From 4630a223b28ea1cd614a6d508a664acc6c8a2a08 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 10 Jun 2015 20:14:12 +0300 Subject: [PATCH 001/427] Initial Commit --- Postbox.xcodeproj/project.pbxproj | 414 ++++++++++++++++++ .../contents.xcworkspacedata | 7 + Postbox/Info.plist | 26 ++ Postbox/Postbox.h | 19 + PostboxTests/Info.plist | 24 + PostboxTests/PostboxTests.swift | 36 ++ 6 files changed, 526 insertions(+) create mode 100644 Postbox.xcodeproj/project.pbxproj create mode 100644 Postbox.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 Postbox/Info.plist create mode 100644 Postbox/Postbox.h create mode 100644 PostboxTests/Info.plist create mode 100644 PostboxTests/PostboxTests.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj new file mode 100644 index 0000000000..62a58cc91d --- /dev/null +++ b/Postbox.xcodeproj/project.pbxproj @@ -0,0 +1,414 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 46; + objects = { + +/* Begin PBXBuildFile section */ + D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; + D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; + D0E3A75D1B28A7E300A402D9 /* PostboxTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + D0E3A7571B28A7E300A402D9 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = D0E3A7411B28A7E300A402D9 /* Project object */; + proxyType = 1; + remoteGlobalIDString = D0E3A7491B28A7E300A402D9; + remoteInfo = Postbox; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXFileReference section */ + D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + D0E3A74F1B28A7E300A402D9 /* Postbox.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Postbox.h; sourceTree = ""; }; + D0E3A7551B28A7E300A402D9 /* PostboxTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = PostboxTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + D0E3A75B1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostboxTests.swift; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + D0E3A7461B28A7E300A402D9 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + D0E3A7521B28A7E300A402D9 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + D0E3A7401B28A7E300A402D9 = { + isa = PBXGroup; + children = ( + D0E3A74C1B28A7E300A402D9 /* Postbox */, + D0E3A7591B28A7E300A402D9 /* PostboxTests */, + D0E3A74B1B28A7E300A402D9 /* Products */, + ); + sourceTree = ""; + }; + D0E3A74B1B28A7E300A402D9 /* Products */ = { + isa = PBXGroup; + children = ( + D0E3A74A1B28A7E300A402D9 /* Postbox.framework */, + D0E3A7551B28A7E300A402D9 /* PostboxTests.xctest */, + ); + name = Products; + sourceTree = ""; + }; + D0E3A74C1B28A7E300A402D9 /* Postbox */ = { + isa = PBXGroup; + children = ( + D0E3A74F1B28A7E300A402D9 /* Postbox.h */, + D0E3A74D1B28A7E300A402D9 /* Supporting Files */, + ); + path = Postbox; + sourceTree = ""; + }; + D0E3A74D1B28A7E300A402D9 /* Supporting Files */ = { + isa = PBXGroup; + children = ( + D0E3A74E1B28A7E300A402D9 /* Info.plist */, + ); + name = "Supporting Files"; + sourceTree = ""; + }; + D0E3A7591B28A7E300A402D9 /* PostboxTests */ = { + isa = PBXGroup; + children = ( + D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */, + D0E3A75A1B28A7E300A402D9 /* Supporting Files */, + ); + path = PostboxTests; + sourceTree = ""; + }; + D0E3A75A1B28A7E300A402D9 /* Supporting Files */ = { + isa = PBXGroup; + children = ( + D0E3A75B1B28A7E300A402D9 /* Info.plist */, + ); + name = "Supporting Files"; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXHeadersBuildPhase section */ + D0E3A7471B28A7E300A402D9 /* Headers */ = { + isa = PBXHeadersBuildPhase; + buildActionMask = 2147483647; + files = ( + D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXHeadersBuildPhase section */ + +/* Begin PBXNativeTarget section */ + D0E3A7491B28A7E300A402D9 /* Postbox */ = { + isa = PBXNativeTarget; + buildConfigurationList = D0E3A7601B28A7E300A402D9 /* Build configuration list for PBXNativeTarget "Postbox" */; + buildPhases = ( + D0E3A7451B28A7E300A402D9 /* Sources */, + D0E3A7461B28A7E300A402D9 /* Frameworks */, + D0E3A7471B28A7E300A402D9 /* Headers */, + D0E3A7481B28A7E300A402D9 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = Postbox; + productName = Postbox; + productReference = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; + productType = "com.apple.product-type.framework"; + }; + D0E3A7541B28A7E300A402D9 /* PostboxTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = D0E3A7631B28A7E300A402D9 /* Build configuration list for PBXNativeTarget "PostboxTests" */; + buildPhases = ( + D0E3A7511B28A7E300A402D9 /* Sources */, + D0E3A7521B28A7E300A402D9 /* Frameworks */, + D0E3A7531B28A7E300A402D9 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + D0E3A7581B28A7E300A402D9 /* PBXTargetDependency */, + ); + name = PostboxTests; + productName = PostboxTests; + productReference = D0E3A7551B28A7E300A402D9 /* PostboxTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + D0E3A7411B28A7E300A402D9 /* Project object */ = { + isa = PBXProject; + attributes = { + LastUpgradeCheck = 0630; + ORGANIZATIONNAME = Telegram; + TargetAttributes = { + D0E3A7491B28A7E300A402D9 = { + CreatedOnToolsVersion = 6.3.1; + }; + D0E3A7541B28A7E300A402D9 = { + CreatedOnToolsVersion = 6.3.1; + }; + }; + }; + buildConfigurationList = D0E3A7441B28A7E300A402D9 /* Build configuration list for PBXProject "Postbox" */; + compatibilityVersion = "Xcode 3.2"; + developmentRegion = English; + hasScannedForEncodings = 0; + knownRegions = ( + en, + ); + mainGroup = D0E3A7401B28A7E300A402D9; + productRefGroup = D0E3A74B1B28A7E300A402D9 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + D0E3A7491B28A7E300A402D9 /* Postbox */, + D0E3A7541B28A7E300A402D9 /* PostboxTests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + D0E3A7481B28A7E300A402D9 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + D0E3A7531B28A7E300A402D9 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + D0E3A7451B28A7E300A402D9 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + D0E3A7511B28A7E300A402D9 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + D0E3A75D1B28A7E300A402D9 /* PostboxTests.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + D0E3A7581B28A7E300A402D9 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = D0E3A7491B28A7E300A402D9 /* Postbox */; + targetProxy = D0E3A7571B28A7E300A402D9 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + D0E3A75E1B28A7E300A402D9 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 1; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_SYMBOLS_PRIVATE_EXTERN = NO; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 8.3; + MTL_ENABLE_DEBUG_INFO = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + TARGETED_DEVICE_FAMILY = "1,2"; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = Debug; + }; + D0E3A75F1B28A7E300A402D9 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 1; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 8.3; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + VALIDATE_PRODUCT = YES; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = Release; + }; + D0E3A7611B28A7E300A402D9 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + DEFINES_MODULE = YES; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + INFOPLIST_FILE = Postbox/Info.plist; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + }; + name = Debug; + }; + D0E3A7621B28A7E300A402D9 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + DEFINES_MODULE = YES; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + INFOPLIST_FILE = Postbox/Info.plist; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + }; + name = Release; + }; + D0E3A7641B28A7E300A402D9 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + FRAMEWORK_SEARCH_PATHS = ( + "$(SDKROOT)/Developer/Library/Frameworks", + "$(inherited)", + ); + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + INFOPLIST_FILE = PostboxTests/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + D0E3A7651B28A7E300A402D9 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + FRAMEWORK_SEARCH_PATHS = ( + "$(SDKROOT)/Developer/Library/Frameworks", + "$(inherited)", + ); + INFOPLIST_FILE = PostboxTests/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + D0E3A7441B28A7E300A402D9 /* Build configuration list for PBXProject "Postbox" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + D0E3A75E1B28A7E300A402D9 /* Debug */, + D0E3A75F1B28A7E300A402D9 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + D0E3A7601B28A7E300A402D9 /* Build configuration list for PBXNativeTarget "Postbox" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + D0E3A7611B28A7E300A402D9 /* Debug */, + D0E3A7621B28A7E300A402D9 /* Release */, + ); + defaultConfigurationIsVisible = 0; + }; + D0E3A7631B28A7E300A402D9 /* Build configuration list for PBXNativeTarget "PostboxTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + D0E3A7641B28A7E300A402D9 /* Debug */, + D0E3A7651B28A7E300A402D9 /* Release */, + ); + defaultConfigurationIsVisible = 0; + }; +/* End XCConfigurationList section */ + }; + rootObject = D0E3A7411B28A7E300A402D9 /* Project object */; +} diff --git a/Postbox.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/Postbox.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000000..088de55706 --- /dev/null +++ b/Postbox.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/Postbox/Info.plist b/Postbox/Info.plist new file mode 100644 index 0000000000..f6228f96e7 --- /dev/null +++ b/Postbox/Info.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + org.telegram.$(PRODUCT_NAME:rfc1034identifier) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + FMWK + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + $(CURRENT_PROJECT_VERSION) + NSPrincipalClass + + + diff --git a/Postbox/Postbox.h b/Postbox/Postbox.h new file mode 100644 index 0000000000..98ca9cb15e --- /dev/null +++ b/Postbox/Postbox.h @@ -0,0 +1,19 @@ +// +// Postbox.h +// Postbox +// +// Created by Peter on 10/06/15. +// Copyright (c) 2015 Telegram. All rights reserved. +// + +#import + +//! Project version number for Postbox. +FOUNDATION_EXPORT double PostboxVersionNumber; + +//! Project version string for Postbox. +FOUNDATION_EXPORT const unsigned char PostboxVersionString[]; + +// In this header, you should import all the public headers of your framework using statements like #import + + diff --git a/PostboxTests/Info.plist b/PostboxTests/Info.plist new file mode 100644 index 0000000000..29dacae6b5 --- /dev/null +++ b/PostboxTests/Info.plist @@ -0,0 +1,24 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + org.telegram.$(PRODUCT_NAME:rfc1034identifier) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + BNDL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + + diff --git a/PostboxTests/PostboxTests.swift b/PostboxTests/PostboxTests.swift new file mode 100644 index 0000000000..d2545f03f8 --- /dev/null +++ b/PostboxTests/PostboxTests.swift @@ -0,0 +1,36 @@ +// +// PostboxTests.swift +// PostboxTests +// +// Created by Peter on 10/06/15. +// Copyright (c) 2015 Telegram. All rights reserved. +// + +import UIKit +import XCTest + +class PostboxTests: XCTestCase { + + override func setUp() { + super.setUp() + // Put setup code here. This method is called before the invocation of each test method in the class. + } + + override func tearDown() { + // Put teardown code here. This method is called after the invocation of each test method in the class. + super.tearDown() + } + + func testExample() { + // This is an example of a functional test case. + XCTAssert(true, "Pass") + } + + func testPerformanceExample() { + // This is an example of a performance test case. + self.measureBlock() { + // Put the code you want to measure the time of here. + } + } + +} From cc75497a0c03aaad362c90bfe3be1940626fe7ff Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 8 Jul 2015 17:16:47 +0300 Subject: [PATCH 002/427] Lot's of changes --- .gitmodules | 3 + Postbox.xcodeproj/project.pbxproj | 174 +- Postbox/Config/Postbox.xcconfig | 2 + Postbox/MessageView.swift | 365 + Postbox/MurMurHash32.h | 9 + Postbox/MurMurHash32.m | 98 + Postbox/Postbox.h | 2 +- Postbox/PostboxPrivate/module.modulemap | 7 + .../sqlcipher/SQLite-Bridging.h | 56 + .../sqlcipher/SQLite-Bridging.m | 217 + .../PostboxPrivate/sqlcipher/fts3_tokenizer.h | 161 + Postbox/PostboxPrivate/sqlcipher/sqlite3.c | 151821 +++++++++++++++ Postbox/PostboxPrivate/sqlcipher/sqlite3.h | 7494 + Postbox/PostboxPrivate/sqlcipher/sqlite3ext.h | 487 + Postbox/module.private.modulemap | 3 + PostboxTests/CodingTests.swift | 159 + PostboxTests/PostboxTests.swift | 665 +- submodules/sqlite.swift | 1 + 18 files changed, 161703 insertions(+), 21 deletions(-) create mode 100644 .gitmodules create mode 100644 Postbox/Config/Postbox.xcconfig create mode 100644 Postbox/MessageView.swift create mode 100644 Postbox/MurMurHash32.h create mode 100644 Postbox/MurMurHash32.m create mode 100644 Postbox/PostboxPrivate/module.modulemap create mode 100644 Postbox/PostboxPrivate/sqlcipher/SQLite-Bridging.h create mode 100644 Postbox/PostboxPrivate/sqlcipher/SQLite-Bridging.m create mode 100644 Postbox/PostboxPrivate/sqlcipher/fts3_tokenizer.h create mode 100644 Postbox/PostboxPrivate/sqlcipher/sqlite3.c create mode 100644 Postbox/PostboxPrivate/sqlcipher/sqlite3.h create mode 100644 Postbox/PostboxPrivate/sqlcipher/sqlite3ext.h create mode 100644 Postbox/module.private.modulemap create mode 100644 PostboxTests/CodingTests.swift create mode 160000 submodules/sqlite.swift diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..98e5887e06 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "submodules/sqlite.swift"] + path = submodules/sqlite.swift + url = https://github.com/stephencelis/SQLite.swift.git diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 62a58cc91d..cb95fe5cc6 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -7,9 +7,35 @@ objects = { /* Begin PBXBuildFile section */ + D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageView.swift */; }; + D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044E15D1B2ACB9C001EE087 /* CodingTests.swift */; }; + D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */ = {isa = PBXBuildFile; fileRef = D044E1621B2AD677001EE087 /* MurMurHash32.m */; }; + D044E1641B2AD718001EE087 /* MurMurHash32.h in Headers */ = {isa = PBXBuildFile; fileRef = D044E1611B2AD667001EE087 /* MurMurHash32.h */; settings = {ATTRIBUTES = (Public, ); }; }; + D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; }; + D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516411B2D9CEF00AE42E0 /* sqlite3.h */; }; + D07516461B2D9CEF00AE42E0 /* sqlite3ext.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */; }; + D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */ = {isa = PBXBuildFile; fileRef = D075165B1B2EC5B000AE42E0 /* module.private.modulemap */; }; + D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */ = {isa = PBXBuildFile; fileRef = D075165D1B2EC5B500AE42E0 /* module.modulemap */; }; + D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516611B2EC7FE00AE42E0 /* Database.swift */; }; + D075166B1B2EC7FE00AE42E0 /* Expression.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516621B2EC7FE00AE42E0 /* Expression.swift */; }; + D075166C1B2EC7FE00AE42E0 /* FTS.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516631B2EC7FE00AE42E0 /* FTS.swift */; }; + D075166E1B2EC7FE00AE42E0 /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516651B2EC7FE00AE42E0 /* Functions.swift */; }; + D075166F1B2EC7FE00AE42E0 /* Query.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516661B2EC7FE00AE42E0 /* Query.swift */; }; + D07516701B2EC7FE00AE42E0 /* Schema.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516671B2EC7FE00AE42E0 /* Schema.swift */; }; + D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516681B2EC7FE00AE42E0 /* Statement.swift */; }; + D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516691B2EC7FE00AE42E0 /* Value.swift */; }; + D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; + D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; + D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; + D07FC7D31B4A3D6B0010B3F7 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; D0E3A75D1B28A7E300A402D9 /* PostboxTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */; }; + D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7811B28ADD000A402D9 /* Postbox.swift */; }; + D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; + D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; + D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; + D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -23,12 +49,39 @@ /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ + D003E4E51B38DBDB00C22CBC /* MessageView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageView.swift; sourceTree = ""; }; + D044E15D1B2ACB9C001EE087 /* CodingTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CodingTests.swift; sourceTree = ""; }; + D044E1611B2AD667001EE087 /* MurMurHash32.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MurMurHash32.h; sourceTree = ""; }; + D044E1621B2AD677001EE087 /* MurMurHash32.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MurMurHash32.m; sourceTree = ""; }; + D07516401B2D9CEF00AE42E0 /* sqlite3.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = sqlite3.c; sourceTree = ""; }; + D07516411B2D9CEF00AE42E0 /* sqlite3.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3.h; sourceTree = ""; }; + D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3ext.h; sourceTree = ""; }; + D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Postbox.xcconfig; path = Postbox/Config/Postbox.xcconfig; sourceTree = ""; }; + D075165B1B2EC5B000AE42E0 /* module.private.modulemap */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = "sourcecode.module-map"; path = module.private.modulemap; sourceTree = ""; }; + D075165D1B2EC5B500AE42E0 /* module.modulemap */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = "sourcecode.module-map"; path = module.modulemap; sourceTree = ""; }; + D07516611B2EC7FE00AE42E0 /* Database.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Database.swift; path = submodules/sqlite.swift/SQLite/Database.swift; sourceTree = SOURCE_ROOT; }; + D07516621B2EC7FE00AE42E0 /* Expression.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Expression.swift; path = submodules/sqlite.swift/SQLite/Expression.swift; sourceTree = SOURCE_ROOT; }; + D07516631B2EC7FE00AE42E0 /* FTS.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = FTS.swift; path = submodules/sqlite.swift/SQLite/FTS.swift; sourceTree = SOURCE_ROOT; }; + D07516651B2EC7FE00AE42E0 /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; + D07516661B2EC7FE00AE42E0 /* Query.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Query.swift; path = submodules/sqlite.swift/SQLite/Query.swift; sourceTree = SOURCE_ROOT; }; + D07516671B2EC7FE00AE42E0 /* Schema.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Schema.swift; path = submodules/sqlite.swift/SQLite/Schema.swift; sourceTree = SOURCE_ROOT; }; + D07516681B2EC7FE00AE42E0 /* Statement.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Statement.swift; path = submodules/sqlite.swift/SQLite/Statement.swift; sourceTree = SOURCE_ROOT; }; + D07516691B2EC7FE00AE42E0 /* Value.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Value.swift; path = submodules/sqlite.swift/SQLite/Value.swift; sourceTree = SOURCE_ROOT; }; + D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; + D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; + D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; + D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../SSignalKit/build/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; D0E3A74F1B28A7E300A402D9 /* Postbox.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Postbox.h; sourceTree = ""; }; D0E3A7551B28A7E300A402D9 /* PostboxTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = PostboxTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A75B1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostboxTests.swift; sourceTree = ""; }; + D0E3A7811B28ADD000A402D9 /* Postbox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Postbox.swift; sourceTree = ""; }; + D0E3A7831B28AE0900A402D9 /* Peer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Peer.swift; sourceTree = ""; }; + D0E3A7871B28AE9C00A402D9 /* Coding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Coding.swift; sourceTree = ""; }; + D0E3A79D1B28B50400A402D9 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = ""; }; + D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -43,6 +96,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + D07FC7D31B4A3D6B0010B3F7 /* SwiftSignalKit.framework in Frameworks */, D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -50,9 +104,56 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + D07515FC1B2C44A200AE42E0 /* thirdparty */ = { + isa = PBXGroup; + children = ( + D07516731B2EC8C700AE42E0 /* sqlite.swift */, + ); + name = thirdparty; + sourceTree = ""; + }; + D075163D1B2D9CEF00AE42E0 /* PostboxPrivate */ = { + isa = PBXGroup; + children = ( + D075165D1B2EC5B500AE42E0 /* module.modulemap */, + D075163F1B2D9CEF00AE42E0 /* sqlcipher */, + ); + path = PostboxPrivate; + sourceTree = ""; + }; + D075163F1B2D9CEF00AE42E0 /* sqlcipher */ = { + isa = PBXGroup; + children = ( + D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */, + D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */, + D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */, + D07516401B2D9CEF00AE42E0 /* sqlite3.c */, + D07516411B2D9CEF00AE42E0 /* sqlite3.h */, + D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */, + ); + path = sqlcipher; + sourceTree = ""; + }; + D07516731B2EC8C700AE42E0 /* sqlite.swift */ = { + isa = PBXGroup; + children = ( + D07516611B2EC7FE00AE42E0 /* Database.swift */, + D07516621B2EC7FE00AE42E0 /* Expression.swift */, + D07516631B2EC7FE00AE42E0 /* FTS.swift */, + D07516651B2EC7FE00AE42E0 /* Functions.swift */, + D07516661B2EC7FE00AE42E0 /* Query.swift */, + D07516671B2EC7FE00AE42E0 /* Schema.swift */, + D07516681B2EC7FE00AE42E0 /* Statement.swift */, + D07516691B2EC7FE00AE42E0 /* Value.swift */, + ); + name = sqlite.swift; + sourceTree = ""; + }; D0E3A7401B28A7E300A402D9 = { isa = PBXGroup; children = ( + D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */, + D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */, D0E3A74C1B28A7E300A402D9 /* Postbox */, D0E3A7591B28A7E300A402D9 /* PostboxTests */, D0E3A74B1B28A7E300A402D9 /* Products */, @@ -71,7 +172,15 @@ D0E3A74C1B28A7E300A402D9 /* Postbox */ = { isa = PBXGroup; children = ( - D0E3A74F1B28A7E300A402D9 /* Postbox.h */, + D075165B1B2EC5B000AE42E0 /* module.private.modulemap */, + D075163D1B2D9CEF00AE42E0 /* PostboxPrivate */, + D07515FC1B2C44A200AE42E0 /* thirdparty */, + D0E3A7871B28AE9C00A402D9 /* Coding.swift */, + D0E3A7831B28AE0900A402D9 /* Peer.swift */, + D0E3A79D1B28B50400A402D9 /* Message.swift */, + D0E3A7A11B28B7DC00A402D9 /* Media.swift */, + D003E4E51B38DBDB00C22CBC /* MessageView.swift */, + D0E3A7811B28ADD000A402D9 /* Postbox.swift */, D0E3A74D1B28A7E300A402D9 /* Supporting Files */, ); path = Postbox; @@ -80,6 +189,9 @@ D0E3A74D1B28A7E300A402D9 /* Supporting Files */ = { isa = PBXGroup; children = ( + D044E1611B2AD667001EE087 /* MurMurHash32.h */, + D044E1621B2AD677001EE087 /* MurMurHash32.m */, + D0E3A74F1B28A7E300A402D9 /* Postbox.h */, D0E3A74E1B28A7E300A402D9 /* Info.plist */, ); name = "Supporting Files"; @@ -90,6 +202,7 @@ children = ( D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */, D0E3A75A1B28A7E300A402D9 /* Supporting Files */, + D044E15D1B2ACB9C001EE087 /* CodingTests.swift */, ); path = PostboxTests; sourceTree = ""; @@ -109,7 +222,12 @@ isa = PBXHeadersBuildPhase; buildActionMask = 2147483647; files = ( + D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */, + D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */, + D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */, D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */, + D07516461B2D9CEF00AE42E0 /* sqlite3ext.h in Headers */, + D044E1641B2AD718001EE087 /* MurMurHash32.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -158,6 +276,7 @@ D0E3A7411B28A7E300A402D9 /* Project object */ = { isa = PBXProject; attributes = { + LastSwiftUpdateCheck = 0700; LastUpgradeCheck = 0630; ORGANIZATIONNAME = Telegram; TargetAttributes = { @@ -209,6 +328,25 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + D07516701B2EC7FE00AE42E0 /* Schema.swift in Sources */, + D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */, + D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, + D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, + D075166E1B2EC7FE00AE42E0 /* Functions.swift in Sources */, + D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, + D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */, + D075166B1B2EC7FE00AE42E0 /* Expression.swift in Sources */, + D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, + D075166C1B2EC7FE00AE42E0 /* FTS.swift in Sources */, + D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, + D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */, + D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */, + D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, + D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, + D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, + D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, + D075166F1B2EC7FE00AE42E0 /* Query.swift in Sources */, + D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -216,6 +354,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */, D0E3A75D1B28A7E300A402D9 /* PostboxTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -233,6 +372,7 @@ /* Begin XCBuildConfiguration section */ D0E3A75E1B28A7E300A402D9 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; @@ -281,6 +421,7 @@ }; D0E3A75F1B28A7E300A402D9 /* Release */ = { isa = XCBuildConfiguration; + baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; @@ -322,29 +463,55 @@ }; D0E3A7611B28A7E300A402D9 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; buildSettings = { + CLANG_ENABLE_MODULES = YES; DEFINES_MODULE = YES; DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; DYLIB_INSTALL_NAME_BASE = "@rpath"; + FRAMEWORK_SEARCH_PATHS = ( + "$(inherited)", + "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", + ); INFOPLIST_FILE = Postbox/Info.plist; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + OTHER_CFLAGS = ( + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_HAS_CODEC=1", + "-DSQLCIPHER_CRYPTO_CC=1", + "-DSQLITE_ENABLE_FTS3", + ); PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; }; name = Debug; }; D0E3A7621B28A7E300A402D9 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { + CLANG_ENABLE_MODULES = YES; DEFINES_MODULE = YES; DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; DYLIB_INSTALL_NAME_BASE = "@rpath"; + FRAMEWORK_SEARCH_PATHS = ( + "$(inherited)", + "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", + ); INFOPLIST_FILE = Postbox/Info.plist; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + OTHER_CFLAGS = ( + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_HAS_CODEC=1", + "-DSQLCIPHER_CRYPTO_CC=1", + "-DSQLITE_ENABLE_FTS3", + ); PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; }; @@ -352,10 +519,12 @@ }; D0E3A7641B28A7E300A402D9 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; buildSettings = { FRAMEWORK_SEARCH_PATHS = ( "$(SDKROOT)/Developer/Library/Frameworks", "$(inherited)", + "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", ); GCC_PREPROCESSOR_DEFINITIONS = ( "DEBUG=1", @@ -373,6 +542,7 @@ FRAMEWORK_SEARCH_PATHS = ( "$(SDKROOT)/Developer/Library/Frameworks", "$(inherited)", + "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", ); INFOPLIST_FILE = PostboxTests/Info.plist; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; @@ -399,6 +569,7 @@ D0E3A7621B28A7E300A402D9 /* Release */, ); defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; }; D0E3A7631B28A7E300A402D9 /* Build configuration list for PBXNativeTarget "PostboxTests" */ = { isa = XCConfigurationList; @@ -407,6 +578,7 @@ D0E3A7651B28A7E300A402D9 /* Release */, ); defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; }; /* End XCConfigurationList section */ }; diff --git a/Postbox/Config/Postbox.xcconfig b/Postbox/Config/Postbox.xcconfig new file mode 100644 index 0000000000..17c706bde8 --- /dev/null +++ b/Postbox/Config/Postbox.xcconfig @@ -0,0 +1,2 @@ +SWIFT_INCLUDE_PATHS = $(SRCROOT)/Postbox +MODULEMAP_PRIVATE_FILE = $(SRCROOT)/Postbox/module.private.modulemap diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift new file mode 100644 index 0000000000..173d734d2e --- /dev/null +++ b/Postbox/MessageView.swift @@ -0,0 +1,365 @@ +import Foundation + +public final class MutableMessageView: Printable { + public struct RemoveContext { + var invalidEarlier: Set + var invalidLater: Set + var removedMessages: Bool + + init() { + self.invalidEarlier = [] + self.invalidLater = [] + self.removedMessages = false + } + } + + let namespaces: [MessageId.Namespace] + let count: Int + var earlier: [MessageId.Namespace : Message] = [:] + var later: [MessageId.Namespace : Message] = [:] + var messages: [Message] + + public init(namespaces: [MessageId.Namespace], count: Int, earlier: [MessageId.Namespace : Message], messages: [Message], later: [MessageId.Namespace : Message]) { + self.namespaces = namespaces + self.count = count + self.earlier = earlier + self.later = later + self.messages = messages + } + + public func add(message: Message) { + if self.messages.count == 0 { + self.messages.append(message) + } else { + var first = MessageIndex(self.messages[self.messages.count - 1]) + var last = MessageIndex(self.messages[0]) + + var next: MessageIndex? + for namespace in self.namespaces { + if let message = later[namespace] { + let messageIndex = MessageIndex(message) + if next == nil || messageIndex < next! { + next = messageIndex + } + } + } + + let index = MessageIndex(message) + + if index < last { + let earlierMessage = self.earlier[message.id.namespace] + if earlierMessage == nil || earlierMessage!.id.id < message.id.id { + if self.messages.count < self.count { + self.messages.insert(message, atIndex: 0) + } else { + self.earlier[message.id.namespace] = message + } + } + } else if index > first { + if next != nil && index > next! { + let laterMessage = self.later[message.id.namespace] + if laterMessage == nil || laterMessage!.id.id > message.id.id { + if self.messages.count < self.count { + self.messages.append(message) + } else { + self.later[message.id.namespace] = message + } + } + } else { + self.messages.append(message) + if self.messages.count > self.count { + let earliest = self.messages[0] + self.earlier[earliest.id.namespace] = earliest + self.messages.removeAtIndex(0) + } + } + } else if index != last && index != first { + var i = self.messages.count + while i >= 1 { + if MessageIndex(self.messages[i - 1]) < index { + break + } + i-- + } + self.messages.insert(message, atIndex: i) + if self.messages.count > self.count { + let earliest = self.messages[0] + self.earlier[earliest.id.namespace] = earliest + self.messages.removeAtIndex(0) + } + } + } + } + + public func remove(ids: Set, context: RemoveContext? = nil) -> RemoveContext { + var updatedContext = RemoveContext() + if let context = context { + updatedContext = context + } + + for (_, message) in self.earlier { + if ids.contains(message.id) { + updatedContext.invalidEarlier.insert(message.id.namespace) + } + } + + for (_, message) in self.later { + if ids.contains(message.id) { + updatedContext.invalidLater.insert(message.id.namespace) + } + } + + if self.messages.count != 0 { + var i = self.messages.count - 1 + while i >= 0 { + if ids.contains(self.messages[i].id) { + self.messages.removeAtIndex(i) + updatedContext.removedMessages = true + } + i-- + } + } + + return updatedContext + } + + public func complete(context: RemoveContext, fetchEarlier: (MessageId.Namespace, MessageId.Id?, Int) -> [Message], fetchLater: (MessageId.Namespace, MessageId.Id?, Int) -> [Message]) { + if context.removedMessages { + var addedMessages: [Message] = [] + + var latestAnchor: MessageIndex? + if let lastMessage = self.messages.last { + latestAnchor = MessageIndex(lastMessage) + } + + if latestAnchor == nil { + var laterMessages: [Message] = [] + for (_, message) in self.later { + let messageIndex = MessageIndex(message) + if latestAnchor == nil || latestAnchor! > messageIndex { + latestAnchor = messageIndex + } + } + } + + for namespace in self.namespaces { + if let later = self.later[namespace] { + addedMessages += fetchLater(namespace, later.id.id - 1, self.count) + } + if let earlier = self.earlier[namespace] { + addedMessages += fetchEarlier(namespace, earlier.id.id + 1, self.count) + } + } + + addedMessages += self.messages + addedMessages.sort({ MessageIndex($0) < MessageIndex($1) }) + var i = addedMessages.count - 1 + while i >= 1 { + if addedMessages[i].id == addedMessages[i - 1].id { + addedMessages.removeAtIndex(i) + } + i-- + } + self.messages = [] + + var anchorIndex = addedMessages.count - 1 + if let latestAnchor = latestAnchor { + var i = addedMessages.count - 1 + while i >= 0 { + if MessageIndex(addedMessages[i]) <= latestAnchor { + anchorIndex = i + break + } + i-- + } + } + + self.later.removeAll(keepCapacity: true) + + if anchorIndex + 1 < addedMessages.count { + for namespace in self.namespaces { + var i = anchorIndex + 1 + while i < addedMessages.count { + if addedMessages[i].id.namespace == namespace { + self.later[namespace] = addedMessages[i] + break + } + i++ + } + } + } + + i = anchorIndex + while i >= 0 && i > anchorIndex - self.count { + self.messages.insert(addedMessages[i], atIndex: 0) + i-- + } + + self.earlier.removeAll(keepCapacity: true) + if anchorIndex - self.count >= 0 { + for namespace in self.namespaces { + i = anchorIndex - self.count + while i >= 0 { + if addedMessages[i].id.namespace == namespace { + self.earlier[namespace] = addedMessages[i] + break + } + i-- + } + } + } + } + else { + for namespace in context.invalidEarlier { + var earlyId: MessageId.Id? + var i = 0 + while i < self.messages.count { + if self.messages[i].id.namespace == namespace { + earlyId = self.messages[i].id.id + break + } + i++ + } + + let earlierMessages = fetchEarlier(namespace, earlyId, 1) + if earlierMessages.count == 0 { + self.earlier.removeValueForKey(namespace) + } else { + self.earlier[namespace] = earlierMessages[0] + } + } + + for namespace in context.invalidLater { + var lateId: MessageId.Id? + var i = self.messages.count - 1 + while i >= 0 { + if self.messages[i].id.namespace == namespace { + lateId = self.messages[i].id.id + break + } + i-- + } + + let laterMessages = fetchLater(namespace, lateId, 1) + if laterMessages.count == 0 { + self.later.removeValueForKey(namespace) + } else { + self.later[namespace] = laterMessages[0] + } + } + } + } + + public var description: String { + var string = "" + string += "...(" + var first = true + for namespace in self.namespaces { + if let value = self.earlier[namespace] { + if first { + first = false + } else { + string += ", " + } + string += "\(namespace): \(value.id.id)—\(value.timestamp)" + } + } + string += ") —— " + + string += "[" + first = true + for message in self.messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" + } + string += "]" + + string += " —— (" + first = true + for namespace in self.namespaces { + if let value = self.later[namespace] { + if first { + first = false + } else { + string += ", " + } + string += "\(namespace): \(value.id.id)—\(value.timestamp)" + } + } + string += ")..." + + return string + } +} + +public final class MessageView: Printable { + public let hasEarlier: Bool + private let earlierIds: [MessageIndex] + public let hasLater: Bool + private let laterIds: [MessageIndex] + public let messages: [Message] + + init(_ mutableView: MutableMessageView) { + self.hasEarlier = mutableView.earlier.count != 0 + self.hasLater = mutableView.later.count != 0 + self.messages = mutableView.messages + + var earlierIds: [MessageIndex] = [] + for (_, message) in mutableView.earlier { + earlierIds.append(MessageIndex(message)) + } + self.earlierIds = earlierIds + + var laterIds: [MessageIndex] = [] + for (_, message) in mutableView.later { + laterIds.append(MessageIndex(message)) + } + self.laterIds = laterIds + } + + public var description: String { + var string = "" + if self.hasEarlier { + string += "more(" + var first = true + for id in self.earlierIds { + if first { + first = false + } else { + string += ", " + } + string += "\(id.id.namespace): \(id.id.id)—\(id.timestamp)" + } + string += ") " + } + string += "[" + var first = true + for message in self.messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" + } + string += "]" + if self.hasLater { + string += " more(" + var first = true + for id in self.laterIds { + if first { + first = false + } else { + string += ", " + } + string += "\(id.id.namespace): \(id.id.id)—\(id.timestamp)" + } + string += ")" + } + return string + } +} diff --git a/Postbox/MurMurHash32.h b/Postbox/MurMurHash32.h new file mode 100644 index 0000000000..42512ef238 --- /dev/null +++ b/Postbox/MurMurHash32.h @@ -0,0 +1,9 @@ +#ifndef Postbox_MurMurHash32_h +#define Postbox_MurMurHash32_h + +#import + +int32_t murMurHash32(void *bytes, int length); +int32_t murMurHashString32(const char *s); + +#endif diff --git a/Postbox/MurMurHash32.m b/Postbox/MurMurHash32.m new file mode 100644 index 0000000000..25afc09f39 --- /dev/null +++ b/Postbox/MurMurHash32.m @@ -0,0 +1,98 @@ +#import "MurMurHash32.h" + +#include +#include + +#define FORCE_INLINE __attribute__((always_inline)) + +static inline uint32_t rotl32 ( uint32_t x, int8_t r ) +{ + return (x << r) | (x >> (32 - r)); +} + +#define ROTL32(x,y) rotl32(x,y) + +static FORCE_INLINE uint32_t getblock ( const uint32_t * p, int i ) +{ + return p[i]; +} + +static FORCE_INLINE uint32_t fmix ( uint32_t h ) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +static void murMurHash32Impl(const void *key, int len, uint32_t seed, void *out) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 4; + + uint32_t h1 = seed; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *)(data + nblocks*4); + + for(int i = -nblocks; i; i++) + { + uint32_t k1 = getblock(blocks,i); + + k1 *= c1; + k1 = ROTL32(k1,15); + k1 *= c2; + + h1 ^= k1; + h1 = ROTL32(h1,13); + h1 = h1*5+0xe6546b64; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*4); + + uint32_t k1 = 0; + + switch(len & 3) + { + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; + case 1: k1 ^= tail[0]; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + + h1 = fmix(h1); + + *(uint32_t*)out = h1; +} + +int32_t murMurHash32(void *bytes, int length) +{ + int32_t result = 0; + murMurHash32Impl(bytes, length, -137723950, &result); + + return result; +} + +int32_t murMurHashString32(const char *s) +{ + int32_t result = 0; + murMurHash32Impl(s, (int)strlen(s), -137723950, &result); + + return result; +} diff --git a/Postbox/Postbox.h b/Postbox/Postbox.h index 98ca9cb15e..5cfad5bac5 100644 --- a/Postbox/Postbox.h +++ b/Postbox/Postbox.h @@ -16,4 +16,4 @@ FOUNDATION_EXPORT const unsigned char PostboxVersionString[]; // In this header, you should import all the public headers of your framework using statements like #import - +#import diff --git a/Postbox/PostboxPrivate/module.modulemap b/Postbox/PostboxPrivate/module.modulemap new file mode 100644 index 0000000000..b4494ba2ae --- /dev/null +++ b/Postbox/PostboxPrivate/module.modulemap @@ -0,0 +1,7 @@ +module sqlcipher { + header "sqlcipher/sqlite3.h" + header "sqlcipher/sqlite3ext.h" + header "sqlcipher/SQLite-Bridging.h" + header "sqlcipher/fts3_tokenizer.h" + export * +} diff --git a/Postbox/PostboxPrivate/sqlcipher/SQLite-Bridging.h b/Postbox/PostboxPrivate/sqlcipher/SQLite-Bridging.h new file mode 100644 index 0000000000..6bb9f2de3c --- /dev/null +++ b/Postbox/PostboxPrivate/sqlcipher/SQLite-Bridging.h @@ -0,0 +1,56 @@ +// +// SQLite.swift +// https://github.com/stephencelis/SQLite.swift +// Copyright (c) 2014-2015 Stephen Celis. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + +@import Foundation; + +typedef struct SQLiteHandle SQLiteHandle; +typedef struct SQLiteContext SQLiteContext; +typedef struct SQLiteValue SQLiteValue; + +NS_ASSUME_NONNULL_BEGIN +typedef int (^_SQLiteBusyHandlerCallback)(int times); +int _SQLiteBusyHandler(SQLiteHandle * handle, _SQLiteBusyHandlerCallback __nullable callback); + +typedef void (^_SQLiteTraceCallback)(const char * SQL); +void _SQLiteTrace(SQLiteHandle * handle, _SQLiteTraceCallback __nullable callback); + +typedef void (^_SQLiteUpdateHookCallback)(int operation, const char * db, const char * table, long long rowid); +void _SQLiteUpdateHook(SQLiteHandle * handle, _SQLiteUpdateHookCallback __nullable callback); + +typedef int (^_SQLiteCommitHookCallback)(); +void _SQLiteCommitHook(SQLiteHandle * handle, _SQLiteCommitHookCallback __nullable callback); + +typedef void (^_SQLiteRollbackHookCallback)(); +void _SQLiteRollbackHook(SQLiteHandle * handle, _SQLiteRollbackHookCallback __nullable callback); + +typedef void (^_SQLiteCreateFunctionCallback)(SQLiteContext * context, int argc, SQLiteValue * __nonnull * __nonnull argv); +int _SQLiteCreateFunction(SQLiteHandle * handle, const char * name, int argc, int deterministic, _SQLiteCreateFunctionCallback __nullable callback); + +typedef int (^_SQLiteCreateCollationCallback)(const char * lhs, const char * rhs); +int _SQLiteCreateCollation(SQLiteHandle * handle, const char * name, _SQLiteCreateCollationCallback __nullable callback); + +typedef NSString * __nullable (^_SQLiteTokenizerNextCallback)(const char * input, int * inputOffset, int * inputLength); +int _SQLiteRegisterTokenizer(SQLiteHandle * db, const char * module, const char * tokenizer, __nullable _SQLiteTokenizerNextCallback callback); +NS_ASSUME_NONNULL_END + diff --git a/Postbox/PostboxPrivate/sqlcipher/SQLite-Bridging.m b/Postbox/PostboxPrivate/sqlcipher/SQLite-Bridging.m new file mode 100644 index 0000000000..26b6897dd4 --- /dev/null +++ b/Postbox/PostboxPrivate/sqlcipher/SQLite-Bridging.m @@ -0,0 +1,217 @@ +// +// SQLite.swift +// https://github.com/stephencelis/SQLite.swift +// Copyright (c) 2014-2015 Stephen Celis. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + +#import "SQLite-Bridging.h" + +#import "fts3_tokenizer.h" + +static int __SQLiteBusyHandler(void * context, int tries) { + return ((__bridge _SQLiteBusyHandlerCallback)context)(tries); +} + +int _SQLiteBusyHandler(SQLiteHandle * handle, _SQLiteBusyHandlerCallback callback) { + if (callback) { + return sqlite3_busy_handler((sqlite3 *)handle, __SQLiteBusyHandler, (__bridge void *)callback); + } else { + return sqlite3_busy_handler((sqlite3 *)handle, 0, 0); + } +} + +static void __SQLiteTrace(void * context, const char * SQL) { + ((__bridge _SQLiteTraceCallback)context)(SQL); +} + +void _SQLiteTrace(SQLiteHandle * handle, _SQLiteTraceCallback callback) { + if (callback) { + sqlite3_trace((sqlite3 *)handle, __SQLiteTrace, (__bridge void *)callback); + } else { + sqlite3_trace((sqlite3 *)handle, 0, 0); + } +} + +static void __SQLiteUpdateHook(void * context, int operation, const char * db, const char * table, long long rowid) { + ((__bridge _SQLiteUpdateHookCallback)context)(operation, db, table, rowid); +} + +void _SQLiteUpdateHook(SQLiteHandle * handle, _SQLiteUpdateHookCallback callback) { + sqlite3_update_hook((sqlite3 *)handle, __SQLiteUpdateHook, (__bridge void *)callback); +} + +static int __SQLiteCommitHook(void * context) { + return ((__bridge _SQLiteCommitHookCallback)context)(); +} + +void _SQLiteCommitHook(SQLiteHandle * handle, _SQLiteCommitHookCallback callback) { + sqlite3_commit_hook((sqlite3 *)handle, __SQLiteCommitHook, (__bridge void *)callback); +} + +static void __SQLiteRollbackHook(void * context) { + ((__bridge _SQLiteRollbackHookCallback)context)(); +} + +void _SQLiteRollbackHook(SQLiteHandle * handle, _SQLiteRollbackHookCallback callback) { + sqlite3_rollback_hook((sqlite3 *)handle, __SQLiteRollbackHook, (__bridge void *)callback); +} + +static void __SQLiteCreateFunction(sqlite3_context * context, int argc, sqlite3_value ** argv) { + ((__bridge _SQLiteCreateFunctionCallback)sqlite3_user_data(context))((SQLiteContext *)context, argc, (SQLiteValue **)argv); +} + +int _SQLiteCreateFunction(SQLiteHandle * handle, const char * name, int argc, int deterministic, _SQLiteCreateFunctionCallback callback) { + if (callback) { + int flags = SQLITE_UTF8; + if (deterministic) { +#ifdef SQLITE_DETERMINISTIC + flags |= SQLITE_DETERMINISTIC; +#endif + } + return sqlite3_create_function_v2((sqlite3 *)handle, name, -1, flags, (__bridge void *)callback, &__SQLiteCreateFunction, 0, 0, 0); + } else { + return sqlite3_create_function_v2((sqlite3 *)handle, name, 0, 0, 0, 0, 0, 0, 0); + } +} + +static int __SQLiteCreateCollation(void * context, int len_lhs, const void * lhs, int len_rhs, const void * rhs) { + return ((__bridge _SQLiteCreateCollationCallback)context)(lhs, rhs); +} + +int _SQLiteCreateCollation(SQLiteHandle * handle, const char * name, _SQLiteCreateCollationCallback callback) { + if (callback) { + return sqlite3_create_collation_v2((sqlite3 *)handle, name, SQLITE_UTF8, (__bridge void *)callback, &__SQLiteCreateCollation, 0); + } else { + return sqlite3_create_collation_v2((sqlite3 *)handle, name, 0, 0, 0, 0); + } +} + +#pragma mark - FTS + +typedef struct __SQLiteTokenizer { + sqlite3_tokenizer base; + __unsafe_unretained _SQLiteTokenizerNextCallback callback; +} __SQLiteTokenizer; + +typedef struct __SQLiteTokenizerCursor { + void * base; + const char * input; + int inputOffset; + int inputLength; + int idx; +} __SQLiteTokenizerCursor; + +static NSMutableDictionary * __SQLiteTokenizerMap; + +static int __SQLiteTokenizerCreate(int argc, const char * const * argv, sqlite3_tokenizer ** ppTokenizer) { + __SQLiteTokenizer * tokenizer = (__SQLiteTokenizer *)sqlite3_malloc(sizeof(__SQLiteTokenizer)); + if (!tokenizer) { + return SQLITE_NOMEM; + } + memset(tokenizer, 0, sizeof(* tokenizer)); // FIXME: needed? + + NSString * key = [NSString stringWithUTF8String:argv[0]]; + tokenizer->callback = [__SQLiteTokenizerMap objectForKey:key]; + if (!tokenizer->callback) { + return SQLITE_ERROR; + } + + *ppTokenizer = &tokenizer->base; + return SQLITE_OK; +} + +static int __SQLiteTokenizerDestroy(sqlite3_tokenizer * pTokenizer) { + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +static int __SQLiteTokenizerOpen(sqlite3_tokenizer * pTokenizer, const char * pInput, int nBytes, sqlite3_tokenizer_cursor ** ppCursor) { + __SQLiteTokenizerCursor * cursor = (__SQLiteTokenizerCursor *)sqlite3_malloc(sizeof(__SQLiteTokenizerCursor)); + if (!cursor) { + return SQLITE_NOMEM; + } + + cursor->input = pInput; + cursor->inputOffset = 0; + cursor->inputLength = 0; + cursor->idx = 0; + + *ppCursor = (sqlite3_tokenizer_cursor *)cursor; + return SQLITE_OK; +} + +static int __SQLiteTokenizerClose(sqlite3_tokenizer_cursor * pCursor) { + sqlite3_free(pCursor); + return SQLITE_OK; +} + +static int __SQLiteTokenizerNext(sqlite3_tokenizer_cursor * pCursor, const char ** ppToken, int * pnBytes, int * piStartOffset, int * piEndOffset, int * piPosition) { + __SQLiteTokenizerCursor * cursor = (__SQLiteTokenizerCursor *)pCursor; + __SQLiteTokenizer * tokenizer = (__SQLiteTokenizer *)cursor->base; + + cursor->inputOffset += cursor->inputLength; + const char * input = cursor->input + cursor->inputOffset; + const char * token = [tokenizer->callback(input, &cursor->inputOffset, &cursor->inputLength) cStringUsingEncoding:NSUTF8StringEncoding]; + if (!token) { + return SQLITE_DONE; + } + + *ppToken = token; + *pnBytes = (int)strlen(token); + *piStartOffset = cursor->inputOffset; + *piEndOffset = cursor->inputOffset + cursor->inputLength; + *piPosition = cursor->idx++; + return SQLITE_OK; +} + +static const sqlite3_tokenizer_module __SQLiteTokenizerModule = { + 0, + __SQLiteTokenizerCreate, + __SQLiteTokenizerDestroy, + __SQLiteTokenizerOpen, + __SQLiteTokenizerClose, + __SQLiteTokenizerNext +}; + +int _SQLiteRegisterTokenizer(SQLiteHandle * db, const char * moduleName, const char * submoduleName, _SQLiteTokenizerNextCallback callback) { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + __SQLiteTokenizerMap = [NSMutableDictionary new]; + }); + + sqlite3_stmt * stmt; + int status = sqlite3_prepare_v2((sqlite3 *)db, "SELECT fts3_tokenizer(?, ?)", -1, &stmt, 0); + if (status != SQLITE_OK ){ + return status; + } + const sqlite3_tokenizer_module * pModule = &__SQLiteTokenizerModule; + sqlite3_bind_text(stmt, 1, moduleName, -1, SQLITE_STATIC); + sqlite3_bind_blob(stmt, 2, &pModule, sizeof(pModule), SQLITE_STATIC); + sqlite3_step(stmt); + status = sqlite3_finalize(stmt); + if (status != SQLITE_OK ){ + return status; + } + + [__SQLiteTokenizerMap setObject:[callback copy] forKey:[NSString stringWithUTF8String:submoduleName]]; + + return SQLITE_OK; +} diff --git a/Postbox/PostboxPrivate/sqlcipher/fts3_tokenizer.h b/Postbox/PostboxPrivate/sqlcipher/fts3_tokenizer.h new file mode 100644 index 0000000000..4a40b2b385 --- /dev/null +++ b/Postbox/PostboxPrivate/sqlcipher/fts3_tokenizer.h @@ -0,0 +1,161 @@ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ + +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +#include "sqlite3.h" + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + + /* + ** Structure version. Should always be set to 0 or 1. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; + +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); + + +#endif /* _FTS3_TOKENIZER_H_ */ diff --git a/Postbox/PostboxPrivate/sqlcipher/sqlite3.c b/Postbox/PostboxPrivate/sqlcipher/sqlite3.c new file mode 100644 index 0000000000..0052bfcdb8 --- /dev/null +++ b/Postbox/PostboxPrivate/sqlcipher/sqlite3.c @@ -0,0 +1,151821 @@ +/****************************************************************************** +** This file is an amalgamation of many separate C source files from SQLite +** version 3.8.6. By combining all the individual C code files into this +** single large file, the entire code can be compiled as a single translation +** unit. This allows many compilers to do optimizations that would not be +** possible if the files were compiled separately. Performance improvements +** of 5% or more are commonly seen when SQLite is compiled as a single +** translation unit. +** +** This file is all you need to compile SQLite. To use SQLite in other +** programs, you need this file and the "sqlite3.h" header file that defines +** the programming interface to the SQLite library. (If you do not have +** the "sqlite3.h" header file at hand, you will find a copy embedded within +** the text of this file. Search for "Begin file sqlite3.h" to find the start +** of the embedded sqlite3.h header file.) Additional code files may be needed +** if you want a wrapper to interface SQLite with your choice of programming +** language. The code for the "sqlite3" command-line shell is also in a +** separate file. This file contains only code for the core SQLite library. +*/ +#define SQLITE_CORE 1 +#define SQLITE_AMALGAMATION 1 +#ifndef SQLITE_PRIVATE +# define SQLITE_PRIVATE static +#endif +#ifndef SQLITE_API +# define SQLITE_API +#endif +/************** Begin file sqliteInt.h ***************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Internal interface definitions for SQLite. +** +*/ +#ifndef _SQLITEINT_H_ +#define _SQLITEINT_H_ + +/* +** These #defines should enable >2GB file support on POSIX if the +** underlying operating system supports it. If the OS lacks +** large file support, or if the OS is windows, these should be no-ops. +** +** Ticket #2739: The _LARGEFILE_SOURCE macro must appear before any +** system #includes. Hence, this block of code must be the very first +** code in all source files. +** +** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch +** on the compiler command line. This is necessary if you are compiling +** on a recent machine (ex: Red Hat 7.2) but you want your code to work +** on an older machine (ex: Red Hat 6.0). If you compile on Red Hat 7.2 +** without this option, LFS is enable. But LFS does not exist in the kernel +** in Red Hat 6.0, so the code won't work. Hence, for maximum binary +** portability you should omit LFS. +** +** The previous paragraph was written in 2005. (This paragraph is written +** on 2008-11-28.) These days, all Linux kernels support large files, so +** you should probably leave LFS enabled. But some embedded platforms might +** lack LFS in which case the SQLITE_DISABLE_LFS macro might still be useful. +** +** Similar is true for Mac OS X. LFS is only supported on Mac OS X 9 and later. +*/ +#ifndef SQLITE_DISABLE_LFS +# define _LARGE_FILE 1 +# ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +# endif +# define _LARGEFILE_SOURCE 1 +#endif + +/* +** For MinGW, check to see if we can include the header file containing its +** version information, among other things. Normally, this internal MinGW +** header file would [only] be included automatically by other MinGW header +** files; however, the contained version information is now required by this +** header file to work around binary compatibility issues (see below) and +** this is the only known way to reliably obtain it. This entire #if block +** would be completely unnecessary if there was any other way of detecting +** MinGW via their preprocessor (e.g. if they customized their GCC to define +** some MinGW-specific macros). When compiling for MinGW, either the +** _HAVE_MINGW_H or _HAVE__MINGW_H (note the extra underscore) macro must be +** defined; otherwise, detection of conditions specific to MinGW will be +** disabled. +*/ +#if defined(_HAVE_MINGW_H) +# include "mingw.h" +#elif defined(_HAVE__MINGW_H) +# include "_mingw.h" +#endif + +/* +** For MinGW version 4.x (and higher), check to see if the _USE_32BIT_TIME_T +** define is required to maintain binary compatibility with the MSVC runtime +** library in use (e.g. for Windows XP). +*/ +#if !defined(_USE_32BIT_TIME_T) && !defined(_USE_64BIT_TIME_T) && \ + defined(_WIN32) && !defined(_WIN64) && \ + defined(__MINGW_MAJOR_VERSION) && __MINGW_MAJOR_VERSION >= 4 && \ + defined(__MSVCRT__) +# define _USE_32BIT_TIME_T +#endif + +/* The public SQLite interface. The _FILE_OFFSET_BITS macro must appear +** first in QNX. Also, the _USE_32BIT_TIME_T macro must appear first for +** MinGW. +*/ +/************** Include sqlite3.h in the middle of sqliteInt.h ***************/ +/************** Begin file sqlite3.h *****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the SQLite library +** presents to client programs. If a C-function, structure, datatype, +** or constant definition does not appear in this file, then it is +** not a published API of SQLite, is subject to change without +** notice, and should not be referenced by programs that use SQLite. +** +** Some of the definitions that are in this file are marked as +** "experimental". Experimental interfaces are normally new +** features recently added to SQLite. We do not anticipate changes +** to experimental interfaces but reserve the right to make minor changes +** if experience from use "in the wild" suggest such changes are prudent. +** +** The official C-language API documentation for SQLite is derived +** from comments in this file. This file is the authoritative source +** on how SQLite interfaces are suppose to operate. +** +** The name of this file under configuration management is "sqlite.h.in". +** The makefile makes some minor changes to this file (such as inserting +** the version number) and changes its name to "sqlite3.h" as +** part of the build process. +*/ +#ifndef _SQLITE3_H_ +#define _SQLITE3_H_ +#include /* Needed for the definition of va_list */ + +/* +** Make sure we can call this stuff from C++. +*/ +#if 0 +extern "C" { +#endif + + +/* +** Add the ability to override 'extern' +*/ +#ifndef SQLITE_EXTERN +# define SQLITE_EXTERN extern +#endif + +#ifndef SQLITE_API +# define SQLITE_API +#endif + + +/* +** These no-op macros are used in front of interfaces to mark those +** interfaces as either deprecated or experimental. New applications +** should not use deprecated interfaces - they are support for backwards +** compatibility only. Application writers should be aware that +** experimental interfaces are subject to change in point releases. +** +** These macros used to resolve to various kinds of compiler magic that +** would generate warning messages when they were used. But that +** compiler magic ended up generating such a flurry of bug reports +** that we have taken it all out and gone back to using simple +** noop macros. +*/ +#define SQLITE_DEPRECATED +#define SQLITE_EXPERIMENTAL + +/* +** Ensure these symbols were not defined by some previous header file. +*/ +#ifdef SQLITE_VERSION +# undef SQLITE_VERSION +#endif +#ifdef SQLITE_VERSION_NUMBER +# undef SQLITE_VERSION_NUMBER +#endif + +/* +** CAPI3REF: Compile-Time Library Version Numbers +** +** ^(The [SQLITE_VERSION] C preprocessor macro in the sqlite3.h header +** evaluates to a string literal that is the SQLite version in the +** format "X.Y.Z" where X is the major version number (always 3 for +** SQLite3) and Y is the minor version number and Z is the release number.)^ +** ^(The [SQLITE_VERSION_NUMBER] C preprocessor macro resolves to an integer +** with the value (X*1000000 + Y*1000 + Z) where X, Y, and Z are the same +** numbers used in [SQLITE_VERSION].)^ +** The SQLITE_VERSION_NUMBER for any given release of SQLite will also +** be larger than the release from which it is derived. Either Y will +** be held constant and Z will be incremented or else Y will be incremented +** and Z will be reset to zero. +** +** Since version 3.6.18, SQLite source code has been stored in the +** Fossil configuration management +** system. ^The SQLITE_SOURCE_ID macro evaluates to +** a string which identifies a particular check-in of SQLite +** within its configuration management system. ^The SQLITE_SOURCE_ID +** string contains the date and time of the check-in (UTC) and an SHA1 +** hash of the entire source tree. +** +** See also: [sqlite3_libversion()], +** [sqlite3_libversion_number()], [sqlite3_sourceid()], +** [sqlite_version()] and [sqlite_source_id()]. +*/ +#define SQLITE_VERSION "3.8.6" +#define SQLITE_VERSION_NUMBER 3008006 +#define SQLITE_SOURCE_ID "2014-08-15 11:46:33 9491ba7d738528f168657adb43a198238abde19e" + +/* +** CAPI3REF: Run-Time Library Version Numbers +** KEYWORDS: sqlite3_version, sqlite3_sourceid +** +** These interfaces provide the same information as the [SQLITE_VERSION], +** [SQLITE_VERSION_NUMBER], and [SQLITE_SOURCE_ID] C preprocessor macros +** but are associated with the library instead of the header file. ^(Cautious +** programmers might include assert() statements in their application to +** verify that values returned by these interfaces match the macros in +** the header, and thus insure that the application is +** compiled with matching library and header files. +** +** 
+** assert( sqlite3_libversion_number()==SQLITE_VERSION_NUMBER );
+** assert( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)==0 );
+** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
+**
)^ +** +** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] +** macro. ^The sqlite3_libversion() function returns a pointer to the +** to the sqlite3_version[] string constant. The sqlite3_libversion() +** function is provided for use in DLLs since DLL users usually do not have +** direct access to string constants within the DLL. ^The +** sqlite3_libversion_number() function returns an integer equal to +** [SQLITE_VERSION_NUMBER]. ^The sqlite3_sourceid() function returns +** a pointer to a string constant whose value is the same as the +** [SQLITE_SOURCE_ID] C preprocessor macro. +** +** See also: [sqlite_version()] and [sqlite_source_id()]. +*/ +SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +SQLITE_API const char *sqlite3_libversion(void); +SQLITE_API const char *sqlite3_sourceid(void); +SQLITE_API int sqlite3_libversion_number(void); + +/* +** CAPI3REF: Run-Time Library Compilation Options Diagnostics +** +** ^The sqlite3_compileoption_used() function returns 0 or 1 +** indicating whether the specified option was defined at +** compile time. ^The SQLITE_ prefix may be omitted from the +** option name passed to sqlite3_compileoption_used(). +** +** ^The sqlite3_compileoption_get() function allows iterating +** over the list of options that were defined at compile time by +** returning the N-th compile time option string. ^If N is out of range, +** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ +** prefix is omitted from any strings returned by +** sqlite3_compileoption_get(). +** +** ^Support for the diagnostic functions sqlite3_compileoption_used() +** and sqlite3_compileoption_get() may be omitted by specifying the +** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. +** +** See also: SQL functions [sqlite_compileoption_used()] and +** [sqlite_compileoption_get()] and the [compile_options pragma]. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +SQLITE_API int sqlite3_compileoption_used(const char *zOptName); +SQLITE_API const char *sqlite3_compileoption_get(int N); +#endif + +/* +** CAPI3REF: Test To See If The Library Is Threadsafe +** +** ^The sqlite3_threadsafe() function returns zero if and only if +** SQLite was compiled with mutexing code omitted due to the +** [SQLITE_THREADSAFE] compile-time option being set to 0. +** +** SQLite can be compiled with or without mutexes. When +** the [SQLITE_THREADSAFE] C preprocessor macro is 1 or 2, mutexes +** are enabled and SQLite is threadsafe. When the +** [SQLITE_THREADSAFE] macro is 0, +** the mutexes are omitted. Without the mutexes, it is not safe +** to use SQLite concurrently from more than one thread. +** +** Enabling mutexes incurs a measurable performance penalty. +** So if speed is of utmost importance, it makes sense to disable +** the mutexes. But for maximum safety, mutexes should be enabled. +** ^The default behavior is for mutexes to be enabled. +** +** This interface can be used by an application to make sure that the +** version of SQLite that it is linking against was compiled with +** the desired setting of the [SQLITE_THREADSAFE] macro. +** +** This interface only reports on the compile-time mutex setting +** of the [SQLITE_THREADSAFE] flag. If SQLite is compiled with +** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but +** can be fully or partially disabled using a call to [sqlite3_config()] +** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD], +** or [SQLITE_CONFIG_MUTEX]. ^(The return value of the +** sqlite3_threadsafe() function shows only the compile-time setting of +** thread safety, not any run-time changes to that setting made by +** sqlite3_config(). In other words, the return value from sqlite3_threadsafe() +** is unchanged by calls to sqlite3_config().)^ +** +** See the [threading mode] documentation for additional information. +*/ +SQLITE_API int sqlite3_threadsafe(void); + +/* +** CAPI3REF: Database Connection Handle +** KEYWORDS: {database connection} {database connections} +** +** Each open SQLite database is represented by a pointer to an instance of +** the opaque structure named "sqlite3". It is useful to think of an sqlite3 +** pointer as an object. The [sqlite3_open()], [sqlite3_open16()], and +** [sqlite3_open_v2()] interfaces are its constructors, and [sqlite3_close()] +** and [sqlite3_close_v2()] are its destructors. There are many other +** interfaces (such as +** [sqlite3_prepare_v2()], [sqlite3_create_function()], and +** [sqlite3_busy_timeout()] to name but three) that are methods on an +** sqlite3 object. +*/ +typedef struct sqlite3 sqlite3; + +/* +** CAPI3REF: 64-Bit Integer Types +** KEYWORDS: sqlite_int64 sqlite_uint64 +** +** Because there is no cross-platform way to specify 64-bit integer types +** SQLite includes typedefs for 64-bit signed and unsigned integers. +** +** The sqlite3_int64 and sqlite3_uint64 are the preferred type definitions. +** The sqlite_int64 and sqlite_uint64 types are supported for backwards +** compatibility only. +** +** ^The sqlite3_int64 and sqlite_int64 types can store integer values +** between -9223372036854775808 and +9223372036854775807 inclusive. ^The +** sqlite3_uint64 and sqlite_uint64 types can store integer values +** between 0 and +18446744073709551615 inclusive. +*/ +#ifdef SQLITE_INT64_TYPE + typedef SQLITE_INT64_TYPE sqlite_int64; + typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +#elif defined(_MSC_VER) || defined(__BORLANDC__) + typedef __int64 sqlite_int64; + typedef unsigned __int64 sqlite_uint64; +#else + typedef long long int sqlite_int64; + typedef unsigned long long int sqlite_uint64; +#endif +typedef sqlite_int64 sqlite3_int64; +typedef sqlite_uint64 sqlite3_uint64; + +/* +** If compiling for a processor that lacks floating point support, +** substitute integer for floating-point. +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# define double sqlite3_int64 +#endif + +/* +** CAPI3REF: Closing A Database Connection +** +** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors +** for the [sqlite3] object. +** ^Calls to sqlite3_close() and sqlite3_close_v2() return [SQLITE_OK] if +** the [sqlite3] object is successfully destroyed and all associated +** resources are deallocated. +** +** ^If the database connection is associated with unfinalized prepared +** statements or unfinished sqlite3_backup objects then sqlite3_close() +** will leave the database connection open and return [SQLITE_BUSY]. +** ^If sqlite3_close_v2() is called with unfinalized prepared statements +** and/or unfinished sqlite3_backups, then the database connection becomes +** an unusable "zombie" which will automatically be deallocated when the +** last prepared statement is finalized or the last sqlite3_backup is +** finished. The sqlite3_close_v2() interface is intended for use with +** host languages that are garbage collected, and where the order in which +** destructors are called is arbitrary. +** +** Applications should [sqlite3_finalize | finalize] all [prepared statements], +** [sqlite3_blob_close | close] all [BLOB handles], and +** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated +** with the [sqlite3] object prior to attempting to close the object. ^If +** sqlite3_close_v2() is called on a [database connection] that still has +** outstanding [prepared statements], [BLOB handles], and/or +** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation +** of resources is deferred until all [prepared statements], [BLOB handles], +** and [sqlite3_backup] objects are also destroyed. +** +** ^If an [sqlite3] object is destroyed while a transaction is open, +** the transaction is automatically rolled back. +** +** The C parameter to [sqlite3_close(C)] and [sqlite3_close_v2(C)] +** must be either a NULL +** pointer or an [sqlite3] object pointer obtained +** from [sqlite3_open()], [sqlite3_open16()], or +** [sqlite3_open_v2()], and not previously closed. +** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer +** argument is a harmless no-op. +*/ +SQLITE_API int sqlite3_close(sqlite3*); +SQLITE_API int sqlite3_close_v2(sqlite3*); + +/* +** The type for a callback function. +** This is legacy and deprecated. It is included for historical +** compatibility and is not documented. +*/ +typedef int (*sqlite3_callback)(void*,int,char**, char**); + +/* +** CAPI3REF: One-Step Query Execution Interface +** +** The sqlite3_exec() interface is a convenience wrapper around +** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()], +** that allows an application to run multiple statements of SQL +** without having to use a lot of C code. +** +** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, +** semicolon-separate SQL statements passed into its 2nd argument, +** in the context of the [database connection] passed in as its 1st +** argument. ^If the callback function of the 3rd argument to +** sqlite3_exec() is not NULL, then it is invoked for each result row +** coming out of the evaluated SQL statements. ^The 4th argument to +** sqlite3_exec() is relayed through to the 1st argument of each +** callback invocation. ^If the callback pointer to sqlite3_exec() +** is NULL, then no callback is ever invoked and result rows are +** ignored. +** +** ^If an error occurs while evaluating the SQL statements passed into +** sqlite3_exec(), then execution of the current statement stops and +** subsequent statements are skipped. ^If the 5th parameter to sqlite3_exec() +** is not NULL then any error message is written into memory obtained +** from [sqlite3_malloc()] and passed back through the 5th parameter. +** To avoid memory leaks, the application should invoke [sqlite3_free()] +** on error message strings returned through the 5th parameter of +** of sqlite3_exec() after the error message string is no longer needed. +** ^If the 5th parameter to sqlite3_exec() is not NULL and no errors +** occur, then sqlite3_exec() sets the pointer in its 5th parameter to +** NULL before returning. +** +** ^If an sqlite3_exec() callback returns non-zero, the sqlite3_exec() +** routine returns SQLITE_ABORT without invoking the callback again and +** without running any subsequent SQL statements. +** +** ^The 2nd argument to the sqlite3_exec() callback function is the +** number of columns in the result. ^The 3rd argument to the sqlite3_exec() +** callback is an array of pointers to strings obtained as if from +** [sqlite3_column_text()], one for each column. ^If an element of a +** result row is NULL then the corresponding string pointer for the +** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the +** sqlite3_exec() callback is an array of pointers to strings where each +** entry represents the name of corresponding result column as obtained +** from [sqlite3_column_name()]. +** +** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer +** to an empty string, or a pointer that contains only whitespace and/or +** SQL comments, then no SQL statements are evaluated and the database +** is not changed. +** +** Restrictions: +** +**
    +**
  • The application must insure that the 1st parameter to sqlite3_exec() +** is a valid and open [database connection]. +**
  • The application must not close the [database connection] specified by +** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not modify the SQL statement text passed into +** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
+*/ +SQLITE_API int sqlite3_exec( + sqlite3*, /* An open database */ + const char *sql, /* SQL to be evaluated */ + int (*callback)(void*,int,char**,char**), /* Callback function */ + void *, /* 1st argument to callback */ + char **errmsg /* Error msg written here */ +); + +/* +** CAPI3REF: Result Codes +** KEYWORDS: {result code definitions} +** +** Many SQLite functions return an integer result code from the set shown +** here in order to indicate success or failure. +** +** New error codes may be added in future versions of SQLite. +** +** See also: [extended result code definitions] +*/ +#define SQLITE_OK 0 /* Successful result */ +/* beginning-of-error-codes */ +#define SQLITE_ERROR 1 /* SQL error or missing database */ +#define SQLITE_INTERNAL 2 /* Internal logic error in SQLite */ +#define SQLITE_PERM 3 /* Access permission denied */ +#define SQLITE_ABORT 4 /* Callback routine requested an abort */ +#define SQLITE_BUSY 5 /* The database file is locked */ +#define SQLITE_LOCKED 6 /* A table in the database is locked */ +#define SQLITE_NOMEM 7 /* A malloc() failed */ +#define SQLITE_READONLY 8 /* Attempt to write a readonly database */ +#define SQLITE_INTERRUPT 9 /* Operation terminated by sqlite3_interrupt()*/ +#define SQLITE_IOERR 10 /* Some kind of disk I/O error occurred */ +#define SQLITE_CORRUPT 11 /* The database disk image is malformed */ +#define SQLITE_NOTFOUND 12 /* Unknown opcode in sqlite3_file_control() */ +#define SQLITE_FULL 13 /* Insertion failed because database is full */ +#define SQLITE_CANTOPEN 14 /* Unable to open the database file */ +#define SQLITE_PROTOCOL 15 /* Database lock protocol error */ +#define SQLITE_EMPTY 16 /* Database is empty */ +#define SQLITE_SCHEMA 17 /* The database schema changed */ +#define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ +#define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ +#define SQLITE_MISMATCH 20 /* Data type mismatch */ +#define SQLITE_MISUSE 21 /* Library used incorrectly */ +#define SQLITE_NOLFS 22 /* Uses OS features not supported on host */ +#define SQLITE_AUTH 23 /* Authorization denied */ +#define SQLITE_FORMAT 24 /* Auxiliary database format error */ +#define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ +#define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ +#define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ +#define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ +/* end-of-error-codes */ + +/* +** CAPI3REF: Extended Result Codes +** KEYWORDS: {extended result code definitions} +** +** In its default configuration, SQLite API routines return one of 30 integer +** [result codes]. However, experience has shown that many of +** these result codes are too coarse-grained. They do not provide as +** much information about problems as programmers might like. In an effort to +** address this, newer versions of SQLite (version 3.3.8 and later) include +** support for additional result codes that provide more detailed information +** about errors. These [extended result codes] are enabled or disabled +** on a per database connection basis using the +** [sqlite3_extended_result_codes()] API. Or, the extended code for +** the most recent error can be obtained using +** [sqlite3_extended_errcode()]. +*/ +#define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) +#define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) +#define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) +#define SQLITE_IOERR_FSYNC (SQLITE_IOERR | (4<<8)) +#define SQLITE_IOERR_DIR_FSYNC (SQLITE_IOERR | (5<<8)) +#define SQLITE_IOERR_TRUNCATE (SQLITE_IOERR | (6<<8)) +#define SQLITE_IOERR_FSTAT (SQLITE_IOERR | (7<<8)) +#define SQLITE_IOERR_UNLOCK (SQLITE_IOERR | (8<<8)) +#define SQLITE_IOERR_RDLOCK (SQLITE_IOERR | (9<<8)) +#define SQLITE_IOERR_DELETE (SQLITE_IOERR | (10<<8)) +#define SQLITE_IOERR_BLOCKED (SQLITE_IOERR | (11<<8)) +#define SQLITE_IOERR_NOMEM (SQLITE_IOERR | (12<<8)) +#define SQLITE_IOERR_ACCESS (SQLITE_IOERR | (13<<8)) +#define SQLITE_IOERR_CHECKRESERVEDLOCK (SQLITE_IOERR | (14<<8)) +#define SQLITE_IOERR_LOCK (SQLITE_IOERR | (15<<8)) +#define SQLITE_IOERR_CLOSE (SQLITE_IOERR | (16<<8)) +#define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8)) +#define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) +#define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) +#define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) +#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) +#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) +#define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) +#define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) +#define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) +#define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) +#define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) +#define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) +#define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) + +/* +** CAPI3REF: Flags For File Open Operations +** +** These bit values are intended for use in the +** 3rd parameter to the [sqlite3_open_v2()] interface and +** in the 4th parameter to the [sqlite3_vfs.xOpen] method. +*/ +#define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_CREATE 0x00000004 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ +#define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ +#define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ +#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MEMORY 0x00000080 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ +#define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ +#define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ +#define SQLITE_OPEN_MAIN_JOURNAL 0x00000800 /* VFS only */ +#define SQLITE_OPEN_TEMP_JOURNAL 0x00001000 /* VFS only */ +#define SQLITE_OPEN_SUBJOURNAL 0x00002000 /* VFS only */ +#define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 /* VFS only */ +#define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ + +/* Reserved: 0x00F00000 */ + +/* +** CAPI3REF: Device Characteristics +** +** The xDeviceCharacteristics method of the [sqlite3_io_methods] +** object returns an integer which is a vector of these +** bit values expressing I/O characteristics of the mass storage +** device that holds the file that the [sqlite3_io_methods] +** refers to. +** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). The SQLITE_IOCAP_POWERSAFE_OVERWRITE property means that +** after reboot following a crash or power loss, the only bytes in a +** file that were written at the application level might have changed +** and that adjacent bytes, even bytes within the same sector are +** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN +** flag indicate that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. +*/ +#define SQLITE_IOCAP_ATOMIC 0x00000001 +#define SQLITE_IOCAP_ATOMIC512 0x00000002 +#define SQLITE_IOCAP_ATOMIC1K 0x00000004 +#define SQLITE_IOCAP_ATOMIC2K 0x00000008 +#define SQLITE_IOCAP_ATOMIC4K 0x00000010 +#define SQLITE_IOCAP_ATOMIC8K 0x00000020 +#define SQLITE_IOCAP_ATOMIC16K 0x00000040 +#define SQLITE_IOCAP_ATOMIC32K 0x00000080 +#define SQLITE_IOCAP_ATOMIC64K 0x00000100 +#define SQLITE_IOCAP_SAFE_APPEND 0x00000200 +#define SQLITE_IOCAP_SEQUENTIAL 0x00000400 +#define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 +#define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 + +/* +** CAPI3REF: File Locking Levels +** +** SQLite uses one of these integer values as the second +** argument to calls it makes to the xLock() and xUnlock() methods +** of an [sqlite3_io_methods] object. +*/ +#define SQLITE_LOCK_NONE 0 +#define SQLITE_LOCK_SHARED 1 +#define SQLITE_LOCK_RESERVED 2 +#define SQLITE_LOCK_PENDING 3 +#define SQLITE_LOCK_EXCLUSIVE 4 + +/* +** CAPI3REF: Synchronization Type Flags +** +** When SQLite invokes the xSync() method of an +** [sqlite3_io_methods] object it uses a combination of +** these integer values as the second argument. +** +** When the SQLITE_SYNC_DATAONLY flag is used, it means that the +** sync operation only needs to flush data to mass storage. Inode +** information need not be flushed. If the lower four bits of the flag +** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics. +** If the lower four bits equal SQLITE_SYNC_FULL, that means +** to use Mac OS X style fullsync instead of fsync(). +** +** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags +** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL +** settings. The [synchronous pragma] determines when calls to the +** xSync VFS method occur and applies uniformly across all platforms. +** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how +** energetic or rigorous or forceful the sync operations are and +** only make a difference on Mac OSX for the default SQLite code. +** (Third-party VFS implementations might also make the distinction +** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the +** operating systems natively supported by SQLite, only Mac OSX +** cares about the difference.) +*/ +#define SQLITE_SYNC_NORMAL 0x00002 +#define SQLITE_SYNC_FULL 0x00003 +#define SQLITE_SYNC_DATAONLY 0x00010 + +/* +** CAPI3REF: OS Interface Open File Handle +** +** An [sqlite3_file] object represents an open file in the +** [sqlite3_vfs | OS interface layer]. Individual OS interface +** implementations will +** want to subclass this object by appending additional fields +** for their own use. The pMethods entry is a pointer to an +** [sqlite3_io_methods] object that defines methods for performing +** I/O operations on the open file. +*/ +typedef struct sqlite3_file sqlite3_file; +struct sqlite3_file { + const struct sqlite3_io_methods *pMethods; /* Methods for an open file */ +}; + +/* +** CAPI3REF: OS Interface File Virtual Methods Object +** +** Every file opened by the [sqlite3_vfs.xOpen] method populates an +** [sqlite3_file] object (or, more commonly, a subclass of the +** [sqlite3_file] object) with a pointer to an instance of this object. +** This object defines the methods used to perform various operations +** against the open file represented by the [sqlite3_file] object. +** +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element +** to a non-NULL pointer, then the sqlite3_io_methods.xClose method +** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The +** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] +** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element +** to NULL. +** +** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or +** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). +** The second choice is a Mac OS X style fullsync. The [SQLITE_SYNC_DATAONLY] +** flag may be ORed in to indicate that only the data of the file +** and not its inode needs to be synced. +** +** The integer values to xLock() and xUnlock() are one of +**
    +**
  • [SQLITE_LOCK_NONE], +**
  • [SQLITE_LOCK_SHARED], +**
  • [SQLITE_LOCK_RESERVED], +**
  • [SQLITE_LOCK_PENDING], or +**
  • [SQLITE_LOCK_EXCLUSIVE]. +**
+** xLock() increases the lock. xUnlock() decreases the lock. +** The xCheckReservedLock() method checks whether any database connection, +** either in this process or in some other process, is holding a RESERVED, +** PENDING, or EXCLUSIVE lock on the file. It returns true +** if such a lock exists and false otherwise. +** +** The xFileControl() method is a generic interface that allows custom +** VFS implementations to directly control an open file using the +** [sqlite3_file_control()] interface. The second "op" argument is an +** integer opcode. The third argument is a generic pointer intended to +** point to a structure that may contain arguments or space in which to +** write return values. Potential uses for xFileControl() might be +** functions to enable blocking locks with timeouts, to change the +** locking strategy (for example to use dot-file locks), to inquire +** about the status of a lock, or to break stale locks. The SQLite +** core reserves all opcodes less than 100 for its own use. +** A [file control opcodes | list of opcodes] less than 100 is available. +** Applications that define a custom xFileControl method should use opcodes +** greater than 100 to avoid conflicts. VFS implementations should +** return [SQLITE_NOTFOUND] for file control opcodes that they do not +** recognize. +** +** The xSectorSize() method returns the sector size of the +** device that underlies the file. The sector size is the +** minimum write that can be performed without disturbing +** other bytes in the file. The xDeviceCharacteristics() +** method returns a bit vector describing behaviors of the +** underlying device: +** +**
    +**
  • [SQLITE_IOCAP_ATOMIC] +**
  • [SQLITE_IOCAP_ATOMIC512] +**
  • [SQLITE_IOCAP_ATOMIC1K] +**
  • [SQLITE_IOCAP_ATOMIC2K] +**
  • [SQLITE_IOCAP_ATOMIC4K] +**
  • [SQLITE_IOCAP_ATOMIC8K] +**
  • [SQLITE_IOCAP_ATOMIC16K] +**
  • [SQLITE_IOCAP_ATOMIC32K] +**
  • [SQLITE_IOCAP_ATOMIC64K] +**
  • [SQLITE_IOCAP_SAFE_APPEND] +**
  • [SQLITE_IOCAP_SEQUENTIAL] +**
+** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). +** +** If xRead() returns SQLITE_IOERR_SHORT_READ it must also fill +** in the unread portions of the buffer with zeros. A VFS that +** fails to zero-fill short reads might seem to work. However, +** failure to zero-fill short reads will eventually lead to +** database corruption. +*/ +typedef struct sqlite3_io_methods sqlite3_io_methods; +struct sqlite3_io_methods { + int iVersion; + int (*xClose)(sqlite3_file*); + int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); + int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst); + int (*xTruncate)(sqlite3_file*, sqlite3_int64 size); + int (*xSync)(sqlite3_file*, int flags); + int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize); + int (*xLock)(sqlite3_file*, int); + int (*xUnlock)(sqlite3_file*, int); + int (*xCheckReservedLock)(sqlite3_file*, int *pResOut); + int (*xFileControl)(sqlite3_file*, int op, void *pArg); + int (*xSectorSize)(sqlite3_file*); + int (*xDeviceCharacteristics)(sqlite3_file*); + /* Methods above are valid for version 1 */ + int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); + int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); + void (*xShmBarrier)(sqlite3_file*); + int (*xShmUnmap)(sqlite3_file*, int deleteFlag); + /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ + /* Additional methods may be added in future releases */ +}; + +/* +** CAPI3REF: Standard File Control Opcodes +** KEYWORDS: {file control opcodes} {file control opcode} +** +** These integer constants are opcodes for the xFileControl method +** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] +** interface. +** +** The [SQLITE_FCNTL_LOCKSTATE] opcode is used for debugging. This +** opcode causes the xFileControl method to write the current state of +** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], +** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) +** into an integer that the pArg argument points to. This capability +** is used during testing and only needs to be supported when SQLITE_TEST +** is defined. +**
    +**
  • [[SQLITE_FCNTL_SIZE_HINT]] +** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS +** layer a hint of how large the database file will grow to be during the +** current transaction. This hint is not guaranteed to be accurate but it +** is often close. The underlying VFS might choose to preallocate database +** file space based on this hint in order to help writes to the database +** file run faster. +** +**
  • [[SQLITE_FCNTL_CHUNK_SIZE]] +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. +** +**
  • [[SQLITE_FCNTL_FILE_POINTER]] +** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with a particular database +** connection. See the [sqlite3_file_control()] documentation for +** additional information. +** +**
  • [[SQLITE_FCNTL_SYNC_OMITTED]] +** No longer in use. +** +**
  • [[SQLITE_FCNTL_SYNC]] +** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and +** sent to the VFS immediately before the xSync method is invoked on a +** database file descriptor. Or, if the xSync method is not invoked +** because the user has configured SQLite with +** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place +** of the xSync method. In most cases, the pointer argument passed with +** this file-control is NULL. However, if the database file is being synced +** as part of a multi-database commit, the argument points to a nul-terminated +** string containing the transactions master-journal file name. VFSes that +** do not need this signal should silently ignore this opcode. Applications +** should not call [sqlite3_file_control()] with this opcode as doing so may +** disrupt the operation of the specialized VFSes that do require it. +** +**
  • [[SQLITE_FCNTL_COMMIT_PHASETWO]] +** The [SQLITE_FCNTL_COMMIT_PHASETWO] opcode is generated internally by SQLite +** and sent to the VFS after a transaction has been committed immediately +** but before the database is unlocked. VFSes that do not need this signal +** should silently ignore this opcode. Applications should not call +** [sqlite3_file_control()] with this opcode as doing so may disrupt the +** operation of the specialized VFSes that do require it. +** +**
  • [[SQLITE_FCNTL_WIN32_AV_RETRY]] +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to provide robustness in the presence of +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete operations up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows these two values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer i the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** +**
  • [[SQLITE_FCNTL_PERSIST_WAL]] +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary +** write ahead log and shared memory files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. +** +**
  • [[SQLITE_FCNTL_POWERSAFE_OVERWRITE]] +** ^The [SQLITE_FCNTL_POWERSAFE_OVERWRITE] opcode is used to set or query the +** persistent "powersafe-overwrite" or "PSOW" setting. The PSOW setting +** determines the [SQLITE_IOCAP_POWERSAFE_OVERWRITE] bit of the +** xDeviceCharacteristics methods. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable zero-damage mode or 1 to enable zero-damage +** mode. If the integer is -1, then it is overwritten with the current +** zero-damage mode setting. +** +**
  • [[SQLITE_FCNTL_OVERWRITE]] +** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening +** a write transaction to indicate that, unless it is rolled back for some +** reason, the entire database file will be overwritten by the current +** transaction. This is used by VACUUM operations. +** +**
  • [[SQLITE_FCNTL_VFSNAME]] +** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of +** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** final bottom-level VFS are written into memory obtained from +** [sqlite3_malloc()] and the result is stored in the char* variable +** that the fourth parameter of [sqlite3_file_control()] points to. +** The caller is responsible for freeing the memory when done. As with +** all file-control actions, there is no guarantee that this will actually +** do anything. Callers should initialize the char* variable to a NULL +** pointer in case this file-control is not implemented. This file-control +** is intended for diagnostic use only. +** +**
  • [[SQLITE_FCNTL_PRAGMA]] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^The argument +** to the [SQLITE_FCNTL_PRAGMA] file control is an array of +** pointers to strings (char**) in which the second element of the array +** is the name of the pragma and the third element is the argument to the +** pragma or NULL if the pragma has no argument. ^The handler for an +** [SQLITE_FCNTL_PRAGMA] file control can optionally make the first element +** of the char** argument point to a string obtained from [sqlite3_mprintf()] +** or the equivalent and that string will become the result of the pragma or +** the error message if the pragma fails. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the PRAGMA itself and the parser generates a no-op +** prepared statement. ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means +** that the VFS encountered an error while handling the [PRAGMA] and the +** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] +** file control occurs at the beginning of pragma statement analysis and so +** it is able to override built-in [PRAGMA] statements. +** +**
  • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connections busy-handler callback. The argument is of type (void **) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connections +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
  • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** +**
  • [[SQLITE_FCNTL_TRACE]] +** The [SQLITE_FCNTL_TRACE] file control provides advisory information +** to the VFS about what the higher layers of the SQLite stack are doing. +** This file control is used by some VFS activity tracing [shims]. +** The argument is a zero-terminated string. Higher layers in the +** SQLite stack may generate instances of this file control if +** the [SQLITE_USE_FCNTL_TRACE] compile-time option is enabled. +** +**
  • [[SQLITE_FCNTL_HAS_MOVED]] +** The [SQLITE_FCNTL_HAS_MOVED] file control interprets its argument as a +** pointer to an integer and it writes a boolean into that integer depending +** on whether or not the file has been renamed, moved, or deleted since it +** was first opened. +** +**
  • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This +** opcode causes the xFileControl method to swap the file handle with the one +** pointed to by the pArg argument. This capability is used during testing +** and only needs to be supported when SQLITE_TEST is defined. +** +**
+*/ +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_GET_LOCKPROXYFILE 2 +#define SQLITE_SET_LOCKPROXYFILE 3 +#define SQLITE_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 +#define SQLITE_FCNTL_PERSIST_WAL 10 +#define SQLITE_FCNTL_OVERWRITE 11 +#define SQLITE_FCNTL_VFSNAME 12 +#define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_TRACE 19 +#define SQLITE_FCNTL_HAS_MOVED 20 +#define SQLITE_FCNTL_SYNC 21 +#define SQLITE_FCNTL_COMMIT_PHASETWO 22 +#define SQLITE_FCNTL_WIN32_SET_HANDLE 23 + +/* +** CAPI3REF: Mutex Handle +** +** The mutex module within SQLite defines [sqlite3_mutex] to be an +** abstract type for a mutex object. The SQLite core never looks +** at the internal representation of an [sqlite3_mutex]. It only +** deals with pointers to the [sqlite3_mutex] object. +** +** Mutexes are created using [sqlite3_mutex_alloc()]. +*/ +typedef struct sqlite3_mutex sqlite3_mutex; + +/* +** CAPI3REF: OS Interface Object +** +** An instance of the sqlite3_vfs object defines the interface between +** the SQLite core and the underlying operating system. The "vfs" +** in the name of the object stands for "virtual file system". See +** the [VFS | VFS documentation] for further information. +** +** The value of the iVersion field is initially 1 but may be larger in +** future versions of SQLite. Additional fields may be appended to this +** object when the iVersion value is increased. Note that the structure +** of the sqlite3_vfs object changes in the transaction between +** SQLite version 3.5.9 and 3.6.0 and yet the iVersion field was not +** modified. +** +** The szOsFile field is the size of the subclassed [sqlite3_file] +** structure used by this VFS. mxPathname is the maximum length of +** a pathname in this VFS. +** +** Registered sqlite3_vfs objects are kept on a linked list formed by +** the pNext pointer. The [sqlite3_vfs_register()] +** and [sqlite3_vfs_unregister()] interfaces manage this list +** in a thread-safe way. The [sqlite3_vfs_find()] interface +** searches the list. Neither the application code nor the VFS +** implementation should use the pNext pointer. +** +** The pNext field is the only field in the sqlite3_vfs +** structure that SQLite will ever modify. SQLite will only access +** or modify this field while holding a particular static mutex. +** The application should never modify anything within the sqlite3_vfs +** object once the object has been registered. +** +** The zName field holds the name of the VFS module. The name must +** be unique across all VFS modules. +** +** [[sqlite3_vfs.xOpen]] +** ^SQLite guarantees that the zFilename parameter to xOpen +** is either a NULL pointer or string obtained +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 11 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that +** the string will be valid and unchanged until xClose() is +** called. Because of the previous sentence, +** the [sqlite3_file] can safely store a pointer to the +** filename if it needs to remember the filename for some reason. +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the +** xFilename parameter is NULL it will also be the case that the +** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. +** +** The flags argument to xOpen() includes all bits set in +** the flags argument to [sqlite3_open_v2()]. Or if [sqlite3_open()] +** or [sqlite3_open16()] is used, then flags includes at least +** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. +** If xOpen() opens a file read-only then it sets *pOutFlags to +** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. +** +** ^(SQLite will also add one of the following flags to the xOpen() +** call, depending on the object being opened: +** +**
    +**
  • [SQLITE_OPEN_MAIN_DB] +**
  • [SQLITE_OPEN_MAIN_JOURNAL] +**
  • [SQLITE_OPEN_TEMP_DB] +**
  • [SQLITE_OPEN_TEMP_JOURNAL] +**
  • [SQLITE_OPEN_TRANSIENT_DB] +**
  • [SQLITE_OPEN_SUBJOURNAL] +**
  • [SQLITE_OPEN_MASTER_JOURNAL] +**
  • [SQLITE_OPEN_WAL] +**
)^ +** +** The file I/O implementation can use the object type flags to +** change the way it deals with files. For example, an application +** that does not care about crash recovery or rollback might make +** the open of a journal file a no-op. Writes to this journal would +** also be no-ops, and any attempt to read the journal would return +** SQLITE_IOERR. Or the implementation might recognize that a database +** file will be doing page-aligned sector reads and writes in a random +** order and set up its I/O subsystem accordingly. +** +** SQLite might also add one of the following flags to the xOpen method: +** +**
    +**
  • [SQLITE_OPEN_DELETEONCLOSE] +**
  • [SQLITE_OPEN_EXCLUSIVE] +**
+** +** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. +** +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** with the [SQLITE_OPEN_CREATE] flag, which are both directly +** analogous to the O_EXCL and O_CREAT flags of the POSIX open() +** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the +** SQLITE_OPEN_CREATE, is used to indicate that file should always +** be created, and that it is an error if it already exists. +** It is not used to indicate the file should be opened +** for exclusive access. +** +** ^At least szOsFile bytes of memory are allocated by SQLite +** to hold the [sqlite3_file] structure passed as the third +** argument to xOpen. The xOpen method does not have to +** allocate the structure; it should just fill it in. Note that +** the xOpen method must set the sqlite3_file.pMethods to either +** a valid [sqlite3_io_methods] object or to NULL. xOpen must do +** this even if the open fails. SQLite expects that the sqlite3_file.pMethods +** element will be valid after xOpen returns regardless of the success +** or failure of the xOpen call. +** +** [[sqlite3_vfs.xAccess]] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to +** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] +** to test whether a file is at least readable. The file can be a +** directory. +** +** ^SQLite will always allocate at least mxPathname+1 bytes for the +** output buffer xFullPathname. The exact size of the output buffer +** is also passed as a parameter to both methods. If the output buffer +** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is +** handled as a fatal error by SQLite, vfs implementations should endeavor +** to prevent this by setting mxPathname to a sufficiently large value. +** +** The xRandomness(), xSleep(), xCurrentTime(), and xCurrentTimeInt64() +** interfaces are not strictly a part of the filesystem, but they are +** included in the VFS structure for completeness. +** The xRandomness() function attempts to return nBytes bytes +** of good-quality randomness into zOut. The return value is +** the actual number of bytes of randomness obtained. +** The xSleep() method causes the calling thread to sleep for at +** least the number of microseconds given. ^The xCurrentTime() +** method returns a Julian Day Number for the current date and time as +** a floating point value. +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian +** Day Number multiplied by 86400000 (the number of milliseconds in +** a 24-hour day). +** ^SQLite will use the xCurrentTimeInt64() method to get the current +** date and time if that method is available (if iVersion is 2 or +** greater and the function pointer is not NULL) and will fall back +** to xCurrentTime() if xCurrentTimeInt64() is unavailable. +** +** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** are not used by the SQLite core. These optional interfaces are provided +** by some VFSes to facilitate testing of the VFS code. By overriding +** system calls with functions under its control, a test program can +** simulate faults and error conditions that would otherwise be difficult +** or impossible to induce. The set of system calls that can be overridden +** varies from one VFS to another, and from one version of the same VFS to the +** next. Applications that use these interfaces must be prepared for any +** or all of these interfaces to be NULL or for their behavior to change +** from one release to the next. Applications must not attempt to access +** any of these methods if the iVersion of the VFS is less than 3. +*/ +typedef struct sqlite3_vfs sqlite3_vfs; +typedef void (*sqlite3_syscall_ptr)(void); +struct sqlite3_vfs { + int iVersion; /* Structure version number (currently 3) */ + int szOsFile; /* Size of subclassed sqlite3_file */ + int mxPathname; /* Maximum file pathname length */ + sqlite3_vfs *pNext; /* Next registered VFS */ + const char *zName; /* Name of this virtual file system */ + void *pAppData; /* Pointer to application-specific data */ + int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int flags, int *pOutFlags); + int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); + int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); + int (*xFullPathname)(sqlite3_vfs*, const char *zName, int nOut, char *zOut); + void *(*xDlOpen)(sqlite3_vfs*, const char *zFilename); + void (*xDlError)(sqlite3_vfs*, int nByte, char *zErrMsg); + void (*(*xDlSym)(sqlite3_vfs*,void*, const char *zSymbol))(void); + void (*xDlClose)(sqlite3_vfs*, void*); + int (*xRandomness)(sqlite3_vfs*, int nByte, char *zOut); + int (*xSleep)(sqlite3_vfs*, int microseconds); + int (*xCurrentTime)(sqlite3_vfs*, double*); + int (*xGetLastError)(sqlite3_vfs*, int, char *); + /* + ** The methods above are in version 1 of the sqlite_vfs object + ** definition. Those that follow are added in version 2 or later + */ + int (*xCurrentTimeInt64)(sqlite3_vfs*, sqlite3_int64*); + /* + ** The methods above are in versions 1 and 2 of the sqlite_vfs object. + ** Those below are for version 3 and greater. + */ + int (*xSetSystemCall)(sqlite3_vfs*, const char *zName, sqlite3_syscall_ptr); + sqlite3_syscall_ptr (*xGetSystemCall)(sqlite3_vfs*, const char *zName); + const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName); + /* + ** The methods above are in versions 1 through 3 of the sqlite_vfs object. + ** New fields may be appended in figure versions. The iVersion + ** value will increment whenever this happens. + */ +}; + +/* +** CAPI3REF: Flags for the xAccess VFS method +** +** These integer constants can be used as the third parameter to +** the xAccess method of an [sqlite3_vfs] object. They determine +** what kind of permissions the xAccess method is looking for. +** With SQLITE_ACCESS_EXISTS, the xAccess method +** simply checks whether the file exists. +** With SQLITE_ACCESS_READWRITE, the xAccess method +** checks whether the named directory is both readable and writable +** (in other words, if files can be added, removed, and renamed within +** the directory). +** The SQLITE_ACCESS_READWRITE constant is currently used only by the +** [temp_store_directory pragma], though this could change in a future +** release of SQLite. +** With SQLITE_ACCESS_READ, the xAccess method +** checks whether the file is readable. The SQLITE_ACCESS_READ constant is +** currently unused, though it might be used in a future release of +** SQLite. +*/ +#define SQLITE_ACCESS_EXISTS 0 +#define SQLITE_ACCESS_READWRITE 1 /* Used by PRAGMA temp_store_directory */ +#define SQLITE_ACCESS_READ 2 /* Unused */ + +/* +** CAPI3REF: Flags for the xShmLock VFS method +** +** These integer constants define the various locking operations +** allowed by the xShmLock method of [sqlite3_io_methods]. The +** following are the only legal combinations of flags to the +** xShmLock method: +** +**
    +**
  • SQLITE_SHM_LOCK | SQLITE_SHM_SHARED +**
  • SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE +**
  • SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED +**
  • SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE +**
+** +** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as +** was given no the corresponding lock. +** +** The xShmLock method can transition between unlocked and SHARED or +** between unlocked and EXCLUSIVE. It cannot transition between SHARED +** and EXCLUSIVE. +*/ +#define SQLITE_SHM_UNLOCK 1 +#define SQLITE_SHM_LOCK 2 +#define SQLITE_SHM_SHARED 4 +#define SQLITE_SHM_EXCLUSIVE 8 + +/* +** CAPI3REF: Maximum xShmLock index +** +** The xShmLock method on [sqlite3_io_methods] may use values +** between 0 and this upper bound as its "offset" argument. +** The SQLite core will never attempt to acquire or release a +** lock outside of this range +*/ +#define SQLITE_SHM_NLOCK 8 + + +/* +** CAPI3REF: Initialize The SQLite Library +** +** ^The sqlite3_initialize() routine initializes the +** SQLite library. ^The sqlite3_shutdown() routine +** deallocates any resources that were allocated by sqlite3_initialize(). +** These routines are designed to aid in process initialization and +** shutdown on embedded systems. Workstation applications using +** SQLite normally do not need to invoke either of these routines. +** +** A call to sqlite3_initialize() is an "effective" call if it is +** the first time sqlite3_initialize() is invoked during the lifetime of +** the process, or if it is the first time sqlite3_initialize() is invoked +** following a call to sqlite3_shutdown(). ^(Only an effective call +** of sqlite3_initialize() does any initialization. All other calls +** are harmless no-ops.)^ +** +** A call to sqlite3_shutdown() is an "effective" call if it is the first +** call to sqlite3_shutdown() since the last sqlite3_initialize(). ^(Only +** an effective call to sqlite3_shutdown() does any deinitialization. +** All other valid calls to sqlite3_shutdown() are harmless no-ops.)^ +** +** The sqlite3_initialize() interface is threadsafe, but sqlite3_shutdown() +** is not. The sqlite3_shutdown() interface must only be called from a +** single thread. All open [database connections] must be closed and all +** other SQLite resources must be deallocated prior to invoking +** sqlite3_shutdown(). +** +** Among other things, ^sqlite3_initialize() will invoke +** sqlite3_os_init(). Similarly, ^sqlite3_shutdown() +** will invoke sqlite3_os_end(). +** +** ^The sqlite3_initialize() routine returns [SQLITE_OK] on success. +** ^If for some reason, sqlite3_initialize() is unable to initialize +** the library (perhaps it is unable to allocate a needed resource such +** as a mutex) it returns an [error code] other than [SQLITE_OK]. +** +** ^The sqlite3_initialize() routine is called internally by many other +** SQLite interfaces so that an application usually does not need to +** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] +** calls sqlite3_initialize() so the SQLite library will be automatically +** initialized when [sqlite3_open()] is called if it has not be initialized +** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] +** compile-time option, then the automatic calls to sqlite3_initialize() +** are omitted and the application must call sqlite3_initialize() directly +** prior to using any other SQLite interface. For maximum portability, +** it is recommended that applications always invoke sqlite3_initialize() +** directly prior to using any other SQLite interface. Future releases +** of SQLite may require this. In other words, the behavior exhibited +** when SQLite is compiled with [SQLITE_OMIT_AUTOINIT] might become the +** default behavior in some future release of SQLite. +** +** The sqlite3_os_init() routine does operating-system specific +** initialization of the SQLite library. The sqlite3_os_end() +** routine undoes the effect of sqlite3_os_init(). Typical tasks +** performed by these routines include allocation or deallocation +** of static resources, initialization of global variables, +** setting up a default [sqlite3_vfs] module, or setting up +** a default configuration using [sqlite3_config()]. +** +** The application should never invoke either sqlite3_os_init() +** or sqlite3_os_end() directly. The application should only invoke +** sqlite3_initialize() and sqlite3_shutdown(). The sqlite3_os_init() +** interface is called automatically by sqlite3_initialize() and +** sqlite3_os_end() is called by sqlite3_shutdown(). Appropriate +** implementations for sqlite3_os_init() and sqlite3_os_end() +** are built into SQLite when it is compiled for Unix, Windows, or OS/2. +** When [custom builds | built for other platforms] +** (using the [SQLITE_OS_OTHER=1] compile-time +** option) the application must supply a suitable implementation for +** sqlite3_os_init() and sqlite3_os_end(). An application-supplied +** implementation of sqlite3_os_init() or sqlite3_os_end() +** must return [SQLITE_OK] on success and some other [error code] upon +** failure. +*/ +SQLITE_API int sqlite3_initialize(void); +SQLITE_API int sqlite3_shutdown(void); +SQLITE_API int sqlite3_os_init(void); +SQLITE_API int sqlite3_os_end(void); + +/* +** CAPI3REF: Configuring The SQLite Library +** +** The sqlite3_config() interface is used to make global configuration +** changes to SQLite in order to tune SQLite to the specific needs of +** the application. The default configuration is recommended for most +** applications and so this routine is usually not necessary. It is +** provided to support rare applications with unusual needs. +** +** The sqlite3_config() interface is not threadsafe. The application +** must insure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running. Furthermore, sqlite3_config() +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** +** The first argument to sqlite3_config() is an integer +** [configuration option] that determines +** what property of SQLite is to be configured. Subsequent arguments +** vary depending on the [configuration option] +** in the first argument. +** +** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. +** ^If the option is unknown or SQLite is unable to set the option +** then this routine returns a non-zero [error code]. +*/ +SQLITE_API int sqlite3_config(int, ...); + +/* +** CAPI3REF: Configure database connections +** +** The sqlite3_db_config() interface is used to make configuration +** changes to a [database connection]. The interface is similar to +** [sqlite3_config()] except that the changes apply to a single +** [database connection] (specified in the first argument). +** +** The second argument to sqlite3_db_config(D,V,...) is the +** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code +** that indicates what aspect of the [database connection] is being configured. +** Subsequent arguments vary depending on the configuration verb. +** +** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if +** the call is considered successful. +*/ +SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Memory Allocation Routines +** +** An instance of this object defines the interface between SQLite +** and low-level memory allocation routines. +** +** This object is used in only one place in the SQLite interface. +** A pointer to an instance of this object is the argument to +** [sqlite3_config()] when the configuration option is +** [SQLITE_CONFIG_MALLOC] or [SQLITE_CONFIG_GETMALLOC]. +** By creating an instance of this object +** and passing it to [sqlite3_config]([SQLITE_CONFIG_MALLOC]) +** during configuration, an application can specify an alternative +** memory allocation subsystem for SQLite to use for all of its +** dynamic memory needs. +** +** Note that SQLite comes with several [built-in memory allocators] +** that are perfectly adequate for the overwhelming majority of applications +** and that this object is only useful to a tiny minority of applications +** with specialized memory allocation requirements. This object is +** also used during testing of SQLite in order to specify an alternative +** memory allocator that simulates memory out-of-memory conditions in +** order to verify that SQLite recovers gracefully from such +** conditions. +** +** The xMalloc, xRealloc, and xFree methods must work like the +** malloc(), realloc() and free() functions from the standard C library. +** ^SQLite guarantees that the second argument to +** xRealloc is always a value returned by a prior call to xRoundup. +** +** xSize should return the allocated size of a memory allocation +** previously obtained from xMalloc or xRealloc. The allocated size +** is always at least as big as the requested size but may be larger. +** +** The xRoundup method returns what would be the allocated size of +** a memory allocation given a particular requested size. Most memory +** allocators round up memory allocations at least to the next multiple +** of 8. Some allocators round up to a larger multiple or to a power of 2. +** Every memory allocation request coming in through [sqlite3_malloc()] +** or [sqlite3_realloc()] first calls xRoundup. If xRoundup returns 0, +** that causes the corresponding memory allocation to fail. +** +** The xInit method initializes the memory allocator. For example, +** it might allocate any require mutexes or initialize internal data +** structures. The xShutdown method is invoked (indirectly) by +** [sqlite3_shutdown()] and should deallocate any resources acquired +** by xInit. The pAppData pointer is used as the only parameter to +** xInit and xShutdown. +** +** SQLite holds the [SQLITE_MUTEX_STATIC_MASTER] mutex when it invokes +** the xInit method, so the xInit method need not be threadsafe. The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. For all other methods, SQLite +** holds the [SQLITE_MUTEX_STATIC_MEM] mutex as long as the +** [SQLITE_CONFIG_MEMSTATUS] configuration option is turned on (which +** it is by default) and so the methods are automatically serialized. +** However, if [SQLITE_CONFIG_MEMSTATUS] is disabled, then the other +** methods must be threadsafe or else make their own arrangements for +** serialization. +** +** SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +*/ +typedef struct sqlite3_mem_methods sqlite3_mem_methods; +struct sqlite3_mem_methods { + void *(*xMalloc)(int); /* Memory allocation function */ + void (*xFree)(void*); /* Free a prior allocation */ + void *(*xRealloc)(void*,int); /* Resize an allocation */ + int (*xSize)(void*); /* Return the size of an allocation */ + int (*xRoundup)(int); /* Round up request size to allocation size */ + int (*xInit)(void*); /* Initialize the memory allocator */ + void (*xShutdown)(void*); /* Deinitialize the memory allocator */ + void *pAppData; /* Argument to xInit() and xShutdown() */ +}; + +/* +** CAPI3REF: Configuration Options +** KEYWORDS: {configuration option} +** +** These constants are the available integer configuration options that +** can be passed as the first argument to the [sqlite3_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_config()] to make sure that +** the call worked. The [sqlite3_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
+** [[SQLITE_CONFIG_SINGLETHREAD]]
SQLITE_CONFIG_SINGLETHREAD
+**
There are no arguments to this option. ^This option sets the +** [threading mode] to Single-thread. In other words, it disables +** all mutexing and puts SQLite into a mode where it can only be used +** by a single thread. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to change the [threading mode] from its default +** value of Single-thread and so [sqlite3_config()] will return +** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD +** configuration option.
+** +** [[SQLITE_CONFIG_MULTITHREAD]]
SQLITE_CONFIG_MULTITHREAD
+**
There are no arguments to this option. ^This option sets the +** [threading mode] to Multi-thread. In other words, it disables +** mutexing on [database connection] and [prepared statement] objects. +** The application is responsible for serializing access to +** [database connections] and [prepared statements]. But other mutexes +** are enabled so that SQLite will be safe to use in a multi-threaded +** environment as long as no two threads attempt to use the same +** [database connection] at the same time. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Multi-thread [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_MULTITHREAD configuration option.
+** +** [[SQLITE_CONFIG_SERIALIZED]]
SQLITE_CONFIG_SERIALIZED
+**
There are no arguments to this option. ^This option sets the +** [threading mode] to Serialized. In other words, this option enables +** all mutexes including the recursive +** mutexes on [database connection] and [prepared statement] objects. +** In this mode (which is the default when SQLite is compiled with +** [SQLITE_THREADSAFE=1]) the SQLite library will itself serialize access +** to [database connections] and [prepared statements] so that the +** application is free to use the same [database connection] or the +** same [prepared statement] in different threads at the same time. +** ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Serialized [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_SERIALIZED configuration option.
+** +** [[SQLITE_CONFIG_MALLOC]]
SQLITE_CONFIG_MALLOC
+**
^(This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mem_methods] structure. The argument specifies +** alternative low-level memory allocation routines to be used in place of +** the memory allocation routines built into SQLite.)^ ^SQLite makes +** its own private copy of the content of the [sqlite3_mem_methods] structure +** before the [sqlite3_config()] call returns.
+** +** [[SQLITE_CONFIG_GETMALLOC]]
SQLITE_CONFIG_GETMALLOC
+**
^(This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] +** structure is filled with the currently defined memory allocation routines.)^ +** This option can be used to overload the default memory allocation +** routines with a wrapper that simulations memory allocation failure or +** tracks memory usage, for example.
+** +** [[SQLITE_CONFIG_MEMSTATUS]]
SQLITE_CONFIG_MEMSTATUS
+**
^This option takes single argument of type int, interpreted as a +** boolean, which enables or disables the collection of memory allocation +** statistics. ^(When memory allocation statistics are disabled, the +** following SQLite interfaces become non-operational: +**
    +**
  • [sqlite3_memory_used()] +**
  • [sqlite3_memory_highwater()] +**
  • [sqlite3_soft_heap_limit64()] +**
  • [sqlite3_status()] +**
)^ +** ^Memory allocation statistics are enabled by default unless SQLite is +** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory +** allocation statistics are disabled by default. +**
+** +** [[SQLITE_CONFIG_SCRATCH]]
SQLITE_CONFIG_SCRATCH
+**
^This option specifies a static memory buffer that SQLite can use for +** scratch memory. There are three arguments: A pointer an 8-byte +** aligned memory buffer from which the scratch allocations will be +** drawn, the size of each scratch allocation (sz), +** and the maximum number of scratch allocations (N). The sz +** argument must be a multiple of 16. +** The first argument must be a pointer to an 8-byte aligned buffer +** of at least sz*N bytes of memory. +** ^SQLite will use no more than two scratch buffers per thread. So +** N should be set to twice the expected maximum number of threads. +** ^SQLite will never require a scratch buffer that is more than 6 +** times the database page size. ^If SQLite needs needs additional +** scratch memory beyond what is provided by this configuration option, then +** [sqlite3_malloc()] will be used to obtain the memory needed.
+** +** [[SQLITE_CONFIG_PAGECACHE]]
SQLITE_CONFIG_PAGECACHE
+**
^This option specifies a static memory buffer that SQLite can use for +** the database page cache with the default page cache implementation. +** This configuration should not be used if an application-define page +** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. +** There are three arguments to this option: A pointer to 8-byte aligned +** memory, the size of each page buffer (sz), and the number of pages (N). +** The sz argument should be the size of the largest database page +** (a power of two between 512 and 32768) plus a little extra for each +** page header. ^The page header size is 20 to 40 bytes depending on +** the host architecture. ^It is harmless, apart from the wasted memory, +** to make sz a little too large. The first +** argument should point to an allocation of at least sz*N bytes of memory. +** ^SQLite will use the memory provided by the first argument to satisfy its +** memory needs for the first N pages that it adds to cache. ^If additional +** page cache memory is needed beyond what is provided by this option, then +** SQLite goes to [sqlite3_malloc()] for the additional storage space. +** The pointer in the first argument must +** be aligned to an 8-byte boundary or subsequent behavior of SQLite +** will be undefined.
+** +** [[SQLITE_CONFIG_HEAP]]
SQLITE_CONFIG_HEAP
+**
^This option specifies a static memory buffer that SQLite will use +** for all of its dynamic memory allocation needs beyond those provided +** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** There are three arguments: An 8-byte aligned pointer to the memory, +** the number of bytes in the memory buffer, and the minimum allocation size. +** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts +** to using its default memory allocator (the system malloc() implementation), +** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the +** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or +** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory +** allocator is engaged to handle all of SQLites memory allocation needs. +** The first pointer (the memory pointer) must be aligned to an 8-byte +** boundary or subsequent behavior of SQLite will be undefined. +** The minimum allocation size is capped at 2**12. Reasonable values +** for the minimum allocation size are 2**5 through 2**8.
+** +** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
+**
^(This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mutex_methods] structure. The argument specifies +** alternative low-level mutex routines to be used in place +** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the +** content of the [sqlite3_mutex_methods] structure before the call to +** [sqlite3_config()] returns. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will +** return [SQLITE_ERROR].
+** +** [[SQLITE_CONFIG_GETMUTEX]]
SQLITE_CONFIG_GETMUTEX
+**
^(This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mutex_methods] structure. The +** [sqlite3_mutex_methods] +** structure is filled with the currently defined mutex routines.)^ +** This option can be used to overload the default mutex allocation +** routines with a wrapper used to track mutex usage for performance +** profiling or testing, for example. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will +** return [SQLITE_ERROR].
+** +** [[SQLITE_CONFIG_LOOKASIDE]]
SQLITE_CONFIG_LOOKASIDE
+**
^(This option takes two arguments that determine the default +** memory allocation for the lookaside memory allocator on each +** [database connection]. The first argument is the +** size of each lookaside buffer slot and the second is the number of +** slots allocated to each database connection.)^ ^(This option sets the +** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** verb to [sqlite3_db_config()] can be used to change the lookaside +** configuration on individual connections.)^
+** +** [[SQLITE_CONFIG_PCACHE2]]
SQLITE_CONFIG_PCACHE2
+**
^(This option takes a single argument which is a pointer to +** an [sqlite3_pcache_methods2] object. This object specifies the interface +** to a custom page cache implementation.)^ ^SQLite makes a copy of the +** object and uses it for page cache memory allocations.
+** +** [[SQLITE_CONFIG_GETPCACHE2]]
SQLITE_CONFIG_GETPCACHE2
+**
^(This option takes a single argument which is a pointer to an +** [sqlite3_pcache_methods2] object. SQLite copies of the current +** page cache implementation into that object.)^
+** +** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
+**
The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +** function with a call signature of void(*)(void*,int,const char*), +** and a pointer to void. ^If the function pointer is not NULL, it is +** invoked by [sqlite3_log()] to process each logging event. ^If the +** function pointer is NULL, the [sqlite3_log()] interface becomes a no-op. +** ^The void pointer that is the second argument to SQLITE_CONFIG_LOG is +** passed through as the first parameter to the application-defined logger +** function whenever that function is invoked. ^The second parameter to +** the logger function is a copy of the first parameter to the corresponding +** [sqlite3_log()] call and is intended to be a [result code] or an +** [extended result code]. ^The third parameter passed to the logger is +** log message after formatting via [sqlite3_snprintf()]. +** The SQLite logging interface is not reentrant; the logger function +** supplied by the application must not invoke any SQLite interface. +** In a multi-threaded application, the application-defined logger +** function must be threadsafe.
+** +** [[SQLITE_CONFIG_URI]]
SQLITE_CONFIG_URI +**
^(This option takes a single argument of type int. If non-zero, then +** URI handling is globally enabled. If the parameter is zero, then URI handling +** is globally disabled.)^ ^If URI handling is globally enabled, all filenames +** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +** specified as part of [ATTACH] commands are interpreted as URIs, regardless +** of whether or not the [SQLITE_OPEN_URI] flag is set when the database +** connection is opened. ^If it is globally disabled, filenames are +** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the +** database connection is opened. ^(By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** [SQLITE_USE_URI] symbol defined.)^ +** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
SQLITE_CONFIG_COVERING_INDEX_SCAN +**
^This option takes a single integer argument which is interpreted as +** a boolean in order to enable or disable the use of covering indices for +** full table scans in the query optimizer. ^The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** +** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] +**
SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE +**
These options are obsolete and should not be used by new code. +** They are retained for backwards compatibility but are now no-ops. +**
+** +** [[SQLITE_CONFIG_SQLLOG]] +**
SQLITE_CONFIG_SQLLOG +**
This option is only available if sqlite is compiled with the +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
+** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
SQLITE_CONFIG_MMAP_SIZE +**
^SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** ^The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ +** ^If either argument to this option is negative, then that argument is +** changed to its compile-time default. +** +** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] +**
SQLITE_CONFIG_WIN32_HEAPSIZE +**
^This option is only available if SQLite is compiled for Windows +** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. +** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** that specifies the maximum size of the created heap. +**
+*/ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* void*, int sz, int N */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ + +/* +** CAPI3REF: Database Connection Configuration Options +** +** These constants are the available integer configuration options that +** can be passed as the second argument to the [sqlite3_db_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_db_config()] to make sure that +** the call worked. ^The [sqlite3_db_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
+**
SQLITE_DBCONFIG_LOOKASIDE
+**
^This option takes three additional arguments that determine the +** [lookaside memory allocator] configuration for the [database connection]. +** ^The first argument (the third parameter to [sqlite3_db_config()] is a +** pointer to a memory buffer to use for lookaside memory. +** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb +** may be NULL in which case SQLite will allocate the +** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the +** size of each lookaside buffer slot. ^The third argument is the number of +** slots. The size of the buffer in the first argument must be greater than +** or equal to the product of the second and third arguments. The buffer +** must be aligned to an 8-byte boundary. ^If the second argument to +** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^
+** +**
SQLITE_DBCONFIG_ENABLE_FKEY
+**
^This option is used to enable or disable the enforcement of +** [foreign key constraints]. There should be two additional arguments. +** The first argument is an integer which is 0 to disable FK enforcement, +** positive to enable FK enforcement or negative to leave FK enforcement +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether FK enforcement is off or on +** following this call. The second parameter may be a NULL pointer, in +** which case the FK enforcement setting is not reported back.
+** +**
SQLITE_DBCONFIG_ENABLE_TRIGGER
+**
^This option is used to enable or disable [CREATE TRIGGER | triggers]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable triggers, +** positive to enable triggers or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether triggers are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the trigger setting is not reported back.
+** +**
+*/ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ + + +/* +** CAPI3REF: Enable Or Disable Extended Result Codes +** +** ^The sqlite3_extended_result_codes() routine enables or disables the +** [extended result codes] feature of SQLite. ^The extended result +** codes are disabled by default for historical compatibility. +*/ +SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); + +/* +** CAPI3REF: Last Insert Rowid +** +** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables) +** has a unique 64-bit signed +** integer key called the [ROWID | "rowid"]. ^The rowid is always available +** as an undeclared column named ROWID, OID, or _ROWID_ as long as those +** names are not also used by explicitly declared columns. ^If +** the table has a column of type [INTEGER PRIMARY KEY] then that column +** is another alias for the rowid. +** +** ^The sqlite3_last_insert_rowid(D) interface returns the [rowid] of the +** most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. +** ^Inserts into [WITHOUT ROWID] tables are not recorded. +** ^If no successful [INSERT]s into rowid tables +** have ever occurred on the database connection D, +** then sqlite3_last_insert_rowid(D) returns zero. +** +** ^(If an [INSERT] occurs within a trigger or within a [virtual table] +** method, then this routine will return the [rowid] of the inserted +** row as long as the trigger or virtual table method is running. +** But once the trigger or virtual table method ends, the value returned +** by this routine reverts to what it was before the trigger or virtual +** table method began.)^ +** +** ^An [INSERT] that fails due to a constraint violation is not a +** successful [INSERT] and does not change the value returned by this +** routine. ^Thus INSERT OR FAIL, INSERT OR IGNORE, INSERT OR ROLLBACK, +** and INSERT OR ABORT make no changes to the return value of this +** routine when their insertion fails. ^(When INSERT OR REPLACE +** encounters a constraint violation, it does not fail. The +** INSERT continues to completion after deleting rows that caused +** the constraint problem so INSERT OR REPLACE will always change +** the return value of this interface.)^ +** +** ^For the purposes of this routine, an [INSERT] is considered to +** be successful even if it is subsequently rolled back. +** +** This function is accessible to SQL statements via the +** [last_insert_rowid() SQL function]. +** +** If a separate thread performs a new [INSERT] on the same +** database connection while the [sqlite3_last_insert_rowid()] +** function is running and thus changes the last insert [rowid], +** then the value returned by [sqlite3_last_insert_rowid()] is +** unpredictable and might not equal either the old or the new +** last insert [rowid]. +*/ +SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); + +/* +** CAPI3REF: Count The Number Of Rows Modified +** +** ^This function returns the number of database rows that were changed +** or inserted or deleted by the most recently completed SQL statement +** on the [database connection] specified by the first parameter. +** ^(Only changes that are directly specified by the [INSERT], [UPDATE], +** or [DELETE] statement are counted. Auxiliary changes caused by +** triggers or [foreign key actions] are not counted.)^ Use the +** [sqlite3_total_changes()] function to find the total number of changes +** including changes caused by triggers and foreign key actions. +** +** ^Changes to a view that are simulated by an [INSTEAD OF trigger] +** are not counted. Only real table changes are counted. +** +** ^(A "row change" is a change to a single row of a single table +** caused by an INSERT, DELETE, or UPDATE statement. Rows that +** are changed as side effects of [REPLACE] constraint resolution, +** rollback, ABORT processing, [DROP TABLE], or by any other +** mechanisms do not count as direct row changes.)^ +** +** A "trigger context" is a scope of execution that begins and +** ends with the script of a [CREATE TRIGGER | trigger]. +** Most SQL statements are +** evaluated outside of any trigger. This is the "top level" +** trigger context. If a trigger fires from the top level, a +** new trigger context is entered for the duration of that one +** trigger. Subtriggers create subcontexts for their duration. +** +** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does +** not create a new trigger context. +** +** ^This function returns the number of direct row changes in the +** most recent INSERT, UPDATE, or DELETE statement within the same +** trigger context. +** +** ^Thus, when called from the top level, this function returns the +** number of changes in the most recent INSERT, UPDATE, or DELETE +** that also occurred at the top level. ^(Within the body of a trigger, +** the sqlite3_changes() interface can be called to find the number of +** changes in the most recently completed INSERT, UPDATE, or DELETE +** statement within the body of the same trigger. +** However, the number returned does not include changes +** caused by subtriggers since those have their own context.)^ +** +** See also the [sqlite3_total_changes()] interface, the +** [count_changes pragma], and the [changes() SQL function]. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_changes()] is running then the value returned +** is unpredictable and not meaningful. +*/ +SQLITE_API int sqlite3_changes(sqlite3*); + +/* +** CAPI3REF: Total Number Of Rows Modified +** +** ^This function returns the number of row changes caused by [INSERT], +** [UPDATE] or [DELETE] statements since the [database connection] was opened. +** ^(The count returned by sqlite3_total_changes() includes all changes +** from all [CREATE TRIGGER | trigger] contexts and changes made by +** [foreign key actions]. However, +** the count does not include changes used to implement [REPLACE] constraints, +** do rollbacks or ABORT processing, or [DROP TABLE] processing. The +** count does not include rows of views that fire an [INSTEAD OF trigger], +** though if the INSTEAD OF trigger makes changes of its own, those changes +** are counted.)^ +** ^The sqlite3_total_changes() function counts the changes as soon as +** the statement that makes them is completed (when the statement handle +** is passed to [sqlite3_reset()] or [sqlite3_finalize()]). +** +** See also the [sqlite3_changes()] interface, the +** [count_changes pragma], and the [total_changes() SQL function]. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_total_changes()] is running then the value +** returned is unpredictable and not meaningful. +*/ +SQLITE_API int sqlite3_total_changes(sqlite3*); + +/* +** CAPI3REF: Interrupt A Long-Running Query +** +** ^This function causes any pending database operation to abort and +** return at its earliest opportunity. This routine is typically +** called in response to a user action such as pressing "Cancel" +** or Ctrl-C where the user wants a long query operation to halt +** immediately. +** +** ^It is safe to call this routine from a thread different from the +** thread that is currently running the database operation. But it +** is not safe to call this routine with a [database connection] that +** is closed or might close before sqlite3_interrupt() returns. +** +** ^If an SQL operation is very nearly finished at the time when +** sqlite3_interrupt() is called, then it might not have an opportunity +** to be interrupted and might continue to completion. +** +** ^An SQL operation that is interrupted will return [SQLITE_INTERRUPT]. +** ^If the interrupted SQL operation is an INSERT, UPDATE, or DELETE +** that is inside an explicit transaction, then the entire transaction +** will be rolled back automatically. +** +** ^The sqlite3_interrupt(D) call is in effect until all currently running +** SQL statements on [database connection] D complete. ^Any new SQL statements +** that are started after the sqlite3_interrupt() call and before the +** running statements reaches zero are interrupted as if they had been +** running prior to the sqlite3_interrupt() call. ^New SQL statements +** that are started after the running statement count reaches zero are +** not effected by the sqlite3_interrupt(). +** ^A call to sqlite3_interrupt(D) that occurs when there are no running +** SQL statements is a no-op and has no effect on SQL statements +** that are started after the sqlite3_interrupt() call returns. +** +** If the database connection closes while [sqlite3_interrupt()] +** is running then bad things will likely happen. +*/ +SQLITE_API void sqlite3_interrupt(sqlite3*); + +/* +** CAPI3REF: Determine If An SQL Statement Is Complete +** +** These routines are useful during command-line input to determine if the +** currently entered text seems to form a complete SQL statement or +** if additional input is needed before sending the text into +** SQLite for parsing. ^These routines return 1 if the input string +** appears to be a complete SQL statement. ^A statement is judged to be +** complete if it ends with a semicolon token and is not a prefix of a +** well-formed CREATE TRIGGER statement. ^Semicolons that are embedded within +** string literals or quoted identifier names or comments are not +** independent tokens (they are part of the token in which they are +** embedded) and thus do not count as a statement terminator. ^Whitespace +** and comments that follow the final semicolon are ignored. +** +** ^These routines return 0 if the statement is incomplete. ^If a +** memory allocation fails, then SQLITE_NOMEM is returned. +** +** ^These routines do not parse the SQL statements thus +** will not detect syntactically incorrect SQL. +** +** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior +** to invoking sqlite3_complete16() then sqlite3_initialize() is invoked +** automatically by sqlite3_complete16(). If that initialization fails, +** then the return value from sqlite3_complete16() will be non-zero +** regardless of whether or not the input SQL is complete.)^ +** +** The input to [sqlite3_complete()] must be a zero-terminated +** UTF-8 string. +** +** The input to [sqlite3_complete16()] must be a zero-terminated +** UTF-16 string in native byte order. +*/ +SQLITE_API int sqlite3_complete(const char *sql); +SQLITE_API int sqlite3_complete16(const void *sql); + +/* +** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. +** +** ^If the busy callback is NULL, then [SQLITE_BUSY] +** is returned immediately upon encountering the lock. ^If the busy callback +** is not NULL, then the callback might be invoked with two arguments. +** +** ^The first argument to the busy handler is a copy of the void* pointer which +** is the third argument to sqlite3_busy_handler(). ^The second argument to +** the busy handler callback is the number of times that the busy handler has +** been invoked for the same locking event. ^If the +** busy callback returns 0, then no additional attempts are made to +** access the database and [SQLITE_BUSY] is returned +** to the application. +** ^If the callback returns non-zero, then another attempt +** is made to access the database and the cycle repeats. +** +** The presence of a busy handler does not guarantee that it will be invoked +** when there is lock contention. ^If SQLite determines that invoking the busy +** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] +** to the application instead of invoking the +** busy handler. +** Consider a scenario where one process is holding a read lock that +** it is trying to promote to a reserved lock and +** a second process is holding a reserved lock that it is trying +** to promote to an exclusive lock. The first process cannot proceed +** because it is blocked by the second and the second process cannot +** proceed because it is blocked by the first. If both processes +** invoke the busy handlers, neither will make any progress. Therefore, +** SQLite returns [SQLITE_BUSY] for the first process, hoping that this +** will induce the first process to release its read lock and allow +** the second process to proceed. +** +** ^The default busy callback is NULL. +** +** ^(There can only be a single busy handler defined for each +** [database connection]. Setting a new busy handler clears any +** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. +** +** The busy callback should not take any actions which modify the +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions +** result in undefined behavior. +** +** A busy handler must not close the database connection +** or [prepared statement] that invoked the busy handler. +*/ +SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); + +/* +** CAPI3REF: Set A Busy Timeout +** +** ^This routine sets a [sqlite3_busy_handler | busy handler] that sleeps +** for a specified amount of time when a table is locked. ^The handler +** will sleep multiple times until at least "ms" milliseconds of sleeping +** have accumulated. ^After at least "ms" milliseconds of sleeping, +** the handler returns 0 which causes [sqlite3_step()] to return +** [SQLITE_BUSY]. +** +** ^Calling this routine with an argument less than or equal to zero +** turns off all busy handlers. +** +** ^(There can only be a single busy handler for a particular +** [database connection] any any given moment. If another busy handler +** was defined (using [sqlite3_busy_handler()]) prior to calling +** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] +*/ +SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); + +/* +** CAPI3REF: Convenience Routines For Running Queries +** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. +** +** Definition: A result table is memory data structure created by the +** [sqlite3_get_table()] interface. A result table records the +** complete query results from one or more queries. +** +** The table conceptually has a number of rows and columns. But +** these numbers are not part of the result table itself. These +** numbers are obtained separately. Let N be the number of rows +** and M be the number of columns. +** +** A result table is an array of pointers to zero-terminated UTF-8 strings. +** There are (N+1)*M elements in the array. The first M pointers point +** to zero-terminated strings that contain the names of the columns. +** The remaining entries all point to query results. NULL values result +** in NULL pointers. All other values are in their UTF-8 zero-terminated +** string representation as returned by [sqlite3_column_text()]. +** +** A result table might consist of one or more memory allocations. +** It is not safe to pass a result table directly to [sqlite3_free()]. +** A result table should be deallocated using [sqlite3_free_table()]. +** +** ^(As an example of the result table format, suppose a query result +** is as follows: +** +** 
+**        Name        | Age
+**        -----------------------
+**        Alice       | 43
+**        Bob         | 28
+**        Cindy       | 21
+**
+** +** There are two column (M==2) and three rows (N==3). Thus the +** result table has 8 entries. Suppose the result table is stored +** in an array names azResult. Then azResult holds this content: +** +** 
+**        azResult[0] = "Name";
+**        azResult[1] = "Age";
+**        azResult[2] = "Alice";
+**        azResult[3] = "43";
+**        azResult[4] = "Bob";
+**        azResult[5] = "28";
+**        azResult[6] = "Cindy";
+**        azResult[7] = "21";
+**
)^ +** +** ^The sqlite3_get_table() function evaluates one or more +** semicolon-separated SQL statements in the zero-terminated UTF-8 +** string of its 2nd parameter and returns a result table to the +** pointer given in its 3rd parameter. +** +** After the application has finished with the result from sqlite3_get_table(), +** it must pass the result table pointer to sqlite3_free_table() in order to +** release the memory that was malloced. Because of the way the +** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling +** function must not try to call [sqlite3_free()] directly. Only +** [sqlite3_free_table()] is able to release the memory properly and safely. +** +** The sqlite3_get_table() interface is implemented as a wrapper around +** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access +** to any internal data structures of SQLite. It uses only the public +** interface defined here. As a consequence, errors that occur in the +** wrapper layer outside of the internal [sqlite3_exec()] call are not +** reflected in subsequent calls to [sqlite3_errcode()] or +** [sqlite3_errmsg()]. +*/ +SQLITE_API int sqlite3_get_table( + sqlite3 *db, /* An open database */ + const char *zSql, /* SQL to be evaluated */ + char ***pazResult, /* Results of the query */ + int *pnRow, /* Number of result rows written here */ + int *pnColumn, /* Number of result columns written here */ + char **pzErrmsg /* Error msg written here */ +); +SQLITE_API void sqlite3_free_table(char **result); + +/* +** CAPI3REF: Formatted String Printing Functions +** +** These routines are work-alikes of the "printf()" family of functions +** from the standard C library. +** +** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their +** results into memory obtained from [sqlite3_malloc()]. +** The strings returned by these two routines should be +** released by [sqlite3_free()]. ^Both routines return a +** NULL pointer if [sqlite3_malloc()] is unable to allocate enough +** memory to hold the resulting string. +** +** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from +** the standard C library. The result is written into the +** buffer supplied as the second parameter whose size is given by +** the first parameter. Note that the order of the +** first two parameters is reversed from snprintf().)^ This is an +** historical accident that cannot be fixed without breaking +** backwards compatibility. ^(Note also that sqlite3_snprintf() +** returns a pointer to its buffer instead of the number of +** characters actually written into the buffer.)^ We admit that +** the number of characters written would be a more useful return +** value but we cannot change the implementation of sqlite3_snprintf() +** now without breaking compatibility. +** +** ^As long as the buffer size is greater than zero, sqlite3_snprintf() +** guarantees that the buffer is always zero-terminated. ^The first +** parameter "n" is the total size of the buffer, including space for +** the zero terminator. So the longest string that can be completely +** written will be n-1 characters. +** +** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf(). +** +** These routines all implement some additional formatting +** options that are useful for constructing SQL statements. +** All of the usual printf() formatting options apply. In addition, there +** is are "%q", "%Q", and "%z" options. +** +** ^(The %q option works like %s in that it substitutes a nul-terminated +** string from the argument list. But %q also doubles every '\'' character. +** %q is designed for use inside a string literal.)^ By doubling each '\'' +** character it escapes that character and allows it to be inserted into +** the string. +** +** For example, assume the string variable zText contains text as follows: +** +** 
+**  char *zText = "It's a happy day!";
+**
+** +** One can use this text in an SQL statement as follows: +** +** 
+**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES('%q')", zText);
+**  sqlite3_exec(db, zSQL, 0, 0, 0);
+**  sqlite3_free(zSQL);
+**
+** +** Because the %q format string is used, the '\'' character in zText +** is escaped and the SQL generated is as follows: +** +** 
+**  INSERT INTO table1 VALUES('It''s a happy day!')
+**
+** +** This is correct. Had we used %s instead of %q, the generated SQL +** would have looked like this: +** +** 
+**  INSERT INTO table1 VALUES('It's a happy day!');
+**
+** +** This second example is an SQL syntax error. As a general rule you should +** always use %q instead of %s when inserting text into a string literal. +** +** ^(The %Q option works like %q except it also adds single quotes around +** the outside of the total string. Additionally, if the parameter in the +** argument list is a NULL pointer, %Q substitutes the text "NULL" (without +** single quotes).)^ So, for example, one could say: +** +** 
+**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
+**  sqlite3_exec(db, zSQL, 0, 0, 0);
+**  sqlite3_free(zSQL);
+**
+** +** The code above will render a correct SQL statement in the zSQL +** variable even if the zText variable is a NULL pointer. +** +** ^(The "%z" formatting option works like "%s" but with the +** addition that after the string has been read and copied into +** the result, [sqlite3_free()] is called on the input string.)^ +*/ +SQLITE_API char *sqlite3_mprintf(const char*,...); +SQLITE_API char *sqlite3_vmprintf(const char*, va_list); +SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); + +/* +** CAPI3REF: Memory Allocation Subsystem +** +** The SQLite core uses these three routines for all of its own +** internal memory allocation needs. "Core" in the previous sentence +** does not include operating-system specific VFS implementation. The +** Windows VFS uses native malloc() and free() for some operations. +** +** ^The sqlite3_malloc() routine returns a pointer to a block +** of memory at least N bytes in length, where N is the parameter. +** ^If sqlite3_malloc() is unable to obtain sufficient free +** memory, it returns a NULL pointer. ^If the parameter N to +** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns +** a NULL pointer. +** +** ^Calling sqlite3_free() with a pointer previously returned +** by sqlite3_malloc() or sqlite3_realloc() releases that memory so +** that it might be reused. ^The sqlite3_free() routine is +** a no-op if is called with a NULL pointer. Passing a NULL pointer +** to sqlite3_free() is harmless. After being freed, memory +** should neither be read nor written. Even reading previously freed +** memory might result in a segmentation fault or other severe error. +** Memory corruption, a segmentation fault, or other severe error +** might result if sqlite3_free() is called with a non-NULL pointer that +** was not obtained from sqlite3_malloc() or sqlite3_realloc(). +** +** ^(The sqlite3_realloc() interface attempts to resize a +** prior memory allocation to be at least N bytes, where N is the +** second parameter. The memory allocation to be resized is the first +** parameter.)^ ^ If the first parameter to sqlite3_realloc() +** is a NULL pointer then its behavior is identical to calling +** sqlite3_malloc(N) where N is the second parameter to sqlite3_realloc(). +** ^If the second parameter to sqlite3_realloc() is zero or +** negative then the behavior is exactly the same as calling +** sqlite3_free(P) where P is the first parameter to sqlite3_realloc(). +** ^sqlite3_realloc() returns a pointer to a memory allocation +** of at least N bytes in size or NULL if sufficient memory is unavailable. +** ^If M is the size of the prior allocation, then min(N,M) bytes +** of the prior allocation are copied into the beginning of buffer returned +** by sqlite3_realloc() and the prior allocation is freed. +** ^If sqlite3_realloc() returns NULL, then the prior allocation +** is not freed. +** +** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. +** +** In SQLite version 3.5.0 and 3.5.1, it was possible to define +** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in +** implementation of these routines to be omitted. That capability +** is no longer provided. Only built-in memory allocators can be used. +** +** Prior to SQLite version 3.7.10, the Windows OS interface layer called +** the system malloc() and free() directly when converting +** filenames between the UTF-8 encoding used by SQLite +** and whatever filename encoding is used by the particular Windows +** installation. Memory allocation errors were detected, but +** they were reported back as [SQLITE_CANTOPEN] or +** [SQLITE_IOERR] rather than [SQLITE_NOMEM]. +** +** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()] +** must be either NULL or else pointers obtained from a prior +** invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have +** not yet been released. +** +** The application must not read or write any part of +** a block of memory after it has been released using +** [sqlite3_free()] or [sqlite3_realloc()]. +*/ +SQLITE_API void *sqlite3_malloc(int); +SQLITE_API void *sqlite3_realloc(void*, int); +SQLITE_API void sqlite3_free(void*); + +/* +** CAPI3REF: Memory Allocator Statistics +** +** SQLite provides these two interfaces for reporting on the status +** of the [sqlite3_malloc()], [sqlite3_free()], and [sqlite3_realloc()] +** routines, which form the built-in memory allocation subsystem. +** +** ^The [sqlite3_memory_used()] routine returns the number of bytes +** of memory currently outstanding (malloced but not freed). +** ^The [sqlite3_memory_highwater()] routine returns the maximum +** value of [sqlite3_memory_used()] since the high-water mark +** was last reset. ^The values returned by [sqlite3_memory_used()] and +** [sqlite3_memory_highwater()] include any overhead +** added by SQLite in its implementation of [sqlite3_malloc()], +** but not overhead added by the any underlying system library +** routines that [sqlite3_malloc()] may call. +** +** ^The memory high-water mark is reset to the current value of +** [sqlite3_memory_used()] if and only if the parameter to +** [sqlite3_memory_highwater()] is true. ^The value returned +** by [sqlite3_memory_highwater(1)] is the high-water mark +** prior to the reset. +*/ +SQLITE_API sqlite3_int64 sqlite3_memory_used(void); +SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); + +/* +** CAPI3REF: Pseudo-Random Number Generator +** +** SQLite contains a high-quality pseudo-random number generator (PRNG) used to +** select random [ROWID | ROWIDs] when inserting new records into a table that +** already uses the largest possible [ROWID]. The PRNG is also used for +** the build-in random() and randomblob() SQL functions. This interface allows +** applications to access the same PRNG for other purposes. +** +** ^A call to this routine stores N bytes of randomness into buffer P. +** ^If N is less than one, then P can be a NULL pointer. +** +** ^If this routine has not been previously called or if the previous +** call had N less than one, then the PRNG is seeded using randomness +** obtained from the xRandomness method of the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more then +** the pseudo-randomness is generated +** internally and without recourse to the [sqlite3_vfs] xRandomness +** method. +*/ +SQLITE_API void sqlite3_randomness(int N, void *P); + +/* +** CAPI3REF: Compile-Time Authorization Callbacks +** +** ^This routine registers an authorizer callback with a particular +** [database connection], supplied in the first argument. +** ^The authorizer callback is invoked as SQL statements are being compiled +** by [sqlite3_prepare()] or its variants [sqlite3_prepare_v2()], +** [sqlite3_prepare16()] and [sqlite3_prepare16_v2()]. ^At various +** points during the compilation process, as logic is being created +** to perform various actions, the authorizer callback is invoked to +** see if those actions are allowed. ^The authorizer callback should +** return [SQLITE_OK] to allow the action, [SQLITE_IGNORE] to disallow the +** specific action but allow the SQL statement to continue to be +** compiled, or [SQLITE_DENY] to cause the entire SQL statement to be +** rejected with an error. ^If the authorizer callback returns +** any value other than [SQLITE_IGNORE], [SQLITE_OK], or [SQLITE_DENY] +** then the [sqlite3_prepare_v2()] or equivalent call that triggered +** the authorizer will fail with an error message. +** +** When the callback returns [SQLITE_OK], that means the operation +** requested is ok. ^When the callback returns [SQLITE_DENY], the +** [sqlite3_prepare_v2()] or equivalent call that triggered the +** authorizer will fail with an error message explaining that +** access is denied. +** +** ^The first parameter to the authorizer callback is a copy of the third +** parameter to the sqlite3_set_authorizer() interface. ^The second parameter +** to the callback is an integer [SQLITE_COPY | action code] that specifies +** the particular action to be authorized. ^The third through sixth parameters +** to the callback are zero-terminated strings that contain additional +** details about the action to be authorized. +** +** ^If the action code is [SQLITE_READ] +** and the callback returns [SQLITE_IGNORE] then the +** [prepared statement] statement is constructed to substitute +** a NULL value in place of the table column that would have +** been read if [SQLITE_OK] had been returned. The [SQLITE_IGNORE] +** return can be used to deny an untrusted user access to individual +** columns of a table. +** ^If the action code is [SQLITE_DELETE] and the callback returns +** [SQLITE_IGNORE] then the [DELETE] operation proceeds but the +** [truncate optimization] is disabled and all rows are deleted individually. +** +** An authorizer is used when [sqlite3_prepare | preparing] +** SQL statements from an untrusted source, to ensure that the SQL statements +** do not try to access data they are not allowed to see, or that they do not +** try to execute malicious statements that damage the database. For +** example, an application may allow a user to enter arbitrary +** SQL queries for evaluation by a database. But the application does +** not want the user to be able to make arbitrary changes to the +** database. An authorizer could then be put in place while the +** user-entered SQL is being [sqlite3_prepare | prepared] that +** disallows everything except [SELECT] statements. +** +** Applications that need to process SQL from untrusted sources +** might also consider lowering resource limits using [sqlite3_limit()] +** and limiting database size using the [max_page_count] [PRAGMA] +** in addition to using an authorizer. +** +** ^(Only a single authorizer can be in place on a database connection +** at a time. Each call to sqlite3_set_authorizer overrides the +** previous call.)^ ^Disable the authorizer by installing a NULL callback. +** The authorizer is disabled by default. +** +** The authorizer callback must not do anything that will modify +** the database connection that invoked the authorizer callback. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^When [sqlite3_prepare_v2()] is used to prepare a statement, the +** statement might be re-prepared during [sqlite3_step()] due to a +** schema change. Hence, the application should ensure that the +** correct authorizer callback remains in place during the [sqlite3_step()]. +** +** ^Note that the authorizer callback is invoked only during +** [sqlite3_prepare()] or its variants. Authorization is not +** performed during statement evaluation in [sqlite3_step()], unless +** as stated in the previous paragraph, sqlite3_step() invokes +** sqlite3_prepare_v2() to reprepare a statement after a schema change. +*/ +SQLITE_API int sqlite3_set_authorizer( + sqlite3*, + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), + void *pUserData +); + +/* +** CAPI3REF: Authorizer Return Codes +** +** The [sqlite3_set_authorizer | authorizer callback function] must +** return either [SQLITE_OK] or one of these two constants in order +** to signal SQLite whether or not the action is permitted. See the +** [sqlite3_set_authorizer | authorizer documentation] for additional +** information. +** +** Note that SQLITE_IGNORE is also used as a [conflict resolution mode] +** returned from the [sqlite3_vtab_on_conflict()] interface. +*/ +#define SQLITE_DENY 1 /* Abort the SQL statement with an error */ +#define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ + +/* +** CAPI3REF: Authorizer Action Codes +** +** The [sqlite3_set_authorizer()] interface registers a callback function +** that is invoked to authorize certain SQL statement actions. The +** second parameter to the callback is an integer code that specifies +** what action is being authorized. These are the integer action codes that +** the authorizer callback may be passed. +** +** These action code values signify what kind of operation is to be +** authorized. The 3rd and 4th parameters to the authorization +** callback function will be parameters or NULL depending on which of these +** codes is used as the second parameter. ^(The 5th parameter to the +** authorizer callback is the name of the database ("main", "temp", +** etc.) if applicable.)^ ^The 6th parameter to the authorizer callback +** is the name of the inner-most trigger or view that is responsible for +** the access attempt or NULL if this access attempt is directly from +** top-level SQL code. +*/ +/******************************************* 3rd ************ 4th ***********/ +#define SQLITE_CREATE_INDEX 1 /* Index Name Table Name */ +#define SQLITE_CREATE_TABLE 2 /* Table Name NULL */ +#define SQLITE_CREATE_TEMP_INDEX 3 /* Index Name Table Name */ +#define SQLITE_CREATE_TEMP_TABLE 4 /* Table Name NULL */ +#define SQLITE_CREATE_TEMP_TRIGGER 5 /* Trigger Name Table Name */ +#define SQLITE_CREATE_TEMP_VIEW 6 /* View Name NULL */ +#define SQLITE_CREATE_TRIGGER 7 /* Trigger Name Table Name */ +#define SQLITE_CREATE_VIEW 8 /* View Name NULL */ +#define SQLITE_DELETE 9 /* Table Name NULL */ +#define SQLITE_DROP_INDEX 10 /* Index Name Table Name */ +#define SQLITE_DROP_TABLE 11 /* Table Name NULL */ +#define SQLITE_DROP_TEMP_INDEX 12 /* Index Name Table Name */ +#define SQLITE_DROP_TEMP_TABLE 13 /* Table Name NULL */ +#define SQLITE_DROP_TEMP_TRIGGER 14 /* Trigger Name Table Name */ +#define SQLITE_DROP_TEMP_VIEW 15 /* View Name NULL */ +#define SQLITE_DROP_TRIGGER 16 /* Trigger Name Table Name */ +#define SQLITE_DROP_VIEW 17 /* View Name NULL */ +#define SQLITE_INSERT 18 /* Table Name NULL */ +#define SQLITE_PRAGMA 19 /* Pragma Name 1st arg or NULL */ +#define SQLITE_READ 20 /* Table Name Column Name */ +#define SQLITE_SELECT 21 /* NULL NULL */ +#define SQLITE_TRANSACTION 22 /* Operation NULL */ +#define SQLITE_UPDATE 23 /* Table Name Column Name */ +#define SQLITE_ATTACH 24 /* Filename NULL */ +#define SQLITE_DETACH 25 /* Database Name NULL */ +#define SQLITE_ALTER_TABLE 26 /* Database Name Table Name */ +#define SQLITE_REINDEX 27 /* Index Name NULL */ +#define SQLITE_ANALYZE 28 /* Table Name NULL */ +#define SQLITE_CREATE_VTABLE 29 /* Table Name Module Name */ +#define SQLITE_DROP_VTABLE 30 /* Table Name Module Name */ +#define SQLITE_FUNCTION 31 /* NULL Function Name */ +#define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ +#define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ + +/* +** CAPI3REF: Tracing And Profiling Functions +** +** These routines register callback functions that can be used for +** tracing and profiling the execution of SQL statements. +** +** ^The callback function registered by sqlite3_trace() is invoked at +** various times when an SQL statement is being run by [sqlite3_step()]. +** ^The sqlite3_trace() callback is invoked with a UTF-8 rendering of the +** SQL statement text as the statement first begins executing. +** ^(Additional sqlite3_trace() callbacks might occur +** as each triggered subprogram is entered. The callbacks for triggers +** contain a UTF-8 SQL comment that identifies the trigger.)^ +** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** +** ^The callback function registered by sqlite3_profile() is invoked +** as each SQL statement finishes. ^The profile callback contains +** the original statement text and an estimate of wall-clock time +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. The +** sqlite3_profile() function is considered experimental and is +** subject to change in future versions of SQLite. +*/ +SQLITE_API void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); +SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, + void(*xProfile)(void*,const char*,sqlite3_uint64), void*); + +/* +** CAPI3REF: Query Progress Callbacks +** +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this +** interface is to keep a GUI updated during a large query. +** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the approximate number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. ^If N is less than one then the progress +** handler is disabled. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. +** +** ^If the progress callback returns non-zero, the operation is +** interrupted. This feature can be used to implement a +** "Cancel" button on a GUI progress dialog box. +** +** The progress handler callback must not do anything that will modify +** the database connection that invoked the progress handler. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +*/ +SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); + +/* +** CAPI3REF: Opening A New Database Connection +** +** ^These routines open an SQLite database file as specified by the +** filename argument. ^The filename argument is interpreted as UTF-8 for +** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte +** order for sqlite3_open16(). ^(A [database connection] handle is usually +** returned in *ppDb, even if an error occurs. The only exception is that +** if SQLite is unable to allocate memory to hold the [sqlite3] object, +** a NULL will be written into *ppDb instead of a pointer to the [sqlite3] +** object.)^ ^(If the database is opened (and/or created) successfully, then +** [SQLITE_OK] is returned. Otherwise an [error code] is returned.)^ ^The +** [sqlite3_errmsg()] or [sqlite3_errmsg16()] routines can be used to obtain +** an English language description of the error following a failure of any +** of the sqlite3_open() routines. +** +** ^The default encoding for the database will be UTF-8 if +** sqlite3_open() or sqlite3_open_v2() is called and +** UTF-16 in the native byte order if sqlite3_open16() is used. +** +** Whether or not an error occurs when it is opened, resources +** associated with the [database connection] handle should be released by +** passing it to [sqlite3_close()] when it is no longer required. +** +** The sqlite3_open_v2() interface works like sqlite3_open() +** except that it accepts two additional parameters for additional control +** over the new database connection. ^(The flags parameter to +** sqlite3_open_v2() can take one of +** the following three values, optionally combined with the +** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE], +** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^ +** +**
+** ^(
[SQLITE_OPEN_READONLY]
+**
The database is opened in read-only mode. If the database does not +** already exist, an error is returned.
)^ +** +** ^(
[SQLITE_OPEN_READWRITE]
+**
The database is opened for reading and writing if possible, or reading +** only if the file is write protected by the operating system. In either +** case the database must already exist, otherwise an error is returned.
)^ +** +** ^(
[SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]
+**
The database is opened for reading and writing, and is created if +** it does not already exist. This is the behavior that is always used for +** sqlite3_open() and sqlite3_open16().
)^ +**
+** +** If the 3rd parameter to sqlite3_open_v2() is not one of the +** combinations shown above optionally combined with other +** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] +** then the behavior is undefined. +** +** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection +** opens in the multi-thread [threading mode] as long as the single-thread +** mode has not been set at compile-time or start-time. ^If the +** [SQLITE_OPEN_FULLMUTEX] flag is set then the database connection opens +** in the serialized [threading mode] unless single-thread was +** previously selected at compile-time or start-time. +** ^The [SQLITE_OPEN_SHAREDCACHE] flag causes the database connection to be +** eligible to use [shared cache mode], regardless of whether or not shared +** cache is enabled using [sqlite3_enable_shared_cache()]. ^The +** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not +** participate in [shared cache mode] even if it is enabled. +** +** ^The fourth parameter to sqlite3_open_v2() is the name of the +** [sqlite3_vfs] object that defines the operating system interface that +** the new database connection should use. ^If the fourth parameter is +** a NULL pointer then the default [sqlite3_vfs] object is used. +** +** ^If the filename is ":memory:", then a private, temporary in-memory database +** is created for the connection. ^This in-memory database will vanish when +** the database connection is closed. Future versions of SQLite might +** make use of additional special filenames that begin with the ":" character. +** It is recommended that when a database filename actually does begin with +** a ":" character you should prefix the filename with a pathname such as +** "./" to avoid ambiguity. +** +** ^If the filename is an empty string, then a private, temporary +** on-disk database will be created. ^This private database will be +** automatically deleted as soon as the database connection is closed. +** +** [[URI filenames in sqlite3_open()]]

URI Filenames

+** +** ^If [URI filename] interpretation is enabled, and the filename argument +** begins with "file:", then the filename is interpreted as a URI. ^URI +** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is +** set in the fourth argument to sqlite3_open_v2(), or if it has +** been enabled globally using the [SQLITE_CONFIG_URI] option with the +** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. +** As of SQLite version 3.7.7, URI filename interpretation is turned off +** by default, but future releases of SQLite might enable URI filename +** interpretation by default. See "[URI filenames]" for additional +** information. +** +** URI filenames are parsed according to RFC 3986. ^If the URI contains an +** authority, then it must be either an empty string or the string +** "localhost". ^If the authority is not an empty string or "localhost", an +** error is returned to the caller. ^The fragment component of a URI, if +** present, is ignored. +** +** ^SQLite uses the path component of the URI as the name of the disk file +** which contains the database. ^If the path begins with a '/' character, +** then it is interpreted as an absolute path. ^If the path does not begin +** with a '/' (meaning that the authority section is omitted from the URI) +** then the path is interpreted as a relative path. +** ^On windows, the first component of an absolute path +** is a drive specification (e.g. "C:"). +** +** [[core URI query parameters]] +** The query component of a URI may contain parameters that are interpreted +** either by SQLite itself, or by a [VFS | custom VFS implementation]. +** SQLite interprets the following three query parameters: +** +**
    +**
  • vfs: ^The "vfs" parameter may be used to specify the name of +** a VFS object that provides the operating system interface that should +** be used to access the database file on disk. ^If this option is set to +** an empty string the default VFS object is used. ^Specifying an unknown +** VFS is an error. ^If sqlite3_open_v2() is used and the vfs option is +** present, then the VFS specified by the option takes precedence over +** the value passed as the fourth parameter to sqlite3_open_v2(). +** +**
  • mode: ^(The mode parameter may be set to either "ro", "rw", +** "rwc", or "memory". Attempting to set it to any other value is +** an error)^. +** ^If "ro" is specified, then the database is opened for read-only +** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the +** third argument to sqlite3_open_v2(). ^If the mode option is set to +** "rw", then the database is opened for read-write (but not create) +** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had +** been set. ^Value "rwc" is equivalent to setting both +** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If the mode option is +** set to "memory" then a pure [in-memory database] that never reads +** or writes from disk is used. ^It is an error to specify a value for +** the mode parameter that is less restrictive than that specified by +** the flags passed in the third parameter to sqlite3_open_v2(). +** +**
  • cache: ^The cache parameter may be set to either "shared" or +** "private". ^Setting it to "shared" is equivalent to setting the +** SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to +** sqlite3_open_v2(). ^Setting the cache parameter to "private" is +** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. +** ^If sqlite3_open_v2() is used and the "cache" parameter is present in +** a URI filename, its value overrides any behavior requested by setting +** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
  • psow: ^The psow parameter may be "true" (or "on" or "yes" or +** "1") or "false" (or "off" or "no" or "0") to indicate that the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. ^The psow query +** parameter only works for the built-in unix and Windows VFSes. +** +**
  • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
  • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** +**
+** +** ^Specifying an unknown parameter in the query component of a URI is not an +** error. Future versions of SQLite might understand additional query +** parameters. See "[query parameters with special meaning to SQLite]" for +** additional information. +** +** [[URI filename examples]]

URI filename examples

+** +** +**
URI filenames Results +**
file:data.db +** Open the file "data.db" in the current directory. +**
file:/home/fred/data.db
+** file:///home/fred/data.db
+** file://localhost/home/fred/data.db
+** Open the database file "/home/fred/data.db". +**
file://darkstar/home/fred/data.db +** An error. "darkstar" is not a recognized authority. +**
+** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db +** Windows only: Open the file "data.db" on fred's desktop on drive +** C:. Note that the %20 escaping in this example is not strictly +** necessary - space characters can be used literally +** in URI filenames. +**
file:data.db?mode=ro&cache=private +** Open file "data.db" in the current directory for read-only access. +** Regardless of whether or not shared-cache mode is enabled by +** default, use a private cache. +**
file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. +**
file:data.db?mode=readonly +** An error. "readonly" is not a valid option for the "mode" parameter. +**
+** +** ^URI hexadecimal escape sequences (%HH) are supported within the path and +** query components of a URI. A hexadecimal escape sequence consists of a +** percent sign - "%" - followed by exactly two hexadecimal digits +** specifying an octet value. ^Before the path or query components of a +** URI filename are interpreted, they are encoded using UTF-8 and all +** hexadecimal escape sequences replaced by a single byte containing the +** corresponding octet. If this process generates an invalid UTF-8 encoding, +** the results are undefined. +** +** Note to Windows users: The encoding used for the filename argument +** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever +** codepage is currently defined. Filenames containing international +** characters must be converted to UTF-8 prior to passing them into +** sqlite3_open() or sqlite3_open_v2(). +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling sqlite3_open() or sqlite3_open_v2(). Otherwise, various +** features that require the use of temporary files may fail. +** +** See also: [sqlite3_temp_directory] +*/ +SQLITE_API int sqlite3_open( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ +); +SQLITE_API int sqlite3_open16( + const void *filename, /* Database filename (UTF-16) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ +); +SQLITE_API int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +); + +/* +** CAPI3REF: Obtain Values For URI Parameters +** +** These are utility routines, useful to VFS implementations, that check +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of that query parameter. +** +** If F is the database filename pointer passed into the xOpen() method of +** a VFS implementation when the flags parameter to xOpen() has one or +** more of the [SQLITE_OPEN_URI] or [SQLITE_OPEN_MAIN_DB] bits set and +** P is the name of the query parameter, then +** sqlite3_uri_parameter(F,P) returns the value of the P +** parameter if it exists or a NULL pointer if P does not appear as a +** query parameter on F. If P is a query parameter of F +** has no explicit value, then sqlite3_uri_parameter(F,P) returns +** a pointer to an empty string. +** +** The sqlite3_uri_boolean(F,P,B) routine assumes that P is a boolean +** parameter and returns true (1) or false (0) according to the value +** of P. The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the +** value of query parameter P is one of "yes", "true", or "on" in any +** case or if the value begins with a non-zero number. The +** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of +** query parameter P is one of "no", "false", or "off" in any case or +** if the value begins with a numeric zero. If P is not a query +** parameter on F or if the value of P is does not match any of the +** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). +** +** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a +** 64-bit signed integer and returns that integer, or D if P does not +** exist. If the value of P is something other than an integer, then +** zero is returned. +** +** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and +** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and +** is not a database file pathname pointer that SQLite passed into the xOpen +** VFS method, then the behavior of this routine is undefined and probably +** undesirable. +*/ +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); + + +/* +** CAPI3REF: Error Codes And Messages +** +** ^The sqlite3_errcode() interface returns the numeric [result code] or +** [extended result code] for the most recent failed sqlite3_* API call +** associated with a [database connection]. If a prior API call failed +** but the most recent API call succeeded, the return value from +** sqlite3_errcode() is undefined. ^The sqlite3_extended_errcode() +** interface is the same except that it always returns the +** [extended result code] even when extended result codes are +** disabled. +** +** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language +** text that describes the error, as either UTF-8 or UTF-16 respectively. +** ^(Memory to hold the error message string is managed internally. +** The application does not need to worry about freeing the result. +** However, the error string might be overwritten or deallocated by +** subsequent calls to other SQLite interface functions.)^ +** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** +** When the serialized [threading mode] is in use, it might be the +** case that a second error occurs on a separate thread in between +** the time of the first error and the call to these interfaces. +** When that happens, the second error will be reported since these +** interfaces always report the most recent result. To avoid +** this, each thread can obtain exclusive use of the [database connection] D +** by invoking [sqlite3_mutex_enter]([sqlite3_db_mutex](D)) before beginning +** to use D and invoking [sqlite3_mutex_leave]([sqlite3_db_mutex](D)) after +** all calls to the interfaces listed here are completed. +** +** If an interface fails with SQLITE_MISUSE, that means the interface +** was invoked incorrectly by the application. In that case, the +** error code and message may or may not be set. +*/ +SQLITE_API int sqlite3_errcode(sqlite3 *db); +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); +SQLITE_API const char *sqlite3_errmsg(sqlite3*); +SQLITE_API const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *sqlite3_errstr(int); + +/* +** CAPI3REF: SQL Statement Object +** KEYWORDS: {prepared statement} {prepared statements} +** +** An instance of this object represents a single SQL statement. +** This object is variously known as a "prepared statement" or a +** "compiled SQL statement" or simply as a "statement". +** +** The life of a statement object goes something like this: +** +**
    +**
  1. Create the object using [sqlite3_prepare_v2()] or a related +** function. +**
  2. Bind values to [host parameters] using the sqlite3_bind_*() +** interfaces. +**
  3. Run the SQL by calling [sqlite3_step()] one or more times. +**
  4. Reset the statement using [sqlite3_reset()] then go back +** to step 2. Do this zero or more times. +**
  5. Destroy the object using [sqlite3_finalize()]. +**
+** +** Refer to documentation on individual methods above for additional +** information. +*/ +typedef struct sqlite3_stmt sqlite3_stmt; + +/* +** CAPI3REF: Run-time Limits +** +** ^(This interface allows the size of various constructs to be limited +** on a connection by connection basis. The first parameter is the +** [database connection] whose limit is to be set or queried. The +** second parameter is one of the [limit categories] that define a +** class of constructs to be size limited. The third parameter is the +** new limit for that construct.)^ +** +** ^If the new limit is a negative number, the limit is unchanged. +** ^(For each limit category SQLITE_LIMIT_NAME there is a +** [limits | hard upper bound] +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. +** (The "_LIMIT_" in the name is changed to "_MAX_".))^ +** ^Attempts to increase a limit above its hard upper bound are +** silently truncated to the hard upper bound. +** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** +** Run-time limits are intended for use in applications that manage +** both their own internal database and also databases that are controlled +** by untrusted external sources. An example application might be a +** web browser that has its own databases for storing history and +** separate databases controlled by JavaScript applications downloaded +** off the Internet. The internal databases can be given the +** large, default limits. Databases managed by external sources can +** be given much smaller limits designed to prevent a denial of service +** attack. Developers might also want to use the [sqlite3_set_authorizer()] +** interface to further control untrusted SQL. The size of the database +** created by an untrusted script can be contained using the +** [max_page_count] [PRAGMA]. +** +** New run-time limit categories may be added in future releases. +*/ +SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); + +/* +** CAPI3REF: Run-Time Limit Categories +** KEYWORDS: {limit category} {*limit categories} +** +** These constants define various performance limits +** that can be lowered at run-time using [sqlite3_limit()]. +** The synopsis of the meanings of the various limits is shown below. +** Additional information is available at [limits | Limits in SQLite]. +** +**
+** [[SQLITE_LIMIT_LENGTH]] ^(
SQLITE_LIMIT_LENGTH
+**
The maximum size of any string or BLOB or table row, in bytes.
)^ +** +** [[SQLITE_LIMIT_SQL_LENGTH]] ^(
SQLITE_LIMIT_SQL_LENGTH
+**
The maximum length of an SQL statement, in bytes.
)^ +** +** [[SQLITE_LIMIT_COLUMN]] ^(
SQLITE_LIMIT_COLUMN
+**
The maximum number of columns in a table definition or in the +** result set of a [SELECT] or the maximum number of columns in an index +** or in an ORDER BY or GROUP BY clause.
)^ +** +** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(
SQLITE_LIMIT_EXPR_DEPTH
+**
The maximum depth of the parse tree on any expression.
)^ +** +** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(
SQLITE_LIMIT_COMPOUND_SELECT
+**
The maximum number of terms in a compound SELECT statement.
)^ +** +** [[SQLITE_LIMIT_VDBE_OP]] ^(
SQLITE_LIMIT_VDBE_OP
+**
The maximum number of instructions in a virtual machine program +** used to implement an SQL statement. This limit is not currently +** enforced, though that might be added in some future release of +** SQLite.
)^ +** +** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
SQLITE_LIMIT_FUNCTION_ARG
+**
The maximum number of arguments on a function.
)^ +** +** [[SQLITE_LIMIT_ATTACHED]] ^(
SQLITE_LIMIT_ATTACHED
+**
The maximum number of [ATTACH | attached databases].)^
+** +** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]] +** ^(
SQLITE_LIMIT_LIKE_PATTERN_LENGTH
+**
The maximum length of the pattern argument to the [LIKE] or +** [GLOB] operators.
)^ +** +** [[SQLITE_LIMIT_VARIABLE_NUMBER]] +** ^(
SQLITE_LIMIT_VARIABLE_NUMBER
+**
The maximum index number of any [parameter] in an SQL statement.)^ +** +** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
SQLITE_LIMIT_TRIGGER_DEPTH
+**
The maximum depth of recursion for triggers.
)^ +**
+*/ +#define SQLITE_LIMIT_LENGTH 0 +#define SQLITE_LIMIT_SQL_LENGTH 1 +#define SQLITE_LIMIT_COLUMN 2 +#define SQLITE_LIMIT_EXPR_DEPTH 3 +#define SQLITE_LIMIT_COMPOUND_SELECT 4 +#define SQLITE_LIMIT_VDBE_OP 5 +#define SQLITE_LIMIT_FUNCTION_ARG 6 +#define SQLITE_LIMIT_ATTACHED 7 +#define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 +#define SQLITE_LIMIT_VARIABLE_NUMBER 9 +#define SQLITE_LIMIT_TRIGGER_DEPTH 10 + +/* +** CAPI3REF: Compiling An SQL Statement +** KEYWORDS: {SQL statement compiler} +** +** To execute an SQL query, it must first be compiled into a byte-code +** program using one of these routines. +** +** The first argument, "db", is a [database connection] obtained from a +** prior successful call to [sqlite3_open()], [sqlite3_open_v2()] or +** [sqlite3_open16()]. The database connection must not have been closed. +** +** The second argument, "zSql", is the statement to be compiled, encoded +** as either UTF-8 or UTF-16. The sqlite3_prepare() and sqlite3_prepare_v2() +** interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2() +** use UTF-16. +** +** ^If the nByte argument is less than zero, then zSql is read up to the +** first zero terminator. ^If nByte is non-negative, then it is the maximum +** number of bytes read from zSql. ^When nByte is non-negative, the +** zSql string ends at either the first '\000' or '\u0000' character or +** the nByte-th byte, whichever comes first. If the caller knows +** that the supplied string is nul-terminated, then there is a small +** performance advantage to be gained by passing an nByte parameter that +** is equal to the number of bytes in the input string including +** the nul-terminator bytes as this saves SQLite from having to +** make a copy of the input string. +** +** ^If pzTail is not NULL then *pzTail is made to point to the first byte +** past the end of the first SQL statement in zSql. These routines only +** compile the first statement in zSql, so *pzTail is left pointing to +** what remains uncompiled. +** +** ^*ppStmt is left pointing to a compiled [prepared statement] that can be +** executed using [sqlite3_step()]. ^If there is an error, *ppStmt is set +** to NULL. ^If the input text contains no SQL (if the input is an empty +** string or a comment) then *ppStmt is set to NULL. +** The calling procedure is responsible for deleting the compiled +** SQL statement using [sqlite3_finalize()] after it has finished with it. +** ppStmt may not be NULL. +** +** ^On success, the sqlite3_prepare() family of routines return [SQLITE_OK]; +** otherwise an [error code] is returned. +** +** The sqlite3_prepare_v2() and sqlite3_prepare16_v2() interfaces are +** recommended for all new programs. The two older interfaces are retained +** for backwards compatibility, but their use is discouraged. +** ^In the "v2" interfaces, the prepared statement +** that is returned (the [sqlite3_stmt] object) contains a copy of the +** original SQL text. This causes the [sqlite3_step()] interface to +** behave differently in three ways: +** +**
    +**
  1. +** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it +** always used to do, [sqlite3_step()] will automatically recompile the SQL +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. +**
    2. +** +**
  2. +** ^When an error occurs, [sqlite3_step()] will return one of the detailed +** [error codes] or [extended error codes]. ^The legacy behavior was that +** [sqlite3_step()] would only return a generic [SQLITE_ERROR] result code +** and the application would have to make a second call to [sqlite3_reset()] +** in order to find the underlying cause of the problem. With the "v2" prepare +** interfaces, the underlying reason for the error is returned immediately. +**
    3. +** +**
  3. +** ^If the specific value bound to [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. +**
    4. +**
+*/ +SQLITE_API int sqlite3_prepare( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); + +/* +** CAPI3REF: Retrieving Statement SQL +** +** ^This interface can be used to retrieve a saved copy of the original +** SQL text used to create a [prepared statement] if that statement was +** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. +*/ +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Determine If An SQL Statement Writes The Database +** +** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if +** and only if the [prepared statement] X makes no direct changes to +** the content of the database file. +** +** Note that [application-defined SQL functions] or +** [virtual tables] might change the database indirectly as a side effect. +** ^(For example, if an application defines a function "eval()" that +** calls [sqlite3_exec()], then the following SQL statement would +** change the database file through side-effects: +** +** 
+**    SELECT eval('DELETE FROM t1') FROM t2;
+**
+** +** But because the [SELECT] statement does not change the database file +** directly, sqlite3_stmt_readonly() would still return true.)^ +** +** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK], +** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true, +** since the statements themselves do not actually modify the database but +** rather they control the timing of when other statements modify the +** database. ^The [ATTACH] and [DETACH] statements also cause +** sqlite3_stmt_readonly() to return true since, while those statements +** change the configuration of a database connection, they do not make +** changes to the content of the database files on disk. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** +** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the +** [prepared statement] S has been stepped at least once using +** [sqlite3_step(S)] but has not run to completion and/or has not +** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) +** interface returns false if S is a NULL pointer. If S is not a +** NULL pointer and is not a pointer to a valid [prepared statement] +** object, then the behavior is undefined and probably undesirable. +** +** This interface can be used in combination [sqlite3_next_stmt()] +** to locate all prepared statements associated with a database +** connection that are in need of being reset. This can be used, +** for example, in diagnostic routines to search for prepared +** statements that are holding a transaction open. +*/ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); + +/* +** CAPI3REF: Dynamically Typed Value Object +** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} +** +** SQLite uses the sqlite3_value object to represent all values +** that can be stored in a database table. SQLite uses dynamic typing +** for the values it stores. ^Values stored in sqlite3_value objects +** can be integers, floating point values, strings, BLOBs, or NULL. +** +** An sqlite3_value object may be either "protected" or "unprotected". +** Some interfaces require a protected sqlite3_value. Other interfaces +** will accept either a protected or an unprotected sqlite3_value. +** Every interface that accepts sqlite3_value arguments specifies +** whether or not it requires a protected sqlite3_value. +** +** The terms "protected" and "unprotected" refer to whether or not +** a mutex is held. An internal mutex is held for a protected +** sqlite3_value object but no mutex is held for an unprotected +** sqlite3_value object. If SQLite is compiled to be single-threaded +** (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0) +** or if SQLite is run in one of reduced mutex modes +** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD] +** then there is no distinction between protected and unprotected +** sqlite3_value objects and they can be used interchangeably. However, +** for maximum code portability it is recommended that applications +** still make the distinction between protected and unprotected +** sqlite3_value objects even when not strictly required. +** +** ^The sqlite3_value objects that are passed as parameters into the +** implementation of [application-defined SQL functions] are protected. +** ^The sqlite3_value object returned by +** [sqlite3_column_value()] is unprotected. +** Unprotected sqlite3_value objects may only be used with +** [sqlite3_result_value()] and [sqlite3_bind_value()]. +** The [sqlite3_value_blob | sqlite3_value_type()] family of +** interfaces require protected sqlite3_value objects. +*/ +typedef struct Mem sqlite3_value; + +/* +** CAPI3REF: SQL Function Context Object +** +** The context in which an SQL function executes is stored in an +** sqlite3_context object. ^A pointer to an sqlite3_context object +** is always first parameter to [application-defined SQL functions]. +** The application-defined SQL function implementation will pass this +** pointer through into calls to [sqlite3_result_int | sqlite3_result()], +** [sqlite3_aggregate_context()], [sqlite3_user_data()], +** [sqlite3_context_db_handle()], [sqlite3_get_auxdata()], +** and/or [sqlite3_set_auxdata()]. +*/ +typedef struct sqlite3_context sqlite3_context; + +/* +** CAPI3REF: Binding Values To Prepared Statements +** KEYWORDS: {host parameter} {host parameters} {host parameter name} +** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding} +** +** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, +** literals may be replaced by a [parameter] that matches one of following +** templates: +** +**
    +**
  • ? +**
  • ?NNN +**
  • :VVV +**
  • @VVV +**
  • $VVV +**
+** +** In the templates above, NNN represents an integer literal, +** and VVV represents an alphanumeric identifier.)^ ^The values of these +** parameters (also called "host parameter names" or "SQL parameters") +** can be set using the sqlite3_bind_*() routines defined here. +** +** ^The first argument to the sqlite3_bind_*() routines is always +** a pointer to the [sqlite3_stmt] object returned from +** [sqlite3_prepare_v2()] or its variants. +** +** ^The second argument is the index of the SQL parameter to be set. +** ^The leftmost SQL parameter has an index of 1. ^When the same named +** SQL parameter is used more than once, second and subsequent +** occurrences have the same index as the first occurrence. +** ^The index for named parameters can be looked up using the +** [sqlite3_bind_parameter_index()] API if desired. ^The index +** for "?NNN" parameters is the value of NNN. +** ^The NNN value must be between 1 and the [sqlite3_limit()] +** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). +** +** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). +** +** ^(In those routines that have a fourth argument, its value is the +** number of bytes in the parameter. To be clear: the value is the +** number of bytes in the value, not the number of characters.)^ +** ^If the fourth parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** is negative, then the length of the string is +** the number of bytes up to the first zero terminator. +** If the fourth parameter to sqlite3_bind_blob() is negative, then +** the behavior is undefined. +** If a non-negative fourth parameter is provided to sqlite3_bind_text() +** or sqlite3_bind_text16() then that parameter must be the byte offset +** where the NUL terminator would occur assuming the string were NUL +** terminated. If any NUL characters occur at byte offsets less than +** the value of the fourth parameter then the resulting string value will +** contain embedded NULs. The result of expressions involving strings +** with embedded NULs is undefined. +** +** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and +** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or +** string after SQLite has finished with it. ^The destructor is called +** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), +** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** ^If the fifth argument is +** the special value [SQLITE_STATIC], then SQLite assumes that the +** information is in static, unmanaged space and does not need to be freed. +** ^If the fifth argument has the value [SQLITE_TRANSIENT], then +** SQLite makes its own private copy of the data immediately, before +** the sqlite3_bind_*() routine returns. +** +** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that +** is filled with zeroes. ^A zeroblob uses a fixed amount of memory +** (just an integer to hold its size) while it is being processed. +** Zeroblobs are intended to serve as placeholders for BLOBs whose +** content is later written using +** [sqlite3_blob_open | incremental BLOB I/O] routines. +** ^A negative value for the zeroblob results in a zero-length BLOB. +** +** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer +** for the [prepared statement] or with a prepared statement for which +** [sqlite3_step()] has been called more recently than [sqlite3_reset()], +** then the call will return [SQLITE_MISUSE]. If any sqlite3_bind_() +** routine is passed a [prepared statement] that has been finalized, the +** result is undefined and probably harmful. +** +** ^Bindings are not cleared by the [sqlite3_reset()] routine. +** ^Unbound parameters are interpreted as NULL. +** +** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an +** [error code] if anything goes wrong. +** ^[SQLITE_RANGE] is returned if the parameter +** index is out of range. ^[SQLITE_NOMEM] is returned if malloc() fails. +** +** See also: [sqlite3_bind_parameter_count()], +** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); +SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); +SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); +SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); +SQLITE_API int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); +SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); + +/* +** CAPI3REF: Number Of SQL Parameters +** +** ^This routine can be used to find the number of [SQL parameters] +** in a [prepared statement]. SQL parameters are tokens of the +** form "?", "?NNN", ":AAA", "$AAA", or "@AAA" that serve as +** placeholders for values that are [sqlite3_bind_blob | bound] +** to the parameters at a later time. +** +** ^(This routine actually returns the index of the largest (rightmost) +** parameter. For all forms except ?NNN, this will correspond to the +** number of unique parameters. If parameters of the ?NNN form are used, +** there may be gaps in the list.)^ +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_name()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); + +/* +** CAPI3REF: Name Of A Host Parameter +** +** ^The sqlite3_bind_parameter_name(P,N) interface returns +** the name of the N-th [SQL parameter] in the [prepared statement] P. +** ^(SQL parameters of the form "?NNN" or ":AAA" or "@AAA" or "$AAA" +** have a name which is the string "?NNN" or ":AAA" or "@AAA" or "$AAA" +** respectively. +** In other words, the initial ":" or "$" or "@" or "?" +** is included as part of the name.)^ +** ^Parameters of the form "?" without a following integer have no name +** and are referred to as "nameless" or "anonymous parameters". +** +** ^The first host parameter has an index of 1, not 0. +** +** ^If the value N is out of range or if the N-th parameter is +** nameless, then NULL is returned. ^The returned string is +** always in UTF-8 encoding even if the named parameter was +** originally specified as UTF-16 in [sqlite3_prepare16()] or +** [sqlite3_prepare16_v2()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); + +/* +** CAPI3REF: Index Of A Parameter With A Given Name +** +** ^Return the index of an SQL parameter given its name. ^The +** index value returned is suitable for use as the second +** parameter to [sqlite3_bind_blob|sqlite3_bind()]. ^A zero +** is returned if no matching parameter is found. ^The parameter +** name must be given in UTF-8 even if the original statement +** was prepared from UTF-16 text using [sqlite3_prepare16_v2()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); + +/* +** CAPI3REF: Reset All Bindings On A Prepared Statement +** +** ^Contrary to the intuition of many, [sqlite3_reset()] does not reset +** the [sqlite3_bind_blob | bindings] on a [prepared statement]. +** ^Use this routine to reset all host parameters to NULL. +*/ +SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); + +/* +** CAPI3REF: Number Of Columns In A Result Set +** +** ^Return the number of columns in the result set returned by the +** [prepared statement]. ^This routine returns 0 if pStmt is an SQL +** statement that does not return data (for example an [UPDATE]). +** +** See also: [sqlite3_data_count()] +*/ +SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Column Names In A Result Set +** +** ^These routines return the name assigned to a particular column +** in the result set of a [SELECT] statement. ^The sqlite3_column_name() +** interface returns a pointer to a zero-terminated UTF-8 string +** and sqlite3_column_name16() returns a pointer to a zero-terminated +** UTF-16 string. ^The first parameter is the [prepared statement] +** that implements the [SELECT] statement. ^The second parameter is the +** column number. ^The leftmost column is number 0. +** +** ^The returned string pointer is valid until either the [prepared statement] +** is destroyed by [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the next call to +** sqlite3_column_name() or sqlite3_column_name16() on the same column. +** +** ^If sqlite3_malloc() fails during the processing of either routine +** (for example during a conversion from UTF-8 to UTF-16) then a +** NULL pointer is returned. +** +** ^The name of a result column is the value of the "AS" clause for +** that column, if there is an AS clause. If there is no AS clause +** then the name of the column is unspecified and may change from +** one release of SQLite to the next. +*/ +SQLITE_API const char *sqlite3_column_name(sqlite3_stmt*, int N); +SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); + +/* +** CAPI3REF: Source Of Data In A Query Result +** +** ^These routines provide a means to determine the database, table, and +** table column that is the origin of a particular result column in +** [SELECT] statement. +** ^The name of the database or table or column can be returned as +** either a UTF-8 or UTF-16 string. ^The _database_ routines return +** the database name, the _table_ routines return the table name, and +** the origin_ routines return the column name. +** ^The returned string is valid until the [prepared statement] is destroyed +** using [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the same information is requested +** again in a different encoding. +** +** ^The names returned are the original un-aliased names of the +** database, table, and column. +** +** ^The first argument to these interfaces is a [prepared statement]. +** ^These functions return information about the Nth result column returned by +** the statement, where N is the second function argument. +** ^The left-most column is column 0 for these routines. +** +** ^If the Nth column returned by the statement is an expression or +** subquery and is not a column value, then all of these functions return +** NULL. ^These routine might also return NULL if a memory allocation error +** occurs. ^Otherwise, they return the name of the attached database, table, +** or column that query result column was extracted from. +** +** ^As with all other SQLite APIs, those whose names end with "16" return +** UTF-16 encoded strings and the other functions return UTF-8. +** +** ^These APIs are only available if the library was compiled with the +** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol. +** +** If two or more threads call one or more of these routines against the same +** prepared statement and column at the same time then the results are +** undefined. +** +** If two or more threads call one or more +** [sqlite3_column_database_name | column metadata interfaces] +** for the same [prepared statement] and result column +** at the same time then the results are undefined. +*/ +SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); + +/* +** CAPI3REF: Declared Datatype Of A Query Result +** +** ^(The first parameter is a [prepared statement]. +** If this statement is a [SELECT] statement and the Nth column of the +** returned result set of that [SELECT] is a table column (not an +** expression or subquery) then the declared type of the table +** column is returned.)^ ^If the Nth column of the result set is an +** expression or subquery, then a NULL pointer is returned. +** ^The returned string is always UTF-8 encoded. +** +** ^(For example, given the database schema: +** +** CREATE TABLE t1(c1 VARIANT); +** +** and the following statement to be compiled: +** +** SELECT c1 + 1, c1 FROM t1; +** +** this routine would return the string "VARIANT" for the second result +** column (i==1), and a NULL pointer for the first result column (i==0).)^ +** +** ^SQLite uses dynamic run-time typing. ^So just because a column +** is declared to contain a particular type does not mean that the +** data stored in that column is of the declared type. SQLite is +** strongly typed, but the typing is dynamic not static. ^Type +** is associated with individual values, not with the containers +** used to hold those values. +*/ +SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); + +/* +** CAPI3REF: Evaluate An SQL Statement +** +** After a [prepared statement] has been prepared using either +** [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or one of the legacy +** interfaces [sqlite3_prepare()] or [sqlite3_prepare16()], this function +** must be called one or more times to evaluate the statement. +** +** The details of the behavior of the sqlite3_step() interface depend +** on whether the statement was prepared using the newer "v2" interface +** [sqlite3_prepare_v2()] and [sqlite3_prepare16_v2()] or the older legacy +** interface [sqlite3_prepare()] and [sqlite3_prepare16()]. The use of the +** new "v2" interface is recommended for new applications but the legacy +** interface will continue to be supported. +** +** ^In the legacy interface, the return value will be either [SQLITE_BUSY], +** [SQLITE_DONE], [SQLITE_ROW], [SQLITE_ERROR], or [SQLITE_MISUSE]. +** ^With the "v2" interface, any of the other [result codes] or +** [extended result codes] might be returned as well. +** +** ^[SQLITE_BUSY] means that the database engine was unable to acquire the +** database locks it needs to do its job. ^If the statement is a [COMMIT] +** or occurs outside of an explicit transaction, then you can retry the +** statement. If the statement is not a [COMMIT] and occurs within an +** explicit transaction then you should rollback the transaction before +** continuing. +** +** ^[SQLITE_DONE] means that the statement has finished executing +** successfully. sqlite3_step() should not be called again on this virtual +** machine without first calling [sqlite3_reset()] to reset the virtual +** machine back to its initial state. +** +** ^If the SQL statement being executed returns any data, then [SQLITE_ROW] +** is returned each time a new row of data is ready for processing by the +** caller. The values may be accessed using the [column access functions]. +** sqlite3_step() is called again to retrieve the next row of data. +** +** ^[SQLITE_ERROR] means that a run-time error (such as a constraint +** violation) has occurred. sqlite3_step() should not be called again on +** the VM. More information may be found by calling [sqlite3_errmsg()]. +** ^With the legacy interface, a more specific error code (for example, +** [SQLITE_INTERRUPT], [SQLITE_SCHEMA], [SQLITE_CORRUPT], and so forth) +** can be obtained by calling [sqlite3_reset()] on the +** [prepared statement]. ^In the "v2" interface, +** the more specific error code is returned directly by sqlite3_step(). +** +** [SQLITE_MISUSE] means that the this routine was called inappropriately. +** Perhaps it was called on a [prepared statement] that has +** already been [sqlite3_finalize | finalized] or on one that had +** previously returned [SQLITE_ERROR] or [SQLITE_DONE]. Or it could +** be the case that the same database connection is being used by two or +** more threads at the same moment in time. +** +** For all versions of SQLite up to and including 3.6.23.1, a call to +** [sqlite3_reset()] was required after sqlite3_step() returned anything +** other than [SQLITE_ROW] before any subsequent invocation of +** sqlite3_step(). Failure to reset the prepared statement using +** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from +** sqlite3_step(). But after version 3.6.23.1, sqlite3_step() began +** calling [sqlite3_reset()] automatically in this circumstance rather +** than returning [SQLITE_MISUSE]. This is not considered a compatibility +** break because any application that ever receives an SQLITE_MISUSE error +** is broken by definition. The [SQLITE_OMIT_AUTORESET] compile-time option +** can be used to restore the legacy behavior. +** +** Goofy Interface Alert: In the legacy interface, the sqlite3_step() +** API always returns a generic error code, [SQLITE_ERROR], following any +** error other than [SQLITE_BUSY] and [SQLITE_MISUSE]. You must call +** [sqlite3_reset()] or [sqlite3_finalize()] in order to find one of the +** specific [error codes] that better describes the error. +** We admit that this is a goofy design. The problem has been fixed +** with the "v2" interface. If you prepare all of your SQL statements +** using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] instead +** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces, +** then the more specific [error codes] are returned directly +** by sqlite3_step(). The use of the "v2" interface is recommended. +*/ +SQLITE_API int sqlite3_step(sqlite3_stmt*); + +/* +** CAPI3REF: Number of columns in a result set +** +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. +** +** See also: [sqlite3_column_count()] +*/ +SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Fundamental Datatypes +** KEYWORDS: SQLITE_TEXT +** +** ^(Every value in SQLite has one of five fundamental datatypes: +** +**
    +**
  • 64-bit signed integer +**
  • 64-bit IEEE floating point number +**
  • string +**
  • BLOB +**
  • NULL +**
)^ +** +** These constants are codes for each of those types. +** +** Note that the SQLITE_TEXT constant was also used in SQLite version 2 +** for a completely different meaning. Software that links against both +** SQLite version 2 and SQLite version 3 should use SQLITE3_TEXT, not +** SQLITE_TEXT. +*/ +#define SQLITE_INTEGER 1 +#define SQLITE_FLOAT 2 +#define SQLITE_BLOB 4 +#define SQLITE_NULL 5 +#ifdef SQLITE_TEXT +# undef SQLITE_TEXT +#else +# define SQLITE_TEXT 3 +#endif +#define SQLITE3_TEXT 3 + +/* +** CAPI3REF: Result Values From A Query +** KEYWORDS: {column access functions} +** +** These routines form the "result set" interface. +** +** ^These routines return information about a single column of the current +** result row of a query. ^In every case the first argument is a pointer +** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*] +** that was returned from [sqlite3_prepare_v2()] or one of its variants) +** and the second argument is the index of the column for which information +** should be returned. ^The leftmost column of the result set has the index 0. +** ^The number of columns in the result can be determined using +** [sqlite3_column_count()]. +** +** If the SQL statement does not currently point to a valid row, or if the +** column index is out of range, the result is undefined. +** These routines may only be called when the most recent call to +** [sqlite3_step()] has returned [SQLITE_ROW] and neither +** [sqlite3_reset()] nor [sqlite3_finalize()] have been called subsequently. +** If any of these routines are called after [sqlite3_reset()] or +** [sqlite3_finalize()] or after [sqlite3_step()] has returned +** something other than [SQLITE_ROW], the results are undefined. +** If [sqlite3_step()] or [sqlite3_reset()] or [sqlite3_finalize()] +** are called from a different thread while any of these routines +** are pending, then the results are undefined. +** +** ^The sqlite3_column_type() routine returns the +** [SQLITE_INTEGER | datatype code] for the initial data type +** of the result column. ^The returned value is one of [SQLITE_INTEGER], +** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL]. The value +** returned by sqlite3_column_type() is only meaningful if no type +** conversions have occurred as described below. After a type conversion, +** the value returned by sqlite3_column_type() is undefined. Future +** versions of SQLite may change the behavior of sqlite3_column_type() +** following a type conversion. +** +** ^If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-16 string, then sqlite3_column_bytes() converts +** the string to UTF-8 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes() uses +** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of +** bytes in the string, not the number of characters. +** +** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), +** even empty strings, are always zero-terminated. ^The return +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. +** +** ^The object returned by [sqlite3_column_value()] is an +** [unprotected sqlite3_value] object. An unprotected sqlite3_value object +** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()]. +** If the [unprotected sqlite3_value] object returned by +** [sqlite3_column_value()] is used in any other way, including calls +** to routines like [sqlite3_value_int()], [sqlite3_value_text()], +** or [sqlite3_value_bytes()], then the behavior is undefined. +** +** These routines attempt to convert the value where appropriate. ^For +** example, if the internal representation is FLOAT and a text result +** is requested, [sqlite3_snprintf()] is used internally to perform the +** conversion automatically. ^(The following table details the conversions +** that are applied: +** +**
+** +**
Internal
Type 		Requested
Type 		Conversion +** +**
NULL INTEGER Result is 0 +**
NULL FLOAT Result is 0.0 +**
NULL TEXT Result is a NULL pointer +**
NULL BLOB Result is a NULL pointer +**
INTEGER FLOAT Convert from integer to float +**
INTEGER TEXT ASCII rendering of the integer +**
INTEGER BLOB Same as INTEGER->TEXT +**
FLOAT INTEGER [CAST] to INTEGER +**
FLOAT TEXT ASCII rendering of the float +**
FLOAT BLOB [CAST] to BLOB +**
TEXT INTEGER [CAST] to INTEGER +**
TEXT FLOAT [CAST] to REAL +**
TEXT BLOB No change +**
BLOB INTEGER [CAST] to INTEGER +**
BLOB FLOAT [CAST] to REAL +**
BLOB TEXT Add a zero terminator if needed +**
+**
)^ +** +** The table above makes reference to standard C library functions atoi() +** and atof(). SQLite does not really use these functions. It has its +** own equivalent internal routines. The atoi() and atof() names are +** used in the table for brevity and because they are familiar to most +** C programmers. +** +** Note that when type conversions occur, pointers returned by prior +** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or +** sqlite3_column_text16() may be invalidated. +** Type conversions and pointer invalidations might occur +** in the following cases: +** +**
    +**
  • The initial content is a BLOB and sqlite3_column_text() or +** sqlite3_column_text16() is called. A zero-terminator might +** need to be added to the string.
    • +**
  • The initial content is UTF-8 text and sqlite3_column_bytes16() or +** sqlite3_column_text16() is called. The content must be converted +** to UTF-16.
    • +**
  • The initial content is UTF-16 text and sqlite3_column_bytes() or +** sqlite3_column_text() is called. The content must be converted +** to UTF-8.
    • +**
+** +** ^Conversions between UTF-16be and UTF-16le are always done in place and do +** not invalidate a prior pointer, though of course the content of the buffer +** that the prior pointer references will have been modified. Other kinds +** of conversion are done in place when it is possible, but sometimes they +** are not possible and in those cases prior pointers are invalidated. +** +** The safest and easiest to remember policy is to invoke these routines +** in one of the following ways: +** +**
    +**
  • sqlite3_column_text() followed by sqlite3_column_bytes()
    • +**
  • sqlite3_column_blob() followed by sqlite3_column_bytes()
    • +**
  • sqlite3_column_text16() followed by sqlite3_column_bytes16()
    • +**
+** +** In other words, you should call sqlite3_column_text(), +** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result +** into the desired format, then invoke sqlite3_column_bytes() or +** sqlite3_column_bytes16() to find the size of the result. Do not mix calls +** to sqlite3_column_text() or sqlite3_column_blob() with calls to +** sqlite3_column_bytes16(), and do not mix calls to sqlite3_column_text16() +** with calls to sqlite3_column_bytes(). +** +** ^The pointers returned are valid until a type conversion occurs as +** described above, or until [sqlite3_step()] or [sqlite3_reset()] or +** [sqlite3_finalize()] is called. ^The memory space used to hold strings +** and BLOBs is freed automatically. Do not pass the pointers returned +** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into +** [sqlite3_free()]. +** +** ^(If a memory allocation error occurs during the evaluation of any +** of these routines, a default value is returned. The default value +** is either the integer 0, the floating point number 0.0, or a NULL +** pointer. Subsequent calls to [sqlite3_errcode()] will return +** [SQLITE_NOMEM].)^ +*/ +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); +SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_int(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); + +/* +** CAPI3REF: Destroy A Prepared Statement Object +** +** ^The sqlite3_finalize() function is called to delete a [prepared statement]. +** ^If the most recent evaluation of the statement encountered no errors +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. +** +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. +*/ +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Reset A Prepared Statement Object +** +** The sqlite3_reset() function is called to reset a [prepared statement] +** object back to its initial state, ready to be re-executed. +** ^Any SQL statement variables that had values bound to them using +** the [sqlite3_bind_blob | sqlite3_bind_*() API] retain their values. +** Use [sqlite3_clear_bindings()] to reset the bindings. +** +** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S +** back to the beginning of its program. +** +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], +** or if [sqlite3_step(S)] has never before been called on S, +** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S indicated an error, then +** [sqlite3_reset(S)] returns an appropriate [error code]. +** +** ^The [sqlite3_reset(S)] interface does not change the values +** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. +*/ +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Create Or Redefine SQL Functions +** KEYWORDS: {function creation routines} +** KEYWORDS: {application-defined SQL function} +** KEYWORDS: {application-defined SQL functions} +** +** ^These functions (collectively known as "function creation routines") +** are used to add SQL functions or aggregates or to redefine the behavior +** of existing SQL functions or aggregates. The only differences between +** these routines are the text encoding expected for +** the second parameter (the name of the function being created) +** and the presence or absence of a destructor callback for +** the application data pointer. +** +** ^The first parameter is the [database connection] to which the SQL +** function is to be added. ^If an application uses more than one database +** connection then application-defined SQL functions must be added +** to each database connection separately. +** +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. +** +** ^The third parameter (nArg) +** is the number of arguments that the SQL function or +** aggregate takes. ^If this parameter is -1, then the SQL function or +** aggregate may take any number of arguments between 0 and the limit +** set by [sqlite3_limit]([SQLITE_LIMIT_FUNCTION_ARG]). If the third +** parameter is less than -1 or greater than 127 then the behavior is +** undefined. +** +** ^The fourth parameter, eTextRep, specifies what +** [SQLITE_UTF8 | text encoding] this SQL function prefers for +** its parameters. The application should set this parameter to +** [SQLITE_UTF16LE] if the function implementation invokes +** [sqlite3_value_text16le()] on an input, or [SQLITE_UTF16BE] if the +** implementation invokes [sqlite3_value_text16be()] on an input, or +** [SQLITE_UTF16] if [sqlite3_value_text16()] is used, or [SQLITE_UTF8] +** otherwise. ^The same SQL function may be registered multiple times using +** different preferred text encodings, with different implementations for +** each encoding. +** ^When multiple implementations of the same function are available, SQLite +** will pick the one that involves the least amount of data conversion. +** +** ^The fourth parameter may optionally be ORed with [SQLITE_DETERMINISTIC] +** to signal that the function will always return the same result given +** the same inputs within a single SQL statement. Most SQL functions are +** deterministic. The built-in [random()] SQL function is an example of a +** function that is not deterministic. The SQLite query planner is able to +** perform additional optimizations on deterministic functions, so use +** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. +** +** ^(The fifth parameter is an arbitrary pointer. The implementation of the +** function can gain access to this pointer using [sqlite3_user_data()].)^ +** +** ^The sixth, seventh and eighth parameters, xFunc, xStep and xFinal, are +** pointers to C-language functions that implement the SQL function or +** aggregate. ^A scalar SQL function requires an implementation of the xFunc +** callback only; NULL pointers must be passed as the xStep and xFinal +** parameters. ^An aggregate SQL function requires an implementation of xStep +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL pointers for all three function +** callbacks. +** +** ^(If the ninth parameter to sqlite3_create_function_v2() is not NULL, +** then it is destructor for the application data pointer. +** The destructor is invoked when the function is deleted, either by being +** overloaded or when the database connection closes.)^ +** ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. +** ^When the destructor callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the application data +** pointer which was the fifth parameter to sqlite3_create_function_v2(). +** +** ^It is permitted to register multiple implementations of the same +** functions with the same name but with either differing numbers of +** arguments or differing preferred text encodings. ^SQLite will use +** the implementation that most closely matches the way in which the +** SQL function is used. ^A function implementation with a non-negative +** nArg parameter is a better match than a function implementation with +** a negative nArg. ^A function where the preferred text encoding +** matches the database encoding is a better +** match than a function where the encoding is different. +** ^A function where the encoding difference is between UTF16le and UTF16be +** is a closer match than a function where the encoding difference is +** between UTF8 and UTF16. +** +** ^Built-in functions may be overloaded by new application-defined functions. +** +** ^An application-defined function is permitted to call other +** SQLite interfaces. However, such calls must not +** close the database connection nor finalize or reset the prepared +** statement in which the function is running. +*/ +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) +); + +/* +** CAPI3REF: Text Encodings +** +** These constant define integer codes that represent the various +** text encodings supported by SQLite. +*/ +#define SQLITE_UTF8 1 +#define SQLITE_UTF16LE 2 +#define SQLITE_UTF16BE 3 +#define SQLITE_UTF16 4 /* Use native byte order */ +#define SQLITE_ANY 5 /* Deprecated */ +#define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ + +/* +** CAPI3REF: Function Flags +** +** These constants may be ORed together with the +** [SQLITE_UTF8 | preferred text encoding] as the fourth argument +** to [sqlite3_create_function()], [sqlite3_create_function16()], or +** [sqlite3_create_function_v2()]. +*/ +#define SQLITE_DETERMINISTIC 0x800 + +/* +** CAPI3REF: Deprecated Functions +** DEPRECATED +** +** These functions are [deprecated]. In order to maintain +** backwards compatibility with older code, these functions continue +** to be supported. However, new applications should avoid +** the use of these functions. To help encourage people to avoid +** using these functions, we are not going to tell you what they do. +*/ +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); +SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); +#endif + +/* +** CAPI3REF: Obtaining SQL Function Parameter Values +** +** The C-language implementation of SQL functions and aggregates uses +** this set of interface routines to access the parameter values on +** the function or aggregate. +** +** The xFunc (for scalar functions) or xStep (for aggregates) parameters +** to [sqlite3_create_function()] and [sqlite3_create_function16()] +** define callbacks that implement the SQL functions and aggregates. +** The 3rd parameter to these callbacks is an array of pointers to +** [protected sqlite3_value] objects. There is one [sqlite3_value] object for +** each parameter to the SQL function. These routines are used to +** extract values from the [sqlite3_value] objects. +** +** These routines work only with [protected sqlite3_value] objects. +** Any attempt to use these routines on an [unprotected sqlite3_value] +** object results in undefined behavior. +** +** ^These routines work just like the corresponding [column access functions] +** except that these routines take a single [protected sqlite3_value] object +** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. +** +** ^The sqlite3_value_text16() interface extracts a UTF-16 string +** in the native byte-order of the host machine. ^The +** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces +** extract UTF-16 strings as big-endian and little-endian respectively. +** +** ^(The sqlite3_value_numeric_type() interface attempts to apply +** numeric affinity to the value. This means that an attempt is +** made to convert the value to an integer or floating point. If +** such a conversion is possible without loss of information (in other +** words, if the value is a string that looks like a number) +** then the conversion is performed. Otherwise no conversion occurs. +** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ +** +** Please pay particular attention to the fact that the pointer returned +** from [sqlite3_value_blob()], [sqlite3_value_text()], or +** [sqlite3_value_text16()] can be invalidated by a subsequent call to +** [sqlite3_value_bytes()], [sqlite3_value_bytes16()], [sqlite3_value_text()], +** or [sqlite3_value_text16()]. +** +** These routines must be called from the same thread as +** the SQL function that supplied the [sqlite3_value*] parameters. +*/ +SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); +SQLITE_API double sqlite3_value_double(sqlite3_value*); +SQLITE_API int sqlite3_value_int(sqlite3_value*); +SQLITE_API sqlite3_int64 sqlite3_value_int64(sqlite3_value*); +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); +SQLITE_API int sqlite3_value_type(sqlite3_value*); +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); + +/* +** CAPI3REF: Obtain Aggregate Function Context +** +** Implementations of aggregate SQL functions use this +** routine to allocate memory for storing their state. +** +** ^The first time the sqlite3_aggregate_context(C,N) routine is called +** for a particular aggregate function, SQLite +** allocates N of memory, zeroes out that memory, and returns a pointer +** to the new memory. ^On second and subsequent calls to +** sqlite3_aggregate_context() for the same aggregate function instance, +** the same buffer is returned. Sqlite3_aggregate_context() is normally +** called once for each invocation of the xStep callback and then one +** last time when the xFinal callback is invoked. ^(When no rows match +** an aggregate query, the xStep() callback of the aggregate function +** implementation is never called and xFinal() is called exactly once. +** In those cases, sqlite3_aggregate_context() might be called for the +** first time from within xFinal().)^ +** +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. +** +** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is +** determined by the N parameter on first successful call. Changing the +** value of N in subsequent call to sqlite3_aggregate_context() within +** the same aggregate function instance will not resize the memory +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. +** +** ^SQLite automatically frees the memory allocated by +** sqlite3_aggregate_context() when the aggregate query concludes. +** +** The first parameter must be a copy of the +** [sqlite3_context | SQL function context] that is the first parameter +** to the xStep or xFinal callback routine that implements the aggregate +** function. +** +** This routine must be called from the same thread in which +** the aggregate SQL function is running. +*/ +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); + +/* +** CAPI3REF: User Data For Functions +** +** ^The sqlite3_user_data() interface returns a copy of +** the pointer that was the pUserData parameter (the 5th parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +** +** This routine must be called from the same thread in which +** the application-defined function is running. +*/ +SQLITE_API void *sqlite3_user_data(sqlite3_context*); + +/* +** CAPI3REF: Database Connection For Functions +** +** ^The sqlite3_context_db_handle() interface returns a copy of +** the pointer to the [database connection] (the 1st parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +*/ +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); + +/* +** CAPI3REF: Function Auxiliary Data +** +** These functions may be used by (non-aggregate) SQL functions to +** associate metadata with argument values. If the same value is passed to +** multiple invocations of the same SQL function during query execution, under +** some circumstances the associated metadata may be preserved. An example +** of where this might be useful is in a regular-expression matching +** function. The compiled version of the regular expression can be stored as +** metadata associated with the pattern string. +** Then as long as the pattern string remains the same, +** the compiled regular expression can be reused on multiple +** invocations of the same function. +** +** ^The sqlite3_get_auxdata() interface returns a pointer to the metadata +** associated by the sqlite3_set_auxdata() function with the Nth argument +** value to the application-defined function. ^If there is no metadata +** associated with the function argument, this sqlite3_get_auxdata() interface +** returns a NULL pointer. +** +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th +** argument of the application-defined function. ^Subsequent +** calls to sqlite3_get_auxdata(C,N) return P from the most recent +** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or +** NULL if the metadata has been discarded. +** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, +** SQLite will invoke the destructor function X with parameter P exactly +** once, when the metadata is discarded. +** SQLite is free to discard the metadata at any time, including:
    +**
  • when the corresponding function parameter changes, or +**
  • when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement, or +**
  • when sqlite3_set_auxdata() is invoked again on the same parameter, or +**
  • during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.
)^ +** +** Note the last bullet in particular. The destructor X in +** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the +** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() +** should be called near the end of the function implementation and the +** function implementation should not make any use of P after +** sqlite3_set_auxdata() has been called. +** +** ^(In practice, metadata is preserved between function calls for +** function parameters that are compile-time constants, including literal +** values and [parameters] and expressions composed from the same.)^ +** +** These routines must be called from the same thread in which +** the SQL function is running. +*/ +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); +SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); + + +/* +** CAPI3REF: Constants Defining Special Destructor Behavior +** +** These are special values for the destructor that is passed in as the +** final argument to routines like [sqlite3_result_blob()]. ^If the destructor +** argument is SQLITE_STATIC, it means that the content pointer is constant +** and will never change. It does not need to be destroyed. ^The +** SQLITE_TRANSIENT value means that the content will likely change in +** the near future and that SQLite should make its own private copy of +** the content before returning. +** +** The typedef is necessary to work around problems in certain +** C++ compilers. +*/ +typedef void (*sqlite3_destructor_type)(void*); +#define SQLITE_STATIC ((sqlite3_destructor_type)0) +#define SQLITE_TRANSIENT ((sqlite3_destructor_type)-1) + +/* +** CAPI3REF: Setting The Result Of An SQL Function +** +** These routines are used by the xFunc or xFinal callbacks that +** implement SQL functions and aggregates. See +** [sqlite3_create_function()] and [sqlite3_create_function16()] +** for additional information. +** +** These functions work very much like the [parameter binding] family of +** functions used to bind values to host parameters in prepared statements. +** Refer to the [SQL parameter] documentation for additional information. +** +** ^The sqlite3_result_blob() interface sets the result from +** an application-defined function to be the BLOB whose content is pointed +** to by the second parameter and which is N bytes long where N is the +** third parameter. +** +** ^The sqlite3_result_zeroblob() interfaces set the result of +** the application-defined function to be a BLOB containing all zero +** bytes and N bytes in size, where N is the value of the 2nd parameter. +** +** ^The sqlite3_result_double() interface sets the result from +** an application-defined function to be a floating point value specified +** by its 2nd argument. +** +** ^The sqlite3_result_error() and sqlite3_result_error16() functions +** cause the implemented SQL function to throw an exception. +** ^SQLite uses the string pointed to by the +** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() +** as the text of an error message. ^SQLite interprets the error +** message string from sqlite3_result_error() as UTF-8. ^SQLite +** interprets the string from sqlite3_result_error16() as UTF-16 in native +** byte order. ^If the third parameter to sqlite3_result_error() +** or sqlite3_result_error16() is negative then SQLite takes as the error +** message all text up through the first zero character. +** ^If the third parameter to sqlite3_result_error() or +** sqlite3_result_error16() is non-negative then SQLite takes that many +** bytes (not characters) from the 2nd parameter as the error message. +** ^The sqlite3_result_error() and sqlite3_result_error16() +** routines make a private copy of the error message text before +** they return. Hence, the calling function can deallocate or +** modify the text after they return without harm. +** ^The sqlite3_result_error_code() function changes the error code +** returned by SQLite as a result of an error in a function. ^By default, +** the error code is SQLITE_ERROR. ^A subsequent call to sqlite3_result_error() +** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. +** +** ^The sqlite3_result_error_toobig() interface causes SQLite to throw an +** error indicating that a string or BLOB is too long to represent. +** +** ^The sqlite3_result_error_nomem() interface causes SQLite to throw an +** error indicating that a memory allocation failed. +** +** ^The sqlite3_result_int() interface sets the return value +** of the application-defined function to be the 32-bit signed integer +** value given in the 2nd argument. +** ^The sqlite3_result_int64() interface sets the return value +** of the application-defined function to be the 64-bit signed integer +** value given in the 2nd argument. +** +** ^The sqlite3_result_null() interface sets the return value +** of the application-defined function to be NULL. +** +** ^The sqlite3_result_text(), sqlite3_result_text16(), +** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces +** set the return value of the application-defined function to be +** a text string which is represented as UTF-8, UTF-16 native byte order, +** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^SQLite takes the text result from the application from +** the 2nd parameter of the sqlite3_result_text* interfaces. +** ^If the 3rd parameter to the sqlite3_result_text* interfaces +** is negative, then SQLite takes result text from the 2nd parameter +** through the first zero character. +** ^If the 3rd parameter to the sqlite3_result_text* interfaces +** is non-negative, then as many bytes (not characters) of the text +** pointed to by the 2nd parameter are taken as the application-defined +** function result. If the 3rd parameter is non-negative, then it +** must be the byte offset into the string where the NUL terminator would +** appear if the string where NUL terminated. If any NUL characters occur +** in the string at a byte offset that is less than the value of the 3rd +** parameter, then the resulting string will contain embedded NULs and the +** result of expressions operating on strings with embedded NULs is undefined. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that +** function as the destructor on the text or BLOB result when it has +** finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces or to +** sqlite3_result_blob is the special constant SQLITE_STATIC, then SQLite +** assumes that the text or BLOB result is in constant space and does not +** copy the content of the parameter nor call a destructor on the content +** when it has finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is the special constant SQLITE_TRANSIENT +** then SQLite makes a copy of the result into space obtained from +** from [sqlite3_malloc()] before it returns. +** +** ^The sqlite3_result_value() interface sets the result of +** the application-defined function to be a copy the +** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The +** sqlite3_result_value() interface makes a copy of the [sqlite3_value] +** so that the [sqlite3_value] specified in the parameter may change or +** be deallocated after sqlite3_result_value() returns without harm. +** ^A [protected sqlite3_value] object may always be used where an +** [unprotected sqlite3_value] object is required, so either +** kind of [sqlite3_value] object can be used with this interface. +** +** If these routines are called from within the different thread +** than the one containing the application-defined function that received +** the [sqlite3_context] pointer, the results are undefined. +*/ +SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_double(sqlite3_context*, double); +SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); +SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context*); +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context*); +SQLITE_API void sqlite3_result_error_code(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); +SQLITE_API void sqlite3_result_null(sqlite3_context*); +SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); + +/* +** CAPI3REF: Define New Collating Sequences +** +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. +** +** ^The name of the collation is a UTF-8 string +** for sqlite3_create_collation() and sqlite3_create_collation_v2() +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. +** +** ^(The third argument (eTextRep) must be one of the constants: +**
    +**
  • [SQLITE_UTF8], +**
  • [SQLITE_UTF16LE], +**
  • [SQLITE_UTF16BE], +**
  • [SQLITE_UTF16], or +**
  • [SQLITE_UTF16_ALIGNED]. +**
)^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCallback. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. +** +** ^The fourth argument, pArg, is an application data pointer that is passed +** through as the first argument to the collating function callback. +** +** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCallback argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The collating function must return an +** integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must always return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
    +**
  1. If A==B then B==A. +**
  2. If A==B and B==C then A==C. +**
  3. If A<B THEN B>A. +**
  4. If A<B and B<C then A<C. +**
+** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. +** +** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. +** +** ^The xDestroy callback is not called if the +** sqlite3_create_collation_v2() function fails. Applications that invoke +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** check the return code and dispose of the application data pointer +** themselves rather than expecting SQLite to deal with it for them. +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards +** compatibility. +** +** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. +*/ +SQLITE_API int sqlite3_create_collation( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*) +); +SQLITE_API int sqlite3_create_collation_v2( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDestroy)(void*) +); +SQLITE_API int sqlite3_create_collation16( + sqlite3*, + const void *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*) +); + +/* +** CAPI3REF: Collation Needed Callbacks +** +** ^To avoid having to register all collation sequences before a database +** can be used, a single callback function may be registered with the +** [database connection] to be invoked whenever an undefined collation +** sequence is required. +** +** ^If the function is registered using the sqlite3_collation_needed() API, +** then it is passed the names of undefined collation sequences as strings +** encoded in UTF-8. ^If sqlite3_collation_needed16() is used, +** the names are passed as UTF-16 in machine native byte order. +** ^A call to either function replaces the existing collation-needed callback. +** +** ^(When the callback is invoked, the first argument passed is a copy +** of the second argument to sqlite3_collation_needed() or +** sqlite3_collation_needed16(). The second argument is the database +** connection. The third argument is one of [SQLITE_UTF8], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE], indicating the most desirable form of the collation +** sequence function required. The fourth parameter is the name of the +** required collation sequence.)^ +** +** The callback function should register the desired collation using +** [sqlite3_create_collation()], [sqlite3_create_collation16()], or +** [sqlite3_create_collation_v2()]. +*/ +SQLITE_API int sqlite3_collation_needed( + sqlite3*, + void*, + void(*)(void*,sqlite3*,int eTextRep,const char*) +); +SQLITE_API int sqlite3_collation_needed16( + sqlite3*, + void*, + void(*)(void*,sqlite3*,int eTextRep,const void*) +); + +#ifdef SQLITE_HAS_CODEC +/* +** Specify the key for an encrypted database. This routine should be +** called right after sqlite3_open(). +** +** The code to implement this API is not available in the public release +** of SQLite. +*/ +SQLITE_API int sqlite3_key( + sqlite3 *db, /* Database to be rekeyed */ + const void *pKey, int nKey /* The key */ +); +SQLITE_API int sqlite3_key_v2( + sqlite3 *db, /* Database to be rekeyed */ + const char *zDbName, /* Name of the database */ + const void *pKey, int nKey /* The key */ +); + +/* +** Change the key on an open database. If the current database is not +** encrypted, this routine will encrypt it. If pNew==0 or nNew==0, the +** database is decrypted. +** +** The code to implement this API is not available in the public release +** of SQLite. +*/ +SQLITE_API int sqlite3_rekey( + sqlite3 *db, /* Database to be rekeyed */ + const void *pKey, int nKey /* The new key */ +); +SQLITE_API int sqlite3_rekey_v2( + sqlite3 *db, /* Database to be rekeyed */ + const char *zDbName, /* Name of the database */ + const void *pKey, int nKey /* The new key */ +); + +/* +** Specify the activation key for a SEE database. Unless +** activated, none of the SEE routines will work. +*/ +SQLITE_API void sqlite3_activate_see( + const char *zPassPhrase /* Activation phrase */ +); +#endif + +#ifdef SQLITE_ENABLE_CEROD +/* +** Specify the activation key for a CEROD database. Unless +** activated, none of the CEROD routines will work. +*/ +SQLITE_API void sqlite3_activate_cerod( + const char *zPassPhrase /* Activation phrase */ +); +#endif + +/* +** CAPI3REF: Suspend Execution For A Short Time +** +** The sqlite3_sleep() function causes the current thread to suspend execution +** for at least a number of milliseconds specified in its parameter. +** +** If the operating system does not support sleep requests with +** millisecond time resolution, then the time will be rounded up to +** the nearest second. The number of milliseconds of sleep actually +** requested from the operating system is returned. +** +** ^SQLite implements this interface by calling the xSleep() +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. +*/ +SQLITE_API int sqlite3_sleep(int); + +/* +** CAPI3REF: Name Of The Folder Holding Temporary Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all temporary files +** created by SQLite when using a built-in [sqlite3_vfs | VFS] +** will be placed in that directory.)^ ^If this variable +** is a NULL pointer, then SQLite performs a search for an appropriate +** temporary file directory. +** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [temp_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [temp_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various +** features that require the use of temporary files may fail. Here is an +** example of how to do this using C++ with the Windows Runtime: +** +** 
+** LPCWSTR zPath = Windows::Storage::ApplicationData::Current->
+**       TemporaryFolder->Path->Data();
+** char zPathBuf[MAX_PATH + 1];
+** memset(zPathBuf, 0, sizeof(zPathBuf));
+** WideCharToMultiByte(CP_UTF8, 0, zPath, -1, zPathBuf, sizeof(zPathBuf),
+**       NULL, NULL);
+** sqlite3_temp_directory = sqlite3_mprintf("%s", zPathBuf);
+**
+*/ +SQLITE_API char *sqlite3_temp_directory; + +/* +** CAPI3REF: Name Of The Folder Holding Database Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all database files +** specified with a relative pathname and created or accessed by +** SQLite when using a built-in windows [sqlite3_vfs | VFS] will be assumed +** to be relative to that directory.)^ ^If this variable is a NULL +** pointer, then SQLite assumes that all database files specified +** with a relative pathname are relative to the current directory +** for the process. Only the windows VFS makes use of this global +** variable; it is ignored by the unix VFS. +** +** Changing the value of this variable while a database connection is +** open can result in a corrupt database. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [data_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [data_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [data_store_directory pragma] should be avoided. +*/ +SQLITE_API char *sqlite3_data_directory; + +/* +** CAPI3REF: Test For Auto-Commit Mode +** KEYWORDS: {autocommit mode} +** +** ^The sqlite3_get_autocommit() interface returns non-zero or +** zero if the given database connection is or is not in autocommit mode, +** respectively. ^Autocommit mode is on by default. +** ^Autocommit mode is disabled by a [BEGIN] statement. +** ^Autocommit mode is re-enabled by a [COMMIT] or [ROLLBACK]. +** +** If certain kinds of errors occur on a statement within a multi-statement +** transaction (errors including [SQLITE_FULL], [SQLITE_IOERR], +** [SQLITE_NOMEM], [SQLITE_BUSY], and [SQLITE_INTERRUPT]) then the +** transaction might be rolled back automatically. The only way to +** find out whether SQLite automatically rolled back the transaction after +** an error is to use this function. +** +** If another thread changes the autocommit status of the database +** connection while this routine is running, then the return value +** is undefined. +*/ +SQLITE_API int sqlite3_get_autocommit(sqlite3*); + +/* +** CAPI3REF: Find The Database Handle Of A Prepared Statement +** +** ^The sqlite3_db_handle interface returns the [database connection] handle +** to which a [prepared statement] belongs. ^The [database connection] +** returned by sqlite3_db_handle is the same [database connection] +** that was the first argument +** to the [sqlite3_prepare_v2()] call (or its variants) that was used to +** create the statement in the first place. +*/ +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); + +/* +** CAPI3REF: Return The Filename For A Database Connection +** +** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename +** associated with database N of connection D. ^The main database file +** has the name "main". If there is no attached database N on the database +** connection D, or if database N is a temporary or in-memory database, then +** a NULL pointer is returned. +** +** ^The filename returned by this function is the output of the +** xFullPathname method of the [VFS]. ^In other words, the filename +** will be an absolute pathname, even if the filename used +** to open the database originally was a URI or relative pathname. +*/ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine if a database is read-only +** +** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N +** of connection D is read-only, 0 if it is read/write, or -1 if N is not +** the name of a database on connection D. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Find the next prepared statement +** +** ^This interface returns a pointer to the next [prepared statement] after +** pStmt associated with the [database connection] pDb. ^If pStmt is NULL +** then this interface returns a pointer to the first prepared statement +** associated with the database connection pDb. ^If no prepared statement +** satisfies the conditions of this routine, it returns NULL. +** +** The [database connection] pointer D in a call to +** [sqlite3_next_stmt(D,S)] must refer to an open database +** connection and in particular must not be a NULL pointer. +*/ +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Commit And Rollback Notification Callbacks +** +** ^The sqlite3_commit_hook() interface registers a callback +** function to be invoked whenever a transaction is [COMMIT | committed]. +** ^Any callback set by a previous call to sqlite3_commit_hook() +** for the same database connection is overridden. +** ^The sqlite3_rollback_hook() interface registers a callback +** function to be invoked whenever a transaction is [ROLLBACK | rolled back]. +** ^Any callback set by a previous call to sqlite3_rollback_hook() +** for the same database connection is overridden. +** ^The pArg argument is passed through to the callback. +** ^If the callback on a commit hook function returns non-zero, +** then the commit is converted into a rollback. +** +** ^The sqlite3_commit_hook(D,C,P) and sqlite3_rollback_hook(D,C,P) functions +** return the P argument from the previous call of the same function +** on the same [database connection] D, or NULL for +** the first call for each function on D. +** +** The commit and rollback hook callbacks are not reentrant. +** The callback implementation must not do anything that will modify +** the database connection that invoked the callback. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the commit +** or rollback hook in the first place. +** Note that running any other SQL statements, including SELECT statements, +** or merely calling [sqlite3_prepare_v2()] and [sqlite3_step()] will modify +** the database connections for the meaning of "modify" in this paragraph. +** +** ^Registering a NULL function disables the callback. +** +** ^When the commit hook callback routine returns zero, the [COMMIT] +** operation is allowed to continue normally. ^If the commit hook +** returns non-zero, then the [COMMIT] is converted into a [ROLLBACK]. +** ^The rollback hook is invoked on a rollback that results from a commit +** hook returning non-zero, just as it would be with any other rollback. +** +** ^For the purposes of this API, a transaction is said to have been +** rolled back if an explicit "ROLLBACK" statement is executed, or +** an error or constraint causes an implicit rollback to occur. +** ^The rollback callback is not invoked if a transaction is +** automatically rolled back because the database connection is closed. +** +** See also the [sqlite3_update_hook()] interface. +*/ +SQLITE_API void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); +SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); + +/* +** CAPI3REF: Data Change Notification Callbacks +** +** ^The sqlite3_update_hook() interface registers a callback function +** with the [database connection] identified by the first argument +** to be invoked whenever a row is updated, inserted or deleted in +** a rowid table. +** ^Any callback set by a previous call to this function +** for the same database connection is overridden. +** +** ^The second argument is a pointer to the function to invoke when a +** row is updated, inserted or deleted in a rowid table. +** ^The first argument to the callback is a copy of the third argument +** to sqlite3_update_hook(). +** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], +** or [SQLITE_UPDATE], depending on the operation that caused the callback +** to be invoked. +** ^The third and fourth arguments to the callback contain pointers to the +** database and table name containing the affected row. +** ^The final callback parameter is the [rowid] of the row. +** ^In the case of an update, this is the [rowid] after the update takes place. +** +** ^(The update hook is not invoked when internal system tables are +** modified (i.e. sqlite_master and sqlite_sequence).)^ +** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. +** +** ^In the current implementation, the update hook +** is not invoked when duplication rows are deleted because of an +** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook +** invoked when rows are deleted using the [truncate optimization]. +** The exceptions defined in this paragraph might change in a future +** release of SQLite. +** +** The update hook implementation must not do anything that will modify +** the database connection that invoked the update hook. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the update hook. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^The sqlite3_update_hook(D,C,P) function +** returns the P argument from the previous call +** on the same [database connection] D, or NULL for +** the first call on D. +** +** See also the [sqlite3_commit_hook()] and [sqlite3_rollback_hook()] +** interfaces. +*/ +SQLITE_API void *sqlite3_update_hook( + sqlite3*, + void(*)(void *,int ,char const *,char const *,sqlite3_int64), + void* +); + +/* +** CAPI3REF: Enable Or Disable Shared Pager Cache +** +** ^(This routine enables or disables the sharing of the database cache +** and schema data structures between [database connection | connections] +** to the same database. Sharing is enabled if the argument is true +** and disabled if the argument is false.)^ +** +** ^Cache sharing is enabled and disabled for an entire process. +** This is a change as of SQLite version 3.5.0. In prior versions of SQLite, +** sharing was enabled or disabled for each thread separately. +** +** ^(The cache sharing mode set by this interface effects all subsequent +** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. +** Existing database connections continue use the sharing mode +** that was in effect at the time they were opened.)^ +** +** ^(This routine returns [SQLITE_OK] if shared cache was enabled or disabled +** successfully. An [error code] is returned otherwise.)^ +** +** ^Shared cache is disabled by default. But this might change in +** future releases of SQLite. Applications that care about shared +** cache setting should set it explicitly. +** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** +** See Also: [SQLite Shared-Cache Mode] +*/ +SQLITE_API int sqlite3_enable_shared_cache(int); + +/* +** CAPI3REF: Attempt To Free Heap Memory +** +** ^The sqlite3_release_memory() interface attempts to free N bytes +** of heap memory by deallocating non-essential memory allocations +** held by the database library. Memory used to cache database +** pages to improve performance is an example of non-essential memory. +** ^sqlite3_release_memory() returns the number of bytes actually freed, +** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** See also: [sqlite3_db_release_memory()] +*/ +SQLITE_API int sqlite3_release_memory(int); + +/* +** CAPI3REF: Free Memory Used By A Database Connection +** +** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap +** memory as possible from database connection D. Unlike the +** [sqlite3_release_memory()] interface, this interface is in effect even +** when the [SQLITE_ENABLE_MEMORY_MANAGEMENT] compile-time option is +** omitted. +** +** See also: [sqlite3_release_memory()] +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3*); + +/* +** CAPI3REF: Impose A Limit On Heap Size +** +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. +** +** ^The return value from sqlite3_soft_heap_limit64() is the size of +** the soft heap limit prior to the call, or negative in the case of an +** error. ^If the argument N is negative +** then no change is made to the soft heap limit. Hence, the current +** size of the soft heap limit can be determined by invoking +** sqlite3_soft_heap_limit64() with a negative argument. +** +** ^If the argument N is zero then the soft heap limit is disabled. +** +** ^(The soft heap limit is not enforced in the current implementation +** if one or more of following conditions are true: +** +**
    +**
  • The soft heap limit is set to zero. +**
  • Memory accounting is disabled using a combination of the +** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and +** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. +**
  • An alternative page cache implementation is specified using +** [sqlite3_config]([SQLITE_CONFIG_PCACHE2],...). +**
  • The page cache allocates from its own memory pool supplied +** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than +** from the heap. +**
)^ +** +** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] +** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], +** the soft heap limit is enforced on every memory allocation. Without +** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced +** when memory is allocated by the page cache. Testing suggests that because +** the page cache is the predominate memory user in SQLite, most +** applications will achieve adequate soft heap limit enforcement without +** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** The circumstances under which SQLite will enforce the soft heap limit may +** changes in future releases of SQLite. +*/ +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + + +/* +** CAPI3REF: Extract Metadata About A Column Of A Table +** +** ^This routine returns metadata about a specific column of a specific +** database table accessible using the [database connection] handle +** passed as the first function argument. +** +** ^The column is identified by the second, third and fourth parameters to +** this function. ^The second parameter is either the name of the database +** (i.e. "main", "temp", or an attached database) containing the specified +** table or NULL. ^If it is NULL, then all attached databases are searched +** for the table using the same algorithm used by the database engine to +** resolve unqualified table references. +** +** ^The third and fourth parameters to this function are the table and column +** name of the desired column, respectively. Neither of these parameters +** may be NULL. +** +** ^Metadata is returned by writing to the memory locations passed as the 5th +** and subsequent parameters to this function. ^Any of these arguments may be +** NULL, in which case the corresponding element of metadata is omitted. +** +** ^(
+** +**
Parameter Output
Type 		Description +** +**
5th const char* Data type +**
6th const char* Name of default collation sequence +**
7th int True if column has a NOT NULL constraint +**
8th int True if column is part of the PRIMARY KEY +**
9th int True if column is [AUTOINCREMENT] +**
+**
)^ +** +** ^The memory pointed to by the character pointers returned for the +** declaration type and collation sequence is valid only until the next +** call to any SQLite API function. +** +** ^If the specified table is actually a view, an [error code] is returned. +** +** ^If the specified column is "rowid", "oid" or "_rowid_" and an +** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output +** parameters are set for the explicitly declared column. ^(If there is no +** explicitly declared [INTEGER PRIMARY KEY] column, then the output +** parameters are set as follows: +** +** 
+**     data type: "INTEGER"
+**     collation sequence: "BINARY"
+**     not null: 0
+**     primary key: 1
+**     auto increment: 0
+**
)^ +** +** ^(This function may load one or more schemas from database files. If an +** error occurs during this process, or if the requested table or column +** cannot be found, an [error code] is returned and an error message left +** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^ +** +** ^This API is only available if the library was compiled with the +** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined. +*/ +SQLITE_API int sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +); + +/* +** CAPI3REF: Load An Extension +** +** ^This interface loads an SQLite extension library from the named file. +** +** ^The sqlite3_load_extension() interface attempts to load an +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. +** +** ^The entry point is zProc. +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ +** ^The sqlite3_load_extension() interface returns +** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. +** ^If an error occurs and pzErrMsg is not 0, then the +** [sqlite3_load_extension()] interface shall attempt to +** fill *pzErrMsg with error message text stored in memory +** obtained from [sqlite3_malloc()]. The calling function +** should free this memory by calling [sqlite3_free()]. +** +** ^Extension loading must be enabled using +** [sqlite3_enable_load_extension()] prior to calling this API, +** otherwise an error will be returned. +** +** See also the [load_extension() SQL function]. +*/ +SQLITE_API int sqlite3_load_extension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Derived from zFile if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +); + +/* +** CAPI3REF: Enable Or Disable Extension Loading +** +** ^So as not to open security holes in older applications that are +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API +** is provided to turn the [sqlite3_load_extension()] mechanism on and off. +** +** ^Extension loading is off by default. +** ^Call the sqlite3_enable_load_extension() routine with onoff==1 +** to turn extension loading on and call it with onoff==0 to turn +** it back off again. +*/ +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); + +/* +** CAPI3REF: Automatically Load Statically Linked Extensions +** +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked [SQLite extension] +** that is to be automatically loaded into all new database connections. +** +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects and integer result as if the signature of the +** entry point where as follows: +** +** 
+**    int xEntryPoint(
+**      sqlite3 *db,
+**      const char **pzErrMsg,
+**      const struct sqlite3_api_routines *pThunk
+**    );
+**
)^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()] +** and [sqlite3_cancel_auto_extension()] +*/ +SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); + +/* +** CAPI3REF: Cancel Automatic Extension Loading +** +** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the +** initialization routine X that was registered using a prior call to +** [sqlite3_auto_extension(X)]. ^The [sqlite3_cancel_auto_extension(X)] +** routine returns 1 if initialization routine X was successfully +** unregistered and it returns 0 if X was not on the list of initialization +** routines. +*/ +SQLITE_API int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); + +/* +** CAPI3REF: Reset Automatic Extension Loading +** +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. +*/ +SQLITE_API void sqlite3_reset_auto_extension(void); + +/* +** The interface to the virtual-table mechanism is currently considered +** to be experimental. The interface might change in incompatible ways. +** If this is a problem for you, do not use the interface at this time. +** +** When the virtual-table mechanism stabilizes, we will declare the +** interface fixed, support it indefinitely, and remove this comment. +*/ + +/* +** Structures used by the virtual table interface +*/ +typedef struct sqlite3_vtab sqlite3_vtab; +typedef struct sqlite3_index_info sqlite3_index_info; +typedef struct sqlite3_vtab_cursor sqlite3_vtab_cursor; +typedef struct sqlite3_module sqlite3_module; + +/* +** CAPI3REF: Virtual Table Object +** KEYWORDS: sqlite3_module {virtual table module} +** +** This structure, sometimes called a "virtual table module", +** defines the implementation of a [virtual tables]. +** This structure consists mostly of methods for the module. +** +** ^A virtual table module is created by filling in a persistent +** instance of this structure and passing a pointer to that instance +** to [sqlite3_create_module()] or [sqlite3_create_module_v2()]. +** ^The registration remains valid until it is replaced by a different +** module or until the [database connection] closes. The content +** of this structure must not change while it is registered with +** any database connection. +*/ +struct sqlite3_module { + int iVersion; + int (*xCreate)(sqlite3*, void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, char**); + int (*xConnect)(sqlite3*, void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, char**); + int (*xBestIndex)(sqlite3_vtab *pVTab, sqlite3_index_info*); + int (*xDisconnect)(sqlite3_vtab *pVTab); + int (*xDestroy)(sqlite3_vtab *pVTab); + int (*xOpen)(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor); + int (*xClose)(sqlite3_vtab_cursor*); + int (*xFilter)(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); + int (*xNext)(sqlite3_vtab_cursor*); + int (*xEof)(sqlite3_vtab_cursor*); + int (*xColumn)(sqlite3_vtab_cursor*, sqlite3_context*, int); + int (*xRowid)(sqlite3_vtab_cursor*, sqlite3_int64 *pRowid); + int (*xUpdate)(sqlite3_vtab *, int, sqlite3_value **, sqlite3_int64 *); + int (*xBegin)(sqlite3_vtab *pVTab); + int (*xSync)(sqlite3_vtab *pVTab); + int (*xCommit)(sqlite3_vtab *pVTab); + int (*xRollback)(sqlite3_vtab *pVTab); + int (*xFindFunction)(sqlite3_vtab *pVtab, int nArg, const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg); + int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 1 of the sqlite_module object. Those + ** below are for version 2 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); +}; + +/* +** CAPI3REF: Virtual Table Indexing Information +** KEYWORDS: sqlite3_index_info +** +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to +** pass information into and receive the reply from the [xBestIndex] +** method of a [virtual table module]. The fields under **Inputs** are the +** inputs to xBestIndex and are read-only. xBestIndex inserts its +** results into the **Outputs** fields. +** +** ^(The aConstraint[] array records WHERE clause constraints of the form: +** +**
column OP expr
+** +** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in +** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the +** expr on the right-hand side can be evaluated (and thus the constraint +** is usable) and false if it cannot.)^ +** +** ^The optimizer automatically inverts terms of the form "expr OP column" +** and makes other simplifications to the WHERE clause in an attempt to +** get as many WHERE clause terms into the form shown above as possible. +** ^The aConstraint[] array only reports WHERE clause terms that are +** relevant to the particular virtual table being queried. +** +** ^Information about the ORDER BY clause is stored in aOrderBy[]. +** ^Each term of aOrderBy records a column of the ORDER BY clause. +** +** The [xBestIndex] method must fill aConstraintUsage[] with information +** about what parameters to pass to xFilter. ^If argvIndex>0 then +** the right-hand side of the corresponding aConstraint[] is evaluated +** and becomes the argvIndex-th entry in argv. ^(If aConstraintUsage[].omit +** is true, then the constraint is assumed to be fully handled by the +** virtual table and is not checked again by SQLite.)^ +** +** ^The idxNum and idxPtr values are recorded and passed into the +** [xFilter] method. +** ^[sqlite3_free()] is used to free idxPtr if and only if +** needToFreeIdxPtr is true. +** +** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in +** the correct order to satisfy the ORDER BY clause so that no separate +** sorting step is required. +** +** ^The estimatedCost value is an estimate of the cost of a particular +** strategy. A cost of N indicates that the cost of the strategy is similar +** to a linear scan of an SQLite table with N rows. A cost of log(N) +** indicates that the expense of the operation is similar to that of a +** binary search on a unique indexed field of an SQLite table with N rows. +** +** ^The estimatedRows value is an estimate of the number of rows that +** will be returned by the strategy. +** +** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info +** structure for SQLite version 3.8.2. If a virtual table extension is +** used with an SQLite version earlier than 3.8.2, the results of attempting +** to read or write the estimatedRows field are undefined (but are likely +** to included crashing the application). The estimatedRows field should +** therefore only be used if [sqlite3_libversion_number()] returns a +** value greater than or equal to 3008002. +*/ +struct sqlite3_index_info { + /* Inputs */ + int nConstraint; /* Number of entries in aConstraint */ + struct sqlite3_index_constraint { + int iColumn; /* Column on left-hand side of constraint */ + unsigned char op; /* Constraint operator */ + unsigned char usable; /* True if this constraint is usable */ + int iTermOffset; /* Used internally - xBestIndex should ignore */ + } *aConstraint; /* Table of WHERE clause constraints */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + struct sqlite3_index_orderby { + int iColumn; /* Column number */ + unsigned char desc; /* True for DESC. False for ASC. */ + } *aOrderBy; /* The ORDER BY clause */ + /* Outputs */ + struct sqlite3_index_constraint_usage { + int argvIndex; /* if >0, constraint is part of argv to xFilter */ + unsigned char omit; /* Do not code a test for this constraint */ + } *aConstraintUsage; + int idxNum; /* Number used to identify the index */ + char *idxStr; /* String, possibly obtained from sqlite3_malloc */ + int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ + int orderByConsumed; /* True if output is already ordered */ + double estimatedCost; /* Estimated cost of using this index */ + /* Fields below are only available in SQLite 3.8.2 and later */ + sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ +}; + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros defined the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the wHERE clause of +** a query that uses a [virtual table]. +*/ +#define SQLITE_INDEX_CONSTRAINT_EQ 2 +#define SQLITE_INDEX_CONSTRAINT_GT 4 +#define SQLITE_INDEX_CONSTRAINT_LE 8 +#define SQLITE_INDEX_CONSTRAINT_LT 16 +#define SQLITE_INDEX_CONSTRAINT_GE 32 +#define SQLITE_INDEX_CONSTRAINT_MATCH 64 + +/* +** CAPI3REF: Register A Virtual Table Implementation +** +** ^These routines are used to register a new [virtual table module] name. +** ^Module names must be registered before +** creating a new [virtual table] using the module and before using a +** preexisting [virtual table] for the module. +** +** ^The module name is registered on the [database connection] specified +** by the first parameter. ^The name of the module is given by the +** second parameter. ^The third parameter is a pointer to +** the implementation of the [virtual table module]. ^The fourth +** parameter is an arbitrary client data pointer that is passed through +** into the [xCreate] and [xConnect] methods of the virtual table module +** when a new virtual table is be being created or reinitialized. +** +** ^The sqlite3_create_module_v2() interface has a fifth parameter which +** is a pointer to a destructor for the pClientData. ^SQLite will +** invoke the destructor function (if it is not NULL) when SQLite +** no longer needs the pClientData pointer. ^The destructor will also +** be invoked if the call to sqlite3_create_module_v2() fails. +** ^The sqlite3_create_module() +** interface is equivalent to sqlite3_create_module_v2() with a NULL +** destructor. +*/ +SQLITE_API int sqlite3_create_module( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData /* Client data for xCreate/xConnect */ +); +SQLITE_API int sqlite3_create_module_v2( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData, /* Client data for xCreate/xConnect */ + void(*xDestroy)(void*) /* Module destructor function */ +); + +/* +** CAPI3REF: Virtual Table Instance Object +** KEYWORDS: sqlite3_vtab +** +** Every [virtual table module] implementation uses a subclass +** of this object to describe a particular instance +** of the [virtual table]. Each subclass will +** be tailored to the specific needs of the module implementation. +** The purpose of this superclass is to define certain fields that are +** common to all module implementations. +** +** ^Virtual tables methods can set an error message by assigning a +** string obtained from [sqlite3_mprintf()] to zErrMsg. The method should +** take care that any prior string is freed by a call to [sqlite3_free()] +** prior to assigning a new string to zErrMsg. ^After the error message +** is delivered up to the client application, the string will be automatically +** freed by sqlite3_free() and the zErrMsg field will be zeroed. +*/ +struct sqlite3_vtab { + const sqlite3_module *pModule; /* The module for this virtual table */ + int nRef; /* NO LONGER USED */ + char *zErrMsg; /* Error message from sqlite3_mprintf() */ + /* Virtual table implementations will typically add additional fields */ +}; + +/* +** CAPI3REF: Virtual Table Cursor Object +** KEYWORDS: sqlite3_vtab_cursor {virtual table cursor} +** +** Every [virtual table module] implementation uses a subclass of the +** following structure to describe cursors that point into the +** [virtual table] and are used +** to loop through the virtual table. Cursors are created using the +** [sqlite3_module.xOpen | xOpen] method of the module and are destroyed +** by the [sqlite3_module.xClose | xClose] method. Cursors are used +** by the [xFilter], [xNext], [xEof], [xColumn], and [xRowid] methods +** of the module. Each module implementation will define +** the content of a cursor structure to suit its own needs. +** +** This superclass exists in order to define fields of the cursor that +** are common to all implementations. +*/ +struct sqlite3_vtab_cursor { + sqlite3_vtab *pVtab; /* Virtual table of this cursor */ + /* Virtual table implementations will typically add additional fields */ +}; + +/* +** CAPI3REF: Declare The Schema Of A Virtual Table +** +** ^The [xCreate] and [xConnect] methods of a +** [virtual table module] call this interface +** to declare the format (the names and datatypes of the columns) of +** the virtual tables they implement. +*/ +SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); + +/* +** CAPI3REF: Overload A Function For A Virtual Table +** +** ^(Virtual tables can provide alternative implementations of functions +** using the [xFindFunction] method of the [virtual table module]. +** But global versions of those functions +** must exist in order to be overloaded.)^ +** +** ^(This API makes sure a global version of a function with a particular +** name and number of parameters exists. If no such function exists +** before this API is called, a new function is created.)^ ^The implementation +** of the new function always causes an exception to be thrown. So +** the new function is not good for anything by itself. Its only +** purpose is to be a placeholder function that can be overloaded +** by a [virtual table]. +*/ +SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); + +/* +** The interface to the virtual-table mechanism defined above (back up +** to a comment remarkably similar to this one) is currently considered +** to be experimental. The interface might change in incompatible ways. +** If this is a problem for you, do not use the interface at this time. +** +** When the virtual-table mechanism stabilizes, we will declare the +** interface fixed, support it indefinitely, and remove this comment. +*/ + +/* +** CAPI3REF: A Handle To An Open BLOB +** KEYWORDS: {BLOB handle} {BLOB handles} +** +** An instance of this object represents an open BLOB on which +** [sqlite3_blob_open | incremental BLOB I/O] can be performed. +** ^Objects of this type are created by [sqlite3_blob_open()] +** and destroyed by [sqlite3_blob_close()]. +** ^The [sqlite3_blob_read()] and [sqlite3_blob_write()] interfaces +** can be used to read or write small subsections of the BLOB. +** ^The [sqlite3_blob_bytes()] interface returns the size of the BLOB in bytes. +*/ +typedef struct sqlite3_blob sqlite3_blob; + +/* +** CAPI3REF: Open A BLOB For Incremental I/O +** +** ^(This interfaces opens a [BLOB handle | handle] to the BLOB located +** in row iRow, column zColumn, table zTable in database zDb; +** in other words, the same BLOB that would be selected by: +** +** 
+**     SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow;
+**
)^ +** +** ^If the flags parameter is non-zero, then the BLOB is opened for read +** and write access. ^If it is zero, the BLOB is opened for read access. +** ^It is not possible to open a column that is part of an index or primary +** key for writing. ^If [foreign key constraints] are enabled, it is +** not possible to open a column that is part of a [child key] for writing. +** +** ^Note that the database name is not the filename that contains +** the database but rather the symbolic name of the database that +** appears after the AS keyword when the database is connected using [ATTACH]. +** ^For the main database file, the database name is "main". +** ^For TEMP tables, the database name is "temp". +** +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is written +** to *ppBlob. Otherwise an [error code] is returned and *ppBlob is set +** to be a null pointer.)^ +** ^This function sets the [database connection] error code and message +** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()] and related +** functions. ^Note that the *ppBlob variable is always initialized in a +** way that makes it safe to invoke [sqlite3_blob_close()] on *ppBlob +** regardless of the success or failure of this routine. +** +** ^(If the row that a BLOB handle points to is modified by an +** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects +** then the BLOB handle is marked as "expired". +** This is true if any column of the row is changed, even a column +** other than the one the BLOB handle is open on.)^ +** ^Calls to [sqlite3_blob_read()] and [sqlite3_blob_write()] for +** an expired BLOB handle fail with a return code of [SQLITE_ABORT]. +** ^(Changes written into a BLOB prior to the BLOB expiring are not +** rolled back by the expiration of the BLOB. Such changes will eventually +** commit if the transaction continues to completion.)^ +** +** ^Use the [sqlite3_blob_bytes()] interface to determine the size of +** the opened blob. ^The size of a blob may not be changed by this +** interface. Use the [UPDATE] SQL command to change the size of a +** blob. +** +** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] +** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. +** +** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces +** and the built-in [zeroblob] SQL function can be used, if desired, +** to create an empty, zero-filled blob in which to read or write using +** this interface. +** +** To avoid a resource leak, every open [BLOB handle] should eventually +** be released by a call to [sqlite3_blob_close()]. +*/ +SQLITE_API int sqlite3_blob_open( + sqlite3*, + const char *zDb, + const char *zTable, + const char *zColumn, + sqlite3_int64 iRow, + int flags, + sqlite3_blob **ppBlob +); + +/* +** CAPI3REF: Move a BLOB Handle to a New Row +** +** ^This function is used to move an existing blob handle so that it points +** to a different row of the same database table. ^The new row is identified +** by the rowid value passed as the second argument. Only the row can be +** changed. ^The database, table and column on which the blob handle is open +** remain the same. Moving an existing blob handle to a new row can be +** faster than closing the existing handle and opening a new one. +** +** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - +** it must exist and there must be either a blob or text value stored in +** the nominated column.)^ ^If the new row is not present in the table, or if +** it does not contain a blob or text value, or if another error occurs, an +** SQLite error code is returned and the blob handle is considered aborted. +** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return +** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle +** always returns zero. +** +** ^This function sets the database handle error code and message. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); + +/* +** CAPI3REF: Close A BLOB Handle +** +** ^Closes an open [BLOB handle]. +** +** ^Closing a BLOB shall cause the current transaction to commit +** if there are no other BLOBs, no pending prepared statements, and the +** database connection is in [autocommit mode]. +** ^If any writes were made to the BLOB, they might be held in cache +** until the close operation if they will fit. +** +** ^(Closing the BLOB often forces the changes +** out to disk and so if any I/O errors occur, they will likely occur +** at the time when the BLOB is closed. Any errors that occur during +** closing are reported as a non-zero return value.)^ +** +** ^(The BLOB is closed unconditionally. Even if this routine returns +** an error code, the BLOB is still closed.)^ +** +** ^Calling this routine with a null pointer (such as would be returned +** by a failed call to [sqlite3_blob_open()]) is a harmless no-op. +*/ +SQLITE_API int sqlite3_blob_close(sqlite3_blob *); + +/* +** CAPI3REF: Return The Size Of An Open BLOB +** +** ^Returns the size in bytes of the BLOB accessible via the +** successfully opened [BLOB handle] in its only argument. ^The +** incremental blob I/O routines can only read or overwriting existing +** blob content; they cannot change the size of a blob. +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +*/ +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); + +/* +** CAPI3REF: Read Data From A BLOB Incrementally +** +** ^(This function is used to read data from an open [BLOB handle] into a +** caller-supplied buffer. N bytes of data are copied into buffer Z +** from the open BLOB, starting at offset iOffset.)^ +** +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is read. ^If N or iOffset is +** less than zero, [SQLITE_ERROR] is returned and no data is read. +** ^The size of the blob (and hence the maximum value of N+iOffset) +** can be determined using the [sqlite3_blob_bytes()] interface. +** +** ^An attempt to read from an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. +** +** ^(On success, sqlite3_blob_read() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_write()]. +*/ +SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); + +/* +** CAPI3REF: Write Data Into A BLOB Incrementally +** +** ^This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. ^N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset. +** +** ^If the [BLOB handle] passed as the first argument was not opened for +** writing (the flags parameter to [sqlite3_blob_open()] was zero), +** this function returns [SQLITE_READONLY]. +** +** ^This function may only modify the contents of the BLOB; it is +** not possible to increase the size of a BLOB using this API. +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is written. ^If N is +** less than zero [SQLITE_ERROR] is returned and no data is written. +** The size of the BLOB (and hence the maximum value of N+iOffset) +** can be determined using the [sqlite3_blob_bytes()] interface. +** +** ^An attempt to write to an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred +** before the [BLOB handle] expired are not rolled back by the +** expiration of the handle, though of course those changes might +** have been overwritten by the statement that expired the BLOB handle +** or by other independent statements. +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_read()]. +*/ +SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); + +/* +** CAPI3REF: Virtual File System Objects +** +** A virtual filesystem (VFS) is an [sqlite3_vfs] object +** that SQLite uses to interact +** with the underlying operating system. Most SQLite builds come with a +** single default VFS that is appropriate for the host computer. +** New VFSes can be registered and existing VFSes can be unregistered. +** The following interfaces are provided. +** +** ^The sqlite3_vfs_find() interface returns a pointer to a VFS given its name. +** ^Names are case sensitive. +** ^Names are zero-terminated UTF-8 strings. +** ^If there is no match, a NULL pointer is returned. +** ^If zVfsName is NULL then the default VFS is returned. +** +** ^New VFSes are registered with sqlite3_vfs_register(). +** ^Each new VFS becomes the default VFS if the makeDflt flag is set. +** ^The same VFS can be registered multiple times without injury. +** ^To make an existing VFS into the default VFS, register it again +** with the makeDflt flag set. If two different VFSes with the +** same name are registered, the behavior is undefined. If a +** VFS is registered with a name that is NULL or an empty string, +** then the behavior is undefined. +** +** ^Unregister a VFS with the sqlite3_vfs_unregister() interface. +** ^(If the default VFS is unregistered, another VFS is chosen as +** the default. The choice for the new VFS is arbitrary.)^ +*/ +SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName); +SQLITE_API int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); +SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); + +/* +** CAPI3REF: Mutexes +** +** The SQLite core uses these routines for thread +** synchronization. Though they are intended for internal +** use by SQLite, code that links against SQLite is +** permitted to use any of these routines. +** +** The SQLite source code contains multiple implementations +** of these mutex routines. An appropriate implementation +** is selected automatically at compile-time. ^(The following +** implementations are available in the SQLite core: +** +**
    +**
  • SQLITE_MUTEX_PTHREADS +**
  • SQLITE_MUTEX_W32 +**
  • SQLITE_MUTEX_NOOP +**
)^ +** +** ^The SQLITE_MUTEX_NOOP implementation is a set of routines +** that does no real locking and is appropriate for use in +** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix +** and Windows. +** +** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex +** implementation is included with the library. In this case the +** application must supply a custom mutex implementation using the +** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function +** before calling sqlite3_initialize() or any other public sqlite3_ +** function that calls sqlite3_initialize().)^ +** +** ^The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. ^If it returns NULL +** that means that a mutex could not be allocated. ^SQLite +** will unwind its stack and return an error. ^(The argument +** to sqlite3_mutex_alloc() is one of these integer constants: +** +**
    +**
  • SQLITE_MUTEX_FAST +**
  • SQLITE_MUTEX_RECURSIVE +**
  • SQLITE_MUTEX_STATIC_MASTER +**
  • SQLITE_MUTEX_STATIC_MEM +**
  • SQLITE_MUTEX_STATIC_OPEN +**
  • SQLITE_MUTEX_STATIC_PRNG +**
  • SQLITE_MUTEX_STATIC_LRU +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
)^ +** +** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) +** cause sqlite3_mutex_alloc() to create +** a new mutex. ^The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. ^SQLite will only request a recursive mutex in +** cases where it really needs one. ^If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other +** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return +** a pointer to a static preexisting mutex. ^Six static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. ^But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +** +** ^The sqlite3_mutex_free() routine deallocates a previously +** allocated dynamic mutex. ^SQLite is careful to deallocate every +** dynamic mutex that it allocates. The dynamic mutexes must not be in +** use when they are deallocated. Attempting to deallocate a static +** mutex results in undefined behavior. ^SQLite never deallocates +** a static mutex. +** +** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. ^If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] +** upon successful entry. ^(Mutexes created using +** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. +** In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter.)^ ^(If the same thread tries to enter any other +** kind of mutex more than once, the behavior is undefined. +** SQLite will never exhibit +** such behavior in its own use of mutexes.)^ +** +** ^(Some systems (for example, Windows 95) do not support the operation +** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable behavior.)^ +** +** ^The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. ^(The behavior +** is undefined if the mutex is not currently entered by the +** calling thread or is not currently allocated. SQLite will +** never do either.)^ +** +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or +** sqlite3_mutex_leave() is a NULL pointer, then all three routines +** behave as no-ops. +** +** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. +*/ +SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int); +SQLITE_API void sqlite3_mutex_free(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_try(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); + +/* +** CAPI3REF: Mutex Methods Object +** +** An instance of this structure defines the low-level routines +** used to allocate and use mutexes. +** +** Usually, the default mutex implementations provided by SQLite are +** sufficient, however the user has the option of substituting a custom +** implementation for specialized deployments or systems for which SQLite +** does not provide a suitable implementation. In this case, the user +** creates and populates an instance of this structure to pass +** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. +** Additionally, an instance of this structure can be used as an +** output variable when querying the system for the current mutex +** implementation, using the [SQLITE_CONFIG_GETMUTEX] option. +** +** ^The xMutexInit method defined by this structure is invoked as +** part of system initialization by the sqlite3_initialize() function. +** ^The xMutexInit routine is called by SQLite exactly once for each +** effective call to [sqlite3_initialize()]. +** +** ^The xMutexEnd method defined by this structure is invoked as +** part of system shutdown by the sqlite3_shutdown() function. The +** implementation of this method is expected to release all outstanding +** resources obtained by the mutex methods implementation, especially +** those obtained by the xMutexInit method. ^The xMutexEnd() +** interface is invoked exactly once for each call to [sqlite3_shutdown()]. +** +** ^(The remaining seven methods defined by this structure (xMutexAlloc, +** xMutexFree, xMutexEnter, xMutexTry, xMutexLeave, xMutexHeld and +** xMutexNotheld) implement the following interfaces (respectively): +** +**
    +**
  • [sqlite3_mutex_alloc()]
    • +**
  • [sqlite3_mutex_free()]
    • +**
  • [sqlite3_mutex_enter()]
    • +**
  • [sqlite3_mutex_try()]
    • +**
  • [sqlite3_mutex_leave()]
    • +**
  • [sqlite3_mutex_held()]
    • +**
  • [sqlite3_mutex_notheld()]
    • +**
)^ +** +** The only difference is that the public sqlite3_XXX functions enumerated +** above silently ignore any invocations that pass a NULL pointer instead +** of a valid mutex handle. The implementations of the methods defined +** by this structure are not required to handle this case, the results +** of passing a NULL pointer instead of a valid mutex handle are undefined +** (i.e. it is acceptable to provide an implementation that segfaults if +** it is passed a NULL pointer). +** +** The xMutexInit() method must be threadsafe. ^It must be harmless to +** invoke xMutexInit() multiple times within the same process and without +** intervening calls to xMutexEnd(). Second and subsequent calls to +** xMutexInit() must be no-ops. +** +** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory +** allocation for a static mutex. ^However xMutexAlloc() may use SQLite +** memory allocation for a fast or recursive mutex. +** +** ^SQLite will invoke the xMutexEnd() method when [sqlite3_shutdown()] is +** called, but only if the prior call to xMutexInit returned SQLITE_OK. +** If xMutexInit fails in any way, it is expected to clean up after itself +** prior to returning. +*/ +typedef struct sqlite3_mutex_methods sqlite3_mutex_methods; +struct sqlite3_mutex_methods { + int (*xMutexInit)(void); + int (*xMutexEnd)(void); + sqlite3_mutex *(*xMutexAlloc)(int); + void (*xMutexFree)(sqlite3_mutex *); + void (*xMutexEnter)(sqlite3_mutex *); + int (*xMutexTry)(sqlite3_mutex *); + void (*xMutexLeave)(sqlite3_mutex *); + int (*xMutexHeld)(sqlite3_mutex *); + int (*xMutexNotheld)(sqlite3_mutex *); +}; + +/* +** CAPI3REF: Mutex Verification Routines +** +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines +** are intended for use inside assert() statements. ^The SQLite core +** never uses these routines except inside an assert() and applications +** are advised to follow the lead of the core. ^The SQLite core only +** provides implementations for these routines when it is compiled +** with the SQLITE_DEBUG flag. ^External mutex implementations +** are only required to provide these routines if SQLITE_DEBUG is +** defined and if NDEBUG is not defined. +** +** ^These routines should return true if the mutex in their argument +** is held or not held, respectively, by the calling thread. +** +** ^The implementation is not required to provide versions of these +** routines that actually work. If the implementation does not provide working +** versions of these routines, it should at least provide stubs that always +** return true so that one does not get spurious assertion failures. +** +** ^If the argument to sqlite3_mutex_held() is a NULL pointer then +** the routine should return 1. This seems counter-intuitive since +** clearly the mutex cannot be held if it does not exist. But +** the reason the mutex does not exist is because the build is not +** using mutexes. And we do not want the assert() containing the +** call to sqlite3_mutex_held() to fail, so a non-zero return is +** the appropriate thing to do. ^The sqlite3_mutex_notheld() +** interface should also return 1 when given a NULL pointer. +*/ +#ifndef NDEBUG +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); +#endif + +/* +** CAPI3REF: Mutex Types +** +** The [sqlite3_mutex_alloc()] interface takes a single argument +** which is one of these integer constants. +** +** The set of static mutexes may change from one SQLite release to the +** next. Applications that override the built-in mutex logic must be +** prepared to accommodate additional static mutexes. +*/ +#define SQLITE_MUTEX_FAST 0 +#define SQLITE_MUTEX_RECURSIVE 1 +#define SQLITE_MUTEX_STATIC_MASTER 2 +#define SQLITE_MUTEX_STATIC_MEM 3 /* sqlite3_malloc() */ +#define SQLITE_MUTEX_STATIC_MEM2 4 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */ +#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_random() */ +#define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ +#define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ + +/* +** CAPI3REF: Retrieve the mutex for a database connection +** +** ^This interface returns a pointer the [sqlite3_mutex] object that +** serializes access to the [database connection] given in the argument +** when the [threading mode] is Serialized. +** ^If the [threading mode] is Single-thread or Multi-thread then this +** routine returns a NULL pointer. +*/ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); + +/* +** CAPI3REF: Low-Level Control Of Database Files +** +** ^The [sqlite3_file_control()] interface makes a direct call to the +** xFileControl method for the [sqlite3_io_methods] object associated +** with a particular database identified by the second argument. ^The +** name of the database is "main" for the main database or "temp" for the +** TEMP database, or the name that appears after the AS keyword for +** databases that are added using the [ATTACH] SQL command. +** ^A NULL pointer can be used in place of "main" to refer to the +** main database file. +** ^The third and fourth parameters to this routine +** are passed directly through to the second and third parameters of +** the xFileControl method. ^The return value of the xFileControl +** method becomes the return value of this routine. +** +** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes +** a pointer to the underlying [sqlite3_file] object to be written into +** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER +** case is a short-circuit path which does not actually invoke the +** underlying sqlite3_io_methods.xFileControl method. +** +** ^If the second parameter (zDbName) does not match the name of any +** open database file, then SQLITE_ERROR is returned. ^This error +** code is not remembered and will not be recalled by [sqlite3_errcode()] +** or [sqlite3_errmsg()]. The underlying xFileControl method might +** also return SQLITE_ERROR. There is no way to distinguish between +** an incorrect zDbName and an SQLITE_ERROR return from the underlying +** xFileControl method. +** +** See also: [SQLITE_FCNTL_LOCKSTATE] +*/ +SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); + +/* +** CAPI3REF: Testing Interface +** +** ^The sqlite3_test_control() interface is used to read out internal +** state of SQLite and to inject faults into SQLite for testing +** purposes. ^The first parameter is an operation code that determines +** the number, meaning, and operation of all subsequent parameters. +** +** This interface is not for use by applications. It exists solely +** for verifying the correct operation of the SQLite library. Depending +** on how the SQLite library is compiled, this interface might not exist. +** +** The details of the operation codes, their meanings, the parameters +** they take, and what they do are all subject to change without notice. +** Unlike most of the SQLite API, this function is not guaranteed to +** operate consistently from one release to the next. +*/ +SQLITE_API int sqlite3_test_control(int op, ...); + +/* +** CAPI3REF: Testing Interface Operation Codes +** +** These constants are the valid operation code parameters used +** as the first argument to [sqlite3_test_control()]. +** +** These parameters and their meanings are subject to change +** without notice. These values are for testing purposes only. +** Applications should not use any of these parameters or the +** [sqlite3_test_control()] interface. +*/ +#define SQLITE_TESTCTRL_FIRST 5 +#define SQLITE_TESTCTRL_PRNG_SAVE 5 +#define SQLITE_TESTCTRL_PRNG_RESTORE 6 +#define SQLITE_TESTCTRL_PRNG_RESET 7 +#define SQLITE_TESTCTRL_BITVEC_TEST 8 +#define SQLITE_TESTCTRL_FAULT_INSTALL 9 +#define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 +#define SQLITE_TESTCTRL_PENDING_BYTE 11 +#define SQLITE_TESTCTRL_ASSERT 12 +#define SQLITE_TESTCTRL_ALWAYS 13 +#define SQLITE_TESTCTRL_RESERVE 14 +#define SQLITE_TESTCTRL_OPTIMIZATIONS 15 +#define SQLITE_TESTCTRL_ISKEYWORD 16 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 +#define SQLITE_TESTCTRL_NEVER_CORRUPT 20 +#define SQLITE_TESTCTRL_VDBE_COVERAGE 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_ISINIT 23 +#define SQLITE_TESTCTRL_LAST 23 + +/* +** CAPI3REF: SQLite Runtime Status +** +** ^This interface is used to retrieve runtime status information +** about the performance of SQLite, and optionally to reset various +** highwater marks. ^The first argument is an integer code for +** the specific parameter to measure. ^(Recognized integer codes +** are of the form [status parameters | SQLITE_STATUS_...].)^ +** ^The current value of the parameter is returned into *pCurrent. +** ^The highest recorded value is returned in *pHighwater. ^If the +** resetFlag is true, then the highest record value is reset after +** *pHighwater is written. ^(Some parameters do not record the highest +** value. For those parameters +** nothing is written into *pHighwater and the resetFlag is ignored.)^ +** ^(Other parameters record only the highwater mark and not the current +** value. For these latter parameters nothing is written into *pCurrent.)^ +** +** ^The sqlite3_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** +** This routine is threadsafe but is not atomic. This routine can be +** called while other threads are running the same or different SQLite +** interfaces. However the values returned in *pCurrent and +** *pHighwater reflect the status of SQLite at different points in time +** and it is possible that another thread might change the parameter +** in between the times when *pCurrent and *pHighwater are written. +** +** See also: [sqlite3_db_status()] +*/ +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); + + +/* +** CAPI3REF: Status Parameters +** KEYWORDS: {status parameters} +** +** These integer constants designate various run-time status parameters +** that can be returned by [sqlite3_status()]. +** +**
+** [[SQLITE_STATUS_MEMORY_USED]] ^(
SQLITE_STATUS_MEMORY_USED
+**
This parameter is the current amount of memory checked out +** using [sqlite3_malloc()], either directly or indirectly. The +** figure includes calls made to [sqlite3_malloc()] by the application +** and internal memory usage by the SQLite library. Scratch memory +** controlled by [SQLITE_CONFIG_SCRATCH] and auxiliary page-cache +** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in +** this parameter. The amount returned is the sum of the allocation +** sizes as reported by the xSize method in [sqlite3_mem_methods].
)^ +** +** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
SQLITE_STATUS_MALLOC_SIZE
+**
This parameter records the largest memory allocation request +** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their +** internal equivalents). Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
)^ +** +** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
SQLITE_STATUS_MALLOC_COUNT
+**
This parameter records the number of separate memory allocations +** currently checked out.
)^ +** +** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
SQLITE_STATUS_PAGECACHE_USED
+**
This parameter returns the number of pages used out of the +** [pagecache memory allocator] that was configured using +** [SQLITE_CONFIG_PAGECACHE]. The +** value returned is in pages, not in bytes.
)^ +** +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] +** ^(
SQLITE_STATUS_PAGECACHE_OVERFLOW
+**
This parameter returns the number of bytes of page cache +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] +** buffer and where forced to overflow to [sqlite3_malloc()]. The +** returned value includes allocations that overflowed because they +** where too large (they were larger than the "sz" parameter to +** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because +** no space was left in the page cache.
)^ +** +** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
SQLITE_STATUS_PAGECACHE_SIZE
+**
This parameter records the largest memory allocation request +** handed to [pagecache memory allocator]. Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
)^ +** +** [[SQLITE_STATUS_SCRATCH_USED]] ^(
SQLITE_STATUS_SCRATCH_USED
+**
This parameter returns the number of allocations used out of the +** [scratch memory allocator] configured using +** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not +** in bytes. Since a single thread may only have one scratch allocation +** outstanding at time, this parameter also reports the number of threads +** using scratch memory at the same time.
)^ +** +** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
SQLITE_STATUS_SCRATCH_OVERFLOW
+**
This parameter returns the number of bytes of scratch memory +** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] +** buffer and where forced to overflow to [sqlite3_malloc()]. The values +** returned include overflows because the requested allocation was too +** larger (that is, because the requested allocation was larger than the +** "sz" parameter to [SQLITE_CONFIG_SCRATCH]) and because no scratch buffer +** slots were available. +**
)^ +** +** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(
SQLITE_STATUS_SCRATCH_SIZE
+**
This parameter records the largest memory allocation request +** handed to [scratch memory allocator]. Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
)^ +** +** [[SQLITE_STATUS_PARSER_STACK]] ^(
SQLITE_STATUS_PARSER_STACK
+**
This parameter records the deepest parser stack. It is only +** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
)^ +**
+** +** New status parameters may be added from time to time. +*/ +#define SQLITE_STATUS_MEMORY_USED 0 +#define SQLITE_STATUS_PAGECACHE_USED 1 +#define SQLITE_STATUS_PAGECACHE_OVERFLOW 2 +#define SQLITE_STATUS_SCRATCH_USED 3 +#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 +#define SQLITE_STATUS_MALLOC_SIZE 5 +#define SQLITE_STATUS_PARSER_STACK 6 +#define SQLITE_STATUS_PAGECACHE_SIZE 7 +#define SQLITE_STATUS_SCRATCH_SIZE 8 +#define SQLITE_STATUS_MALLOC_COUNT 9 + +/* +** CAPI3REF: Database Connection Status +** +** ^This interface is used to retrieve runtime status information +** about a single [database connection]. ^The first argument is the +** database connection object to be interrogated. ^The second argument +** is an integer constant, taken from the set of +** [SQLITE_DBSTATUS options], that +** determines the parameter to interrogate. The set of +** [SQLITE_DBSTATUS options] is likely +** to grow in future releases of SQLite. +** +** ^The current value of the requested parameter is written into *pCur +** and the highest instantaneous value is written into *pHiwtr. ^If +** the resetFlg is true, then the highest instantaneous value is +** reset back down to the current value. +** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** +** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. +*/ +SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); + +/* +** CAPI3REF: Status Parameters for database connections +** KEYWORDS: {SQLITE_DBSTATUS options} +** +** These constants are the available integer "verbs" that can be passed as +** the second argument to the [sqlite3_db_status()] interface. +** +** New verbs may be added in future releases of SQLite. Existing verbs +** might be discontinued. Applications should check the return code from +** [sqlite3_db_status()] to make sure that the call worked. +** The [sqlite3_db_status()] interface will return a non-zero error code +** if a discontinued or unsupported verb is invoked. +** +**
+** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(
SQLITE_DBSTATUS_LOOKASIDE_USED
+**
This parameter returns the number of lookaside memory slots currently +** checked out.
)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
SQLITE_DBSTATUS_LOOKASIDE_HIT
+**
This parameter returns the number malloc attempts that were +** satisfied using lookaside memory. Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] +** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
+**
This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to the amount of +** memory requested being larger than the lookaside slot size. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] +** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
+**
This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to all lookaside +** memory already being in use. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
SQLITE_DBSTATUS_CACHE_USED
+**
This parameter returns the approximate number of of bytes of heap +** memory used by all pager caches associated with the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +** +** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
SQLITE_DBSTATUS_SCHEMA_USED
+**
This parameter returns the approximate number of of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** [[SQLITE_DBSTATUS_STMT_USED]] ^(
SQLITE_DBSTATUS_STMT_USED
+**
This parameter returns the approximate number of of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
SQLITE_DBSTATUS_CACHE_HIT
+**
This parameter returns the number of pager cache hits that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
SQLITE_DBSTATUS_CACHE_MISS
+**
This parameter returns the number of pager cache misses that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
SQLITE_DBSTATUS_CACHE_WRITE
+**
This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
+** +** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(
SQLITE_DBSTATUS_DEFERRED_FKS
+**
This parameter returns zero for the current value if and only if +** all foreign key constraints (deferred or immediate) have been +** resolved.)^ ^The highwater mark is always 0. +**
+**
+*/ +#define SQLITE_DBSTATUS_LOOKASIDE_USED 0 +#define SQLITE_DBSTATUS_CACHE_USED 1 +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 +#define SQLITE_DBSTATUS_CACHE_HIT 7 +#define SQLITE_DBSTATUS_CACHE_MISS 8 +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_DEFERRED_FKS 10 +#define SQLITE_DBSTATUS_MAX 10 /* Largest defined DBSTATUS */ + + +/* +** CAPI3REF: Prepared Statement Status +** +** ^(Each prepared statement maintains various +** [SQLITE_STMTSTATUS counters] that measure the number +** of times it has performed specific operations.)^ These counters can +** be used to monitor the performance characteristics of the prepared +** statements. For example, if the number of table steps greatly exceeds +** the number of table searches or result rows, that would tend to indicate +** that the prepared statement is using a full table scan rather than +** an index. +** +** ^(This interface is used to retrieve and reset counter values from +** a [prepared statement]. The first argument is the prepared statement +** object to be interrogated. The second argument +** is an integer code for a specific [SQLITE_STMTSTATUS counter] +** to be interrogated.)^ +** ^The current value of the requested counter is returned. +** ^If the resetFlg is true, then the counter is reset to zero after this +** interface call returns. +** +** See also: [sqlite3_status()] and [sqlite3_db_status()]. +*/ +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); + +/* +** CAPI3REF: Status Parameters for prepared statements +** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters} +** +** These preprocessor macros define integer codes that name counter +** values associated with the [sqlite3_stmt_status()] interface. +** The meanings of the various counters are as follows: +** +**
+** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
SQLITE_STMTSTATUS_FULLSCAN_STEP
+**
^This is the number of times that SQLite has stepped forward in +** a table as part of a full table scan. Large numbers for this counter +** may indicate opportunities for performance improvement through +** careful use of indices.
+** +** [[SQLITE_STMTSTATUS_SORT]]
SQLITE_STMTSTATUS_SORT
+**
^This is the number of sort operations that have occurred. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance through careful use of indices.
+** +** [[SQLITE_STMTSTATUS_AUTOINDEX]]
SQLITE_STMTSTATUS_AUTOINDEX
+**
^This is the number of rows inserted into transient indices that +** were created automatically in order to help joins run faster. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance by adding permanent indices that do not +** need to be reinitialized each time the statement is run.
+** +** [[SQLITE_STMTSTATUS_VM_STEP]]
SQLITE_STMTSTATUS_VM_STEP
+**
^This is the number of virtual machine operations executed +** by the prepared statement if that number is less than or equal +** to 2147483647. The number of virtual machine operations can be +** used as a proxy for the total work done by the prepared statement. +** If the number of virtual machine operations exceeds 2147483647 +** then the value returned by this statement status code is undefined. +**
+**
+*/ +#define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 +#define SQLITE_STMTSTATUS_SORT 2 +#define SQLITE_STMTSTATUS_AUTOINDEX 3 +#define SQLITE_STMTSTATUS_VM_STEP 4 + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache type is opaque. It is implemented by +** the pluggable module. The SQLite core has no knowledge of +** its size or internal structure and never deals with the +** sqlite3_pcache object except by holding and passing pointers +** to the object. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache sqlite3_pcache; + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache_page object represents a single page in the +** page cache. The page cache will allocate instances of this +** object. Various methods of the page cache use pointers to instances +** of this object as parameters or as their return value. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache_page sqlite3_pcache_page; +struct sqlite3_pcache_page { + void *pBuf; /* The content of the page */ + void *pExtra; /* Extra information associated with the page */ +}; + +/* +** CAPI3REF: Application Defined Page Cache. +** KEYWORDS: {page cache} +** +** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can +** register an alternative page cache implementation by passing in an +** instance of the sqlite3_pcache_methods2 structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which +** that memory is allocated and released, and the policies used to +** determine exactly which parts of a database file are cached and for +** how long. +** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** +** ^(The contents of the sqlite3_pcache_methods2 structure are copied to an +** internal buffer by SQLite within the call to [sqlite3_config]. Hence +** the application may discard the parameter after the call to +** [sqlite3_config()] returns.)^ +** +** [[the xInit() page cache method]] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ +** (usually only once during the lifetime of the process). ^(The xInit() +** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^ +** The intent of the xInit() method is to set up global data structures +** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ +** +** [[the xShutdown() page cache method]] +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up +** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. +** +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. All other methods must be threadsafe +** in multithreaded applications. +** +** ^SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +** +** [[the xCreate() page cache methods]] +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, +** though this is not guaranteed. ^The +** first parameter, szPage, is the size in bytes of the pages that must +** be allocated by the cache. ^szPage will always a power of two. ^The +** second parameter szExtra is a number of bytes of extra storage +** associated with each page cache entry. ^The szExtra parameter will +** a number less than 250. SQLite will use the +** extra szExtra bytes on each page to store metadata about the underlying +** database page on disk. The value passed into szExtra depends +** on the SQLite version, the target platform, and how SQLite was compiled. +** ^The third argument to xCreate(), bPurgeable, is true if the cache being +** created will be used to cache database pages of a file stored on disk, or +** false if it is used for an in-memory database. The cache implementation +** does not have to do anything special based with the value of bPurgeable; +** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will +** never invoke xUnpin() except to deliberately delete a page. +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will +** never contain any unpinned pages. +** +** [[the xCachesize() page cache method]] +** ^(The xCachesize() method may be called at any time by SQLite to set the +** suggested maximum cache-size (number of pages stored by) the cache +** instance passed as the first argument. This is the value configured using +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable +** parameter, the implementation is not required to do anything with this +** value; it is advisory only. +** +** [[the xPagecount() page cache methods]] +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. +** +** [[the xFetch() page cache methods]] +** The xFetch() method locates a page in the cache and returns a pointer to +** an sqlite3_pcache_page object associated with that page, or a NULL pointer. +** The pBuf element of the returned sqlite3_pcache_page object will be a +** pointer to a buffer of szPage bytes used to store the content of a +** single database page. The pExtra element of sqlite3_pcache_page will be +** a pointer to the szExtra bytes of extra storage that SQLite has requested +** for each entry in the page cache. +** +** The page to be fetched is determined by the key. ^The minimum key value +** is 1. After it has been retrieved using xFetch, the page is considered +** to be "pinned". +** +** If the requested page is already in the page cache, then the page cache +** implementation must return a pointer to the page buffer with its content +** intact. If the requested page is not already in the cache, then the +** cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: +** +** +**
createFlag Behavior when page is not already in cache +**
0 Do not allocate a new page. Return NULL. +**
1 Allocate a new page if it easy and convenient to do so. +** Otherwise return NULL. +**
2 Make every effort to allocate a new page. Only return +** NULL if allocating a new page is effectively impossible. +**
+** +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the to xFetch() calls, SQLite may +** attempt to unpin one or more cache pages by spilling the content of +** pinned pages to disk and synching the operating system disk cache. +** +** [[the xUnpin() page cache method]] +** ^xUnpin() is called by SQLite with a pointer to a currently pinned page +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation +** may choose to evict unpinned pages at any time. +** +** The cache must not perform any reference counting. A single +** call to xUnpin() unpins the page regardless of the number of prior calls +** to xFetch(). +** +** [[the xRekey() page cache methods]] +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be +** discarded. ^Any prior cache entry associated with newKey is guaranteed not +** to be pinned. +** +** When SQLite calls the xTruncate() method, the cache must discard all +** existing cache entries with page numbers (keys) greater than or equal +** to the value of the iLimit parameter passed to xTruncate(). If any +** of these pages are pinned, they are implicitly unpinned, meaning that +** they can be safely discarded. +** +** [[the xDestroy() page cache method]] +** ^The xDestroy() method is used to delete a cache allocated by xCreate(). +** All resources associated with the specified cache should be freed. ^After +** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] +** handle invalid, and will not use it with any other sqlite3_pcache_methods2 +** functions. +** +** [[the xShrink() page cache method]] +** ^SQLite invokes the xShrink() method when it wants the page cache to +** free up as much of heap memory as possible. The page cache implementation +** is not obligated to free any memory, but well-behaved implementations should +** do their best. +*/ +typedef struct sqlite3_pcache_methods2 sqlite3_pcache_methods2; +struct sqlite3_pcache_methods2 { + int iVersion; + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int szExtra, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard); + void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, + unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); + void (*xShrink)(sqlite3_pcache*); +}; + +/* +** This is the obsolete pcache_methods object that has now been replaced +** by sqlite3_pcache_methods2. This object is not used by SQLite. It is +** retained in the header file for backwards compatibility only. +*/ +typedef struct sqlite3_pcache_methods sqlite3_pcache_methods; +struct sqlite3_pcache_methods { + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + void *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, void*, int discard); + void (*xRekey)(sqlite3_pcache*, void*, unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); +}; + + +/* +** CAPI3REF: Online Backup Object +** +** The sqlite3_backup object records state information about an ongoing +** online backup operation. ^The sqlite3_backup object is created by +** a call to [sqlite3_backup_init()] and is destroyed by a call to +** [sqlite3_backup_finish()]. +** +** See Also: [Using the SQLite Online Backup API] +*/ +typedef struct sqlite3_backup sqlite3_backup; + +/* +** CAPI3REF: Online Backup API. +** +** The backup API copies the content of one database into another. +** It is useful either for creating backups of databases or +** for copying in-memory databases to or from persistent files. +** +** See Also: [Using the SQLite Online Backup API] +** +** ^SQLite holds a write transaction open on the destination database file +** for the duration of the backup operation. +** ^The source database is read-locked only while it is being read; +** it is not locked continuously for the entire backup operation. +** ^Thus, the backup may be performed on a live source database without +** preventing other database connections from +** reading or writing to the source database while the backup is underway. +** +** ^(To perform a backup operation: +**
    +**
  1. sqlite3_backup_init() is called once to initialize the +** backup, +**
  2. sqlite3_backup_step() is called one or more times to transfer +** the data between the two databases, and finally +**
  3. sqlite3_backup_finish() is called to release all resources +** associated with the backup operation. +**
)^ +** There should be exactly one call to sqlite3_backup_finish() for each +** successful call to sqlite3_backup_init(). +** +** [[sqlite3_backup_init()]] sqlite3_backup_init() +** +** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the +** [database connection] associated with the destination database +** and the database name, respectively. +** ^The database name is "main" for the main database, "temp" for the +** temporary database, or the name specified after the AS keyword in +** an [ATTACH] statement for an attached database. +** ^The S and M arguments passed to +** sqlite3_backup_init(D,N,S,M) identify the [database connection] +** and database name of the source database, respectively. +** ^The source and destination [database connections] (parameters S and D) +** must be different or else sqlite3_backup_init(D,N,S,M) will fail with +** an error. +** +** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is +** returned and an error code and error message are stored in the +** destination [database connection] D. +** ^The error code and message for the failed call to sqlite3_backup_init() +** can be retrieved using the [sqlite3_errcode()], [sqlite3_errmsg()], and/or +** [sqlite3_errmsg16()] functions. +** ^A successful call to sqlite3_backup_init() returns a pointer to an +** [sqlite3_backup] object. +** ^The [sqlite3_backup] object may be used with the sqlite3_backup_step() and +** sqlite3_backup_finish() functions to perform the specified backup +** operation. +** +** [[sqlite3_backup_step()]] sqlite3_backup_step() +** +** ^Function sqlite3_backup_step(B,N) will copy up to N pages between +** the source and destination databases specified by [sqlite3_backup] object B. +** ^If N is negative, all remaining source pages are copied. +** ^If sqlite3_backup_step(B,N) successfully copies N pages and there +** are still more pages to be copied, then the function returns [SQLITE_OK]. +** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages +** from source to destination, then it returns [SQLITE_DONE]. +** ^If an error occurs while running sqlite3_backup_step(B,N), +** then an [error code] is returned. ^As well as [SQLITE_OK] and +** [SQLITE_DONE], a call to sqlite3_backup_step() may return [SQLITE_READONLY], +** [SQLITE_NOMEM], [SQLITE_BUSY], [SQLITE_LOCKED], or an +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX] extended error code. +** +** ^(The sqlite3_backup_step() might return [SQLITE_READONLY] if +**
    +**
  1. the destination database was opened read-only, or +**
  2. the destination database is using write-ahead-log journaling +** and the destination and source page sizes differ, or +**
  3. the destination database is an in-memory database and the +** destination and source page sizes differ. +**
)^ +** +** ^If sqlite3_backup_step() cannot obtain a required file-system lock, then +** the [sqlite3_busy_handler | busy-handler function] +** is invoked (if one is specified). ^If the +** busy-handler returns non-zero before the lock is available, then +** [SQLITE_BUSY] is returned to the caller. ^In this case the call to +** sqlite3_backup_step() can be retried later. ^If the source +** [database connection] +** is being used to write to the source database when sqlite3_backup_step() +** is called, then [SQLITE_LOCKED] is returned immediately. ^Again, in this +** case the call to sqlite3_backup_step() can be retried later on. ^(If +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX], [SQLITE_NOMEM], or +** [SQLITE_READONLY] is returned, then +** there is no point in retrying the call to sqlite3_backup_step(). These +** errors are considered fatal.)^ The application must accept +** that the backup operation has failed and pass the backup operation handle +** to the sqlite3_backup_finish() to release associated resources. +** +** ^The first call to sqlite3_backup_step() obtains an exclusive lock +** on the destination file. ^The exclusive lock is not released until either +** sqlite3_backup_finish() is called or the backup operation is complete +** and sqlite3_backup_step() returns [SQLITE_DONE]. ^Every call to +** sqlite3_backup_step() obtains a [shared lock] on the source database that +** lasts for the duration of the sqlite3_backup_step() call. +** ^Because the source database is not locked between calls to +** sqlite3_backup_step(), the source database may be modified mid-way +** through the backup process. ^If the source database is modified by an +** external process or via a database connection other than the one being +** used by the backup operation, then the backup will be automatically +** restarted by the next call to sqlite3_backup_step(). ^If the source +** database is modified by the using the same database connection as is used +** by the backup operation, then the backup database is automatically +** updated at the same time. +** +** [[sqlite3_backup_finish()]] sqlite3_backup_finish() +** +** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the +** application wishes to abandon the backup operation, the application +** should destroy the [sqlite3_backup] by passing it to sqlite3_backup_finish(). +** ^The sqlite3_backup_finish() interfaces releases all +** resources associated with the [sqlite3_backup] object. +** ^If sqlite3_backup_step() has not yet returned [SQLITE_DONE], then any +** active write-transaction on the destination database is rolled back. +** The [sqlite3_backup] object is invalid +** and may not be used following a call to sqlite3_backup_finish(). +** +** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no +** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() completed. +** ^If an out-of-memory condition or IO error occurred during any prior +** sqlite3_backup_step() call on the same [sqlite3_backup] object, then +** sqlite3_backup_finish() returns the corresponding [error code]. +** +** ^A return of [SQLITE_BUSY] or [SQLITE_LOCKED] from sqlite3_backup_step() +** is not a permanent error and does not affect the return value of +** sqlite3_backup_finish(). +** +** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]] +** sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** +** ^Each call to sqlite3_backup_step() sets two values inside +** the [sqlite3_backup] object: the number of pages still to be backed +** up and the total number of pages in the source database file. +** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces +** retrieve these two values, respectively. +** +** ^The values returned by these functions are only updated by +** sqlite3_backup_step(). ^If the source database is modified during a backup +** operation, then the values are not updated to account for any extra +** pages that need to be updated or the size of the source database file +** changing. +** +** Concurrent Usage of Database Handles +** +** ^The source [database connection] may be used by the application for other +** purposes while a backup operation is underway or being initialized. +** ^If SQLite is compiled and configured to support threadsafe database +** connections, then the source database connection may be used concurrently +** from within other threads. +** +** However, the application must guarantee that the destination +** [database connection] is not passed to any other API (by any thread) after +** sqlite3_backup_init() is called and before the corresponding call to +** sqlite3_backup_finish(). SQLite does not currently check to see +** if the application incorrectly accesses the destination [database connection] +** and so no error code is reported, but the operations may malfunction +** nevertheless. Use of the destination database connection while a +** backup is in progress might also also cause a mutex deadlock. +** +** If running in [shared cache mode], the application must +** guarantee that the shared cache used by the destination database +** is not accessed while the backup is running. In practice this means +** that the application must guarantee that the disk file being +** backed up to is not accessed by any connection within the process, +** not just the specific connection that was passed to sqlite3_backup_init(). +** +** The [sqlite3_backup] object itself is partially threadsafe. Multiple +** threads may safely make multiple concurrent calls to sqlite3_backup_step(). +** However, the sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** APIs are not strictly speaking threadsafe. If they are invoked at the +** same time as another thread is invoking sqlite3_backup_step() it is +** possible that they return invalid values. +*/ +SQLITE_API sqlite3_backup *sqlite3_backup_init( + sqlite3 *pDest, /* Destination database handle */ + const char *zDestName, /* Destination database name */ + sqlite3 *pSource, /* Source database handle */ + const char *zSourceName /* Source database name */ +); +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage); +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); + +/* +** CAPI3REF: Unlock Notification +** +** ^When running in shared-cache mode, a database operation may fail with +** an [SQLITE_LOCKED] error if the required locks on the shared-cache or +** individual tables within the shared-cache cannot be obtained. See +** [SQLite Shared-Cache Mode] for a description of shared-cache locking. +** ^This API may be used to register a callback that SQLite will invoke +** when the connection currently holding the required lock relinquishes it. +** ^This API is only available if the library was compiled with the +** [SQLITE_ENABLE_UNLOCK_NOTIFY] C-preprocessor symbol defined. +** +** See Also: [Using the SQLite Unlock Notification Feature]. +** +** ^Shared-cache locks are released when a database connection concludes +** its current transaction, either by committing it or rolling it back. +** +** ^When a connection (known as the blocked connection) fails to obtain a +** shared-cache lock and SQLITE_LOCKED is returned to the caller, the +** identity of the database connection (the blocking connection) that +** has locked the required resource is stored internally. ^After an +** application receives an SQLITE_LOCKED error, it may call the +** sqlite3_unlock_notify() method with the blocked connection handle as +** the first argument to register for a callback that will be invoked +** when the blocking connections current transaction is concluded. ^The +** callback is invoked from within the [sqlite3_step] or [sqlite3_close] +** call that concludes the blocking connections transaction. +** +** ^(If sqlite3_unlock_notify() is called in a multi-threaded application, +** there is a chance that the blocking connection will have already +** concluded its transaction by the time sqlite3_unlock_notify() is invoked. +** If this happens, then the specified callback is invoked immediately, +** from within the call to sqlite3_unlock_notify().)^ +** +** ^If the blocked connection is attempting to obtain a write-lock on a +** shared-cache table, and more than one other connection currently holds +** a read-lock on the same table, then SQLite arbitrarily selects one of +** the other connections to use as the blocking connection. +** +** ^(There may be at most one unlock-notify callback registered by a +** blocked connection. If sqlite3_unlock_notify() is called when the +** blocked connection already has a registered unlock-notify callback, +** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is +** called with a NULL pointer as its second argument, then any existing +** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback may also be canceled by closing the blocked +** connection using [sqlite3_close()]. +** +** The unlock-notify callback is not reentrant. If an application invokes +** any sqlite3_xxx API functions from within an unlock-notify callback, a +** crash or deadlock may be the result. +** +** ^Unless deadlock is detected (see below), sqlite3_unlock_notify() always +** returns SQLITE_OK. +** +** Callback Invocation Details +** +** When an unlock-notify callback is registered, the application provides a +** single void* pointer that is passed to the callback when it is invoked. +** However, the signature of the callback function allows SQLite to pass +** it an array of void* context pointers. The first argument passed to +** an unlock-notify callback is a pointer to an array of void* pointers, +** and the second is the number of entries in the array. +** +** When a blocking connections transaction is concluded, there may be +** more than one blocked connection that has registered for an unlock-notify +** callback. ^If two or more such blocked connections have specified the +** same callback function, then instead of invoking the callback function +** multiple times, it is invoked once with the set of void* context pointers +** specified by the blocked connections bundled together into an array. +** This gives the application an opportunity to prioritize any actions +** related to the set of unblocked database connections. +** +** Deadlock Detection +** +** Assuming that after registering for an unlock-notify callback a +** database waits for the callback to be issued before taking any further +** action (a reasonable assumption), then using this API may cause the +** application to deadlock. For example, if connection X is waiting for +** connection Y's transaction to be concluded, and similarly connection +** Y is waiting on connection X's transaction, then neither connection +** will proceed and the system may remain deadlocked indefinitely. +** +** To avoid this scenario, the sqlite3_unlock_notify() performs deadlock +** detection. ^If a given call to sqlite3_unlock_notify() would put the +** system in a deadlocked state, then SQLITE_LOCKED is returned and no +** unlock-notify callback is registered. The system is said to be in +** a deadlocked state if connection A has registered for an unlock-notify +** callback on the conclusion of connection B's transaction, and connection +** B has itself registered for an unlock-notify callback when connection +** A's transaction is concluded. ^Indirect deadlock is also detected, so +** the system is also considered to be deadlocked if connection B has +** registered for an unlock-notify callback on the conclusion of connection +** C's transaction, where connection C is waiting on connection A. ^Any +** number of levels of indirection are allowed. +** +** The "DROP TABLE" Exception +** +** When a call to [sqlite3_step()] returns SQLITE_LOCKED, it is almost +** always appropriate to call sqlite3_unlock_notify(). There is however, +** one exception. When executing a "DROP TABLE" or "DROP INDEX" statement, +** SQLite checks if there are any currently executing SELECT statements +** that belong to the same connection. If there are, SQLITE_LOCKED is +** returned. In this case there is no "blocking connection", so invoking +** sqlite3_unlock_notify() results in the unlock-notify callback being +** invoked immediately. If the application then re-attempts the "DROP TABLE" +** or "DROP INDEX" query, an infinite loop might be the result. +** +** One way around this problem is to check the extended error code returned +** by an sqlite3_step() call. ^(If there is a blocking connection, then the +** extended error code is set to SQLITE_LOCKED_SHAREDCACHE. Otherwise, in +** the special "DROP TABLE/INDEX" case, the extended error code is just +** SQLITE_LOCKED.)^ +*/ +SQLITE_API int sqlite3_unlock_notify( + sqlite3 *pBlocked, /* Waiting connection */ + void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ + void *pNotifyArg /* Argument to pass to xNotify */ +); + + +/* +** CAPI3REF: String Comparison +** +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. +*/ +SQLITE_API int sqlite3_stricmp(const char *, const char *); +SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); + +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); + +/* +** CAPI3REF: Error Logging Interface +** +** ^The [sqlite3_log()] interface writes a message into the [error log] +** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. +** ^If logging is enabled, the zFormat string and subsequent arguments are +** used with [sqlite3_snprintf()] to generate the final output string. +** +** The sqlite3_log() interface is intended for use by extensions such as +** virtual tables, collating functions, and SQL functions. While there is +** nothing to prevent an application from calling sqlite3_log(), doing so +** is considered bad form. +** +** The zFormat string must not be NULL. +** +** To avoid deadlocks and other threading problems, the sqlite3_log() routine +** will not use dynamically allocated memory. The log message is stored in +** a fixed-length buffer on the stack. If the log message is longer than +** a few hundred characters, it will be truncated to the length of the +** buffer. +*/ +SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); + +/* +** CAPI3REF: Write-Ahead Log Commit Hook +** +** ^The [sqlite3_wal_hook()] function is used to register a callback that +** will be invoked each time a database connection commits data to a +** [write-ahead log] (i.e. whenever a transaction is committed in +** [journal_mode | journal_mode=WAL mode]). +** +** ^The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released, so the implementation +** may read, write or [checkpoint] the database as required. +** +** ^The first parameter passed to the callback function when it is invoked +** is a copy of the third parameter passed to sqlite3_wal_hook() when +** registering the callback. ^The second is a copy of the database handle. +** ^The third parameter is the name of the database that was written to - +** either "main" or the name of an [ATTACH]-ed database. ^The fourth parameter +** is the number of pages currently in the write-ahead log file, +** including those that were just committed. +** +** The callback function should normally return [SQLITE_OK]. ^If an error +** code is returned, that error will propagate back up through the +** SQLite code base to cause the statement that provoked the callback +** to report an error, though the commit will have still occurred. If the +** callback returns [SQLITE_ROW] or [SQLITE_DONE], or if it returns a value +** that does not correspond to any valid SQLite error code, the results +** are undefined. +** +** A single database handle may have at most a single write-ahead log callback +** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any +** previously registered write-ahead log callback. ^Note that the +** [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will +** those overwrite any prior [sqlite3_wal_hook()] settings. +*/ +SQLITE_API void *sqlite3_wal_hook( + sqlite3*, + int(*)(void *,sqlite3*,const char*,int), + void* +); + +/* +** CAPI3REF: Configure an auto-checkpoint +** +** ^The [sqlite3_wal_autocheckpoint(D,N)] is a wrapper around +** [sqlite3_wal_hook()] that causes any database on [database connection] D +** to automatically [checkpoint] +** after committing a transaction if there are N or +** more frames in the [write-ahead log] file. ^Passing zero or +** a negative value as the nFrame parameter disables automatic +** checkpoints entirely. +** +** ^The callback registered by this function replaces any existing callback +** registered using [sqlite3_wal_hook()]. ^Likewise, registering a callback +** using [sqlite3_wal_hook()] disables the automatic checkpoint mechanism +** configured by this function. +** +** ^The [wal_autocheckpoint pragma] can be used to invoke this interface +** from SQL. +** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** +** ^Every new [database connection] defaults to having the auto-checkpoint +** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] +** pages. The use of this interface +** is only necessary if the default setting is found to be suboptimal +** for a particular application. +*/ +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); + +/* +** CAPI3REF: Checkpoint a database +** +** ^The [sqlite3_wal_checkpoint(D,X)] interface causes database named X +** on [database connection] D to be [checkpointed]. ^If X is NULL or an +** empty string, then a checkpoint is run on all databases of +** connection D. ^If the database connection D is not in +** [WAL | write-ahead log mode] then this interface is a harmless no-op. +** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a +** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. +** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL +** or RESET checkpoint. +** +** ^The [wal_checkpoint pragma] can be used to invoke this interface +** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] can be used to cause this interface to be +** run whenever the WAL reaches a certain size threshold. +** +** See also: [sqlite3_wal_checkpoint_v2()] +*/ +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Checkpoint a database +** +** Run a checkpoint operation on WAL database zDb attached to database +** handle db. The specific operation is determined by the value of the +** eMode parameter: +** +**
+**
SQLITE_CHECKPOINT_PASSIVE
+** Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish. Sync the db file if all frames in the log +** are checkpointed. This mode is the same as calling +** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] +** is never invoked. +** +**
SQLITE_CHECKPOINT_FULL
+** This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no +** database writer and all readers are reading from the most recent database +** snapshot. It then checkpoints all frames in the log file and syncs the +** database file. This call blocks database writers while it is running, +** but not database readers. +** +**
SQLITE_CHECKPOINT_RESTART
+** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after +** checkpointing the log file it blocks (calls the +** [sqlite3_busy_handler|busy-handler callback]) +** until all readers are reading from the database file only. This ensures +** that the next client to write to the database file restarts the log file +** from the beginning. This call blocks database writers while it is running, +** but not database readers. +**
+** +** If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to +** the total number of checkpointed frames (including any that were already +** checkpointed when this function is called). *pnLog and *pnCkpt may be +** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. +** If no values are available because of an error, they are both set to -1 +** before returning to communicate this to the caller. +** +** All calls obtain an exclusive "checkpoint" lock on the database file. If +** any other process is running a checkpoint operation at the same time, the +** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** busy-handler configured, it will not be invoked in this case. +** +** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive +** "writer" lock on the database file. If the writer lock cannot be obtained +** immediately, and a busy-handler is configured, it is invoked and the writer +** lock retried until either the busy-handler returns 0 or the lock is +** successfully obtained. The busy-handler is also invoked while waiting for +** database readers as described above. If the busy-handler returns 0 before +** the writer lock is obtained or while waiting for database readers, the +** checkpoint operation proceeds from that point in the same way as +** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible +** without blocking any further. SQLITE_BUSY is returned in this case. +** +** If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. If +** an SQLITE_BUSY error is encountered when processing one or more of the +** attached WAL databases, the operation is still attempted on any remaining +** attached databases and SQLITE_BUSY is returned to the caller. If any other +** error occurs while processing an attached database, processing is abandoned +** and the error code returned to the caller immediately. If no error +** (SQLITE_BUSY or otherwise) is encountered while processing the attached +** databases, SQLITE_OK is returned. +** +** If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** zDb is not NULL (or a zero length string) and is not the name of any +** attached database, SQLITE_ERROR is returned to the caller. +*/ +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +); + +/* +** CAPI3REF: Checkpoint operation parameters +** +** These constants can be used as the 3rd parameter to +** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] +** documentation for additional information about the meaning and use of +** each of these values. +*/ +#define SQLITE_CHECKPOINT_PASSIVE 0 +#define SQLITE_CHECKPOINT_FULL 1 +#define SQLITE_CHECKPOINT_RESTART 2 + +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function may be called by either the [xConnect] or [xCreate] method +** of a [virtual table] implementation to configure +** various facets of the virtual table interface. +** +** If this interface is invoked outside the context of an xConnect or +** xCreate virtual table method then the behavior is undefined. +** +** At present, there is only one option that may be configured using +** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options +** may be added in the future. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Virtual Table Configuration Options +** +** These macros define the various options to the +** [sqlite3_vtab_config()] interface that [virtual table] implementations +** can use to customize and optimize their behavior. +** +**
+**
SQLITE_VTAB_CONSTRAINT_SUPPORT +**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, +** where X is an integer. If X is zero, then the [virtual table] whose +** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not +** support constraints. In this configuration (which is the default) if +** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If X is non-zero, then the virtual table implementation guarantees +** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before +** any modifications to internal or persistent data structures have been made. +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** is able to roll back a statement or database transaction, and abandon +** or continue processing the current SQL statement as appropriate. +** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns +** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode +** had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
+*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the [xUpdate] method +** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The +** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL], +** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode +** of the SQL statement that triggered the call to the [xUpdate] method of the +** [virtual table]. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Conflict resolution modes +** KEYWORDS: {conflict resolution mode} +** +** These constants are returned by [sqlite3_vtab_on_conflict()] to +** inform a [virtual table] implementation what the [ON CONFLICT] mode +** is for the SQL statement being evaluated. +** +** Note that the [SQLITE_IGNORE] constant is also used as a potential +** return value from the [sqlite3_set_authorizer()] callback and that +** [SQLITE_ABORT] is also a [result code]. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 // Also an error code */ +#define SQLITE_REPLACE 5 + + + +/* +** Undo the hack that converts floating point types to integer for +** builds on processors without floating point support. +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# undef double +#endif + +#if 0 +} /* End of the 'extern "C"' block */ +#endif +#endif /* _SQLITE3_H_ */ + +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#if 0 +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visiblity */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ + + +/************** End of sqlite3.h *********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* +** Include the configuration header output by 'configure' if we're using the +** autoconf-based build +*/ +#ifdef _HAVE_SQLITE_CONFIG_H +#include "config.h" +#endif + +/************** Include sqliteLimit.h in the middle of sqliteInt.h ***********/ +/************** Begin file sqliteLimit.h *************************************/ +/* +** 2007 May 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file defines various limits of what SQLite can process. +*/ + +/* +** The maximum length of a TEXT or BLOB in bytes. This also +** limits the size of a row in a table or index. +** +** The hard limit is the ability of a 32-bit signed integer +** to count the size: 2^31-1 or 2147483647. +*/ +#ifndef SQLITE_MAX_LENGTH +# define SQLITE_MAX_LENGTH 1000000000 +#endif + +/* +** This is the maximum number of +** +** * Columns in a table +** * Columns in an index +** * Columns in a view +** * Terms in the SET clause of an UPDATE statement +** * Terms in the result set of a SELECT statement +** * Terms in the GROUP BY or ORDER BY clauses of a SELECT statement. +** * Terms in the VALUES clause of an INSERT statement +** +** The hard upper limit here is 32676. Most database people will +** tell you that in a well-normalized database, you usually should +** not have more than a dozen or so columns in any table. And if +** that is the case, there is no point in having more than a few +** dozen values in any of the other situations described above. +*/ +#ifndef SQLITE_MAX_COLUMN +# define SQLITE_MAX_COLUMN 2000 +#endif + +/* +** The maximum length of a single SQL statement in bytes. +** +** It used to be the case that setting this value to zero would +** turn the limit off. That is no longer true. It is not possible +** to turn this limit off. +*/ +#ifndef SQLITE_MAX_SQL_LENGTH +# define SQLITE_MAX_SQL_LENGTH 1000000000 +#endif + +/* +** The maximum depth of an expression tree. This is limited to +** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might +** want to place more severe limits on the complexity of an +** expression. +** +** A value of 0 used to mean that the limit was not enforced. +** But that is no longer true. The limit is now strictly enforced +** at all times. +*/ +#ifndef SQLITE_MAX_EXPR_DEPTH +# define SQLITE_MAX_EXPR_DEPTH 1000 +#endif + +/* +** The maximum number of terms in a compound SELECT statement. +** The code generator for compound SELECT statements does one +** level of recursion for each term. A stack overflow can result +** if the number of terms is too large. In practice, most SQL +** never has more than 3 or 4 terms. Use a value of 0 to disable +** any limit on the number of terms in a compount SELECT. +*/ +#ifndef SQLITE_MAX_COMPOUND_SELECT +# define SQLITE_MAX_COMPOUND_SELECT 500 +#endif + +/* +** The maximum number of opcodes in a VDBE program. +** Not currently enforced. +*/ +#ifndef SQLITE_MAX_VDBE_OP +# define SQLITE_MAX_VDBE_OP 25000 +#endif + +/* +** The maximum number of arguments to an SQL function. +*/ +#ifndef SQLITE_MAX_FUNCTION_ARG +# define SQLITE_MAX_FUNCTION_ARG 127 +#endif + +/* +** The maximum number of in-memory pages to use for the main database +** table and for temporary tables. The SQLITE_DEFAULT_CACHE_SIZE +*/ +#ifndef SQLITE_DEFAULT_CACHE_SIZE +# define SQLITE_DEFAULT_CACHE_SIZE 2000 +#endif +#ifndef SQLITE_DEFAULT_TEMP_CACHE_SIZE +# define SQLITE_DEFAULT_TEMP_CACHE_SIZE 500 +#endif + +/* +** The default number of frames to accumulate in the log file before +** checkpointing the database in WAL mode. +*/ +#ifndef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT +# define SQLITE_DEFAULT_WAL_AUTOCHECKPOINT 1000 +#endif + +/* +** The maximum number of attached databases. This must be between 0 +** and 62. The upper bound on 62 is because a 64-bit integer bitmap +** is used internally to track attached databases. +*/ +#ifndef SQLITE_MAX_ATTACHED +# define SQLITE_MAX_ATTACHED 10 +#endif + + +/* +** The maximum value of a ?nnn wildcard that the parser will accept. +*/ +#ifndef SQLITE_MAX_VARIABLE_NUMBER +# define SQLITE_MAX_VARIABLE_NUMBER 999 +#endif + +/* Maximum page size. The upper bound on this value is 65536. This a limit +** imposed by the use of 16-bit offsets within each page. +** +** Earlier versions of SQLite allowed the user to change this value at +** compile time. This is no longer permitted, on the grounds that it creates +** a library that is technically incompatible with an SQLite library +** compiled with a different limit. If a process operating on a database +** with a page-size of 65536 bytes crashes, then an instance of SQLite +** compiled with the default page-size limit will not be able to rollback +** the aborted transaction. This could lead to database corruption. +*/ +#ifdef SQLITE_MAX_PAGE_SIZE +# undef SQLITE_MAX_PAGE_SIZE +#endif +#define SQLITE_MAX_PAGE_SIZE 65536 + + +/* +** The default size of a database page. +*/ +#ifndef SQLITE_DEFAULT_PAGE_SIZE +# define SQLITE_DEFAULT_PAGE_SIZE 1024 +#endif +#if SQLITE_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE +# undef SQLITE_DEFAULT_PAGE_SIZE +# define SQLITE_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE +#endif + +/* +** Ordinarily, if no value is explicitly provided, SQLite creates databases +** with page size SQLITE_DEFAULT_PAGE_SIZE. However, based on certain +** device characteristics (sector-size and atomic write() support), +** SQLite may choose a larger value. This constant is the maximum value +** SQLite will choose on its own. +*/ +#ifndef SQLITE_MAX_DEFAULT_PAGE_SIZE +# define SQLITE_MAX_DEFAULT_PAGE_SIZE 8192 +#endif +#if SQLITE_MAX_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE +# undef SQLITE_MAX_DEFAULT_PAGE_SIZE +# define SQLITE_MAX_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE +#endif + + +/* +** Maximum number of pages in one database file. +** +** This is really just the default value for the max_page_count pragma. +** This value can be lowered (or raised) at run-time using that the +** max_page_count macro. +*/ +#ifndef SQLITE_MAX_PAGE_COUNT +# define SQLITE_MAX_PAGE_COUNT 1073741823 +#endif + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Maximum depth of recursion for triggers. +** +** A value of 1 means that a trigger program will not be able to itself +** fire any triggers. A value of 0 means that no trigger programs at all +** may be executed. +*/ +#ifndef SQLITE_MAX_TRIGGER_DEPTH +# define SQLITE_MAX_TRIGGER_DEPTH 1000 +#endif + +/************** End of sqliteLimit.h *****************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* Disable nuisance warnings on Borland compilers */ +#if defined(__BORLANDC__) +#pragma warn -rch /* unreachable code */ +#pragma warn -ccc /* Condition is always true or false */ +#pragma warn -aus /* Assigned value is never used */ +#pragma warn -csu /* Comparing signed and unsigned */ +#pragma warn -spa /* Suspicious pointer arithmetic */ +#endif + +/* Needed for various definitions... */ +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif + +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE +#endif + +/* +** Include standard header files as necessary +*/ +#ifdef HAVE_STDINT_H +#include +#endif +#ifdef HAVE_INTTYPES_H +#include +#endif + +/* +** The following macros are used to cast pointers to integers and +** integers to pointers. The way you do this varies from one compiler +** to the next, so we have developed the following set of #if statements +** to generate appropriate macros for a wide range of compilers. +** +** The correct "ANSI" way to do this is to use the intptr_t type. +** Unfortunately, that typedef is not available on all compilers, or +** if it is available, it requires an #include of specific headers +** that vary from one machine to the next. +** +** Ticket #3860: The llvm-gcc-4.2 compiler from Apple chokes on +** the ((void*)&((char*)0)[X]) construct. But MSVC chokes on ((void*)(X)). +** So we have to define the macros in different ways depending on the +** compiler. +*/ +#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +# define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) +#elif !defined(__GNUC__) /* Works for compilers other than LLVM */ +# define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) +# define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) +#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) +#else /* Generates a warning - but it always works */ +# define SQLITE_INT_TO_PTR(X) ((void*)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(X)) +#endif + +/* +** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. +** 0 means mutexes are permanently disable and the library is never +** threadsafe. 1 means the library is serialized which is the highest +** level of threadsafety. 2 means the library is multithreaded - multiple +** threads can use SQLite as long as no two threads try to use the same +** database connection at the same time. +** +** Older versions of SQLite used an optional THREADSAFE macro. +** We support that for legacy. +*/ +#if !defined(SQLITE_THREADSAFE) +# if defined(THREADSAFE) +# define SQLITE_THREADSAFE THREADSAFE +# else +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ +# endif +#endif + +/* +** Powersafe overwrite is on by default. But can be turned off using +** the -DSQLITE_POWERSAFE_OVERWRITE=0 command-line option. +*/ +#ifndef SQLITE_POWERSAFE_OVERWRITE +# define SQLITE_POWERSAFE_OVERWRITE 1 +#endif + +/* +** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. +** It determines whether or not the features related to +** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can +** be overridden at runtime using the sqlite3_config() API. +*/ +#if !defined(SQLITE_DEFAULT_MEMSTATUS) +# define SQLITE_DEFAULT_MEMSTATUS 1 +#endif + +/* +** Exactly one of the following macros must be defined in order to +** specify which memory allocation subsystem to use. +** +** SQLITE_SYSTEM_MALLOC // Use normal system malloc() +** SQLITE_WIN32_MALLOC // Use Win32 native heap API +** SQLITE_ZERO_MALLOC // Use a stub allocator that always fails +** SQLITE_MEMDEBUG // Debugging version of system malloc() +** +** On Windows, if the SQLITE_WIN32_MALLOC_VALIDATE macro is defined and the +** assert() macro is enabled, each call into the Win32 native heap subsystem +** will cause HeapValidate to be called. If heap validation should fail, an +** assertion will be triggered. +** +** If none of the above are defined, then set SQLITE_SYSTEM_MALLOC as +** the default. +*/ +#if defined(SQLITE_SYSTEM_MALLOC) \ + + defined(SQLITE_WIN32_MALLOC) \ + + defined(SQLITE_ZERO_MALLOC) \ + + defined(SQLITE_MEMDEBUG)>1 +# error "Two or more of the following compile-time configuration options\ + are defined but at most one is allowed:\ + SQLITE_SYSTEM_MALLOC, SQLITE_WIN32_MALLOC, SQLITE_MEMDEBUG,\ + SQLITE_ZERO_MALLOC" +#endif +#if defined(SQLITE_SYSTEM_MALLOC) \ + + defined(SQLITE_WIN32_MALLOC) \ + + defined(SQLITE_ZERO_MALLOC) \ + + defined(SQLITE_MEMDEBUG)==0 +# define SQLITE_SYSTEM_MALLOC 1 +#endif + +/* +** If SQLITE_MALLOC_SOFT_LIMIT is not zero, then try to keep the +** sizes of memory allocations below this value where possible. +*/ +#if !defined(SQLITE_MALLOC_SOFT_LIMIT) +# define SQLITE_MALLOC_SOFT_LIMIT 1024 +#endif + +/* +** We need to define _XOPEN_SOURCE as follows in order to enable +** recursive mutexes on most Unix systems and fchmod() on OpenBSD. +** But _XOPEN_SOURCE define causes problems for Mac OS X, so omit +** it. +*/ +#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) +# define _XOPEN_SOURCE 600 +#endif + +/* +** NDEBUG and SQLITE_DEBUG are opposites. It should always be true that +** defined(NDEBUG)==!defined(SQLITE_DEBUG). If this is not currently true, +** make it true by defining or undefining NDEBUG. +** +** Setting NDEBUG makes the code smaller and faster by disabling the +** assert() statements in the code. So we want the default action +** to be for NDEBUG to be set and NDEBUG to be undefined only if SQLITE_DEBUG +** is set. Thus NDEBUG becomes an opt-in rather than an opt-out +** feature. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif + +/* +** Enable SQLITE_ENABLE_EXPLAIN_COMMENTS if SQLITE_DEBUG is turned on. +*/ +#if !defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) && defined(SQLITE_DEBUG) +# define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 +#endif + +/* +** The testcase() macro is used to aid in coverage testing. When +** doing coverage testing, the condition inside the argument to +** testcase() must be evaluated both true and false in order to +** get full branch coverage. The testcase() macro is inserted +** to help ensure adequate test coverage in places where simple +** condition/decision coverage is inadequate. For example, testcase() +** can be used to make sure boundary values are tested. For +** bitmask tests, testcase() can be used to make sure each bit +** is significant and used at least once. On switch statements +** where multiple cases go to the same block of code, testcase() +** can insure that all cases are evaluated. +** +*/ +#ifdef SQLITE_COVERAGE_TEST +SQLITE_PRIVATE void sqlite3Coverage(int); +# define testcase(X) if( X ){ sqlite3Coverage(__LINE__); } +#else +# define testcase(X) +#endif + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +/* +** Sometimes we need a small amount of code such as a variable initialization +** to setup for a later assert() statement. We do not want this code to +** appear when assert() is disabled. The following macro is therefore +** used to contain that setup code. The "VVA" acronym stands for +** "Verification, Validation, and Accreditation". In other words, the +** code within VVA_ONLY() will only run during verification processes. +*/ +#ifndef NDEBUG +# define VVA_ONLY(X) X +#else +# define VVA_ONLY(X) +#endif + +/* +** The ALWAYS and NEVER macros surround boolean expressions which +** are intended to always be true or false, respectively. Such +** expressions could be omitted from the code completely. But they +** are included in a few cases in order to enhance the resilience +** of SQLite to unexpected behavior - to make the code "self-healing" +** or "ductile" rather than being "brittle" and crashing at the first +** hint of unplanned behavior. +** +** In other words, ALWAYS and NEVER are added for defensive code. +** +** When doing coverage testing ALWAYS and NEVER are hard-coded to +** be true and false so that the unreachable code they specify will +** not be counted as untested code. +*/ +#if defined(SQLITE_COVERAGE_TEST) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + +/* +** Return true (non-zero) if the input is a integer that is too large +** to fit in 32-bits. This macro is used inside of various testcase() +** macros to verify that we have tested SQLite for large-file support. +*/ +#define IS_BIG_INT(X) (((X)&~(i64)0xffffffff)!=0) + +/* +** The macro unlikely() is a hint that surrounds a boolean +** expression that is usually false. Macro likely() surrounds +** a boolean expression that is usually true. These hints could, +** in theory, be used by the compiler to generate better code, but +** currently they are just comments for human readers. +*/ +#define likely(X) (X) +#define unlikely(X) (X) + +/************** Include hash.h in the middle of sqliteInt.h ******************/ +/************** Begin file hash.h ********************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. +*/ +#ifndef _SQLITE_HASH_H_ +#define _SQLITE_HASH_H_ + +/* Forward declarations of structures. */ +typedef struct Hash Hash; +typedef struct HashElem HashElem; + +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, some of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +** +** All elements of the hash table are on a single doubly-linked list. +** Hash.first points to the head of this list. +** +** There are Hash.htsize buckets. Each bucket points to a spot in +** the global doubly-linked list. The contents of the bucket are the +** element pointed to plus the next _ht.count-1 elements in the list. +** +** Hash.htsize and Hash.ht may be zero. In that case lookup is done +** by a linear search of the global list. For small tables, the +** Hash.ht table is never allocated because if there are few elements +** in the table, it is faster to do a linear search than to manage +** the hash table. +*/ +struct Hash { + unsigned int htsize; /* Number of buckets in the hash table */ + unsigned int count; /* Number of entries in this table */ + HashElem *first; /* The first element of the array */ + struct _ht { /* the hash table */ + int count; /* Number of entries with this hash */ + HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; + +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct HashElem { + HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + const char *pKey; int nKey; /* Key associated with this element */ +}; + +/* +** Access routines. To delete, insert a NULL pointer. +*/ +SQLITE_PRIVATE void sqlite3HashInit(Hash*); +SQLITE_PRIVATE void *sqlite3HashInsert(Hash*, const char *pKey, int nKey, void *pData); +SQLITE_PRIVATE void *sqlite3HashFind(const Hash*, const char *pKey, int nKey); +SQLITE_PRIVATE void sqlite3HashClear(Hash*); + +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Hash h; +** HashElem *p; +** ... +** for(p=sqliteHashFirst(&h); p; p=sqliteHashNext(p)){ +** SomeStructure *pData = sqliteHashData(p); +** // do something with pData +** } +*/ +#define sqliteHashFirst(H) ((H)->first) +#define sqliteHashNext(E) ((E)->next) +#define sqliteHashData(E) ((E)->data) +/* #define sqliteHashKey(E) ((E)->pKey) // NOT USED */ +/* #define sqliteHashKeysize(E) ((E)->nKey) // NOT USED */ + +/* +** Number of entries in a hash table +*/ +/* #define sqliteHashCount(H) ((H)->count) // NOT USED */ + +#endif /* _SQLITE_HASH_H_ */ + +/************** End of hash.h ************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include parse.h in the middle of sqliteInt.h *****************/ +/************** Begin file parse.h *******************************************/ +#define TK_SEMI 1 +#define TK_EXPLAIN 2 +#define TK_QUERY 3 +#define TK_PLAN 4 +#define TK_BEGIN 5 +#define TK_TRANSACTION 6 +#define TK_DEFERRED 7 +#define TK_IMMEDIATE 8 +#define TK_EXCLUSIVE 9 +#define TK_COMMIT 10 +#define TK_END 11 +#define TK_ROLLBACK 12 +#define TK_SAVEPOINT 13 +#define TK_RELEASE 14 +#define TK_TO 15 +#define TK_TABLE 16 +#define TK_CREATE 17 +#define TK_IF 18 +#define TK_NOT 19 +#define TK_EXISTS 20 +#define TK_TEMP 21 +#define TK_LP 22 +#define TK_RP 23 +#define TK_AS 24 +#define TK_WITHOUT 25 +#define TK_COMMA 26 +#define TK_ID 27 +#define TK_INDEXED 28 +#define TK_ABORT 29 +#define TK_ACTION 30 +#define TK_AFTER 31 +#define TK_ANALYZE 32 +#define TK_ASC 33 +#define TK_ATTACH 34 +#define TK_BEFORE 35 +#define TK_BY 36 +#define TK_CASCADE 37 +#define TK_CAST 38 +#define TK_COLUMNKW 39 +#define TK_CONFLICT 40 +#define TK_DATABASE 41 +#define TK_DESC 42 +#define TK_DETACH 43 +#define TK_EACH 44 +#define TK_FAIL 45 +#define TK_FOR 46 +#define TK_IGNORE 47 +#define TK_INITIALLY 48 +#define TK_INSTEAD 49 +#define TK_LIKE_KW 50 +#define TK_MATCH 51 +#define TK_NO 52 +#define TK_KEY 53 +#define TK_OF 54 +#define TK_OFFSET 55 +#define TK_PRAGMA 56 +#define TK_RAISE 57 +#define TK_RECURSIVE 58 +#define TK_REPLACE 59 +#define TK_RESTRICT 60 +#define TK_ROW 61 +#define TK_TRIGGER 62 +#define TK_VACUUM 63 +#define TK_VIEW 64 +#define TK_VIRTUAL 65 +#define TK_WITH 66 +#define TK_REINDEX 67 +#define TK_RENAME 68 +#define TK_CTIME_KW 69 +#define TK_ANY 70 +#define TK_OR 71 +#define TK_AND 72 +#define TK_IS 73 +#define TK_BETWEEN 74 +#define TK_IN 75 +#define TK_ISNULL 76 +#define TK_NOTNULL 77 +#define TK_NE 78 +#define TK_EQ 79 +#define TK_GT 80 +#define TK_LE 81 +#define TK_LT 82 +#define TK_GE 83 +#define TK_ESCAPE 84 +#define TK_BITAND 85 +#define TK_BITOR 86 +#define TK_LSHIFT 87 +#define TK_RSHIFT 88 +#define TK_PLUS 89 +#define TK_MINUS 90 +#define TK_STAR 91 +#define TK_SLASH 92 +#define TK_REM 93 +#define TK_CONCAT 94 +#define TK_COLLATE 95 +#define TK_BITNOT 96 +#define TK_STRING 97 +#define TK_JOIN_KW 98 +#define TK_CONSTRAINT 99 +#define TK_DEFAULT 100 +#define TK_NULL 101 +#define TK_PRIMARY 102 +#define TK_UNIQUE 103 +#define TK_CHECK 104 +#define TK_REFERENCES 105 +#define TK_AUTOINCR 106 +#define TK_ON 107 +#define TK_INSERT 108 +#define TK_DELETE 109 +#define TK_UPDATE 110 +#define TK_SET 111 +#define TK_DEFERRABLE 112 +#define TK_FOREIGN 113 +#define TK_DROP 114 +#define TK_UNION 115 +#define TK_ALL 116 +#define TK_EXCEPT 117 +#define TK_INTERSECT 118 +#define TK_SELECT 119 +#define TK_VALUES 120 +#define TK_DISTINCT 121 +#define TK_DOT 122 +#define TK_FROM 123 +#define TK_JOIN 124 +#define TK_USING 125 +#define TK_ORDER 126 +#define TK_GROUP 127 +#define TK_HAVING 128 +#define TK_LIMIT 129 +#define TK_WHERE 130 +#define TK_INTO 131 +#define TK_INTEGER 132 +#define TK_FLOAT 133 +#define TK_BLOB 134 +#define TK_VARIABLE 135 +#define TK_CASE 136 +#define TK_WHEN 137 +#define TK_THEN 138 +#define TK_ELSE 139 +#define TK_INDEX 140 +#define TK_ALTER 141 +#define TK_ADD 142 +#define TK_TO_TEXT 143 +#define TK_TO_BLOB 144 +#define TK_TO_NUMERIC 145 +#define TK_TO_INT 146 +#define TK_TO_REAL 147 +#define TK_ISNOT 148 +#define TK_END_OF_FILE 149 +#define TK_ILLEGAL 150 +#define TK_SPACE 151 +#define TK_UNCLOSED_STRING 152 +#define TK_FUNCTION 153 +#define TK_COLUMN 154 +#define TK_AGG_FUNCTION 155 +#define TK_AGG_COLUMN 156 +#define TK_UMINUS 157 +#define TK_UPLUS 158 +#define TK_REGISTER 159 + +/************** End of parse.h ***********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +#include +#include +#include +#include +#include + +/* +** If compiling for a processor that lacks floating point support, +** substitute integer for floating-point +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# define double sqlite_int64 +# define float sqlite_int64 +# define LONGDOUBLE_TYPE sqlite_int64 +# ifndef SQLITE_BIG_DBL +# define SQLITE_BIG_DBL (((sqlite3_int64)1)<<50) +# endif +# define SQLITE_OMIT_DATETIME_FUNCS 1 +# define SQLITE_OMIT_TRACE 1 +# undef SQLITE_MIXED_ENDIAN_64BIT_FLOAT +# undef SQLITE_HAVE_ISNAN +#endif +#ifndef SQLITE_BIG_DBL +# define SQLITE_BIG_DBL (1e99) +#endif + +/* +** OMIT_TEMPDB is set to 1 if SQLITE_OMIT_TEMPDB is defined, or 0 +** afterward. Having this macro allows us to cause the C compiler +** to omit code used by TEMP tables without messy #ifndef statements. +*/ +#ifdef SQLITE_OMIT_TEMPDB +#define OMIT_TEMPDB 1 +#else +#define OMIT_TEMPDB 0 +#endif + +/* +** The "file format" number is an integer that is incremented whenever +** the VDBE-level file format changes. The following macros define the +** the default file format for new databases and the maximum file format +** that the library can read. +*/ +#define SQLITE_MAX_FILE_FORMAT 4 +#ifndef SQLITE_DEFAULT_FILE_FORMAT +# define SQLITE_DEFAULT_FILE_FORMAT 4 +#endif + +/* +** Determine whether triggers are recursive by default. This can be +** changed at run-time using a pragma. +*/ +#ifndef SQLITE_DEFAULT_RECURSIVE_TRIGGERS +# define SQLITE_DEFAULT_RECURSIVE_TRIGGERS 0 +#endif + +/* +** Provide a default value for SQLITE_TEMP_STORE in case it is not specified +** on the command-line +*/ +#ifndef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 1 +# define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ +#endif + +/* +** GCC does not define the offsetof() macro so we'll have to do it +** ourselves. +*/ +#ifndef offsetof +#define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) +#endif + +/* +** Macros to compute minimum and maximum of two numbers. +*/ +#define MIN(A,B) ((A)<(B)?(A):(B)) +#define MAX(A,B) ((A)>(B)?(A):(B)) + +/* +** Check to see if this machine uses EBCDIC. (Yes, believe it or +** not, there are still machines out there that use EBCDIC.) +*/ +#if 'A' == '\301' +# define SQLITE_EBCDIC 1 +#else +# define SQLITE_ASCII 1 +#endif + +/* +** Integers of known sizes. These typedefs might change for architectures +** where the sizes very. Preprocessor macros are available so that the +** types can be conveniently redefined at compile-type. Like this: +** +** cc '-DUINTPTR_TYPE=long long int' ... +*/ +#ifndef UINT32_TYPE +# ifdef HAVE_UINT32_T +# define UINT32_TYPE uint32_t +# else +# define UINT32_TYPE unsigned int +# endif +#endif +#ifndef UINT16_TYPE +# ifdef HAVE_UINT16_T +# define UINT16_TYPE uint16_t +# else +# define UINT16_TYPE unsigned short int +# endif +#endif +#ifndef INT16_TYPE +# ifdef HAVE_INT16_T +# define INT16_TYPE int16_t +# else +# define INT16_TYPE short int +# endif +#endif +#ifndef UINT8_TYPE +# ifdef HAVE_UINT8_T +# define UINT8_TYPE uint8_t +# else +# define UINT8_TYPE unsigned char +# endif +#endif +#ifndef INT8_TYPE +# ifdef HAVE_INT8_T +# define INT8_TYPE int8_t +# else +# define INT8_TYPE signed char +# endif +#endif +#ifndef LONGDOUBLE_TYPE +# define LONGDOUBLE_TYPE long double +#endif +typedef sqlite_int64 i64; /* 8-byte signed integer */ +typedef sqlite_uint64 u64; /* 8-byte unsigned integer */ +typedef UINT32_TYPE u32; /* 4-byte unsigned integer */ +typedef UINT16_TYPE u16; /* 2-byte unsigned integer */ +typedef INT16_TYPE i16; /* 2-byte signed integer */ +typedef UINT8_TYPE u8; /* 1-byte unsigned integer */ +typedef INT8_TYPE i8; /* 1-byte signed integer */ + +/* +** SQLITE_MAX_U32 is a u64 constant that is the maximum u64 value +** that can be stored in a u32 without loss of data. The value +** is 0x00000000ffffffff. But because of quirks of some compilers, we +** have to specify the value in the less intuitive manner shown: +*/ +#define SQLITE_MAX_U32 ((((u64)1)<<32)-1) + +/* +** The datatype used to store estimates of the number of rows in a +** table or index. This is an unsigned integer type. For 99.9% of +** the world, a 32-bit integer is sufficient. But a 64-bit integer +** can be used at compile-time if desired. +*/ +#ifdef SQLITE_64BIT_STATS + typedef u64 tRowcnt; /* 64-bit only if requested at compile-time */ +#else + typedef u32 tRowcnt; /* 32-bit is the default */ +#endif + +/* +** Estimated quantities used for query planning are stored as 16-bit +** logarithms. For quantity X, the value stored is 10*log2(X). This +** gives a possible range of values of approximately 1.0e986 to 1e-986. +** But the allowed values are "grainy". Not every value is representable. +** For example, quantities 16 and 17 are both represented by a LogEst +** of 40. However, since LogEst quantaties are suppose to be estimates, +** not exact values, this imprecision is not a problem. +** +** "LogEst" is short for "Logarithmic Estimate". +** +** Examples: +** 1 -> 0 20 -> 43 10000 -> 132 +** 2 -> 10 25 -> 46 25000 -> 146 +** 3 -> 16 100 -> 66 1000000 -> 199 +** 4 -> 20 1000 -> 99 1048576 -> 200 +** 10 -> 33 1024 -> 100 4294967296 -> 320 +** +** The LogEst can be negative to indicate fractional values. +** Examples: +** +** 0.5 -> -10 0.1 -> -33 0.0625 -> -40 +*/ +typedef INT16_TYPE LogEst; + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. +*/ +#ifdef SQLITE_AMALGAMATION +SQLITE_PRIVATE const int sqlite3one = 1; +#else +SQLITE_PRIVATE const int sqlite3one; +#endif +#if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) +# define SQLITE_BYTEORDER 1234 +# define SQLITE_BIGENDIAN 0 +# define SQLITE_LITTLEENDIAN 1 +# define SQLITE_UTF16NATIVE SQLITE_UTF16LE +#endif +#if (defined(sparc) || defined(__ppc__)) \ + && !defined(SQLITE_RUNTIME_BYTEORDER) +# define SQLITE_BYTEORDER 4321 +# define SQLITE_BIGENDIAN 1 +# define SQLITE_LITTLEENDIAN 0 +# define SQLITE_UTF16NATIVE SQLITE_UTF16BE +#endif +#if !defined(SQLITE_BYTEORDER) +# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ +# define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) +# define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) +# define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) +#endif + +/* +** Constants for the largest and smallest possible 64-bit signed integers. +** These macros are designed to work correctly on both 32-bit and 64-bit +** compilers. +*/ +#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +/* +** Round up a number to the next larger multiple of 8. This is used +** to force 8-byte alignment on 64-bit architectures. +*/ +#define ROUND8(x) (((x)+7)&~7) + +/* +** Round down to the nearest multiple of 8 +*/ +#define ROUNDDOWN8(x) ((x)&~7) + +/* +** Assert that the pointer X is aligned to an 8-byte boundary. This +** macro is used only within assert() to verify that the code gets +** all alignment restrictions correct. +** +** Except, if SQLITE_4_BYTE_ALIGNED_MALLOC is defined, then the +** underlying malloc() implemention might return us 4-byte aligned +** pointers. In that case, only verify 4-byte alignment. +*/ +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC +# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&3)==0) +#else +# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) +#endif + +/* +** Disable MMAP on platforms where it is known to not work +*/ +#if defined(__OpenBSD__) || defined(__QNXNTO__) +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +#endif + +/* +** Default maximum size of memory used by memory-mapped I/O in the VFS +*/ +#ifdef __APPLE__ +# include +# if TARGET_OS_IPHONE +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif +#ifndef SQLITE_MAX_MMAP_SIZE +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ +# else +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +# define SQLITE_MAX_MMAP_SIZE_xc 1 /* exclude from ctime.c */ +#endif + +/* +** The default MMAP_SIZE is zero on all platforms. Or, even if a larger +** default MMAP_SIZE is specified at compile-time, make sure that it does +** not exceed the maximum mmap size. +*/ +#ifndef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE 0 +# define SQLITE_DEFAULT_MMAP_SIZE_xc 1 /* Exclude from ctime.c */ +#endif +#if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE +# undef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE +#endif + +/* +** Only one of SQLITE_ENABLE_STAT3 or SQLITE_ENABLE_STAT4 can be defined. +** Priority is given to SQLITE_ENABLE_STAT4. If either are defined, also +** define SQLITE_ENABLE_STAT3_OR_STAT4 +*/ +#ifdef SQLITE_ENABLE_STAT4 +# undef SQLITE_ENABLE_STAT3 +# define SQLITE_ENABLE_STAT3_OR_STAT4 1 +#elif SQLITE_ENABLE_STAT3 +# define SQLITE_ENABLE_STAT3_OR_STAT4 1 +#elif SQLITE_ENABLE_STAT3_OR_STAT4 +# undef SQLITE_ENABLE_STAT3_OR_STAT4 +#endif + +/* +** An instance of the following structure is used to store the busy-handler +** callback for a given sqlite handle. +** +** The sqlite.busyHandler member of the sqlite struct contains the busy +** callback for the database handle. Each pager opened via the sqlite +** handle is passed a pointer to sqlite.busyHandler. The busy-handler +** callback is currently invoked only from within pager.c. +*/ +typedef struct BusyHandler BusyHandler; +struct BusyHandler { + int (*xFunc)(void *,int); /* The busy callback */ + void *pArg; /* First arg to busy callback */ + int nBusy; /* Incremented with each busy call */ +}; + +/* +** Name of the master database table. The master database table +** is a special table that holds the names and attributes of all +** user tables and indices. +*/ +#define MASTER_NAME "sqlite_master" +#define TEMP_MASTER_NAME "sqlite_temp_master" + +/* +** The root-page of the master database table. +*/ +#define MASTER_ROOT 1 + +/* +** The name of the schema table. +*/ +#define SCHEMA_TABLE(x) ((!OMIT_TEMPDB)&&(x==1)?TEMP_MASTER_NAME:MASTER_NAME) + +/* +** A convenience macro that returns the number of elements in +** an array. +*/ +#define ArraySize(X) ((int)(sizeof(X)/sizeof(X[0]))) + +/* +** Determine if the argument is a power of two +*/ +#define IsPowerOfTwo(X) (((X)&((X)-1))==0) + +/* +** The following value as a destructor means to use sqlite3DbFree(). +** The sqlite3DbFree() routine requires two parameters instead of the +** one parameter that destructors normally want. So we have to introduce +** this magic value that the code knows to handle differently. Any +** pointer will work here as long as it is distinct from SQLITE_STATIC +** and SQLITE_TRANSIENT. +*/ +#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3MallocSize) + +/* +** When SQLITE_OMIT_WSD is defined, it means that the target platform does +** not support Writable Static Data (WSD) such as global and static variables. +** All variables must either be on the stack or dynamically allocated from +** the heap. When WSD is unsupported, the variable declarations scattered +** throughout the SQLite code must become constants instead. The SQLITE_WSD +** macro is used for this purpose. And instead of referencing the variable +** directly, we use its constant as a key to lookup the run-time allocated +** buffer that holds real variable. The constant is also the initializer +** for the run-time allocated buffer. +** +** In the usual case where WSD is supported, the SQLITE_WSD and GLOBAL +** macros become no-ops and have zero performance impact. +*/ +#ifdef SQLITE_OMIT_WSD + #define SQLITE_WSD const + #define GLOBAL(t,v) (*(t*)sqlite3_wsd_find((void*)&(v), sizeof(v))) + #define sqlite3GlobalConfig GLOBAL(struct Sqlite3Config, sqlite3Config) +SQLITE_API int sqlite3_wsd_init(int N, int J); +SQLITE_API void *sqlite3_wsd_find(void *K, int L); +#else + #define SQLITE_WSD + #define GLOBAL(t,v) v + #define sqlite3GlobalConfig sqlite3Config +#endif + +/* +** The following macros are used to suppress compiler warnings and to +** make it clear to human readers when a function parameter is deliberately +** left unused within the body of a function. This usually happens when +** a function is called via a function pointer. For example the +** implementation of an SQL aggregate step callback may not use the +** parameter indicating the number of arguments passed to the aggregate, +** if it knows that this is enforced elsewhere. +** +** When a function parameter is not used at all within the body of a function, +** it is generally named "NotUsed" or "NotUsed2" to make things even clearer. +** However, these macros may also be used to suppress warnings related to +** parameters that may or may not be used depending on compilation options. +** For example those parameters only used in assert() statements. In these +** cases the parameters are named as per the usual conventions. +*/ +#define UNUSED_PARAMETER(x) (void)(x) +#define UNUSED_PARAMETER2(x,y) UNUSED_PARAMETER(x),UNUSED_PARAMETER(y) + +/* +** Forward references to structures +*/ +typedef struct AggInfo AggInfo; +typedef struct AuthContext AuthContext; +typedef struct AutoincInfo AutoincInfo; +typedef struct Bitvec Bitvec; +typedef struct CollSeq CollSeq; +typedef struct Column Column; +typedef struct Db Db; +typedef struct Schema Schema; +typedef struct Expr Expr; +typedef struct ExprList ExprList; +typedef struct ExprSpan ExprSpan; +typedef struct FKey FKey; +typedef struct FuncDestructor FuncDestructor; +typedef struct FuncDef FuncDef; +typedef struct FuncDefHash FuncDefHash; +typedef struct IdList IdList; +typedef struct Index Index; +typedef struct IndexSample IndexSample; +typedef struct KeyClass KeyClass; +typedef struct KeyInfo KeyInfo; +typedef struct Lookaside Lookaside; +typedef struct LookasideSlot LookasideSlot; +typedef struct Module Module; +typedef struct NameContext NameContext; +typedef struct Parse Parse; +typedef struct PrintfArguments PrintfArguments; +typedef struct RowSet RowSet; +typedef struct Savepoint Savepoint; +typedef struct Select Select; +typedef struct SelectDest SelectDest; +typedef struct SrcList SrcList; +typedef struct StrAccum StrAccum; +typedef struct Table Table; +typedef struct TableLock TableLock; +typedef struct Token Token; +typedef struct Trigger Trigger; +typedef struct TriggerPrg TriggerPrg; +typedef struct TriggerStep TriggerStep; +typedef struct UnpackedRecord UnpackedRecord; +typedef struct VTable VTable; +typedef struct VtabCtx VtabCtx; +typedef struct Walker Walker; +typedef struct WhereInfo WhereInfo; +typedef struct With With; + +/* +** Defer sourcing vdbe.h and btree.h until after the "u8" and +** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque +** pointer types (i.e. FuncDef) defined above. +*/ +/************** Include btree.h in the middle of sqliteInt.h *****************/ +/************** Begin file btree.h *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite B-Tree file +** subsystem. See comments in the source code for a detailed description +** of what each interface routine does. +*/ +#ifndef _BTREE_H_ +#define _BTREE_H_ + +/* TODO: This definition is just included so other modules compile. It +** needs to be revisited. +*/ +#define SQLITE_N_BTREE_META 10 + +/* +** If defined as non-zero, auto-vacuum is enabled by default. Otherwise +** it must be turned on for each database using "PRAGMA auto_vacuum = 1". +*/ +#ifndef SQLITE_DEFAULT_AUTOVACUUM + #define SQLITE_DEFAULT_AUTOVACUUM 0 +#endif + +#define BTREE_AUTOVACUUM_NONE 0 /* Do not do auto-vacuum */ +#define BTREE_AUTOVACUUM_FULL 1 /* Do full auto-vacuum */ +#define BTREE_AUTOVACUUM_INCR 2 /* Incremental vacuum */ + +/* +** Forward declarations of structure +*/ +typedef struct Btree Btree; +typedef struct BtCursor BtCursor; +typedef struct BtShared BtShared; + + +SQLITE_PRIVATE int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use with this b-tree */ + const char *zFilename, /* Name of database file to open */ + sqlite3 *db, /* Associated database connection */ + Btree **ppBtree, /* Return open Btree* here */ + int flags, /* Flags */ + int vfsFlags /* Flags passed through to VFS open */ +); + +/* The flags parameter to sqlite3BtreeOpen can be the bitwise or of the +** following values. +** +** NOTE: These values must match the corresponding PAGER_ values in +** pager.h. +*/ +#define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ +#define BTREE_MEMORY 2 /* This is an in-memory DB */ +#define BTREE_SINGLE 4 /* The file contains at most 1 b-tree */ +#define BTREE_UNORDERED 8 /* Use of a hash implementation is OK */ + +SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); +SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); +#if SQLITE_MAX_MMAP_SIZE>0 +SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#endif +SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags(Btree*,unsigned); +SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*); +SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); +SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); +SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); +SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree*); +SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree*); +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); +#endif +SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); +SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int); +SQLITE_PRIVATE int sqlite3BtreeCommit(Btree*); +SQLITE_PRIVATE int sqlite3BtreeRollback(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree*, int*, int flags); +SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree*); +SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree*); +SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree*); +SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); +SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *pBtree); +SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock); +SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *, int, int); + +SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *); +SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *); +SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *, Btree *); + +SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); + +/* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR +** of the flags shown below. +** +** Every SQLite table must have either BTREE_INTKEY or BTREE_BLOBKEY set. +** With BTREE_INTKEY, the table key is a 64-bit integer and arbitrary data +** is stored in the leaves. (BTREE_INTKEY is used for SQL tables.) With +** BTREE_BLOBKEY, the key is an arbitrary BLOB and no content is stored +** anywhere - the key is the content. (BTREE_BLOBKEY is used for SQL +** indices.) +*/ +#define BTREE_INTKEY 1 /* Table has only 64-bit signed integer keys */ +#define BTREE_BLOBKEY 2 /* Table has keys only - no data */ + +SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); +SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, int*); +SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree*, int); + +SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); +SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); + +SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); + +/* +** The second parameter to sqlite3BtreeGetMeta or sqlite3BtreeUpdateMeta +** should be one of the following values. The integer values are assigned +** to constants so that the offset of the corresponding field in an +** SQLite database header may be found using the following formula: +** +** offset = 36 + (idx * 4) +** +** For example, the free-page-count field is located at byte offset 36 of +** the database file header. The incr-vacuum-flag field is located at +** byte offset 64 (== 36+4*7). +*/ +#define BTREE_FREE_PAGE_COUNT 0 +#define BTREE_SCHEMA_VERSION 1 +#define BTREE_FILE_FORMAT 2 +#define BTREE_DEFAULT_CACHE_SIZE 3 +#define BTREE_LARGEST_ROOT_PAGE 4 +#define BTREE_TEXT_ENCODING 5 +#define BTREE_USER_VERSION 6 +#define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 + +/* +** Values that may be OR'd together to form the second argument of an +** sqlite3BtreeCursorHints() call. +*/ +#define BTREE_BULKLOAD 0x00000001 + +SQLITE_PRIVATE int sqlite3BtreeCursor( + Btree*, /* BTree containing table to open */ + int iTable, /* Index of root page */ + int wrFlag, /* 1 for writing. 0 for read-only */ + struct KeyInfo*, /* First argument to compare function */ + BtCursor *pCursor /* Space to write cursor structure */ +); +SQLITE_PRIVATE int sqlite3BtreeCursorSize(void); +SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*); + +SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( + BtCursor*, + UnpackedRecord *pUnKey, + i64 intKey, + int bias, + int *pRes +); +SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*, int*); +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, + const void *pData, int nData, + int nZero, int bias, int seekResult); +SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor*, i64 *pSize); +SQLITE_PRIVATE int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*); +SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt); +SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt); +SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor*, u32 *pSize); +SQLITE_PRIVATE int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*); + +SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); +SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); + +SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); +SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); +SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); +SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); +SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); +SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); + +#ifndef NDEBUG +SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*); +#endif + +#ifndef SQLITE_OMIT_BTREECOUNT +SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *, i64 *); +#endif + +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3BtreeCursorInfo(BtCursor*, int*, int); +SQLITE_PRIVATE void sqlite3BtreeCursorList(Btree*); +#endif + +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); +#endif + +/* +** If we are not using shared cache, then there is no need to +** use mutexes to access the BtShared structures. So make the +** Enter and Leave procedures no-ops. +*/ +#ifndef SQLITE_OMIT_SHARED_CACHE +SQLITE_PRIVATE void sqlite3BtreeEnter(Btree*); +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3*); +#else +# define sqlite3BtreeEnter(X) +# define sqlite3BtreeEnterAll(X) +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); +SQLITE_PRIVATE void sqlite3BtreeLeave(Btree*); +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3*); +#ifndef NDEBUG + /* These routines are used inside assert() statements only. */ +SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree*); +SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3*); +SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); +#endif +#else + +# define sqlite3BtreeSharable(X) 0 +# define sqlite3BtreeLeave(X) +# define sqlite3BtreeEnterCursor(X) +# define sqlite3BtreeLeaveCursor(X) +# define sqlite3BtreeLeaveAll(X) + +# define sqlite3BtreeHoldsMutex(X) 1 +# define sqlite3BtreeHoldsAllMutexes(X) 1 +# define sqlite3SchemaMutexHeld(X,Y,Z) 1 +#endif + + +#endif /* _BTREE_H_ */ + +/************** End of btree.h ***********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include vdbe.h in the middle of sqliteInt.h ******************/ +/************** Begin file vdbe.h ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Header file for the Virtual DataBase Engine (VDBE) +** +** This header defines the interface to the virtual database engine +** or VDBE. The VDBE implements an abstract machine that runs a +** simple program to access and modify the underlying database. +*/ +#ifndef _SQLITE_VDBE_H_ +#define _SQLITE_VDBE_H_ +/* #include */ + +/* +** A single VDBE is an opaque structure named "Vdbe". Only routines +** in the source file sqliteVdbe.c are allowed to see the insides +** of this structure. +*/ +typedef struct Vdbe Vdbe; + +/* +** The names of the following types declared in vdbeInt.h are required +** for the VdbeOp definition. +*/ +typedef struct Mem Mem; +typedef struct SubProgram SubProgram; + +/* +** A single instruction of the virtual machine has an opcode +** and as many as three operands. The instruction is recorded +** as an instance of the following structure: +*/ +struct VdbeOp { + u8 opcode; /* What operation to perform */ + signed char p4type; /* One of the P4_xxx constants for p4 */ + u8 opflags; /* Mask of the OPFLG_* flags in opcodes.h */ + u8 p5; /* Fifth parameter is an unsigned character */ + int p1; /* First operand */ + int p2; /* Second parameter (often the jump destination) */ + int p3; /* The third parameter */ + union { /* fourth parameter */ + int i; /* Integer value if p4type==P4_INT32 */ + void *p; /* Generic pointer */ + char *z; /* Pointer to data for string (char array) types */ + i64 *pI64; /* Used when p4type is P4_INT64 */ + double *pReal; /* Used when p4type is P4_REAL */ + FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ + CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ + Mem *pMem; /* Used when p4type is P4_MEM */ + VTable *pVtab; /* Used when p4type is P4_VTAB */ + KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ + int *ai; /* Used when p4type is P4_INTARRAY */ + SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + int (*xAdvance)(BtCursor *, int *); + } p4; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + char *zComment; /* Comment to improve readability */ +#endif +#ifdef VDBE_PROFILE + u32 cnt; /* Number of times this instruction was executed */ + u64 cycles; /* Total time spent executing this instruction */ +#endif +#ifdef SQLITE_VDBE_COVERAGE + int iSrcLine; /* Source-code line that generated this opcode */ +#endif +}; +typedef struct VdbeOp VdbeOp; + + +/* +** A sub-routine used to implement a trigger program. +*/ +struct SubProgram { + VdbeOp *aOp; /* Array of opcodes for sub-program */ + int nOp; /* Elements in aOp[] */ + int nMem; /* Number of memory cells required */ + int nCsr; /* Number of cursors required */ + int nOnce; /* Number of OP_Once instructions */ + void *token; /* id that may be used to recursive triggers */ + SubProgram *pNext; /* Next sub-program already visited */ +}; + +/* +** A smaller version of VdbeOp used for the VdbeAddOpList() function because +** it takes up less space. +*/ +struct VdbeOpList { + u8 opcode; /* What operation to perform */ + signed char p1; /* First operand */ + signed char p2; /* Second parameter (often the jump destination) */ + signed char p3; /* Third parameter */ +}; +typedef struct VdbeOpList VdbeOpList; + +/* +** Allowed values of VdbeOp.p4type +*/ +#define P4_NOTUSED 0 /* The P4 parameter is not used */ +#define P4_DYNAMIC (-1) /* Pointer to a string obtained from sqliteMalloc() */ +#define P4_STATIC (-2) /* Pointer to a static string */ +#define P4_COLLSEQ (-4) /* P4 is a pointer to a CollSeq structure */ +#define P4_FUNCDEF (-5) /* P4 is a pointer to a FuncDef structure */ +#define P4_KEYINFO (-6) /* P4 is a pointer to a KeyInfo structure */ +#define P4_MEM (-8) /* P4 is a pointer to a Mem* structure */ +#define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ +#define P4_VTAB (-10) /* P4 is a pointer to an sqlite3_vtab structure */ +#define P4_MPRINTF (-11) /* P4 is a string obtained from sqlite3_mprintf() */ +#define P4_REAL (-12) /* P4 is a 64-bit floating point value */ +#define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ +#define P4_INT32 (-14) /* P4 is a 32-bit signed integer */ +#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ +#define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ +#define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ + +/* Error message codes for OP_Halt */ +#define P5_ConstraintNotNull 1 +#define P5_ConstraintUnique 2 +#define P5_ConstraintCheck 3 +#define P5_ConstraintFK 4 + +/* +** The Vdbe.aColName array contains 5n Mem structures, where n is the +** number of columns of data returned by the statement. +*/ +#define COLNAME_NAME 0 +#define COLNAME_DECLTYPE 1 +#define COLNAME_DATABASE 2 +#define COLNAME_TABLE 3 +#define COLNAME_COLUMN 4 +#ifdef SQLITE_ENABLE_COLUMN_METADATA +# define COLNAME_N 5 /* Number of COLNAME_xxx symbols */ +#else +# ifdef SQLITE_OMIT_DECLTYPE +# define COLNAME_N 1 /* Store only the name */ +# else +# define COLNAME_N 2 /* Store the name and decltype */ +# endif +#endif + +/* +** The following macro converts a relative address in the p2 field +** of a VdbeOp structure into a negative number so that +** sqlite3VdbeAddOpList() knows that the address is relative. Calling +** the macro again restores the address. +*/ +#define ADDR(X) (-1-(X)) + +/* +** The makefile scans the vdbe.c source file and creates the "opcodes.h" +** header file that defines a number for each opcode used by the VDBE. +*/ +/************** Include opcodes.h in the middle of vdbe.h ********************/ +/************** Begin file opcodes.h *****************************************/ +/* Automatically generated. Do not edit */ +/* See the mkopcodeh.awk script for details */ +#define OP_Function 1 /* synopsis: r[P3]=func(r[P2@P5]) */ +#define OP_Savepoint 2 +#define OP_AutoCommit 3 +#define OP_Transaction 4 +#define OP_SorterNext 5 +#define OP_PrevIfOpen 6 +#define OP_NextIfOpen 7 +#define OP_Prev 8 +#define OP_Next 9 +#define OP_AggStep 10 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_Checkpoint 11 +#define OP_JournalMode 12 +#define OP_Vacuum 13 +#define OP_VFilter 14 /* synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 15 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 16 +#define OP_Gosub 17 +#define OP_Return 18 +#define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ +#define OP_InitCoroutine 20 +#define OP_EndCoroutine 21 +#define OP_Yield 22 +#define OP_HaltIfNull 23 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 24 +#define OP_Integer 25 /* synopsis: r[P2]=P1 */ +#define OP_Int64 26 /* synopsis: r[P2]=P4 */ +#define OP_String 27 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 28 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 29 /* synopsis: r[P1]=NULL */ +#define OP_Blob 30 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 31 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 32 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 33 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 34 /* synopsis: r[P2]=r[P1] */ +#define OP_ResultRow 35 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 36 +#define OP_AddImm 37 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_MustBeInt 38 +#define OP_RealAffinity 39 +#define OP_Permutation 40 +#define OP_Compare 41 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_Jump 42 +#define OP_Once 43 +#define OP_If 44 +#define OP_IfNot 45 +#define OP_Column 46 /* synopsis: r[P3]=PX */ +#define OP_Affinity 47 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 48 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 49 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 50 +#define OP_SetCookie 51 +#define OP_ReopenIdx 52 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 53 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 54 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenAutoindex 55 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 56 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 57 +#define OP_OpenPseudo 58 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 59 +#define OP_SeekLT 60 /* synopsis: key=r[P3@P4] */ +#define OP_SeekLE 61 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGE 62 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGT 63 /* synopsis: key=r[P3@P4] */ +#define OP_Seek 64 /* synopsis: intkey=r[P2] */ +#define OP_NoConflict 65 /* synopsis: key=r[P3@P4] */ +#define OP_NotFound 66 /* synopsis: key=r[P3@P4] */ +#define OP_Found 67 /* synopsis: key=r[P3@P4] */ +#define OP_NotExists 68 /* synopsis: intkey=r[P3] */ +#define OP_Sequence 69 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 70 /* synopsis: r[P2]=rowid */ +#define OP_Or 71 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ +#define OP_And 72 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ +#define OP_Insert 73 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_InsertInt 74 /* synopsis: intkey=P3 data=r[P2] */ +#define OP_Delete 75 +#define OP_IsNull 76 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ +#define OP_NotNull 77 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ +#define OP_Ne 78 /* same as TK_NE, synopsis: if r[P1]!=r[P3] goto P2 */ +#define OP_Eq 79 /* same as TK_EQ, synopsis: if r[P1]==r[P3] goto P2 */ +#define OP_Gt 80 /* same as TK_GT, synopsis: if r[P1]>r[P3] goto P2 */ +#define OP_Le 81 /* same as TK_LE, synopsis: if r[P1]<=r[P3] goto P2 */ +#define OP_Lt 82 /* same as TK_LT, synopsis: if r[P1]=r[P3] goto P2 */ +#define OP_ResetCount 84 +#define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 89 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 90 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 91 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 92 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 93 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 94 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_SorterCompare 95 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_BitNot 96 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ +#define OP_String8 97 /* same as TK_STRING, synopsis: r[P2]='P4' */ +#define OP_SorterData 98 /* synopsis: r[P2]=data */ +#define OP_RowKey 99 /* synopsis: r[P2]=key */ +#define OP_RowData 100 /* synopsis: r[P2]=data */ +#define OP_Rowid 101 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 102 +#define OP_Last 103 +#define OP_SorterSort 104 +#define OP_Sort 105 +#define OP_Rewind 106 +#define OP_SorterInsert 107 +#define OP_IdxInsert 108 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 109 /* synopsis: key=r[P2@P3] */ +#define OP_IdxRowid 110 /* synopsis: r[P2]=rowid */ +#define OP_IdxLE 111 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGT 112 /* synopsis: key=r[P3@P4] */ +#define OP_IdxLT 113 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGE 114 /* synopsis: key=r[P3@P4] */ +#define OP_Destroy 115 +#define OP_Clear 116 +#define OP_ResetSorter 117 +#define OP_CreateIndex 118 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_CreateTable 119 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_ParseSchema 120 +#define OP_LoadAnalysis 121 +#define OP_DropTable 122 +#define OP_DropIndex 123 +#define OP_DropTrigger 124 +#define OP_IntegrityCk 125 +#define OP_RowSetAdd 126 /* synopsis: rowset(P1)=r[P2] */ +#define OP_RowSetRead 127 /* synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 128 /* synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 129 +#define OP_Param 130 +#define OP_FkCounter 131 /* synopsis: fkctr[P1]+=P2 */ +#define OP_FkIfZero 132 /* synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_Real 133 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_MemMax 134 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_IfPos 135 /* synopsis: if r[P1]>0 goto P2 */ +#define OP_IfNeg 136 /* synopsis: r[P1]+=P3, if r[P1]<0 goto P2 */ +#define OP_IfZero 137 /* synopsis: r[P1]+=P3, if r[P1]==0 goto P2 */ +#define OP_AggFinal 138 /* synopsis: accum=r[P1] N=P2 */ +#define OP_IncrVacuum 139 +#define OP_Expire 140 +#define OP_TableLock 141 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 142 +#define OP_ToText 143 /* same as TK_TO_TEXT */ +#define OP_ToBlob 144 /* same as TK_TO_BLOB */ +#define OP_ToNumeric 145 /* same as TK_TO_NUMERIC */ +#define OP_ToInt 146 /* same as TK_TO_INT */ +#define OP_ToReal 147 /* same as TK_TO_REAL */ +#define OP_VCreate 148 +#define OP_VDestroy 149 +#define OP_VOpen 150 +#define OP_VColumn 151 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VNext 152 +#define OP_VRename 153 +#define OP_Pagecount 154 +#define OP_MaxPgcnt 155 +#define OP_Init 156 /* synopsis: Start at P2 */ +#define OP_Noop 157 +#define OP_Explain 158 + + +/* Properties such as "out2" or "jump" that are specified in +** comments following the "case" for each opcode in the vdbe.c +** are encoded into bitvectors as follows: +*/ +#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */ +#define OPFLG_OUT2_PRERELEASE 0x0002 /* out2-prerelease: */ +#define OPFLG_IN1 0x0004 /* in1: P1 is an input */ +#define OPFLG_IN2 0x0008 /* in2: P2 is an input */ +#define OPFLG_IN3 0x0010 /* in3: P3 is an input */ +#define OPFLG_OUT2 0x0020 /* out2: P2 is an output */ +#define OPFLG_OUT3 0x0040 /* out3: P3 is an output */ +#define OPFLG_INITIALIZER {\ +/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,\ +/* 8 */ 0x01, 0x01, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00,\ +/* 16 */ 0x01, 0x01, 0x04, 0x24, 0x01, 0x04, 0x05, 0x10,\ +/* 24 */ 0x00, 0x02, 0x02, 0x02, 0x02, 0x00, 0x02, 0x02,\ +/* 32 */ 0x00, 0x00, 0x20, 0x00, 0x00, 0x04, 0x05, 0x04,\ +/* 40 */ 0x00, 0x00, 0x01, 0x01, 0x05, 0x05, 0x00, 0x00,\ +/* 48 */ 0x00, 0x02, 0x02, 0x10, 0x00, 0x00, 0x00, 0x00,\ +/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11,\ +/* 64 */ 0x08, 0x11, 0x11, 0x11, 0x11, 0x02, 0x02, 0x4c,\ +/* 72 */ 0x4c, 0x00, 0x00, 0x00, 0x05, 0x05, 0x15, 0x15,\ +/* 80 */ 0x15, 0x15, 0x15, 0x15, 0x00, 0x4c, 0x4c, 0x4c,\ +/* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x00,\ +/* 96 */ 0x24, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01,\ +/* 104 */ 0x01, 0x01, 0x01, 0x08, 0x08, 0x00, 0x02, 0x01,\ +/* 112 */ 0x01, 0x01, 0x01, 0x02, 0x00, 0x00, 0x02, 0x02,\ +/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x45,\ +/* 128 */ 0x15, 0x01, 0x02, 0x00, 0x01, 0x02, 0x08, 0x05,\ +/* 136 */ 0x05, 0x05, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04,\ +/* 144 */ 0x04, 0x04, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00,\ +/* 152 */ 0x01, 0x00, 0x02, 0x02, 0x01, 0x00, 0x00,} + +/************** End of opcodes.h *********************************************/ +/************** Continuing where we left off in vdbe.h ***********************/ + +/* +** Prototypes for the VDBE interface. See comments on the implementation +** for a description of what each of these routines does. +*/ +SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); +SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5); +SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); +SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe*, int addr); +SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); +SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); +SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); +SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); +SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); +SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *, int); +#endif +SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); +SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe*); +SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, int); +SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); +SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); +SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); +#ifndef SQLITE_OMIT_TRACE +SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); +#endif +SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); + +SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); +SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); +SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); + +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); +SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); + +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); +#endif + +/* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on +** each VDBE opcode. +** +** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op +** comments in VDBE programs that show key decision points in the code +** generator. +*/ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe*, const char*, ...); +# define VdbeComment(X) sqlite3VdbeComment X +SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); +# define VdbeNoopComment(X) sqlite3VdbeNoopComment X +# ifdef SQLITE_ENABLE_MODULE_COMMENTS +# define VdbeModuleComment(X) sqlite3VdbeNoopComment X +# else +# define VdbeModuleComment(X) +# endif +#else +# define VdbeComment(X) +# define VdbeNoopComment(X) +# define VdbeModuleComment(X) +#endif + +/* +** The VdbeCoverage macros are used to set a coverage testing point +** for VDBE branch instructions. The coverage testing points are line +** numbers in the sqlite3.c source file. VDBE branch coverage testing +** only works with an amalagmation build. That's ok since a VDBE branch +** coverage build designed for testing the test suite only. No application +** should ever ship with VDBE branch coverage measuring turned on. +** +** VdbeCoverage(v) // Mark the previously coded instruction +** // as a branch +** +** VdbeCoverageIf(v, conditional) // Mark previous if conditional true +** +** VdbeCoverageAlwaysTaken(v) // Previous branch is always taken +** +** VdbeCoverageNeverTaken(v) // Previous branch is never taken +** +** Every VDBE branch operation must be tagged with one of the macros above. +** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and +** -DSQLITE_DEBUG then an ALWAYS() will fail in the vdbeTakeBranch() +** routine in vdbe.c, alerting the developer to the missed tag. +*/ +#ifdef SQLITE_VDBE_COVERAGE +SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); +# define VdbeCoverage(v) sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageIf(v,x) if(x)sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageAlwaysTaken(v) sqlite3VdbeSetLineNumber(v,2); +# define VdbeCoverageNeverTaken(v) sqlite3VdbeSetLineNumber(v,1); +# define VDBE_OFFSET_LINENO(x) (__LINE__+x) +#else +# define VdbeCoverage(v) +# define VdbeCoverageIf(v,x) +# define VdbeCoverageAlwaysTaken(v) +# define VdbeCoverageNeverTaken(v) +# define VDBE_OFFSET_LINENO(x) 0 +#endif + +#endif + +/************** End of vdbe.h ************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include pager.h in the middle of sqliteInt.h *****************/ +/************** Begin file pager.h *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite page cache +** subsystem. The page cache subsystem reads and writes a file a page +** at a time and provides a journal for rollback. +*/ + +#ifndef _PAGER_H_ +#define _PAGER_H_ + +/* +** Default maximum size for persistent journal files. A negative +** value means no limit. This value may be overridden using the +** sqlite3PagerJournalSizeLimit() API. See also "PRAGMA journal_size_limit". +*/ +#ifndef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT + #define SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT -1 +#endif + +/* +** The type used to represent a page number. The first page in a file +** is called page 1. 0 is used to represent "not a page". +*/ +typedef u32 Pgno; + +/* +** Each open file is managed by a separate instance of the "Pager" structure. +*/ +typedef struct Pager Pager; + +/* +** Handle type for pages. +*/ +typedef struct PgHdr DbPage; + +/* +** Page number PAGER_MJ_PGNO is never used in an SQLite database (it is +** reserved for working around a windows/posix incompatibility). It is +** used in the journal to signify that the remainder of the journal file +** is devoted to storing a master journal name - there are no more pages to +** roll back. See comments for function writeMasterJournal() in pager.c +** for details. +*/ +#define PAGER_MJ_PGNO(x) ((Pgno)((PENDING_BYTE/((x)->pageSize))+1)) + +/* +** Allowed values for the flags parameter to sqlite3PagerOpen(). +** +** NOTE: These values must match the corresponding BTREE_ values in btree.h. +*/ +#define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ +#define PAGER_MEMORY 0x0002 /* In-memory database */ + +/* +** Valid values for the second argument to sqlite3PagerLockingMode(). +*/ +#define PAGER_LOCKINGMODE_QUERY -1 +#define PAGER_LOCKINGMODE_NORMAL 0 +#define PAGER_LOCKINGMODE_EXCLUSIVE 1 + +/* +** Numeric constants that encode the journalmode. +*/ +#define PAGER_JOURNALMODE_QUERY (-1) /* Query the value of journalmode */ +#define PAGER_JOURNALMODE_DELETE 0 /* Commit by deleting journal file */ +#define PAGER_JOURNALMODE_PERSIST 1 /* Commit by zeroing journal header */ +#define PAGER_JOURNALMODE_OFF 2 /* Journal omitted. */ +#define PAGER_JOURNALMODE_TRUNCATE 3 /* Commit by truncating journal */ +#define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ +#define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ + +/* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_GET_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_GET_READONLY 0x02 /* Read-only page is acceptable */ + +/* +** Flags for sqlite3PagerSetFlags() +*/ +#define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ +#define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ +#define PAGER_SYNCHRONOUS_FULL 0x03 /* PRAGMA synchronous=FULL */ +#define PAGER_SYNCHRONOUS_MASK 0x03 /* Mask for three values above */ +#define PAGER_FULLFSYNC 0x04 /* PRAGMA fullfsync=ON */ +#define PAGER_CKPT_FULLFSYNC 0x08 /* PRAGMA checkpoint_fullfsync=ON */ +#define PAGER_CACHESPILL 0x10 /* PRAGMA cache_spill=ON */ +#define PAGER_FLAGS_MASK 0x1c /* All above except SYNCHRONOUS */ + +/* +** The remainder of this file contains the declarations of the functions +** that make up the Pager sub-system API. See source code comments for +** a detailed description of each routine. +*/ + +/* Open and close a Pager connection. */ +SQLITE_PRIVATE int sqlite3PagerOpen( + sqlite3_vfs*, + Pager **ppPager, + const char*, + int, + int, + int, + void(*)(DbPage*) +); +SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); + +/* Functions used to configure a Pager object. */ +SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); +SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); +SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); +SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); +SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); +SQLITE_PRIVATE void sqlite3PagerShrink(Pager*); +SQLITE_PRIVATE void sqlite3PagerSetFlags(Pager*,unsigned); +SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *, int); +SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *, int); +SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*); +SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager*); +SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *, i64); +SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager*); + +/* Functions used to obtain and release page references. */ +SQLITE_PRIVATE int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); +#define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0) +SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); +SQLITE_PRIVATE void sqlite3PagerRef(DbPage*); +SQLITE_PRIVATE void sqlite3PagerUnref(DbPage*); +SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage*); + +/* Operations on page references. */ +SQLITE_PRIVATE int sqlite3PagerWrite(DbPage*); +SQLITE_PRIVATE void sqlite3PagerDontWrite(DbPage*); +SQLITE_PRIVATE int sqlite3PagerMovepage(Pager*,DbPage*,Pgno,int); +SQLITE_PRIVATE int sqlite3PagerPageRefcount(DbPage*); +SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *); +SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *); + +/* Functions used to manage pager transactions and savepoints. */ +SQLITE_PRIVATE void sqlite3PagerPagecount(Pager*, int*); +SQLITE_PRIVATE int sqlite3PagerBegin(Pager*, int exFlag, int); +SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); +SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager*); +SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zMaster); +SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager*); +SQLITE_PRIVATE int sqlite3PagerRollback(Pager*); +SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int n); +SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); +SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager); + +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); +SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); +SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager); +#endif + +#ifdef SQLITE_ENABLE_ZIPVFS +SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager); +#endif + +/* Functions used to query pager state and configuration. */ +SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager*); +SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); +SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); +SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); +SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager*); +SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); +SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); +SQLITE_PRIVATE int sqlite3PagerNosync(Pager*); +SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); +SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *); +SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); + +/* Functions used to truncate the database file. */ +SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); + +#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) +SQLITE_PRIVATE void *sqlite3PagerCodec(DbPage *); +#endif + +/* Functions to support testing and debugging. */ +#if !defined(NDEBUG) || defined(SQLITE_TEST) +SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage*); +SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage*); +#endif +#ifdef SQLITE_TEST +SQLITE_PRIVATE int *sqlite3PagerStats(Pager*); +SQLITE_PRIVATE void sqlite3PagerRefdump(Pager*); + void disable_simulated_io_errors(void); + void enable_simulated_io_errors(void); +#else +# define disable_simulated_io_errors() +# define enable_simulated_io_errors() +#endif + +#endif /* _PAGER_H_ */ + +/************** End of pager.h ***********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include pcache.h in the middle of sqliteInt.h ****************/ +/************** Begin file pcache.h ******************************************/ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite page cache +** subsystem. +*/ + +#ifndef _PCACHE_H_ + +typedef struct PgHdr PgHdr; +typedef struct PCache PCache; + +/* +** Every page in the cache is controlled by an instance of the following +** structure. +*/ +struct PgHdr { + sqlite3_pcache_page *pPage; /* Pcache object page handle */ + void *pData; /* Page data */ + void *pExtra; /* Extra content */ + PgHdr *pDirty; /* Transient list of dirty pages */ + Pager *pPager; /* The pager this page is part of */ + Pgno pgno; /* Page number for this page */ +#ifdef SQLITE_CHECK_PAGES + u32 pageHash; /* Hash of page content */ +#endif + u16 flags; /* PGHDR flags defined below */ + + /********************************************************************** + ** Elements above are public. All that follows is private to pcache.c + ** and should not be accessed by other modules. + */ + i16 nRef; /* Number of users of this page */ + PCache *pCache; /* Cache that owns this page */ + + PgHdr *pDirtyNext; /* Next element in list of dirty pages */ + PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ +}; + +/* Bit values for PgHdr.flags */ +#define PGHDR_DIRTY 0x002 /* Page has changed */ +#define PGHDR_NEED_SYNC 0x004 /* Fsync the rollback journal before + ** writing this page to the database */ +#define PGHDR_NEED_READ 0x008 /* Content is unread */ +#define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ +#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ + +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + +/* Initialize and shutdown the page cache subsystem */ +SQLITE_PRIVATE int sqlite3PcacheInitialize(void); +SQLITE_PRIVATE void sqlite3PcacheShutdown(void); + +/* Page cache buffer management: +** These routines implement SQLITE_CONFIG_PAGECACHE. +*/ +SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *, int sz, int n); + +/* Create a new pager cache. +** Under memory stress, invoke xStress to try to make pages clean. +** Only clean and unpinned pages can be reclaimed. +*/ +SQLITE_PRIVATE void sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + int (*xStress)(void*, PgHdr*), /* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *pToInit /* Preallocated space for the PCache */ +); + +/* Modify the page-size after the cache has been created. */ +SQLITE_PRIVATE void sqlite3PcacheSetPageSize(PCache *, int); + +/* Return the size in bytes of a PCache object. Used to preallocate +** storage space. +*/ +SQLITE_PRIVATE int sqlite3PcacheSize(void); + +/* One release per successful fetch. Page is pinned until released. +** Reference counted. +*/ +SQLITE_PRIVATE int sqlite3PcacheFetch(PCache*, Pgno, int createFlag, PgHdr**); +SQLITE_PRIVATE void sqlite3PcacheRelease(PgHdr*); + +SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ +SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ +SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ +SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ + +/* Change a page number. Used by incr-vacuum. */ +SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr*, Pgno); + +/* Remove all pages with pgno>x. Reset the cache if x==0 */ +SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache*, Pgno x); + +/* Get a list of all dirty pages in the cache, sorted by page number */ +SQLITE_PRIVATE PgHdr *sqlite3PcacheDirtyList(PCache*); + +/* Reset and close the cache object */ +SQLITE_PRIVATE void sqlite3PcacheClose(PCache*); + +/* Clear flags from pages of the page cache */ +SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *); + +/* Discard the contents of the cache */ +SQLITE_PRIVATE void sqlite3PcacheClear(PCache*); + +/* Return the total number of outstanding page references */ +SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache*); + +/* Increment the reference count of an existing page */ +SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr*); + +SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr*); + +/* Return the total number of pages stored in the cache */ +SQLITE_PRIVATE int sqlite3PcachePagecount(PCache*); + +#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) +/* Iterate through all dirty pages currently stored in the cache. This +** interface is only available if SQLITE_CHECK_PAGES is defined when the +** library is built. +*/ +SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); +#endif + +/* Set and get the suggested cache-size for the specified pager-cache. +** +** If no global maximum is configured, then the system attempts to limit +** the total number of pages cached by purgeable pager-caches to the sum +** of the suggested cache-sizes. +*/ +SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *, int); +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *); +#endif + +/* Free up as much memory as possible from the page cache */ +SQLITE_PRIVATE void sqlite3PcacheShrink(PCache*); + +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* Try to return memory used by the pcache module to the main memory heap */ +SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int); +#endif + +#ifdef SQLITE_TEST +SQLITE_PRIVATE void sqlite3PcacheStats(int*,int*,int*,int*); +#endif + +SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); + +#endif /* _PCACHE_H_ */ + +/************** End of pcache.h **********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/************** Include os.h in the middle of sqliteInt.h ********************/ +/************** Begin file os.h **********************************************/ +/* +** 2001 September 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file (together with is companion C source-code file +** "os.c") attempt to abstract the underlying operating system so that +** the SQLite library will work on both POSIX and windows systems. +** +** This header file is #include-ed by sqliteInt.h and thus ends up +** being included by every source file. +*/ +#ifndef _SQLITE_OS_H_ +#define _SQLITE_OS_H_ + +/* +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. +*/ +/************** Include os_setup.h in the middle of os.h *********************/ +/************** Begin file os_setup.h ****************************************/ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains pre-processor directives related to operating system +** detection and/or setup. +*/ +#ifndef _OS_SETUP_H_ +#define _OS_SETUP_H_ + +/* +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of SQLITE_OS_UNIX, +** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of +** the three will be 1. The other two will be 0. +*/ +#if defined(SQLITE_OS_OTHER) +# if SQLITE_OS_OTHER==1 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# else +# undef SQLITE_OS_OTHER +# endif +#endif +#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) +# define SQLITE_OS_OTHER 0 +# ifndef SQLITE_OS_WIN +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +# else +# define SQLITE_OS_UNIX 0 +# endif +#else +# ifndef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# endif +#endif + +#endif /* _OS_SETUP_H_ */ + +/************** End of os_setup.h ********************************************/ +/************** Continuing where we left off in os.h *************************/ + +/* If the SET_FULLSYNC macro is not defined above, then make it +** a no-op +*/ +#ifndef SET_FULLSYNC +# define SET_FULLSYNC(x,y) +#endif + +/* +** The default size of a disk sector +*/ +#ifndef SQLITE_DEFAULT_SECTOR_SIZE +# define SQLITE_DEFAULT_SECTOR_SIZE 4096 +#endif + +/* +** Temporary files are named starting with this prefix followed by 16 random +** alphanumeric characters, and no file extension. They are stored in the +** OS's standard temporary file directory, and are deleted prior to exit. +** If sqlite is being embedded in another program, you may wish to change the +** prefix to reflect your program's name, so that if your program exits +** prematurely, old temporary files can be easily identified. This can be done +** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. +** +** 2006-10-31: The default prefix used to be "sqlite_". But then +** Mcafee started using SQLite in their anti-virus product and it +** started putting files with the "sqlite" name in the c:/temp folder. +** This annoyed many windows users. Those users would then do a +** Google search for "sqlite", find the telephone numbers of the +** developers and call to wake them up at night and complain. +** For this reason, the default name prefix is changed to be "sqlite" +** spelled backwards. So the temp files are still identified, but +** anybody smart enough to figure out the code is also likely smart +** enough to know that calling the developer will not help get rid +** of the file. +*/ +#ifndef SQLITE_TEMP_FILE_PREFIX +# define SQLITE_TEMP_FILE_PREFIX "etilqs_" +#endif + +/* +** The following values may be passed as the second argument to +** sqlite3OsLock(). The various locks exhibit the following semantics: +** +** SHARED: Any number of processes may hold a SHARED lock simultaneously. +** RESERVED: A single process may hold a RESERVED lock on a file at +** any time. Other processes may hold and obtain new SHARED locks. +** PENDING: A single process may hold a PENDING lock on a file at +** any one time. Existing SHARED locks may persist, but no new +** SHARED locks may be obtained by other processes. +** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. +** +** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a +** process that requests an EXCLUSIVE lock may actually obtain a PENDING +** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to +** sqlite3OsLock(). +*/ +#define NO_LOCK 0 +#define SHARED_LOCK 1 +#define RESERVED_LOCK 2 +#define PENDING_LOCK 3 +#define EXCLUSIVE_LOCK 4 + +/* +** File Locking Notes: (Mostly about windows but also some info for Unix) +** +** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because +** those functions are not available. So we use only LockFile() and +** UnlockFile(). +** +** LockFile() prevents not just writing but also reading by other processes. +** A SHARED_LOCK is obtained by locking a single randomly-chosen +** byte out of a specific range of bytes. The lock byte is obtained at +** random so two separate readers can probably access the file at the +** same time, unless they are unlucky and choose the same lock byte. +** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. +** There can only be one writer. A RESERVED_LOCK is obtained by locking +** a single byte of the file that is designated as the reserved lock byte. +** A PENDING_LOCK is obtained by locking a designated byte different from +** the RESERVED_LOCK byte. +** +** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, +** which means we can use reader/writer locks. When reader/writer locks +** are used, the lock is placed on the same range of bytes that is used +** for probabilistic locking in Win95/98/ME. Hence, the locking scheme +** will support two or more Win95 readers or two or more WinNT readers. +** But a single Win95 reader will lock out all WinNT readers and a single +** WinNT reader will lock out all other Win95 readers. +** +** The following #defines specify the range of bytes used for locking. +** SHARED_SIZE is the number of bytes available in the pool from which +** a random byte is selected for a shared lock. The pool of bytes for +** shared locks begins at SHARED_FIRST. +** +** The same locking strategy and +** byte ranges are used for Unix. This leaves open the possiblity of having +** clients on win95, winNT, and unix all talking to the same shared file +** and all locking correctly. To do so would require that samba (or whatever +** tool is being used for file sharing) implements locks correctly between +** windows and unix. I'm guessing that isn't likely to happen, but by +** using the same locking range we are at least open to the possibility. +** +** Locking in windows is manditory. For this reason, we cannot store +** actual data in the bytes used for locking. The pager never allocates +** the pages involved in locking therefore. SHARED_SIZE is selected so +** that all locks will fit on a single page even at the minimum page size. +** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE +** is set high so that we don't have to allocate an unused page except +** for very large databases. But one should test the page skipping logic +** by setting PENDING_BYTE low and running the entire regression suite. +** +** Changing the value of PENDING_BYTE results in a subtly incompatible +** file format. Depending on how it is changed, you might not notice +** the incompatibility right away, even running a full regression test. +** The default location of PENDING_BYTE is the first byte past the +** 1GB boundary. +** +*/ +#ifdef SQLITE_OMIT_WSD +# define PENDING_BYTE (0x40000000) +#else +# define PENDING_BYTE sqlite3PendingByte +#endif +#define RESERVED_BYTE (PENDING_BYTE+1) +#define SHARED_FIRST (PENDING_BYTE+2) +#define SHARED_SIZE 510 + +/* +** Wrapper around OS specific sqlite3_os_init() function. +*/ +SQLITE_PRIVATE int sqlite3OsInit(void); + +/* +** Functions for accessing sqlite3_file methods +*/ +SQLITE_PRIVATE int sqlite3OsClose(sqlite3_file*); +SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); +SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); +SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); +SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); +#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 +SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); +SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); +SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); + + +/* +** Functions for accessing sqlite3_vfs methods +*/ +SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); +SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); +SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); +SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); +SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); +SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); + +/* +** Convenience functions for opening and closing files using +** sqlite3_malloc() to obtain space for the file-handle structure. +*/ +SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); +SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); + +#endif /* _SQLITE_OS_H_ */ + +/************** End of os.h **************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include mutex.h in the middle of sqliteInt.h *****************/ +/************** Begin file mutex.h *******************************************/ +/* +** 2007 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the common header for all mutex implementations. +** The sqliteInt.h header #includes this file so that it is available +** to all source files. We break it out in an effort to keep the code +** better organized. +** +** NOTE: source files should *not* #include this header file directly. +** Source files should #include the sqliteInt.h file and let that file +** include this one indirectly. +*/ + + +/* +** Figure out what version of the code to use. The choices are +** +** SQLITE_MUTEX_OMIT No mutex logic. Not even stubs. The +** mutexes implemention cannot be overridden +** at start-time. +** +** SQLITE_MUTEX_NOOP For single-threaded applications. No +** mutual exclusion is provided. But this +** implementation can be overridden at +** start-time. +** +** SQLITE_MUTEX_PTHREADS For multi-threaded applications on Unix. +** +** SQLITE_MUTEX_W32 For multi-threaded applications on Win32. +*/ +#if !SQLITE_THREADSAFE +# define SQLITE_MUTEX_OMIT +#endif +#if SQLITE_THREADSAFE && !defined(SQLITE_MUTEX_NOOP) +# if SQLITE_OS_UNIX +# define SQLITE_MUTEX_PTHREADS +# elif SQLITE_OS_WIN +# define SQLITE_MUTEX_W32 +# else +# define SQLITE_MUTEX_NOOP +# endif +#endif + +#ifdef SQLITE_MUTEX_OMIT +/* +** If this is a no-op implementation, implement everything as macros. +*/ +#define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) +#define sqlite3_mutex_free(X) +#define sqlite3_mutex_enter(X) +#define sqlite3_mutex_try(X) SQLITE_OK +#define sqlite3_mutex_leave(X) +#define sqlite3_mutex_held(X) ((void)(X),1) +#define sqlite3_mutex_notheld(X) ((void)(X),1) +#define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) +#define sqlite3MutexInit() SQLITE_OK +#define sqlite3MutexEnd() +#define MUTEX_LOGIC(X) +#else +#define MUTEX_LOGIC(X) X +#endif /* defined(SQLITE_MUTEX_OMIT) */ + +/************** End of mutex.h ***********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + + +/* +** Each database file to be accessed by the system is an instance +** of the following structure. There are normally two of these structures +** in the sqlite.aDb[] array. aDb[0] is the main database file and +** aDb[1] is the database file used to hold temporary tables. Additional +** databases may be attached. +*/ +struct Db { + char *zName; /* Name of this database */ + Btree *pBt; /* The B*Tree structure for this database file */ + u8 safety_level; /* How aggressive at syncing data to disk */ + Schema *pSchema; /* Pointer to database schema (possibly shared) */ +}; + +/* +** An instance of the following structure stores a database schema. +** +** Most Schema objects are associated with a Btree. The exception is +** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing. +** In shared cache mode, a single Schema object can be shared by multiple +** Btrees that refer to the same underlying BtShared object. +** +** Schema objects are automatically deallocated when the last Btree that +** references them is destroyed. The TEMP Schema is manually freed by +** sqlite3_close(). +* +** A thread must be holding a mutex on the corresponding Btree in order +** to access Schema content. This implies that the thread must also be +** holding a mutex on the sqlite3 connection pointer that owns the Btree. +** For a TEMP Schema, only the connection mutex is required. +*/ +struct Schema { + int schema_cookie; /* Database schema version number for this file */ + int iGeneration; /* Generation counter. Incremented with each change */ + Hash tblHash; /* All tables indexed by name */ + Hash idxHash; /* All (named) indices indexed by name */ + Hash trigHash; /* All triggers indexed by name */ + Hash fkeyHash; /* All foreign keys by referenced table name */ + Table *pSeqTab; /* The sqlite_sequence table used by AUTOINCREMENT */ + u8 file_format; /* Schema format version for this file */ + u8 enc; /* Text encoding used by this database */ + u16 schemaFlags; /* Flags associated with this schema */ + int cache_size; /* Number of pages to use in the cache */ +}; + +/* +** These macros can be used to test, set, or clear bits in the +** Db.pSchema->flags field. +*/ +#define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))==(P)) +#define DbHasAnyProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))!=0) +#define DbSetProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags|=(P) +#define DbClearProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags&=~(P) + +/* +** Allowed values for the DB.pSchema->flags field. +** +** The DB_SchemaLoaded flag is set after the database schema has been +** read into internal hash tables. +** +** DB_UnresetViews means that one or more views have column names that +** have been filled out. If the schema changes, these column names might +** changes and so the view will need to be reset. +*/ +#define DB_SchemaLoaded 0x0001 /* The schema has been loaded */ +#define DB_UnresetViews 0x0002 /* Some views have defined column names */ +#define DB_Empty 0x0004 /* The file is empty (length 0 bytes) */ + +/* +** The number of different kinds of things that can be limited +** using the sqlite3_limit() interface. +*/ +#define SQLITE_N_LIMIT (SQLITE_LIMIT_TRIGGER_DEPTH+1) + +/* +** Lookaside malloc is a set of fixed-size buffers that can be used +** to satisfy small transient memory allocation requests for objects +** associated with a particular database connection. The use of +** lookaside malloc provides a significant performance enhancement +** (approx 10%) by avoiding numerous malloc/free requests while parsing +** SQL statements. +** +** The Lookaside structure holds configuration information about the +** lookaside malloc subsystem. Each available memory allocation in +** the lookaside subsystem is stored on a linked list of LookasideSlot +** objects. +** +** Lookaside allocations are only allowed for objects that are associated +** with a particular database connection. Hence, schema information cannot +** be stored in lookaside because in shared cache mode the schema information +** is shared by multiple database connections. Therefore, while parsing +** schema information, the Lookaside.bEnabled flag is cleared so that +** lookaside allocations are not used to construct the schema objects. +*/ +struct Lookaside { + u16 sz; /* Size of each buffer in bytes */ + u8 bEnabled; /* False to disable new lookaside allocations */ + u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ + int nOut; /* Number of buffers currently checked out */ + int mxOut; /* Highwater mark for nOut */ + int anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ + LookasideSlot *pFree; /* List of available buffers */ + void *pStart; /* First byte of available memory space */ + void *pEnd; /* First byte past end of available space */ +}; +struct LookasideSlot { + LookasideSlot *pNext; /* Next buffer in the list of free buffers */ +}; + +/* +** A hash table for function definitions. +** +** Hash each FuncDef structure into one of the FuncDefHash.a[] slots. +** Collisions are on the FuncDef.pHash chain. +*/ +struct FuncDefHash { + FuncDef *a[23]; /* Hash table for functions */ +}; + +/* +** Each database connection is an instance of the following structure. +*/ +struct sqlite3 { + sqlite3_vfs *pVfs; /* OS Interface */ + struct Vdbe *pVdbe; /* List of active virtual machines */ + CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ + sqlite3_mutex *mutex; /* Connection mutex */ + Db *aDb; /* All backends */ + int nDb; /* Number of backends currently in use */ + int flags; /* Miscellaneous flags. See below */ + i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 szMmap; /* Default mmap_size setting */ + unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ + int errCode; /* Most recent error code (SQLITE_*) */ + int errMask; /* & result codes with this before returning */ + u16 dbOptFlags; /* Flags to enable/disable optimizations */ + u8 autoCommit; /* The auto-commit flag. */ + u8 temp_store; /* 1: file 2: memory 0: default */ + u8 mallocFailed; /* True if we have seen a malloc failure */ + u8 dfltLockMode; /* Default locking-mode for attached dbs */ + signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ + u8 suppressErr; /* Do not issue error messages if true */ + u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ + u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ + int nextPagesize; /* Pagesize after VACUUM if >0 */ + u32 magic; /* Magic number for detect library misuse */ + int nChange; /* Value returned by sqlite3_changes() */ + int nTotalChange; /* Value returned by sqlite3_total_changes() */ + int aLimit[SQLITE_N_LIMIT]; /* Limits */ + struct sqlite3InitInfo { /* Information used during initialization */ + int newTnum; /* Rootpage of table being initialized */ + u8 iDb; /* Which db file is being initialized */ + u8 busy; /* TRUE if currently initializing */ + u8 orphanTrigger; /* Last statement is orphaned TEMP trigger */ + } init; + int nVdbeActive; /* Number of VDBEs currently running */ + int nVdbeRead; /* Number of active VDBEs that read or write */ + int nVdbeWrite; /* Number of active VDBEs that read and write */ + int nVdbeExec; /* Number of nested calls to VdbeExec() */ + int nExtension; /* Number of loaded extensions */ + void **aExtension; /* Array of shared library handles */ + void (*xTrace)(void*,const char*); /* Trace function */ + void *pTraceArg; /* Argument to the trace function */ + void (*xProfile)(void*,const char*,u64); /* Profiling function */ + void *pProfileArg; /* Argument to profile function */ + void *pCommitArg; /* Argument to xCommitCallback() */ + int (*xCommitCallback)(void*); /* Invoked at every commit. */ + void *pRollbackArg; /* Argument to xRollbackCallback() */ + void (*xRollbackCallback)(void*); /* Invoked at every commit. */ + void *pUpdateArg; + void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); +#ifndef SQLITE_OMIT_WAL + int (*xWalCallback)(void *, sqlite3 *, const char *, int); + void *pWalArg; +#endif + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*); + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*); + void *pCollNeededArg; + sqlite3_value *pErr; /* Most recent error message */ + union { + volatile int isInterrupted; /* True if sqlite3_interrupt has been called */ + double notUsed1; /* Spacer */ + } u1; + Lookaside lookaside; /* Lookaside malloc configuration */ +#ifndef SQLITE_OMIT_AUTHORIZATION + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + /* Access authorization function */ + void *pAuthArg; /* 1st argument to the access auth function */ +#endif +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + int (*xProgress)(void *); /* The progress callback */ + void *pProgressArg; /* Argument to the progress callback */ + unsigned nProgressOps; /* Number of opcodes for progress callback */ +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + int nVTrans; /* Allocated size of aVTrans */ + Hash aModule; /* populated by sqlite3_create_module() */ + VtabCtx *pVtabCtx; /* Context for active vtab connect/create */ + VTable **aVTrans; /* Virtual tables with open transactions */ + VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ +#endif + FuncDefHash aFunc; /* Hash table of connection functions */ + Hash aCollSeq; /* All collating sequences */ + BusyHandler busyHandler; /* Busy callback */ + Db aDbStatic[2]; /* Static space for the 2 default backends */ + Savepoint *pSavepoint; /* List of active savepoints */ + int busyTimeout; /* Busy handler timeout, in msec */ + int nSavepoint; /* Number of non-transaction savepoints */ + int nStatement; /* Number of nested statement-transactions */ + i64 nDeferredCons; /* Net deferred constraints this transaction. */ + i64 nDeferredImmCons; /* Net deferred immediate constraints */ + int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ + +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + /* The following variables are all protected by the STATIC_MASTER + ** mutex, not by sqlite3.mutex. They are used by code in notify.c. + ** + ** When X.pUnlockConnection==Y, that means that X is waiting for Y to + ** unlock so that it can proceed. + ** + ** When X.pBlockingConnection==Y, that means that something that X tried + ** tried to do recently failed with an SQLITE_LOCKED error due to locks + ** held by Y. + */ + sqlite3 *pBlockingConnection; /* Connection that caused SQLITE_LOCKED */ + sqlite3 *pUnlockConnection; /* Connection to watch for unlock */ + void *pUnlockArg; /* Argument to xUnlockNotify */ + void (*xUnlockNotify)(void **, int); /* Unlock notify callback */ + sqlite3 *pNextBlocked; /* Next in list of all blocked connections */ +#endif +}; + +/* +** A macro to discover the encoding of a database. +*/ +#define ENC(db) ((db)->aDb[0].pSchema->enc) + +/* +** Possible values for the sqlite3.flags. +*/ +#define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ +#define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ +#define SQLITE_FullFSync 0x00000004 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00000008 /* Use full fsync for checkpoint */ +#define SQLITE_CacheSpill 0x00000010 /* OK to spill pager cache */ +#define SQLITE_FullColNames 0x00000020 /* Show full column names on SELECT */ +#define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ +#define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ + /* DELETE, or UPDATE and return */ + /* the count using a callback. */ +#define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ + /* result set is empty */ +#define SQLITE_SqlTrace 0x00000200 /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing 0x00000400 /* Debug listings of VDBE programs */ +#define SQLITE_WriteSchema 0x00000800 /* OK to update SQLITE_MASTER */ +#define SQLITE_VdbeAddopTrace 0x00001000 /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_IgnoreChecks 0x00002000 /* Do not enforce check constraints */ +#define SQLITE_ReadUncommitted 0x0004000 /* For shared-cache mode */ +#define SQLITE_LegacyFileFmt 0x00008000 /* Create new databases in format 1 */ +#define SQLITE_RecoveryMode 0x00010000 /* Ignore schema errors */ +#define SQLITE_ReverseOrder 0x00020000 /* Reverse unordered SELECTs */ +#define SQLITE_RecTriggers 0x00040000 /* Enable recursive triggers */ +#define SQLITE_ForeignKeys 0x00080000 /* Enforce foreign key constraints */ +#define SQLITE_AutoIndex 0x00100000 /* Enable automatic indexes */ +#define SQLITE_PreferBuiltin 0x00200000 /* Preference to built-in funcs */ +#define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ +#define SQLITE_EnableTrigger 0x00800000 /* True to enable triggers */ +#define SQLITE_DeferFKs 0x01000000 /* Defer all FK constraints */ +#define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ +#define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ + + +/* +** Bits of the sqlite3.dbOptFlags field that are used by the +** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to +** selectively disable various optimizations. +*/ +#define SQLITE_QueryFlattener 0x0001 /* Query flattening */ +#define SQLITE_ColumnCache 0x0002 /* Column cache */ +#define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ +/* not used 0x0010 // Was: SQLITE_IdxRealAsInt */ +#define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ +#define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ +#define SQLITE_Transitive 0x0200 /* Transitive constraints */ +#define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ +#define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ +#define SQLITE_AdjustOutEst 0x1000 /* Adjust output estimates using WHERE */ +#define SQLITE_AllOpts 0xffff /* All optimizations */ + +/* +** Macros for testing whether or not optimizations are enabled or disabled. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +#define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) +#define OptimizationEnabled(db, mask) (((db)->dbOptFlags&(mask))==0) +#else +#define OptimizationDisabled(db, mask) 0 +#define OptimizationEnabled(db, mask) 1 +#endif + +/* +** Return true if it OK to factor constant expressions into the initialization +** code. The argument is a Parse object for the code generator. +*/ +#define ConstFactorOk(P) ((P)->okConstFactor) + +/* +** Possible values for the sqlite.magic field. +** The numbers are obtained at random and have no special meaning, other +** than being distinct from one another. +*/ +#define SQLITE_MAGIC_OPEN 0xa029a697 /* Database is open */ +#define SQLITE_MAGIC_CLOSED 0x9f3c2d33 /* Database is closed */ +#define SQLITE_MAGIC_SICK 0x4b771290 /* Error and awaiting close */ +#define SQLITE_MAGIC_BUSY 0xf03b7906 /* Database currently in use */ +#define SQLITE_MAGIC_ERROR 0xb5357930 /* An SQLITE_MISUSE error occurred */ +#define SQLITE_MAGIC_ZOMBIE 0x64cffc7f /* Close with last statement close */ + +/* +** Each SQL function is defined by an instance of the following +** structure. A pointer to this structure is stored in the sqlite.aFunc +** hash table. When multiple functions have the same name, the hash table +** points to a linked list of these structures. +*/ +struct FuncDef { + i16 nArg; /* Number of arguments. -1 means unlimited */ + u16 funcFlags; /* Some combination of SQLITE_FUNC_* */ + void *pUserData; /* User data parameter */ + FuncDef *pNext; /* Next function with same name */ + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */ + void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */ + void (*xFinalize)(sqlite3_context*); /* Aggregate finalizer */ + char *zName; /* SQL name of the function. */ + FuncDef *pHash; /* Next with a different name but the same hash */ + FuncDestructor *pDestructor; /* Reference counted destructor function */ +}; + +/* +** This structure encapsulates a user-function destructor callback (as +** configured using create_function_v2()) and a reference counter. When +** create_function_v2() is called to create a function with a destructor, +** a single object of this type is allocated. FuncDestructor.nRef is set to +** the number of FuncDef objects created (either 1 or 3, depending on whether +** or not the specified encoding is SQLITE_ANY). The FuncDef.pDestructor +** member of each of the new FuncDef objects is set to point to the allocated +** FuncDestructor. +** +** Thereafter, when one of the FuncDef objects is deleted, the reference +** count on this object is decremented. When it reaches 0, the destructor +** is invoked and the FuncDestructor structure freed. +*/ +struct FuncDestructor { + int nRef; + void (*xDestroy)(void *); + void *pUserData; +}; + +/* +** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There +** are assert() statements in the code to verify this. +*/ +#define SQLITE_FUNC_ENCMASK 0x003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +#define SQLITE_FUNC_LIKE 0x004 /* Candidate for the LIKE optimization */ +#define SQLITE_FUNC_CASE 0x008 /* Case-sensitive LIKE-type function */ +#define SQLITE_FUNC_EPHEM 0x010 /* Ephemeral. Delete with VDBE */ +#define SQLITE_FUNC_NEEDCOLL 0x020 /* sqlite3GetFuncCollSeq() might be called */ +#define SQLITE_FUNC_LENGTH 0x040 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x080 /* Built-in typeof() function */ +#define SQLITE_FUNC_COUNT 0x100 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ +#define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ +#define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ + +/* +** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are +** used to create the initializers for the FuncDef structures. +** +** FUNCTION(zName, nArg, iArg, bNC, xFunc) +** Used to create a scalar function definition of a function zName +** implemented by C function xFunc that accepts nArg arguments. The +** value passed as iArg is cast to a (void*) and made available +** as the user-data (sqlite3_user_data()) for the function. If +** argument bNC is true, then the SQLITE_FUNC_NEEDCOLL flag is set. +** +** VFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. +** +** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) +** Used to create an aggregate function definition implemented by +** the C functions xStep and xFinal. The first four parameters +** are interpreted in the same way as the first 4 parameters to +** FUNCTION(). +** +** LIKEFUNC(zName, nArg, pArg, flags) +** Used to create a scalar function definition of a function zName +** that accepts nArg arguments and is implemented by a call to C +** function likeFunc. Argument pArg is cast to a (void *) and made +** available as the function user-data (sqlite3_user_data()). The +** FuncDef.flags variable is set to the value passed as the flags +** parameter. +*/ +#define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ + {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + pArg, 0, xFunc, 0, 0, #zName, 0, 0} +#define LIKEFUNC(zName, nArg, arg, flags) \ + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ + (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} +#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} + +/* +** All current savepoints are stored in a linked list starting at +** sqlite3.pSavepoint. The first element in the list is the most recently +** opened savepoint. Savepoints are added to the list by the vdbe +** OP_Savepoint instruction. +*/ +struct Savepoint { + char *zName; /* Savepoint name (nul-terminated) */ + i64 nDeferredCons; /* Number of deferred fk violations */ + i64 nDeferredImmCons; /* Number of deferred imm fk. */ + Savepoint *pNext; /* Parent savepoint (if any) */ +}; + +/* +** The following are used as the second parameter to sqlite3Savepoint(), +** and as the P1 argument to the OP_Savepoint instruction. +*/ +#define SAVEPOINT_BEGIN 0 +#define SAVEPOINT_RELEASE 1 +#define SAVEPOINT_ROLLBACK 2 + + +/* +** Each SQLite module (virtual table definition) is defined by an +** instance of the following structure, stored in the sqlite3.aModule +** hash table. +*/ +struct Module { + const sqlite3_module *pModule; /* Callback pointers */ + const char *zName; /* Name passed to create_module() */ + void *pAux; /* pAux passed to create_module() */ + void (*xDestroy)(void *); /* Module destructor function */ +}; + +/* +** information about each column of an SQL table is held in an instance +** of this structure. +*/ +struct Column { + char *zName; /* Name of this column */ + Expr *pDflt; /* Default value of this column */ + char *zDflt; /* Original text of the default value */ + char *zType; /* Data type for this column */ + char *zColl; /* Collating sequence. If NULL, use the default */ + u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ + char affinity; /* One of the SQLITE_AFF_... values */ + u8 szEst; /* Estimated size of this column. INT==1 */ + u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ +}; + +/* Allowed values for Column.colFlags: +*/ +#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ + +/* +** A "Collating Sequence" is defined by an instance of the following +** structure. Conceptually, a collating sequence consists of a name and +** a comparison routine that defines the order of that sequence. +** +** If CollSeq.xCmp is NULL, it means that the +** collating sequence is undefined. Indices built on an undefined +** collating sequence may not be read or written. +*/ +struct CollSeq { + char *zName; /* Name of the collating sequence, UTF-8 encoded */ + u8 enc; /* Text encoding handled by xCmp() */ + void *pUser; /* First argument to xCmp() */ + int (*xCmp)(void*,int, const void*, int, const void*); + void (*xDel)(void*); /* Destructor for pUser */ +}; + +/* +** A sort order can be either ASC or DESC. +*/ +#define SQLITE_SO_ASC 0 /* Sort in ascending order */ +#define SQLITE_SO_DESC 1 /* Sort in ascending order */ + +/* +** Column affinity types. +** +** These used to have mnemonic name like 'i' for SQLITE_AFF_INTEGER and +** 't' for SQLITE_AFF_TEXT. But we can save a little space and improve +** the speed a little by numbering the values consecutively. +** +** But rather than start with 0 or 1, we begin with 'a'. That way, +** when multiple affinity types are concatenated into a string and +** used as the P4 operand, they will be more readable. +** +** Note also that the numeric types are grouped together so that testing +** for a numeric type is a single comparison. +*/ +#define SQLITE_AFF_TEXT 'a' +#define SQLITE_AFF_NONE 'b' +#define SQLITE_AFF_NUMERIC 'c' +#define SQLITE_AFF_INTEGER 'd' +#define SQLITE_AFF_REAL 'e' + +#define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) + +/* +** The SQLITE_AFF_MASK values masks off the significant bits of an +** affinity value. +*/ +#define SQLITE_AFF_MASK 0x67 + +/* +** Additional bit values that can be ORed with an affinity without +** changing the affinity. +** +** The SQLITE_NOTNULL flag is a combination of NULLEQ and JUMPIFNULL. +** It causes an assert() to fire if either operand to a comparison +** operator is NULL. It is added to certain comparison operators to +** prove that the operands are always NOT NULL. +*/ +#define SQLITE_JUMPIFNULL 0x08 /* jumps if either operand is NULL */ +#define SQLITE_STOREP2 0x10 /* Store result in reg[P2] rather than jump */ +#define SQLITE_NULLEQ 0x80 /* NULL=NULL */ +#define SQLITE_NOTNULL 0x88 /* Assert that operands are never NULL */ + +/* +** An object of this type is created for each virtual table present in +** the database schema. +** +** If the database schema is shared, then there is one instance of this +** structure for each database connection (sqlite3*) that uses the shared +** schema. This is because each database connection requires its own unique +** instance of the sqlite3_vtab* handle used to access the virtual table +** implementation. sqlite3_vtab* handles can not be shared between +** database connections, even when the rest of the in-memory database +** schema is shared, as the implementation often stores the database +** connection handle passed to it via the xConnect() or xCreate() method +** during initialization internally. This database connection handle may +** then be used by the virtual table implementation to access real tables +** within the database. So that they appear as part of the callers +** transaction, these accesses need to be made via the same database +** connection as that used to execute SQL operations on the virtual table. +** +** All VTable objects that correspond to a single table in a shared +** database schema are initially stored in a linked-list pointed to by +** the Table.pVTable member variable of the corresponding Table object. +** When an sqlite3_prepare() operation is required to access the virtual +** table, it searches the list for the VTable that corresponds to the +** database connection doing the preparing so as to use the correct +** sqlite3_vtab* handle in the compiled query. +** +** When an in-memory Table object is deleted (for example when the +** schema is being reloaded for some reason), the VTable objects are not +** deleted and the sqlite3_vtab* handles are not xDisconnect()ed +** immediately. Instead, they are moved from the Table.pVTable list to +** another linked list headed by the sqlite3.pDisconnect member of the +** corresponding sqlite3 structure. They are then deleted/xDisconnected +** next time a statement is prepared using said sqlite3*. This is done +** to avoid deadlock issues involving multiple sqlite3.mutex mutexes. +** Refer to comments above function sqlite3VtabUnlockList() for an +** explanation as to why it is safe to add an entry to an sqlite3.pDisconnect +** list without holding the corresponding sqlite3.mutex mutex. +** +** The memory for objects of this type is always allocated by +** sqlite3DbMalloc(), using the connection handle stored in VTable.db as +** the first argument. +*/ +struct VTable { + sqlite3 *db; /* Database connection associated with this table */ + Module *pMod; /* Pointer to module implementation */ + sqlite3_vtab *pVtab; /* Pointer to vtab instance */ + int nRef; /* Number of pointers to this structure */ + u8 bConstraint; /* True if constraints are supported */ + int iSavepoint; /* Depth of the SAVEPOINT stack */ + VTable *pNext; /* Next in linked list (see above) */ +}; + +/* +** Each SQL table is represented in memory by an instance of the +** following structure. +** +** Table.zName is the name of the table. The case of the original +** CREATE TABLE statement is stored, but case is not significant for +** comparisons. +** +** Table.nCol is the number of columns in this table. Table.aCol is a +** pointer to an array of Column structures, one for each column. +** +** If the table has an INTEGER PRIMARY KEY, then Table.iPKey is the index of +** the column that is that key. Otherwise Table.iPKey is negative. Note +** that the datatype of the PRIMARY KEY must be INTEGER for this field to +** be set. An INTEGER PRIMARY KEY is used as the rowid for each row of +** the table. If a table has no INTEGER PRIMARY KEY, then a random rowid +** is generated for each row of the table. TF_HasPrimaryKey is set if +** the table has any PRIMARY KEY, INTEGER or otherwise. +** +** Table.tnum is the page number for the root BTree page of the table in the +** database file. If Table.iDb is the index of the database table backend +** in sqlite.aDb[]. 0 is for the main database and 1 is for the file that +** holds temporary tables and indices. If TF_Ephemeral is set +** then the table is stored in a file that is automatically deleted +** when the VDBE cursor to the table is closed. In this case Table.tnum +** refers VDBE cursor number that holds the table open, not to the root +** page number. Transient tables are used to hold the results of a +** sub-query that appears instead of a real table name in the FROM clause +** of a SELECT statement. +*/ +struct Table { + char *zName; /* Name of the table or view */ + Column *aCol; /* Information about each column */ + Index *pIndex; /* List of SQL indexes on this table. */ + Select *pSelect; /* NULL for tables. Points to definition if a view. */ + FKey *pFKey; /* Linked list of all foreign keys in this table */ + char *zColAff; /* String defining the affinity of each column */ +#ifndef SQLITE_OMIT_CHECK + ExprList *pCheck; /* All CHECK constraints */ +#endif + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ + int tnum; /* Root BTree node for this table (see note above) */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ + i16 nCol; /* Number of columns in this table */ + u16 nRef; /* Number of pointers to this Table */ + LogEst szTabRow; /* Estimated size of each table row in bytes */ +#ifdef SQLITE_ENABLE_COSTMULT + LogEst costMult; /* Cost multiplier for using this table */ +#endif + u8 tabFlags; /* Mask of TF_* values */ + u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ +#ifndef SQLITE_OMIT_ALTERTABLE + int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + int nModuleArg; /* Number of arguments to the module */ + char **azModuleArg; /* Text of all module args. [0] is module name */ + VTable *pVTable; /* List of VTable objects. */ +#endif + Trigger *pTrigger; /* List of triggers stored in pSchema */ + Schema *pSchema; /* Schema that contains this table */ + Table *pNextZombie; /* Next on the Parse.pZombieTab list */ +}; + +/* +** Allowed values for Table.tabFlags. +*/ +#define TF_Readonly 0x01 /* Read-only system table */ +#define TF_Ephemeral 0x02 /* An ephemeral table */ +#define TF_HasPrimaryKey 0x04 /* Table has a primary key */ +#define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ +#define TF_Virtual 0x10 /* Is a virtual table */ +#define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */ + + +/* +** Test to see whether or not a table is a virtual table. This is +** done as a macro so that it will be optimized out when virtual +** table support is omitted from the build. +*/ +#ifndef SQLITE_OMIT_VIRTUALTABLE +# define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +#else +# define IsVirtual(X) 0 +# define IsHiddenColumn(X) 0 +#endif + +/* Does the table have a rowid */ +#define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) + +/* +** Each foreign key constraint is an instance of the following structure. +** +** A foreign key is associated with two tables. The "from" table is +** the table that contains the REFERENCES clause that creates the foreign +** key. The "to" table is the table that is named in the REFERENCES clause. +** Consider this example: +** +** CREATE TABLE ex1( +** a INTEGER PRIMARY KEY, +** b INTEGER CONSTRAINT fk1 REFERENCES ex2(x) +** ); +** +** For foreign key "fk1", the from-table is "ex1" and the to-table is "ex2". +** Equivalent names: +** +** from-table == child-table +** to-table == parent-table +** +** Each REFERENCES clause generates an instance of the following structure +** which is attached to the from-table. The to-table need not exist when +** the from-table is created. The existence of the to-table is not checked. +** +** The list of all parents for child Table X is held at X.pFKey. +** +** A list of all children for a table named Z (which might not even exist) +** is held in Schema.fkeyHash with a hash key of Z. +*/ +struct FKey { + Table *pFrom; /* Table containing the REFERENCES clause (aka: Child) */ + FKey *pNextFrom; /* Next FKey with the same in pFrom. Next parent of pFrom */ + char *zTo; /* Name of table that the key points to (aka: Parent) */ + FKey *pNextTo; /* Next with the same zTo. Next child of zTo. */ + FKey *pPrevTo; /* Previous with the same zTo */ + int nCol; /* Number of columns in this key */ + /* EV: R-30323-21917 */ + u8 isDeferred; /* True if constraint checking is deferred till COMMIT */ + u8 aAction[2]; /* ON DELETE and ON UPDATE actions, respectively */ + Trigger *apTrigger[2];/* Triggers for aAction[] actions */ + struct sColMap { /* Mapping of columns in pFrom to columns in zTo */ + int iFrom; /* Index of column in pFrom */ + char *zCol; /* Name of column in zTo. If NULL use PRIMARY KEY */ + } aCol[1]; /* One entry for each of nCol columns */ +}; + +/* +** SQLite supports many different ways to resolve a constraint +** error. ROLLBACK processing means that a constraint violation +** causes the operation in process to fail and for the current transaction +** to be rolled back. ABORT processing means the operation in process +** fails and any prior changes from that one operation are backed out, +** but the transaction is not rolled back. FAIL processing means that +** the operation in progress stops and returns an error code. But prior +** changes due to the same operation are not backed out and no rollback +** occurs. IGNORE means that the particular row that caused the constraint +** error is not inserted or updated. Processing continues and no error +** is returned. REPLACE means that preexisting database rows that caused +** a UNIQUE constraint violation are removed so that the new insert or +** update can proceed. Processing continues and no error is reported. +** +** RESTRICT, SETNULL, and CASCADE actions apply only to foreign keys. +** RESTRICT is the same as ABORT for IMMEDIATE foreign keys and the +** same as ROLLBACK for DEFERRED keys. SETNULL means that the foreign +** key is set to NULL. CASCADE means that a DELETE or UPDATE of the +** referenced table row is propagated into the row that holds the +** foreign key. +** +** The following symbolic values are used to record which type +** of action to take. +*/ +#define OE_None 0 /* There is no constraint to check */ +#define OE_Rollback 1 /* Fail the operation and rollback the transaction */ +#define OE_Abort 2 /* Back out changes but do no rollback transaction */ +#define OE_Fail 3 /* Stop the operation but leave all prior changes */ +#define OE_Ignore 4 /* Ignore the error. Do not do the INSERT or UPDATE */ +#define OE_Replace 5 /* Delete existing record, then do INSERT or UPDATE */ + +#define OE_Restrict 6 /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */ +#define OE_SetNull 7 /* Set the foreign key value to NULL */ +#define OE_SetDflt 8 /* Set the foreign key value to its default */ +#define OE_Cascade 9 /* Cascade the changes */ + +#define OE_Default 10 /* Do whatever the default action is */ + + +/* +** An instance of the following structure is passed as the first +** argument to sqlite3VdbeKeyCompare and is used to control the +** comparison of the two index keys. +** +** Note that aSortOrder[] and aColl[] have nField+1 slots. There +** are nField slots for the columns of an index then one extra slot +** for the rowid at the end. +*/ +struct KeyInfo { + u32 nRef; /* Number of references to this KeyInfo object */ + u8 enc; /* Text encoding - one of the SQLITE_UTF* values */ + u16 nField; /* Number of key columns in the index */ + u16 nXField; /* Number of columns beyond the key columns */ + sqlite3 *db; /* The database connection */ + u8 *aSortOrder; /* Sort order for each column. */ + CollSeq *aColl[1]; /* Collating sequence for each term of the key */ +}; + +/* +** An instance of the following structure holds information about a +** single index record that has already been parsed out into individual +** values. +** +** A record is an object that contains one or more fields of data. +** Records are used to store the content of a table row and to store +** the key of an index. A blob encoding of a record is created by +** the OP_MakeRecord opcode of the VDBE and is disassembled by the +** OP_Column opcode. +** +** This structure holds a record that has already been disassembled +** into its constituent fields. +** +** The r1 and r2 member variables are only used by the optimized comparison +** functions vdbeRecordCompareInt() and vdbeRecordCompareString(). +*/ +struct UnpackedRecord { + KeyInfo *pKeyInfo; /* Collation and sort-order information */ + u16 nField; /* Number of entries in apMem[] */ + i8 default_rc; /* Comparison result if keys are equal */ + u8 isCorrupt; /* Corruption detected by xRecordCompare() */ + Mem *aMem; /* Values */ + int r1; /* Value to return if (lhs > rhs) */ + int r2; /* Value to return if (rhs < lhs) */ +}; + + +/* +** Each SQL index is represented in memory by an +** instance of the following structure. +** +** The columns of the table that are to be indexed are described +** by the aiColumn[] field of this structure. For example, suppose +** we have the following table and index: +** +** CREATE TABLE Ex1(c1 int, c2 int, c3 text); +** CREATE INDEX Ex2 ON Ex1(c3,c1); +** +** In the Table structure describing Ex1, nCol==3 because there are +** three columns in the table. In the Index structure describing +** Ex2, nColumn==2 since 2 of the 3 columns of Ex1 are indexed. +** The value of aiColumn is {2, 0}. aiColumn[0]==2 because the +** first column to be indexed (c3) has an index of 2 in Ex1.aCol[]. +** The second column to be indexed (c1) has an index of 0 in +** Ex1.aCol[], hence Ex2.aiColumn[1]==0. +** +** The Index.onError field determines whether or not the indexed columns +** must be unique and what to do if they are not. When Index.onError=OE_None, +** it means this is not a unique index. Otherwise it is a unique index +** and the value of Index.onError indicate the which conflict resolution +** algorithm to employ whenever an attempt is made to insert a non-unique +** element. +*/ +struct Index { + char *zName; /* Name of this index */ + i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ + LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ + Table *pTable; /* The SQL table being indexed */ + char *zColAff; /* String defining the affinity of each column */ + Index *pNext; /* The next index associated with the same table */ + Schema *pSchema; /* Schema containing this index */ + u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ + char **azColl; /* Array of collation sequence names for index */ + Expr *pPartIdxWhere; /* WHERE clause for partial indices */ + KeyInfo *pKeyInfo; /* A KeyInfo object suitable for this index */ + int tnum; /* DB Page containing root of this index */ + LogEst szIdxRow; /* Estimated average row size in bytes */ + u16 nKeyCol; /* Number of columns forming the key */ + u16 nColumn; /* Number of columns stored in the index */ + u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + unsigned idxType:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned bUnordered:1; /* Use this index for == or IN queries only */ + unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ + unsigned isResized:1; /* True if resizeIndexObject() has been called */ + unsigned isCovering:1; /* True if this is a covering index */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int nSample; /* Number of elements in aSample[] */ + int nSampleCol; /* Size of IndexSample.anEq[] and so on */ + tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ + IndexSample *aSample; /* Samples of the left-most key */ +#endif +}; + +/* +** Allowed values for Index.idxType +*/ +#define SQLITE_IDXTYPE_APPDEF 0 /* Created using CREATE INDEX */ +#define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */ +#define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */ + +/* Return true if index X is a PRIMARY KEY index */ +#define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY) + +/* Return true if index X is a UNIQUE index */ +#define IsUniqueIndex(X) ((X)->onError!=OE_None) + +/* +** Each sample stored in the sqlite_stat3 table is represented in memory +** using a structure of this type. See documentation at the top of the +** analyze.c source file for additional information. +*/ +struct IndexSample { + void *p; /* Pointer to sampled record */ + int n; /* Size of record in bytes */ + tRowcnt *anEq; /* Est. number of rows where the key equals this sample */ + tRowcnt *anLt; /* Est. number of rows where key is less than this sample */ + tRowcnt *anDLt; /* Est. number of distinct keys less than this sample */ +}; + +/* +** Each token coming out of the lexer is an instance of +** this structure. Tokens are also used as part of an expression. +** +** Note if Token.z==0 then Token.dyn and Token.n are undefined and +** may contain random values. Do not make any assumptions about Token.dyn +** and Token.n when Token.z==0. +*/ +struct Token { + const char *z; /* Text of the token. Not NULL-terminated! */ + unsigned int n; /* Number of characters in this token */ +}; + +/* +** An instance of this structure contains information needed to generate +** code for a SELECT that contains aggregate functions. +** +** If Expr.op==TK_AGG_COLUMN or TK_AGG_FUNCTION then Expr.pAggInfo is a +** pointer to this structure. The Expr.iColumn field is the index in +** AggInfo.aCol[] or AggInfo.aFunc[] of information needed to generate +** code for that node. +** +** AggInfo.pGroupBy and AggInfo.aFunc.pExpr point to fields within the +** original Select structure that describes the SELECT statement. These +** fields do not need to be freed when deallocating the AggInfo structure. +*/ +struct AggInfo { + u8 directMode; /* Direct rendering mode means take data directly + ** from source tables rather than from accumulators */ + u8 useSortingIdx; /* In direct mode, reference the sorting index rather + ** than the source table */ + int sortingIdx; /* Cursor number of the sorting index */ + int sortingIdxPTab; /* Cursor number of pseudo-table */ + int nSortingColumn; /* Number of columns in the sorting index */ + int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + ExprList *pGroupBy; /* The group by clause */ + struct AggInfo_col { /* For each column used in source tables */ + Table *pTab; /* Source table */ + int iTable; /* Cursor number of the source table */ + int iColumn; /* Column number within the source table */ + int iSorterColumn; /* Column number in the sorting index */ + int iMem; /* Memory location that acts as accumulator */ + Expr *pExpr; /* The original expression */ + } *aCol; + int nColumn; /* Number of used entries in aCol[] */ + int nAccumulator; /* Number of columns that show through to the output. + ** Additional columns are used only as parameters to + ** aggregate functions */ + struct AggInfo_func { /* For each aggregate function */ + Expr *pExpr; /* Expression encoding the function */ + FuncDef *pFunc; /* The aggregate function implementation */ + int iMem; /* Memory location that acts as accumulator */ + int iDistinct; /* Ephemeral table used to enforce DISTINCT */ + } *aFunc; + int nFunc; /* Number of entries in aFunc[] */ +}; + +/* +** The datatype ynVar is a signed integer, either 16-bit or 32-bit. +** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater +** than 32767 we have to make it 32-bit. 16-bit is preferred because +** it uses less memory in the Expr object, which is a big memory user +** in systems with lots of prepared statements. And few applications +** need more than about 10 or 20 variables. But some extreme users want +** to have prepared statements with over 32767 variables, and for them +** the option is available (at compile-time). +*/ +#if SQLITE_MAX_VARIABLE_NUMBER<=32767 +typedef i16 ynVar; +#else +typedef int ynVar; +#endif + +/* +** Each node of an expression in the parse tree is an instance +** of this structure. +** +** Expr.op is the opcode. The integer parser token codes are reused +** as opcodes here. For example, the parser defines TK_GE to be an integer +** code representing the ">=" operator. This same integer code is reused +** to represent the greater-than-or-equal-to operator in the expression +** tree. +** +** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, +** or TK_STRING), then Expr.token contains the text of the SQL literal. If +** the expression is a variable (TK_VARIABLE), then Expr.token contains the +** variable name. Finally, if the expression is an SQL function (TK_FUNCTION), +** then Expr.token contains the name of the function. +** +** Expr.pRight and Expr.pLeft are the left and right subexpressions of a +** binary operator. Either or both may be NULL. +** +** Expr.x.pList is a list of arguments if the expression is an SQL function, +** a CASE expression or an IN expression of the form " IN (, ...)". +** Expr.x.pSelect is used if the expression is a sub-select or an expression of +** the form " IN (SELECT ...)". If the EP_xIsSelect bit is set in the +** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is +** valid. +** +** An expression of the form ID or ID.ID refers to a column in a table. +** For such expressions, Expr.op is set to TK_COLUMN and Expr.iTable is +** the integer cursor number of a VDBE cursor pointing to that table and +** Expr.iColumn is the column number for the specific column. If the +** expression is used as a result in an aggregate SELECT, then the +** value is also stored in the Expr.iAgg column in the aggregate so that +** it can be accessed after all aggregates are computed. +** +** If the expression is an unbound variable marker (a question mark +** character '?' in the original SQL) then the Expr.iTable holds the index +** number for that variable. +** +** If the expression is a subquery then Expr.iColumn holds an integer +** register number containing the result of the subquery. If the +** subquery gives a constant result, then iTable is -1. If the subquery +** gives a different answer at different times during statement processing +** then iTable is the address of a subroutine that computes the subquery. +** +** If the Expr is of type OP_Column, and the table it is selecting from +** is a disk table or the "old.*" pseudo-table, then pTab points to the +** corresponding table definition. +** +** ALLOCATION NOTES: +** +** Expr objects can use a lot of memory space in database schema. To +** help reduce memory requirements, sometimes an Expr object will be +** truncated. And to reduce the number of memory allocations, sometimes +** two or more Expr objects will be stored in a single memory allocation, +** together with Expr.zToken strings. +** +** If the EP_Reduced and EP_TokenOnly flags are set when +** an Expr object is truncated. When EP_Reduced is set, then all +** the child Expr objects in the Expr.pLeft and Expr.pRight subtrees +** are contained within the same memory allocation. Note, however, that +** the subtrees in Expr.x.pList or Expr.x.pSelect are always separately +** allocated, regardless of whether or not EP_Reduced is set. +*/ +struct Expr { + u8 op; /* Operation performed by this node */ + char affinity; /* The affinity of the column or 0 if not a column */ + u32 flags; /* Various flags. EP_* See below */ + union { + char *zToken; /* Token value. Zero terminated and dequoted */ + int iValue; /* Non-negative integer value if EP_IntValue */ + } u; + + /* If the EP_TokenOnly flag is set in the Expr.flags mask, then no + ** space is allocated for the fields below this point. An attempt to + ** access them will result in a segfault or malfunction. + *********************************************************************/ + + Expr *pLeft; /* Left subnode */ + Expr *pRight; /* Right subnode */ + union { + ExprList *pList; /* op = IN, EXISTS, SELECT, CASE, FUNCTION, BETWEEN */ + Select *pSelect; /* EP_xIsSelect and op = IN, EXISTS, SELECT */ + } x; + + /* If the EP_Reduced flag is set in the Expr.flags mask, then no + ** space is allocated for the fields below this point. An attempt to + ** access them will result in a segfault or malfunction. + *********************************************************************/ + +#if SQLITE_MAX_EXPR_DEPTH>0 + int nHeight; /* Height of the tree headed by this node */ +#endif + int iTable; /* TK_COLUMN: cursor number of table holding column + ** TK_REGISTER: register number + ** TK_TRIGGER: 1 -> new, 0 -> old + ** EP_Unlikely: 1000 times likelihood */ + ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. + ** TK_VARIABLE: variable number (always >= 1). */ + i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ + i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ + u8 op2; /* TK_REGISTER: original value of Expr.op + ** TK_COLUMN: the value of p5 for OP_Column + ** TK_AGG_FUNCTION: nesting depth */ + AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ + Table *pTab; /* Table for TK_COLUMN expressions. */ +}; + +/* +** The following are the meanings of bits in the Expr.flags field. +*/ +#define EP_FromJoin 0x000001 /* Originated in ON or USING clause of a join */ +#define EP_Agg 0x000002 /* Contains one or more aggregate functions */ +#define EP_Resolved 0x000004 /* IDs have been resolved to COLUMNs */ +#define EP_Error 0x000008 /* Expression contains one or more errors */ +#define EP_Distinct 0x000010 /* Aggregate function with DISTINCT keyword */ +#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ +#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ +#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ +#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ +#define EP_Generic 0x000200 /* Ignore COLLATE or affinity on this tree */ +#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ +#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ +#define EP_Skip 0x001000 /* COLLATE, AS, or UNLIKELY */ +#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ +#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ +#define EP_Static 0x008000 /* Held in memory not obtained from malloc() */ +#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ +#define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ +#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ +#define EP_Constant 0x080000 /* Node is a constant */ + +/* +** These macros can be used to test, set, or clear bits in the +** Expr.flags field. +*/ +#define ExprHasProperty(E,P) (((E)->flags&(P))!=0) +#define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) +#define ExprSetProperty(E,P) (E)->flags|=(P) +#define ExprClearProperty(E,P) (E)->flags&=~(P) + +/* The ExprSetVVAProperty() macro is used for Verification, Validation, +** and Accreditation only. It works like ExprSetProperty() during VVA +** processes but is a no-op for delivery. +*/ +#ifdef SQLITE_DEBUG +# define ExprSetVVAProperty(E,P) (E)->flags|=(P) +#else +# define ExprSetVVAProperty(E,P) +#endif + +/* +** Macros to determine the number of bytes required by a normal Expr +** struct, an Expr struct with the EP_Reduced flag set in Expr.flags +** and an Expr struct with the EP_TokenOnly flag set. +*/ +#define EXPR_FULLSIZE sizeof(Expr) /* Full size */ +#define EXPR_REDUCEDSIZE offsetof(Expr,iTable) /* Common features */ +#define EXPR_TOKENONLYSIZE offsetof(Expr,pLeft) /* Fewer features */ + +/* +** Flags passed to the sqlite3ExprDup() function. See the header comment +** above sqlite3ExprDup() for details. +*/ +#define EXPRDUP_REDUCE 0x0001 /* Used reduced-size Expr nodes */ + +/* +** A list of expressions. Each expression may optionally have a +** name. An expr/name combination can be used in several ways, such +** as the list of "expr AS ID" fields following a "SELECT" or in the +** list of "ID = expr" items in an UPDATE. A list of expressions can +** also be used as the argument to a function, in which case the a.zName +** field is not used. +** +** By default the Expr.zSpan field holds a human-readable description of +** the expression that is used in the generation of error messages and +** column labels. In this case, Expr.zSpan is typically the text of a +** column expression as it exists in a SELECT statement. However, if +** the bSpanIsTab flag is set, then zSpan is overloaded to mean the name +** of the result column in the form: DATABASE.TABLE.COLUMN. This later +** form is used for name resolution with nested FROM clauses. +*/ +struct ExprList { + int nExpr; /* Number of expressions on the list */ + struct ExprList_item { /* For each expression in the list */ + Expr *pExpr; /* The list of expressions */ + char *zName; /* Token associated with this expression */ + char *zSpan; /* Original text of the expression */ + u8 sortOrder; /* 1 for DESC or 0 for ASC */ + unsigned done :1; /* A flag to indicate when processing is finished */ + unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ + unsigned reusable :1; /* Constant expression is reusable */ + union { + struct { + u16 iOrderByCol; /* For ORDER BY, column number in result set */ + u16 iAlias; /* Index into Parse.aAlias[] for zName */ + } x; + int iConstExprReg; /* Register in which Expr value is cached */ + } u; + } *a; /* Alloc a power of two greater or equal to nExpr */ +}; + +/* +** An instance of this structure is used by the parser to record both +** the parse tree for an expression and the span of input text for an +** expression. +*/ +struct ExprSpan { + Expr *pExpr; /* The expression parse tree */ + const char *zStart; /* First character of input text */ + const char *zEnd; /* One character past the end of input text */ +}; + +/* +** An instance of this structure can hold a simple list of identifiers, +** such as the list "a,b,c" in the following statements: +** +** INSERT INTO t(a,b,c) VALUES ...; +** CREATE INDEX idx ON t(a,b,c); +** CREATE TRIGGER trig BEFORE UPDATE ON t(a,b,c) ...; +** +** The IdList.a.idx field is used when the IdList represents the list of +** column names after a table name in an INSERT statement. In the statement +** +** INSERT INTO t(a,b,c) ... +** +** If "a" is the k-th column of table "t", then IdList.a[0].idx==k. +*/ +struct IdList { + struct IdList_item { + char *zName; /* Name of the identifier */ + int idx; /* Index in some Table.aCol[] of a column named zName */ + } *a; + int nId; /* Number of identifiers on the list */ +}; + +/* +** The bitmask datatype defined below is used for various optimizations. +** +** Changing this from a 64-bit to a 32-bit type limits the number of +** tables in a join to 32 instead of 64. But it also reduces the size +** of the library by 738 bytes on ix86. +*/ +typedef u64 Bitmask; + +/* +** The number of bits in a Bitmask. "BMS" means "BitMask Size". +*/ +#define BMS ((int)(sizeof(Bitmask)*8)) + +/* +** A bit in a Bitmask +*/ +#define MASKBIT(n) (((Bitmask)1)<<(n)) +#define MASKBIT32(n) (((unsigned int)1)<<(n)) + +/* +** The following structure describes the FROM clause of a SELECT statement. +** Each table or subquery in the FROM clause is a separate element of +** the SrcList.a[] array. +** +** With the addition of multiple database support, the following structure +** can also be used to describe a particular table such as the table that +** is modified by an INSERT, DELETE, or UPDATE statement. In standard SQL, +** such a table must be a simple name: ID. But in SQLite, the table can +** now be identified by a database name, a dot, then the table name: ID.ID. +** +** The jointype starts out showing the join type between the current table +** and the next table on the list. The parser builds the list this way. +** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each +** jointype expresses the join between the table and the previous table. +** +** In the colUsed field, the high-order bit (bit 63) is set if the table +** contains more than 63 columns and the 64-th or later column is used. +*/ +struct SrcList { + int nSrc; /* Number of tables or subqueries in the FROM clause */ + u32 nAlloc; /* Number of entries allocated in a[] below */ + struct SrcList_item { + Schema *pSchema; /* Schema to which this item is fixed */ + char *zDatabase; /* Name of database holding this table */ + char *zName; /* Name of the table */ + char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ + Table *pTab; /* An SQL table corresponding to zName */ + Select *pSelect; /* A SELECT statement used in place of a table name */ + int addrFillSub; /* Address of subroutine to manifest a subquery */ + int regReturn; /* Register holding return address of addrFillSub */ + int regResult; /* Registers holding results of a co-routine */ + u8 jointype; /* Type of join between this able and the previous */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ +#ifndef SQLITE_OMIT_EXPLAIN + u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ +#endif + int iCursor; /* The VDBE cursor number used to access this table */ + Expr *pOn; /* The ON clause of a join */ + IdList *pUsing; /* The USING clause of a join */ + Bitmask colUsed; /* Bit N (1<" clause */ + Index *pIndex; /* Index structure corresponding to zIndex, if any */ + } a[1]; /* One entry for each identifier on the list */ +}; + +/* +** Permitted values of the SrcList.a.jointype field +*/ +#define JT_INNER 0x0001 /* Any kind of inner or cross join */ +#define JT_CROSS 0x0002 /* Explicit use of the CROSS keyword */ +#define JT_NATURAL 0x0004 /* True for a "natural" join */ +#define JT_LEFT 0x0008 /* Left outer join */ +#define JT_RIGHT 0x0010 /* Right outer join */ +#define JT_OUTER 0x0020 /* The "OUTER" keyword is present */ +#define JT_ERROR 0x0040 /* unknown or unsupported join type */ + + +/* +** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin() +** and the WhereInfo.wctrlFlags member. +*/ +#define WHERE_ORDERBY_NORMAL 0x0000 /* No-op */ +#define WHERE_ORDERBY_MIN 0x0001 /* ORDER BY processing for min() func */ +#define WHERE_ORDERBY_MAX 0x0002 /* ORDER BY processing for max() func */ +#define WHERE_ONEPASS_DESIRED 0x0004 /* Want to do one-pass UPDATE/DELETE */ +#define WHERE_DUPLICATES_OK 0x0008 /* Ok to return a row more than once */ +#define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ +#define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ +#define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ +#define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ +#define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ +#define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ +#define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ +#define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ +#define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ + +/* Allowed return values from sqlite3WhereIsDistinct() +*/ +#define WHERE_DISTINCT_NOOP 0 /* DISTINCT keyword not used */ +#define WHERE_DISTINCT_UNIQUE 1 /* No duplicates */ +#define WHERE_DISTINCT_ORDERED 2 /* All duplicates are adjacent */ +#define WHERE_DISTINCT_UNORDERED 3 /* Duplicates are scattered */ + +/* +** A NameContext defines a context in which to resolve table and column +** names. The context consists of a list of tables (the pSrcList) field and +** a list of named expression (pEList). The named expression list may +** be NULL. The pSrc corresponds to the FROM clause of a SELECT or +** to the table being operated on by INSERT, UPDATE, or DELETE. The +** pEList corresponds to the result set of a SELECT and is NULL for +** other statements. +** +** NameContexts can be nested. When resolving names, the inner-most +** context is searched first. If no match is found, the next outer +** context is checked. If there is still no match, the next context +** is checked. This process continues until either a match is found +** or all contexts are check. When a match is found, the nRef member of +** the context containing the match is incremented. +** +** Each subquery gets a new NameContext. The pNext field points to the +** NameContext in the parent query. Thus the process of scanning the +** NameContext list corresponds to searching through successively outer +** subqueries looking for a match. +*/ +struct NameContext { + Parse *pParse; /* The parser */ + SrcList *pSrcList; /* One or more tables used to resolve names */ + ExprList *pEList; /* Optional list of result-set columns */ + AggInfo *pAggInfo; /* Information about aggregates at this level */ + NameContext *pNext; /* Next outer name context. NULL for outermost */ + int nRef; /* Number of names resolved by this context */ + int nErr; /* Number of errors encountered while resolving names */ + u8 ncFlags; /* Zero or more NC_* flags defined below */ +}; + +/* +** Allowed values for the NameContext, ncFlags field. +*/ +#define NC_AllowAgg 0x01 /* Aggregate functions are allowed here */ +#define NC_HasAgg 0x02 /* One or more aggregate functions seen */ +#define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ +#define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ +#define NC_PartIdx 0x10 /* True if resolving a partial index WHERE */ + +/* +** An instance of the following structure contains all information +** needed to generate code for a single SELECT statement. +** +** nLimit is set to -1 if there is no LIMIT clause. nOffset is set to 0. +** If there is a LIMIT clause, the parser sets nLimit to the value of the +** limit and nOffset to the value of the offset (or 0 if there is not +** offset). But later on, nLimit and nOffset become the memory locations +** in the VDBE that record the limit and offset counters. +** +** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes. +** These addresses must be stored so that we can go back and fill in +** the P4_KEYINFO and P2 parameters later. Neither the KeyInfo nor +** the number of columns in P2 can be computed at the same time +** as the OP_OpenEphm instruction is coded because not +** enough information about the compound query is known at that point. +** The KeyInfo for addrOpenTran[0] and [1] contains collating sequences +** for the result set. The KeyInfo for addrOpenEphm[2] contains collating +** sequences for the ORDER BY clause. +*/ +struct Select { + ExprList *pEList; /* The fields of the result */ + u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ + u16 selFlags; /* Various SF_* values */ + int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ + int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ + u64 nSelectRow; /* Estimated number of result rows */ + SrcList *pSrc; /* The FROM clause */ + Expr *pWhere; /* The WHERE clause */ + ExprList *pGroupBy; /* The GROUP BY clause */ + Expr *pHaving; /* The HAVING clause */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Select *pPrior; /* Prior select in a compound select statement */ + Select *pNext; /* Next select to the left in a compound */ + Expr *pLimit; /* LIMIT expression. NULL means not used. */ + Expr *pOffset; /* OFFSET expression. NULL means not used. */ + With *pWith; /* WITH clause attached to this select. Or NULL. */ +}; + +/* +** Allowed values for Select.selFlags. The "SF" prefix stands for +** "Select Flag". +*/ +#define SF_Distinct 0x0001 /* Output should be DISTINCT */ +#define SF_Resolved 0x0002 /* Identifiers have been resolved */ +#define SF_Aggregate 0x0004 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ + /* 0x0040 NOT USED */ +#define SF_Values 0x0080 /* Synthesized from VALUES clause */ + /* 0x0100 NOT USED */ +#define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ +#define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ +#define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ +#define SF_Compound 0x1000 /* Part of a compound query */ + + +/* +** The results of a SELECT can be distributed in several ways, as defined +** by one of the following macros. The "SRT" prefix means "SELECT Result +** Type". +** +** SRT_Union Store results as a key in a temporary index +** identified by pDest->iSDParm. +** +** SRT_Except Remove results from the temporary index pDest->iSDParm. +** +** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result +** set is not empty. +** +** SRT_Discard Throw the results away. This is used by SELECT +** statements within triggers whose only purpose is +** the side-effects of functions. +** +** All of the above are free to ignore their ORDER BY clause. Those that +** follow must honor the ORDER BY clause. +** +** SRT_Output Generate a row of output (using the OP_ResultRow +** opcode) for each row in the result set. +** +** SRT_Mem Only valid if the result is a single column. +** Store the first column of the first result row +** in register pDest->iSDParm then abandon the rest +** of the query. This destination implies "LIMIT 1". +** +** SRT_Set The result must be a single column. Store each +** row of result as the key in table pDest->iSDParm. +** Apply the affinity pDest->affSdst before storing +** results. Used to implement "IN (SELECT ...)". +** +** SRT_EphemTab Create an temporary table pDest->iSDParm and store +** the result there. The cursor is left open after +** returning. This is like SRT_Table except that +** this destination uses OP_OpenEphemeral to create +** the table first. +** +** SRT_Coroutine Generate a co-routine that returns a new row of +** results each time it is invoked. The entry point +** of the co-routine is stored in register pDest->iSDParm +** and the result row is stored in pDest->nDest registers +** starting with pDest->iSdst. +** +** SRT_Table Store results in temporary table pDest->iSDParm. +** SRT_Fifo This is like SRT_EphemTab except that the table +** is assumed to already be open. SRT_Fifo has +** the additional property of being able to ignore +** the ORDER BY clause. +** +** SRT_DistFifo Store results in a temporary table pDest->iSDParm. +** But also use temporary table pDest->iSDParm+1 as +** a record of all prior results and ignore any duplicate +** rows. Name means: "Distinct Fifo". +** +** SRT_Queue Store results in priority queue pDest->iSDParm (really +** an index). Append a sequence number so that all entries +** are distinct. +** +** SRT_DistQueue Store results in priority queue pDest->iSDParm only if +** the same record has never been stored before. The +** index at pDest->iSDParm+1 hold all prior stores. +*/ +#define SRT_Union 1 /* Store result as keys in an index */ +#define SRT_Except 2 /* Remove result from a UNION index */ +#define SRT_Exists 3 /* Store 1 if the result is not empty */ +#define SRT_Discard 4 /* Do not save the results anywhere */ +#define SRT_Fifo 5 /* Store result as data with an automatic rowid */ +#define SRT_DistFifo 6 /* Like SRT_Fifo, but unique results only */ +#define SRT_Queue 7 /* Store result in an queue */ +#define SRT_DistQueue 8 /* Like SRT_Queue, but unique results only */ + +/* The ORDER BY clause is ignored for all of the above */ +#define IgnorableOrderby(X) ((X->eDest)<=SRT_DistQueue) + +#define SRT_Output 9 /* Output each row of result */ +#define SRT_Mem 10 /* Store result in a memory cell */ +#define SRT_Set 11 /* Store results as keys in an index */ +#define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 13 /* Generate a single row of result */ +#define SRT_Table 14 /* Store result as data with an automatic rowid */ + +/* +** An instance of this object describes where to put of the results of +** a SELECT statement. +*/ +struct SelectDest { + u8 eDest; /* How to dispose of the results. On of SRT_* above. */ + char affSdst; /* Affinity used when eDest==SRT_Set */ + int iSDParm; /* A parameter used by the eDest disposal method */ + int iSdst; /* Base register where results are written */ + int nSdst; /* Number of registers allocated */ + ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ +}; + +/* +** During code generation of statements that do inserts into AUTOINCREMENT +** tables, the following information is attached to the Table.u.autoInc.p +** pointer of each autoincrement table to record some side information that +** the code generator needs. We have to keep per-table autoincrement +** information in case inserts are down within triggers. Triggers do not +** normally coordinate their activities, but we do need to coordinate the +** loading and saving of autoincrement information. +*/ +struct AutoincInfo { + AutoincInfo *pNext; /* Next info block in a list of them all */ + Table *pTab; /* Table this info block refers to */ + int iDb; /* Index in sqlite3.aDb[] of database holding pTab */ + int regCtr; /* Memory register holding the rowid counter */ +}; + +/* +** Size of the column cache +*/ +#ifndef SQLITE_N_COLCACHE +# define SQLITE_N_COLCACHE 10 +#endif + +/* +** At least one instance of the following structure is created for each +** trigger that may be fired while parsing an INSERT, UPDATE or DELETE +** statement. All such objects are stored in the linked list headed at +** Parse.pTriggerPrg and deleted once statement compilation has been +** completed. +** +** A Vdbe sub-program that implements the body and WHEN clause of trigger +** TriggerPrg.pTrigger, assuming a default ON CONFLICT clause of +** TriggerPrg.orconf, is stored in the TriggerPrg.pProgram variable. +** The Parse.pTriggerPrg list never contains two entries with the same +** values for both pTrigger and orconf. +** +** The TriggerPrg.aColmask[0] variable is set to a mask of old.* columns +** accessed (or set to 0 for triggers fired as a result of INSERT +** statements). Similarly, the TriggerPrg.aColmask[1] variable is set to +** a mask of new.* columns used by the program. +*/ +struct TriggerPrg { + Trigger *pTrigger; /* Trigger this program was coded from */ + TriggerPrg *pNext; /* Next entry in Parse.pTriggerPrg list */ + SubProgram *pProgram; /* Program implementing pTrigger/orconf */ + int orconf; /* Default ON CONFLICT policy */ + u32 aColmask[2]; /* Masks of old.*, new.* columns accessed */ +}; + +/* +** The yDbMask datatype for the bitmask of all attached databases. +*/ +#if SQLITE_MAX_ATTACHED>30 + typedef unsigned char yDbMask[(SQLITE_MAX_ATTACHED+9)/8]; +# define DbMaskTest(M,I) (((M)[(I)/8]&(1<<((I)&7)))!=0) +# define DbMaskZero(M) memset((M),0,sizeof(M)) +# define DbMaskSet(M,I) (M)[(I)/8]|=(1<<((I)&7)) +# define DbMaskAllZero(M) sqlite3DbMaskAllZero(M) +# define DbMaskNonZero(M) (sqlite3DbMaskAllZero(M)==0) +#else + typedef unsigned int yDbMask; +# define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) +# define DbMaskZero(M) (M)=0 +# define DbMaskSet(M,I) (M)|=(((yDbMask)1)<<(I)) +# define DbMaskAllZero(M) (M)==0 +# define DbMaskNonZero(M) (M)!=0 +#endif + +/* +** An SQL parser context. A copy of this structure is passed through +** the parser and down into all the parser action routine in order to +** carry around information that is global to the entire parse. +** +** The structure is divided into two parts. When the parser and code +** generate call themselves recursively, the first part of the structure +** is constant but the second part is reset at the beginning and end of +** each recursion. +** +** The nTableLock and aTableLock variables are only used if the shared-cache +** feature is enabled (if sqlite3Tsd()->useSharedData is true). They are +** used to store the set of table-locks required by the statement being +** compiled. Function sqlite3TableLock() is used to add entries to the +** list. +*/ +struct Parse { + sqlite3 *db; /* The main database structure */ + char *zErrMsg; /* An error message */ + Vdbe *pVdbe; /* An engine for executing database bytecode */ + int rc; /* Return code from execution */ + u8 colNamesSet; /* TRUE after OP_ColumnName has been issued to pVdbe */ + u8 checkSchema; /* Causes schema cookie check after an error */ + u8 nested; /* Number of nested calls to the parser/code generator */ + u8 nTempReg; /* Number of temporary registers in aTempReg[] */ + u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ + u8 mayAbort; /* True if statement may throw an ABORT exception */ + u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ + u8 okConstFactor; /* OK to factor out constants */ + int aTempReg[8]; /* Holding area for temporary registers */ + int nRangeReg; /* Size of the temporary register block */ + int iRangeReg; /* First register in temporary register block */ + int nErr; /* Number of errors seen */ + int nTab; /* Number of previously allocated VDBE cursors */ + int nMem; /* Number of memory cells used so far */ + int nSet; /* Number of sets used so far */ + int nOnce; /* Number of OP_Once instructions so far */ + int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ + int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ + int ckBase; /* Base register of data during check constraints */ + int iPartIdxTab; /* Table corresponding to a partial index */ + int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ + int iCacheCnt; /* Counter used to generate aColCache[].lru values */ + int nLabel; /* Number of labels used */ + int *aLabel; /* Space to hold the labels */ + struct yColCache { + int iTable; /* Table cursor number */ + i16 iColumn; /* Table column number */ + u8 tempReg; /* iReg is a temp register that needs to be freed */ + int iLevel; /* Nesting level */ + int iReg; /* Reg with value of this column. 0 means none. */ + int lru; /* Least recently used entry has the smallest value */ + } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ + ExprList *pConstExpr;/* Constant expressions */ + Token constraintName;/* Name of the constraint currently being parsed */ + yDbMask writeMask; /* Start a write transaction on these databases */ + yDbMask cookieMask; /* Bitmask of schema verified databases */ + int cookieValue[SQLITE_MAX_ATTACHED+2]; /* Values of cookies to verify */ + int regRowid; /* Register holding rowid of CREATE TABLE entry */ + int regRoot; /* Register holding root page number for new objects */ + int nMaxArg; /* Max args passed to user function by sub-program */ +#ifndef SQLITE_OMIT_SHARED_CACHE + int nTableLock; /* Number of locks in aTableLock */ + TableLock *aTableLock; /* Required table locks for shared-cache mode */ +#endif + AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ + + /* Information used while coding trigger programs. */ + Parse *pToplevel; /* Parse structure for main program (or NULL) */ + Table *pTriggerTab; /* Table triggers are being coded for */ + int addrCrTab; /* Address of OP_CreateTable opcode on CREATE TABLE */ + int addrSkipPK; /* Address of instruction to skip PRIMARY KEY index */ + u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ + u32 oldmask; /* Mask of old.* columns referenced */ + u32 newmask; /* Mask of new.* columns referenced */ + u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ + u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ + u8 disableTriggers; /* True to disable triggers */ + + /************************************************************************ + ** Above is constant between recursions. Below is reset before and after + ** each recursion. The boundary between these two regions is determined + ** using offsetof(Parse,nVar) so the nVar field must be the first field + ** in the recursive region. + ************************************************************************/ + + int nVar; /* Number of '?' variables seen in the SQL so far */ + int nzVar; /* Number of available slots in azVar[] */ + u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ + u8 bFreeWith; /* True if pWith should be freed with parser */ + u8 explain; /* True if the EXPLAIN flag is found on the query */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ + int nVtabLock; /* Number of virtual tables to lock */ +#endif + int nAlias; /* Number of aliased result set columns */ + int nHeight; /* Expression tree height of current sub-select */ +#ifndef SQLITE_OMIT_EXPLAIN + int iSelectId; /* ID of current select for EXPLAIN output */ + int iNextSelectId; /* Next available select ID for EXPLAIN output */ +#endif + char **azVar; /* Pointers to names of parameters */ + Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ + const char *zTail; /* All SQL text past the last semicolon parsed */ + Table *pNewTable; /* A table being constructed by CREATE TABLE */ + Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ + const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ + Token sNameToken; /* Token with unqualified schema object name */ + Token sLastToken; /* The last token parsed */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + Token sArg; /* Complete text of a module argument */ + Table **apVtabLock; /* Pointer to virtual tables needing locking */ +#endif + Table *pZombieTab; /* List of Table objects to delete after code gen */ + TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ + With *pWith; /* Current WITH clause, or NULL */ +}; + +/* +** Return true if currently inside an sqlite3_declare_vtab() call. +*/ +#ifdef SQLITE_OMIT_VIRTUALTABLE + #define IN_DECLARE_VTAB 0 +#else + #define IN_DECLARE_VTAB (pParse->declareVtab) +#endif + +/* +** An instance of the following structure can be declared on a stack and used +** to save the Parse.zAuthContext value so that it can be restored later. +*/ +struct AuthContext { + const char *zAuthContext; /* Put saved Parse.zAuthContext here */ + Parse *pParse; /* The Parse structure */ +}; + +/* +** Bitfield flags for P5 value in various opcodes. +*/ +#define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ +#define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ +#define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ +#define OPFLAG_APPEND 0x08 /* This is likely to be an append */ +#define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ +#define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ +#define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ +#define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ +#define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ +#define OPFLAG_P2ISREG 0x02 /* P2 to OP_Open** is a register number */ +#define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ + +/* + * Each trigger present in the database schema is stored as an instance of + * struct Trigger. + * + * Pointers to instances of struct Trigger are stored in two ways. + * 1. In the "trigHash" hash table (part of the sqlite3* that represents the + * database). This allows Trigger structures to be retrieved by name. + * 2. All triggers associated with a single table form a linked list, using the + * pNext member of struct Trigger. A pointer to the first element of the + * linked list is stored as the "pTrigger" member of the associated + * struct Table. + * + * The "step_list" member points to the first element of a linked list + * containing the SQL statements specified as the trigger program. + */ +struct Trigger { + char *zName; /* The name of the trigger */ + char *table; /* The table or view to which the trigger applies */ + u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT */ + u8 tr_tm; /* One of TRIGGER_BEFORE, TRIGGER_AFTER */ + Expr *pWhen; /* The WHEN clause of the expression (may be NULL) */ + IdList *pColumns; /* If this is an UPDATE OF trigger, + the is stored here */ + Schema *pSchema; /* Schema containing the trigger */ + Schema *pTabSchema; /* Schema containing the table */ + TriggerStep *step_list; /* Link list of trigger program steps */ + Trigger *pNext; /* Next trigger associated with the table */ +}; + +/* +** A trigger is either a BEFORE or an AFTER trigger. The following constants +** determine which. +** +** If there are multiple triggers, you might of some BEFORE and some AFTER. +** In that cases, the constants below can be ORed together. +*/ +#define TRIGGER_BEFORE 1 +#define TRIGGER_AFTER 2 + +/* + * An instance of struct TriggerStep is used to store a single SQL statement + * that is a part of a trigger-program. + * + * Instances of struct TriggerStep are stored in a singly linked list (linked + * using the "pNext" member) referenced by the "step_list" member of the + * associated struct Trigger instance. The first element of the linked list is + * the first step of the trigger-program. + * + * The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or + * "SELECT" statement. The meanings of the other members is determined by the + * value of "op" as follows: + * + * (op == TK_INSERT) + * orconf -> stores the ON CONFLICT algorithm + * pSelect -> If this is an INSERT INTO ... SELECT ... statement, then + * this stores a pointer to the SELECT statement. Otherwise NULL. + * target -> A token holding the quoted name of the table to insert into. + * pExprList -> If this is an INSERT INTO ... VALUES ... statement, then + * this stores values to be inserted. Otherwise NULL. + * pIdList -> If this is an INSERT INTO ... () VALUES ... + * statement, then this stores the column-names to be + * inserted into. + * + * (op == TK_DELETE) + * target -> A token holding the quoted name of the table to delete from. + * pWhere -> The WHERE clause of the DELETE statement if one is specified. + * Otherwise NULL. + * + * (op == TK_UPDATE) + * target -> A token holding the quoted name of the table to update rows of. + * pWhere -> The WHERE clause of the UPDATE statement if one is specified. + * Otherwise NULL. + * pExprList -> A list of the columns to update and the expressions to update + * them to. See sqlite3Update() documentation of "pChanges" + * argument. + * + */ +struct TriggerStep { + u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT */ + u8 orconf; /* OE_Rollback etc. */ + Trigger *pTrig; /* The trigger that this step is a part of */ + Select *pSelect; /* SELECT statment or RHS of INSERT INTO .. SELECT ... */ + Token target; /* Target table for DELETE, UPDATE, INSERT */ + Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ + ExprList *pExprList; /* SET clause for UPDATE. */ + IdList *pIdList; /* Column names for INSERT */ + TriggerStep *pNext; /* Next in the link-list */ + TriggerStep *pLast; /* Last element in link-list. Valid for 1st elem only */ +}; + +/* +** The following structure contains information used by the sqliteFix... +** routines as they walk the parse tree to make database references +** explicit. +*/ +typedef struct DbFixer DbFixer; +struct DbFixer { + Parse *pParse; /* The parsing context. Error messages written here */ + Schema *pSchema; /* Fix items to this schema */ + int bVarOnly; /* Check for variable references only */ + const char *zDb; /* Make sure all objects are contained in this database */ + const char *zType; /* Type of the container - used for error messages */ + const Token *pName; /* Name of the container - used for error messages */ +}; + +/* +** An objected used to accumulate the text of a string where we +** do not necessarily know how big the string will be in the end. +*/ +struct StrAccum { + sqlite3 *db; /* Optional database for lookaside. Can be NULL */ + char *zBase; /* A base allocation. Not from malloc. */ + char *zText; /* The string collected so far */ + int nChar; /* Length of the string so far */ + int nAlloc; /* Amount of space allocated in zText */ + int mxAlloc; /* Maximum allowed string length */ + u8 useMalloc; /* 0: none, 1: sqlite3DbMalloc, 2: sqlite3_malloc */ + u8 accError; /* STRACCUM_NOMEM or STRACCUM_TOOBIG */ +}; +#define STRACCUM_NOMEM 1 +#define STRACCUM_TOOBIG 2 + +/* +** A pointer to this structure is used to communicate information +** from sqlite3Init and OP_ParseSchema into the sqlite3InitCallback. +*/ +typedef struct { + sqlite3 *db; /* The database being initialized */ + char **pzErrMsg; /* Error message stored here */ + int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ + int rc; /* Result code stored here */ +} InitData; + +/* +** Structure containing global configuration data for the SQLite library. +** +** This structure also contains some state information. +*/ +struct Sqlite3Config { + int bMemstat; /* True to enable memory status */ + int bCoreMutex; /* True to enable core mutexing */ + int bFullMutex; /* True to enable full mutexing */ + int bOpenUri; /* True to interpret filenames as URIs */ + int bUseCis; /* Use covering indices for full-scans */ + int mxStrlen; /* Maximum string length */ + int neverCorrupt; /* Database is always well-formed */ + int szLookaside; /* Default lookaside buffer size */ + int nLookaside; /* Default lookaside buffer count */ + sqlite3_mem_methods m; /* Low-level memory allocation interface */ + sqlite3_mutex_methods mutex; /* Low-level mutex interface */ + sqlite3_pcache_methods2 pcache2; /* Low-level page-cache interface */ + void *pHeap; /* Heap storage space */ + int nHeap; /* Size of pHeap[] */ + int mnReq, mxReq; /* Min and max heap requests sizes */ + sqlite3_int64 szMmap; /* mmap() space per open file */ + sqlite3_int64 mxMmap; /* Maximum value for szMmap */ + void *pScratch; /* Scratch memory */ + int szScratch; /* Size of each scratch buffer */ + int nScratch; /* Number of scratch buffers */ + void *pPage; /* Page cache memory */ + int szPage; /* Size of each page in pPage[] */ + int nPage; /* Number of pages in pPage[] */ + int mxParserStack; /* maximum depth of the parser stack */ + int sharedCacheEnabled; /* true if shared-cache mode enabled */ + /* The above might be initialized to non-zero. The following need to always + ** initially be zero, however. */ + int isInit; /* True after initialization has finished */ + int inProgress; /* True while initialization in progress */ + int isMutexInit; /* True after mutexes are initialized */ + int isMallocInit; /* True after malloc is initialized */ + int isPCacheInit; /* True after malloc is initialized */ + int nRefInitMutex; /* Number of users of pInitMutex */ + sqlite3_mutex *pInitMutex; /* Mutex used by sqlite3_initialize() */ + void (*xLog)(void*,int,const char*); /* Function for logging */ + void *pLogArg; /* First argument to xLog() */ +#ifdef SQLITE_ENABLE_SQLLOG + void(*xSqllog)(void*,sqlite3*,const char*, int); + void *pSqllogArg; +#endif +#ifdef SQLITE_VDBE_COVERAGE + /* The following callback (if not NULL) is invoked on every VDBE branch + ** operation. Set the callback using SQLITE_TESTCTRL_VDBE_COVERAGE. + */ + void (*xVdbeBranch)(void*,int iSrcLine,u8 eThis,u8 eMx); /* Callback */ + void *pVdbeBranchArg; /* 1st argument */ +#endif +#ifndef SQLITE_OMIT_BUILTIN_TEST + int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif + int bLocaltimeFault; /* True to fail localtime() calls */ +}; + +/* +** This macro is used inside of assert() statements to indicate that +** the assert is only valid on a well-formed database. Instead of: +** +** assert( X ); +** +** One writes: +** +** assert( X || CORRUPT_DB ); +** +** CORRUPT_DB is true during normal operation. CORRUPT_DB does not indicate +** that the database is definitely corrupt, only that it might be corrupt. +** For most test cases, CORRUPT_DB is set to false using a special +** sqlite3_test_control(). This enables assert() statements to prove +** things that are always true for well-formed databases. +*/ +#define CORRUPT_DB (sqlite3Config.neverCorrupt==0) + +/* +** Context pointer passed down through the tree-walk. +*/ +struct Walker { + int (*xExprCallback)(Walker*, Expr*); /* Callback for expressions */ + int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ + void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ + Parse *pParse; /* Parser context. */ + int walkerDepth; /* Number of subqueries */ + union { /* Extra data for callback */ + NameContext *pNC; /* Naming context */ + int i; /* Integer value */ + SrcList *pSrcList; /* FROM clause */ + struct SrcCount *pSrcCount; /* Counting column references */ + } u; +}; + +/* Forward declarations */ +SQLITE_PRIVATE int sqlite3WalkExpr(Walker*, Expr*); +SQLITE_PRIVATE int sqlite3WalkExprList(Walker*, ExprList*); +SQLITE_PRIVATE int sqlite3WalkSelect(Walker*, Select*); +SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker*, Select*); +SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker*, Select*); + +/* +** Return code from the parse-tree walking primitives and their +** callbacks. +*/ +#define WRC_Continue 0 /* Continue down into children */ +#define WRC_Prune 1 /* Omit children but continue walking siblings */ +#define WRC_Abort 2 /* Abandon the tree walk */ + +/* +** An instance of this structure represents a set of one or more CTEs +** (common table expressions) created by a single WITH clause. +*/ +struct With { + int nCte; /* Number of CTEs in the WITH clause */ + With *pOuter; /* Containing WITH clause, or NULL */ + struct Cte { /* For each CTE in the WITH clause.... */ + char *zName; /* Name of this CTE */ + ExprList *pCols; /* List of explicit column names, or NULL */ + Select *pSelect; /* The definition of this CTE */ + const char *zErr; /* Error message for circular references */ + } a[1]; +}; + +/* +** Assuming zIn points to the first byte of a UTF-8 character, +** advance zIn to point to the first byte of the next UTF-8 character. +*/ +#define SQLITE_SKIP_UTF8(zIn) { \ + if( (*(zIn++))>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){ zIn++; } \ + } \ +} + +/* +** The SQLITE_*_BKPT macros are substitutes for the error codes with +** the same name but without the _BKPT suffix. These macros invoke +** routines that report the line-number on which the error originated +** using sqlite3_log(). The routines also provide a convenient place +** to set a debugger breakpoint. +*/ +SQLITE_PRIVATE int sqlite3CorruptError(int); +SQLITE_PRIVATE int sqlite3MisuseError(int); +SQLITE_PRIVATE int sqlite3CantopenError(int); +#define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__) +#define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__) +#define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__) + + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +/* +** The ctype.h header is needed for non-ASCII systems. It is also +** needed by FTS3 when FTS3 is included in the amalgamation. +*/ +#if !defined(SQLITE_ASCII) || \ + (defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_AMALGAMATION)) +# include +#endif + +/* +** The following macros mimic the standard library functions toupper(), +** isspace(), isalnum(), isdigit() and isxdigit(), respectively. The +** sqlite versions only work for ASCII characters, regardless of locale. +*/ +#ifdef SQLITE_ASCII +# define sqlite3Toupper(x) ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20)) +# define sqlite3Isspace(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x01) +# define sqlite3Isalnum(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x06) +# define sqlite3Isalpha(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x02) +# define sqlite3Isdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x04) +# define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) +# define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) +#else +# define sqlite3Toupper(x) toupper((unsigned char)(x)) +# define sqlite3Isspace(x) isspace((unsigned char)(x)) +# define sqlite3Isalnum(x) isalnum((unsigned char)(x)) +# define sqlite3Isalpha(x) isalpha((unsigned char)(x)) +# define sqlite3Isdigit(x) isdigit((unsigned char)(x)) +# define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) +# define sqlite3Tolower(x) tolower((unsigned char)(x)) +#endif + +/* +** Internal function prototypes +*/ +#define sqlite3StrICmp sqlite3_stricmp +SQLITE_PRIVATE int sqlite3Strlen30(const char*); +#define sqlite3StrNICmp sqlite3_strnicmp + +SQLITE_PRIVATE int sqlite3MallocInit(void); +SQLITE_PRIVATE void sqlite3MallocEnd(void); +SQLITE_PRIVATE void *sqlite3Malloc(int); +SQLITE_PRIVATE void *sqlite3MallocZero(int); +SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3*, int); +SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3*, int); +SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3*,const char*); +SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3*,const char*, int); +SQLITE_PRIVATE void *sqlite3Realloc(void*, int); +SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *, void *, int); +SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *, void *, int); +SQLITE_PRIVATE void sqlite3DbFree(sqlite3*, void*); +SQLITE_PRIVATE int sqlite3MallocSize(void*); +SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3*, void*); +SQLITE_PRIVATE void *sqlite3ScratchMalloc(int); +SQLITE_PRIVATE void sqlite3ScratchFree(void*); +SQLITE_PRIVATE void *sqlite3PageMalloc(int); +SQLITE_PRIVATE void sqlite3PageFree(void*); +SQLITE_PRIVATE void sqlite3MemSetDefault(void); +SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); +SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); + +/* +** On systems with ample stack space and that support alloca(), make +** use of alloca() to obtain space for large automatic objects. By default, +** obtain space from malloc(). +** +** The alloca() routine never returns NULL. This will cause code paths +** that deal with sqlite3StackAlloc() failures to be unreachable. +*/ +#ifdef SQLITE_USE_ALLOCA +# define sqlite3StackAllocRaw(D,N) alloca(N) +# define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) +# define sqlite3StackFree(D,P) +#else +# define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) +# define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) +# define sqlite3StackFree(D,P) sqlite3DbFree(D,P) +#endif + +#ifdef SQLITE_ENABLE_MEMSYS3 +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void); +#endif + + +#ifndef SQLITE_MUTEX_OMIT +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void); +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3NoopMutex(void); +SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int); +SQLITE_PRIVATE int sqlite3MutexInit(void); +SQLITE_PRIVATE int sqlite3MutexEnd(void); +#endif + +SQLITE_PRIVATE int sqlite3StatusValue(int); +SQLITE_PRIVATE void sqlite3StatusAdd(int, int); +SQLITE_PRIVATE void sqlite3StatusSet(int, int); + +#ifndef SQLITE_OMIT_FLOATING_POINT +SQLITE_PRIVATE int sqlite3IsNaN(double); +#else +# define sqlite3IsNaN(X) 0 +#endif + +/* +** An instance of the following structure holds information about SQL +** functions arguments that are the parameters to the printf() function. +*/ +struct PrintfArguments { + int nArg; /* Total number of arguments */ + int nUsed; /* Number of arguments used so far */ + sqlite3_value **apArg; /* The argument values */ +}; + +#define SQLITE_PRINTF_INTERNAL 0x01 +#define SQLITE_PRINTF_SQLFUNC 0x02 +SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, u32, const char*, va_list); +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, u32, const char*, ...); +SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); +SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); +SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3*,char*,const char*,...); +#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3DebugPrintf(const char*, ...); +#endif +#if defined(SQLITE_TEST) +SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); +#endif + +/* Output formatting for SQLITE_TESTCTRL_EXPLAIN */ +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +SQLITE_PRIVATE void sqlite3ExplainBegin(Vdbe*); +SQLITE_PRIVATE void sqlite3ExplainPrintf(Vdbe*, const char*, ...); +SQLITE_PRIVATE void sqlite3ExplainNL(Vdbe*); +SQLITE_PRIVATE void sqlite3ExplainPush(Vdbe*); +SQLITE_PRIVATE void sqlite3ExplainPop(Vdbe*); +SQLITE_PRIVATE void sqlite3ExplainFinish(Vdbe*); +SQLITE_PRIVATE void sqlite3ExplainSelect(Vdbe*, Select*); +SQLITE_PRIVATE void sqlite3ExplainExpr(Vdbe*, Expr*); +SQLITE_PRIVATE void sqlite3ExplainExprList(Vdbe*, ExprList*); +SQLITE_PRIVATE const char *sqlite3VdbeExplanation(Vdbe*); +#else +# define sqlite3ExplainBegin(X) +# define sqlite3ExplainSelect(A,B) +# define sqlite3ExplainExpr(A,B) +# define sqlite3ExplainExprList(A,B) +# define sqlite3ExplainFinish(X) +# define sqlite3VdbeExplanation(X) 0 +#endif + + +SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*, ...); +SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); +SQLITE_PRIVATE int sqlite3Dequote(char*); +SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); +SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*, char **); +SQLITE_PRIVATE void sqlite3FinishCoding(Parse*); +SQLITE_PRIVATE int sqlite3GetTempReg(Parse*); +SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int); +SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int); +SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int); +SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*); +SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); +SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*); +SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); +SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*); +SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); +SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); +SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); +SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); +SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); +SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); +SQLITE_PRIVATE int sqlite3InitCallback(void*, int, char**, char**); +SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); +SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3*); +SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int); +SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); +SQLITE_PRIVATE void sqlite3BeginParse(Parse*,int); +SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); +SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); +SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); +SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index*, i16); +SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); +SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*); +SQLITE_PRIVATE void sqlite3AddNotNull(Parse*, int); +SQLITE_PRIVATE void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int); +SQLITE_PRIVATE void sqlite3AddCheckConstraint(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3AddColumnType(Parse*,Token*); +SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,ExprSpan*); +SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); +SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*); +SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, + sqlite3_vfs**,char**,char **); +SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); +SQLITE_PRIVATE int sqlite3CodeOnce(Parse *); + +#ifdef SQLITE_OMIT_BUILTIN_TEST +# define sqlite3FaultSim(X) SQLITE_OK +#else +SQLITE_PRIVATE int sqlite3FaultSim(int); +#endif + +SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32); +SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec*, u32); +SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec*, u32); +SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec*, u32, void*); +SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec*); +SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec*); +SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); + +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); +SQLITE_PRIVATE void sqlite3RowSetClear(RowSet*); +SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); +SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); +SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); + +SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); + +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); +#else +# define sqlite3ViewGetColumnNames(A,B) 0 +#endif + +#if SQLITE_MAX_ATTACHED>30 +SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask); +#endif +SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); +SQLITE_PRIVATE void sqlite3CodeDropTable(Parse*, Table*, int, int); +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); +#ifndef SQLITE_OMIT_AUTOINCREMENT +SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse); +SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); +#else +# define sqlite3AutoincrementBegin(X) +# define sqlite3AutoincrementEnd(X) +#endif +SQLITE_PRIVATE void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int); +SQLITE_PRIVATE void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); +SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*); +SQLITE_PRIVATE int sqlite3IdListIndex(IdList*,const char*); +SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int); +SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*); +SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, + Token*, Select*, Expr*, IdList*); +SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); +SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, struct SrcList_item *); +SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList*); +SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); +SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3*, IdList*); +SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3*, SrcList*); +SQLITE_PRIVATE Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**); +SQLITE_PRIVATE Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, + Expr*, int, int); +SQLITE_PRIVATE void sqlite3DropIndex(Parse*, SrcList*, int); +SQLITE_PRIVATE int sqlite3Select(Parse*, Select*, SelectDest*); +SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, + Expr*,ExprList*,u16,Expr*,Expr*); +SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); +SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); +SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) +SQLITE_PRIVATE Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); +#endif +SQLITE_PRIVATE void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); +SQLITE_PRIVATE void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); +SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); +SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); +SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); +SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); +SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); +SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); +SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse*, int, int, int); +SQLITE_PRIVATE void sqlite3ExprCachePush(Parse*); +SQLITE_PRIVATE void sqlite3ExprCachePop(Parse*); +SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); +SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); +SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); +SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprCodeAtInit(Parse*, Expr*, int, u8); +SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); +SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); +SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, u8); +#define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ +#define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ +SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse*, Expr*, int, int); +SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse*, Expr*, int, int); +SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3*,const char*, const char*); +SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*); +SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *); +SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3*,const char*, const char*); +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); +SQLITE_PRIVATE void sqlite3Vacuum(Parse*); +SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*); +SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, Token*); +SQLITE_PRIVATE int sqlite3ExprCompare(Expr*, Expr*, int); +SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList*, ExprList*, int); +SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Expr*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); +SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); +SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse*); +SQLITE_PRIVATE void sqlite3PrngSaveState(void); +SQLITE_PRIVATE void sqlite3PrngRestoreState(void); +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*,int); +SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int); +SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); +SQLITE_PRIVATE void sqlite3BeginTransaction(Parse*, int); +SQLITE_PRIVATE void sqlite3CommitTransaction(Parse*); +SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse*); +SQLITE_PRIVATE void sqlite3Savepoint(Parse*, int, Token*); +SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *); +SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*); +SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); +SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); +SQLITE_PRIVATE int sqlite3IsRowid(const char*); +SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); +SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); +SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); +SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, + u8,u8,int,int*); +SQLITE_PRIVATE void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); +SQLITE_PRIVATE int sqlite3OpenTableAndIndices(Parse*, Table*, int, int, u8*, int*, int*); +SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse*, int, int); +SQLITE_PRIVATE void sqlite3MultiWrite(Parse*); +SQLITE_PRIVATE void sqlite3MayAbort(Parse*); +SQLITE_PRIVATE void sqlite3HaltConstraint(Parse*, int, int, char*, i8, u8); +SQLITE_PRIVATE void sqlite3UniqueConstraint(Parse*, int, Index*); +SQLITE_PRIVATE void sqlite3RowidConstraint(Parse*, int, Table*); +SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3*,Expr*,int); +SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); +SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); +SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3*,IdList*); +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,Select*,int); +SQLITE_PRIVATE void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*); +SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8); +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3*); +SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void); +SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void); +SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3*); +SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); +SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); + +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) +SQLITE_PRIVATE void sqlite3MaterializeView(Parse*, Table*, Expr*, int); +#endif + +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3BeginTrigger(Parse*, Token*,Token*,int,int,IdList*,SrcList*, + Expr*,int, int); +SQLITE_PRIVATE void sqlite3FinishTrigger(Parse*, TriggerStep*, Token*); +SQLITE_PRIVATE void sqlite3DropTrigger(Parse*, SrcList*, int); +SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse*, Trigger*); +SQLITE_PRIVATE Trigger *sqlite3TriggersExist(Parse *, Table*, int, ExprList*, int *pMask); +SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *, Table *); +SQLITE_PRIVATE void sqlite3CodeRowTrigger(Parse*, Trigger *, int, ExprList*, int, Table *, + int, int, int); +SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, int, int, int); + void sqliteViewTriggers(Parse*, Table*, Expr*, int, ExprList*); +SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*, + Select*,u8); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*); +SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); +SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); +# define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) +#else +# define sqlite3TriggersExist(B,C,D,E,F) 0 +# define sqlite3DeleteTrigger(A,B) +# define sqlite3DropTriggerPtr(A,B) +# define sqlite3UnlinkAndDeleteTrigger(A,B,C) +# define sqlite3CodeRowTrigger(A,B,C,D,E,F,G,H,I) +# define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F) +# define sqlite3TriggerList(X, Y) 0 +# define sqlite3ParseToplevel(p) p +# define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 +#endif + +SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*); +SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int); +SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse*, int); +#ifndef SQLITE_OMIT_AUTHORIZATION +SQLITE_PRIVATE void sqlite3AuthRead(Parse*,Expr*,Schema*,SrcList*); +SQLITE_PRIVATE int sqlite3AuthCheck(Parse*,int, const char*, const char*, const char*); +SQLITE_PRIVATE void sqlite3AuthContextPush(Parse*, AuthContext*, const char*); +SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext*); +SQLITE_PRIVATE int sqlite3AuthReadCol(Parse*, const char *, const char *, int); +#else +# define sqlite3AuthRead(a,b,c,d) +# define sqlite3AuthCheck(a,b,c,d,e) SQLITE_OK +# define sqlite3AuthContextPush(a,b,c) +# define sqlite3AuthContextPop(a) ((void)(a)) +#endif +SQLITE_PRIVATE void sqlite3Attach(Parse*, Expr*, Expr*, Expr*); +SQLITE_PRIVATE void sqlite3Detach(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*); +SQLITE_PRIVATE int sqlite3FixSrcList(DbFixer*, SrcList*); +SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); +SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); +SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); +SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); +SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); +SQLITE_PRIVATE int sqlite3Atoi(const char*); +SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); +SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); +SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8**); +SQLITE_PRIVATE LogEst sqlite3LogEst(u64); +SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst,LogEst); +#ifndef SQLITE_OMIT_VIRTUALTABLE +SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); +#endif +SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); + +/* +** Routines to read and write variable-length integers. These used to +** be defined locally, but now we use the varint routines in the util.c +** file. Code should use the MACRO forms below, as the Varint32 versions +** are coded to assume the single byte case is already handled (which +** the MACRO form does). +*/ +SQLITE_PRIVATE int sqlite3PutVarint(unsigned char*, u64); +SQLITE_PRIVATE int sqlite3PutVarint32(unsigned char*, u32); +SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *, u64 *); +SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *, u32 *); +SQLITE_PRIVATE int sqlite3VarintLen(u64 v); + +/* +** The header of a record consists of a sequence variable-length integers. +** These integers are almost always small and are encoded as a single byte. +** The following macros take advantage this fact to provide a fast encode +** and decode of the integers in a record header. It is faster for the common +** case where the integer is a single byte. It is a little slower when the +** integer is two or more bytes. But overall it is faster. +** +** The following expressions are equivalent: +** +** x = sqlite3GetVarint32( A, &B ); +** x = sqlite3PutVarint32( A, B ); +** +** x = getVarint32( A, B ); +** x = putVarint32( A, B ); +** +*/ +#define getVarint32(A,B) \ + (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) +#define putVarint32(A,B) \ + (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ + sqlite3PutVarint32((A),(B))) +#define getVarint sqlite3GetVarint +#define putVarint sqlite3PutVarint + + +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *, Index *); +SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); +SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); +SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); +SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); +SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); +SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); +SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...); +SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +SQLITE_PRIVATE u8 sqlite3HexToInt(int h); +SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); + +#if defined(SQLITE_TEST) +SQLITE_PRIVATE const char *sqlite3ErrName(int); +#endif + +SQLITE_PRIVATE const char *sqlite3ErrStr(int); +SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); +SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); +SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); +SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); +SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); +SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *, const char *); +SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, int); +SQLITE_PRIVATE int sqlite3AddInt64(i64*,i64); +SQLITE_PRIVATE int sqlite3SubInt64(i64*,i64); +SQLITE_PRIVATE int sqlite3MulInt64(i64*,i64); +SQLITE_PRIVATE int sqlite3AbsInt32(int); +#ifdef SQLITE_ENABLE_8_3_NAMES +SQLITE_PRIVATE void sqlite3FileSuffix3(const char*, char*); +#else +# define sqlite3FileSuffix3(X,Y) +#endif +SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z,u8); + +SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); +SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); +SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, + void(*)(void*)); +SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); +SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); +SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *); +SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8); +SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **); +SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); +#ifndef SQLITE_AMALGAMATION +SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; +SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; +SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; +SQLITE_PRIVATE const Token sqlite3IntTokens[]; +SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; +SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; +#ifndef SQLITE_OMIT_WSD +SQLITE_PRIVATE int sqlite3PendingByte; +#endif +#endif +SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3*, int, int, int); +SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*); +SQLITE_PRIVATE void sqlite3AlterFunctions(void); +SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); +SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); +SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*); +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *, Expr *, int, int); +SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); +SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); +SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); +SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); +SQLITE_PRIVATE void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); +SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); +SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *, Table *, int, int); +SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *); +SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *, SrcList *); +SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); +SQLITE_PRIVATE char sqlite3AffinityType(const char*, u8*); +SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*); +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler*); +SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*); +SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *, const char *); +SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3*,int iDB); +SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3*,Index*); +SQLITE_PRIVATE void sqlite3DefaultRowEst(Index*); +SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3*, int); +SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*); +SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse*, int, int); +SQLITE_PRIVATE void sqlite3SchemaClear(void *); +SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *, Btree *); +SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3*,int,int); +SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo*); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo*); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); +#endif +SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, + void (*)(sqlite3_context*,int,sqlite3_value **), + void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), + FuncDestructor *pDestructor +); +SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); +SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); + +SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, char*, int, int); +SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum*,const char*,int); +SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum*,const char*); +SQLITE_PRIVATE void sqlite3AppendSpace(StrAccum*,int); +SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); +SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum*); +SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest*,int,int); +SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); + +SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *); +SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void); +SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); +SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); +SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord*); +SQLITE_PRIVATE int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**); +#endif + +/* +** The interface to the LEMON-generated parser +*/ +SQLITE_PRIVATE void *sqlite3ParserAlloc(void*(*)(size_t)); +SQLITE_PRIVATE void sqlite3ParserFree(void*, void(*)(void*)); +SQLITE_PRIVATE void sqlite3Parser(void*, int, Token, Parse*); +#ifdef YYTRACKMAXSTACKDEPTH +SQLITE_PRIVATE int sqlite3ParserStackPeak(void*); +#endif + +SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3*); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3*); +#else +# define sqlite3CloseExtensions(X) +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE +SQLITE_PRIVATE void sqlite3TableLock(Parse *, int, int, u8, const char *); +#else + #define sqlite3TableLock(v,w,x,y,z) +#endif + +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); +#endif + +#ifdef SQLITE_OMIT_VIRTUALTABLE +# define sqlite3VtabClear(Y) +# define sqlite3VtabSync(X,Y) SQLITE_OK +# define sqlite3VtabRollback(X) +# define sqlite3VtabCommit(X) +# define sqlite3VtabInSync(db) 0 +# define sqlite3VtabLock(X) +# define sqlite3VtabUnlock(X) +# define sqlite3VtabUnlockList(X) +# define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK +# define sqlite3GetVTable(X,Y) ((VTable*)0) +#else +SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table*); +SQLITE_PRIVATE void sqlite3VtabDisconnect(sqlite3 *db, Table *p); +SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, Vdbe*); +SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db); +SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); +SQLITE_PRIVATE void sqlite3VtabLock(VTable *); +SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); +SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3*); +SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *, int, int); +SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); +SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); +# define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) +#endif +SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); +SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int); +SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*); +SQLITE_PRIVATE void sqlite3VtabArgInit(Parse*); +SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse*, Token*); +SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3*, int, const char *, char **); +SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse*, Table*); +SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3*, int, const char *); +SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *, VTable *); +SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); +SQLITE_PRIVATE void sqlite3InvalidFunction(sqlite3_context*,int,sqlite3_value**); +SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); +SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); +SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); +SQLITE_PRIVATE void sqlite3ParserReset(Parse*); +SQLITE_PRIVATE int sqlite3Reprepare(Vdbe*); +SQLITE_PRIVATE void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); +SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); +SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3*); +SQLITE_PRIVATE const char *sqlite3JournalModename(int); +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); +SQLITE_PRIVATE int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); +#endif +#ifndef SQLITE_OMIT_CTE +SQLITE_PRIVATE With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*); +SQLITE_PRIVATE void sqlite3WithDelete(sqlite3*,With*); +SQLITE_PRIVATE void sqlite3WithPush(Parse*, With*, u8); +#else +#define sqlite3WithPush(x,y,z) +#define sqlite3WithDelete(x,y) +#endif + +/* Declarations for functions in fkey.c. All of these are replaced by +** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign +** key functionality is available. If OMIT_TRIGGER is defined but +** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In +** this case foreign keys are parsed, but no other functionality is +** provided (enforcement of FK constraints requires the triggers sub-system). +*/ +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) +SQLITE_PRIVATE void sqlite3FkCheck(Parse*, Table*, int, int, int*, int); +SQLITE_PRIVATE void sqlite3FkDropTable(Parse*, SrcList *, Table*); +SQLITE_PRIVATE void sqlite3FkActions(Parse*, Table*, ExprList*, int, int*, int); +SQLITE_PRIVATE int sqlite3FkRequired(Parse*, Table*, int*, int); +SQLITE_PRIVATE u32 sqlite3FkOldmask(Parse*, Table*); +SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *); +#else + #define sqlite3FkActions(a,b,c,d,e,f) + #define sqlite3FkCheck(a,b,c,d,e,f) + #define sqlite3FkDropTable(a,b,c) + #define sqlite3FkOldmask(a,b) 0 + #define sqlite3FkRequired(a,b,c,d) 0 +#endif +#ifndef SQLITE_OMIT_FOREIGN_KEY +SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *, Table*); +SQLITE_PRIVATE int sqlite3FkLocateIndex(Parse*,Table*,FKey*,Index**,int**); +#else + #define sqlite3FkDelete(a,b) + #define sqlite3FkLocateIndex(a,b,c,d,e) +#endif + + +/* +** Available fault injectors. Should be numbered beginning with 0. +*/ +#define SQLITE_FAULTINJECTOR_MALLOC 0 +#define SQLITE_FAULTINJECTOR_COUNT 1 + +/* +** The interface to the code in fault.c used for identifying "benign" +** malloc failures. This is only present if SQLITE_OMIT_BUILTIN_TEST +** is not defined. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +SQLITE_PRIVATE void sqlite3BeginBenignMalloc(void); +SQLITE_PRIVATE void sqlite3EndBenignMalloc(void); +#else + #define sqlite3BeginBenignMalloc() + #define sqlite3EndBenignMalloc() +#endif + +/* +** Allowed return values from sqlite3FindInIndex() +*/ +#define IN_INDEX_ROWID 1 /* Search the rowid of the table */ +#define IN_INDEX_EPH 2 /* Search an ephemeral b-tree */ +#define IN_INDEX_INDEX_ASC 3 /* Existing index ASCENDING */ +#define IN_INDEX_INDEX_DESC 4 /* Existing index DESCENDING */ +#define IN_INDEX_NOOP 5 /* No table available. Use comparisons */ +/* +** Allowed flags for the 3rd parameter to sqlite3FindInIndex(). +*/ +#define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ +#define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ +#define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*); + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE +SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); +SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *); +SQLITE_PRIVATE int sqlite3JournalExists(sqlite3_file *p); +#else + #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile) + #define sqlite3JournalExists(p) 1 +#endif + +SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *); +SQLITE_PRIVATE int sqlite3MemJournalSize(void); +SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *); + +#if SQLITE_MAX_EXPR_DEPTH>0 +SQLITE_PRIVATE void sqlite3ExprSetHeight(Parse *pParse, Expr *p); +SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *); +SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse*, int); +#else + #define sqlite3ExprSetHeight(x,y) + #define sqlite3SelectExprHeight(x) 0 + #define sqlite3ExprCheckHeight(x,y) +#endif + +SQLITE_PRIVATE u32 sqlite3Get4byte(const u8*); +SQLITE_PRIVATE void sqlite3Put4byte(u8*, u32); + +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY +SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *, sqlite3 *); +SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db); +SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db); +#else + #define sqlite3ConnectionBlocked(x,y) + #define sqlite3ConnectionUnlocked(x) + #define sqlite3ConnectionClosed(x) +#endif + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3ParserTrace(FILE*, char *); +#endif + +/* +** If the SQLITE_ENABLE IOTRACE exists then the global variable +** sqlite3IoTrace is a pointer to a printf-like routine used to +** print I/O tracing messages. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +# define IOTRACE(A) if( sqlite3IoTrace ){ sqlite3IoTrace A; } +SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe*); +SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*,...); +#else +# define IOTRACE(A) +# define sqlite3VdbeIOTraceSql(X) +#endif + +/* +** These routines are available for the mem2.c debugging memory allocator +** only. They are used to verify that different "types" of memory +** allocations are properly tracked by the system. +** +** sqlite3MemdebugSetType() sets the "type" of an allocation to one of +** the MEMTYPE_* macros defined below. The type must be a bitmask with +** a single bit set. +** +** sqlite3MemdebugHasType() returns true if any of the bits in its second +** argument match the type set by the previous sqlite3MemdebugSetType(). +** sqlite3MemdebugHasType() is intended for use inside assert() statements. +** +** sqlite3MemdebugNoType() returns true if none of the bits in its second +** argument match the type set by the previous sqlite3MemdebugSetType(). +** +** Perhaps the most important point is the difference between MEMTYPE_HEAP +** and MEMTYPE_LOOKASIDE. If an allocation is MEMTYPE_LOOKASIDE, that means +** it might have been allocated by lookaside, except the allocation was +** too large or lookaside was already full. It is important to verify +** that allocations that might have been satisfied by lookaside are not +** passed back to non-lookaside free() routines. Asserts such as the +** example above are placed on the non-lookaside free() routines to verify +** this constraint. +** +** All of this is no-op for a production build. It only comes into +** play when the SQLITE_MEMDEBUG compile-time option is used. +*/ +#ifdef SQLITE_MEMDEBUG +SQLITE_PRIVATE void sqlite3MemdebugSetType(void*,u8); +SQLITE_PRIVATE int sqlite3MemdebugHasType(void*,u8); +SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); +#else +# define sqlite3MemdebugSetType(X,Y) /* no-op */ +# define sqlite3MemdebugHasType(X,Y) 1 +# define sqlite3MemdebugNoType(X,Y) 1 +#endif +#define MEMTYPE_HEAP 0x01 /* General heap allocations */ +#define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ +#define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ +#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ +#define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ + +#endif /* _SQLITEINT_H_ */ + +/************** End of sqliteInt.h *******************************************/ +/************** Begin file crypto.c ******************************************/ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC + +/* #include */ +/************** Include btreeInt.h in the middle of crypto.c *****************/ +/************** Begin file btreeInt.h ****************************************/ +/* +** 2004 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a external (disk-based) database using BTrees. +** For a detailed discussion of BTrees, refer to +** +** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3: +** "Sorting And Searching", pages 473-480. Addison-Wesley +** Publishing Company, Reading, Massachusetts. +** +** The basic idea is that each page of the file contains N database +** entries and N+1 pointers to subpages. +** +** ---------------------------------------------------------------- +** | Ptr(0) | Key(0) | Ptr(1) | Key(1) | ... | Key(N-1) | Ptr(N) | +** ---------------------------------------------------------------- +** +** All of the keys on the page that Ptr(0) points to have values less +** than Key(0). All of the keys on page Ptr(1) and its subpages have +** values greater than Key(0) and less than Key(1). All of the keys +** on Ptr(N) and its subpages have values greater than Key(N-1). And +** so forth. +** +** Finding a particular key requires reading O(log(M)) pages from the +** disk where M is the number of entries in the tree. +** +** In this implementation, a single file can hold one or more separate +** BTrees. Each BTree is identified by the index of its root page. The +** key and data for any entry are combined to form the "payload". A +** fixed amount of payload can be carried directly on the database +** page. If the payload is larger than the preset amount then surplus +** bytes are stored on overflow pages. The payload for an entry +** and the preceding pointer are combined to form a "Cell". Each +** page has a small header which contains the Ptr(N) pointer and other +** information such as the size of key and data. +** +** FORMAT DETAILS +** +** The file is divided into pages. The first page is called page 1, +** the second is page 2, and so forth. A page number of zero indicates +** "no such page". The page size can be any power of 2 between 512 and 65536. +** Each page can be either a btree page, a freelist page, an overflow +** page, or a pointer-map page. +** +** The first page is always a btree page. The first 100 bytes of the first +** page contain a special header (the "file header") that describes the file. +** The format of the file header is as follows: +** +** OFFSET SIZE DESCRIPTION +** 0 16 Header string: "SQLite format 3\000" +** 16 2 Page size in bytes. (1 means 65536) +** 18 1 File format write version +** 19 1 File format read version +** 20 1 Bytes of unused space at the end of each page +** 21 1 Max embedded payload fraction (must be 64) +** 22 1 Min embedded payload fraction (must be 32) +** 23 1 Min leaf payload fraction (must be 32) +** 24 4 File change counter +** 28 4 Reserved for future use +** 32 4 First freelist page +** 36 4 Number of freelist pages in the file +** 40 60 15 4-byte meta values passed to higher layers +** +** 40 4 Schema cookie +** 44 4 File format of schema layer +** 48 4 Size of page cache +** 52 4 Largest root-page (auto/incr_vacuum) +** 56 4 1=UTF-8 2=UTF16le 3=UTF16be +** 60 4 User version +** 64 4 Incremental vacuum mode +** 68 4 Application-ID +** 72 20 unused +** 92 4 The version-valid-for number +** 96 4 SQLITE_VERSION_NUMBER +** +** All of the integer values are big-endian (most significant byte first). +** +** The file change counter is incremented when the database is changed +** This counter allows other processes to know when the file has changed +** and thus when they need to flush their cache. +** +** The max embedded payload fraction is the amount of the total usable +** space in a page that can be consumed by a single cell for standard +** B-tree (non-LEAFDATA) tables. A value of 255 means 100%. The default +** is to limit the maximum cell size so that at least 4 cells will fit +** on one page. Thus the default max embedded payload fraction is 64. +** +** If the payload for a cell is larger than the max payload, then extra +** payload is spilled to overflow pages. Once an overflow page is allocated, +** as many bytes as possible are moved into the overflow pages without letting +** the cell size drop below the min embedded payload fraction. +** +** The min leaf payload fraction is like the min embedded payload fraction +** except that it applies to leaf nodes in a LEAFDATA tree. The maximum +** payload fraction for a LEAFDATA tree is always 100% (or 255) and it +** not specified in the header. +** +** Each btree pages is divided into three sections: The header, the +** cell pointer array, and the cell content area. Page 1 also has a 100-byte +** file header that occurs before the page header. +** +** |----------------| +** | file header | 100 bytes. Page 1 only. +** |----------------| +** | page header | 8 bytes for leaves. 12 bytes for interior nodes +** |----------------| +** | cell pointer | | 2 bytes per cell. Sorted order. +** | array | | Grows downward +** | | v +** |----------------| +** | unallocated | +** | space | +** |----------------| ^ Grows upwards +** | cell content | | Arbitrary order interspersed with freeblocks. +** | area | | and free space fragments. +** |----------------| +** +** The page headers looks like this: +** +** OFFSET SIZE DESCRIPTION +** 0 1 Flags. 1: intkey, 2: zerodata, 4: leafdata, 8: leaf +** 1 2 byte offset to the first freeblock +** 3 2 number of cells on this page +** 5 2 first byte of the cell content area +** 7 1 number of fragmented free bytes +** 8 4 Right child (the Ptr(N) value). Omitted on leaves. +** +** The flags define the format of this btree page. The leaf flag means that +** this page has no children. The zerodata flag means that this page carries +** only keys and no data. The intkey flag means that the key is a integer +** which is stored in the key size entry of the cell header rather than in +** the payload area. +** +** The cell pointer array begins on the first byte after the page header. +** The cell pointer array contains zero or more 2-byte numbers which are +** offsets from the beginning of the page to the cell content in the cell +** content area. The cell pointers occur in sorted order. The system strives +** to keep free space after the last cell pointer so that new cells can +** be easily added without having to defragment the page. +** +** Cell content is stored at the very end of the page and grows toward the +** beginning of the page. +** +** Unused space within the cell content area is collected into a linked list of +** freeblocks. Each freeblock is at least 4 bytes in size. The byte offset +** to the first freeblock is given in the header. Freeblocks occur in +** increasing order. Because a freeblock must be at least 4 bytes in size, +** any group of 3 or fewer unused bytes in the cell content area cannot +** exist on the freeblock chain. A group of 3 or fewer free bytes is called +** a fragment. The total number of bytes in all fragments is recorded. +** in the page header at offset 7. +** +** SIZE DESCRIPTION +** 2 Byte offset of the next freeblock +** 2 Bytes in this freeblock +** +** Cells are of variable length. Cells are stored in the cell content area at +** the end of the page. Pointers to the cells are in the cell pointer array +** that immediately follows the page header. Cells is not necessarily +** contiguous or in order, but cell pointers are contiguous and in order. +** +** Cell content makes use of variable length integers. A variable +** length integer is 1 to 9 bytes where the lower 7 bits of each +** byte are used. The integer consists of all bytes that have bit 8 set and +** the first byte with bit 8 clear. The most significant byte of the integer +** appears first. A variable-length integer may not be more than 9 bytes long. +** As a special case, all 8 bytes of the 9th byte are used as data. This +** allows a 64-bit integer to be encoded in 9 bytes. +** +** 0x00 becomes 0x00000000 +** 0x7f becomes 0x0000007f +** 0x81 0x00 becomes 0x00000080 +** 0x82 0x00 becomes 0x00000100 +** 0x80 0x7f becomes 0x0000007f +** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 +** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 +** +** Variable length integers are used for rowids and to hold the number of +** bytes of key and data in a btree cell. +** +** The content of a cell looks like this: +** +** SIZE DESCRIPTION +** 4 Page number of the left child. Omitted if leaf flag is set. +** var Number of bytes of data. Omitted if the zerodata flag is set. +** var Number of bytes of key. Or the key itself if intkey flag is set. +** * Payload +** 4 First page of the overflow chain. Omitted if no overflow +** +** Overflow pages form a linked list. Each page except the last is completely +** filled with data (pagesize - 4 bytes). The last page can have as little +** as 1 byte of data. +** +** SIZE DESCRIPTION +** 4 Page number of next overflow page +** * Data +** +** Freelist pages come in two subtypes: trunk pages and leaf pages. The +** file header points to the first in a linked list of trunk page. Each trunk +** page points to multiple leaf pages. The content of a leaf page is +** unspecified. A trunk page looks like this: +** +** SIZE DESCRIPTION +** 4 Page number of next trunk page +** 4 Number of leaf pointers on this page +** * zero or more pages numbers of leaves +*/ + + +/* The following value is the maximum cell size assuming a maximum page +** size give above. +*/ +#define MX_CELL_SIZE(pBt) ((int)(pBt->pageSize-8)) + +/* The maximum number of cells on a single page of the database. This +** assumes a minimum cell size of 6 bytes (4 bytes for the cell itself +** plus 2 bytes for the index to the cell in the page header). Such +** small cells will be rare, but they are possible. +*/ +#define MX_CELL(pBt) ((pBt->pageSize-8)/6) + +/* Forward declarations */ +typedef struct MemPage MemPage; +typedef struct BtLock BtLock; + +/* +** This is a magic string that appears at the beginning of every +** SQLite database in order to identify the file as a real database. +** +** You can change this value at compile-time by specifying a +** -DSQLITE_FILE_HEADER="..." on the compiler command-line. The +** header must be exactly 16 bytes including the zero-terminator so +** the string itself should be 15 characters long. If you change +** the header, then your custom library will not be able to read +** databases generated by the standard tools and the standard tools +** will not be able to read databases created by your custom library. +*/ +#ifndef SQLITE_FILE_HEADER /* 123456789 123456 */ +# define SQLITE_FILE_HEADER "SQLite format 3" +#endif + +/* +** Page type flags. An ORed combination of these flags appear as the +** first byte of on-disk image of every BTree page. +*/ +#define PTF_INTKEY 0x01 +#define PTF_ZERODATA 0x02 +#define PTF_LEAFDATA 0x04 +#define PTF_LEAF 0x08 + +/* +** As each page of the file is loaded into memory, an instance of the following +** structure is appended and initialized to zero. This structure stores +** information about the page that is decoded from the raw file page. +** +** The pParent field points back to the parent page. This allows us to +** walk up the BTree from any leaf to the root. Care must be taken to +** unref() the parent page pointer when this page is no longer referenced. +** The pageDestructor() routine handles that chore. +** +** Access to all fields of this structure is controlled by the mutex +** stored in MemPage.pBt->mutex. +*/ +struct MemPage { + u8 isInit; /* True if previously initialized. MUST BE FIRST! */ + u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ + u8 intKey; /* True if intkey flag is set */ + u8 leaf; /* True if leaf flag is set */ + u8 hasData; /* True if this page stores data */ + u8 hdrOffset; /* 100 for page 1. 0 otherwise */ + u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ + u8 max1bytePayload; /* min(maxLocal,127) */ + u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ + u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ + u16 cellOffset; /* Index in aData of first cell pointer */ + u16 nFree; /* Number of free bytes on the page */ + u16 nCell; /* Number of cells on this page, local and ovfl */ + u16 maskPage; /* Mask for page offset */ + u16 aiOvfl[5]; /* Insert the i-th overflow cell before the aiOvfl-th + ** non-overflow cell */ + u8 *apOvfl[5]; /* Pointers to the body of overflow cells */ + BtShared *pBt; /* Pointer to BtShared that this page is part of */ + u8 *aData; /* Pointer to disk image of the page data */ + u8 *aDataEnd; /* One byte past the end of usable data */ + u8 *aCellIdx; /* The cell index area */ + DbPage *pDbPage; /* Pager page handle */ + Pgno pgno; /* Page number for this page */ +}; + +/* +** The in-memory image of a disk page has the auxiliary information appended +** to the end. EXTRA_SIZE is the number of bytes of space needed to hold +** that extra information. +*/ +#define EXTRA_SIZE sizeof(MemPage) + +/* +** A linked list of the following structures is stored at BtShared.pLock. +** Locks are added (or upgraded from READ_LOCK to WRITE_LOCK) when a cursor +** is opened on the table with root page BtShared.iTable. Locks are removed +** from this list when a transaction is committed or rolled back, or when +** a btree handle is closed. +*/ +struct BtLock { + Btree *pBtree; /* Btree handle holding this lock */ + Pgno iTable; /* Root page of table */ + u8 eLock; /* READ_LOCK or WRITE_LOCK */ + BtLock *pNext; /* Next in BtShared.pLock list */ +}; + +/* Candidate values for BtLock.eLock */ +#define READ_LOCK 1 +#define WRITE_LOCK 2 + +/* A Btree handle +** +** A database connection contains a pointer to an instance of +** this object for every database file that it has open. This structure +** is opaque to the database connection. The database connection cannot +** see the internals of this structure and only deals with pointers to +** this structure. +** +** For some database files, the same underlying database cache might be +** shared between multiple connections. In that case, each connection +** has it own instance of this object. But each instance of this object +** points to the same BtShared object. The database cache and the +** schema associated with the database file are all contained within +** the BtShared object. +** +** All fields in this structure are accessed under sqlite3.mutex. +** The pBt pointer itself may not be changed while there exists cursors +** in the referenced BtShared that point back to this Btree since those +** cursors have to go through this Btree to find their BtShared and +** they often do so without holding sqlite3.mutex. +*/ +struct Btree { + sqlite3 *db; /* The database connection holding this btree */ + BtShared *pBt; /* Sharable content of this btree */ + u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ + u8 sharable; /* True if we can share pBt with another db */ + u8 locked; /* True if db currently has pBt locked */ + int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ + int nBackup; /* Number of backup operations reading this btree */ + Btree *pNext; /* List of other sharable Btrees from the same db */ + Btree *pPrev; /* Back pointer of the same list */ +#ifndef SQLITE_OMIT_SHARED_CACHE + BtLock lock; /* Object used to lock page 1 */ +#endif +}; + +/* +** Btree.inTrans may take one of the following values. +** +** If the shared-data extension is enabled, there may be multiple users +** of the Btree structure. At most one of these may open a write transaction, +** but any number may have active read transactions. +*/ +#define TRANS_NONE 0 +#define TRANS_READ 1 +#define TRANS_WRITE 2 + +/* +** An instance of this object represents a single database file. +** +** A single database file can be in use at the same time by two +** or more database connections. When two or more connections are +** sharing the same database file, each connection has it own +** private Btree object for the file and each of those Btrees points +** to this one BtShared object. BtShared.nRef is the number of +** connections currently sharing this database file. +** +** Fields in this structure are accessed under the BtShared.mutex +** mutex, except for nRef and pNext which are accessed under the +** global SQLITE_MUTEX_STATIC_MASTER mutex. The pPager field +** may not be modified once it is initially set as long as nRef>0. +** The pSchema field may be set once under BtShared.mutex and +** thereafter is unchanged as long as nRef>0. +** +** isPending: +** +** If a BtShared client fails to obtain a write-lock on a database +** table (because there exists one or more read-locks on the table), +** the shared-cache enters 'pending-lock' state and isPending is +** set to true. +** +** The shared-cache leaves the 'pending lock' state when either of +** the following occur: +** +** 1) The current writer (BtShared.pWriter) concludes its transaction, OR +** 2) The number of locks held by other connections drops to zero. +** +** while in the 'pending-lock' state, no connection may start a new +** transaction. +** +** This feature is included to help prevent writer-starvation. +*/ +struct BtShared { + Pager *pPager; /* The page cache */ + sqlite3 *db; /* Database connection currently using this Btree */ + BtCursor *pCursor; /* A list of all open cursors */ + MemPage *pPage1; /* First page of the database */ + u8 openFlags; /* Flags to sqlite3BtreeOpen() */ +#ifndef SQLITE_OMIT_AUTOVACUUM + u8 autoVacuum; /* True if auto-vacuum is enabled */ + u8 incrVacuum; /* True if incr-vacuum is enabled */ + u8 bDoTruncate; /* True to truncate db on commit */ +#endif + u8 inTransaction; /* Transaction state */ + u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ + u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ + u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ + u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ + u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ + u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ + u32 pageSize; /* Total number of bytes on a page */ + u32 usableSize; /* Number of usable bytes on each page */ + int nTransaction; /* Number of open transactions (read + write) */ + u32 nPage; /* Number of pages in the database */ + void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ + void (*xFreeSchema)(void*); /* Destructor for BtShared.pSchema */ + sqlite3_mutex *mutex; /* Non-recursive mutex required to access this object */ + Bitvec *pHasContent; /* Set of pages moved to free-list this transaction */ +#ifndef SQLITE_OMIT_SHARED_CACHE + int nRef; /* Number of references to this structure */ + BtShared *pNext; /* Next on a list of sharable BtShared structs */ + BtLock *pLock; /* List of locks held on this shared-btree struct */ + Btree *pWriter; /* Btree with currently open write transaction */ +#endif + u8 *pTmpSpace; /* BtShared.pageSize bytes of space for tmp use */ +}; + +/* +** Allowed values for BtShared.btsFlags +*/ +#define BTS_READ_ONLY 0x0001 /* Underlying file is readonly */ +#define BTS_PAGESIZE_FIXED 0x0002 /* Page size can no longer be changed */ +#define BTS_SECURE_DELETE 0x0004 /* PRAGMA secure_delete is enabled */ +#define BTS_INITIALLY_EMPTY 0x0008 /* Database was empty at trans start */ +#define BTS_NO_WAL 0x0010 /* Do not open write-ahead-log files */ +#define BTS_EXCLUSIVE 0x0020 /* pWriter has an exclusive lock */ +#define BTS_PENDING 0x0040 /* Waiting for read-locks to clear */ + +/* +** An instance of the following structure is used to hold information +** about a cell. The parseCellPtr() function fills in this structure +** based on information extract from the raw disk page. +*/ +typedef struct CellInfo CellInfo; +struct CellInfo { + i64 nKey; /* The key for INTKEY tables, or number of bytes in key */ + u8 *pCell; /* Pointer to the start of cell content */ + u32 nData; /* Number of bytes of data */ + u32 nPayload; /* Total amount of payload */ + u16 nHeader; /* Size of the cell content header in bytes */ + u16 nLocal; /* Amount of payload held locally */ + u16 iOverflow; /* Offset to overflow page number. Zero if no overflow */ + u16 nSize; /* Size of the cell content on the main b-tree page */ +}; + +/* +** Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than +** this will be declared corrupt. This value is calculated based on a +** maximum database size of 2^31 pages a minimum fanout of 2 for a +** root-node and 3 for all other internal nodes. +** +** If a tree that appears to be taller than this is encountered, it is +** assumed that the database is corrupt. +*/ +#define BTCURSOR_MAX_DEPTH 20 + +/* +** A cursor is a pointer to a particular entry within a particular +** b-tree within a database file. +** +** The entry is identified by its MemPage and the index in +** MemPage.aCell[] of the entry. +** +** A single database file can be shared by two more database connections, +** but cursors cannot be shared. Each cursor is associated with a +** particular database connection identified BtCursor.pBtree.db. +** +** Fields in this structure are accessed under the BtShared.mutex +** found at self->pBt->mutex. +*/ +struct BtCursor { + Btree *pBtree; /* The Btree to which this cursor belongs */ + BtShared *pBt; /* The BtShared this cursor points to */ + BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ + struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ + Pgno *aOverflow; /* Cache of overflow page locations */ + CellInfo info; /* A parse of the cell we are pointing at */ + i64 nKey; /* Size of pKey, or last integer key */ + void *pKey; /* Saved key that was cursor last known position */ + Pgno pgnoRoot; /* The root page of this tree */ + int nOvflAlloc; /* Allocated size of aOverflow[] array */ + int skipNext; /* Prev() is noop if negative. Next() is noop if positive */ + u8 curFlags; /* zero or more BTCF_* flags defined below */ + u8 eState; /* One of the CURSOR_XXX constants (see below) */ + u8 hints; /* As configured by CursorSetHints() */ + i16 iPage; /* Index of current page in apPage */ + u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ + MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ +}; + +/* +** Legal values for BtCursor.curFlags +*/ +#define BTCF_WriteFlag 0x01 /* True if a write cursor */ +#define BTCF_ValidNKey 0x02 /* True if info.nKey is valid */ +#define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ +#define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ +#define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ + +/* +** Potential values for BtCursor.eState. +** +** CURSOR_INVALID: +** Cursor does not point to a valid entry. This can happen (for example) +** because the table is empty or because BtreeCursorFirst() has not been +** called. +** +** CURSOR_VALID: +** Cursor points to a valid entry. getPayload() etc. may be called. +** +** CURSOR_SKIPNEXT: +** Cursor is valid except that the Cursor.skipNext field is non-zero +** indicating that the next sqlite3BtreeNext() or sqlite3BtreePrevious() +** operation should be a no-op. +** +** CURSOR_REQUIRESEEK: +** The table that this cursor was opened on still exists, but has been +** modified since the cursor was last used. The cursor position is saved +** in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in +** this state, restoreCursorPosition() can be called to attempt to +** seek the cursor to the saved position. +** +** CURSOR_FAULT: +** A unrecoverable error (an I/O error or a malloc failure) has occurred +** on a different connection that shares the BtShared cache with this +** cursor. The error has left the cache in an inconsistent state. +** Do nothing else with this cursor. Any attempt to use the cursor +** should return the error code stored in BtCursor.skip +*/ +#define CURSOR_INVALID 0 +#define CURSOR_VALID 1 +#define CURSOR_SKIPNEXT 2 +#define CURSOR_REQUIRESEEK 3 +#define CURSOR_FAULT 4 + +/* +** The database page the PENDING_BYTE occupies. This page is never used. +*/ +# define PENDING_BYTE_PAGE(pBt) PAGER_MJ_PGNO(pBt) + +/* +** These macros define the location of the pointer-map entry for a +** database page. The first argument to each is the number of usable +** bytes on each page of the database (often 1024). The second is the +** page number to look up in the pointer map. +** +** PTRMAP_PAGENO returns the database page number of the pointer-map +** page that stores the required pointer. PTRMAP_PTROFFSET returns +** the offset of the requested map entry. +** +** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page, +** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be +** used to test if pgno is a pointer-map page. PTRMAP_ISPAGE implements +** this test. +*/ +#define PTRMAP_PAGENO(pBt, pgno) ptrmapPageno(pBt, pgno) +#define PTRMAP_PTROFFSET(pgptrmap, pgno) (5*(pgno-pgptrmap-1)) +#define PTRMAP_ISPAGE(pBt, pgno) (PTRMAP_PAGENO((pBt),(pgno))==(pgno)) + +/* +** The pointer map is a lookup table that identifies the parent page for +** each child page in the database file. The parent page is the page that +** contains a pointer to the child. Every page in the database contains +** 0 or 1 parent pages. (In this context 'database page' refers +** to any page that is not part of the pointer map itself.) Each pointer map +** entry consists of a single byte 'type' and a 4 byte parent page number. +** The PTRMAP_XXX identifiers below are the valid types. +** +** The purpose of the pointer map is to facility moving pages from one +** position in the file to another as part of autovacuum. When a page +** is moved, the pointer in its parent must be updated to point to the +** new location. The pointer map is used to locate the parent page quickly. +** +** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not +** used in this case. +** +** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number +** is not used in this case. +** +** PTRMAP_OVERFLOW1: The database page is the first page in a list of +** overflow pages. The page number identifies the page that +** contains the cell with a pointer to this overflow page. +** +** PTRMAP_OVERFLOW2: The database page is the second or later page in a list of +** overflow pages. The page-number identifies the previous +** page in the overflow page list. +** +** PTRMAP_BTREE: The database page is a non-root btree page. The page number +** identifies the parent page in the btree. +*/ +#define PTRMAP_ROOTPAGE 1 +#define PTRMAP_FREEPAGE 2 +#define PTRMAP_OVERFLOW1 3 +#define PTRMAP_OVERFLOW2 4 +#define PTRMAP_BTREE 5 + +/* A bunch of assert() statements to check the transaction state variables +** of handle p (type Btree*) are internally consistent. +*/ +#define btreeIntegrity(p) \ + assert( p->pBt->inTransaction!=TRANS_NONE || p->pBt->nTransaction==0 ); \ + assert( p->pBt->inTransaction>=p->inTrans ); + + +/* +** The ISAUTOVACUUM macro is used within balance_nonroot() to determine +** if the database supports auto-vacuum or not. Because it is used +** within an expression that is an argument to another macro +** (sqliteMallocRaw), it is not possible to use conditional compilation. +** So, this macro is defined instead. +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define ISAUTOVACUUM (pBt->autoVacuum) +#else +#define ISAUTOVACUUM 0 +#endif + + +/* +** This structure is passed around through all the sanity checking routines +** in order to keep track of some global state information. +** +** The aRef[] array is allocated so that there is 1 bit for each page in +** the database. As the integrity-check proceeds, for each page used in +** the database the corresponding bit is set. This allows integrity-check to +** detect pages that are used twice and orphaned pages (both of which +** indicate corruption). +*/ +typedef struct IntegrityCk IntegrityCk; +struct IntegrityCk { + BtShared *pBt; /* The tree being checked out */ + Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ + u8 *aPgRef; /* 1 bit per page in the db (see above) */ + Pgno nPage; /* Number of pages in the database */ + int mxErr; /* Stop accumulating errors when this reaches zero */ + int nErr; /* Number of messages written to zErrMsg so far */ + int mallocFailed; /* A memory allocation error has occurred */ + StrAccum errMsg; /* Accumulate the error message text here */ +}; + +/* +** Routines to read or write a two- and four-byte big-endian integer values. +*/ +#define get2byte(x) ((x)[0]<<8 | (x)[1]) +#define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v)) +#define get4byte sqlite3Get4byte +#define put4byte sqlite3Put4byte + +/************** End of btreeInt.h ********************************************/ +/************** Continuing where we left off in crypto.c *********************/ +/************** Include crypto.h in the middle of crypto.c *******************/ +/************** Begin file crypto.h ******************************************/ +/* +** SQLCipher +** crypto.h developed by Stephen Lombardo (Zetetic LLC) +** sjlombardo at zetetic dot net +** http://zetetic.net +** +** Copyright (c) 2008, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifndef CRYPTO_H +#define CRYPTO_H + +#if !defined (SQLCIPHER_CRYPTO_CC) \ + && !defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) \ + && !defined (SQLCIPHER_CRYPTO_OPENSSL) +#define SQLCIPHER_CRYPTO_OPENSSL +#endif + +#define FILE_HEADER_SZ 16 + +#ifndef CIPHER_VERSION +#define CIPHER_VERSION "3.2.0" +#endif + +#ifndef CIPHER +#define CIPHER "aes-256-cbc" +#endif + +#define CIPHER_DECRYPT 0 +#define CIPHER_ENCRYPT 1 + +#define CIPHER_READ_CTX 0 +#define CIPHER_WRITE_CTX 1 +#define CIPHER_READWRITE_CTX 2 + +#ifndef PBKDF2_ITER +#define PBKDF2_ITER 64000 +#endif + +/* possible flags for cipher_ctx->flags */ +#define CIPHER_FLAG_HMAC 0x01 +#define CIPHER_FLAG_LE_PGNO 0x02 +#define CIPHER_FLAG_BE_PGNO 0x04 + +#ifndef DEFAULT_CIPHER_FLAGS +#define DEFAULT_CIPHER_FLAGS CIPHER_FLAG_HMAC | CIPHER_FLAG_LE_PGNO +#endif + + +/* by default, sqlcipher will use a reduced number of iterations to generate + the HMAC key / or transform a raw cipher key + */ +#ifndef FAST_PBKDF2_ITER +#define FAST_PBKDF2_ITER 2 +#endif + +/* this if a fixed random array that will be xor'd with the database salt to ensure that the + salt passed to the HMAC key derivation function is not the same as that used to derive + the encryption key. This can be overridden at compile time but it will make the resulting + binary incompatible with the default builds when using HMAC. A future version of SQLcipher + will likely allow this to be defined at runtime via pragma */ +#ifndef HMAC_SALT_MASK +#define HMAC_SALT_MASK 0x3a +#endif + +#ifndef CIPHER_MAX_IV_SZ +#define CIPHER_MAX_IV_SZ 16 +#endif + +#ifndef CIPHER_MAX_KEY_SZ +#define CIPHER_MAX_KEY_SZ 64 +#endif + + +#ifdef CODEC_DEBUG +#define CODEC_TRACE(X) {printf X;fflush(stdout);} +#else +#define CODEC_TRACE(X) +#endif + +#ifdef CODEC_DEBUG_PAGEDATA +#define CODEC_HEXDUMP(DESC,BUFFER,LEN) \ + { \ + int __pctr; \ + printf(DESC); \ + for(__pctr=0; __pctr < LEN; __pctr++) { \ + if(__pctr % 16 == 0) printf("\n%05x: ",__pctr); \ + printf("%02x ",((unsigned char*) BUFFER)[__pctr]); \ + } \ + printf("\n"); \ + fflush(stdout); \ + } +#else +#define CODEC_HEXDUMP(DESC,BUFFER,LEN) +#endif + +/* extensions defined in pager.c */ +SQLITE_PRIVATE void sqlite3pager_get_codec(Pager *pPager, void **ctx); +SQLITE_PRIVATE int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno); +SQLITE_PRIVATE sqlite3_file *sqlite3Pager_get_fd(Pager *pPager); +SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetCodec( + Pager *pPager, + void *(*xCodec)(void*,void*,Pgno,int), + void (*xCodecSizeChng)(void*,int,int), + void (*xCodecFree)(void*), + void *pCodec +); +SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetError(Pager *pPager, int error); +/* end extensions defined in pager.c */ + +/* +** Simple shared routines for converting hex char strings to binary data + */ +static int cipher_hex2int(char c) { + return (c>='0' && c<='9') ? (c)-'0' : + (c>='A' && c<='F') ? (c)-'A'+10 : + (c>='a' && c<='f') ? (c)-'a'+10 : 0; +} + +static void cipher_hex2bin(const unsigned char *hex, int sz, unsigned char *out){ + int i; + for(i = 0; i < sz; i += 2){ + out[i/2] = (cipher_hex2int(hex[i])<<4) | cipher_hex2int(hex[i+1]); + } +} + +static void cipher_bin2hex(const unsigned char* in, int sz, char *out) { + int i; + for(i=0; i < sz; i++) { + sqlite3_snprintf(3, out + (i*2), "%02x ", in[i]); + } +} + +/* extensions defined in crypto_impl.c */ +typedef struct codec_ctx codec_ctx; + +/* activation and initialization */ +void sqlcipher_activate(); +void sqlcipher_deactivate(); +int sqlcipher_codec_ctx_init(codec_ctx **, Db *, Pager *, sqlite3_file *, const void *, int); +void sqlcipher_codec_ctx_free(codec_ctx **); +int sqlcipher_codec_key_derive(codec_ctx *); +int sqlcipher_codec_key_copy(codec_ctx *, int); + +/* page cipher implementation */ +int sqlcipher_page_cipher(codec_ctx *, int, Pgno, int, int, unsigned char *, unsigned char *); + +/* context setters & getters */ +void sqlcipher_codec_ctx_set_error(codec_ctx *, int); + +int sqlcipher_codec_ctx_set_pass(codec_ctx *, const void *, int, int); +void sqlcipher_codec_get_keyspec(codec_ctx *, void **zKey, int *nKey); + +int sqlcipher_codec_ctx_set_pagesize(codec_ctx *, int); +int sqlcipher_codec_ctx_get_pagesize(codec_ctx *); +int sqlcipher_codec_ctx_get_reservesize(codec_ctx *); + +void sqlcipher_set_default_kdf_iter(int iter); +int sqlcipher_get_default_kdf_iter(); + +int sqlcipher_codec_ctx_set_kdf_iter(codec_ctx *, int, int); +int sqlcipher_codec_ctx_get_kdf_iter(codec_ctx *ctx, int); + +void* sqlcipher_codec_ctx_get_kdf_salt(codec_ctx *ctx); + +int sqlcipher_codec_ctx_set_fast_kdf_iter(codec_ctx *, int, int); +int sqlcipher_codec_ctx_get_fast_kdf_iter(codec_ctx *, int); + +int sqlcipher_codec_ctx_set_cipher(codec_ctx *, const char *, int); +const char* sqlcipher_codec_ctx_get_cipher(codec_ctx *ctx, int for_ctx); + +void* sqlcipher_codec_ctx_get_data(codec_ctx *); + +void sqlcipher_exportFunc(sqlite3_context *, int, sqlite3_value **); + +void sqlcipher_set_default_use_hmac(int use); +int sqlcipher_get_default_use_hmac(); + +void sqlcipher_set_hmac_salt_mask(unsigned char mask); +unsigned char sqlcipher_get_hmac_salt_mask(); + +int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use); +int sqlcipher_codec_ctx_get_use_hmac(codec_ctx *ctx, int for_ctx); + +int sqlcipher_codec_ctx_set_flag(codec_ctx *ctx, unsigned int flag); +int sqlcipher_codec_ctx_unset_flag(codec_ctx *ctx, unsigned int flag); +int sqlcipher_codec_ctx_get_flag(codec_ctx *ctx, unsigned int flag, int for_ctx); + +const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx); +int sqlcipher_codec_ctx_migrate(codec_ctx *ctx); +int sqlcipher_codec_add_random(codec_ctx *ctx, const char *data, int random_sz); +int sqlcipher_cipher_profile(sqlite3 *db, const char *destination); +static void sqlcipher_profile_callback(void *file, const char *sql, sqlite3_uint64 run_time); +static int sqlcipher_codec_get_store_pass(codec_ctx *ctx); +static void sqlcipher_codec_get_pass(codec_ctx *ctx, void **zKey, int *nKey); +static void sqlcipher_codec_set_store_pass(codec_ctx *ctx, int value); + +#endif +#endif +/* END SQLCIPHER */ + +/************** End of crypto.h **********************************************/ +/************** Continuing where we left off in crypto.c *********************/ + +static const char* codec_get_cipher_version() { + return CIPHER_VERSION; +} + +/* Generate code to return a string value */ +static void codec_vdbe_return_static_string(Parse *pParse, const char *zLabel, const char *value){ + Vdbe *v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, value, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); +} + +static int codec_set_btree_to_codec_pagesize(sqlite3 *db, Db *pDb, codec_ctx *ctx) { + int rc, page_sz, reserve_sz; + + page_sz = sqlcipher_codec_ctx_get_pagesize(ctx); + reserve_sz = sqlcipher_codec_ctx_get_reservesize(ctx); + + sqlite3_mutex_enter(db->mutex); + db->nextPagesize = page_sz; + + /* before forcing the page size we need to unset the BTS_PAGESIZE_FIXED flag, else + sqliteBtreeSetPageSize will block the change */ + pDb->pBt->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; + CODEC_TRACE(("codec_set_btree_to_codec_pagesize: sqlite3BtreeSetPageSize() size=%d reserve=%d\n", page_sz, reserve_sz)); + rc = sqlite3BtreeSetPageSize(pDb->pBt, page_sz, reserve_sz, 0); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +static int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, int for_ctx) { + struct Db *pDb = &db->aDb[nDb]; + CODEC_TRACE(("codec_set_pass_key: entered db=%p nDb=%d zKey=%s nKey=%d for_ctx=%d\n", db, nDb, (char *)zKey, nKey, for_ctx)); + if(pDb->pBt) { + codec_ctx *ctx; + sqlite3pager_get_codec(pDb->pBt->pBt->pPager, (void **) &ctx); + if(ctx) return sqlcipher_codec_ctx_set_pass(ctx, zKey, nKey, for_ctx); + } + return SQLITE_ERROR; +} + +int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLeft, const char *zRight) { + struct Db *pDb = &db->aDb[iDb]; + codec_ctx *ctx = NULL; + int rc; + + if(pDb->pBt) { + sqlite3pager_get_codec(pDb->pBt->pBt->pPager, (void **) &ctx); + } + + CODEC_TRACE(("sqlcipher_codec_pragma: entered db=%p iDb=%d pParse=%p zLeft=%s zRight=%s ctx=%p\n", db, iDb, pParse, zLeft, zRight, ctx)); + + if( sqlite3StrICmp(zLeft, "cipher_store_pass")==0 && zRight ) { + sqlcipher_codec_set_store_pass(ctx, sqlite3GetBoolean(zRight, 1)); + } else + if( sqlite3StrICmp(zLeft, "cipher_store_pass")==0 && !zRight ) { + char *store_pass_value = sqlite3_mprintf("%d", sqlcipher_codec_get_store_pass(ctx)); + codec_vdbe_return_static_string(pParse, "cipher_store_pass", store_pass_value); + sqlite3_free(store_pass_value); + } + if( sqlite3StrICmp(zLeft, "cipher_profile")== 0 && zRight ){ + char *profile_status = sqlite3_mprintf("%d", sqlcipher_cipher_profile(db, zRight)); + codec_vdbe_return_static_string(pParse, "cipher_profile", profile_status); + sqlite3_free(profile_status); + } else + if( sqlite3StrICmp(zLeft, "cipher_add_random")==0 && zRight ){ + if(ctx) { + char *add_random_status = sqlite3_mprintf("%d", sqlcipher_codec_add_random(ctx, zRight, sqlite3Strlen30(zRight))); + codec_vdbe_return_static_string(pParse, "cipher_add_random", add_random_status); + sqlite3_free(add_random_status); + } + } else + if( sqlite3StrICmp(zLeft, "cipher_migrate")==0 && !zRight ){ + if(ctx){ + char *migrate_status = sqlite3_mprintf("%d", sqlcipher_codec_ctx_migrate(ctx)); + codec_vdbe_return_static_string(pParse, "cipher_migrate", migrate_status); + sqlite3_free(migrate_status); + } + } else + if( sqlite3StrICmp(zLeft, "cipher_provider")==0 && !zRight ){ + if(ctx) { codec_vdbe_return_static_string(pParse, "cipher_provider", + sqlcipher_codec_get_cipher_provider(ctx)); + } + } else + if( sqlite3StrICmp(zLeft, "cipher_version")==0 && !zRight ){ + codec_vdbe_return_static_string(pParse, "cipher_version", codec_get_cipher_version()); + }else + if( sqlite3StrICmp(zLeft, "cipher")==0 ){ + if(ctx) { + if( zRight ) { + sqlcipher_codec_ctx_set_cipher(ctx, zRight, 2); // change cipher for both + }else { + codec_vdbe_return_static_string(pParse, "cipher", + sqlcipher_codec_ctx_get_cipher(ctx, 2)); + } + } + }else + if( sqlite3StrICmp(zLeft, "rekey_cipher")==0 && zRight ){ + if(ctx) sqlcipher_codec_ctx_set_cipher(ctx, zRight, 1); // change write cipher only + }else + if( sqlite3StrICmp(zLeft,"cipher_default_kdf_iter")==0 ){ + if( zRight ) { + sqlcipher_set_default_kdf_iter(atoi(zRight)); // change default KDF iterations + } else { + char *kdf_iter = sqlite3_mprintf("%d", sqlcipher_get_default_kdf_iter()); + codec_vdbe_return_static_string(pParse, "cipher_default_kdf_iter", kdf_iter); + sqlite3_free(kdf_iter); + } + }else + if( sqlite3StrICmp(zLeft, "kdf_iter")==0 ){ + if(ctx) { + if( zRight ) { + sqlcipher_codec_ctx_set_kdf_iter(ctx, atoi(zRight), 2); // change of RW PBKDF2 iteration + } else { + char *kdf_iter = sqlite3_mprintf("%d", sqlcipher_codec_ctx_get_kdf_iter(ctx, 2)); + codec_vdbe_return_static_string(pParse, "kdf_iter", kdf_iter); + sqlite3_free(kdf_iter); + } + } + }else + if( sqlite3StrICmp(zLeft, "fast_kdf_iter")==0){ + if(ctx) { + if( zRight ) { + sqlcipher_codec_ctx_set_fast_kdf_iter(ctx, atoi(zRight), 2); // change of RW PBKDF2 iteration + } else { + char *fast_kdf_iter = sqlite3_mprintf("%d", sqlcipher_codec_ctx_get_fast_kdf_iter(ctx, 2)); + codec_vdbe_return_static_string(pParse, "fast_kdf_iter", fast_kdf_iter); + sqlite3_free(fast_kdf_iter); + } + } + }else + if( sqlite3StrICmp(zLeft, "rekey_kdf_iter")==0 && zRight ){ + if(ctx) sqlcipher_codec_ctx_set_kdf_iter(ctx, atoi(zRight), 1); // write iterations only + }else + if( sqlite3StrICmp(zLeft,"cipher_page_size")==0 ){ + if(ctx) { + if( zRight ) { + int size = atoi(zRight); + rc = sqlcipher_codec_ctx_set_pagesize(ctx, size); + if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); + rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx); + if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); + } else { + char * page_size = sqlite3_mprintf("%d", sqlcipher_codec_ctx_get_pagesize(ctx)); + codec_vdbe_return_static_string(pParse, "cipher_page_size", page_size); + sqlite3_free(page_size); + } + } + }else + if( sqlite3StrICmp(zLeft,"cipher_default_use_hmac")==0 ){ + if( zRight ) { + sqlcipher_set_default_use_hmac(sqlite3GetBoolean(zRight,1)); + } else { + char *default_use_hmac = sqlite3_mprintf("%d", sqlcipher_get_default_use_hmac()); + codec_vdbe_return_static_string(pParse, "cipher_default_use_hmac", default_use_hmac); + sqlite3_free(default_use_hmac); + } + }else + if( sqlite3StrICmp(zLeft,"cipher_use_hmac")==0 ){ + if(ctx) { + if( zRight ) { + rc = sqlcipher_codec_ctx_set_use_hmac(ctx, sqlite3GetBoolean(zRight,1)); + if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); + /* since the use of hmac has changed, the page size may also change */ + rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx); + if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); + } else { + char *hmac_flag = sqlite3_mprintf("%d", sqlcipher_codec_ctx_get_use_hmac(ctx, 2)); + codec_vdbe_return_static_string(pParse, "cipher_use_hmac", hmac_flag); + sqlite3_free(hmac_flag); + } + } + }else + if( sqlite3StrICmp(zLeft,"cipher_hmac_pgno")==0 ){ + if(ctx) { + if(zRight) { + // clear both pgno endian flags + if(sqlite3StrICmp(zRight, "le") == 0) { + sqlcipher_codec_ctx_unset_flag(ctx, CIPHER_FLAG_BE_PGNO); + sqlcipher_codec_ctx_set_flag(ctx, CIPHER_FLAG_LE_PGNO); + } else if(sqlite3StrICmp(zRight, "be") == 0) { + sqlcipher_codec_ctx_unset_flag(ctx, CIPHER_FLAG_LE_PGNO); + sqlcipher_codec_ctx_set_flag(ctx, CIPHER_FLAG_BE_PGNO); + } else if(sqlite3StrICmp(zRight, "native") == 0) { + sqlcipher_codec_ctx_unset_flag(ctx, CIPHER_FLAG_LE_PGNO); + sqlcipher_codec_ctx_unset_flag(ctx, CIPHER_FLAG_BE_PGNO); + } + } else { + if(sqlcipher_codec_ctx_get_flag(ctx, CIPHER_FLAG_LE_PGNO, 2)) { + codec_vdbe_return_static_string(pParse, "cipher_hmac_pgno", "le"); + } else if(sqlcipher_codec_ctx_get_flag(ctx, CIPHER_FLAG_BE_PGNO, 2)) { + codec_vdbe_return_static_string(pParse, "cipher_hmac_pgno", "be"); + } else { + codec_vdbe_return_static_string(pParse, "cipher_hmac_pgno", "native"); + } + } + } + }else + if( sqlite3StrICmp(zLeft,"cipher_hmac_salt_mask")==0 ){ + if(ctx) { + if(zRight) { + if (sqlite3StrNICmp(zRight ,"x'", 2) == 0 && sqlite3Strlen30(zRight) == 5) { + unsigned char mask = 0; + const unsigned char *hex = (const unsigned char *)zRight+2; + cipher_hex2bin(hex,2,&mask); + sqlcipher_set_hmac_salt_mask(mask); + } + } else { + char *hmac_salt_mask = sqlite3_mprintf("%02x", sqlcipher_get_hmac_salt_mask()); + codec_vdbe_return_static_string(pParse, "cipher_hmac_salt_mask", hmac_salt_mask); + sqlite3_free(hmac_salt_mask); + } + } + }else { + return 0; + } + return 1; +} + + +/* + * sqlite3Codec can be called in multiple modes. + * encrypt mode - expected to return a pointer to the + * encrypted data without altering pData. + * decrypt mode - expected to return a pointer to pData, with + * the data decrypted in the input buffer + */ +void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { + codec_ctx *ctx = (codec_ctx *) iCtx; + int offset = 0, rc = 0; + int page_sz = sqlcipher_codec_ctx_get_pagesize(ctx); + unsigned char *pData = (unsigned char *) data; + void *buffer = sqlcipher_codec_ctx_get_data(ctx); + void *kdf_salt = sqlcipher_codec_ctx_get_kdf_salt(ctx); + CODEC_TRACE(("sqlite3Codec: entered pgno=%d, mode=%d, page_sz=%d\n", pgno, mode, page_sz)); + + /* call to derive keys if not present yet */ + if((rc = sqlcipher_codec_key_derive(ctx)) != SQLITE_OK) { + sqlcipher_codec_ctx_set_error(ctx, rc); + return NULL; + } + + if(pgno == 1) offset = FILE_HEADER_SZ; /* adjust starting pointers in data page for header offset on first page*/ + + CODEC_TRACE(("sqlite3Codec: switch mode=%d offset=%d\n", mode, offset)); + switch(mode) { + case 0: /* decrypt */ + case 2: + case 3: + if(pgno == 1) memcpy(buffer, SQLITE_FILE_HEADER, FILE_HEADER_SZ); /* copy file header to the first 16 bytes of the page */ + rc = sqlcipher_page_cipher(ctx, CIPHER_READ_CTX, pgno, CIPHER_DECRYPT, page_sz - offset, pData + offset, (unsigned char*)buffer + offset); + if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); + memcpy(pData, buffer, page_sz); /* copy buffer data back to pData and return */ + return pData; + break; + case 6: /* encrypt */ + if(pgno == 1) memcpy(buffer, kdf_salt, FILE_HEADER_SZ); /* copy salt to output buffer */ + rc = sqlcipher_page_cipher(ctx, CIPHER_WRITE_CTX, pgno, CIPHER_ENCRYPT, page_sz - offset, pData + offset, (unsigned char*)buffer + offset); + if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); + return buffer; /* return persistent buffer data, pData remains intact */ + break; + case 7: + if(pgno == 1) memcpy(buffer, kdf_salt, FILE_HEADER_SZ); /* copy salt to output buffer */ + rc = sqlcipher_page_cipher(ctx, CIPHER_READ_CTX, pgno, CIPHER_ENCRYPT, page_sz - offset, pData + offset, (unsigned char*)buffer + offset); + if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); + return buffer; /* return persistent buffer data, pData remains intact */ + break; + default: + return pData; + break; + } +} + +SQLITE_PRIVATE void sqlite3FreeCodecArg(void *pCodecArg) { + codec_ctx *ctx = (codec_ctx *) pCodecArg; + if(pCodecArg == NULL) return; + sqlcipher_codec_ctx_free(&ctx); // wipe and free allocated memory for the context + sqlcipher_deactivate(); /* cleanup related structures, OpenSSL etc, when codec is detatched */ +} + +SQLITE_PRIVATE int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { + struct Db *pDb = &db->aDb[nDb]; + + CODEC_TRACE(("sqlite3CodecAttach: entered nDb=%d zKey=%s, nKey=%d\n", nDb, (char *)zKey, nKey)); + + + if(nKey && zKey && pDb->pBt) { + int rc; + Pager *pPager = pDb->pBt->pBt->pPager; + sqlite3_file *fd = sqlite3Pager_get_fd(pPager); + codec_ctx *ctx; + + sqlcipher_activate(); /* perform internal initialization for sqlcipher */ + + sqlite3_mutex_enter(db->mutex); + + /* point the internal codec argument against the contet to be prepared */ + rc = sqlcipher_codec_ctx_init(&ctx, pDb, pDb->pBt->pBt->pPager, fd, zKey, nKey); + + if(rc != SQLITE_OK) return rc; /* initialization failed, do not attach potentially corrupted context */ + + sqlite3pager_sqlite3PagerSetCodec(sqlite3BtreePager(pDb->pBt), sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); + + codec_set_btree_to_codec_pagesize(db, pDb, ctx); + + /* force secure delete. This has the benefit of wiping internal data when deleted + and also ensures that all pages are written to disk (i.e. not skipped by + sqlite3PagerDontWrite optimizations) */ + sqlite3BtreeSecureDelete(pDb->pBt, 1); + + /* if fd is null, then this is an in-memory database and + we dont' want to overwrite the AutoVacuum settings + if not null, then set to the default */ + if(fd != NULL) { + sqlite3BtreeSetAutoVacuum(pDb->pBt, SQLITE_DEFAULT_AUTOVACUUM); + } + sqlite3_mutex_leave(db->mutex); + } + return SQLITE_OK; +} + +SQLITE_API void sqlite3_activate_see(const char* in) { + /* do nothing, security enhancements are always active */ +} + +static int sqlcipher_find_db_index(sqlite3 *db, const char *zDb) { + int db_index; + if(zDb == NULL){ + return 0; + } + for(db_index = 0; db_index < db->nDb; db_index++) { + struct Db *pDb = &db->aDb[db_index]; + if(strcmp(pDb->zName, zDb) == 0) { + return db_index; + } + } + return 0; +} + +SQLITE_API int sqlite3_key(sqlite3 *db, const void *pKey, int nKey) { + CODEC_TRACE(("sqlite3_key entered: db=%p pKey=%s nKey=%d\n", db, (char *)pKey, nKey)); + return sqlite3_key_v2(db, "main", pKey, nKey); +} + +SQLITE_API int sqlite3_key_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { + CODEC_TRACE(("sqlite3_key_v2: entered db=%p zDb=%s pKey=%s nKey=%d\n", db, zDb, (char *)pKey, nKey)); + /* attach key if db and pKey are not null and nKey is > 0 */ + if(db && pKey && nKey) { + int db_index = sqlcipher_find_db_index(db, zDb); + return sqlite3CodecAttach(db, db_index, pKey, nKey); + } + return SQLITE_ERROR; +} + +SQLITE_API int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { + CODEC_TRACE(("sqlite3_rekey entered: db=%p pKey=%s nKey=%d\n", db, (char *)pKey, nKey)); + return sqlite3_rekey_v2(db, "main", pKey, nKey); +} + +/* sqlite3_rekey_v2 +** Given a database, this will reencrypt the database using a new key. +** There is only one possible modes of operation - to encrypt a database +** that is already encrpyted. If the database is not already encrypted +** this should do nothing +** The proposed logic for this function follows: +** 1. Determine if the database is already encryptped +** 2. If there is NOT already a key present do nothing +** 3. If there is a key present, re-encrypt the database with the new key +*/ +SQLITE_API int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { + CODEC_TRACE(("sqlite3_rekey_v2: entered db=%p zDb=%s pKey=%s, nKey=%d\n", db, zDb, (char *)pKey, nKey)); + if(db && pKey && nKey) { + int db_index = sqlcipher_find_db_index(db, zDb); + struct Db *pDb = &db->aDb[db_index]; + CODEC_TRACE(("sqlite3_rekey_v2: database pDb=%p db_index:%d\n", pDb, db_index)); + if(pDb->pBt) { + codec_ctx *ctx; + int rc, page_count; + Pgno pgno; + PgHdr *page; + Pager *pPager = pDb->pBt->pBt->pPager; + + sqlite3pager_get_codec(pDb->pBt->pBt->pPager, (void **) &ctx); + + if(ctx == NULL) { + /* there was no codec attached to this database, so this should do nothing! */ + CODEC_TRACE(("sqlite3_rekey_v2: no codec attached to db, exiting\n")); + return SQLITE_OK; + } + + sqlite3_mutex_enter(db->mutex); + + codec_set_pass_key(db, db_index, pKey, nKey, CIPHER_WRITE_CTX); + + /* do stuff here to rewrite the database + ** 1. Create a transaction on the database + ** 2. Iterate through each page, reading it and then writing it. + ** 3. If that goes ok then commit and put ctx->rekey into ctx->key + ** note: don't deallocate rekey since it may be used in a subsequent iteration + */ + rc = sqlite3BtreeBeginTrans(pDb->pBt, 1); /* begin write transaction */ + sqlite3PagerPagecount(pPager, &page_count); + for(pgno = 1; rc == SQLITE_OK && pgno <= (unsigned int)page_count; pgno++) { /* pgno's start at 1 see pager.c:pagerAcquire */ + if(!sqlite3pager_is_mj_pgno(pPager, pgno)) { /* skip this page (see pager.c:pagerAcquire for reasoning) */ + rc = sqlite3PagerGet(pPager, pgno, &page); + if(rc == SQLITE_OK) { /* write page see pager_incr_changecounter for example */ + rc = sqlite3PagerWrite(page); + if(rc == SQLITE_OK) { + sqlite3PagerUnref(page); + } else { + CODEC_TRACE(("sqlite3_rekey_v2: error %d occurred writing page %d\n", rc, pgno)); + } + } else { + CODEC_TRACE(("sqlite3_rekey_v2: error %d occurred getting page %d\n", rc, pgno)); + } + } + } + + /* if commit was successful commit and copy the rekey data to current key, else rollback to release locks */ + if(rc == SQLITE_OK) { + CODEC_TRACE(("sqlite3_rekey_v2: committing\n")); + rc = sqlite3BtreeCommit(pDb->pBt); + sqlcipher_codec_key_copy(ctx, CIPHER_WRITE_CTX); + } else { + CODEC_TRACE(("sqlite3_rekey_v2: rollback\n")); + sqlite3BtreeRollback(pDb->pBt, SQLITE_ABORT_ROLLBACK); + } + + sqlite3_mutex_leave(db->mutex); + } + return SQLITE_OK; + } + return SQLITE_ERROR; +} + +SQLITE_PRIVATE void sqlite3CodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { + struct Db *pDb = &db->aDb[nDb]; + CODEC_TRACE(("sqlite3CodecGetKey: entered db=%p, nDb=%d\n", db, nDb)); + if( pDb->pBt ) { + codec_ctx *ctx; + sqlite3pager_get_codec(pDb->pBt->pBt->pPager, (void **) &ctx); + if(ctx) { + if(sqlcipher_codec_get_store_pass(ctx) == 1) { + sqlcipher_codec_get_pass(ctx, zKey, nKey); + } else { + sqlcipher_codec_get_keyspec(ctx, zKey, nKey); + } + } else { + *zKey = NULL; + *nKey = 0; + } + } +} + +#ifndef OMIT_EXPORT + +/* + * Implementation of an "export" function that allows a caller + * to duplicate the main database to an attached database. This is intended + * as a conveneince for users who need to: + * + * 1. migrate from an non-encrypted database to an encrypted database + * 2. move from an encrypted database to a non-encrypted database + * 3. convert beween the various flavors of encrypted databases. + * + * This implementation is based heavily on the procedure and code used + * in vacuum.c, but is exposed as a function that allows export to any + * named attached database. + */ + +/* +** Finalize a prepared statement. If there was an error, store the +** text of the error message in *pzErrMsg. Return the result code. +** +** Based on vacuumFinalize from vacuum.c +*/ +static int sqlcipher_finalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ + int rc; + rc = sqlite3VdbeFinalize((Vdbe*)pStmt); + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + return rc; +} + +/* +** Execute zSql on database db. Return an error code. +** +** Based on execSql from vacuum.c +*/ +static int sqlcipher_execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + VVA_ONLY( int rc; ) + if( !zSql ){ + return SQLITE_NOMEM; + } + if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + return sqlite3_errcode(db); + } + VVA_ONLY( rc = ) sqlite3_step(pStmt); + assert( rc!=SQLITE_ROW ); + return sqlcipher_finalize(db, pStmt, pzErrMsg); +} + +/* +** Execute zSql on database db. The statement returns exactly +** one column. Execute this as SQL on the same database. +** +** Based on execExecSql from vacuum.c +*/ +static int sqlcipher_execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + int rc; + + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + rc = sqlcipher_execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); + if( rc!=SQLITE_OK ){ + sqlcipher_finalize(db, pStmt, pzErrMsg); + return rc; + } + } + + return sqlcipher_finalize(db, pStmt, pzErrMsg); +} + +/* + * copy database and schema from the main database to an attached database + * + * Based on sqlite3RunVacuum from vacuum.c +*/ +void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3 *db = sqlite3_context_db_handle(context); + const char* attachedDb = (const char*) sqlite3_value_text(argv[0]); + int saved_flags; /* Saved value of the db->flags */ + int saved_nChange; /* Saved value of db->nChange */ + int saved_nTotalChange; /* Saved value of db->nTotalChange */ + void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ + int rc = SQLITE_OK; /* Return code from service routines */ + char *zSql = NULL; /* SQL statements */ + char *pzErrMsg = NULL; + + saved_flags = db->flags; + saved_nChange = db->nChange; + saved_nTotalChange = db->nTotalChange; + saved_xTrace = db->xTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); + db->xTrace = 0; + + /* Query the schema of the main database. Create a mirror schema + ** in the temporary database. + */ + zSql = sqlite3_mprintf( + "SELECT 'CREATE TABLE %s.' || substr(sql,14) " + " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" + " AND rootpage>0" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = sqlite3_mprintf( + "SELECT 'CREATE INDEX %s.' || substr(sql,14)" + " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %%' " + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = sqlite3_mprintf( + "SELECT 'CREATE UNIQUE INDEX %s.' || substr(sql,21) " + " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %%'" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + /* Loop through the tables in the main database. For each, do + ** an "INSERT INTO rekey_db.xxx SELECT * FROM main.xxx;" to copy + ** the contents to the temporary database. + */ + zSql = sqlite3_mprintf( + "SELECT 'INSERT INTO %s.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';'" + "FROM main.sqlite_master " + "WHERE type = 'table' AND name!='sqlite_sequence' " + " AND rootpage>0" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + /* Copy over the sequence table + */ + zSql = sqlite3_mprintf( + "SELECT 'DELETE FROM %s.' || quote(name) || ';' " + "FROM %s.sqlite_master WHERE name='sqlite_sequence' " + , attachedDb, attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = sqlite3_mprintf( + "SELECT 'INSERT INTO %s.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';' " + "FROM %s.sqlite_master WHERE name=='sqlite_sequence';" + , attachedDb, attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + /* Copy the triggers, views, and virtual tables from the main database + ** over to the temporary database. None of these objects has any + ** associated storage, so all we have to do is copy their entries + ** from the SQLITE_MASTER table. + */ + zSql = sqlite3_mprintf( + "INSERT INTO %s.sqlite_master " + " SELECT type, name, tbl_name, rootpage, sql" + " FROM main.sqlite_master" + " WHERE type='view' OR type='trigger'" + " OR (type='table' AND rootpage=0)" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = NULL; +end_of_export: + db->flags = saved_flags; + db->nChange = saved_nChange; + db->nTotalChange = saved_nTotalChange; + db->xTrace = saved_xTrace; + + sqlite3_free(zSql); + + if(rc) { + if(pzErrMsg != NULL) { + sqlite3_result_error(context, pzErrMsg, -1); + sqlite3DbFree(db, pzErrMsg); + } else { + sqlite3_result_error(context, sqlite3ErrStr(rc), -1); + } + } +} + +#endif + +/* END SQLCIPHER */ +#endif + +/************** End of crypto.c **********************************************/ +/************** Begin file crypto_impl.c *************************************/ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC + +/************** Include sqlcipher.h in the middle of crypto_impl.c ***********/ +/************** Begin file sqlcipher.h ***************************************/ +/* +** SQLCipher +** sqlcipher.h developed by Stephen Lombardo (Zetetic LLC) +** sjlombardo at zetetic dot net +** http://zetetic.net +** +** Copyright (c) 2008, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifndef SQLCIPHER_H +#define SQLCIPHER_H + + +typedef struct { + int (*activate)(void *ctx); + int (*deactivate)(void *ctx); + const char* (*get_provider_name)(void *ctx); + int (*add_random)(void *ctx, void *buffer, int length); + int (*random)(void *ctx, void *buffer, int length); + int (*hmac)(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out); + int (*kdf)(void *ctx, const unsigned char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key); + int (*cipher)(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out); + int (*set_cipher)(void *ctx, const char *cipher_name); + const char* (*get_cipher)(void *ctx); + int (*get_key_sz)(void *ctx); + int (*get_iv_sz)(void *ctx); + int (*get_block_sz)(void *ctx); + int (*get_hmac_sz)(void *ctx); + int (*ctx_copy)(void *target_ctx, void *source_ctx); + int (*ctx_cmp)(void *c1, void *c2); + int (*ctx_init)(void **ctx); + int (*ctx_free)(void **ctx); +} sqlcipher_provider; + +/* utility functions */ +void sqlcipher_free(void *ptr, int sz); +void* sqlcipher_malloc(int sz); +void* sqlcipher_memset(void *v, unsigned char value, int len); +int sqlcipher_ismemset(const void *v, unsigned char value, int len); +int sqlcipher_memcmp(const void *v0, const void *v1, int len); +void sqlcipher_free(void *, int); + +/* provider interfaces */ +int sqlcipher_register_provider(sqlcipher_provider *p); +sqlcipher_provider* sqlcipher_get_provider(); + +#endif +#endif +/* END SQLCIPHER */ + + +/************** End of sqlcipher.h *******************************************/ +/************** Continuing where we left off in crypto_impl.c ****************/ +#ifndef OMIT_MEMLOCK +#if defined(__unix__) || defined(__APPLE__) +#include +#elif defined(_WIN32) +# include +#endif +#endif + +/* the default implementation of SQLCipher uses a cipher_ctx + to keep track of read / write state separately. The following + struct and associated functions are defined here */ +typedef struct { + int store_pass; + int derive_key; + int kdf_iter; + int fast_kdf_iter; + int key_sz; + int iv_sz; + int block_sz; + int pass_sz; + int reserve_sz; + int hmac_sz; + int keyspec_sz; + unsigned int flags; + unsigned char *key; + unsigned char *hmac_key; + unsigned char *pass; + char *keyspec; + sqlcipher_provider *provider; + void *provider_ctx; +} cipher_ctx; + +static unsigned int default_flags = DEFAULT_CIPHER_FLAGS; +static unsigned char hmac_salt_mask = HMAC_SALT_MASK; +static int default_kdf_iter = PBKDF2_ITER; +static unsigned int sqlcipher_activate_count = 0; +static sqlite3_mutex* sqlcipher_provider_mutex = NULL; +static sqlcipher_provider *default_provider = NULL; + +struct codec_ctx { + int kdf_salt_sz; + int page_sz; + unsigned char *kdf_salt; + unsigned char *hmac_kdf_salt; + unsigned char *buffer; + Btree *pBt; + cipher_ctx *read_ctx; + cipher_ctx *write_ctx; + unsigned int skip_read_hmac; + unsigned int need_kdf_salt; +}; + +int sqlcipher_register_provider(sqlcipher_provider *p) { + sqlite3_mutex_enter(sqlcipher_provider_mutex); + if(default_provider != NULL && default_provider != p) { + /* only free the current registerd provider if it has been initialized + and it isn't a pointer to the same provider passed to the function + (i.e. protect against a caller calling register twice for the same provider) */ + sqlcipher_free(default_provider, sizeof(sqlcipher_provider)); + } + default_provider = p; + sqlite3_mutex_leave(sqlcipher_provider_mutex); + return SQLITE_OK; +} + +/* return a pointer to the currently registered provider. This will + allow an application to fetch the current registered provider and + make minor changes to it */ +sqlcipher_provider* sqlcipher_get_provider() { + return default_provider; +} + +void sqlcipher_activate() { + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + + if(sqlcipher_provider_mutex == NULL) { + /* allocate a new mutex to guard access to the provider */ + sqlcipher_provider_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + } + + /* check to see if there is a provider registered at this point + if there no provider registered at this point, register the + default provider */ + if(sqlcipher_get_provider() == NULL) { + sqlcipher_provider *p = sqlcipher_malloc(sizeof(sqlcipher_provider)); +#if defined (SQLCIPHER_CRYPTO_CC) + extern int sqlcipher_cc_setup(sqlcipher_provider *p); + sqlcipher_cc_setup(p); +#elif defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) + extern int sqlcipher_ltc_setup(sqlcipher_provider *p); + sqlcipher_ltc_setup(p); +#elif defined (SQLCIPHER_CRYPTO_OPENSSL) + extern int sqlcipher_openssl_setup(sqlcipher_provider *p); + sqlcipher_openssl_setup(p); +#else +#error "NO DEFAULT SQLCIPHER CRYPTO PROVIDER DEFINED" +#endif + sqlcipher_register_provider(p); + } + + sqlcipher_activate_count++; /* increment activation count */ + + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); +} + +void sqlcipher_deactivate() { + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlcipher_activate_count--; + /* if no connections are using sqlcipher, cleanup globals */ + if(sqlcipher_activate_count < 1) { + sqlite3_mutex_enter(sqlcipher_provider_mutex); + if(default_provider != NULL) { + sqlcipher_free(default_provider, sizeof(sqlcipher_provider)); + default_provider = NULL; + } + sqlite3_mutex_leave(sqlcipher_provider_mutex); + + /* last connection closed, free provider mutex*/ + sqlite3_mutex_free(sqlcipher_provider_mutex); + sqlcipher_provider_mutex = NULL; + + sqlcipher_activate_count = 0; /* reset activation count */ + } + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); +} + +/* constant time memset using volitile to avoid having the memset + optimized out by the compiler. + Note: As suggested by Joachim Schipper (joachim.schipper@fox-it.com) +*/ +void* sqlcipher_memset(void *v, unsigned char value, int len) { + int i = 0; + volatile unsigned char *a = v; + + if (v == NULL) return v; + + for(i = 0; i < len; i++) { + a[i] = value; + } + + return v; +} + +/* constant time memory check tests every position of a memory segement + matches a single value (i.e. the memory is all zeros) + returns 0 if match, 1 of no match */ +int sqlcipher_ismemset(const void *v, unsigned char value, int len) { + const unsigned char *a = v; + int i = 0, result = 0; + + for(i = 0; i < len; i++) { + result |= a[i] ^ value; + } + + return (result != 0); +} + +/* constant time memory comparison routine. + returns 0 if match, 1 if no match */ +int sqlcipher_memcmp(const void *v0, const void *v1, int len) { + const unsigned char *a0 = v0, *a1 = v1; + int i = 0, result = 0; + + for(i = 0; i < len; i++) { + result |= a0[i] ^ a1[i]; + } + + return (result != 0); +} + +/** + * Free and wipe memory. Uses SQLites internal sqlite3_free so that memory + * can be countend and memory leak detection works in the test suite. + * If ptr is not null memory will be freed. + * If sz is greater than zero, the memory will be overwritten with zero before it is freed + * If sz is > 0, and not compiled with OMIT_MEMLOCK, system will attempt to unlock the + * memory segment so it can be paged + */ +void sqlcipher_free(void *ptr, int sz) { + if(ptr) { + if(sz > 0) { + sqlcipher_memset(ptr, 0, sz); +#ifndef OMIT_MEMLOCK +#if defined(__unix__) || defined(__APPLE__) + munlock(ptr, sz); +#elif defined(_WIN32) +#if !(defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP || WINAPI_FAMILY == WINAPI_FAMILY_APP)) +VirtualUnlock(ptr, sz); +#endif +#endif +#endif + } + sqlite3_free(ptr); + } +} + +/** + * allocate memory. Uses sqlite's internall malloc wrapper so memory can be + * reference counted and leak detection works. Unless compiled with OMIT_MEMLOCK + * attempts to lock the memory pages so sensitive information won't be swapped + */ +void* sqlcipher_malloc(int sz) { + void *ptr = sqlite3Malloc(sz); + sqlcipher_memset(ptr, 0, sz); +#ifndef OMIT_MEMLOCK + if(ptr) { +#if defined(__unix__) || defined(__APPLE__) + mlock(ptr, sz); +#elif defined(_WIN32) +#if !(defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP || WINAPI_FAMILY == WINAPI_FAMILY_APP)) + VirtualLock(ptr, sz); +#endif +#endif + } +#endif + return ptr; +} + + +/** + * Initialize new cipher_ctx struct. This function will allocate memory + * for the cipher context and for the key + * + * returns SQLITE_OK if initialization was successful + * returns SQLITE_NOMEM if an error occured allocating memory + */ +static int sqlcipher_cipher_ctx_init(cipher_ctx **iCtx) { + int rc; + cipher_ctx *ctx; + *iCtx = (cipher_ctx *) sqlcipher_malloc(sizeof(cipher_ctx)); + ctx = *iCtx; + if(ctx == NULL) return SQLITE_NOMEM; + + ctx->provider = (sqlcipher_provider *) sqlcipher_malloc(sizeof(sqlcipher_provider)); + if(ctx->provider == NULL) return SQLITE_NOMEM; + + /* make a copy of the provider to be used for the duration of the context */ + sqlite3_mutex_enter(sqlcipher_provider_mutex); + memcpy(ctx->provider, default_provider, sizeof(sqlcipher_provider)); + sqlite3_mutex_leave(sqlcipher_provider_mutex); + + if((rc = ctx->provider->ctx_init(&ctx->provider_ctx)) != SQLITE_OK) return rc; + ctx->key = (unsigned char *) sqlcipher_malloc(CIPHER_MAX_KEY_SZ); + ctx->hmac_key = (unsigned char *) sqlcipher_malloc(CIPHER_MAX_KEY_SZ); + if(ctx->key == NULL) return SQLITE_NOMEM; + if(ctx->hmac_key == NULL) return SQLITE_NOMEM; + + /* setup default flags */ + ctx->flags = default_flags; + + return SQLITE_OK; +} + +/** + * Free and wipe memory associated with a cipher_ctx + */ +static void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { + cipher_ctx *ctx = *iCtx; + CODEC_TRACE(("cipher_ctx_free: entered iCtx=%p\n", iCtx)); + ctx->provider->ctx_free(&ctx->provider_ctx); + sqlcipher_free(ctx->provider, sizeof(sqlcipher_provider)); + sqlcipher_free(ctx->key, ctx->key_sz); + sqlcipher_free(ctx->hmac_key, ctx->key_sz); + sqlcipher_free(ctx->pass, ctx->pass_sz); + sqlcipher_free(ctx->keyspec, ctx->keyspec_sz); + sqlcipher_free(ctx, sizeof(cipher_ctx)); +} + +/** + * Compare one cipher_ctx to another. + * + * returns 0 if all the parameters (except the derived key data) are the same + * returns 1 otherwise + */ +static int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { + int are_equal = ( + c1->iv_sz == c2->iv_sz + && c1->kdf_iter == c2->kdf_iter + && c1->fast_kdf_iter == c2->fast_kdf_iter + && c1->key_sz == c2->key_sz + && c1->pass_sz == c2->pass_sz + && c1->flags == c2->flags + && c1->hmac_sz == c2->hmac_sz + && c1->provider->ctx_cmp(c1->provider_ctx, c2->provider_ctx) + && ( + c1->pass == c2->pass + || !sqlcipher_memcmp((const unsigned char*)c1->pass, + (const unsigned char*)c2->pass, + c1->pass_sz) + )); + + CODEC_TRACE(("sqlcipher_cipher_ctx_cmp: entered \ + c1=%p c2=%p \ + c1->iv_sz=%d c2->iv_sz=%d \ + c1->kdf_iter=%d c2->kdf_iter=%d \ + c1->fast_kdf_iter=%d c2->fast_kdf_iter=%d \ + c1->key_sz=%d c2->key_sz=%d \ + c1->pass_sz=%d c2->pass_sz=%d \ + c1->flags=%d c2->flags=%d \ + c1->hmac_sz=%d c2->hmac_sz=%d \ + c1->provider_ctx=%p c2->provider_ctx=%p \ + c1->pass=%p c2->pass=%p \ + c1->pass=%s c2->pass=%s \ + provider->ctx_cmp=%d \ + sqlcipher_memcmp=%d \ + are_equal=%d \ + \n", + c1, c2, + c1->iv_sz, c2->iv_sz, + c1->kdf_iter, c2->kdf_iter, + c1->fast_kdf_iter, c2->fast_kdf_iter, + c1->key_sz, c2->key_sz, + c1->pass_sz, c2->pass_sz, + c1->flags, c2->flags, + c1->hmac_sz, c2->hmac_sz, + c1->provider_ctx, c2->provider_ctx, + c1->pass, c2->pass, + c1->pass, c2->pass, + c1->provider->ctx_cmp(c1->provider_ctx, c2->provider_ctx), + sqlcipher_memcmp((const unsigned char*)c1->pass, + (const unsigned char*)c2->pass, + c1->pass_sz), + are_equal + )); + + return !are_equal; /* return 0 if they are the same, 1 otherwise */ +} + +/** + * Copy one cipher_ctx to another. For instance, assuming that read_ctx is a + * fully initialized context, you could copy it to write_ctx and all yet data + * and pass information across + * + * returns SQLITE_OK if initialization was successful + * returns SQLITE_NOMEM if an error occured allocating memory + */ +static int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { + void *key = target->key; + void *hmac_key = target->hmac_key; + void *provider = target->provider; + void *provider_ctx = target->provider_ctx; + + CODEC_TRACE(("sqlcipher_cipher_ctx_copy: entered target=%p, source=%p\n", target, source)); + sqlcipher_free(target->pass, target->pass_sz); + sqlcipher_free(target->keyspec, target->keyspec_sz); + memcpy(target, source, sizeof(cipher_ctx)); + + target->key = key; //restore pointer to previously allocated key data + memcpy(target->key, source->key, CIPHER_MAX_KEY_SZ); + + target->hmac_key = hmac_key; //restore pointer to previously allocated hmac key data + memcpy(target->hmac_key, source->hmac_key, CIPHER_MAX_KEY_SZ); + + target->provider = provider; // restore pointer to previouly allocated provider; + memcpy(target->provider, source->provider, sizeof(sqlcipher_provider)); + + target->provider_ctx = provider_ctx; // restore pointer to previouly allocated provider context; + target->provider->ctx_copy(target->provider_ctx, source->provider_ctx); + + if(source->pass && source->pass_sz) { + target->pass = sqlcipher_malloc(source->pass_sz); + if(target->pass == NULL) return SQLITE_NOMEM; + memcpy(target->pass, source->pass, source->pass_sz); + } + if(source->keyspec && source->keyspec_sz) { + target->keyspec = sqlcipher_malloc(source->keyspec_sz); + if(target->keyspec == NULL) return SQLITE_NOMEM; + memcpy(target->keyspec, source->keyspec, source->keyspec_sz); + } + return SQLITE_OK; +} + +/** + * Set the keyspec for the cipher_ctx + * + * returns SQLITE_OK if assignment was successfull + * returns SQLITE_NOMEM if an error occured allocating memory + */ +static int sqlcipher_cipher_ctx_set_keyspec(cipher_ctx *ctx, const unsigned char *key, int key_sz, const unsigned char *salt, int salt_sz) { + + /* free, zero existing pointers and size */ + sqlcipher_free(ctx->keyspec, ctx->keyspec_sz); + ctx->keyspec = NULL; + ctx->keyspec_sz = 0; + + /* establic a hex-formated key specification, containing the raw encryption key and + the salt used to generate it */ + ctx->keyspec_sz = ((key_sz + salt_sz) * 2) + 3; + ctx->keyspec = sqlcipher_malloc(ctx->keyspec_sz); + if(ctx->keyspec == NULL) return SQLITE_NOMEM; + + ctx->keyspec[0] = 'x'; + ctx->keyspec[1] = '\''; + cipher_bin2hex(key, key_sz, ctx->keyspec + 2); + cipher_bin2hex(salt, salt_sz, ctx->keyspec + (key_sz * 2) + 2); + ctx->keyspec[ctx->keyspec_sz - 1] = '\''; + return SQLITE_OK; +} + +static int sqlcipher_codec_get_store_pass(codec_ctx *ctx) { + return ctx->read_ctx->store_pass; +} + +static void sqlcipher_codec_set_store_pass(codec_ctx *ctx, int value) { + ctx->read_ctx->store_pass = value; +} + +static void sqlcipher_codec_get_pass(codec_ctx *ctx, void **zKey, int *nKey) { + *zKey = ctx->read_ctx->pass; + *nKey = ctx->read_ctx->pass_sz; +} + +/** + * Set the passphrase for the cipher_ctx + * + * returns SQLITE_OK if assignment was successfull + * returns SQLITE_NOMEM if an error occured allocating memory + */ +static int sqlcipher_cipher_ctx_set_pass(cipher_ctx *ctx, const void *zKey, int nKey) { + + /* free, zero existing pointers and size */ + sqlcipher_free(ctx->pass, ctx->pass_sz); + ctx->pass = NULL; + ctx->pass_sz = 0; + + if(zKey && nKey) { /* if new password is provided, copy it */ + ctx->pass_sz = nKey; + ctx->pass = sqlcipher_malloc(nKey); + if(ctx->pass == NULL) return SQLITE_NOMEM; + memcpy(ctx->pass, zKey, nKey); + } + return SQLITE_OK; +} + +int sqlcipher_codec_ctx_set_pass(codec_ctx *ctx, const void *zKey, int nKey, int for_ctx) { + cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + int rc; + + if((rc = sqlcipher_cipher_ctx_set_pass(c_ctx, zKey, nKey)) != SQLITE_OK) return rc; + c_ctx->derive_key = 1; + + if(for_ctx == 2) + if((rc = sqlcipher_cipher_ctx_copy( for_ctx ? ctx->read_ctx : ctx->write_ctx, c_ctx)) != SQLITE_OK) + return rc; + + return SQLITE_OK; +} + +int sqlcipher_codec_ctx_set_cipher(codec_ctx *ctx, const char *cipher_name, int for_ctx) { + cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + int rc; + + c_ctx->provider->set_cipher(c_ctx->provider_ctx, cipher_name); + + c_ctx->key_sz = c_ctx->provider->get_key_sz(c_ctx->provider_ctx); + c_ctx->iv_sz = c_ctx->provider->get_iv_sz(c_ctx->provider_ctx); + c_ctx->block_sz = c_ctx->provider->get_block_sz(c_ctx->provider_ctx); + c_ctx->hmac_sz = c_ctx->provider->get_hmac_sz(c_ctx->provider_ctx); + c_ctx->derive_key = 1; + + if(for_ctx == 2) + if((rc = sqlcipher_cipher_ctx_copy( for_ctx ? ctx->read_ctx : ctx->write_ctx, c_ctx)) != SQLITE_OK) + return rc; + + return SQLITE_OK; +} + +const char* sqlcipher_codec_ctx_get_cipher(codec_ctx *ctx, int for_ctx) { + cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + return c_ctx->provider->get_cipher(c_ctx->provider_ctx); +} + +/* set the global default KDF iteration */ +void sqlcipher_set_default_kdf_iter(int iter) { + default_kdf_iter = iter; +} + +int sqlcipher_get_default_kdf_iter() { + return default_kdf_iter; +} + +int sqlcipher_codec_ctx_set_kdf_iter(codec_ctx *ctx, int kdf_iter, int for_ctx) { + cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + int rc; + + c_ctx->kdf_iter = kdf_iter; + c_ctx->derive_key = 1; + + if(for_ctx == 2) + if((rc = sqlcipher_cipher_ctx_copy( for_ctx ? ctx->read_ctx : ctx->write_ctx, c_ctx)) != SQLITE_OK) + return rc; + + return SQLITE_OK; +} + +int sqlcipher_codec_ctx_get_kdf_iter(codec_ctx *ctx, int for_ctx) { + cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + return c_ctx->kdf_iter; +} + +int sqlcipher_codec_ctx_set_fast_kdf_iter(codec_ctx *ctx, int fast_kdf_iter, int for_ctx) { + cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + int rc; + + c_ctx->fast_kdf_iter = fast_kdf_iter; + c_ctx->derive_key = 1; + + if(for_ctx == 2) + if((rc = sqlcipher_cipher_ctx_copy( for_ctx ? ctx->read_ctx : ctx->write_ctx, c_ctx)) != SQLITE_OK) + return rc; + + return SQLITE_OK; +} + +int sqlcipher_codec_ctx_get_fast_kdf_iter(codec_ctx *ctx, int for_ctx) { + cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + return c_ctx->fast_kdf_iter; +} + +/* set the global default flag for HMAC */ +void sqlcipher_set_default_use_hmac(int use) { + if(use) default_flags |= CIPHER_FLAG_HMAC; + else default_flags &= ~CIPHER_FLAG_HMAC; +} + +int sqlcipher_get_default_use_hmac() { + return (default_flags & CIPHER_FLAG_HMAC) != 0; +} + +void sqlcipher_set_hmac_salt_mask(unsigned char mask) { + hmac_salt_mask = mask; +} + +unsigned char sqlcipher_get_hmac_salt_mask() { + return hmac_salt_mask; +} + +/* set the codec flag for whether this individual database should be using hmac */ +int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use) { + int reserve = CIPHER_MAX_IV_SZ; /* base reserve size will be IV only */ + + if(use) reserve += ctx->read_ctx->hmac_sz; /* if reserve will include hmac, update that size */ + + /* calculate the amount of reserve needed in even increments of the cipher block size */ + + reserve = ((reserve % ctx->read_ctx->block_sz) == 0) ? reserve : + ((reserve / ctx->read_ctx->block_sz) + 1) * ctx->read_ctx->block_sz; + + CODEC_TRACE(("sqlcipher_codec_ctx_set_use_hmac: use=%d block_sz=%d md_size=%d reserve=%d\n", + use, ctx->read_ctx->block_sz, ctx->read_ctx->hmac_sz, reserve)); + + + if(use) { + sqlcipher_codec_ctx_set_flag(ctx, CIPHER_FLAG_HMAC); + } else { + sqlcipher_codec_ctx_unset_flag(ctx, CIPHER_FLAG_HMAC); + } + + ctx->write_ctx->reserve_sz = ctx->read_ctx->reserve_sz = reserve; + + return SQLITE_OK; +} + +int sqlcipher_codec_ctx_get_use_hmac(codec_ctx *ctx, int for_ctx) { + cipher_ctx * c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + return (c_ctx->flags & CIPHER_FLAG_HMAC) != 0; +} + +int sqlcipher_codec_ctx_set_flag(codec_ctx *ctx, unsigned int flag) { + ctx->write_ctx->flags |= flag; + ctx->read_ctx->flags |= flag; + return SQLITE_OK; +} + +int sqlcipher_codec_ctx_unset_flag(codec_ctx *ctx, unsigned int flag) { + ctx->write_ctx->flags &= ~flag; + ctx->read_ctx->flags &= ~flag; + return SQLITE_OK; +} + +int sqlcipher_codec_ctx_get_flag(codec_ctx *ctx, unsigned int flag, int for_ctx) { + cipher_ctx * c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + return (c_ctx->flags & flag) != 0; +} + +void sqlcipher_codec_ctx_set_error(codec_ctx *ctx, int error) { + CODEC_TRACE(("sqlcipher_codec_ctx_set_error: ctx=%p, error=%d\n", ctx, error)); + sqlite3pager_sqlite3PagerSetError(ctx->pBt->pBt->pPager, error); + ctx->pBt->pBt->db->errCode = error; +} + +int sqlcipher_codec_ctx_get_reservesize(codec_ctx *ctx) { + return ctx->read_ctx->reserve_sz; +} + +void* sqlcipher_codec_ctx_get_data(codec_ctx *ctx) { + return ctx->buffer; +} + +void* sqlcipher_codec_ctx_get_kdf_salt(codec_ctx *ctx) { + return ctx->kdf_salt; +} + +void sqlcipher_codec_get_keyspec(codec_ctx *ctx, void **zKey, int *nKey) { + *zKey = ctx->read_ctx->keyspec; + *nKey = ctx->read_ctx->keyspec_sz; +} + +int sqlcipher_codec_ctx_set_pagesize(codec_ctx *ctx, int size) { + /* attempt to free the existing page buffer */ + sqlcipher_free(ctx->buffer,ctx->page_sz); + ctx->page_sz = size; + + /* pre-allocate a page buffer of PageSize bytes. This will + be used as a persistent buffer for encryption and decryption + operations to avoid overhead of multiple memory allocations*/ + ctx->buffer = sqlcipher_malloc(size); + if(ctx->buffer == NULL) return SQLITE_NOMEM; + + return SQLITE_OK; +} + +int sqlcipher_codec_ctx_get_pagesize(codec_ctx *ctx) { + return ctx->page_sz; +} + +int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, sqlite3_file *fd, const void *zKey, int nKey) { + int rc; + codec_ctx *ctx; + *iCtx = sqlcipher_malloc(sizeof(codec_ctx)); + ctx = *iCtx; + + if(ctx == NULL) return SQLITE_NOMEM; + + ctx->pBt = pDb->pBt; /* assign pointer to database btree structure */ + + /* allocate space for salt data. Then read the first 16 bytes + directly off the database file. This is the salt for the + key derivation function. If we get a short read allocate + a new random salt value */ + ctx->kdf_salt_sz = FILE_HEADER_SZ; + ctx->kdf_salt = sqlcipher_malloc(ctx->kdf_salt_sz); + if(ctx->kdf_salt == NULL) return SQLITE_NOMEM; + + /* allocate space for separate hmac salt data. We want the + HMAC derivation salt to be different than the encryption + key derivation salt */ + ctx->hmac_kdf_salt = sqlcipher_malloc(ctx->kdf_salt_sz); + if(ctx->hmac_kdf_salt == NULL) return SQLITE_NOMEM; + + + /* + Always overwrite page size and set to the default because the first page of the database + in encrypted and thus sqlite can't effectively determine the pagesize. this causes an issue in + cases where bytes 16 & 17 of the page header are a power of 2 as reported by John Lehman + */ + if((rc = sqlcipher_codec_ctx_set_pagesize(ctx, SQLITE_DEFAULT_PAGE_SIZE)) != SQLITE_OK) return rc; + + if((rc = sqlcipher_cipher_ctx_init(&ctx->read_ctx)) != SQLITE_OK) return rc; + if((rc = sqlcipher_cipher_ctx_init(&ctx->write_ctx)) != SQLITE_OK) return rc; + + if(fd == NULL || sqlite3OsRead(fd, ctx->kdf_salt, FILE_HEADER_SZ, 0) != SQLITE_OK) { + ctx->need_kdf_salt = 1; + } + + if((rc = sqlcipher_codec_ctx_set_cipher(ctx, CIPHER, 0)) != SQLITE_OK) return rc; + if((rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, default_kdf_iter, 0)) != SQLITE_OK) return rc; + if((rc = sqlcipher_codec_ctx_set_fast_kdf_iter(ctx, FAST_PBKDF2_ITER, 0)) != SQLITE_OK) return rc; + if((rc = sqlcipher_codec_ctx_set_pass(ctx, zKey, nKey, 0)) != SQLITE_OK) return rc; + + /* Note that use_hmac is a special case that requires recalculation of page size + so we call set_use_hmac to perform setup */ + if((rc = sqlcipher_codec_ctx_set_use_hmac(ctx, default_flags & CIPHER_FLAG_HMAC)) != SQLITE_OK) return rc; + + if((rc = sqlcipher_cipher_ctx_copy(ctx->write_ctx, ctx->read_ctx)) != SQLITE_OK) return rc; + + return SQLITE_OK; +} + +/** + * Free and wipe memory associated with a cipher_ctx, including the allocated + * read_ctx and write_ctx. + */ +void sqlcipher_codec_ctx_free(codec_ctx **iCtx) { + codec_ctx *ctx = *iCtx; + CODEC_TRACE(("codec_ctx_free: entered iCtx=%p\n", iCtx)); + sqlcipher_free(ctx->kdf_salt, ctx->kdf_salt_sz); + sqlcipher_free(ctx->hmac_kdf_salt, ctx->kdf_salt_sz); + sqlcipher_free(ctx->buffer, 0); + sqlcipher_cipher_ctx_free(&ctx->read_ctx); + sqlcipher_cipher_ctx_free(&ctx->write_ctx); + sqlcipher_free(ctx, sizeof(codec_ctx)); +} + +/** convert a 32bit unsigned integer to little endian byte ordering */ +static void sqlcipher_put4byte_le(unsigned char *p, u32 v) { + p[0] = (u8)v; + p[1] = (u8)(v>>8); + p[2] = (u8)(v>>16); + p[3] = (u8)(v>>24); +} + +static int sqlcipher_page_hmac(cipher_ctx *ctx, Pgno pgno, unsigned char *in, int in_sz, unsigned char *out) { + unsigned char pgno_raw[sizeof(pgno)]; + /* we may convert page number to consistent representation before calculating MAC for + compatibility across big-endian and little-endian platforms. + + Note: The public release of sqlcipher 2.0.0 to 2.0.6 had a bug where the bytes of pgno + were used directly in the MAC. SQLCipher convert's to little endian by default to preserve + backwards compatibility on the most popular platforms, but can optionally be configured + to use either big endian or native byte ordering via pragma. */ + + if(ctx->flags & CIPHER_FLAG_LE_PGNO) { /* compute hmac using little endian pgno*/ + sqlcipher_put4byte_le(pgno_raw, pgno); + } else if(ctx->flags & CIPHER_FLAG_BE_PGNO) { /* compute hmac using big endian pgno */ + sqlite3Put4byte(pgno_raw, pgno); /* sqlite3Put4byte converts 32bit uint to big endian */ + } else { /* use native byte ordering */ + memcpy(pgno_raw, &pgno, sizeof(pgno)); + } + + /* include the encrypted page data, initialization vector, and page number in HMAC. This will + prevent both tampering with the ciphertext, manipulation of the IV, or resequencing otherwise + valid pages out of order in a database */ + ctx->provider->hmac( + ctx->provider_ctx, ctx->hmac_key, + ctx->key_sz, in, + in_sz, (unsigned char*) &pgno_raw, + sizeof(pgno), out); + return SQLITE_OK; +} + +/* + * ctx - codec context + * pgno - page number in database + * size - size in bytes of input and output buffers + * mode - 1 to encrypt, 0 to decrypt + * in - pointer to input bytes + * out - pouter to output bytes + */ +int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int page_sz, unsigned char *in, unsigned char *out) { + cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; + unsigned char *iv_in, *iv_out, *hmac_in, *hmac_out, *out_start; + int size; + + /* calculate some required positions into various buffers */ + size = page_sz - c_ctx->reserve_sz; /* adjust size to useable size and memset reserve at end of page */ + iv_out = out + size; + iv_in = in + size; + + /* hmac will be written immediately after the initialization vector. the remainder of the page reserve will contain + random bytes. note, these pointers are only valid when using hmac */ + hmac_in = in + size + c_ctx->iv_sz; + hmac_out = out + size + c_ctx->iv_sz; + out_start = out; /* note the original position of the output buffer pointer, as out will be rewritten during encryption */ + + CODEC_TRACE(("codec_cipher:entered pgno=%d, mode=%d, size=%d\n", pgno, mode, size)); + CODEC_HEXDUMP("codec_cipher: input page data", in, page_sz); + + /* the key size should never be zero. If it is, error out. */ + if(c_ctx->key_sz == 0) { + CODEC_TRACE(("codec_cipher: error possible context corruption, key_sz is zero for pgno=%d\n", pgno)); + sqlcipher_memset(out, 0, page_sz); + return SQLITE_ERROR; + } + + if(mode == CIPHER_ENCRYPT) { + /* start at front of the reserve block, write random data to the end */ + if(c_ctx->provider->random(c_ctx->provider_ctx, iv_out, c_ctx->reserve_sz) != SQLITE_OK) return SQLITE_ERROR; + } else { /* CIPHER_DECRYPT */ + memcpy(iv_out, iv_in, c_ctx->iv_sz); /* copy the iv from the input to output buffer */ + } + + if((c_ctx->flags & CIPHER_FLAG_HMAC) && (mode == CIPHER_DECRYPT) && !ctx->skip_read_hmac) { + if(sqlcipher_page_hmac(c_ctx, pgno, in, size + c_ctx->iv_sz, hmac_out) != SQLITE_OK) { + sqlcipher_memset(out, 0, page_sz); + CODEC_TRACE(("codec_cipher: hmac operations failed for pgno=%d\n", pgno)); + return SQLITE_ERROR; + } + + CODEC_TRACE(("codec_cipher: comparing hmac on in=%p out=%p hmac_sz=%d\n", hmac_in, hmac_out, c_ctx->hmac_sz)); + if(sqlcipher_memcmp(hmac_in, hmac_out, c_ctx->hmac_sz) != 0) { /* the hmac check failed */ + if(sqlcipher_ismemset(in, 0, page_sz) == 0) { + /* first check if the entire contents of the page is zeros. If so, this page + resulted from a short read (i.e. sqlite attempted to pull a page after the end of the file. these + short read failures must be ignored for autovaccum mode to work so wipe the output buffer + and return SQLITE_OK to skip the decryption step. */ + CODEC_TRACE(("codec_cipher: zeroed page (short read) for pgno %d, encryption but returning SQLITE_OK\n", pgno)); + sqlcipher_memset(out, 0, page_sz); + return SQLITE_OK; + } else { + /* if the page memory is not all zeros, it means the there was data and a hmac on the page. + since the check failed, the page was either tampered with or corrupted. wipe the output buffer, + and return SQLITE_ERROR to the caller */ + CODEC_TRACE(("codec_cipher: hmac check failed for pgno=%d returning SQLITE_ERROR\n", pgno)); + sqlcipher_memset(out, 0, page_sz); + return SQLITE_ERROR; + } + } + } + + c_ctx->provider->cipher(c_ctx->provider_ctx, mode, c_ctx->key, c_ctx->key_sz, iv_out, in, size, out); + + if((c_ctx->flags & CIPHER_FLAG_HMAC) && (mode == CIPHER_ENCRYPT)) { + sqlcipher_page_hmac(c_ctx, pgno, out_start, size + c_ctx->iv_sz, hmac_out); + } + + CODEC_HEXDUMP("codec_cipher: output page data", out_start, page_sz); + + return SQLITE_OK; +} + +/** + * Derive an encryption key for a cipher contex key based on the raw password. + * + * If the raw key data is formated as x'hex' and there are exactly enough hex chars to fill + * the key (i.e 64 hex chars for a 256 bit key) then the key data will be used directly. + + * Else, if the raw key data is formated as x'hex' and there are exactly enough hex chars to fill + * the key and the salt (i.e 92 hex chars for a 256 bit key and 16 byte salt) then it will be unpacked + * as the key followed by the salt. + * + * Otherwise, a key data will be derived using PBKDF2 + * + * returns SQLITE_OK if initialization was successful + * returns SQLITE_ERROR if the key could't be derived (for instance if pass is NULL or pass_sz is 0) + */ +static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { + int rc; + CODEC_TRACE(("cipher_ctx_key_derive: entered c_ctx->pass=%s, c_ctx->pass_sz=%d \ + ctx->kdf_salt=%p ctx->kdf_salt_sz=%d c_ctx->kdf_iter=%d \ + ctx->hmac_kdf_salt=%p, c_ctx->fast_kdf_iter=%d c_ctx->key_sz=%d\n", + c_ctx->pass, c_ctx->pass_sz, ctx->kdf_salt, ctx->kdf_salt_sz, c_ctx->kdf_iter, + ctx->hmac_kdf_salt, c_ctx->fast_kdf_iter, c_ctx->key_sz)); + + + if(c_ctx->pass && c_ctx->pass_sz) { // if pass is not null + + if(ctx->need_kdf_salt) { + if(ctx->read_ctx->provider->random(ctx->read_ctx->provider_ctx, ctx->kdf_salt, FILE_HEADER_SZ) != SQLITE_OK) return SQLITE_ERROR; + ctx->need_kdf_salt = 0; + } + if (c_ctx->pass_sz == ((c_ctx->key_sz * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0) { + int n = c_ctx->pass_sz - 3; /* adjust for leading x' and tailing ' */ + const unsigned char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ + CODEC_TRACE(("cipher_ctx_key_derive: using raw key from hex\n")); + cipher_hex2bin(z, n, c_ctx->key); + } else if (c_ctx->pass_sz == (((c_ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0) { + const unsigned char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ + CODEC_TRACE(("cipher_ctx_key_derive: using raw key from hex\n")); + cipher_hex2bin(z, (c_ctx->key_sz * 2), c_ctx->key); + cipher_hex2bin(z + (c_ctx->key_sz * 2), (ctx->kdf_salt_sz * 2), ctx->kdf_salt); + } else { + CODEC_TRACE(("cipher_ctx_key_derive: deriving key using full PBKDF2 with %d iterations\n", c_ctx->kdf_iter)); + c_ctx->provider->kdf(c_ctx->provider_ctx, c_ctx->pass, c_ctx->pass_sz, + ctx->kdf_salt, ctx->kdf_salt_sz, c_ctx->kdf_iter, + c_ctx->key_sz, c_ctx->key); + } + + /* set the context "keyspec" containing the hex-formatted key and salt to be used when attaching databases */ + if((rc = sqlcipher_cipher_ctx_set_keyspec(c_ctx, c_ctx->key, c_ctx->key_sz, ctx->kdf_salt, ctx->kdf_salt_sz)) != SQLITE_OK) return rc; + + /* if this context is setup to use hmac checks, generate a seperate and different + key for HMAC. In this case, we use the output of the previous KDF as the input to + this KDF run. This ensures a distinct but predictable HMAC key. */ + if(c_ctx->flags & CIPHER_FLAG_HMAC) { + int i; + + /* start by copying the kdf key into the hmac salt slot + then XOR it with the fixed hmac salt defined at compile time + this ensures that the salt passed in to derive the hmac key, while + easy to derive and publically known, is not the same as the salt used + to generate the encryption key */ + memcpy(ctx->hmac_kdf_salt, ctx->kdf_salt, ctx->kdf_salt_sz); + for(i = 0; i < ctx->kdf_salt_sz; i++) { + ctx->hmac_kdf_salt[i] ^= hmac_salt_mask; + } + + CODEC_TRACE(("cipher_ctx_key_derive: deriving hmac key from encryption key using PBKDF2 with %d iterations\n", + c_ctx->fast_kdf_iter)); + + + c_ctx->provider->kdf(c_ctx->provider_ctx, c_ctx->key, c_ctx->key_sz, + ctx->hmac_kdf_salt, ctx->kdf_salt_sz, c_ctx->fast_kdf_iter, + c_ctx->key_sz, c_ctx->hmac_key); + } + + c_ctx->derive_key = 0; + return SQLITE_OK; + }; + return SQLITE_ERROR; +} + +int sqlcipher_codec_key_derive(codec_ctx *ctx) { + /* derive key on first use if necessary */ + if(ctx->read_ctx->derive_key) { + if(sqlcipher_cipher_ctx_key_derive(ctx, ctx->read_ctx) != SQLITE_OK) return SQLITE_ERROR; + } + + if(ctx->write_ctx->derive_key) { + if(sqlcipher_cipher_ctx_cmp(ctx->write_ctx, ctx->read_ctx) == 0) { + /* the relevant parameters are the same, just copy read key */ + if(sqlcipher_cipher_ctx_copy(ctx->write_ctx, ctx->read_ctx) != SQLITE_OK) return SQLITE_ERROR; + } else { + if(sqlcipher_cipher_ctx_key_derive(ctx, ctx->write_ctx) != SQLITE_OK) return SQLITE_ERROR; + } + } + + /* TODO: wipe and free passphrase after key derivation */ + if(ctx->read_ctx->store_pass != 1) { + sqlcipher_cipher_ctx_set_pass(ctx->read_ctx, NULL, 0); + sqlcipher_cipher_ctx_set_pass(ctx->write_ctx, NULL, 0); + } + + return SQLITE_OK; +} + +int sqlcipher_codec_key_copy(codec_ctx *ctx, int source) { + if(source == CIPHER_READ_CTX) { + return sqlcipher_cipher_ctx_copy(ctx->write_ctx, ctx->read_ctx); + } else { + return sqlcipher_cipher_ctx_copy(ctx->read_ctx, ctx->write_ctx); + } +} + +const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx) { + return ctx->read_ctx->provider->get_provider_name(ctx->read_ctx); +} + + +static int sqlcipher_check_connection(const char *filename, char *key, int key_sz, char *sql, int *user_version) { + int rc; + sqlite3 *db = NULL; + sqlite3_stmt *statement = NULL; + char *query_user_version = "PRAGMA user_version;"; + + rc = sqlite3_open(filename, &db); + if(rc != SQLITE_OK){ + goto cleanup; + } + rc = sqlite3_key(db, key, key_sz); + if(rc != SQLITE_OK){ + goto cleanup; + } + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if(rc != SQLITE_OK){ + goto cleanup; + } + rc = sqlite3_prepare(db, query_user_version, -1, &statement, NULL); + if(rc != SQLITE_OK){ + goto cleanup; + } + rc = sqlite3_step(statement); + if(rc == SQLITE_ROW){ + *user_version = sqlite3_column_int(statement, 0); + rc = SQLITE_OK; + } + +cleanup: + if(statement){ + sqlite3_finalize(statement); + } + if(db){ + sqlite3_close(db); + } + return rc; +} + +int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { + u32 meta; + int rc = 0; + int command_idx = 0; + int password_sz; + int saved_flags; + int saved_nChange; + int saved_nTotalChange; + void (*saved_xTrace)(void*,const char*); + Db *pDb = 0; + sqlite3 *db = ctx->pBt->db; + const char *db_filename = sqlite3_db_filename(db, "main"); + char *migrated_db_filename = sqlite3_mprintf("%s-migrated", db_filename); + char *pragma_hmac_off = "PRAGMA cipher_use_hmac = OFF;"; + char *pragma_4k_kdf_iter = "PRAGMA kdf_iter = 4000;"; + char *pragma_1x_and_4k; + char *set_user_version; + char *key; + int key_sz; + int user_version = 0; + int upgrade_1x_format = 0; + int upgrade_4k_format = 0; + static const unsigned char aCopy[] = { + BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ + BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ + BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ + BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ + }; + + + key_sz = ctx->read_ctx->pass_sz + 1; + key = sqlcipher_malloc(key_sz); + memset(key, 0, key_sz); + memcpy(key, ctx->read_ctx->pass, ctx->read_ctx->pass_sz); + + if(db_filename){ + const char* commands[5]; + char *attach_command = sqlite3_mprintf("ATTACH DATABASE '%s-migrated' as migrate KEY '%q';", + db_filename, key); + + int rc = sqlcipher_check_connection(db_filename, key, ctx->read_ctx->pass_sz, "", &user_version); + if(rc == SQLITE_OK){ + CODEC_TRACE(("No upgrade required - exiting\n")); + goto exit; + } + + // Version 2 - check for 4k with hmac format + rc = sqlcipher_check_connection(db_filename, key, ctx->read_ctx->pass_sz, pragma_4k_kdf_iter, &user_version); + if(rc == SQLITE_OK) { + CODEC_TRACE(("Version 2 format found\n")); + upgrade_4k_format = 1; + } + + // Version 1 - check both no hmac and 4k together + pragma_1x_and_4k = sqlite3_mprintf("%s%s", pragma_hmac_off, + pragma_4k_kdf_iter); + rc = sqlcipher_check_connection(db_filename, key, ctx->read_ctx->pass_sz, pragma_1x_and_4k, &user_version); + sqlite3_free(pragma_1x_and_4k); + if(rc == SQLITE_OK) { + CODEC_TRACE(("Version 1 format found\n")); + upgrade_1x_format = 1; + upgrade_4k_format = 1; + } + + if(upgrade_1x_format == 0 && upgrade_4k_format == 0) { + CODEC_TRACE(("Upgrade format not determined\n")); + goto handle_error; + } + + set_user_version = sqlite3_mprintf("PRAGMA migrate.user_version = %d;", user_version); + commands[0] = upgrade_4k_format == 1 ? pragma_4k_kdf_iter : ""; + commands[1] = upgrade_1x_format == 1 ? pragma_hmac_off : ""; + commands[2] = attach_command; + commands[3] = "SELECT sqlcipher_export('migrate');"; + commands[4] = set_user_version; + + for(command_idx = 0; command_idx < ArraySize(commands); command_idx++){ + const char *command = commands[command_idx]; + if(strcmp(command, "") == 0){ + continue; + } + rc = sqlite3_exec(db, command, NULL, NULL, NULL); + if(rc != SQLITE_OK){ + break; + } + } + sqlite3_free(attach_command); + sqlite3_free(set_user_version); + sqlcipher_free(key, key_sz); + + if(rc == SQLITE_OK){ + Btree *pDest; + Btree *pSrc; + int i = 0; + + if( !db->autoCommit ){ + CODEC_TRACE(("cannot migrate from within a transaction")); + goto handle_error; + } + if( db->nVdbeActive>1 ){ + CODEC_TRACE(("cannot migrate - SQL statements in progress")); + goto handle_error; + } + + /* Save the current value of the database flags so that it can be + ** restored before returning. Then set the writable-schema flag, and + ** disable CHECK and foreign key constraints. */ + saved_flags = db->flags; + saved_nChange = db->nChange; + saved_nTotalChange = db->nTotalChange; + saved_xTrace = db->xTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); + db->xTrace = 0; + + pDest = db->aDb[0].pBt; + pDb = &(db->aDb[db->nDb-1]); + pSrc = pDb->pBt; + + rc = sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); + rc = sqlite3BtreeBeginTrans(pSrc, 2); + rc = sqlite3BtreeBeginTrans(pDest, 2); + + assert( 1==sqlite3BtreeIsInTrans(pDest) ); + assert( 1==sqlite3BtreeIsInTrans(pSrc) ); + + sqlite3CodecGetKey(db, db->nDb - 1, (void**)&key, &password_sz); + sqlite3CodecAttach(db, 0, key, password_sz); + sqlite3pager_get_codec(pDest->pBt->pPager, (void**)&ctx); + + ctx->skip_read_hmac = 1; + for(i=0; iskip_read_hmac = 0; + if( rc!=SQLITE_OK ) goto handle_error; + rc = sqlite3BtreeCommit(pDest); + + db->flags = saved_flags; + db->nChange = saved_nChange; + db->nTotalChange = saved_nTotalChange; + db->xTrace = saved_xTrace; + db->autoCommit = 1; + if( pDb ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; + } + sqlite3ResetAllSchemasOfConnection(db); + remove(migrated_db_filename); + sqlite3_free(migrated_db_filename); + } else { + CODEC_TRACE(("*** migration failure** \n\n")); + } + + } + goto exit; + + handle_error: + CODEC_TRACE(("An error occurred attempting to migrate the database\n")); + rc = SQLITE_ERROR; + + exit: + return rc; +} + +int sqlcipher_codec_add_random(codec_ctx *ctx, const char *zRight, int random_sz){ + const char *suffix = &zRight[random_sz-1]; + int n = random_sz - 3; /* adjust for leading x' and tailing ' */ + if (n > 0 && + sqlite3StrNICmp((const char *)zRight ,"x'", 2) == 0 && + sqlite3StrNICmp(suffix, "'", 1) == 0 && + n % 2 == 0) { + int rc = 0; + int buffer_sz = n / 2; + unsigned char *random; + const unsigned char *z = (const unsigned char *)zRight + 2; /* adjust lead offset of x' */ + CODEC_TRACE(("sqlcipher_codec_add_random: using raw random blob from hex\n")); + random = sqlcipher_malloc(buffer_sz); + memset(random, 0, buffer_sz); + cipher_hex2bin(z, n, random); + rc = ctx->read_ctx->provider->add_random(ctx->read_ctx->provider_ctx, random, buffer_sz); + sqlcipher_free(random, buffer_sz); + return rc; + } + return SQLITE_ERROR; +} + +int sqlcipher_cipher_profile(sqlite3 *db, const char *destination){ + FILE *f; + if( strcmp(destination,"stdout")==0 ){ + f = stdout; + }else if( strcmp(destination, "stderr")==0 ){ + f = stderr; + }else if( strcmp(destination, "off")==0 ){ + f = 0; + }else{ + f = fopen(destination, "wb"); + if( f==0 ){ + return SQLITE_ERROR; + } + } + sqlite3_profile(db, sqlcipher_profile_callback, f); + return SQLITE_OK; +} + +static void sqlcipher_profile_callback(void *file, const char *sql, sqlite3_uint64 run_time){ + FILE *f = (FILE*)file; + double elapsed = run_time/1000000.0; + if( f ) fprintf(f, "Elapsed time:%.3f ms - %s\n", elapsed, sql); +} + + +#endif +/* END SQLCIPHER */ + +/************** End of crypto_impl.c *****************************************/ +/************** Begin file crypto_libtomcrypt.c ******************************/ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_LIBTOMCRYPT +#include + +#define FORTUNA_MAX_SZ 32 +static prng_state prng; +static unsigned int ltc_init = 0; +static unsigned int ltc_ref_count = 0; +static sqlite3_mutex* ltc_rand_mutex = NULL; + +static int sqlcipher_ltc_add_random(void *ctx, void *buffer, int length) { + int rc = 0; + int data_to_read = length; + int block_sz = data_to_read < FORTUNA_MAX_SZ ? data_to_read : FORTUNA_MAX_SZ; + const unsigned char * data = (const unsigned char *)buffer; +#ifndef SQLCIPHER_LTC_NO_MUTEX_RAND + sqlite3_mutex_enter(ltc_rand_mutex); +#endif + while(data_to_read > 0){ + rc = fortuna_add_entropy(data, block_sz, &prng); + rc = rc != CRYPT_OK ? SQLITE_ERROR : SQLITE_OK; + if(rc != SQLITE_OK){ + break; + } + data_to_read -= block_sz; + data += block_sz; + block_sz = data_to_read < FORTUNA_MAX_SZ ? data_to_read : FORTUNA_MAX_SZ; + } + fortuna_ready(&prng); +#ifndef SQLCIPHER_LTC_NO_MUTEX_RAND + sqlite3_mutex_leave(ltc_rand_mutex); +#endif + return rc; +} + +static int sqlcipher_ltc_activate(void *ctx) { + unsigned char random_buffer[FORTUNA_MAX_SZ]; +#ifndef SQLCIPHER_LTC_NO_MUTEX_RAND + if(ltc_rand_mutex == NULL){ + ltc_rand_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + } + sqlite3_mutex_enter(ltc_rand_mutex); +#endif + sqlcipher_memset(random_buffer, 0, FORTUNA_MAX_SZ); + if(ltc_init == 0) { + if(register_prng(&fortuna_desc) != CRYPT_OK) return SQLITE_ERROR; + if(register_cipher(&rijndael_desc) != CRYPT_OK) return SQLITE_ERROR; + if(register_hash(&sha1_desc) != CRYPT_OK) return SQLITE_ERROR; + if(fortuna_start(&prng) != CRYPT_OK) { + return SQLITE_ERROR; + } + ltc_init = 1; + } + ltc_ref_count++; +#ifndef SQLCIPHER_TEST + sqlite3_randomness(FORTUNA_MAX_SZ, random_buffer); +#endif +#ifndef SQLCIPHER_LTC_NO_MUTEX_RAND + sqlite3_mutex_leave(ltc_rand_mutex); +#endif + if(sqlcipher_ltc_add_random(ctx, random_buffer, FORTUNA_MAX_SZ) != SQLITE_OK) { + return SQLITE_ERROR; + } + sqlcipher_memset(random_buffer, 0, FORTUNA_MAX_SZ); + return SQLITE_OK; +} + +static int sqlcipher_ltc_deactivate(void *ctx) { +#ifndef SQLCIPHER_LTC_NO_MUTEX_RAND + sqlite3_mutex_enter(ltc_rand_mutex); +#endif + ltc_ref_count--; + if(ltc_ref_count == 0){ + fortuna_done(&prng); + sqlcipher_memset((void *)&prng, 0, sizeof(prng)); +#ifndef SQLCIPHER_LTC_NO_MUTEX_RAND + sqlite3_mutex_leave(ltc_rand_mutex); + sqlite3_mutex_free(ltc_rand_mutex); + ltc_rand_mutex = NULL; +#endif + } +#ifndef SQLCIPHER_LTC_NO_MUTEX_RAND + else { + sqlite3_mutex_leave(ltc_rand_mutex); + } +#endif + return SQLITE_OK; +} + +static const char* sqlcipher_ltc_get_provider_name(void *ctx) { + return "libtomcrypt"; +} + +static int sqlcipher_ltc_random(void *ctx, void *buffer, int length) { +#ifndef SQLCIPHER_LTC_NO_MUTEX_RAND + sqlite3_mutex_enter(ltc_rand_mutex); +#endif + fortuna_read(buffer, length, &prng); +#ifndef SQLCIPHER_LTC_NO_MUTEX_RAND + sqlite3_mutex_leave(ltc_rand_mutex); +#endif + return SQLITE_OK; +} + +static int sqlcipher_ltc_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + int rc, hash_idx; + hmac_state hmac; + unsigned long outlen = key_sz; + + hash_idx = find_hash("sha1"); + if((rc = hmac_init(&hmac, hash_idx, hmac_key, key_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_process(&hmac, in, in_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_process(&hmac, in2, in2_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_done(&hmac, out, &outlen)) != CRYPT_OK) return SQLITE_ERROR; + return SQLITE_OK; +} + +static int sqlcipher_ltc_kdf(void *ctx, const unsigned char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { + int rc, hash_idx; + unsigned long outlen = key_sz; + unsigned long random_buffer_sz = sizeof(char) * 256; + unsigned char *random_buffer = sqlcipher_malloc(random_buffer_sz); + sqlcipher_memset(random_buffer, 0, random_buffer_sz); + + hash_idx = find_hash("sha1"); + if((rc = pkcs_5_alg2(pass, pass_sz, salt, salt_sz, + workfactor, hash_idx, key, &outlen)) != CRYPT_OK) { + return SQLITE_ERROR; + } + if((rc = pkcs_5_alg2(key, key_sz, salt, salt_sz, + 1, hash_idx, random_buffer, &random_buffer_sz)) != CRYPT_OK) { + return SQLITE_ERROR; + } + sqlcipher_ltc_add_random(ctx, random_buffer, random_buffer_sz); + sqlcipher_free(random_buffer, random_buffer_sz); + return SQLITE_OK; +} + +static const char* sqlcipher_ltc_get_cipher(void *ctx) { + return "rijndael"; +} + +static int sqlcipher_ltc_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + int rc, cipher_idx; + symmetric_CBC cbc; + + if((cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx))) == -1) return SQLITE_ERROR; + if((rc = cbc_start(cipher_idx, iv, key, key_sz, 0, &cbc)) != CRYPT_OK) return SQLITE_ERROR; + rc = mode == 1 ? cbc_encrypt(in, out, in_sz, &cbc) : cbc_decrypt(in, out, in_sz, &cbc); + if(rc != CRYPT_OK) return SQLITE_ERROR; + cbc_done(&cbc); + return SQLITE_OK; +} + +static int sqlcipher_ltc_set_cipher(void *ctx, const char *cipher_name) { + return SQLITE_OK; +} + +static int sqlcipher_ltc_get_key_sz(void *ctx) { + int cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx)); + return cipher_descriptor[cipher_idx].max_key_length; +} + +static int sqlcipher_ltc_get_iv_sz(void *ctx) { + int cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx)); + return cipher_descriptor[cipher_idx].block_length; +} + +static int sqlcipher_ltc_get_block_sz(void *ctx) { + int cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx)); + return cipher_descriptor[cipher_idx].block_length; +} + +static int sqlcipher_ltc_get_hmac_sz(void *ctx) { + int hash_idx = find_hash("sha1"); + return hash_descriptor[hash_idx].hashsize; +} + +static int sqlcipher_ltc_ctx_copy(void *target_ctx, void *source_ctx) { + return SQLITE_OK; +} + +static int sqlcipher_ltc_ctx_cmp(void *c1, void *c2) { + return 1; +} + +static int sqlcipher_ltc_ctx_init(void **ctx) { + sqlcipher_ltc_activate(NULL); + return SQLITE_OK; +} + +static int sqlcipher_ltc_ctx_free(void **ctx) { + sqlcipher_ltc_deactivate(&ctx); + return SQLITE_OK; +} + +int sqlcipher_ltc_setup(sqlcipher_provider *p) { + p->activate = sqlcipher_ltc_activate; + p->deactivate = sqlcipher_ltc_deactivate; + p->get_provider_name = sqlcipher_ltc_get_provider_name; + p->random = sqlcipher_ltc_random; + p->hmac = sqlcipher_ltc_hmac; + p->kdf = sqlcipher_ltc_kdf; + p->cipher = sqlcipher_ltc_cipher; + p->set_cipher = sqlcipher_ltc_set_cipher; + p->get_cipher = sqlcipher_ltc_get_cipher; + p->get_key_sz = sqlcipher_ltc_get_key_sz; + p->get_iv_sz = sqlcipher_ltc_get_iv_sz; + p->get_block_sz = sqlcipher_ltc_get_block_sz; + p->get_hmac_sz = sqlcipher_ltc_get_hmac_sz; + p->ctx_copy = sqlcipher_ltc_ctx_copy; + p->ctx_cmp = sqlcipher_ltc_ctx_cmp; + p->ctx_init = sqlcipher_ltc_ctx_init; + p->ctx_free = sqlcipher_ltc_ctx_free; + p->add_random = sqlcipher_ltc_add_random; + return SQLITE_OK; +} + +#endif +#endif +/* END SQLCIPHER */ + +/************** End of crypto_libtomcrypt.c **********************************/ +/************** Begin file crypto_openssl.c **********************************/ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_OPENSSL +#include +#include +#include + +typedef struct { + EVP_CIPHER *evp_cipher; +} openssl_ctx; + + +static unsigned int openssl_external_init = 0; +static unsigned int openssl_init_count = 0; +static sqlite3_mutex* openssl_rand_mutex = NULL; + +static int sqlcipher_openssl_add_random(void *ctx, void *buffer, int length) { +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_enter(openssl_rand_mutex); +#endif + RAND_add(buffer, length, 0); +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_leave(openssl_rand_mutex); +#endif + return SQLITE_OK; +} + +/* activate and initialize sqlcipher. Most importantly, this will automatically + intialize OpenSSL's EVP system if it hasn't already be externally. Note that + this function may be called multiple times as new codecs are intiialized. + Thus it performs some basic counting to ensure that only the last and final + sqlcipher_openssl_deactivate() will free the EVP structures. +*/ +static int sqlcipher_openssl_activate(void *ctx) { + /* initialize openssl and increment the internal init counter + but only if it hasn't been initalized outside of SQLCipher by this program + e.g. on startup */ + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + + if(openssl_init_count == 0 && EVP_get_cipherbyname(CIPHER) != NULL) { + /* if openssl has not yet been initialized by this library, but + a call to get_cipherbyname works, then the openssl library + has been initialized externally already. */ + openssl_external_init = 1; + } + + if(openssl_init_count == 0 && openssl_external_init == 0) { + /* if the library was not externally initialized, then should be now */ + OpenSSL_add_all_algorithms(); + } + +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + if(openssl_rand_mutex == NULL) { + /* allocate a mutex to guard against concurrent calls to RAND_bytes() */ + openssl_rand_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + } +#endif + + openssl_init_count++; + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + return SQLITE_OK; +} + +/* deactivate SQLCipher, most imporantly decremeting the activation count and + freeing the EVP structures on the final deactivation to ensure that + OpenSSL memory is cleaned up */ +static int sqlcipher_openssl_deactivate(void *ctx) { + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + openssl_init_count--; + + if(openssl_init_count == 0) { + if(openssl_external_init == 0) { + /* if OpenSSL hasn't be initialized externally, and the counter reaches zero + after it's decremented, release EVP memory + Note: this code will only be reached if OpensSSL_add_all_algorithms() + is called by SQLCipher internally. This should prevent SQLCipher from + "cleaning up" openssl when it was initialized externally by the program */ + EVP_cleanup(); + } +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_free(openssl_rand_mutex); + openssl_rand_mutex = NULL; +#endif + } + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + return SQLITE_OK; +} + +static const char* sqlcipher_openssl_get_provider_name(void *ctx) { + return "openssl"; +} + +/* generate a defined number of random bytes */ +static int sqlcipher_openssl_random (void *ctx, void *buffer, int length) { + int rc = 0; + /* concurrent calls to RAND_bytes can cause a crash under some openssl versions when a + naive application doesn't use CRYPTO_set_locking_callback and + CRYPTO_THREADID_set_callback to ensure openssl thread safety. + This is simple workaround to prevent this common crash + but a more proper solution is that applications setup platform-appropriate + thread saftey in openssl externally */ +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_enter(openssl_rand_mutex); +#endif + rc = RAND_bytes((unsigned char *)buffer, length); +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_leave(openssl_rand_mutex); +#endif + return (rc == 1) ? SQLITE_OK : SQLITE_ERROR; +} + +static int sqlcipher_openssl_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + HMAC_CTX hctx; + unsigned int outlen; + HMAC_CTX_init(&hctx); + HMAC_Init_ex(&hctx, hmac_key, key_sz, EVP_sha1(), NULL); + HMAC_Update(&hctx, in, in_sz); + HMAC_Update(&hctx, in2, in2_sz); + HMAC_Final(&hctx, out, &outlen); + HMAC_CTX_cleanup(&hctx); + return SQLITE_OK; +} + +static int sqlcipher_openssl_kdf(void *ctx, const unsigned char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { + PKCS5_PBKDF2_HMAC_SHA1((const char *)pass, pass_sz, salt, salt_sz, workfactor, key_sz, key); + return SQLITE_OK; +} + +static int sqlcipher_openssl_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + EVP_CIPHER_CTX ectx; + int tmp_csz, csz; + + EVP_CipherInit(&ectx, ((openssl_ctx *)ctx)->evp_cipher, NULL, NULL, mode); + EVP_CIPHER_CTX_set_padding(&ectx, 0); // no padding + EVP_CipherInit(&ectx, NULL, key, iv, mode); + EVP_CipherUpdate(&ectx, out, &tmp_csz, in, in_sz); + csz = tmp_csz; + out += tmp_csz; + EVP_CipherFinal(&ectx, out, &tmp_csz); + csz += tmp_csz; + EVP_CIPHER_CTX_cleanup(&ectx); + assert(in_sz == csz); + return SQLITE_OK; +} + +static int sqlcipher_openssl_set_cipher(void *ctx, const char *cipher_name) { + openssl_ctx *o_ctx = (openssl_ctx *)ctx; + o_ctx->evp_cipher = (EVP_CIPHER *) EVP_get_cipherbyname(cipher_name); + return SQLITE_OK; +} + +static const char* sqlcipher_openssl_get_cipher(void *ctx) { + return EVP_CIPHER_name(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_key_sz(void *ctx) { + return EVP_CIPHER_key_length(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_iv_sz(void *ctx) { + return EVP_CIPHER_iv_length(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_block_sz(void *ctx) { + return EVP_CIPHER_block_size(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_hmac_sz(void *ctx) { + return EVP_MD_size(EVP_sha1()); +} + +static int sqlcipher_openssl_ctx_copy(void *target_ctx, void *source_ctx) { + memcpy(target_ctx, source_ctx, sizeof(openssl_ctx)); + return SQLITE_OK; +} + +static int sqlcipher_openssl_ctx_cmp(void *c1, void *c2) { + return ((openssl_ctx *)c1)->evp_cipher == ((openssl_ctx *)c2)->evp_cipher; +} + +static int sqlcipher_openssl_ctx_init(void **ctx) { + *ctx = sqlcipher_malloc(sizeof(openssl_ctx)); + if(*ctx == NULL) return SQLITE_NOMEM; + sqlcipher_openssl_activate(*ctx); + return SQLITE_OK; +} + +static int sqlcipher_openssl_ctx_free(void **ctx) { + sqlcipher_openssl_deactivate(*ctx); + sqlcipher_free(*ctx, sizeof(openssl_ctx)); + return SQLITE_OK; +} + +int sqlcipher_openssl_setup(sqlcipher_provider *p) { + p->activate = sqlcipher_openssl_activate; + p->deactivate = sqlcipher_openssl_deactivate; + p->get_provider_name = sqlcipher_openssl_get_provider_name; + p->random = sqlcipher_openssl_random; + p->hmac = sqlcipher_openssl_hmac; + p->kdf = sqlcipher_openssl_kdf; + p->cipher = sqlcipher_openssl_cipher; + p->set_cipher = sqlcipher_openssl_set_cipher; + p->get_cipher = sqlcipher_openssl_get_cipher; + p->get_key_sz = sqlcipher_openssl_get_key_sz; + p->get_iv_sz = sqlcipher_openssl_get_iv_sz; + p->get_block_sz = sqlcipher_openssl_get_block_sz; + p->get_hmac_sz = sqlcipher_openssl_get_hmac_sz; + p->ctx_copy = sqlcipher_openssl_ctx_copy; + p->ctx_cmp = sqlcipher_openssl_ctx_cmp; + p->ctx_init = sqlcipher_openssl_ctx_init; + p->ctx_free = sqlcipher_openssl_ctx_free; + p->add_random = sqlcipher_openssl_add_random; + return SQLITE_OK; +} + +#endif +#endif +/* END SQLCIPHER */ + +/************** End of crypto_openssl.c **************************************/ +/************** Begin file crypto_cc.c ***************************************/ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_CC +#include +#include + +static int sqlcipher_cc_add_random(void *ctx, void *buffer, int length) { + return SQLITE_OK; +} + +/* generate a defined number of random bytes */ +static int sqlcipher_cc_random (void *ctx, void *buffer, int length) { + return (SecRandomCopyBytes(kSecRandomDefault, length, (uint8_t *)buffer) == 0) ? SQLITE_OK : SQLITE_ERROR; +} + +static const char* sqlcipher_cc_get_provider_name(void *ctx) { + return "commoncrypto"; +} + +static int sqlcipher_cc_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + CCHmacContext hmac_context; + CCHmacInit(&hmac_context, kCCHmacAlgSHA1, hmac_key, key_sz); + CCHmacUpdate(&hmac_context, in, in_sz); + CCHmacUpdate(&hmac_context, in2, in2_sz); + CCHmacFinal(&hmac_context, out); + return SQLITE_OK; +} + +static int sqlcipher_cc_kdf(void *ctx, const unsigned char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { + CCKeyDerivationPBKDF(kCCPBKDF2, (const char *)pass, pass_sz, salt, salt_sz, kCCPRFHmacAlgSHA1, workfactor, key, key_sz); + return SQLITE_OK; +} + +static int sqlcipher_cc_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + CCCryptorRef cryptor; + size_t tmp_csz, csz; + CCOperation op = mode == CIPHER_ENCRYPT ? kCCEncrypt : kCCDecrypt; + + CCCryptorCreate(op, kCCAlgorithmAES128, 0, key, kCCKeySizeAES256, iv, &cryptor); + CCCryptorUpdate(cryptor, in, in_sz, out, in_sz, &tmp_csz); + csz = tmp_csz; + out += tmp_csz; + CCCryptorFinal(cryptor, out, in_sz - csz, &tmp_csz); + csz += tmp_csz; + CCCryptorRelease(cryptor); + assert(size == csz); + + return SQLITE_OK; +} + +static int sqlcipher_cc_set_cipher(void *ctx, const char *cipher_name) { + return SQLITE_OK; +} + +static const char* sqlcipher_cc_get_cipher(void *ctx) { + return "aes-256-cbc"; +} + +static int sqlcipher_cc_get_key_sz(void *ctx) { + return kCCKeySizeAES256; +} + +static int sqlcipher_cc_get_iv_sz(void *ctx) { + return kCCBlockSizeAES128; +} + +static int sqlcipher_cc_get_block_sz(void *ctx) { + return kCCBlockSizeAES128; +} + +static int sqlcipher_cc_get_hmac_sz(void *ctx) { + return CC_SHA1_DIGEST_LENGTH; +} + +static int sqlcipher_cc_ctx_copy(void *target_ctx, void *source_ctx) { + return SQLITE_OK; +} + +static int sqlcipher_cc_ctx_cmp(void *c1, void *c2) { + return 1; /* always indicate contexts are the same */ +} + +static int sqlcipher_cc_ctx_init(void **ctx) { + return SQLITE_OK; +} + +static int sqlcipher_cc_ctx_free(void **ctx) { + return SQLITE_OK; +} + +int sqlcipher_cc_setup(sqlcipher_provider *p) { + p->random = sqlcipher_cc_random; + p->get_provider_name = sqlcipher_cc_get_provider_name; + p->hmac = sqlcipher_cc_hmac; + p->kdf = sqlcipher_cc_kdf; + p->cipher = sqlcipher_cc_cipher; + p->set_cipher = sqlcipher_cc_set_cipher; + p->get_cipher = sqlcipher_cc_get_cipher; + p->get_key_sz = sqlcipher_cc_get_key_sz; + p->get_iv_sz = sqlcipher_cc_get_iv_sz; + p->get_block_sz = sqlcipher_cc_get_block_sz; + p->get_hmac_sz = sqlcipher_cc_get_hmac_sz; + p->ctx_copy = sqlcipher_cc_ctx_copy; + p->ctx_cmp = sqlcipher_cc_ctx_cmp; + p->ctx_init = sqlcipher_cc_ctx_init; + p->ctx_free = sqlcipher_cc_ctx_free; + p->add_random = sqlcipher_cc_add_random; + return SQLITE_OK; +} + +#endif +#endif +/* END SQLCIPHER */ + +/************** End of crypto_cc.c *******************************************/ +/************** Begin file global.c ******************************************/ +/* +** 2008 June 13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains definitions of global variables and contants. +*/ + +/* An array to map all upper-case characters into their corresponding +** lower-case character. +** +** SQLite only considers US-ASCII (or EBCDIC) characters. We do not +** handle case conversions for the UTF character set since the tables +** involved are nearly as big or bigger than SQLite itself. +*/ +SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { +#ifdef SQLITE_ASCII + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121, + 122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107, + 108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125, + 126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161, + 162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179, + 180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197, + 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233, + 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251, + 252,253,254,255 +#endif +#ifdef SQLITE_EBCDIC + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, /* 0x */ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, /* 1x */ + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, /* 2x */ + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, /* 3x */ + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, /* 4x */ + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, /* 5x */ + 96, 97, 66, 67, 68, 69, 70, 71, 72, 73,106,107,108,109,110,111, /* 6x */ + 112, 81, 82, 83, 84, 85, 86, 87, 88, 89,122,123,124,125,126,127, /* 7x */ + 128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, /* 8x */ + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,156,159, /* 9x */ + 160,161,162,163,164,165,166,167,168,169,170,171,140,141,142,175, /* Ax */ + 176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191, /* Bx */ + 192,129,130,131,132,133,134,135,136,137,202,203,204,205,206,207, /* Cx */ + 208,145,146,147,148,149,150,151,152,153,218,219,220,221,222,223, /* Dx */ + 224,225,162,163,164,165,166,167,168,169,232,203,204,205,206,207, /* Ex */ + 239,240,241,242,243,244,245,246,247,248,249,219,220,221,222,255, /* Fx */ +#endif +}; + +/* +** The following 256 byte lookup table is used to support SQLites built-in +** equivalents to the following standard library functions: +** +** isspace() 0x01 +** isalpha() 0x02 +** isdigit() 0x04 +** isalnum() 0x06 +** isxdigit() 0x08 +** toupper() 0x20 +** SQLite identifier character 0x40 +** +** Bit 0x20 is set if the mapped character requires translation to upper +** case. i.e. if the character is a lower-case ASCII character. +** If x is a lower-case ASCII character, then its upper-case equivalent +** is (x - 0x20). Therefore toupper() can be implemented as: +** +** (x & ~(map[x]&0x20)) +** +** Standard function tolower() is implemented using the sqlite3UpperToLower[] +** array. tolower() is used more often than toupper() by SQLite. +** +** Bit 0x40 is set if the character non-alphanumeric and can be used in an +** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any +** non-ASCII UTF character. Hence the test for whether or not a character is +** part of an identifier is 0x46. +** +** SQLite's versions are identical to the standard versions assuming a +** locale of "C". They are implemented as macros in sqliteInt.h. +*/ +#ifdef SQLITE_ASCII +SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ + 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ + 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ + 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ + + 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ + 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ + 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ +}; +#endif + +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 0 +#endif + +#ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN +# define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 +#endif + +/* +** The following singleton contains the global configuration for +** the SQLite library. +*/ +SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { + SQLITE_DEFAULT_MEMSTATUS, /* bMemstat */ + 1, /* bCoreMutex */ + SQLITE_THREADSAFE==1, /* bFullMutex */ + SQLITE_USE_URI, /* bOpenUri */ + SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ + 0x7ffffffe, /* mxStrlen */ + 0, /* neverCorrupt */ + 128, /* szLookaside */ + 500, /* nLookaside */ + {0,0,0,0,0,0,0,0}, /* m */ + {0,0,0,0,0,0,0,0,0}, /* mutex */ + {0,0,0,0,0,0,0,0,0,0,0,0,0},/* pcache2 */ + (void*)0, /* pHeap */ + 0, /* nHeap */ + 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ + SQLITE_MAX_MMAP_SIZE, /* mxMmap */ + (void*)0, /* pScratch */ + 0, /* szScratch */ + 0, /* nScratch */ + (void*)0, /* pPage */ + 0, /* szPage */ + 0, /* nPage */ + 0, /* mxParserStack */ + 0, /* sharedCacheEnabled */ + /* All the rest should always be initialized to zero */ + 0, /* isInit */ + 0, /* inProgress */ + 0, /* isMutexInit */ + 0, /* isMallocInit */ + 0, /* isPCacheInit */ + 0, /* nRefInitMutex */ + 0, /* pInitMutex */ + 0, /* xLog */ + 0, /* pLogArg */ +#ifdef SQLITE_ENABLE_SQLLOG + 0, /* xSqllog */ + 0, /* pSqllogArg */ +#endif +#ifdef SQLITE_VDBE_COVERAGE + 0, /* xVdbeBranch */ + 0, /* pVbeBranchArg */ +#endif +#ifndef SQLITE_OMIT_BUILTIN_TEST + 0, /* xTestCallback */ +#endif + 0 /* bLocaltimeFault */ +}; + +/* +** Hash table for global functions - functions common to all +** database connections. After initialization, this table is +** read-only. +*/ +SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; + +/* +** Constant tokens for values 0 and 1. +*/ +SQLITE_PRIVATE const Token sqlite3IntTokens[] = { + { "0", 1 }, + { "1", 1 } +}; + + +/* +** The value of the "pending" byte must be 0x40000000 (1 byte past the +** 1-gibabyte boundary) in a compatible database. SQLite never uses +** the database page that contains the pending byte. It never attempts +** to read or write that page. The pending byte page is set assign +** for use by the VFS layers as space for managing file locks. +** +** During testing, it is often desirable to move the pending byte to +** a different position in the file. This allows code that has to +** deal with the pending byte to run on files that are much smaller +** than 1 GiB. The sqlite3_test_control() interface can be used to +** move the pending byte. +** +** IMPORTANT: Changing the pending byte to any value other than +** 0x40000000 results in an incompatible database file format! +** Changing the pending byte during operating results in undefined +** and dileterious behavior. +*/ +#ifndef SQLITE_OMIT_WSD +SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; +#endif + +/* +** Properties of opcodes. The OPFLG_INITIALIZER macro is +** created by mkopcodeh.awk during compilation. Data is obtained +** from the comments following the "case OP_xxxx:" statements in +** the vdbe.c file. +*/ +SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; + +/************** End of global.c **********************************************/ +/************** Begin file ctime.c *******************************************/ +/* +** 2010 February 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements routines used to report what compile-time options +** SQLite was built with. +*/ + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + + +/* +** An array of names of all compile-time options. This array should +** be sorted A-Z. +** +** This array looks large, but in a typical installation actually uses +** only a handful of compile-time options, so most times this array is usually +** rather short and uses little memory space. +*/ +static const char * const azCompileOpt[] = { + +/* These macros are provided to "stringify" the value of the define +** for those options in which the value is meaningful. */ +#define CTIMEOPT_VAL_(opt) #opt +#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) + +#ifdef SQLITE_32BIT_ROWID + "32BIT_ROWID", +#endif +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC + "4_BYTE_ALIGNED_MALLOC", +#endif +#ifdef SQLITE_CASE_SENSITIVE_LIKE + "CASE_SENSITIVE_LIKE", +#endif +#ifdef SQLITE_CHECK_PAGES + "CHECK_PAGES", +#endif +#ifdef SQLITE_COVERAGE_TEST + "COVERAGE_TEST", +#endif +#ifdef SQLITE_DEBUG + "DEBUG", +#endif +#ifdef SQLITE_DEFAULT_LOCKING_MODE + "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), +#endif +#if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), +#endif +#ifdef SQLITE_DISABLE_DIRSYNC + "DISABLE_DIRSYNC", +#endif +#ifdef SQLITE_DISABLE_LFS + "DISABLE_LFS", +#endif +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + "ENABLE_ATOMIC_WRITE", +#endif +#ifdef SQLITE_ENABLE_CEROD + "ENABLE_CEROD", +#endif +#ifdef SQLITE_ENABLE_COLUMN_METADATA + "ENABLE_COLUMN_METADATA", +#endif +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + "ENABLE_EXPENSIVE_ASSERT", +#endif +#ifdef SQLITE_ENABLE_FTS1 + "ENABLE_FTS1", +#endif +#ifdef SQLITE_ENABLE_FTS2 + "ENABLE_FTS2", +#endif +#ifdef SQLITE_ENABLE_FTS3 + "ENABLE_FTS3", +#endif +#ifdef SQLITE_ENABLE_FTS3_PARENTHESIS + "ENABLE_FTS3_PARENTHESIS", +#endif +#ifdef SQLITE_ENABLE_FTS4 + "ENABLE_FTS4", +#endif +#ifdef SQLITE_ENABLE_ICU + "ENABLE_ICU", +#endif +#ifdef SQLITE_ENABLE_IOTRACE + "ENABLE_IOTRACE", +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + "ENABLE_LOAD_EXTENSION", +#endif +#ifdef SQLITE_ENABLE_LOCKING_STYLE + "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), +#endif +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + "ENABLE_MEMORY_MANAGEMENT", +#endif +#ifdef SQLITE_ENABLE_MEMSYS3 + "ENABLE_MEMSYS3", +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + "ENABLE_MEMSYS5", +#endif +#ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK + "ENABLE_OVERSIZE_CELL_CHECK", +#endif +#ifdef SQLITE_ENABLE_RTREE + "ENABLE_RTREE", +#endif +#if defined(SQLITE_ENABLE_STAT4) + "ENABLE_STAT4", +#elif defined(SQLITE_ENABLE_STAT3) + "ENABLE_STAT3", +#endif +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + "ENABLE_UNLOCK_NOTIFY", +#endif +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + "ENABLE_UPDATE_DELETE_LIMIT", +#endif +#ifdef SQLITE_HAS_CODEC + "HAS_CODEC", +#endif +#ifdef SQLITE_HAVE_ISNAN + "HAVE_ISNAN", +#endif +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + "HOMEGROWN_RECURSIVE_MUTEX", +#endif +#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + "IGNORE_AFP_LOCK_ERRORS", +#endif +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + "IGNORE_FLOCK_LOCK_ERRORS", +#endif +#ifdef SQLITE_INT64_TYPE + "INT64_TYPE", +#endif +#ifdef SQLITE_LOCK_TRACE + "LOCK_TRACE", +#endif +#if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif +#ifdef SQLITE_MAX_SCHEMA_RETRY + "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), +#endif +#ifdef SQLITE_MEMDEBUG + "MEMDEBUG", +#endif +#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT + "MIXED_ENDIAN_64BIT_FLOAT", +#endif +#ifdef SQLITE_NO_SYNC + "NO_SYNC", +#endif +#ifdef SQLITE_OMIT_ALTERTABLE + "OMIT_ALTERTABLE", +#endif +#ifdef SQLITE_OMIT_ANALYZE + "OMIT_ANALYZE", +#endif +#ifdef SQLITE_OMIT_ATTACH + "OMIT_ATTACH", +#endif +#ifdef SQLITE_OMIT_AUTHORIZATION + "OMIT_AUTHORIZATION", +#endif +#ifdef SQLITE_OMIT_AUTOINCREMENT + "OMIT_AUTOINCREMENT", +#endif +#ifdef SQLITE_OMIT_AUTOINIT + "OMIT_AUTOINIT", +#endif +#ifdef SQLITE_OMIT_AUTOMATIC_INDEX + "OMIT_AUTOMATIC_INDEX", +#endif +#ifdef SQLITE_OMIT_AUTORESET + "OMIT_AUTORESET", +#endif +#ifdef SQLITE_OMIT_AUTOVACUUM + "OMIT_AUTOVACUUM", +#endif +#ifdef SQLITE_OMIT_BETWEEN_OPTIMIZATION + "OMIT_BETWEEN_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_BLOB_LITERAL + "OMIT_BLOB_LITERAL", +#endif +#ifdef SQLITE_OMIT_BTREECOUNT + "OMIT_BTREECOUNT", +#endif +#ifdef SQLITE_OMIT_BUILTIN_TEST + "OMIT_BUILTIN_TEST", +#endif +#ifdef SQLITE_OMIT_CAST + "OMIT_CAST", +#endif +#ifdef SQLITE_OMIT_CHECK + "OMIT_CHECK", +#endif +#ifdef SQLITE_OMIT_COMPLETE + "OMIT_COMPLETE", +#endif +#ifdef SQLITE_OMIT_COMPOUND_SELECT + "OMIT_COMPOUND_SELECT", +#endif +#ifdef SQLITE_OMIT_CTE + "OMIT_CTE", +#endif +#ifdef SQLITE_OMIT_DATETIME_FUNCS + "OMIT_DATETIME_FUNCS", +#endif +#ifdef SQLITE_OMIT_DECLTYPE + "OMIT_DECLTYPE", +#endif +#ifdef SQLITE_OMIT_DEPRECATED + "OMIT_DEPRECATED", +#endif +#ifdef SQLITE_OMIT_DISKIO + "OMIT_DISKIO", +#endif +#ifdef SQLITE_OMIT_EXPLAIN + "OMIT_EXPLAIN", +#endif +#ifdef SQLITE_OMIT_FLAG_PRAGMAS + "OMIT_FLAG_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_FLOATING_POINT + "OMIT_FLOATING_POINT", +#endif +#ifdef SQLITE_OMIT_FOREIGN_KEY + "OMIT_FOREIGN_KEY", +#endif +#ifdef SQLITE_OMIT_GET_TABLE + "OMIT_GET_TABLE", +#endif +#ifdef SQLITE_OMIT_INCRBLOB + "OMIT_INCRBLOB", +#endif +#ifdef SQLITE_OMIT_INTEGRITY_CHECK + "OMIT_INTEGRITY_CHECK", +#endif +#ifdef SQLITE_OMIT_LIKE_OPTIMIZATION + "OMIT_LIKE_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + "OMIT_LOAD_EXTENSION", +#endif +#ifdef SQLITE_OMIT_LOCALTIME + "OMIT_LOCALTIME", +#endif +#ifdef SQLITE_OMIT_LOOKASIDE + "OMIT_LOOKASIDE", +#endif +#ifdef SQLITE_OMIT_MEMORYDB + "OMIT_MEMORYDB", +#endif +#ifdef SQLITE_OMIT_OR_OPTIMIZATION + "OMIT_OR_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_PAGER_PRAGMAS + "OMIT_PAGER_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_PRAGMA + "OMIT_PRAGMA", +#endif +#ifdef SQLITE_OMIT_PROGRESS_CALLBACK + "OMIT_PROGRESS_CALLBACK", +#endif +#ifdef SQLITE_OMIT_QUICKBALANCE + "OMIT_QUICKBALANCE", +#endif +#ifdef SQLITE_OMIT_REINDEX + "OMIT_REINDEX", +#endif +#ifdef SQLITE_OMIT_SCHEMA_PRAGMAS + "OMIT_SCHEMA_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + "OMIT_SCHEMA_VERSION_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_SHARED_CACHE + "OMIT_SHARED_CACHE", +#endif +#ifdef SQLITE_OMIT_SUBQUERY + "OMIT_SUBQUERY", +#endif +#ifdef SQLITE_OMIT_TCL_VARIABLE + "OMIT_TCL_VARIABLE", +#endif +#ifdef SQLITE_OMIT_TEMPDB + "OMIT_TEMPDB", +#endif +#ifdef SQLITE_OMIT_TRACE + "OMIT_TRACE", +#endif +#ifdef SQLITE_OMIT_TRIGGER + "OMIT_TRIGGER", +#endif +#ifdef SQLITE_OMIT_TRUNCATE_OPTIMIZATION + "OMIT_TRUNCATE_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_UTF16 + "OMIT_UTF16", +#endif +#ifdef SQLITE_OMIT_VACUUM + "OMIT_VACUUM", +#endif +#ifdef SQLITE_OMIT_VIEW + "OMIT_VIEW", +#endif +#ifdef SQLITE_OMIT_VIRTUALTABLE + "OMIT_VIRTUALTABLE", +#endif +#ifdef SQLITE_OMIT_WAL + "OMIT_WAL", +#endif +#ifdef SQLITE_OMIT_WSD + "OMIT_WSD", +#endif +#ifdef SQLITE_OMIT_XFER_OPT + "OMIT_XFER_OPT", +#endif +#ifdef SQLITE_PERFORMANCE_TRACE + "PERFORMANCE_TRACE", +#endif +#ifdef SQLITE_PROXY_DEBUG + "PROXY_DEBUG", +#endif +#ifdef SQLITE_RTREE_INT_ONLY + "RTREE_INT_ONLY", +#endif +#ifdef SQLITE_SECURE_DELETE + "SECURE_DELETE", +#endif +#ifdef SQLITE_SMALL_STACK + "SMALL_STACK", +#endif +#ifdef SQLITE_SOUNDEX + "SOUNDEX", +#endif +#ifdef SQLITE_SYSTEM_MALLOC + "SYSTEM_MALLOC", +#endif +#ifdef SQLITE_TCL + "TCL", +#endif +#if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) + "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), +#endif +#ifdef SQLITE_TEST + "TEST", +#endif +#if defined(SQLITE_THREADSAFE) + "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), +#endif +#ifdef SQLITE_USE_ALLOCA + "USE_ALLOCA", +#endif +#ifdef SQLITE_WIN32_MALLOC + "WIN32_MALLOC", +#endif +#ifdef SQLITE_ZERO_MALLOC + "ZERO_MALLOC" +#endif +}; + +/* +** Given the name of a compile-time option, return true if that option +** was used and false if not. +** +** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix +** is not required for a match. +*/ +SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ + int i, n; + if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; + n = sqlite3Strlen30(zOptName); + + /* Since ArraySize(azCompileOpt) is normally in single digits, a + ** linear search is adequate. No need for a binary search. */ + for(i=0; i=0 && NaDb[] (or -1) */ + u8 nullRow; /* True if pointing to a row with no data */ + u8 rowidIsValid; /* True if lastRowid is valid */ + u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + Bool isEphemeral:1; /* True for an ephemeral table */ + Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ + Bool isTable:1; /* True if a table requiring integer keys */ + Bool isOrdered:1; /* True if the underlying table is BTREE_UNORDERED */ + Pgno pgnoRoot; /* Root page of the open btree cursor */ + sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ + i64 seqCount; /* Sequence counter */ + i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ + i64 lastRowid; /* Rowid being deleted by OP_Delete */ + VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ + + /* Cached information about the header for the data record that the + ** cursor is currently pointing to. Only valid if cacheStatus matches + ** Vdbe.cacheCtr. Vdbe.cacheCtr will never take on the value of + ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that + ** the cache is out of date. + ** + ** aRow might point to (ephemeral) data for the current row, or it might + ** be NULL. + */ + u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ + u32 payloadSize; /* Total number of bytes in the record */ + u32 szRow; /* Byte available in aRow */ + u32 iHdrOffset; /* Offset to next unparsed byte of the header */ + const u8 *aRow; /* Data for the current row, if all on one page */ + u32 aType[1]; /* Type values for all entries in the record */ + /* 2*nField extra array elements allocated for aType[], beyond the one + ** static element declared in the structure. nField total array slots for + ** aType[] and nField+1 array slots for aOffset[] */ +}; +typedef struct VdbeCursor VdbeCursor; + +/* +** When a sub-program is executed (OP_Program), a structure of this type +** is allocated to store the current value of the program counter, as +** well as the current memory cell array and various other frame specific +** values stored in the Vdbe struct. When the sub-program is finished, +** these values are copied back to the Vdbe from the VdbeFrame structure, +** restoring the state of the VM to as it was before the sub-program +** began executing. +** +** The memory for a VdbeFrame object is allocated and managed by a memory +** cell in the parent (calling) frame. When the memory cell is deleted or +** overwritten, the VdbeFrame object is not freed immediately. Instead, it +** is linked into the Vdbe.pDelFrame list. The contents of the Vdbe.pDelFrame +** list is deleted when the VM is reset in VdbeHalt(). The reason for doing +** this instead of deleting the VdbeFrame immediately is to avoid recursive +** calls to sqlite3VdbeMemRelease() when the memory cells belonging to the +** child frame are released. +** +** The currently executing frame is stored in Vdbe.pFrame. Vdbe.pFrame is +** set to NULL if the currently executing frame is the main program. +*/ +typedef struct VdbeFrame VdbeFrame; +struct VdbeFrame { + Vdbe *v; /* VM this frame belongs to */ + VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ + Op *aOp; /* Program instructions for parent frame */ + Mem *aMem; /* Array of memory cells for parent frame */ + u8 *aOnceFlag; /* Array of OP_Once flags for parent frame */ + VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ + void *token; /* Copy of SubProgram.token */ + i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ + int nCursor; /* Number of entries in apCsr */ + int pc; /* Program Counter in parent (calling) frame */ + int nOp; /* Size of aOp array */ + int nMem; /* Number of entries in aMem */ + int nOnceFlag; /* Number of entries in aOnceFlag */ + int nChildMem; /* Number of memory cells for child frame */ + int nChildCsr; /* Number of cursors for child frame */ + int nChange; /* Statement changes (Vdbe.nChanges) */ +}; + +#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) + +/* +** A value for VdbeCursor.cacheValid that means the cache is always invalid. +*/ +#define CACHE_STALE 0 + +/* +** Internally, the vdbe manipulates nearly all SQL values as Mem +** structures. Each Mem struct may cache multiple representations (string, +** integer etc.) of the same value. +*/ +struct Mem { + sqlite3 *db; /* The associated database connection */ + char *z; /* String or BLOB value */ + double r; /* Real value */ + union { + i64 i; /* Integer value used when MEM_Int is set in flags */ + int nZero; /* Used when bit MEM_Zero is set in flags */ + FuncDef *pDef; /* Used only when flags==MEM_Agg */ + RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ + VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ + } u; + int n; /* Number of characters in string value, excluding '\0' */ + u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ + u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ +#ifdef SQLITE_DEBUG + Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ + void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ +#endif + void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ + char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ +}; + +/* One or more of the following flags are set to indicate the validOK +** representations of the value stored in the Mem struct. +** +** If the MEM_Null flag is set, then the value is an SQL NULL value. +** No other flags may be set in this case. +** +** If the MEM_Str flag is set then Mem.z points at a string representation. +** Usually this is encoded in the same unicode encoding as the main +** database (see below for exceptions). If the MEM_Term flag is also +** set, then the string is nul terminated. The MEM_Int and MEM_Real +** flags may coexist with the MEM_Str flag. +*/ +#define MEM_Null 0x0001 /* Value is NULL */ +#define MEM_Str 0x0002 /* Value is a string */ +#define MEM_Int 0x0004 /* Value is an integer */ +#define MEM_Real 0x0008 /* Value is a real number */ +#define MEM_Blob 0x0010 /* Value is a BLOB */ +#define MEM_AffMask 0x001f /* Mask of affinity bits */ +#define MEM_RowSet 0x0020 /* Value is a RowSet object */ +#define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ +#define MEM_Undefined 0x0080 /* Value is undefined */ +#define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ +#define MEM_TypeMask 0x01ff /* Mask of type bits */ + + +/* Whenever Mem contains a valid string or blob representation, one of +** the following flags must be set to determine the memory management +** policy for Mem.z. The MEM_Term flag tells us whether or not the +** string is \000 or \u0000 terminated +*/ +#define MEM_Term 0x0200 /* String rep is nul terminated */ +#define MEM_Dyn 0x0400 /* Need to call Mem.xDel() on Mem.z */ +#define MEM_Static 0x0800 /* Mem.z points to a static string */ +#define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ +#define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ +#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ +#ifdef SQLITE_OMIT_INCRBLOB + #undef MEM_Zero + #define MEM_Zero 0x0000 +#endif + +/* +** Clear any existing type flags from a Mem and replace them with f +*/ +#define MemSetTypeFlag(p, f) \ + ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) + +/* +** Return true if a memory cell is not marked as invalid. This macro +** is for use inside assert() statements only. +*/ +#ifdef SQLITE_DEBUG +#define memIsValid(M) ((M)->flags & MEM_Undefined)==0 +#endif + +/* +** Each auxilliary data pointer stored by a user defined function +** implementation calling sqlite3_set_auxdata() is stored in an instance +** of this structure. All such structures associated with a single VM +** are stored in a linked list headed at Vdbe.pAuxData. All are destroyed +** when the VM is halted (if not before). +*/ +struct AuxData { + int iOp; /* Instruction number of OP_Function opcode */ + int iArg; /* Index of function argument. */ + void *pAux; /* Aux data pointer */ + void (*xDelete)(void *); /* Destructor for the aux data */ + AuxData *pNext; /* Next element in list */ +}; + +/* +** The "context" argument for a installable function. A pointer to an +** instance of this structure is the first argument to the routines used +** implement the SQL functions. +** +** There is a typedef for this structure in sqlite.h. So all routines, +** even the public interface to SQLite, can use a pointer to this structure. +** But this file is the only place where the internal details of this +** structure are known. +** +** This structure is defined inside of vdbeInt.h because it uses substructures +** (Mem) which are only defined there. +*/ +struct sqlite3_context { + FuncDef *pFunc; /* Pointer to function information. MUST BE FIRST */ + Mem s; /* The return value is stored here */ + Mem *pMem; /* Memory cell used to store aggregate context */ + CollSeq *pColl; /* Collating sequence */ + Vdbe *pVdbe; /* The VM that owns this context */ + int iOp; /* Instruction number of OP_Function */ + int isError; /* Error code returned by the function. */ + u8 skipFlag; /* Skip skip accumulator loading if true */ + u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ +}; + +/* +** An Explain object accumulates indented output which is helpful +** in describing recursive data structures. +*/ +struct Explain { + Vdbe *pVdbe; /* Attach the explanation to this Vdbe */ + StrAccum str; /* The string being accumulated */ + int nIndent; /* Number of elements in aIndent */ + u16 aIndent[100]; /* Levels of indentation */ + char zBase[100]; /* Initial space */ +}; + +/* A bitfield type for use inside of structures. Always follow with :N where +** N is the number of bits. +*/ +typedef unsigned bft; /* Bit Field Type */ + +/* +** An instance of the virtual machine. This structure contains the complete +** state of the virtual machine. +** +** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare() +** is really a pointer to an instance of this structure. +** +** The Vdbe.inVtabMethod variable is set to non-zero for the duration of +** any virtual table method invocations made by the vdbe program. It is +** set to 2 for xDestroy method calls and 1 for all other methods. This +** variable is used for two purposes: to allow xDestroy methods to execute +** "DROP TABLE" statements and to prevent some nasty side effects of +** malloc failure when SQLite is invoked recursively by a virtual table +** method function. +*/ +struct Vdbe { + sqlite3 *db; /* The database connection that owns this statement */ + Op *aOp; /* Space to hold the virtual machine's program */ + Mem *aMem; /* The memory locations */ + Mem **apArg; /* Arguments to currently executing user function */ + Mem *aColName; /* Column names to return */ + Mem *pResultSet; /* Pointer to an array of results */ + Parse *pParse; /* Parsing context used to create this Vdbe */ + int nMem; /* Number of memory locations currently allocated */ + int nOp; /* Number of instructions in the program */ + int nCursor; /* Number of slots in apCsr[] */ + u32 magic; /* Magic number for sanity checking */ + char *zErrMsg; /* Error message written here */ + Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ + VdbeCursor **apCsr; /* One element of this array for each open cursor */ + Mem *aVar; /* Values for the OP_Variable opcode. */ + char **azVar; /* Name of variables */ + ynVar nVar; /* Number of entries in aVar[] */ + ynVar nzVar; /* Number of entries in azVar[] */ + u32 cacheCtr; /* VdbeCursor row cache generation counter */ + int pc; /* The program counter */ + int rc; /* Value to return */ + u16 nResColumn; /* Number of columns in one row of the result set */ + u8 errorAction; /* Recovery action to do in case of an error */ + u8 minWriteFileFormat; /* Minimum file format for writable database files */ + bft explain:2; /* True if EXPLAIN present on SQL command */ + bft inVtabMethod:2; /* See comments above */ + bft changeCntOn:1; /* True to update the change-counter */ + bft expired:1; /* True if the VM needs to be recompiled */ + bft runOnlyOnce:1; /* Automatically expire on reset */ + bft usesStmtJournal:1; /* True if uses a statement journal */ + bft readOnly:1; /* True for statements that do not write */ + bft bIsReader:1; /* True for statements that read */ + bft isPrepareV2:1; /* True if prepared with prepare_v2() */ + bft doingRerun:1; /* True if rerunning after an auto-reprepare */ + int nChange; /* Number of db changes made since last reset */ + yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ + yDbMask lockMask; /* Subset of btreeMask that requires a lock */ + int iStatement; /* Statement number (or 0 if has not opened stmt) */ + u32 aCounter[5]; /* Counters used by sqlite3_stmt_status() */ +#ifndef SQLITE_OMIT_TRACE + i64 startTime; /* Time when query started - used for profiling */ +#endif + i64 iCurrentTime; /* Value of julianday('now') for this statement */ + i64 nFkConstraint; /* Number of imm. FK constraints this VM */ + i64 nStmtDefCons; /* Number of def. constraints when stmt started */ + i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ + char *zSql; /* Text of the SQL statement that generated this */ + void *pFree; /* Free this when deleting the vdbe */ +#ifdef SQLITE_ENABLE_TREE_EXPLAIN + Explain *pExplain; /* The explainer */ + char *zExplain; /* Explanation of data structures */ +#endif + VdbeFrame *pFrame; /* Parent frame */ + VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ + int nFrame; /* Number of frames in pFrame list */ + u32 expmask; /* Binding to these vars invalidates VM */ + SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ + int nOnceFlag; /* Size of array aOnceFlag[] */ + u8 *aOnceFlag; /* Flags for OP_Once */ + AuxData *pAuxData; /* Linked list of auxdata allocations */ +}; + +/* +** The following are allowed values for Vdbe.magic +*/ +#define VDBE_MAGIC_INIT 0x26bceaa5 /* Building a VDBE program */ +#define VDBE_MAGIC_RUN 0xbdf20da3 /* VDBE is ready to execute */ +#define VDBE_MAGIC_HALT 0x519c2973 /* VDBE has completed execution */ +#define VDBE_MAGIC_DEAD 0xb606c3c8 /* The VDBE has been deallocated */ + +/* +** Function prototypes +*/ +SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); +void sqliteVdbePopStack(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); +#endif +SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int); +SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); +SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); + +int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); +SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); +SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); +SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *, int); +SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem*, const Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); +SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem*, Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); +SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem*, i64); +#ifdef SQLITE_OMIT_FLOATING_POINT +# define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 +#else +SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem*, double); +#endif +SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); +SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, int); +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); +SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); +SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,int,Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); +SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p); +#define VdbeMemDynamic(X) \ + (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) +#define VdbeMemRelease(X) \ + if( VdbeMemDynamic(X) ) sqlite3VdbeMemReleaseExternal(X); +SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); +SQLITE_PRIVATE const char *sqlite3OpcodeName(int); +SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); +SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); +SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); +SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p); + +SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); +SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); +SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); +SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); +SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); +SQLITE_PRIVATE int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); +SQLITE_PRIVATE int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); +SQLITE_PRIVATE int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe*); +#else +# define sqlite3VdbeEnter(X) +# define sqlite3VdbeLeave(X) +#endif + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe*,Mem*); +SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem*); +#endif + +#ifndef SQLITE_OMIT_FOREIGN_KEY +SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); +#else +# define sqlite3VdbeCheckFk(p,i) 0 +#endif + +SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf); +#endif +SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem); + +#ifndef SQLITE_OMIT_INCRBLOB +SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); + #define ExpandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0) +#else + #define sqlite3VdbeMemExpandBlob(x) SQLITE_OK + #define ExpandBlob(P) SQLITE_OK +#endif + +#endif /* !defined(_VDBEINT_H_) */ + +/************** End of vdbeInt.h *********************************************/ +/************** Continuing where we left off in status.c *********************/ + +/* +** Variables in which to record status information. +*/ +typedef struct sqlite3StatType sqlite3StatType; +static SQLITE_WSD struct sqlite3StatType { + int nowValue[10]; /* Current value */ + int mxValue[10]; /* Maximum value */ +} sqlite3Stat = { {0,}, {0,} }; + + +/* The "wsdStat" macro will resolve to the status information +** state vector. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdStat can refer directly +** to the "sqlite3Stat" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdStatInit sqlite3StatType *x = &GLOBAL(sqlite3StatType,sqlite3Stat) +# define wsdStat x[0] +#else +# define wsdStatInit +# define wsdStat sqlite3Stat +#endif + +/* +** Return the current value of a status parameter. +*/ +SQLITE_PRIVATE int sqlite3StatusValue(int op){ + wsdStatInit; + assert( op>=0 && op=0 && opwsdStat.mxValue[op] ){ + wsdStat.mxValue[op] = wsdStat.nowValue[op]; + } +} + +/* +** Set the value of a status to X. +*/ +SQLITE_PRIVATE void sqlite3StatusSet(int op, int X){ + wsdStatInit; + assert( op>=0 && opwsdStat.mxValue[op] ){ + wsdStat.mxValue[op] = wsdStat.nowValue[op]; + } +} + +/* +** Query status information. +** +** This implementation assumes that reading or writing an aligned +** 32-bit integer is an atomic operation. If that assumption is not true, +** then this routine is not threadsafe. +*/ +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ + wsdStatInit; + if( op<0 || op>=ArraySize(wsdStat.nowValue) ){ + return SQLITE_MISUSE_BKPT; + } + *pCurrent = wsdStat.nowValue[op]; + *pHighwater = wsdStat.mxValue[op]; + if( resetFlag ){ + wsdStat.mxValue[op] = wsdStat.nowValue[op]; + } + return SQLITE_OK; +} + +/* +** Query status information for a single database connection +*/ +SQLITE_API int sqlite3_db_status( + sqlite3 *db, /* The database connection whose status is desired */ + int op, /* Status verb */ + int *pCurrent, /* Write current value here */ + int *pHighwater, /* Write high-water mark here */ + int resetFlag /* Reset high-water mark if true */ +){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_mutex_enter(db->mutex); + switch( op ){ + case SQLITE_DBSTATUS_LOOKASIDE_USED: { + *pCurrent = db->lookaside.nOut; + *pHighwater = db->lookaside.mxOut; + if( resetFlag ){ + db->lookaside.mxOut = db->lookaside.nOut; + } + break; + } + + case SQLITE_DBSTATUS_LOOKASIDE_HIT: + case SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE: + case SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL: { + testcase( op==SQLITE_DBSTATUS_LOOKASIDE_HIT ); + testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE ); + testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL ); + assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 ); + assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 ); + *pCurrent = 0; + *pHighwater = db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT]; + if( resetFlag ){ + db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0; + } + break; + } + + /* + ** Return an approximation for the amount of memory currently used + ** by all pagers associated with the given database connection. The + ** highwater mark is meaningless and is returned as zero. + */ + case SQLITE_DBSTATUS_CACHE_USED: { + int totalUsed = 0; + int i; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + totalUsed += sqlite3PagerMemUsed(pPager); + } + } + sqlite3BtreeLeaveAll(db); + *pCurrent = totalUsed; + *pHighwater = 0; + break; + } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store the schema for all databases (main, temp, and any ATTACHed + ** databases. *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_SCHEMA_USED: { + int i; /* Used to iterate through schemas */ + int nByte = 0; /* Used to accumulate return value */ + + sqlite3BtreeEnterAll(db); + db->pnBytesFreed = &nByte; + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( ALWAYS(pSchema!=0) ){ + HashElem *p; + + nByte += sqlite3GlobalConfig.m.xRoundup(sizeof(HashElem)) * ( + pSchema->tblHash.count + + pSchema->trigHash.count + + pSchema->idxHash.count + + pSchema->fkeyHash.count + ); + nByte += sqlite3MallocSize(pSchema->tblHash.ht); + nByte += sqlite3MallocSize(pSchema->trigHash.ht); + nByte += sqlite3MallocSize(pSchema->idxHash.ht); + nByte += sqlite3MallocSize(pSchema->fkeyHash.ht); + + for(p=sqliteHashFirst(&pSchema->trigHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTrigger(db, (Trigger*)sqliteHashData(p)); + } + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTable(db, (Table *)sqliteHashData(p)); + } + } + } + db->pnBytesFreed = 0; + sqlite3BtreeLeaveAll(db); + + *pHighwater = 0; + *pCurrent = nByte; + break; + } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store all prepared statements. + ** *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_STMT_USED: { + struct Vdbe *pVdbe; /* Used to iterate through VMs */ + int nByte = 0; /* Used to accumulate return value */ + + db->pnBytesFreed = &nByte; + for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ + sqlite3VdbeClearObject(db, pVdbe); + sqlite3DbFree(db, pVdbe); + } + db->pnBytesFreed = 0; + + *pHighwater = 0; + *pCurrent = nByte; + + break; + } + + /* + ** Set *pCurrent to the total cache hits or misses encountered by all + ** pagers the database handle is connected to. *pHighwater is always set + ** to zero. + */ + case SQLITE_DBSTATUS_CACHE_HIT: + case SQLITE_DBSTATUS_CACHE_MISS: + case SQLITE_DBSTATUS_CACHE_WRITE:{ + int i; + int nRet = 0; + assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); + assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); + + for(i=0; inDb; i++){ + if( db->aDb[i].pBt ){ + Pager *pPager = sqlite3BtreePager(db->aDb[i].pBt); + sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet); + } + } + *pHighwater = 0; + *pCurrent = nRet; + break; + } + + /* Set *pCurrent to non-zero if there are unresolved deferred foreign + ** key constraints. Set *pCurrent to zero if all foreign key constraints + ** have been satisfied. The *pHighwater is always set to zero. + */ + case SQLITE_DBSTATUS_DEFERRED_FKS: { + *pHighwater = 0; + *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0; + break; + } + + default: { + rc = SQLITE_ERROR; + } + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/************** End of status.c **********************************************/ +/************** Begin file date.c ********************************************/ +/* +** 2003 October 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement date and time +** functions for SQLite. +** +** There is only one exported symbol in this file - the function +** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. +** All other code has file scope. +** +** SQLite processes all times and dates as Julian Day numbers. The +** dates and times are stored as the number of days since noon +** in Greenwich on November 24, 4714 B.C. according to the Gregorian +** calendar system. +** +** 1970-01-01 00:00:00 is JD 2440587.5 +** 2000-01-01 00:00:00 is JD 2451544.5 +** +** This implemention requires years to be expressed as a 4-digit number +** which means that only dates between 0000-01-01 and 9999-12-31 can +** be represented, even though julian day numbers allow a much wider +** range of dates. +** +** The Gregorian calendar system is used for all dates and times, +** even those that predate the Gregorian calendar. Historians usually +** use the Julian calendar for dates prior to 1582-10-15 and for some +** dates afterwards, depending on locale. Beware of this difference. +** +** The conversion algorithms are implemented based on descriptions +** in the following text: +** +** Jean Meeus +** Astronomical Algorithms, 2nd Edition, 1998 +** ISBM 0-943396-61-1 +** Willmann-Bell, Inc +** Richmond, Virginia (USA) +*/ +/* #include */ +/* #include */ +#include + +#ifndef SQLITE_OMIT_DATETIME_FUNCS + + +/* +** A structure for holding a single date and time. +*/ +typedef struct DateTime DateTime; +struct DateTime { + sqlite3_int64 iJD; /* The julian day number times 86400000 */ + int Y, M, D; /* Year, month, and day */ + int h, m; /* Hour and minutes */ + int tz; /* Timezone offset in minutes */ + double s; /* Seconds */ + char validYMD; /* True (1) if Y,M,D are valid */ + char validHMS; /* True (1) if h,m,s are valid */ + char validJD; /* True (1) if iJD is valid */ + char validTZ; /* True (1) if tz is valid */ +}; + + +/* +** Convert zDate into one or more integers. Additional arguments +** come in groups of 5 as follows: +** +** N number of digits in the integer +** min minimum allowed value of the integer +** max maximum allowed value of the integer +** nextC first character after the integer +** pVal where to write the integers value. +** +** Conversions continue until one with nextC==0 is encountered. +** The function returns the number of successful conversions. +*/ +static int getDigits(const char *zDate, ...){ + va_list ap; + int val; + int N; + int min; + int max; + int nextC; + int *pVal; + int cnt = 0; + va_start(ap, zDate); + do{ + N = va_arg(ap, int); + min = va_arg(ap, int); + max = va_arg(ap, int); + nextC = va_arg(ap, int); + pVal = va_arg(ap, int*); + val = 0; + while( N-- ){ + if( !sqlite3Isdigit(*zDate) ){ + goto end_getDigits; + } + val = val*10 + *zDate - '0'; + zDate++; + } + if( valmax || (nextC!=0 && nextC!=*zDate) ){ + goto end_getDigits; + } + *pVal = val; + zDate++; + cnt++; + }while( nextC ); +end_getDigits: + va_end(ap); + return cnt; +} + +/* +** Parse a timezone extension on the end of a date-time. +** The extension is of the form: +** +** (+/-)HH:MM +** +** Or the "zulu" notation: +** +** Z +** +** If the parse is successful, write the number of minutes +** of change in p->tz and return 0. If a parser error occurs, +** return non-zero. +** +** A missing specifier is not considered an error. +*/ +static int parseTimezone(const char *zDate, DateTime *p){ + int sgn = 0; + int nHr, nMn; + int c; + while( sqlite3Isspace(*zDate) ){ zDate++; } + p->tz = 0; + c = *zDate; + if( c=='-' ){ + sgn = -1; + }else if( c=='+' ){ + sgn = +1; + }else if( c=='Z' || c=='z' ){ + zDate++; + goto zulu_time; + }else{ + return c!=0; + } + zDate++; + if( getDigits(zDate, 2, 0, 14, ':', &nHr, 2, 0, 59, 0, &nMn)!=2 ){ + return 1; + } + zDate += 5; + p->tz = sgn*(nMn + nHr*60); +zulu_time: + while( sqlite3Isspace(*zDate) ){ zDate++; } + return *zDate!=0; +} + +/* +** Parse times of the form HH:MM or HH:MM:SS or HH:MM:SS.FFFF. +** The HH, MM, and SS must each be exactly 2 digits. The +** fractional seconds FFFF can be one or more digits. +** +** Return 1 if there is a parsing error and 0 on success. +*/ +static int parseHhMmSs(const char *zDate, DateTime *p){ + int h, m, s; + double ms = 0.0; + if( getDigits(zDate, 2, 0, 24, ':', &h, 2, 0, 59, 0, &m)!=2 ){ + return 1; + } + zDate += 5; + if( *zDate==':' ){ + zDate++; + if( getDigits(zDate, 2, 0, 59, 0, &s)!=1 ){ + return 1; + } + zDate += 2; + if( *zDate=='.' && sqlite3Isdigit(zDate[1]) ){ + double rScale = 1.0; + zDate++; + while( sqlite3Isdigit(*zDate) ){ + ms = ms*10.0 + *zDate - '0'; + rScale *= 10.0; + zDate++; + } + ms /= rScale; + } + }else{ + s = 0; + } + p->validJD = 0; + p->validHMS = 1; + p->h = h; + p->m = m; + p->s = s + ms; + if( parseTimezone(zDate, p) ) return 1; + p->validTZ = (p->tz!=0)?1:0; + return 0; +} + +/* +** Convert from YYYY-MM-DD HH:MM:SS to julian day. We always assume +** that the YYYY-MM-DD is according to the Gregorian calendar. +** +** Reference: Meeus page 61 +*/ +static void computeJD(DateTime *p){ + int Y, M, D, A, B, X1, X2; + + if( p->validJD ) return; + if( p->validYMD ){ + Y = p->Y; + M = p->M; + D = p->D; + }else{ + Y = 2000; /* If no YMD specified, assume 2000-Jan-01 */ + M = 1; + D = 1; + } + if( M<=2 ){ + Y--; + M += 12; + } + A = Y/100; + B = 2 - A + (A/4); + X1 = 36525*(Y+4716)/100; + X2 = 306001*(M+1)/10000; + p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); + p->validJD = 1; + if( p->validHMS ){ + p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000); + if( p->validTZ ){ + p->iJD -= p->tz*60000; + p->validYMD = 0; + p->validHMS = 0; + p->validTZ = 0; + } + } +} + +/* +** Parse dates of the form +** +** YYYY-MM-DD HH:MM:SS.FFF +** YYYY-MM-DD HH:MM:SS +** YYYY-MM-DD HH:MM +** YYYY-MM-DD +** +** Write the result into the DateTime structure and return 0 +** on success and 1 if the input string is not a well-formed +** date. +*/ +static int parseYyyyMmDd(const char *zDate, DateTime *p){ + int Y, M, D, neg; + + if( zDate[0]=='-' ){ + zDate++; + neg = 1; + }else{ + neg = 0; + } + if( getDigits(zDate,4,0,9999,'-',&Y,2,1,12,'-',&M,2,1,31,0,&D)!=3 ){ + return 1; + } + zDate += 10; + while( sqlite3Isspace(*zDate) || 'T'==*(u8*)zDate ){ zDate++; } + if( parseHhMmSs(zDate, p)==0 ){ + /* We got the time */ + }else if( *zDate==0 ){ + p->validHMS = 0; + }else{ + return 1; + } + p->validJD = 0; + p->validYMD = 1; + p->Y = neg ? -Y : Y; + p->M = M; + p->D = D; + if( p->validTZ ){ + computeJD(p); + } + return 0; +} + +/* +** Set the time to the current time reported by the VFS. +** +** Return the number of errors. +*/ +static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ + p->iJD = sqlite3StmtCurrentTime(context); + if( p->iJD>0 ){ + p->validJD = 1; + return 0; + }else{ + return 1; + } +} + +/* +** Attempt to parse the given string into a Julian Day Number. Return +** the number of errors. +** +** The following are acceptable forms for the input string: +** +** YYYY-MM-DD HH:MM:SS.FFF +/-HH:MM +** DDDD.DD +** now +** +** In the first form, the +/-HH:MM is always optional. The fractional +** seconds extension (the ".FFF") is optional. The seconds portion +** (":SS.FFF") is option. The year and date can be omitted as long +** as there is a time string. The time string can be omitted as long +** as there is a year and date. +*/ +static int parseDateOrTime( + sqlite3_context *context, + const char *zDate, + DateTime *p +){ + double r; + if( parseYyyyMmDd(zDate,p)==0 ){ + return 0; + }else if( parseHhMmSs(zDate, p)==0 ){ + return 0; + }else if( sqlite3StrICmp(zDate,"now")==0){ + return setDateTimeToCurrent(context, p); + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ + p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); + p->validJD = 1; + return 0; + } + return 1; +} + +/* +** Compute the Year, Month, and Day from the julian day number. +*/ +static void computeYMD(DateTime *p){ + int Z, A, B, C, D, E, X1; + if( p->validYMD ) return; + if( !p->validJD ){ + p->Y = 2000; + p->M = 1; + p->D = 1; + }else{ + Z = (int)((p->iJD + 43200000)/86400000); + A = (int)((Z - 1867216.25)/36524.25); + A = Z + 1 + A - (A/4); + B = A + 1524; + C = (int)((B - 122.1)/365.25); + D = (36525*C)/100; + E = (int)((B-D)/30.6001); + X1 = (int)(30.6001*E); + p->D = B - D - X1; + p->M = E<14 ? E-1 : E-13; + p->Y = p->M>2 ? C - 4716 : C - 4715; + } + p->validYMD = 1; +} + +/* +** Compute the Hour, Minute, and Seconds from the julian day number. +*/ +static void computeHMS(DateTime *p){ + int s; + if( p->validHMS ) return; + computeJD(p); + s = (int)((p->iJD + 43200000) % 86400000); + p->s = s/1000.0; + s = (int)p->s; + p->s -= s; + p->h = s/3600; + s -= p->h*3600; + p->m = s/60; + p->s += s - p->m*60; + p->validHMS = 1; +} + +/* +** Compute both YMD and HMS +*/ +static void computeYMD_HMS(DateTime *p){ + computeYMD(p); + computeHMS(p); +} + +/* +** Clear the YMD and HMS and the TZ +*/ +static void clearYMD_HMS_TZ(DateTime *p){ + p->validYMD = 0; + p->validHMS = 0; + p->validTZ = 0; +} + +/* +** On recent Windows platforms, the localtime_s() function is available +** as part of the "Secure CRT". It is essentially equivalent to +** localtime_r() available under most POSIX platforms, except that the +** order of the parameters is reversed. +** +** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. +** +** If the user has not indicated to use localtime_r() or localtime_s() +** already, check for an MSVC build environment that provides +** localtime_s(). +*/ +#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ + defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#define HAVE_LOCALTIME_S 1 +#endif + +#ifndef SQLITE_OMIT_LOCALTIME +/* +** The following routine implements the rough equivalent of localtime_r() +** using whatever operating-system specific localtime facility that +** is available. This routine returns 0 on success and +** non-zero on any kind of error. +** +** If the sqlite3GlobalConfig.bLocaltimeFault variable is true then this +** routine will always fail. +** +** EVIDENCE-OF: R-62172-00036 In this implementation, the standard C +** library function localtime_r() is used to assist in the calculation of +** local time. +*/ +static int osLocaltime(time_t *t, struct tm *pTm){ + int rc; +#if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \ + && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S) + struct tm *pX; +#if SQLITE_THREADSAFE>0 + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + sqlite3_mutex_enter(mutex); + pX = localtime(t); +#ifndef SQLITE_OMIT_BUILTIN_TEST + if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; +#endif + if( pX ) *pTm = *pX; + sqlite3_mutex_leave(mutex); + rc = pX==0; +#else +#ifndef SQLITE_OMIT_BUILTIN_TEST + if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; +#endif +#if defined(HAVE_LOCALTIME_R) && HAVE_LOCALTIME_R + rc = localtime_r(t, pTm)==0; +#else + rc = localtime_s(pTm, t); +#endif /* HAVE_LOCALTIME_R */ +#endif /* HAVE_LOCALTIME_R || HAVE_LOCALTIME_S */ + return rc; +} +#endif /* SQLITE_OMIT_LOCALTIME */ + + +#ifndef SQLITE_OMIT_LOCALTIME +/* +** Compute the difference (in milliseconds) between localtime and UTC +** (a.k.a. GMT) for the time value p where p is in UTC. If no error occurs, +** return this value and set *pRc to SQLITE_OK. +** +** Or, if an error does occur, set *pRc to SQLITE_ERROR. The returned value +** is undefined in this case. +*/ +static sqlite3_int64 localtimeOffset( + DateTime *p, /* Date at which to calculate offset */ + sqlite3_context *pCtx, /* Write error here if one occurs */ + int *pRc /* OUT: Error code. SQLITE_OK or ERROR */ +){ + DateTime x, y; + time_t t; + struct tm sLocal; + + /* Initialize the contents of sLocal to avoid a compiler warning. */ + memset(&sLocal, 0, sizeof(sLocal)); + + x = *p; + computeYMD_HMS(&x); + if( x.Y<1971 || x.Y>=2038 ){ + /* EVIDENCE-OF: R-55269-29598 The localtime_r() C function normally only + ** works for years between 1970 and 2037. For dates outside this range, + ** SQLite attempts to map the year into an equivalent year within this + ** range, do the calculation, then map the year back. + */ + x.Y = 2000; + x.M = 1; + x.D = 1; + x.h = 0; + x.m = 0; + x.s = 0.0; + } else { + int s = (int)(x.s + 0.5); + x.s = s; + } + x.tz = 0; + x.validJD = 0; + computeJD(&x); + t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); + if( osLocaltime(&t, &sLocal) ){ + sqlite3_result_error(pCtx, "local time unavailable", -1); + *pRc = SQLITE_ERROR; + return 0; + } + y.Y = sLocal.tm_year + 1900; + y.M = sLocal.tm_mon + 1; + y.D = sLocal.tm_mday; + y.h = sLocal.tm_hour; + y.m = sLocal.tm_min; + y.s = sLocal.tm_sec; + y.validYMD = 1; + y.validHMS = 1; + y.validJD = 0; + y.validTZ = 0; + computeJD(&y); + *pRc = SQLITE_OK; + return y.iJD - x.iJD; +} +#endif /* SQLITE_OMIT_LOCALTIME */ + +/* +** Process a modifier to a date-time stamp. The modifiers are +** as follows: +** +** NNN days +** NNN hours +** NNN minutes +** NNN.NNNN seconds +** NNN months +** NNN years +** start of month +** start of year +** start of week +** start of day +** weekday N +** unixepoch +** localtime +** utc +** +** Return 0 on success and 1 if there is any kind of error. If the error +** is in a system call (i.e. localtime()), then an error message is written +** to context pCtx. If the error is an unrecognized modifier, no error is +** written to pCtx. +*/ +static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ + int rc = 1; + int n; + double r; + char *z, zBuf[30]; + z = zBuf; + for(n=0; niJD += localtimeOffset(p, pCtx, &rc); + clearYMD_HMS_TZ(p); + } + break; + } +#endif + case 'u': { + /* + ** unixepoch + ** + ** Treat the current value of p->iJD as the number of + ** seconds since 1970. Convert to a real julian day number. + */ + if( strcmp(z, "unixepoch")==0 && p->validJD ){ + p->iJD = (p->iJD + 43200)/86400 + 21086676*(i64)10000000; + clearYMD_HMS_TZ(p); + rc = 0; + } +#ifndef SQLITE_OMIT_LOCALTIME + else if( strcmp(z, "utc")==0 ){ + sqlite3_int64 c1; + computeJD(p); + c1 = localtimeOffset(p, pCtx, &rc); + if( rc==SQLITE_OK ){ + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + } + } +#endif + break; + } + case 'w': { + /* + ** weekday N + ** + ** Move the date to the same time on the next occurrence of + ** weekday N where 0==Sunday, 1==Monday, and so forth. If the + ** date is already on the appropriate weekday, this is a no-op. + */ + if( strncmp(z, "weekday ", 8)==0 + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) + && (n=(int)r)==r && n>=0 && r<7 ){ + sqlite3_int64 Z; + computeYMD_HMS(p); + p->validTZ = 0; + p->validJD = 0; + computeJD(p); + Z = ((p->iJD + 129600000)/86400000) % 7; + if( Z>n ) Z -= 7; + p->iJD += (n - Z)*86400000; + clearYMD_HMS_TZ(p); + rc = 0; + } + break; + } + case 's': { + /* + ** start of TTTTT + ** + ** Move the date backwards to the beginning of the current day, + ** or month or year. + */ + if( strncmp(z, "start of ", 9)!=0 ) break; + z += 9; + computeYMD(p); + p->validHMS = 1; + p->h = p->m = 0; + p->s = 0.0; + p->validTZ = 0; + p->validJD = 0; + if( strcmp(z,"month")==0 ){ + p->D = 1; + rc = 0; + }else if( strcmp(z,"year")==0 ){ + computeYMD(p); + p->M = 1; + p->D = 1; + rc = 0; + }else if( strcmp(z,"day")==0 ){ + rc = 0; + } + break; + } + case '+': + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + double rRounder; + for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} + if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ + rc = 1; + break; + } + if( z[n]==':' ){ + /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the + ** specified number of hours, minutes, seconds, and fractional seconds + ** to the time. The ".FFF" may be omitted. The ":SS.FFF" may be + ** omitted. + */ + const char *z2 = z; + DateTime tx; + sqlite3_int64 day; + if( !sqlite3Isdigit(*z2) ) z2++; + memset(&tx, 0, sizeof(tx)); + if( parseHhMmSs(z2, &tx) ) break; + computeJD(&tx); + tx.iJD -= 43200000; + day = tx.iJD/86400000; + tx.iJD -= day*86400000; + if( z[0]=='-' ) tx.iJD = -tx.iJD; + computeJD(p); + clearYMD_HMS_TZ(p); + p->iJD += tx.iJD; + rc = 0; + break; + } + z += n; + while( sqlite3Isspace(*z) ) z++; + n = sqlite3Strlen30(z); + if( n>10 || n<3 ) break; + if( z[n-1]=='s' ){ z[n-1] = 0; n--; } + computeJD(p); + rc = 0; + rRounder = r<0 ? -0.5 : +0.5; + if( n==3 && strcmp(z,"day")==0 ){ + p->iJD += (sqlite3_int64)(r*86400000.0 + rRounder); + }else if( n==4 && strcmp(z,"hour")==0 ){ + p->iJD += (sqlite3_int64)(r*(86400000.0/24.0) + rRounder); + }else if( n==6 && strcmp(z,"minute")==0 ){ + p->iJD += (sqlite3_int64)(r*(86400000.0/(24.0*60.0)) + rRounder); + }else if( n==6 && strcmp(z,"second")==0 ){ + p->iJD += (sqlite3_int64)(r*(86400000.0/(24.0*60.0*60.0)) + rRounder); + }else if( n==5 && strcmp(z,"month")==0 ){ + int x, y; + computeYMD_HMS(p); + p->M += (int)r; + x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; + p->Y += x; + p->M -= x*12; + p->validJD = 0; + computeJD(p); + y = (int)r; + if( y!=r ){ + p->iJD += (sqlite3_int64)((r - y)*30.0*86400000.0 + rRounder); + } + }else if( n==4 && strcmp(z,"year")==0 ){ + int y = (int)r; + computeYMD_HMS(p); + p->Y += y; + p->validJD = 0; + computeJD(p); + if( y!=r ){ + p->iJD += (sqlite3_int64)((r - y)*365.0*86400000.0 + rRounder); + } + }else{ + rc = 1; + } + clearYMD_HMS_TZ(p); + break; + } + default: { + break; + } + } + return rc; +} + +/* +** Process time function arguments. argv[0] is a date-time stamp. +** argv[1] and following are modifiers. Parse them all and write +** the resulting time into the DateTime structure p. Return 0 +** on success and 1 if there are any errors. +** +** If there are zero parameters (if even argv[0] is undefined) +** then assume a default value of "now" for argv[0]. +*/ +static int isDate( + sqlite3_context *context, + int argc, + sqlite3_value **argv, + DateTime *p +){ + int i; + const unsigned char *z; + int eType; + memset(p, 0, sizeof(*p)); + if( argc==0 ){ + return setDateTimeToCurrent(context, p); + } + if( (eType = sqlite3_value_type(argv[0]))==SQLITE_FLOAT + || eType==SQLITE_INTEGER ){ + p->iJD = (sqlite3_int64)(sqlite3_value_double(argv[0])*86400000.0 + 0.5); + p->validJD = 1; + }else{ + z = sqlite3_value_text(argv[0]); + if( !z || parseDateOrTime(context, (char*)z, p) ){ + return 1; + } + } + for(i=1; iaLimit[SQLITE_LIMIT_LENGTH]+1 ); + testcase( n==(u64)db->aLimit[SQLITE_LIMIT_LENGTH] ); + if( n(u64)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + return; + }else{ + z = sqlite3DbMallocRaw(db, (int)n); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + } + computeJD(&x); + computeYMD_HMS(&x); + for(i=j=0; zFmt[i]; i++){ + if( zFmt[i]!='%' ){ + z[j++] = zFmt[i]; + }else{ + i++; + switch( zFmt[i] ){ + case 'd': sqlite3_snprintf(3, &z[j],"%02d",x.D); j+=2; break; + case 'f': { + double s = x.s; + if( s>59.999 ) s = 59.999; + sqlite3_snprintf(7, &z[j],"%06.3f", s); + j += sqlite3Strlen30(&z[j]); + break; + } + case 'H': sqlite3_snprintf(3, &z[j],"%02d",x.h); j+=2; break; + case 'W': /* Fall thru */ + case 'j': { + int nDay; /* Number of days since 1st day of year */ + DateTime y = x; + y.validJD = 0; + y.M = 1; + y.D = 1; + computeJD(&y); + nDay = (int)((x.iJD-y.iJD+43200000)/86400000); + if( zFmt[i]=='W' ){ + int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ + wd = (int)(((x.iJD+43200000)/86400000)%7); + sqlite3_snprintf(3, &z[j],"%02d",(nDay+7-wd)/7); + j += 2; + }else{ + sqlite3_snprintf(4, &z[j],"%03d",nDay+1); + j += 3; + } + break; + } + case 'J': { + sqlite3_snprintf(20, &z[j],"%.16g",x.iJD/86400000.0); + j+=sqlite3Strlen30(&z[j]); + break; + } + case 'm': sqlite3_snprintf(3, &z[j],"%02d",x.M); j+=2; break; + case 'M': sqlite3_snprintf(3, &z[j],"%02d",x.m); j+=2; break; + case 's': { + sqlite3_snprintf(30,&z[j],"%lld", + (i64)(x.iJD/1000 - 21086676*(i64)10000)); + j += sqlite3Strlen30(&z[j]); + break; + } + case 'S': sqlite3_snprintf(3,&z[j],"%02d",(int)x.s); j+=2; break; + case 'w': { + z[j++] = (char)(((x.iJD+129600000)/86400000) % 7) + '0'; + break; + } + case 'Y': { + sqlite3_snprintf(5,&z[j],"%04d",x.Y); j+=sqlite3Strlen30(&z[j]); + break; + } + default: z[j++] = '%'; break; + } + } + } + z[j] = 0; + sqlite3_result_text(context, z, -1, + z==zBuf ? SQLITE_TRANSIENT : SQLITE_DYNAMIC); +} + +/* +** current_time() +** +** This function returns the same value as time('now'). +*/ +static void ctimeFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + timeFunc(context, 0, 0); +} + +/* +** current_date() +** +** This function returns the same value as date('now'). +*/ +static void cdateFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + dateFunc(context, 0, 0); +} + +/* +** current_timestamp() +** +** This function returns the same value as datetime('now'). +*/ +static void ctimestampFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + datetimeFunc(context, 0, 0); +} +#endif /* !defined(SQLITE_OMIT_DATETIME_FUNCS) */ + +#ifdef SQLITE_OMIT_DATETIME_FUNCS +/* +** If the library is compiled to omit the full-scale date and time +** handling (to get a smaller binary), the following minimal version +** of the functions current_time(), current_date() and current_timestamp() +** are included instead. This is to support column declarations that +** include "DEFAULT CURRENT_TIME" etc. +** +** This function uses the C-library functions time(), gmtime() +** and strftime(). The format string to pass to strftime() is supplied +** as the user-data for the function. +*/ +static void currentTimeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + time_t t; + char *zFormat = (char *)sqlite3_user_data(context); + sqlite3 *db; + sqlite3_int64 iT; + struct tm *pTm; + struct tm sNow; + char zBuf[20]; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + iT = sqlite3StmtCurrentTime(context); + if( iT<=0 ) return; + t = iT/1000 - 10000*(sqlite3_int64)21086676; +#ifdef HAVE_GMTIME_R + pTm = gmtime_r(&t, &sNow); +#else + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + pTm = gmtime(&t); + if( pTm ) memcpy(&sNow, pTm, sizeof(sNow)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +#endif + if( pTm ){ + strftime(zBuf, 20, zFormat, &sNow); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + } +} +#endif + +/* +** This function registered all of the above C functions as SQL +** functions. This should be the only routine in this file with +** external linkage. +*/ +SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ + static SQLITE_WSD FuncDef aDateTimeFuncs[] = { +#ifndef SQLITE_OMIT_DATETIME_FUNCS + FUNCTION(julianday, -1, 0, 0, juliandayFunc ), + FUNCTION(date, -1, 0, 0, dateFunc ), + FUNCTION(time, -1, 0, 0, timeFunc ), + FUNCTION(datetime, -1, 0, 0, datetimeFunc ), + FUNCTION(strftime, -1, 0, 0, strftimeFunc ), + FUNCTION(current_time, 0, 0, 0, ctimeFunc ), + FUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), + FUNCTION(current_date, 0, 0, 0, cdateFunc ), +#else + STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), + STR_FUNCTION(current_date, 0, "%Y-%m-%d", 0, currentTimeFunc), + STR_FUNCTION(current_timestamp, 0, "%Y-%m-%d %H:%M:%S", 0, currentTimeFunc), +#endif + }; + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aDateTimeFuncs); + + for(i=0; ipMethods ){ + rc = pId->pMethods->xClose(pId); + pId->pMethods = 0; + } + return rc; +} +SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file *id, void *pBuf, int amt, i64 offset){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xRead(id, pBuf, amt, offset); +} +SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file *id, const void *pBuf, int amt, i64 offset){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xWrite(id, pBuf, amt, offset); +} +SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file *id, i64 size){ + return id->pMethods->xTruncate(id, size); +} +SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file *id, int flags){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xSync(id, flags); +} +SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFileSize(id, pSize); +} +SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file *id, int lockType){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xLock(id, lockType); +} +SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file *id, int lockType){ + return id->pMethods->xUnlock(id, lockType); +} +SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xCheckReservedLock(id, pResOut); +} + +/* +** Use sqlite3OsFileControl() when we are doing something that might fail +** and we need to know about the failures. Use sqlite3OsFileControlHint() +** when simply tossing information over the wall to the VFS and we do not +** really care if the VFS receives and understands the information since it +** is only a hint and can be safely ignored. The sqlite3OsFileControlHint() +** routine has no return value since the return value would be meaningless. +*/ +SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ +#ifdef SQLITE_TEST + if( op!=SQLITE_FCNTL_COMMIT_PHASETWO ){ + /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite + ** is using a regular VFS, it is called after the corresponding + ** transaction has been committed. Injecting a fault at this point + ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM + ** but the transaction is committed anyway. + ** + ** The core must call OsFileControl() though, not OsFileControlHint(), + ** as if a custom VFS (e.g. zipvfs) returns an error here, it probably + ** means the commit really has failed and an error should be returned + ** to the user. */ + DO_OS_MALLOC_TEST(id); + } +#endif + return id->pMethods->xFileControl(id, op, pArg); +} +SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file *id, int op, void *pArg){ + (void)id->pMethods->xFileControl(id, op, pArg); +} + +SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id){ + int (*xSectorSize)(sqlite3_file*) = id->pMethods->xSectorSize; + return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); +} +SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ + return id->pMethods->xDeviceCharacteristics(id); +} +SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int offset, int n, int flags){ + return id->pMethods->xShmLock(id, offset, n, flags); +} +SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id){ + id->pMethods->xShmBarrier(id); +} +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int deleteFlag){ + return id->pMethods->xShmUnmap(id, deleteFlag); +} +SQLITE_PRIVATE int sqlite3OsShmMap( + sqlite3_file *id, /* Database file handle */ + int iPage, + int pgsz, + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Pointer to mapping */ +){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* The real implementation of xFetch and xUnfetch */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); +} +#else +/* No-op stubs to use when memory-mapped I/O is disabled */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + *pp = 0; + return SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return SQLITE_OK; +} +#endif + +/* +** The next group of routines are convenience wrappers around the +** VFS methods. +*/ +SQLITE_PRIVATE int sqlite3OsOpen( + sqlite3_vfs *pVfs, + const char *zPath, + sqlite3_file *pFile, + int flags, + int *pFlagsOut +){ + int rc; + DO_OS_MALLOC_TEST(0); + /* 0x87f7f is a mask of SQLITE_OPEN_ flags that are valid to be passed + ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, + ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before + ** reaching the VFS. */ + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f7f, pFlagsOut); + assert( rc==SQLITE_OK || pFile->pMethods==0 ); + return rc; +} +SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + DO_OS_MALLOC_TEST(0); + assert( dirSync==0 || dirSync==1 ); + return pVfs->xDelete(pVfs, zPath, dirSync); +} +SQLITE_PRIVATE int sqlite3OsAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + DO_OS_MALLOC_TEST(0); + return pVfs->xAccess(pVfs, zPath, flags, pResOut); +} +SQLITE_PRIVATE int sqlite3OsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nPathOut, + char *zPathOut +){ + DO_OS_MALLOC_TEST(0); + zPathOut[0] = 0; + return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut); +} +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return pVfs->xDlOpen(pVfs, zPath); +} +SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + pVfs->xDlError(pVfs, nByte, zBufOut); +} +SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *pVfs, void *pHdle, const char *zSym))(void){ + return pVfs->xDlSym(pVfs, pHdle, zSym); +} +SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *pVfs, void *pHandle){ + pVfs->xDlClose(pVfs, pHandle); +} +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return pVfs->xRandomness(pVfs, nByte, zBufOut); +} +SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ + return pVfs->xSleep(pVfs, nMicro); +} +SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ + int rc; + /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() + ** method to get the current date and time if that method is available + ** (if iVersion is 2 or greater and the function pointer is not NULL) and + ** will fall back to xCurrentTime() if xCurrentTimeInt64() is + ** unavailable. + */ + if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ + rc = pVfs->xCurrentTimeInt64(pVfs, pTimeOut); + }else{ + double r; + rc = pVfs->xCurrentTime(pVfs, &r); + *pTimeOut = (sqlite3_int64)(r*86400000.0); + } + return rc; +} + +SQLITE_PRIVATE int sqlite3OsOpenMalloc( + sqlite3_vfs *pVfs, + const char *zFile, + sqlite3_file **ppFile, + int flags, + int *pOutFlags +){ + int rc = SQLITE_NOMEM; + sqlite3_file *pFile; + pFile = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile); + if( pFile ){ + rc = sqlite3OsOpen(pVfs, zFile, pFile, flags, pOutFlags); + if( rc!=SQLITE_OK ){ + sqlite3_free(pFile); + }else{ + *ppFile = pFile; + } + } + return rc; +} +SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *pFile){ + int rc = SQLITE_OK; + assert( pFile ); + rc = sqlite3OsClose(pFile); + sqlite3_free(pFile); + return rc; +} + +/* +** This function is a wrapper around the OS specific implementation of +** sqlite3_os_init(). The purpose of the wrapper is to provide the +** ability to simulate a malloc failure, so that the handling of an +** error in sqlite3_os_init() by the upper layers can be tested. +*/ +SQLITE_PRIVATE int sqlite3OsInit(void){ + void *p = sqlite3_malloc(10); + if( p==0 ) return SQLITE_NOMEM; + sqlite3_free(p); + return sqlite3_os_init(); +} + +/* +** The list of all registered VFS implementations. +*/ +static sqlite3_vfs * SQLITE_WSD vfsList = 0; +#define vfsList GLOBAL(sqlite3_vfs *, vfsList) + +/* +** Locate a VFS by name. If no name is given, simply return the +** first VFS on the list. +*/ +SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfs){ + sqlite3_vfs *pVfs = 0; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex; +#endif +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return 0; +#endif +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + sqlite3_mutex_enter(mutex); + for(pVfs = vfsList; pVfs; pVfs=pVfs->pNext){ + if( zVfs==0 ) break; + if( strcmp(zVfs, pVfs->zName)==0 ) break; + } + sqlite3_mutex_leave(mutex); + return pVfs; +} + +/* +** Unlink a VFS from the linked list +*/ +static void vfsUnlink(sqlite3_vfs *pVfs){ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ); + if( pVfs==0 ){ + /* No-op */ + }else if( vfsList==pVfs ){ + vfsList = pVfs->pNext; + }else if( vfsList ){ + sqlite3_vfs *p = vfsList; + while( p->pNext && p->pNext!=pVfs ){ + p = p->pNext; + } + if( p->pNext==pVfs ){ + p->pNext = pVfs->pNext; + } + } +} + +/* +** Register a VFS with the system. It is harmless to register the same +** VFS multiple times. The new VFS becomes the default if makeDflt is +** true. +*/ +SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ + MUTEX_LOGIC(sqlite3_mutex *mutex;) +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + sqlite3_mutex_enter(mutex); + vfsUnlink(pVfs); + if( makeDflt || vfsList==0 ){ + pVfs->pNext = vfsList; + vfsList = pVfs; + }else{ + pVfs->pNext = vfsList->pNext; + vfsList->pNext = pVfs; + } + assert(vfsList); + sqlite3_mutex_leave(mutex); + return SQLITE_OK; +} + +/* +** Unregister a VFS so that it is no longer accessible. +*/ +SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + sqlite3_mutex_enter(mutex); + vfsUnlink(pVfs); + sqlite3_mutex_leave(mutex); + return SQLITE_OK; +} + +/************** End of os.c **************************************************/ +/************** Begin file fault.c *******************************************/ +/* +** 2008 Jan 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code to support the concept of "benign" +** malloc failures (when the xMalloc() or xRealloc() method of the +** sqlite3_mem_methods structure fails to allocate a block of memory +** and returns 0). +** +** Most malloc failures are non-benign. After they occur, SQLite +** abandons the current operation and returns an error code (usually +** SQLITE_NOMEM) to the user. However, sometimes a fault is not necessarily +** fatal. For example, if a malloc fails while resizing a hash table, this +** is completely recoverable simply by not carrying out the resize. The +** hash table will continue to function normally. So a malloc failure +** during a hash table resize is a benign fault. +*/ + + +#ifndef SQLITE_OMIT_BUILTIN_TEST + +/* +** Global variables. +*/ +typedef struct BenignMallocHooks BenignMallocHooks; +static SQLITE_WSD struct BenignMallocHooks { + void (*xBenignBegin)(void); + void (*xBenignEnd)(void); +} sqlite3Hooks = { 0, 0 }; + +/* The "wsdHooks" macro will resolve to the appropriate BenignMallocHooks +** structure. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdHooks can refer directly +** to the "sqlite3Hooks" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdHooksInit \ + BenignMallocHooks *x = &GLOBAL(BenignMallocHooks,sqlite3Hooks) +# define wsdHooks x[0] +#else +# define wsdHooksInit +# define wsdHooks sqlite3Hooks +#endif + + +/* +** Register hooks to call when sqlite3BeginBenignMalloc() and +** sqlite3EndBenignMalloc() are called, respectively. +*/ +SQLITE_PRIVATE void sqlite3BenignMallocHooks( + void (*xBenignBegin)(void), + void (*xBenignEnd)(void) +){ + wsdHooksInit; + wsdHooks.xBenignBegin = xBenignBegin; + wsdHooks.xBenignEnd = xBenignEnd; +} + +/* +** This (sqlite3EndBenignMalloc()) is called by SQLite code to indicate that +** subsequent malloc failures are benign. A call to sqlite3EndBenignMalloc() +** indicates that subsequent malloc failures are non-benign. +*/ +SQLITE_PRIVATE void sqlite3BeginBenignMalloc(void){ + wsdHooksInit; + if( wsdHooks.xBenignBegin ){ + wsdHooks.xBenignBegin(); + } +} +SQLITE_PRIVATE void sqlite3EndBenignMalloc(void){ + wsdHooksInit; + if( wsdHooks.xBenignEnd ){ + wsdHooks.xBenignEnd(); + } +} + +#endif /* #ifndef SQLITE_OMIT_BUILTIN_TEST */ + +/************** End of fault.c ***********************************************/ +/************** Begin file mem0.c ********************************************/ +/* +** 2008 October 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains a no-op memory allocation drivers for use when +** SQLITE_ZERO_MALLOC is defined. The allocation drivers implemented +** here always fail. SQLite will not operate with these drivers. These +** are merely placeholders. Real drivers must be substituted using +** sqlite3_config() before SQLite will operate. +*/ + +/* +** This version of the memory allocator is the default. It is +** used when no other memory allocator is specified using compile-time +** macros. +*/ +#ifdef SQLITE_ZERO_MALLOC + +/* +** No-op versions of all memory allocation routines +*/ +static void *sqlite3MemMalloc(int nByte){ return 0; } +static void sqlite3MemFree(void *pPrior){ return; } +static void *sqlite3MemRealloc(void *pPrior, int nByte){ return 0; } +static int sqlite3MemSize(void *pPrior){ return 0; } +static int sqlite3MemRoundup(int n){ return n; } +static int sqlite3MemInit(void *NotUsed){ return SQLITE_OK; } +static void sqlite3MemShutdown(void *NotUsed){ return; } + +/* +** This routine is the only routine in this file with external linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +SQLITE_PRIVATE void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +#endif /* SQLITE_ZERO_MALLOC */ + +/************** End of mem0.c ************************************************/ +/************** Begin file mem1.c ********************************************/ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains low-level memory allocation drivers for when +** SQLite will use the standard C-library malloc/realloc/free interface +** to obtain the memory it needs. +** +** This file contains implementations of the low-level memory allocation +** routines specified in the sqlite3_mem_methods object. The content of +** this file is only used if SQLITE_SYSTEM_MALLOC is defined. The +** SQLITE_SYSTEM_MALLOC macro is defined automatically if neither the +** SQLITE_MEMDEBUG nor the SQLITE_WIN32_MALLOC macros are defined. The +** default configuration is to use memory allocation routines in this +** file. +** +** C-preprocessor macro summary: +** +** HAVE_MALLOC_USABLE_SIZE The configure script sets this symbol if +** the malloc_usable_size() interface exists +** on the target platform. Or, this symbol +** can be set manually, if desired. +** If an equivalent interface exists by +** a different name, using a separate -D +** option to rename it. +** +** SQLITE_WITHOUT_ZONEMALLOC Some older macs lack support for the zone +** memory allocator. Set this symbol to enable +** building on older macs. +** +** SQLITE_WITHOUT_MSIZE Set this symbol to disable the use of +** _msize() on windows systems. This might +** be necessary when compiling for Delphi, +** for example. +*/ + +/* +** This version of the memory allocator is the default. It is +** used when no other memory allocator is specified using compile-time +** macros. +*/ +#ifdef SQLITE_SYSTEM_MALLOC +#if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) + +/* +** Use the zone allocator available on apple products unless the +** SQLITE_WITHOUT_ZONEMALLOC symbol is defined. +*/ +#include +#include +#include +static malloc_zone_t* _sqliteZone_; +#define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) +#define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); +#define SQLITE_REALLOC(x,y) malloc_zone_realloc(_sqliteZone_, (x), (y)) +#define SQLITE_MALLOCSIZE(x) \ + (_sqliteZone_ ? _sqliteZone_->size(_sqliteZone_,x) : malloc_size(x)) + +#else /* if not __APPLE__ */ + +/* +** Use standard C library malloc and free on non-Apple systems. +** Also used by Apple systems if SQLITE_WITHOUT_ZONEMALLOC is defined. +*/ +#define SQLITE_MALLOC(x) malloc(x) +#define SQLITE_FREE(x) free(x) +#define SQLITE_REALLOC(x,y) realloc((x),(y)) + +/* +** The malloc.h header file is needed for malloc_usable_size() function +** on some systems (e.g. Linux). +*/ +#if defined(HAVE_MALLOC_H) && defined(HAVE_MALLOC_USABLE_SIZE) +# define SQLITE_USE_MALLOC_H +# define SQLITE_USE_MALLOC_USABLE_SIZE +/* +** The MSVCRT has malloc_usable_size(), but it is called _msize(). The +** use of _msize() is automatic, but can be disabled by compiling with +** -DSQLITE_WITHOUT_MSIZE. Using the _msize() function also requires +** the malloc.h header file. +*/ +#elif defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE) +# define SQLITE_USE_MALLOC_H +# define SQLITE_USE_MSIZE +#endif + +/* +** Include the malloc.h header file, if necessary. Also set define macro +** SQLITE_MALLOCSIZE to the appropriate function name, which is _msize() +** for MSVC and malloc_usable_size() for most other systems (e.g. Linux). +** The memory size function can always be overridden manually by defining +** the macro SQLITE_MALLOCSIZE to the desired function name. +*/ +#if defined(SQLITE_USE_MALLOC_H) +# include +# if defined(SQLITE_USE_MALLOC_USABLE_SIZE) +# if !defined(SQLITE_MALLOCSIZE) +# define SQLITE_MALLOCSIZE(x) malloc_usable_size(x) +# endif +# elif defined(SQLITE_USE_MSIZE) +# if !defined(SQLITE_MALLOCSIZE) +# define SQLITE_MALLOCSIZE _msize +# endif +# endif +#endif /* defined(SQLITE_USE_MALLOC_H) */ + +#endif /* __APPLE__ or not __APPLE__ */ + +/* +** Like malloc(), but remember the size of the allocation +** so that we can find it later using sqlite3MemSize(). +** +** For this low-level routine, we are guaranteed that nByte>0 because +** cases of nByte<=0 will be intercepted and dealt with by higher level +** routines. +*/ +static void *sqlite3MemMalloc(int nByte){ +#ifdef SQLITE_MALLOCSIZE + void *p = SQLITE_MALLOC( nByte ); + if( p==0 ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); + } + return p; +#else + sqlite3_int64 *p; + assert( nByte>0 ); + nByte = ROUND8(nByte); + p = SQLITE_MALLOC( nByte+8 ); + if( p ){ + p[0] = nByte; + p++; + }else{ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); + } + return (void *)p; +#endif +} + +/* +** Like free() but works for allocations obtained from sqlite3MemMalloc() +** or sqlite3MemRealloc(). +** +** For this low-level routine, we already know that pPrior!=0 since +** cases where pPrior==0 will have been intecepted and dealt with +** by higher-level routines. +*/ +static void sqlite3MemFree(void *pPrior){ +#ifdef SQLITE_MALLOCSIZE + SQLITE_FREE(pPrior); +#else + sqlite3_int64 *p = (sqlite3_int64*)pPrior; + assert( pPrior!=0 ); + p--; + SQLITE_FREE(p); +#endif +} + +/* +** Report the allocated size of a prior return from xMalloc() +** or xRealloc(). +*/ +static int sqlite3MemSize(void *pPrior){ +#ifdef SQLITE_MALLOCSIZE + return pPrior ? (int)SQLITE_MALLOCSIZE(pPrior) : 0; +#else + sqlite3_int64 *p; + if( pPrior==0 ) return 0; + p = (sqlite3_int64*)pPrior; + p--; + return (int)p[0]; +#endif +} + +/* +** Like realloc(). Resize an allocation previously obtained from +** sqlite3MemMalloc(). +** +** For this low-level interface, we know that pPrior!=0. Cases where +** pPrior==0 while have been intercepted by higher-level routine and +** redirected to xMalloc. Similarly, we know that nByte>0 becauses +** cases where nByte<=0 will have been intercepted by higher-level +** routines and redirected to xFree. +*/ +static void *sqlite3MemRealloc(void *pPrior, int nByte){ +#ifdef SQLITE_MALLOCSIZE + void *p = SQLITE_REALLOC(pPrior, nByte); + if( p==0 ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, + "failed memory resize %u to %u bytes", + SQLITE_MALLOCSIZE(pPrior), nByte); + } + return p; +#else + sqlite3_int64 *p = (sqlite3_int64*)pPrior; + assert( pPrior!=0 && nByte>0 ); + assert( nByte==ROUND8(nByte) ); /* EV: R-46199-30249 */ + p--; + p = SQLITE_REALLOC(p, nByte+8 ); + if( p ){ + p[0] = nByte; + p++; + }else{ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, + "failed memory resize %u to %u bytes", + sqlite3MemSize(pPrior), nByte); + } + return (void*)p; +#endif +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int sqlite3MemRoundup(int n){ + return ROUND8(n); +} + +/* +** Initialize this module. +*/ +static int sqlite3MemInit(void *NotUsed){ +#if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) + int cpuCount; + size_t len; + if( _sqliteZone_ ){ + return SQLITE_OK; + } + len = sizeof(cpuCount); + /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); + if( cpuCount>1 ){ + /* defer MT decisions to system malloc */ + _sqliteZone_ = malloc_default_zone(); + }else{ + /* only 1 core, use our own zone to contention over global locks, + ** e.g. we have our own dedicated locks */ + bool success; + malloc_zone_t* newzone = malloc_create_zone(4096, 0); + malloc_set_zone_name(newzone, "Sqlite_Heap"); + do{ + success = OSAtomicCompareAndSwapPtrBarrier(NULL, newzone, + (void * volatile *)&_sqliteZone_); + }while(!_sqliteZone_); + if( !success ){ + /* somebody registered a zone first */ + malloc_destroy_zone(newzone); + } + } +#endif + UNUSED_PARAMETER(NotUsed); + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void sqlite3MemShutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return; +} + +/* +** This routine is the only routine in this file with external linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +SQLITE_PRIVATE void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +#endif /* SQLITE_SYSTEM_MALLOC */ + +/************** End of mem1.c ************************************************/ +/************** Begin file mem2.c ********************************************/ +/* +** 2007 August 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains low-level memory allocation drivers for when +** SQLite will use the standard C-library malloc/realloc/free interface +** to obtain the memory it needs while adding lots of additional debugging +** information to each allocation in order to help detect and fix memory +** leaks and memory usage errors. +** +** This file contains implementations of the low-level memory allocation +** routines specified in the sqlite3_mem_methods object. +*/ + +/* +** This version of the memory allocator is used only if the +** SQLITE_MEMDEBUG macro is defined +*/ +#ifdef SQLITE_MEMDEBUG + +/* +** The backtrace functionality is only available with GLIBC +*/ +#ifdef __GLIBC__ + extern int backtrace(void**,int); + extern void backtrace_symbols_fd(void*const*,int,int); +#else +# define backtrace(A,B) 1 +# define backtrace_symbols_fd(A,B,C) +#endif +/* #include */ + +/* +** Each memory allocation looks like this: +** +** ------------------------------------------------------------------------ +** | Title | backtrace pointers | MemBlockHdr | allocation | EndGuard | +** ------------------------------------------------------------------------ +** +** The application code sees only a pointer to the allocation. We have +** to back up from the allocation pointer to find the MemBlockHdr. The +** MemBlockHdr tells us the size of the allocation and the number of +** backtrace pointers. There is also a guard word at the end of the +** MemBlockHdr. +*/ +struct MemBlockHdr { + i64 iSize; /* Size of this allocation */ + struct MemBlockHdr *pNext, *pPrev; /* Linked list of all unfreed memory */ + char nBacktrace; /* Number of backtraces on this alloc */ + char nBacktraceSlots; /* Available backtrace slots */ + u8 nTitle; /* Bytes of title; includes '\0' */ + u8 eType; /* Allocation type code */ + int iForeGuard; /* Guard word for sanity */ +}; + +/* +** Guard words +*/ +#define FOREGUARD 0x80F5E153 +#define REARGUARD 0xE4676B53 + +/* +** Number of malloc size increments to track. +*/ +#define NCSIZE 1000 + +/* +** All of the static variables used by this module are collected +** into a single structure named "mem". This is to keep the +** static variables organized and to reduce namespace pollution +** when this module is combined with other in the amalgamation. +*/ +static struct { + + /* + ** Mutex to control access to the memory allocation subsystem. + */ + sqlite3_mutex *mutex; + + /* + ** Head and tail of a linked list of all outstanding allocations + */ + struct MemBlockHdr *pFirst; + struct MemBlockHdr *pLast; + + /* + ** The number of levels of backtrace to save in new allocations. + */ + int nBacktrace; + void (*xBacktrace)(int, int, void **); + + /* + ** Title text to insert in front of each block + */ + int nTitle; /* Bytes of zTitle to save. Includes '\0' and padding */ + char zTitle[100]; /* The title text */ + + /* + ** sqlite3MallocDisallow() increments the following counter. + ** sqlite3MallocAllow() decrements it. + */ + int disallow; /* Do not allow memory allocation */ + + /* + ** Gather statistics on the sizes of memory allocations. + ** nAlloc[i] is the number of allocation attempts of i*8 + ** bytes. i==NCSIZE is the number of allocation attempts for + ** sizes more than NCSIZE*8 bytes. + */ + int nAlloc[NCSIZE]; /* Total number of allocations */ + int nCurrent[NCSIZE]; /* Current number of allocations */ + int mxCurrent[NCSIZE]; /* Highwater mark for nCurrent */ + +} mem; + + +/* +** Adjust memory usage statistics +*/ +static void adjustStats(int iSize, int increment){ + int i = ROUND8(iSize)/8; + if( i>NCSIZE-1 ){ + i = NCSIZE - 1; + } + if( increment>0 ){ + mem.nAlloc[i]++; + mem.nCurrent[i]++; + if( mem.nCurrent[i]>mem.mxCurrent[i] ){ + mem.mxCurrent[i] = mem.nCurrent[i]; + } + }else{ + mem.nCurrent[i]--; + assert( mem.nCurrent[i]>=0 ); + } +} + +/* +** Given an allocation, find the MemBlockHdr for that allocation. +** +** This routine checks the guards at either end of the allocation and +** if they are incorrect it asserts. +*/ +static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){ + struct MemBlockHdr *p; + int *pInt; + u8 *pU8; + int nReserve; + + p = (struct MemBlockHdr*)pAllocation; + p--; + assert( p->iForeGuard==(int)FOREGUARD ); + nReserve = ROUND8(p->iSize); + pInt = (int*)pAllocation; + pU8 = (u8*)pAllocation; + assert( pInt[nReserve/sizeof(int)]==(int)REARGUARD ); + /* This checks any of the "extra" bytes allocated due + ** to rounding up to an 8 byte boundary to ensure + ** they haven't been overwritten. + */ + while( nReserve-- > p->iSize ) assert( pU8[nReserve]==0x65 ); + return p; +} + +/* +** Return the number of bytes currently allocated at address p. +*/ +static int sqlite3MemSize(void *p){ + struct MemBlockHdr *pHdr; + if( !p ){ + return 0; + } + pHdr = sqlite3MemsysGetHeader(p); + return (int)pHdr->iSize; +} + +/* +** Initialize the memory allocation subsystem. +*/ +static int sqlite3MemInit(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( (sizeof(struct MemBlockHdr)&7) == 0 ); + if( !sqlite3GlobalConfig.bMemstat ){ + /* If memory status is enabled, then the malloc.c wrapper will already + ** hold the STATIC_MEM mutex when the routines here are invoked. */ + mem.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + return SQLITE_OK; +} + +/* +** Deinitialize the memory allocation subsystem. +*/ +static void sqlite3MemShutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + mem.mutex = 0; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int sqlite3MemRoundup(int n){ + return ROUND8(n); +} + +/* +** Fill a buffer with pseudo-random bytes. This is used to preset +** the content of a new memory allocation to unpredictable values and +** to clear the content of a freed allocation to unpredictable values. +*/ +static void randomFill(char *pBuf, int nByte){ + unsigned int x, y, r; + x = SQLITE_PTR_TO_INT(pBuf); + y = nByte | 1; + while( nByte >= 4 ){ + x = (x>>1) ^ (-(int)(x&1) & 0xd0000001); + y = y*1103515245 + 12345; + r = x ^ y; + *(int*)pBuf = r; + pBuf += 4; + nByte -= 4; + } + while( nByte-- > 0 ){ + x = (x>>1) ^ (-(int)(x&1) & 0xd0000001); + y = y*1103515245 + 12345; + r = x ^ y; + *(pBuf++) = r & 0xff; + } +} + +/* +** Allocate nByte bytes of memory. +*/ +static void *sqlite3MemMalloc(int nByte){ + struct MemBlockHdr *pHdr; + void **pBt; + char *z; + int *pInt; + void *p = 0; + int totalSize; + int nReserve; + sqlite3_mutex_enter(mem.mutex); + assert( mem.disallow==0 ); + nReserve = ROUND8(nByte); + totalSize = nReserve + sizeof(*pHdr) + sizeof(int) + + mem.nBacktrace*sizeof(void*) + mem.nTitle; + p = malloc(totalSize); + if( p ){ + z = p; + pBt = (void**)&z[mem.nTitle]; + pHdr = (struct MemBlockHdr*)&pBt[mem.nBacktrace]; + pHdr->pNext = 0; + pHdr->pPrev = mem.pLast; + if( mem.pLast ){ + mem.pLast->pNext = pHdr; + }else{ + mem.pFirst = pHdr; + } + mem.pLast = pHdr; + pHdr->iForeGuard = FOREGUARD; + pHdr->eType = MEMTYPE_HEAP; + pHdr->nBacktraceSlots = mem.nBacktrace; + pHdr->nTitle = mem.nTitle; + if( mem.nBacktrace ){ + void *aAddr[40]; + pHdr->nBacktrace = backtrace(aAddr, mem.nBacktrace+1)-1; + memcpy(pBt, &aAddr[1], pHdr->nBacktrace*sizeof(void*)); + assert(pBt[0]); + if( mem.xBacktrace ){ + mem.xBacktrace(nByte, pHdr->nBacktrace-1, &aAddr[1]); + } + }else{ + pHdr->nBacktrace = 0; + } + if( mem.nTitle ){ + memcpy(z, mem.zTitle, mem.nTitle); + } + pHdr->iSize = nByte; + adjustStats(nByte, +1); + pInt = (int*)&pHdr[1]; + pInt[nReserve/sizeof(int)] = REARGUARD; + randomFill((char*)pInt, nByte); + memset(((char*)pInt)+nByte, 0x65, nReserve-nByte); + p = (void*)pInt; + } + sqlite3_mutex_leave(mem.mutex); + return p; +} + +/* +** Free memory. +*/ +static void sqlite3MemFree(void *pPrior){ + struct MemBlockHdr *pHdr; + void **pBt; + char *z; + assert( sqlite3GlobalConfig.bMemstat || sqlite3GlobalConfig.bCoreMutex==0 + || mem.mutex!=0 ); + pHdr = sqlite3MemsysGetHeader(pPrior); + pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + sqlite3_mutex_enter(mem.mutex); + if( pHdr->pPrev ){ + assert( pHdr->pPrev->pNext==pHdr ); + pHdr->pPrev->pNext = pHdr->pNext; + }else{ + assert( mem.pFirst==pHdr ); + mem.pFirst = pHdr->pNext; + } + if( pHdr->pNext ){ + assert( pHdr->pNext->pPrev==pHdr ); + pHdr->pNext->pPrev = pHdr->pPrev; + }else{ + assert( mem.pLast==pHdr ); + mem.pLast = pHdr->pPrev; + } + z = (char*)pBt; + z -= pHdr->nTitle; + adjustStats((int)pHdr->iSize, -1); + randomFill(z, sizeof(void*)*pHdr->nBacktraceSlots + sizeof(*pHdr) + + (int)pHdr->iSize + sizeof(int) + pHdr->nTitle); + free(z); + sqlite3_mutex_leave(mem.mutex); +} + +/* +** Change the size of an existing memory allocation. +** +** For this debugging implementation, we *always* make a copy of the +** allocation into a new place in memory. In this way, if the +** higher level code is using pointer to the old allocation, it is +** much more likely to break and we are much more liking to find +** the error. +*/ +static void *sqlite3MemRealloc(void *pPrior, int nByte){ + struct MemBlockHdr *pOldHdr; + void *pNew; + assert( mem.disallow==0 ); + assert( (nByte & 7)==0 ); /* EV: R-46199-30249 */ + pOldHdr = sqlite3MemsysGetHeader(pPrior); + pNew = sqlite3MemMalloc(nByte); + if( pNew ){ + memcpy(pNew, pPrior, (int)(nByteiSize ? nByte : pOldHdr->iSize)); + if( nByte>pOldHdr->iSize ){ + randomFill(&((char*)pNew)[pOldHdr->iSize], nByte - (int)pOldHdr->iSize); + } + sqlite3MemFree(pPrior); + } + return pNew; +} + +/* +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +SQLITE_PRIVATE void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +/* +** Set the "type" of an allocation. +*/ +SQLITE_PRIVATE void sqlite3MemdebugSetType(void *p, u8 eType){ + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); + pHdr->eType = eType; + } +} + +/* +** Return TRUE if the mask of type in eType matches the type of the +** allocation p. Also return true if p==NULL. +** +** This routine is designed for use within an assert() statement, to +** verify the type of an allocation. For example: +** +** assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); +*/ +SQLITE_PRIVATE int sqlite3MemdebugHasType(void *p, u8 eType){ + int rc = 1; + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ + if( (pHdr->eType&eType)==0 ){ + rc = 0; + } + } + return rc; +} + +/* +** Return TRUE if the mask of type in eType matches no bits of the type of the +** allocation p. Also return true if p==NULL. +** +** This routine is designed for use within an assert() statement, to +** verify the type of an allocation. For example: +** +** assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); +*/ +SQLITE_PRIVATE int sqlite3MemdebugNoType(void *p, u8 eType){ + int rc = 1; + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ + if( (pHdr->eType&eType)!=0 ){ + rc = 0; + } + } + return rc; +} + +/* +** Set the number of backtrace levels kept for each allocation. +** A value of zero turns off backtracing. The number is always rounded +** up to a multiple of 2. +*/ +SQLITE_PRIVATE void sqlite3MemdebugBacktrace(int depth){ + if( depth<0 ){ depth = 0; } + if( depth>20 ){ depth = 20; } + depth = (depth+1)&0xfe; + mem.nBacktrace = depth; +} + +SQLITE_PRIVATE void sqlite3MemdebugBacktraceCallback(void (*xBacktrace)(int, int, void **)){ + mem.xBacktrace = xBacktrace; +} + +/* +** Set the title string for subsequent allocations. +*/ +SQLITE_PRIVATE void sqlite3MemdebugSettitle(const char *zTitle){ + unsigned int n = sqlite3Strlen30(zTitle) + 1; + sqlite3_mutex_enter(mem.mutex); + if( n>=sizeof(mem.zTitle) ) n = sizeof(mem.zTitle)-1; + memcpy(mem.zTitle, zTitle, n); + mem.zTitle[n] = 0; + mem.nTitle = ROUND8(n); + sqlite3_mutex_leave(mem.mutex); +} + +SQLITE_PRIVATE void sqlite3MemdebugSync(){ + struct MemBlockHdr *pHdr; + for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ + void **pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + mem.xBacktrace((int)pHdr->iSize, pHdr->nBacktrace-1, &pBt[1]); + } +} + +/* +** Open the file indicated and write a log of all unfreed memory +** allocations into that log. +*/ +SQLITE_PRIVATE void sqlite3MemdebugDump(const char *zFilename){ + FILE *out; + struct MemBlockHdr *pHdr; + void **pBt; + int i; + out = fopen(zFilename, "w"); + if( out==0 ){ + fprintf(stderr, "** Unable to output memory debug output log: %s **\n", + zFilename); + return; + } + for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ + char *z = (char*)pHdr; + z -= pHdr->nBacktraceSlots*sizeof(void*) + pHdr->nTitle; + fprintf(out, "**** %lld bytes at %p from %s ****\n", + pHdr->iSize, &pHdr[1], pHdr->nTitle ? z : "???"); + if( pHdr->nBacktrace ){ + fflush(out); + pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(out)); + fprintf(out, "\n"); + } + } + fprintf(out, "COUNTS:\n"); + for(i=0; i=1 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( size==mem3.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + memsys3UnlinkFromList(i, &mem3.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + memsys3UnlinkFromList(i, &mem3.aiHash[hash]); + } +} + +/* +** Link the chunk at mem3.aPool[i] so that is on the list rooted +** at *pRoot. +*/ +static void memsys3LinkIntoList(u32 i, u32 *pRoot){ + assert( sqlite3_mutex_held(mem3.mutex) ); + mem3.aPool[i].u.list.next = *pRoot; + mem3.aPool[i].u.list.prev = 0; + if( *pRoot ){ + mem3.aPool[*pRoot].u.list.prev = i; + } + *pRoot = i; +} + +/* +** Link the chunk at index i into either the appropriate +** small chunk list, or into the large chunk hash table. +*/ +static void memsys3Link(u32 i){ + u32 size, hash; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( i>=1 ); + assert( (mem3.aPool[i-1].u.hdr.size4x & 1)==0 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( size==mem3.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + memsys3LinkIntoList(i, &mem3.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + memsys3LinkIntoList(i, &mem3.aiHash[hash]); + } +} + +/* +** If the STATIC_MEM mutex is not already held, obtain it now. The mutex +** will already be held (obtained by code in malloc.c) if +** sqlite3GlobalConfig.bMemStat is true. +*/ +static void memsys3Enter(void){ + if( sqlite3GlobalConfig.bMemstat==0 && mem3.mutex==0 ){ + mem3.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + sqlite3_mutex_enter(mem3.mutex); +} +static void memsys3Leave(void){ + sqlite3_mutex_leave(mem3.mutex); +} + +/* +** Called when we are unable to satisfy an allocation of nBytes. +*/ +static void memsys3OutOfMemory(int nByte){ + if( !mem3.alarmBusy ){ + mem3.alarmBusy = 1; + assert( sqlite3_mutex_held(mem3.mutex) ); + sqlite3_mutex_leave(mem3.mutex); + sqlite3_release_memory(nByte); + sqlite3_mutex_enter(mem3.mutex); + mem3.alarmBusy = 0; + } +} + + +/* +** Chunk i is a free chunk that has been unlinked. Adjust its +** size parameters for check-out and return a pointer to the +** user portion of the chunk. +*/ +static void *memsys3Checkout(u32 i, u32 nBlock){ + u32 x; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( i>=1 ); + assert( mem3.aPool[i-1].u.hdr.size4x/4==nBlock ); + assert( mem3.aPool[i+nBlock-1].u.hdr.prevSize==nBlock ); + x = mem3.aPool[i-1].u.hdr.size4x; + mem3.aPool[i-1].u.hdr.size4x = nBlock*4 | 1 | (x&2); + mem3.aPool[i+nBlock-1].u.hdr.prevSize = nBlock; + mem3.aPool[i+nBlock-1].u.hdr.size4x |= 2; + return &mem3.aPool[i]; +} + +/* +** Carve a piece off of the end of the mem3.iMaster free chunk. +** Return a pointer to the new allocation. Or, if the master chunk +** is not large enough, return 0. +*/ +static void *memsys3FromMaster(u32 nBlock){ + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( mem3.szMaster>=nBlock ); + if( nBlock>=mem3.szMaster-1 ){ + /* Use the entire master */ + void *p = memsys3Checkout(mem3.iMaster, mem3.szMaster); + mem3.iMaster = 0; + mem3.szMaster = 0; + mem3.mnMaster = 0; + return p; + }else{ + /* Split the master block. Return the tail. */ + u32 newi, x; + newi = mem3.iMaster + mem3.szMaster - nBlock; + assert( newi > mem3.iMaster+1 ); + mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.prevSize = nBlock; + mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.size4x |= 2; + mem3.aPool[newi-1].u.hdr.size4x = nBlock*4 + 1; + mem3.szMaster -= nBlock; + mem3.aPool[newi-1].u.hdr.prevSize = mem3.szMaster; + x = mem3.aPool[mem3.iMaster-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iMaster-1].u.hdr.size4x = mem3.szMaster*4 | x; + if( mem3.szMaster < mem3.mnMaster ){ + mem3.mnMaster = mem3.szMaster; + } + return (void*)&mem3.aPool[newi]; + } +} + +/* +** *pRoot is the head of a list of free chunks of the same size +** or same size hash. In other words, *pRoot is an entry in either +** mem3.aiSmall[] or mem3.aiHash[]. +** +** This routine examines all entries on the given list and tries +** to coalesce each entries with adjacent free chunks. +** +** If it sees a chunk that is larger than mem3.iMaster, it replaces +** the current mem3.iMaster with the new larger chunk. In order for +** this mem3.iMaster replacement to work, the master chunk must be +** linked into the hash tables. That is not the normal state of +** affairs, of course. The calling routine must link the master +** chunk before invoking this routine, then must unlink the (possibly +** changed) master chunk once this routine has finished. +*/ +static void memsys3Merge(u32 *pRoot){ + u32 iNext, prev, size, i, x; + + assert( sqlite3_mutex_held(mem3.mutex) ); + for(i=*pRoot; i>0; i=iNext){ + iNext = mem3.aPool[i].u.list.next; + size = mem3.aPool[i-1].u.hdr.size4x; + assert( (size&1)==0 ); + if( (size&2)==0 ){ + memsys3UnlinkFromList(i, pRoot); + assert( i > mem3.aPool[i-1].u.hdr.prevSize ); + prev = i - mem3.aPool[i-1].u.hdr.prevSize; + if( prev==iNext ){ + iNext = mem3.aPool[prev].u.list.next; + } + memsys3Unlink(prev); + size = i + size/4 - prev; + x = mem3.aPool[prev-1].u.hdr.size4x & 2; + mem3.aPool[prev-1].u.hdr.size4x = size*4 | x; + mem3.aPool[prev+size-1].u.hdr.prevSize = size; + memsys3Link(prev); + i = prev; + }else{ + size /= 4; + } + if( size>mem3.szMaster ){ + mem3.iMaster = i; + mem3.szMaster = size; + } + } +} + +/* +** Return a block of memory of at least nBytes in size. +** Return NULL if unable. +** +** This function assumes that the necessary mutexes, if any, are +** already held by the caller. Hence "Unsafe". +*/ +static void *memsys3MallocUnsafe(int nByte){ + u32 i; + u32 nBlock; + u32 toFree; + + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( sizeof(Mem3Block)==8 ); + if( nByte<=12 ){ + nBlock = 2; + }else{ + nBlock = (nByte + 11)/8; + } + assert( nBlock>=2 ); + + /* STEP 1: + ** Look for an entry of the correct size in either the small + ** chunk table or in the large chunk hash table. This is + ** successful most of the time (about 9 times out of 10). + */ + if( nBlock <= MX_SMALL ){ + i = mem3.aiSmall[nBlock-2]; + if( i>0 ){ + memsys3UnlinkFromList(i, &mem3.aiSmall[nBlock-2]); + return memsys3Checkout(i, nBlock); + } + }else{ + int hash = nBlock % N_HASH; + for(i=mem3.aiHash[hash]; i>0; i=mem3.aPool[i].u.list.next){ + if( mem3.aPool[i-1].u.hdr.size4x/4==nBlock ){ + memsys3UnlinkFromList(i, &mem3.aiHash[hash]); + return memsys3Checkout(i, nBlock); + } + } + } + + /* STEP 2: + ** Try to satisfy the allocation by carving a piece off of the end + ** of the master chunk. This step usually works if step 1 fails. + */ + if( mem3.szMaster>=nBlock ){ + return memsys3FromMaster(nBlock); + } + + + /* STEP 3: + ** Loop through the entire memory pool. Coalesce adjacent free + ** chunks. Recompute the master chunk as the largest free chunk. + ** Then try again to satisfy the allocation by carving a piece off + ** of the end of the master chunk. This step happens very + ** rarely (we hope!) + */ + for(toFree=nBlock*16; toFree<(mem3.nPool*16); toFree *= 2){ + memsys3OutOfMemory(toFree); + if( mem3.iMaster ){ + memsys3Link(mem3.iMaster); + mem3.iMaster = 0; + mem3.szMaster = 0; + } + for(i=0; i=nBlock ){ + return memsys3FromMaster(nBlock); + } + } + } + + /* If none of the above worked, then we fail. */ + return 0; +} + +/* +** Free an outstanding memory allocation. +** +** This function assumes that the necessary mutexes, if any, are +** already held by the caller. Hence "Unsafe". +*/ +static void memsys3FreeUnsafe(void *pOld){ + Mem3Block *p = (Mem3Block*)pOld; + int i; + u32 size, x; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] ); + i = p - mem3.aPool; + assert( (mem3.aPool[i-1].u.hdr.size4x&1)==1 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( i+size<=mem3.nPool+1 ); + mem3.aPool[i-1].u.hdr.size4x &= ~1; + mem3.aPool[i+size-1].u.hdr.prevSize = size; + mem3.aPool[i+size-1].u.hdr.size4x &= ~2; + memsys3Link(i); + + /* Try to expand the master using the newly freed chunk */ + if( mem3.iMaster ){ + while( (mem3.aPool[mem3.iMaster-1].u.hdr.size4x&2)==0 ){ + size = mem3.aPool[mem3.iMaster-1].u.hdr.prevSize; + mem3.iMaster -= size; + mem3.szMaster += size; + memsys3Unlink(mem3.iMaster); + x = mem3.aPool[mem3.iMaster-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iMaster-1].u.hdr.size4x = mem3.szMaster*4 | x; + mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.prevSize = mem3.szMaster; + } + x = mem3.aPool[mem3.iMaster-1].u.hdr.size4x & 2; + while( (mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.size4x&1)==0 ){ + memsys3Unlink(mem3.iMaster+mem3.szMaster); + mem3.szMaster += mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.size4x/4; + mem3.aPool[mem3.iMaster-1].u.hdr.size4x = mem3.szMaster*4 | x; + mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.prevSize = mem3.szMaster; + } + } +} + +/* +** Return the size of an outstanding allocation, in bytes. The +** size returned omits the 8-byte header overhead. This only +** works for chunks that are currently checked out. +*/ +static int memsys3Size(void *p){ + Mem3Block *pBlock; + if( p==0 ) return 0; + pBlock = (Mem3Block*)p; + assert( (pBlock[-1].u.hdr.size4x&1)!=0 ); + return (pBlock[-1].u.hdr.size4x&~3)*2 - 4; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int memsys3Roundup(int n){ + if( n<=12 ){ + return 12; + }else{ + return ((n+11)&~7) - 4; + } +} + +/* +** Allocate nBytes of memory. +*/ +static void *memsys3Malloc(int nBytes){ + sqlite3_int64 *p; + assert( nBytes>0 ); /* malloc.c filters out 0 byte requests */ + memsys3Enter(); + p = memsys3MallocUnsafe(nBytes); + memsys3Leave(); + return (void*)p; +} + +/* +** Free memory. +*/ +static void memsys3Free(void *pPrior){ + assert( pPrior ); + memsys3Enter(); + memsys3FreeUnsafe(pPrior); + memsys3Leave(); +} + +/* +** Change the size of an existing memory allocation +*/ +static void *memsys3Realloc(void *pPrior, int nBytes){ + int nOld; + void *p; + if( pPrior==0 ){ + return sqlite3_malloc(nBytes); + } + if( nBytes<=0 ){ + sqlite3_free(pPrior); + return 0; + } + nOld = memsys3Size(pPrior); + if( nBytes<=nOld && nBytes>=nOld-128 ){ + return pPrior; + } + memsys3Enter(); + p = memsys3MallocUnsafe(nBytes); + if( p ){ + if( nOld>1)!=(size&1) ){ + fprintf(out, "%p tail checkout bit is incorrect\n", &mem3.aPool[i]); + assert( 0 ); + break; + } + if( size&1 ){ + fprintf(out, "%p %6d bytes checked out\n", &mem3.aPool[i], (size/4)*8-8); + }else{ + fprintf(out, "%p %6d bytes free%s\n", &mem3.aPool[i], (size/4)*8-8, + i==mem3.iMaster ? " **master**" : ""); + } + } + for(i=0; i0; j=mem3.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem3.aPool[j], + (mem3.aPool[j-1].u.hdr.size4x/4)*8-8); + } + fprintf(out, "\n"); + } + for(i=0; i0; j=mem3.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem3.aPool[j], + (mem3.aPool[j-1].u.hdr.size4x/4)*8-8); + } + fprintf(out, "\n"); + } + fprintf(out, "master=%d\n", mem3.iMaster); + fprintf(out, "nowUsed=%d\n", mem3.nPool*8 - mem3.szMaster*8); + fprintf(out, "mxUsed=%d\n", mem3.nPool*8 - mem3.mnMaster*8); + sqlite3_mutex_leave(mem3.mutex); + if( out==stdout ){ + fflush(stdout); + }else{ + fclose(out); + } +#else + UNUSED_PARAMETER(zFilename); +#endif +} + +/* +** This routine is the only routine in this file with external +** linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. The +** arguments specify the block of memory to manage. +** +** This routine is only called by sqlite3_config(), and therefore +** is not required to be threadsafe (it is not). +*/ +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ + static const sqlite3_mem_methods mempoolMethods = { + memsys3Malloc, + memsys3Free, + memsys3Realloc, + memsys3Size, + memsys3Roundup, + memsys3Init, + memsys3Shutdown, + 0 + }; + return &mempoolMethods; +} + +#endif /* SQLITE_ENABLE_MEMSYS3 */ + +/************** End of mem3.c ************************************************/ +/************** Begin file mem5.c ********************************************/ +/* +** 2007 October 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement a memory +** allocation subsystem for use by SQLite. +** +** This version of the memory allocation subsystem omits all +** use of malloc(). The application gives SQLite a block of memory +** before calling sqlite3_initialize() from which allocations +** are made and returned by the xMalloc() and xRealloc() +** implementations. Once sqlite3_initialize() has been called, +** the amount of memory available to SQLite is fixed and cannot +** be changed. +** +** This version of the memory allocation subsystem is included +** in the build only if SQLITE_ENABLE_MEMSYS5 is defined. +** +** This memory allocator uses the following algorithm: +** +** 1. All memory allocations sizes are rounded up to a power of 2. +** +** 2. If two adjacent free blocks are the halves of a larger block, +** then the two blocks are coalesed into the single larger block. +** +** 3. New memory is allocated from the first available free block. +** +** This algorithm is described in: J. M. Robson. "Bounds for Some Functions +** Concerning Dynamic Storage Allocation". Journal of the Association for +** Computing Machinery, Volume 21, Number 8, July 1974, pages 491-499. +** +** Let n be the size of the largest allocation divided by the minimum +** allocation size (after rounding all sizes up to a power of 2.) Let M +** be the maximum amount of memory ever outstanding at one time. Let +** N be the total amount of memory available for allocation. Robson +** proved that this memory allocator will never breakdown due to +** fragmentation as long as the following constraint holds: +** +** N >= M*(1 + log2(n)/2) - n + 1 +** +** The sqlite3_status() logic tracks the maximum values of n and M so +** that an application can, at any time, verify this constraint. +*/ + +/* +** This version of the memory allocator is used only when +** SQLITE_ENABLE_MEMSYS5 is defined. +*/ +#ifdef SQLITE_ENABLE_MEMSYS5 + +/* +** A minimum allocation is an instance of the following structure. +** Larger allocations are an array of these structures where the +** size of the array is a power of 2. +** +** The size of this object must be a power of two. That fact is +** verified in memsys5Init(). +*/ +typedef struct Mem5Link Mem5Link; +struct Mem5Link { + int next; /* Index of next free chunk */ + int prev; /* Index of previous free chunk */ +}; + +/* +** Maximum size of any allocation is ((1<=0 && i=0 && iLogsize<=LOGMAX ); + assert( (mem5.aCtrl[i] & CTRL_LOGSIZE)==iLogsize ); + + next = MEM5LINK(i)->next; + prev = MEM5LINK(i)->prev; + if( prev<0 ){ + mem5.aiFreelist[iLogsize] = next; + }else{ + MEM5LINK(prev)->next = next; + } + if( next>=0 ){ + MEM5LINK(next)->prev = prev; + } +} + +/* +** Link the chunk at mem5.aPool[i] so that is on the iLogsize +** free list. +*/ +static void memsys5Link(int i, int iLogsize){ + int x; + assert( sqlite3_mutex_held(mem5.mutex) ); + assert( i>=0 && i=0 && iLogsize<=LOGMAX ); + assert( (mem5.aCtrl[i] & CTRL_LOGSIZE)==iLogsize ); + + x = MEM5LINK(i)->next = mem5.aiFreelist[iLogsize]; + MEM5LINK(i)->prev = -1; + if( x>=0 ){ + assert( xprev = i; + } + mem5.aiFreelist[iLogsize] = i; +} + +/* +** If the STATIC_MEM mutex is not already held, obtain it now. The mutex +** will already be held (obtained by code in malloc.c) if +** sqlite3GlobalConfig.bMemStat is true. +*/ +static void memsys5Enter(void){ + sqlite3_mutex_enter(mem5.mutex); +} +static void memsys5Leave(void){ + sqlite3_mutex_leave(mem5.mutex); +} + +/* +** Return the size of an outstanding allocation, in bytes. The +** size returned omits the 8-byte header overhead. This only +** works for chunks that are currently checked out. +*/ +static int memsys5Size(void *p){ + int iSize = 0; + if( p ){ + int i = (int)(((u8 *)p-mem5.zPool)/mem5.szAtom); + assert( i>=0 && i0 ); + + /* Keep track of the maximum allocation request. Even unfulfilled + ** requests are counted */ + if( (u32)nByte>mem5.maxRequest ){ + mem5.maxRequest = nByte; + } + + /* Abort if the requested allocation size is larger than the largest + ** power of two that we can represent using 32-bit signed integers. + */ + if( nByte > 0x40000000 ){ + return 0; + } + + /* Round nByte up to the next valid power of two */ + for(iFullSz=mem5.szAtom, iLogsize=0; iFullSzLOGMAX ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes", nByte); + return 0; + } + i = mem5.aiFreelist[iBin]; + memsys5Unlink(i, iBin); + while( iBin>iLogsize ){ + int newSize; + + iBin--; + newSize = 1 << iBin; + mem5.aCtrl[i+newSize] = CTRL_FREE | iBin; + memsys5Link(i+newSize, iBin); + } + mem5.aCtrl[i] = iLogsize; + + /* Update allocator performance statistics. */ + mem5.nAlloc++; + mem5.totalAlloc += iFullSz; + mem5.totalExcess += iFullSz - nByte; + mem5.currentCount++; + mem5.currentOut += iFullSz; + if( mem5.maxCount=0 && iBlock0 ); + assert( mem5.currentOut>=(size*mem5.szAtom) ); + mem5.currentCount--; + mem5.currentOut -= size*mem5.szAtom; + assert( mem5.currentOut>0 || mem5.currentCount==0 ); + assert( mem5.currentCount>0 || mem5.currentOut==0 ); + + mem5.aCtrl[iBlock] = CTRL_FREE | iLogsize; + while( ALWAYS(iLogsize>iLogsize) & 1 ){ + iBuddy = iBlock - size; + }else{ + iBuddy = iBlock + size; + } + assert( iBuddy>=0 ); + if( (iBuddy+(1<mem5.nBlock ) break; + if( mem5.aCtrl[iBuddy]!=(CTRL_FREE | iLogsize) ) break; + memsys5Unlink(iBuddy, iLogsize); + iLogsize++; + if( iBuddy0 ){ + memsys5Enter(); + p = memsys5MallocUnsafe(nBytes); + memsys5Leave(); + } + return (void*)p; +} + +/* +** Free memory. +** +** The outer layer memory allocator prevents this routine from +** being called with pPrior==0. +*/ +static void memsys5Free(void *pPrior){ + assert( pPrior!=0 ); + memsys5Enter(); + memsys5FreeUnsafe(pPrior); + memsys5Leave(); +} + +/* +** Change the size of an existing memory allocation. +** +** The outer layer memory allocator prevents this routine from +** being called with pPrior==0. +** +** nBytes is always a value obtained from a prior call to +** memsys5Round(). Hence nBytes is always a non-negative power +** of two. If nBytes==0 that means that an oversize allocation +** (an allocation larger than 0x40000000) was requested and this +** routine should return 0 without freeing pPrior. +*/ +static void *memsys5Realloc(void *pPrior, int nBytes){ + int nOld; + void *p; + assert( pPrior!=0 ); + assert( (nBytes&(nBytes-1))==0 ); /* EV: R-46199-30249 */ + assert( nBytes>=0 ); + if( nBytes==0 ){ + return 0; + } + nOld = memsys5Size(pPrior); + if( nBytes<=nOld ){ + return pPrior; + } + memsys5Enter(); + p = memsys5MallocUnsafe(nBytes); + if( p ){ + memcpy(p, pPrior, nOld); + memsys5FreeUnsafe(pPrior); + } + memsys5Leave(); + return p; +} + +/* +** Round up a request size to the next valid allocation size. If +** the allocation is too large to be handled by this allocation system, +** return 0. +** +** All allocations must be a power of two and must be expressed by a +** 32-bit signed integer. Hence the largest allocation is 0x40000000 +** or 1073741824 bytes. +*/ +static int memsys5Roundup(int n){ + int iFullSz; + if( n > 0x40000000 ) return 0; + for(iFullSz=mem5.szAtom; iFullSz 0 +** memsys5Log(2) -> 1 +** memsys5Log(4) -> 2 +** memsys5Log(5) -> 3 +** memsys5Log(8) -> 3 +** memsys5Log(9) -> 4 +*/ +static int memsys5Log(int iValue){ + int iLog; + for(iLog=0; (iLog<(int)((sizeof(int)*8)-1)) && (1<mem5.szAtom ){ + mem5.szAtom = mem5.szAtom << 1; + } + + mem5.nBlock = (nByte / (mem5.szAtom+sizeof(u8))); + mem5.zPool = zByte; + mem5.aCtrl = (u8 *)&mem5.zPool[mem5.nBlock*mem5.szAtom]; + + for(ii=0; ii<=LOGMAX; ii++){ + mem5.aiFreelist[ii] = -1; + } + + iOffset = 0; + for(ii=LOGMAX; ii>=0; ii--){ + int nAlloc = (1<mem5.nBlock); + } + + /* If a mutex is required for normal operation, allocate one */ + if( sqlite3GlobalConfig.bMemstat==0 ){ + mem5.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void memsys5Shutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + mem5.mutex = 0; + return; +} + +#ifdef SQLITE_TEST +/* +** Open the file indicated and write a log of all unfreed memory +** allocations into that log. +*/ +SQLITE_PRIVATE void sqlite3Memsys5Dump(const char *zFilename){ + FILE *out; + int i, j, n; + int nMinLog; + + if( zFilename==0 || zFilename[0]==0 ){ + out = stdout; + }else{ + out = fopen(zFilename, "w"); + if( out==0 ){ + fprintf(stderr, "** Unable to output memory debug output log: %s **\n", + zFilename); + return; + } + } + memsys5Enter(); + nMinLog = memsys5Log(mem5.szAtom); + for(i=0; i<=LOGMAX && i+nMinLog<32; i++){ + for(n=0, j=mem5.aiFreelist[i]; j>=0; j = MEM5LINK(j)->next, n++){} + fprintf(out, "freelist items of size %d: %d\n", mem5.szAtom << i, n); + } + fprintf(out, "mem5.nAlloc = %llu\n", mem5.nAlloc); + fprintf(out, "mem5.totalAlloc = %llu\n", mem5.totalAlloc); + fprintf(out, "mem5.totalExcess = %llu\n", mem5.totalExcess); + fprintf(out, "mem5.currentOut = %u\n", mem5.currentOut); + fprintf(out, "mem5.currentCount = %u\n", mem5.currentCount); + fprintf(out, "mem5.maxOut = %u\n", mem5.maxOut); + fprintf(out, "mem5.maxCount = %u\n", mem5.maxCount); + fprintf(out, "mem5.maxRequest = %u\n", mem5.maxRequest); + memsys5Leave(); + if( out==stdout ){ + fflush(stdout); + }else{ + fclose(out); + } +} +#endif + +/* +** This routine is the only routine in this file with external +** linkage. It returns a pointer to a static sqlite3_mem_methods +** struct populated with the memsys5 methods. +*/ +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ + static const sqlite3_mem_methods memsys5Methods = { + memsys5Malloc, + memsys5Free, + memsys5Realloc, + memsys5Size, + memsys5Roundup, + memsys5Init, + memsys5Shutdown, + 0 + }; + return &memsys5Methods; +} + +#endif /* SQLITE_ENABLE_MEMSYS5 */ + +/************** End of mem5.c ************************************************/ +/************** Begin file mutex.c *******************************************/ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes. +** +** This file contains code that is common across all mutex implementations. +*/ + +#if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT) +/* +** For debugging purposes, record when the mutex subsystem is initialized +** and uninitialized so that we can assert() if there is an attempt to +** allocate a mutex while the system is uninitialized. +*/ +static SQLITE_WSD int mutexIsInit = 0; +#endif /* SQLITE_DEBUG */ + + +#ifndef SQLITE_MUTEX_OMIT +/* +** Initialize the mutex system. +*/ +SQLITE_PRIVATE int sqlite3MutexInit(void){ + int rc = SQLITE_OK; + if( !sqlite3GlobalConfig.mutex.xMutexAlloc ){ + /* If the xMutexAlloc method has not been set, then the user did not + ** install a mutex implementation via sqlite3_config() prior to + ** sqlite3_initialize() being called. This block copies pointers to + ** the default implementation into the sqlite3GlobalConfig structure. + */ + sqlite3_mutex_methods const *pFrom; + sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; + + if( sqlite3GlobalConfig.bCoreMutex ){ + pFrom = sqlite3DefaultMutex(); + }else{ + pFrom = sqlite3NoopMutex(); + } + memcpy(pTo, pFrom, offsetof(sqlite3_mutex_methods, xMutexAlloc)); + memcpy(&pTo->xMutexFree, &pFrom->xMutexFree, + sizeof(*pTo) - offsetof(sqlite3_mutex_methods, xMutexFree)); + pTo->xMutexAlloc = pFrom->xMutexAlloc; + } + rc = sqlite3GlobalConfig.mutex.xMutexInit(); + +#ifdef SQLITE_DEBUG + GLOBAL(int, mutexIsInit) = 1; +#endif + + return rc; +} + +/* +** Shutdown the mutex system. This call frees resources allocated by +** sqlite3MutexInit(). +*/ +SQLITE_PRIVATE int sqlite3MutexEnd(void){ + int rc = SQLITE_OK; + if( sqlite3GlobalConfig.mutex.xMutexEnd ){ + rc = sqlite3GlobalConfig.mutex.xMutexEnd(); + } + +#ifdef SQLITE_DEBUG + GLOBAL(int, mutexIsInit) = 0; +#endif + + return rc; +} + +/* +** Retrieve a pointer to a static mutex or allocate a new dynamic one. +*/ +SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int id){ +#ifndef SQLITE_OMIT_AUTOINIT + if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; +#endif + return sqlite3GlobalConfig.mutex.xMutexAlloc(id); +} + +SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ + if( !sqlite3GlobalConfig.bCoreMutex ){ + return 0; + } + assert( GLOBAL(int, mutexIsInit) ); + return sqlite3GlobalConfig.mutex.xMutexAlloc(id); +} + +/* +** Free a dynamic mutex. +*/ +SQLITE_API void sqlite3_mutex_free(sqlite3_mutex *p){ + if( p ){ + sqlite3GlobalConfig.mutex.xMutexFree(p); + } +} + +/* +** Obtain the mutex p. If some other thread already has the mutex, block +** until it can be obtained. +*/ +SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex *p){ + if( p ){ + sqlite3GlobalConfig.mutex.xMutexEnter(p); + } +} + +/* +** Obtain the mutex p. If successful, return SQLITE_OK. Otherwise, if another +** thread holds the mutex and it cannot be obtained, return SQLITE_BUSY. +*/ +SQLITE_API int sqlite3_mutex_try(sqlite3_mutex *p){ + int rc = SQLITE_OK; + if( p ){ + return sqlite3GlobalConfig.mutex.xMutexTry(p); + } + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was previously +** entered by the same thread. The behavior is undefined if the mutex +** is not currently entered. If a NULL pointer is passed as an argument +** this function is a no-op. +*/ +SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ + if( p ){ + sqlite3GlobalConfig.mutex.xMutexLeave(p); + } +} + +#ifndef NDEBUG +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex *p){ + return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); +} +SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ + return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); +} +#endif + +#endif /* !defined(SQLITE_MUTEX_OMIT) */ + +/************** End of mutex.c ***********************************************/ +/************** Begin file mutex_noop.c **************************************/ +/* +** 2008 October 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes. +** +** This implementation in this file does not provide any mutual +** exclusion and is thus suitable for use only in applications +** that use SQLite in a single thread. The routines defined +** here are place-holders. Applications can substitute working +** mutex routines at start-time using the +** +** sqlite3_config(SQLITE_CONFIG_MUTEX,...) +** +** interface. +** +** If compiled with SQLITE_DEBUG, then additional logic is inserted +** that does error checking on mutexes to make sure they are being +** called correctly. +*/ + +#ifndef SQLITE_MUTEX_OMIT + +#ifndef SQLITE_DEBUG +/* +** Stub routines for all mutex methods. +** +** This routines provide no mutual exclusion or error checking. +*/ +static int noopMutexInit(void){ return SQLITE_OK; } +static int noopMutexEnd(void){ return SQLITE_OK; } +static sqlite3_mutex *noopMutexAlloc(int id){ + UNUSED_PARAMETER(id); + return (sqlite3_mutex*)8; +} +static void noopMutexFree(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } +static void noopMutexEnter(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } +static int noopMutexTry(sqlite3_mutex *p){ + UNUSED_PARAMETER(p); + return SQLITE_OK; +} +static void noopMutexLeave(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } + +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3NoopMutex(void){ + static const sqlite3_mutex_methods sMutex = { + noopMutexInit, + noopMutexEnd, + noopMutexAlloc, + noopMutexFree, + noopMutexEnter, + noopMutexTry, + noopMutexLeave, + + 0, + 0, + }; + + return &sMutex; +} +#endif /* !SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +** In this implementation, error checking is provided for testing +** and debugging purposes. The mutexes still do not provide any +** mutual exclusion. +*/ + +/* +** The mutex object +*/ +typedef struct sqlite3_debug_mutex { + int id; /* The mutex type */ + int cnt; /* Number of entries without a matching leave */ +} sqlite3_debug_mutex; + +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +static int debugMutexHeld(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + return p==0 || p->cnt>0; +} +static int debugMutexNotheld(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + return p==0 || p->cnt==0; +} + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int debugMutexInit(void){ return SQLITE_OK; } +static int debugMutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. +*/ +static sqlite3_mutex *debugMutexAlloc(int id){ + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1]; + sqlite3_debug_mutex *pNew = 0; + switch( id ){ + case SQLITE_MUTEX_FAST: + case SQLITE_MUTEX_RECURSIVE: { + pNew = sqlite3Malloc(sizeof(*pNew)); + if( pNew ){ + pNew->id = id; + pNew->cnt = 0; + } + break; + } + default: { + assert( id-2 >= 0 ); + assert( id-2 < (int)(sizeof(aStatic)/sizeof(aStatic[0])) ); + pNew = &aStatic[id-2]; + pNew->id = id; + break; + } + } + return (sqlite3_mutex*)pNew; +} + +/* +** This routine deallocates a previously allocated mutex. +*/ +static void debugMutexFree(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( p->cnt==0 ); + assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + sqlite3_free(p); +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void debugMutexEnter(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(pX) ); + p->cnt++; +} +static int debugMutexTry(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(pX) ); + p->cnt++; + return SQLITE_OK; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void debugMutexLeave(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( debugMutexHeld(pX) ); + p->cnt--; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(pX) ); +} + +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3NoopMutex(void){ + static const sqlite3_mutex_methods sMutex = { + debugMutexInit, + debugMutexEnd, + debugMutexAlloc, + debugMutexFree, + debugMutexEnter, + debugMutexTry, + debugMutexLeave, + + debugMutexHeld, + debugMutexNotheld + }; + + return &sMutex; +} +#endif /* SQLITE_DEBUG */ + +/* +** If compiled with SQLITE_MUTEX_NOOP, then the no-op mutex implementation +** is used regardless of the run-time threadsafety setting. +*/ +#ifdef SQLITE_MUTEX_NOOP +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ + return sqlite3NoopMutex(); +} +#endif /* defined(SQLITE_MUTEX_NOOP) */ +#endif /* !defined(SQLITE_MUTEX_OMIT) */ + +/************** End of mutex_noop.c ******************************************/ +/************** Begin file mutex_unix.c **************************************/ +/* +** 2007 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes for pthreads +*/ + +/* +** The code in this file is only used if we are compiling threadsafe +** under unix with pthreads. +** +** Note that this implementation requires a version of pthreads that +** supports recursive mutexes. +*/ +#ifdef SQLITE_MUTEX_PTHREADS + +#include + +/* +** The sqlite3_mutex.id, sqlite3_mutex.nRef, and sqlite3_mutex.owner fields +** are necessary under two condidtions: (1) Debug builds and (2) using +** home-grown mutexes. Encapsulate these conditions into a single #define. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_HOMEGROWN_RECURSIVE_MUTEX) +# define SQLITE_MUTEX_NREF 1 +#else +# define SQLITE_MUTEX_NREF 0 +#endif + +/* +** Each recursive mutex is an instance of the following structure. +*/ +struct sqlite3_mutex { + pthread_mutex_t mutex; /* Mutex controlling the lock */ +#if SQLITE_MUTEX_NREF + int id; /* Mutex type */ + volatile int nRef; /* Number of entrances */ + volatile pthread_t owner; /* Thread that is within this mutex */ + int trace; /* True to trace changes */ +#endif +}; +#if SQLITE_MUTEX_NREF +#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0, 0, (pthread_t)0, 0 } +#else +#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER } +#endif + +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use only inside assert() statements. On some platforms, +** there might be race conditions that can cause these routines to +** deliver incorrect results. In particular, if pthread_equal() is +** not an atomic operation, then these routines might delivery +** incorrect results. On most platforms, pthread_equal() is a +** comparison of two integers and is therefore atomic. But we are +** told that HPUX is not such a platform. If so, then these routines +** will not always work correctly on HPUX. +** +** On those platforms where pthread_equal() is not atomic, SQLite +** should be compiled without -DSQLITE_DEBUG and with -DNDEBUG to +** make sure no assert() statements are evaluated and hence these +** routines are never called. +*/ +#if !defined(NDEBUG) || defined(SQLITE_DEBUG) +static int pthreadMutexHeld(sqlite3_mutex *p){ + return (p->nRef!=0 && pthread_equal(p->owner, pthread_self())); +} +static int pthreadMutexNotheld(sqlite3_mutex *p){ + return p->nRef==0 || pthread_equal(p->owner, pthread_self())==0; +} +#endif + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int pthreadMutexInit(void){ return SQLITE_OK; } +static int pthreadMutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. SQLite +** will unwind its stack and return an error. The argument +** to sqlite3_mutex_alloc() is one of these integer constants: +** +**
    +**
  • SQLITE_MUTEX_FAST +**
  • SQLITE_MUTEX_RECURSIVE +**
  • SQLITE_MUTEX_STATIC_MASTER +**
  • SQLITE_MUTEX_STATIC_MEM +**
  • SQLITE_MUTEX_STATIC_OPEN +**
  • SQLITE_MUTEX_STATIC_PRNG +**
  • SQLITE_MUTEX_STATIC_LRU +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 +**
+** +** The first two constants cause sqlite3_mutex_alloc() to create +** a new mutex. The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. But SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** The other allowed parameters to sqlite3_mutex_alloc() each return +** a pointer to a static preexisting mutex. Six static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +*/ +static sqlite3_mutex *pthreadMutexAlloc(int iType){ + static sqlite3_mutex staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER + }; + sqlite3_mutex *p; + switch( iType ){ + case SQLITE_MUTEX_RECURSIVE: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, we will have to + ** build our own. See below. */ + pthread_mutex_init(&p->mutex, 0); +#else + /* Use a recursive mutex if it is available */ + pthread_mutexattr_t recursiveAttr; + pthread_mutexattr_init(&recursiveAttr); + pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&p->mutex, &recursiveAttr); + pthread_mutexattr_destroy(&recursiveAttr); +#endif +#if SQLITE_MUTEX_NREF + p->id = iType; +#endif + } + break; + } + case SQLITE_MUTEX_FAST: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ +#if SQLITE_MUTEX_NREF + p->id = iType; +#endif + pthread_mutex_init(&p->mutex, 0); + } + break; + } + default: { + assert( iType-2 >= 0 ); + assert( iType-2 < ArraySize(staticMutexes) ); + p = &staticMutexes[iType-2]; +#if SQLITE_MUTEX_NREF + p->id = iType; +#endif + break; + } + } + return p; +} + + +/* +** This routine deallocates a previously +** allocated mutex. SQLite is careful to deallocate every +** mutex that it allocates. +*/ +static void pthreadMutexFree(sqlite3_mutex *p){ + assert( p->nRef==0 ); + assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + pthread_mutex_destroy(&p->mutex); + sqlite3_free(p); +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void pthreadMutexEnter(sqlite3_mutex *p){ + assert( p->id==SQLITE_MUTEX_RECURSIVE || pthreadMutexNotheld(p) ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, then we have to grow + ** our own. This implementation assumes that pthread_equal() + ** is atomic - that it cannot be deceived into thinking self + ** and p->owner are equal if p->owner changes between two values + ** that are not equal to self while the comparison is taking place. + ** This implementation also assumes a coherent cache - that + ** separate processes cannot read different values from the same + ** address at the same time. If either of these two conditions + ** are not met, then the mutexes will fail and problems will result. + */ + { + pthread_t self = pthread_self(); + if( p->nRef>0 && pthread_equal(p->owner, self) ){ + p->nRef++; + }else{ + pthread_mutex_lock(&p->mutex); + assert( p->nRef==0 ); + p->owner = self; + p->nRef = 1; + } + } +#else + /* Use the built-in recursive mutexes if they are available. + */ + pthread_mutex_lock(&p->mutex); +#if SQLITE_MUTEX_NREF + assert( p->nRef>0 || p->owner==0 ); + p->owner = pthread_self(); + p->nRef++; +#endif +#endif + +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif +} +static int pthreadMutexTry(sqlite3_mutex *p){ + int rc; + assert( p->id==SQLITE_MUTEX_RECURSIVE || pthreadMutexNotheld(p) ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, then we have to grow + ** our own. This implementation assumes that pthread_equal() + ** is atomic - that it cannot be deceived into thinking self + ** and p->owner are equal if p->owner changes between two values + ** that are not equal to self while the comparison is taking place. + ** This implementation also assumes a coherent cache - that + ** separate processes cannot read different values from the same + ** address at the same time. If either of these two conditions + ** are not met, then the mutexes will fail and problems will result. + */ + { + pthread_t self = pthread_self(); + if( p->nRef>0 && pthread_equal(p->owner, self) ){ + p->nRef++; + rc = SQLITE_OK; + }else if( pthread_mutex_trylock(&p->mutex)==0 ){ + assert( p->nRef==0 ); + p->owner = self; + p->nRef = 1; + rc = SQLITE_OK; + }else{ + rc = SQLITE_BUSY; + } + } +#else + /* Use the built-in recursive mutexes if they are available. + */ + if( pthread_mutex_trylock(&p->mutex)==0 ){ +#if SQLITE_MUTEX_NREF + p->owner = pthread_self(); + p->nRef++; +#endif + rc = SQLITE_OK; + }else{ + rc = SQLITE_BUSY; + } +#endif + +#ifdef SQLITE_DEBUG + if( rc==SQLITE_OK && p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void pthreadMutexLeave(sqlite3_mutex *p){ + assert( pthreadMutexHeld(p) ); +#if SQLITE_MUTEX_NREF + p->nRef--; + if( p->nRef==0 ) p->owner = 0; +#endif + assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + if( p->nRef==0 ){ + pthread_mutex_unlock(&p->mutex); + } +#else + pthread_mutex_unlock(&p->mutex); +#endif + +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("leave mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif +} + +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ + static const sqlite3_mutex_methods sMutex = { + pthreadMutexInit, + pthreadMutexEnd, + pthreadMutexAlloc, + pthreadMutexFree, + pthreadMutexEnter, + pthreadMutexTry, + pthreadMutexLeave, +#ifdef SQLITE_DEBUG + pthreadMutexHeld, + pthreadMutexNotheld +#else + 0, + 0 +#endif + }; + + return &sMutex; +} + +#endif /* SQLITE_MUTEX_PTHREADS */ + +/************** End of mutex_unix.c ******************************************/ +/************** Begin file mutex_w32.c ***************************************/ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes for Win32. +*/ + +#if SQLITE_OS_WIN +/* +** Include code that is common to all os_*.c files +*/ +/************** Include os_common.h in the middle of mutex_w32.c *************/ +/************** Begin file os_common.h ***************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains macros and a little bit of code that is common to +** all of the platform-specific files (os_*.c) and is #included into those +** files. +** +** This file should be #included by the os_*.c files only. It is not a +** general purpose header file. +*/ +#ifndef _OS_COMMON_H_ +#define _OS_COMMON_H_ + +/* +** At least two bugs have slipped in because we changed the MEMORY_DEBUG +** macro to SQLITE_DEBUG and some older makefiles have not yet made the +** switch. The following code should catch this problem at compile-time. +*/ +#ifdef MEMORY_DEBUG +# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." +#endif + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X +#else +# define OSTRACE(X) +#endif + +/* +** Macros for performance tracing. Normally turned off. Only works +** on i486 hardware. +*/ +#ifdef SQLITE_PERFORMANCE_TRACE + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +/************** Include hwtime.h in the middle of os_common.h ****************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 class CPUs. +*/ +#ifndef _HWTIME_H_ +#define _HWTIME_H_ + +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; + } + +#elif (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + #error Need implementation of sqlite3Hwtime() for your platform. + + /* + ** To compile without implementing sqlite3Hwtime() for your platform, + ** you can remove the above #error and use the following + ** stub function. You will lose timing support for many + ** of the debugging and testing utilities, but it should at + ** least compile and run. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(_HWTIME_H_) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in os_common.h ******************/ + +static sqlite_uint64 g_start; +static sqlite_uint64 g_elapsed; +#define TIMER_START g_start=sqlite3Hwtime() +#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start +#define TIMER_ELAPSED g_elapsed +#else +#define TIMER_START +#define TIMER_END +#define TIMER_ELAPSED ((sqlite_uint64)0) +#endif + +/* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ +SQLITE_API int sqlite3_diskfull_pending = 0; +SQLITE_API int sqlite3_diskfull = 0; +#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) +#define SimulateIOError(CODE) \ + if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ + || sqlite3_io_error_pending-- == 1 ) \ + { local_ioerr(); CODE; } +static void local_ioerr(){ + IOTRACE(("IOERR\n")); + sqlite3_io_error_hit++; + if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; +} +#define SimulateDiskfullError(CODE) \ + if( sqlite3_diskfull_pending ){ \ + if( sqlite3_diskfull_pending == 1 ){ \ + local_ioerr(); \ + sqlite3_diskfull = 1; \ + sqlite3_io_error_hit = 1; \ + CODE; \ + }else{ \ + sqlite3_diskfull_pending--; \ + } \ + } +#else +#define SimulateIOErrorBenign(X) +#define SimulateIOError(A) +#define SimulateDiskfullError(A) +#endif + +/* +** When testing, keep a count of the number of open files. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_open_file_count = 0; +#define OpenCounter(X) sqlite3_open_file_count+=(X) +#else +#define OpenCounter(X) +#endif + +#endif /* !defined(_OS_COMMON_H_) */ + +/************** End of os_common.h *******************************************/ +/************** Continuing where we left off in mutex_w32.c ******************/ + +/* +** Include the header file for the Windows VFS. +*/ +/************** Include os_win.h in the middle of mutex_w32.c ****************/ +/************** Begin file os_win.h ******************************************/ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Windows. +*/ +#ifndef _OS_WIN_H_ +#define _OS_WIN_H_ + +/* +** Include the primary Windows SDK header file. +*/ +/* #include "windows.h" */ + +#ifdef __CYGWIN__ +# include +# include /* amalgamator: dontcache */ +#endif + +/* +** Determine if we are dealing with Windows NT. +** +** We ought to be able to determine if we are compiling for Windows 9x or +** Windows NT using the _WIN32_WINNT macro as follows: +** +** #if defined(_WIN32_WINNT) +** # define SQLITE_OS_WINNT 1 +** #else +** # define SQLITE_OS_WINNT 0 +** #endif +** +** However, Visual Studio 2005 does not set _WIN32_WINNT by default, as +** it ought to, so the above test does not work. We'll just assume that +** everything is Windows NT unless the programmer explicitly says otherwise +** by setting SQLITE_OS_WINNT to 0. +*/ +#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) +# define SQLITE_OS_WINNT 1 +#endif + +/* +** Determine if we are dealing with Windows CE - which has a much reduced +** API. +*/ +#if defined(_WIN32_WCE) +# define SQLITE_OS_WINCE 1 +#else +# define SQLITE_OS_WINCE 0 +#endif + +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + +#endif /* _OS_WIN_H_ */ + +/************** End of os_win.h **********************************************/ +/************** Continuing where we left off in mutex_w32.c ******************/ +#endif + +/* +** The code in this file is only used if we are compiling multithreaded +** on a Win32 system. +*/ +#ifdef SQLITE_MUTEX_W32 + +/* +** Each recursive mutex is an instance of the following structure. +*/ +struct sqlite3_mutex { + CRITICAL_SECTION mutex; /* Mutex controlling the lock */ + int id; /* Mutex type */ +#ifdef SQLITE_DEBUG + volatile int nRef; /* Number of enterances */ + volatile DWORD owner; /* Thread holding this mutex */ + volatile int trace; /* True to trace changes */ +#endif +}; + +/* +** These are the initializer values used when declaring a "static" mutex +** on Win32. It should be noted that all mutexes require initialization +** on the Win32 platform. +*/ +#define SQLITE_W32_MUTEX_INITIALIZER { 0 } + +#ifdef SQLITE_DEBUG +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \ + 0L, (DWORD)0, 0 } +#else +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } +#endif + +#ifdef SQLITE_DEBUG +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use only inside assert() statements. +*/ +static int winMutexHeld(sqlite3_mutex *p){ + return p->nRef!=0 && p->owner==GetCurrentThreadId(); +} + +static int winMutexNotheld2(sqlite3_mutex *p, DWORD tid){ + return p->nRef==0 || p->owner!=tid; +} + +static int winMutexNotheld(sqlite3_mutex *p){ + DWORD tid = GetCurrentThreadId(); + return winMutexNotheld2(p, tid); +} +#endif + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static sqlite3_mutex winMutex_staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER +}; + +static int winMutex_isInit = 0; +static int winMutex_isNt = -1; /* <0 means "need to query" */ + +/* As the winMutexInit() and winMutexEnd() functions are called as part +** of the sqlite3_initialize() and sqlite3_shutdown() processing, the +** "interlocked" magic used here is probably not strictly necessary. +*/ +static LONG volatile winMutex_lock = 0; + +SQLITE_API int sqlite3_win32_is_nt(void); /* os_win.c */ +SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ + +static int winMutexInit(void){ + /* The first to increment to 1 does actual initialization */ + if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ + int i; + for(i=0; i +**
  • SQLITE_MUTEX_FAST +**
  • SQLITE_MUTEX_RECURSIVE +**
  • SQLITE_MUTEX_STATIC_MASTER +**
  • SQLITE_MUTEX_STATIC_MEM +**
  • SQLITE_MUTEX_STATIC_OPEN +**
  • SQLITE_MUTEX_STATIC_PRNG +**
  • SQLITE_MUTEX_STATIC_LRU +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 +** +** +** The first two constants cause sqlite3_mutex_alloc() to create +** a new mutex. The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. But SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** The other allowed parameters to sqlite3_mutex_alloc() each return +** a pointer to a static preexisting mutex. Six static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +*/ +static sqlite3_mutex *winMutexAlloc(int iType){ + sqlite3_mutex *p; + + switch( iType ){ + case SQLITE_MUTEX_FAST: + case SQLITE_MUTEX_RECURSIVE: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ +#ifdef SQLITE_DEBUG + p->id = iType; +#ifdef SQLITE_WIN32_MUTEX_TRACE_DYNAMIC + p->trace = 1; +#endif +#endif +#if SQLITE_OS_WINRT + InitializeCriticalSectionEx(&p->mutex, 0, 0); +#else + InitializeCriticalSection(&p->mutex); +#endif + } + break; + } + default: { + assert( iType-2 >= 0 ); + assert( iType-2 < ArraySize(winMutex_staticMutexes) ); + assert( winMutex_isInit==1 ); + p = &winMutex_staticMutexes[iType-2]; +#ifdef SQLITE_DEBUG + p->id = iType; +#ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC + p->trace = 1; +#endif +#endif + break; + } + } + return p; +} + + +/* +** This routine deallocates a previously +** allocated mutex. SQLite is careful to deallocate every +** mutex that it allocates. +*/ +static void winMutexFree(sqlite3_mutex *p){ + assert( p ); +#ifdef SQLITE_DEBUG + assert( p->nRef==0 && p->owner==0 ); + assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); +#endif + assert( winMutex_isInit==1 ); + DeleteCriticalSection(&p->mutex); + sqlite3_free(p); +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void winMutexEnter(sqlite3_mutex *p){ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); +#endif +#ifdef SQLITE_DEBUG + assert( p ); + assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); +#else + assert( p ); +#endif + assert( winMutex_isInit==1 ); + EnterCriticalSection(&p->mutex); +#ifdef SQLITE_DEBUG + assert( p->nRef>0 || p->owner==0 ); + p->owner = tid; + p->nRef++; + if( p->trace ){ + OSTRACE(("ENTER-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", + tid, p, p->trace, p->nRef)); + } +#endif +} + +static int winMutexTry(sqlite3_mutex *p){ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); +#endif + int rc = SQLITE_BUSY; + assert( p ); + assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); + /* + ** The sqlite3_mutex_try() routine is very rarely used, and when it + ** is used it is merely an optimization. So it is OK for it to always + ** fail. + ** + ** The TryEnterCriticalSection() interface is only available on WinNT. + ** And some windows compilers complain if you try to use it without + ** first doing some #defines that prevent SQLite from building on Win98. + ** For that reason, we will omit this optimization for now. See + ** ticket #2685. + */ +#if defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0400 + assert( winMutex_isInit==1 ); + assert( winMutex_isNt>=-1 && winMutex_isNt<=1 ); + if( winMutex_isNt<0 ){ + winMutex_isNt = sqlite3_win32_is_nt(); + } + assert( winMutex_isNt==0 || winMutex_isNt==1 ); + if( winMutex_isNt && TryEnterCriticalSection(&p->mutex) ){ +#ifdef SQLITE_DEBUG + p->owner = tid; + p->nRef++; +#endif + rc = SQLITE_OK; + } +#else + UNUSED_PARAMETER(p); +#endif +#ifdef SQLITE_DEBUG + if( p->trace ){ + OSTRACE(("TRY-MUTEX tid=%lu, mutex=%p (%d), owner=%lu, nRef=%d, rc=%s\n", + tid, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); + } +#endif + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void winMutexLeave(sqlite3_mutex *p){ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); +#endif + assert( p ); +#ifdef SQLITE_DEBUG + assert( p->nRef>0 ); + assert( p->owner==tid ); + p->nRef--; + if( p->nRef==0 ) p->owner = 0; + assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); +#endif + assert( winMutex_isInit==1 ); + LeaveCriticalSection(&p->mutex); +#ifdef SQLITE_DEBUG + if( p->trace ){ + OSTRACE(("LEAVE-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", + tid, p, p->trace, p->nRef)); + } +#endif +} + +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ + static const sqlite3_mutex_methods sMutex = { + winMutexInit, + winMutexEnd, + winMutexAlloc, + winMutexFree, + winMutexEnter, + winMutexTry, + winMutexLeave, +#ifdef SQLITE_DEBUG + winMutexHeld, + winMutexNotheld +#else + 0, + 0 +#endif + }; + return &sMutex; +} + +#endif /* SQLITE_MUTEX_W32 */ + +/************** End of mutex_w32.c *******************************************/ +/************** Begin file malloc.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Memory allocation functions used throughout sqlite. +*/ +/* #include */ + +/* +** Attempt to release up to n bytes of non-essential memory currently +** held by SQLite. An example of non-essential memory is memory used to +** cache database pages that are not currently in use. +*/ +SQLITE_API int sqlite3_release_memory(int n){ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + return sqlite3PcacheReleaseMemory(n); +#else + /* IMPLEMENTATION-OF: R-34391-24921 The sqlite3_release_memory() routine + ** is a no-op returning zero if SQLite is not compiled with + ** SQLITE_ENABLE_MEMORY_MANAGEMENT. */ + UNUSED_PARAMETER(n); + return 0; +#endif +} + +/* +** An instance of the following object records the location of +** each unused scratch buffer. +*/ +typedef struct ScratchFreeslot { + struct ScratchFreeslot *pNext; /* Next unused scratch buffer */ +} ScratchFreeslot; + +/* +** State information local to the memory allocation subsystem. +*/ +static SQLITE_WSD struct Mem0Global { + sqlite3_mutex *mutex; /* Mutex to serialize access */ + + /* + ** The alarm callback and its arguments. The mem0.mutex lock will + ** be held while the callback is running. Recursive calls into + ** the memory subsystem are allowed, but no new callbacks will be + ** issued. + */ + sqlite3_int64 alarmThreshold; + void (*alarmCallback)(void*, sqlite3_int64,int); + void *alarmArg; + + /* + ** Pointers to the end of sqlite3GlobalConfig.pScratch memory + ** (so that a range test can be used to determine if an allocation + ** being freed came from pScratch) and a pointer to the list of + ** unused scratch allocations. + */ + void *pScratchEnd; + ScratchFreeslot *pScratchFree; + u32 nScratchFree; + + /* + ** True if heap is nearly "full" where "full" is defined by the + ** sqlite3_soft_heap_limit() setting. + */ + int nearlyFull; +} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; + +#define mem0 GLOBAL(struct Mem0Global, mem0) + +/* +** This routine runs when the memory allocator sees that the +** total memory allocation is about to exceed the soft heap +** limit. +*/ +static void softHeapLimitEnforcer( + void *NotUsed, + sqlite3_int64 NotUsed2, + int allocSize +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + sqlite3_release_memory(allocSize); +} + +/* +** Change the alarm callback +*/ +static int sqlite3MemoryAlarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + int nUsed; + sqlite3_mutex_enter(mem0.mutex); + mem0.alarmCallback = xCallback; + mem0.alarmArg = pArg; + mem0.alarmThreshold = iThreshold; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed); + sqlite3_mutex_leave(mem0.mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Deprecated external interface. Internal/core SQLite code +** should call sqlite3MemoryAlarm. +*/ +SQLITE_API int sqlite3_memory_alarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + return sqlite3MemoryAlarm(xCallback, pArg, iThreshold); +} +#endif + +/* +** Set the soft heap-size limit for the library. Passing a zero or +** negative value indicates no limit. +*/ +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; + sqlite3_int64 excess; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return -1; +#endif + sqlite3_mutex_enter(mem0.mutex); + priorLimit = mem0.alarmThreshold; + sqlite3_mutex_leave(mem0.mutex); + if( n<0 ) return priorLimit; + if( n>0 ){ + sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, n); + }else{ + sqlite3MemoryAlarm(0, 0, 0); + } + excess = sqlite3_memory_used() - n; + if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); + return priorLimit; +} +SQLITE_API void sqlite3_soft_heap_limit(int n){ + if( n<0 ) n = 0; + sqlite3_soft_heap_limit64(n); +} + +/* +** Initialize the memory allocation subsystem. +*/ +SQLITE_PRIVATE int sqlite3MallocInit(void){ + if( sqlite3GlobalConfig.m.xMalloc==0 ){ + sqlite3MemSetDefault(); + } + memset(&mem0, 0, sizeof(mem0)); + if( sqlite3GlobalConfig.bCoreMutex ){ + mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 + && sqlite3GlobalConfig.nScratch>0 ){ + int i, n, sz; + ScratchFreeslot *pSlot; + sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch); + sqlite3GlobalConfig.szScratch = sz; + pSlot = (ScratchFreeslot*)sqlite3GlobalConfig.pScratch; + n = sqlite3GlobalConfig.nScratch; + mem0.pScratchFree = pSlot; + mem0.nScratchFree = n; + for(i=0; ipNext = (ScratchFreeslot*)(sz+(char*)pSlot); + pSlot = pSlot->pNext; + } + pSlot->pNext = 0; + mem0.pScratchEnd = (void*)&pSlot[1]; + }else{ + mem0.pScratchEnd = 0; + sqlite3GlobalConfig.pScratch = 0; + sqlite3GlobalConfig.szScratch = 0; + sqlite3GlobalConfig.nScratch = 0; + } + if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512 + || sqlite3GlobalConfig.nPage<1 ){ + sqlite3GlobalConfig.pPage = 0; + sqlite3GlobalConfig.szPage = 0; + sqlite3GlobalConfig.nPage = 0; + } + return sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData); +} + +/* +** Return true if the heap is currently under memory pressure - in other +** words if the amount of heap used is close to the limit set by +** sqlite3_soft_heap_limit(). +*/ +SQLITE_PRIVATE int sqlite3HeapNearlyFull(void){ + return mem0.nearlyFull; +} + +/* +** Deinitialize the memory allocation subsystem. +*/ +SQLITE_PRIVATE void sqlite3MallocEnd(void){ + if( sqlite3GlobalConfig.m.xShutdown ){ + sqlite3GlobalConfig.m.xShutdown(sqlite3GlobalConfig.m.pAppData); + } + memset(&mem0, 0, sizeof(mem0)); +} + +/* +** Return the amount of memory currently checked out. +*/ +SQLITE_API sqlite3_int64 sqlite3_memory_used(void){ + int n, mx; + sqlite3_int64 res; + sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, 0); + res = (sqlite3_int64)n; /* Work around bug in Borland C. Ticket #3216 */ + return res; +} + +/* +** Return the maximum amount of memory that has ever been +** checked out since either the beginning of this process +** or since the most recent reset. +*/ +SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ + int n, mx; + sqlite3_int64 res; + sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, resetFlag); + res = (sqlite3_int64)mx; /* Work around bug in Borland C. Ticket #3216 */ + return res; +} + +/* +** Trigger the alarm +*/ +static void sqlite3MallocAlarm(int nByte){ + void (*xCallback)(void*,sqlite3_int64,int); + sqlite3_int64 nowUsed; + void *pArg; + if( mem0.alarmCallback==0 ) return; + xCallback = mem0.alarmCallback; + nowUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + pArg = mem0.alarmArg; + mem0.alarmCallback = 0; + sqlite3_mutex_leave(mem0.mutex); + xCallback(pArg, nowUsed, nByte); + sqlite3_mutex_enter(mem0.mutex); + mem0.alarmCallback = xCallback; + mem0.alarmArg = pArg; +} + +/* +** Do a memory allocation with statistics and alarms. Assume the +** lock is already held. +*/ +static int mallocWithAlarm(int n, void **pp){ + int nFull; + void *p; + assert( sqlite3_mutex_held(mem0.mutex) ); + nFull = sqlite3GlobalConfig.m.xRoundup(n); + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); + if( mem0.alarmCallback!=0 ){ + int nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.alarmThreshold - nFull ){ + mem0.nearlyFull = 1; + sqlite3MallocAlarm(nFull); + }else{ + mem0.nearlyFull = 0; + } + } + p = sqlite3GlobalConfig.m.xMalloc(nFull); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + if( p==0 && mem0.alarmCallback ){ + sqlite3MallocAlarm(nFull); + p = sqlite3GlobalConfig.m.xMalloc(nFull); + } +#endif + if( p ){ + nFull = sqlite3MallocSize(p); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, 1); + } + *pp = p; + return nFull; +} + +/* +** Allocate memory. This routine is like sqlite3_malloc() except that it +** assumes the memory subsystem has already been initialized. +*/ +SQLITE_PRIVATE void *sqlite3Malloc(int n){ + void *p; + if( n<=0 /* IMP: R-65312-04917 */ + || n>=0x7fffff00 + ){ + /* A memory allocation of a number of bytes which is near the maximum + ** signed integer value might cause an integer overflow inside of the + ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving + ** 255 bytes of overhead. SQLite itself will never use anything near + ** this amount. The only way to reach the limit is with sqlite3_malloc() */ + p = 0; + }else if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + mallocWithAlarm(n, &p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + p = sqlite3GlobalConfig.m.xMalloc(n); + } + assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-04675-44850 */ + return p; +} + +/* +** This version of the memory allocation is for use by the application. +** First make sure the memory subsystem is initialized, then do the +** allocation. +*/ +SQLITE_API void *sqlite3_malloc(int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return sqlite3Malloc(n); +} + +/* +** Each thread may only have a single outstanding allocation from +** xScratchMalloc(). We verify this constraint in the single-threaded +** case by setting scratchAllocOut to 1 when an allocation +** is outstanding clearing it when the allocation is freed. +*/ +#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) +static int scratchAllocOut = 0; +#endif + + +/* +** Allocate memory that is to be used and released right away. +** This routine is similar to alloca() in that it is not intended +** for situations where the memory might be held long-term. This +** routine is intended to get memory to old large transient data +** structures that would not normally fit on the stack of an +** embedded processor. +*/ +SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ + void *p; + assert( n>0 ); + + sqlite3_mutex_enter(mem0.mutex); + if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ + p = mem0.pScratchFree; + mem0.pScratchFree = mem0.pScratchFree->pNext; + mem0.nScratchFree--; + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1); + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + sqlite3_mutex_leave(mem0.mutex); + }else{ + if( sqlite3GlobalConfig.bMemstat ){ + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + n = mallocWithAlarm(n, &p); + if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3_mutex_leave(mem0.mutex); + p = sqlite3GlobalConfig.m.xMalloc(n); + } + sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); + } + assert( sqlite3_mutex_notheld(mem0.mutex) ); + + +#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) + /* Verify that no more than two scratch allocations per thread + ** are outstanding at one time. (This is only checked in the + ** single-threaded case since checking in the multi-threaded case + ** would be much more complicated.) */ + assert( scratchAllocOut<=1 ); + if( p ) scratchAllocOut++; +#endif + + return p; +} +SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ + if( p ){ + +#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) + /* Verify that no more than two scratch allocation per thread + ** is outstanding at one time. (This is only checked in the + ** single-threaded case since checking in the multi-threaded case + ** would be much more complicated.) */ + assert( scratchAllocOut>=1 && scratchAllocOut<=2 ); + scratchAllocOut--; +#endif + + if( p>=sqlite3GlobalConfig.pScratch && ppNext = mem0.pScratchFree; + mem0.pScratchFree = pSlot; + mem0.nScratchFree++; + assert( mem0.nScratchFree <= (u32)sqlite3GlobalConfig.nScratch ); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, -1); + sqlite3_mutex_leave(mem0.mutex); + }else{ + /* Release memory back to the heap */ + assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_SCRATCH) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + if( sqlite3GlobalConfig.bMemstat ){ + int iSize = sqlite3MallocSize(p); + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); + sqlite3GlobalConfig.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3GlobalConfig.m.xFree(p); + } + } + } +} + +/* +** TRUE if p is a lookaside memory allocation from db +*/ +#ifndef SQLITE_OMIT_LOOKASIDE +static int isLookaside(sqlite3 *db, void *p){ + return p>=db->lookaside.pStart && plookaside.pEnd; +} +#else +#define isLookaside(A,B) 0 +#endif + +/* +** Return the size of a memory allocation previously obtained from +** sqlite3Malloc() or sqlite3_malloc(). +*/ +SQLITE_PRIVATE int sqlite3MallocSize(void *p){ + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); + return sqlite3GlobalConfig.m.xSize(p); +} +SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + if( isLookaside(db, p) ){ + return db->lookaside.sz; + }else{ + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + return sqlite3GlobalConfig.m.xSize(p); + } +} + +/* +** Free memory previously obtained from sqlite3Malloc(). +*/ +SQLITE_API void sqlite3_free(void *p){ + if( p==0 ) return; /* IMP: R-49053-54554 */ + assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p)); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); + sqlite3GlobalConfig.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3GlobalConfig.m.xFree(p); + } +} + +/* +** Free memory that might be associated with a particular database +** connection. +*/ +SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ + assert( db==0 || sqlite3_mutex_held(db->mutex) ); + if( p==0 ) return; + if( db ){ + if( db->pnBytesFreed ){ + *db->pnBytesFreed += sqlite3DbMallocSize(db, p); + return; + } + if( isLookaside(db, p) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; +#if SQLITE_DEBUG + /* Trash all content in the buffer being freed */ + memset(p, 0xaa, db->lookaside.sz); +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + db->lookaside.nOut--; + return; + } + } + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); +} + +/* +** Change the size of an existing memory allocation +*/ +SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ + int nOld, nNew, nDiff; + void *pNew; + if( pOld==0 ){ + return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ + } + if( nBytes<=0 ){ + sqlite3_free(pOld); /* IMP: R-31593-10574 */ + return 0; + } + if( nBytes>=0x7fffff00 ){ + /* The 0x7ffff00 limit term is explained in comments on sqlite3Malloc() */ + return 0; + } + nOld = sqlite3MallocSize(pOld); + /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second + ** argument to xRealloc is always a value returned by a prior call to + ** xRoundup. */ + nNew = sqlite3GlobalConfig.m.xRoundup(nBytes); + if( nOld==nNew ){ + pNew = pOld; + }else if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, nBytes); + nDiff = nNew - nOld; + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= + mem0.alarmThreshold-nDiff ){ + sqlite3MallocAlarm(nDiff); + } + assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + if( pNew==0 && mem0.alarmCallback ){ + sqlite3MallocAlarm(nBytes); + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + } + if( pNew ){ + nNew = sqlite3MallocSize(pNew); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nNew-nOld); + } + sqlite3_mutex_leave(mem0.mutex); + }else{ + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + } + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-04675-44850 */ + return pNew; +} + +/* +** The public interface to sqlite3Realloc. Make sure that the memory +** subsystem is initialized prior to invoking sqliteRealloc. +*/ +SQLITE_API void *sqlite3_realloc(void *pOld, int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return sqlite3Realloc(pOld, n); +} + + +/* +** Allocate and zero memory. +*/ +SQLITE_PRIVATE void *sqlite3MallocZero(int n){ + void *p = sqlite3Malloc(n); + if( p ){ + memset(p, 0, n); + } + return p; +} + +/* +** Allocate and zero memory. If the allocation fails, make +** the mallocFailed flag in the connection pointer. +*/ +SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3 *db, int n){ + void *p = sqlite3DbMallocRaw(db, n); + if( p ){ + memset(p, 0, n); + } + return p; +} + +/* +** Allocate and zero memory. If the allocation fails, make +** the mallocFailed flag in the connection pointer. +** +** If db!=0 and db->mallocFailed is true (indicating a prior malloc +** failure on the same database connection) then always return 0. +** Hence for a particular database connection, once malloc starts +** failing, it fails consistently until mallocFailed is reset. +** This is an important assumption. There are many places in the +** code that do things like this: +** +** int *a = (int*)sqlite3DbMallocRaw(db, 100); +** int *b = (int*)sqlite3DbMallocRaw(db, 200); +** if( b ) a[10] = 9; +** +** In other words, if a subsequent malloc (ex: "b") worked, it is assumed +** that all prior mallocs (ex: "a") worked too. +*/ +SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, int n){ + void *p; + assert( db==0 || sqlite3_mutex_held(db->mutex) ); + assert( db==0 || db->pnBytesFreed==0 ); +#ifndef SQLITE_OMIT_LOOKASIDE + if( db ){ + LookasideSlot *pBuf; + if( db->mallocFailed ){ + return 0; + } + if( db->lookaside.bEnabled ){ + if( n>db->lookaside.sz ){ + db->lookaside.anStat[1]++; + }else if( (pBuf = db->lookaside.pFree)==0 ){ + db->lookaside.anStat[2]++; + }else{ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.nOut++; + db->lookaside.anStat[0]++; + if( db->lookaside.nOut>db->lookaside.mxOut ){ + db->lookaside.mxOut = db->lookaside.nOut; + } + return (void*)pBuf; + } + } + } +#else + if( db && db->mallocFailed ){ + return 0; + } +#endif + p = sqlite3Malloc(n); + if( !p && db ){ + db->mallocFailed = 1; + } + sqlite3MemdebugSetType(p, MEMTYPE_DB | + ((db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + return p; +} + +/* +** Resize the block of memory pointed to by p to n bytes. If the +** resize fails, set the mallocFailed flag in the connection object. +*/ +SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ + void *pNew = 0; + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + if( db->mallocFailed==0 ){ + if( p==0 ){ + return sqlite3DbMallocRaw(db, n); + } + if( isLookaside(db, p) ){ + if( n<=db->lookaside.sz ){ + return p; + } + pNew = sqlite3DbMallocRaw(db, n); + if( pNew ){ + memcpy(pNew, p, db->lookaside.sz); + sqlite3DbFree(db, p); + } + }else{ + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + pNew = sqlite3_realloc(p, n); + if( !pNew ){ + sqlite3MemdebugSetType(p, MEMTYPE_DB|MEMTYPE_HEAP); + db->mallocFailed = 1; + } + sqlite3MemdebugSetType(pNew, MEMTYPE_DB | + (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + } + } + return pNew; +} + +/* +** Attempt to reallocate p. If the reallocation fails, then free p +** and set the mallocFailed flag in the database connection. +*/ +SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, int n){ + void *pNew; + pNew = sqlite3DbRealloc(db, p, n); + if( !pNew ){ + sqlite3DbFree(db, p); + } + return pNew; +} + +/* +** Make a copy of a string in memory obtained from sqliteMalloc(). These +** functions call sqlite3MallocRaw() directly instead of sqliteMalloc(). This +** is because when memory debugging is turned on, these two functions are +** called via macros that record the current file and line number in the +** ThreadData structure. +*/ +SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3 *db, const char *z){ + char *zNew; + size_t n; + if( z==0 ){ + return 0; + } + n = sqlite3Strlen30(z) + 1; + assert( (n&0x7fffffff)==n ); + zNew = sqlite3DbMallocRaw(db, (int)n); + if( zNew ){ + memcpy(zNew, z, n); + } + return zNew; +} +SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, int n){ + char *zNew; + if( z==0 ){ + return 0; + } + assert( (n&0x7fffffff)==n ); + zNew = sqlite3DbMallocRaw(db, n+1); + if( zNew ){ + memcpy(zNew, z, n); + zNew[n] = 0; + } + return zNew; +} + +/* +** Create a string from the zFromat argument and the va_list that follows. +** Store the string in memory obtained from sqliteMalloc() and make *pz +** point to that string. +*/ +SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *z; + + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + sqlite3DbFree(db, *pz); + *pz = z; +} + + +/* +** This function must be called before exiting any API function (i.e. +** returning control to the user) that has called sqlite3_malloc or +** sqlite3_realloc. +** +** The returned value is normally a copy of the second argument to this +** function. However, if a malloc() failure has occurred since the previous +** invocation SQLITE_NOMEM is returned instead. +** +** If the first argument, db, is not NULL and a malloc() error has occurred, +** then the connection error-code (the value returned by sqlite3_errcode()) +** is set to SQLITE_NOMEM. +*/ +SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ + /* If the db handle is not NULL, then we must hold the connection handle + ** mutex here. Otherwise the read (and possible write) of db->mallocFailed + ** is unsafe, as is the call to sqlite3Error(). + */ + assert( !db || sqlite3_mutex_held(db->mutex) ); + if( db && (db->mallocFailed || rc==SQLITE_IOERR_NOMEM) ){ + sqlite3Error(db, SQLITE_NOMEM, 0); + db->mallocFailed = 0; + rc = SQLITE_NOMEM; + } + return rc & (db ? db->errMask : 0xff); +} + +/************** End of malloc.c **********************************************/ +/************** Begin file printf.c ******************************************/ +/* +** The "printf" code that follows dates from the 1980's. It is in +** the public domain. The original comments are included here for +** completeness. They are very out-of-date but might be useful as +** an historical reference. Most of the "enhancements" have been backed +** out so that the functionality is now the same as standard printf(). +** +************************************************************************** +** +** This file contains code for a set of "printf"-like routines. These +** routines format strings much like the printf() from the standard C +** library, though the implementation here has enhancements to support +** SQLlite. +*/ + +/* +** Conversion types fall into various categories as defined by the +** following enumeration. +*/ +#define etRADIX 1 /* Integer types. %d, %x, %o, and so forth */ +#define etFLOAT 2 /* Floating point. %f */ +#define etEXP 3 /* Exponentional notation. %e and %E */ +#define etGENERIC 4 /* Floating or exponential, depending on exponent. %g */ +#define etSIZE 5 /* Return number of characters processed so far. %n */ +#define etSTRING 6 /* Strings. %s */ +#define etDYNSTRING 7 /* Dynamically allocated strings. %z */ +#define etPERCENT 8 /* Percent symbol. %% */ +#define etCHARX 9 /* Characters. %c */ +/* The rest are extensions, not normally found in printf() */ +#define etSQLESCAPE 10 /* Strings with '\'' doubled. %q */ +#define etSQLESCAPE2 11 /* Strings with '\'' doubled and enclosed in '', + NULL pointers replaced by SQL NULL. %Q */ +#define etTOKEN 12 /* a pointer to a Token structure */ +#define etSRCLIST 13 /* a pointer to a SrcList */ +#define etPOINTER 14 /* The %p conversion */ +#define etSQLESCAPE3 15 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 16 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ + +#define etINVALID 0 /* Any unrecognized conversion type */ + + +/* +** An "etByte" is an 8-bit unsigned value. +*/ +typedef unsigned char etByte; + +/* +** Each builtin conversion character (ex: the 'd' in "%d") is described +** by an instance of the following structure +*/ +typedef struct et_info { /* Information about each format field */ + char fmttype; /* The format field code letter */ + etByte base; /* The base for radix conversion */ + etByte flags; /* One or more of FLAG_ constants below */ + etByte type; /* Conversion paradigm */ + etByte charset; /* Offset into aDigits[] of the digits string */ + etByte prefix; /* Offset into aPrefix[] of the prefix string */ +} et_info; + +/* +** Allowed values for et_info.flags +*/ +#define FLAG_SIGNED 1 /* True if the value to convert is signed */ +#define FLAG_INTERN 2 /* True if for internal use only */ +#define FLAG_STRING 4 /* Allow infinity precision */ + + +/* +** The following table is searched linearly, so it is good to put the +** most frequently used conversion types first. +*/ +static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; +static const char aPrefix[] = "-x0\000X0"; +static const et_info fmtinfo[] = { + { 'd', 10, 1, etRADIX, 0, 0 }, + { 's', 0, 4, etSTRING, 0, 0 }, + { 'g', 0, 1, etGENERIC, 30, 0 }, + { 'z', 0, 4, etDYNSTRING, 0, 0 }, + { 'q', 0, 4, etSQLESCAPE, 0, 0 }, + { 'Q', 0, 4, etSQLESCAPE2, 0, 0 }, + { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, + { 'c', 0, 0, etCHARX, 0, 0 }, + { 'o', 8, 0, etRADIX, 0, 2 }, + { 'u', 10, 0, etRADIX, 0, 0 }, + { 'x', 16, 0, etRADIX, 16, 1 }, + { 'X', 16, 0, etRADIX, 0, 4 }, +#ifndef SQLITE_OMIT_FLOATING_POINT + { 'f', 0, 1, etFLOAT, 0, 0 }, + { 'e', 0, 1, etEXP, 30, 0 }, + { 'E', 0, 1, etEXP, 14, 0 }, + { 'G', 0, 1, etGENERIC, 14, 0 }, +#endif + { 'i', 10, 1, etRADIX, 0, 0 }, + { 'n', 0, 0, etSIZE, 0, 0 }, + { '%', 0, 0, etPERCENT, 0, 0 }, + { 'p', 16, 0, etPOINTER, 0, 1 }, + +/* All the rest have the FLAG_INTERN bit set and are thus for internal +** use only */ + { 'T', 0, 2, etTOKEN, 0, 0 }, + { 'S', 0, 2, etSRCLIST, 0, 0 }, + { 'r', 10, 3, etORDINAL, 0, 0 }, +}; + +/* +** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point +** conversions will work. +*/ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** "*val" is a double such that 0.1 <= *val < 10.0 +** Return the ascii code for the leading digit of *val, then +** multiply "*val" by 10.0 to renormalize. +** +** Example: +** input: *val = 3.14159 +** output: *val = 1.4159 function return = '3' +** +** The counter *cnt is incremented each time. After counter exceeds +** 16 (the number of significant digits in a 64-bit float) '0' is +** always returned. +*/ +static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ + int digit; + LONGDOUBLE_TYPE d; + if( (*cnt)<=0 ) return '0'; + (*cnt)--; + digit = (int)*val; + d = digit; + digit += '0'; + *val = (*val - d)*10.0; + return (char)digit; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + +/* +** Set the StrAccum object to an error mode. +*/ +static void setStrAccumError(StrAccum *p, u8 eError){ + p->accError = eError; + p->nAlloc = 0; +} + +/* +** Extra argument values from a PrintfArguments object +*/ +static sqlite3_int64 getIntArg(PrintfArguments *p){ + if( p->nArg<=p->nUsed ) return 0; + return sqlite3_value_int64(p->apArg[p->nUsed++]); +} +static double getDoubleArg(PrintfArguments *p){ + if( p->nArg<=p->nUsed ) return 0.0; + return sqlite3_value_double(p->apArg[p->nUsed++]); +} +static char *getTextArg(PrintfArguments *p){ + if( p->nArg<=p->nUsed ) return 0; + return (char*)sqlite3_value_text(p->apArg[p->nUsed++]); +} + + +/* +** On machines with a small stack size, you can redefine the +** SQLITE_PRINT_BUF_SIZE to be something smaller, if desired. +*/ +#ifndef SQLITE_PRINT_BUF_SIZE +# define SQLITE_PRINT_BUF_SIZE 70 +#endif +#define etBUFSIZE SQLITE_PRINT_BUF_SIZE /* Size of the output buffer */ + +/* +** Render a string given by "fmt" into the StrAccum object. +*/ +SQLITE_PRIVATE void sqlite3VXPrintf( + StrAccum *pAccum, /* Accumulate results here */ + u32 bFlags, /* SQLITE_PRINTF_* flags */ + const char *fmt, /* Format string */ + va_list ap /* arguments */ +){ + int c; /* Next character in the format string */ + char *bufpt; /* Pointer to the conversion buffer */ + int precision; /* Precision of the current field */ + int length; /* Length of the field */ + int idx; /* A general purpose loop counter */ + int width; /* Width of the current field */ + etByte flag_leftjustify; /* True if "-" flag is present */ + etByte flag_plussign; /* True if "+" flag is present */ + etByte flag_blanksign; /* True if " " flag is present */ + etByte flag_alternateform; /* True if "#" flag is present */ + etByte flag_altform2; /* True if "!" flag is present */ + etByte flag_zeropad; /* True if field width constant starts with zero */ + etByte flag_long; /* True if "l" flag is present */ + etByte flag_longlong; /* True if the "ll" flag is present */ + etByte done; /* Loop termination flag */ + etByte xtype = 0; /* Conversion paradigm */ + u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ + u8 useIntern; /* Ok to use internal conversions (ex: %T) */ + char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ + sqlite_uint64 longvalue; /* Value for integer types */ + LONGDOUBLE_TYPE realvalue; /* Value for real types */ + const et_info *infop; /* Pointer to the appropriate info structure */ + char *zOut; /* Rendering buffer */ + int nOut; /* Size of the rendering buffer */ + char *zExtra; /* Malloced memory used by some conversion */ +#ifndef SQLITE_OMIT_FLOATING_POINT + int exp, e2; /* exponent of real numbers */ + int nsd; /* Number of significant digits returned */ + double rounder; /* Used for rounding floating point values */ + etByte flag_dp; /* True if decimal point should be shown */ + etByte flag_rtz; /* True if trailing zeros should be removed */ +#endif + PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ + char buf[etBUFSIZE]; /* Conversion buffer */ + + bufpt = 0; + if( bFlags ){ + if( (bArgList = (bFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ + pArgList = va_arg(ap, PrintfArguments*); + } + useIntern = bFlags & SQLITE_PRINTF_INTERNAL; + }else{ + bArgList = useIntern = 0; + } + for(; (c=(*fmt))!=0; ++fmt){ + if( c!='%' ){ + bufpt = (char *)fmt; + while( (c=(*++fmt))!='%' && c!=0 ){}; + sqlite3StrAccumAppend(pAccum, bufpt, (int)(fmt - bufpt)); + if( c==0 ) break; + } + if( (c=(*++fmt))==0 ){ + sqlite3StrAccumAppend(pAccum, "%", 1); + break; + } + /* Find out what flags are present */ + flag_leftjustify = flag_plussign = flag_blanksign = + flag_alternateform = flag_altform2 = flag_zeropad = 0; + done = 0; + do{ + switch( c ){ + case '-': flag_leftjustify = 1; break; + case '+': flag_plussign = 1; break; + case ' ': flag_blanksign = 1; break; + case '#': flag_alternateform = 1; break; + case '!': flag_altform2 = 1; break; + case '0': flag_zeropad = 1; break; + default: done = 1; break; + } + }while( !done && (c=(*++fmt))!=0 ); + /* Get the field width */ + width = 0; + if( c=='*' ){ + if( bArgList ){ + width = (int)getIntArg(pArgList); + }else{ + width = va_arg(ap,int); + } + if( width<0 ){ + flag_leftjustify = 1; + width = -width; + } + c = *++fmt; + }else{ + while( c>='0' && c<='9' ){ + width = width*10 + c - '0'; + c = *++fmt; + } + } + /* Get the precision */ + if( c=='.' ){ + precision = 0; + c = *++fmt; + if( c=='*' ){ + if( bArgList ){ + precision = (int)getIntArg(pArgList); + }else{ + precision = va_arg(ap,int); + } + if( precision<0 ) precision = -precision; + c = *++fmt; + }else{ + while( c>='0' && c<='9' ){ + precision = precision*10 + c - '0'; + c = *++fmt; + } + } + }else{ + precision = -1; + } + /* Get the conversion type modifier */ + if( c=='l' ){ + flag_long = 1; + c = *++fmt; + if( c=='l' ){ + flag_longlong = 1; + c = *++fmt; + }else{ + flag_longlong = 0; + } + }else{ + flag_long = flag_longlong = 0; + } + /* Fetch the info entry for the field */ + infop = &fmtinfo[0]; + xtype = etINVALID; + for(idx=0; idxflags & FLAG_INTERN)==0 ){ + xtype = infop->type; + }else{ + return; + } + break; + } + } + zExtra = 0; + + /* + ** At this point, variables are initialized as follows: + ** + ** flag_alternateform TRUE if a '#' is present. + ** flag_altform2 TRUE if a '!' is present. + ** flag_plussign TRUE if a '+' is present. + ** flag_leftjustify TRUE if a '-' is present or if the + ** field width was negative. + ** flag_zeropad TRUE if the width began with 0. + ** flag_long TRUE if the letter 'l' (ell) prefixed + ** the conversion character. + ** flag_longlong TRUE if the letter 'll' (ell ell) prefixed + ** the conversion character. + ** flag_blanksign TRUE if a ' ' is present. + ** width The specified field width. This is + ** always non-negative. Zero is the default. + ** precision The specified precision. The default + ** is -1. + ** xtype The class of the conversion. + ** infop Pointer to the appropriate info struct. + */ + switch( xtype ){ + case etPOINTER: + flag_longlong = sizeof(char*)==sizeof(i64); + flag_long = sizeof(char*)==sizeof(long int); + /* Fall through into the next case */ + case etORDINAL: + case etRADIX: + if( infop->flags & FLAG_SIGNED ){ + i64 v; + if( bArgList ){ + v = getIntArg(pArgList); + }else if( flag_longlong ){ + v = va_arg(ap,i64); + }else if( flag_long ){ + v = va_arg(ap,long int); + }else{ + v = va_arg(ap,int); + } + if( v<0 ){ + if( v==SMALLEST_INT64 ){ + longvalue = ((u64)1)<<63; + }else{ + longvalue = -v; + } + prefix = '-'; + }else{ + longvalue = v; + if( flag_plussign ) prefix = '+'; + else if( flag_blanksign ) prefix = ' '; + else prefix = 0; + } + }else{ + if( bArgList ){ + longvalue = (u64)getIntArg(pArgList); + }else if( flag_longlong ){ + longvalue = va_arg(ap,u64); + }else if( flag_long ){ + longvalue = va_arg(ap,unsigned long int); + }else{ + longvalue = va_arg(ap,unsigned int); + } + prefix = 0; + } + if( longvalue==0 ) flag_alternateform = 0; + if( flag_zeropad && precision=4 || (longvalue/10)%10==1 ){ + x = 0; + } + *(--bufpt) = zOrd[x*2+1]; + *(--bufpt) = zOrd[x*2]; + } + { + const char *cset = &aDigits[infop->charset]; + u8 base = infop->base; + do{ /* Convert to ascii */ + *(--bufpt) = cset[longvalue%base]; + longvalue = longvalue/base; + }while( longvalue>0 ); + } + length = (int)(&zOut[nOut-1]-bufpt); + for(idx=precision-length; idx>0; idx--){ + *(--bufpt) = '0'; /* Zero pad */ + } + if( prefix ) *(--bufpt) = prefix; /* Add sign */ + if( flag_alternateform && infop->prefix ){ /* Add "0" or "0x" */ + const char *pre; + char x; + pre = &aPrefix[infop->prefix]; + for(; (x=(*pre))!=0; pre++) *(--bufpt) = x; + } + length = (int)(&zOut[nOut-1]-bufpt); + break; + case etFLOAT: + case etEXP: + case etGENERIC: + if( bArgList ){ + realvalue = getDoubleArg(pArgList); + }else{ + realvalue = va_arg(ap,double); + } +#ifdef SQLITE_OMIT_FLOATING_POINT + length = 0; +#else + if( precision<0 ) precision = 6; /* Set default precision */ + if( realvalue<0.0 ){ + realvalue = -realvalue; + prefix = '-'; + }else{ + if( flag_plussign ) prefix = '+'; + else if( flag_blanksign ) prefix = ' '; + else prefix = 0; + } + if( xtype==etGENERIC && precision>0 ) precision--; + for(idx=precision, rounder=0.5; idx>0; idx--, rounder*=0.1){} + if( xtype==etFLOAT ) realvalue += rounder; + /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ + exp = 0; + if( sqlite3IsNaN((double)realvalue) ){ + bufpt = "NaN"; + length = 3; + break; + } + if( realvalue>0.0 ){ + LONGDOUBLE_TYPE scale = 1.0; + while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} + while( realvalue>=1e64*scale && exp<=350 ){ scale *= 1e64; exp+=64; } + while( realvalue>=1e8*scale && exp<=350 ){ scale *= 1e8; exp+=8; } + while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } + realvalue /= scale; + while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } + while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } + if( exp>350 ){ + if( prefix=='-' ){ + bufpt = "-Inf"; + }else if( prefix=='+' ){ + bufpt = "+Inf"; + }else{ + bufpt = "Inf"; + } + length = sqlite3Strlen30(bufpt); + break; + } + } + bufpt = buf; + /* + ** If the field type is etGENERIC, then convert to either etEXP + ** or etFLOAT, as appropriate. + */ + if( xtype!=etFLOAT ){ + realvalue += rounder; + if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } + } + if( xtype==etGENERIC ){ + flag_rtz = !flag_alternateform; + if( exp<-4 || exp>precision ){ + xtype = etEXP; + }else{ + precision = precision - exp; + xtype = etFLOAT; + } + }else{ + flag_rtz = flag_altform2; + } + if( xtype==etEXP ){ + e2 = 0; + }else{ + e2 = exp; + } + if( MAX(e2,0)+precision+width > etBUFSIZE - 15 ){ + bufpt = zExtra = sqlite3Malloc( MAX(e2,0)+precision+width+15 ); + if( bufpt==0 ){ + setStrAccumError(pAccum, STRACCUM_NOMEM); + return; + } + } + zOut = bufpt; + nsd = 16 + flag_altform2*10; + flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; + /* The sign in front of the number */ + if( prefix ){ + *(bufpt++) = prefix; + } + /* Digits prior to the decimal point */ + if( e2<0 ){ + *(bufpt++) = '0'; + }else{ + for(; e2>=0; e2--){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } + } + /* The decimal point */ + if( flag_dp ){ + *(bufpt++) = '.'; + } + /* "0" digits after the decimal point but before the first + ** significant digit of the number */ + for(e2++; e2<0; precision--, e2++){ + assert( precision>0 ); + *(bufpt++) = '0'; + } + /* Significant digits after the decimal point */ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } + /* Remove trailing zeros and the "." if no digits follow the "." */ + if( flag_rtz && flag_dp ){ + while( bufpt[-1]=='0' ) *(--bufpt) = 0; + assert( bufpt>zOut ); + if( bufpt[-1]=='.' ){ + if( flag_altform2 ){ + *(bufpt++) = '0'; + }else{ + *(--bufpt) = 0; + } + } + } + /* Add the "eNNN" suffix */ + if( xtype==etEXP ){ + *(bufpt++) = aDigits[infop->charset]; + if( exp<0 ){ + *(bufpt++) = '-'; exp = -exp; + }else{ + *(bufpt++) = '+'; + } + if( exp>=100 ){ + *(bufpt++) = (char)((exp/100)+'0'); /* 100's digit */ + exp %= 100; + } + *(bufpt++) = (char)(exp/10+'0'); /* 10's digit */ + *(bufpt++) = (char)(exp%10+'0'); /* 1's digit */ + } + *bufpt = 0; + + /* The converted number is in buf[] and zero terminated. Output it. + ** Note that the number is in the usual order, not reversed as with + ** integer conversions. */ + length = (int)(bufpt-zOut); + bufpt = zOut; + + /* Special case: Add leading zeros if the flag_zeropad flag is + ** set and we are not left justified */ + if( flag_zeropad && !flag_leftjustify && length < width){ + int i; + int nPad = width - length; + for(i=width; i>=nPad; i--){ + bufpt[i] = bufpt[i-nPad]; + } + i = prefix!=0; + while( nPad-- ) bufpt[i++] = '0'; + length = width; + } +#endif /* !defined(SQLITE_OMIT_FLOATING_POINT) */ + break; + case etSIZE: + if( !bArgList ){ + *(va_arg(ap,int*)) = pAccum->nChar; + } + length = width = 0; + break; + case etPERCENT: + buf[0] = '%'; + bufpt = buf; + length = 1; + break; + case etCHARX: + if( bArgList ){ + bufpt = getTextArg(pArgList); + c = bufpt ? bufpt[0] : 0; + }else{ + c = va_arg(ap,int); + } + buf[0] = (char)c; + if( precision>=0 ){ + for(idx=1; idx=0 ){ + for(length=0; lengthetBUFSIZE ){ + bufpt = zExtra = sqlite3Malloc( n ); + if( bufpt==0 ){ + setStrAccumError(pAccum, STRACCUM_NOMEM); + return; + } + }else{ + bufpt = buf; + } + j = 0; + if( needQuote ) bufpt[j++] = q; + k = i; + for(i=0; i=0 && precisionn ){ + sqlite3StrAccumAppend(pAccum, (const char*)pToken->z, pToken->n); + } + length = width = 0; + break; + } + case etSRCLIST: { + SrcList *pSrc = va_arg(ap, SrcList*); + int k = va_arg(ap, int); + struct SrcList_item *pItem = &pSrc->a[k]; + assert( bArgList==0 ); + assert( k>=0 && knSrc ); + if( pItem->zDatabase ){ + sqlite3StrAccumAppendAll(pAccum, pItem->zDatabase); + sqlite3StrAccumAppend(pAccum, ".", 1); + } + sqlite3StrAccumAppendAll(pAccum, pItem->zName); + length = width = 0; + break; + } + default: { + assert( xtype==etINVALID ); + return; + } + }/* End switch over the format type */ + /* + ** The text of the conversion is pointed to by "bufpt" and is + ** "length" characters long. The field width is "width". Do + ** the output. + */ + width -= length; + if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + sqlite3StrAccumAppend(pAccum, bufpt, length); + if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + + if( zExtra ) sqlite3_free(zExtra); + }/* End for loop over the format string */ +} /* End of function */ + +/* +** Enlarge the memory allocation on a StrAccum object so that it is +** able to accept at least N more bytes of text. +** +** Return the number of bytes of text that StrAccum is able to accept +** after the attempted enlargement. The value returned might be zero. +*/ +static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ + char *zNew; + assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ + if( p->accError ){ + testcase(p->accError==STRACCUM_TOOBIG); + testcase(p->accError==STRACCUM_NOMEM); + return 0; + } + if( !p->useMalloc ){ + N = p->nAlloc - p->nChar - 1; + setStrAccumError(p, STRACCUM_TOOBIG); + return N; + }else{ + char *zOld = (p->zText==p->zBase ? 0 : p->zText); + i64 szNew = p->nChar; + szNew += N + 1; + if( szNew > p->mxAlloc ){ + sqlite3StrAccumReset(p); + setStrAccumError(p, STRACCUM_TOOBIG); + return 0; + }else{ + p->nAlloc = (int)szNew; + } + if( p->useMalloc==1 ){ + zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); + }else{ + zNew = sqlite3_realloc(zOld, p->nAlloc); + } + if( zNew ){ + assert( p->zText!=0 || p->nChar==0 ); + if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); + p->zText = zNew; + }else{ + sqlite3StrAccumReset(p); + setStrAccumError(p, STRACCUM_NOMEM); + return 0; + } + } + return N; +} + +/* +** Append N space characters to the given string buffer. +*/ +SQLITE_PRIVATE void sqlite3AppendSpace(StrAccum *p, int N){ + if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; + while( (N--)>0 ) p->zText[p->nChar++] = ' '; +} + +/* +** The StrAccum "p" is not large enough to accept N new bytes of z[]. +** So enlarge if first, then do the append. +** +** This is a helper routine to sqlite3StrAccumAppend() that does special-case +** work (enlarging the buffer) using tail recursion, so that the +** sqlite3StrAccumAppend() routine can use fast calling semantics. +*/ +static void enlargeAndAppend(StrAccum *p, const char *z, int N){ + N = sqlite3StrAccumEnlarge(p, N); + if( N>0 ){ + memcpy(&p->zText[p->nChar], z, N); + p->nChar += N; + } +} + +/* +** Append N bytes of text from z to the StrAccum object. Increase the +** size of the memory allocation for StrAccum if necessary. +*/ +SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ + assert( z!=0 ); + assert( p->zText!=0 || p->nChar==0 || p->accError ); + assert( N>=0 ); + assert( p->accError==0 || p->nAlloc==0 ); + if( p->nChar+N >= p->nAlloc ){ + enlargeAndAppend(p,z,N); + return; + } + assert( p->zText ); + memcpy(&p->zText[p->nChar], z, N); + p->nChar += N; +} + +/* +** Append the complete text of zero-terminated string z[] to the p string. +*/ +SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum *p, const char *z){ + sqlite3StrAccumAppend(p, z, sqlite3Strlen30(z)); +} + + +/* +** Finish off a string by making sure it is zero-terminated. +** Return a pointer to the resulting string. Return a NULL +** pointer if any kind of error was encountered. +*/ +SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ + if( p->zText ){ + p->zText[p->nChar] = 0; + if( p->useMalloc && p->zText==p->zBase ){ + if( p->useMalloc==1 ){ + p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + }else{ + p->zText = sqlite3_malloc(p->nChar+1); + } + if( p->zText ){ + memcpy(p->zText, p->zBase, p->nChar+1); + }else{ + setStrAccumError(p, STRACCUM_NOMEM); + } + } + } + return p->zText; +} + +/* +** Reset an StrAccum string. Reclaim all malloced memory. +*/ +SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum *p){ + if( p->zText!=p->zBase ){ + if( p->useMalloc==1 ){ + sqlite3DbFree(p->db, p->zText); + }else{ + sqlite3_free(p->zText); + } + } + p->zText = 0; +} + +/* +** Initialize a string accumulator +*/ +SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum *p, char *zBase, int n, int mx){ + p->zText = p->zBase = zBase; + p->db = 0; + p->nChar = 0; + p->nAlloc = n; + p->mxAlloc = mx; + p->useMalloc = 1; + p->accError = 0; +} + +/* +** Print into memory obtained from sqliteMalloc(). Use the internal +** %-conversion extensions. +*/ +SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list ap){ + char *z; + char zBase[SQLITE_PRINT_BUF_SIZE]; + StrAccum acc; + assert( db!=0 ); + sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), + db->aLimit[SQLITE_LIMIT_LENGTH]); + acc.db = db; + sqlite3VXPrintf(&acc, SQLITE_PRINTF_INTERNAL, zFormat, ap); + z = sqlite3StrAccumFinish(&acc); + if( acc.accError==STRACCUM_NOMEM ){ + db->mallocFailed = 1; + } + return z; +} + +/* +** Print into memory obtained from sqliteMalloc(). Use the internal +** %-conversion extensions. +*/ +SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + return z; +} + +/* +** Like sqlite3MPrintf(), but call sqlite3DbFree() on zStr after formatting +** the string and before returnning. This routine is intended to be used +** to modify an existing string. For example: +** +** x = sqlite3MPrintf(db, x, "prefix %s suffix", x); +** +*/ +SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3 *db, char *zStr, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + sqlite3DbFree(db, zStr); + return z; +} + +/* +** Print into memory obtained from sqlite3_malloc(). Omit the internal +** %-conversion extensions. +*/ +SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ + char *z; + char zBase[SQLITE_PRINT_BUF_SIZE]; + StrAccum acc; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); + acc.useMalloc = 2; + sqlite3VXPrintf(&acc, 0, zFormat, ap); + z = sqlite3StrAccumFinish(&acc); + return z; +} + +/* +** Print into memory obtained from sqlite3_malloc()(). Omit the internal +** %-conversion extensions. +*/ +SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){ + va_list ap; + char *z; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + return z; +} + +/* +** sqlite3_snprintf() works like snprintf() except that it ignores the +** current locale settings. This is important for SQLite because we +** are not able to use a "," as the decimal point in place of "." as +** specified by some locales. +** +** Oops: The first two arguments of sqlite3_snprintf() are backwards +** from the snprintf() standard. Unfortunately, it is too late to change +** this without breaking compatibility, so we just have to live with the +** mistake. +** +** sqlite3_vsnprintf() is the varargs version. +*/ +SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ + StrAccum acc; + if( n<=0 ) return zBuf; + sqlite3StrAccumInit(&acc, zBuf, n, 0); + acc.useMalloc = 0; + sqlite3VXPrintf(&acc, 0, zFormat, ap); + return sqlite3StrAccumFinish(&acc); +} +SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ + char *z; + va_list ap; + va_start(ap,zFormat); + z = sqlite3_vsnprintf(n, zBuf, zFormat, ap); + va_end(ap); + return z; +} + +/* +** This is the routine that actually formats the sqlite3_log() message. +** We house it in a separate routine from sqlite3_log() to avoid using +** stack space on small-stack systems when logging is disabled. +** +** sqlite3_log() must render into a static buffer. It cannot dynamically +** allocate memory because it might be called while the memory allocator +** mutex is held. +*/ +static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ + StrAccum acc; /* String accumulator */ + char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ + + sqlite3StrAccumInit(&acc, zMsg, sizeof(zMsg), 0); + acc.useMalloc = 0; + sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, + sqlite3StrAccumFinish(&acc)); +} + +/* +** Format and write a message to the log if logging is enabled. +*/ +SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...){ + va_list ap; /* Vararg list */ + if( sqlite3GlobalConfig.xLog ){ + va_start(ap, zFormat); + renderLogMsg(iErrCode, zFormat, ap); + va_end(ap); + } +} + +#if defined(SQLITE_DEBUG) +/* +** A version of printf() that understands %lld. Used for debugging. +** The printf() built into some versions of windows does not understand %lld +** and segfaults if you give it a long long int. +*/ +SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ + va_list ap; + StrAccum acc; + char zBuf[500]; + sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); + acc.useMalloc = 0; + va_start(ap,zFormat); + sqlite3VXPrintf(&acc, 0, zFormat, ap); + va_end(ap); + sqlite3StrAccumFinish(&acc); + fprintf(stdout,"%s", zBuf); + fflush(stdout); +} +#endif + +/* +** variable-argument wrapper around sqlite3VXPrintf(). +*/ +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ...){ + va_list ap; + va_start(ap,zFormat); + sqlite3VXPrintf(p, bFlags, zFormat, ap); + va_end(ap); +} + +/************** End of printf.c **********************************************/ +/************** Begin file random.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement a pseudo-random number +** generator (PRNG) for SQLite. +** +** Random numbers are used by some of the database backends in order +** to generate random integer keys for tables or random filenames. +*/ + + +/* All threads share a single random number generator. +** This structure is the current state of the generator. +*/ +static SQLITE_WSD struct sqlite3PrngType { + unsigned char isInit; /* True if initialized */ + unsigned char i, j; /* State variables */ + unsigned char s[256]; /* State variables */ +} sqlite3Prng; + +/* +** Return N random bytes. +*/ +SQLITE_API void sqlite3_randomness(int N, void *pBuf){ + unsigned char t; + unsigned char *zBuf = pBuf; + + /* The "wsdPrng" macro will resolve to the pseudo-random number generator + ** state vector. If writable static data is unsupported on the target, + ** we have to locate the state vector at run-time. In the more common + ** case where writable static data is supported, wsdPrng can refer directly + ** to the "sqlite3Prng" state vector declared above. + */ +#ifdef SQLITE_OMIT_WSD + struct sqlite3PrngType *p = &GLOBAL(struct sqlite3PrngType, sqlite3Prng); +# define wsdPrng p[0] +#else +# define wsdPrng sqlite3Prng +#endif + +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); + sqlite3_mutex_enter(mutex); +#endif + + if( N<=0 ){ + wsdPrng.isInit = 0; + sqlite3_mutex_leave(mutex); + return; + } + + /* Initialize the state of the random number generator once, + ** the first time this routine is called. The seed value does + ** not need to contain a lot of randomness since we are not + ** trying to do secure encryption or anything like that... + ** + ** Nothing in this file or anywhere else in SQLite does any kind of + ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random + ** number generator) not as an encryption device. + */ + if( !wsdPrng.isInit ){ + int i; + char k[256]; + wsdPrng.j = 0; + wsdPrng.i = 0; + sqlite3OsRandomness(sqlite3_vfs_find(0), 256, k); + for(i=0; i<256; i++){ + wsdPrng.s[i] = (u8)i; + } + for(i=0; i<256; i++){ + wsdPrng.j += wsdPrng.s[i] + k[i]; + t = wsdPrng.s[wsdPrng.j]; + wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; + wsdPrng.s[i] = t; + } + wsdPrng.isInit = 1; + } + + assert( N>0 ); + do{ + wsdPrng.i++; + t = wsdPrng.s[wsdPrng.i]; + wsdPrng.j += t; + wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; + wsdPrng.s[wsdPrng.j] = t; + t += wsdPrng.s[wsdPrng.i]; + *(zBuf++) = wsdPrng.s[t]; + }while( --N ); + sqlite3_mutex_leave(mutex); +} + +#ifndef SQLITE_OMIT_BUILTIN_TEST +/* +** For testing purposes, we sometimes want to preserve the state of +** PRNG and restore the PRNG to its saved state at a later time, or +** to reset the PRNG to its initial state. These routines accomplish +** those tasks. +** +** The sqlite3_test_control() interface calls these routines to +** control the PRNG. +*/ +static SQLITE_WSD struct sqlite3PrngType sqlite3SavedPrng; +SQLITE_PRIVATE void sqlite3PrngSaveState(void){ + memcpy( + &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), + &GLOBAL(struct sqlite3PrngType, sqlite3Prng), + sizeof(sqlite3Prng) + ); +} +SQLITE_PRIVATE void sqlite3PrngRestoreState(void){ + memcpy( + &GLOBAL(struct sqlite3PrngType, sqlite3Prng), + &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), + sizeof(sqlite3Prng) + ); +} +#endif /* SQLITE_OMIT_BUILTIN_TEST */ + +/************** End of random.c **********************************************/ +/************** Begin file utf.c *********************************************/ +/* +** 2004 April 13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains routines used to translate between UTF-8, +** UTF-16, UTF-16BE, and UTF-16LE. +** +** Notes on UTF-8: +** +** Byte-0 Byte-1 Byte-2 Byte-3 Value +** 0xxxxxxx 00000000 00000000 0xxxxxxx +** 110yyyyy 10xxxxxx 00000000 00000yyy yyxxxxxx +** 1110zzzz 10yyyyyy 10xxxxxx 00000000 zzzzyyyy yyxxxxxx +** 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx 000uuuuu zzzzyyyy yyxxxxxx +** +** +** Notes on UTF-16: (with wwww+1==uuuuu) +** +** Word-0 Word-1 Value +** 110110ww wwzzzzyy 110111yy yyxxxxxx 000uuuuu zzzzyyyy yyxxxxxx +** zzzzyyyy yyxxxxxx 00000000 zzzzyyyy yyxxxxxx +** +** +** BOM or Byte Order Mark: +** 0xff 0xfe little-endian utf-16 follows +** 0xfe 0xff big-endian utf-16 follows +** +*/ +/* #include */ + +#ifndef SQLITE_AMALGAMATION +/* +** The following constant value is used by the SQLITE_BIGENDIAN and +** SQLITE_LITTLEENDIAN macros. +*/ +SQLITE_PRIVATE const int sqlite3one = 1; +#endif /* SQLITE_AMALGAMATION */ + +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. +*/ +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} + +#define WRITE_UTF16LE(zOut, c) { \ + if( c<=0xFFFF ){ \ + *zOut++ = (u8)(c&0x00FF); \ + *zOut++ = (u8)((c>>8)&0x00FF); \ + }else{ \ + *zOut++ = (u8)(((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0)); \ + *zOut++ = (u8)(0x00D8 + (((c-0x10000)>>18)&0x03)); \ + *zOut++ = (u8)(c&0x00FF); \ + *zOut++ = (u8)(0x00DC + ((c>>8)&0x03)); \ + } \ +} + +#define WRITE_UTF16BE(zOut, c) { \ + if( c<=0xFFFF ){ \ + *zOut++ = (u8)((c>>8)&0x00FF); \ + *zOut++ = (u8)(c&0x00FF); \ + }else{ \ + *zOut++ = (u8)(0x00D8 + (((c-0x10000)>>18)&0x03)); \ + *zOut++ = (u8)(((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0)); \ + *zOut++ = (u8)(0x00DC + ((c>>8)&0x03)); \ + *zOut++ = (u8)(c&0x00FF); \ + } \ +} + +#define READ_UTF16LE(zIn, TERM, c){ \ + c = (*zIn++); \ + c += ((*zIn++)<<8); \ + if( c>=0xD800 && c<0xE000 && TERM ){ \ + int c2 = (*zIn++); \ + c2 += ((*zIn++)<<8); \ + c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \ + } \ +} + +#define READ_UTF16BE(zIn, TERM, c){ \ + c = ((*zIn++)<<8); \ + c += (*zIn++); \ + if( c>=0xD800 && c<0xE000 && TERM ){ \ + int c2 = ((*zIn++)<<8); \ + c2 += (*zIn++); \ + c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \ + } \ +} + +/* +** Translate a single UTF-8 character. Return the unicode value. +** +** During translation, assume that the byte that zTerm points +** is a 0x00. +** +** Write a pointer to the next unread byte back into *pzNext. +** +** Notes On Invalid UTF-8: +** +** * This routine never allows a 7-bit character (0x00 through 0x7f) to +** be encoded as a multi-byte character. Any multi-byte character that +** attempts to encode a value between 0x00 and 0x7f is rendered as 0xfffd. +** +** * This routine never allows a UTF16 surrogate value to be encoded. +** If a multi-byte character attempts to encode a value between +** 0xd800 and 0xe000 then it is rendered as 0xfffd. +** +** * Bytes in the range of 0x80 through 0xbf which occur as the first +** byte of a character are interpreted as single-byte characters +** and rendered as themselves even though they are technically +** invalid characters. +** +** * This routine accepts over-length UTF8 encodings +** for unicode values 0x80 and greater. It does not change over-length +** encodings to 0xfffd as some systems recommend. +*/ +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } +SQLITE_PRIVATE u32 sqlite3Utf8Read( + const unsigned char **pz /* Pointer to string from which to read char */ +){ + unsigned int c; + + /* Same as READ_UTF8() above but without the zTerm parameter. + ** For this routine, we assume the UTF8 string is always zero-terminated. + */ + c = *((*pz)++); + if( c>=0xc0 ){ + c = sqlite3Utf8Trans1[c-0xc0]; + while( (*(*pz) & 0xc0)==0x80 ){ + c = (c<<6) + (0x3f & *((*pz)++)); + } + if( c<0x80 + || (c&0xFFFFF800)==0xD800 + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } + } + return c; +} + + + + +/* +** If the TRANSLATE_TRACE macro is defined, the value of each Mem is +** printed on stderr on the way into and out of sqlite3VdbeMemTranslate(). +*/ +/* #define TRANSLATE_TRACE 1 */ + +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine transforms the internal text encoding used by pMem to +** desiredEnc. It is an error if the string is already of the desired +** encoding, or if *pMem does not contain a string value. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ + int len; /* Maximum length of output string in bytes */ + unsigned char *zOut; /* Output buffer */ + unsigned char *zIn; /* Input iterator */ + unsigned char *zTerm; /* End of input */ + unsigned char *z; /* Output iterator */ + unsigned int c; + + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( pMem->flags&MEM_Str ); + assert( pMem->enc!=desiredEnc ); + assert( pMem->enc!=0 ); + assert( pMem->n>=0 ); + +#if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) + { + char zBuf[100]; + sqlite3VdbeMemPrettyPrint(pMem, zBuf); + fprintf(stderr, "INPUT: %s\n", zBuf); + } +#endif + + /* If the translation is between UTF-16 little and big endian, then + ** all that is required is to swap the byte order. This case is handled + ** differently from the others. + */ + if( pMem->enc!=SQLITE_UTF8 && desiredEnc!=SQLITE_UTF8 ){ + u8 temp; + int rc; + rc = sqlite3VdbeMemMakeWriteable(pMem); + if( rc!=SQLITE_OK ){ + assert( rc==SQLITE_NOMEM ); + return SQLITE_NOMEM; + } + zIn = (u8*)pMem->z; + zTerm = &zIn[pMem->n&~1]; + while( zInenc = desiredEnc; + goto translate_out; + } + + /* Set len to the maximum number of bytes required in the output buffer. */ + if( desiredEnc==SQLITE_UTF8 ){ + /* When converting from UTF-16, the maximum growth results from + ** translating a 2-byte character to a 4-byte UTF-8 character. + ** A single byte is required for the output string + ** nul-terminator. + */ + pMem->n &= ~1; + len = pMem->n * 2 + 1; + }else{ + /* When converting from UTF-8 to UTF-16 the maximum growth is caused + ** when a 1-byte UTF-8 character is translated into a 2-byte UTF-16 + ** character. Two bytes are required in the output buffer for the + ** nul-terminator. + */ + len = pMem->n * 2 + 2; + } + + /* Set zIn to point at the start of the input buffer and zTerm to point 1 + ** byte past the end. + ** + ** Variable zOut is set to point at the output buffer, space obtained + ** from sqlite3_malloc(). + */ + zIn = (u8*)pMem->z; + zTerm = &zIn[pMem->n]; + zOut = sqlite3DbMallocRaw(pMem->db, len); + if( !zOut ){ + return SQLITE_NOMEM; + } + z = zOut; + + if( pMem->enc==SQLITE_UTF8 ){ + if( desiredEnc==SQLITE_UTF16LE ){ + /* UTF-8 -> UTF-16 Little-endian */ + while( zIn UTF-16 Big-endian */ + while( zInn = (int)(z - zOut); + *z++ = 0; + }else{ + assert( desiredEnc==SQLITE_UTF8 ); + if( pMem->enc==SQLITE_UTF16LE ){ + /* UTF-16 Little-endian -> UTF-8 */ + while( zIn UTF-8 */ + while( zInn = (int)(z - zOut); + } + *z = 0; + assert( (pMem->n+(desiredEnc==SQLITE_UTF8?1:2))<=len ); + + sqlite3VdbeMemRelease(pMem); + pMem->flags &= ~(MEM_Static|MEM_Dyn|MEM_Ephem); + pMem->enc = desiredEnc; + pMem->flags |= (MEM_Term); + pMem->z = (char*)zOut; + pMem->zMalloc = pMem->z; + +translate_out: +#if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) + { + char zBuf[100]; + sqlite3VdbeMemPrettyPrint(pMem, zBuf); + fprintf(stderr, "OUTPUT: %s\n", zBuf); + } +#endif + return SQLITE_OK; +} + +/* +** This routine checks for a byte-order mark at the beginning of the +** UTF-16 string stored in *pMem. If one is present, it is removed and +** the encoding of the Mem adjusted. This routine does not do any +** byte-swapping, it just sets Mem.enc appropriately. +** +** The allocation (static, dynamic etc.) and encoding of the Mem may be +** changed by this function. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem){ + int rc = SQLITE_OK; + u8 bom = 0; + + assert( pMem->n>=0 ); + if( pMem->n>1 ){ + u8 b1 = *(u8 *)pMem->z; + u8 b2 = *(((u8 *)pMem->z) + 1); + if( b1==0xFE && b2==0xFF ){ + bom = SQLITE_UTF16BE; + } + if( b1==0xFF && b2==0xFE ){ + bom = SQLITE_UTF16LE; + } + } + + if( bom ){ + rc = sqlite3VdbeMemMakeWriteable(pMem); + if( rc==SQLITE_OK ){ + pMem->n -= 2; + memmove(pMem->z, &pMem->z[2], pMem->n); + pMem->z[pMem->n] = '\0'; + pMem->z[pMem->n+1] = '\0'; + pMem->flags |= MEM_Term; + pMem->enc = bom; + } + } + return rc; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** pZ is a UTF-8 encoded unicode string. If nByte is less than zero, +** return the number of unicode characters in pZ up to (but not including) +** the first 0x00 byte. If nByte is not less than zero, return the +** number of unicode characters in the first nByte of pZ (or up to +** the first 0x00, whichever comes first). +*/ +SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *zIn, int nByte){ + int r = 0; + const u8 *z = (const u8*)zIn; + const u8 *zTerm; + if( nByte>=0 ){ + zTerm = &z[nByte]; + }else{ + zTerm = (const u8*)(-1); + } + assert( z<=zTerm ); + while( *z!=0 && zmallocFailed ){ + sqlite3VdbeMemRelease(&m); + m.z = 0; + } + assert( (m.flags & MEM_Term)!=0 || db->mallocFailed ); + assert( (m.flags & MEM_Str)!=0 || db->mallocFailed ); + assert( m.z || db->mallocFailed ); + return m.z; +} + +/* +** zIn is a UTF-16 encoded unicode string at least nChar characters long. +** Return the number of bytes in the first nChar unicode characters +** in pZ. nChar must be non-negative. +*/ +SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *zIn, int nChar){ + int c; + unsigned char const *z = zIn; + int n = 0; + + if( SQLITE_UTF16NATIVE==SQLITE_UTF16BE ){ + while( n0 && n<=4 ); + z[0] = 0; + z = zBuf; + c = sqlite3Utf8Read((const u8**)&z); + t = i; + if( i>=0xD800 && i<=0xDFFF ) t = 0xFFFD; + if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD; + assert( c==t ); + assert( (z-zBuf)==n ); + } + for(i=0; i<0x00110000; i++){ + if( i>=0xD800 && i<0xE000 ) continue; + z = zBuf; + WRITE_UTF16LE(z, i); + n = (int)(z-zBuf); + assert( n>0 && n<=4 ); + z[0] = 0; + z = zBuf; + READ_UTF16LE(z, 1, c); + assert( c==i ); + assert( (z-zBuf)==n ); + } + for(i=0; i<0x00110000; i++){ + if( i>=0xD800 && i<0xE000 ) continue; + z = zBuf; + WRITE_UTF16BE(z, i); + n = (int)(z-zBuf); + assert( n>0 && n<=4 ); + z[0] = 0; + z = zBuf; + READ_UTF16BE(z, 1, c); + assert( c==i ); + assert( (z-zBuf)==n ); + } +} +#endif /* SQLITE_TEST */ +#endif /* SQLITE_OMIT_UTF16 */ + +/************** End of utf.c *************************************************/ +/************** Begin file util.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Utility functions used throughout sqlite. +** +** This file contains functions for allocating memory, comparing +** strings, and stuff like that. +** +*/ +/* #include */ +#ifdef SQLITE_HAVE_ISNAN +# include +#endif + +/* +** Routine needed to support the testcase() macro. +*/ +#ifdef SQLITE_COVERAGE_TEST +SQLITE_PRIVATE void sqlite3Coverage(int x){ + static unsigned dummy = 0; + dummy += (unsigned)x; +} +#endif + +/* +** Give a callback to the test harness that can be used to simulate faults +** in places where it is difficult or expensive to do so purely by means +** of inputs. +** +** The intent of the integer argument is to let the fault simulator know +** which of multiple sqlite3FaultSim() calls has been hit. +** +** Return whatever integer value the test callback returns, or return +** SQLITE_OK if no test callback is installed. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ + int (*xCallback)(int) = sqlite3GlobalConfig.xTestCallback; + return xCallback ? xCallback(iTest) : SQLITE_OK; +} +#endif + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Return true if the floating point value is Not a Number (NaN). +** +** Use the math library isnan() function if compiled with SQLITE_HAVE_ISNAN. +** Otherwise, we have our own implementation that works on most systems. +*/ +SQLITE_PRIVATE int sqlite3IsNaN(double x){ + int rc; /* The value return */ +#if !defined(SQLITE_HAVE_ISNAN) + /* + ** Systems that support the isnan() library function should probably + ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have + ** found that many systems do not have a working isnan() function so + ** this implementation is provided as an alternative. + ** + ** This NaN test sometimes fails if compiled on GCC with -ffast-math. + ** On the other hand, the use of -ffast-math comes with the following + ** warning: + ** + ** This option [-ffast-math] should never be turned on by any + ** -O option since it can result in incorrect output for programs + ** which depend on an exact implementation of IEEE or ISO + ** rules/specifications for math functions. + ** + ** Under MSVC, this NaN test may fail if compiled with a floating- + ** point precision mode other than /fp:precise. From the MSDN + ** documentation: + ** + ** The compiler [with /fp:precise] will properly handle comparisons + ** involving NaN. For example, x != x evaluates to true if x is NaN + ** ... + */ +#ifdef __FAST_MATH__ +# error SQLite will not work correctly with the -ffast-math option of GCC. +#endif + volatile double y = x; + volatile double z = y; + rc = (y!=z); +#else /* if defined(SQLITE_HAVE_ISNAN) */ + rc = isnan(x); +#endif /* SQLITE_HAVE_ISNAN */ + testcase( rc ); + return rc; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +** +** The value returned will never be negative. Nor will it ever be greater +** than the actual length of the string. For very long strings (greater +** than 1GiB) the value returned might be less than the true string length. +*/ +SQLITE_PRIVATE int sqlite3Strlen30(const char *z){ + const char *z2 = z; + if( z==0 ) return 0; + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); +} + +/* +** Set the most recent error code and error string for the sqlite +** handle "db". The error code is set to "err_code". +** +** If it is not NULL, string zFormat specifies the format of the +** error string in the style of the printf functions: The following +** format characters are allowed: +** +** %s Insert a string +** %z A string that should be freed after use +** %d Insert an integer +** %T Insert a token +** %S Insert the first element of a SrcList +** +** zFormat and any string tokens that follow it are assumed to be +** encoded in UTF-8. +** +** To clear the most recent error for sqlite handle "db", sqlite3Error +** should be called with err_code set to SQLITE_OK and zFormat set +** to NULL. +*/ +SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ...){ + assert( db!=0 ); + db->errCode = err_code; + if( zFormat && (db->pErr || (db->pErr = sqlite3ValueNew(db))!=0) ){ + char *z; + va_list ap; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + sqlite3ValueSetStr(db->pErr, -1, z, SQLITE_UTF8, SQLITE_DYNAMIC); + }else if( db->pErr ){ + sqlite3ValueSetNull(db->pErr); + } +} + +/* +** Add an error message to pParse->zErrMsg and increment pParse->nErr. +** The following formatting characters are allowed: +** +** %s Insert a string +** %z A string that should be freed after use +** %d Insert an integer +** %T Insert a token +** %S Insert the first element of a SrcList +** +** This function should be used to report any error that occurs whilst +** compiling an SQL statement (i.e. within sqlite3_prepare()). The +** last thing the sqlite3_prepare() function does is copy the error +** stored by this function into the database handle using sqlite3Error(). +** Function sqlite3Error() should be used during statement execution +** (sqlite3_step() etc.). +*/ +SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ + char *zMsg; + va_list ap; + sqlite3 *db = pParse->db; + va_start(ap, zFormat); + zMsg = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + if( db->suppressErr ){ + sqlite3DbFree(db, zMsg); + }else{ + pParse->nErr++; + sqlite3DbFree(db, pParse->zErrMsg); + pParse->zErrMsg = zMsg; + pParse->rc = SQLITE_ERROR; + } +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** The input string must be zero-terminated. A new zero-terminator +** is added to the dequoted string. +** +** The return value is -1 if no dequoting occurs or the length of the +** dequoted string, exclusive of the zero terminator, if dequoting does +** occur. +** +** 2002-Feb-14: This routine is extended to remove MS-Access style +** brackets from around identifers. For example: "[a-b-c]" becomes +** "a-b-c". +*/ +SQLITE_PRIVATE int sqlite3Dequote(char *z){ + char quote; + int i, j; + if( z==0 ) return -1; + quote = z[0]; + switch( quote ){ + case '\'': break; + case '"': break; + case '`': break; /* For MySQL compatibility */ + case '[': quote = ']'; break; /* For MS SqlServer compatibility */ + default: return -1; + } + for(i=1, j=0;; i++){ + assert( z[i] ); + if( z[i]==quote ){ + if( z[i+1]==quote ){ + z[j++] = quote; + i++; + }else{ + break; + } + }else{ + z[j++] = z[i]; + } + } + z[j] = 0; + return j; +} + +/* Convenient short-hand */ +#define UpperToLower sqlite3UpperToLower + +/* +** Some systems have stricmp(). Others have strcasecmp(). Because +** there is no consistency, we will define our own. +** +** IMPLEMENTATION-OF: R-30243-02494 The sqlite3_stricmp() and +** sqlite3_strnicmp() APIs allow applications and extensions to compare +** the contents of two buffers containing UTF-8 strings in a +** case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. +*/ +SQLITE_API int sqlite3_stricmp(const char *zLeft, const char *zRight){ + register unsigned char *a, *b; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } + return UpperToLower[*a] - UpperToLower[*b]; +} +SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ + register unsigned char *a, *b; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } + return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; +} + +/* +** The string z[] is an text representation of a real number. +** Convert this string to a double and write it into *pResult. +** +** The string z[] is length bytes in length (bytes, not characters) and +** uses the encoding enc. The string is not necessarily zero-terminated. +** +** Return TRUE if the result is a valid real number (or integer) and FALSE +** if the string is empty or contains extraneous text. Valid numbers +** are in one of these formats: +** +** [+-]digits[E[+-]digits] +** [+-]digits.[digits][E[+-]digits] +** [+-].digits[E[+-]digits] +** +** Leading and trailing whitespace is ignored for the purpose of determining +** validity. +** +** If some prefix of the input string is a valid number, this routine +** returns FALSE but it still converts the prefix and writes the result +** into *pResult. +*/ +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ +#ifndef SQLITE_OMIT_FLOATING_POINT + int incr; + const char *zEnd = z + length; + /* sign * significand * (10 ^ (esign * exponent)) */ + int sign = 1; /* sign of significand */ + i64 s = 0; /* significand */ + int d = 0; /* adjust exponent for shifting decimal point */ + int esign = 1; /* sign of exponent */ + int e = 0; /* exponent */ + int eValid = 1; /* True exponent is either not used or is well-formed */ + double result; + int nDigits = 0; + int nonNum = 0; + + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + *pResult = 0.0; /* Default return value, in case of an error */ + + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + int i; + incr = 2; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i=zEnd ) return 0; + + /* get sign of significand */ + if( *z=='-' ){ + sign = -1; + z+=incr; + }else if( *z=='+' ){ + z+=incr; + } + + /* skip leading zeroes */ + while( z=zEnd ) goto do_atof_calc; + + /* if decimal point is present */ + if( *z=='.' ){ + z+=incr; + /* copy digits from after decimal to significand + ** (decrease exponent by d to shift decimal right) */ + while( z=zEnd ) goto do_atof_calc; + + /* if exponent is present */ + if( *z=='e' || *z=='E' ){ + z+=incr; + eValid = 0; + if( z>=zEnd ) goto do_atof_calc; + /* get sign of exponent */ + if( *z=='-' ){ + esign = -1; + z+=incr; + }else if( *z=='+' ){ + z+=incr; + } + /* copy digits to exponent */ + while( z0 ){ + while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10; + }else{ + while( !(s%10) && e>0 ) e--,s/=10; + } + + /* adjust the sign of significand */ + s = sign<0 ? -s : s; + + /* if exponent, scale significand as appropriate + ** and store in result. */ + if( e ){ + LONGDOUBLE_TYPE scale = 1.0; + /* attempt to handle extremely small/large numbers better */ + if( e>307 && e<342 ){ + while( e%308 ) { scale *= 1.0e+1; e -= 1; } + if( esign<0 ){ + result = s / scale; + result /= 1.0e+308; + }else{ + result = s * scale; + result *= 1.0e+308; + } + }else if( e>=342 ){ + if( esign<0 ){ + result = 0.0*s; + }else{ + result = 1e308*1e308*s; /* Infinity */ + } + }else{ + /* 1.0e+22 is the largest power of 10 than can be + ** represented exactly. */ + while( e%22 ) { scale *= 1.0e+1; e -= 1; } + while( e>0 ) { scale *= 1.0e+22; e -= 22; } + if( esign<0 ){ + result = s / scale; + }else{ + result = s * scale; + } + } + } else { + result = (double)s; + } + } + + /* store the result */ + *pResult = result; + + /* return true if number and no extra non-whitespace chracters after */ + return z>=zEnd && nDigits>0 && eValid && nonNum==0; +#else + return !sqlite3Atoi64(z, pResult, length, enc); +#endif /* SQLITE_OMIT_FLOATING_POINT */ +} + +/* +** Compare the 19-character string zNum against the text representation +** value 2^63: 9223372036854775808. Return negative, zero, or positive +** if zNum is less than, equal to, or greater than the string. +** Note that zNum must contain exactly 19 characters. +** +** Unlike memcmp() this routine is guaranteed to return the difference +** in the values of the last digit if the only difference is in the +** last digit. So, for example, +** +** compare2pow63("9223372036854775800", 1) +** +** will return -8. +*/ +static int compare2pow63(const char *zNum, int incr){ + int c = 0; + int i; + /* 012345678901234567 */ + const char *pow63 = "922337203685477580"; + for(i=0; c==0 && i<18; i++){ + c = (zNum[i*incr]-pow63[i])*10; + } + if( c==0 ){ + c = zNum[18*incr] - '8'; + testcase( c==(-1) ); + testcase( c==0 ); + testcase( c==(+1) ); + } + return c; +} + +/* +** Convert zNum to a 64-bit signed integer. zNum must be decimal. This +** routine does *not* accept hexadecimal notation. +** +** If the zNum value is representable as a 64-bit twos-complement +** integer, then write that value into *pNum and return 0. +** +** If zNum is exactly 9223372036854775808, return 2. This special +** case is broken out because while 9223372036854775808 cannot be a +** signed 64-bit integer, its negative -9223372036854775808 can be. +** +** If zNum is too big for a 64-bit integer and is not +** 9223372036854775808 or if zNum contains any non-numeric text, +** then return 1. +** +** length is the number of bytes in the string (bytes, not characters). +** The string is not necessarily zero-terminated. The encoding is +** given by enc. +*/ +SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ + int incr; + u64 u = 0; + int neg = 0; /* assume positive */ + int i; + int c = 0; + int nonNum = 0; + const char *zStart; + const char *zEnd = zNum + length; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + incr = 2; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i='0' && c<='9'; i+=incr){ + u = u*10 + c - '0'; + } + if( u>LARGEST_INT64 ){ + *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; + }else if( neg ){ + *pNum = -(i64)u; + }else{ + *pNum = (i64)u; + } + testcase( i==18 ); + testcase( i==19 ); + testcase( i==20 ); + if( (c!=0 && &zNum[i]19*incr || nonNum ){ + /* zNum is empty or contains non-numeric text or is longer + ** than 19 digits (thus guaranteeing that it is too large) */ + return 1; + }else if( i<19*incr ){ + /* Less than 19 digits, so we know that it fits in 64 bits */ + assert( u<=LARGEST_INT64 ); + return 0; + }else{ + /* zNum is a 19-digit numbers. Compare it against 9223372036854775808. */ + c = compare2pow63(zNum, incr); + if( c<0 ){ + /* zNum is less than 9223372036854775808 so it fits */ + assert( u<=LARGEST_INT64 ); + return 0; + }else if( c>0 ){ + /* zNum is greater than 9223372036854775808 so it overflows */ + return 1; + }else{ + /* zNum is exactly 9223372036854775808. Fits if negative. The + ** special case 2 overflow if positive */ + assert( u-1==LARGEST_INT64 ); + return neg ? 0 : 2; + } + } +} + +/* +** Transform a UTF-8 integer literal, in either decimal or hexadecimal, +** into a 64-bit signed integer. This routine accepts hexadecimal literals, +** whereas sqlite3Atoi64() does not. +** +** Returns: +** +** 0 Successful transformation. Fits in a 64-bit signed integer. +** 1 Integer too large for a 64-bit signed integer or is malformed +** 2 Special case of 9223372036854775808 +*/ +SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' + && (z[1]=='x' || z[1]=='X') + && sqlite3Isxdigit(z[2]) + ){ + u64 u = 0; + int i, k; + for(i=2; z[i]=='0'; i++){} + for(k=i; sqlite3Isxdigit(z[k]); k++){ + u = u*16 + sqlite3HexToInt(z[k]); + } + memcpy(pOut, &u, 8); + return (z[k]==0 && k-i<=16) ? 0 : 1; + }else +#endif /* SQLITE_OMIT_HEX_INTEGER */ + { + return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); + } +} + +/* +** If zNum represents an integer that will fit in 32-bits, then set +** *pValue to that integer and return true. Otherwise return false. +** +** This routine accepts both decimal and hexadecimal notation for integers. +** +** Any non-numeric characters that following zNum are ignored. +** This is different from sqlite3Atoi64() which requires the +** input number to be zero-terminated. +*/ +SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ + sqlite_int64 v = 0; + int i, c; + int neg = 0; + if( zNum[0]=='-' ){ + neg = 1; + zNum++; + }else if( zNum[0]=='+' ){ + zNum++; + } +#ifndef SQLITE_OMIT_HEX_INTEGER + else if( zNum[0]=='0' + && (zNum[1]=='x' || zNum[1]=='X') + && sqlite3Isxdigit(zNum[2]) + ){ + u32 u = 0; + zNum += 2; + while( zNum[0]=='0' ) zNum++; + for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + u = u*16 + sqlite3HexToInt(zNum[i]); + } + if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ + memcpy(pValue, &u, 4); + return 1; + }else{ + return 0; + } + } +#endif + for(i=0; i<11 && (c = zNum[i] - '0')>=0 && c<=9; i++){ + v = v*10 + c; + } + + /* The longest decimal representation of a 32 bit integer is 10 digits: + ** + ** 1234567890 + ** 2^31 -> 2147483648 + */ + testcase( i==10 ); + if( i>10 ){ + return 0; + } + testcase( v-neg==2147483647 ); + if( v-neg>2147483647 ){ + return 0; + } + if( neg ){ + v = -v; + } + *pValue = (int)v; + return 1; +} + +/* +** Return a 32-bit integer value extracted from a string. If the +** string is not an integer, just return 0. +*/ +SQLITE_PRIVATE int sqlite3Atoi(const char *z){ + int x = 0; + if( z ) sqlite3GetInt32(z, &x); + return x; +} + +/* +** The variable-length integer encoding is as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** C = xxxxxxxx 8 bits of data +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** 28 bits - BBBA +** 35 bits - BBBBA +** 42 bits - BBBBBA +** 49 bits - BBBBBBA +** 56 bits - BBBBBBBA +** 64 bits - BBBBBBBBC +*/ + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data write will be between 1 and 9 bytes. The number +** of bytes written is returned. +** +** A variable-length integer consists of the lower 7 bits of each byte +** for all bytes that have the 8th bit set and one byte with the 8th +** bit clear. Except, if we get to the 9th byte, it stores the full +** 8 bits and is the last byte. +*/ +SQLITE_PRIVATE int sqlite3PutVarint(unsigned char *p, u64 v){ + int i, j, n; + u8 buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (u8)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + assert( n<=9 ); + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + +/* +** This routine is a faster version of sqlite3PutVarint() that only +** works for 32-bit positive integers and which is optimized for +** the common case of small integers. A MACRO version, putVarint32, +** is provided which inlines the single-byte case. All code should use +** the MACRO version as this function assumes the single-byte case has +** already been handled. +*/ +SQLITE_PRIVATE int sqlite3PutVarint32(unsigned char *p, u32 v){ +#ifndef putVarint32 + if( (v & ~0x7f)==0 ){ + p[0] = v; + return 1; + } +#endif + if( (v & ~0x3fff)==0 ){ + p[0] = (u8)((v>>7) | 0x80); + p[1] = (u8)(v & 0x7f); + return 2; + } + return sqlite3PutVarint(p, v); +} + +/* +** Bitmasks used by sqlite3GetVarint(). These precomputed constants +** are defined here rather than simply putting the constant expressions +** inline in order to work around bugs in the RVT compiler. +** +** SLOT_2_0 A mask for (0x7f<<14) | 0x7f +** +** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0 +*/ +#define SLOT_2_0 0x001fc07f +#define SLOT_4_2_0 0xf01fc07f + + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read. The value is stored in *v. +*/ +SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ + u32 a,b,s; + + a = *p; + /* a: p0 (unmasked) */ + if (!(a&0x80)) + { + *v = a; + return 1; + } + + p++; + b = *p; + /* b: p1 (unmasked) */ + if (!(b&0x80)) + { + a &= 0x7f; + a = a<<7; + a |= b; + *v = a; + return 2; + } + + /* Verify that constants are precomputed correctly */ + assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); + assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); + + p++; + a = a<<14; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_2_0; + b &= 0x7f; + b = b<<7; + a |= b; + *v = a; + return 3; + } + + /* CSE1 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p1<<14 | p3 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_2_0; + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + *v = a; + return 4; + } + + /* a: p0<<14 | p2 (masked) */ + /* b: p1<<14 | p3 (unmasked) */ + /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + b &= SLOT_2_0; + s = a; + /* s: p0<<14 | p2 (masked) */ + + p++; + a = a<<14; + a |= *p; + /* a: p0<<28 | p2<<14 | p4 (unmasked) */ + if (!(a&0x80)) + { + /* we can skip these cause they were (effectively) done above in calc'ing s */ + /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + /* b &= (0x7f<<14)|(0x7f); */ + b = b<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 5; + } + + /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + s = s<<7; + s |= b; + /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + + p++; + b = b<<14; + b |= *p; + /* b: p1<<28 | p3<<14 | p5 (unmasked) */ + if (!(b&0x80)) + { + /* we can skip this cause it was (effectively) done above in calc'ing s */ + /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + a &= SLOT_2_0; + a = a<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 6; + } + + p++; + a = a<<14; + a |= *p; + /* a: p2<<28 | p4<<14 | p6 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_4_2_0; + b &= SLOT_2_0; + b = b<<7; + a |= b; + s = s>>11; + *v = ((u64)s)<<32 | a; + return 7; + } + + /* CSE2 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p3<<28 | p5<<14 | p7 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_4_2_0; + /* moved CSE2 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + s = s>>4; + *v = ((u64)s)<<32 | a; + return 8; + } + + p++; + a = a<<15; + a |= *p; + /* a: p4<<29 | p6<<15 | p8 (unmasked) */ + + /* moved CSE2 up */ + /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ + b &= SLOT_2_0; + b = b<<8; + a |= b; + + s = s<<4; + b = p[-4]; + b &= 0x7f; + b = b>>3; + s |= b; + + *v = ((u64)s)<<32 | a; + + return 9; +} + +/* +** Read a 32-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read. The value is stored in *v. +** +** If the varint stored in p[0] is larger than can fit in a 32-bit unsigned +** integer, then set *v to 0xffffffff. +** +** A MACRO version, getVarint32, is provided which inlines the +** single-byte case. All code should use the MACRO version as +** this function assumes the single-byte case has already been handled. +*/ +SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ + u32 a,b; + + /* The 1-byte case. Overwhelmingly the most common. Handled inline + ** by the getVarin32() macro */ + a = *p; + /* a: p0 (unmasked) */ +#ifndef getVarint32 + if (!(a&0x80)) + { + /* Values between 0 and 127 */ + *v = a; + return 1; + } +#endif + + /* The 2-byte case */ + p++; + b = *p; + /* b: p1 (unmasked) */ + if (!(b&0x80)) + { + /* Values between 128 and 16383 */ + a &= 0x7f; + a = a<<7; + *v = a | b; + return 2; + } + + /* The 3-byte case */ + p++; + a = a<<14; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + /* Values between 16384 and 2097151 */ + a &= (0x7f<<14)|(0x7f); + b &= 0x7f; + b = b<<7; + *v = a | b; + return 3; + } + + /* A 32-bit varint is used to store size information in btrees. + ** Objects are rarely larger than 2MiB limit of a 3-byte varint. + ** A 3-byte varint is sufficient, for example, to record the size + ** of a 1048569-byte BLOB or string. + ** + ** We only unroll the first 1-, 2-, and 3- byte cases. The very + ** rare larger cases can be handled by the slower 64-bit varint + ** routine. + */ +#if 1 + { + u64 v64; + u8 n; + + p -= 2; + n = sqlite3GetVarint(p, &v64); + assert( n>3 && n<=9 ); + if( (v64 & SQLITE_MAX_U32)!=v64 ){ + *v = 0xffffffff; + }else{ + *v = (u32)v64; + } + return n; + } + +#else + /* For following code (kept for historical record only) shows an + ** unrolling for the 3- and 4-byte varint cases. This code is + ** slightly faster, but it is also larger and much harder to test. + */ + p++; + b = b<<14; + b |= *p; + /* b: p1<<14 | p3 (unmasked) */ + if (!(b&0x80)) + { + /* Values between 2097152 and 268435455 */ + b &= (0x7f<<14)|(0x7f); + a &= (0x7f<<14)|(0x7f); + a = a<<7; + *v = a | b; + return 4; + } + + p++; + a = a<<14; + a |= *p; + /* a: p0<<28 | p2<<14 | p4 (unmasked) */ + if (!(a&0x80)) + { + /* Values between 268435456 and 34359738367 */ + a &= SLOT_4_2_0; + b &= SLOT_4_2_0; + b = b<<7; + *v = a | b; + return 5; + } + + /* We can only reach this point when reading a corrupt database + ** file. In that case we are not in any hurry. Use the (relatively + ** slow) general-purpose sqlite3GetVarint() routine to extract the + ** value. */ + { + u64 v64; + u8 n; + + p -= 4; + n = sqlite3GetVarint(p, &v64); + assert( n>5 && n<=9 ); + *v = (u32)v64; + return n; + } +#endif +} + +/* +** Return the number of bytes that will be needed to store the given +** 64-bit integer. +*/ +SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 && ALWAYS(i<9) ); + return i; +} + + +/* +** Read or write a four-byte big-endian integer value. +*/ +SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ + testcase( p[0]&0x80 ); + return ((unsigned)p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3]; +} +SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ + p[0] = (u8)(v>>24); + p[1] = (u8)(v>>16); + p[2] = (u8)(v>>8); + p[3] = (u8)v; +} + + + +/* +** Translate a single byte of Hex into an integer. +** This routine only works if h really is a valid hexadecimal +** character: 0..9a..fA..F +*/ +SQLITE_PRIVATE u8 sqlite3HexToInt(int h){ + assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_ASCII + h += 9*(1&(h>>6)); +#endif +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#endif + return (u8)(h & 0xf); +} + +#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) +/* +** Convert a BLOB literal of the form "x'hhhhhh'" into its binary +** value. Return a pointer to its binary value. Space to hold the +** binary value has been obtained from malloc and must be freed by +** the calling routine. +*/ +SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ + char *zBlob; + int i; + + zBlob = (char *)sqlite3DbMallocRaw(db, n/2 + 1); + n--; + if( zBlob ){ + for(i=0; imagic; + if( magic!=SQLITE_MAGIC_OPEN ){ + if( sqlite3SafetyCheckSickOrOk(db) ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + logBadConnection("unopened"); + } + return 0; + }else{ + return 1; + } +} +SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ + u32 magic; + magic = db->magic; + if( magic!=SQLITE_MAGIC_SICK && + magic!=SQLITE_MAGIC_OPEN && + magic!=SQLITE_MAGIC_BUSY ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + logBadConnection("invalid"); + return 0; + }else{ + return 1; + } +} + +/* +** Attempt to add, substract, or multiply the 64-bit signed value iB against +** the other 64-bit signed integer at *pA and store the result in *pA. +** Return 0 on success. Or if the operation would have resulted in an +** overflow, leave *pA unchanged and return 1. +*/ +SQLITE_PRIVATE int sqlite3AddInt64(i64 *pA, i64 iB){ + i64 iA = *pA; + testcase( iA==0 ); testcase( iA==1 ); + testcase( iB==-1 ); testcase( iB==0 ); + if( iB>=0 ){ + testcase( iA>0 && LARGEST_INT64 - iA == iB ); + testcase( iA>0 && LARGEST_INT64 - iA == iB - 1 ); + if( iA>0 && LARGEST_INT64 - iA < iB ) return 1; + }else{ + testcase( iA<0 && -(iA + LARGEST_INT64) == iB + 1 ); + testcase( iA<0 && -(iA + LARGEST_INT64) == iB + 2 ); + if( iA<0 && -(iA + LARGEST_INT64) > iB + 1 ) return 1; + } + *pA += iB; + return 0; +} +SQLITE_PRIVATE int sqlite3SubInt64(i64 *pA, i64 iB){ + testcase( iB==SMALLEST_INT64+1 ); + if( iB==SMALLEST_INT64 ){ + testcase( (*pA)==(-1) ); testcase( (*pA)==0 ); + if( (*pA)>=0 ) return 1; + *pA -= iB; + return 0; + }else{ + return sqlite3AddInt64(pA, -iB); + } +} +#define TWOPOWER32 (((i64)1)<<32) +#define TWOPOWER31 (((i64)1)<<31) +SQLITE_PRIVATE int sqlite3MulInt64(i64 *pA, i64 iB){ + i64 iA = *pA; + i64 iA1, iA0, iB1, iB0, r; + + iA1 = iA/TWOPOWER32; + iA0 = iA % TWOPOWER32; + iB1 = iB/TWOPOWER32; + iB0 = iB % TWOPOWER32; + if( iA1==0 ){ + if( iB1==0 ){ + *pA *= iB; + return 0; + } + r = iA0*iB1; + }else if( iB1==0 ){ + r = iA1*iB0; + }else{ + /* If both iA1 and iB1 are non-zero, overflow will result */ + return 1; + } + testcase( r==(-TWOPOWER31)-1 ); + testcase( r==(-TWOPOWER31) ); + testcase( r==TWOPOWER31 ); + testcase( r==TWOPOWER31-1 ); + if( r<(-TWOPOWER31) || r>=TWOPOWER31 ) return 1; + r *= TWOPOWER32; + if( sqlite3AddInt64(&r, iA0*iB0) ) return 1; + *pA = r; + return 0; +} + +/* +** Compute the absolute value of a 32-bit signed integer, of possible. Or +** if the integer has a value of -2147483648, return +2147483647 +*/ +SQLITE_PRIVATE int sqlite3AbsInt32(int x){ + if( x>=0 ) return x; + if( x==(int)0x80000000 ) return 0x7fffffff; + return -x; +} + +#ifdef SQLITE_ENABLE_8_3_NAMES +/* +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa +*/ +SQLITE_PRIVATE void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBaseFilename, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = sqlite3Strlen30(z); + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); + } +} +#endif + +/* +** Find (an approximate) sum of two LogEst values. This computation is +** not a simple "+" operator because LogEst is stored as a logarithmic +** value. +** +*/ +SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ + static const unsigned char x[] = { + 10, 10, /* 0,1 */ + 9, 9, /* 2,3 */ + 8, 8, /* 4,5 */ + 7, 7, 7, /* 6,7,8 */ + 6, 6, 6, /* 9,10,11 */ + 5, 5, 5, /* 12-14 */ + 4, 4, 4, 4, /* 15-18 */ + 3, 3, 3, 3, 3, 3, /* 19-24 */ + 2, 2, 2, 2, 2, 2, 2, /* 25-31 */ + }; + if( a>=b ){ + if( a>b+49 ) return a; + if( a>b+31 ) return a+1; + return a+x[a-b]; + }else{ + if( b>a+49 ) return b; + if( b>a+31 ) return b+1; + return b+x[b-a]; + } +} + +/* +** Convert an integer into a LogEst. In other words, compute an +** approximation for 10*log2(x). +*/ +SQLITE_PRIVATE LogEst sqlite3LogEst(u64 x){ + static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; + LogEst y = 40; + if( x<8 ){ + if( x<2 ) return 0; + while( x<8 ){ y -= 10; x <<= 1; } + }else{ + while( x>255 ){ y += 40; x >>= 4; } + while( x>15 ){ y += 10; x >>= 1; } + } + return a[x&7] + y - 10; +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Convert a double into a LogEst +** In other words, compute an approximation for 10*log2(x). +*/ +SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double x){ + u64 a; + LogEst e; + assert( sizeof(x)==8 && sizeof(a)==8 ); + if( x<=1 ) return 0; + if( x<=2000000000 ) return sqlite3LogEst((u64)x); + memcpy(&a, &x, 8); + e = (a>>52) - 1022; + return e*10; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Convert a LogEst into an integer. +*/ +SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ + u64 n; + if( x<10 ) return 1; + n = x%10; + x /= 10; + if( n>=5 ) n -= 2; + else if( n>=1 ) n -= 1; + if( x>=3 ){ + return x>60 ? (u64)LARGEST_INT64 : (n+8)<<(x-3); + } + return (n+8)>>(3-x); +} + +/************** End of util.c ************************************************/ +/************** Begin file hash.c ********************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of generic hash-tables +** used in SQLite. +*/ +/* #include */ + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. +** +** "pNew" is a pointer to the hash table that is to be initialized. +*/ +SQLITE_PRIVATE void sqlite3HashInit(Hash *pNew){ + assert( pNew!=0 ); + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} + +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +SQLITE_PRIVATE void sqlite3HashClear(Hash *pH){ + HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + sqlite3_free(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + HashElem *next_elem = elem->next; + sqlite3_free(elem); + elem = next_elem; + } + pH->count = 0; +} + +/* +** The hashing function. +*/ +static unsigned int strHash(const char *z, int nKey){ + unsigned int h = 0; + assert( nKey>=0 ); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ sqlite3UpperToLower[(unsigned char)*z++]; + nKey--; + } + return h; +} + + +/* Link pNew element into the hash table pH. If pEntry!=0 then also +** insert pNew into the pEntry hash bucket. +*/ +static void insertElement( + Hash *pH, /* The complete hash table */ + struct _ht *pEntry, /* The entry into which pNew is inserted */ + HashElem *pNew /* The element to be inserted */ +){ + HashElem *pHead; /* First element already in pEntry */ + if( pEntry ){ + pHead = pEntry->count ? pEntry->chain : 0; + pEntry->count++; + pEntry->chain = pNew; + }else{ + pHead = 0; + } + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; + } +} + + +/* Resize the hash table so that it cantains "new_size" buckets. +** +** The hash table might fail to resize if sqlite3_malloc() fails or +** if the new size is the same as the prior size. +** Return TRUE if the resize occurs and false if not. +*/ +static int rehash(Hash *pH, unsigned int new_size){ + struct _ht *new_ht; /* The new hash table */ + HashElem *elem, *next_elem; /* For looping over existing elements */ + +#if SQLITE_MALLOC_SOFT_LIMIT>0 + if( new_size*sizeof(struct _ht)>SQLITE_MALLOC_SOFT_LIMIT ){ + new_size = SQLITE_MALLOC_SOFT_LIMIT/sizeof(struct _ht); + } + if( new_size==pH->htsize ) return 0; +#endif + + /* The inability to allocates space for a larger hash table is + ** a performance hit but it is not a fatal error. So mark the + ** allocation as a benign. Use sqlite3Malloc()/memset(0) instead of + ** sqlite3MallocZero() to make the allocation, as sqlite3MallocZero() + ** only zeroes the requested number of bytes whereas this module will + ** use the actual amount of space allocated for the hash table (which + ** may be larger than the requested amount). + */ + sqlite3BeginBenignMalloc(); + new_ht = (struct _ht *)sqlite3Malloc( new_size*sizeof(struct _ht) ); + sqlite3EndBenignMalloc(); + + if( new_ht==0 ) return 0; + sqlite3_free(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size = sqlite3MallocSize(new_ht)/sizeof(struct _ht); + memset(new_ht, 0, new_size*sizeof(struct _ht)); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + unsigned int h = strHash(elem->pKey, elem->nKey) % new_size; + next_elem = elem->next; + insertElement(pH, &new_ht[h], elem); + } + return 1; +} + +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. The hash for this key has +** already been computed and is passed as the 4th parameter. +*/ +static HashElem *findElementGivenHash( + const Hash *pH, /* The pH to be searched */ + const char *pKey, /* The key we are searching for */ + int nKey, /* Bytes in key (not counting zero terminator) */ + unsigned int h /* The hash for this key. */ +){ + HashElem *elem; /* Used to loop thru the element list */ + int count; /* Number of elements left to test */ + + if( pH->ht ){ + struct _ht *pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + }else{ + elem = pH->first; + count = pH->count; + } + while( count-- && ALWAYS(elem) ){ + if( elem->nKey==nKey && sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ + return elem; + } + elem = elem->next; + } + return 0; +} + +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void removeElementGivenHash( + Hash *pH, /* The pH containing "elem" */ + HashElem* elem, /* The element to be removed from the pH */ + unsigned int h /* Hash value for the element */ +){ + struct _ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; + }else{ + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + if( pH->ht ){ + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + pEntry->count--; + assert( pEntry->count>=0 ); + } + sqlite3_free( elem ); + pH->count--; + if( pH->count==0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + sqlite3HashClear(pH); + } +} + +/* Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +SQLITE_PRIVATE void *sqlite3HashFind(const Hash *pH, const char *pKey, int nKey){ + HashElem *elem; /* The element that matches key */ + unsigned int h; /* A hash on key */ + + assert( pH!=0 ); + assert( pKey!=0 ); + assert( nKey>=0 ); + if( pH->ht ){ + h = strHash(pKey, nKey) % pH->htsize; + }else{ + h = 0; + } + elem = findElementGivenHash(pH, pKey, nKey, h); + return elem ? elem->data : 0; +} + +/* Insert an element into the hash table pH. The key is pKey,nKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created and NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. +*/ +SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, void *data){ + unsigned int h; /* the hash of the key modulo hash table size */ + HashElem *elem; /* Used to loop thru the element list */ + HashElem *new_elem; /* New element added to the pH */ + + assert( pH!=0 ); + assert( pKey!=0 ); + assert( nKey>=0 ); + if( pH->htsize ){ + h = strHash(pKey, nKey) % pH->htsize; + }else{ + h = 0; + } + elem = findElementGivenHash(pH,pKey,nKey,h); + if( elem ){ + void *old_data = elem->data; + if( data==0 ){ + removeElementGivenHash(pH,elem,h); + }else{ + elem->data = data; + elem->pKey = pKey; + assert(nKey==elem->nKey); + } + return old_data; + } + if( data==0 ) return 0; + new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); + if( new_elem==0 ) return data; + new_elem->pKey = pKey; + new_elem->nKey = nKey; + new_elem->data = data; + pH->count++; + if( pH->count>=10 && pH->count > 2*pH->htsize ){ + if( rehash(pH, pH->count*2) ){ + assert( pH->htsize>0 ); + h = strHash(pKey, nKey) % pH->htsize; + } + } + if( pH->ht ){ + insertElement(pH, &pH->ht[h], new_elem); + }else{ + insertElement(pH, 0, new_elem); + } + return 0; +} + +/************** End of hash.c ************************************************/ +/************** Begin file opcodes.c *****************************************/ +/* Automatically generated. Do not edit */ +/* See the mkopcodec.awk script for details. */ +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) +#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) || defined(SQLITE_DEBUG) +# define OpHelp(X) "\0" X +#else +# define OpHelp(X) +#endif +SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ + static const char *const azName[] = { "?", + /* 1 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), + /* 2 */ "Savepoint" OpHelp(""), + /* 3 */ "AutoCommit" OpHelp(""), + /* 4 */ "Transaction" OpHelp(""), + /* 5 */ "SorterNext" OpHelp(""), + /* 6 */ "PrevIfOpen" OpHelp(""), + /* 7 */ "NextIfOpen" OpHelp(""), + /* 8 */ "Prev" OpHelp(""), + /* 9 */ "Next" OpHelp(""), + /* 10 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 11 */ "Checkpoint" OpHelp(""), + /* 12 */ "JournalMode" OpHelp(""), + /* 13 */ "Vacuum" OpHelp(""), + /* 14 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 15 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 16 */ "Goto" OpHelp(""), + /* 17 */ "Gosub" OpHelp(""), + /* 18 */ "Return" OpHelp(""), + /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), + /* 20 */ "InitCoroutine" OpHelp(""), + /* 21 */ "EndCoroutine" OpHelp(""), + /* 22 */ "Yield" OpHelp(""), + /* 23 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 24 */ "Halt" OpHelp(""), + /* 25 */ "Integer" OpHelp("r[P2]=P1"), + /* 26 */ "Int64" OpHelp("r[P2]=P4"), + /* 27 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 28 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 29 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 30 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 31 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 32 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 33 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 34 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 35 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 36 */ "CollSeq" OpHelp(""), + /* 37 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 38 */ "MustBeInt" OpHelp(""), + /* 39 */ "RealAffinity" OpHelp(""), + /* 40 */ "Permutation" OpHelp(""), + /* 41 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 42 */ "Jump" OpHelp(""), + /* 43 */ "Once" OpHelp(""), + /* 44 */ "If" OpHelp(""), + /* 45 */ "IfNot" OpHelp(""), + /* 46 */ "Column" OpHelp("r[P3]=PX"), + /* 47 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 48 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 49 */ "Count" OpHelp("r[P2]=count()"), + /* 50 */ "ReadCookie" OpHelp(""), + /* 51 */ "SetCookie" OpHelp(""), + /* 52 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 53 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 54 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 55 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 56 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 57 */ "SorterOpen" OpHelp(""), + /* 58 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 59 */ "Close" OpHelp(""), + /* 60 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 61 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 62 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 63 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 64 */ "Seek" OpHelp("intkey=r[P2]"), + /* 65 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 66 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 67 */ "Found" OpHelp("key=r[P3@P4]"), + /* 68 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 69 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 70 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 71 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), + /* 72 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), + /* 73 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 74 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), + /* 75 */ "Delete" OpHelp(""), + /* 76 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), + /* 77 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), + /* 78 */ "Ne" OpHelp("if r[P1]!=r[P3] goto P2"), + /* 79 */ "Eq" OpHelp("if r[P1]==r[P3] goto P2"), + /* 80 */ "Gt" OpHelp("if r[P1]>r[P3] goto P2"), + /* 81 */ "Le" OpHelp("if r[P1]<=r[P3] goto P2"), + /* 82 */ "Lt" OpHelp("if r[P1]=r[P3] goto P2"), + /* 84 */ "ResetCount" OpHelp(""), + /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 89 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 90 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 91 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 92 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 93 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 94 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 95 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 96 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), + /* 97 */ "String8" OpHelp("r[P2]='P4'"), + /* 98 */ "SorterData" OpHelp("r[P2]=data"), + /* 99 */ "RowKey" OpHelp("r[P2]=key"), + /* 100 */ "RowData" OpHelp("r[P2]=data"), + /* 101 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 102 */ "NullRow" OpHelp(""), + /* 103 */ "Last" OpHelp(""), + /* 104 */ "SorterSort" OpHelp(""), + /* 105 */ "Sort" OpHelp(""), + /* 106 */ "Rewind" OpHelp(""), + /* 107 */ "SorterInsert" OpHelp(""), + /* 108 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 109 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 110 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 111 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 112 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 113 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 114 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 115 */ "Destroy" OpHelp(""), + /* 116 */ "Clear" OpHelp(""), + /* 117 */ "ResetSorter" OpHelp(""), + /* 118 */ "CreateIndex" OpHelp("r[P2]=root iDb=P1"), + /* 119 */ "CreateTable" OpHelp("r[P2]=root iDb=P1"), + /* 120 */ "ParseSchema" OpHelp(""), + /* 121 */ "LoadAnalysis" OpHelp(""), + /* 122 */ "DropTable" OpHelp(""), + /* 123 */ "DropIndex" OpHelp(""), + /* 124 */ "DropTrigger" OpHelp(""), + /* 125 */ "IntegrityCk" OpHelp(""), + /* 126 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 127 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 128 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 129 */ "Program" OpHelp(""), + /* 130 */ "Param" OpHelp(""), + /* 131 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 132 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 133 */ "Real" OpHelp("r[P2]=P4"), + /* 134 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 135 */ "IfPos" OpHelp("if r[P1]>0 goto P2"), + /* 136 */ "IfNeg" OpHelp("r[P1]+=P3, if r[P1]<0 goto P2"), + /* 137 */ "IfZero" OpHelp("r[P1]+=P3, if r[P1]==0 goto P2"), + /* 138 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 139 */ "IncrVacuum" OpHelp(""), + /* 140 */ "Expire" OpHelp(""), + /* 141 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 142 */ "VBegin" OpHelp(""), + /* 143 */ "ToText" OpHelp(""), + /* 144 */ "ToBlob" OpHelp(""), + /* 145 */ "ToNumeric" OpHelp(""), + /* 146 */ "ToInt" OpHelp(""), + /* 147 */ "ToReal" OpHelp(""), + /* 148 */ "VCreate" OpHelp(""), + /* 149 */ "VDestroy" OpHelp(""), + /* 150 */ "VOpen" OpHelp(""), + /* 151 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 152 */ "VNext" OpHelp(""), + /* 153 */ "VRename" OpHelp(""), + /* 154 */ "Pagecount" OpHelp(""), + /* 155 */ "MaxPgcnt" OpHelp(""), + /* 156 */ "Init" OpHelp("Start at P2"), + /* 157 */ "Noop" OpHelp(""), + /* 158 */ "Explain" OpHelp(""), + }; + return azName[i]; +} +#endif + +/************** End of opcodes.c *********************************************/ +/************** Begin file os_unix.c *****************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains the VFS implementation for unix-like operating systems +** include Linux, MacOSX, *BSD, QNX, VxWorks, AIX, HPUX, and others. +** +** There are actually several different VFS implementations in this file. +** The differences are in the way that file locking is done. The default +** implementation uses Posix Advisory Locks. Alternative implementations +** use flock(), dot-files, various proprietary locking schemas, or simply +** skip locking all together. +** +** This source file is organized into divisions where the logic for various +** subfunctions is contained within the appropriate division. PLEASE +** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed +** in the correct division and should be clearly labeled. +** +** The layout of divisions is as follows: +** +** * General-purpose declarations and utility functions. +** * Unique file ID logic used by VxWorks. +** * Various locking primitive implementations (all except proxy locking): +** + for Posix Advisory Locks +** + for no-op locks +** + for dot-file locks +** + for flock() locking +** + for named semaphore locks (VxWorks only) +** + for AFP filesystem locks (MacOSX only) +** * sqlite3_file methods not associated with locking. +** * Definitions of sqlite3_io_methods objects for all locking +** methods plus "finder" functions for each locking method. +** * sqlite3_vfs method implementations. +** * Locking primitives for the proxy uber-locking-method. (MacOSX only) +** * Definitions of sqlite3_vfs objects for all locking methods +** plus implementations of sqlite3_os_init() and sqlite3_os_end(). +*/ +#if SQLITE_OS_UNIX /* This file is used on unix only */ + +/* +** There are various methods for file locking used for concurrency +** control: +** +** 1. POSIX locking (the default), +** 2. No locking, +** 3. Dot-file locking, +** 4. flock() locking, +** 5. AFP locking (OSX only), +** 6. Named POSIX semaphores (VXWorks only), +** 7. proxy locking. (OSX only) +** +** Styles 4, 5, and 7 are only available of SQLITE_ENABLE_LOCKING_STYLE +** is defined to 1. The SQLITE_ENABLE_LOCKING_STYLE also enables automatic +** selection of the appropriate locking style based on the filesystem +** where the database is located. +*/ +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif + +/* +** Define the OS_VXWORKS pre-processor macro to 1 if building on +** vxworks, or 0 otherwise. +*/ +#ifndef OS_VXWORKS +# if defined(__RTP__) || defined(_WRS_KERNEL) +# define OS_VXWORKS 1 +# else +# define OS_VXWORKS 0 +# endif +#endif + +/* +** standard include files. +*/ +#include +#include +#include +#include +/* #include */ +#include +#include +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +# include +#endif + +#if SQLITE_ENABLE_LOCKING_STYLE || OS_VXWORKS +# include +# if OS_VXWORKS +# include +# include +# else +# include +# include +# endif +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + +#if defined(__APPLE__) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) +# include +#endif + +#ifdef HAVE_UTIME +# include +#endif + +/* +** Allowed values of unixFile.fsFlags +*/ +#define SQLITE_FSFLAGS_IS_MSDOS 0x1 + +/* +** If we are to be thread-safe, include the pthreads header and define +** the SQLITE_UNIX_THREADS macro. +*/ +#if SQLITE_THREADSAFE +/* # include */ +# define SQLITE_UNIX_THREADS 1 +#endif + +/* +** Default permissions when creating a new file +*/ +#ifndef SQLITE_DEFAULT_FILE_PERMISSIONS +# define SQLITE_DEFAULT_FILE_PERMISSIONS 0644 +#endif + +/* +** Default permissions when creating auto proxy dir +*/ +#ifndef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +# define SQLITE_DEFAULT_PROXYDIR_PERMISSIONS 0755 +#endif + +/* +** Maximum supported path-length. +*/ +#define MAX_PATHNAME 512 + +/* +** Only set the lastErrno if the error code is a real error and not +** a normal expected return code of SQLITE_BUSY or SQLITE_OK +*/ +#define IS_LOCK_ERROR(x) ((x != SQLITE_OK) && (x != SQLITE_BUSY)) + +/* Forward references */ +typedef struct unixShm unixShm; /* Connection shared memory */ +typedef struct unixShmNode unixShmNode; /* Shared memory instance */ +typedef struct unixInodeInfo unixInodeInfo; /* An i-node */ +typedef struct UnixUnusedFd UnixUnusedFd; /* An unused file descriptor */ + +/* +** Sometimes, after a file handle is closed by SQLite, the file descriptor +** cannot be closed immediately. In these cases, instances of the following +** structure are used to store the file descriptor while waiting for an +** opportunity to either close or reuse it. +*/ +struct UnixUnusedFd { + int fd; /* File descriptor to close */ + int flags; /* Flags this file descriptor was opened with */ + UnixUnusedFd *pNext; /* Next unused file descriptor on same file */ +}; + +/* +** The unixFile structure is subclass of sqlite3_file specific to the unix +** VFS implementations. +*/ +typedef struct unixFile unixFile; +struct unixFile { + sqlite3_io_methods const *pMethod; /* Always the first entry */ + sqlite3_vfs *pVfs; /* The VFS that created this unixFile */ + unixInodeInfo *pInode; /* Info about locks on this inode */ + int h; /* The file descriptor */ + unsigned char eFileLock; /* The type of lock held on this fd */ + unsigned short int ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ + int lastErrno; /* The unix errno from last I/O error */ + void *lockingContext; /* Locking style specific state */ + UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ + const char *zPath; /* Name of the file */ + unixShm *pShm; /* Shared memory segment information */ + int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ +#endif +#ifdef __QNXNTO__ + int sectorSize; /* Device sector size */ + int deviceCharacteristics; /* Precomputed device characteristics */ +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + int openFlags; /* The flags specified at open() */ +#endif +#if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) + unsigned fsFlags; /* cached details from statfs() */ +#endif +#if OS_VXWORKS + struct vxworksFileId *pId; /* Unique file ID */ +#endif +#ifdef SQLITE_DEBUG + /* The next group of variables are used to track whether or not the + ** transaction counter in bytes 24-27 of database files are updated + ** whenever any part of the database changes. An assertion fault will + ** occur if a file is updated without also updating the transaction + ** counter. This test is made to avoid new problems similar to the + ** one described by ticket #3584. + */ + unsigned char transCntrChng; /* True if the transaction counter changed */ + unsigned char dbUpdate; /* True if any part of database file changed */ + unsigned char inNormalWrite; /* True if in a normal write operation */ + +#endif + +#ifdef SQLITE_TEST + /* In test mode, increase the size of this structure a bit so that + ** it is larger than the struct CrashFile defined in test6.c. + */ + char aPadding[32]; +#endif +}; + +/* This variable holds the process id (pid) from when the xRandomness() +** method was called. If xOpen() is called from a different process id, +** indicating that a fork() has occurred, the PRNG will be reset. +*/ +static int randomnessPid = 0; + +/* +** Allowed values for the unixFile.ctrlFlags bitmask: +*/ +#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ +#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ +#define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ +#ifndef SQLITE_DISABLE_DIRSYNC +# define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ +#else +# define UNIXFILE_DIRSYNC 0x00 +#endif +#define UNIXFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ +#define UNIXFILE_DELETE 0x20 /* Delete on close */ +#define UNIXFILE_URI 0x40 /* Filename might have query parameters */ +#define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings have been issued */ + +/* +** Include code that is common to all os_*.c files +*/ +/************** Include os_common.h in the middle of os_unix.c ***************/ +/************** Begin file os_common.h ***************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains macros and a little bit of code that is common to +** all of the platform-specific files (os_*.c) and is #included into those +** files. +** +** This file should be #included by the os_*.c files only. It is not a +** general purpose header file. +*/ +#ifndef _OS_COMMON_H_ +#define _OS_COMMON_H_ + +/* +** At least two bugs have slipped in because we changed the MEMORY_DEBUG +** macro to SQLITE_DEBUG and some older makefiles have not yet made the +** switch. The following code should catch this problem at compile-time. +*/ +#ifdef MEMORY_DEBUG +# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." +#endif + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X +#else +# define OSTRACE(X) +#endif + +/* +** Macros for performance tracing. Normally turned off. Only works +** on i486 hardware. +*/ +#ifdef SQLITE_PERFORMANCE_TRACE + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +/************** Include hwtime.h in the middle of os_common.h ****************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 class CPUs. +*/ +#ifndef _HWTIME_H_ +#define _HWTIME_H_ + +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; + } + +#elif (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + #error Need implementation of sqlite3Hwtime() for your platform. + + /* + ** To compile without implementing sqlite3Hwtime() for your platform, + ** you can remove the above #error and use the following + ** stub function. You will lose timing support for many + ** of the debugging and testing utilities, but it should at + ** least compile and run. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(_HWTIME_H_) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in os_common.h ******************/ + +static sqlite_uint64 g_start; +static sqlite_uint64 g_elapsed; +#define TIMER_START g_start=sqlite3Hwtime() +#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start +#define TIMER_ELAPSED g_elapsed +#else +#define TIMER_START +#define TIMER_END +#define TIMER_ELAPSED ((sqlite_uint64)0) +#endif + +/* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ +SQLITE_API int sqlite3_diskfull_pending = 0; +SQLITE_API int sqlite3_diskfull = 0; +#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) +#define SimulateIOError(CODE) \ + if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ + || sqlite3_io_error_pending-- == 1 ) \ + { local_ioerr(); CODE; } +static void local_ioerr(){ + IOTRACE(("IOERR\n")); + sqlite3_io_error_hit++; + if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; +} +#define SimulateDiskfullError(CODE) \ + if( sqlite3_diskfull_pending ){ \ + if( sqlite3_diskfull_pending == 1 ){ \ + local_ioerr(); \ + sqlite3_diskfull = 1; \ + sqlite3_io_error_hit = 1; \ + CODE; \ + }else{ \ + sqlite3_diskfull_pending--; \ + } \ + } +#else +#define SimulateIOErrorBenign(X) +#define SimulateIOError(A) +#define SimulateDiskfullError(A) +#endif + +/* +** When testing, keep a count of the number of open files. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_open_file_count = 0; +#define OpenCounter(X) sqlite3_open_file_count+=(X) +#else +#define OpenCounter(X) +#endif + +#endif /* !defined(_OS_COMMON_H_) */ + +/************** End of os_common.h *******************************************/ +/************** Continuing where we left off in os_unix.c ********************/ + +/* +** Define various macros that are missing from some systems. +*/ +#ifndef O_LARGEFILE +# define O_LARGEFILE 0 +#endif +#ifdef SQLITE_DISABLE_LFS +# undef O_LARGEFILE +# define O_LARGEFILE 0 +#endif +#ifndef O_NOFOLLOW +# define O_NOFOLLOW 0 +#endif +#ifndef O_BINARY +# define O_BINARY 0 +#endif + +/* +** The threadid macro resolves to the thread-id or to 0. Used for +** testing and debugging only. +*/ +#if SQLITE_THREADSAFE +#define threadid pthread_self() +#else +#define threadid 0 +#endif + +/* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + +/* +** Different Unix systems declare open() in different ways. Same use +** open(const char*,int,mode_t). Others use open(const char*,int,...). +** The difference is important when using a pointer to the function. +** +** The safest way to deal with the problem is to always use this wrapper +** which always has the same well-defined interface. +*/ +static int posixOpen(const char *zFile, int flags, int mode){ + return open(zFile, flags, mode); +} + +/* +** On some systems, calls to fchown() will trigger a message in a security +** log if they come from non-root processes. So avoid calling fchown() if +** we are not running as root. +*/ +static int posixFchown(int fd, uid_t uid, gid_t gid){ +#if OS_VXWORKS + return 0; +#else + return geteuid() ? 0 : fchown(fd,uid,gid); +#endif +} + +/* Forward reference */ +static int openDirectory(const char*, int*); +static int unixGetpagesize(void); + +/* +** Many system calls are accessed through pointer-to-functions so that +** they may be overridden at runtime to facilitate fault injection during +** testing and sandboxing. The following array holds the names and pointers +** to all overrideable system calls. +*/ +static struct unix_syscall { + const char *zName; /* Name of the system call */ + sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ + sqlite3_syscall_ptr pDefault; /* Default value */ +} aSyscall[] = { + { "open", (sqlite3_syscall_ptr)posixOpen, 0 }, +#define osOpen ((int(*)(const char*,int,int))aSyscall[0].pCurrent) + + { "close", (sqlite3_syscall_ptr)close, 0 }, +#define osClose ((int(*)(int))aSyscall[1].pCurrent) + + { "access", (sqlite3_syscall_ptr)access, 0 }, +#define osAccess ((int(*)(const char*,int))aSyscall[2].pCurrent) + + { "getcwd", (sqlite3_syscall_ptr)getcwd, 0 }, +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[3].pCurrent) + + { "stat", (sqlite3_syscall_ptr)stat, 0 }, +#define osStat ((int(*)(const char*,struct stat*))aSyscall[4].pCurrent) + +/* +** The DJGPP compiler environment looks mostly like Unix, but it +** lacks the fcntl() system call. So redefine fcntl() to be something +** that always succeeds. This means that locking does not occur under +** DJGPP. But it is DOS - what did you expect? +*/ +#ifdef __DJGPP__ + { "fstat", 0, 0 }, +#define osFstat(a,b,c) 0 +#else + { "fstat", (sqlite3_syscall_ptr)fstat, 0 }, +#define osFstat ((int(*)(int,struct stat*))aSyscall[5].pCurrent) +#endif + + { "ftruncate", (sqlite3_syscall_ptr)ftruncate, 0 }, +#define osFtruncate ((int(*)(int,off_t))aSyscall[6].pCurrent) + + { "fcntl", (sqlite3_syscall_ptr)fcntl, 0 }, +#define osFcntl ((int(*)(int,int,...))aSyscall[7].pCurrent) + + { "read", (sqlite3_syscall_ptr)read, 0 }, +#define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) + +#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) + { "pread", (sqlite3_syscall_ptr)pread, 0 }, +#else + { "pread", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPread ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[9].pCurrent) + +#if defined(USE_PREAD64) + { "pread64", (sqlite3_syscall_ptr)pread64, 0 }, +#else + { "pread64", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPread64 ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[10].pCurrent) + + { "write", (sqlite3_syscall_ptr)write, 0 }, +#define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) + +#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) + { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, +#else + { "pwrite", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPwrite ((ssize_t(*)(int,const void*,size_t,off_t))\ + aSyscall[12].pCurrent) + +#if defined(USE_PREAD64) + { "pwrite64", (sqlite3_syscall_ptr)pwrite64, 0 }, +#else + { "pwrite64", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ + aSyscall[13].pCurrent) + + { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) + +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + { "fallocate", (sqlite3_syscall_ptr)posix_fallocate, 0 }, +#else + { "fallocate", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osFallocate ((int(*)(int,off_t,off_t))aSyscall[15].pCurrent) + + { "unlink", (sqlite3_syscall_ptr)unlink, 0 }, +#define osUnlink ((int(*)(const char*))aSyscall[16].pCurrent) + + { "openDirectory", (sqlite3_syscall_ptr)openDirectory, 0 }, +#define osOpenDirectory ((int(*)(const char*,int*))aSyscall[17].pCurrent) + + { "mkdir", (sqlite3_syscall_ptr)mkdir, 0 }, +#define osMkdir ((int(*)(const char*,mode_t))aSyscall[18].pCurrent) + + { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, +#define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) + + { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, +#define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +#if HAVE_MREMAP + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) + { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, +#define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent) + +#endif + +}; /* End of the overrideable system calls */ + +/* +** This is the xSetSystemCall() method of sqlite3_vfs for all of the +** "unix" VFSes. Return SQLITE_OK opon successfully updating the +** system call pointer, or SQLITE_NOTFOUND if there is no configurable +** system call named zName. +*/ +static int unixSetSystemCall( + sqlite3_vfs *pNotUsed, /* The VFS pointer. Not used */ + const char *zName, /* Name of system call to override */ + sqlite3_syscall_ptr pNewFunc /* Pointer to new system call value */ +){ + unsigned int i; + int rc = SQLITE_NOTFOUND; + + UNUSED_PARAMETER(pNotUsed); + if( zName==0 ){ + /* If no zName is given, restore all system calls to their default + ** settings and return NULL + */ + rc = SQLITE_OK; + for(i=0; i=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break; + osClose(fd); + sqlite3_log(SQLITE_WARNING, + "attempt to open \"%s\" as file descriptor %d", z, fd); + fd = -1; + if( osOpen("/dev/null", f, m)<0 ) break; + } + if( fd>=0 ){ + if( m!=0 ){ + struct stat statbuf; + if( osFstat(fd, &statbuf)==0 + && statbuf.st_size==0 + && (statbuf.st_mode&0777)!=m + ){ + osFchmod(fd, m); + } + } +#if defined(FD_CLOEXEC) && (!defined(O_CLOEXEC) || O_CLOEXEC==0) + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); +#endif + } + return fd; +} + +/* +** Helper functions to obtain and relinquish the global mutex. The +** global mutex is used to protect the unixInodeInfo and +** vxworksFileId objects used by this file, all of which may be +** shared by multiple threads. +** +** Function unixMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() +** statements. e.g. +** +** unixEnterMutex() +** assert( unixMutexHeld() ); +** unixEnterLeave() +*/ +static void unixEnterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +static void unixLeaveMutex(void){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +#ifdef SQLITE_DEBUG +static int unixMutexHeld(void) { + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +#endif + + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +/* +** Helper function for printing out trace information from debugging +** binaries. This returns the string represetation of the supplied +** integer lock-type. +*/ +static const char *azFileLock(int eFileLock){ + switch( eFileLock ){ + case NO_LOCK: return "NONE"; + case SHARED_LOCK: return "SHARED"; + case RESERVED_LOCK: return "RESERVED"; + case PENDING_LOCK: return "PENDING"; + case EXCLUSIVE_LOCK: return "EXCLUSIVE"; + } + return "ERROR"; +} +#endif + +#ifdef SQLITE_LOCK_TRACE +/* +** Print out information about all locking operations. +** +** This routine is used for troubleshooting locks on multithreaded +** platforms. Enable by compiling with the -DSQLITE_LOCK_TRACE +** command-line option on the compiler. This code is normally +** turned off. +*/ +static int lockTrace(int fd, int op, struct flock *p){ + char *zOpName, *zType; + int s; + int savedErrno; + if( op==F_GETLK ){ + zOpName = "GETLK"; + }else if( op==F_SETLK ){ + zOpName = "SETLK"; + }else{ + s = osFcntl(fd, op, p); + sqlite3DebugPrintf("fcntl unknown %d %d %d\n", fd, op, s); + return s; + } + if( p->l_type==F_RDLCK ){ + zType = "RDLCK"; + }else if( p->l_type==F_WRLCK ){ + zType = "WRLCK"; + }else if( p->l_type==F_UNLCK ){ + zType = "UNLCK"; + }else{ + assert( 0 ); + } + assert( p->l_whence==SEEK_SET ); + s = osFcntl(fd, op, p); + savedErrno = errno; + sqlite3DebugPrintf("fcntl %d %d %s %s %d %d %d %d\n", + threadid, fd, zOpName, zType, (int)p->l_start, (int)p->l_len, + (int)p->l_pid, s); + if( s==(-1) && op==F_SETLK && (p->l_type==F_RDLCK || p->l_type==F_WRLCK) ){ + struct flock l2; + l2 = *p; + osFcntl(fd, F_GETLK, &l2); + if( l2.l_type==F_RDLCK ){ + zType = "RDLCK"; + }else if( l2.l_type==F_WRLCK ){ + zType = "WRLCK"; + }else if( l2.l_type==F_UNLCK ){ + zType = "UNLCK"; + }else{ + assert( 0 ); + } + sqlite3DebugPrintf("fcntl-failure-reason: %s %d %d %d\n", + zType, (int)l2.l_start, (int)l2.l_len, (int)l2.l_pid); + } + errno = savedErrno; + return s; +} +#undef osFcntl +#define osFcntl lockTrace +#endif /* SQLITE_LOCK_TRACE */ + +/* +** Retry ftruncate() calls that fail due to EINTR +*/ +static int robust_ftruncate(int h, sqlite3_int64 sz){ + int rc; + do{ rc = osFtruncate(h,sz); }while( rc<0 && errno==EINTR ); + return rc; +} + +/* +** This routine translates a standard POSIX errno code into something +** useful to the clients of the sqlite3 functions. Specifically, it is +** intended to translate a variety of "try again" errors into SQLITE_BUSY +** and a variety of "please close the file descriptor NOW" errors into +** SQLITE_IOERR +** +** Errors during initialization of locks, or file system support for locks, +** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. +*/ +static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { + switch (posixError) { +#if 0 + /* At one point this code was not commented out. In theory, this branch + ** should never be hit, as this function should only be called after + ** a locking-related function (i.e. fcntl()) has returned non-zero with + ** the value of errno as the first argument. Since a system call has failed, + ** errno should be non-zero. + ** + ** Despite this, if errno really is zero, we still don't want to return + ** SQLITE_OK. The system call failed, and *some* SQLite error should be + ** propagated back to the caller. Commenting this branch out means errno==0 + ** will be handled by the "default:" case below. + */ + case 0: + return SQLITE_OK; +#endif + + case EAGAIN: + case ETIMEDOUT: + case EBUSY: + case EINTR: + case ENOLCK: + /* random NFS retry error, unless during file system support + * introspection, in which it actually means what it says */ + return SQLITE_BUSY; + + case EACCES: + /* EACCES is like EAGAIN during locking operations, but not any other time*/ + if( (sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + (sqliteIOErr == SQLITE_IOERR_RDLOCK) || + (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ){ + return SQLITE_BUSY; + } + /* else fall through */ + case EPERM: + return SQLITE_PERM; + +#if EOPNOTSUPP!=ENOTSUP + case EOPNOTSUPP: + /* something went terribly awry, unless during file system support + * introspection, in which it actually means what it says */ +#endif +#ifdef ENOTSUP + case ENOTSUP: + /* invalid fd, unless during file system support introspection, in which + * it actually means what it says */ +#endif + case EIO: + case EBADF: + case EINVAL: + case ENOTCONN: + case ENODEV: + case ENXIO: + case ENOENT: +#ifdef ESTALE /* ESTALE is not defined on Interix systems */ + case ESTALE: +#endif + case ENOSYS: + /* these should force the client to close the file and reconnect */ + + default: + return sqliteIOErr; + } +} + + +/****************************************************************************** +****************** Begin Unique File ID Utility Used By VxWorks *************** +** +** On most versions of unix, we can get a unique ID for a file by concatenating +** the device number and the inode number. But this does not work on VxWorks. +** On VxWorks, a unique file id must be based on the canonical filename. +** +** A pointer to an instance of the following structure can be used as a +** unique file ID in VxWorks. Each instance of this structure contains +** a copy of the canonical filename. There is also a reference count. +** The structure is reclaimed when the number of pointers to it drops to +** zero. +** +** There are never very many files open at one time and lookups are not +** a performance-critical path, so it is sufficient to put these +** structures on a linked list. +*/ +struct vxworksFileId { + struct vxworksFileId *pNext; /* Next in a list of them all */ + int nRef; /* Number of references to this one */ + int nName; /* Length of the zCanonicalName[] string */ + char *zCanonicalName; /* Canonical filename */ +}; + +#if OS_VXWORKS +/* +** All unique filenames are held on a linked list headed by this +** variable: +*/ +static struct vxworksFileId *vxworksFileList = 0; + +/* +** Simplify a filename into its canonical form +** by making the following changes: +** +** * removing any trailing and duplicate / +** * convert /./ into just / +** * convert /A/../ where A is any simple name into just / +** +** Changes are made in-place. Return the new name length. +** +** The original filename is in z[0..n-1]. Return the number of +** characters in the simplified name. +*/ +static int vxworksSimplifyName(char *z, int n){ + int i, j; + while( n>1 && z[n-1]=='/' ){ n--; } + for(i=j=0; i0 && z[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; + } + } + z[j++] = z[i]; + } + z[j] = 0; + return j; +} + +/* +** Find a unique file ID for the given absolute pathname. Return +** a pointer to the vxworksFileId object. This pointer is the unique +** file ID. +** +** The nRef field of the vxworksFileId object is incremented before +** the object is returned. A new vxworksFileId object is created +** and added to the global list if necessary. +** +** If a memory allocation error occurs, return NULL. +*/ +static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ + struct vxworksFileId *pNew; /* search key and new file ID */ + struct vxworksFileId *pCandidate; /* For looping over existing file IDs */ + int n; /* Length of zAbsoluteName string */ + + assert( zAbsoluteName[0]=='/' ); + n = (int)strlen(zAbsoluteName); + pNew = sqlite3_malloc( sizeof(*pNew) + (n+1) ); + if( pNew==0 ) return 0; + pNew->zCanonicalName = (char*)&pNew[1]; + memcpy(pNew->zCanonicalName, zAbsoluteName, n+1); + n = vxworksSimplifyName(pNew->zCanonicalName, n); + + /* Search for an existing entry that matching the canonical name. + ** If found, increment the reference count and return a pointer to + ** the existing file ID. + */ + unixEnterMutex(); + for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ + if( pCandidate->nName==n + && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 + ){ + sqlite3_free(pNew); + pCandidate->nRef++; + unixLeaveMutex(); + return pCandidate; + } + } + + /* No match was found. We will make a new file ID */ + pNew->nRef = 1; + pNew->nName = n; + pNew->pNext = vxworksFileList; + vxworksFileList = pNew; + unixLeaveMutex(); + return pNew; +} + +/* +** Decrement the reference count on a vxworksFileId object. Free +** the object when the reference count reaches zero. +*/ +static void vxworksReleaseFileId(struct vxworksFileId *pId){ + unixEnterMutex(); + assert( pId->nRef>0 ); + pId->nRef--; + if( pId->nRef==0 ){ + struct vxworksFileId **pp; + for(pp=&vxworksFileList; *pp && *pp!=pId; pp = &((*pp)->pNext)){} + assert( *pp==pId ); + *pp = pId->pNext; + sqlite3_free(pId); + } + unixLeaveMutex(); +} +#endif /* OS_VXWORKS */ +/*************** End of Unique File ID Utility Used By VxWorks **************** +******************************************************************************/ + + +/****************************************************************************** +*************************** Posix Advisory Locking **************************** +** +** POSIX advisory locks are broken by design. ANSI STD 1003.1 (1996) +** section 6.5.2.2 lines 483 through 490 specify that when a process +** sets or clears a lock, that operation overrides any prior locks set +** by the same process. It does not explicitly say so, but this implies +** that it overrides locks set by the same process using a different +** file descriptor. Consider this test case: +** +** int fd1 = open("./file1", O_RDWR|O_CREAT, 0644); +** int fd2 = open("./file2", O_RDWR|O_CREAT, 0644); +** +** Suppose ./file1 and ./file2 are really the same file (because +** one is a hard or symbolic link to the other) then if you set +** an exclusive lock on fd1, then try to get an exclusive lock +** on fd2, it works. I would have expected the second lock to +** fail since there was already a lock on the file due to fd1. +** But not so. Since both locks came from the same process, the +** second overrides the first, even though they were on different +** file descriptors opened on different file names. +** +** This means that we cannot use POSIX locks to synchronize file access +** among competing threads of the same process. POSIX locks will work fine +** to synchronize access for threads in separate processes, but not +** threads within the same process. +** +** To work around the problem, SQLite has to manage file locks internally +** on its own. Whenever a new database is opened, we have to find the +** specific inode of the database file (the inode is determined by the +** st_dev and st_ino fields of the stat structure that fstat() fills in) +** and check for locks already existing on that inode. When locks are +** created or removed, we have to look at our own internal record of the +** locks to see if another thread has previously set a lock on that same +** inode. +** +** (Aside: The use of inode numbers as unique IDs does not work on VxWorks. +** For VxWorks, we have to use the alternative unique ID system based on +** canonical filename and implemented in the previous division.) +** +** The sqlite3_file structure for POSIX is no longer just an integer file +** descriptor. It is now a structure that holds the integer file +** descriptor and a pointer to a structure that describes the internal +** locks on the corresponding inode. There is one locking structure +** per inode, so if the same inode is opened twice, both unixFile structures +** point to the same locking structure. The locking structure keeps +** a reference count (so we will know when to delete it) and a "cnt" +** field that tells us its internal lock status. cnt==0 means the +** file is unlocked. cnt==-1 means the file has an exclusive lock. +** cnt>0 means there are cnt shared locks on the file. +** +** Any attempt to lock or unlock a file first checks the locking +** structure. The fcntl() system call is only invoked to set a +** POSIX lock if the internal lock structure transitions between +** a locked and an unlocked state. +** +** But wait: there are yet more problems with POSIX advisory locks. +** +** If you close a file descriptor that points to a file that has locks, +** all locks on that file that are owned by the current process are +** released. To work around this problem, each unixInodeInfo object +** maintains a count of the number of pending locks on tha inode. +** When an attempt is made to close an unixFile, if there are +** other unixFile open on the same inode that are holding locks, the call +** to close() the file descriptor is deferred until all of the locks clear. +** The unixInodeInfo structure keeps a list of file descriptors that need to +** be closed and that list is walked (and cleared) when the last lock +** clears. +** +** Yet another problem: LinuxThreads do not play well with posix locks. +** +** Many older versions of linux use the LinuxThreads library which is +** not posix compliant. Under LinuxThreads, a lock created by thread +** A cannot be modified or overridden by a different thread B. +** Only thread A can modify the lock. Locking behavior is correct +** if the appliation uses the newer Native Posix Thread Library (NPTL) +** on linux - with NPTL a lock created by thread A can override locks +** in thread B. But there is no way to know at compile-time which +** threading library is being used. So there is no way to know at +** compile-time whether or not thread A can override locks on thread B. +** One has to do a run-time check to discover the behavior of the +** current process. +** +** SQLite used to support LinuxThreads. But support for LinuxThreads +** was dropped beginning with version 3.7.0. SQLite will still work with +** LinuxThreads provided that (1) there is no more than one connection +** per database file in the same process and (2) database connections +** do not move across threads. +*/ + +/* +** An instance of the following structure serves as the key used +** to locate a particular unixInodeInfo object. +*/ +struct unixFileId { + dev_t dev; /* Device number */ +#if OS_VXWORKS + struct vxworksFileId *pId; /* Unique file ID for vxworks. */ +#else + ino_t ino; /* Inode number */ +#endif +}; + +/* +** An instance of the following structure is allocated for each open +** inode. Or, on LinuxThreads, there is one of these structures for +** each inode opened by each thread. +** +** A single inode can have multiple file descriptors, so each unixFile +** structure contains a pointer to an instance of this object and this +** object keeps a count of the number of unixFile pointing to it. +*/ +struct unixInodeInfo { + struct unixFileId fileId; /* The lookup key */ + int nShared; /* Number of SHARED locks held */ + unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ + unsigned char bProcessLock; /* An exclusive process lock is held */ + int nRef; /* Number of pointers to this structure */ + unixShmNode *pShmNode; /* Shared memory associated with this inode */ + int nLock; /* Number of outstanding file locks */ + UnixUnusedFd *pUnused; /* Unused file descriptors to close */ + unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ + unixInodeInfo *pPrev; /* .... doubly linked */ +#if SQLITE_ENABLE_LOCKING_STYLE + unsigned long long sharedByte; /* for AFP simulated shared lock */ +#endif +#if OS_VXWORKS + sem_t *pSem; /* Named POSIX semaphore */ + char aSemName[MAX_PATHNAME+2]; /* Name of that semaphore */ +#endif +}; + +/* +** A lists of all unixInodeInfo objects. +*/ +static unixInodeInfo *inodeList = 0; + +/* +** +** This function - unixLogError_x(), is only ever called via the macro +** unixLogError(). +** +** It is invoked after an error occurs in an OS function and errno has been +** set. It logs a message using sqlite3_log() containing the current value of +** errno and, if possible, the human-readable equivalent from strerror() or +** strerror_r(). +** +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed (e.g. "unlink", "open") and the associated file-system path, +** if any. +*/ +#define unixLogError(a,b,c) unixLogErrorAtLine(a,b,c,__LINE__) +static int unixLogErrorAtLine( + int errcode, /* SQLite error code */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char *zErr; /* Message from strerror() or equivalent */ + int iErrno = errno; /* Saved syscall error number */ + + /* If this is not a threadsafe build (SQLITE_THREADSAFE==0), then use + ** the strerror() function to obtain the human-readable error message + ** equivalent to errno. Otherwise, use strerror_r(). + */ +#if SQLITE_THREADSAFE && defined(HAVE_STRERROR_R) + char aErr[80]; + memset(aErr, 0, sizeof(aErr)); + zErr = aErr; + + /* If STRERROR_R_CHAR_P (set by autoconf scripts) or __USE_GNU is defined, + ** assume that the system provides the GNU version of strerror_r() that + ** returns a pointer to a buffer containing the error message. That pointer + ** may point to aErr[], or it may point to some static storage somewhere. + ** Otherwise, assume that the system provides the POSIX version of + ** strerror_r(), which always writes an error message into aErr[]. + ** + ** If the code incorrectly assumes that it is the POSIX version that is + ** available, the error message will often be an empty string. Not a + ** huge problem. Incorrectly concluding that the GNU version is available + ** could lead to a segfault though. + */ +#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) + zErr = +# endif + strerror_r(iErrno, aErr, sizeof(aErr)-1); + +#elif SQLITE_THREADSAFE + /* This is a threadsafe build, but strerror_r() is not available. */ + zErr = ""; +#else + /* Non-threadsafe build, use strerror(). */ + zErr = strerror(iErrno); +#endif + + if( zPath==0 ) zPath = ""; + sqlite3_log(errcode, + "os_unix.c:%d: (%d) %s(%s) - %s", + iLine, iErrno, zFunc, zPath, zErr + ); + + return errcode; +} + +/* +** Close a file descriptor. +** +** We assume that close() almost always works, since it is only in a +** very sick application or on a very sick platform that it might fail. +** If it does fail, simply leak the file descriptor, but do log the +** error. +** +** Note that it is not safe to retry close() after EINTR since the +** file descriptor might have already been reused by another thread. +** So we don't even try to recover from an EINTR. Just log the error +** and move on. +*/ +static void robust_close(unixFile *pFile, int h, int lineno){ + if( osClose(h) ){ + unixLogErrorAtLine(SQLITE_IOERR_CLOSE, "close", + pFile ? pFile->zPath : 0, lineno); + } +} + +/* +** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. +*/ +static void closePendingFds(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + UnixUnusedFd *p; + UnixUnusedFd *pNext; + for(p=pInode->pUnused; p; p=pNext){ + pNext = p->pNext; + robust_close(pFile, p->fd, __LINE__); + sqlite3_free(p); + } + pInode->pUnused = 0; +} + +/* +** Release a unixInodeInfo structure previously allocated by findInodeInfo(). +** +** The mutex entered using the unixEnterMutex() function must be held +** when this function is called. +*/ +static void releaseInodeInfo(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + assert( unixMutexHeld() ); + if( ALWAYS(pInode) ){ + pInode->nRef--; + if( pInode->nRef==0 ){ + assert( pInode->pShmNode==0 ); + closePendingFds(pFile); + if( pInode->pPrev ){ + assert( pInode->pPrev->pNext==pInode ); + pInode->pPrev->pNext = pInode->pNext; + }else{ + assert( inodeList==pInode ); + inodeList = pInode->pNext; + } + if( pInode->pNext ){ + assert( pInode->pNext->pPrev==pInode ); + pInode->pNext->pPrev = pInode->pPrev; + } + sqlite3_free(pInode); + } + } +} + +/* +** Given a file descriptor, locate the unixInodeInfo object that +** describes that file descriptor. Create a new one if necessary. The +** return value might be uninitialized if an error occurs. +** +** The mutex entered using the unixEnterMutex() function must be held +** when this function is called. +** +** Return an appropriate error code. +*/ +static int findInodeInfo( + unixFile *pFile, /* Unix file with file desc used in the key */ + unixInodeInfo **ppInode /* Return the unixInodeInfo object here */ +){ + int rc; /* System call return code */ + int fd; /* The file descriptor for pFile */ + struct unixFileId fileId; /* Lookup key for the unixInodeInfo */ + struct stat statbuf; /* Low-level file information */ + unixInodeInfo *pInode = 0; /* Candidate unixInodeInfo object */ + + assert( unixMutexHeld() ); + + /* Get low-level information about the file that we can used to + ** create a unique name for the file. + */ + fd = pFile->h; + rc = osFstat(fd, &statbuf); + if( rc!=0 ){ + pFile->lastErrno = errno; +#ifdef EOVERFLOW + if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; +#endif + return SQLITE_IOERR; + } + +#ifdef __APPLE__ + /* On OS X on an msdos filesystem, the inode number is reported + ** incorrectly for zero-size files. See ticket #3260. To work + ** around this problem (we consider it a bug in OS X, not SQLite) + ** we always increase the file size to 1 by writing a single byte + ** prior to accessing the inode number. The one byte written is + ** an ASCII 'S' character which also happens to be the first byte + ** in the header of every SQLite database. In this way, if there + ** is a race condition such that another thread has already populated + ** the first page of the database, no damage is done. + */ + if( statbuf.st_size==0 && (pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS)!=0 ){ + do{ rc = osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); + if( rc!=1 ){ + pFile->lastErrno = errno; + return SQLITE_IOERR; + } + rc = osFstat(fd, &statbuf); + if( rc!=0 ){ + pFile->lastErrno = errno; + return SQLITE_IOERR; + } + } +#endif + + memset(&fileId, 0, sizeof(fileId)); + fileId.dev = statbuf.st_dev; +#if OS_VXWORKS + fileId.pId = pFile->pId; +#else + fileId.ino = statbuf.st_ino; +#endif + pInode = inodeList; + while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ + pInode = pInode->pNext; + } + if( pInode==0 ){ + pInode = sqlite3_malloc( sizeof(*pInode) ); + if( pInode==0 ){ + return SQLITE_NOMEM; + } + memset(pInode, 0, sizeof(*pInode)); + memcpy(&pInode->fileId, &fileId, sizeof(fileId)); + pInode->nRef = 1; + pInode->pNext = inodeList; + pInode->pPrev = 0; + if( inodeList ) inodeList->pPrev = pInode; + inodeList = pInode; + }else{ + pInode->nRef++; + } + *ppInode = pInode; + return SQLITE_OK; +} + +/* +** Return TRUE if pFile has been renamed or unlinked since it was first opened. +*/ +static int fileHasMoved(unixFile *pFile){ +#if OS_VXWORKS + return pFile->pInode!=0 && pFile->pId!=pFile->pInode->fileId.pId; +#else + struct stat buf; + return pFile->pInode!=0 && + (osStat(pFile->zPath, &buf)!=0 || buf.st_ino!=pFile->pInode->fileId.ino); +#endif +} + + +/* +** Check a unixFile that is a database. Verify the following: +** +** (1) There is exactly one hard link on the file +** (2) The file is not a symbolic link +** (3) The file has not been renamed or unlinked +** +** Issue sqlite3_log(SQLITE_WARNING,...) messages if anything is not right. +*/ +static void verifyDbFile(unixFile *pFile){ + struct stat buf; + int rc; + if( pFile->ctrlFlags & UNIXFILE_WARNED ){ + /* One or more of the following warnings have already been issued. Do not + ** repeat them so as not to clutter the error log */ + return; + } + rc = osFstat(pFile->h, &buf); + if( rc!=0 ){ + sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink>1 ){ + sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( fileHasMoved(pFile) ){ + sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } +} + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + unixEnterMutex(); /* Because pFile->pInode is shared across threads */ + + /* Check if a thread in this process holds such a lock */ + if( pFile->pInode->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. + */ +#ifndef __DJGPP__ + if( !reserved && !pFile->pInode->bProcessLock ){ + struct flock lock; + lock.l_whence = SEEK_SET; + lock.l_start = RESERVED_BYTE; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pFile->h, F_GETLK, &lock) ){ + rc = SQLITE_IOERR_CHECKRESERVEDLOCK; + pFile->lastErrno = errno; + } else if( lock.l_type!=F_UNLCK ){ + reserved = 1; + } + } +#endif + + unixLeaveMutex(); + OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; +} + +/* +** Attempt to set a system-lock on the file pFile. The lock is +** described by pLock. +** +** If the pFile was opened read/write from unix-excl, then the only lock +** ever obtained is an exclusive lock, and it is obtained exactly once +** the first time any lock is attempted. All subsequent system locking +** operations become no-ops. Locking operations still happen internally, +** in order to coordinate access between separate database connections +** within this process, but all of that is handled in memory and the +** operating system does not participate. +** +** This function is a pass-through to fcntl(F_SETLK) if pFile is using +** any VFS other than "unix-excl" or if pFile is opened on "unix-excl" +** and is read-only. +** +** Zero is returned if the call completes successfully, or -1 if a call +** to fcntl() fails. In this case, errno is set appropriately (by fcntl()). +*/ +static int unixFileLock(unixFile *pFile, struct flock *pLock){ + int rc; + unixInodeInfo *pInode = pFile->pInode; + assert( unixMutexHeld() ); + assert( pInode!=0 ); + if( ((pFile->ctrlFlags & UNIXFILE_EXCL)!=0 || pInode->bProcessLock) + && ((pFile->ctrlFlags & UNIXFILE_RDONLY)==0) + ){ + if( pInode->bProcessLock==0 ){ + struct flock lock; + assert( pInode->nLock==0 ); + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + lock.l_type = F_WRLCK; + rc = osFcntl(pFile->h, F_SETLK, &lock); + if( rc<0 ) return rc; + pInode->bProcessLock = 1; + pInode->nLock++; + }else{ + rc = 0; + } + }else{ + rc = osFcntl(pFile->h, F_SETLK, pLock); + } + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int unixLock(sqlite3_file *id, int eFileLock){ + /* The following describes the implementation of the various locks and + ** lock transitions in terms of the POSIX advisory shared and exclusive + ** lock primitives (called read-locks and write-locks below, to avoid + ** confusion with SQLite lock names). The algorithms are complicated + ** slightly in order to be compatible with windows systems simultaneously + ** accessing the same database file, in case that is ever required. + ** + ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved + ** byte', each single bytes at well known offsets, and the 'shared byte + ** range', a range of 510 bytes at a well known offset. + ** + ** To obtain a SHARED lock, a read-lock is obtained on the 'pending + ** byte'. If this is successful, a random byte from the 'shared byte + ** range' is read-locked and the lock on the 'pending byte' released. + ** + ** A process may only obtain a RESERVED lock after it has a SHARED lock. + ** A RESERVED lock is implemented by grabbing a write-lock on the + ** 'reserved byte'. + ** + ** A process may only obtain a PENDING lock after it has obtained a + ** SHARED lock. A PENDING lock is implemented by obtaining a write-lock + ** on the 'pending byte'. This ensures that no new SHARED locks can be + ** obtained, but existing SHARED locks are allowed to persist. A process + ** does not have to obtain a RESERVED lock on the way to a PENDING lock. + ** This property is used by the algorithm for rolling back a journal file + ** after a crash. + ** + ** An EXCLUSIVE lock, obtained after a PENDING lock is held, is + ** implemented by obtaining a write-lock on the entire 'shared byte + ** range'. Since all other locks require a read-lock on one of the bytes + ** within this range, this ensures that no other locks are held on the + ** database. + ** + ** The reason a single byte cannot be used instead of the 'shared byte + ** range' is that some versions of windows do not support read-locks. By + ** locking a random byte from a range, concurrent SHARED locks may exist + ** even if the locking primitive used is always a write-lock. + */ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + struct flock lock; + int tErrno = 0; + + assert( pFile ); + OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, + azFileLock(eFileLock), azFileLock(pFile->eFileLock), + azFileLock(pFile->pInode->eFileLock), pFile->pInode->nShared , getpid())); + + /* If there is already a lock of this type or more restrictive on the + ** unixFile, do nothing. Don't use the end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. + */ + if( pFile->eFileLock>=eFileLock ){ + OSTRACE(("LOCK %d %s ok (already held) (unix)\n", pFile->h, + azFileLock(eFileLock))); + return SQLITE_OK; + } + + /* Make sure the locking sequence is correct. + ** (1) We never move from unlocked to anything higher than shared lock. + ** (2) SQLite never explicitly requests a pendig lock. + ** (3) A shared lock is always held when a reserve lock is requested. + */ + assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); + assert( eFileLock!=PENDING_LOCK ); + assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); + + /* This mutex is needed because pFile->pInode is shared across threads + */ + unixEnterMutex(); + pInode = pFile->pInode; + + /* If some thread using this PID has a lock via a different unixFile* + ** handle that precludes the requested lock, return BUSY. + */ + if( (pFile->eFileLock!=pInode->eFileLock && + (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) + ){ + rc = SQLITE_BUSY; + goto end_lock; + } + + /* If a SHARED lock is requested, and some thread using this PID already + ** has a SHARED or RESERVED lock, then increment reference counts and + ** return SQLITE_OK. + */ + if( eFileLock==SHARED_LOCK && + (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ + assert( eFileLock==SHARED_LOCK ); + assert( pFile->eFileLock==0 ); + assert( pInode->nShared>0 ); + pFile->eFileLock = SHARED_LOCK; + pInode->nShared++; + pInode->nLock++; + goto end_lock; + } + + + /* A PENDING lock is needed before acquiring a SHARED lock and before + ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will + ** be released. + */ + lock.l_len = 1L; + lock.l_whence = SEEK_SET; + if( eFileLock==SHARED_LOCK + || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLocklastErrno = tErrno; + } + goto end_lock; + } + } + + + /* If control gets to this point, then actually go ahead and make + ** operating system calls for the specified lock. + */ + if( eFileLock==SHARED_LOCK ){ + assert( pInode->nShared==0 ); + assert( pInode->eFileLock==0 ); + assert( rc==SQLITE_OK ); + + /* Now get the read-lock */ + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( unixFileLock(pFile, &lock) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + } + + /* Drop the temporary PENDING lock */ + lock.l_start = PENDING_BYTE; + lock.l_len = 1L; + lock.l_type = F_UNLCK; + if( unixFileLock(pFile, &lock) && rc==SQLITE_OK ){ + /* This could happen with a network mount */ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + } + + if( rc ){ + if( rc!=SQLITE_BUSY ){ + pFile->lastErrno = tErrno; + } + goto end_lock; + }else{ + pFile->eFileLock = SHARED_LOCK; + pInode->nLock++; + pInode->nShared = 1; + } + }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + }else{ + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file + ** already. + */ + assert( 0!=pFile->eFileLock ); + lock.l_type = F_WRLCK; + + assert( eFileLock==RESERVED_LOCK || eFileLock==EXCLUSIVE_LOCK ); + if( eFileLock==RESERVED_LOCK ){ + lock.l_start = RESERVED_BYTE; + lock.l_len = 1L; + }else{ + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + } + + if( unixFileLock(pFile, &lock) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( rc!=SQLITE_BUSY ){ + pFile->lastErrno = tErrno; + } + } + } + + +#ifdef SQLITE_DEBUG + /* Set up the transaction-counter change checking flags when + ** transitioning from a SHARED to a RESERVED lock. The change + ** from SHARED to RESERVED marks the beginning of a normal + ** write operation (not a hot journal rollback). + */ + if( rc==SQLITE_OK + && pFile->eFileLock<=SHARED_LOCK + && eFileLock==RESERVED_LOCK + ){ + pFile->transCntrChng = 0; + pFile->dbUpdate = 0; + pFile->inNormalWrite = 1; + } +#endif + + + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + pInode->eFileLock = eFileLock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; + } + +end_lock: + unixLeaveMutex(); + OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); + return rc; +} + +/* +** Add the file descriptor used by file handle pFile to the corresponding +** pUnused list. +*/ +static void setPendingFd(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + UnixUnusedFd *p = pFile->pUnused; + p->pNext = pInode->pUnused; + pInode->pUnused = p; + pFile->h = -1; + pFile->pUnused = 0; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** If handleNFSUnlock is true, then on downgrading an EXCLUSIVE_LOCK to SHARED +** the byte range is divided into 2 parts and the first part is unlocked then +** set to a read lock, then the other part is simply unlocked. This works +** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to +** remove the write lock on a region when a read lock is set. +*/ +static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + struct flock lock; + int rc = SQLITE_OK; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, eFileLock, + pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, + getpid())); + + assert( eFileLock<=SHARED_LOCK ); + if( pFile->eFileLock<=eFileLock ){ + return SQLITE_OK; + } + unixEnterMutex(); + pInode = pFile->pInode; + assert( pInode->nShared!=0 ); + if( pFile->eFileLock>SHARED_LOCK ){ + assert( pInode->eFileLock==pFile->eFileLock ); + +#ifdef SQLITE_DEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ + pFile->inNormalWrite = 0; +#endif + + /* downgrading to a shared lock on NFS involves clearing the write lock + ** before establishing the readlock - to avoid a race condition we downgrade + ** the lock in 2 blocks, so that part of the range will be covered by a + ** write lock until the rest is covered by a read lock: + ** 1: [WWWWW] + ** 2: [....W] + ** 3: [RRRRW] + ** 4: [RRRR.] + */ + if( eFileLock==SHARED_LOCK ){ + +#if !defined(__APPLE__) || !SQLITE_ENABLE_LOCKING_STYLE + (void)handleNFSUnlock; + assert( handleNFSUnlock==0 ); +#endif +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + if( handleNFSUnlock ){ + int tErrno; /* Error code from system call errors */ + off_t divSize = SHARED_SIZE - 1; + + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; + } + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; + } + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST+divSize; + lock.l_len = SHARED_SIZE-divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; + } + }else +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + { + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( unixFileLock(pFile, &lock) ){ + /* In theory, the call to unixFileLock() cannot fail because another + ** process is holding an incompatible lock. If it does, this + ** indicates that the other process is not following the locking + ** protocol. If this happens, return SQLITE_IOERR_RDLOCK. Returning + ** SQLITE_BUSY would confuse the upper layer (in practice it causes + ** an assert to fail). */ + rc = SQLITE_IOERR_RDLOCK; + pFile->lastErrno = errno; + goto end_unlock; + } + } + } + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = PENDING_BYTE; + lock.l_len = 2L; assert( PENDING_BYTE+1==RESERVED_BYTE ); + if( unixFileLock(pFile, &lock)==0 ){ + pInode->eFileLock = SHARED_LOCK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + pFile->lastErrno = errno; + goto end_unlock; + } + } + if( eFileLock==NO_LOCK ){ + /* Decrement the shared lock counter. Release the lock using an + ** OS call only when all threads in this same process have released + ** the lock. + */ + pInode->nShared--; + if( pInode->nShared==0 ){ + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = lock.l_len = 0L; + if( unixFileLock(pFile, &lock)==0 ){ + pInode->eFileLock = NO_LOCK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + pFile->lastErrno = errno; + pInode->eFileLock = NO_LOCK; + pFile->eFileLock = NO_LOCK; + } + } + + /* Decrement the count of locks against this same file. When the + ** count reaches zero, close any other file descriptors whose close + ** was deferred because of outstanding locks. + */ + pInode->nLock--; + assert( pInode->nLock>=0 ); + if( pInode->nLock==0 ){ + closePendingFds(pFile); + } + } + +end_unlock: + unixLeaveMutex(); + if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int unixUnlock(sqlite3_file *id, int eFileLock){ +#if SQLITE_MAX_MMAP_SIZE>0 + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); +#endif + return posixUnlock(id, eFileLock, 0); +} + +#if SQLITE_MAX_MMAP_SIZE>0 +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); +#endif + +/* +** This function performs the parts of the "close file" operation +** common to all locking schemes. It closes the directory and file +** handles, if they are valid, and sets all fields of the unixFile +** structure to 0. +** +** It is *not* necessary to hold the mutex when this routine is called, +** even on VxWorks. A mutex will be acquired on VxWorks by the +** vxworksReleaseFileId() routine. +*/ +static int closeUnixFile(sqlite3_file *id){ + unixFile *pFile = (unixFile*)id; +#if SQLITE_MAX_MMAP_SIZE>0 + unixUnmapfile(pFile); +#endif + if( pFile->h>=0 ){ + robust_close(pFile, pFile->h, __LINE__); + pFile->h = -1; + } +#if OS_VXWORKS + if( pFile->pId ){ + if( pFile->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(pFile->pId->zCanonicalName); + } + vxworksReleaseFileId(pFile->pId); + pFile->pId = 0; + } +#endif +#ifdef SQLITE_UNLINK_AFTER_CLOSE + if( pFile->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(pFile->zPath); + sqlite3_free(*(char**)&pFile->zPath); + pFile->zPath = 0; + } +#endif + OSTRACE(("CLOSE %-3d\n", pFile->h)); + OpenCounter(-1); + sqlite3_free(pFile->pUnused); + memset(pFile, 0, sizeof(unixFile)); + return SQLITE_OK; +} + +/* +** Close a file. +*/ +static int unixClose(sqlite3_file *id){ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile *)id; + verifyDbFile(pFile); + unixUnlock(id, NO_LOCK); + unixEnterMutex(); + + /* unixFile.pInode is always valid here. Otherwise, a different close + ** routine (e.g. nolockClose()) would be called instead. + */ + assert( pFile->pInode->nLock>0 || pFile->pInode->bProcessLock==0 ); + if( ALWAYS(pFile->pInode) && pFile->pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->pUnused list. It will be automatically closed + ** when the last lock is cleared. + */ + setPendingFd(pFile); + } + releaseInodeInfo(pFile); + rc = closeUnixFile(id); + unixLeaveMutex(); + return rc; +} + +/************** End of the posix advisory lock implementation ***************** +******************************************************************************/ + +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int nolockCheckReservedLock(sqlite3_file *NotUsed, int *pResOut){ + UNUSED_PARAMETER(NotUsed); + *pResOut = 0; + return SQLITE_OK; +} +static int nolockLock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} +static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} + +/* +** Close the file. +*/ +static int nolockClose(sqlite3_file *id) { + return closeUnixFile(id); +} + +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ + +/****************************************************************************** +************************* Begin dot-file Locking ****************************** +** +** The dotfile locking implementation uses the existence of separate lock +** files (really a directory) to control access to the database. This works +** on just about every filesystem imaginable. But there are serious downsides: +** +** (1) There is zero concurrency. A single reader blocks all other +** connections from reading or writing the database. +** +** (2) An application crash or power loss can leave stale lock files +** sitting around that need to be cleared manually. +** +** Nevertheless, a dotlock is an appropriate locking mode for use if no +** other locking strategy is available. +** +** Dotfile locking works by creating a subdirectory in the same directory as +** the database and with the same name but with a ".lock" extension added. +** The existence of a lock directory implies an EXCLUSIVE lock. All other +** lock types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. +*/ + +/* +** The file suffix added to the data base filename in order to create the +** lock directory. +*/ +#define DOTLOCK_SUFFIX ".lock" + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +** +** In dotfile locking, either a lock exists or it does not. So in this +** variation of CheckReservedLock(), *pResOut is set to true if any lock +** is held on the file and false if the file is unlocked. +*/ +static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->eFileLock>SHARED_LOCK ){ + /* Either this connection or some other connection in the same process + ** holds a lock on the file. No need to check further. */ + reserved = 1; + }else{ + /* The lock is held if and only if the lockfile exists */ + const char *zLockFile = (const char*)pFile->lockingContext; + reserved = osAccess(zLockFile, 0)==0; + } + OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +** +** With dotfile locking, we really only support state (4): EXCLUSIVE. +** But we track the other locking levels internally. +*/ +static int dotlockLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + char *zLockFile = (char *)pFile->lockingContext; + int rc = SQLITE_OK; + + + /* If we have any lock, then the lock file already exists. All we have + ** to do is adjust our internal record of the lock level. + */ + if( pFile->eFileLock > NO_LOCK ){ + pFile->eFileLock = eFileLock; + /* Always update the timestamp on the old file */ +#ifdef HAVE_UTIME + utime(zLockFile, NULL); +#else + utimes(zLockFile, NULL); +#endif + return SQLITE_OK; + } + + /* grab an exclusive lock */ + rc = osMkdir(zLockFile, 0777); + if( rc<0 ){ + /* failed to open/create the lock directory */ + int tErrno = errno; + if( EEXIST == tErrno ){ + rc = SQLITE_BUSY; + } else { + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + } + return rc; + } + + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** When the locking level reaches NO_LOCK, delete the lock file. +*/ +static int dotlockUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + char *zLockFile = (char *)pFile->lockingContext; + int rc; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, + pFile->eFileLock, getpid())); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* To downgrade to shared, simply update our internal notion of the + ** lock state. No need to mess with the file on disk. + */ + if( eFileLock==SHARED_LOCK ){ + pFile->eFileLock = SHARED_LOCK; + return SQLITE_OK; + } + + /* To fully unlock the database, delete the lock file */ + assert( eFileLock==NO_LOCK ); + rc = osRmdir(zLockFile); + if( rc<0 && errno==ENOTDIR ) rc = osUnlink(zLockFile); + if( rc<0 ){ + int tErrno = errno; + rc = 0; + if( ENOENT != tErrno ){ + rc = SQLITE_IOERR_UNLOCK; + } + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + return rc; + } + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; +} + +/* +** Close a file. Make sure the lock has been released before closing. +*/ +static int dotlockClose(sqlite3_file *id) { + int rc = SQLITE_OK; + if( id ){ + unixFile *pFile = (unixFile*)id; + dotlockUnlock(id, NO_LOCK); + sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); + } + return rc; +} +/****************** End of the dot-file lock implementation ******************* +******************************************************************************/ + +/****************************************************************************** +************************** Begin flock Locking ******************************** +** +** Use the flock() system call to do file locking. +** +** flock() locking is like dot-file locking in that the various +** fine-grain locking levels supported by SQLite are collapsed into +** a single exclusive lock. In other words, SHARED, RESERVED, and +** PENDING locks are the same thing as an EXCLUSIVE lock. SQLite +** still works when you do this, but concurrency is reduced since +** only a single process can be reading the database at a time. +** +** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off or if +** compiling for VXWORKS. +*/ +#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS + +/* +** Retry flock() calls that fail with EINTR +*/ +#ifdef EINTR +static int robust_flock(int fd, int op){ + int rc; + do{ rc = flock(fd,op); }while( rc<0 && errno==EINTR ); + return rc; +} +#else +# define robust_flock(a,b) flock(a,b) +#endif + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + /* attempt to get the lock */ + int lrc = robust_flock(pFile->h, LOCK_EX | LOCK_NB); + if( !lrc ){ + /* got the lock, unlock it */ + lrc = robust_flock(pFile->h, LOCK_UN); + if ( lrc ) { + int tErrno = errno; + /* unlock failed with an error */ + lrc = SQLITE_IOERR_UNLOCK; + if( IS_LOCK_ERROR(lrc) ){ + pFile->lastErrno = tErrno; + rc = lrc; + } + } + } else { + int tErrno = errno; + reserved = 1; + /* someone else might have it reserved */ + lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(lrc) ){ + pFile->lastErrno = tErrno; + rc = lrc; + } + } + } + OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved)); + +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ + rc = SQLITE_OK; + reserved=1; + } +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** flock() only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int flockLock(sqlite3_file *id, int eFileLock) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + + assert( pFile ); + + /* if we already have a lock, it is exclusive. + ** Just adjust level and punt on outta here. */ + if (pFile->eFileLock > NO_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* grab an exclusive lock */ + + if (robust_flock(pFile->h, LOCK_EX | LOCK_NB)) { + int tErrno = errno; + /* didn't get, must be busy */ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + } else { + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + } + OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ + rc = SQLITE_BUSY; + } +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + return rc; +} + + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int flockUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, eFileLock, + pFile->eFileLock, getpid())); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* shared can just be set because we always have an exclusive */ + if (eFileLock==SHARED_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* no, really, unlock. */ + if( robust_flock(pFile->h, LOCK_UN) ){ +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + return SQLITE_OK; +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + return SQLITE_IOERR_UNLOCK; + }else{ + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; + } +} + +/* +** Close a file. +*/ +static int flockClose(sqlite3_file *id) { + int rc = SQLITE_OK; + if( id ){ + flockUnlock(id, NO_LOCK); + rc = closeUnixFile(id); + } + return rc; +} + +#endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ + +/******************* End of the flock lock implementation ********************* +******************************************************************************/ + +/****************************************************************************** +************************ Begin Named Semaphore Locking ************************ +** +** Named semaphore locking is only supported on VxWorks. +** +** Semaphore locking is like dot-lock and flock in that it really only +** supports EXCLUSIVE locking. Only a single process can read or write +** the database file at a time. This reduces potential concurrency, but +** makes the lock implementation much easier. +*/ +#if OS_VXWORKS + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + sem_t *pSem = pFile->pInode->pSem; + + if( sem_trywait(pSem)==-1 ){ + int tErrno = errno; + if( EAGAIN != tErrno ){ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); + pFile->lastErrno = tErrno; + } else { + /* someone else has the lock when we are in NO_LOCK */ + reserved = (pFile->eFileLock < SHARED_LOCK); + } + }else{ + /* we could have it if we want it */ + sem_post(pSem); + } + } + OSTRACE(("TEST WR-LOCK %d %d %d (sem)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** Semaphore locks only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int semLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + sem_t *pSem = pFile->pInode->pSem; + int rc = SQLITE_OK; + + /* if we already have a lock, it is exclusive. + ** Just adjust level and punt on outta here. */ + if (pFile->eFileLock > NO_LOCK) { + pFile->eFileLock = eFileLock; + rc = SQLITE_OK; + goto sem_end_lock; + } + + /* lock semaphore now but bail out when already locked. */ + if( sem_trywait(pSem)==-1 ){ + rc = SQLITE_BUSY; + goto sem_end_lock; + } + + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + + sem_end_lock: + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int semUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + sem_t *pSem = pFile->pInode->pSem; + + assert( pFile ); + assert( pSem ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, + pFile->eFileLock, getpid())); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* shared can just be set because we always have an exclusive */ + if (eFileLock==SHARED_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* no, really unlock. */ + if ( sem_post(pSem)==-1 ) { + int rc, tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + return rc; + } + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; +} + +/* + ** Close a file. + */ +static int semClose(sqlite3_file *id) { + if( id ){ + unixFile *pFile = (unixFile*)id; + semUnlock(id, NO_LOCK); + assert( pFile ); + unixEnterMutex(); + releaseInodeInfo(pFile); + unixLeaveMutex(); + closeUnixFile(id); + } + return SQLITE_OK; +} + +#endif /* OS_VXWORKS */ +/* +** Named semaphore locking is only available on VxWorks. +** +*************** End of the named semaphore lock implementation **************** +******************************************************************************/ + + +/****************************************************************************** +*************************** Begin AFP Locking ********************************* +** +** AFP is the Apple Filing Protocol. AFP is a network filesystem found +** on Apple Macintosh computers - both OS9 and OSX. +** +** Third-party implementations of AFP are available. But this code here +** only works on OSX. +*/ + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** The afpLockingContext structure contains all afp lock specific state +*/ +typedef struct afpLockingContext afpLockingContext; +struct afpLockingContext { + int reserved; + const char *dbPath; /* Name of the open file */ +}; + +struct ByteRangeLockPB2 +{ + unsigned long long offset; /* offset to first byte to lock */ + unsigned long long length; /* nbr of bytes to lock */ + unsigned long long retRangeStart; /* nbr of 1st byte locked if successful */ + unsigned char unLockFlag; /* 1 = unlock, 0 = lock */ + unsigned char startEndFlag; /* 1=rel to end of fork, 0=rel to start */ + int fd; /* file desc to assoc this lock with */ +}; + +#define afpfsByteRangeLock2FSCTL _IOWR('z', 23, struct ByteRangeLockPB2) + +/* +** This is a utility for setting or clearing a bit-range lock on an +** AFP filesystem. +** +** Return SQLITE_OK on success, SQLITE_BUSY on failure. +*/ +static int afpSetLock( + const char *path, /* Name of the file to be locked or unlocked */ + unixFile *pFile, /* Open file descriptor on path */ + unsigned long long offset, /* First byte to be locked */ + unsigned long long length, /* Number of bytes to lock */ + int setLockFlag /* True to set lock. False to clear lock */ +){ + struct ByteRangeLockPB2 pb; + int err; + + pb.unLockFlag = setLockFlag ? 0 : 1; + pb.startEndFlag = 0; + pb.offset = offset; + pb.length = length; + pb.fd = pFile->h; + + OSTRACE(("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", + (setLockFlag?"ON":"OFF"), pFile->h, (pb.fd==-1?"[testval-1]":""), + offset, length)); + err = fsctl(path, afpfsByteRangeLock2FSCTL, &pb, 0); + if ( err==-1 ) { + int rc; + int tErrno = errno; + OSTRACE(("AFPSETLOCK failed to fsctl() '%s' %d %s\n", + path, tErrno, strerror(tErrno))); +#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + rc = SQLITE_BUSY; +#else + rc = sqliteErrorFromPosixError(tErrno, + setLockFlag ? SQLITE_IOERR_LOCK : SQLITE_IOERR_UNLOCK); +#endif /* SQLITE_IGNORE_AFP_LOCK_ERRORS */ + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + return rc; + } else { + return SQLITE_OK; + } +} + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + afpLockingContext *context; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + context = (afpLockingContext *) pFile->lockingContext; + if( context->reserved ){ + *pResOut = 1; + return SQLITE_OK; + } + unixEnterMutex(); /* Because pFile->pInode is shared across threads */ + + /* Check if a thread in this process holds such a lock */ + if( pFile->pInode->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. + */ + if( !reserved ){ + /* lock the RESERVED byte */ + int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( SQLITE_OK==lrc ){ + /* if we succeeded in taking the reserved lock, unlock it to restore + ** the original state */ + lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + } else { + /* if we failed to get the lock then someone else must have it */ + reserved = 1; + } + if( IS_LOCK_ERROR(lrc) ){ + rc=lrc; + } + } + + unixLeaveMutex(); + OSTRACE(("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int afpLock(sqlite3_file *id, int eFileLock){ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode = pFile->pInode; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + + assert( pFile ); + OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, + azFileLock(eFileLock), azFileLock(pFile->eFileLock), + azFileLock(pInode->eFileLock), pInode->nShared , getpid())); + + /* If there is already a lock of this type or more restrictive on the + ** unixFile, do nothing. Don't use the afp_end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. + */ + if( pFile->eFileLock>=eFileLock ){ + OSTRACE(("LOCK %d %s ok (already held) (afp)\n", pFile->h, + azFileLock(eFileLock))); + return SQLITE_OK; + } + + /* Make sure the locking sequence is correct + ** (1) We never move from unlocked to anything higher than shared lock. + ** (2) SQLite never explicitly requests a pendig lock. + ** (3) A shared lock is always held when a reserve lock is requested. + */ + assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); + assert( eFileLock!=PENDING_LOCK ); + assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); + + /* This mutex is needed because pFile->pInode is shared across threads + */ + unixEnterMutex(); + pInode = pFile->pInode; + + /* If some thread using this PID has a lock via a different unixFile* + ** handle that precludes the requested lock, return BUSY. + */ + if( (pFile->eFileLock!=pInode->eFileLock && + (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) + ){ + rc = SQLITE_BUSY; + goto afp_end_lock; + } + + /* If a SHARED lock is requested, and some thread using this PID already + ** has a SHARED or RESERVED lock, then increment reference counts and + ** return SQLITE_OK. + */ + if( eFileLock==SHARED_LOCK && + (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ + assert( eFileLock==SHARED_LOCK ); + assert( pFile->eFileLock==0 ); + assert( pInode->nShared>0 ); + pFile->eFileLock = SHARED_LOCK; + pInode->nShared++; + pInode->nLock++; + goto afp_end_lock; + } + + /* A PENDING lock is needed before acquiring a SHARED lock and before + ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will + ** be released. + */ + if( eFileLock==SHARED_LOCK + || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLockdbPath, pFile, PENDING_BYTE, 1, 1); + if (failed) { + rc = failed; + goto afp_end_lock; + } + } + + /* If control gets to this point, then actually go ahead and make + ** operating system calls for the specified lock. + */ + if( eFileLock==SHARED_LOCK ){ + int lrc1, lrc2, lrc1Errno = 0; + long lk, mask; + + assert( pInode->nShared==0 ); + assert( pInode->eFileLock==0 ); + + mask = (sizeof(long)==8) ? LARGEST_INT64 : 0x7fffffff; + /* Now get the read-lock SHARED_LOCK */ + /* note that the quality of the randomness doesn't matter that much */ + lk = random(); + pInode->sharedByte = (lk & mask)%(SHARED_SIZE - 1); + lrc1 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST+pInode->sharedByte, 1, 1); + if( IS_LOCK_ERROR(lrc1) ){ + lrc1Errno = pFile->lastErrno; + } + /* Drop the temporary PENDING lock */ + lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); + + if( IS_LOCK_ERROR(lrc1) ) { + pFile->lastErrno = lrc1Errno; + rc = lrc1; + goto afp_end_lock; + } else if( IS_LOCK_ERROR(lrc2) ){ + rc = lrc2; + goto afp_end_lock; + } else if( lrc1 != SQLITE_OK ) { + rc = lrc1; + } else { + pFile->eFileLock = SHARED_LOCK; + pInode->nLock++; + pInode->nShared = 1; + } + }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + }else{ + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file + ** already. + */ + int failed = 0; + assert( 0!=pFile->eFileLock ); + if (eFileLock >= RESERVED_LOCK && pFile->eFileLock < RESERVED_LOCK) { + /* Acquire a RESERVED lock */ + failed = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( !failed ){ + context->reserved = 1; + } + } + if (!failed && eFileLock == EXCLUSIVE_LOCK) { + /* Acquire an EXCLUSIVE lock */ + + /* Remove the shared lock before trying the range. we'll need to + ** reestablish the shared lock if we can't get the afpUnlock + */ + if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + + pInode->sharedByte, 1, 0)) ){ + int failed2 = SQLITE_OK; + /* now attemmpt to get the exclusive lock range */ + failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, + SHARED_SIZE, 1); + if( failed && (failed2 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST + pInode->sharedByte, 1, 1)) ){ + /* Can't reestablish the shared lock. Sqlite can't deal, this is + ** a critical I/O error + */ + rc = ((failed & SQLITE_IOERR) == SQLITE_IOERR) ? failed2 : + SQLITE_IOERR_LOCK; + goto afp_end_lock; + } + }else{ + rc = failed; + } + } + if( failed ){ + rc = failed; + } + } + + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + pInode->eFileLock = eFileLock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; + } + +afp_end_lock: + unixLeaveMutex(); + OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int afpUnlock(sqlite3_file *id, int eFileLock) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + int skipShared = 0; +#ifdef SQLITE_TEST + int h = pFile->h; +#endif + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, + pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, + getpid())); + + assert( eFileLock<=SHARED_LOCK ); + if( pFile->eFileLock<=eFileLock ){ + return SQLITE_OK; + } + unixEnterMutex(); + pInode = pFile->pInode; + assert( pInode->nShared!=0 ); + if( pFile->eFileLock>SHARED_LOCK ){ + assert( pInode->eFileLock==pFile->eFileLock ); + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + +#ifdef SQLITE_DEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ + assert( pFile->inNormalWrite==0 + || pFile->dbUpdate==0 + || pFile->transCntrChng==1 ); + pFile->inNormalWrite = 0; +#endif + + if( pFile->eFileLock==EXCLUSIVE_LOCK ){ + rc = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); + if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1) ){ + /* only re-establish the shared lock if necessary */ + int sharedLockByte = SHARED_FIRST+pInode->sharedByte; + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 1); + } else { + skipShared = 1; + } + } + if( rc==SQLITE_OK && pFile->eFileLock>=PENDING_LOCK ){ + rc = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); + } + if( rc==SQLITE_OK && pFile->eFileLock>=RESERVED_LOCK && context->reserved ){ + rc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + if( !rc ){ + context->reserved = 0; + } + } + if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1)){ + pInode->eFileLock = SHARED_LOCK; + } + } + if( rc==SQLITE_OK && eFileLock==NO_LOCK ){ + + /* Decrement the shared lock counter. Release the lock using an + ** OS call only when all threads in this same process have released + ** the lock. + */ + unsigned long long sharedLockByte = SHARED_FIRST+pInode->sharedByte; + pInode->nShared--; + if( pInode->nShared==0 ){ + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + if( !skipShared ){ + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); + } + if( !rc ){ + pInode->eFileLock = NO_LOCK; + pFile->eFileLock = NO_LOCK; + } + } + if( rc==SQLITE_OK ){ + pInode->nLock--; + assert( pInode->nLock>=0 ); + if( pInode->nLock==0 ){ + closePendingFds(pFile); + } + } + } + + unixLeaveMutex(); + if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + return rc; +} + +/* +** Close a file & cleanup AFP specific locking context +*/ +static int afpClose(sqlite3_file *id) { + int rc = SQLITE_OK; + if( id ){ + unixFile *pFile = (unixFile*)id; + afpUnlock(id, NO_LOCK); + unixEnterMutex(); + if( pFile->pInode && pFile->pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); + } + releaseInodeInfo(pFile); + sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); + unixLeaveMutex(); + } + return rc; +} + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The code above is the AFP lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. If you don't compile for a mac, then the "unix-afp" +** VFS is not available. +** +********************* End of the AFP lock implementation ********************** +******************************************************************************/ + +/****************************************************************************** +*************************** Begin NFS Locking ********************************/ + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* + ** Lower the locking level on file descriptor pFile to eFileLock. eFileLock + ** must be either NO_LOCK or SHARED_LOCK. + ** + ** If the locking level of the file descriptor is already at or below + ** the requested locking level, this routine is a no-op. + */ +static int nfsUnlock(sqlite3_file *id, int eFileLock){ + return posixUnlock(id, eFileLock, 1); +} + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The code above is the NFS lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. +** +********************* End of the NFS lock implementation ********************** +******************************************************************************/ + +/****************************************************************************** +**************** Non-locking sqlite3_file methods ***************************** +** +** The next division contains implementations for all methods of the +** sqlite3_file object other than the locking methods. The locking +** methods were defined in divisions above (one locking method per +** division). Those methods that are common to all locking modes +** are gather together into this division. +*/ + +/* +** Seek to the offset passed as the second argument, then read cnt +** bytes into pBuf. Return the number of bytes actually read. +** +** NB: If you define USE_PREAD or USE_PREAD64, then it might also +** be necessary to define _XOPEN_SOURCE to be 500. This varies from +** one system to another. Since SQLite does not define USE_PREAD +** any any form by default, we will not attempt to define _XOPEN_SOURCE. +** See tickets #2741 and #2681. +** +** To avoid stomping the errno value on a failed read the lastErrno value +** is set before returning. +*/ +static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ + int got; + int prior = 0; +#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) + i64 newOffset; +#endif + TIMER_START; + assert( cnt==(cnt&0x1ffff) ); + assert( id->h>2 ); + cnt &= 0x1ffff; + do{ +#if defined(USE_PREAD) + got = osPread(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); +#elif defined(USE_PREAD64) + got = osPread64(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); +#else + newOffset = lseek(id->h, offset, SEEK_SET); + SimulateIOError( newOffset-- ); + if( newOffset!=offset ){ + if( newOffset == -1 ){ + ((unixFile*)id)->lastErrno = errno; + }else{ + ((unixFile*)id)->lastErrno = 0; + } + return -1; + } + got = osRead(id->h, pBuf, cnt); +#endif + if( got==cnt ) break; + if( got<0 ){ + if( errno==EINTR ){ got = 1; continue; } + prior = 0; + ((unixFile*)id)->lastErrno = errno; + break; + }else if( got>0 ){ + cnt -= got; + offset += got; + prior += got; + pBuf = (void*)(got + (char*)pBuf); + } + }while( got>0 ); + TIMER_END; + OSTRACE(("READ %-3d %5d %7lld %llu\n", + id->h, got+prior, offset-prior, TIMER_ELAPSED)); + return got+prior; +} + +/* +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. +*/ +static int unixRead( + sqlite3_file *id, + void *pBuf, + int amt, + sqlite3_int64 offset +){ + unixFile *pFile = (unixFile *)id; + int got; + assert( id ); + assert( offset>=0 ); + assert( amt>0 ); + + /* If this is a database file (not a journal, master-journal or temp + ** file), the bytes in the locking range should never be read or written. */ +#if 0 + assert( pFile->pUnused==0 + || offset>=PENDING_BYTE+512 + || offset+amt<=PENDING_BYTE + ); +#endif + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + + got = seekAndRead(pFile, offset, pBuf, amt); + if( got==amt ){ + return SQLITE_OK; + }else if( got<0 ){ + /* lastErrno set by seekAndRead */ + return SQLITE_IOERR_READ; + }else{ + pFile->lastErrno = 0; /* not a system error */ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[got], 0, amt-got); + return SQLITE_IOERR_SHORT_READ; + } +} + +/* +** Attempt to seek the file-descriptor passed as the first argument to +** absolute offset iOff, then attempt to write nBuf bytes of data from +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** return the actual number of bytes written (which may be less than +** nBuf). +*/ +static int seekAndWriteFd( + int fd, /* File descriptor to write to */ + i64 iOff, /* File offset to begin writing at */ + const void *pBuf, /* Copy data from this buffer to the file */ + int nBuf, /* Size of buffer pBuf in bytes */ + int *piErrno /* OUT: Error number if error occurs */ +){ + int rc = 0; /* Value returned by system call */ + + assert( nBuf==(nBuf&0x1ffff) ); + assert( fd>2 ); + nBuf &= 0x1ffff; + TIMER_START; + +#if defined(USE_PREAD) + do{ rc = osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); +#elif defined(USE_PREAD64) + do{ rc = osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); +#else + do{ + i64 iSeek = lseek(fd, iOff, SEEK_SET); + SimulateIOError( iSeek-- ); + + if( iSeek!=iOff ){ + if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); + return -1; + } + rc = osWrite(fd, pBuf, nBuf); + }while( rc<0 && errno==EINTR ); +#endif + + TIMER_END; + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); + + if( rc<0 && piErrno ) *piErrno = errno; + return rc; +} + + +/* +** Seek to the offset in id->offset then read cnt bytes into pBuf. +** Return the number of bytes actually read. Update the offset. +** +** To avoid stomping the errno value on a failed write the lastErrno value +** is set before returning. +*/ +static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ + return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); +} + + +/* +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. +*/ +static int unixWrite( + sqlite3_file *id, + const void *pBuf, + int amt, + sqlite3_int64 offset +){ + unixFile *pFile = (unixFile*)id; + int wrote = 0; + assert( id ); + assert( amt>0 ); + + /* If this is a database file (not a journal, master-journal or temp + ** file), the bytes in the locking range should never be read or written. */ +#if 0 + assert( pFile->pUnused==0 + || offset>=PENDING_BYTE+512 + || offset+amt<=PENDING_BYTE + ); +#endif + +#ifdef SQLITE_DEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) then record the fact that the database + ** has changed. If the transaction counter is modified, record that + ** fact too. + */ + if( pFile->inNormalWrite ){ + pFile->dbUpdate = 1; /* The database has been modified */ + if( offset<=24 && offset+amt>=27 ){ + int rc; + char oldCntr[4]; + SimulateIOErrorBenign(1); + rc = seekAndRead(pFile, 24, oldCntr, 4); + SimulateIOErrorBenign(0); + if( rc!=4 || memcmp(oldCntr, &((char*)pBuf)[24-offset], 4)!=0 ){ + pFile->transCntrChng = 1; /* The transaction counter has changed */ + } + } + } +#endif + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ + amt -= wrote; + offset += wrote; + pBuf = &((char*)pBuf)[wrote]; + } + SimulateIOError(( wrote=(-1), amt=1 )); + SimulateDiskfullError(( wrote=0, amt=1 )); + + if( amt>0 ){ + if( wrote<0 && pFile->lastErrno!=ENOSPC ){ + /* lastErrno set by seekAndWrite */ + return SQLITE_IOERR_WRITE; + }else{ + pFile->lastErrno = 0; /* not a system error */ + return SQLITE_FULL; + } + } + + return SQLITE_OK; +} + +#ifdef SQLITE_TEST +/* +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occurring at the right times. +*/ +SQLITE_API int sqlite3_sync_count = 0; +SQLITE_API int sqlite3_fullsync_count = 0; +#endif + +/* +** We do not trust systems to provide a working fdatasync(). Some do. +** Others do no. To be safe, we will stick with the (slightly slower) +** fsync(). If you know that your system does support fdatasync() correctly, +** then simply compile with -Dfdatasync=fdatasync +*/ +#if !defined(fdatasync) +# define fdatasync fsync +#endif + +/* +** Define HAVE_FULLFSYNC to 0 or 1 depending on whether or not +** the F_FULLFSYNC macro is defined. F_FULLFSYNC is currently +** only available on Mac OS X. But that could change. +*/ +#ifdef F_FULLFSYNC +# define HAVE_FULLFSYNC 1 +#else +# define HAVE_FULLFSYNC 0 +#endif + + +/* +** The fsync() system call does not work as advertised on many +** unix systems. The following procedure is an attempt to make +** it work better. +** +** The SQLITE_NO_SYNC macro disables all fsync()s. This is useful +** for testing when we want to run through the test suite quickly. +** You are strongly advised *not* to deploy with SQLITE_NO_SYNC +** enabled, however, since with SQLITE_NO_SYNC enabled, an OS crash +** or power failure will likely corrupt the database file. +** +** SQLite sets the dataOnly flag if the size of the file is unchanged. +** The idea behind dataOnly is that it should only write the file content +** to disk, not the inode. We only set dataOnly if the file size is +** unchanged since the file size is part of the inode. However, +** Ted Ts'o tells us that fdatasync() will also write the inode if the +** file size has changed. The only real difference between fdatasync() +** and fsync(), Ted tells us, is that fdatasync() will not flush the +** inode if the mtime or owner or other inode attributes have changed. +** We only care about the file size, not the other file attributes, so +** as far as SQLite is concerned, an fdatasync() is always adequate. +** So, we always use fdatasync() if it is available, regardless of +** the value of the dataOnly flag. +*/ +static int full_fsync(int fd, int fullSync, int dataOnly){ + int rc; + + /* The following "ifdef/elif/else/" block has the same structure as + ** the one below. It is replicated here solely to avoid cluttering + ** up the real code with the UNUSED_PARAMETER() macros. + */ +#ifdef SQLITE_NO_SYNC + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(fullSync); + UNUSED_PARAMETER(dataOnly); +#elif HAVE_FULLFSYNC + UNUSED_PARAMETER(dataOnly); +#else + UNUSED_PARAMETER(fullSync); + UNUSED_PARAMETER(dataOnly); +#endif + + /* Record the number of times that we do a normal fsync() and + ** FULLSYNC. This is used during testing to verify that this procedure + ** gets called with the correct arguments. + */ +#ifdef SQLITE_TEST + if( fullSync ) sqlite3_fullsync_count++; + sqlite3_sync_count++; +#endif + + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op + */ +#ifdef SQLITE_NO_SYNC + rc = SQLITE_OK; +#elif HAVE_FULLFSYNC + if( fullSync ){ + rc = osFcntl(fd, F_FULLFSYNC, 0); + }else{ + rc = 1; + } + /* If the FULLFSYNC failed, fall back to attempting an fsync(). + ** It shouldn't be possible for fullfsync to fail on the local + ** file system (on OSX), so failure indicates that FULLFSYNC + ** isn't supported for this file system. So, attempt an fsync + ** and (for now) ignore the overhead of a superfluous fcntl call. + ** It'd be better to detect fullfsync support once and avoid + ** the fcntl call every time sync is called. + */ + if( rc ) rc = fsync(fd); + +#elif defined(__APPLE__) + /* fdatasync() on HFS+ doesn't yet flush the file size if it changed correctly + ** so currently we default to the macro that redefines fdatasync to fsync + */ + rc = fsync(fd); +#else + rc = fdatasync(fd); +#if OS_VXWORKS + if( rc==-1 && errno==ENOTSUP ){ + rc = fsync(fd); + } +#endif /* OS_VXWORKS */ +#endif /* ifdef SQLITE_NO_SYNC elif HAVE_FULLFSYNC */ + + if( OS_VXWORKS && rc!= -1 ){ + rc = 0; + } + return rc; +} + +/* +** Open a file descriptor to the directory containing file zFilename. +** If successful, *pFd is set to the opened file descriptor and +** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM +** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined +** value. +** +** The directory file descriptor is used for only one thing - to +** fsync() a directory to make sure file creation and deletion events +** are flushed to disk. Such fsyncs are not needed on newer +** journaling filesystems, but are required on older filesystems. +** +** This routine can be overridden using the xSetSysCall interface. +** The ability to override this routine was added in support of the +** chromium sandbox. Opening a directory is a security risk (we are +** told) so making it overrideable allows the chromium sandbox to +** replace this routine with a harmless no-op. To make this routine +** a no-op, replace it with a stub that returns SQLITE_OK but leaves +** *pFd set to a negative number. +** +** If SQLITE_OK is returned, the caller is responsible for closing +** the file descriptor *pFd using close(). +*/ +static int openDirectory(const char *zFilename, int *pFd){ + int ii; + int fd = -1; + char zDirname[MAX_PATHNAME+1]; + + sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); + for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); + if( ii>0 ){ + zDirname[ii] = '\0'; + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); + if( fd>=0 ){ + OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); + } + } + *pFd = fd; + return (fd>=0?SQLITE_OK:unixLogError(SQLITE_CANTOPEN_BKPT, "open", zDirname)); +} + +/* +** Make sure all writes to a particular file are committed to disk. +** +** If dataOnly==0 then both the file itself and its metadata (file +** size, access time, etc) are synced. If dataOnly!=0 then only the +** file data is synced. +** +** Under Unix, also make sure that the directory entry for the file +** has been created by fsync-ing the directory that contains the file. +** If we do not do this and we encounter a power failure, the directory +** entry for the journal might not exist after we reboot. The next +** SQLite to access the file will not know that the journal exists (because +** the directory entry for the journal was never created) and the transaction +** will not roll back - possibly leading to database corruption. +*/ +static int unixSync(sqlite3_file *id, int flags){ + int rc; + unixFile *pFile = (unixFile*)id; + + int isDataOnly = (flags&SQLITE_SYNC_DATAONLY); + int isFullsync = (flags&0x0F)==SQLITE_SYNC_FULL; + + /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ + assert((flags&0x0F)==SQLITE_SYNC_NORMAL + || (flags&0x0F)==SQLITE_SYNC_FULL + ); + + /* Unix cannot, but some systems may return SQLITE_FULL from here. This + ** line is to test that doing so does not cause any problems. + */ + SimulateDiskfullError( return SQLITE_FULL ); + + assert( pFile ); + OSTRACE(("SYNC %-3d\n", pFile->h)); + rc = full_fsync(pFile->h, isFullsync, isDataOnly); + SimulateIOError( rc=1 ); + if( rc ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_FSYNC, "full_fsync", pFile->zPath); + } + + /* Also fsync the directory containing the file if the DIRSYNC flag + ** is set. This is a one-time occurrence. Many systems (examples: AIX) + ** are unable to fsync a directory, so ignore errors on the fsync. + */ + if( pFile->ctrlFlags & UNIXFILE_DIRSYNC ){ + int dirfd; + OSTRACE(("DIRSYNC %s (have_fullfsync=%d fullsync=%d)\n", pFile->zPath, + HAVE_FULLFSYNC, isFullsync)); + rc = osOpenDirectory(pFile->zPath, &dirfd); + if( rc==SQLITE_OK && dirfd>=0 ){ + full_fsync(dirfd, 0, 0); + robust_close(pFile, dirfd, __LINE__); + }else if( rc==SQLITE_CANTOPEN ){ + rc = SQLITE_OK; + } + pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; + } + return rc; +} + +/* +** Truncate an open file to a specified size +*/ +static int unixTruncate(sqlite3_file *id, i64 nByte){ + unixFile *pFile = (unixFile *)id; + int rc; + assert( pFile ); + SimulateIOError( return SQLITE_IOERR_TRUNCATE ); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk>0 ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + rc = robust_ftruncate(pFile->h, (off_t)nByte); + if( rc ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + }else{ +#ifdef SQLITE_DEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) and we truncate the file to zero length, + ** that effectively updates the change counter. This might happen + ** when restoring a database using the backup API from a zero-length + ** source. + */ + if( pFile->inNormalWrite && nByte==0 ){ + pFile->transCntrChng = 1; + } +#endif + +#if SQLITE_MAX_MMAP_SIZE>0 + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nBytemmapSize ){ + pFile->mmapSize = nByte; + } +#endif + + return SQLITE_OK; + } +} + +/* +** Determine the current size of a file in bytes +*/ +static int unixFileSize(sqlite3_file *id, i64 *pSize){ + int rc; + struct stat buf; + assert( id ); + rc = osFstat(((unixFile*)id)->h, &buf); + SimulateIOError( rc=1 ); + if( rc!=0 ){ + ((unixFile*)id)->lastErrno = errno; + return SQLITE_IOERR_FSTAT; + } + *pSize = buf.st_size; + + /* When opening a zero-size database, the findInodeInfo() procedure + ** writes a single byte into that file in order to work around a bug + ** in the OS-X msdos filesystem. In order to avoid problems with upper + ** layers, we need to report this file size as zero even though it is + ** really 1. Ticket #3260. + */ + if( *pSize==1 ) *pSize = 0; + + + return SQLITE_OK; +} + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) +/* +** Handler for proxy-locking file-control verbs. Defined below in the +** proxying locking division. +*/ +static int proxyFileControl(sqlite3_file*,int,void*); +#endif + +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +** file-control operation. Enlarge the database to nBytes in size +** (rounded up to the next chunk-size). If the database is already +** nBytes or larger, this routine is a no-op. +*/ +static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + if( pFile->szChunk>0 ){ + i64 nSize; /* Required file size */ + struct stat buf; /* Used to hold return values of fstat() */ + + if( osFstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; + + nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; + if( nSize>(i64)buf.st_size ){ + +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + /* The code below is handling the return value of osFallocate() + ** correctly. posix_fallocate() is defined to "returns zero on success, + ** or an error number on failure". See the manpage for details. */ + int err; + do{ + err = osFallocate(pFile->h, buf.st_size, nSize-buf.st_size); + }while( err==EINTR ); + if( err ) return SQLITE_IOERR_WRITE; +#else + /* If the OS does not have posix_fallocate(), fake it. First use + ** ftruncate() to set the file size, then write a single byte to + ** the last byte in each block within the extended region. This + ** is the same technique used by glibc to implement posix_fallocate() + ** on systems that do not have a real fallocate() system call. + */ + int nBlk = buf.st_blksize; /* File-system block size */ + i64 iWrite; /* Next offset to write to */ + + if( robust_ftruncate(pFile->h, nSize) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; + while( iWrite0 + if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } +#endif + + return SQLITE_OK; +} + +/* +** If *pArg is inititially negative then this is a query. Set *pArg to +** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. +** +** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. +*/ +static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ + if( *pArg<0 ){ + *pArg = (pFile->ctrlFlags & mask)!=0; + }else if( (*pArg)==0 ){ + pFile->ctrlFlags &= ~mask; + }else{ + pFile->ctrlFlags |= mask; + } +} + +/* Forward declaration */ +static int unixGetTempname(int nBuf, char *zBuf); + +/* +** Information and control of an open file handle. +*/ +static int unixFileControl(sqlite3_file *id, int op, void *pArg){ + unixFile *pFile = (unixFile*)id; + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = pFile->eFileLock; + return SQLITE_OK; + } + case SQLITE_LAST_ERRNO: { + *(int*)pArg = pFile->lastErrno; + return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + pFile->szChunk = *(int *)pArg; + return SQLITE_OK; + } + case SQLITE_FCNTL_SIZE_HINT: { + int rc; + SimulateIOErrorBenign(1); + rc = fcntlSizeHint(pFile, *(i64 *)pArg); + SimulateIOErrorBenign(0); + return rc; + } + case SQLITE_FCNTL_PERSIST_WAL: { + unixModeBit(pFile, UNIXFILE_PERSIST_WAL, (int*)pArg); + return SQLITE_OK; + } + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + unixModeBit(pFile, UNIXFILE_PSOW, (int*)pArg); + return SQLITE_OK; + } + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); + return SQLITE_OK; + } + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = sqlite3_malloc( pFile->pVfs->mxPathname ); + if( zTFile ){ + unixGetTempname(pFile->pVfs->mxPathname, zTFile); + *(char**)pArg = zTFile; + } + return SQLITE_OK; + } + case SQLITE_FCNTL_HAS_MOVED: { + *(int*)pArg = fileHasMoved(pFile); + return SQLITE_OK; + } +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + int rc = SQLITE_OK; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ + pFile->mmapSizeMax = newLimit; + if( pFile->mmapSize>0 ){ + unixUnmapfile(pFile); + rc = unixMapfile(pFile, -1); + } + } + return rc; + } +#endif +#ifdef SQLITE_DEBUG + /* The pager calls this method to signal that it has done + ** a rollback and that the database is therefore unchanged and + ** it hence it is OK for the transaction change counter to be + ** unchanged. + */ + case SQLITE_FCNTL_DB_UNCHANGED: { + ((unixFile*)id)->dbUpdate = 0; + return SQLITE_OK; + } +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + case SQLITE_SET_LOCKPROXYFILE: + case SQLITE_GET_LOCKPROXYFILE: { + return proxyFileControl(id,op,pArg); + } +#endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ + } + return SQLITE_NOTFOUND; +} + +/* +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. +** +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. +*/ +#ifndef __QNXNTO__ +static int unixSectorSize(sqlite3_file *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return SQLITE_DEFAULT_SECTOR_SIZE; +} +#endif + +/* +** The following version of unixSectorSize() is optimized for QNX. +*/ +#ifdef __QNXNTO__ +#include +#include +static int unixSectorSize(sqlite3_file *id){ + unixFile *pFile = (unixFile*)id; + if( pFile->sectorSize == 0 ){ + struct statvfs fsInfo; + + /* Set defaults for non-supported filesystems */ + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + pFile->deviceCharacteristics = 0; + if( fstatvfs(pFile->h, &fsInfo) == -1 ) { + return pFile->sectorSize; + } + + if( !strcmp(fsInfo.f_basetype, "tmp") ) { + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC4K | /* All ram filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "etfs") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* etfs cluster size writes are atomic */ + (pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) | + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx6") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC | /* All filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx4") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "dos") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else{ + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC512 | /* blocks are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + 0; + } + } + /* Last chance verification. If the sector size isn't a multiple of 512 + ** then it isn't valid.*/ + if( pFile->sectorSize % 512 != 0 ){ + pFile->deviceCharacteristics = 0; + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + } + return pFile->sectorSize; +} +#endif /* __QNXNTO__ */ + +/* +** Return the device characteristics for the file. +** +** This VFS is set up to return SQLITE_IOCAP_POWERSAFE_OVERWRITE by default. +** However, that choice is contraversial since technically the underlying +** file system does not always provide powersafe overwrites. (In other +** words, after a power-loss event, parts of the file that were never +** written might end up being altered.) However, non-PSOW behavior is very, +** very rare. And asserting PSOW makes a large reduction in the amount +** of required I/O for journaling, since a lot of padding is eliminated. +** Hence, while POWERSAFE_OVERWRITE is on by default, there is a file-control +** available to turn it off and URI query parameter available to turn it off. +*/ +static int unixDeviceCharacteristics(sqlite3_file *id){ + unixFile *p = (unixFile*)id; + int rc = 0; +#ifdef __QNXNTO__ + if( p->sectorSize==0 ) unixSectorSize(id); + rc = p->deviceCharacteristics; +#endif + if( p->ctrlFlags & UNIXFILE_PSOW ){ + rc |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; + } + return rc; +} + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + +/* +** Return the system page size. +** +** This function should not be called directly by other code in this file. +** Instead, it should be called via macro osGetpagesize(). +*/ +static int unixGetpagesize(void){ +#if defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +#endif /* !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 */ + +#ifndef SQLITE_OMIT_WAL + +/* +** Object used to represent an shared memory buffer. +** +** When multiple threads all reference the same wal-index, each thread +** has its own unixShm object, but they all point to a single instance +** of this unixShmNode object. In other words, each wal-index is opened +** only once per process. +** +** Each unixShmNode object is connected to a single unixInodeInfo object. +** We could coalesce this object into unixInodeInfo, but that would mean +** every open file that does not use shared memory (in other words, most +** open files) would have to carry around this extra information. So +** the unixInodeInfo object contains a pointer to this unixShmNode object +** and the unixShmNode object is created only when needed. +** +** unixMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** +** The following fields are read-only after the object is created: +** +** fid +** zFilename +** +** Either unixShmNode.mutex must be held or unixShmNode.nRef==0 and +** unixMutexHeld() is true when reading or writing any other field +** in this structure. +*/ +struct unixShmNode { + unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ + sqlite3_mutex *mutex; /* Mutex to access this object */ + char *zFilename; /* Name of the mmapped file */ + int h; /* Open file descriptor */ + int szRegion; /* Size of shared-memory regions */ + u16 nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + char **apRegion; /* Array of mapped shared-memory regions */ + int nRef; /* Number of unixShm objects pointing to this */ + unixShm *pFirst; /* All unixShm objects pointing to this */ +#ifdef SQLITE_DEBUG + u8 exclMask; /* Mask of exclusive locks held */ + u8 sharedMask; /* Mask of shared locks held */ + u8 nextShmId; /* Next available unixShm.id value */ +#endif +}; + +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** The following fields are initialized when this object is created and +** are read-only thereafter: +** +** unixShm.pFile +** unixShm.id +** +** All other fields are read/write. The unixShm.pFile->mutex must be held +** while accessing any read/write fields. +*/ +struct unixShm { + unixShmNode *pShmNode; /* The underlying unixShmNode object */ + unixShm *pNext; /* Next unixShm with the same unixShmNode */ + u8 hasMutex; /* True if holding the unixShmNode mutex */ + u8 id; /* Id of this connection within its unixShmNode */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ +}; + +/* +** Constants used for locking +*/ +#define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ +#define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ + +/* +** Apply posix advisory locks for all bytes from ofst through ofst+n-1. +** +** Locks block if the mask is exactly UNIX_SHM_C and are non-blocking +** otherwise. +*/ +static int unixShmSystemLock( + unixShmNode *pShmNode, /* Apply locks to this open shared-memory segment */ + int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ + int ofst, /* First byte of the locking range */ + int n /* Number of bytes to lock */ +){ + struct flock f; /* The posix advisory locking structure */ + int rc = SQLITE_OK; /* Result code form fcntl() */ + + /* Access to the unixShmNode object is serialized by the caller */ + assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); + + /* Shared locks never span more than one byte */ + assert( n==1 || lockType!=F_RDLCK ); + + /* Locks are within range */ + assert( n>=1 && nh>=0 ){ + /* Initialize the locking parameters */ + memset(&f, 0, sizeof(f)); + f.l_type = lockType; + f.l_whence = SEEK_SET; + f.l_start = ofst; + f.l_len = n; + + rc = osFcntl(pShmNode->h, F_SETLK, &f); + rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; + } + + /* Update the global lock state and do debug tracing */ +#ifdef SQLITE_DEBUG + { u16 mask; + OSTRACE(("SHM-LOCK ")); + mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; + pShmNode->sharedMask &= ~mask; + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock %d ok", ofst)); + pShmNode->exclMask &= ~mask; + pShmNode->sharedMask |= mask; + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d ok", ofst)); + pShmNode->exclMask |= mask; + pShmNode->sharedMask &= ~mask; + } + }else{ + if( lockType==F_UNLCK ){ + OSTRACE(("unlock %d failed", ofst)); + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock failed")); + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d failed", ofst)); + } + } + OSTRACE((" - afterwards %03x,%03x\n", + pShmNode->sharedMask, pShmNode->exclMask)); + } +#endif + + return rc; +} + +/* +** Return the minimum number of 32KB shm regions that should be mapped at +** a time, assuming that each mapping must be an integer multiple of the +** current system page-size. +** +** Usually, this is 1. The exception seems to be systems that are configured +** to use 64KB pages - in this case each mapping must cover at least two +** shm regions. +*/ +static int unixShmRegionPerMap(void){ + int shmsz = 32*1024; /* SHM region size */ + int pgsz = osGetpagesize(); /* System page size */ + assert( ((pgsz-1)&pgsz)==0 ); /* Page size must be a power of 2 */ + if( pgszpInode->pShmNode; + assert( unixMutexHeld() ); + if( p && p->nRef==0 ){ + int nShmPerMap = unixShmRegionPerMap(); + int i; + assert( p->pInode==pFd->pInode ); + sqlite3_mutex_free(p->mutex); + for(i=0; inRegion; i+=nShmPerMap){ + if( p->h>=0 ){ + osMunmap(p->apRegion[i], p->szRegion); + }else{ + sqlite3_free(p->apRegion[i]); + } + } + sqlite3_free(p->apRegion); + if( p->h>=0 ){ + robust_close(pFd, p->h, __LINE__); + p->h = -1; + } + p->pInode->pShmNode = 0; + sqlite3_free(p); + } +} + +/* +** Open a shared-memory area associated with open database file pDbFd. +** This particular implementation uses mmapped files. +** +** The file used to implement shared-memory is in the same directory +** as the open database file and has the same name as the open database +** file with the "-shm" suffix added. For example, if the database file +** is "/home/user1/config.db" then the file that is created and mmapped +** for shared memory will be called "/home/user1/config.db-shm". +** +** Another approach to is to use files in /dev/shm or /dev/tmp or an +** some other tmpfs mount. But if a file in a different directory +** from the database file is used, then differing access permissions +** or a chroot() might cause two different processes on the same +** database to end up using different files for shared memory - +** meaning that their memory would not really be shared - resulting +** in database corruption. Nevertheless, this tmpfs file usage +** can be enabled at compile-time using -DSQLITE_SHM_DIRECTORY="/dev/shm" +** or the equivalent. The use of the SQLITE_SHM_DIRECTORY compile-time +** option results in an incompatible build of SQLite; builds of SQLite +** that with differing SQLITE_SHM_DIRECTORY settings attempt to use the +** same database file at the same time, database corruption will likely +** result. The SQLITE_SHM_DIRECTORY compile-time option is considered +** "unsupported" and may go away in a future SQLite release. +** +** When opening a new shared-memory file, if no other instances of that +** file are currently open, in this process or in other processes, then +** the file must be truncated to zero length or have its header cleared. +** +** If the original database file (pDbFd) is using the "unix-excl" VFS +** that means that an exclusive lock is held on the database file and +** that no other processes are able to read or write the database. In +** that case, we do not really need shared memory. No shared memory +** file is created. The shared memory will be simulated with heap memory. +*/ +static int unixOpenSharedMemory(unixFile *pDbFd){ + struct unixShm *p = 0; /* The connection to be opened */ + struct unixShmNode *pShmNode; /* The underlying mmapped file */ + int rc; /* Result code */ + unixInodeInfo *pInode; /* The inode of fd */ + char *zShmFilename; /* Name of the file used for SHM */ + int nShmFilename; /* Size of the SHM filename in bytes */ + + /* Allocate space for the new unixShm object. */ + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) return SQLITE_NOMEM; + memset(p, 0, sizeof(*p)); + assert( pDbFd->pShm==0 ); + + /* Check to see if a unixShmNode object already exists. Reuse an existing + ** one if present. Create a new one if necessary. + */ + unixEnterMutex(); + pInode = pDbFd->pInode; + pShmNode = pInode->pShmNode; + if( pShmNode==0 ){ + struct stat sStat; /* fstat() info for database file */ + + /* Call fstat() to figure out the permissions on the database file. If + ** a new *-shm file is created, an attempt will be made to create it + ** with the same permissions. + */ + if( osFstat(pDbFd->h, &sStat) && pInode->bProcessLock==0 ){ + rc = SQLITE_IOERR_FSTAT; + goto shm_open_err; + } + +#ifdef SQLITE_SHM_DIRECTORY + nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 31; +#else + nShmFilename = 6 + (int)strlen(pDbFd->zPath); +#endif + pShmNode = sqlite3_malloc( sizeof(*pShmNode) + nShmFilename ); + if( pShmNode==0 ){ + rc = SQLITE_NOMEM; + goto shm_open_err; + } + memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); + zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1]; +#ifdef SQLITE_SHM_DIRECTORY + sqlite3_snprintf(nShmFilename, zShmFilename, + SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", + (u32)sStat.st_ino, (u32)sStat.st_dev); +#else + sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); +#endif + pShmNode->h = -1; + pDbFd->pInode->pShmNode = pShmNode; + pShmNode->pInode = pDbFd->pInode; + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_NOMEM; + goto shm_open_err; + } + + if( pInode->bProcessLock==0 ){ + int openFlags = O_RDWR | O_CREAT; + if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + openFlags = O_RDONLY; + pShmNode->isReadonly = 1; + } + pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); + if( pShmNode->h<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + goto shm_open_err; + } + + /* If this process is running as root, make sure that the SHM file + ** is owned by the same user that owns the original database. Otherwise, + ** the original owner will not be able to connect. + */ + osFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); + + /* Check to see if another process is holding the dead-man switch. + ** If not, truncate the file to zero length. + */ + rc = SQLITE_OK; + if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ + if( robust_ftruncate(pShmNode->h, 0) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); + } + } + if( rc==SQLITE_OK ){ + rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); + } + if( rc ) goto shm_open_err; + } + } + + /* Make the new connection a child of the unixShmNode */ + p->pShmNode = pShmNode; +#ifdef SQLITE_DEBUG + p->id = pShmNode->nextShmId++; +#endif + pShmNode->nRef++; + pDbFd->pShm = p; + unixLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the unixEnterMutex() mutex and the pointer from the + ** new (struct unixShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** mutex. + */ + sqlite3_mutex_enter(pShmNode->mutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->mutex); + return SQLITE_OK; + + /* Jump here on any error */ +shm_open_err: + unixShmPurge(pDbFd); /* This call frees pShmNode if required */ + sqlite3_free(p); + unixLeaveMutex(); + return rc; +} + +/* +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the bExtend parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** bExtend is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. +*/ +static int unixShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + unixFile *pDbFd = (unixFile*)fd; + unixShm *p; + unixShmNode *pShmNode; + int rc = SQLITE_OK; + int nShmPerMap = unixShmRegionPerMap(); + int nReqRegion; + + /* If the shared-memory file has not yet been opened, open it now. */ + if( pDbFd->pShm==0 ){ + rc = unixOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + } + + p = pDbFd->pShm; + pShmNode = p->pShmNode; + sqlite3_mutex_enter(pShmNode->mutex); + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + assert( pShmNode->pInode==pDbFd->pInode ); + assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); + + /* Minimum number of regions required to be mapped. */ + nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap; + + if( pShmNode->nRegionszRegion = szRegion; + + if( pShmNode->h>=0 ){ + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + if( osFstat(pShmNode->h, &sStat) ){ + rc = SQLITE_IOERR_SHMSIZE; + goto shmpage_out; + } + + if( sStat.st_sizeh, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ + const char *zFile = pShmNode->zFilename; + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); + goto shmpage_out; + } + } + } + } + } + + /* Map the requested memory region into this processes address space. */ + apNew = (char **)sqlite3_realloc( + pShmNode->apRegion, nReqRegion*sizeof(char *) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM; + goto shmpage_out; + } + pShmNode->apRegion = apNew; + while( pShmNode->nRegionh>=0 ){ + pMem = osMmap(0, nMap, + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, + MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion + ); + if( pMem==MAP_FAILED ){ + rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); + goto shmpage_out; + } + }else{ + pMem = sqlite3_malloc(szRegion); + if( pMem==0 ){ + rc = SQLITE_NOMEM; + goto shmpage_out; + } + memset(pMem, 0, szRegion); + } + + for(i=0; iapRegion[pShmNode->nRegion+i] = &((char*)pMem)[szRegion*i]; + } + pShmNode->nRegion += nShmPerMap; + } + } + +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + *pp = pShmNode->apRegion[iRegion]; + }else{ + *pp = 0; + } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; + sqlite3_mutex_leave(pShmNode->mutex); + return rc; +} + +/* +** Change the lock state for a shared-memory segment. +** +** Note that the relationship between SHAREd and EXCLUSIVE locks is a little +** different here than in posix. In xShmLock(), one can go from unlocked +** to shared and back or from unlocked to exclusive and back. But one may +** not go from shared to exclusive or from exclusive to shared. +*/ +static int unixShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + unixFile *pDbFd = (unixFile*)fd; /* Connection holding shared memory */ + unixShm *p = pDbFd->pShm; /* The shared memory being locked */ + unixShm *pX; /* For looping over all siblings */ + unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ + + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); + + mask = (1<<(ofst+n)) - (1<1 || mask==(1<mutex); + if( flags & SQLITE_SHM_UNLOCK ){ + u16 allMask = 0; /* Mask of locks held by siblings */ + + /* See if any siblings hold this same lock */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); + allMask |= pX->sharedMask; + } + + /* Unlock the system-level locks */ + if( (mask & allMask)==0 ){ + rc = unixShmSystemLock(pShmNode, F_UNLCK, ofst+UNIX_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + }else if( flags & SQLITE_SHM_SHARED ){ + u16 allShared = 0; /* Union of locks held by connections other than "p" */ + + /* Find out which shared locks are already held by sibling connections. + ** If any sibling already holds an exclusive lock, go ahead and return + ** SQLITE_BUSY. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + allShared |= pX->sharedMask; + } + + /* Get shared locks at the system level, if necessary */ + if( rc==SQLITE_OK ){ + if( (allShared & mask)==0 ){ + rc = unixShmSystemLock(pShmNode, F_RDLCK, ofst+UNIX_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + } + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + } + + /* Get the exclusive locks at the system level. Then if successful + ** also mark the local connection as being locked. + */ + if( rc==SQLITE_OK ){ + rc = unixShmSystemLock(pShmNode, F_WRLCK, ofst+UNIX_SHM_BASE, n); + if( rc==SQLITE_OK ){ + assert( (p->sharedMask & mask)==0 ); + p->exclMask |= mask; + } + } + } + sqlite3_mutex_leave(pShmNode->mutex); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", + p->id, getpid(), p->sharedMask, p->exclMask)); + return rc; +} + +/* +** Implement a memory barrier or memory fence on shared memory. +** +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. +*/ +static void unixShmBarrier( + sqlite3_file *fd /* Database file holding the shared memory */ +){ + UNUSED_PARAMETER(fd); + unixEnterMutex(); + unixLeaveMutex(); +} + +/* +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. +** +** If there is no shared memory associated with the connection then this +** routine is a harmless no-op. +*/ +static int unixShmUnmap( + sqlite3_file *fd, /* The underlying database file */ + int deleteFlag /* Delete shared-memory if true */ +){ + unixShm *p; /* The connection to be closed */ + unixShmNode *pShmNode; /* The underlying shared-memory file */ + unixShm **pp; /* For looping over sibling connections */ + unixFile *pDbFd; /* The underlying database file */ + + pDbFd = (unixFile*)fd; + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; + + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); + + /* Remove connection p from the set of connections associated + ** with pShmNode */ + sqlite3_mutex_enter(pShmNode->mutex); + for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + sqlite3_mutex_leave(pShmNode->mutex); + + /* If pShmNode->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + unixEnterMutex(); + assert( pShmNode->nRef>0 ); + pShmNode->nRef--; + if( pShmNode->nRef==0 ){ + if( deleteFlag && pShmNode->h>=0 ) osUnlink(pShmNode->zFilename); + unixShmPurge(pDbFd); + } + unixLeaveMutex(); + + return SQLITE_OK; +} + + +#else +# define unixShmMap 0 +# define unixShmLock 0 +# define unixShmBarrier 0 +# define unixShmUnmap 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** If it is currently memory mapped, unmap file pFd. +*/ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); + if( pFd->pMapRegion ){ + osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapSizeActual = 0; + } +} + +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapSizeActual +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + const char *zErr = "mmap"; + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapSizeActual; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapSizeMax ); + assert( nNew>0 ); + assert( pFd->mmapSizeActual>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); + + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + + if( pOrig ){ +#if HAVE_MREMAP + i64 nReuse = pFd->mmapSize; +#else + const int szSyspage = osGetpagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); +#endif + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = 0; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + } + + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapSizeMax = 0; + } + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapSizeActual = nNew; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int unixMapfile(unixFile *pFd, i64 nByte){ + i64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; + } + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + + if( nMap!=pFd->mmapSize ){ + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ + unixUnmapfile(pFd); + } + } + + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ +#endif + *pp = 0; + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#else + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(p); + UNUSED_PARAMETER(iOff); +#endif + return SQLITE_OK; +} + +/* +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ + +/* +** This division contains definitions of sqlite3_io_methods objects that +** implement various file locking strategies. It also contains definitions +** of "finder" functions. A finder-function is used to locate the appropriate +** sqlite3_io_methods object for a particular database file. The pAppData +** field of the sqlite3_vfs VFS objects are initialized to be pointers to +** the correct finder-function for that VFS. +** +** Most finder functions return a pointer to a fixed sqlite3_io_methods +** object. The only interesting finder-function is autolockIoFinder, which +** looks at the filesystem type and tries to guess the best locking +** strategy from that. +** +** For finder-funtion F, two objects are created: +** +** (1) The real finder-function named "FImpt()". +** +** (2) A constant pointer to this function named just "F". +** +** +** A pointer to the F pointer is used as the pAppData value for VFS +** objects. We have to do this instead of letting pAppData point +** directly at the finder-function since C90 rules prevent a void* +** from be cast into a function pointer. +** +** +** Each instance of this macro generates two objects: +** +** * A constant sqlite3_io_methods object call METHOD that has locking +** methods CLOSE, LOCK, UNLOCK, CKRESLOCK. +** +** * An I/O method finder function called FINDER that returns a pointer +** to the METHOD object in the previous bullet. +*/ +#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK) \ +static const sqlite3_io_methods METHOD = { \ + VERSION, /* iVersion */ \ + CLOSE, /* xClose */ \ + unixRead, /* xRead */ \ + unixWrite, /* xWrite */ \ + unixTruncate, /* xTruncate */ \ + unixSync, /* xSync */ \ + unixFileSize, /* xFileSize */ \ + LOCK, /* xLock */ \ + UNLOCK, /* xUnlock */ \ + CKLOCK, /* xCheckReservedLock */ \ + unixFileControl, /* xFileControl */ \ + unixSectorSize, /* xSectorSize */ \ + unixDeviceCharacteristics, /* xDeviceCapabilities */ \ + unixShmMap, /* xShmMap */ \ + unixShmLock, /* xShmLock */ \ + unixShmBarrier, /* xShmBarrier */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ +}; \ +static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ + UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ + return &METHOD; \ +} \ +static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ + = FINDER##Impl; + +/* +** Here are all of the sqlite3_io_methods objects for each of the +** locking strategies. Functions that return pointers to these methods +** are also created. +*/ +IOMETHODS( + posixIoFinder, /* Finder function name */ + posixIoMethods, /* sqlite3_io_methods object name */ + 3, /* shared memory and mmap are enabled */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + unixUnlock, /* xUnlock method */ + unixCheckReservedLock /* xCheckReservedLock method */ +) +IOMETHODS( + nolockIoFinder, /* Finder function name */ + nolockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + nolockClose, /* xClose method */ + nolockLock, /* xLock method */ + nolockUnlock, /* xUnlock method */ + nolockCheckReservedLock /* xCheckReservedLock method */ +) +IOMETHODS( + dotlockIoFinder, /* Finder function name */ + dotlockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + dotlockClose, /* xClose method */ + dotlockLock, /* xLock method */ + dotlockUnlock, /* xUnlock method */ + dotlockCheckReservedLock /* xCheckReservedLock method */ +) + +#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS +IOMETHODS( + flockIoFinder, /* Finder function name */ + flockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + flockClose, /* xClose method */ + flockLock, /* xLock method */ + flockUnlock, /* xUnlock method */ + flockCheckReservedLock /* xCheckReservedLock method */ +) +#endif + +#if OS_VXWORKS +IOMETHODS( + semIoFinder, /* Finder function name */ + semIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + semClose, /* xClose method */ + semLock, /* xLock method */ + semUnlock, /* xUnlock method */ + semCheckReservedLock /* xCheckReservedLock method */ +) +#endif + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + afpIoFinder, /* Finder function name */ + afpIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + afpClose, /* xClose method */ + afpLock, /* xLock method */ + afpUnlock, /* xUnlock method */ + afpCheckReservedLock /* xCheckReservedLock method */ +) +#endif + +/* +** The proxy locking method is a "super-method" in the sense that it +** opens secondary file descriptors for the conch and lock files and +** it uses proxy, dot-file, AFP, and flock() locking methods on those +** secondary files. For this reason, the division that implements +** proxy locking is located much further down in the file. But we need +** to go ahead and define the sqlite3_io_methods and finder function +** for proxy locking here. So we forward declare the I/O methods. +*/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +static int proxyClose(sqlite3_file*); +static int proxyLock(sqlite3_file*, int); +static int proxyUnlock(sqlite3_file*, int); +static int proxyCheckReservedLock(sqlite3_file*, int*); +IOMETHODS( + proxyIoFinder, /* Finder function name */ + proxyIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + proxyClose, /* xClose method */ + proxyLock, /* xLock method */ + proxyUnlock, /* xUnlock method */ + proxyCheckReservedLock /* xCheckReservedLock method */ +) +#endif + +/* nfs lockd on OSX 10.3+ doesn't clear write locks when a read lock is set */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + nfsIoFinder, /* Finder function name */ + nfsIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + nfsUnlock, /* xUnlock method */ + unixCheckReservedLock /* xCheckReservedLock method */ +) +#endif + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** This "finder" function attempts to determine the best locking strategy +** for the database file "filePath". It then returns the sqlite3_io_methods +** object that implements that strategy. +** +** This is for MacOSX only. +*/ +static const sqlite3_io_methods *autolockIoFinderImpl( + const char *filePath, /* name of the database file */ + unixFile *pNew /* open file object for the database file */ +){ + static const struct Mapping { + const char *zFilesystem; /* Filesystem type name */ + const sqlite3_io_methods *pMethods; /* Appropriate locking method */ + } aMap[] = { + { "hfs", &posixIoMethods }, + { "ufs", &posixIoMethods }, + { "afpfs", &afpIoMethods }, + { "smbfs", &afpIoMethods }, + { "webdav", &nolockIoMethods }, + { 0, 0 } + }; + int i; + struct statfs fsInfo; + struct flock lockInfo; + + if( !filePath ){ + /* If filePath==NULL that means we are dealing with a transient file + ** that does not need to be locked. */ + return &nolockIoMethods; + } + if( statfs(filePath, &fsInfo) != -1 ){ + if( fsInfo.f_flags & MNT_RDONLY ){ + return &nolockIoMethods; + } + for(i=0; aMap[i].zFilesystem; i++){ + if( strcmp(fsInfo.f_fstypename, aMap[i].zFilesystem)==0 ){ + return aMap[i].pMethods; + } + } + } + + /* Default case. Handles, amongst others, "nfs". + ** Test byte-range lock using fcntl(). If the call succeeds, + ** assume that the file-system supports POSIX style locks. + */ + lockInfo.l_len = 1; + lockInfo.l_start = 0; + lockInfo.l_whence = SEEK_SET; + lockInfo.l_type = F_RDLCK; + if( osFcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { + if( strcmp(fsInfo.f_fstypename, "nfs")==0 ){ + return &nfsIoMethods; + } else { + return &posixIoMethods; + } + }else{ + return &dotlockIoMethods; + } +} +static const sqlite3_io_methods + *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + +#if OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE +/* +** This "finder" function attempts to determine the best locking strategy +** for the database file "filePath". It then returns the sqlite3_io_methods +** object that implements that strategy. +** +** This is for VXWorks only. +*/ +static const sqlite3_io_methods *autolockIoFinderImpl( + const char *filePath, /* name of the database file */ + unixFile *pNew /* the open file object */ +){ + struct flock lockInfo; + + if( !filePath ){ + /* If filePath==NULL that means we are dealing with a transient file + ** that does not need to be locked. */ + return &nolockIoMethods; + } + + /* Test if fcntl() is supported and use POSIX style locks. + ** Otherwise fall back to the named semaphore method. + */ + lockInfo.l_len = 1; + lockInfo.l_start = 0; + lockInfo.l_whence = SEEK_SET; + lockInfo.l_type = F_RDLCK; + if( osFcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { + return &posixIoMethods; + }else{ + return &semIoMethods; + } +} +static const sqlite3_io_methods + *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; + +#endif /* OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE */ + +/* +** An abstract type for a pointer to a IO method finder function: +*/ +typedef const sqlite3_io_methods *(*finder_type)(const char*,unixFile*); + + +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** +** +** This division contains the implementation of methods on the +** sqlite3_vfs object. +*/ + +/* +** Initialize the contents of the unixFile structure pointed to by pId. +*/ +static int fillInUnixFile( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + int h, /* Open file descriptor of file being opened */ + sqlite3_file *pId, /* Write to the unixFile structure here */ + const char *zFilename, /* Name of the file being opened */ + int ctrlFlags /* Zero or more UNIXFILE_* values */ +){ + const sqlite3_io_methods *pLockingStyle; + unixFile *pNew = (unixFile *)pId; + int rc = SQLITE_OK; + + assert( pNew->pInode==NULL ); + + /* Usually the path zFilename should not be a relative pathname. The + ** exception is when opening the proxy "conch" file in builds that + ** include the special Apple locking styles. + */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + assert( zFilename==0 || zFilename[0]=='/' + || pVfs->pAppData==(void*)&autolockIoFinder ); +#else + assert( zFilename==0 || zFilename[0]=='/' ); +#endif + + /* No locking occurs in temporary files */ + assert( zFilename!=0 || (ctrlFlags & UNIXFILE_NOLOCK)!=0 ); + + OSTRACE(("OPEN %-3d %s\n", h, zFilename)); + pNew->h = h; + pNew->pVfs = pVfs; + pNew->zPath = zFilename; + pNew->ctrlFlags = (u8)ctrlFlags; +#if SQLITE_MAX_MMAP_SIZE>0 + pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif + if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), + "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + pNew->ctrlFlags |= UNIXFILE_PSOW; + } + if( strcmp(pVfs->zName,"unix-excl")==0 ){ + pNew->ctrlFlags |= UNIXFILE_EXCL; + } + +#if OS_VXWORKS + pNew->pId = vxworksFindFileId(zFilename); + if( pNew->pId==0 ){ + ctrlFlags |= UNIXFILE_NOLOCK; + rc = SQLITE_NOMEM; + } +#endif + + if( ctrlFlags & UNIXFILE_NOLOCK ){ + pLockingStyle = &nolockIoMethods; + }else{ + pLockingStyle = (**(finder_type*)pVfs->pAppData)(zFilename, pNew); +#if SQLITE_ENABLE_LOCKING_STYLE + /* Cache zFilename in the locking context (AFP and dotlock override) for + ** proxyLock activation is possible (remote proxy is based on db name) + ** zFilename remains valid until file is closed, to support */ + pNew->lockingContext = (void*)zFilename; +#endif + } + + if( pLockingStyle == &posixIoMethods +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + || pLockingStyle == &nfsIoMethods +#endif + ){ + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( rc!=SQLITE_OK ){ + /* If an error occurred in findInodeInfo(), close the file descriptor + ** immediately, before releasing the mutex. findInodeInfo() may fail + ** in two scenarios: + ** + ** (a) A call to fstat() failed. + ** (b) A malloc failed. + ** + ** Scenario (b) may only occur if the process is holding no other + ** file descriptors open on the same file. If there were other file + ** descriptors on this file, then no malloc would be required by + ** findInodeInfo(). If this is the case, it is quite safe to close + ** handle h - as it is guaranteed that no posix locks will be released + ** by doing so. + ** + ** If scenario (a) caused the error then things are not so safe. The + ** implicit assumption here is that if fstat() fails, things are in + ** such bad shape that dropping a lock or two doesn't matter much. + */ + robust_close(pNew, h, __LINE__); + h = -1; + } + unixLeaveMutex(); + } + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + else if( pLockingStyle == &afpIoMethods ){ + /* AFP locking uses the file path so it needs to be included in + ** the afpLockingContext. + */ + afpLockingContext *pCtx; + pNew->lockingContext = pCtx = sqlite3_malloc( sizeof(*pCtx) ); + if( pCtx==0 ){ + rc = SQLITE_NOMEM; + }else{ + /* NB: zFilename exists and remains valid until the file is closed + ** according to requirement F11141. So we do not need to make a + ** copy of the filename. */ + pCtx->dbPath = zFilename; + pCtx->reserved = 0; + srandomdev(); + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew->lockingContext); + robust_close(pNew, h, __LINE__); + h = -1; + } + unixLeaveMutex(); + } + } +#endif + + else if( pLockingStyle == &dotlockIoMethods ){ + /* Dotfile locking uses the file path so it needs to be included in + ** the dotlockLockingContext + */ + char *zLockFile; + int nFilename; + assert( zFilename!=0 ); + nFilename = (int)strlen(zFilename) + 6; + zLockFile = (char *)sqlite3_malloc(nFilename); + if( zLockFile==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); + } + pNew->lockingContext = zLockFile; + } + +#if OS_VXWORKS + else if( pLockingStyle == &semIoMethods ){ + /* Named semaphore locking uses the file path so it needs to be + ** included in the semLockingContext + */ + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( (rc==SQLITE_OK) && (pNew->pInode->pSem==NULL) ){ + char *zSemName = pNew->pInode->aSemName; + int n; + sqlite3_snprintf(MAX_PATHNAME, zSemName, "/%s.sem", + pNew->pId->zCanonicalName); + for( n=1; zSemName[n]; n++ ) + if( zSemName[n]=='/' ) zSemName[n] = '_'; + pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); + if( pNew->pInode->pSem == SEM_FAILED ){ + rc = SQLITE_NOMEM; + pNew->pInode->aSemName[0] = '\0'; + } + } + unixLeaveMutex(); + } +#endif + + pNew->lastErrno = 0; +#if OS_VXWORKS + if( rc!=SQLITE_OK ){ + if( h>=0 ) robust_close(pNew, h, __LINE__); + h = -1; + osUnlink(zFilename); + pNew->ctrlFlags |= UNIXFILE_DELETE; + } +#endif + if( rc!=SQLITE_OK ){ + if( h>=0 ) robust_close(pNew, h, __LINE__); + }else{ + pNew->pMethod = pLockingStyle; + OpenCounter(+1); + verifyDbFile(pNew); + } + return rc; +} + +/* +** Return the name of a directory in which to put temporary files. +** If no suitable temporary file directory can be found, return NULL. +*/ +static const char *unixTempFileDir(void){ + static const char *azDirs[] = { + 0, + 0, + 0, + "/var/tmp", + "/usr/tmp", + "/tmp", + 0 /* List terminator */ + }; + unsigned int i; + struct stat buf; + const char *zDir = 0; + + azDirs[0] = sqlite3_temp_directory; + if( !azDirs[1] ) azDirs[1] = getenv("SQLITE_TMPDIR"); + if( !azDirs[2] ) azDirs[2] = getenv("TMPDIR"); + for(i=0; imxPathname bytes. +*/ +static int unixGetTempname(int nBuf, char *zBuf){ + static const unsigned char zChars[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789"; + unsigned int i, j; + const char *zDir; + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. + */ + SimulateIOError( return SQLITE_IOERR ); + + zDir = unixTempFileDir(); + if( zDir==0 ) zDir = "."; + + /* Check that the output buffer is large enough for the temporary file + ** name. If it is not, return SQLITE_ERROR. + */ + if( (strlen(zDir) + strlen(SQLITE_TEMP_FILE_PREFIX) + 18) >= (size_t)nBuf ){ + return SQLITE_ERROR; + } + + do{ + sqlite3_snprintf(nBuf-18, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX, zDir); + j = (int)strlen(zBuf); + sqlite3_randomness(15, &zBuf[j]); + for(i=0; i<15; i++, j++){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + zBuf[j+1] = 0; + }while( osAccess(zBuf,0)==0 ); + return SQLITE_OK; +} + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) +/* +** Routine to transform a unixFile into a proxy-locking unixFile. +** Implementation in the proxy-lock division, but used by unixOpen() +** if SQLITE_PREFER_PROXY_LOCKING is defined. +*/ +static int proxyTransformUnixFile(unixFile*, const char*); +#endif + +/* +** Search for an unused file descriptor that was opened on the database +** file (not a journal or master-journal file) identified by pathname +** zPath with SQLITE_OPEN_XXX flags matching those passed as the second +** argument to this function. +** +** Such a file descriptor may exist if a database connection was closed +** but the associated file descriptor could not be closed because some +** other file descriptor open on the same file is holding a file-lock. +** Refer to comments in the unixClose() function and the lengthy comment +** describing "Posix Advisory Locking" at the start of this file for +** further details. Also, ticket #4018. +** +** If a suitable file descriptor is found, then it is returned. If no +** such file descriptor is located, -1 is returned. +*/ +static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ + UnixUnusedFd *pUnused = 0; + + /* Do not search for an unused file descriptor on vxworks. Not because + ** vxworks would not benefit from the change (it might, we're not sure), + ** but because no way to test it is currently available. It is better + ** not to risk breaking vxworks support for the sake of such an obscure + ** feature. */ +#if !OS_VXWORKS + struct stat sStat; /* Results of stat() call */ + + /* A stat() call may fail for various reasons. If this happens, it is + ** almost certain that an open() call on the same path will also fail. + ** For this reason, if an error occurs in the stat() call here, it is + ** ignored and -1 is returned. The caller will try to open a new file + ** descriptor on the same path, fail, and return an error to SQLite. + ** + ** Even if a subsequent open() call does succeed, the consequences of + ** not searching for a resusable file descriptor are not dire. */ + if( 0==osStat(zPath, &sStat) ){ + unixInodeInfo *pInode; + + unixEnterMutex(); + pInode = inodeList; + while( pInode && (pInode->fileId.dev!=sStat.st_dev + || pInode->fileId.ino!=sStat.st_ino) ){ + pInode = pInode->pNext; + } + if( pInode ){ + UnixUnusedFd **pp; + for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); + pUnused = *pp; + if( pUnused ){ + *pp = pUnused->pNext; + } + } + unixLeaveMutex(); + } +#endif /* if !OS_VXWORKS */ + return pUnused; +} + +/* +** This function is called by unixOpen() to determine the unix permissions +** to create new files with. If no error occurs, then SQLITE_OK is returned +** and a value suitable for passing as the third argument to open(2) is +** written to *pMode. If an IO error occurs, an SQLite error code is +** returned and the value of *pMode is not modified. +** +** In most cases cases, this routine sets *pMode to 0, which will become +** an indication to robust_open() to create the file using +** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. +** But if the file being opened is a WAL or regular journal file, then +** this function queries the file-system for the permissions on the +** corresponding database file and sets *pMode to this value. Whenever +** possible, WAL and journal files are created using the same permissions +** as the associated database file. +** +** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the +** original filename is unavailable. But 8_3_NAMES is only used for +** FAT filesystems and permissions do not matter there, so just use +** the default permissions. +*/ +static int findCreateFileMode( + const char *zPath, /* Path of file (possibly) being created */ + int flags, /* Flags passed as 4th argument to xOpen() */ + mode_t *pMode, /* OUT: Permissions to open file with */ + uid_t *pUid, /* OUT: uid to set on the file */ + gid_t *pGid /* OUT: gid to set on the file */ +){ + int rc = SQLITE_OK; /* Return Code */ + *pMode = 0; + *pUid = 0; + *pGid = 0; + if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + char zDb[MAX_PATHNAME+1]; /* Database file path */ + int nDb; /* Number of valid bytes in zDb */ + struct stat sStat; /* Output of stat() on database file */ + + /* zPath is a path to a WAL or journal file. The following block derives + ** the path to the associated database file from zPath. This block handles + ** the following naming conventions: + ** + ** "-journal" + ** "-wal" + ** "-journalNN" + ** "-walNN" + ** + ** where NN is a decimal number. The NN naming schemes are + ** used by the test_multiplex.c module. + */ + nDb = sqlite3Strlen30(zPath) - 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + while( nDb>0 && sqlite3Isalnum(zPath[nDb]) ) nDb--; + if( nDb==0 || zPath[nDb]!='-' ) return SQLITE_OK; +#else + while( zPath[nDb]!='-' ){ + assert( nDb>0 ); + assert( zPath[nDb]!='\n' ); + nDb--; + } +#endif + memcpy(zDb, zPath, nDb); + zDb[nDb] = '\0'; + + if( 0==osStat(zDb, &sStat) ){ + *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; + }else{ + rc = SQLITE_IOERR_FSTAT; + } + }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ + *pMode = 0600; + } + return rc; +} + +/* +** Open the file zPath. +** +** Previously, the SQLite OS layer used three functions in place of this +** one: +** +** sqlite3OsOpenReadWrite(); +** sqlite3OsOpenReadOnly(); +** sqlite3OsOpenExclusive(); +** +** These calls correspond to the following combinations of flags: +** +** ReadWrite() -> (READWRITE | CREATE) +** ReadOnly() -> (READONLY) +** OpenExclusive() -> (READWRITE | CREATE | EXCLUSIVE) +** +** The old OpenExclusive() accepted a boolean argument - "delFlag". If +** true, the file was configured to be automatically deleted when the +** file handle closed. To achieve the same effect using this new +** interface, add the DELETEONCLOSE flag to those specified above for +** OpenExclusive(). +*/ +static int unixOpen( + sqlite3_vfs *pVfs, /* The VFS for which this is the xOpen method */ + const char *zPath, /* Pathname of file to be opened */ + sqlite3_file *pFile, /* The file descriptor to be filled in */ + int flags, /* Input flags to control the opening */ + int *pOutFlags /* Output flags returned to SQLite core */ +){ + unixFile *p = (unixFile *)pFile; + int fd = -1; /* File descriptor returned by open() */ + int openFlags = 0; /* Flags to pass to open() */ + int eType = flags&0xFFFFFF00; /* Type of file to open */ + int noLock; /* True to omit locking primitives */ + int rc = SQLITE_OK; /* Function Return Code */ + int ctrlFlags = 0; /* UNIXFILE_* flags */ + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); + int isReadonly = (flags & SQLITE_OPEN_READONLY); + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); +#if SQLITE_ENABLE_LOCKING_STYLE + int isAutoProxy = (flags & SQLITE_OPEN_AUTOPROXY); +#endif +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + struct statfs fsInfo; +#endif + + /* If creating a master or main-file journal, this function will open + ** a file-descriptor on the directory too. The first time unixSync() + ** is called the directory file descriptor will be fsync()ed and close()d. + */ + int syncDir = (isCreate && ( + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char zTmpname[MAX_PATHNAME+2]; + const char *zName = zPath; + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, WAL file and master journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); + + /* Detect a pid change and reset the PRNG. There is a race condition + ** here such that two or more threads all trying to open databases at + ** the same instant might all reset the PRNG. But multiple resets + ** are harmless. + */ + if( randomnessPid!=getpid() ){ + randomnessPid = getpid(); + sqlite3_randomness(0,0); + } + + memset(p, 0, sizeof(unixFile)); + + if( eType==SQLITE_OPEN_MAIN_DB ){ + UnixUnusedFd *pUnused; + pUnused = findReusableFd(zName, flags); + if( pUnused ){ + fd = pUnused->fd; + }else{ + pUnused = sqlite3_malloc(sizeof(*pUnused)); + if( !pUnused ){ + return SQLITE_NOMEM; + } + } + p->pUnused = pUnused; + + /* Database filenames are double-zero terminated if they are not + ** URIs with parameters. Hence, they can always be passed into + ** sqlite3_uri_parameter(). */ + assert( (flags & SQLITE_OPEN_URI) || zName[strlen(zName)+1]==0 ); + + }else if( !zName ){ + /* If zName is NULL, the upper layer is requesting a temp file. */ + assert(isDelete && !syncDir); + rc = unixGetTempname(MAX_PATHNAME+2, zTmpname); + if( rc!=SQLITE_OK ){ + return rc; + } + zName = zTmpname; + + /* Generated temporary filenames are always double-zero terminated + ** for use by sqlite3_uri_parameter(). */ + assert( zName[strlen(zName)+1]==0 ); + } + + /* Determine the value of the flags parameter passed to POSIX function + ** open(). These must be calculated even if open() is not called, as + ** they may be stored as part of the file handle and used by the + ** 'conch file' locking functions later on. */ + if( isReadonly ) openFlags |= O_RDONLY; + if( isReadWrite ) openFlags |= O_RDWR; + if( isCreate ) openFlags |= O_CREAT; + if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); + openFlags |= (O_LARGEFILE|O_BINARY); + + if( fd<0 ){ + mode_t openMode; /* Permissions to create file with */ + uid_t uid; /* Userid for the file */ + gid_t gid; /* Groupid for the file */ + rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid); + if( rc!=SQLITE_OK ){ + assert( !p->pUnused ); + assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); + return rc; + } + fd = robust_open(zName, openFlags, openMode); + OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); + if( fd<0 && errno!=EISDIR && isReadWrite && !isExclusive ){ + /* Failed to open the file for read/write access. Try read-only. */ + flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + openFlags &= ~(O_RDWR|O_CREAT); + flags |= SQLITE_OPEN_READONLY; + openFlags |= O_RDONLY; + isReadonly = 1; + fd = robust_open(zName, openFlags, openMode); + } + if( fd<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + goto open_finished; + } + + /* If this process is running as root and if creating a new rollback + ** journal or WAL file, set the ownership of the journal or WAL to be + ** the same as the original database. + */ + if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + osFchown(fd, uid, gid); + } + } + assert( fd>=0 ); + if( pOutFlags ){ + *pOutFlags = flags; + } + + if( p->pUnused ){ + p->pUnused->fd = fd; + p->pUnused->flags = flags; + } + + if( isDelete ){ +#if OS_VXWORKS + zPath = zName; +#elif defined(SQLITE_UNLINK_AFTER_CLOSE) + zPath = sqlite3_mprintf("%s", zName); + if( zPath==0 ){ + robust_close(p, fd, __LINE__); + return SQLITE_NOMEM; + } +#else + osUnlink(zName); +#endif + } +#if SQLITE_ENABLE_LOCKING_STYLE + else{ + p->openFlags = openFlags; + } +#endif + + noLock = eType!=SQLITE_OPEN_MAIN_DB; + + +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + if( fstatfs(fd, &fsInfo) == -1 ){ + ((unixFile*)pFile)->lastErrno = errno; + robust_close(p, fd, __LINE__); + return SQLITE_IOERR_ACCESS; + } + if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; + } +#endif + + /* Set up appropriate ctrlFlags */ + if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; + if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; + if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; + if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; + if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; + +#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_PREFER_PROXY_LOCKING + isAutoProxy = 1; +#endif + if( isAutoProxy && (zPath!=NULL) && (!noLock) && pVfs->xOpen ){ + char *envforce = getenv("SQLITE_FORCE_PROXY_LOCKING"); + int useProxy = 0; + + /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means + ** never use proxy, NULL means use proxy for non-local files only. */ + if( envforce!=NULL ){ + useProxy = atoi(envforce)>0; + }else{ + if( statfs(zPath, &fsInfo) == -1 ){ + /* In theory, the close(fd) call is sub-optimal. If the file opened + ** with fd is a database file, and there are other connections open + ** on that file that are currently holding advisory locks on it, + ** then the call to close() will cancel those locks. In practice, + ** we're assuming that statfs() doesn't fail very often. At least + ** not while other file descriptors opened by the same process on + ** the same file are working. */ + p->lastErrno = errno; + robust_close(p, fd, __LINE__); + rc = SQLITE_IOERR_ACCESS; + goto open_finished; + } + useProxy = !(fsInfo.f_flags&MNT_LOCAL); + } + if( useProxy ){ + rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); + if( rc==SQLITE_OK ){ + rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); + if( rc!=SQLITE_OK ){ + /* Use unixClose to clean up the resources added in fillInUnixFile + ** and clear all the structure's references. Specifically, + ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op + */ + unixClose(pFile); + return rc; + } + } + goto open_finished; + } + } +#endif + + rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); + +open_finished: + if( rc!=SQLITE_OK ){ + sqlite3_free(p->pUnused); + } + return rc; +} + + +/* +** Delete the file at zPath. If the dirSync argument is true, fsync() +** the directory after deleting the file. +*/ +static int unixDelete( + sqlite3_vfs *NotUsed, /* VFS containing this as the xDelete method */ + const char *zPath, /* Name of file to be deleted */ + int dirSync /* If true, fsync() directory after deleting file */ +){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(NotUsed); + SimulateIOError(return SQLITE_IOERR_DELETE); + if( osUnlink(zPath)==(-1) ){ + if( errno==ENOENT +#if OS_VXWORKS + || errno==0x380003 +#endif + ){ + rc = SQLITE_IOERR_DELETE_NOENT; + }else{ + rc = unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); + } + return rc; + } +#ifndef SQLITE_DISABLE_DIRSYNC + if( (dirSync & 1)!=0 ){ + int fd; + rc = osOpenDirectory(zPath, &fd); + if( rc==SQLITE_OK ){ +#if OS_VXWORKS + if( fsync(fd)==-1 ) +#else + if( fsync(fd) ) +#endif + { + rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); + } + robust_close(0, fd, __LINE__); + }else if( rc==SQLITE_CANTOPEN ){ + rc = SQLITE_OK; + } + } +#endif + return rc; +} + +/* +** Test the existence of or access permissions of file zPath. The +** test performed depends on the value of flags: +** +** SQLITE_ACCESS_EXISTS: Return 1 if the file exists +** SQLITE_ACCESS_READWRITE: Return 1 if the file is read and writable. +** SQLITE_ACCESS_READONLY: Return 1 if the file is readable. +** +** Otherwise return 0. +*/ +static int unixAccess( + sqlite3_vfs *NotUsed, /* The VFS containing this xAccess method */ + const char *zPath, /* Path of the file to examine */ + int flags, /* What do we want to learn about the zPath file? */ + int *pResOut /* Write result boolean here */ +){ + int amode = 0; + UNUSED_PARAMETER(NotUsed); + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + switch( flags ){ + case SQLITE_ACCESS_EXISTS: + amode = F_OK; + break; + case SQLITE_ACCESS_READWRITE: + amode = W_OK|R_OK; + break; + case SQLITE_ACCESS_READ: + amode = R_OK; + break; + + default: + assert(!"Invalid flags argument"); + } + *pResOut = (osAccess(zPath, amode)==0); + if( flags==SQLITE_ACCESS_EXISTS && *pResOut ){ + struct stat buf; + if( 0==osStat(zPath, &buf) && buf.st_size==0 ){ + *pResOut = 0; + } + } + return SQLITE_OK; +} + + +/* +** Turn a relative pathname into a full pathname. The relative path +** is stored as a nul-terminated string in the buffer pointed to by +** zPath. +** +** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes +** (in this case, MAX_PATHNAME bytes). The full-path is written to +** this buffer before returning. +*/ +static int unixFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zPath, /* Possibly relative input path */ + int nOut, /* Size of output buffer in bytes */ + char *zOut /* Output buffer */ +){ + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); + + assert( pVfs->mxPathname==MAX_PATHNAME ); + UNUSED_PARAMETER(pVfs); + + zOut[nOut-1] = '\0'; + if( zPath[0]=='/' ){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); + }else{ + int nCwd; + if( osGetcwd(zOut, nOut-1)==0 ){ + return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + } + nCwd = (int)strlen(zOut); + sqlite3_snprintf(nOut-nCwd, &zOut[nCwd], "/%s", zPath); + } + return SQLITE_OK; +} + + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. +*/ +#include +static void *unixDlOpen(sqlite3_vfs *NotUsed, const char *zFilename){ + UNUSED_PARAMETER(NotUsed); + return dlopen(zFilename, RTLD_NOW | RTLD_GLOBAL); +} + +/* +** SQLite calls this function immediately after a call to unixDlSym() or +** unixDlOpen() fails (returns a null pointer). If a more detailed error +** message is available, it is written to zBufOut. If no error message +** is available, zBufOut is left unmodified and SQLite uses a default +** error message. +*/ +static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ + const char *zErr; + UNUSED_PARAMETER(NotUsed); + unixEnterMutex(); + zErr = dlerror(); + if( zErr ){ + sqlite3_snprintf(nBuf, zBufOut, "%s", zErr); + } + unixLeaveMutex(); +} +static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ + /* + ** GCC with -pedantic-errors says that C90 does not allow a void* to be + ** cast into a pointer to a function. And yet the library dlsym() routine + ** returns a void* which is really a pointer to a function. So how do we + ** use dlsym() with -pedantic-errors? + ** + ** Variable x below is defined to be a pointer to a function taking + ** parameters void* and const char* and returning a pointer to a function. + ** We initialize x by assigning it a pointer to the dlsym() function. + ** (That assignment requires a cast.) Then we call the function that + ** x points to. + ** + ** This work-around is unlikely to work correctly on any system where + ** you really cannot cast a function pointer into void*. But then, on the + ** other hand, dlsym() will not work on such a system either, so we have + ** not really lost anything. + */ + void (*(*x)(void*,const char*))(void); + UNUSED_PARAMETER(NotUsed); + x = (void(*(*)(void*,const char*))(void))dlsym; + return (*x)(p, zSym); +} +static void unixDlClose(sqlite3_vfs *NotUsed, void *pHandle){ + UNUSED_PARAMETER(NotUsed); + dlclose(pHandle); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define unixDlOpen 0 + #define unixDlError 0 + #define unixDlSym 0 + #define unixDlClose 0 +#endif + +/* +** Write nBuf bytes of random data to the supplied buffer zBuf. +*/ +static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ + UNUSED_PARAMETER(NotUsed); + assert((size_t)nBuf>=(sizeof(time_t)+sizeof(int))); + + /* We have to initialize zBuf to prevent valgrind from reporting + ** errors. The reports issued by valgrind are incorrect - we would + ** prefer that the randomness be increased by making use of the + ** uninitialized space in zBuf - but valgrind errors tend to worry + ** some users. Rather than argue, it seems easier just to initialize + ** the whole array and silence valgrind, even if that means less randomness + ** in the random seed. + ** + ** When testing, initializing zBuf[] to zero is all we do. That means + ** that we always use the same random number sequence. This makes the + ** tests repeatable. + */ + memset(zBuf, 0, nBuf); + randomnessPid = getpid(); +#if !defined(SQLITE_TEST) + { + int fd, got; + fd = robust_open("/dev/urandom", O_RDONLY, 0); + if( fd<0 ){ + time_t t; + time(&t); + memcpy(zBuf, &t, sizeof(t)); + memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid)); + assert( sizeof(t)+sizeof(randomnessPid)<=(size_t)nBuf ); + nBuf = sizeof(t) + sizeof(randomnessPid); + }else{ + do{ got = osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR ); + robust_close(0, fd, __LINE__); + } + } +#endif + return nBuf; +} + + +/* +** Sleep for a little while. Return the amount of time slept. +** The argument is the number of microseconds we want to sleep. +** The return value is the number of microseconds of sleep actually +** requested from the underlying operating system, a number which +** might be greater than or equal to the argument, but not less +** than the argument. +*/ +static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ +#if OS_VXWORKS + struct timespec sp; + + sp.tv_sec = microseconds / 1000000; + sp.tv_nsec = (microseconds % 1000000) * 1000; + nanosleep(&sp, NULL); + UNUSED_PARAMETER(NotUsed); + return microseconds; +#elif defined(HAVE_USLEEP) && HAVE_USLEEP + usleep(microseconds); + UNUSED_PARAMETER(NotUsed); + return microseconds; +#else + int seconds = (microseconds+999999)/1000000; + sleep(seconds); + UNUSED_PARAMETER(NotUsed); + return seconds*1000000; +#endif +} + +/* +** The following variable, if set to a non-zero value, is interpreted as +** the number of seconds since 1970 and is used to set the result of +** sqlite3OsCurrentTime() during testing. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +#endif + +/* +** Find the current time (in Universal Coordinated Time). Write into *piNow +** the current time and date as a Julian Day number times 86_400_000. In +** other words, write into *piNow the number of milliseconds since the Julian +** epoch of noon in Greenwich on November 24, 4714 B.C according to the +** proleptic Gregorian calendar. +** +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** cannot be found. +*/ +static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + int rc = SQLITE_OK; +#if defined(NO_GETTOD) + time_t t; + time(&t); + *piNow = ((sqlite3_int64)t)*1000 + unixEpoch; +#elif OS_VXWORKS + struct timespec sNow; + clock_gettime(CLOCK_REALTIME, &sNow); + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; +#else + struct timeval sNow; + if( gettimeofday(&sNow, 0)==0 ){ + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + }else{ + rc = SQLITE_ERROR; + } +#endif + +#ifdef SQLITE_TEST + if( sqlite3_current_time ){ + *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; + } +#endif + UNUSED_PARAMETER(NotUsed); + return rc; +} + +/* +** Find the current time (in Universal Coordinated Time). Write the +** current time and date as a Julian Day number into *prNow and +** return 0. Return 1 if the time and date cannot be found. +*/ +static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ + sqlite3_int64 i = 0; + int rc; + UNUSED_PARAMETER(NotUsed); + rc = unixCurrentTimeInt64(0, &i); + *prNow = i/86400000.0; + return rc; +} + +/* +** We added the xGetLastError() method with the intention of providing +** better low-level error messages when operating-system problems come up +** during SQLite operation. But so far, none of that has been implemented +** in the core. So this routine is never called. For now, it is merely +** a place-holder. +*/ +static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ + UNUSED_PARAMETER(NotUsed); + UNUSED_PARAMETER(NotUsed2); + UNUSED_PARAMETER(NotUsed3); + return 0; +} + + +/* +************************ End of sqlite3_vfs methods *************************** +******************************************************************************/ + +/****************************************************************************** +************************** Begin Proxy Locking ******************************** +** +** Proxy locking is a "uber-locking-method" in this sense: It uses the +** other locking methods on secondary lock files. Proxy locking is a +** meta-layer over top of the primitive locking implemented above. For +** this reason, the division that implements of proxy locking is deferred +** until late in the file (here) after all of the other I/O methods have +** been defined - so that the primitive locking methods are available +** as services to help with the implementation of proxy locking. +** +**** +** +** The default locking schemes in SQLite use byte-range locks on the +** database file to coordinate safe, concurrent access by multiple readers +** and writers [http://sqlite.org/lockingv3.html]. The five file locking +** states (UNLOCKED, PENDING, SHARED, RESERVED, EXCLUSIVE) are implemented +** as POSIX read & write locks over fixed set of locations (via fsctl), +** on AFP and SMB only exclusive byte-range locks are available via fsctl +** with _IOWR('z', 23, struct ByteRangeLockPB2) to track the same 5 states. +** To simulate a F_RDLCK on the shared range, on AFP a randomly selected +** address in the shared range is taken for a SHARED lock, the entire +** shared range is taken for an EXCLUSIVE lock): +** +** PENDING_BYTE 0x40000000 +** RESERVED_BYTE 0x40000001 +** SHARED_RANGE 0x40000002 -> 0x40000200 +** +** This works well on the local file system, but shows a nearly 100x +** slowdown in read performance on AFP because the AFP client disables +** the read cache when byte-range locks are present. Enabling the read +** cache exposes a cache coherency problem that is present on all OS X +** supported network file systems. NFS and AFP both observe the +** close-to-open semantics for ensuring cache coherency +** [http://nfs.sourceforge.net/#faq_a8], which does not effectively +** address the requirements for concurrent database access by multiple +** readers and writers +** [http://www.nabble.com/SQLite-on-NFS-cache-coherency-td15655701.html]. +** +** To address the performance and cache coherency issues, proxy file locking +** changes the way database access is controlled by limiting access to a +** single host at a time and moving file locks off of the database file +** and onto a proxy file on the local file system. +** +** +** Using proxy locks +** ----------------- +** +** C APIs +** +** sqlite3_file_control(db, dbname, SQLITE_SET_LOCKPROXYFILE, +** | ":auto:"); +** sqlite3_file_control(db, dbname, SQLITE_GET_LOCKPROXYFILE, &); +** +** +** SQL pragmas +** +** PRAGMA [database.]lock_proxy_file= | :auto: +** PRAGMA [database.]lock_proxy_file +** +** Specifying ":auto:" means that if there is a conch file with a matching +** host ID in it, the proxy path in the conch file will be used, otherwise +** a proxy path based on the user's temp dir +** (via confstr(_CS_DARWIN_USER_TEMP_DIR,...)) will be used and the +** actual proxy file name is generated from the name and path of the +** database file. For example: +** +** For database path "/Users/me/foo.db" +** The lock path will be "/sqliteplocks/_Users_me_foo.db:auto:") +** +** Once a lock proxy is configured for a database connection, it can not +** be removed, however it may be switched to a different proxy path via +** the above APIs (assuming the conch file is not being held by another +** connection or process). +** +** +** How proxy locking works +** ----------------------- +** +** Proxy file locking relies primarily on two new supporting files: +** +** * conch file to limit access to the database file to a single host +** at a time +** +** * proxy file to act as a proxy for the advisory locks normally +** taken on the database +** +** The conch file - to use a proxy file, sqlite must first "hold the conch" +** by taking an sqlite-style shared lock on the conch file, reading the +** contents and comparing the host's unique host ID (see below) and lock +** proxy path against the values stored in the conch. The conch file is +** stored in the same directory as the database file and the file name +** is patterned after the database file name as ".-conch". +** If the conch file does not exist, or it's contents do not match the +** host ID and/or proxy path, then the lock is escalated to an exclusive +** lock and the conch file contents is updated with the host ID and proxy +** path and the lock is downgraded to a shared lock again. If the conch +** is held by another process (with a shared lock), the exclusive lock +** will fail and SQLITE_BUSY is returned. +** +** The proxy file - a single-byte file used for all advisory file locks +** normally taken on the database file. This allows for safe sharing +** of the database file for multiple readers and writers on the same +** host (the conch ensures that they all use the same local lock file). +** +** Requesting the lock proxy does not immediately take the conch, it is +** only taken when the first request to lock database file is made. +** This matches the semantics of the traditional locking behavior, where +** opening a connection to a database file does not take a lock on it. +** The shared lock and an open file descriptor are maintained until +** the connection to the database is closed. +** +** The proxy file and the lock file are never deleted so they only need +** to be created the first time they are used. +** +** Configuration options +** --------------------- +** +** SQLITE_PREFER_PROXY_LOCKING +** +** Database files accessed on non-local file systems are +** automatically configured for proxy locking, lock files are +** named automatically using the same logic as +** PRAGMA lock_proxy_file=":auto:" +** +** SQLITE_PROXY_DEBUG +** +** Enables the logging of error messages during host id file +** retrieval and creation +** +** LOCKPROXYDIR +** +** Overrides the default directory used for lock proxy files that +** are named automatically via the ":auto:" setting +** +** SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +** +** Permissions to use when creating a directory for storing the +** lock proxy files, only used when LOCKPROXYDIR is not set. +** +** +** As mentioned above, when compiled with SQLITE_PREFER_PROXY_LOCKING, +** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will +** force proxy locking to be used for every database file opened, and 0 +** will force automatic proxy locking to be disabled for all database +** files (explicity calling the SQLITE_SET_LOCKPROXYFILE pragma or +** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). +*/ + +/* +** Proxy locking is only available on MacOSX +*/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + +/* +** The proxyLockingContext has the path and file structures for the remote +** and local proxy files in it +*/ +typedef struct proxyLockingContext proxyLockingContext; +struct proxyLockingContext { + unixFile *conchFile; /* Open conch file */ + char *conchFilePath; /* Name of the conch file */ + unixFile *lockProxy; /* Open proxy lock file */ + char *lockProxyPath; /* Name of the proxy lock file */ + char *dbPath; /* Name of the open file */ + int conchHeld; /* 1 if the conch is held, -1 if lockless */ + void *oldLockingContext; /* Original lockingcontext to restore on close */ + sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ +}; + +/* +** The proxy lock file path for the database at dbPath is written into lPath, +** which must point to valid, writable memory large enough for a maxLen length +** file path. +*/ +static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ + int len; + int dbLen; + int i; + +#ifdef LOCKPROXYDIR + len = strlcpy(lPath, LOCKPROXYDIR, maxLen); +#else +# ifdef _CS_DARWIN_USER_TEMP_DIR + { + if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ + OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", + lPath, errno, getpid())); + return SQLITE_IOERR_LOCK; + } + len = strlcat(lPath, "sqliteplocks", maxLen); + } +# else + len = strlcpy(lPath, "/tmp/", maxLen); +# endif +#endif + + if( lPath[len-1]!='/' ){ + len = strlcat(lPath, "/", maxLen); + } + + /* transform the db path to a unique cache name */ + dbLen = (int)strlen(dbPath); + for( i=0; i 0) ){ + /* only mkdir if leaf dir != "." or "/" or ".." */ + if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') + || (i-start==2 && buf[start] != '.' && buf[start+1] != '.') ){ + buf[i]='\0'; + if( osMkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ + int err=errno; + if( err!=EEXIST ) { + OSTRACE(("CREATELOCKPATH FAILED creating %s, " + "'%s' proxy lock path=%s pid=%d\n", + buf, strerror(err), lockPath, getpid())); + return err; + } + } + } + start=i+1; + } + buf[i] = lockPath[i]; + } + OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n", lockPath, getpid())); + return 0; +} + +/* +** Create a new VFS file descriptor (stored in memory obtained from +** sqlite3_malloc) and open the file named "path" in the file descriptor. +** +** The caller is responsible not only for closing the file descriptor +** but also for freeing the memory associated with the file descriptor. +*/ +static int proxyCreateUnixFile( + const char *path, /* path for the new unixFile */ + unixFile **ppFile, /* unixFile created and returned by ref */ + int islockfile /* if non zero missing dirs will be created */ +) { + int fd = -1; + unixFile *pNew; + int rc = SQLITE_OK; + int openFlags = O_RDWR | O_CREAT; + sqlite3_vfs dummyVfs; + int terrno = 0; + UnixUnusedFd *pUnused = NULL; + + /* 1. first try to open/create the file + ** 2. if that fails, and this is a lock file (not-conch), try creating + ** the parent directories and then try again. + ** 3. if that fails, try to open the file read-only + ** otherwise return BUSY (if lock file) or CANTOPEN for the conch file + */ + pUnused = findReusableFd(path, openFlags); + if( pUnused ){ + fd = pUnused->fd; + }else{ + pUnused = sqlite3_malloc(sizeof(*pUnused)); + if( !pUnused ){ + return SQLITE_NOMEM; + } + } + if( fd<0 ){ + fd = robust_open(path, openFlags, 0); + terrno = errno; + if( fd<0 && errno==ENOENT && islockfile ){ + if( proxyCreateLockPath(path) == SQLITE_OK ){ + fd = robust_open(path, openFlags, 0); + } + } + } + if( fd<0 ){ + openFlags = O_RDONLY; + fd = robust_open(path, openFlags, 0); + terrno = errno; + } + if( fd<0 ){ + if( islockfile ){ + return SQLITE_BUSY; + } + switch (terrno) { + case EACCES: + return SQLITE_PERM; + case EIO: + return SQLITE_IOERR_LOCK; /* even though it is the conch */ + default: + return SQLITE_CANTOPEN_BKPT; + } + } + + pNew = (unixFile *)sqlite3_malloc(sizeof(*pNew)); + if( pNew==NULL ){ + rc = SQLITE_NOMEM; + goto end_create_proxy; + } + memset(pNew, 0, sizeof(unixFile)); + pNew->openFlags = openFlags; + memset(&dummyVfs, 0, sizeof(dummyVfs)); + dummyVfs.pAppData = (void*)&autolockIoFinder; + dummyVfs.zName = "dummy"; + pUnused->fd = fd; + pUnused->flags = openFlags; + pNew->pUnused = pUnused; + + rc = fillInUnixFile(&dummyVfs, fd, (sqlite3_file*)pNew, path, 0); + if( rc==SQLITE_OK ){ + *ppFile = pNew; + return SQLITE_OK; + } +end_create_proxy: + robust_close(pNew, fd, __LINE__); + sqlite3_free(pNew); + sqlite3_free(pUnused); + return rc; +} + +#ifdef SQLITE_TEST +/* simulate multiple hosts by creating unique hostid file paths */ +SQLITE_API int sqlite3_hostid_num = 0; +#endif + +#define PROXY_HOSTIDLEN 16 /* conch file host id length */ + +/* Not always defined in the headers as it ought to be */ +extern int gethostuuid(uuid_t id, const struct timespec *wait); + +/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN +** bytes of writable memory. +*/ +static int proxyGetHostID(unsigned char *pHostID, int *pError){ + assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); + memset(pHostID, 0, PROXY_HOSTIDLEN); +#if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\ + && __MAC_OS_X_VERSION_MIN_REQUIRED<1050 + { + static const struct timespec timeout = {1, 0}; /* 1 sec timeout */ + if( gethostuuid(pHostID, &timeout) ){ + int err = errno; + if( pError ){ + *pError = err; + } + return SQLITE_IOERR; + } + } +#else + UNUSED_PARAMETER(pError); +#endif +#ifdef SQLITE_TEST + /* simulate multiple hosts by creating unique hostid file paths */ + if( sqlite3_hostid_num != 0){ + pHostID[0] = (char)(pHostID[0] + (char)(sqlite3_hostid_num & 0xFF)); + } +#endif + + return SQLITE_OK; +} + +/* The conch file contains the header, host id and lock file path + */ +#define PROXY_CONCHVERSION 2 /* 1-byte header, 16-byte host id, path */ +#define PROXY_HEADERLEN 1 /* conch file header length */ +#define PROXY_PATHINDEX (PROXY_HEADERLEN+PROXY_HOSTIDLEN) +#define PROXY_MAXCONCHLEN (PROXY_HEADERLEN+PROXY_HOSTIDLEN+MAXPATHLEN) + +/* +** Takes an open conch file, copies the contents to a new path and then moves +** it back. The newly created file's file descriptor is assigned to the +** conch file structure and finally the original conch file descriptor is +** closed. Returns zero if successful. +*/ +static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + char tPath[MAXPATHLEN]; + char buf[PROXY_MAXCONCHLEN]; + char *cPath = pCtx->conchFilePath; + size_t readLen = 0; + size_t pathLen = 0; + char errmsg[64] = ""; + int fd = -1; + int rc = -1; + UNUSED_PARAMETER(myHostID); + + /* create a new path by replace the trailing '-conch' with '-break' */ + pathLen = strlcpy(tPath, cPath, MAXPATHLEN); + if( pathLen>MAXPATHLEN || pathLen<6 || + (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ + sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen); + goto end_breaklock; + } + /* read the conch content */ + readLen = osPread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0); + if( readLenh, __LINE__); + conchFile->h = fd; + conchFile->openFlags = O_RDWR | O_CREAT; + +end_breaklock: + if( rc ){ + if( fd>=0 ){ + osUnlink(tPath); + robust_close(pFile, fd, __LINE__); + } + fprintf(stderr, "failed to break stale lock on %s, %s\n", cPath, errmsg); + } + return rc; +} + +/* Take the requested lock on the conch file and break a stale lock if the +** host id matches. +*/ +static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + int nTries = 0; + struct timespec conchModTime; + + memset(&conchModTime, 0, sizeof(conchModTime)); + do { + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + nTries ++; + if( rc==SQLITE_BUSY ){ + /* If the lock failed (busy): + * 1st try: get the mod time of the conch, wait 0.5s and try again. + * 2nd try: fail if the mod time changed or host id is different, wait + * 10 sec and try again + * 3rd try: break the lock unless the mod time has changed. + */ + struct stat buf; + if( osFstat(conchFile->h, &buf) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_LOCK; + } + + if( nTries==1 ){ + conchModTime = buf.st_mtimespec; + usleep(500000); /* wait 0.5 sec and try the lock again*/ + continue; + } + + assert( nTries>1 ); + if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || + conchModTime.tv_nsec != buf.st_mtimespec.tv_nsec ){ + return SQLITE_BUSY; + } + + if( nTries==2 ){ + char tBuf[PROXY_MAXCONCHLEN]; + int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); + if( len<0 ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_LOCK; + } + if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ + /* don't break the lock if the host id doesn't match */ + if( 0!=memcmp(&tBuf[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN) ){ + return SQLITE_BUSY; + } + }else{ + /* don't break the lock on short read or a version mismatch */ + return SQLITE_BUSY; + } + usleep(10000000); /* wait 10 sec and try the lock again */ + continue; + } + + assert( nTries==3 ); + if( 0==proxyBreakConchLock(pFile, myHostID) ){ + rc = SQLITE_OK; + if( lockType==EXCLUSIVE_LOCK ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); + } + if( !rc ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + } + } + } + } while( rc==SQLITE_BUSY && nTries<3 ); + + return rc; +} + +/* Takes the conch by taking a shared lock and read the contents conch, if +** lockPath is non-NULL, the host ID and lock file path must match. A NULL +** lockPath means that the lockPath in the conch file will be used if the +** host IDs match, or a new lock path will be generated automatically +** and written to the conch file. +*/ +static int proxyTakeConch(unixFile *pFile){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + + if( pCtx->conchHeld!=0 ){ + return SQLITE_OK; + }else{ + unixFile *conchFile = pCtx->conchFile; + uuid_t myHostID; + int pError = 0; + char readBuf[PROXY_MAXCONCHLEN]; + char lockPath[MAXPATHLEN]; + char *tempLockPath = NULL; + int rc = SQLITE_OK; + int createConch = 0; + int hostIdMatch = 0; + int readLen = 0; + int tryOldLockPath = 0; + int forceNewLockPath = 0; + + OSTRACE(("TAKECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), getpid())); + + rc = proxyGetHostID(myHostID, &pError); + if( (rc&0xff)==SQLITE_IOERR ){ + pFile->lastErrno = pError; + goto end_takeconch; + } + rc = proxyConchLock(pFile, myHostID, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + goto end_takeconch; + } + /* read the existing conch file */ + readLen = seekAndRead((unixFile*)conchFile, 0, readBuf, PROXY_MAXCONCHLEN); + if( readLen<0 ){ + /* I/O error: lastErrno set by seekAndRead */ + pFile->lastErrno = conchFile->lastErrno; + rc = SQLITE_IOERR_READ; + goto end_takeconch; + }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || + readBuf[0]!=(char)PROXY_CONCHVERSION ){ + /* a short read or version format mismatch means we need to create a new + ** conch file. + */ + createConch = 1; + } + /* if the host id matches and the lock path already exists in the conch + ** we'll try to use the path there, if we can't open that path, we'll + ** retry with a new auto-generated path + */ + do { /* in case we need to try again for an :auto: named lock file */ + + if( !createConch && !forceNewLockPath ){ + hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, + PROXY_HOSTIDLEN); + /* if the conch has data compare the contents */ + if( !pCtx->lockProxyPath ){ + /* for auto-named local lock file, just check the host ID and we'll + ** use the local lock file path that's already in there + */ + if( hostIdMatch ){ + size_t pathLen = (readLen - PROXY_PATHINDEX); + + if( pathLen>=MAXPATHLEN ){ + pathLen=MAXPATHLEN-1; + } + memcpy(lockPath, &readBuf[PROXY_PATHINDEX], pathLen); + lockPath[pathLen] = 0; + tempLockPath = lockPath; + tryOldLockPath = 1; + /* create a copy of the lock path if the conch is taken */ + goto end_takeconch; + } + }else if( hostIdMatch + && !strncmp(pCtx->lockProxyPath, &readBuf[PROXY_PATHINDEX], + readLen-PROXY_PATHINDEX) + ){ + /* conch host and lock path match */ + goto end_takeconch; + } + } + + /* if the conch isn't writable and doesn't match, we can't take it */ + if( (conchFile->openFlags&O_RDWR) == 0 ){ + rc = SQLITE_BUSY; + goto end_takeconch; + } + + /* either the conch didn't match or we need to create a new one */ + if( !pCtx->lockProxyPath ){ + proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); + tempLockPath = lockPath; + /* create a copy of the lock path _only_ if the conch is taken */ + } + + /* update conch with host and path (this will fail if other process + ** has a shared lock already), if the host id matches, use the big + ** stick. + */ + futimes(conchFile->h, NULL); + if( hostIdMatch && !createConch ){ + if( conchFile->pInode && conchFile->pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + } else { + rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); + } + }else{ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, EXCLUSIVE_LOCK); + } + if( rc==SQLITE_OK ){ + char writeBuffer[PROXY_MAXCONCHLEN]; + int writeSize = 0; + + writeBuffer[0] = (char)PROXY_CONCHVERSION; + memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); + if( pCtx->lockProxyPath!=NULL ){ + strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, MAXPATHLEN); + }else{ + strlcpy(&writeBuffer[PROXY_PATHINDEX], tempLockPath, MAXPATHLEN); + } + writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); + robust_ftruncate(conchFile->h, writeSize); + rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); + fsync(conchFile->h); + /* If we created a new conch file (not just updated the contents of a + ** valid conch file), try to match the permissions of the database + */ + if( rc==SQLITE_OK && createConch ){ + struct stat buf; + int err = osFstat(pFile->h, &buf); + if( err==0 ){ + mode_t cmode = buf.st_mode&(S_IRUSR|S_IWUSR | S_IRGRP|S_IWGRP | + S_IROTH|S_IWOTH); + /* try to match the database file R/W permissions, ignore failure */ +#ifndef SQLITE_PROXY_DEBUG + osFchmod(conchFile->h, cmode); +#else + do{ + rc = osFchmod(conchFile->h, cmode); + }while( rc==(-1) && errno==EINTR ); + if( rc!=0 ){ + int code = errno; + fprintf(stderr, "fchmod %o FAILED with %d %s\n", + cmode, code, strerror(code)); + } else { + fprintf(stderr, "fchmod %o SUCCEDED\n",cmode); + } + }else{ + int code = errno; + fprintf(stderr, "STAT FAILED[%d] with %d %s\n", + err, code, strerror(code)); +#endif + } + } + } + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); + + end_takeconch: + OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); + if( rc==SQLITE_OK && pFile->openFlags ){ + int fd; + if( pFile->h>=0 ){ + robust_close(pFile, pFile->h, __LINE__); + } + pFile->h = -1; + fd = robust_open(pCtx->dbPath, pFile->openFlags, 0); + OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); + if( fd>=0 ){ + pFile->h = fd; + }else{ + rc=SQLITE_CANTOPEN_BKPT; /* SQLITE_BUSY? proxyTakeConch called + during locking */ + } + } + if( rc==SQLITE_OK && !pCtx->lockProxy ){ + char *path = tempLockPath ? tempLockPath : pCtx->lockProxyPath; + rc = proxyCreateUnixFile(path, &pCtx->lockProxy, 1); + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && tryOldLockPath ){ + /* we couldn't create the proxy lock file with the old lock file path + ** so try again via auto-naming + */ + forceNewLockPath = 1; + tryOldLockPath = 0; + continue; /* go back to the do {} while start point, try again */ + } + } + if( rc==SQLITE_OK ){ + /* Need to make a copy of path if we extracted the value + ** from the conch file or the path was allocated on the stack + */ + if( tempLockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); + if( !pCtx->lockProxyPath ){ + rc = SQLITE_NOMEM; + } + } + } + if( rc==SQLITE_OK ){ + pCtx->conchHeld = 1; + + if( pCtx->lockProxy->pMethod == &afpIoMethods ){ + afpLockingContext *afpCtx; + afpCtx = (afpLockingContext *)pCtx->lockProxy->lockingContext; + afpCtx->dbPath = pCtx->lockProxyPath; + } + } else { + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + OSTRACE(("TAKECONCH %d %s\n", conchFile->h, + rc==SQLITE_OK?"ok":"failed")); + return rc; + } while (1); /* in case we need to retry the :auto: lock file - + ** we should never get here except via the 'continue' call. */ + } +} + +/* +** If pFile holds a lock on a conch file, then release that lock. +*/ +static int proxyReleaseConch(unixFile *pFile){ + int rc = SQLITE_OK; /* Subroutine return code */ + proxyLockingContext *pCtx; /* The locking context for the proxy lock */ + unixFile *conchFile; /* Name of the conch file */ + + pCtx = (proxyLockingContext *)pFile->lockingContext; + conchFile = pCtx->conchFile; + OSTRACE(("RELEASECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + getpid())); + if( pCtx->conchHeld>0 ){ + rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + pCtx->conchHeld = 0; + OSTRACE(("RELEASECONCH %d %s\n", conchFile->h, + (rc==SQLITE_OK ? "ok" : "failed"))); + return rc; +} + +/* +** Given the name of a database file, compute the name of its conch file. +** Store the conch filename in memory obtained from sqlite3_malloc(). +** Make *pConchPath point to the new name. Return SQLITE_OK on success +** or SQLITE_NOMEM if unable to obtain memory. +** +** The caller is responsible for ensuring that the allocated memory +** space is eventually freed. +** +** *pConchPath is set to NULL if a memory allocation error occurs. +*/ +static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ + int i; /* Loop counter */ + int len = (int)strlen(dbPath); /* Length of database filename - dbPath */ + char *conchPath; /* buffer in which to construct conch name */ + + /* Allocate space for the conch filename and initialize the name to + ** the name of the original database file. */ + *pConchPath = conchPath = (char *)sqlite3_malloc(len + 8); + if( conchPath==0 ){ + return SQLITE_NOMEM; + } + memcpy(conchPath, dbPath, len+1); + + /* now insert a "." before the last / character */ + for( i=(len-1); i>=0; i-- ){ + if( conchPath[i]=='/' ){ + i++; + break; + } + } + conchPath[i]='.'; + while ( ilockingContext; + char *oldPath = pCtx->lockProxyPath; + int rc = SQLITE_OK; + + if( pFile->eFileLock!=NO_LOCK ){ + return SQLITE_BUSY; + } + + /* nothing to do if the path is NULL, :auto: or matches the existing path */ + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") || + (oldPath && !strncmp(oldPath, path, MAXPATHLEN)) ){ + return SQLITE_OK; + }else{ + unixFile *lockProxy = pCtx->lockProxy; + pCtx->lockProxy=NULL; + pCtx->conchHeld = 0; + if( lockProxy!=NULL ){ + rc=lockProxy->pMethod->xClose((sqlite3_file *)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + } + sqlite3_free(oldPath); + pCtx->lockProxyPath = sqlite3DbStrDup(0, path); + } + + return rc; +} + +/* +** pFile is a file that has been opened by a prior xOpen call. dbPath +** is a string buffer at least MAXPATHLEN+1 characters in size. +** +** This routine find the filename associated with pFile and writes it +** int dbPath. +*/ +static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ +#if defined(__APPLE__) + if( pFile->pMethod == &afpIoMethods ){ + /* afp style keeps a reference to the db path in the filePath field + ** of the struct */ + assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, MAXPATHLEN); + } else +#endif + if( pFile->pMethod == &dotlockIoMethods ){ + /* dot lock style uses the locking context to store the dot lock + ** file path */ + int len = strlen((char *)pFile->lockingContext) - strlen(DOTLOCK_SUFFIX); + memcpy(dbPath, (char *)pFile->lockingContext, len + 1); + }else{ + /* all other styles use the locking context to store the db file path */ + assert( strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strlcpy(dbPath, (char *)pFile->lockingContext, MAXPATHLEN); + } + return SQLITE_OK; +} + +/* +** Takes an already filled in unix file and alters it so all file locking +** will be performed on the local proxy lock file. The following fields +** are preserved in the locking context so that they can be restored and +** the unix structure properly cleaned up at close time: +** ->lockingContext +** ->pMethod +*/ +static int proxyTransformUnixFile(unixFile *pFile, const char *path) { + proxyLockingContext *pCtx; + char dbPath[MAXPATHLEN+1]; /* Name of the database file */ + char *lockPath=NULL; + int rc = SQLITE_OK; + + if( pFile->eFileLock!=NO_LOCK ){ + return SQLITE_BUSY; + } + proxyGetDbPathForUnixFile(pFile, dbPath); + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") ){ + lockPath=NULL; + }else{ + lockPath=(char *)path; + } + + OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, + (lockPath ? lockPath : ":auto:"), getpid())); + + pCtx = sqlite3_malloc( sizeof(*pCtx) ); + if( pCtx==0 ){ + return SQLITE_NOMEM; + } + memset(pCtx, 0, sizeof(*pCtx)); + + rc = proxyCreateConchPathname(dbPath, &pCtx->conchFilePath); + if( rc==SQLITE_OK ){ + rc = proxyCreateUnixFile(pCtx->conchFilePath, &pCtx->conchFile, 0); + if( rc==SQLITE_CANTOPEN && ((pFile->openFlags&O_RDWR) == 0) ){ + /* if (a) the open flags are not O_RDWR, (b) the conch isn't there, and + ** (c) the file system is read-only, then enable no-locking access. + ** Ugh, since O_RDONLY==0x0000 we test for !O_RDWR since unixOpen asserts + ** that openFlags will have only one of O_RDONLY or O_RDWR. + */ + struct statfs fsInfo; + struct stat conchInfo; + int goLockless = 0; + + if( osStat(pCtx->conchFilePath, &conchInfo) == -1 ) { + int err = errno; + if( (err==ENOENT) && (statfs(dbPath, &fsInfo) != -1) ){ + goLockless = (fsInfo.f_flags&MNT_RDONLY) == MNT_RDONLY; + } + } + if( goLockless ){ + pCtx->conchHeld = -1; /* read only FS/ lockless */ + rc = SQLITE_OK; + } + } + } + if( rc==SQLITE_OK && lockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); + } + + if( rc==SQLITE_OK ){ + pCtx->dbPath = sqlite3DbStrDup(0, dbPath); + if( pCtx->dbPath==NULL ){ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + /* all memory is allocated, proxys are created and assigned, + ** switch the locking context and pMethod then return. + */ + pCtx->oldLockingContext = pFile->lockingContext; + pFile->lockingContext = pCtx; + pCtx->pOldMethod = pFile->pMethod; + pFile->pMethod = &proxyIoMethods; + }else{ + if( pCtx->conchFile ){ + pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); + sqlite3_free(pCtx->conchFile); + } + sqlite3DbFree(0, pCtx->lockProxyPath); + sqlite3_free(pCtx->conchFilePath); + sqlite3_free(pCtx); + } + OSTRACE(("TRANSPROXY %d %s\n", pFile->h, + (rc==SQLITE_OK ? "ok" : "failed"))); + return rc; +} + + +/* +** This routine handles sqlite3_file_control() calls that are specific +** to proxy locking. +*/ +static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_GET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + if( pFile->pMethod == &proxyIoMethods ){ + proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; + proxyTakeConch(pFile); + if( pCtx->lockProxyPath ){ + *(const char **)pArg = pCtx->lockProxyPath; + }else{ + *(const char **)pArg = ":auto: (not held)"; + } + } else { + *(const char **)pArg = NULL; + } + return SQLITE_OK; + } + case SQLITE_SET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + int rc = SQLITE_OK; + int isProxyStyle = (pFile->pMethod == &proxyIoMethods); + if( pArg==NULL || (const char *)pArg==0 ){ + if( isProxyStyle ){ + /* turn off proxy locking - not supported */ + rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; + }else{ + /* turn off proxy locking - already off - NOOP */ + rc = SQLITE_OK; + } + }else{ + const char *proxyPath = (const char *)pArg; + if( isProxyStyle ){ + proxyLockingContext *pCtx = + (proxyLockingContext*)pFile->lockingContext; + if( !strcmp(pArg, ":auto:") + || (pCtx->lockProxyPath && + !strncmp(pCtx->lockProxyPath, proxyPath, MAXPATHLEN)) + ){ + rc = SQLITE_OK; + }else{ + rc = switchLockProxyPath(pFile, proxyPath); + } + }else{ + /* turn on proxy file locking */ + rc = proxyTransformUnixFile(pFile, proxyPath); + } + } + return rc; + } + default: { + assert( 0 ); /* The call assures that only valid opcodes are sent */ + } + } + /*NOTREACHED*/ + return SQLITE_ERROR; +} + +/* +** Within this division (the proxying locking implementation) the procedures +** above this point are all utilities. The lock-related methods of the +** proxy-locking sqlite3_io_method object follow. +*/ + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int proxyCheckReservedLock(sqlite3_file *id, int *pResOut) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + return proxy->pMethod->xCheckReservedLock((sqlite3_file*)proxy, pResOut); + }else{ /* conchHeld < 0 is lockless */ + pResOut=0; + } + } + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int proxyLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xLock((sqlite3_file*)proxy, eFileLock); + pFile->eFileLock = proxy->eFileLock; + }else{ + /* conchHeld < 0 is lockless */ + } + } + return rc; +} + + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int proxyUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xUnlock((sqlite3_file*)proxy, eFileLock); + pFile->eFileLock = proxy->eFileLock; + }else{ + /* conchHeld < 0 is lockless */ + } + } + return rc; +} + +/* +** Close a file that uses proxy locks. +*/ +static int proxyClose(sqlite3_file *id) { + if( id ){ + unixFile *pFile = (unixFile*)id; + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *lockProxy = pCtx->lockProxy; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + + if( lockProxy ){ + rc = lockProxy->pMethod->xUnlock((sqlite3_file*)lockProxy, NO_LOCK); + if( rc ) return rc; + rc = lockProxy->pMethod->xClose((sqlite3_file*)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + pCtx->lockProxy = 0; + } + if( conchFile ){ + if( pCtx->conchHeld ){ + rc = proxyReleaseConch(pFile); + if( rc ) return rc; + } + rc = conchFile->pMethod->xClose((sqlite3_file*)conchFile); + if( rc ) return rc; + sqlite3_free(conchFile); + } + sqlite3DbFree(0, pCtx->lockProxyPath); + sqlite3_free(pCtx->conchFilePath); + sqlite3DbFree(0, pCtx->dbPath); + /* restore the original locking context and pMethod then close it */ + pFile->lockingContext = pCtx->oldLockingContext; + pFile->pMethod = pCtx->pOldMethod; + sqlite3_free(pCtx); + return pFile->pMethod->xClose(id); + } + return SQLITE_OK; +} + + + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The proxy locking style is intended for use with AFP filesystems. +** And since AFP is only supported on MacOSX, the proxy locking is also +** restricted to MacOSX. +** +** +******************* End of the proxy lock implementation ********************** +******************************************************************************/ + +/* +** Initialize the operating system interface. +** +** This routine registers all VFS implementations for unix-like operating +** systems. This routine, and the sqlite3_os_end() routine that follows, +** should be the only routines in this file that are visible from other +** files. +** +** This routine is called once during SQLite initialization and by a +** single thread. The memory allocation and mutex subsystems have not +** necessarily been initialized when this routine is called, and so they +** should not be used. +*/ +SQLITE_API int sqlite3_os_init(void){ + /* + ** The following macro defines an initializer for an sqlite3_vfs object. + ** The name of the VFS is NAME. The pAppData is a pointer to a pointer + ** to the "finder" function. (pAppData is a pointer to a pointer because + ** silly C90 rules prohibit a void* from being cast to a function pointer + ** and so we have to go through the intermediate pointer to avoid problems + ** when compiling with -pedantic-errors on GCC.) + ** + ** The FINDER parameter to this macro is the name of the pointer to the + ** finder-function. The finder-function returns a pointer to the + ** sqlite_io_methods object that implements the desired locking + ** behaviors. See the division above that contains the IOMETHODS + ** macro for addition information on finder-functions. + ** + ** Most finders simply return a pointer to a fixed sqlite3_io_methods + ** object. But the "autolockIoFinder" available on MacOSX does a little + ** more than that; it looks at the filesystem type that hosts the + ** database file and tries to choose an locking method appropriate for + ** that filesystem time. + */ + #define UNIXVFS(VFSNAME, FINDER) { \ + 3, /* iVersion */ \ + sizeof(unixFile), /* szOsFile */ \ + MAX_PATHNAME, /* mxPathname */ \ + 0, /* pNext */ \ + VFSNAME, /* zName */ \ + (void*)&FINDER, /* pAppData */ \ + unixOpen, /* xOpen */ \ + unixDelete, /* xDelete */ \ + unixAccess, /* xAccess */ \ + unixFullPathname, /* xFullPathname */ \ + unixDlOpen, /* xDlOpen */ \ + unixDlError, /* xDlError */ \ + unixDlSym, /* xDlSym */ \ + unixDlClose, /* xDlClose */ \ + unixRandomness, /* xRandomness */ \ + unixSleep, /* xSleep */ \ + unixCurrentTime, /* xCurrentTime */ \ + unixGetLastError, /* xGetLastError */ \ + unixCurrentTimeInt64, /* xCurrentTimeInt64 */ \ + unixSetSystemCall, /* xSetSystemCall */ \ + unixGetSystemCall, /* xGetSystemCall */ \ + unixNextSystemCall, /* xNextSystemCall */ \ + } + + /* + ** All default VFSes for unix are contained in the following array. + ** + ** Note that the sqlite3_vfs.pNext field of the VFS object is modified + ** by the SQLite core when the VFS is registered. So the following + ** array cannot be const. + */ + static sqlite3_vfs aVfs[] = { +#if SQLITE_ENABLE_LOCKING_STYLE && (OS_VXWORKS || defined(__APPLE__)) + UNIXVFS("unix", autolockIoFinder ), +#else + UNIXVFS("unix", posixIoFinder ), +#endif + UNIXVFS("unix-none", nolockIoFinder ), + UNIXVFS("unix-dotfile", dotlockIoFinder ), + UNIXVFS("unix-excl", posixIoFinder ), +#if OS_VXWORKS + UNIXVFS("unix-namedsem", semIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + UNIXVFS("unix-posix", posixIoFinder ), +#if !OS_VXWORKS + UNIXVFS("unix-flock", flockIoFinder ), +#endif +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + UNIXVFS("unix-afp", afpIoFinder ), + UNIXVFS("unix-nfs", nfsIoFinder ), + UNIXVFS("unix-proxy", proxyIoFinder ), +#endif + }; + unsigned int i; /* Loop counter */ + + /* Double-check that the aSyscall[] array has been constructed + ** correctly. See ticket [bb3a86e890c8e96ab] */ + assert( ArraySize(aSyscall)==25 ); + + /* Register all VFSes defined in the aVfs[] array */ + for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ + sqlite3_vfs_register(&aVfs[i], i==0); + } + return SQLITE_OK; +} + +/* +** Shutdown the operating system interface. +** +** Some operating systems might need to do some cleanup in this routine, +** to release dynamically allocated objects. But not on unix. +** This routine is a no-op for unix. +*/ +SQLITE_API int sqlite3_os_end(void){ + return SQLITE_OK; +} + +#endif /* SQLITE_OS_UNIX */ + +/************** End of os_unix.c *********************************************/ +/************** Begin file os_win.c ******************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Windows. +*/ +#if SQLITE_OS_WIN /* This file is used for Windows only */ + +/* +** Include code that is common to all os_*.c files +*/ +/************** Include os_common.h in the middle of os_win.c ****************/ +/************** Begin file os_common.h ***************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains macros and a little bit of code that is common to +** all of the platform-specific files (os_*.c) and is #included into those +** files. +** +** This file should be #included by the os_*.c files only. It is not a +** general purpose header file. +*/ +#ifndef _OS_COMMON_H_ +#define _OS_COMMON_H_ + +/* +** At least two bugs have slipped in because we changed the MEMORY_DEBUG +** macro to SQLITE_DEBUG and some older makefiles have not yet made the +** switch. The following code should catch this problem at compile-time. +*/ +#ifdef MEMORY_DEBUG +# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." +#endif + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X +#else +# define OSTRACE(X) +#endif + +/* +** Macros for performance tracing. Normally turned off. Only works +** on i486 hardware. +*/ +#ifdef SQLITE_PERFORMANCE_TRACE + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +/************** Include hwtime.h in the middle of os_common.h ****************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 class CPUs. +*/ +#ifndef _HWTIME_H_ +#define _HWTIME_H_ + +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; + } + +#elif (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + #error Need implementation of sqlite3Hwtime() for your platform. + + /* + ** To compile without implementing sqlite3Hwtime() for your platform, + ** you can remove the above #error and use the following + ** stub function. You will lose timing support for many + ** of the debugging and testing utilities, but it should at + ** least compile and run. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(_HWTIME_H_) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in os_common.h ******************/ + +static sqlite_uint64 g_start; +static sqlite_uint64 g_elapsed; +#define TIMER_START g_start=sqlite3Hwtime() +#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start +#define TIMER_ELAPSED g_elapsed +#else +#define TIMER_START +#define TIMER_END +#define TIMER_ELAPSED ((sqlite_uint64)0) +#endif + +/* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ +SQLITE_API int sqlite3_diskfull_pending = 0; +SQLITE_API int sqlite3_diskfull = 0; +#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) +#define SimulateIOError(CODE) \ + if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ + || sqlite3_io_error_pending-- == 1 ) \ + { local_ioerr(); CODE; } +static void local_ioerr(){ + IOTRACE(("IOERR\n")); + sqlite3_io_error_hit++; + if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; +} +#define SimulateDiskfullError(CODE) \ + if( sqlite3_diskfull_pending ){ \ + if( sqlite3_diskfull_pending == 1 ){ \ + local_ioerr(); \ + sqlite3_diskfull = 1; \ + sqlite3_io_error_hit = 1; \ + CODE; \ + }else{ \ + sqlite3_diskfull_pending--; \ + } \ + } +#else +#define SimulateIOErrorBenign(X) +#define SimulateIOError(A) +#define SimulateDiskfullError(A) +#endif + +/* +** When testing, keep a count of the number of open files. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_open_file_count = 0; +#define OpenCounter(X) sqlite3_open_file_count+=(X) +#else +#define OpenCounter(X) +#endif + +#endif /* !defined(_OS_COMMON_H_) */ + +/************** End of os_common.h *******************************************/ +/************** Continuing where we left off in os_win.c *********************/ + +/* +** Include the header file for the Windows VFS. +*/ + +/* +** Compiling and using WAL mode requires several APIs that are only +** available in Windows platforms based on the NT kernel. +*/ +#if !SQLITE_OS_WINNT && !defined(SQLITE_OMIT_WAL) +# error "WAL mode requires support from the Windows NT kernel, compile\ + with SQLITE_OMIT_WAL." +#endif + +/* +** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(SQLITE_WIN32_NO_ANSI) +# define SQLITE_WIN32_HAS_ANSI +#endif + +/* +** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT) && \ + !defined(SQLITE_WIN32_NO_WIDE) +# define SQLITE_WIN32_HAS_WIDE +#endif + +/* +** Make sure at least one set of Win32 APIs is available. +*/ +#if !defined(SQLITE_WIN32_HAS_ANSI) && !defined(SQLITE_WIN32_HAS_WIDE) +# error "At least one of SQLITE_WIN32_HAS_ANSI and SQLITE_WIN32_HAS_WIDE\ + must be defined." +#endif + +/* +** Define the required Windows SDK version constants if they are not +** already available. +*/ +#ifndef NTDDI_WIN8 +# define NTDDI_WIN8 0x06020000 +#endif + +#ifndef NTDDI_WINBLUE +# define NTDDI_WINBLUE 0x06030000 +#endif + +/* +** Check to see if the GetVersionEx[AW] functions are deprecated on the +** target system. GetVersionEx was first deprecated in Win8.1. +*/ +#ifndef SQLITE_WIN32_GETVERSIONEX +# if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINBLUE +# define SQLITE_WIN32_GETVERSIONEX 0 /* GetVersionEx() is deprecated */ +# else +# define SQLITE_WIN32_GETVERSIONEX 1 /* GetVersionEx() is current */ +# endif +#endif + +/* +** This constant should already be defined (in the "WinDef.h" SDK file). +*/ +#ifndef MAX_PATH +# define MAX_PATH (260) +#endif + +/* +** Maximum pathname length (in chars) for Win32. This should normally be +** MAX_PATH. +*/ +#ifndef SQLITE_WIN32_MAX_PATH_CHARS +# define SQLITE_WIN32_MAX_PATH_CHARS (MAX_PATH) +#endif + +/* +** This constant should already be defined (in the "WinNT.h" SDK file). +*/ +#ifndef UNICODE_STRING_MAX_CHARS +# define UNICODE_STRING_MAX_CHARS (32767) +#endif + +/* +** Maximum pathname length (in chars) for WinNT. This should normally be +** UNICODE_STRING_MAX_CHARS. +*/ +#ifndef SQLITE_WINNT_MAX_PATH_CHARS +# define SQLITE_WINNT_MAX_PATH_CHARS (UNICODE_STRING_MAX_CHARS) +#endif + +/* +** Maximum pathname length (in bytes) for Win32. The MAX_PATH macro is in +** characters, so we allocate 4 bytes per character assuming worst-case of +** 4-bytes-per-character for UTF8. +*/ +#ifndef SQLITE_WIN32_MAX_PATH_BYTES +# define SQLITE_WIN32_MAX_PATH_BYTES (SQLITE_WIN32_MAX_PATH_CHARS*4) +#endif + +/* +** Maximum pathname length (in bytes) for WinNT. This should normally be +** UNICODE_STRING_MAX_CHARS * sizeof(WCHAR). +*/ +#ifndef SQLITE_WINNT_MAX_PATH_BYTES +# define SQLITE_WINNT_MAX_PATH_BYTES \ + (sizeof(WCHAR) * SQLITE_WINNT_MAX_PATH_CHARS) +#endif + +/* +** Maximum error message length (in chars) for WinRT. +*/ +#ifndef SQLITE_WIN32_MAX_ERRMSG_CHARS +# define SQLITE_WIN32_MAX_ERRMSG_CHARS (1024) +#endif + +/* +** Returns non-zero if the character should be treated as a directory +** separator. +*/ +#ifndef winIsDirSep +# define winIsDirSep(a) (((a) == '/') || ((a) == '\\')) +#endif + +/* +** This macro is used when a local variable is set to a value that is +** [sometimes] not used by the code (e.g. via conditional compilation). +*/ +#ifndef UNUSED_VARIABLE_VALUE +# define UNUSED_VARIABLE_VALUE(x) (void)(x) +#endif + +/* +** Returns the character that should be used as the directory separator. +*/ +#ifndef winGetDirSep +# define winGetDirSep() '\\' +#endif + +/* +** Do we need to manually define the Win32 file mapping APIs for use with WAL +** mode (e.g. these APIs are available in the Windows CE SDK; however, they +** are not present in the header file)? +*/ +#if SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) +/* +** Two of the file mapping APIs are different under WinRT. Figure out which +** set we need. +*/ +#if SQLITE_OS_WINRT +WINBASEAPI HANDLE WINAPI CreateFileMappingFromApp(HANDLE, \ + LPSECURITY_ATTRIBUTES, ULONG, ULONG64, LPCWSTR); + +WINBASEAPI LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); +#else +#if defined(SQLITE_WIN32_HAS_ANSI) +WINBASEAPI HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCSTR); +#endif /* defined(SQLITE_WIN32_HAS_ANSI) */ + +#if defined(SQLITE_WIN32_HAS_WIDE) +WINBASEAPI HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCWSTR); +#endif /* defined(SQLITE_WIN32_HAS_WIDE) */ + +WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +#endif /* SQLITE_OS_WINRT */ + +/* +** This file mapping API is common to both Win32 and WinRT. +*/ +WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); +#endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ + +/* +** Some Microsoft compilers lack this definition. +*/ +#ifndef INVALID_FILE_ATTRIBUTES +# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) +#endif + +#ifndef FILE_FLAG_MASK +# define FILE_FLAG_MASK (0xFF3C0000) +#endif + +#ifndef FILE_ATTRIBUTE_MASK +# define FILE_ATTRIBUTE_MASK (0x0003FFF7) +#endif + +#ifndef SQLITE_OMIT_WAL +/* Forward references to structures used for WAL */ +typedef struct winShm winShm; /* A connection to shared-memory */ +typedef struct winShmNode winShmNode; /* A region of shared-memory */ +#endif + +/* +** WinCE lacks native support for file locking so we have to fake it +** with some code of our own. +*/ +#if SQLITE_OS_WINCE +typedef struct winceLock { + int nReaders; /* Number of reader locks obtained */ + BOOL bPending; /* Indicates a pending lock has been obtained */ + BOOL bReserved; /* Indicates a reserved lock has been obtained */ + BOOL bExclusive; /* Indicates an exclusive lock has been obtained */ +} winceLock; +#endif + +/* +** The winFile structure is a subclass of sqlite3_file* specific to the win32 +** portability layer. +*/ +typedef struct winFile winFile; +struct winFile { + const sqlite3_io_methods *pMethod; /*** Must be first ***/ + sqlite3_vfs *pVfs; /* The VFS used to open this file */ + HANDLE h; /* Handle for accessing the file */ + u8 locktype; /* Type of lock currently held on this file */ + short sharedLockByte; /* Randomly chosen byte used as a shared lock */ + u8 ctrlFlags; /* Flags. See WINFILE_* below */ + DWORD lastErrno; /* The Windows errno from the last I/O error */ +#ifndef SQLITE_OMIT_WAL + winShm *pShm; /* Instance of shared memory on this file */ +#endif + const char *zPath; /* Full pathname of this file */ + int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ +#if SQLITE_OS_WINCE + LPWSTR zDeleteOnClose; /* Name of file to delete when closing */ + HANDLE hMutex; /* Mutex used to control access to shared lock */ + HANDLE hShared; /* Shared memory segment used for locking */ + winceLock local; /* Locks obtained by this instance of winFile */ + winceLock *shared; /* Global shared lock memory for the file */ +#endif +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ +#endif +}; + +/* +** Allowed values for winFile.ctrlFlags +*/ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ +#define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ +#define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ + +/* + * The size of the buffer used by sqlite3_win32_write_debug(). + */ +#ifndef SQLITE_WIN32_DBG_BUF_SIZE +# define SQLITE_WIN32_DBG_BUF_SIZE ((int)(4096-sizeof(DWORD))) +#endif + +/* + * The value used with sqlite3_win32_set_directory() to specify that + * the data directory should be changed. + */ +#ifndef SQLITE_WIN32_DATA_DIRECTORY_TYPE +# define SQLITE_WIN32_DATA_DIRECTORY_TYPE (1) +#endif + +/* + * The value used with sqlite3_win32_set_directory() to specify that + * the temporary directory should be changed. + */ +#ifndef SQLITE_WIN32_TEMP_DIRECTORY_TYPE +# define SQLITE_WIN32_TEMP_DIRECTORY_TYPE (2) +#endif + +/* + * If compiled with SQLITE_WIN32_MALLOC on Windows, we will use the + * various Win32 API heap functions instead of our own. + */ +#ifdef SQLITE_WIN32_MALLOC + +/* + * If this is non-zero, an isolated heap will be created by the native Win32 + * allocator subsystem; otherwise, the default process heap will be used. This + * setting has no effect when compiling for WinRT. By default, this is enabled + * and an isolated heap will be created to store all allocated data. + * + ****************************************************************************** + * WARNING: It is important to note that when this setting is non-zero and the + * winMemShutdown function is called (e.g. by the sqlite3_shutdown + * function), all data that was allocated using the isolated heap will + * be freed immediately and any attempt to access any of that freed + * data will almost certainly result in an immediate access violation. + ****************************************************************************** + */ +#ifndef SQLITE_WIN32_HEAP_CREATE +# define SQLITE_WIN32_HEAP_CREATE (TRUE) +#endif + +/* + * The initial size of the Win32-specific heap. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_SIZE +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_DEFAULT_CACHE_SIZE) * \ + (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) +#endif + +/* + * The maximum size of the Win32-specific heap. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_SIZE +# define SQLITE_WIN32_HEAP_MAX_SIZE (0) +#endif + +/* + * The extra flags to use in calls to the Win32 heap APIs. This value may be + * zero for the default behavior. + */ +#ifndef SQLITE_WIN32_HEAP_FLAGS +# define SQLITE_WIN32_HEAP_FLAGS (0) +#endif + + +/* +** The winMemData structure stores information required by the Win32-specific +** sqlite3_mem_methods implementation. +*/ +typedef struct winMemData winMemData; +struct winMemData { +#ifndef NDEBUG + u32 magic1; /* Magic number to detect structure corruption. */ +#endif + HANDLE hHeap; /* The handle to our heap. */ + BOOL bOwned; /* Do we own the heap (i.e. destroy it on shutdown)? */ +#ifndef NDEBUG + u32 magic2; /* Magic number to detect structure corruption. */ +#endif +}; + +#ifndef NDEBUG +#define WINMEM_MAGIC1 0x42b2830b +#define WINMEM_MAGIC2 0xbd4d7cf4 +#endif + +static struct winMemData win_mem_data = { +#ifndef NDEBUG + WINMEM_MAGIC1, +#endif + NULL, FALSE +#ifndef NDEBUG + ,WINMEM_MAGIC2 +#endif +}; + +#ifndef NDEBUG +#define winMemAssertMagic1() assert( win_mem_data.magic1==WINMEM_MAGIC1 ) +#define winMemAssertMagic2() assert( win_mem_data.magic2==WINMEM_MAGIC2 ) +#define winMemAssertMagic() winMemAssertMagic1(); winMemAssertMagic2(); +#else +#define winMemAssertMagic() +#endif + +#define winMemGetDataPtr() &win_mem_data +#define winMemGetHeap() win_mem_data.hHeap +#define winMemGetOwned() win_mem_data.bOwned + +static void *winMemMalloc(int nBytes); +static void winMemFree(void *pPrior); +static void *winMemRealloc(void *pPrior, int nBytes); +static int winMemSize(void *p); +static int winMemRoundup(int n); +static int winMemInit(void *pAppData); +static void winMemShutdown(void *pAppData); + +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetWin32(void); +#endif /* SQLITE_WIN32_MALLOC */ + +/* +** The following variable is (normally) set once and never changes +** thereafter. It records whether the operating system is Win9x +** or WinNT. +** +** 0: Operating system unknown. +** 1: Operating system is Win9x. +** 2: Operating system is WinNT. +** +** In order to facilitate testing on a WinNT system, the test fixture +** can manually set this value to 1 to emulate Win98 behavior. +*/ +#ifdef SQLITE_TEST +SQLITE_API LONG volatile sqlite3_os_type = 0; +#else +static LONG volatile sqlite3_os_type = 0; +#endif + +#ifndef SYSCALL +# define SYSCALL sqlite3_syscall_ptr +#endif + +/* +** This function is not available on Windows CE or WinRT. + */ + +#if SQLITE_OS_WINCE || SQLITE_OS_WINRT +# define osAreFileApisANSI() 1 +#endif + +/* +** Many system calls are accessed through pointer-to-functions so that +** they may be overridden at runtime to facilitate fault injection during +** testing and sandboxing. The following array holds the names and pointers +** to all overrideable system calls. +*/ +static struct win_syscall { + const char *zName; /* Name of the system call */ + sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ + sqlite3_syscall_ptr pDefault; /* Default value */ +} aSyscall[] = { +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "AreFileApisANSI", (SYSCALL)AreFileApisANSI, 0 }, +#else + { "AreFileApisANSI", (SYSCALL)0, 0 }, +#endif + +#ifndef osAreFileApisANSI +#define osAreFileApisANSI ((BOOL(WINAPI*)(VOID))aSyscall[0].pCurrent) +#endif + +#if SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "CharLowerW", (SYSCALL)CharLowerW, 0 }, +#else + { "CharLowerW", (SYSCALL)0, 0 }, +#endif + +#define osCharLowerW ((LPWSTR(WINAPI*)(LPWSTR))aSyscall[1].pCurrent) + +#if SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "CharUpperW", (SYSCALL)CharUpperW, 0 }, +#else + { "CharUpperW", (SYSCALL)0, 0 }, +#endif + +#define osCharUpperW ((LPWSTR(WINAPI*)(LPWSTR))aSyscall[2].pCurrent) + + { "CloseHandle", (SYSCALL)CloseHandle, 0 }, + +#define osCloseHandle ((BOOL(WINAPI*)(HANDLE))aSyscall[3].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "CreateFileA", (SYSCALL)CreateFileA, 0 }, +#else + { "CreateFileA", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileA ((HANDLE(WINAPI*)(LPCSTR,DWORD,DWORD, \ + LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[4].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "CreateFileW", (SYSCALL)CreateFileW, 0 }, +#else + { "CreateFileW", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ + LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) + +#if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ + !defined(SQLITE_OMIT_WAL)) + { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, +#else + { "CreateFileMappingA", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingA ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) + +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + !defined(SQLITE_OMIT_WAL)) + { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, +#else + { "CreateFileMappingW", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, +#else + { "CreateMutexW", (SYSCALL)0, 0 }, +#endif + +#define osCreateMutexW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,BOOL, \ + LPCWSTR))aSyscall[8].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "DeleteFileA", (SYSCALL)DeleteFileA, 0 }, +#else + { "DeleteFileA", (SYSCALL)0, 0 }, +#endif + +#define osDeleteFileA ((BOOL(WINAPI*)(LPCSTR))aSyscall[9].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "DeleteFileW", (SYSCALL)DeleteFileW, 0 }, +#else + { "DeleteFileW", (SYSCALL)0, 0 }, +#endif + +#define osDeleteFileW ((BOOL(WINAPI*)(LPCWSTR))aSyscall[10].pCurrent) + +#if SQLITE_OS_WINCE + { "FileTimeToLocalFileTime", (SYSCALL)FileTimeToLocalFileTime, 0 }, +#else + { "FileTimeToLocalFileTime", (SYSCALL)0, 0 }, +#endif + +#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(CONST FILETIME*, \ + LPFILETIME))aSyscall[11].pCurrent) + +#if SQLITE_OS_WINCE + { "FileTimeToSystemTime", (SYSCALL)FileTimeToSystemTime, 0 }, +#else + { "FileTimeToSystemTime", (SYSCALL)0, 0 }, +#endif + +#define osFileTimeToSystemTime ((BOOL(WINAPI*)(CONST FILETIME*, \ + LPSYSTEMTIME))aSyscall[12].pCurrent) + + { "FlushFileBuffers", (SYSCALL)FlushFileBuffers, 0 }, + +#define osFlushFileBuffers ((BOOL(WINAPI*)(HANDLE))aSyscall[13].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "FormatMessageA", (SYSCALL)FormatMessageA, 0 }, +#else + { "FormatMessageA", (SYSCALL)0, 0 }, +#endif + +#define osFormatMessageA ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPSTR, \ + DWORD,va_list*))aSyscall[14].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "FormatMessageW", (SYSCALL)FormatMessageW, 0 }, +#else + { "FormatMessageW", (SYSCALL)0, 0 }, +#endif + +#define osFormatMessageW ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPWSTR, \ + DWORD,va_list*))aSyscall[15].pCurrent) + +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "FreeLibrary", (SYSCALL)FreeLibrary, 0 }, +#else + { "FreeLibrary", (SYSCALL)0, 0 }, +#endif + +#define osFreeLibrary ((BOOL(WINAPI*)(HMODULE))aSyscall[16].pCurrent) + + { "GetCurrentProcessId", (SYSCALL)GetCurrentProcessId, 0 }, + +#define osGetCurrentProcessId ((DWORD(WINAPI*)(VOID))aSyscall[17].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) + { "GetDiskFreeSpaceA", (SYSCALL)GetDiskFreeSpaceA, 0 }, +#else + { "GetDiskFreeSpaceA", (SYSCALL)0, 0 }, +#endif + +#define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ + LPDWORD))aSyscall[18].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, +#else + { "GetDiskFreeSpaceW", (SYSCALL)0, 0 }, +#endif + +#define osGetDiskFreeSpaceW ((BOOL(WINAPI*)(LPCWSTR,LPDWORD,LPDWORD,LPDWORD, \ + LPDWORD))aSyscall[19].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "GetFileAttributesA", (SYSCALL)GetFileAttributesA, 0 }, +#else + { "GetFileAttributesA", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, +#else + { "GetFileAttributesW", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesW ((DWORD(WINAPI*)(LPCWSTR))aSyscall[21].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "GetFileAttributesExW", (SYSCALL)GetFileAttributesExW, 0 }, +#else + { "GetFileAttributesExW", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ + LPVOID))aSyscall[22].pCurrent) + +#if !SQLITE_OS_WINRT + { "GetFileSize", (SYSCALL)GetFileSize, 0 }, +#else + { "GetFileSize", (SYSCALL)0, 0 }, +#endif + +#define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) + { "GetFullPathNameA", (SYSCALL)GetFullPathNameA, 0 }, +#else + { "GetFullPathNameA", (SYSCALL)0, 0 }, +#endif + +#define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ + LPSTR*))aSyscall[24].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, +#else + { "GetFullPathNameW", (SYSCALL)0, 0 }, +#endif + +#define osGetFullPathNameW ((DWORD(WINAPI*)(LPCWSTR,DWORD,LPWSTR, \ + LPWSTR*))aSyscall[25].pCurrent) + + { "GetLastError", (SYSCALL)GetLastError, 0 }, + +#define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) + +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) +#if SQLITE_OS_WINCE + /* The GetProcAddressA() routine is only available on Windows CE. */ + { "GetProcAddressA", (SYSCALL)GetProcAddressA, 0 }, +#else + /* All other Windows platforms expect GetProcAddress() to take + ** an ANSI string regardless of the _UNICODE setting */ + { "GetProcAddressA", (SYSCALL)GetProcAddress, 0 }, +#endif +#else + { "GetProcAddressA", (SYSCALL)0, 0 }, +#endif + +#define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ + LPCSTR))aSyscall[27].pCurrent) + +#if !SQLITE_OS_WINRT + { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, +#else + { "GetSystemInfo", (SYSCALL)0, 0 }, +#endif + +#define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) + + { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, + +#define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) + +#if !SQLITE_OS_WINCE + { "GetSystemTimeAsFileTime", (SYSCALL)GetSystemTimeAsFileTime, 0 }, +#else + { "GetSystemTimeAsFileTime", (SYSCALL)0, 0 }, +#endif + +#define osGetSystemTimeAsFileTime ((VOID(WINAPI*)( \ + LPFILETIME))aSyscall[30].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "GetTempPathA", (SYSCALL)GetTempPathA, 0 }, +#else + { "GetTempPathA", (SYSCALL)0, 0 }, +#endif + +#define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, +#else + { "GetTempPathW", (SYSCALL)0, 0 }, +#endif + +#define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) + +#if !SQLITE_OS_WINRT + { "GetTickCount", (SYSCALL)GetTickCount, 0 }, +#else + { "GetTickCount", (SYSCALL)0, 0 }, +#endif + +#define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) && defined(SQLITE_WIN32_GETVERSIONEX) && \ + SQLITE_WIN32_GETVERSIONEX + { "GetVersionExA", (SYSCALL)GetVersionExA, 0 }, +#else + { "GetVersionExA", (SYSCALL)0, 0 }, +#endif + +#define osGetVersionExA ((BOOL(WINAPI*)( \ + LPOSVERSIONINFOA))aSyscall[34].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX + { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, +#else + { "GetVersionExW", (SYSCALL)0, 0 }, +#endif + +#define osGetVersionExW ((BOOL(WINAPI*)( \ + LPOSVERSIONINFOW))aSyscall[35].pCurrent) + + { "HeapAlloc", (SYSCALL)HeapAlloc, 0 }, + +#define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ + SIZE_T))aSyscall[36].pCurrent) + +#if !SQLITE_OS_WINRT + { "HeapCreate", (SYSCALL)HeapCreate, 0 }, +#else + { "HeapCreate", (SYSCALL)0, 0 }, +#endif + +#define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ + SIZE_T))aSyscall[37].pCurrent) + +#if !SQLITE_OS_WINRT + { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, +#else + { "HeapDestroy", (SYSCALL)0, 0 }, +#endif + +#define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[38].pCurrent) + + { "HeapFree", (SYSCALL)HeapFree, 0 }, + +#define osHeapFree ((BOOL(WINAPI*)(HANDLE,DWORD,LPVOID))aSyscall[39].pCurrent) + + { "HeapReAlloc", (SYSCALL)HeapReAlloc, 0 }, + +#define osHeapReAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD,LPVOID, \ + SIZE_T))aSyscall[40].pCurrent) + + { "HeapSize", (SYSCALL)HeapSize, 0 }, + +#define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ + LPCVOID))aSyscall[41].pCurrent) + +#if !SQLITE_OS_WINRT + { "HeapValidate", (SYSCALL)HeapValidate, 0 }, +#else + { "HeapValidate", (SYSCALL)0, 0 }, +#endif + +#define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ + LPCVOID))aSyscall[42].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "HeapCompact", (SYSCALL)HeapCompact, 0 }, +#else + { "HeapCompact", (SYSCALL)0, 0 }, +#endif + +#define osHeapCompact ((UINT(WINAPI*)(HANDLE,DWORD))aSyscall[43].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) && !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadLibraryA", (SYSCALL)LoadLibraryA, 0 }, +#else + { "LoadLibraryA", (SYSCALL)0, 0 }, +#endif + +#define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[44].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, +#else + { "LoadLibraryW", (SYSCALL)0, 0 }, +#endif + +#define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[45].pCurrent) + +#if !SQLITE_OS_WINRT + { "LocalFree", (SYSCALL)LocalFree, 0 }, +#else + { "LocalFree", (SYSCALL)0, 0 }, +#endif + +#define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[46].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "LockFile", (SYSCALL)LockFile, 0 }, +#else + { "LockFile", (SYSCALL)0, 0 }, +#endif + +#ifndef osLockFile +#define osLockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + DWORD))aSyscall[47].pCurrent) +#endif + +#if !SQLITE_OS_WINCE + { "LockFileEx", (SYSCALL)LockFileEx, 0 }, +#else + { "LockFileEx", (SYSCALL)0, 0 }, +#endif + +#ifndef osLockFileEx +#define osLockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD,DWORD, \ + LPOVERLAPPED))aSyscall[48].pCurrent) +#endif + +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL)) + { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, +#else + { "MapViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osMapViewOfFile ((LPVOID(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + SIZE_T))aSyscall[49].pCurrent) + + { "MultiByteToWideChar", (SYSCALL)MultiByteToWideChar, 0 }, + +#define osMultiByteToWideChar ((int(WINAPI*)(UINT,DWORD,LPCSTR,int,LPWSTR, \ + int))aSyscall[50].pCurrent) + + { "QueryPerformanceCounter", (SYSCALL)QueryPerformanceCounter, 0 }, + +#define osQueryPerformanceCounter ((BOOL(WINAPI*)( \ + LARGE_INTEGER*))aSyscall[51].pCurrent) + + { "ReadFile", (SYSCALL)ReadFile, 0 }, + +#define osReadFile ((BOOL(WINAPI*)(HANDLE,LPVOID,DWORD,LPDWORD, \ + LPOVERLAPPED))aSyscall[52].pCurrent) + + { "SetEndOfFile", (SYSCALL)SetEndOfFile, 0 }, + +#define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[53].pCurrent) + +#if !SQLITE_OS_WINRT + { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, +#else + { "SetFilePointer", (SYSCALL)0, 0 }, +#endif + +#define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ + DWORD))aSyscall[54].pCurrent) + +#if !SQLITE_OS_WINRT + { "Sleep", (SYSCALL)Sleep, 0 }, +#else + { "Sleep", (SYSCALL)0, 0 }, +#endif + +#define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[55].pCurrent) + + { "SystemTimeToFileTime", (SYSCALL)SystemTimeToFileTime, 0 }, + +#define osSystemTimeToFileTime ((BOOL(WINAPI*)(CONST SYSTEMTIME*, \ + LPFILETIME))aSyscall[56].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "UnlockFile", (SYSCALL)UnlockFile, 0 }, +#else + { "UnlockFile", (SYSCALL)0, 0 }, +#endif + +#ifndef osUnlockFile +#define osUnlockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + DWORD))aSyscall[57].pCurrent) +#endif + +#if !SQLITE_OS_WINCE + { "UnlockFileEx", (SYSCALL)UnlockFileEx, 0 }, +#else + { "UnlockFileEx", (SYSCALL)0, 0 }, +#endif + +#define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + LPOVERLAPPED))aSyscall[58].pCurrent) + +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) + { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, +#else + { "UnmapViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osUnmapViewOfFile ((BOOL(WINAPI*)(LPCVOID))aSyscall[59].pCurrent) + + { "WideCharToMultiByte", (SYSCALL)WideCharToMultiByte, 0 }, + +#define osWideCharToMultiByte ((int(WINAPI*)(UINT,DWORD,LPCWSTR,int,LPSTR,int, \ + LPCSTR,LPBOOL))aSyscall[60].pCurrent) + + { "WriteFile", (SYSCALL)WriteFile, 0 }, + +#define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ + LPOVERLAPPED))aSyscall[61].pCurrent) + +#if SQLITE_OS_WINRT + { "CreateEventExW", (SYSCALL)CreateEventExW, 0 }, +#else + { "CreateEventExW", (SYSCALL)0, 0 }, +#endif + +#define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ + DWORD,DWORD))aSyscall[62].pCurrent) + +#if !SQLITE_OS_WINRT + { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, +#else + { "WaitForSingleObject", (SYSCALL)0, 0 }, +#endif + +#define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ + DWORD))aSyscall[63].pCurrent) + +#if SQLITE_OS_WINRT + { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, +#else + { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, +#endif + +#define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ + BOOL))aSyscall[64].pCurrent) + +#if SQLITE_OS_WINRT + { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, +#else + { "SetFilePointerEx", (SYSCALL)0, 0 }, +#endif + +#define osSetFilePointerEx ((BOOL(WINAPI*)(HANDLE,LARGE_INTEGER, \ + PLARGE_INTEGER,DWORD))aSyscall[65].pCurrent) + +#if SQLITE_OS_WINRT + { "GetFileInformationByHandleEx", (SYSCALL)GetFileInformationByHandleEx, 0 }, +#else + { "GetFileInformationByHandleEx", (SYSCALL)0, 0 }, +#endif + +#define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ + FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) + +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) + { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, +#else + { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, +#endif + +#define osMapViewOfFileFromApp ((LPVOID(WINAPI*)(HANDLE,ULONG,ULONG64, \ + SIZE_T))aSyscall[67].pCurrent) + +#if SQLITE_OS_WINRT + { "CreateFile2", (SYSCALL)CreateFile2, 0 }, +#else + { "CreateFile2", (SYSCALL)0, 0 }, +#endif + +#define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ + LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[68].pCurrent) + +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, +#else + { "LoadPackagedLibrary", (SYSCALL)0, 0 }, +#endif + +#define osLoadPackagedLibrary ((HMODULE(WINAPI*)(LPCWSTR, \ + DWORD))aSyscall[69].pCurrent) + +#if SQLITE_OS_WINRT + { "GetTickCount64", (SYSCALL)GetTickCount64, 0 }, +#else + { "GetTickCount64", (SYSCALL)0, 0 }, +#endif + +#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[70].pCurrent) + +#if SQLITE_OS_WINRT + { "GetNativeSystemInfo", (SYSCALL)GetNativeSystemInfo, 0 }, +#else + { "GetNativeSystemInfo", (SYSCALL)0, 0 }, +#endif + +#define osGetNativeSystemInfo ((VOID(WINAPI*)( \ + LPSYSTEM_INFO))aSyscall[71].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "OutputDebugStringA", (SYSCALL)OutputDebugStringA, 0 }, +#else + { "OutputDebugStringA", (SYSCALL)0, 0 }, +#endif + +#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[72].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "OutputDebugStringW", (SYSCALL)OutputDebugStringW, 0 }, +#else + { "OutputDebugStringW", (SYSCALL)0, 0 }, +#endif + +#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[73].pCurrent) + + { "GetProcessHeap", (SYSCALL)GetProcessHeap, 0 }, + +#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) + +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) + { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, +#else + { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ + LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) + +/* +** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" +** is really just a macro that uses a compiler intrinsic (e.g. x64). +** So do not try to make this is into a redefinable interface. +*/ +#if defined(InterlockedCompareExchange) + { "InterlockedCompareExchange", (SYSCALL)0, 0 }, + +#define osInterlockedCompareExchange InterlockedCompareExchange +#else + { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, + +#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG volatile*, \ + LONG,LONG))aSyscall[76].pCurrent) +#endif /* defined(InterlockedCompareExchange) */ + +}; /* End of the overrideable system calls */ + +/* +** This is the xSetSystemCall() method of sqlite3_vfs for all of the +** "win32" VFSes. Return SQLITE_OK opon successfully updating the +** system call pointer, or SQLITE_NOTFOUND if there is no configurable +** system call named zName. +*/ +static int winSetSystemCall( + sqlite3_vfs *pNotUsed, /* The VFS pointer. Not used */ + const char *zName, /* Name of system call to override */ + sqlite3_syscall_ptr pNewFunc /* Pointer to new system call value */ +){ + unsigned int i; + int rc = SQLITE_NOTFOUND; + + UNUSED_PARAMETER(pNotUsed); + if( zName==0 ){ + /* If no zName is given, restore all system calls to their default + ** settings and return NULL + */ + rc = SQLITE_OK; + for(i=0; i0 ){ + memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); + memcpy(zDbgBuf, zBuf, nMin); + osOutputDebugStringA(zDbgBuf); + }else{ + osOutputDebugStringA(zBuf); + } +#elif defined(SQLITE_WIN32_HAS_WIDE) + memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); + if ( osMultiByteToWideChar( + osAreFileApisANSI() ? CP_ACP : CP_OEMCP, 0, zBuf, + nMin, (LPWSTR)zDbgBuf, SQLITE_WIN32_DBG_BUF_SIZE/sizeof(WCHAR))<=0 ){ + return; + } + osOutputDebugStringW((LPCWSTR)zDbgBuf); +#else + if( nMin>0 ){ + memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); + memcpy(zDbgBuf, zBuf, nMin); + fprintf(stderr, "%s", zDbgBuf); + }else{ + fprintf(stderr, "%s", zBuf); + } +#endif +} + +/* +** The following routine suspends the current thread for at least ms +** milliseconds. This is equivalent to the Win32 Sleep() interface. +*/ +#if SQLITE_OS_WINRT +static HANDLE sleepObj = NULL; +#endif + +SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds){ +#if SQLITE_OS_WINRT + if ( sleepObj==NULL ){ + sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, + SYNCHRONIZE); + } + assert( sleepObj!=NULL ); + osWaitForSingleObjectEx(sleepObj, milliseconds, FALSE); +#else + osSleep(milliseconds); +#endif +} + +/* +** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, +** or WinCE. Return false (zero) for Win95, Win98, or WinME. +** +** Here is an interesting observation: Win95, Win98, and WinME lack +** the LockFileEx() API. But we can still statically link against that +** API as long as we don't call it when running Win95/98/ME. A call to +** this routine is used to determine if the host is Win95/98/ME or +** WinNT/2K/XP so that we will know whether or not we can safely call +** the LockFileEx() API. +*/ + +#if !defined(SQLITE_WIN32_GETVERSIONEX) || !SQLITE_WIN32_GETVERSIONEX +# define osIsNT() (1) +#elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) +# define osIsNT() (1) +#elif !defined(SQLITE_WIN32_HAS_WIDE) +# define osIsNT() (0) +#else +# define osIsNT() ((sqlite3_os_type==2) || sqlite3_win32_is_nt()) +#endif + +/* +** This function determines if the machine is running a version of Windows +** based on the NT kernel. +*/ +SQLITE_API int sqlite3_win32_is_nt(void){ +#if defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX + if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WIN8 + OSVERSIONINFOW sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExW(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#elif defined(SQLITE_WIN32_HAS_ANSI) + OSVERSIONINFOA sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExA(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#endif + } + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#elif SQLITE_TEST + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#else + return 1; +#endif +} + +#ifdef SQLITE_WIN32_MALLOC +/* +** Allocate nBytes of memory. +*/ +static void *winMemMalloc(int nBytes){ + HANDLE hHeap; + void *p; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + assert( nBytes>=0 ); + p = osHeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + if( !p ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapAlloc %u bytes (%lu), heap=%p", + nBytes, osGetLastError(), (void*)hHeap); + } + return p; +} + +/* +** Free memory. +*/ +static void winMemFree(void *pPrior){ + HANDLE hHeap; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); +#endif + if( !pPrior ) return; /* Passing NULL to HeapFree is undefined. */ + if( !osHeapFree(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapFree block %p (%lu), heap=%p", + pPrior, osGetLastError(), (void*)hHeap); + } +} + +/* +** Change the size of an existing memory allocation +*/ +static void *winMemRealloc(void *pPrior, int nBytes){ + HANDLE hHeap; + void *p; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); +#endif + assert( nBytes>=0 ); + if( !pPrior ){ + p = osHeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + }else{ + p = osHeapReAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior, (SIZE_T)nBytes); + } + if( !p ){ + sqlite3_log(SQLITE_NOMEM, "failed to %s %u bytes (%lu), heap=%p", + pPrior ? "HeapReAlloc" : "HeapAlloc", nBytes, osGetLastError(), + (void*)hHeap); + } + return p; +} + +/* +** Return the size of an outstanding allocation, in bytes. +*/ +static int winMemSize(void *p){ + HANDLE hHeap; + SIZE_T n; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, p) ); +#endif + if( !p ) return 0; + n = osHeapSize(hHeap, SQLITE_WIN32_HEAP_FLAGS, p); + if( n==(SIZE_T)-1 ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapSize block %p (%lu), heap=%p", + p, osGetLastError(), (void*)hHeap); + return 0; + } + return (int)n; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int winMemRoundup(int n){ + return n; +} + +/* +** Initialize this module. +*/ +static int winMemInit(void *pAppData){ + winMemData *pWinMemData = (winMemData *)pAppData; + + if( !pWinMemData ) return SQLITE_ERROR; + assert( pWinMemData->magic1==WINMEM_MAGIC1 ); + assert( pWinMemData->magic2==WINMEM_MAGIC2 ); + +#if !SQLITE_OS_WINRT && SQLITE_WIN32_HEAP_CREATE + if( !pWinMemData->hHeap ){ + DWORD dwInitialSize = SQLITE_WIN32_HEAP_INIT_SIZE; + DWORD dwMaximumSize = (DWORD)sqlite3GlobalConfig.nHeap; + if( dwMaximumSize==0 ){ + dwMaximumSize = SQLITE_WIN32_HEAP_MAX_SIZE; + }else if( dwInitialSize>dwMaximumSize ){ + dwInitialSize = dwMaximumSize; + } + pWinMemData->hHeap = osHeapCreate(SQLITE_WIN32_HEAP_FLAGS, + dwInitialSize, dwMaximumSize); + if( !pWinMemData->hHeap ){ + sqlite3_log(SQLITE_NOMEM, + "failed to HeapCreate (%lu), flags=%u, initSize=%lu, maxSize=%lu", + osGetLastError(), SQLITE_WIN32_HEAP_FLAGS, dwInitialSize, + dwMaximumSize); + return SQLITE_NOMEM; + } + pWinMemData->bOwned = TRUE; + assert( pWinMemData->bOwned ); + } +#else + pWinMemData->hHeap = osGetProcessHeap(); + if( !pWinMemData->hHeap ){ + sqlite3_log(SQLITE_NOMEM, + "failed to GetProcessHeap (%lu)", osGetLastError()); + return SQLITE_NOMEM; + } + pWinMemData->bOwned = FALSE; + assert( !pWinMemData->bOwned ); +#endif + assert( pWinMemData->hHeap!=0 ); + assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void winMemShutdown(void *pAppData){ + winMemData *pWinMemData = (winMemData *)pAppData; + + if( !pWinMemData ) return; + assert( pWinMemData->magic1==WINMEM_MAGIC1 ); + assert( pWinMemData->magic2==WINMEM_MAGIC2 ); + + if( pWinMemData->hHeap ){ + assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + if( pWinMemData->bOwned ){ + if( !osHeapDestroy(pWinMemData->hHeap) ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapDestroy (%lu), heap=%p", + osGetLastError(), (void*)pWinMemData->hHeap); + } + pWinMemData->bOwned = FALSE; + } + pWinMemData->hHeap = NULL; + } +} + +/* +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. The +** arguments specify the block of memory to manage. +** +** This routine is only called by sqlite3_config(), and therefore +** is not required to be threadsafe (it is not). +*/ +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetWin32(void){ + static const sqlite3_mem_methods winMemMethods = { + winMemMalloc, + winMemFree, + winMemRealloc, + winMemSize, + winMemRoundup, + winMemInit, + winMemShutdown, + &win_mem_data + }; + return &winMemMethods; +} + +SQLITE_PRIVATE void sqlite3MemSetDefault(void){ + sqlite3_config(SQLITE_CONFIG_MALLOC, sqlite3MemGetWin32()); +} +#endif /* SQLITE_WIN32_MALLOC */ + +/* +** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). +** +** Space to hold the returned string is obtained from malloc. +*/ +static LPWSTR winUtf8ToUnicode(const char *zFilename){ + int nChar; + LPWSTR zWideFilename; + + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideFilename = sqlite3MallocZero( nChar*sizeof(zWideFilename[0]) ); + if( zWideFilename==0 ){ + return 0; + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, + nChar); + if( nChar==0 ){ + sqlite3_free(zWideFilename); + zWideFilename = 0; + } + return zWideFilename; +} + +/* +** Convert Microsoft Unicode to UTF-8. Space to hold the returned string is +** obtained from sqlite3_malloc(). +*/ +static char *winUnicodeToUtf8(LPCWSTR zWideFilename){ + int nByte; + char *zFilename; + + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); + if( nByte == 0 ){ + return 0; + } + zFilename = sqlite3MallocZero( nByte ); + if( zFilename==0 ){ + return 0; + } + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, + 0, 0); + if( nByte == 0 ){ + sqlite3_free(zFilename); + zFilename = 0; + } + return zFilename; +} + +/* +** Convert an ANSI string to Microsoft Unicode, based on the +** current codepage settings for file apis. +** +** Space to hold the returned string is obtained +** from sqlite3_malloc. +*/ +static LPWSTR winMbcsToUnicode(const char *zFilename){ + int nByte; + LPWSTR zMbcsFilename; + int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; + + nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, NULL, + 0)*sizeof(WCHAR); + if( nByte==0 ){ + return 0; + } + zMbcsFilename = sqlite3MallocZero( nByte*sizeof(zMbcsFilename[0]) ); + if( zMbcsFilename==0 ){ + return 0; + } + nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, + nByte); + if( nByte==0 ){ + sqlite3_free(zMbcsFilename); + zMbcsFilename = 0; + } + return zMbcsFilename; +} + +/* +** Convert Microsoft Unicode to multi-byte character string, based on the +** user's ANSI codepage. +** +** Space to hold the returned string is obtained from +** sqlite3_malloc(). +*/ +static char *winUnicodeToMbcs(LPCWSTR zWideFilename){ + int nByte; + char *zFilename; + int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; + + nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); + if( nByte == 0 ){ + return 0; + } + zFilename = sqlite3MallocZero( nByte ); + if( zFilename==0 ){ + return 0; + } + nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, + nByte, 0, 0); + if( nByte == 0 ){ + sqlite3_free(zFilename); + zFilename = 0; + } + return zFilename; +} + +/* +** Convert multibyte character string to UTF-8. Space to hold the +** returned string is obtained from sqlite3_malloc(). +*/ +SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ + char *zFilenameUtf8; + LPWSTR zTmpWide; + + zTmpWide = winMbcsToUnicode(zFilename); + if( zTmpWide==0 ){ + return 0; + } + zFilenameUtf8 = winUnicodeToUtf8(zTmpWide); + sqlite3_free(zTmpWide); + return zFilenameUtf8; +} + +/* +** Convert UTF-8 to multibyte character string. Space to hold the +** returned string is obtained from sqlite3_malloc(). +*/ +SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ + char *zFilenameMbcs; + LPWSTR zTmpWide; + + zTmpWide = winUtf8ToUnicode(zFilename); + if( zTmpWide==0 ){ + return 0; + } + zFilenameMbcs = winUnicodeToMbcs(zTmpWide); + sqlite3_free(zTmpWide); + return zFilenameMbcs; +} + +/* +** This function sets the data directory or the temporary directory based on +** the provided arguments. The type argument must be 1 in order to set the +** data directory or 2 in order to set the temporary directory. The zValue +** argument is the name of the directory to use. The return value will be +** SQLITE_OK if successful. +*/ +SQLITE_API int sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ + char **ppDirectory = 0; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){ + ppDirectory = &sqlite3_data_directory; + }else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){ + ppDirectory = &sqlite3_temp_directory; + } + assert( !ppDirectory || type==SQLITE_WIN32_DATA_DIRECTORY_TYPE + || type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE + ); + assert( !ppDirectory || sqlite3MemdebugHasType(*ppDirectory, MEMTYPE_HEAP) ); + if( ppDirectory ){ + char *zValueUtf8 = 0; + if( zValue && zValue[0] ){ + zValueUtf8 = winUnicodeToUtf8(zValue); + if ( zValueUtf8==0 ){ + return SQLITE_NOMEM; + } + } + sqlite3_free(*ppDirectory); + *ppDirectory = zValueUtf8; + return SQLITE_OK; + } + return SQLITE_ERROR; +} + +/* +** The return value of winGetLastErrorMsg +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). +*/ +static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ + /* FormatMessage returns 0 on failure. Otherwise it + ** returns the number of TCHARs written to the output + ** buffer, excluding the terminating null char. + */ + DWORD dwLen = 0; + char *zOut = 0; + + if( osIsNT() ){ +#if SQLITE_OS_WINRT + WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1]; + dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + zTempWide, + SQLITE_WIN32_MAX_ERRMSG_CHARS, + 0); +#else + LPWSTR zTempWide = NULL; + dwLen = osFormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + (LPWSTR) &zTempWide, + 0, + 0); +#endif + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + sqlite3BeginBenignMalloc(); + zOut = winUnicodeToUtf8(zTempWide); + sqlite3EndBenignMalloc(); +#if !SQLITE_OS_WINRT + /* free the system buffer allocated by FormatMessage */ + osLocalFree(zTempWide); +#endif + } + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + char *zTemp = NULL; + dwLen = osFormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + (LPSTR) &zTemp, + 0, + 0); + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + sqlite3BeginBenignMalloc(); + zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + sqlite3EndBenignMalloc(); + /* free the system buffer allocated by FormatMessage */ + osLocalFree(zTemp); + } + } +#endif + if( 0 == dwLen ){ + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%lx (%lu)", lastErrno, lastErrno); + }else{ + /* copy a maximum of nBuf chars to output buffer */ + sqlite3_snprintf(nBuf, zBuf, "%s", zOut); + /* free the UTF8 buffer */ + sqlite3_free(zOut); + } + return 0; +} + +/* +** +** This function - winLogErrorAtLine() - is only ever called via the macro +** winLogError(). +** +** This routine is invoked after an error occurs in an OS function. +** It logs a message using sqlite3_log() containing the current value of +** error code and, if possible, the human-readable equivalent from +** FormatMessage. +** +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed and the associated file-system path, if any. +*/ +#define winLogError(a,b,c,d) winLogErrorAtLine(a,b,c,d,__LINE__) +static int winLogErrorAtLine( + int errcode, /* SQLite error code */ + DWORD lastErrno, /* Win32 last error */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char zMsg[500]; /* Human readable error text */ + int i; /* Loop counter */ + + zMsg[0] = 0; + winGetLastErrorMsg(lastErrno, sizeof(zMsg), zMsg); + assert( errcode!=SQLITE_OK ); + if( zPath==0 ) zPath = ""; + for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} + zMsg[i] = 0; + sqlite3_log(errcode, + "os_win.c:%d: (%lu) %s(%s) - %s", + iLine, lastErrno, zFunc, zPath, zMsg + ); + + return errcode; +} + +/* +** The number of times that a ReadFile(), WriteFile(), and DeleteFile() +** will be retried following a locking error - probably caused by +** antivirus software. Also the initial delay before the first retry. +** The delay increases linearly with each retry. +*/ +#ifndef SQLITE_WIN32_IOERR_RETRY +# define SQLITE_WIN32_IOERR_RETRY 10 +#endif +#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY +# define SQLITE_WIN32_IOERR_RETRY_DELAY 25 +#endif +static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; +static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; + +/* +** The "winIoerrCanRetry1" macro is used to determine if a particular I/O +** error code obtained via GetLastError() is eligible to be retried. It +** must accept the error code DWORD as its only argument and should return +** non-zero if the error code is transient in nature and the operation +** responsible for generating the original error might succeed upon being +** retried. The argument to this macro should be a variable. +** +** Additionally, a macro named "winIoerrCanRetry2" may be defined. If it +** is defined, it will be consulted only when the macro "winIoerrCanRetry1" +** returns zero. The "winIoerrCanRetry2" macro is completely optional and +** may be used to include additional error codes in the set that should +** result in the failing I/O operation being retried by the caller. If +** defined, the "winIoerrCanRetry2" macro must exhibit external semantics +** identical to those of the "winIoerrCanRetry1" macro. +*/ +#if !defined(winIoerrCanRetry1) +#define winIoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ + ((a)==ERROR_SHARING_VIOLATION) || \ + ((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_DEV_NOT_EXIST) || \ + ((a)==ERROR_NETNAME_DELETED) || \ + ((a)==ERROR_SEM_TIMEOUT) || \ + ((a)==ERROR_NETWORK_UNREACHABLE)) +#endif + +/* +** If a ReadFile() or WriteFile() error occurs, invoke this routine +** to see if it should be retried. Return TRUE to retry. Return FALSE +** to give up with an error. +*/ +static int winRetryIoerr(int *pnRetry, DWORD *pError){ + DWORD e = osGetLastError(); + if( *pnRetry>=winIoerrRetry ){ + if( pError ){ + *pError = e; + } + return 0; + } + if( winIoerrCanRetry1(e) ){ + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#if defined(winIoerrCanRetry2) + else if( winIoerrCanRetry2(e) ){ + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#endif + if( pError ){ + *pError = e; + } + return 0; +} + +/* +** Log a I/O error retry episode. +*/ +static void winLogIoerr(int nRetry){ + if( nRetry ){ + sqlite3_log(SQLITE_IOERR, + "delayed %dms for lock/sharing conflict", + winIoerrRetryDelay*nRetry*(nRetry+1)/2 + ); + } +} + +#if SQLITE_OS_WINCE +/************************************************************************* +** This section contains code for WinCE only. +*/ +#if !defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API +/* +** The MSVC CRT on Windows CE may not have a localtime() function. So +** create a substitute. +*/ +/* #include */ +struct tm *__cdecl localtime(const time_t *t) +{ + static struct tm y; + FILETIME uTm, lTm; + SYSTEMTIME pTm; + sqlite3_int64 t64; + t64 = *t; + t64 = (t64 + 11644473600)*10000000; + uTm.dwLowDateTime = (DWORD)(t64 & 0xFFFFFFFF); + uTm.dwHighDateTime= (DWORD)(t64 >> 32); + osFileTimeToLocalFileTime(&uTm,&lTm); + osFileTimeToSystemTime(&lTm,&pTm); + y.tm_year = pTm.wYear - 1900; + y.tm_mon = pTm.wMonth - 1; + y.tm_wday = pTm.wDayOfWeek; + y.tm_mday = pTm.wDay; + y.tm_hour = pTm.wHour; + y.tm_min = pTm.wMinute; + y.tm_sec = pTm.wSecond; + return &y; +} +#endif + +#define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] + +/* +** Acquire a lock on the handle h +*/ +static void winceMutexAcquire(HANDLE h){ + DWORD dwErr; + do { + dwErr = osWaitForSingleObject(h, INFINITE); + } while (dwErr != WAIT_OBJECT_0 && dwErr != WAIT_ABANDONED); +} +/* +** Release a lock acquired by winceMutexAcquire() +*/ +#define winceMutexRelease(h) ReleaseMutex(h) + +/* +** Create the mutex and shared memory used for locking in the file +** descriptor pFile +*/ +static int winceCreateLock(const char *zFilename, winFile *pFile){ + LPWSTR zTok; + LPWSTR zName; + DWORD lastErrno; + BOOL bLogged = FALSE; + BOOL bInit = TRUE; + + zName = winUtf8ToUnicode(zFilename); + if( zName==0 ){ + /* out of memory */ + return SQLITE_IOERR_NOMEM; + } + + /* Initialize the local lockdata */ + memset(&pFile->local, 0, sizeof(pFile->local)); + + /* Replace the backslashes from the filename and lowercase it + ** to derive a mutex name. */ + zTok = osCharLowerW(zName); + for (;*zTok;zTok++){ + if (*zTok == '\\') *zTok = '_'; + } + + /* Create/open the named mutex */ + pFile->hMutex = osCreateMutexW(NULL, FALSE, zName); + if (!pFile->hMutex){ + pFile->lastErrno = osGetLastError(); + sqlite3_free(zName); + return winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock1", zFilename); + } + + /* Acquire the mutex before continuing */ + winceMutexAcquire(pFile->hMutex); + + /* Since the names of named mutexes, semaphores, file mappings etc are + ** case-sensitive, take advantage of that by uppercasing the mutex name + ** and using that as the shared filemapping name. + */ + osCharUpperW(zName); + pFile->hShared = osCreateFileMappingW(INVALID_HANDLE_VALUE, NULL, + PAGE_READWRITE, 0, sizeof(winceLock), + zName); + + /* Set a flag that indicates we're the first to create the memory so it + ** must be zero-initialized */ + lastErrno = osGetLastError(); + if (lastErrno == ERROR_ALREADY_EXISTS){ + bInit = FALSE; + } + + sqlite3_free(zName); + + /* If we succeeded in making the shared memory handle, map it. */ + if( pFile->hShared ){ + pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, + FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); + /* If mapping failed, close the shared memory handle and erase it */ + if( !pFile->shared ){ + pFile->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock2", zFilename); + bLogged = TRUE; + osCloseHandle(pFile->hShared); + pFile->hShared = NULL; + } + } + + /* If shared memory could not be created, then close the mutex and fail */ + if( pFile->hShared==NULL ){ + if( !bLogged ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock3", zFilename); + bLogged = TRUE; + } + winceMutexRelease(pFile->hMutex); + osCloseHandle(pFile->hMutex); + pFile->hMutex = NULL; + return SQLITE_IOERR; + } + + /* Initialize the shared memory if we're supposed to */ + if( bInit ){ + memset(pFile->shared, 0, sizeof(winceLock)); + } + + winceMutexRelease(pFile->hMutex); + return SQLITE_OK; +} + +/* +** Destroy the part of winFile that deals with wince locks +*/ +static void winceDestroyLock(winFile *pFile){ + if (pFile->hMutex){ + /* Acquire the mutex */ + winceMutexAcquire(pFile->hMutex); + + /* The following blocks should probably assert in debug mode, but they + are to cleanup in case any locks remained open */ + if (pFile->local.nReaders){ + pFile->shared->nReaders --; + } + if (pFile->local.bReserved){ + pFile->shared->bReserved = FALSE; + } + if (pFile->local.bPending){ + pFile->shared->bPending = FALSE; + } + if (pFile->local.bExclusive){ + pFile->shared->bExclusive = FALSE; + } + + /* De-reference and close our copy of the shared memory handle */ + osUnmapViewOfFile(pFile->shared); + osCloseHandle(pFile->hShared); + + /* Done with the mutex */ + winceMutexRelease(pFile->hMutex); + osCloseHandle(pFile->hMutex); + pFile->hMutex = NULL; + } +} + +/* +** An implementation of the LockFile() API of Windows for CE +*/ +static BOOL winceLockFile( + LPHANDLE phFile, + DWORD dwFileOffsetLow, + DWORD dwFileOffsetHigh, + DWORD nNumberOfBytesToLockLow, + DWORD nNumberOfBytesToLockHigh +){ + winFile *pFile = HANDLE_TO_WINFILE(phFile); + BOOL bReturn = FALSE; + + UNUSED_PARAMETER(dwFileOffsetHigh); + UNUSED_PARAMETER(nNumberOfBytesToLockHigh); + + if (!pFile->hMutex) return TRUE; + winceMutexAcquire(pFile->hMutex); + + /* Wanting an exclusive lock? */ + if (dwFileOffsetLow == (DWORD)SHARED_FIRST + && nNumberOfBytesToLockLow == (DWORD)SHARED_SIZE){ + if (pFile->shared->nReaders == 0 && pFile->shared->bExclusive == 0){ + pFile->shared->bExclusive = TRUE; + pFile->local.bExclusive = TRUE; + bReturn = TRUE; + } + } + + /* Want a read-only lock? */ + else if (dwFileOffsetLow == (DWORD)SHARED_FIRST && + nNumberOfBytesToLockLow == 1){ + if (pFile->shared->bExclusive == 0){ + pFile->local.nReaders ++; + if (pFile->local.nReaders == 1){ + pFile->shared->nReaders ++; + } + bReturn = TRUE; + } + } + + /* Want a pending lock? */ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToLockLow == 1){ + /* If no pending lock has been acquired, then acquire it */ + if (pFile->shared->bPending == 0) { + pFile->shared->bPending = TRUE; + pFile->local.bPending = TRUE; + bReturn = TRUE; + } + } + + /* Want a reserved lock? */ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToLockLow == 1){ + if (pFile->shared->bReserved == 0) { + pFile->shared->bReserved = TRUE; + pFile->local.bReserved = TRUE; + bReturn = TRUE; + } + } + + winceMutexRelease(pFile->hMutex); + return bReturn; +} + +/* +** An implementation of the UnlockFile API of Windows for CE +*/ +static BOOL winceUnlockFile( + LPHANDLE phFile, + DWORD dwFileOffsetLow, + DWORD dwFileOffsetHigh, + DWORD nNumberOfBytesToUnlockLow, + DWORD nNumberOfBytesToUnlockHigh +){ + winFile *pFile = HANDLE_TO_WINFILE(phFile); + BOOL bReturn = FALSE; + + UNUSED_PARAMETER(dwFileOffsetHigh); + UNUSED_PARAMETER(nNumberOfBytesToUnlockHigh); + + if (!pFile->hMutex) return TRUE; + winceMutexAcquire(pFile->hMutex); + + /* Releasing a reader lock or an exclusive lock */ + if (dwFileOffsetLow == (DWORD)SHARED_FIRST){ + /* Did we have an exclusive lock? */ + if (pFile->local.bExclusive){ + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE); + pFile->local.bExclusive = FALSE; + pFile->shared->bExclusive = FALSE; + bReturn = TRUE; + } + + /* Did we just have a reader lock? */ + else if (pFile->local.nReaders){ + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE + || nNumberOfBytesToUnlockLow == 1); + pFile->local.nReaders --; + if (pFile->local.nReaders == 0) + { + pFile->shared->nReaders --; + } + bReturn = TRUE; + } + } + + /* Releasing a pending lock */ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToUnlockLow == 1){ + if (pFile->local.bPending){ + pFile->local.bPending = FALSE; + pFile->shared->bPending = FALSE; + bReturn = TRUE; + } + } + /* Releasing a reserved lock */ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToUnlockLow == 1){ + if (pFile->local.bReserved) { + pFile->local.bReserved = FALSE; + pFile->shared->bReserved = FALSE; + bReturn = TRUE; + } + } + + winceMutexRelease(pFile->hMutex); + return bReturn; +} +/* +** End of the special code for wince +*****************************************************************************/ +#endif /* SQLITE_OS_WINCE */ + +/* +** Lock a file region. +*/ +static BOOL winLockFile( + LPHANDLE phFile, + DWORD flags, + DWORD offsetLow, + DWORD offsetHigh, + DWORD numBytesLow, + DWORD numBytesHigh +){ +#if SQLITE_OS_WINCE + /* + ** NOTE: Windows CE is handled differently here due its lack of the Win32 + ** API LockFile. + */ + return winceLockFile(phFile, offsetLow, offsetHigh, + numBytesLow, numBytesHigh); +#else + if( osIsNT() ){ + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = offsetLow; + ovlp.OffsetHigh = offsetHigh; + return osLockFileEx(*phFile, flags, 0, numBytesLow, numBytesHigh, &ovlp); + }else{ + return osLockFile(*phFile, offsetLow, offsetHigh, numBytesLow, + numBytesHigh); + } +#endif +} + +/* +** Unlock a file region. + */ +static BOOL winUnlockFile( + LPHANDLE phFile, + DWORD offsetLow, + DWORD offsetHigh, + DWORD numBytesLow, + DWORD numBytesHigh +){ +#if SQLITE_OS_WINCE + /* + ** NOTE: Windows CE is handled differently here due its lack of the Win32 + ** API UnlockFile. + */ + return winceUnlockFile(phFile, offsetLow, offsetHigh, + numBytesLow, numBytesHigh); +#else + if( osIsNT() ){ + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = offsetLow; + ovlp.OffsetHigh = offsetHigh; + return osUnlockFileEx(*phFile, 0, numBytesLow, numBytesHigh, &ovlp); + }else{ + return osUnlockFile(*phFile, offsetLow, offsetHigh, numBytesLow, + numBytesHigh); + } +#endif +} + +/***************************************************************************** +** The next group of routines implement the I/O methods specified +** by the sqlite3_io_methods object. +******************************************************************************/ + +/* +** Some Microsoft compilers lack this definition. +*/ +#ifndef INVALID_SET_FILE_POINTER +# define INVALID_SET_FILE_POINTER ((DWORD)-1) +#endif + +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ +#if !SQLITE_OS_WINRT + LONG upperBits; /* Most sig. 32 bits of new offset */ + LONG lowerBits; /* Least sig. 32 bits of new offset */ + DWORD dwRet; /* Value returned by SetFilePointer() */ + DWORD lastErrno; /* Value returned by GetLastError() */ + + OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); + + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); + lowerBits = (LONG)(iOffset & 0xffffffff); + + /* API oddity: If successful, SetFilePointer() returns a dword + ** containing the lower 32-bits of the new file-offset. Or, if it fails, + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occurred, it is also necessary to call + ** GetLastError(). + */ + dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + + if( (dwRet==INVALID_SET_FILE_POINTER + && ((lastErrno = osGetLastError())!=NO_ERROR)) ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, + "winSeekFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); + return 1; + } + + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); + return 0; +#else + /* + ** Same as above, except that this implementation works for WinRT. + */ + + LARGE_INTEGER x; /* The new offset */ + BOOL bRet; /* Value returned by SetFilePointerEx() */ + + x.QuadPart = iOffset; + bRet = osSetFilePointerEx(pFile->h, x, 0, FILE_BEGIN); + + if(!bRet){ + pFile->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, + "winSeekFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); + return 1; + } + + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); + return 0; +#endif +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* Forward references to VFS helper methods used for memory mapped files */ +static int winMapfile(winFile*, sqlite3_int64); +static int winUnmapfile(winFile*); +#endif + +/* +** Close a file. +** +** It is reported that an attempt to close a handle might sometimes +** fail. This is a very unreasonable result, but Windows is notorious +** for being unreasonable so I do not doubt that it might happen. If +** the close fails, we pause for 100 milliseconds and try again. As +** many as MX_CLOSE_ATTEMPT attempts to close the handle are made before +** giving up and returning an error. +*/ +#define MX_CLOSE_ATTEMPT 3 +static int winClose(sqlite3_file *id){ + int rc, cnt = 0; + winFile *pFile = (winFile*)id; + + assert( id!=0 ); +#ifndef SQLITE_OMIT_WAL + assert( pFile->pShm==0 ); +#endif + assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + OSTRACE(("CLOSE file=%p\n", pFile->h)); + +#if SQLITE_MAX_MMAP_SIZE>0 + winUnmapfile(pFile); +#endif + + do{ + rc = osCloseHandle(pFile->h); + /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ + }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); +#if SQLITE_OS_WINCE +#define WINCE_DELETION_ATTEMPTS 3 + winceDestroyLock(pFile); + if( pFile->zDeleteOnClose ){ + int cnt = 0; + while( + osDeleteFileW(pFile->zDeleteOnClose)==0 + && osGetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff + && cnt++ < WINCE_DELETION_ATTEMPTS + ){ + sqlite3_win32_sleep(100); /* Wait a little before trying again */ + } + sqlite3_free(pFile->zDeleteOnClose); + } +#endif + if( rc ){ + pFile->h = NULL; + } + OpenCounter(-1); + OSTRACE(("CLOSE file=%p, rc=%s\n", pFile->h, rc ? "ok" : "failed")); + return rc ? SQLITE_OK + : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), + "winClose", pFile->zPath); +} + +/* +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. +*/ +static int winRead( + sqlite3_file *id, /* File to read from */ + void *pBuf, /* Write content into this buffer */ + int amt, /* Number of bytes to read */ + sqlite3_int64 offset /* Begin reading at this offset */ +){ +#if !SQLITE_OS_WINCE + OVERLAPPED overlapped; /* The offset for ReadFile. */ +#endif + winFile *pFile = (winFile*)id; /* file handle */ + DWORD nRead; /* Number of bytes actually read from file */ + int nRetry = 0; /* Number of retrys */ + + assert( id!=0 ); + assert( amt>0 ); + assert( offset>=0 ); + SimulateIOError(return SQLITE_IOERR_READ); + OSTRACE(("READ file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + OSTRACE(("READ-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + +#if SQLITE_OS_WINCE + if( winSeekFile(pFile, offset) ){ + OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); + return SQLITE_FULL; + } + while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ +#else + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + while( !osReadFile(pFile->h, pBuf, amt, &nRead, &overlapped) && + osGetLastError()!=ERROR_HANDLE_EOF ){ +#endif + DWORD lastErrno; + if( winRetryIoerr(&nRetry, &lastErrno) ) continue; + pFile->lastErrno = lastErrno; + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_READ\n", pFile->h)); + return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, + "winRead", pFile->zPath); + } + winLogIoerr(nRetry); + if( nRead<(DWORD)amt ){ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[nRead], 0, amt-nRead); + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_SHORT_READ\n", pFile->h)); + return SQLITE_IOERR_SHORT_READ; + } + + OSTRACE(("READ file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; +} + +/* +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. +*/ +static int winWrite( + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ +){ + int rc = 0; /* True if error has occurred, else false */ + winFile *pFile = (winFile*)id; /* File handle */ + int nRetry = 0; /* Number of retries */ + + assert( amt>0 ); + assert( pFile ); + SimulateIOError(return SQLITE_IOERR_WRITE); + SimulateDiskfullError(return SQLITE_FULL); + + OSTRACE(("WRITE file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + OSTRACE(("WRITE-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + +#if SQLITE_OS_WINCE + rc = winSeekFile(pFile, offset); + if( rc==0 ){ +#else + { +#endif +#if !SQLITE_OS_WINCE + OVERLAPPED overlapped; /* The offset for WriteFile. */ +#endif + u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ + int nRem = amt; /* Number of bytes yet to be written */ + DWORD nWrite; /* Bytes written by each WriteFile() call */ + DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ + +#if !SQLITE_OS_WINCE + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif + + while( nRem>0 ){ +#if SQLITE_OS_WINCE + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ +#else + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ +#endif + if( winRetryIoerr(&nRetry, &lastErrno) ) continue; + break; + } + assert( nWrite==0 || nWrite<=(DWORD)nRem ); + if( nWrite==0 || nWrite>(DWORD)nRem ){ + lastErrno = osGetLastError(); + break; + } +#if !SQLITE_OS_WINCE + offset += nWrite; + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif + aRem += nWrite; + nRem -= nWrite; + } + if( nRem>0 ){ + pFile->lastErrno = lastErrno; + rc = 1; + } + } + + if( rc ){ + if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) + || ( pFile->lastErrno==ERROR_DISK_FULL )){ + OSTRACE(("WRITE file=%p, rc=SQLITE_FULL\n", pFile->h)); + return winLogError(SQLITE_FULL, pFile->lastErrno, + "winWrite1", pFile->zPath); + } + OSTRACE(("WRITE file=%p, rc=SQLITE_IOERR_WRITE\n", pFile->h)); + return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, + "winWrite2", pFile->zPath); + }else{ + winLogIoerr(nRetry); + } + OSTRACE(("WRITE file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; +} + +/* +** Truncate an open file to a specified size +*/ +static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ + winFile *pFile = (winFile*)id; /* File handle object */ + int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; + + assert( pFile ); + SimulateIOError(return SQLITE_IOERR_TRUNCATE); + OSTRACE(("TRUNCATE file=%p, size=%lld, lock=%d\n", + pFile->h, nByte, pFile->locktype)); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk>0 ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ + if( winSeekFile(pFile, nByte) ){ + rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; + rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, + "winTruncate2", pFile->zPath); + } + +#if SQLITE_MAX_MMAP_SIZE>0 + /* If the file was truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; + } +#endif + + OSTRACE(("TRUNCATE file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; +} + +#ifdef SQLITE_TEST +/* +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occuring at the right times. +*/ +SQLITE_API int sqlite3_sync_count = 0; +SQLITE_API int sqlite3_fullsync_count = 0; +#endif + +/* +** Make sure all writes to a particular file are committed to disk. +*/ +static int winSync(sqlite3_file *id, int flags){ +#ifndef SQLITE_NO_SYNC + /* + ** Used only when SQLITE_NO_SYNC is not defined. + */ + BOOL rc; +#endif +#if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || \ + (defined(SQLITE_TEST) && defined(SQLITE_DEBUG)) + /* + ** Used when SQLITE_NO_SYNC is not defined and by the assert() and/or + ** OSTRACE() macros. + */ + winFile *pFile = (winFile*)id; +#else + UNUSED_PARAMETER(id); +#endif + + assert( pFile ); + /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ + assert((flags&0x0F)==SQLITE_SYNC_NORMAL + || (flags&0x0F)==SQLITE_SYNC_FULL + ); + + /* Unix cannot, but some systems may return SQLITE_FULL from here. This + ** line is to test that doing so does not cause any problems. + */ + SimulateDiskfullError( return SQLITE_FULL ); + + OSTRACE(("SYNC file=%p, flags=%x, lock=%d\n", + pFile->h, flags, pFile->locktype)); + +#ifndef SQLITE_TEST + UNUSED_PARAMETER(flags); +#else + if( (flags&0x0F)==SQLITE_SYNC_FULL ){ + sqlite3_fullsync_count++; + } + sqlite3_sync_count++; +#endif + + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op + */ +#ifdef SQLITE_NO_SYNC + OSTRACE(("SYNC-NOP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; +#else + rc = osFlushFileBuffers(pFile->h); + SimulateIOError( rc=FALSE ); + if( rc ){ + OSTRACE(("SYNC file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC file=%p, rc=SQLITE_IOERR_FSYNC\n", pFile->h)); + return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, + "winSync", pFile->zPath); + } +#endif +} + +/* +** Determine the current size of a file in bytes +*/ +static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ + winFile *pFile = (winFile*)id; + int rc = SQLITE_OK; + + assert( id!=0 ); + assert( pSize!=0 ); + SimulateIOError(return SQLITE_IOERR_FSTAT); + OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); + +#if SQLITE_OS_WINRT + { + FILE_STANDARD_INFO info; + if( osGetFileInformationByHandleEx(pFile->h, FileStandardInfo, + &info, sizeof(info)) ){ + *pSize = info.EndOfFile.QuadPart; + }else{ + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, + "winFileSize", pFile->zPath); + } + } +#else + { + DWORD upperBits; + DWORD lowerBits; + DWORD lastErrno; + + lowerBits = osGetFileSize(pFile->h, &upperBits); + *pSize = (((sqlite3_int64)upperBits)<<32) + lowerBits; + if( (lowerBits == INVALID_FILE_SIZE) + && ((lastErrno = osGetLastError())!=NO_ERROR) ){ + pFile->lastErrno = lastErrno; + rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, + "winFileSize", pFile->zPath); + } + } +#endif + OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", + pFile->h, pSize, *pSize, sqlite3ErrName(rc))); + return rc; +} + +/* +** LOCKFILE_FAIL_IMMEDIATELY is undefined on some Windows systems. +*/ +#ifndef LOCKFILE_FAIL_IMMEDIATELY +# define LOCKFILE_FAIL_IMMEDIATELY 1 +#endif + +#ifndef LOCKFILE_EXCLUSIVE_LOCK +# define LOCKFILE_EXCLUSIVE_LOCK 2 +#endif + +/* +** Historically, SQLite has used both the LockFile and LockFileEx functions. +** When the LockFile function was used, it was always expected to fail +** immediately if the lock could not be obtained. Also, it always expected to +** obtain an exclusive lock. These flags are used with the LockFileEx function +** and reflect those expectations; therefore, they should not be changed. +*/ +#ifndef SQLITE_LOCKFILE_FLAGS +# define SQLITE_LOCKFILE_FLAGS (LOCKFILE_FAIL_IMMEDIATELY | \ + LOCKFILE_EXCLUSIVE_LOCK) +#endif + +/* +** Currently, SQLite never calls the LockFileEx function without wanting the +** call to fail immediately if the lock cannot be obtained. +*/ +#ifndef SQLITE_LOCKFILEEX_FLAGS +# define SQLITE_LOCKFILEEX_FLAGS (LOCKFILE_FAIL_IMMEDIATELY) +#endif + +/* +** Acquire a reader lock. +** Different API routines are called depending on whether or not this +** is Win9x or WinNT. +*/ +static int winGetReadLock(winFile *pFile){ + int res; + OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); + if( osIsNT() ){ +#if SQLITE_OS_WINCE + /* + ** NOTE: Windows CE is handled differently here due its lack of the Win32 + ** API LockFileEx. + */ + res = winceLockFile(&pFile->h, SHARED_FIRST, 0, 1, 0); +#else + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS, SHARED_FIRST, 0, + SHARED_SIZE, 0); +#endif + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + int lk; + sqlite3_randomness(sizeof(lk), &lk); + pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, + SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); + } +#endif + if( res == 0 ){ + pFile->lastErrno = osGetLastError(); + /* No need to log a failure to lock */ + } + OSTRACE(("READ-LOCK file=%p, result=%d\n", pFile->h, res)); + return res; +} + +/* +** Undo a readlock +*/ +static int winUnlockReadLock(winFile *pFile){ + int res; + DWORD lastErrno; + OSTRACE(("READ-UNLOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); + if( osIsNT() ){ + res = winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + res = winUnlockFile(&pFile->h, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); + } +#endif + if( res==0 && ((lastErrno = osGetLastError())!=ERROR_NOT_LOCKED) ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, + "winUnlockReadLock", pFile->zPath); + } + OSTRACE(("READ-UNLOCK file=%p, result=%d\n", pFile->h, res)); + return res; +} + +/* +** Lock the file with the lock specified by parameter locktype - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. The winUnlock() routine +** erases all locks at once and returns us immediately to locking level 0. +** It is not possible to lower the locking level one step at a time. You +** must go straight to locking level 0. +*/ +static int winLock(sqlite3_file *id, int locktype){ + int rc = SQLITE_OK; /* Return code from subroutines */ + int res = 1; /* Result of a Windows lock call */ + int newLocktype; /* Set pFile->locktype to this value before exiting */ + int gotPendingLock = 0;/* True if we acquired a PENDING lock this time */ + winFile *pFile = (winFile*)id; + DWORD lastErrno = NO_ERROR; + + assert( id!=0 ); + OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); + + /* If there is already a lock of this type or more restrictive on the + ** OsFile, do nothing. Don't use the end_lock: exit path, as + ** sqlite3OsEnterMutex() hasn't been called yet. + */ + if( pFile->locktype>=locktype ){ + OSTRACE(("LOCK-HELD file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + + /* Make sure the locking sequence is correct + */ + assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); + assert( locktype!=PENDING_LOCK ); + assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); + + /* Lock the PENDING_LOCK byte if we need to acquire a PENDING lock or + ** a SHARED lock. If we are acquiring a SHARED lock, the acquisition of + ** the PENDING_LOCK byte is temporary. + */ + newLocktype = pFile->locktype; + if( (pFile->locktype==NO_LOCK) + || ( (locktype==EXCLUSIVE_LOCK) + && (pFile->locktype==RESERVED_LOCK)) + ){ + int cnt = 3; + while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, + PENDING_BYTE, 0, 1, 0))==0 ){ + /* Try 3 times to get the pending lock. This is needed to work + ** around problems caused by indexing and/or anti-virus software on + ** Windows systems. + ** If you are using this code as a model for alternative VFSes, do not + ** copy this retry logic. It is a hack intended for Windows only. + */ + lastErrno = osGetLastError(); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, result=%d\n", + pFile->h, cnt, res)); + if( lastErrno==ERROR_INVALID_HANDLE ){ + pFile->lastErrno = lastErrno; + rc = SQLITE_IOERR_LOCK; + OSTRACE(("LOCK-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(rc))); + return rc; + } + if( cnt ) sqlite3_win32_sleep(1); + } + gotPendingLock = res; + if( !res ){ + lastErrno = osGetLastError(); + } + } + + /* Acquire a shared lock + */ + if( locktype==SHARED_LOCK && res ){ + assert( pFile->locktype==NO_LOCK ); + res = winGetReadLock(pFile); + if( res ){ + newLocktype = SHARED_LOCK; + }else{ + lastErrno = osGetLastError(); + } + } + + /* Acquire a RESERVED lock + */ + if( locktype==RESERVED_LOCK && res ){ + assert( pFile->locktype==SHARED_LOCK ); + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, RESERVED_BYTE, 0, 1, 0); + if( res ){ + newLocktype = RESERVED_LOCK; + }else{ + lastErrno = osGetLastError(); + } + } + + /* Acquire a PENDING lock + */ + if( locktype==EXCLUSIVE_LOCK && res ){ + newLocktype = PENDING_LOCK; + gotPendingLock = 0; + } + + /* Acquire an EXCLUSIVE lock + */ + if( locktype==EXCLUSIVE_LOCK && res ){ + assert( pFile->locktype>=SHARED_LOCK ); + res = winUnlockReadLock(pFile); + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, + SHARED_SIZE, 0); + if( res ){ + newLocktype = EXCLUSIVE_LOCK; + }else{ + lastErrno = osGetLastError(); + winGetReadLock(pFile); + } + } + + /* If we are holding a PENDING lock that ought to be released, then + ** release it now. + */ + if( gotPendingLock && locktype==SHARED_LOCK ){ + winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); + } + + /* Update the state of the lock has held in the file descriptor then + ** return the appropriate result code. + */ + if( res ){ + rc = SQLITE_OK; + }else{ + pFile->lastErrno = lastErrno; + rc = SQLITE_BUSY; + OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", + pFile->h, locktype, newLocktype)); + } + pFile->locktype = (u8)newLocktype; + OSTRACE(("LOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); + return rc; +} + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, return +** non-zero, otherwise zero. +*/ +static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ + int res; + winFile *pFile = (winFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); + + assert( id!=0 ); + if( pFile->locktype>=RESERVED_LOCK ){ + res = 1; + OSTRACE(("TEST-WR-LOCK file=%p, result=%d (local)\n", pFile->h, res)); + }else{ + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); + if( res ){ + winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); + } + res = !res; + OSTRACE(("TEST-WR-LOCK file=%p, result=%d (remote)\n", pFile->h, res)); + } + *pResOut = res; + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + pFile->h, pResOut, *pResOut)); + return SQLITE_OK; +} + +/* +** Lower the locking level on file descriptor id to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** It is not possible for this routine to fail if the second argument +** is NO_LOCK. If the second argument is SHARED_LOCK then this routine +** might return SQLITE_IOERR; +*/ +static int winUnlock(sqlite3_file *id, int locktype){ + int type; + winFile *pFile = (winFile*)id; + int rc = SQLITE_OK; + assert( pFile!=0 ); + assert( locktype<=SHARED_LOCK ); + OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); + type = pFile->locktype; + if( type>=EXCLUSIVE_LOCK ){ + winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + if( locktype==SHARED_LOCK && !winGetReadLock(pFile) ){ + /* This should never happen. We should always be able to + ** reacquire the read lock */ + rc = winLogError(SQLITE_IOERR_UNLOCK, osGetLastError(), + "winUnlock", pFile->zPath); + } + } + if( type>=RESERVED_LOCK ){ + winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); + } + if( locktype==NO_LOCK && type>=SHARED_LOCK ){ + winUnlockReadLock(pFile); + } + if( type>=PENDING_LOCK ){ + winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); + } + pFile->locktype = (u8)locktype; + OSTRACE(("UNLOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); + return rc; +} + +/* +** If *pArg is inititially negative then this is a query. Set *pArg to +** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. +** +** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. +*/ +static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ + if( *pArg<0 ){ + *pArg = (pFile->ctrlFlags & mask)!=0; + }else if( (*pArg)==0 ){ + pFile->ctrlFlags &= ~mask; + }else{ + pFile->ctrlFlags |= mask; + } +} + +/* Forward references to VFS helper methods used for temporary files */ +static int winGetTempname(sqlite3_vfs *, char **); +static int winIsDir(const void *); +static BOOL winIsDriveLetterAndColon(const char *); + +/* +** Control and query of the open file handle. +*/ +static int winFileControl(sqlite3_file *id, int op, void *pArg){ + winFile *pFile = (winFile*)id; + OSTRACE(("FCNTL file=%p, op=%d, pArg=%p\n", pFile->h, op, pArg)); + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = pFile->locktype; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_LAST_ERRNO: { + *(int*)pArg = (int)pFile->lastErrno; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + pFile->szChunk = *(int *)pArg; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_SIZE_HINT: { + if( pFile->szChunk>0 ){ + sqlite3_int64 oldSz; + int rc = winFileSize(id, &oldSz); + if( rc==SQLITE_OK ){ + sqlite3_int64 newSz = *(sqlite3_int64*)pArg; + if( newSz>oldSz ){ + SimulateIOErrorBenign(1); + rc = winTruncate(id, newSz); + SimulateIOErrorBenign(0); + } + } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_PERSIST_WAL: { + winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + winModeBit(pFile, WINFILE_PSOW, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_WIN32_AV_RETRY: { + int *a = (int*)pArg; + if( a[0]>0 ){ + winIoerrRetry = a[0]; + }else{ + a[0] = winIoerrRetry; + } + if( a[1]>0 ){ + winIoerrRetryDelay = a[1]; + }else{ + a[1] = winIoerrRetryDelay; + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } +#ifdef SQLITE_TEST + case SQLITE_FCNTL_WIN32_SET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + HANDLE hOldFile = pFile->h; + pFile->h = *phFile; + *phFile = hOldFile; + OSTRACE(("FCNTL oldFile=%p, newFile=%p, rc=SQLITE_OK\n", + hOldFile, pFile->h)); + return SQLITE_OK; + } +#endif + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = 0; + int rc = winGetTempname(pFile->pVfs, &zTFile); + if( rc==SQLITE_OK ){ + *(char**)pArg = zTFile; + } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; + } +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + int rc = SQLITE_OK; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ + pFile->mmapSizeMax = newLimit; + if( pFile->mmapSize>0 ){ + winUnmapfile(pFile); + rc = winMapfile(pFile, -1); + } + } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; + } +#endif + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); + return SQLITE_NOTFOUND; +} + +/* +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. +** +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. +*/ +static int winSectorSize(sqlite3_file *id){ + (void)id; + return SQLITE_DEFAULT_SECTOR_SIZE; +} + +/* +** Return a vector of device characteristics. +*/ +static int winDeviceCharacteristics(sqlite3_file *id){ + winFile *p = (winFile*)id; + return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | + ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); +} + +/* +** Windows will only let you create file view mappings +** on allocation size granularity boundaries. +** During sqlite3_os_init() we do a GetSystemInfo() +** to get the granularity size. +*/ +static SYSTEM_INFO winSysInfo; + +#ifndef SQLITE_OMIT_WAL + +/* +** Helper functions to obtain and relinquish the global mutex. The +** global mutex is used to protect the winLockInfo objects used by +** this file, all of which may be shared by multiple threads. +** +** Function winShmMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() +** statements. e.g. +** +** winShmEnterMutex() +** assert( winShmMutexHeld() ); +** winShmLeaveMutex() +*/ +static void winShmEnterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +static void winShmLeaveMutex(void){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +#ifndef NDEBUG +static int winShmMutexHeld(void) { + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +#endif + +/* +** Object used to represent a single file opened and mmapped to provide +** shared memory. When multiple threads all reference the same +** log-summary, each thread has its own winFile object, but they all +** point to a single instance of this object. In other words, each +** log-summary is opened only once per process. +** +** winShmMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** pNext +** +** The following fields are read-only after the object is created: +** +** fid +** zFilename +** +** Either winShmNode.mutex must be held or winShmNode.nRef==0 and +** winShmMutexHeld() is true when reading or writing any other field +** in this structure. +** +*/ +struct winShmNode { + sqlite3_mutex *mutex; /* Mutex to access this object */ + char *zFilename; /* Name of the file */ + winFile hFile; /* File handle from winOpen */ + + int szRegion; /* Size of shared-memory regions */ + int nRegion; /* Size of array apRegion */ + struct ShmRegion { + HANDLE hMap; /* File handle from CreateFileMapping */ + void *pMap; + } *aRegion; + DWORD lastErrno; /* The Windows errno from the last I/O error */ + + int nRef; /* Number of winShm objects pointing to this */ + winShm *pFirst; /* All winShm objects pointing to this */ + winShmNode *pNext; /* Next in list of all winShmNode objects */ +#ifdef SQLITE_DEBUG + u8 nextShmId; /* Next available winShm.id value */ +#endif +}; + +/* +** A global array of all winShmNode objects. +** +** The winShmMutexHeld() must be true while reading or writing this list. +*/ +static winShmNode *winShmNodeList = 0; + +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** The following fields are initialized when this object is created and +** are read-only thereafter: +** +** winShm.pShmNode +** winShm.id +** +** All other fields are read/write. The winShm.pShmNode->mutex must be held +** while accessing any read/write fields. +*/ +struct winShm { + winShmNode *pShmNode; /* The underlying winShmNode object */ + winShm *pNext; /* Next winShm with the same winShmNode */ + u8 hasMutex; /* True if holding the winShmNode mutex */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ +#ifdef SQLITE_DEBUG + u8 id; /* Id of this connection with its winShmNode */ +#endif +}; + +/* +** Constants used for locking +*/ +#define WIN_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ +#define WIN_SHM_DMS (WIN_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ + +/* +** Apply advisory locks for all n bytes beginning at ofst. +*/ +#define _SHM_UNLCK 1 +#define _SHM_RDLCK 2 +#define _SHM_WRLCK 3 +static int winShmSystemLock( + winShmNode *pFile, /* Apply locks to this open shared-memory segment */ + int lockType, /* _SHM_UNLCK, _SHM_RDLCK, or _SHM_WRLCK */ + int ofst, /* Offset to first byte to be locked/unlocked */ + int nByte /* Number of bytes to lock or unlock */ +){ + int rc = 0; /* Result code form Lock/UnlockFileEx() */ + + /* Access to the winShmNode object is serialized by the caller */ + assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); + + OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", + pFile->hFile.h, lockType, ofst, nByte)); + + /* Release/Acquire the system-level lock */ + if( lockType==_SHM_UNLCK ){ + rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); + }else{ + /* Initialize the locking parameters */ + DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; + if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); + } + + if( rc!= 0 ){ + rc = SQLITE_OK; + }else{ + pFile->lastErrno = osGetLastError(); + rc = SQLITE_BUSY; + } + + OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", + pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); + + return rc; +} + +/* Forward references to VFS methods */ +static int winOpen(sqlite3_vfs*,const char*,sqlite3_file*,int,int*); +static int winDelete(sqlite3_vfs *,const char*,int); + +/* +** Purge the winShmNodeList list of all entries with winShmNode.nRef==0. +** +** This is not a VFS shared-memory method; it is a utility function called +** by VFS shared-memory methods. +*/ +static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ + winShmNode **pp; + winShmNode *p; + assert( winShmMutexHeld() ); + OSTRACE(("SHM-PURGE pid=%lu, deleteFlag=%d\n", + osGetCurrentProcessId(), deleteFlag)); + pp = &winShmNodeList; + while( (p = *pp)!=0 ){ + if( p->nRef==0 ){ + int i; + if( p->mutex ){ sqlite3_mutex_free(p->mutex); } + for(i=0; inRegion; i++){ + BOOL bRc = osUnmapViewOfFile(p->aRegion[i].pMap); + OSTRACE(("SHM-PURGE-UNMAP pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); + UNUSED_VARIABLE_VALUE(bRc); + bRc = osCloseHandle(p->aRegion[i].hMap); + OSTRACE(("SHM-PURGE-CLOSE pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); + UNUSED_VARIABLE_VALUE(bRc); + } + if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ + SimulateIOErrorBenign(1); + winClose((sqlite3_file *)&p->hFile); + SimulateIOErrorBenign(0); + } + if( deleteFlag ){ + SimulateIOErrorBenign(1); + sqlite3BeginBenignMalloc(); + winDelete(pVfs, p->zFilename, 0); + sqlite3EndBenignMalloc(); + SimulateIOErrorBenign(0); + } + *pp = p->pNext; + sqlite3_free(p->aRegion); + sqlite3_free(p); + }else{ + pp = &p->pNext; + } + } +} + +/* +** Open the shared-memory area associated with database file pDbFd. +** +** When opening a new shared-memory file, if no other instances of that +** file are currently open, in this process or in other processes, then +** the file must be truncated to zero length or have its header cleared. +*/ +static int winOpenSharedMemory(winFile *pDbFd){ + struct winShm *p; /* The connection to be opened */ + struct winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc; /* Result code */ + struct winShmNode *pNew; /* Newly allocated winShmNode */ + int nName; /* Size of zName in bytes */ + + assert( pDbFd->pShm==0 ); /* Not previously opened */ + + /* Allocate space for the new sqlite3_shm object. Also speculatively + ** allocate space for a new winShmNode and filename. + */ + p = sqlite3MallocZero( sizeof(*p) ); + if( p==0 ) return SQLITE_IOERR_NOMEM; + nName = sqlite3Strlen30(pDbFd->zPath); + pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); + if( pNew==0 ){ + sqlite3_free(p); + return SQLITE_IOERR_NOMEM; + } + pNew->zFilename = (char*)&pNew[1]; + sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); + + /* Look to see if there is an existing winShmNode that can be used. + ** If no matching winShmNode currently exists, create a new one. + */ + winShmEnterMutex(); + for(pShmNode = winShmNodeList; pShmNode; pShmNode=pShmNode->pNext){ + /* TBD need to come up with better match here. Perhaps + ** use FILE_ID_BOTH_DIR_INFO Structure. + */ + if( sqlite3StrICmp(pShmNode->zFilename, pNew->zFilename)==0 ) break; + } + if( pShmNode ){ + sqlite3_free(pNew); + }else{ + pShmNode = pNew; + pNew = 0; + ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; + pShmNode->pNext = winShmNodeList; + winShmNodeList = pShmNode; + + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_IOERR_NOMEM; + goto shm_open_err; + } + + rc = winOpen(pDbFd->pVfs, + pShmNode->zFilename, /* Name of the file (UTF-8) */ + (sqlite3_file*)&pShmNode->hFile, /* File handle here */ + SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + 0); + if( SQLITE_OK!=rc ){ + goto shm_open_err; + } + + /* Check to see if another process is holding the dead-man switch. + ** If not, truncate the file to zero length. + */ + if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ + rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + "winOpenShm", pDbFd->zPath); + } + } + if( rc==SQLITE_OK ){ + winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, _SHM_RDLCK, WIN_SHM_DMS, 1); + } + if( rc ) goto shm_open_err; + } + + /* Make the new connection a child of the winShmNode */ + p->pShmNode = pShmNode; +#ifdef SQLITE_DEBUG + p->id = pShmNode->nextShmId++; +#endif + pShmNode->nRef++; + pDbFd->pShm = p; + winShmLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the winShmEnterMutex() mutex and the pointer from the + ** new (struct winShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** mutex. + */ + sqlite3_mutex_enter(pShmNode->mutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->mutex); + return SQLITE_OK; + + /* Jump here on any error */ +shm_open_err: + winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); + winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ + sqlite3_free(p); + sqlite3_free(pNew); + winShmLeaveMutex(); + return rc; +} + +/* +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. +*/ +static int winShmUnmap( + sqlite3_file *fd, /* Database holding shared memory */ + int deleteFlag /* Delete after closing if true */ +){ + winFile *pDbFd; /* Database holding shared-memory */ + winShm *p; /* The connection to be closed */ + winShmNode *pShmNode; /* The underlying shared-memory file */ + winShm **pp; /* For looping over sibling connections */ + + pDbFd = (winFile*)fd; + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; + + /* Remove connection p from the set of connections associated + ** with pShmNode */ + sqlite3_mutex_enter(pShmNode->mutex); + for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + sqlite3_mutex_leave(pShmNode->mutex); + + /* If pShmNode->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + winShmEnterMutex(); + assert( pShmNode->nRef>0 ); + pShmNode->nRef--; + if( pShmNode->nRef==0 ){ + winShmPurge(pDbFd->pVfs, deleteFlag); + } + winShmLeaveMutex(); + + return SQLITE_OK; +} + +/* +** Change the lock state for a shared-memory segment. +*/ +static int winShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ + winShm *p = pDbFd->pShm; /* The shared memory being locked */ + winShm *pX; /* For looping over all siblings */ + winShmNode *pShmNode = p->pShmNode; + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ + + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + + mask = (u16)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); + if( flags & SQLITE_SHM_UNLOCK ){ + u16 allMask = 0; /* Mask of locks held by siblings */ + + /* See if any siblings hold this same lock */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); + allMask |= pX->sharedMask; + } + + /* Unlock the system-level locks */ + if( (mask & allMask)==0 ){ + rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + }else if( flags & SQLITE_SHM_SHARED ){ + u16 allShared = 0; /* Union of locks held by connections other than "p" */ + + /* Find out which shared locks are already held by sibling connections. + ** If any sibling already holds an exclusive lock, go ahead and return + ** SQLITE_BUSY. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + allShared |= pX->sharedMask; + } + + /* Get shared locks at the system level, if necessary */ + if( rc==SQLITE_OK ){ + if( (allShared & mask)==0 ){ + rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + } + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + } + + /* Get the exclusive locks at the system level. Then if successful + ** also mark the local connection as being locked. + */ + if( rc==SQLITE_OK ){ + rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); + if( rc==SQLITE_OK ){ + assert( (p->sharedMask & mask)==0 ); + p->exclMask |= mask; + } + } + } + sqlite3_mutex_leave(pShmNode->mutex); + OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc))); + return rc; +} + +/* +** Implement a memory barrier or memory fence on shared memory. +** +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. +*/ +static void winShmBarrier( + sqlite3_file *fd /* Database holding the shared memory */ +){ + UNUSED_PARAMETER(fd); + /* MemoryBarrier(); // does not work -- do not know why not */ + winShmEnterMutex(); + winShmLeaveMutex(); +} + +/* +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the isWrite parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** isWrite is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. +*/ +static int winShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int isWrite, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + winFile *pDbFd = (winFile*)fd; + winShm *p = pDbFd->pShm; + winShmNode *pShmNode; + int rc = SQLITE_OK; + + if( !p ){ + rc = winOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + p = pDbFd->pShm; + } + pShmNode = p->pShmNode; + + sqlite3_mutex_enter(pShmNode->mutex); + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + + if( pShmNode->nRegion<=iRegion ){ + struct ShmRegion *apNew; /* New aRegion[] array */ + int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ + sqlite3_int64 sz; /* Current size of wal-index file */ + + pShmNode->szRegion = szRegion; + + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), + "winShmMap1", pDbFd->zPath); + goto shmpage_out; + } + + if( szhFile, nByte); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), + "winShmMap2", pDbFd->zPath); + goto shmpage_out; + } + } + + /* Map the requested memory region into this processes address space. */ + apNew = (struct ShmRegion *)sqlite3_realloc( + pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM; + goto shmpage_out; + } + pShmNode->aRegion = apNew; + + while( pShmNode->nRegion<=iRegion ){ + HANDLE hMap = NULL; /* file-mapping handle */ + void *pMap = 0; /* Mapped memory region */ + +#if SQLITE_OS_WINRT + hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, + NULL, PAGE_READWRITE, nByte, NULL + ); +#elif defined(SQLITE_WIN32_HAS_WIDE) + hMap = osCreateFileMappingW(pShmNode->hFile.h, + NULL, PAGE_READWRITE, 0, nByte, NULL + ); +#elif defined(SQLITE_WIN32_HAS_ANSI) + hMap = osCreateFileMappingA(pShmNode->hFile.h, + NULL, PAGE_READWRITE, 0, nByte, NULL + ); +#endif + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, nByte, + hMap ? "ok" : "failed")); + if( hMap ){ + int iOffset = pShmNode->nRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; +#if SQLITE_OS_WINRT + pMap = osMapViewOfFileFromApp(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + iOffset - iOffsetShift, szRegion + iOffsetShift + ); +#else + pMap = osMapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + 0, iOffset - iOffsetShift, szRegion + iOffsetShift + ); +#endif + OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, iOffset, + szRegion, pMap ? "ok" : "failed")); + } + if( !pMap ){ + pShmNode->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_SHMMAP, pShmNode->lastErrno, + "winShmMap3", pDbFd->zPath); + if( hMap ) osCloseHandle(hMap); + goto shmpage_out; + } + + pShmNode->aRegion[pShmNode->nRegion].pMap = pMap; + pShmNode->aRegion[pShmNode->nRegion].hMap = hMap; + pShmNode->nRegion++; + } + } + +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + int iOffset = iRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; + char *p = (char *)pShmNode->aRegion[iRegion].pMap; + *pp = (void *)&p[iOffsetShift]; + }else{ + *pp = 0; + } + sqlite3_mutex_leave(pShmNode->mutex); + return rc; +} + +#else +# define winShmMap 0 +# define winShmLock 0 +# define winShmBarrier 0 +# define winShmUnmap 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ + +/* +** Cleans up the mapped region of the specified file, if any. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +static int winUnmapfile(winFile *pFile){ + assert( pFile!=0 ); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " + "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", + osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, + pFile->mmapSize, pFile->mmapSizeActual, pFile->mmapSizeMax)); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), pFile, + pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmapfile1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFile, pFile->hMap)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmapfile2", pFile->zPath); + } + pFile->hMap = NULL; + } + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile)); + return SQLITE_OK; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, size=%lld\n", + osGetCurrentProcessId(), pFd, nByte)); + + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_IOERR_FSTAT; + } + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + + winUnmapfile(pFd); + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#if SQLITE_OS_WINRT + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#endif + if( pFd->hMap==NULL ){ + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile1", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-CREATE pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return SQLITE_OK; + } + assert( (nMap % winSysInfo.dwPageSize)==0 ); + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, (SIZE_T)nMap); +#else + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); +#endif + if( pNew==NULL ){ + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile2", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-MAP pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return SQLITE_OK; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapSizeActual = nMap; + } + + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling winUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ +#endif + *pp = 0; + + OSTRACE(("FETCH pid=%lu, pFile=%p, offset=%lld, amount=%d, pp=%p\n", + osGetCurrentProcessId(), fd, iOff, nAmt, pp)); + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ){ + OSTRACE(("FETCH pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return rc; + } + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + + OSTRACE(("FETCH pid=%lu, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd, pp, *pp)); + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to winFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the winFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, offset=%lld, p=%p\n", + osGetCurrentProcessId(), pFd, iOff, p)); + + if( p ){ + pFd->nFetchOut--; + }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can can be omitted - potentially improving + ** performance. */ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#endif + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd)); + return SQLITE_OK; +} + +/* +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ + +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32. +*/ +static const sqlite3_io_methods winIoMethod = { + 3, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winLock, /* xLock */ + winUnlock, /* xUnlock */ + winCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ +}; + +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** +** +** This division contains the implementation of methods on the +** sqlite3_vfs object. +*/ + +#if defined(__CYGWIN__) +/* +** Convert a filename from whatever the underlying operating system +** supports for filenames into UTF-8. Space to hold the result is +** obtained from malloc and must be freed by the calling function. +*/ +static char *winConvertToUtf8Filename(const void *zFilename){ + char *zConverted = 0; + if( osIsNT() ){ + zConverted = winUnicodeToUtf8(zFilename); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + zConverted = sqlite3_win32_mbcs_to_utf8(zFilename); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} +#endif + +/* +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function. +*/ +static void *winConvertFromUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( osIsNT() ){ + zConverted = winUtf8ToUnicode(zFilename); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + zConverted = sqlite3_win32_utf8_to_mbcs(zFilename); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} + +/* +** This function returns non-zero if the specified UTF-8 string buffer +** ends with a directory separator character or one was successfully +** added to it. +*/ +static int winMakeEndInDirSep(int nBuf, char *zBuf){ + if( zBuf ){ + int nLen = sqlite3Strlen30(zBuf); + if( nLen>0 ){ + if( winIsDirSep(zBuf[nLen-1]) ){ + return 1; + }else if( nLen+1mxPathname; nBuf = nMax + 2; + zBuf = sqlite3MallocZero( nBuf ); + if( !zBuf ){ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); + return SQLITE_IOERR_NOMEM; + } + + /* Figure out the effective temporary directory. First, check if one + ** has been explicitly set by the application; otherwise, use the one + ** configured by the operating system. + */ + nDir = nMax - (nPre + 15); + assert( nDir>0 ); + if( sqlite3_temp_directory ){ + int nDirLen = sqlite3Strlen30(sqlite3_temp_directory); + if( nDirLen>0 ){ + if( !winIsDirSep(sqlite3_temp_directory[nDirLen-1]) ){ + nDirLen++; + } + if( nDirLen>nDir ){ + sqlite3_free(zBuf); + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); + return winLogError(SQLITE_ERROR, 0, "winGetTempname1", 0); + } + sqlite3_snprintf(nMax, zBuf, "%s", sqlite3_temp_directory); + } + } +#if defined(__CYGWIN__) + else{ + static const char *azDirs[] = { + 0, /* getenv("SQLITE_TMPDIR") */ + 0, /* getenv("TMPDIR") */ + 0, /* getenv("TMP") */ + 0, /* getenv("TEMP") */ + 0, /* getenv("USERPROFILE") */ + "/var/tmp", + "/usr/tmp", + "/tmp", + ".", + 0 /* List terminator */ + }; + unsigned int i; + const char *zDir = 0; + + if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); + if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); + if( !azDirs[2] ) azDirs[2] = getenv("TMP"); + if( !azDirs[3] ) azDirs[3] = getenv("TEMP"); + if( !azDirs[4] ) azDirs[4] = getenv("USERPROFILE"); + for(i=0; i/etilqs_XXXXXXXXXXXXXXX\0\0" + ** + ** If not, return SQLITE_ERROR. The number 17 is used here in order to + ** account for the space used by the 15 character random suffix and the + ** two trailing NUL characters. The final directory separator character + ** has already added if it was not already present. + */ + nLen = sqlite3Strlen30(zBuf); + if( (nLen + nPre + 17) > nBuf ){ + sqlite3_free(zBuf); + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); + return winLogError(SQLITE_ERROR, 0, "winGetTempname5", 0); + } + + sqlite3_snprintf(nBuf-16-nLen, zBuf+nLen, SQLITE_TEMP_FILE_PREFIX); + + j = sqlite3Strlen30(zBuf); + sqlite3_randomness(15, &zBuf[j]); + for(i=0; i<15; i++, j++){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + zBuf[j+1] = 0; + *pzBuf = zBuf; + + OSTRACE(("TEMP-FILENAME name=%s, rc=SQLITE_OK\n", zBuf)); + return SQLITE_OK; +} + +/* +** Return TRUE if the named file is really a directory. Return false if +** it is something other than a directory, or if there is any kind of memory +** allocation failure. +*/ +static int winIsDir(const void *zConverted){ + DWORD attr; + int rc = 0; + DWORD lastErrno; + + if( osIsNT() ){ + int cnt = 0; + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, + GetFileExInfoStandard, + &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} + if( !rc ){ + return 0; /* Invalid name? */ + } + attr = sAttrData.dwFileAttributes; +#if SQLITE_OS_WINCE==0 + }else{ + attr = osGetFileAttributesA((char*)zConverted); +#endif + } + return (attr!=INVALID_FILE_ATTRIBUTES) && (attr&FILE_ATTRIBUTE_DIRECTORY); +} + +/* +** Open a file. +*/ +static int winOpen( + sqlite3_vfs *pVfs, /* Used to get maximum path name length */ + const char *zName, /* Name of the file (UTF-8) */ + sqlite3_file *id, /* Write the SQLite file handle here */ + int flags, /* Open mode flags */ + int *pOutFlags /* Status return flags */ +){ + HANDLE h; + DWORD lastErrno = 0; + DWORD dwDesiredAccess; + DWORD dwShareMode; + DWORD dwCreationDisposition; + DWORD dwFlagsAndAttributes = 0; +#if SQLITE_OS_WINCE + int isTemp = 0; +#endif + winFile *pFile = (winFile*)id; + void *zConverted; /* Filename in OS encoding */ + const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ + int cnt = 0; + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char *zTmpname = 0; /* For temporary filename, if necessary. */ + + int rc = SQLITE_OK; /* Function Return Code */ +#if !defined(NDEBUG) || SQLITE_OS_WINCE + int eType = flags&0xFFFFFF00; /* Type of file to open */ +#endif + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); + int isReadonly = (flags & SQLITE_OPEN_READONLY); + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); + +#ifndef NDEBUG + int isOpenJournal = (isCreate && ( + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); +#endif + + OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", + zUtf8Name, id, flags, pOutFlags)); + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, WAL file and master journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); + + assert( pFile!=0 ); + memset(pFile, 0, sizeof(winFile)); + pFile->h = INVALID_HANDLE_VALUE; + +#if SQLITE_OS_WINRT + if( !zUtf8Name && !sqlite3_temp_directory ){ + sqlite3_log(SQLITE_ERROR, + "sqlite3_temp_directory variable should be set for WinRT"); + } +#endif + + /* If the second argument to this function is NULL, generate a + ** temporary file name to use + */ + if( !zUtf8Name ){ + assert( isDelete && !isOpenJournal ); + rc = winGetTempname(pVfs, &zTmpname); + if( rc!=SQLITE_OK ){ + OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); + return rc; + } + zUtf8Name = zTmpname; + } + + /* Database filenames are double-zero terminated if they are not + ** URIs with parameters. Hence, they can always be passed into + ** sqlite3_uri_parameter(). + */ + assert( (eType!=SQLITE_OPEN_MAIN_DB) || (flags & SQLITE_OPEN_URI) || + zUtf8Name[sqlite3Strlen30(zUtf8Name)+1]==0 ); + + /* Convert the filename to the system encoding. */ + zConverted = winConvertFromUtf8Filename(zUtf8Name); + if( zConverted==0 ){ + sqlite3_free(zTmpname); + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); + return SQLITE_IOERR_NOMEM; + } + + if( winIsDir(zConverted) ){ + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8Name)); + return SQLITE_CANTOPEN_ISDIR; + } + + if( isReadWrite ){ + dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + }else{ + dwDesiredAccess = GENERIC_READ; + } + + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is + ** created. SQLite doesn't use it to indicate "exclusive access" + ** as it is usually understood. + */ + if( isExclusive ){ + /* Creates a new file, only if it does not already exist. */ + /* If the file exists, it fails. */ + dwCreationDisposition = CREATE_NEW; + }else if( isCreate ){ + /* Open existing file, or create if it doesn't exist */ + dwCreationDisposition = OPEN_ALWAYS; + }else{ + /* Opens a file, only if it exists. */ + dwCreationDisposition = OPEN_EXISTING; + } + + dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; + + if( isDelete ){ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes = FILE_ATTRIBUTE_HIDDEN; + isTemp = 1; +#else + dwFlagsAndAttributes = FILE_ATTRIBUTE_TEMPORARY + | FILE_ATTRIBUTE_HIDDEN + | FILE_FLAG_DELETE_ON_CLOSE; +#endif + }else{ + dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + } + /* Reports from the internet are that performance is always + ** better if FILE_FLAG_RANDOM_ACCESS is used. Ticket #2699. */ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; +#endif + + if( osIsNT() ){ +#if SQLITE_OS_WINRT + CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; + extendedParameters.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS); + extendedParameters.dwFileAttributes = + dwFlagsAndAttributes & FILE_ATTRIBUTE_MASK; + extendedParameters.dwFileFlags = dwFlagsAndAttributes & FILE_FLAG_MASK; + extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; + extendedParameters.lpSecurityAttributes = NULL; + extendedParameters.hTemplateFile = NULL; + while( (h = osCreateFile2((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, + dwCreationDisposition, + &extendedParameters))==INVALID_HANDLE_VALUE && + winRetryIoerr(&cnt, &lastErrno) ){ + /* Noop */ + } +#else + while( (h = osCreateFileW((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL))==INVALID_HANDLE_VALUE && + winRetryIoerr(&cnt, &lastErrno) ){ + /* Noop */ + } +#endif + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + while( (h = osCreateFileA((LPCSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL))==INVALID_HANDLE_VALUE && + winRetryIoerr(&cnt, &lastErrno) ){ + /* Noop */ + } + } +#endif + winLogIoerr(cnt); + + OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, + dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + + if( h==INVALID_HANDLE_VALUE ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_CANTOPEN, pFile->lastErrno, "winOpen", zUtf8Name); + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + if( isReadWrite && !isExclusive ){ + return winOpen(pVfs, zName, id, + ((flags|SQLITE_OPEN_READONLY) & + ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), + pOutFlags); + }else{ + return SQLITE_CANTOPEN_BKPT; + } + } + + if( pOutFlags ){ + if( isReadWrite ){ + *pOutFlags = SQLITE_OPEN_READWRITE; + }else{ + *pOutFlags = SQLITE_OPEN_READONLY; + } + } + + OSTRACE(("OPEN file=%p, name=%s, access=%lx, pOutFlags=%p, *pOutFlags=%d, " + "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? + *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + +#if SQLITE_OS_WINCE + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK + ){ + osCloseHandle(h); + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; + } + if( isTemp ){ + pFile->zDeleteOnClose = zConverted; + }else +#endif + { + sqlite3_free(zConverted); + } + + sqlite3_free(zTmpname); + pFile->pMethod = &winIoMethod; + pFile->pVfs = pVfs; + pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } + if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + pFile->ctrlFlags |= WINFILE_PSOW; + } + pFile->lastErrno = NO_ERROR; + pFile->zPath = zName; +#if SQLITE_MAX_MMAP_SIZE>0 + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif + + OpenCounter(+1); + return rc; +} + +/* +** Delete the named file. +** +** Note that Windows does not allow a file to be deleted if some other +** process has it open. Sometimes a virus scanner or indexing program +** will open a journal file shortly after it is created in order to do +** whatever it does. While this other process is holding the +** file open, we will be unable to delete it. To work around this +** problem, we delay 100 milliseconds and try to delete again. Up +** to MX_DELETION_ATTEMPTs deletion attempts are run before giving +** up and returning an error. +*/ +static int winDelete( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to delete */ + int syncDir /* Not used on win32 */ +){ + int cnt = 0; + int rc; + DWORD attr; + DWORD lastErrno = 0; + void *zConverted; + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(syncDir); + + SimulateIOError(return SQLITE_IOERR_DELETE); + OSTRACE(("DELETE name=%s, syncDir=%d\n", zFilename, syncDir)); + + zConverted = winConvertFromUtf8Filename(zFilename); + if( zConverted==0 ){ + OSTRACE(("DELETE name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); + return SQLITE_IOERR_NOMEM; + } + if( osIsNT() ){ + do { +#if SQLITE_OS_WINRT + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + if ( osGetFileAttributesExW(zConverted, GetFileExInfoStandard, + &sAttrData) ){ + attr = sAttrData.dwFileAttributes; + }else{ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } +#else + attr = osGetFileAttributesW(zConverted); +#endif + if ( attr==INVALID_FILE_ATTRIBUTES ){ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileW(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !winRetryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + do { + attr = osGetFileAttributesA(zConverted); + if ( attr==INVALID_FILE_ATTRIBUTES ){ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileA(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !winRetryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); + } +#endif + if( rc && rc!=SQLITE_IOERR_DELETE_NOENT ){ + rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); + }else{ + winLogIoerr(cnt); + } + sqlite3_free(zConverted); + OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); + return rc; +} + +/* +** Check the existence and status of a file. +*/ +static int winAccess( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pResOut /* OUT: Result */ +){ + DWORD attr; + int rc = 0; + DWORD lastErrno = 0; + void *zConverted; + UNUSED_PARAMETER(pVfs); + + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", + zFilename, flags, pResOut)); + + zConverted = winConvertFromUtf8Filename(zFilename); + if( zConverted==0 ){ + OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); + return SQLITE_IOERR_NOMEM; + } + if( osIsNT() ){ + int cnt = 0; + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, + GetFileExInfoStandard, + &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} + if( rc ){ + /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file + ** as if it does not exist. + */ + if( flags==SQLITE_ACCESS_EXISTS + && sAttrData.nFileSizeHigh==0 + && sAttrData.nFileSizeLow==0 ){ + attr = INVALID_FILE_ATTRIBUTES; + }else{ + attr = sAttrData.dwFileAttributes; + } + }else{ + winLogIoerr(cnt); + if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ + sqlite3_free(zConverted); + return winLogError(SQLITE_IOERR_ACCESS, lastErrno, "winAccess", + zFilename); + }else{ + attr = INVALID_FILE_ATTRIBUTES; + } + } + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + attr = osGetFileAttributesA((char*)zConverted); + } +#endif + sqlite3_free(zConverted); + switch( flags ){ + case SQLITE_ACCESS_READ: + case SQLITE_ACCESS_EXISTS: + rc = attr!=INVALID_FILE_ATTRIBUTES; + break; + case SQLITE_ACCESS_READWRITE: + rc = attr!=INVALID_FILE_ATTRIBUTES && + (attr & FILE_ATTRIBUTE_READONLY)==0; + break; + default: + assert(!"Invalid flags argument"); + } + *pResOut = rc; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); + return SQLITE_OK; +} + +/* +** Returns non-zero if the specified path name starts with a drive letter +** followed by a colon character. +*/ +static BOOL winIsDriveLetterAndColon( + const char *zPathname +){ + return ( sqlite3Isalpha(zPathname[0]) && zPathname[1]==':' ); +} + +/* +** Returns non-zero if the specified path name should be used verbatim. If +** non-zero is returned from this function, the calling function must simply +** use the provided path name verbatim -OR- resolve it into a full path name +** using the GetFullPathName Win32 API function (if available). +*/ +static BOOL winIsVerbatimPathname( + const char *zPathname +){ + /* + ** If the path name starts with a forward slash or a backslash, it is either + ** a legal UNC name, a volume relative path, or an absolute path name in the + ** "Unix" format on Windows. There is no easy way to differentiate between + ** the final two cases; therefore, we return the safer return value of TRUE + ** so that callers of this function will simply use it verbatim. + */ + if ( winIsDirSep(zPathname[0]) ){ + return TRUE; + } + + /* + ** If the path name starts with a letter and a colon it is either a volume + ** relative path or an absolute path. Callers of this function must not + ** attempt to treat it as a relative path name (i.e. they should simply use + ** it verbatim). + */ + if ( winIsDriveLetterAndColon(zPathname) ){ + return TRUE; + } + + /* + ** If we get to this point, the path name should almost certainly be a purely + ** relative one (i.e. not a UNC name, not absolute, and not volume relative). + */ + return FALSE; +} + +/* +** Turn a relative pathname into a full pathname. Write the full +** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname +** bytes in size. +*/ +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + +#if defined(__CYGWIN__) + SimulateIOError( return SQLITE_ERROR ); + UNUSED_PARAMETER(nFull); + assert( nFull>=pVfs->mxPathname ); + if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ + /* + ** NOTE: We are dealing with a relative path name and the data + ** directory has been set. Therefore, use it as the basis + ** for converting the relative path name to an absolute + ** one by prepending the data directory and a slash. + */ + char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); + if( !zOut ){ + return SQLITE_IOERR_NOMEM; + } + if( cygwin_conv_path( + (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | + CCP_RELATIVE, zRelative, zOut, pVfs->mxPathname+1)<0 ){ + sqlite3_free(zOut); + return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, + "winFullPathname1", zRelative); + }else{ + char *zUtf8 = winConvertToUtf8Filename(zOut); + if( !zUtf8 ){ + sqlite3_free(zOut); + return SQLITE_IOERR_NOMEM; + } + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", + sqlite3_data_directory, winGetDirSep(), zUtf8); + sqlite3_free(zUtf8); + sqlite3_free(zOut); + } + }else{ + char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); + if( !zOut ){ + return SQLITE_IOERR_NOMEM; + } + if( cygwin_conv_path( + (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), + zRelative, zOut, pVfs->mxPathname+1)<0 ){ + sqlite3_free(zOut); + return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, + "winFullPathname2", zRelative); + }else{ + char *zUtf8 = winConvertToUtf8Filename(zOut); + if( !zUtf8 ){ + sqlite3_free(zOut); + return SQLITE_IOERR_NOMEM; + } + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); + sqlite3_free(zUtf8); + sqlite3_free(zOut); + } + } + return SQLITE_OK; +#endif + +#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && !defined(__CYGWIN__) + SimulateIOError( return SQLITE_ERROR ); + /* WinCE has no concept of a relative pathname, or so I am told. */ + /* WinRT has no way to convert a relative path to an absolute one. */ + if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ + /* + ** NOTE: We are dealing with a relative path name and the data + ** directory has been set. Therefore, use it as the basis + ** for converting the relative path name to an absolute + ** one by prepending the data directory and a backslash. + */ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", + sqlite3_data_directory, winGetDirSep(), zRelative); + }else{ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zRelative); + } + return SQLITE_OK; +#endif + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + DWORD nByte; + void *zConverted; + char *zOut; + + /* If this path name begins with "/X:", where "X" is any alphabetic + ** character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ + zRelative++; + } + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); + if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ + /* + ** NOTE: We are dealing with a relative path name and the data + ** directory has been set. Therefore, use it as the basis + ** for converting the relative path name to an absolute + ** one by prepending the data directory and a backslash. + */ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", + sqlite3_data_directory, winGetDirSep(), zRelative); + return SQLITE_OK; + } + zConverted = winConvertFromUtf8Filename(zRelative); + if( zConverted==0 ){ + return SQLITE_IOERR_NOMEM; + } + if( osIsNT() ){ + LPWSTR zTemp; + nByte = osGetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname1", zRelative); + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + sqlite3_free(zConverted); + return SQLITE_IOERR_NOMEM; + } + nByte = osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname2", zRelative); + } + sqlite3_free(zConverted); + zOut = winUnicodeToUtf8(zTemp); + sqlite3_free(zTemp); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + char *zTemp; + nByte = osGetFullPathNameA((char*)zConverted, 0, 0, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname3", zRelative); + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + sqlite3_free(zConverted); + return SQLITE_IOERR_NOMEM; + } + nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname4", zRelative); + } + sqlite3_free(zConverted); + zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + sqlite3_free(zTemp); + } +#endif + if( zOut ){ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut); + sqlite3_free(zOut); + return SQLITE_OK; + }else{ + return SQLITE_IOERR_NOMEM; + } +#endif +} + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. +*/ +static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ + HANDLE h; +#if defined(__CYGWIN__) + int nFull = pVfs->mxPathname+1; + char *zFull = sqlite3MallocZero( nFull ); + void *zConverted = 0; + if( zFull==0 ){ + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + if( winFullPathname(pVfs, zFilename, nFull, zFull)!=SQLITE_OK ){ + sqlite3_free(zFull); + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + zConverted = winConvertFromUtf8Filename(zFull); + sqlite3_free(zFull); +#else + void *zConverted = winConvertFromUtf8Filename(zFilename); + UNUSED_PARAMETER(pVfs); +#endif + if( zConverted==0 ){ + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + if( osIsNT() ){ +#if SQLITE_OS_WINRT + h = osLoadPackagedLibrary((LPCWSTR)zConverted, 0); +#else + h = osLoadLibraryW((LPCWSTR)zConverted); +#endif + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + h = osLoadLibraryA((char*)zConverted); + } +#endif + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)h)); + sqlite3_free(zConverted); + return (void*)h; +} +static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ + UNUSED_PARAMETER(pVfs); + winGetLastErrorMsg(osGetLastError(), nBuf, zBufOut); +} +static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ + FARPROC proc; + UNUSED_PARAMETER(pVfs); + proc = osGetProcAddressA((HANDLE)pH, zSym); + OSTRACE(("DLSYM handle=%p, symbol=%s, address=%p\n", + (void*)pH, zSym, (void*)proc)); + return (void(*)(void))proc; +} +static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ + UNUSED_PARAMETER(pVfs); + osFreeLibrary((HANDLE)pHandle); + OSTRACE(("DLCLOSE handle=%p\n", (void*)pHandle)); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define winDlOpen 0 + #define winDlError 0 + #define winDlSym 0 + #define winDlClose 0 +#endif + + +/* +** Write up to nBuf bytes of randomness into zBuf. +*/ +static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + int n = 0; + UNUSED_PARAMETER(pVfs); +#if defined(SQLITE_TEST) + n = nBuf; + memset(zBuf, 0, nBuf); +#else + if( sizeof(SYSTEMTIME)<=nBuf-n ){ + SYSTEMTIME x; + osGetSystemTime(&x); + memcpy(&zBuf[n], &x, sizeof(x)); + n += sizeof(x); + } + if( sizeof(DWORD)<=nBuf-n ){ + DWORD pid = osGetCurrentProcessId(); + memcpy(&zBuf[n], &pid, sizeof(pid)); + n += sizeof(pid); + } +#if SQLITE_OS_WINRT + if( sizeof(ULONGLONG)<=nBuf-n ){ + ULONGLONG cnt = osGetTickCount64(); + memcpy(&zBuf[n], &cnt, sizeof(cnt)); + n += sizeof(cnt); + } +#else + if( sizeof(DWORD)<=nBuf-n ){ + DWORD cnt = osGetTickCount(); + memcpy(&zBuf[n], &cnt, sizeof(cnt)); + n += sizeof(cnt); + } +#endif + if( sizeof(LARGE_INTEGER)<=nBuf-n ){ + LARGE_INTEGER i; + osQueryPerformanceCounter(&i); + memcpy(&zBuf[n], &i, sizeof(i)); + n += sizeof(i); + } +#endif + return n; +} + + +/* +** Sleep for a little while. Return the amount of time slept. +*/ +static int winSleep(sqlite3_vfs *pVfs, int microsec){ + sqlite3_win32_sleep((microsec+999)/1000); + UNUSED_PARAMETER(pVfs); + return ((microsec+999)/1000)*1000; +} + +/* +** The following variable, if set to a non-zero value, is interpreted as +** the number of seconds since 1970 and is used to set the result of +** sqlite3OsCurrentTime() during testing. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +#endif + +/* +** Find the current time (in Universal Coordinated Time). Write into *piNow +** the current time and date as a Julian Day number times 86_400_000. In +** other words, write into *piNow the number of milliseconds since the Julian +** epoch of noon in Greenwich on November 24, 4714 B.C according to the +** proleptic Gregorian calendar. +** +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** cannot be found. +*/ +static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ + /* FILETIME structure is a 64-bit value representing the number of + 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). + */ + FILETIME ft; + static const sqlite3_int64 winFiletimeEpoch = 23058135*(sqlite3_int64)8640000; +#ifdef SQLITE_TEST + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; +#endif + /* 2^32 - to avoid use of LL and warnings in gcc */ + static const sqlite3_int64 max32BitValue = + (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + + (sqlite3_int64)294967296; + +#if SQLITE_OS_WINCE + SYSTEMTIME time; + osGetSystemTime(&time); + /* if SystemTimeToFileTime() fails, it returns zero. */ + if (!osSystemTimeToFileTime(&time,&ft)){ + return SQLITE_ERROR; + } +#else + osGetSystemTimeAsFileTime( &ft ); +#endif + + *piNow = winFiletimeEpoch + + ((((sqlite3_int64)ft.dwHighDateTime)*max32BitValue) + + (sqlite3_int64)ft.dwLowDateTime)/(sqlite3_int64)10000; + +#ifdef SQLITE_TEST + if( sqlite3_current_time ){ + *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; + } +#endif + UNUSED_PARAMETER(pVfs); + return SQLITE_OK; +} + +/* +** Find the current time (in Universal Coordinated Time). Write the +** current time and date as a Julian Day number into *prNow and +** return 0. Return 1 if the time and date cannot be found. +*/ +static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ + int rc; + sqlite3_int64 i; + rc = winCurrentTimeInt64(pVfs, &i); + if( !rc ){ + *prNow = i/86400000.0; + } + return rc; +} + +/* +** The idea is that this function works like a combination of +** GetLastError() and FormatMessage() on Windows (or errno and +** strerror_r() on Unix). After an error is returned by an OS +** function, SQLite calls this function with zBuf pointing to +** a buffer of nBuf bytes. The OS layer should populate the +** buffer with a nul-terminated UTF-8 encoded error message +** describing the last IO error to have occurred within the calling +** thread. +** +** If the error message is too large for the supplied buffer, +** it should be truncated. The return value of xGetLastError +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). If non-zero is returned, +** then it is not necessary to include the nul-terminator character +** in the output buffer. +** +** Not supplying an error message will have no adverse effect +** on SQLite. It is fine to have an implementation that never +** returns an error message: +** +** int xGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ +** assert(zBuf[0]=='\0'); +** return 0; +** } +** +** However if an error message is supplied, it will be incorporated +** by sqlite into the error message available to the user using +** sqlite3_errmsg(), possibly making IO errors easier to debug. +*/ +static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + UNUSED_PARAMETER(pVfs); + return winGetLastErrorMsg(osGetLastError(), nBuf, zBuf); +} + +/* +** Initialize and deinitialize the operating system interface. +*/ +SQLITE_API int sqlite3_os_init(void){ + static sqlite3_vfs winVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32", /* zName */ + 0, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath", /* zName */ + 0, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif + + /* Double-check that the aSyscall[] array has been constructed + ** correctly. See ticket [bb3a86e890c8e96ab] */ + assert( ArraySize(aSyscall)==77 ); + + /* get memory map allocation granularity */ + memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); +#if SQLITE_OS_WINRT + osGetNativeSystemInfo(&winSysInfo); +#else + osGetSystemInfo(&winSysInfo); +#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); + + sqlite3_vfs_register(&winVfs, 1); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathVfs, 0); +#endif + + return SQLITE_OK; +} + +SQLITE_API int sqlite3_os_end(void){ +#if SQLITE_OS_WINRT + if( sleepObj!=NULL ){ + osCloseHandle(sleepObj); + sleepObj = NULL; + } +#endif + return SQLITE_OK; +} + +#endif /* SQLITE_OS_WIN */ + +/************** End of os_win.c **********************************************/ +/************** Begin file bitvec.c ******************************************/ +/* +** 2008 February 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements an object that represents a fixed-length +** bitmap. Bits are numbered starting with 1. +** +** A bitmap is used to record which pages of a database file have been +** journalled during a transaction, or which pages have the "dont-write" +** property. Usually only a few pages are meet either condition. +** So the bitmap is usually sparse and has low cardinality. +** But sometimes (for example when during a DROP of a large table) most +** or all of the pages in a database can get journalled. In those cases, +** the bitmap becomes dense with high cardinality. The algorithm needs +** to handle both cases well. +** +** The size of the bitmap is fixed when the object is created. +** +** All bits are clear when the bitmap is created. Individual bits +** may be set or cleared one at a time. +** +** Test operations are about 100 times more common that set operations. +** Clear operations are exceedingly rare. There are usually between +** 5 and 500 set operations per Bitvec object, though the number of sets can +** sometimes grow into tens of thousands or larger. The size of the +** Bitvec object is the number of pages in the database file at the +** start of a transaction, and is thus usually less than a few thousand, +** but can be as large as 2 billion for a really big database. +*/ + +/* Size of the Bitvec structure in bytes. */ +#define BITVEC_SZ 512 + +/* Round the union size down to the nearest pointer boundary, since that's how +** it will be aligned within the Bitvec struct. */ +#define BITVEC_USIZE (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) + +/* Type of the array "element" for the bitmap representation. +** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. +** Setting this to the "natural word" size of your CPU may improve +** performance. */ +#define BITVEC_TELEM u8 +/* Size, in bits, of the bitmap element. */ +#define BITVEC_SZELEM 8 +/* Number of elements in a bitmap array. */ +#define BITVEC_NELEM (BITVEC_USIZE/sizeof(BITVEC_TELEM)) +/* Number of bits in the bitmap array. */ +#define BITVEC_NBIT (BITVEC_NELEM*BITVEC_SZELEM) + +/* Number of u32 values in hash table. */ +#define BITVEC_NINT (BITVEC_USIZE/sizeof(u32)) +/* Maximum number of entries in hash table before +** sub-dividing and re-hashing. */ +#define BITVEC_MXHASH (BITVEC_NINT/2) +/* Hashing function for the aHash representation. +** Empirical testing showed that the *37 multiplier +** (an arbitrary prime)in the hash function provided +** no fewer collisions than the no-op *1. */ +#define BITVEC_HASH(X) (((X)*1)%BITVEC_NINT) + +#define BITVEC_NPTR (BITVEC_USIZE/sizeof(Bitvec *)) + + +/* +** A bitmap is an instance of the following structure. +** +** This bitmap records the existence of zero or more bits +** with values between 1 and iSize, inclusive. +** +** There are three possible representations of the bitmap. +** If iSize<=BITVEC_NBIT, then Bitvec.u.aBitmap[] is a straight +** bitmap. The least significant bit is bit 1. +** +** If iSize>BITVEC_NBIT and iDivisor==0 then Bitvec.u.aHash[] is +** a hash table that will hold up to BITVEC_MXHASH distinct values. +** +** Otherwise, the value i is redirected into one of BITVEC_NPTR +** sub-bitmaps pointed to by Bitvec.u.apSub[]. Each subbitmap +** handles up to iDivisor separate values of i. apSub[0] holds +** values between 1 and iDivisor. apSub[1] holds values between +** iDivisor+1 and 2*iDivisor. apSub[N] holds values between +** N*iDivisor+1 and (N+1)*iDivisor. Each subbitmap is normalized +** to hold deal with values between 1 and iDivisor. +*/ +struct Bitvec { + u32 iSize; /* Maximum bit index. Max iSize is 4,294,967,296. */ + u32 nSet; /* Number of bits that are set - only valid for aHash + ** element. Max is BITVEC_NINT. For BITVEC_SZ of 512, + ** this would be 125. */ + u32 iDivisor; /* Number of bits handled by each apSub[] entry. */ + /* Should >=0 for apSub element. */ + /* Max iDivisor is max(u32) / BITVEC_NPTR + 1. */ + /* For a BITVEC_SZ of 512, this would be 34,359,739. */ + union { + BITVEC_TELEM aBitmap[BITVEC_NELEM]; /* Bitmap representation */ + u32 aHash[BITVEC_NINT]; /* Hash table representation */ + Bitvec *apSub[BITVEC_NPTR]; /* Recursive representation */ + } u; +}; + +/* +** Create a new bitmap object able to handle bits between 0 and iSize, +** inclusive. Return a pointer to the new object. Return NULL if +** malloc fails. +*/ +SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32 iSize){ + Bitvec *p; + assert( sizeof(*p)==BITVEC_SZ ); + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + p->iSize = iSize; + } + return p; +} + +/* +** Check to see if the i-th bit is set. Return true or false. +** If p is NULL (if the bitmap has not been created) or if +** i is out of range, then return false. +*/ +SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ + if( p==0 ) return 0; + if( i>p->iSize || i==0 ) return 0; + i--; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return 0; + } + } + if( p->iSize<=BITVEC_NBIT ){ + return (p->u.aBitmap[i/BITVEC_SZELEM] & (1<<(i&(BITVEC_SZELEM-1))))!=0; + } else{ + u32 h = BITVEC_HASH(i++); + while( p->u.aHash[h] ){ + if( p->u.aHash[h]==i ) return 1; + h = (h+1) % BITVEC_NINT; + } + return 0; + } +} + +/* +** Set the i-th bit. Return 0 on success and an error code if +** anything goes wrong. +** +** This routine might cause sub-bitmaps to be allocated. Failing +** to get the memory needed to hold the sub-bitmap is the only +** that can go wrong with an insert, assuming p and i are valid. +** +** The calling function must ensure that p is a valid Bitvec object +** and that the value for "i" is within range of the Bitvec object. +** Otherwise the behavior is undefined. +*/ +SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ + u32 h; + if( p==0 ) return SQLITE_OK; + assert( i>0 ); + assert( i<=p->iSize ); + i--; + while((p->iSize > BITVEC_NBIT) && p->iDivisor) { + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + if( p->u.apSub[bin]==0 ){ + p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); + if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM; + } + p = p->u.apSub[bin]; + } + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] |= 1 << (i&(BITVEC_SZELEM-1)); + return SQLITE_OK; + } + h = BITVEC_HASH(i++); + /* if there wasn't a hash collision, and this doesn't */ + /* completely fill the hash, then just add it without */ + /* worring about sub-dividing and re-hashing. */ + if( !p->u.aHash[h] ){ + if (p->nSet<(BITVEC_NINT-1)) { + goto bitvec_set_end; + } else { + goto bitvec_set_rehash; + } + } + /* there was a collision, check to see if it's already */ + /* in hash, if not, try to find a spot for it */ + do { + if( p->u.aHash[h]==i ) return SQLITE_OK; + h++; + if( h>=BITVEC_NINT ) h = 0; + } while( p->u.aHash[h] ); + /* we didn't find it in the hash. h points to the first */ + /* available free spot. check to see if this is going to */ + /* make our hash too "full". */ +bitvec_set_rehash: + if( p->nSet>=BITVEC_MXHASH ){ + unsigned int j; + int rc; + u32 *aiValues = sqlite3StackAllocRaw(0, sizeof(p->u.aHash)); + if( aiValues==0 ){ + return SQLITE_NOMEM; + }else{ + memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); + memset(p->u.apSub, 0, sizeof(p->u.apSub)); + p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR; + rc = sqlite3BitvecSet(p, i); + for(j=0; jnSet++; + p->u.aHash[h] = i; + return SQLITE_OK; +} + +/* +** Clear the i-th bit. +** +** pBuf must be a pointer to at least BITVEC_SZ bytes of temporary storage +** that BitvecClear can use to rebuilt its hash table. +*/ +SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec *p, u32 i, void *pBuf){ + if( p==0 ) return; + assert( i>0 ); + i--; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return; + } + } + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] &= ~(1 << (i&(BITVEC_SZELEM-1))); + }else{ + unsigned int j; + u32 *aiValues = pBuf; + memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); + memset(p->u.aHash, 0, sizeof(p->u.aHash)); + p->nSet = 0; + for(j=0; jnSet++; + while( p->u.aHash[h] ){ + h++; + if( h>=BITVEC_NINT ) h = 0; + } + p->u.aHash[h] = aiValues[j]; + } + } + } +} + +/* +** Destroy a bitmap object. Reclaim all memory used. +*/ +SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec *p){ + if( p==0 ) return; + if( p->iDivisor ){ + unsigned int i; + for(i=0; iu.apSub[i]); + } + } + sqlite3_free(p); +} + +/* +** Return the value of the iSize parameter specified when Bitvec *p +** was created. +*/ +SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ + return p->iSize; +} + +#ifndef SQLITE_OMIT_BUILTIN_TEST +/* +** Let V[] be an array of unsigned characters sufficient to hold +** up to N bits. Let I be an integer between 0 and N. 0<=I>3] |= (1<<(I&7)) +#define CLEARBIT(V,I) V[I>>3] &= ~(1<<(I&7)) +#define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 + +/* +** This routine runs an extensive test of the Bitvec code. +** +** The input is an array of integers that acts as a program +** to test the Bitvec. The integers are opcodes followed +** by 0, 1, or 3 operands, depending on the opcode. Another +** opcode follows immediately after the last operand. +** +** There are 6 opcodes numbered from 0 through 5. 0 is the +** "halt" opcode and causes the test to end. +** +** 0 Halt and return the number of errors +** 1 N S X Set N bits beginning with S and incrementing by X +** 2 N S X Clear N bits beginning with S and incrementing by X +** 3 N Set N randomly chosen bits +** 4 N Clear N randomly chosen bits +** 5 N S X Set N bits from S increment X in array only, not in bitvec +** +** The opcodes 1 through 4 perform set and clear operations are performed +** on both a Bitvec object and on a linear array of bits obtained from malloc. +** Opcode 5 works on the linear array only, not on the Bitvec. +** Opcode 5 is used to deliberately induce a fault in order to +** confirm that error detection works. +** +** At the conclusion of the test the linear array is compared +** against the Bitvec object. If there are any differences, +** an error is returned. If they are the same, zero is returned. +** +** If a memory allocation error occurs, return -1. +*/ +SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ + Bitvec *pBitvec = 0; + unsigned char *pV = 0; + int rc = -1; + int i, nx, pc, op; + void *pTmpSpace; + + /* Allocate the Bitvec to be tested and a linear array of + ** bits to act as the reference */ + pBitvec = sqlite3BitvecCreate( sz ); + pV = sqlite3MallocZero( (sz+7)/8 + 1 ); + pTmpSpace = sqlite3_malloc(BITVEC_SZ); + if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; + + /* NULL pBitvec tests */ + sqlite3BitvecSet(0, 1); + sqlite3BitvecClear(0, 1, pTmpSpace); + + /* Run the program */ + pc = 0; + while( (op = aOp[pc])!=0 ){ + switch( op ){ + case 1: + case 2: + case 5: { + nx = 4; + i = aOp[pc+2] - 1; + aOp[pc+2] += aOp[pc+3]; + break; + } + case 3: + case 4: + default: { + nx = 2; + sqlite3_randomness(sizeof(i), &i); + break; + } + } + if( (--aOp[pc+1]) > 0 ) nx = 0; + pc += nx; + i = (i & 0x7fffffff)%sz; + if( (op & 1)!=0 ){ + SETBIT(pV, (i+1)); + if( op!=5 ){ + if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end; + } + }else{ + CLEARBIT(pV, (i+1)); + sqlite3BitvecClear(pBitvec, i+1, pTmpSpace); + } + } + + /* Test to make sure the linear array exactly matches the + ** Bitvec object. Start with the assumption that they do + ** match (rc==0). Change rc to non-zero if a discrepancy + ** is found. + */ + rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) + + sqlite3BitvecTest(pBitvec, 0) + + (sqlite3BitvecSize(pBitvec) - sz); + for(i=1; i<=sz; i++){ + if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ + rc = i; + break; + } + } + + /* Free allocated structure */ +bitvec_end: + sqlite3_free(pTmpSpace); + sqlite3_free(pV); + sqlite3BitvecDestroy(pBitvec); + return rc; +} +#endif /* SQLITE_OMIT_BUILTIN_TEST */ + +/************** End of bitvec.c **********************************************/ +/************** Begin file pcache.c ******************************************/ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements that page cache. +*/ + +/* +** A complete page cache is an instance of this structure. +*/ +struct PCache { + PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ + PgHdr *pSynced; /* Last synced page in dirty page list */ + int nRef; /* Number of referenced pages */ + int szCache; /* Configured cache size */ + int szPage; /* Size of every page in this cache */ + int szExtra; /* Size of extra space for each page */ + u8 bPurgeable; /* True if pages are on backing store */ + u8 eCreate; /* eCreate value for for xFetch() */ + int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ + void *pStress; /* Argument to xStress */ + sqlite3_pcache *pCache; /* Pluggable cache module */ + PgHdr *pPage1; /* Reference to page 1 */ +}; + +/* +** Some of the assert() macros in this code are too expensive to run +** even during normal debugging. Use them only rarely on long-running +** tests. Enable the expensive asserts using the +** -DSQLITE_ENABLE_EXPENSIVE_ASSERT=1 compile-time option. +*/ +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT +# define expensive_assert(X) assert(X) +#else +# define expensive_assert(X) +#endif + +/********************************** Linked List Management ********************/ + +#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) +/* +** Check that the pCache->pSynced variable is set correctly. If it +** is not, either fail an assert or return zero. Otherwise, return +** non-zero. This is only used in debugging builds, as follows: +** +** expensive_assert( pcacheCheckSynced(pCache) ); +*/ +static int pcacheCheckSynced(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirtyTail; p!=pCache->pSynced; p=p->pDirtyPrev){ + assert( p->nRef || (p->flags&PGHDR_NEED_SYNC) ); + } + return (p==0 || p->nRef || (p->flags&PGHDR_NEED_SYNC)==0); +} +#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */ + +/* +** Remove page pPage from the list of dirty pages. +*/ +static void pcacheRemoveFromDirtyList(PgHdr *pPage){ + PCache *p = pPage->pCache; + + assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); + assert( pPage->pDirtyPrev || pPage==p->pDirty ); + + /* Update the PCache1.pSynced variable if necessary. */ + if( p->pSynced==pPage ){ + PgHdr *pSynced = pPage->pDirtyPrev; + while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ + pSynced = pSynced->pDirtyPrev; + } + p->pSynced = pSynced; + } + + if( pPage->pDirtyNext ){ + pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; + }else{ + assert( pPage==p->pDirtyTail ); + p->pDirtyTail = pPage->pDirtyPrev; + } + if( pPage->pDirtyPrev ){ + pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; + }else{ + assert( pPage==p->pDirty ); + p->pDirty = pPage->pDirtyNext; + if( p->pDirty==0 && p->bPurgeable ){ + assert( p->eCreate==1 ); + p->eCreate = 2; + } + } + pPage->pDirtyNext = 0; + pPage->pDirtyPrev = 0; + + expensive_assert( pcacheCheckSynced(p) ); +} + +/* +** Add page pPage to the head of the dirty list (PCache1.pDirty is set to +** pPage). +*/ +static void pcacheAddToDirtyList(PgHdr *pPage){ + PCache *p = pPage->pCache; + + assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); + + pPage->pDirtyNext = p->pDirty; + if( pPage->pDirtyNext ){ + assert( pPage->pDirtyNext->pDirtyPrev==0 ); + pPage->pDirtyNext->pDirtyPrev = pPage; + }else if( p->bPurgeable ){ + assert( p->eCreate==2 ); + p->eCreate = 1; + } + p->pDirty = pPage; + if( !p->pDirtyTail ){ + p->pDirtyTail = pPage; + } + if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ + p->pSynced = pPage; + } + expensive_assert( pcacheCheckSynced(p) ); +} + +/* +** Wrapper around the pluggable caches xUnpin method. If the cache is +** being used for an in-memory database, this function is a no-op. +*/ +static void pcacheUnpin(PgHdr *p){ + PCache *pCache = p->pCache; + if( pCache->bPurgeable ){ + if( p->pgno==1 ){ + pCache->pPage1 = 0; + } + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 0); + } +} + +/*************************************************** General Interfaces ****** +** +** Initialize and shutdown the page cache subsystem. Neither of these +** functions are threadsafe. +*/ +SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + /* IMPLEMENTATION-OF: R-26801-64137 If the xInit() method is NULL, then the + ** built-in default page cache is used instead of the application defined + ** page cache. */ + sqlite3PCacheSetDefault(); + } + return sqlite3GlobalConfig.pcache2.xInit(sqlite3GlobalConfig.pcache2.pArg); +} +SQLITE_PRIVATE void sqlite3PcacheShutdown(void){ + if( sqlite3GlobalConfig.pcache2.xShutdown ){ + /* IMPLEMENTATION-OF: R-26000-56589 The xShutdown() method may be NULL. */ + sqlite3GlobalConfig.pcache2.xShutdown(sqlite3GlobalConfig.pcache2.pArg); + } +} + +/* +** Return the size in bytes of a PCache object. +*/ +SQLITE_PRIVATE int sqlite3PcacheSize(void){ return sizeof(PCache); } + +/* +** Create a new PCache object. Storage space to hold the object +** has already been allocated and is passed in as the p pointer. +** The caller discovers how much space needs to be allocated by +** calling sqlite3PcacheSize(). +*/ +SQLITE_PRIVATE void sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + int (*xStress)(void*,PgHdr*),/* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *p /* Preallocated space for the PCache */ +){ + memset(p, 0, sizeof(PCache)); + p->szPage = szPage; + p->szExtra = szExtra; + p->bPurgeable = bPurgeable; + p->eCreate = 2; + p->xStress = xStress; + p->pStress = pStress; + p->szCache = 100; +} + +/* +** Change the page size for PCache object. The caller must ensure that there +** are no outstanding page references when this function is called. +*/ +SQLITE_PRIVATE void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ + assert( pCache->nRef==0 && pCache->pDirty==0 ); + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); + pCache->pCache = 0; + pCache->pPage1 = 0; + } + pCache->szPage = szPage; +} + +/* +** Compute the number of pages of cache requested. +*/ +static int numberOfCachePages(PCache *p){ + if( p->szCache>=0 ){ + return p->szCache; + }else{ + return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); + } +} + +/* +** Try to obtain a page from the cache. +*/ +SQLITE_PRIVATE int sqlite3PcacheFetch( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + int createFlag, /* If true, create page if it does not exist already */ + PgHdr **ppPage /* Write the page here */ +){ + sqlite3_pcache_page *pPage; + PgHdr *pPgHdr = 0; + int eCreate; + + assert( pCache!=0 ); + assert( createFlag==1 || createFlag==0 ); + assert( pgno>0 ); + + /* If the pluggable cache (sqlite3_pcache*) has not been allocated, + ** allocate it now. + */ + if( !pCache->pCache ){ + sqlite3_pcache *p; + if( !createFlag ){ + *ppPage = 0; + return SQLITE_OK; + } + p = sqlite3GlobalConfig.pcache2.xCreate( + pCache->szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable + ); + if( !p ){ + return SQLITE_NOMEM; + } + sqlite3GlobalConfig.pcache2.xCachesize(p, numberOfCachePages(pCache)); + pCache->pCache = p; + } + + /* eCreate defines what to do if the page does not exist. + ** 0 Do not allocate a new page. (createFlag==0) + ** 1 Allocate a new page if doing so is inexpensive. + ** (createFlag==1 AND bPurgeable AND pDirty) + ** 2 Allocate a new page even it doing so is difficult. + ** (createFlag==1 AND !(bPurgeable AND pDirty) + */ + eCreate = createFlag==0 ? 0 : pCache->eCreate; + assert( (createFlag*(1+(!pCache->bPurgeable||!pCache->pDirty)))==eCreate ); + pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); + if( !pPage && eCreate==1 ){ + PgHdr *pPg; + + /* Find a dirty page to write-out and recycle. First try to find a + ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC + ** cleared), but if that is not possible settle for any other + ** unreferenced dirty page. + */ + expensive_assert( pcacheCheckSynced(pCache) ); + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + pPg=pPg->pDirtyPrev + ); + pCache->pSynced = pPg; + if( !pPg ){ + for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); + } + if( pPg ){ + int rc; +#ifdef SQLITE_LOG_CACHE_SPILL + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), + numberOfCachePages(pCache)); +#endif + rc = pCache->xStress(pCache->pStress, pPg); + if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ + return rc; + } + } + + pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); + } + + if( pPage ){ + pPgHdr = (PgHdr *)pPage->pExtra; + + if( !pPgHdr->pPage ){ + memset(pPgHdr, 0, sizeof(PgHdr)); + pPgHdr->pPage = pPage; + pPgHdr->pData = pPage->pBuf; + pPgHdr->pExtra = (void *)&pPgHdr[1]; + memset(pPgHdr->pExtra, 0, pCache->szExtra); + pPgHdr->pCache = pCache; + pPgHdr->pgno = pgno; + } + assert( pPgHdr->pCache==pCache ); + assert( pPgHdr->pgno==pgno ); + assert( pPgHdr->pData==pPage->pBuf ); + assert( pPgHdr->pExtra==(void *)&pPgHdr[1] ); + + if( 0==pPgHdr->nRef ){ + pCache->nRef++; + } + pPgHdr->nRef++; + if( pgno==1 ){ + pCache->pPage1 = pPgHdr; + } + } + *ppPage = pPgHdr; + return (pPgHdr==0 && eCreate) ? SQLITE_NOMEM : SQLITE_OK; +} + +/* +** Decrement the reference count on a page. If the page is clean and the +** reference count drops to 0, then it is made elible for recycling. +*/ +SQLITE_PRIVATE void sqlite3PcacheRelease(PgHdr *p){ + assert( p->nRef>0 ); + p->nRef--; + if( p->nRef==0 ){ + PCache *pCache = p->pCache; + pCache->nRef--; + if( (p->flags&PGHDR_DIRTY)==0 ){ + pcacheUnpin(p); + }else{ + /* Move the page to the head of the dirty list. */ + pcacheRemoveFromDirtyList(p); + pcacheAddToDirtyList(p); + } + } +} + +/* +** Increase the reference count of a supplied page by 1. +*/ +SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ + assert(p->nRef>0); + p->nRef++; +} + +/* +** Drop a page from the cache. There must be exactly one reference to the +** page. This function deletes that reference, so after it returns the +** page pointed to by p is invalid. +*/ +SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ + PCache *pCache; + assert( p->nRef==1 ); + if( p->flags&PGHDR_DIRTY ){ + pcacheRemoveFromDirtyList(p); + } + pCache = p->pCache; + pCache->nRef--; + if( p->pgno==1 ){ + pCache->pPage1 = 0; + } + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 1); +} + +/* +** Make sure the page is marked as dirty. If it isn't dirty already, +** make it so. +*/ +SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ + p->flags &= ~PGHDR_DONT_WRITE; + assert( p->nRef>0 ); + if( 0==(p->flags & PGHDR_DIRTY) ){ + p->flags |= PGHDR_DIRTY; + pcacheAddToDirtyList( p); + } +} + +/* +** Make sure the page is marked as clean. If it isn't clean already, +** make it so. +*/ +SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ + if( (p->flags & PGHDR_DIRTY) ){ + pcacheRemoveFromDirtyList(p); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); + if( p->nRef==0 ){ + pcacheUnpin(p); + } + } +} + +/* +** Make every page in the cache clean. +*/ +SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache *pCache){ + PgHdr *p; + while( (p = pCache->pDirty)!=0 ){ + sqlite3PcacheMakeClean(p); + } +} + +/* +** Clear the PGHDR_NEED_SYNC flag from all dirty pages. +*/ +SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~PGHDR_NEED_SYNC; + } + pCache->pSynced = pCache->pDirtyTail; +} + +/* +** Change the page number of page p to newPgno. +*/ +SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ + PCache *pCache = p->pCache; + assert( p->nRef>0 ); + assert( newPgno>0 ); + sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); + p->pgno = newPgno; + if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ + pcacheRemoveFromDirtyList(p); + pcacheAddToDirtyList(p); + } +} + +/* +** Drop every cache entry whose page number is greater than "pgno". The +** caller must ensure that there are no outstanding references to any pages +** other than page 1 with a page number greater than pgno. +** +** If there is a reference to page 1 and the pgno parameter passed to this +** function is 0, then the data area associated with page 1 is zeroed, but +** the page object is not dropped. +*/ +SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ + if( pCache->pCache ){ + PgHdr *p; + PgHdr *pNext; + for(p=pCache->pDirty; p; p=pNext){ + pNext = p->pDirtyNext; + /* This routine never gets call with a positive pgno except right + ** after sqlite3PcacheCleanAll(). So if there are dirty pages, + ** it must be that pgno==0. + */ + assert( p->pgno>0 ); + if( ALWAYS(p->pgno>pgno) ){ + assert( p->flags&PGHDR_DIRTY ); + sqlite3PcacheMakeClean(p); + } + } + if( pgno==0 && pCache->pPage1 ){ + memset(pCache->pPage1->pData, 0, pCache->szPage); + pgno = 1; + } + sqlite3GlobalConfig.pcache2.xTruncate(pCache->pCache, pgno+1); + } +} + +/* +** Close a cache. +*/ +SQLITE_PRIVATE void sqlite3PcacheClose(PCache *pCache){ + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); + } +} + +/* +** Discard the contents of the cache. +*/ +SQLITE_PRIVATE void sqlite3PcacheClear(PCache *pCache){ + sqlite3PcacheTruncate(pCache, 0); +} + +/* +** Merge two lists of pages connected by pDirty and in pgno order. +** Do not both fixing the pDirtyPrev pointers. +*/ +static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ + PgHdr result, *pTail; + pTail = &result; + while( pA && pB ){ + if( pA->pgnopgno ){ + pTail->pDirty = pA; + pTail = pA; + pA = pA->pDirty; + }else{ + pTail->pDirty = pB; + pTail = pB; + pB = pB->pDirty; + } + } + if( pA ){ + pTail->pDirty = pA; + }else if( pB ){ + pTail->pDirty = pB; + }else{ + pTail->pDirty = 0; + } + return result.pDirty; +} + +/* +** Sort the list of pages in accending order by pgno. Pages are +** connected by pDirty pointers. The pDirtyPrev pointers are +** corrupted by this sort. +** +** Since there cannot be more than 2^31 distinct pages in a database, +** there cannot be more than 31 buckets required by the merge sorter. +** One extra bucket is added to catch overflow in case something +** ever changes to make the previous sentence incorrect. +*/ +#define N_SORT_BUCKET 32 +static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ + PgHdr *a[N_SORT_BUCKET], *p; + int i; + memset(a, 0, sizeof(a)); + while( pIn ){ + p = pIn; + pIn = p->pDirty; + p->pDirty = 0; + for(i=0; ALWAYS(ipDirty; p; p=p->pDirtyNext){ + p->pDirty = p->pDirtyNext; + } + return pcacheSortDirtyList(pCache->pDirty); +} + +/* +** Return the total number of referenced pages held by the cache. +*/ +SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ + return pCache->nRef; +} + +/* +** Return the number of references to the page supplied as an argument. +*/ +SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr *p){ + return p->nRef; +} + +/* +** Return the total number of pages in the cache. +*/ +SQLITE_PRIVATE int sqlite3PcachePagecount(PCache *pCache){ + int nPage = 0; + if( pCache->pCache ){ + nPage = sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); + } + return nPage; +} + +#ifdef SQLITE_TEST +/* +** Get the suggested cache-size value. +*/ +SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *pCache){ + return numberOfCachePages(pCache); +} +#endif + +/* +** Set the suggested cache-size value. +*/ +SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ + pCache->szCache = mxPage; + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, + numberOfCachePages(pCache)); + } +} + +/* +** Free up as much memory as possible from the page cache. +*/ +SQLITE_PRIVATE void sqlite3PcacheShrink(PCache *pCache){ + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); + } +} + +#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) +/* +** For all dirty pages currently in the cache, invoke the specified +** callback. This is only used if the SQLITE_CHECK_PAGES macro is +** defined. +*/ +SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)){ + PgHdr *pDirty; + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext){ + xIter(pDirty); + } +} +#endif + +/************** End of pcache.c **********************************************/ +/************** Begin file pcache1.c *****************************************/ +/* +** 2008 November 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements the default page cache implementation (the +** sqlite3_pcache interface). It also contains part of the implementation +** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. +** If the default page cache implementation is overriden, then neither of +** these two features are available. +*/ + + +typedef struct PCache1 PCache1; +typedef struct PgHdr1 PgHdr1; +typedef struct PgFreeslot PgFreeslot; +typedef struct PGroup PGroup; + +/* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set +** of one or more PCaches that are able to recycle each others unpinned +** pages when they are under memory pressure. A PGroup is an instance of +** the following object. +** +** This page cache implementation works in one of two modes: +** +** (1) Every PCache is the sole member of its own PGroup. There is +** one PGroup per PCache. +** +** (2) There is a single global PGroup that all PCaches are a member +** of. +** +** Mode 1 uses more memory (since PCache instances are not able to rob +** unused pages from other PCaches) but it also operates without a mutex, +** and is therefore often faster. Mode 2 requires a mutex in order to be +** threadsafe, but recycles pages more efficiently. +** +** For mode (1), PGroup.mutex is NULL. For mode (2) there is only a single +** PGroup which is the pcache1.grp global variable and its mutex is +** SQLITE_MUTEX_STATIC_LRU. +*/ +struct PGroup { + sqlite3_mutex *mutex; /* MUTEX_STATIC_LRU or NULL */ + unsigned int nMaxPage; /* Sum of nMax for purgeable caches */ + unsigned int nMinPage; /* Sum of nMin for purgeable caches */ + unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ + unsigned int nCurrentPage; /* Number of purgeable pages allocated */ + PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ +}; + +/* Each page cache is an instance of the following object. Every +** open database file (including each in-memory database and each +** temporary or transient database) has a single page cache which +** is an instance of this object. +** +** Pointers to structures of this type are cast and returned as +** opaque sqlite3_pcache* handles. +*/ +struct PCache1 { + /* Cache configuration parameters. Page size (szPage) and the purgeable + ** flag (bPurgeable) are set when the cache is created. nMax may be + ** modified at any time by a call to the pcache1Cachesize() method. + ** The PGroup mutex must be held when accessing nMax. + */ + PGroup *pGroup; /* PGroup this cache belongs to */ + int szPage; /* Size of allocated pages in bytes */ + int szExtra; /* Size of extra space in bytes */ + int bPurgeable; /* True if cache is purgeable */ + unsigned int nMin; /* Minimum number of pages reserved */ + unsigned int nMax; /* Configured "cache_size" value */ + unsigned int n90pct; /* nMax*9/10 */ + unsigned int iMaxKey; /* Largest key seen since xTruncate() */ + + /* Hash table of all pages. The following variables may only be accessed + ** when the accessor is holding the PGroup mutex. + */ + unsigned int nRecyclable; /* Number of pages in the LRU list */ + unsigned int nPage; /* Total number of pages in apHash */ + unsigned int nHash; /* Number of slots in apHash[] */ + PgHdr1 **apHash; /* Hash table for fast lookup by key */ +}; + +/* +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; + unsigned int iKey; /* Key value (page number) */ + u8 isPinned; /* Page in use, not on the LRU list */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; + +/* +** Free slots in the allocator used to divide up the buffer provided using +** the SQLITE_CONFIG_PAGECACHE mechanism. +*/ +struct PgFreeslot { + PgFreeslot *pNext; /* Next free slot */ +}; + +/* +** Global data used by this cache. +*/ +static SQLITE_WSD struct PCacheGlobal { + PGroup grp; /* The global PGroup for mode (2) */ + + /* Variables related to SQLITE_CONFIG_PAGECACHE settings. The + ** szSlot, nSlot, pStart, pEnd, nReserve, and isInit values are all + ** fixed at sqlite3_initialize() time and do not require mutex protection. + ** The nFreeSlot and pFree values do require mutex protection. + */ + int isInit; /* True if initialized */ + int szSlot; /* Size of each free slot */ + int nSlot; /* The number of pcache slots */ + int nReserve; /* Try to keep nFreeSlot above this */ + void *pStart, *pEnd; /* Bounds of pagecache malloc range */ + /* Above requires no mutex. Use mutex below for variable that follow. */ + sqlite3_mutex *mutex; /* Mutex for accessing the following: */ + PgFreeslot *pFree; /* Free page blocks */ + int nFreeSlot; /* Number of unused pcache slots */ + /* The following value requires a mutex to change. We skip the mutex on + ** reading because (1) most platforms read a 32-bit integer atomically and + ** (2) even if an incorrect value is read, no great harm is done since this + ** is really just an optimization. */ + int bUnderPressure; /* True if low on PAGECACHE memory */ +} pcache1_g; + +/* +** All code in this file should access the global structure above via the +** alias "pcache1". This ensures that the WSD emulation is used when +** compiling for systems that do not support real WSD. +*/ +#define pcache1 (GLOBAL(struct PCacheGlobal, pcache1_g)) + +/* +** Macros to enter and leave the PCache LRU mutex. +*/ +#define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) +#define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) + +/******************************************************************************/ +/******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + +/* +** This function is called during initialization if a static buffer is +** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE +** verb to sqlite3_config(). Parameter pBuf points to an allocation large +** enough to contain 'n' buffers of 'sz' bytes each. +** +** This routine is called from sqlite3_initialize() and so it is guaranteed +** to be serialized already. There is no need for further mutexing. +*/ +SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ + if( pcache1.isInit ){ + PgFreeslot *p; + sz = ROUNDDOWN8(sz); + pcache1.szSlot = sz; + pcache1.nSlot = pcache1.nFreeSlot = n; + pcache1.nReserve = n>90 ? 10 : (n/10 + 1); + pcache1.pStart = pBuf; + pcache1.pFree = 0; + pcache1.bUnderPressure = 0; + while( n-- ){ + p = (PgFreeslot*)pBuf; + p->pNext = pcache1.pFree; + pcache1.pFree = p; + pBuf = (void*)&((char*)pBuf)[sz]; + } + pcache1.pEnd = pBuf; + } +} + +/* +** Malloc function used within this file to allocate space from the buffer +** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no +** such buffer exists or there is no space left in it, this function falls +** back to sqlite3Malloc(). +** +** Multiple threads can run this routine at the same time. Global variables +** in pcache1 need to be protected via mutex. +*/ +static void *pcache1Alloc(int nByte){ + void *p = 0; + assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + if( nByte<=pcache1.szSlot ){ + sqlite3_mutex_enter(pcache1.mutex); + p = (PgHdr1 *)pcache1.pFree; + if( p ){ + pcache1.pFree = pcache1.pFree->pNext; + pcache1.nFreeSlot--; + pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); + } + sqlite3_mutex_leave(pcache1.mutex); + } + if( p==0 ){ + /* Memory is not available in the SQLITE_CONFIG_PAGECACHE pool. Get + ** it from sqlite3Malloc instead. + */ + p = sqlite3Malloc(nByte); +#ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS + if( p ){ + int sz = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); + sqlite3_mutex_leave(pcache1.mutex); + } +#endif + sqlite3MemdebugSetType(p, MEMTYPE_PCACHE); + } + return p; +} + +/* +** Free an allocated buffer obtained from pcache1Alloc(). +*/ +static int pcache1Free(void *p){ + int nFreed = 0; + if( p==0 ) return 0; + if( p>=pcache1.pStart && ppNext = pcache1.pFree; + pcache1.pFree = pSlot; + pcache1.nFreeSlot++; + pcache1.bUnderPressure = pcache1.nFreeSlot=pcache1.pStart && ppGroup->mutex) ); + pcache1LeaveMutex(pCache->pGroup); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + pPg = pcache1Alloc(pCache->szPage); + p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); + if( !pPg || !p ){ + pcache1Free(pPg); + sqlite3_free(p); + pPg = 0; + } +#else + pPg = pcache1Alloc(sizeof(PgHdr1) + pCache->szPage + pCache->szExtra); + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; +#endif + pcache1EnterMutex(pCache->pGroup); + + if( pPg ){ + p->page.pBuf = pPg; + p->page.pExtra = &p[1]; + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage++; + } + return p; + } + return 0; +} + +/* +** Free a page object allocated by pcache1AllocPage(). +** +** The pointer is allowed to be NULL, which is prudent. But it turns out +** that the current implementation happens to never call this routine +** with a NULL pointer, so we mark the NULL test with ALWAYS(). +*/ +static void pcache1FreePage(PgHdr1 *p){ + if( ALWAYS(p) ){ + PCache1 *pCache = p->pCache; + assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + pcache1Free(p->page.pBuf); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + sqlite3_free(p); +#endif + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage--; + } + } +} + +/* +** Malloc function used by SQLite to obtain space from the buffer configured +** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer +** exists, this function falls back to sqlite3Malloc(). +*/ +SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){ + return pcache1Alloc(sz); +} + +/* +** Free an allocated buffer obtained from sqlite3PageMalloc(). +*/ +SQLITE_PRIVATE void sqlite3PageFree(void *p){ + pcache1Free(p); +} + + +/* +** Return true if it desirable to avoid allocating a new page cache +** entry. +** +** If memory was allocated specifically to the page cache using +** SQLITE_CONFIG_PAGECACHE but that memory has all been used, then +** it is desirable to avoid allocating a new page cache entry because +** presumably SQLITE_CONFIG_PAGECACHE was suppose to be sufficient +** for all page cache needs and we should not need to spill the +** allocation onto the heap. +** +** Or, the heap is used for all page cache memory but the heap is +** under memory pressure, then again it is desirable to avoid +** allocating a new page cache entry in order to avoid stressing +** the heap even further. +*/ +static int pcache1UnderMemoryPressure(PCache1 *pCache){ + if( pcache1.nSlot && (pCache->szPage+pCache->szExtra)<=pcache1.szSlot ){ + return pcache1.bUnderPressure; + }else{ + return sqlite3HeapNearlyFull(); + } +} + +/******************************************************************************/ +/******** General Implementation Functions ************************************/ + +/* +** This function is used to resize the hash table used by the cache passed +** as the first argument. +** +** The PCache mutex must be held when this function is called. +*/ +static int pcache1ResizeHash(PCache1 *p){ + PgHdr1 **apNew; + unsigned int nNew; + unsigned int i; + + assert( sqlite3_mutex_held(p->pGroup->mutex) ); + + nNew = p->nHash*2; + if( nNew<256 ){ + nNew = 256; + } + + pcache1LeaveMutex(p->pGroup); + if( p->nHash ){ sqlite3BeginBenignMalloc(); } + apNew = (PgHdr1 **)sqlite3MallocZero(sizeof(PgHdr1 *)*nNew); + if( p->nHash ){ sqlite3EndBenignMalloc(); } + pcache1EnterMutex(p->pGroup); + if( apNew ){ + for(i=0; inHash; i++){ + PgHdr1 *pPage; + PgHdr1 *pNext = p->apHash[i]; + while( (pPage = pNext)!=0 ){ + unsigned int h = pPage->iKey % nNew; + pNext = pPage->pNext; + pPage->pNext = apNew[h]; + apNew[h] = pPage; + } + } + sqlite3_free(p->apHash); + p->apHash = apNew; + p->nHash = nNew; + } + + return (p->apHash ? SQLITE_OK : SQLITE_NOMEM); +} + +/* +** This function is used internally to remove the page pPage from the +** PGroup LRU list, if is part of it. If pPage is not part of the PGroup +** LRU list, then this function is a no-op. +** +** The PGroup mutex must be held when this function is called. +*/ +static void pcache1PinPage(PgHdr1 *pPage){ + PCache1 *pCache; + PGroup *pGroup; + + assert( pPage!=0 ); + assert( pPage->isPinned==0 ); + pCache = pPage->pCache; + pGroup = pCache->pGroup; + assert( pPage->pLruNext || pPage==pGroup->pLruTail ); + assert( pPage->pLruPrev || pPage==pGroup->pLruHead ); + assert( sqlite3_mutex_held(pGroup->mutex) ); + if( pPage->pLruPrev ){ + pPage->pLruPrev->pLruNext = pPage->pLruNext; + }else{ + pGroup->pLruHead = pPage->pLruNext; + } + if( pPage->pLruNext ){ + pPage->pLruNext->pLruPrev = pPage->pLruPrev; + }else{ + pGroup->pLruTail = pPage->pLruPrev; + } + pPage->pLruNext = 0; + pPage->pLruPrev = 0; + pPage->isPinned = 1; + pCache->nRecyclable--; +} + + +/* +** Remove the page supplied as an argument from the hash table +** (PCache1.apHash structure) that it is currently stored in. +** +** The PGroup mutex must be held when this function is called. +*/ +static void pcache1RemoveFromHash(PgHdr1 *pPage){ + unsigned int h; + PCache1 *pCache = pPage->pCache; + PgHdr1 **pp; + + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + h = pPage->iKey % pCache->nHash; + for(pp=&pCache->apHash[h]; (*pp)!=pPage; pp=&(*pp)->pNext); + *pp = (*pp)->pNext; + + pCache->nPage--; +} + +/* +** If there are currently more than nMaxPage pages allocated, try +** to recycle pages to reduce the number allocated to nMaxPage. +*/ +static void pcache1EnforceMaxPage(PGroup *pGroup){ + assert( sqlite3_mutex_held(pGroup->mutex) ); + while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ + PgHdr1 *p = pGroup->pLruTail; + assert( p->pCache->pGroup==pGroup ); + assert( p->isPinned==0 ); + pcache1PinPage(p); + pcache1RemoveFromHash(p); + pcache1FreePage(p); + } +} + +/* +** Discard all pages from cache pCache with a page number (key value) +** greater than or equal to iLimit. Any pinned pages that meet this +** criteria are unpinned before they are discarded. +** +** The PCache mutex must be held when this function is called. +*/ +static void pcache1TruncateUnsafe( + PCache1 *pCache, /* The cache to truncate */ + unsigned int iLimit /* Drop pages with this pgno or larger */ +){ + TESTONLY( unsigned int nPage = 0; ) /* To assert pCache->nPage is correct */ + unsigned int h; + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + for(h=0; hnHash; h++){ + PgHdr1 **pp = &pCache->apHash[h]; + PgHdr1 *pPage; + while( (pPage = *pp)!=0 ){ + if( pPage->iKey>=iLimit ){ + pCache->nPage--; + *pp = pPage->pNext; + if( !pPage->isPinned ) pcache1PinPage(pPage); + pcache1FreePage(pPage); + }else{ + pp = &pPage->pNext; + TESTONLY( nPage++; ) + } + } + } + assert( pCache->nPage==nPage ); +} + +/******************************************************************************/ +/******** sqlite3_pcache Methods **********************************************/ + +/* +** Implementation of the sqlite3_pcache.xInit method. +*/ +static int pcache1Init(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( pcache1.isInit==0 ); + memset(&pcache1, 0, sizeof(pcache1)); + if( sqlite3GlobalConfig.bCoreMutex ){ + pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); + } + pcache1.grp.mxPinned = 10; + pcache1.isInit = 1; + return SQLITE_OK; +} + +/* +** Implementation of the sqlite3_pcache.xShutdown method. +** Note that the static mutex allocated in xInit does +** not need to be freed. +*/ +static void pcache1Shutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( pcache1.isInit!=0 ); + memset(&pcache1, 0, sizeof(pcache1)); +} + +/* +** Implementation of the sqlite3_pcache.xCreate method. +** +** Allocate a new cache. +*/ +static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ + PCache1 *pCache; /* The newly created page cache */ + PGroup *pGroup; /* The group the new page cache will belong to */ + int sz; /* Bytes of memory required to allocate the new cache */ + + /* + ** The separateCache variable is true if each PCache has its own private + ** PGroup. In other words, separateCache is true for mode (1) where no + ** mutexing is required. + ** + ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT + ** + ** * Always use a unified cache in single-threaded applications + ** + ** * Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off) + ** use separate caches (mode-1) + */ +#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 + const int separateCache = 0; +#else + int separateCache = sqlite3GlobalConfig.bCoreMutex>0; +#endif + + assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); + assert( szExtra < 300 ); + + sz = sizeof(PCache1) + sizeof(PGroup)*separateCache; + pCache = (PCache1 *)sqlite3MallocZero(sz); + if( pCache ){ + if( separateCache ){ + pGroup = (PGroup*)&pCache[1]; + pGroup->mxPinned = 10; + }else{ + pGroup = &pcache1.grp; + } + pCache->pGroup = pGroup; + pCache->szPage = szPage; + pCache->szExtra = szExtra; + pCache->bPurgeable = (bPurgeable ? 1 : 0); + if( bPurgeable ){ + pCache->nMin = 10; + pcache1EnterMutex(pGroup); + pGroup->nMinPage += pCache->nMin; + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pcache1LeaveMutex(pGroup); + } + } + return (sqlite3_pcache *)pCache; +} + +/* +** Implementation of the sqlite3_pcache.xCachesize method. +** +** Configure the cache_size limit for a cache. +*/ +static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ + PCache1 *pCache = (PCache1 *)p; + if( pCache->bPurgeable ){ + PGroup *pGroup = pCache->pGroup; + pcache1EnterMutex(pGroup); + pGroup->nMaxPage += (nMax - pCache->nMax); + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pCache->nMax = nMax; + pCache->n90pct = pCache->nMax*9/10; + pcache1EnforceMaxPage(pGroup); + pcache1LeaveMutex(pGroup); + } +} + +/* +** Implementation of the sqlite3_pcache.xShrink method. +** +** Free up as much memory as possible. +*/ +static void pcache1Shrink(sqlite3_pcache *p){ + PCache1 *pCache = (PCache1*)p; + if( pCache->bPurgeable ){ + PGroup *pGroup = pCache->pGroup; + int savedMaxPage; + pcache1EnterMutex(pGroup); + savedMaxPage = pGroup->nMaxPage; + pGroup->nMaxPage = 0; + pcache1EnforceMaxPage(pGroup); + pGroup->nMaxPage = savedMaxPage; + pcache1LeaveMutex(pGroup); + } +} + +/* +** Implementation of the sqlite3_pcache.xPagecount method. +*/ +static int pcache1Pagecount(sqlite3_pcache *p){ + int n; + PCache1 *pCache = (PCache1*)p; + pcache1EnterMutex(pCache->pGroup); + n = pCache->nPage; + pcache1LeaveMutex(pCache->pGroup); + return n; +} + +/* +** Implementation of the sqlite3_pcache.xFetch method. +** +** Fetch a page by key value. +** +** Whether or not a new page may be allocated by this function depends on +** the value of the createFlag argument. 0 means do not allocate a new +** page. 1 means allocate a new page if space is easily available. 2 +** means to try really hard to allocate a new page. +** +** For a non-purgeable cache (a cache used as the storage for an in-memory +** database) there is really no difference between createFlag 1 and 2. So +** the calling function (pcache.c) will never have a createFlag of 1 on +** a non-purgeable cache. +** +** There are three different approaches to obtaining space for a page, +** depending on the value of parameter createFlag (which may be 0, 1 or 2). +** +** 1. Regardless of the value of createFlag, the cache is searched for a +** copy of the requested page. If one is found, it is returned. +** +** 2. If createFlag==0 and the page is not already in the cache, NULL is +** returned. +** +** 3. If createFlag is 1, and the page is not already in the cache, then +** return NULL (do not allocate a new page) if any of the following +** conditions are true: +** +** (a) the number of pages pinned by the cache is greater than +** PCache1.nMax, or +** +** (b) the number of pages pinned by the cache is greater than +** the sum of nMax for all purgeable caches, less the sum of +** nMin for all other purgeable caches, or +** +** 4. If none of the first three conditions apply and the cache is marked +** as purgeable, and if one of the following is true: +** +** (a) The number of pages allocated for the cache is already +** PCache1.nMax, or +** +** (b) The number of pages allocated for all purgeable caches is +** already equal to or greater than the sum of nMax for all +** purgeable caches, +** +** (c) The system is under memory pressure and wants to avoid +** unnecessary pages cache entry allocations +** +** then attempt to recycle a page from the LRU list. If it is the right +** size, return the recycled buffer. Otherwise, free the buffer and +** proceed to step 5. +** +** 5. Otherwise, allocate and return a new page buffer. +*/ +static sqlite3_pcache_page *pcache1Fetch( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + unsigned int nPinned; + PCache1 *pCache = (PCache1 *)p; + PGroup *pGroup; + PgHdr1 *pPage = 0; + + assert( offsetof(PgHdr1,page)==0 ); + assert( pCache->bPurgeable || createFlag!=1 ); + assert( pCache->bPurgeable || pCache->nMin==0 ); + assert( pCache->bPurgeable==0 || pCache->nMin==10 ); + assert( pCache->nMin==0 || pCache->bPurgeable ); + pcache1EnterMutex(pGroup = pCache->pGroup); + + /* Step 1: Search the hash table for an existing entry. */ + if( pCache->nHash>0 ){ + unsigned int h = iKey % pCache->nHash; + for(pPage=pCache->apHash[h]; pPage&&pPage->iKey!=iKey; pPage=pPage->pNext); + } + + /* Step 2: Abort if no existing page is found and createFlag is 0 */ + if( pPage ){ + if( !pPage->isPinned ) pcache1PinPage(pPage); + goto fetch_out; + } + if( createFlag==0 ){ + goto fetch_out; + } + + /* The pGroup local variable will normally be initialized by the + ** pcache1EnterMutex() macro above. But if SQLITE_MUTEX_OMIT is defined, + ** then pcache1EnterMutex() is a no-op, so we have to initialize the + ** local variable here. Delaying the initialization of pGroup is an + ** optimization: The common case is to exit the module before reaching + ** this point. + */ +#ifdef SQLITE_MUTEX_OMIT + pGroup = pCache->pGroup; +#endif + + /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ + assert( pCache->nPage >= pCache->nRecyclable ); + nPinned = pCache->nPage - pCache->nRecyclable; + assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); + assert( pCache->n90pct == pCache->nMax*9/10 ); + if( createFlag==1 && ( + nPinned>=pGroup->mxPinned + || nPinned>=pCache->n90pct + || pcache1UnderMemoryPressure(pCache) + )){ + goto fetch_out; + } + + if( pCache->nPage>=pCache->nHash && pcache1ResizeHash(pCache) ){ + goto fetch_out; + } + assert( pCache->nHash>0 && pCache->apHash ); + + /* Step 4. Try to recycle a page. */ + if( pCache->bPurgeable && pGroup->pLruTail && ( + (pCache->nPage+1>=pCache->nMax) + || pGroup->nCurrentPage>=pGroup->nMaxPage + || pcache1UnderMemoryPressure(pCache) + )){ + PCache1 *pOther; + pPage = pGroup->pLruTail; + assert( pPage->isPinned==0 ); + pcache1RemoveFromHash(pPage); + pcache1PinPage(pPage); + pOther = pPage->pCache; + + /* We want to verify that szPage and szExtra are the same for pOther + ** and pCache. Assert that we can verify this by comparing sums. */ + assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); + assert( pCache->szExtra<512 ); + assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); + assert( pOther->szExtra<512 ); + + if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ + pcache1FreePage(pPage); + pPage = 0; + }else{ + pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); + } + } + + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. + */ + if( !pPage ){ + if( createFlag==1 ) sqlite3BeginBenignMalloc(); + pPage = pcache1AllocPage(pCache); + if( createFlag==1 ) sqlite3EndBenignMalloc(); + } + + if( pPage ){ + unsigned int h = iKey % pCache->nHash; + pCache->nPage++; + pPage->iKey = iKey; + pPage->pNext = pCache->apHash[h]; + pPage->pCache = pCache; + pPage->pLruPrev = 0; + pPage->pLruNext = 0; + pPage->isPinned = 1; + *(void **)pPage->page.pExtra = 0; + pCache->apHash[h] = pPage; + } + +fetch_out: + if( pPage && iKey>pCache->iMaxKey ){ + pCache->iMaxKey = iKey; + } + pcache1LeaveMutex(pGroup); + return (sqlite3_pcache_page*)pPage; +} + + +/* +** Implementation of the sqlite3_pcache.xUnpin method. +** +** Mark a page as unpinned (eligible for asynchronous recycling). +*/ +static void pcache1Unpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int reuseUnlikely +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = (PgHdr1 *)pPg; + PGroup *pGroup = pCache->pGroup; + + assert( pPage->pCache==pCache ); + pcache1EnterMutex(pGroup); + + /* It is an error to call this function if the page is already + ** part of the PGroup LRU list. + */ + assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); + assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage ); + assert( pPage->isPinned==1 ); + + if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ + pcache1RemoveFromHash(pPage); + pcache1FreePage(pPage); + }else{ + /* Add the page to the PGroup LRU list. */ + if( pGroup->pLruHead ){ + pGroup->pLruHead->pLruPrev = pPage; + pPage->pLruNext = pGroup->pLruHead; + pGroup->pLruHead = pPage; + }else{ + pGroup->pLruTail = pPage; + pGroup->pLruHead = pPage; + } + pCache->nRecyclable++; + pPage->isPinned = 0; + } + + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xRekey method. +*/ +static void pcache1Rekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned int iOld, + unsigned int iNew +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = (PgHdr1 *)pPg; + PgHdr1 **pp; + unsigned int h; + assert( pPage->iKey==iOld ); + assert( pPage->pCache==pCache ); + + pcache1EnterMutex(pCache->pGroup); + + h = iOld%pCache->nHash; + pp = &pCache->apHash[h]; + while( (*pp)!=pPage ){ + pp = &(*pp)->pNext; + } + *pp = pPage->pNext; + + h = iNew%pCache->nHash; + pPage->iKey = iNew; + pPage->pNext = pCache->apHash[h]; + pCache->apHash[h] = pPage; + if( iNew>pCache->iMaxKey ){ + pCache->iMaxKey = iNew; + } + + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xTruncate method. +** +** Discard all unpinned pages in the cache with a page number equal to +** or greater than parameter iLimit. Any pinned pages with a page number +** equal to or greater than iLimit are implicitly unpinned. +*/ +static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ + PCache1 *pCache = (PCache1 *)p; + pcache1EnterMutex(pCache->pGroup); + if( iLimit<=pCache->iMaxKey ){ + pcache1TruncateUnsafe(pCache, iLimit); + pCache->iMaxKey = iLimit-1; + } + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xDestroy method. +** +** Destroy a cache allocated using pcache1Create(). +*/ +static void pcache1Destroy(sqlite3_pcache *p){ + PCache1 *pCache = (PCache1 *)p; + PGroup *pGroup = pCache->pGroup; + assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); + pcache1EnterMutex(pGroup); + pcache1TruncateUnsafe(pCache, 0); + assert( pGroup->nMaxPage >= pCache->nMax ); + pGroup->nMaxPage -= pCache->nMax; + assert( pGroup->nMinPage >= pCache->nMin ); + pGroup->nMinPage -= pCache->nMin; + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pcache1EnforceMaxPage(pGroup); + pcache1LeaveMutex(pGroup); + sqlite3_free(pCache->apHash); + sqlite3_free(pCache); +} + +/* +** This function is called during initialization (sqlite3_initialize()) to +** install the default pluggable cache module, assuming the user has not +** already provided an alternative. +*/ +SQLITE_PRIVATE void sqlite3PCacheSetDefault(void){ + static const sqlite3_pcache_methods2 defaultMethods = { + 1, /* iVersion */ + 0, /* pArg */ + pcache1Init, /* xInit */ + pcache1Shutdown, /* xShutdown */ + pcache1Create, /* xCreate */ + pcache1Cachesize, /* xCachesize */ + pcache1Pagecount, /* xPagecount */ + pcache1Fetch, /* xFetch */ + pcache1Unpin, /* xUnpin */ + pcache1Rekey, /* xRekey */ + pcache1Truncate, /* xTruncate */ + pcache1Destroy, /* xDestroy */ + pcache1Shrink /* xShrink */ + }; + sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultMethods); +} + +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* +** This function is called to free superfluous dynamically allocated memory +** held by the pager system. Memory in use by any SQLite pager allocated +** by the current thread may be sqlite3_free()ed. +** +** nReq is the number of bytes of memory required. Once this much has +** been released, the function returns. The return value is the total number +** of bytes of memory released. +*/ +SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ + int nFree = 0; + assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); + assert( sqlite3_mutex_notheld(pcache1.mutex) ); + if( pcache1.pStart==0 ){ + PgHdr1 *p; + pcache1EnterMutex(&pcache1.grp); + while( (nReq<0 || nFreepage.pBuf); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + nFree += sqlite3MemSize(p); +#endif + assert( p->isPinned==0 ); + pcache1PinPage(p); + pcache1RemoveFromHash(p); + pcache1FreePage(p); + } + pcache1LeaveMutex(&pcache1.grp); + } + return nFree; +} +#endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ + +#ifdef SQLITE_TEST +/* +** This function is used by test procedures to inspect the internal state +** of the global cache. +*/ +SQLITE_PRIVATE void sqlite3PcacheStats( + int *pnCurrent, /* OUT: Total number of pages cached */ + int *pnMax, /* OUT: Global maximum cache size */ + int *pnMin, /* OUT: Sum of PCache1.nMin for purgeable caches */ + int *pnRecyclable /* OUT: Total number of pages available for recycling */ +){ + PgHdr1 *p; + int nRecyclable = 0; + for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){ + assert( p->isPinned==0 ); + nRecyclable++; + } + *pnCurrent = pcache1.grp.nCurrentPage; + *pnMax = (int)pcache1.grp.nMaxPage; + *pnMin = (int)pcache1.grp.nMinPage; + *pnRecyclable = nRecyclable; +} +#endif + +/************** End of pcache1.c *********************************************/ +/************** Begin file rowset.c ******************************************/ +/* +** 2008 December 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module implements an object we call a "RowSet". +** +** The RowSet object is a collection of rowids. Rowids +** are inserted into the RowSet in an arbitrary order. Inserts +** can be intermixed with tests to see if a given rowid has been +** previously inserted into the RowSet. +** +** After all inserts are finished, it is possible to extract the +** elements of the RowSet in sorted order. Once this extraction +** process has started, no new elements may be inserted. +** +** Hence, the primitive operations for a RowSet are: +** +** CREATE +** INSERT +** TEST +** SMALLEST +** DESTROY +** +** The CREATE and DESTROY primitives are the constructor and destructor, +** obviously. The INSERT primitive adds a new element to the RowSet. +** TEST checks to see if an element is already in the RowSet. SMALLEST +** extracts the least value from the RowSet. +** +** The INSERT primitive might allocate additional memory. Memory is +** allocated in chunks so most INSERTs do no allocation. There is an +** upper bound on the size of allocated memory. No memory is freed +** until DESTROY. +** +** The TEST primitive includes a "batch" number. The TEST primitive +** will only see elements that were inserted before the last change +** in the batch number. In other words, if an INSERT occurs between +** two TESTs where the TESTs have the same batch nubmer, then the +** value added by the INSERT will not be visible to the second TEST. +** The initial batch number is zero, so if the very first TEST contains +** a non-zero batch number, it will see all prior INSERTs. +** +** No INSERTs may occurs after a SMALLEST. An assertion will fail if +** that is attempted. +** +** The cost of an INSERT is roughly constant. (Sometime new memory +** has to be allocated on an INSERT.) The cost of a TEST with a new +** batch number is O(NlogN) where N is the number of elements in the RowSet. +** The cost of a TEST using the same batch number is O(logN). The cost +** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST +** primitives are constant time. The cost of DESTROY is O(N). +** +** There is an added cost of O(N) when switching between TEST and +** SMALLEST primitives. +*/ + + +/* +** Target size for allocation chunks. +*/ +#define ROWSET_ALLOCATION_SIZE 1024 + +/* +** The number of rowset entries per allocation chunk. +*/ +#define ROWSET_ENTRY_PER_CHUNK \ + ((ROWSET_ALLOCATION_SIZE-8)/sizeof(struct RowSetEntry)) + +/* +** Each entry in a RowSet is an instance of the following object. +** +** This same object is reused to store a linked list of trees of RowSetEntry +** objects. In that alternative use, pRight points to the next entry +** in the list, pLeft points to the tree, and v is unused. The +** RowSet.pForest value points to the head of this forest list. +*/ +struct RowSetEntry { + i64 v; /* ROWID value for this entry */ + struct RowSetEntry *pRight; /* Right subtree (larger entries) or list */ + struct RowSetEntry *pLeft; /* Left subtree (smaller entries) */ +}; + +/* +** RowSetEntry objects are allocated in large chunks (instances of the +** following structure) to reduce memory allocation overhead. The +** chunks are kept on a linked list so that they can be deallocated +** when the RowSet is destroyed. +*/ +struct RowSetChunk { + struct RowSetChunk *pNextChunk; /* Next chunk on list of them all */ + struct RowSetEntry aEntry[ROWSET_ENTRY_PER_CHUNK]; /* Allocated entries */ +}; + +/* +** A RowSet in an instance of the following structure. +** +** A typedef of this structure if found in sqliteInt.h. +*/ +struct RowSet { + struct RowSetChunk *pChunk; /* List of all chunk allocations */ + sqlite3 *db; /* The database connection */ + struct RowSetEntry *pEntry; /* List of entries using pRight */ + struct RowSetEntry *pLast; /* Last entry on the pEntry list */ + struct RowSetEntry *pFresh; /* Source of new entry objects */ + struct RowSetEntry *pForest; /* List of binary trees of entries */ + u16 nFresh; /* Number of objects on pFresh */ + u16 rsFlags; /* Various flags */ + int iBatch; /* Current insert batch */ +}; + +/* +** Allowed values for RowSet.rsFlags +*/ +#define ROWSET_SORTED 0x01 /* True if RowSet.pEntry is sorted */ +#define ROWSET_NEXT 0x02 /* True if sqlite3RowSetNext() has been called */ + +/* +** Turn bulk memory into a RowSet object. N bytes of memory +** are available at pSpace. The db pointer is used as a memory context +** for any subsequent allocations that need to occur. +** Return a pointer to the new RowSet object. +** +** It must be the case that N is sufficient to make a Rowset. If not +** an assertion fault occurs. +** +** If N is larger than the minimum, use the surplus as an initial +** allocation of entries available to be filled. +*/ +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){ + RowSet *p; + assert( N >= ROUND8(sizeof(*p)) ); + p = pSpace; + p->pChunk = 0; + p->db = db; + p->pEntry = 0; + p->pLast = 0; + p->pForest = 0; + p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); + p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); + p->rsFlags = ROWSET_SORTED; + p->iBatch = 0; + return p; +} + +/* +** Deallocate all chunks from a RowSet. This frees all memory that +** the RowSet has allocated over its lifetime. This routine is +** the destructor for the RowSet. +*/ +SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ + struct RowSetChunk *pChunk, *pNextChunk; + for(pChunk=p->pChunk; pChunk; pChunk = pNextChunk){ + pNextChunk = pChunk->pNextChunk; + sqlite3DbFree(p->db, pChunk); + } + p->pChunk = 0; + p->nFresh = 0; + p->pEntry = 0; + p->pLast = 0; + p->pForest = 0; + p->rsFlags = ROWSET_SORTED; +} + +/* +** Allocate a new RowSetEntry object that is associated with the +** given RowSet. Return a pointer to the new and completely uninitialized +** objected. +** +** In an OOM situation, the RowSet.db->mallocFailed flag is set and this +** routine returns NULL. +*/ +static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ + assert( p!=0 ); + if( p->nFresh==0 ){ + struct RowSetChunk *pNew; + pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); + if( pNew==0 ){ + return 0; + } + pNew->pNextChunk = p->pChunk; + p->pChunk = pNew; + p->pFresh = pNew->aEntry; + p->nFresh = ROWSET_ENTRY_PER_CHUNK; + } + p->nFresh--; + return p->pFresh++; +} + +/* +** Insert a new value into a RowSet. +** +** The mallocFailed flag of the database connection is set if a +** memory allocation fails. +*/ +SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet *p, i64 rowid){ + struct RowSetEntry *pEntry; /* The new entry */ + struct RowSetEntry *pLast; /* The last prior entry */ + + /* This routine is never called after sqlite3RowSetNext() */ + assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); + + pEntry = rowSetEntryAlloc(p); + if( pEntry==0 ) return; + pEntry->v = rowid; + pEntry->pRight = 0; + pLast = p->pLast; + if( pLast ){ + if( (p->rsFlags & ROWSET_SORTED)!=0 && rowid<=pLast->v ){ + p->rsFlags &= ~ROWSET_SORTED; + } + pLast->pRight = pEntry; + }else{ + p->pEntry = pEntry; + } + p->pLast = pEntry; +} + +/* +** Merge two lists of RowSetEntry objects. Remove duplicates. +** +** The input lists are connected via pRight pointers and are +** assumed to each already be in sorted order. +*/ +static struct RowSetEntry *rowSetEntryMerge( + struct RowSetEntry *pA, /* First sorted list to be merged */ + struct RowSetEntry *pB /* Second sorted list to be merged */ +){ + struct RowSetEntry head; + struct RowSetEntry *pTail; + + pTail = &head; + while( pA && pB ){ + assert( pA->pRight==0 || pA->v<=pA->pRight->v ); + assert( pB->pRight==0 || pB->v<=pB->pRight->v ); + if( pA->vv ){ + pTail->pRight = pA; + pA = pA->pRight; + pTail = pTail->pRight; + }else if( pB->vv ){ + pTail->pRight = pB; + pB = pB->pRight; + pTail = pTail->pRight; + }else{ + pA = pA->pRight; + } + } + if( pA ){ + assert( pA->pRight==0 || pA->v<=pA->pRight->v ); + pTail->pRight = pA; + }else{ + assert( pB==0 || pB->pRight==0 || pB->v<=pB->pRight->v ); + pTail->pRight = pB; + } + return head.pRight; +} + +/* +** Sort all elements on the list of RowSetEntry objects into order of +** increasing v. +*/ +static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ + unsigned int i; + struct RowSetEntry *pNext, *aBucket[40]; + + memset(aBucket, 0, sizeof(aBucket)); + while( pIn ){ + pNext = pIn->pRight; + pIn->pRight = 0; + for(i=0; aBucket[i]; i++){ + pIn = rowSetEntryMerge(aBucket[i], pIn); + aBucket[i] = 0; + } + aBucket[i] = pIn; + pIn = pNext; + } + pIn = 0; + for(i=0; ipLeft ){ + struct RowSetEntry *p; + rowSetTreeToList(pIn->pLeft, ppFirst, &p); + p->pRight = pIn; + }else{ + *ppFirst = pIn; + } + if( pIn->pRight ){ + rowSetTreeToList(pIn->pRight, &pIn->pRight, ppLast); + }else{ + *ppLast = pIn; + } + assert( (*ppLast)->pRight==0 ); +} + + +/* +** Convert a sorted list of elements (connected by pRight) into a binary +** tree with depth of iDepth. A depth of 1 means the tree contains a single +** node taken from the head of *ppList. A depth of 2 means a tree with +** three nodes. And so forth. +** +** Use as many entries from the input list as required and update the +** *ppList to point to the unused elements of the list. If the input +** list contains too few elements, then construct an incomplete tree +** and leave *ppList set to NULL. +** +** Return a pointer to the root of the constructed binary tree. +*/ +static struct RowSetEntry *rowSetNDeepTree( + struct RowSetEntry **ppList, + int iDepth +){ + struct RowSetEntry *p; /* Root of the new tree */ + struct RowSetEntry *pLeft; /* Left subtree */ + if( *ppList==0 ){ + return 0; + } + if( iDepth==1 ){ + p = *ppList; + *ppList = p->pRight; + p->pLeft = p->pRight = 0; + return p; + } + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ + return pLeft; + } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); + return p; +} + +/* +** Convert a sorted list of elements into a binary tree. Make the tree +** as deep as it needs to be in order to contain the entire list. +*/ +static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ + int iDepth; /* Depth of the tree so far */ + struct RowSetEntry *p; /* Current tree root */ + struct RowSetEntry *pLeft; /* Left subtree */ + + assert( pList!=0 ); + p = pList; + pList = p->pRight; + p->pLeft = p->pRight = 0; + for(iDepth=1; pList; iDepth++){ + pLeft = p; + p = pList; + pList = p->pRight; + p->pLeft = pLeft; + p->pRight = rowSetNDeepTree(&pList, iDepth); + } + return p; +} + +/* +** Take all the entries on p->pEntry and on the trees in p->pForest and +** sort them all together into one big ordered list on p->pEntry. +** +** This routine should only be called once in the life of a RowSet. +*/ +static void rowSetToList(RowSet *p){ + + /* This routine is called only once */ + assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); + + if( (p->rsFlags & ROWSET_SORTED)==0 ){ + p->pEntry = rowSetEntrySort(p->pEntry); + } + + /* While this module could theoretically support it, sqlite3RowSetNext() + ** is never called after sqlite3RowSetText() for the same RowSet. So + ** there is never a forest to deal with. Should this change, simply + ** remove the assert() and the #if 0. */ + assert( p->pForest==0 ); +#if 0 + while( p->pForest ){ + struct RowSetEntry *pTree = p->pForest->pLeft; + if( pTree ){ + struct RowSetEntry *pHead, *pTail; + rowSetTreeToList(pTree, &pHead, &pTail); + p->pEntry = rowSetEntryMerge(p->pEntry, pHead); + } + p->pForest = p->pForest->pRight; + } +#endif + p->rsFlags |= ROWSET_NEXT; /* Verify this routine is never called again */ +} + +/* +** Extract the smallest element from the RowSet. +** Write the element into *pRowid. Return 1 on success. Return +** 0 if the RowSet is already empty. +** +** After this routine has been called, the sqlite3RowSetInsert() +** routine may not be called again. +*/ +SQLITE_PRIVATE int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ + assert( p!=0 ); + + /* Merge the forest into a single sorted list on first call */ + if( (p->rsFlags & ROWSET_NEXT)==0 ) rowSetToList(p); + + /* Return the next entry on the list */ + if( p->pEntry ){ + *pRowid = p->pEntry->v; + p->pEntry = p->pEntry->pRight; + if( p->pEntry==0 ){ + sqlite3RowSetClear(p); + } + return 1; + }else{ + return 0; + } +} + +/* +** Check to see if element iRowid was inserted into the rowset as +** part of any insert batch prior to iBatch. Return 1 or 0. +** +** If this is the first test of a new batch and if there exist entires +** on pRowSet->pEntry, then sort those entires into the forest at +** pRowSet->pForest so that they can be tested. +*/ +SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ + struct RowSetEntry *p, *pTree; + + /* This routine is never called after sqlite3RowSetNext() */ + assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); + + /* Sort entries into the forest on the first test of a new batch + */ + if( iBatch!=pRowSet->iBatch ){ + p = pRowSet->pEntry; + if( p ){ + struct RowSetEntry **ppPrevTree = &pRowSet->pForest; + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ + p = rowSetEntrySort(p); + } + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + ppPrevTree = &pTree->pRight; + if( pTree->pLeft==0 ){ + pTree->pLeft = rowSetListToTree(p); + break; + }else{ + struct RowSetEntry *pAux, *pTail; + rowSetTreeToList(pTree->pLeft, &pAux, &pTail); + pTree->pLeft = 0; + p = rowSetEntryMerge(pAux, p); + } + } + if( pTree==0 ){ + *ppPrevTree = pTree = rowSetEntryAlloc(pRowSet); + if( pTree ){ + pTree->v = 0; + pTree->pRight = 0; + pTree->pLeft = rowSetListToTree(p); + } + } + pRowSet->pEntry = 0; + pRowSet->pLast = 0; + pRowSet->rsFlags |= ROWSET_SORTED; + } + pRowSet->iBatch = iBatch; + } + + /* Test to see if the iRowid value appears anywhere in the forest. + ** Return 1 if it does and 0 if not. + */ + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + p = pTree->pLeft; + while( p ){ + if( p->vpRight; + }else if( p->v>iRowid ){ + p = p->pLeft; + }else{ + return 1; + } + } + } + return 0; +} + +/************** End of rowset.c **********************************************/ +/************** Begin file pager.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of the page cache subsystem or "pager". +** +** The pager is used to access a database disk file. It implements +** atomic commit and rollback through the use of a journal file that +** is separate from the database file. The pager also implements file +** locking to prevent two processes from writing the same database +** file simultaneously, or one process from reading the database while +** another is writing. +*/ +#ifndef SQLITE_OMIT_DISKIO +/************** Include wal.h in the middle of pager.c ***********************/ +/************** Begin file wal.h *********************************************/ +/* +** 2010 February 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface to the write-ahead logging +** system. Refer to the comments below and the header comment attached to +** the implementation of each function in log.c for further details. +*/ + +#ifndef _WAL_H_ +#define _WAL_H_ + + +/* Additional values that can be added to the sync_flags argument of +** sqlite3WalFrames(): +*/ +#define WAL_SYNC_TRANSACTIONS 0x20 /* Sync at the end of each transaction */ +#define SQLITE_SYNC_MASK 0x13 /* Mask off the SQLITE_SYNC_* values */ + +#ifdef SQLITE_OMIT_WAL +# define sqlite3WalOpen(x,y,z) 0 +# define sqlite3WalLimit(x,y) +# define sqlite3WalClose(w,x,y,z) 0 +# define sqlite3WalBeginReadTransaction(y,z) 0 +# define sqlite3WalEndReadTransaction(z) +# define sqlite3WalDbsize(y) 0 +# define sqlite3WalBeginWriteTransaction(y) 0 +# define sqlite3WalEndWriteTransaction(x) 0 +# define sqlite3WalUndo(x,y,z) 0 +# define sqlite3WalSavepoint(y,z) +# define sqlite3WalSavepointUndo(y,z) 0 +# define sqlite3WalFrames(u,v,w,x,y,z) 0 +# define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0 +# define sqlite3WalCallback(z) 0 +# define sqlite3WalExclusiveMode(y,z) 0 +# define sqlite3WalHeapMemory(z) 0 +# define sqlite3WalFramesize(z) 0 +# define sqlite3WalFindFrame(x,y,z) 0 +#else + +#define WAL_SAVEPOINT_NDATA 4 + +/* Connection to a write-ahead log (WAL) file. +** There is one object of this type for each pager. +*/ +typedef struct Wal Wal; + +/* Open and close a connection to a write-ahead log. */ +SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); +SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); + +/* Set the limiting size of a WAL file. */ +SQLITE_PRIVATE void sqlite3WalLimit(Wal*, i64); + +/* Used by readers to open (lock) and close (unlock) a snapshot. A +** snapshot is like a read-transaction. It is the state of the database +** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and +** preserves the current state even if the other threads or processes +** write to or checkpoint the WAL. sqlite3WalCloseSnapshot() closes the +** transaction and releases the lock. +*/ +SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *); +SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal); + +/* Read a page from the write-ahead log, if it is present. */ +SQLITE_PRIVATE int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +SQLITE_PRIVATE int sqlite3WalReadFrame(Wal *, u32, int, u8 *); + +/* If the WAL is not empty, return the size of the database. */ +SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal); + +/* Obtain or release the WRITER lock. */ +SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal); +SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal); + +/* Undo any frames written (but not committed) to the log */ +SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx); + +/* Return an integer that records the current (uncommitted) write +** position in the WAL */ +SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData); + +/* Move the write position of the WAL back to iFrame. Called in +** response to a ROLLBACK TO command. */ +SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData); + +/* Write a frame or frames to the log. */ +SQLITE_PRIVATE int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); + +/* Copy pages from the log to the database file */ +SQLITE_PRIVATE int sqlite3WalCheckpoint( + Wal *pWal, /* Write-ahead log connection */ + int eMode, /* One of PASSIVE, FULL and RESTART */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of buffer nBuf */ + u8 *zBuf, /* Temporary buffer to use */ + int *pnLog, /* OUT: Number of frames in WAL */ + int *pnCkpt /* OUT: Number of backfilled frames in WAL */ +); + +/* Return the value to pass to a sqlite3_wal_hook callback, the +** number of frames in the WAL at the point of the last commit since +** sqlite3WalCallback() was called. If no commits have occurred since +** the last call, then return 0. +*/ +SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal); + +/* Tell the wal layer that an EXCLUSIVE lock has been obtained (or released) +** by the pager layer on the database file. +*/ +SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op); + +/* Return true if the argument is non-NULL and the WAL module is using +** heap-memory for the wal-index. Otherwise, if the argument is NULL or the +** WAL module is using shared-memory, return false. +*/ +SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); + +#ifdef SQLITE_ENABLE_ZIPVFS +/* If the WAL file is not empty, return the number of bytes of content +** stored in each frame (i.e. the db page-size when the WAL was created). +*/ +SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal); +#endif + +#endif /* ifndef SQLITE_OMIT_WAL */ +#endif /* _WAL_H_ */ + +/************** End of wal.h *************************************************/ +/************** Continuing where we left off in pager.c **********************/ + + +/******************* NOTES ON THE DESIGN OF THE PAGER ************************ +** +** This comment block describes invariants that hold when using a rollback +** journal. These invariants do not apply for journal_mode=WAL, +** journal_mode=MEMORY, or journal_mode=OFF. +** +** Within this comment block, a page is deemed to have been synced +** automatically as soon as it is written when PRAGMA synchronous=OFF. +** Otherwise, the page is not synced until the xSync method of the VFS +** is called successfully on the file containing the page. +** +** Definition: A page of the database file is said to be "overwriteable" if +** one or more of the following are true about the page: +** +** (a) The original content of the page as it was at the beginning of +** the transaction has been written into the rollback journal and +** synced. +** +** (b) The page was a freelist leaf page at the start of the transaction. +** +** (c) The page number is greater than the largest page that existed in +** the database file at the start of the transaction. +** +** (1) A page of the database file is never overwritten unless one of the +** following are true: +** +** (a) The page and all other pages on the same sector are overwriteable. +** +** (b) The atomic page write optimization is enabled, and the entire +** transaction other than the update of the transaction sequence +** number consists of a single page change. +** +** (2) The content of a page written into the rollback journal exactly matches +** both the content in the database when the rollback journal was written +** and the content in the database at the beginning of the current +** transaction. +** +** (3) Writes to the database file are an integer multiple of the page size +** in length and are aligned on a page boundary. +** +** (4) Reads from the database file are either aligned on a page boundary and +** an integer multiple of the page size in length or are taken from the +** first 100 bytes of the database file. +** +** (5) All writes to the database file are synced prior to the rollback journal +** being deleted, truncated, or zeroed. +** +** (6) If a master journal file is used, then all writes to the database file +** are synced prior to the master journal being deleted. +** +** Definition: Two databases (or the same database at two points it time) +** are said to be "logically equivalent" if they give the same answer to +** all queries. Note in particular the content of freelist leaf +** pages can be changed arbitarily without effecting the logical equivalence +** of the database. +** +** (7) At any time, if any subset, including the empty set and the total set, +** of the unsynced changes to a rollback journal are removed and the +** journal is rolled back, the resulting database file will be logical +** equivalent to the database file at the beginning of the transaction. +** +** (8) When a transaction is rolled back, the xTruncate method of the VFS +** is called to restore the database file to the same size it was at +** the beginning of the transaction. (In some VFSes, the xTruncate +** method is a no-op, but that does not change the fact the SQLite will +** invoke it.) +** +** (9) Whenever the database file is modified, at least one bit in the range +** of bytes from 24 through 39 inclusive will be changed prior to releasing +** the EXCLUSIVE lock, thus signaling other connections on the same +** database to flush their caches. +** +** (10) The pattern of bits in bytes 24 through 39 shall not repeat in less +** than one billion transactions. +** +** (11) A database file is well-formed at the beginning and at the conclusion +** of every transaction. +** +** (12) An EXCLUSIVE lock is held on the database file when writing to +** the database file. +** +** (13) A SHARED lock is held on the database file while reading any +** content out of the database file. +** +******************************************************************************/ + +/* +** Macros for troubleshooting. Normally turned off +*/ +#if 0 +int sqlite3PagerTrace=1; /* True to enable tracing */ +#define sqlite3DebugPrintf printf +#define PAGERTRACE(X) if( sqlite3PagerTrace ){ sqlite3DebugPrintf X; } +#else +#define PAGERTRACE(X) +#endif + +/* +** The following two macros are used within the PAGERTRACE() macros above +** to print out file-descriptors. +** +** PAGERID() takes a pointer to a Pager struct as its argument. The +** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file +** struct as its argument. +*/ +#define PAGERID(p) ((int)(p->fd)) +#define FILEHANDLEID(fd) ((int)fd) + +/* +** The Pager.eState variable stores the current 'state' of a pager. A +** pager may be in any one of the seven states shown in the following +** state diagram. +** +** OPEN <------+------+ +** | | | +** V | | +** +---------> READER-------+ | +** | | | +** | V | +** |<-------WRITER_LOCKED------> ERROR +** | | ^ +** | V | +** |<------WRITER_CACHEMOD-------->| +** | | | +** | V | +** |<-------WRITER_DBMOD---------->| +** | | | +** | V | +** +<------WRITER_FINISHED-------->+ +** +** +** List of state transitions and the C [function] that performs each: +** +** OPEN -> READER [sqlite3PagerSharedLock] +** READER -> OPEN [pager_unlock] +** +** READER -> WRITER_LOCKED [sqlite3PagerBegin] +** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal] +** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] +** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] +** WRITER_*** -> READER [pager_end_transaction] +** +** WRITER_*** -> ERROR [pager_error] +** ERROR -> OPEN [pager_unlock] +** +** +** OPEN: +** +** The pager starts up in this state. Nothing is guaranteed in this +** state - the file may or may not be locked and the database size is +** unknown. The database may not be read or written. +** +** * No read or write transaction is active. +** * Any lock, or no lock at all, may be held on the database file. +** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. +** +** READER: +** +** In this state all the requirements for reading the database in +** rollback (non-WAL) mode are met. Unless the pager is (or recently +** was) in exclusive-locking mode, a user-level read transaction is +** open. The database size is known in this state. +** +** A connection running with locking_mode=normal enters this state when +** it opens a read-transaction on the database and returns to state +** OPEN after the read-transaction is completed. However a connection +** running in locking_mode=exclusive (including temp databases) remains in +** this state even after the read-transaction is closed. The only way +** a locking_mode=exclusive connection can transition from READER to OPEN +** is via the ERROR state (see below). +** +** * A read transaction may be active (but a write-transaction cannot). +** * A SHARED or greater lock is held on the database file. +** * The dbSize variable may be trusted (even if a user-level read +** transaction is not active). The dbOrigSize and dbFileSize variables +** may not be trusted at this point. +** * If the database is a WAL database, then the WAL connection is open. +** * Even if a read-transaction is not open, it is guaranteed that +** there is no hot-journal in the file-system. +** +** WRITER_LOCKED: +** +** The pager moves to this state from READER when a write-transaction +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual +** modifications to the cache or database have taken place. +** +** In rollback mode, a RESERVED or (if the transaction was opened with +** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database +** file. +** +** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. +** If the connection is running with locking_mode=exclusive, an attempt +** is made to obtain an EXCLUSIVE lock on the database file. +** +** * A write transaction is active. +** * If the connection is open in rollback-mode, a RESERVED or greater +** lock is held on the database file. +** * If the connection is open in WAL-mode, a WAL write transaction +** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully +** called). +** * The dbSize, dbOrigSize and dbFileSize variables are all valid. +** * The contents of the pager cache have not been modified. +** * The journal file may or may not be open. +** * Nothing (not even the first header) has been written to the journal. +** +** WRITER_CACHEMOD: +** +** A pager moves from WRITER_LOCKED state to this state when a page is +** first modified by the upper layer. In rollback mode the journal file +** is opened (if it is not already open) and a header written to the +** start of it. The database file on disk has not been modified. +** +** * A write transaction is active. +** * A RESERVED or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** to it, but the header has not been synced to disk. +** * The contents of the page cache have been modified. +** +** WRITER_DBMOD: +** +** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state +** when it modifies the contents of the database file. WAL connections +** never enter this state (since they do not modify the database file, +** just the log file). +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** and synced to disk. +** * The contents of the page cache have been modified (and possibly +** written to disk). +** +** WRITER_FINISHED: +** +** It is not possible for a WAL connection to enter this state. +** +** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD +** state after the entire transaction has been successfully written into the +** database file. In this state the transaction may be committed simply +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper +** layer must either commit or rollback the transaction. +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * All writing and syncing of journal and database data has finished. +** If no error occurred, all that remains is to finalize the journal to +** commit the transaction. If an error did occur, the caller will need +** to rollback the transaction. +** +** ERROR: +** +** The ERROR state is entered when an IO or disk-full error (including +** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it +** difficult to be sure that the in-memory pager state (cache contents, +** db size etc.) are consistent with the contents of the file-system. +** +** Temporary pager files may enter the ERROR state, but in-memory pagers +** cannot. +** +** For example, if an IO error occurs while performing a rollback, +** the contents of the page-cache may be left in an inconsistent state. +** At this point it would be dangerous to change back to READER state +** (as usually happens after a rollback). Any subsequent readers might +** report database corruption (due to the inconsistent cache), and if +** they upgrade to writers, they may inadvertently corrupt the database +** file. To avoid this hazard, the pager switches into the ERROR state +** instead of READER following such an error. +** +** Once it has entered the ERROR state, any attempt to use the pager +** to read or write data returns an error. Eventually, once all +** outstanding transactions have been abandoned, the pager is able to +** transition back to OPEN state, discarding the contents of the +** page-cache and any other in-memory state at the same time. Everything +** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** when a read-transaction is next opened on the pager (transitioning +** the pager into READER state). At that point the system has recovered +** from the error. +** +** Specifically, the pager jumps into the ERROR state if: +** +** 1. An error occurs while attempting a rollback. This happens in +** function sqlite3PagerRollback(). +** +** 2. An error occurs while attempting to finalize a journal file +** following a commit in function sqlite3PagerCommitPhaseTwo(). +** +** 3. An error occurs while attempting to write to the journal or +** database file in function pagerStress() in order to free up +** memory. +** +** In other cases, the error is returned to the b-tree layer. The b-tree +** layer then attempts a rollback operation. If the error condition +** persists, the pager enters the ERROR state via condition (1) above. +** +** Condition (3) is necessary because it can be triggered by a read-only +** statement executed within a transaction. In this case, if the error +** code were simply returned to the user, the b-tree layer would not +** automatically attempt a rollback, as it assumes that an error in a +** read-only statement cannot leave the pager in an internally inconsistent +** state. +** +** * The Pager.errCode variable is set to something other than SQLITE_OK. +** * There are one or more outstanding references to pages (after the +** last reference is dropped the pager should move back to OPEN state). +** * The pager is not an in-memory pager. +** +** +** Notes: +** +** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the +** connection is open in WAL mode. A WAL connection is always in one +** of the first four states. +** +** * Normally, a connection open in exclusive mode is never in PAGER_OPEN +** state. There are two exceptions: immediately after exclusive-mode has +** been turned on (and before any read or write transactions are +** executed), and when the pager is leaving the "error state". +** +** * See also: assert_pager_state(). +*/ +#define PAGER_OPEN 0 +#define PAGER_READER 1 +#define PAGER_WRITER_LOCKED 2 +#define PAGER_WRITER_CACHEMOD 3 +#define PAGER_WRITER_DBMOD 4 +#define PAGER_WRITER_FINISHED 5 +#define PAGER_ERROR 6 + +/* +** The Pager.eLock variable is almost always set to one of the +** following locking-states, according to the lock currently held on +** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** This variable is kept up to date as locks are taken and released by +** the pagerLockDb() and pagerUnlockDb() wrappers. +** +** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY +** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not +** the operation was successful. In these circumstances pagerLockDb() and +** pagerUnlockDb() take a conservative approach - eLock is always updated +** when unlocking the file, and only updated when locking the file if the +** VFS call is successful. This way, the Pager.eLock variable may be set +** to a less exclusive (lower) value than the lock that is actually held +** at the system level, but it is never set to a more exclusive value. +** +** This is usually safe. If an xUnlock fails or appears to fail, there may +** be a few redundant xLock() calls or a lock may be held for longer than +** required, but nothing really goes wrong. +** +** The exception is when the database file is unlocked as the pager moves +** from ERROR to OPEN state. At this point there may be a hot-journal file +** in the file-system that needs to be rolled back (as part of a OPEN->SHARED +** transition, by the same pager or any other). If the call to xUnlock() +** fails at this point and the pager is left holding an EXCLUSIVE lock, this +** can confuse the call to xCheckReservedLock() call made later as part +** of hot-journal detection. +** +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may +** return true because the caller itself is holding an EXCLUSIVE lock (but +** doesn't know it because of a previous error in xUnlock). If this happens +** a hot-journal may be mistaken for a journal being created by an active +** transaction in another process, causing SQLite to read from the database +** without rolling it back. +** +** To work around this, if a call to xUnlock() fails when unlocking the +** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It +** is only changed back to a real locking state after a successful call +** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE +** lock on the database file before attempting to roll it back. See function +** PagerSharedLock() for more detail. +** +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** PAGER_OPEN state. +*/ +#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) + +/* +** A macro used for invoking the codec if there is one +*/ +#ifdef SQLITE_HAS_CODEC +# define CODEC1(P,D,N,X,E) \ + if( P->xCodec && P->xCodec(P->pCodec,D,N,X)==0 ){ E; } +# define CODEC2(P,D,N,X,E,O) \ + if( P->xCodec==0 ){ O=(char*)D; }else \ + if( (O=(char*)(P->xCodec(P->pCodec,D,N,X)))==0 ){ E; } +#else +# define CODEC1(P,D,N,X,E) /* NO-OP */ +# define CODEC2(P,D,N,X,E,O) O=(char*)D +#endif + +/* +** The maximum allowed sector size. 64KiB. If the xSectorsize() method +** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. +** This could conceivably cause corruption following a power failure on +** such a system. This is currently an undocumented limit. +*/ +#define MAX_SECTOR_SIZE 0x10000 + +/* +** An instance of the following structure is allocated for each active +** savepoint and statement transaction in the system. All such structures +** are stored in the Pager.aSavepoint[] array, which is allocated and +** resized using sqlite3Realloc(). +** +** When a savepoint is created, the PagerSavepoint.iHdrOffset field is +** set to 0. If a journal-header is written into the main journal while +** the savepoint is active, then iHdrOffset is set to the byte offset +** immediately following the last journal record written into the main +** journal before the journal-header. This is required during savepoint +** rollback (see pagerPlaybackSavepoint()). +*/ +typedef struct PagerSavepoint PagerSavepoint; +struct PagerSavepoint { + i64 iOffset; /* Starting offset in main journal */ + i64 iHdrOffset; /* See above */ + Bitvec *pInSavepoint; /* Set of pages in this savepoint */ + Pgno nOrig; /* Original number of pages in file */ + Pgno iSubRec; /* Index of first record in sub-journal */ +#ifndef SQLITE_OMIT_WAL + u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */ +#endif +}; + +/* +** Bits of the Pager.doNotSpill flag. See further description below. +*/ +#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ +#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ +#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ + +/* +** A open page cache is an instance of struct Pager. A description of +** some of the more important member variables follows: +** +** eState +** +** The current 'state' of the pager object. See the comment and state +** diagram above for a description of the pager state. +** +** eLock +** +** For a real on-disk database, the current lock held on the database file - +** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** +** For a temporary or in-memory database (neither of which require any +** locks), this variable is always set to EXCLUSIVE_LOCK. Since such +** databases always have Pager.exclusiveMode==1, this tricks the pager +** logic into thinking that it already has all the locks it will ever +** need (and no reason to release them). +** +** In some (obscure) circumstances, this variable may also be set to +** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for +** details. +** +** changeCountDone +** +** This boolean variable is used to make sure that the change-counter +** (the 4-byte header field at byte offset 24 of the database file) is +** not updated more often than necessary. +** +** It is set to true when the change-counter field is updated, which +** can only happen if an exclusive lock is held on the database file. +** It is cleared (set to false) whenever an exclusive lock is +** relinquished on the database file. Each time a transaction is committed, +** The changeCountDone flag is inspected. If it is true, the work of +** updating the change-counter is omitted for the current transaction. +** +** This mechanism means that when running in exclusive mode, a connection +** need only update the change-counter once, for the first transaction +** committed. +** +** setMaster +** +** When PagerCommitPhaseOne() is called to commit a transaction, it may +** (or may not) specify a master-journal name to be written into the +** journal file before it is synced to disk. +** +** Whether or not a journal file contains a master-journal pointer affects +** the way in which the journal file is finalized after the transaction is +** committed or rolled back when running in "journal_mode=PERSIST" mode. +** If a journal file does not contain a master-journal pointer, it is +** finalized by overwriting the first journal header with zeroes. If +** it does contain a master-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were +** running in "journal_mode=truncate" mode. +** +** Journal files that contain master journal pointers cannot be finalized +** simply by overwriting the first journal-header with zeroes, as the +** master journal pointer could interfere with hot-journal rollback of any +** subsequently interrupted transaction that reuses the journal file. +** +** The flag is cleared as soon as the journal file is finalized (either +** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the +** journal file from being successfully finalized, the setMaster flag +** is cleared anyway (and the pager will move to ERROR state). +** +** doNotSpill +** +** This variables control the behavior of cache-spills (calls made by +** the pcache module to the pagerStress() routine to write cached data +** to the file-system in order to free up memory). +** +** When bits SPILLFLAG_OFF or SPILLFLAG_ROLLBACK of doNotSpill are set, +** writing to the database from pagerStress() is disabled altogether. +** The SPILLFLAG_ROLLBACK case is done in a very obscure case that +** comes up during savepoint rollback that requires the pcache module +** to allocate a new page to prevent the journal file from being written +** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF +** case is a user preference. +** +** If the SPILLFLAG_NOSYNC bit is set, writing to the database from pagerStress() +** is permitted, but syncing the journal file is not. This flag is set +** by sqlite3PagerWrite() when the file-system sector-size is larger than +** the database page-size in order to prevent a journal sync from happening +** in between the journalling of two pages on the same sector. +** +** subjInMemory +** +** This is a boolean variable. If true, then any required sub-journal +** is opened as an in-memory journal file. If false, then in-memory +** sub-journals are only used for in-memory pager files. +** +** This variable is updated by the upper layer each time a new +** write-transaction is opened. +** +** dbSize, dbOrigSize, dbFileSize +** +** Variable dbSize is set to the number of pages in the database file. +** It is valid in PAGER_READER and higher states (all states except for +** OPEN and ERROR). +** +** dbSize is set based on the size of the database file, which may be +** larger than the size of the database (the value stored at offset +** 28 of the database header by the btree). If the size of the file +** is not an integer multiple of the page-size, the value stored in +** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2). +** Except, any file that is greater than 0 bytes in size is considered +** to have at least one page. (i.e. a 1KB file with 2K page-size leads +** to dbSize==1). +** +** During a write-transaction, if pages with page-numbers greater than +** dbSize are modified in the cache, dbSize is updated accordingly. +** Similarly, if the database is truncated using PagerTruncateImage(), +** dbSize is updated. +** +** Variables dbOrigSize and dbFileSize are valid in states +** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize +** variable at the start of the transaction. It is used during rollback, +** and to determine whether or not pages need to be journalled before +** being modified. +** +** Throughout a write-transaction, dbFileSize contains the size of +** the file on disk in pages. It is set to a copy of dbSize when the +** write-transaction is first opened, and updated when VFS calls are made +** to write or truncate the database file on disk. +** +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), +** pager_truncate() is called. The pager_truncate() call uses xFilesize() +** to measure the database file on disk, and then truncates it if required. +** dbFileSize is not used when rolling back a transaction. In this case +** pager_truncate() is called unconditionally (which means there may be +** a call to xFilesize() that is not strictly required). In either case, +** pager_truncate() may cause the file to become smaller or larger. +** +** dbHintSize +** +** The dbHintSize variable is used to limit the number of calls made to +** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** +** dbHintSize is set to a copy of the dbSize variable when a +** write-transaction is opened (at the same time as dbFileSize and +** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, +** dbHintSize is increased to the number of pages that correspond to the +** size-hint passed to the method call. See pager_write_pagelist() for +** details. +** +** errCode +** +** The Pager.errCode variable is only ever used in PAGER_ERROR state. It +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** sub-codes. +*/ +struct Pager { + sqlite3_vfs *pVfs; /* OS functions to use for IO */ + u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ + u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ + u8 useJournal; /* Use a rollback journal on this file */ + u8 noSync; /* Do not sync the journal if true */ + u8 fullSync; /* Do extra syncs of the journal for robustness */ + u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ + u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ + u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ + u8 tempFile; /* zFilename is a temporary or immutable file */ + u8 noLock; /* Do not lock (except in WAL mode) */ + u8 readOnly; /* True for a read-only database */ + u8 memDb; /* True to inhibit all file I/O */ + + /************************************************************************** + ** The following block contains those class members that change during + ** routine opertion. Class members not in this block are either fixed + ** when the pager is first created or else only change when there is a + ** significant mode change (such as changing the page_size, locking_mode, + ** or the journal_mode). From another view, these class members describe + ** the "state" of the pager, while other class members describe the + ** "configuration" of the pager. + */ + u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ + u8 eLock; /* Current lock held on database file */ + u8 changeCountDone; /* Set after incrementing the change-counter */ + u8 setMaster; /* True if a m-j name has been written to jrnl */ + u8 doNotSpill; /* Do not spill the cache when non-zero */ + u8 subjInMemory; /* True to use in-memory sub-journals */ + Pgno dbSize; /* Number of pages in the database */ + Pgno dbOrigSize; /* dbSize before the current transaction */ + Pgno dbFileSize; /* Number of pages in the database file */ + Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */ + int errCode; /* One of several kinds of errors */ + int nRec; /* Pages journalled since last j-header written */ + u32 cksumInit; /* Quasi-random value added to every checksum */ + u32 nSubRec; /* Number of records written to sub-journal */ + Bitvec *pInJournal; /* One bit for each page in the database file */ + sqlite3_file *fd; /* File descriptor for database */ + sqlite3_file *jfd; /* File descriptor for main journal */ + sqlite3_file *sjfd; /* File descriptor for sub-journal */ + i64 journalOff; /* Current write offset in the journal file */ + i64 journalHdr; /* Byte offset to previous journal header */ + sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ + PagerSavepoint *aSavepoint; /* Array of active savepoints */ + int nSavepoint; /* Number of elements in aSavepoint[] */ + char dbFileVers[16]; /* Changes whenever database file changes */ + + u8 bUseFetch; /* True to use xFetch() */ + int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 szMmap; /* Desired maximum mmap size */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ + /* + ** End of the routinely-changing class members + ***************************************************************************/ + + u16 nExtra; /* Add this many bytes to each in-memory page */ + i16 nReserve; /* Number of unused bytes at end of each page */ + u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ + u32 sectorSize; /* Assumed sector size during rollback */ + int pageSize; /* Number of bytes in a page */ + Pgno mxPgno; /* Maximum allowed size of the database */ + i64 journalSizeLimit; /* Size limit for persistent journal files */ + char *zFilename; /* Name of the database file */ + char *zJournal; /* Name of the journal file */ + int (*xBusyHandler)(void*); /* Function to call when busy */ + void *pBusyHandlerArg; /* Context argument for xBusyHandler */ + int aStat[3]; /* Total cache hits, misses and writes */ +#ifdef SQLITE_TEST + int nRead; /* Database pages read */ +#endif + void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ +#ifdef SQLITE_HAS_CODEC + void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ + void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */ + void (*xCodecFree)(void*); /* Destructor for the codec */ + void *pCodec; /* First argument to xCodec... methods */ +#endif + char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ + PCache *pPCache; /* Pointer to page cache object */ +#ifndef SQLITE_OMIT_WAL + Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ + char *zWal; /* File name for write-ahead log */ +#endif +}; + +/* +** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** or CACHE_WRITE to sqlite3_db_status(). +*/ +#define PAGER_STAT_HIT 0 +#define PAGER_STAT_MISS 1 +#define PAGER_STAT_WRITE 2 + +/* +** The following global variables hold counters used for +** testing purposes only. These variables do not exist in +** a non-testing build. These variables are not thread-safe. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_pager_readdb_count = 0; /* Number of full pages read from DB */ +SQLITE_API int sqlite3_pager_writedb_count = 0; /* Number of full pages written to DB */ +SQLITE_API int sqlite3_pager_writej_count = 0; /* Number of pages written to journal */ +# define PAGER_INCR(v) v++ +#else +# define PAGER_INCR(v) +#endif + + + +/* +** Journal files begin with the following magic string. The data +** was obtained from /dev/random. It is used only as a sanity check. +** +** Since version 2.8.0, the journal format contains additional sanity +** checking information. If the power fails while the journal is being +** written, semi-random garbage data might appear in the journal +** file after power is restored. If an attempt is then made +** to roll the journal back, the database could be corrupted. The additional +** sanity checking data is an attempt to discover the garbage in the +** journal and ignore it. +** +** The sanity checking information for the new journal format consists +** of a 32-bit checksum on each page of data. The checksum covers both +** the page number and the pPager->pageSize bytes of data for the page. +** This cksum is initialized to a 32-bit random value that appears in the +** journal file right after the header. The random initializer is important, +** because garbage data that appears at the end of a journal is likely +** data that was once in other files that have now been deleted. If the +** garbage data came from an obsolete journal file, the checksums might +** be correct. But by initializing the checksum to random value which +** is different for every journal, we minimize that risk. +*/ +static const unsigned char aJournalMagic[] = { + 0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd7, +}; + +/* +** The size of the of each page record in the journal is given by +** the following macro. +*/ +#define JOURNAL_PG_SZ(pPager) ((pPager->pageSize) + 8) + +/* +** The journal header size for this pager. This is usually the same +** size as a single disk sector. See also setSectorSize(). +*/ +#define JOURNAL_HDR_SZ(pPager) (pPager->sectorSize) + +/* +** The macro MEMDB is true if we are dealing with an in-memory database. +** We do this as a macro so that if the SQLITE_OMIT_MEMORYDB macro is set, +** the value of MEMDB will be a constant and the compiler will optimize +** out code that would never execute. +*/ +#ifdef SQLITE_OMIT_MEMORYDB +# define MEMDB 0 +#else +# define MEMDB pPager->memDb +#endif + +/* +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch +** interfaces to access the database using memory-mapped I/O. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +# define USEFETCH(x) ((x)->bUseFetch) +#else +# define USEFETCH(x) 0 +#endif + +/* +** The maximum legal page number is (2^31 - 1). +*/ +#define PAGER_MAX_PGNO 2147483647 + +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods) + +/* +** Return true if this pager uses a write-ahead log instead of the usual +** rollback journal. Otherwise false. +*/ +#ifndef SQLITE_OMIT_WAL +static int pagerUseWal(Pager *pPager){ + return (pPager->pWal!=0); +} +#else +# define pagerUseWal(x) 0 +# define pagerRollbackWal(x) 0 +# define pagerWalFrames(v,w,x,y) 0 +# define pagerOpenWalIfPresent(z) SQLITE_OK +# define pagerBeginReadTransaction(z) SQLITE_OK +#endif + +#ifndef NDEBUG +/* +** Usage: +** +** assert( assert_pager_state(pPager) ); +** +** This function runs many asserts to try to find inconsistencies in +** the internal state of the Pager object. +*/ +static int assert_pager_state(Pager *p){ + Pager *pPager = p; + + /* State must be valid. */ + assert( p->eState==PAGER_OPEN + || p->eState==PAGER_READER + || p->eState==PAGER_WRITER_LOCKED + || p->eState==PAGER_WRITER_CACHEMOD + || p->eState==PAGER_WRITER_DBMOD + || p->eState==PAGER_WRITER_FINISHED + || p->eState==PAGER_ERROR + ); + + /* Regardless of the current state, a temp-file connection always behaves + ** as if it has an exclusive lock on the database file. It never updates + ** the change-counter field, so the changeCountDone flag is always set. + */ + assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); + assert( p->tempFile==0 || pPager->changeCountDone ); + + /* If the useJournal flag is clear, the journal-mode must be "OFF". + ** And if the journal-mode is "OFF", the journal file must not be open. + */ + assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); + assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); + + /* Check that MEMDB implies noSync. And an in-memory journal. Since + ** this means an in-memory pager performs no IO at all, it cannot encounter + ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing + ** a journal file. (although the in-memory journal implementation may + ** return SQLITE_IOERR_NOMEM while the journal file is being written). It + ** is therefore not possible for an in-memory pager to enter the ERROR + ** state. + */ + if( MEMDB ){ + assert( p->noSync ); + assert( p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_MEMORY + ); + assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); + assert( pagerUseWal(p)==0 ); + } + + /* If changeCountDone is set, a RESERVED lock or greater must be held + ** on the file. + */ + assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); + assert( p->eLock!=PENDING_LOCK ); + + switch( p->eState ){ + case PAGER_OPEN: + assert( !MEMDB ); + assert( pPager->errCode==SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); + break; + + case PAGER_READER: + assert( pPager->errCode==SQLITE_OK ); + assert( p->eLock!=UNKNOWN_LOCK ); + assert( p->eLock>=SHARED_LOCK ); + break; + + case PAGER_WRITER_LOCKED: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + assert( p->eLock>=RESERVED_LOCK ); + } + assert( pPager->dbSize==pPager->dbOrigSize ); + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + assert( pPager->setMaster==0 ); + break; + + case PAGER_WRITER_CACHEMOD: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + /* It is possible that if journal_mode=wal here that neither the + ** journal file nor the WAL file are open. This happens during + ** a rollback transaction that switches from journal_mode=off + ** to journal_mode=wal. + */ + assert( p->eLock>=RESERVED_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + } + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + break; + + case PAGER_WRITER_DBMOD: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + assert( pPager->dbOrigSize<=pPager->dbHintSize ); + break; + + case PAGER_WRITER_FINISHED: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + break; + + case PAGER_ERROR: + /* There must be at least one outstanding reference to the pager if + ** in ERROR state. Otherwise the pager should have already dropped + ** back to OPEN state. + */ + assert( pPager->errCode!=SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + break; + } + + return 1; +} +#endif /* ifndef NDEBUG */ + +#ifdef SQLITE_DEBUG +/* +** Return a pointer to a human readable string in a static buffer +** containing the state of the Pager object passed as an argument. This +** is intended to be used within debuggers. For example, as an alternative +** to "print *pPager" in gdb: +** +** (gdb) printf "%s", print_pager_state(pPager) +*/ +static char *print_pager_state(Pager *p){ + static char zRet[1024]; + + sqlite3_snprintf(1024, zRet, + "Filename: %s\n" + "State: %s errCode=%d\n" + "Lock: %s\n" + "Locking mode: locking_mode=%s\n" + "Journal mode: journal_mode=%s\n" + "Backing store: tempFile=%d memDb=%d useJournal=%d\n" + "Journal: journalOff=%lld journalHdr=%lld\n" + "Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n" + , p->zFilename + , p->eState==PAGER_OPEN ? "OPEN" : + p->eState==PAGER_READER ? "READER" : + p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" : + p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : + p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : + p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : + p->eState==PAGER_ERROR ? "ERROR" : "?error?" + , (int)p->errCode + , p->eLock==NO_LOCK ? "NO_LOCK" : + p->eLock==RESERVED_LOCK ? "RESERVED" : + p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : + p->eLock==SHARED_LOCK ? "SHARED" : + p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?" + , p->exclusiveMode ? "exclusive" : "normal" + , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : + p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : + p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" : + p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" : + p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : + p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" + , (int)p->tempFile, (int)p->memDb, (int)p->useJournal + , p->journalOff, p->journalHdr + , (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize + ); + + return zRet; +} +#endif + +/* +** Return true if it is necessary to write page *pPg into the sub-journal. +** A page needs to be written into the sub-journal if there exists one +** or more open savepoints for which: +** +** * The page-number is less than or equal to PagerSavepoint.nOrig, and +** * The bit corresponding to the page-number is not set in +** PagerSavepoint.pInSavepoint. +*/ +static int subjRequiresPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + PagerSavepoint *p; + Pgno pgno = pPg->pgno; + int i; + for(i=0; inSavepoint; i++){ + p = &pPager->aSavepoint[i]; + if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){ + return 1; + } + } + return 0; +} + +/* +** Return true if the page is already in the journal file. +*/ +static int pageInJournal(Pager *pPager, PgHdr *pPg){ + return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno); +} + +/* +** Read a 32-bit integer from the given file descriptor. Store the integer +** that is read in *pRes. Return SQLITE_OK if everything worked, or an +** error code is something goes wrong. +** +** All values are stored on disk as big-endian. +*/ +static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){ + unsigned char ac[4]; + int rc = sqlite3OsRead(fd, ac, sizeof(ac), offset); + if( rc==SQLITE_OK ){ + *pRes = sqlite3Get4byte(ac); + } + return rc; +} + +/* +** Write a 32-bit integer into a string buffer in big-endian byte order. +*/ +#define put32bits(A,B) sqlite3Put4byte((u8*)A,B) + + +/* +** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK +** on success or an error code is something goes wrong. +*/ +static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ + char ac[4]; + put32bits(ac, val); + return sqlite3OsWrite(fd, ac, 4, offset); +} + +/* +** Unlock the database file to level eLock, which must be either NO_LOCK +** or SHARED_LOCK. Regardless of whether or not the call to xUnlock() +** succeeds, set the Pager.eLock variable to match the (attempted) new lock. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it. See the comment above the #define of +** UNKNOWN_LOCK for an explanation of this. +*/ +static int pagerUnlockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( !pPager->exclusiveMode || pPager->eLock==eLock ); + assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); + if( isOpen(pPager->fd) ){ + assert( pPager->eLock>=eLock ); + rc = pPager->noLock ? SQLITE_OK : sqlite3OsUnlock(pPager->fd, eLock); + if( pPager->eLock!=UNKNOWN_LOCK ){ + pPager->eLock = (u8)eLock; + } + IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) + } + return rc; +} + +/* +** Lock the database file to level eLock, which must be either SHARED_LOCK, +** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the +** Pager.eLock variable to the new locking state. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation +** of this. +*/ +static int pagerLockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); + if( pPager->eLockeLock==UNKNOWN_LOCK ){ + rc = pPager->noLock ? SQLITE_OK : sqlite3OsLock(pPager->fd, eLock); + if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ + pPager->eLock = (u8)eLock; + IOTRACE(("LOCK %p %d\n", pPager, eLock)) + } + } + return rc; +} + +/* +** This function determines whether or not the atomic-write optimization +** can be used with this pager. The optimization can be used if: +** +** (a) the value returned by OsDeviceCharacteristics() indicates that +** a database page may be written atomically, and +** (b) the value returned by OsSectorSize() is less than or equal +** to the page size. +** +** The optimization is also always enabled for temporary files. It is +** an error to call this function if pPager is opened on an in-memory +** database. +** +** If the optimization cannot be used, 0 is returned. If it can be used, +** then the value returned is the size of the journal file when it +** contains rollback data for exactly one page. +*/ +#ifdef SQLITE_ENABLE_ATOMIC_WRITE +static int jrnlBufferSize(Pager *pPager){ + assert( !MEMDB ); + if( !pPager->tempFile ){ + int dc; /* Device characteristics */ + int nSector; /* Sector size */ + int szPage; /* Page size */ + + assert( isOpen(pPager->fd) ); + dc = sqlite3OsDeviceCharacteristics(pPager->fd); + nSector = pPager->sectorSize; + szPage = pPager->pageSize; + + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + if( 0==(dc&(SQLITE_IOCAP_ATOMIC|(szPage>>8)) || nSector>szPage) ){ + return 0; + } + } + + return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); +} +#endif + +/* +** If SQLITE_CHECK_PAGES is defined then we do some sanity checking +** on the cache using a hash function. This is used for testing +** and debugging only. +*/ +#ifdef SQLITE_CHECK_PAGES +/* +** Return a 32-bit hash of the page data for pPage. +*/ +static u32 pager_datahash(int nByte, unsigned char *pData){ + u32 hash = 0; + int i; + for(i=0; ipPager->pageSize, (unsigned char *)pPage->pData); +} +static void pager_set_pagehash(PgHdr *pPage){ + pPage->pageHash = pager_pagehash(pPage); +} + +/* +** The CHECK_PAGE macro takes a PgHdr* as an argument. If SQLITE_CHECK_PAGES +** is defined, and NDEBUG is not defined, an assert() statement checks +** that the page is either dirty or still matches the calculated page-hash. +*/ +#define CHECK_PAGE(x) checkPage(x) +static void checkPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + assert( pPager->eState!=PAGER_ERROR ); + assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); +} + +#else +#define pager_datahash(X,Y) 0 +#define pager_pagehash(X) 0 +#define pager_set_pagehash(X) +#define CHECK_PAGE(x) +#endif /* SQLITE_CHECK_PAGES */ + +/* +** When this is called the journal file for pager pPager must be open. +** This function attempts to read a master journal file name from the +** end of the file and, if successful, copies it into memory supplied +** by the caller. See comments above writeMasterJournal() for the format +** used to store a master journal file name at the end of a journal file. +** +** zMaster must point to a buffer of at least nMaster bytes allocated by +** the caller. This should be sqlite3_vfs.mxPathname+1 (to ensure there is +** enough space to write the master journal name). If the master journal +** name in the journal is longer than nMaster bytes (including a +** nul-terminator), then this is handled as if no master journal name +** were present in the journal. +** +** If a master journal file name is present at the end of the journal +** file, then it is copied into the buffer pointed to by zMaster. A +** nul-terminator byte is appended to the buffer following the master +** journal file name. +** +** If it is determined that no master journal file name is present +** zMaster[0] is set to 0 and SQLITE_OK returned. +** +** If an error occurs while reading from the journal file, an SQLite +** error code is returned. +*/ +static int readMasterJournal(sqlite3_file *pJrnl, char *zMaster, u32 nMaster){ + int rc; /* Return code */ + u32 len; /* Length in bytes of master journal name */ + i64 szJ; /* Total size in bytes of journal file pJrnl */ + u32 cksum; /* MJ checksum value read from journal */ + u32 u; /* Unsigned loop counter */ + unsigned char aMagic[8]; /* A buffer to hold the magic header */ + zMaster[0] = '\0'; + + if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) + || szJ<16 + || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) + || len>=nMaster + || len==0 + || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) + || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) + || memcmp(aMagic, aJournalMagic, 8) + || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zMaster, len, szJ-16-len)) + ){ + return rc; + } + + /* See if the checksum matches the master journal name */ + for(u=0; ujournalOff, assuming a sector +** size of pPager->sectorSize bytes. +** +** i.e for a sector size of 512: +** +** Pager.journalOff Return value +** --------------------------------------- +** 0 0 +** 512 512 +** 100 512 +** 2000 2048 +** +*/ +static i64 journalHdrOffset(Pager *pPager){ + i64 offset = 0; + i64 c = pPager->journalOff; + if( c ){ + offset = ((c-1)/JOURNAL_HDR_SZ(pPager) + 1) * JOURNAL_HDR_SZ(pPager); + } + assert( offset%JOURNAL_HDR_SZ(pPager)==0 ); + assert( offset>=c ); + assert( (offset-c)jfd) ); + if( pPager->journalOff ){ + const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ + + IOTRACE(("JZEROHDR %p\n", pPager)) + if( doTruncate || iLimit==0 ){ + rc = sqlite3OsTruncate(pPager->jfd, 0); + }else{ + static const char zeroHdr[28] = {0}; + rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0); + } + if( rc==SQLITE_OK && !pPager->noSync ){ + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags); + } + + /* At this point the transaction is committed but the write lock + ** is still held on the file. If there is a size limit configured for + ** the persistent journal and the journal file currently consumes more + ** space than that limit allows for, truncate it now. There is no need + ** to sync the file following this operation. + */ + if( rc==SQLITE_OK && iLimit>0 ){ + i64 sz; + rc = sqlite3OsFileSize(pPager->jfd, &sz); + if( rc==SQLITE_OK && sz>iLimit ){ + rc = sqlite3OsTruncate(pPager->jfd, iLimit); + } + } + } + return rc; +} + +/* +** The journal file must be open when this routine is called. A journal +** header (JOURNAL_HDR_SZ bytes) is written into the journal file at the +** current location. +** +** The format for the journal header is as follows: +** - 8 bytes: Magic identifying journal format. +** - 4 bytes: Number of records in journal, or -1 no-sync mode is on. +** - 4 bytes: Random number used for page hash. +** - 4 bytes: Initial database page count. +** - 4 bytes: Sector size used by the process that wrote this journal. +** - 4 bytes: Database page size. +** +** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space. +*/ +static int writeJournalHdr(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + char *zHeader = pPager->pTmpSpace; /* Temporary space used to build header */ + u32 nHeader = (u32)pPager->pageSize;/* Size of buffer pointed to by zHeader */ + u32 nWrite; /* Bytes of header sector written */ + int ii; /* Loop counter */ + + assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ + + if( nHeader>JOURNAL_HDR_SZ(pPager) ){ + nHeader = JOURNAL_HDR_SZ(pPager); + } + + /* If there are active savepoints and any of them were created + ** since the most recent journal header was written, update the + ** PagerSavepoint.iHdrOffset fields now. + */ + for(ii=0; iinSavepoint; ii++){ + if( pPager->aSavepoint[ii].iHdrOffset==0 ){ + pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff; + } + } + + pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager); + + /* + ** Write the nRec Field - the number of page records that follow this + ** journal header. Normally, zero is written to this value at this time. + ** After the records are added to the journal (and the journal synced, + ** if in full-sync mode), the zero is overwritten with the true number + ** of records (see syncJournal()). + ** + ** A faster alternative is to write 0xFFFFFFFF to the nRec field. When + ** reading the journal this value tells SQLite to assume that the + ** rest of the journal file contains valid page records. This assumption + ** is dangerous, as if a failure occurred whilst writing to the journal + ** file it may contain some garbage data. There are two scenarios + ** where this risk can be ignored: + ** + ** * When the pager is in no-sync mode. Corruption can follow a + ** power failure in this case anyway. + ** + ** * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees + ** that garbage data is never appended to the journal file. + */ + assert( isOpen(pPager->fd) || pPager->noSync ); + if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + ){ + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff); + }else{ + memset(zHeader, 0, sizeof(aJournalMagic)+4); + } + + /* The random check-hash initializer */ + sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); + /* The initial database size */ + put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); + /* The assumed sector size for this process */ + put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize); + + /* The page size */ + put32bits(&zHeader[sizeof(aJournalMagic)+16], pPager->pageSize); + + /* Initializing the tail of the buffer is not necessary. Everything + ** works find if the following memset() is omitted. But initializing + ** the memory prevents valgrind from complaining, so we are willing to + ** take the performance hit. + */ + memset(&zHeader[sizeof(aJournalMagic)+20], 0, + nHeader-(sizeof(aJournalMagic)+20)); + + /* In theory, it is only necessary to write the 28 bytes that the + ** journal header consumes to the journal file here. Then increment the + ** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next + ** record is written to the following sector (leaving a gap in the file + ** that will be implicitly filled in by the OS). + ** + ** However it has been discovered that on some systems this pattern can + ** be significantly slower than contiguously writing data to the file, + ** even if that means explicitly writing data to the block of + ** (JOURNAL_HDR_SZ - 28) bytes that will not be used. So that is what + ** is done. + ** + ** The loop is required here in case the sector-size is larger than the + ** database page size. Since the zHeader buffer is only Pager.pageSize + ** bytes in size, more than one call to sqlite3OsWrite() may be required + ** to populate the entire journal header sector. + */ + for(nWrite=0; rc==SQLITE_OK&&nWritejournalHdr, nHeader)) + rc = sqlite3OsWrite(pPager->jfd, zHeader, nHeader, pPager->journalOff); + assert( pPager->journalHdr <= pPager->journalOff ); + pPager->journalOff += nHeader; + } + + return rc; +} + +/* +** The journal file must be open when this is called. A journal header file +** (JOURNAL_HDR_SZ bytes) is read from the current location in the journal +** file. The current location in the journal file is given by +** pPager->journalOff. See comments above function writeJournalHdr() for +** a description of the journal header format. +** +** If the header is read successfully, *pNRec is set to the number of +** page records following this header and *pDbSize is set to the size of the +** database before the transaction began, in pages. Also, pPager->cksumInit +** is set to the value read from the journal header. SQLITE_OK is returned +** in this case. +** +** If the journal header file appears to be corrupted, SQLITE_DONE is +** returned and *pNRec and *PDbSize are undefined. If JOURNAL_HDR_SZ bytes +** cannot be read from the journal file an error code is returned. +*/ +static int readJournalHdr( + Pager *pPager, /* Pager object */ + int isHot, + i64 journalSize, /* Size of the open journal file in bytes */ + u32 *pNRec, /* OUT: Value read from the nRec field */ + u32 *pDbSize /* OUT: Value of original database size field */ +){ + int rc; /* Return code */ + unsigned char aMagic[8]; /* A buffer to hold the magic header */ + i64 iHdrOff; /* Offset of journal header being read */ + + assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ + + /* Advance Pager.journalOff to the start of the next sector. If the + ** journal file is too small for there to be a header stored at this + ** point, return SQLITE_DONE. + */ + pPager->journalOff = journalHdrOffset(pPager); + if( pPager->journalOff+JOURNAL_HDR_SZ(pPager) > journalSize ){ + return SQLITE_DONE; + } + iHdrOff = pPager->journalOff; + + /* Read in the first 8 bytes of the journal header. If they do not match + ** the magic string found at the start of each journal header, return + ** SQLITE_DONE. If an IO error occurs, return an error code. Otherwise, + ** proceed. + */ + if( isHot || iHdrOff!=pPager->journalHdr ){ + rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), iHdrOff); + if( rc ){ + return rc; + } + if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))!=0 ){ + return SQLITE_DONE; + } + } + + /* Read the first three 32-bit fields of the journal header: The nRec + ** field, the checksum-initializer and the database size at the start + ** of the transaction. Return an error code if anything goes wrong. + */ + if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+8, pNRec)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+12, &pPager->cksumInit)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+16, pDbSize)) + ){ + return rc; + } + + if( pPager->journalOff==0 ){ + u32 iPageSize; /* Page-size field of journal header */ + u32 iSectorSize; /* Sector-size field of journal header */ + + /* Read the page-size and sector-size journal header fields. */ + if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+20, &iSectorSize)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+24, &iPageSize)) + ){ + return rc; + } + + /* Versions of SQLite prior to 3.5.8 set the page-size field of the + ** journal header to zero. In this case, assume that the Pager.pageSize + ** variable is already set to the correct page size. + */ + if( iPageSize==0 ){ + iPageSize = pPager->pageSize; + } + + /* Check that the values read from the page-size and sector-size fields + ** are within range. To be 'in range', both values need to be a power + ** of two greater than or equal to 512 or 32, and not greater than their + ** respective compile time maximum limits. + */ + if( iPageSize<512 || iSectorSize<32 + || iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE + || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 + ){ + /* If the either the page-size or sector-size in the journal-header is + ** invalid, then the process that wrote the journal-header must have + ** crashed before the header was synced. In this case stop reading + ** the journal file here. + */ + return SQLITE_DONE; + } + + /* Update the page-size to match the value read from the journal. + ** Use a testcase() macro to make sure that malloc failure within + ** PagerSetPagesize() is tested. + */ + rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); + testcase( rc!=SQLITE_OK ); + + /* Update the assumed sector-size to match the value used by + ** the process that created this journal. If this journal was + ** created by a process other than this one, then this routine + ** is being called from within pager_playback(). The local value + ** of Pager.sectorSize is restored at the end of that routine. + */ + pPager->sectorSize = iSectorSize; + } + + pPager->journalOff += JOURNAL_HDR_SZ(pPager); + return rc; +} + + +/* +** Write the supplied master journal name into the journal file for pager +** pPager at the current location. The master journal name must be the last +** thing written to a journal file. If the pager is in full-sync mode, the +** journal file descriptor is advanced to the next sector boundary before +** anything is written. The format is: +** +** + 4 bytes: PAGER_MJ_PGNO. +** + N bytes: Master journal filename in utf-8. +** + 4 bytes: N (length of master journal name in bytes, no nul-terminator). +** + 4 bytes: Master journal name checksum. +** + 8 bytes: aJournalMagic[]. +** +** The master journal page checksum is the sum of the bytes in the master +** journal name, where each byte is interpreted as a signed 8-bit integer. +** +** If zMaster is a NULL pointer (occurs for a single database transaction), +** this call is a no-op. +*/ +static int writeMasterJournal(Pager *pPager, const char *zMaster){ + int rc; /* Return code */ + int nMaster; /* Length of string zMaster */ + i64 iHdrOff; /* Offset of header in journal file */ + i64 jrnlSize; /* Size of journal file on disk */ + u32 cksum = 0; /* Checksum of string zMaster */ + + assert( pPager->setMaster==0 ); + assert( !pagerUseWal(pPager) ); + + if( !zMaster + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || !isOpen(pPager->jfd) + ){ + return SQLITE_OK; + } + pPager->setMaster = 1; + assert( pPager->journalHdr <= pPager->journalOff ); + + /* Calculate the length in bytes and the checksum of zMaster */ + for(nMaster=0; zMaster[nMaster]; nMaster++){ + cksum += zMaster[nMaster]; + } + + /* If in full-sync mode, advance to the next disk sector before writing + ** the master journal name. This is in case the previous page written to + ** the journal has already been synced. + */ + if( pPager->fullSync ){ + pPager->journalOff = journalHdrOffset(pPager); + } + iHdrOff = pPager->journalOff; + + /* Write the master journal data to the end of the journal file. If + ** an error occurs, return the error code to the caller. + */ + if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_MJ_PGNO(pPager)))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, zMaster, nMaster, iHdrOff+4))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster, nMaster))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster+4, cksum))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, iHdrOff+4+nMaster+8))) + ){ + return rc; + } + pPager->journalOff += (nMaster+20); + + /* If the pager is in peristent-journal mode, then the physical + ** journal-file may extend past the end of the master-journal name + ** and 8 bytes of magic data just written to the file. This is + ** dangerous because the code to rollback a hot-journal file + ** will not be able to find the master-journal name to determine + ** whether or not the journal is hot. + ** + ** Easiest thing to do in this scenario is to truncate the journal + ** file to the required size. + */ + if( SQLITE_OK==(rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize)) + && jrnlSize>pPager->journalOff + ){ + rc = sqlite3OsTruncate(pPager->jfd, pPager->journalOff); + } + return rc; +} + +/* +** Find a page in the hash table given its page number. Return +** a pointer to the page or NULL if the requested page is not +** already in memory. +*/ +static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ + PgHdr *p = 0; /* Return value */ + + /* It is not possible for a call to PcacheFetch() with createFlag==0 to + ** fail, since no attempt to allocate dynamic memory will be made. + */ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &p); + return p; +} + +/* +** Discard the entire contents of the in-memory page-cache. +*/ +static void pager_reset(Pager *pPager){ + sqlite3BackupRestart(pPager->pBackup); + sqlite3PcacheClear(pPager->pPCache); +} + +/* +** Free all structures in the Pager.aSavepoint[] array and set both +** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal +** if it is open and the pager is not in exclusive mode. +*/ +static void releaseAllSavepoints(Pager *pPager){ + int ii; /* Iterator for looping through Pager.aSavepoint */ + for(ii=0; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); + } + if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){ + sqlite3OsClose(pPager->sjfd); + } + sqlite3_free(pPager->aSavepoint); + pPager->aSavepoint = 0; + pPager->nSavepoint = 0; + pPager->nSubRec = 0; +} + +/* +** Set the bit number pgno in the PagerSavepoint.pInSavepoint +** bitvecs of all open savepoints. Return SQLITE_OK if successful +** or SQLITE_NOMEM if a malloc failure occurs. +*/ +static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ + int ii; /* Loop counter */ + int rc = SQLITE_OK; /* Result code */ + + for(ii=0; iinSavepoint; ii++){ + PagerSavepoint *p = &pPager->aSavepoint[ii]; + if( pgno<=p->nOrig ){ + rc |= sqlite3BitvecSet(p->pInSavepoint, pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + } + } + return rc; +} + +/* +** This function is a no-op if the pager is in exclusive mode and not +** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN +** state. +** +** If the pager is not in exclusive-access mode, the database file is +** completely unlocked. If the file is unlocked and the file-system does +** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is +** closed (if it is open). +** +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to +** the OPEN state. Regardless of whether the pager is in exclusive-mode +** or not, any journal file left in the file-system will be treated +** as a hot-journal and rolled back the next time a read-transaction +** is opened (by this or by any other connection). +*/ +static void pager_unlock(Pager *pPager){ + + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR + ); + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + releaseAllSavepoints(pPager); + + if( pagerUseWal(pPager) ){ + assert( !isOpen(pPager->jfd) ); + sqlite3WalEndReadTransaction(pPager->pWal); + pPager->eState = PAGER_OPEN; + }else if( !pPager->exclusiveMode ){ + int rc; /* Error code returned by pagerUnlockDb() */ + int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; + + /* If the operating system support deletion of open files, then + ** close the journal file when dropping the database lock. Otherwise + ** another connection with journal_mode=delete might delete the file + ** out from under us. + */ + assert( (PAGER_JOURNALMODE_MEMORY & 5)!=1 ); + assert( (PAGER_JOURNALMODE_OFF & 5)!=1 ); + assert( (PAGER_JOURNALMODE_WAL & 5)!=1 ); + assert( (PAGER_JOURNALMODE_DELETE & 5)!=1 ); + assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); + assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); + if( 0==(iDc & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN) + || 1!=(pPager->journalMode & 5) + ){ + sqlite3OsClose(pPager->jfd); + } + + /* If the pager is in the ERROR state and the call to unlock the database + ** file fails, set the current lock to UNKNOWN_LOCK. See the comment + ** above the #define for UNKNOWN_LOCK for an explanation of why this + ** is necessary. + */ + rc = pagerUnlockDb(pPager, NO_LOCK); + if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ + pPager->eLock = UNKNOWN_LOCK; + } + + /* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here + ** without clearing the error code. This is intentional - the error + ** code is cleared and the cache reset in the block below. + */ + assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + } + + /* If Pager.errCode is set, the contents of the pager cache cannot be + ** trusted. Now that there are no outstanding references to the pager, + ** it can safely move back to PAGER_OPEN state. This happens in both + ** normal and exclusive-locking mode. + */ + if( pPager->errCode ){ + assert( !MEMDB ); + pager_reset(pPager); + pPager->changeCountDone = pPager->tempFile; + pPager->eState = PAGER_OPEN; + pPager->errCode = SQLITE_OK; + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + } + + pPager->journalOff = 0; + pPager->journalHdr = 0; + pPager->setMaster = 0; +} + +/* +** This function is called whenever an IOERR or FULL error that requires +** the pager to transition into the ERROR state may ahve occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. +** +** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the +** IOERR sub-codes, the pager enters the ERROR state and the error code +** is stored in Pager.errCode. While the pager remains in the ERROR state, +** all major API calls on the Pager will immediately return Pager.errCode. +** +** The ERROR state indicates that the contents of the pager-cache +** cannot be trusted. This state can be cleared by completely discarding +** the contents of the pager-cache. If a transaction was active when +** the persistent error occurred, then the rollback journal may need +** to be replayed to restore the contents of the database file (as if +** it were a hot-journal). +*/ +static int pager_error(Pager *pPager, int rc){ + int rc2 = rc & 0xff; + assert( rc==SQLITE_OK || !MEMDB ); + assert( + pPager->errCode==SQLITE_FULL || + pPager->errCode==SQLITE_OK || + (pPager->errCode & 0xff)==SQLITE_IOERR + ); + if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ + pPager->errCode = rc; + pPager->eState = PAGER_ERROR; + } + return rc; +} + +static int pager_truncate(Pager *pPager, Pgno nPage); + +/* +** This routine ends a transaction. A transaction is usually ended by +** either a COMMIT or a ROLLBACK operation. This routine may be called +** after rollback of a hot-journal, or if an error occurs while opening +** the journal file or writing the very first journal-header of a +** database transaction. +** +** This routine is never called in PAGER_ERROR state. If it is called +** in PAGER_NONE or PAGER_SHARED state and the lock held is less +** exclusive than a RESERVED lock, it is a no-op. +** +** Otherwise, any active savepoints are released. +** +** If the journal file is open, then it is "finalized". Once a journal +** file has been finalized it is not possible to use it to roll back a +** transaction. Nor will it be considered to be a hot-journal by this +** or any other database connection. Exactly how a journal is finalized +** depends on whether or not the pager is running in exclusive mode and +** the current journal-mode (Pager.journalMode value), as follows: +** +** journalMode==MEMORY +** Journal file descriptor is simply closed. This destroys an +** in-memory journal. +** +** journalMode==TRUNCATE +** Journal file is truncated to zero bytes in size. +** +** journalMode==PERSIST +** The first 28 bytes of the journal file are zeroed. This invalidates +** the first journal header in the file, and hence the entire journal +** file. An invalid journal file cannot be rolled back. +** +** journalMode==DELETE +** The journal file is closed and deleted using sqlite3OsDelete(). +** +** If the pager is running in exclusive mode, this method of finalizing +** the journal file is never used. Instead, if the journalMode is +** DELETE and the pager is in exclusive mode, the method described under +** journalMode==PERSIST is used instead. +** +** After the journal is finalized, the pager moves to PAGER_READER state. +** If running in non-exclusive rollback mode, the lock on the file is +** downgraded to a SHARED_LOCK. +** +** SQLITE_OK is returned if no error occurs. If an error occurs during +** any of the IO operations to finalize the journal file or unlock the +** database then the IO error code is returned to the user. If the +** operation to finalize the journal file fails, then the code still +** tries to unlock the database file if not in exclusive mode. If the +** unlock operation fails as well, then the first error code related +** to the first error encountered (the journal finalization one) is +** returned. +*/ +static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ + int rc = SQLITE_OK; /* Error code from journal finalization operation */ + int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ + + /* Do nothing if the pager does not have an open write transaction + ** or at least a RESERVED lock. This function may be called when there + ** is no write-transaction active but a RESERVED or greater lock is + ** held under two circumstances: + ** + ** 1. After a successful hot-journal rollback, it is called with + ** eState==PAGER_NONE and eLock==EXCLUSIVE_LOCK. + ** + ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE + ** lock switches back to locking_mode=normal and then executes a + ** read-transaction, this function is called with eState==PAGER_READER + ** and eLock==EXCLUSIVE_LOCK when the read-transaction is closed. + */ + assert( assert_pager_state(pPager) ); + assert( pPager->eState!=PAGER_ERROR ); + if( pPager->eStateeLockjfd) || pPager->pInJournal==0 ); + if( isOpen(pPager->jfd) ){ + assert( !pagerUseWal(pPager) ); + + /* Finalize the journal file. */ + if( sqlite3IsMemJournal(pPager->jfd) ){ + assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); + sqlite3OsClose(pPager->jfd); + }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ + if( pPager->journalOff==0 ){ + rc = SQLITE_OK; + }else{ + rc = sqlite3OsTruncate(pPager->jfd, 0); + } + pPager->journalOff = 0; + }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST + || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) + ){ + rc = zeroJournalHdr(pPager, hasMaster); + pPager->journalOff = 0; + }else{ + /* This branch may be executed with Pager.journalMode==MEMORY if + ** a hot-journal was just rolled back. In this case the journal + ** file should be closed and deleted. If this connection writes to + ** the database file, it will do so using an in-memory journal. + */ + int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd)); + assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + sqlite3OsClose(pPager->jfd); + if( bDelete ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + } + } + +#ifdef SQLITE_CHECK_PAGES + sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); + if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ + PgHdr *p = pager_lookup(pPager, 1); + if( p ){ + p->pageHash = 0; + sqlite3PagerUnrefNotNull(p); + } + } +#endif + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + pPager->nRec = 0; + sqlite3PcacheCleanAll(pPager->pPCache); + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + + if( pagerUseWal(pPager) ){ + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + ** lock held on the database file. + */ + rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); + assert( rc2==SQLITE_OK ); + }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){ + /* This branch is taken when committing a transaction in rollback-journal + ** mode if the database file on disk is larger than the database image. + ** At this point the journal has been finalized and the transaction + ** successfully committed, but the EXCLUSIVE lock is still held on the + ** file. So it is safe to truncate the database file to its minimum + ** required size. */ + assert( pPager->eLock==EXCLUSIVE_LOCK ); + rc = pager_truncate(pPager, pPager->dbSize); + } + + if( rc==SQLITE_OK && bCommit && isOpen(pPager->fd) ){ + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_COMMIT_PHASETWO, 0); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + } + + if( !pPager->exclusiveMode + && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) + ){ + rc2 = pagerUnlockDb(pPager, SHARED_LOCK); + pPager->changeCountDone = 0; + } + pPager->eState = PAGER_READER; + pPager->setMaster = 0; + + return (rc==SQLITE_OK?rc2:rc); +} + +/* +** Execute a rollback if a transaction is active and unlock the +** database file. +** +** If the pager has already entered the ERROR state, do not attempt +** the rollback at this time. Instead, pager_unlock() is called. The +** call to pager_unlock() will discard all in-memory pages, unlock +** the database file and move the pager back to OPEN state. If this +** means that there is a hot-journal left in the file-system, the next +** connection to obtain a shared lock on the pager (which may be this one) +** will roll it back. +** +** If the pager has not already entered the ERROR state, but an IO or +** malloc error occurs during a rollback, then this will itself cause +** the pager to enter the ERROR state. Which will be cleared by the +** call to pager_unlock(), as described above. +*/ +static void pagerUnlockAndRollback(Pager *pPager){ + if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_LOCKED ){ + sqlite3BeginBenignMalloc(); + sqlite3PagerRollback(pPager); + sqlite3EndBenignMalloc(); + }else if( !pPager->exclusiveMode ){ + assert( pPager->eState==PAGER_READER ); + pager_end_transaction(pPager, 0, 0); + } + } + pager_unlock(pPager); +} + +/* +** Parameter aData must point to a buffer of pPager->pageSize bytes +** of data. Compute and return a checksum based ont the contents of the +** page of data and the current value of pPager->cksumInit. +** +** This is not a real checksum. It is really just the sum of the +** random initial value (pPager->cksumInit) and every 200th byte +** of the page data, starting with byte offset (pPager->pageSize%200). +** Each byte is interpreted as an 8-bit unsigned integer. +** +** Changing the formula used to compute this checksum results in an +** incompatible journal file format. +** +** If journal corruption occurs due to a power failure, the most likely +** scenario is that one end or the other of the record will be changed. +** It is much less likely that the two ends of the journal record will be +** correct and the middle be corrupt. Thus, this "checksum" scheme, +** though fast and simple, catches the mostly likely kind of corruption. +*/ +static u32 pager_cksum(Pager *pPager, const u8 *aData){ + u32 cksum = pPager->cksumInit; /* Checksum value to return */ + int i = pPager->pageSize-200; /* Loop counter */ + while( i>0 ){ + cksum += aData[i]; + i -= 200; + } + return cksum; +} + +/* +** Report the current page size and number of reserved bytes back +** to the codec. +*/ +#ifdef SQLITE_HAS_CODEC +static void pagerReportSize(Pager *pPager){ + if( pPager->xCodecSizeChng ){ + pPager->xCodecSizeChng(pPager->pCodec, pPager->pageSize, + (int)pPager->nReserve); + } +} +#else +# define pagerReportSize(X) /* No-op if we do not support a codec */ +#endif + +/* +** Read a single page from either the journal file (if isMainJrnl==1) or +** from the sub-journal (if isMainJrnl==0) and playback that page. +** The page begins at offset *pOffset into the file. The *pOffset +** value is increased to the start of the next page in the journal. +** +** The main rollback journal uses checksums - the statement journal does +** not. +** +** If the page number of the page record read from the (sub-)journal file +** is greater than the current value of Pager.dbSize, then playback is +** skipped and SQLITE_OK is returned. +** +** If pDone is not NULL, then it is a record of pages that have already +** been played back. If the page at *pOffset has already been played back +** (if the corresponding pDone bit is set) then skip the playback. +** Make sure the pDone bit corresponding to the *pOffset page is set +** prior to returning. +** +** If the page record is successfully read from the (sub-)journal file +** and played back, then SQLITE_OK is returned. If an IO error occurs +** while reading the record from the (sub-)journal file or while writing +** to the database file, then the IO error code is returned. If data +** is successfully read from the (sub-)journal file but appears to be +** corrupted, SQLITE_DONE is returned. Data is considered corrupted in +** two circumstances: +** +** * If the record page-number is illegal (0 or PAGER_MJ_PGNO), or +** * If the record is being rolled back from the main journal file +** and the checksum field does not match the record content. +** +** Neither of these two scenarios are possible during a savepoint rollback. +** +** If this is a savepoint rollback, then memory may have to be dynamically +** allocated by this function. If this is the case and an allocation fails, +** SQLITE_NOMEM is returned. +*/ +static int pager_playback_one_page( + Pager *pPager, /* The pager being played back */ + i64 *pOffset, /* Offset of record to playback */ + Bitvec *pDone, /* Bitvec of pages already played back */ + int isMainJrnl, /* 1 -> main journal. 0 -> sub-journal. */ + int isSavepnt /* True for a savepoint rollback */ +){ + int rc; + PgHdr *pPg; /* An existing page in the cache */ + Pgno pgno; /* The page number of a page in journal */ + u32 cksum; /* Checksum used for sanity checking */ + char *aData; /* Temporary storage for the page */ + sqlite3_file *jfd; /* The file descriptor for the journal file */ + int isSynced; /* True if journal page is synced */ + + assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ + assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ + assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */ + assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */ + + aData = pPager->pTmpSpace; + assert( aData ); /* Temp storage must have already been allocated */ + assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); + + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + ** or savepoint rollback done at the request of the caller) or this is + ** a hot-journal rollback. If it is a hot-journal rollback, the pager + ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback + ** only reads from the main journal, not the sub-journal. + */ + assert( pPager->eState>=PAGER_WRITER_CACHEMOD + || (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK) + ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl ); + + /* Read the page number and page data from the journal or sub-journal + ** file. Return an error code to the caller if an IO error occurs. + */ + jfd = isMainJrnl ? pPager->jfd : pPager->sjfd; + rc = read32bits(jfd, *pOffset, &pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsRead(jfd, (u8*)aData, pPager->pageSize, (*pOffset)+4); + if( rc!=SQLITE_OK ) return rc; + *pOffset += pPager->pageSize + 4 + isMainJrnl*4; + + /* Sanity checking on the page. This is more important that I originally + ** thought. If a power failure occurs while the journal is being written, + ** it could cause invalid data to be written into the journal. We need to + ** detect this invalid data (with high probability) and ignore it. + */ + if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ + assert( !isSavepnt ); + return SQLITE_DONE; + } + if( pgno>(Pgno)pPager->dbSize || sqlite3BitvecTest(pDone, pgno) ){ + return SQLITE_OK; + } + if( isMainJrnl ){ + rc = read32bits(jfd, (*pOffset)-4, &cksum); + if( rc ) return rc; + if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){ + return SQLITE_DONE; + } + } + + /* If this page has already been played by before during the current + ** rollback, then don't bother to play it back again. + */ + if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ + return rc; + } + + /* When playing back page 1, restore the nReserve setting + */ + if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){ + pPager->nReserve = ((u8*)aData)[20]; + pagerReportSize(pPager); + } + + /* If the pager is in CACHEMOD state, then there must be a copy of this + ** page in the pager cache. In this case just update the pager cache, + ** not the database file. The page is left marked dirty in this case. + ** + ** An exception to the above rule: If the database is in no-sync mode + ** and a page is moved during an incremental vacuum then the page may + ** not be in the pager cache. Later: if a malloc() or IO error occurs + ** during a Movepage() call, then the page may not be in the cache + ** either. So the condition described in the above paragraph is not + ** assert()able. + ** + ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the + ** pager cache if it exists and the main file. The page is then marked + ** not dirty. Since this code is only executed in PAGER_OPEN state for + ** a hot-journal rollback, it is guaranteed that the page-cache is empty + ** if the pager is in OPEN state. + ** + ** Ticket #1171: The statement journal might contain page content that is + ** different from the page content at the start of the transaction. + ** This occurs when a page is changed prior to the start of a statement + ** then changed again within the statement. When rolling back such a + ** statement we must not write to the original database unless we know + ** for certain that original page contents are synced into the main rollback + ** journal. Otherwise, a power loss might leave modified data in the + ** database file without an entry in the rollback journal that can + ** restore the database to its original form. Two conditions must be + ** met before writing to the database files. (1) the database must be + ** locked. (2) we know that the original page content is fully synced + ** in the main journal either because the page is not in cache or else + ** the page is marked as needSync==0. + ** + ** 2008-04-14: When attempting to vacuum a corrupt database file, it + ** is possible to fail a statement on a database that does not yet exist. + ** Do not attempt to write if database file has never been opened. + */ + if( pagerUseWal(pPager) ){ + pPg = 0; + }else{ + pPg = pager_lookup(pPager, pgno); + } + assert( pPg || !MEMDB ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 ); + PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", + PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), + (isMainJrnl?"main-journal":"sub-journal") + )); + if( isMainJrnl ){ + isSynced = pPager->noSync || (*pOffset <= pPager->journalHdr); + }else{ + isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); + } + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + && isSynced + ){ + i64 ofst = (pgno-1)*(i64)pPager->pageSize; + testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); + assert( !pagerUseWal(pPager) ); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + if( pPager->pBackup ){ + CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM); + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); + CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData); + } + }else if( !isMainJrnl && pPg==0 ){ + /* If this is a rollback of a savepoint and data was not written to + ** the database and the page is not in-memory, there is a potential + ** problem. When the page is next fetched by the b-tree layer, it + ** will be read from the database file, which may or may not be + ** current. + ** + ** There are a couple of different ways this can happen. All are quite + ** obscure. When running in synchronous mode, this can only happen + ** if the page is on the free-list at the start of the transaction, then + ** populated, then moved using sqlite3PagerMovepage(). + ** + ** The solution is to add an in-memory page to the cache containing + ** the data just read from the sub-journal. Mark the page as dirty + ** and if the pager requires a journal-sync, then mark the page as + ** requiring a journal-sync before it is written. + */ + assert( isSavepnt ); + assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)==0 ); + pPager->doNotSpill |= SPILLFLAG_ROLLBACK; + rc = sqlite3PagerAcquire(pPager, pgno, &pPg, 1); + assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 ); + pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK; + if( rc!=SQLITE_OK ) return rc; + pPg->flags &= ~PGHDR_NEED_READ; + sqlite3PcacheMakeDirty(pPg); + } + if( pPg ){ + /* No page should ever be explicitly rolled back that is in use, except + ** for page 1 which is held in use in order to keep the lock on the + ** database active. However such a page may be rolled back as a result + ** of an internal error resulting in an automatic call to + ** sqlite3PagerRollback(). + */ + void *pData; + pData = pPg->pData; + memcpy(pData, (u8*)aData, pPager->pageSize); + pPager->xReiniter(pPg); + if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){ + /* If the contents of this page were just restored from the main + ** journal file, then its content must be as they were when the + ** transaction was first opened. In this case we can mark the page + ** as clean, since there will be no need to write it out to the + ** database. + ** + ** There is one exception to this rule. If the page is being rolled + ** back as part of a savepoint (or statement) rollback from an + ** unsynced portion of the main journal file, then it is not safe + ** to mark the page as clean. This is because marking the page as + ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is + ** already in the journal file (recorded in Pager.pInJournal) and + ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to + ** again within this transaction, it will be marked as dirty but + ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially + ** be written out into the database file before its journal file + ** segment is synced. If a crash occurs during or following this, + ** database corruption may ensue. + */ + assert( !pagerUseWal(pPager) ); + sqlite3PcacheMakeClean(pPg); + } + pager_set_pagehash(pPg); + + /* If this was page 1, then restore the value of Pager.dbFileVers. + ** Do this before any decoding. */ + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); + } + + /* Decode the page just read from disk */ + CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM); + sqlite3PcacheRelease(pPg); + } + return rc; +} + +/* +** Parameter zMaster is the name of a master journal file. A single journal +** file that referred to the master journal file has just been rolled back. +** This routine checks if it is possible to delete the master journal file, +** and does so if it is. +** +** Argument zMaster may point to Pager.pTmpSpace. So that buffer is not +** available for use within this function. +** +** When a master journal file is created, it is populated with the names +** of all of its child journals, one after another, formatted as utf-8 +** encoded text. The end of each child journal file is marked with a +** nul-terminator byte (0x00). i.e. the entire contents of a master journal +** file for a transaction involving two databases might be: +** +** "/home/bill/a.db-journal\x00/home/bill/b.db-journal\x00" +** +** A master journal file may only be deleted once all of its child +** journals have been rolled back. +** +** This function reads the contents of the master-journal file into +** memory and loops through each of the child journal names. For +** each child journal, it checks if: +** +** * if the child journal exists, and if so +** * if the child journal contains a reference to master journal +** file zMaster +** +** If a child journal can be found that matches both of the criteria +** above, this function returns without doing anything. Otherwise, if +** no such child journal can be found, file zMaster is deleted from +** the file-system using sqlite3OsDelete(). +** +** If an IO error within this function, an error code is returned. This +** function allocates memory by calling sqlite3Malloc(). If an allocation +** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors +** occur, SQLITE_OK is returned. +** +** TODO: This function allocates a single block of memory to load +** the entire contents of the master journal file. This could be +** a couple of kilobytes or so - potentially larger than the page +** size. +*/ +static int pager_delmaster(Pager *pPager, const char *zMaster){ + sqlite3_vfs *pVfs = pPager->pVfs; + int rc; /* Return code */ + sqlite3_file *pMaster; /* Malloc'd master-journal file descriptor */ + sqlite3_file *pJournal; /* Malloc'd child-journal file descriptor */ + char *zMasterJournal = 0; /* Contents of master journal file */ + i64 nMasterJournal; /* Size of master journal file */ + char *zJournal; /* Pointer to one journal within MJ file */ + char *zMasterPtr; /* Space to hold MJ filename from a journal file */ + int nMasterPtr; /* Amount of space allocated to zMasterPtr[] */ + + /* Allocate space for both the pJournal and pMaster file descriptors. + ** If successful, open the master journal file for reading. + */ + pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); + pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); + if( !pMaster ){ + rc = SQLITE_NOMEM; + }else{ + const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); + rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); + } + if( rc!=SQLITE_OK ) goto delmaster_out; + + /* Load the entire master journal file into space obtained from + ** sqlite3_malloc() and pointed to by zMasterJournal. Also obtain + ** sufficient space (in zMasterPtr) to hold the names of master + ** journal files extracted from regular rollback-journals. + */ + rc = sqlite3OsFileSize(pMaster, &nMasterJournal); + if( rc!=SQLITE_OK ) goto delmaster_out; + nMasterPtr = pVfs->mxPathname+1; + zMasterJournal = sqlite3Malloc((int)nMasterJournal + nMasterPtr + 1); + if( !zMasterJournal ){ + rc = SQLITE_NOMEM; + goto delmaster_out; + } + zMasterPtr = &zMasterJournal[nMasterJournal+1]; + rc = sqlite3OsRead(pMaster, zMasterJournal, (int)nMasterJournal, 0); + if( rc!=SQLITE_OK ) goto delmaster_out; + zMasterJournal[nMasterJournal] = 0; + + zJournal = zMasterJournal; + while( (zJournal-zMasterJournal)pageSize bytes). +** If the file on disk is currently larger than nPage pages, then use the VFS +** xTruncate() method to truncate it. +** +** Or, it might might be the case that the file on disk is smaller than +** nPage pages. Some operating system implementations can get confused if +** you try to truncate a file to some size that is larger than it +** currently is, so detect this case and write a single zero byte to +** the end of the new file instead. +** +** If successful, return SQLITE_OK. If an IO error occurs while modifying +** the database file, return the error code to the caller. +*/ +static int pager_truncate(Pager *pPager, Pgno nPage){ + int rc = SQLITE_OK; + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState!=PAGER_READER ); + + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ + i64 currentSize, newSize; + int szPage = pPager->pageSize; + assert( pPager->eLock==EXCLUSIVE_LOCK ); + /* TODO: Is it safe to use Pager.dbFileSize here? */ + rc = sqlite3OsFileSize(pPager->fd, ¤tSize); + newSize = szPage*(i64)nPage; + if( rc==SQLITE_OK && currentSize!=newSize ){ + if( currentSize>newSize ){ + rc = sqlite3OsTruncate(pPager->fd, newSize); + }else if( (currentSize+szPage)<=newSize ){ + char *pTmp = pPager->pTmpSpace; + memset(pTmp, 0, szPage); + testcase( (newSize-szPage) == currentSize ); + testcase( (newSize-szPage) > currentSize ); + rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage); + } + if( rc==SQLITE_OK ){ + pPager->dbFileSize = nPage; + } + } + } + return rc; +} + +/* +** Return a sanitized version of the sector-size of OS file pFile. The +** return value is guaranteed to lie between 32 and MAX_SECTOR_SIZE. +*/ +SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *pFile){ + int iRet = sqlite3OsSectorSize(pFile); + if( iRet<32 ){ + iRet = 512; + }else if( iRet>MAX_SECTOR_SIZE ){ + assert( MAX_SECTOR_SIZE>=512 ); + iRet = MAX_SECTOR_SIZE; + } + return iRet; +} + +/* +** Set the value of the Pager.sectorSize variable for the given +** pager based on the value returned by the xSectorSize method +** of the open database file. The sector size will be used used +** to determine the size and alignment of journal header and +** master journal pointers within created journal files. +** +** For temporary files the effective sector size is always 512 bytes. +** +** Otherwise, for non-temporary files, the effective sector size is +** the value returned by the xSectorSize() method rounded up to 32 if +** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it +** is greater than MAX_SECTOR_SIZE. +** +** If the file has the SQLITE_IOCAP_POWERSAFE_OVERWRITE property, then set +** the effective sector size to its minimum value (512). The purpose of +** pPager->sectorSize is to define the "blast radius" of bytes that +** might change if a crash occurs while writing to a single byte in +** that range. But with POWERSAFE_OVERWRITE, the blast radius is zero +** (that is what POWERSAFE_OVERWRITE means), so we minimize the sector +** size. For backwards compatibility of the rollback journal file format, +** we cannot reduce the effective sector size below 512. +*/ +static void setSectorSize(Pager *pPager){ + assert( isOpen(pPager->fd) || pPager->tempFile ); + + if( pPager->tempFile + || (sqlite3OsDeviceCharacteristics(pPager->fd) & + SQLITE_IOCAP_POWERSAFE_OVERWRITE)!=0 + ){ + /* Sector size doesn't matter for temporary files. Also, the file + ** may not have been opened yet, in which case the OsSectorSize() + ** call will segfault. */ + pPager->sectorSize = 512; + }else{ + pPager->sectorSize = sqlite3SectorSize(pPager->fd); + } +} + +/* +** Playback the journal and thus restore the database file to +** the state it was in before we started making changes. +** +** The journal file format is as follows: +** +** (1) 8 byte prefix. A copy of aJournalMagic[]. +** (2) 4 byte big-endian integer which is the number of valid page records +** in the journal. If this value is 0xffffffff, then compute the +** number of page records from the journal size. +** (3) 4 byte big-endian integer which is the initial value for the +** sanity checksum. +** (4) 4 byte integer which is the number of pages to truncate the +** database to during a rollback. +** (5) 4 byte big-endian integer which is the sector size. The header +** is this many bytes in size. +** (6) 4 byte big-endian integer which is the page size. +** (7) zero padding out to the next sector size. +** (8) Zero or more pages instances, each as follows: +** + 4 byte page number. +** + pPager->pageSize bytes of data. +** + 4 byte checksum +** +** When we speak of the journal header, we mean the first 7 items above. +** Each entry in the journal is an instance of the 8th item. +** +** Call the value from the second bullet "nRec". nRec is the number of +** valid page entries in the journal. In most cases, you can compute the +** value of nRec from the size of the journal file. But if a power +** failure occurred while the journal was being written, it could be the +** case that the size of the journal file had already been increased but +** the extra entries had not yet made it safely to disk. In such a case, +** the value of nRec computed from the file size would be too large. For +** that reason, we always use the nRec value in the header. +** +** If the nRec value is 0xffffffff it means that nRec should be computed +** from the file size. This value is used when the user selects the +** no-sync option for the journal. A power failure could lead to corruption +** in this case. But for things like temporary table (which will be +** deleted when the power is restored) we don't care. +** +** If the file opened as the journal file is not a well-formed +** journal file then all pages up to the first corrupted page are rolled +** back (or no pages if the journal header is corrupted). The journal file +** is then deleted and SQLITE_OK returned, just as if no corruption had +** been encountered. +** +** If an I/O or malloc() error occurs, the journal-file is not deleted +** and an error code is returned. +** +** The isHot parameter indicates that we are trying to rollback a journal +** that might be a hot journal. Or, it could be that the journal is +** preserved because of JOURNALMODE_PERSIST or JOURNALMODE_TRUNCATE. +** If the journal really is hot, reset the pager cache prior rolling +** back any content. If the journal is merely persistent, no reset is +** needed. +*/ +static int pager_playback(Pager *pPager, int isHot){ + sqlite3_vfs *pVfs = pPager->pVfs; + i64 szJ; /* Size of the journal file in bytes */ + u32 nRec; /* Number of Records in the journal */ + u32 u; /* Unsigned loop counter */ + Pgno mxPg = 0; /* Size of the original file in pages */ + int rc; /* Result code of a subroutine */ + int res = 1; /* Value returned by sqlite3OsAccess() */ + char *zMaster = 0; /* Name of master journal file if any */ + int needPagerReset; /* True to reset page prior to first page rollback */ + int nPlayback = 0; /* Total number of pages restored from journal */ + + /* Figure out how many records are in the journal. Abort early if + ** the journal is empty. + */ + assert( isOpen(pPager->jfd) ); + rc = sqlite3OsFileSize(pPager->jfd, &szJ); + if( rc!=SQLITE_OK ){ + goto end_playback; + } + + /* Read the master journal name from the journal, if it is present. + ** If a master journal file name is specified, but the file is not + ** present on disk, then the journal is not hot and does not need to be + ** played back. + ** + ** TODO: Technically the following is an error because it assumes that + ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that + ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, + ** mxPathname is 512, which is the same as the minimum allowable value + ** for pageSize. + */ + zMaster = pPager->pTmpSpace; + rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); + if( rc==SQLITE_OK && zMaster[0] ){ + rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); + } + zMaster = 0; + if( rc!=SQLITE_OK || !res ){ + goto end_playback; + } + pPager->journalOff = 0; + needPagerReset = isHot; + + /* This loop terminates either when a readJournalHdr() or + ** pager_playback_one_page() call returns SQLITE_DONE or an IO error + ** occurs. + */ + while( 1 ){ + /* Read the next journal header from the journal file. If there are + ** not enough bytes left in the journal file for a complete header, or + ** it is corrupted, then a process must have failed while writing it. + ** This indicates nothing more needs to be rolled back. + */ + rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + goto end_playback; + } + + /* If nRec is 0xffffffff, then this journal was created by a process + ** working in no-sync mode. This means that the rest of the journal + ** file consists of pages, there are no more journal headers. Compute + ** the value of nRec based on this assumption. + */ + if( nRec==0xffffffff ){ + assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ); + nRec = (int)((szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager)); + } + + /* If nRec is 0 and this rollback is of a transaction created by this + ** process and if this is the final header in the journal, then it means + ** that this part of the journal was being filled but has not yet been + ** synced to disk. Compute the number of pages based on the remaining + ** size of the file. + ** + ** The third term of the test was added to fix ticket #2565. + ** When rolling back a hot journal, nRec==0 always means that the next + ** chunk of the journal contains zero pages to be rolled back. But + ** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in + ** the journal, it means that the journal might contain additional + ** pages that need to be rolled back and that the number of pages + ** should be computed based on the journal file size. + */ + if( nRec==0 && !isHot && + pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){ + nRec = (int)((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager)); + } + + /* If this is the first header read from the journal, truncate the + ** database file back to its original size. + */ + if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){ + rc = pager_truncate(pPager, mxPg); + if( rc!=SQLITE_OK ){ + goto end_playback; + } + pPager->dbSize = mxPg; + } + + /* Copy original pages out of the journal and back into the + ** database file and/or page cache. + */ + for(u=0; ujournalOff,0,1,0); + if( rc==SQLITE_OK ){ + nPlayback++; + }else{ + if( rc==SQLITE_DONE ){ + pPager->journalOff = szJ; + break; + }else if( rc==SQLITE_IOERR_SHORT_READ ){ + /* If the journal has been truncated, simply stop reading and + ** processing the journal. This might happen if the journal was + ** not completely written and synced prior to a crash. In that + ** case, the database should have never been written in the + ** first place so it is OK to simply abandon the rollback. */ + rc = SQLITE_OK; + goto end_playback; + }else{ + /* If we are unable to rollback, quit and return the error + ** code. This will cause the pager to enter the error state + ** so that no further harm will be done. Perhaps the next + ** process to come along will be able to rollback the database. + */ + goto end_playback; + } + } + } + } + /*NOTREACHED*/ + assert( 0 ); + +end_playback: + /* Following a rollback, the database file should be back in its original + ** state prior to the start of the transaction, so invoke the + ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the + ** assertion that the transaction counter was modified. + */ +#ifdef SQLITE_DEBUG + if( pPager->fd->pMethods ){ + sqlite3OsFileControlHint(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0); + } +#endif + + /* If this playback is happening automatically as a result of an IO or + ** malloc error that occurred after the change-counter was updated but + ** before the transaction was committed, then the change-counter + ** modification may just have been reverted. If this happens in exclusive + ** mode, then subsequent transactions performed by the connection will not + ** update the change-counter at all. This may lead to cache inconsistency + ** problems for other processes at some point in the future. So, just + ** in case this has happened, clear the changeCountDone flag now. + */ + pPager->changeCountDone = pPager->tempFile; + + if( rc==SQLITE_OK ){ + zMaster = pPager->pTmpSpace; + rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); + testcase( rc!=SQLITE_OK ); + } + if( rc==SQLITE_OK + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ + rc = sqlite3PagerSync(pPager, 0); + } + if( rc==SQLITE_OK ){ + rc = pager_end_transaction(pPager, zMaster[0]!='\0', 0); + testcase( rc!=SQLITE_OK ); + } + if( rc==SQLITE_OK && zMaster[0] && res ){ + /* If there was a master journal and this routine will return success, + ** see if it is possible to delete the master journal. + */ + rc = pager_delmaster(pPager, zMaster); + testcase( rc!=SQLITE_OK ); + } + if( isHot && nPlayback ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_ROLLBACK, "recovered %d pages from %s", + nPlayback, pPager->zJournal); + } + + /* The Pager.sectorSize variable may have been updated while rolling + ** back a journal created by a process with a different sector size + ** value. Reset it to the correct value for this process. + */ + setSectorSize(pPager); + return rc; +} + + +/* +** Read the content for page pPg out of the database file and into +** pPg->pData. A shared lock or greater must be held on the database +** file before this function is called. +** +** If page 1 is read, then the value of Pager.dbFileVers[] is set to +** the value read from the database file. +** +** If an IO error occurs, then the IO error is returned to the caller. +** Otherwise, SQLITE_OK is returned. +*/ +static int readDbPage(PgHdr *pPg, u32 iFrame){ + Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ + Pgno pgno = pPg->pgno; /* Page number to read */ + int rc = SQLITE_OK; /* Return code */ + int pgsz = pPager->pageSize; /* Number of bytes to read */ + + assert( pPager->eState>=PAGER_READER && !MEMDB ); + assert( isOpen(pPager->fd) ); + +#ifndef SQLITE_OMIT_WAL + if( iFrame ){ + /* Try to pull the page from the write-ahead log. */ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else +#endif + { + i64 iOffset = (pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + } + + if( pgno==1 ){ + if( rc ){ + /* If the read is unsuccessful, set the dbFileVers[] to something + ** that will never be a valid file version. dbFileVers[] is a copy + ** of bytes 24..39 of the database. Bytes 28..31 should always be + ** zero or the size of the database in page. Bytes 32..35 and 35..39 + ** should be page numbers which are never 0xffffffff. So filling + ** pPager->dbFileVers[] with all 0xff bytes should suffice. + ** + ** For an encrypted database, the situation is more complex: bytes + ** 24..39 of the database are white noise. But the probability of + ** white noising equaling 16 bytes of 0xff is vanishingly small so + ** we should still be ok. + */ + memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); + }else{ + u8 *dbFileVers = &((u8*)pPg->pData)[24]; + memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); + } + } + CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); + + PAGER_INCR(sqlite3_pager_readdb_count); + PAGER_INCR(pPager->nRead); + IOTRACE(("PGIN %p %d\n", pPager, pgno)); + PAGERTRACE(("FETCH %d page %d hash(%08x)\n", + PAGERID(pPager), pgno, pager_pagehash(pPg))); + + return rc; +} + +/* +** Update the value of the change-counter at offsets 24 and 92 in +** the header and the sqlite version number at offset 96. +** +** This is an unconditional update. See also the pager_incr_changecounter() +** routine which only updates the change-counter if the update is actually +** needed, as determined by the pPager->changeCountDone state variable. +*/ +static void pager_write_changecounter(PgHdr *pPg){ + u32 change_counter; + + /* Increment the value just read and write it back to byte 24. */ + change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1; + put32bits(((char*)pPg->pData)+24, change_counter); + + /* Also store the SQLite version number in bytes 96..99 and in + ** bytes 92..95 store the change counter for which the version number + ** is valid. */ + put32bits(((char*)pPg->pData)+92, change_counter); + put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER); +} + +#ifndef SQLITE_OMIT_WAL +/* +** This function is invoked once for each page that has already been +** written into the log file when a WAL transaction is rolled back. +** Parameter iPg is the page number of said page. The pCtx argument +** is actually a pointer to the Pager structure. +** +** If page iPg is present in the cache, and has no outstanding references, +** it is discarded. Otherwise, if there are one or more outstanding +** references, the page content is reloaded from the database. If the +** attempt to reload content from the database is required and fails, +** return an SQLite error code. Otherwise, SQLITE_OK. +*/ +static int pagerUndoCallback(void *pCtx, Pgno iPg){ + int rc = SQLITE_OK; + Pager *pPager = (Pager *)pCtx; + PgHdr *pPg; + + assert( pagerUseWal(pPager) ); + pPg = sqlite3PagerLookup(pPager, iPg); + if( pPg ){ + if( sqlite3PcachePageRefcount(pPg)==1 ){ + sqlite3PcacheDrop(pPg); + }else{ + u32 iFrame = 0; + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } + if( rc==SQLITE_OK ){ + pPager->xReiniter(pPg); + } + sqlite3PagerUnrefNotNull(pPg); + } + } + + /* Normally, if a transaction is rolled back, any backup processes are + ** updated as data is copied out of the rollback journal and into the + ** database. This is not generally possible with a WAL database, as + ** rollback involves simply truncating the log file. Therefore, if one + ** or more frames have already been written to the log (and therefore + ** also copied into the backup databases) as part of this transaction, + ** the backups must be restarted. + */ + sqlite3BackupRestart(pPager->pBackup); + + return rc; +} + +/* +** This function is called to rollback a transaction on a WAL database. +*/ +static int pagerRollbackWal(Pager *pPager){ + int rc; /* Return Code */ + PgHdr *pList; /* List of dirty pages to revert */ + + /* For all pages in the cache that are currently dirty or have already + ** been written (but not committed) to the log file, do one of the + ** following: + ** + ** + Discard the cached page (if refcount==0), or + ** + Reload page content from the database (if refcount>0). + */ + pPager->dbSize = pPager->dbOrigSize; + rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); + pList = sqlite3PcacheDirtyList(pPager->pPCache); + while( pList && rc==SQLITE_OK ){ + PgHdr *pNext = pList->pDirty; + rc = pagerUndoCallback((void *)pPager, pList->pgno); + pList = pNext; + } + + return rc; +} + +/* +** This function is a wrapper around sqlite3WalFrames(). As well as logging +** the contents of the list of pages headed by pList (connected by pDirty), +** this function notifies any active backup processes that the pages have +** changed. +** +** The list of pages passed into this routine is always sorted by page number. +** Hence, if page 1 appears anywhere on the list, it will be the first page. +*/ +static int pagerWalFrames( + Pager *pPager, /* Pager object */ + PgHdr *pList, /* List of frames to log */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit /* True if this is a commit */ +){ + int rc; /* Return code */ + int nList; /* Number of pages in pList */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES) + PgHdr *p; /* For looping over pages */ +#endif + + assert( pPager->pWal ); + assert( pList ); +#ifdef SQLITE_DEBUG + /* Verify that the page list is in accending order */ + for(p=pList; p && p->pDirty; p=p->pDirty){ + assert( p->pgno < p->pDirty->pgno ); + } +#endif + + assert( pList->pDirty==0 || isCommit ); + if( isCommit ){ + /* If a WAL transaction is being committed, there is no point in writing + ** any pages with page numbers greater than nTruncate into the WAL file. + ** They will never be read by any client. So remove them from the pDirty + ** list here. */ + PgHdr *p; + PgHdr **ppNext = &pList; + nList = 0; + for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ + if( p->pgno<=nTruncate ){ + ppNext = &p->pDirty; + nList++; + } + } + assert( pList ); + }else{ + nList = 1; + } + pPager->aStat[PAGER_STAT_WRITE] += nList; + + if( pList->pgno==1 ) pager_write_changecounter(pList); + rc = sqlite3WalFrames(pPager->pWal, + pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags + ); + if( rc==SQLITE_OK && pPager->pBackup ){ + PgHdr *p; + for(p=pList; p; p=p->pDirty){ + sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); + } + } + +#ifdef SQLITE_CHECK_PAGES + pList = sqlite3PcacheDirtyList(pPager->pPCache); + for(p=pList; p; p=p->pDirty){ + pager_set_pagehash(p); + } +#endif + + return rc; +} + +/* +** Begin a read transaction on the WAL. +** +** This routine used to be called "pagerOpenSnapshot()" because it essentially +** makes a snapshot of the database at the current point in time and preserves +** that snapshot for use by the reader in spite of concurrently changes by +** other writers or checkpointers. +*/ +static int pagerBeginReadTransaction(Pager *pPager){ + int rc; /* Return code */ + int changed = 0; /* True if cache must be reset */ + + assert( pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + + /* sqlite3WalEndReadTransaction() was not called for the previous + ** transaction in locking_mode=EXCLUSIVE. So call it now. If we + ** are in locking_mode=NORMAL and EndRead() was previously called, + ** the duplicate call is harmless. + */ + sqlite3WalEndReadTransaction(pPager->pWal); + + rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); + if( rc!=SQLITE_OK || changed ){ + pager_reset(pPager); + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + } + + return rc; +} +#endif + +/* +** This function is called as part of the transition from PAGER_OPEN +** to PAGER_READER state to determine the size of the database file +** in pages (assuming the page size currently stored in Pager.pageSize). +** +** If no error occurs, SQLITE_OK is returned and the size of the database +** in pages is stored in *pnPage. Otherwise, an error code (perhaps +** SQLITE_IOERR_FSTAT) is returned and *pnPage is left unmodified. +*/ +static int pagerPagecount(Pager *pPager, Pgno *pnPage){ + Pgno nPage; /* Value to return via *pnPage */ + + /* Query the WAL sub-system for the database size. The WalDbsize() + ** function returns zero if the WAL is not open (i.e. Pager.pWal==0), or + ** if the database size is not available. The database size is not + ** available from the WAL sub-system if the log file is empty or + ** contains no valid committed transactions. + */ + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK ); + nPage = sqlite3WalDbsize(pPager->pWal); + + /* If the database size was not available from the WAL sub-system, + ** determine it based on the size of the database file. If the size + ** of the database file is not an integer multiple of the page-size, + ** round down to the nearest page. Except, any file larger than 0 + ** bytes in size is considered to contain at least one page. + */ + if( nPage==0 ){ + i64 n = 0; /* Size of db file in bytes */ + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( isOpen(pPager->fd) ){ + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; + } + } + nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); + } + + /* If the current number of pages in the file is greater than the + ** configured maximum pager number, increase the allowed limit so + ** that the file can be read. + */ + if( nPage>pPager->mxPgno ){ + pPager->mxPgno = (Pgno)nPage; + } + + *pnPage = nPage; + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_WAL +/* +** Check if the *-wal file that corresponds to the database opened by pPager +** exists if the database is not empy, or verify that the *-wal file does +** not exist (by deleting it) if the database file is empty. +** +** If the database is not empty and the *-wal file exists, open the pager +** in WAL mode. If the database is empty or if no *-wal file exists and +** if no error occurs, make sure Pager.journalMode is not set to +** PAGER_JOURNALMODE_WAL. +** +** Return SQLITE_OK or an error code. +** +** The caller must hold a SHARED lock on the database file to call this +** function. Because an EXCLUSIVE lock on the db file is required to delete +** a WAL on a none-empty database, this ensures there is no race condition +** between the xAccess() below and an xDelete() being executed by some +** other connection. +*/ +static int pagerOpenWalIfPresent(Pager *pPager){ + int rc = SQLITE_OK; + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK ); + + if( !pPager->tempFile ){ + int isWal; /* True if WAL file exists */ + Pgno nPage; /* Size of the database file */ + + rc = pagerPagecount(pPager, &nPage); + if( rc ) return rc; + if( nPage==0 ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); + if( rc==SQLITE_IOERR_DELETE_NOENT ) rc = SQLITE_OK; + isWal = 0; + }else{ + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &isWal + ); + } + if( rc==SQLITE_OK ){ + if( isWal ){ + testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); + rc = sqlite3PagerOpenWal(pPager, 0); + }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + pPager->journalMode = PAGER_JOURNALMODE_DELETE; + } + } + } + return rc; +} +#endif + +/* +** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback +** the entire master journal file. The case pSavepoint==NULL occurs when +** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction +** savepoint. +** +** When pSavepoint is not NULL (meaning a non-transaction savepoint is +** being rolled back), then the rollback consists of up to three stages, +** performed in the order specified: +** +** * Pages are played back from the main journal starting at byte +** offset PagerSavepoint.iOffset and continuing to +** PagerSavepoint.iHdrOffset, or to the end of the main journal +** file if PagerSavepoint.iHdrOffset is zero. +** +** * If PagerSavepoint.iHdrOffset is not zero, then pages are played +** back starting from the journal header immediately following +** PagerSavepoint.iHdrOffset to the end of the main journal file. +** +** * Pages are then played back from the sub-journal file, starting +** with the PagerSavepoint.iSubRec and continuing to the end of +** the journal file. +** +** Throughout the rollback process, each time a page is rolled back, the +** corresponding bit is set in a bitvec structure (variable pDone in the +** implementation below). This is used to ensure that a page is only +** rolled back the first time it is encountered in either journal. +** +** If pSavepoint is NULL, then pages are only played back from the main +** journal file. There is no need for a bitvec in this case. +** +** In either case, before playback commences the Pager.dbSize variable +** is reset to the value that it held at the start of the savepoint +** (or transaction). No page with a page-number greater than this value +** is played back. If one is encountered it is simply skipped. +*/ +static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ + i64 szJ; /* Effective size of the main journal */ + i64 iHdrOff; /* End of first segment of main-journal records */ + int rc = SQLITE_OK; /* Return code */ + Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ + + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + + /* Allocate a bitvec to use to store the set of pages rolled back */ + if( pSavepoint ){ + pDone = sqlite3BitvecCreate(pSavepoint->nOrig); + if( !pDone ){ + return SQLITE_NOMEM; + } + } + + /* Set the database size back to the value it was before the savepoint + ** being reverted was opened. + */ + pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; + pPager->changeCountDone = pPager->tempFile; + + if( !pSavepoint && pagerUseWal(pPager) ){ + return pagerRollbackWal(pPager); + } + + /* Use pPager->journalOff as the effective size of the main rollback + ** journal. The actual file might be larger than this in + ** PAGER_JOURNALMODE_TRUNCATE or PAGER_JOURNALMODE_PERSIST. But anything + ** past pPager->journalOff is off-limits to us. + */ + szJ = pPager->journalOff; + assert( pagerUseWal(pPager)==0 || szJ==0 ); + + /* Begin by rolling back records from the main journal starting at + ** PagerSavepoint.iOffset and continuing to the next journal header. + ** There might be records in the main journal that have a page number + ** greater than the current database size (pPager->dbSize) but those + ** will be skipped automatically. Pages are added to pDone as they + ** are played back. + */ + if( pSavepoint && !pagerUseWal(pPager) ){ + iHdrOff = pSavepoint->iHdrOffset ? pSavepoint->iHdrOffset : szJ; + pPager->journalOff = pSavepoint->iOffset; + while( rc==SQLITE_OK && pPager->journalOffjournalOff, pDone, 1, 1); + } + assert( rc!=SQLITE_DONE ); + }else{ + pPager->journalOff = 0; + } + + /* Continue rolling back records out of the main journal starting at + ** the first journal header seen and continuing until the effective end + ** of the main journal file. Continue to skip out-of-range pages and + ** continue adding pages rolled back to pDone. + */ + while( rc==SQLITE_OK && pPager->journalOffjournalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff" + ** test is related to ticket #2565. See the discussion in the + ** pager_playback() function for additional information. + */ + if( nJRec==0 + && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff + ){ + nJRec = (u32)((szJ - pPager->journalOff)/JOURNAL_PG_SZ(pPager)); + } + for(ii=0; rc==SQLITE_OK && iijournalOffjournalOff, pDone, 1, 1); + } + assert( rc!=SQLITE_DONE ); + } + assert( rc!=SQLITE_OK || pPager->journalOff>=szJ ); + + /* Finally, rollback pages from the sub-journal. Page that were + ** previously rolled back out of the main journal (and are hence in pDone) + ** will be skipped. Out-of-range pages are also skipped. + */ + if( pSavepoint ){ + u32 ii; /* Loop counter */ + i64 offset = (i64)pSavepoint->iSubRec*(4+pPager->pageSize); + + if( pagerUseWal(pPager) ){ + rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData); + } + for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && iinSubRec; ii++){ + assert( offset==(i64)ii*(4+pPager->pageSize) ); + rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1); + } + assert( rc!=SQLITE_DONE ); + } + + sqlite3BitvecDestroy(pDone); + if( rc==SQLITE_OK ){ + pPager->journalOff = szJ; + } + + return rc; +} + +/* +** Change the maximum number of in-memory pages that are allowed. +*/ +SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ + sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); +} + +/* +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. +*/ +static void pagerFixMaplimit(Pager *pPager){ +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) && fd->pMethods->iVersion>=3 ){ + sqlite3_int64 sz; + sz = pPager->szMmap; + pPager->bUseFetch = (sz>0); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); + } +#endif +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 szMmap){ + pPager->szMmap = szMmap; + pagerFixMaplimit(pPager); +} + +/* +** Free as much memory as possible from the pager. +*/ +SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ + sqlite3PcacheShrink(pPager->pPCache); +} + +/* +** Adjust settings of the pager to those specified in the pgFlags parameter. +** +** The "level" in pgFlags & PAGER_SYNCHRONOUS_MASK sets the robustness +** of the database to damage due to OS crashes or power failures by +** changing the number of syncs()s when writing the journals. +** There are three levels: +** +** OFF sqlite3OsSync() is never called. This is the default +** for temporary and transient files. +** +** NORMAL The journal is synced once before writes begin on the +** database. This is normally adequate protection, but +** it is theoretically possible, though very unlikely, +** that an inopertune power failure could leave the journal +** in a state which would cause damage to the database +** when it is rolled back. +** +** FULL The journal is synced twice before writes begin on the +** database (with some additional information - the nRec field +** of the journal header - being written in between the two +** syncs). If we assume that writing a +** single disk sector is atomic, then this mode provides +** assurance that the journal will not be corrupted to the +** point of causing damage to the database during rollback. +** +** The above is for a rollback-journal mode. For WAL mode, OFF continues +** to mean that no syncs ever occur. NORMAL means that the WAL is synced +** prior to the start of checkpoint and that the database file is synced +** at the conclusion of the checkpoint if the entire content of the WAL +** was written back into the database. But no sync operations occur for +** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL +** file is synced following each commit operation, in addition to the +** syncs associated with NORMAL. +** +** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The +** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync +** using fcntl(F_FULLFSYNC). SQLITE_SYNC_NORMAL means to do an +** ordinary fsync() call. There is no difference between SQLITE_SYNC_FULL +** and SQLITE_SYNC_NORMAL on platforms other than MacOSX. But the +** synchronous=FULL versus synchronous=NORMAL setting determines when +** the xSync primitive is called and is relevant to all platforms. +** +** Numeric values associated with these states are OFF==1, NORMAL=2, +** and FULL=3. +*/ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +SQLITE_PRIVATE void sqlite3PagerSetFlags( + Pager *pPager, /* The pager to set safety level for */ + unsigned pgFlags /* Various flags */ +){ + unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; + assert( level>=1 && level<=3 ); + pPager->noSync = (level==1 || pPager->tempFile) ?1:0; + pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; + if( pPager->noSync ){ + pPager->syncFlags = 0; + pPager->ckptSyncFlags = 0; + }else if( pgFlags & PAGER_FULLFSYNC ){ + pPager->syncFlags = SQLITE_SYNC_FULL; + pPager->ckptSyncFlags = SQLITE_SYNC_FULL; + }else if( pgFlags & PAGER_CKPT_FULLFSYNC ){ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = SQLITE_SYNC_FULL; + }else{ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; + } + pPager->walSyncFlags = pPager->syncFlags; + if( pPager->fullSync ){ + pPager->walSyncFlags |= WAL_SYNC_TRANSACTIONS; + } + if( pgFlags & PAGER_CACHESPILL ){ + pPager->doNotSpill &= ~SPILLFLAG_OFF; + }else{ + pPager->doNotSpill |= SPILLFLAG_OFF; + } +} +#endif + +/* +** The following global variable is incremented whenever the library +** attempts to open a temporary file. This information is used for +** testing and analysis only. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_opentemp_count = 0; +#endif + +/* +** Open a temporary file. +** +** Write the file descriptor into *pFile. Return SQLITE_OK on success +** or some other error code if we fail. The OS will automatically +** delete the temporary file when it is closed. +** +** The flags passed to the VFS layer xOpen() call are those specified +** by parameter vfsFlags ORed with the following: +** +** SQLITE_OPEN_READWRITE +** SQLITE_OPEN_CREATE +** SQLITE_OPEN_EXCLUSIVE +** SQLITE_OPEN_DELETEONCLOSE +*/ +static int pagerOpentemp( + Pager *pPager, /* The pager object */ + sqlite3_file *pFile, /* Write the file descriptor here */ + int vfsFlags /* Flags passed through to the VFS */ +){ + int rc; /* Return code */ + +#ifdef SQLITE_TEST + sqlite3_opentemp_count++; /* Used for testing and analysis only */ +#endif + + vfsFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + rc = sqlite3OsOpen(pPager->pVfs, 0, pFile, vfsFlags, 0); + assert( rc!=SQLITE_OK || isOpen(pFile) ); + return rc; +} + +/* +** Set the busy handler function. +** +** The pager invokes the busy-handler if sqlite3OsLock() returns +** SQLITE_BUSY when trying to upgrade from no-lock to a SHARED lock, +** or when trying to upgrade from a RESERVED lock to an EXCLUSIVE +** lock. It does *not* invoke the busy handler when upgrading from +** SHARED to RESERVED, or when upgrading from SHARED to EXCLUSIVE +** (which occurs during hot-journal rollback). Summary: +** +** Transition | Invokes xBusyHandler +** -------------------------------------------------------- +** NO_LOCK -> SHARED_LOCK | Yes +** SHARED_LOCK -> RESERVED_LOCK | No +** SHARED_LOCK -> EXCLUSIVE_LOCK | No +** RESERVED_LOCK -> EXCLUSIVE_LOCK | Yes +** +** If the busy-handler callback returns non-zero, the lock is +** retried. If it returns zero, then the SQLITE_BUSY error is +** returned to the caller of the pager API function. +*/ +SQLITE_PRIVATE void sqlite3PagerSetBusyhandler( + Pager *pPager, /* Pager object */ + int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ + void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ +){ + pPager->xBusyHandler = xBusyHandler; + pPager->pBusyHandlerArg = pBusyHandlerArg; + + if( isOpen(pPager->fd) ){ + void **ap = (void **)&pPager->xBusyHandler; + assert( ((int(*)(void *))(ap[0]))==xBusyHandler ); + assert( ap[1]==pBusyHandlerArg ); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap); + } +} + +/* +** Change the page size used by the Pager object. The new page size +** is passed in *pPageSize. +** +** If the pager is in the error state when this function is called, it +** is a no-op. The value returned is the error state error code (i.e. +** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). +** +** Otherwise, if all of the following are true: +** +** * the new page size (value of *pPageSize) is valid (a power +** of two between 512 and SQLITE_MAX_PAGE_SIZE, inclusive), and +** +** * there are no outstanding page references, and +** +** * the database is either not an in-memory database or it is +** an in-memory database that currently consists of zero pages. +** +** then the pager object page size is set to *pPageSize. +** +** If the page size is changed, then this function uses sqlite3PagerMalloc() +** to obtain a new Pager.pTmpSpace buffer. If this allocation attempt +** fails, SQLITE_NOMEM is returned and the page size remains unchanged. +** In all other cases, SQLITE_OK is returned. +** +** If the page size is not changed, either because one of the enumerated +** conditions above is not true, the pager was in error state when this +** function was called, or because the memory allocation attempt failed, +** then *pPageSize is set to the old, retained page size before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ + int rc = SQLITE_OK; + + /* It is not possible to do a full assert_pager_state() here, as this + ** function may be called from within PagerOpen(), before the state + ** of the Pager object is internally consistent. + ** + ** At one point this function returned an error if the pager was in + ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that + ** there is at least one outstanding page reference, this function + ** is a no-op for that case anyhow. + */ + + u32 pageSize = *pPageSize; + assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); + if( (pPager->memDb==0 || pPager->dbSize==0) + && sqlite3PcacheRefCount(pPager->pPCache)==0 + && pageSize && pageSize!=(u32)pPager->pageSize + ){ + char *pNew = NULL; /* New temp space */ + i64 nByte = 0; + + if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){ + rc = sqlite3OsFileSize(pPager->fd, &nByte); + } + if( rc==SQLITE_OK ){ + pNew = (char *)sqlite3PageMalloc(pageSize); + if( !pNew ) rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + pager_reset(pPager); + pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); + pPager->pageSize = pageSize; + sqlite3PageFree(pPager->pTmpSpace); + pPager->pTmpSpace = pNew; + sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + } + } + + *pPageSize = pPager->pageSize; + if( rc==SQLITE_OK ){ + if( nReserve<0 ) nReserve = pPager->nReserve; + assert( nReserve>=0 && nReserve<1000 ); + pPager->nReserve = (i16)nReserve; + pagerReportSize(pPager); + pagerFixMaplimit(pPager); + } + return rc; +} + +/* +** Return a pointer to the "temporary page" buffer held internally +** by the pager. This is a buffer that is big enough to hold the +** entire content of a database page. This buffer is used internally +** during rollback and will be overwritten whenever a rollback +** occurs. But other modules are free to use it too, as long as +** no rollbacks are happening. +*/ +SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager *pPager){ + return pPager->pTmpSpace; +} + +/* +** Attempt to set the maximum database page count if mxPage is positive. +** Make no changes if mxPage is zero or negative. And never reduce the +** maximum page count below the current size of the database. +** +** Regardless of mxPage, return the current maximum page count. +*/ +SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ + if( mxPage>0 ){ + pPager->mxPgno = mxPage; + } + assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ + assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */ + return pPager->mxPgno; +} + +/* +** The following set of routines are used to disable the simulated +** I/O error mechanism. These routines are used to avoid simulated +** errors in places where we do not care about errors. +** +** Unless -DSQLITE_TEST=1 is used, these routines are all no-ops +** and generate no code. +*/ +#ifdef SQLITE_TEST +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_hit; +static int saved_cnt; +void disable_simulated_io_errors(void){ + saved_cnt = sqlite3_io_error_pending; + sqlite3_io_error_pending = -1; +} +void enable_simulated_io_errors(void){ + sqlite3_io_error_pending = saved_cnt; +} +#else +# define disable_simulated_io_errors() +# define enable_simulated_io_errors() +#endif + +/* +** Read the first N bytes from the beginning of the file into memory +** that pDest points to. +** +** If the pager was opened on a transient file (zFilename==""), or +** opened on a file less than N bytes in size, the output buffer is +** zeroed and SQLITE_OK returned. The rationale for this is that this +** function is used to read database headers, and a new transient or +** zero sized database has a header than consists entirely of zeroes. +** +** If any IO error apart from SQLITE_IOERR_SHORT_READ is encountered, +** the error code is returned to the caller and the contents of the +** output buffer undefined. +*/ +SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ + int rc = SQLITE_OK; + memset(pDest, 0, N); + assert( isOpen(pPager->fd) || pPager->tempFile ); + + /* This routine is only called by btree immediately after creating + ** the Pager object. There has not been an opportunity to transition + ** to WAL mode yet. + */ + assert( !pagerUseWal(pPager) ); + + if( isOpen(pPager->fd) ){ + IOTRACE(("DBHDR %p 0 %d\n", pPager, N)) + rc = sqlite3OsRead(pPager->fd, pDest, N, 0); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + } + return rc; +} + +/* +** This function may only be called when a read-transaction is open on +** the pager. It returns the total number of pages in the database. +** +** However, if the file is between 1 and bytes in size, then +** this is considered a 1 page file. +*/ +SQLITE_PRIVATE void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ + assert( pPager->eState>=PAGER_READER ); + assert( pPager->eState!=PAGER_WRITER_FINISHED ); + *pnPage = (int)pPager->dbSize; +} + + +/* +** Try to obtain a lock of type locktype on the database file. If +** a similar or greater lock is already held, this function is a no-op +** (returning SQLITE_OK immediately). +** +** Otherwise, attempt to obtain the lock using sqlite3OsLock(). Invoke +** the busy callback if the lock is currently not available. Repeat +** until the busy callback returns false or until the attempt to +** obtain the lock succeeds. +** +** Return SQLITE_OK on success and an error code if we cannot obtain +** the lock. If the lock is obtained successfully, set the Pager.state +** variable to locktype before returning. +*/ +static int pager_wait_on_lock(Pager *pPager, int locktype){ + int rc; /* Return code */ + + /* Check that this is either a no-op (because the requested lock is + ** already held, or one of the transistions that the busy-handler + ** may be invoked during, according to the comment above + ** sqlite3PagerSetBusyhandler(). + */ + assert( (pPager->eLock>=locktype) + || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) + || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) + ); + + do { + rc = pagerLockDb(pPager, locktype); + }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); + return rc; +} + +/* +** Function assertTruncateConstraint(pPager) checks that one of the +** following is true for all dirty pages currently in the page-cache: +** +** a) The page number is less than or equal to the size of the +** current database image, in pages, OR +** +** b) if the page content were written at this time, it would not +** be necessary to write the current content out to the sub-journal +** (as determined by function subjRequiresPage()). +** +** If the condition asserted by this function were not true, and the +** dirty page were to be discarded from the cache via the pagerStress() +** routine, pagerStress() would not write the current page content to +** the database file. If a savepoint transaction were rolled back after +** this happened, the correct behavior would be to restore the current +** content of the page. However, since this content is not present in either +** the database file or the portion of the rollback journal and +** sub-journal rolled back the content could not be restored and the +** database image would become corrupt. It is therefore fortunate that +** this circumstance cannot arise. +*/ +#if defined(SQLITE_DEBUG) +static void assertTruncateConstraintCb(PgHdr *pPg){ + assert( pPg->flags&PGHDR_DIRTY ); + assert( !subjRequiresPage(pPg) || pPg->pgno<=pPg->pPager->dbSize ); +} +static void assertTruncateConstraint(Pager *pPager){ + sqlite3PcacheIterateDirty(pPager->pPCache, assertTruncateConstraintCb); +} +#else +# define assertTruncateConstraint(pPager) +#endif + +/* +** Truncate the in-memory database file image to nPage pages. This +** function does not actually modify the database file on disk. It +** just sets the internal state of the pager object so that the +** truncation will be done when the current transaction is committed. +** +** This function is only called right before committing a transaction. +** Once this function has been called, the transaction must either be +** rolled back or committed. It is not safe to call this function and +** then continue writing to the database. +*/ +SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ + assert( pPager->dbSize>=nPage ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + pPager->dbSize = nPage; + + /* At one point the code here called assertTruncateConstraint() to + ** ensure that all pages being truncated away by this operation are, + ** if one or more savepoints are open, present in the savepoint + ** journal so that they can be restored if the savepoint is rolled + ** back. This is no longer necessary as this function is now only + ** called right before committing a transaction. So although the + ** Pager object may still have open savepoints (Pager.nSavepoint!=0), + ** they cannot be rolled back. So the assertTruncateConstraint() call + ** is no longer correct. */ +} + + +/* +** This function is called before attempting a hot-journal rollback. It +** syncs the journal file to disk, then sets pPager->journalHdr to the +** size of the journal file so that the pager_playback() routine knows +** that the entire journal file has been synced. +** +** Syncing a hot-journal to disk before attempting to roll it back ensures +** that if a power-failure occurs during the rollback, the process that +** attempts rollback following system recovery sees the same journal +** content as this process. +** +** If everything goes as planned, SQLITE_OK is returned. Otherwise, +** an SQLite error code. +*/ +static int pagerSyncHotJournal(Pager *pPager){ + int rc = SQLITE_OK; + if( !pPager->noSync ){ + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_NORMAL); + } + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileSize(pPager->jfd, &pPager->journalHdr); + } + return rc; +} + +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ + PgHdr *p; /* Memory mapped page to return */ + + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); + return SQLITE_NOMEM; + } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pgno = pgno; + p->pData = pData; + pPager->nMmapOut++; + + return SQLITE_OK; +} + +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); +} + +/* +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. +*/ +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pMmapFreelist; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + + +/* +** Shutdown the page cache. Free all memory and close all files. +** +** If a transaction was in progress when this routine is called, that +** transaction is rolled back. All outstanding pages are invalidated +** and their memory is freed. Any attempt to use a page associated +** with this page cache after this function returns will likely +** result in a coredump. +** +** This function always succeeds. If a transaction is active an attempt +** is made to roll it back. If an error occurs during the rollback +** a hot journal may be left in the filesystem but no error is returned +** to the caller. +*/ +SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ + u8 *pTmp = (u8 *)pPager->pTmpSpace; + + assert( assert_pager_state(pPager) ); + disable_simulated_io_errors(); + sqlite3BeginBenignMalloc(); + pagerFreeMapHdrs(pPager); + /* pPager->errCode = 0; */ + pPager->exclusiveMode = 0; +#ifndef SQLITE_OMIT_WAL + sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); + pPager->pWal = 0; +#endif + pager_reset(pPager); + if( MEMDB ){ + pager_unlock(pPager); + }else{ + /* If it is open, sync the journal file before calling UnlockAndRollback. + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs + ** while this is happening, the database could become corrupt. + ** + ** If an error occurs while trying to sync the journal, shift the pager + ** into the ERROR state. This causes UnlockAndRollback to unlock the + ** database and close the journal file without attempting to roll it + ** back or finalize it. The next database user will have to do hot-journal + ** rollback before accessing the database file. + */ + if( isOpen(pPager->jfd) ){ + pager_error(pPager, pagerSyncHotJournal(pPager)); + } + pagerUnlockAndRollback(pPager); + } + sqlite3EndBenignMalloc(); + enable_simulated_io_errors(); + PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); + IOTRACE(("CLOSE %p\n", pPager)) + sqlite3OsClose(pPager->jfd); + sqlite3OsClose(pPager->fd); + sqlite3PageFree(pTmp); + sqlite3PcacheClose(pPager->pPCache); + +#ifdef SQLITE_HAS_CODEC + if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); +#endif + + assert( !pPager->aSavepoint && !pPager->pInJournal ); + assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); + + sqlite3_free(pPager); + return SQLITE_OK; +} + +#if !defined(NDEBUG) || defined(SQLITE_TEST) +/* +** Return the page number for page pPg. +*/ +SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage *pPg){ + return pPg->pgno; +} +#endif + +/* +** Increment the reference count for page pPg. +*/ +SQLITE_PRIVATE void sqlite3PagerRef(DbPage *pPg){ + sqlite3PcacheRef(pPg); +} + +/* +** Sync the journal. In other words, make sure all the pages that have +** been written to the journal have actually reached the surface of the +** disk and can be restored in the event of a hot-journal rollback. +** +** If the Pager.noSync flag is set, then this function is a no-op. +** Otherwise, the actions required depend on the journal-mode and the +** device characteristics of the file-system, as follows: +** +** * If the journal file is an in-memory journal file, no action need +** be taken. +** +** * Otherwise, if the device does not support the SAFE_APPEND property, +** then the nRec field of the most recently written journal header +** is updated to contain the number of journal records that have +** been written following it. If the pager is operating in full-sync +** mode, then the journal file is synced before this field is updated. +** +** * If the device does not support the SEQUENTIAL property, then +** journal file is synced. +** +** Or, in pseudo-code: +** +** if( NOT ){ +** if( NOT SAFE_APPEND ){ +** if( ) xSync(); +** +** } +** if( NOT SEQUENTIAL ) xSync(); +** } +** +** If successful, this routine clears the PGHDR_NEED_SYNC flag of every +** page currently held in memory before returning SQLITE_OK. If an IO +** error is encountered, then the IO error code is returned to the caller. +*/ +static int syncJournal(Pager *pPager, int newHdr){ + int rc; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + assert( !pagerUseWal(pPager) ); + + rc = sqlite3PagerExclusiveLock(pPager); + if( rc!=SQLITE_OK ) return rc; + + if( !pPager->noSync ){ + assert( !pPager->tempFile ); + if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + assert( isOpen(pPager->jfd) ); + + if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + /* This block deals with an obscure problem. If the last connection + ** that wrote to this database was operating in persistent-journal + ** mode, then the journal file may at this point actually be larger + ** than Pager.journalOff bytes. If the next thing in the journal + ** file happens to be a journal-header (written as part of the + ** previous connection's transaction), and a crash or power-failure + ** occurs after nRec is updated but before this connection writes + ** anything else to the journal file (or commits/rolls back its + ** transaction), then SQLite may become confused when doing the + ** hot-journal rollback following recovery. It may roll back all + ** of this connections data, then proceed to rolling back the old, + ** out-of-date data that follows it. Database corruption. + ** + ** To work around this, if the journal file does appear to contain + ** a valid header following Pager.journalOff, then write a 0x00 + ** byte to the start of it to prevent it from being recognized. + ** + ** Variable iNextHdrOffset is set to the offset at which this + ** problematic header will occur, if it exists. aMagic is used + ** as a temporary buffer to inspect the first couple of bytes of + ** the potential journal header. + */ + i64 iNextHdrOffset; + u8 aMagic[8]; + u8 zHeader[sizeof(aJournalMagic)+4]; + + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + put32bits(&zHeader[sizeof(aJournalMagic)], pPager->nRec); + + iNextHdrOffset = journalHdrOffset(pPager); + rc = sqlite3OsRead(pPager->jfd, aMagic, 8, iNextHdrOffset); + if( rc==SQLITE_OK && 0==memcmp(aMagic, aJournalMagic, 8) ){ + static const u8 zerobyte = 0; + rc = sqlite3OsWrite(pPager->jfd, &zerobyte, 1, iNextHdrOffset); + } + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ + return rc; + } + + /* Write the nRec value into the journal file header. If in + ** full-synchronous mode, sync the journal first. This ensures that + ** all data has really hit the disk before nRec is updated to mark + ** it as a candidate for rollback. + ** + ** This is not required if the persistent media supports the + ** SAFE_APPEND property. Because in this case it is not possible + ** for garbage data to be appended to the file, the nRec field + ** is populated with 0xFFFFFFFF when the journal header is written + ** and never needs to be updated. + */ + if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); + IOTRACE(("JSYNC %p\n", pPager)) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + if( rc!=SQLITE_OK ) return rc; + } + IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr)); + rc = sqlite3OsWrite( + pPager->jfd, zHeader, sizeof(zHeader), pPager->journalHdr + ); + if( rc!=SQLITE_OK ) return rc; + } + if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); + IOTRACE(("JSYNC %p\n", pPager)) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| + (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) + ); + if( rc!=SQLITE_OK ) return rc; + } + + pPager->journalHdr = pPager->journalOff; + if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + if( rc!=SQLITE_OK ) return rc; + } + }else{ + pPager->journalHdr = pPager->journalOff; + } + } + + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + ** all pages. + */ + sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->eState = PAGER_WRITER_DBMOD; + assert( assert_pager_state(pPager) ); + return SQLITE_OK; +} + +/* +** The argument is the first in a linked list of dirty pages connected +** by the PgHdr.pDirty pointer. This function writes each one of the +** in-memory pages in the list to the database file. The argument may +** be NULL, representing an empty list. In this case this function is +** a no-op. +** +** The pager must hold at least a RESERVED lock when this function +** is called. Before writing anything to the database file, this lock +** is upgraded to an EXCLUSIVE lock. If the lock cannot be obtained, +** SQLITE_BUSY is returned and no data is written to the database file. +** +** If the pager is a temp-file pager and the actual file-system file +** is not yet open, it is created and opened before any data is +** written out. +** +** Once the lock has been upgraded and, if necessary, the file opened, +** the pages are written out to the database file in list order. Writing +** a page is skipped if it meets either of the following criteria: +** +** * The page number is greater than Pager.dbSize, or +** * The PGHDR_DONT_WRITE flag is set on the page. +** +** If writing out a page causes the database file to grow, Pager.dbFileSize +** is updated accordingly. If page 1 is written out, then the value cached +** in Pager.dbFileVers[] is updated to match the new value stored in +** the database file. +** +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs, an IO error code is returned. Or, if the EXCLUSIVE lock cannot +** be obtained, SQLITE_BUSY is returned. +*/ +static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ + int rc = SQLITE_OK; /* Return code */ + + /* This function is only called for rollback pagers in WRITER_DBMOD state. */ + assert( !pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->eLock==EXCLUSIVE_LOCK ); + + /* If the file is a temp-file has not yet been opened, open it now. It + ** is not possible for rc to be other than SQLITE_OK if this branch + ** is taken, as pager_wait_on_lock() is a no-op for temp-files. + */ + if( !isOpen(pPager->fd) ){ + assert( pPager->tempFile && rc==SQLITE_OK ); + rc = pagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); + } + + /* Before the first write, give the VFS a hint of what the final + ** file size will be. + */ + assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); + if( rc==SQLITE_OK + && pPager->dbHintSizedbSize + && (pList->pDirty || pList->pgno>pPager->dbHintSize) + ){ + sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); + pPager->dbHintSize = pPager->dbSize; + } + + while( rc==SQLITE_OK && pList ){ + Pgno pgno = pList->pgno; + + /* If there are dirty pages in the page cache with page numbers greater + ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to + ** make the file smaller (presumably by auto-vacuum code). Do not write + ** any such pages to the file. + ** + ** Also, do not write out any page that has the PGHDR_DONT_WRITE flag + ** set (set by sqlite3PagerDontWrite()). + */ + if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ + i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ + char *pData; /* Data to write */ + + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); + if( pList->pgno==1 ) pager_write_changecounter(pList); + + /* Encode the database */ + CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); + + /* Write out the page data. */ + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + + /* If page 1 was just written, update Pager.dbFileVers to match + ** the value now stored in the database file. If writing this + ** page caused the database file to grow, update dbFileSize. + */ + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers)); + } + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + pPager->aStat[PAGER_STAT_WRITE]++; + + /* Update any backup objects copying the contents of this pager. */ + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); + + PAGERTRACE(("STORE %d page %d hash(%08x)\n", + PAGERID(pPager), pgno, pager_pagehash(pList))); + IOTRACE(("PGOUT %p %d\n", pPager, pgno)); + PAGER_INCR(sqlite3_pager_writedb_count); + }else{ + PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); + } + pager_set_pagehash(pList); + pList = pList->pDirty; + } + + return rc; +} + +/* +** Ensure that the sub-journal file is open. If it is already open, this +** function is a no-op. +** +** SQLITE_OK is returned if everything goes according to plan. An +** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() +** fails. +*/ +static int openSubJournal(Pager *pPager){ + int rc = SQLITE_OK; + if( !isOpen(pPager->sjfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ + sqlite3MemJournalOpen(pPager->sjfd); + }else{ + rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL); + } + } + return rc; +} + +/* +** Append a record of the current state of page pPg to the sub-journal. +** It is the callers responsibility to use subjRequiresPage() to check +** that it is really required before calling this function. +** +** If successful, set the bit corresponding to pPg->pgno in the bitvecs +** for all open savepoints before returning. +** +** This function returns SQLITE_OK if everything is successful, an IO +** error code if the attempt to write to the sub-journal fails, or +** SQLITE_NOMEM if a malloc fails while setting a bit in a savepoint +** bitvec. +*/ +static int subjournalPage(PgHdr *pPg){ + int rc = SQLITE_OK; + Pager *pPager = pPg->pPager; + if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + + /* Open the sub-journal, if it has not already been opened */ + assert( pPager->useJournal ); + assert( isOpen(pPager->jfd) || pagerUseWal(pPager) ); + assert( isOpen(pPager->sjfd) || pPager->nSubRec==0 ); + assert( pagerUseWal(pPager) + || pageInJournal(pPager, pPg) + || pPg->pgno>pPager->dbOrigSize + ); + rc = openSubJournal(pPager); + + /* If the sub-journal was opened successfully (or was already open), + ** write the journal record into the file. */ + if( rc==SQLITE_OK ){ + void *pData = pPg->pData; + i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); + char *pData2; + + CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); + rc = write32bits(pPager->sjfd, offset, pPg->pgno); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4); + } + } + } + if( rc==SQLITE_OK ){ + pPager->nSubRec++; + assert( pPager->nSavepoint>0 ); + rc = addToSavepointBitvecs(pPager, pPg->pgno); + } + return rc; +} + +/* +** This function is called by the pcache layer when it has reached some +** soft memory limit. The first argument is a pointer to a Pager object +** (cast as a void*). The pager is always 'purgeable' (not an in-memory +** database). The second argument is a reference to a page that is +** currently dirty but has no outstanding references. The page +** is always associated with the Pager object passed as the first +** argument. +** +** The job of this function is to make pPg clean by writing its contents +** out to the database file, if possible. This may involve syncing the +** journal file. +** +** If successful, sqlite3PcacheMakeClean() is called on the page and +** SQLITE_OK returned. If an IO error occurs while trying to make the +** page clean, the IO error code is returned. If the page cannot be +** made clean for some other reason, but no error occurs, then SQLITE_OK +** is returned by sqlite3PcacheMakeClean() is not called. +*/ +static int pagerStress(void *p, PgHdr *pPg){ + Pager *pPager = (Pager *)p; + int rc = SQLITE_OK; + + assert( pPg->pPager==pPager ); + assert( pPg->flags&PGHDR_DIRTY ); + + /* The doNotSpill NOSYNC bit is set during times when doing a sync of + ** journal (and adding a new header) is not allowed. This occurs + ** during calls to sqlite3PagerWrite() while trying to journal multiple + ** pages belonging to the same sector. + ** + ** The doNotSpill ROLLBACK and OFF bits inhibits all cache spilling + ** regardless of whether or not a sync is required. This is set during + ** a rollback or by user request, respectively. + ** + ** Spilling is also prohibited when in an error state since that could + ** lead to database corruption. In the current implementaton it + ** is impossible for sqlite3PcacheFetch() to be called with createFlag==1 + ** while in the error state, hence it is impossible for this routine to + ** be called in the error state. Nevertheless, we include a NEVER() + ** test for the error state as a safeguard against future changes. + */ + if( NEVER(pPager->errCode) ) return SQLITE_OK; + testcase( pPager->doNotSpill & SPILLFLAG_ROLLBACK ); + testcase( pPager->doNotSpill & SPILLFLAG_OFF ); + testcase( pPager->doNotSpill & SPILLFLAG_NOSYNC ); + if( pPager->doNotSpill + && ((pPager->doNotSpill & (SPILLFLAG_ROLLBACK|SPILLFLAG_OFF))!=0 + || (pPg->flags & PGHDR_NEED_SYNC)!=0) + ){ + return SQLITE_OK; + } + + pPg->pDirty = 0; + if( pagerUseWal(pPager) ){ + /* Write a single frame for this page to the log. */ + if( subjRequiresPage(pPg) ){ + rc = subjournalPage(pPg); + } + if( rc==SQLITE_OK ){ + rc = pagerWalFrames(pPager, pPg, 0, 0); + } + }else{ + + /* Sync the journal file if required. */ + if( pPg->flags&PGHDR_NEED_SYNC + || pPager->eState==PAGER_WRITER_CACHEMOD + ){ + rc = syncJournal(pPager, 1); + } + + /* If the page number of this page is larger than the current size of + ** the database image, it may need to be written to the sub-journal. + ** This is because the call to pager_write_pagelist() below will not + ** actually write data to the file in this case. + ** + ** Consider the following sequence of events: + ** + ** BEGIN; + ** + ** + ** SAVEPOINT sp; + ** + ** pagerStress(page X) + ** ROLLBACK TO sp; + ** + ** If (X>Y), then when pagerStress is called page X will not be written + ** out to the database file, but will be dropped from the cache. Then, + ** following the "ROLLBACK TO sp" statement, reading page X will read + ** data from the database file. This will be the copy of page X as it + ** was when the transaction started, not as it was when "SAVEPOINT sp" + ** was executed. + ** + ** The solution is to write the current data for page X into the + ** sub-journal file now (if it is not already there), so that it will + ** be restored to its current value when the "ROLLBACK TO sp" is + ** executed. + */ + if( NEVER( + rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) + ) ){ + rc = subjournalPage(pPg); + } + + /* Write the contents of the page out to the database file. */ + if( rc==SQLITE_OK ){ + assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); + rc = pager_write_pagelist(pPager, pPg); + } + } + + /* Mark the page as clean. */ + if( rc==SQLITE_OK ){ + PAGERTRACE(("STRESS %d page %d\n", PAGERID(pPager), pPg->pgno)); + sqlite3PcacheMakeClean(pPg); + } + + return pager_error(pPager, rc); +} + + +/* +** Allocate and initialize a new Pager object and put a pointer to it +** in *ppPager. The pager should eventually be freed by passing it +** to sqlite3PagerClose(). +** +** The zFilename argument is the path to the database file to open. +** If zFilename is NULL then a randomly-named temporary file is created +** and used as the file to be cached. Temporary files are be deleted +** automatically when they are closed. If zFilename is ":memory:" then +** all information is held in cache. It is never written to disk. +** This can be used to implement an in-memory database. +** +** The nExtra parameter specifies the number of bytes of space allocated +** along with each page reference. This space is available to the user +** via the sqlite3PagerGetExtra() API. +** +** The flags argument is used to specify properties that affect the +** operation of the pager. It should be passed some bitwise combination +** of the PAGER_* flags. +** +** The vfsFlags parameter is a bitmask to pass to the flags parameter +** of the xOpen() method of the supplied VFS when opening files. +** +** If the pager object is allocated and the specified file opened +** successfully, SQLITE_OK is returned and *ppPager set to point to +** the new pager object. If an error occurs, *ppPager is set to NULL +** and error code returned. This function may return SQLITE_NOMEM +** (sqlite3Malloc() is used to allocate memory), SQLITE_CANTOPEN or +** various SQLITE_IO_XXX errors. +*/ +SQLITE_PRIVATE int sqlite3PagerOpen( + sqlite3_vfs *pVfs, /* The virtual file system to use */ + Pager **ppPager, /* OUT: Return the Pager structure here */ + const char *zFilename, /* Name of the database file to open */ + int nExtra, /* Extra bytes append to each in-memory page */ + int flags, /* flags controlling this file */ + int vfsFlags, /* flags passed through to sqlite3_vfs.xOpen() */ + void (*xReinit)(DbPage*) /* Function to reinitialize pages */ +){ + u8 *pPtr; + Pager *pPager = 0; /* Pager object to allocate and return */ + int rc = SQLITE_OK; /* Return code */ + int tempFile = 0; /* True for temp files (incl. in-memory files) */ + int memDb = 0; /* True if this is an in-memory file */ + int readOnly = 0; /* True if this is a read-only file */ + int journalFileSize; /* Bytes to allocate for each journal fd */ + char *zPathname = 0; /* Full path to database file */ + int nPathname = 0; /* Number of bytes in zPathname */ + int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ + int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ + u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + const char *zUri = 0; /* URI args to copy */ + int nUri = 0; /* Number of bytes of URI args at *zUri */ + + /* Figure out how much space is required for each journal file-handle + ** (there are two of them, the main journal and the sub-journal). This + ** is the maximum space required for an in-memory journal file handle + ** and a regular journal file-handle. Note that a "regular journal-handle" + ** may be a wrapper capable of caching the first portion of the journal + ** file in memory to implement the atomic-write optimization (see + ** source file journal.c). + */ + if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){ + journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); + }else{ + journalFileSize = ROUND8(sqlite3MemJournalSize()); + } + + /* Set the output variable to NULL in case an error occurs. */ + *ppPager = 0; + +#ifndef SQLITE_OMIT_MEMORYDB + if( flags & PAGER_MEMORY ){ + memDb = 1; + if( zFilename && zFilename[0] ){ + zPathname = sqlite3DbStrDup(0, zFilename); + if( zPathname==0 ) return SQLITE_NOMEM; + nPathname = sqlite3Strlen30(zPathname); + zFilename = 0; + } + } +#endif + + /* Compute and store the full pathname in an allocated buffer pointed + ** to by zPathname, length nPathname. Or, if this is a temporary file, + ** leave both nPathname and zPathname set to 0. + */ + if( zFilename && zFilename[0] ){ + const char *z; + nPathname = pVfs->mxPathname+1; + zPathname = sqlite3DbMallocRaw(0, nPathname*2); + if( zPathname==0 ){ + return SQLITE_NOMEM; + } + zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ + rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); + nPathname = sqlite3Strlen30(zPathname); + z = zUri = &zFilename[sqlite3Strlen30(zFilename)+1]; + while( *z ){ + z += sqlite3Strlen30(z)+1; + z += sqlite3Strlen30(z)+1; + } + nUri = (int)(&z[1] - zUri); + assert( nUri>=0 ); + if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ + /* This branch is taken when the journal path required by + ** the database being opened will be more than pVfs->mxPathname + ** bytes in length. This means the database cannot be opened, + ** as it will not be possible to open the journal file or even + ** check for a hot-journal before reading. + */ + rc = SQLITE_CANTOPEN_BKPT; + } + if( rc!=SQLITE_OK ){ + sqlite3DbFree(0, zPathname); + return rc; + } + } + + /* Allocate memory for the Pager structure, PCache object, the + ** three file descriptors, the database file name and the journal + ** file name. The layout in memory is as follows: + ** + ** Pager object (sizeof(Pager) bytes) + ** PCache object (sqlite3PcacheSize() bytes) + ** Database file handle (pVfs->szOsFile bytes) + ** Sub-journal file handle (journalFileSize bytes) + ** Main journal file handle (journalFileSize bytes) + ** Database file name (nPathname+1 bytes) + ** Journal file name (nPathname+8+1 bytes) + */ + pPtr = (u8 *)sqlite3MallocZero( + ROUND8(sizeof(*pPager)) + /* Pager structure */ + ROUND8(pcacheSize) + /* PCache object */ + ROUND8(pVfs->szOsFile) + /* The main db file */ + journalFileSize * 2 + /* The two journal files */ + nPathname + 1 + nUri + /* zFilename */ + nPathname + 8 + 2 /* zJournal */ +#ifndef SQLITE_OMIT_WAL + + nPathname + 4 + 2 /* zWal */ +#endif + ); + assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); + if( !pPtr ){ + sqlite3DbFree(0, zPathname); + return SQLITE_NOMEM; + } + pPager = (Pager*)(pPtr); + pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); + pPager->fd = (sqlite3_file*)(pPtr += ROUND8(pcacheSize)); + pPager->sjfd = (sqlite3_file*)(pPtr += ROUND8(pVfs->szOsFile)); + pPager->jfd = (sqlite3_file*)(pPtr += journalFileSize); + pPager->zFilename = (char*)(pPtr += journalFileSize); + assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); + + /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ + if( zPathname ){ + assert( nPathname>0 ); + pPager->zJournal = (char*)(pPtr += nPathname + 1 + nUri); + memcpy(pPager->zFilename, zPathname, nPathname); + if( nUri ) memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); + memcpy(pPager->zJournal, zPathname, nPathname); + memcpy(&pPager->zJournal[nPathname], "-journal\000", 8+2); + sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); +#ifndef SQLITE_OMIT_WAL + pPager->zWal = &pPager->zJournal[nPathname+8+1]; + memcpy(pPager->zWal, zPathname, nPathname); + memcpy(&pPager->zWal[nPathname], "-wal\000", 4+1); + sqlite3FileSuffix3(pPager->zFilename, pPager->zWal); +#endif + sqlite3DbFree(0, zPathname); + } + pPager->pVfs = pVfs; + pPager->vfsFlags = vfsFlags; + + /* Open the pager file. + */ + if( zFilename && zFilename[0] ){ + int fout = 0; /* VFS flags returned by xOpen() */ + rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); + assert( !memDb ); + readOnly = (fout&SQLITE_OPEN_READONLY); + + /* If the file was successfully opened for read/write access, + ** choose a default page size in case we have to create the + ** database file. The default page size is the maximum of: + ** + ** + SQLITE_DEFAULT_PAGE_SIZE, + ** + The value returned by sqlite3OsSectorSize() + ** + The largest page size that can be written atomically. + */ + if( rc==SQLITE_OK ){ + int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + if( !readOnly ){ + setSectorSize(pPager); + assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); + if( szPageDfltsectorSize ){ + if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ + szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; + }else{ + szPageDflt = (u32)pPager->sectorSize; + } + } +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int ii; + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); + for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ + if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ + szPageDflt = ii; + } + } + } +#endif + } + pPager->noLock = sqlite3_uri_boolean(zFilename, "nolock", 0); + if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 + || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ + vfsFlags |= SQLITE_OPEN_READONLY; + goto act_like_temp_file; + } + } + }else{ + /* If a temporary file is requested, it is not opened immediately. + ** In this case we accept the default page size and delay actually + ** opening the file until the first call to OsWrite(). + ** + ** This branch is also run for an in-memory database. An in-memory + ** database is the same as a temp-file that is never written out to + ** disk and uses an in-memory rollback journal. + ** + ** This branch also runs for files marked as immutable. + */ +act_like_temp_file: + tempFile = 1; + pPager->eState = PAGER_READER; /* Pretend we already have a lock */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE locking mode */ + pPager->noLock = 1; /* Do no locking */ + readOnly = (vfsFlags&SQLITE_OPEN_READONLY); + } + + /* The following call to PagerSetPagesize() serves to set the value of + ** Pager.pageSize and to allocate the Pager.pTmpSpace buffer. + */ + if( rc==SQLITE_OK ){ + assert( pPager->memDb==0 ); + rc = sqlite3PagerSetPagesize(pPager, &szPageDflt, -1); + testcase( rc!=SQLITE_OK ); + } + + /* If an error occurred in either of the blocks above, free the + ** Pager structure and close the file. + */ + if( rc!=SQLITE_OK ){ + assert( !pPager->pTmpSpace ); + sqlite3OsClose(pPager->fd); + sqlite3_free(pPager); + return rc; + } + + /* Initialize the PCache object. */ + assert( nExtra<1000 ); + nExtra = ROUND8(nExtra); + sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + + PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename)); + IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) + + pPager->useJournal = (u8)useJournal; + /* pPager->stmtOpen = 0; */ + /* pPager->stmtInUse = 0; */ + /* pPager->nRef = 0; */ + /* pPager->stmtSize = 0; */ + /* pPager->stmtJSize = 0; */ + /* pPager->nPage = 0; */ + pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; + /* pPager->state = PAGER_UNLOCK; */ + /* pPager->errMask = 0; */ + pPager->tempFile = (u8)tempFile; + assert( tempFile==PAGER_LOCKINGMODE_NORMAL + || tempFile==PAGER_LOCKINGMODE_EXCLUSIVE ); + assert( PAGER_LOCKINGMODE_EXCLUSIVE==1 ); + pPager->exclusiveMode = (u8)tempFile; + pPager->changeCountDone = pPager->tempFile; + pPager->memDb = (u8)memDb; + pPager->readOnly = (u8)readOnly; + assert( useJournal || pPager->tempFile ); + pPager->noSync = pPager->tempFile; + if( pPager->noSync ){ + assert( pPager->fullSync==0 ); + assert( pPager->syncFlags==0 ); + assert( pPager->walSyncFlags==0 ); + assert( pPager->ckptSyncFlags==0 ); + }else{ + pPager->fullSync = 1; + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; + pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; + } + /* pPager->pFirst = 0; */ + /* pPager->pFirstSynced = 0; */ + /* pPager->pLast = 0; */ + pPager->nExtra = (u16)nExtra; + pPager->journalSizeLimit = SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT; + assert( isOpen(pPager->fd) || tempFile ); + setSectorSize(pPager); + if( !useJournal ){ + pPager->journalMode = PAGER_JOURNALMODE_OFF; + }else if( memDb ){ + pPager->journalMode = PAGER_JOURNALMODE_MEMORY; + } + /* pPager->xBusyHandler = 0; */ + /* pPager->pBusyHandlerArg = 0; */ + pPager->xReiniter = xReinit; + /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ + + *ppPager = pPager; + return SQLITE_OK; +} + + +/* Verify that the database file has not be deleted or renamed out from +** under the pager. Return SQLITE_OK if the database is still were it ought +** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error +** code from sqlite3OsAccess()) if the database has gone missing. +*/ +static int databaseIsUnmoved(Pager *pPager){ + int bHasMoved = 0; + int rc; + + if( pPager->tempFile ) return SQLITE_OK; + if( pPager->dbSize==0 ) return SQLITE_OK; + assert( pPager->zFilename && pPager->zFilename[0] ); + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved); + if( rc==SQLITE_NOTFOUND ){ + /* If the HAS_MOVED file-control is unimplemented, assume that the file + ** has not been moved. That is the historical behavior of SQLite: prior to + ** version 3.8.3, it never checked */ + rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bHasMoved ){ + rc = SQLITE_READONLY_DBMOVED; + } + return rc; +} + + +/* +** This function is called after transitioning from PAGER_UNLOCK to +** PAGER_SHARED state. It tests if there is a hot journal present in +** the file-system for the given pager. A hot journal is one that +** needs to be played back. According to this function, a hot-journal +** file exists if the following criteria are met: +** +** * The journal file exists in the file system, and +** * No process holds a RESERVED or greater lock on the database file, and +** * The database file itself is greater than 0 bytes in size, and +** * The first byte of the journal file exists and is not 0x00. +** +** If the current size of the database file is 0 but a journal file +** exists, that is probably an old journal left over from a prior +** database with the same name. In this case the journal file is +** just deleted using OsDelete, *pExists is set to 0 and SQLITE_OK +** is returned. +** +** This routine does not check if there is a master journal filename +** at the end of the file. If there is, and that master journal file +** does not exist, then the journal file is not really hot. In this +** case this routine will return a false-positive. The pager_playback() +** routine will discover that the journal file is not really hot and +** will not roll it back. +** +** If a hot-journal file is found to exist, *pExists is set to 1 and +** SQLITE_OK returned. If no hot-journal file is present, *pExists is +** set to 0 and SQLITE_OK returned. If an IO error occurs while trying +** to determine whether or not a hot-journal file exists, the IO error +** code is returned and the value of *pExists is undefined. +*/ +static int hasHotJournal(Pager *pPager, int *pExists){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int rc = SQLITE_OK; /* Return code */ + int exists = 1; /* True if a journal file is present */ + int jrnlOpen = !!isOpen(pPager->jfd); + + assert( pPager->useJournal ); + assert( isOpen(pPager->fd) ); + assert( pPager->eState==PAGER_OPEN ); + + assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & + SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN + )); + + *pExists = 0; + if( !jrnlOpen ){ + rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); + } + if( rc==SQLITE_OK && exists ){ + int locked = 0; /* True if some process holds a RESERVED lock */ + + /* Race condition here: Another process might have been holding the + ** the RESERVED lock and have a journal open at the sqlite3OsAccess() + ** call above, but then delete the journal and drop the lock before + ** we get to the following sqlite3OsCheckReservedLock() call. If that + ** is the case, this routine might think there is a hot journal when + ** in fact there is none. This results in a false-positive which will + ** be dealt with by the playback routine. Ticket #3883. + */ + rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); + if( rc==SQLITE_OK && !locked ){ + Pgno nPage; /* Number of pages in database file */ + + rc = pagerPagecount(pPager, &nPage); + if( rc==SQLITE_OK ){ + /* If the database is zero pages in size, that means that either (1) the + ** journal is a remnant from a prior database with the same name where + ** the database file but not the journal was deleted, or (2) the initial + ** transaction that populates a new database is being rolled back. + ** In either case, the journal file can be deleted. However, take care + ** not to delete the journal file if it is already open due to + ** journal_mode=PERSIST. + */ + if( nPage==0 && !jrnlOpen ){ + sqlite3BeginBenignMalloc(); + if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ + sqlite3OsDelete(pVfs, pPager->zJournal, 0); + if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); + } + sqlite3EndBenignMalloc(); + }else{ + /* The journal file exists and no other connection has a reserved + ** or greater lock on the database file. Now check that there is + ** at least one non-zero bytes at the start of the journal file. + ** If there is, then we consider this journal to be hot. If not, + ** it can be ignored. + */ + if( !jrnlOpen ){ + int f = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL; + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &f); + } + if( rc==SQLITE_OK ){ + u8 first = 0; + rc = sqlite3OsRead(pPager->jfd, (void *)&first, 1, 0); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + if( !jrnlOpen ){ + sqlite3OsClose(pPager->jfd); + } + *pExists = (first!=0); + }else if( rc==SQLITE_CANTOPEN ){ + /* If we cannot open the rollback journal file in order to see if + ** its has a zero header, that might be due to an I/O error, or + ** it might be due to the race condition described above and in + ** ticket #3883. Either way, assume that the journal is hot. + ** This might be a false positive. But if it is, then the + ** automatic journal playback and recovery mechanism will deal + ** with it under an EXCLUSIVE lock where we do not need to + ** worry so much with race conditions. + */ + *pExists = 1; + rc = SQLITE_OK; + } + } + } + } + } + + return rc; +} + +/* +** This function is called to obtain a shared lock on the database file. +** It is illegal to call sqlite3PagerAcquire() until after this function +** has been successfully called. If a shared-lock is already held when +** this function is called, it is a no-op. +** +** The following operations are also performed by this function. +** +** 1) If the pager is currently in PAGER_OPEN state (no lock held +** on the database file), then an attempt is made to obtain a +** SHARED lock on the database file. Immediately after obtaining +** the SHARED lock, the file-system is checked for a hot-journal, +** which is played back if present. Following any hot-journal +** rollback, the contents of the cache are validated by checking +** the 'change-counter' field of the database file header and +** discarded if they are found to be invalid. +** +** 2) If the pager is running in exclusive-mode, and there are currently +** no outstanding references to any pages, and is in the error state, +** then an attempt is made to clear the error state by discarding +** the contents of the page cache and rolling back any open journal +** file. +** +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs while locking the database, checking for a hot-journal file or +** rolling back a journal file, the IO error code is returned. +*/ +SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + + /* This routine is only called from b-tree and only when there are no + ** outstanding pages. This implies that the pager state should either + ** be OPEN or READER. READER is only possible if the pager is or was in + ** exclusive access mode. + */ + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } + + if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ + int bHotJournal = 1; /* True if there exists a hot journal-file */ + + assert( !MEMDB ); + + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); + goto failed; + } + + /* If a journal file exists, and there is no RESERVED lock on the + ** database file, then it either needs to be played back or deleted. + */ + if( pPager->eLock<=SHARED_LOCK ){ + rc = hasHotJournal(pPager, &bHotJournal); + } + if( rc!=SQLITE_OK ){ + goto failed; + } + if( bHotJournal ){ + if( pPager->readOnly ){ + rc = SQLITE_READONLY_ROLLBACK; + goto failed; + } + + /* Get an EXCLUSIVE lock on the database file. At this point it is + ** important that a RESERVED lock is not obtained on the way to the + ** EXCLUSIVE lock. If it were, another process might open the + ** database file, detect the RESERVED lock, and conclude that the + ** database is safe to read while this process is still rolling the + ** hot-journal back. + ** + ** Because the intermediate RESERVED lock is not requested, any + ** other process attempting to access the database file will get to + ** this point in the code and fail to obtain its own EXCLUSIVE lock + ** on the database file. + ** + ** Unless the pager is in locking_mode=exclusive mode, the lock is + ** downgraded to SHARED_LOCK before this function returns. + */ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + goto failed; + } + + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist + ** mode (and also for journal_mode=truncate on some systems). + ** + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it + ** may mean that the pager was in the error-state when this + ** function was called and the journal file does not exist. + */ + if( !isOpen(pPager->jfd) ){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int bExists; /* True if journal file exists */ + rc = sqlite3OsAccess( + pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists ){ + int fout = 0; + int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + assert( !pPager->tempFile ); + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ + rc = SQLITE_CANTOPEN_BKPT; + sqlite3OsClose(pPager->jfd); + } + } + } + + /* Playback and delete the journal. Drop the database write + ** lock and reacquire the read lock. Purge the cache before + ** playing back the hot-journal so that we don't end up with + ** an inconsistent cache. Sync the hot journal before playing + ** it back since the process that crashed and left the hot journal + ** probably did not sync it and we are required to always sync + ** the journal before playing it back. + */ + if( isOpen(pPager->jfd) ){ + assert( rc==SQLITE_OK ); + rc = pagerSyncHotJournal(pPager); + if( rc==SQLITE_OK ){ + rc = pager_playback(pPager, 1); + pPager->eState = PAGER_OPEN; + } + }else if( !pPager->exclusiveMode ){ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + if( rc!=SQLITE_OK ){ + /* This branch is taken if an error occurs while trying to open + ** or roll back a hot-journal while holding an EXCLUSIVE lock. The + ** pager_unlock() routine will be called before returning to unlock + ** the file. If the unlock attempt fails, then Pager.eLock must be + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). + ** + ** In order to get pager_unlock() to do this, set Pager.eState to + ** PAGER_ERROR now. This is not actually counted as a transition + ** to ERROR state in the state diagram at the top of this file, + ** since we know that the same call to pager_unlock() will very + ** shortly transition the pager object to the OPEN state. Calling + ** assert_pager_state() would fail now, as it should not be possible + ** to be in ERROR state when there are zero outstanding page + ** references. + */ + pager_error(pPager, rc); + goto failed; + } + + assert( pPager->eState==PAGER_OPEN ); + assert( (pPager->eLock==SHARED_LOCK) + || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) + ); + } + + if( !pPager->tempFile && ( + pPager->pBackup + || sqlite3PcachePagecount(pPager->pPCache)>0 + || USEFETCH(pPager) + )){ + /* The shared-lock has just been acquired on the database file + ** and there are already pages in the cache (from a previous + ** read or write transaction). Check to see if the database + ** has been modified. If the database has changed, flush the + ** cache. + ** + ** Database changes is detected by looking at 15 bytes beginning + ** at offset 24 into the file. The first 4 of these 16 bytes are + ** a 32-bit counter that is incremented with each change. The + ** other bytes change randomly with each file change when + ** a codec is in use. + ** + ** There is a vanishingly small chance that a change will not be + ** detected. The chance of an undetected change is so small that + ** it can be neglected. + */ + Pgno nPage = 0; + char dbFileVers[sizeof(pPager->dbFileVers)]; + + rc = pagerPagecount(pPager, &nPage); + if( rc ) goto failed; + + if( nPage>0 ){ + IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ + goto failed; + } + }else{ + memset(dbFileVers, 0, sizeof(dbFileVers)); + } + + if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ + pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + if( USEFETCH(pPager) ){ + sqlite3OsUnfetch(pPager->fd, 0, 0); + } + } + } + + /* If there is a WAL file in the file-system, open this database in WAL + ** mode. Otherwise, the following function call is a no-op. + */ + rc = pagerOpenWalIfPresent(pPager); +#ifndef SQLITE_OMIT_WAL + assert( pPager->pWal==0 || rc==SQLITE_OK ); +#endif + } + + if( pagerUseWal(pPager) ){ + assert( rc==SQLITE_OK ); + rc = pagerBeginReadTransaction(pPager); + } + + if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + rc = pagerPagecount(pPager, &pPager->dbSize); + } + + failed: + if( rc!=SQLITE_OK ){ + assert( !MEMDB ); + pager_unlock(pPager); + assert( pPager->eState==PAGER_OPEN ); + }else{ + pPager->eState = PAGER_READER; + } + return rc; +} + +/* +** If the reference count has reached zero, rollback any active +** transaction and unlock the pager. +** +** Except, in locking_mode=EXCLUSIVE when there is nothing to in +** the rollback journal, the unlock is not performed and there is +** nothing to rollback, so this routine is a no-op. +*/ +static void pagerUnlockIfUnused(Pager *pPager){ + if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + pagerUnlockAndRollback(pPager); + } +} + +/* +** Acquire a reference to page number pgno in pager pPager (a page +** reference has type DbPage*). If the requested reference is +** successfully obtained, it is copied to *ppPage and SQLITE_OK returned. +** +** If the requested page is already in the cache, it is returned. +** Otherwise, a new page object is allocated and populated with data +** read from the database file. In some cases, the pcache module may +** choose not to allocate a new page object and may reuse an existing +** object with no outstanding references. +** +** The extra data appended to a page is always initialized to zeros the +** first time a page is loaded into memory. If the page requested is +** already in the cache when this function is called, then the extra +** data is left as it was when the page object was last used. +** +** If the database image is smaller than the requested page or if a +** non-zero value is passed as the noContent parameter and the +** requested page is not already stored in the cache, then no +** actual disk read occurs. In this case the memory image of the +** page is initialized to all zeros. +** +** If noContent is true, it means that we do not care about the contents +** of the page. This occurs in two scenarios: +** +** a) When reading a free-list leaf page from the database, and +** +** b) When a savepoint is being rolled back and we need to load +** a new page into the cache to be filled with the data read +** from the savepoint journal. +** +** If noContent is true, then the data returned is zeroed instead of +** being read from the database. Additionally, the bits corresponding +** to pgno in Pager.pInJournal (bitvec of pages already written to the +** journal file) and the PagerSavepoint.pInSavepoint bitvecs of any open +** savepoints are set. This means if the page is made writable at any +** point in the future, using a call to sqlite3PagerWrite(), its contents +** will not be journaled. This saves IO. +** +** The acquisition might fail for several reasons. In all cases, +** an appropriate error code is returned and *ppPage is set to NULL. +** +** See also sqlite3PagerLookup(). Both this routine and Lookup() attempt +** to find a page in the in-memory cache first. If the page is not already +** in memory, this routine goes to disk to read it in whereas Lookup() +** just returns 0. This routine acquires a read-lock the first time it +** has to go to disk, and could also playback an old journal if necessary. +** Since Lookup() never goes to disk, it never has to deal with locks +** or journal files. +*/ +SQLITE_PRIVATE int sqlite3PagerAcquire( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + const int noContent = (flags & PAGER_GET_NOCONTENT); + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) +#ifdef SQLITE_HAS_CODEC + && pPager->xCodec==0 +#endif + ); + + assert( pPager->eState>=PAGER_READER ); + assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); + + if( pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } + + /* If the pager is in the error state, return an error immediately. + ** Otherwise, request the page from the PCache layer. */ + if( pPager->errCode!=SQLITE_OK ){ + rc = pPager->errCode; + }else{ + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( bMmapOk && iFrame==0 ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER ){ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } + } + + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); + } + + if( rc!=SQLITE_OK ){ + /* Either the call to sqlite3PcacheFetch() returned an error or the + ** pager was already in the error-state when this function was called. + ** Set pPg to 0 and jump to the exception handler. */ + pPg = 0; + goto pager_acquire_err; + } + assert( (*ppPage)->pgno==pgno ); + assert( (*ppPage)->pPager==pPager || (*ppPage)->pPager==0 ); + + if( (*ppPage)->pPager && !noContent ){ + /* In this case the pcache already contains an initialized copy of + ** the page. Return without further ado. */ + assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); + pPager->aStat[PAGER_STAT_HIT]++; + return SQLITE_OK; + + }else{ + /* The pager cache has created a new page. Its content needs to + ** be initialized. */ + + pPg = *ppPage; + pPg->pPager = pPager; + + /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page + ** number greater than this, or the unused locking-page, is requested. */ + if( pgno>PAGER_MAX_PGNO || pgno==PAGER_MJ_PGNO(pPager) ){ + rc = SQLITE_CORRUPT_BKPT; + goto pager_acquire_err; + } + + if( MEMDB || pPager->dbSizefd) ){ + if( pgno>pPager->mxPgno ){ + rc = SQLITE_FULL; + goto pager_acquire_err; + } + if( noContent ){ + /* Failure to set the bits in the InJournal bit-vectors is benign. + ** It merely means that we might do some extra work to journal a + ** page that does not need to be journaled. Nevertheless, be sure + ** to test the case where a malloc error occurs while trying to set + ** a bit in a bit vector. + */ + sqlite3BeginBenignMalloc(); + if( pgno<=pPager->dbOrigSize ){ + TESTONLY( rc = ) sqlite3BitvecSet(pPager->pInJournal, pgno); + testcase( rc==SQLITE_NOMEM ); + } + TESTONLY( rc = ) addToSavepointBitvecs(pPager, pgno); + testcase( rc==SQLITE_NOMEM ); + sqlite3EndBenignMalloc(); + } + memset(pPg->pData, 0, pPager->pageSize); + IOTRACE(("ZERO %p %d\n", pPager, pgno)); + }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + assert( pPg->pPager==pPager ); + pPager->aStat[PAGER_STAT_MISS]++; + rc = readDbPage(pPg, iFrame); + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } + } + pager_set_pagehash(pPg); + } + + return SQLITE_OK; + +pager_acquire_err: + assert( rc!=SQLITE_OK ); + if( pPg ){ + sqlite3PcacheDrop(pPg); + } + pagerUnlockIfUnused(pPager); + + *ppPage = 0; + return rc; +} + +/* +** Acquire a page if it is already in the in-memory cache. Do +** not read the page from disk. Return a pointer to the page, +** or 0 if the page is not in cache. +** +** See also sqlite3PagerGet(). The difference between this routine +** and sqlite3PagerGet() is that _get() will go to the disk and read +** in the page if the page is not already in cache. This routine +** returns NULL if the page is not in cache or if a disk I/O error +** has ever happened. +*/ +SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ + PgHdr *pPg = 0; + assert( pPager!=0 ); + assert( pgno!=0 ); + assert( pPager->pPCache!=0 ); + assert( pPager->eState>=PAGER_READER && pPager->eState!=PAGER_ERROR ); + sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + return pPg; +} + +/* +** Release a page reference. +** +** If the number of references to the page drop to zero, then the +** page is added to the LRU list. When all references to all pages +** are released, a rollback occurs and the lock on the database is +** removed. +*/ +SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage *pPg){ + Pager *pPager; + assert( pPg!=0 ); + pPager = pPg->pPager; + if( pPg->flags & PGHDR_MMAP ){ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } + pagerUnlockIfUnused(pPager); +} +SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ + if( pPg ) sqlite3PagerUnrefNotNull(pPg); +} + +/* +** This function is called at the start of every write transaction. +** There must already be a RESERVED or EXCLUSIVE lock on the database +** file when this routine is called. +** +** Open the journal file for pager pPager and write a journal header +** to the start of it. If there are active savepoints, open the sub-journal +** as well. This function is only used when the journal file is being +** opened to write a rollback log for a transaction. It is not used +** when opening a hot journal file to roll it back. +** +** If the journal file is already open (as it may be in exclusive mode), +** then this function just writes a journal header to the start of the +** already open file. +** +** Whether or not the journal file is opened by this function, the +** Pager.pInJournal bitvec structure is allocated. +** +** Return SQLITE_OK if everything is successful. Otherwise, return +** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or +** an IO error code if opening or writing the journal file fails. +*/ +static int pager_open_journal(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ + + assert( pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + assert( pPager->pInJournal==0 ); + + /* If already in the error state, this function is a no-op. But on + ** the other hand, this routine is never called if we are already in + ** an error state. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + + if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pInJournal==0 ){ + return SQLITE_NOMEM; + } + + /* Open the journal file if it is not already open. */ + if( !isOpen(pPager->jfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->jfd); + }else{ + const int flags = /* VFS flags to open journal file */ + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + (pPager->tempFile ? + (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): + (SQLITE_OPEN_MAIN_JOURNAL) + ); + + /* Verify that the database still has the same name as it did when + ** it was originally opened. */ + rc = databaseIsUnmoved(pPager); + if( rc==SQLITE_OK ){ +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + rc = sqlite3JournalOpen( + pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + ); +#else + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); +#endif + } + } + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + } + + + /* Write the first journal header to the journal file and open + ** the sub-journal if necessary. + */ + if( rc==SQLITE_OK ){ + /* TODO: Check if all of these are really required. */ + pPager->nRec = 0; + pPager->journalOff = 0; + pPager->setMaster = 0; + pPager->journalHdr = 0; + rc = writeJournalHdr(pPager); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + }else{ + assert( pPager->eState==PAGER_WRITER_LOCKED ); + pPager->eState = PAGER_WRITER_CACHEMOD; + } + + return rc; +} + +/* +** Begin a write-transaction on the specified pager object. If a +** write-transaction has already been opened, this function is a no-op. +** +** If the exFlag argument is false, then acquire at least a RESERVED +** lock on the database file. If exFlag is true, then acquire at least +** an EXCLUSIVE lock. If such a lock is already held, no locking +** functions need be called. +** +** If the subjInMemory argument is non-zero, then any sub-journal opened +** within this transaction will be opened as an in-memory file. This +** has no effect if the sub-journal is already opened (as it may be when +** running in exclusive mode) or if the transaction does not require a +** sub-journal. If the subjInMemory argument is zero, then any required +** sub-journal is implemented in-memory if pPager is an in-memory database, +** or using a temporary file otherwise. +*/ +SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ + int rc = SQLITE_OK; + + if( pPager->errCode ) return pPager->errCode; + assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; + + if( ALWAYS(pPager->eState==PAGER_READER) ){ + assert( pPager->pInJournal==0 ); + + if( pagerUseWal(pPager) ){ + /* If the pager is configured to use locking_mode=exclusive, and an + ** exclusive lock on the database is not already held, obtain it now. + */ + if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + return rc; + } + sqlite3WalExclusiveMode(pPager->pWal, 1); + } + + /* Grab the write lock on the log file. If successful, upgrade to + ** PAGER_RESERVED state. Otherwise, return an error code to the caller. + ** The busy-handler is not invoked if another connection already + ** holds the write-lock. If possible, the upper layer will call it. + */ + rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + }else{ + /* Obtain a RESERVED lock on the database file. If the exFlag parameter + ** is true, then immediately upgrade this to an EXCLUSIVE lock. The + ** busy-handler callback can be used when upgrading to the EXCLUSIVE + ** lock, but not when obtaining the RESERVED lock. + */ + rc = pagerLockDb(pPager, RESERVED_LOCK); + if( rc==SQLITE_OK && exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + } + + if( rc==SQLITE_OK ){ + /* Change to WRITER_LOCKED state. + ** + ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD + ** when it has an open transaction, but never to DBMOD or FINISHED. + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in + ** WAL mode. + */ + pPager->eState = PAGER_WRITER_LOCKED; + pPager->dbHintSize = pPager->dbSize; + pPager->dbFileSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; + pPager->journalOff = 0; + } + + assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); + assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + } + + PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); + return rc; +} + +/* +** Mark a single data page as writeable. The page is written into the +** main journal or sub-journal as required. If the page is written into +** one of the journals, the corresponding bit is set in the +** Pager.pInJournal bitvec and the PagerSavepoint.pInSavepoint bitvecs +** of any open savepoints as appropriate. +*/ +static int pager_write(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + int rc = SQLITE_OK; + int inJournal; + + /* This routine is not called unless a write-transaction has already + ** been started. The journal file may or may not be open at this point. + ** It is never called in the ERROR state. + */ + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + assert( pPager->errCode==0 ); + assert( pPager->readOnly==0 ); + + CHECK_PAGE(pPg); + + /* The journal file needs to be opened. Higher level routines have already + ** obtained the necessary locks to begin the write-transaction, but the + ** rollback journal might not yet be open. Open it now if this is the case. + ** + ** This is done before calling sqlite3PcacheMakeDirty() on the page. + ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then + ** an error might occur and the pager would end up in WRITER_LOCKED state + ** with pages marked as dirty in the cache. + */ + if( pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_open_journal(pPager); + if( rc!=SQLITE_OK ) return rc; + } + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( assert_pager_state(pPager) ); + + /* Mark the page as dirty. If the page has already been written + ** to the journal then we can return right away. + */ + sqlite3PcacheMakeDirty(pPg); + inJournal = pageInJournal(pPager, pPg); + if( inJournal && (pPager->nSavepoint==0 || !subjRequiresPage(pPg)) ){ + assert( !pagerUseWal(pPager) ); + }else{ + + /* The transaction journal now exists and we have a RESERVED or an + ** EXCLUSIVE lock on the main database file. Write the current page to + ** the transaction journal if it is not there already. + */ + if( !inJournal && !pagerUseWal(pPager) ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ + u32 cksum; + char *pData2; + i64 iOff = pPager->journalOff; + + /* We should never write to the journal file the page that + ** contains the database locks. The following assert verifies + ** that we do not. */ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + + assert( pPager->journalHdr<=pPager->journalOff ); + CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + cksum = pager_cksum(pPager, (u8*)pData2); + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + + pPager->journalOff += 8 + pPager->pageSize; + pPager->nRec++; + assert( pPager->pInJournal!=0 ); + rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + rc |= addToSavepointBitvecs(pPager, pPg->pgno); + if( rc!=SQLITE_OK ){ + assert( rc==SQLITE_NOMEM ); + return rc; + } + }else{ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ + pPg->flags |= PGHDR_NEED_SYNC; + } + PAGERTRACE(("APPEND %d page %d needSync=%d\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); + } + } + + /* If the statement journal is open and the page is not in it, + ** then write the current page to the statement journal. Note that + ** the statement journal format differs from the standard journal format + ** in that it omits the checksums and the header. + */ + if( pPager->nSavepoint>0 && subjRequiresPage(pPg) ){ + rc = subjournalPage(pPg); + } + } + + /* Update the database size and return. + */ + if( pPager->dbSizepgno ){ + pPager->dbSize = pPg->pgno; + } + return rc; +} + +/* +** Mark a data page as writeable. This routine must be called before +** making changes to a page. The caller must check the return value +** of this function and be careful not to change any page data unless +** this routine returns SQLITE_OK. +** +** The difference between this function and pager_write() is that this +** function also deals with the special case where 2 or more pages +** fit on a single disk sector. In this case all co-resident pages +** must have been written to the journal file before returning. +** +** If an error occurs, SQLITE_NOMEM or an IO error code is returned +** as appropriate. Otherwise, SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3PagerWrite(DbPage *pDbPage){ + int rc = SQLITE_OK; + + PgHdr *pPg = pDbPage; + Pager *pPager = pPg->pPager; + + assert( (pPg->flags & PGHDR_MMAP)==0 ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPager) ); + + if( pPager->sectorSize > (u32)pPager->pageSize ){ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + + /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow + ** a journal header to be written between the pages journaled by + ** this function. + */ + assert( !MEMDB ); + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)==0 ); + pPager->doNotSpill |= SPILLFLAG_NOSYNC; + + /* This trick assumes that both the page-size and sector-size are + ** an integer power of 2. It sets variable pg1 to the identifier + ** of the first page of the sector pPg is located on. + */ + pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; + + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; + } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); + + for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ + if( pg!=PAGER_MJ_PGNO(pPager) ){ + rc = sqlite3PagerGet(pPager, pg, &pPage); + if( rc==SQLITE_OK ){ + rc = pager_write(pPage); + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + ** starting at pg1, then it needs to be set for all of them. Because + ** writing to any of these nPage pages may damage the others, the + ** journal file must contain sync()ed copies of all of them + ** before any of them can be written out to the database file. + */ + if( rc==SQLITE_OK && needSync ){ + assert( !MEMDB ); + for(ii=0; iiflags |= PGHDR_NEED_SYNC; + sqlite3PagerUnrefNotNull(pPage); + } + } + } + + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)!=0 ); + pPager->doNotSpill &= ~SPILLFLAG_NOSYNC; + }else{ + rc = pager_write(pDbPage); + } + return rc; +} + +/* +** Return TRUE if the page given in the argument was previously passed +** to sqlite3PagerWrite(). In other words, return TRUE if it is ok +** to change the content of the page. +*/ +#ifndef NDEBUG +SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ + return pPg->flags&PGHDR_DIRTY; +} +#endif + +/* +** A call to this routine tells the pager that it is not necessary to +** write the information on page pPg back to the disk, even though +** that page might be marked as dirty. This happens, for example, when +** the page has been added as a leaf of the freelist and so its +** content no longer matters. +** +** The overlying software layer calls this routine when all of the data +** on the given page is unused. The pager marks the page as clean so +** that it does not get written to disk. +** +** Tests show that this optimization can quadruple the speed of large +** DELETE operations. +*/ +SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + if( (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ + PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); + IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) + pPg->flags |= PGHDR_DONT_WRITE; + pager_set_pagehash(pPg); + } +} + +/* +** This routine is called to increment the value of the database file +** change-counter, stored as a 4-byte big-endian integer starting at +** byte offset 24 of the pager file. The secondary change counter at +** 92 is also updated, as is the SQLite version number at offset 96. +** +** But this only happens if the pPager->changeCountDone flag is false. +** To avoid excess churning of page 1, the update only happens once. +** See also the pager_write_changecounter() routine that does an +** unconditional update of the change counters. +** +** If the isDirectMode flag is zero, then this is done by calling +** sqlite3PagerWrite() on page 1, then modifying the contents of the +** page data. In this case the file will be updated when the current +** transaction is committed. +** +** The isDirectMode flag may only be non-zero if the library was compiled +** with the SQLITE_ENABLE_ATOMIC_WRITE macro defined. In this case, +** if isDirect is non-zero, then the database file is updated directly +** by writing an updated version of page 1 using a call to the +** sqlite3OsWrite() function. +*/ +static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ + int rc = SQLITE_OK; + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + + /* Declare and initialize constant integer 'isDirect'. If the + ** atomic-write optimization is enabled in this build, then isDirect + ** is initialized to the value passed as the isDirectMode parameter + ** to this function. Otherwise, it is always set to zero. + ** + ** The idea is that if the atomic-write optimization is not + ** enabled at compile time, the compiler can omit the tests of + ** 'isDirect' below, as well as the block enclosed in the + ** "if( isDirect )" condition. + */ +#ifndef SQLITE_ENABLE_ATOMIC_WRITE +# define DIRECT_MODE 0 + assert( isDirectMode==0 ); + UNUSED_PARAMETER(isDirectMode); +#else +# define DIRECT_MODE isDirectMode +#endif + + if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){ + PgHdr *pPgHdr; /* Reference to page 1 */ + + assert( !pPager->tempFile && isOpen(pPager->fd) ); + + /* Open page 1 of the file for writing. */ + rc = sqlite3PagerGet(pPager, 1, &pPgHdr); + assert( pPgHdr==0 || rc==SQLITE_OK ); + + /* If page one was fetched successfully, and this function is not + ** operating in direct-mode, make page 1 writable. When not in + ** direct mode, page 1 is always held in cache and hence the PagerGet() + ** above is always successful - hence the ALWAYS on rc==SQLITE_OK. + */ + if( !DIRECT_MODE && ALWAYS(rc==SQLITE_OK) ){ + rc = sqlite3PagerWrite(pPgHdr); + } + + if( rc==SQLITE_OK ){ + /* Actually do the update of the change counter */ + pager_write_changecounter(pPgHdr); + + /* If running in direct mode, write the contents of page 1 to the file. */ + if( DIRECT_MODE ){ + const void *zBuf; + assert( pPager->dbFileSize>0 ); + CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; + } + if( rc==SQLITE_OK ){ + /* Update the pager's copy of the change-counter. Otherwise, the + ** next time a read transaction is opened the cache will be + ** flushed (as the change-counter values will not match). */ + const void *pCopy = (const void *)&((const char *)zBuf)[24]; + memcpy(&pPager->dbFileVers, pCopy, sizeof(pPager->dbFileVers)); + pPager->changeCountDone = 1; + } + }else{ + pPager->changeCountDone = 1; + } + } + + /* Release the page reference. */ + sqlite3PagerUnref(pPgHdr); + } + return rc; +} + +/* +** Sync the database file to disk. This is a no-op for in-memory databases +** or pages with the Pager.noSync flag set. +** +** If successful, or if called on a pager for which it is a no-op, this +** function returns SQLITE_OK. Otherwise, an IO error code is returned. +*/ +SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zMaster){ + int rc = SQLITE_OK; + + if( isOpen(pPager->fd) ){ + void *pArg = (void*)zMaster; + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC, pArg); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + } + if( rc==SQLITE_OK && !pPager->noSync ){ + assert( !MEMDB ); + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); + } + return rc; +} + +/* +** This function may only be called while a write-transaction is active in +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** the database file, an attempt is made to obtain one. +** +** If the EXCLUSIVE lock is already held or the attempt to obtain it is +** successful, or the connection is in WAL mode, SQLITE_OK is returned. +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** returned. +*/ +SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ + int rc = SQLITE_OK; + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + return rc; +} + +/* +** Sync the database file for the pager pPager. zMaster points to the name +** of a master journal file that should be written into the individual +** journal file. zMaster may be NULL, which is interpreted as no master +** journal (a single database transaction). +** +** This routine ensures that: +** +** * The database file change-counter is updated, +** * the journal is synced (unless the atomic-write optimization is used), +** * all dirty pages are written to the database file, +** * the database file is truncated (if required), and +** * the database file synced. +** +** The only thing that remains to commit the transaction is to finalize +** (delete, truncate or zero the first part of) the journal file (or +** delete the master journal file if specified). +** +** Note that if zMaster==NULL, this does not overwrite a previous value +** passed to an sqlite3PagerCommitPhaseOne() call. +** +** If the final parameter - noSync - is true, then the database file itself +** is not synced. The caller must call sqlite3PagerSync() directly to +** sync the database file before calling CommitPhaseTwo() to delete the +** journal file in this case. +*/ +SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( + Pager *pPager, /* Pager object */ + const char *zMaster, /* If not NULL, the master journal name */ + int noSync /* True to omit the xSync on the db file */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_ERROR + ); + assert( assert_pager_state(pPager) ); + + /* If a prior error occurred, report that error again. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + + PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", + pPager->zFilename, zMaster, pPager->dbSize)); + + /* If no database changes have been made, return early. */ + if( pPager->eStatepBackup); + }else{ + if( pagerUseWal(pPager) ){ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + PgHdr *pPageOne = 0; + if( pList==0 ){ + /* Must have at least one page for the WAL commit flag. + ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ + rc = sqlite3PagerGet(pPager, 1, &pPageOne); + pList = pPageOne; + pList->pDirty = 0; + } + assert( rc==SQLITE_OK ); + if( ALWAYS(pList) ){ + rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1); + } + sqlite3PagerUnref(pPageOne); + if( rc==SQLITE_OK ){ + sqlite3PcacheCleanAll(pPager->pPCache); + } + }else{ + /* The following block updates the change-counter. Exactly how it + ** does this depends on whether or not the atomic-update optimization + ** was enabled at compile time, and if this transaction meets the + ** runtime criteria to use the operation: + ** + ** * The file-system supports the atomic-write property for + ** blocks of size page-size, and + ** * This commit is not part of a multi-file transaction, and + ** * Exactly one page has been modified and store in the journal file. + ** + ** If the optimization was not enabled at compile time, then the + ** pager_incr_changecounter() function is called to update the change + ** counter in 'indirect-mode'. If the optimization is compiled in but + ** is not applicable to this transaction, call sqlite3JournalCreate() + ** to make sure the journal file has actually been created, then call + ** pager_incr_changecounter() to update the change-counter in indirect + ** mode. + ** + ** Otherwise, if the optimization is both enabled and applicable, + ** then call pager_incr_changecounter() to update the change-counter + ** in 'direct' mode. In this case the journal file will never be + ** created for this transaction. + */ + #ifdef SQLITE_ENABLE_ATOMIC_WRITE + PgHdr *pPg; + assert( isOpen(pPager->jfd) + || pPager->journalMode==PAGER_JOURNALMODE_OFF + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + if( !zMaster && isOpen(pPager->jfd) + && pPager->journalOff==jrnlBufferSize(pPager) + && pPager->dbSize>=pPager->dbOrigSize + && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) + ){ + /* Update the db file change counter via the direct-write method. The + ** following call will modify the in-memory representation of page 1 + ** to include the updated change counter and then write page 1 + ** directly to the database file. Because of the atomic-write + ** property of the host file-system, this is safe. + */ + rc = pager_incr_changecounter(pPager, 1); + }else{ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc==SQLITE_OK ){ + rc = pager_incr_changecounter(pPager, 0); + } + } + #else + rc = pager_incr_changecounter(pPager, 0); + #endif + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* Write the master journal name into the journal file. If a master + ** journal file name has already been written to the journal file, + ** or if zMaster is NULL (no master journal), then this call is a no-op. + */ + rc = writeMasterJournal(pPager, zMaster); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* Sync the journal file and write all dirty pages to the database. + ** If the atomic-update optimization is being used, this sync will not + ** create the journal file or perform any real IO. + ** + ** Because the change-counter page was just modified, unless the + ** atomic-update optimization is used it is almost certain that the + ** journal requires a sync here. However, in locking_mode=exclusive + ** on a system under memory pressure it is just possible that this is + ** not the case. In this case it is likely enough that the redundant + ** xSync() call will be changed to a no-op by the OS anyhow. + */ + rc = syncJournal(pPager, 0); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); + if( rc!=SQLITE_OK ){ + assert( rc!=SQLITE_IOERR_BLOCKED ); + goto commit_phase_one_exit; + } + sqlite3PcacheCleanAll(pPager->pPCache); + + /* If the file on disk is smaller than the database image, use + ** pager_truncate to grow the file here. This can happen if the database + ** image was extended as part of the current transaction and then the + ** last page in the db image moved to the free-list. In this case the + ** last page is never written out to disk, leaving the database file + ** undersized. Fix this now if it is the case. */ + if( pPager->dbSize>pPager->dbFileSize ){ + Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + rc = pager_truncate(pPager, nNew); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + } + + /* Finally, sync the database file. */ + if( !noSync ){ + rc = sqlite3PagerSync(pPager, zMaster); + } + IOTRACE(("DBSYNC %p\n", pPager)) + } + } + +commit_phase_one_exit: + if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ + pPager->eState = PAGER_WRITER_FINISHED; + } + return rc; +} + + +/* +** When this function is called, the database file has been completely +** updated to reflect the changes made by the current transaction and +** synced to disk. The journal file still exists in the file-system +** though, and if a failure occurs at this point it will eventually +** be used as a hot-journal and the current transaction rolled back. +** +** This function finalizes the journal file, either by deleting, +** truncating or partially zeroing it, so that it cannot be used +** for hot-journal rollback. Once this is done the transaction is +** irrevocably committed. +** +** If an error occurs, an IO error code is returned and the pager +** moves into the error state. Otherwise, SQLITE_OK is returned. +*/ +SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + + /* This routine should not be called if a prior error has occurred. + ** But if (due to a coding error elsewhere in the system) it does get + ** called, just return the same error code without doing anything. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_FINISHED + || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) + ); + assert( assert_pager_state(pPager) ); + + /* An optimization. If the database was not actually modified during + ** this transaction, the pager is running in exclusive-mode and is + ** using persistent journals, then this function is a no-op. + ** + ** The start of the journal file currently contains a single journal + ** header with the nRec field set to 0. If such a journal is used as + ** a hot-journal during hot-journal rollback, 0 changes will be made + ** to the database file. So there is no need to zero the journal + ** header. Since the pager is in exclusive mode, there is no need + ** to drop any locks either. + */ + if( pPager->eState==PAGER_WRITER_LOCKED + && pPager->exclusiveMode + && pPager->journalMode==PAGER_JOURNALMODE_PERSIST + ){ + assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); + pPager->eState = PAGER_READER; + return SQLITE_OK; + } + + PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + rc = pager_end_transaction(pPager, pPager->setMaster, 1); + return pager_error(pPager, rc); +} + +/* +** If a write transaction is open, then all changes made within the +** transaction are reverted and the current write-transaction is closed. +** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR +** state if an error occurs. +** +** If the pager is already in PAGER_ERROR state when this function is called, +** it returns Pager.errCode immediately. No work is performed in this case. +** +** Otherwise, in rollback mode, this function performs two functions: +** +** 1) It rolls back the journal file, restoring all database file and +** in-memory cache pages to the state they were in when the transaction +** was opened, and +** +** 2) It finalizes the journal file, so that it is not used for hot +** rollback at any point in the future. +** +** Finalization of the journal file (task 2) is only performed if the +** rollback is successful. +** +** In WAL mode, all cache-entries containing data modified within the +** current transaction are either expelled from the cache or reverted to +** their pre-transaction state by re-reading data from the database or +** WAL files. The WAL transaction is then closed. +*/ +SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); + + /* PagerRollback() is a no-op if called in READER or OPEN state. If + ** the pager is already in the ERROR state, the rollback is not + ** attempted here. Instead, the error code is returned to the caller. + */ + assert( assert_pager_state(pPager) ); + if( pPager->eState==PAGER_ERROR ) return pPager->errCode; + if( pPager->eState<=PAGER_READER ) return SQLITE_OK; + + if( pagerUseWal(pPager) ){ + int rc2; + rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); + rc2 = pager_end_transaction(pPager, pPager->setMaster, 0); + if( rc==SQLITE_OK ) rc = rc2; + }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ + int eState = pPager->eState; + rc = pager_end_transaction(pPager, 0, 0); + if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ + /* This can happen using journal_mode=off. Move the pager to the error + ** state to indicate that the contents of the cache may not be trusted. + ** Any active readers will get SQLITE_ABORT. + */ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + return rc; + } + }else{ + rc = pager_playback(pPager, 0); + } + + assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT + || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR + || rc==SQLITE_CANTOPEN + ); + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error persistent. + */ + return pager_error(pPager, rc); +} + +/* +** Return TRUE if the database file is opened read-only. Return FALSE +** if the database is (in theory) writable. +*/ +SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager *pPager){ + return pPager->readOnly; +} + +/* +** Return the number of references to the pager. +*/ +SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ + return sqlite3PcacheRefCount(pPager->pPCache); +} + +/* +** Return the approximate number of bytes of memory currently +** used by the pager and its associated cache. +*/ +SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager *pPager){ + int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr) + + 5*sizeof(void*); + return perPageSize*sqlite3PcachePagecount(pPager->pPCache) + + sqlite3MallocSize(pPager) + + pPager->pageSize; +} + +/* +** Return the number of references to the specified page. +*/ +SQLITE_PRIVATE int sqlite3PagerPageRefcount(DbPage *pPage){ + return sqlite3PcachePageRefcount(pPage); +} + +#ifdef SQLITE_TEST +/* +** This routine is used for testing and analysis only. +*/ +SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ + static int a[11]; + a[0] = sqlite3PcacheRefCount(pPager->pPCache); + a[1] = sqlite3PcachePagecount(pPager->pPCache); + a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); + a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; + a[4] = pPager->eState; + a[5] = pPager->errCode; + a[6] = pPager->aStat[PAGER_STAT_HIT]; + a[7] = pPager->aStat[PAGER_STAT_MISS]; + a[8] = 0; /* Used to be pPager->nOvfl */ + a[9] = pPager->nRead; + a[10] = pPager->aStat[PAGER_STAT_WRITE]; + return a; +} +#endif + +/* +** Parameter eStat must be either SQLITE_DBSTATUS_CACHE_HIT or +** SQLITE_DBSTATUS_CACHE_MISS. Before returning, *pnVal is incremented by the +** current cache hit or miss count, according to the value of eStat. If the +** reset parameter is non-zero, the cache hit or miss count is zeroed before +** returning. +*/ +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ + + assert( eStat==SQLITE_DBSTATUS_CACHE_HIT + || eStat==SQLITE_DBSTATUS_CACHE_MISS + || eStat==SQLITE_DBSTATUS_CACHE_WRITE + ); + + assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS ); + assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE ); + assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 && PAGER_STAT_WRITE==2 ); + + *pnVal += pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT]; + if( reset ){ + pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT] = 0; + } +} + +/* +** Return true if this is an in-memory pager. +*/ +SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ + return MEMDB; +} + +/* +** Check that there are at least nSavepoint savepoints open. If there are +** currently less than nSavepoints open, then open one or more savepoints +** to make up the difference. If the number of savepoints is already +** equal to nSavepoint, then this function is a no-op. +** +** If a memory allocation fails, SQLITE_NOMEM is returned. If an error +** occurs while opening the sub-journal file, then an IO error code is +** returned. Otherwise, SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ + int rc = SQLITE_OK; /* Return code */ + int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + + if( nSavepoint>nCurrent && pPager->useJournal ){ + int ii; /* Iterator variable */ + PagerSavepoint *aNew; /* New Pager.aSavepoint array */ + + /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM + ** if the allocation fails. Otherwise, zero the new portion in case a + ** malloc failure occurs while populating it in the for(...) loop below. + */ + aNew = (PagerSavepoint *)sqlite3Realloc( + pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint + ); + if( !aNew ){ + return SQLITE_NOMEM; + } + memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); + pPager->aSavepoint = aNew; + + /* Populate the PagerSavepoint structures just allocated. */ + for(ii=nCurrent; iidbSize; + if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ + aNew[ii].iOffset = pPager->journalOff; + }else{ + aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); + } + aNew[ii].iSubRec = pPager->nSubRec; + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + if( !aNew[ii].pInSavepoint ){ + return SQLITE_NOMEM; + } + if( pagerUseWal(pPager) ){ + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); + } + pPager->nSavepoint = ii+1; + } + assert( pPager->nSavepoint==nSavepoint ); + assertTruncateConstraint(pPager); + } + + return rc; +} + +/* +** This function is called to rollback or release (commit) a savepoint. +** The savepoint to release or rollback need not be the most recently +** created savepoint. +** +** Parameter op is always either SAVEPOINT_ROLLBACK or SAVEPOINT_RELEASE. +** If it is SAVEPOINT_RELEASE, then release and destroy the savepoint with +** index iSavepoint. If it is SAVEPOINT_ROLLBACK, then rollback all changes +** that have occurred since the specified savepoint was created. +** +** The savepoint to rollback or release is identified by parameter +** iSavepoint. A value of 0 means to operate on the outermost savepoint +** (the first created). A value of (Pager.nSavepoint-1) means operate +** on the most recently created savepoint. If iSavepoint is greater than +** (Pager.nSavepoint-1), then this function is a no-op. +** +** If a negative value is passed to this function, then the current +** transaction is rolled back. This is different to calling +** sqlite3PagerRollback() because this function does not terminate +** the transaction or unlock the database, it just restores the +** contents of the database to its original state. +** +** In any case, all savepoints with an index greater than iSavepoint +** are destroyed. If this is a release operation (op==SAVEPOINT_RELEASE), +** then savepoint iSavepoint is also destroyed. +** +** This function may return SQLITE_NOMEM if a memory allocation fails, +** or an IO error code if an IO error occurs while rolling back a +** savepoint. If no errors occur, SQLITE_OK is returned. +*/ +SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ + int rc = pPager->errCode; /* Return code */ + + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); + + if( rc==SQLITE_OK && iSavepointnSavepoint ){ + int ii; /* Iterator variable */ + int nNew; /* Number of remaining savepoints after this op. */ + + /* Figure out how many savepoints will still be active after this + ** operation. Store this value in nNew. Then free resources associated + ** with any savepoints that are destroyed by this operation. + */ + nNew = iSavepoint + (( op==SAVEPOINT_RELEASE ) ? 0 : 1); + for(ii=nNew; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); + } + pPager->nSavepoint = nNew; + + /* If this is a release of the outermost savepoint, truncate + ** the sub-journal to zero bytes in size. */ + if( op==SAVEPOINT_RELEASE ){ + if( nNew==0 && isOpen(pPager->sjfd) ){ + /* Only truncate if it is an in-memory sub-journal. */ + if( sqlite3IsMemJournal(pPager->sjfd) ){ + rc = sqlite3OsTruncate(pPager->sjfd, 0); + assert( rc==SQLITE_OK ); + } + pPager->nSubRec = 0; + } + } + /* Else this is a rollback operation, playback the specified savepoint. + ** If this is a temp-file, it is possible that the journal file has + ** not yet been opened. In this case there have been no changes to + ** the database file, so the playback operation can be skipped. + */ + else if( pagerUseWal(pPager) || isOpen(pPager->jfd) ){ + PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; + rc = pagerPlaybackSavepoint(pPager, pSavepoint); + assert(rc!=SQLITE_DONE); + } + } + + return rc; +} + +/* +** Return the full pathname of the database file. +** +** Except, if the pager is in-memory only, then return an empty string if +** nullIfMemDb is true. This routine is called with nullIfMemDb==1 when +** used to report the filename to the user, for compatibility with legacy +** behavior. But when the Btree needs to know the filename for matching to +** shared cache, it uses nullIfMemDb==0 so that in-memory databases can +** participate in shared-cache. +*/ +SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager, int nullIfMemDb){ + return (nullIfMemDb && pPager->memDb) ? "" : pPager->zFilename; +} + +/* +** Return the VFS structure for the pager. +*/ +SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ + return pPager->pVfs; +} + +/* +** Return the file handle for the database file associated +** with the pager. This might return NULL if the file has +** not yet been opened. +*/ +SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ + return pPager->fd; +} + +/* +** Return the full pathname of the journal file. +*/ +SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ + return pPager->zJournal; +} + +/* +** Return true if fsync() calls are disabled for this pager. Return FALSE +** if fsync()s are executed normally. +*/ +SQLITE_PRIVATE int sqlite3PagerNosync(Pager *pPager){ + return pPager->noSync; +} + +#ifdef SQLITE_HAS_CODEC +/* +** Set or retrieve the codec for this pager +*/ +SQLITE_PRIVATE void sqlite3PagerSetCodec( + Pager *pPager, + void *(*xCodec)(void*,void*,Pgno,int), + void (*xCodecSizeChng)(void*,int,int), + void (*xCodecFree)(void*), + void *pCodec +){ + if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); + pPager->xCodec = pPager->memDb ? 0 : xCodec; + pPager->xCodecSizeChng = xCodecSizeChng; + pPager->xCodecFree = xCodecFree; + pPager->pCodec = pCodec; + pagerReportSize(pPager); +} +SQLITE_PRIVATE void *sqlite3PagerGetCodec(Pager *pPager){ + return pPager->pCodec; +} + +/* +** This function is called by the wal module when writing page content +** into the log file. +** +** This function returns a pointer to a buffer containing the encrypted +** page content. If a malloc fails, this function may return NULL. +*/ +SQLITE_PRIVATE void *sqlite3PagerCodec(PgHdr *pPg){ + void *aData = 0; + CODEC2(pPg->pPager, pPg->pData, pPg->pgno, 6, return 0, aData); + return aData; +} + +/* +** Return the current pager state +*/ +SQLITE_PRIVATE int sqlite3PagerState(Pager *pPager){ + return pPager->eState; +} +#endif /* SQLITE_HAS_CODEC */ + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Move the page pPg to location pgno in the file. +** +** There must be no references to the page previously located at +** pgno (which we call pPgOld) though that page is allowed to be +** in cache. If the page previously located at pgno is not already +** in the rollback journal, it is not put there by by this routine. +** +** References to the page pPg remain valid. Updating any +** meta-data associated with pPg (i.e. data stored in the nExtra bytes +** allocated along with the page) is the responsibility of the caller. +** +** A transaction must be active when this routine is called. It used to be +** required that a statement transaction was not active, but this restriction +** has been removed (CREATE INDEX needs to move a page when a statement +** transaction is active). +** +** If the fourth argument, isCommit, is non-zero, then this page is being +** moved as part of a database reorganization just before the transaction +** is being committed. In this case, it is guaranteed that the database page +** pPg refers to will not be written to again within this transaction. +** +** This function may return SQLITE_NOMEM or an IO error code if an error +** occurs. Otherwise, it returns SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ + PgHdr *pPgOld; /* The page being overwritten. */ + Pgno needSyncPgno = 0; /* Old value of pPg->pgno, if sync is required */ + int rc; /* Return code */ + Pgno origPgno; /* The original page number */ + + assert( pPg->nRef>0 ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + + /* In order to be able to rollback, an in-memory database must journal + ** the page we are moving from. + */ + if( MEMDB ){ + rc = sqlite3PagerWrite(pPg); + if( rc ) return rc; + } + + /* If the page being moved is dirty and has not been saved by the latest + ** savepoint, then save the current contents of the page into the + ** sub-journal now. This is required to handle the following scenario: + ** + ** BEGIN; + ** + ** SAVEPOINT one; + ** + ** ROLLBACK TO one; + ** + ** If page X were not written to the sub-journal here, it would not + ** be possible to restore its contents when the "ROLLBACK TO one" + ** statement were is processed. + ** + ** subjournalPage() may need to allocate space to store pPg->pgno into + ** one or more savepoint bitvecs. This is the reason this function + ** may return SQLITE_NOMEM. + */ + if( pPg->flags&PGHDR_DIRTY + && subjRequiresPage(pPg) + && SQLITE_OK!=(rc = subjournalPage(pPg)) + ){ + return rc; + } + + PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", + PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno)); + IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno)) + + /* If the journal needs to be sync()ed before page pPg->pgno can + ** be written to, store pPg->pgno in local variable needSyncPgno. + ** + ** If the isCommit flag is set, there is no need to remember that + ** the journal needs to be sync()ed before database page pPg->pgno + ** can be written to. The caller has already promised not to write to it. + */ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ + needSyncPgno = pPg->pgno; + assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || + pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); + assert( pPg->flags&PGHDR_DIRTY ); + } + + /* If the cache contains a page with page-number pgno, remove it + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for + ** page pgno before the 'move' operation, it needs to be retained + ** for the page moved there. + */ + pPg->flags &= ~PGHDR_NEED_SYNC; + pPgOld = pager_lookup(pPager, pgno); + assert( !pPgOld || pPgOld->nRef==1 ); + if( pPgOld ){ + pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); + if( MEMDB ){ + /* Do not discard pages from an in-memory database since we might + ** need to rollback later. Just move the page out of the way. */ + sqlite3PcacheMove(pPgOld, pPager->dbSize+1); + }else{ + sqlite3PcacheDrop(pPgOld); + } + } + + origPgno = pPg->pgno; + sqlite3PcacheMove(pPg, pgno); + sqlite3PcacheMakeDirty(pPg); + + /* For an in-memory database, make sure the original page continues + ** to exist, in case the transaction needs to roll back. Use pPgOld + ** as the original page since it has already been allocated. + */ + if( MEMDB ){ + assert( pPgOld ); + sqlite3PcacheMove(pPgOld, origPgno); + sqlite3PagerUnrefNotNull(pPgOld); + } + + if( needSyncPgno ){ + /* If needSyncPgno is non-zero, then the journal file needs to be + ** sync()ed before any data is written to database file page needSyncPgno. + ** Currently, no such page exists in the page-cache and the + ** "is journaled" bitvec flag has been set. This needs to be remedied by + ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC + ** flag. + ** + ** If the attempt to load the page into the page-cache fails, (due + ** to a malloc() or IO failure), clear the bit in the pInJournal[] + ** array. Otherwise, if the page is loaded and written again in + ** this transaction, it may be written to the database file before + ** it is synced into the journal file. This way, it may end up in + ** the journal file twice, but that is not a problem. + */ + PgHdr *pPgHdr; + rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); + if( rc!=SQLITE_OK ){ + if( needSyncPgno<=pPager->dbOrigSize ){ + assert( pPager->pTmpSpace!=0 ); + sqlite3BitvecClear(pPager->pInJournal, needSyncPgno, pPager->pTmpSpace); + } + return rc; + } + pPgHdr->flags |= PGHDR_NEED_SYNC; + sqlite3PcacheMakeDirty(pPgHdr); + sqlite3PagerUnrefNotNull(pPgHdr); + } + + return SQLITE_OK; +} +#endif + +/* +** Return a pointer to the data for the specified page. +*/ +SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *pPg){ + assert( pPg->nRef>0 || pPg->pPager->memDb ); + return pPg->pData; +} + +/* +** Return a pointer to the Pager.nExtra bytes of "extra" space +** allocated along with the specified page. +*/ +SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *pPg){ + return pPg->pExtra; +} + +/* +** Get/set the locking-mode for this pager. Parameter eMode must be one +** of PAGER_LOCKINGMODE_QUERY, PAGER_LOCKINGMODE_NORMAL or +** PAGER_LOCKINGMODE_EXCLUSIVE. If the parameter is not _QUERY, then +** the locking-mode is set to the value specified. +** +** The returned value is either PAGER_LOCKINGMODE_NORMAL or +** PAGER_LOCKINGMODE_EXCLUSIVE, indicating the current (possibly updated) +** locking-mode. +*/ +SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *pPager, int eMode){ + assert( eMode==PAGER_LOCKINGMODE_QUERY + || eMode==PAGER_LOCKINGMODE_NORMAL + || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); + assert( PAGER_LOCKINGMODE_QUERY<0 ); + assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 ); + assert( pPager->exclusiveMode || 0==sqlite3WalHeapMemory(pPager->pWal) ); + if( eMode>=0 && !pPager->tempFile && !sqlite3WalHeapMemory(pPager->pWal) ){ + pPager->exclusiveMode = (u8)eMode; + } + return (int)pPager->exclusiveMode; +} + +/* +** Set the journal-mode for this pager. Parameter eMode must be one of: +** +** PAGER_JOURNALMODE_DELETE +** PAGER_JOURNALMODE_TRUNCATE +** PAGER_JOURNALMODE_PERSIST +** PAGER_JOURNALMODE_OFF +** PAGER_JOURNALMODE_MEMORY +** PAGER_JOURNALMODE_WAL +** +** The journalmode is set to the value specified if the change is allowed. +** The change may be disallowed for the following reasons: +** +** * An in-memory database can only have its journal_mode set to _OFF +** or _MEMORY. +** +** * Temporary databases cannot have _WAL journalmode. +** +** The returned indicate the current (possibly updated) journal-mode. +*/ +SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ + u8 eOld = pPager->journalMode; /* Prior journalmode */ + +#ifdef SQLITE_DEBUG + /* The print_pager_state() routine is intended to be used by the debugger + ** only. We invoke it once here to suppress a compiler warning. */ + print_pager_state(pPager); +#endif + + + /* The eMode parameter is always valid */ + assert( eMode==PAGER_JOURNALMODE_DELETE + || eMode==PAGER_JOURNALMODE_TRUNCATE + || eMode==PAGER_JOURNALMODE_PERSIST + || eMode==PAGER_JOURNALMODE_OFF + || eMode==PAGER_JOURNALMODE_WAL + || eMode==PAGER_JOURNALMODE_MEMORY ); + + /* This routine is only called from the OP_JournalMode opcode, and + ** the logic there will never allow a temporary file to be changed + ** to WAL mode. + */ + assert( pPager->tempFile==0 || eMode!=PAGER_JOURNALMODE_WAL ); + + /* Do allow the journalmode of an in-memory database to be set to + ** anything other than MEMORY or OFF + */ + if( MEMDB ){ + assert( eOld==PAGER_JOURNALMODE_MEMORY || eOld==PAGER_JOURNALMODE_OFF ); + if( eMode!=PAGER_JOURNALMODE_MEMORY && eMode!=PAGER_JOURNALMODE_OFF ){ + eMode = eOld; + } + } + + if( eMode!=eOld ){ + + /* Change the journal mode. */ + assert( pPager->eState!=PAGER_ERROR ); + pPager->journalMode = (u8)eMode; + + /* When transistioning from TRUNCATE or PERSIST to any other journal + ** mode except WAL, unless the pager is in locking_mode=exclusive mode, + ** delete the journal file. + */ + assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); + assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); + assert( (PAGER_JOURNALMODE_DELETE & 5)==0 ); + assert( (PAGER_JOURNALMODE_MEMORY & 5)==4 ); + assert( (PAGER_JOURNALMODE_OFF & 5)==0 ); + assert( (PAGER_JOURNALMODE_WAL & 5)==5 ); + + assert( isOpen(pPager->fd) || pPager->exclusiveMode ); + if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ + + /* In this case we would like to delete the journal file. If it is + ** not possible, then that is not a problem. Deleting the journal file + ** here is an optimization only. + ** + ** Before deleting the journal file, obtain a RESERVED lock on the + ** database file. This ensures that the journal file is not deleted + ** while it is in use by some other client. + */ + sqlite3OsClose(pPager->jfd); + if( pPager->eLock>=RESERVED_LOCK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + }else{ + int rc = SQLITE_OK; + int state = pPager->eState; + assert( state==PAGER_OPEN || state==PAGER_READER ); + if( state==PAGER_OPEN ){ + rc = sqlite3PagerSharedLock(pPager); + } + if( pPager->eState==PAGER_READER ){ + assert( rc==SQLITE_OK ); + rc = pagerLockDb(pPager, RESERVED_LOCK); + } + if( rc==SQLITE_OK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + if( rc==SQLITE_OK && state==PAGER_READER ){ + pagerUnlockDb(pPager, SHARED_LOCK); + }else if( state==PAGER_OPEN ){ + pager_unlock(pPager); + } + assert( state==pPager->eState ); + } + } + } + + /* Return the new journal mode */ + return (int)pPager->journalMode; +} + +/* +** Return the current journal mode. +*/ +SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager *pPager){ + return (int)pPager->journalMode; +} + +/* +** Return TRUE if the pager is in a state where it is OK to change the +** journalmode. Journalmode changes can only happen when the database +** is unmodified. +*/ +SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; + if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; + return 1; +} + +/* +** Get/set the size-limit used for persistent journal files. +** +** Setting the size limit to -1 means no limit is enforced. +** An attempt to set a limit smaller than -1 is a no-op. +*/ +SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ + if( iLimit>=-1 ){ + pPager->journalSizeLimit = iLimit; + sqlite3WalLimit(pPager->pWal, iLimit); + } + return pPager->journalSizeLimit; +} + +/* +** Return a pointer to the pPager->pBackup variable. The backup module +** in backup.c maintains the content of this variable. This module +** uses it opaquely as an argument to sqlite3BackupRestart() and +** sqlite3BackupUpdate() only. +*/ +SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ + return &pPager->pBackup; +} + +#ifndef SQLITE_OMIT_VACUUM +/* +** Unless this is an in-memory or temporary database, clear the pager cache. +*/ +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *pPager){ + if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); +} +#endif + +#ifndef SQLITE_OMIT_WAL +/* +** This function is called when the user invokes "PRAGMA wal_checkpoint", +** "PRAGMA wal_blocking_checkpoint" or calls the sqlite3_wal_checkpoint() +** or wal_blocking_checkpoint() API functions. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; + if( pPager->pWal ){ + rc = sqlite3WalCheckpoint(pPager->pWal, eMode, + pPager->xBusyHandler, pPager->pBusyHandlerArg, + pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, + pnLog, pnCkpt + ); + } + return rc; +} + +SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager){ + return sqlite3WalCallback(pPager->pWal); +} + +/* +** Return true if the underlying VFS for the given pager supports the +** primitives necessary for write-ahead logging. +*/ +SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ + const sqlite3_io_methods *pMethods = pPager->fd->pMethods; + return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); +} + +/* +** Attempt to take an exclusive lock on the database file. If a PENDING lock +** is obtained instead, immediately release it. +*/ +static int pagerExclusiveLock(Pager *pPager){ + int rc; /* Return code */ + + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + /* If the attempt to grab the exclusive lock failed, release the + ** pending lock that may have been obtained instead. */ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + return rc; +} + +/* +** Call sqlite3WalOpen() to open the WAL handle. If the pager is in +** exclusive-locking mode when this function is called, take an EXCLUSIVE +** lock on the database file and use heap-memory to store the wal-index +** in. Otherwise, use the normal shared-memory. +*/ +static int pagerOpenWal(Pager *pPager){ + int rc = SQLITE_OK; + + assert( pPager->pWal==0 && pPager->tempFile==0 ); + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + + /* If the pager is already in exclusive-mode, the WAL module will use + ** heap-memory for the wal-index instead of the VFS shared-memory + ** implementation. Take the exclusive lock now, before opening the WAL + ** file, to make sure this is safe. + */ + if( pPager->exclusiveMode ){ + rc = pagerExclusiveLock(pPager); + } + + /* Open the connection to the log file. If this operation fails, + ** (e.g. due to malloc() failure), return an error code. + */ + if( rc==SQLITE_OK ){ + rc = sqlite3WalOpen(pPager->pVfs, + pPager->fd, pPager->zWal, pPager->exclusiveMode, + pPager->journalSizeLimit, &pPager->pWal + ); + } + pagerFixMaplimit(pPager); + + return rc; +} + + +/* +** The caller must be holding a SHARED lock on the database file to call +** this function. +** +** If the pager passed as the first argument is open on a real database +** file (not a temp file or an in-memory database), and the WAL file +** is not already open, make an attempt to open it now. If successful, +** return SQLITE_OK. If an error occurs or the VFS used by the pager does +** not support the xShmXXX() methods, return an error code. *pbOpen is +** not modified in either case. +** +** If the pager is open on a temp-file (or in-memory database), or if +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK +** without doing anything. +*/ +SQLITE_PRIVATE int sqlite3PagerOpenWal( + Pager *pPager, /* Pager object */ + int *pbOpen /* OUT: Set to true if call is a no-op */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pbOpen ); + assert( pPager->eState==PAGER_READER || !pbOpen ); + assert( pbOpen==0 || *pbOpen==0 ); + assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); + + if( !pPager->tempFile && !pPager->pWal ){ + if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; + + /* Close any rollback journal previously open */ + sqlite3OsClose(pPager->jfd); + + rc = pagerOpenWal(pPager); + if( rc==SQLITE_OK ){ + pPager->journalMode = PAGER_JOURNALMODE_WAL; + pPager->eState = PAGER_OPEN; + } + }else{ + *pbOpen = 1; + } + + return rc; +} + +/* +** This function is called to close the connection to the log file prior +** to switching from WAL to rollback mode. +** +** Before closing the log file, this function attempts to take an +** EXCLUSIVE lock on the database file. If this cannot be obtained, an +** error (SQLITE_BUSY) is returned and the log connection is not closed. +** If successful, the EXCLUSIVE lock is not released before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ + int rc = SQLITE_OK; + + assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); + + /* If the log file is not already open, but does exist in the file-system, + ** it may need to be checkpointed before the connection can switch to + ** rollback mode. Open it now so this can happen. + */ + if( !pPager->pWal ){ + int logexists = 0; + rc = pagerLockDb(pPager, SHARED_LOCK); + if( rc==SQLITE_OK ){ + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists + ); + } + if( rc==SQLITE_OK && logexists ){ + rc = pagerOpenWal(pPager); + } + } + + /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on + ** the database file, the log and log-summary files will be deleted. + */ + if( rc==SQLITE_OK && pPager->pWal ){ + rc = pagerExclusiveLock(pPager); + if( rc==SQLITE_OK ){ + rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, + pPager->pageSize, (u8*)pPager->pTmpSpace); + pPager->pWal = 0; + pagerFixMaplimit(pPager); + } + } + return rc; +} + +#endif /* !SQLITE_OMIT_WAL */ + +#ifdef SQLITE_ENABLE_ZIPVFS +/* +** A read-lock must be held on the pager when this function is called. If +** the pager is in WAL mode and the WAL file currently contains one or more +** frames, return the size in bytes of the page images stored within the +** WAL frames. Otherwise, if this is not a WAL database or the WAL file +** is empty, return 0. +*/ +SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ + assert( pPager->eState==PAGER_READER ); + return sqlite3WalFramesize(pPager->pWal); +} +#endif + +#endif /* SQLITE_OMIT_DISKIO */ + +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +SQLITE_PRIVATE void sqlite3pager_get_codec(Pager *pPager, void **ctx) { + *ctx = pPager->pCodec; +} + +SQLITE_PRIVATE int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno) { + return (PAGER_MJ_PGNO(pPager) == pgno) ? 1 : 0; +} + +SQLITE_PRIVATE sqlite3_file *sqlite3Pager_get_fd(Pager *pPager) { + return (isOpen(pPager->fd)) ? pPager->fd : NULL; +} + +SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetCodec( + Pager *pPager, + void *(*xCodec)(void*,void*,Pgno,int), + void (*xCodecSizeChng)(void*,int,int), + void (*xCodecFree)(void*), + void *pCodec +){ + sqlite3PagerSetCodec(pPager, xCodec, xCodecSizeChng, xCodecFree, pCodec); +} + +SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) { + pPager->errCode = error; +} + +#endif +/* END SQLCIPHER */ + + +/************** End of pager.c ***********************************************/ +/************** Begin file wal.c *********************************************/ +/* +** 2010 February 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of a write-ahead log (WAL) used in +** "journal_mode=WAL" mode. +** +** WRITE-AHEAD LOG (WAL) FILE FORMAT +** +** A WAL file consists of a header followed by zero or more "frames". +** Each frame records the revised content of a single page from the +** database file. All changes to the database are recorded by writing +** frames into the WAL. Transactions commit when a frame is written that +** contains a commit marker. A single WAL can and usually does record +** multiple transactions. Periodically, the content of the WAL is +** transferred back into the database file in an operation called a +** "checkpoint". +** +** A single WAL file can be used multiple times. In other words, the +** WAL can fill up with frames and then be checkpointed and then new +** frames can overwrite the old ones. A WAL always grows from beginning +** toward the end. Checksums and counters attached to each frame are +** used to determine which frames within the WAL are valid and which +** are leftovers from prior checkpoints. +** +** The WAL header is 32 bytes in size and consists of the following eight +** big-endian 32-bit unsigned integer values: +** +** 0: Magic number. 0x377f0682 or 0x377f0683 +** 4: File format version. Currently 3007000 +** 8: Database page size. Example: 1024 +** 12: Checkpoint sequence number +** 16: Salt-1, random integer incremented with each checkpoint +** 20: Salt-2, a different random integer changing with each ckpt +** 24: Checksum-1 (first part of checksum for first 24 bytes of header). +** 28: Checksum-2 (second part of checksum for first 24 bytes of header). +** +** Immediately following the wal-header are zero or more frames. Each +** frame consists of a 24-byte frame-header followed by a bytes +** of page data. The frame-header is six big-endian 32-bit unsigned +** integer values, as follows: +** +** 0: Page number. +** 4: For commit records, the size of the database image in pages +** after the commit. For all other records, zero. +** 8: Salt-1 (copied from the header) +** 12: Salt-2 (copied from the header) +** 16: Checksum-1. +** 20: Checksum-2. +** +** A frame is considered valid if and only if the following conditions are +** true: +** +** (1) The salt-1 and salt-2 values in the frame-header match +** salt values in the wal-header +** +** (2) The checksum values in the final 8 bytes of the frame-header +** exactly match the checksum computed consecutively on the +** WAL header and the first 8 bytes and the content of all frames +** up to and including the current frame. +** +** The checksum is computed using 32-bit big-endian integers if the +** magic number in the first 4 bytes of the WAL is 0x377f0683 and it +** is computed using little-endian if the magic number is 0x377f0682. +** The checksum values are always stored in the frame header in a +** big-endian format regardless of which byte order is used to compute +** the checksum. The checksum is computed by interpreting the input as +** an even number of unsigned 32-bit integers: x[0] through x[N]. The +** algorithm used for the checksum is as follows: +** +** for i from 0 to n-1 step 2: +** s0 += x[i] + s1; +** s1 += x[i+1] + s0; +** endfor +** +** Note that s0 and s1 are both weighted checksums using fibonacci weights +** in reverse order (the largest fibonacci weight occurs on the first element +** of the sequence being summed.) The s1 value spans all 32-bit +** terms of the sequence whereas s0 omits the final term. +** +** On a checkpoint, the WAL is first VFS.xSync-ed, then valid content of the +** WAL is transferred into the database, then the database is VFS.xSync-ed. +** The VFS.xSync operations serve as write barriers - all writes launched +** before the xSync must complete before any write that launches after the +** xSync begins. +** +** After each checkpoint, the salt-1 value is incremented and the salt-2 +** value is randomized. This prevents old and new frames in the WAL from +** being considered valid at the same time and being checkpointing together +** following a crash. +** +** READER ALGORITHM +** +** To read a page from the database (call it page number P), a reader +** first checks the WAL to see if it contains page P. If so, then the +** last valid instance of page P that is a followed by a commit frame +** or is a commit frame itself becomes the value read. If the WAL +** contains no copies of page P that are valid and which are a commit +** frame or are followed by a commit frame, then page P is read from +** the database file. +** +** To start a read transaction, the reader records the index of the last +** valid frame in the WAL. The reader uses this recorded "mxFrame" value +** for all subsequent read operations. New transactions can be appended +** to the WAL, but as long as the reader uses its original mxFrame value +** and ignores the newly appended content, it will see a consistent snapshot +** of the database from a single point in time. This technique allows +** multiple concurrent readers to view different versions of the database +** content simultaneously. +** +** The reader algorithm in the previous paragraphs works correctly, but +** because frames for page P can appear anywhere within the WAL, the +** reader has to scan the entire WAL looking for page P frames. If the +** WAL is large (multiple megabytes is typical) that scan can be slow, +** and read performance suffers. To overcome this problem, a separate +** data structure called the wal-index is maintained to expedite the +** search for frames of a particular page. +** +** WAL-INDEX FORMAT +** +** Conceptually, the wal-index is shared memory, though VFS implementations +** might choose to implement the wal-index using a mmapped file. Because +** the wal-index is shared memory, SQLite does not support journal_mode=WAL +** on a network filesystem. All users of the database must be able to +** share memory. +** +** The wal-index is transient. After a crash, the wal-index can (and should +** be) reconstructed from the original WAL file. In fact, the VFS is required +** to either truncate or zero the header of the wal-index when the last +** connection to it closes. Because the wal-index is transient, it can +** use an architecture-specific format; it does not have to be cross-platform. +** Hence, unlike the database and WAL file formats which store all values +** as big endian, the wal-index can store multi-byte values in the native +** byte order of the host computer. +** +** The purpose of the wal-index is to answer this question quickly: Given +** a page number P and a maximum frame index M, return the index of the +** last frame in the wal before frame M for page P in the WAL, or return +** NULL if there are no frames for page P in the WAL prior to M. +** +** The wal-index consists of a header region, followed by an one or +** more index blocks. +** +** The wal-index header contains the total number of frames within the WAL +** in the mxFrame field. +** +** Each index block except for the first contains information on +** HASHTABLE_NPAGE frames. The first index block contains information on +** HASHTABLE_NPAGE_ONE frames. The values of HASHTABLE_NPAGE_ONE and +** HASHTABLE_NPAGE are selected so that together the wal-index header and +** first index block are the same size as all other index blocks in the +** wal-index. +** +** Each index block contains two sections, a page-mapping that contains the +** database page number associated with each wal frame, and a hash-table +** that allows readers to query an index block for a specific page number. +** The page-mapping is an array of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE +** for the first index block) 32-bit page numbers. The first entry in the +** first index-block contains the database page number corresponding to the +** first frame in the WAL file. The first entry in the second index block +** in the WAL file corresponds to the (HASHTABLE_NPAGE_ONE+1)th frame in +** the log, and so on. +** +** The last index block in a wal-index usually contains less than the full +** complement of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE) page-numbers, +** depending on the contents of the WAL file. This does not change the +** allocated size of the page-mapping array - the page-mapping array merely +** contains unused entries. +** +** Even without using the hash table, the last frame for page P +** can be found by scanning the page-mapping sections of each index block +** starting with the last index block and moving toward the first, and +** within each index block, starting at the end and moving toward the +** beginning. The first entry that equals P corresponds to the frame +** holding the content for that page. +** +** The hash table consists of HASHTABLE_NSLOT 16-bit unsigned integers. +** HASHTABLE_NSLOT = 2*HASHTABLE_NPAGE, and there is one entry in the +** hash table for each page number in the mapping section, so the hash +** table is never more than half full. The expected number of collisions +** prior to finding a match is 1. Each entry of the hash table is an +** 1-based index of an entry in the mapping section of the same +** index block. Let K be the 1-based index of the largest entry in +** the mapping section. (For index blocks other than the last, K will +** always be exactly HASHTABLE_NPAGE (4096) and for the last index block +** K will be (mxFrame%HASHTABLE_NPAGE).) Unused slots of the hash table +** contain a value of 0. +** +** To look for page P in the hash table, first compute a hash iKey on +** P as follows: +** +** iKey = (P * 383) % HASHTABLE_NSLOT +** +** Then start scanning entries of the hash table, starting with iKey +** (wrapping around to the beginning when the end of the hash table is +** reached) until an unused hash slot is found. Let the first unused slot +** be at index iUnused. (iUnused might be less than iKey if there was +** wrap-around.) Because the hash table is never more than half full, +** the search is guaranteed to eventually hit an unused entry. Let +** iMax be the value between iKey and iUnused, closest to iUnused, +** where aHash[iMax]==P. If there is no iMax entry (if there exists +** no hash slot such that aHash[i]==p) then page P is not in the +** current index block. Otherwise the iMax-th mapping entry of the +** current index block corresponds to the last entry that references +** page P. +** +** A hash search begins with the last index block and moves toward the +** first index block, looking for entries corresponding to page P. On +** average, only two or three slots in each index block need to be +** examined in order to either find the last entry for page P, or to +** establish that no such entry exists in the block. Each index block +** holds over 4000 entries. So two or three index blocks are sufficient +** to cover a typical 10 megabyte WAL file, assuming 1K pages. 8 or 10 +** comparisons (on average) suffice to either locate a frame in the +** WAL or to establish that the frame does not exist in the WAL. This +** is much faster than scanning the entire 10MB WAL. +** +** Note that entries are added in order of increasing K. Hence, one +** reader might be using some value K0 and a second reader that started +** at a later time (after additional transactions were added to the WAL +** and to the wal-index) might be using a different value K1, where K1>K0. +** Both readers can use the same hash table and mapping section to get +** the correct result. There may be entries in the hash table with +** K>K0 but to the first reader, those entries will appear to be unused +** slots in the hash table and so the first reader will get an answer as +** if no values greater than K0 had ever been inserted into the hash table +** in the first place - which is what reader one wants. Meanwhile, the +** second reader using K1 will see additional values that were inserted +** later, which is exactly what reader two wants. +** +** When a rollback occurs, the value of K is decreased. Hash table entries +** that correspond to frames greater than the new K value are removed +** from the hash table at this point. +*/ +#ifndef SQLITE_OMIT_WAL + + +/* +** Trace output macros +*/ +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3WalTrace = 0; +# define WALTRACE(X) if(sqlite3WalTrace) sqlite3DebugPrintf X +#else +# define WALTRACE(X) +#endif + +/* +** The maximum (and only) versions of the wal and wal-index formats +** that may be interpreted by this version of SQLite. +** +** If a client begins recovering a WAL file and finds that (a) the checksum +** values in the wal-header are correct and (b) the version field is not +** WAL_MAX_VERSION, recovery fails and SQLite returns SQLITE_CANTOPEN. +** +** Similarly, if a client successfully reads a wal-index header (i.e. the +** checksum test is successful) and finds that the version field is not +** WALINDEX_MAX_VERSION, then no read-transaction is opened and SQLite +** returns SQLITE_CANTOPEN. +*/ +#define WAL_MAX_VERSION 3007000 +#define WALINDEX_MAX_VERSION 3007000 + +/* +** Indices of various locking bytes. WAL_NREADER is the number +** of available reader locks and should be at least 3. +*/ +#define WAL_WRITE_LOCK 0 +#define WAL_ALL_BUT_WRITE 1 +#define WAL_CKPT_LOCK 1 +#define WAL_RECOVER_LOCK 2 +#define WAL_READ_LOCK(I) (3+(I)) +#define WAL_NREADER (SQLITE_SHM_NLOCK-3) + + +/* Object declarations */ +typedef struct WalIndexHdr WalIndexHdr; +typedef struct WalIterator WalIterator; +typedef struct WalCkptInfo WalCkptInfo; + + +/* +** The following object holds a copy of the wal-index header content. +** +** The actual header in the wal-index consists of two copies of this +** object. +** +** The szPage value can be any power of 2 between 512 and 32768, inclusive. +** Or it can be 1 to represent a 65536-byte page. The latter case was +** added in 3.7.1 when support for 64K pages was added. +*/ +struct WalIndexHdr { + u32 iVersion; /* Wal-index version */ + u32 unused; /* Unused (padding) field */ + u32 iChange; /* Counter incremented each transaction */ + u8 isInit; /* 1 when initialized */ + u8 bigEndCksum; /* True if checksums in WAL are big-endian */ + u16 szPage; /* Database page size in bytes. 1==64K */ + u32 mxFrame; /* Index of last valid frame in the WAL */ + u32 nPage; /* Size of database in pages */ + u32 aFrameCksum[2]; /* Checksum of last frame in log */ + u32 aSalt[2]; /* Two salt values copied from WAL header */ + u32 aCksum[2]; /* Checksum over all prior fields */ +}; + +/* +** A copy of the following object occurs in the wal-index immediately +** following the second copy of the WalIndexHdr. This object stores +** information used by checkpoint. +** +** nBackfill is the number of frames in the WAL that have been written +** back into the database. (We call the act of moving content from WAL to +** database "backfilling".) The nBackfill number is never greater than +** WalIndexHdr.mxFrame. nBackfill can only be increased by threads +** holding the WAL_CKPT_LOCK lock (which includes a recovery thread). +** However, a WAL_WRITE_LOCK thread can move the value of nBackfill from +** mxFrame back to zero when the WAL is reset. +** +** There is one entry in aReadMark[] for each reader lock. If a reader +** holds read-lock K, then the value in aReadMark[K] is no greater than +** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) +** for any aReadMark[] means that entry is unused. aReadMark[0] is +** a special case; its value is never used and it exists as a place-holder +** to avoid having to offset aReadMark[] indexs by one. Readers holding +** WAL_READ_LOCK(0) always ignore the entire WAL and read all content +** directly from the database. +** +** The value of aReadMark[K] may only be changed by a thread that +** is holding an exclusive lock on WAL_READ_LOCK(K). Thus, the value of +** aReadMark[K] cannot changed while there is a reader is using that mark +** since the reader will be holding a shared lock on WAL_READ_LOCK(K). +** +** The checkpointer may only transfer frames from WAL to database where +** the frame numbers are less than or equal to every aReadMark[] that is +** in use (that is, every aReadMark[j] for which there is a corresponding +** WAL_READ_LOCK(j)). New readers (usually) pick the aReadMark[] with the +** largest value and will increase an unused aReadMark[] to mxFrame if there +** is not already an aReadMark[] equal to mxFrame. The exception to the +** previous sentence is when nBackfill equals mxFrame (meaning that everything +** in the WAL has been backfilled into the database) then new readers +** will choose aReadMark[0] which has value 0 and hence such reader will +** get all their all content directly from the database file and ignore +** the WAL. +** +** Writers normally append new frames to the end of the WAL. However, +** if nBackfill equals mxFrame (meaning that all WAL content has been +** written back into the database) and if no readers are using the WAL +** (in other words, if there are no WAL_READ_LOCK(i) where i>0) then +** the writer will first "reset" the WAL back to the beginning and start +** writing new content beginning at frame 1. +** +** We assume that 32-bit loads are atomic and so no locks are needed in +** order to read from any aReadMark[] entries. +*/ +struct WalCkptInfo { + u32 nBackfill; /* Number of WAL frames backfilled into DB */ + u32 aReadMark[WAL_NREADER]; /* Reader marks */ +}; +#define READMARK_NOT_USED 0xffffffff + + +/* A block of WALINDEX_LOCK_RESERVED bytes beginning at +** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems +** only support mandatory file-locks, we do not read or write data +** from the region of the file on which locks are applied. +*/ +#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2 + sizeof(WalCkptInfo)) +#define WALINDEX_LOCK_RESERVED 16 +#define WALINDEX_HDR_SIZE (WALINDEX_LOCK_OFFSET+WALINDEX_LOCK_RESERVED) + +/* Size of header before each frame in wal */ +#define WAL_FRAME_HDRSIZE 24 + +/* Size of write ahead log header, including checksum. */ +/* #define WAL_HDRSIZE 24 */ +#define WAL_HDRSIZE 32 + +/* WAL magic value. Either this value, or the same value with the least +** significant bit also set (WAL_MAGIC | 0x00000001) is stored in 32-bit +** big-endian format in the first 4 bytes of a WAL file. +** +** If the LSB is set, then the checksums for each frame within the WAL +** file are calculated by treating all data as an array of 32-bit +** big-endian words. Otherwise, they are calculated by interpreting +** all data as 32-bit little-endian words. +*/ +#define WAL_MAGIC 0x377f0682 + +/* +** Return the offset of frame iFrame in the write-ahead log file, +** assuming a database page size of szPage bytes. The offset returned +** is to the start of the write-ahead log frame-header. +*/ +#define walFrameOffset(iFrame, szPage) ( \ + WAL_HDRSIZE + ((iFrame)-1)*(i64)((szPage)+WAL_FRAME_HDRSIZE) \ +) + +/* +** An open write-ahead log file is represented by an instance of the +** following object. +*/ +struct Wal { + sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ + sqlite3_file *pDbFd; /* File handle for the database file */ + sqlite3_file *pWalFd; /* File handle for WAL file */ + u32 iCallback; /* Value to pass to log callback (or 0) */ + i64 mxWalSize; /* Truncate WAL to this size upon reset */ + int nWiData; /* Size of array apWiData */ + int szFirstBlock; /* Size of first block written to WAL file */ + volatile u32 **apWiData; /* Pointer to wal-index content in memory */ + u32 szPage; /* Database page size */ + i16 readLock; /* Which read lock is being held. -1 for none */ + u8 syncFlags; /* Flags to use to sync header writes */ + u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ + u8 writeLock; /* True if in a write transaction */ + u8 ckptLock; /* True if holding a checkpoint lock */ + u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ + u8 truncateOnCommit; /* True to truncate WAL file on commit */ + u8 syncHeader; /* Fsync the WAL header if true */ + u8 padToSectorBoundary; /* Pad transactions out to the next sector */ + WalIndexHdr hdr; /* Wal-index header for current transaction */ + const char *zWalName; /* Name of WAL file */ + u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ +#ifdef SQLITE_DEBUG + u8 lockError; /* True if a locking error has occurred */ +#endif +}; + +/* +** Candidate values for Wal.exclusiveMode. +*/ +#define WAL_NORMAL_MODE 0 +#define WAL_EXCLUSIVE_MODE 1 +#define WAL_HEAPMEMORY_MODE 2 + +/* +** Possible values for WAL.readOnly +*/ +#define WAL_RDWR 0 /* Normal read/write connection */ +#define WAL_RDONLY 1 /* The WAL file is readonly */ +#define WAL_SHM_RDONLY 2 /* The SHM file is readonly */ + +/* +** Each page of the wal-index mapping contains a hash-table made up of +** an array of HASHTABLE_NSLOT elements of the following type. +*/ +typedef u16 ht_slot; + +/* +** This structure is used to implement an iterator that loops through +** all frames in the WAL in database page order. Where two or more frames +** correspond to the same database page, the iterator visits only the +** frame most recently written to the WAL (in other words, the frame with +** the largest index). +** +** The internals of this structure are only accessed by: +** +** walIteratorInit() - Create a new iterator, +** walIteratorNext() - Step an iterator, +** walIteratorFree() - Free an iterator. +** +** This functionality is used by the checkpoint code (see walCheckpoint()). +*/ +struct WalIterator { + int iPrior; /* Last result returned from the iterator */ + int nSegment; /* Number of entries in aSegment[] */ + struct WalSegment { + int iNext; /* Next slot in aIndex[] not yet returned */ + ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ + u32 *aPgno; /* Array of page numbers. */ + int nEntry; /* Nr. of entries in aPgno[] and aIndex[] */ + int iZero; /* Frame number associated with aPgno[0] */ + } aSegment[1]; /* One for every 32KB page in the wal-index */ +}; + +/* +** Define the parameters of the hash tables in the wal-index file. There +** is a hash-table following every HASHTABLE_NPAGE page numbers in the +** wal-index. +** +** Changing any of these constants will alter the wal-index format and +** create incompatibilities. +*/ +#define HASHTABLE_NPAGE 4096 /* Must be power of 2 */ +#define HASHTABLE_HASH_1 383 /* Should be prime */ +#define HASHTABLE_NSLOT (HASHTABLE_NPAGE*2) /* Must be a power of 2 */ + +/* +** The block of page numbers associated with the first hash-table in a +** wal-index is smaller than usual. This is so that there is a complete +** hash-table on each aligned 32KB page of the wal-index. +*/ +#define HASHTABLE_NPAGE_ONE (HASHTABLE_NPAGE - (WALINDEX_HDR_SIZE/sizeof(u32))) + +/* The wal-index is divided into pages of WALINDEX_PGSZ bytes each. */ +#define WALINDEX_PGSZ ( \ + sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \ +) + +/* +** Obtain a pointer to the iPage'th page of the wal-index. The wal-index +** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are +** numbered from zero. +** +** If this call is successful, *ppPage is set to point to the wal-index +** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, +** then an SQLite error code is returned and *ppPage is set to 0. +*/ +static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ + int rc = SQLITE_OK; + + /* Enlarge the pWal->apWiData[] array if required */ + if( pWal->nWiData<=iPage ){ + int nByte = sizeof(u32*)*(iPage+1); + volatile u32 **apNew; + apNew = (volatile u32 **)sqlite3_realloc((void *)pWal->apWiData, nByte); + if( !apNew ){ + *ppPage = 0; + return SQLITE_NOMEM; + } + memset((void*)&apNew[pWal->nWiData], 0, + sizeof(u32*)*(iPage+1-pWal->nWiData)); + pWal->apWiData = apNew; + pWal->nWiData = iPage+1; + } + + /* Request a pointer to the required page from the VFS */ + if( pWal->apWiData[iPage]==0 ){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM; + }else{ + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + if( rc==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; + rc = SQLITE_OK; + } + } + } + + *ppPage = pWal->apWiData[iPage]; + assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); + return rc; +} + +/* +** Return a pointer to the WalCkptInfo structure in the wal-index. +*/ +static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + return (volatile WalCkptInfo*)&(pWal->apWiData[0][sizeof(WalIndexHdr)/2]); +} + +/* +** Return a pointer to the WalIndexHdr structure in the wal-index. +*/ +static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + return (volatile WalIndexHdr*)pWal->apWiData[0]; +} + +/* +** The argument to this macro must be of type u32. On a little-endian +** architecture, it returns the u32 value that results from interpreting +** the 4 bytes as a big-endian value. On a big-endian architecture, it +** returns the value that would be produced by intepreting the 4 bytes +** of the input value as a little-endian integer. +*/ +#define BYTESWAP32(x) ( \ + (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ +) + +/* +** Generate or extend an 8 byte checksum based on the data in +** array aByte[] and the initial values of aIn[0] and aIn[1] (or +** initial values of 0 and 0 if aIn==NULL). +** +** The checksum is written back into aOut[] before returning. +** +** nByte must be a positive multiple of 8. +*/ +static void walChecksumBytes( + int nativeCksum, /* True for native byte-order, false for non-native */ + u8 *a, /* Content to be checksummed */ + int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ + const u32 *aIn, /* Initial checksum value input */ + u32 *aOut /* OUT: Final checksum value output */ +){ + u32 s1, s2; + u32 *aData = (u32 *)a; + u32 *aEnd = (u32 *)&a[nByte]; + + if( aIn ){ + s1 = aIn[0]; + s2 = aIn[1]; + }else{ + s1 = s2 = 0; + } + + assert( nByte>=8 ); + assert( (nByte&0x00000007)==0 ); + + if( nativeCksum ){ + do { + s1 += *aData++ + s2; + s2 += *aData++ + s1; + }while( aDataexclusiveMode!=WAL_HEAPMEMORY_MODE ){ + sqlite3OsShmBarrier(pWal->pDbFd); + } +} + +/* +** Write the header information in pWal->hdr into the wal-index. +** +** The checksum on pWal->hdr is updated before it is written. +*/ +static void walIndexWriteHdr(Wal *pWal){ + volatile WalIndexHdr *aHdr = walIndexHdr(pWal); + const int nCksum = offsetof(WalIndexHdr, aCksum); + + assert( pWal->writeLock ); + pWal->hdr.isInit = 1; + pWal->hdr.iVersion = WALINDEX_MAX_VERSION; + walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); + memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); + walShmBarrier(pWal); + memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); +} + +/* +** This function encodes a single frame header and writes it to a buffer +** supplied by the caller. A frame-header is made up of a series of +** 4-byte big-endian integers, as follows: +** +** 0: Page number. +** 4: For commit records, the size of the database image in pages +** after the commit. For all other records, zero. +** 8: Salt-1 (copied from the wal-header) +** 12: Salt-2 (copied from the wal-header) +** 16: Checksum-1. +** 20: Checksum-2. +*/ +static void walEncodeFrame( + Wal *pWal, /* The write-ahead log */ + u32 iPage, /* Database page number for frame */ + u32 nTruncate, /* New db size (or 0 for non-commit frames) */ + u8 *aData, /* Pointer to page data */ + u8 *aFrame /* OUT: Write encoded frame here */ +){ + int nativeCksum; /* True for native byte-order checksums */ + u32 *aCksum = pWal->hdr.aFrameCksum; + assert( WAL_FRAME_HDRSIZE==24 ); + sqlite3Put4byte(&aFrame[0], iPage); + sqlite3Put4byte(&aFrame[4], nTruncate); + memcpy(&aFrame[8], pWal->hdr.aSalt, 8); + + nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); + walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); + + sqlite3Put4byte(&aFrame[16], aCksum[0]); + sqlite3Put4byte(&aFrame[20], aCksum[1]); +} + +/* +** Check to see if the frame with header in aFrame[] and content +** in aData[] is valid. If it is a valid frame, fill *piPage and +** *pnTruncate and return true. Return if the frame is not valid. +*/ +static int walDecodeFrame( + Wal *pWal, /* The write-ahead log */ + u32 *piPage, /* OUT: Database page number for frame */ + u32 *pnTruncate, /* OUT: New db size (or 0 if not commit) */ + u8 *aData, /* Pointer to page data (for checksum) */ + u8 *aFrame /* Frame data */ +){ + int nativeCksum; /* True for native byte-order checksums */ + u32 *aCksum = pWal->hdr.aFrameCksum; + u32 pgno; /* Page number of the frame */ + assert( WAL_FRAME_HDRSIZE==24 ); + + /* A frame is only valid if the salt values in the frame-header + ** match the salt values in the wal-header. + */ + if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){ + return 0; + } + + /* A frame is only valid if the page number is creater than zero. + */ + pgno = sqlite3Get4byte(&aFrame[0]); + if( pgno==0 ){ + return 0; + } + + /* A frame is only valid if a checksum of the WAL header, + ** all prior frams, the first 16 bytes of this frame-header, + ** and the frame-data matches the checksum in the last 8 + ** bytes of this frame-header. + */ + nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); + walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); + if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) + || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) + ){ + /* Checksum failed. */ + return 0; + } + + /* If we reach this point, the frame is valid. Return the page number + ** and the new database size. + */ + *piPage = pgno; + *pnTruncate = sqlite3Get4byte(&aFrame[4]); + return 1; +} + + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +/* +** Names of locks. This routine is used to provide debugging output and is not +** a part of an ordinary build. +*/ +static const char *walLockName(int lockIdx){ + if( lockIdx==WAL_WRITE_LOCK ){ + return "WRITE-LOCK"; + }else if( lockIdx==WAL_CKPT_LOCK ){ + return "CKPT-LOCK"; + }else if( lockIdx==WAL_RECOVER_LOCK ){ + return "RECOVER-LOCK"; + }else{ + static char zName[15]; + sqlite3_snprintf(sizeof(zName), zName, "READ-LOCK[%d]", + lockIdx-WAL_READ_LOCK(0)); + return zName; + } +} +#endif /*defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */ + + +/* +** Set or release locks on the WAL. Locks are either shared or exclusive. +** A lock cannot be moved directly between shared and exclusive - it must go +** through the unlocked state first. +** +** In locking_mode=EXCLUSIVE, all of these routines become no-ops. +*/ +static int walLockShared(Wal *pWal, int lockIdx){ + int rc; + if( pWal->exclusiveMode ) return SQLITE_OK; + rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, + SQLITE_SHM_LOCK | SQLITE_SHM_SHARED); + WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal, + walLockName(lockIdx), rc ? "failed" : "ok")); + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) + return rc; +} +static void walUnlockShared(Wal *pWal, int lockIdx){ + if( pWal->exclusiveMode ) return; + (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, + SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); + WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); +} +static int walLockExclusive(Wal *pWal, int lockIdx, int n){ + int rc; + if( pWal->exclusiveMode ) return SQLITE_OK; + rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, + SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); + WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, + walLockName(lockIdx), n, rc ? "failed" : "ok")); + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) + return rc; +} +static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ + if( pWal->exclusiveMode ) return; + (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, + SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); + WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal, + walLockName(lockIdx), n)); +} + +/* +** Compute a hash on a page number. The resulting hash value must land +** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances +** the hash to the next value in the event of a collision. +*/ +static int walHash(u32 iPage){ + assert( iPage>0 ); + assert( (HASHTABLE_NSLOT & (HASHTABLE_NSLOT-1))==0 ); + return (iPage*HASHTABLE_HASH_1) & (HASHTABLE_NSLOT-1); +} +static int walNextHash(int iPriorHash){ + return (iPriorHash+1)&(HASHTABLE_NSLOT-1); +} + +/* +** Return pointers to the hash table and page number array stored on +** page iHash of the wal-index. The wal-index is broken into 32KB pages +** numbered starting from 0. +** +** Set output variable *paHash to point to the start of the hash table +** in the wal-index file. Set *piZero to one less than the frame +** number of the first frame indexed by this hash table. If a +** slot in the hash table is set to N, it refers to frame number +** (*piZero+N) in the log. +** +** Finally, set *paPgno so that *paPgno[1] is the page number of the +** first frame indexed by the hash table, frame (*piZero+1). +*/ +static int walHashGet( + Wal *pWal, /* WAL handle */ + int iHash, /* Find the iHash'th table */ + volatile ht_slot **paHash, /* OUT: Pointer to hash index */ + volatile u32 **paPgno, /* OUT: Pointer to page number array */ + u32 *piZero /* OUT: Frame associated with *paPgno[0] */ +){ + int rc; /* Return code */ + volatile u32 *aPgno; + + rc = walIndexPage(pWal, iHash, &aPgno); + assert( rc==SQLITE_OK || iHash>0 ); + + if( rc==SQLITE_OK ){ + u32 iZero; + volatile ht_slot *aHash; + + aHash = (volatile ht_slot *)&aPgno[HASHTABLE_NPAGE]; + if( iHash==0 ){ + aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; + iZero = 0; + }else{ + iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + } + + *paPgno = &aPgno[-1]; + *paHash = aHash; + *piZero = iZero; + } + return rc; +} + +/* +** Return the number of the wal-index page that contains the hash-table +** and page-number array that contain entries corresponding to WAL frame +** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages +** are numbered starting from 0. +*/ +static int walFramePage(u32 iFrame){ + int iHash = (iFrame+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1) / HASHTABLE_NPAGE; + assert( (iHash==0 || iFrame>HASHTABLE_NPAGE_ONE) + && (iHash>=1 || iFrame<=HASHTABLE_NPAGE_ONE) + && (iHash<=1 || iFrame>(HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE)) + && (iHash>=2 || iFrame<=HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE) + && (iHash<=2 || iFrame>(HASHTABLE_NPAGE_ONE+2*HASHTABLE_NPAGE)) + ); + return iHash; +} + +/* +** Return the page number associated with frame iFrame in this WAL. +*/ +static u32 walFramePgno(Wal *pWal, u32 iFrame){ + int iHash = walFramePage(iFrame); + if( iHash==0 ){ + return pWal->apWiData[0][WALINDEX_HDR_SIZE/sizeof(u32) + iFrame - 1]; + } + return pWal->apWiData[iHash][(iFrame-1-HASHTABLE_NPAGE_ONE)%HASHTABLE_NPAGE]; +} + +/* +** Remove entries from the hash table that point to WAL slots greater +** than pWal->hdr.mxFrame. +** +** This function is called whenever pWal->hdr.mxFrame is decreased due +** to a rollback or savepoint. +** +** At most only the hash table containing pWal->hdr.mxFrame needs to be +** updated. Any later hash tables will be automatically cleared when +** pWal->hdr.mxFrame advances to the point where those hash tables are +** actually needed. +*/ +static void walCleanupHash(Wal *pWal){ + volatile ht_slot *aHash = 0; /* Pointer to hash table to clear */ + volatile u32 *aPgno = 0; /* Page number array for hash table */ + u32 iZero = 0; /* frame == (aHash[x]+iZero) */ + int iLimit = 0; /* Zero values greater than this */ + int nByte; /* Number of bytes to zero in aPgno[] */ + int i; /* Used to iterate through aHash[] */ + + assert( pWal->writeLock ); + testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE-1 ); + testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE ); + testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE+1 ); + + if( pWal->hdr.mxFrame==0 ) return; + + /* Obtain pointers to the hash-table and page-number array containing + ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed + ** that the page said hash-table and array reside on is already mapped. + */ + assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); + assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); + walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &aHash, &aPgno, &iZero); + + /* Zero all hash-table entries that correspond to frame numbers greater + ** than pWal->hdr.mxFrame. + */ + iLimit = pWal->hdr.mxFrame - iZero; + assert( iLimit>0 ); + for(i=0; iiLimit ){ + aHash[i] = 0; + } + } + + /* Zero the entries in the aPgno array that correspond to frames with + ** frame numbers greater than pWal->hdr.mxFrame. + */ + nByte = (int)((char *)aHash - (char *)&aPgno[iLimit+1]); + memset((void *)&aPgno[iLimit+1], 0, nByte); + +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + /* Verify that the every entry in the mapping region is still reachable + ** via the hash table even after the cleanup. + */ + if( iLimit ){ + int i; /* Loop counter */ + int iKey; /* Hash key */ + for(i=1; i<=iLimit; i++){ + for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ + if( aHash[iKey]==i ) break; + } + assert( aHash[iKey]==i ); + } + } +#endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ +} + + +/* +** Set an entry in the wal-index that will map database page number +** pPage into WAL frame iFrame. +*/ +static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ + int rc; /* Return code */ + u32 iZero = 0; /* One less than frame number of aPgno[1] */ + volatile u32 *aPgno = 0; /* Page number array */ + volatile ht_slot *aHash = 0; /* Hash table */ + + rc = walHashGet(pWal, walFramePage(iFrame), &aHash, &aPgno, &iZero); + + /* Assuming the wal-index file was successfully mapped, populate the + ** page number array and hash table entry. + */ + if( rc==SQLITE_OK ){ + int iKey; /* Hash table key */ + int idx; /* Value to write to hash-table slot */ + int nCollide; /* Number of hash collisions */ + + idx = iFrame - iZero; + assert( idx <= HASHTABLE_NSLOT/2 + 1 ); + + /* If this is the first entry to be added to this hash-table, zero the + ** entire hash table and aPgno[] array before proceding. + */ + if( idx==1 ){ + int nByte = (int)((u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]); + memset((void*)&aPgno[1], 0, nByte); + } + + /* If the entry in aPgno[] is already set, then the previous writer + ** must have exited unexpectedly in the middle of a transaction (after + ** writing one or more dirty pages to the WAL to free up memory). + ** Remove the remnants of that writers uncommitted transaction from + ** the hash-table before writing any new entries. + */ + if( aPgno[idx] ){ + walCleanupHash(pWal); + assert( !aPgno[idx] ); + } + + /* Write the aPgno[] array entry and the hash-table slot. */ + nCollide = idx; + for(iKey=walHash(iPage); aHash[iKey]; iKey=walNextHash(iKey)){ + if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; + } + aPgno[idx] = iPage; + aHash[iKey] = (ht_slot)idx; + +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + /* Verify that the number of entries in the hash table exactly equals + ** the number of entries in the mapping region. + */ + { + int i; /* Loop counter */ + int nEntry = 0; /* Number of entries in the hash table */ + for(i=0; ickptLock==1 || pWal->ckptLock==0 ); + assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); + assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); + assert( pWal->writeLock ); + iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; + nLock = SQLITE_SHM_NLOCK - iLock; + rc = walLockExclusive(pWal, iLock, nLock); + if( rc ){ + return rc; + } + WALTRACE(("WAL%p: recovery begin...\n", pWal)); + + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + + rc = sqlite3OsFileSize(pWal->pWalFd, &nSize); + if( rc!=SQLITE_OK ){ + goto recovery_error; + } + + if( nSize>WAL_HDRSIZE ){ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + int iFrame; /* Index of last frame read */ + i64 iOffset; /* Next offset to read from log file */ + int szPage; /* Page size according to the log */ + u32 magic; /* Magic value read from WAL header */ + u32 version; /* Magic value read from WAL header */ + int isValid; /* True if this frame is valid */ + + /* Read in the WAL header. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + goto recovery_error; + } + + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** data. Similarly, if the 'magic' value is invalid, ignore the whole + ** WAL file. + */ + magic = sqlite3Get4byte(&aBuf[0]); + szPage = sqlite3Get4byte(&aBuf[8]); + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 + ){ + goto finished; + } + pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); + pWal->szPage = szPage; + pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); + memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + + /* Verify that the WAL header checksum is correct */ + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum + ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) + ){ + goto finished; + } + + /* Verify that the version number on the WAL format is one that + ** are able to understand */ + version = sqlite3Get4byte(&aBuf[4]); + if( version!=WAL_MAX_VERSION ){ + rc = SQLITE_CANTOPEN_BKPT; + goto finished; + } + + /* Malloc a buffer to read frames into. */ + szFrame = szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc(szFrame); + if( !aFrame ){ + rc = SQLITE_NOMEM; + goto recovery_error; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + + /* Read all frames from the log file. */ + iFrame = 0; + for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){ + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + iFrame++; + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); + if( !isValid ) break; + rc = walIndexAppend(pWal, iFrame, pgno); + if( rc!=SQLITE_OK ) break; + + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + } + } + + sqlite3_free(aFrame); + } + +finished: + if( rc==SQLITE_OK ){ + volatile WalCkptInfo *pInfo; + int i; + pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; + pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; + walIndexWriteHdr(pWal); + + /* Reset the checkpoint-header. This is safe because this thread is + ** currently holding locks that exclude all other readers, writers and + ** checkpointers. + */ + pInfo = walCkptInfo(pWal); + pInfo->nBackfill = 0; + pInfo->aReadMark[0] = 0; + for(i=1; iaReadMark[i] = READMARK_NOT_USED; + if( pWal->hdr.mxFrame ) pInfo->aReadMark[1] = pWal->hdr.mxFrame; + + /* If more than one frame was recovered from the log file, report an + ** event via sqlite3_log(). This is to help with identifying performance + ** problems caused by applications routinely shutting down without + ** checkpointing the log file. + */ + if( pWal->hdr.nPage ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, + "recovered %d frames from WAL file %s", + pWal->hdr.mxFrame, pWal->zWalName + ); + } + } + +recovery_error: + WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); + walUnlockExclusive(pWal, iLock, nLock); + return rc; +} + +/* +** Close an open wal-index. +*/ +static void walIndexClose(Wal *pWal, int isDelete){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void *)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + }else{ + sqlite3OsShmUnmap(pWal->pDbFd, isDelete); + } +} + +/* +** Open a connection to the WAL file zWalName. The database file must +** already be opened on connection pDbFd. The buffer that zWalName points +** to must remain valid for the lifetime of the returned Wal* handle. +** +** A SHARED lock should be held on the database file when this function +** is called. The purpose of this SHARED lock is to prevent any other +** client from unlinking the WAL or wal-index file. If another process +** were to do this just after this client opened one of these files, the +** system would be badly broken. +** +** If the log file is successfully opened, SQLITE_OK is returned and +** *ppWal is set to point to a new WAL handle. If an error occurs, +** an SQLite error code is returned and *ppWal is left unmodified. +*/ +SQLITE_PRIVATE int sqlite3WalOpen( + sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ + sqlite3_file *pDbFd, /* The open database file */ + const char *zWalName, /* Name of the WAL file */ + int bNoShm, /* True to run in heap-memory mode */ + i64 mxWalSize, /* Truncate WAL to this size on reset */ + Wal **ppWal /* OUT: Allocated Wal handle */ +){ + int rc; /* Return Code */ + Wal *pRet; /* Object to allocate and return */ + int flags; /* Flags passed to OsOpen() */ + + assert( zWalName && zWalName[0] ); + assert( pDbFd ); + + /* In the amalgamation, the os_unix.c and os_win.c source files come before + ** this source file. Verify that the #defines of the locking byte offsets + ** in os_unix.c and os_win.c agree with the WALINDEX_LOCK_OFFSET value. + */ +#ifdef WIN_SHM_BASE + assert( WIN_SHM_BASE==WALINDEX_LOCK_OFFSET ); +#endif +#ifdef UNIX_SHM_BASE + assert( UNIX_SHM_BASE==WALINDEX_LOCK_OFFSET ); +#endif + + + /* Allocate an instance of struct Wal to return. */ + *ppWal = 0; + pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile); + if( !pRet ){ + return SQLITE_NOMEM; + } + + pRet->pVfs = pVfs; + pRet->pWalFd = (sqlite3_file *)&pRet[1]; + pRet->pDbFd = pDbFd; + pRet->readLock = -1; + pRet->mxWalSize = mxWalSize; + pRet->zWalName = zWalName; + pRet->syncHeader = 1; + pRet->padToSectorBoundary = 1; + pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); + + /* Open file handle on the write-ahead log file. */ + flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); + if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ + pRet->readOnly = WAL_RDONLY; + } + + if( rc!=SQLITE_OK ){ + walIndexClose(pRet, 0); + sqlite3OsClose(pRet->pWalFd); + sqlite3_free(pRet); + }else{ + int iDC = sqlite3OsDeviceCharacteristics(pDbFd); + if( iDC & SQLITE_IOCAP_SEQUENTIAL ){ pRet->syncHeader = 0; } + if( iDC & SQLITE_IOCAP_POWERSAFE_OVERWRITE ){ + pRet->padToSectorBoundary = 0; + } + *ppWal = pRet; + WALTRACE(("WAL%d: opened\n", pRet)); + } + return rc; +} + +/* +** Change the size to which the WAL file is trucated on each reset. +*/ +SQLITE_PRIVATE void sqlite3WalLimit(Wal *pWal, i64 iLimit){ + if( pWal ) pWal->mxWalSize = iLimit; +} + +/* +** Find the smallest page number out of all pages held in the WAL that +** has not been returned by any prior invocation of this method on the +** same WalIterator object. Write into *piFrame the frame index where +** that page was last written into the WAL. Write into *piPage the page +** number. +** +** Return 0 on success. If there are no pages in the WAL with a page +** number larger than *piPage, then return 1. +*/ +static int walIteratorNext( + WalIterator *p, /* Iterator */ + u32 *piPage, /* OUT: The page number of the next page */ + u32 *piFrame /* OUT: Wal frame index of next page */ +){ + u32 iMin; /* Result pgno must be greater than iMin */ + u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */ + int i; /* For looping through segments */ + + iMin = p->iPrior; + assert( iMin<0xffffffff ); + for(i=p->nSegment-1; i>=0; i--){ + struct WalSegment *pSegment = &p->aSegment[i]; + while( pSegment->iNextnEntry ){ + u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; + if( iPg>iMin ){ + if( iPgiZero + pSegment->aIndex[pSegment->iNext]; + } + break; + } + pSegment->iNext++; + } + } + + *piPage = p->iPrior = iRet; + return (iRet==0xFFFFFFFF); +} + +/* +** This function merges two sorted lists into a single sorted list. +** +** aLeft[] and aRight[] are arrays of indices. The sort key is +** aContent[aLeft[]] and aContent[aRight[]]. Upon entry, the following +** is guaranteed for all J0 && nRight>0 ); + while( iRight=nRight || aContent[aLeft[iLeft]]=nLeft || aContent[aLeft[iLeft]]>dbpage ); + assert( iRight>=nRight || aContent[aRight[iRight]]>dbpage ); + } + + *paRight = aLeft; + *pnRight = iOut; + memcpy(aLeft, aTmp, sizeof(aTmp[0])*iOut); +} + +/* +** Sort the elements in list aList using aContent[] as the sort key. +** Remove elements with duplicate keys, preferring to keep the +** larger aList[] values. +** +** The aList[] entries are indices into aContent[]. The values in +** aList[] are to be sorted so that for all J0 ); + assert( HASHTABLE_NPAGE==(1<<(ArraySize(aSub)-1)) ); + + for(iList=0; iListaList && p->nList<=(1<aList==&aList[iList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); + } + aSub[iSub].aList = aMerge; + aSub[iSub].nList = nMerge; + } + + for(iSub++; iSubnList<=(1<aList==&aList[nList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); + } + } + assert( aMerge==aList ); + *pnList = nMerge; + +#ifdef SQLITE_DEBUG + { + int i; + for(i=1; i<*pnList; i++){ + assert( aContent[aList[i]] > aContent[aList[i-1]] ); + } + } +#endif +} + +/* +** Free an iterator allocated by walIteratorInit(). +*/ +static void walIteratorFree(WalIterator *p){ + sqlite3ScratchFree(p); +} + +/* +** Construct a WalInterator object that can be used to loop over all +** pages in the WAL in ascending order. The caller must hold the checkpoint +** lock. +** +** On success, make *pp point to the newly allocated WalInterator object +** return SQLITE_OK. Otherwise, return an error code. If this routine +** returns an error, the value of *pp is undefined. +** +** The calling routine should invoke walIteratorFree() to destroy the +** WalIterator object when it has finished with it. +*/ +static int walIteratorInit(Wal *pWal, WalIterator **pp){ + WalIterator *p; /* Return value */ + int nSegment; /* Number of segments to merge */ + u32 iLast; /* Last frame in log */ + int nByte; /* Number of bytes to allocate */ + int i; /* Iterator variable */ + ht_slot *aTmp; /* Temp space used by merge-sort */ + int rc = SQLITE_OK; /* Return Code */ + + /* This routine only runs while holding the checkpoint lock. And + ** it only runs if there is actually content in the log (mxFrame>0). + */ + assert( pWal->ckptLock && pWal->hdr.mxFrame>0 ); + iLast = pWal->hdr.mxFrame; + + /* Allocate space for the WalIterator object. */ + nSegment = walFramePage(iLast) + 1; + nByte = sizeof(WalIterator) + + (nSegment-1)*sizeof(struct WalSegment) + + iLast*sizeof(ht_slot); + p = (WalIterator *)sqlite3ScratchMalloc(nByte); + if( !p ){ + return SQLITE_NOMEM; + } + memset(p, 0, nByte); + p->nSegment = nSegment; + + /* Allocate temporary space used by the merge-sort routine. This block + ** of memory will be freed before this function returns. + */ + aTmp = (ht_slot *)sqlite3ScratchMalloc( + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) + ); + if( !aTmp ){ + rc = SQLITE_NOMEM; + } + + for(i=0; rc==SQLITE_OK && iaSegment[p->nSegment])[iZero]; + iZero++; + + for(j=0; jaSegment[i].iZero = iZero; + p->aSegment[i].nEntry = nEntry; + p->aSegment[i].aIndex = aIndex; + p->aSegment[i].aPgno = (u32 *)aPgno; + } + } + sqlite3ScratchFree(aTmp); + + if( rc!=SQLITE_OK ){ + walIteratorFree(p); + } + *pp = p; + return rc; +} + +/* +** Attempt to obtain the exclusive WAL lock defined by parameters lockIdx and +** n. If the attempt fails and parameter xBusy is not NULL, then it is a +** busy-handler function. Invoke it and retry the lock until either the +** lock is successfully obtained or the busy-handler returns 0. +*/ +static int walBusyLock( + Wal *pWal, /* WAL connection */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int lockIdx, /* Offset of first byte to lock */ + int n /* Number of bytes to lock */ +){ + int rc; + do { + rc = walLockExclusive(pWal, lockIdx, n); + }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) ); + return rc; +} + +/* +** The cache of the wal-index header must be valid to call this function. +** Return the page-size in bytes used by the database. +*/ +static int walPagesize(Wal *pWal){ + return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); +} + +/* +** Copy as much content as we can from the WAL back into the database file +** in response to an sqlite3_wal_checkpoint() request or the equivalent. +** +** The amount of information copies from WAL to database might be limited +** by active readers. This routine will never overwrite a database page +** that a concurrent reader might be using. +** +** All I/O barrier operations (a.k.a fsyncs) occur in this routine when +** SQLite is in WAL-mode in synchronous=NORMAL. That means that if +** checkpoints are always run by a background thread or background +** process, foreground threads will never block on a lengthy fsync call. +** +** Fsync is called on the WAL before writing content out of the WAL and +** into the database. This ensures that if the new content is persistent +** in the WAL and can be recovered following a power-loss or hard reset. +** +** Fsync is also called on the database file if (and only if) the entire +** WAL content is copied into the database file. This second fsync makes +** it safe to delete the WAL since the new content will persist in the +** database file. +** +** This routine uses and updates the nBackfill field of the wal-index header. +** This is the only routine tha will increase the value of nBackfill. +** (A WAL reset or recovery will revert nBackfill to zero, but not increase +** its value.) +** +** The caller must be holding sufficient locks to ensure that no other +** checkpoint is running (in any other thread or process) at the same +** time. +*/ +static int walCheckpoint( + Wal *pWal, /* Wal connection */ + int eMode, /* One of PASSIVE, FULL or RESTART */ + int (*xBusyCall)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags for OsSync() (or 0) */ + u8 *zBuf /* Temporary buffer to use */ +){ + int rc; /* Return code */ + int szPage; /* Database page-size */ + WalIterator *pIter = 0; /* Wal iterator context */ + u32 iDbpage = 0; /* Next database page to write */ + u32 iFrame = 0; /* Wal frame containing data for iDbpage */ + u32 mxSafeFrame; /* Max frame that can be backfilled */ + u32 mxPage; /* Max database page to write */ + int i; /* Loop counter */ + volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ + + szPage = walPagesize(pWal); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + pInfo = walCkptInfo(pWal); + if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK; + + /* Allocate the iterator */ + rc = walIteratorInit(pWal, &pIter); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( pIter ); + + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; + + /* Compute in mxSafeFrame the index of the last frame of the WAL that is + ** safe to write into the database. Frames beyond mxSafeFrame might + ** overwrite database pages that are in use by active readers and thus + ** cannot be backfilled from the WAL. + */ + mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; + for(i=1; iaReadMark[i]; + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; + xBusy = 0; + }else{ + goto walcheckpoint_out; + } + } + } + + if( pInfo->nBackfillnBackfill; + + /* Sync the WAL to disk */ + if( sync_flags ){ + rc = sqlite3OsSync(pWal->pWalFd, sync_flags); + } + + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } + } + + + /* Iterate through the contents of the WAL, copying data to the db file. */ + while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + i64 iOffset; + assert( walFramePgno(pWal, iFrame)==iDbpage ); + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; + iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ + rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + iOffset = (iDbpage-1)*(i64)szPage; + testcase( IS_BIG_INT(iOffset) ); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + } + + /* If work was actually accomplished... */ + if( rc==SQLITE_OK ){ + if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ + i64 szDb = pWal->hdr.nPage*(i64)szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + if( rc==SQLITE_OK && sync_flags ){ + rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + } + } + if( rc==SQLITE_OK ){ + pInfo->nBackfill = mxSafeFrame; + } + } + + /* Release the reader lock held while backfilling */ + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); + } + + if( rc==SQLITE_BUSY ){ + /* Reset the return code so as not to report a checkpoint failure + ** just because there are active readers. */ + rc = SQLITE_OK; + } + + /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal + ** file has been copied into the database file, then block until all + ** readers have finished using the wal file. This ensures that the next + ** process to write to the database restarts the wal file. + */ + if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + assert( pWal->writeLock ); + if( pInfo->nBackfillhdr.mxFrame ){ + rc = SQLITE_BUSY; + }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ + assert( mxSafeFrame==pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + } + } + } + + walcheckpoint_out: + walIteratorFree(pIter); + return rc; +} + +/* +** If the WAL file is currently larger than nMax bytes in size, truncate +** it to exactly nMax bytes. If an error occurs while doing so, ignore it. +*/ +static void walLimitSize(Wal *pWal, i64 nMax){ + i64 sz; + int rx; + sqlite3BeginBenignMalloc(); + rx = sqlite3OsFileSize(pWal->pWalFd, &sz); + if( rx==SQLITE_OK && (sz > nMax ) ){ + rx = sqlite3OsTruncate(pWal->pWalFd, nMax); + } + sqlite3EndBenignMalloc(); + if( rx ){ + sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); + } +} + +/* +** Close a connection to a log file. +*/ +SQLITE_PRIVATE int sqlite3WalClose( + Wal *pWal, /* Wal to close */ + int sync_flags, /* Flags to pass to OsSync() (or 0) */ + int nBuf, + u8 *zBuf /* Buffer of at least nBuf bytes */ +){ + int rc = SQLITE_OK; + if( pWal ){ + int isDelete = 0; /* True to unlink wal and wal-index files */ + + /* If an EXCLUSIVE lock can be obtained on the database file (using the + ** ordinary, rollback-mode locking methods, this guarantees that the + ** connection associated with this log file is the only connection to + ** the database. In this case checkpoint the database and unlink both + ** the wal and wal-index files. + ** + ** The EXCLUSIVE lock is not released before returning. + */ + rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); + if( rc==SQLITE_OK ){ + if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; + } + rc = sqlite3WalCheckpoint( + pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 + ); + if( rc==SQLITE_OK ){ + int bPersist = -1; + sqlite3OsFileControlHint( + pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist + ); + if( bPersist!=1 ){ + /* Try to delete the WAL file if the checkpoint completed and + ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal + ** mode (!bPersist) */ + isDelete = 1; + }else if( pWal->mxWalSize>=0 ){ + /* Try to truncate the WAL file to zero bytes if the checkpoint + ** completed and fsynced (rc==SQLITE_OK) and we are in persistent + ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a + ** non-negative value (pWal->mxWalSize>=0). Note that we truncate + ** to zero bytes as truncating to the journal_size_limit might + ** leave a corrupt WAL file on disk. */ + walLimitSize(pWal, 0); + } + } + } + + walIndexClose(pWal, isDelete); + sqlite3OsClose(pWal->pWalFd); + if( isDelete ){ + sqlite3BeginBenignMalloc(); + sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0); + sqlite3EndBenignMalloc(); + } + WALTRACE(("WAL%p: closed\n", pWal)); + sqlite3_free((void *)pWal->apWiData); + sqlite3_free(pWal); + } + return rc; +} + +/* +** Try to read the wal-index header. Return 0 on success and 1 if +** there is a problem. +** +** The wal-index is in shared memory. Another thread or process might +** be writing the header at the same time this procedure is trying to +** read it, which might result in inconsistency. A dirty read is detected +** by verifying that both copies of the header are the same and also by +** a checksum on the header. +** +** If and only if the read is consistent and the header is different from +** pWal->hdr, then pWal->hdr is updated to the content of the new header +** and *pChanged is set to 1. +** +** If the checksum cannot be verified return non-zero. If the header +** is read successfully and the checksum verified, return zero. +*/ +static int walIndexTryHdr(Wal *pWal, int *pChanged){ + u32 aCksum[2]; /* Checksum on the header content */ + WalIndexHdr h1, h2; /* Two copies of the header content */ + WalIndexHdr volatile *aHdr; /* Header in shared memory */ + + /* The first page of the wal-index must be mapped at this point. */ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + + /* Read the header. This might happen concurrently with a write to the + ** same area of shared memory on a different CPU in a SMP, + ** meaning it is possible that an inconsistent snapshot is read + ** from the file. If this happens, return non-zero. + ** + ** There are two copies of the header at the beginning of the wal-index. + ** When reading, read [0] first then [1]. Writes are in the reverse order. + ** Memory barriers are used to prevent the compiler or the hardware from + ** reordering the reads and writes. + */ + aHdr = walIndexHdr(pWal); + memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); + walShmBarrier(pWal); + memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); + + if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ + return 1; /* Dirty read */ + } + if( h1.isInit==0 ){ + return 1; /* Malformed header - probably all zeros */ + } + walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum); + if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){ + return 1; /* Checksum does not match */ + } + + if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){ + *pChanged = 1; + memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr)); + pWal->szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( pWal->szPage<=32768 ); + testcase( pWal->szPage>=65536 ); + } + + /* The header was successfully read. Return zero. */ + return 0; +} + +/* +** Read the wal-index header from the wal-index and into pWal->hdr. +** If the wal-header appears to be corrupt, try to reconstruct the +** wal-index from the WAL before returning. +** +** Set *pChanged to 1 if the wal-index header value in pWal->hdr is +** changed by this opertion. If pWal->hdr is unchanged, set *pChanged +** to 0. +** +** If the wal-index header is successfully read, return SQLITE_OK. +** Otherwise an SQLite error code. +*/ +static int walIndexReadHdr(Wal *pWal, int *pChanged){ + int rc; /* Return code */ + int badHdr; /* True if a header read failed */ + volatile u32 *page0; /* Chunk of wal-index containing header */ + + /* Ensure that page 0 of the wal-index (the page that contains the + ** wal-index header) is mapped. Return early if an error occurs here. + */ + assert( pChanged ); + rc = walIndexPage(pWal, 0, &page0); + if( rc!=SQLITE_OK ){ + return rc; + }; + assert( page0 || pWal->writeLock==0 ); + + /* If the first page of the wal-index has been mapped, try to read the + ** wal-index header immediately, without holding any lock. This usually + ** works, but may fail if the wal-index header is corrupt or currently + ** being modified by another thread or process. + */ + badHdr = (page0 ? walIndexTryHdr(pWal, pChanged) : 1); + + /* If the first attempt failed, it might have been due to a race + ** with a writer. So get a WRITE lock and try again. + */ + assert( badHdr==0 || pWal->writeLock==0 ); + if( badHdr ){ + if( pWal->readOnly & WAL_SHM_RDONLY ){ + if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ + walUnlockShared(pWal, WAL_WRITE_LOCK); + rc = SQLITE_READONLY_RECOVERY; + } + }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ + badHdr = walIndexTryHdr(pWal, pChanged); + if( badHdr ){ + /* If the wal-index header is still malformed even while holding + ** a WRITE lock, it can only mean that the header is corrupted and + ** needs to be reconstructed. So run recovery to do exactly that. + */ + rc = walIndexRecover(pWal); + *pChanged = 1; + } + } + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + } + } + + /* If the header is read successfully, check the version number to make + ** sure the wal-index was not constructed with some future format that + ** this version of SQLite cannot understand. + */ + if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ + rc = SQLITE_CANTOPEN_BKPT; + } + + return rc; +} + +/* +** This is the value that walTryBeginRead returns when it needs to +** be retried. +*/ +#define WAL_RETRY (-1) + +/* +** Attempt to start a read transaction. This might fail due to a race or +** other transient condition. When that happens, it returns WAL_RETRY to +** indicate to the caller that it is safe to retry immediately. +** +** On success return SQLITE_OK. On a permanent failure (such an +** I/O error or an SQLITE_BUSY because another process is running +** recovery) return a positive error code. +** +** The useWal parameter is true to force the use of the WAL and disable +** the case where the WAL is bypassed because it has been completely +** checkpointed. If useWal==0 then this routine calls walIndexReadHdr() +** to make a copy of the wal-index header into pWal->hdr. If the +** wal-index header has changed, *pChanged is set to 1 (as an indication +** to the caller that the local paget cache is obsolete and needs to be +** flushed.) When useWal==1, the wal-index header is assumed to already +** be loaded and the pChanged parameter is unused. +** +** The caller must set the cnt parameter to the number of prior calls to +** this routine during the current read attempt that returned WAL_RETRY. +** This routine will start taking more aggressive measures to clear the +** race conditions after multiple WAL_RETRY returns, and after an excessive +** number of errors will ultimately return SQLITE_PROTOCOL. The +** SQLITE_PROTOCOL return indicates that some other process has gone rogue +** and is not honoring the locking protocol. There is a vanishingly small +** chance that SQLITE_PROTOCOL could be returned because of a run of really +** bad luck when there is lots of contention for the wal-index, but that +** possibility is so small that it can be safely neglected, we believe. +** +** On success, this routine obtains a read lock on +** WAL_READ_LOCK(pWal->readLock). The pWal->readLock integer is +** in the range 0 <= pWal->readLock < WAL_NREADER. If pWal->readLock==(-1) +** that means the Wal does not hold any read lock. The reader must not +** access any database page that is modified by a WAL frame up to and +** including frame number aReadMark[pWal->readLock]. The reader will +** use WAL frames up to and including pWal->hdr.mxFrame if pWal->readLock>0 +** Or if pWal->readLock==0, then the reader will ignore the WAL +** completely and get all content directly from the database file. +** If the useWal parameter is 1 then the WAL will never be ignored and +** this routine will always set pWal->readLock>0 on success. +** When the read transaction is completed, the caller must release the +** lock on WAL_READ_LOCK(pWal->readLock) and set pWal->readLock to -1. +** +** This routine uses the nBackfill and aReadMark[] fields of the header +** to select a particular WAL_READ_LOCK() that strives to let the +** checkpoint process do as much work as possible. This routine might +** update values of the aReadMark[] array in the header, but if it does +** so it takes care to hold an exclusive lock on the corresponding +** WAL_READ_LOCK() while changing values. +*/ +static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ + volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ + u32 mxReadMark; /* Largest aReadMark[] value */ + int mxI; /* Index of largest aReadMark[] value */ + int i; /* Loop counter */ + int rc = SQLITE_OK; /* Return code */ + + assert( pWal->readLock<0 ); /* Not currently locked */ + + /* Take steps to avoid spinning forever if there is a protocol error. + ** + ** Circumstances that cause a RETRY should only last for the briefest + ** instances of time. No I/O or other system calls are done while the + ** locks are held, so the locks should not be held for very long. But + ** if we are unlucky, another process that is holding a lock might get + ** paged out or take a page-fault that is time-consuming to resolve, + ** during the few nanoseconds that it is holding the lock. In that case, + ** it might take longer than normal for the lock to free. + ** + ** After 5 RETRYs, we begin calling sqlite3OsSleep(). The first few + ** calls to sqlite3OsSleep() have a delay of 1 microsecond. Really this + ** is more of a scheduler yield than an actual delay. But on the 10th + ** an subsequent retries, the delays start becoming longer and longer, + ** so that on the 100th (and last) RETRY we delay for 323 milliseconds. + ** The total delay time before giving up is less than 10 seconds. + */ + if( cnt>5 ){ + int nDelay = 1; /* Pause time in microseconds */ + if( cnt>100 ){ + VVA_ONLY( pWal->lockError = 1; ) + return SQLITE_PROTOCOL; + } + if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; + sqlite3OsSleep(pWal->pVfs, nDelay); + } + + if( !useWal ){ + rc = walIndexReadHdr(pWal, pChanged); + if( rc==SQLITE_BUSY ){ + /* If there is not a recovery running in another thread or process + ** then convert BUSY errors to WAL_RETRY. If recovery is known to + ** be running, convert BUSY to BUSY_RECOVERY. There is a race here + ** which might cause WAL_RETRY to be returned even if BUSY_RECOVERY + ** would be technically correct. But the race is benign since with + ** WAL_RETRY this routine will be called again and will probably be + ** right on the second iteration. + */ + if( pWal->apWiData[0]==0 ){ + /* This branch is taken when the xShmMap() method returns SQLITE_BUSY. + ** We assume this is a transient condition, so return WAL_RETRY. The + ** xShmMap() implementation used by the default unix and win32 VFS + ** modules may return SQLITE_BUSY due to a race condition in the + ** code that determines whether or not the shared-memory region + ** must be zeroed before the requested page is returned. + */ + rc = WAL_RETRY; + }else if( SQLITE_OK==(rc = walLockShared(pWal, WAL_RECOVER_LOCK)) ){ + walUnlockShared(pWal, WAL_RECOVER_LOCK); + rc = WAL_RETRY; + }else if( rc==SQLITE_BUSY ){ + rc = SQLITE_BUSY_RECOVERY; + } + } + if( rc!=SQLITE_OK ){ + return rc; + } + } + + pInfo = walCkptInfo(pWal); + if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ){ + /* The WAL has been completely backfilled (or it is empty). + ** and can be safely ignored. + */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + walShmBarrier(pWal); + if( rc==SQLITE_OK ){ + if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ + /* It is not safe to allow the reader to continue here if frames + ** may have been appended to the log before READ_LOCK(0) was obtained. + ** When holding READ_LOCK(0), the reader ignores the entire log file, + ** which implies that the database file contains a trustworthy + ** snapshoT. Since holding READ_LOCK(0) prevents a checkpoint from + ** happening, this is usually correct. + ** + ** However, if frames have been appended to the log (or if the log + ** is wrapped and written for that matter) before the READ_LOCK(0) + ** is obtained, that is not necessarily true. A checkpointer may + ** have started to backfill the appended frames but crashed before + ** it finished. Leaving a corrupt image in the database file. + */ + walUnlockShared(pWal, WAL_READ_LOCK(0)); + return WAL_RETRY; + } + pWal->readLock = 0; + return SQLITE_OK; + }else if( rc!=SQLITE_BUSY ){ + return rc; + } + } + + /* If we get this far, it means that the reader will want to use + ** the WAL to get at content from recent commits. The job now is + ** to select one of the aReadMark[] entries that is closest to + ** but not exceeding pWal->hdr.mxFrame and lock that entry. + */ + mxReadMark = 0; + mxI = 0; + for(i=1; iaReadMark[i]; + if( mxReadMark<=thisMark && thisMark<=pWal->hdr.mxFrame ){ + assert( thisMark!=READMARK_NOT_USED ); + mxReadMark = thisMark; + mxI = i; + } + } + /* There was once an "if" here. The extra "{" is to preserve indentation. */ + { + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkhdr.mxFrame || mxI==0) + ){ + for(i=1; iaReadMark[i] = pWal->hdr.mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; + }else if( rc!=SQLITE_BUSY ){ + return rc; + } + } + } + if( mxI==0 ){ + assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + } + + rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + if( rc ){ + return rc==SQLITE_BUSY ? WAL_RETRY : rc; + } + /* Now that the read-lock has been obtained, check that neither the + ** value in the aReadMark[] array or the contents of the wal-index + ** header have changed. + ** + ** It is necessary to check that the wal-index header did not change + ** between the time it was read and when the shared-lock was obtained + ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility + ** that the log file may have been wrapped by a writer, or that frames + ** that occur later in the log than pWal->hdr.mxFrame may have been + ** copied into the database by a checkpointer. If either of these things + ** happened, then reading the database with the current value of + ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry + ** instead. + ** + ** This does not guarantee that the copy of the wal-index header is up to + ** date before proceeding. That would not be possible without somehow + ** blocking writers. It only guarantees that a dangerous checkpoint or + ** log-wrap (either of which would require an exclusive lock on + ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. + */ + walShmBarrier(pWal); + if( pInfo->aReadMark[mxI]!=mxReadMark + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ + walUnlockShared(pWal, WAL_READ_LOCK(mxI)); + return WAL_RETRY; + }else{ + assert( mxReadMark<=pWal->hdr.mxFrame ); + pWal->readLock = (i16)mxI; + } + } + return rc; +} + +/* +** Begin a read transaction on the database. +** +** This routine used to be called sqlite3OpenSnapshot() and with good reason: +** it takes a snapshot of the state of the WAL and wal-index for the current +** instant in time. The current thread will continue to use this snapshot. +** Other threads might append new content to the WAL and wal-index but +** that extra content is ignored by the current thread. +** +** If the database contents have changes since the previous read +** transaction, then *pChanged is set to 1 before returning. The +** Pager layer will use this to know that is cache is stale and +** needs to be flushed. +*/ +SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ + int rc; /* Return code */ + int cnt = 0; /* Number of TryBeginRead attempts */ + + do{ + rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); + }while( rc==WAL_RETRY ); + testcase( (rc&0xff)==SQLITE_BUSY ); + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + return rc; +} + +/* +** Finish with a read transaction. All this does is release the +** read-lock. +*/ +SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ + sqlite3WalEndWriteTransaction(pWal); + if( pWal->readLock>=0 ){ + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + pWal->readLock = -1; + } +} + +/* +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. +** +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. +*/ +SQLITE_PRIVATE int sqlite3WalFindFrame( + Wal *pWal, /* WAL handle */ + Pgno pgno, /* Database page number to read data for */ + u32 *piRead /* OUT: Frame number (or zero) */ +){ + u32 iRead = 0; /* If !=0, WAL frame to return data from */ + u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ + int iHash; /* Used to loop through N hash tables */ + + /* This routine is only be called from within a read transaction. */ + assert( pWal->readLock>=0 || pWal->lockError ); + + /* If the "last page" field of the wal-index header snapshot is 0, then + ** no data will be read from the wal under any circumstances. Return early + ** in this case as an optimization. Likewise, if pWal->readLock==0, + ** then the WAL is ignored by the reader so return early, as if the + ** WAL were empty. + */ + if( iLast==0 || pWal->readLock==0 ){ + *piRead = 0; + return SQLITE_OK; + } + + /* Search the hash table or tables for an entry matching page number + ** pgno. Each iteration of the following for() loop searches one + ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). + ** + ** This code might run concurrently to the code in walIndexAppend() + ** that adds entries to the wal-index (and possibly to this hash + ** table). This means the value just read from the hash + ** slot (aHash[iKey]) may have been added before or after the + ** current read transaction was opened. Values added after the + ** read transaction was opened may have been written incorrectly - + ** i.e. these slots may contain garbage data. However, we assume + ** that any slots written before the current read transaction was + ** opened remain unmodified. + ** + ** For the reasons above, the if(...) condition featured in the inner + ** loop of the following block is more stringent that would be required + ** if we had exclusive access to the hash-table: + ** + ** (aPgno[iFrame]==pgno): + ** This condition filters out normal hash-table collisions. + ** + ** (iFrame<=iLast): + ** This condition filters out entries that were added to the hash + ** table after the current read-transaction had started. + */ + for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ + volatile ht_slot *aHash; /* Pointer to hash table */ + volatile u32 *aPgno; /* Pointer to array of page numbers */ + u32 iZero; /* Frame number corresponding to aPgno[0] */ + int iKey; /* Hash slot index */ + int nCollide; /* Number of hash collisions remaining */ + int rc; /* Error code */ + + rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + if( rc!=SQLITE_OK ){ + return rc; + } + nCollide = HASHTABLE_NSLOT; + for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ + u32 iFrame = aHash[iKey] + iZero; + if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ + /* assert( iFrame>iRead ); -- not true if there is corruption */ + iRead = iFrame; + } + if( (nCollide--)==0 ){ + return SQLITE_CORRUPT_BKPT; + } + } + } + +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + /* If expensive assert() statements are available, do a linear search + ** of the wal-index file content. Make sure the results agree with the + ** result obtained using the hash indexes above. */ + { + u32 iRead2 = 0; + u32 iTest; + for(iTest=iLast; iTest>0; iTest--){ + if( walFramePgno(pWal, iTest)==pgno ){ + iRead2 = iTest; + break; + } + } + assert( iRead==iRead2 ); + } +#endif + + *piRead = iRead; + return SQLITE_OK; +} + +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +SQLITE_PRIVATE int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} + +/* +** Return the size of the database in pages (or zero, if unknown). +*/ +SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ + if( pWal && ALWAYS(pWal->readLock>=0) ){ + return pWal->hdr.nPage; + } + return 0; +} + + +/* +** This function starts a write transaction on the WAL. +** +** A read transaction must have already been started by a prior call +** to sqlite3WalBeginReadTransaction(). +** +** If another thread or process has written into the database since +** the read transaction was started, then it is not possible for this +** thread to write as doing so would cause a fork. So this routine +** returns SQLITE_BUSY in that case and no write transaction is started. +** +** There can only be a single writer active at a time. +*/ +SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ + int rc; + + /* Cannot start a write transaction without first holding a read + ** transaction. */ + assert( pWal->readLock>=0 ); + + if( pWal->readOnly ){ + return SQLITE_READONLY; + } + + /* Only one writer allowed at a time. Get the write lock. Return + ** SQLITE_BUSY if unable. + */ + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + if( rc ){ + return rc; + } + pWal->writeLock = 1; + + /* If another connection has written to the database file since the + ** time the read transaction on this connection was started, then + ** the write is disallowed. + */ + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + rc = SQLITE_BUSY_SNAPSHOT; + } + + return rc; +} + +/* +** End a write transaction. The commit has already been done. This +** routine merely releases the lock. +*/ +SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){ + if( pWal->writeLock ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + pWal->truncateOnCommit = 0; + } + return SQLITE_OK; +} + +/* +** If any data has been written (but not committed) to the log file, this +** function moves the write-pointer back to the start of the transaction. +** +** Additionally, the callback function is invoked for each frame written +** to the WAL since the start of the transaction. If the callback returns +** other than SQLITE_OK, it is not invoked again and the error code is +** returned to the caller. +** +** Otherwise, if the callback function does not return an error, this +** function returns SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ + int rc = SQLITE_OK; + if( ALWAYS(pWal->writeLock) ){ + Pgno iMax = pWal->hdr.mxFrame; + Pgno iFrame; + + /* Restore the clients cache of the wal-index header to the state it + ** was in before the client began writing to the database. + */ + memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + for(iFrame=pWal->hdr.mxFrame+1; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + iFrame++ + ){ + /* This call cannot fail. Unless the page for which the page number + ** is passed as the second argument is (a) in the cache and + ** (b) has an outstanding reference, then xUndo is either a no-op + ** (if (a) is false) or simply expels the page from the cache (if (b) + ** is false). + ** + ** If the upper layer is doing a rollback, it is guaranteed that there + ** are no outstanding references to any page other than page 1. And + ** page 1 is never written to the log until the transaction is + ** committed. As a result, the call to xUndo may not fail. + */ + assert( walFramePgno(pWal, iFrame)!=1 ); + rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + } + if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); + } + assert( rc==SQLITE_OK ); + return rc; +} + +/* +** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 +** values. This function populates the array with values required to +** "rollback" the write position of the WAL handle back to the current +** point in the event of a savepoint rollback (via WalSavepointUndo()). +*/ +SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ + assert( pWal->writeLock ); + aWalData[0] = pWal->hdr.mxFrame; + aWalData[1] = pWal->hdr.aFrameCksum[0]; + aWalData[2] = pWal->hdr.aFrameCksum[1]; + aWalData[3] = pWal->nCkpt; +} + +/* +** Move the write position of the WAL back to the point identified by +** the values in the aWalData[] array. aWalData must point to an array +** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated +** by a call to WalSavepoint(). +*/ +SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ + int rc = SQLITE_OK; + + assert( pWal->writeLock ); + assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame ); + + if( aWalData[3]!=pWal->nCkpt ){ + /* This savepoint was opened immediately after the write-transaction + ** was started. Right after that, the writer decided to wrap around + ** to the start of the log. Update the savepoint values to match. + */ + aWalData[0] = 0; + aWalData[3] = pWal->nCkpt; + } + + if( aWalData[0]hdr.mxFrame ){ + pWal->hdr.mxFrame = aWalData[0]; + pWal->hdr.aFrameCksum[0] = aWalData[1]; + pWal->hdr.aFrameCksum[1] = aWalData[2]; + walCleanupHash(pWal); + } + + return rc; +} + + +/* +** This function is called just before writing a set of frames to the log +** file (see sqlite3WalFrames()). It checks to see if, instead of appending +** to the current log file, it is possible to overwrite the start of the +** existing log file with the new frames (i.e. "reset" the log). If so, +** it sets pWal->hdr.mxFrame to 0. Otherwise, pWal->hdr.mxFrame is left +** unchanged. +** +** SQLITE_OK is returned if no error is encountered (regardless of whether +** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned +** if an error occurs. +*/ +static int walRestartLog(Wal *pWal){ + int rc = SQLITE_OK; + int cnt; + + if( pWal->readLock==0 ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + assert( pInfo->nBackfill==pWal->hdr.mxFrame ); + if( pInfo->nBackfill>0 ){ + u32 salt1; + sqlite3_randomness(4, &salt1); + rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + /* If all readers are using WAL_READ_LOCK(0) (in other words if no + ** readers are currently using the WAL), then the transactions + ** frames will overwrite the start of the existing log. Update the + ** wal-index header to reflect this. + ** + ** In theory it would be Ok to update the cache of the header only + ** at this point. But updating the actual wal-index header is also + ** safe and means there is no special case for sqlite3WalUndo() + ** to handle if this transaction is rolled back. + */ + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + aSalt[1] = salt1; + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + }else if( rc!=SQLITE_BUSY ){ + return rc; + } + } + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + cnt = 0; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + }while( rc==WAL_RETRY ); + assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + } + return rc; +} + +/* +** Information about the current state of the WAL file and where +** the next fsync should occur - passed from sqlite3WalFrames() into +** walWriteToLog(). +*/ +typedef struct WalWriter { + Wal *pWal; /* The complete WAL information */ + sqlite3_file *pFd; /* The WAL file to which we write */ + sqlite3_int64 iSyncPoint; /* Fsync at this offset */ + int syncFlags; /* Flags for the fsync */ + int szPage; /* Size of one page */ +} WalWriter; + +/* +** Write iAmt bytes of content into the WAL file beginning at iOffset. +** Do a sync when crossing the p->iSyncPoint boundary. +** +** In other words, if iSyncPoint is in between iOffset and iOffset+iAmt, +** first write the part before iSyncPoint, then sync, then write the +** rest. +*/ +static int walWriteToLog( + WalWriter *p, /* WAL to write to */ + void *pContent, /* Content to be written */ + int iAmt, /* Number of bytes to write */ + sqlite3_int64 iOffset /* Start writing at this offset */ +){ + int rc; + if( iOffsetiSyncPoint && iOffset+iAmt>=p->iSyncPoint ){ + int iFirstAmt = (int)(p->iSyncPoint - iOffset); + rc = sqlite3OsWrite(p->pFd, pContent, iFirstAmt, iOffset); + if( rc ) return rc; + iOffset += iFirstAmt; + iAmt -= iFirstAmt; + pContent = (void*)(iFirstAmt + (char*)pContent); + assert( p->syncFlags & (SQLITE_SYNC_NORMAL|SQLITE_SYNC_FULL) ); + rc = sqlite3OsSync(p->pFd, p->syncFlags & SQLITE_SYNC_MASK); + if( iAmt==0 || rc ) return rc; + } + rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset); + return rc; +} + +/* +** Write out a single frame of the WAL +*/ +static int walWriteOneFrame( + WalWriter *p, /* Where to write the frame */ + PgHdr *pPage, /* The page of the frame to be written */ + int nTruncate, /* The commit flag. Usually 0. >0 for commit */ + sqlite3_int64 iOffset /* Byte offset at which to write */ +){ + int rc; /* Result code from subfunctions */ + void *pData; /* Data actually written */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ +#if defined(SQLITE_HAS_CODEC) + if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM; +#else + pData = pPage->pData; +#endif + walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); + rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); + if( rc ) return rc; + /* Write the page data */ + rc = walWriteToLog(p, pData, p->szPage, iOffset+sizeof(aFrame)); + return rc; +} + +/* +** Write a set of frames to the log. The caller must hold the write-lock +** on the log file (obtained using sqlite3WalBeginWriteTransaction()). +*/ +SQLITE_PRIVATE int sqlite3WalFrames( + Wal *pWal, /* Wal handle to write to */ + int szPage, /* Database page-size in bytes */ + PgHdr *pList, /* List of dirty pages to write */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ +){ + int rc; /* Used to catch return codes */ + u32 iFrame; /* Next frame address */ + PgHdr *p; /* Iterator to run through pList with. */ + PgHdr *pLast = 0; /* Last frame in list */ + int nExtra = 0; /* Number of extra copies of last page */ + int szFrame; /* The size of a single frame */ + i64 iOffset; /* Next byte to write in WAL file */ + WalWriter w; /* The writer */ + + assert( pList ); + assert( pWal->writeLock ); + + /* If this frame set completes a transaction, then nTruncate>0. If + ** nTruncate==0 then this frame set does not complete the transaction. */ + assert( (isCommit!=0)==(nTruncate!=0) ); + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) + { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} + WALTRACE(("WAL%p: frame write begin. %d frames. mxFrame=%d. %s\n", + pWal, cnt, pWal->hdr.mxFrame, isCommit ? "Commit" : "Spill")); + } +#endif + + /* See if it is possible to write these frames into the start of the + ** log file, instead of appending to it at pWal->hdr.mxFrame. + */ + if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){ + return rc; + } + + /* If this is the first frame written into the log, write the WAL + ** header to the start of the WAL file. See comments at the top of + ** this source file for a description of the WAL header format. + */ + iFrame = pWal->hdr.mxFrame; + if( iFrame==0 ){ + u8 aWalHdr[WAL_HDRSIZE]; /* Buffer to assemble wal-header in */ + u32 aCksum[2]; /* Checksum for wal-header */ + + sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); + sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); + sqlite3Put4byte(&aWalHdr[8], szPage); + sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); + if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt); + memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); + walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); + sqlite3Put4byte(&aWalHdr[24], aCksum[0]); + sqlite3Put4byte(&aWalHdr[28], aCksum[1]); + + pWal->szPage = szPage; + pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; + pWal->hdr.aFrameCksum[0] = aCksum[0]; + pWal->hdr.aFrameCksum[1] = aCksum[1]; + pWal->truncateOnCommit = 1; + + rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0); + WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok")); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Sync the header (unless SQLITE_IOCAP_SEQUENTIAL is true or unless + ** all syncing is turned off by PRAGMA synchronous=OFF). Otherwise + ** an out-of-order write following a WAL restart could result in + ** database corruption. See the ticket: + ** + ** http://localhost:591/sqlite/info/ff5be73dee + */ + if( pWal->syncHeader && sync_flags ){ + rc = sqlite3OsSync(pWal->pWalFd, sync_flags & SQLITE_SYNC_MASK); + if( rc ) return rc; + } + } + assert( (int)pWal->szPage==szPage ); + + /* Setup information needed to write frames into the WAL */ + w.pWal = pWal; + w.pFd = pWal->pWalFd; + w.iSyncPoint = 0; + w.syncFlags = sync_flags; + w.szPage = szPage; + iOffset = walFrameOffset(iFrame+1, szPage); + szFrame = szPage + WAL_FRAME_HDRSIZE; + + /* Write all frames into the log file exactly once */ + for(p=pList; p; p=p->pDirty){ + int nDbSize; /* 0 normally. Positive == commit flag */ + iFrame++; + assert( iOffset==walFrameOffset(iFrame, szPage) ); + nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; + rc = walWriteOneFrame(&w, p, nDbSize, iOffset); + if( rc ) return rc; + pLast = p; + iOffset += szFrame; + } + + /* If this is the end of a transaction, then we might need to pad + ** the transaction and/or sync the WAL file. + ** + ** Padding and syncing only occur if this set of frames complete a + ** transaction and if PRAGMA synchronous=FULL. If synchronous==NORMAL + ** or synchonous==OFF, then no padding or syncing are needed. + ** + ** If SQLITE_IOCAP_POWERSAFE_OVERWRITE is defined, then padding is not + ** needed and only the sync is done. If padding is needed, then the + ** final frame is repeated (with its commit mark) until the next sector + ** boundary is crossed. Only the part of the WAL prior to the last + ** sector boundary is synced; the part of the last frame that extends + ** past the sector boundary is written after the sync. + */ + if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){ + if( pWal->padToSectorBoundary ){ + int sectorSize = sqlite3SectorSize(pWal->pWalFd); + w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; + while( iOffsettruncateOnCommit && pWal->mxWalSize>=0 ){ + i64 sz = pWal->mxWalSize; + if( walFrameOffset(iFrame+nExtra+1, szPage)>pWal->mxWalSize ){ + sz = walFrameOffset(iFrame+nExtra+1, szPage); + } + walLimitSize(pWal, sz); + pWal->truncateOnCommit = 0; + } + + /* Append data to the wal-index. It is not necessary to lock the + ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index + ** guarantees that there are no other writers, and no data that may + ** be in use by existing readers is being overwritten. + */ + iFrame = pWal->hdr.mxFrame; + for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ + iFrame++; + rc = walIndexAppend(pWal, iFrame, p->pgno); + } + while( rc==SQLITE_OK && nExtra>0 ){ + iFrame++; + nExtra--; + rc = walIndexAppend(pWal, iFrame, pLast->pgno); + } + + if( rc==SQLITE_OK ){ + /* Update the private copy of the header. */ + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + pWal->hdr.mxFrame = iFrame; + if( isCommit ){ + pWal->hdr.iChange++; + pWal->hdr.nPage = nTruncate; + } + /* If this is a commit, update the wal-index header too. */ + if( isCommit ){ + walIndexWriteHdr(pWal); + pWal->iCallback = iFrame; + } + } + + WALTRACE(("WAL%p: frame write %s\n", pWal, rc ? "failed" : "ok")); + return rc; +} + +/* +** This routine is called to implement sqlite3_wal_checkpoint() and +** related interfaces. +** +** Obtain a CHECKPOINT lock and then backfill as much information as +** we can from WAL into the database. +** +** If parameter xBusy is not NULL, it is a pointer to a busy-handler +** callback. In this case this function runs a blocking checkpoint. +*/ +SQLITE_PRIVATE int sqlite3WalCheckpoint( + Wal *pWal, /* Wal connection */ + int eMode, /* PASSIVE, FULL or RESTART */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of temporary buffer */ + u8 *zBuf, /* Temporary buffer to use */ + int *pnLog, /* OUT: Number of frames in WAL */ + int *pnCkpt /* OUT: Number of backfilled frames in WAL */ +){ + int rc; /* Return code */ + int isChanged = 0; /* True if a new wal-index header is loaded */ + int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ + + assert( pWal->ckptLock==0 ); + assert( pWal->writeLock==0 ); + + if( pWal->readOnly ) return SQLITE_READONLY; + WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + if( rc ){ + /* Usually this is SQLITE_BUSY meaning that another thread or process + ** is already running a checkpoint, or maybe a recovery. But it might + ** also be SQLITE_IOERR. */ + return rc; + } + pWal->ckptLock = 1; + + /* If this is a blocking-checkpoint, then obtain the write-lock as well + ** to prevent any writers from running while the checkpoint is underway. + ** This has to be done before the call to walIndexReadHdr() below. + ** + ** If the writer lock cannot be obtained, then a passive checkpoint is + ** run instead. Since the checkpointer is not holding the writer lock, + ** there is no point in blocking waiting for any readers. Assuming no + ** other error occurs, this function will return SQLITE_BUSY to the caller. + */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + rc = SQLITE_OK; + } + } + + /* Read the wal-index header. */ + if( rc==SQLITE_OK ){ + rc = walIndexReadHdr(pWal, &isChanged); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } + } + + /* Copy data from the log to the database file. */ + if( rc==SQLITE_OK ){ + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf); + } + + /* If no error occurred, set the output variables. */ + if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ + if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; + if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); + } + } + + if( isChanged ){ + /* If a new wal-index header was loaded before the checkpoint was + ** performed, then the pager-cache associated with pWal is now + ** out of date. So zero the cached wal-index header to ensure that + ** next time the pager opens a snapshot on this database it knows that + ** the cache needs to be reset. + */ + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + } + + /* Release the locks. */ + sqlite3WalEndWriteTransaction(pWal); + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + pWal->ckptLock = 0; + WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); + return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); +} + +/* Return the value to pass to a sqlite3_wal_hook callback, the +** number of frames in the WAL at the point of the last commit since +** sqlite3WalCallback() was called. If no commits have occurred since +** the last call, then return 0. +*/ +SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal){ + u32 ret = 0; + if( pWal ){ + ret = pWal->iCallback; + pWal->iCallback = 0; + } + return (int)ret; +} + +/* +** This function is called to change the WAL subsystem into or out +** of locking_mode=EXCLUSIVE. +** +** If op is zero, then attempt to change from locking_mode=EXCLUSIVE +** into locking_mode=NORMAL. This means that we must acquire a lock +** on the pWal->readLock byte. If the WAL is already in locking_mode=NORMAL +** or if the acquisition of the lock fails, then return 0. If the +** transition out of exclusive-mode is successful, return 1. This +** operation must occur while the pager is still holding the exclusive +** lock on the main database file. +** +** If op is one, then change from locking_mode=NORMAL into +** locking_mode=EXCLUSIVE. This means that the pWal->readLock must +** be released. Return 1 if the transition is made and 0 if the +** WAL is already in exclusive-locking mode - meaning that this +** routine is a no-op. The pager must already hold the exclusive lock +** on the main database file before invoking this operation. +** +** If op is negative, then do a dry-run of the op==1 case but do +** not actually change anything. The pager uses this to see if it +** should acquire the database exclusive lock prior to invoking +** the op==1 case. +*/ +SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ + int rc; + assert( pWal->writeLock==0 ); + assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 ); + + /* pWal->readLock is usually set, but might be -1 if there was a + ** prior error while attempting to acquire are read-lock. This cannot + ** happen if the connection is actually in exclusive mode (as no xShmLock + ** locks are taken in this case). Nor should the pager attempt to + ** upgrade to exclusive-mode following such an error. + */ + assert( pWal->readLock>=0 || pWal->lockError ); + assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); + + if( op==0 ){ + if( pWal->exclusiveMode ){ + pWal->exclusiveMode = 0; + if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){ + pWal->exclusiveMode = 1; + } + rc = pWal->exclusiveMode==0; + }else{ + /* Already in locking_mode=NORMAL */ + rc = 0; + } + }else if( op>0 ){ + assert( pWal->exclusiveMode==0 ); + assert( pWal->readLock>=0 ); + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + pWal->exclusiveMode = 1; + rc = 1; + }else{ + rc = pWal->exclusiveMode==0; + } + return rc; +} + +/* +** Return true if the argument is non-NULL and the WAL module is using +** heap-memory for the wal-index. Otherwise, if the argument is NULL or the +** WAL module is using shared-memory, return false. +*/ +SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ + return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); +} + +#ifdef SQLITE_ENABLE_ZIPVFS +/* +** If the argument is not NULL, it points to a Wal object that holds a +** read-lock. This function returns the database page-size if it is known, +** or zero if it is not (or if pWal is NULL). +*/ +SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ + assert( pWal==0 || pWal->readLock>=0 ); + return (pWal ? pWal->szPage : 0); +} +#endif + +#endif /* #ifndef SQLITE_OMIT_WAL */ + +/************** End of wal.c *************************************************/ +/************** Begin file btmutex.c *****************************************/ +/* +** 2007 August 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to implement mutexes on Btree objects. +** This code really belongs in btree.c. But btree.c is getting too +** big and we want to break it down some. This packaged seemed like +** a good breakout. +*/ +#ifndef SQLITE_OMIT_SHARED_CACHE +#if SQLITE_THREADSAFE + +/* +** Obtain the BtShared mutex associated with B-Tree handle p. Also, +** set BtShared.db to the database handle associated with p and the +** p->locked boolean to true. +*/ +static void lockBtreeMutex(Btree *p){ + assert( p->locked==0 ); + assert( sqlite3_mutex_notheld(p->pBt->mutex) ); + assert( sqlite3_mutex_held(p->db->mutex) ); + + sqlite3_mutex_enter(p->pBt->mutex); + p->pBt->db = p->db; + p->locked = 1; +} + +/* +** Release the BtShared mutex associated with B-Tree handle p and +** clear the p->locked boolean. +*/ +static void unlockBtreeMutex(Btree *p){ + BtShared *pBt = p->pBt; + assert( p->locked==1 ); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( sqlite3_mutex_held(p->db->mutex) ); + assert( p->db==pBt->db ); + + sqlite3_mutex_leave(pBt->mutex); + p->locked = 0; +} + +/* +** Enter a mutex on the given BTree object. +** +** If the object is not sharable, then no mutex is ever required +** and this routine is a no-op. The underlying mutex is non-recursive. +** But we keep a reference count in Btree.wantToLock so the behavior +** of this interface is recursive. +** +** To avoid deadlocks, multiple Btrees are locked in the same order +** by all database connections. The p->pNext is a list of other +** Btrees belonging to the same database connection as the p Btree +** which need to be locked after p. If we cannot get a lock on +** p, then first unlock all of the others on p->pNext, then wait +** for the lock to become available on p, then relock all of the +** subsequent Btrees that desire a lock. +*/ +SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ + Btree *pLater; + + /* Some basic sanity checking on the Btree. The list of Btrees + ** connected by pNext and pPrev should be in sorted order by + ** Btree.pBt value. All elements of the list should belong to + ** the same connection. Only shared Btrees are on the list. */ + assert( p->pNext==0 || p->pNext->pBt>p->pBt ); + assert( p->pPrev==0 || p->pPrev->pBtpBt ); + assert( p->pNext==0 || p->pNext->db==p->db ); + assert( p->pPrev==0 || p->pPrev->db==p->db ); + assert( p->sharable || (p->pNext==0 && p->pPrev==0) ); + + /* Check for locking consistency */ + assert( !p->locked || p->wantToLock>0 ); + assert( p->sharable || p->wantToLock==0 ); + + /* We should already hold a lock on the database connection */ + assert( sqlite3_mutex_held(p->db->mutex) ); + + /* Unless the database is sharable and unlocked, then BtShared.db + ** should already be set correctly. */ + assert( (p->locked==0 && p->sharable) || p->pBt->db==p->db ); + + if( !p->sharable ) return; + p->wantToLock++; + if( p->locked ) return; + + /* In most cases, we should be able to acquire the lock we + ** want without having to go throught the ascending lock + ** procedure that follows. Just be sure not to block. + */ + if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){ + p->pBt->db = p->db; + p->locked = 1; + return; + } + + /* To avoid deadlock, first release all locks with a larger + ** BtShared address. Then acquire our lock. Then reacquire + ** the other BtShared locks that we used to hold in ascending + ** order. + */ + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + assert( pLater->sharable ); + assert( pLater->pNext==0 || pLater->pNext->pBt>pLater->pBt ); + assert( !pLater->locked || pLater->wantToLock>0 ); + if( pLater->locked ){ + unlockBtreeMutex(pLater); + } + } + lockBtreeMutex(p); + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + if( pLater->wantToLock ){ + lockBtreeMutex(pLater); + } + } +} + +/* +** Exit the recursive mutex on a Btree. +*/ +SQLITE_PRIVATE void sqlite3BtreeLeave(Btree *p){ + if( p->sharable ){ + assert( p->wantToLock>0 ); + p->wantToLock--; + if( p->wantToLock==0 ){ + unlockBtreeMutex(p); + } + } +} + +#ifndef NDEBUG +/* +** Return true if the BtShared mutex is held on the btree, or if the +** B-Tree is not marked as sharable. +** +** This routine is used only from within assert() statements. +*/ +SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ + assert( p->sharable==0 || p->locked==0 || p->wantToLock>0 ); + assert( p->sharable==0 || p->locked==0 || p->db==p->pBt->db ); + assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->pBt->mutex) ); + assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->db->mutex) ); + + return (p->sharable==0 || p->locked); +} +#endif + + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Enter and leave a mutex on a Btree given a cursor owned by that +** Btree. These entry points are used by incremental I/O and can be +** omitted if that module is not used. +*/ +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ + sqlite3BtreeEnter(pCur->pBtree); +} +SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ + sqlite3BtreeLeave(pCur->pBtree); +} +#endif /* SQLITE_OMIT_INCRBLOB */ + + +/* +** Enter the mutex on every Btree associated with a database +** connection. This is needed (for example) prior to parsing +** a statement since we will be comparing table and column names +** against all schemas and we do not want those schemas being +** reset out from under us. +** +** There is a corresponding leave-all procedures. +** +** Enter the mutexes in accending order by BtShared pointer address +** to avoid the possibility of deadlock when two threads with +** two or more btrees in common both try to lock all their btrees +** at the same instant. +*/ +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ + int i; + Btree *p; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p ) sqlite3BtreeEnter(p); + } +} +SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ + int i; + Btree *p; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p ) sqlite3BtreeLeave(p); + } +} + +/* +** Return true if a particular Btree requires a lock. Return FALSE if +** no lock is ever required since it is not sharable. +*/ +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ + return p->sharable; +} + +#ifndef NDEBUG +/* +** Return true if the current thread holds the database connection +** mutex and all required BtShared mutexes. +** +** This routine is used inside assert() statements only. +*/ +SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ + int i; + if( !sqlite3_mutex_held(db->mutex) ){ + return 0; + } + for(i=0; inDb; i++){ + Btree *p; + p = db->aDb[i].pBt; + if( p && p->sharable && + (p->wantToLock==0 || !sqlite3_mutex_held(p->pBt->mutex)) ){ + return 0; + } + } + return 1; +} +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Return true if the correct mutexes are held for accessing the +** db->aDb[iDb].pSchema structure. The mutexes required for schema +** access are: +** +** (1) The mutex on db +** (2) if iDb!=1, then the mutex on db->aDb[iDb].pBt. +** +** If pSchema is not NULL, then iDb is computed from pSchema and +** db using sqlite3SchemaToIndex(). +*/ +SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){ + Btree *p; + assert( db!=0 ); + if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema); + assert( iDb>=0 && iDbnDb ); + if( !sqlite3_mutex_held(db->mutex) ) return 0; + if( iDb==1 ) return 1; + p = db->aDb[iDb].pBt; + assert( p!=0 ); + return p->sharable==0 || p->locked==1; +} +#endif /* NDEBUG */ + +#else /* SQLITE_THREADSAFE>0 above. SQLITE_THREADSAFE==0 below */ +/* +** The following are special cases for mutex enter routines for use +** in single threaded applications that use shared cache. Except for +** these two routines, all mutex operations are no-ops in that case and +** are null #defines in btree.h. +** +** If shared cache is disabled, then all btree mutex routines, including +** the ones below, are no-ops and are null #defines in btree.h. +*/ + +SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ + p->pBt->db = p->db; +} +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ + int i; + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + p->pBt->db = p->db; + } + } +} +#endif /* if SQLITE_THREADSAFE */ +#endif /* ifndef SQLITE_OMIT_SHARED_CACHE */ + +/************** End of btmutex.c *********************************************/ +/************** Begin file btree.c *******************************************/ +/* +** 2004 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a external (disk-based) database using BTrees. +** See the header comment on "btreeInt.h" for additional information. +** Including a description of file format and an overview of operation. +*/ + +/* +** The header string that appears at the beginning of every +** SQLite database. +*/ +static const char zMagicHeader[] = SQLITE_FILE_HEADER; + +/* +** Set this global variable to 1 to enable tracing using the TRACE +** macro. +*/ +#if 0 +int sqlite3BtreeTrace=1; /* True to enable tracing */ +# define TRACE(X) if(sqlite3BtreeTrace){printf X;fflush(stdout);} +#else +# define TRACE(X) +#endif + +/* +** Extract a 2-byte big-endian integer from an array of unsigned bytes. +** But if the value is zero, make it 65536. +** +** This routine is used to extract the "offset to cell content area" value +** from the header of a btree page. If the page size is 65536 and the page +** is empty, the offset should be 65536, but the 2-byte value stores zero. +** This routine makes the necessary adjustment to 65536. +*/ +#define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) + +/* +** Values passed as the 5th argument to allocateBtreePage() +*/ +#define BTALLOC_ANY 0 /* Allocate any page */ +#define BTALLOC_EXACT 1 /* Allocate exact page if possible */ +#define BTALLOC_LE 2 /* Allocate any page <= the parameter */ + +/* +** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not +** defined, or 0 if it is. For example: +** +** bIncrVacuum = IfNotOmitAV(pBtShared->incrVacuum); +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define IfNotOmitAV(expr) (expr) +#else +#define IfNotOmitAV(expr) 0 +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** A list of BtShared objects that are eligible for participation +** in shared cache. This variable has file scope during normal builds, +** but the test harness needs to access it so we make it global for +** test builds. +** +** Access to this variable is protected by SQLITE_MUTEX_STATIC_MASTER. +*/ +#ifdef SQLITE_TEST +SQLITE_PRIVATE BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; +#else +static BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; +#endif +#endif /* SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Enable or disable the shared pager and schema features. +** +** This routine has no effect on existing database connections. +** The shared cache setting effects only future calls to +** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). +*/ +SQLITE_API int sqlite3_enable_shared_cache(int enable){ + sqlite3GlobalConfig.sharedCacheEnabled = enable; + return SQLITE_OK; +} +#endif + + + +#ifdef SQLITE_OMIT_SHARED_CACHE + /* + ** The functions querySharedCacheTableLock(), setSharedCacheTableLock(), + ** and clearAllSharedCacheTableLocks() + ** manipulate entries in the BtShared.pLock linked list used to store + ** shared-cache table level locks. If the library is compiled with the + ** shared-cache feature disabled, then there is only ever one user + ** of each BtShared structure and so this locking is not necessary. + ** So define the lock related functions as no-ops. + */ + #define querySharedCacheTableLock(a,b,c) SQLITE_OK + #define setSharedCacheTableLock(a,b,c) SQLITE_OK + #define clearAllSharedCacheTableLocks(a) + #define downgradeAllSharedCacheTableLocks(a) + #define hasSharedCacheTableLock(a,b,c,d) 1 + #define hasReadConflicts(a, b) 0 +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE + +#ifdef SQLITE_DEBUG +/* +**** This function is only used as part of an assert() statement. *** +** +** Check to see if pBtree holds the required locks to read or write to the +** table with root page iRoot. Return 1 if it does and 0 if not. +** +** For example, when writing to a table with root-page iRoot via +** Btree connection pBtree: +** +** assert( hasSharedCacheTableLock(pBtree, iRoot, 0, WRITE_LOCK) ); +** +** When writing to an index that resides in a sharable database, the +** caller should have first obtained a lock specifying the root page of +** the corresponding table. This makes things a bit more complicated, +** as this module treats each table as a separate structure. To determine +** the table corresponding to the index being written, this +** function has to search through the database schema. +** +** Instead of a lock on the table/index rooted at page iRoot, the caller may +** hold a write-lock on the schema table (root page 1). This is also +** acceptable. +*/ +static int hasSharedCacheTableLock( + Btree *pBtree, /* Handle that must hold lock */ + Pgno iRoot, /* Root page of b-tree */ + int isIndex, /* True if iRoot is the root of an index b-tree */ + int eLockType /* Required lock type (READ_LOCK or WRITE_LOCK) */ +){ + Schema *pSchema = (Schema *)pBtree->pBt->pSchema; + Pgno iTab = 0; + BtLock *pLock; + + /* If this database is not shareable, or if the client is reading + ** and has the read-uncommitted flag set, then no lock is required. + ** Return true immediately. + */ + if( (pBtree->sharable==0) + || (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommitted)) + ){ + return 1; + } + + /* If the client is reading or writing an index and the schema is + ** not loaded, then it is too difficult to actually check to see if + ** the correct locks are held. So do not bother - just return true. + ** This case does not come up very often anyhow. + */ + if( isIndex && (!pSchema || (pSchema->schemaFlags&DB_SchemaLoaded)==0) ){ + return 1; + } + + /* Figure out the root-page that the lock should be held on. For table + ** b-trees, this is just the root page of the b-tree being read or + ** written. For index b-trees, it is the root page of the associated + ** table. */ + if( isIndex ){ + HashElem *p; + for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){ + Index *pIdx = (Index *)sqliteHashData(p); + if( pIdx->tnum==(int)iRoot ){ + iTab = pIdx->pTable->tnum; + } + } + }else{ + iTab = iRoot; + } + + /* Search for the required lock. Either a write-lock on root-page iTab, a + ** write-lock on the schema table, or (if the client is reading) a + ** read-lock on iTab will suffice. Return 1 if any of these are found. */ + for(pLock=pBtree->pBt->pLock; pLock; pLock=pLock->pNext){ + if( pLock->pBtree==pBtree + && (pLock->iTable==iTab || (pLock->eLock==WRITE_LOCK && pLock->iTable==1)) + && pLock->eLock>=eLockType + ){ + return 1; + } + } + + /* Failed to find the required lock. */ + return 0; +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +**** This function may be used as part of assert() statements only. **** +** +** Return true if it would be illegal for pBtree to write into the +** table or index rooted at iRoot because other shared connections are +** simultaneously reading that same table or index. +** +** It is illegal for pBtree to write if some other Btree object that +** shares the same BtShared object is currently reading or writing +** the iRoot table. Except, if the other Btree object has the +** read-uncommitted flag set, then it is OK for the other object to +** have a read cursor. +** +** For example, before writing to any part of the table or index +** rooted at page iRoot, one should call: +** +** assert( !hasReadConflicts(pBtree, iRoot) ); +*/ +static int hasReadConflicts(Btree *pBtree, Pgno iRoot){ + BtCursor *p; + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( p->pgnoRoot==iRoot + && p->pBtree!=pBtree + && 0==(p->pBtree->db->flags & SQLITE_ReadUncommitted) + ){ + return 1; + } + } + return 0; +} +#endif /* #ifdef SQLITE_DEBUG */ + +/* +** Query to see if Btree handle p may obtain a lock of type eLock +** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return +** SQLITE_OK if the lock may be obtained (by calling +** setSharedCacheTableLock()), or SQLITE_LOCKED if not. +*/ +static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ + BtShared *pBt = p->pBt; + BtLock *pIter; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); + assert( !(p->db->flags&SQLITE_ReadUncommitted)||eLock==WRITE_LOCK||iTab==1 ); + + /* If requesting a write-lock, then the Btree must have an open write + ** transaction on this file. And, obviously, for this to be so there + ** must be an open write transaction on the file itself. + */ + assert( eLock==READ_LOCK || (p==pBt->pWriter && p->inTrans==TRANS_WRITE) ); + assert( eLock==READ_LOCK || pBt->inTransaction==TRANS_WRITE ); + + /* This routine is a no-op if the shared-cache is not enabled */ + if( !p->sharable ){ + return SQLITE_OK; + } + + /* If some other connection is holding an exclusive lock, the + ** requested lock may not be obtained. + */ + if( pBt->pWriter!=p && (pBt->btsFlags & BTS_EXCLUSIVE)!=0 ){ + sqlite3ConnectionBlocked(p->db, pBt->pWriter->db); + return SQLITE_LOCKED_SHAREDCACHE; + } + + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + /* The condition (pIter->eLock!=eLock) in the following if(...) + ** statement is a simplification of: + ** + ** (eLock==WRITE_LOCK || pIter->eLock==WRITE_LOCK) + ** + ** since we know that if eLock==WRITE_LOCK, then no other connection + ** may hold a WRITE_LOCK on any table in this file (since there can + ** only be a single writer). + */ + assert( pIter->eLock==READ_LOCK || pIter->eLock==WRITE_LOCK ); + assert( eLock==READ_LOCK || pIter->pBtree==p || pIter->eLock==READ_LOCK); + if( pIter->pBtree!=p && pIter->iTable==iTab && pIter->eLock!=eLock ){ + sqlite3ConnectionBlocked(p->db, pIter->pBtree->db); + if( eLock==WRITE_LOCK ){ + assert( p==pBt->pWriter ); + pBt->btsFlags |= BTS_PENDING; + } + return SQLITE_LOCKED_SHAREDCACHE; + } + } + return SQLITE_OK; +} +#endif /* !SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Add a lock on the table with root-page iTable to the shared-btree used +** by Btree handle p. Parameter eLock must be either READ_LOCK or +** WRITE_LOCK. +** +** This function assumes the following: +** +** (a) The specified Btree object p is connected to a sharable +** database (one with the BtShared.sharable flag set), and +** +** (b) No other Btree objects hold a lock that conflicts +** with the requested lock (i.e. querySharedCacheTableLock() has +** already been called and returned SQLITE_OK). +** +** SQLITE_OK is returned if the lock is added successfully. SQLITE_NOMEM +** is returned if a malloc attempt fails. +*/ +static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ + BtShared *pBt = p->pBt; + BtLock *pLock = 0; + BtLock *pIter; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); + + /* A connection with the read-uncommitted flag set will never try to + ** obtain a read-lock using this function. The only read-lock obtained + ** by a connection in read-uncommitted mode is on the sqlite_master + ** table, and that lock is obtained in BtreeBeginTrans(). */ + assert( 0==(p->db->flags&SQLITE_ReadUncommitted) || eLock==WRITE_LOCK ); + + /* This function should only be called on a sharable b-tree after it + ** has been determined that no other b-tree holds a conflicting lock. */ + assert( p->sharable ); + assert( SQLITE_OK==querySharedCacheTableLock(p, iTable, eLock) ); + + /* First search the list for an existing lock on this table. */ + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->iTable==iTable && pIter->pBtree==p ){ + pLock = pIter; + break; + } + } + + /* If the above search did not find a BtLock struct associating Btree p + ** with table iTable, allocate one and link it into the list. + */ + if( !pLock ){ + pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); + if( !pLock ){ + return SQLITE_NOMEM; + } + pLock->iTable = iTable; + pLock->pBtree = p; + pLock->pNext = pBt->pLock; + pBt->pLock = pLock; + } + + /* Set the BtLock.eLock variable to the maximum of the current lock + ** and the requested lock. This means if a write-lock was already held + ** and a read-lock requested, we don't incorrectly downgrade the lock. + */ + assert( WRITE_LOCK>READ_LOCK ); + if( eLock>pLock->eLock ){ + pLock->eLock = eLock; + } + + return SQLITE_OK; +} +#endif /* !SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Release all the table locks (locks obtained via calls to +** the setSharedCacheTableLock() procedure) held by Btree object p. +** +** This function assumes that Btree p has an open read or write +** transaction. If it does not, then the BTS_PENDING flag +** may be incorrectly cleared. +*/ +static void clearAllSharedCacheTableLocks(Btree *p){ + BtShared *pBt = p->pBt; + BtLock **ppIter = &pBt->pLock; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->sharable || 0==*ppIter ); + assert( p->inTrans>0 ); + + while( *ppIter ){ + BtLock *pLock = *ppIter; + assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); + assert( pLock->pBtree->inTrans>=pLock->eLock ); + if( pLock->pBtree==p ){ + *ppIter = pLock->pNext; + assert( pLock->iTable!=1 || pLock==&p->lock ); + if( pLock->iTable!=1 ){ + sqlite3_free(pLock); + } + }else{ + ppIter = &pLock->pNext; + } + } + + assert( (pBt->btsFlags & BTS_PENDING)==0 || pBt->pWriter ); + if( pBt->pWriter==p ){ + pBt->pWriter = 0; + pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); + }else if( pBt->nTransaction==2 ){ + /* This function is called when Btree p is concluding its + ** transaction. If there currently exists a writer, and p is not + ** that writer, then the number of locks held by connections other + ** than the writer must be about to drop to zero. In this case + ** set the BTS_PENDING flag to 0. + ** + ** If there is not currently a writer, then BTS_PENDING must + ** be zero already. So this next line is harmless in that case. + */ + pBt->btsFlags &= ~BTS_PENDING; + } +} + +/* +** This function changes all write-locks held by Btree p into read-locks. +*/ +static void downgradeAllSharedCacheTableLocks(Btree *p){ + BtShared *pBt = p->pBt; + if( pBt->pWriter==p ){ + BtLock *pLock; + pBt->pWriter = 0; + pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + assert( pLock->eLock==READ_LOCK || pLock->pBtree==p ); + pLock->eLock = READ_LOCK; + } + } +} + +#endif /* SQLITE_OMIT_SHARED_CACHE */ + +static void releasePage(MemPage *pPage); /* Forward reference */ + +/* +***** This routine is used inside of assert() only **** +** +** Verify that the cursor holds the mutex on its BtShared +*/ +#ifdef SQLITE_DEBUG +static int cursorHoldsMutex(BtCursor *p){ + return sqlite3_mutex_held(p->pBt->mutex); +} +#endif + +/* +** Invalidate the overflow cache of the cursor passed as the first argument. +** on the shared btree structure pBt. +*/ +#define invalidateOverflowCache(pCur) (pCur->curFlags &= ~BTCF_ValidOvfl) + +/* +** Invalidate the overflow page-list cache for all cursors opened +** on the shared btree structure pBt. +*/ +static void invalidateAllOverflowCache(BtShared *pBt){ + BtCursor *p; + assert( sqlite3_mutex_held(pBt->mutex) ); + for(p=pBt->pCursor; p; p=p->pNext){ + invalidateOverflowCache(p); + } +} + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** This function is called before modifying the contents of a table +** to invalidate any incrblob cursors that are open on the +** row or one of the rows being modified. +** +** If argument isClearTable is true, then the entire contents of the +** table is about to be deleted. In this case invalidate all incrblob +** cursors open on any row within the table with root-page pgnoRoot. +** +** Otherwise, if argument isClearTable is false, then the row with +** rowid iRow is being replaced or deleted. In this case invalidate +** only those incrblob cursors open on that specific row. +*/ +static void invalidateIncrblobCursors( + Btree *pBtree, /* The database file to check */ + i64 iRow, /* The rowid that might be changing */ + int isClearTable /* True if all rows are being deleted */ +){ + BtCursor *p; + BtShared *pBt = pBtree->pBt; + assert( sqlite3BtreeHoldsMutex(pBtree) ); + for(p=pBt->pCursor; p; p=p->pNext){ + if( (p->curFlags & BTCF_Incrblob)!=0 && (isClearTable || p->info.nKey==iRow) ){ + p->eState = CURSOR_INVALID; + } + } +} + +#else + /* Stub function when INCRBLOB is omitted */ + #define invalidateIncrblobCursors(x,y,z) +#endif /* SQLITE_OMIT_INCRBLOB */ + +/* +** Set bit pgno of the BtShared.pHasContent bitvec. This is called +** when a page that previously contained data becomes a free-list leaf +** page. +** +** The BtShared.pHasContent bitvec exists to work around an obscure +** bug caused by the interaction of two useful IO optimizations surrounding +** free-list leaf pages: +** +** 1) When all data is deleted from a page and the page becomes +** a free-list leaf page, the page is not written to the database +** (as free-list leaf pages contain no meaningful data). Sometimes +** such a page is not even journalled (as it will not be modified, +** why bother journalling it?). +** +** 2) When a free-list leaf page is reused, its content is not read +** from the database or written to the journal file (why should it +** be, if it is not at all meaningful?). +** +** By themselves, these optimizations work fine and provide a handy +** performance boost to bulk delete or insert operations. However, if +** a page is moved to the free-list and then reused within the same +** transaction, a problem comes up. If the page is not journalled when +** it is moved to the free-list and it is also not journalled when it +** is extracted from the free-list and reused, then the original data +** may be lost. In the event of a rollback, it may not be possible +** to restore the database to its original configuration. +** +** The solution is the BtShared.pHasContent bitvec. Whenever a page is +** moved to become a free-list leaf page, the corresponding bit is +** set in the bitvec. Whenever a leaf page is extracted from the free-list, +** optimization 2 above is omitted if the corresponding bit is already +** set in BtShared.pHasContent. The contents of the bitvec are cleared +** at the end of every transaction. +*/ +static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ + int rc = SQLITE_OK; + if( !pBt->pHasContent ){ + assert( pgno<=pBt->nPage ); + pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage); + if( !pBt->pHasContent ){ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){ + rc = sqlite3BitvecSet(pBt->pHasContent, pgno); + } + return rc; +} + +/* +** Query the BtShared.pHasContent vector. +** +** This function is called when a free-list leaf page is removed from the +** free-list for reuse. It returns false if it is safe to retrieve the +** page from the pager layer with the 'no-content' flag set. True otherwise. +*/ +static int btreeGetHasContent(BtShared *pBt, Pgno pgno){ + Bitvec *p = pBt->pHasContent; + return (p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTest(p, pgno))); +} + +/* +** Clear (destroy) the BtShared.pHasContent bitvec. This should be +** invoked at the conclusion of each write-transaction. +*/ +static void btreeClearHasContent(BtShared *pBt){ + sqlite3BitvecDestroy(pBt->pHasContent); + pBt->pHasContent = 0; +} + +/* +** Release all of the apPage[] pages for a cursor. +*/ +static void btreeReleaseAllCursorPages(BtCursor *pCur){ + int i; + for(i=0; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + pCur->apPage[i] = 0; + } + pCur->iPage = -1; +} + + +/* +** Save the current cursor position in the variables BtCursor.nKey +** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. +** +** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) +** prior to calling this routine. +*/ +static int saveCursorPosition(BtCursor *pCur){ + int rc; + + assert( CURSOR_VALID==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); + assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ + + /* If this is an intKey table, then the above call to BtreeKeySize() + ** stores the integer key in pCur->nKey. In this case this value is + ** all that is required. Otherwise, if pCur is not open on an intKey + ** table, then malloc space for and store the pCur->nKey bytes of key + ** data. + */ + if( 0==pCur->apPage[0]->intKey ){ + void *pKey = sqlite3Malloc( (int)pCur->nKey ); + if( pKey ){ + rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); + if( rc==SQLITE_OK ){ + pCur->pKey = pKey; + }else{ + sqlite3_free(pKey); + } + }else{ + rc = SQLITE_NOMEM; + } + } + assert( !pCur->apPage[0]->intKey || !pCur->pKey ); + + if( rc==SQLITE_OK ){ + btreeReleaseAllCursorPages(pCur); + pCur->eState = CURSOR_REQUIRESEEK; + } + + invalidateOverflowCache(pCur); + return rc; +} + +/* +** Save the positions of all cursors (except pExcept) that are open on +** the table with root-page iRoot. Usually, this is called just before cursor +** pExcept is used to modify the table (BtreeDelete() or BtreeInsert()). +*/ +static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ + BtCursor *p; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pExcept==0 || pExcept->pBt==pBt ); + for(p=pBt->pCursor; p; p=p->pNext){ + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ + if( p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( SQLITE_OK!=rc ){ + return rc; + } + }else{ + testcase( p->iPage>0 ); + btreeReleaseAllCursorPages(p); + } + } + } + return SQLITE_OK; +} + +/* +** Clear the current cursor position. +*/ +SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + sqlite3_free(pCur->pKey); + pCur->pKey = 0; + pCur->eState = CURSOR_INVALID; +} + +/* +** In this version of BtreeMoveto, pKey is a packed index record +** such as is generated by the OP_MakeRecord opcode. Unpack the +** record and then call BtreeMovetoUnpacked() to do the work. +*/ +static int btreeMoveto( + BtCursor *pCur, /* Cursor open on the btree to be searched */ + const void *pKey, /* Packed key if the btree is an index */ + i64 nKey, /* Integer key for tables. Size of pKey for indices */ + int bias, /* Bias search to the high end */ + int *pRes /* Write search results here */ +){ + int rc; /* Status code */ + UnpackedRecord *pIdxKey; /* Unpacked index key */ + char aSpace[200]; /* Temp space for pIdxKey - to avoid a malloc */ + char *pFree = 0; + + if( pKey ){ + assert( nKey==(i64)(int)nKey ); + pIdxKey = sqlite3VdbeAllocUnpackedRecord( + pCur->pKeyInfo, aSpace, sizeof(aSpace), &pFree + ); + if( pIdxKey==0 ) return SQLITE_NOMEM; + sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); + if( pIdxKey->nField==0 ){ + sqlite3DbFree(pCur->pKeyInfo->db, pFree); + return SQLITE_CORRUPT_BKPT; + } + }else{ + pIdxKey = 0; + } + rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); + if( pFree ){ + sqlite3DbFree(pCur->pKeyInfo->db, pFree); + } + return rc; +} + +/* +** Restore the cursor to the position it was in (or as close to as possible) +** when saveCursorPosition() was called. Note that this call deletes the +** saved position info stored by saveCursorPosition(), so there can be +** at most one effective restoreCursorPosition() call after each +** saveCursorPosition(). +*/ +static int btreeRestoreCursorPosition(BtCursor *pCur){ + int rc; + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState>=CURSOR_REQUIRESEEK ); + if( pCur->eState==CURSOR_FAULT ){ + return pCur->skipNext; + } + pCur->eState = CURSOR_INVALID; + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &pCur->skipNext); + if( rc==SQLITE_OK ){ + sqlite3_free(pCur->pKey); + pCur->pKey = 0; + assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); + if( pCur->skipNext && pCur->eState==CURSOR_VALID ){ + pCur->eState = CURSOR_SKIPNEXT; + } + } + return rc; +} + +#define restoreCursorPosition(p) \ + (p->eState>=CURSOR_REQUIRESEEK ? \ + btreeRestoreCursorPosition(p) : \ + SQLITE_OK) + +/* +** Determine whether or not a cursor has moved from the position it +** was last placed at. Cursors can move when the row they are pointing +** at is deleted out from under them. +** +** This routine returns an error code if something goes wrong. The +** integer *pHasMoved is set as follows: +** +** 0: The cursor is unchanged +** 1: The cursor is still pointing at the same row, but the pointers +** returned by sqlite3BtreeKeyFetch() or sqlite3BtreeDataFetch() +** might now be invalid because of a balance() or other change to the +** b-tree. +** 2: The cursor is no longer pointing to the row. The row might have +** been deleted out from under the cursor. +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ + int rc; + + if( pCur->eState==CURSOR_VALID ){ + *pHasMoved = 0; + return SQLITE_OK; + } + rc = restoreCursorPosition(pCur); + if( rc ){ + *pHasMoved = 2; + return rc; + } + if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){ + *pHasMoved = 2; + }else{ + *pHasMoved = 1; + } + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Given a page number of a regular database page, return the page +** number for the pointer-map page that contains the entry for the +** input page number. +** +** Return 0 (not a valid page) for pgno==1 since there is +** no pointer map associated with page 1. The integrity_check logic +** requires that ptrmapPageno(*,1)!=1. +*/ +static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ + int nPagesPerMapPage; + Pgno iPtrMap, ret; + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno<2 ) return 0; + nPagesPerMapPage = (pBt->usableSize/5)+1; + iPtrMap = (pgno-2)/nPagesPerMapPage; + ret = (iPtrMap*nPagesPerMapPage) + 2; + if( ret==PENDING_BYTE_PAGE(pBt) ){ + ret++; + } + return ret; +} + +/* +** Write an entry into the pointer map. +** +** This routine updates the pointer map entry for page number 'key' +** so that it maps to type 'eType' and parent page number 'pgno'. +** +** If *pRC is initially non-zero (non-SQLITE_OK) then this routine is +** a no-op. If an error occurs, the appropriate error code is written +** into *pRC. +*/ +static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ + DbPage *pDbPage; /* The pointer map page */ + u8 *pPtrmap; /* The pointer map data */ + Pgno iPtrmap; /* The pointer map page number */ + int offset; /* Offset in pointer map page */ + int rc; /* Return code from subfunctions */ + + if( *pRC ) return; + + assert( sqlite3_mutex_held(pBt->mutex) ); + /* The master-journal page number must never be used as a pointer map page */ + assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); + + assert( pBt->autoVacuum ); + if( key==0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } + assert( offset <= (int)pBt->usableSize-5 ); + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); + + if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ + TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + *pRC= rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK ){ + pPtrmap[offset] = eType; + put4byte(&pPtrmap[offset+1], parent); + } + } + +ptrmap_exit: + sqlite3PagerUnref(pDbPage); +} + +/* +** Read an entry from the pointer map. +** +** This routine retrieves the pointer map entry for page 'key', writing +** the type and parent page number to *pEType and *pPgno respectively. +** An error code is returned if something goes wrong, otherwise SQLITE_OK. +*/ +static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ + DbPage *pDbPage; /* The pointer map page */ + int iPtrmap; /* Pointer map page index */ + u8 *pPtrmap; /* Pointer map page data */ + int offset; /* Offset of entry in pointer map */ + int rc; + + assert( sqlite3_mutex_held(pBt->mutex) ); + + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + if( rc!=0 ){ + return rc; + } + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); + + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + sqlite3PagerUnref(pDbPage); + return SQLITE_CORRUPT_BKPT; + } + assert( offset <= (int)pBt->usableSize-5 ); + assert( pEType!=0 ); + *pEType = pPtrmap[offset]; + if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); + + sqlite3PagerUnref(pDbPage); + if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_BKPT; + return SQLITE_OK; +} + +#else /* if defined SQLITE_OMIT_AUTOVACUUM */ + #define ptrmapPut(w,x,y,z,rc) + #define ptrmapGet(w,x,y,z) SQLITE_OK + #define ptrmapPutOvflPtr(x, y, rc) +#endif + +/* +** Given a btree page and a cell index (0 means the first cell on +** the page, 1 means the second cell, and so forth) return a pointer +** to the cell content. +** +** This routine works only for pages that do not contain overflow cells. +*/ +#define findCell(P,I) \ + ((P)->aData + ((P)->maskPage & get2byte(&(P)->aCellIdx[2*(I)]))) +#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) + + +/* +** This a more complex version of findCell() that works for +** pages that do contain overflow cells. +*/ +static u8 *findOverflowCell(MemPage *pPage, int iCell){ + int i; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + for(i=pPage->nOverflow-1; i>=0; i--){ + int k; + k = pPage->aiOvfl[i]; + if( k<=iCell ){ + if( k==iCell ){ + return pPage->apOvfl[i]; + } + iCell--; + } + } + return findCell(pPage, iCell); +} + +/* +** Parse a cell content block and fill in the CellInfo structure. There +** are two versions of this function. btreeParseCell() takes a +** cell index as the second argument and btreeParseCellPtr() +** takes a pointer to the body of the cell as its second argument. +** +** Within this file, the parseCell() macro can be called instead of +** btreeParseCellPtr(). Using some compilers, this will be faster. +*/ +static void btreeParseCellPtr( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + u16 n; /* Number bytes in cell content header */ + u32 nPayload; /* Number of bytes of cell payload */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + + pInfo->pCell = pCell; + assert( pPage->leaf==0 || pPage->leaf==1 ); + n = pPage->childPtrSize; + assert( n==4-4*pPage->leaf ); + if( pPage->intKey ){ + if( pPage->hasData ){ + assert( n==0 ); + n = getVarint32(pCell, nPayload); + }else{ + nPayload = 0; + } + n += getVarint(&pCell[n], (u64*)&pInfo->nKey); + pInfo->nData = nPayload; + }else{ + pInfo->nData = 0; + n += getVarint32(&pCell[n], nPayload); + pInfo->nKey = nPayload; + } + pInfo->nPayload = nPayload; + pInfo->nHeader = n; + testcase( nPayload==pPage->maxLocal ); + testcase( nPayload==pPage->maxLocal+1 ); + if( likely(nPayload<=pPage->maxLocal) ){ + /* This is the (easy) common case where the entire payload fits + ** on the local page. No overflow is required. + */ + if( (pInfo->nSize = (u16)(n+nPayload))<4 ) pInfo->nSize = 4; + pInfo->nLocal = (u16)nPayload; + pInfo->iOverflow = 0; + }else{ + /* If the payload will not fit completely on the local page, we have + ** to decide how much to store locally and how much to spill onto + ** overflow pages. The strategy is to minimize the amount of unused + ** space on overflow pages while keeping the amount of local storage + ** in between minLocal and maxLocal. + ** + ** Warning: changing the way overflow payload is distributed in any + ** way will result in an incompatible file format. + */ + int minLocal; /* Minimum amount of payload held locally */ + int maxLocal; /* Maximum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + + minLocal = pPage->minLocal; + maxLocal = pPage->maxLocal; + surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize - 4); + testcase( surplus==maxLocal ); + testcase( surplus==maxLocal+1 ); + if( surplus <= maxLocal ){ + pInfo->nLocal = (u16)surplus; + }else{ + pInfo->nLocal = (u16)minLocal; + } + pInfo->iOverflow = (u16)(pInfo->nLocal + n); + pInfo->nSize = pInfo->iOverflow + 4; + } +} +#define parseCell(pPage, iCell, pInfo) \ + btreeParseCellPtr((pPage), findCell((pPage), (iCell)), (pInfo)) +static void btreeParseCell( + MemPage *pPage, /* Page containing the cell */ + int iCell, /* The cell index. First cell is 0 */ + CellInfo *pInfo /* Fill in this structure */ +){ + parseCell(pPage, iCell, pInfo); +} + +/* +** Compute the total number of bytes that a Cell needs in the cell +** data area of the btree-page. The return number includes the cell +** data header and the local payload, but not any overflow page or +** the space used by the cell pointer. +*/ +static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ + u8 *pIter = &pCell[pPage->childPtrSize]; + u32 nSize; + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + btreeParseCellPtr(pPage, pCell, &debuginfo); +#endif + + if( pPage->intKey ){ + u8 *pEnd; + if( pPage->hasData ){ + pIter += getVarint32(pIter, nSize); + }else{ + nSize = 0; + } + + /* pIter now points at the 64-bit integer key value, a variable length + ** integer. The following block moves pIter to point at the first byte + ** past the end of the key value. */ + pEnd = &pIter[9]; + while( (*pIter++)&0x80 && pItermaxLocal ); + testcase( nSize==pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4; + } + nSize += (u32)(pIter - pCell); + + /* The minimum size of any cell is 4 bytes. */ + if( nSize<4 ){ + nSize = 4; + } + + assert( nSize==debuginfo.nSize ); + return (u16)nSize; +} + +#ifdef SQLITE_DEBUG +/* This variation on cellSizePtr() is used inside of assert() statements +** only. */ +static u16 cellSize(MemPage *pPage, int iCell){ + return cellSizePtr(pPage, findCell(pPage, iCell)); +} +#endif + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** If the cell pCell, part of page pPage contains a pointer +** to an overflow page, insert an entry into the pointer-map +** for the overflow page. +*/ +static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ + CellInfo info; + if( *pRC ) return; + assert( pCell!=0 ); + btreeParseCellPtr(pPage, pCell, &info); + assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); + if( info.iOverflow ){ + Pgno ovfl = get4byte(&pCell[info.iOverflow]); + ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); + } +} +#endif + + +/* +** Defragment the page given. All Cells are moved to the +** end of the page and all free space is collected into one +** big FreeBlk that occurs in between the header and cell +** pointer array and the cell content area. +*/ +static int defragmentPage(MemPage *pPage){ + int i; /* Loop counter */ + int pc; /* Address of a i-th cell */ + int hdr; /* Offset to the page header */ + int size; /* Size of a cell */ + int usableSize; /* Number of usable bytes on a page */ + int cellOffset; /* Offset to the cell pointer array */ + int cbrk; /* Offset to the cell content area */ + int nCell; /* Number of cells on the page */ + unsigned char *data; /* The page data */ + unsigned char *temp; /* Temp area for cell content */ + int iCellFirst; /* First allowable cell index */ + int iCellLast; /* Last possible cell index */ + + + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( pPage->pBt!=0 ); + assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); + assert( pPage->nOverflow==0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + data = pPage->aData; + hdr = pPage->hdrOffset; + cellOffset = pPage->cellOffset; + nCell = pPage->nCell; + assert( nCell==get2byte(&data[hdr+3]) ); + usableSize = pPage->pBt->usableSize; + cbrk = get2byte(&data[hdr+5]); + memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); + cbrk = usableSize; + iCellFirst = cellOffset + 2*nCell; + iCellLast = usableSize - 4; + for(i=0; iiCellLast ){ + return SQLITE_CORRUPT_BKPT; + } +#endif + assert( pc>=iCellFirst && pc<=iCellLast ); + size = cellSizePtr(pPage, &temp[pc]); + cbrk -= size; +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + if( cbrkusableSize ){ + return SQLITE_CORRUPT_BKPT; + } +#endif + assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); + testcase( cbrk+size==usableSize ); + testcase( pc+size==usableSize ); + memcpy(&data[cbrk], &temp[pc], size); + put2byte(pAddr, cbrk); + } + assert( cbrk>=iCellFirst ); + put2byte(&data[hdr+5], cbrk); + data[hdr+1] = 0; + data[hdr+2] = 0; + data[hdr+7] = 0; + memset(&data[iCellFirst], 0, cbrk-iCellFirst); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + if( cbrk-iCellFirst!=pPage->nFree ){ + return SQLITE_CORRUPT_BKPT; + } + return SQLITE_OK; +} + +/* +** Allocate nByte bytes of space from within the B-Tree page passed +** as the first argument. Write into *pIdx the index into pPage->aData[] +** of the first byte of allocated space. Return either SQLITE_OK or +** an error code (usually SQLITE_CORRUPT). +** +** The caller guarantees that there is sufficient space to make the +** allocation. This routine might need to defragment in order to bring +** all the space together, however. This routine will avoid using +** the first two bytes past the cell pointer area since presumably this +** allocation is being made in order to insert a new cell, so we will +** also end up needing a new cell pointer. +*/ +static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ + const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ + u8 * const data = pPage->aData; /* Local cache of pPage->aData */ + int nFrag; /* Number of fragmented bytes on pPage */ + int top; /* First byte of cell content area */ + int gap; /* First byte of gap between cell pointers and cell content */ + int rc; /* Integer return code */ + int usableSize; /* Usable size of the page */ + + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( pPage->pBt ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( nByte>=0 ); /* Minimum cell size is 4 */ + assert( pPage->nFree>=nByte ); + assert( pPage->nOverflow==0 ); + usableSize = pPage->pBt->usableSize; + assert( nByte < usableSize-8 ); + + nFrag = data[hdr+7]; + assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); + gap = pPage->cellOffset + 2*pPage->nCell; + top = get2byteNotZero(&data[hdr+5]); + if( gap>top ) return SQLITE_CORRUPT_BKPT; + testcase( gap+2==top ); + testcase( gap+1==top ); + testcase( gap==top ); + + if( nFrag>=60 ){ + /* Always defragment highly fragmented pages */ + rc = defragmentPage(pPage); + if( rc ) return rc; + top = get2byteNotZero(&data[hdr+5]); + }else if( gap+2<=top ){ + /* Search the freelist looking for a free slot big enough to satisfy + ** the request. The allocation is made from the first free slot in + ** the list that is large enough to accommodate it. + */ + int pc, addr; + for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&data[addr], &data[pc], 2); + data[hdr+7] = (u8)(nFrag + x); + }else if( size+pc > usableSize ){ + return SQLITE_CORRUPT_BKPT; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&data[pc+2], x); + } + *pIdx = pc + x; + return SQLITE_OK; + } + } + } + + /* Check to make sure there is enough space in the gap to satisfy + ** the allocation. If not, defragment. + */ + testcase( gap+2+nByte==top ); + if( gap+2+nByte>top ){ + rc = defragmentPage(pPage); + if( rc ) return rc; + top = get2byteNotZero(&data[hdr+5]); + assert( gap+nByte<=top ); + } + + + /* Allocate memory from the gap in between the cell pointer array + ** and the cell content area. The btreeInitPage() call has already + ** validated the freelist. Given that the freelist is valid, there + ** is no way that the allocation can extend off the end of the page. + ** The assert() below verifies the previous sentence. + */ + top -= nByte; + put2byte(&data[hdr+5], top); + assert( top+nByte <= (int)pPage->pBt->usableSize ); + *pIdx = top; + return SQLITE_OK; +} + +/* +** Return a section of the pPage->aData to the freelist. +** The first byte of the new free block is pPage->aDisk[start] +** and the size of the block is "size" bytes. +** +** Most of the effort here is involved in coalesing adjacent +** free blocks into a single big free block. +*/ +static int freeSpace(MemPage *pPage, int start, int size){ + int addr, pbegin, hdr; + int iLast; /* Largest possible freeblock offset */ + unsigned char *data = pPage->aData; + + assert( pPage->pBt!=0 ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( start>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( (start + size) <= (int)pPage->pBt->usableSize ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( size>=0 ); /* Minimum cell size is 4 */ + + if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[start], 0, size); + } + + /* Add the space back into the linked list of freeblocks. Note that + ** even though the freeblock list was checked by btreeInitPage(), + ** btreeInitPage() did not detect overlapping cells or + ** freeblocks that overlapped cells. Nor does it detect when the + ** cell content area exceeds the value in the page header. If these + ** situations arise, then subsequent insert operations might corrupt + ** the freelist. So we do need to check for corruption while scanning + ** the freelist. + */ + hdr = pPage->hdrOffset; + addr = hdr + 1; + iLast = pPage->pBt->usableSize - 4; + assert( start<=iLast ); + while( (pbegin = get2byte(&data[addr]))0 ){ + if( pbeginiLast ){ + return SQLITE_CORRUPT_BKPT; + } + assert( pbegin>addr || pbegin==0 ); + put2byte(&data[addr], start); + put2byte(&data[start], pbegin); + put2byte(&data[start+2], size); + pPage->nFree = pPage->nFree + (u16)size; + + /* Coalesce adjacent free blocks */ + addr = hdr + 1; + while( (pbegin = get2byte(&data[addr]))>0 ){ + int pnext, psize, x; + assert( pbegin>addr ); + assert( pbegin <= (int)pPage->pBt->usableSize-4 ); + pnext = get2byte(&data[pbegin]); + psize = get2byte(&data[pbegin+2]); + if( pbegin + psize + 3 >= pnext && pnext>0 ){ + int frag = pnext - (pbegin+psize); + if( (frag<0) || (frag>(int)data[hdr+7]) ){ + return SQLITE_CORRUPT_BKPT; + } + data[hdr+7] -= (u8)frag; + x = get2byte(&data[pnext]); + put2byte(&data[pbegin], x); + x = pnext + get2byte(&data[pnext+2]) - pbegin; + put2byte(&data[pbegin+2], x); + }else{ + addr = pbegin; + } + } + + /* If the cell content area begins with a freeblock, remove it. */ + if( data[hdr+1]==data[hdr+5] && data[hdr+2]==data[hdr+6] ){ + int top; + pbegin = get2byte(&data[hdr+1]); + memcpy(&data[hdr+1], &data[pbegin], 2); + top = get2byte(&data[hdr+5]) + get2byte(&data[pbegin+2]); + put2byte(&data[hdr+5], top); + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + return SQLITE_OK; +} + +/* +** Decode the flags byte (the first byte of the header) for a page +** and initialize fields of the MemPage structure accordingly. +** +** Only the following combinations are supported. Anything different +** indicates a corrupt database files: +** +** PTF_ZERODATA +** PTF_ZERODATA | PTF_LEAF +** PTF_LEAFDATA | PTF_INTKEY +** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF +*/ +static int decodeFlags(MemPage *pPage, int flagByte){ + BtShared *pBt; /* A copy of pPage->pBt */ + + assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); + flagByte &= ~PTF_LEAF; + pPage->childPtrSize = 4-4*pPage->leaf; + pBt = pPage->pBt; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + pPage->intKey = 1; + pPage->hasData = pPage->leaf; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else if( flagByte==PTF_ZERODATA ){ + pPage->intKey = 0; + pPage->hasData = 0; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + }else{ + return SQLITE_CORRUPT_BKPT; + } + pPage->max1bytePayload = pBt->max1bytePayload; + return SQLITE_OK; +} + +/* +** Initialize the auxiliary information for a disk block. +** +** Return SQLITE_OK on success. If we see that the page does +** not contain a well-formed database page, then return +** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not +** guarantee that the page is well-formed. It only shows that +** we failed to detect any corruption. +*/ +static int btreeInitPage(MemPage *pPage){ + + assert( pPage->pBt!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + + if( !pPage->isInit ){ + u16 pc; /* Address of a freeblock within pPage->aData[] */ + u8 hdr; /* Offset to beginning of page header */ + u8 *data; /* Equal to pPage->aData */ + BtShared *pBt; /* The main btree structure */ + int usableSize; /* Amount of usable space on each page */ + u16 cellOffset; /* Offset from start of page to first cell pointer */ + int nFree; /* Number of unused bytes on the page */ + int top; /* First byte of the cell content area */ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ + + pBt = pPage->pBt; + + hdr = pPage->hdrOffset; + data = pPage->aData; + if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nOverflow = 0; + usableSize = pBt->usableSize; + pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; + pPage->aDataEnd = &data[usableSize]; + pPage->aCellIdx = &data[cellOffset]; + top = get2byteNotZero(&data[hdr+5]); + pPage->nCell = get2byte(&data[hdr+3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ + return SQLITE_CORRUPT_BKPT; + } + testcase( pPage->nCell==MX_CELL(pBt) ); + + /* A malformed database page might cause us to read past the end + ** of page when parsing a cell. + ** + ** The following block of code checks early to see if a cell extends + ** past the end of a page boundary and causes SQLITE_CORRUPT to be + ** returned if it does. + */ + iCellFirst = cellOffset + 2*pPage->nCell; + iCellLast = usableSize - 4; +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + { + int i; /* Index into the cell pointer array */ + int sz; /* Size of a cell */ + + if( !pPage->leaf ) iCellLast--; + for(i=0; inCell; i++){ + pc = get2byte(&data[cellOffset+i*2]); + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + if( pciCellLast ){ + return SQLITE_CORRUPT_BKPT; + } + sz = cellSizePtr(pPage, &data[pc]); + testcase( pc+sz==usableSize ); + if( pc+sz>usableSize ){ + return SQLITE_CORRUPT_BKPT; + } + } + if( !pPage->leaf ) iCellLast++; + } +#endif + + /* Compute the total free space on the page */ + pc = get2byte(&data[hdr+1]); + nFree = data[hdr+7] + top; + while( pc>0 ){ + u16 next, size; + if( pciCellLast ){ + /* Start of free block is off the page */ + return SQLITE_CORRUPT_BKPT; + } + next = get2byte(&data[pc]); + size = get2byte(&data[pc+2]); + if( (next>0 && next<=pc+size+3) || pc+size>usableSize ){ + /* Free blocks must be in ascending order. And the last byte of + ** the free-block must lie on the database page. */ + return SQLITE_CORRUPT_BKPT; + } + nFree = nFree + size; + pc = next; + } + + /* At this point, nFree contains the sum of the offset to the start + ** of the cell-content area plus the number of free bytes within + ** the cell-content area. If this is greater than the usable-size + ** of the page, then the page must be corrupted. This check also + ** serves to verify that the offset to the start of the cell-content + ** area, according to the page header, lies within the page. + */ + if( nFree>usableSize ){ + return SQLITE_CORRUPT_BKPT; + } + pPage->nFree = (u16)(nFree - iCellFirst); + pPage->isInit = 1; + } + return SQLITE_OK; +} + +/* +** Set up a raw page so that it looks like a database page holding +** no entries. +*/ +static void zeroPage(MemPage *pPage, int flags){ + unsigned char *data = pPage->aData; + BtShared *pBt = pPage->pBt; + u8 hdr = pPage->hdrOffset; + u16 first; + + assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage) == data ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pBt->btsFlags & BTS_SECURE_DELETE ){ + memset(&data[hdr], 0, pBt->usableSize - hdr); + } + data[hdr] = (char)flags; + first = hdr + ((flags&PTF_LEAF)==0 ? 12 : 8); + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pBt->usableSize); + pPage->nFree = (u16)(pBt->usableSize - first); + decodeFlags(pPage, flags); + pPage->cellOffset = first; + pPage->aDataEnd = &data[pBt->usableSize]; + pPage->aCellIdx = &data[first]; + pPage->nOverflow = 0; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nCell = 0; + pPage->isInit = 1; +} + + +/* +** Convert a DbPage obtained from the pager into a MemPage used by +** the btree layer. +*/ +static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ + MemPage *pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + pPage->aData = sqlite3PagerGetData(pDbPage); + pPage->pDbPage = pDbPage; + pPage->pBt = pBt; + pPage->pgno = pgno; + pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; + return pPage; +} + +/* +** Get a page from the pager. Initialize the MemPage.pBt and +** MemPage.aData elements if needed. +** +** If the noContent flag is set, it means that we do not care about +** the content of the page at this time. So do not go to the disk +** to fetch the content. Just fill in the content with zeros for now. +** If in the future we call sqlite3PagerWrite() on this page, that +** means we have started to be concerned about content and the disk +** read should occur at that point. +*/ +static int btreeGetPage( + BtShared *pBt, /* The btree */ + Pgno pgno, /* Number of the page to fetch */ + MemPage **ppPage, /* Return the page in this parameter */ + int flags /* PAGER_GET_NOCONTENT or PAGER_GET_READONLY */ +){ + int rc; + DbPage *pDbPage; + + assert( flags==0 || flags==PAGER_GET_NOCONTENT || flags==PAGER_GET_READONLY ); + assert( sqlite3_mutex_held(pBt->mutex) ); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); + if( rc ) return rc; + *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); + return SQLITE_OK; +} + +/* +** Retrieve a page from the pager cache. If the requested page is not +** already in the pager cache return NULL. Initialize the MemPage.pBt and +** MemPage.aData elements if needed. +*/ +static MemPage *btreePageLookup(BtShared *pBt, Pgno pgno){ + DbPage *pDbPage; + assert( sqlite3_mutex_held(pBt->mutex) ); + pDbPage = sqlite3PagerLookup(pBt->pPager, pgno); + if( pDbPage ){ + return btreePageFromDbPage(pDbPage, pgno, pBt); + } + return 0; +} + +/* +** Return the size of the database file in pages. If there is any kind of +** error, return ((unsigned int)-1). +*/ +static Pgno btreePagecount(BtShared *pBt){ + return pBt->nPage; +} +SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ + assert( sqlite3BtreeHoldsMutex(p) ); + assert( ((p->pBt->nPage)&0x8000000)==0 ); + return btreePagecount(p->pBt); +} + +/* +** Get a page from the pager and initialize it. This routine is just a +** convenience wrapper around separate calls to btreeGetPage() and +** btreeInitPage(). +** +** If an error occurs, then the value *ppPage is set to is undefined. It +** may remain unchanged, or it may be set to an invalid value. +*/ +static int getAndInitPage( + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + int bReadonly /* PAGER_GET_READONLY or 0 */ +){ + int rc; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( bReadonly==PAGER_GET_READONLY || bReadonly==0 ); + + if( pgno>btreePagecount(pBt) ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = btreeGetPage(pBt, pgno, ppPage, bReadonly); + if( rc==SQLITE_OK && (*ppPage)->isInit==0 ){ + rc = btreeInitPage(*ppPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + } + } + } + + testcase( pgno==0 ); + assert( pgno!=0 || rc==SQLITE_CORRUPT ); + return rc; +} + +/* +** Release a MemPage. This should be called once for each prior +** call to btreeGetPage. +*/ +static void releasePage(MemPage *pPage){ + if( pPage ){ + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefNotNull(pPage->pDbPage); + } +} + +/* +** During a rollback, when the pager reloads information into the cache +** so that the cache is restored to its original state at the start of +** the transaction, for each page restored this routine is called. +** +** This routine needs to reset the extra data section at the end of the +** page to agree with the restored data. +*/ +static void pageReinit(DbPage *pData){ + MemPage *pPage; + pPage = (MemPage *)sqlite3PagerGetExtra(pData); + assert( sqlite3PagerPageRefcount(pData)>0 ); + if( pPage->isInit ){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->isInit = 0; + if( sqlite3PagerPageRefcount(pData)>1 ){ + /* pPage might not be a btree page; it might be an overflow page + ** or ptrmap page or a free page. In those cases, the following + ** call to btreeInitPage() will likely return SQLITE_CORRUPT. + ** But no harm is done by this. And it is very important that + ** btreeInitPage() be called on every btree page so we make + ** the call for every page that comes in for re-initing. */ + btreeInitPage(pPage); + } + } +} + +/* +** Invoke the busy handler for a btree. +*/ +static int btreeInvokeBusyHandler(void *pArg){ + BtShared *pBt = (BtShared*)pArg; + assert( pBt->db ); + assert( sqlite3_mutex_held(pBt->db->mutex) ); + return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); +} + +/* +** Open a database file. +** +** zFilename is the name of the database file. If zFilename is NULL +** then an ephemeral database is created. The ephemeral database might +** be exclusively in memory, or it might use a disk-based memory cache. +** Either way, the ephemeral database will be automatically deleted +** when sqlite3BtreeClose() is called. +** +** If zFilename is ":memory:" then an in-memory database is created +** that is automatically destroyed when it is closed. +** +** The "flags" parameter is a bitmask that might contain bits like +** BTREE_OMIT_JOURNAL and/or BTREE_MEMORY. +** +** If the database is already opened in the same database connection +** and we are in shared cache mode, then the open will fail with an +** SQLITE_CONSTRAINT error. We cannot allow two or more BtShared +** objects in the same database connection since doing so will lead +** to problems with locking. +*/ +SQLITE_PRIVATE int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use for this b-tree */ + const char *zFilename, /* Name of the file containing the BTree database */ + sqlite3 *db, /* Associated database handle */ + Btree **ppBtree, /* Pointer to new Btree object written here */ + int flags, /* Options */ + int vfsFlags /* Flags passed through to sqlite3_vfs.xOpen() */ +){ + BtShared *pBt = 0; /* Shared part of btree structure */ + Btree *p; /* Handle to return */ + sqlite3_mutex *mutexOpen = 0; /* Prevents a race condition. Ticket #3537 */ + int rc = SQLITE_OK; /* Result code from this function */ + u8 nReserve; /* Byte of unused space on each page */ + unsigned char zDbHeader[100]; /* Database header content */ + + /* True if opening an ephemeral, temporary database */ + const int isTempDb = zFilename==0 || zFilename[0]==0; + + /* Set the variable isMemdb to true for an in-memory database, or + ** false for a file-based database. + */ +#ifdef SQLITE_OMIT_MEMORYDB + const int isMemdb = 0; +#else + const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0) + || (isTempDb && sqlite3TempInMemory(db)) + || (vfsFlags & SQLITE_OPEN_MEMORY)!=0; +#endif + + assert( db!=0 ); + assert( pVfs!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ + + /* Only a BTREE_SINGLE database can be BTREE_UNORDERED */ + assert( (flags & BTREE_UNORDERED)==0 || (flags & BTREE_SINGLE)!=0 ); + + /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ + assert( (flags & BTREE_SINGLE)==0 || isTempDb ); + + if( isMemdb ){ + flags |= BTREE_MEMORY; + } + if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ + vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; + } + p = sqlite3MallocZero(sizeof(Btree)); + if( !p ){ + return SQLITE_NOMEM; + } + p->inTrans = TRANS_NONE; + p->db = db; +#ifndef SQLITE_OMIT_SHARED_CACHE + p->lock.pBtree = p; + p->lock.iTable = 1; +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* + ** If this Btree is a candidate for shared cache, try to find an + ** existing BtShared object that we can share with + */ + if( isTempDb==0 && (isMemdb==0 || (vfsFlags&SQLITE_OPEN_URI)!=0) ){ + if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ + int nFullPathname = pVfs->mxPathname+1; + char *zFullPathname = sqlite3Malloc(nFullPathname); + MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) + p->sharable = 1; + if( !zFullPathname ){ + sqlite3_free(p); + return SQLITE_NOMEM; + } + if( isMemdb ){ + memcpy(zFullPathname, zFilename, sqlite3Strlen30(zFilename)+1); + }else{ + rc = sqlite3OsFullPathname(pVfs, zFilename, + nFullPathname, zFullPathname); + if( rc ){ + sqlite3_free(zFullPathname); + sqlite3_free(p); + return rc; + } + } +#if SQLITE_THREADSAFE + mutexOpen = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_OPEN); + sqlite3_mutex_enter(mutexOpen); + mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(mutexShared); +#endif + for(pBt=GLOBAL(BtShared*,sqlite3SharedCacheList); pBt; pBt=pBt->pNext){ + assert( pBt->nRef>0 ); + if( 0==strcmp(zFullPathname, sqlite3PagerFilename(pBt->pPager, 0)) + && sqlite3PagerVfs(pBt->pPager)==pVfs ){ + int iDb; + for(iDb=db->nDb-1; iDb>=0; iDb--){ + Btree *pExisting = db->aDb[iDb].pBt; + if( pExisting && pExisting->pBt==pBt ){ + sqlite3_mutex_leave(mutexShared); + sqlite3_mutex_leave(mutexOpen); + sqlite3_free(zFullPathname); + sqlite3_free(p); + return SQLITE_CONSTRAINT; + } + } + p->pBt = pBt; + pBt->nRef++; + break; + } + } + sqlite3_mutex_leave(mutexShared); + sqlite3_free(zFullPathname); + } +#ifdef SQLITE_DEBUG + else{ + /* In debug mode, we mark all persistent databases as sharable + ** even when they are not. This exercises the locking code and + ** gives more opportunity for asserts(sqlite3_mutex_held()) + ** statements to find locking problems. + */ + p->sharable = 1; + } +#endif + } +#endif + if( pBt==0 ){ + /* + ** The following asserts make sure that structures used by the btree are + ** the right size. This is to guard against size changes that result + ** when compiling on a different architecture. + */ + assert( sizeof(i64)==8 || sizeof(i64)==4 ); + assert( sizeof(u64)==8 || sizeof(u64)==4 ); + assert( sizeof(u32)==4 ); + assert( sizeof(u16)==2 ); + assert( sizeof(Pgno)==4 ); + + pBt = sqlite3MallocZero( sizeof(*pBt) ); + if( pBt==0 ){ + rc = SQLITE_NOMEM; + goto btree_open_out; + } + rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, + EXTRA_SIZE, flags, vfsFlags, pageReinit); + if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); + rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); + } + if( rc!=SQLITE_OK ){ + goto btree_open_out; + } + pBt->openFlags = (u8)flags; + pBt->db = db; + sqlite3PagerSetBusyhandler(pBt->pPager, btreeInvokeBusyHandler, pBt); + p->pBt = pBt; + + pBt->pCursor = 0; + pBt->pPage1 = 0; + if( sqlite3PagerIsreadonly(pBt->pPager) ) pBt->btsFlags |= BTS_READ_ONLY; +#ifdef SQLITE_SECURE_DELETE + pBt->btsFlags |= BTS_SECURE_DELETE; +#endif + pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); + if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE + || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ + pBt->pageSize = 0; +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the magic name ":memory:" will create an in-memory database, then + ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if + ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if + ** SQLITE_OMIT_MEMORYDB has been defined, then ":memory:" is just a + ** regular file-name. In this case the auto-vacuum applies as per normal. + */ + if( zFilename && !isMemdb ){ + pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0); + pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0); + } +#endif + nReserve = 0; + }else{ + nReserve = zDbHeader[20]; + pBt->btsFlags |= BTS_PAGESIZE_FIXED; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->autoVacuum = (get4byte(&zDbHeader[36 + 4*4])?1:0); + pBt->incrVacuum = (get4byte(&zDbHeader[36 + 7*4])?1:0); +#endif + } + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); + if( rc ) goto btree_open_out; + pBt->usableSize = pBt->pageSize - nReserve; + assert( (pBt->pageSize & 7)==0 ); /* 8-byte alignment of pageSize */ + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* Add the new BtShared object to the linked list sharable BtShareds. + */ + if( p->sharable ){ + MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) + pBt->nRef = 1; + MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);) + if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ + pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); + if( pBt->mutex==0 ){ + rc = SQLITE_NOMEM; + db->mallocFailed = 0; + goto btree_open_out; + } + } + sqlite3_mutex_enter(mutexShared); + pBt->pNext = GLOBAL(BtShared*,sqlite3SharedCacheList); + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt; + sqlite3_mutex_leave(mutexShared); + } +#endif + } + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* If the new Btree uses a sharable pBtShared, then link the new + ** Btree into the list of all sharable Btrees for the same connection. + ** The list is kept in ascending order by pBt address. + */ + if( p->sharable ){ + int i; + Btree *pSib; + for(i=0; inDb; i++){ + if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ + while( pSib->pPrev ){ pSib = pSib->pPrev; } + if( p->pBtpBt ){ + p->pNext = pSib; + p->pPrev = 0; + pSib->pPrev = p; + }else{ + while( pSib->pNext && pSib->pNext->pBtpBt ){ + pSib = pSib->pNext; + } + p->pNext = pSib->pNext; + p->pPrev = pSib; + if( p->pNext ){ + p->pNext->pPrev = p; + } + pSib->pNext = p; + } + break; + } + } + } +#endif + *ppBtree = p; + +btree_open_out: + if( rc!=SQLITE_OK ){ + if( pBt && pBt->pPager ){ + sqlite3PagerClose(pBt->pPager); + } + sqlite3_free(pBt); + sqlite3_free(p); + *ppBtree = 0; + }else{ + /* If the B-Tree was successfully opened, set the pager-cache size to the + ** default value. Except, when opening on an existing shared pager-cache, + ** do not change the pager-cache size. + */ + if( sqlite3BtreeSchema(p, 0, 0)==0 ){ + sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE); + } + } + if( mutexOpen ){ + assert( sqlite3_mutex_held(mutexOpen) ); + sqlite3_mutex_leave(mutexOpen); + } + return rc; +} + +/* +** Decrement the BtShared.nRef counter. When it reaches zero, +** remove the BtShared structure from the sharing list. Return +** true if the BtShared.nRef counter reaches zero and return +** false if it is still positive. +*/ +static int removeFromSharingList(BtShared *pBt){ +#ifndef SQLITE_OMIT_SHARED_CACHE + MUTEX_LOGIC( sqlite3_mutex *pMaster; ) + BtShared *pList; + int removed = 0; + + assert( sqlite3_mutex_notheld(pBt->mutex) ); + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + sqlite3_mutex_enter(pMaster); + pBt->nRef--; + if( pBt->nRef<=0 ){ + if( GLOBAL(BtShared*,sqlite3SharedCacheList)==pBt ){ + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt->pNext; + }else{ + pList = GLOBAL(BtShared*,sqlite3SharedCacheList); + while( ALWAYS(pList) && pList->pNext!=pBt ){ + pList=pList->pNext; + } + if( ALWAYS(pList) ){ + pList->pNext = pBt->pNext; + } + } + if( SQLITE_THREADSAFE ){ + sqlite3_mutex_free(pBt->mutex); + } + removed = 1; + } + sqlite3_mutex_leave(pMaster); + return removed; +#else + return 1; +#endif +} + +/* +** Make sure pBt->pTmpSpace points to an allocation of +** MX_CELL_SIZE(pBt) bytes. +*/ +static void allocateTempSpace(BtShared *pBt){ + if( !pBt->pTmpSpace ){ + pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); + + /* One of the uses of pBt->pTmpSpace is to format cells before + ** inserting them into a leaf page (function fillInCell()). If + ** a cell is less than 4 bytes in size, it is rounded up to 4 bytes + ** by the various routines that manipulate binary cells. Which + ** can mean that fillInCell() only initializes the first 2 or 3 + ** bytes of pTmpSpace, but that the first 4 bytes are copied from + ** it into a database page. This is not actually a problem, but it + ** does cause a valgrind error when the 1 or 2 bytes of unitialized + ** data is passed to system call write(). So to avoid this error, + ** zero the first 4 bytes of temp space here. */ + if( pBt->pTmpSpace ) memset(pBt->pTmpSpace, 0, 4); + } +} + +/* +** Free the pBt->pTmpSpace allocation +*/ +static void freeTempSpace(BtShared *pBt){ + sqlite3PageFree( pBt->pTmpSpace); + pBt->pTmpSpace = 0; +} + +/* +** Close an open database and invalidate all cursors. +*/ +SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ + BtShared *pBt = p->pBt; + BtCursor *pCur; + + /* Close all cursors opened via this handle. */ + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + pCur = pBt->pCursor; + while( pCur ){ + BtCursor *pTmp = pCur; + pCur = pCur->pNext; + if( pTmp->pBtree==p ){ + sqlite3BtreeCloseCursor(pTmp); + } + } + + /* Rollback any active transaction and free the handle structure. + ** The call to sqlite3BtreeRollback() drops any table-locks held by + ** this handle. + */ + sqlite3BtreeRollback(p, SQLITE_OK); + sqlite3BtreeLeave(p); + + /* If there are still other outstanding references to the shared-btree + ** structure, return now. The remainder of this procedure cleans + ** up the shared-btree. + */ + assert( p->wantToLock==0 && p->locked==0 ); + if( !p->sharable || removeFromSharingList(pBt) ){ + /* The pBt is no longer on the sharing list, so we can access + ** it without having to hold the mutex. + ** + ** Clean out and delete the BtShared object. + */ + assert( !pBt->pCursor ); + sqlite3PagerClose(pBt->pPager); + if( pBt->xFreeSchema && pBt->pSchema ){ + pBt->xFreeSchema(pBt->pSchema); + } + sqlite3DbFree(0, pBt->pSchema); + freeTempSpace(pBt); + sqlite3_free(pBt); + } + +#ifndef SQLITE_OMIT_SHARED_CACHE + assert( p->wantToLock==0 ); + assert( p->locked==0 ); + if( p->pPrev ) p->pPrev->pNext = p->pNext; + if( p->pNext ) p->pNext->pPrev = p->pPrev; +#endif + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Change the limit on the number of pages allowed in the cache. +** +** The maximum number of cache pages is set to the absolute +** value of mxPage. If mxPage is negative, the pager will +** operate asynchronously - it will not stop to do fsync()s +** to insure data is written to the disk surface before +** continuing. Transactions still work if synchronous is off, +** and the database cannot be corrupted if this program +** crashes. But if the operating system crashes or there is +** an abrupt power failure when synchronous is off, the database +** could be left in an inconsistent and unrecoverable state. +** Synchronous is on by default so database corruption is not +** normally a worry. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetCachesize(pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapLimit(pBt->pPager, szMmap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** Change the way data is synced to disk in order to increase or decrease +** how well the database resists damage due to OS crashes and power +** failures. Level 1 is the same as asynchronous (no syncs() occur and +** there is a high probability of damage) Level 2 is the default. There +** is a very low but non-zero probability of damage. Level 3 reduces the +** probability of damage to near zero but with a write performance reduction. +*/ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags( + Btree *p, /* The btree to set the safety level on */ + unsigned pgFlags /* Various PAGER_* flags */ +){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetFlags(pBt->pPager, pgFlags); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} +#endif + +/* +** Return TRUE if the given btree is set to safety level 1. In other +** words, return TRUE if no sync() occurs on the disk files. +*/ +SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree *p){ + BtShared *pBt = p->pBt; + int rc; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + assert( pBt && pBt->pPager ); + rc = sqlite3PagerNosync(pBt->pPager); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Change the default pages size and the number of reserved bytes per page. +** Or, if the page size has already been fixed, return SQLITE_READONLY +** without changing anything. +** +** The page size must be a power of 2 between 512 and 65536. If the page +** size supplied does not meet this constraint then the page size is not +** changed. +** +** Page sizes are constrained to be a power of two so that the region +** of the database file used for locking (beginning at PENDING_BYTE, +** the first byte past the 1GB boundary, 0x40000000) needs to occur +** at the beginning of a page. +** +** If parameter nReserve is less than zero, then the number of reserved +** bytes per page is left unchanged. +** +** If the iFix!=0 then the BTS_PAGESIZE_FIXED flag is set so that the page size +** and autovacuum mode can no longer be changed. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ + int rc = SQLITE_OK; + BtShared *pBt = p->pBt; + assert( nReserve>=-1 && nReserve<=255 ); + sqlite3BtreeEnter(p); + if( pBt->btsFlags & BTS_PAGESIZE_FIXED ){ + sqlite3BtreeLeave(p); + return SQLITE_READONLY; + } + if( nReserve<0 ){ + nReserve = pBt->pageSize - pBt->usableSize; + } + assert( nReserve>=0 && nReserve<=255 ); + if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && + ((pageSize-1)&pageSize)==0 ){ + assert( (pageSize & 7)==0 ); + assert( !pBt->pPage1 && !pBt->pCursor ); + pBt->pageSize = (u32)pageSize; + freeTempSpace(pBt); + } + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); + pBt->usableSize = pBt->pageSize - (u16)nReserve; + if( iFix ) pBt->btsFlags |= BTS_PAGESIZE_FIXED; + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Return the currently defined page size +*/ +SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree *p){ + return p->pBt->pageSize; +} + +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +/* +** This function is similar to sqlite3BtreeGetReserve(), except that it +** may only be called if it is guaranteed that the b-tree mutex is already +** held. +** +** This is useful in one special case in the backup API code where it is +** known that the shared b-tree mutex is held, but the mutex on the +** database handle that owns *p is not. In this case if sqlite3BtreeEnter() +** were to be called, it might collide with some other operation on the +** database handle that owns *p, causing undefined behavior. +*/ +SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p){ + assert( sqlite3_mutex_held(p->pBt->mutex) ); + return p->pBt->pageSize - p->pBt->usableSize; +} +#endif /* SQLITE_HAS_CODEC || SQLITE_DEBUG */ + +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) +/* +** Return the number of bytes of space at the end of every page that +** are intentually left unused. This is the "reserved" space that is +** sometimes used by extensions. +*/ +SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree *p){ + int n; + sqlite3BtreeEnter(p); + n = p->pBt->pageSize - p->pBt->usableSize; + sqlite3BtreeLeave(p); + return n; +} + +/* +** Set the maximum page count for a database if mxPage is positive. +** No changes are made if mxPage is 0 or negative. +** Regardless of the value of mxPage, return the maximum page count. +*/ +SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree *p, int mxPage){ + int n; + sqlite3BtreeEnter(p); + n = sqlite3PagerMaxPageCount(p->pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return n; +} + +/* +** Set the BTS_SECURE_DELETE flag if newFlag is 0 or 1. If newFlag is -1, +** then make no changes. Always return the value of the BTS_SECURE_DELETE +** setting after the change. +*/ +SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ + int b; + if( p==0 ) return 0; + sqlite3BtreeEnter(p); + if( newFlag>=0 ){ + p->pBt->btsFlags &= ~BTS_SECURE_DELETE; + if( newFlag ) p->pBt->btsFlags |= BTS_SECURE_DELETE; + } + b = (p->pBt->btsFlags & BTS_SECURE_DELETE)!=0; + sqlite3BtreeLeave(p); + return b; +} +#endif /* !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) */ + +/* +** Change the 'auto-vacuum' property of the database. If the 'autoVacuum' +** parameter is non-zero, then auto-vacuum mode is enabled. If zero, it +** is disabled. The default value for the auto-vacuum property is +** determined by the SQLITE_DEFAULT_AUTOVACUUM macro. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ +#ifdef SQLITE_OMIT_AUTOVACUUM + return SQLITE_READONLY; +#else + BtShared *pBt = p->pBt; + int rc = SQLITE_OK; + u8 av = (u8)autoVacuum; + + sqlite3BtreeEnter(p); + if( (pBt->btsFlags & BTS_PAGESIZE_FIXED)!=0 && (av ?1:0)!=pBt->autoVacuum ){ + rc = SQLITE_READONLY; + }else{ + pBt->autoVacuum = av ?1:0; + pBt->incrVacuum = av==2 ?1:0; + } + sqlite3BtreeLeave(p); + return rc; +#endif +} + +/* +** Return the value of the 'auto-vacuum' property. If auto-vacuum is +** enabled 1 is returned. Otherwise 0. +*/ +SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *p){ +#ifdef SQLITE_OMIT_AUTOVACUUM + return BTREE_AUTOVACUUM_NONE; +#else + int rc; + sqlite3BtreeEnter(p); + rc = ( + (!p->pBt->autoVacuum)?BTREE_AUTOVACUUM_NONE: + (!p->pBt->incrVacuum)?BTREE_AUTOVACUUM_FULL: + BTREE_AUTOVACUUM_INCR + ); + sqlite3BtreeLeave(p); + return rc; +#endif +} + + +/* +** Get a reference to pPage1 of the database file. This will +** also acquire a readlock on that file. +** +** SQLITE_OK is returned on success. If the file is not a +** well-formed database file, then SQLITE_CORRUPT is returned. +** SQLITE_BUSY is returned if the database is locked. SQLITE_NOMEM +** is returned if we run out of memory. +*/ +static int lockBtree(BtShared *pBt){ + int rc; /* Result code from subfunctions */ + MemPage *pPage1; /* Page 1 of the database file */ + int nPage; /* Number of pages in the database */ + int nPageFile = 0; /* Number of pages in the database file */ + int nPageHeader; /* Number of pages in the database according to hdr */ + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pBt->pPage1==0 ); + rc = sqlite3PagerSharedLock(pBt->pPager); + if( rc!=SQLITE_OK ) return rc; + rc = btreeGetPage(pBt, 1, &pPage1, 0); + if( rc!=SQLITE_OK ) return rc; + + /* Do some checking to help insure the file we opened really is + ** a valid database file. + */ + nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); + sqlite3PagerPagecount(pBt->pPager, &nPageFile); + if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ + nPage = nPageFile; + } + if( nPage>0 ){ + u32 pageSize; + u32 usableSize; + u8 *page1 = pPage1->aData; + rc = SQLITE_NOTADB; + if( memcmp(page1, zMagicHeader, 16)!=0 ){ + goto page1_init_failed; + } + +#ifdef SQLITE_OMIT_WAL + if( page1[18]>1 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } + if( page1[19]>1 ){ + goto page1_init_failed; + } +#else + if( page1[18]>2 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } + if( page1[19]>2 ){ + goto page1_init_failed; + } + + /* If the write version is set to 2, this database should be accessed + ** in WAL mode. If the log is not already open, open it now. Then + ** return SQLITE_OK and return without populating BtShared.pPage1. + ** The caller detects this and calls this function again. This is + ** required as the version of page 1 currently in the page1 buffer + ** may not be the latest version - there may be a newer one in the log + ** file. + */ + if( page1[19]==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){ + int isOpen = 0; + rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); + if( rc!=SQLITE_OK ){ + goto page1_init_failed; + }else if( isOpen==0 ){ + releasePage(pPage1); + return SQLITE_OK; + } + rc = SQLITE_NOTADB; + } +#endif + + /* The maximum embedded fraction must be exactly 25%. And the minimum + ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. + ** The original design allowed these amounts to vary, but as of + ** version 3.6.0, we require them to be fixed. + */ + if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ + goto page1_init_failed; + } + pageSize = (page1[16]<<8) | (page1[17]<<16); + if( ((pageSize-1)&pageSize)!=0 + || pageSize>SQLITE_MAX_PAGE_SIZE + || pageSize<=256 + ){ + goto page1_init_failed; + } + assert( (pageSize & 7)==0 ); + usableSize = pageSize - page1[20]; + if( (u32)pageSize!=pBt->pageSize ){ + /* After reading the first page of the database assuming a page size + ** of BtShared.pageSize, we have discovered that the page-size is + ** actually pageSize. Unlock the database, leave pBt->pPage1 at + ** zero and return SQLITE_OK. The caller will call this function + ** again with the correct page-size. + */ + releasePage(pPage1); + pBt->usableSize = usableSize; + pBt->pageSize = pageSize; + freeTempSpace(pBt); + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, + pageSize-usableSize); + return rc; + } + if( (pBt->db->flags & SQLITE_RecoveryMode)==0 && nPage>nPageFile ){ + rc = SQLITE_CORRUPT_BKPT; + goto page1_init_failed; + } + if( usableSize<480 ){ + goto page1_init_failed; + } + pBt->pageSize = pageSize; + pBt->usableSize = usableSize; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0); + pBt->incrVacuum = (get4byte(&page1[36 + 7*4])?1:0); +#endif + } + + /* maxLocal is the maximum amount of payload to store locally for + ** a cell. Make sure it is small enough so that at least minFanout + ** cells can will fit on one page. We assume a 10-byte page header. + ** Besides the payload, the cell must store: + ** 2-byte pointer to the cell + ** 4-byte child pointer + ** 9-byte nKey value + ** 4-byte nData value + ** 4-byte overflow page pointer + ** So a cell consists of a 2-byte pointer, a header which is as much as + ** 17 bytes long, 0 to N bytes of payload, and an optional 4 byte overflow + ** page pointer. + */ + pBt->maxLocal = (u16)((pBt->usableSize-12)*64/255 - 23); + pBt->minLocal = (u16)((pBt->usableSize-12)*32/255 - 23); + pBt->maxLeaf = (u16)(pBt->usableSize - 35); + pBt->minLeaf = (u16)((pBt->usableSize-12)*32/255 - 23); + if( pBt->maxLocal>127 ){ + pBt->max1bytePayload = 127; + }else{ + pBt->max1bytePayload = (u8)pBt->maxLocal; + } + assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); + pBt->pPage1 = pPage1; + pBt->nPage = nPage; + return SQLITE_OK; + +page1_init_failed: + releasePage(pPage1); + pBt->pPage1 = 0; + return rc; +} + +#ifndef NDEBUG +/* +** Return the number of cursors open on pBt. This is for use +** in assert() expressions, so it is only compiled if NDEBUG is not +** defined. +** +** Only write cursors are counted if wrOnly is true. If wrOnly is +** false then all cursors are counted. +** +** For the purposes of this routine, a cursor is any cursor that +** is capable of reading or writing to the databse. Cursors that +** have been tripped into the CURSOR_FAULT state are not counted. +*/ +static int countValidCursors(BtShared *pBt, int wrOnly){ + BtCursor *pCur; + int r = 0; + for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ + if( (wrOnly==0 || (pCur->curFlags & BTCF_WriteFlag)!=0) + && pCur->eState!=CURSOR_FAULT ) r++; + } + return r; +} +#endif + +/* +** If there are no outstanding cursors and we are not in the middle +** of a transaction but there is a read lock on the database, then +** this routine unrefs the first page of the database file which +** has the effect of releasing the read lock. +** +** If there is a transaction in progress, this routine is a no-op. +*/ +static void unlockBtreeIfUnused(BtShared *pBt){ + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); + if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ + assert( pBt->pPage1->aData ); + assert( sqlite3PagerRefcount(pBt->pPager)==1 ); + assert( pBt->pPage1->aData ); + releasePage(pBt->pPage1); + pBt->pPage1 = 0; + } +} + +/* +** If pBt points to an empty file then convert that empty file +** into a new empty database by initializing the first page of +** the database. +*/ +static int newDatabase(BtShared *pBt){ + MemPage *pP1; + unsigned char *data; + int rc; + + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pBt->nPage>0 ){ + return SQLITE_OK; + } + pP1 = pBt->pPage1; + assert( pP1!=0 ); + data = pP1->aData; + rc = sqlite3PagerWrite(pP1->pDbPage); + if( rc ) return rc; + memcpy(data, zMagicHeader, sizeof(zMagicHeader)); + assert( sizeof(zMagicHeader)==16 ); + data[16] = (u8)((pBt->pageSize>>8)&0xff); + data[17] = (u8)((pBt->pageSize>>16)&0xff); + data[18] = 1; + data[19] = 1; + assert( pBt->usableSize<=pBt->pageSize && pBt->usableSize+255>=pBt->pageSize); + data[20] = (u8)(pBt->pageSize - pBt->usableSize); + data[21] = 64; + data[22] = 32; + data[23] = 32; + memset(&data[24], 0, 100-24); + zeroPage(pP1, PTF_INTKEY|PTF_LEAF|PTF_LEAFDATA ); + pBt->btsFlags |= BTS_PAGESIZE_FIXED; +#ifndef SQLITE_OMIT_AUTOVACUUM + assert( pBt->autoVacuum==1 || pBt->autoVacuum==0 ); + assert( pBt->incrVacuum==1 || pBt->incrVacuum==0 ); + put4byte(&data[36 + 4*4], pBt->autoVacuum); + put4byte(&data[36 + 7*4], pBt->incrVacuum); +#endif + pBt->nPage = 1; + data[31] = 1; + return SQLITE_OK; +} + +/* +** Initialize the first page of the database file (creating a database +** consisting of a single page and no schema objects). Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +*/ +SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ + int rc; + sqlite3BtreeEnter(p); + p->pBt->nPage = 0; + rc = newDatabase(p->pBt); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Attempt to start a new transaction. A write-transaction +** is started if the second argument is nonzero, otherwise a read- +** transaction. If the second argument is 2 or more and exclusive +** transaction is started, meaning that no other process is allowed +** to access the database. A preexisting transaction may not be +** upgraded to exclusive by calling this routine a second time - the +** exclusivity flag only works for a new transaction. +** +** A write-transaction must be started before attempting any +** changes to the database. None of the following routines +** will work unless a transaction is started first: +** +** sqlite3BtreeCreateTable() +** sqlite3BtreeCreateIndex() +** sqlite3BtreeClearTable() +** sqlite3BtreeDropTable() +** sqlite3BtreeInsert() +** sqlite3BtreeDelete() +** sqlite3BtreeUpdateMeta() +** +** If an initial attempt to acquire the lock fails because of lock contention +** and the database was previously unlocked, then invoke the busy handler +** if there is one. But if there was previously a read-lock, do not +** invoke the busy handler - just return SQLITE_BUSY. SQLITE_BUSY is +** returned when there is already a read-lock in order to avoid a deadlock. +** +** Suppose there are two processes A and B. A has a read lock and B has +** a reserved lock. B tries to promote to exclusive but is blocked because +** of A's read lock. A tries to promote to reserved but is blocked by B. +** One or the other of the two processes must give way or there can be +** no progress. By returning SQLITE_BUSY and not invoking the busy callback +** when A already has a read lock, we encourage A to give up and let B +** proceed. +*/ +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ + sqlite3 *pBlock = 0; + BtShared *pBt = p->pBt; + int rc = SQLITE_OK; + + sqlite3BtreeEnter(p); + btreeIntegrity(p); + + /* If the btree is already in a write-transaction, or it + ** is already in a read-transaction and a read-transaction + ** is requested, this is a no-op. + */ + if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ + goto trans_begun; + } + assert( pBt->inTransaction==TRANS_WRITE || IfNotOmitAV(pBt->bDoTruncate)==0 ); + + /* Write transactions are not possible on a read-only database */ + if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ + rc = SQLITE_READONLY; + goto trans_begun; + } + +#ifndef SQLITE_OMIT_SHARED_CACHE + /* If another database handle has already opened a write transaction + ** on this shared-btree structure and a second write transaction is + ** requested, return SQLITE_LOCKED. + */ + if( (wrflag && pBt->inTransaction==TRANS_WRITE) + || (pBt->btsFlags & BTS_PENDING)!=0 + ){ + pBlock = pBt->pWriter->db; + }else if( wrflag>1 ){ + BtLock *pIter; + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->pBtree!=p ){ + pBlock = pIter->pBtree->db; + break; + } + } + } + if( pBlock ){ + sqlite3ConnectionBlocked(p->db, pBlock); + rc = SQLITE_LOCKED_SHAREDCACHE; + goto trans_begun; + } +#endif + + /* Any read-only or read-write transaction implies a read-lock on + ** page 1. So if some other shared-cache client already has a write-lock + ** on page 1, the transaction cannot be opened. */ + rc = querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK); + if( SQLITE_OK!=rc ) goto trans_begun; + + pBt->btsFlags &= ~BTS_INITIALLY_EMPTY; + if( pBt->nPage==0 ) pBt->btsFlags |= BTS_INITIALLY_EMPTY; + do { + /* Call lockBtree() until either pBt->pPage1 is populated or + ** lockBtree() returns something other than SQLITE_OK. lockBtree() + ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after + ** reading page 1 it discovers that the page-size of the database + ** file is not pBt->pageSize. In this case lockBtree() will update + ** pBt->pageSize to the page-size of the file on disk. + */ + while( pBt->pPage1==0 && SQLITE_OK==(rc = lockBtree(pBt)) ); + + if( rc==SQLITE_OK && wrflag ){ + if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ + rc = SQLITE_READONLY; + }else{ + rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); + if( rc==SQLITE_OK ){ + rc = newDatabase(pBt); + } + } + } + + if( rc!=SQLITE_OK ){ + unlockBtreeIfUnused(pBt); + } + }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && + btreeInvokeBusyHandler(pBt) ); + + if( rc==SQLITE_OK ){ + if( p->inTrans==TRANS_NONE ){ + pBt->nTransaction++; +#ifndef SQLITE_OMIT_SHARED_CACHE + if( p->sharable ){ + assert( p->lock.pBtree==p && p->lock.iTable==1 ); + p->lock.eLock = READ_LOCK; + p->lock.pNext = pBt->pLock; + pBt->pLock = &p->lock; + } +#endif + } + p->inTrans = (wrflag?TRANS_WRITE:TRANS_READ); + if( p->inTrans>pBt->inTransaction ){ + pBt->inTransaction = p->inTrans; + } + if( wrflag ){ + MemPage *pPage1 = pBt->pPage1; +#ifndef SQLITE_OMIT_SHARED_CACHE + assert( !pBt->pWriter ); + pBt->pWriter = p; + pBt->btsFlags &= ~BTS_EXCLUSIVE; + if( wrflag>1 ) pBt->btsFlags |= BTS_EXCLUSIVE; +#endif + + /* If the db-size header field is incorrect (as it may be if an old + ** client has been writing the database file), update it now. Doing + ** this sooner rather than later means the database size can safely + ** re-read the database size from page 1 if a savepoint or transaction + ** rollback occurs within the transaction. + */ + if( pBt->nPage!=get4byte(&pPage1->aData[28]) ){ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pPage1->aData[28], pBt->nPage); + } + } + } + } + + +trans_begun: + if( rc==SQLITE_OK && wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + } + + btreeIntegrity(p); + sqlite3BtreeLeave(p); + return rc; +} + +#ifndef SQLITE_OMIT_AUTOVACUUM + +/* +** Set the pointer-map entries for all children of page pPage. Also, if +** pPage contains cells that point to overflow pages, set the pointer +** map entries for the overflow pages as well. +*/ +static int setChildPtrmaps(MemPage *pPage){ + int i; /* Counter variable */ + int nCell; /* Number of cells in page pPage */ + int rc; /* Return code */ + BtShared *pBt = pPage->pBt; + u8 isInitOrig = pPage->isInit; + Pgno pgno = pPage->pgno; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + rc = btreeInitPage(pPage); + if( rc!=SQLITE_OK ){ + goto set_child_ptrmaps_out; + } + nCell = pPage->nCell; + + for(i=0; ileaf ){ + Pgno childPgno = get4byte(pCell); + ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); + } + } + + if( !pPage->leaf ){ + Pgno childPgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); + } + +set_child_ptrmaps_out: + pPage->isInit = isInitOrig; + return rc; +} + +/* +** Somewhere on pPage is a pointer to page iFrom. Modify this pointer so +** that it points to iTo. Parameter eType describes the type of pointer to +** be modified, as follows: +** +** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child +** page of pPage. +** +** PTRMAP_OVERFLOW1: pPage is a btree-page. The pointer points at an overflow +** page pointed to by one of the cells on pPage. +** +** PTRMAP_OVERFLOW2: pPage is an overflow-page. The pointer points at the next +** overflow page in the list. +*/ +static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + if( eType==PTRMAP_OVERFLOW2 ){ + /* The pointer is always the first 4 bytes of the page in this case. */ + if( get4byte(pPage->aData)!=iFrom ){ + return SQLITE_CORRUPT_BKPT; + } + put4byte(pPage->aData, iTo); + }else{ + u8 isInitOrig = pPage->isInit; + int i; + int nCell; + + btreeInitPage(pPage); + nCell = pPage->nCell; + + for(i=0; iaData+pPage->maskPage + && iFrom==get4byte(&pCell[info.iOverflow]) + ){ + put4byte(&pCell[info.iOverflow], iTo); + break; + } + }else{ + if( get4byte(pCell)==iFrom ){ + put4byte(pCell, iTo); + break; + } + } + } + + if( i==nCell ){ + if( eType!=PTRMAP_BTREE || + get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){ + return SQLITE_CORRUPT_BKPT; + } + put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); + } + + pPage->isInit = isInitOrig; + } + return SQLITE_OK; +} + + +/* +** Move the open database page pDbPage to location iFreePage in the +** database. The pDbPage reference remains valid. +** +** The isCommit flag indicates that there is no need to remember that +** the journal needs to be sync()ed before database page pDbPage->pgno +** can be written to. The caller has already promised not to write to that +** page. +*/ +static int relocatePage( + BtShared *pBt, /* Btree */ + MemPage *pDbPage, /* Open page to move */ + u8 eType, /* Pointer map 'type' entry for pDbPage */ + Pgno iPtrPage, /* Pointer map 'page-no' entry for pDbPage */ + Pgno iFreePage, /* The location to move pDbPage to */ + int isCommit /* isCommit flag passed to sqlite3PagerMovepage */ +){ + MemPage *pPtrPage; /* The page that contains a pointer to pDbPage */ + Pgno iDbPage = pDbPage->pgno; + Pager *pPager = pBt->pPager; + int rc; + + assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || + eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pDbPage->pBt==pBt ); + + /* Move page iDbPage from its current location to page number iFreePage */ + TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", + iDbPage, iFreePage, iPtrPage, eType)); + rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); + if( rc!=SQLITE_OK ){ + return rc; + } + pDbPage->pgno = iFreePage; + + /* If pDbPage was a btree-page, then it may have child pages and/or cells + ** that point to overflow pages. The pointer map entries for all these + ** pages need to be changed. + ** + ** If pDbPage is an overflow page, then the first 4 bytes may store a + ** pointer to a subsequent overflow page. If this is the case, then + ** the pointer map needs to be updated for the subsequent overflow page. + */ + if( eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ){ + rc = setChildPtrmaps(pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + Pgno nextOvfl = get4byte(pDbPage->aData); + if( nextOvfl!=0 ){ + ptrmapPut(pBt, nextOvfl, PTRMAP_OVERFLOW2, iFreePage, &rc); + if( rc!=SQLITE_OK ){ + return rc; + } + } + } + + /* Fix the database pointer on page iPtrPage that pointed at iDbPage so + ** that it points at iFreePage. Also fix the pointer map entry for + ** iPtrPage. + */ + if( eType!=PTRMAP_ROOTPAGE ){ + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3PagerWrite(pPtrPage->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pPtrPage); + return rc; + } + rc = modifyPagePointer(pPtrPage, iDbPage, iFreePage, eType); + releasePage(pPtrPage); + if( rc==SQLITE_OK ){ + ptrmapPut(pBt, iFreePage, eType, iPtrPage, &rc); + } + } + return rc; +} + +/* Forward declaration required by incrVacuumStep(). */ +static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); + +/* +** Perform a single step of an incremental-vacuum. If successful, return +** SQLITE_OK. If there is no work to do (and therefore no point in +** calling this function again), return SQLITE_DONE. Or, if an error +** occurs, return some other error code. +** +** More specificly, this function attempts to re-organize the database so +** that the last page of the file currently in use is no longer in use. +** +** Parameter nFin is the number of pages that this database would contain +** were this function called until it returns SQLITE_DONE. +** +** If the bCommit parameter is non-zero, this function assumes that the +** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE +** or an error. bCommit is passed true for an auto-vacuum-on-commmit +** operation, or false for an incremental vacuum. +*/ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ + Pgno nFreeList; /* Number of pages still on the free-list */ + int rc; + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( iLastPg>nFin ); + + if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){ + u8 eType; + Pgno iPtrPage; + + nFreeList = get4byte(&pBt->pPage1->aData[36]); + if( nFreeList==0 ){ + return SQLITE_DONE; + } + + rc = ptrmapGet(pBt, iLastPg, &eType, &iPtrPage); + if( rc!=SQLITE_OK ){ + return rc; + } + if( eType==PTRMAP_ROOTPAGE ){ + return SQLITE_CORRUPT_BKPT; + } + + if( eType==PTRMAP_FREEPAGE ){ + if( bCommit==0 ){ + /* Remove the page from the files free-list. This is not required + ** if bCommit is non-zero. In that case, the free-list will be + ** truncated to zero after this function returns, so it doesn't + ** matter if it still contains some garbage entries. + */ + Pgno iFreePg; + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, BTALLOC_EXACT); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( iFreePg==iLastPg ); + releasePage(pFreePg); + } + } else { + Pgno iFreePg; /* Index of free page to move pLastPg to */ + MemPage *pLastPg; + u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ + Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ + + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If bCommit is zero, this loop runs exactly once and page pLastPg + ** is swapped with the first free page pulled off the free list. + ** + ** On the other hand, if bCommit is greater than zero, then keep + ** looping until a free-page located within the first nFin pages + ** of the file is found. + */ + if( bCommit==0 ){ + eMode = BTALLOC_LE; + iNear = nFin; + } + do { + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode); + if( rc!=SQLITE_OK ){ + releasePage(pLastPg); + return rc; + } + releasePage(pFreePg); + }while( bCommit && iFreePg>nFin ); + assert( iFreePgbDoTruncate = 1; + pBt->nPage = iLastPg; + } + return SQLITE_OK; +} + +/* +** The database opened by the first argument is an auto-vacuum database +** nOrig pages in size containing nFree free pages. Return the expected +** size of the database in pages following an auto-vacuum operation. +*/ +static Pgno finalDbSize(BtShared *pBt, Pgno nOrig, Pgno nFree){ + int nEntry; /* Number of entries on one ptrmap page */ + Pgno nPtrmap; /* Number of PtrMap pages to be freed */ + Pgno nFin; /* Return value */ + + nEntry = pBt->usableSize/5; + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFinpBt; + + sqlite3BtreeEnter(p); + assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE ); + if( !pBt->autoVacuum ){ + rc = SQLITE_DONE; + }else{ + Pgno nOrig = btreePagecount(pBt); + Pgno nFree = get4byte(&pBt->pPage1->aData[36]); + Pgno nFin = finalDbSize(pBt, nOrig, nFree); + + if( nOrig0 ){ + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[28], pBt->nPage); + } + }else{ + rc = SQLITE_DONE; + } + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** This routine is called prior to sqlite3PagerCommit when a transaction +** is committed for an auto-vacuum database. +** +** If SQLITE_OK is returned, then *pnTrunc is set to the number of pages +** the database file should be truncated to during the commit process. +** i.e. the database has been reorganized so that only the first *pnTrunc +** pages are in use. +*/ +static int autoVacuumCommit(BtShared *pBt){ + int rc = SQLITE_OK; + Pager *pPager = pBt->pPager; + VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) ); + + assert( sqlite3_mutex_held(pBt->mutex) ); + invalidateAllOverflowCache(pBt); + assert(pBt->autoVacuum); + if( !pBt->incrVacuum ){ + Pgno nFin; /* Number of pages in database after autovacuuming */ + Pgno nFree; /* Number of pages on the freelist initially */ + Pgno iFree; /* The next page to be freed */ + Pgno nOrig; /* Database size before freeing */ + + nOrig = btreePagecount(pBt); + if( PTRMAP_ISPAGE(pBt, nOrig) || nOrig==PENDING_BYTE_PAGE(pBt) ){ + /* It is not possible to create a database for which the final page + ** is either a pointer-map page or the pending-byte page. If one + ** is encountered, this indicates corruption. + */ + return SQLITE_CORRUPT_BKPT; + } + + nFree = get4byte(&pBt->pPage1->aData[36]); + nFin = finalDbSize(pBt, nOrig, nFree); + if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; + if( nFinnFin && rc==SQLITE_OK; iFree--){ + rc = incrVacuumStep(pBt, nFin, iFree, 1); + } + if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[32], 0); + put4byte(&pBt->pPage1->aData[36], 0); + put4byte(&pBt->pPage1->aData[28], nFin); + pBt->bDoTruncate = 1; + pBt->nPage = nFin; + } + if( rc!=SQLITE_OK ){ + sqlite3PagerRollback(pPager); + } + } + + assert( nRef>=sqlite3PagerRefcount(pPager) ); + return rc; +} + +#else /* ifndef SQLITE_OMIT_AUTOVACUUM */ +# define setChildPtrmaps(x) SQLITE_OK +#endif + +/* +** This routine does the first phase of a two-phase commit. This routine +** causes a rollback journal to be created (if it does not already exist) +** and populated with enough information so that if a power loss occurs +** the database can be restored to its original state by playing back +** the journal. Then the contents of the journal are flushed out to +** the disk. After the journal is safely on oxide, the changes to the +** database are written into the database file and flushed to oxide. +** At the end of this call, the rollback journal still exists on the +** disk and we are still holding all locks, so the transaction has not +** committed. See sqlite3BtreeCommitPhaseTwo() for the second phase of the +** commit process. +** +** This call is a no-op if no write-transaction is currently active on pBt. +** +** Otherwise, sync the database file for the btree pBt. zMaster points to +** the name of a master journal file that should be written into the +** individual journal file, or is NULL, indicating no master journal file +** (single database transaction). +** +** When this is called, the master journal should already have been +** created, populated with this journal pointer and synced to disk. +** +** Once this is routine has returned, the only thing required to commit +** the write-transaction for this database file is to delete the journal. +*/ +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ + int rc = SQLITE_OK; + if( p->inTrans==TRANS_WRITE ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + rc = autoVacuumCommit(pBt); + if( rc!=SQLITE_OK ){ + sqlite3BtreeLeave(p); + return rc; + } + } + if( pBt->bDoTruncate ){ + sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); + } +#endif + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); + sqlite3BtreeLeave(p); + } + return rc; +} + +/* +** This function is called from both BtreeCommitPhaseTwo() and BtreeRollback() +** at the conclusion of a transaction. +*/ +static void btreeEndTransaction(Btree *p){ + BtShared *pBt = p->pBt; + sqlite3 *db = p->db; + assert( sqlite3BtreeHoldsMutex(p) ); + +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->bDoTruncate = 0; +#endif + if( p->inTrans>TRANS_NONE && db->nVdbeRead>1 ){ + /* If there are other active statements that belong to this database + ** handle, downgrade to a read-only transaction. The other statements + ** may still be reading from the database. */ + downgradeAllSharedCacheTableLocks(p); + p->inTrans = TRANS_READ; + }else{ + /* If the handle had any kind of transaction open, decrement the + ** transaction count of the shared btree. If the transaction count + ** reaches 0, set the shared state to TRANS_NONE. The unlockBtreeIfUnused() + ** call below will unlock the pager. */ + if( p->inTrans!=TRANS_NONE ){ + clearAllSharedCacheTableLocks(p); + pBt->nTransaction--; + if( 0==pBt->nTransaction ){ + pBt->inTransaction = TRANS_NONE; + } + } + + /* Set the current transaction state to TRANS_NONE and unlock the + ** pager if this call closed the only read or write transaction. */ + p->inTrans = TRANS_NONE; + unlockBtreeIfUnused(pBt); + } + + btreeIntegrity(p); +} + +/* +** Commit the transaction currently in progress. +** +** This routine implements the second phase of a 2-phase commit. The +** sqlite3BtreeCommitPhaseOne() routine does the first phase and should +** be invoked prior to calling this routine. The sqlite3BtreeCommitPhaseOne() +** routine did all the work of writing information out to disk and flushing the +** contents so that they are written onto the disk platter. All this +** routine has to do is delete or truncate or zero the header in the +** the rollback journal (which causes the transaction to commit) and +** drop locks. +** +** Normally, if an error occurs while the pager layer is attempting to +** finalize the underlying journal file, this function returns an error and +** the upper layer will attempt a rollback. However, if the second argument +** is non-zero then this b-tree transaction is part of a multi-file +** transaction. In this case, the transaction has already been committed +** (by deleting a master journal file) and the caller will ignore this +** functions return code. So, even if an error occurs in the pager layer, +** reset the b-tree objects internal state to indicate that the write +** transaction has been closed. This is quite safe, as the pager will have +** transitioned to the error state. +** +** This will release the write lock on the database file. If there +** are no active cursors, it also releases the read lock. +*/ +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ + + if( p->inTrans==TRANS_NONE ) return SQLITE_OK; + sqlite3BtreeEnter(p); + btreeIntegrity(p); + + /* If the handle has a write-transaction open, commit the shared-btrees + ** transaction and set the shared state to TRANS_READ. + */ + if( p->inTrans==TRANS_WRITE ){ + int rc; + BtShared *pBt = p->pBt; + assert( pBt->inTransaction==TRANS_WRITE ); + assert( pBt->nTransaction>0 ); + rc = sqlite3PagerCommitPhaseTwo(pBt->pPager); + if( rc!=SQLITE_OK && bCleanup==0 ){ + sqlite3BtreeLeave(p); + return rc; + } + pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); + } + + btreeEndTransaction(p); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + +/* +** Do both phases of a commit. +*/ +SQLITE_PRIVATE int sqlite3BtreeCommit(Btree *p){ + int rc; + sqlite3BtreeEnter(p); + rc = sqlite3BtreeCommitPhaseOne(p, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeCommitPhaseTwo(p, 0); + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** This routine sets the state to CURSOR_FAULT and the error +** code to errCode for every cursor on BtShared that pBtree +** references. +** +** Every cursor is tripped, including cursors that belong +** to other database connections that happen to be sharing +** the cache with pBtree. +** +** This routine gets called when a rollback occurs. +** All cursors using the same cache must be tripped +** to prevent them from trying to use the btree after +** the rollback. The rollback may have deleted tables +** or moved root pages, so it is not sufficient to +** save the state of the cursor. The cursor must be +** invalidated. +*/ +SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ + BtCursor *p; + if( pBtree==0 ) return; + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + int i; + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + for(i=0; i<=p->iPage; i++){ + releasePage(p->apPage[i]); + p->apPage[i] = 0; + } + } + sqlite3BtreeLeave(pBtree); +} + +/* +** Rollback the transaction in progress. All cursors will be +** invalided by this operation. Any attempt to use a cursor +** that was open at the beginning of this operation will result +** in an error. +** +** This will release the write lock on the database file. If there +** are no active cursors, it also releases the read lock. +*/ +SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode){ + int rc; + BtShared *pBt = p->pBt; + MemPage *pPage1; + + sqlite3BtreeEnter(p); + if( tripCode==SQLITE_OK ){ + rc = tripCode = saveAllCursors(pBt, 0, 0); + }else{ + rc = SQLITE_OK; + } + if( tripCode ){ + sqlite3BtreeTripAllCursors(p, tripCode); + } + btreeIntegrity(p); + + if( p->inTrans==TRANS_WRITE ){ + int rc2; + + assert( TRANS_WRITE==pBt->inTransaction ); + rc2 = sqlite3PagerRollback(pBt->pPager); + if( rc2!=SQLITE_OK ){ + rc = rc2; + } + + /* The rollback may have destroyed the pPage1->aData value. So + ** call btreeGetPage() on page 1 again to make + ** sure pPage1->aData is set correctly. */ + if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + int nPage = get4byte(28+(u8*)pPage1->aData); + testcase( nPage==0 ); + if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); + testcase( pBt->nPage!=nPage ); + pBt->nPage = nPage; + releasePage(pPage1); + } + assert( countValidCursors(pBt, 1)==0 ); + pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); + } + + btreeEndTransaction(p); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Start a statement subtransaction. The subtransaction can can be rolled +** back independently of the main transaction. You must start a transaction +** before starting a subtransaction. The subtransaction is ended automatically +** if the main transaction commits or rolls back. +** +** Statement subtransactions are used around individual SQL statements +** that are contained within a BEGIN...COMMIT block. If a constraint +** error occurs within the statement, the effect of that one statement +** can be rolled back without having to rollback the entire transaction. +** +** A statement sub-transaction is implemented as an anonymous savepoint. The +** value passed as the second parameter is the total number of savepoints, +** including the new anonymous savepoint, open on the B-Tree. i.e. if there +** are no active savepoints and no other statement-transactions open, +** iStatement is 1. This anonymous savepoint can be released or rolled back +** using the sqlite3BtreeSavepoint() function. +*/ +SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( iStatement>0 ); + assert( iStatement>p->db->nSavepoint ); + assert( pBt->inTransaction==TRANS_WRITE ); + /* At the pager level, a statement transaction is a savepoint with + ** an index greater than all savepoints created explicitly using + ** SQL statements. It is illegal to open, release or rollback any + ** such savepoints while the statement transaction savepoint is active. + */ + rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** The second argument to this function, op, is always SAVEPOINT_ROLLBACK +** or SAVEPOINT_RELEASE. This function either releases or rolls back the +** savepoint identified by parameter iSavepoint, depending on the value +** of op. +** +** Normally, iSavepoint is greater than or equal to zero. However, if op is +** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the +** contents of the entire transaction are rolled back. This is different +** from a normal transaction rollback, as no locks are released and the +** transaction remains open. +*/ +SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ + int rc = SQLITE_OK; + if( p && p->inTrans==TRANS_WRITE ){ + BtShared *pBt = p->pBt; + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); + sqlite3BtreeEnter(p); + rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + if( rc==SQLITE_OK ){ + if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ + pBt->nPage = 0; + } + rc = newDatabase(pBt); + pBt->nPage = get4byte(28 + pBt->pPage1->aData); + + /* The database size was written into the offset 28 of the header + ** when the transaction started, so we know that the value at offset + ** 28 is nonzero. */ + assert( pBt->nPage>0 ); + } + sqlite3BtreeLeave(p); + } + return rc; +} + +/* +** Create a new cursor for the BTree whose root is on the page +** iTable. If a read-only cursor is requested, it is assumed that +** the caller already has at least a read-only transaction open +** on the database already. If a write-cursor is requested, then +** the caller is assumed to have an open write transaction. +** +** If wrFlag==0, then the cursor can only be used for reading. +** If wrFlag==1, then the cursor can be used for reading or for +** writing if other conditions for writing are also met. These +** are the conditions that must be met in order for writing to +** be allowed: +** +** 1: The cursor must have been opened with wrFlag==1 +** +** 2: Other database connections that share the same pager cache +** but which are not in the READ_UNCOMMITTED state may not have +** cursors open with wrFlag==0 on the same table. Otherwise +** the changes made by this write cursor would be visible to +** the read cursors in the other database connection. +** +** 3: The database must be writable (not on read-only media) +** +** 4: There must be an active transaction. +** +** No checking is done to make sure that page iTable really is the +** root page of a b-tree. If it is not, then the cursor acquired +** will not work correctly. +** +** It is assumed that the sqlite3BtreeCursorZero() has been called +** on pCur to initialize the memory space prior to invoking this routine. +*/ +static int btreeCursor( + Btree *p, /* The btree */ + int iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to comparison function */ + BtCursor *pCur /* Space for new cursor */ +){ + BtShared *pBt = p->pBt; /* Shared b-tree handle */ + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( wrFlag==0 || wrFlag==1 ); + + /* The following assert statements verify that if this is a sharable + ** b-tree database, the connection is holding the required table locks, + ** and that no other connection has any open cursor that conflicts with + ** this lock. */ + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, wrFlag+1) ); + assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); + + /* Assert that the caller has opened the required transaction. */ + assert( p->inTrans>TRANS_NONE ); + assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); + assert( pBt->pPage1 && pBt->pPage1->aData ); + + if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){ + return SQLITE_READONLY; + } + if( iTable==1 && btreePagecount(pBt)==0 ){ + assert( wrFlag==0 ); + iTable = 0; + } + + /* Now that no other errors can occur, finish filling in the BtCursor + ** variables and link the cursor into the BtShared list. */ + pCur->pgnoRoot = (Pgno)iTable; + pCur->iPage = -1; + pCur->pKeyInfo = pKeyInfo; + pCur->pBtree = p; + pCur->pBt = pBt; + assert( wrFlag==0 || wrFlag==BTCF_WriteFlag ); + pCur->curFlags = wrFlag; + pCur->pNext = pBt->pCursor; + if( pCur->pNext ){ + pCur->pNext->pPrev = pCur; + } + pBt->pCursor = pCur; + pCur->eState = CURSOR_INVALID; + return SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3BtreeCursor( + Btree *p, /* The btree */ + int iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to xCompare() */ + BtCursor *pCur /* Write new cursor here */ +){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Return the size of a BtCursor object in bytes. +** +** This interfaces is needed so that users of cursors can preallocate +** sufficient storage to hold a cursor. The BtCursor object is opaque +** to users so they cannot do the sizeof() themselves - they must call +** this routine. +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){ + return ROUND8(sizeof(BtCursor)); +} + +/* +** Initialize memory that will be converted into a BtCursor object. +** +** The simple approach here would be to memset() the entire object +** to zero. But it turns out that the apPage[] and aiIdx[] arrays +** do not need to be zeroed and they are large, so we can save a lot +** of run-time by skipping the initialization of those elements. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){ + memset(p, 0, offsetof(BtCursor, iPage)); +} + +/* +** Close a cursor. The read lock on the database file is released +** when the last cursor is closed. +*/ +SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ + Btree *pBtree = pCur->pBtree; + if( pBtree ){ + int i; + BtShared *pBt = pCur->pBt; + sqlite3BtreeEnter(pBtree); + sqlite3BtreeClearCursor(pCur); + if( pCur->pPrev ){ + pCur->pPrev->pNext = pCur->pNext; + }else{ + pBt->pCursor = pCur->pNext; + } + if( pCur->pNext ){ + pCur->pNext->pPrev = pCur->pPrev; + } + for(i=0; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + } + unlockBtreeIfUnused(pBt); + sqlite3DbFree(pBtree->db, pCur->aOverflow); + /* sqlite3_free(pCur); */ + sqlite3BtreeLeave(pBtree); + } + return SQLITE_OK; +} + +/* +** Make sure the BtCursor* given in the argument has a valid +** BtCursor.info structure. If it is not already valid, call +** btreeParseCell() to fill it in. +** +** BtCursor.info is a cache of the information in the current cell. +** Using this cache reduces the number of calls to btreeParseCell(). +** +** 2007-06-25: There is a bug in some versions of MSVC that cause the +** compiler to crash when getCellInfo() is implemented as a macro. +** But there is a measureable speed advantage to using the macro on gcc +** (when less compiler optimizations like -Os or -O0 are used and the +** compiler is not doing agressive inlining.) So we use a real function +** for MSVC and a macro for everything else. Ticket #2457. +*/ +#ifndef NDEBUG + static void assertCellInfo(BtCursor *pCur){ + CellInfo info; + int iPage = pCur->iPage; + memset(&info, 0, sizeof(info)); + btreeParseCell(pCur->apPage[iPage], pCur->aiIdx[iPage], &info); + assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 ); + } +#else + #define assertCellInfo(x) +#endif +#ifdef _MSC_VER + /* Use a real function in MSVC to work around bugs in that compiler. */ + static void getCellInfo(BtCursor *pCur){ + if( pCur->info.nSize==0 ){ + int iPage = pCur->iPage; + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); + pCur->curFlags |= BTCF_ValidNKey; + }else{ + assertCellInfo(pCur); + } + } +#else /* if not _MSC_VER */ + /* Use a macro in all other compilers so that the function is inlined */ +#define getCellInfo(pCur) \ + if( pCur->info.nSize==0 ){ \ + int iPage = pCur->iPage; \ + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ + pCur->curFlags |= BTCF_ValidNKey; \ + }else{ \ + assertCellInfo(pCur); \ + } +#endif /* _MSC_VER */ + +#ifndef NDEBUG /* The next routine used only within assert() statements */ +/* +** Return true if the given BtCursor is valid. A valid cursor is one +** that is currently pointing to a row in a (non-empty) table. +** This is a verification routine is used only within assert() statements. +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor *pCur){ + return pCur && pCur->eState==CURSOR_VALID; +} +#endif /* NDEBUG */ + +/* +** Set *pSize to the size of the buffer needed to hold the value of +** the key for the current entry. If the cursor is not pointing +** to a valid entry, *pSize is set to 0. +** +** For a table with the INTKEY flag set, this routine returns the key +** itself, not the number of bytes in the key. +** +** The caller must position the cursor prior to invoking this routine. +** +** This routine cannot fail. It always returns SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID ); + if( pCur->eState!=CURSOR_VALID ){ + *pSize = 0; + }else{ + getCellInfo(pCur); + *pSize = pCur->info.nKey; + } + return SQLITE_OK; +} + +/* +** Set *pSize to the number of bytes of data in the entry the +** cursor currently points to. +** +** The caller must guarantee that the cursor is pointing to a non-NULL +** valid entry. In other words, the calling procedure must guarantee +** that the cursor has Cursor.eState==CURSOR_VALID. +** +** Failure is not possible. This function always returns SQLITE_OK. +** It might just as well be a procedure (returning void) but we continue +** to return an integer result code for historical reasons. +*/ +SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + *pSize = pCur->info.nData; + return SQLITE_OK; +} + +/* +** Given the page number of an overflow page in the database (parameter +** ovfl), this function finds the page number of the next page in the +** linked list of overflow pages. If possible, it uses the auto-vacuum +** pointer-map data instead of reading the content of page ovfl to do so. +** +** If an error occurs an SQLite error code is returned. Otherwise: +** +** The page number of the next overflow page in the linked list is +** written to *pPgnoNext. If page ovfl is the last page in its linked +** list, *pPgnoNext is set to zero. +** +** If ppPage is not NULL, and a reference to the MemPage object corresponding +** to page number pOvfl was obtained, then *ppPage is set to point to that +** reference. It is the responsibility of the caller to call releasePage() +** on *ppPage to free the reference. In no reference was obtained (because +** the pointer-map was used to obtain the value for *pPgnoNext), then +** *ppPage is set to zero. +*/ +static int getOverflowPage( + BtShared *pBt, /* The database file */ + Pgno ovfl, /* Current overflow page number */ + MemPage **ppPage, /* OUT: MemPage handle (may be NULL) */ + Pgno *pPgnoNext /* OUT: Next overflow page number */ +){ + Pgno next = 0; + MemPage *pPage = 0; + int rc = SQLITE_OK; + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert(pPgnoNext); + +#ifndef SQLITE_OMIT_AUTOVACUUM + /* Try to find the next page in the overflow list using the + ** autovacuum pointer-map pages. Guess that the next page in + ** the overflow list is page number (ovfl+1). If that guess turns + ** out to be wrong, fall back to loading the data of page + ** number ovfl to determine the next page number. + */ + if( pBt->autoVacuum ){ + Pgno pgno; + Pgno iGuess = ovfl+1; + u8 eType; + + while( PTRMAP_ISPAGE(pBt, iGuess) || iGuess==PENDING_BYTE_PAGE(pBt) ){ + iGuess++; + } + + if( iGuess<=btreePagecount(pBt) ){ + rc = ptrmapGet(pBt, iGuess, &eType, &pgno); + if( rc==SQLITE_OK && eType==PTRMAP_OVERFLOW2 && pgno==ovfl ){ + next = iGuess; + rc = SQLITE_DONE; + } + } + } +#endif + + assert( next==0 || rc==SQLITE_DONE ); + if( rc==SQLITE_OK ){ + rc = btreeGetPage(pBt, ovfl, &pPage, (ppPage==0) ? PAGER_GET_READONLY : 0); + assert( rc==SQLITE_OK || pPage==0 ); + if( rc==SQLITE_OK ){ + next = get4byte(pPage->aData); + } + } + + *pPgnoNext = next; + if( ppPage ){ + *ppPage = pPage; + }else{ + releasePage(pPage); + } + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Copy data from a buffer to a page, or from a page to a buffer. +** +** pPayload is a pointer to data stored on database page pDbPage. +** If argument eOp is false, then nByte bytes of data are copied +** from pPayload to the buffer pointed at by pBuf. If eOp is true, +** then sqlite3PagerWrite() is called on pDbPage and nByte bytes +** of data are copied from the buffer pBuf to pPayload. +** +** SQLITE_OK is returned on success, otherwise an error code. +*/ +static int copyPayload( + void *pPayload, /* Pointer to page data */ + void *pBuf, /* Pointer to buffer */ + int nByte, /* Number of bytes to copy */ + int eOp, /* 0 -> copy from page, 1 -> copy to page */ + DbPage *pDbPage /* Page containing pPayload */ +){ + if( eOp ){ + /* Copy data from buffer to page (a write operation) */ + int rc = sqlite3PagerWrite(pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + memcpy(pPayload, pBuf, nByte); + }else{ + /* Copy data from page to buffer (a read operation) */ + memcpy(pBuf, pPayload, nByte); + } + return SQLITE_OK; +} + +/* +** This function is used to read or overwrite payload information +** for the entry that the pCur cursor is pointing to. The eOp +** argument is interpreted as follows: +** +** 0: The operation is a read. Populate the overflow cache. +** 1: The operation is a write. Populate the overflow cache. +** 2: The operation is a read. Do not populate the overflow cache. +** +** A total of "amt" bytes are read or written beginning at "offset". +** Data is read to or from the buffer pBuf. +** +** The content being read or written might appear on the main page +** or be scattered out on multiple overflow pages. +** +** If the current cursor entry uses one or more overflow pages and the +** eOp argument is not 2, this function may allocate space for and lazily +** popluates the overflow page-list cache array (BtCursor.aOverflow). +** Subsequent calls use this cache to make seeking to the supplied offset +** more efficient. +** +** Once an overflow page-list cache has been allocated, it may be +** invalidated if some other cursor writes to the same table, or if +** the cursor is moved to a different row. Additionally, in auto-vacuum +** mode, the following events may invalidate an overflow page-list cache. +** +** * An incremental vacuum, +** * A commit in auto_vacuum="full" mode, +** * Creating a table (may require moving an overflow page). +*/ +static int accessPayload( + BtCursor *pCur, /* Cursor pointing to entry to read from */ + u32 offset, /* Begin reading this far into payload */ + u32 amt, /* Read this many bytes */ + unsigned char *pBuf, /* Write the bytes into this buffer */ + int eOp /* zero to read. non-zero to write. */ +){ + unsigned char *aPayload; + int rc = SQLITE_OK; + u32 nKey; + int iIdx = 0; + MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ + BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ +#ifdef SQLITE_DIRECT_OVERFLOW_READ + int bEnd; /* True if reading to end of data */ +#endif + + assert( pPage ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->aiIdx[pCur->iPage]nCell ); + assert( cursorHoldsMutex(pCur) ); + assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ + + getCellInfo(pCur); + aPayload = pCur->info.pCell + pCur->info.nHeader; + nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); +#ifdef SQLITE_DIRECT_OVERFLOW_READ + bEnd = (offset+amt==nKey+pCur->info.nData); +#endif + + if( NEVER(offset+amt > nKey+pCur->info.nData) + || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ){ + /* Trying to read or write past the end of the data is an error */ + return SQLITE_CORRUPT_BKPT; + } + + /* Check if data must be read/written to/from the btree page itself. */ + if( offsetinfo.nLocal ){ + int a = amt; + if( a+offset>pCur->info.nLocal ){ + a = pCur->info.nLocal - offset; + } + rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage); + offset = 0; + pBuf += a; + amt -= a; + }else{ + offset -= pCur->info.nLocal; + } + + if( rc==SQLITE_OK && amt>0 ){ + const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ + Pgno nextPage; + + nextPage = get4byte(&aPayload[pCur->info.nLocal]); + + /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. + ** Except, do not allocate aOverflow[] for eOp==2. + ** + ** The aOverflow[] array is sized at one entry for each overflow page + ** in the overflow chain. The page number of the first overflow page is + ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array + ** means "not yet known" (the cache is lazily populated). + */ + if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ + int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; + if( nOvfl>pCur->nOvflAlloc ){ + Pgno *aNew = (Pgno*)sqlite3DbRealloc( + pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno) + ); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pCur->nOvflAlloc = nOvfl*2; + pCur->aOverflow = aNew; + } + } + if( rc==SQLITE_OK ){ + memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); + pCur->curFlags |= BTCF_ValidOvfl; + } + } + + /* If the overflow page-list cache has been allocated and the + ** entry for the first required overflow page is valid, skip + ** directly to it. + */ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[offset/ovflSize] ){ + iIdx = (offset/ovflSize); + nextPage = pCur->aOverflow[iIdx]; + offset = (offset%ovflSize); + } + + for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ + + /* If required, populate the overflow page-list cache. */ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ + assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); + pCur->aOverflow[iIdx] = nextPage; + } + + if( offset>=ovflSize ){ + /* The only reason to read this page is to obtain the page + ** number for the next page in the overflow chain. The page + ** data is not required. So first try to lookup the overflow + ** page-list cache, if any, then fall back to the getOverflowPage() + ** function. + ** + ** Note that the aOverflow[] array must be allocated because eOp!=2 + ** here. If eOp==2, then offset==0 and this branch is never taken. + */ + assert( eOp!=2 ); + assert( pCur->curFlags & BTCF_ValidOvfl ); + if( pCur->aOverflow[iIdx+1] ){ + nextPage = pCur->aOverflow[iIdx+1]; + }else{ + rc = getOverflowPage(pBt, nextPage, 0, &nextPage); + } + offset -= ovflSize; + }else{ + /* Need to read this page properly. It contains some of the + ** range of data that is being read (eOp==0) or written (eOp!=0). + */ +#ifdef SQLITE_DIRECT_OVERFLOW_READ + sqlite3_file *fd; +#endif + int a = amt; + if( a + offset > ovflSize ){ + a = ovflSize - offset; + } + +#ifdef SQLITE_DIRECT_OVERFLOW_READ + /* If all the following are true: + ** + ** 1) this is a read operation, and + ** 2) data is required from the start of this overflow page, and + ** 3) the database is file-backed, and + ** 4) there is no open write-transaction, and + ** 5) the database is not a WAL database, + ** 6) all data from the page is being read. + ** + ** then data can be read directly from the database file into the + ** output buffer, bypassing the page-cache altogether. This speeds + ** up loading large records that span many overflow pages. + */ + if( (eOp&0x01)==0 /* (1) */ + && offset==0 /* (2) */ + && (bEnd || a==ovflSize) /* (6) */ + && pBt->inTransaction==TRANS_READ /* (4) */ + && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ + && pBt->pPage1->aData[19]==0x01 /* (5) */ + ){ + u8 aSave[4]; + u8 *aWrite = &pBuf[-4]; + memcpy(aSave, aWrite, 4); + rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); + nextPage = get4byte(aWrite); + memcpy(aWrite, aSave, 4); + }else +#endif + + { + DbPage *pDbPage; + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) + ); + if( rc==SQLITE_OK ){ + aPayload = sqlite3PagerGetData(pDbPage); + nextPage = get4byte(aPayload); + rc = copyPayload(&aPayload[offset+4], pBuf, a, (eOp&0x01), pDbPage); + sqlite3PagerUnref(pDbPage); + offset = 0; + } + } + amt -= a; + pBuf += a; + } + } + } + + if( rc==SQLITE_OK && amt>0 ){ + return SQLITE_CORRUPT_BKPT; + } + return rc; +} + +/* +** Read part of the key associated with cursor pCur. Exactly +** "amt" bytes will be transfered into pBuf[]. The transfer +** begins at "offset". +** +** The caller must ensure that pCur is pointing to a valid row +** in the table. +** +** Return SQLITE_OK on success or an error code if anything goes +** wrong. An error is returned if "offset+amt" is larger than +** the available payload. +*/ +SQLITE_PRIVATE int sqlite3BtreeKey(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0); +} + +/* +** Read part of the data associated with cursor pCur. Exactly +** "amt" bytes will be transfered into pBuf[]. The transfer +** begins at "offset". +** +** Return SQLITE_OK on success or an error code if anything goes +** wrong. An error is returned if "offset+amt" is larger than +** the available payload. +*/ +SQLITE_PRIVATE int sqlite3BtreeData(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + int rc; + +#ifndef SQLITE_OMIT_INCRBLOB + if ( pCur->eState==CURSOR_INVALID ){ + return SQLITE_ABORT; + } +#endif + + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + rc = accessPayload(pCur, offset, amt, pBuf, 0); + } + return rc; +} + +/* +** Return a pointer to payload information from the entry that the +** pCur cursor is pointing to. The pointer is to the beginning of +** the key if index btrees (pPage->intKey==0) and is the data for +** table btrees (pPage->intKey==1). The number of bytes of available +** key/data is written into *pAmt. If *pAmt==0, then the value +** returned will not be a valid pointer. +** +** This routine is an optimization. It is common for the entire key +** and data to fit on the local page and for there to be no overflow +** pages. When that is so, this routine can be used to access the +** key and data without making a copy. If the key and/or data spills +** onto overflow pages, then accessPayload() must be used to reassemble +** the key/data and copy it into a preallocated buffer. +** +** The pointer returned by this routine looks directly into the cached +** page of the database. The data might change or move the next time +** any btree routine is called. +*/ +static const void *fetchPayload( + BtCursor *pCur, /* Cursor pointing to entry to read from */ + u32 *pAmt /* Write the number of available bytes here */ +){ + assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); + assert( pCur->eState==CURSOR_VALID ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( cursorHoldsMutex(pCur) ); + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + assert( pCur->info.nSize>0 ); + *pAmt = pCur->info.nLocal; + return (void*)(pCur->info.pCell + pCur->info.nHeader); +} + + +/* +** For the entry that cursor pCur is point to, return as +** many bytes of the key or data as are available on the local +** b-tree page. Write the number of available bytes into *pAmt. +** +** The pointer returned is ephemeral. The key/data may move +** or be destroyed on the next call to any Btree routine, +** including calls from other threads against the same cache. +** Hence, a mutex on the BtShared should be held prior to calling +** this routine. +** +** These routines is used to get quick access to key and data +** in the common case where no overflow pages are used. +*/ +SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor *pCur, u32 *pAmt){ + return fetchPayload(pCur, pAmt); +} +SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ + return fetchPayload(pCur, pAmt); +} + + +/* +** Move the cursor down to a new child page. The newPgno argument is the +** page number of the child page to move to. +** +** This function returns SQLITE_CORRUPT if the page-header flags field of +** the new child page does not match the flags field of the parent (i.e. +** if an intkey page appears to be the parent of a non-intkey page, or +** vice-versa). +*/ +static int moveToChild(BtCursor *pCur, u32 newPgno){ + int rc; + int i = pCur->iPage; + MemPage *pNewPage; + BtShared *pBt = pCur->pBt; + + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPageiPage>=0 ); + if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ + return SQLITE_CORRUPT_BKPT; + } + rc = getAndInitPage(pBt, newPgno, &pNewPage, + (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); + if( rc ) return rc; + pCur->apPage[i+1] = pNewPage; + pCur->aiIdx[i+1] = 0; + pCur->iPage++; + + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){ + return SQLITE_CORRUPT_BKPT; + } + return SQLITE_OK; +} + +#if 0 +/* +** Page pParent is an internal (non-leaf) tree page. This function +** asserts that page number iChild is the left-child if the iIdx'th +** cell in page pParent. Or, if iIdx is equal to the total number of +** cells in pParent, that page number iChild is the right-child of +** the page. +*/ +static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ + assert( iIdx<=pParent->nCell ); + if( iIdx==pParent->nCell ){ + assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild ); + }else{ + assert( get4byte(findCell(pParent, iIdx))==iChild ); + } +} +#else +# define assertParentIndex(x,y,z) +#endif + +/* +** Move the cursor up to the parent page. +** +** pCur->idx is set to the cell index that contains the pointer +** to the page we are coming from. If we are coming from the +** right-most child page then pCur->idx is set to one more than +** the largest cell index. +*/ +static void moveToParent(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>0 ); + assert( pCur->apPage[pCur->iPage] ); + + /* UPDATE: It is actually possible for the condition tested by the assert + ** below to be untrue if the database file is corrupt. This can occur if + ** one cursor has modified page pParent while a reference to it is held + ** by a second cursor. Which can only happen if a single page is linked + ** into more than one b-tree structure in a corrupt database. */ +#if 0 + assertParentIndex( + pCur->apPage[pCur->iPage-1], + pCur->aiIdx[pCur->iPage-1], + pCur->apPage[pCur->iPage]->pgno + ); +#endif + testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); + + releasePage(pCur->apPage[pCur->iPage]); + pCur->iPage--; + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); +} + +/* +** Move the cursor to point to the root page of its b-tree structure. +** +** If the table has a virtual root page, then the cursor is moved to point +** to the virtual root page instead of the actual root page. A table has a +** virtual root page when the actual root page contains no cells and a +** single child page. This can only happen with the table rooted at page 1. +** +** If the b-tree structure is empty, the cursor state is set to +** CURSOR_INVALID. Otherwise, the cursor is set to point to the first +** cell located on the root (or virtual root) page and the cursor state +** is set to CURSOR_VALID. +** +** If this function returns successfully, it may be assumed that the +** page-header flags indicate that the [virtual] root-page is the expected +** kind of b-tree page (i.e. if when opening the cursor the caller did not +** specify a KeyInfo structure the flags byte is set to 0x05 or 0x0D, +** indicating a table b-tree, or if the caller did specify a KeyInfo +** structure the flags byte is set to 0x02 or 0x0A, indicating an index +** b-tree). +*/ +static int moveToRoot(BtCursor *pCur){ + MemPage *pRoot; + int rc = SQLITE_OK; + + assert( cursorHoldsMutex(pCur) ); + assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; + } + sqlite3BtreeClearCursor(pCur); + } + + if( pCur->iPage>=0 ){ + while( pCur->iPage ) releasePage(pCur->apPage[pCur->iPage--]); + }else if( pCur->pgnoRoot==0 ){ + pCur->eState = CURSOR_INVALID; + return SQLITE_OK; + }else{ + rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0], + (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); + if( rc!=SQLITE_OK ){ + pCur->eState = CURSOR_INVALID; + return rc; + } + pCur->iPage = 0; + } + pRoot = pCur->apPage[0]; + assert( pRoot->pgno==pCur->pgnoRoot ); + + /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor + ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is + ** NULL, the caller expects a table b-tree. If this is not the case, + ** return an SQLITE_CORRUPT error. + ** + ** Earlier versions of SQLite assumed that this test could not fail + ** if the root page was already loaded when this function was called (i.e. + ** if pCur->iPage>=0). But this is not so if the database is corrupted + ** in such a way that page pRoot is linked into a second b-tree table + ** (or the freelist). */ + assert( pRoot->intKey==1 || pRoot->intKey==0 ); + if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ + return SQLITE_CORRUPT_BKPT; + } + + pCur->aiIdx[0] = 0; + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); + + if( pRoot->nCell>0 ){ + pCur->eState = CURSOR_VALID; + }else if( !pRoot->leaf ){ + Pgno subpage; + if( pRoot->pgno!=1 ) return SQLITE_CORRUPT_BKPT; + subpage = get4byte(&pRoot->aData[pRoot->hdrOffset+8]); + pCur->eState = CURSOR_VALID; + rc = moveToChild(pCur, subpage); + }else{ + pCur->eState = CURSOR_INVALID; + } + return rc; +} + +/* +** Move the cursor down to the left-most leaf entry beneath the +** entry to which it is currently pointing. +** +** The left-most leaf is the one with the smallest key - the first +** in ascending order. +*/ +static int moveToLeftmost(BtCursor *pCur){ + Pgno pgno; + int rc = SQLITE_OK; + MemPage *pPage; + + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + assert( pCur->aiIdx[pCur->iPage]nCell ); + pgno = get4byte(findCell(pPage, pCur->aiIdx[pCur->iPage])); + rc = moveToChild(pCur, pgno); + } + return rc; +} + +/* +** Move the cursor down to the right-most leaf entry beneath the +** page to which it is currently pointing. Notice the difference +** between moveToLeftmost() and moveToRightmost(). moveToLeftmost() +** finds the left-most entry beneath the *entry* whereas moveToRightmost() +** finds the right-most entry beneath the *page*. +** +** The right-most entry is the one with the largest key - the last +** key in ascending order. +*/ +static int moveToRightmost(BtCursor *pCur){ + Pgno pgno; + int rc = SQLITE_OK; + MemPage *pPage = 0; + + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + pCur->aiIdx[pCur->iPage] = pPage->nCell; + rc = moveToChild(pCur, pgno); + } + if( rc==SQLITE_OK ){ + pCur->aiIdx[pCur->iPage] = pPage->nCell-1; + pCur->info.nSize = 0; + pCur->curFlags &= ~BTCF_ValidNKey; + } + return rc; +} + +/* Move the cursor to the first entry in the table. Return SQLITE_OK +** on success. Set *pRes to 0 if the cursor actually points to something +** or set *pRes to 1 if the table is empty. +*/ +SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + if( pCur->eState==CURSOR_INVALID ){ + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); + *pRes = 1; + }else{ + assert( pCur->apPage[pCur->iPage]->nCell>0 ); + *pRes = 0; + rc = moveToLeftmost(pCur); + } + } + return rc; +} + +/* Move the cursor to the last entry in the table. Return SQLITE_OK +** on success. Set *pRes to 0 if the cursor actually points to something +** or set *pRes to 1 if the table is empty. +*/ +SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* If the cursor already points to the last entry, this is a no-op. */ + if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ +#ifdef SQLITE_DEBUG + /* This block serves to assert() that the cursor really does point + ** to the last entry in the b-tree. */ + int ii; + for(ii=0; iiiPage; ii++){ + assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); + } + assert( pCur->aiIdx[pCur->iPage]==pCur->apPage[pCur->iPage]->nCell-1 ); + assert( pCur->apPage[pCur->iPage]->leaf ); +#endif + return SQLITE_OK; + } + + rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + if( CURSOR_INVALID==pCur->eState ){ + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); + *pRes = 1; + }else{ + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + + } + } + return rc; +} + +/* Move the cursor so that it points to an entry near the key +** specified by pIdxKey or intKey. Return a success code. +** +** For INTKEY tables, the intKey parameter is used. pIdxKey +** must be NULL. For index tables, pIdxKey is used and intKey +** is ignored. +** +** If an exact match is not found, then the cursor is always +** left pointing at a leaf page which would hold the entry if it +** were present. The cursor might point to an entry that comes +** before or after the key. +** +** An integer is written into *pRes which is the result of +** comparing the key with the entry to which the cursor is +** pointing. The meaning of the integer written into +** *pRes is as follows: +** +** *pRes<0 The cursor is left pointing at an entry that +** is smaller than intKey/pIdxKey or if the table is empty +** and the cursor is therefore left point to nothing. +** +** *pRes==0 The cursor is left pointing at an entry that +** exactly matches intKey/pIdxKey. +** +** *pRes>0 The cursor is left pointing at an entry that +** is larger than intKey/pIdxKey. +** +*/ +SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( + BtCursor *pCur, /* The cursor to be moved */ + UnpackedRecord *pIdxKey, /* Unpacked index key */ + i64 intKey, /* The table key */ + int biasRight, /* If true, bias the search to the high end */ + int *pRes /* Write search results here */ +){ + int rc; + RecordCompare xRecordCompare; + + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( pRes ); + assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); + + /* If the cursor is already positioned at the point we are trying + ** to move to, then just return without doing any work */ + if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 + && pCur->apPage[0]->intKey + ){ + if( pCur->info.nKey==intKey ){ + *pRes = 0; + return SQLITE_OK; + } + if( (pCur->curFlags & BTCF_AtLast)!=0 && pCur->info.nKeyisCorrupt = 0; + assert( pIdxKey->default_rc==1 + || pIdxKey->default_rc==0 + || pIdxKey->default_rc==-1 + ); + }else{ + xRecordCompare = 0; /* All keys are integers */ + } + + rc = moveToRoot(pCur); + if( rc ){ + return rc; + } + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage] ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->isInit ); + assert( pCur->eState==CURSOR_INVALID || pCur->apPage[pCur->iPage]->nCell>0 ); + if( pCur->eState==CURSOR_INVALID ){ + *pRes = -1; + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); + return SQLITE_OK; + } + assert( pCur->apPage[0]->intKey || pIdxKey ); + for(;;){ + int lwr, upr, idx, c; + Pgno chldPg; + MemPage *pPage = pCur->apPage[pCur->iPage]; + u8 *pCell; /* Pointer to current cell in pPage */ + + /* pPage->nCell must be greater than zero. If this is the root-page + ** the cursor would have been INVALID above and this for(;;) loop + ** not run. If this is not the root-page, then the moveToChild() routine + ** would have already detected db corruption. Similarly, pPage must + ** be the right kind (index or table) of b-tree page. Otherwise + ** a moveToChild() or moveToRoot() call would have detected corruption. */ + assert( pPage->nCell>0 ); + assert( pPage->intKey==(pIdxKey==0) ); + lwr = 0; + upr = pPage->nCell-1; + assert( biasRight==0 || biasRight==1 ); + idx = upr>>(1-biasRight); /* idx = biasRight ? upr : (lwr+upr)/2; */ + pCur->aiIdx[pCur->iPage] = (u16)idx; + if( xRecordCompare==0 ){ + for(;;){ + i64 nCellKey; + pCell = findCell(pPage, idx) + pPage->childPtrSize; + if( pPage->hasData ){ + while( 0x80 <= *(pCell++) ){ + if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; + } + } + getVarint(pCell, (u64*)&nCellKey); + if( nCellKeyupr ){ c = -1; break; } + }else if( nCellKey>intKey ){ + upr = idx-1; + if( lwr>upr ){ c = +1; break; } + }else{ + assert( nCellKey==intKey ); + pCur->curFlags |= BTCF_ValidNKey; + pCur->info.nKey = nCellKey; + pCur->aiIdx[pCur->iPage] = (u16)idx; + if( !pPage->leaf ){ + lwr = idx; + goto moveto_next_layer; + }else{ + *pRes = 0; + rc = SQLITE_OK; + goto moveto_finish; + } + } + assert( lwr+upr>=0 ); + idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2; */ + } + }else{ + for(;;){ + int nCell; + pCell = findCell(pPage, idx) + pPage->childPtrSize; + + /* The maximum supported page-size is 65536 bytes. This means that + ** the maximum number of record bytes stored on an index B-Tree + ** page is less than 16384 bytes and may be stored as a 2-byte + ** varint. This information is used to attempt to avoid parsing + ** the entire cell by checking for the cases where the record is + ** stored entirely within the b-tree page by inspecting the first + ** 2 bytes of the cell. + */ + nCell = pCell[0]; + if( nCell<=pPage->max1bytePayload ){ + /* This branch runs if the record-size field of the cell is a + ** single byte varint and the record fits entirely on the main + ** b-tree page. */ + testcase( pCell+nCell+1==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey, 0); + }else if( !(pCell[1] & 0x80) + && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal + ){ + /* The record-size field is a 2 byte varint and the record + ** fits entirely on the main b-tree page. */ + testcase( pCell+nCell+2==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey, 0); + }else{ + /* The record flows over onto one or more overflow pages. In + ** this case the whole cell needs to be parsed, a buffer allocated + ** and accessPayload() used to retrieve the record into the + ** buffer before VdbeRecordCompare() can be called. */ + void *pCellKey; + u8 * const pCellBody = pCell - pPage->childPtrSize; + btreeParseCellPtr(pPage, pCellBody, &pCur->info); + nCell = (int)pCur->info.nKey; + pCellKey = sqlite3Malloc( nCell ); + if( pCellKey==0 ){ + rc = SQLITE_NOMEM; + goto moveto_finish; + } + pCur->aiIdx[pCur->iPage] = (u16)idx; + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 2); + if( rc ){ + sqlite3_free(pCellKey); + goto moveto_finish; + } + c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); + sqlite3_free(pCellKey); + } + assert( pIdxKey->isCorrupt==0 || c==0 ); + if( c<0 ){ + lwr = idx+1; + }else if( c>0 ){ + upr = idx-1; + }else{ + assert( c==0 ); + *pRes = 0; + rc = SQLITE_OK; + pCur->aiIdx[pCur->iPage] = (u16)idx; + if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT; + goto moveto_finish; + } + if( lwr>upr ) break; + assert( lwr+upr>=0 ); + idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2 */ + } + } + assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) ); + assert( pPage->isInit ); + if( pPage->leaf ){ + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + pCur->aiIdx[pCur->iPage] = (u16)idx; + *pRes = c; + rc = SQLITE_OK; + goto moveto_finish; + } +moveto_next_layer: + if( lwr>=pPage->nCell ){ + chldPg = get4byte(&pPage->aData[pPage->hdrOffset+8]); + }else{ + chldPg = get4byte(findCell(pPage, lwr)); + } + pCur->aiIdx[pCur->iPage] = (u16)lwr; + rc = moveToChild(pCur, chldPg); + if( rc ) break; + } +moveto_finish: + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + return rc; +} + + +/* +** Return TRUE if the cursor is not pointing at an entry of the table. +** +** TRUE will be returned after a call to sqlite3BtreeNext() moves +** past the last entry in the table or sqlite3BtreePrev() moves past +** the first entry. TRUE is also returned if the table is empty. +*/ +SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor *pCur){ + /* TODO: What if the cursor is in CURSOR_REQUIRESEEK but all table entries + ** have been deleted? This API will need to change to return an error code + ** as well as the boolean result value. + */ + return (CURSOR_VALID!=pCur->eState); +} + +/* +** Advance the cursor to the next entry in the database. If +** successful then set *pRes=0. If the cursor +** was already pointing to the last entry in the database before +** this routine was called, then set *pRes=1. +** +** The calling function will set *pRes to 0 or 1. The initial *pRes value +** will be 1 if the cursor being stepped corresponds to an SQL index and +** if this routine could have been skipped if that SQL index had been +** a unique index. Otherwise the caller will have set *pRes to zero. +** Zero is the common case. The btree implementation is free to use the +** initial *pRes value as a hint to improve performance, but the current +** SQLite btree implementation does not. (Note that the comdb2 btree +** implementation does use this hint, however.) +*/ +SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ + int rc; + int idx; + MemPage *pPage; + + assert( cursorHoldsMutex(pCur) ); + assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); + assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + if( pCur->eState!=CURSOR_VALID ){ + invalidateOverflowCache(pCur); + rc = restoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + *pRes = 0; + return rc; + } + if( CURSOR_INVALID==pCur->eState ){ + *pRes = 1; + return SQLITE_OK; + } + if( pCur->skipNext ){ + assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT ); + pCur->eState = CURSOR_VALID; + if( pCur->skipNext>0 ){ + pCur->skipNext = 0; + *pRes = 0; + return SQLITE_OK; + } + pCur->skipNext = 0; + } + } + + pPage = pCur->apPage[pCur->iPage]; + idx = ++pCur->aiIdx[pCur->iPage]; + assert( pPage->isInit ); + + /* If the database file is corrupt, it is possible for the value of idx + ** to be invalid here. This can only occur if a second cursor modifies + ** the page while cursor pCur is holding a reference to it. Which can + ** only happen if the database is corrupt in such a way as to link the + ** page into more than one b-tree structure. */ + testcase( idx>pPage->nCell ); + + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + if( idx>=pPage->nCell ){ + if( !pPage->leaf ){ + rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); + if( rc ){ + *pRes = 0; + return rc; + } + rc = moveToLeftmost(pCur); + *pRes = 0; + return rc; + } + do{ + if( pCur->iPage==0 ){ + *pRes = 1; + pCur->eState = CURSOR_INVALID; + return SQLITE_OK; + } + moveToParent(pCur); + pPage = pCur->apPage[pCur->iPage]; + }while( pCur->aiIdx[pCur->iPage]>=pPage->nCell ); + *pRes = 0; + if( pPage->intKey ){ + rc = sqlite3BtreeNext(pCur, pRes); + }else{ + rc = SQLITE_OK; + } + return rc; + } + *pRes = 0; + if( pPage->leaf ){ + return SQLITE_OK; + } + rc = moveToLeftmost(pCur); + return rc; +} + + +/* +** Step the cursor to the back to the previous entry in the database. If +** successful then set *pRes=0. If the cursor +** was already pointing to the first entry in the database before +** this routine was called, then set *pRes=1. +** +** The calling function will set *pRes to 0 or 1. The initial *pRes value +** will be 1 if the cursor being stepped corresponds to an SQL index and +** if this routine could have been skipped if that SQL index had been +** a unique index. Otherwise the caller will have set *pRes to zero. +** Zero is the common case. The btree implementation is free to use the +** initial *pRes value as a hint to improve performance, but the current +** SQLite btree implementation does not. (Note that the comdb2 btree +** implementation does use this hint, however.) +*/ +SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ + int rc; + MemPage *pPage; + + assert( cursorHoldsMutex(pCur) ); + assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); + assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl); + if( pCur->eState!=CURSOR_VALID ){ + if( ALWAYS(pCur->eState>=CURSOR_REQUIRESEEK) ){ + rc = btreeRestoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + *pRes = 0; + return rc; + } + } + if( CURSOR_INVALID==pCur->eState ){ + *pRes = 1; + return SQLITE_OK; + } + if( pCur->skipNext ){ + assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT ); + pCur->eState = CURSOR_VALID; + if( pCur->skipNext<0 ){ + pCur->skipNext = 0; + *pRes = 0; + return SQLITE_OK; + } + pCur->skipNext = 0; + } + } + + pPage = pCur->apPage[pCur->iPage]; + assert( pPage->isInit ); + if( !pPage->leaf ){ + int idx = pCur->aiIdx[pCur->iPage]; + rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); + if( rc ){ + *pRes = 0; + return rc; + } + rc = moveToRightmost(pCur); + }else{ + while( pCur->aiIdx[pCur->iPage]==0 ){ + if( pCur->iPage==0 ){ + pCur->eState = CURSOR_INVALID; + *pRes = 1; + return SQLITE_OK; + } + moveToParent(pCur); + } + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + + pCur->aiIdx[pCur->iPage]--; + pPage = pCur->apPage[pCur->iPage]; + if( pPage->intKey && !pPage->leaf ){ + rc = sqlite3BtreePrevious(pCur, pRes); + }else{ + rc = SQLITE_OK; + } + } + *pRes = 0; + return rc; +} + +/* +** Allocate a new page from the database file. +** +** The new page is marked as dirty. (In other words, sqlite3PagerWrite() +** has already been called on the new page.) The new page has also +** been referenced and the calling routine is responsible for calling +** sqlite3PagerUnref() on the new page when it is done. +** +** SQLITE_OK is returned on success. Any other return value indicates +** an error. *ppPage and *pPgno are undefined in the event of an error. +** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. +** +** If the "nearby" parameter is not 0, then an effort is made to +** locate a page close to the page number "nearby". This can be used in an +** attempt to keep related pages close to each other in the database file, +** which in turn can make database access faster. +** +** If the eMode parameter is BTALLOC_EXACT and the nearby page exists +** anywhere on the free-list, then it is guaranteed to be returned. If +** eMode is BTALLOC_LT then the page returned will be less than or equal +** to nearby if any such page exists. If eMode is BTALLOC_ANY then there +** are no restrictions on which page is returned. +*/ +static int allocateBtreePage( + BtShared *pBt, /* The btree */ + MemPage **ppPage, /* Store pointer to the allocated page here */ + Pgno *pPgno, /* Store the page number here */ + Pgno nearby, /* Search for a page near this one */ + u8 eMode /* BTALLOC_EXACT, BTALLOC_LT, or BTALLOC_ANY */ +){ + MemPage *pPage1; + int rc; + u32 n; /* Number of pages on the freelist */ + u32 k; /* Number of leaves on the trunk of the freelist */ + MemPage *pTrunk = 0; + MemPage *pPrevTrunk = 0; + Pgno mxPage; /* Total size of the database file */ + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); + pPage1 = pBt->pPage1; + mxPage = btreePagecount(pBt); + n = get4byte(&pPage1->aData[36]); + testcase( n==mxPage-1 ); + if( n>=mxPage ){ + return SQLITE_CORRUPT_BKPT; + } + if( n>0 ){ + /* There are pages on the freelist. Reuse one of those pages. */ + Pgno iTrunk; + u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ + + /* If eMode==BTALLOC_EXACT and a query of the pointer-map + ** shows that the page 'nearby' is somewhere on the free-list, then + ** the entire-list will be searched for that page. + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + if( eMode==BTALLOC_EXACT ){ + if( nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } + } + }else if( eMode==BTALLOC_LE ){ + searchList = 1; + } +#endif + + /* Decrement the free-list count by 1. Set iTrunk to the index of the + ** first free-list trunk page. iPrevTrunk is initially 1. + */ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) return rc; + put4byte(&pPage1->aData[36], n-1); + + /* The code within this loop is run only once if the 'searchList' variable + ** is not true. Otherwise, it runs once for each trunk-page on the + ** free-list until the page 'nearby' is located (eMode==BTALLOC_EXACT) + ** or until a page less than 'nearby' is located (eMode==BTALLOC_LT) + */ + do { + pPrevTrunk = pTrunk; + if( pPrevTrunk ){ + iTrunk = get4byte(&pPrevTrunk->aData[0]); + }else{ + iTrunk = get4byte(&pPage1->aData[32]); + } + testcase( iTrunk==mxPage ); + if( iTrunk>mxPage ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + } + if( rc ){ + pTrunk = 0; + goto end_allocate_page; + } + assert( pTrunk!=0 ); + assert( pTrunk->aData!=0 ); + + k = get4byte(&pTrunk->aData[4]); /* # of leaves on this trunk page */ + if( k==0 && !searchList ){ + /* The trunk has no leaves and the list is not being searched. + ** So extract the trunk page itself and use it as the newly + ** allocated page */ + assert( pPrevTrunk==0 ); + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc ){ + goto end_allocate_page; + } + *pPgno = iTrunk; + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + *ppPage = pTrunk; + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + }else if( k>(u32)(pBt->usableSize/4 - 2) ){ + /* Value of k is out of range. Database corruption */ + rc = SQLITE_CORRUPT_BKPT; + goto end_allocate_page; +#ifndef SQLITE_OMIT_AUTOVACUUM + }else if( searchList + && (nearby==iTrunk || (iTrunkpDbPage); + if( rc ){ + goto end_allocate_page; + } + if( k==0 ){ + if( !pPrevTrunk ){ + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } + memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); + } + }else{ + /* The trunk page is required by the caller but it contains + ** pointers to free-list leaves. The first leaf becomes a trunk + ** page in this case. + */ + MemPage *pNewTrunk; + Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); + if( iNewTrunk>mxPage ){ + rc = SQLITE_CORRUPT_BKPT; + goto end_allocate_page; + } + testcase( iNewTrunk==mxPage ); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } + rc = sqlite3PagerWrite(pNewTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pNewTrunk); + goto end_allocate_page; + } + memcpy(&pNewTrunk->aData[0], &pTrunk->aData[0], 4); + put4byte(&pNewTrunk->aData[4], k-1); + memcpy(&pNewTrunk->aData[8], &pTrunk->aData[12], (k-1)*4); + releasePage(pNewTrunk); + if( !pPrevTrunk ){ + assert( sqlite3PagerIswriteable(pPage1->pDbPage) ); + put4byte(&pPage1->aData[32], iNewTrunk); + }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc ){ + goto end_allocate_page; + } + put4byte(&pPrevTrunk->aData[0], iNewTrunk); + } + } + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); +#endif + }else if( k>0 ){ + /* Extract a leaf from the trunk */ + u32 closest; + Pgno iPage; + unsigned char *aData = pTrunk->aData; + if( nearby>0 ){ + u32 i; + closest = 0; + if( eMode==BTALLOC_LE ){ + for(i=0; imxPage ){ + rc = SQLITE_CORRUPT_BKPT; + goto end_allocate_page; + } + testcase( iPage==mxPage ); + if( !searchList + || (iPage==nearby || (iPagepgno, n-1)); + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc ) goto end_allocate_page; + if( closestpDbPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + } + } + searchList = 0; + } + } + releasePage(pPrevTrunk); + pPrevTrunk = 0; + }while( searchList ); + }else{ + /* There are no pages on the freelist, so append a new page to the + ** database image. + ** + ** Normally, new pages allocated by this block can be requested from the + ** pager layer with the 'no-content' flag set. This prevents the pager + ** from trying to read the pages content from disk. However, if the + ** current transaction has already run one or more incremental-vacuum + ** steps, then the page we are about to allocate may contain content + ** that is required in the event of a rollback. In this case, do + ** not set the no-content flag. This causes the pager to load and journal + ** the current page content before overwriting it. + ** + ** Note that the pager will not actually attempt to load or journal + ** content for any page that really does lie past the end of the database + ** file on disk. So the effects of disabling the no-content optimization + ** here are confined to those pages that lie between the end of the + ** database image and the end of the database file. + */ + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate)) ? PAGER_GET_NOCONTENT : 0; + + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc ) return rc; + pBt->nPage++; + if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ) pBt->nPage++; + +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, pBt->nPage) ){ + /* If *pPgno refers to a pointer-map page, allocate two new pages + ** at the end of the file instead of one. The first allocated page + ** becomes a new pointer-map page, the second is used by the caller. + */ + MemPage *pPg = 0; + TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); + assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg->pDbPage); + releasePage(pPg); + } + if( rc ) return rc; + pBt->nPage++; + if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ){ pBt->nPage++; } + } +#endif + put4byte(28 + (u8*)pBt->pPage1->aData, pBt->nPage); + *pPgno = pBt->nPage; + + assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); + if( rc ) return rc; + rc = sqlite3PagerWrite((*ppPage)->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + } + TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); + } + + assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); + +end_allocate_page: + releasePage(pTrunk); + releasePage(pPrevTrunk); + if( rc==SQLITE_OK ){ + if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ + releasePage(*ppPage); + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; + } + (*ppPage)->isInit = 0; + }else{ + *ppPage = 0; + } + assert( rc!=SQLITE_OK || sqlite3PagerIswriteable((*ppPage)->pDbPage) ); + return rc; +} + +/* +** This function is used to add page iPage to the database file free-list. +** It is assumed that the page is not already a part of the free-list. +** +** The value passed as the second argument to this function is optional. +** If the caller happens to have a pointer to the MemPage object +** corresponding to page iPage handy, it may pass it as the second value. +** Otherwise, it may pass NULL. +** +** If a pointer to a MemPage object is passed as the second argument, +** its reference count is not altered by this function. +*/ +static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ + MemPage *pTrunk = 0; /* Free-list trunk page */ + Pgno iTrunk = 0; /* Page number of free-list trunk page */ + MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ + MemPage *pPage; /* Page being freed. May be NULL. */ + int rc; /* Return Code */ + int nFree; /* Initial number of pages on free-list */ + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( iPage>1 ); + assert( !pMemPage || pMemPage->pgno==iPage ); + + if( pMemPage ){ + pPage = pMemPage; + sqlite3PagerRef(pPage->pDbPage); + }else{ + pPage = btreePageLookup(pBt, iPage); + } + + /* Increment the free page count on pPage1 */ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) goto freepage_out; + nFree = get4byte(&pPage1->aData[36]); + put4byte(&pPage1->aData[36], nFree+1); + + if( pBt->btsFlags & BTS_SECURE_DELETE ){ + /* If the secure_delete option is enabled, then + ** always fully overwrite deleted information with zeros. + */ + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) + ){ + goto freepage_out; + } + memset(pPage->aData, 0, pPage->pBt->pageSize); + } + + /* If the database supports auto-vacuum, write an entry in the pointer-map + ** to indicate that the page is free. + */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); + if( rc ) goto freepage_out; + } + + /* Now manipulate the actual database free-list structure. There are two + ** possibilities. If the free-list is currently empty, or if the first + ** trunk page in the free-list is full, then this page will become a + ** new free-list trunk page. Otherwise, it will become a leaf of the + ** first trunk page in the current free-list. This block tests if it + ** is possible to add the page as a new free-list leaf. + */ + if( nFree!=0 ){ + u32 nLeaf; /* Initial number of leaf cells on trunk page */ + + iTrunk = get4byte(&pPage1->aData[32]); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + if( rc!=SQLITE_OK ){ + goto freepage_out; + } + + nLeaf = get4byte(&pTrunk->aData[4]); + assert( pBt->usableSize>32 ); + if( nLeaf > (u32)pBt->usableSize/4 - 2 ){ + rc = SQLITE_CORRUPT_BKPT; + goto freepage_out; + } + if( nLeaf < (u32)pBt->usableSize/4 - 8 ){ + /* In this case there is room on the trunk page to insert the page + ** being freed as a new leaf. + ** + ** Note that the trunk page is not really full until it contains + ** usableSize/4 - 2 entries, not usableSize/4 - 8 entries as we have + ** coded. But due to a coding error in versions of SQLite prior to + ** 3.6.0, databases with freelist trunk pages holding more than + ** usableSize/4 - 8 entries will be reported as corrupt. In order + ** to maintain backwards compatibility with older versions of SQLite, + ** we will continue to restrict the number of entries to usableSize/4 - 8 + ** for now. At some point in the future (once everyone has upgraded + ** to 3.6.0 or later) we should consider fixing the conditional above + ** to read "usableSize/4-2" instead of "usableSize/4-8". + */ + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pTrunk->aData[4], nLeaf+1); + put4byte(&pTrunk->aData[8+nLeaf*4], iPage); + if( pPage && (pBt->btsFlags & BTS_SECURE_DELETE)==0 ){ + sqlite3PagerDontWrite(pPage->pDbPage); + } + rc = btreeSetHasContent(pBt, iPage); + } + TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); + goto freepage_out; + } + } + + /* If control flows to this point, then it was not possible to add the + ** the page being freed as a leaf page of the first trunk in the free-list. + ** Possibly because the free-list is empty, or possibly because the + ** first trunk in the free-list is full. Either way, the page being freed + ** will become the new first trunk page in the free-list. + */ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + goto freepage_out; + } + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + goto freepage_out; + } + put4byte(pPage->aData, iTrunk); + put4byte(&pPage->aData[4], 0); + put4byte(&pPage1->aData[32], iPage); + TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); + +freepage_out: + if( pPage ){ + pPage->isInit = 0; + } + releasePage(pPage); + releasePage(pTrunk); + return rc; +} +static void freePage(MemPage *pPage, int *pRC){ + if( (*pRC)==SQLITE_OK ){ + *pRC = freePage2(pPage->pBt, pPage, pPage->pgno); + } +} + +/* +** Free any overflow pages associated with the given Cell. +*/ +static int clearCell(MemPage *pPage, unsigned char *pCell){ + BtShared *pBt = pPage->pBt; + CellInfo info; + Pgno ovflPgno; + int rc; + int nOvfl; + u32 ovflPageSize; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + btreeParseCellPtr(pPage, pCell, &info); + if( info.iOverflow==0 ){ + return SQLITE_OK; /* No overflow pages. Return without doing anything */ + } + if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){ + return SQLITE_CORRUPT_BKPT; /* Cell extends past end of page */ + } + ovflPgno = get4byte(&pCell[info.iOverflow]); + assert( pBt->usableSize > 4 ); + ovflPageSize = pBt->usableSize - 4; + nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; + assert( ovflPgno==0 || nOvfl>0 ); + while( nOvfl-- ){ + Pgno iNext = 0; + MemPage *pOvfl = 0; + if( ovflPgno<2 || ovflPgno>btreePagecount(pBt) ){ + /* 0 is not a legal page number and page 1 cannot be an + ** overflow page. Therefore if ovflPgno<2 or past the end of the + ** file the database must be corrupt. */ + return SQLITE_CORRUPT_BKPT; + } + if( nOvfl ){ + rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); + if( rc ) return rc; + } + + if( ( pOvfl || ((pOvfl = btreePageLookup(pBt, ovflPgno))!=0) ) + && sqlite3PagerPageRefcount(pOvfl->pDbPage)!=1 + ){ + /* There is no reason any cursor should have an outstanding reference + ** to an overflow page belonging to a cell that is being deleted/updated. + ** So if there exists more than one reference to this page, then it + ** must not really be an overflow page and the database must be corrupt. + ** It is helpful to detect this before calling freePage2(), as + ** freePage2() may zero the page contents if secure-delete mode is + ** enabled. If this 'overflow' page happens to be a page that the + ** caller is iterating through or using in some other way, this + ** can be problematic. + */ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = freePage2(pBt, pOvfl, ovflPgno); + } + + if( pOvfl ){ + sqlite3PagerUnref(pOvfl->pDbPage); + } + if( rc ) return rc; + ovflPgno = iNext; + } + return SQLITE_OK; +} + +/* +** Create the byte sequence used to represent a cell on page pPage +** and write that byte sequence into pCell[]. Overflow pages are +** allocated and filled in as necessary. The calling procedure +** is responsible for making sure sufficient space has been allocated +** for pCell[]. +** +** Note that pCell does not necessary need to point to the pPage->aData +** area. pCell might point to some temporary storage. The cell will +** be constructed in this temporary area then copied into pPage->aData +** later. +*/ +static int fillInCell( + MemPage *pPage, /* The page that contains the cell */ + unsigned char *pCell, /* Complete text of the cell */ + const void *pKey, i64 nKey, /* The key */ + const void *pData,int nData, /* The data */ + int nZero, /* Extra zero bytes to append to pData */ + int *pnSize /* Write cell size here */ +){ + int nPayload; + const u8 *pSrc; + int nSrc, n, rc; + int spaceLeft; + MemPage *pOvfl = 0; + MemPage *pToRelease = 0; + unsigned char *pPrior; + unsigned char *pPayload; + BtShared *pBt = pPage->pBt; + Pgno pgnoOvfl = 0; + int nHeader; + CellInfo info; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + + /* pPage is not necessarily writeable since pCell might be auxiliary + ** buffer space that is separate from the pPage buffer area */ + assert( pCellaData || pCell>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + /* Fill in the header. */ + nHeader = 0; + if( !pPage->leaf ){ + nHeader += 4; + } + if( pPage->hasData ){ + nHeader += putVarint32(&pCell[nHeader], nData+nZero); + }else{ + nData = nZero = 0; + } + nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); + btreeParseCellPtr(pPage, pCell, &info); + assert( info.nHeader==nHeader ); + assert( info.nKey==nKey ); + assert( info.nData==(u32)(nData+nZero) ); + + /* Fill in the payload */ + nPayload = nData + nZero; + if( pPage->intKey ){ + pSrc = pData; + nSrc = nData; + nData = 0; + }else{ + if( NEVER(nKey>0x7fffffff || pKey==0) ){ + return SQLITE_CORRUPT_BKPT; + } + nPayload += (int)nKey; + pSrc = pKey; + nSrc = (int)nKey; + } + *pnSize = info.nSize; + spaceLeft = info.nLocal; + pPayload = &pCell[nHeader]; + pPrior = &pCell[info.iOverflow]; + + while( nPayload>0 ){ + if( spaceLeft==0 ){ +#ifndef SQLITE_OMIT_AUTOVACUUM + Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */ + if( pBt->autoVacuum ){ + do{ + pgnoOvfl++; + } while( + PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) + ); + } +#endif + rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the database supports auto-vacuum, and the second or subsequent + ** overflow page is being allocated, add an entry to the pointer-map + ** for that page now. + ** + ** If this is the first overflow page, then write a partial entry + ** to the pointer-map. If we write nothing to this pointer-map slot, + ** then the optimistic overflow chain processing in clearCell() + ** may misinterpret the uninitialized values and delete the + ** wrong pages from the database. + */ + if( pBt->autoVacuum && rc==SQLITE_OK ){ + u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1); + ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc); + if( rc ){ + releasePage(pOvfl); + } + } +#endif + if( rc ){ + releasePage(pToRelease); + return rc; + } + + /* If pToRelease is not zero than pPrior points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPrior is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPrioraData || pPrior>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + put4byte(pPrior, pgnoOvfl); + releasePage(pToRelease); + pToRelease = pOvfl; + pPrior = pOvfl->aData; + put4byte(pPrior, 0); + pPayload = &pOvfl->aData[4]; + spaceLeft = pBt->usableSize - 4; + } + n = nPayload; + if( n>spaceLeft ) n = spaceLeft; + + /* If pToRelease is not zero than pPayload points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPayload is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + if( nSrc>0 ){ + if( n>nSrc ) n = nSrc; + assert( pSrc ); + memcpy(pPayload, pSrc, n); + }else{ + memset(pPayload, 0, n); + } + nPayload -= n; + pPayload += n; + pSrc += n; + nSrc -= n; + spaceLeft -= n; + if( nSrc==0 ){ + nSrc = nData; + pSrc = pData; + } + } + releasePage(pToRelease); + return SQLITE_OK; +} + +/* +** Remove the i-th cell from pPage. This routine effects pPage only. +** The cell content is not freed or deallocated. It is assumed that +** the cell content has been copied someplace else. This routine just +** removes the reference to the cell from pPage. +** +** "sz" must be the number of bytes in the cell. +*/ +static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ + u32 pc; /* Offset to cell content of cell being deleted */ + u8 *data; /* pPage->aData */ + u8 *ptr; /* Used to move bytes around within data[] */ + int rc; /* The return code */ + int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ + + if( *pRC ) return; + + assert( idx>=0 && idxnCell ); + assert( sz==cellSize(pPage, idx) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + data = pPage->aData; + ptr = &pPage->aCellIdx[2*idx]; + pc = get2byte(ptr); + hdr = pPage->hdrOffset; + testcase( pc==get2byte(&data[hdr+5]) ); + testcase( pc+sz==pPage->pBt->usableSize ); + if( pc < (u32)get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + rc = freeSpace(pPage, pc, sz); + if( rc ){ + *pRC = rc; + return; + } + pPage->nCell--; + memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); + put2byte(&data[hdr+3], pPage->nCell); + pPage->nFree += 2; +} + +/* +** Insert a new cell on pPage at cell index "i". pCell points to the +** content of the cell. +** +** If the cell content will fit on the page, then put it there. If it +** will not fit, then make a copy of the cell content into pTemp if +** pTemp is not null. Regardless of pTemp, allocate a new entry +** in pPage->apOvfl[] and make it point to the cell content (either +** in pTemp or the original pCell) and also record its index. +** Allocating a new entry in pPage->aCell[] implies that +** pPage->nOverflow is incremented. +** +** If nSkip is non-zero, then do not copy the first nSkip bytes of the +** cell. The caller will overwrite them after this function returns. If +** nSkip is non-zero, then pCell may not point to an invalid memory location +** (but pCell+nSkip is always valid). +*/ +static void insertCell( + MemPage *pPage, /* Page into which we are copying */ + int i, /* New cell becomes the i-th cell of the page */ + u8 *pCell, /* Content of the new cell */ + int sz, /* Bytes of content in pCell */ + u8 *pTemp, /* Temp storage space for pCell, if needed */ + Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ + int *pRC /* Read and write return code from here */ +){ + int idx = 0; /* Where to write new cell content in data[] */ + int j; /* Loop counter */ + int end; /* First byte past the last cell pointer in data[] */ + int ins; /* Index in data[] where new cell pointer is inserted */ + int cellOffset; /* Address of first cell pointer in data[] */ + u8 *data; /* The content of the whole page */ + int nSkip = (iChild ? 4 : 0); + + if( *pRC ) return; + + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + /* The cell should normally be sized correctly. However, when moving a + ** malformed cell from a leaf page to an interior page, if the cell size + ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size + ** might be less than 8 (leaf-size + pointer) on the interior node. Hence + ** the term after the || in the following assert(). */ + assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); + if( pPage->nOverflow || sz+2>pPage->nFree ){ + if( pTemp ){ + memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip); + pCell = pTemp; + } + if( iChild ){ + put4byte(pCell, iChild); + } + j = pPage->nOverflow++; + assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) ); + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = (u16)i; + }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + data = pPage->aData; + cellOffset = pPage->cellOffset; + end = cellOffset + 2*pPage->nCell; + ins = cellOffset + 2*i; + rc = allocateSpace(pPage, sz, &idx); + if( rc ){ *pRC = rc; return; } + /* The allocateSpace() routine guarantees the following two properties + ** if it returns success */ + assert( idx >= end+2 ); + assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nCell++; + pPage->nFree -= (u16)(2 + sz); + memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip); + if( iChild ){ + put4byte(&data[idx], iChild); + } + memmove(&data[ins+2], &data[ins], end-ins); + put2byte(&data[ins], idx); + put2byte(&data[pPage->hdrOffset+3], pPage->nCell); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pPage->pBt->autoVacuum ){ + /* The cell may contain a pointer to an overflow page. If so, write + ** the entry for the overflow page into the pointer map. + */ + ptrmapPutOvflPtr(pPage, pCell, pRC); + } +#endif + } +} + +/* +** Add a list of cells to a page. The page should be initially empty. +** The cells are guaranteed to fit on the page. +*/ +static void assemblePage( + MemPage *pPage, /* The page to be assemblied */ + int nCell, /* The number of cells to add to this page */ + u8 **apCell, /* Pointers to cell bodies */ + u16 *aSize /* Sizes of the cells */ +){ + int i; /* Loop counter */ + u8 *pCellptr; /* Address of next cell pointer */ + int cellbody; /* Address of next cell body */ + u8 * const data = pPage->aData; /* Pointer to data for pPage */ + const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ + const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ + + assert( pPage->nOverflow==0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt) + && (int)MX_CELL(pPage->pBt)<=10921); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + + /* Check that the page has just been zeroed by zeroPage() */ + assert( pPage->nCell==0 ); + assert( get2byteNotZero(&data[hdr+5])==nUsable ); + + pCellptr = &pPage->aCellIdx[nCell*2]; + cellbody = nUsable; + for(i=nCell-1; i>=0; i--){ + u16 sz = aSize[i]; + pCellptr -= 2; + cellbody -= sz; + put2byte(pCellptr, cellbody); + memcpy(&data[cellbody], apCell[i], sz); + } + put2byte(&data[hdr+3], nCell); + put2byte(&data[hdr+5], cellbody); + pPage->nFree -= (nCell*2 + nUsable - cellbody); + pPage->nCell = (u16)nCell; +} + +/* +** The following parameters determine how many adjacent pages get involved +** in a balancing operation. NN is the number of neighbors on either side +** of the page that participate in the balancing operation. NB is the +** total number of pages that participate, including the target page and +** NN neighbors on either side. +** +** The minimum value of NN is 1 (of course). Increasing NN above 1 +** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance +** in exchange for a larger degradation in INSERT and UPDATE performance. +** The value of NN appears to give the best results overall. +*/ +#define NN 1 /* Number of neighbors on either side of pPage */ +#define NB (NN*2+1) /* Total pages involved in the balance */ + + +#ifndef SQLITE_OMIT_QUICKBALANCE +/* +** This version of balance() handles the common special case where +** a new entry is being inserted on the extreme right-end of the +** tree, in other words, when the new entry will become the largest +** entry in the tree. +** +** Instead of trying to balance the 3 right-most leaf pages, just add +** a new page to the right-hand side and put the one new entry in +** that page. This leaves the right side of the tree somewhat +** unbalanced. But odds are that we will be inserting new entries +** at the end soon afterwards so the nearly empty page will quickly +** fill up. On average. +** +** pPage is the leaf page which is the right-most page in the tree. +** pParent is its parent. pPage must have a single overflow entry +** which is also the right-most entry on the page. +** +** The pSpace buffer is used to store a temporary copy of the divider +** cell that will be inserted into pParent. Such a cell consists of a 4 +** byte page number followed by a variable length integer. In other +** words, at most 13 bytes. Hence the pSpace buffer must be at +** least 13 bytes in size. +*/ +static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ + BtShared *const pBt = pPage->pBt; /* B-Tree Database */ + MemPage *pNew; /* Newly allocated page */ + int rc; /* Return Code */ + Pgno pgnoNew; /* Page number of pNew */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + assert( pPage->nOverflow==1 ); + + /* This error condition is now caught prior to reaching this function */ + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; + + /* Allocate a new page. This page will become the right-sibling of + ** pPage. Make the parent page writable, so that the new divider cell + ** may be inserted. If both these operations are successful, proceed. + */ + rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); + + if( rc==SQLITE_OK ){ + + u8 *pOut = &pSpace[4]; + u8 *pCell = pPage->apOvfl[0]; + u16 szCell = cellSizePtr(pPage, pCell); + u8 *pStop; + + assert( sqlite3PagerIswriteable(pNew->pDbPage) ); + assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); + zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); + assemblePage(pNew, 1, &pCell, &szCell); + + /* If this is an auto-vacuum database, update the pointer map + ** with entries for the new page, and any pointer from the + ** cell on the page to an overflow page. If either of these + ** operations fails, the return code is set, but the contents + ** of the parent page are still manipulated by thh code below. + ** That is Ok, at this point the parent page is guaranteed to + ** be marked as dirty. Returning an error code will cause a + ** rollback, undoing any changes made to the parent page. + */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); + if( szCell>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pCell, &rc); + } + } + + /* Create a divider cell to insert into pParent. The divider cell + ** consists of a 4-byte page number (the page number of pPage) and + ** a variable length key value (which must be the same value as the + ** largest key on pPage). + ** + ** To find the largest key value on pPage, first find the right-most + ** cell on pPage. The first two fields of this cell are the + ** record-length (a variable length integer at most 32-bits in size) + ** and the key value (a variable length integer, may have any value). + ** The first of the while(...) loops below skips over the record-length + ** field. The second while(...) loop copies the key value from the + ** cell on pPage into the pSpace buffer. + */ + pCell = findCell(pPage, pPage->nCell-1); + pStop = &pCell[9]; + while( (*(pCell++)&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno, &rc); + + /* Set the right-child pointer of pParent to point to the new page. */ + put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); + + /* Release the reference to the new page. */ + releasePage(pNew); + } + + return rc; +} +#endif /* SQLITE_OMIT_QUICKBALANCE */ + +#if 0 +/* +** This function does not contribute anything to the operation of SQLite. +** it is sometimes activated temporarily while debugging code responsible +** for setting pointer-map entries. +*/ +static int ptrmapCheckPages(MemPage **apPage, int nPage){ + int i, j; + for(i=0; ipBt; + assert( pPage->isInit ); + + for(j=0; jnCell; j++){ + CellInfo info; + u8 *z; + + z = findCell(pPage, j); + btreeParseCellPtr(pPage, z, &info); + if( info.iOverflow ){ + Pgno ovfl = get4byte(&z[info.iOverflow]); + ptrmapGet(pBt, ovfl, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_OVERFLOW1 ); + } + if( !pPage->leaf ){ + Pgno child = get4byte(z); + ptrmapGet(pBt, child, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_BTREE ); + } + } + if( !pPage->leaf ){ + Pgno child = get4byte(&pPage->aData[pPage->hdrOffset+8]); + ptrmapGet(pBt, child, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_BTREE ); + } + } + return 1; +} +#endif + +/* +** This function is used to copy the contents of the b-tree node stored +** on page pFrom to page pTo. If page pFrom was not a leaf page, then +** the pointer-map entries for each child page are updated so that the +** parent page stored in the pointer map is page pTo. If pFrom contained +** any cells with overflow page pointers, then the corresponding pointer +** map entries are also updated so that the parent page is page pTo. +** +** If pFrom is currently carrying any overflow cells (entries in the +** MemPage.apOvfl[] array), they are not copied to pTo. +** +** Before returning, page pTo is reinitialized using btreeInitPage(). +** +** The performance of this function is not critical. It is only used by +** the balance_shallower() and balance_deeper() procedures, neither of +** which are called often under normal circumstances. +*/ +static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ + if( (*pRC)==SQLITE_OK ){ + BtShared * const pBt = pFrom->pBt; + u8 * const aFrom = pFrom->aData; + u8 * const aTo = pTo->aData; + int const iFromHdr = pFrom->hdrOffset; + int const iToHdr = ((pTo->pgno==1) ? 100 : 0); + int rc; + int iData; + + + assert( pFrom->isInit ); + assert( pFrom->nFree>=iToHdr ); + assert( get2byte(&aFrom[iFromHdr+5]) <= (int)pBt->usableSize ); + + /* Copy the b-tree node content from page pFrom to page pTo. */ + iData = get2byte(&aFrom[iFromHdr+5]); + memcpy(&aTo[iData], &aFrom[iData], pBt->usableSize-iData); + memcpy(&aTo[iToHdr], &aFrom[iFromHdr], pFrom->cellOffset + 2*pFrom->nCell); + + /* Reinitialize page pTo so that the contents of the MemPage structure + ** match the new data. The initialization of pTo can actually fail under + ** fairly obscure circumstances, even though it is a copy of initialized + ** page pFrom. + */ + pTo->isInit = 0; + rc = btreeInitPage(pTo); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + + /* If this is an auto-vacuum database, update the pointer-map entries + ** for any b-tree or overflow pages that pTo now contains the pointers to. + */ + if( ISAUTOVACUUM ){ + *pRC = setChildPtrmaps(pTo); + } + } +} + +/* +** This routine redistributes cells on the iParentIdx'th child of pParent +** (hereafter "the page") and up to 2 siblings so that all pages have about the +** same amount of free space. Usually a single sibling on either side of the +** page are used in the balancing, though both siblings might come from one +** side if the page is the first or last child of its parent. If the page +** has fewer than 2 siblings (something which can only happen if the page +** is a root page or a child of a root page) then all available siblings +** participate in the balancing. +** +** The number of siblings of the page might be increased or decreased by +** one or two in an effort to keep pages nearly full but not over full. +** +** Note that when this routine is called, some of the cells on the page +** might not actually be stored in MemPage.aData[]. This can happen +** if the page is overfull. This routine ensures that all cells allocated +** to the page and its siblings fit into MemPage.aData[] before returning. +** +** In the course of balancing the page and its siblings, cells may be +** inserted into or removed from the parent page (pParent). Doing so +** may cause the parent page to become overfull or underfull. If this +** happens, it is the responsibility of the caller to invoke the correct +** balancing routine to fix this problem (see the balance() routine). +** +** If this routine fails for any reason, it might leave the database +** in a corrupted state. So if this routine fails, the database should +** be rolled back. +** +** The third argument to this function, aOvflSpace, is a pointer to a +** buffer big enough to hold one page. If while inserting cells into the parent +** page (pParent) the parent page becomes overfull, this buffer is +** used to store the parent's overflow cells. Because this function inserts +** a maximum of four divider cells into the parent page, and the maximum +** size of a cell stored within an internal node is always less than 1/4 +** of the page-size, the aOvflSpace[] buffer is guaranteed to be large +** enough for all overflow cells. +** +** If aOvflSpace is set to a null pointer, this function returns +** SQLITE_NOMEM. +*/ +#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) +#pragma optimize("", off) +#endif +static int balance_nonroot( + MemPage *pParent, /* Parent page of siblings being balanced */ + int iParentIdx, /* Index of "the page" in pParent */ + u8 *aOvflSpace, /* page-size bytes of space for parent ovfl */ + int isRoot, /* True if pParent is a root-page */ + int bBulk /* True if this call is part of a bulk load */ +){ + BtShared *pBt; /* The whole database */ + int nCell = 0; /* Number of cells in apCell[] */ + int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ + int nNew = 0; /* Number of pages in apNew[] */ + int nOld; /* Number of pages in apOld[] */ + int i, j, k; /* Loop counters */ + int nxDiv; /* Next divider slot in pParent->aCell[] */ + int rc = SQLITE_OK; /* The return code */ + u16 leafCorrection; /* 4 if pPage is a leaf. 0 if not */ + int leafData; /* True if pPage is a leaf of a LEAFDATA tree */ + int usableSpace; /* Bytes in pPage beyond the header */ + int pageFlags; /* Value of pPage->aData[0] */ + int subtotal; /* Subtotal of bytes in cells on one page */ + int iSpace1 = 0; /* First unused byte of aSpace1[] */ + int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ + int szScratch; /* Size of scratch memory requested */ + MemPage *apOld[NB]; /* pPage and up to two siblings */ + MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ + MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ + u8 *pRight; /* Location in parent of right-sibling pointer */ + u8 *apDiv[NB-1]; /* Divider cells in pParent */ + int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ + int szNew[NB+2]; /* Combined size of cells place on i-th page */ + u8 **apCell = 0; /* All cells begin balanced */ + u16 *szCell; /* Local size of all cells in apCell[] */ + u8 *aSpace1; /* Space for copies of dividers cells */ + Pgno pgno; /* Temp var to store a page number in */ + + pBt = pParent->pBt; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + +#if 0 + TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno)); +#endif + + /* At this point pParent may have at most one overflow cell. And if + ** this overflow cell is present, it must be the cell with + ** index iParentIdx. This scenario comes about when this function + ** is called (indirectly) from sqlite3BtreeDelete(). + */ + assert( pParent->nOverflow==0 || pParent->nOverflow==1 ); + assert( pParent->nOverflow==0 || pParent->aiOvfl[0]==iParentIdx ); + + if( !aOvflSpace ){ + return SQLITE_NOMEM; + } + + /* Find the sibling pages to balance. Also locate the cells in pParent + ** that divide the siblings. An attempt is made to find NN siblings on + ** either side of pPage. More siblings are taken from one side, however, + ** if there are fewer than NN siblings on the other side. If pParent + ** has NB or fewer children then all children of pParent are taken. + ** + ** This loop also drops the divider cells from the parent page. This + ** way, the remainder of the function does not have to deal with any + ** overflow cells in the parent page, since if any existed they will + ** have already been removed. + */ + i = pParent->nOverflow + pParent->nCell; + if( i<2 ){ + nxDiv = 0; + }else{ + assert( bBulk==0 || bBulk==1 ); + if( iParentIdx==0 ){ + nxDiv = 0; + }else if( iParentIdx==i ){ + nxDiv = i-2+bBulk; + }else{ + assert( bBulk==0 ); + nxDiv = iParentIdx-1; + } + i = 2-bBulk; + } + nOld = i+1; + if( (i+nxDiv-pParent->nOverflow)==pParent->nCell ){ + pRight = &pParent->aData[pParent->hdrOffset+8]; + }else{ + pRight = findCell(pParent, i+nxDiv-pParent->nOverflow); + } + pgno = get4byte(pRight); + while( 1 ){ + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); + if( rc ){ + memset(apOld, 0, (i+1)*sizeof(MemPage*)); + goto balance_cleanup; + } + nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; + if( (i--)==0 ) break; + + if( i+nxDiv==pParent->aiOvfl[0] && pParent->nOverflow ){ + apDiv[i] = pParent->apOvfl[0]; + pgno = get4byte(apDiv[i]); + szNew[i] = cellSizePtr(pParent, apDiv[i]); + pParent->nOverflow = 0; + }else{ + apDiv[i] = findCell(pParent, i+nxDiv-pParent->nOverflow); + pgno = get4byte(apDiv[i]); + szNew[i] = cellSizePtr(pParent, apDiv[i]); + + /* Drop the cell from the parent page. apDiv[i] still points to + ** the cell within the parent, even though it has been dropped. + ** This is safe because dropping a cell only overwrites the first + ** four bytes of it, and this function does not need the first + ** four bytes of the divider cell. So the pointer is safe to use + ** later on. + ** + ** But not if we are in secure-delete mode. In secure-delete mode, + ** the dropCell() routine will overwrite the entire cell with zeroes. + ** In this case, temporarily copy the cell into the aOvflSpace[] + ** buffer. It will be copied out again as soon as the aSpace[] buffer + ** is allocated. */ + if( pBt->btsFlags & BTS_SECURE_DELETE ){ + int iOff; + + iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); + if( (iOff+szNew[i])>(int)pBt->usableSize ){ + rc = SQLITE_CORRUPT_BKPT; + memset(apOld, 0, (i+1)*sizeof(MemPage*)); + goto balance_cleanup; + }else{ + memcpy(&aOvflSpace[iOff], apDiv[i], szNew[i]); + apDiv[i] = &aOvflSpace[apDiv[i]-pParent->aData]; + } + } + dropCell(pParent, i+nxDiv-pParent->nOverflow, szNew[i], &rc); + } + } + + /* Make nMaxCells a multiple of 4 in order to preserve 8-byte + ** alignment */ + nMaxCells = (nMaxCells + 3)&~3; + + /* + ** Allocate space for memory structures + */ + k = pBt->pageSize + ROUND8(sizeof(MemPage)); + szScratch = + nMaxCells*sizeof(u8*) /* apCell */ + + nMaxCells*sizeof(u16) /* szCell */ + + pBt->pageSize /* aSpace1 */ + + k*nOld; /* Page copies (apCopy) */ + apCell = sqlite3ScratchMalloc( szScratch ); + if( apCell==0 ){ + rc = SQLITE_NOMEM; + goto balance_cleanup; + } + szCell = (u16*)&apCell[nMaxCells]; + aSpace1 = (u8*)&szCell[nMaxCells]; + assert( EIGHT_BYTE_ALIGNMENT(aSpace1) ); + + /* + ** Load pointers to all cells on sibling pages and the divider cells + ** into the local apCell[] array. Make copies of the divider cells + ** into space obtained from aSpace1[] and remove the divider cells + ** from pParent. + ** + ** If the siblings are on leaf pages, then the child pointers of the + ** divider cells are stripped from the cells before they are copied + ** into aSpace1[]. In this way, all cells in apCell[] are without + ** child pointers. If siblings are not leaves, then all cell in + ** apCell[] include child pointers. Either way, all cells in apCell[] + ** are alike. + ** + ** leafCorrection: 4 if pPage is a leaf. 0 if pPage is not a leaf. + ** leafData: 1 if pPage holds key+data and pParent holds only keys. + */ + leafCorrection = apOld[0]->leaf*4; + leafData = apOld[0]->hasData; + for(i=0; ipageSize + k*i]; + memcpy(pOld, apOld[i], sizeof(MemPage)); + pOld->aData = (void*)&pOld[1]; + memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); + + limit = pOld->nCell+pOld->nOverflow; + if( pOld->nOverflow>0 ){ + for(j=0; jaData; + u16 maskPage = pOld->maskPage; + u16 cellOffset = pOld->cellOffset; + for(j=0; jmaxLocal+23 ); + assert( iSpace1 <= (int)pBt->pageSize ); + memcpy(pTemp, apDiv[i], sz); + apCell[nCell] = pTemp+leafCorrection; + assert( leafCorrection==0 || leafCorrection==4 ); + szCell[nCell] = szCell[nCell] - leafCorrection; + if( !pOld->leaf ){ + assert( leafCorrection==0 ); + assert( pOld->hdrOffset==0 ); + /* The right pointer of the child page pOld becomes the left + ** pointer of the divider cell */ + memcpy(apCell[nCell], &pOld->aData[8], 4); + }else{ + assert( leafCorrection==4 ); + if( szCell[nCell]<4 ){ + /* Do not allow any cells smaller than 4 bytes. */ + szCell[nCell] = 4; + } + } + nCell++; + } + } + + /* + ** Figure out the number of pages needed to hold all nCell cells. + ** Store this number in "k". Also compute szNew[] which is the total + ** size of all cells on the i-th page and cntNew[] which is the index + ** in apCell[] of the cell that divides page i from page i+1. + ** cntNew[k] should equal nCell. + ** + ** Values computed by this block: + ** + ** k: The total number of sibling pages + ** szNew[i]: Spaced used on the i-th sibling page. + ** cntNew[i]: Index in apCell[] and szCell[] for the first cell to + ** the right of the i-th sibling page. + ** usableSpace: Number of bytes of space available on each sibling. + ** + */ + usableSpace = pBt->usableSize - 12 + leafCorrection; + for(subtotal=k=i=0; i usableSpace ){ + szNew[k] = subtotal - szCell[i]; + cntNew[k] = i; + if( leafData ){ i--; } + subtotal = 0; + k++; + if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + } + } + szNew[k] = subtotal; + cntNew[k] = nCell; + k++; + + /* + ** The packing computed by the previous block is biased toward the siblings + ** on the left side. The left siblings are always nearly full, while the + ** right-most sibling might be nearly empty. This block of code attempts + ** to adjust the packing of siblings to get a better balance. + ** + ** This adjustment is more than an optimization. The packing above might + ** be so out of balance as to be illegal. For example, the right-most + ** sibling might be completely empty. This adjustment is not optional. + */ + for(i=k-1; i>0; i--){ + int szRight = szNew[i]; /* Size of sibling on the right */ + int szLeft = szNew[i-1]; /* Size of sibling on the left */ + int r; /* Index of right-most cell in left sibling */ + int d; /* Index of first cell to the left of right sibling */ + + r = cntNew[i-1] - 1; + d = r + 1 - leafData; + assert( d0) or pPage is + ** a virtual root page. A virtual root page is when the real root + ** page is page 1 and we are the only child of that page. + ** + ** UPDATE: The assert() below is not necessarily true if the database + ** file is corrupt. The corruption will be detected and reported later + ** in this procedure so there is no need to act upon it now. + */ +#if 0 + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); +#endif + + TRACE(("BALANCE: old: %d %d %d ", + apOld[0]->pgno, + nOld>=2 ? apOld[1]->pgno : 0, + nOld>=3 ? apOld[2]->pgno : 0 + )); + + /* + ** Allocate k new pages. Reuse old pages where possible. + */ + if( apOld[0]->pgno<=1 ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + pageFlags = apOld[0]->aData[0]; + for(i=0; ipDbPage); + nNew++; + if( rc ) goto balance_cleanup; + }else{ + assert( i>0 ); + rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); + if( rc ) goto balance_cleanup; + apNew[i] = pNew; + nNew++; + + /* Set the pointer-map entry for the new sibling page. */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); + if( rc!=SQLITE_OK ){ + goto balance_cleanup; + } + } + } + } + + /* Free any old pages that were not reused as new pages. + */ + while( ipgno; + int minI = i; + for(j=i+1; jpgno<(unsigned)minV ){ + minI = j; + minV = apNew[j]->pgno; + } + } + if( minI>i ){ + MemPage *pT; + pT = apNew[i]; + apNew[i] = apNew[minI]; + apNew[minI] = pT; + } + } + TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", + apNew[0]->pgno, szNew[0], + nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); + + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + put4byte(pRight, apNew[nNew-1]->pgno); + + /* + ** Evenly distribute the data in apCell[] across the new pages. + ** Insert divider cells into pParent as necessary. + */ + j = 0; + for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); + assert( pNew->nOverflow==0 ); + + j = cntNew[i]; + + /* If the sibling page assembled above was not the right-most sibling, + ** insert a divider cell into the parent page. + */ + assert( ileaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + btreeParseCellPtr(pNew, apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** Note that this can never happen in an SQLite data file, as all + ** cells are at least 4 bytes. It only happens in b-trees used + ** to evaluate "IN (SELECT ...)" and similar clauses. + */ + if( szCell[j]==4 ){ + assert(leafCorrection==4); + sz = cellSizePtr(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + + j++; + nxDiv++; + } + } + assert( j==nCell ); + assert( nOld>0 ); + assert( nNew>0 ); + if( (pageFlags & PTF_LEAF)==0 ){ + u8 *zChild = &apCopy[nOld-1]->aData[8]; + memcpy(&apNew[nNew-1]->aData[8], zChild, 4); + } + + if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ + /* The root page of the b-tree now contains no cells. The only sibling + ** page is the right-child of the parent. Copy the contents of the + ** child page into the parent, decreasing the overall height of the + ** b-tree structure by one. This is described as the "balance-shallower" + ** sub-algorithm in some documentation. + ** + ** If this is an auto-vacuum database, the call to copyNodeContent() + ** sets all pointer-map entries corresponding to database image pages + ** for which the pointer is stored within the content being copied. + ** + ** The second assert below verifies that the child page is defragmented + ** (it must be, as it was just reconstructed using assemblePage()). This + ** is important if the parent page happens to be page 1 of the database + ** image. */ + assert( nNew==1 ); + assert( apNew[0]->nFree == + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); + }else if( ISAUTOVACUUM ){ + /* Fix the pointer-map entries for all the cells that were shifted around. + ** There are several different types of pointer-map entries that need to + ** be dealt with by this routine. Some of these have been set already, but + ** many have not. The following is a summary: + ** + ** 1) The entries associated with new sibling pages that were not + ** siblings when this function was called. These have already + ** been set. We don't need to worry about old siblings that were + ** moved to the free-list - the freePage() code has taken care + ** of those. + ** + ** 2) The pointer-map entries associated with the first overflow + ** page in any overflow chains used by new divider cells. These + ** have also already been taken care of by the insertCell() code. + ** + ** 3) If the sibling pages are not leaves, then the child pages of + ** cells stored on the sibling pages may need to be updated. + ** + ** 4) If the sibling pages are not internal intkey nodes, then any + ** overflow pages used by these cells may need to be updated + ** (internal intkey nodes never contain pointers to overflow pages). + ** + ** 5) If the sibling pages are not leaves, then the pointer-map + ** entries for the right-child pages of each sibling may need + ** to be updated. + ** + ** Cases 1 and 2 are dealt with above by other code. The next + ** block deals with cases 3 and 4 and the one after that, case 5. Since + ** setting a pointer map entry is a relatively expensive operation, this + ** code only sets pointer map entries for child or overflow pages that have + ** actually moved between pages. */ + MemPage *pNew = apNew[0]; + MemPage *pOld = apCopy[0]; + int nOverflow = pOld->nOverflow; + int iNextOld = pOld->nCell + nOverflow; + int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1); + j = 0; /* Current 'old' sibling page */ + k = 0; /* Current 'new' sibling page */ + for(i=0; inCell + pOld->nOverflow; + if( pOld->nOverflow ){ + nOverflow = pOld->nOverflow; + iOverflow = i + !leafData + pOld->aiOvfl[0]; + } + isDivider = !leafData; + } + + assert(nOverflow>0 || iOverflowaiOvfl[0]==pOld->aiOvfl[1]-1); + assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1); + if( i==iOverflow ){ + isDivider = 1; + if( (--nOverflow)>0 ){ + iOverflow++; + } + } + + if( i==cntNew[k] ){ + /* Cell i is the cell immediately following the last cell on new + ** sibling page k. If the siblings are not leaf pages of an + ** intkey b-tree, then cell i is a divider cell. */ + pNew = apNew[++k]; + if( !leafData ) continue; + } + assert( jpgno!=pNew->pgno ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( szCell[i]>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, apCell[i], &rc); + } + } + } + + if( !leafCorrection ){ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); + } + } + +#if 0 + /* The ptrmapCheckPages() contains assert() statements that verify that + ** all pointer map pages are set correctly. This is helpful while + ** debugging. This is usually disabled because a corrupt database may + ** cause an assert() statement to fail. */ + ptrmapCheckPages(apNew, nNew); + ptrmapCheckPages(&pParent, 1); +#endif + } + + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, nCell)); + + /* + ** Cleanup before returning. + */ +balance_cleanup: + sqlite3ScratchFree(apCell); + for(i=0; i= 1700 && defined(_M_ARM) +#pragma optimize("", on) +#endif + + +/* +** This function is called when the root page of a b-tree structure is +** overfull (has one or more overflow pages). +** +** A new child page is allocated and the contents of the current root +** page, including overflow cells, are copied into the child. The root +** page is then overwritten to make it an empty page with the right-child +** pointer pointing to the new page. +** +** Before returning, all pointer-map entries corresponding to pages +** that the new child-page now contains pointers to are updated. The +** entry corresponding to the new right-child pointer of the root +** page is also updated. +** +** If successful, *ppChild is set to contain a reference to the child +** page and SQLITE_OK is returned. In this case the caller is required +** to call releasePage() on *ppChild exactly once. If an error occurs, +** an error code is returned and *ppChild is set to 0. +*/ +static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ + int rc; /* Return value from subprocedures */ + MemPage *pChild = 0; /* Pointer to a new child page */ + Pgno pgnoChild = 0; /* Page number of the new child page */ + BtShared *pBt = pRoot->pBt; /* The BTree */ + + assert( pRoot->nOverflow>0 ); + assert( sqlite3_mutex_held(pBt->mutex) ); + + /* Make pRoot, the root page of the b-tree, writable. Allocate a new + ** page that will become the new right-child of pPage. Copy the contents + ** of the node stored on pRoot into the new child page. + */ + rc = sqlite3PagerWrite(pRoot->pDbPage); + if( rc==SQLITE_OK ){ + rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); + copyNodeContent(pRoot, pChild, &rc); + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); + } + } + if( rc ){ + *ppChild = 0; + releasePage(pChild); + return rc; + } + assert( sqlite3PagerIswriteable(pChild->pDbPage) ); + assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); + assert( pChild->nCell==pRoot->nCell ); + + TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); + + /* Copy the overflow cells from pRoot to pChild */ + memcpy(pChild->aiOvfl, pRoot->aiOvfl, + pRoot->nOverflow*sizeof(pRoot->aiOvfl[0])); + memcpy(pChild->apOvfl, pRoot->apOvfl, + pRoot->nOverflow*sizeof(pRoot->apOvfl[0])); + pChild->nOverflow = pRoot->nOverflow; + + /* Zero the contents of pRoot. Then install pChild as the right-child. */ + zeroPage(pRoot, pChild->aData[0] & ~PTF_LEAF); + put4byte(&pRoot->aData[pRoot->hdrOffset+8], pgnoChild); + + *ppChild = pChild; + return SQLITE_OK; +} + +/* +** The page that pCur currently points to has just been modified in +** some way. This function figures out if this modification means the +** tree needs to be balanced, and if so calls the appropriate balancing +** routine. Balancing routines are: +** +** balance_quick() +** balance_deeper() +** balance_nonroot() +*/ +static int balance(BtCursor *pCur){ + int rc = SQLITE_OK; + const int nMin = pCur->pBt->usableSize * 2 / 3; + u8 aBalanceQuickSpace[13]; + u8 *pFree = 0; + + TESTONLY( int balance_quick_called = 0 ); + TESTONLY( int balance_deeper_called = 0 ); + + do { + int iPage = pCur->iPage; + MemPage *pPage = pCur->apPage[iPage]; + + if( iPage==0 ){ + if( pPage->nOverflow ){ + /* The root page of the b-tree is overfull. In this case call the + ** balance_deeper() function to create a new child for the root-page + ** and copy the current contents of the root-page to it. The + ** next iteration of the do-loop will balance the child page. + */ + assert( (balance_deeper_called++)==0 ); + rc = balance_deeper(pPage, &pCur->apPage[1]); + if( rc==SQLITE_OK ){ + pCur->iPage = 1; + pCur->aiIdx[0] = 0; + pCur->aiIdx[1] = 0; + assert( pCur->apPage[1]->nOverflow ); + } + }else{ + break; + } + }else if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ + break; + }else{ + MemPage * const pParent = pCur->apPage[iPage-1]; + int const iIdx = pCur->aiIdx[iPage-1]; + + rc = sqlite3PagerWrite(pParent->pDbPage); + if( rc==SQLITE_OK ){ +#ifndef SQLITE_OMIT_QUICKBALANCE + if( pPage->hasData + && pPage->nOverflow==1 + && pPage->aiOvfl[0]==pPage->nCell + && pParent->pgno!=1 + && pParent->nCell==iIdx + ){ + /* Call balance_quick() to create a new sibling of pPage on which + ** to store the overflow cell. balance_quick() inserts a new cell + ** into pParent, which may cause pParent overflow. If this + ** happens, the next interation of the do-loop will balance pParent + ** use either balance_nonroot() or balance_deeper(). Until this + ** happens, the overflow cell is stored in the aBalanceQuickSpace[] + ** buffer. + ** + ** The purpose of the following assert() is to check that only a + ** single call to balance_quick() is made for each call to this + ** function. If this were not verified, a subtle bug involving reuse + ** of the aBalanceQuickSpace[] might sneak in. + */ + assert( (balance_quick_called++)==0 ); + rc = balance_quick(pParent, pPage, aBalanceQuickSpace); + }else +#endif + { + /* In this case, call balance_nonroot() to redistribute cells + ** between pPage and up to 2 of its sibling pages. This involves + ** modifying the contents of pParent, which may cause pParent to + ** become overfull or underfull. The next iteration of the do-loop + ** will balance the parent page to correct this. + ** + ** If the parent page becomes overfull, the overflow cell or cells + ** are stored in the pSpace buffer allocated immediately below. + ** A subsequent iteration of the do-loop will deal with this by + ** calling balance_nonroot() (balance_deeper() may be called first, + ** but it doesn't deal with overflow cells - just moves them to a + ** different page). Once this subsequent call to balance_nonroot() + ** has completed, it is safe to release the pSpace buffer used by + ** the previous call, as the overflow cell data will have been + ** copied either into the body of a database page or into the new + ** pSpace buffer passed to the latter call to balance_nonroot(). + */ + u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize); + rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, pCur->hints); + if( pFree ){ + /* If pFree is not NULL, it points to the pSpace buffer used + ** by a previous call to balance_nonroot(). Its contents are + ** now stored either on real database pages or within the + ** new pSpace buffer, so it may be safely freed here. */ + sqlite3PageFree(pFree); + } + + /* The pSpace buffer will be freed after the next call to + ** balance_nonroot(), or just before this function returns, whichever + ** comes first. */ + pFree = pSpace; + } + } + + pPage->nOverflow = 0; + + /* The next iteration of the do-loop balances the parent page. */ + releasePage(pPage); + pCur->iPage--; + } + }while( rc==SQLITE_OK ); + + if( pFree ){ + sqlite3PageFree(pFree); + } + return rc; +} + + +/* +** Insert a new record into the BTree. The key is given by (pKey,nKey) +** and the data is given by (pData,nData). The cursor is used only to +** define what table the record should be inserted into. The cursor +** is left pointing at a random location. +** +** For an INTKEY table, only the nKey value of the key is used. pKey is +** ignored. For a ZERODATA table, the pData and nData are both ignored. +** +** If the seekResult parameter is non-zero, then a successful call to +** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already +** been performed. seekResult is the search result returned (a negative +** number if pCur points at an entry that is smaller than (pKey, nKey), or +** a positive value if pCur points at an etry that is larger than +** (pKey, nKey)). +** +** If the seekResult parameter is non-zero, then the caller guarantees that +** cursor pCur is pointing at the existing copy of a row that is to be +** overwritten. If the seekResult parameter is 0, then cursor pCur may +** point to any entry or to no entry at all and so this function has to seek +** the cursor before the new key can be inserted. +*/ +SQLITE_PRIVATE int sqlite3BtreeInsert( + BtCursor *pCur, /* Insert data into the table of this cursor */ + const void *pKey, i64 nKey, /* The key of the new record */ + const void *pData, int nData, /* The data of the new record */ + int nZero, /* Number of extra 0 bytes to append to data */ + int appendBias, /* True if this is likely an append */ + int seekResult /* Result of prior MovetoUnpacked() call */ +){ + int rc; + int loc = seekResult; /* -1: before desired location +1: after */ + int szNew = 0; + int idx; + MemPage *pPage; + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; + unsigned char *oldCell; + unsigned char *newCell = 0; + + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; + } + + assert( cursorHoldsMutex(pCur) ); + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE + && (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); + + /* Assert that the caller has been consistent. If this cursor was opened + ** expecting an index b-tree, then the caller should be inserting blob + ** keys with no associated data. If the cursor was opened expecting an + ** intkey table, the caller should be inserting integer keys with a + ** blob of associated data. */ + assert( (pKey==0)==(pCur->pKeyInfo==0) ); + + /* Save the positions of any other cursors open on this table. + ** + ** In some cases, the call to btreeMoveto() below is a no-op. For + ** example, when inserting data into a table with auto-generated integer + ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the + ** integer key to use. It then calls this function to actually insert the + ** data into the intkey B-Tree. In this case btreeMoveto() recognizes + ** that the cursor is already where it needs to be and returns without + ** doing any work. To avoid thwarting these optimizations, it is important + ** not to clear the cursor here. + */ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + + if( pCur->pKeyInfo==0 ){ + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced */ + invalidateIncrblobCursors(p, nKey, 0); + + /* If the cursor is currently on the last row and we are appending a + ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() + ** call */ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){ + loc = -1; + } + } + + if( !loc ){ + rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); + if( rc ) return rc; + } + assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); + + pPage = pCur->apPage[pCur->iPage]; + assert( pPage->intKey || nKey>=0 ); + assert( pPage->leaf || !pPage->intKey ); + + TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", + pCur->pgnoRoot, nKey, nData, pPage->pgno, + loc==0 ? "overwrite" : "new entry")); + assert( pPage->isInit ); + allocateTempSpace(pBt); + newCell = pBt->pTmpSpace; + if( newCell==0 ) return SQLITE_NOMEM; + rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); + if( rc ) goto end_insert; + assert( szNew==cellSizePtr(pPage, newCell) ); + assert( szNew <= MX_CELL_SIZE(pBt) ); + idx = pCur->aiIdx[pCur->iPage]; + if( loc==0 ){ + u16 szOld; + assert( idxnCell ); + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ){ + goto end_insert; + } + oldCell = findCell(pPage, idx); + if( !pPage->leaf ){ + memcpy(newCell, oldCell, 4); + } + szOld = cellSizePtr(pPage, oldCell); + rc = clearCell(pPage, oldCell); + dropCell(pPage, idx, szOld, &rc); + if( rc ) goto end_insert; + }else if( loc<0 && pPage->nCell>0 ){ + assert( pPage->leaf ); + idx = ++pCur->aiIdx[pCur->iPage]; + }else{ + assert( pPage->leaf ); + } + insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); + + /* If no error has occurred and pPage has an overflow cell, call balance() + ** to redistribute the cells within the tree. Since balance() may move + ** the cursor, zero the BtCursor.info.nSize and BTCF_ValidNKey + ** variables. + ** + ** Previous versions of SQLite called moveToRoot() to move the cursor + ** back to the root page as balance() used to invalidate the contents + ** of BtCursor.apPage[] and BtCursor.aiIdx[]. Instead of doing that, + ** set the cursor state to "invalid". This makes common insert operations + ** slightly faster. + ** + ** There is a subtle but important optimization here too. When inserting + ** multiple records into an intkey b-tree using a single cursor (as can + ** happen while processing an "INSERT INTO ... SELECT" statement), it + ** is advantageous to leave the cursor pointing to the last entry in + ** the b-tree if possible. If the cursor is left pointing to the last + ** entry in the table, and the next row inserted has an integer key + ** larger than the largest existing key, it is possible to insert the + ** row without seeking the cursor. This can be a big performance boost. + */ + pCur->info.nSize = 0; + if( rc==SQLITE_OK && pPage->nOverflow ){ + pCur->curFlags &= ~(BTCF_ValidNKey); + rc = balance(pCur); + + /* Must make sure nOverflow is reset to zero even if the balance() + ** fails. Internal data structure corruption will result otherwise. + ** Also, set the cursor state to invalid. This stops saveCursorPosition() + ** from trying to save the current position of the cursor. */ + pCur->apPage[pCur->iPage]->nOverflow = 0; + pCur->eState = CURSOR_INVALID; + } + assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); + +end_insert: + return rc; +} + +/* +** Delete the entry that the cursor is pointing to. The cursor +** is left pointing at a arbitrary location. +*/ +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; + int rc; /* Return code */ + MemPage *pPage; /* Page to delete cell from */ + unsigned char *pCell; /* Pointer to cell to delete */ + int iCellIdx; /* Index of cell to delete */ + int iCellDepth; /* Depth of node containing pCell */ + + assert( cursorHoldsMutex(pCur) ); + assert( pBt->inTransaction==TRANS_WRITE ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( pCur->curFlags & BTCF_WriteFlag ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); + assert( !hasReadConflicts(p, pCur->pgnoRoot) ); + + if( NEVER(pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell) + || NEVER(pCur->eState!=CURSOR_VALID) + ){ + return SQLITE_ERROR; /* Something has gone awry. */ + } + + iCellDepth = pCur->iPage; + iCellIdx = pCur->aiIdx[iCellDepth]; + pPage = pCur->apPage[iCellDepth]; + pCell = findCell(pPage, iCellIdx); + + /* If the page containing the entry to delete is not a leaf page, move + ** the cursor to the largest entry in the tree that is smaller than + ** the entry being deleted. This cell will replace the cell being deleted + ** from the internal node. The 'previous' entry is used for this instead + ** of the 'next' entry, as the previous entry is always a part of the + ** sub-tree headed by the child page of the cell being deleted. This makes + ** balancing the tree following the delete operation easier. */ + if( !pPage->leaf ){ + int notUsed = 0; + rc = sqlite3BtreePrevious(pCur, ¬Used); + if( rc ) return rc; + } + + /* Save the positions of any other cursors open on this table before + ** making any modifications. Make the page containing the entry to be + ** deleted writable. Then free any overflow pages associated with the + ** entry and finally remove the cell itself from within the page. + */ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + + /* If this is a delete operation to remove a row from a table b-tree, + ** invalidate any incrblob cursors open on the row being deleted. */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, pCur->info.nKey, 0); + } + + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ) return rc; + rc = clearCell(pPage, pCell); + dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc); + if( rc ) return rc; + + /* If the cell deleted was not located on a leaf page, then the cursor + ** is currently pointing to the largest entry in the sub-tree headed + ** by the child-page of the cell that was just deleted from an internal + ** node. The cell from the leaf node needs to be moved to the internal + ** node to replace the deleted cell. */ + if( !pPage->leaf ){ + MemPage *pLeaf = pCur->apPage[pCur->iPage]; + int nCell; + Pgno n = pCur->apPage[iCellDepth+1]->pgno; + unsigned char *pTmp; + + pCell = findCell(pLeaf, pLeaf->nCell-1); + nCell = cellSizePtr(pLeaf, pCell); + assert( MX_CELL_SIZE(pBt) >= nCell ); + + allocateTempSpace(pBt); + pTmp = pBt->pTmpSpace; + + rc = sqlite3PagerWrite(pLeaf->pDbPage); + insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); + if( rc ) return rc; + } + + /* Balance the tree. If the entry deleted was located on a leaf page, + ** then the cursor still points to that page. In this case the first + ** call to balance() repairs the tree, and the if(...) condition is + ** never true. + ** + ** Otherwise, if the entry deleted was on an internal node page, then + ** pCur is pointing to the leaf page from which a cell was removed to + ** replace the cell deleted from the internal node. This is slightly + ** tricky as the leaf node may be underfull, and the internal node may + ** be either under or overfull. In this case run the balancing algorithm + ** on the leaf node first. If the balance proceeds far enough up the + ** tree that we can be sure that any problem in the internal node has + ** been corrected, so be it. Otherwise, after balancing the leaf node, + ** walk the cursor up the tree to the internal node and balance it as + ** well. */ + rc = balance(pCur); + if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){ + while( pCur->iPage>iCellDepth ){ + releasePage(pCur->apPage[pCur->iPage--]); + } + rc = balance(pCur); + } + + if( rc==SQLITE_OK ){ + moveToRoot(pCur); + } + return rc; +} + +/* +** Create a new BTree table. Write into *piTable the page +** number for the root page of the new table. +** +** The type of type is determined by the flags parameter. Only the +** following values of flags are currently in use. Other values for +** flags might not work: +** +** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys +** BTREE_ZERODATA Used for SQL indices +*/ +static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ + BtShared *pBt = p->pBt; + MemPage *pRoot; + Pgno pgnoRoot; + int rc; + int ptfFlags; /* Page-type flage for the root page of new table */ + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( pBt->inTransaction==TRANS_WRITE ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); + +#ifdef SQLITE_OMIT_AUTOVACUUM + rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); + if( rc ){ + return rc; + } +#else + if( pBt->autoVacuum ){ + Pgno pgnoMove; /* Move a page here to make room for the root-page */ + MemPage *pPageMove; /* The page to move to. */ + + /* Creating a new table may probably require moving an existing database + ** to make room for the new tables root page. In case this page turns + ** out to be an overflow page, delete all overflow page-map caches + ** held by open cursors. + */ + invalidateAllOverflowCache(pBt); + + /* Read the value of meta[3] from the database to determine where the + ** root page of the new table should go. meta[3] is the largest root-page + ** created so far, so the new root-page is (meta[3]+1). + */ + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); + pgnoRoot++; + + /* The new root-page may not be allocated on a pointer-map page, or the + ** PENDING_BYTE page. + */ + while( pgnoRoot==PTRMAP_PAGENO(pBt, pgnoRoot) || + pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ + pgnoRoot++; + } + assert( pgnoRoot>=3 ); + + /* Allocate a page. The page that currently resides at pgnoRoot will + ** be moved to the allocated page (unless the allocated page happens + ** to reside at pgnoRoot). + */ + rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, BTALLOC_EXACT); + if( rc!=SQLITE_OK ){ + return rc; + } + + if( pgnoMove!=pgnoRoot ){ + /* pgnoRoot is the page that will be used for the root-page of + ** the new table (assuming an error did not occur). But we were + ** allocated pgnoMove. If required (i.e. if it was not allocated + ** by extending the file), the current page at position pgnoMove + ** is already journaled. + */ + u8 eType = 0; + Pgno iPtrPage = 0; + + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); + releasePage(pPageMove); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Move the page currently at pgnoRoot to pgnoMove. */ + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); + if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ + rc = SQLITE_CORRUPT_BKPT; + } + if( rc!=SQLITE_OK ){ + releasePage(pRoot); + return rc; + } + assert( eType!=PTRMAP_ROOTPAGE ); + assert( eType!=PTRMAP_FREEPAGE ); + rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove, 0); + releasePage(pRoot); + + /* Obtain the page at pgnoRoot */ + if( rc!=SQLITE_OK ){ + return rc; + } + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3PagerWrite(pRoot->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pRoot); + return rc; + } + }else{ + pRoot = pPageMove; + } + + /* Update the pointer-map and meta-data with the new root-page number. */ + ptrmapPut(pBt, pgnoRoot, PTRMAP_ROOTPAGE, 0, &rc); + if( rc ){ + releasePage(pRoot); + return rc; + } + + /* When the new root page was allocated, page 1 was made writable in + ** order either to increase the database filesize, or to decrement the + ** freelist count. Hence, the sqlite3BtreeUpdateMeta() call cannot fail. + */ + assert( sqlite3PagerIswriteable(pBt->pPage1->pDbPage) ); + rc = sqlite3BtreeUpdateMeta(p, 4, pgnoRoot); + if( NEVER(rc) ){ + releasePage(pRoot); + return rc; + } + + }else{ + rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); + if( rc ) return rc; + } +#endif + assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); + if( createTabFlags & BTREE_INTKEY ){ + ptfFlags = PTF_INTKEY | PTF_LEAFDATA | PTF_LEAF; + }else{ + ptfFlags = PTF_ZERODATA | PTF_LEAF; + } + zeroPage(pRoot, ptfFlags); + sqlite3PagerUnref(pRoot->pDbPage); + assert( (pBt->openFlags & BTREE_SINGLE)==0 || pgnoRoot==2 ); + *piTable = (int)pgnoRoot; + return SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCreateTable(p, piTable, flags); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Erase the given database page and all its children. Return +** the page to the freelist. +*/ +static int clearDatabasePage( + BtShared *pBt, /* The BTree that contains the table */ + Pgno pgno, /* Page number to clear */ + int freePageFlag, /* Deallocate page if true */ + int *pnChange /* Add number of Cells freed to this counter */ +){ + MemPage *pPage; + int rc; + unsigned char *pCell; + int i; + int hdr; + + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } + + rc = getAndInitPage(pBt, pgno, &pPage, 0); + if( rc ) return rc; + hdr = pPage->hdrOffset; + for(i=0; inCell; i++){ + pCell = findCell(pPage, i); + if( !pPage->leaf ){ + rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); + if( rc ) goto cleardatabasepage_out; + } + rc = clearCell(pPage, pCell); + if( rc ) goto cleardatabasepage_out; + } + if( !pPage->leaf ){ + rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); + if( rc ) goto cleardatabasepage_out; + }else if( pnChange ){ + assert( pPage->intKey ); + *pnChange += pPage->nCell; + } + if( freePageFlag ){ + freePage(pPage, &rc); + }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ + zeroPage(pPage, pPage->aData[hdr] | PTF_LEAF); + } + +cleardatabasepage_out: + releasePage(pPage); + return rc; +} + +/* +** Delete all information from a single table in the database. iTable is +** the page number of the root of the table. After this routine returns, +** the root page is empty, but still exists. +** +** This routine will fail with SQLITE_LOCKED if there are any open +** read cursors on the table. Open write cursors are moved to the +** root of the table. +** +** If pnChange is not NULL, then table iTable must be an intkey table. The +** integer value pointed to by pnChange is incremented by the number of +** entries in the table. +*/ +SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + + rc = saveAllCursors(pBt, (Pgno)iTable, 0); + + if( SQLITE_OK==rc ){ + /* Invalidate all incrblob cursors open on table iTable (assuming iTable + ** is the root of a table b-tree - if it is not, the following call is + ** a no-op). */ + invalidateIncrblobCursors(p, 0, 1); + rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Delete all information from the single table that pCur is open on. +** +** This routine only work for pCur on an ephemeral table. +*/ +SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor *pCur){ + return sqlite3BtreeClearTable(pCur->pBtree, pCur->pgnoRoot, 0); +} + +/* +** Erase all information in a table and add the root of the table to +** the freelist. Except, the root of the principle table (the one on +** page 1) is never added to the freelist. +** +** This routine will fail with SQLITE_LOCKED if there are any open +** cursors on the table. +** +** If AUTOVACUUM is enabled and the page at iTable is not the last +** root page in the database file, then the last root page +** in the database file is moved into the slot formerly occupied by +** iTable and that last slot formerly occupied by the last root page +** is added to the freelist instead of iTable. In this say, all +** root pages are kept at the beginning of the database file, which +** is necessary for AUTOVACUUM to work right. *piMoved is set to the +** page number that used to be the last root page in the file before +** the move. If no page gets moved, *piMoved is set to 0. +** The last root page is recorded in meta[3] and the value of +** meta[3] is updated by this procedure. +*/ +static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ + int rc; + MemPage *pPage = 0; + BtShared *pBt = p->pBt; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->inTrans==TRANS_WRITE ); + + /* It is illegal to drop a table if any cursors are open on the + ** database. This is because in auto-vacuum mode the backend may + ** need to move another root-page to fill a gap left by the deleted + ** root page. If an open cursor was using this page a problem would + ** occur. + ** + ** This error is caught long before control reaches this point. + */ + if( NEVER(pBt->pCursor) ){ + sqlite3ConnectionBlocked(p->db, pBt->pCursor->pBtree->db); + return SQLITE_LOCKED_SHAREDCACHE; + } + + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + if( rc ) return rc; + rc = sqlite3BtreeClearTable(p, iTable, 0); + if( rc ){ + releasePage(pPage); + return rc; + } + + *piMoved = 0; + + if( iTable>1 ){ +#ifdef SQLITE_OMIT_AUTOVACUUM + freePage(pPage, &rc); + releasePage(pPage); +#else + if( pBt->autoVacuum ){ + Pgno maxRootPgno; + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); + + if( iTable==maxRootPgno ){ + /* If the table being dropped is the table with the largest root-page + ** number in the database, put the root page on the free list. + */ + freePage(pPage, &rc); + releasePage(pPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + /* The table being dropped does not have the largest root-page + ** number in the database. So move the page that does into the + ** gap left by the deleted root-page. + */ + MemPage *pMove; + releasePage(pPage); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + pMove = 0; + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + freePage(pMove, &rc); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + *piMoved = maxRootPgno; + } + + /* Set the new 'max-root-page' value in the database header. This + ** is the old value less one, less one more if that happens to + ** be a root-page number, less one again if that is the + ** PENDING_BYTE_PAGE. + */ + maxRootPgno--; + while( maxRootPgno==PENDING_BYTE_PAGE(pBt) + || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ + maxRootPgno--; + } + assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); + + rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); + }else{ + freePage(pPage, &rc); + releasePage(pPage); + } +#endif + }else{ + /* If sqlite3BtreeDropTable was called on page 1. + ** This really never should happen except in a corrupt + ** database. + */ + zeroPage(pPage, PTF_INTKEY|PTF_LEAF ); + releasePage(pPage); + } + return rc; +} +SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeDropTable(p, iTable, piMoved); + sqlite3BtreeLeave(p); + return rc; +} + + +/* +** This function may only be called if the b-tree connection already +** has a read or write transaction open on the database. +** +** Read the meta-information out of a database file. Meta[0] +** is the number of free pages currently in the database. Meta[1] +** through meta[15] are available for use by higher layers. Meta[0] +** is read-only, the others are read/write. +** +** The schema layer numbers meta values differently. At the schema +** layer (and the SetCookie and ReadCookie opcodes) the number of +** free pages is not visible. So Cookie[0] is the same as Meta[1]. +*/ +SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ + BtShared *pBt = p->pBt; + + sqlite3BtreeEnter(p); + assert( p->inTrans>TRANS_NONE ); + assert( SQLITE_OK==querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK) ); + assert( pBt->pPage1 ); + assert( idx>=0 && idx<=15 ); + + *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + + /* If auto-vacuum is disabled in this build and this is an auto-vacuum + ** database, mark the database as read-only. */ +#ifdef SQLITE_OMIT_AUTOVACUUM + if( idx==BTREE_LARGEST_ROOT_PAGE && *pMeta>0 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } +#endif + + sqlite3BtreeLeave(p); +} + +/* +** Write meta-information back into the database. Meta[0] is +** read-only and may not be written. +*/ +SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ + BtShared *pBt = p->pBt; + unsigned char *pP1; + int rc; + assert( idx>=1 && idx<=15 ); + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + assert( pBt->pPage1!=0 ); + pP1 = pBt->pPage1->aData; + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pP1[36 + idx*4], iMeta); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( idx==BTREE_INCR_VACUUM ){ + assert( pBt->autoVacuum || iMeta==0 ); + assert( iMeta==0 || iMeta==1 ); + pBt->incrVacuum = (u8)iMeta; + } +#endif + } + sqlite3BtreeLeave(p); + return rc; +} + +#ifndef SQLITE_OMIT_BTREECOUNT +/* +** The first argument, pCur, is a cursor opened on some b-tree. Count the +** number of entries in the b-tree and write the result to *pnEntry. +** +** SQLITE_OK is returned if the operation is successfully executed. +** Otherwise, if an error is encountered (i.e. an IO error or database +** corruption) an SQLite error code is returned. +*/ +SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ + i64 nEntry = 0; /* Value to return in *pnEntry */ + int rc; /* Return code */ + + if( pCur->pgnoRoot==0 ){ + *pnEntry = 0; + return SQLITE_OK; + } + rc = moveToRoot(pCur); + + /* Unless an error occurs, the following loop runs one iteration for each + ** page in the B-Tree structure (not including overflow pages). + */ + while( rc==SQLITE_OK ){ + int iIdx; /* Index of child node in parent */ + MemPage *pPage; /* Current page of the b-tree */ + + /* If this is a leaf page or the tree is not an int-key tree, then + ** this page contains countable entries. Increment the entry counter + ** accordingly. + */ + pPage = pCur->apPage[pCur->iPage]; + if( pPage->leaf || !pPage->intKey ){ + nEntry += pPage->nCell; + } + + /* pPage is a leaf node. This loop navigates the cursor so that it + ** points to the first interior cell that it points to the parent of + ** the next page in the tree that has not yet been visited. The + ** pCur->aiIdx[pCur->iPage] value is set to the index of the parent cell + ** of the page, or to the number of cells in the page if the next page + ** to visit is the right-child of its parent. + ** + ** If all pages in the tree have been visited, return SQLITE_OK to the + ** caller. + */ + if( pPage->leaf ){ + do { + if( pCur->iPage==0 ){ + /* All pages of the b-tree have been visited. Return successfully. */ + *pnEntry = nEntry; + return SQLITE_OK; + } + moveToParent(pCur); + }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell ); + + pCur->aiIdx[pCur->iPage]++; + pPage = pCur->apPage[pCur->iPage]; + } + + /* Descend to the child node of the cell that the cursor currently + ** points at. This is the right-child if (iIdx==pPage->nCell). + */ + iIdx = pCur->aiIdx[pCur->iPage]; + if( iIdx==pPage->nCell ){ + rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); + }else{ + rc = moveToChild(pCur, get4byte(findCell(pPage, iIdx))); + } + } + + /* An error has occurred. Return an error code. */ + return rc; +} +#endif + +/* +** Return the pager associated with a BTree. This routine is used for +** testing and debugging only. +*/ +SQLITE_PRIVATE Pager *sqlite3BtreePager(Btree *p){ + return p->pBt->pPager; +} + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** Append a message to the error message string. +*/ +static void checkAppendMsg( + IntegrityCk *pCheck, + char *zMsg1, + const char *zFormat, + ... +){ + va_list ap; + if( !pCheck->mxErr ) return; + pCheck->mxErr--; + pCheck->nErr++; + va_start(ap, zFormat); + if( pCheck->errMsg.nChar ){ + sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); + } + if( zMsg1 ){ + sqlite3StrAccumAppendAll(&pCheck->errMsg, zMsg1); + } + sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); + va_end(ap); + if( pCheck->errMsg.accError==STRACCUM_NOMEM ){ + pCheck->mallocFailed = 1; + } +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK + +/* +** Return non-zero if the bit in the IntegrityCk.aPgRef[] array that +** corresponds to page iPg is already set. +*/ +static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + return (pCheck->aPgRef[iPg/8] & (1 << (iPg & 0x07))); +} + +/* +** Set the bit in the IntegrityCk.aPgRef[] array that corresponds to page iPg. +*/ +static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07)); +} + + +/* +** Add 1 to the reference count for page iPage. If this is the second +** reference to the page, add an error message to pCheck->zErrMsg. +** Return 1 if there are 2 ore more references to the page and 0 if +** if this is the first reference to the page. +** +** Also check that the page number is in bounds. +*/ +static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ + if( iPage==0 ) return 1; + if( iPage>pCheck->nPage ){ + checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); + return 1; + } + if( getPageReferenced(pCheck, iPage) ){ + checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage); + return 1; + } + setPageReferenced(pCheck, iPage); + return 0; +} + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Check that the entry in the pointer-map for page iChild maps to +** page iParent, pointer type ptrType. If not, append an error message +** to pCheck. +*/ +static void checkPtrmap( + IntegrityCk *pCheck, /* Integrity check context */ + Pgno iChild, /* Child page number */ + u8 eType, /* Expected pointer map type */ + Pgno iParent, /* Expected pointer map parent page number */ + char *zContext /* Context description (used for error msg) */ +){ + int rc; + u8 ePtrmapType; + Pgno iPtrmapParent; + + rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->mallocFailed = 1; + checkAppendMsg(pCheck, zContext, "Failed to read ptrmap key=%d", iChild); + return; + } + + if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ + checkAppendMsg(pCheck, zContext, + "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", + iChild, eType, iParent, ePtrmapType, iPtrmapParent); + } +} +#endif + +/* +** Check the integrity of the freelist or of an overflow page list. +** Verify that the number of pages on the list is N. +*/ +static void checkList( + IntegrityCk *pCheck, /* Integrity checking context */ + int isFreeList, /* True for a freelist. False for overflow page list */ + int iPage, /* Page number for first page in the list */ + int N, /* Expected number of pages in the list */ + char *zContext /* Context for error messages */ +){ + int i; + int expected = N; + int iFirst = iPage; + while( N-- > 0 && pCheck->mxErr ){ + DbPage *pOvflPage; + unsigned char *pOvflData; + if( iPage<1 ){ + checkAppendMsg(pCheck, zContext, + "%d of %d pages missing from overflow list starting at %d", + N+1, expected, iFirst); + break; + } + if( checkRef(pCheck, iPage, zContext) ) break; + if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){ + checkAppendMsg(pCheck, zContext, "failed to get page %d", iPage); + break; + } + pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); + if( isFreeList ){ + int n = get4byte(&pOvflData[4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pCheck->pBt->autoVacuum ){ + checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); + } +#endif + if( n>(int)pCheck->pBt->usableSize/4-2 ){ + checkAppendMsg(pCheck, zContext, + "freelist leaf count too big on page %d", iPage); + N--; + }else{ + for(i=0; ipBt->autoVacuum ){ + checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext); + } +#endif + checkRef(pCheck, iFreePage, zContext); + } + N -= n; + } + } +#ifndef SQLITE_OMIT_AUTOVACUUM + else{ + /* If this database supports auto-vacuum and iPage is not the last + ** page in this overflow list, check that the pointer-map entry for + ** the following page matches iPage. + */ + if( pCheck->pBt->autoVacuum && N>0 ){ + i = get4byte(pOvflData); + checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext); + } + } +#endif + iPage = get4byte(pOvflData); + sqlite3PagerUnref(pOvflPage); + } +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** Do various sanity checks on a single page of a tree. Return +** the tree depth. Root pages return 0. Parents of root pages +** return 1, and so forth. +** +** These checks are done: +** +** 1. Make sure that cells and freeblocks do not overlap +** but combine to completely cover the page. +** NO 2. Make sure cell keys are in order. +** NO 3. Make sure no key is less than or equal to zLowerBound. +** NO 4. Make sure no key is greater than or equal to zUpperBound. +** 5. Check the integrity of overflow pages. +** 6. Recursively call checkTreePage on all children. +** 7. Verify that the depth of all children is the same. +** 8. Make sure this page is at least 33% full or else it is +** the root of the tree. +*/ +static int checkTreePage( + IntegrityCk *pCheck, /* Context for the sanity check */ + int iPage, /* Page number of the page to check */ + char *zParentContext, /* Parent context */ + i64 *pnParentMinKey, + i64 *pnParentMaxKey +){ + MemPage *pPage; + int i, rc, depth, d2, pgno, cnt; + int hdr, cellStart; + int nCell; + u8 *data; + BtShared *pBt; + int usableSize; + char zContext[100]; + char *hit = 0; + i64 nMinKey = 0; + i64 nMaxKey = 0; + + sqlite3_snprintf(sizeof(zContext), zContext, "Page %d: ", iPage); + + /* Check that the page exists + */ + pBt = pCheck->pBt; + usableSize = pBt->usableSize; + if( iPage==0 ) return 0; + if( checkRef(pCheck, iPage, zParentContext) ) return 0; + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + checkAppendMsg(pCheck, zContext, + "unable to get the page. error code=%d", rc); + return 0; + } + + /* Clear MemPage.isInit to make sure the corruption detection code in + ** btreeInitPage() is executed. */ + pPage->isInit = 0; + if( (rc = btreeInitPage(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ + checkAppendMsg(pCheck, zContext, + "btreeInitPage() returns error code %d", rc); + releasePage(pPage); + return 0; + } + + /* Check out all the cells. + */ + depth = 0; + for(i=0; inCell && pCheck->mxErr; i++){ + u8 *pCell; + u32 sz; + CellInfo info; + + /* Check payload overflow pages + */ + sqlite3_snprintf(sizeof(zContext), zContext, + "On tree page %d cell %d: ", iPage, i); + pCell = findCell(pPage,i); + btreeParseCellPtr(pPage, pCell, &info); + sz = info.nData; + if( !pPage->intKey ) sz += (int)info.nKey; + /* For intKey pages, check that the keys are in order. + */ + else if( i==0 ) nMinKey = nMaxKey = info.nKey; + else{ + if( info.nKey <= nMaxKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); + } + nMaxKey = info.nKey; + } + assert( sz==info.nPayload ); + if( (sz>info.nLocal) + && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) + ){ + int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4); + Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext); + } +#endif + checkList(pCheck, 0, pgnoOvfl, nPage, zContext); + } + + /* Check sanity of left child page. + */ + if( !pPage->leaf ){ + pgno = get4byte(pCell); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + } +#endif + d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0 ? NULL : &nMaxKey); + if( i>0 && d2!=depth ){ + checkAppendMsg(pCheck, zContext, "Child page depth differs"); + } + depth = d2; + } + } + + if( !pPage->leaf ){ + pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + sqlite3_snprintf(sizeof(zContext), zContext, + "On page %d at right child: ", iPage); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + } +#endif + checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell ? NULL : &nMaxKey); + } + + /* For intKey leaf pages, check that the min/max keys are in order + ** with any left/parent/right pages. + */ + if( pPage->leaf && pPage->intKey ){ + /* if we are a left child page */ + if( pnParentMinKey ){ + /* if we are the left most child page */ + if( !pnParentMaxKey ){ + if( nMaxKey > *pnParentMinKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (max larger than parent min of %lld)", + nMaxKey, *pnParentMinKey); + } + }else{ + if( nMinKey <= *pnParentMinKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (min less than parent min of %lld)", + nMinKey, *pnParentMinKey); + } + if( nMaxKey > *pnParentMaxKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (max larger than parent max of %lld)", + nMaxKey, *pnParentMaxKey); + } + *pnParentMinKey = nMaxKey; + } + /* else if we're a right child page */ + } else if( pnParentMaxKey ){ + if( nMinKey <= *pnParentMaxKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (min less than parent max of %lld)", + nMinKey, *pnParentMaxKey); + } + } + } + + /* Check for complete coverage of the page + */ + data = pPage->aData; + hdr = pPage->hdrOffset; + hit = sqlite3PageMalloc( pBt->pageSize ); + if( hit==0 ){ + pCheck->mallocFailed = 1; + }else{ + int contentOffset = get2byteNotZero(&data[hdr+5]); + assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ + memset(hit+contentOffset, 0, usableSize-contentOffset); + memset(hit, 1, contentOffset); + nCell = get2byte(&data[hdr+3]); + cellStart = hdr + 12 - 4*pPage->leaf; + for(i=0; i=usableSize ){ + checkAppendMsg(pCheck, 0, + "Corruption detected in cell %d on page %d",i,iPage); + }else{ + for(j=pc+size-1; j>=pc; j--) hit[j]++; + } + } + i = get2byte(&data[hdr+1]); + while( i>0 ){ + int size, j; + assert( i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + size = get2byte(&data[i+2]); + assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ + for(j=i+size-1; j>=i; j--) hit[j]++; + j = get2byte(&data[i]); + assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ + assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + i = j; + } + for(i=cnt=0; i1 ){ + checkAppendMsg(pCheck, 0, + "Multiple uses for byte %d of page %d", i, iPage); + break; + } + } + if( cnt!=data[hdr+7] ){ + checkAppendMsg(pCheck, 0, + "Fragmentation of %d bytes reported as %d on page %d", + cnt, data[hdr+7], iPage); + } + } + sqlite3PageFree(hit); + releasePage(pPage); + return depth+1; +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** This routine does a complete check of the given BTree file. aRoot[] is +** an array of pages numbers were each page number is the root page of +** a table. nRoot is the number of entries in aRoot. +** +** A read-only or read-write transaction must be opened before calling +** this function. +** +** Write the number of error seen in *pnErr. Except for some memory +** allocation errors, an error message held in memory obtained from +** malloc is returned if *pnErr is non-zero. If *pnErr==0 then NULL is +** returned. If a memory allocation error occurs, NULL is returned. +*/ +SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( + Btree *p, /* The btree to be checked */ + int *aRoot, /* An array of root pages numbers for individual trees */ + int nRoot, /* Number of entries in aRoot[] */ + int mxErr, /* Stop reporting errors after this many */ + int *pnErr /* Write number of errors seen to this variable */ +){ + Pgno i; + int nRef; + IntegrityCk sCheck; + BtShared *pBt = p->pBt; + char zErr[100]; + + sqlite3BtreeEnter(p); + assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); + nRef = sqlite3PagerRefcount(pBt->pPager); + sCheck.pBt = pBt; + sCheck.pPager = pBt->pPager; + sCheck.nPage = btreePagecount(sCheck.pBt); + sCheck.mxErr = mxErr; + sCheck.nErr = 0; + sCheck.mallocFailed = 0; + *pnErr = 0; + if( sCheck.nPage==0 ){ + sqlite3BtreeLeave(p); + return 0; + } + + sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); + if( !sCheck.aPgRef ){ + *pnErr = 1; + sqlite3BtreeLeave(p); + return 0; + } + i = PENDING_BYTE_PAGE(pBt); + if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); + sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); + sCheck.errMsg.useMalloc = 2; + + /* Check the integrity of the freelist + */ + checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), + get4byte(&pBt->pPage1->aData[36]), "Main freelist: "); + + /* Check all the tables. + */ + for(i=0; (int)iautoVacuum && aRoot[i]>1 ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); + } +#endif + checkTreePage(&sCheck, aRoot[i], "List of tree roots: ", NULL, NULL); + } + + /* Make sure every page in the file is referenced + */ + for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ +#ifdef SQLITE_OMIT_AUTOVACUUM + if( getPageReferenced(&sCheck, i)==0 ){ + checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + } +#else + /* If the database supports auto-vacuum, make sure no tables contain + ** references to pointer-map pages. + */ + if( getPageReferenced(&sCheck, i)==0 && + (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + } + if( getPageReferenced(&sCheck, i)!=0 && + (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i); + } +#endif + } + + /* Make sure this analysis did not leave any unref() pages. + ** This is an internal consistency check; an integrity check + ** of the integrity check. + */ + if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ + checkAppendMsg(&sCheck, 0, + "Outstanding page count goes from %d to %d during this analysis", + nRef, sqlite3PagerRefcount(pBt->pPager) + ); + } + + /* Clean up and report errors. + */ + sqlite3BtreeLeave(p); + sqlite3_free(sCheck.aPgRef); + if( sCheck.mallocFailed ){ + sqlite3StrAccumReset(&sCheck.errMsg); + *pnErr = sCheck.nErr+1; + return 0; + } + *pnErr = sCheck.nErr; + if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg); + return sqlite3StrAccumFinish(&sCheck.errMsg); +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +/* +** Return the full pathname of the underlying database file. Return +** an empty string if the database is in-memory or a TEMP database. +** +** The pager filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. +*/ +SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *p){ + assert( p->pBt->pPager!=0 ); + return sqlite3PagerFilename(p->pBt->pPager, 1); +} + +/* +** Return the pathname of the journal file for this database. The return +** value of this routine is the same regardless of whether the journal file +** has been created or not. +** +** The pager journal filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. +*/ +SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *p){ + assert( p->pBt->pPager!=0 ); + return sqlite3PagerJournalname(p->pBt->pPager); +} + +/* +** Return non-zero if a transaction is active. +*/ +SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree *p){ + assert( p==0 || sqlite3_mutex_held(p->db->mutex) ); + return (p && (p->inTrans==TRANS_WRITE)); +} + +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on the Btree passed as the first argument. +** +** Return SQLITE_LOCKED if this or any other connection has an open +** transaction on the shared-cache the argument Btree is connected to. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; + if( p ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( pBt->inTransaction!=TRANS_NONE ){ + rc = SQLITE_LOCKED; + }else{ + rc = sqlite3PagerCheckpoint(pBt->pPager, eMode, pnLog, pnCkpt); + } + sqlite3BtreeLeave(p); + } + return rc; +} +#endif + +/* +** Return non-zero if a read (or write) transaction is active. +*/ +SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree *p){ + assert( p ); + assert( sqlite3_mutex_held(p->db->mutex) ); + return p->inTrans!=TRANS_NONE; +} + +SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree *p){ + assert( p ); + assert( sqlite3_mutex_held(p->db->mutex) ); + return p->nBackup!=0; +} + +/* +** This function returns a pointer to a blob of memory associated with +** a single shared-btree. The memory is used by client code for its own +** purposes (for example, to store a high-level schema associated with +** the shared-btree). The btree layer manages reference counting issues. +** +** The first time this is called on a shared-btree, nBytes bytes of memory +** are allocated, zeroed, and returned to the caller. For each subsequent +** call the nBytes parameter is ignored and a pointer to the same blob +** of memory returned. +** +** If the nBytes parameter is 0 and the blob of memory has not yet been +** allocated, a null pointer is returned. If the blob has already been +** allocated, it is returned as normal. +** +** Just before the shared-btree is closed, the function passed as the +** xFree argument when the memory allocation was made is invoked on the +** blob of allocated memory. The xFree function should not call sqlite3_free() +** on the memory, the btree layer does that. +*/ +SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( !pBt->pSchema && nBytes ){ + pBt->pSchema = sqlite3DbMallocZero(0, nBytes); + pBt->xFreeSchema = xFree; + } + sqlite3BtreeLeave(p); + return pBt->pSchema; +} + +/* +** Return SQLITE_LOCKED_SHAREDCACHE if another user of the same shared +** btree as the argument handle holds an exclusive lock on the +** sqlite_master table. Otherwise SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *p){ + int rc; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + rc = querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK); + assert( rc==SQLITE_OK || rc==SQLITE_LOCKED_SHAREDCACHE ); + sqlite3BtreeLeave(p); + return rc; +} + + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Obtain a lock on the table whose root page is iTab. The +** lock is a write lock if isWritelock is true or a read lock +** if it is false. +*/ +SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ + int rc = SQLITE_OK; + assert( p->inTrans!=TRANS_NONE ); + if( p->sharable ){ + u8 lockType = READ_LOCK + isWriteLock; + assert( READ_LOCK+1==WRITE_LOCK ); + assert( isWriteLock==0 || isWriteLock==1 ); + + sqlite3BtreeEnter(p); + rc = querySharedCacheTableLock(p, iTab, lockType); + if( rc==SQLITE_OK ){ + rc = setSharedCacheTableLock(p, iTab, lockType); + } + sqlite3BtreeLeave(p); + } + return rc; +} +#endif + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Argument pCsr must be a cursor opened for writing on an +** INTKEY table currently pointing at a valid table entry. +** This function modifies the data stored as part of that entry. +** +** Only the data content may only be modified, it is not possible to +** change the length of the data stored. If this function is called with +** parameters that attempt to write past the end of the existing data, +** no modifications are made and SQLITE_CORRUPT is returned. +*/ +SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ + int rc; + assert( cursorHoldsMutex(pCsr) ); + assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); + assert( pCsr->curFlags & BTCF_Incrblob ); + + rc = restoreCursorPosition(pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( pCsr->eState!=CURSOR_REQUIRESEEK ); + if( pCsr->eState!=CURSOR_VALID ){ + return SQLITE_ABORT; + } + + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. + */ + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); + + /* Check some assumptions: + ** (a) the cursor is open for writing, + ** (b) there is a read/write transaction open, + ** (c) the connection holds a write-lock on the table (if required), + ** (d) there are no conflicting read-locks, and + ** (e) the cursor points at a valid row of an intKey table. + */ + if( (pCsr->curFlags & BTCF_WriteFlag)==0 ){ + return SQLITE_READONLY; + } + assert( (pCsr->pBt->btsFlags & BTS_READ_ONLY)==0 + && pCsr->pBt->inTransaction==TRANS_WRITE ); + assert( hasSharedCacheTableLock(pCsr->pBtree, pCsr->pgnoRoot, 0, 2) ); + assert( !hasReadConflicts(pCsr->pBtree, pCsr->pgnoRoot) ); + assert( pCsr->apPage[pCsr->iPage]->intKey ); + + return accessPayload(pCsr, offset, amt, (unsigned char *)z, 1); +} + +/* +** Mark this cursor as an incremental blob cursor. +*/ +SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ + pCur->curFlags |= BTCF_Incrblob; +} +#endif + +/* +** Set both the "read version" (single byte at byte offset 18) and +** "write version" (single byte at byte offset 19) fields in the database +** header to iVersion. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ + BtShared *pBt = pBtree->pBt; + int rc; /* Return code */ + + assert( iVersion==1 || iVersion==2 ); + + /* If setting the version fields to 1, do not automatically open the + ** WAL connection, even if the version fields are currently set to 2. + */ + pBt->btsFlags &= ~BTS_NO_WAL; + if( iVersion==1 ) pBt->btsFlags |= BTS_NO_WAL; + + rc = sqlite3BtreeBeginTrans(pBtree, 0); + if( rc==SQLITE_OK ){ + u8 *aData = pBt->pPage1->aData; + if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ + rc = sqlite3BtreeBeginTrans(pBtree, 2); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + aData[18] = (u8)iVersion; + aData[19] = (u8)iVersion; + } + } + } + } + + pBt->btsFlags &= ~BTS_NO_WAL; + return rc; +} + +/* +** set the mask of hint flags for cursor pCsr. Currently the only valid +** values are 0 and BTREE_BULKLOAD. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ + assert( mask==BTREE_BULKLOAD || mask==0 ); + pCsr->hints = mask; +} + +/* +** Return true if the given Btree is read-only. +*/ +SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ + return (p->pBt->btsFlags & BTS_READ_ONLY)!=0; +} + +/************** End of btree.c ***********************************************/ +/************** Begin file backup.c ******************************************/ +/* +** 2009 January 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the implementation of the sqlite3_backup_XXX() +** API functions and the related features. +*/ + +/* +** Structure allocated for each backup operation. +*/ +struct sqlite3_backup { + sqlite3* pDestDb; /* Destination database handle */ + Btree *pDest; /* Destination b-tree file */ + u32 iDestSchema; /* Original schema cookie in destination */ + int bDestLocked; /* True once a write-transaction is open on pDest */ + + Pgno iNext; /* Page number of the next source page to copy */ + sqlite3* pSrcDb; /* Source database handle */ + Btree *pSrc; /* Source b-tree file */ + + int rc; /* Backup process error code */ + + /* These two variables are set by every call to backup_step(). They are + ** read by calls to backup_remaining() and backup_pagecount(). + */ + Pgno nRemaining; /* Number of pages left to copy */ + Pgno nPagecount; /* Total number of pages to copy */ + + int isAttached; /* True once backup has been registered with pager */ + sqlite3_backup *pNext; /* Next backup associated with source pager */ +}; + +/* +** THREAD SAFETY NOTES: +** +** Once it has been created using backup_init(), a single sqlite3_backup +** structure may be accessed via two groups of thread-safe entry points: +** +** * Via the sqlite3_backup_XXX() API function backup_step() and +** backup_finish(). Both these functions obtain the source database +** handle mutex and the mutex associated with the source BtShared +** structure, in that order. +** +** * Via the BackupUpdate() and BackupRestart() functions, which are +** invoked by the pager layer to report various state changes in +** the page cache associated with the source database. The mutex +** associated with the source database BtShared structure will always +** be held when either of these functions are invoked. +** +** The other sqlite3_backup_XXX() API functions, backup_remaining() and +** backup_pagecount() are not thread-safe functions. If they are called +** while some other thread is calling backup_step() or backup_finish(), +** the values returned may be invalid. There is no way for a call to +** BackupUpdate() or BackupRestart() to interfere with backup_remaining() +** or backup_pagecount(). +** +** Depending on the SQLite configuration, the database handles and/or +** the Btree objects may have their own mutexes that require locking. +** Non-sharable Btrees (in-memory databases for example), do not have +** associated mutexes. +*/ + +/* +** Return a pointer corresponding to database zDb (i.e. "main", "temp") +** in connection handle pDb. If such a database cannot be found, return +** a NULL pointer and write an error message to pErrorDb. +** +** If the "temp" database is requested, it may need to be opened by this +** function. If an error occurs while doing so, return 0 and write an +** error message to pErrorDb. +*/ +static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ + int i = sqlite3FindDbName(pDb, zDb); + + if( i==1 ){ + Parse *pParse; + int rc = 0; + pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); + if( pParse==0 ){ + sqlite3Error(pErrorDb, SQLITE_NOMEM, "out of memory"); + rc = SQLITE_NOMEM; + }else{ + pParse->db = pDb; + if( sqlite3OpenTempDatabase(pParse) ){ + sqlite3Error(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); + rc = SQLITE_ERROR; + } + sqlite3DbFree(pErrorDb, pParse->zErrMsg); + sqlite3ParserReset(pParse); + sqlite3StackFree(pErrorDb, pParse); + } + if( rc ){ + return 0; + } + } + + if( i<0 ){ + sqlite3Error(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); + return 0; + } + + return pDb->aDb[i].pBt; +} + +/* +** Attempt to set the page size of the destination to match the page size +** of the source. +*/ +static int setDestPgsz(sqlite3_backup *p){ + int rc; + rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),-1,0); + return rc; +} + +/* +** Create an sqlite3_backup process to copy the contents of zSrcDb from +** connection handle pSrcDb to zDestDb in pDestDb. If successful, return +** a pointer to the new sqlite3_backup object. +** +** If an error occurs, NULL is returned and an error code and error message +** stored in database handle pDestDb. +*/ +SQLITE_API sqlite3_backup *sqlite3_backup_init( + sqlite3* pDestDb, /* Database to write to */ + const char *zDestDb, /* Name of database within pDestDb */ + sqlite3* pSrcDb, /* Database connection to read from */ + const char *zSrcDb /* Name of database within pSrcDb */ +){ + sqlite3_backup *p; /* Value to return */ + + /* Lock the source database handle. The destination database + ** handle is not locked in this routine, but it is locked in + ** sqlite3_backup_step(). The user is required to ensure that no + ** other thread accesses the destination handle for the duration + ** of the backup operation. Any attempt to use the destination + ** database connection while a backup is in progress may cause + ** a malfunction or a deadlock. + */ + sqlite3_mutex_enter(pSrcDb->mutex); + sqlite3_mutex_enter(pDestDb->mutex); + + if( pSrcDb==pDestDb ){ + sqlite3Error( + pDestDb, SQLITE_ERROR, "source and destination must be distinct" + ); + p = 0; + }else { + /* Allocate space for a new sqlite3_backup object... + ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ + p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); + if( !p ){ + sqlite3Error(pDestDb, SQLITE_NOMEM, 0); + } + } + + /* If the allocation succeeded, populate the new object. */ + if( p ){ + p->pSrc = findBtree(pDestDb, pSrcDb, zSrcDb); + p->pDest = findBtree(pDestDb, pDestDb, zDestDb); + p->pDestDb = pDestDb; + p->pSrcDb = pSrcDb; + p->iNext = 1; + p->isAttached = 0; + + if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){ + /* One (or both) of the named databases did not exist or an OOM + ** error was hit. The error has already been written into the + ** pDestDb handle. All that is left to do here is free the + ** sqlite3_backup structure. + */ + sqlite3_free(p); + p = 0; + } + } + if( p ){ + p->pSrc->nBackup++; + } + + sqlite3_mutex_leave(pDestDb->mutex); + sqlite3_mutex_leave(pSrcDb->mutex); + return p; +} + +/* +** Argument rc is an SQLite error code. Return true if this error is +** considered fatal if encountered during a backup operation. All errors +** are considered fatal except for SQLITE_BUSY and SQLITE_LOCKED. +*/ +static int isFatalError(int rc){ + return (rc!=SQLITE_OK && rc!=SQLITE_BUSY && ALWAYS(rc!=SQLITE_LOCKED)); +} + +/* +** Parameter zSrcData points to a buffer containing the data for +** page iSrcPg from the source database. Copy this data into the +** destination database. +*/ +static int backupOnePage( + sqlite3_backup *p, /* Backup handle */ + Pgno iSrcPg, /* Source database page to backup */ + const u8 *zSrcData, /* Source database page data */ + int bUpdate /* True for an update, false otherwise */ +){ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); + const int nSrcPgsz = sqlite3BtreeGetPageSize(p->pSrc); + int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); + const int nCopy = MIN(nSrcPgsz, nDestPgsz); + const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; +#ifdef SQLITE_HAS_CODEC + /* Use BtreeGetReserveNoMutex() for the source b-tree, as although it is + ** guaranteed that the shared-mutex is held by this thread, handle + ** p->pSrc may not actually be the owner. */ + int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); + int nDestReserve = sqlite3BtreeGetReserve(p->pDest); +#endif + int rc = SQLITE_OK; + i64 iOff; + + assert( sqlite3BtreeGetReserveNoMutex(p->pSrc)>=0 ); + assert( p->bDestLocked ); + assert( !isFatalError(p->rc) ); + assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); + assert( zSrcData ); + + /* Catch the case where the destination is an in-memory database and the + ** page sizes of the source and destination differ. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ + rc = SQLITE_READONLY; + } + +#ifdef SQLITE_HAS_CODEC + /* Backup is not possible if the page size of the destination is changing + ** and a codec is in use. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ + rc = SQLITE_READONLY; + } + + /* Backup is not possible if the number of bytes of reserve space differ + ** between source and destination. If there is a difference, try to + ** fix the destination to agree with the source. If that is not possible, + ** then the backup cannot proceed. + */ + if( nSrcReserve!=nDestReserve ){ + u32 newPgsz = nSrcPgsz; + rc = sqlite3PagerSetPagesize(pDestPager, &newPgsz, nSrcReserve); + if( rc==SQLITE_OK && newPgsz!=nSrcPgsz ) rc = SQLITE_READONLY; + } +#endif + + /* This loop runs once for each destination page spanned by the source + ** page. For each iteration, variable iOff is set to the byte offset + ** of the destination page. + */ + for(iOff=iEnd-(i64)nSrcPgsz; rc==SQLITE_OK && iOffpDest->pBt) ) continue; + if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg)) + && SQLITE_OK==(rc = sqlite3PagerWrite(pDestPg)) + ){ + const u8 *zIn = &zSrcData[iOff%nSrcPgsz]; + u8 *zDestData = sqlite3PagerGetData(pDestPg); + u8 *zOut = &zDestData[iOff%nDestPgsz]; + + /* Copy the data from the source page into the destination page. + ** Then clear the Btree layer MemPage.isInit flag. Both this module + ** and the pager code use this trick (clearing the first byte + ** of the page 'extra' space to invalidate the Btree layers + ** cached parse of the page). MemPage.isInit is marked + ** "MUST BE FIRST" for this purpose. + */ + memcpy(zOut, zIn, nCopy); + ((u8 *)sqlite3PagerGetExtra(pDestPg))[0] = 0; + if( iOff==0 && bUpdate==0 ){ + sqlite3Put4byte(&zOut[28], sqlite3BtreeLastPage(p->pSrc)); + } + } + sqlite3PagerUnref(pDestPg); + } + + return rc; +} + +/* +** If pFile is currently larger than iSize bytes, then truncate it to +** exactly iSize bytes. If pFile is not larger than iSize bytes, then +** this function is a no-op. +** +** Return SQLITE_OK if everything is successful, or an SQLite error +** code if an error occurs. +*/ +static int backupTruncateFile(sqlite3_file *pFile, i64 iSize){ + i64 iCurrent; + int rc = sqlite3OsFileSize(pFile, &iCurrent); + if( rc==SQLITE_OK && iCurrent>iSize ){ + rc = sqlite3OsTruncate(pFile, iSize); + } + return rc; +} + +/* +** Register this backup object with the associated source pager for +** callbacks when pages are changed or the cache invalidated. +*/ +static void attachBackupObject(sqlite3_backup *p){ + sqlite3_backup **pp; + assert( sqlite3BtreeHoldsMutex(p->pSrc) ); + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + p->pNext = *pp; + *pp = p; + p->isAttached = 1; +} + +/* +** Copy nPage pages from the source b-tree to the destination. +*/ +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ + int rc; + int destMode; /* Destination journal mode */ + int pgszSrc = 0; /* Source page size */ + int pgszDest = 0; /* Destination page size */ + + sqlite3_mutex_enter(p->pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + if( p->pDestDb ){ + sqlite3_mutex_enter(p->pDestDb->mutex); + } + + rc = p->rc; + if( !isFatalError(rc) ){ + Pager * const pSrcPager = sqlite3BtreePager(p->pSrc); /* Source pager */ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); /* Dest pager */ + int ii; /* Iterator variable */ + int nSrcPage = -1; /* Size of source db in pages */ + int bCloseTrans = 0; /* True if src db requires unlocking */ + + /* If the source pager is currently in a write-transaction, return + ** SQLITE_BUSY immediately. + */ + if( p->pDestDb && p->pSrc->pBt->inTransaction==TRANS_WRITE ){ + rc = SQLITE_BUSY; + }else{ + rc = SQLITE_OK; + } + + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) + ){ + p->bDestLocked = 1; + sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); + } + + /* If there is no open read-transaction on the source database, open + ** one now. If a transaction is opened here, then it will be closed + ** before this function exits. + */ + if( rc==SQLITE_OK && 0==sqlite3BtreeIsInReadTrans(p->pSrc) ){ + rc = sqlite3BtreeBeginTrans(p->pSrc, 0); + bCloseTrans = 1; + } + + /* Do not allow backup if the destination database is in WAL mode + ** and the page sizes are different between source and destination */ + pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); + pgszDest = sqlite3BtreeGetPageSize(p->pDest); + destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); + if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ + rc = SQLITE_READONLY; + } + + /* Now that there is a read-lock on the source database, query the + ** source pager for the number of pages in the database. + */ + nSrcPage = (int)sqlite3BtreeLastPage(p->pSrc); + assert( nSrcPage>=0 ); + for(ii=0; (nPage<0 || iiiNext<=(Pgno)nSrcPage && !rc; ii++){ + const Pgno iSrcPg = p->iNext; /* Source page number */ + if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ + DbPage *pSrcPg; /* Source page object */ + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_GET_READONLY); + if( rc==SQLITE_OK ){ + rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); + sqlite3PagerUnref(pSrcPg); + } + } + p->iNext++; + } + if( rc==SQLITE_OK ){ + p->nPagecount = nSrcPage; + p->nRemaining = nSrcPage+1-p->iNext; + if( p->iNext>(Pgno)nSrcPage ){ + rc = SQLITE_DONE; + }else if( !p->isAttached ){ + attachBackupObject(p); + } + } + + /* Update the schema version field in the destination database. This + ** is to make sure that the schema-version really does change in + ** the case where the source and destination databases have the + ** same schema version. + */ + if( rc==SQLITE_DONE ){ + if( nSrcPage==0 ){ + rc = sqlite3BtreeNewDb(p->pDest); + nSrcPage = 1; + } + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1); + } + if( rc==SQLITE_OK ){ + if( p->pDestDb ){ + sqlite3ResetAllSchemasOfConnection(p->pDestDb); + } + if( destMode==PAGER_JOURNALMODE_WAL ){ + rc = sqlite3BtreeSetVersion(p->pDest, 2); + } + } + if( rc==SQLITE_OK ){ + int nDestTruncate; + /* Set nDestTruncate to the final number of pages in the destination + ** database. The complication here is that the destination page + ** size may be different to the source page size. + ** + ** If the source page size is smaller than the destination page size, + ** round up. In this case the call to sqlite3OsTruncate() below will + ** fix the size of the file. However it is important to call + ** sqlite3PagerTruncateImage() here so that any pages in the + ** destination file that lie beyond the nDestTruncate page mark are + ** journalled by PagerCommitPhaseOne() before they are destroyed + ** by the file truncation. + */ + assert( pgszSrc==sqlite3BtreeGetPageSize(p->pSrc) ); + assert( pgszDest==sqlite3BtreeGetPageSize(p->pDest) ); + if( pgszSrcpDest->pBt) ){ + nDestTruncate--; + } + }else{ + nDestTruncate = nSrcPage * (pgszSrc/pgszDest); + } + assert( nDestTruncate>0 ); + + if( pgszSrc= iSize || ( + nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) + && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest + )); + + /* This block ensures that all data required to recreate the original + ** database has been stored in the journal for pDestPager and the + ** journal synced to disk. So at this point we may safely modify + ** the database file in any way, knowing that if a power failure + ** occurs, the original database will be reconstructed from the + ** journal file. */ + sqlite3PagerPagecount(pDestPager, &nDstPage); + for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ + if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ + DbPage *pPg; + rc = sqlite3PagerGet(pDestPager, iPg, &pPg); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg); + sqlite3PagerUnref(pPg); + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + } + + /* Write the extra pages and truncate the database file as required */ + iEnd = MIN(PENDING_BYTE + pgszDest, iSize); + for( + iOff=PENDING_BYTE+pgszSrc; + rc==SQLITE_OK && iOffpDest, 0)) + ){ + rc = SQLITE_DONE; + } + } + } + + /* If bCloseTrans is true, then this function opened a read transaction + ** on the source database. Close the read transaction here. There is + ** no need to check the return values of the btree methods here, as + ** "committing" a read-only transaction cannot fail. + */ + if( bCloseTrans ){ + TESTONLY( int rc2 ); + TESTONLY( rc2 = ) sqlite3BtreeCommitPhaseOne(p->pSrc, 0); + TESTONLY( rc2 |= ) sqlite3BtreeCommitPhaseTwo(p->pSrc, 0); + assert( rc2==SQLITE_OK ); + } + + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM; + } + p->rc = rc; + } + if( p->pDestDb ){ + sqlite3_mutex_leave(p->pDestDb->mutex); + } + sqlite3BtreeLeave(p->pSrc); + sqlite3_mutex_leave(p->pSrcDb->mutex); + return rc; +} + +/* +** Release all resources associated with an sqlite3_backup* handle. +*/ +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ + sqlite3_backup **pp; /* Ptr to head of pagers backup list */ + sqlite3 *pSrcDb; /* Source database connection */ + int rc; /* Value to return */ + + /* Enter the mutexes */ + if( p==0 ) return SQLITE_OK; + pSrcDb = p->pSrcDb; + sqlite3_mutex_enter(pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + if( p->pDestDb ){ + sqlite3_mutex_enter(p->pDestDb->mutex); + } + + /* Detach this backup from the source pager. */ + if( p->pDestDb ){ + p->pSrc->nBackup--; + } + if( p->isAttached ){ + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + while( *pp!=p ){ + pp = &(*pp)->pNext; + } + *pp = p->pNext; + } + + /* If a transaction is still open on the Btree, roll it back. */ + sqlite3BtreeRollback(p->pDest, SQLITE_OK); + + /* Set the error code of the destination database handle. */ + rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; + if( p->pDestDb ){ + sqlite3Error(p->pDestDb, rc, 0); + + /* Exit the mutexes and free the backup context structure. */ + sqlite3LeaveMutexAndCloseZombie(p->pDestDb); + } + sqlite3BtreeLeave(p->pSrc); + if( p->pDestDb ){ + /* EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ + sqlite3_free(p); + } + sqlite3LeaveMutexAndCloseZombie(pSrcDb); + return rc; +} + +/* +** Return the number of pages still to be backed up as of the most recent +** call to sqlite3_backup_step(). +*/ +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ + return p->nRemaining; +} + +/* +** Return the total number of pages in the source database as of the most +** recent call to sqlite3_backup_step(). +*/ +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ + return p->nPagecount; +} + +/* +** This function is called after the contents of page iPage of the +** source database have been modified. If page iPage has already been +** copied into the destination database, then the data written to the +** destination is now invalidated. The destination copy of iPage needs +** to be updated with the new data before the backup operation is +** complete. +** +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. +*/ +SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ + sqlite3_backup *p; /* Iterator variable */ + for(p=pBackup; p; p=p->pNext){ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + if( !isFatalError(p->rc) && iPageiNext ){ + /* The backup process p has already copied page iPage. But now it + ** has been modified by a transaction on the source pager. Copy + ** the new data into the backup. + */ + int rc; + assert( p->pDestDb ); + sqlite3_mutex_enter(p->pDestDb->mutex); + rc = backupOnePage(p, iPage, aData, 1); + sqlite3_mutex_leave(p->pDestDb->mutex); + assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED ); + if( rc!=SQLITE_OK ){ + p->rc = rc; + } + } + } +} + +/* +** Restart the backup process. This is called when the pager layer +** detects that the database has been modified by an external database +** connection. In this case there is no way of knowing which of the +** pages that have been copied into the destination database are still +** valid and which are not, so the entire process needs to be restarted. +** +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. +*/ +SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *pBackup){ + sqlite3_backup *p; /* Iterator variable */ + for(p=pBackup; p; p=p->pNext){ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + p->iNext = 1; + } +} + +#ifndef SQLITE_OMIT_VACUUM +/* +** Copy the complete content of pBtFrom into pBtTo. A transaction +** must be active for both files. +** +** The size of file pTo may be reduced by this operation. If anything +** goes wrong, the transaction on pTo is rolled back. If successful, the +** transaction is committed before returning. +*/ +SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ + int rc; + sqlite3_file *pFd; /* File descriptor for database pTo */ + sqlite3_backup b; + sqlite3BtreeEnter(pTo); + sqlite3BtreeEnter(pFrom); + + assert( sqlite3BtreeIsInTrans(pTo) ); + pFd = sqlite3PagerFile(sqlite3BtreePager(pTo)); + if( pFd->pMethods ){ + i64 nByte = sqlite3BtreeGetPageSize(pFrom)*(i64)sqlite3BtreeLastPage(pFrom); + rc = sqlite3OsFileControl(pFd, SQLITE_FCNTL_OVERWRITE, &nByte); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc ) goto copy_finished; + } + + /* Set up an sqlite3_backup object. sqlite3_backup.pDestDb must be set + ** to 0. This is used by the implementations of sqlite3_backup_step() + ** and sqlite3_backup_finish() to detect that they are being called + ** from this function, not directly by the user. + */ + memset(&b, 0, sizeof(b)); + b.pSrcDb = pFrom->db; + b.pSrc = pFrom; + b.pDest = pTo; + b.iNext = 1; + + /* 0x7FFFFFFF is the hard limit for the number of pages in a database + ** file. By passing this as the number of pages to copy to + ** sqlite3_backup_step(), we can guarantee that the copy finishes + ** within a single call (unless an error occurs). The assert() statement + ** checks this assumption - (p->rc) should be set to either SQLITE_DONE + ** or an error code. + */ + sqlite3_backup_step(&b, 0x7FFFFFFF); + assert( b.rc!=SQLITE_OK ); + rc = sqlite3_backup_finish(&b); + if( rc==SQLITE_OK ){ + pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; + }else{ + sqlite3PagerClearCache(sqlite3BtreePager(b.pDest)); + } + + assert( sqlite3BtreeIsInTrans(pTo)==0 ); +copy_finished: + sqlite3BtreeLeave(pFrom); + sqlite3BtreeLeave(pTo); + return rc; +} +#endif /* SQLITE_OMIT_VACUUM */ + +/************** End of backup.c **********************************************/ +/************** Begin file vdbemem.c *****************************************/ +/* +** 2004 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code use to manipulate "Mem" structure. A "Mem" +** stores a single value in the VDBE. Mem is an opaque structure visible +** only within the VDBE. Interface routines refer to a Mem using the +** name sqlite_value +*/ + +#ifdef SQLITE_DEBUG +/* +** Check invariants on a Mem object. +** +** This routine is intended for use inside of assert() statements, like +** this: assert( sqlite3VdbeCheckMemInvariants(pMem) ); +*/ +SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ + /* The MEM_Dyn bit is set if and only if Mem.xDel is a non-NULL destructor + ** function for Mem.z + */ + assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); + assert( (p->flags & MEM_Dyn)!=0 || p->xDel==0 ); + + /* If p holds a string or blob, the Mem.z must point to exactly + ** one of the following: + ** + ** (1) Memory in Mem.zMalloc and managed by the Mem object + ** (2) Memory to be freed using Mem.xDel + ** (3) An ephermal string or blob + ** (4) A static string or blob + */ + if( (p->flags & (MEM_Str|MEM_Blob)) && p->z!=0 ){ + assert( + ((p->z==p->zMalloc)? 1 : 0) + + ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + + ((p->flags&MEM_Static)!=0 ? 1 : 0) == 1 + ); + } + + return 1; +} +#endif + + +/* +** If pMem is an object with a valid string representation, this routine +** ensures the internal encoding for the string representation is +** 'desiredEnc', one of SQLITE_UTF8, SQLITE_UTF16LE or SQLITE_UTF16BE. +** +** If pMem is not a string object, or the encoding of the string +** representation is already stored using the requested encoding, then this +** routine is a no-op. +** +** SQLITE_OK is returned if the conversion is successful (or not required). +** SQLITE_NOMEM may be returned if a malloc() fails during conversion +** between formats. +*/ +SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ +#ifndef SQLITE_OMIT_UTF16 + int rc; +#endif + assert( (pMem->flags&MEM_RowSet)==0 ); + assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE + || desiredEnc==SQLITE_UTF16BE ); + if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){ + return SQLITE_OK; + } + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); +#ifdef SQLITE_OMIT_UTF16 + return SQLITE_ERROR; +#else + + /* MemTranslate() may return SQLITE_OK or SQLITE_NOMEM. If NOMEM is returned, + ** then the encoding of the value may not have changed. + */ + rc = sqlite3VdbeMemTranslate(pMem, (u8)desiredEnc); + assert(rc==SQLITE_OK || rc==SQLITE_NOMEM); + assert(rc==SQLITE_OK || pMem->enc!=desiredEnc); + assert(rc==SQLITE_NOMEM || pMem->enc==desiredEnc); + return rc; +#endif +} + +/* +** Make sure pMem->z points to a writable allocation of at least +** min(n,32) bytes. +** +** If the bPreserve argument is true, then copy of the content of +** pMem->z into the new allocation. pMem must be either a string or +** blob if bPreserve is true. If bPreserve is false, any prior content +** in pMem->z is discarded. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ + assert( sqlite3VdbeCheckMemInvariants(pMem) ); + assert( (pMem->flags&MEM_RowSet)==0 ); + + /* If the bPreserve flag is set to true, then the memory cell must already + ** contain a valid string or blob value. */ + assert( bPreserve==0 || pMem->flags&(MEM_Blob|MEM_Str) ); + testcase( bPreserve && pMem->z==0 ); + + if( pMem->zMalloc==0 || sqlite3DbMallocSize(pMem->db, pMem->zMalloc)z==pMem->zMalloc ){ + pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); + bPreserve = 0; + }else{ + sqlite3DbFree(pMem->db, pMem->zMalloc); + pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); + } + if( pMem->zMalloc==0 ){ + VdbeMemRelease(pMem); + pMem->z = 0; + pMem->flags = MEM_Null; + return SQLITE_NOMEM; + } + } + + if( pMem->z && bPreserve && pMem->z!=pMem->zMalloc ){ + memcpy(pMem->zMalloc, pMem->z, pMem->n); + } + if( (pMem->flags&MEM_Dyn)!=0 ){ + assert( pMem->xDel!=0 && pMem->xDel!=SQLITE_DYNAMIC ); + pMem->xDel((void *)(pMem->z)); + } + + pMem->z = pMem->zMalloc; + pMem->flags &= ~(MEM_Dyn|MEM_Ephem|MEM_Static); + pMem->xDel = 0; + return SQLITE_OK; +} + +/* +** Make the given Mem object MEM_Dyn. In other words, make it so +** that any TEXT or BLOB content is stored in memory obtained from +** malloc(). In this way, we know that the memory is safe to be +** overwritten or altered. +** +** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ + int f; + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( (pMem->flags&MEM_RowSet)==0 ); + ExpandBlob(pMem); + f = pMem->flags; + if( (f&(MEM_Str|MEM_Blob)) && pMem->z!=pMem->zMalloc ){ + if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ + return SQLITE_NOMEM; + } + pMem->z[pMem->n] = 0; + pMem->z[pMem->n+1] = 0; + pMem->flags |= MEM_Term; +#ifdef SQLITE_DEBUG + pMem->pScopyFrom = 0; +#endif + } + + return SQLITE_OK; +} + +/* +** If the given Mem* has a zero-filled tail, turn it into an ordinary +** blob stored in dynamically allocated space. +*/ +#ifndef SQLITE_OMIT_INCRBLOB +SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ + if( pMem->flags & MEM_Zero ){ + int nByte; + assert( pMem->flags&MEM_Blob ); + assert( (pMem->flags&MEM_RowSet)==0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + + /* Set nByte to the number of bytes required to store the expanded blob. */ + nByte = pMem->n + pMem->u.nZero; + if( nByte<=0 ){ + nByte = 1; + } + if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ + return SQLITE_NOMEM; + } + + memset(&pMem->z[pMem->n], 0, pMem->u.nZero); + pMem->n += pMem->u.nZero; + pMem->flags &= ~(MEM_Zero|MEM_Term); + } + return SQLITE_OK; +} +#endif + + +/* +** Make sure the given Mem is \u0000 terminated. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + if( (pMem->flags & MEM_Term)!=0 || (pMem->flags & MEM_Str)==0 ){ + return SQLITE_OK; /* Nothing to do */ + } + if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ + return SQLITE_NOMEM; + } + pMem->z[pMem->n] = 0; + pMem->z[pMem->n+1] = 0; + pMem->flags |= MEM_Term; + return SQLITE_OK; +} + +/* +** Add MEM_Str to the set of representations for the given Mem. Numbers +** are converted using sqlite3_snprintf(). Converting a BLOB to a string +** is a no-op. +** +** Existing representations MEM_Int and MEM_Real are *not* invalidated. +** +** A MEM_Null value will never be passed to this function. This function is +** used for converting values to text for returning to the user (i.e. via +** sqlite3_value_text()), or for ensuring that values to be used as btree +** keys are strings. In the former case a NULL pointer is returned the +** user and the later is an internal programming error. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, int enc){ + int rc = SQLITE_OK; + int fg = pMem->flags; + const int nByte = 32; + + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( !(fg&MEM_Zero) ); + assert( !(fg&(MEM_Str|MEM_Blob)) ); + assert( fg&(MEM_Int|MEM_Real) ); + assert( (pMem->flags&MEM_RowSet)==0 ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + + if( sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + return SQLITE_NOMEM; + } + + /* For a Real or Integer, use sqlite3_mprintf() to produce the UTF-8 + ** string representation of the value. Then, if the required encoding + ** is UTF-16le or UTF-16be do a translation. + ** + ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. + */ + if( fg & MEM_Int ){ + sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); + }else{ + assert( fg & MEM_Real ); + sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->r); + } + pMem->n = sqlite3Strlen30(pMem->z); + pMem->enc = SQLITE_UTF8; + pMem->flags |= MEM_Str|MEM_Term; + sqlite3VdbeChangeEncoding(pMem, enc); + return rc; +} + +/* +** Memory cell pMem contains the context of an aggregate function. +** This routine calls the finalize method for that function. The +** result of the aggregate is stored back into pMem. +** +** Return SQLITE_ERROR if the finalizer reports an error. SQLITE_OK +** otherwise. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ + int rc = SQLITE_OK; + if( ALWAYS(pFunc && pFunc->xFinalize) ){ + sqlite3_context ctx; + assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); + ctx.s.flags = MEM_Null; + ctx.s.db = pMem->db; + ctx.pMem = pMem; + ctx.pFunc = pFunc; + pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ + assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); + sqlite3DbFree(pMem->db, pMem->zMalloc); + memcpy(pMem, &ctx.s, sizeof(ctx.s)); + rc = ctx.isError; + } + return rc; +} + +/* +** If the memory cell contains a string value that must be freed by +** invoking an external callback, free it now. Calling this function +** does not free any Mem.zMalloc buffer. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p){ + assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); + if( p->flags&MEM_Agg ){ + sqlite3VdbeMemFinalize(p, p->u.pDef); + assert( (p->flags & MEM_Agg)==0 ); + sqlite3VdbeMemRelease(p); + }else if( p->flags&MEM_Dyn ){ + assert( (p->flags&MEM_RowSet)==0 ); + assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); + p->xDel((void *)p->z); + p->xDel = 0; + }else if( p->flags&MEM_RowSet ){ + sqlite3RowSetClear(p->u.pRowSet); + }else if( p->flags&MEM_Frame ){ + sqlite3VdbeMemSetNull(p); + } +} + +/* +** Release any memory held by the Mem. This may leave the Mem in an +** inconsistent state, for example with (Mem.z==0) and +** (Mem.flags==MEM_Str). +*/ +SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p){ + assert( sqlite3VdbeCheckMemInvariants(p) ); + VdbeMemRelease(p); + if( p->zMalloc ){ + sqlite3DbFree(p->db, p->zMalloc); + p->zMalloc = 0; + } + p->z = 0; + assert( p->xDel==0 ); /* Zeroed by VdbeMemRelease() above */ +} + +/* +** Convert a 64-bit IEEE double into a 64-bit signed integer. +** If the double is out of range of a 64-bit signed integer then +** return the closest available 64-bit signed integer. +*/ +static i64 doubleToInt64(double r){ +#ifdef SQLITE_OMIT_FLOATING_POINT + /* When floating-point is omitted, double and int64 are the same thing */ + return r; +#else + /* + ** Many compilers we encounter do not define constants for the + ** minimum and maximum 64-bit integers, or they define them + ** inconsistently. And many do not understand the "LL" notation. + ** So we define our own static constants here using nothing + ** larger than a 32-bit integer constant. + */ + static const i64 maxInt = LARGEST_INT64; + static const i64 minInt = SMALLEST_INT64; + + if( r<=(double)minInt ){ + return minInt; + }else if( r>=(double)maxInt ){ + return maxInt; + }else{ + return (i64)r; + } +#endif +} + +/* +** Return some kind of integer value which is the best we can do +** at representing the value that *pMem describes as an integer. +** If pMem is an integer, then the value is exact. If pMem is +** a floating-point then the value returned is the integer part. +** If pMem is a string or blob, then we make an attempt to convert +** it into a integer and return that. If pMem represents an +** an SQL-NULL value, return 0. +** +** If pMem represents a string value, its encoding might be changed. +*/ +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ + int flags; + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + flags = pMem->flags; + if( flags & MEM_Int ){ + return pMem->u.i; + }else if( flags & MEM_Real ){ + return doubleToInt64(pMem->r); + }else if( flags & (MEM_Str|MEM_Blob) ){ + i64 value = 0; + assert( pMem->z || pMem->n==0 ); + testcase( pMem->z==0 ); + sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); + return value; + }else{ + return 0; + } +} + +/* +** Return the best representation of pMem that we can get into a +** double. If pMem is already a double or an integer, return its +** value. If it is a string or blob, try to convert it to a double. +** If it is a NULL, return 0.0. +*/ +SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + if( pMem->flags & MEM_Real ){ + return pMem->r; + }else if( pMem->flags & MEM_Int ){ + return (double)pMem->u.i; + }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + double val = (double)0; + sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); + return val; + }else{ + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + return (double)0; + } +} + +/* +** The MEM structure is already a MEM_Real. Try to also make it a +** MEM_Int if we can. +*/ +SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ + assert( pMem->flags & MEM_Real ); + assert( (pMem->flags & MEM_RowSet)==0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + pMem->u.i = doubleToInt64(pMem->r); + + /* Only mark the value as an integer if + ** + ** (1) the round-trip conversion real->int->real is a no-op, and + ** (2) The integer is neither the largest nor the smallest + ** possible integer (ticket #3922) + ** + ** The second and third terms in the following conditional enforces + ** the second condition under the assumption that addition overflow causes + ** values to wrap around. + */ + if( pMem->r==(double)pMem->u.i + && pMem->u.i>SMALLEST_INT64 + && pMem->u.iflags |= MEM_Int; + } +} + +/* +** Convert pMem to type integer. Invalidate any prior representations. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( (pMem->flags & MEM_RowSet)==0 ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + pMem->u.i = sqlite3VdbeIntValue(pMem); + MemSetTypeFlag(pMem, MEM_Int); + return SQLITE_OK; +} + +/* +** Convert pMem so that it is of type MEM_Real. +** Invalidate any prior representations. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + pMem->r = sqlite3VdbeRealValue(pMem); + MemSetTypeFlag(pMem, MEM_Real); + return SQLITE_OK; +} + +/* +** Convert pMem so that it has types MEM_Real or MEM_Int or both. +** Invalidate any prior representations. +** +** Every effort is made to force the conversion, even if the input +** is a string that does not look completely like a number. Convert +** as much of the string as we can and ignore the rest. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ + assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ + MemSetTypeFlag(pMem, MEM_Int); + }else{ + pMem->r = sqlite3VdbeRealValue(pMem); + MemSetTypeFlag(pMem, MEM_Real); + sqlite3VdbeIntegerAffinity(pMem); + } + } + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); + pMem->flags &= ~(MEM_Str|MEM_Blob); + return SQLITE_OK; +} + +/* +** Delete any previous value and set the value stored in *pMem to NULL. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){ + if( pMem->flags & MEM_Frame ){ + VdbeFrame *pFrame = pMem->u.pFrame; + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; + } + if( pMem->flags & MEM_RowSet ){ + sqlite3RowSetClear(pMem->u.pRowSet); + } + MemSetTypeFlag(pMem, MEM_Null); +} +SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value *p){ + sqlite3VdbeMemSetNull((Mem*)p); +} + +/* +** Delete any previous value and set the value to be a BLOB of length +** n containing all zeros. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ + sqlite3VdbeMemRelease(pMem); + pMem->flags = MEM_Blob|MEM_Zero; + pMem->n = 0; + if( n<0 ) n = 0; + pMem->u.nZero = n; + pMem->enc = SQLITE_UTF8; + +#ifdef SQLITE_OMIT_INCRBLOB + sqlite3VdbeMemGrow(pMem, n, 0); + if( pMem->z ){ + pMem->n = n; + memset(pMem->z, 0, n); + } +#endif +} + +/* +** Delete any previous value and set the value stored in *pMem to val, +** manifest type INTEGER. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ + sqlite3VdbeMemRelease(pMem); + pMem->u.i = val; + pMem->flags = MEM_Int; +} + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Delete any previous value and set the value stored in *pMem to val, +** manifest type REAL. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ + if( sqlite3IsNaN(val) ){ + sqlite3VdbeMemSetNull(pMem); + }else{ + sqlite3VdbeMemRelease(pMem); + pMem->r = val; + pMem->flags = MEM_Real; + } +} +#endif + +/* +** Delete any previous value and set the value of pMem to be an +** empty boolean index. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem *pMem){ + sqlite3 *db = pMem->db; + assert( db!=0 ); + assert( (pMem->flags & MEM_RowSet)==0 ); + sqlite3VdbeMemRelease(pMem); + pMem->zMalloc = sqlite3DbMallocRaw(db, 64); + if( db->mallocFailed ){ + pMem->flags = MEM_Null; + }else{ + assert( pMem->zMalloc ); + pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, + sqlite3DbMallocSize(db, pMem->zMalloc)); + assert( pMem->u.pRowSet!=0 ); + pMem->flags = MEM_RowSet; + } +} + +/* +** Return true if the Mem object contains a TEXT or BLOB that is +** too large - whose size exceeds SQLITE_MAX_LENGTH. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ + assert( p->db!=0 ); + if( p->flags & (MEM_Str|MEM_Blob) ){ + int n = p->n; + if( p->flags & MEM_Zero ){ + n += p->u.nZero; + } + return n>p->db->aLimit[SQLITE_LIMIT_LENGTH]; + } + return 0; +} + +#ifdef SQLITE_DEBUG +/* +** This routine prepares a memory cell for modication by breaking +** its link to a shallow copy and by marking any current shallow +** copies of this cell as invalid. +** +** This is used for testing and debugging only - to make sure shallow +** copies are not misused. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ + int i; + Mem *pX; + for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ + if( pX->pScopyFrom==pMem ){ + pX->flags |= MEM_Undefined; + pX->pScopyFrom = 0; + } + } + pMem->pScopyFrom = 0; +} +#endif /* SQLITE_DEBUG */ + +/* +** Size of struct Mem not including the Mem.zMalloc member. +*/ +#define MEMCELLSIZE offsetof(Mem,zMalloc) + +/* +** Make an shallow copy of pFrom into pTo. Prior contents of +** pTo are freed. The pFrom->z field is not duplicated. If +** pFrom->z is used, then pTo->z points to the same thing as pFrom->z +** and flags gets srcType (either MEM_Ephem or MEM_Static). +*/ +SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ + assert( (pFrom->flags & MEM_RowSet)==0 ); + VdbeMemRelease(pTo); + memcpy(pTo, pFrom, MEMCELLSIZE); + pTo->xDel = 0; + if( (pFrom->flags&MEM_Static)==0 ){ + pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); + assert( srcType==MEM_Ephem || srcType==MEM_Static ); + pTo->flags |= srcType; + } +} + +/* +** Make a full copy of pFrom into pTo. Prior contents of pTo are +** freed before the copy is made. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ + int rc = SQLITE_OK; + + assert( (pFrom->flags & MEM_RowSet)==0 ); + VdbeMemRelease(pTo); + memcpy(pTo, pFrom, MEMCELLSIZE); + pTo->flags &= ~MEM_Dyn; + pTo->xDel = 0; + + if( pTo->flags&(MEM_Str|MEM_Blob) ){ + if( 0==(pFrom->flags&MEM_Static) ){ + pTo->flags |= MEM_Ephem; + rc = sqlite3VdbeMemMakeWriteable(pTo); + } + } + + return rc; +} + +/* +** Transfer the contents of pFrom to pTo. Any existing value in pTo is +** freed. If pFrom contains ephemeral data, a copy is made. +** +** pFrom contains an SQL NULL when this routine returns. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ + assert( pFrom->db==0 || sqlite3_mutex_held(pFrom->db->mutex) ); + assert( pTo->db==0 || sqlite3_mutex_held(pTo->db->mutex) ); + assert( pFrom->db==0 || pTo->db==0 || pFrom->db==pTo->db ); + + sqlite3VdbeMemRelease(pTo); + memcpy(pTo, pFrom, sizeof(Mem)); + pFrom->flags = MEM_Null; + pFrom->xDel = 0; + pFrom->zMalloc = 0; +} + +/* +** Change the value of a Mem to be a string or a BLOB. +** +** The memory management strategy depends on the value of the xDel +** parameter. If the value passed is SQLITE_TRANSIENT, then the +** string is copied into a (possibly existing) buffer managed by the +** Mem structure. Otherwise, any existing buffer is freed and the +** pointer copied. +** +** If the string is too large (if it exceeds the SQLITE_LIMIT_LENGTH +** size limit) then no memory allocation occurs. If the string can be +** stored without allocating memory, then it is. If a memory allocation +** is required to store the string, then value of pMem is unchanged. In +** either case, SQLITE_TOOBIG is returned. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemSetStr( + Mem *pMem, /* Memory cell to set to string value */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*) /* Destructor function */ +){ + int nByte = n; /* New value for pMem->n */ + int iLimit; /* Maximum allowed string or blob size */ + u16 flags = 0; /* New value for pMem->flags */ + + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( (pMem->flags & MEM_RowSet)==0 ); + + /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ + if( !z ){ + sqlite3VdbeMemSetNull(pMem); + return SQLITE_OK; + } + + if( pMem->db ){ + iLimit = pMem->db->aLimit[SQLITE_LIMIT_LENGTH]; + }else{ + iLimit = SQLITE_MAX_LENGTH; + } + flags = (enc==0?MEM_Blob:MEM_Str); + if( nByte<0 ){ + assert( enc!=0 ); + if( enc==SQLITE_UTF8 ){ + for(nByte=0; nByte<=iLimit && z[nByte]; nByte++){} + }else{ + for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} + } + flags |= MEM_Term; + } + + /* The following block sets the new values of Mem.z and Mem.xDel. It + ** also sets a flag in local variable "flags" to indicate the memory + ** management (one of MEM_Dyn or MEM_Static). + */ + if( xDel==SQLITE_TRANSIENT ){ + int nAlloc = nByte; + if( flags&MEM_Term ){ + nAlloc += (enc==SQLITE_UTF8?1:2); + } + if( nByte>iLimit ){ + return SQLITE_TOOBIG; + } + if( sqlite3VdbeMemGrow(pMem, nAlloc, 0) ){ + return SQLITE_NOMEM; + } + memcpy(pMem->z, z, nAlloc); + }else if( xDel==SQLITE_DYNAMIC ){ + sqlite3VdbeMemRelease(pMem); + pMem->zMalloc = pMem->z = (char *)z; + pMem->xDel = 0; + }else{ + sqlite3VdbeMemRelease(pMem); + pMem->z = (char *)z; + pMem->xDel = xDel; + flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + } + + pMem->n = nByte; + pMem->flags = flags; + pMem->enc = (enc==0 ? SQLITE_UTF8 : enc); + +#ifndef SQLITE_OMIT_UTF16 + if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ + return SQLITE_NOMEM; + } +#endif + + if( nByte>iLimit ){ + return SQLITE_TOOBIG; + } + + return SQLITE_OK; +} + +/* +** Move data out of a btree key or data field and into a Mem structure. +** The data or key is taken from the entry that pCur is currently pointing +** to. offset and amt determine what portion of the data or key to retrieve. +** key is true to get the key or false to get data. The result is written +** into the pMem element. +** +** The pMem structure is assumed to be uninitialized. Any prior content +** is overwritten without being freed. +** +** If this routine fails for any reason (malloc returns NULL or unable +** to read from the disk) then the pMem is left in an inconsistent state. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( + BtCursor *pCur, /* Cursor pointing at record to retrieve. */ + u32 offset, /* Offset from the start of data to return bytes from. */ + u32 amt, /* Number of bytes to return. */ + int key, /* If true, retrieve from the btree key, not data. */ + Mem *pMem /* OUT: Return data in this Mem structure. */ +){ + char *zData; /* Data from the btree layer */ + u32 available = 0; /* Number of bytes available on the local btree page */ + int rc = SQLITE_OK; /* Return code */ + + assert( sqlite3BtreeCursorIsValid(pCur) ); + + /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() + ** that both the BtShared and database handle mutexes are held. */ + assert( (pMem->flags & MEM_RowSet)==0 ); + if( key ){ + zData = (char *)sqlite3BtreeKeyFetch(pCur, &available); + }else{ + zData = (char *)sqlite3BtreeDataFetch(pCur, &available); + } + assert( zData!=0 ); + + if( offset+amt<=available ){ + sqlite3VdbeMemRelease(pMem); + pMem->z = &zData[offset]; + pMem->flags = MEM_Blob|MEM_Ephem; + pMem->n = (int)amt; + }else if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ + if( key ){ + rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); + }else{ + rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); + } + if( rc==SQLITE_OK ){ + pMem->z[amt] = 0; + pMem->z[amt+1] = 0; + pMem->flags = MEM_Blob|MEM_Term; + pMem->n = (int)amt; + }else{ + sqlite3VdbeMemRelease(pMem); + } + } + + return rc; +} + +/* This function is only available internally, it is not part of the +** external API. It works in a similar way to sqlite3_value_text(), +** except the data returned is in the encoding specified by the second +** parameter, which must be one of SQLITE_UTF16BE, SQLITE_UTF16LE or +** SQLITE_UTF8. +** +** (2006-02-16:) The enc value can be or-ed with SQLITE_UTF16_ALIGNED. +** If that is the case, then the result must be aligned on an even byte +** boundary. +*/ +SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ + if( !pVal ) return 0; + + assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); + assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); + assert( (pVal->flags & MEM_RowSet)==0 ); + + if( pVal->flags&MEM_Null ){ + return 0; + } + assert( (MEM_Blob>>3) == MEM_Str ); + pVal->flags |= (pVal->flags & MEM_Blob)>>3; + ExpandBlob(pVal); + if( pVal->flags&MEM_Str ){ + sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); + if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ + assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); + if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ + return 0; + } + } + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ + }else{ + assert( (pVal->flags&MEM_Blob)==0 ); + sqlite3VdbeMemStringify(pVal, enc); + assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); + } + assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 + || pVal->db->mallocFailed ); + if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ + return pVal->z; + }else{ + return 0; + } +} + +/* +** Create a new sqlite3_value object. +*/ +SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *db){ + Mem *p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p ){ + p->flags = MEM_Null; + p->db = db; + } + return p; +} + +/* +** Context object passed by sqlite3Stat4ProbeSetValue() through to +** valueNew(). See comments above valueNew() for details. +*/ +struct ValueNewStat4Ctx { + Parse *pParse; + Index *pIdx; + UnpackedRecord **ppRec; + int iVal; +}; + +/* +** Allocate and return a pointer to a new sqlite3_value object. If +** the second argument to this function is NULL, the object is allocated +** by calling sqlite3ValueNew(). +** +** Otherwise, if the second argument is non-zero, then this function is +** being called indirectly by sqlite3Stat4ProbeSetValue(). If it has not +** already been allocated, allocate the UnpackedRecord structure that +** that function will return to its caller here. Then return a pointer +** an sqlite3_value within the UnpackedRecord.a[] array. +*/ +static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( p ){ + UnpackedRecord *pRec = p->ppRec[0]; + + if( pRec==0 ){ + Index *pIdx = p->pIdx; /* Index being probed */ + int nByte; /* Bytes of space to allocate */ + int i; /* Counter variable */ + int nCol = pIdx->nColumn; /* Number of index columns including rowid */ + + nByte = sizeof(Mem) * nCol + ROUND8(sizeof(UnpackedRecord)); + pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); + if( pRec ){ + pRec->pKeyInfo = sqlite3KeyInfoOfIndex(p->pParse, pIdx); + if( pRec->pKeyInfo ){ + assert( pRec->pKeyInfo->nField+pRec->pKeyInfo->nXField==nCol ); + assert( pRec->pKeyInfo->enc==ENC(db) ); + pRec->aMem = (Mem *)((u8*)pRec + ROUND8(sizeof(UnpackedRecord))); + for(i=0; iaMem[i].flags = MEM_Null; + pRec->aMem[i].db = db; + } + }else{ + sqlite3DbFree(db, pRec); + pRec = 0; + } + } + if( pRec==0 ) return 0; + p->ppRec[0] = pRec; + } + + pRec->nField = p->iVal+1; + return &pRec->aMem[p->iVal]; + } +#else + UNUSED_PARAMETER(p); +#endif /* defined(SQLITE_ENABLE_STAT3_OR_STAT4) */ + return sqlite3ValueNew(db); +} + +/* +** Extract a value from the supplied expression in the manner described +** above sqlite3ValueFromExpr(). Allocate the sqlite3_value object +** using valueNew(). +** +** If pCtx is NULL and an error occurs after the sqlite3_value object +** has been allocated, it is freed before returning. Or, if pCtx is not +** NULL, it is assumed that the caller will free any allocated object +** in all cases. +*/ +static int valueFromExpr( + sqlite3 *db, /* The database connection */ + Expr *pExpr, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal, /* Write the new value here */ + struct ValueNewStat4Ctx *pCtx /* Second argument for valueNew() */ +){ + int op; + char *zVal = 0; + sqlite3_value *pVal = 0; + int negInt = 1; + const char *zNeg = ""; + int rc = SQLITE_OK; + + if( !pExpr ){ + *ppVal = 0; + return SQLITE_OK; + } + op = pExpr->op; + if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; + + /* Handle negative integers in a single step. This is needed in the + ** case when the value is -9223372036854775808. + */ + if( op==TK_UMINUS + && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ + pExpr = pExpr->pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + + if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ + pVal = valueNew(db, pCtx); + if( pVal==0 ) goto no_mem; + if( ExprHasProperty(pExpr, EP_IntValue) ){ + sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); + }else{ + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); + if( zVal==0 ) goto no_mem; + sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + } + if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ + sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + }else{ + sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); + } + if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; + if( enc!=SQLITE_UTF8 ){ + rc = sqlite3VdbeChangeEncoding(pVal, enc); + } + }else if( op==TK_UMINUS ) { + /* This branch happens for multiple negative signs. Ex: -(-5) */ + if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) + && pVal!=0 + ){ + sqlite3VdbeMemNumerify(pVal); + if( pVal->u.i==SMALLEST_INT64 ){ + pVal->flags &= ~MEM_Int; + pVal->flags |= MEM_Real; + pVal->r = (double)SMALLEST_INT64; + }else{ + pVal->u.i = -pVal->u.i; + } + pVal->r = -pVal->r; + sqlite3ValueApplyAffinity(pVal, affinity, enc); + } + }else if( op==TK_NULL ){ + pVal = valueNew(db, pCtx); + if( pVal==0 ) goto no_mem; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + else if( op==TK_BLOB ){ + int nVal; + assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); + assert( pExpr->u.zToken[1]=='\'' ); + pVal = valueNew(db, pCtx); + if( !pVal ) goto no_mem; + zVal = &pExpr->u.zToken[2]; + nVal = sqlite3Strlen30(zVal)-1; + assert( zVal[nVal]=='\'' ); + sqlite3VdbeMemSetStr(pVal, sqlite3HexToBlob(db, zVal, nVal), nVal/2, + 0, SQLITE_DYNAMIC); + } +#endif + + *ppVal = pVal; + return rc; + +no_mem: + db->mallocFailed = 1; + sqlite3DbFree(db, zVal); + assert( *ppVal==0 ); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( pCtx==0 ) sqlite3ValueFree(pVal); +#else + assert( pCtx==0 ); sqlite3ValueFree(pVal); +#endif + return SQLITE_NOMEM; +} + +/* +** Create a new sqlite3_value object, containing the value of pExpr. +** +** This only works for very simple expressions that consist of one constant +** token (i.e. "5", "5.1", "'a string'"). If the expression can +** be converted directly into a value, then the value is allocated and +** a pointer written to *ppVal. The caller is responsible for deallocating +** the value by passing it to sqlite3ValueFree() later on. If the expression +** cannot be converted to a value, then *ppVal is set to NULL. +*/ +SQLITE_PRIVATE int sqlite3ValueFromExpr( + sqlite3 *db, /* The database connection */ + Expr *pExpr, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* Write the new value here */ +){ + return valueFromExpr(db, pExpr, enc, affinity, ppVal, 0); +} + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** The implementation of the sqlite_record() function. This function accepts +** a single argument of any type. The return value is a formatted database +** record (a blob) containing the argument value. +** +** This is used to convert the value stored in the 'sample' column of the +** sqlite_stat3 table to the record format SQLite uses internally. +*/ +static void recordFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const int file_format = 1; + int iSerial; /* Serial type */ + int nSerial; /* Bytes of space for iSerial as varint */ + int nVal; /* Bytes of space required for argv[0] */ + int nRet; + sqlite3 *db; + u8 *aRet; + + UNUSED_PARAMETER( argc ); + iSerial = sqlite3VdbeSerialType(argv[0], file_format); + nSerial = sqlite3VarintLen(iSerial); + nVal = sqlite3VdbeSerialTypeLen(iSerial); + db = sqlite3_context_db_handle(context); + + nRet = 1 + nSerial + nVal; + aRet = sqlite3DbMallocRaw(db, nRet); + if( aRet==0 ){ + sqlite3_result_error_nomem(context); + }else{ + aRet[0] = nSerial+1; + sqlite3PutVarint(&aRet[1], iSerial); + sqlite3VdbeSerialPut(&aRet[1+nSerial], argv[0], iSerial); + sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); + sqlite3DbFree(db, aRet); + } +} + +/* +** Register built-in functions used to help read ANALYZE data. +*/ +SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void){ + static SQLITE_WSD FuncDef aAnalyzeTableFuncs[] = { + FUNCTION(sqlite_record, 1, 0, 0, recordFunc), + }; + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAnalyzeTableFuncs); + for(i=0; idb; + + /* Skip over any TK_COLLATE nodes */ + pExpr = sqlite3ExprSkipCollate(pExpr); + + if( !pExpr ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + sqlite3VdbeMemSetNull((Mem*)pVal); + } + }else if( pExpr->op==TK_VARIABLE + || NEVER(pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) + ){ + Vdbe *v; + int iBindVar = pExpr->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); + if( (v = pParse->pReprepare)!=0 ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); + if( rc==SQLITE_OK ){ + sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); + } + pVal->db = pParse->db; + } + } + }else{ + rc = valueFromExpr(db, pExpr, ENC(db), affinity, &pVal, pAlloc); + } + + assert( pVal==0 || pVal->db==db ); + *ppVal = pVal; + return rc; +} + +/* +** This function is used to allocate and populate UnpackedRecord +** structures intended to be compared against sample index keys stored +** in the sqlite_stat4 table. +** +** A single call to this function attempts to populates field iVal (leftmost +** is 0 etc.) of the unpacked record with a value extracted from expression +** pExpr. Extraction of values is possible if: +** +** * (pExpr==0). In this case the value is assumed to be an SQL NULL, +** +** * The expression is a bound variable, and this is a reprepare, or +** +** * The sqlite3ValueFromExpr() function is able to extract a value +** from the expression (i.e. the expression is a literal value). +** +** If a value can be extracted, the affinity passed as the 5th argument +** is applied to it before it is copied into the UnpackedRecord. Output +** parameter *pbOk is set to true if a value is extracted, or false +** otherwise. +** +** When this function is called, *ppRec must either point to an object +** allocated by an earlier call to this function, or must be NULL. If it +** is NULL and a value can be successfully extracted, a new UnpackedRecord +** is allocated (and *ppRec set to point to it) before returning. +** +** Unless an error is encountered, SQLITE_OK is returned. It is not an +** error if a value cannot be extracted from pExpr. If an error does +** occur, an SQLite error code is returned. +*/ +SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( + Parse *pParse, /* Parse context */ + Index *pIdx, /* Index being probed */ + UnpackedRecord **ppRec, /* IN/OUT: Probe record */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + int iVal, /* Array element to populate */ + int *pbOk /* OUT: True if value was extracted */ +){ + int rc; + sqlite3_value *pVal = 0; + struct ValueNewStat4Ctx alloc; + + alloc.pParse = pParse; + alloc.pIdx = pIdx; + alloc.ppRec = ppRec; + alloc.iVal = iVal; + + rc = stat4ValueFromExpr(pParse, pExpr, affinity, &alloc, &pVal); + assert( pVal==0 || pVal->db==pParse->db ); + *pbOk = (pVal!=0); + return rc; +} + +/* +** Attempt to extract a value from expression pExpr using the methods +** as described for sqlite3Stat4ProbeSetValue() above. +** +** If successful, set *ppVal to point to a new value object and return +** SQLITE_OK. If no value can be extracted, but no other error occurs +** (e.g. OOM), return SQLITE_OK and set *ppVal to NULL. Or, if an error +** does occur, return an SQLite error code. The final value of *ppVal +** is undefined in this case. +*/ +SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr( + Parse *pParse, /* Parse context */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* OUT: New value object (or NULL) */ +){ + return stat4ValueFromExpr(pParse, pExpr, affinity, 0, ppVal); +} + +/* +** Extract the iCol-th column from the nRec-byte record in pRec. Write +** the column value into *ppVal. If *ppVal is initially NULL then a new +** sqlite3_value object is allocated. +** +** If *ppVal is initially NULL then the caller is responsible for +** ensuring that the value written into *ppVal is eventually freed. +*/ +SQLITE_PRIVATE int sqlite3Stat4Column( + sqlite3 *db, /* Database handle */ + const void *pRec, /* Pointer to buffer containing record */ + int nRec, /* Size of buffer pRec in bytes */ + int iCol, /* Column to extract */ + sqlite3_value **ppVal /* OUT: Extracted value */ +){ + u32 t; /* a column type code */ + int nHdr; /* Size of the header in the record */ + int iHdr; /* Next unread header byte */ + int iField; /* Next unread data byte */ + int szField; /* Size of the current data field */ + int i; /* Column index */ + u8 *a = (u8*)pRec; /* Typecast byte array */ + Mem *pMem = *ppVal; /* Write result into this Mem object */ + + assert( iCol>0 ); + iHdr = getVarint32(a, nHdr); + if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + iField = nHdr; + for(i=0; i<=iCol; i++){ + iHdr += getVarint32(&a[iHdr], t); + testcase( iHdr==nHdr ); + testcase( iHdr==nHdr+1 ); + if( iHdr>nHdr ) return SQLITE_CORRUPT_BKPT; + szField = sqlite3VdbeSerialTypeLen(t); + iField += szField; + } + testcase( iField==nRec ); + testcase( iField==nRec+1 ); + if( iField>nRec ) return SQLITE_CORRUPT_BKPT; + if( pMem==0 ){ + pMem = *ppVal = sqlite3ValueNew(db); + if( pMem==0 ) return SQLITE_NOMEM; + } + sqlite3VdbeSerialGet(&a[iField-szField], t, pMem); + pMem->enc = ENC(db); + return SQLITE_OK; +} + +/* +** Unless it is NULL, the argument must be an UnpackedRecord object returned +** by an earlier call to sqlite3Stat4ProbeSetValue(). This call deletes +** the object. +*/ +SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ + if( pRec ){ + int i; + int nCol = pRec->pKeyInfo->nField+pRec->pKeyInfo->nXField; + Mem *aMem = pRec->aMem; + sqlite3 *db = aMem[0].db; + for(i=0; ipKeyInfo); + sqlite3DbFree(db, pRec); + } +} +#endif /* ifdef SQLITE_ENABLE_STAT4 */ + +/* +** Change the string value of an sqlite3_value object +*/ +SQLITE_PRIVATE void sqlite3ValueSetStr( + sqlite3_value *v, /* Value to be set */ + int n, /* Length of string z */ + const void *z, /* Text of the new string */ + u8 enc, /* Encoding to use */ + void (*xDel)(void*) /* Destructor for the string */ +){ + if( v ) sqlite3VdbeMemSetStr((Mem *)v, z, n, enc, xDel); +} + +/* +** Free an sqlite3_value object +*/ +SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value *v){ + if( !v ) return; + sqlite3VdbeMemRelease((Mem *)v); + sqlite3DbFree(((Mem*)v)->db, v); +} + +/* +** Return the number of bytes in the sqlite3_value object assuming +** that it uses the encoding "enc" +*/ +SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ + Mem *p = (Mem*)pVal; + if( (p->flags & MEM_Blob)!=0 || sqlite3ValueText(pVal, enc) ){ + if( p->flags & MEM_Zero ){ + return p->n + p->u.nZero; + }else{ + return p->n; + } + } + return 0; +} + +/************** End of vdbemem.c *********************************************/ +/************** Begin file vdbeaux.c *****************************************/ +/* +** 2003 September 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used for creating, destroying, and populating +** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior +** to version 2.8.7, all this code was combined into the vdbe.c source file. +** But that file was getting too big so this subroutines were split out. +*/ + +/* +** Create a new virtual database engine. +*/ +SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ + sqlite3 *db = pParse->db; + Vdbe *p; + p = sqlite3DbMallocZero(db, sizeof(Vdbe) ); + if( p==0 ) return 0; + p->db = db; + if( db->pVdbe ){ + db->pVdbe->pPrev = p; + } + p->pNext = db->pVdbe; + p->pPrev = 0; + db->pVdbe = p; + p->magic = VDBE_MAGIC_INIT; + p->pParse = pParse; + assert( pParse->aLabel==0 ); + assert( pParse->nLabel==0 ); + assert( pParse->nOpAlloc==0 ); + return p; +} + +/* +** Remember the SQL string for a prepared statement. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ + assert( isPrepareV2==1 || isPrepareV2==0 ); + if( p==0 ) return; +#if defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_ENABLE_SQLLOG) + if( !isPrepareV2 ) return; +#endif + assert( p->zSql==0 ); + p->zSql = sqlite3DbStrNDup(p->db, z, n); + p->isPrepareV2 = (u8)isPrepareV2; +} + +/* +** Return the SQL associated with a prepared statement +*/ +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + return (p && p->isPrepareV2) ? p->zSql : 0; +} + +/* +** Swap all content between two VDBE structures. +*/ +SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ + Vdbe tmp, *pTmp; + char *zTmp; + tmp = *pA; + *pA = *pB; + *pB = tmp; + pTmp = pA->pNext; + pA->pNext = pB->pNext; + pB->pNext = pTmp; + pTmp = pA->pPrev; + pA->pPrev = pB->pPrev; + pB->pPrev = pTmp; + zTmp = pA->zSql; + pA->zSql = pB->zSql; + pB->zSql = zTmp; + pB->isPrepareV2 = pA->isPrepareV2; +} + +/* +** Resize the Vdbe.aOp array so that it is at least nOp elements larger +** than its current size. nOp is guaranteed to be less than or equal +** to 1024/sizeof(Op). +** +** If an out-of-memory error occurs while resizing the array, return +** SQLITE_NOMEM. In this case Vdbe.aOp and Parse.nOpAlloc remain +** unchanged (this is so that any opcodes already allocated can be +** correctly deallocated along with the rest of the Vdbe). +*/ +static int growOpArray(Vdbe *v, int nOp){ + VdbeOp *pNew; + Parse *p = v->pParse; + + /* The SQLITE_TEST_REALLOC_STRESS compile-time option is designed to force + ** more frequent reallocs and hence provide more opportunities for + ** simulated OOM faults. SQLITE_TEST_REALLOC_STRESS is generally used + ** during testing only. With SQLITE_TEST_REALLOC_STRESS grow the op array + ** by the minimum* amount required until the size reaches 512. Normal + ** operation (without SQLITE_TEST_REALLOC_STRESS) is to double the current + ** size of the op array or add 1KB of space, whichever is smaller. */ +#ifdef SQLITE_TEST_REALLOC_STRESS + int nNew = (p->nOpAlloc>=512 ? p->nOpAlloc*2 : p->nOpAlloc+nOp); +#else + int nNew = (p->nOpAlloc ? p->nOpAlloc*2 : (int)(1024/sizeof(Op))); + UNUSED_PARAMETER(nOp); +#endif + + assert( nOp<=(1024/sizeof(Op)) ); + assert( nNew>=(p->nOpAlloc+nOp) ); + pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); + if( pNew ){ + p->nOpAlloc = sqlite3DbMallocSize(p->db, pNew)/sizeof(Op); + v->aOp = pNew; + } + return (pNew ? SQLITE_OK : SQLITE_NOMEM); +} + +#ifdef SQLITE_DEBUG +/* This routine is just a convenient place to set a breakpoint that will +** fire after each opcode is inserted and displayed using +** "PRAGMA vdbe_addoptrace=on". +*/ +static void test_addop_breakpoint(void){ + static int n = 0; + n++; +} +#endif + +/* +** Add a new instruction to the list of instructions current in the +** VDBE. Return the address of the new instruction. +** +** Parameters: +** +** p Pointer to the VDBE +** +** op The opcode for this instruction +** +** p1, p2, p3 Operands +** +** Use the sqlite3VdbeResolveLabel() function to fix an address and +** the sqlite3VdbeChangeP4() function to change the value of the P4 +** operand. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ + int i; + VdbeOp *pOp; + + i = p->nOp; + assert( p->magic==VDBE_MAGIC_INIT ); + assert( op>0 && op<0xff ); + if( p->pParse->nOpAlloc<=i ){ + if( growOpArray(p, 1) ){ + return 1; + } + } + p->nOp++; + pOp = &p->aOp[i]; + pOp->opcode = (u8)op; + pOp->p5 = 0; + pOp->p1 = p1; + pOp->p2 = p2; + pOp->p3 = p3; + pOp->p4.p = 0; + pOp->p4type = P4_NOTUSED; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + pOp->zComment = 0; +#endif +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + int jj, kk; + Parse *pParse = p->pParse; + for(jj=kk=0; jjaColCache + jj; + if( x->iLevel>pParse->iCacheLevel || x->iReg==0 ) continue; + printf(" r[%d]={%d:%d}", x->iReg, x->iTable, x->iColumn); + kk++; + } + if( kk ) printf("\n"); + sqlite3VdbePrintOp(0, i, &p->aOp[i]); + test_addop_breakpoint(); + } +#endif +#ifdef VDBE_PROFILE + pOp->cycles = 0; + pOp->cnt = 0; +#endif +#ifdef SQLITE_VDBE_COVERAGE + pOp->iSrcLine = 0; +#endif + return i; +} +SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe *p, int op){ + return sqlite3VdbeAddOp3(p, op, 0, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){ + return sqlite3VdbeAddOp3(p, op, p1, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ + return sqlite3VdbeAddOp3(p, op, p1, p2, 0); +} + + +/* +** Add an opcode that includes the p4 value as a pointer. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp4( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + const char *zP4, /* The P4 operand */ + int p4type /* P4 operand type */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + sqlite3VdbeChangeP4(p, addr, zP4, p4type); + return addr; +} + +/* +** Add an OP_ParseSchema opcode. This routine is broken out from +** sqlite3VdbeAddOp4() since it needs to also needs to mark all btrees +** as having been used. +** +** The zWhere string must have been obtained from sqlite3_malloc(). +** This routine will take ownership of the allocated memory. +*/ +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ + int j; + int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); + sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); +} + +/* +** Add an opcode that includes the p4 value as an integer. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32); + return addr; +} + +/* +** Create a new symbolic label for an instruction that has yet to be +** coded. The symbolic label is really just a negative number. The +** label can be used as the P2 value of an operation. Later, when +** the label is resolved to a specific address, the VDBE will scan +** through its operation list and change all values of P2 which match +** the label into the resolved address. +** +** The VDBE knows that a P2 value is a label because labels are +** always negative and P2 values are suppose to be non-negative. +** Hence, a negative P2 value is a label that has yet to be resolved. +** +** Zero is returned if a malloc() fails. +*/ +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ + Parse *p = v->pParse; + int i = p->nLabel++; + assert( v->magic==VDBE_MAGIC_INIT ); + if( (i & (i-1))==0 ){ + p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, + (i*2+1)*sizeof(p->aLabel[0])); + } + if( p->aLabel ){ + p->aLabel[i] = -1; + } + return -1-i; +} + +/* +** Resolve label "x" to be the address of the next instruction to +** be inserted. The parameter "x" must have been obtained from +** a prior call to sqlite3VdbeMakeLabel(). +*/ +SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ + Parse *p = v->pParse; + int j = -1-x; + assert( v->magic==VDBE_MAGIC_INIT ); + assert( jnLabel ); + if( ALWAYS(j>=0) && p->aLabel ){ + p->aLabel[j] = v->nOp; + } + p->iFixedOp = v->nOp - 1; +} + +/* +** Mark the VDBE as one that can only be run one time. +*/ +SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe *p){ + p->runOnlyOnce = 1; +} + +#ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ + +/* +** The following type and function are used to iterate through all opcodes +** in a Vdbe main program and each of the sub-programs (triggers) it may +** invoke directly or indirectly. It should be used as follows: +** +** Op *pOp; +** VdbeOpIter sIter; +** +** memset(&sIter, 0, sizeof(sIter)); +** sIter.v = v; // v is of type Vdbe* +** while( (pOp = opIterNext(&sIter)) ){ +** // Do something with pOp +** } +** sqlite3DbFree(v->db, sIter.apSub); +** +*/ +typedef struct VdbeOpIter VdbeOpIter; +struct VdbeOpIter { + Vdbe *v; /* Vdbe to iterate through the opcodes of */ + SubProgram **apSub; /* Array of subprograms */ + int nSub; /* Number of entries in apSub */ + int iAddr; /* Address of next instruction to return */ + int iSub; /* 0 = main program, 1 = first sub-program etc. */ +}; +static Op *opIterNext(VdbeOpIter *p){ + Vdbe *v = p->v; + Op *pRet = 0; + Op *aOp; + int nOp; + + if( p->iSub<=p->nSub ){ + + if( p->iSub==0 ){ + aOp = v->aOp; + nOp = v->nOp; + }else{ + aOp = p->apSub[p->iSub-1]->aOp; + nOp = p->apSub[p->iSub-1]->nOp; + } + assert( p->iAddriAddr]; + p->iAddr++; + if( p->iAddr==nOp ){ + p->iSub++; + p->iAddr = 0; + } + + if( pRet->p4type==P4_SUBPROGRAM ){ + int nByte = (p->nSub+1)*sizeof(SubProgram*); + int j; + for(j=0; jnSub; j++){ + if( p->apSub[j]==pRet->p4.pProgram ) break; + } + if( j==p->nSub ){ + p->apSub = sqlite3DbReallocOrFree(v->db, p->apSub, nByte); + if( !p->apSub ){ + pRet = 0; + }else{ + p->apSub[p->nSub++] = pRet->p4.pProgram; + } + } + } + } + + return pRet; +} + +/* +** Check if the program stored in the VM associated with pParse may +** throw an ABORT exception (causing the statement, but not entire transaction +** to be rolled back). This condition is true if the main program or any +** sub-programs contains any of the following: +** +** * OP_Halt with P1=SQLITE_CONSTRAINT and P2=OE_Abort. +** * OP_HaltIfNull with P1=SQLITE_CONSTRAINT and P2=OE_Abort. +** * OP_Destroy +** * OP_VUpdate +** * OP_VRename +** * OP_FkCounter with P2==0 (immediate foreign key constraint) +** +** Then check that the value of Parse.mayAbort is true if an +** ABORT may be thrown, or false otherwise. Return true if it does +** match, or false otherwise. This function is intended to be used as +** part of an assert statement in the compiler. Similar to: +** +** assert( sqlite3VdbeAssertMayAbort(pParse->pVdbe, pParse->mayAbort) ); +*/ +SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ + int hasAbort = 0; + Op *pOp; + VdbeOpIter sIter; + memset(&sIter, 0, sizeof(sIter)); + sIter.v = v; + + while( (pOp = opIterNext(&sIter))!=0 ){ + int opcode = pOp->opcode; + if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename +#ifndef SQLITE_OMIT_FOREIGN_KEY + || (opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1) +#endif + || ((opcode==OP_Halt || opcode==OP_HaltIfNull) + && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) + ){ + hasAbort = 1; + break; + } + } + sqlite3DbFree(v->db, sIter.apSub); + + /* Return true if hasAbort==mayAbort. Or if a malloc failure occurred. + ** If malloc failed, then the while() loop above may not have iterated + ** through all opcodes and hasAbort may be set incorrectly. Return + ** true for this case to prevent the assert() in the callers frame + ** from failing. */ + return ( v->db->mallocFailed || hasAbort==mayAbort ); +} +#endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ + +/* +** Loop through the program looking for P2 values that are negative +** on jump instructions. Each such value is a label. Resolve the +** label by setting the P2 value to its correct non-zero value. +** +** This routine is called once after all opcodes have been inserted. +** +** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument +** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by +** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. +** +** The Op.opflags field is set on all opcodes. +*/ +static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ + int i; + int nMaxArgs = *pMaxFuncArgs; + Op *pOp; + Parse *pParse = p->pParse; + int *aLabel = pParse->aLabel; + p->readOnly = 1; + p->bIsReader = 0; + for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){ + u8 opcode = pOp->opcode; + + /* NOTE: Be sure to update mkopcodeh.awk when adding or removing + ** cases from this switch! */ + switch( opcode ){ + case OP_Function: + case OP_AggStep: { + if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5; + break; + } + case OP_Transaction: { + if( pOp->p2!=0 ) p->readOnly = 0; + /* fall thru */ + } + case OP_AutoCommit: + case OP_Savepoint: { + p->bIsReader = 1; + break; + } +#ifndef SQLITE_OMIT_WAL + case OP_Checkpoint: +#endif + case OP_Vacuum: + case OP_JournalMode: { + p->readOnly = 0; + p->bIsReader = 1; + break; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + case OP_VUpdate: { + if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; + break; + } + case OP_VFilter: { + int n; + assert( p->nOp - i >= 3 ); + assert( pOp[-1].opcode==OP_Integer ); + n = pOp[-1].p1; + if( n>nMaxArgs ) nMaxArgs = n; + break; + } +#endif + case OP_Next: + case OP_NextIfOpen: + case OP_SorterNext: { + pOp->p4.xAdvance = sqlite3BtreeNext; + pOp->p4type = P4_ADVANCE; + break; + } + case OP_Prev: + case OP_PrevIfOpen: { + pOp->p4.xAdvance = sqlite3BtreePrevious; + pOp->p4type = P4_ADVANCE; + break; + } + } + + pOp->opflags = sqlite3OpcodeProperty[opcode]; + if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){ + assert( -1-pOp->p2nLabel ); + pOp->p2 = aLabel[-1-pOp->p2]; + } + } + sqlite3DbFree(p->db, pParse->aLabel); + pParse->aLabel = 0; + pParse->nLabel = 0; + *pMaxFuncArgs = nMaxArgs; + assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) ); +} + +/* +** Return the address of the next instruction to be inserted. +*/ +SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ + assert( p->magic==VDBE_MAGIC_INIT ); + return p->nOp; +} + +/* +** This function returns a pointer to the array of opcodes associated with +** the Vdbe passed as the first argument. It is the callers responsibility +** to arrange for the returned array to be eventually freed using the +** vdbeFreeOpArray() function. +** +** Before returning, *pnOp is set to the number of entries in the returned +** array. Also, *pnMaxArg is set to the larger of its current value and +** the number of entries in the Vdbe.apArg[] array required to execute the +** returned program. +*/ +SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ + VdbeOp *aOp = p->aOp; + assert( aOp && !p->db->mallocFailed ); + + /* Check that sqlite3VdbeUsesBtree() was not called on this VM */ + assert( DbMaskAllZero(p->btreeMask) ); + + resolveP2Values(p, pnMaxArg); + *pnOp = p->nOp; + p->aOp = 0; + return aOp; +} + +/* +** Add a whole list of operations to the operation stack. Return the +** address of the first operation added. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ + int addr; + assert( p->magic==VDBE_MAGIC_INIT ); + if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ + return 0; + } + addr = p->nOp; + if( ALWAYS(nOp>0) ){ + int i; + VdbeOpList const *pIn = aOp; + for(i=0; ip2; + VdbeOp *pOut = &p->aOp[i+addr]; + pOut->opcode = pIn->opcode; + pOut->p1 = pIn->p1; + if( p2<0 ){ + assert( sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP ); + pOut->p2 = addr + ADDR(p2); + }else{ + pOut->p2 = p2; + } + pOut->p3 = pIn->p3; + pOut->p4type = P4_NOTUSED; + pOut->p4.p = 0; + pOut->p5 = 0; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + pOut->zComment = 0; +#endif +#ifdef SQLITE_VDBE_COVERAGE + pOut->iSrcLine = iLineno+i; +#else + (void)iLineno; +#endif +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); + } +#endif + } + p->nOp += nOp; + } + return addr; +} + +/* +** Change the value of the P1 operand for a specific instruction. +** This routine is useful when a large program is loaded from a +** static array using sqlite3VdbeAddOpList but we want to make a +** few minor changes to the program. +*/ +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ + assert( p!=0 ); + if( ((u32)p->nOp)>addr ){ + p->aOp[addr].p1 = val; + } +} + +/* +** Change the value of the P2 operand for a specific instruction. +** This routine is useful for setting a jump destination. +*/ +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ + assert( p!=0 ); + if( ((u32)p->nOp)>addr ){ + p->aOp[addr].p2 = val; + } +} + +/* +** Change the value of the P3 operand for a specific instruction. +*/ +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ + assert( p!=0 ); + if( ((u32)p->nOp)>addr ){ + p->aOp[addr].p3 = val; + } +} + +/* +** Change the value of the P5 operand for the most recently +** added operation. +*/ +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ + assert( p!=0 ); + if( p->aOp ){ + assert( p->nOp>0 ); + p->aOp[p->nOp-1].p5 = val; + } +} + +/* +** Change the P2 operand of instruction addr so that it points to +** the address of the next instruction to be coded. +*/ +SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ + sqlite3VdbeChangeP2(p, addr, p->nOp); + p->pParse->iFixedOp = p->nOp - 1; +} + + +/* +** If the input FuncDef structure is ephemeral, then free it. If +** the FuncDef is not ephermal, then do nothing. +*/ +static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ + if( ALWAYS(pDef) && (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ + sqlite3DbFree(db, pDef); + } +} + +static void vdbeFreeOpArray(sqlite3 *, Op *, int); + +/* +** Delete a P4 value if necessary. +*/ +static void freeP4(sqlite3 *db, int p4type, void *p4){ + if( p4 ){ + assert( db ); + switch( p4type ){ + case P4_REAL: + case P4_INT64: + case P4_DYNAMIC: + case P4_INTARRAY: { + sqlite3DbFree(db, p4); + break; + } + case P4_KEYINFO: { + if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4); + break; + } + case P4_MPRINTF: { + if( db->pnBytesFreed==0 ) sqlite3_free(p4); + break; + } + case P4_FUNCDEF: { + freeEphemeralFunction(db, (FuncDef*)p4); + break; + } + case P4_MEM: { + if( db->pnBytesFreed==0 ){ + sqlite3ValueFree((sqlite3_value*)p4); + }else{ + Mem *p = (Mem*)p4; + sqlite3DbFree(db, p->zMalloc); + sqlite3DbFree(db, p); + } + break; + } + case P4_VTAB : { + if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); + break; + } + } + } +} + +/* +** Free the space allocated for aOp and any p4 values allocated for the +** opcodes contained within. If aOp is not NULL it is assumed to contain +** nOp entries. +*/ +static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ + if( aOp ){ + Op *pOp; + for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ + freeP4(db, pOp->p4type, pOp->p4.p); +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + sqlite3DbFree(db, pOp->zComment); +#endif + } + } + sqlite3DbFree(db, aOp); +} + +/* +** Link the SubProgram object passed as the second argument into the linked +** list at Vdbe.pSubProgram. This list is used to delete all sub-program +** objects when the VM is no longer required. +*/ +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ + p->pNext = pVdbe->pProgram; + pVdbe->pProgram = p; +} + +/* +** Change the opcode at addr into OP_Noop +*/ +SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ + if( addrnOp ){ + VdbeOp *pOp = &p->aOp[addr]; + sqlite3 *db = p->db; + freeP4(db, pOp->p4type, pOp->p4.p); + memset(pOp, 0, sizeof(pOp[0])); + pOp->opcode = OP_Noop; + if( addr==p->nOp-1 ) p->nOp--; + } +} + +/* +** Remove the last opcode inserted +*/ +SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ + if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ + sqlite3VdbeChangeToNoop(p, p->nOp-1); + return 1; + }else{ + return 0; + } +} + +/* +** Change the value of the P4 operand for a specific instruction. +** This routine is useful when a large program is loaded from a +** static array using sqlite3VdbeAddOpList but we want to make a +** few minor changes to the program. +** +** If n>=0 then the P4 operand is dynamic, meaning that a copy of +** the string is made into memory obtained from sqlite3_malloc(). +** A value of n==0 means copy bytes of zP4 up to and including the +** first null byte. If n>0 then copy n+1 bytes of zP4. +** +** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points +** to a string or structure that is guaranteed to exist for the lifetime of +** the Vdbe. In these cases we can just copy the pointer. +** +** If addr<0 then change P4 on the most recently inserted instruction. +*/ +SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ + Op *pOp; + sqlite3 *db; + assert( p!=0 ); + db = p->db; + assert( p->magic==VDBE_MAGIC_INIT ); + if( p->aOp==0 || db->mallocFailed ){ + if( n!=P4_VTAB ){ + freeP4(db, n, (void*)*(char**)&zP4); + } + return; + } + assert( p->nOp>0 ); + assert( addrnOp ); + if( addr<0 ){ + addr = p->nOp - 1; + } + pOp = &p->aOp[addr]; + assert( pOp->p4type==P4_NOTUSED + || pOp->p4type==P4_INT32 + || pOp->p4type==P4_KEYINFO ); + freeP4(db, pOp->p4type, pOp->p4.p); + pOp->p4.p = 0; + if( n==P4_INT32 ){ + /* Note: this cast is safe, because the origin data point was an int + ** that was cast to a (const char *). */ + pOp->p4.i = SQLITE_PTR_TO_INT(zP4); + pOp->p4type = P4_INT32; + }else if( zP4==0 ){ + pOp->p4.p = 0; + pOp->p4type = P4_NOTUSED; + }else if( n==P4_KEYINFO ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = P4_KEYINFO; + }else if( n==P4_VTAB ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = P4_VTAB; + sqlite3VtabLock((VTable *)zP4); + assert( ((VTable *)zP4)->db==p->db ); + }else if( n<0 ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = (signed char)n; + }else{ + if( n==0 ) n = sqlite3Strlen30(zP4); + pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); + pOp->p4type = P4_DYNAMIC; + } +} + +/* +** Set the P4 on the most recently added opcode to the KeyInfo for the +** index given. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse *pParse, Index *pIdx){ + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + assert( pIdx!=0 ); + sqlite3VdbeChangeP4(v, -1, (char*)sqlite3KeyInfoOfIndex(pParse, pIdx), + P4_KEYINFO); +} + +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +/* +** Change the comment on the most recently coded instruction. Or +** insert a No-op and add the comment to that new instruction. This +** makes the code easier to read during debugging. None of this happens +** in a production build. +*/ +static void vdbeVComment(Vdbe *p, const char *zFormat, va_list ap){ + assert( p->nOp>0 || p->aOp==0 ); + assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); + if( p->nOp ){ + assert( p->aOp ); + sqlite3DbFree(p->db, p->aOp[p->nOp-1].zComment); + p->aOp[p->nOp-1].zComment = sqlite3VMPrintf(p->db, zFormat, ap); + } +} +SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe *p, const char *zFormat, ...){ + va_list ap; + if( p ){ + va_start(ap, zFormat); + vdbeVComment(p, zFormat, ap); + va_end(ap); + } +} +SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ + va_list ap; + if( p ){ + sqlite3VdbeAddOp0(p, OP_Noop); + va_start(ap, zFormat); + vdbeVComment(p, zFormat, ap); + va_end(ap); + } +} +#endif /* NDEBUG */ + +#ifdef SQLITE_VDBE_COVERAGE +/* +** Set the value if the iSrcLine field for the previously coded instruction. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ + sqlite3VdbeGetOp(v,-1)->iSrcLine = iLine; +} +#endif /* SQLITE_VDBE_COVERAGE */ + +/* +** Return the opcode for a given address. If the address is -1, then +** return the most recently inserted opcode. +** +** If a memory allocation error has occurred prior to the calling of this +** routine, then a pointer to a dummy VdbeOp will be returned. That opcode +** is readable but not writable, though it is cast to a writable value. +** The return of a dummy opcode allows the call to continue functioning +** after a OOM fault without having to check to see if the return from +** this routine is a valid pointer. But because the dummy.opcode is 0, +** dummy will never be written to. This is verified by code inspection and +** by running with Valgrind. +*/ +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ + /* C89 specifies that the constant "dummy" will be initialized to all + ** zeros, which is correct. MSVC generates a warning, nevertheless. */ + static VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ + assert( p->magic==VDBE_MAGIC_INIT ); + if( addr<0 ){ + addr = p->nOp - 1; + } + assert( (addr>=0 && addrnOp) || p->db->mallocFailed ); + if( p->db->mallocFailed ){ + return (VdbeOp*)&dummy; + }else{ + return &p->aOp[addr]; + } +} + +#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) +/* +** Return an integer value for one of the parameters to the opcode pOp +** determined by character c. +*/ +static int translateP(char c, const Op *pOp){ + if( c=='1' ) return pOp->p1; + if( c=='2' ) return pOp->p2; + if( c=='3' ) return pOp->p3; + if( c=='4' ) return pOp->p4.i; + return pOp->p5; +} + +/* +** Compute a string for the "comment" field of a VDBE opcode listing. +** +** The Synopsis: field in comments in the vdbe.c source file gets converted +** to an extra string that is appended to the sqlite3OpcodeName(). In the +** absence of other comments, this synopsis becomes the comment on the opcode. +** Some translation occurs: +** +** "PX" -> "r[X]" +** "PX@PY" -> "r[X..X+Y-1]" or "r[x]" if y is 0 or 1 +** "PX@PY+1" -> "r[X..X+Y]" or "r[x]" if y is 0 +** "PY..PY" -> "r[X..Y]" or "r[x]" if y<=x +*/ +static int displayComment( + const Op *pOp, /* The opcode to be commented */ + const char *zP4, /* Previously obtained value for P4 */ + char *zTemp, /* Write result here */ + int nTemp /* Space available in zTemp[] */ +){ + const char *zOpName; + const char *zSynopsis; + int nOpName; + int ii, jj; + zOpName = sqlite3OpcodeName(pOp->opcode); + nOpName = sqlite3Strlen30(zOpName); + if( zOpName[nOpName+1] ){ + int seenCom = 0; + char c; + zSynopsis = zOpName += nOpName + 1; + for(ii=jj=0; jjzComment); + seenCom = 1; + }else{ + int v1 = translateP(c, pOp); + int v2; + sqlite3_snprintf(nTemp-jj, zTemp+jj, "%d", v1); + if( strncmp(zSynopsis+ii+1, "@P", 2)==0 ){ + ii += 3; + jj += sqlite3Strlen30(zTemp+jj); + v2 = translateP(zSynopsis[ii], pOp); + if( strncmp(zSynopsis+ii+1,"+1",2)==0 ){ + ii += 2; + v2++; + } + if( v2>1 ){ + sqlite3_snprintf(nTemp-jj, zTemp+jj, "..%d", v1+v2-1); + } + }else if( strncmp(zSynopsis+ii+1, "..P3", 4)==0 && pOp->p3==0 ){ + ii += 4; + } + } + jj += sqlite3Strlen30(zTemp+jj); + }else{ + zTemp[jj++] = c; + } + } + if( !seenCom && jjzComment ){ + sqlite3_snprintf(nTemp-jj, zTemp+jj, "; %s", pOp->zComment); + jj += sqlite3Strlen30(zTemp+jj); + } + if( jjzComment ){ + sqlite3_snprintf(nTemp, zTemp, "%s", pOp->zComment); + jj = sqlite3Strlen30(zTemp); + }else{ + zTemp[0] = 0; + jj = 0; + } + return jj; +} +#endif /* SQLITE_DEBUG */ + + +#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ + || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) +/* +** Compute a string that describes the P4 parameter for an opcode. +** Use zTemp for any required temporary buffer space. +*/ +static char *displayP4(Op *pOp, char *zTemp, int nTemp){ + char *zP4 = zTemp; + assert( nTemp>=20 ); + switch( pOp->p4type ){ + case P4_KEYINFO: { + int i, j; + KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; + assert( pKeyInfo->aSortOrder!=0 ); + sqlite3_snprintf(nTemp, zTemp, "k(%d", pKeyInfo->nField); + i = sqlite3Strlen30(zTemp); + for(j=0; jnField; j++){ + CollSeq *pColl = pKeyInfo->aColl[j]; + const char *zColl = pColl ? pColl->zName : "nil"; + int n = sqlite3Strlen30(zColl); + if( n==6 && memcmp(zColl,"BINARY",6)==0 ){ + zColl = "B"; + n = 1; + } + if( i+n>nTemp-6 ){ + memcpy(&zTemp[i],",...",4); + break; + } + zTemp[i++] = ','; + if( pKeyInfo->aSortOrder[j] ){ + zTemp[i++] = '-'; + } + memcpy(&zTemp[i], zColl, n+1); + i += n; + } + zTemp[i++] = ')'; + zTemp[i] = 0; + assert( ip4.pColl; + sqlite3_snprintf(nTemp, zTemp, "(%.20s)", pColl->zName); + break; + } + case P4_FUNCDEF: { + FuncDef *pDef = pOp->p4.pFunc; + sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); + break; + } + case P4_INT64: { + sqlite3_snprintf(nTemp, zTemp, "%lld", *pOp->p4.pI64); + break; + } + case P4_INT32: { + sqlite3_snprintf(nTemp, zTemp, "%d", pOp->p4.i); + break; + } + case P4_REAL: { + sqlite3_snprintf(nTemp, zTemp, "%.16g", *pOp->p4.pReal); + break; + } + case P4_MEM: { + Mem *pMem = pOp->p4.pMem; + if( pMem->flags & MEM_Str ){ + zP4 = pMem->z; + }else if( pMem->flags & MEM_Int ){ + sqlite3_snprintf(nTemp, zTemp, "%lld", pMem->u.i); + }else if( pMem->flags & MEM_Real ){ + sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->r); + }else if( pMem->flags & MEM_Null ){ + sqlite3_snprintf(nTemp, zTemp, "NULL"); + }else{ + assert( pMem->flags & MEM_Blob ); + zP4 = "(blob)"; + } + break; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + case P4_VTAB: { + sqlite3_vtab *pVtab = pOp->p4.pVtab->pVtab; + sqlite3_snprintf(nTemp, zTemp, "vtab:%p:%p", pVtab, pVtab->pModule); + break; + } +#endif + case P4_INTARRAY: { + sqlite3_snprintf(nTemp, zTemp, "intarray"); + break; + } + case P4_SUBPROGRAM: { + sqlite3_snprintf(nTemp, zTemp, "program"); + break; + } + case P4_ADVANCE: { + zTemp[0] = 0; + break; + } + default: { + zP4 = pOp->p4.z; + if( zP4==0 ){ + zP4 = zTemp; + zTemp[0] = 0; + } + } + } + assert( zP4!=0 ); + return zP4; +} +#endif + +/* +** Declare to the Vdbe that the BTree object at db->aDb[i] is used. +** +** The prepared statements need to know in advance the complete set of +** attached databases that will be use. A mask of these databases +** is maintained in p->btreeMask. The p->lockMask value is the subset of +** p->btreeMask of databases that will require a lock. +*/ +SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ + assert( i>=0 && idb->nDb && i<(int)sizeof(yDbMask)*8 ); + assert( i<(int)sizeof(p->btreeMask)*8 ); + DbMaskSet(p->btreeMask, i); + if( i!=1 && sqlite3BtreeSharable(p->db->aDb[i].pBt) ){ + DbMaskSet(p->lockMask, i); + } +} + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +/* +** If SQLite is compiled to support shared-cache mode and to be threadsafe, +** this routine obtains the mutex associated with each BtShared structure +** that may be accessed by the VM passed as an argument. In doing so it also +** sets the BtShared.db member of each of the BtShared structures, ensuring +** that the correct busy-handler callback is invoked if required. +** +** If SQLite is not threadsafe but does support shared-cache mode, then +** sqlite3BtreeEnter() is invoked to set the BtShared.db variables +** of all of BtShared structures accessible via the database handle +** associated with the VM. +** +** If SQLite is not threadsafe and does not support shared-cache mode, this +** function is a no-op. +** +** The p->btreeMask field is a bitmask of all btrees that the prepared +** statement p will ever use. Let N be the number of bits in p->btreeMask +** corresponding to btrees that use shared cache. Then the runtime of +** this routine is N*N. But as N is rarely more than 1, this should not +** be a problem. +*/ +SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe *p){ + int i; + sqlite3 *db; + Db *aDb; + int nDb; + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ + db = p->db; + aDb = db->aDb; + nDb = db->nDb; + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ + sqlite3BtreeEnter(aDb[i].pBt); + } + } +} +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +/* +** Unlock all of the btrees previously locked by a call to sqlite3VdbeEnter(). +*/ +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ + int i; + sqlite3 *db; + Db *aDb; + int nDb; + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ + db = p->db; + aDb = db->aDb; + nDb = db->nDb; + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ + sqlite3BtreeLeave(aDb[i].pBt); + } + } +} +#endif + +#if defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) +/* +** Print a single opcode. This routine is used for debugging only. +*/ +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ + char *zP4; + char zPtr[50]; + char zCom[100]; + static const char *zFormat1 = "%4d %-13s %4d %4d %4d %-13s %.2X %s\n"; + if( pOut==0 ) pOut = stdout; + zP4 = displayP4(pOp, zPtr, sizeof(zPtr)); +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + displayComment(pOp, zP4, zCom, sizeof(zCom)); +#else + zCom[0] = 0; +#endif + /* NB: The sqlite3OpcodeName() function is implemented by code created + ** by the mkopcodeh.awk and mkopcodec.awk scripts which extract the + ** information from the vdbe.c source text */ + fprintf(pOut, zFormat1, pc, + sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, zP4, pOp->p5, + zCom + ); + fflush(pOut); +} +#endif + +/* +** Release an array of N Mem elements +*/ +static void releaseMemArray(Mem *p, int N){ + if( p && N ){ + Mem *pEnd; + sqlite3 *db = p->db; + u8 malloc_failed = db->mallocFailed; + if( db->pnBytesFreed ){ + for(pEnd=&p[N]; pzMalloc); + } + return; + } + for(pEnd=&p[N]; pflags & MEM_Agg ); + testcase( p->flags & MEM_Dyn ); + testcase( p->flags & MEM_Frame ); + testcase( p->flags & MEM_RowSet ); + if( p->flags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ + sqlite3VdbeMemRelease(p); + }else if( p->zMalloc ){ + sqlite3DbFree(db, p->zMalloc); + p->zMalloc = 0; + } + + p->flags = MEM_Undefined; + } + db->mallocFailed = malloc_failed; + } +} + +/* +** Delete a VdbeFrame object and its contents. VdbeFrame objects are +** allocated by the OP_Program opcode in sqlite3VdbeExec(). +*/ +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ + int i; + Mem *aMem = VdbeFrameMem(p); + VdbeCursor **apCsr = (VdbeCursor **)&aMem[p->nChildMem]; + for(i=0; inChildCsr; i++){ + sqlite3VdbeFreeCursor(p->v, apCsr[i]); + } + releaseMemArray(aMem, p->nChildMem); + sqlite3DbFree(p->v->db, p); +} + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Give a listing of the program in the virtual machine. +** +** The interface is the same as sqlite3VdbeExec(). But instead of +** running the code, it invokes the callback once for each instruction. +** This feature is used to implement "EXPLAIN". +** +** When p->explain==1, each instruction is listed. When +** p->explain==2, only OP_Explain instructions are listed and these +** are shown in a different format. p->explain==2 is used to implement +** EXPLAIN QUERY PLAN. +** +** When p->explain==1, first the main program is listed, then each of +** the trigger subprograms are listed one by one. +*/ +SQLITE_PRIVATE int sqlite3VdbeList( + Vdbe *p /* The VDBE */ +){ + int nRow; /* Stop when row count reaches this */ + int nSub = 0; /* Number of sub-vdbes seen so far */ + SubProgram **apSub = 0; /* Array of sub-vdbes */ + Mem *pSub = 0; /* Memory cell hold array of subprogs */ + sqlite3 *db = p->db; /* The database connection */ + int i; /* Loop counter */ + int rc = SQLITE_OK; /* Return code */ + Mem *pMem = &p->aMem[1]; /* First Mem of result set */ + + assert( p->explain ); + assert( p->magic==VDBE_MAGIC_RUN ); + assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY || p->rc==SQLITE_NOMEM ); + + /* Even though this opcode does not use dynamic strings for + ** the result, result columns may become dynamic if the user calls + ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. + */ + releaseMemArray(pMem, 8); + p->pResultSet = 0; + + if( p->rc==SQLITE_NOMEM ){ + /* This happens if a malloc() inside a call to sqlite3_column_text() or + ** sqlite3_column_text16() failed. */ + db->mallocFailed = 1; + return SQLITE_ERROR; + } + + /* When the number of output rows reaches nRow, that means the + ** listing has finished and sqlite3_step() should return SQLITE_DONE. + ** nRow is the sum of the number of rows in the main program, plus + ** the sum of the number of rows in all trigger subprograms encountered + ** so far. The nRow value will increase as new trigger subprograms are + ** encountered, but p->pc will eventually catch up to nRow. + */ + nRow = p->nOp; + if( p->explain==1 ){ + /* The first 8 memory cells are used for the result set. So we will + ** commandeer the 9th cell to use as storage for an array of pointers + ** to trigger subprograms. The VDBE is guaranteed to have at least 9 + ** cells. */ + assert( p->nMem>9 ); + pSub = &p->aMem[9]; + if( pSub->flags&MEM_Blob ){ + /* On the first call to sqlite3_step(), pSub will hold a NULL. It is + ** initialized to a BLOB by the P4_SUBPROGRAM processing logic below */ + nSub = pSub->n/sizeof(Vdbe*); + apSub = (SubProgram **)pSub->z; + } + for(i=0; inOp; + } + } + + do{ + i = p->pc++; + }while( iexplain==2 && p->aOp[i].opcode!=OP_Explain ); + if( i>=nRow ){ + p->rc = SQLITE_OK; + rc = SQLITE_DONE; + }else if( db->u1.isInterrupted ){ + p->rc = SQLITE_INTERRUPT; + rc = SQLITE_ERROR; + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(p->rc)); + }else{ + char *zP4; + Op *pOp; + if( inOp ){ + /* The output line number is small enough that we are still in the + ** main program. */ + pOp = &p->aOp[i]; + }else{ + /* We are currently listing subprograms. Figure out which one and + ** pick up the appropriate opcode. */ + int j; + i -= p->nOp; + for(j=0; i>=apSub[j]->nOp; j++){ + i -= apSub[j]->nOp; + } + pOp = &apSub[j]->aOp[i]; + } + if( p->explain==1 ){ + pMem->flags = MEM_Int; + pMem->u.i = i; /* Program counter */ + pMem++; + + pMem->flags = MEM_Static|MEM_Str|MEM_Term; + pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */ + assert( pMem->z!=0 ); + pMem->n = sqlite3Strlen30(pMem->z); + pMem->enc = SQLITE_UTF8; + pMem++; + + /* When an OP_Program opcode is encounter (the only opcode that has + ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms + ** kept in p->aMem[9].z to hold the new program - assuming this subprogram + ** has not already been seen. + */ + if( pOp->p4type==P4_SUBPROGRAM ){ + int nByte = (nSub+1)*sizeof(SubProgram*); + int j; + for(j=0; jp4.pProgram ) break; + } + if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, nSub!=0) ){ + apSub = (SubProgram **)pSub->z; + apSub[nSub++] = pOp->p4.pProgram; + pSub->flags |= MEM_Blob; + pSub->n = nSub*sizeof(SubProgram*); + } + } + } + + pMem->flags = MEM_Int; + pMem->u.i = pOp->p1; /* P1 */ + pMem++; + + pMem->flags = MEM_Int; + pMem->u.i = pOp->p2; /* P2 */ + pMem++; + + pMem->flags = MEM_Int; + pMem->u.i = pOp->p3; /* P3 */ + pMem++; + + if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ + assert( p->db->mallocFailed ); + return SQLITE_ERROR; + } + pMem->flags = MEM_Str|MEM_Term; + zP4 = displayP4(pOp, pMem->z, 32); + if( zP4!=pMem->z ){ + sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); + }else{ + assert( pMem->z!=0 ); + pMem->n = sqlite3Strlen30(pMem->z); + pMem->enc = SQLITE_UTF8; + } + pMem++; + + if( p->explain==1 ){ + if( sqlite3VdbeMemGrow(pMem, 4, 0) ){ + assert( p->db->mallocFailed ); + return SQLITE_ERROR; + } + pMem->flags = MEM_Str|MEM_Term; + pMem->n = 2; + sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5); /* P5 */ + pMem->enc = SQLITE_UTF8; + pMem++; + +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + if( sqlite3VdbeMemGrow(pMem, 500, 0) ){ + assert( p->db->mallocFailed ); + return SQLITE_ERROR; + } + pMem->flags = MEM_Str|MEM_Term; + pMem->n = displayComment(pOp, zP4, pMem->z, 500); + pMem->enc = SQLITE_UTF8; +#else + pMem->flags = MEM_Null; /* Comment */ +#endif + } + + p->nResColumn = 8 - 4*(p->explain-1); + p->pResultSet = &p->aMem[1]; + p->rc = SQLITE_OK; + rc = SQLITE_ROW; + } + return rc; +} +#endif /* SQLITE_OMIT_EXPLAIN */ + +#ifdef SQLITE_DEBUG +/* +** Print the SQL that was used to generate a VDBE program. +*/ +SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe *p){ + const char *z = 0; + if( p->zSql ){ + z = p->zSql; + }else if( p->nOp>=1 ){ + const VdbeOp *pOp = &p->aOp[0]; + if( pOp->opcode==OP_Init && pOp->p4.z!=0 ){ + z = pOp->p4.z; + while( sqlite3Isspace(*z) ) z++; + } + } + if( z ) printf("SQL: [%s]\n", z); +} +#endif + +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) +/* +** Print an IOTRACE message showing SQL content. +*/ +SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ + int nOp = p->nOp; + VdbeOp *pOp; + if( sqlite3IoTrace==0 ) return; + if( nOp<1 ) return; + pOp = &p->aOp[0]; + if( pOp->opcode==OP_Init && pOp->p4.z!=0 ){ + int i, j; + char z[1000]; + sqlite3_snprintf(sizeof(z), z, "%s", pOp->p4.z); + for(i=0; sqlite3Isspace(z[i]); i++){} + for(j=0; z[i]; i++){ + if( sqlite3Isspace(z[i]) ){ + if( z[i-1]!=' ' ){ + z[j++] = ' '; + } + }else{ + z[j++] = z[i]; + } + } + z[j] = 0; + sqlite3IoTrace("SQL %s\n", z); + } +} +#endif /* !SQLITE_OMIT_TRACE && SQLITE_ENABLE_IOTRACE */ + +/* +** Allocate space from a fixed size buffer and return a pointer to +** that space. If insufficient space is available, return NULL. +** +** The pBuf parameter is the initial value of a pointer which will +** receive the new memory. pBuf is normally NULL. If pBuf is not +** NULL, it means that memory space has already been allocated and that +** this routine should not allocate any new memory. When pBuf is not +** NULL simply return pBuf. Only allocate new memory space when pBuf +** is NULL. +** +** nByte is the number of bytes of space needed. +** +** *ppFrom points to available space and pEnd points to the end of the +** available space. When space is allocated, *ppFrom is advanced past +** the end of the allocated space. +** +** *pnByte is a counter of the number of bytes of space that have failed +** to allocate. If there is insufficient space in *ppFrom to satisfy the +** request, then increment *pnByte by the amount of the request. +*/ +static void *allocSpace( + void *pBuf, /* Where return pointer will be stored */ + int nByte, /* Number of bytes to allocate */ + u8 **ppFrom, /* IN/OUT: Allocate from *ppFrom */ + u8 *pEnd, /* Pointer to 1 byte past the end of *ppFrom buffer */ + int *pnByte /* If allocation cannot be made, increment *pnByte */ +){ + assert( EIGHT_BYTE_ALIGNMENT(*ppFrom) ); + if( pBuf ) return pBuf; + nByte = ROUND8(nByte); + if( &(*ppFrom)[nByte] <= pEnd ){ + pBuf = (void*)*ppFrom; + *ppFrom += nByte; + }else{ + *pnByte += nByte; + } + return pBuf; +} + +/* +** Rewind the VDBE back to the beginning in preparation for +** running it. +*/ +SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + int i; +#endif + assert( p!=0 ); + assert( p->magic==VDBE_MAGIC_INIT ); + + /* There should be at least one opcode. + */ + assert( p->nOp>0 ); + + /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ + p->magic = VDBE_MAGIC_RUN; + +#ifdef SQLITE_DEBUG + for(i=1; inMem; i++){ + assert( p->aMem[i].db==p->db ); + } +#endif + p->pc = -1; + p->rc = SQLITE_OK; + p->errorAction = OE_Abort; + p->magic = VDBE_MAGIC_RUN; + p->nChange = 0; + p->cacheCtr = 1; + p->minWriteFileFormat = 255; + p->iStatement = 0; + p->nFkConstraint = 0; +#ifdef VDBE_PROFILE + for(i=0; inOp; i++){ + p->aOp[i].cnt = 0; + p->aOp[i].cycles = 0; + } +#endif +} + +/* +** Prepare a virtual machine for execution for the first time after +** creating the virtual machine. This involves things such +** as allocating stack space and initializing the program counter. +** After the VDBE has be prepped, it can be executed by one or more +** calls to sqlite3VdbeExec(). +** +** This function may be called exact once on a each virtual machine. +** After this routine is called the VM has been "packaged" and is ready +** to run. After this routine is called, futher calls to +** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects +** the Vdbe from the Parse object that helped generate it so that the +** the Vdbe becomes an independent entity and the Parse object can be +** destroyed. +** +** Use the sqlite3VdbeRewind() procedure to restore a virtual machine back +** to its initial state after it has been run. +*/ +SQLITE_PRIVATE void sqlite3VdbeMakeReady( + Vdbe *p, /* The VDBE */ + Parse *pParse /* Parsing context */ +){ + sqlite3 *db; /* The database connection */ + int nVar; /* Number of parameters */ + int nMem; /* Number of VM memory registers */ + int nCursor; /* Number of cursors required */ + int nArg; /* Number of arguments in subprograms */ + int nOnce; /* Number of OP_Once instructions */ + int n; /* Loop counter */ + u8 *zCsr; /* Memory available for allocation */ + u8 *zEnd; /* First byte past allocated memory */ + int nByte; /* How much extra memory is needed */ + + assert( p!=0 ); + assert( p->nOp>0 ); + assert( pParse!=0 ); + assert( p->magic==VDBE_MAGIC_INIT ); + assert( pParse==p->pParse ); + db = p->db; + assert( db->mallocFailed==0 ); + nVar = pParse->nVar; + nMem = pParse->nMem; + nCursor = pParse->nTab; + nArg = pParse->nMaxArg; + nOnce = pParse->nOnce; + if( nOnce==0 ) nOnce = 1; /* Ensure at least one byte in p->aOnceFlag[] */ + + /* For each cursor required, also allocate a memory cell. Memory + ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by + ** the vdbe program. Instead they are used to allocate space for + ** VdbeCursor/BtCursor structures. The blob of memory associated with + ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1) + ** stores the blob of memory associated with cursor 1, etc. + ** + ** See also: allocateCursor(). + */ + nMem += nCursor; + + /* Allocate space for memory registers, SQL variables, VDBE cursors and + ** an array to marshal SQL function arguments in. + */ + zCsr = (u8*)&p->aOp[p->nOp]; /* Memory avaliable for allocation */ + zEnd = (u8*)&p->aOp[pParse->nOpAlloc]; /* First byte past end of zCsr[] */ + + resolveP2Values(p, &nArg); + p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); + if( pParse->explain && nMem<10 ){ + nMem = 10; + } + memset(zCsr, 0, zEnd-zCsr); + zCsr += (zCsr - (u8*)0)&7; + assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + p->expired = 0; + + /* Memory for registers, parameters, cursor, etc, is allocated in two + ** passes. On the first pass, we try to reuse unused space at the + ** end of the opcode array. If we are unable to satisfy all memory + ** requirements by reusing the opcode array tail, then the second + ** pass will fill in the rest using a fresh allocation. + ** + ** This two-pass approach that reuses as much memory as possible from + ** the leftover space at the end of the opcode array can significantly + ** reduce the amount of memory held by a prepared statement. + */ + do { + nByte = 0; + p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); + p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); + p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); + p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); + p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), + &zCsr, zEnd, &nByte); + p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); + if( nByte ){ + p->pFree = sqlite3DbMallocZero(db, nByte); + } + zCsr = p->pFree; + zEnd = &zCsr[nByte]; + }while( nByte && !db->mallocFailed ); + + p->nCursor = nCursor; + p->nOnceFlag = nOnce; + if( p->aVar ){ + p->nVar = (ynVar)nVar; + for(n=0; naVar[n].flags = MEM_Null; + p->aVar[n].db = db; + } + } + if( p->azVar ){ + p->nzVar = pParse->nzVar; + memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); + memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); + } + if( p->aMem ){ + p->aMem--; /* aMem[] goes from 1..nMem */ + p->nMem = nMem; /* not from 0..nMem-1 */ + for(n=1; n<=nMem; n++){ + p->aMem[n].flags = MEM_Undefined; + p->aMem[n].db = db; + } + } + p->explain = pParse->explain; + sqlite3VdbeRewind(p); +} + +/* +** Close a VDBE cursor and release all the resources that cursor +** happens to hold. +*/ +SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ + if( pCx==0 ){ + return; + } + sqlite3VdbeSorterClose(p->db, pCx); + if( pCx->pBt ){ + sqlite3BtreeClose(pCx->pBt); + /* The pCx->pCursor will be close automatically, if it exists, by + ** the call above. */ + }else if( pCx->pCursor ){ + sqlite3BtreeCloseCursor(pCx->pCursor); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pCx->pVtabCursor ){ + sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; + const sqlite3_module *pModule = pVtabCursor->pVtab->pModule; + p->inVtabMethod = 1; + pModule->xClose(pVtabCursor); + p->inVtabMethod = 0; + } +#endif +} + +/* +** Copy the values stored in the VdbeFrame structure to its Vdbe. This +** is used, for example, when a trigger sub-program is halted to restore +** control to the main program. +*/ +SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ + Vdbe *v = pFrame->v; + v->aOnceFlag = pFrame->aOnceFlag; + v->nOnceFlag = pFrame->nOnceFlag; + v->aOp = pFrame->aOp; + v->nOp = pFrame->nOp; + v->aMem = pFrame->aMem; + v->nMem = pFrame->nMem; + v->apCsr = pFrame->apCsr; + v->nCursor = pFrame->nCursor; + v->db->lastRowid = pFrame->lastRowid; + v->nChange = pFrame->nChange; + return pFrame->pc; +} + +/* +** Close all cursors. +** +** Also release any dynamic memory held by the VM in the Vdbe.aMem memory +** cell array. This is necessary as the memory cell array may contain +** pointers to VdbeFrame objects, which may in turn contain pointers to +** open cursors. +*/ +static void closeAllCursors(Vdbe *p){ + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + sqlite3VdbeFrameRestore(pFrame); + } + p->pFrame = 0; + p->nFrame = 0; + + if( p->apCsr ){ + int i; + for(i=0; inCursor; i++){ + VdbeCursor *pC = p->apCsr[i]; + if( pC ){ + sqlite3VdbeFreeCursor(p, pC); + p->apCsr[i] = 0; + } + } + } + if( p->aMem ){ + releaseMemArray(&p->aMem[1], p->nMem); + } + while( p->pDelFrame ){ + VdbeFrame *pDel = p->pDelFrame; + p->pDelFrame = pDel->pParent; + sqlite3VdbeFrameDelete(pDel); + } + + /* Delete any auxdata allocations made by the VM */ + sqlite3VdbeDeleteAuxData(p, -1, 0); + assert( p->pAuxData==0 ); +} + +/* +** Clean up the VM after execution. +** +** This routine will automatically close any cursors, lists, and/or +** sorters that were left open. It also deletes the values of +** variables in the aVar[] array. +*/ +static void Cleanup(Vdbe *p){ + sqlite3 *db = p->db; + +#ifdef SQLITE_DEBUG + /* Execute assert() statements to ensure that the Vdbe.apCsr[] and + ** Vdbe.aMem[] arrays have already been cleaned up. */ + int i; + if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 ); + if( p->aMem ){ + for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); + } +#endif + + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; + p->pResultSet = 0; +} + +/* +** Set the number of result columns that will be returned by this SQL +** statement. This is now set at compile time, rather than during +** execution of the vdbe program so that sqlite3_column_count() can +** be called on an SQL statement before sqlite3_step(). +*/ +SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ + Mem *pColName; + int n; + sqlite3 *db = p->db; + + releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + sqlite3DbFree(db, p->aColName); + n = nResColumn*COLNAME_N; + p->nResColumn = (u16)nResColumn; + p->aColName = pColName = (Mem*)sqlite3DbMallocZero(db, sizeof(Mem)*n ); + if( p->aColName==0 ) return; + while( n-- > 0 ){ + pColName->flags = MEM_Null; + pColName->db = p->db; + pColName++; + } +} + +/* +** Set the name of the idx'th column to be returned by the SQL statement. +** zName must be a pointer to a nul terminated string. +** +** This call must be made after a call to sqlite3VdbeSetNumCols(). +** +** The final parameter, xDel, must be one of SQLITE_DYNAMIC, SQLITE_STATIC +** or SQLITE_TRANSIENT. If it is SQLITE_DYNAMIC, then the buffer pointed +** to by zName will be freed by sqlite3DbFree() when the vdbe is destroyed. +*/ +SQLITE_PRIVATE int sqlite3VdbeSetColName( + Vdbe *p, /* Vdbe being configured */ + int idx, /* Index of column zName applies to */ + int var, /* One of the COLNAME_* constants */ + const char *zName, /* Pointer to buffer containing name */ + void (*xDel)(void*) /* Memory management strategy for zName */ +){ + int rc; + Mem *pColName; + assert( idxnResColumn ); + assert( vardb->mallocFailed ){ + assert( !zName || xDel!=SQLITE_DYNAMIC ); + return SQLITE_NOMEM; + } + assert( p->aColName!=0 ); + pColName = &(p->aColName[idx+var*p->nResColumn]); + rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); + assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); + return rc; +} + +/* +** A read or write transaction may or may not be active on database handle +** db. If a transaction is active, commit it. If there is a +** write-transaction spanning more than one database file, this routine +** takes care of the master journal trickery. +*/ +static int vdbeCommit(sqlite3 *db, Vdbe *p){ + int i; + int nTrans = 0; /* Number of databases with an active write-transaction */ + int rc = SQLITE_OK; + int needXcommit = 0; + +#ifdef SQLITE_OMIT_VIRTUALTABLE + /* With this option, sqlite3VtabSync() is defined to be simply + ** SQLITE_OK so p is not used. + */ + UNUSED_PARAMETER(p); +#endif + + /* Before doing anything else, call the xSync() callback for any + ** virtual module tables written in this transaction. This has to + ** be done before determining whether a master journal file is + ** required, as an xSync() callback may add an attached database + ** to the transaction. + */ + rc = sqlite3VtabSync(db, p); + + /* This loop determines (a) if the commit hook should be invoked and + ** (b) how many database files have open write transactions, not + ** including the temp database. (b) is important because if more than + ** one database file has an open write transaction, a master journal + ** file is required for an atomic commit. + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeIsInTrans(pBt) ){ + needXcommit = 1; + if( i!=1 ) nTrans++; + sqlite3BtreeEnter(pBt); + rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); + } + } + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If there are any write-transactions at all, invoke the commit hook */ + if( needXcommit && db->xCommitCallback ){ + rc = db->xCommitCallback(db->pCommitArg); + if( rc ){ + return SQLITE_CONSTRAINT_COMMITHOOK; + } + } + + /* The simple case - no more than one database file (not counting the + ** TEMP database) has a transaction active. There is no need for the + ** master-journal. + ** + ** If the return value of sqlite3BtreeGetFilename() is a zero length + ** string, it means the main database is :memory: or a temp file. In + ** that case we do not support atomic multi-file commits, so use the + ** simple case then too. + */ + if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) + || nTrans<=1 + ){ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + } + } + + /* Do the commit only if all databases successfully complete phase 1. + ** If one of the BtreeCommitPhaseOne() calls fails, this indicates an + ** IO error while deleting or truncating a journal file. It is unlikely, + ** but could happen. In this case abandon processing and return the error. + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + rc = sqlite3BtreeCommitPhaseTwo(pBt, 0); + } + } + if( rc==SQLITE_OK ){ + sqlite3VtabCommit(db); + } + } + + /* The complex case - There is a multi-file write-transaction active. + ** This requires a master journal file to ensure the transaction is + ** committed atomicly. + */ +#ifndef SQLITE_OMIT_DISKIO + else{ + sqlite3_vfs *pVfs = db->pVfs; + int needSync = 0; + char *zMaster = 0; /* File-name for the master journal */ + char const *zMainFile = sqlite3BtreeGetFilename(db->aDb[0].pBt); + sqlite3_file *pMaster = 0; + i64 offset = 0; + int res; + int retryCount = 0; + int nMainFile; + + /* Select a master journal file name */ + nMainFile = sqlite3Strlen30(zMainFile); + zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XXz", zMainFile); + if( zMaster==0 ) return SQLITE_NOMEM; + do { + u32 iRandom; + if( retryCount ){ + if( retryCount>100 ){ + sqlite3_log(SQLITE_FULL, "MJ delete: %s", zMaster); + sqlite3OsDelete(pVfs, zMaster, 0); + break; + }else if( retryCount==1 ){ + sqlite3_log(SQLITE_FULL, "MJ collide: %s", zMaster); + } + } + retryCount++; + sqlite3_randomness(sizeof(iRandom), &iRandom); + sqlite3_snprintf(13, &zMaster[nMainFile], "-mj%06X9%02X", + (iRandom>>8)&0xffffff, iRandom&0xff); + /* The antipenultimate character of the master journal name must + ** be "9" to avoid name collisions when using 8+3 filenames. */ + assert( zMaster[sqlite3Strlen30(zMaster)-3]=='9' ); + sqlite3FileSuffix3(zMainFile, zMaster); + rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); + }while( rc==SQLITE_OK && res ); + if( rc==SQLITE_OK ){ + /* Open the master journal. */ + rc = sqlite3OsOpenMalloc(pVfs, zMaster, &pMaster, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + SQLITE_OPEN_EXCLUSIVE|SQLITE_OPEN_MASTER_JOURNAL, 0 + ); + } + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, zMaster); + return rc; + } + + /* Write the name of each database file in the transaction into the new + ** master journal file. If an error occurs at this point close + ** and delete the master journal file. All the individual journal files + ** still have 'null' as the master journal pointer, so they will roll + ** back independently if a failure occurs. + */ + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeIsInTrans(pBt) ){ + char const *zFile = sqlite3BtreeGetJournalname(pBt); + if( zFile==0 ){ + continue; /* Ignore TEMP and :memory: databases */ + } + assert( zFile[0]!=0 ); + if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ + needSync = 1; + } + rc = sqlite3OsWrite(pMaster, zFile, sqlite3Strlen30(zFile)+1, offset); + offset += sqlite3Strlen30(zFile)+1; + if( rc!=SQLITE_OK ){ + sqlite3OsCloseFree(pMaster); + sqlite3OsDelete(pVfs, zMaster, 0); + sqlite3DbFree(db, zMaster); + return rc; + } + } + } + + /* Sync the master journal file. If the IOCAP_SEQUENTIAL device + ** flag is set this is not required. + */ + if( needSync + && 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) + && SQLITE_OK!=(rc = sqlite3OsSync(pMaster, SQLITE_SYNC_NORMAL)) + ){ + sqlite3OsCloseFree(pMaster); + sqlite3OsDelete(pVfs, zMaster, 0); + sqlite3DbFree(db, zMaster); + return rc; + } + + /* Sync all the db files involved in the transaction. The same call + ** sets the master journal pointer in each individual journal. If + ** an error occurs here, do not delete the master journal file. + ** + ** If the error occurs during the first call to + ** sqlite3BtreeCommitPhaseOne(), then there is a chance that the + ** master journal file will be orphaned. But we cannot delete it, + ** in case the master journal file name was written into the journal + ** file before the failure occurred. + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, zMaster); + } + } + sqlite3OsCloseFree(pMaster); + assert( rc!=SQLITE_BUSY ); + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, zMaster); + return rc; + } + + /* Delete the master journal file. This commits the transaction. After + ** doing this the directory is synced again before any individual + ** transaction files are deleted. + */ + rc = sqlite3OsDelete(pVfs, zMaster, 1); + sqlite3DbFree(db, zMaster); + zMaster = 0; + if( rc ){ + return rc; + } + + /* All files and directories have already been synced, so the following + ** calls to sqlite3BtreeCommitPhaseTwo() are only closing files and + ** deleting or truncating journals. If something goes wrong while + ** this is happening we don't really care. The integrity of the + ** transaction is already guaranteed, but some stray 'cold' journals + ** may be lying around. Returning an error code won't help matters. + */ + disable_simulated_io_errors(); + sqlite3BeginBenignMalloc(); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + sqlite3BtreeCommitPhaseTwo(pBt, 1); + } + } + sqlite3EndBenignMalloc(); + enable_simulated_io_errors(); + + sqlite3VtabCommit(db); + } +#endif + + return rc; +} + +/* +** This routine checks that the sqlite3.nVdbeActive count variable +** matches the number of vdbe's in the list sqlite3.pVdbe that are +** currently active. An assertion fails if the two counts do not match. +** This is an internal self-check only - it is not an essential processing +** step. +** +** This is a no-op if NDEBUG is defined. +*/ +#ifndef NDEBUG +static void checkActiveVdbeCnt(sqlite3 *db){ + Vdbe *p; + int cnt = 0; + int nWrite = 0; + int nRead = 0; + p = db->pVdbe; + while( p ){ + if( sqlite3_stmt_busy((sqlite3_stmt*)p) ){ + cnt++; + if( p->readOnly==0 ) nWrite++; + if( p->bIsReader ) nRead++; + } + p = p->pNext; + } + assert( cnt==db->nVdbeActive ); + assert( nWrite==db->nVdbeWrite ); + assert( nRead==db->nVdbeRead ); +} +#else +#define checkActiveVdbeCnt(x) +#endif + +/* +** If the Vdbe passed as the first argument opened a statement-transaction, +** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or +** SAVEPOINT_RELEASE. If it is SAVEPOINT_ROLLBACK, then the statement +** transaction is rolled back. If eOp is SAVEPOINT_RELEASE, then the +** statement transaction is committed. +** +** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. +** Otherwise SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ + sqlite3 *const db = p->db; + int rc = SQLITE_OK; + + /* If p->iStatement is greater than zero, then this Vdbe opened a + ** statement transaction that should be closed here. The only exception + ** is that an IO error may have occurred, causing an emergency rollback. + ** In this case (db->nStatement==0), and there is nothing to do. + */ + if( db->nStatement && p->iStatement ){ + int i; + const int iSavepoint = p->iStatement-1; + + assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); + assert( db->nStatement>0 ); + assert( p->iStatement==(db->nStatement+db->nSavepoint) ); + + for(i=0; inDb; i++){ + int rc2 = SQLITE_OK; + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + if( eOp==SAVEPOINT_ROLLBACK ){ + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc2==SQLITE_OK ){ + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); + } + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + db->nStatement--; + p->iStatement = 0; + + if( rc==SQLITE_OK ){ + if( eOp==SAVEPOINT_ROLLBACK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc==SQLITE_OK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + } + } + + /* If the statement transaction is being rolled back, also restore the + ** database handles deferred constraint counter to the value it had when + ** the statement transaction was opened. */ + if( eOp==SAVEPOINT_ROLLBACK ){ + db->nDeferredCons = p->nStmtDefCons; + db->nDeferredImmCons = p->nStmtDefImmCons; + } + } + return rc; +} + +/* +** This function is called when a transaction opened by the database +** handle associated with the VM passed as an argument is about to be +** committed. If there are outstanding deferred foreign key constraint +** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. +** +** If there are outstanding FK violations and this function returns +** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY +** and write an error message to it. Then return SQLITE_ERROR. +*/ +#ifndef SQLITE_OMIT_FOREIGN_KEY +SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ + sqlite3 *db = p->db; + if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) + || (!deferred && p->nFkConstraint>0) + ){ + p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; + p->errorAction = OE_Abort; + sqlite3SetString(&p->zErrMsg, db, "FOREIGN KEY constraint failed"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} +#endif + +/* +** This routine is called the when a VDBE tries to halt. If the VDBE +** has made changes and is in autocommit mode, then commit those +** changes. If a rollback is needed, then do the rollback. +** +** This routine is the only way to move the state of a VM from +** SQLITE_MAGIC_RUN to SQLITE_MAGIC_HALT. It is harmless to +** call this on a VM that is in the SQLITE_MAGIC_HALT state. +** +** Return an error code. If the commit could not complete because of +** lock contention, return SQLITE_BUSY. If SQLITE_BUSY is returned, it +** means the close did not happen and needs to be repeated. +*/ +SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ + int rc; /* Used to store transient return codes */ + sqlite3 *db = p->db; + + /* This function contains the logic that determines if a statement or + ** transaction will be committed or rolled back as a result of the + ** execution of this virtual machine. + ** + ** If any of the following errors occur: + ** + ** SQLITE_NOMEM + ** SQLITE_IOERR + ** SQLITE_FULL + ** SQLITE_INTERRUPT + ** + ** Then the internal cache might have been left in an inconsistent + ** state. We need to rollback the statement transaction, if there is + ** one, or the complete transaction if there is no statement transaction. + */ + + if( p->db->mallocFailed ){ + p->rc = SQLITE_NOMEM; + } + if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag); + closeAllCursors(p); + if( p->magic!=VDBE_MAGIC_RUN ){ + return SQLITE_OK; + } + checkActiveVdbeCnt(db); + + /* No commit or rollback needed if the program never started or if the + ** SQL statement does not read or write a database file. */ + if( p->pc>=0 && p->bIsReader ){ + int mrc; /* Primary error code from p->rc */ + int eStatementOp = 0; + int isSpecialError; /* Set to true if a 'special' error */ + + /* Lock all btrees used by the statement */ + sqlite3VdbeEnter(p); + + /* Check for one of the special errors */ + mrc = p->rc & 0xff; + isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR + || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; + if( isSpecialError ){ + /* If the query was read-only and the error code is SQLITE_INTERRUPT, + ** no rollback is necessary. Otherwise, at least a savepoint + ** transaction must be rolled back to restore the database to a + ** consistent state. + ** + ** Even if the statement is read-only, it is important to perform + ** a statement or transaction rollback operation. If the error + ** occurred while writing to the journal, sub-journal or database + ** file as part of an effort to free up cache space (see function + ** pagerStress() in pager.c), the rollback is required to restore + ** the pager to a consistent state. + */ + if( !p->readOnly || mrc!=SQLITE_INTERRUPT ){ + if( (mrc==SQLITE_NOMEM || mrc==SQLITE_FULL) && p->usesStmtJournal ){ + eStatementOp = SAVEPOINT_ROLLBACK; + }else{ + /* We are forced to roll back the active transaction. Before doing + ** so, abort any other statements this handle currently has active. + */ + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); + sqlite3CloseSavepoints(db); + db->autoCommit = 1; + } + } + } + + /* Check for immediate foreign key violations. */ + if( p->rc==SQLITE_OK ){ + sqlite3VdbeCheckFk(p, 0); + } + + /* If the auto-commit flag is set and this is the only active writer + ** VM, then we do either a commit or rollback of the current transaction. + ** + ** Note: This block also runs if one of the special errors handled + ** above has occurred. + */ + if( !sqlite3VtabInSync(db) + && db->autoCommit + && db->nVdbeWrite==(p->readOnly==0) + ){ + if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ + rc = sqlite3VdbeCheckFk(p, 1); + if( rc!=SQLITE_OK ){ + if( NEVER(p->readOnly) ){ + sqlite3VdbeLeave(p); + return SQLITE_ERROR; + } + rc = SQLITE_CONSTRAINT_FOREIGNKEY; + }else{ + /* The auto-commit flag is true, the vdbe program was successful + ** or hit an 'OR FAIL' constraint and there are no deferred foreign + ** key constraints to hold up the transaction. This means a commit + ** is required. */ + rc = vdbeCommit(db, p); + } + if( rc==SQLITE_BUSY && p->readOnly ){ + sqlite3VdbeLeave(p); + return SQLITE_BUSY; + }else if( rc!=SQLITE_OK ){ + p->rc = rc; + sqlite3RollbackAll(db, SQLITE_OK); + }else{ + db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~SQLITE_DeferFKs; + sqlite3CommitInternalChanges(db); + } + }else{ + sqlite3RollbackAll(db, SQLITE_OK); + } + db->nStatement = 0; + }else if( eStatementOp==0 ){ + if( p->rc==SQLITE_OK || p->errorAction==OE_Fail ){ + eStatementOp = SAVEPOINT_RELEASE; + }else if( p->errorAction==OE_Abort ){ + eStatementOp = SAVEPOINT_ROLLBACK; + }else{ + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); + sqlite3CloseSavepoints(db); + db->autoCommit = 1; + } + } + + /* If eStatementOp is non-zero, then a statement transaction needs to + ** be committed or rolled back. Call sqlite3VdbeCloseStatement() to + ** do so. If this operation returns an error, and the current statement + ** error code is SQLITE_OK or SQLITE_CONSTRAINT, then promote the + ** current statement error code. + */ + if( eStatementOp ){ + rc = sqlite3VdbeCloseStatement(p, eStatementOp); + if( rc ){ + if( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ){ + p->rc = rc; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; + } + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); + sqlite3CloseSavepoints(db); + db->autoCommit = 1; + } + } + + /* If this was an INSERT, UPDATE or DELETE and no statement transaction + ** has been rolled back, update the database connection change-counter. + */ + if( p->changeCntOn ){ + if( eStatementOp!=SAVEPOINT_ROLLBACK ){ + sqlite3VdbeSetChanges(db, p->nChange); + }else{ + sqlite3VdbeSetChanges(db, 0); + } + p->nChange = 0; + } + + /* Release the locks */ + sqlite3VdbeLeave(p); + } + + /* We have successfully halted and closed the VM. Record this fact. */ + if( p->pc>=0 ){ + db->nVdbeActive--; + if( !p->readOnly ) db->nVdbeWrite--; + if( p->bIsReader ) db->nVdbeRead--; + assert( db->nVdbeActive>=db->nVdbeRead ); + assert( db->nVdbeRead>=db->nVdbeWrite ); + assert( db->nVdbeWrite>=0 ); + } + p->magic = VDBE_MAGIC_HALT; + checkActiveVdbeCnt(db); + if( p->db->mallocFailed ){ + p->rc = SQLITE_NOMEM; + } + + /* If the auto-commit flag is set to true, then any locks that were held + ** by connection db have now been released. Call sqlite3ConnectionUnlocked() + ** to invoke any required unlock-notify callbacks. + */ + if( db->autoCommit ){ + sqlite3ConnectionUnlocked(db); + } + + assert( db->nVdbeActive>0 || db->autoCommit==0 || db->nStatement==0 ); + return (p->rc==SQLITE_BUSY ? SQLITE_BUSY : SQLITE_OK); +} + + +/* +** Each VDBE holds the result of the most recent sqlite3_step() call +** in p->rc. This routine sets that result back to SQLITE_OK. +*/ +SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe *p){ + p->rc = SQLITE_OK; +} + +/* +** Copy the error code and error message belonging to the VDBE passed +** as the first argument to its database handle (so that they will be +** returned by calls to sqlite3_errcode() and sqlite3_errmsg()). +** +** This function does not clear the VDBE error code or message, just +** copies them to the database handle. +*/ +SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p){ + sqlite3 *db = p->db; + int rc = p->rc; + if( p->zErrMsg ){ + u8 mallocFailed = db->mallocFailed; + sqlite3BeginBenignMalloc(); + if( db->pErr==0 ) db->pErr = sqlite3ValueNew(db); + sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); + sqlite3EndBenignMalloc(); + db->mallocFailed = mallocFailed; + db->errCode = rc; + }else{ + sqlite3Error(db, rc, 0); + } + return rc; +} + +#ifdef SQLITE_ENABLE_SQLLOG +/* +** If an SQLITE_CONFIG_SQLLOG hook is registered and the VM has been run, +** invoke it. +*/ +static void vdbeInvokeSqllog(Vdbe *v){ + if( sqlite3GlobalConfig.xSqllog && v->rc==SQLITE_OK && v->zSql && v->pc>=0 ){ + char *zExpanded = sqlite3VdbeExpandSql(v, v->zSql); + assert( v->db->init.busy==0 ); + if( zExpanded ){ + sqlite3GlobalConfig.xSqllog( + sqlite3GlobalConfig.pSqllogArg, v->db, zExpanded, 1 + ); + sqlite3DbFree(v->db, zExpanded); + } + } +} +#else +# define vdbeInvokeSqllog(x) +#endif + +/* +** Clean up a VDBE after execution but do not delete the VDBE just yet. +** Write any error messages into *pzErrMsg. Return the result code. +** +** After this routine is run, the VDBE should be ready to be executed +** again. +** +** To look at it another way, this routine resets the state of the +** virtual machine from VDBE_MAGIC_RUN or VDBE_MAGIC_HALT back to +** VDBE_MAGIC_INIT. +*/ +SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ + sqlite3 *db; + db = p->db; + + /* If the VM did not run to completion or if it encountered an + ** error, then it might not have been halted properly. So halt + ** it now. + */ + sqlite3VdbeHalt(p); + + /* If the VDBE has be run even partially, then transfer the error code + ** and error message from the VDBE into the main database structure. But + ** if the VDBE has just been set to run but has not actually executed any + ** instructions yet, leave the main database error information unchanged. + */ + if( p->pc>=0 ){ + vdbeInvokeSqllog(p); + sqlite3VdbeTransferError(p); + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; + if( p->runOnlyOnce ) p->expired = 1; + }else if( p->rc && p->expired ){ + /* The expired flag was set on the VDBE before the first call + ** to sqlite3_step(). For consistency (since sqlite3_step() was + ** called), set the database error in this case as well. + */ + sqlite3Error(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; + } + + /* Reclaim all memory used by the VDBE + */ + Cleanup(p); + + /* Save profiling information from this VDBE run. + */ +#ifdef VDBE_PROFILE + { + FILE *out = fopen("vdbe_profile.out", "a"); + if( out ){ + int i; + fprintf(out, "---- "); + for(i=0; inOp; i++){ + fprintf(out, "%02x", p->aOp[i].opcode); + } + fprintf(out, "\n"); + if( p->zSql ){ + char c, pc = 0; + fprintf(out, "-- "); + for(i=0; (c = p->zSql[i])!=0; i++){ + if( pc=='\n' ) fprintf(out, "-- "); + putc(c, out); + pc = c; + } + if( pc!='\n' ) fprintf(out, "\n"); + } + for(i=0; inOp; i++){ + char zHdr[100]; + sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", + p->aOp[i].cnt, + p->aOp[i].cycles, + p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 + ); + fprintf(out, "%s", zHdr); + sqlite3VdbePrintOp(out, i, &p->aOp[i]); + } + fclose(out); + } + } +#endif + p->iCurrentTime = 0; + p->magic = VDBE_MAGIC_INIT; + return p->rc & db->errMask; +} + +/* +** Clean up and delete a VDBE after execution. Return an integer which is +** the result code. Write any error message text into *pzErrMsg. +*/ +SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ + int rc = SQLITE_OK; + if( p->magic==VDBE_MAGIC_RUN || p->magic==VDBE_MAGIC_HALT ){ + rc = sqlite3VdbeReset(p); + assert( (rc & p->db->errMask)==rc ); + } + sqlite3VdbeDelete(p); + return rc; +} + +/* +** If parameter iOp is less than zero, then invoke the destructor for +** all auxiliary data pointers currently cached by the VM passed as +** the first argument. +** +** Or, if iOp is greater than or equal to zero, then the destructor is +** only invoked for those auxiliary data pointers created by the user +** function invoked by the OP_Function opcode at instruction iOp of +** VM pVdbe, and only then if: +** +** * the associated function parameter is the 32nd or later (counting +** from left to right), or +** +** * the corresponding bit in argument mask is clear (where the first +** function parameter corrsponds to bit 0 etc.). +*/ +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){ + AuxData **pp = &pVdbe->pAuxData; + while( *pp ){ + AuxData *pAux = *pp; + if( (iOp<0) + || (pAux->iOp==iOp && (pAux->iArg>31 || !(mask & MASKBIT32(pAux->iArg)))) + ){ + testcase( pAux->iArg==31 ); + if( pAux->xDelete ){ + pAux->xDelete(pAux->pAux); + } + *pp = pAux->pNext; + sqlite3DbFree(pVdbe->db, pAux); + }else{ + pp= &pAux->pNext; + } + } +} + +/* +** Free all memory associated with the Vdbe passed as the second argument, +** except for object itself, which is preserved. +** +** The difference between this function and sqlite3VdbeDelete() is that +** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with +** the database connection and frees the object itself. +*/ +SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ + SubProgram *pSub, *pNext; + int i; + assert( p->db==0 || p->db==db ); + releaseMemArray(p->aVar, p->nVar); + releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + for(pSub=p->pProgram; pSub; pSub=pNext){ + pNext = pSub->pNext; + vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); + sqlite3DbFree(db, pSub); + } + for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); + vdbeFreeOpArray(db, p->aOp, p->nOp); + sqlite3DbFree(db, p->aColName); + sqlite3DbFree(db, p->zSql); + sqlite3DbFree(db, p->pFree); +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) + sqlite3DbFree(db, p->zExplain); + sqlite3DbFree(db, p->pExplain); +#endif +} + +/* +** Delete an entire VDBE. +*/ +SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ + sqlite3 *db; + + if( NEVER(p==0) ) return; + db = p->db; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3VdbeClearObject(db, p); + if( p->pPrev ){ + p->pPrev->pNext = p->pNext; + }else{ + assert( db->pVdbe==p ); + db->pVdbe = p->pNext; + } + if( p->pNext ){ + p->pNext->pPrev = p->pPrev; + } + p->magic = VDBE_MAGIC_DEAD; + p->db = 0; + sqlite3DbFree(db, p); +} + +/* +** Make sure the cursor p is ready to read or write the row to which it +** was last positioned. Return an error code if an OOM fault or I/O error +** prevents us from positioning the cursor to its correct position. +** +** If a MoveTo operation is pending on the given cursor, then do that +** MoveTo now. If no move is pending, check to see if the row has been +** deleted out from under the cursor and if it has, mark the row as +** a NULL row. +** +** If the cursor is already pointing to the correct row and that row has +** not been deleted out from under the cursor, then this routine is a no-op. +*/ +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ + if( p->deferredMoveto ){ + int res, rc; +#ifdef SQLITE_TEST + extern int sqlite3_search_count; +#endif + assert( p->isTable ); + rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); + if( rc ) return rc; + p->lastRowid = p->movetoTarget; + if( res!=0 ) return SQLITE_CORRUPT_BKPT; + p->rowidIsValid = 1; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + p->deferredMoveto = 0; + p->cacheStatus = CACHE_STALE; + }else if( p->pCursor ){ + int hasMoved; + int rc = sqlite3BtreeCursorHasMoved(p->pCursor, &hasMoved); + if( rc ) return rc; + if( hasMoved ){ + p->cacheStatus = CACHE_STALE; + if( hasMoved==2 ) p->nullRow = 1; + } + } + return SQLITE_OK; +} + +/* +** The following functions: +** +** sqlite3VdbeSerialType() +** sqlite3VdbeSerialTypeLen() +** sqlite3VdbeSerialLen() +** sqlite3VdbeSerialPut() +** sqlite3VdbeSerialGet() +** +** encapsulate the code that serializes values for storage in SQLite +** data and index records. Each serialized value consists of a +** 'serial-type' and a blob of data. The serial type is an 8-byte unsigned +** integer, stored as a varint. +** +** In an SQLite index record, the serial type is stored directly before +** the blob of data that it corresponds to. In a table record, all serial +** types are stored at the start of the record, and the blobs of data at +** the end. Hence these functions allow the caller to handle the +** serial-type and data blob separately. +** +** The following table describes the various storage classes for data: +** +** serial type bytes of data type +** -------------- --------------- --------------- +** 0 0 NULL +** 1 1 signed integer +** 2 2 signed integer +** 3 3 signed integer +** 4 4 signed integer +** 5 6 signed integer +** 6 8 signed integer +** 7 8 IEEE float +** 8 0 Integer constant 0 +** 9 0 Integer constant 1 +** 10,11 reserved for expansion +** N>=12 and even (N-12)/2 BLOB +** N>=13 and odd (N-13)/2 text +** +** The 8 and 9 types were added in 3.3.0, file format 4. Prior versions +** of SQLite will not understand those serial types. +*/ + +/* +** Return the serial-type for the value stored in pMem. +*/ +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ + int flags = pMem->flags; + u32 n; + + if( flags&MEM_Null ){ + return 0; + } + if( flags&MEM_Int ){ + /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ +# define MAX_6BYTE ((((i64)0x00008000)<<32)-1) + i64 i = pMem->u.i; + u64 u; + if( i<0 ){ + if( i<(-MAX_6BYTE) ) return 6; + /* Previous test prevents: u = -(-9223372036854775808) */ + u = -i; + }else{ + u = i; + } + if( u<=127 ){ + return ((i&1)==i && file_format>=4) ? 8+(u32)u : 1; + } + if( u<=32767 ) return 2; + if( u<=8388607 ) return 3; + if( u<=2147483647 ) return 4; + if( u<=MAX_6BYTE ) return 5; + return 6; + } + if( flags&MEM_Real ){ + return 7; + } + assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); + assert( pMem->n>=0 ); + n = (u32)pMem->n; + if( flags & MEM_Zero ){ + n += pMem->u.nZero; + } + return ((n*2) + 12 + ((flags&MEM_Str)!=0)); +} + +/* +** Return the length of the data corresponding to the supplied serial-type. +*/ +SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ + if( serial_type>=12 ){ + return (serial_type-12)/2; + }else{ + static const u8 aSize[] = { 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 }; + return aSize[serial_type]; + } +} + +/* +** If we are on an architecture with mixed-endian floating +** points (ex: ARM7) then swap the lower 4 bytes with the +** upper 4 bytes. Return the result. +** +** For most architectures, this is a no-op. +** +** (later): It is reported to me that the mixed-endian problem +** on ARM7 is an issue with GCC, not with the ARM7 chip. It seems +** that early versions of GCC stored the two words of a 64-bit +** float in the wrong order. And that error has been propagated +** ever since. The blame is not necessarily with GCC, though. +** GCC might have just copying the problem from a prior compiler. +** I am also told that newer versions of GCC that follow a different +** ABI get the byte order right. +** +** Developers using SQLite on an ARM7 should compile and run their +** application using -DSQLITE_DEBUG=1 at least once. With DEBUG +** enabled, some asserts below will ensure that the byte order of +** floating point values is correct. +** +** (2007-08-30) Frank van Vugt has studied this problem closely +** and has send his findings to the SQLite developers. Frank +** writes that some Linux kernels offer floating point hardware +** emulation that uses only 32-bit mantissas instead of a full +** 48-bits as required by the IEEE standard. (This is the +** CONFIG_FPE_FASTFPE option.) On such systems, floating point +** byte swapping becomes very complicated. To avoid problems, +** the necessary byte swapping is carried out using a 64-bit integer +** rather than a 64-bit float. Frank assures us that the code here +** works for him. We, the developers, have no way to independently +** verify this, but Frank seems to know what he is talking about +** so we trust him. +*/ +#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT +static u64 floatSwap(u64 in){ + union { + u64 r; + u32 i[2]; + } u; + u32 t; + + u.r = in; + t = u.i[0]; + u.i[0] = u.i[1]; + u.i[1] = t; + return u.r; +} +# define swapMixedEndianFloat(X) X = floatSwap(X) +#else +# define swapMixedEndianFloat(X) +#endif + +/* +** Write the serialized data blob for the value stored in pMem into +** buf. It is assumed that the caller has allocated sufficient space. +** Return the number of bytes written. +** +** nBuf is the amount of space left in buf[]. The caller is responsible +** for allocating enough space to buf[] to hold the entire field, exclusive +** of the pMem->u.nZero bytes for a MEM_Zero value. +** +** Return the number of bytes actually written into buf[]. The number +** of bytes in the zero-filled tail is included in the return value only +** if those bytes were zeroed in buf[]. +*/ +SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ + u32 len; + + /* Integer and Real */ + if( serial_type<=7 && serial_type>0 ){ + u64 v; + u32 i; + if( serial_type==7 ){ + assert( sizeof(v)==sizeof(pMem->r) ); + memcpy(&v, &pMem->r, sizeof(v)); + swapMixedEndianFloat(v); + }else{ + v = pMem->u.i; + } + len = i = sqlite3VdbeSerialTypeLen(serial_type); + while( i-- ){ + buf[i] = (u8)(v&0xFF); + v >>= 8; + } + return len; + } + + /* String or blob */ + if( serial_type>=12 ){ + assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) + == (int)sqlite3VdbeSerialTypeLen(serial_type) ); + len = pMem->n; + memcpy(buf, pMem->z, len); + return len; + } + + /* NULL or constants 0 or 1 */ + return 0; +} + +/* Input "x" is a sequence of unsigned characters that represent a +** big-endian integer. Return the equivalent native integer +*/ +#define ONE_BYTE_INT(x) ((i8)(x)[0]) +#define TWO_BYTE_INT(x) (256*(i8)((x)[0])|(x)[1]) +#define THREE_BYTE_INT(x) (65536*(i8)((x)[0])|((x)[1]<<8)|(x)[2]) +#define FOUR_BYTE_UINT(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) + +/* +** Deserialize the data blob pointed to by buf as serial type serial_type +** and store the result in pMem. Return the number of bytes read. +*/ +SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( + const unsigned char *buf, /* Buffer to deserialize from */ + u32 serial_type, /* Serial type to deserialize */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x; + u32 y; + switch( serial_type ){ + case 10: /* Reserved for future use */ + case 11: /* Reserved for future use */ + case 0: { /* NULL */ + pMem->flags = MEM_Null; + break; + } + case 1: { /* 1-byte signed integer */ + pMem->u.i = ONE_BYTE_INT(buf); + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 1; + } + case 2: { /* 2-byte signed integer */ + pMem->u.i = TWO_BYTE_INT(buf); + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 2; + } + case 3: { /* 3-byte signed integer */ + pMem->u.i = THREE_BYTE_INT(buf); + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 3; + } + case 4: { /* 4-byte signed integer */ + y = FOUR_BYTE_UINT(buf); + pMem->u.i = (i64)*(int*)&y; + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 4; + } + case 5: { /* 6-byte signed integer */ + pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 6; + } + case 6: /* 8-byte signed integer */ + case 7: { /* IEEE floating point */ +#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) + /* Verify that integers and floating point values use the same + ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is + ** defined that 64-bit floating point values really are mixed + ** endian. + */ + static const u64 t1 = ((u64)0x3ff00000)<<32; + static const double r1 = 1.0; + u64 t2 = t1; + swapMixedEndianFloat(t2); + assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); +#endif + x = FOUR_BYTE_UINT(buf); + y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) | y; + if( serial_type==6 ){ + pMem->u.i = *(i64*)&x; + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + }else{ + assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->r, &x, sizeof(x)); + pMem->flags = sqlite3IsNaN(pMem->r) ? MEM_Null : MEM_Real; + } + return 8; + } + case 8: /* Integer 0 */ + case 9: { /* Integer 1 */ + pMem->u.i = serial_type-8; + pMem->flags = MEM_Int; + return 0; + } + default: { + static const u16 aFlag[] = { MEM_Blob|MEM_Ephem, MEM_Str|MEM_Ephem }; + u32 len = (serial_type-12)/2; + pMem->z = (char *)buf; + pMem->n = len; + pMem->xDel = 0; + pMem->flags = aFlag[serial_type&1]; + return len; + } + } + return 0; +} + +/* +** This routine is used to allocate sufficient space for an UnpackedRecord +** structure large enough to be used with sqlite3VdbeRecordUnpack() if +** the first argument is a pointer to KeyInfo structure pKeyInfo. +** +** The space is either allocated using sqlite3DbMallocRaw() or from within +** the unaligned buffer passed via the second and third arguments (presumably +** stack space). If the former, then *ppFree is set to a pointer that should +** be eventually freed by the caller using sqlite3DbFree(). Or, if the +** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL +** before returning. +** +** If an OOM error occurs, NULL is returned. +*/ +SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( + KeyInfo *pKeyInfo, /* Description of the record */ + char *pSpace, /* Unaligned space available */ + int szSpace, /* Size of pSpace[] in bytes */ + char **ppFree /* OUT: Caller should free this pointer */ +){ + UnpackedRecord *p; /* Unpacked record to return */ + int nOff; /* Increment pSpace by nOff to align it */ + int nByte; /* Number of bytes required for *p */ + + /* We want to shift the pointer pSpace up such that it is 8-byte aligned. + ** Thus, we need to calculate a value, nOff, between 0 and 7, to shift + ** it by. If pSpace is already 8-byte aligned, nOff should be zero. + */ + nOff = (8 - (SQLITE_PTR_TO_INT(pSpace) & 7)) & 7; + nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nField+1); + if( nByte>szSpace+nOff ){ + p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); + *ppFree = (char *)p; + if( !p ) return 0; + }else{ + p = (UnpackedRecord*)&pSpace[nOff]; + *ppFree = 0; + } + + p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; + assert( pKeyInfo->aSortOrder!=0 ); + p->pKeyInfo = pKeyInfo; + p->nField = pKeyInfo->nField + 1; + return p; +} + +/* +** Given the nKey-byte encoding of a record in pKey[], populate the +** UnpackedRecord structure indicated by the fourth argument with the +** contents of the decoded record. +*/ +SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( + KeyInfo *pKeyInfo, /* Information about the record format */ + int nKey, /* Size of the binary record */ + const void *pKey, /* The binary record */ + UnpackedRecord *p /* Populate this structure before returning. */ +){ + const unsigned char *aKey = (const unsigned char *)pKey; + int d; + u32 idx; /* Offset in aKey[] to read from */ + u16 u; /* Unsigned loop counter */ + u32 szHdr; + Mem *pMem = p->aMem; + + p->default_rc = 0; + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + idx = getVarint32(aKey, szHdr); + d = szHdr; + u = 0; + while( idxnField && d<=nKey ){ + u32 serial_type; + + idx += getVarint32(&aKey[idx], serial_type); + pMem->enc = pKeyInfo->enc; + pMem->db = pKeyInfo->db; + /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ + pMem->zMalloc = 0; + d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); + pMem++; + u++; + } + assert( u<=pKeyInfo->nField + 1 ); + p->nField = u; +} + +#if SQLITE_DEBUG +/* +** This function compares two index or table record keys in the same way +** as the sqlite3VdbeRecordCompare() routine. Unlike VdbeRecordCompare(), +** this function deserializes and compares values using the +** sqlite3VdbeSerialGet() and sqlite3MemCompare() functions. It is used +** in assert() statements to ensure that the optimized code in +** sqlite3VdbeRecordCompare() returns results with these two primitives. +*/ +static int vdbeRecordCompareDebug( + int nKey1, const void *pKey1, /* Left key */ + const UnpackedRecord *pPKey2 /* Right key */ +){ + u32 d1; /* Offset into aKey[] of next data element */ + u32 idx1; /* Offset into aKey[] of next header element */ + u32 szHdr1; /* Number of bytes in header */ + int i = 0; + int rc = 0; + const unsigned char *aKey1 = (const unsigned char *)pKey1; + KeyInfo *pKeyInfo; + Mem mem1; + + pKeyInfo = pPKey2->pKeyInfo; + mem1.enc = pKeyInfo->enc; + mem1.db = pKeyInfo->db; + /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ + VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + + /* Compilers may complain that mem1.u.i is potentially uninitialized. + ** We could initialize it, as shown here, to silence those complaints. + ** But in fact, mem1.u.i will never actually be used uninitialized, and doing + ** the unnecessary initialization has a measurable negative performance + ** impact, since this routine is a very high runner. And so, we choose + ** to ignore the compiler warnings and leave this variable uninitialized. + */ + /* mem1.u.i = 0; // not needed, here to silence compiler warning */ + + idx1 = getVarint32(aKey1, szHdr1); + d1 = szHdr1; + assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); + assert( pKeyInfo->aSortOrder!=0 ); + assert( pKeyInfo->nField>0 ); + assert( idx1<=szHdr1 || CORRUPT_DB ); + do{ + u32 serial_type1; + + /* Read the serial types for the next element in each key. */ + idx1 += getVarint32( aKey1+idx1, serial_type1 ); + + /* Verify that there is enough key space remaining to avoid + ** a buffer overread. The "d1+serial_type1+2" subexpression will + ** always be greater than or equal to the amount of required key space. + ** Use that approximation to avoid the more expensive call to + ** sqlite3VdbeSerialTypeLen() in the common case. + */ + if( d1+serial_type1+2>(u32)nKey1 + && d1+sqlite3VdbeSerialTypeLen(serial_type1)>(u32)nKey1 + ){ + break; + } + + /* Extract the values to be compared. + */ + d1 += sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); + + /* Do the comparison + */ + rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], pKeyInfo->aColl[i]); + if( rc!=0 ){ + assert( mem1.zMalloc==0 ); /* See comment below */ + if( pKeyInfo->aSortOrder[i] ){ + rc = -rc; /* Invert the result for DESC sort order. */ + } + return rc; + } + i++; + }while( idx1nField ); + + /* No memory allocation is ever used on mem1. Prove this using + ** the following assert(). If the assert() fails, it indicates a + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). + */ + assert( mem1.zMalloc==0 ); + + /* rc==0 here means that one of the keys ran out of fields and + ** all the fields up to that point were equal. Return the the default_rc + ** value. */ + return pPKey2->default_rc; +} +#endif + +/* +** Both *pMem1 and *pMem2 contain string values. Compare the two values +** using the collation sequence pColl. As usual, return a negative , zero +** or positive value if *pMem1 is less than, equal to or greater than +** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". +*/ +static int vdbeCompareMemString( + const Mem *pMem1, + const Mem *pMem2, + const CollSeq *pColl +){ + if( pMem1->enc==pColl->enc ){ + /* The strings are already in the correct encoding. Call the + ** comparison function directly */ + return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); + }else{ + int rc; + const void *v1, *v2; + int n1, n2; + Mem c1; + Mem c2; + memset(&c1, 0, sizeof(c1)); + memset(&c2, 0, sizeof(c2)); + sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); + sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); + v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); + n1 = v1==0 ? 0 : c1.n; + v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); + n2 = v2==0 ? 0 : c2.n; + rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); + sqlite3VdbeMemRelease(&c1); + sqlite3VdbeMemRelease(&c2); + return rc; + } +} + +/* +** Compare the values contained by the two memory cells, returning +** negative, zero or positive if pMem1 is less than, equal to, or greater +** than pMem2. Sorting order is NULL's first, followed by numbers (integers +** and reals) sorted numerically, followed by text ordered by the collating +** sequence pColl and finally blob's ordered by memcmp(). +** +** Two NULL values are considered equal by this function. +*/ +SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ + int rc; + int f1, f2; + int combined_flags; + + f1 = pMem1->flags; + f2 = pMem2->flags; + combined_flags = f1|f2; + assert( (combined_flags & MEM_RowSet)==0 ); + + /* If one value is NULL, it is less than the other. If both values + ** are NULL, return 0. + */ + if( combined_flags&MEM_Null ){ + return (f2&MEM_Null) - (f1&MEM_Null); + } + + /* If one value is a number and the other is not, the number is less. + ** If both are numbers, compare as reals if one is a real, or as integers + ** if both values are integers. + */ + if( combined_flags&(MEM_Int|MEM_Real) ){ + double r1, r2; + if( (f1 & f2 & MEM_Int)!=0 ){ + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return 1; + return 0; + } + if( (f1&MEM_Real)!=0 ){ + r1 = pMem1->r; + }else if( (f1&MEM_Int)!=0 ){ + r1 = (double)pMem1->u.i; + }else{ + return 1; + } + if( (f2&MEM_Real)!=0 ){ + r2 = pMem2->r; + }else if( (f2&MEM_Int)!=0 ){ + r2 = (double)pMem2->u.i; + }else{ + return -1; + } + if( r1r2 ) return 1; + return 0; + } + + /* If one value is a string and the other is a blob, the string is less. + ** If both are strings, compare using the collating functions. + */ + if( combined_flags&MEM_Str ){ + if( (f1 & MEM_Str)==0 ){ + return 1; + } + if( (f2 & MEM_Str)==0 ){ + return -1; + } + + assert( pMem1->enc==pMem2->enc ); + assert( pMem1->enc==SQLITE_UTF8 || + pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); + + /* The collation sequence must be defined at this point, even if + ** the user deletes the collation sequence after the vdbe program is + ** compiled (this was not always the case). + */ + assert( !pColl || pColl->xCmp ); + + if( pColl ){ + return vdbeCompareMemString(pMem1, pMem2, pColl); + } + /* If a NULL pointer was passed as the collate function, fall through + ** to the blob case and use memcmp(). */ + } + + /* Both values must be blobs. Compare using memcmp(). */ + rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); + if( rc==0 ){ + rc = pMem1->n - pMem2->n; + } + return rc; +} + + +/* +** The first argument passed to this function is a serial-type that +** corresponds to an integer - all values between 1 and 9 inclusive +** except 7. The second points to a buffer containing an integer value +** serialized according to serial_type. This function deserializes +** and returns the value. +*/ +static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ + u32 y; + assert( CORRUPT_DB || (serial_type>=1 && serial_type<=9 && serial_type!=7) ); + switch( serial_type ){ + case 0: + case 1: + testcase( aKey[0]&0x80 ); + return ONE_BYTE_INT(aKey); + case 2: + testcase( aKey[0]&0x80 ); + return TWO_BYTE_INT(aKey); + case 3: + testcase( aKey[0]&0x80 ); + return THREE_BYTE_INT(aKey); + case 4: { + testcase( aKey[0]&0x80 ); + y = FOUR_BYTE_UINT(aKey); + return (i64)*(int*)&y; + } + case 5: { + testcase( aKey[0]&0x80 ); + return FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); + } + case 6: { + u64 x = FOUR_BYTE_UINT(aKey); + testcase( aKey[0]&0x80 ); + x = (x<<32) | FOUR_BYTE_UINT(aKey+4); + return (i64)*(i64*)&x; + } + } + + return (serial_type - 8); +} + +/* +** This function compares the two table rows or index records +** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero +** or positive integer if key1 is less than, equal to or +** greater than key2. The {nKey1, pKey1} key must be a blob +** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 +** key must be a parsed key such as obtained from +** sqlite3VdbeParseRecord. +** +** If argument bSkip is non-zero, it is assumed that the caller has already +** determined that the first fields of the keys are equal. +** +** Key1 and Key2 do not have to contain the same number of fields. If all +** fields that appear in both keys are equal, then pPKey2->default_rc is +** returned. +** +** If database corruption is discovered, set pPKey2->isCorrupt to non-zero +** and return 0. +*/ +SQLITE_PRIVATE int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2, /* Right key */ + int bSkip /* If true, skip the first field */ +){ + u32 d1; /* Offset into aKey[] of next data element */ + int i; /* Index of next field to compare */ + u32 szHdr1; /* Size of record header in bytes */ + u32 idx1; /* Offset of first type in header */ + int rc = 0; /* Return value */ + Mem *pRhs = pPKey2->aMem; /* Next field of pPKey2 to compare */ + KeyInfo *pKeyInfo = pPKey2->pKeyInfo; + const unsigned char *aKey1 = (const unsigned char *)pKey1; + Mem mem1; + + /* If bSkip is true, then the caller has already determined that the first + ** two elements in the keys are equal. Fix the various stack variables so + ** that this routine begins comparing at the second field. */ + if( bSkip ){ + u32 s1; + idx1 = 1 + getVarint32(&aKey1[1], s1); + szHdr1 = aKey1[0]; + d1 = szHdr1 + sqlite3VdbeSerialTypeLen(s1); + i = 1; + pRhs++; + }else{ + idx1 = getVarint32(aKey1, szHdr1); + d1 = szHdr1; + if( d1>(unsigned)nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } + i = 0; + } + + VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField + || CORRUPT_DB ); + assert( pPKey2->pKeyInfo->aSortOrder!=0 ); + assert( pPKey2->pKeyInfo->nField>0 ); + assert( idx1<=szHdr1 || CORRUPT_DB ); + do{ + u32 serial_type; + + /* RHS is an integer */ + if( pRhs->flags & MEM_Int ){ + serial_type = aKey1[idx1]; + testcase( serial_type==12 ); + if( serial_type>=12 ){ + rc = +1; + }else if( serial_type==0 ){ + rc = -1; + }else if( serial_type==7 ){ + double rhs = (double)pRhs->u.i; + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + if( mem1.rrhs ){ + rc = +1; + } + }else{ + i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); + i64 rhs = pRhs->u.i; + if( lhsrhs ){ + rc = +1; + } + } + } + + /* RHS is real */ + else if( pRhs->flags & MEM_Real ){ + serial_type = aKey1[idx1]; + if( serial_type>=12 ){ + rc = +1; + }else if( serial_type==0 ){ + rc = -1; + }else{ + double rhs = pRhs->r; + double lhs; + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + if( serial_type==7 ){ + lhs = mem1.r; + }else{ + lhs = (double)mem1.u.i; + } + if( lhsrhs ){ + rc = +1; + } + } + } + + /* RHS is a string */ + else if( pRhs->flags & MEM_Str ){ + getVarint32(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); + if( serial_type<12 ){ + rc = -1; + }else if( !(serial_type & 0x01) ){ + rc = +1; + }else{ + mem1.n = (serial_type - 12) / 2; + testcase( (d1+mem1.n)==(unsigned)nKey1 ); + testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); + if( (d1+mem1.n) > (unsigned)nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + }else if( pKeyInfo->aColl[i] ){ + mem1.enc = pKeyInfo->enc; + mem1.db = pKeyInfo->db; + mem1.flags = MEM_Str; + mem1.z = (char*)&aKey1[d1]; + rc = vdbeCompareMemString(&mem1, pRhs, pKeyInfo->aColl[i]); + }else{ + int nCmp = MIN(mem1.n, pRhs->n); + rc = memcmp(&aKey1[d1], pRhs->z, nCmp); + if( rc==0 ) rc = mem1.n - pRhs->n; + } + } + } + + /* RHS is a blob */ + else if( pRhs->flags & MEM_Blob ){ + getVarint32(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); + if( serial_type<12 || (serial_type & 0x01) ){ + rc = -1; + }else{ + int nStr = (serial_type - 12) / 2; + testcase( (d1+nStr)==(unsigned)nKey1 ); + testcase( (d1+nStr+1)==(unsigned)nKey1 ); + if( (d1+nStr) > (unsigned)nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + }else{ + int nCmp = MIN(nStr, pRhs->n); + rc = memcmp(&aKey1[d1], pRhs->z, nCmp); + if( rc==0 ) rc = nStr - pRhs->n; + } + } + } + + /* RHS is null */ + else{ + serial_type = aKey1[idx1]; + rc = (serial_type!=0); + } + + if( rc!=0 ){ + if( pKeyInfo->aSortOrder[i] ){ + rc = -rc; + } + assert( CORRUPT_DB + || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) + || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + || pKeyInfo->db->mallocFailed + ); + assert( mem1.zMalloc==0 ); /* See comment below */ + return rc; + } + + i++; + pRhs++; + d1 += sqlite3VdbeSerialTypeLen(serial_type); + idx1 += sqlite3VarintLen(serial_type); + }while( idx1<(unsigned)szHdr1 && inField && d1<=(unsigned)nKey1 ); + + /* No memory allocation is ever used on mem1. Prove this using + ** the following assert(). If the assert() fails, it indicates a + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ + assert( mem1.zMalloc==0 ); + + /* rc==0 here means that one or both of the keys ran out of fields and + ** all the fields up to that point were equal. Return the the default_rc + ** value. */ + assert( CORRUPT_DB + || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) + || pKeyInfo->db->mallocFailed + ); + return pPKey2->default_rc; +} + +/* +** This function is an optimized version of sqlite3VdbeRecordCompare() +** that (a) the first field of pPKey2 is an integer, and (b) the +** size-of-header varint at the start of (pKey1/nKey1) fits in a single +** byte (i.e. is less than 128). +** +** To avoid concerns about buffer overreads, this routine is only used +** on schemas where the maximum valid header size is 63 bytes or less. +*/ +static int vdbeRecordCompareInt( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2, /* Right key */ + int bSkip /* Ignored */ +){ + const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; + int serial_type = ((const u8*)pKey1)[1]; + int res; + u32 y; + u64 x; + i64 v = pPKey2->aMem[0].u.i; + i64 lhs; + UNUSED_PARAMETER(bSkip); + + assert( bSkip==0 ); + assert( (*(u8*)pKey1)<=0x3F || CORRUPT_DB ); + switch( serial_type ){ + case 1: { /* 1-byte signed integer */ + lhs = ONE_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 2: { /* 2-byte signed integer */ + lhs = TWO_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 3: { /* 3-byte signed integer */ + lhs = THREE_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 4: { /* 4-byte signed integer */ + y = FOUR_BYTE_UINT(aKey); + lhs = (i64)*(int*)&y; + testcase( lhs<0 ); + break; + } + case 5: { /* 6-byte signed integer */ + lhs = FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 6: { /* 8-byte signed integer */ + x = FOUR_BYTE_UINT(aKey); + x = (x<<32) | FOUR_BYTE_UINT(aKey+4); + lhs = *(i64*)&x; + testcase( lhs<0 ); + break; + } + case 8: + lhs = 0; + break; + case 9: + lhs = 1; + break; + + /* This case could be removed without changing the results of running + ** this code. Including it causes gcc to generate a faster switch + ** statement (since the range of switch targets now starts at zero and + ** is contiguous) but does not cause any duplicate code to be generated + ** (as gcc is clever enough to combine the two like cases). Other + ** compilers might be similar. */ + case 0: case 7: + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + + default: + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + } + + if( v>lhs ){ + res = pPKey2->r1; + }else if( vr2; + }else if( pPKey2->nField>1 ){ + /* The first fields of the two keys are equal. Compare the trailing + ** fields. */ + res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + }else{ + /* The first fields of the two keys are equal and there are no trailing + ** fields. Return pPKey2->default_rc in this case. */ + res = pPKey2->default_rc; + } + + assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) + || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + || CORRUPT_DB + ); + return res; +} + +/* +** This function is an optimized version of sqlite3VdbeRecordCompare() +** that (a) the first field of pPKey2 is a string, that (b) the first field +** uses the collation sequence BINARY and (c) that the size-of-header varint +** at the start of (pKey1/nKey1) fits in a single byte. +*/ +static int vdbeRecordCompareString( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2, /* Right key */ + int bSkip +){ + const u8 *aKey1 = (const u8*)pKey1; + int serial_type; + int res; + UNUSED_PARAMETER(bSkip); + + assert( bSkip==0 ); + getVarint32(&aKey1[1], serial_type); + + if( serial_type<12 ){ + res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ + }else if( !(serial_type & 0x01) ){ + res = pPKey2->r2; /* (pKey1/nKey1) is a blob */ + }else{ + int nCmp; + int nStr; + int szHdr = aKey1[0]; + + nStr = (serial_type-12) / 2; + if( (szHdr + nStr) > nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } + nCmp = MIN( pPKey2->aMem[0].n, nStr ); + res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); + + if( res==0 ){ + res = nStr - pPKey2->aMem[0].n; + if( res==0 ){ + if( pPKey2->nField>1 ){ + res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + }else{ + res = pPKey2->default_rc; + } + }else if( res>0 ){ + res = pPKey2->r2; + }else{ + res = pPKey2->r1; + } + }else if( res>0 ){ + res = pPKey2->r2; + }else{ + res = pPKey2->r1; + } + } + + assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) + || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + || CORRUPT_DB + || pPKey2->pKeyInfo->db->mallocFailed + ); + return res; +} + +/* +** Return a pointer to an sqlite3VdbeRecordCompare() compatible function +** suitable for comparing serialized records to the unpacked record passed +** as the only argument. +*/ +SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ + /* varintRecordCompareInt() and varintRecordCompareString() both assume + ** that the size-of-header varint that occurs at the start of each record + ** fits in a single byte (i.e. is 127 or less). varintRecordCompareInt() + ** also assumes that it is safe to overread a buffer by at least the + ** maximum possible legal header size plus 8 bytes. Because there is + ** guaranteed to be at least 74 (but not 136) bytes of padding following each + ** buffer passed to varintRecordCompareInt() this makes it convenient to + ** limit the size of the header to 64 bytes in cases where the first field + ** is an integer. + ** + ** The easiest way to enforce this limit is to consider only records with + ** 13 fields or less. If the first field is an integer, the maximum legal + ** header size is (12*5 + 1 + 1) bytes. */ + if( (p->pKeyInfo->nField + p->pKeyInfo->nXField)<=13 ){ + int flags = p->aMem[0].flags; + if( p->pKeyInfo->aSortOrder[0] ){ + p->r1 = 1; + p->r2 = -1; + }else{ + p->r1 = -1; + p->r2 = 1; + } + if( (flags & MEM_Int) ){ + return vdbeRecordCompareInt; + } + testcase( flags & MEM_Real ); + testcase( flags & MEM_Null ); + testcase( flags & MEM_Blob ); + if( (flags & (MEM_Real|MEM_Null|MEM_Blob))==0 && p->pKeyInfo->aColl[0]==0 ){ + assert( flags & MEM_Str ); + return vdbeRecordCompareString; + } + } + + return sqlite3VdbeRecordCompare; +} + +/* +** pCur points at an index entry created using the OP_MakeRecord opcode. +** Read the rowid (the last field in the record) and store it in *rowid. +** Return SQLITE_OK if everything works, or an error code otherwise. +** +** pCur might be pointing to text obtained from a corrupt database file. +** So the content cannot be trusted. Do appropriate checks on the content. +*/ +SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ + i64 nCellKey = 0; + int rc; + u32 szHdr; /* Size of the header */ + u32 typeRowid; /* Serial type of the rowid */ + u32 lenRowid; /* Size of the rowid */ + Mem m, v; + + UNUSED_PARAMETER(db); + + /* Get the size of the index entry. Only indices entries of less + ** than 2GiB are support - anything large must be database corruption. + ** Any corruption is detected in sqlite3BtreeParseCellPtr(), though, so + ** this code can safely assume that nCellKey is 32-bits + */ + assert( sqlite3BtreeCursorIsValid(pCur) ); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); + assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ + assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); + + /* Read in the complete content of the index entry */ + memset(&m, 0, sizeof(m)); + rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m); + if( rc ){ + return rc; + } + + /* The index entry must begin with a header size */ + (void)getVarint32((u8*)m.z, szHdr); + testcase( szHdr==3 ); + testcase( szHdr==m.n ); + if( unlikely(szHdr<3 || (int)szHdr>m.n) ){ + goto idx_rowid_corruption; + } + + /* The last field of the index should be an integer - the ROWID. + ** Verify that the last entry really is an integer. */ + (void)getVarint32((u8*)&m.z[szHdr-1], typeRowid); + testcase( typeRowid==1 ); + testcase( typeRowid==2 ); + testcase( typeRowid==3 ); + testcase( typeRowid==4 ); + testcase( typeRowid==5 ); + testcase( typeRowid==6 ); + testcase( typeRowid==8 ); + testcase( typeRowid==9 ); + if( unlikely(typeRowid<1 || typeRowid>9 || typeRowid==7) ){ + goto idx_rowid_corruption; + } + lenRowid = sqlite3VdbeSerialTypeLen(typeRowid); + testcase( (u32)m.n==szHdr+lenRowid ); + if( unlikely((u32)m.npCursor; + Mem m; + + assert( sqlite3BtreeCursorIsValid(pCur) ); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); + assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ + /* nCellKey will always be between 0 and 0xffffffff because of the way + ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ + if( nCellKey<=0 || nCellKey>0x7fffffff ){ + *res = 0; + return SQLITE_CORRUPT_BKPT; + } + memset(&m, 0, sizeof(m)); + rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (u32)nCellKey, 1, &m); + if( rc ){ + return rc; + } + *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked, 0); + sqlite3VdbeMemRelease(&m); + return SQLITE_OK; +} + +/* +** This routine sets the value to be returned by subsequent calls to +** sqlite3_changes() on the database handle 'db'. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *db, int nChange){ + assert( sqlite3_mutex_held(db->mutex) ); + db->nChange = nChange; + db->nTotalChange += nChange; +} + +/* +** Set a flag in the vdbe to update the change counter when it is finalised +** or reset. +*/ +SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ + v->changeCntOn = 1; +} + +/* +** Mark every prepared statement associated with a database connection +** as expired. +** +** An expired statement means that recompilation of the statement is +** recommend. Statements expire when things happen that make their +** programs obsolete. Removing user-defined functions or collating +** sequences, or changing an authorization function are the types of +** things that make prepared statements obsolete. +*/ +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db){ + Vdbe *p; + for(p = db->pVdbe; p; p=p->pNext){ + p->expired = 1; + } +} + +/* +** Return the database associated with the Vdbe. +*/ +SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe *v){ + return v->db; +} + +/* +** Return a pointer to an sqlite3_value structure containing the value bound +** parameter iVar of VM v. Except, if the value is an SQL NULL, return +** 0 instead. Unless it is NULL, apply affinity aff (one of the SQLITE_AFF_* +** constants) to the value before returning it. +** +** The returned value must be freed by the caller using sqlite3ValueFree(). +*/ +SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){ + assert( iVar>0 ); + if( v ){ + Mem *pMem = &v->aVar[iVar-1]; + if( 0==(pMem->flags & MEM_Null) ){ + sqlite3_value *pRet = sqlite3ValueNew(v->db); + if( pRet ){ + sqlite3VdbeMemCopy((Mem *)pRet, pMem); + sqlite3ValueApplyAffinity(pRet, aff, SQLITE_UTF8); + } + return pRet; + } + } + return 0; +} + +/* +** Configure SQL variable iVar so that binding a new value to it signals +** to sqlite3_reoptimize() that re-preparing the statement may result +** in a better query plan. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ + assert( iVar>0 ); + if( iVar>32 ){ + v->expmask = 0xffffffff; + }else{ + v->expmask |= ((u32)1 << (iVar-1)); + } +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored +** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored +** in memory obtained from sqlite3DbMalloc). +*/ +SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/************** End of vdbeaux.c *********************************************/ +/************** Begin file vdbeapi.c *****************************************/ +/* +** 2004 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code use to implement APIs that are part of the +** VDBE. +*/ + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Return TRUE (non-zero) of the statement supplied as an argument needs +** to be recompiled. A statement needs to be recompiled whenever the +** execution environment changes in a way that would alter the program +** that sqlite3_prepare() generates. For example, if new functions or +** collating sequences are registered or if an authorizer function is +** added or changed. +*/ +SQLITE_API int sqlite3_expired(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + return p==0 || p->expired; +} +#endif + +/* +** Check on a Vdbe to make sure it has not been finalized. Log +** an error and return true if it has been finalized (or is otherwise +** invalid). Return false if it is ok. +*/ +static int vdbeSafety(Vdbe *p){ + if( p->db==0 ){ + sqlite3_log(SQLITE_MISUSE, "API called with finalized prepared statement"); + return 1; + }else{ + return 0; + } +} +static int vdbeSafetyNotNull(Vdbe *p){ + if( p==0 ){ + sqlite3_log(SQLITE_MISUSE, "API called with NULL prepared statement"); + return 1; + }else{ + return vdbeSafety(p); + } +} + +/* +** The following routine destroys a virtual machine that is created by +** the sqlite3_compile() routine. The integer returned is an SQLITE_ +** success/failure code that describes the result of executing the virtual +** machine. +** +** This routine sets the error code and string returned by +** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). +*/ +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ + int rc; + if( pStmt==0 ){ + /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL + ** pointer is a harmless no-op. */ + rc = SQLITE_OK; + }else{ + Vdbe *v = (Vdbe*)pStmt; + sqlite3 *db = v->db; + if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; + sqlite3_mutex_enter(db->mutex); + rc = sqlite3VdbeFinalize(v); + rc = sqlite3ApiExit(db, rc); + sqlite3LeaveMutexAndCloseZombie(db); + } + return rc; +} + +/* +** Terminate the current execution of an SQL statement and reset it +** back to its starting state so that it can be reused. A success code from +** the prior execution is returned. +** +** This routine sets the error code and string returned by +** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). +*/ +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ + int rc; + if( pStmt==0 ){ + rc = SQLITE_OK; + }else{ + Vdbe *v = (Vdbe*)pStmt; + sqlite3_mutex_enter(v->db->mutex); + rc = sqlite3VdbeReset(v); + sqlite3VdbeRewind(v); + assert( (rc & (v->db->errMask))==rc ); + rc = sqlite3ApiExit(v->db, rc); + sqlite3_mutex_leave(v->db->mutex); + } + return rc; +} + +/* +** Set all the parameters in the compiled SQL statement to NULL. +*/ +SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ + int i; + int rc = SQLITE_OK; + Vdbe *p = (Vdbe*)pStmt; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = ((Vdbe*)pStmt)->db->mutex; +#endif + sqlite3_mutex_enter(mutex); + for(i=0; inVar; i++){ + sqlite3VdbeMemRelease(&p->aVar[i]); + p->aVar[i].flags = MEM_Null; + } + if( p->isPrepareV2 && p->expmask ){ + p->expired = 1; + } + sqlite3_mutex_leave(mutex); + return rc; +} + + +/**************************** sqlite3_value_ ******************************* +** The following routines extract information from a Mem or sqlite3_value +** structure. +*/ +SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ + Mem *p = (Mem*)pVal; + if( p->flags & (MEM_Blob|MEM_Str) ){ + sqlite3VdbeMemExpandBlob(p); + p->flags |= MEM_Blob; + return p->n ? p->z : 0; + }else{ + return sqlite3_value_text(pVal); + } +} +SQLITE_API int sqlite3_value_bytes(sqlite3_value *pVal){ + return sqlite3ValueBytes(pVal, SQLITE_UTF8); +} +SQLITE_API int sqlite3_value_bytes16(sqlite3_value *pVal){ + return sqlite3ValueBytes(pVal, SQLITE_UTF16NATIVE); +} +SQLITE_API double sqlite3_value_double(sqlite3_value *pVal){ + return sqlite3VdbeRealValue((Mem*)pVal); +} +SQLITE_API int sqlite3_value_int(sqlite3_value *pVal){ + return (int)sqlite3VdbeIntValue((Mem*)pVal); +} +SQLITE_API sqlite_int64 sqlite3_value_int64(sqlite3_value *pVal){ + return sqlite3VdbeIntValue((Mem*)pVal); +} +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value *pVal){ + return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_value_text16(sqlite3_value* pVal){ + return sqlite3ValueText(pVal, SQLITE_UTF16NATIVE); +} +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value *pVal){ + return sqlite3ValueText(pVal, SQLITE_UTF16BE); +} +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ + return sqlite3ValueText(pVal, SQLITE_UTF16LE); +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ + static const u8 aType[] = { + SQLITE_BLOB, /* 0x00 */ + SQLITE_NULL, /* 0x01 */ + SQLITE_TEXT, /* 0x02 */ + SQLITE_NULL, /* 0x03 */ + SQLITE_INTEGER, /* 0x04 */ + SQLITE_NULL, /* 0x05 */ + SQLITE_INTEGER, /* 0x06 */ + SQLITE_NULL, /* 0x07 */ + SQLITE_FLOAT, /* 0x08 */ + SQLITE_NULL, /* 0x09 */ + SQLITE_FLOAT, /* 0x0a */ + SQLITE_NULL, /* 0x0b */ + SQLITE_INTEGER, /* 0x0c */ + SQLITE_NULL, /* 0x0d */ + SQLITE_INTEGER, /* 0x0e */ + SQLITE_NULL, /* 0x0f */ + SQLITE_BLOB, /* 0x10 */ + SQLITE_NULL, /* 0x11 */ + SQLITE_TEXT, /* 0x12 */ + SQLITE_NULL, /* 0x13 */ + SQLITE_INTEGER, /* 0x14 */ + SQLITE_NULL, /* 0x15 */ + SQLITE_INTEGER, /* 0x16 */ + SQLITE_NULL, /* 0x17 */ + SQLITE_FLOAT, /* 0x18 */ + SQLITE_NULL, /* 0x19 */ + SQLITE_FLOAT, /* 0x1a */ + SQLITE_NULL, /* 0x1b */ + SQLITE_INTEGER, /* 0x1c */ + SQLITE_NULL, /* 0x1d */ + SQLITE_INTEGER, /* 0x1e */ + SQLITE_NULL, /* 0x1f */ + }; + return aType[pVal->flags&MEM_AffMask]; +} + +/**************************** sqlite3_result_ ******************************* +** The following routines are used by user-defined functions to specify +** the function result. +** +** The setStrOrError() funtion calls sqlite3VdbeMemSetStr() to store the +** result as a string or blob but if the string or blob is too large, it +** then sets the error code to SQLITE_TOOBIG +*/ +static void setResultStrOrError( + sqlite3_context *pCtx, /* Function context */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*) /* Destructor function */ +){ + if( sqlite3VdbeMemSetStr(&pCtx->s, z, n, enc, xDel)==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(pCtx); + } +} +SQLITE_API void sqlite3_result_blob( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( n>=0 ); + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, 0, xDel); +} +SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetDouble(&pCtx->s, rVal); +} +SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + pCtx->isError = SQLITE_ERROR; + pCtx->fErrorOrAux = 1; + sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + pCtx->isError = SQLITE_ERROR; + pCtx->fErrorOrAux = 1; + sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); +} +#endif +SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetInt64(&pCtx->s, (i64)iVal); +} +SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetInt64(&pCtx->s, iVal); +} +SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetNull(&pCtx->s); +} +SQLITE_API void sqlite3_result_text( + sqlite3_context *pCtx, + const char *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API void sqlite3_result_text16( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); +} +SQLITE_API void sqlite3_result_text16be( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); +} +SQLITE_API void sqlite3_result_text16le( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemCopy(&pCtx->s, pValue); +} +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetZeroBlob(&pCtx->s, n); +} +SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ + pCtx->isError = errCode; + pCtx->fErrorOrAux = 1; + if( pCtx->s.flags & MEM_Null ){ + sqlite3VdbeMemSetStr(&pCtx->s, sqlite3ErrStr(errCode), -1, + SQLITE_UTF8, SQLITE_STATIC); + } +} + +/* Force an SQLITE_TOOBIG error. */ +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + pCtx->isError = SQLITE_TOOBIG; + pCtx->fErrorOrAux = 1; + sqlite3VdbeMemSetStr(&pCtx->s, "string or blob too big", -1, + SQLITE_UTF8, SQLITE_STATIC); +} + +/* An SQLITE_NOMEM error. */ +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetNull(&pCtx->s); + pCtx->isError = SQLITE_NOMEM; + pCtx->fErrorOrAux = 1; + pCtx->s.db->mallocFailed = 1; +} + +/* +** This function is called after a transaction has been committed. It +** invokes callbacks registered with sqlite3_wal_hook() as required. +*/ +static int doWalCallbacks(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_WAL + int i; + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + int nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ + rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry); + } + } + } +#endif + return rc; +} + +/* +** Execute the statement pStmt, either until a row of data is ready, the +** statement is completely executed or an error occurs. +** +** This routine implements the bulk of the logic behind the sqlite_step() +** API. The only thing omitted is the automatic recompile if a +** schema change has occurred. That detail is handled by the +** outer sqlite3_step() wrapper procedure. +*/ +static int sqlite3Step(Vdbe *p){ + sqlite3 *db; + int rc; + + assert(p); + if( p->magic!=VDBE_MAGIC_RUN ){ + /* We used to require that sqlite3_reset() be called before retrying + ** sqlite3_step() after any error or after SQLITE_DONE. But beginning + ** with version 3.7.0, we changed this so that sqlite3_reset() would + ** be called automatically instead of throwing the SQLITE_MISUSE error. + ** This "automatic-reset" change is not technically an incompatibility, + ** since any application that receives an SQLITE_MISUSE is broken by + ** definition. + ** + ** Nevertheless, some published applications that were originally written + ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE + ** returns, and those were broken by the automatic-reset change. As a + ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the + ** legacy behavior of returning SQLITE_MISUSE for cases where the + ** previous sqlite3_step() returned something other than a SQLITE_LOCKED + ** or SQLITE_BUSY error. + */ +#ifdef SQLITE_OMIT_AUTORESET + if( p->rc==SQLITE_BUSY || p->rc==SQLITE_LOCKED ){ + sqlite3_reset((sqlite3_stmt*)p); + }else{ + return SQLITE_MISUSE_BKPT; + } +#else + sqlite3_reset((sqlite3_stmt*)p); +#endif + } + + /* Check that malloc() has not failed. If it has, return early. */ + db = p->db; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM; + return SQLITE_NOMEM; + } + + if( p->pc<=0 && p->expired ){ + p->rc = SQLITE_SCHEMA; + rc = SQLITE_ERROR; + goto end_of_step; + } + if( p->pc<0 ){ + /* If there are no other statements currently running, then + ** reset the interrupt flag. This prevents a call to sqlite3_interrupt + ** from interrupting a statement that has not yet started. + */ + if( db->nVdbeActive==0 ){ + db->u1.isInterrupted = 0; + } + + assert( db->nVdbeWrite>0 || db->autoCommit==0 + || (db->nDeferredCons==0 && db->nDeferredImmCons==0) + ); + +#ifndef SQLITE_OMIT_TRACE + if( db->xProfile && !db->init.busy ){ + sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); + } +#endif + + db->nVdbeActive++; + if( p->readOnly==0 ) db->nVdbeWrite++; + if( p->bIsReader ) db->nVdbeRead++; + p->pc = 0; + } +#ifndef SQLITE_OMIT_EXPLAIN + if( p->explain ){ + rc = sqlite3VdbeList(p); + }else +#endif /* SQLITE_OMIT_EXPLAIN */ + { + db->nVdbeExec++; + rc = sqlite3VdbeExec(p); + db->nVdbeExec--; + } + +#ifndef SQLITE_OMIT_TRACE + /* Invoke the profile callback if there is one + */ + if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ + sqlite3_int64 iNow; + sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); + db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); + } +#endif + + if( rc==SQLITE_DONE ){ + assert( p->rc==SQLITE_OK ); + p->rc = doWalCallbacks(db); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + } + } + + db->errCode = rc; + if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ + p->rc = SQLITE_NOMEM; + } +end_of_step: + /* At this point local variable rc holds the value that should be + ** returned if this statement was compiled using the legacy + ** sqlite3_prepare() interface. According to the docs, this can only + ** be one of the values in the first assert() below. Variable p->rc + ** contains the value that would be returned if sqlite3_finalize() + ** were called on statement p. + */ + assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR + || rc==SQLITE_BUSY || rc==SQLITE_MISUSE + ); + assert( p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE ); + if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ + /* If this statement was prepared using sqlite3_prepare_v2(), and an + ** error has occurred, then return the error code in p->rc to the + ** caller. Set the error code in the database handle to the same value. + */ + rc = sqlite3VdbeTransferError(p); + } + return (rc&db->errMask); +} + +/* +** This is the top-level implementation of sqlite3_step(). Call +** sqlite3Step() to do most of the work. If a schema error occurs, +** call sqlite3Reprepare() and try again. +*/ +SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ + int rc = SQLITE_OK; /* Result from sqlite3Step() */ + int rc2 = SQLITE_OK; /* Result from sqlite3Reprepare() */ + Vdbe *v = (Vdbe*)pStmt; /* the prepared statement */ + int cnt = 0; /* Counter to prevent infinite loop of reprepares */ + sqlite3 *db; /* The database connection */ + + if( vdbeSafetyNotNull(v) ){ + return SQLITE_MISUSE_BKPT; + } + db = v->db; + sqlite3_mutex_enter(db->mutex); + v->doingRerun = 0; + while( (rc = sqlite3Step(v))==SQLITE_SCHEMA + && cnt++ < SQLITE_MAX_SCHEMA_RETRY + && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ + sqlite3_reset(pStmt); + v->doingRerun = 1; + assert( v->expired==0 ); + } + if( rc2!=SQLITE_OK ){ + /* This case occurs after failing to recompile an sql statement. + ** The error message from the SQL compiler has already been loaded + ** into the database handle. This block copies the error message + ** from the database handle into the statement and sets the statement + ** program counter to 0 to ensure that when the statement is + ** finalized or reset the parser error message is available via + ** sqlite3_errmsg() and sqlite3_errcode(). + */ + const char *zErr = (const char *)sqlite3_value_text(db->pErr); + assert( zErr!=0 || db->mallocFailed ); + sqlite3DbFree(db, v->zErrMsg); + if( !db->mallocFailed ){ + v->zErrMsg = sqlite3DbStrDup(db, zErr); + v->rc = rc2; + } else { + v->zErrMsg = 0; + v->rc = rc = SQLITE_NOMEM; + } + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + + +/* +** Extract the user data from a sqlite3_context structure and return a +** pointer to it. +*/ +SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ + assert( p && p->pFunc ); + return p->pFunc->pUserData; +} + +/* +** Extract the user data from a sqlite3_context structure and return a +** pointer to it. +** +** IMPLEMENTATION-OF: R-46798-50301 The sqlite3_context_db_handle() interface +** returns a copy of the pointer to the database connection (the 1st +** parameter) of the sqlite3_create_function() and +** sqlite3_create_function16() routines that originally registered the +** application defined function. +*/ +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ + assert( p && p->pFunc ); + return p->s.db; +} + +/* +** Return the current time for a statement +*/ +SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ + Vdbe *v = p->pVdbe; + int rc; + if( v->iCurrentTime==0 ){ + rc = sqlite3OsCurrentTimeInt64(p->s.db->pVfs, &v->iCurrentTime); + if( rc ) v->iCurrentTime = 0; + } + return v->iCurrentTime; +} + +/* +** The following is the implementation of an SQL function that always +** fails with an error message stating that the function is used in the +** wrong context. The sqlite3_overload_function() API might construct +** SQL function that use this routine so that the functions will exist +** for name resolution but are actually overloaded by the xFindFunction +** method of virtual tables. +*/ +SQLITE_PRIVATE void sqlite3InvalidFunction( + sqlite3_context *context, /* The function calling context */ + int NotUsed, /* Number of arguments to the function */ + sqlite3_value **NotUsed2 /* Value of each argument */ +){ + const char *zName = context->pFunc->zName; + char *zErr; + UNUSED_PARAMETER2(NotUsed, NotUsed2); + zErr = sqlite3_mprintf( + "unable to use function %s in the requested context", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); +} + +/* +** Allocate or return the aggregate context for a user function. A new +** context is allocated on the first call. Subsequent calls return the +** same context that was returned on prior calls. +*/ +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ + Mem *pMem; + assert( p && p->pFunc && p->pFunc->xStep ); + assert( sqlite3_mutex_held(p->s.db->mutex) ); + pMem = p->pMem; + testcase( nByte<0 ); + if( (pMem->flags & MEM_Agg)==0 ){ + if( nByte<=0 ){ + sqlite3VdbeMemReleaseExternal(pMem); + pMem->flags = MEM_Null; + pMem->z = 0; + }else{ + sqlite3VdbeMemGrow(pMem, nByte, 0); + pMem->flags = MEM_Agg; + pMem->u.pDef = p->pFunc; + if( pMem->z ){ + memset(pMem->z, 0, nByte); + } + } + } + return (void*)pMem->z; +} + +/* +** Return the auxilary data pointer, if any, for the iArg'th argument to +** the user-function defined by pCtx. +*/ +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ + AuxData *pAuxData; + + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ + if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; + } + + return (pAuxData ? pAuxData->pAux : 0); +} + +/* +** Set the auxilary data pointer and delete function, for the iArg'th +** argument to the user-function defined by pCtx. Any previous value is +** deleted by calling the delete function specified when it was set. +*/ +SQLITE_API void sqlite3_set_auxdata( + sqlite3_context *pCtx, + int iArg, + void *pAux, + void (*xDelete)(void*) +){ + AuxData *pAuxData; + Vdbe *pVdbe = pCtx->pVdbe; + + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + if( iArg<0 ) goto failed; + + for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ + if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; + } + if( pAuxData==0 ){ + pAuxData = sqlite3DbMallocZero(pVdbe->db, sizeof(AuxData)); + if( !pAuxData ) goto failed; + pAuxData->iOp = pCtx->iOp; + pAuxData->iArg = iArg; + pAuxData->pNext = pVdbe->pAuxData; + pVdbe->pAuxData = pAuxData; + if( pCtx->fErrorOrAux==0 ){ + pCtx->isError = 0; + pCtx->fErrorOrAux = 1; + } + }else if( pAuxData->xDelete ){ + pAuxData->xDelete(pAuxData->pAux); + } + + pAuxData->pAux = pAux; + pAuxData->xDelete = xDelete; + return; + +failed: + if( xDelete ){ + xDelete(pAux); + } +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Return the number of times the Step function of a aggregate has been +** called. +** +** This function is deprecated. Do not use it for new code. It is +** provide only to avoid breaking legacy code. New aggregate function +** implementations should keep their own counts within their aggregate +** context. +*/ +SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ + assert( p && p->pMem && p->pFunc && p->pFunc->xStep ); + return p->pMem->n; +} +#endif + +/* +** Return the number of columns in the result set for the statement pStmt. +*/ +SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ + Vdbe *pVm = (Vdbe *)pStmt; + return pVm ? pVm->nResColumn : 0; +} + +/* +** Return the number of values available from the current row of the +** currently executing statement pStmt. +*/ +SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ + Vdbe *pVm = (Vdbe *)pStmt; + if( pVm==0 || pVm->pResultSet==0 ) return 0; + return pVm->nResColumn; +} + +/* +** Return a pointer to static memory containing an SQL NULL value. +*/ +static const Mem *columnNullValue(void){ + /* Even though the Mem structure contains an element + ** of type i64, on certain architectures (x86) with certain compiler + ** switches (-Os), gcc may align this Mem object on a 4-byte boundary + ** instead of an 8-byte one. This all works fine, except that when + ** running with SQLITE_DEBUG defined the SQLite code sometimes assert()s + ** that a Mem structure is located on an 8-byte boundary. To prevent + ** these assert()s from failing, when building with SQLITE_DEBUG defined + ** using gcc, we force nullMem to be 8-byte aligned using the magical + ** __attribute__((aligned(8))) macro. */ + static const Mem nullMem +#if defined(SQLITE_DEBUG) && defined(__GNUC__) + __attribute__((aligned(8))) +#endif + = {0, "", (double)0, {0}, 0, MEM_Null, 0, +#ifdef SQLITE_DEBUG + 0, 0, /* pScopyFrom, pFiller */ +#endif + 0, 0 }; + return &nullMem; +} + +/* +** Check to see if column iCol of the given statement is valid. If +** it is, return a pointer to the Mem for the value of that column. +** If iCol is not valid, return a pointer to a Mem which has a value +** of NULL. +*/ +static Mem *columnMem(sqlite3_stmt *pStmt, int i){ + Vdbe *pVm; + Mem *pOut; + + pVm = (Vdbe *)pStmt; + if( pVm && pVm->pResultSet!=0 && inResColumn && i>=0 ){ + sqlite3_mutex_enter(pVm->db->mutex); + pOut = &pVm->pResultSet[i]; + }else{ + if( pVm && ALWAYS(pVm->db) ){ + sqlite3_mutex_enter(pVm->db->mutex); + sqlite3Error(pVm->db, SQLITE_RANGE, 0); + } + pOut = (Mem*)columnNullValue(); + } + return pOut; +} + +/* +** This function is called after invoking an sqlite3_value_XXX function on a +** column value (i.e. a value returned by evaluating an SQL expression in the +** select list of a SELECT statement) that may cause a malloc() failure. If +** malloc() has failed, the threads mallocFailed flag is cleared and the result +** code of statement pStmt set to SQLITE_NOMEM. +** +** Specifically, this is called from within: +** +** sqlite3_column_int() +** sqlite3_column_int64() +** sqlite3_column_text() +** sqlite3_column_text16() +** sqlite3_column_real() +** sqlite3_column_bytes() +** sqlite3_column_bytes16() +** sqiite3_column_blob() +*/ +static void columnMallocFailure(sqlite3_stmt *pStmt) +{ + /* If malloc() failed during an encoding conversion within an + ** sqlite3_column_XXX API, then set the return code of the statement to + ** SQLITE_NOMEM. The next call to _step() (if any) will return SQLITE_ERROR + ** and _finalize() will return NOMEM. + */ + Vdbe *p = (Vdbe *)pStmt; + if( p ){ + p->rc = sqlite3ApiExit(p->db, p->rc); + sqlite3_mutex_leave(p->db->mutex); + } +} + +/**************************** sqlite3_column_ ******************************* +** The following routines are used to access elements of the current row +** in the result set. +*/ +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ + const void *val; + val = sqlite3_value_blob( columnMem(pStmt,i) ); + /* Even though there is no encoding conversion, value_blob() might + ** need to call malloc() to expand the result of a zeroblob() + ** expression. + */ + columnMallocFailure(pStmt); + return val; +} +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ + int val = sqlite3_value_bytes( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ + int val = sqlite3_value_bytes16( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API double sqlite3_column_double(sqlite3_stmt *pStmt, int i){ + double val = sqlite3_value_double( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API int sqlite3_column_int(sqlite3_stmt *pStmt, int i){ + int val = sqlite3_value_int( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API sqlite_int64 sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ + sqlite_int64 val = sqlite3_value_int64( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt *pStmt, int i){ + const unsigned char *val = sqlite3_value_text( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt *pStmt, int i){ + Mem *pOut = columnMem(pStmt, i); + if( pOut->flags&MEM_Static ){ + pOut->flags &= ~MEM_Static; + pOut->flags |= MEM_Ephem; + } + columnMallocFailure(pStmt); + return (sqlite3_value *)pOut; +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ + const void *val = sqlite3_value_text16( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ + int iType = sqlite3_value_type( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return iType; +} + +/* +** Convert the N-th element of pStmt->pColName[] into a string using +** xFunc() then return that string. If N is out of range, return 0. +** +** There are up to 5 names for each column. useType determines which +** name is returned. Here are the names: +** +** 0 The column name as it should be displayed for output +** 1 The datatype name for the column +** 2 The name of the database that the column derives from +** 3 The name of the table that the column derives from +** 4 The name of the table column that the result column derives from +** +** If the result is not a simple column reference (if it is an expression +** or a constant) then useTypes 2, 3, and 4 return NULL. +*/ +static const void *columnName( + sqlite3_stmt *pStmt, + int N, + const void *(*xFunc)(Mem*), + int useType +){ + const void *ret = 0; + Vdbe *p = (Vdbe *)pStmt; + int n; + sqlite3 *db = p->db; + + assert( db!=0 ); + n = sqlite3_column_count(pStmt); + if( N=0 ){ + N += useType*n; + sqlite3_mutex_enter(db->mutex); + assert( db->mallocFailed==0 ); + ret = xFunc(&p->aColName[N]); + /* A malloc may have failed inside of the xFunc() call. If this + ** is the case, clear the mallocFailed flag and return NULL. + */ + if( db->mallocFailed ){ + db->mallocFailed = 0; + ret = 0; + } + sqlite3_mutex_leave(db->mutex); + } + return ret; +} + +/* +** Return the name of the Nth column of the result set returned by SQL +** statement pStmt. +*/ +SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_NAME); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_NAME); +} +#endif + +/* +** Constraint: If you have ENABLE_COLUMN_METADATA then you must +** not define OMIT_DECLTYPE. +*/ +#if defined(SQLITE_OMIT_DECLTYPE) && defined(SQLITE_ENABLE_COLUMN_METADATA) +# error "Must not define both SQLITE_OMIT_DECLTYPE \ + and SQLITE_ENABLE_COLUMN_METADATA" +#endif + +#ifndef SQLITE_OMIT_DECLTYPE +/* +** Return the column declaration type (if applicable) of the 'i'th column +** of the result set of SQL statement pStmt. +*/ +SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DECLTYPE); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DECLTYPE); +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_DECLTYPE */ + +#ifdef SQLITE_ENABLE_COLUMN_METADATA +/* +** Return the name of the database from which a result column derives. +** NULL is returned if the result column is an expression or constant or +** anything else which is not an unabiguous reference to a database column. +*/ +SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DATABASE); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DATABASE); +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the name of the table from which a result column derives. +** NULL is returned if the result column is an expression or constant or +** anything else which is not an unabiguous reference to a database column. +*/ +SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_TABLE); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_TABLE); +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the name of the table column from which a result column derives. +** NULL is returned if the result column is an expression or constant or +** anything else which is not an unabiguous reference to a database column. +*/ +SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_COLUMN); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_COLUMN); +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_ENABLE_COLUMN_METADATA */ + + +/******************************* sqlite3_bind_ *************************** +** +** Routines used to attach values to wildcards in a compiled SQL statement. +*/ +/* +** Unbind the value bound to variable i in virtual machine p. This is the +** the same as binding a NULL value to the column. If the "i" parameter is +** out of range, then SQLITE_RANGE is returned. Othewise SQLITE_OK. +** +** A successful evaluation of this routine acquires the mutex on p. +** the mutex is released if any kind of error occurs. +** +** The error code stored in database p->db is overwritten with the return +** value in any case. +*/ +static int vdbeUnbind(Vdbe *p, int i){ + Mem *pVar; + if( vdbeSafetyNotNull(p) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(p->db->mutex); + if( p->magic!=VDBE_MAGIC_RUN || p->pc>=0 ){ + sqlite3Error(p->db, SQLITE_MISUSE, 0); + sqlite3_mutex_leave(p->db->mutex); + sqlite3_log(SQLITE_MISUSE, + "bind on a busy prepared statement: [%s]", p->zSql); + return SQLITE_MISUSE_BKPT; + } + if( i<1 || i>p->nVar ){ + sqlite3Error(p->db, SQLITE_RANGE, 0); + sqlite3_mutex_leave(p->db->mutex); + return SQLITE_RANGE; + } + i--; + pVar = &p->aVar[i]; + sqlite3VdbeMemRelease(pVar); + pVar->flags = MEM_Null; + sqlite3Error(p->db, SQLITE_OK, 0); + + /* If the bit corresponding to this variable in Vdbe.expmask is set, then + ** binding a new value to this variable invalidates the current query plan. + ** + ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host + ** parameter in the WHERE clause might influence the choice of query plan + ** for a statement, then the statement will be automatically recompiled, + ** as if there had been a schema change, on the first sqlite3_step() call + ** following any change to the bindings of that parameter. + */ + if( p->isPrepareV2 && + ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff) + ){ + p->expired = 1; + } + return SQLITE_OK; +} + +/* +** Bind a text or BLOB value. +*/ +static int bindText( + sqlite3_stmt *pStmt, /* The statement to bind against */ + int i, /* Index of the parameter to bind */ + const void *zData, /* Pointer to the data to be bound */ + int nData, /* Number of bytes of data to be bound */ + void (*xDel)(void*), /* Destructor for the data */ + u8 encoding /* Encoding for the data */ +){ + Vdbe *p = (Vdbe *)pStmt; + Mem *pVar; + int rc; + + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + if( zData!=0 ){ + pVar = &p->aVar[i-1]; + rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + if( rc==SQLITE_OK && encoding!=0 ){ + rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + } + sqlite3Error(p->db, rc, 0); + rc = sqlite3ApiExit(p->db, rc); + } + sqlite3_mutex_leave(p->db->mutex); + }else if( xDel!=SQLITE_STATIC && xDel!=SQLITE_TRANSIENT ){ + xDel((void*)zData); + } + return rc; +} + + +/* +** Bind a blob value to an SQL statement variable. +*/ +SQLITE_API int sqlite3_bind_blob( + sqlite3_stmt *pStmt, + int i, + const void *zData, + int nData, + void (*xDel)(void*) +){ + return bindText(pStmt, i, zData, nData, xDel, 0); +} +SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); + sqlite3_mutex_leave(p->db->mutex); + } + return rc; +} +SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ + return sqlite3_bind_int64(p, i, (i64)iValue); +} +SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); + sqlite3_mutex_leave(p->db->mutex); + } + return rc; +} +SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ + int rc; + Vdbe *p = (Vdbe*)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3_mutex_leave(p->db->mutex); + } + return rc; +} +SQLITE_API int sqlite3_bind_text( + sqlite3_stmt *pStmt, + int i, + const char *zData, + int nData, + void (*xDel)(void*) +){ + return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int sqlite3_bind_text16( + sqlite3_stmt *pStmt, + int i, + const void *zData, + int nData, + void (*xDel)(void*) +){ + return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ + int rc; + switch( sqlite3_value_type((sqlite3_value*)pValue) ){ + case SQLITE_INTEGER: { + rc = sqlite3_bind_int64(pStmt, i, pValue->u.i); + break; + } + case SQLITE_FLOAT: { + rc = sqlite3_bind_double(pStmt, i, pValue->r); + break; + } + case SQLITE_BLOB: { + if( pValue->flags & MEM_Zero ){ + rc = sqlite3_bind_zeroblob(pStmt, i, pValue->u.nZero); + }else{ + rc = sqlite3_bind_blob(pStmt, i, pValue->z, pValue->n,SQLITE_TRANSIENT); + } + break; + } + case SQLITE_TEXT: { + rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE_TRANSIENT, + pValue->enc); + break; + } + default: { + rc = sqlite3_bind_null(pStmt, i); + break; + } + } + return rc; +} +SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); + sqlite3_mutex_leave(p->db->mutex); + } + return rc; +} + +/* +** Return the number of wildcards that can be potentially bound to. +** This routine is added to support DBD::SQLite. +*/ +SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + return p ? p->nVar : 0; +} + +/* +** Return the name of a wildcard parameter. Return NULL if the index +** is out of range or if the wildcard is unnamed. +** +** The result is always UTF-8. +*/ +SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ + Vdbe *p = (Vdbe*)pStmt; + if( p==0 || i<1 || i>p->nzVar ){ + return 0; + } + return p->azVar[i-1]; +} + +/* +** Given a wildcard parameter name, return the index of the variable +** with that name. If there is no variable with the given name, +** return 0. +*/ +SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ + int i; + if( p==0 ){ + return 0; + } + if( zName ){ + for(i=0; inzVar; i++){ + const char *z = p->azVar[i]; + if( z && strncmp(z,zName,nName)==0 && z[nName]==0 ){ + return i+1; + } + } + } + return 0; +} +SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ + return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName)); +} + +/* +** Transfer all bindings from the first statement over to the second. +*/ +SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ + Vdbe *pFrom = (Vdbe*)pFromStmt; + Vdbe *pTo = (Vdbe*)pToStmt; + int i; + assert( pTo->db==pFrom->db ); + assert( pTo->nVar==pFrom->nVar ); + sqlite3_mutex_enter(pTo->db->mutex); + for(i=0; inVar; i++){ + sqlite3VdbeMemMove(&pTo->aVar[i], &pFrom->aVar[i]); + } + sqlite3_mutex_leave(pTo->db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Deprecated external interface. Internal/core SQLite code +** should call sqlite3TransferBindings. +** +** Is is misuse to call this routine with statements from different +** database connections. But as this is a deprecated interface, we +** will not bother to check for that condition. +** +** If the two statements contain a different number of bindings, then +** an SQLITE_ERROR is returned. Nothing else can go wrong, so otherwise +** SQLITE_OK is returned. +*/ +SQLITE_API int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ + Vdbe *pFrom = (Vdbe*)pFromStmt; + Vdbe *pTo = (Vdbe*)pToStmt; + if( pFrom->nVar!=pTo->nVar ){ + return SQLITE_ERROR; + } + if( pTo->isPrepareV2 && pTo->expmask ){ + pTo->expired = 1; + } + if( pFrom->isPrepareV2 && pFrom->expmask ){ + pFrom->expired = 1; + } + return sqlite3TransferBindings(pFromStmt, pToStmt); +} +#endif + +/* +** Return the sqlite3* database handle to which the prepared statement given +** in the argument belongs. This is the same database handle that was +** the first argument to the sqlite3_prepare() that was used to create +** the statement in the first place. +*/ +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->db : 0; +} + +/* +** Return true if the prepared statement is guaranteed to not modify the +** database. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; +} + +/* +** Return true if the prepared statement is in need of being reset. +*/ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ + Vdbe *v = (Vdbe*)pStmt; + return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN; +} + +/* +** Return a pointer to the next prepared statement after pStmt associated +** with database connection pDb. If pStmt is NULL, return the first +** prepared statement for the database connection. Return NULL if there +** are no more. +*/ +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ + sqlite3_stmt *pNext; + sqlite3_mutex_enter(pDb->mutex); + if( pStmt==0 ){ + pNext = (sqlite3_stmt*)pDb->pVdbe; + }else{ + pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext; + } + sqlite3_mutex_leave(pDb->mutex); + return pNext; +} + +/* +** Return the value of a status counter for a prepared statement +*/ +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ + Vdbe *pVdbe = (Vdbe*)pStmt; + u32 v = pVdbe->aCounter[op]; + if( resetFlag ) pVdbe->aCounter[op] = 0; + return (int)v; +} + +/************** End of vdbeapi.c *********************************************/ +/************** Begin file vdbetrace.c ***************************************/ +/* +** 2009 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to insert the values of host parameters +** (aka "wildcards") into the SQL text output by sqlite3_trace(). +** +** The Vdbe parse-tree explainer is also found here. +*/ + +#ifndef SQLITE_OMIT_TRACE + +/* +** zSql is a zero-terminated string of UTF-8 SQL text. Return the number of +** bytes in this text up to but excluding the first character in +** a host parameter. If the text contains no host parameters, return +** the total number of bytes in the text. +*/ +static int findNextHostParameter(const char *zSql, int *pnToken){ + int tokenType; + int nTotal = 0; + int n; + + *pnToken = 0; + while( zSql[0] ){ + n = sqlite3GetToken((u8*)zSql, &tokenType); + assert( n>0 && tokenType!=TK_ILLEGAL ); + if( tokenType==TK_VARIABLE ){ + *pnToken = n; + break; + } + nTotal += n; + zSql += n; + } + return nTotal; +} + +/* +** This function returns a pointer to a nul-terminated string in memory +** obtained from sqlite3DbMalloc(). If sqlite3.nVdbeExec is 1, then the +** string contains a copy of zRawSql but with host parameters expanded to +** their current bindings. Or, if sqlite3.nVdbeExec is greater than 1, +** then the returned string holds a copy of zRawSql with "-- " prepended +** to each line of text. +** +** If the SQLITE_TRACE_SIZE_LIMIT macro is defined to an integer, then +** then long strings and blobs are truncated to that many bytes. This +** can be used to prevent unreasonably large trace strings when dealing +** with large (multi-megabyte) strings and blobs. +** +** The calling function is responsible for making sure the memory returned +** is eventually freed. +** +** ALGORITHM: Scan the input string looking for host parameters in any of +** these forms: ?, ?N, $A, @A, :A. Take care to avoid text within +** string literals, quoted identifier names, and comments. For text forms, +** the host parameter index is found by scanning the perpared +** statement for the corresponding OP_Variable opcode. Once the host +** parameter index is known, locate the value in p->aVar[]. Then render +** the value as a literal in place of the host parameter name. +*/ +SQLITE_PRIVATE char *sqlite3VdbeExpandSql( + Vdbe *p, /* The prepared statement being evaluated */ + const char *zRawSql /* Raw text of the SQL statement */ +){ + sqlite3 *db; /* The database connection */ + int idx = 0; /* Index of a host parameter */ + int nextIndex = 1; /* Index of next ? host parameter */ + int n; /* Length of a token prefix */ + int nToken; /* Length of the parameter token */ + int i; /* Loop counter */ + Mem *pVar; /* Value of a host parameter */ + StrAccum out; /* Accumulate the output here */ + char zBase[100]; /* Initial working space */ + + db = p->db; + sqlite3StrAccumInit(&out, zBase, sizeof(zBase), + db->aLimit[SQLITE_LIMIT_LENGTH]); + out.db = db; + if( db->nVdbeExec>1 ){ + while( *zRawSql ){ + const char *zStart = zRawSql; + while( *(zRawSql++)!='\n' && *zRawSql ); + sqlite3StrAccumAppend(&out, "-- ", 3); + assert( (zRawSql - zStart) > 0 ); + sqlite3StrAccumAppend(&out, zStart, (int)(zRawSql-zStart)); + } + }else{ + while( zRawSql[0] ){ + n = findNextHostParameter(zRawSql, &nToken); + assert( n>0 ); + sqlite3StrAccumAppend(&out, zRawSql, n); + zRawSql += n; + assert( zRawSql[0] || nToken==0 ); + if( nToken==0 ) break; + if( zRawSql[0]=='?' ){ + if( nToken>1 ){ + assert( sqlite3Isdigit(zRawSql[1]) ); + sqlite3GetInt32(&zRawSql[1], &idx); + }else{ + idx = nextIndex; + } + }else{ + assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' ); + testcase( zRawSql[0]==':' ); + testcase( zRawSql[0]=='$' ); + testcase( zRawSql[0]=='@' ); + idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken); + assert( idx>0 ); + } + zRawSql += nToken; + nextIndex = idx + 1; + assert( idx>0 && idx<=p->nVar ); + pVar = &p->aVar[idx-1]; + if( pVar->flags & MEM_Null ){ + sqlite3StrAccumAppend(&out, "NULL", 4); + }else if( pVar->flags & MEM_Int ){ + sqlite3XPrintf(&out, 0, "%lld", pVar->u.i); + }else if( pVar->flags & MEM_Real ){ + sqlite3XPrintf(&out, 0, "%!.15g", pVar->r); + }else if( pVar->flags & MEM_Str ){ + int nOut; /* Number of bytes of the string text to include in output */ +#ifndef SQLITE_OMIT_UTF16 + u8 enc = ENC(db); + Mem utf8; + if( enc!=SQLITE_UTF8 ){ + memset(&utf8, 0, sizeof(utf8)); + utf8.db = db; + sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); + sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); + pVar = &utf8; + } +#endif + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ){ + nOut = SQLITE_TRACE_SIZE_LIMIT; + while( nOutn && (pVar->z[nOut]&0xc0)==0x80 ){ nOut++; } + } +#endif + sqlite3XPrintf(&out, 0, "'%.*q'", nOut, pVar->z); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOutn ){ + sqlite3XPrintf(&out, 0, "/*+%d bytes*/", pVar->n-nOut); + } +#endif +#ifndef SQLITE_OMIT_UTF16 + if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); +#endif + }else if( pVar->flags & MEM_Zero ){ + sqlite3XPrintf(&out, 0, "zeroblob(%d)", pVar->u.nZero); + }else{ + int nOut; /* Number of bytes of the blob to include in output */ + assert( pVar->flags & MEM_Blob ); + sqlite3StrAccumAppend(&out, "x'", 2); + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; +#endif + for(i=0; iz[i]&0xff); + } + sqlite3StrAccumAppend(&out, "'", 1); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOutn ){ + sqlite3XPrintf(&out, 0, "/*+%d bytes*/", pVar->n-nOut); + } +#endif + } + } + } + return sqlite3StrAccumFinish(&out); +} + +#endif /* #ifndef SQLITE_OMIT_TRACE */ + +/***************************************************************************** +** The following code implements the data-structure explaining logic +** for the Vdbe. +*/ + +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) + +/* +** Allocate a new Explain object +*/ +SQLITE_PRIVATE void sqlite3ExplainBegin(Vdbe *pVdbe){ + if( pVdbe ){ + Explain *p; + sqlite3BeginBenignMalloc(); + p = (Explain *)sqlite3MallocZero( sizeof(Explain) ); + if( p ){ + p->pVdbe = pVdbe; + sqlite3_free(pVdbe->pExplain); + pVdbe->pExplain = p; + sqlite3StrAccumInit(&p->str, p->zBase, sizeof(p->zBase), + SQLITE_MAX_LENGTH); + p->str.useMalloc = 2; + }else{ + sqlite3EndBenignMalloc(); + } + } +} + +/* +** Return true if the Explain ends with a new-line. +*/ +static int endsWithNL(Explain *p){ + return p && p->str.zText && p->str.nChar + && p->str.zText[p->str.nChar-1]=='\n'; +} + +/* +** Append text to the indentation +*/ +SQLITE_PRIVATE void sqlite3ExplainPrintf(Vdbe *pVdbe, const char *zFormat, ...){ + Explain *p; + if( pVdbe && (p = pVdbe->pExplain)!=0 ){ + va_list ap; + if( p->nIndent && endsWithNL(p) ){ + int n = p->nIndent; + if( n>ArraySize(p->aIndent) ) n = ArraySize(p->aIndent); + sqlite3AppendSpace(&p->str, p->aIndent[n-1]); + } + va_start(ap, zFormat); + sqlite3VXPrintf(&p->str, SQLITE_PRINTF_INTERNAL, zFormat, ap); + va_end(ap); + } +} + +/* +** Append a '\n' if there is not already one. +*/ +SQLITE_PRIVATE void sqlite3ExplainNL(Vdbe *pVdbe){ + Explain *p; + if( pVdbe && (p = pVdbe->pExplain)!=0 && !endsWithNL(p) ){ + sqlite3StrAccumAppend(&p->str, "\n", 1); + } +} + +/* +** Push a new indentation level. Subsequent lines will be indented +** so that they begin at the current cursor position. +*/ +SQLITE_PRIVATE void sqlite3ExplainPush(Vdbe *pVdbe){ + Explain *p; + if( pVdbe && (p = pVdbe->pExplain)!=0 ){ + if( p->str.zText && p->nIndentaIndent) ){ + const char *z = p->str.zText; + int i = p->str.nChar-1; + int x; + while( i>=0 && z[i]!='\n' ){ i--; } + x = (p->str.nChar - 1) - i; + if( p->nIndent && xaIndent[p->nIndent-1] ){ + x = p->aIndent[p->nIndent-1]; + } + p->aIndent[p->nIndent] = x; + } + p->nIndent++; + } +} + +/* +** Pop the indentation stack by one level. +*/ +SQLITE_PRIVATE void sqlite3ExplainPop(Vdbe *p){ + if( p && p->pExplain ) p->pExplain->nIndent--; +} + +/* +** Free the indentation structure +*/ +SQLITE_PRIVATE void sqlite3ExplainFinish(Vdbe *pVdbe){ + if( pVdbe && pVdbe->pExplain ){ + sqlite3_free(pVdbe->zExplain); + sqlite3ExplainNL(pVdbe); + pVdbe->zExplain = sqlite3StrAccumFinish(&pVdbe->pExplain->str); + sqlite3_free(pVdbe->pExplain); + pVdbe->pExplain = 0; + sqlite3EndBenignMalloc(); + } +} + +/* +** Return the explanation of a virtual machine. +*/ +SQLITE_PRIVATE const char *sqlite3VdbeExplanation(Vdbe *pVdbe){ + return (pVdbe && pVdbe->zExplain) ? pVdbe->zExplain : 0; +} +#endif /* defined(SQLITE_DEBUG) */ + +/************** End of vdbetrace.c *******************************************/ +/************** Begin file vdbe.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** The code in this file implements the function that runs the +** bytecode of a prepared statement. +** +** Various scripts scan this source file in order to generate HTML +** documentation, headers files, or other derived files. The formatting +** of the code in this file is, therefore, important. See other comments +** in this file for details. If in doubt, do not deviate from existing +** commenting and indentation practices when changing or adding code. +*/ + +/* +** Invoke this macro on memory cells just prior to changing the +** value of the cell. This macro verifies that shallow copies are +** not misused. A shallow copy of a string or blob just copies a +** pointer to the string or blob, not the content. If the original +** is changed while the copy is still in use, the string or blob might +** be changed out from under the copy. This macro verifies that nothing +** like that ever happens. +*/ +#ifdef SQLITE_DEBUG +# define memAboutToChange(P,M) sqlite3VdbeMemAboutToChange(P,M) +#else +# define memAboutToChange(P,M) +#endif + +/* +** The following global variable is incremented every time a cursor +** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test +** procedures use this information to make sure that indices are +** working correctly. This variable has no function other than to +** help verify the correct operation of the library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_search_count = 0; +#endif + +/* +** When this global variable is positive, it gets decremented once before +** each instruction in the VDBE. When it reaches zero, the u1.isInterrupted +** field of the sqlite3 structure is set in order to simulate an interrupt. +** +** This facility is used for testing purposes only. It does not function +** in an ordinary build. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_interrupt_count = 0; +#endif + +/* +** The next global variable is incremented each type the OP_Sort opcode +** is executed. The test procedures use this information to make sure that +** sorting is occurring or not occurring at appropriate times. This variable +** has no function other than to help verify the correct operation of the +** library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_sort_count = 0; +#endif + +/* +** The next global variable records the size of the largest MEM_Blob +** or MEM_Str that has been used by a VDBE opcode. The test procedures +** use this information to make sure that the zero-blob functionality +** is working correctly. This variable has no function other than to +** help verify the correct operation of the library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_max_blobsize = 0; +static void updateMaxBlobsize(Mem *p){ + if( (p->flags & (MEM_Str|MEM_Blob))!=0 && p->n>sqlite3_max_blobsize ){ + sqlite3_max_blobsize = p->n; + } +} +#endif + +/* +** The next global variable is incremented each time the OP_Found opcode +** is executed. This is used to test whether or not the foreign key +** operation implemented using OP_FkIsZero is working. This variable +** has no function other than to help verify the correct operation of the +** library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_found_count = 0; +#endif + +/* +** Test a register to see if it exceeds the current maximum blob size. +** If it does, record the new maximum blob size. +*/ +#if defined(SQLITE_TEST) && !defined(SQLITE_OMIT_BUILTIN_TEST) +# define UPDATE_MAX_BLOBSIZE(P) updateMaxBlobsize(P) +#else +# define UPDATE_MAX_BLOBSIZE(P) +#endif + +/* +** Invoke the VDBE coverage callback, if that callback is defined. This +** feature is used for test suite validation only and does not appear an +** production builds. +** +** M is an integer, 2 or 3, that indices how many different ways the +** branch can go. It is usually 2. "I" is the direction the branch +** goes. 0 means falls through. 1 means branch is taken. 2 means the +** second alternative branch is taken. +** +** iSrcLine is the source code line (from the __LINE__ macro) that +** generated the VDBE instruction. This instrumentation assumes that all +** source code is in a single file (the amalgamation). Special values 1 +** and 2 for the iSrcLine parameter mean that this particular branch is +** always taken or never taken, respectively. +*/ +#if !defined(SQLITE_VDBE_COVERAGE) +# define VdbeBranchTaken(I,M) +#else +# define VdbeBranchTaken(I,M) vdbeTakeBranch(pOp->iSrcLine,I,M) + static void vdbeTakeBranch(int iSrcLine, u8 I, u8 M){ + if( iSrcLine<=2 && ALWAYS(iSrcLine>0) ){ + M = iSrcLine; + /* Assert the truth of VdbeCoverageAlwaysTaken() and + ** VdbeCoverageNeverTaken() */ + assert( (M & I)==I ); + }else{ + if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ + sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg, + iSrcLine,I,M); + } + } +#endif + +/* +** Convert the given register into a string if it isn't one +** already. Return non-zero if a malloc() fails. +*/ +#define Stringify(P, enc) \ + if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc)) \ + { goto no_mem; } + +/* +** An ephemeral string value (signified by the MEM_Ephem flag) contains +** a pointer to a dynamically allocated string where some other entity +** is responsible for deallocating that string. Because the register +** does not control the string, it might be deleted without the register +** knowing it. +** +** This routine converts an ephemeral string into a dynamically allocated +** string that the register itself controls. In other words, it +** converts an MEM_Ephem string into a string with P.z==P.zMalloc. +*/ +#define Deephemeralize(P) \ + if( ((P)->flags&MEM_Ephem)!=0 \ + && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} + +/* Return true if the cursor was opened using the OP_OpenSorter opcode. */ +#define isSorter(x) ((x)->pSorter!=0) + +/* +** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL +** if we run out of memory. +*/ +static VdbeCursor *allocateCursor( + Vdbe *p, /* The virtual machine */ + int iCur, /* Index of the new VdbeCursor */ + int nField, /* Number of fields in the table or index */ + int iDb, /* Database the cursor belongs to, or -1 */ + int isBtreeCursor /* True for B-Tree. False for pseudo-table or vtab */ +){ + /* Find the memory cell that will be used to store the blob of memory + ** required for this VdbeCursor structure. It is convenient to use a + ** vdbe memory cell to manage the memory allocation required for a + ** VdbeCursor structure for the following reasons: + ** + ** * Sometimes cursor numbers are used for a couple of different + ** purposes in a vdbe program. The different uses might require + ** different sized allocations. Memory cells provide growable + ** allocations. + ** + ** * When using ENABLE_MEMORY_MANAGEMENT, memory cell buffers can + ** be freed lazily via the sqlite3_release_memory() API. This + ** minimizes the number of malloc calls made by the system. + ** + ** Memory cells for cursors are allocated at the top of the address + ** space. Memory cell (p->nMem) corresponds to cursor 0. Space for + ** cursor 1 is managed by memory cell (p->nMem-1), etc. + */ + Mem *pMem = &p->aMem[p->nMem-iCur]; + + int nByte; + VdbeCursor *pCx = 0; + nByte = + ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + + (isBtreeCursor?sqlite3BtreeCursorSize():0); + + assert( iCurnCursor ); + if( p->apCsr[iCur] ){ + sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); + p->apCsr[iCur] = 0; + } + if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; + memset(pCx, 0, sizeof(VdbeCursor)); + pCx->iDb = iDb; + pCx->nField = nField; + if( isBtreeCursor ){ + pCx->pCursor = (BtCursor*) + &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; + sqlite3BtreeCursorZero(pCx->pCursor); + } + } + return pCx; +} + +/* +** Try to convert a value into a numeric representation if we can +** do so without loss of information. In other words, if the string +** looks like a number, convert it into a number. If it does not +** look like a number, leave it alone. +*/ +static void applyNumericAffinity(Mem *pRec){ + double rValue; + i64 iValue; + u8 enc = pRec->enc; + if( (pRec->flags&MEM_Str)==0 ) return; + if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; + if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ + pRec->u.i = iValue; + pRec->flags |= MEM_Int; + }else{ + pRec->r = rValue; + pRec->flags |= MEM_Real; + } +} +#define ApplyNumericAffinity(X) \ + if(((X)->flags&(MEM_Real|MEM_Int))==0){applyNumericAffinity(X);} + +/* +** Processing is determine by the affinity parameter: +** +** SQLITE_AFF_INTEGER: +** SQLITE_AFF_REAL: +** SQLITE_AFF_NUMERIC: +** Try to convert pRec to an integer representation or a +** floating-point representation if an integer representation +** is not possible. Note that the integer representation is +** always preferred, even if the affinity is REAL, because +** an integer representation is more space efficient on disk. +** +** SQLITE_AFF_TEXT: +** Convert pRec to a text representation. +** +** SQLITE_AFF_NONE: +** No-op. pRec is unchanged. +*/ +static void applyAffinity( + Mem *pRec, /* The value to apply affinity to */ + char affinity, /* The affinity to be applied */ + u8 enc /* Use this text encoding */ +){ + if( affinity==SQLITE_AFF_TEXT ){ + /* Only attempt the conversion to TEXT if there is an integer or real + ** representation (blob and NULL do not get converted) but no string + ** representation. + */ + if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ + sqlite3VdbeMemStringify(pRec, enc); + } + pRec->flags &= ~(MEM_Real|MEM_Int); + }else if( affinity!=SQLITE_AFF_NONE ){ + assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL + || affinity==SQLITE_AFF_NUMERIC ); + ApplyNumericAffinity(pRec); + if( pRec->flags & MEM_Real ){ + sqlite3VdbeIntegerAffinity(pRec); + } + } +} + +/* +** Try to convert the type of a function argument or a result column +** into a numeric representation. Use either INTEGER or REAL whichever +** is appropriate. But only do the conversion if it is possible without +** loss of information and return the revised type of the argument. +*/ +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ + int eType = sqlite3_value_type(pVal); + if( eType==SQLITE_TEXT ){ + Mem *pMem = (Mem*)pVal; + applyNumericAffinity(pMem); + eType = sqlite3_value_type(pVal); + } + return eType; +} + +/* +** Exported version of applyAffinity(). This one works on sqlite3_value*, +** not the internal Mem* type. +*/ +SQLITE_PRIVATE void sqlite3ValueApplyAffinity( + sqlite3_value *pVal, + u8 affinity, + u8 enc +){ + applyAffinity((Mem *)pVal, affinity, enc); +} + +/* +** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or +** none. +** +** Unlike applyNumericAffinity(), this routine does not modify pMem->flags. +** But it does set pMem->r and pMem->u.i appropriately. +*/ +static u16 numericType(Mem *pMem){ + if( pMem->flags & (MEM_Int|MEM_Real) ){ + return pMem->flags & (MEM_Int|MEM_Real); + } + if( pMem->flags & (MEM_Str|MEM_Blob) ){ + if( sqlite3AtoF(pMem->z, &pMem->r, pMem->n, pMem->enc)==0 ){ + return 0; + } + if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ + return MEM_Int; + } + return MEM_Real; + } + return 0; +} + +#ifdef SQLITE_DEBUG +/* +** Write a nice string representation of the contents of cell pMem +** into buffer zBuf, length nBuf. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ + char *zCsr = zBuf; + int f = pMem->flags; + + static const char *const encnames[] = {"(X)", "(8)", "(16LE)", "(16BE)"}; + + if( f&MEM_Blob ){ + int i; + char c; + if( f & MEM_Dyn ){ + c = 'z'; + assert( (f & (MEM_Static|MEM_Ephem))==0 ); + }else if( f & MEM_Static ){ + c = 't'; + assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); + }else if( f & MEM_Ephem ){ + c = 'e'; + assert( (f & (MEM_Static|MEM_Dyn))==0 ); + }else{ + c = 's'; + } + + sqlite3_snprintf(100, zCsr, "%c", c); + zCsr += sqlite3Strlen30(zCsr); + sqlite3_snprintf(100, zCsr, "%d[", pMem->n); + zCsr += sqlite3Strlen30(zCsr); + for(i=0; i<16 && in; i++){ + sqlite3_snprintf(100, zCsr, "%02X", ((int)pMem->z[i] & 0xFF)); + zCsr += sqlite3Strlen30(zCsr); + } + for(i=0; i<16 && in; i++){ + char z = pMem->z[i]; + if( z<32 || z>126 ) *zCsr++ = '.'; + else *zCsr++ = z; + } + + sqlite3_snprintf(100, zCsr, "]%s", encnames[pMem->enc]); + zCsr += sqlite3Strlen30(zCsr); + if( f & MEM_Zero ){ + sqlite3_snprintf(100, zCsr,"+%dz",pMem->u.nZero); + zCsr += sqlite3Strlen30(zCsr); + } + *zCsr = '\0'; + }else if( f & MEM_Str ){ + int j, k; + zBuf[0] = ' '; + if( f & MEM_Dyn ){ + zBuf[1] = 'z'; + assert( (f & (MEM_Static|MEM_Ephem))==0 ); + }else if( f & MEM_Static ){ + zBuf[1] = 't'; + assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); + }else if( f & MEM_Ephem ){ + zBuf[1] = 'e'; + assert( (f & (MEM_Static|MEM_Dyn))==0 ); + }else{ + zBuf[1] = 's'; + } + k = 2; + sqlite3_snprintf(100, &zBuf[k], "%d", pMem->n); + k += sqlite3Strlen30(&zBuf[k]); + zBuf[k++] = '['; + for(j=0; j<15 && jn; j++){ + u8 c = pMem->z[j]; + if( c>=0x20 && c<0x7f ){ + zBuf[k++] = c; + }else{ + zBuf[k++] = '.'; + } + } + zBuf[k++] = ']'; + sqlite3_snprintf(100,&zBuf[k], encnames[pMem->enc]); + k += sqlite3Strlen30(&zBuf[k]); + zBuf[k++] = 0; + } +} +#endif + +#ifdef SQLITE_DEBUG +/* +** Print the value of a register for tracing purposes: +*/ +static void memTracePrint(Mem *p){ + if( p->flags & MEM_Undefined ){ + printf(" undefined"); + }else if( p->flags & MEM_Null ){ + printf(" NULL"); + }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ + printf(" si:%lld", p->u.i); + }else if( p->flags & MEM_Int ){ + printf(" i:%lld", p->u.i); +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( p->flags & MEM_Real ){ + printf(" r:%g", p->r); +#endif + }else if( p->flags & MEM_RowSet ){ + printf(" (rowset)"); + }else{ + char zBuf[200]; + sqlite3VdbeMemPrettyPrint(p, zBuf); + printf(" %s", zBuf); + } +} +static void registerTrace(int iReg, Mem *p){ + printf("REG[%d] = ", iReg); + memTracePrint(p); + printf("\n"); +} +#endif + +#ifdef SQLITE_DEBUG +# define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M) +#else +# define REGISTER_TRACE(R,M) +#endif + + +#ifdef VDBE_PROFILE + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +/************** Include hwtime.h in the middle of vdbe.c *********************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 class CPUs. +*/ +#ifndef _HWTIME_H_ +#define _HWTIME_H_ + +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; + } + +#elif (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + #error Need implementation of sqlite3Hwtime() for your platform. + + /* + ** To compile without implementing sqlite3Hwtime() for your platform, + ** you can remove the above #error and use the following + ** stub function. You will lose timing support for many + ** of the debugging and testing utilities, but it should at + ** least compile and run. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(_HWTIME_H_) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in vdbe.c ***********************/ + +#endif + +#ifndef NDEBUG +/* +** This function is only called from within an assert() expression. It +** checks that the sqlite3.nTransaction variable is correctly set to +** the number of non-transaction savepoints currently in the +** linked list starting at sqlite3.pSavepoint. +** +** Usage: +** +** assert( checkSavepointCount(db) ); +*/ +static int checkSavepointCount(sqlite3 *db){ + int n = 0; + Savepoint *p; + for(p=db->pSavepoint; p; p=p->pNext) n++; + assert( n==(db->nSavepoint + db->isTransactionSavepoint) ); + return 1; +} +#endif + + +/* +** Execute as much of a VDBE program as we can. +** This is the core of sqlite3_step(). +*/ +SQLITE_PRIVATE int sqlite3VdbeExec( + Vdbe *p /* The VDBE */ +){ + int pc=0; /* The program counter */ + Op *aOp = p->aOp; /* Copy of p->aOp */ + Op *pOp; /* Current operation */ + int rc = SQLITE_OK; /* Value to return */ + sqlite3 *db = p->db; /* The database */ + u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */ + u8 encoding = ENC(db); /* The database encoding */ + int iCompare = 0; /* Result of last OP_Compare operation */ + unsigned nVmStep = 0; /* Number of virtual machine steps */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + unsigned nProgressLimit = 0;/* Invoke xProgress() when nVmStep reaches this */ +#endif + Mem *aMem = p->aMem; /* Copy of p->aMem */ + Mem *pIn1 = 0; /* 1st input operand */ + Mem *pIn2 = 0; /* 2nd input operand */ + Mem *pIn3 = 0; /* 3rd input operand */ + Mem *pOut = 0; /* Output operand */ + int *aPermute = 0; /* Permutation of columns for OP_Compare */ + i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ +#ifdef VDBE_PROFILE + u64 start; /* CPU clock count at start of opcode */ +#endif + /*** INSERT STACK UNION HERE ***/ + + assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ + sqlite3VdbeEnter(p); + if( p->rc==SQLITE_NOMEM ){ + /* This happens if a malloc() inside a call to sqlite3_column_text() or + ** sqlite3_column_text16() failed. */ + goto no_mem; + } + assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY ); + assert( p->bIsReader || p->readOnly!=0 ); + p->rc = SQLITE_OK; + p->iCurrentTime = 0; + assert( p->explain==0 ); + p->pResultSet = 0; + db->busyHandler.nBusy = 0; + if( db->u1.isInterrupted ) goto abort_due_to_interrupt; + sqlite3VdbeIOTraceSql(p); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + assert( 0 < db->nProgressOps ); + nProgressLimit = (unsigned)p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; + if( nProgressLimit==0 ){ + nProgressLimit = db->nProgressOps; + }else{ + nProgressLimit %= (unsigned)db->nProgressOps; + } + } +#endif +#ifdef SQLITE_DEBUG + sqlite3BeginBenignMalloc(); + if( p->pc==0 + && (p->db->flags & (SQLITE_VdbeListing|SQLITE_VdbeEQP|SQLITE_VdbeTrace))!=0 + ){ + int i; + int once = 1; + sqlite3VdbePrintSql(p); + if( p->db->flags & SQLITE_VdbeListing ){ + printf("VDBE Program Listing:\n"); + for(i=0; inOp; i++){ + sqlite3VdbePrintOp(stdout, i, &aOp[i]); + } + } + if( p->db->flags & SQLITE_VdbeEQP ){ + for(i=0; inOp; i++){ + if( aOp[i].opcode==OP_Explain ){ + if( once ) printf("VDBE Query Plan:\n"); + printf("%s\n", aOp[i].p4.z); + once = 0; + } + } + } + if( p->db->flags & SQLITE_VdbeTrace ) printf("VDBE Trace:\n"); + } + sqlite3EndBenignMalloc(); +#endif + for(pc=p->pc; rc==SQLITE_OK; pc++){ + assert( pc>=0 && pcnOp ); + if( db->mallocFailed ) goto no_mem; +#ifdef VDBE_PROFILE + start = sqlite3Hwtime(); +#endif + nVmStep++; + pOp = &aOp[pc]; + + /* Only allow tracing if SQLITE_DEBUG is defined. + */ +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeTrace ){ + sqlite3VdbePrintOp(stdout, pc, pOp); + } +#endif + + + /* Check to see if we need to simulate an interrupt. This only happens + ** if we have a special test build. + */ +#ifdef SQLITE_TEST + if( sqlite3_interrupt_count>0 ){ + sqlite3_interrupt_count--; + if( sqlite3_interrupt_count==0 ){ + sqlite3_interrupt(db); + } + } +#endif + + /* On any opcode with the "out2-prerelease" tag, free any + ** external allocations out of mem[p2] and set mem[p2] to be + ** an undefined integer. Opcodes will either fill in the integer + ** value or convert mem[p2] to a different type. + */ + assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); + if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); + pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); + VdbeMemRelease(pOut); + pOut->flags = MEM_Int; + } + + /* Sanity checking on other operands */ +#ifdef SQLITE_DEBUG + if( (pOp->opflags & OPFLG_IN1)!=0 ){ + assert( pOp->p1>0 ); + assert( pOp->p1<=(p->nMem-p->nCursor) ); + assert( memIsValid(&aMem[pOp->p1]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); + REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); + } + if( (pOp->opflags & OPFLG_IN2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); + assert( memIsValid(&aMem[pOp->p2]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); + REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); + } + if( (pOp->opflags & OPFLG_IN3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( memIsValid(&aMem[pOp->p3]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); + REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); + } + if( (pOp->opflags & OPFLG_OUT2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); + memAboutToChange(p, &aMem[pOp->p2]); + } + if( (pOp->opflags & OPFLG_OUT3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); + memAboutToChange(p, &aMem[pOp->p3]); + } +#endif + + switch( pOp->opcode ){ + +/***************************************************************************** +** What follows is a massive switch statement where each case implements a +** separate instruction in the virtual machine. If we follow the usual +** indentation conventions, each case should be indented by 6 spaces. But +** that is a lot of wasted space on the left margin. So the code within +** the switch statement will break with convention and be flush-left. Another +** big comment (similar to this one) will mark the point in the code where +** we transition back to normal indentation. +** +** The formatting of each case is important. The makefile for SQLite +** generates two C files "opcodes.h" and "opcodes.c" by scanning this +** file looking for lines that begin with "case OP_". The opcodes.h files +** will be filled with #defines that give unique integer values to each +** opcode and the opcodes.c file is filled with an array of strings where +** each string is the symbolic name for the corresponding opcode. If the +** case statement is followed by a comment of the form "/# same as ... #/" +** that comment is used to determine the particular value of the opcode. +** +** Other keywords in the comment that follows each case are used to +** construct the OPFLG_INITIALIZER value that initializes opcodeProperty[]. +** Keywords include: in1, in2, in3, out2_prerelease, out2, out3. See +** the mkopcodeh.awk script for additional information. +** +** Documentation about VDBE opcodes is generated by scanning this file +** for lines of that contain "Opcode:". That line and all subsequent +** comment lines are used in the generation of the opcode.html documentation +** file. +** +** SUMMARY: +** +** Formatting is important to scripts that scan this file. +** Do not deviate from the formatting style currently in use. +** +*****************************************************************************/ + +/* Opcode: Goto * P2 * * * +** +** An unconditional jump to address P2. +** The next instruction executed will be +** the one at index P2 from the beginning of +** the program. +** +** The P1 parameter is not actually used by this opcode. However, it +** is sometimes set to 1 instead of 0 as a hint to the command-line shell +** that this Goto is the bottom of a loop and that the lines from P2 down +** to the current line should be indented for EXPLAIN output. +*/ +case OP_Goto: { /* jump */ + pc = pOp->p2 - 1; + + /* Opcodes that are used as the bottom of a loop (OP_Next, OP_Prev, + ** OP_VNext, OP_RowSetNext, or OP_SorterNext) all jump here upon + ** completion. Check to see if sqlite3_interrupt() has been called + ** or if the progress callback needs to be invoked. + ** + ** This code uses unstructured "goto" statements and does not look clean. + ** But that is not due to sloppy coding habits. The code is written this + ** way for performance, to avoid having to run the interrupt and progress + ** checks on every opcode. This helps sqlite3_step() to run about 1.5% + ** faster according to "valgrind --tool=cachegrind" */ +check_for_interrupt: + if( db->u1.isInterrupted ) goto abort_due_to_interrupt; +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + /* Call the progress callback if it is configured and the required number + ** of VDBE ops have been executed (either since this invocation of + ** sqlite3VdbeExec() or since last time the progress callback was called). + ** If the progress callback returns non-zero, exit the virtual machine with + ** a return code SQLITE_ABORT. + */ + if( db->xProgress!=0 && nVmStep>=nProgressLimit ){ + assert( db->nProgressOps!=0 ); + nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); + if( db->xProgress(db->pProgressArg) ){ + rc = SQLITE_INTERRUPT; + goto vdbe_error_halt; + } + } +#endif + + break; +} + +/* Opcode: Gosub P1 P2 * * * +** +** Write the current address onto register P1 +** and then jump to address P2. +*/ +case OP_Gosub: { /* jump */ + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + pIn1 = &aMem[pOp->p1]; + assert( VdbeMemDynamic(pIn1)==0 ); + memAboutToChange(p, pIn1); + pIn1->flags = MEM_Int; + pIn1->u.i = pc; + REGISTER_TRACE(pOp->p1, pIn1); + pc = pOp->p2 - 1; + break; +} + +/* Opcode: Return P1 * * * * +** +** Jump to the next instruction after the address in register P1. After +** the jump, register P1 becomes undefined. +*/ +case OP_Return: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags==MEM_Int ); + pc = (int)pIn1->u.i; + pIn1->flags = MEM_Undefined; + break; +} + +/* Opcode: InitCoroutine P1 P2 P3 * * +** +** Set up register P1 so that it will Yield to the coroutine +** located at address P3. +** +** If P2!=0 then the coroutine implementation immediately follows +** this opcode. So jump over the coroutine implementation to +** address P2. +** +** See also: EndCoroutine +*/ +case OP_InitCoroutine: { /* jump */ + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p2>=0 && pOp->p2nOp ); + assert( pOp->p3>=0 && pOp->p3nOp ); + pOut = &aMem[pOp->p1]; + assert( !VdbeMemDynamic(pOut) ); + pOut->u.i = pOp->p3 - 1; + pOut->flags = MEM_Int; + if( pOp->p2 ) pc = pOp->p2 - 1; + break; +} + +/* Opcode: EndCoroutine P1 * * * * +** +** The instruction at the address in register P1 is a Yield. +** Jump to the P2 parameter of that Yield. +** After the jump, register P1 becomes undefined. +** +** See also: InitCoroutine +*/ +case OP_EndCoroutine: { /* in1 */ + VdbeOp *pCaller; + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags==MEM_Int ); + assert( pIn1->u.i>=0 && pIn1->u.inOp ); + pCaller = &aOp[pIn1->u.i]; + assert( pCaller->opcode==OP_Yield ); + assert( pCaller->p2>=0 && pCaller->p2nOp ); + pc = pCaller->p2 - 1; + pIn1->flags = MEM_Undefined; + break; +} + +/* Opcode: Yield P1 P2 * * * +** +** Swap the program counter with the value in register P1. This +** has the effect of yielding to a coroutine. +** +** If the coroutine that is launched by this instruction ends with +** Yield or Return then continue to the next instruction. But if +** the coroutine launched by this instruction ends with +** EndCoroutine, then jump to P2 rather than continuing with the +** next instruction. +** +** See also: InitCoroutine +*/ +case OP_Yield: { /* in1, jump */ + int pcDest; + pIn1 = &aMem[pOp->p1]; + assert( VdbeMemDynamic(pIn1)==0 ); + pIn1->flags = MEM_Int; + pcDest = (int)pIn1->u.i; + pIn1->u.i = pc; + REGISTER_TRACE(pOp->p1, pIn1); + pc = pcDest; + break; +} + +/* Opcode: HaltIfNull P1 P2 P3 P4 P5 +** Synopsis: if r[P3]=null halt +** +** Check the value in register P3. If it is NULL then Halt using +** parameter P1, P2, and P4 as if this were a Halt instruction. If the +** value in register P3 is not NULL, then this routine is a no-op. +** The P5 parameter should be 1. +*/ +case OP_HaltIfNull: { /* in3 */ + pIn3 = &aMem[pOp->p3]; + if( (pIn3->flags & MEM_Null)==0 ) break; + /* Fall through into OP_Halt */ +} + +/* Opcode: Halt P1 P2 * P4 P5 +** +** Exit immediately. All open cursors, etc are closed +** automatically. +** +** P1 is the result code returned by sqlite3_exec(), sqlite3_reset(), +** or sqlite3_finalize(). For a normal halt, this should be SQLITE_OK (0). +** For errors, it can be some other value. If P1!=0 then P2 will determine +** whether or not to rollback the current transaction. Do not rollback +** if P2==OE_Fail. Do the rollback if P2==OE_Rollback. If P2==OE_Abort, +** then back out all changes that have occurred during this execution of the +** VDBE, but do not rollback the transaction. +** +** If P4 is not null then it is an error message string. +** +** P5 is a value between 0 and 4, inclusive, that modifies the P4 string. +** +** 0: (no change) +** 1: NOT NULL contraint failed: P4 +** 2: UNIQUE constraint failed: P4 +** 3: CHECK constraint failed: P4 +** 4: FOREIGN KEY constraint failed: P4 +** +** If P5 is not zero and P4 is NULL, then everything after the ":" is +** omitted. +** +** There is an implied "Halt 0 0 0" instruction inserted at the very end of +** every program. So a jump past the last instruction of the program +** is the same as executing Halt. +*/ +case OP_Halt: { + const char *zType; + const char *zLogFmt; + + if( pOp->p1==SQLITE_OK && p->pFrame ){ + /* Halt the sub-program. Return control to the parent frame. */ + VdbeFrame *pFrame = p->pFrame; + p->pFrame = pFrame->pParent; + p->nFrame--; + sqlite3VdbeSetChanges(db, p->nChange); + pc = sqlite3VdbeFrameRestore(pFrame); + lastRowid = db->lastRowid; + if( pOp->p2==OE_Ignore ){ + /* Instruction pc is the OP_Program that invoked the sub-program + ** currently being halted. If the p2 instruction of this OP_Halt + ** instruction is set to OE_Ignore, then the sub-program is throwing + ** an IGNORE exception. In this case jump to the address specified + ** as the p2 of the calling OP_Program. */ + pc = p->aOp[pc].p2-1; + } + aOp = p->aOp; + aMem = p->aMem; + break; + } + p->rc = pOp->p1; + p->errorAction = (u8)pOp->p2; + p->pc = pc; + if( p->rc ){ + if( pOp->p5 ){ + static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK", + "FOREIGN KEY" }; + assert( pOp->p5>=1 && pOp->p5<=4 ); + testcase( pOp->p5==1 ); + testcase( pOp->p5==2 ); + testcase( pOp->p5==3 ); + testcase( pOp->p5==4 ); + zType = azType[pOp->p5-1]; + }else{ + zType = 0; + } + assert( zType!=0 || pOp->p4.z!=0 ); + zLogFmt = "abort at %d in [%s]: %s"; + if( zType && pOp->p4.z ){ + sqlite3SetString(&p->zErrMsg, db, "%s constraint failed: %s", + zType, pOp->p4.z); + }else if( pOp->p4.z ){ + sqlite3SetString(&p->zErrMsg, db, "%s", pOp->p4.z); + }else{ + sqlite3SetString(&p->zErrMsg, db, "%s constraint failed", zType); + } + sqlite3_log(pOp->p1, zLogFmt, pc, p->zSql, p->zErrMsg); + } + rc = sqlite3VdbeHalt(p); + assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); + if( rc==SQLITE_BUSY ){ + p->rc = rc = SQLITE_BUSY; + }else{ + assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ); + assert( rc==SQLITE_OK || db->nDeferredCons>0 || db->nDeferredImmCons>0 ); + rc = p->rc ? SQLITE_ERROR : SQLITE_DONE; + } + goto vdbe_return; +} + +/* Opcode: Integer P1 P2 * * * +** Synopsis: r[P2]=P1 +** +** The 32-bit integer value P1 is written into register P2. +*/ +case OP_Integer: { /* out2-prerelease */ + pOut->u.i = pOp->p1; + break; +} + +/* Opcode: Int64 * P2 * P4 * +** Synopsis: r[P2]=P4 +** +** P4 is a pointer to a 64-bit integer value. +** Write that value into register P2. +*/ +case OP_Int64: { /* out2-prerelease */ + assert( pOp->p4.pI64!=0 ); + pOut->u.i = *pOp->p4.pI64; + break; +} + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* Opcode: Real * P2 * P4 * +** Synopsis: r[P2]=P4 +** +** P4 is a pointer to a 64-bit floating point value. +** Write that value into register P2. +*/ +case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ + pOut->flags = MEM_Real; + assert( !sqlite3IsNaN(*pOp->p4.pReal) ); + pOut->r = *pOp->p4.pReal; + break; +} +#endif + +/* Opcode: String8 * P2 * P4 * +** Synopsis: r[P2]='P4' +** +** P4 points to a nul terminated UTF-8 string. This opcode is transformed +** into a String before it is executed for the first time. During +** this transformation, the length of string P4 is computed and stored +** as the P1 parameter. +*/ +case OP_String8: { /* same as TK_STRING, out2-prerelease */ + assert( pOp->p4.z!=0 ); + pOp->opcode = OP_String; + pOp->p1 = sqlite3Strlen30(pOp->p4.z); + +#ifndef SQLITE_OMIT_UTF16 + if( encoding!=SQLITE_UTF8 ){ + rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); + if( rc==SQLITE_TOOBIG ) goto too_big; + if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; + assert( pOut->zMalloc==pOut->z ); + assert( VdbeMemDynamic(pOut)==0 ); + pOut->zMalloc = 0; + pOut->flags |= MEM_Static; + if( pOp->p4type==P4_DYNAMIC ){ + sqlite3DbFree(db, pOp->p4.z); + } + pOp->p4type = P4_DYNAMIC; + pOp->p4.z = pOut->z; + pOp->p1 = pOut->n; + } +#endif + if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + /* Fall through to the next case, OP_String */ +} + +/* Opcode: String P1 P2 * P4 * +** Synopsis: r[P2]='P4' (len=P1) +** +** The string value P4 of length P1 (bytes) is stored in register P2. +*/ +case OP_String: { /* out2-prerelease */ + assert( pOp->p4.z!=0 ); + pOut->flags = MEM_Str|MEM_Static|MEM_Term; + pOut->z = pOp->p4.z; + pOut->n = pOp->p1; + pOut->enc = encoding; + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Null P1 P2 P3 * * +** Synopsis: r[P2..P3]=NULL +** +** Write a NULL into registers P2. If P3 greater than P2, then also write +** NULL into register P3 and every register in between P2 and P3. If P3 +** is less than P2 (typically P3 is zero) then only register P2 is +** set to NULL. +** +** If the P1 value is non-zero, then also set the MEM_Cleared flag so that +** NULL values will not compare equal even if SQLITE_NULLEQ is set on +** OP_Ne or OP_Eq. +*/ +case OP_Null: { /* out2-prerelease */ + int cnt; + u16 nullFlag; + cnt = pOp->p3-pOp->p2; + assert( pOp->p3<=(p->nMem-p->nCursor) ); + pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; + while( cnt>0 ){ + pOut++; + memAboutToChange(p, pOut); + VdbeMemRelease(pOut); + pOut->flags = nullFlag; + cnt--; + } + break; +} + +/* Opcode: SoftNull P1 * * * * +** Synopsis: r[P1]=NULL +** +** Set register P1 to have the value NULL as seen by the OP_MakeRecord +** instruction, but do not free any string or blob memory associated with +** the register, so that if the value was a string or blob that was +** previously copied using OP_SCopy, the copies will continue to be valid. +*/ +case OP_SoftNull: { + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + pOut = &aMem[pOp->p1]; + pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined; + break; +} + +/* Opcode: Blob P1 P2 * P4 * +** Synopsis: r[P2]=P4 (len=P1) +** +** P4 points to a blob of data P1 bytes long. Store this +** blob in register P2. +*/ +case OP_Blob: { /* out2-prerelease */ + assert( pOp->p1 <= SQLITE_MAX_LENGTH ); + sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + pOut->enc = encoding; + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Variable P1 P2 * P4 * +** Synopsis: r[P2]=parameter(P1,P4) +** +** Transfer the values of bound parameter P1 into register P2 +** +** If the parameter is named, then its name appears in P4. +** The P4 value is used by sqlite3_bind_parameter_name(). +*/ +case OP_Variable: { /* out2-prerelease */ + Mem *pVar; /* Value being transferred */ + + assert( pOp->p1>0 && pOp->p1<=p->nVar ); + assert( pOp->p4.z==0 || pOp->p4.z==p->azVar[pOp->p1-1] ); + pVar = &p->aVar[pOp->p1 - 1]; + if( sqlite3VdbeMemTooBig(pVar) ){ + goto too_big; + } + sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Move P1 P2 P3 * * +** Synopsis: r[P2@P3]=r[P1@P3] +** +** Move the P3 values in register P1..P1+P3-1 over into +** registers P2..P2+P3-1. Registers P1..P1+P3-1 are +** left holding a NULL. It is an error for register ranges +** P1..P1+P3-1 and P2..P2+P3-1 to overlap. It is an error +** for P3 to be less than 1. +*/ +case OP_Move: { + char *zMalloc; /* Holding variable for allocated memory */ + int n; /* Number of registers left to copy */ + int p1; /* Register to copy from */ + int p2; /* Register to copy to */ + + n = pOp->p3; + p1 = pOp->p1; + p2 = pOp->p2; + assert( n>0 && p1>0 && p2>0 ); + assert( p1+n<=p2 || p2+n<=p1 ); + + pIn1 = &aMem[p1]; + pOut = &aMem[p2]; + do{ + assert( pOut<=&aMem[(p->nMem-p->nCursor)] ); + assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); + assert( memIsValid(pIn1) ); + memAboutToChange(p, pOut); + VdbeMemRelease(pOut); + zMalloc = pOut->zMalloc; + memcpy(pOut, pIn1, sizeof(Mem)); +#ifdef SQLITE_DEBUG + if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrom<&aMem[p1+pOp->p3] ){ + pOut->pScopyFrom += p1 - pOp->p2; + } +#endif + pIn1->flags = MEM_Undefined; + pIn1->xDel = 0; + pIn1->zMalloc = zMalloc; + REGISTER_TRACE(p2++, pOut); + pIn1++; + pOut++; + }while( --n ); + break; +} + +/* Opcode: Copy P1 P2 P3 * * +** Synopsis: r[P2@P3+1]=r[P1@P3+1] +** +** Make a copy of registers P1..P1+P3 into registers P2..P2+P3. +** +** This instruction makes a deep copy of the value. A duplicate +** is made of any string or blob constant. See also OP_SCopy. +*/ +case OP_Copy: { + int n; + + n = pOp->p3; + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + assert( pOut!=pIn1 ); + while( 1 ){ + sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); + Deephemeralize(pOut); +#ifdef SQLITE_DEBUG + pOut->pScopyFrom = 0; +#endif + REGISTER_TRACE(pOp->p2+pOp->p3-n, pOut); + if( (n--)==0 ) break; + pOut++; + pIn1++; + } + break; +} + +/* Opcode: SCopy P1 P2 * * * +** Synopsis: r[P2]=r[P1] +** +** Make a shallow copy of register P1 into register P2. +** +** This instruction makes a shallow copy of the value. If the value +** is a string or blob, then the copy is only a pointer to the +** original and hence if the original changes so will the copy. +** Worse, if the original is deallocated, the copy becomes invalid. +** Thus the program must guarantee that the original will not change +** during the lifetime of the copy. Use OP_Copy to make a complete +** copy. +*/ +case OP_SCopy: { /* out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + assert( pOut!=pIn1 ); + sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); +#ifdef SQLITE_DEBUG + if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; +#endif + break; +} + +/* Opcode: ResultRow P1 P2 * * * +** Synopsis: output=r[P1@P2] +** +** The registers P1 through P1+P2-1 contain a single row of +** results. This opcode causes the sqlite3_step() call to terminate +** with an SQLITE_ROW return code and it sets up the sqlite3_stmt +** structure to provide access to the r(P1)..r(P1+P2-1) values as +** the result row. +*/ +case OP_ResultRow: { + Mem *pMem; + int i; + assert( p->nResColumn==pOp->p2 ); + assert( pOp->p1>0 ); + assert( pOp->p1+pOp->p2<=(p->nMem-p->nCursor)+1 ); + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + /* Run the progress counter just before returning. + */ + if( db->xProgress!=0 + && nVmStep>=nProgressLimit + && db->xProgress(db->pProgressArg)!=0 + ){ + rc = SQLITE_INTERRUPT; + goto vdbe_error_halt; + } +#endif + + /* If this statement has violated immediate foreign key constraints, do + ** not return the number of rows modified. And do not RELEASE the statement + ** transaction. It needs to be rolled back. */ + if( SQLITE_OK!=(rc = sqlite3VdbeCheckFk(p, 0)) ){ + assert( db->flags&SQLITE_CountRows ); + assert( p->usesStmtJournal ); + break; + } + + /* If the SQLITE_CountRows flag is set in sqlite3.flags mask, then + ** DML statements invoke this opcode to return the number of rows + ** modified to the user. This is the only way that a VM that + ** opens a statement transaction may invoke this opcode. + ** + ** In case this is such a statement, close any statement transaction + ** opened by this VM before returning control to the user. This is to + ** ensure that statement-transactions are always nested, not overlapping. + ** If the open statement-transaction is not closed here, then the user + ** may step another VM that opens its own statement transaction. This + ** may lead to overlapping statement transactions. + ** + ** The statement transaction is never a top-level transaction. Hence + ** the RELEASE call below can never fail. + */ + assert( p->iStatement==0 || db->flags&SQLITE_CountRows ); + rc = sqlite3VdbeCloseStatement(p, SAVEPOINT_RELEASE); + if( NEVER(rc!=SQLITE_OK) ){ + break; + } + + /* Invalidate all ephemeral cursor row caches */ + p->cacheCtr = (p->cacheCtr + 2)|1; + + /* Make sure the results of the current row are \000 terminated + ** and have an assigned type. The results are de-ephemeralized as + ** a side effect. + */ + pMem = p->pResultSet = &aMem[pOp->p1]; + for(i=0; ip2; i++){ + assert( memIsValid(&pMem[i]) ); + Deephemeralize(&pMem[i]); + assert( (pMem[i].flags & MEM_Ephem)==0 + || (pMem[i].flags & (MEM_Str|MEM_Blob))==0 ); + sqlite3VdbeMemNulTerminate(&pMem[i]); + REGISTER_TRACE(pOp->p1+i, &pMem[i]); + } + if( db->mallocFailed ) goto no_mem; + + /* Return SQLITE_ROW + */ + p->pc = pc + 1; + rc = SQLITE_ROW; + goto vdbe_return; +} + +/* Opcode: Concat P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]+r[P1] +** +** Add the text in register P1 onto the end of the text in +** register P2 and store the result in register P3. +** If either the P1 or P2 text are NULL then store NULL in P3. +** +** P3 = P2 || P1 +** +** It is illegal for P1 and P3 to be the same register. Sometimes, +** if P3 is the same register as P2, the implementation is able +** to avoid a memcpy(). +*/ +case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ + i64 nByte; + + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + pOut = &aMem[pOp->p3]; + assert( pIn1!=pOut ); + if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); + break; + } + if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; + Stringify(pIn1, encoding); + Stringify(pIn2, encoding); + nByte = pIn1->n + pIn2->n; + if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ + goto no_mem; + } + MemSetTypeFlag(pOut, MEM_Str); + if( pOut!=pIn2 ){ + memcpy(pOut->z, pIn2->z, pIn2->n); + } + memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); + pOut->z[nByte]=0; + pOut->z[nByte+1] = 0; + pOut->flags |= MEM_Term; + pOut->n = (int)nByte; + pOut->enc = encoding; + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Add P1 P2 P3 * * +** Synopsis: r[P3]=r[P1]+r[P2] +** +** Add the value in register P1 to the value in register P2 +** and store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: Multiply P1 P2 P3 * * +** Synopsis: r[P3]=r[P1]*r[P2] +** +** +** Multiply the value in register P1 by the value in register P2 +** and store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: Subtract P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]-r[P1] +** +** Subtract the value in register P1 from the value in register P2 +** and store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: Divide P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]/r[P1] +** +** Divide the value in register P1 by the value in register P2 +** and store the result in register P3 (P3=P2/P1). If the value in +** register P1 is zero, then the result is NULL. If either input is +** NULL, the result is NULL. +*/ +/* Opcode: Remainder P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]%r[P1] +** +** Compute the remainder after integer register P2 is divided by +** register P1 and store the result in register P3. +** If the value in register P1 is zero the result is NULL. +** If either operand is NULL, the result is NULL. +*/ +case OP_Add: /* same as TK_PLUS, in1, in2, out3 */ +case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ +case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ +case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ +case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ + char bIntint; /* Started out as two integer operands */ + u16 flags; /* Combined MEM_* flags from both inputs */ + u16 type1; /* Numeric type of left operand */ + u16 type2; /* Numeric type of right operand */ + i64 iA; /* Integer value of left operand */ + i64 iB; /* Integer value of right operand */ + double rA; /* Real value of left operand */ + double rB; /* Real value of right operand */ + + pIn1 = &aMem[pOp->p1]; + type1 = numericType(pIn1); + pIn2 = &aMem[pOp->p2]; + type2 = numericType(pIn2); + pOut = &aMem[pOp->p3]; + flags = pIn1->flags | pIn2->flags; + if( (flags & MEM_Null)!=0 ) goto arithmetic_result_is_null; + if( (type1 & type2 & MEM_Int)!=0 ){ + iA = pIn1->u.i; + iB = pIn2->u.i; + bIntint = 1; + switch( pOp->opcode ){ + case OP_Add: if( sqlite3AddInt64(&iB,iA) ) goto fp_math; break; + case OP_Subtract: if( sqlite3SubInt64(&iB,iA) ) goto fp_math; break; + case OP_Multiply: if( sqlite3MulInt64(&iB,iA) ) goto fp_math; break; + case OP_Divide: { + if( iA==0 ) goto arithmetic_result_is_null; + if( iA==-1 && iB==SMALLEST_INT64 ) goto fp_math; + iB /= iA; + break; + } + default: { + if( iA==0 ) goto arithmetic_result_is_null; + if( iA==-1 ) iA = 1; + iB %= iA; + break; + } + } + pOut->u.i = iB; + MemSetTypeFlag(pOut, MEM_Int); + }else{ + bIntint = 0; +fp_math: + rA = sqlite3VdbeRealValue(pIn1); + rB = sqlite3VdbeRealValue(pIn2); + switch( pOp->opcode ){ + case OP_Add: rB += rA; break; + case OP_Subtract: rB -= rA; break; + case OP_Multiply: rB *= rA; break; + case OP_Divide: { + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + if( rA==(double)0 ) goto arithmetic_result_is_null; + rB /= rA; + break; + } + default: { + iA = (i64)rA; + iB = (i64)rB; + if( iA==0 ) goto arithmetic_result_is_null; + if( iA==-1 ) iA = 1; + rB = (double)(iB % iA); + break; + } + } +#ifdef SQLITE_OMIT_FLOATING_POINT + pOut->u.i = rB; + MemSetTypeFlag(pOut, MEM_Int); +#else + if( sqlite3IsNaN(rB) ){ + goto arithmetic_result_is_null; + } + pOut->r = rB; + MemSetTypeFlag(pOut, MEM_Real); + if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ + sqlite3VdbeIntegerAffinity(pOut); + } +#endif + } + break; + +arithmetic_result_is_null: + sqlite3VdbeMemSetNull(pOut); + break; +} + +/* Opcode: CollSeq P1 * * P4 +** +** P4 is a pointer to a CollSeq struct. If the next call to a user function +** or aggregate calls sqlite3GetFuncCollSeq(), this collation sequence will +** be returned. This is used by the built-in min(), max() and nullif() +** functions. +** +** If P1 is not zero, then it is a register that a subsequent min() or +** max() aggregate will set to 1 if the current row is not the minimum or +** maximum. The P1 register is initialized to 0 by this instruction. +** +** The interface used by the implementation of the aforementioned functions +** to retrieve the collation sequence set by this opcode is not available +** publicly, only to user functions defined in func.c. +*/ +case OP_CollSeq: { + assert( pOp->p4type==P4_COLLSEQ ); + if( pOp->p1 ){ + sqlite3VdbeMemSetInt64(&aMem[pOp->p1], 0); + } + break; +} + +/* Opcode: Function P1 P2 P3 P4 P5 +** Synopsis: r[P3]=func(r[P2@P5]) +** +** Invoke a user function (P4 is a pointer to a Function structure that +** defines the function) with P5 arguments taken from register P2 and +** successors. The result of the function is stored in register P3. +** Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** See also: AggStep and AggFinal +*/ +case OP_Function: { + int i; + Mem *pArg; + sqlite3_context ctx; + sqlite3_value **apVal; + int n; + + n = pOp->p5; + apVal = p->apArg; + assert( apVal || n==0 ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); + + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); + assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); + pArg = &aMem[pOp->p2]; + for(i=0; ip2+i, pArg); + } + + assert( pOp->p4type==P4_FUNCDEF ); + ctx.pFunc = pOp->p4.pFunc; + ctx.iOp = pc; + ctx.pVdbe = p; + + /* The output cell may already have a buffer allocated. Move + ** the pointer to ctx.s so in case the user-function can use + ** the already allocated buffer instead of allocating a new one. + */ + memcpy(&ctx.s, pOut, sizeof(Mem)); + pOut->flags = MEM_Null; + pOut->xDel = 0; + pOut->zMalloc = 0; + MemSetTypeFlag(&ctx.s, MEM_Null); + + ctx.fErrorOrAux = 0; + if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + assert( pOp>aOp ); + assert( pOp[-1].p4type==P4_COLLSEQ ); + assert( pOp[-1].opcode==OP_CollSeq ); + ctx.pColl = pOp[-1].p4.pColl; + } + db->lastRowid = lastRowid; + (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ + lastRowid = db->lastRowid; + + if( db->mallocFailed ){ + /* Even though a malloc() has failed, the implementation of the + ** user function may have called an sqlite3_result_XXX() function + ** to return a value. The following call releases any resources + ** associated with such a value. + */ + sqlite3VdbeMemRelease(&ctx.s); + goto no_mem; + } + + /* If the function returned an error, throw an exception */ + if( ctx.fErrorOrAux ){ + if( ctx.isError ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); + rc = ctx.isError; + } + sqlite3VdbeDeleteAuxData(p, pc, pOp->p1); + } + + /* Copy the result of the function into register P3 */ + sqlite3VdbeChangeEncoding(&ctx.s, encoding); + assert( pOut->flags==MEM_Null ); + memcpy(pOut, &ctx.s, sizeof(Mem)); + if( sqlite3VdbeMemTooBig(pOut) ){ + goto too_big; + } + +#if 0 + /* The app-defined function has done something that as caused this + ** statement to expire. (Perhaps the function called sqlite3_exec() + ** with a CREATE TABLE statement.) + */ + if( p->expired ) rc = SQLITE_ABORT; +#endif + + REGISTER_TRACE(pOp->p3, pOut); + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: BitAnd P1 P2 P3 * * +** Synopsis: r[P3]=r[P1]&r[P2] +** +** Take the bit-wise AND of the values in register P1 and P2 and +** store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: BitOr P1 P2 P3 * * +** Synopsis: r[P3]=r[P1]|r[P2] +** +** Take the bit-wise OR of the values in register P1 and P2 and +** store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: ShiftLeft P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]<>r[P1] +** +** Shift the integer value in register P2 to the right by the +** number of bits specified by the integer in register P1. +** Store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +case OP_BitAnd: /* same as TK_BITAND, in1, in2, out3 */ +case OP_BitOr: /* same as TK_BITOR, in1, in2, out3 */ +case OP_ShiftLeft: /* same as TK_LSHIFT, in1, in2, out3 */ +case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ + i64 iA; + u64 uA; + i64 iB; + u8 op; + + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + pOut = &aMem[pOp->p3]; + if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); + break; + } + iA = sqlite3VdbeIntValue(pIn2); + iB = sqlite3VdbeIntValue(pIn1); + op = pOp->opcode; + if( op==OP_BitAnd ){ + iA &= iB; + }else if( op==OP_BitOr ){ + iA |= iB; + }else if( iB!=0 ){ + assert( op==OP_ShiftRight || op==OP_ShiftLeft ); + + /* If shifting by a negative amount, shift in the other direction */ + if( iB<0 ){ + assert( OP_ShiftRight==OP_ShiftLeft+1 ); + op = 2*OP_ShiftLeft + 1 - op; + iB = iB>(-64) ? -iB : 64; + } + + if( iB>=64 ){ + iA = (iA>=0 || op==OP_ShiftLeft) ? 0 : -1; + }else{ + memcpy(&uA, &iA, sizeof(uA)); + if( op==OP_ShiftLeft ){ + uA <<= iB; + }else{ + uA >>= iB; + /* Sign-extend on a right shift of a negative number */ + if( iA<0 ) uA |= ((((u64)0xffffffff)<<32)|0xffffffff) << (64-iB); + } + memcpy(&iA, &uA, sizeof(iA)); + } + } + pOut->u.i = iA; + MemSetTypeFlag(pOut, MEM_Int); + break; +} + +/* Opcode: AddImm P1 P2 * * * +** Synopsis: r[P1]=r[P1]+P2 +** +** Add the constant P2 to the value in register P1. +** The result is always an integer. +** +** To force any register to be an integer, just add 0. +*/ +case OP_AddImm: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); + sqlite3VdbeMemIntegerify(pIn1); + pIn1->u.i += pOp->p2; + break; +} + +/* Opcode: MustBeInt P1 P2 * * * +** +** Force the value in register P1 to be an integer. If the value +** in P1 is not an integer and cannot be converted into an integer +** without data loss, then jump immediately to P2, or if P2==0 +** raise an SQLITE_MISMATCH exception. +*/ +case OP_MustBeInt: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + if( (pIn1->flags & MEM_Int)==0 ){ + applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); + VdbeBranchTaken((pIn1->flags&MEM_Int)==0, 2); + if( (pIn1->flags & MEM_Int)==0 ){ + if( pOp->p2==0 ){ + rc = SQLITE_MISMATCH; + goto abort_due_to_error; + }else{ + pc = pOp->p2 - 1; + break; + } + } + } + MemSetTypeFlag(pIn1, MEM_Int); + break; +} + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* Opcode: RealAffinity P1 * * * * +** +** If register P1 holds an integer convert it to a real value. +** +** This opcode is used when extracting information from a column that +** has REAL affinity. Such column values may still be stored as +** integers, for space efficiency, but after extraction we want them +** to have only a real value. +*/ +case OP_RealAffinity: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + if( pIn1->flags & MEM_Int ){ + sqlite3VdbeMemRealify(pIn1); + } + break; +} +#endif + +#ifndef SQLITE_OMIT_CAST +/* Opcode: ToText P1 * * * * +** +** Force the value in register P1 to be text. +** If the value is numeric, convert it to a string using the +** equivalent of sprintf(). Blob values are unchanged and +** are afterwards simply interpreted as text. +** +** A NULL value is not changed by this routine. It remains NULL. +*/ +case OP_ToText: { /* same as TK_TO_TEXT, in1 */ + pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); + if( pIn1->flags & MEM_Null ) break; + assert( MEM_Str==(MEM_Blob>>3) ); + pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; + applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); + rc = ExpandBlob(pIn1); + assert( pIn1->flags & MEM_Str || db->mallocFailed ); + pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + UPDATE_MAX_BLOBSIZE(pIn1); + break; +} + +/* Opcode: ToBlob P1 * * * * +** +** Force the value in register P1 to be a BLOB. +** If the value is numeric, convert it to a string first. +** Strings are simply reinterpreted as blobs with no change +** to the underlying data. +** +** A NULL value is not changed by this routine. It remains NULL. +*/ +case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ + pIn1 = &aMem[pOp->p1]; + if( pIn1->flags & MEM_Null ) break; + if( (pIn1->flags & MEM_Blob)==0 ){ + applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); + assert( pIn1->flags & MEM_Str || db->mallocFailed ); + MemSetTypeFlag(pIn1, MEM_Blob); + }else{ + pIn1->flags &= ~(MEM_TypeMask&~MEM_Blob); + } + UPDATE_MAX_BLOBSIZE(pIn1); + break; +} + +/* Opcode: ToNumeric P1 * * * * +** +** Force the value in register P1 to be numeric (either an +** integer or a floating-point number.) +** If the value is text or blob, try to convert it to an using the +** equivalent of atoi() or atof() and store 0 if no such conversion +** is possible. +** +** A NULL value is not changed by this routine. It remains NULL. +*/ +case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ + pIn1 = &aMem[pOp->p1]; + sqlite3VdbeMemNumerify(pIn1); + break; +} +#endif /* SQLITE_OMIT_CAST */ + +/* Opcode: ToInt P1 * * * * +** +** Force the value in register P1 to be an integer. If +** The value is currently a real number, drop its fractional part. +** If the value is text or blob, try to convert it to an integer using the +** equivalent of atoi() and store 0 if no such conversion is possible. +** +** A NULL value is not changed by this routine. It remains NULL. +*/ +case OP_ToInt: { /* same as TK_TO_INT, in1 */ + pIn1 = &aMem[pOp->p1]; + if( (pIn1->flags & MEM_Null)==0 ){ + sqlite3VdbeMemIntegerify(pIn1); + } + break; +} + +#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) +/* Opcode: ToReal P1 * * * * +** +** Force the value in register P1 to be a floating point number. +** If The value is currently an integer, convert it. +** If the value is text or blob, try to convert it to an integer using the +** equivalent of atoi() and store 0.0 if no such conversion is possible. +** +** A NULL value is not changed by this routine. It remains NULL. +*/ +case OP_ToReal: { /* same as TK_TO_REAL, in1 */ + pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); + if( (pIn1->flags & MEM_Null)==0 ){ + sqlite3VdbeMemRealify(pIn1); + } + break; +} +#endif /* !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) */ + +/* Opcode: Lt P1 P2 P3 P4 P5 +** Synopsis: if r[P1]r[P3] goto P2 +** +** This works just like the Lt opcode except that the jump is taken if +** the content of register P3 is greater than the content of +** register P1. See the Lt opcode for additional information. +*/ +/* Opcode: Ge P1 P2 P3 P4 P5 +** Synopsis: if r[P1]>=r[P3] goto P2 +** +** This works just like the Lt opcode except that the jump is taken if +** the content of register P3 is greater than or equal to the content of +** register P1. See the Lt opcode for additional information. +*/ +case OP_Eq: /* same as TK_EQ, jump, in1, in3 */ +case OP_Ne: /* same as TK_NE, jump, in1, in3 */ +case OP_Lt: /* same as TK_LT, jump, in1, in3 */ +case OP_Le: /* same as TK_LE, jump, in1, in3 */ +case OP_Gt: /* same as TK_GT, jump, in1, in3 */ +case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ + int res; /* Result of the comparison of pIn1 against pIn3 */ + char affinity; /* Affinity to use for comparison */ + u16 flags1; /* Copy of initial value of pIn1->flags */ + u16 flags3; /* Copy of initial value of pIn3->flags */ + + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; + flags1 = pIn1->flags; + flags3 = pIn3->flags; + if( (flags1 | flags3)&MEM_Null ){ + /* One or both operands are NULL */ + if( pOp->p5 & SQLITE_NULLEQ ){ + /* If SQLITE_NULLEQ is set (which will only happen if the operator is + ** OP_Eq or OP_Ne) then take the jump or not depending on whether + ** or not both operands are null. + */ + assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); + assert( (flags1 & MEM_Cleared)==0 ); + assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 ); + if( (flags1&MEM_Null)!=0 + && (flags3&MEM_Null)!=0 + && (flags3&MEM_Cleared)==0 + ){ + res = 0; /* Results are equal */ + }else{ + res = 1; /* Results are not equal */ + } + }else{ + /* SQLITE_NULLEQ is clear and at least one operand is NULL, + ** then the result is always NULL. + ** The jump is taken if the SQLITE_JUMPIFNULL bit is set. + */ + if( pOp->p5 & SQLITE_STOREP2 ){ + pOut = &aMem[pOp->p2]; + MemSetTypeFlag(pOut, MEM_Null); + REGISTER_TRACE(pOp->p2, pOut); + }else{ + VdbeBranchTaken(2,3); + if( pOp->p5 & SQLITE_JUMPIFNULL ){ + pc = pOp->p2-1; + } + } + break; + } + }else{ + /* Neither operand is NULL. Do a comparison. */ + affinity = pOp->p5 & SQLITE_AFF_MASK; + if( affinity ){ + applyAffinity(pIn1, affinity, encoding); + applyAffinity(pIn3, affinity, encoding); + if( db->mallocFailed ) goto no_mem; + } + + assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); + ExpandBlob(pIn1); + ExpandBlob(pIn3); + res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); + } + switch( pOp->opcode ){ + case OP_Eq: res = res==0; break; + case OP_Ne: res = res!=0; break; + case OP_Lt: res = res<0; break; + case OP_Le: res = res<=0; break; + case OP_Gt: res = res>0; break; + default: res = res>=0; break; + } + + if( pOp->p5 & SQLITE_STOREP2 ){ + pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); + MemSetTypeFlag(pOut, MEM_Int); + pOut->u.i = res; + REGISTER_TRACE(pOp->p2, pOut); + }else{ + VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); + if( res ){ + pc = pOp->p2-1; + } + } + /* Undo any changes made by applyAffinity() to the input registers. */ + pIn1->flags = (pIn1->flags&~MEM_TypeMask) | (flags1&MEM_TypeMask); + pIn3->flags = (pIn3->flags&~MEM_TypeMask) | (flags3&MEM_TypeMask); + break; +} + +/* Opcode: Permutation * * * P4 * +** +** Set the permutation used by the OP_Compare operator to be the array +** of integers in P4. +** +** The permutation is only valid until the next OP_Compare that has +** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should +** occur immediately prior to the OP_Compare. +*/ +case OP_Permutation: { + assert( pOp->p4type==P4_INTARRAY ); + assert( pOp->p4.ai ); + aPermute = pOp->p4.ai; + break; +} + +/* Opcode: Compare P1 P2 P3 P4 P5 +** Synopsis: r[P1@P3] <-> r[P2@P3] +** +** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this +** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of +** the comparison for use by the next OP_Jump instruct. +** +** If P5 has the OPFLAG_PERMUTE bit set, then the order of comparison is +** determined by the most recent OP_Permutation operator. If the +** OPFLAG_PERMUTE bit is clear, then register are compared in sequential +** order. +** +** P4 is a KeyInfo structure that defines collating sequences and sort +** orders for the comparison. The permutation applies to registers +** only. The KeyInfo elements are used sequentially. +** +** The comparison is a sort comparison, so NULLs compare equal, +** NULLs are less than numbers, numbers are less than strings, +** and strings are less than blobs. +*/ +case OP_Compare: { + int n; + int i; + int p1; + int p2; + const KeyInfo *pKeyInfo; + int idx; + CollSeq *pColl; /* Collating sequence to use on this term */ + int bRev; /* True for DESCENDING sort order */ + + if( (pOp->p5 & OPFLAG_PERMUTE)==0 ) aPermute = 0; + n = pOp->p3; + pKeyInfo = pOp->p4.pKeyInfo; + assert( n>0 ); + assert( pKeyInfo!=0 ); + p1 = pOp->p1; + p2 = pOp->p2; +#if SQLITE_DEBUG + if( aPermute ){ + int k, mx = 0; + for(k=0; kmx ) mx = aPermute[k]; + assert( p1>0 && p1+mx<=(p->nMem-p->nCursor)+1 ); + assert( p2>0 && p2+mx<=(p->nMem-p->nCursor)+1 ); + }else{ + assert( p1>0 && p1+n<=(p->nMem-p->nCursor)+1 ); + assert( p2>0 && p2+n<=(p->nMem-p->nCursor)+1 ); + } +#endif /* SQLITE_DEBUG */ + for(i=0; inField ); + pColl = pKeyInfo->aColl[i]; + bRev = pKeyInfo->aSortOrder[i]; + iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); + if( iCompare ){ + if( bRev ) iCompare = -iCompare; + break; + } + } + aPermute = 0; + break; +} + +/* Opcode: Jump P1 P2 P3 * * +** +** Jump to the instruction at address P1, P2, or P3 depending on whether +** in the most recent OP_Compare instruction the P1 vector was less than +** equal to, or greater than the P2 vector, respectively. +*/ +case OP_Jump: { /* jump */ + if( iCompare<0 ){ + pc = pOp->p1 - 1; VdbeBranchTaken(0,3); + }else if( iCompare==0 ){ + pc = pOp->p2 - 1; VdbeBranchTaken(1,3); + }else{ + pc = pOp->p3 - 1; VdbeBranchTaken(2,3); + } + break; +} + +/* Opcode: And P1 P2 P3 * * +** Synopsis: r[P3]=(r[P1] && r[P2]) +** +** Take the logical AND of the values in registers P1 and P2 and +** write the result into register P3. +** +** If either P1 or P2 is 0 (false) then the result is 0 even if +** the other input is NULL. A NULL and true or two NULLs give +** a NULL output. +*/ +/* Opcode: Or P1 P2 P3 * * +** Synopsis: r[P3]=(r[P1] || r[P2]) +** +** Take the logical OR of the values in register P1 and P2 and +** store the answer in register P3. +** +** If either P1 or P2 is nonzero (true) then the result is 1 (true) +** even if the other input is NULL. A NULL and false or two NULLs +** give a NULL output. +*/ +case OP_And: /* same as TK_AND, in1, in2, out3 */ +case OP_Or: { /* same as TK_OR, in1, in2, out3 */ + int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ + int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ + + pIn1 = &aMem[pOp->p1]; + if( pIn1->flags & MEM_Null ){ + v1 = 2; + }else{ + v1 = sqlite3VdbeIntValue(pIn1)!=0; + } + pIn2 = &aMem[pOp->p2]; + if( pIn2->flags & MEM_Null ){ + v2 = 2; + }else{ + v2 = sqlite3VdbeIntValue(pIn2)!=0; + } + if( pOp->opcode==OP_And ){ + static const unsigned char and_logic[] = { 0, 0, 0, 0, 1, 2, 0, 2, 2 }; + v1 = and_logic[v1*3+v2]; + }else{ + static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 }; + v1 = or_logic[v1*3+v2]; + } + pOut = &aMem[pOp->p3]; + if( v1==2 ){ + MemSetTypeFlag(pOut, MEM_Null); + }else{ + pOut->u.i = v1; + MemSetTypeFlag(pOut, MEM_Int); + } + break; +} + +/* Opcode: Not P1 P2 * * * +** Synopsis: r[P2]= !r[P1] +** +** Interpret the value in register P1 as a boolean value. Store the +** boolean complement in register P2. If the value in register P1 is +** NULL, then a NULL is stored in P2. +*/ +case OP_Not: { /* same as TK_NOT, in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); + }else{ + sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1)); + } + break; +} + +/* Opcode: BitNot P1 P2 * * * +** Synopsis: r[P1]= ~r[P1] +** +** Interpret the content of register P1 as an integer. Store the +** ones-complement of the P1 value into register P2. If P1 holds +** a NULL then store a NULL in P2. +*/ +case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); + }else{ + sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1)); + } + break; +} + +/* Opcode: Once P1 P2 * * * +** +** Check the "once" flag number P1. If it is set, jump to instruction P2. +** Otherwise, set the flag and fall through to the next instruction. +** In other words, this opcode causes all following opcodes up through P2 +** (but not including P2) to run just once and to be skipped on subsequent +** times through the loop. +** +** All "once" flags are initially cleared whenever a prepared statement +** first begins to run. +*/ +case OP_Once: { /* jump */ + assert( pOp->p1nOnceFlag ); + VdbeBranchTaken(p->aOnceFlag[pOp->p1]!=0, 2); + if( p->aOnceFlag[pOp->p1] ){ + pc = pOp->p2-1; + }else{ + p->aOnceFlag[pOp->p1] = 1; + } + break; +} + +/* Opcode: If P1 P2 P3 * * +** +** Jump to P2 if the value in register P1 is true. The value +** is considered true if it is numeric and non-zero. If the value +** in P1 is NULL then take the jump if and only if P3 is non-zero. +*/ +/* Opcode: IfNot P1 P2 P3 * * +** +** Jump to P2 if the value in register P1 is False. The value +** is considered false if it has a numeric value of zero. If the value +** in P1 is NULL then take the jump if and only if P3 is non-zero. +*/ +case OP_If: /* jump, in1 */ +case OP_IfNot: { /* jump, in1 */ + int c; + pIn1 = &aMem[pOp->p1]; + if( pIn1->flags & MEM_Null ){ + c = pOp->p3; + }else{ +#ifdef SQLITE_OMIT_FLOATING_POINT + c = sqlite3VdbeIntValue(pIn1)!=0; +#else + c = sqlite3VdbeRealValue(pIn1)!=0.0; +#endif + if( pOp->opcode==OP_IfNot ) c = !c; + } + VdbeBranchTaken(c!=0, 2); + if( c ){ + pc = pOp->p2-1; + } + break; +} + +/* Opcode: IsNull P1 P2 * * * +** Synopsis: if r[P1]==NULL goto P2 +** +** Jump to P2 if the value in register P1 is NULL. +*/ +case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ + pIn1 = &aMem[pOp->p1]; + VdbeBranchTaken( (pIn1->flags & MEM_Null)!=0, 2); + if( (pIn1->flags & MEM_Null)!=0 ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: NotNull P1 P2 * * * +** Synopsis: if r[P1]!=NULL goto P2 +** +** Jump to P2 if the value in register P1 is not NULL. +*/ +case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ + pIn1 = &aMem[pOp->p1]; + VdbeBranchTaken( (pIn1->flags & MEM_Null)==0, 2); + if( (pIn1->flags & MEM_Null)==0 ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: Column P1 P2 P3 P4 P5 +** Synopsis: r[P3]=PX +** +** Interpret the data that cursor P1 points to as a structure built using +** the MakeRecord instruction. (See the MakeRecord opcode for additional +** information about the format of the data.) Extract the P2-th column +** from this record. If there are less that (P2+1) +** values in the record, extract a NULL. +** +** The value extracted is stored in register P3. +** +** If the column contains fewer than P2 fields, then extract a NULL. Or, +** if the P4 argument is a P4_MEM use the value of the P4 argument as +** the result. +** +** If the OPFLAG_CLEARCACHE bit is set on P5 and P1 is a pseudo-table cursor, +** then the cache of the cursor is reset prior to extracting the column. +** The first OP_Column against a pseudo-table after the value of the content +** register has changed should have this bit set. +** +** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when +** the result is guaranteed to only be used as the argument of a length() +** or typeof() function, respectively. The loading of large blobs can be +** skipped for length() and all content loading can be skipped for typeof(). +*/ +case OP_Column: { + i64 payloadSize64; /* Number of bytes in the record */ + int p2; /* column number to retrieve */ + VdbeCursor *pC; /* The VDBE cursor */ + BtCursor *pCrsr; /* The BTree cursor */ + u32 *aType; /* aType[i] holds the numeric type of the i-th column */ + u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ + int len; /* The length of the serialized data for the column */ + int i; /* Loop counter */ + Mem *pDest; /* Where to write the extracted value */ + Mem sMem; /* For storing the record being decoded */ + const u8 *zData; /* Part of the record being decoded */ + const u8 *zHdr; /* Next unparsed byte of the header */ + const u8 *zEndHdr; /* Pointer to first byte after the header */ + u32 offset; /* Offset into the data */ + u32 szField; /* Number of bytes in the content of a field */ + u32 avail; /* Number of bytes of available data */ + u32 t; /* A type code from the record header */ + Mem *pReg; /* PseudoTable input register */ + + p2 = pOp->p2; + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( p2nField ); + aType = pC->aType; + aOffset = aType + pC->nField; +#ifndef SQLITE_OMIT_VIRTUALTABLE + assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */ +#endif + pCrsr = pC->pCursor; + assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* pCrsr NULL on PseudoTables */ + assert( pCrsr!=0 || pC->nullRow ); /* pC->nullRow on PseudoTables */ + + /* If the cursor cache is stale, bring it up-to-date */ + rc = sqlite3VdbeCursorMoveto(pC); + if( rc ) goto abort_due_to_error; + if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){ + if( pC->nullRow ){ + if( pCrsr==0 ){ + assert( pC->pseudoTableReg>0 ); + pReg = &aMem[pC->pseudoTableReg]; + assert( pReg->flags & MEM_Blob ); + assert( memIsValid(pReg) ); + pC->payloadSize = pC->szRow = avail = pReg->n; + pC->aRow = (u8*)pReg->z; + }else{ + MemSetTypeFlag(pDest, MEM_Null); + goto op_column_out; + } + }else{ + assert( pCrsr ); + if( pC->isTable==0 ){ + assert( sqlite3BtreeCursorIsValid(pCrsr) ); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64); + assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ + /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the + ** payload size, so it is impossible for payloadSize64 to be + ** larger than 32 bits. */ + assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 ); + pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail); + pC->payloadSize = (u32)payloadSize64; + }else{ + assert( sqlite3BtreeCursorIsValid(pCrsr) ); + VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize); + assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ + pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail); + } + assert( avail<=65536 ); /* Maximum page size is 64KiB */ + if( pC->payloadSize <= (u32)avail ){ + pC->szRow = pC->payloadSize; + }else{ + pC->szRow = avail; + } + if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + } + pC->cacheStatus = p->cacheCtr; + pC->iHdrOffset = getVarint32(pC->aRow, offset); + pC->nHdrParsed = 0; + aOffset[0] = offset; + if( availaRow does not have to hold the entire row, but it does at least + ** need to cover the header of the record. If pC->aRow does not contain + ** the complete header, then set it to zero, forcing the header to be + ** dynamically allocated. */ + pC->aRow = 0; + pC->szRow = 0; + } + + /* Make sure a corrupt database has not given us an oversize header. + ** Do this now to avoid an oversize memory allocation. + ** + ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte + ** types use so much data space that there can only be 4096 and 32 of + ** them, respectively. So the maximum header length results from a + ** 3-byte type for each of the maximum of 32768 columns plus three + ** extra bytes for the header length itself. 32768*3 + 3 = 98307. + */ + if( offset > 98307 || offset > pC->payloadSize ){ + rc = SQLITE_CORRUPT_BKPT; + goto op_column_error; + } + } + + /* Make sure at least the first p2+1 entries of the header have been + ** parsed and valid information is in aOffset[] and aType[]. + */ + if( pC->nHdrParsed<=p2 ){ + /* If there is more header available for parsing in the record, try + ** to extract additional fields up through the p2+1-th field + */ + if( pC->iHdrOffsetaRow==0 ){ + memset(&sMem, 0, sizeof(sMem)); + rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], + !pC->isTable, &sMem); + if( rc!=SQLITE_OK ){ + goto op_column_error; + } + zData = (u8*)sMem.z; + }else{ + zData = pC->aRow; + } + + /* Fill in aType[i] and aOffset[i] values through the p2-th field. */ + i = pC->nHdrParsed; + offset = aOffset[i]; + zHdr = zData + pC->iHdrOffset; + zEndHdr = zData + aOffset[0]; + assert( i<=p2 && zHdrnHdrParsed = i; + pC->iHdrOffset = (u32)(zHdr - zData); + if( pC->aRow==0 ){ + sqlite3VdbeMemRelease(&sMem); + sMem.flags = MEM_Null; + } + + /* If we have read more header data than was contained in the header, + ** or if the end of the last field appears to be past the end of the + ** record, or if the end of the last field appears to be before the end + ** of the record (when all fields present), then we must be dealing + ** with a corrupt database. + */ + if( (zHdr > zEndHdr) + || (offset > pC->payloadSize) + || (zHdr==zEndHdr && offset!=pC->payloadSize) + ){ + rc = SQLITE_CORRUPT_BKPT; + goto op_column_error; + } + } + + /* If after trying to extra new entries from the header, nHdrParsed is + ** still not up to p2, that means that the record has fewer than p2 + ** columns. So the result will be either the default value or a NULL. + */ + if( pC->nHdrParsed<=p2 ){ + if( pOp->p4type==P4_MEM ){ + sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); + }else{ + MemSetTypeFlag(pDest, MEM_Null); + } + goto op_column_out; + } + } + + /* Extract the content for the p2+1-th column. Control can only + ** reach this point if aOffset[p2], aOffset[p2+1], and aType[p2] are + ** all valid. + */ + assert( p2nHdrParsed ); + assert( rc==SQLITE_OK ); + assert( sqlite3VdbeCheckMemInvariants(pDest) ); + if( pC->szRow>=aOffset[p2+1] ){ + /* This is the common case where the desired content fits on the original + ** page - where the content is not on an overflow page */ + VdbeMemRelease(pDest); + sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest); + }else{ + /* This branch happens only when content is on overflow pages */ + t = aType[p2]; + if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 + && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) + || (len = sqlite3VdbeSerialTypeLen(t))==0 + ){ + /* Content is irrelevant for the typeof() function and for + ** the length(X) function if X is a blob. So we might as well use + ** bogus content rather than reading content from disk. NULL works + ** for text and blob and whatever is in the payloadSize64 variable + ** will work for everything else. Content is also irrelevant if + ** the content length is 0. */ + zData = t<=13 ? (u8*)&payloadSize64 : 0; + sMem.zMalloc = 0; + }else{ + memset(&sMem, 0, sizeof(sMem)); + sqlite3VdbeMemMove(&sMem, pDest); + rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, + &sMem); + if( rc!=SQLITE_OK ){ + goto op_column_error; + } + zData = (u8*)sMem.z; + } + sqlite3VdbeSerialGet(zData, t, pDest); + /* If we dynamically allocated space to hold the data (in the + ** sqlite3VdbeMemFromBtree() call above) then transfer control of that + ** dynamically allocated space over to the pDest structure. + ** This prevents a memory copy. */ + if( sMem.zMalloc ){ + assert( sMem.z==sMem.zMalloc ); + assert( VdbeMemDynamic(pDest)==0 ); + assert( (pDest->flags & (MEM_Blob|MEM_Str))==0 || pDest->z==sMem.z ); + pDest->flags &= ~(MEM_Ephem|MEM_Static); + pDest->flags |= MEM_Term; + pDest->z = sMem.z; + pDest->zMalloc = sMem.zMalloc; + } + } + pDest->enc = encoding; + +op_column_out: + Deephemeralize(pDest); +op_column_error: + UPDATE_MAX_BLOBSIZE(pDest); + REGISTER_TRACE(pOp->p3, pDest); + break; +} + +/* Opcode: Affinity P1 P2 * P4 * +** Synopsis: affinity(r[P1@P2]) +** +** Apply affinities to a range of P2 registers starting with P1. +** +** P4 is a string that is P2 characters long. The nth character of the +** string indicates the column affinity that should be used for the nth +** memory cell in the range. +*/ +case OP_Affinity: { + const char *zAffinity; /* The affinity to be applied */ + char cAff; /* A single character of affinity */ + + zAffinity = pOp->p4.z; + assert( zAffinity!=0 ); + assert( zAffinity[pOp->p2]==0 ); + pIn1 = &aMem[pOp->p1]; + while( (cAff = *(zAffinity++))!=0 ){ + assert( pIn1 <= &p->aMem[(p->nMem-p->nCursor)] ); + assert( memIsValid(pIn1) ); + applyAffinity(pIn1, cAff, encoding); + pIn1++; + } + break; +} + +/* Opcode: MakeRecord P1 P2 P3 P4 * +** Synopsis: r[P3]=mkrec(r[P1@P2]) +** +** Convert P2 registers beginning with P1 into the [record format] +** use as a data record in a database table or as a key +** in an index. The OP_Column opcode can decode the record later. +** +** P4 may be a string that is P2 characters long. The nth character of the +** string indicates the column affinity that should be used for the nth +** field of the index key. +** +** The mapping from character to affinity is given by the SQLITE_AFF_ +** macros defined in sqliteInt.h. +** +** If P4 is NULL then all index fields have the affinity NONE. +*/ +case OP_MakeRecord: { + u8 *zNewRecord; /* A buffer to hold the data for the new record */ + Mem *pRec; /* The new record */ + u64 nData; /* Number of bytes of data space */ + int nHdr; /* Number of bytes of header space */ + i64 nByte; /* Data space required for this record */ + int nZero; /* Number of zero bytes at the end of the record */ + int nVarint; /* Number of bytes in a varint */ + u32 serial_type; /* Type field */ + Mem *pData0; /* First field to be combined into the record */ + Mem *pLast; /* Last field of the record */ + int nField; /* Number of fields in the record */ + char *zAffinity; /* The affinity string for the record */ + int file_format; /* File format to use for encoding */ + int i; /* Space used in zNewRecord[] header */ + int j; /* Space used in zNewRecord[] content */ + int len; /* Length of a field */ + + /* Assuming the record contains N fields, the record format looks + ** like this: + ** + ** ------------------------------------------------------------------------ + ** | hdr-size | type 0 | type 1 | ... | type N-1 | data0 | ... | data N-1 | + ** ------------------------------------------------------------------------ + ** + ** Data(0) is taken from register P1. Data(1) comes from register P1+1 + ** and so froth. + ** + ** Each type field is a varint representing the serial type of the + ** corresponding data element (see sqlite3VdbeSerialType()). The + ** hdr-size field is also a varint which is the offset from the beginning + ** of the record to data0. + */ + nData = 0; /* Number of bytes of data space */ + nHdr = 0; /* Number of bytes of header space */ + nZero = 0; /* Number of zero bytes at the end of the record */ + nField = pOp->p1; + zAffinity = pOp->p4.z; + assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem-p->nCursor)+1 ); + pData0 = &aMem[nField]; + nField = pOp->p2; + pLast = &pData0[nField-1]; + file_format = p->minWriteFileFormat; + + /* Identify the output register */ + assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); + + /* Apply the requested affinity to all inputs + */ + assert( pData0<=pLast ); + if( zAffinity ){ + pRec = pData0; + do{ + applyAffinity(pRec++, *(zAffinity++), encoding); + assert( zAffinity[0]==0 || pRec<=pLast ); + }while( zAffinity[0] ); + } + + /* Loop through the elements that will make up the record to figure + ** out how much space is required for the new record. + */ + pRec = pLast; + do{ + assert( memIsValid(pRec) ); + serial_type = sqlite3VdbeSerialType(pRec, file_format); + len = sqlite3VdbeSerialTypeLen(serial_type); + if( pRec->flags & MEM_Zero ){ + if( nData ){ + sqlite3VdbeMemExpandBlob(pRec); + }else{ + nZero += pRec->u.nZero; + len -= pRec->u.nZero; + } + } + nData += len; + testcase( serial_type==127 ); + testcase( serial_type==128 ); + nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); + }while( (--pRec)>=pData0 ); + + /* Add the initial header varint and total the size */ + testcase( nHdr==126 ); + testcase( nHdr==127 ); + if( nHdr<=126 ){ + /* The common case */ + nHdr += 1; + }else{ + /* Rare case of a really large header */ + nVarint = sqlite3VarintLen(nHdr); + nHdr += nVarint; + if( nVarintdb->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + + /* Make sure the output register has a buffer large enough to store + ** the new record. The output register (pOp->p3) is not allowed to + ** be one of the input registers (because the following call to + ** sqlite3VdbeMemGrow() could clobber the value before it is used). + */ + if( sqlite3VdbeMemGrow(pOut, (int)nByte, 0) ){ + goto no_mem; + } + zNewRecord = (u8 *)pOut->z; + + /* Write the record */ + i = putVarint32(zNewRecord, nHdr); + j = nHdr; + assert( pData0<=pLast ); + pRec = pData0; + do{ + serial_type = sqlite3VdbeSerialType(pRec, file_format); + i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ + j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ + }while( (++pRec)<=pLast ); + assert( i==nHdr ); + assert( j==nByte ); + + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + pOut->n = (int)nByte; + pOut->flags = MEM_Blob; + pOut->xDel = 0; + if( nZero ){ + pOut->u.nZero = nZero; + pOut->flags |= MEM_Zero; + } + pOut->enc = SQLITE_UTF8; /* In case the blob is ever converted to text */ + REGISTER_TRACE(pOp->p3, pOut); + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Count P1 P2 * * * +** Synopsis: r[P2]=count() +** +** Store the number of entries (an integer value) in the table or index +** opened by cursor P1 in register P2 +*/ +#ifndef SQLITE_OMIT_BTREECOUNT +case OP_Count: { /* out2-prerelease */ + i64 nEntry; + BtCursor *pCrsr; + + pCrsr = p->apCsr[pOp->p1]->pCursor; + assert( pCrsr ); + nEntry = 0; /* Not needed. Only used to silence a warning. */ + rc = sqlite3BtreeCount(pCrsr, &nEntry); + pOut->u.i = nEntry; + break; +} +#endif + +/* Opcode: Savepoint P1 * * P4 * +** +** Open, release or rollback the savepoint named by parameter P4, depending +** on the value of P1. To open a new savepoint, P1==0. To release (commit) an +** existing savepoint, P1==1, or to rollback an existing savepoint P1==2. +*/ +case OP_Savepoint: { + int p1; /* Value of P1 operand */ + char *zName; /* Name of savepoint */ + int nName; + Savepoint *pNew; + Savepoint *pSavepoint; + Savepoint *pTmp; + int iSavepoint; + int ii; + + p1 = pOp->p1; + zName = pOp->p4.z; + + /* Assert that the p1 parameter is valid. Also that if there is no open + ** transaction, then there cannot be any savepoints. + */ + assert( db->pSavepoint==0 || db->autoCommit==0 ); + assert( p1==SAVEPOINT_BEGIN||p1==SAVEPOINT_RELEASE||p1==SAVEPOINT_ROLLBACK ); + assert( db->pSavepoint || db->isTransactionSavepoint==0 ); + assert( checkSavepointCount(db) ); + assert( p->bIsReader ); + + if( p1==SAVEPOINT_BEGIN ){ + if( db->nVdbeWrite>0 ){ + /* A new savepoint cannot be created if there are active write + ** statements (i.e. open read/write incremental blob handles). + */ + sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + }else{ + nName = sqlite3Strlen30(zName); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* This call is Ok even if this savepoint is actually a transaction + ** savepoint (and therefore should not prompt xSavepoint()) callbacks. + ** If this is a transaction savepoint being opened, it is guaranteed + ** that the db->aVTrans[] array is empty. */ + assert( db->autoCommit==0 || db->nVTrans==0 ); + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, + db->nStatement+db->nSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; +#endif + + /* Create a new savepoint structure. */ + pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+nName+1); + if( pNew ){ + pNew->zName = (char *)&pNew[1]; + memcpy(pNew->zName, zName, nName+1); + + /* If there is no open transaction, then mark this as a special + ** "transaction savepoint". */ + if( db->autoCommit ){ + db->autoCommit = 0; + db->isTransactionSavepoint = 1; + }else{ + db->nSavepoint++; + } + + /* Link the new savepoint into the database handle's list. */ + pNew->pNext = db->pSavepoint; + db->pSavepoint = pNew; + pNew->nDeferredCons = db->nDeferredCons; + pNew->nDeferredImmCons = db->nDeferredImmCons; + } + } + }else{ + iSavepoint = 0; + + /* Find the named savepoint. If there is no such savepoint, then an + ** an error is returned to the user. */ + for( + pSavepoint = db->pSavepoint; + pSavepoint && sqlite3StrICmp(pSavepoint->zName, zName); + pSavepoint = pSavepoint->pNext + ){ + iSavepoint++; + } + if( !pSavepoint ){ + sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName); + rc = SQLITE_ERROR; + }else if( db->nVdbeWrite>0 && p1==SAVEPOINT_RELEASE ){ + /* It is not possible to release (commit) a savepoint if there are + ** active write statements. + */ + sqlite3SetString(&p->zErrMsg, db, + "cannot release savepoint - SQL statements in progress" + ); + rc = SQLITE_BUSY; + }else{ + + /* Determine whether or not this is a transaction savepoint. If so, + ** and this is a RELEASE command, then the current transaction + ** is committed. + */ + int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint; + if( isTransaction && p1==SAVEPOINT_RELEASE ){ + if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + goto vdbe_return; + } + db->autoCommit = 1; + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = pc; + db->autoCommit = 0; + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; + } + db->isTransactionSavepoint = 0; + rc = p->rc; + }else{ + iSavepoint = db->nSavepoint - iSavepoint - 1; + if( p1==SAVEPOINT_ROLLBACK ){ + for(ii=0; iinDb; ii++){ + sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + } + } + for(ii=0; iinDb; ii++){ + rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + } + if( p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ + sqlite3ExpirePreparedStatements(db); + sqlite3ResetAllSchemasOfConnection(db); + db->flags = (db->flags | SQLITE_InternChanges); + } + } + + /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all + ** savepoints nested inside of the savepoint being operated on. */ + while( db->pSavepoint!=pSavepoint ){ + pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + db->nSavepoint--; + } + + /* If it is a RELEASE, then destroy the savepoint being operated on + ** too. If it is a ROLLBACK TO, then set the number of deferred + ** constraint violations present in the database to the value stored + ** when the savepoint was created. */ + if( p1==SAVEPOINT_RELEASE ){ + assert( pSavepoint==db->pSavepoint ); + db->pSavepoint = pSavepoint->pNext; + sqlite3DbFree(db, pSavepoint); + if( !isTransaction ){ + db->nSavepoint--; + } + }else{ + db->nDeferredCons = pSavepoint->nDeferredCons; + db->nDeferredImmCons = pSavepoint->nDeferredImmCons; + } + + if( !isTransaction ){ + rc = sqlite3VtabSavepoint(db, p1, iSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + } + } + } + + break; +} + +/* Opcode: AutoCommit P1 P2 * * * +** +** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll +** back any currently active btree transactions. If there are any active +** VMs (apart from this one), then a ROLLBACK fails. A COMMIT fails if +** there are active writing VMs or active VMs that use shared cache. +** +** This instruction causes the VM to halt. +*/ +case OP_AutoCommit: { + int desiredAutoCommit; + int iRollback; + int turnOnAC; + + desiredAutoCommit = pOp->p1; + iRollback = pOp->p2; + turnOnAC = desiredAutoCommit && !db->autoCommit; + assert( desiredAutoCommit==1 || desiredAutoCommit==0 ); + assert( desiredAutoCommit==1 || iRollback==0 ); + assert( db->nVdbeActive>0 ); /* At least this one VM is active */ + assert( p->bIsReader ); + +#if 0 + if( turnOnAC && iRollback && db->nVdbeActive>1 ){ + /* If this instruction implements a ROLLBACK and other VMs are + ** still running, and a transaction is active, return an error indicating + ** that the other VMs must complete first. + */ + sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + }else +#endif + if( turnOnAC && !iRollback && db->nVdbeWrite>0 ){ + /* If this instruction implements a COMMIT and other VMs are writing + ** return an error indicating that the other VMs must complete first. + */ + sqlite3SetString(&p->zErrMsg, db, "cannot commit transaction - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + }else if( desiredAutoCommit!=db->autoCommit ){ + if( iRollback ){ + assert( desiredAutoCommit==1 ); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); + db->autoCommit = 1; + }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + goto vdbe_return; + }else{ + db->autoCommit = (u8)desiredAutoCommit; + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = pc; + db->autoCommit = (u8)(1-desiredAutoCommit); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; + } + } + assert( db->nStatement==0 ); + sqlite3CloseSavepoints(db); + if( p->rc==SQLITE_OK ){ + rc = SQLITE_DONE; + }else{ + rc = SQLITE_ERROR; + } + goto vdbe_return; + }else{ + sqlite3SetString(&p->zErrMsg, db, + (!desiredAutoCommit)?"cannot start a transaction within a transaction":( + (iRollback)?"cannot rollback - no transaction is active": + "cannot commit - no transaction is active")); + + rc = SQLITE_ERROR; + } + break; +} + +/* Opcode: Transaction P1 P2 P3 P4 P5 +** +** Begin a transaction on database P1 if a transaction is not already +** active. +** If P2 is non-zero, then a write-transaction is started, or if a +** read-transaction is already active, it is upgraded to a write-transaction. +** If P2 is zero, then a read-transaction is started. +** +** P1 is the index of the database file on which the transaction is +** started. Index 0 is the main database file and index 1 is the +** file used for temporary tables. Indices of 2 or more are used for +** attached databases. +** +** If a write-transaction is started and the Vdbe.usesStmtJournal flag is +** true (this flag is set if the Vdbe may modify more than one row and may +** throw an ABORT exception), a statement transaction may also be opened. +** More specifically, a statement transaction is opened iff the database +** connection is currently not in autocommit mode, or if there are other +** active statements. A statement transaction allows the changes made by this +** VDBE to be rolled back after an error without having to roll back the +** entire transaction. If no error is encountered, the statement transaction +** will automatically commit when the VDBE halts. +** +** If P5!=0 then this opcode also checks the schema cookie against P3 +** and the schema generation counter against P4. +** The cookie changes its value whenever the database schema changes. +** This operation is used to detect when that the cookie has changed +** and that the current process needs to reread the schema. If the schema +** cookie in P3 differs from the schema cookie in the database header or +** if the schema generation counter in P4 differs from the current +** generation counter, then an SQLITE_SCHEMA error is raised and execution +** halts. The sqlite3_step() wrapper function might then reprepare the +** statement and rerun it from the beginning. +*/ +case OP_Transaction: { + Btree *pBt; + int iMeta; + int iGen; + + assert( p->bIsReader ); + assert( p->readOnly==0 || pOp->p2==0 ); + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); + if( pOp->p2 && (db->flags & SQLITE_QueryOnly)!=0 ){ + rc = SQLITE_READONLY; + goto abort_due_to_error; + } + pBt = db->aDb[pOp->p1].pBt; + + if( pBt ){ + rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); + if( rc==SQLITE_BUSY ){ + p->pc = pc; + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; + } + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + + if( pOp->p2 && p->usesStmtJournal + && (db->autoCommit==0 || db->nVdbeRead>1) + ){ + assert( sqlite3BtreeIsInTrans(pBt) ); + if( p->iStatement==0 ){ + assert( db->nStatement>=0 && db->nSavepoint>=0 ); + db->nStatement++; + p->iStatement = db->nSavepoint + db->nStatement; + } + + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, p->iStatement-1); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginStmt(pBt, p->iStatement); + } + + /* Store the current value of the database handles deferred constraint + ** counter. If the statement transaction needs to be rolled back, + ** the value of this counter needs to be restored too. */ + p->nStmtDefCons = db->nDeferredCons; + p->nStmtDefImmCons = db->nDeferredImmCons; + } + + /* Gather the schema version number for checking */ + sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta); + iGen = db->aDb[pOp->p1].pSchema->iGeneration; + }else{ + iGen = iMeta = 0; + } + assert( pOp->p5==0 || pOp->p4type==P4_INT32 ); + if( pOp->p5 && (iMeta!=pOp->p3 || iGen!=pOp->p4.i) ){ + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, "database schema has changed"); + /* If the schema-cookie from the database file matches the cookie + ** stored with the in-memory representation of the schema, do + ** not reload the schema from the database file. + ** + ** If virtual-tables are in use, this is not just an optimization. + ** Often, v-tables store their data in other SQLite tables, which + ** are queried from within xNext() and other v-table methods using + ** prepared queries. If such a query is out-of-date, we do not want to + ** discard the database schema, as the user code implementing the + ** v-table would have to be ready for the sqlite3_vtab structure itself + ** to be invalidated whenever sqlite3_step() is called from within + ** a v-table method. + */ + if( db->aDb[pOp->p1].pSchema->schema_cookie!=iMeta ){ + sqlite3ResetOneSchema(db, pOp->p1); + } + p->expired = 1; + rc = SQLITE_SCHEMA; + } + break; +} + +/* Opcode: ReadCookie P1 P2 P3 * * +** +** Read cookie number P3 from database P1 and write it into register P2. +** P3==1 is the schema version. P3==2 is the database format. +** P3==3 is the recommended pager cache size, and so forth. P1==0 is +** the main database file and P1==1 is the database file used to store +** temporary tables. +** +** There must be a read-lock on the database (either a transaction +** must be started or there must be an open cursor) before +** executing this instruction. +*/ +case OP_ReadCookie: { /* out2-prerelease */ + int iMeta; + int iDb; + int iCookie; + + assert( p->bIsReader ); + iDb = pOp->p1; + iCookie = pOp->p3; + assert( pOp->p3=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 ); + assert( DbMaskTest(p->btreeMask, iDb) ); + + sqlite3BtreeGetMeta(db->aDb[iDb].pBt, iCookie, (u32 *)&iMeta); + pOut->u.i = iMeta; + break; +} + +/* Opcode: SetCookie P1 P2 P3 * * +** +** Write the content of register P3 (interpreted as an integer) +** into cookie number P2 of database P1. P2==1 is the schema version. +** P2==2 is the database format. P2==3 is the recommended pager cache +** size, and so forth. P1==0 is the main database file and P1==1 is the +** database file used to store temporary tables. +** +** A transaction must be started before executing this opcode. +*/ +case OP_SetCookie: { /* in3 */ + Db *pDb; + assert( pOp->p2p1>=0 && pOp->p1nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); + assert( p->readOnly==0 ); + pDb = &db->aDb[pOp->p1]; + assert( pDb->pBt!=0 ); + assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); + pIn3 = &aMem[pOp->p3]; + sqlite3VdbeMemIntegerify(pIn3); + /* See note about index shifting on OP_ReadCookie */ + rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); + if( pOp->p2==BTREE_SCHEMA_VERSION ){ + /* When the schema cookie changes, record the new cookie internally */ + pDb->pSchema->schema_cookie = (int)pIn3->u.i; + db->flags |= SQLITE_InternChanges; + }else if( pOp->p2==BTREE_FILE_FORMAT ){ + /* Record changes in the file format */ + pDb->pSchema->file_format = (u8)pIn3->u.i; + } + if( pOp->p1==1 ){ + /* Invalidate all prepared statements whenever the TEMP database + ** schema is changed. Ticket #1644 */ + sqlite3ExpirePreparedStatements(db); + p->expired = 0; + } + break; +} + +/* Opcode: OpenRead P1 P2 P3 P4 P5 +** Synopsis: root=P2 iDb=P3 +** +** Open a read-only cursor for the database table whose root page is +** P2 in a database file. The database file is determined by P3. +** P3==0 means the main database, P3==1 means the database used for +** temporary tables, and P3>1 means used the corresponding attached +** database. Give the new cursor an identifier of P1. The P1 +** values need not be contiguous but all P1 values should be small integers. +** It is an error for P1 to be negative. +** +** If P5!=0 then use the content of register P2 as the root page, not +** the value of P2 itself. +** +** There will be a read lock on the database whenever there is an +** open cursor. If the database was unlocked prior to this instruction +** then a read lock is acquired as part of this instruction. A read +** lock allows other processes to read the database but prohibits +** any other process from modifying the database. The read lock is +** released when all cursors are closed. If this instruction attempts +** to get a read lock but fails, the script terminates with an +** SQLITE_BUSY error code. +** +** The P4 value may be either an integer (P4_INT32) or a pointer to +** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo +** structure, then said structure defines the content and collating +** sequence of the index being opened. Otherwise, if P4 is an integer +** value, it is set to the number of columns in the table. +** +** See also: OpenWrite, ReopenIdx +*/ +/* Opcode: ReopenIdx P1 P2 P3 P4 P5 +** Synopsis: root=P2 iDb=P3 +** +** The ReopenIdx opcode works exactly like ReadOpen except that it first +** checks to see if the cursor on P1 is already open with a root page +** number of P2 and if it is this opcode becomes a no-op. In other words, +** if the cursor is already open, do not reopen it. +** +** The ReopenIdx opcode may only be used with P5==0 and with P4 being +** a P4_KEYINFO object. Furthermore, the P3 value must be the same as +** every other ReopenIdx or OpenRead for the same cursor number. +** +** See the OpenRead opcode documentation for additional information. +*/ +/* Opcode: OpenWrite P1 P2 P3 P4 P5 +** Synopsis: root=P2 iDb=P3 +** +** Open a read/write cursor named P1 on the table or index whose root +** page is P2. Or if P5!=0 use the content of register P2 to find the +** root page. +** +** The P4 value may be either an integer (P4_INT32) or a pointer to +** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo +** structure, then said structure defines the content and collating +** sequence of the index being opened. Otherwise, if P4 is an integer +** value, it is set to the number of columns in the table, or to the +** largest index of any column of the table that is actually used. +** +** This instruction works just like OpenRead except that it opens the cursor +** in read/write mode. For a given table, there can be one or more read-only +** cursors or a single read/write cursor but not both. +** +** See also OpenRead. +*/ +case OP_ReopenIdx: { + VdbeCursor *pCur; + + assert( pOp->p5==0 ); + assert( pOp->p4type==P4_KEYINFO ); + pCur = p->apCsr[pOp->p1]; + if( pCur && pCur->pgnoRoot==(u32)pOp->p2 ){ + assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ + break; + } + /* If the cursor is not currently open or is open on a different + ** index, then fall through into OP_OpenRead to force a reopen */ +} +case OP_OpenRead: +case OP_OpenWrite: { + int nField; + KeyInfo *pKeyInfo; + int p2; + int iDb; + int wrFlag; + Btree *pX; + VdbeCursor *pCur; + Db *pDb; + + assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR))==pOp->p5 ); + assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 ); + assert( p->bIsReader ); + assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx + || p->readOnly==0 ); + + if( p->expired ){ + rc = SQLITE_ABORT; + break; + } + + nField = 0; + pKeyInfo = 0; + p2 = pOp->p2; + iDb = pOp->p3; + assert( iDb>=0 && iDbnDb ); + assert( DbMaskTest(p->btreeMask, iDb) ); + pDb = &db->aDb[iDb]; + pX = pDb->pBt; + assert( pX!=0 ); + if( pOp->opcode==OP_OpenWrite ){ + wrFlag = 1; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( pDb->pSchema->file_format < p->minWriteFileFormat ){ + p->minWriteFileFormat = pDb->pSchema->file_format; + } + }else{ + wrFlag = 0; + } + if( pOp->p5 & OPFLAG_P2ISREG ){ + assert( p2>0 ); + assert( p2<=(p->nMem-p->nCursor) ); + pIn2 = &aMem[p2]; + assert( memIsValid(pIn2) ); + assert( (pIn2->flags & MEM_Int)!=0 ); + sqlite3VdbeMemIntegerify(pIn2); + p2 = (int)pIn2->u.i; + /* The p2 value always comes from a prior OP_CreateTable opcode and + ** that opcode will always set the p2 value to 2 or more or else fail. + ** If there were a failure, the prepared statement would have halted + ** before reaching this instruction. */ + if( NEVER(p2<2) ) { + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } + } + if( pOp->p4type==P4_KEYINFO ){ + pKeyInfo = pOp->p4.pKeyInfo; + assert( pKeyInfo->enc==ENC(db) ); + assert( pKeyInfo->db==db ); + nField = pKeyInfo->nField+pKeyInfo->nXField; + }else if( pOp->p4type==P4_INT32 ){ + nField = pOp->p4.i; + } + assert( pOp->p1>=0 ); + assert( nField>=0 ); + testcase( nField==0 ); /* Table with INTEGER PRIMARY KEY and nothing else */ + pCur = allocateCursor(p, pOp->p1, nField, iDb, 1); + if( pCur==0 ) goto no_mem; + pCur->nullRow = 1; + pCur->isOrdered = 1; + pCur->pgnoRoot = p2; + rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); + pCur->pKeyInfo = pKeyInfo; + assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); + sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR)); + + /* Since it performs no memory allocation or IO, the only value that + ** sqlite3BtreeCursor() may return is SQLITE_OK. */ + assert( rc==SQLITE_OK ); + + /* Set the VdbeCursor.isTable variable. Previous versions of + ** SQLite used to check if the root-page flags were sane at this point + ** and report database corruption if they were not, but this check has + ** since moved into the btree layer. */ + pCur->isTable = pOp->p4type!=P4_KEYINFO; + break; +} + +/* Opcode: OpenEphemeral P1 P2 * P4 P5 +** Synopsis: nColumn=P2 +** +** Open a new cursor P1 to a transient table. +** The cursor is always opened read/write even if +** the main database is read-only. The ephemeral +** table is deleted automatically when the cursor is closed. +** +** P2 is the number of columns in the ephemeral table. +** The cursor points to a BTree table if P4==0 and to a BTree index +** if P4 is not 0. If P4 is not NULL, it points to a KeyInfo structure +** that defines the format of keys in the index. +** +** The P5 parameter can be a mask of the BTREE_* flags defined +** in btree.h. These flags control aspects of the operation of +** the btree. The BTREE_OMIT_JOURNAL and BTREE_SINGLE flags are +** added automatically. +*/ +/* Opcode: OpenAutoindex P1 P2 * P4 * +** Synopsis: nColumn=P2 +** +** This opcode works the same as OP_OpenEphemeral. It has a +** different name to distinguish its use. Tables created using +** by this opcode will be used for automatically created transient +** indices in joins. +*/ +case OP_OpenAutoindex: +case OP_OpenEphemeral: { + VdbeCursor *pCx; + KeyInfo *pKeyInfo; + + static const int vfsFlags = + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_TRANSIENT_DB; + assert( pOp->p1>=0 ); + assert( pOp->p2>=0 ); + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + if( pCx==0 ) goto no_mem; + pCx->nullRow = 1; + pCx->isEphemeral = 1; + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, + BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pCx->pBt, 1); + } + if( rc==SQLITE_OK ){ + /* If a transient index is required, create it by calling + ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before + ** opening it. If a transient table is required, just use the + ** automatically created table with root-page 1 (an BLOB_INTKEY table). + */ + if( (pKeyInfo = pOp->p4.pKeyInfo)!=0 ){ + int pgno; + assert( pOp->p4type==P4_KEYINFO ); + rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY | pOp->p5); + if( rc==SQLITE_OK ){ + assert( pgno==MASTER_ROOT+1 ); + assert( pKeyInfo->db==db ); + assert( pKeyInfo->enc==ENC(db) ); + pCx->pKeyInfo = pKeyInfo; + rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, pKeyInfo, pCx->pCursor); + } + pCx->isTable = 0; + }else{ + rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, 1, 0, pCx->pCursor); + pCx->isTable = 1; + } + } + pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); + break; +} + +/* Opcode: SorterOpen P1 P2 * P4 * +** +** This opcode works like OP_OpenEphemeral except that it opens +** a transient index that is specifically designed to sort large +** tables using an external merge-sort algorithm. +*/ +case OP_SorterOpen: { + VdbeCursor *pCx; + + assert( pOp->p1>=0 ); + assert( pOp->p2>=0 ); + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + if( pCx==0 ) goto no_mem; + pCx->pKeyInfo = pOp->p4.pKeyInfo; + assert( pCx->pKeyInfo->db==db ); + assert( pCx->pKeyInfo->enc==ENC(db) ); + rc = sqlite3VdbeSorterInit(db, pCx); + break; +} + +/* Opcode: OpenPseudo P1 P2 P3 * * +** Synopsis: P3 columns in r[P2] +** +** Open a new cursor that points to a fake table that contains a single +** row of data. The content of that one row is the content of memory +** register P2. In other words, cursor P1 becomes an alias for the +** MEM_Blob content contained in register P2. +** +** A pseudo-table created by this opcode is used to hold a single +** row output from the sorter so that the row can be decomposed into +** individual columns using the OP_Column opcode. The OP_Column opcode +** is the only cursor opcode that works with a pseudo-table. +** +** P3 is the number of fields in the records that will be stored by +** the pseudo-table. +*/ +case OP_OpenPseudo: { + VdbeCursor *pCx; + + assert( pOp->p1>=0 ); + assert( pOp->p3>=0 ); + pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, 0); + if( pCx==0 ) goto no_mem; + pCx->nullRow = 1; + pCx->pseudoTableReg = pOp->p2; + pCx->isTable = 1; + assert( pOp->p5==0 ); + break; +} + +/* Opcode: Close P1 * * * * +** +** Close a cursor previously opened as P1. If P1 is not +** currently open, this instruction is a no-op. +*/ +case OP_Close: { + assert( pOp->p1>=0 && pOp->p1nCursor ); + sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); + p->apCsr[pOp->p1] = 0; + break; +} + +/* Opcode: SeekGE P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as the key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the smallest entry that +** is greater than or equal to the key value. If there are no records +** greater than or equal to the key and P2 is not zero, then jump to P2. +** +** This opcode leaves the cursor configured to move in forward order, +** from the beginning toward the end. In other words, the cursor is +** configured to use Next, not Prev. +** +** See also: Found, NotFound, SeekLt, SeekGt, SeekLe +*/ +/* Opcode: SeekGT P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the smallest entry that +** is greater than the key value. If there are no records greater than +** the key and P2 is not zero, then jump to P2. +** +** This opcode leaves the cursor configured to move in forward order, +** from the beginning toward the end. In other words, the cursor is +** configured to use Next, not Prev. +** +** See also: Found, NotFound, SeekLt, SeekGe, SeekLe +*/ +/* Opcode: SeekLT P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the largest entry that +** is less than the key value. If there are no records less than +** the key and P2 is not zero, then jump to P2. +** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use Prev, not Next. +** +** See also: Found, NotFound, SeekGt, SeekGe, SeekLe +*/ +/* Opcode: SeekLE P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the largest entry that +** is less than or equal to the key value. If there are no records +** less than or equal to the key and P2 is not zero, then jump to P2. +** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use Prev, not Next. +** +** See also: Found, NotFound, SeekGt, SeekGe, SeekLt +*/ +case OP_SeekLT: /* jump, in3 */ +case OP_SeekLE: /* jump, in3 */ +case OP_SeekGE: /* jump, in3 */ +case OP_SeekGT: { /* jump, in3 */ + int res; + int oc; + VdbeCursor *pC; + UnpackedRecord r; + int nField; + i64 iKey; /* The rowid we are to seek to */ + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p2!=0 ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->pseudoTableReg==0 ); + assert( OP_SeekLE == OP_SeekLT+1 ); + assert( OP_SeekGE == OP_SeekLT+2 ); + assert( OP_SeekGT == OP_SeekLT+3 ); + assert( pC->isOrdered ); + assert( pC->pCursor!=0 ); + oc = pOp->opcode; + pC->nullRow = 0; +#ifdef SQLITE_DEBUG + pC->seekOp = pOp->opcode; +#endif + if( pC->isTable ){ + /* The input value in P3 might be of any type: integer, real, string, + ** blob, or NULL. But it needs to be an integer before we can do + ** the seek, so covert it. */ + pIn3 = &aMem[pOp->p3]; + ApplyNumericAffinity(pIn3); + iKey = sqlite3VdbeIntValue(pIn3); + pC->rowidIsValid = 0; + + /* If the P3 value could not be converted into an integer without + ** loss of information, then special processing is required... */ + if( (pIn3->flags & MEM_Int)==0 ){ + if( (pIn3->flags & MEM_Real)==0 ){ + /* If the P3 value cannot be converted into any kind of a number, + ** then the seek is not possible, so jump to P2 */ + pc = pOp->p2 - 1; VdbeBranchTaken(1,2); + break; + } + + /* If the approximation iKey is larger than the actual real search + ** term, substitute >= for > and < for <=. e.g. if the search term + ** is 4.9 and the integer approximation 5: + ** + ** (x > 4.9) -> (x >= 5) + ** (x <= 4.9) -> (x < 5) + */ + if( pIn3->r<(double)iKey ){ + assert( OP_SeekGE==(OP_SeekGT-1) ); + assert( OP_SeekLT==(OP_SeekLE-1) ); + assert( (OP_SeekLE & 0x0001)==(OP_SeekGT & 0x0001) ); + if( (oc & 0x0001)==(OP_SeekGT & 0x0001) ) oc--; + } + + /* If the approximation iKey is smaller than the actual real search + ** term, substitute <= for < and > for >=. */ + else if( pIn3->r>(double)iKey ){ + assert( OP_SeekLE==(OP_SeekLT+1) ); + assert( OP_SeekGT==(OP_SeekGE+1) ); + assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) ); + if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++; + } + } + rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + if( res==0 ){ + pC->rowidIsValid = 1; + pC->lastRowid = iKey; + } + }else{ + nField = pOp->p4.i; + assert( pOp->p4type==P4_INT32 ); + assert( nField>0 ); + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)nField; + + /* The next line of code computes as follows, only faster: + ** if( oc==OP_SeekGT || oc==OP_SeekLE ){ + ** r.default_rc = -1; + ** }else{ + ** r.default_rc = +1; + ** } + */ + r.default_rc = ((1 & (oc - OP_SeekLT)) ? -1 : +1); + assert( oc!=OP_SeekGT || r.default_rc==-1 ); + assert( oc!=OP_SeekLE || r.default_rc==-1 ); + assert( oc!=OP_SeekGE || r.default_rc==+1 ); + assert( oc!=OP_SeekLT || r.default_rc==+1 ); + + r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ipCursor, &r, 0, 0, &res); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + pC->rowidIsValid = 0; + } + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); + if( res<0 || (res==0 && oc==OP_SeekGT) ){ + res = 0; + rc = sqlite3BtreeNext(pC->pCursor, &res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + pC->rowidIsValid = 0; + }else{ + res = 0; + } + }else{ + assert( oc==OP_SeekLT || oc==OP_SeekLE ); + if( res>0 || (res==0 && oc==OP_SeekLT) ){ + res = 0; + rc = sqlite3BtreePrevious(pC->pCursor, &res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + pC->rowidIsValid = 0; + }else{ + /* res might be negative because the table is empty. Check to + ** see if this is the case. + */ + res = sqlite3BtreeEof(pC->pCursor); + } + } + assert( pOp->p2>0 ); + VdbeBranchTaken(res!=0,2); + if( res ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: Seek P1 P2 * * * +** Synopsis: intkey=r[P2] +** +** P1 is an open table cursor and P2 is a rowid integer. Arrange +** for P1 to move so that it points to the rowid given by P2. +** +** This is actually a deferred seek. Nothing actually happens until +** the cursor is used to read a record. That way, if no reads +** occur, no unnecessary I/O happens. +*/ +case OP_Seek: { /* in2 */ + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->pCursor!=0 ); + assert( pC->isTable ); + pC->nullRow = 0; + pIn2 = &aMem[pOp->p2]; + pC->movetoTarget = sqlite3VdbeIntValue(pIn2); + pC->rowidIsValid = 0; + pC->deferredMoveto = 1; + break; +} + + +/* Opcode: Found P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** is a prefix of any entry in P1 then a jump is made to P2 and +** P1 is left pointing at the matching entry. +** +** This operation leaves the cursor in a state where it can be +** advanced in the forward direction. The Next instruction will work, +** but not the Prev instruction. +** +** See also: NotFound, NoConflict, NotExists. SeekGe +*/ +/* Opcode: NotFound P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** is not the prefix of any entry in P1 then a jump is made to P2. If P1 +** does contain an entry whose prefix matches the P3/P4 record then control +** falls through to the next instruction and P1 is left pointing at the +** matching entry. +** +** This operation leaves the cursor in a state where it cannot be +** advanced in either direction. In other words, the Next and Prev +** opcodes do not work after this operation. +** +** See also: Found, NotExists, NoConflict +*/ +/* Opcode: NoConflict P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** contains any NULL value, jump immediately to P2. If all terms of the +** record are not-NULL then a check is done to determine if any row in the +** P1 index btree has a matching key prefix. If there are no matches, jump +** immediately to P2. If there is a match, fall through and leave the P1 +** cursor pointing to the matching row. +** +** This opcode is similar to OP_NotFound with the exceptions that the +** branch is always taken if any part of the search key input is NULL. +** +** This operation leaves the cursor in a state where it cannot be +** advanced in either direction. In other words, the Next and Prev +** opcodes do not work after this operation. +** +** See also: NotFound, Found, NotExists +*/ +case OP_NoConflict: /* jump, in3 */ +case OP_NotFound: /* jump, in3 */ +case OP_Found: { /* jump, in3 */ + int alreadyExists; + int ii; + VdbeCursor *pC; + int res; + char *pFree; + UnpackedRecord *pIdxKey; + UnpackedRecord r; + char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*4 + 7]; + +#ifdef SQLITE_TEST + if( pOp->opcode!=OP_NoConflict ) sqlite3_found_count++; +#endif + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); +#ifdef SQLITE_DEBUG + pC->seekOp = pOp->opcode; +#endif + pIn3 = &aMem[pOp->p3]; + assert( pC->pCursor!=0 ); + assert( pC->isTable==0 ); + pFree = 0; /* Not needed. Only used to suppress a compiler warning. */ + if( pOp->p4.i>0 ){ + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)pOp->p4.i; + r.aMem = pIn3; + for(ii=0; iip3+ii, &r.aMem[ii]); +#endif + } + pIdxKey = &r; + }else{ + pIdxKey = sqlite3VdbeAllocUnpackedRecord( + pC->pKeyInfo, aTempRec, sizeof(aTempRec), &pFree + ); + if( pIdxKey==0 ) goto no_mem; + assert( pIn3->flags & MEM_Blob ); + assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ + sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); + } + pIdxKey->default_rc = 0; + if( pOp->opcode==OP_NoConflict ){ + /* For the OP_NoConflict opcode, take the jump if any of the + ** input fields are NULL, since any key with a NULL will not + ** conflict */ + for(ii=0; iip2 - 1; VdbeBranchTaken(1,2); + break; + } + } + } + rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, pIdxKey, 0, 0, &res); + if( pOp->p4.i==0 ){ + sqlite3DbFree(db, pFree); + } + if( rc!=SQLITE_OK ){ + break; + } + pC->seekResult = res; + alreadyExists = (res==0); + pC->nullRow = 1-alreadyExists; + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + if( pOp->opcode==OP_Found ){ + VdbeBranchTaken(alreadyExists!=0,2); + if( alreadyExists ) pc = pOp->p2 - 1; + }else{ + VdbeBranchTaken(alreadyExists==0,2); + if( !alreadyExists ) pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: NotExists P1 P2 P3 * * +** Synopsis: intkey=r[P3] +** +** P1 is the index of a cursor open on an SQL table btree (with integer +** keys). P3 is an integer rowid. If P1 does not contain a record with +** rowid P3 then jump immediately to P2. If P1 does contain a record +** with rowid P3 then leave the cursor pointing at that record and fall +** through to the next instruction. +** +** The OP_NotFound opcode performs the same operation on index btrees +** (with arbitrary multi-value keys). +** +** This opcode leaves the cursor in a state where it cannot be advanced +** in either direction. In other words, the Next and Prev opcodes will +** not work following this opcode. +** +** See also: Found, NotFound, NoConflict +*/ +case OP_NotExists: { /* jump, in3 */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + u64 iKey; + + pIn3 = &aMem[pOp->p3]; + assert( pIn3->flags & MEM_Int ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); +#ifdef SQLITE_DEBUG + pC->seekOp = 0; +#endif + assert( pC->isTable ); + assert( pC->pseudoTableReg==0 ); + pCrsr = pC->pCursor; + assert( pCrsr!=0 ); + res = 0; + iKey = pIn3->u.i; + rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); + pC->lastRowid = pIn3->u.i; + pC->rowidIsValid = res==0 ?1:0; + pC->nullRow = 0; + pC->cacheStatus = CACHE_STALE; + pC->deferredMoveto = 0; + VdbeBranchTaken(res!=0,2); + if( res!=0 ){ + pc = pOp->p2 - 1; + assert( pC->rowidIsValid==0 ); + } + pC->seekResult = res; + break; +} + +/* Opcode: Sequence P1 P2 * * * +** Synopsis: r[P2]=cursor[P1].ctr++ +** +** Find the next available sequence number for cursor P1. +** Write the sequence number into register P2. +** The sequence number on the cursor is incremented after this +** instruction. +*/ +case OP_Sequence: { /* out2-prerelease */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( p->apCsr[pOp->p1]!=0 ); + pOut->u.i = p->apCsr[pOp->p1]->seqCount++; + break; +} + + +/* Opcode: NewRowid P1 P2 P3 * * +** Synopsis: r[P2]=rowid +** +** Get a new integer record number (a.k.a "rowid") used as the key to a table. +** The record number is not previously used as a key in the database +** table that cursor P1 points to. The new record number is written +** written to register P2. +** +** If P3>0 then P3 is a register in the root frame of this VDBE that holds +** the largest previously generated record number. No new record numbers are +** allowed to be less than this value. When this value reaches its maximum, +** an SQLITE_FULL error is generated. The P3 register is updated with the ' +** generated record number. This P3 mechanism is used to help implement the +** AUTOINCREMENT feature. +*/ +case OP_NewRowid: { /* out2-prerelease */ + i64 v; /* The new rowid */ + VdbeCursor *pC; /* Cursor of table to get the new rowid */ + int res; /* Result of an sqlite3BtreeLast() */ + int cnt; /* Counter to limit the number of searches */ + Mem *pMem; /* Register holding largest rowid for AUTOINCREMENT */ + VdbeFrame *pFrame; /* Root frame of VDBE */ + + v = 0; + res = 0; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + if( NEVER(pC->pCursor==0) ){ + /* The zero initialization above is all that is needed */ + }else{ + /* The next rowid or record number (different terms for the same + ** thing) is obtained in a two-step algorithm. + ** + ** First we attempt to find the largest existing rowid and add one + ** to that. But if the largest existing rowid is already the maximum + ** positive integer, we have to fall through to the second + ** probabilistic algorithm + ** + ** The second algorithm is to select a rowid at random and see if + ** it already exists in the table. If it does not exist, we have + ** succeeded. If the random rowid does exist, we select a new one + ** and try again, up to 100 times. + */ + assert( pC->isTable ); + +#ifdef SQLITE_32BIT_ROWID +# define MAX_ROWID 0x7fffffff +#else + /* Some compilers complain about constants of the form 0x7fffffffffffffff. + ** Others complain about 0x7ffffffffffffffffLL. The following macro seems + ** to provide the constant while making all compilers happy. + */ +# define MAX_ROWID (i64)( (((u64)0x7fffffff)<<32) | (u64)0xffffffff ) +#endif + + if( !pC->useRandomRowid ){ + rc = sqlite3BtreeLast(pC->pCursor, &res); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + if( res ){ + v = 1; /* IMP: R-61914-48074 */ + }else{ + assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); + rc = sqlite3BtreeKeySize(pC->pCursor, &v); + assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ + if( v>=MAX_ROWID ){ + pC->useRandomRowid = 1; + }else{ + v++; /* IMP: R-29538-34987 */ + } + } + } + +#ifndef SQLITE_OMIT_AUTOINCREMENT + if( pOp->p3 ){ + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3>0 ); + if( p->pFrame ){ + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3<=pFrame->nMem ); + pMem = &pFrame->aMem[pOp->p3]; + }else{ + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3<=(p->nMem-p->nCursor) ); + pMem = &aMem[pOp->p3]; + memAboutToChange(p, pMem); + } + assert( memIsValid(pMem) ); + + REGISTER_TRACE(pOp->p3, pMem); + sqlite3VdbeMemIntegerify(pMem); + assert( (pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ + if( pMem->u.i==MAX_ROWID || pC->useRandomRowid ){ + rc = SQLITE_FULL; /* IMP: R-12275-61338 */ + goto abort_due_to_error; + } + if( vu.i+1 ){ + v = pMem->u.i + 1; + } + pMem->u.i = v; + } +#endif + if( pC->useRandomRowid ){ + /* IMPLEMENTATION-OF: R-07677-41881 If the largest ROWID is equal to the + ** largest possible integer (9223372036854775807) then the database + ** engine starts picking positive candidate ROWIDs at random until + ** it finds one that is not previously used. */ + assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is + ** an AUTOINCREMENT table. */ + /* on the first attempt, simply do one more than previous */ + v = lastRowid; + v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ + v++; /* ensure non-zero */ + cnt = 0; + while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, + 0, &res))==SQLITE_OK) + && (res==0) + && (++cnt<100)){ + /* collision - try another random rowid */ + sqlite3_randomness(sizeof(v), &v); + if( cnt<5 ){ + /* try "small" random rowids for the initial attempts */ + v &= 0xffffff; + }else{ + v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ + } + v++; /* ensure non-zero */ + } + if( rc==SQLITE_OK && res==0 ){ + rc = SQLITE_FULL; /* IMP: R-38219-53002 */ + goto abort_due_to_error; + } + assert( v>0 ); /* EV: R-40812-03570 */ + } + pC->rowidIsValid = 0; + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + } + pOut->u.i = v; + break; +} + +/* Opcode: Insert P1 P2 P3 P4 P5 +** Synopsis: intkey=r[P3] data=r[P2] +** +** Write an entry into the table of cursor P1. A new entry is +** created if it doesn't already exist or the data for an existing +** entry is overwritten. The data is the value MEM_Blob stored in register +** number P2. The key is stored in register P3. The key must +** be a MEM_Int. +** +** If the OPFLAG_NCHANGE flag of P5 is set, then the row change count is +** incremented (otherwise not). If the OPFLAG_LASTROWID flag of P5 is set, +** then rowid is stored for subsequent return by the +** sqlite3_last_insert_rowid() function (otherwise it is unmodified). +** +** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of +** the last seek operation (OP_NotExists) was a success, then this +** operation will not attempt to find the appropriate row before doing +** the insert but will instead overwrite the row that the cursor is +** currently pointing to. Presumably, the prior OP_NotExists opcode +** has already positioned the cursor correctly. This is an optimization +** that boosts performance by avoiding redundant seeks. +** +** If the OPFLAG_ISUPDATE flag is set, then this opcode is part of an +** UPDATE operation. Otherwise (if the flag is clear) then this opcode +** is part of an INSERT operation. The difference is only important to +** the update hook. +** +** Parameter P4 may point to a string containing the table-name, or +** may be NULL. If it is not NULL, then the update-hook +** (sqlite3.xUpdateCallback) is invoked following a successful insert. +** +** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically +** allocated, then ownership of P2 is transferred to the pseudo-cursor +** and register P2 becomes ephemeral. If the cursor is changed, the +** value of register P2 will then change. Make sure this does not +** cause any problems.) +** +** This instruction only works on tables. The equivalent instruction +** for indices is OP_IdxInsert. +*/ +/* Opcode: InsertInt P1 P2 P3 P4 P5 +** Synopsis: intkey=P3 data=r[P2] +** +** This works exactly like OP_Insert except that the key is the +** integer value P3, not the value of the integer stored in register P3. +*/ +case OP_Insert: +case OP_InsertInt: { + Mem *pData; /* MEM cell holding data for the record to be inserted */ + Mem *pKey; /* MEM cell holding key for the record */ + i64 iKey; /* The integer ROWID or key for the record to be inserted */ + VdbeCursor *pC; /* Cursor to table into which insert is written */ + int nZero; /* Number of zero-bytes to append */ + int seekResult; /* Result of prior seek or 0 if no USESEEKRESULT flag */ + const char *zDb; /* database name - used by the update hook */ + const char *zTbl; /* Table name - used by the opdate hook */ + int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ + + pData = &aMem[pOp->p2]; + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( memIsValid(pData) ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->pCursor!=0 ); + assert( pC->pseudoTableReg==0 ); + assert( pC->isTable ); + REGISTER_TRACE(pOp->p2, pData); + + if( pOp->opcode==OP_Insert ){ + pKey = &aMem[pOp->p3]; + assert( pKey->flags & MEM_Int ); + assert( memIsValid(pKey) ); + REGISTER_TRACE(pOp->p3, pKey); + iKey = pKey->u.i; + }else{ + assert( pOp->opcode==OP_InsertInt ); + iKey = pOp->p3; + } + + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey; + if( pData->flags & MEM_Null ){ + pData->z = 0; + pData->n = 0; + }else{ + assert( pData->flags & (MEM_Blob|MEM_Str) ); + } + seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0); + if( pData->flags & MEM_Zero ){ + nZero = pData->u.nZero; + }else{ + nZero = 0; + } + rc = sqlite3BtreeInsert(pC->pCursor, 0, iKey, + pData->z, pData->n, nZero, + (pOp->p5 & OPFLAG_APPEND)!=0, seekResult + ); + pC->rowidIsValid = 0; + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + + /* Invoke the update-hook if required. */ + if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){ + zDb = db->aDb[pC->iDb].zName; + zTbl = pOp->p4.z; + op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); + assert( pC->isTable ); + db->xUpdateCallback(db->pUpdateArg, op, zDb, zTbl, iKey); + assert( pC->iDb>=0 ); + } + break; +} + +/* Opcode: Delete P1 P2 * P4 * +** +** Delete the record at which the P1 cursor is currently pointing. +** +** The cursor will be left pointing at either the next or the previous +** record in the table. If it is left pointing at the next record, then +** the next Next instruction will be a no-op. Hence it is OK to delete +** a record from within a Next loop. +** +** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is +** incremented (otherwise not). +** +** P1 must not be pseudo-table. It has to be a real table with +** multiple rows. +** +** If P4 is not NULL, then it is the name of the table that P1 is +** pointing to. The update hook will be invoked, if it exists. +** If P4 is not NULL then the P1 cursor must have been positioned +** using OP_NotFound prior to invoking this opcode. +*/ +case OP_Delete: { + i64 iKey; + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ + iKey = pC->lastRowid; /* Only used for the update hook */ + + /* The OP_Delete opcode always follows an OP_NotExists or OP_Last or + ** OP_Column on the same table without any intervening operations that + ** might move or invalidate the cursor. Hence cursor pC is always pointing + ** to the row to be deleted and the sqlite3VdbeCursorMoveto() operation + ** below is always a no-op and cannot fail. We will run it anyhow, though, + ** to guard against future changes to the code generator. + **/ + assert( pC->deferredMoveto==0 ); + rc = sqlite3VdbeCursorMoveto(pC); + if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + + rc = sqlite3BtreeDelete(pC->pCursor); + pC->cacheStatus = CACHE_STALE; + + /* Invoke the update-hook if required. */ + if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){ + db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, + db->aDb[pC->iDb].zName, pOp->p4.z, iKey); + assert( pC->iDb>=0 ); + } + if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++; + break; +} +/* Opcode: ResetCount * * * * * +** +** The value of the change counter is copied to the database handle +** change counter (returned by subsequent calls to sqlite3_changes()). +** Then the VMs internal change counter resets to 0. +** This is used by trigger programs. +*/ +case OP_ResetCount: { + sqlite3VdbeSetChanges(db, p->nChange); + p->nChange = 0; + break; +} + +/* Opcode: SorterCompare P1 P2 P3 P4 +** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 +** +** P1 is a sorter cursor. This instruction compares a prefix of the +** record blob in register P3 against a prefix of the entry that +** the sorter cursor currently points to. Only the first P4 fields +** of r[P3] and the sorter record are compared. +** +** If either P3 or the sorter contains a NULL in one of their significant +** fields (not counting the P4 fields at the end which are ignored) then +** the comparison is assumed to be equal. +** +** Fall through to next instruction if the two records compare equal to +** each other. Jump to P2 if they are different. +*/ +case OP_SorterCompare: { + VdbeCursor *pC; + int res; + int nKeyCol; + + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + assert( pOp->p4type==P4_INT32 ); + pIn3 = &aMem[pOp->p3]; + nKeyCol = pOp->p4.i; + rc = sqlite3VdbeSorterCompare(pC, pIn3, nKeyCol, &res); + VdbeBranchTaken(res!=0,2); + if( res ){ + pc = pOp->p2-1; + } + break; +}; + +/* Opcode: SorterData P1 P2 * * * +** Synopsis: r[P2]=data +** +** Write into register P2 the current sorter data for sorter cursor P1. +*/ +case OP_SorterData: { + VdbeCursor *pC; + + pOut = &aMem[pOp->p2]; + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + rc = sqlite3VdbeSorterRowkey(pC, pOut); + assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); + break; +} + +/* Opcode: RowData P1 P2 * * * +** Synopsis: r[P2]=data +** +** Write into register P2 the complete row data for cursor P1. +** There is no interpretation of the data. +** It is just copied onto the P2 register exactly as +** it is found in the database file. +** +** If the P1 cursor must be pointing to a valid row (not a NULL row) +** of a real table, not a pseudo-table. +*/ +/* Opcode: RowKey P1 P2 * * * +** Synopsis: r[P2]=key +** +** Write into register P2 the complete row key for cursor P1. +** There is no interpretation of the data. +** The key is copied onto the P2 register exactly as +** it is found in the database file. +** +** If the P1 cursor must be pointing to a valid row (not a NULL row) +** of a real table, not a pseudo-table. +*/ +case OP_RowKey: +case OP_RowData: { + VdbeCursor *pC; + BtCursor *pCrsr; + u32 n; + i64 n64; + + pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); + + /* Note that RowKey and RowData are really exactly the same instruction */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC)==0 ); + assert( pC->isTable || pOp->opcode!=OP_RowData ); + assert( pC->isTable==0 || pOp->opcode==OP_RowData ); + assert( pC!=0 ); + assert( pC->nullRow==0 ); + assert( pC->pseudoTableReg==0 ); + assert( pC->pCursor!=0 ); + pCrsr = pC->pCursor; + assert( sqlite3BtreeCursorIsValid(pCrsr) ); + + /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or + ** OP_Rewind/Op_Next with no intervening instructions that might invalidate + ** the cursor. Hence the following sqlite3VdbeCursorMoveto() call is always + ** a no-op and can never fail. But we leave it in place as a safety. + */ + assert( pC->deferredMoveto==0 ); + rc = sqlite3VdbeCursorMoveto(pC); + if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + + if( pC->isTable==0 ){ + assert( !pC->isTable ); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64); + assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ + if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + n = (u32)n64; + }else{ + VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &n); + assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ + if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + } + if( sqlite3VdbeMemGrow(pOut, n, 0) ){ + goto no_mem; + } + pOut->n = n; + MemSetTypeFlag(pOut, MEM_Blob); + if( pC->isTable==0 ){ + rc = sqlite3BtreeKey(pCrsr, 0, n, pOut->z); + }else{ + rc = sqlite3BtreeData(pCrsr, 0, n, pOut->z); + } + pOut->enc = SQLITE_UTF8; /* In case the blob is ever cast to text */ + UPDATE_MAX_BLOBSIZE(pOut); + REGISTER_TRACE(pOp->p2, pOut); + break; +} + +/* Opcode: Rowid P1 P2 * * * +** Synopsis: r[P2]=rowid +** +** Store in register P2 an integer which is the key of the table entry that +** P1 is currently point to. +** +** P1 can be either an ordinary table or a virtual table. There used to +** be a separate OP_VRowid opcode for use with virtual tables, but this +** one opcode now works for both table types. +*/ +case OP_Rowid: { /* out2-prerelease */ + VdbeCursor *pC; + i64 v; + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->pseudoTableReg==0 || pC->nullRow ); + if( pC->nullRow ){ + pOut->flags = MEM_Null; + break; + }else if( pC->deferredMoveto ){ + v = pC->movetoTarget; +#ifndef SQLITE_OMIT_VIRTUALTABLE + }else if( pC->pVtabCursor ){ + pVtab = pC->pVtabCursor->pVtab; + pModule = pVtab->pModule; + assert( pModule->xRowid ); + rc = pModule->xRowid(pC->pVtabCursor, &v); + sqlite3VtabImportErrmsg(p, pVtab); +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + }else{ + assert( pC->pCursor!=0 ); + rc = sqlite3VdbeCursorMoveto(pC); + if( rc ) goto abort_due_to_error; + if( pC->rowidIsValid ){ + v = pC->lastRowid; + }else{ + rc = sqlite3BtreeKeySize(pC->pCursor, &v); + assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */ + } + } + pOut->u.i = v; + break; +} + +/* Opcode: NullRow P1 * * * * +** +** Move the cursor P1 to a null row. Any OP_Column operations +** that occur while the cursor is on the null row will always +** write a NULL. +*/ +case OP_NullRow: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pC->nullRow = 1; + pC->rowidIsValid = 0; + pC->cacheStatus = CACHE_STALE; + if( pC->pCursor ){ + sqlite3BtreeClearCursor(pC->pCursor); + } + break; +} + +/* Opcode: Last P1 P2 * * * +** +** The next use of the Rowid or Column or Prev instruction for P1 +** will refer to the last entry in the database table or index. +** If the table or index is empty and P2>0, then jump immediately to P2. +** If P2 is 0 or if the table or index is not empty, fall through +** to the following instruction. +** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use Prev, not Next. +*/ +case OP_Last: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pCrsr = pC->pCursor; + res = 0; + assert( pCrsr!=0 ); + rc = sqlite3BtreeLast(pCrsr, &res); + pC->nullRow = (u8)res; + pC->deferredMoveto = 0; + pC->rowidIsValid = 0; + pC->cacheStatus = CACHE_STALE; +#ifdef SQLITE_DEBUG + pC->seekOp = OP_Last; +#endif + if( pOp->p2>0 ){ + VdbeBranchTaken(res!=0,2); + if( res ) pc = pOp->p2 - 1; + } + break; +} + + +/* Opcode: Sort P1 P2 * * * +** +** This opcode does exactly the same thing as OP_Rewind except that +** it increments an undocumented global variable used for testing. +** +** Sorting is accomplished by writing records into a sorting index, +** then rewinding that index and playing it back from beginning to +** end. We use the OP_Sort opcode instead of OP_Rewind to do the +** rewinding so that the global variable will be incremented and +** regression tests can determine whether or not the optimizer is +** correctly optimizing out sorts. +*/ +case OP_SorterSort: /* jump */ +case OP_Sort: { /* jump */ +#ifdef SQLITE_TEST + sqlite3_sort_count++; + sqlite3_search_count--; +#endif + p->aCounter[SQLITE_STMTSTATUS_SORT]++; + /* Fall through into OP_Rewind */ +} +/* Opcode: Rewind P1 P2 * * * +** +** The next use of the Rowid or Column or Next instruction for P1 +** will refer to the first entry in the database table or index. +** If the table or index is empty and P2>0, then jump immediately to P2. +** If P2 is 0 or if the table or index is not empty, fall through +** to the following instruction. +** +** This opcode leaves the cursor configured to move in forward order, +** from the beginning toward the end. In other words, the cursor is +** configured to use Next, not Prev. +*/ +case OP_Rewind: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); + res = 1; +#ifdef SQLITE_DEBUG + pC->seekOp = OP_Rewind; +#endif + if( isSorter(pC) ){ + rc = sqlite3VdbeSorterRewind(db, pC, &res); + }else{ + pCrsr = pC->pCursor; + assert( pCrsr ); + rc = sqlite3BtreeFirst(pCrsr, &res); + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + pC->rowidIsValid = 0; + } + pC->nullRow = (u8)res; + assert( pOp->p2>0 && pOp->p2nOp ); + VdbeBranchTaken(res!=0,2); + if( res ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: Next P1 P2 P3 P4 P5 +** +** Advance cursor P1 so that it points to the next key/data pair in its +** table or index. If there are no more key/value pairs then fall through +** to the following instruction. But if the cursor advance was successful, +** jump immediately to P2. +** +** The Next opcode is only valid following an SeekGT, SeekGE, or +** OP_Rewind opcode used to position the cursor. Next is not allowed +** to follow SeekLT, SeekLE, or OP_Last. +** +** The P1 cursor must be for a real table, not a pseudo-table. P1 must have +** been opened prior to this opcode or the program will segfault. +** +** The P3 value is a hint to the btree implementation. If P3==1, that +** means P1 is an SQL index and that this instruction could have been +** omitted if that index had been unique. P3 is usually 0. P3 is +** always either 0 or 1. +** +** P4 is always of type P4_ADVANCE. The function pointer points to +** sqlite3BtreeNext(). +** +** If P5 is positive and the jump is taken, then event counter +** number P5-1 in the prepared statement is incremented. +** +** See also: Prev, NextIfOpen +*/ +/* Opcode: NextIfOpen P1 P2 P3 P4 P5 +** +** This opcode works just like Next except that if cursor P1 is not +** open it behaves a no-op. +*/ +/* Opcode: Prev P1 P2 P3 P4 P5 +** +** Back up cursor P1 so that it points to the previous key/data pair in its +** table or index. If there is no previous key/value pairs then fall through +** to the following instruction. But if the cursor backup was successful, +** jump immediately to P2. +** +** +** The Prev opcode is only valid following an SeekLT, SeekLE, or +** OP_Last opcode used to position the cursor. Prev is not allowed +** to follow SeekGT, SeekGE, or OP_Rewind. +** +** The P1 cursor must be for a real table, not a pseudo-table. If P1 is +** not open then the behavior is undefined. +** +** The P3 value is a hint to the btree implementation. If P3==1, that +** means P1 is an SQL index and that this instruction could have been +** omitted if that index had been unique. P3 is usually 0. P3 is +** always either 0 or 1. +** +** P4 is always of type P4_ADVANCE. The function pointer points to +** sqlite3BtreePrevious(). +** +** If P5 is positive and the jump is taken, then event counter +** number P5-1 in the prepared statement is incremented. +*/ +/* Opcode: PrevIfOpen P1 P2 P3 P4 P5 +** +** This opcode works just like Prev except that if cursor P1 is not +** open it behaves a no-op. +*/ +case OP_SorterNext: { /* jump */ + VdbeCursor *pC; + int res; + + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + res = 0; + rc = sqlite3VdbeSorterNext(db, pC, &res); + goto next_tail; +case OP_PrevIfOpen: /* jump */ +case OP_NextIfOpen: /* jump */ + if( p->apCsr[pOp->p1]==0 ) break; + /* Fall through */ +case OP_Prev: /* jump */ +case OP_Next: /* jump */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p5aCounter) ); + pC = p->apCsr[pOp->p1]; + res = pOp->p3; + assert( pC!=0 ); + assert( pC->deferredMoveto==0 ); + assert( pC->pCursor ); + assert( res==0 || (res==1 && pC->isTable==0) ); + testcase( res==1 ); + assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); + assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); + assert( pOp->opcode!=OP_NextIfOpen || pOp->p4.xAdvance==sqlite3BtreeNext ); + assert( pOp->opcode!=OP_PrevIfOpen || pOp->p4.xAdvance==sqlite3BtreePrevious); + + /* The Next opcode is only used after SeekGT, SeekGE, and Rewind. + ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ + assert( pOp->opcode!=OP_Next || pOp->opcode!=OP_NextIfOpen + || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE + || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found); + assert( pOp->opcode!=OP_Prev || pOp->opcode!=OP_PrevIfOpen + || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE + || pC->seekOp==OP_Last ); + + rc = pOp->p4.xAdvance(pC->pCursor, &res); +next_tail: + pC->cacheStatus = CACHE_STALE; + VdbeBranchTaken(res==0,2); + if( res==0 ){ + pC->nullRow = 0; + pc = pOp->p2 - 1; + p->aCounter[pOp->p5]++; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + }else{ + pC->nullRow = 1; + } + pC->rowidIsValid = 0; + goto check_for_interrupt; +} + +/* Opcode: IdxInsert P1 P2 P3 * P5 +** Synopsis: key=r[P2] +** +** Register P2 holds an SQL index key made using the +** MakeRecord instructions. This opcode writes that key +** into the index P1. Data for the entry is nil. +** +** P3 is a flag that provides a hint to the b-tree layer that this +** insert is likely to be an append. +** +** If P5 has the OPFLAG_NCHANGE bit set, then the change counter is +** incremented by this instruction. If the OPFLAG_NCHANGE bit is clear, +** then the change counter is unchanged. +** +** If P5 has the OPFLAG_USESEEKRESULT bit set, then the cursor must have +** just done a seek to the spot where the new entry is to be inserted. +** This flag avoids doing an extra seek. +** +** This instruction only works for indices. The equivalent instruction +** for tables is OP_Insert. +*/ +case OP_SorterInsert: /* in2 */ +case OP_IdxInsert: { /* in2 */ + VdbeCursor *pC; + BtCursor *pCrsr; + int nKey; + const char *zKey; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( isSorter(pC)==(pOp->opcode==OP_SorterInsert) ); + pIn2 = &aMem[pOp->p2]; + assert( pIn2->flags & MEM_Blob ); + pCrsr = pC->pCursor; + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; + assert( pCrsr!=0 ); + assert( pC->isTable==0 ); + rc = ExpandBlob(pIn2); + if( rc==SQLITE_OK ){ + if( isSorter(pC) ){ + rc = sqlite3VdbeSorterWrite(db, pC, pIn2); + }else{ + nKey = pIn2->n; + zKey = pIn2->z; + rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, + ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) + ); + assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; + } + } + break; +} + +/* Opcode: IdxDelete P1 P2 P3 * * +** Synopsis: key=r[P2@P3] +** +** The content of P3 registers starting at register P2 form +** an unpacked index key. This opcode removes that entry from the +** index opened by cursor P1. +*/ +case OP_IdxDelete: { + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + UnpackedRecord r; + + assert( pOp->p3>0 ); + assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem-p->nCursor)+1 ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pCrsr = pC->pCursor; + assert( pCrsr!=0 ); + assert( pOp->p5==0 ); + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)pOp->p3; + r.default_rc = 0; + r.aMem = &aMem[pOp->p2]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ideferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; + break; +} + +/* Opcode: IdxRowid P1 P2 * * * +** Synopsis: r[P2]=rowid +** +** Write into register P2 an integer which is the last entry in the record at +** the end of the index key pointed to by cursor P1. This integer should be +** the rowid of the table entry to which this index entry points. +** +** See also: Rowid, MakeRecord. +*/ +case OP_IdxRowid: { /* out2-prerelease */ + BtCursor *pCrsr; + VdbeCursor *pC; + i64 rowid; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pCrsr = pC->pCursor; + assert( pCrsr!=0 ); + pOut->flags = MEM_Null; + rc = sqlite3VdbeCursorMoveto(pC); + if( NEVER(rc) ) goto abort_due_to_error; + assert( pC->deferredMoveto==0 ); + assert( pC->isTable==0 ); + if( !pC->nullRow ){ + rowid = 0; /* Not needed. Only used to silence a warning. */ + rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + pOut->u.i = rowid; + pOut->flags = MEM_Int; + } + break; +} + +/* Opcode: IdxGE P1 P2 P3 P4 P5 +** Synopsis: key=r[P3@P4] +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY. Compare this key value against the index +** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID +** fields at the end. +** +** If the P1 index entry is greater than or equal to the key value +** then jump to P2. Otherwise fall through to the next instruction. +*/ +/* Opcode: IdxGT P1 P2 P3 P4 P5 +** Synopsis: key=r[P3@P4] +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY. Compare this key value against the index +** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID +** fields at the end. +** +** If the P1 index entry is greater than the key value +** then jump to P2. Otherwise fall through to the next instruction. +*/ +/* Opcode: IdxLT P1 P2 P3 P4 P5 +** Synopsis: key=r[P3@P4] +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY or ROWID. Compare this key value against +** the index that P1 is currently pointing to, ignoring the PRIMARY KEY or +** ROWID on the P1 index. +** +** If the P1 index entry is less than the key value then jump to P2. +** Otherwise fall through to the next instruction. +*/ +/* Opcode: IdxLE P1 P2 P3 P4 P5 +** Synopsis: key=r[P3@P4] +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY or ROWID. Compare this key value against +** the index that P1 is currently pointing to, ignoring the PRIMARY KEY or +** ROWID on the P1 index. +** +** If the P1 index entry is less than or equal to the key value then jump +** to P2. Otherwise fall through to the next instruction. +*/ +case OP_IdxLE: /* jump */ +case OP_IdxGT: /* jump */ +case OP_IdxLT: /* jump */ +case OP_IdxGE: { /* jump */ + VdbeCursor *pC; + int res; + UnpackedRecord r; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->isOrdered ); + assert( pC->pCursor!=0); + assert( pC->deferredMoveto==0 ); + assert( pOp->p5==0 || pOp->p5==1 ); + assert( pOp->p4type==P4_INT32 ); + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)pOp->p4.i; + if( pOp->opcodeopcode==OP_IdxLE || pOp->opcode==OP_IdxGT ); + r.default_rc = -1; + }else{ + assert( pOp->opcode==OP_IdxGE || pOp->opcode==OP_IdxLT ); + r.default_rc = 0; + } + r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iopcode&1)==(OP_IdxLT&1) ){ + assert( pOp->opcode==OP_IdxLE || pOp->opcode==OP_IdxLT ); + res = -res; + }else{ + assert( pOp->opcode==OP_IdxGE || pOp->opcode==OP_IdxGT ); + res++; + } + VdbeBranchTaken(res>0,2); + if( res>0 ){ + pc = pOp->p2 - 1 ; + } + break; +} + +/* Opcode: Destroy P1 P2 P3 * * +** +** Delete an entire database table or index whose root page in the database +** file is given by P1. +** +** The table being destroyed is in the main database file if P3==0. If +** P3==1 then the table to be clear is in the auxiliary database file +** that is used to store tables create using CREATE TEMPORARY TABLE. +** +** If AUTOVACUUM is enabled then it is possible that another root page +** might be moved into the newly deleted root page in order to keep all +** root pages contiguous at the beginning of the database. The former +** value of the root page that moved - its value before the move occurred - +** is stored in register P2. If no page +** movement was required (because the table being dropped was already +** the last one in the database) then a zero is stored in register P2. +** If AUTOVACUUM is disabled then a zero is stored in register P2. +** +** See also: Clear +*/ +case OP_Destroy: { /* out2-prerelease */ + int iMoved; + int iCnt; + Vdbe *pVdbe; + int iDb; + + assert( p->readOnly==0 ); +#ifndef SQLITE_OMIT_VIRTUALTABLE + iCnt = 0; + for(pVdbe=db->pVdbe; pVdbe; pVdbe = pVdbe->pNext){ + if( pVdbe->magic==VDBE_MAGIC_RUN && pVdbe->bIsReader + && pVdbe->inVtabMethod<2 && pVdbe->pc>=0 + ){ + iCnt++; + } + } +#else + iCnt = db->nVdbeRead; +#endif + pOut->flags = MEM_Null; + if( iCnt>1 ){ + rc = SQLITE_LOCKED; + p->errorAction = OE_Abort; + }else{ + iDb = pOp->p3; + assert( iCnt==1 ); + assert( DbMaskTest(p->btreeMask, iDb) ); + iMoved = 0; /* Not needed. Only to silence a warning. */ + rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved); + pOut->flags = MEM_Int; + pOut->u.i = iMoved; +#ifndef SQLITE_OMIT_AUTOVACUUM + if( rc==SQLITE_OK && iMoved!=0 ){ + sqlite3RootPageMoved(db, iDb, iMoved, pOp->p1); + /* All OP_Destroy operations occur on the same btree */ + assert( resetSchemaOnFault==0 || resetSchemaOnFault==iDb+1 ); + resetSchemaOnFault = iDb+1; + } +#endif + } + break; +} + +/* Opcode: Clear P1 P2 P3 +** +** Delete all contents of the database table or index whose root page +** in the database file is given by P1. But, unlike Destroy, do not +** remove the table or index from the database file. +** +** The table being clear is in the main database file if P2==0. If +** P2==1 then the table to be clear is in the auxiliary database file +** that is used to store tables create using CREATE TEMPORARY TABLE. +** +** If the P3 value is non-zero, then the table referred to must be an +** intkey table (an SQL table, not an index). In this case the row change +** count is incremented by the number of rows in the table being cleared. +** If P3 is greater than zero, then the value stored in register P3 is +** also incremented by the number of rows in the table being cleared. +** +** See also: Destroy +*/ +case OP_Clear: { + int nChange; + + nChange = 0; + assert( p->readOnly==0 ); + assert( DbMaskTest(p->btreeMask, pOp->p2) ); + rc = sqlite3BtreeClearTable( + db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &nChange : 0) + ); + if( pOp->p3 ){ + p->nChange += nChange; + if( pOp->p3>0 ){ + assert( memIsValid(&aMem[pOp->p3]) ); + memAboutToChange(p, &aMem[pOp->p3]); + aMem[pOp->p3].u.i += nChange; + } + } + break; +} + +/* Opcode: ResetSorter P1 * * * * +** +** Delete all contents from the ephemeral table or sorter +** that is open on cursor P1. +** +** This opcode only works for cursors used for sorting and +** opened with OP_OpenEphemeral or OP_SorterOpen. +*/ +case OP_ResetSorter: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + if( pC->pSorter ){ + sqlite3VdbeSorterReset(db, pC->pSorter); + }else{ + assert( pC->isEphemeral ); + rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + } + break; +} + +/* Opcode: CreateTable P1 P2 * * * +** Synopsis: r[P2]=root iDb=P1 +** +** Allocate a new table in the main database file if P1==0 or in the +** auxiliary database file if P1==1 or in an attached database if +** P1>1. Write the root page number of the new table into +** register P2 +** +** The difference between a table and an index is this: A table must +** have a 4-byte integer key and can have arbitrary data. An index +** has an arbitrary key but no data. +** +** See also: CreateIndex +*/ +/* Opcode: CreateIndex P1 P2 * * * +** Synopsis: r[P2]=root iDb=P1 +** +** Allocate a new index in the main database file if P1==0 or in the +** auxiliary database file if P1==1 or in an attached database if +** P1>1. Write the root page number of the new table into +** register P2. +** +** See documentation on OP_CreateTable for additional information. +*/ +case OP_CreateIndex: /* out2-prerelease */ +case OP_CreateTable: { /* out2-prerelease */ + int pgno; + int flags; + Db *pDb; + + pgno = 0; + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); + assert( p->readOnly==0 ); + pDb = &db->aDb[pOp->p1]; + assert( pDb->pBt!=0 ); + if( pOp->opcode==OP_CreateTable ){ + /* flags = BTREE_INTKEY; */ + flags = BTREE_INTKEY; + }else{ + flags = BTREE_BLOBKEY; + } + rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, flags); + pOut->u.i = pgno; + break; +} + +/* Opcode: ParseSchema P1 * * P4 * +** +** Read and parse all entries from the SQLITE_MASTER table of database P1 +** that match the WHERE clause P4. +** +** This opcode invokes the parser to create a new virtual machine, +** then runs the new virtual machine. It is thus a re-entrant opcode. +*/ +case OP_ParseSchema: { + int iDb; + const char *zMaster; + char *zSql; + InitData initData; + + /* Any prepared statement that invokes this opcode will hold mutexes + ** on every btree. This is a prerequisite for invoking + ** sqlite3InitCallback(). + */ +#ifdef SQLITE_DEBUG + for(iDb=0; iDbnDb; iDb++){ + assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); + } +#endif + + iDb = pOp->p1; + assert( iDb>=0 && iDbnDb ); + assert( DbHasProperty(db, iDb, DB_SchemaLoaded) ); + /* Used to be a conditional */ { + zMaster = SCHEMA_TABLE(iDb); + initData.db = db; + initData.iDb = pOp->p1; + initData.pzErrMsg = &p->zErrMsg; + zSql = sqlite3MPrintf(db, + "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", + db->aDb[iDb].zName, zMaster, pOp->p4.z); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + assert( db->init.busy==0 ); + db->init.busy = 1; + initData.rc = SQLITE_OK; + assert( !db->mallocFailed ); + rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); + if( rc==SQLITE_OK ) rc = initData.rc; + sqlite3DbFree(db, zSql); + db->init.busy = 0; + } + } + if( rc ) sqlite3ResetAllSchemasOfConnection(db); + if( rc==SQLITE_NOMEM ){ + goto no_mem; + } + break; +} + +#if !defined(SQLITE_OMIT_ANALYZE) +/* Opcode: LoadAnalysis P1 * * * * +** +** Read the sqlite_stat1 table for database P1 and load the content +** of that table into the internal index hash table. This will cause +** the analysis to be used when preparing all subsequent queries. +*/ +case OP_LoadAnalysis: { + assert( pOp->p1>=0 && pOp->p1nDb ); + rc = sqlite3AnalysisLoad(db, pOp->p1); + break; +} +#endif /* !defined(SQLITE_OMIT_ANALYZE) */ + +/* Opcode: DropTable P1 * * P4 * +** +** Remove the internal (in-memory) data structures that describe +** the table named P4 in database P1. This is called after a table +** is dropped from disk (using the Destroy opcode) in order to keep +** the internal representation of the +** schema consistent with what is on disk. +*/ +case OP_DropTable: { + sqlite3UnlinkAndDeleteTable(db, pOp->p1, pOp->p4.z); + break; +} + +/* Opcode: DropIndex P1 * * P4 * +** +** Remove the internal (in-memory) data structures that describe +** the index named P4 in database P1. This is called after an index +** is dropped from disk (using the Destroy opcode) +** in order to keep the internal representation of the +** schema consistent with what is on disk. +*/ +case OP_DropIndex: { + sqlite3UnlinkAndDeleteIndex(db, pOp->p1, pOp->p4.z); + break; +} + +/* Opcode: DropTrigger P1 * * P4 * +** +** Remove the internal (in-memory) data structures that describe +** the trigger named P4 in database P1. This is called after a trigger +** is dropped from disk (using the Destroy opcode) in order to keep +** the internal representation of the +** schema consistent with what is on disk. +*/ +case OP_DropTrigger: { + sqlite3UnlinkAndDeleteTrigger(db, pOp->p1, pOp->p4.z); + break; +} + + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* Opcode: IntegrityCk P1 P2 P3 * P5 +** +** Do an analysis of the currently open database. Store in +** register P1 the text of an error message describing any problems. +** If no problems are found, store a NULL in register P1. +** +** The register P3 contains the maximum number of allowed errors. +** At most reg(P3) errors will be reported. +** In other words, the analysis stops as soon as reg(P1) errors are +** seen. Reg(P1) is updated with the number of errors remaining. +** +** The root page numbers of all tables in the database are integer +** stored in reg(P1), reg(P1+1), reg(P1+2), .... There are P2 tables +** total. +** +** If P5 is not zero, the check is done on the auxiliary database +** file, not the main database file. +** +** This opcode is used to implement the integrity_check pragma. +*/ +case OP_IntegrityCk: { + int nRoot; /* Number of tables to check. (Number of root pages.) */ + int *aRoot; /* Array of rootpage numbers for tables to be checked */ + int j; /* Loop counter */ + int nErr; /* Number of errors reported */ + char *z; /* Text of the error report */ + Mem *pnErr; /* Register keeping track of errors remaining */ + + assert( p->bIsReader ); + nRoot = pOp->p2; + assert( nRoot>0 ); + aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(nRoot+1) ); + if( aRoot==0 ) goto no_mem; + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + pnErr = &aMem[pOp->p3]; + assert( (pnErr->flags & MEM_Int)!=0 ); + assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); + pIn1 = &aMem[pOp->p1]; + for(j=0; jp5nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p5) ); + z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, + (int)pnErr->u.i, &nErr); + sqlite3DbFree(db, aRoot); + pnErr->u.i -= nErr; + sqlite3VdbeMemSetNull(pIn1); + if( nErr==0 ){ + assert( z==0 ); + }else if( z==0 ){ + goto no_mem; + }else{ + sqlite3VdbeMemSetStr(pIn1, z, -1, SQLITE_UTF8, sqlite3_free); + } + UPDATE_MAX_BLOBSIZE(pIn1); + sqlite3VdbeChangeEncoding(pIn1, encoding); + break; +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +/* Opcode: RowSetAdd P1 P2 * * * +** Synopsis: rowset(P1)=r[P2] +** +** Insert the integer value held by register P2 into a boolean index +** held in register P1. +** +** An assertion fails if P2 is not an integer. +*/ +case OP_RowSetAdd: { /* in1, in2 */ + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + assert( (pIn2->flags & MEM_Int)!=0 ); + if( (pIn1->flags & MEM_RowSet)==0 ){ + sqlite3VdbeMemSetRowSet(pIn1); + if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + } + sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i); + break; +} + +/* Opcode: RowSetRead P1 P2 P3 * * +** Synopsis: r[P3]=rowset(P1) +** +** Extract the smallest value from boolean index P1 and put that value into +** register P3. Or, if boolean index P1 is initially empty, leave P3 +** unchanged and jump to instruction P2. +*/ +case OP_RowSetRead: { /* jump, in1, out3 */ + i64 val; + + pIn1 = &aMem[pOp->p1]; + if( (pIn1->flags & MEM_RowSet)==0 + || sqlite3RowSetNext(pIn1->u.pRowSet, &val)==0 + ){ + /* The boolean index is empty */ + sqlite3VdbeMemSetNull(pIn1); + pc = pOp->p2 - 1; + VdbeBranchTaken(1,2); + }else{ + /* A value was pulled from the index */ + sqlite3VdbeMemSetInt64(&aMem[pOp->p3], val); + VdbeBranchTaken(0,2); + } + goto check_for_interrupt; +} + +/* Opcode: RowSetTest P1 P2 P3 P4 +** Synopsis: if r[P3] in rowset(P1) goto P2 +** +** Register P3 is assumed to hold a 64-bit integer value. If register P1 +** contains a RowSet object and that RowSet object contains +** the value held in P3, jump to register P2. Otherwise, insert the +** integer in P3 into the RowSet and continue on to the +** next opcode. +** +** The RowSet object is optimized for the case where successive sets +** of integers, where each set contains no duplicates. Each set +** of values is identified by a unique P4 value. The first set +** must have P4==0, the final set P4=-1. P4 must be either -1 or +** non-negative. For non-negative values of P4 only the lower 4 +** bits are significant. +** +** This allows optimizations: (a) when P4==0 there is no need to test +** the rowset object for P3, as it is guaranteed not to contain it, +** (b) when P4==-1 there is no need to insert the value, as it will +** never be tested for, and (c) when a value that is part of set X is +** inserted, there is no need to search to see if the same value was +** previously inserted as part of set X (only if it was previously +** inserted as part of some other set). +*/ +case OP_RowSetTest: { /* jump, in1, in3 */ + int iSet; + int exists; + + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; + iSet = pOp->p4.i; + assert( pIn3->flags&MEM_Int ); + + /* If there is anything other than a rowset object in memory cell P1, + ** delete it now and initialize P1 with an empty rowset + */ + if( (pIn1->flags & MEM_RowSet)==0 ){ + sqlite3VdbeMemSetRowSet(pIn1); + if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + } + + assert( pOp->p4type==P4_INT32 ); + assert( iSet==-1 || iSet>=0 ); + if( iSet ){ + exists = sqlite3RowSetTest(pIn1->u.pRowSet, iSet, pIn3->u.i); + VdbeBranchTaken(exists!=0,2); + if( exists ){ + pc = pOp->p2 - 1; + break; + } + } + if( iSet>=0 ){ + sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i); + } + break; +} + + +#ifndef SQLITE_OMIT_TRIGGER + +/* Opcode: Program P1 P2 P3 P4 P5 +** +** Execute the trigger program passed as P4 (type P4_SUBPROGRAM). +** +** P1 contains the address of the memory cell that contains the first memory +** cell in an array of values used as arguments to the sub-program. P2 +** contains the address to jump to if the sub-program throws an IGNORE +** exception using the RAISE() function. Register P3 contains the address +** of a memory cell in this (the parent) VM that is used to allocate the +** memory required by the sub-vdbe at runtime. +** +** P4 is a pointer to the VM containing the trigger program. +** +** If P5 is non-zero, then recursive program invocation is enabled. +*/ +case OP_Program: { /* jump */ + int nMem; /* Number of memory registers for sub-program */ + int nByte; /* Bytes of runtime space required for sub-program */ + Mem *pRt; /* Register to allocate runtime space */ + Mem *pMem; /* Used to iterate through memory cells */ + Mem *pEnd; /* Last memory cell in new array */ + VdbeFrame *pFrame; /* New vdbe frame to execute in */ + SubProgram *pProgram; /* Sub-program to execute */ + void *t; /* Token identifying trigger */ + + pProgram = pOp->p4.pProgram; + pRt = &aMem[pOp->p3]; + assert( pProgram->nOp>0 ); + + /* If the p5 flag is clear, then recursive invocation of triggers is + ** disabled for backwards compatibility (p5 is set if this sub-program + ** is really a trigger, not a foreign key action, and the flag set + ** and cleared by the "PRAGMA recursive_triggers" command is clear). + ** + ** It is recursive invocation of triggers, at the SQL level, that is + ** disabled. In some cases a single trigger may generate more than one + ** SubProgram (if the trigger may be executed with more than one different + ** ON CONFLICT algorithm). SubProgram structures associated with a + ** single trigger all have the same value for the SubProgram.token + ** variable. */ + if( pOp->p5 ){ + t = pProgram->token; + for(pFrame=p->pFrame; pFrame && pFrame->token!=t; pFrame=pFrame->pParent); + if( pFrame ) break; + } + + if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){ + rc = SQLITE_ERROR; + sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion"); + break; + } + + /* Register pRt is used to store the memory required to save the state + ** of the current program, and the memory required at runtime to execute + ** the trigger program. If this trigger has been fired before, then pRt + ** is already allocated. Otherwise, it must be initialized. */ + if( (pRt->flags&MEM_Frame)==0 ){ + /* SubProgram.nMem is set to the number of memory cells used by the + ** program stored in SubProgram.aOp. As well as these, one memory + ** cell is required for each cursor used by the program. Set local + ** variable nMem (and later, VdbeFrame.nChildMem) to this value. + */ + nMem = pProgram->nMem + pProgram->nCsr; + nByte = ROUND8(sizeof(VdbeFrame)) + + nMem * sizeof(Mem) + + pProgram->nCsr * sizeof(VdbeCursor *) + + pProgram->nOnce * sizeof(u8); + pFrame = sqlite3DbMallocZero(db, nByte); + if( !pFrame ){ + goto no_mem; + } + sqlite3VdbeMemRelease(pRt); + pRt->flags = MEM_Frame; + pRt->u.pFrame = pFrame; + + pFrame->v = p; + pFrame->nChildMem = nMem; + pFrame->nChildCsr = pProgram->nCsr; + pFrame->pc = pc; + pFrame->aMem = p->aMem; + pFrame->nMem = p->nMem; + pFrame->apCsr = p->apCsr; + pFrame->nCursor = p->nCursor; + pFrame->aOp = p->aOp; + pFrame->nOp = p->nOp; + pFrame->token = pProgram->token; + pFrame->aOnceFlag = p->aOnceFlag; + pFrame->nOnceFlag = p->nOnceFlag; + + pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; + for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ + pMem->flags = MEM_Undefined; + pMem->db = db; + } + }else{ + pFrame = pRt->u.pFrame; + assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem ); + assert( pProgram->nCsr==pFrame->nChildCsr ); + assert( pc==pFrame->pc ); + } + + p->nFrame++; + pFrame->pParent = p->pFrame; + pFrame->lastRowid = lastRowid; + pFrame->nChange = p->nChange; + p->nChange = 0; + p->pFrame = pFrame; + p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; + p->nMem = pFrame->nChildMem; + p->nCursor = (u16)pFrame->nChildCsr; + p->apCsr = (VdbeCursor **)&aMem[p->nMem+1]; + p->aOp = aOp = pProgram->aOp; + p->nOp = pProgram->nOp; + p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; + p->nOnceFlag = pProgram->nOnce; + pc = -1; + memset(p->aOnceFlag, 0, p->nOnceFlag); + + break; +} + +/* Opcode: Param P1 P2 * * * +** +** This opcode is only ever present in sub-programs called via the +** OP_Program instruction. Copy a value currently stored in a memory +** cell of the calling (parent) frame to cell P2 in the current frames +** address space. This is used by trigger programs to access the new.* +** and old.* values. +** +** The address of the cell in the parent frame is determined by adding +** the value of the P1 argument to the value of the P1 argument to the +** calling OP_Program instruction. +*/ +case OP_Param: { /* out2-prerelease */ + VdbeFrame *pFrame; + Mem *pIn; + pFrame = p->pFrame; + pIn = &pFrame->aMem[pOp->p1 + pFrame->aOp[pFrame->pc].p1]; + sqlite3VdbeMemShallowCopy(pOut, pIn, MEM_Ephem); + break; +} + +#endif /* #ifndef SQLITE_OMIT_TRIGGER */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY +/* Opcode: FkCounter P1 P2 * * * +** Synopsis: fkctr[P1]+=P2 +** +** Increment a "constraint counter" by P2 (P2 may be negative or positive). +** If P1 is non-zero, the database constraint counter is incremented +** (deferred foreign key constraints). Otherwise, if P1 is zero, the +** statement counter is incremented (immediate foreign key constraints). +*/ +case OP_FkCounter: { + if( db->flags & SQLITE_DeferFKs ){ + db->nDeferredImmCons += pOp->p2; + }else if( pOp->p1 ){ + db->nDeferredCons += pOp->p2; + }else{ + p->nFkConstraint += pOp->p2; + } + break; +} + +/* Opcode: FkIfZero P1 P2 * * * +** Synopsis: if fkctr[P1]==0 goto P2 +** +** This opcode tests if a foreign key constraint-counter is currently zero. +** If so, jump to instruction P2. Otherwise, fall through to the next +** instruction. +** +** If P1 is non-zero, then the jump is taken if the database constraint-counter +** is zero (the one that counts deferred constraint violations). If P1 is +** zero, the jump is taken if the statement constraint-counter is zero +** (immediate foreign key constraint violations). +*/ +case OP_FkIfZero: { /* jump */ + if( pOp->p1 ){ + VdbeBranchTaken(db->nDeferredCons==0 && db->nDeferredImmCons==0, 2); + if( db->nDeferredCons==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1; + }else{ + VdbeBranchTaken(p->nFkConstraint==0 && db->nDeferredImmCons==0, 2); + if( p->nFkConstraint==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1; + } + break; +} +#endif /* #ifndef SQLITE_OMIT_FOREIGN_KEY */ + +#ifndef SQLITE_OMIT_AUTOINCREMENT +/* Opcode: MemMax P1 P2 * * * +** Synopsis: r[P1]=max(r[P1],r[P2]) +** +** P1 is a register in the root frame of this VM (the root frame is +** different from the current frame if this instruction is being executed +** within a sub-program). Set the value of register P1 to the maximum of +** its current value and the value in register P2. +** +** This instruction throws an error if the memory cell is not initially +** an integer. +*/ +case OP_MemMax: { /* in2 */ + VdbeFrame *pFrame; + if( p->pFrame ){ + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + pIn1 = &pFrame->aMem[pOp->p1]; + }else{ + pIn1 = &aMem[pOp->p1]; + } + assert( memIsValid(pIn1) ); + sqlite3VdbeMemIntegerify(pIn1); + pIn2 = &aMem[pOp->p2]; + sqlite3VdbeMemIntegerify(pIn2); + if( pIn1->u.iu.i){ + pIn1->u.i = pIn2->u.i; + } + break; +} +#endif /* SQLITE_OMIT_AUTOINCREMENT */ + +/* Opcode: IfPos P1 P2 * * * +** Synopsis: if r[P1]>0 goto P2 +** +** If the value of register P1 is 1 or greater, jump to P2. +** +** It is illegal to use this instruction on a register that does +** not contain an integer. An assertion fault will result if you try. +*/ +case OP_IfPos: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + VdbeBranchTaken( pIn1->u.i>0, 2); + if( pIn1->u.i>0 ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: IfNeg P1 P2 P3 * * +** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 +** +** Register P1 must contain an integer. Add literal P3 to the value in +** register P1 then if the value of register P1 is less than zero, jump to P2. +*/ +case OP_IfNeg: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + pIn1->u.i += pOp->p3; + VdbeBranchTaken(pIn1->u.i<0, 2); + if( pIn1->u.i<0 ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: IfZero P1 P2 P3 * * +** Synopsis: r[P1]+=P3, if r[P1]==0 goto P2 +** +** The register P1 must contain an integer. Add literal P3 to the +** value in register P1. If the result is exactly 0, jump to P2. +*/ +case OP_IfZero: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + pIn1->u.i += pOp->p3; + VdbeBranchTaken(pIn1->u.i==0, 2); + if( pIn1->u.i==0 ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: AggStep * P2 P3 P4 P5 +** Synopsis: accum=r[P3] step(r[P2@P5]) +** +** Execute the step function for an aggregate. The +** function has P5 arguments. P4 is a pointer to the FuncDef +** structure that specifies the function. Use register +** P3 as the accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +*/ +case OP_AggStep: { + int n; + int i; + Mem *pMem; + Mem *pRec; + sqlite3_context ctx; + sqlite3_value **apVal; + + n = pOp->p5; + assert( n>=0 ); + pRec = &aMem[pOp->p2]; + apVal = p->apArg; + assert( apVal || n==0 ); + for(i=0; ip4.pFunc; + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + ctx.pMem = pMem = &aMem[pOp->p3]; + pMem->n++; + ctx.s.flags = MEM_Null; + ctx.s.z = 0; + ctx.s.zMalloc = 0; + ctx.s.xDel = 0; + ctx.s.db = db; + ctx.isError = 0; + ctx.pColl = 0; + ctx.skipFlag = 0; + if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + assert( pOp>p->aOp ); + assert( pOp[-1].p4type==P4_COLLSEQ ); + assert( pOp[-1].opcode==OP_CollSeq ); + ctx.pColl = pOp[-1].p4.pColl; + } + (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */ + if( ctx.isError ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); + rc = ctx.isError; + } + if( ctx.skipFlag ){ + assert( pOp[-1].opcode==OP_CollSeq ); + i = pOp[-1].p1; + if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); + } + + sqlite3VdbeMemRelease(&ctx.s); + + break; +} + +/* Opcode: AggFinal P1 P2 * P4 * +** Synopsis: accum=r[P1] N=P2 +** +** Execute the finalizer function for an aggregate. P1 is +** the memory location that is the accumulator for the aggregate. +** +** P2 is the number of arguments that the step function takes and +** P4 is a pointer to the FuncDef for this function. The P2 +** argument is not used by this opcode. It is only there to disambiguate +** functions that can take varying numbers of arguments. The +** P4 argument is only needed for the degenerate case where +** the step function was not previously called. +*/ +case OP_AggFinal: { + Mem *pMem; + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + pMem = &aMem[pOp->p1]; + assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); + rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); + if( rc ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(pMem)); + } + sqlite3VdbeChangeEncoding(pMem, encoding); + UPDATE_MAX_BLOBSIZE(pMem); + if( sqlite3VdbeMemTooBig(pMem) ){ + goto too_big; + } + break; +} + +#ifndef SQLITE_OMIT_WAL +/* Opcode: Checkpoint P1 P2 P3 * * +** +** Checkpoint database P1. This is a no-op if P1 is not currently in +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL +** or RESTART. Write 1 or 0 into mem[P3] if the checkpoint returns +** SQLITE_BUSY or not, respectively. Write the number of pages in the +** WAL after the checkpoint into mem[P3+1] and the number of pages +** in the WAL that have been checkpointed after the checkpoint +** completes into mem[P3+2]. However on an error, mem[P3+1] and +** mem[P3+2] are initialized to -1. +*/ +case OP_Checkpoint: { + int i; /* Loop counter */ + int aRes[3]; /* Results */ + Mem *pMem; /* Write results here */ + + assert( p->readOnly==0 ); + aRes[0] = 0; + aRes[1] = aRes[2] = -1; + assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE + || pOp->p2==SQLITE_CHECKPOINT_FULL + || pOp->p2==SQLITE_CHECKPOINT_RESTART + ); + rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); + if( rc==SQLITE_BUSY ){ + rc = SQLITE_OK; + aRes[0] = 1; + } + for(i=0, pMem = &aMem[pOp->p3]; i<3; i++, pMem++){ + sqlite3VdbeMemSetInt64(pMem, (i64)aRes[i]); + } + break; +}; +#endif + +#ifndef SQLITE_OMIT_PRAGMA +/* Opcode: JournalMode P1 P2 P3 * * +** +** Change the journal mode of database P1 to P3. P3 must be one of the +** PAGER_JOURNALMODE_XXX values. If changing between the various rollback +** modes (delete, truncate, persist, off and memory), this is a simple +** operation. No IO is required. +** +** If changing into or out of WAL mode the procedure is more complicated. +** +** Write a string containing the final journal-mode to register P2. +*/ +case OP_JournalMode: { /* out2-prerelease */ + Btree *pBt; /* Btree to change journal mode of */ + Pager *pPager; /* Pager associated with pBt */ + int eNew; /* New journal mode */ + int eOld; /* The old journal mode */ +#ifndef SQLITE_OMIT_WAL + const char *zFilename; /* Name of database file for pPager */ +#endif + + eNew = pOp->p3; + assert( eNew==PAGER_JOURNALMODE_DELETE + || eNew==PAGER_JOURNALMODE_TRUNCATE + || eNew==PAGER_JOURNALMODE_PERSIST + || eNew==PAGER_JOURNALMODE_OFF + || eNew==PAGER_JOURNALMODE_MEMORY + || eNew==PAGER_JOURNALMODE_WAL + || eNew==PAGER_JOURNALMODE_QUERY + ); + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( p->readOnly==0 ); + + pBt = db->aDb[pOp->p1].pBt; + pPager = sqlite3BtreePager(pBt); + eOld = sqlite3PagerGetJournalMode(pPager); + if( eNew==PAGER_JOURNALMODE_QUERY ) eNew = eOld; + if( !sqlite3PagerOkToChangeJournalMode(pPager) ) eNew = eOld; + +#ifndef SQLITE_OMIT_WAL + zFilename = sqlite3PagerFilename(pPager, 1); + + /* Do not allow a transition to journal_mode=WAL for a database + ** in temporary storage or if the VFS does not support shared memory + */ + if( eNew==PAGER_JOURNALMODE_WAL + && (sqlite3Strlen30(zFilename)==0 /* Temp file */ + || !sqlite3PagerWalSupported(pPager)) /* No shared-memory support */ + ){ + eNew = eOld; + } + + if( (eNew!=eOld) + && (eOld==PAGER_JOURNALMODE_WAL || eNew==PAGER_JOURNALMODE_WAL) + ){ + if( !db->autoCommit || db->nVdbeRead>1 ){ + rc = SQLITE_ERROR; + sqlite3SetString(&p->zErrMsg, db, + "cannot change %s wal mode from within a transaction", + (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") + ); + break; + }else{ + + if( eOld==PAGER_JOURNALMODE_WAL ){ + /* If leaving WAL mode, close the log file. If successful, the call + ** to PagerCloseWal() checkpoints and deletes the write-ahead-log + ** file. An EXCLUSIVE lock may still be held on the database file + ** after a successful return. + */ + rc = sqlite3PagerCloseWal(pPager); + if( rc==SQLITE_OK ){ + sqlite3PagerSetJournalMode(pPager, eNew); + } + }else if( eOld==PAGER_JOURNALMODE_MEMORY ){ + /* Cannot transition directly from MEMORY to WAL. Use mode OFF + ** as an intermediate */ + sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF); + } + + /* Open a transaction on the database file. Regardless of the journal + ** mode, this transaction always uses a rollback journal. + */ + assert( sqlite3BtreeIsInTrans(pBt)==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1)); + } + } + } +#endif /* ifndef SQLITE_OMIT_WAL */ + + if( rc ){ + eNew = eOld; + } + eNew = sqlite3PagerSetJournalMode(pPager, eNew); + + pOut = &aMem[pOp->p2]; + pOut->flags = MEM_Str|MEM_Static|MEM_Term; + pOut->z = (char *)sqlite3JournalModename(eNew); + pOut->n = sqlite3Strlen30(pOut->z); + pOut->enc = SQLITE_UTF8; + sqlite3VdbeChangeEncoding(pOut, encoding); + break; +}; +#endif /* SQLITE_OMIT_PRAGMA */ + +#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) +/* Opcode: Vacuum * * * * * +** +** Vacuum the entire database. This opcode will cause other virtual +** machines to be created and run. It may not be called from within +** a transaction. +*/ +case OP_Vacuum: { + assert( p->readOnly==0 ); + rc = sqlite3RunVacuum(&p->zErrMsg, db); + break; +} +#endif + +#if !defined(SQLITE_OMIT_AUTOVACUUM) +/* Opcode: IncrVacuum P1 P2 * * * +** +** Perform a single step of the incremental vacuum procedure on +** the P1 database. If the vacuum has finished, jump to instruction +** P2. Otherwise, fall through to the next instruction. +*/ +case OP_IncrVacuum: { /* jump */ + Btree *pBt; + + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); + assert( p->readOnly==0 ); + pBt = db->aDb[pOp->p1].pBt; + rc = sqlite3BtreeIncrVacuum(pBt); + VdbeBranchTaken(rc==SQLITE_DONE,2); + if( rc==SQLITE_DONE ){ + pc = pOp->p2 - 1; + rc = SQLITE_OK; + } + break; +} +#endif + +/* Opcode: Expire P1 * * * * +** +** Cause precompiled statements to expire. When an expired statement +** is executed using sqlite3_step() it will either automatically +** reprepare itself (if it was originally created using sqlite3_prepare_v2()) +** or it will fail with SQLITE_SCHEMA. +** +** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, +** then only the currently executing statement is expired. +*/ +case OP_Expire: { + if( !pOp->p1 ){ + sqlite3ExpirePreparedStatements(db); + }else{ + p->expired = 1; + } + break; +} + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* Opcode: TableLock P1 P2 P3 P4 * +** Synopsis: iDb=P1 root=P2 write=P3 +** +** Obtain a lock on a particular table. This instruction is only used when +** the shared-cache feature is enabled. +** +** P1 is the index of the database in sqlite3.aDb[] of the database +** on which the lock is acquired. A readlock is obtained if P3==0 or +** a write lock if P3==1. +** +** P2 contains the root-page of the table to lock. +** +** P4 contains a pointer to the name of the table being locked. This is only +** used to generate an error message if the lock cannot be obtained. +*/ +case OP_TableLock: { + u8 isWriteLock = (u8)pOp->p3; + if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommitted) ){ + int p1 = pOp->p1; + assert( p1>=0 && p1nDb ); + assert( DbMaskTest(p->btreeMask, p1) ); + assert( isWriteLock==0 || isWriteLock==1 ); + rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); + if( (rc&0xFF)==SQLITE_LOCKED ){ + const char *z = pOp->p4.z; + sqlite3SetString(&p->zErrMsg, db, "database table is locked: %s", z); + } + } + break; +} +#endif /* SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VBegin * * * P4 * +** +** P4 may be a pointer to an sqlite3_vtab structure. If so, call the +** xBegin method for that table. +** +** Also, whether or not P4 is set, check that this is not being called from +** within a callback to a virtual table xSync() method. If it is, the error +** code will be set to SQLITE_LOCKED. +*/ +case OP_VBegin: { + VTable *pVTab; + pVTab = pOp->p4.pVtab; + rc = sqlite3VtabBegin(db, pVTab); + if( pVTab ) sqlite3VtabImportErrmsg(p, pVTab->pVtab); + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VCreate P1 * * P4 * +** +** P4 is the name of a virtual table in database P1. Call the xCreate method +** for that table. +*/ +case OP_VCreate: { + rc = sqlite3VtabCallCreate(db, pOp->p1, pOp->p4.z, &p->zErrMsg); + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VDestroy P1 * * P4 * +** +** P4 is the name of a virtual table in database P1. Call the xDestroy method +** of that table. +*/ +case OP_VDestroy: { + p->inVtabMethod = 2; + rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); + p->inVtabMethod = 0; + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VOpen P1 * * P4 * +** +** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. +** P1 is a cursor number. This opcode opens a cursor to the virtual +** table and stores that cursor in P1. +*/ +case OP_VOpen: { + VdbeCursor *pCur; + sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab *pVtab; + sqlite3_module *pModule; + + assert( p->bIsReader ); + pCur = 0; + pVtabCursor = 0; + pVtab = pOp->p4.pVtab->pVtab; + pModule = (sqlite3_module *)pVtab->pModule; + assert(pVtab && pModule); + rc = pModule->xOpen(pVtab, &pVtabCursor); + sqlite3VtabImportErrmsg(p, pVtab); + if( SQLITE_OK==rc ){ + /* Initialize sqlite3_vtab_cursor base class */ + pVtabCursor->pVtab = pVtab; + + /* Initialize vdbe cursor object */ + pCur = allocateCursor(p, pOp->p1, 0, -1, 0); + if( pCur ){ + pCur->pVtabCursor = pVtabCursor; + }else{ + db->mallocFailed = 1; + pModule->xClose(pVtabCursor); + } + } + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VFilter P1 P2 P3 P4 * +** Synopsis: iplan=r[P3] zplan='P4' +** +** P1 is a cursor opened using VOpen. P2 is an address to jump to if +** the filtered result set is empty. +** +** P4 is either NULL or a string that was generated by the xBestIndex +** method of the module. The interpretation of the P4 string is left +** to the module implementation. +** +** This opcode invokes the xFilter method on the virtual table specified +** by P1. The integer query plan parameter to xFilter is stored in register +** P3. Register P3+1 stores the argc parameter to be passed to the +** xFilter method. Registers P3+2..P3+1+argc are the argc +** additional parameters which are passed to +** xFilter as argv. Register P3+2 becomes argv[0] when passed to xFilter. +** +** A jump is made to P2 if the result set after filtering would be empty. +*/ +case OP_VFilter: { /* jump */ + int nArg; + int iQuery; + const sqlite3_module *pModule; + Mem *pQuery; + Mem *pArgc; + sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab *pVtab; + VdbeCursor *pCur; + int res; + int i; + Mem **apArg; + + pQuery = &aMem[pOp->p3]; + pArgc = &pQuery[1]; + pCur = p->apCsr[pOp->p1]; + assert( memIsValid(pQuery) ); + REGISTER_TRACE(pOp->p3, pQuery); + assert( pCur->pVtabCursor ); + pVtabCursor = pCur->pVtabCursor; + pVtab = pVtabCursor->pVtab; + pModule = pVtab->pModule; + + /* Grab the index number and argc parameters */ + assert( (pQuery->flags&MEM_Int)!=0 && pArgc->flags==MEM_Int ); + nArg = (int)pArgc->u.i; + iQuery = (int)pQuery->u.i; + + /* Invoke the xFilter method */ + { + res = 0; + apArg = p->apArg; + for(i = 0; iinVtabMethod = 1; + rc = pModule->xFilter(pVtabCursor, iQuery, pOp->p4.z, nArg, apArg); + p->inVtabMethod = 0; + sqlite3VtabImportErrmsg(p, pVtab); + if( rc==SQLITE_OK ){ + res = pModule->xEof(pVtabCursor); + } + VdbeBranchTaken(res!=0,2); + if( res ){ + pc = pOp->p2 - 1; + } + } + pCur->nullRow = 0; + + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VColumn P1 P2 P3 * * +** Synopsis: r[P3]=vcolumn(P2) +** +** Store the value of the P2-th column of +** the row of the virtual-table that the +** P1 cursor is pointing to into register P3. +*/ +case OP_VColumn: { + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + Mem *pDest; + sqlite3_context sContext; + + VdbeCursor *pCur = p->apCsr[pOp->p1]; + assert( pCur->pVtabCursor ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); + if( pCur->nullRow ){ + sqlite3VdbeMemSetNull(pDest); + break; + } + pVtab = pCur->pVtabCursor->pVtab; + pModule = pVtab->pModule; + assert( pModule->xColumn ); + memset(&sContext, 0, sizeof(sContext)); + + /* The output cell may already have a buffer allocated. Move + ** the current contents to sContext.s so in case the user-function + ** can use the already allocated buffer instead of allocating a + ** new one. + */ + sqlite3VdbeMemMove(&sContext.s, pDest); + MemSetTypeFlag(&sContext.s, MEM_Null); + + rc = pModule->xColumn(pCur->pVtabCursor, &sContext, pOp->p2); + sqlite3VtabImportErrmsg(p, pVtab); + if( sContext.isError ){ + rc = sContext.isError; + } + + /* Copy the result of the function to the P3 register. We + ** do this regardless of whether or not an error occurred to ensure any + ** dynamic allocation in sContext.s (a Mem struct) is released. + */ + sqlite3VdbeChangeEncoding(&sContext.s, encoding); + sqlite3VdbeMemMove(pDest, &sContext.s); + REGISTER_TRACE(pOp->p3, pDest); + UPDATE_MAX_BLOBSIZE(pDest); + + if( sqlite3VdbeMemTooBig(pDest) ){ + goto too_big; + } + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VNext P1 P2 * * * +** +** Advance virtual table P1 to the next row in its result set and +** jump to instruction P2. Or, if the virtual table has reached +** the end of its result set, then fall through to the next instruction. +*/ +case OP_VNext: { /* jump */ + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + int res; + VdbeCursor *pCur; + + res = 0; + pCur = p->apCsr[pOp->p1]; + assert( pCur->pVtabCursor ); + if( pCur->nullRow ){ + break; + } + pVtab = pCur->pVtabCursor->pVtab; + pModule = pVtab->pModule; + assert( pModule->xNext ); + + /* Invoke the xNext() method of the module. There is no way for the + ** underlying implementation to return an error if one occurs during + ** xNext(). Instead, if an error occurs, true is returned (indicating that + ** data is available) and the error code returned when xColumn or + ** some other method is next invoked on the save virtual table cursor. + */ + p->inVtabMethod = 1; + rc = pModule->xNext(pCur->pVtabCursor); + p->inVtabMethod = 0; + sqlite3VtabImportErrmsg(p, pVtab); + if( rc==SQLITE_OK ){ + res = pModule->xEof(pCur->pVtabCursor); + } + VdbeBranchTaken(!res,2); + if( !res ){ + /* If there is data, jump to P2 */ + pc = pOp->p2 - 1; + } + goto check_for_interrupt; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VRename P1 * * P4 * +** +** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. +** This opcode invokes the corresponding xRename method. The value +** in register P1 is passed as the zName argument to the xRename method. +*/ +case OP_VRename: { + sqlite3_vtab *pVtab; + Mem *pName; + + pVtab = pOp->p4.pVtab->pVtab; + pName = &aMem[pOp->p1]; + assert( pVtab->pModule->xRename ); + assert( memIsValid(pName) ); + assert( p->readOnly==0 ); + REGISTER_TRACE(pOp->p1, pName); + assert( pName->flags & MEM_Str ); + testcase( pName->enc==SQLITE_UTF8 ); + testcase( pName->enc==SQLITE_UTF16BE ); + testcase( pName->enc==SQLITE_UTF16LE ); + rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); + if( rc==SQLITE_OK ){ + rc = pVtab->pModule->xRename(pVtab, pName->z); + sqlite3VtabImportErrmsg(p, pVtab); + p->expired = 0; + } + break; +} +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VUpdate P1 P2 P3 P4 P5 +** Synopsis: data=r[P3@P2] +** +** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. +** This opcode invokes the corresponding xUpdate method. P2 values +** are contiguous memory cells starting at P3 to pass to the xUpdate +** invocation. The value in register (P3+P2-1) corresponds to the +** p2th element of the argv array passed to xUpdate. +** +** The xUpdate method will do a DELETE or an INSERT or both. +** The argv[0] element (which corresponds to memory cell P3) +** is the rowid of a row to delete. If argv[0] is NULL then no +** deletion occurs. The argv[1] element is the rowid of the new +** row. This can be NULL to have the virtual table select the new +** rowid for itself. The subsequent elements in the array are +** the values of columns in the new row. +** +** If P2==1 then no insert is performed. argv[0] is the rowid of +** a row to delete. +** +** P1 is a boolean flag. If it is set to true and the xUpdate call +** is successful, then the value returned by sqlite3_last_insert_rowid() +** is set to the value of the rowid for the row just inserted. +** +** P5 is the error actions (OE_Replace, OE_Fail, OE_Ignore, etc) to +** apply in the case of a constraint failure on an insert or update. +*/ +case OP_VUpdate: { + sqlite3_vtab *pVtab; + sqlite3_module *pModule; + int nArg; + int i; + sqlite_int64 rowid; + Mem **apArg; + Mem *pX; + + assert( pOp->p2==1 || pOp->p5==OE_Fail || pOp->p5==OE_Rollback + || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace + ); + assert( p->readOnly==0 ); + pVtab = pOp->p4.pVtab->pVtab; + pModule = (sqlite3_module *)pVtab->pModule; + nArg = pOp->p2; + assert( pOp->p4type==P4_VTAB ); + if( ALWAYS(pModule->xUpdate) ){ + u8 vtabOnConflict = db->vtabOnConflict; + apArg = p->apArg; + pX = &aMem[pOp->p3]; + for(i=0; ivtabOnConflict = pOp->p5; + rc = pModule->xUpdate(pVtab, nArg, apArg, &rowid); + db->vtabOnConflict = vtabOnConflict; + sqlite3VtabImportErrmsg(p, pVtab); + if( rc==SQLITE_OK && pOp->p1 ){ + assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); + db->lastRowid = lastRowid = rowid; + } + if( (rc&0xff)==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ + if( pOp->p5==OE_Ignore ){ + rc = SQLITE_OK; + }else{ + p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); + } + }else{ + p->nChange++; + } + } + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* Opcode: Pagecount P1 P2 * * * +** +** Write the current number of pages in database P1 to memory cell P2. +*/ +case OP_Pagecount: { /* out2-prerelease */ + pOut->u.i = sqlite3BtreeLastPage(db->aDb[pOp->p1].pBt); + break; +} +#endif + + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* Opcode: MaxPgcnt P1 P2 P3 * * +** +** Try to set the maximum page count for database P1 to the value in P3. +** Do not let the maximum page count fall below the current page count and +** do not change the maximum page count value if P3==0. +** +** Store the maximum page count after the change in register P2. +*/ +case OP_MaxPgcnt: { /* out2-prerelease */ + unsigned int newMax; + Btree *pBt; + + pBt = db->aDb[pOp->p1].pBt; + newMax = 0; + if( pOp->p3 ){ + newMax = sqlite3BtreeLastPage(pBt); + if( newMax < (unsigned)pOp->p3 ) newMax = (unsigned)pOp->p3; + } + pOut->u.i = sqlite3BtreeMaxPageCount(pBt, newMax); + break; +} +#endif + + +/* Opcode: Init * P2 * P4 * +** Synopsis: Start at P2 +** +** Programs contain a single instance of this opcode as the very first +** opcode. +** +** If tracing is enabled (by the sqlite3_trace()) interface, then +** the UTF-8 string contained in P4 is emitted on the trace callback. +** Or if P4 is blank, use the string returned by sqlite3_sql(). +** +** If P2 is not zero, jump to instruction P2. +*/ +case OP_Init: { /* jump */ + char *zTrace; + char *z; + + if( pOp->p2 ){ + pc = pOp->p2 - 1; + } +#ifndef SQLITE_OMIT_TRACE + if( db->xTrace + && !p->doingRerun + && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ + z = sqlite3VdbeExpandSql(p, zTrace); + db->xTrace(db->pTraceArg, z); + sqlite3DbFree(db, z); + } +#ifdef SQLITE_USE_FCNTL_TRACE + zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); + if( zTrace ){ + int i; + for(i=0; inDb; i++){ + if( DbMaskTest(p->btreeMask, i)==0 ) continue; + sqlite3_file_control(db, db->aDb[i].zName, SQLITE_FCNTL_TRACE, zTrace); + } + } +#endif /* SQLITE_USE_FCNTL_TRACE */ +#ifdef SQLITE_DEBUG + if( (db->flags & SQLITE_SqlTrace)!=0 + && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ + sqlite3DebugPrintf("SQL-trace: %s\n", zTrace); + } +#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_OMIT_TRACE */ + break; +} + + +/* Opcode: Noop * * * * * +** +** Do nothing. This instruction is often useful as a jump +** destination. +*/ +/* +** The magic Explain opcode are only inserted when explain==2 (which +** is to say when the EXPLAIN QUERY PLAN syntax is used.) +** This opcode records information from the optimizer. It is the +** the same as a no-op. This opcodesnever appears in a real VM program. +*/ +default: { /* This is really OP_Noop and OP_Explain */ + assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain ); + break; +} + +/***************************************************************************** +** The cases of the switch statement above this line should all be indented +** by 6 spaces. But the left-most 6 spaces have been removed to improve the +** readability. From this point on down, the normal indentation rules are +** restored. +*****************************************************************************/ + } + +#ifdef VDBE_PROFILE + { + u64 endTime = sqlite3Hwtime(); + if( endTime>start ) pOp->cycles += endTime - start; + pOp->cnt++; + } +#endif + + /* The following code adds nothing to the actual functionality + ** of the program. It is only here for testing and debugging. + ** On the other hand, it does burn CPU cycles every time through + ** the evaluator loop. So we can leave it out when NDEBUG is defined. + */ +#ifndef NDEBUG + assert( pc>=-1 && pcnOp ); + +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeTrace ){ + if( rc!=0 ) printf("rc=%d\n",rc); + if( pOp->opflags & (OPFLG_OUT2_PRERELEASE|OPFLG_OUT2) ){ + registerTrace(pOp->p2, &aMem[pOp->p2]); + } + if( pOp->opflags & OPFLG_OUT3 ){ + registerTrace(pOp->p3, &aMem[pOp->p3]); + } + } +#endif /* SQLITE_DEBUG */ +#endif /* NDEBUG */ + } /* The end of the for(;;) loop the loops through opcodes */ + + /* If we reach this point, it means that execution is finished with + ** an error of some kind. + */ +vdbe_error_halt: + assert( rc ); + p->rc = rc; + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(rc, "statement aborts at %d: [%s] %s", + pc, p->zSql, p->zErrMsg); + sqlite3VdbeHalt(p); + if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; + rc = SQLITE_ERROR; + if( resetSchemaOnFault>0 ){ + sqlite3ResetOneSchema(db, resetSchemaOnFault-1); + } + + /* This is the only way out of this procedure. We have to + ** release the mutexes on btrees that were acquired at the + ** top. */ +vdbe_return: + db->lastRowid = lastRowid; + testcase( nVmStep>0 ); + p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; + sqlite3VdbeLeave(p); + return rc; + + /* Jump to here if a string or blob larger than SQLITE_MAX_LENGTH + ** is encountered. + */ +too_big: + sqlite3SetString(&p->zErrMsg, db, "string or blob too big"); + rc = SQLITE_TOOBIG; + goto vdbe_error_halt; + + /* Jump to here if a malloc() fails. + */ +no_mem: + db->mallocFailed = 1; + sqlite3SetString(&p->zErrMsg, db, "out of memory"); + rc = SQLITE_NOMEM; + goto vdbe_error_halt; + + /* Jump to here for any other kind of fatal error. The "rc" variable + ** should hold the error number. + */ +abort_due_to_error: + assert( p->zErrMsg==0 ); + if( db->mallocFailed ) rc = SQLITE_NOMEM; + if( rc!=SQLITE_IOERR_NOMEM ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); + } + goto vdbe_error_halt; + + /* Jump to here if the sqlite3_interrupt() API sets the interrupt + ** flag. + */ +abort_due_to_interrupt: + assert( db->u1.isInterrupted ); + rc = SQLITE_INTERRUPT; + p->rc = rc; + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); + goto vdbe_error_halt; +} + + +/************** End of vdbe.c ************************************************/ +/************** Begin file vdbeblob.c ****************************************/ +/* +** 2007 May 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to implement incremental BLOB I/O. +*/ + + +#ifndef SQLITE_OMIT_INCRBLOB + +/* +** Valid sqlite3_blob* handles point to Incrblob structures. +*/ +typedef struct Incrblob Incrblob; +struct Incrblob { + int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ + int nByte; /* Size of open blob, in bytes */ + int iOffset; /* Byte offset of blob in cursor data */ + int iCol; /* Table column this handle is open on */ + BtCursor *pCsr; /* Cursor pointing at blob row */ + sqlite3_stmt *pStmt; /* Statement holding cursor open */ + sqlite3 *db; /* The associated database */ +}; + + +/* +** This function is used by both blob_open() and blob_reopen(). It seeks +** the b-tree cursor associated with blob handle p to point to row iRow. +** If successful, SQLITE_OK is returned and subsequent calls to +** sqlite3_blob_read() or sqlite3_blob_write() access the specified row. +** +** If an error occurs, or if the specified row does not exist or does not +** contain a value of type TEXT or BLOB in the column nominated when the +** blob handle was opened, then an error code is returned and *pzErr may +** be set to point to a buffer containing an error message. It is the +** responsibility of the caller to free the error message buffer using +** sqlite3DbFree(). +** +** If an error does occur, then the b-tree cursor is closed. All subsequent +** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will +** immediately return SQLITE_ABORT. +*/ +static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ + int rc; /* Error code */ + char *zErr = 0; /* Error message */ + Vdbe *v = (Vdbe *)p->pStmt; + + /* Set the value of the SQL statements only variable to integer iRow. + ** This is done directly instead of using sqlite3_bind_int64() to avoid + ** triggering asserts related to mutexes. + */ + assert( v->aVar[0].flags&MEM_Int ); + v->aVar[0].u.i = iRow; + + rc = sqlite3_step(p->pStmt); + if( rc==SQLITE_ROW ){ + VdbeCursor *pC = v->apCsr[0]; + u32 type = pC->aType[p->iCol]; + if( type<12 ){ + zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", + type==0?"null": type==7?"real": "integer" + ); + rc = SQLITE_ERROR; + sqlite3_finalize(p->pStmt); + p->pStmt = 0; + }else{ + p->iOffset = pC->aType[p->iCol + pC->nField]; + p->nByte = sqlite3VdbeSerialTypeLen(type); + p->pCsr = pC->pCursor; + sqlite3BtreeIncrblobCursor(p->pCsr); + } + } + + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + }else if( p->pStmt ){ + rc = sqlite3_finalize(p->pStmt); + p->pStmt = 0; + if( rc==SQLITE_OK ){ + zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow); + rc = SQLITE_ERROR; + }else{ + zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db)); + } + } + + assert( rc!=SQLITE_OK || zErr==0 ); + assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE ); + + *pzErr = zErr; + return rc; +} + +/* +** Open a blob handle. +*/ +SQLITE_API int sqlite3_blob_open( + sqlite3* db, /* The database connection */ + const char *zDb, /* The attached database containing the blob */ + const char *zTable, /* The table containing the blob */ + const char *zColumn, /* The column containing the blob */ + sqlite_int64 iRow, /* The row containing the glob */ + int flags, /* True -> read/write access, false -> read-only */ + sqlite3_blob **ppBlob /* Handle for accessing the blob returned here */ +){ + int nAttempt = 0; + int iCol; /* Index of zColumn in row-record */ + + /* This VDBE program seeks a btree cursor to the identified + ** db/table/row entry. The reason for using a vdbe program instead + ** of writing code to use the b-tree layer directly is that the + ** vdbe program will take advantage of the various transaction, + ** locking and error handling infrastructure built into the vdbe. + ** + ** After seeking the cursor, the vdbe executes an OP_ResultRow. + ** Code external to the Vdbe then "borrows" the b-tree cursor and + ** uses it to implement the blob_read(), blob_write() and + ** blob_bytes() functions. + ** + ** The sqlite3_blob_close() function finalizes the vdbe program, + ** which closes the b-tree cursor and (possibly) commits the + ** transaction. + */ + static const int iLn = VDBE_OFFSET_LINENO(4); + static const VdbeOpList openBlob[] = { + /* {OP_Transaction, 0, 0, 0}, // 0: Inserted separately */ + {OP_TableLock, 0, 0, 0}, /* 1: Acquire a read or write lock */ + /* One of the following two instructions is replaced by an OP_Noop. */ + {OP_OpenRead, 0, 0, 0}, /* 2: Open cursor 0 for reading */ + {OP_OpenWrite, 0, 0, 0}, /* 3: Open cursor 0 for read/write */ + {OP_Variable, 1, 1, 1}, /* 4: Push the rowid to the stack */ + {OP_NotExists, 0, 10, 1}, /* 5: Seek the cursor */ + {OP_Column, 0, 0, 1}, /* 6 */ + {OP_ResultRow, 1, 0, 0}, /* 7 */ + {OP_Goto, 0, 4, 0}, /* 8 */ + {OP_Close, 0, 0, 0}, /* 9 */ + {OP_Halt, 0, 0, 0}, /* 10 */ + }; + + int rc = SQLITE_OK; + char *zErr = 0; + Table *pTab; + Parse *pParse = 0; + Incrblob *pBlob = 0; + + flags = !!flags; /* flags = (flags ? 1 : 0); */ + *ppBlob = 0; + + sqlite3_mutex_enter(db->mutex); + + pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); + if( !pBlob ) goto blob_open_out; + pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); + if( !pParse ) goto blob_open_out; + + do { + memset(pParse, 0, sizeof(Parse)); + pParse->db = db; + sqlite3DbFree(db, zErr); + zErr = 0; + + sqlite3BtreeEnterAll(db); + pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); + if( pTab && IsVirtual(pTab) ){ + pTab = 0; + sqlite3ErrorMsg(pParse, "cannot open virtual table: %s", zTable); + } + if( pTab && !HasRowid(pTab) ){ + pTab = 0; + sqlite3ErrorMsg(pParse, "cannot open table without rowid: %s", zTable); + } +#ifndef SQLITE_OMIT_VIEW + if( pTab && pTab->pSelect ){ + pTab = 0; + sqlite3ErrorMsg(pParse, "cannot open view: %s", zTable); + } +#endif + if( !pTab ){ + if( pParse->zErrMsg ){ + sqlite3DbFree(db, zErr); + zErr = pParse->zErrMsg; + pParse->zErrMsg = 0; + } + rc = SQLITE_ERROR; + sqlite3BtreeLeaveAll(db); + goto blob_open_out; + } + + /* Now search pTab for the exact column. */ + for(iCol=0; iColnCol; iCol++) { + if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){ + break; + } + } + if( iCol==pTab->nCol ){ + sqlite3DbFree(db, zErr); + zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn); + rc = SQLITE_ERROR; + sqlite3BtreeLeaveAll(db); + goto blob_open_out; + } + + /* If the value is being opened for writing, check that the + ** column is not indexed, and that it is not part of a foreign key. + ** It is against the rules to open a column to which either of these + ** descriptions applies for writing. */ + if( flags ){ + const char *zFault = 0; + Index *pIdx; +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( db->flags&SQLITE_ForeignKeys ){ + /* Check that the column is not part of an FK child key definition. It + ** is not necessary to check if it is part of a parent key, as parent + ** key columns must be indexed. The check below will pick up this + ** case. */ + FKey *pFKey; + for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + int j; + for(j=0; jnCol; j++){ + if( pFKey->aCol[j].iFrom==iCol ){ + zFault = "foreign key"; + } + } + } + } +#endif + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int j; + for(j=0; jnKeyCol; j++){ + if( pIdx->aiColumn[j]==iCol ){ + zFault = "indexed"; + } + } + } + if( zFault ){ + sqlite3DbFree(db, zErr); + zErr = sqlite3MPrintf(db, "cannot open %s column for writing", zFault); + rc = SQLITE_ERROR; + sqlite3BtreeLeaveAll(db); + goto blob_open_out; + } + } + + pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(pParse); + assert( pBlob->pStmt || db->mallocFailed ); + if( pBlob->pStmt ){ + Vdbe *v = (Vdbe *)pBlob->pStmt; + int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + + + sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, + pTab->pSchema->schema_cookie, + pTab->pSchema->iGeneration); + sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); + + /* Make sure a mutex is held on the table to be accessed */ + sqlite3VdbeUsesBtree(v, iDb); + + /* Configure the OP_TableLock instruction */ +#ifdef SQLITE_OMIT_SHARED_CACHE + sqlite3VdbeChangeToNoop(v, 1); +#else + sqlite3VdbeChangeP1(v, 1, iDb); + sqlite3VdbeChangeP2(v, 1, pTab->tnum); + sqlite3VdbeChangeP3(v, 1, flags); + sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); +#endif + + /* Remove either the OP_OpenWrite or OpenRead. Set the P2 + ** parameter of the other to pTab->tnum. */ + sqlite3VdbeChangeToNoop(v, 3 - flags); + sqlite3VdbeChangeP2(v, 2 + flags, pTab->tnum); + sqlite3VdbeChangeP3(v, 2 + flags, iDb); + + /* Configure the number of columns. Configure the cursor to + ** think that the table has one more column than it really + ** does. An OP_Column to retrieve this imaginary column will + ** always return an SQL NULL. This is useful because it means + ** we can invoke OP_Column to fill in the vdbe cursors type + ** and offset cache without causing any IO. + */ + sqlite3VdbeChangeP4(v, 2+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); + sqlite3VdbeChangeP2(v, 6, pTab->nCol); + if( !db->mallocFailed ){ + pParse->nVar = 1; + pParse->nMem = 1; + pParse->nTab = 1; + sqlite3VdbeMakeReady(v, pParse); + } + } + + pBlob->flags = flags; + pBlob->iCol = iCol; + pBlob->db = db; + sqlite3BtreeLeaveAll(db); + if( db->mallocFailed ){ + goto blob_open_out; + } + sqlite3_bind_int64(pBlob->pStmt, 1, iRow); + rc = blobSeekToRow(pBlob, iRow, &zErr); + } while( (++nAttempt)mallocFailed==0 ){ + *ppBlob = (sqlite3_blob *)pBlob; + }else{ + if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); + sqlite3DbFree(db, pBlob); + } + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + sqlite3ParserReset(pParse); + sqlite3StackFree(db, pParse); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Close a blob handle that was previously created using +** sqlite3_blob_open(). +*/ +SQLITE_API int sqlite3_blob_close(sqlite3_blob *pBlob){ + Incrblob *p = (Incrblob *)pBlob; + int rc; + sqlite3 *db; + + if( p ){ + db = p->db; + sqlite3_mutex_enter(db->mutex); + rc = sqlite3_finalize(p->pStmt); + sqlite3DbFree(db, p); + sqlite3_mutex_leave(db->mutex); + }else{ + rc = SQLITE_OK; + } + return rc; +} + +/* +** Perform a read or write operation on a blob +*/ +static int blobReadWrite( + sqlite3_blob *pBlob, + void *z, + int n, + int iOffset, + int (*xCall)(BtCursor*, u32, u32, void*) +){ + int rc; + Incrblob *p = (Incrblob *)pBlob; + Vdbe *v; + sqlite3 *db; + + if( p==0 ) return SQLITE_MISUSE_BKPT; + db = p->db; + sqlite3_mutex_enter(db->mutex); + v = (Vdbe*)p->pStmt; + + if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ + /* Request is out of range. Return a transient error. */ + rc = SQLITE_ERROR; + sqlite3Error(db, SQLITE_ERROR, 0); + }else if( v==0 ){ + /* If there is no statement handle, then the blob-handle has + ** already been invalidated. Return SQLITE_ABORT in this case. + */ + rc = SQLITE_ABORT; + }else{ + /* Call either BtreeData() or BtreePutData(). If SQLITE_ABORT is + ** returned, clean-up the statement handle. + */ + assert( db == v->db ); + sqlite3BtreeEnterCursor(p->pCsr); + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); + sqlite3BtreeLeaveCursor(p->pCsr); + if( rc==SQLITE_ABORT ){ + sqlite3VdbeFinalize(v); + p->pStmt = 0; + }else{ + db->errCode = rc; + v->rc = rc; + } + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Read data from a blob handle. +*/ +SQLITE_API int sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ + return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreeData); +} + +/* +** Write data to a blob handle. +*/ +SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ + return blobReadWrite(pBlob, (void *)z, n, iOffset, sqlite3BtreePutData); +} + +/* +** Query a blob handle for the size of the data. +** +** The Incrblob.nByte field is fixed for the lifetime of the Incrblob +** so no mutex is required for access. +*/ +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ + Incrblob *p = (Incrblob *)pBlob; + return (p && p->pStmt) ? p->nByte : 0; +} + +/* +** Move an existing blob handle to point to a different row of the same +** database table. +** +** If an error occurs, or if the specified row does not exist or does not +** contain a blob or text value, then an error code is returned and the +** database handle error code and message set. If this happens, then all +** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) +** immediately return SQLITE_ABORT. +*/ +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ + int rc; + Incrblob *p = (Incrblob *)pBlob; + sqlite3 *db; + + if( p==0 ) return SQLITE_MISUSE_BKPT; + db = p->db; + sqlite3_mutex_enter(db->mutex); + + if( p->pStmt==0 ){ + /* If there is no statement handle, then the blob-handle has + ** already been invalidated. Return SQLITE_ABORT in this case. + */ + rc = SQLITE_ABORT; + }else{ + char *zErr; + rc = blobSeekToRow(p, iRow, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + } + assert( rc!=SQLITE_SCHEMA ); + } + + rc = sqlite3ApiExit(db, rc); + assert( rc==SQLITE_OK || p->pStmt==0 ); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#endif /* #ifndef SQLITE_OMIT_INCRBLOB */ + +/************** End of vdbeblob.c ********************************************/ +/************** Begin file vdbesort.c ****************************************/ +/* +** 2011 July 9 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code for the VdbeSorter object, used in concert with +** a VdbeCursor to sort large numbers of keys (as may be required, for +** example, by CREATE INDEX statements on tables too large to fit in main +** memory). +*/ + + + +typedef struct VdbeSorterIter VdbeSorterIter; +typedef struct SorterRecord SorterRecord; +typedef struct FileWriter FileWriter; + +/* +** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: +** +** As keys are added to the sorter, they are written to disk in a series +** of sorted packed-memory-arrays (PMAs). The size of each PMA is roughly +** the same as the cache-size allowed for temporary databases. In order +** to allow the caller to extract keys from the sorter in sorted order, +** all PMAs currently stored on disk must be merged together. This comment +** describes the data structure used to do so. The structure supports +** merging any number of arrays in a single pass with no redundant comparison +** operations. +** +** The aIter[] array contains an iterator for each of the PMAs being merged. +** An aIter[] iterator either points to a valid key or else is at EOF. For +** the purposes of the paragraphs below, we assume that the array is actually +** N elements in size, where N is the smallest power of 2 greater to or equal +** to the number of iterators being merged. The extra aIter[] elements are +** treated as if they are empty (always at EOF). +** +** The aTree[] array is also N elements in size. The value of N is stored in +** the VdbeSorter.nTree variable. +** +** The final (N/2) elements of aTree[] contain the results of comparing +** pairs of iterator keys together. Element i contains the result of +** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the +** aTree element is set to the index of it. +** +** For the purposes of this comparison, EOF is considered greater than any +** other key value. If the keys are equal (only possible with two EOF +** values), it doesn't matter which index is stored. +** +** The (N/4) elements of aTree[] that precede the final (N/2) described +** above contains the index of the smallest of each block of 4 iterators. +** And so on. So that aTree[1] contains the index of the iterator that +** currently points to the smallest key value. aTree[0] is unused. +** +** Example: +** +** aIter[0] -> Banana +** aIter[1] -> Feijoa +** aIter[2] -> Elderberry +** aIter[3] -> Currant +** aIter[4] -> Grapefruit +** aIter[5] -> Apple +** aIter[6] -> Durian +** aIter[7] -> EOF +** +** aTree[] = { X, 5 0, 5 0, 3, 5, 6 } +** +** The current element is "Apple" (the value of the key indicated by +** iterator 5). When the Next() operation is invoked, iterator 5 will +** be advanced to the next key in its segment. Say the next key is +** "Eggplant": +** +** aIter[5] -> Eggplant +** +** The contents of aTree[] are updated first by comparing the new iterator +** 5 key to the current key of iterator 4 (still "Grapefruit"). The iterator +** 5 value is still smaller, so aTree[6] is set to 5. And so on up the tree. +** The value of iterator 6 - "Durian" - is now smaller than that of iterator +** 5, so aTree[3] is set to 6. Key 0 is smaller than key 6 (BananaaAlloc); + sqlite3DbFree(db, pIter->aBuffer); + memset(pIter, 0, sizeof(VdbeSorterIter)); +} + +/* +** Read nByte bytes of data from the stream of data iterated by object p. +** If successful, set *ppOut to point to a buffer containing the data +** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite +** error code. +** +** The buffer indicated by *ppOut may only be considered valid until the +** next call to this function. +*/ +static int vdbeSorterIterRead( + sqlite3 *db, /* Database handle (for malloc) */ + VdbeSorterIter *p, /* Iterator */ + int nByte, /* Bytes of data to read */ + u8 **ppOut /* OUT: Pointer to buffer containing data */ +){ + int iBuf; /* Offset within buffer to read from */ + int nAvail; /* Bytes of data available in buffer */ + assert( p->aBuffer ); + + /* If there is no more data to be read from the buffer, read the next + ** p->nBuffer bytes of data from the file into it. Or, if there are less + ** than p->nBuffer bytes remaining in the PMA, read all remaining data. */ + iBuf = p->iReadOff % p->nBuffer; + if( iBuf==0 ){ + int nRead; /* Bytes to read from disk */ + int rc; /* sqlite3OsRead() return code */ + + /* Determine how many bytes of data to read. */ + if( (p->iEof - p->iReadOff) > (i64)p->nBuffer ){ + nRead = p->nBuffer; + }else{ + nRead = (int)(p->iEof - p->iReadOff); + } + assert( nRead>0 ); + + /* Read data from the file. Return early if an error occurs. */ + rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff); + assert( rc!=SQLITE_IOERR_SHORT_READ ); + if( rc!=SQLITE_OK ) return rc; + } + nAvail = p->nBuffer - iBuf; + + if( nByte<=nAvail ){ + /* The requested data is available in the in-memory buffer. In this + ** case there is no need to make a copy of the data, just return a + ** pointer into the buffer to the caller. */ + *ppOut = &p->aBuffer[iBuf]; + p->iReadOff += nByte; + }else{ + /* The requested data is not all available in the in-memory buffer. + ** In this case, allocate space at p->aAlloc[] to copy the requested + ** range into. Then return a copy of pointer p->aAlloc to the caller. */ + int nRem; /* Bytes remaining to copy */ + + /* Extend the p->aAlloc[] allocation if required. */ + if( p->nAllocnAlloc*2; + while( nByte>nNew ) nNew = nNew*2; + p->aAlloc = sqlite3DbReallocOrFree(db, p->aAlloc, nNew); + if( !p->aAlloc ) return SQLITE_NOMEM; + p->nAlloc = nNew; + } + + /* Copy as much data as is available in the buffer into the start of + ** p->aAlloc[]. */ + memcpy(p->aAlloc, &p->aBuffer[iBuf], nAvail); + p->iReadOff += nAvail; + nRem = nByte - nAvail; + + /* The following loop copies up to p->nBuffer bytes per iteration into + ** the p->aAlloc[] buffer. */ + while( nRem>0 ){ + int rc; /* vdbeSorterIterRead() return code */ + int nCopy; /* Number of bytes to copy */ + u8 *aNext; /* Pointer to buffer to copy data from */ + + nCopy = nRem; + if( nRem>p->nBuffer ) nCopy = p->nBuffer; + rc = vdbeSorterIterRead(db, p, nCopy, &aNext); + if( rc!=SQLITE_OK ) return rc; + assert( aNext!=p->aAlloc ); + memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); + nRem -= nCopy; + } + + *ppOut = p->aAlloc; + } + + return SQLITE_OK; +} + +/* +** Read a varint from the stream of data accessed by p. Set *pnOut to +** the value read. +*/ +static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ + int iBuf; + + iBuf = p->iReadOff % p->nBuffer; + if( iBuf && (p->nBuffer-iBuf)>=9 ){ + p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + }else{ + u8 aVarint[16], *a; + int i = 0, rc; + do{ + rc = vdbeSorterIterRead(db, p, 1, &a); + if( rc ) return rc; + aVarint[(i++)&0xf] = a[0]; + }while( (a[0]&0x80)!=0 ); + sqlite3GetVarint(aVarint, pnOut); + } + + return SQLITE_OK; +} + + +/* +** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if +** no error occurs, or an SQLite error code if one does. +*/ +static int vdbeSorterIterNext( + sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */ + VdbeSorterIter *pIter /* Iterator to advance */ +){ + int rc; /* Return Code */ + u64 nRec = 0; /* Size of record in bytes */ + + if( pIter->iReadOff>=pIter->iEof ){ + /* This is an EOF condition */ + vdbeSorterIterZero(db, pIter); + return SQLITE_OK; + } + + rc = vdbeSorterIterVarint(db, pIter, &nRec); + if( rc==SQLITE_OK ){ + pIter->nKey = (int)nRec; + rc = vdbeSorterIterRead(db, pIter, (int)nRec, &pIter->aKey); + } + + return rc; +} + +/* +** Initialize iterator pIter to scan through the PMA stored in file pFile +** starting at offset iStart and ending at offset iEof-1. This function +** leaves the iterator pointing to the first key in the PMA (or EOF if the +** PMA is empty). +*/ +static int vdbeSorterIterInit( + sqlite3 *db, /* Database handle */ + const VdbeSorter *pSorter, /* Sorter object */ + i64 iStart, /* Start offset in pFile */ + VdbeSorterIter *pIter, /* Iterator to populate */ + i64 *pnByte /* IN/OUT: Increment this value by PMA size */ +){ + int rc = SQLITE_OK; + int nBuf; + + nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + + assert( pSorter->iWriteOff>iStart ); + assert( pIter->aAlloc==0 ); + assert( pIter->aBuffer==0 ); + pIter->pFile = pSorter->pTemp1; + pIter->iReadOff = iStart; + pIter->nAlloc = 128; + pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); + pIter->nBuffer = nBuf; + pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + + if( !pIter->aBuffer ){ + rc = SQLITE_NOMEM; + }else{ + int iBuf; + + iBuf = iStart % nBuf; + if( iBuf ){ + int nRead = nBuf - iBuf; + if( (iStart + nRead) > pSorter->iWriteOff ){ + nRead = (int)(pSorter->iWriteOff - iStart); + } + rc = sqlite3OsRead( + pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + ); + } + + if( rc==SQLITE_OK ){ + u64 nByte; /* Size of PMA in bytes */ + pIter->iEof = pSorter->iWriteOff; + rc = vdbeSorterIterVarint(db, pIter, &nByte); + pIter->iEof = pIter->iReadOff + nByte; + *pnByte += nByte; + } + } + + if( rc==SQLITE_OK ){ + rc = vdbeSorterIterNext(db, pIter); + } + return rc; +} + + +/* +** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, +** size nKey2 bytes). Argument pKeyInfo supplies the collation functions +** used by the comparison. If an error occurs, return an SQLite error code. +** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive +** value, depending on whether key1 is smaller, equal to or larger than key2. +** +** If the bOmitRowid argument is non-zero, assume both keys end in a rowid +** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid +** is true and key1 contains even a single NULL value, it is considered to +** be less than key2. Even if key2 also contains NULL values. +** +** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace +** has been allocated and contains an unpacked record that is used as key2. +*/ +static void vdbeSorterCompare( + const VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ + int nKeyCol, /* Num of columns. 0 means "all" */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2, /* Right side of comparison */ + int *pRes /* OUT: Result of comparison */ +){ + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + VdbeSorter *pSorter = pCsr->pSorter; + UnpackedRecord *r2 = pSorter->pUnpacked; + int i; + + if( pKey2 ){ + sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); + } + + if( nKeyCol ){ + r2->nField = nKeyCol; + for(i=0; iaMem[i].flags & MEM_Null ){ + *pRes = -1; + return; + } + } + assert( r2->default_rc==0 ); + } + + *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); +} + +/* +** This function is called to compare two iterator keys when merging +** multiple b-tree segments. Parameter iOut is the index of the aTree[] +** value to recalculate. +*/ +static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ + VdbeSorter *pSorter = pCsr->pSorter; + int i1; + int i2; + int iRes; + VdbeSorterIter *p1; + VdbeSorterIter *p2; + + assert( iOutnTree && iOut>0 ); + + if( iOut>=(pSorter->nTree/2) ){ + i1 = (iOut - pSorter->nTree/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pSorter->aTree[iOut*2]; + i2 = pSorter->aTree[iOut*2+1]; + } + + p1 = &pSorter->aIter[i1]; + p2 = &pSorter->aIter[i2]; + + if( p1->pFile==0 ){ + iRes = i2; + }else if( p2->pFile==0 ){ + iRes = i1; + }else{ + int res; + assert( pCsr->pSorter->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ + vdbeSorterCompare( + pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + ); + if( res<=0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pSorter->aTree[iOut] = iRes; + return SQLITE_OK; +} + +/* +** Initialize the temporary index cursor just opened as a sorter cursor. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ + int pgsz; /* Page size of main database */ + int mxCache; /* Cache size */ + VdbeSorter *pSorter; /* The new sorter */ + char *d; /* Dummy */ + + assert( pCsr->pKeyInfo && pCsr->pBt==0 ); + pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); + if( pSorter==0 ){ + return SQLITE_NOMEM; + } + + pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); + if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; + assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + + if( !sqlite3TempInMemory(db) ){ + pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCachemxPmaSize = mxCache * pgsz; + } + + return SQLITE_OK; +} + +/* +** Free the list of sorted records starting at pRecord. +*/ +static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ + SorterRecord *p; + SorterRecord *pNext; + for(p=pRecord; p; p=pNext){ + pNext = p->pNext; + sqlite3DbFree(db, p); + } +} + +/* +** Reset a sorting cursor back to its original empty state. +*/ +SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ + if( pSorter->aIter ){ + int i; + for(i=0; inTree; i++){ + vdbeSorterIterZero(db, &pSorter->aIter[i]); + } + sqlite3DbFree(db, pSorter->aIter); + pSorter->aIter = 0; + } + if( pSorter->pTemp1 ){ + sqlite3OsCloseFree(pSorter->pTemp1); + pSorter->pTemp1 = 0; + } + vdbeSorterRecordFree(db, pSorter->pRecord); + pSorter->pRecord = 0; + pSorter->iWriteOff = 0; + pSorter->iReadOff = 0; + pSorter->nInMemory = 0; + pSorter->nTree = 0; + pSorter->nPMA = 0; + pSorter->aTree = 0; +} + + +/* +** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. +*/ +SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ + VdbeSorter *pSorter = pCsr->pSorter; + if( pSorter ){ + sqlite3VdbeSorterReset(db, pSorter); + sqlite3DbFree(db, pSorter->pUnpacked); + sqlite3DbFree(db, pSorter); + pCsr->pSorter = 0; + } +} + +/* +** Allocate space for a file-handle and open a temporary file. If successful, +** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. +** Otherwise, set *ppFile to 0 and return an SQLite error code. +*/ +static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ + int dummy; + return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy + ); +} + +/* +** Merge the two sorted lists p1 and p2 into a single list. +** Set *ppOut to the head of the new list. +*/ +static void vdbeSorterMerge( + const VdbeCursor *pCsr, /* For pKeyInfo */ + SorterRecord *p1, /* First list to merge */ + SorterRecord *p2, /* Second list to merge */ + SorterRecord **ppOut /* OUT: Head of merged list */ +){ + SorterRecord *pFinal = 0; + SorterRecord **pp = &pFinal; + void *pVal2 = p2 ? p2->pVal : 0; + + while( p1 && p2 ){ + int res; + vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + if( res<=0 ){ + *pp = p1; + pp = &p1->pNext; + p1 = p1->pNext; + pVal2 = 0; + }else{ + *pp = p2; + pp = &p2->pNext; + p2 = p2->pNext; + if( p2==0 ) break; + pVal2 = p2->pVal; + } + } + *pp = p1 ? p1 : p2; + *ppOut = pFinal; +} + +/* +** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK +** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error +** occurs. +*/ +static int vdbeSorterSort(const VdbeCursor *pCsr){ + int i; + SorterRecord **aSlot; + SorterRecord *p; + VdbeSorter *pSorter = pCsr->pSorter; + + aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); + if( !aSlot ){ + return SQLITE_NOMEM; + } + + p = pSorter->pRecord; + while( p ){ + SorterRecord *pNext = p->pNext; + p->pNext = 0; + for(i=0; aSlot[i]; i++){ + vdbeSorterMerge(pCsr, p, aSlot[i], &p); + aSlot[i] = 0; + } + aSlot[i] = p; + p = pNext; + } + + p = 0; + for(i=0; i<64; i++){ + vdbeSorterMerge(pCsr, p, aSlot[i], &p); + } + pSorter->pRecord = p; + + sqlite3_free(aSlot); + return SQLITE_OK; +} + +/* +** Initialize a file-writer object. +*/ +static void fileWriterInit( + sqlite3 *db, /* Database (for malloc) */ + sqlite3_file *pFile, /* File to write to */ + FileWriter *p, /* Object to populate */ + i64 iStart /* Offset of pFile to begin writing at */ +){ + int nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + + memset(p, 0, sizeof(FileWriter)); + p->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + if( !p->aBuffer ){ + p->eFWErr = SQLITE_NOMEM; + }else{ + p->iBufEnd = p->iBufStart = (iStart % nBuf); + p->iWriteOff = iStart - p->iBufStart; + p->nBuffer = nBuf; + p->pFile = pFile; + } +} + +/* +** Write nData bytes of data to the file-write object. Return SQLITE_OK +** if successful, or an SQLite error code if an error occurs. +*/ +static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ + int nRem = nData; + while( nRem>0 && p->eFWErr==0 ){ + int nCopy = nRem; + if( nCopy>(p->nBuffer - p->iBufEnd) ){ + nCopy = p->nBuffer - p->iBufEnd; + } + + memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy); + p->iBufEnd += nCopy; + if( p->iBufEnd==p->nBuffer ){ + p->eFWErr = sqlite3OsWrite(p->pFile, + &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, + p->iWriteOff + p->iBufStart + ); + p->iBufStart = p->iBufEnd = 0; + p->iWriteOff += p->nBuffer; + } + assert( p->iBufEndnBuffer ); + + nRem -= nCopy; + } +} + +/* +** Flush any buffered data to disk and clean up the file-writer object. +** The results of using the file-writer after this call are undefined. +** Return SQLITE_OK if flushing the buffered data succeeds or is not +** required. Otherwise, return an SQLite error code. +** +** Before returning, set *piEof to the offset immediately following the +** last byte written to the file. +*/ +static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){ + int rc; + if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ + p->eFWErr = sqlite3OsWrite(p->pFile, + &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, + p->iWriteOff + p->iBufStart + ); + } + *piEof = (p->iWriteOff + p->iBufEnd); + sqlite3DbFree(db, p->aBuffer); + rc = p->eFWErr; + memset(p, 0, sizeof(FileWriter)); + return rc; +} + +/* +** Write value iVal encoded as a varint to the file-write object. Return +** SQLITE_OK if successful, or an SQLite error code if an error occurs. +*/ +static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ + int nByte; + u8 aByte[10]; + nByte = sqlite3PutVarint(aByte, iVal); + fileWriterWrite(p, aByte, nByte); +} + +/* +** Write the current contents of the in-memory linked-list to a PMA. Return +** SQLITE_OK if successful, or an SQLite error code otherwise. +** +** The format of a PMA is: +** +** * A varint. This varint contains the total number of bytes of content +** in the PMA (not including the varint itself). +** +** * One or more records packed end-to-end in order of ascending keys. +** Each record consists of a varint followed by a blob of data (the +** key). The varint is the number of bytes in the blob of data. +*/ +static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + VdbeSorter *pSorter = pCsr->pSorter; + FileWriter writer; + + memset(&writer, 0, sizeof(FileWriter)); + + if( pSorter->nInMemory==0 ){ + assert( pSorter->pRecord==0 ); + return rc; + } + + rc = vdbeSorterSort(pCsr); + + /* If the first temporary PMA file has not been opened, open it now. */ + if( rc==SQLITE_OK && pSorter->pTemp1==0 ){ + rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); + assert( rc!=SQLITE_OK || pSorter->pTemp1 ); + assert( pSorter->iWriteOff==0 ); + assert( pSorter->nPMA==0 ); + } + + if( rc==SQLITE_OK ){ + SorterRecord *p; + SorterRecord *pNext = 0; + + fileWriterInit(db, pSorter->pTemp1, &writer, pSorter->iWriteOff); + pSorter->nPMA++; + fileWriterWriteVarint(&writer, pSorter->nInMemory); + for(p=pSorter->pRecord; p; p=pNext){ + pNext = p->pNext; + fileWriterWriteVarint(&writer, p->nVal); + fileWriterWrite(&writer, p->pVal, p->nVal); + sqlite3DbFree(db, p); + } + pSorter->pRecord = p; + rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); + } + + return rc; +} + +/* +** Add a record to the sorter. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterWrite( + sqlite3 *db, /* Database handle */ + const VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal /* Memory cell containing record */ +){ + VdbeSorter *pSorter = pCsr->pSorter; + int rc = SQLITE_OK; /* Return Code */ + SorterRecord *pNew; /* New list element */ + + assert( pSorter ); + pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; + + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->pVal = (void *)&pNew[1]; + memcpy(pNew->pVal, pVal->z, pVal->n); + pNew->nVal = pVal->n; + pNew->pNext = pSorter->pRecord; + pSorter->pRecord = pNew; + } + + /* See if the contents of the sorter should now be written out. They + ** are written out when either of the following are true: + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * cache-size), or + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. + */ + if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( + (pSorter->nInMemory>pSorter->mxPmaSize) + || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + )){ +#ifdef SQLITE_DEBUG + i64 nExpect = pSorter->iWriteOff + + sqlite3VarintLen(pSorter->nInMemory) + + pSorter->nInMemory; +#endif + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + } + + return rc; +} + +/* +** Helper function for sqlite3VdbeSorterRewind(). +*/ +static int vdbeSorterInitMerge( + sqlite3 *db, /* Database handle */ + const VdbeCursor *pCsr, /* Cursor handle for this sorter */ + i64 *pnByte /* Sum of bytes in all opened PMAs */ +){ + VdbeSorter *pSorter = pCsr->pSorter; + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterator through aIter[] */ + i64 nByte = 0; /* Total bytes in all opened PMAs */ + + /* Initialize the iterators. */ + for(i=0; iaIter[i]; + rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); + pSorter->iReadOff = pIter->iEof; + assert( rc!=SQLITE_OK || pSorter->iReadOff<=pSorter->iWriteOff ); + if( rc!=SQLITE_OK || pSorter->iReadOff>=pSorter->iWriteOff ) break; + } + + /* Initialize the aTree[] array. */ + for(i=pSorter->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pCsr, i); + } + + *pnByte = nByte; + return rc; +} + +/* +** Once the sorter has been populated, this function is called to prepare +** for iterating through its contents in sorted order. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ + VdbeSorter *pSorter = pCsr->pSorter; + int rc; /* Return code */ + sqlite3_file *pTemp2 = 0; /* Second temp file to use */ + i64 iWrite2 = 0; /* Write offset for pTemp2 */ + int nIter; /* Number of iterators used */ + int nByte; /* Bytes of space required for aIter/aTree */ + int N = 2; /* Power of 2 >= nIter */ + + assert( pSorter ); + + /* If no data has been written to disk, then do not do so now. Instead, + ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly + ** from the in-memory list. */ + if( pSorter->nPMA==0 ){ + *pbEof = !pSorter->pRecord; + assert( pSorter->aTree==0 ); + return vdbeSorterSort(pCsr); + } + + /* Write the current in-memory list to a PMA. */ + rc = vdbeSorterListToPMA(db, pCsr); + if( rc!=SQLITE_OK ) return rc; + + /* Allocate space for aIter[] and aTree[]. */ + nIter = pSorter->nPMA; + if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT; + assert( nIter>0 ); + while( NaIter = (VdbeSorterIter *)sqlite3DbMallocZero(db, nByte); + if( !pSorter->aIter ) return SQLITE_NOMEM; + pSorter->aTree = (int *)&pSorter->aIter[N]; + pSorter->nTree = N; + + do { + int iNew; /* Index of new, merged, PMA */ + + for(iNew=0; + rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNTnPMA; + iNew++ + ){ + int rc2; /* Return code from fileWriterFinish() */ + FileWriter writer; /* Object used to write to disk */ + i64 nWrite; /* Number of bytes in new PMA */ + + memset(&writer, 0, sizeof(FileWriter)); + + /* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1, + ** initialize an iterator for each of them and break out of the loop. + ** These iterators will be incrementally merged as the VDBE layer calls + ** sqlite3VdbeSorterNext(). + ** + ** Otherwise, if pTemp1 contains more than SORTER_MAX_MERGE_COUNT PMAs, + ** initialize interators for SORTER_MAX_MERGE_COUNT of them. These PMAs + ** are merged into a single PMA that is written to file pTemp2. + */ + rc = vdbeSorterInitMerge(db, pCsr, &nWrite); + assert( rc!=SQLITE_OK || pSorter->aIter[ pSorter->aTree[1] ].pFile ); + if( rc!=SQLITE_OK || pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ + break; + } + + /* Open the second temp file, if it is not already open. */ + if( pTemp2==0 ){ + assert( iWrite2==0 ); + rc = vdbeSorterOpenTempFile(db, &pTemp2); + } + + if( rc==SQLITE_OK ){ + int bEof = 0; + fileWriterInit(db, pTemp2, &writer, iWrite2); + fileWriterWriteVarint(&writer, nWrite); + while( rc==SQLITE_OK && bEof==0 ){ + VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + assert( pIter->pFile ); + + fileWriterWriteVarint(&writer, pIter->nKey); + fileWriterWrite(&writer, pIter->aKey, pIter->nKey); + rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); + } + rc2 = fileWriterFinish(db, &writer, &iWrite2); + if( rc==SQLITE_OK ) rc = rc2; + } + } + + if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ + break; + }else{ + sqlite3_file *pTmp = pSorter->pTemp1; + pSorter->nPMA = iNew; + pSorter->pTemp1 = pTemp2; + pTemp2 = pTmp; + pSorter->iWriteOff = iWrite2; + pSorter->iReadOff = 0; + iWrite2 = 0; + } + }while( rc==SQLITE_OK ); + + if( pTemp2 ){ + sqlite3OsCloseFree(pTemp2); + } + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + return rc; +} + +/* +** Advance to the next element in the sorter. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ + VdbeSorter *pSorter = pCsr->pSorter; + int rc; /* Return code */ + + if( pSorter->aTree ){ + int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ + rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); + if( rc==SQLITE_OK ){ + int i; /* Index of aTree[] to recalculate */ + VdbeSorterIter *pIter1; /* First iterator to compare */ + VdbeSorterIter *pIter2; /* Second iterator to compare */ + u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ + + /* Find the first two iterators to compare. The one that was just + ** advanced (iPrev) and the one next to it in the array. */ + pIter1 = &pSorter->aIter[(iPrev & 0xFFFE)]; + pIter2 = &pSorter->aIter[(iPrev | 0x0001)]; + pKey2 = pIter2->aKey; + + for(i=(pSorter->nTree+iPrev)/2; i>0; i=i/2){ + /* Compare pIter1 and pIter2. Store the result in variable iRes. */ + int iRes; + if( pIter1->pFile==0 ){ + iRes = +1; + }else if( pIter2->pFile==0 ){ + iRes = -1; + }else{ + vdbeSorterCompare(pCsr, 0, + pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes + ); + } + + /* If pIter1 contained the smaller value, set aTree[i] to its index. + ** Then set pIter2 to the next iterator to compare to pIter1. In this + ** case there is no cache of pIter2 in pSorter->pUnpacked, so set + ** pKey2 to point to the record belonging to pIter2. + ** + ** Alternatively, if pIter2 contains the smaller of the two values, + ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() + ** was actually called above, then pSorter->pUnpacked now contains + ** a value equivalent to pIter2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pIter2 again. */ + if( iRes<=0 ){ + pSorter->aTree[i] = (int)(pIter1 - pSorter->aIter); + pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; + pKey2 = pIter2->aKey; + }else{ + if( pIter1->pFile ) pKey2 = 0; + pSorter->aTree[i] = (int)(pIter2 - pSorter->aIter); + pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; + } + + } + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + } + }else{ + SorterRecord *pFree = pSorter->pRecord; + pSorter->pRecord = pFree->pNext; + pFree->pNext = 0; + vdbeSorterRecordFree(db, pFree); + *pbEof = !pSorter->pRecord; + rc = SQLITE_OK; + } + return rc; +} + +/* +** Return a pointer to a buffer owned by the sorter that contains the +** current key. +*/ +static void *vdbeSorterRowkey( + const VdbeSorter *pSorter, /* Sorter object */ + int *pnKey /* OUT: Size of current key in bytes */ +){ + void *pKey; + if( pSorter->aTree ){ + VdbeSorterIter *pIter; + pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + *pnKey = pIter->nKey; + pKey = pIter->aKey; + }else{ + *pnKey = pSorter->pRecord->nVal; + pKey = pSorter->pRecord->pVal; + } + return pKey; +} + +/* +** Copy the current sorter key into the memory cell pOut. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ + VdbeSorter *pSorter = pCsr->pSorter; + void *pKey; int nKey; /* Sorter key to copy into pOut */ + + pKey = vdbeSorterRowkey(pSorter, &nKey); + if( sqlite3VdbeMemGrow(pOut, nKey, 0) ){ + return SQLITE_NOMEM; + } + pOut->n = nKey; + MemSetTypeFlag(pOut, MEM_Blob); + memcpy(pOut->z, pKey, nKey); + + return SQLITE_OK; +} + +/* +** Compare the key in memory cell pVal with the key that the sorter cursor +** passed as the first argument currently points to. For the purposes of +** the comparison, ignore the rowid field at the end of each record. +** +** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). +** Otherwise, set *pRes to a negative, zero or positive value if the +** key in pVal is smaller than, equal to or larger than the current sorter +** key. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterCompare( + const VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal, /* Value to compare to current sorter key */ + int nKeyCol, /* Only compare this many fields */ + int *pRes /* OUT: Result of comparison */ +){ + VdbeSorter *pSorter = pCsr->pSorter; + void *pKey; int nKey; /* Sorter key to compare pVal with */ + + pKey = vdbeSorterRowkey(pSorter, &nKey); + vdbeSorterCompare(pCsr, nKeyCol, pVal->z, pVal->n, pKey, nKey, pRes); + return SQLITE_OK; +} + +/************** End of vdbesort.c ********************************************/ +/************** Begin file journal.c *****************************************/ +/* +** 2007 August 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a special kind of sqlite3_file object used +** by SQLite to create journal files if the atomic-write optimization +** is enabled. +** +** The distinctive characteristic of this sqlite3_file is that the +** actual on disk file is created lazily. When the file is created, +** the caller specifies a buffer size for an in-memory buffer to +** be used to service read() and write() requests. The actual file +** on disk is not created or populated until either: +** +** 1) The in-memory representation grows too large for the allocated +** buffer, or +** 2) The sqlite3JournalCreate() function is called. +*/ +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + + +/* +** A JournalFile object is a subclass of sqlite3_file used by +** as an open file handle for journal files. +*/ +struct JournalFile { + sqlite3_io_methods *pMethod; /* I/O methods on journal files */ + int nBuf; /* Size of zBuf[] in bytes */ + char *zBuf; /* Space to buffer journal writes */ + int iSize; /* Amount of zBuf[] currently used */ + int flags; /* xOpen flags */ + sqlite3_vfs *pVfs; /* The "real" underlying VFS */ + sqlite3_file *pReal; /* The "real" underlying file descriptor */ + const char *zJournal; /* Name of the journal file */ +}; +typedef struct JournalFile JournalFile; + +/* +** If it does not already exists, create and populate the on-disk file +** for JournalFile p. +*/ +static int createFile(JournalFile *p){ + int rc = SQLITE_OK; + if( !p->pReal ){ + sqlite3_file *pReal = (sqlite3_file *)&p[1]; + rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); + if( rc==SQLITE_OK ){ + p->pReal = pReal; + if( p->iSize>0 ){ + assert(p->iSize<=p->nBuf); + rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); + } + if( rc!=SQLITE_OK ){ + /* If an error occurred while writing to the file, close it before + ** returning. This way, SQLite uses the in-memory journal data to + ** roll back changes made to the internal page-cache before this + ** function was called. */ + sqlite3OsClose(pReal); + p->pReal = 0; + } + } + } + return rc; +} + +/* +** Close the file. +*/ +static int jrnlClose(sqlite3_file *pJfd){ + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + sqlite3OsClose(p->pReal); + } + sqlite3_free(p->zBuf); + return SQLITE_OK; +} + +/* +** Read data from the file. +*/ +static int jrnlRead( + sqlite3_file *pJfd, /* The journal file from which to read */ + void *zBuf, /* Put the results here */ + int iAmt, /* Number of bytes to read */ + sqlite_int64 iOfst /* Begin reading at this offset */ +){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); + }else if( (iAmt+iOfst)>p->iSize ){ + rc = SQLITE_IOERR_SHORT_READ; + }else{ + memcpy(zBuf, &p->zBuf[iOfst], iAmt); + } + return rc; +} + +/* +** Write data to the file. +*/ +static int jrnlWrite( + sqlite3_file *pJfd, /* The journal file into which to write */ + const void *zBuf, /* Take data to be written from here */ + int iAmt, /* Number of bytes to write */ + sqlite_int64 iOfst /* Begin writing at this offset into the file */ +){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( !p->pReal && (iOfst+iAmt)>p->nBuf ){ + rc = createFile(p); + } + if( rc==SQLITE_OK ){ + if( p->pReal ){ + rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); + }else{ + memcpy(&p->zBuf[iOfst], zBuf, iAmt); + if( p->iSize<(iOfst+iAmt) ){ + p->iSize = (iOfst+iAmt); + } + } + } + return rc; +} + +/* +** Truncate the file. +*/ +static int jrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsTruncate(p->pReal, size); + }else if( sizeiSize ){ + p->iSize = size; + } + return rc; +} + +/* +** Sync the file. +*/ +static int jrnlSync(sqlite3_file *pJfd, int flags){ + int rc; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsSync(p->pReal, flags); + }else{ + rc = SQLITE_OK; + } + return rc; +} + +/* +** Query the size of the file in bytes. +*/ +static int jrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsFileSize(p->pReal, pSize); + }else{ + *pSize = (sqlite_int64) p->iSize; + } + return rc; +} + +/* +** Table of methods for JournalFile sqlite3_file object. +*/ +static struct sqlite3_io_methods JournalFileMethods = { + 1, /* iVersion */ + jrnlClose, /* xClose */ + jrnlRead, /* xRead */ + jrnlWrite, /* xWrite */ + jrnlTruncate, /* xTruncate */ + jrnlSync, /* xSync */ + jrnlFileSize, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0 /* xShmUnmap */ +}; + +/* +** Open a journal file. +*/ +SQLITE_PRIVATE int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nBuf /* Bytes buffered before opening the file */ +){ + JournalFile *p = (JournalFile *)pJfd; + memset(p, 0, sqlite3JournalSize(pVfs)); + if( nBuf>0 ){ + p->zBuf = sqlite3MallocZero(nBuf); + if( !p->zBuf ){ + return SQLITE_NOMEM; + } + }else{ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + } + p->pMethod = &JournalFileMethods; + p->nBuf = nBuf; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; +} + +/* +** If the argument p points to a JournalFile structure, and the underlying +** file has not yet been created, create it now. +*/ +SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ + if( p->pMethods!=&JournalFileMethods ){ + return SQLITE_OK; + } + return createFile((JournalFile *)p); +} + +/* +** The file-handle passed as the only argument is guaranteed to be an open +** file. It may or may not be of class JournalFile. If the file is a +** JournalFile, and the underlying file on disk has not yet been opened, +** return 0. Otherwise, return 1. +*/ +SQLITE_PRIVATE int sqlite3JournalExists(sqlite3_file *p){ + return (p->pMethods!=&JournalFileMethods || ((JournalFile *)p)->pReal!=0); +} + +/* +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. +*/ +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return (pVfs->szOsFile+sizeof(JournalFile)); +} +#endif + +/************** End of journal.c *********************************************/ +/************** Begin file memjournal.c **************************************/ +/* +** 2008 October 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code use to implement an in-memory rollback journal. +** The in-memory rollback journal is used to journal transactions for +** ":memory:" databases and when the journal_mode=MEMORY pragma is used. +*/ + +/* Forward references to internal structures */ +typedef struct MemJournal MemJournal; +typedef struct FilePoint FilePoint; +typedef struct FileChunk FileChunk; + +/* Space to hold the rollback journal is allocated in increments of +** this many bytes. +** +** The size chosen is a little less than a power of two. That way, +** the FileChunk object will have a size that almost exactly fills +** a power-of-two allocation. This mimimizes wasted space in power-of-two +** memory allocators. +*/ +#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) + +/* +** The rollback journal is composed of a linked list of these structures. +*/ +struct FileChunk { + FileChunk *pNext; /* Next chunk in the journal */ + u8 zChunk[JOURNAL_CHUNKSIZE]; /* Content of this chunk */ +}; + +/* +** An instance of this object serves as a cursor into the rollback journal. +** The cursor can be either for reading or writing. +*/ +struct FilePoint { + sqlite3_int64 iOffset; /* Offset from the beginning of the file */ + FileChunk *pChunk; /* Specific chunk into which cursor points */ +}; + +/* +** This subclass is a subclass of sqlite3_file. Each open memory-journal +** is an instance of this class. +*/ +struct MemJournal { + sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + FileChunk *pFirst; /* Head of in-memory chunk-list */ + FilePoint endpoint; /* Pointer to the end of the file */ + FilePoint readpoint; /* Pointer to the end of the last xRead() */ +}; + +/* +** Read data from the in-memory journal file. This is the implementation +** of the sqlite3_vfs.xRead method. +*/ +static int memjrnlRead( + sqlite3_file *pJfd, /* The journal file from which to read */ + void *zBuf, /* Put the results here */ + int iAmt, /* Number of bytes to read */ + sqlite_int64 iOfst /* Begin reading at this offset */ +){ + MemJournal *p = (MemJournal *)pJfd; + u8 *zOut = zBuf; + int nRead = iAmt; + int iChunkOffset; + FileChunk *pChunk; + + /* SQLite never tries to read past the end of a rollback journal file */ + assert( iOfst+iAmt<=p->endpoint.iOffset ); + + if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ + sqlite3_int64 iOff = 0; + for(pChunk=p->pFirst; + ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst; + pChunk=pChunk->pNext + ){ + iOff += JOURNAL_CHUNKSIZE; + } + }else{ + pChunk = p->readpoint.pChunk; + } + + iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE); + do { + int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset; + int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset)); + memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); + zOut += nCopy; + nRead -= iSpace; + iChunkOffset = 0; + } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); + p->readpoint.iOffset = iOfst+iAmt; + p->readpoint.pChunk = pChunk; + + return SQLITE_OK; +} + +/* +** Write data to the file. +*/ +static int memjrnlWrite( + sqlite3_file *pJfd, /* The journal file into which to write */ + const void *zBuf, /* Take data to be written from here */ + int iAmt, /* Number of bytes to write */ + sqlite_int64 iOfst /* Begin writing at this offset into the file */ +){ + MemJournal *p = (MemJournal *)pJfd; + int nWrite = iAmt; + u8 *zWrite = (u8 *)zBuf; + + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required by sqlite. + */ + assert( iOfst==p->endpoint.iOffset ); + UNUSED_PARAMETER(iOfst); + + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE); + int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset); + + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM; + } + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; + } + + memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; + } + + return SQLITE_OK; +} + +/* +** Truncate the file. +*/ +static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ + MemJournal *p = (MemJournal *)pJfd; + FileChunk *pChunk; + assert(size==0); + UNUSED_PARAMETER(size); + pChunk = p->pFirst; + while( pChunk ){ + FileChunk *pTmp = pChunk; + pChunk = pChunk->pNext; + sqlite3_free(pTmp); + } + sqlite3MemJournalOpen(pJfd); + return SQLITE_OK; +} + +/* +** Close the file. +*/ +static int memjrnlClose(sqlite3_file *pJfd){ + memjrnlTruncate(pJfd, 0); + return SQLITE_OK; +} + + +/* +** Sync the file. +** +** Syncing an in-memory journal is a no-op. And, in fact, this routine +** is never called in a working implementation. This implementation +** exists purely as a contingency, in case some malfunction in some other +** part of SQLite causes Sync to be called by mistake. +*/ +static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} + +/* +** Query the size of the file in bytes. +*/ +static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ + MemJournal *p = (MemJournal *)pJfd; + *pSize = (sqlite_int64) p->endpoint.iOffset; + return SQLITE_OK; +} + +/* +** Table of methods for MemJournal sqlite3_file object. +*/ +static const struct sqlite3_io_methods MemJournalMethods = { + 1, /* iVersion */ + memjrnlClose, /* xClose */ + memjrnlRead, /* xRead */ + memjrnlWrite, /* xWrite */ + memjrnlTruncate, /* xTruncate */ + memjrnlSync, /* xSync */ + memjrnlFileSize, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/* +** Open a journal file. +*/ +SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *pJfd){ + MemJournal *p = (MemJournal *)pJfd; + assert( EIGHT_BYTE_ALIGNMENT(p) ); + memset(p, 0, sqlite3MemJournalSize()); + p->pMethod = (sqlite3_io_methods*)&MemJournalMethods; +} + +/* +** Return true if the file-handle passed as an argument is +** an in-memory journal +*/ +SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *pJfd){ + return pJfd->pMethods==&MemJournalMethods; +} + +/* +** Return the number of bytes required to store a MemJournal file descriptor. +*/ +SQLITE_PRIVATE int sqlite3MemJournalSize(void){ + return sizeof(MemJournal); +} + +/************** End of memjournal.c ******************************************/ +/************** Begin file walker.c ******************************************/ +/* +** 2008 August 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains routines used for walking the parser tree for +** an SQL statement. +*/ +/* #include */ +/* #include */ + + +/* +** Walk an expression tree. Invoke the callback once for each node +** of the expression, while decending. (In other words, the callback +** is invoked before visiting children.) +** +** The return value from the callback should be one of the WRC_* +** constants to specify how to proceed with the walk. +** +** WRC_Continue Continue descending down the tree. +** +** WRC_Prune Do not descend into child nodes. But allow +** the walk to continue with sibling nodes. +** +** WRC_Abort Do no more callbacks. Unwind the stack and +** return the top-level walk call. +** +** The return value from this routine is WRC_Abort to abandon the tree walk +** and WRC_Continue to continue. +*/ +SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ + int rc; + if( pExpr==0 ) return WRC_Continue; + testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); + testcase( ExprHasProperty(pExpr, EP_Reduced) ); + rc = pWalker->xExprCallback(pWalker, pExpr); + if( rc==WRC_Continue + && !ExprHasProperty(pExpr,EP_TokenOnly) ){ + if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; + }else{ + if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + } + } + return rc & WRC_Abort; +} + +/* +** Call sqlite3WalkExpr() for every expression in list p or until +** an abort request is seen. +*/ +SQLITE_PRIVATE int sqlite3WalkExprList(Walker *pWalker, ExprList *p){ + int i; + struct ExprList_item *pItem; + if( p ){ + for(i=p->nExpr, pItem=p->a; i>0; i--, pItem++){ + if( sqlite3WalkExpr(pWalker, pItem->pExpr) ) return WRC_Abort; + } + } + return WRC_Continue; +} + +/* +** Walk all expressions associated with SELECT statement p. Do +** not invoke the SELECT callback on p, but do (of course) invoke +** any expr callbacks and SELECT callbacks that come from subqueries. +** Return WRC_Abort or WRC_Continue. +*/ +SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){ + if( sqlite3WalkExprList(pWalker, p->pEList) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pWhere) ) return WRC_Abort; + if( sqlite3WalkExprList(pWalker, p->pGroupBy) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pHaving) ) return WRC_Abort; + if( sqlite3WalkExprList(pWalker, p->pOrderBy) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pLimit) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pOffset) ) return WRC_Abort; + return WRC_Continue; +} + +/* +** Walk the parse trees associated with all subqueries in the +** FROM clause of SELECT statement p. Do not invoke the select +** callback on p, but do invoke it on each FROM clause subquery +** and on any subqueries further down in the tree. Return +** WRC_Abort or WRC_Continue; +*/ +SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ + SrcList *pSrc; + int i; + struct SrcList_item *pItem; + + pSrc = p->pSrc; + if( ALWAYS(pSrc) ){ + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ + return WRC_Abort; + } + } + } + return WRC_Continue; +} + +/* +** Call sqlite3WalkExpr() for every expression in Select statement p. +** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and +** on the compound select chain, p->pPrior. +** +** If it is not NULL, the xSelectCallback() callback is invoked before +** the walk of the expressions and FROM clause. The xSelectCallback2() +** method, if it is not NULL, is invoked following the walk of the +** expressions and FROM clause. +** +** Return WRC_Continue under normal conditions. Return WRC_Abort if +** there is an abort request. +** +** If the Walker does not have an xSelectCallback() then this routine +** is a no-op returning WRC_Continue. +*/ +SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ + int rc; + if( p==0 || (pWalker->xSelectCallback==0 && pWalker->xSelectCallback2==0) ){ + return WRC_Continue; + } + rc = WRC_Continue; + pWalker->walkerDepth++; + while( p ){ + if( pWalker->xSelectCallback ){ + rc = pWalker->xSelectCallback(pWalker, p); + if( rc ) break; + } + if( sqlite3WalkSelectExpr(pWalker, p) + || sqlite3WalkSelectFrom(pWalker, p) + ){ + pWalker->walkerDepth--; + return WRC_Abort; + } + if( pWalker->xSelectCallback2 ){ + pWalker->xSelectCallback2(pWalker, p); + } + p = p->pPrior; + } + pWalker->walkerDepth--; + return rc & WRC_Abort; +} + +/************** End of walker.c **********************************************/ +/************** Begin file resolve.c *****************************************/ +/* +** 2008 August 18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains routines used for walking the parser tree and +** resolve all identifiers by associating them with a particular +** table and column. +*/ +/* #include */ +/* #include */ + +/* +** Walk the expression tree pExpr and increase the aggregate function +** depth (the Expr.op2 field) by N on every TK_AGG_FUNCTION node. +** This needs to occur when copying a TK_AGG_FUNCTION node from an +** outer query into an inner subquery. +** +** incrAggFunctionDepth(pExpr,n) is the main routine. incrAggDepth(..) +** is a helper function - a callback for the tree walker. +*/ +static int incrAggDepth(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.i; + return WRC_Continue; +} +static void incrAggFunctionDepth(Expr *pExpr, int N){ + if( N>0 ){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = incrAggDepth; + w.u.i = N; + sqlite3WalkExpr(&w, pExpr); + } +} + +/* +** Turn the pExpr expression into an alias for the iCol-th column of the +** result set in pEList. +** +** If the result set column is a simple column reference, then this routine +** makes an exact copy. But for any other kind of expression, this +** routine make a copy of the result set column as the argument to the +** TK_AS operator. The TK_AS operator causes the expression to be +** evaluated just once and then reused for each alias. +** +** The reason for suppressing the TK_AS term when the expression is a simple +** column reference is so that the column reference will be recognized as +** usable by indices within the WHERE clause processing logic. +** +** The TK_AS operator is inhibited if zType[0]=='G'. This means +** that in a GROUP BY clause, the expression is evaluated twice. Hence: +** +** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x +** +** Is equivalent to: +** +** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 +** +** The result of random()%5 in the GROUP BY clause is probably different +** from the result in the result-set. On the other hand Standard SQL does +** not allow the GROUP BY clause to contain references to result-set columns. +** So this should never come up in well-formed queries. +** +** If the reference is followed by a COLLATE operator, then make sure +** the COLLATE operator is preserved. For example: +** +** SELECT a+b, c+d FROM t1 ORDER BY 1 COLLATE nocase; +** +** Should be transformed into: +** +** SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase; +** +** The nSubquery parameter specifies how many levels of subquery the +** alias is removed from the original expression. The usually value is +** zero but it might be more if the alias is contained within a subquery +** of the original expression. The Expr.op2 field of TK_AGG_FUNCTION +** structures must be increased by the nSubquery amount. +*/ +static void resolveAlias( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* A result set */ + int iCol, /* A column in the result set. 0..pEList->nExpr-1 */ + Expr *pExpr, /* Transform this into an alias to the result set */ + const char *zType, /* "GROUP" or "ORDER" or "" */ + int nSubquery /* Number of subqueries that the label is moving */ +){ + Expr *pOrig; /* The iCol-th column of the result set */ + Expr *pDup; /* Copy of pOrig */ + sqlite3 *db; /* The database connection */ + + assert( iCol>=0 && iColnExpr ); + pOrig = pEList->a[iCol].pExpr; + assert( pOrig!=0 ); + assert( pOrig->flags & EP_Resolved ); + db = pParse->db; + pDup = sqlite3ExprDup(db, pOrig, 0); + if( pDup==0 ) return; + if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ + incrAggFunctionDepth(pDup, nSubquery); + pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); + if( pDup==0 ) return; + ExprSetProperty(pDup, EP_Skip); + if( pEList->a[iCol].u.x.iAlias==0 ){ + pEList->a[iCol].u.x.iAlias = (u16)(++pParse->nAlias); + } + pDup->iTable = pEList->a[iCol].u.x.iAlias; + } + if( pExpr->op==TK_COLLATE ){ + pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); + } + + /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This + ** prevents ExprDelete() from deleting the Expr structure itself, + ** allowing it to be repopulated by the memcpy() on the following line. + ** The pExpr->u.zToken might point into memory that will be freed by the + ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to + ** make a copy of the token before doing the sqlite3DbFree(). + */ + ExprSetProperty(pExpr, EP_Static); + sqlite3ExprDelete(db, pExpr); + memcpy(pExpr, pDup, sizeof(*pExpr)); + if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){ + assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 ); + pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); + pExpr->flags |= EP_MemToken; + } + sqlite3DbFree(db, pDup); +} + + +/* +** Return TRUE if the name zCol occurs anywhere in the USING clause. +** +** Return FALSE if the USING clause is NULL or if it does not contain +** zCol. +*/ +static int nameInUsingClause(IdList *pUsing, const char *zCol){ + if( pUsing ){ + int k; + for(k=0; knId; k++){ + if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ) return 1; + } + } + return 0; +} + +/* +** Subqueries stores the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". +** Check to see if the zSpan given to this routine matches the zDb, zTab, +** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will +** match anything. +*/ +SQLITE_PRIVATE int sqlite3MatchSpanName( + const char *zSpan, + const char *zCol, + const char *zTab, + const char *zDb +){ + int n; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ + return 0; + } + zSpan += n+1; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zTab && (sqlite3StrNICmp(zSpan, zTab, n)!=0 || zTab[n]!=0) ){ + return 0; + } + zSpan += n+1; + if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ + return 0; + } + return 1; +} + +/* +** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up +** that name in the set of source tables in pSrcList and make the pExpr +** expression node refer back to that source column. The following changes +** are made to pExpr: +** +** pExpr->iDb Set the index in db->aDb[] of the database X +** (even if X is implied). +** pExpr->iTable Set to the cursor number for the table obtained +** from pSrcList. +** pExpr->pTab Points to the Table structure of X.Y (even if +** X and/or Y are implied.) +** pExpr->iColumn Set to the column number within the table. +** pExpr->op Set to TK_COLUMN. +** pExpr->pLeft Any expression this points to is deleted +** pExpr->pRight Any expression this points to is deleted. +** +** The zDb variable is the name of the database (the "X"). This value may be +** NULL meaning that name is of the form Y.Z or Z. Any available database +** can be used. The zTable variable is the name of the table (the "Y"). This +** value can be NULL if zDb is also NULL. If zTable is NULL it +** means that the form of the name is Z and that columns from any table +** can be used. +** +** If the name cannot be resolved unambiguously, leave an error message +** in pParse and return WRC_Abort. Return WRC_Prune on success. +*/ +static int lookupName( + Parse *pParse, /* The parsing context */ + const char *zDb, /* Name of the database containing table, or NULL */ + const char *zTab, /* Name of table containing column, or NULL */ + const char *zCol, /* Name of the column. */ + NameContext *pNC, /* The name context used to resolve the name */ + Expr *pExpr /* Make this EXPR node point to the selected column */ +){ + int i, j; /* Loop counters */ + int cnt = 0; /* Number of matching column names */ + int cntTab = 0; /* Number of matching table names */ + int nSubquery = 0; /* How many levels of subquery */ + sqlite3 *db = pParse->db; /* The database connection */ + struct SrcList_item *pItem; /* Use for looping over pSrcList items */ + struct SrcList_item *pMatch = 0; /* The matching pSrcList item */ + NameContext *pTopNC = pNC; /* First namecontext in the list */ + Schema *pSchema = 0; /* Schema of the expression */ + int isTrigger = 0; /* True if resolved to a trigger column */ + Table *pTab = 0; /* Table hold the row */ + Column *pCol; /* A column of pTab */ + + assert( pNC ); /* the name context cannot be NULL. */ + assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + + /* Initialize the node to no-match */ + pExpr->iTable = -1; + pExpr->pTab = 0; + ExprSetVVAProperty(pExpr, EP_NoReduce); + + /* Translate the schema name in zDb into a pointer to the corresponding + ** schema. If not found, pSchema will remain NULL and nothing will match + ** resulting in an appropriate error message toward the end of this routine + */ + if( zDb ){ + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IsCheck ); + if( (pNC->ncFlags & (NC_PartIdx|NC_IsCheck))!=0 ){ + /* Silently ignore database qualifiers inside CHECK constraints and partial + ** indices. Do not raise errors because that might break legacy and + ** because it does not hurt anything to just ignore the database name. */ + zDb = 0; + }else{ + for(i=0; inDb; i++){ + assert( db->aDb[i].zName ); + if( sqlite3StrICmp(db->aDb[i].zName,zDb)==0 ){ + pSchema = db->aDb[i].pSchema; + break; + } + } + } + } + + /* Start at the inner-most context and move outward until a match is found */ + while( pNC && cnt==0 ){ + ExprList *pEList; + SrcList *pSrcList = pNC->pSrcList; + + if( pSrcList ){ + for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ + pTab = pItem->pTab; + assert( pTab!=0 && pTab->zName!=0 ); + assert( pTab->nCol>0 ); + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_NestedFrom)!=0 ){ + int hit = 0; + pEList = pItem->pSelect->pEList; + for(j=0; jnExpr; j++){ + if( sqlite3MatchSpanName(pEList->a[j].zSpan, zCol, zTab, zDb) ){ + cnt++; + cntTab = 2; + pMatch = pItem; + pExpr->iColumn = j; + hit = 1; + } + } + if( hit || zTab==0 ) continue; + } + if( zDb && pTab->pSchema!=pSchema ){ + continue; + } + if( zTab ){ + const char *zTabName = pItem->zAlias ? pItem->zAlias : pTab->zName; + assert( zTabName!=0 ); + if( sqlite3StrICmp(zTabName, zTab)!=0 ){ + continue; + } + } + if( 0==(cntTab++) ){ + pMatch = pItem; + } + for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ + if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + /* If there has been exactly one prior match and this match + ** is for the right-hand table of a NATURAL JOIN or is in a + ** USING clause, then skip this match. + */ + if( cnt==1 ){ + if( pItem->jointype & JT_NATURAL ) continue; + if( nameInUsingClause(pItem->pUsing, zCol) ) continue; + } + cnt++; + pMatch = pItem; + /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ + pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; + break; + } + } + } + if( pMatch ){ + pExpr->iTable = pMatch->iCursor; + pExpr->pTab = pMatch->pTab; + pSchema = pExpr->pTab->pSchema; + } + } /* if( pSrcList ) */ + +#ifndef SQLITE_OMIT_TRIGGER + /* If we have not already resolved the name, then maybe + ** it is a new.* or old.* trigger argument reference + */ + if( zDb==0 && zTab!=0 && cntTab==0 && pParse->pTriggerTab!=0 ){ + int op = pParse->eTriggerOp; + assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); + if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){ + pExpr->iTable = 1; + pTab = pParse->pTriggerTab; + }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){ + pExpr->iTable = 0; + pTab = pParse->pTriggerTab; + }else{ + pTab = 0; + } + + if( pTab ){ + int iCol; + pSchema = pTab->pSchema; + cntTab++; + for(iCol=0, pCol=pTab->aCol; iColnCol; iCol++, pCol++){ + if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + if( iCol==pTab->iPKey ){ + iCol = -1; + } + break; + } + } + if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && HasRowid(pTab) ){ + /* IMP: R-51414-32910 */ + /* IMP: R-44911-55124 */ + iCol = -1; + } + if( iColnCol ){ + cnt++; + if( iCol<0 ){ + pExpr->affinity = SQLITE_AFF_INTEGER; + }else if( pExpr->iTable==0 ){ + testcase( iCol==31 ); + testcase( iCol==32 ); + pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<iColumn = (i16)iCol; + pExpr->pTab = pTab; + isTrigger = 1; + } + } + } +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ + + /* + ** Perhaps the name is a reference to the ROWID + */ + if( cnt==0 && cntTab==1 && pMatch && sqlite3IsRowid(zCol) + && HasRowid(pMatch->pTab) ){ + cnt = 1; + pExpr->iColumn = -1; /* IMP: R-44911-55124 */ + pExpr->affinity = SQLITE_AFF_INTEGER; + } + + /* + ** If the input is of the form Z (not Y.Z or X.Y.Z) then the name Z + ** might refer to an result-set alias. This happens, for example, when + ** we are resolving names in the WHERE clause of the following command: + ** + ** SELECT a+b AS x FROM table WHERE x<10; + ** + ** In cases like this, replace pExpr with a copy of the expression that + ** forms the result set entry ("a+b" in the example) and return immediately. + ** Note that the expression in the result set should have already been + ** resolved by the time the WHERE clause is resolved. + ** + ** The ability to use an output result-set column in the WHERE, GROUP BY, + ** or HAVING clauses, or as part of a larger expression in the ORDRE BY + ** clause is not standard SQL. This is a (goofy) SQLite extension, that + ** is supported for backwards compatibility only. TO DO: Issue a warning + ** on sqlite3_log() whenever the capability is used. + */ + if( (pEList = pNC->pEList)!=0 + && zTab==0 + && cnt==0 + ){ + for(j=0; jnExpr; j++){ + char *zAs = pEList->a[j].zName; + if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + Expr *pOrig; + assert( pExpr->pLeft==0 && pExpr->pRight==0 ); + assert( pExpr->x.pList==0 ); + assert( pExpr->x.pSelect==0 ); + pOrig = pEList->a[j].pExpr; + if( (pNC->ncFlags&NC_AllowAgg)==0 && ExprHasProperty(pOrig, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); + return WRC_Abort; + } + resolveAlias(pParse, pEList, j, pExpr, "", nSubquery); + cnt = 1; + pMatch = 0; + assert( zTab==0 && zDb==0 ); + goto lookupname_end; + } + } + } + + /* Advance to the next name context. The loop will exit when either + ** we have a match (cnt>0) or when we run out of name contexts. + */ + if( cnt==0 ){ + pNC = pNC->pNext; + nSubquery++; + } + } + + /* + ** If X and Y are NULL (in other words if only the column name Z is + ** supplied) and the value of Z is enclosed in double-quotes, then + ** Z is a string literal if it doesn't match any column names. In that + ** case, we need to return right away and not make any changes to + ** pExpr. + ** + ** Because no reference was made to outer contexts, the pNC->nRef + ** fields are not changed in any context. + */ + if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){ + pExpr->op = TK_STRING; + pExpr->pTab = 0; + return WRC_Prune; + } + + /* + ** cnt==0 means there was not match. cnt>1 means there were two or + ** more matches. Either way, we have an error. + */ + if( cnt!=1 ){ + const char *zErr; + zErr = cnt==0 ? "no such column" : "ambiguous column name"; + if( zDb ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); + }else if( zTab ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); + } + pParse->checkSchema = 1; + pTopNC->nErr++; + } + + /* If a column from a table in pSrcList is referenced, then record + ** this fact in the pSrcList.a[].colUsed bitmask. Column 0 causes + ** bit 0 to be set. Column 1 sets bit 1. And so forth. If the + ** column number is greater than the number of bits in the bitmask + ** then set the high-order bit of the bitmask. + */ + if( pExpr->iColumn>=0 && pMatch!=0 ){ + int n = pExpr->iColumn; + testcase( n==BMS-1 ); + if( n>=BMS ){ + n = BMS-1; + } + assert( pMatch->iCursor==pExpr->iTable ); + pMatch->colUsed |= ((Bitmask)1)<pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + pExpr->op = (isTrigger ? TK_TRIGGER : TK_COLUMN); +lookupname_end: + if( cnt==1 ){ + assert( pNC!=0 ); + if( pExpr->op!=TK_AS ){ + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + } + /* Increment the nRef value on all name contexts from TopNC up to + ** the point where the name matched. */ + for(;;){ + assert( pTopNC!=0 ); + pTopNC->nRef++; + if( pTopNC==pNC ) break; + pTopNC = pTopNC->pNext; + } + return WRC_Prune; + } else { + return WRC_Abort; + } +} + +/* +** Allocate and return a pointer to an expression to load the column iCol +** from datasource iSrc in SrcList pSrc. +*/ +SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ + Expr *p = sqlite3ExprAlloc(db, TK_COLUMN, 0, 0); + if( p ){ + struct SrcList_item *pItem = &pSrc->a[iSrc]; + p->pTab = pItem->pTab; + p->iTable = pItem->iCursor; + if( p->pTab->iPKey==iCol ){ + p->iColumn = -1; + }else{ + p->iColumn = (ynVar)iCol; + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + ExprSetProperty(p, EP_Resolved); + } + return p; +} + +/* +** Report an error that an expression is not valid for a partial index WHERE +** clause. +*/ +static void notValidPartIdxWhere( + Parse *pParse, /* Leave error message here */ + NameContext *pNC, /* The name context */ + const char *zMsg /* Type of error */ +){ + if( (pNC->ncFlags & NC_PartIdx)!=0 ){ + sqlite3ErrorMsg(pParse, "%s prohibited in partial index WHERE clauses", + zMsg); + } +} + +#ifndef SQLITE_OMIT_CHECK +/* +** Report an error that an expression is not valid for a CHECK constraint. +*/ +static void notValidCheckConstraint( + Parse *pParse, /* Leave error message here */ + NameContext *pNC, /* The name context */ + const char *zMsg /* Type of error */ +){ + if( (pNC->ncFlags & NC_IsCheck)!=0 ){ + sqlite3ErrorMsg(pParse,"%s prohibited in CHECK constraints", zMsg); + } +} +#else +# define notValidCheckConstraint(P,N,M) +#endif + +/* +** Expression p should encode a floating point value between 1.0 and 0.0. +** Return 1024 times this value. Or return -1 if p is not a floating point +** value between 1.0 and 0.0. +*/ +static int exprProbability(Expr *p){ + double r = -1.0; + if( p->op!=TK_FLOAT ) return -1; + sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); + assert( r>=0.0 ); + if( r>1.0 ) return -1; + return (int)(r*1000.0); +} + +/* +** This routine is callback for sqlite3WalkExpr(). +** +** Resolve symbolic names into TK_COLUMN operators for the current +** node in the expression tree. Return 0 to continue the search down +** the tree or 2 to abort the tree walk. +** +** This routine also does error checking and name resolution for +** function names. The operator for aggregate functions is changed +** to TK_AGG_FUNCTION. +*/ +static int resolveExprStep(Walker *pWalker, Expr *pExpr){ + NameContext *pNC; + Parse *pParse; + + pNC = pWalker->u.pNC; + assert( pNC!=0 ); + pParse = pNC->pParse; + assert( pParse==pWalker->pParse ); + + if( ExprHasProperty(pExpr, EP_Resolved) ) return WRC_Prune; + ExprSetProperty(pExpr, EP_Resolved); +#ifndef NDEBUG + if( pNC->pSrcList && pNC->pSrcList->nAlloc>0 ){ + SrcList *pSrcList = pNC->pSrcList; + int i; + for(i=0; ipSrcList->nSrc; i++){ + assert( pSrcList->a[i].iCursor>=0 && pSrcList->a[i].iCursornTab); + } + } +#endif + switch( pExpr->op ){ + +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) + /* The special operator TK_ROW means use the rowid for the first + ** column in the FROM clause. This is used by the LIMIT and ORDER BY + ** clause processing on UPDATE and DELETE statements. + */ + case TK_ROW: { + SrcList *pSrcList = pNC->pSrcList; + struct SrcList_item *pItem; + assert( pSrcList && pSrcList->nSrc==1 ); + pItem = pSrcList->a; + pExpr->op = TK_COLUMN; + pExpr->pTab = pItem->pTab; + pExpr->iTable = pItem->iCursor; + pExpr->iColumn = -1; + pExpr->affinity = SQLITE_AFF_INTEGER; + break; + } +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ + + /* A lone identifier is the name of a column. + */ + case TK_ID: { + return lookupName(pParse, 0, 0, pExpr->u.zToken, pNC, pExpr); + } + + /* A table name and column name: ID.ID + ** Or a database, table and column: ID.ID.ID + */ + case TK_DOT: { + const char *zColumn; + const char *zTable; + const char *zDb; + Expr *pRight; + + /* if( pSrcList==0 ) break; */ + pRight = pExpr->pRight; + if( pRight->op==TK_ID ){ + zDb = 0; + zTable = pExpr->pLeft->u.zToken; + zColumn = pRight->u.zToken; + }else{ + assert( pRight->op==TK_DOT ); + zDb = pExpr->pLeft->u.zToken; + zTable = pRight->pLeft->u.zToken; + zColumn = pRight->pRight->u.zToken; + } + return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + } + + /* Resolve function names + */ + case TK_FUNCTION: { + ExprList *pList = pExpr->x.pList; /* The argument list */ + int n = pList ? pList->nExpr : 0; /* Number of arguments */ + int no_such_func = 0; /* True if no such function exists */ + int wrong_num_args = 0; /* True if wrong number of arguments */ + int is_agg = 0; /* True if is an aggregate function */ + int auth; /* Authorization to use the function */ + int nId; /* Number of characters in function name */ + const char *zId; /* The function name. */ + FuncDef *pDef; /* Information about the function */ + u8 enc = ENC(pParse->db); /* The database encoding */ + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + notValidPartIdxWhere(pParse, pNC, "functions"); + zId = pExpr->u.zToken; + nId = sqlite3Strlen30(zId); + pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); + if( pDef==0 ){ + pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0); + if( pDef==0 ){ + no_such_func = 1; + }else{ + wrong_num_args = 1; + } + }else{ + is_agg = pDef->xFunc==0; + if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ + ExprSetProperty(pExpr, EP_Unlikely|EP_Skip); + if( n==2 ){ + pExpr->iTable = exprProbability(pList->a[1].pExpr); + if( pExpr->iTable<0 ){ + sqlite3ErrorMsg(pParse, "second argument to likelihood() must be a " + "constant between 0.0 and 1.0"); + pNC->nErr++; + } + }else{ + /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is equivalent to + ** likelihood(X, 0.0625). + ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is short-hand for + ** likelihood(X,0.0625). + ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand for + ** likelihood(X,0.9375). + ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to + ** likelihood(X,0.9375). */ + /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ + pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; + } + } + } +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pDef ){ + auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %s", + pDef->zName); + pNC->nErr++; + } + pExpr->op = TK_NULL; + return WRC_Prune; + } + if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ) ExprSetProperty(pExpr,EP_Constant); + } +#endif + if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ + sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); + pNC->nErr++; + is_agg = 0; + }else if( no_such_func && pParse->db->init.busy==0 ){ + sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); + pNC->nErr++; + }else if( wrong_num_args ){ + sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", + nId, zId); + pNC->nErr++; + } + if( is_agg ) pNC->ncFlags &= ~NC_AllowAgg; + sqlite3WalkExprList(pWalker, pList); + if( is_agg ){ + NameContext *pNC2 = pNC; + pExpr->op = TK_AGG_FUNCTION; + pExpr->op2 = 0; + while( pNC2 && !sqlite3FunctionUsesThisSrc(pExpr, pNC2->pSrcList) ){ + pExpr->op2++; + pNC2 = pNC2->pNext; + } + if( pNC2 ) pNC2->ncFlags |= NC_HasAgg; + pNC->ncFlags |= NC_AllowAgg; + } + /* FIX ME: Compute pExpr->affinity based on the expected return + ** type of the function + */ + return WRC_Prune; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_SELECT: + case TK_EXISTS: testcase( pExpr->op==TK_EXISTS ); +#endif + case TK_IN: { + testcase( pExpr->op==TK_IN ); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + int nRef = pNC->nRef; + notValidCheckConstraint(pParse, pNC, "subqueries"); + notValidPartIdxWhere(pParse, pNC, "subqueries"); + sqlite3WalkSelect(pWalker, pExpr->x.pSelect); + assert( pNC->nRef>=nRef ); + if( nRef!=pNC->nRef ){ + ExprSetProperty(pExpr, EP_VarSelect); + } + } + break; + } + case TK_VARIABLE: { + notValidCheckConstraint(pParse, pNC, "parameters"); + notValidPartIdxWhere(pParse, pNC, "parameters"); + break; + } + } + return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; +} + +/* +** pEList is a list of expressions which are really the result set of the +** a SELECT statement. pE is a term in an ORDER BY or GROUP BY clause. +** This routine checks to see if pE is a simple identifier which corresponds +** to the AS-name of one of the terms of the expression list. If it is, +** this routine return an integer between 1 and N where N is the number of +** elements in pEList, corresponding to the matching entry. If there is +** no match, or if pE is not a simple identifier, then this routine +** return 0. +** +** pEList has been resolved. pE has not. +*/ +static int resolveAsName( + Parse *pParse, /* Parsing context for error messages */ + ExprList *pEList, /* List of expressions to scan */ + Expr *pE /* Expression we are trying to match */ +){ + int i; /* Loop counter */ + + UNUSED_PARAMETER(pParse); + + if( pE->op==TK_ID ){ + char *zCol = pE->u.zToken; + for(i=0; inExpr; i++){ + char *zAs = pEList->a[i].zName; + if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + return i+1; + } + } + } + return 0; +} + +/* +** pE is a pointer to an expression which is a single term in the +** ORDER BY of a compound SELECT. The expression has not been +** name resolved. +** +** At the point this routine is called, we already know that the +** ORDER BY term is not an integer index into the result set. That +** case is handled by the calling routine. +** +** Attempt to match pE against result set columns in the left-most +** SELECT statement. Return the index i of the matching column, +** as an indication to the caller that it should sort by the i-th column. +** The left-most column is 1. In other words, the value returned is the +** same integer value that would be used in the SQL statement to indicate +** the column. +** +** If there is no match, return 0. Return -1 if an error occurs. +*/ +static int resolveOrderByTermToExprList( + Parse *pParse, /* Parsing context for error messages */ + Select *pSelect, /* The SELECT statement with the ORDER BY clause */ + Expr *pE /* The specific ORDER BY term */ +){ + int i; /* Loop counter */ + ExprList *pEList; /* The columns of the result set */ + NameContext nc; /* Name context for resolving pE */ + sqlite3 *db; /* Database connection */ + int rc; /* Return code from subprocedures */ + u8 savedSuppErr; /* Saved value of db->suppressErr */ + + assert( sqlite3ExprIsInteger(pE, &i)==0 ); + pEList = pSelect->pEList; + + /* Resolve all names in the ORDER BY term expression + */ + memset(&nc, 0, sizeof(nc)); + nc.pParse = pParse; + nc.pSrcList = pSelect->pSrc; + nc.pEList = pEList; + nc.ncFlags = NC_AllowAgg; + nc.nErr = 0; + db = pParse->db; + savedSuppErr = db->suppressErr; + db->suppressErr = 1; + rc = sqlite3ResolveExprNames(&nc, pE); + db->suppressErr = savedSuppErr; + if( rc ) return 0; + + /* Try to match the ORDER BY expression against an expression + ** in the result set. Return an 1-based index of the matching + ** result-set entry. + */ + for(i=0; inExpr; i++){ + if( sqlite3ExprCompare(pEList->a[i].pExpr, pE, -1)<2 ){ + return i+1; + } + } + + /* If no match, return 0. */ + return 0; +} + +/* +** Generate an ORDER BY or GROUP BY term out-of-range error. +*/ +static void resolveOutOfRangeError( + Parse *pParse, /* The error context into which to write the error */ + const char *zType, /* "ORDER" or "GROUP" */ + int i, /* The index (1-based) of the term out of range */ + int mx /* Largest permissible value of i */ +){ + sqlite3ErrorMsg(pParse, + "%r %s BY term out of range - should be " + "between 1 and %d", i, zType, mx); +} + +/* +** Analyze the ORDER BY clause in a compound SELECT statement. Modify +** each term of the ORDER BY clause is a constant integer between 1 +** and N where N is the number of columns in the compound SELECT. +** +** ORDER BY terms that are already an integer between 1 and N are +** unmodified. ORDER BY terms that are integers outside the range of +** 1 through N generate an error. ORDER BY terms that are expressions +** are matched against result set expressions of compound SELECT +** beginning with the left-most SELECT and working toward the right. +** At the first match, the ORDER BY expression is transformed into +** the integer column number. +** +** Return the number of errors seen. +*/ +static int resolveCompoundOrderBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect /* The SELECT statement containing the ORDER BY */ +){ + int i; + ExprList *pOrderBy; + ExprList *pEList; + sqlite3 *db; + int moreToDo = 1; + + pOrderBy = pSelect->pOrderBy; + if( pOrderBy==0 ) return 0; + db = pParse->db; +#if SQLITE_MAX_COLUMN + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); + return 1; + } +#endif + for(i=0; inExpr; i++){ + pOrderBy->a[i].done = 0; + } + pSelect->pNext = 0; + while( pSelect->pPrior ){ + pSelect->pPrior->pNext = pSelect; + pSelect = pSelect->pPrior; + } + while( pSelect && moreToDo ){ + struct ExprList_item *pItem; + moreToDo = 0; + pEList = pSelect->pEList; + assert( pEList!=0 ); + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + int iCol = -1; + Expr *pE, *pDup; + if( pItem->done ) continue; + pE = sqlite3ExprSkipCollate(pItem->pExpr); + if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( iCol<=0 || iCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); + return 1; + } + }else{ + iCol = resolveAsName(pParse, pEList, pE); + if( iCol==0 ){ + pDup = sqlite3ExprDup(db, pE, 0); + if( !db->mallocFailed ){ + assert(pDup); + iCol = resolveOrderByTermToExprList(pParse, pSelect, pDup); + } + sqlite3ExprDelete(db, pDup); + } + } + if( iCol>0 ){ + /* Convert the ORDER BY term into an integer column number iCol, + ** taking care to preserve the COLLATE clause if it exists */ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return 1; + pNew->flags |= EP_IntValue; + pNew->u.iValue = iCol; + if( pItem->pExpr==pE ){ + pItem->pExpr = pNew; + }else{ + assert( pItem->pExpr->op==TK_COLLATE ); + assert( pItem->pExpr->pLeft==pE ); + pItem->pExpr->pLeft = pNew; + } + sqlite3ExprDelete(db, pE); + pItem->u.x.iOrderByCol = (u16)iCol; + pItem->done = 1; + }else{ + moreToDo = 1; + } + } + pSelect = pSelect->pNext; + } + for(i=0; inExpr; i++){ + if( pOrderBy->a[i].done==0 ){ + sqlite3ErrorMsg(pParse, "%r ORDER BY term does not match any " + "column in the result set", i+1); + return 1; + } + } + return 0; +} + +/* +** Check every term in the ORDER BY or GROUP BY clause pOrderBy of +** the SELECT statement pSelect. If any term is reference to a +** result set expression (as determined by the ExprList.a.u.x.iOrderByCol +** field) then convert that term into a copy of the corresponding result set +** column. +** +** If any errors are detected, add an error message to pParse and +** return non-zero. Return zero if no errors are seen. +*/ +SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect, /* The SELECT statement containing the clause */ + ExprList *pOrderBy, /* The ORDER BY or GROUP BY clause to be processed */ + const char *zType /* "ORDER" or "GROUP" */ +){ + int i; + sqlite3 *db = pParse->db; + ExprList *pEList; + struct ExprList_item *pItem; + + if( pOrderBy==0 || pParse->db->mallocFailed ) return 0; +#if SQLITE_MAX_COLUMN + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); + return 1; + } +#endif + pEList = pSelect->pEList; + assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + if( pItem->u.x.iOrderByCol ){ + if( pItem->u.x.iOrderByCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); + return 1; + } + resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, zType,0); + } + } + return 0; +} + +/* +** pOrderBy is an ORDER BY or GROUP BY clause in SELECT statement pSelect. +** The Name context of the SELECT statement is pNC. zType is either +** "ORDER" or "GROUP" depending on which type of clause pOrderBy is. +** +** This routine resolves each term of the clause into an expression. +** If the order-by term is an integer I between 1 and N (where N is the +** number of columns in the result set of the SELECT) then the expression +** in the resolution is a copy of the I-th result-set expression. If +** the order-by term is an identifier that corresponds to the AS-name of +** a result-set expression, then the term resolves to a copy of the +** result-set expression. Otherwise, the expression is resolved in +** the usual way - using sqlite3ResolveExprNames(). +** +** This routine returns the number of errors. If errors occur, then +** an appropriate error message might be left in pParse. (OOM errors +** excepted.) +*/ +static int resolveOrderGroupBy( + NameContext *pNC, /* The name context of the SELECT statement */ + Select *pSelect, /* The SELECT statement holding pOrderBy */ + ExprList *pOrderBy, /* An ORDER BY or GROUP BY clause to resolve */ + const char *zType /* Either "ORDER" or "GROUP", as appropriate */ +){ + int i, j; /* Loop counters */ + int iCol; /* Column number */ + struct ExprList_item *pItem; /* A term of the ORDER BY clause */ + Parse *pParse; /* Parsing context */ + int nResult; /* Number of terms in the result set */ + + if( pOrderBy==0 ) return 0; + nResult = pSelect->pEList->nExpr; + pParse = pNC->pParse; + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + Expr *pE = pItem->pExpr; + Expr *pE2 = sqlite3ExprSkipCollate(pE); + if( zType[0]!='G' ){ + iCol = resolveAsName(pParse, pSelect->pEList, pE2); + if( iCol>0 ){ + /* If an AS-name match is found, mark this ORDER BY column as being + ** a copy of the iCol-th result-set column. The subsequent call to + ** sqlite3ResolveOrderGroupBy() will convert the expression to a + ** copy of the iCol-th result-set expression. */ + pItem->u.x.iOrderByCol = (u16)iCol; + continue; + } + } + if( sqlite3ExprIsInteger(pE2, &iCol) ){ + /* The ORDER BY term is an integer constant. Again, set the column + ** number so that sqlite3ResolveOrderGroupBy() will convert the + ** order-by term to a copy of the result-set expression */ + if( iCol<1 || iCol>0xffff ){ + resolveOutOfRangeError(pParse, zType, i+1, nResult); + return 1; + } + pItem->u.x.iOrderByCol = (u16)iCol; + continue; + } + + /* Otherwise, treat the ORDER BY term as an ordinary expression */ + pItem->u.x.iOrderByCol = 0; + if( sqlite3ResolveExprNames(pNC, pE) ){ + return 1; + } + for(j=0; jpEList->nExpr; j++){ + if( sqlite3ExprCompare(pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ + pItem->u.x.iOrderByCol = j+1; + } + } + } + return sqlite3ResolveOrderGroupBy(pParse, pSelect, pOrderBy, zType); +} + +/* +** Resolve names in the SELECT statement p and all of its descendents. +*/ +static int resolveSelectStep(Walker *pWalker, Select *p){ + NameContext *pOuterNC; /* Context that contains this SELECT */ + NameContext sNC; /* Name context of this SELECT */ + int isCompound; /* True if p is a compound select */ + int nCompound; /* Number of compound terms processed so far */ + Parse *pParse; /* Parsing context */ + ExprList *pEList; /* Result set expression list */ + int i; /* Loop counter */ + ExprList *pGroupBy; /* The GROUP BY clause */ + Select *pLeftmost; /* Left-most of SELECT of a compound */ + sqlite3 *db; /* Database connection */ + + + assert( p!=0 ); + if( p->selFlags & SF_Resolved ){ + return WRC_Prune; + } + pOuterNC = pWalker->u.pNC; + pParse = pWalker->pParse; + db = pParse->db; + + /* Normally sqlite3SelectExpand() will be called first and will have + ** already expanded this SELECT. However, if this is a subquery within + ** an expression, sqlite3ResolveExprNames() will be called without a + ** prior call to sqlite3SelectExpand(). When that happens, let + ** sqlite3SelectPrep() do all of the processing for this SELECT. + ** sqlite3SelectPrep() will invoke both sqlite3SelectExpand() and + ** this routine in the correct order. + */ + if( (p->selFlags & SF_Expanded)==0 ){ + sqlite3SelectPrep(pParse, p, pOuterNC); + return (pParse->nErr || db->mallocFailed) ? WRC_Abort : WRC_Prune; + } + + isCompound = p->pPrior!=0; + nCompound = 0; + pLeftmost = p; + while( p ){ + assert( (p->selFlags & SF_Expanded)!=0 ); + assert( (p->selFlags & SF_Resolved)==0 ); + p->selFlags |= SF_Resolved; + + /* Resolve the expressions in the LIMIT and OFFSET clauses. These + ** are not allowed to refer to any names, so pass an empty NameContext. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + if( sqlite3ResolveExprNames(&sNC, p->pLimit) || + sqlite3ResolveExprNames(&sNC, p->pOffset) ){ + return WRC_Abort; + } + + /* Recursively resolve names in all subqueries + */ + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + if( pItem->pSelect ){ + NameContext *pNC; /* Used to iterate name contexts */ + int nRef = 0; /* Refcount for pOuterNC and outer contexts */ + const char *zSavedContext = pParse->zAuthContext; + + /* Count the total number of references to pOuterNC and all of its + ** parent contexts. After resolving references to expressions in + ** pItem->pSelect, check if this value has changed. If so, then + ** SELECT statement pItem->pSelect must be correlated. Set the + ** pItem->isCorrelated flag if this is the case. */ + for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; + + if( pItem->zName ) pParse->zAuthContext = pItem->zName; + sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + pParse->zAuthContext = zSavedContext; + if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + + for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; + assert( pItem->isCorrelated==0 && nRef<=0 ); + pItem->isCorrelated = (nRef!=0); + } + } + + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to + ** resolve the result-set expression list. + */ + sNC.ncFlags = NC_AllowAgg; + sNC.pSrcList = p->pSrc; + sNC.pNext = pOuterNC; + + /* Resolve names in the result set. */ + pEList = p->pEList; + assert( pEList!=0 ); + for(i=0; inExpr; i++){ + Expr *pX = pEList->a[i].pExpr; + if( sqlite3ResolveExprNames(&sNC, pX) ){ + return WRC_Abort; + } + } + + /* If there are no aggregate functions in the result-set, and no GROUP BY + ** expression, do not allow aggregates in any of the other expressions. + */ + assert( (p->selFlags & SF_Aggregate)==0 ); + pGroupBy = p->pGroupBy; + if( pGroupBy || (sNC.ncFlags & NC_HasAgg)!=0 ){ + p->selFlags |= SF_Aggregate; + }else{ + sNC.ncFlags &= ~NC_AllowAgg; + } + + /* If a HAVING clause is present, then there must be a GROUP BY clause. + */ + if( p->pHaving && !pGroupBy ){ + sqlite3ErrorMsg(pParse, "a GROUP BY clause is required before HAVING"); + return WRC_Abort; + } + + /* Add the output column list to the name-context before parsing the + ** other expressions in the SELECT statement. This is so that + ** expressions in the WHERE clause (etc.) can refer to expressions by + ** aliases in the result set. + ** + ** Minor point: If this is the case, then the expression will be + ** re-evaluated for each reference to it. + */ + sNC.pEList = p->pEList; + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + + /* The ORDER BY and GROUP BY clauses may not refer to terms in + ** outer queries + */ + sNC.pNext = 0; + sNC.ncFlags |= NC_AllowAgg; + + /* Process the ORDER BY clause for singleton SELECT statements. + ** The ORDER BY clause for compounds SELECT statements is handled + ** below, after all of the result-sets for all of the elements of + ** the compound have been resolved. + */ + if( !isCompound && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") ){ + return WRC_Abort; + } + if( db->mallocFailed ){ + return WRC_Abort; + } + + /* Resolve the GROUP BY clause. At the same time, make sure + ** the GROUP BY clause does not contain aggregate functions. + */ + if( pGroupBy ){ + struct ExprList_item *pItem; + + if( resolveOrderGroupBy(&sNC, p, pGroupBy, "GROUP") || db->mallocFailed ){ + return WRC_Abort; + } + for(i=0, pItem=pGroupBy->a; inExpr; i++, pItem++){ + if( ExprHasProperty(pItem->pExpr, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "aggregate functions are not allowed in " + "the GROUP BY clause"); + return WRC_Abort; + } + } + } + + /* Advance to the next term of the compound + */ + p = p->pPrior; + nCompound++; + } + + /* Resolve the ORDER BY on a compound SELECT after all terms of + ** the compound have been resolved. + */ + if( isCompound && resolveCompoundOrderBy(pParse, pLeftmost) ){ + return WRC_Abort; + } + + return WRC_Prune; +} + +/* +** This routine walks an expression tree and resolves references to +** table columns and result-set columns. At the same time, do error +** checking on function usage and set a flag if any aggregate functions +** are seen. +** +** To resolve table columns references we look for nodes (or subtrees) of the +** form X.Y.Z or Y.Z or just Z where +** +** X: The name of a database. Ex: "main" or "temp" or +** the symbolic name assigned to an ATTACH-ed database. +** +** Y: The name of a table in a FROM clause. Or in a trigger +** one of the special names "old" or "new". +** +** Z: The name of a column in table Y. +** +** The node at the root of the subtree is modified as follows: +** +** Expr.op Changed to TK_COLUMN +** Expr.pTab Points to the Table object for X.Y +** Expr.iColumn The column index in X.Y. -1 for the rowid. +** Expr.iTable The VDBE cursor number for X.Y +** +** +** To resolve result-set references, look for expression nodes of the +** form Z (with no X and Y prefix) where the Z matches the right-hand +** size of an AS clause in the result-set of a SELECT. The Z expression +** is replaced by a copy of the left-hand side of the result-set expression. +** Table-name and function resolution occurs on the substituted expression +** tree. For example, in: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY x; +** +** The "x" term of the order by is replaced by "a+b" to render: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY a+b; +** +** Function calls are checked to make sure that the function is +** defined and that the correct number of arguments are specified. +** If the function is an aggregate function, then the NC_HasAgg flag is +** set and the opcode is changed from TK_FUNCTION to TK_AGG_FUNCTION. +** If an expression contains aggregate functions then the EP_Agg +** property on the expression is set. +** +** An error message is left in pParse if anything is amiss. The number +** if errors is returned. +*/ +SQLITE_PRIVATE int sqlite3ResolveExprNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + Expr *pExpr /* The expression to be analyzed. */ +){ + u8 savedHasAgg; + Walker w; + + if( pExpr==0 ) return 0; +#if SQLITE_MAX_EXPR_DEPTH>0 + { + Parse *pParse = pNC->pParse; + if( sqlite3ExprCheckHeight(pParse, pExpr->nHeight+pNC->pParse->nHeight) ){ + return 1; + } + pParse->nHeight += pExpr->nHeight; + } +#endif + savedHasAgg = pNC->ncFlags & NC_HasAgg; + pNC->ncFlags &= ~NC_HasAgg; + memset(&w, 0, sizeof(w)); + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.pParse = pNC->pParse; + w.u.pNC = pNC; + sqlite3WalkExpr(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + pNC->pParse->nHeight -= pExpr->nHeight; +#endif + if( pNC->nErr>0 || w.pParse->nErr>0 ){ + ExprSetProperty(pExpr, EP_Error); + } + if( pNC->ncFlags & NC_HasAgg ){ + ExprSetProperty(pExpr, EP_Agg); + }else if( savedHasAgg ){ + pNC->ncFlags |= NC_HasAgg; + } + return ExprHasProperty(pExpr, EP_Error); +} + + +/* +** Resolve all names in all expressions of a SELECT and in all +** decendents of the SELECT, including compounds off of p->pPrior, +** subqueries in expressions, and subqueries used as FROM clause +** terms. +** +** See sqlite3ResolveExprNames() for a description of the kinds of +** transformations that occur. +** +** All SELECT statements should have been expanded using +** sqlite3SelectExpand() prior to invoking this routine. +*/ +SQLITE_PRIVATE void sqlite3ResolveSelectNames( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + NameContext *pOuterNC /* Name context for parent SELECT statement */ +){ + Walker w; + + assert( p!=0 ); + memset(&w, 0, sizeof(w)); + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.pParse = pParse; + w.u.pNC = pOuterNC; + sqlite3WalkSelect(&w, p); +} + +/* +** Resolve names in expressions that can only reference a single table: +** +** * CHECK constraints +** * WHERE clauses on partial indices +** +** The Expr.iTable value for Expr.op==TK_COLUMN nodes of the expression +** is set to -1 and the Expr.iColumn value is set to the column number. +** +** Any errors cause an error message to be set in pParse. +*/ +SQLITE_PRIVATE void sqlite3ResolveSelfReference( + Parse *pParse, /* Parsing context */ + Table *pTab, /* The table being referenced */ + int type, /* NC_IsCheck or NC_PartIdx */ + Expr *pExpr, /* Expression to resolve. May be NULL. */ + ExprList *pList /* Expression list to resolve. May be NUL. */ +){ + SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ + NameContext sNC; /* Name context for pParse->pNewTable */ + int i; /* Loop counter */ + + assert( type==NC_IsCheck || type==NC_PartIdx ); + memset(&sNC, 0, sizeof(sNC)); + memset(&sSrc, 0, sizeof(sSrc)); + sSrc.nSrc = 1; + sSrc.a[0].zName = pTab->zName; + sSrc.a[0].pTab = pTab; + sSrc.a[0].iCursor = -1; + sNC.pParse = pParse; + sNC.pSrcList = &sSrc; + sNC.ncFlags = type; + if( sqlite3ResolveExprNames(&sNC, pExpr) ) return; + if( pList ){ + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ + return; + } + } + } +} + +/************** End of resolve.c *********************************************/ +/************** Begin file expr.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains routines used for analyzing expressions and +** for generating VDBE code that evaluates expressions in SQLite. +*/ + +/* +** Return the 'affinity' of the expression pExpr if any. +** +** If pExpr is a column, a reference to a column via an 'AS' alias, +** or a sub-select with a column as the return value, then the +** affinity of that column is returned. Otherwise, 0x00 is returned, +** indicating no affinity for the expression. +** +** i.e. the WHERE clause expresssions in the following statements all +** have an affinity: +** +** CREATE TABLE t1(a); +** SELECT * FROM t1 WHERE a; +** SELECT a AS b FROM t1 WHERE b; +** SELECT * FROM t1 WHERE (select a from t1); +*/ +SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ + int op; + pExpr = sqlite3ExprSkipCollate(pExpr); + if( pExpr->flags & EP_Generic ) return 0; + op = pExpr->op; + if( op==TK_SELECT ){ + assert( pExpr->flags&EP_xIsSelect ); + return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); + } +#ifndef SQLITE_OMIT_CAST + if( op==TK_CAST ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + return sqlite3AffinityType(pExpr->u.zToken, 0); + } +#endif + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) + && pExpr->pTab!=0 + ){ + /* op==TK_REGISTER && pExpr->pTab!=0 happens when pExpr was originally + ** a TK_COLUMN but was previously evaluated and cached in a register */ + int j = pExpr->iColumn; + if( j<0 ) return SQLITE_AFF_INTEGER; + assert( pExpr->pTab && jpTab->nCol ); + return pExpr->pTab->aCol[j].affinity; + } + return pExpr->affinity; +} + +/* +** Set the collating sequence for expression pExpr to be the collating +** sequence named by pToken. Return a pointer to a new Expr node that +** implements the COLLATE operator. +** +** If a memory allocation error occurs, that fact is recorded in pParse->db +** and the pExpr parameter is returned unchanged. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* Add the "COLLATE" clause to this expression */ + const Token *pCollName /* Name of collating sequence */ +){ + if( pCollName->n>0 ){ + Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, 1); + if( pNew ){ + pNew->pLeft = pExpr; + pNew->flags |= EP_Collate|EP_Skip; + pExpr = pNew; + } + } + return pExpr; +} +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ + Token s; + assert( zC!=0 ); + s.z = zC; + s.n = sqlite3Strlen30(s.z); + return sqlite3ExprAddCollateToken(pParse, pExpr, &s); +} + +/* +** Skip over any TK_COLLATE or TK_AS operators and any unlikely() +** or likelihood() function at the root of an expression. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ + while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){ + if( ExprHasProperty(pExpr, EP_Unlikely) ){ + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + assert( pExpr->x.pList->nExpr>0 ); + assert( pExpr->op==TK_FUNCTION ); + pExpr = pExpr->x.pList->a[0].pExpr; + }else{ + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ); + pExpr = pExpr->pLeft; + } + } + return pExpr; +} + +/* +** Return the collation sequence for the expression pExpr. If +** there is no defined collating sequence, return NULL. +** +** The collating sequence might be determined by a COLLATE operator +** or by the presence of a column with a defined collating sequence. +** COLLATE operators take first precedence. Left operands take +** precedence over right operands. +*/ +SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ + sqlite3 *db = pParse->db; + CollSeq *pColl = 0; + Expr *p = pExpr; + while( p ){ + int op = p->op; + if( p->flags & EP_Generic ) break; + if( op==TK_CAST || op==TK_UPLUS ){ + p = p->pLeft; + continue; + } + if( op==TK_COLLATE || (op==TK_REGISTER && p->op2==TK_COLLATE) ){ + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); + break; + } + if( p->pTab!=0 + && (op==TK_AGG_COLUMN || op==TK_COLUMN + || op==TK_REGISTER || op==TK_TRIGGER) + ){ + /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally + ** a TK_COLUMN but was previously evaluated and cached in a register */ + int j = p->iColumn; + if( j>=0 ){ + const char *zColl = p->pTab->aCol[j].zColl; + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); + } + break; + } + if( p->flags & EP_Collate ){ + if( ALWAYS(p->pLeft) && (p->pLeft->flags & EP_Collate)!=0 ){ + p = p->pLeft; + }else{ + p = p->pRight; + } + }else{ + break; + } + } + if( sqlite3CheckCollSeq(pParse, pColl) ){ + pColl = 0; + } + return pColl; +} + +/* +** pExpr is an operand of a comparison operator. aff2 is the +** type affinity of the other operand. This routine returns the +** type affinity that should be used for the comparison operator. +*/ +SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ + char aff1 = sqlite3ExprAffinity(pExpr); + if( aff1 && aff2 ){ + /* Both sides of the comparison are columns. If one has numeric + ** affinity, use that. Otherwise use no affinity. + */ + if( sqlite3IsNumericAffinity(aff1) || sqlite3IsNumericAffinity(aff2) ){ + return SQLITE_AFF_NUMERIC; + }else{ + return SQLITE_AFF_NONE; + } + }else if( !aff1 && !aff2 ){ + /* Neither side of the comparison is a column. Compare the + ** results directly. + */ + return SQLITE_AFF_NONE; + }else{ + /* One side is a column, the other is not. Use the columns affinity. */ + assert( aff1==0 || aff2==0 ); + return (aff1 + aff2); + } +} + +/* +** pExpr is a comparison operator. Return the type affinity that should +** be applied to both operands prior to doing the comparison. +*/ +static char comparisonAffinity(Expr *pExpr){ + char aff; + assert( pExpr->op==TK_EQ || pExpr->op==TK_IN || pExpr->op==TK_LT || + pExpr->op==TK_GT || pExpr->op==TK_GE || pExpr->op==TK_LE || + pExpr->op==TK_NE || pExpr->op==TK_IS || pExpr->op==TK_ISNOT ); + assert( pExpr->pLeft ); + aff = sqlite3ExprAffinity(pExpr->pLeft); + if( pExpr->pRight ){ + aff = sqlite3CompareAffinity(pExpr->pRight, aff); + }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); + }else if( !aff ){ + aff = SQLITE_AFF_NONE; + } + return aff; +} + +/* +** pExpr is a comparison expression, eg. '=', '<', IN(...) etc. +** idx_affinity is the affinity of an indexed column. Return true +** if the index with affinity idx_affinity may be used to implement +** the comparison in pExpr. +*/ +SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ + char aff = comparisonAffinity(pExpr); + switch( aff ){ + case SQLITE_AFF_NONE: + return 1; + case SQLITE_AFF_TEXT: + return idx_affinity==SQLITE_AFF_TEXT; + default: + return sqlite3IsNumericAffinity(idx_affinity); + } +} + +/* +** Return the P5 value that should be used for a binary comparison +** opcode (OP_Eq, OP_Ge etc.) used to compare pExpr1 and pExpr2. +*/ +static u8 binaryCompareP5(Expr *pExpr1, Expr *pExpr2, int jumpIfNull){ + u8 aff = (char)sqlite3ExprAffinity(pExpr2); + aff = (u8)sqlite3CompareAffinity(pExpr1, aff) | (u8)jumpIfNull; + return aff; +} + +/* +** Return a pointer to the collation sequence that should be used by +** a binary comparison operator comparing pLeft and pRight. +** +** If the left hand expression has a collating sequence type, then it is +** used. Otherwise the collation sequence for the right hand expression +** is used, or the default (BINARY) if neither expression has a collating +** type. +** +** Argument pRight (but not pLeft) may be a null pointer. In this case, +** it is not considered. +*/ +SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq( + Parse *pParse, + Expr *pLeft, + Expr *pRight +){ + CollSeq *pColl; + assert( pLeft ); + if( pLeft->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pLeft); + }else if( pRight && (pRight->flags & EP_Collate)!=0 ){ + pColl = sqlite3ExprCollSeq(pParse, pRight); + }else{ + pColl = sqlite3ExprCollSeq(pParse, pLeft); + if( !pColl ){ + pColl = sqlite3ExprCollSeq(pParse, pRight); + } + } + return pColl; +} + +/* +** Generate code for a comparison operator. +*/ +static int codeCompare( + Parse *pParse, /* The parsing (and code generating) context */ + Expr *pLeft, /* The left operand */ + Expr *pRight, /* The right operand */ + int opcode, /* The comparison opcode */ + int in1, int in2, /* Register holding operands */ + int dest, /* Jump here if true. */ + int jumpIfNull /* If true, jump if either operand is NULL */ +){ + int p5; + int addr; + CollSeq *p4; + + p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); + p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); + addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, + (void*)p4, P4_COLLSEQ); + sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); + return addr; +} + +#if SQLITE_MAX_EXPR_DEPTH>0 +/* +** Check that argument nHeight is less than or equal to the maximum +** expression depth allowed. If it is not, leave an error message in +** pParse. +*/ +SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse *pParse, int nHeight){ + int rc = SQLITE_OK; + int mxHeight = pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH]; + if( nHeight>mxHeight ){ + sqlite3ErrorMsg(pParse, + "Expression tree is too large (maximum depth %d)", mxHeight + ); + rc = SQLITE_ERROR; + } + return rc; +} + +/* The following three functions, heightOfExpr(), heightOfExprList() +** and heightOfSelect(), are used to determine the maximum height +** of any expression tree referenced by the structure passed as the +** first argument. +** +** If this maximum height is greater than the current value pointed +** to by pnHeight, the second parameter, then set *pnHeight to that +** value. +*/ +static void heightOfExpr(Expr *p, int *pnHeight){ + if( p ){ + if( p->nHeight>*pnHeight ){ + *pnHeight = p->nHeight; + } + } +} +static void heightOfExprList(ExprList *p, int *pnHeight){ + if( p ){ + int i; + for(i=0; inExpr; i++){ + heightOfExpr(p->a[i].pExpr, pnHeight); + } + } +} +static void heightOfSelect(Select *p, int *pnHeight){ + if( p ){ + heightOfExpr(p->pWhere, pnHeight); + heightOfExpr(p->pHaving, pnHeight); + heightOfExpr(p->pLimit, pnHeight); + heightOfExpr(p->pOffset, pnHeight); + heightOfExprList(p->pEList, pnHeight); + heightOfExprList(p->pGroupBy, pnHeight); + heightOfExprList(p->pOrderBy, pnHeight); + heightOfSelect(p->pPrior, pnHeight); + } +} + +/* +** Set the Expr.nHeight variable in the structure passed as an +** argument. An expression with no children, Expr.pList or +** Expr.pSelect member has a height of 1. Any other expression +** has a height equal to the maximum height of any other +** referenced Expr plus one. +*/ +static void exprSetHeight(Expr *p){ + int nHeight = 0; + heightOfExpr(p->pLeft, &nHeight); + heightOfExpr(p->pRight, &nHeight); + if( ExprHasProperty(p, EP_xIsSelect) ){ + heightOfSelect(p->x.pSelect, &nHeight); + }else{ + heightOfExprList(p->x.pList, &nHeight); + } + p->nHeight = nHeight + 1; +} + +/* +** Set the Expr.nHeight variable using the exprSetHeight() function. If +** the height is greater than the maximum allowed expression depth, +** leave an error in pParse. +*/ +SQLITE_PRIVATE void sqlite3ExprSetHeight(Parse *pParse, Expr *p){ + exprSetHeight(p); + sqlite3ExprCheckHeight(pParse, p->nHeight); +} + +/* +** Return the maximum height of any expression tree referenced +** by the select statement passed as an argument. +*/ +SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ + int nHeight = 0; + heightOfSelect(p, &nHeight); + return nHeight; +} +#else + #define exprSetHeight(y) +#endif /* SQLITE_MAX_EXPR_DEPTH>0 */ + +/* +** This routine is the core allocator for Expr nodes. +** +** Construct a new expression node and return a pointer to it. Memory +** for this node and for the pToken argument is a single allocation +** obtained from sqlite3DbMalloc(). The calling function +** is responsible for making sure the node eventually gets freed. +** +** If dequote is true, then the token (if it exists) is dequoted. +** If dequote is false, no dequoting is performance. The deQuote +** parameter is ignored if pToken is NULL or if the token does not +** appear to be quoted. If the quotes were of the form "..." (double-quotes) +** then the EP_DblQuoted flag is set on the expression node. +** +** Special case: If op==TK_INTEGER and pToken points to a string that +** can be translated into a 32-bit integer, then the token is not +** stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage +** is allocated to hold the integer text and the dequote flag is ignored. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprAlloc( + sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + int op, /* Expression opcode */ + const Token *pToken, /* Token argument. Might be NULL */ + int dequote /* True to dequote */ +){ + Expr *pNew; + int nExtra = 0; + int iValue = 0; + + if( pToken ){ + if( op!=TK_INTEGER || pToken->z==0 + || sqlite3GetInt32(pToken->z, &iValue)==0 ){ + nExtra = pToken->n+1; + assert( iValue>=0 ); + } + } + pNew = sqlite3DbMallocZero(db, sizeof(Expr)+nExtra); + if( pNew ){ + pNew->op = (u8)op; + pNew->iAgg = -1; + if( pToken ){ + if( nExtra==0 ){ + pNew->flags |= EP_IntValue; + pNew->u.iValue = iValue; + }else{ + int c; + pNew->u.zToken = (char*)&pNew[1]; + assert( pToken->z!=0 || pToken->n==0 ); + if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); + pNew->u.zToken[pToken->n] = 0; + if( dequote && nExtra>=3 + && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ + sqlite3Dequote(pNew->u.zToken); + if( c=='"' ) pNew->flags |= EP_DblQuoted; + } + } + } +#if SQLITE_MAX_EXPR_DEPTH>0 + pNew->nHeight = 1; +#endif + } + return pNew; +} + +/* +** Allocate a new expression node from a zero-terminated token that has +** already been dequoted. +*/ +SQLITE_PRIVATE Expr *sqlite3Expr( + sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + int op, /* Expression opcode */ + const char *zToken /* Token argument. Might be NULL */ +){ + Token x; + x.z = zToken; + x.n = zToken ? sqlite3Strlen30(zToken) : 0; + return sqlite3ExprAlloc(db, op, &x, 0); +} + +/* +** Attach subtrees pLeft and pRight to the Expr node pRoot. +** +** If pRoot==NULL that means that a memory allocation error has occurred. +** In that case, delete the subtrees pLeft and pRight. +*/ +SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( + sqlite3 *db, + Expr *pRoot, + Expr *pLeft, + Expr *pRight +){ + if( pRoot==0 ){ + assert( db->mallocFailed ); + sqlite3ExprDelete(db, pLeft); + sqlite3ExprDelete(db, pRight); + }else{ + if( pRight ){ + pRoot->pRight = pRight; + pRoot->flags |= EP_Collate & pRight->flags; + } + if( pLeft ){ + pRoot->pLeft = pLeft; + pRoot->flags |= EP_Collate & pLeft->flags; + } + exprSetHeight(pRoot); + } +} + +/* +** Allocate a Expr node which joins as many as two subtrees. +** +** One or both of the subtrees can be NULL. Return a pointer to the new +** Expr node. Or, if an OOM error occurs, set pParse->db->mallocFailed, +** free the subtrees and return NULL. +*/ +SQLITE_PRIVATE Expr *sqlite3PExpr( + Parse *pParse, /* Parsing context */ + int op, /* Expression opcode */ + Expr *pLeft, /* Left operand */ + Expr *pRight, /* Right operand */ + const Token *pToken /* Argument token */ +){ + Expr *p; + if( op==TK_AND && pLeft && pRight ){ + /* Take advantage of short-circuit false optimization for AND */ + p = sqlite3ExprAnd(pParse->db, pLeft, pRight); + }else{ + p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); + sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + } + if( p ) { + sqlite3ExprCheckHeight(pParse, p->nHeight); + } + return p; +} + +/* +** If the expression is always either TRUE or FALSE (respectively), +** then return 1. If one cannot determine the truth value of the +** expression at compile-time return 0. +** +** This is an optimization. If is OK to return 0 here even if +** the expression really is always false or false (a false negative). +** But it is a bug to return 1 if the expression might have different +** boolean values in different circumstances (a false positive.) +** +** Note that if the expression is part of conditional for a +** LEFT JOIN, then we cannot determine at compile-time whether or not +** is it true or false, so always return 0. +*/ +static int exprAlwaysTrue(Expr *p){ + int v = 0; + if( ExprHasProperty(p, EP_FromJoin) ) return 0; + if( !sqlite3ExprIsInteger(p, &v) ) return 0; + return v!=0; +} +static int exprAlwaysFalse(Expr *p){ + int v = 0; + if( ExprHasProperty(p, EP_FromJoin) ) return 0; + if( !sqlite3ExprIsInteger(p, &v) ) return 0; + return v==0; +} + +/* +** Join two expressions using an AND operator. If either expression is +** NULL, then just return the other expression. +** +** If one side or the other of the AND is known to be false, then instead +** of returning an AND expression, just return a constant expression with +** a value of false. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ + if( pLeft==0 ){ + return pRight; + }else if( pRight==0 ){ + return pLeft; + }else if( exprAlwaysFalse(pLeft) || exprAlwaysFalse(pRight) ){ + sqlite3ExprDelete(db, pLeft); + sqlite3ExprDelete(db, pRight); + return sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[0], 0); + }else{ + Expr *pNew = sqlite3ExprAlloc(db, TK_AND, 0, 0); + sqlite3ExprAttachSubtrees(db, pNew, pLeft, pRight); + return pNew; + } +} + +/* +** Construct a new expression node for a function with multiple +** arguments. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ + Expr *pNew; + sqlite3 *db = pParse->db; + assert( pToken ); + pNew = sqlite3ExprAlloc(db, TK_FUNCTION, pToken, 1); + if( pNew==0 ){ + sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */ + return 0; + } + pNew->x.pList = pList; + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + sqlite3ExprSetHeight(pParse, pNew); + return pNew; +} + +/* +** Assign a variable number to an expression that encodes a wildcard +** in the original SQL statement. +** +** Wildcards consisting of a single "?" are assigned the next sequential +** variable number. +** +** Wildcards of the form "?nnn" are assigned the number "nnn". We make +** sure "nnn" is not too be to avoid a denial of service attack when +** the SQL statement comes from an external source. +** +** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number +** as the previous instance of the same wildcard. Or if this is the first +** instance of the wildcard, the next sequenial variable number is +** assigned. +*/ +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ + sqlite3 *db = pParse->db; + const char *z; + + if( pExpr==0 ) return; + assert( !ExprHasProperty(pExpr, EP_IntValue|EP_Reduced|EP_TokenOnly) ); + z = pExpr->u.zToken; + assert( z!=0 ); + assert( z[0]!=0 ); + if( z[1]==0 ){ + /* Wildcard of the form "?". Assign the next variable number */ + assert( z[0]=='?' ); + pExpr->iColumn = (ynVar)(++pParse->nVar); + }else{ + ynVar x = 0; + u32 n = sqlite3Strlen30(z); + if( z[0]=='?' ){ + /* Wildcard of the form "?nnn". Convert "nnn" to an integer and + ** use it as the variable number */ + i64 i; + int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + pExpr->iColumn = x = (ynVar)i; + testcase( i==0 ); + testcase( i==1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", + db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + x = 0; + } + if( i>pParse->nVar ){ + pParse->nVar = (int)i; + } + }else{ + /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable + ** number as the prior appearance of the same name, or if the name + ** has never appeared before, reuse the same variable number + */ + ynVar i; + for(i=0; inzVar; i++){ + if( pParse->azVar[i] && strcmp(pParse->azVar[i],z)==0 ){ + pExpr->iColumn = x = (ynVar)i+1; + break; + } + } + if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); + } + if( x>0 ){ + if( x>pParse->nzVar ){ + char **a; + a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); + if( a==0 ) return; /* Error reported through db->mallocFailed */ + pParse->azVar = a; + memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); + pParse->nzVar = x; + } + if( z[0]!='?' || pParse->azVar[x-1]==0 ){ + sqlite3DbFree(db, pParse->azVar[x-1]); + pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); + } + } + } + if( !pParse->nErr && pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "too many SQL variables"); + } +} + +/* +** Recursively delete an expression tree. +*/ +SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p==0 ) return; + /* Sanity check: Assert that the IntValue is non-negative if it exists */ + assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); + if( !ExprHasProperty(p, EP_TokenOnly) ){ + /* The Expr.x union is never used at the same time as Expr.pRight */ + assert( p->x.pList==0 || p->pRight==0 ); + sqlite3ExprDelete(db, p->pLeft); + sqlite3ExprDelete(db, p->pRight); + if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); + if( ExprHasProperty(p, EP_xIsSelect) ){ + sqlite3SelectDelete(db, p->x.pSelect); + }else{ + sqlite3ExprListDelete(db, p->x.pList); + } + } + if( !ExprHasProperty(p, EP_Static) ){ + sqlite3DbFree(db, p); + } +} + +/* +** Return the number of bytes allocated for the expression structure +** passed as the first argument. This is always one of EXPR_FULLSIZE, +** EXPR_REDUCEDSIZE or EXPR_TOKENONLYSIZE. +*/ +static int exprStructSize(Expr *p){ + if( ExprHasProperty(p, EP_TokenOnly) ) return EXPR_TOKENONLYSIZE; + if( ExprHasProperty(p, EP_Reduced) ) return EXPR_REDUCEDSIZE; + return EXPR_FULLSIZE; +} + +/* +** The dupedExpr*Size() routines each return the number of bytes required +** to store a copy of an expression or expression tree. They differ in +** how much of the tree is measured. +** +** dupedExprStructSize() Size of only the Expr structure +** dupedExprNodeSize() Size of Expr + space for token +** dupedExprSize() Expr + token + subtree components +** +*************************************************************************** +** +** The dupedExprStructSize() function returns two values OR-ed together: +** (1) the space required for a copy of the Expr structure only and +** (2) the EP_xxx flags that indicate what the structure size should be. +** The return values is always one of: +** +** EXPR_FULLSIZE +** EXPR_REDUCEDSIZE | EP_Reduced +** EXPR_TOKENONLYSIZE | EP_TokenOnly +** +** The size of the structure can be found by masking the return value +** of this routine with 0xfff. The flags can be found by masking the +** return value with EP_Reduced|EP_TokenOnly. +** +** Note that with flags==EXPRDUP_REDUCE, this routines works on full-size +** (unreduced) Expr objects as they or originally constructed by the parser. +** During expression analysis, extra information is computed and moved into +** later parts of teh Expr object and that extra information might get chopped +** off if the expression is reduced. Note also that it does not work to +** make a EXPRDUP_REDUCE copy of a reduced expression. It is only legal +** to reduce a pristine expression tree from the parser. The implementation +** of dupedExprStructSize() contain multiple assert() statements that attempt +** to enforce this constraint. +*/ +static int dupedExprStructSize(Expr *p, int flags){ + int nSize; + assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ + assert( EXPR_FULLSIZE<=0xfff ); + assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); + if( 0==(flags&EXPRDUP_REDUCE) ){ + nSize = EXPR_FULLSIZE; + }else{ + assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); + assert( !ExprHasProperty(p, EP_FromJoin) ); + assert( !ExprHasProperty(p, EP_MemToken) ); + assert( !ExprHasProperty(p, EP_NoReduce) ); + if( p->pLeft || p->x.pList ){ + nSize = EXPR_REDUCEDSIZE | EP_Reduced; + }else{ + assert( p->pRight==0 ); + nSize = EXPR_TOKENONLYSIZE | EP_TokenOnly; + } + } + return nSize; +} + +/* +** This function returns the space in bytes required to store the copy +** of the Expr structure and a copy of the Expr.u.zToken string (if that +** string is defined.) +*/ +static int dupedExprNodeSize(Expr *p, int flags){ + int nByte = dupedExprStructSize(p, flags) & 0xfff; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nByte += sqlite3Strlen30(p->u.zToken)+1; + } + return ROUND8(nByte); +} + +/* +** Return the number of bytes required to create a duplicate of the +** expression passed as the first argument. The second argument is a +** mask containing EXPRDUP_XXX flags. +** +** The value returned includes space to create a copy of the Expr struct +** itself and the buffer referred to by Expr.u.zToken, if any. +** +** If the EXPRDUP_REDUCE flag is set, then the return value includes +** space to duplicate all Expr nodes in the tree formed by Expr.pLeft +** and Expr.pRight variables (but not for any structures pointed to or +** descended from the Expr.x.pList or Expr.x.pSelect variables). +*/ +static int dupedExprSize(Expr *p, int flags){ + int nByte = 0; + if( p ){ + nByte = dupedExprNodeSize(p, flags); + if( flags&EXPRDUP_REDUCE ){ + nByte += dupedExprSize(p->pLeft, flags) + dupedExprSize(p->pRight, flags); + } + } + return nByte; +} + +/* +** This function is similar to sqlite3ExprDup(), except that if pzBuffer +** is not NULL then *pzBuffer is assumed to point to a buffer large enough +** to store the copy of expression p, the copies of p->u.zToken +** (if applicable), and the copies of the p->pLeft and p->pRight expressions, +** if any. Before returning, *pzBuffer is set to the first byte passed the +** portion of the buffer copied into by this function. +*/ +static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ + Expr *pNew = 0; /* Value to return */ + if( p ){ + const int isReduced = (flags&EXPRDUP_REDUCE); + u8 *zAlloc; + u32 staticFlag = 0; + + assert( pzBuffer==0 || isReduced ); + + /* Figure out where to write the new Expr structure. */ + if( pzBuffer ){ + zAlloc = *pzBuffer; + staticFlag = EP_Static; + }else{ + zAlloc = sqlite3DbMallocRaw(db, dupedExprSize(p, flags)); + } + pNew = (Expr *)zAlloc; + + if( pNew ){ + /* Set nNewSize to the size allocated for the structure pointed to + ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or + ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed + ** by the copy of the p->u.zToken string (if any). + */ + const unsigned nStructSize = dupedExprStructSize(p, flags); + const int nNewSize = nStructSize & 0xfff; + int nToken; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; + }else{ + nToken = 0; + } + if( isReduced ){ + assert( ExprHasProperty(p, EP_Reduced)==0 ); + memcpy(zAlloc, p, nNewSize); + }else{ + int nSize = exprStructSize(p); + memcpy(zAlloc, p, nSize); + memset(&zAlloc[nSize], 0, EXPR_FULLSIZE-nSize); + } + + /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); + pNew->flags |= staticFlag; + + /* Copy the p->u.zToken string, if any. */ + if( nToken ){ + char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + memcpy(zToken, p->u.zToken, nToken); + } + + if( 0==((p->flags|pNew->flags) & EP_TokenOnly) ){ + /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ + if( ExprHasProperty(p, EP_xIsSelect) ){ + pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, isReduced); + }else{ + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, isReduced); + } + } + + /* Fill in pNew->pLeft and pNew->pRight. */ + if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ + zAlloc += dupedExprNodeSize(p, flags); + if( ExprHasProperty(pNew, EP_Reduced) ){ + pNew->pLeft = exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc); + pNew->pRight = exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc); + } + if( pzBuffer ){ + *pzBuffer = zAlloc; + } + }else{ + if( !ExprHasProperty(p, EP_TokenOnly) ){ + pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); + pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); + } + } + + } + } + return pNew; +} + +/* +** Create and return a deep copy of the object passed as the second +** argument. If an OOM condition is encountered, NULL is returned +** and the db->mallocFailed flag set. +*/ +#ifndef SQLITE_OMIT_CTE +static With *withDup(sqlite3 *db, With *p){ + With *pRet = 0; + if( p ){ + int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + pRet = sqlite3DbMallocZero(db, nByte); + if( pRet ){ + int i; + pRet->nCte = p->nCte; + for(i=0; inCte; i++){ + pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); + pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); + pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName); + } + } + } + return pRet; +} +#else +# define withDup(x,y) 0 +#endif + +/* +** The following group of routines make deep copies of expressions, +** expression lists, ID lists, and select statements. The copies can +** be deleted (by being passed to their respective ...Delete() routines) +** without effecting the originals. +** +** The expression list, ID, and source lists return by sqlite3ExprListDup(), +** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded +** by subsequent calls to sqlite*ListAppend() routines. +** +** Any tables that the SrcList might point to are not duplicated. +** +** The flags parameter contains a combination of the EXPRDUP_XXX flags. +** If the EXPRDUP_REDUCE flag is set, then the structure returned is a +** truncated version of the usual Expr structure that will be stored as +** part of the in-memory representation of the database schema. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ + return exprDup(db, p, flags, 0); +} +SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ + ExprList *pNew; + struct ExprList_item *pItem, *pOldItem; + int i; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + if( pNew==0 ) return 0; + pNew->nExpr = i = p->nExpr; + if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; inExpr; i+=i){} + pNew->a = pItem = sqlite3DbMallocRaw(db, i*sizeof(p->a[0]) ); + if( pItem==0 ){ + sqlite3DbFree(db, pNew); + return 0; + } + pOldItem = p->a; + for(i=0; inExpr; i++, pItem++, pOldItem++){ + Expr *pOldExpr = pOldItem->pExpr; + pItem->pExpr = sqlite3ExprDup(db, pOldExpr, flags); + pItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan); + pItem->sortOrder = pOldItem->sortOrder; + pItem->done = 0; + pItem->bSpanIsTab = pOldItem->bSpanIsTab; + pItem->u = pOldItem->u; + } + return pNew; +} + +/* +** If cursors, triggers, views and subqueries are all omitted from +** the build, then none of the following routines, except for +** sqlite3SelectDup(), can be called. sqlite3SelectDup() is sometimes +** called with a NULL argument. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) \ + || !defined(SQLITE_OMIT_SUBQUERY) +SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ + SrcList *pNew; + int i; + int nByte; + if( p==0 ) return 0; + nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); + pNew = sqlite3DbMallocRaw(db, nByte ); + if( pNew==0 ) return 0; + pNew->nSrc = pNew->nAlloc = p->nSrc; + for(i=0; inSrc; i++){ + struct SrcList_item *pNewItem = &pNew->a[i]; + struct SrcList_item *pOldItem = &p->a[i]; + Table *pTab; + pNewItem->pSchema = pOldItem->pSchema; + pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); + pNewItem->jointype = pOldItem->jointype; + pNewItem->iCursor = pOldItem->iCursor; + pNewItem->addrFillSub = pOldItem->addrFillSub; + pNewItem->regReturn = pOldItem->regReturn; + pNewItem->isCorrelated = pOldItem->isCorrelated; + pNewItem->viaCoroutine = pOldItem->viaCoroutine; + pNewItem->isRecursive = pOldItem->isRecursive; + pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); + pNewItem->notIndexed = pOldItem->notIndexed; + pNewItem->pIndex = pOldItem->pIndex; + pTab = pNewItem->pTab = pOldItem->pTab; + if( pTab ){ + pTab->nRef++; + } + pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); + pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn, flags); + pNewItem->pUsing = sqlite3IdListDup(db, pOldItem->pUsing); + pNewItem->colUsed = pOldItem->colUsed; + } + return pNew; +} +SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ + IdList *pNew; + int i; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + if( pNew==0 ) return 0; + pNew->nId = p->nId; + pNew->a = sqlite3DbMallocRaw(db, p->nId*sizeof(p->a[0]) ); + if( pNew->a==0 ){ + sqlite3DbFree(db, pNew); + return 0; + } + /* Note that because the size of the allocation for p->a[] is not + ** necessarily a power of two, sqlite3IdListAppend() may not be called + ** on the duplicate created by this function. */ + for(i=0; inId; i++){ + struct IdList_item *pNewItem = &pNew->a[i]; + struct IdList_item *pOldItem = &p->a[i]; + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->idx = pOldItem->idx; + } + return pNew; +} +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ + Select *pNew, *pPrior; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); + if( pNew==0 ) return 0; + pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); + pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); + pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); + pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); + pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); + pNew->op = p->op; + pNew->pPrior = pPrior = sqlite3SelectDup(db, p->pPrior, flags); + if( pPrior ) pPrior->pNext = pNew; + pNew->pNext = 0; + pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); + pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = p->nSelectRow; + pNew->pWith = withDup(db, p->pWith); + return pNew; +} +#else +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ + assert( p==0 ); + return 0; +} +#endif + + +/* +** Add a new element to the end of an expression list. If pList is +** initially NULL, then create a new expression list. +** +** If a memory allocation error occurs, the entire list is freed and +** NULL is returned. If non-NULL is returned, then it is guaranteed +** that the new entry was successfully appended. +*/ +SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + Expr *pExpr /* Expression to be appended. Might be NULL */ +){ + sqlite3 *db = pParse->db; + if( pList==0 ){ + pList = sqlite3DbMallocZero(db, sizeof(ExprList) ); + if( pList==0 ){ + goto no_mem; + } + pList->a = sqlite3DbMallocRaw(db, sizeof(pList->a[0])); + if( pList->a==0 ) goto no_mem; + }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ + struct ExprList_item *a; + assert( pList->nExpr>0 ); + a = sqlite3DbRealloc(db, pList->a, pList->nExpr*2*sizeof(pList->a[0])); + if( a==0 ){ + goto no_mem; + } + pList->a = a; + } + assert( pList->a!=0 ); + if( 1 ){ + struct ExprList_item *pItem = &pList->a[pList->nExpr++]; + memset(pItem, 0, sizeof(*pItem)); + pItem->pExpr = pExpr; + } + return pList; + +no_mem: + /* Avoid leaking memory if malloc has failed. */ + sqlite3ExprDelete(db, pExpr); + sqlite3ExprListDelete(db, pList); + return 0; +} + +/* +** Set the ExprList.a[].zName element of the most recently added item +** on the expression list. +** +** pList might be NULL following an OOM error. But pName should never be +** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag +** is set. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetName( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to add the span. */ + Token *pName, /* Name to be added */ + int dequote /* True to cause the name to be dequoted */ +){ + assert( pList!=0 || pParse->db->mallocFailed!=0 ); + if( pList ){ + struct ExprList_item *pItem; + assert( pList->nExpr>0 ); + pItem = &pList->a[pList->nExpr-1]; + assert( pItem->zName==0 ); + pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); + if( dequote && pItem->zName ) sqlite3Dequote(pItem->zName); + } +} + +/* +** Set the ExprList.a[].zSpan element of the most recently added item +** on the expression list. +** +** pList might be NULL following an OOM error. But pSpan should never be +** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag +** is set. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetSpan( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to add the span. */ + ExprSpan *pSpan /* The span to be added */ +){ + sqlite3 *db = pParse->db; + assert( pList!=0 || db->mallocFailed!=0 ); + if( pList ){ + struct ExprList_item *pItem = &pList->a[pList->nExpr-1]; + assert( pList->nExpr>0 ); + assert( db->mallocFailed || pItem->pExpr==pSpan->pExpr ); + sqlite3DbFree(db, pItem->zSpan); + pItem->zSpan = sqlite3DbStrNDup(db, (char*)pSpan->zStart, + (int)(pSpan->zEnd - pSpan->zStart)); + } +} + +/* +** If the expression list pEList contains more than iLimit elements, +** leave an error message in pParse. +*/ +SQLITE_PRIVATE void sqlite3ExprListCheckLength( + Parse *pParse, + ExprList *pEList, + const char *zObject +){ + int mx = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + testcase( pEList && pEList->nExpr==mx ); + testcase( pEList && pEList->nExpr==mx+1 ); + if( pEList && pEList->nExpr>mx ){ + sqlite3ErrorMsg(pParse, "too many columns in %s", zObject); + } +} + +/* +** Delete an entire expression list. +*/ +SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ + int i; + struct ExprList_item *pItem; + if( pList==0 ) return; + assert( pList->a!=0 || pList->nExpr==0 ); + for(pItem=pList->a, i=0; inExpr; i++, pItem++){ + sqlite3ExprDelete(db, pItem->pExpr); + sqlite3DbFree(db, pItem->zName); + sqlite3DbFree(db, pItem->zSpan); + } + sqlite3DbFree(db, pList->a); + sqlite3DbFree(db, pList); +} + +/* +** These routines are Walker callbacks. Walker.u.pi is a pointer +** to an integer. These routines are checking an expression to see +** if it is a constant. Set *Walker.u.pi to 0 if the expression is +** not constant. +** +** These callback routines are used to implement the following: +** +** sqlite3ExprIsConstant() +** sqlite3ExprIsConstantNotJoin() +** sqlite3ExprIsConstantOrFunction() +** +*/ +static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + + /* If pWalker->u.i is 3 then any term of the expression that comes from + ** the ON or USING clauses of a join disqualifies the expression + ** from being considered constant. */ + if( pWalker->u.i==3 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->u.i = 0; + return WRC_Abort; + } + + switch( pExpr->op ){ + /* Consider functions to be constant if all their arguments are constant + ** and either pWalker->u.i==2 or the function as the SQLITE_FUNC_CONST + ** flag. */ + case TK_FUNCTION: + if( pWalker->u.i==2 || ExprHasProperty(pExpr,EP_Constant) ){ + return WRC_Continue; + } + /* Fall through */ + case TK_ID: + case TK_COLUMN: + case TK_AGG_FUNCTION: + case TK_AGG_COLUMN: + testcase( pExpr->op==TK_ID ); + testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_AGG_FUNCTION ); + testcase( pExpr->op==TK_AGG_COLUMN ); + pWalker->u.i = 0; + return WRC_Abort; + default: + testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */ + testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */ + return WRC_Continue; + } +} +static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ + UNUSED_PARAMETER(NotUsed); + pWalker->u.i = 0; + return WRC_Abort; +} +static int exprIsConst(Expr *p, int initFlag){ + Walker w; + memset(&w, 0, sizeof(w)); + w.u.i = initFlag; + w.xExprCallback = exprNodeIsConstant; + w.xSelectCallback = selectNodeIsConstant; + sqlite3WalkExpr(&w, p); + return w.u.i; +} + +/* +** Walk an expression tree. Return 1 if the expression is constant +** and 0 if it involves variables or function calls. +** +** For the purposes of this function, a double-quoted string (ex: "abc") +** is considered a variable but a single-quoted string (ex: 'abc') is +** a constant. +*/ +SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ + return exprIsConst(p, 1); +} + +/* +** Walk an expression tree. Return 1 if the expression is constant +** that does no originate from the ON or USING clauses of a join. +** Return 0 if it involves variables or function calls or terms from +** an ON or USING clause. +*/ +SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ + return exprIsConst(p, 3); +} + +/* +** Walk an expression tree. Return 1 if the expression is constant +** or a function call with constant arguments. Return and 0 if there +** are any variables. +** +** For the purposes of this function, a double-quoted string (ex: "abc") +** is considered a variable but a single-quoted string (ex: 'abc') is +** a constant. +*/ +SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p){ + return exprIsConst(p, 2); +} + +/* +** If the expression p codes a constant integer that is small enough +** to fit in a 32-bit integer, return 1 and put the value of the integer +** in *pValue. If the expression is not an integer or if it is too big +** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +*/ +SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ + int rc = 0; + + /* If an expression is an integer literal that fits in a signed 32-bit + ** integer, then the EP_IntValue flag will have already been set */ + assert( p->op!=TK_INTEGER || (p->flags & EP_IntValue)!=0 + || sqlite3GetInt32(p->u.zToken, &rc)==0 ); + + if( p->flags & EP_IntValue ){ + *pValue = p->u.iValue; + return 1; + } + switch( p->op ){ + case TK_UPLUS: { + rc = sqlite3ExprIsInteger(p->pLeft, pValue); + break; + } + case TK_UMINUS: { + int v; + if( sqlite3ExprIsInteger(p->pLeft, &v) ){ + assert( v!=(-2147483647-1) ); + *pValue = -v; + rc = 1; + } + break; + } + default: break; + } + return rc; +} + +/* +** Return FALSE if there is no chance that the expression can be NULL. +** +** If the expression might be NULL or if the expression is too complex +** to tell return TRUE. +** +** This routine is used as an optimization, to skip OP_IsNull opcodes +** when we know that a value cannot be NULL. Hence, a false positive +** (returning TRUE when in fact the expression can never be NULL) might +** be a small performance hit but is otherwise harmless. On the other +** hand, a false negative (returning FALSE when the result could be NULL) +** will likely result in an incorrect answer. So when in doubt, return +** TRUE. +*/ +SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ + u8 op; + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: + case TK_STRING: + case TK_FLOAT: + case TK_BLOB: + return 0; + case TK_COLUMN: + assert( p->pTab!=0 ); + return p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0; + default: + return 1; + } +} + +/* +** Return TRUE if the given expression is a constant which would be +** unchanged by OP_Affinity with the affinity given in the second +** argument. +** +** This routine is used to determine if the OP_Affinity operation +** can be omitted. When in doubt return FALSE. A false negative +** is harmless. A false positive, however, can result in the wrong +** answer. +*/ +SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ + u8 op; + if( aff==SQLITE_AFF_NONE ) return 1; + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: { + return aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC; + } + case TK_FLOAT: { + return aff==SQLITE_AFF_REAL || aff==SQLITE_AFF_NUMERIC; + } + case TK_STRING: { + return aff==SQLITE_AFF_TEXT; + } + case TK_BLOB: { + return 1; + } + case TK_COLUMN: { + assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */ + return p->iColumn<0 + && (aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC); + } + default: { + return 0; + } + } +} + +/* +** Return TRUE if the given string is a row-id column name. +*/ +SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ + if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1; + if( sqlite3StrICmp(z, "ROWID")==0 ) return 1; + if( sqlite3StrICmp(z, "OID")==0 ) return 1; + return 0; +} + +/* +** Return true if we are able to the IN operator optimization on a +** query of the form +** +** x IN (SELECT ...) +** +** Where the SELECT... clause is as specified by the parameter to this +** routine. +** +** The Select object passed in has already been preprocessed and no +** errors have been found. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +static int isCandidateForInOpt(Select *p){ + SrcList *pSrc; + ExprList *pEList; + Table *pTab; + if( p==0 ) return 0; /* right-hand side of IN is SELECT */ + if( p->pPrior ) return 0; /* Not a compound SELECT */ + if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ + testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); + testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + return 0; /* No DISTINCT keyword and no aggregate functions */ + } + assert( p->pGroupBy==0 ); /* Has no GROUP BY clause */ + if( p->pLimit ) return 0; /* Has no LIMIT clause */ + assert( p->pOffset==0 ); /* No LIMIT means no OFFSET */ + if( p->pWhere ) return 0; /* Has no WHERE clause */ + pSrc = p->pSrc; + assert( pSrc!=0 ); + if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ + if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ + pTab = pSrc->a[0].pTab; + if( NEVER(pTab==0) ) return 0; + assert( pTab->pSelect==0 ); /* FROM clause is not a view */ + if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ + pEList = p->pEList; + if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ + if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */ + return 1; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +/* +** Code an OP_Once instruction and allocate space for its flag. Return the +** address of the new instruction. +*/ +SQLITE_PRIVATE int sqlite3CodeOnce(Parse *pParse){ + Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ + return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++); +} + +/* +** Generate code that checks the left-most column of index table iCur to see if +** it contains any NULL entries. Cause the register at regHasNull to be set +** to a non-NULL value if iCur contains no NULLs. Cause register regHasNull +** to be set to NULL if iCur contains one or more NULL values. +*/ +static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ + int j1; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); + j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + VdbeComment((v, "first_entry_in(%d)", iCur)); + sqlite3VdbeJumpHere(v, j1); +} + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** The argument is an IN operator with a list (not a subquery) on the +** right-hand side. Return TRUE if that list is constant. +*/ +static int sqlite3InRhsIsConstant(Expr *pIn){ + Expr *pLHS; + int res; + assert( !ExprHasProperty(pIn, EP_xIsSelect) ); + pLHS = pIn->pLeft; + pIn->pLeft = 0; + res = sqlite3ExprIsConstant(pIn); + pIn->pLeft = pLHS; + return res; +} +#endif + +/* +** This function is used by the implementation of the IN (...) operator. +** The pX parameter is the expression on the RHS of the IN operator, which +** might be either a list of expressions or a subquery. +** +** The job of this routine is to find or create a b-tree object that can +** be used either to test for membership in the RHS set or to iterate through +** all members of the RHS set, skipping duplicates. +** +** A cursor is opened on the b-tree object that is the RHS of the IN operator +** and pX->iTable is set to the index of that cursor. +** +** The returned value of this function indicates the b-tree type, as follows: +** +** IN_INDEX_ROWID - The cursor was opened on a database table. +** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. +** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. +** IN_INDEX_EPH - The cursor was opened on a specially created and +** populated epheremal table. +** IN_INDEX_NOOP - No cursor was allocated. The IN operator must be +** implemented as a sequence of comparisons. +** +** An existing b-tree might be used if the RHS expression pX is a simple +** subquery such as: +** +** SELECT FROM +** +** If the RHS of the IN operator is a list or a more complex subquery, then +** an ephemeral table might need to be generated from the RHS and then +** pX->iTable made to point to the ephermeral table instead of an +** existing table. +** +** The inFlags parameter must contain exactly one of the bits +** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP. If inFlags contains +** IN_INDEX_MEMBERSHIP, then the generated table will be used for a +** fast membership test. When the IN_INDEX_LOOP bit is set, the +** IN index will be used to loop over all values of the RHS of the +** IN operator. +** +** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate +** through the set members) then the b-tree must not contain duplicates. +** An epheremal table must be used unless the selected is guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or it +** has a UNIQUE constraint or UNIQUE index. +** +** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used +** for fast set membership tests) then an epheremal table must +** be used unless is an INTEGER PRIMARY KEY or an index can +** be found with as its left-most column. +** +** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and +** if the RHS of the IN operator is a list (not a subquery) then this +** routine might decide that creating an ephemeral b-tree for membership +** testing is too expensive and return IN_INDEX_NOOP. In that case, the +** calling routine should implement the IN operator using a sequence +** of Eq or Ne comparison operations. +** +** When the b-tree is being used for membership tests, the calling function +** might need to know whether or not the RHS side of the IN operator +** contains a NULL. If prRhsHasNull is not a NULL pointer and +** if there is any chance that the (...) might contain a NULL value at +** runtime, then a register is allocated and the register number written +** to *prRhsHasNull. If there is no chance that the (...) contains a +** NULL value, then *prRhsHasNull is left unchanged. +** +** If a register is allocated and its location stored in *prRhsHasNull, then +** the value in that register will be NULL if the b-tree contains one or more +** NULL values, and it will be some non-NULL value if the b-tree contains no +** NULL values. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ + Select *p; /* SELECT to the right of IN operator */ + int eType = 0; /* Type of RHS table. IN_INDEX_* */ + int iTab = pParse->nTab++; /* Cursor of the RHS table */ + int mustBeUnique; /* True if RHS must be unique */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ + + assert( pX->op==TK_IN ); + mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; + + /* Check to see if an existing table or index can be used to + ** satisfy the query. This is preferable to generating a new + ** ephemeral table. + */ + p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); + if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) ){ + sqlite3 *db = pParse->db; /* Database connection */ + Table *pTab; /* Table
    . */ + Expr *pExpr; /* Expression */ + i16 iCol; /* Index of column */ + i16 iDb; /* Database idx for pTab */ + + assert( p ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ + pTab = p->pSrc->a[0].pTab; + pExpr = p->pEList->a[0].pExpr; + iCol = (i16)pExpr->iColumn; + + /* Code an OP_Transaction and OP_TableLock for
    . */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + + /* This function is only called from two places. In both cases the vdbe + ** has already been allocated. So assume sqlite3GetVdbe() is always + ** successful here. + */ + assert(v); + if( iCol<0 ){ + int iAddr = sqlite3CodeOnce(pParse); + VdbeCoverage(v); + + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + eType = IN_INDEX_ROWID; + + sqlite3VdbeJumpHere(v, iAddr); + }else{ + Index *pIdx; /* Iterator variable */ + + /* The collation sequence used by the comparison. If an index is to + ** be used in place of a temp-table, it must be ordered according + ** to this collation sequence. */ + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); + + /* Check that the affinity that will be used to perform the + ** comparison is the same as the affinity of the column. If + ** it is not, it is not possible to use any index. + */ + int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); + + for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ + if( (pIdx->aiColumn[0]==iCol) + && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq + && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx))) + ){ + int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; + + if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ + *prRhsHasNull = ++pParse->nMem; + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); + } + sqlite3VdbeJumpHere(v, iAddr); + } + } + } + } + + /* If no preexisting index is available for the IN clause + ** and IN_INDEX_NOOP is an allowed reply + ** and the RHS of the IN operator is a list, not a subquery + ** and the RHS is not contant or has two or fewer terms, + ** then it is not worth creating an ephermeral table to evaluate + ** the IN operator so return IN_INDEX_NOOP. + */ + if( eType==0 + && (inFlags & IN_INDEX_NOOP_OK) + && !ExprHasProperty(pX, EP_xIsSelect) + && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + ){ + eType = IN_INDEX_NOOP; + } + + + if( eType==0 ){ + /* Could not find an existing table or index to use as the RHS b-tree. + ** We will have to generate an ephemeral table to do the job. + */ + u32 savedNQueryLoop = pParse->nQueryLoop; + int rMayHaveNull = 0; + eType = IN_INDEX_EPH; + if( inFlags & IN_INDEX_LOOP ){ + pParse->nQueryLoop = 0; + if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ + eType = IN_INDEX_ROWID; + } + }else if( prRhsHasNull ){ + *prRhsHasNull = rMayHaveNull = ++pParse->nMem; + } + sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); + pParse->nQueryLoop = savedNQueryLoop; + }else{ + pX->iTable = iTab; + } + return eType; +} +#endif + +/* +** Generate code for scalar subqueries used as a subquery expression, EXISTS, +** or IN operators. Examples: +** +** (SELECT a FROM b) -- subquery +** EXISTS (SELECT a FROM b) -- EXISTS subquery +** x IN (4,5,11) -- IN operator with list on right-hand side +** x IN (SELECT a FROM b) -- IN operator with subquery on the right +** +** The pExpr parameter describes the expression that contains the IN +** operator or subquery. +** +** If parameter isRowid is non-zero, then expression pExpr is guaranteed +** to be of the form " IN (?, ?, ?)", where is a reference +** to some integer key column of a table B-Tree. In this case, use an +** intkey B-Tree to store the set of IN(...) values instead of the usual +** (slower) variable length keys B-Tree. +** +** If rMayHaveNull is non-zero, that means that the operation is an IN +** (not a SELECT or EXISTS) and that the RHS might contains NULLs. +** All this routine does is initialize the register given by rMayHaveNull +** to NULL. Calling routines will take care of changing this register +** value to non-NULL if the RHS is NULL-free. +** +** For a SELECT or EXISTS operator, return the register that holds the +** result. For IN operators or if an error occurs, the return value is 0. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3CodeSubselect( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The IN, SELECT, or EXISTS operator */ + int rHasNullFlag, /* Register that records whether NULLs exist in RHS */ + int isRowid /* If true, LHS of IN operator is a rowid */ +){ + int jmpIfDynamic = -1; /* One-time test address */ + int rReg = 0; /* Register storing resulting */ + Vdbe *v = sqlite3GetVdbe(pParse); + if( NEVER(v==0) ) return 0; + sqlite3ExprCachePush(pParse); + + /* This code must be run in its entirety every time it is encountered + ** if any of the following is true: + ** + ** * The right-hand side is a correlated subquery + ** * The right-hand side is an expression list containing variables + ** * We are inside a trigger + ** + ** If all of the above are false, then we can run this code just once + ** save the results, and reuse the same result on subsequent invocations. + */ + if( !ExprHasProperty(pExpr, EP_VarSelect) ){ + jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } + +#ifndef SQLITE_OMIT_EXPLAIN + if( pParse->explain==2 ){ + char *zMsg = sqlite3MPrintf( + pParse->db, "EXECUTE %s%s SUBQUERY %d", jmpIfDynamic>=0?"":"CORRELATED ", + pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId + ); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + } +#endif + + switch( pExpr->op ){ + case TK_IN: { + char affinity; /* Affinity of the LHS of the IN */ + int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ + KeyInfo *pKeyInfo = 0; /* Key information */ + + affinity = sqlite3ExprAffinity(pLeft); + + /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' + ** expression it is handled the same way. An ephemeral table is + ** filled with single-field index keys representing the results + ** from the SELECT or the . + ** + ** If the 'x' expression is a column value, or the SELECT... + ** statement returns a column value, then the affinity of that + ** column is used to build the index keys. If both 'x' and the + ** SELECT... statement are columns, then numeric affinity is used + ** if either column has NUMERIC or INTEGER affinity. If neither + ** 'x' nor the SELECT... statement are columns, then numeric affinity + ** is used. + */ + pExpr->iTable = pParse->nTab++; + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); + pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, 1, 1); + + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* Case 1: expr IN (SELECT ...) + ** + ** Generate code to write the results of the select into the temporary + ** table allocated and opened above. + */ + Select *pSelect = pExpr->x.pSelect; + SelectDest dest; + ExprList *pEList; + + assert( !isRowid ); + sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); + dest.affSdst = (u8)affinity; + assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + pSelect->selFlags &= ~SF_Distinct; + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + if( sqlite3Select(pParse, pSelect, &dest) ){ + sqlite3KeyInfoUnref(pKeyInfo); + return 0; + } + pEList = pSelect->pEList; + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + pKeyInfo->aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, + pEList->a[0].pExpr); + }else if( ALWAYS(pExpr->x.pList!=0) ){ + /* Case 2: expr IN (exprlist) + ** + ** For each expression, build an index key from the evaluation and + ** store it in the temporary table. If is a column, then use + ** that columns affinity when building index keys. If is not + ** a column, use numeric affinity. + */ + int i; + ExprList *pList = pExpr->x.pList; + struct ExprList_item *pItem; + int r1, r2, r3; + + if( !affinity ){ + affinity = SQLITE_AFF_NONE; + } + if( pKeyInfo ){ + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + } + + /* Loop through each expression in . */ + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempReg(pParse); + if( isRowid ) sqlite3VdbeAddOp2(v, OP_Null, 0, r2); + for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ + Expr *pE2 = pItem->pExpr; + int iValToIns; + + /* If the expression is not constant then we will need to + ** disable the test that was generated above that makes sure + ** this code only executes once. Because for a non-constant + ** expression we need to rerun this code each time. + */ + if( jmpIfDynamic>=0 && !sqlite3ExprIsConstant(pE2) ){ + sqlite3VdbeChangeToNoop(v, jmpIfDynamic); + jmpIfDynamic = -1; + } + + /* Evaluate the expression and insert it into the temp table */ + if( isRowid && sqlite3ExprIsInteger(pE2, &iValToIns) ){ + sqlite3VdbeAddOp3(v, OP_InsertInt, pExpr->iTable, r2, iValToIns); + }else{ + r3 = sqlite3ExprCodeTarget(pParse, pE2, r1); + if( isRowid ){ + sqlite3VdbeAddOp2(v, OP_MustBeInt, r3, + sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3); + }else{ + sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, r3, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pExpr->iTable, r2); + } + } + } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); + } + break; + } + + case TK_EXISTS: + case TK_SELECT: + default: { + /* If this has to be a scalar SELECT. Generate code to put the + ** value of this select in a memory cell and record the number + ** of the memory cell in iColumn. If this is an EXISTS, write + ** an integer 0 (not exists) or 1 (exists) into a memory cell + ** and record that memory cell in iColumn. + */ + Select *pSel; /* SELECT statement to encode */ + SelectDest dest; /* How to deal with SELECt result */ + + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); + assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); + + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + pSel = pExpr->x.pSelect; + sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); + if( pExpr->op==TK_SELECT ){ + dest.eDest = SRT_Mem; + sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); + VdbeComment((v, "Init subquery result")); + }else{ + dest.eDest = SRT_Exists; + sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); + VdbeComment((v, "Init EXISTS result")); + } + sqlite3ExprDelete(pParse->db, pSel->pLimit); + pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, + &sqlite3IntTokens[1]); + pSel->iLimit = 0; + if( sqlite3Select(pParse, pSel, &dest) ){ + return 0; + } + rReg = dest.iSDParm; + ExprSetVVAProperty(pExpr, EP_NoReduce); + break; + } + } + + if( rHasNullFlag ){ + sqlite3SetHasNullFlag(v, pExpr->iTable, rHasNullFlag); + } + + if( jmpIfDynamic>=0 ){ + sqlite3VdbeJumpHere(v, jmpIfDynamic); + } + sqlite3ExprCachePop(pParse); + + return rReg; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code for an IN expression. +** +** x IN (SELECT ...) +** x IN (value, value, ...) +** +** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS) +** is an array of zero or more values. The expression is true if the LHS is +** contained within the RHS. The value of the expression is unknown (NULL) +** if the LHS is NULL or if the LHS is not contained within the RHS and the +** RHS contains one or more NULL values. +** +** This routine generates code that jumps to destIfFalse if the LHS is not +** contained within the RHS. If due to NULLs we cannot determine if the LHS +** is contained in the RHS then jump to destIfNull. If the LHS is contained +** within the RHS then fall through. +*/ +static void sqlite3ExprCodeIN( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The IN expression */ + int destIfFalse, /* Jump here if LHS is not contained in the RHS */ + int destIfNull /* Jump here if the results are unknown due to NULLs */ +){ + int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ + char affinity; /* Comparison affinity to use */ + int eType; /* Type of the RHS */ + int r1; /* Temporary use register */ + Vdbe *v; /* Statement under construction */ + + /* Compute the RHS. After this step, the table with cursor + ** pExpr->iTable will contains the values that make up the RHS. + */ + v = pParse->pVdbe; + assert( v!=0 ); /* OOM detected prior to this routine */ + VdbeNoopComment((v, "begin IN expr")); + eType = sqlite3FindInIndex(pParse, pExpr, + IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, + destIfFalse==destIfNull ? 0 : &rRhsHasNull); + + /* Figure out the affinity to use to create a key from the results + ** of the expression. affinityStr stores a static string suitable for + ** P4 of OP_MakeRecord. + */ + affinity = comparisonAffinity(pExpr); + + /* Code the LHS, the from " IN (...)". + */ + sqlite3ExprCachePush(pParse); + r1 = sqlite3GetTempReg(pParse); + sqlite3ExprCode(pParse, pExpr->pLeft, r1); + + /* If sqlite3FindInIndex() did not find or create an index that is + ** suitable for evaluating the IN operator, then evaluate using a + ** sequence of comparisons. + */ + if( eType==IN_INDEX_NOOP ){ + ExprList *pList = pExpr->x.pList; + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + int labelOk = sqlite3VdbeMakeLabel(v); + int r2, regToFree; + int regCkNull = 0; + int ii; + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + if( destIfNull!=destIfFalse ){ + regCkNull = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull); + } + for(ii=0; iinExpr; ii++){ + r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); + if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){ + sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); + } + if( iinExpr-1 || destIfNull!=destIfFalse ){ + sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, iinExpr-1); + VdbeCoverageIf(v, ii==pList->nExpr-1); + sqlite3VdbeChangeP5(v, affinity); + }else{ + assert( destIfNull==destIfFalse ); + sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2, + (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL); + } + sqlite3ReleaseTempReg(pParse, regToFree); + } + if( regCkNull ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + } + sqlite3VdbeResolveLabel(v, labelOk); + sqlite3ReleaseTempReg(pParse, regCkNull); + }else{ + + /* If the LHS is NULL, then the result is either false or NULL depending + ** on whether the RHS is empty or not, respectively. + */ + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + if( destIfNull==destIfFalse ){ + /* Shortcut for the common case where the false and NULL outcomes are + ** the same. */ + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); + }else{ + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeJumpHere(v, addr1); + } + } + + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree + */ + sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + VdbeCoverage(v); + }else{ + /* In this case, the RHS is an index b-tree. + */ + sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); + + /* If the set membership test fails, then the result of the + ** "x IN (...)" expression must be either 0 or NULL. If the set + ** contains no NULL values, then the result is 0. If the set + ** contains one or more NULL values, then the result of the + ** expression is also NULL. + */ + assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); + if( rRhsHasNull==0 ){ + /* This branch runs if it is known at compile time that the RHS + ** cannot contain NULL values. This happens as the result + ** of a "NOT NULL" constraint in the database schema. + ** + ** Also run this branch if NULL is equivalent to FALSE + ** for this particular IN operator. + */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); + VdbeCoverage(v); + }else{ + /* In this branch, the RHS of the IN might contain a NULL and + ** the presence of a NULL on the RHS makes a difference in the + ** outcome. + */ + int j1; + + /* First check to see if the LHS is contained in the RHS. If so, + ** then the answer is TRUE the presence of NULLs in the RHS does + ** not matter. If the LHS is not contained in the RHS, then the + ** answer is NULL if the RHS contains NULLs and the answer is + ** FALSE if the RHS is NULL-free. + */ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeJumpHere(v, j1); + } + } + } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ExprCachePop(pParse); + VdbeComment((v, "end IN expr")); +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +/* +** Duplicate an 8-byte value +*/ +static char *dup8bytes(Vdbe *v, const char *in){ + char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8); + if( out ){ + memcpy(out, in, 8); + } + return out; +} + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Generate an instruction that will put the floating point +** value described by z[0..n-1] into register iMem. +** +** The z[] string will probably not be zero-terminated. But the +** z[n] character is guaranteed to be something that does not look +** like the continuation of the number. +*/ +static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ + if( ALWAYS(z!=0) ){ + double value; + char *zV; + sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ + if( negateFlag ) value = -value; + zV = dup8bytes(v, (char*)&value); + sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); + } +} +#endif + + +/* +** Generate an instruction that will put the integer describe by +** text z[0..n-1] into register iMem. +** +** Expr.u.zToken is always UTF8 and zero-terminated. +*/ +static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ + Vdbe *v = pParse->pVdbe; + if( pExpr->flags & EP_IntValue ){ + int i = pExpr->u.iValue; + assert( i>=0 ); + if( negFlag ) i = -i; + sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); + }else{ + int c; + i64 value; + const char *z = pExpr->u.zToken; + assert( z!=0 ); + c = sqlite3DecOrHexToI64(z, &value); + if( c==0 || (c==2 && negFlag) ){ + char *zV; + if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } + zV = dup8bytes(v, (char*)&value); + sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); + }else{ +#ifdef SQLITE_OMIT_FLOATING_POINT + sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); +#else +#ifndef SQLITE_OMIT_HEX_INTEGER + if( sqlite3_strnicmp(z,"0x",2)==0 ){ + sqlite3ErrorMsg(pParse, "hex literal too big: %s", z); + }else +#endif + { + codeReal(v, z, negFlag, iMem); + } +#endif + } + } +} + +/* +** Clear a cache entry. +*/ +static void cacheEntryClear(Parse *pParse, struct yColCache *p){ + if( p->tempReg ){ + if( pParse->nTempRegaTempReg) ){ + pParse->aTempReg[pParse->nTempReg++] = p->iReg; + } + p->tempReg = 0; + } +} + + +/* +** Record in the column cache that a particular column from a +** particular table is stored in a particular register. +*/ +SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ + int i; + int minLru; + int idxLru; + struct yColCache *p; + + assert( iReg>0 ); /* Register numbers are always positive */ + assert( iCol>=-1 && iCol<32768 ); /* Finite column numbers */ + + /* The SQLITE_ColumnCache flag disables the column cache. This is used + ** for testing only - to verify that SQLite always gets the same answer + ** with and without the column cache. + */ + if( OptimizationDisabled(pParse->db, SQLITE_ColumnCache) ) return; + + /* First replace any existing entry. + ** + ** Actually, the way the column cache is currently used, we are guaranteed + ** that the object will never already be in cache. Verify this guarantee. + */ +#ifndef NDEBUG + for(i=0, p=pParse->aColCache; iiReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); + } +#endif + + /* Find an empty slot and replace it */ + for(i=0, p=pParse->aColCache; iiReg==0 ){ + p->iLevel = pParse->iCacheLevel; + p->iTable = iTab; + p->iColumn = iCol; + p->iReg = iReg; + p->tempReg = 0; + p->lru = pParse->iCacheCnt++; + return; + } + } + + /* Replace the last recently used */ + minLru = 0x7fffffff; + idxLru = -1; + for(i=0, p=pParse->aColCache; ilrulru; + } + } + if( ALWAYS(idxLru>=0) ){ + p = &pParse->aColCache[idxLru]; + p->iLevel = pParse->iCacheLevel; + p->iTable = iTab; + p->iColumn = iCol; + p->iReg = iReg; + p->tempReg = 0; + p->lru = pParse->iCacheCnt++; + return; + } +} + +/* +** Indicate that registers between iReg..iReg+nReg-1 are being overwritten. +** Purge the range of registers from the column cache. +*/ +SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ + int i; + int iLast = iReg + nReg - 1; + struct yColCache *p; + for(i=0, p=pParse->aColCache; iiReg; + if( r>=iReg && r<=iLast ){ + cacheEntryClear(pParse, p); + p->iReg = 0; + } + } +} + +/* +** Remember the current column cache context. Any new entries added +** added to the column cache after this call are removed when the +** corresponding pop occurs. +*/ +SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ + pParse->iCacheLevel++; +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("PUSH to %d\n", pParse->iCacheLevel); + } +#endif +} + +/* +** Remove from the column cache any entries that were added since the +** the previous sqlite3ExprCachePush operation. In other words, restore +** the cache to the state it was in prior the most recent Push. +*/ +SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ + int i; + struct yColCache *p; + assert( pParse->iCacheLevel>=1 ); + pParse->iCacheLevel--; +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("POP to %d\n", pParse->iCacheLevel); + } +#endif + for(i=0, p=pParse->aColCache; iiReg && p->iLevel>pParse->iCacheLevel ){ + cacheEntryClear(pParse, p); + p->iReg = 0; + } + } +} + +/* +** When a cached column is reused, make sure that its register is +** no longer available as a temp register. ticket #3879: that same +** register might be in the cache in multiple places, so be sure to +** get them all. +*/ +static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ + int i; + struct yColCache *p; + for(i=0, p=pParse->aColCache; iiReg==iReg ){ + p->tempReg = 0; + } + } +} + +/* +** Generate code to extract the value of the iCol-th column of a table. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( + Vdbe *v, /* The VDBE under construction */ + Table *pTab, /* The table containing the value */ + int iTabCur, /* The table cursor. Or the PK cursor for WITHOUT ROWID */ + int iCol, /* Index of the column to extract */ + int regOut /* Extract the value into this register */ +){ + if( iCol<0 || iCol==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); + }else{ + int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; + int x = iCol; + if( !HasRowid(pTab) ){ + x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + } + sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); + } + if( iCol>=0 ){ + sqlite3ColumnDefault(v, pTab, iCol, regOut); + } +} + +/* +** Generate code that will extract the iColumn-th column from +** table pTab and store the column value in a register. An effort +** is made to store the column value in register iReg, but this is +** not guaranteed. The location of the column value is returned. +** +** There must be an open cursor to pTab in iTable when this routine +** is called. If iColumn<0 then code is generated that extracts the rowid. +*/ +SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Description of the table we are reading from */ + int iColumn, /* Index of the table column */ + int iTable, /* The cursor pointing to the table */ + int iReg, /* Store results here */ + u8 p5 /* P5 value for OP_Column */ +){ + Vdbe *v = pParse->pVdbe; + int i; + struct yColCache *p; + + for(i=0, p=pParse->aColCache; iiReg>0 && p->iTable==iTable && p->iColumn==iColumn ){ + p->lru = pParse->iCacheCnt++; + sqlite3ExprCachePinRegister(pParse, p->iReg); + return p->iReg; + } + } + assert( v!=0 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); + if( p5 ){ + sqlite3VdbeChangeP5(v, p5); + }else{ + sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + } + return iReg; +} + +/* +** Clear all column cache entries. +*/ +SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse *pParse){ + int i; + struct yColCache *p; + +#if SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("CLEAR\n"); + } +#endif + for(i=0, p=pParse->aColCache; iiReg ){ + cacheEntryClear(pParse, p); + p->iReg = 0; + } + } +} + +/* +** Record the fact that an affinity change has occurred on iCount +** registers starting with iStart. +*/ +SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ + sqlite3ExprCacheRemove(pParse, iStart, iCount); +} + +/* +** Generate code to move content from registers iFrom...iFrom+nReg-1 +** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ + int i; + struct yColCache *p; + assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); + for(i=0, p=pParse->aColCache; iiReg; + if( x>=iFrom && xiReg += iTo-iFrom; + } + } +} + +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +/* +** Return true if any register in the range iFrom..iTo (inclusive) +** is used as part of the column cache. +** +** This routine is used within assert() and testcase() macros only +** and does not appear in a normal build. +*/ +static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ + int i; + struct yColCache *p; + for(i=0, p=pParse->aColCache; iiReg; + if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ + } + return 0; +} +#endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ + +/* +** Convert an expression node to a TK_REGISTER +*/ +static void exprToRegister(Expr *p, int iReg){ + p->op2 = p->op; + p->op = TK_REGISTER; + p->iTable = iReg; + ExprClearProperty(p, EP_Skip); +} + +/* +** Generate code into the current Vdbe to evaluate the given +** expression. Attempt to store the results in register "target". +** Return the register where results are stored. +** +** With this routine, there is no guarantee that results will +** be stored in target. The result might be stored in some other +** register if it is convenient to do so. The calling function +** must check the return code and move the results to the desired +** register. +*/ +SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ + Vdbe *v = pParse->pVdbe; /* The VM under construction */ + int op; /* The opcode being coded */ + int inReg = target; /* Results stored in register inReg */ + int regFree1 = 0; /* If non-zero free this temporary register */ + int regFree2 = 0; /* If non-zero free this temporary register */ + int r1, r2, r3, r4; /* Various register numbers */ + sqlite3 *db = pParse->db; /* The database connection */ + Expr tempX; /* Temporary expression node */ + + assert( target>0 && target<=pParse->nMem ); + if( v==0 ){ + assert( pParse->db->mallocFailed ); + return 0; + } + + if( pExpr==0 ){ + op = TK_NULL; + }else{ + op = pExpr->op; + } + switch( op ){ + case TK_AGG_COLUMN: { + AggInfo *pAggInfo = pExpr->pAggInfo; + struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; + if( !pAggInfo->directMode ){ + assert( pCol->iMem>0 ); + inReg = pCol->iMem; + break; + }else if( pAggInfo->useSortingIdx ){ + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, + pCol->iSorterColumn, target); + break; + } + /* Otherwise, fall thru into the TK_COLUMN case */ + } + case TK_COLUMN: { + int iTab = pExpr->iTable; + if( iTab<0 ){ + if( pParse->ckBase>0 ){ + /* Generating CHECK constraints or inserting into partial index */ + inReg = pExpr->iColumn + pParse->ckBase; + break; + }else{ + /* Deleting from a partial index */ + iTab = pParse->iPartIdxTab; + } + } + inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + pExpr->iColumn, iTab, target, + pExpr->op2); + break; + } + case TK_INTEGER: { + codeInteger(pParse, pExpr, 0, target); + break; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + codeReal(v, pExpr->u.zToken, 0, target); + break; + } +#endif + case TK_STRING: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0, pExpr->u.zToken, 0); + break; + } + case TK_NULL: { + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + int n; + const char *z; + char *zBlob; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); + assert( pExpr->u.zToken[1]=='\'' ); + z = &pExpr->u.zToken[2]; + n = sqlite3Strlen30(z) - 1; + assert( z[n]=='\'' ); + zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); + sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); + break; + } +#endif + case TK_VARIABLE: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken!=0 ); + assert( pExpr->u.zToken[0]!=0 ); + sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); + if( pExpr->u.zToken[1]!=0 ){ + assert( pExpr->u.zToken[0]=='?' + || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); + sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); + } + break; + } + case TK_REGISTER: { + inReg = pExpr->iTable; + break; + } + case TK_AS: { + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + break; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + int aff, to_op; + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + aff = sqlite3AffinityType(pExpr->u.zToken, 0); + to_op = aff - SQLITE_AFF_TEXT + OP_ToText; + assert( to_op==OP_ToText || aff!=SQLITE_AFF_TEXT ); + assert( to_op==OP_ToBlob || aff!=SQLITE_AFF_NONE ); + assert( to_op==OP_ToNumeric || aff!=SQLITE_AFF_NUMERIC ); + assert( to_op==OP_ToInt || aff!=SQLITE_AFF_INTEGER ); + assert( to_op==OP_ToReal || aff!=SQLITE_AFF_REAL ); + testcase( to_op==OP_ToText ); + testcase( to_op==OP_ToBlob ); + testcase( to_op==OP_ToNumeric ); + testcase( to_op==OP_ToInt ); + testcase( to_op==OP_ToReal ); + if( inReg!=target ){ + sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); + inReg = target; + } + sqlite3VdbeAddOp1(v, to_op, inReg); + testcase( usedAsColumnCache(pParse, inReg, inReg) ); + sqlite3ExprCacheAffinityChange(pParse, inReg, 1); + break; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_IS: + case TK_ISNOT: { + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + op = (op==TK_IS) ? TK_EQ : TK_NE; + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_AND: + case TK_OR: + case TK_PLUS: + case TK_STAR: + case TK_MINUS: + case TK_REM: + case TK_BITAND: + case TK_BITOR: + case TK_SLASH: + case TK_LSHIFT: + case TK_RSHIFT: + case TK_CONCAT: { + assert( TK_AND==OP_And ); testcase( op==TK_AND ); + assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + assert( TK_PLUS==OP_Add ); testcase( op==TK_PLUS ); + assert( TK_MINUS==OP_Subtract ); testcase( op==TK_MINUS ); + assert( TK_REM==OP_Remainder ); testcase( op==TK_REM ); + assert( TK_BITAND==OP_BitAnd ); testcase( op==TK_BITAND ); + assert( TK_BITOR==OP_BitOr ); testcase( op==TK_BITOR ); + assert( TK_SLASH==OP_Divide ); testcase( op==TK_SLASH ); + assert( TK_LSHIFT==OP_ShiftLeft ); testcase( op==TK_LSHIFT ); + assert( TK_RSHIFT==OP_ShiftRight ); testcase( op==TK_RSHIFT ); + assert( TK_CONCAT==OP_Concat ); testcase( op==TK_CONCAT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + sqlite3VdbeAddOp3(v, op, r2, r1, target); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_UMINUS: { + Expr *pLeft = pExpr->pLeft; + assert( pLeft ); + if( pLeft->op==TK_INTEGER ){ + codeInteger(pParse, pLeft, 1, target); +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( pLeft->op==TK_FLOAT ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + codeReal(v, pLeft->u.zToken, 1, target); +#endif + }else{ + tempX.op = TK_INTEGER; + tempX.flags = EP_IntValue|EP_TokenOnly; + tempX.u.iValue = 0; + r1 = sqlite3ExprCodeTemp(pParse, &tempX, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free2); + sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); + testcase( regFree2==0 ); + } + inReg = target; + break; + } + case TK_BITNOT: + case TK_NOT: { + assert( TK_BITNOT==OP_BitNot ); testcase( op==TK_BITNOT ); + assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + inReg = target; + sqlite3VdbeAddOp2(v, op, r1, inReg); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + int addr; + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + addr = sqlite3VdbeAddOp1(v, op, r1); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); + sqlite3VdbeAddOp2(v, OP_Integer, 0, target); + sqlite3VdbeJumpHere(v, addr); + break; + } + case TK_AGG_FUNCTION: { + AggInfo *pInfo = pExpr->pAggInfo; + if( pInfo==0 ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); + }else{ + inReg = pInfo->aFunc[pExpr->iAgg].iMem; + } + break; + } + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + int nFarg; /* Number of function arguments */ + FuncDef *pDef; /* The function definition object */ + int nId; /* Length of the function name in bytes */ + const char *zId; /* The function name */ + u32 constMask = 0; /* Mask of function arguments that are constant */ + int i; /* Loop counter */ + u8 enc = ENC(db); /* The text encoding used by this database */ + CollSeq *pColl = 0; /* A collating sequence */ + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + if( ExprHasProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + }else{ + pFarg = pExpr->x.pList; + } + nFarg = pFarg ? pFarg->nExpr : 0; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + zId = pExpr->u.zToken; + nId = sqlite3Strlen30(zId); + pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); + if( pDef==0 || pDef->xFunc==0 ){ + sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); + break; + } + + /* Attempt a direct implementation of the built-in COALESCE() and + ** IFNULL() functions. This avoids unnecessary evalation of + ** arguments past the first non-NULL argument. + */ + if( pDef->funcFlags & SQLITE_FUNC_COALESCE ){ + int endCoalesce = sqlite3VdbeMakeLabel(v); + assert( nFarg>=2 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + for(i=1; ia[i].pExpr, target); + sqlite3ExprCachePop(pParse); + } + sqlite3VdbeResolveLabel(v, endCoalesce); + break; + } + + /* The UNLIKELY() function is a no-op. The result is the value + ** of the first argument. + */ + if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ + assert( nFarg>=1 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + break; + } + + for(i=0; ia[i].pExpr) ){ + testcase( i==31 ); + constMask |= MASKBIT32(i); + } + if( (pDef->funcFlags & SQLITE_FUNC_NEEDCOLL)!=0 && !pColl ){ + pColl = sqlite3ExprCollSeq(pParse, pFarg->a[i].pExpr); + } + } + if( pFarg ){ + if( constMask ){ + r1 = pParse->nMem+1; + pParse->nMem += nFarg; + }else{ + r1 = sqlite3GetTempRange(pParse, nFarg); + } + + /* For length() and typeof() functions with a column argument, + ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG + ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data + ** loading. + */ + if( (pDef->funcFlags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + u8 exprOp; + assert( nFarg==1 ); + assert( pFarg->a[0].pExpr!=0 ); + exprOp = pFarg->a[0].pExpr->op; + if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ + assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); + assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); + testcase( pDef->funcFlags & OPFLAG_LENGTHARG ); + pFarg->a[0].pExpr->op2 = + pDef->funcFlags & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG); + } + } + + sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ + sqlite3ExprCodeExprList(pParse, pFarg, r1, + SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); + sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ + }else{ + r1 = 0; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Possibly overload the function if the first argument is + ** a virtual table column. + ** + ** For infix functions (LIKE, GLOB, REGEXP, and MATCH) use the + ** second argument, not the first, as the argument to test to + ** see if it is a column in a virtual table. This is done because + ** the left operand of infix functions (the operand we want to + ** control overloading) ends up as the second argument to the + ** function. The expression "A glob B" is equivalent to + ** "glob(B,A). We want to use the A in "A glob B" to test + ** for function overloading. But we use the B term in "glob(B,A)". + */ + if( nFarg>=2 && (pExpr->flags & EP_InfixFunc) ){ + pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[1].pExpr); + }else if( nFarg>0 ){ + pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[0].pExpr); + } +#endif + if( pDef->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + if( !pColl ) pColl = db->pDfltColl; + sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp4(v, OP_Function, constMask, r1, target, + (char*)pDef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nFarg); + if( nFarg && constMask==0 ){ + sqlite3ReleaseTempRange(pParse, r1, nFarg); + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: + case TK_SELECT: { + testcase( op==TK_EXISTS ); + testcase( op==TK_SELECT ); + inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); + break; + } + case TK_IN: { + int destIfFalse = sqlite3VdbeMakeLabel(v); + int destIfNull = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + sqlite3VdbeResolveLabel(v, destIfFalse); + sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); + sqlite3VdbeResolveLabel(v, destIfNull); + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pLeft = pExpr->pLeft; + struct ExprList_item *pLItem = pExpr->x.pList->a; + Expr *pRight = pLItem->pExpr; + + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + r3 = sqlite3GetTempReg(pParse); + r4 = sqlite3GetTempReg(pParse); + codeCompare(pParse, pLeft, pRight, OP_Ge, + r1, r2, r3, SQLITE_STOREP2); VdbeCoverage(v); + pLItem++; + pRight = pLItem->pExpr; + sqlite3ReleaseTempReg(pParse, regFree2); + r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); + testcase( regFree2==0 ); + codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); + sqlite3ReleaseTempReg(pParse, r3); + sqlite3ReleaseTempReg(pParse, r4); + break; + } + case TK_COLLATE: + case TK_UPLUS: { + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + break; + } + + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + ** + ** The expression is implemented using an OP_Param opcode. The p1 + ** parameter is set to 0 for an old.rowid reference, or to (i+1) + ** to reference another column of the old.* pseudo-table, where + ** i is the index of the column. For a new.rowid reference, p1 is + ** set to (n+1), where n is the number of columns in each pseudo-table. + ** For a reference to any other column in the new.* pseudo-table, p1 + ** is set to (n+2+i), where n and i are as defined previously. For + ** example, if the table on which triggers are being fired is + ** declared as: + ** + ** CREATE TABLE t1(a, b); + ** + ** Then p1 is interpreted as follows: + ** + ** p1==0 -> old.rowid p1==3 -> new.rowid + ** p1==1 -> old.a p1==4 -> new.a + ** p1==2 -> old.b p1==5 -> new.b + */ + Table *pTab = pExpr->pTab; + int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + pExpr->iColumn; + + assert( pExpr->iTable==0 || pExpr->iTable==1 ); + assert( pExpr->iColumn>=-1 && pExpr->iColumnnCol ); + assert( pTab->iPKey<0 || pExpr->iColumn!=pTab->iPKey ); + assert( p1>=0 && p1<(pTab->nCol*2+2) ); + + sqlite3VdbeAddOp2(v, OP_Param, p1, target); + VdbeComment((v, "%s.%s -> $%d", + (pExpr->iTable ? "new" : "old"), + (pExpr->iColumn<0 ? "rowid" : pExpr->pTab->aCol[pExpr->iColumn].zName), + target + )); + +#ifndef SQLITE_OMIT_FLOATING_POINT + /* If the column has REAL affinity, it may currently be stored as an + ** integer. Use OP_RealAffinity to make sure it is really real. */ + if( pExpr->iColumn>=0 + && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL + ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + } +#endif + break; + } + + + /* + ** Form A: + ** CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form B: + ** CASE WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form A is can be transformed into the equivalent form B as follows: + ** CASE WHEN x=e1 THEN r1 WHEN x=e2 THEN r2 ... + ** WHEN x=eN THEN rN ELSE y END + ** + ** X (if it exists) is in pExpr->pLeft. + ** Y is in the last element of pExpr->x.pList if pExpr->x.pList->nExpr is + ** odd. The Y is also optional. If the number of elements in x.pList + ** is even, then Y is omitted and the "otherwise" result is NULL. + ** Ei is in pExpr->pList->a[i*2] and Ri is pExpr->pList->a[i*2+1]. + ** + ** The result of the expression is the Ri for the first matching Ei, + ** or if there is no matching Ei, the ELSE term Y, or if there is + ** no ELSE term, NULL. + */ + default: assert( op==TK_CASE ); { + int endLabel; /* GOTO label for end of CASE stmt */ + int nextCase; /* GOTO label for next WHEN clause */ + int nExpr; /* 2x number of WHEN terms */ + int i; /* Loop counter */ + ExprList *pEList; /* List of WHEN terms */ + struct ExprList_item *aListelem; /* Array of WHEN terms */ + Expr opCompare; /* The X==Ei expression */ + Expr *pX; /* The X expression */ + Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ + VVA_ONLY( int iCacheLevel = pParse->iCacheLevel; ) + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); + assert(pExpr->x.pList->nExpr > 0); + pEList = pExpr->x.pList; + aListelem = pEList->a; + nExpr = pEList->nExpr; + endLabel = sqlite3VdbeMakeLabel(v); + if( (pX = pExpr->pLeft)!=0 ){ + tempX = *pX; + testcase( pX->op==TK_COLUMN ); + exprToRegister(&tempX, sqlite3ExprCodeTemp(pParse, pX, ®Free1)); + testcase( regFree1==0 ); + opCompare.op = TK_EQ; + opCompare.pLeft = &tempX; + pTest = &opCompare; + /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: + ** The value in regFree1 might get SCopy-ed into the file result. + ** So make sure that the regFree1 register is not reused for other + ** purposes and possibly overwritten. */ + regFree1 = 0; + } + for(i=0; iop==TK_COLUMN ); + sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); + testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); + sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); + sqlite3ExprCachePop(pParse); + sqlite3VdbeResolveLabel(v, nextCase); + } + if( (nExpr&1)!=0 ){ + sqlite3ExprCachePush(pParse); + sqlite3ExprCode(pParse, pEList->a[nExpr-1].pExpr, target); + sqlite3ExprCachePop(pParse); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + assert( db->mallocFailed || pParse->nErr>0 + || pParse->iCacheLevel==iCacheLevel ); + sqlite3VdbeResolveLabel(v, endLabel); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + assert( pExpr->affinity==OE_Rollback + || pExpr->affinity==OE_Abort + || pExpr->affinity==OE_Fail + || pExpr->affinity==OE_Ignore + ); + if( !pParse->pTriggerTab ){ + sqlite3ErrorMsg(pParse, + "RAISE() may only be used within a trigger-program"); + return 0; + } + if( pExpr->affinity==OE_Abort ){ + sqlite3MayAbort(pParse); + } + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + if( pExpr->affinity==OE_Ignore ){ + sqlite3VdbeAddOp4( + v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + VdbeCoverage(v); + }else{ + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, + pExpr->affinity, pExpr->u.zToken, 0, 0); + } + + break; + } +#endif + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); + return inReg; +} + +/* +** Factor out the code of the given expression to initialization time. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeAtInit( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The expression to code when the VDBE initializes */ + int regDest, /* Store the value in this register */ + u8 reusable /* True if this expression is reusable */ +){ + ExprList *p; + assert( ConstFactorOk(pParse) ); + p = pParse->pConstExpr; + pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + p = sqlite3ExprListAppend(pParse, p, pExpr); + if( p ){ + struct ExprList_item *pItem = &p->a[p->nExpr-1]; + pItem->u.iConstExprReg = regDest; + pItem->reusable = reusable; + } + pParse->pConstExpr = p; +} + +/* +** Generate code to evaluate an expression and store the results +** into a register. Return the register number where the results +** are stored. +** +** If the register is a temporary register that can be deallocated, +** then write its number into *pReg. If the result register is not +** a temporary, then set *pReg to zero. +** +** If pExpr is a constant, then this routine might generate this +** code to fill the register in the initialization section of the +** VDBE program, in order to factor it out of the evaluation loop. +*/ +SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ + int r2; + pExpr = sqlite3ExprSkipCollate(pExpr); + if( ConstFactorOk(pParse) + && pExpr->op!=TK_REGISTER + && sqlite3ExprIsConstantNotJoin(pExpr) + ){ + ExprList *p = pParse->pConstExpr; + int i; + *pReg = 0; + if( p ){ + struct ExprList_item *pItem; + for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){ + if( pItem->reusable && sqlite3ExprCompare(pItem->pExpr,pExpr,-1)==0 ){ + return pItem->u.iConstExprReg; + } + } + } + r2 = ++pParse->nMem; + sqlite3ExprCodeAtInit(pParse, pExpr, r2, 1); + }else{ + int r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + if( r2==r1 ){ + *pReg = r1; + }else{ + sqlite3ReleaseTempReg(pParse, r1); + *pReg = 0; + } + } + return r2; +} + +/* +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. +*/ +SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ + int inReg; + + assert( target>0 && target<=pParse->nMem ); + if( pExpr && pExpr->op==TK_REGISTER ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); + }else{ + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + assert( pParse->pVdbe || pParse->db->mallocFailed ); + if( inReg!=target && pParse->pVdbe ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + } + } +} + +/* +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. If the expression is constant, then this routine +** might choose to code the expression at initialization time. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ + if( pParse->okConstFactor && sqlite3ExprIsConstant(pExpr) ){ + sqlite3ExprCodeAtInit(pParse, pExpr, target, 0); + }else{ + sqlite3ExprCode(pParse, pExpr, target); + } +} + +/* +** Generate code that evalutes the given expression and puts the result +** in register target. +** +** Also make a copy of the expression results into another "cache" register +** and modify the expression so that the next time it is evaluated, +** the result is a copy of the cache register. +** +** This routine is used for expressions that are used multiple +** times. They are evaluated once and the results of the expression +** are reused. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ + Vdbe *v = pParse->pVdbe; + int iMem; + + assert( target>0 ); + assert( pExpr->op!=TK_REGISTER ); + sqlite3ExprCode(pParse, pExpr, target); + iMem = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, target, iMem); + exprToRegister(pExpr, iMem); +} + +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +/* +** Generate a human-readable explanation of an expression tree. +*/ +SQLITE_PRIVATE void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ + int op; /* The opcode being coded */ + const char *zBinOp = 0; /* Binary operator */ + const char *zUniOp = 0; /* Unary operator */ + if( pExpr==0 ){ + op = TK_NULL; + }else{ + op = pExpr->op; + } + switch( op ){ + case TK_AGG_COLUMN: { + sqlite3ExplainPrintf(pOut, "AGG{%d:%d}", + pExpr->iTable, pExpr->iColumn); + break; + } + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ + sqlite3ExplainPrintf(pOut, "COLUMN(%d)", pExpr->iColumn); + }else{ + sqlite3ExplainPrintf(pOut, "{%d:%d}", + pExpr->iTable, pExpr->iColumn); + } + break; + } + case TK_INTEGER: { + if( pExpr->flags & EP_IntValue ){ + sqlite3ExplainPrintf(pOut, "%d", pExpr->u.iValue); + }else{ + sqlite3ExplainPrintf(pOut, "%s", pExpr->u.zToken); + } + break; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_STRING: { + sqlite3ExplainPrintf(pOut,"%Q", pExpr->u.zToken); + break; + } + case TK_NULL: { + sqlite3ExplainPrintf(pOut,"NULL"); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_VARIABLE: { + sqlite3ExplainPrintf(pOut,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); + break; + } + case TK_REGISTER: { + sqlite3ExplainPrintf(pOut,"REGISTER(%d)", pExpr->iTable); + break; + } + case TK_AS: { + sqlite3ExplainExpr(pOut, pExpr->pLeft); + break; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + const char *zAff = "unk"; + switch( sqlite3AffinityType(pExpr->u.zToken, 0) ){ + case SQLITE_AFF_TEXT: zAff = "TEXT"; break; + case SQLITE_AFF_NONE: zAff = "NONE"; break; + case SQLITE_AFF_NUMERIC: zAff = "NUMERIC"; break; + case SQLITE_AFF_INTEGER: zAff = "INTEGER"; break; + case SQLITE_AFF_REAL: zAff = "REAL"; break; + } + sqlite3ExplainPrintf(pOut, "CAST-%s(", zAff); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut, ")"); + break; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: zBinOp = "LT"; break; + case TK_LE: zBinOp = "LE"; break; + case TK_GT: zBinOp = "GT"; break; + case TK_GE: zBinOp = "GE"; break; + case TK_NE: zBinOp = "NE"; break; + case TK_EQ: zBinOp = "EQ"; break; + case TK_IS: zBinOp = "IS"; break; + case TK_ISNOT: zBinOp = "ISNOT"; break; + case TK_AND: zBinOp = "AND"; break; + case TK_OR: zBinOp = "OR"; break; + case TK_PLUS: zBinOp = "ADD"; break; + case TK_STAR: zBinOp = "MUL"; break; + case TK_MINUS: zBinOp = "SUB"; break; + case TK_REM: zBinOp = "REM"; break; + case TK_BITAND: zBinOp = "BITAND"; break; + case TK_BITOR: zBinOp = "BITOR"; break; + case TK_SLASH: zBinOp = "DIV"; break; + case TK_LSHIFT: zBinOp = "LSHIFT"; break; + case TK_RSHIFT: zBinOp = "RSHIFT"; break; + case TK_CONCAT: zBinOp = "CONCAT"; break; + + case TK_UMINUS: zUniOp = "UMINUS"; break; + case TK_UPLUS: zUniOp = "UPLUS"; break; + case TK_BITNOT: zUniOp = "BITNOT"; break; + case TK_NOT: zUniOp = "NOT"; break; + case TK_ISNULL: zUniOp = "ISNULL"; break; + case TK_NOTNULL: zUniOp = "NOTNULL"; break; + + case TK_COLLATE: { + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut,".COLLATE(%s)",pExpr->u.zToken); + break; + } + + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + if( ExprHasProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + }else{ + pFarg = pExpr->x.pList; + } + if( op==TK_AGG_FUNCTION ){ + sqlite3ExplainPrintf(pOut, "AGG_FUNCTION%d:%s(", + pExpr->op2, pExpr->u.zToken); + }else{ + sqlite3ExplainPrintf(pOut, "FUNCTION:%s(", pExpr->u.zToken); + } + if( pFarg ){ + sqlite3ExplainExprList(pOut, pFarg); + } + sqlite3ExplainPrintf(pOut, ")"); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: { + sqlite3ExplainPrintf(pOut, "EXISTS("); + sqlite3ExplainSelect(pOut, pExpr->x.pSelect); + sqlite3ExplainPrintf(pOut,")"); + break; + } + case TK_SELECT: { + sqlite3ExplainPrintf(pOut, "("); + sqlite3ExplainSelect(pOut, pExpr->x.pSelect); + sqlite3ExplainPrintf(pOut, ")"); + break; + } + case TK_IN: { + sqlite3ExplainPrintf(pOut, "IN("); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut, ","); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + sqlite3ExplainSelect(pOut, pExpr->x.pSelect); + }else{ + sqlite3ExplainExprList(pOut, pExpr->x.pList); + } + sqlite3ExplainPrintf(pOut, ")"); + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pX = pExpr->pLeft; + Expr *pY = pExpr->x.pList->a[0].pExpr; + Expr *pZ = pExpr->x.pList->a[1].pExpr; + sqlite3ExplainPrintf(pOut, "BETWEEN("); + sqlite3ExplainExpr(pOut, pX); + sqlite3ExplainPrintf(pOut, ","); + sqlite3ExplainExpr(pOut, pY); + sqlite3ExplainPrintf(pOut, ","); + sqlite3ExplainExpr(pOut, pZ); + sqlite3ExplainPrintf(pOut, ")"); + break; + } + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + */ + sqlite3ExplainPrintf(pOut, "%s(%d)", + pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); + break; + } + case TK_CASE: { + sqlite3ExplainPrintf(pOut, "CASE("); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut, ","); + sqlite3ExplainExprList(pOut, pExpr->x.pList); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + const char *zType = "unk"; + switch( pExpr->affinity ){ + case OE_Rollback: zType = "rollback"; break; + case OE_Abort: zType = "abort"; break; + case OE_Fail: zType = "fail"; break; + case OE_Ignore: zType = "ignore"; break; + } + sqlite3ExplainPrintf(pOut, "RAISE-%s(%s)", zType, pExpr->u.zToken); + break; + } +#endif + } + if( zBinOp ){ + sqlite3ExplainPrintf(pOut,"%s(", zBinOp); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut,","); + sqlite3ExplainExpr(pOut, pExpr->pRight); + sqlite3ExplainPrintf(pOut,")"); + }else if( zUniOp ){ + sqlite3ExplainPrintf(pOut,"%s(", zUniOp); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut,")"); + } +} +#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ + +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +/* +** Generate a human-readable explanation of an expression list. +*/ +SQLITE_PRIVATE void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ + int i; + if( pList==0 || pList->nExpr==0 ){ + sqlite3ExplainPrintf(pOut, "(empty-list)"); + return; + }else if( pList->nExpr==1 ){ + sqlite3ExplainExpr(pOut, pList->a[0].pExpr); + }else{ + sqlite3ExplainPush(pOut); + for(i=0; inExpr; i++){ + sqlite3ExplainPrintf(pOut, "item[%d] = ", i); + sqlite3ExplainPush(pOut); + sqlite3ExplainExpr(pOut, pList->a[i].pExpr); + sqlite3ExplainPop(pOut); + if( pList->a[i].zName ){ + sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); + } + if( pList->a[i].bSpanIsTab ){ + sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); + } + if( inExpr-1 ){ + sqlite3ExplainNL(pOut); + } + } + sqlite3ExplainPop(pOut); + } +} +#endif /* SQLITE_DEBUG */ + +/* +** Generate code that pushes the value of every element of the given +** expression list into a sequence of registers beginning at target. +** +** Return the number of elements evaluated. +** +** The SQLITE_ECEL_DUP flag prevents the arguments from being +** filled using OP_SCopy. OP_Copy must be used instead. +** +** The SQLITE_ECEL_FACTOR argument allows constant arguments to be +** factored out into initialization code. +*/ +SQLITE_PRIVATE int sqlite3ExprCodeExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* The expression list to be coded */ + int target, /* Where to write results */ + u8 flags /* SQLITE_ECEL_* flags */ +){ + struct ExprList_item *pItem; + int i, n; + u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; + assert( pList!=0 ); + assert( target>0 ); + assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ + n = pList->nExpr; + if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; + for(pItem=pList->a, i=0; ipExpr; + if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ + sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); + }else{ + int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); + if( inReg!=target+i ){ + VdbeOp *pOp; + Vdbe *v = pParse->pVdbe; + if( copyOp==OP_Copy + && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy + && pOp->p1+pOp->p3+1==inReg + && pOp->p2+pOp->p3+1==target+i + ){ + pOp->p3++; + }else{ + sqlite3VdbeAddOp2(v, copyOp, inReg, target+i); + } + } + } + } + return n; +} + +/* +** Generate code for a BETWEEN operator. +** +** x BETWEEN y AND z +** +** The above is equivalent to +** +** x>=y AND x<=z +** +** Code it as such, taking care to do the common subexpression +** elementation of x. +*/ +static void exprCodeBetween( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The BETWEEN expression */ + int dest, /* Jump here if the jump is taken */ + int jumpIfTrue, /* Take the jump if the BETWEEN is true */ + int jumpIfNull /* Take the jump if the BETWEEN is NULL */ +){ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr compLeft; /* The x>=y term */ + Expr compRight; /* The x<=z term */ + Expr exprX; /* The x subexpression */ + int regFree1 = 0; /* Temporary use register */ + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + exprX = *pExpr->pLeft; + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = &exprX; + compLeft.pRight = pExpr->x.pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = &exprX; + compRight.pRight = pExpr->x.pList->a[1].pExpr; + exprToRegister(&exprX, sqlite3ExprCodeTemp(pParse, &exprX, ®Free1)); + if( jumpIfTrue ){ + sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); + }else{ + sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); + } + sqlite3ReleaseTempReg(pParse, regFree1); + + /* Ensure adequate test coverage */ + testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 ); + testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 ); + testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 ); + testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 ); + testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 ); + testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 ); + testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 ); + testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 ); +} + +/* +** Generate code for a boolean expression such that a jump is made +** to the label "dest" if the expression is true but execution +** continues straight thru if the expression is false. +** +** If the expression evaluates to NULL (neither true nor false), then +** take the jump if the jumpIfNull flag is SQLITE_JUMPIFNULL. +** +** This code depends on the fact that certain token values (ex: TK_EQ) +** are the same as opcode values (ex: OP_Eq) that implement the corresponding +** operation. Special comments in vdbe.c and the mkopcodeh.awk script in +** the make process cause these values to align. Assert()s in the code +** below verify that the numbers are aligned correctly. +*/ +SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ + Vdbe *v = pParse->pVdbe; + int op = 0; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; + + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ + if( NEVER(pExpr==0) ) return; /* No way this can happen */ + op = pExpr->op; + switch( op ){ + case TK_AND: { + int d2 = sqlite3VdbeMakeLabel(v); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprCachePush(pParse); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + sqlite3ExprCachePop(pParse); + break; + } + case TK_OR: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprCachePush(pParse); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3ExprCachePop(pParse); + break; + } + case TK_NOT: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + testcase( jumpIfNull==0 ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_IS: + case TK_ISNOT: { + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + op = (op==TK_IS) ? TK_EQ : TK_NE; + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); + testcase( regFree1==0 ); + break; + } + case TK_BETWEEN: { + testcase( jumpIfNull==0 ); + exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_IN: { + int destIfFalse = sqlite3VdbeMakeLabel(v); + int destIfNull = jumpIfNull ? dest : destIfFalse; + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeResolveLabel(v, destIfFalse); + break; + } +#endif + default: { + if( exprAlwaysTrue(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + }else if( exprAlwaysFalse(pExpr) ){ + /* No-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } + break; + } + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); +} + +/* +** Generate code for a boolean expression such that a jump is made +** to the label "dest" if the expression is false but execution +** continues straight thru if the expression is true. +** +** If the expression evaluates to NULL (neither true nor false) then +** jump if jumpIfNull is SQLITE_JUMPIFNULL or fall through if jumpIfNull +** is 0. +*/ +SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ + Vdbe *v = pParse->pVdbe; + int op = 0; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; + + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ + if( pExpr==0 ) return; + + /* The value of pExpr->op and op are related as follows: + ** + ** pExpr->op op + ** --------- ---------- + ** TK_ISNULL OP_NotNull + ** TK_NOTNULL OP_IsNull + ** TK_NE OP_Eq + ** TK_EQ OP_Ne + ** TK_GT OP_Le + ** TK_LE OP_Gt + ** TK_GE OP_Lt + ** TK_LT OP_Ge + ** + ** For other values of pExpr->op, op is undefined and unused. + ** The value of TK_ and OP_ constants are arranged such that we + ** can compute the mapping above using the following expression. + ** Assert()s verify that the computation is correct. + */ + op = ((pExpr->op+(TK_ISNULL&1))^1)-(TK_ISNULL&1); + + /* Verify correct alignment of TK_ and OP_ constants + */ + assert( pExpr->op!=TK_ISNULL || op==OP_NotNull ); + assert( pExpr->op!=TK_NOTNULL || op==OP_IsNull ); + assert( pExpr->op!=TK_NE || op==OP_Eq ); + assert( pExpr->op!=TK_EQ || op==OP_Ne ); + assert( pExpr->op!=TK_LT || op==OP_Ge ); + assert( pExpr->op!=TK_LE || op==OP_Gt ); + assert( pExpr->op!=TK_GT || op==OP_Le ); + assert( pExpr->op!=TK_GE || op==OP_Lt ); + + switch( pExpr->op ){ + case TK_AND: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprCachePush(pParse); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3ExprCachePop(pParse); + break; + } + case TK_OR: { + int d2 = sqlite3VdbeMakeLabel(v); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprCachePush(pParse); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + sqlite3ExprCachePop(pParse); + break; + } + case TK_NOT: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + testcase( jumpIfNull==0 ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_IS: + case TK_ISNOT: { + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); + testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); + testcase( regFree1==0 ); + break; + } + case TK_BETWEEN: { + testcase( jumpIfNull==0 ); + exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_IN: { + if( jumpIfNull ){ + sqlite3ExprCodeIN(pParse, pExpr, dest, dest); + }else{ + int destIfNull = sqlite3VdbeMakeLabel(v); + sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull); + sqlite3VdbeResolveLabel(v, destIfNull); + } + break; + } +#endif + default: { + if( exprAlwaysFalse(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + }else if( exprAlwaysTrue(pExpr) ){ + /* no-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } + break; + } + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); +} + +/* +** Do a deep comparison of two expression trees. Return 0 if the two +** expressions are completely identical. Return 1 if they differ only +** by a COLLATE operator at the top level. Return 2 if there are differences +** other than the top-level COLLATE operator. +** +** If any subelement of pB has Expr.iTable==(-1) then it is allowed +** to compare equal to an equivalent element in pA with Expr.iTable==iTab. +** +** The pA side might be using TK_REGISTER. If that is the case and pB is +** not using TK_REGISTER but is otherwise equivalent, then still return 0. +** +** Sometimes this routine will return 2 even if the two expressions +** really are equivalent. If we cannot prove that the expressions are +** identical, we return 2 just to be safe. So if this routine +** returns 2, then you do not really know for certain if the two +** expressions are the same. But if you get a 0 or 1 return, then you +** can be sure the expressions are the same. In the places where +** this routine is used, it does not hurt to get an extra 2 - that +** just might result in some slightly slower code. But returning +** an incorrect 0 or 1 could lead to a malfunction. +*/ +SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ + u32 combinedFlags; + if( pA==0 || pB==0 ){ + return pB==pA ? 0 : 2; + } + combinedFlags = pA->flags | pB->flags; + if( combinedFlags & EP_IntValue ){ + if( (pA->flags&pB->flags&EP_IntValue)!=0 && pA->u.iValue==pB->u.iValue ){ + return 0; + } + return 2; + } + if( pA->op!=pB->op ){ + if( pA->op==TK_COLLATE && sqlite3ExprCompare(pA->pLeft, pB, iTab)<2 ){ + return 1; + } + if( pB->op==TK_COLLATE && sqlite3ExprCompare(pA, pB->pLeft, iTab)<2 ){ + return 1; + } + return 2; + } + if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken ){ + if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + return pA->op==TK_COLLATE ? 1 : 2; + } + } + if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; + if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){ + if( combinedFlags & EP_xIsSelect ) return 2; + if( sqlite3ExprCompare(pA->pLeft, pB->pLeft, iTab) ) return 2; + if( sqlite3ExprCompare(pA->pRight, pB->pRight, iTab) ) return 2; + if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2; + if( ALWAYS((combinedFlags & EP_Reduced)==0) ){ + if( pA->iColumn!=pB->iColumn ) return 2; + if( pA->iTable!=pB->iTable + && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2; + } + } + return 0; +} + +/* +** Compare two ExprList objects. Return 0 if they are identical and +** non-zero if they differ in any way. +** +** If any subelement of pB has Expr.iTable==(-1) then it is allowed +** to compare equal to an equivalent element in pA with Expr.iTable==iTab. +** +** This routine might return non-zero for equivalent ExprLists. The +** only consequence will be disabled optimizations. But this routine +** must never return 0 if the two ExprList objects are different, or +** a malfunction will result. +** +** Two NULL pointers are considered to be the same. But a NULL pointer +** always differs from a non-NULL pointer. +*/ +SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ + int i; + if( pA==0 && pB==0 ) return 0; + if( pA==0 || pB==0 ) return 1; + if( pA->nExpr!=pB->nExpr ) return 1; + for(i=0; inExpr; i++){ + Expr *pExprA = pA->a[i].pExpr; + Expr *pExprB = pB->a[i].pExpr; + if( pA->a[i].sortOrder!=pB->a[i].sortOrder ) return 1; + if( sqlite3ExprCompare(pExprA, pExprB, iTab) ) return 1; + } + return 0; +} + +/* +** Return true if we can prove the pE2 will always be true if pE1 is +** true. Return false if we cannot complete the proof or if pE2 might +** be false. Examples: +** +** pE1: x==5 pE2: x==5 Result: true +** pE1: x>0 pE2: x==5 Result: false +** pE1: x=21 pE2: x=21 OR y=43 Result: true +** pE1: x!=123 pE2: x IS NOT NULL Result: true +** pE1: x!=?1 pE2: x IS NOT NULL Result: true +** pE1: x IS NULL pE2: x IS NOT NULL Result: false +** pE1: x IS ?2 pE2: x IS NOT NULL Reuslt: false +** +** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has +** Expr.iTable<0 then assume a table number given by iTab. +** +** When in doubt, return false. Returning true might give a performance +** improvement. Returning false might cause a performance reduction, but +** it will always give the correct answer and is hence always safe. +*/ +SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Expr *pE1, Expr *pE2, int iTab){ + if( sqlite3ExprCompare(pE1, pE2, iTab)==0 ){ + return 1; + } + if( pE2->op==TK_OR + && (sqlite3ExprImpliesExpr(pE1, pE2->pLeft, iTab) + || sqlite3ExprImpliesExpr(pE1, pE2->pRight, iTab) ) + ){ + return 1; + } + if( pE2->op==TK_NOTNULL + && sqlite3ExprCompare(pE1->pLeft, pE2->pLeft, iTab)==0 + && (pE1->op!=TK_ISNULL && pE1->op!=TK_IS) + ){ + return 1; + } + return 0; +} + +/* +** An instance of the following structure is used by the tree walker +** to count references to table columns in the arguments of an +** aggregate function, in order to implement the +** sqlite3FunctionThisSrc() routine. +*/ +struct SrcCount { + SrcList *pSrc; /* One particular FROM clause in a nested query */ + int nThis; /* Number of references to columns in pSrcList */ + int nOther; /* Number of references to columns in other FROM clauses */ +}; + +/* +** Count the number of references to columns. +*/ +static int exprSrcCount(Walker *pWalker, Expr *pExpr){ + /* The NEVER() on the second term is because sqlite3FunctionUsesThisSrc() + ** is always called before sqlite3ExprAnalyzeAggregates() and so the + ** TK_COLUMNs have not yet been converted into TK_AGG_COLUMN. If + ** sqlite3FunctionUsesThisSrc() is used differently in the future, the + ** NEVER() will need to be removed. */ + if( pExpr->op==TK_COLUMN || NEVER(pExpr->op==TK_AGG_COLUMN) ){ + int i; + struct SrcCount *p = pWalker->u.pSrcCount; + SrcList *pSrc = p->pSrc; + for(i=0; inSrc; i++){ + if( pExpr->iTable==pSrc->a[i].iCursor ) break; + } + if( inSrc ){ + p->nThis++; + }else{ + p->nOther++; + } + } + return WRC_Continue; +} + +/* +** Determine if any of the arguments to the pExpr Function reference +** pSrcList. Return true if they do. Also return true if the function +** has no arguments or has only constant arguments. Return false if pExpr +** references columns but not columns of tables found in pSrcList. +*/ +SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr *pExpr, SrcList *pSrcList){ + Walker w; + struct SrcCount cnt; + assert( pExpr->op==TK_AGG_FUNCTION ); + memset(&w, 0, sizeof(w)); + w.xExprCallback = exprSrcCount; + w.u.pSrcCount = &cnt; + cnt.pSrc = pSrcList; + cnt.nThis = 0; + cnt.nOther = 0; + sqlite3WalkExprList(&w, pExpr->x.pList); + return cnt.nThis>0 || cnt.nOther==0; +} + +/* +** Add a new element to the pAggInfo->aCol[] array. Return the index of +** the new element. Return a negative number if malloc fails. +*/ +static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ + int i; + pInfo->aCol = sqlite3ArrayAllocate( + db, + pInfo->aCol, + sizeof(pInfo->aCol[0]), + &pInfo->nColumn, + &i + ); + return i; +} + +/* +** Add a new element to the pAggInfo->aFunc[] array. Return the index of +** the new element. Return a negative number if malloc fails. +*/ +static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ + int i; + pInfo->aFunc = sqlite3ArrayAllocate( + db, + pInfo->aFunc, + sizeof(pInfo->aFunc[0]), + &pInfo->nFunc, + &i + ); + return i; +} + +/* +** This is the xExprCallback for a tree walker. It is used to +** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates +** for additional information. +*/ +static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ + int i; + NameContext *pNC = pWalker->u.pNC; + Parse *pParse = pNC->pParse; + SrcList *pSrcList = pNC->pSrcList; + AggInfo *pAggInfo = pNC->pAggInfo; + + switch( pExpr->op ){ + case TK_AGG_COLUMN: + case TK_COLUMN: { + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_COLUMN ); + /* Check to see if the column is in one of the tables in the FROM + ** clause of the aggregate query */ + if( ALWAYS(pSrcList!=0) ){ + struct SrcList_item *pItem = pSrcList->a; + for(i=0; inSrc; i++, pItem++){ + struct AggInfo_col *pCol; + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + if( pExpr->iTable==pItem->iCursor ){ + /* If we reach this point, it means that pExpr refers to a table + ** that is in the FROM clause of the aggregate query. + ** + ** Make an entry for the column in pAggInfo->aCol[] if there + ** is not an entry there already. + */ + int k; + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable && + pCol->iColumn==pExpr->iColumn ){ + break; + } + } + if( (k>=pAggInfo->nColumn) + && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 + ){ + pCol = &pAggInfo->aCol[k]; + pCol->pTab = pExpr->pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iMem = ++pParse->nMem; + pCol->iSorterColumn = -1; + pCol->pExpr = pExpr; + if( pAggInfo->pGroupBy ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && + pE->iColumn==pExpr->iColumn ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } + } + /* There is now an entry for pExpr in pAggInfo->aCol[] (either + ** because it was there before or because we just created it). + ** Convert the pExpr to be a TK_AGG_COLUMN referring to that + ** pAggInfo->aCol[] entry. + */ + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->pAggInfo = pAggInfo; + pExpr->op = TK_AGG_COLUMN; + pExpr->iAgg = (i16)k; + break; + } /* endif pExpr->iTable==pItem->iCursor */ + } /* end loop over pSrcList */ + } + return WRC_Prune; + } + case TK_AGG_FUNCTION: { + if( (pNC->ncFlags & NC_InAggFunc)==0 + && pWalker->walkerDepth==pExpr->op2 + ){ + /* Check to see if pExpr is a duplicate of another aggregate + ** function that is already in the pAggInfo structure + */ + struct AggInfo_func *pItem = pAggInfo->aFunc; + for(i=0; inFunc; i++, pItem++){ + if( sqlite3ExprCompare(pItem->pExpr, pExpr, -1)==0 ){ + break; + } + } + if( i>=pAggInfo->nFunc ){ + /* pExpr is original. Make a new entry in pAggInfo->aFunc[] + */ + u8 enc = ENC(pParse->db); + i = addAggInfoFunc(pParse->db, pAggInfo); + if( i>=0 ){ + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + pItem = &pAggInfo->aFunc[i]; + pItem->pExpr = pExpr; + pItem->iMem = ++pParse->nMem; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pItem->pFunc = sqlite3FindFunction(pParse->db, + pExpr->u.zToken, sqlite3Strlen30(pExpr->u.zToken), + pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); + if( pExpr->flags & EP_Distinct ){ + pItem->iDistinct = pParse->nTab++; + }else{ + pItem->iDistinct = -1; + } + } + } + /* Make pExpr point to the appropriate pAggInfo->aFunc[] entry + */ + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->iAgg = (i16)i; + pExpr->pAggInfo = pAggInfo; + return WRC_Prune; + }else{ + return WRC_Continue; + } + } + } + return WRC_Continue; +} +static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ + UNUSED_PARAMETER(pWalker); + UNUSED_PARAMETER(pSelect); + return WRC_Continue; +} + +/* +** Analyze the pExpr expression looking for aggregate functions and +** for variables that need to be added to AggInfo object that pNC->pAggInfo +** points to. Additional entries are made on the AggInfo object as +** necessary. +** +** This routine should only be called after the expression has been +** analyzed by sqlite3ResolveExprNames(). +*/ +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = analyzeAggregate; + w.xSelectCallback = analyzeAggregatesInSelect; + w.u.pNC = pNC; + assert( pNC->pSrcList!=0 ); + sqlite3WalkExpr(&w, pExpr); +} + +/* +** Call sqlite3ExprAnalyzeAggregates() for every expression in an +** expression list. Return the number of errors. +** +** If an error is found, the analysis is cut short. +*/ +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext *pNC, ExprList *pList){ + struct ExprList_item *pItem; + int i; + if( pList ){ + for(pItem=pList->a, i=0; inExpr; i++, pItem++){ + sqlite3ExprAnalyzeAggregates(pNC, pItem->pExpr); + } + } +} + +/* +** Allocate a single new register for use to hold some intermediate result. +*/ +SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ + if( pParse->nTempReg==0 ){ + return ++pParse->nMem; + } + return pParse->aTempReg[--pParse->nTempReg]; +} + +/* +** Deallocate a register, making available for reuse for some other +** purpose. +** +** If a register is currently being used by the column cache, then +** the dallocation is deferred until the column cache line that uses +** the register becomes stale. +*/ +SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ + if( iReg && pParse->nTempRegaTempReg) ){ + int i; + struct yColCache *p; + for(i=0, p=pParse->aColCache; iiReg==iReg ){ + p->tempReg = 1; + return; + } + } + pParse->aTempReg[pParse->nTempReg++] = iReg; + } +} + +/* +** Allocate or deallocate a block of nReg consecutive registers +*/ +SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ + int i, n; + i = pParse->iRangeReg; + n = pParse->nRangeReg; + if( nReg<=n ){ + assert( !usedAsColumnCache(pParse, i, i+n-1) ); + pParse->iRangeReg += nReg; + pParse->nRangeReg -= nReg; + }else{ + i = pParse->nMem+1; + pParse->nMem += nReg; + } + return i; +} +SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ + sqlite3ExprCacheRemove(pParse, iReg, nReg); + if( nReg>pParse->nRangeReg ){ + pParse->nRangeReg = nReg; + pParse->iRangeReg = iReg; + } +} + +/* +** Mark all temporary registers as being unavailable for reuse. +*/ +SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ + pParse->nTempReg = 0; + pParse->nRangeReg = 0; +} + +/************** End of expr.c ************************************************/ +/************** Begin file alter.c *******************************************/ +/* +** 2005 February 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that used to generate VDBE code +** that implements the ALTER TABLE command. +*/ + +/* +** The code in this file only exists if we are not omitting the +** ALTER TABLE logic from the build. +*/ +#ifndef SQLITE_OMIT_ALTERTABLE + + +/* +** This function is used by SQL generated to implement the +** ALTER TABLE command. The first argument is the text of a CREATE TABLE or +** CREATE INDEX command. The second is a table name. The table name in +** the CREATE TABLE or CREATE INDEX statement is replaced with the third +** argument and the result returned. Examples: +** +** sqlite_rename_table('CREATE TABLE abc(a, b, c)', 'def') +** -> 'CREATE TABLE def(a, b, c)' +** +** sqlite_rename_table('CREATE INDEX i ON abc(a)', 'def') +** -> 'CREATE INDEX i ON def(a, b, c)' +*/ +static void renameTableFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + unsigned char const *zSql = sqlite3_value_text(argv[0]); + unsigned char const *zTableName = sqlite3_value_text(argv[1]); + + int token; + Token tname; + unsigned char const *zCsr = zSql; + int len = 0; + char *zRet; + + sqlite3 *db = sqlite3_context_db_handle(context); + + UNUSED_PARAMETER(NotUsed); + + /* The principle used to locate the table name in the CREATE TABLE + ** statement is that the table name is the first non-space token that + ** is immediately followed by a TK_LP or TK_USING token. + */ + if( zSql ){ + do { + if( !*zCsr ){ + /* Ran out of input before finding an opening bracket. Return NULL. */ + return; + } + + /* Store the token that zCsr points to in tname. */ + tname.z = (char*)zCsr; + tname.n = len; + + /* Advance zCsr to the next token. Store that token type in 'token', + ** and its length in 'len' (to be used next iteration of this loop). + */ + do { + zCsr += len; + len = sqlite3GetToken(zCsr, &token); + } while( token==TK_SPACE ); + assert( len>0 ); + } while( token!=TK_LP && token!=TK_USING ); + + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), + zSql, zTableName, tname.z+tname.n); + sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); + } +} + +/* +** This C function implements an SQL user function that is used by SQL code +** generated by the ALTER TABLE ... RENAME command to modify the definition +** of any foreign key constraints that use the table being renamed as the +** parent table. It is passed three arguments: +** +** 1) The complete text of the CREATE TABLE statement being modified, +** 2) The old name of the table being renamed, and +** 3) The new name of the table being renamed. +** +** It returns the new CREATE TABLE statement. For example: +** +** sqlite_rename_parent('CREATE TABLE t1(a REFERENCES t2)', 't2', 't3') +** -> 'CREATE TABLE t1(a REFERENCES t3)' +*/ +#ifndef SQLITE_OMIT_FOREIGN_KEY +static void renameParentFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char *zOutput = 0; + char *zResult; + unsigned char const *zInput = sqlite3_value_text(argv[0]); + unsigned char const *zOld = sqlite3_value_text(argv[1]); + unsigned char const *zNew = sqlite3_value_text(argv[2]); + + unsigned const char *z; /* Pointer to token */ + int n; /* Length of token z */ + int token; /* Type of token */ + + UNUSED_PARAMETER(NotUsed); + if( zInput==0 || zOld==0 ) return; + for(z=zInput; *z; z=z+n){ + n = sqlite3GetToken(z, &token); + if( token==TK_REFERENCES ){ + char *zParent; + do { + z += n; + n = sqlite3GetToken(z, &token); + }while( token==TK_SPACE ); + + zParent = sqlite3DbStrNDup(db, (const char *)z, n); + if( zParent==0 ) break; + sqlite3Dequote(zParent); + if( 0==sqlite3StrICmp((const char *)zOld, zParent) ){ + char *zOut = sqlite3MPrintf(db, "%s%.*s\"%w\"", + (zOutput?zOutput:""), (int)(z-zInput), zInput, (const char *)zNew + ); + sqlite3DbFree(db, zOutput); + zOutput = zOut; + zInput = &z[n]; + } + sqlite3DbFree(db, zParent); + } + } + + zResult = sqlite3MPrintf(db, "%s%s", (zOutput?zOutput:""), zInput), + sqlite3_result_text(context, zResult, -1, SQLITE_DYNAMIC); + sqlite3DbFree(db, zOutput); +} +#endif + +#ifndef SQLITE_OMIT_TRIGGER +/* This function is used by SQL generated to implement the +** ALTER TABLE command. The first argument is the text of a CREATE TRIGGER +** statement. The second is a table name. The table name in the CREATE +** TRIGGER statement is replaced with the third argument and the result +** returned. This is analagous to renameTableFunc() above, except for CREATE +** TRIGGER, not CREATE INDEX and CREATE TABLE. +*/ +static void renameTriggerFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + unsigned char const *zSql = sqlite3_value_text(argv[0]); + unsigned char const *zTableName = sqlite3_value_text(argv[1]); + + int token; + Token tname; + int dist = 3; + unsigned char const *zCsr = zSql; + int len = 0; + char *zRet; + sqlite3 *db = sqlite3_context_db_handle(context); + + UNUSED_PARAMETER(NotUsed); + + /* The principle used to locate the table name in the CREATE TRIGGER + ** statement is that the table name is the first token that is immediatedly + ** preceded by either TK_ON or TK_DOT and immediatedly followed by one + ** of TK_WHEN, TK_BEGIN or TK_FOR. + */ + if( zSql ){ + do { + + if( !*zCsr ){ + /* Ran out of input before finding the table name. Return NULL. */ + return; + } + + /* Store the token that zCsr points to in tname. */ + tname.z = (char*)zCsr; + tname.n = len; + + /* Advance zCsr to the next token. Store that token type in 'token', + ** and its length in 'len' (to be used next iteration of this loop). + */ + do { + zCsr += len; + len = sqlite3GetToken(zCsr, &token); + }while( token==TK_SPACE ); + assert( len>0 ); + + /* Variable 'dist' stores the number of tokens read since the most + ** recent TK_DOT or TK_ON. This means that when a WHEN, FOR or BEGIN + ** token is read and 'dist' equals 2, the condition stated above + ** to be met. + ** + ** Note that ON cannot be a database, table or column name, so + ** there is no need to worry about syntax like + ** "CREATE TRIGGER ... ON ON.ON BEGIN ..." etc. + */ + dist++; + if( token==TK_DOT || token==TK_ON ){ + dist = 0; + } + } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) ); + + /* Variable tname now contains the token that is the old table-name + ** in the CREATE TRIGGER statement. + */ + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), + zSql, zTableName, tname.z+tname.n); + sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); + } +} +#endif /* !SQLITE_OMIT_TRIGGER */ + +/* +** Register built-in functions used to help implement ALTER TABLE +*/ +SQLITE_PRIVATE void sqlite3AlterFunctions(void){ + static SQLITE_WSD FuncDef aAlterTableFuncs[] = { + FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), +#ifndef SQLITE_OMIT_TRIGGER + FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), +#endif +#ifndef SQLITE_OMIT_FOREIGN_KEY + FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), +#endif + }; + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAlterTableFuncs); + + for(i=0; i OR name= OR ... +** +** If argument zWhere is NULL, then a pointer string containing the text +** "name=" is returned, where is the quoted version +** of the string passed as argument zConstant. The returned buffer is +** allocated using sqlite3DbMalloc(). It is the responsibility of the +** caller to ensure that it is eventually freed. +** +** If argument zWhere is not NULL, then the string returned is +** " OR name=", where is the contents of zWhere. +** In this case zWhere is passed to sqlite3DbFree() before returning. +** +*/ +static char *whereOrName(sqlite3 *db, char *zWhere, char *zConstant){ + char *zNew; + if( !zWhere ){ + zNew = sqlite3MPrintf(db, "name=%Q", zConstant); + }else{ + zNew = sqlite3MPrintf(db, "%s OR name=%Q", zWhere, zConstant); + sqlite3DbFree(db, zWhere); + } + return zNew; +} + +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) +/* +** Generate the text of a WHERE expression which can be used to select all +** tables that have foreign key constraints that refer to table pTab (i.e. +** constraints for which pTab is the parent table) from the sqlite_master +** table. +*/ +static char *whereForeignKeys(Parse *pParse, Table *pTab){ + FKey *p; + char *zWhere = 0; + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + zWhere = whereOrName(pParse->db, zWhere, p->pFrom->zName); + } + return zWhere; +} +#endif + +/* +** Generate the text of a WHERE expression which can be used to select all +** temporary triggers on table pTab from the sqlite_temp_master table. If +** table pTab has no temporary triggers, or is itself stored in the +** temporary database, NULL is returned. +*/ +static char *whereTempTriggers(Parse *pParse, Table *pTab){ + Trigger *pTrig; + char *zWhere = 0; + const Schema *pTempSchema = pParse->db->aDb[1].pSchema; /* Temp db schema */ + + /* If the table is not located in the temp-db (in which case NULL is + ** returned, loop through the tables list of triggers. For each trigger + ** that is not part of the temp-db schema, add a clause to the WHERE + ** expression being built up in zWhere. + */ + if( pTab->pSchema!=pTempSchema ){ + sqlite3 *db = pParse->db; + for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ + if( pTrig->pSchema==pTempSchema ){ + zWhere = whereOrName(db, zWhere, pTrig->zName); + } + } + } + if( zWhere ){ + char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere); + sqlite3DbFree(pParse->db, zWhere); + zWhere = zNew; + } + return zWhere; +} + +/* +** Generate code to drop and reload the internal representation of table +** pTab from the database, including triggers and temporary triggers. +** Argument zName is the name of the table in the database schema at +** the time the generated code is executed. This can be different from +** pTab->zName if this function is being called to code part of an +** "ALTER TABLE RENAME TO" statement. +*/ +static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ + Vdbe *v; + char *zWhere; + int iDb; /* Index of database containing pTab */ +#ifndef SQLITE_OMIT_TRIGGER + Trigger *pTrig; +#endif + + v = sqlite3GetVdbe(pParse); + if( NEVER(v==0) ) return; + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + assert( iDb>=0 ); + +#ifndef SQLITE_OMIT_TRIGGER + /* Drop any table triggers from the internal schema. */ + for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ + int iTrigDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); + assert( iTrigDb==iDb || iTrigDb==1 ); + sqlite3VdbeAddOp4(v, OP_DropTrigger, iTrigDb, 0, 0, pTrig->zName, 0); + } +#endif + + /* Drop the table and index from the internal schema. */ + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); + + /* Reload the table, index and permanent trigger schemas. */ + zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); + if( !zWhere ) return; + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); + +#ifndef SQLITE_OMIT_TRIGGER + /* Now, if the table is not stored in the temp database, reload any temp + ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. + */ + if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ + sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); + } +#endif +} + +/* +** Parameter zName is the name of a table that is about to be altered +** (either with ALTER TABLE ... RENAME TO or ALTER TABLE ... ADD COLUMN). +** If the table is a system table, this function leaves an error message +** in pParse->zErr (system tables may not be altered) and returns non-zero. +** +** Or, if zName is not a system table, zero is returned. +*/ +static int isSystemTable(Parse *pParse, const char *zName){ + if( sqlite3Strlen30(zName)>6 && 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ + sqlite3ErrorMsg(pParse, "table %s may not be altered", zName); + return 1; + } + return 0; +} + +/* +** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" +** command. +*/ +SQLITE_PRIVATE void sqlite3AlterRenameTable( + Parse *pParse, /* Parser context. */ + SrcList *pSrc, /* The table to rename. */ + Token *pName /* The new table name. */ +){ + int iDb; /* Database that contains the table */ + char *zDb; /* Name of database iDb */ + Table *pTab; /* Table being renamed */ + char *zName = 0; /* NULL-terminated version of pName */ + sqlite3 *db = pParse->db; /* Database connection */ + int nTabName; /* Number of UTF-8 characters in zTabName */ + const char *zTabName; /* Original name of the table */ + Vdbe *v; +#ifndef SQLITE_OMIT_TRIGGER + char *zWhere = 0; /* Where clause to locate temp triggers */ +#endif + VTable *pVTab = 0; /* Non-zero if this is a v-tab with an xRename() */ + int savedDbFlags; /* Saved value of db->flags */ + + savedDbFlags = db->flags; + if( NEVER(db->mallocFailed) ) goto exit_rename_table; + assert( pSrc->nSrc==1 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_rename_table; + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + zDb = db->aDb[iDb].zName; + db->flags |= SQLITE_PreferBuiltin; + + /* Get a NULL terminated version of the new table name. */ + zName = sqlite3NameFromToken(db, pName); + if( !zName ) goto exit_rename_table; + + /* Check that a table or index named 'zName' does not already exist + ** in database iDb. If so, this is an error. + */ + if( sqlite3FindTable(db, zName, zDb) || sqlite3FindIndex(db, zName, zDb) ){ + sqlite3ErrorMsg(pParse, + "there is already another table or index with this name: %s", zName); + goto exit_rename_table; + } + + /* Make sure it is not a system table being altered, or a reserved name + ** that the table is being renamed to. + */ + if( SQLITE_OK!=isSystemTable(pParse, pTab->zName) ){ + goto exit_rename_table; + } + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto + exit_rename_table; + } + +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "view %s may not be altered", pTab->zName); + goto exit_rename_table; + } +#endif + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + goto exit_rename_table; + } +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto exit_rename_table; + } + if( IsVirtual(pTab) ){ + pVTab = sqlite3GetVTable(db, pTab); + if( pVTab->pVtab->pModule->xRename==0 ){ + pVTab = 0; + } + } +#endif + + /* Begin a transaction for database iDb. + ** Then modify the schema cookie (since the ALTER TABLE modifies the + ** schema). Open a statement transaction if the table is a virtual + ** table. + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ){ + goto exit_rename_table; + } + sqlite3BeginWriteOperation(pParse, pVTab!=0, iDb); + sqlite3ChangeCookie(pParse, iDb); + + /* If this is a virtual table, invoke the xRename() function if + ** one is defined. The xRename() callback will modify the names + ** of any resources used by the v-table implementation (including other + ** SQLite tables) that are identified by the name of the virtual table. + */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pVTab ){ + int i = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0); + sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); + sqlite3MayAbort(pParse); + } +#endif + + /* figure out how many UTF-8 characters are in zName */ + zTabName = pTab->zName; + nTabName = sqlite3Utf8CharLen(zTabName, -1); + +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + if( db->flags&SQLITE_ForeignKeys ){ + /* If foreign-key support is enabled, rewrite the CREATE TABLE + ** statements corresponding to all child tables of foreign key constraints + ** for which the renamed table is the parent table. */ + if( (zWhere=whereForeignKeys(pParse, pTab))!=0 ){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".%s SET " + "sql = sqlite_rename_parent(sql, %Q, %Q) " + "WHERE %s;", zDb, SCHEMA_TABLE(iDb), zTabName, zName, zWhere); + sqlite3DbFree(db, zWhere); + } + } +#endif + + /* Modify the sqlite_master table to use the new table name. */ + sqlite3NestedParse(pParse, + "UPDATE %Q.%s SET " +#ifdef SQLITE_OMIT_TRIGGER + "sql = sqlite_rename_table(sql, %Q), " +#else + "sql = CASE " + "WHEN type = 'trigger' THEN sqlite_rename_trigger(sql, %Q)" + "ELSE sqlite_rename_table(sql, %Q) END, " +#endif + "tbl_name = %Q, " + "name = CASE " + "WHEN type='table' THEN %Q " + "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " + "'sqlite_autoindex_' || %Q || substr(name,%d+18) " + "ELSE name END " + "WHERE tbl_name=%Q COLLATE nocase AND " + "(type='table' OR type='index' OR type='trigger');", + zDb, SCHEMA_TABLE(iDb), zName, zName, zName, +#ifndef SQLITE_OMIT_TRIGGER + zName, +#endif + zName, nTabName, zTabName + ); + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* If the sqlite_sequence table exists in this database, then update + ** it with the new table name. + */ + if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", + zDb, zName, pTab->zName); + } +#endif + +#ifndef SQLITE_OMIT_TRIGGER + /* If there are TEMP triggers on this table, modify the sqlite_temp_master + ** table. Don't do this if the table being ALTERed is itself located in + ** the temp database. + */ + if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ + sqlite3NestedParse(pParse, + "UPDATE sqlite_temp_master SET " + "sql = sqlite_rename_trigger(sql, %Q), " + "tbl_name = %Q " + "WHERE %s;", zName, zName, zWhere); + sqlite3DbFree(db, zWhere); + } +#endif + +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + if( db->flags&SQLITE_ForeignKeys ){ + FKey *p; + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + Table *pFrom = p->pFrom; + if( pFrom!=pTab ){ + reloadTableSchema(pParse, p->pFrom, pFrom->zName); + } + } + } +#endif + + /* Drop and reload the internal table schema. */ + reloadTableSchema(pParse, pTab, zName); + +exit_rename_table: + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zName); + db->flags = savedDbFlags; +} + + +/* +** Generate code to make sure the file format number is at least minFormat. +** The generated code will increase the file format number if necessary. +*/ +SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minFormat){ + Vdbe *v; + v = sqlite3GetVdbe(pParse); + /* The VDBE should have been allocated before this routine is called. + ** If that allocation failed, we would have quit before reaching this + ** point */ + if( ALWAYS(v) ){ + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3GetTempReg(pParse); + int j1; + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); + j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); + sqlite3VdbeJumpHere(v, j1); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } +} + +/* +** This function is called after an "ALTER TABLE ... ADD" statement +** has been parsed. Argument pColDef contains the text of the new +** column definition. +** +** The Table structure pParse->pNewTable was extended to include +** the new column during parsing. +*/ +SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ + Table *pNew; /* Copy of pParse->pNewTable */ + Table *pTab; /* Table being altered */ + int iDb; /* Database number */ + const char *zDb; /* Database name */ + const char *zTab; /* Table name */ + char *zCol; /* Null-terminated column definition */ + Column *pCol; /* The new column */ + Expr *pDflt; /* Default value for the new column */ + sqlite3 *db; /* The database connection; */ + + db = pParse->db; + if( pParse->nErr || db->mallocFailed ) return; + pNew = pParse->pNewTable; + assert( pNew ); + + assert( sqlite3BtreeHoldsAllMutexes(db) ); + iDb = sqlite3SchemaToIndex(db, pNew->pSchema); + zDb = db->aDb[iDb].zName; + zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ + pCol = &pNew->aCol[pNew->nCol-1]; + pDflt = pCol->pDflt; + pTab = sqlite3FindTable(db, zTab, zDb); + assert( pTab ); + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + return; + } +#endif + + /* If the default value for the new column was specified with a + ** literal NULL, then set pDflt to 0. This simplifies checking + ** for an SQL NULL default below. + */ + if( pDflt && pDflt->op==TK_NULL ){ + pDflt = 0; + } + + /* Check that the new column is not specified as PRIMARY KEY or UNIQUE. + ** If there is a NOT NULL constraint, then the default value for the + ** column must not be NULL. + */ + if( pCol->colFlags & COLFLAG_PRIMKEY ){ + sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); + return; + } + if( pNew->pIndex ){ + sqlite3ErrorMsg(pParse, "Cannot add a UNIQUE column"); + return; + } + if( (db->flags&SQLITE_ForeignKeys) && pNew->pFKey && pDflt ){ + sqlite3ErrorMsg(pParse, + "Cannot add a REFERENCES column with non-NULL default value"); + return; + } + if( pCol->notNull && !pDflt ){ + sqlite3ErrorMsg(pParse, + "Cannot add a NOT NULL column with default value NULL"); + return; + } + + /* Ensure the default expression is something that sqlite3ValueFromExpr() + ** can handle (i.e. not CURRENT_TIME etc.) + */ + if( pDflt ){ + sqlite3_value *pVal = 0; + if( sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){ + db->mallocFailed = 1; + return; + } + if( !pVal ){ + sqlite3ErrorMsg(pParse, "Cannot add a column with non-constant default"); + return; + } + sqlite3ValueFree(pVal); + } + + /* Modify the CREATE TABLE statement. */ + zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); + if( zCol ){ + char *zEnd = &zCol[pColDef->n-1]; + int savedDbFlags = db->flags; + while( zEnd>zCol && (*zEnd==';' || sqlite3Isspace(*zEnd)) ){ + *zEnd-- = '\0'; + } + db->flags |= SQLITE_PreferBuiltin; + sqlite3NestedParse(pParse, + "UPDATE \"%w\".%s SET " + "sql = substr(sql,1,%d) || ', ' || %Q || substr(sql,%d) " + "WHERE type = 'table' AND name = %Q", + zDb, SCHEMA_TABLE(iDb), pNew->addColOffset, zCol, pNew->addColOffset+1, + zTab + ); + sqlite3DbFree(db, zCol); + db->flags = savedDbFlags; + } + + /* If the default value of the new column is NULL, then set the file + ** format to 2. If the default value of the new column is not NULL, + ** the file format becomes 3. + */ + sqlite3MinimumFileFormat(pParse, iDb, pDflt ? 3 : 2); + + /* Reload the schema of the modified table. */ + reloadTableSchema(pParse, pTab, pTab->zName); +} + +/* +** This function is called by the parser after the table-name in +** an "ALTER TABLE ADD" statement is parsed. Argument +** pSrc is the full-name of the table being altered. +** +** This routine makes a (partial) copy of the Table structure +** for the table being altered and sets Parse.pNewTable to point +** to it. Routines called by the parser as the column definition +** is parsed (i.e. sqlite3AddColumn()) add the new Column data to +** the copy. The copy of the Table structure is deleted by tokenize.c +** after parsing is finished. +** +** Routine sqlite3AlterFinishAddColumn() will be called to complete +** coding the "ALTER TABLE ... ADD" statement. +*/ +SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ + Table *pNew; + Table *pTab; + Vdbe *v; + int iDb; + int i; + int nAlloc; + sqlite3 *db = pParse->db; + + /* Look up the table being altered. */ + assert( pParse->pNewTable==0 ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + if( db->mallocFailed ) goto exit_begin_add_column; + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_begin_add_column; + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "virtual tables may not be altered"); + goto exit_begin_add_column; + } +#endif + + /* Make sure this is not an attempt to ALTER a view. */ + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "Cannot add a column to a view"); + goto exit_begin_add_column; + } + if( SQLITE_OK!=isSystemTable(pParse, pTab->zName) ){ + goto exit_begin_add_column; + } + + assert( pTab->addColOffset>0 ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + + /* Put a copy of the Table struct in Parse.pNewTable for the + ** sqlite3AddColumn() function and friends to modify. But modify + ** the name by adding an "sqlite_altertab_" prefix. By adding this + ** prefix, we insure that the name will not collide with an existing + ** table because user table are not allowed to have the "sqlite_" + ** prefix on their name. + */ + pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table)); + if( !pNew ) goto exit_begin_add_column; + pParse->pNewTable = pNew; + pNew->nRef = 1; + pNew->nCol = pTab->nCol; + assert( pNew->nCol>0 ); + nAlloc = (((pNew->nCol-1)/8)*8)+8; + assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); + pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); + pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); + if( !pNew->aCol || !pNew->zName ){ + db->mallocFailed = 1; + goto exit_begin_add_column; + } + memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); + for(i=0; inCol; i++){ + Column *pCol = &pNew->aCol[i]; + pCol->zName = sqlite3DbStrDup(db, pCol->zName); + pCol->zColl = 0; + pCol->zType = 0; + pCol->pDflt = 0; + pCol->zDflt = 0; + } + pNew->pSchema = db->aDb[iDb].pSchema; + pNew->addColOffset = pTab->addColOffset; + pNew->nRef = 1; + + /* Begin a transaction and increment the schema cookie. */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + v = sqlite3GetVdbe(pParse); + if( !v ) goto exit_begin_add_column; + sqlite3ChangeCookie(pParse, iDb); + +exit_begin_add_column: + sqlite3SrcListDelete(db, pSrc); + return; +} +#endif /* SQLITE_ALTER_TABLE */ + +/************** End of alter.c ***********************************************/ +/************** Begin file analyze.c *****************************************/ +/* +** 2005-07-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code associated with the ANALYZE command. +** +** The ANALYZE command gather statistics about the content of tables +** and indices. These statistics are made available to the query planner +** to help it make better decisions about how to perform queries. +** +** The following system tables are or have been supported: +** +** CREATE TABLE sqlite_stat1(tbl, idx, stat); +** CREATE TABLE sqlite_stat2(tbl, idx, sampleno, sample); +** CREATE TABLE sqlite_stat3(tbl, idx, nEq, nLt, nDLt, sample); +** CREATE TABLE sqlite_stat4(tbl, idx, nEq, nLt, nDLt, sample); +** +** Additional tables might be added in future releases of SQLite. +** The sqlite_stat2 table is not created or used unless the SQLite version +** is between 3.6.18 and 3.7.8, inclusive, and unless SQLite is compiled +** with SQLITE_ENABLE_STAT2. The sqlite_stat2 table is deprecated. +** The sqlite_stat2 table is superseded by sqlite_stat3, which is only +** created and used by SQLite versions 3.7.9 and later and with +** SQLITE_ENABLE_STAT3 defined. The functionality of sqlite_stat3 +** is a superset of sqlite_stat2. The sqlite_stat4 is an enhanced +** version of sqlite_stat3 and is only available when compiled with +** SQLITE_ENABLE_STAT4 and in SQLite versions 3.8.1 and later. It is +** not possible to enable both STAT3 and STAT4 at the same time. If they +** are both enabled, then STAT4 takes precedence. +** +** For most applications, sqlite_stat1 provides all the statisics required +** for the query planner to make good choices. +** +** Format of sqlite_stat1: +** +** There is normally one row per index, with the index identified by the +** name in the idx column. The tbl column is the name of the table to +** which the index belongs. In each such row, the stat column will be +** a string consisting of a list of integers. The first integer in this +** list is the number of rows in the index. (This is the same as the +** number of rows in the table, except for partial indices.) The second +** integer is the average number of rows in the index that have the same +** value in the first column of the index. The third integer is the average +** number of rows in the index that have the same value for the first two +** columns. The N-th integer (for N>1) is the average number of rows in +** the index which have the same value for the first N-1 columns. For +** a K-column index, there will be K+1 integers in the stat column. If +** the index is unique, then the last integer will be 1. +** +** The list of integers in the stat column can optionally be followed +** by the keyword "unordered". The "unordered" keyword, if it is present, +** must be separated from the last integer by a single space. If the +** "unordered" keyword is present, then the query planner assumes that +** the index is unordered and will not use the index for a range query. +** +** If the sqlite_stat1.idx column is NULL, then the sqlite_stat1.stat +** column contains a single integer which is the (estimated) number of +** rows in the table identified by sqlite_stat1.tbl. +** +** Format of sqlite_stat2: +** +** The sqlite_stat2 is only created and is only used if SQLite is compiled +** with SQLITE_ENABLE_STAT2 and if the SQLite version number is between +** 3.6.18 and 3.7.8. The "stat2" table contains additional information +** about the distribution of keys within an index. The index is identified by +** the "idx" column and the "tbl" column is the name of the table to which +** the index belongs. There are usually 10 rows in the sqlite_stat2 +** table for each index. +** +** The sqlite_stat2 entries for an index that have sampleno between 0 and 9 +** inclusive are samples of the left-most key value in the index taken at +** evenly spaced points along the index. Let the number of samples be S +** (10 in the standard build) and let C be the number of rows in the index. +** Then the sampled rows are given by: +** +** rownumber = (i*C*2 + C)/(S*2) +** +** For i between 0 and S-1. Conceptually, the index space is divided into +** S uniform buckets and the samples are the middle row from each bucket. +** +** The format for sqlite_stat2 is recorded here for legacy reference. This +** version of SQLite does not support sqlite_stat2. It neither reads nor +** writes the sqlite_stat2 table. This version of SQLite only supports +** sqlite_stat3. +** +** Format for sqlite_stat3: +** +** The sqlite_stat3 format is a subset of sqlite_stat4. Hence, the +** sqlite_stat4 format will be described first. Further information +** about sqlite_stat3 follows the sqlite_stat4 description. +** +** Format for sqlite_stat4: +** +** As with sqlite_stat2, the sqlite_stat4 table contains histogram data +** to aid the query planner in choosing good indices based on the values +** that indexed columns are compared against in the WHERE clauses of +** queries. +** +** The sqlite_stat4 table contains multiple entries for each index. +** The idx column names the index and the tbl column is the table of the +** index. If the idx and tbl columns are the same, then the sample is +** of the INTEGER PRIMARY KEY. The sample column is a blob which is the +** binary encoding of a key from the index. The nEq column is a +** list of integers. The first integer is the approximate number +** of entries in the index whose left-most column exactly matches +** the left-most column of the sample. The second integer in nEq +** is the approximate number of entries in the index where the +** first two columns match the first two columns of the sample. +** And so forth. nLt is another list of integers that show the approximate +** number of entries that are strictly less than the sample. The first +** integer in nLt contains the number of entries in the index where the +** left-most column is less than the left-most column of the sample. +** The K-th integer in the nLt entry is the number of index entries +** where the first K columns are less than the first K columns of the +** sample. The nDLt column is like nLt except that it contains the +** number of distinct entries in the index that are less than the +** sample. +** +** There can be an arbitrary number of sqlite_stat4 entries per index. +** The ANALYZE command will typically generate sqlite_stat4 tables +** that contain between 10 and 40 samples which are distributed across +** the key space, though not uniformly, and which include samples with +** large nEq values. +** +** Format for sqlite_stat3 redux: +** +** The sqlite_stat3 table is like sqlite_stat4 except that it only +** looks at the left-most column of the index. The sqlite_stat3.sample +** column contains the actual value of the left-most column instead +** of a blob encoding of the complete index key as is found in +** sqlite_stat4.sample. The nEq, nLt, and nDLt entries of sqlite_stat3 +** all contain just a single integer which is the same as the first +** integer in the equivalent columns in sqlite_stat4. +*/ +#ifndef SQLITE_OMIT_ANALYZE + +#if defined(SQLITE_ENABLE_STAT4) +# define IsStat4 1 +# define IsStat3 0 +#elif defined(SQLITE_ENABLE_STAT3) +# define IsStat4 0 +# define IsStat3 1 +#else +# define IsStat4 0 +# define IsStat3 0 +# undef SQLITE_STAT4_SAMPLES +# define SQLITE_STAT4_SAMPLES 1 +#endif +#define IsStat34 (IsStat3+IsStat4) /* 1 for STAT3 or STAT4. 0 otherwise */ + +/* +** This routine generates code that opens the sqlite_statN tables. +** The sqlite_stat1 table is always relevant. sqlite_stat2 is now +** obsolete. sqlite_stat3 and sqlite_stat4 are only opened when +** appropriate compile-time options are provided. +** +** If the sqlite_statN tables do not previously exist, it is created. +** +** Argument zWhere may be a pointer to a buffer containing a table name, +** or it may be a NULL pointer. If it is not NULL, then all entries in +** the sqlite_statN tables associated with the named table are deleted. +** If zWhere==0, then code is generated to delete all stat table entries. +*/ +static void openStatTable( + Parse *pParse, /* Parsing context */ + int iDb, /* The database we are looking in */ + int iStatCur, /* Open the sqlite_stat1 table on this cursor */ + const char *zWhere, /* Delete entries for this table or index */ + const char *zWhereType /* Either "tbl" or "idx" */ +){ + static const struct { + const char *zName; + const char *zCols; + } aTable[] = { + { "sqlite_stat1", "tbl,idx,stat" }, +#if defined(SQLITE_ENABLE_STAT4) + { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, + { "sqlite_stat3", 0 }, +#elif defined(SQLITE_ENABLE_STAT3) + { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, + { "sqlite_stat4", 0 }, +#else + { "sqlite_stat3", 0 }, + { "sqlite_stat4", 0 }, +#endif + }; + int i; + sqlite3 *db = pParse->db; + Db *pDb; + Vdbe *v = sqlite3GetVdbe(pParse); + int aRoot[ArraySize(aTable)]; + u8 aCreateTbl[ArraySize(aTable)]; + + if( v==0 ) return; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3VdbeDb(v)==db ); + pDb = &db->aDb[iDb]; + + /* Create new statistic tables if they do not exist, or clear them + ** if they do already exist. + */ + for(i=0; izName))==0 ){ + if( aTable[i].zCols ){ + /* The sqlite_statN table does not exist. Create it. Note that a + ** side-effect of the CREATE TABLE statement is to leave the rootpage + ** of the new table in register pParse->regRoot. This is important + ** because the OpenWrite opcode below will be needing it. */ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols + ); + aRoot[i] = pParse->regRoot; + aCreateTbl[i] = OPFLAG_P2ISREG; + } + }else{ + /* The table already exists. If zWhere is not NULL, delete all entries + ** associated with the table zWhere. If zWhere is NULL, delete the + ** entire contents of the table. */ + aRoot[i] = pStat->tnum; + aCreateTbl[i] = 0; + sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); + if( zWhere ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE %s=%Q", + pDb->zName, zTab, zWhereType, zWhere + ); + }else{ + /* The sqlite_stat[134] table already exists. Delete all rows. */ + sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb); + } + } + } + + /* Open the sqlite_stat[134] tables for writing. */ + for(i=0; aTable[i].zCols; i++){ + assert( inRowid ){ + sqlite3DbFree(db, p->u.aRowid); + p->nRowid = 0; + } +} +#endif + +/* Initialize the BLOB value of a ROWID +*/ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ + assert( db!=0 ); + if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); + p->u.aRowid = sqlite3DbMallocRaw(db, n); + if( p->u.aRowid ){ + p->nRowid = n; + memcpy(p->u.aRowid, pData, n); + }else{ + p->nRowid = 0; + } +} +#endif + +/* Initialize the INTEGER value of a ROWID. +*/ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +static void sampleSetRowidInt64(sqlite3 *db, Stat4Sample *p, i64 iRowid){ + assert( db!=0 ); + if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); + p->nRowid = 0; + p->u.iRowid = iRowid; +} +#endif + + +/* +** Copy the contents of object (*pFrom) into (*pTo). +*/ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +static void sampleCopy(Stat4Accum *p, Stat4Sample *pTo, Stat4Sample *pFrom){ + pTo->isPSample = pFrom->isPSample; + pTo->iCol = pFrom->iCol; + pTo->iHash = pFrom->iHash; + memcpy(pTo->anEq, pFrom->anEq, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anLt, pFrom->anLt, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anDLt, pFrom->anDLt, sizeof(tRowcnt)*p->nCol); + if( pFrom->nRowid ){ + sampleSetRowid(p->db, pTo, pFrom->nRowid, pFrom->u.aRowid); + }else{ + sampleSetRowidInt64(p->db, pTo, pFrom->u.iRowid); + } +} +#endif + +/* +** Reclaim all memory of a Stat4Accum structure. +*/ +static void stat4Destructor(void *pOld){ + Stat4Accum *p = (Stat4Accum*)pOld; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int i; + for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); + for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); + sampleClear(p->db, &p->current); +#endif + sqlite3DbFree(p->db, p); +} + +/* +** Implementation of the stat_init(N,K,C) SQL function. The three parameters +** are: +** N: The number of columns in the index including the rowid/pk (note 1) +** K: The number of columns in the index excluding the rowid/pk. +** C: The number of rows in the index (note 2) +** +** Note 1: In the special case of the covering index that implements a +** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the +** total number of columns in the table. +** +** Note 2: C is only used for STAT3 and STAT4. +** +** For indexes on ordinary rowid tables, N==K+1. But for indexes on +** WITHOUT ROWID tables, N=K+P where P is the number of columns in the +** PRIMARY KEY of the table. The covering index that implements the +** original WITHOUT ROWID table as N==K as a special case. +** +** This routine allocates the Stat4Accum object in heap memory. The return +** value is a pointer to the the Stat4Accum object encoded as a blob (i.e. +** the size of the blob is sizeof(void*) bytes). +*/ +static void statInit( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Stat4Accum *p; + int nCol; /* Number of columns in index being sampled */ + int nKeyCol; /* Number of key columns */ + int nColUp; /* nCol rounded up for alignment */ + int n; /* Bytes of space to allocate */ + sqlite3 *db; /* Database connection */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int mxSample = SQLITE_STAT4_SAMPLES; +#endif + + /* Decode the three function arguments */ + UNUSED_PARAMETER(argc); + nCol = sqlite3_value_int(argv[0]); + assert( nCol>0 ); + nColUp = sizeof(tRowcnt)<8 ? (nCol+1)&~1 : nCol; + nKeyCol = sqlite3_value_int(argv[1]); + assert( nKeyCol<=nCol ); + assert( nKeyCol>0 ); + + /* Allocate the space required for the Stat4Accum object */ + n = sizeof(*p) + + sizeof(tRowcnt)*nColUp /* Stat4Accum.anEq */ + + sizeof(tRowcnt)*nColUp /* Stat4Accum.anDLt */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + + sizeof(tRowcnt)*nColUp /* Stat4Accum.anLt */ + + sizeof(Stat4Sample)*(nCol+mxSample) /* Stat4Accum.aBest[], a[] */ + + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample) +#endif + ; + db = sqlite3_context_db_handle(context); + p = sqlite3DbMallocZero(db, n); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + p->db = db; + p->nRow = 0; + p->nCol = nCol; + p->nKeyCol = nKeyCol; + p->current.anDLt = (tRowcnt*)&p[1]; + p->current.anEq = &p->current.anDLt[nColUp]; + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + { + u8 *pSpace; /* Allocated space not yet assigned */ + int i; /* Used to iterate through p->aSample[] */ + + p->iGet = -1; + p->mxSample = mxSample; + p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); + p->current.anLt = &p->current.anEq[nColUp]; + p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[2])*0xd0944565; + + /* Set up the Stat4Accum.a[] and aBest[] arrays */ + p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; + p->aBest = &p->a[mxSample]; + pSpace = (u8*)(&p->a[mxSample+nCol]); + for(i=0; i<(mxSample+nCol); i++){ + p->a[i].anEq = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anDLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + } + assert( (pSpace - (u8*)p)==n ); + + for(i=0; iaBest[i].iCol = i; + } + } +#endif + + /* Return a pointer to the allocated object to the caller */ + sqlite3_result_blob(context, p, sizeof(p), stat4Destructor); +} +static const FuncDef statInitFuncdef = { + 2+IsStat34, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statInit, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat_init", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + +#ifdef SQLITE_ENABLE_STAT4 +/* +** pNew and pOld are both candidate non-periodic samples selected for +** the same column (pNew->iCol==pOld->iCol). Ignoring this column and +** considering only any trailing columns and the sample hash value, this +** function returns true if sample pNew is to be preferred over pOld. +** In other words, if we assume that the cardinalities of the selected +** column for pNew and pOld are equal, is pNew to be preferred over pOld. +** +** This function assumes that for each argument sample, the contents of +** the anEq[] array from pSample->anEq[pSample->iCol+1] onwards are valid. +*/ +static int sampleIsBetterPost( + Stat4Accum *pAccum, + Stat4Sample *pNew, + Stat4Sample *pOld +){ + int nCol = pAccum->nCol; + int i; + assert( pNew->iCol==pOld->iCol ); + for(i=pNew->iCol+1; ianEq[i]>pOld->anEq[i] ) return 1; + if( pNew->anEq[i]anEq[i] ) return 0; + } + if( pNew->iHash>pOld->iHash ) return 1; + return 0; +} +#endif + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Return true if pNew is to be preferred over pOld. +** +** This function assumes that for each argument sample, the contents of +** the anEq[] array from pSample->anEq[pSample->iCol] onwards are valid. +*/ +static int sampleIsBetter( + Stat4Accum *pAccum, + Stat4Sample *pNew, + Stat4Sample *pOld +){ + tRowcnt nEqNew = pNew->anEq[pNew->iCol]; + tRowcnt nEqOld = pOld->anEq[pOld->iCol]; + + assert( pOld->isPSample==0 && pNew->isPSample==0 ); + assert( IsStat4 || (pNew->iCol==0 && pOld->iCol==0) ); + + if( (nEqNew>nEqOld) ) return 1; +#ifdef SQLITE_ENABLE_STAT4 + if( nEqNew==nEqOld ){ + if( pNew->iColiCol ) return 1; + return (pNew->iCol==pOld->iCol && sampleIsBetterPost(pAccum, pNew, pOld)); + } + return 0; +#else + return (nEqNew==nEqOld && pNew->iHash>pOld->iHash); +#endif +} + +/* +** Copy the contents of sample *pNew into the p->a[] array. If necessary, +** remove the least desirable sample from p->a[] to make room. +*/ +static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ + Stat4Sample *pSample = 0; + int i; + + assert( IsStat4 || nEqZero==0 ); + +#ifdef SQLITE_ENABLE_STAT4 + if( pNew->isPSample==0 ){ + Stat4Sample *pUpgrade = 0; + assert( pNew->anEq[pNew->iCol]>0 ); + + /* This sample is being added because the prefix that ends in column + ** iCol occurs many times in the table. However, if we have already + ** added a sample that shares this prefix, there is no need to add + ** this one. Instead, upgrade the priority of the highest priority + ** existing sample that shares this prefix. */ + for(i=p->nSample-1; i>=0; i--){ + Stat4Sample *pOld = &p->a[i]; + if( pOld->anEq[pNew->iCol]==0 ){ + if( pOld->isPSample ) return; + assert( pOld->iCol>pNew->iCol ); + assert( sampleIsBetter(p, pNew, pOld) ); + if( pUpgrade==0 || sampleIsBetter(p, pOld, pUpgrade) ){ + pUpgrade = pOld; + } + } + } + if( pUpgrade ){ + pUpgrade->iCol = pNew->iCol; + pUpgrade->anEq[pUpgrade->iCol] = pNew->anEq[pUpgrade->iCol]; + goto find_new_min; + } + } +#endif + + /* If necessary, remove sample iMin to make room for the new sample. */ + if( p->nSample>=p->mxSample ){ + Stat4Sample *pMin = &p->a[p->iMin]; + tRowcnt *anEq = pMin->anEq; + tRowcnt *anLt = pMin->anLt; + tRowcnt *anDLt = pMin->anDLt; + sampleClear(p->db, pMin); + memmove(pMin, &pMin[1], sizeof(p->a[0])*(p->nSample-p->iMin-1)); + pSample = &p->a[p->nSample-1]; + pSample->nRowid = 0; + pSample->anEq = anEq; + pSample->anDLt = anDLt; + pSample->anLt = anLt; + p->nSample = p->mxSample-1; + } + + /* The "rows less-than" for the rowid column must be greater than that + ** for the last sample in the p->a[] array. Otherwise, the samples would + ** be out of order. */ +#ifdef SQLITE_ENABLE_STAT4 + assert( p->nSample==0 + || pNew->anLt[p->nCol-1] > p->a[p->nSample-1].anLt[p->nCol-1] ); +#endif + + /* Insert the new sample */ + pSample = &p->a[p->nSample]; + sampleCopy(p, pSample, pNew); + p->nSample++; + + /* Zero the first nEqZero entries in the anEq[] array. */ + memset(pSample->anEq, 0, sizeof(tRowcnt)*nEqZero); + +#ifdef SQLITE_ENABLE_STAT4 + find_new_min: +#endif + if( p->nSample>=p->mxSample ){ + int iMin = -1; + for(i=0; imxSample; i++){ + if( p->a[i].isPSample ) continue; + if( iMin<0 || sampleIsBetter(p, &p->a[iMin], &p->a[i]) ){ + iMin = i; + } + } + assert( iMin>=0 ); + p->iMin = iMin; + } +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** Field iChng of the index being scanned has changed. So at this point +** p->current contains a sample that reflects the previous row of the +** index. The value of anEq[iChng] and subsequent anEq[] elements are +** correct at this point. +*/ +static void samplePushPrevious(Stat4Accum *p, int iChng){ +#ifdef SQLITE_ENABLE_STAT4 + int i; + + /* Check if any samples from the aBest[] array should be pushed + ** into IndexSample.a[] at this point. */ + for(i=(p->nCol-2); i>=iChng; i--){ + Stat4Sample *pBest = &p->aBest[i]; + pBest->anEq[i] = p->current.anEq[i]; + if( p->nSamplemxSample || sampleIsBetter(p, pBest, &p->a[p->iMin]) ){ + sampleInsert(p, pBest, i); + } + } + + /* Update the anEq[] fields of any samples already collected. */ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } + } +#endif + +#if defined(SQLITE_ENABLE_STAT3) && !defined(SQLITE_ENABLE_STAT4) + if( iChng==0 ){ + tRowcnt nLt = p->current.anLt[0]; + tRowcnt nEq = p->current.anEq[0]; + + /* Check if this is to be a periodic sample. If so, add it. */ + if( (nLt/p->nPSample)!=(nLt+nEq)/p->nPSample ){ + p->current.isPSample = 1; + sampleInsert(p, &p->current, 0); + p->current.isPSample = 0; + }else + + /* Or if it is a non-periodic sample. Add it in this case too. */ + if( p->nSamplemxSample + || sampleIsBetter(p, &p->current, &p->a[p->iMin]) + ){ + sampleInsert(p, &p->current, 0); + } + } +#endif + +#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 + UNUSED_PARAMETER( p ); + UNUSED_PARAMETER( iChng ); +#endif +} + +/* +** Implementation of the stat_push SQL function: stat_push(P,C,R) +** Arguments: +** +** P Pointer to the Stat4Accum object created by stat_init() +** C Index of left-most column to differ from previous row +** R Rowid for the current row. Might be a key record for +** WITHOUT ROWID tables. +** +** This SQL function always returns NULL. It's purpose it to accumulate +** statistical data and/or samples in the Stat4Accum object about the +** index being analyzed. The stat_get() SQL function will later be used to +** extract relevant information for constructing the sqlite_statN tables. +** +** The R parameter is only used for STAT3 and STAT4 +*/ +static void statPush( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + + /* The three function arguments */ + Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); + int iChng = sqlite3_value_int(argv[1]); + + UNUSED_PARAMETER( argc ); + UNUSED_PARAMETER( context ); + assert( p->nCol>0 ); + assert( iChngnCol ); + + if( p->nRow==0 ){ + /* This is the first call to this function. Do initialization. */ + for(i=0; inCol; i++) p->current.anEq[i] = 1; + }else{ + /* Second and subsequent calls get processed here */ + samplePushPrevious(p, iChng); + + /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply + ** to the current row of the index. */ + for(i=0; icurrent.anEq[i]++; + } + for(i=iChng; inCol; i++){ + p->current.anDLt[i]++; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + p->current.anLt[i] += p->current.anEq[i]; +#endif + p->current.anEq[i] = 1; + } + } + p->nRow++; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ + sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); + }else{ + sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), + sqlite3_value_blob(argv[2])); + } + p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; +#endif + +#ifdef SQLITE_ENABLE_STAT4 + { + tRowcnt nLt = p->current.anLt[p->nCol-1]; + + /* Check if this is to be a periodic sample. If so, add it. */ + if( (nLt/p->nPSample)!=(nLt+1)/p->nPSample ){ + p->current.isPSample = 1; + p->current.iCol = 0; + sampleInsert(p, &p->current, p->nCol-1); + p->current.isPSample = 0; + } + + /* Update the aBest[] array. */ + for(i=0; i<(p->nCol-1); i++){ + p->current.iCol = i; + if( i>=iChng || sampleIsBetterPost(p, &p->current, &p->aBest[i]) ){ + sampleCopy(p, &p->aBest[i], &p->current); + } + } + } +#endif +} +static const FuncDef statPushFuncdef = { + 2+IsStat34, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statPush, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat_push", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + +#define STAT_GET_STAT1 0 /* "stat" column of stat1 table */ +#define STAT_GET_ROWID 1 /* "rowid" column of stat[34] entry */ +#define STAT_GET_NEQ 2 /* "neq" column of stat[34] entry */ +#define STAT_GET_NLT 3 /* "nlt" column of stat[34] entry */ +#define STAT_GET_NDLT 4 /* "ndlt" column of stat[34] entry */ + +/* +** Implementation of the stat_get(P,J) SQL function. This routine is +** used to query statistical information that has been gathered into +** the Stat4Accum object by prior calls to stat_push(). The P parameter +** is a BLOB which is decoded into a pointer to the Stat4Accum objects. +** The content to returned is determined by the parameter J +** which is one of the STAT_GET_xxxx values defined above. +** +** If neither STAT3 nor STAT4 are enabled, then J is always +** STAT_GET_STAT1 and is hence omitted and this routine becomes +** a one-parameter function, stat_get(P), that always returns the +** stat1 table entry information. +*/ +static void statGet( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + /* STAT3 and STAT4 have a parameter on this routine. */ + int eCall = sqlite3_value_int(argv[1]); + assert( argc==2 ); + assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ + || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT + || eCall==STAT_GET_NDLT + ); + if( eCall==STAT_GET_STAT1 ) +#else + assert( argc==1 ); +#endif + { + /* Return the value to store in the "stat" column of the sqlite_stat1 + ** table for this index. + ** + ** The value is a string composed of a list of integers describing + ** the index. The first integer in the list is the total number of + ** entries in the index. There is one additional integer in the list + ** for each indexed column. This additional integer is an estimate of + ** the number of rows matched by a stabbing query on the index using + ** a key with the corresponding number of fields. In other words, + ** if the index is on columns (a,b) and the sqlite_stat1 value is + ** "100 10 2", then SQLite estimates that: + ** + ** * the index contains 100 rows, + ** * "WHERE a=?" matches 10 rows, and + ** * "WHERE a=? AND b=?" matches 2 rows. + ** + ** If D is the count of distinct values and K is the total number of + ** rows, then each estimate is computed as: + ** + ** I = (K+D-1)/D + */ + char *z; + int i; + + char *zRet = sqlite3MallocZero( (p->nKeyCol+1)*25 ); + if( zRet==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + sqlite3_snprintf(24, zRet, "%llu", (u64)p->nRow); + z = zRet + sqlite3Strlen30(zRet); + for(i=0; inKeyCol; i++){ + u64 nDistinct = p->current.anDLt[i] + 1; + u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; + sqlite3_snprintf(24, z, " %llu", iVal); + z += sqlite3Strlen30(z); + assert( p->current.anEq[i] ); + } + assert( z[0]=='\0' && z>zRet ); + + sqlite3_result_text(context, zRet, -1, sqlite3_free); + } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + else if( eCall==STAT_GET_ROWID ){ + if( p->iGet<0 ){ + samplePushPrevious(p, 0); + p->iGet = 0; + } + if( p->iGetnSample ){ + Stat4Sample *pS = p->a + p->iGet; + if( pS->nRowid==0 ){ + sqlite3_result_int64(context, pS->u.iRowid); + }else{ + sqlite3_result_blob(context, pS->u.aRowid, pS->nRowid, + SQLITE_TRANSIENT); + } + } + }else{ + tRowcnt *aCnt = 0; + + assert( p->iGetnSample ); + switch( eCall ){ + case STAT_GET_NEQ: aCnt = p->a[p->iGet].anEq; break; + case STAT_GET_NLT: aCnt = p->a[p->iGet].anLt; break; + default: { + aCnt = p->a[p->iGet].anDLt; + p->iGet++; + break; + } + } + + if( IsStat3 ){ + sqlite3_result_int64(context, (i64)aCnt[0]); + }else{ + char *zRet = sqlite3MallocZero(p->nCol * 25); + if( zRet==0 ){ + sqlite3_result_error_nomem(context); + }else{ + int i; + char *z = zRet; + for(i=0; inCol; i++){ + sqlite3_snprintf(24, z, "%llu ", (u64)aCnt[i]); + z += sqlite3Strlen30(z); + } + assert( z[0]=='\0' && z>zRet ); + z[-1] = '\0'; + sqlite3_result_text(context, zRet, -1, sqlite3_free); + } + } + } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( argc ); +#endif +} +static const FuncDef statGetFuncdef = { + 1+IsStat34, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statGet, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat_get", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + +static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ + assert( regOut!=regStat4 && regOut!=regStat4+1 ); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3VdbeAddOp2(v, OP_Integer, iParam, regStat4+1); +#elif SQLITE_DEBUG + assert( iParam==STAT_GET_STAT1 ); +#else + UNUSED_PARAMETER( iParam ); +#endif + sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regOut); + sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 1 + IsStat34); +} + +/* +** Generate code to do an analysis of all indices associated with +** a single table. +*/ +static void analyzeOneTable( + Parse *pParse, /* Parser context */ + Table *pTab, /* Table whose indices are to be analyzed */ + Index *pOnlyIdx, /* If not NULL, only analyze this one index */ + int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ + int iMem, /* Available memory locations begin here */ + int iTab /* Next available cursor */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + Index *pIdx; /* An index to being analyzed */ + int iIdxCur; /* Cursor open on index being analyzed */ + int iTabCur; /* Table cursor */ + Vdbe *v; /* The virtual machine being built up */ + int i; /* Loop counter */ + int jZeroRows = -1; /* Jump from here if number of rows is zero */ + int iDb; /* Index of database containing pTab */ + u8 needTableCnt = 1; /* True to count the table */ + int regNewRowid = iMem++; /* Rowid for the inserted record */ + int regStat4 = iMem++; /* Register to hold Stat4Accum object */ + int regChng = iMem++; /* Index of changed index field */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int regRowid = iMem++; /* Rowid argument passed to stat_push() */ +#endif + int regTemp = iMem++; /* Temporary use register */ + int regTabname = iMem++; /* Register containing table name */ + int regIdxname = iMem++; /* Register containing index name */ + int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ + int regPrev = iMem; /* MUST BE LAST (see below) */ + + pParse->nMem = MAX(pParse->nMem, iMem); + v = sqlite3GetVdbe(pParse); + if( v==0 || NEVER(pTab==0) ){ + return; + } + if( pTab->tnum==0 ){ + /* Do not gather statistics on views or virtual tables */ + return; + } + if( sqlite3_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ + /* Do not gather statistics on system tables */ + return; + } + assert( sqlite3BtreeHoldsAllMutexes(db) ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 ); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); +#ifndef SQLITE_OMIT_AUTHORIZATION + if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, + db->aDb[iDb].zName ) ){ + return; + } +#endif + + /* Establish a read-lock on the table at the shared-cache level. + ** Open a read-only cursor on the table. Also allocate a cursor number + ** to use for scanning indexes (iIdxCur). No index cursor is opened at + ** this time though. */ + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + iTabCur = iTab++; + iIdxCur = iTab++; + pParse->nTab = MAX(pParse->nTab, iTab); + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); + + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int nCol; /* Number of columns in pIdx. "N" */ + int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrNextRow; /* Address of "next_row:" */ + const char *zIdxName; /* Name of the index */ + int nColTest; /* Number of columns to test for changes */ + + if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; + if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ + nCol = pIdx->nKeyCol; + zIdxName = pTab->zName; + nColTest = nCol - 1; + }else{ + nCol = pIdx->nColumn; + zIdxName = pIdx->zName; + nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1; + } + + /* Populate the register containing the index name. */ + sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); + VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); + + /* + ** Pseudo-code for loop that calls stat_push(): + ** + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto chng_addr_0; + ** + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto chng_addr_N + ** + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + ** + ** endDistinctTest: + ** regRowid = idx(rowid) + ** stat_push(P, regChng, regRowid) + ** Next csr + ** if !eof(csr) goto next_row; + ** + ** end_of_scan: + */ + + /* Make sure there are enough memory cells allocated to accommodate + ** the regPrev array and a trailing rowid (the rowid slot is required + ** when building a record to insert into the sample column of + ** the sqlite_stat4 table. */ + pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); + + /* Open a read-only cursor on the index being analyzed. */ + assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); + sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + + /* Invoke the stat_init() function. The arguments are: + ** + ** (1) the number of columns in the index including the rowid + ** (or for a WITHOUT ROWID table, the number of PK columns), + ** (2) the number of columns in the key without the rowid/pk + ** (3) the number of rows in the index, + ** + ** + ** The third argument is only used for STAT3 and STAT4 + */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); +#endif + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); + sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); + sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2+IsStat34); + + /* Implementation of the following: + ** + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto next_push_0; + ** + */ + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); + addrNextRow = sqlite3VdbeCurrentAddr(v); + + if( nColTest>0 ){ + int endDistinctTest = sqlite3VdbeMakeLabel(v); + int *aGotoChng; /* Array of jump instruction addresses */ + aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); + if( aGotoChng==0 ) continue; + + /* + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto endDistinctTest + */ + sqlite3VdbeAddOp0(v, OP_Goto); + addrNextRow = sqlite3VdbeCurrentAddr(v); + if( nColTest==1 && pIdx->nKeyCol==1 && IsUniqueIndex(pIdx) ){ + /* For a single-column UNIQUE index, once we have found a non-NULL + ** row, we know that all the rest will be distinct, so skip + ** subsequent distinctness tests. */ + sqlite3VdbeAddOp2(v, OP_NotNull, regPrev, endDistinctTest); + VdbeCoverage(v); + } + for(i=0; iazColl[i]); + sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + aGotoChng[i] = + sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); + + + /* + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + */ + sqlite3VdbeJumpHere(v, addrNextRow-1); + for(i=0; ipTable); + int j, k, regKey; + regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); + VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); + sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); + } +#endif + assert( regChng==(regStat4+1) ); + sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); + sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2+IsStat34); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + + /* Add the entry to the stat1 table. */ + callStatGet(v, regStat4, STAT_GET_STAT1, regStat1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + + /* Add the entries to the stat3 or stat4 table. */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + { + int regEq = regStat1; + int regLt = regStat1+1; + int regDLt = regStat1+2; + int regSample = regStat1+3; + int regCol = regStat1+4; + int regSampleRowid = regCol + nCol; + int addrNext; + int addrIsNull; + u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + + pParse->nMem = MAX(pParse->nMem, regCol+nCol); + + addrNext = sqlite3VdbeCurrentAddr(v); + callStatGet(v, regStat4, STAT_GET_ROWID, regSampleRowid); + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid); + VdbeCoverage(v); + callStatGet(v, regStat4, STAT_GET_NEQ, regEq); + callStatGet(v, regStat4, STAT_GET_NLT, regLt); + callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); + sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); + /* We know that the regSampleRowid row exists because it was read by + ** the previous loop. Thus the not-found jump of seekOp will never + ** be taken */ + VdbeCoverageNeverTaken(v); +#ifdef SQLITE_ENABLE_STAT3 + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, + pIdx->aiColumn[0], regSample); +#else + for(i=0; iaiColumn[i]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); +#endif + sqlite3VdbeAddOp3(v, OP_MakeRecord, regTabname, 6, regTemp); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regTemp, regNewRowid); + sqlite3VdbeAddOp2(v, OP_Goto, 1, addrNext); /* P1==1 for end-of-loop */ + sqlite3VdbeJumpHere(v, addrIsNull); + } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + + /* End of analysis */ + sqlite3VdbeJumpHere(v, addrRewind); + } + + + /* Create a single sqlite_stat1 entry containing NULL as the index + ** name and the row count as the content. + */ + if( pOnlyIdx==0 && needTableCnt ){ + VdbeComment((v, "%s", pTab->zName)); + sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeJumpHere(v, jZeroRows); + } +} + + +/* +** Generate code that will cause the most recent index analysis to +** be loaded into internal hash tables where is can be used. +*/ +static void loadAnalysis(Parse *pParse, int iDb){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); + } +} + +/* +** Generate code that will do an analysis of an entire database +*/ +static void analyzeDatabase(Parse *pParse, int iDb){ + sqlite3 *db = pParse->db; + Schema *pSchema = db->aDb[iDb].pSchema; /* Schema of database iDb */ + HashElem *k; + int iStatCur; + int iMem; + int iTab; + + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + openStatTable(pParse, iDb, iStatCur, 0, 0); + iMem = pParse->nMem+1; + iTab = pParse->nTab; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + } + loadAnalysis(pParse, iDb); +} + +/* +** Generate code that will do an analysis of a single table in +** a database. If pOnlyIdx is not NULL then it is a single index +** in pTab that should be analyzed. +*/ +static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ + int iDb; + int iStatCur; + + assert( pTab!=0 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + if( pOnlyIdx ){ + openStatTable(pParse, iDb, iStatCur, pOnlyIdx->zName, "idx"); + }else{ + openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); + } + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab); + loadAnalysis(pParse, iDb); +} + +/* +** Generate code for the ANALYZE command. The parser calls this routine +** when it recognizes an ANALYZE command. +** +** ANALYZE -- 1 +** ANALYZE -- 2 +** ANALYZE ?.? -- 3 +** +** Form 1 causes all indices in all attached databases to be analyzed. +** Form 2 analyzes all indices the single database named. +** Form 3 analyzes all indices associated with the named table. +*/ +SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ + sqlite3 *db = pParse->db; + int iDb; + int i; + char *z, *zDb; + Table *pTab; + Index *pIdx; + Token *pTableName; + Vdbe *v; + + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return; + } + + assert( pName2!=0 || pName1==0 ); + if( pName1==0 ){ + /* Form 1: Analyze everything */ + for(i=0; inDb; i++){ + if( i==1 ) continue; /* Do not analyze the TEMP database */ + analyzeDatabase(pParse, i); + } + }else if( pName2->n==0 ){ + /* Form 2: Analyze the database or table named */ + iDb = sqlite3FindDb(db, pName1); + if( iDb>=0 ){ + analyzeDatabase(pParse, iDb); + }else{ + z = sqlite3NameFromToken(db, pName1); + if( z ){ + if( (pIdx = sqlite3FindIndex(db, z, 0))!=0 ){ + analyzeTable(pParse, pIdx->pTable, pIdx); + }else if( (pTab = sqlite3LocateTable(pParse, 0, z, 0))!=0 ){ + analyzeTable(pParse, pTab, 0); + } + sqlite3DbFree(db, z); + } + } + }else{ + /* Form 3: Analyze the fully qualified table name */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); + if( iDb>=0 ){ + zDb = db->aDb[iDb].zName; + z = sqlite3NameFromToken(db, pTableName); + if( z ){ + if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ + analyzeTable(pParse, pIdx->pTable, pIdx); + }else if( (pTab = sqlite3LocateTable(pParse, 0, z, zDb))!=0 ){ + analyzeTable(pParse, pTab, 0); + } + sqlite3DbFree(db, z); + } + } + } + v = sqlite3GetVdbe(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Expire); +} + +/* +** Used to pass information from the analyzer reader through to the +** callback routine. +*/ +typedef struct analysisInfo analysisInfo; +struct analysisInfo { + sqlite3 *db; + const char *zDatabase; +}; + +/* +** The first argument points to a nul-terminated string containing a +** list of space separated integers. Read the first nOut of these into +** the array aOut[]. +*/ +static void decodeIntArray( + char *zIntArray, /* String containing int array to decode */ + int nOut, /* Number of slots in aOut[] */ + tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ + Index *pIndex /* Handle extra flags for this index, if not NULL */ +){ + char *z = zIntArray; + int c; + int i; + tRowcnt v; + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( z==0 ) z = ""; +#else + if( NEVER(z==0) ) z = ""; +#endif + for(i=0; *z && i='0' && c<='9' ){ + v = v*10 + c - '0'; + z++; + } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( aOut ){ + aOut[i] = v; + }else +#else + assert( aOut==0 ); + UNUSED_PARAMETER(aOut); +#endif + { + aLog[i] = sqlite3LogEst(v); + } + if( *z==' ' ) z++; + } +#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 + assert( pIndex!=0 ); +#else + if( pIndex ) +#endif + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + } +#ifdef SQLITE_ENABLE_COSTMULT + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } +#endif + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; + } +} + +/* +** This callback is invoked once for each index when reading the +** sqlite_stat1 table. +** +** argv[0] = name of the table +** argv[1] = name of the index (might be NULL) +** argv[2] = results of analysis - on integer for each column +** +** Entries for which argv[1]==NULL simply record the number of rows in +** the table. +*/ +static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ + analysisInfo *pInfo = (analysisInfo*)pData; + Index *pIndex; + Table *pTable; + const char *z; + + assert( argc==3 ); + UNUSED_PARAMETER2(NotUsed, argc); + + if( argv==0 || argv[0]==0 || argv[2]==0 ){ + return 0; + } + pTable = sqlite3FindTable(pInfo->db, argv[0], pInfo->zDatabase); + if( pTable==0 ){ + return 0; + } + if( argv[1]==0 ){ + pIndex = 0; + }else if( sqlite3_stricmp(argv[0],argv[1])==0 ){ + pIndex = sqlite3PrimaryKeyIndex(pTable); + }else{ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); + } + z = argv[2]; + + if( pIndex ){ + pIndex->bUnordered = 0; + decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + }else{ + Index fakeIdx; + fakeIdx.szIdxRow = pTable->szTabRow; +#ifdef SQLITE_ENABLE_COSTMULT + fakeIdx.pTable = pTable; +#endif + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); + pTable->szTabRow = fakeIdx.szIdxRow; + } + + return 0; +} + +/* +** If the Index.aSample variable is not NULL, delete the aSample[] array +** and its contents. +*/ +SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( pIdx->aSample ){ + int j; + for(j=0; jnSample; j++){ + IndexSample *p = &pIdx->aSample[j]; + sqlite3DbFree(db, p->p); + } + sqlite3DbFree(db, pIdx->aSample); + } + if( db && db->pnBytesFreed==0 ){ + pIdx->nSample = 0; + pIdx->aSample = 0; + } +#else + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pIdx); +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +} + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Populate the pIdx->aAvgEq[] array based on the samples currently +** stored in pIdx->aSample[]. +*/ +static void initAvgEq(Index *pIdx){ + if( pIdx ){ + IndexSample *aSample = pIdx->aSample; + IndexSample *pFinal = &aSample[pIdx->nSample-1]; + int iCol; + int nCol = 1; + if( pIdx->nSampleCol>1 ){ + /* If this is stat4 data, then calculate aAvgEq[] values for all + ** sample columns except the last. The last is always set to 1, as + ** once the trailing PK fields are considered all index keys are + ** unique. */ + nCol = pIdx->nSampleCol-1; + pIdx->aAvgEq[nCol] = 1; + } + for(iCol=0; iColanDLt[iCol]; + + /* Set nSum to the number of distinct (iCol+1) field prefixes that + ** occur in the stat4 table for this index before pFinal. Set + ** sumEq to the sum of the nEq values for column iCol for the same + ** set (adding the value only once where there exist dupicate + ** prefixes). */ + for(i=0; i<(pIdx->nSample-1); i++){ + if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){ + sumEq += aSample[i].anEq[iCol]; + nSum++; + } + } + if( nDLt>nSum ){ + avgEq = (pFinal->anLt[iCol] - sumEq)/(nDLt - nSum); + } + if( avgEq==0 ) avgEq = 1; + pIdx->aAvgEq[iCol] = avgEq; + } + } +} + +/* +** Look up an index by name. Or, if the name of a WITHOUT ROWID table +** is supplied instead, find the PRIMARY KEY index for that table. +*/ +static Index *findIndexOrPrimaryKey( + sqlite3 *db, + const char *zName, + const char *zDb +){ + Index *pIdx = sqlite3FindIndex(db, zName, zDb); + if( pIdx==0 ){ + Table *pTab = sqlite3FindTable(db, zName, zDb); + if( pTab && !HasRowid(pTab) ) pIdx = sqlite3PrimaryKeyIndex(pTab); + } + return pIdx; +} + +/* +** Load the content from either the sqlite_stat4 or sqlite_stat3 table +** into the relevant Index.aSample[] arrays. +** +** Arguments zSql1 and zSql2 must point to SQL statements that return +** data equivalent to the following (statements are different for stat3, +** see the caller of this function for details): +** +** zSql1: SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx +** zSql2: SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4 +** +** where %Q is replaced with the database name before the SQL is executed. +*/ +static int loadStatTbl( + sqlite3 *db, /* Database handle */ + int bStat3, /* Assume single column records only */ + const char *zSql1, /* SQL statement 1 (see above) */ + const char *zSql2, /* SQL statement 2 (see above) */ + const char *zDb /* Database name (e.g. "main") */ +){ + int rc; /* Result codes from subroutines */ + sqlite3_stmt *pStmt = 0; /* An SQL statement being run */ + char *zSql; /* Text of the SQL statement */ + Index *pPrevIdx = 0; /* Previous index in the loop */ + IndexSample *pSample; /* A slot in pIdx->aSample[] */ + + assert( db->lookaside.bEnabled==0 ); + zSql = sqlite3MPrintf(db, zSql1, zDb); + if( !zSql ){ + return SQLITE_NOMEM; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int nIdxCol = 1; /* Number of columns in stat4 records */ + + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nSample; /* Number of samples */ + int nByte; /* Bytes of space required */ + int i; /* Bytes of space required */ + tRowcnt *pSpace; + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + nSample = sqlite3_column_int(pStmt, 1); + pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); + assert( pIdx==0 || bStat3 || pIdx->nSample==0 ); + /* Index.nSample is non-zero at this point if data has already been + ** loaded from the stat4 table. In this case ignore stat3 data. */ + if( pIdx==0 || pIdx->nSample ) continue; + if( bStat3==0 ){ + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; + } + } + pIdx->nSampleCol = nIdxCol; + nByte = sizeof(IndexSample) * nSample; + nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + + pIdx->aSample = sqlite3DbMallocZero(db, nByte); + if( pIdx->aSample==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM; + } + pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; + for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol; + } + assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) ); + } + rc = sqlite3_finalize(pStmt); + if( rc ) return rc; + + zSql = sqlite3MPrintf(db, zSql2, zDb); + if( !zSql ){ + return SQLITE_NOMEM; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nCol = 1; /* Number of columns in index */ + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); + if( pIdx==0 ) continue; + /* This next condition is true if data has already been loaded from + ** the sqlite_stat4 table. In this case ignore stat3 data. */ + nCol = pIdx->nSampleCol; + if( bStat3 && nCol>1 ) continue; + if( pIdx!=pPrevIdx ){ + initAvgEq(pPrevIdx); + pPrevIdx = pIdx; + } + pSample = &pIdx->aSample[pIdx->nSample]; + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); + + /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. + ** This is in case the sample record is corrupted. In that case, the + ** sqlite3VdbeRecordCompare() may read up to two varints past the + ** end of the allocated buffer before it realizes it is dealing with + ** a corrupt record. Adding the two 0x00 bytes prevents this from causing + ** a buffer overread. */ + pSample->n = sqlite3_column_bytes(pStmt, 4); + pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); + if( pSample->p==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM; + } + memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); + pIdx->nSample++; + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) initAvgEq(pPrevIdx); + return rc; +} + +/* +** Load content from the sqlite_stat4 and sqlite_stat3 tables into +** the Index.aSample[] arrays of all indices. +*/ +static int loadStat4(sqlite3 *db, const char *zDb){ + int rc = SQLITE_OK; /* Result codes from subroutines */ + + assert( db->lookaside.bEnabled==0 ); + if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ + rc = loadStatTbl(db, 0, + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", + zDb + ); + } + + if( rc==SQLITE_OK && sqlite3FindTable(db, "sqlite_stat3", zDb) ){ + rc = loadStatTbl(db, 1, + "SELECT idx,count(*) FROM %Q.sqlite_stat3 GROUP BY idx", + "SELECT idx,neq,nlt,ndlt,sqlite_record(sample) FROM %Q.sqlite_stat3", + zDb + ); + } + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** Load the content of the sqlite_stat1 and sqlite_stat3/4 tables. The +** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] +** arrays. The contents of sqlite_stat3/4 are used to populate the +** Index.aSample[] arrays. +** +** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR +** is returned. In this case, even if SQLITE_ENABLE_STAT3/4 was defined +** during compilation and the sqlite_stat3/4 table is present, no data is +** read from it. +** +** If SQLITE_ENABLE_STAT3/4 was defined during compilation and the +** sqlite_stat4 table is not present in the database, SQLITE_ERROR is +** returned. However, in this case, data is read from the sqlite_stat1 +** table (if it is present) before returning. +** +** If an OOM error occurs, this function always sets db->mallocFailed. +** This means if the caller does not care about other errors, the return +** code may be ignored. +*/ +SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ + analysisInfo sInfo; + HashElem *i; + char *zSql; + int rc; + + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 ); + + /* Clear any prior statistics */ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3DefaultRowEst(pIdx); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3DeleteIndexSamples(db, pIdx); + pIdx->aSample = 0; +#endif + } + + /* Check to make sure the sqlite_stat1 table exists */ + sInfo.db = db; + sInfo.zDatabase = db->aDb[iDb].zName; + if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)==0 ){ + return SQLITE_ERROR; + } + + /* Load new statistics out of the sqlite_stat1 table */ + zSql = sqlite3MPrintf(db, + "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); + sqlite3DbFree(db, zSql); + } + + + /* Load the statistics from the sqlite_stat4 table. */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( rc==SQLITE_OK ){ + int lookasideEnabled = db->lookaside.bEnabled; + db->lookaside.bEnabled = 0; + rc = loadStat4(db, sInfo.zDatabase); + db->lookaside.bEnabled = lookasideEnabled; + } +#endif + + if( rc==SQLITE_NOMEM ){ + db->mallocFailed = 1; + } + return rc; +} + + +#endif /* SQLITE_OMIT_ANALYZE */ + +/************** End of analyze.c *********************************************/ +/************** Begin file attach.c ******************************************/ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the ATTACH and DETACH commands. +*/ + +#ifndef SQLITE_OMIT_ATTACH +/* +** Resolve an expression that was part of an ATTACH or DETACH statement. This +** is slightly different from resolving a normal SQL expression, because simple +** identifiers are treated as strings, not possible column names or aliases. +** +** i.e. if the parser sees: +** +** ATTACH DATABASE abc AS def +** +** it treats the two expressions as literal strings 'abc' and 'def' instead of +** looking for columns of the same name. +** +** This only applies to the root node of pExpr, so the statement: +** +** ATTACH DATABASE abc||def AS 'db2' +** +** will fail because neither abc or def can be resolved. +*/ +static int resolveAttachExpr(NameContext *pName, Expr *pExpr) +{ + int rc = SQLITE_OK; + if( pExpr ){ + if( pExpr->op!=TK_ID ){ + rc = sqlite3ResolveExprNames(pName, pExpr); + }else{ + pExpr->op = TK_STRING; + } + } + return rc; +} + +/* +** An SQL user-function registered to do the work of an ATTACH statement. The +** three arguments to the function come directly from an attach statement: +** +** ATTACH DATABASE x AS y KEY z +** +** SELECT sqlite_attach(x, y, z) +** +** If the optional "KEY z" syntax is omitted, an SQL NULL is passed as the +** third argument. +*/ +static void attachFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + int i; + int rc = 0; + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zName; + const char *zFile; + char *zPath = 0; + char *zErr = 0; + unsigned int flags; + Db *aNew; + char *zErrDyn = 0; + sqlite3_vfs *pVfs; + + UNUSED_PARAMETER(NotUsed); + + zFile = (const char *)sqlite3_value_text(argv[0]); + zName = (const char *)sqlite3_value_text(argv[1]); + if( zFile==0 ) zFile = ""; + if( zName==0 ) zName = ""; + + /* Check for the following errors: + ** + ** * Too many attached databases, + ** * Transaction currently open + ** * Specified database name already being used. + */ + if( db->nDb>=db->aLimit[SQLITE_LIMIT_ATTACHED]+2 ){ + zErrDyn = sqlite3MPrintf(db, "too many attached databases - max %d", + db->aLimit[SQLITE_LIMIT_ATTACHED] + ); + goto attach_error; + } + if( !db->autoCommit ){ + zErrDyn = sqlite3MPrintf(db, "cannot ATTACH database within transaction"); + goto attach_error; + } + for(i=0; inDb; i++){ + char *z = db->aDb[i].zName; + assert( z && zName ); + if( sqlite3StrICmp(z, zName)==0 ){ + zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); + goto attach_error; + } + } + + /* Allocate the new entry in the db->aDb[] array and initialize the schema + ** hash tables. + */ + if( db->aDb==db->aDbStatic ){ + aNew = sqlite3DbMallocRaw(db, sizeof(db->aDb[0])*3 ); + if( aNew==0 ) return; + memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); + }else{ + aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); + if( aNew==0 ) return; + } + db->aDb = aNew; + aNew = &db->aDb[db->nDb]; + memset(aNew, 0, sizeof(*aNew)); + + /* Open the database file. If the btree is successfully opened, use + ** it to obtain the database schema. At this point the schema may + ** or may not be initialized. + */ + flags = db->openFlags; + rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + assert( pVfs ); + flags |= SQLITE_OPEN_MAIN_DB; + rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, 0, flags); + sqlite3_free( zPath ); + db->nDb++; + if( rc==SQLITE_CONSTRAINT ){ + rc = SQLITE_ERROR; + zErrDyn = sqlite3MPrintf(db, "database is already attached"); + }else if( rc==SQLITE_OK ){ + Pager *pPager; + aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt); + if( !aNew->pSchema ){ + rc = SQLITE_NOMEM; + }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){ + zErrDyn = sqlite3MPrintf(db, + "attached databases must use the same text encoding as main database"); + rc = SQLITE_ERROR; + } + pPager = sqlite3BtreePager(aNew->pBt); + sqlite3PagerLockingMode(pPager, db->dfltLockMode); + sqlite3BtreeSecureDelete(aNew->pBt, + sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK)); +#endif + } + aNew->safety_level = 3; + aNew->zName = sqlite3DbStrDup(db, zName); + if( rc==SQLITE_OK && aNew->zName==0 ){ + rc = SQLITE_NOMEM; + } + + +#ifdef SQLITE_HAS_CODEC + if( rc==SQLITE_OK ){ + extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); + extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); + int nKey; + char *zKey; + int t = sqlite3_value_type(argv[2]); + switch( t ){ + case SQLITE_INTEGER: + case SQLITE_FLOAT: + zErrDyn = sqlite3DbStrDup(db, "Invalid key value"); + rc = SQLITE_ERROR; + break; + + case SQLITE_TEXT: + case SQLITE_BLOB: + nKey = sqlite3_value_bytes(argv[2]); + zKey = (char *)sqlite3_value_blob(argv[2]); + rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); + break; + + case SQLITE_NULL: + /* No key specified. Use the key from the main database */ + sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); + if( nKey>0 || sqlite3BtreeGetReserve(db->aDb[0].pBt)>0 ){ + rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); + } + break; + } + } +#endif + + /* If the file was opened successfully, read the schema for the new database. + ** If this fails, or if opening the file failed, then close the file and + ** remove the entry from the db->aDb[] array. i.e. put everything back the way + ** we found it. + */ + if( rc==SQLITE_OK ){ + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrDyn); + sqlite3BtreeLeaveAll(db); + } + if( rc ){ + int iDb = db->nDb - 1; + assert( iDb>=2 ); + if( db->aDb[iDb].pBt ){ + sqlite3BtreeClose(db->aDb[iDb].pBt); + db->aDb[iDb].pBt = 0; + db->aDb[iDb].pSchema = 0; + } + sqlite3ResetAllSchemasOfConnection(db); + db->nDb = iDb; + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + db->mallocFailed = 1; + sqlite3DbFree(db, zErrDyn); + zErrDyn = sqlite3MPrintf(db, "out of memory"); + }else if( zErrDyn==0 ){ + zErrDyn = sqlite3MPrintf(db, "unable to open database: %s", zFile); + } + goto attach_error; + } + + return; + +attach_error: + /* Return an error if we get here */ + if( zErrDyn ){ + sqlite3_result_error(context, zErrDyn, -1); + sqlite3DbFree(db, zErrDyn); + } + if( rc ) sqlite3_result_error_code(context, rc); +} + +/* +** An SQL user-function registered to do the work of an DETACH statement. The +** three arguments to the function come directly from a detach statement: +** +** DETACH DATABASE x +** +** SELECT sqlite_detach(x) +*/ +static void detachFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + const char *zName = (const char *)sqlite3_value_text(argv[0]); + sqlite3 *db = sqlite3_context_db_handle(context); + int i; + Db *pDb = 0; + char zErr[128]; + + UNUSED_PARAMETER(NotUsed); + + if( zName==0 ) zName = ""; + for(i=0; inDb; i++){ + pDb = &db->aDb[i]; + if( pDb->pBt==0 ) continue; + if( sqlite3StrICmp(pDb->zName, zName)==0 ) break; + } + + if( i>=db->nDb ){ + sqlite3_snprintf(sizeof(zErr),zErr, "no such database: %s", zName); + goto detach_error; + } + if( i<2 ){ + sqlite3_snprintf(sizeof(zErr),zErr, "cannot detach database %s", zName); + goto detach_error; + } + if( !db->autoCommit ){ + sqlite3_snprintf(sizeof(zErr), zErr, + "cannot DETACH database within transaction"); + goto detach_error; + } + if( sqlite3BtreeIsInReadTrans(pDb->pBt) || sqlite3BtreeIsInBackup(pDb->pBt) ){ + sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName); + goto detach_error; + } + + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; + sqlite3ResetAllSchemasOfConnection(db); + return; + +detach_error: + sqlite3_result_error(context, zErr, -1); +} + +/* +** This procedure generates VDBE code for a single invocation of either the +** sqlite_detach() or sqlite_attach() SQL user functions. +*/ +static void codeAttach( + Parse *pParse, /* The parser context */ + int type, /* Either SQLITE_ATTACH or SQLITE_DETACH */ + FuncDef const *pFunc,/* FuncDef wrapper for detachFunc() or attachFunc() */ + Expr *pAuthArg, /* Expression to pass to authorization callback */ + Expr *pFilename, /* Name of database file */ + Expr *pDbname, /* Name of the database to use internally */ + Expr *pKey /* Database key for encryption extension */ +){ + int rc; + NameContext sName; + Vdbe *v; + sqlite3* db = pParse->db; + int regArgs; + + memset(&sName, 0, sizeof(NameContext)); + sName.pParse = pParse; + + if( + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pFilename)) || + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pDbname)) || + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pKey)) + ){ + pParse->nErr++; + goto attach_end; + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pAuthArg ){ + char *zAuthArg; + if( pAuthArg->op==TK_STRING ){ + zAuthArg = pAuthArg->u.zToken; + }else{ + zAuthArg = 0; + } + rc = sqlite3AuthCheck(pParse, type, zAuthArg, 0, 0); + if(rc!=SQLITE_OK ){ + goto attach_end; + } + } +#endif /* SQLITE_OMIT_AUTHORIZATION */ + + + v = sqlite3GetVdbe(pParse); + regArgs = sqlite3GetTempRange(pParse, 4); + sqlite3ExprCode(pParse, pFilename, regArgs); + sqlite3ExprCode(pParse, pDbname, regArgs+1); + sqlite3ExprCode(pParse, pKey, regArgs+2); + + assert( v || db->mallocFailed ); + if( v ){ + sqlite3VdbeAddOp3(v, OP_Function, 0, regArgs+3-pFunc->nArg, regArgs+3); + assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); + sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); + sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); + + /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this + ** statement only). For DETACH, set it to false (expire all existing + ** statements). + */ + sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH)); + } + +attach_end: + sqlite3ExprDelete(db, pFilename); + sqlite3ExprDelete(db, pDbname); + sqlite3ExprDelete(db, pKey); +} + +/* +** Called by the parser to compile a DETACH statement. +** +** DETACH pDbname +*/ +SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ + static const FuncDef detach_func = { + 1, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + detachFunc, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "sqlite_detach", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ + }; + codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); +} + +/* +** Called by the parser to compile an ATTACH statement. +** +** ATTACH p AS pDbname KEY pKey +*/ +SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){ + static const FuncDef attach_func = { + 3, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + attachFunc, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "sqlite_attach", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ + }; + codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); +} +#endif /* SQLITE_OMIT_ATTACH */ + +/* +** Initialize a DbFixer structure. This routine must be called prior +** to passing the structure to one of the sqliteFixAAAA() routines below. +*/ +SQLITE_PRIVATE void sqlite3FixInit( + DbFixer *pFix, /* The fixer to be initialized */ + Parse *pParse, /* Error messages will be written here */ + int iDb, /* This is the database that must be used */ + const char *zType, /* "view", "trigger", or "index" */ + const Token *pName /* Name of the view, trigger, or index */ +){ + sqlite3 *db; + + db = pParse->db; + assert( db->nDb>iDb ); + pFix->pParse = pParse; + pFix->zDb = db->aDb[iDb].zName; + pFix->pSchema = db->aDb[iDb].pSchema; + pFix->zType = zType; + pFix->pName = pName; + pFix->bVarOnly = (iDb==1); +} + +/* +** The following set of routines walk through the parse tree and assign +** a specific database to all table references where the database name +** was left unspecified in the original SQL statement. The pFix structure +** must have been initialized by a prior call to sqlite3FixInit(). +** +** These routines are used to make sure that an index, trigger, or +** view in one database does not refer to objects in a different database. +** (Exception: indices, triggers, and views in the TEMP database are +** allowed to refer to anything.) If a reference is explicitly made +** to an object in a different database, an error message is added to +** pParse->zErrMsg and these routines return non-zero. If everything +** checks out, these routines return 0. +*/ +SQLITE_PRIVATE int sqlite3FixSrcList( + DbFixer *pFix, /* Context of the fixation */ + SrcList *pList /* The Source list to check and modify */ +){ + int i; + const char *zDb; + struct SrcList_item *pItem; + + if( NEVER(pList==0) ) return 0; + zDb = pFix->zDb; + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pFix->bVarOnly==0 ){ + if( pItem->zDatabase && sqlite3StrICmp(pItem->zDatabase, zDb) ){ + sqlite3ErrorMsg(pFix->pParse, + "%s %T cannot reference objects in database %s", + pFix->zType, pFix->pName, pItem->zDatabase); + return 1; + } + sqlite3DbFree(pFix->pParse->db, pItem->zDatabase); + pItem->zDatabase = 0; + pItem->pSchema = pFix->pSchema; + } +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) + if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; + if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1; +#endif + } + return 0; +} +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) +SQLITE_PRIVATE int sqlite3FixSelect( + DbFixer *pFix, /* Context of the fixation */ + Select *pSelect /* The SELECT statement to be fixed to one database */ +){ + while( pSelect ){ + if( sqlite3FixExprList(pFix, pSelect->pEList) ){ + return 1; + } + if( sqlite3FixSrcList(pFix, pSelect->pSrc) ){ + return 1; + } + if( sqlite3FixExpr(pFix, pSelect->pWhere) ){ + return 1; + } + if( sqlite3FixExprList(pFix, pSelect->pGroupBy) ){ + return 1; + } + if( sqlite3FixExpr(pFix, pSelect->pHaving) ){ + return 1; + } + if( sqlite3FixExprList(pFix, pSelect->pOrderBy) ){ + return 1; + } + if( sqlite3FixExpr(pFix, pSelect->pLimit) ){ + return 1; + } + if( sqlite3FixExpr(pFix, pSelect->pOffset) ){ + return 1; + } + pSelect = pSelect->pPrior; + } + return 0; +} +SQLITE_PRIVATE int sqlite3FixExpr( + DbFixer *pFix, /* Context of the fixation */ + Expr *pExpr /* The expression to be fixed to one database */ +){ + while( pExpr ){ + if( pExpr->op==TK_VARIABLE ){ + if( pFix->pParse->db->init.busy ){ + pExpr->op = TK_NULL; + }else{ + sqlite3ErrorMsg(pFix->pParse, "%s cannot use variables", pFix->zType); + return 1; + } + } + if( ExprHasProperty(pExpr, EP_TokenOnly) ) break; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; + }else{ + if( sqlite3FixExprList(pFix, pExpr->x.pList) ) return 1; + } + if( sqlite3FixExpr(pFix, pExpr->pRight) ){ + return 1; + } + pExpr = pExpr->pLeft; + } + return 0; +} +SQLITE_PRIVATE int sqlite3FixExprList( + DbFixer *pFix, /* Context of the fixation */ + ExprList *pList /* The expression to be fixed to one database */ +){ + int i; + struct ExprList_item *pItem; + if( pList==0 ) return 0; + for(i=0, pItem=pList->a; inExpr; i++, pItem++){ + if( sqlite3FixExpr(pFix, pItem->pExpr) ){ + return 1; + } + } + return 0; +} +#endif + +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE int sqlite3FixTriggerStep( + DbFixer *pFix, /* Context of the fixation */ + TriggerStep *pStep /* The trigger step be fixed to one database */ +){ + while( pStep ){ + if( sqlite3FixSelect(pFix, pStep->pSelect) ){ + return 1; + } + if( sqlite3FixExpr(pFix, pStep->pWhere) ){ + return 1; + } + if( sqlite3FixExprList(pFix, pStep->pExprList) ){ + return 1; + } + pStep = pStep->pNext; + } + return 0; +} +#endif + +/************** End of attach.c **********************************************/ +/************** Begin file auth.c ********************************************/ +/* +** 2003 January 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the sqlite3_set_authorizer() +** API. This facility is an optional feature of the library. Embedded +** systems that do not need this facility may omit it by recompiling +** the library with -DSQLITE_OMIT_AUTHORIZATION=1 +*/ + +/* +** All of the code in this file may be omitted by defining a single +** macro. +*/ +#ifndef SQLITE_OMIT_AUTHORIZATION + +/* +** Set or clear the access authorization function. +** +** The access authorization function is be called during the compilation +** phase to verify that the user has read and/or write access permission on +** various fields of the database. The first argument to the auth function +** is a copy of the 3rd argument to this routine. The second argument +** to the auth function is one of these constants: +** +** SQLITE_CREATE_INDEX +** SQLITE_CREATE_TABLE +** SQLITE_CREATE_TEMP_INDEX +** SQLITE_CREATE_TEMP_TABLE +** SQLITE_CREATE_TEMP_TRIGGER +** SQLITE_CREATE_TEMP_VIEW +** SQLITE_CREATE_TRIGGER +** SQLITE_CREATE_VIEW +** SQLITE_DELETE +** SQLITE_DROP_INDEX +** SQLITE_DROP_TABLE +** SQLITE_DROP_TEMP_INDEX +** SQLITE_DROP_TEMP_TABLE +** SQLITE_DROP_TEMP_TRIGGER +** SQLITE_DROP_TEMP_VIEW +** SQLITE_DROP_TRIGGER +** SQLITE_DROP_VIEW +** SQLITE_INSERT +** SQLITE_PRAGMA +** SQLITE_READ +** SQLITE_SELECT +** SQLITE_TRANSACTION +** SQLITE_UPDATE +** +** The third and fourth arguments to the auth function are the name of +** the table and the column that are being accessed. The auth function +** should return either SQLITE_OK, SQLITE_DENY, or SQLITE_IGNORE. If +** SQLITE_OK is returned, it means that access is allowed. SQLITE_DENY +** means that the SQL statement will never-run - the sqlite3_exec() call +** will return with an error. SQLITE_IGNORE means that the SQL statement +** should run but attempts to read the specified column will return NULL +** and attempts to write the column will be ignored. +** +** Setting the auth function to NULL disables this hook. The default +** setting of the auth function is NULL. +*/ +SQLITE_API int sqlite3_set_authorizer( + sqlite3 *db, + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), + void *pArg +){ + sqlite3_mutex_enter(db->mutex); + db->xAuth = xAuth; + db->pAuthArg = pArg; + sqlite3ExpirePreparedStatements(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Write an error message into pParse->zErrMsg that explains that the +** user-supplied authorization function returned an illegal value. +*/ +static void sqliteAuthBadReturnCode(Parse *pParse){ + sqlite3ErrorMsg(pParse, "authorizer malfunction"); + pParse->rc = SQLITE_ERROR; +} + +/* +** Invoke the authorization callback for permission to read column zCol from +** table zTab in database zDb. This function assumes that an authorization +** callback has been registered (i.e. that sqlite3.xAuth is not NULL). +** +** If SQLITE_IGNORE is returned and pExpr is not NULL, then pExpr is changed +** to an SQL NULL expression. Otherwise, if pExpr is NULL, then SQLITE_IGNORE +** is treated as SQLITE_DENY. In this case an error is left in pParse. +*/ +SQLITE_PRIVATE int sqlite3AuthReadCol( + Parse *pParse, /* The parser context */ + const char *zTab, /* Table name */ + const char *zCol, /* Column name */ + int iDb /* Index of containing database. */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + char *zDb = db->aDb[iDb].zName; /* Name of attached database */ + int rc; /* Auth callback return code */ + + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext); + if( rc==SQLITE_DENY ){ + if( db->nDb>2 || iDb!=0 ){ + sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol); + }else{ + sqlite3ErrorMsg(pParse, "access to %s.%s is prohibited", zTab, zCol); + } + pParse->rc = SQLITE_AUTH; + }else if( rc!=SQLITE_IGNORE && rc!=SQLITE_OK ){ + sqliteAuthBadReturnCode(pParse); + } + return rc; +} + +/* +** The pExpr should be a TK_COLUMN expression. The table referred to +** is in pTabList or else it is the NEW or OLD table of a trigger. +** Check to see if it is OK to read this particular column. +** +** If the auth function returns SQLITE_IGNORE, change the TK_COLUMN +** instruction into a TK_NULL. If the auth function returns SQLITE_DENY, +** then generate an error. +*/ +SQLITE_PRIVATE void sqlite3AuthRead( + Parse *pParse, /* The parser context */ + Expr *pExpr, /* The expression to check authorization on */ + Schema *pSchema, /* The schema of the expression */ + SrcList *pTabList /* All table that pExpr might refer to */ +){ + sqlite3 *db = pParse->db; + Table *pTab = 0; /* The table being read */ + const char *zCol; /* Name of the column of the table */ + int iSrc; /* Index in pTabList->a[] of table being read */ + int iDb; /* The index of the database the expression refers to */ + int iCol; /* Index of column in table */ + + if( db->xAuth==0 ) return; + iDb = sqlite3SchemaToIndex(pParse->db, pSchema); + if( iDb<0 ){ + /* An attempt to read a column out of a subquery or other + ** temporary table. */ + return; + } + + assert( pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER ); + if( pExpr->op==TK_TRIGGER ){ + pTab = pParse->pTriggerTab; + }else{ + assert( pTabList ); + for(iSrc=0; ALWAYS(iSrcnSrc); iSrc++){ + if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ + pTab = pTabList->a[iSrc].pTab; + break; + } + } + } + iCol = pExpr->iColumn; + if( NEVER(pTab==0) ) return; + + if( iCol>=0 ){ + assert( iColnCol ); + zCol = pTab->aCol[iCol].zName; + }else if( pTab->iPKey>=0 ){ + assert( pTab->iPKeynCol ); + zCol = pTab->aCol[pTab->iPKey].zName; + }else{ + zCol = "ROWID"; + } + assert( iDb>=0 && iDbnDb ); + if( SQLITE_IGNORE==sqlite3AuthReadCol(pParse, pTab->zName, zCol, iDb) ){ + pExpr->op = TK_NULL; + } +} + +/* +** Do an authorization check using the code and arguments given. Return +** either SQLITE_OK (zero) or SQLITE_IGNORE or SQLITE_DENY. If SQLITE_DENY +** is returned, then the error count and error message in pParse are +** modified appropriately. +*/ +SQLITE_PRIVATE int sqlite3AuthCheck( + Parse *pParse, + int code, + const char *zArg1, + const char *zArg2, + const char *zArg3 +){ + sqlite3 *db = pParse->db; + int rc; + + /* Don't do any authorization checks if the database is initialising + ** or if the parser is being invoked from within sqlite3_declare_vtab. + */ + if( db->init.busy || IN_DECLARE_VTAB ){ + return SQLITE_OK; + } + + if( db->xAuth==0 ){ + return SQLITE_OK; + } + rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext); + if( rc==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized"); + pParse->rc = SQLITE_AUTH; + }else if( rc!=SQLITE_OK && rc!=SQLITE_IGNORE ){ + rc = SQLITE_DENY; + sqliteAuthBadReturnCode(pParse); + } + return rc; +} + +/* +** Push an authorization context. After this routine is called, the +** zArg3 argument to authorization callbacks will be zContext until +** popped. Or if pParse==0, this routine is a no-op. +*/ +SQLITE_PRIVATE void sqlite3AuthContextPush( + Parse *pParse, + AuthContext *pContext, + const char *zContext +){ + assert( pParse ); + pContext->pParse = pParse; + pContext->zAuthContext = pParse->zAuthContext; + pParse->zAuthContext = zContext; +} + +/* +** Pop an authorization context that was previously pushed +** by sqlite3AuthContextPush +*/ +SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ + if( pContext->pParse ){ + pContext->pParse->zAuthContext = pContext->zAuthContext; + pContext->pParse = 0; + } +} + +#endif /* SQLITE_OMIT_AUTHORIZATION */ + +/************** End of auth.c ************************************************/ +/************** Begin file build.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the SQLite parser +** when syntax rules are reduced. The routines in this file handle the +** following kinds of SQL syntax: +** +** CREATE TABLE +** DROP TABLE +** CREATE INDEX +** DROP INDEX +** creating ID lists +** BEGIN TRANSACTION +** COMMIT +** ROLLBACK +*/ + +/* +** This routine is called when a new SQL statement is beginning to +** be parsed. Initialize the pParse structure as needed. +*/ +SQLITE_PRIVATE void sqlite3BeginParse(Parse *pParse, int explainFlag){ + pParse->explain = (u8)explainFlag; + pParse->nVar = 0; +} + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** The TableLock structure is only used by the sqlite3TableLock() and +** codeTableLocks() functions. +*/ +struct TableLock { + int iDb; /* The database containing the table to be locked */ + int iTab; /* The root page of the table to be locked */ + u8 isWriteLock; /* True for write lock. False for a read lock */ + const char *zName; /* Name of the table */ +}; + +/* +** Record the fact that we want to lock a table at run-time. +** +** The table to be locked has root page iTab and is found in database iDb. +** A read or a write lock can be taken depending on isWritelock. +** +** This routine just records the fact that the lock is desired. The +** code to make the lock occur is generated by a later call to +** codeTableLocks() which occurs during sqlite3FinishCoding(). +*/ +SQLITE_PRIVATE void sqlite3TableLock( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database containing the table to lock */ + int iTab, /* Root page number of the table to be locked */ + u8 isWriteLock, /* True for a write lock */ + const char *zName /* Name of the table to be locked */ +){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + int i; + int nBytes; + TableLock *p; + assert( iDb>=0 ); + + for(i=0; inTableLock; i++){ + p = &pToplevel->aTableLock[i]; + if( p->iDb==iDb && p->iTab==iTab ){ + p->isWriteLock = (p->isWriteLock || isWriteLock); + return; + } + } + + nBytes = sizeof(TableLock) * (pToplevel->nTableLock+1); + pToplevel->aTableLock = + sqlite3DbReallocOrFree(pToplevel->db, pToplevel->aTableLock, nBytes); + if( pToplevel->aTableLock ){ + p = &pToplevel->aTableLock[pToplevel->nTableLock++]; + p->iDb = iDb; + p->iTab = iTab; + p->isWriteLock = isWriteLock; + p->zName = zName; + }else{ + pToplevel->nTableLock = 0; + pToplevel->db->mallocFailed = 1; + } +} + +/* +** Code an OP_TableLock instruction for each table locked by the +** statement (configured by calls to sqlite3TableLock()). +*/ +static void codeTableLocks(Parse *pParse){ + int i; + Vdbe *pVdbe; + + pVdbe = sqlite3GetVdbe(pParse); + assert( pVdbe!=0 ); /* sqlite3GetVdbe cannot fail: VDBE already allocated */ + + for(i=0; inTableLock; i++){ + TableLock *p = &pParse->aTableLock[i]; + int p1 = p->iDb; + sqlite3VdbeAddOp4(pVdbe, OP_TableLock, p1, p->iTab, p->isWriteLock, + p->zName, P4_STATIC); + } +} +#else + #define codeTableLocks(x) +#endif + +/* +** Return TRUE if the given yDbMask object is empty - if it contains no +** 1 bits. This routine is used by the DbMaskAllZero() and DbMaskNotZero() +** macros when SQLITE_MAX_ATTACHED is greater than 30. +*/ +#if SQLITE_MAX_ATTACHED>30 +SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){ + int i; + for(i=0; ipToplevel==0 ); + db = pParse->db; + if( db->mallocFailed ) return; + if( pParse->nested ) return; + if( pParse->nErr ) return; + + /* Begin by generating some termination code at the end of the + ** vdbe program + */ + v = sqlite3GetVdbe(pParse); + assert( !pParse->isMultiWrite + || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); + if( v ){ + while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} + sqlite3VdbeAddOp0(v, OP_Halt); + + /* The cookie mask contains one bit for each database file open. + ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are + ** set for each database that is used. Generate code to start a + ** transaction on each used database and to verify the schema cookie + ** on each used database. + */ + if( db->mallocFailed==0 + && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) + ){ + int iDb, i; + assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + for(iDb=0; iDbnDb; iDb++){ + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pParse->cookieValue[iDb], /* P3 */ + db->aDb[iDb].pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=0; inVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; +#endif + + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the + ** shared-cache feature is enabled. + */ + codeTableLocks(pParse); + + /* Initialize any AUTOINCREMENT data structures required. + */ + sqlite3AutoincrementBegin(pParse); + + /* Code constant expressions that where factored out of inner loops */ + if( pParse->pConstExpr ){ + ExprList *pEL = pParse->pConstExpr; + pParse->okConstFactor = 0; + for(i=0; inExpr; i++){ + sqlite3ExprCode(pParse, pEL->a[i].pExpr, pEL->a[i].u.iConstExprReg); + } + } + + /* Finally, jump back to the beginning of the executable code. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, 1); + } + } + + + /* Get the VDBE program ready for execution + */ + if( v && ALWAYS(pParse->nErr==0) && !db->mallocFailed ){ + assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ + /* A minimum of one cursor is required if autoincrement is used + * See ticket [a696379c1f08866] */ + if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; + sqlite3VdbeMakeReady(v, pParse); + pParse->rc = SQLITE_DONE; + pParse->colNamesSet = 0; + }else{ + pParse->rc = SQLITE_ERROR; + } + pParse->nTab = 0; + pParse->nMem = 0; + pParse->nSet = 0; + pParse->nVar = 0; + DbMaskZero(pParse->cookieMask); +} + +/* +** Run the parser and code generator recursively in order to generate +** code for the SQL statement given onto the end of the pParse context +** currently under construction. When the parser is run recursively +** this way, the final OP_Halt is not appended and other initialization +** and finalization steps are omitted because those are handling by the +** outermost parser. +** +** Not everything is nestable. This facility is designed to permit +** INSERT, UPDATE, and DELETE operations against SQLITE_MASTER. Use +** care if you decide to try to use this routine for some other purposes. +*/ +SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ + va_list ap; + char *zSql; + char *zErrMsg = 0; + sqlite3 *db = pParse->db; +# define SAVE_SZ (sizeof(Parse) - offsetof(Parse,nVar)) + char saveBuf[SAVE_SZ]; + + if( pParse->nErr ) return; + assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ + va_start(ap, zFormat); + zSql = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + if( zSql==0 ){ + return; /* A malloc must have failed */ + } + pParse->nested++; + memcpy(saveBuf, &pParse->nVar, SAVE_SZ); + memset(&pParse->nVar, 0, SAVE_SZ); + sqlite3RunParser(pParse, zSql, &zErrMsg); + sqlite3DbFree(db, zErrMsg); + sqlite3DbFree(db, zSql); + memcpy(&pParse->nVar, saveBuf, SAVE_SZ); + pParse->nested--; +} + +/* +** Locate the in-memory structure that describes a particular database +** table given the name of that table and (optionally) the name of the +** database containing the table. Return NULL if not found. +** +** If zDatabase is 0, all databases are searched for the table and the +** first matching table is returned. (No checking for duplicate table +** names is done.) The search order is TEMP first, then MAIN, then any +** auxiliary databases added using the ATTACH command. +** +** See also sqlite3LocateTable(). +*/ +SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ + Table *p = 0; + int i; + int nName; + assert( zName!=0 ); + nName = sqlite3Strlen30(zName); + /* All mutexes are required for schema access. Make sure we hold them. */ + assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); + if( p ) break; + } + return p; +} + +/* +** Locate the in-memory structure that describes a particular database +** table given the name of that table and (optionally) the name of the +** database containing the table. Return NULL if not found. Also leave an +** error message in pParse->zErrMsg. +** +** The difference between this routine and sqlite3FindTable() is that this +** routine leaves an error message in pParse->zErrMsg where +** sqlite3FindTable() does not. +*/ +SQLITE_PRIVATE Table *sqlite3LocateTable( + Parse *pParse, /* context in which to report errors */ + int isView, /* True if looking for a VIEW rather than a TABLE */ + const char *zName, /* Name of the table we are looking for */ + const char *zDbase /* Name of the database. Might be NULL */ +){ + Table *p; + + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return 0; + } + + p = sqlite3FindTable(pParse->db, zName, zDbase); + if( p==0 ){ + const char *zMsg = isView ? "no such view" : "no such table"; + if( zDbase ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + } + pParse->checkSchema = 1; + } + return p; +} + +/* +** Locate the table identified by *p. +** +** This is a wrapper around sqlite3LocateTable(). The difference between +** sqlite3LocateTable() and this function is that this function restricts +** the search to schema (p->pSchema) if it is not NULL. p->pSchema may be +** non-NULL if it is part of a view or trigger program definition. See +** sqlite3FixSrcList() for details. +*/ +SQLITE_PRIVATE Table *sqlite3LocateTableItem( + Parse *pParse, + int isView, + struct SrcList_item *p +){ + const char *zDb; + assert( p->pSchema==0 || p->zDatabase==0 ); + if( p->pSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + zDb = pParse->db->aDb[iDb].zName; + }else{ + zDb = p->zDatabase; + } + return sqlite3LocateTable(pParse, isView, p->zName, zDb); +} + +/* +** Locate the in-memory structure that describes +** a particular index given the name of that index +** and the name of the database that contains the index. +** Return NULL if not found. +** +** If zDatabase is 0, all databases are searched for the +** table and the first matching index is returned. (No checking +** for duplicate index names is done.) The search order is +** TEMP first, then MAIN, then any auxiliary databases added +** using the ATTACH command. +*/ +SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ + Index *p = 0; + int i; + int nName = sqlite3Strlen30(zName); + /* All mutexes are required for schema access. Make sure we hold them. */ + assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + Schema *pSchema = db->aDb[j].pSchema; + assert( pSchema ); + if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&pSchema->idxHash, zName, nName); + if( p ) break; + } + return p; +} + +/* +** Reclaim the memory used by an index +*/ +static void freeIndex(sqlite3 *db, Index *p){ +#ifndef SQLITE_OMIT_ANALYZE + sqlite3DeleteIndexSamples(db, p); +#endif + if( db==0 || db->pnBytesFreed==0 ) sqlite3KeyInfoUnref(p->pKeyInfo); + sqlite3ExprDelete(db, p->pPartIdxWhere); + sqlite3DbFree(db, p->zColAff); + if( p->isResized ) sqlite3DbFree(db, p->azColl); + sqlite3DbFree(db, p); +} + +/* +** For the index called zIdxName which is found in the database iDb, +** unlike that index from its Table then remove the index from +** the index hash table and free all memory structures associated +** with the index. +*/ +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ + Index *pIndex; + int len; + Hash *pHash; + + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pHash = &db->aDb[iDb].pSchema->idxHash; + len = sqlite3Strlen30(zIdxName); + pIndex = sqlite3HashInsert(pHash, zIdxName, len, 0); + if( ALWAYS(pIndex) ){ + if( pIndex->pTable->pIndex==pIndex ){ + pIndex->pTable->pIndex = pIndex->pNext; + }else{ + Index *p; + /* Justification of ALWAYS(); The index must be on the list of + ** indices. */ + p = pIndex->pTable->pIndex; + while( ALWAYS(p) && p->pNext!=pIndex ){ p = p->pNext; } + if( ALWAYS(p && p->pNext==pIndex) ){ + p->pNext = pIndex->pNext; + } + } + freeIndex(db, pIndex); + } + db->flags |= SQLITE_InternChanges; +} + +/* +** Look through the list of open database files in db->aDb[] and if +** any have been closed, remove them from the list. Reallocate the +** db->aDb[] structure to a smaller size, if possible. +** +** Entry 0 (the "main" database) and entry 1 (the "temp" database) +** are never candidates for being collapsed. +*/ +SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3 *db){ + int i, j; + for(i=j=2; inDb; i++){ + struct Db *pDb = &db->aDb[i]; + if( pDb->pBt==0 ){ + sqlite3DbFree(db, pDb->zName); + pDb->zName = 0; + continue; + } + if( jaDb[j] = db->aDb[i]; + } + j++; + } + memset(&db->aDb[j], 0, (db->nDb-j)*sizeof(db->aDb[j])); + db->nDb = j; + if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ + memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); + sqlite3DbFree(db, db->aDb); + db->aDb = db->aDbStatic; + } +} + +/* +** Reset the schema for the database at index iDb. Also reset the +** TEMP schema. +*/ +SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3 *db, int iDb){ + Db *pDb; + assert( iDbnDb ); + + /* Case 1: Reset the single schema identified by iDb */ + pDb = &db->aDb[iDb]; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( pDb->pSchema!=0 ); + sqlite3SchemaClear(pDb->pSchema); + + /* If any database other than TEMP is reset, then also reset TEMP + ** since TEMP might be holding triggers that reference tables in the + ** other database. + */ + if( iDb!=1 ){ + pDb = &db->aDb[1]; + assert( pDb->pSchema!=0 ); + sqlite3SchemaClear(pDb->pSchema); + } + return; +} + +/* +** Erase all schema information from all attached databases (including +** "main" and "temp") for a single database connection. +*/ +SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){ + int i; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Db *pDb = &db->aDb[i]; + if( pDb->pSchema ){ + sqlite3SchemaClear(pDb->pSchema); + } + } + db->flags &= ~SQLITE_InternChanges; + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); + sqlite3CollapseDatabaseArray(db); +} + +/* +** This routine is called when a commit occurs. +*/ +SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ + db->flags &= ~SQLITE_InternChanges; +} + +/* +** Delete memory allocated for the column names of a table or view (the +** Table.aCol[] array). +*/ +static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ + int i; + Column *pCol; + assert( pTable!=0 ); + if( (pCol = pTable->aCol)!=0 ){ + for(i=0; inCol; i++, pCol++){ + sqlite3DbFree(db, pCol->zName); + sqlite3ExprDelete(db, pCol->pDflt); + sqlite3DbFree(db, pCol->zDflt); + sqlite3DbFree(db, pCol->zType); + sqlite3DbFree(db, pCol->zColl); + } + sqlite3DbFree(db, pTable->aCol); + } +} + +/* +** Remove the memory data structures associated with the given +** Table. No changes are made to disk by this routine. +** +** This routine just deletes the data structure. It does not unlink +** the table data structure from the hash table. But it does destroy +** memory structures of the indices and foreign keys associated with +** the table. +** +** The db parameter is optional. It is needed if the Table object +** contains lookaside memory. (Table objects in the schema do not use +** lookaside memory, but some ephemeral Table objects do.) Or the +** db parameter can be used with db->pnBytesFreed to measure the memory +** used by the Table object. +*/ +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ + Index *pIndex, *pNext; + TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */ + + assert( !pTable || pTable->nRef>0 ); + + /* Do not delete the table until the reference count reaches zero. */ + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; + + /* Record the number of outstanding lookaside allocations in schema Tables + ** prior to doing any free() operations. Since schema Tables do not use + ** lookaside, this number should not change. */ + TESTONLY( nLookaside = (db && (pTable->tabFlags & TF_Ephemeral)==0) ? + db->lookaside.nOut : 0 ); + + /* Delete all indices associated with this table. */ + for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ + pNext = pIndex->pNext; + assert( pIndex->pSchema==pTable->pSchema ); + if( !db || db->pnBytesFreed==0 ){ + char *zName = pIndex->zName; + TESTONLY ( Index *pOld = ) sqlite3HashInsert( + &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + ); + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + assert( pOld==pIndex || pOld==0 ); + } + freeIndex(db, pIndex); + } + + /* Delete any foreign keys attached to this table. */ + sqlite3FkDelete(db, pTable); + + /* Delete the Table structure itself. + */ + sqliteDeleteColumnNames(db, pTable); + sqlite3DbFree(db, pTable->zName); + sqlite3DbFree(db, pTable->zColAff); + sqlite3SelectDelete(db, pTable->pSelect); +#ifndef SQLITE_OMIT_CHECK + sqlite3ExprListDelete(db, pTable->pCheck); +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3VtabClear(db, pTable); +#endif + sqlite3DbFree(db, pTable); + + /* Verify that no lookaside memory was used by schema tables */ + assert( nLookaside==0 || nLookaside==db->lookaside.nOut ); +} + +/* +** Unlink the given table from the hash tables and the delete the +** table structure with all its indices and foreign keys. +*/ +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ + Table *p; + Db *pDb; + + assert( db!=0 ); + assert( iDb>=0 && iDbnDb ); + assert( zTabName ); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ + pDb = &db->aDb[iDb]; + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, + sqlite3Strlen30(zTabName),0); + sqlite3DeleteTable(db, p); + db->flags |= SQLITE_InternChanges; +} + +/* +** Given a token, return a string that consists of the text of that +** token. Space to hold the returned string +** is obtained from sqliteMalloc() and must be freed by the calling +** function. +** +** Any quotation marks (ex: "name", 'name', [name], or `name`) that +** surround the body of the token are removed. +** +** Tokens are often just pointers into the original SQL text and so +** are not \000 terminated and are not persistent. The returned string +** is \000 terminated and is persistent. +*/ +SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, Token *pName){ + char *zName; + if( pName ){ + zName = sqlite3DbStrNDup(db, (char*)pName->z, pName->n); + sqlite3Dequote(zName); + }else{ + zName = 0; + } + return zName; +} + +/* +** Open the sqlite_master table stored in database number iDb for +** writing. The table is opened using cursor 0. +*/ +SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *p, int iDb){ + Vdbe *v = sqlite3GetVdbe(p); + sqlite3TableLock(p, iDb, MASTER_ROOT, 1, SCHEMA_TABLE(iDb)); + sqlite3VdbeAddOp4Int(v, OP_OpenWrite, 0, MASTER_ROOT, iDb, 5); + if( p->nTab==0 ){ + p->nTab = 1; + } +} + +/* +** Parameter zName points to a nul-terminated buffer containing the name +** of a database ("main", "temp" or the name of an attached db). This +** function returns the index of the named database in db->aDb[], or +** -1 if the named db cannot be found. +*/ +SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *db, const char *zName){ + int i = -1; /* Database number */ + if( zName ){ + Db *pDb; + int n = sqlite3Strlen30(zName); + for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ + if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite3Strlen30(pDb->zName) && + 0==sqlite3StrICmp(pDb->zName, zName) ){ + break; + } + } + } + return i; +} + +/* +** The token *pName contains the name of a database (either "main" or +** "temp" or the name of an attached db). This routine returns the +** index of the named database in db->aDb[], or -1 if the named db +** does not exist. +*/ +SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){ + int i; /* Database number */ + char *zName; /* Name we are searching for */ + zName = sqlite3NameFromToken(db, pName); + i = sqlite3FindDbName(db, zName); + sqlite3DbFree(db, zName); + return i; +} + +/* The table or view or trigger name is passed to this routine via tokens +** pName1 and pName2. If the table name was fully qualified, for example: +** +** CREATE TABLE xxx.yyy (...); +** +** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if +** the table name is not fully qualified, i.e.: +** +** CREATE TABLE yyy(...); +** +** Then pName1 is set to "yyy" and pName2 is "". +** +** This routine sets the *ppUnqual pointer to point at the token (pName1 or +** pName2) that stores the unqualified table name. The index of the +** database "xxx" is returned. +*/ +SQLITE_PRIVATE int sqlite3TwoPartName( + Parse *pParse, /* Parsing and code generating context */ + Token *pName1, /* The "xxx" in the name "xxx.yyy" or "xxx" */ + Token *pName2, /* The "yyy" in the name "xxx.yyy" */ + Token **pUnqual /* Write the unqualified object name here */ +){ + int iDb; /* Database holding the object */ + sqlite3 *db = pParse->db; + + if( ALWAYS(pName2!=0) && pName2->n>0 ){ + if( db->init.busy ) { + sqlite3ErrorMsg(pParse, "corrupt database"); + pParse->nErr++; + return -1; + } + *pUnqual = pName2; + iDb = sqlite3FindDb(db, pName1); + if( iDb<0 ){ + sqlite3ErrorMsg(pParse, "unknown database %T", pName1); + pParse->nErr++; + return -1; + } + }else{ + assert( db->init.iDb==0 || db->init.busy ); + iDb = db->init.iDb; + *pUnqual = pName1; + } + return iDb; +} + +/* +** This routine is used to check if the UTF-8 string zName is a legal +** unqualified name for a new schema object (table, index, view or +** trigger). All names are legal except those that begin with the string +** "sqlite_" (in upper, lower or mixed case). This portion of the namespace +** is reserved for internal use. +*/ +SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *pParse, const char *zName){ + if( !pParse->db->init.busy && pParse->nested==0 + && (pParse->db->flags & SQLITE_WriteSchema)==0 + && 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ + sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", zName); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +/* +** Return the PRIMARY KEY index of a table +*/ +SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table *pTab){ + Index *p; + for(p=pTab->pIndex; p && !IsPrimaryKeyIndex(p); p=p->pNext){} + return p; +} + +/* +** Return the column of index pIdx that corresponds to table +** column iCol. Return -1 if not found. +*/ +SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index *pIdx, i16 iCol){ + int i; + for(i=0; inColumn; i++){ + if( iCol==pIdx->aiColumn[i] ) return i; + } + return -1; +} + +/* +** Begin constructing a new table representation in memory. This is +** the first of several action routines that get called in response +** to a CREATE TABLE statement. In particular, this routine is called +** after seeing tokens "CREATE" and "TABLE" and the table name. The isTemp +** flag is true if the table should be stored in the auxiliary database +** file instead of in the main database file. This is normally the case +** when the "TEMP" or "TEMPORARY" keyword occurs in between +** CREATE and TABLE. +** +** The new table record is initialized and put in pParse->pNewTable. +** As more of the CREATE TABLE statement is parsed, additional action +** routines will be called to add more information to this record. +** At the end of the CREATE TABLE statement, the sqlite3EndTable() routine +** is called to complete the construction of the new table record. +*/ +SQLITE_PRIVATE void sqlite3StartTable( + Parse *pParse, /* Parser context */ + Token *pName1, /* First part of the name of the table or view */ + Token *pName2, /* Second part of the name of the table or view */ + int isTemp, /* True if this is a TEMP table */ + int isView, /* True if this is a VIEW */ + int isVirtual, /* True if this is a VIRTUAL table */ + int noErr /* Do nothing if table already exists */ +){ + Table *pTable; + char *zName = 0; /* The name of the new table */ + sqlite3 *db = pParse->db; + Vdbe *v; + int iDb; /* Database number to create the table in */ + Token *pName; /* Unqualified name of the table to create */ + + /* The table or view name to create is passed to this routine via tokens + ** pName1 and pName2. If the table name was fully qualified, for example: + ** + ** CREATE TABLE xxx.yyy (...); + ** + ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if + ** the table name is not fully qualified, i.e.: + ** + ** CREATE TABLE yyy(...); + ** + ** Then pName1 is set to "yyy" and pName2 is "". + ** + ** The call below sets the pName pointer to point at the token (pName1 or + ** pName2) that stores the unqualified table name. The variable iDb is + ** set to the index of the database that the table or view is to be + ** created in. + */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) return; + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ + sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); + return; + } + if( !OMIT_TEMPDB && isTemp ) iDb = 1; + + pParse->sNameToken = *pName; + zName = sqlite3NameFromToken(db, pName); + if( zName==0 ) return; + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + goto begin_table_error; + } + if( db->init.iDb==1 ) isTemp = 1; +#ifndef SQLITE_OMIT_AUTHORIZATION + assert( (isTemp & 1)==isTemp ); + { + int code; + char *zDb = db->aDb[iDb].zName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ + goto begin_table_error; + } + if( isView ){ + if( !OMIT_TEMPDB && isTemp ){ + code = SQLITE_CREATE_TEMP_VIEW; + }else{ + code = SQLITE_CREATE_VIEW; + } + }else{ + if( !OMIT_TEMPDB && isTemp ){ + code = SQLITE_CREATE_TEMP_TABLE; + }else{ + code = SQLITE_CREATE_TABLE; + } + } + if( !isVirtual && sqlite3AuthCheck(pParse, code, zName, 0, zDb) ){ + goto begin_table_error; + } + } +#endif + + /* Make sure the new table name does not collide with an existing + ** index or table name in the same database. Issue an error message if + ** it does. The exception is if the statement being parsed was passed + ** to an sqlite3_declare_vtab() call. In that case only the column names + ** and types will be used, so there is no need to test for namespace + ** collisions. + */ + if( !IN_DECLARE_VTAB ){ + char *zDb = db->aDb[iDb].zName; + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto begin_table_error; + } + pTable = sqlite3FindTable(db, zName, zDb); + if( pTable ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "table %T already exists", pName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + } + goto begin_table_error; + } + if( sqlite3FindIndex(db, zName, zDb)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already an index named %s", zName); + goto begin_table_error; + } + } + + pTable = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTable==0 ){ + db->mallocFailed = 1; + pParse->rc = SQLITE_NOMEM; + pParse->nErr++; + goto begin_table_error; + } + pTable->zName = zName; + pTable->iPKey = -1; + pTable->pSchema = db->aDb[iDb].pSchema; + pTable->nRef = 1; + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + assert( pParse->pNewTable==0 ); + pParse->pNewTable = pTable; + + /* If this is the magic sqlite_sequence table used by autoincrement, + ** then record a pointer to this table in the main database structure + ** so that INSERT can find the table easily. + */ +#ifndef SQLITE_OMIT_AUTOINCREMENT + if( !pParse->nested && strcmp(zName, "sqlite_sequence")==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pTable->pSchema->pSeqTab = pTable; + } +#endif + + /* Begin generating the code that will insert the table record into + ** the SQLITE_MASTER table. Note in particular that we must go ahead + ** and allocate the record number for the table entry now. Before any + ** PRIMARY KEY or UNIQUE keywords are parsed. Those keywords will cause + ** indices to be created and the table record must come before the + ** indices. Hence, the record number for the table must be allocated + ** now. + */ + if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ + int j1; + int fileFormat; + int reg1, reg2, reg3; + sqlite3BeginWriteOperation(pParse, 0, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( isVirtual ){ + sqlite3VdbeAddOp0(v, OP_VBegin); + } +#endif + + /* If the file format and encoding in the database have not been set, + ** set them now. + */ + reg1 = pParse->regRowid = ++pParse->nMem; + reg2 = pParse->regRoot = ++pParse->nMem; + reg3 = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); + fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? + 1 : SQLITE_MAX_FILE_FORMAT; + sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, reg3); + sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, reg3); + sqlite3VdbeJumpHere(v, j1); + + /* This just creates a place-holder record in the sqlite_master table. + ** The record created does not contain anything yet. It will be replaced + ** by the real entry in code generated at sqlite3EndTable(). + ** + ** The rowid for the new entry is left in register pParse->regRowid. + ** The root page number of the new table is left in reg pParse->regRoot. + ** The rowid and root page number values are needed by the code that + ** sqlite3EndTable will generate. + */ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) + if( isView || isVirtual ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, reg2); + }else +#endif + { + pParse->addrCrTab = sqlite3VdbeAddOp2(v, OP_CreateTable, iDb, reg2); + } + sqlite3OpenMasterTable(pParse, iDb); + sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); + sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); + sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeAddOp0(v, OP_Close); + } + + /* Normal (non-error) return. */ + return; + + /* If an error occurs, we jump here */ +begin_table_error: + sqlite3DbFree(db, zName); + return; +} + +/* +** This macro is used to compare two strings in a case-insensitive manner. +** It is slightly faster than calling sqlite3StrICmp() directly, but +** produces larger code. +** +** WARNING: This macro is not compatible with the strcmp() family. It +** returns true if the two strings are equal, otherwise false. +*/ +#define STRICMP(x, y) (\ +sqlite3UpperToLower[*(unsigned char *)(x)]== \ +sqlite3UpperToLower[*(unsigned char *)(y)] \ +&& sqlite3StrICmp((x)+1,(y)+1)==0 ) + +/* +** Add a new column to the table currently being constructed. +** +** The parser calls this routine once for each column declaration +** in a CREATE TABLE statement. sqlite3StartTable() gets called +** first to get things going. Then this routine is called for each +** column. +*/ +SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ + Table *p; + int i; + char *z; + Column *pCol; + sqlite3 *db = pParse->db; + if( (p = pParse->pNewTable)==0 ) return; +#if SQLITE_MAX_COLUMN + if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName); + return; + } +#endif + z = sqlite3NameFromToken(db, pName); + if( z==0 ) return; + for(i=0; inCol; i++){ + if( STRICMP(z, p->aCol[i].zName) ){ + sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); + sqlite3DbFree(db, z); + return; + } + } + if( (p->nCol & 0x7)==0 ){ + Column *aNew; + aNew = sqlite3DbRealloc(db,p->aCol,(p->nCol+8)*sizeof(p->aCol[0])); + if( aNew==0 ){ + sqlite3DbFree(db, z); + return; + } + p->aCol = aNew; + } + pCol = &p->aCol[p->nCol]; + memset(pCol, 0, sizeof(p->aCol[0])); + pCol->zName = z; + + /* If there is no type specified, columns have the default affinity + ** 'NONE'. If there is a type specified, then sqlite3AddColumnType() will + ** be called next to set pCol->affinity correctly. + */ + pCol->affinity = SQLITE_AFF_NONE; + pCol->szEst = 1; + p->nCol++; +} + +/* +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. A "NOT NULL" constraint has +** been seen on a column. This routine sets the notNull flag on +** the column currently under construction. +*/ +SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ + Table *p; + p = pParse->pNewTable; + if( p==0 || NEVER(p->nCol<1) ) return; + p->aCol[p->nCol-1].notNull = (u8)onError; +} + +/* +** Scan the column type name zType (length nType) and return the +** associated affinity type. +** +** This routine does a case-independent search of zType for the +** substrings in the following table. If one of the substrings is +** found, the corresponding affinity is returned. If zType contains +** more than one of the substrings, entries toward the top of +** the table take priority. For example, if zType is 'BLOBINT', +** SQLITE_AFF_INTEGER is returned. +** +** Substring | Affinity +** -------------------------------- +** 'INT' | SQLITE_AFF_INTEGER +** 'CHAR' | SQLITE_AFF_TEXT +** 'CLOB' | SQLITE_AFF_TEXT +** 'TEXT' | SQLITE_AFF_TEXT +** 'BLOB' | SQLITE_AFF_NONE +** 'REAL' | SQLITE_AFF_REAL +** 'FLOA' | SQLITE_AFF_REAL +** 'DOUB' | SQLITE_AFF_REAL +** +** If none of the substrings in the above table are found, +** SQLITE_AFF_NUMERIC is returned. +*/ +SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ + u32 h = 0; + char aff = SQLITE_AFF_NUMERIC; + const char *zChar = 0; + + if( zIn==0 ) return aff; + while( zIn[0] ){ + h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; + zIn++; + if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ + aff = SQLITE_AFF_TEXT; + zChar = zIn; + }else if( h==(('c'<<24)+('l'<<16)+('o'<<8)+'b') ){ /* CLOB */ + aff = SQLITE_AFF_TEXT; + }else if( h==(('t'<<24)+('e'<<16)+('x'<<8)+'t') ){ /* TEXT */ + aff = SQLITE_AFF_TEXT; + }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ + && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ + aff = SQLITE_AFF_NONE; + if( zIn[0]=='(' ) zChar = zIn; +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; + }else if( h==(('f'<<24)+('l'<<16)+('o'<<8)+'a') /* FLOA */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; + }else if( h==(('d'<<24)+('o'<<16)+('u'<<8)+'b') /* DOUB */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; +#endif + }else if( (h&0x00FFFFFF)==(('i'<<16)+('n'<<8)+'t') ){ /* INT */ + aff = SQLITE_AFF_INTEGER; + break; + } + } + + /* If pszEst is not NULL, store an estimate of the field size. The + ** estimate is scaled so that the size of an integer is 1. */ + if( pszEst ){ + *pszEst = 1; /* default size is approx 4 bytes */ + if( aff<=SQLITE_AFF_NONE ){ + if( zChar ){ + while( zChar[0] ){ + if( sqlite3Isdigit(zChar[0]) ){ + int v = 0; + sqlite3GetInt32(zChar, &v); + v = v/4 + 1; + if( v>255 ) v = 255; + *pszEst = v; /* BLOB(k), VARCHAR(k), CHAR(k) -> r=(k/4+1) */ + break; + } + zChar++; + } + }else{ + *pszEst = 5; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ + } + } + } + return aff; +} + +/* +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. The pFirst token is the first +** token in the sequence of tokens that describe the type of the +** column currently under construction. pLast is the last token +** in the sequence. Use this information to construct a string +** that contains the typename of the column and store that string +** in zType. +*/ +SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){ + Table *p; + Column *pCol; + + p = pParse->pNewTable; + if( p==0 || NEVER(p->nCol<1) ) return; + pCol = &p->aCol[p->nCol-1]; + assert( pCol->zType==0 ); + pCol->zType = sqlite3NameFromToken(pParse->db, pType); + pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst); +} + +/* +** The expression is the default value for the most recently added column +** of the table currently under construction. +** +** Default value expressions must be constant. Raise an exception if this +** is not the case. +** +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. +*/ +SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ + Table *p; + Column *pCol; + sqlite3 *db = pParse->db; + p = pParse->pNewTable; + if( p!=0 ){ + pCol = &(p->aCol[p->nCol-1]); + if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){ + sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", + pCol->zName); + }else{ + /* A copy of pExpr is used instead of the original, as pExpr contains + ** tokens that point to volatile memory. The 'span' of the expression + ** is required by pragma table_info. + */ + sqlite3ExprDelete(db, pCol->pDflt); + pCol->pDflt = sqlite3ExprDup(db, pSpan->pExpr, EXPRDUP_REDUCE); + sqlite3DbFree(db, pCol->zDflt); + pCol->zDflt = sqlite3DbStrNDup(db, (char*)pSpan->zStart, + (int)(pSpan->zEnd - pSpan->zStart)); + } + } + sqlite3ExprDelete(db, pSpan->pExpr); +} + +/* +** Designate the PRIMARY KEY for the table. pList is a list of names +** of columns that form the primary key. If pList is NULL, then the +** most recently added column of the table is the primary key. +** +** A table can have at most one primary key. If the table already has +** a primary key (and this is the second primary key) then create an +** error. +** +** If the PRIMARY KEY is on a single column whose datatype is INTEGER, +** then we will try to use that column as the rowid. Set the Table.iPKey +** field of the table under construction to be the index of the +** INTEGER PRIMARY KEY column. Table.iPKey is set to -1 if there is +** no INTEGER PRIMARY KEY. +** +** If the key is not an INTEGER PRIMARY KEY, then create a unique +** index for the key. No index is created for INTEGER PRIMARY KEYs. +*/ +SQLITE_PRIVATE void sqlite3AddPrimaryKey( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List of field names to be indexed */ + int onError, /* What to do with a uniqueness conflict */ + int autoInc, /* True if the AUTOINCREMENT keyword is present */ + int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ +){ + Table *pTab = pParse->pNewTable; + char *zType = 0; + int iCol = -1, i; + int nTerm; + if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; + if( pTab->tabFlags & TF_HasPrimaryKey ){ + sqlite3ErrorMsg(pParse, + "table \"%s\" has more than one primary key", pTab->zName); + goto primary_key_exit; + } + pTab->tabFlags |= TF_HasPrimaryKey; + if( pList==0 ){ + iCol = pTab->nCol - 1; + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; + zType = pTab->aCol[iCol].zType; + nTerm = 1; + }else{ + nTerm = pList->nExpr; + for(i=0; inCol; iCol++){ + if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; + zType = pTab->aCol[iCol].zType; + break; + } + } + } + } + if( nTerm==1 + && zType && sqlite3StrICmp(zType, "INTEGER")==0 + && sortOrder==SQLITE_SO_ASC + ){ + pTab->iPKey = iCol; + pTab->keyConf = (u8)onError; + assert( autoInc==0 || autoInc==1 ); + pTab->tabFlags |= autoInc*TF_Autoincrement; + if( pList ) pParse->iPkSortOrder = pList->a[0].sortOrder; + }else if( autoInc ){ +#ifndef SQLITE_OMIT_AUTOINCREMENT + sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " + "INTEGER PRIMARY KEY"); +#endif + }else{ + Vdbe *v = pParse->pVdbe; + Index *p; + if( v ) pParse->addrSkipPK = sqlite3VdbeAddOp0(v, OP_Noop); + p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, + 0, sortOrder, 0); + if( p ){ + p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; + if( v ) sqlite3VdbeJumpHere(v, pParse->addrSkipPK); + } + pList = 0; + } + +primary_key_exit: + sqlite3ExprListDelete(pParse->db, pList); + return; +} + +/* +** Add a new CHECK constraint to the table currently under construction. +*/ +SQLITE_PRIVATE void sqlite3AddCheckConstraint( + Parse *pParse, /* Parsing context */ + Expr *pCheckExpr /* The check expression */ +){ +#ifndef SQLITE_OMIT_CHECK + Table *pTab = pParse->pNewTable; + sqlite3 *db = pParse->db; + if( pTab && !IN_DECLARE_VTAB + && !sqlite3BtreeIsReadonly(db->aDb[db->init.iDb].pBt) + ){ + pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); + if( pParse->constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + } + }else +#endif + { + sqlite3ExprDelete(pParse->db, pCheckExpr); + } +} + +/* +** Set the collation function of the most recently parsed table column +** to the CollSeq given. +*/ +SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ + Table *p; + int i; + char *zColl; /* Dequoted name of collation sequence */ + sqlite3 *db; + + if( (p = pParse->pNewTable)==0 ) return; + i = p->nCol-1; + db = pParse->db; + zColl = sqlite3NameFromToken(db, pToken); + if( !zColl ) return; + + if( sqlite3LocateCollSeq(pParse, zColl) ){ + Index *pIdx; + sqlite3DbFree(db, p->aCol[i].zColl); + p->aCol[i].zColl = zColl; + + /* If the column is declared as " PRIMARY KEY COLLATE ", + ** then an index may have been created on this column before the + ** collation type was added. Correct this if it is the case. + */ + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->nKeyCol==1 ); + if( pIdx->aiColumn[0]==i ){ + pIdx->azColl[0] = p->aCol[i].zColl; + } + } + }else{ + sqlite3DbFree(db, zColl); + } +} + +/* +** This function returns the collation sequence for database native text +** encoding identified by the string zName, length nName. +** +** If the requested collation sequence is not available, or not available +** in the database native encoding, the collation factory is invoked to +** request it. If the collation factory does not supply such a sequence, +** and the sequence is available in another text encoding, then that is +** returned instead. +** +** If no versions of the requested collations sequence are available, or +** another error occurs, NULL is returned and an error message written into +** pParse. +** +** This routine is a wrapper around sqlite3FindCollSeq(). This routine +** invokes the collation factory if the named collation cannot be found +** and generates an error message. +** +** See also: sqlite3FindCollSeq(), sqlite3GetCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ + sqlite3 *db = pParse->db; + u8 enc = ENC(db); + u8 initbusy = db->init.busy; + CollSeq *pColl; + + pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); + if( !initbusy && (!pColl || !pColl->xCmp) ){ + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); + } + + return pColl; +} + + +/* +** Generate code that will increment the schema cookie. +** +** The schema cookie is used to determine when the schema for the +** database changes. After each schema change, the cookie value +** changes. When a process first reads the schema it records the +** cookie. Thereafter, whenever it goes to access the database, +** it checks the cookie to make sure the schema has not changed +** since it was last read. +** +** This plan is not completely bullet-proof. It is possible for +** the schema to change multiple times and for the cookie to be +** set back to prior value. But schema changes are infrequent +** and the probability of hitting the same cookie value is only +** 1 chance in 2^32. So we're safe enough. +*/ +SQLITE_PRIVATE void sqlite3ChangeCookie(Parse *pParse, int iDb){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + sqlite3VdbeAddOp2(v, OP_Integer, db->aDb[iDb].pSchema->schema_cookie+1, r1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, r1); + sqlite3ReleaseTempReg(pParse, r1); +} + +/* +** Measure the number of characters needed to output the given +** identifier. The number returned includes any quotes used +** but does not include the null terminator. +** +** The estimate is conservative. It might be larger that what is +** really needed. +*/ +static int identLength(const char *z){ + int n; + for(n=0; *z; n++, z++){ + if( *z=='"' ){ n++; } + } + return n + 2; +} + +/* +** The first parameter is a pointer to an output buffer. The second +** parameter is a pointer to an integer that contains the offset at +** which to write into the output buffer. This function copies the +** nul-terminated string pointed to by the third parameter, zSignedIdent, +** to the specified offset in the buffer and updates *pIdx to refer +** to the first byte after the last byte written before returning. +** +** If the string zSignedIdent consists entirely of alpha-numeric +** characters, does not begin with a digit and is not an SQL keyword, +** then it is copied to the output buffer exactly as it is. Otherwise, +** it is quoted using double-quotes. +*/ +static void identPut(char *z, int *pIdx, char *zSignedIdent){ + unsigned char *zIdent = (unsigned char*)zSignedIdent; + int i, j, needQuote; + i = *pIdx; + + for(j=0; zIdent[j]; j++){ + if( !sqlite3Isalnum(zIdent[j]) && zIdent[j]!='_' ) break; + } + needQuote = sqlite3Isdigit(zIdent[0]) + || sqlite3KeywordCode(zIdent, j)!=TK_ID + || zIdent[j]!=0 + || j==0; + + if( needQuote ) z[i++] = '"'; + for(j=0; zIdent[j]; j++){ + z[i++] = zIdent[j]; + if( zIdent[j]=='"' ) z[i++] = '"'; + } + if( needQuote ) z[i++] = '"'; + z[i] = 0; + *pIdx = i; +} + +/* +** Generate a CREATE TABLE statement appropriate for the given +** table. Memory to hold the text of the statement is obtained +** from sqliteMalloc() and must be freed by the calling function. +*/ +static char *createTableStmt(sqlite3 *db, Table *p){ + int i, k, n; + char *zStmt; + char *zSep, *zSep2, *zEnd; + Column *pCol; + n = 0; + for(pCol = p->aCol, i=0; inCol; i++, pCol++){ + n += identLength(pCol->zName) + 5; + } + n += identLength(p->zName); + if( n<50 ){ + zSep = ""; + zSep2 = ","; + zEnd = ")"; + }else{ + zSep = "\n "; + zSep2 = ",\n "; + zEnd = "\n)"; + } + n += 35 + 6*p->nCol; + zStmt = sqlite3DbMallocRaw(0, n); + if( zStmt==0 ){ + db->mallocFailed = 1; + return 0; + } + sqlite3_snprintf(n, zStmt, "CREATE TABLE "); + k = sqlite3Strlen30(zStmt); + identPut(zStmt, &k, p->zName); + zStmt[k++] = '('; + for(pCol=p->aCol, i=0; inCol; i++, pCol++){ + static const char * const azType[] = { + /* SQLITE_AFF_TEXT */ " TEXT", + /* SQLITE_AFF_NONE */ "", + /* SQLITE_AFF_NUMERIC */ " NUM", + /* SQLITE_AFF_INTEGER */ " INT", + /* SQLITE_AFF_REAL */ " REAL" + }; + int len; + const char *zType; + + sqlite3_snprintf(n-k, &zStmt[k], zSep); + k += sqlite3Strlen30(&zStmt[k]); + zSep = zSep2; + identPut(zStmt, &k, pCol->zName); + assert( pCol->affinity-SQLITE_AFF_TEXT >= 0 ); + assert( pCol->affinity-SQLITE_AFF_TEXT < ArraySize(azType) ); + testcase( pCol->affinity==SQLITE_AFF_TEXT ); + testcase( pCol->affinity==SQLITE_AFF_NONE ); + testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); + testcase( pCol->affinity==SQLITE_AFF_INTEGER ); + testcase( pCol->affinity==SQLITE_AFF_REAL ); + + zType = azType[pCol->affinity - SQLITE_AFF_TEXT]; + len = sqlite3Strlen30(zType); + assert( pCol->affinity==SQLITE_AFF_NONE + || pCol->affinity==sqlite3AffinityType(zType, 0) ); + memcpy(&zStmt[k], zType, len); + k += len; + assert( k<=n ); + } + sqlite3_snprintf(n-k, &zStmt[k], "%s", zEnd); + return zStmt; +} + +/* +** Resize an Index object to hold N columns total. Return SQLITE_OK +** on success and SQLITE_NOMEM on an OOM error. +*/ +static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ + char *zExtra; + int nByte; + if( pIdx->nColumn>=N ) return SQLITE_OK; + assert( pIdx->isResized==0 ); + nByte = (sizeof(char*) + sizeof(i16) + 1)*N; + zExtra = sqlite3DbMallocZero(db, nByte); + if( zExtra==0 ) return SQLITE_NOMEM; + memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); + pIdx->azColl = (char**)zExtra; + zExtra += sizeof(char*)*N; + memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn); + pIdx->aiColumn = (i16*)zExtra; + zExtra += sizeof(i16)*N; + memcpy(zExtra, pIdx->aSortOrder, pIdx->nColumn); + pIdx->aSortOrder = (u8*)zExtra; + pIdx->nColumn = N; + pIdx->isResized = 1; + return SQLITE_OK; +} + +/* +** Estimate the total row width for a table. +*/ +static void estimateTableWidth(Table *pTab){ + unsigned wTable = 0; + const Column *pTabCol; + int i; + for(i=pTab->nCol, pTabCol=pTab->aCol; i>0; i--, pTabCol++){ + wTable += pTabCol->szEst; + } + if( pTab->iPKey<0 ) wTable++; + pTab->szTabRow = sqlite3LogEst(wTable*4); +} + +/* +** Estimate the average size of a row for an index. +*/ +static void estimateIndexWidth(Index *pIdx){ + unsigned wIndex = 0; + int i; + const Column *aCol = pIdx->pTable->aCol; + for(i=0; inColumn; i++){ + i16 x = pIdx->aiColumn[i]; + assert( xpTable->nCol ); + wIndex += x<0 ? 1 : aCol[pIdx->aiColumn[i]].szEst; + } + pIdx->szIdxRow = sqlite3LogEst(wIndex*4); +} + +/* Return true if value x is found any of the first nCol entries of aiCol[] +*/ +static int hasColumn(const i16 *aiCol, int nCol, int x){ + while( nCol-- > 0 ) if( x==*(aiCol++) ) return 1; + return 0; +} + +/* +** This routine runs at the end of parsing a CREATE TABLE statement that +** has a WITHOUT ROWID clause. The job of this routine is to convert both +** internal schema data structures and the generated VDBE code so that they +** are appropriate for a WITHOUT ROWID table instead of a rowid table. +** Changes include: +** +** (1) Convert the OP_CreateTable into an OP_CreateIndex. There is +** no rowid btree for a WITHOUT ROWID. Instead, the canonical +** data storage is a covering index btree. +** (2) Bypass the creation of the sqlite_master table entry +** for the PRIMARY KEY as the the primary key index is now +** identified by the sqlite_master table entry of the table itself. +** (3) Set the Index.tnum of the PRIMARY KEY Index object in the +** schema to the rootpage from the main table. +** (4) Set all columns of the PRIMARY KEY schema object to be NOT NULL. +** (5) Add all table columns to the PRIMARY KEY Index object +** so that the PRIMARY KEY is a covering index. The surplus +** columns are part of KeyInfo.nXField and are not used for +** sorting or lookup or uniqueness checks. +** (6) Replace the rowid tail on all automatically generated UNIQUE +** indices with the PRIMARY KEY columns. +*/ +static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ + Index *pIdx; + Index *pPk; + int nPk; + int i, j; + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + + /* Convert the OP_CreateTable opcode that would normally create the + ** root-page for the table into a OP_CreateIndex opcode. The index + ** created will become the PRIMARY KEY index. + */ + if( pParse->addrCrTab ){ + assert( v ); + sqlite3VdbeGetOp(v, pParse->addrCrTab)->opcode = OP_CreateIndex; + } + + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master + ** table entry. + */ + if( pParse->addrSkipPK ){ + assert( v ); + sqlite3VdbeGetOp(v, pParse->addrSkipPK)->opcode = OP_Goto; + } + + /* Locate the PRIMARY KEY index. Or, if this table was originally + ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. + */ + if( pTab->iPKey>=0 ){ + ExprList *pList; + pList = sqlite3ExprListAppend(pParse, 0, 0); + if( pList==0 ) return; + pList->a[0].zName = sqlite3DbStrDup(pParse->db, + pTab->aCol[pTab->iPKey].zName); + pList->a[0].sortOrder = pParse->iPkSortOrder; + assert( pParse->pNewTable==pTab ); + pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); + if( pPk==0 ) return; + pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY; + pTab->iPKey = -1; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + } + pPk->isCovering = 1; + assert( pPk!=0 ); + nPk = pPk->nKeyCol; + + /* Make sure every column of the PRIMARY KEY is NOT NULL */ + for(i=0; iaCol[pPk->aiColumn[i]].notNull = 1; + } + pPk->uniqNotNull = 1; + + /* The root page of the PRIMARY KEY is the table root page */ + pPk->tnum = pTab->tnum; + + /* Update the in-memory representation of all UNIQUE indices by converting + ** the final rowid column into one or more columns of the PRIMARY KEY. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int n; + if( IsPrimaryKeyIndex(pIdx) ) continue; + for(i=n=0; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ) n++; + } + if( n==0 ){ + /* This index is a superset of the primary key */ + pIdx->nColumn = pIdx->nKeyCol; + continue; + } + if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; + for(i=0, j=pIdx->nKeyCol; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ){ + pIdx->aiColumn[j] = pPk->aiColumn[i]; + pIdx->azColl[j] = pPk->azColl[i]; + j++; + } + } + assert( pIdx->nColumn>=pIdx->nKeyCol+n ); + assert( pIdx->nColumn>=j ); + } + + /* Add all table columns to the PRIMARY KEY index + */ + if( nPknCol ){ + if( resizeIndexObject(db, pPk, pTab->nCol) ) return; + for(i=0, j=nPk; inCol; i++){ + if( !hasColumn(pPk->aiColumn, j, i) ){ + assert( jnColumn ); + pPk->aiColumn[j] = i; + pPk->azColl[j] = "BINARY"; + j++; + } + } + assert( pPk->nColumn==j ); + assert( pTab->nCol==j ); + }else{ + pPk->nColumn = pTab->nCol; + } +} + +/* +** This routine is called to report the final ")" that terminates +** a CREATE TABLE statement. +** +** The table structure that other action routines have been building +** is added to the internal hash tables, assuming no errors have +** occurred. +** +** An entry for the table is made in the master table on disk, unless +** this is a temporary table or db->init.busy==1. When db->init.busy==1 +** it means we are reading the sqlite_master table because we just +** connected to the database or because the sqlite_master table has +** recently changed, so the entry for this table already exists in +** the sqlite_master table. We do not want to create it again. +** +** If the pSelect argument is not NULL, it means that this routine +** was called to create a table generated from a +** "CREATE TABLE ... AS SELECT ..." statement. The column names of +** the new table will match the result set of the SELECT. +*/ +SQLITE_PRIVATE void sqlite3EndTable( + Parse *pParse, /* Parse context */ + Token *pCons, /* The ',' token after the last column defn. */ + Token *pEnd, /* The ')' before options in the CREATE TABLE */ + u8 tabOpts, /* Extra table options. Usually 0. */ + Select *pSelect /* Select from a "CREATE ... AS SELECT" */ +){ + Table *p; /* The new table */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb; /* Database in which the table lives */ + Index *pIdx; /* An implied index of the table */ + + if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ + return; + } + p = pParse->pNewTable; + if( p==0 ) return; + + assert( !db->init.busy || !pSelect ); + + /* If the db->init.busy is 1 it means we are reading the SQL off the + ** "sqlite_master" or "sqlite_temp_master" table on the disk. + ** So do not write to the disk again. Extract the root page number + ** for the table from the db->init.newTnum field. (The page number + ** should have been put there by the sqliteOpenCb routine.) + */ + if( db->init.busy ){ + p->tnum = db->init.newTnum; + } + + /* Special processing for WITHOUT ROWID Tables */ + if( tabOpts & TF_WithoutRowid ){ + if( (p->tabFlags & TF_Autoincrement) ){ + sqlite3ErrorMsg(pParse, + "AUTOINCREMENT not allowed on WITHOUT ROWID tables"); + return; + } + if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ + sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); + }else{ + p->tabFlags |= TF_WithoutRowid; + convertToWithoutRowidTable(pParse, p); + } + } + + iDb = sqlite3SchemaToIndex(db, p->pSchema); + +#ifndef SQLITE_OMIT_CHECK + /* Resolve names in all CHECK constraint expressions. + */ + if( p->pCheck ){ + sqlite3ResolveSelfReference(pParse, p, NC_IsCheck, 0, p->pCheck); + } +#endif /* !defined(SQLITE_OMIT_CHECK) */ + + /* Estimate the average row size for the table and for all implied indices */ + estimateTableWidth(p); + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + estimateIndexWidth(pIdx); + } + + /* If not initializing, then create a record for the new table + ** in the SQLITE_MASTER table of the database. + ** + ** If this is a TEMPORARY table, write the entry into the auxiliary + ** file instead of into the main database file. + */ + if( !db->init.busy ){ + int n; + Vdbe *v; + char *zType; /* "view" or "table" */ + char *zType2; /* "VIEW" or "TABLE" */ + char *zStmt; /* Text of the CREATE TABLE or CREATE VIEW statement */ + + v = sqlite3GetVdbe(pParse); + if( NEVER(v==0) ) return; + + sqlite3VdbeAddOp1(v, OP_Close, 0); + + /* + ** Initialize zType for the new view or table. + */ + if( p->pSelect==0 ){ + /* A regular table */ + zType = "table"; + zType2 = "TABLE"; +#ifndef SQLITE_OMIT_VIEW + }else{ + /* A view */ + zType = "view"; + zType2 = "VIEW"; +#endif + } + + /* If this is a CREATE TABLE xx AS SELECT ..., execute the SELECT + ** statement to populate the new table. The root-page number for the + ** new table is in register pParse->regRoot. + ** + ** Once the SELECT has been coded by sqlite3Select(), it is in a + ** suitable state to query for the column names and types to be used + ** by the new table. + ** + ** A shared-cache write-lock is not required to write to the new table, + ** as a schema-lock must have already been obtained to create it. Since + ** a schema-lock excludes all other database users, the write-lock would + ** be redundant. + */ + if( pSelect ){ + SelectDest dest; + Table *pSelTab; + + assert(pParse->nTab==1); + sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); + pParse->nTab = 2; + sqlite3SelectDestInit(&dest, SRT_Table, 1); + sqlite3Select(pParse, pSelect, &dest); + sqlite3VdbeAddOp1(v, OP_Close, 1); + if( pParse->nErr==0 ){ + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); + if( pSelTab==0 ) return; + assert( p->aCol==0 ); + p->nCol = pSelTab->nCol; + p->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + } + } + + /* Compute the complete text of the CREATE statement */ + if( pSelect ){ + zStmt = createTableStmt(db, p); + }else{ + Token *pEnd2 = tabOpts ? &pParse->sLastToken : pEnd; + n = (int)(pEnd2->z - pParse->sNameToken.z); + if( pEnd2->z[0]!=';' ) n += pEnd2->n; + zStmt = sqlite3MPrintf(db, + "CREATE %s %.*s", zType2, n, pParse->sNameToken.z + ); + } + + /* A slot for the record has already been allocated in the + ** SQLITE_MASTER table. We just need to update that slot with all + ** the information we've collected. + */ + sqlite3NestedParse(pParse, + "UPDATE %Q.%s " + "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q " + "WHERE rowid=#%d", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + zType, + p->zName, + p->zName, + pParse->regRoot, + zStmt, + pParse->regRowid + ); + sqlite3DbFree(db, zStmt); + sqlite3ChangeCookie(pParse, iDb); + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* Check to see if we need to create an sqlite_sequence table for + ** keeping track of autoincrement keys. + */ + if( p->tabFlags & TF_Autoincrement ){ + Db *pDb = &db->aDb[iDb]; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( pDb->pSchema->pSeqTab==0 ){ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.sqlite_sequence(name,seq)", + pDb->zName + ); + } + } +#endif + + /* Reparse everything to update our internal data structures */ + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName)); + } + + + /* Add the table to the in-memory representation of the database. + */ + if( db->init.busy ){ + Table *pOld; + Schema *pSchema = p->pSchema; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, + sqlite3Strlen30(p->zName),p); + if( pOld ){ + assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ + db->mallocFailed = 1; + return; + } + pParse->pNewTable = 0; + db->flags |= SQLITE_InternChanges; + +#ifndef SQLITE_OMIT_ALTERTABLE + if( !p->pSelect ){ + const char *zName = (const char *)pParse->sNameToken.z; + int nName; + assert( !pSelect && pCons && pEnd ); + if( pCons->z==0 ){ + pCons = pEnd; + } + nName = (int)((const char *)pCons->z - zName); + p->addColOffset = 13 + sqlite3Utf8CharLen(zName, nName); + } +#endif + } +} + +#ifndef SQLITE_OMIT_VIEW +/* +** The parser calls this routine in order to create a new VIEW +*/ +SQLITE_PRIVATE void sqlite3CreateView( + Parse *pParse, /* The parsing context */ + Token *pBegin, /* The CREATE token that begins the statement */ + Token *pName1, /* The token that holds the name of the view */ + Token *pName2, /* The token that holds the name of the view */ + Select *pSelect, /* A SELECT statement that will become the new view */ + int isTemp, /* TRUE for a TEMPORARY view */ + int noErr /* Suppress error messages if VIEW already exists */ +){ + Table *p; + int n; + const char *z; + Token sEnd; + DbFixer sFix; + Token *pName = 0; + int iDb; + sqlite3 *db = pParse->db; + + if( pParse->nVar>0 ){ + sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); + sqlite3SelectDelete(db, pSelect); + return; + } + sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); + p = pParse->pNewTable; + if( p==0 || pParse->nErr ){ + sqlite3SelectDelete(db, pSelect); + return; + } + sqlite3TwoPartName(pParse, pName1, pName2, &pName); + iDb = sqlite3SchemaToIndex(db, p->pSchema); + sqlite3FixInit(&sFix, pParse, iDb, "view", pName); + if( sqlite3FixSelect(&sFix, pSelect) ){ + sqlite3SelectDelete(db, pSelect); + return; + } + + /* Make a copy of the entire SELECT statement that defines the view. + ** This will force all the Expr.token.z values to be dynamically + ** allocated rather than point to the input string - which means that + ** they will persist after the current sqlite3_exec() call returns. + */ + p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + sqlite3SelectDelete(db, pSelect); + if( db->mallocFailed ){ + return; + } + if( !db->init.busy ){ + sqlite3ViewGetColumnNames(pParse, p); + } + + /* Locate the end of the CREATE VIEW statement. Make sEnd point to + ** the end. + */ + sEnd = pParse->sLastToken; + if( ALWAYS(sEnd.z[0]!=0) && sEnd.z[0]!=';' ){ + sEnd.z += sEnd.n; + } + sEnd.n = 0; + n = (int)(sEnd.z - pBegin->z); + z = pBegin->z; + while( ALWAYS(n>0) && sqlite3Isspace(z[n-1]) ){ n--; } + sEnd.z = &z[n-1]; + sEnd.n = 1; + + /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ + sqlite3EndTable(pParse, 0, &sEnd, 0, 0); + return; +} +#endif /* SQLITE_OMIT_VIEW */ + +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +/* +** The Table structure pTable is really a VIEW. Fill in the names of +** the columns of the view in the pTable structure. Return the number +** of errors. If an error is seen leave an error message in pParse->zErrMsg. +*/ +SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ + Table *pSelTab; /* A fake table from which we get the result set */ + Select *pSel; /* Copy of the SELECT that implements the view */ + int nErr = 0; /* Number of errors encountered */ + int n; /* Temporarily holds the number of cursors assigned */ + sqlite3 *db = pParse->db; /* Database connection for malloc errors */ + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + + assert( pTable ); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3VtabCallConnect(pParse, pTable) ){ + return SQLITE_ERROR; + } + if( IsVirtual(pTable) ) return 0; +#endif + +#ifndef SQLITE_OMIT_VIEW + /* A positive nCol means the columns names for this view are + ** already known. + */ + if( pTable->nCol>0 ) return 0; + + /* A negative nCol is a special marker meaning that we are currently + ** trying to compute the column names. If we enter this routine with + ** a negative nCol, it means two or more views form a loop, like this: + ** + ** CREATE VIEW one AS SELECT * FROM two; + ** CREATE VIEW two AS SELECT * FROM one; + ** + ** Actually, the error above is now caught prior to reaching this point. + ** But the following test is still important as it does come up + ** in the following: + ** + ** CREATE TABLE main.ex1(a); + ** CREATE TEMP VIEW ex1 AS SELECT a FROM ex1; + ** SELECT * FROM temp.ex1; + */ + if( pTable->nCol<0 ){ + sqlite3ErrorMsg(pParse, "view %s is circularly defined", pTable->zName); + return 1; + } + assert( pTable->nCol>=0 ); + + /* If we get this far, it means we need to compute the table names. + ** Note that the call to sqlite3ResultSetOfSelect() will expand any + ** "*" elements in the results set of the view and will assign cursors + ** to the elements of the FROM clause. But we do not want these changes + ** to be permanent. So the computation is done on a copy of the SELECT + ** statement that defines the view. + */ + assert( pTable->pSelect ); + pSel = sqlite3SelectDup(db, pTable->pSelect, 0); + if( pSel ){ + u8 enableLookaside = db->lookaside.bEnabled; + n = pParse->nTab; + sqlite3SrcListAssignCursors(pParse, pSel->pSrc); + pTable->nCol = -1; + db->lookaside.bEnabled = 0; +#ifndef SQLITE_OMIT_AUTHORIZATION + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + db->xAuth = xAuth; +#else + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); +#endif + db->lookaside.bEnabled = enableLookaside; + pParse->nTab = n; + if( pSelTab ){ + assert( pTable->aCol==0 ); + pTable->nCol = pSelTab->nCol; + pTable->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + pTable->pSchema->schemaFlags |= DB_UnresetViews; + }else{ + pTable->nCol = 0; + nErr++; + } + sqlite3SelectDelete(db, pSel); + } else { + nErr++; + } +#endif /* SQLITE_OMIT_VIEW */ + return nErr; +} +#endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ + +#ifndef SQLITE_OMIT_VIEW +/* +** Clear the column names from every VIEW in database idx. +*/ +static void sqliteViewResetAll(sqlite3 *db, int idx){ + HashElem *i; + assert( sqlite3SchemaMutexHeld(db, idx, 0) ); + if( !DbHasProperty(db, idx, DB_UnresetViews) ) return; + for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + if( pTab->pSelect ){ + sqliteDeleteColumnNames(db, pTab); + pTab->aCol = 0; + pTab->nCol = 0; + } + } + DbClearProperty(db, idx, DB_UnresetViews); +} +#else +# define sqliteViewResetAll(A,B) +#endif /* SQLITE_OMIT_VIEW */ + +/* +** This function is called by the VDBE to adjust the internal schema +** used by SQLite when the btree layer moves a table root page. The +** root-page of a table or index in database iDb has changed from iFrom +** to iTo. +** +** Ticket #1728: The symbol table might still contain information +** on tables and/or indices that are the process of being deleted. +** If you are unlucky, one of those deleted indices or tables might +** have the same rootpage number as the real table or index that is +** being moved. So we cannot stop searching after the first match +** because the first match might be for one of the deleted indices +** or tables and not the table/index that is actually being moved. +** We must continue looping until all tables and indices with +** rootpage==iFrom have been converted to have a rootpage of iTo +** in order to be certain that we got the right one. +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, int iFrom, int iTo){ + HashElem *pElem; + Hash *pHash; + Db *pDb; + + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pDb = &db->aDb[iDb]; + pHash = &pDb->pSchema->tblHash; + for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ + Table *pTab = sqliteHashData(pElem); + if( pTab->tnum==iFrom ){ + pTab->tnum = iTo; + } + } + pHash = &pDb->pSchema->idxHash; + for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ + Index *pIdx = sqliteHashData(pElem); + if( pIdx->tnum==iFrom ){ + pIdx->tnum = iTo; + } + } +} +#endif + +/* +** Write code to erase the table with root-page iTable from database iDb. +** Also write code to modify the sqlite_master table and internal schema +** if a root-page of another table is moved by the btree-layer whilst +** erasing iTable (this can happen with an auto-vacuum database). +*/ +static void destroyRootPage(Parse *pParse, int iTable, int iDb){ + Vdbe *v = sqlite3GetVdbe(pParse); + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); + sqlite3MayAbort(pParse); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* OP_Destroy stores an in integer r1. If this integer + ** is non-zero, then it is the root page number of a table moved to + ** location iTable. The following code modifies the sqlite_master table to + ** reflect this. + ** + ** The "#NNN" in the SQL is a special constant that means whatever value + ** is in register NNN. See grammar rules associated with the TK_REGISTER + ** token for additional information. + */ + sqlite3NestedParse(pParse, + "UPDATE %Q.%s SET rootpage=%d WHERE #%d AND rootpage=#%d", + pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable, r1, r1); +#endif + sqlite3ReleaseTempReg(pParse, r1); +} + +/* +** Write VDBE code to erase table pTab and all associated indices on disk. +** Code to update the sqlite_master tables and internal schema definitions +** in case a root-page belonging to another table is moved by the btree layer +** is also added (this can happen with an auto-vacuum database). +*/ +static void destroyTable(Parse *pParse, Table *pTab){ +#ifdef SQLITE_OMIT_AUTOVACUUM + Index *pIdx; + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + destroyRootPage(pParse, pTab->tnum, iDb); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + destroyRootPage(pParse, pIdx->tnum, iDb); + } +#else + /* If the database may be auto-vacuum capable (if SQLITE_OMIT_AUTOVACUUM + ** is not defined), then it is important to call OP_Destroy on the + ** table and index root-pages in order, starting with the numerically + ** largest root-page number. This guarantees that none of the root-pages + ** to be destroyed is relocated by an earlier OP_Destroy. i.e. if the + ** following were coded: + ** + ** OP_Destroy 4 0 + ** ... + ** OP_Destroy 5 0 + ** + ** and root page 5 happened to be the largest root-page number in the + ** database, then root page 5 would be moved to page 4 by the + ** "OP_Destroy 4 0" opcode. The subsequent "OP_Destroy 5 0" would hit + ** a free-list page. + */ + int iTab = pTab->tnum; + int iDestroyed = 0; + + while( 1 ){ + Index *pIdx; + int iLargest = 0; + + if( iDestroyed==0 || iTabpIndex; pIdx; pIdx=pIdx->pNext){ + int iIdx = pIdx->tnum; + assert( pIdx->pSchema==pTab->pSchema ); + if( (iDestroyed==0 || (iIdxiLargest ){ + iLargest = iIdx; + } + } + if( iLargest==0 ){ + return; + }else{ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + assert( iDb>=0 && iDbdb->nDb ); + destroyRootPage(pParse, iLargest, iDb); + iDestroyed = iLargest; + } + } +#endif +} + +/* +** Remove entries from the sqlite_statN tables (for N in (1,2,3)) +** after a DROP INDEX or DROP TABLE command. +*/ +static void sqlite3ClearStatTables( + Parse *pParse, /* The parsing context */ + int iDb, /* The database number */ + const char *zType, /* "idx" or "tbl" */ + const char *zName /* Name of index or table */ +){ + int i; + const char *zDbName = pParse->db->aDb[iDb].zName; + for(i=1; i<=4; i++){ + char zTab[24]; + sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); + if( sqlite3FindTable(pParse->db, zTab, zDbName) ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE %s=%Q", + zDbName, zTab, zType, zName + ); + } + } +} + +/* +** Generate code to drop a table. +*/ +SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ + Vdbe *v; + sqlite3 *db = pParse->db; + Trigger *pTrigger; + Db *pDb = &db->aDb[iDb]; + + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp0(v, OP_VBegin); + } +#endif + + /* Drop all triggers associated with the table being dropped. Code + ** is generated to remove entries from sqlite_master and/or + ** sqlite_temp_master if required. + */ + pTrigger = sqlite3TriggerList(pParse, pTab); + while( pTrigger ){ + assert( pTrigger->pSchema==pTab->pSchema || + pTrigger->pSchema==db->aDb[1].pSchema ); + sqlite3DropTriggerPtr(pParse, pTrigger); + pTrigger = pTrigger->pNext; + } + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* Remove any entries of the sqlite_sequence table associated with + ** the table being dropped. This is done before the table is dropped + ** at the btree level, in case the sqlite_sequence table needs to + ** move as a result of the drop (can happen in auto-vacuum mode). + */ + if( pTab->tabFlags & TF_Autoincrement ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", + pDb->zName, pTab->zName + ); + } +#endif + + /* Drop all SQLITE_MASTER table and index entries that refer to the + ** table. The program name loops through the master table and deletes + ** every row that refers to a table of the same name as the one being + ** dropped. Triggers are handled separately because a trigger can be + ** created in the temp database that refers to a table in another + ** database. + */ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", + pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + if( !isView && !IsVirtual(pTab) ){ + destroyTable(pParse, pTab); + } + + /* Remove the table entry from SQLite's internal schema and modify + ** the schema cookie. + */ + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); + } + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); + sqlite3ChangeCookie(pParse, iDb); + sqliteViewResetAll(db, iDb); +} + +/* +** This routine is called to do the work of a DROP TABLE statement. +** pName is the name of the table to be dropped. +*/ +SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ + Table *pTab; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; + + if( db->mallocFailed ){ + goto exit_drop_table; + } + assert( pParse->nErr==0 ); + assert( pName->nSrc==1 ); + if( noErr ) db->suppressErr++; + pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); + if( noErr ) db->suppressErr--; + + if( pTab==0 ){ + if( noErr ) sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + goto exit_drop_table; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 && iDbnDb ); + + /* If pTab is a virtual table, call ViewGetColumnNames() to ensure + ** it is initialized. + */ + if( IsVirtual(pTab) && sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto exit_drop_table; + } +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code; + const char *zTab = SCHEMA_TABLE(iDb); + const char *zDb = db->aDb[iDb].zName; + const char *zArg2 = 0; + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ + goto exit_drop_table; + } + if( isView ){ + if( !OMIT_TEMPDB && iDb==1 ){ + code = SQLITE_DROP_TEMP_VIEW; + }else{ + code = SQLITE_DROP_VIEW; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + }else if( IsVirtual(pTab) ){ + code = SQLITE_DROP_VTABLE; + zArg2 = sqlite3GetVTable(db, pTab)->pMod->zName; +#endif + }else{ + if( !OMIT_TEMPDB && iDb==1 ){ + code = SQLITE_DROP_TEMP_TABLE; + }else{ + code = SQLITE_DROP_TABLE; + } + } + if( sqlite3AuthCheck(pParse, code, pTab->zName, zArg2, zDb) ){ + goto exit_drop_table; + } + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb) ){ + goto exit_drop_table; + } + } +#endif + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && sqlite3StrNICmp(pTab->zName, "sqlite_stat", 11)!=0 ){ + sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); + goto exit_drop_table; + } + +#ifndef SQLITE_OMIT_VIEW + /* Ensure DROP TABLE is not used on a view, and DROP VIEW is not used + ** on a table. + */ + if( isView && pTab->pSelect==0 ){ + sqlite3ErrorMsg(pParse, "use DROP TABLE to delete table %s", pTab->zName); + goto exit_drop_table; + } + if( !isView && pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "use DROP VIEW to delete view %s", pTab->zName); + goto exit_drop_table; + } +#endif + + /* Generate code to remove the table from the master table + ** on disk. + */ + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); + sqlite3FkDropTable(pParse, pName, pTab); + sqlite3CodeDropTable(pParse, pTab, iDb, isView); + } + +exit_drop_table: + sqlite3SrcListDelete(db, pName); +} + +/* +** This routine is called to create a new foreign key on the table +** currently under construction. pFromCol determines which columns +** in the current table point to the foreign key. If pFromCol==0 then +** connect the key to the last column inserted. pTo is the name of +** the table referred to (a.k.a the "parent" table). pToCol is a list +** of tables in the parent pTo table. flags contains all +** information about the conflict resolution algorithms specified +** in the ON DELETE, ON UPDATE and ON INSERT clauses. +** +** An FKey structure is created and added to the table currently +** under construction in the pParse->pNewTable field. +** +** The foreign key is set for IMMEDIATE processing. A subsequent call +** to sqlite3DeferForeignKey() might change this to DEFERRED. +*/ +SQLITE_PRIVATE void sqlite3CreateForeignKey( + Parse *pParse, /* Parsing context */ + ExprList *pFromCol, /* Columns in this table that point to other table */ + Token *pTo, /* Name of the other table */ + ExprList *pToCol, /* Columns in the other table */ + int flags /* Conflict resolution algorithms. */ +){ + sqlite3 *db = pParse->db; +#ifndef SQLITE_OMIT_FOREIGN_KEY + FKey *pFKey = 0; + FKey *pNextTo; + Table *p = pParse->pNewTable; + int nByte; + int i; + int nCol; + char *z; + + assert( pTo!=0 ); + if( p==0 || IN_DECLARE_VTAB ) goto fk_end; + if( pFromCol==0 ){ + int iCol = p->nCol-1; + if( NEVER(iCol<0) ) goto fk_end; + if( pToCol && pToCol->nExpr!=1 ){ + sqlite3ErrorMsg(pParse, "foreign key on %s" + " should reference only one column of table %T", + p->aCol[iCol].zName, pTo); + goto fk_end; + } + nCol = 1; + }else if( pToCol && pToCol->nExpr!=pFromCol->nExpr ){ + sqlite3ErrorMsg(pParse, + "number of columns in foreign key does not match the number of " + "columns in the referenced table"); + goto fk_end; + }else{ + nCol = pFromCol->nExpr; + } + nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; + if( pToCol ){ + for(i=0; inExpr; i++){ + nByte += sqlite3Strlen30(pToCol->a[i].zName) + 1; + } + } + pFKey = sqlite3DbMallocZero(db, nByte ); + if( pFKey==0 ){ + goto fk_end; + } + pFKey->pFrom = p; + pFKey->pNextFrom = p->pFKey; + z = (char*)&pFKey->aCol[nCol]; + pFKey->zTo = z; + memcpy(z, pTo->z, pTo->n); + z[pTo->n] = 0; + sqlite3Dequote(z); + z += pTo->n+1; + pFKey->nCol = nCol; + if( pFromCol==0 ){ + pFKey->aCol[0].iFrom = p->nCol-1; + }else{ + for(i=0; inCol; j++){ + if( sqlite3StrICmp(p->aCol[j].zName, pFromCol->a[i].zName)==0 ){ + pFKey->aCol[i].iFrom = j; + break; + } + } + if( j>=p->nCol ){ + sqlite3ErrorMsg(pParse, + "unknown column \"%s\" in foreign key definition", + pFromCol->a[i].zName); + goto fk_end; + } + } + } + if( pToCol ){ + for(i=0; ia[i].zName); + pFKey->aCol[i].zCol = z; + memcpy(z, pToCol->a[i].zName, n); + z[n] = 0; + z += n+1; + } + } + pFKey->isDeferred = 0; + pFKey->aAction[0] = (u8)(flags & 0xff); /* ON DELETE action */ + pFKey->aAction[1] = (u8)((flags >> 8 ) & 0xff); /* ON UPDATE action */ + + assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); + pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, + pFKey->zTo, sqlite3Strlen30(pFKey->zTo), (void *)pFKey + ); + if( pNextTo==pFKey ){ + db->mallocFailed = 1; + goto fk_end; + } + if( pNextTo ){ + assert( pNextTo->pPrevTo==0 ); + pFKey->pNextTo = pNextTo; + pNextTo->pPrevTo = pFKey; + } + + /* Link the foreign key to the table as the last step. + */ + p->pFKey = pFKey; + pFKey = 0; + +fk_end: + sqlite3DbFree(db, pFKey); +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + sqlite3ExprListDelete(db, pFromCol); + sqlite3ExprListDelete(db, pToCol); +} + +/* +** This routine is called when an INITIALLY IMMEDIATE or INITIALLY DEFERRED +** clause is seen as part of a foreign key definition. The isDeferred +** parameter is 1 for INITIALLY DEFERRED and 0 for INITIALLY IMMEDIATE. +** The behavior of the most recently created foreign key is adjusted +** accordingly. +*/ +SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse *pParse, int isDeferred){ +#ifndef SQLITE_OMIT_FOREIGN_KEY + Table *pTab; + FKey *pFKey; + if( (pTab = pParse->pNewTable)==0 || (pFKey = pTab->pFKey)==0 ) return; + assert( isDeferred==0 || isDeferred==1 ); /* EV: R-30323-21917 */ + pFKey->isDeferred = (u8)isDeferred; +#endif +} + +/* +** Generate code that will erase and refill index *pIdx. This is +** used to initialize a newly created index or to recompute the +** content of an index in response to a REINDEX command. +** +** if memRootPage is not negative, it means that the index is newly +** created. The register specified by memRootPage contains the +** root page number of the index. If memRootPage is negative, then +** the index already exists and must be cleared before being refilled and +** the root page number of the index is taken from pIndex->tnum. +*/ +static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ + Table *pTab = pIndex->pTable; /* The table that is indexed */ + int iTab = pParse->nTab++; /* Btree cursor used for pTab */ + int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ + int iSorter; /* Cursor opened by OpenSorter (if in use) */ + int addr1; /* Address of top of loop */ + int addr2; /* Address to jump to for next iteration */ + int tnum; /* Root page of index */ + int iPartIdxLabel; /* Jump to this label to skip a row */ + Vdbe *v; /* Generate code into this virtual machine */ + KeyInfo *pKey; /* KeyInfo for index */ + int regRecord; /* Register holding assemblied index record */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, + db->aDb[iDb].zName ) ){ + return; + } +#endif + + /* Require a write-lock on the table to perform this operation */ + sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); + + v = sqlite3GetVdbe(pParse); + if( v==0 ) return; + if( memRootPage>=0 ){ + tnum = memRootPage; + }else{ + tnum = pIndex->tnum; + } + pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); + + /* Open the sorter cursor if we are to use one. */ + iSorter = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*) + sqlite3KeyInfoRef(pKey), P4_KEYINFO); + + /* Open the table. Loop through all rows of the table, inserting index + ** records into the sorter. */ + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); VdbeCoverage(v); + regRecord = sqlite3GetTempReg(pParse); + + sqlite3GenerateIndexKey(pParse,pIndex,iTab,regRecord,0,&iPartIdxLabel,0,0); + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); + sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, + (char *)pKey, P4_KEYINFO); + sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); + + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); + assert( pKey!=0 || db->mallocFailed || pParse->nErr ); + if( IsUniqueIndex(pIndex) && pKey!=0 ){ + int j2 = sqlite3VdbeCurrentAddr(v) + 3; + sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, + pIndex->nKeyCol); VdbeCoverage(v); + sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); + }else{ + addr2 = sqlite3VdbeCurrentAddr(v); + } + sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + + sqlite3VdbeAddOp1(v, OP_Close, iTab); + sqlite3VdbeAddOp1(v, OP_Close, iIdx); + sqlite3VdbeAddOp1(v, OP_Close, iSorter); +} + +/* +** Allocate heap space to hold an Index object with nCol columns. +** +** Increase the allocation size to provide an extra nExtra bytes +** of 8-byte aligned space after the Index object and return a +** pointer to this extra space in *ppExtra. +*/ +SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( + sqlite3 *db, /* Database connection */ + i16 nCol, /* Total number of columns in the index */ + int nExtra, /* Number of bytes of extra space to alloc */ + char **ppExtra /* Pointer to the "extra" space */ +){ + Index *p; /* Allocated index object */ + int nByte; /* Bytes of space for Index object + arrays */ + + nByte = ROUND8(sizeof(Index)) + /* Index structure */ + ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ + sizeof(i16)*nCol + /* Index.aiColumn */ + sizeof(u8)*nCol); /* Index.aSortOrder */ + p = sqlite3DbMallocZero(db, nByte + nExtra); + if( p ){ + char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); + p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->aSortOrder = (u8*)pExtra; + p->nColumn = nCol; + p->nKeyCol = nCol - 1; + *ppExtra = ((char*)p) + nByte; + } + return p; +} + +/* +** Create a new index for an SQL table. pName1.pName2 is the name of the index +** and pTblList is the name of the table that is to be indexed. Both will +** be NULL for a primary key or an index that is created to satisfy a +** UNIQUE constraint. If pTable and pIndex are NULL, use pParse->pNewTable +** as the table to be indexed. pParse->pNewTable is a table that is +** currently being constructed by a CREATE TABLE statement. +** +** pList is a list of columns to be indexed. pList will be NULL if this +** is a primary key or unique-constraint on the most recent column added +** to the table currently under construction. +** +** If the index is created successfully, return a pointer to the new Index +** structure. This is used by sqlite3AddPrimaryKey() to mark the index +** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY) +*/ +SQLITE_PRIVATE Index *sqlite3CreateIndex( + Parse *pParse, /* All information about this parse */ + Token *pName1, /* First part of index name. May be NULL */ + Token *pName2, /* Second part of index name. May be NULL */ + SrcList *pTblName, /* Table to index. Use pParse->pNewTable if 0 */ + ExprList *pList, /* A list of columns to be indexed */ + int onError, /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + Token *pStart, /* The CREATE token that begins this statement */ + Expr *pPIWhere, /* WHERE clause for partial indices */ + int sortOrder, /* Sort order of primary key when pList==NULL */ + int ifNotExist /* Omit error if index already exists */ +){ + Index *pRet = 0; /* Pointer to return */ + Table *pTab = 0; /* Table to be indexed */ + Index *pIndex = 0; /* The index to be created */ + char *zName = 0; /* Name of the index */ + int nName; /* Number of characters in zName */ + int i, j; + DbFixer sFix; /* For assigning database names to pTable */ + int sortOrderMask; /* 1 to honor DESC in index. 0 to ignore. */ + sqlite3 *db = pParse->db; + Db *pDb; /* The specific table containing the indexed database */ + int iDb; /* Index of the database that is being written */ + Token *pName = 0; /* Unqualified name of the index to create */ + struct ExprList_item *pListItem; /* For looping over pList */ + const Column *pTabCol; /* A column in the table */ + int nExtra = 0; /* Space allocated for zExtra[] */ + int nExtraCol; /* Number of extra columns needed */ + char *zExtra = 0; /* Extra space after the Index object */ + Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ + + assert( pParse->nErr==0 ); /* Never called with prior errors */ + if( db->mallocFailed || IN_DECLARE_VTAB ){ + goto exit_create_index; + } + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto exit_create_index; + } + + /* + ** Find the table that is to be indexed. Return early if not found. + */ + if( pTblName!=0 ){ + + /* Use the two-part index name to determine the database + ** to search for the table. 'Fix' the table name to this db + ** before looking up the table. + */ + assert( pName1 && pName2 ); + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) goto exit_create_index; + assert( pName && pName->z ); + +#ifndef SQLITE_OMIT_TEMPDB + /* If the index name was unqualified, check if the table + ** is a temp table. If so, set the database to 1. Do not do this + ** if initialising a database schema. + */ + if( !db->init.busy ){ + pTab = sqlite3SrcListLookup(pParse, pTblName); + if( pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } + } +#endif + + sqlite3FixInit(&sFix, pParse, iDb, "index", pName); + if( sqlite3FixSrcList(&sFix, pTblName) ){ + /* Because the parser constructs pTblName from a single identifier, + ** sqlite3FixSrcList can never fail. */ + assert(0); + } + pTab = sqlite3LocateTableItem(pParse, 0, &pTblName->a[0]); + assert( db->mallocFailed==0 || pTab==0 ); + if( pTab==0 ) goto exit_create_index; + if( iDb==1 && db->aDb[iDb].pSchema!=pTab->pSchema ){ + sqlite3ErrorMsg(pParse, + "cannot create a TEMP index on non-TEMP table \"%s\"", + pTab->zName); + goto exit_create_index; + } + if( !HasRowid(pTab) ) pPk = sqlite3PrimaryKeyIndex(pTab); + }else{ + assert( pName==0 ); + assert( pStart==0 ); + pTab = pParse->pNewTable; + if( !pTab ) goto exit_create_index; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + } + pDb = &db->aDb[iDb]; + + assert( pTab!=0 ); + assert( pParse->nErr==0 ); + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ + sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); + goto exit_create_index; + } +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "views may not be indexed"); + goto exit_create_index; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "virtual tables may not be indexed"); + goto exit_create_index; + } +#endif + + /* + ** Find the name of the index. Make sure there is not already another + ** index or table with the same name. + ** + ** Exception: If we are reading the names of permanent indices from the + ** sqlite_master table (because some other process changed the schema) and + ** one of the index names collides with the name of a temporary table or + ** index, then we will continue to process this index. + ** + ** If pName==0 it means that we are + ** dealing with a primary key or UNIQUE constraint. We have to invent our + ** own name. + */ + if( pName ){ + zName = sqlite3NameFromToken(db, pName); + if( zName==0 ) goto exit_create_index; + assert( pName->z!=0 ); + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + goto exit_create_index; + } + if( !db->init.busy ){ + if( sqlite3FindTable(db, zName, 0)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); + goto exit_create_index; + } + } + if( sqlite3FindIndex(db, zName, pDb->zName)!=0 ){ + if( !ifNotExist ){ + sqlite3ErrorMsg(pParse, "index %s already exists", zName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + } + goto exit_create_index; + } + }else{ + int n; + Index *pLoop; + for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} + zName = sqlite3MPrintf(db, "sqlite_autoindex_%s_%d", pTab->zName, n); + if( zName==0 ){ + goto exit_create_index; + } + } + + /* Check for authorization to create an index. + */ +#ifndef SQLITE_OMIT_AUTHORIZATION + { + const char *zDb = pDb->zName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ + goto exit_create_index; + } + i = SQLITE_CREATE_INDEX; + if( !OMIT_TEMPDB && iDb==1 ) i = SQLITE_CREATE_TEMP_INDEX; + if( sqlite3AuthCheck(pParse, i, zName, pTab->zName, zDb) ){ + goto exit_create_index; + } + } +#endif + + /* If pList==0, it means this routine was called to make a primary + ** key out of the last column added to the table under construction. + ** So create a fake list to simulate this. + */ + if( pList==0 ){ + pList = sqlite3ExprListAppend(pParse, 0, 0); + if( pList==0 ) goto exit_create_index; + pList->a[0].zName = sqlite3DbStrDup(pParse->db, + pTab->aCol[pTab->nCol-1].zName); + pList->a[0].sortOrder = (u8)sortOrder; + } + + /* Figure out how many bytes of space are required to store explicitly + ** specified collation sequence names. + */ + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + if( pExpr ){ + assert( pExpr->op==TK_COLLATE ); + nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); + } + } + + /* + ** Allocate the index structure. + */ + nName = sqlite3Strlen30(zName); + nExtraCol = pPk ? pPk->nKeyCol : 1; + pIndex = sqlite3AllocateIndexObject(db, pList->nExpr + nExtraCol, + nName + nExtra + 1, &zExtra); + if( db->mallocFailed ){ + goto exit_create_index; + } + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); + pIndex->zName = zExtra; + zExtra += nName + 1; + memcpy(pIndex->zName, zName, nName+1); + pIndex->pTable = pTab; + pIndex->onError = (u8)onError; + pIndex->uniqNotNull = onError!=OE_None; + pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE; + pIndex->pSchema = db->aDb[iDb].pSchema; + pIndex->nKeyCol = pList->nExpr; + if( pPIWhere ){ + sqlite3ResolveSelfReference(pParse, pTab, NC_PartIdx, pPIWhere, 0); + pIndex->pPartIdxWhere = pPIWhere; + pPIWhere = 0; + } + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + + /* Check to see if we should honor DESC requests on index columns + */ + if( pDb->pSchema->file_format>=4 ){ + sortOrderMask = -1; /* Honor DESC */ + }else{ + sortOrderMask = 0; /* Ignore DESC */ + } + + /* Scan the names of the columns of the table to be indexed and + ** load the column indices into the Index structure. Report an error + ** if any column is not found. + ** + ** TODO: Add a test to make sure that the same column is not named + ** more than once within the same index. Only the first instance of + ** the column will ever be used by the optimizer. Note that using the + ** same column more than once cannot be an error because that would + ** break backwards compatibility - it needs to be a warning. + */ + for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ + const char *zColName = pListItem->zName; + int requestedSortOrder; + char *zColl; /* Collation sequence name */ + + for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ + if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; + } + if( j>=pTab->nCol ){ + sqlite3ErrorMsg(pParse, "table %s has no column named %s", + pTab->zName, zColName); + pParse->checkSchema = 1; + goto exit_create_index; + } + assert( pTab->nCol<=0x7fff && j<=0x7fff ); + pIndex->aiColumn[i] = (i16)j; + if( pListItem->pExpr ){ + int nColl; + assert( pListItem->pExpr->op==TK_COLLATE ); + zColl = pListItem->pExpr->u.zToken; + nColl = sqlite3Strlen30(zColl) + 1; + assert( nExtra>=nColl ); + memcpy(zExtra, zColl, nColl); + zColl = zExtra; + zExtra += nColl; + nExtra -= nColl; + }else{ + zColl = pTab->aCol[j].zColl; + if( !zColl ) zColl = "BINARY"; + } + if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ + goto exit_create_index; + } + pIndex->azColl[i] = zColl; + requestedSortOrder = pListItem->sortOrder & sortOrderMask; + pIndex->aSortOrder[i] = (u8)requestedSortOrder; + if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; + } + if( pPk ){ + for(j=0; jnKeyCol; j++){ + int x = pPk->aiColumn[j]; + if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ + pIndex->nColumn--; + }else{ + pIndex->aiColumn[i] = x; + pIndex->azColl[i] = pPk->azColl[j]; + pIndex->aSortOrder[i] = pPk->aSortOrder[j]; + i++; + } + } + assert( i==pIndex->nColumn ); + }else{ + pIndex->aiColumn[i] = -1; + pIndex->azColl[i] = "BINARY"; + } + sqlite3DefaultRowEst(pIndex); + if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); + + if( pTab==pParse->pNewTable ){ + /* This routine has been called to create an automatic index as a + ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or + ** a PRIMARY KEY or UNIQUE clause following the column definitions. + ** i.e. one of: + ** + ** CREATE TABLE t(x PRIMARY KEY, y); + ** CREATE TABLE t(x, y, UNIQUE(x, y)); + ** + ** Either way, check to see if the table already has such an index. If + ** so, don't bother creating this one. This only applies to + ** automatically created indices. Users can do as they wish with + ** explicit indices. + ** + ** Two UNIQUE or PRIMARY KEY constraints are considered equivalent + ** (and thus suppressing the second one) even if they have different + ** sort orders. + ** + ** If there are different collating sequences or if the columns of + ** the constraint occur in different orders, then the constraints are + ** considered distinct and both result in separate indices. + */ + Index *pIdx; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int k; + assert( IsUniqueIndex(pIdx) ); + assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF ); + assert( IsUniqueIndex(pIndex) ); + + if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue; + for(k=0; knKeyCol; k++){ + const char *z1; + const char *z2; + if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; + z1 = pIdx->azColl[k]; + z2 = pIndex->azColl[k]; + if( z1!=z2 && sqlite3StrICmp(z1, z2) ) break; + } + if( k==pIdx->nKeyCol ){ + if( pIdx->onError!=pIndex->onError ){ + /* This constraint creates the same index as a previous + ** constraint specified somewhere in the CREATE TABLE statement. + ** However the ON CONFLICT clauses are different. If both this + ** constraint and the previous equivalent constraint have explicit + ** ON CONFLICT clauses this is an error. Otherwise, use the + ** explicitly specified behavior for the index. + */ + if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ + sqlite3ErrorMsg(pParse, + "conflicting ON CONFLICT clauses specified", 0); + } + if( pIdx->onError==OE_Default ){ + pIdx->onError = pIndex->onError; + } + } + goto exit_create_index; + } + } + } + + /* Link the new Index structure to its table and to the other + ** in-memory database structures. + */ + if( db->init.busy ){ + Index *p; + assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + p = sqlite3HashInsert(&pIndex->pSchema->idxHash, + pIndex->zName, sqlite3Strlen30(pIndex->zName), + pIndex); + if( p ){ + assert( p==pIndex ); /* Malloc must have failed */ + db->mallocFailed = 1; + goto exit_create_index; + } + db->flags |= SQLITE_InternChanges; + if( pTblName!=0 ){ + pIndex->tnum = db->init.newTnum; + } + } + + /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the + ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then + ** emit code to allocate the index rootpage on disk and make an entry for + ** the index in the sqlite_master table and populate the index with + ** content. But, do not do this if we are simply reading the sqlite_master + ** table to parse the schema, or if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table. + ** + ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY + ** or UNIQUE index in a CREATE TABLE statement. Since the table + ** has just been created, it contains no data and the index initialization + ** step can be skipped. + */ + else if( pParse->nErr==0 && (HasRowid(pTab) || pTblName!=0) ){ + Vdbe *v; + char *zStmt; + int iMem = ++pParse->nMem; + + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto exit_create_index; + + + /* Create the rootpage for the index + */ + sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); + + /* Gather the complete text of the CREATE INDEX statement into + ** the zStmt variable + */ + if( pStart ){ + int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; + if( pName->z[n-1]==';' ) n--; + /* A named index with an explicit CREATE INDEX statement */ + zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", + onError==OE_None ? "" : " UNIQUE", n, pName->z); + }else{ + /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ + /* zStmt = sqlite3MPrintf(""); */ + zStmt = 0; + } + + /* Add an entry in sqlite_master for this index + */ + sqlite3NestedParse(pParse, + "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + pIndex->zName, + pTab->zName, + iMem, + zStmt + ); + sqlite3DbFree(db, zStmt); + + /* Fill the index with data and reparse the schema. Code an OP_Expire + ** to invalidate all pre-compiled statements. + */ + if( pTblName ){ + sqlite3RefillIndex(pParse, pIndex, iMem); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); + sqlite3VdbeAddOp1(v, OP_Expire, 0); + } + } + + /* When adding an index to the list of indices for a table, make + ** sure all indices labeled OE_Replace come after all those labeled + ** OE_Ignore. This is necessary for the correct constraint check + ** processing (in sqlite3GenerateConstraintChecks()) as part of + ** UPDATE and INSERT statements. + */ + if( db->init.busy || pTblName==0 ){ + if( onError!=OE_Replace || pTab->pIndex==0 + || pTab->pIndex->onError==OE_Replace){ + pIndex->pNext = pTab->pIndex; + pTab->pIndex = pIndex; + }else{ + Index *pOther = pTab->pIndex; + while( pOther->pNext && pOther->pNext->onError!=OE_Replace ){ + pOther = pOther->pNext; + } + pIndex->pNext = pOther->pNext; + pOther->pNext = pIndex; + } + pRet = pIndex; + pIndex = 0; + } + + /* Clean up before exiting */ +exit_create_index: + if( pIndex ) freeIndex(db, pIndex); + sqlite3ExprDelete(db, pPIWhere); + sqlite3ExprListDelete(db, pList); + sqlite3SrcListDelete(db, pTblName); + sqlite3DbFree(db, zName); + return pRet; +} + +/* +** Fill the Index.aiRowEst[] array with default information - information +** to be used when we have not run the ANALYZE command. +** +** aiRowEst[0] is suppose to contain the number of elements in the index. +** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the +** number of rows in the table that match any particular value of the +** first column of the index. aiRowEst[2] is an estimate of the number +** of rows that match any particular combination of the first 2 columns +** of the index. And so forth. It must always be the case that +* +** aiRowEst[N]<=aiRowEst[N-1] +** aiRowEst[N]>=1 +** +** Apart from that, we have little to go on besides intuition as to +** how aiRowEst[] should be initialized. The numbers generated here +** are based on typical values found in actual indices. +*/ +SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ + /* 10, 9, 8, 7, 6 */ + LogEst aVal[] = { 33, 32, 30, 28, 26 }; + LogEst *a = pIdx->aiRowLogEst; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + int i; + + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table. Or 10, if the estimated number of rows + ** in the table is less than that. */ + a[0] = pIdx->pTable->nRowLogEst; + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); + } + + assert( 0==sqlite3LogEst(1) ); + if( IsUniqueIndex(pIdx) ) a[pIdx->nKeyCol] = 0; +} + +/* +** This routine will drop an existing named index. This routine +** implements the DROP INDEX statement. +*/ +SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ + Index *pIndex; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; + + assert( pParse->nErr==0 ); /* Never called with prior errors */ + if( db->mallocFailed ){ + goto exit_drop_index; + } + assert( pName->nSrc==1 ); + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto exit_drop_index; + } + pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); + if( pIndex==0 ){ + if( !ifExists ){ + sqlite3ErrorMsg(pParse, "no such index: %S", pName, 0); + }else{ + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + } + pParse->checkSchema = 1; + goto exit_drop_index; + } + if( pIndex->idxType!=SQLITE_IDXTYPE_APPDEF ){ + sqlite3ErrorMsg(pParse, "index associated with UNIQUE " + "or PRIMARY KEY constraint cannot be dropped", 0); + goto exit_drop_index; + } + iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code = SQLITE_DROP_INDEX; + Table *pTab = pIndex->pTable; + const char *zDb = db->aDb[iDb].zName; + const char *zTab = SCHEMA_TABLE(iDb); + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ + goto exit_drop_index; + } + if( !OMIT_TEMPDB && iDb ) code = SQLITE_DROP_TEMP_INDEX; + if( sqlite3AuthCheck(pParse, code, pIndex->zName, pTab->zName, zDb) ){ + goto exit_drop_index; + } + } +#endif + + /* Generate code to remove the index and from the master table */ + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName + ); + sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); + sqlite3ChangeCookie(pParse, iDb); + destroyRootPage(pParse, pIndex->tnum, iDb); + sqlite3VdbeAddOp4(v, OP_DropIndex, iDb, 0, 0, pIndex->zName, 0); + } + +exit_drop_index: + sqlite3SrcListDelete(db, pName); +} + +/* +** pArray is a pointer to an array of objects. Each object in the +** array is szEntry bytes in size. This routine uses sqlite3DbRealloc() +** to extend the array so that there is space for a new object at the end. +** +** When this function is called, *pnEntry contains the current size of +** the array (in entries - so the allocation is ((*pnEntry) * szEntry) bytes +** in total). +** +** If the realloc() is successful (i.e. if no OOM condition occurs), the +** space allocated for the new object is zeroed, *pnEntry updated to +** reflect the new size of the array and a pointer to the new allocation +** returned. *pIdx is set to the index of the new array entry in this case. +** +** Otherwise, if the realloc() fails, *pIdx is set to -1, *pnEntry remains +** unchanged and a copy of pArray returned. +*/ +SQLITE_PRIVATE void *sqlite3ArrayAllocate( + sqlite3 *db, /* Connection to notify of malloc failures */ + void *pArray, /* Array of objects. Might be reallocated */ + int szEntry, /* Size of each object in the array */ + int *pnEntry, /* Number of objects currently in use */ + int *pIdx /* Write the index of a new slot here */ +){ + char *z; + int n = *pnEntry; + if( (n & (n-1))==0 ){ + int sz = (n==0) ? 1 : 2*n; + void *pNew = sqlite3DbRealloc(db, pArray, sz*szEntry); + if( pNew==0 ){ + *pIdx = -1; + return pArray; + } + pArray = pNew; + } + z = (char*)pArray; + memset(&z[n * szEntry], 0, szEntry); + *pIdx = n; + ++*pnEntry; + return pArray; +} + +/* +** Append a new element to the given IdList. Create a new IdList if +** need be. +** +** A new IdList is returned, or NULL if malloc() fails. +*/ +SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){ + int i; + if( pList==0 ){ + pList = sqlite3DbMallocZero(db, sizeof(IdList) ); + if( pList==0 ) return 0; + } + pList->a = sqlite3ArrayAllocate( + db, + pList->a, + sizeof(pList->a[0]), + &pList->nId, + &i + ); + if( i<0 ){ + sqlite3IdListDelete(db, pList); + return 0; + } + pList->a[i].zName = sqlite3NameFromToken(db, pToken); + return pList; +} + +/* +** Delete an IdList. +*/ +SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ + int i; + if( pList==0 ) return; + for(i=0; inId; i++){ + sqlite3DbFree(db, pList->a[i].zName); + } + sqlite3DbFree(db, pList->a); + sqlite3DbFree(db, pList); +} + +/* +** Return the index in pList of the identifier named zId. Return -1 +** if not found. +*/ +SQLITE_PRIVATE int sqlite3IdListIndex(IdList *pList, const char *zName){ + int i; + if( pList==0 ) return -1; + for(i=0; inId; i++){ + if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i; + } + return -1; +} + +/* +** Expand the space allocated for the given SrcList object by +** creating nExtra new slots beginning at iStart. iStart is zero based. +** New slots are zeroed. +** +** For example, suppose a SrcList initially contains two entries: A,B. +** To append 3 new entries onto the end, do this: +** +** sqlite3SrcListEnlarge(db, pSrclist, 3, 2); +** +** After the call above it would contain: A, B, nil, nil, nil. +** If the iStart argument had been 1 instead of 2, then the result +** would have been: A, nil, nil, nil, B. To prepend the new slots, +** the iStart value would be 0. The result then would +** be: nil, nil, nil, A, B. +** +** If a memory allocation fails the SrcList is unchanged. The +** db->mallocFailed flag will be set to true. +*/ +SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( + sqlite3 *db, /* Database connection to notify of OOM errors */ + SrcList *pSrc, /* The SrcList to be enlarged */ + int nExtra, /* Number of new slots to add to pSrc->a[] */ + int iStart /* Index in pSrc->a[] of first new slot */ +){ + int i; + + /* Sanity checking on calling parameters */ + assert( iStart>=0 ); + assert( nExtra>=1 ); + assert( pSrc!=0 ); + assert( iStart<=pSrc->nSrc ); + + /* Allocate additional space if needed */ + if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ + SrcList *pNew; + int nAlloc = pSrc->nSrc+nExtra; + int nGot; + pNew = sqlite3DbRealloc(db, pSrc, + sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); + if( pNew==0 ){ + assert( db->mallocFailed ); + return pSrc; + } + pSrc = pNew; + nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; + pSrc->nAlloc = nGot; + } + + /* Move existing slots that come after the newly inserted slots + ** out of the way */ + for(i=pSrc->nSrc-1; i>=iStart; i--){ + pSrc->a[i+nExtra] = pSrc->a[i]; + } + pSrc->nSrc += nExtra; + + /* Zero the newly allocated slots */ + memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); + for(i=iStart; ia[i].iCursor = -1; + } + + /* Return a pointer to the enlarged SrcList */ + return pSrc; +} + + +/* +** Append a new table name to the given SrcList. Create a new SrcList if +** need be. A new entry is created in the SrcList even if pTable is NULL. +** +** A SrcList is returned, or NULL if there is an OOM error. The returned +** SrcList might be the same as the SrcList that was input or it might be +** a new one. If an OOM error does occurs, then the prior value of pList +** that is input to this routine is automatically freed. +** +** If pDatabase is not null, it means that the table has an optional +** database name prefix. Like this: "database.table". The pDatabase +** points to the table name and the pTable points to the database name. +** The SrcList.a[].zName field is filled with the table name which might +** come from pTable (if pDatabase is NULL) or from pDatabase. +** SrcList.a[].zDatabase is filled with the database name from pTable, +** or with NULL if no database is specified. +** +** In other words, if call like this: +** +** sqlite3SrcListAppend(D,A,B,0); +** +** Then B is a table name and the database name is unspecified. If called +** like this: +** +** sqlite3SrcListAppend(D,A,B,C); +** +** Then C is the table name and B is the database name. If C is defined +** then so is B. In other words, we never have a case where: +** +** sqlite3SrcListAppend(D,A,0,C); +** +** Both pTable and pDatabase are assumed to be quoted. They are dequoted +** before being added to the SrcList. +*/ +SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( + sqlite3 *db, /* Connection to notify of malloc failures */ + SrcList *pList, /* Append to this SrcList. NULL creates a new SrcList */ + Token *pTable, /* Table to append */ + Token *pDatabase /* Database of the table */ +){ + struct SrcList_item *pItem; + assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ + if( pList==0 ){ + pList = sqlite3DbMallocZero(db, sizeof(SrcList) ); + if( pList==0 ) return 0; + pList->nAlloc = 1; + } + pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); + if( db->mallocFailed ){ + sqlite3SrcListDelete(db, pList); + return 0; + } + pItem = &pList->a[pList->nSrc-1]; + if( pDatabase && pDatabase->z==0 ){ + pDatabase = 0; + } + if( pDatabase ){ + Token *pTemp = pDatabase; + pDatabase = pTable; + pTable = pTemp; + } + pItem->zName = sqlite3NameFromToken(db, pTable); + pItem->zDatabase = sqlite3NameFromToken(db, pDatabase); + return pList; +} + +/* +** Assign VdbeCursor index numbers to all tables in a SrcList +*/ +SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ + int i; + struct SrcList_item *pItem; + assert(pList || pParse->db->mallocFailed ); + if( pList ){ + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pItem->iCursor>=0 ) break; + pItem->iCursor = pParse->nTab++; + if( pItem->pSelect ){ + sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); + } + } + } +} + +/* +** Delete an entire SrcList including all its substructure. +*/ +SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ + int i; + struct SrcList_item *pItem; + if( pList==0 ) return; + for(pItem=pList->a, i=0; inSrc; i++, pItem++){ + sqlite3DbFree(db, pItem->zDatabase); + sqlite3DbFree(db, pItem->zName); + sqlite3DbFree(db, pItem->zAlias); + sqlite3DbFree(db, pItem->zIndex); + sqlite3DeleteTable(db, pItem->pTab); + sqlite3SelectDelete(db, pItem->pSelect); + sqlite3ExprDelete(db, pItem->pOn); + sqlite3IdListDelete(db, pItem->pUsing); + } + sqlite3DbFree(db, pList); +} + +/* +** This routine is called by the parser to add a new term to the +** end of a growing FROM clause. The "p" parameter is the part of +** the FROM clause that has already been constructed. "p" is NULL +** if this is the first term of the FROM clause. pTable and pDatabase +** are the name of the table and database named in the FROM clause term. +** pDatabase is NULL if the database name qualifier is missing - the +** usual case. If the term has a alias, then pAlias points to the +** alias token. If the term is a subquery, then pSubquery is the +** SELECT statement that the subquery encodes. The pTable and +** pDatabase parameters are NULL for subqueries. The pOn and pUsing +** parameters are the content of the ON and USING clauses. +** +** Return a new SrcList which encodes is the FROM with the new +** term added. +*/ +SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( + Parse *pParse, /* Parsing context */ + SrcList *p, /* The left part of the FROM clause already seen */ + Token *pTable, /* Name of the table to add to the FROM clause */ + Token *pDatabase, /* Name of the database containing pTable */ + Token *pAlias, /* The right-hand side of the AS subexpression */ + Select *pSubquery, /* A subquery used in place of a table name */ + Expr *pOn, /* The ON clause of a join */ + IdList *pUsing /* The USING clause of a join */ +){ + struct SrcList_item *pItem; + sqlite3 *db = pParse->db; + if( !p && (pOn || pUsing) ){ + sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", + (pOn ? "ON" : "USING") + ); + goto append_from_error; + } + p = sqlite3SrcListAppend(db, p, pTable, pDatabase); + if( p==0 || NEVER(p->nSrc==0) ){ + goto append_from_error; + } + pItem = &p->a[p->nSrc-1]; + assert( pAlias!=0 ); + if( pAlias->n ){ + pItem->zAlias = sqlite3NameFromToken(db, pAlias); + } + pItem->pSelect = pSubquery; + pItem->pOn = pOn; + pItem->pUsing = pUsing; + return p; + + append_from_error: + assert( p==0 ); + sqlite3ExprDelete(db, pOn); + sqlite3IdListDelete(db, pUsing); + sqlite3SelectDelete(db, pSubquery); + return 0; +} + +/* +** Add an INDEXED BY or NOT INDEXED clause to the most recently added +** element of the source-list passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ + assert( pIndexedBy!=0 ); + if( p && ALWAYS(p->nSrc>0) ){ + struct SrcList_item *pItem = &p->a[p->nSrc-1]; + assert( pItem->notIndexed==0 && pItem->zIndex==0 ); + if( pIndexedBy->n==1 && !pIndexedBy->z ){ + /* A "NOT INDEXED" clause was supplied. See parse.y + ** construct "indexed_opt" for details. */ + pItem->notIndexed = 1; + }else{ + pItem->zIndex = sqlite3NameFromToken(pParse->db, pIndexedBy); + } + } +} + +/* +** When building up a FROM clause in the parser, the join operator +** is initially attached to the left operand. But the code generator +** expects the join operator to be on the right operand. This routine +** Shifts all join operators from left to right for an entire FROM +** clause. +** +** Example: Suppose the join is like this: +** +** A natural cross join B +** +** The operator is "natural cross join". The A and B operands are stored +** in p->a[0] and p->a[1], respectively. The parser initially stores the +** operator with A. This routine shifts that operator over to B. +*/ +SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ + if( p ){ + int i; + assert( p->a || p->nSrc==0 ); + for(i=p->nSrc-1; i>0; i--){ + p->a[i].jointype = p->a[i-1].jointype; + } + p->a[0].jointype = 0; + } +} + +/* +** Begin a transaction +*/ +SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ + sqlite3 *db; + Vdbe *v; + int i; + + assert( pParse!=0 ); + db = pParse->db; + assert( db!=0 ); +/* if( db->aDb[0].pBt==0 ) return; */ + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ){ + return; + } + v = sqlite3GetVdbe(pParse); + if( !v ) return; + if( type!=TK_DEFERRED ){ + for(i=0; inDb; i++){ + sqlite3VdbeAddOp2(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1); + sqlite3VdbeUsesBtree(v, i); + } + } + sqlite3VdbeAddOp2(v, OP_AutoCommit, 0, 0); +} + +/* +** Commit a transaction +*/ +SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ + Vdbe *v; + + assert( pParse!=0 ); + assert( pParse->db!=0 ); + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "COMMIT", 0, 0) ){ + return; + } + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 0); + } +} + +/* +** Rollback a transaction +*/ +SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse *pParse){ + Vdbe *v; + + assert( pParse!=0 ); + assert( pParse->db!=0 ); + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "ROLLBACK", 0, 0) ){ + return; + } + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 1); + } +} + +/* +** This function is called by the parser when it parses a command to create, +** release or rollback an SQL savepoint. +*/ +SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){ + char *zName = sqlite3NameFromToken(pParse->db, pName); + if( zName ){ + Vdbe *v = sqlite3GetVdbe(pParse); +#ifndef SQLITE_OMIT_AUTHORIZATION + static const char * const az[] = { "BEGIN", "RELEASE", "ROLLBACK" }; + assert( !SAVEPOINT_BEGIN && SAVEPOINT_RELEASE==1 && SAVEPOINT_ROLLBACK==2 ); +#endif + if( !v || sqlite3AuthCheck(pParse, SQLITE_SAVEPOINT, az[op], zName, 0) ){ + sqlite3DbFree(pParse->db, zName); + return; + } + sqlite3VdbeAddOp4(v, OP_Savepoint, op, 0, 0, zName, P4_DYNAMIC); + } +} + +/* +** Make sure the TEMP database is open and available for use. Return +** the number of errors. Leave any error messages in the pParse structure. +*/ +SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ + sqlite3 *db = pParse->db; + if( db->aDb[1].pBt==0 && !pParse->explain ){ + int rc; + Btree *pBt; + static const int flags = + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_TEMP_DB; + + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pBt, 0, flags); + if( rc!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "unable to open a temporary database " + "file for storing temporary tables"); + pParse->rc = rc; + return 1; + } + db->aDb[1].pBt = pBt; + assert( db->aDb[1].pSchema ); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ + db->mallocFailed = 1; + return 1; + } + } + return 0; +} + +/* +** Record the fact that the schema cookie will need to be verified +** for database iDb. The code to actually verify the schema cookie +** will occur at the end of the top-level VDBE and will be generated +** later, by sqlite3FinishCoding(). +*/ +SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3 *db = pToplevel->db; + + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDbcookieMask, iDb)==0 ){ + DbMaskSet(pToplevel->cookieMask, iDb); + pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; + if( !OMIT_TEMPDB && iDb==1 ){ + sqlite3OpenTempDatabase(pToplevel); + } + } +} + +/* +** If argument zDb is NULL, then call sqlite3CodeVerifySchema() for each +** attached database. Otherwise, invoke it for the database named zDb only. +*/ +SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ + sqlite3 *db = pParse->db; + int i; + for(i=0; inDb; i++){ + Db *pDb = &db->aDb[i]; + if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zName)) ){ + sqlite3CodeVerifySchema(pParse, i); + } + } +} + +/* +** Generate VDBE code that prepares for doing an operation that +** might change the database. +** +** This routine starts a new transaction if we are not already within +** a transaction. If we are already within a transaction, then a checkpoint +** is set if the setStatement parameter is true. A checkpoint should +** be set for operations that might fail (due to a constraint) part of +** the way through and which will need to undo some writes without having to +** rollback the whole transaction. For operations where all constraints +** can be checked before any changes are made to the database, it is never +** necessary to undo a write and the checkpoint should not be set. +*/ +SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3CodeVerifySchema(pParse, iDb); + DbMaskSet(pToplevel->writeMask, iDb); + pToplevel->isMultiWrite |= setStatement; +} + +/* +** Indicate that the statement currently under construction might write +** more than one entry (example: deleting one row then inserting another, +** inserting multiple rows in a table, or inserting a row and index entries.) +** If an abort occurs after some of these writes have completed, then it will +** be necessary to undo the completed writes. +*/ +SQLITE_PRIVATE void sqlite3MultiWrite(Parse *pParse){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pToplevel->isMultiWrite = 1; +} + +/* +** The code generator calls this routine if is discovers that it is +** possible to abort a statement prior to completion. In order to +** perform this abort without corrupting the database, we need to make +** sure that the statement is protected by a statement transaction. +** +** Technically, we only need to set the mayAbort flag if the +** isMultiWrite flag was previously set. There is a time dependency +** such that the abort must occur after the multiwrite. This makes +** some statements involving the REPLACE conflict resolution algorithm +** go a little faster. But taking advantage of this time dependency +** makes it more difficult to prove that the code is correct (in +** particular, it prevents us from writing an effective +** implementation of sqlite3AssertMayAbort()) and so we have chosen +** to take the safe route and skip the optimization. +*/ +SQLITE_PRIVATE void sqlite3MayAbort(Parse *pParse){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pToplevel->mayAbort = 1; +} + +/* +** Code an OP_Halt that causes the vdbe to return an SQLITE_CONSTRAINT +** error. The onError parameter determines which (if any) of the statement +** and/or current transaction is rolled back. +*/ +SQLITE_PRIVATE void sqlite3HaltConstraint( + Parse *pParse, /* Parsing context */ + int errCode, /* extended error code */ + int onError, /* Constraint type */ + char *p4, /* Error message */ + i8 p4type, /* P4_STATIC or P4_TRANSIENT */ + u8 p5Errmsg /* P5_ErrMsg type */ +){ + Vdbe *v = sqlite3GetVdbe(pParse); + assert( (errCode&0xff)==SQLITE_CONSTRAINT ); + if( onError==OE_Abort ){ + sqlite3MayAbort(pParse); + } + sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); + if( p5Errmsg ) sqlite3VdbeChangeP5(v, p5Errmsg); +} + +/* +** Code an OP_Halt due to UNIQUE or PRIMARY KEY constraint violation. +*/ +SQLITE_PRIVATE void sqlite3UniqueConstraint( + Parse *pParse, /* Parsing context */ + int onError, /* Constraint type */ + Index *pIdx /* The index that triggers the constraint */ +){ + char *zErr; + int j; + StrAccum errMsg; + Table *pTab = pIdx->pTable; + + sqlite3StrAccumInit(&errMsg, 0, 0, 200); + errMsg.db = pParse->db; + for(j=0; jnKeyCol; j++){ + char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); + sqlite3StrAccumAppendAll(&errMsg, pTab->zName); + sqlite3StrAccumAppend(&errMsg, ".", 1); + sqlite3StrAccumAppendAll(&errMsg, zCol); + } + zErr = sqlite3StrAccumFinish(&errMsg); + sqlite3HaltConstraint(pParse, + IsPrimaryKeyIndex(pIdx) ? SQLITE_CONSTRAINT_PRIMARYKEY + : SQLITE_CONSTRAINT_UNIQUE, + onError, zErr, P4_DYNAMIC, P5_ConstraintUnique); +} + + +/* +** Code an OP_Halt due to non-unique rowid. +*/ +SQLITE_PRIVATE void sqlite3RowidConstraint( + Parse *pParse, /* Parsing context */ + int onError, /* Conflict resolution algorithm */ + Table *pTab /* The table with the non-unique rowid */ +){ + char *zMsg; + int rc; + if( pTab->iPKey>=0 ){ + zMsg = sqlite3MPrintf(pParse->db, "%s.%s", pTab->zName, + pTab->aCol[pTab->iPKey].zName); + rc = SQLITE_CONSTRAINT_PRIMARYKEY; + }else{ + zMsg = sqlite3MPrintf(pParse->db, "%s.rowid", pTab->zName); + rc = SQLITE_CONSTRAINT_ROWID; + } + sqlite3HaltConstraint(pParse, rc, onError, zMsg, P4_DYNAMIC, + P5_ConstraintUnique); +} + +/* +** Check to see if pIndex uses the collating sequence pColl. Return +** true if it does and false if it does not. +*/ +#ifndef SQLITE_OMIT_REINDEX +static int collationMatch(const char *zColl, Index *pIndex){ + int i; + assert( zColl!=0 ); + for(i=0; inColumn; i++){ + const char *z = pIndex->azColl[i]; + assert( z!=0 || pIndex->aiColumn[i]<0 ); + if( pIndex->aiColumn[i]>=0 && 0==sqlite3StrICmp(z, zColl) ){ + return 1; + } + } + return 0; +} +#endif + +/* +** Recompute all indices of pTab that use the collating sequence pColl. +** If pColl==0 then recompute all indices of pTab. +*/ +#ifndef SQLITE_OMIT_REINDEX +static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ + Index *pIndex; /* An index associated with pTab */ + + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( zColl==0 || collationMatch(zColl, pIndex) ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); + } + } +} +#endif + +/* +** Recompute all indices of all tables in all databases where the +** indices use the collating sequence pColl. If pColl==0 then recompute +** all indices everywhere. +*/ +#ifndef SQLITE_OMIT_REINDEX +static void reindexDatabases(Parse *pParse, char const *zColl){ + Db *pDb; /* A single database */ + int iDb; /* The database index number */ + sqlite3 *db = pParse->db; /* The database connection */ + HashElem *k; /* For looping over tables in pDb */ + Table *pTab; /* A table in the database */ + + assert( sqlite3BtreeHoldsAllMutexes(db) ); /* Needed for schema access */ + for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ + assert( pDb!=0 ); + for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + reindexTable(pParse, pTab, zColl); + } + } +} +#endif + +/* +** Generate code for the REINDEX command. +** +** REINDEX -- 1 +** REINDEX -- 2 +** REINDEX ?.? -- 3 +** REINDEX ?.? -- 4 +** +** Form 1 causes all indices in all attached databases to be rebuilt. +** Form 2 rebuilds all indices in all databases that use the named +** collating function. Forms 3 and 4 rebuild the named index or all +** indices associated with the named table. +*/ +#ifndef SQLITE_OMIT_REINDEX +SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ + CollSeq *pColl; /* Collating sequence to be reindexed, or NULL */ + char *z; /* Name of a table or index */ + const char *zDb; /* Name of the database */ + Table *pTab; /* A table in the database */ + Index *pIndex; /* An index associated with pTab */ + int iDb; /* The database index number */ + sqlite3 *db = pParse->db; /* The database connection */ + Token *pObjName; /* Name of the table or index to be reindexed */ + + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return; + } + + if( pName1==0 ){ + reindexDatabases(pParse, 0); + return; + }else if( NEVER(pName2==0) || pName2->z==0 ){ + char *zColl; + assert( pName1->z ); + zColl = sqlite3NameFromToken(pParse->db, pName1); + if( !zColl ) return; + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); + if( pColl ){ + reindexDatabases(pParse, zColl); + sqlite3DbFree(db, zColl); + return; + } + sqlite3DbFree(db, zColl); + } + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); + if( iDb<0 ) return; + z = sqlite3NameFromToken(db, pObjName); + if( z==0 ) return; + zDb = db->aDb[iDb].zName; + pTab = sqlite3FindTable(db, z, zDb); + if( pTab ){ + reindexTable(pParse, pTab, 0); + sqlite3DbFree(db, z); + return; + } + pIndex = sqlite3FindIndex(db, z, zDb); + sqlite3DbFree(db, z); + if( pIndex ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); + return; + } + sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); +} +#endif + +/* +** Return a KeyInfo structure that is appropriate for the given Index. +** +** The KeyInfo structure for an index is cached in the Index object. +** So there might be multiple references to the returned pointer. The +** caller should not try to modify the KeyInfo object. +** +** The caller should invoke sqlite3KeyInfoUnref() on the returned object +** when it has finished using it. +*/ +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ + if( pParse->nErr ) return 0; +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pIdx->pKeyInfo && pIdx->pKeyInfo->db!=pParse->db ){ + sqlite3KeyInfoUnref(pIdx->pKeyInfo); + pIdx->pKeyInfo = 0; + } +#endif + if( pIdx->pKeyInfo==0 ){ + int i; + int nCol = pIdx->nColumn; + int nKey = pIdx->nKeyCol; + KeyInfo *pKey; + if( pIdx->uniqNotNull ){ + pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); + }else{ + pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); + } + if( pKey ){ + assert( sqlite3KeyInfoIsWriteable(pKey) ); + for(i=0; iazColl[i]; + assert( zColl!=0 ); + pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : + sqlite3LocateCollSeq(pParse, zColl); + pKey->aSortOrder[i] = pIdx->aSortOrder[i]; + } + if( pParse->nErr ){ + sqlite3KeyInfoUnref(pKey); + }else{ + pIdx->pKeyInfo = pKey; + } + } + } + return sqlite3KeyInfoRef(pIdx->pKeyInfo); +} + +#ifndef SQLITE_OMIT_CTE +/* +** This routine is invoked once per CTE by the parser while parsing a +** WITH clause. +*/ +SQLITE_PRIVATE With *sqlite3WithAdd( + Parse *pParse, /* Parsing context */ + With *pWith, /* Existing WITH clause, or NULL */ + Token *pName, /* Name of the common-table */ + ExprList *pArglist, /* Optional column name list for the table */ + Select *pQuery /* Query used to initialize the table */ +){ + sqlite3 *db = pParse->db; + With *pNew; + char *zName; + + /* Check that the CTE name is unique within this WITH clause. If + ** not, store an error in the Parse structure. */ + zName = sqlite3NameFromToken(pParse->db, pName); + if( zName && pWith ){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ + sqlite3ErrorMsg(pParse, "duplicate WITH table name: %s", zName); + } + } + } + + if( pWith ){ + int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + pNew = sqlite3DbRealloc(db, pWith, nByte); + }else{ + pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + } + assert( zName!=0 || pNew==0 ); + assert( db->mallocFailed==0 || pNew==0 ); + + if( pNew==0 ){ + sqlite3ExprListDelete(db, pArglist); + sqlite3SelectDelete(db, pQuery); + sqlite3DbFree(db, zName); + pNew = pWith; + }else{ + pNew->a[pNew->nCte].pSelect = pQuery; + pNew->a[pNew->nCte].pCols = pArglist; + pNew->a[pNew->nCte].zName = zName; + pNew->a[pNew->nCte].zErr = 0; + pNew->nCte++; + } + + return pNew; +} + +/* +** Free the contents of the With object passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3WithDelete(sqlite3 *db, With *pWith){ + if( pWith ){ + int i; + for(i=0; inCte; i++){ + struct Cte *pCte = &pWith->a[i]; + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); + } + sqlite3DbFree(db, pWith); + } +} +#endif /* !defined(SQLITE_OMIT_CTE) */ + +/************** End of build.c ***********************************************/ +/************** Begin file callback.c ****************************************/ +/* +** 2005 May 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains functions used to access the internal hash tables +** of user defined functions and collation sequences. +*/ + + +/* +** Invoke the 'collation needed' callback to request a collation sequence +** in the encoding enc of name zName, length nName. +*/ +static void callCollNeeded(sqlite3 *db, int enc, const char *zName){ + assert( !db->xCollNeeded || !db->xCollNeeded16 ); + if( db->xCollNeeded ){ + char *zExternal = sqlite3DbStrDup(db, zName); + if( !zExternal ) return; + db->xCollNeeded(db->pCollNeededArg, db, enc, zExternal); + sqlite3DbFree(db, zExternal); + } +#ifndef SQLITE_OMIT_UTF16 + if( db->xCollNeeded16 ){ + char const *zExternal; + sqlite3_value *pTmp = sqlite3ValueNew(db); + sqlite3ValueSetStr(pTmp, -1, zName, SQLITE_UTF8, SQLITE_STATIC); + zExternal = sqlite3ValueText(pTmp, SQLITE_UTF16NATIVE); + if( zExternal ){ + db->xCollNeeded16(db->pCollNeededArg, db, (int)ENC(db), zExternal); + } + sqlite3ValueFree(pTmp); + } +#endif +} + +/* +** This routine is called if the collation factory fails to deliver a +** collation function in the best encoding but there may be other versions +** of this collation function (for other text encodings) available. Use one +** of these instead if they exist. Avoid a UTF-8 <-> UTF-16 conversion if +** possible. +*/ +static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ + CollSeq *pColl2; + char *z = pColl->zName; + int i; + static const u8 aEnc[] = { SQLITE_UTF16BE, SQLITE_UTF16LE, SQLITE_UTF8 }; + for(i=0; i<3; i++){ + pColl2 = sqlite3FindCollSeq(db, aEnc[i], z, 0); + if( pColl2->xCmp!=0 ){ + memcpy(pColl, pColl2, sizeof(CollSeq)); + pColl->xDel = 0; /* Do not copy the destructor */ + return SQLITE_OK; + } + } + return SQLITE_ERROR; +} + +/* +** This function is responsible for invoking the collation factory callback +** or substituting a collation sequence of a different encoding when the +** requested collation sequence is not available in the desired encoding. +** +** If it is not NULL, then pColl must point to the database native encoding +** collation sequence with name zName, length nName. +** +** The return value is either the collation sequence to be used in database +** db for collation type name zName, length nName, or NULL, if no collation +** sequence can be found. If no collation is found, leave an error message. +** +** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( + Parse *pParse, /* Parsing context */ + u8 enc, /* The desired encoding for the collating sequence */ + CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ + const char *zName /* Collating sequence name */ +){ + CollSeq *p; + sqlite3 *db = pParse->db; + + p = pColl; + if( !p ){ + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( !p || !p->xCmp ){ + /* No collation sequence of this type for this encoding is registered. + ** Call the collation factory to see if it can supply us with one. + */ + callCollNeeded(db, enc, zName); + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( p && !p->xCmp && synthCollSeq(db, p) ){ + p = 0; + } + assert( !p || p->xCmp ); + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + } + return p; +} + +/* +** This routine is called on a collation sequence before it is used to +** check that it is defined. An undefined collation sequence exists when +** a database is loaded that contains references to collation sequences +** that have not been defined by sqlite3_create_collation() etc. +** +** If required, this routine calls the 'collation needed' callback to +** request a definition of the collating sequence. If this doesn't work, +** an equivalent collating sequence that uses a text encoding different +** from the main database is substituted, if one is available. +*/ +SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ + if( pColl ){ + const char *zName = pColl->zName; + sqlite3 *db = pParse->db; + CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); + if( !p ){ + return SQLITE_ERROR; + } + assert( p==pColl ); + } + return SQLITE_OK; +} + + + +/* +** Locate and return an entry from the db.aCollSeq hash table. If the entry +** specified by zName and nName is not found and parameter 'create' is +** true, then create a new entry. Otherwise return NULL. +** +** Each pointer stored in the sqlite3.aCollSeq hash table contains an +** array of three CollSeq structures. The first is the collation sequence +** prefferred for UTF-8, the second UTF-16le, and the third UTF-16be. +** +** Stored immediately after the three collation sequences is a copy of +** the collation sequence name. A pointer to this string is stored in +** each collation sequence structure. +*/ +static CollSeq *findCollSeqEntry( + sqlite3 *db, /* Database connection */ + const char *zName, /* Name of the collating sequence */ + int create /* Create a new entry if true */ +){ + CollSeq *pColl; + int nName = sqlite3Strlen30(zName); + pColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + + if( 0==pColl && create ){ + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 ); + if( pColl ){ + CollSeq *pDel = 0; + pColl[0].zName = (char*)&pColl[3]; + pColl[0].enc = SQLITE_UTF8; + pColl[1].zName = (char*)&pColl[3]; + pColl[1].enc = SQLITE_UTF16LE; + pColl[2].zName = (char*)&pColl[3]; + pColl[2].enc = SQLITE_UTF16BE; + memcpy(pColl[0].zName, zName, nName); + pColl[0].zName[nName] = 0; + pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, nName, pColl); + + /* If a malloc() failure occurred in sqlite3HashInsert(), it will + ** return the pColl pointer to be deleted (because it wasn't added + ** to the hash table). + */ + assert( pDel==0 || pDel==pColl ); + if( pDel!=0 ){ + db->mallocFailed = 1; + sqlite3DbFree(db, pDel); + pColl = 0; + } + } + } + return pColl; +} + +/* +** Parameter zName points to a UTF-8 encoded string nName bytes long. +** Return the CollSeq* pointer for the collation sequence named zName +** for the encoding 'enc' from the database 'db'. +** +** If the entry specified is not found and 'create' is true, then create a +** new entry. Otherwise return NULL. +** +** A separate function sqlite3LocateCollSeq() is a wrapper around +** this routine. sqlite3LocateCollSeq() invokes the collation factory +** if necessary and generates an error message if the collating sequence +** cannot be found. +** +** See also: sqlite3LocateCollSeq(), sqlite3GetCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( + sqlite3 *db, + u8 enc, + const char *zName, + int create +){ + CollSeq *pColl; + if( zName ){ + pColl = findCollSeqEntry(db, zName, create); + }else{ + pColl = db->pDfltColl; + } + assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE ); + if( pColl ) pColl += enc-1; + return pColl; +} + +/* During the search for the best function definition, this procedure +** is called to test how well the function passed as the first argument +** matches the request for a function with nArg arguments in a system +** that uses encoding enc. The value returned indicates how well the +** request is matched. A higher value indicates a better match. +** +** If nArg is -1 that means to only return a match (non-zero) if p->nArg +** is also -1. In other words, we are searching for a function that +** takes a variable number of arguments. +** +** If nArg is -2 that means that we are searching for any function +** regardless of the number of arguments it uses, so return a positive +** match score for any +** +** The returned value is always between 0 and 6, as follows: +** +** 0: Not a match. +** 1: UTF8/16 conversion required and function takes any number of arguments. +** 2: UTF16 byte order change required and function takes any number of args. +** 3: encoding matches and function takes any number of arguments +** 4: UTF8/16 conversion required - argument count matches exactly +** 5: UTF16 byte order conversion required - argument count matches exactly +** 6: Perfect match: encoding and argument count match exactly. +** +** If nArg==(-2) then any function with a non-null xStep or xFunc is +** a perfect match and any function with both xStep and xFunc NULL is +** a non-match. +*/ +#define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ +static int matchQuality( + FuncDef *p, /* The function we are evaluating for match quality */ + int nArg, /* Desired number of arguments. (-1)==any */ + u8 enc /* Desired text encoding */ +){ + int match; + + /* nArg of -2 is a special case */ + if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH; + + /* Wrong number of arguments means "no match" */ + if( p->nArg!=nArg && p->nArg>=0 ) return 0; + + /* Give a better score to a function with a specific number of arguments + ** than to function that accepts any number of arguments. */ + if( p->nArg==nArg ){ + match = 4; + }else{ + match = 1; + } + + /* Bonus points if the text encoding matches */ + if( enc==(p->funcFlags & SQLITE_FUNC_ENCMASK) ){ + match += 2; /* Exact encoding match */ + }else if( (enc & p->funcFlags & 2)!=0 ){ + match += 1; /* Both are UTF16, but with different byte orders */ + } + + return match; +} + +/* +** Search a FuncDefHash for a function with the given name. Return +** a pointer to the matching FuncDef if found, or 0 if there is no match. +*/ +static FuncDef *functionSearch( + FuncDefHash *pHash, /* Hash table to search */ + int h, /* Hash of the name */ + const char *zFunc, /* Name of function */ + int nFunc /* Number of bytes in zFunc */ +){ + FuncDef *p; + for(p=pHash->a[h]; p; p=p->pHash){ + if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ + return p; + } + } + return 0; +} + +/* +** Insert a new FuncDef into a FuncDefHash hash table. +*/ +SQLITE_PRIVATE void sqlite3FuncDefInsert( + FuncDefHash *pHash, /* The hash table into which to insert */ + FuncDef *pDef /* The function definition to insert */ +){ + FuncDef *pOther; + int nName = sqlite3Strlen30(pDef->zName); + u8 c1 = (u8)pDef->zName[0]; + int h = (sqlite3UpperToLower[c1] + nName) % ArraySize(pHash->a); + pOther = functionSearch(pHash, h, pDef->zName, nName); + if( pOther ){ + assert( pOther!=pDef && pOther->pNext!=pDef ); + pDef->pNext = pOther->pNext; + pOther->pNext = pDef; + }else{ + pDef->pNext = 0; + pDef->pHash = pHash->a[h]; + pHash->a[h] = pDef; + } +} + + + +/* +** Locate a user function given a name, a number of arguments and a flag +** indicating whether the function prefers UTF-16 over UTF-8. Return a +** pointer to the FuncDef structure that defines that function, or return +** NULL if the function does not exist. +** +** If the createFlag argument is true, then a new (blank) FuncDef +** structure is created and liked into the "db" structure if a +** no matching function previously existed. +** +** If nArg is -2, then the first valid function found is returned. A +** function is valid if either xFunc or xStep is non-zero. The nArg==(-2) +** case is used to see if zName is a valid function name for some number +** of arguments. If nArg is -2, then createFlag must be 0. +** +** If createFlag is false, then a function with the required name and +** number of arguments may be returned even if the eTextRep flag does not +** match that requested. +*/ +SQLITE_PRIVATE FuncDef *sqlite3FindFunction( + sqlite3 *db, /* An open database */ + const char *zName, /* Name of the function. Not null-terminated */ + int nName, /* Number of characters in the name */ + int nArg, /* Number of arguments. -1 means any number */ + u8 enc, /* Preferred text encoding */ + u8 createFlag /* Create new entry if true and does not otherwise exist */ +){ + FuncDef *p; /* Iterator variable */ + FuncDef *pBest = 0; /* Best match found so far */ + int bestScore = 0; /* Score of best match */ + int h; /* Hash value */ + + assert( nArg>=(-2) ); + assert( nArg>=(-1) || createFlag==0 ); + h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); + + /* First search for a match amongst the application-defined functions. + */ + p = functionSearch(&db->aFunc, h, zName, nName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ + pBest = p; + bestScore = score; + } + p = p->pNext; + } + + /* If no match is found, search the built-in functions. + ** + ** If the SQLITE_PreferBuiltin flag is set, then search the built-in + ** functions even if a prior app-defined function was found. And give + ** priority to built-in functions. + ** + ** Except, if createFlag is true, that means that we are trying to + ** install a new function. Whatever FuncDef structure is returned it will + ** have fields overwritten with new information appropriate for the + ** new function. But the FuncDefs for built-in functions are read-only. + ** So we must not search for built-ins when creating a new function. + */ + if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + bestScore = 0; + p = functionSearch(pHash, h, zName, nName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ + pBest = p; + bestScore = score; + } + p = p->pNext; + } + } + + /* If the createFlag parameter is true and the search did not reveal an + ** exact match for the name, number of arguments and encoding, then add a + ** new entry to the hash table and return it. + */ + if( createFlag && bestScorezName = (char *)&pBest[1]; + pBest->nArg = (u16)nArg; + pBest->funcFlags = enc; + memcpy(pBest->zName, zName, nName); + pBest->zName[nName] = 0; + sqlite3FuncDefInsert(&db->aFunc, pBest); + } + + if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){ + return pBest; + } + return 0; +} + +/* +** Free all resources held by the schema structure. The void* argument points +** at a Schema struct. This function does not call sqlite3DbFree(db, ) on the +** pointer itself, it just cleans up subsidiary resources (i.e. the contents +** of the schema hash tables). +** +** The Schema.cache_size variable is not cleared. +*/ +SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ + Hash temp1; + Hash temp2; + HashElem *pElem; + Schema *pSchema = (Schema *)p; + + temp1 = pSchema->tblHash; + temp2 = pSchema->trigHash; + sqlite3HashInit(&pSchema->trigHash); + sqlite3HashClear(&pSchema->idxHash); + for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ + sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); + } + sqlite3HashClear(&temp2); + sqlite3HashInit(&pSchema->tblHash); + for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ + Table *pTab = sqliteHashData(pElem); + sqlite3DeleteTable(0, pTab); + } + sqlite3HashClear(&temp1); + sqlite3HashClear(&pSchema->fkeyHash); + pSchema->pSeqTab = 0; + if( pSchema->schemaFlags & DB_SchemaLoaded ){ + pSchema->iGeneration++; + pSchema->schemaFlags &= ~DB_SchemaLoaded; + } +} + +/* +** Find and return the schema associated with a BTree. Create +** a new one if necessary. +*/ +SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ + Schema * p; + if( pBt ){ + p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaClear); + }else{ + p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); + } + if( !p ){ + db->mallocFailed = 1; + }else if ( 0==p->file_format ){ + sqlite3HashInit(&p->tblHash); + sqlite3HashInit(&p->idxHash); + sqlite3HashInit(&p->trigHash); + sqlite3HashInit(&p->fkeyHash); + p->enc = SQLITE_UTF8; + } + return p; +} + +/************** End of callback.c ********************************************/ +/************** Begin file delete.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** in order to generate code for DELETE FROM statements. +*/ + +/* +** While a SrcList can in general represent multiple tables and subqueries +** (as in the FROM clause of a SELECT statement) in this case it contains +** the name of a single table, as one might find in an INSERT, DELETE, +** or UPDATE statement. Look up that table in the symbol table and +** return a pointer. Set an error message and return NULL if the table +** name is not found or if any other error occurs. +** +** The following fields are initialized appropriate in pSrc: +** +** pSrc->a[0].pTab Pointer to the Table object +** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one +** +*/ +SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ + struct SrcList_item *pItem = pSrc->a; + Table *pTab; + assert( pItem && pSrc->nSrc==1 ); + pTab = sqlite3LocateTableItem(pParse, 0, pItem); + sqlite3DeleteTable(pParse->db, pItem->pTab); + pItem->pTab = pTab; + if( pTab ){ + pTab->nRef++; + } + if( sqlite3IndexedByLookup(pParse, pItem) ){ + pTab = 0; + } + return pTab; +} + +/* +** Check to make sure the given table is writable. If it is not +** writable, generate an error message and return 1. If it is +** writable return 0; +*/ +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ + /* A table is not writable under the following circumstances: + ** + ** 1) It is a virtual table and no implementation of the xUpdate method + ** has been provided, or + ** 2) It is a system table (i.e. sqlite_master), this call is not + ** part of a nested parse and writable_schema pragma has not + ** been specified. + ** + ** In either case leave an error message in pParse and return non-zero. + */ + if( ( IsVirtual(pTab) + && sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ) + || ( (pTab->tabFlags & TF_Readonly)!=0 + && (pParse->db->flags & SQLITE_WriteSchema)==0 + && pParse->nested==0 ) + ){ + sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); + return 1; + } + +#ifndef SQLITE_OMIT_VIEW + if( !viewOk && pTab->pSelect ){ + sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); + return 1; + } +#endif + return 0; +} + + +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) +/* +** Evaluate a view and store its result in an ephemeral table. The +** pWhere argument is an optional WHERE clause that restricts the +** set of rows in the view that are to be added to the ephemeral table. +*/ +SQLITE_PRIVATE void sqlite3MaterializeView( + Parse *pParse, /* Parsing context */ + Table *pView, /* View definition */ + Expr *pWhere, /* Optional WHERE clause to be added */ + int iCur /* Cursor number for ephemerial table */ +){ + SelectDest dest; + Select *pSel; + SrcList *pFrom; + sqlite3 *db = pParse->db; + int iDb = sqlite3SchemaToIndex(db, pView->pSchema); + pWhere = sqlite3ExprDup(db, pWhere, 0); + pFrom = sqlite3SrcListAppend(db, 0, 0, 0); + if( pFrom ){ + assert( pFrom->nSrc==1 ); + pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + assert( pFrom->a[0].pOn==0 ); + assert( pFrom->a[0].pUsing==0 ); + } + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); + sqlite3Select(pParse, pSel, &dest); + sqlite3SelectDelete(db, pSel); +} +#endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ + +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) +/* +** Generate an expression tree to implement the WHERE, ORDER BY, +** and LIMIT/OFFSET portion of DELETE and UPDATE statements. +** +** DELETE FROM table_wxyz WHERE a<5 ORDER BY a LIMIT 1; +** \__________________________/ +** pLimitWhere (pInClause) +*/ +SQLITE_PRIVATE Expr *sqlite3LimitWhere( + Parse *pParse, /* The parser context */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, /* The ORDER BY clause. May be null */ + Expr *pLimit, /* The LIMIT clause. May be null */ + Expr *pOffset, /* The OFFSET clause. May be null */ + char *zStmtType /* Either DELETE or UPDATE. For err msgs. */ +){ + Expr *pWhereRowid = NULL; /* WHERE rowid .. */ + Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ + Expr *pSelectRowid = NULL; /* SELECT rowid ... */ + ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ + Select *pSelect = NULL; /* Complete SELECT tree */ + + /* Check that there isn't an ORDER BY without a LIMIT clause. + */ + if( pOrderBy && (pLimit == 0) ) { + sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); + goto limit_where_cleanup_2; + } + + /* We only need to generate a select expression if there + ** is a limit/offset term to enforce. + */ + if( pLimit == 0 ) { + /* if pLimit is null, pOffset will always be null as well. */ + assert( pOffset == 0 ); + return pWhere; + } + + /* Generate a select expression tree to enforce the limit/offset + ** term for the DELETE or UPDATE statement. For example: + ** DELETE FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** becomes: + ** DELETE FROM table_a WHERE rowid IN ( + ** SELECT rowid FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** ); + */ + + pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); + if( pSelectRowid == 0 ) goto limit_where_cleanup_2; + pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid); + if( pEList == 0 ) goto limit_where_cleanup_2; + + /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree + ** and the SELECT subtree. */ + pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); + if( pSelectSrc == 0 ) { + sqlite3ExprListDelete(pParse->db, pEList); + goto limit_where_cleanup_2; + } + + /* generate the SELECT expression tree. */ + pSelect = sqlite3SelectNew(pParse,pEList,pSelectSrc,pWhere,0,0, + pOrderBy,0,pLimit,pOffset); + if( pSelect == 0 ) return 0; + + /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ + pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); + if( pWhereRowid == 0 ) goto limit_where_cleanup_1; + pInClause = sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0); + if( pInClause == 0 ) goto limit_where_cleanup_1; + + pInClause->x.pSelect = pSelect; + pInClause->flags |= EP_xIsSelect; + sqlite3ExprSetHeight(pParse, pInClause); + return pInClause; + + /* something went wrong. clean up anything allocated. */ +limit_where_cleanup_1: + sqlite3SelectDelete(pParse->db, pSelect); + return 0; + +limit_where_cleanup_2: + sqlite3ExprDelete(pParse->db, pWhere); + sqlite3ExprListDelete(pParse->db, pOrderBy); + sqlite3ExprDelete(pParse->db, pLimit); + sqlite3ExprDelete(pParse->db, pOffset); + return 0; +} +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) */ + /* && !defined(SQLITE_OMIT_SUBQUERY) */ + +/* +** Generate code for a DELETE FROM statement. +** +** DELETE FROM table_wxyz WHERE a<5 AND b NOT NULL; +** \________/ \________________/ +** pTabList pWhere +*/ +SQLITE_PRIVATE void sqlite3DeleteFrom( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* The table from which we should delete things */ + Expr *pWhere /* The WHERE clause. May be null */ +){ + Vdbe *v; /* The virtual database engine */ + Table *pTab; /* The table from which records will be deleted */ + const char *zDb; /* Name of database holding pTab */ + int i; /* Loop counter */ + WhereInfo *pWInfo; /* Information about the WHERE clause */ + Index *pIdx; /* For looping over indices of the table */ + int iTabCur; /* Cursor number for the table */ + int iDataCur; /* VDBE cursor for the canonical data source */ + int iIdxCur; /* Cursor number of the first index */ + int nIdx; /* Number of indices */ + sqlite3 *db; /* Main database structure */ + AuthContext sContext; /* Authorization context */ + NameContext sNC; /* Name context to resolve expressions in */ + int iDb; /* Database number */ + int memCnt = -1; /* Memory cell used for change counting */ + int rcauth; /* Value returned by authorization callback */ + int okOnePass; /* True for one-pass algorithm without the FIFO */ + int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ + u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ + Index *pPk; /* The PRIMARY KEY index on the table */ + int iPk = 0; /* First of nPk registers holding PRIMARY KEY value */ + i16 nPk = 1; /* Number of columns in the PRIMARY KEY */ + int iKey; /* Memory cell holding key of row to be deleted */ + i16 nKey; /* Number of memory cells in the row key */ + int iEphCur = 0; /* Ephemeral table holding all primary key values */ + int iRowSet = 0; /* Register for rowset of rows to delete */ + int addrBypass = 0; /* Address of jump over the delete logic */ + int addrLoop = 0; /* Top of the delete loop */ + int addrDelete = 0; /* Jump directly to the delete logic */ + int addrEphOpen = 0; /* Instruction to open the Ephermeral table */ + +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True if attempting to delete from a view */ + Trigger *pTrigger; /* List of table triggers, if required */ +#endif + + memset(&sContext, 0, sizeof(sContext)); + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ + goto delete_from_cleanup; + } + assert( pTabList->nSrc==1 ); + + /* Locate the table which we want to delete. This table has to be + ** put in an SrcList structure because some of the subroutines we + ** will be calling are designed to work with multiple tables and expect + ** an SrcList* parameter instead of just a Table* parameter. + */ + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ) goto delete_from_cleanup; + + /* Figure out if we have any triggers and if the table being + ** deleted from is a view + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + isView = pTab->pSelect!=0; +#else +# define pTrigger 0 +# define isView 0 +#endif +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + + /* If pTab is really a view, make sure it has been initialized. + */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto delete_from_cleanup; + } + + if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + goto delete_from_cleanup; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDbnDb ); + zDb = db->aDb[iDb].zName; + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb); + assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); + if( rcauth==SQLITE_DENY ){ + goto delete_from_cleanup; + } + assert(!isView || pTrigger); + + /* Assign cursor numbers to the table and all its indices. + */ + assert( pTabList->nSrc==1 ); + iTabCur = pTabList->a[0].iCursor = pParse->nTab++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ + pParse->nTab++; + } + + /* Start the view context + */ + if( isView ){ + sqlite3AuthContextPush(pParse, &sContext, pTab->zName); + } + + /* Begin generating code. + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ){ + goto delete_from_cleanup; + } + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, 1, iDb); + + /* If we are trying to delete from a view, realize that view into + ** a ephemeral table. + */ +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) + if( isView ){ + sqlite3MaterializeView(pParse, pTab, pWhere, iTabCur); + iDataCur = iIdxCur = iTabCur; + } +#endif + + /* Resolve the column names in the WHERE clause. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + if( sqlite3ResolveExprNames(&sNC, pWhere) ){ + goto delete_from_cleanup; + } + + /* Initialize the counter of the number of rows deleted, if + ** we are counting rows. + */ + if( db->flags & SQLITE_CountRows ){ + memCnt = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt); + } + +#ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION + /* Special case: A DELETE without a WHERE clause deletes everything. + ** It is easier just to erase the whole table. Prior to version 3.6.5, + ** this optimization caused the row change count (the value returned by + ** API function sqlite3_count_changes) to be set incorrectly. */ + if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) + && 0==sqlite3FkRequired(pParse, pTab, 0, 0) + ){ + assert( !isView ); + sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt, + pTab->zName, P4_STATIC); + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->pSchema==pTab->pSchema ); + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); + } + }else +#endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ + { + if( HasRowid(pTab) ){ + /* For a rowid table, initialize the RowSet to an empty set */ + pPk = 0; + nPk = 1; + iRowSet = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); + }else{ + /* For a WITHOUT ROWID table, create an ephermeral table used to + ** hold all primary keys for rows to be deleted. */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + nPk = pPk->nKeyCol; + iPk = pParse->nMem+1; + pParse->nMem += nPk; + iEphCur = pParse->nTab++; + addrEphOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEphCur, nPk); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + + /* Construct a query to find the rowid or primary key for every row + ** to be deleted, based on the WHERE clause. + */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, + WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, + iTabCur+1); + if( pWInfo==0 ) goto delete_from_cleanup; + okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + + /* Keep track of the number of rows to be deleted */ + if( db->flags & SQLITE_CountRows ){ + sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); + } + + /* Extract the rowid or primary key for the current row */ + if( pPk ){ + for(i=0; iaiColumn[i], iPk+i); + } + iKey = iPk; + }else{ + iKey = pParse->nMem + 1; + iKey = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iTabCur, iKey, 0); + if( iKey>pParse->nMem ) pParse->nMem = iKey; + } + + if( okOnePass ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only + ** one, so just keep it in its register(s) and fall through to the + ** delete code. + */ + nKey = nPk; /* OP_Found will use an unpacked key */ + aToOpen = sqlite3DbMallocRaw(db, nIdx+2); + if( aToOpen==0 ){ + sqlite3WhereEnd(pWInfo); + goto delete_from_cleanup; + } + memset(aToOpen, 1, nIdx+1); + aToOpen[nIdx+1] = 0; + if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; + if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; + if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); + addrDelete = sqlite3VdbeAddOp0(v, OP_Goto); /* Jump to DELETE logic */ + }else if( pPk ){ + /* Construct a composite key for the row to be deleted and remember it */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + }else{ + /* Get the rowid of the row to be deleted and remember it in the RowSet */ + nKey = 1; /* OP_Seek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } + + /* End of the WHERE loop */ + sqlite3WhereEnd(pWInfo); + if( okOnePass ){ + /* Bypass the delete logic below if the WHERE loop found zero rows */ + addrBypass = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBypass); + sqlite3VdbeJumpHere(v, addrDelete); + } + + /* Unless this is a view, open cursors for the table we are + ** deleting from and all its indices. If this is a view, then the + ** only effect this statement has is to fire the INSTEAD OF + ** triggers. + */ + if( !isView ){ + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, + &iDataCur, &iIdxCur); + assert( pPk || iDataCur==iTabCur ); + assert( pPk || iIdxCur==iDataCur+1 ); + } + + /* Set up a loop over the rowids/primary-keys that were found in the + ** where-clause loop above. + */ + if( okOnePass ){ + /* Just one row. Hence the top-of-loop is a no-op */ + assert( nKey==nPk ); /* OP_Found will use an unpacked key */ + if( aToOpen[iDataCur-iTabCur] ){ + assert( pPk!=0 ); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); + VdbeCoverage(v); + } + }else if( pPk ){ + addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_RowKey, iEphCur, iKey); + assert( nKey==0 ); /* OP_Found will use a composite key */ + }else{ + addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); + VdbeCoverage(v); + assert( nKey==1 ); + } + + /* Delete the row */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, OE_Abort); + sqlite3MayAbort(pParse); + }else +#endif + { + int count = (pParse->nested==0); /* True to count changes */ + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + iKey, nKey, count, OE_Default, okOnePass); + } + + /* End of the loop over all rowids/primary-keys. */ + if( okOnePass ){ + sqlite3VdbeResolveLabel(v, addrBypass); + }else if( pPk ){ + sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrLoop); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrLoop); + sqlite3VdbeJumpHere(v, addrLoop); + } + + /* Close the cursors open on the table and its indexes. */ + if( !isView && !IsVirtual(pTab) ){ + if( !pPk ) sqlite3VdbeAddOp1(v, OP_Close, iDataCur); + for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ + sqlite3VdbeAddOp1(v, OP_Close, iIdxCur + i); + } + } + } /* End non-truncate path */ + + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. + */ + if( pParse->nested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* Return the number of rows that were deleted. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. + */ + if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, memCnt, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC); + } + +delete_from_cleanup: + sqlite3AuthContextPop(&sContext); + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprDelete(db, pWhere); + sqlite3DbFree(db, aToOpen); + return; +} +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** thely may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif + +/* +** This routine generates VDBE code that causes a single row of a +** single table to be deleted. Both the original table entry and +** all indices are removed. +** +** Preconditions: +** +** 1. iDataCur is an open cursor on the btree that is the canonical data +** store for the table. (This will be either the table itself, +** in the case of a rowid table, or the PRIMARY KEY index in the case +** of a WITHOUT ROWID table.) +** +** 2. Read/write cursors for all indices of pTab must be open as +** cursor number iIdxCur+i for the i-th index. +** +** 3. The primary key for the row to be deleted must be stored in a +** sequence of nPk memory cells starting at iPk. If nPk==0 that means +** that a search record formed from OP_MakeRecord is contained in the +** single memory location iPk. +*/ +SQLITE_PRIVATE void sqlite3GenerateRowDelete( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table containing the row to be deleted */ + Trigger *pTrigger, /* List of triggers to (potentially) fire */ + int iDataCur, /* Cursor from which column data is extracted */ + int iIdxCur, /* First index cursor */ + int iPk, /* First memory cell containing the PRIMARY KEY */ + i16 nPk, /* Number of PRIMARY KEY memory cells */ + u8 count, /* If non-zero, increment the row change counter */ + u8 onconf, /* Default ON CONFLICT policy for triggers */ + u8 bNoSeek /* iDataCur is already pointing to the row to delete */ +){ + Vdbe *v = pParse->pVdbe; /* Vdbe */ + int iOld = 0; /* First register in OLD.* array */ + int iLabel; /* Label resolved to end of generated code */ + u8 opSeek; /* Seek opcode */ + + /* Vdbe is guaranteed to have been allocated by this stage. */ + assert( v ); + VdbeModuleComment((v, "BEGIN: GenRowDel(%d,%d,%d,%d)", + iDataCur, iIdxCur, iPk, (int)nPk)); + + /* Seek cursor iCur to the row to delete. If this row no longer exists + ** (this can happen if a trigger program has already deleted it), do + ** not attempt to delete it or fire any DELETE triggers. */ + iLabel = sqlite3VdbeMakeLabel(v); + opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + if( !bNoSeek ){ + sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); + VdbeCoverageIf(v, opSeek==OP_NotExists); + VdbeCoverageIf(v, opSeek==OP_NotFound); + } + + /* If there are any triggers to fire, allocate a range of registers to + ** use for the old.* references in the triggers. */ + if( sqlite3FkRequired(pParse, pTab, 0, 0) || pTrigger ){ + u32 mask; /* Mask of OLD.* columns in use */ + int iCol; /* Iterator used while populating OLD.* */ + int addrStart; /* Start of BEFORE trigger programs */ + + /* TODO: Could use temporary registers here. Also could attempt to + ** avoid copying the contents of the rowid register. */ + mask = sqlite3TriggerColmask( + pParse, pTrigger, 0, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onconf + ); + mask |= sqlite3FkOldmask(pParse, pTab); + iOld = pParse->nMem+1; + pParse->nMem += (1 + pTab->nCol); + + /* Populate the OLD.* pseudo-table register array. These values will be + ** used by any BEFORE and AFTER triggers that exist. */ + sqlite3VdbeAddOp2(v, OP_Copy, iPk, iOld); + for(iCol=0; iColnCol; iCol++){ + testcase( mask!=0xffffffff && iCol==31 ); + testcase( mask!=0xffffffff && iCol==32 ); + if( mask==0xffffffff || (iCol<=31 && (mask & MASKBIT32(iCol))!=0) ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, iCol, iOld+iCol+1); + } + } + + /* Invoke BEFORE DELETE trigger programs. */ + addrStart = sqlite3VdbeCurrentAddr(v); + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_BEFORE, pTab, iOld, onconf, iLabel + ); + + /* If any BEFORE triggers were coded, then seek the cursor to the + ** row to be deleted again. It may be that the BEFORE triggers moved + ** the cursor or of already deleted the row that the cursor was + ** pointing to. + */ + if( addrStartpSelect==0 ){ + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); + if( count ){ + sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); + } + } + + /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to + ** handle rows (possibly in other tables) that refer via a foreign key + ** to the row just deleted. */ + sqlite3FkActions(pParse, pTab, 0, iOld, 0, 0); + + /* Invoke AFTER DELETE trigger programs. */ + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel + ); + + /* Jump here if the row had already been deleted before any BEFORE + ** trigger programs were invoked. Or if a trigger program throws a + ** RAISE(IGNORE) exception. */ + sqlite3VdbeResolveLabel(v, iLabel); + VdbeModuleComment((v, "END: GenRowDel()")); +} + +/* +** This routine generates VDBE code that causes the deletion of all +** index entries associated with a single row of a single table, pTab +** +** Preconditions: +** +** 1. A read/write cursor "iDataCur" must be open on the canonical storage +** btree for the table pTab. (This will be either the table itself +** for rowid tables or to the primary key index for WITHOUT ROWID +** tables.) +** +** 2. Read/write cursors for all indices of pTab must be open as +** cursor number iIdxCur+i for the i-th index. (The pTab->pIndex +** index is the 0-th index.) +** +** 3. The "iDataCur" cursor must be already be positioned on the row +** that is to be deleted. +*/ +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Table containing the row to be deleted */ + int iDataCur, /* Cursor of table holding data. */ + int iIdxCur, /* First index cursor */ + int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ +){ + int i; /* Index loop counter */ + int r1 = -1; /* Register holding an index key */ + int iPartIdxLabel; /* Jump destination for skipping partial index entries */ + Index *pIdx; /* Current index */ + Index *pPrior = 0; /* Prior index */ + Vdbe *v; /* The prepared statement under construction */ + Index *pPk; /* PRIMARY KEY index, or NULL for rowid tables */ + + v = pParse->pVdbe; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ + assert( iIdxCur+i!=iDataCur || pPk==pIdx ); + if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; + if( pIdx==pPk ) continue; + VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, + &iPartIdxLabel, pPrior, r1); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); + pPrior = pIdx; + } +} + +/* +** Generate code that will assemble an index key and stores it in register +** regOut. The key with be for index pIdx which is an index on pTab. +** iCur is the index of a cursor open on the pTab table and pointing to +** the entry that needs indexing. If pTab is a WITHOUT ROWID table, then +** iCur must be the cursor of the PRIMARY KEY index. +** +** Return a register number which is the first in a block of +** registers that holds the elements of the index key. The +** block of registers has already been deallocated by the time +** this routine returns. +** +** If *piPartIdxLabel is not NULL, fill it in with a label and jump +** to that label if pIdx is a partial index that should be skipped. +** The label should be resolved using sqlite3ResolvePartIdxLabel(). +** A partial index should be skipped if its WHERE clause evaluates +** to false or null. If pIdx is not a partial index, *piPartIdxLabel +** will be set to zero which is an empty label that is ignored by +** sqlite3ResolvePartIdxLabel(). +** +** The pPrior and regPrior parameters are used to implement a cache to +** avoid unnecessary register loads. If pPrior is not NULL, then it is +** a pointer to a different index for which an index key has just been +** computed into register regPrior. If the current pIdx index is generating +** its key into the same sequence of registers and if pPrior and pIdx share +** a column in common, then the register corresponding to that column already +** holds the correct value and the loading of that register is skipped. +** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK +** on a table with multiple indices, and especially with the ROWID or +** PRIMARY KEY columns of the index. +*/ +SQLITE_PRIVATE int sqlite3GenerateIndexKey( + Parse *pParse, /* Parsing context */ + Index *pIdx, /* The index for which to generate a key */ + int iDataCur, /* Cursor number from which to take column data */ + int regOut, /* Put the new key into this register if not 0 */ + int prefixOnly, /* Compute only a unique prefix of the key */ + int *piPartIdxLabel, /* OUT: Jump to this label to skip partial index */ + Index *pPrior, /* Previously generated index key */ + int regPrior /* Register holding previous generated key */ +){ + Vdbe *v = pParse->pVdbe; + int j; + Table *pTab = pIdx->pTable; + int regBase; + int nCol; + + if( piPartIdxLabel ){ + if( pIdx->pPartIdxWhere ){ + *piPartIdxLabel = sqlite3VdbeMakeLabel(v); + pParse->iPartIdxTab = iDataCur; + sqlite3ExprCachePush(pParse); + sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + SQLITE_JUMPIFNULL); + }else{ + *piPartIdxLabel = 0; + } + } + nCol = (prefixOnly && pIdx->uniqNotNull) ? pIdx->nKeyCol : pIdx->nColumn; + regBase = sqlite3GetTempRange(pParse, nCol); + if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; + for(j=0; jaiColumn[j]==pIdx->aiColumn[j] ) continue; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], + regBase+j); + /* If the column affinity is REAL but the number is an integer, then it + ** might be stored in the table as an integer (using a compact + ** representation) then converted to REAL by an OP_RealAffinity opcode. + ** But we are getting ready to store this value back into an index, where + ** it should be converted by to INTEGER again. So omit the OP_RealAffinity + ** opcode if it is present */ + sqlite3VdbeDeletePriorOpcode(v, OP_RealAffinity); + } + if( regOut ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); + } + sqlite3ReleaseTempRange(pParse, regBase, nCol); + return regBase; +} + +/* +** If a prior call to sqlite3GenerateIndexKey() generated a jump-over label +** because it was a partial index, then this routine should be called to +** resolve that label. +*/ +SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ + if( iLabel ){ + sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); + sqlite3ExprCachePop(pParse); + } +} + +/************** End of delete.c **********************************************/ +/************** Begin file func.c ********************************************/ +/* +** 2002 February 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C-language implementions for many of the SQL +** functions of SQLite. (Some function, and in particular the date and +** time functions, are implemented separately.) +*/ +/* #include */ +/* #include */ + +/* +** Return the collating function associated with a function. +*/ +static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ + return context->pColl; +} + +/* +** Indicate that the accumulator load should be skipped on this +** iteration of the aggregate loop. +*/ +static void sqlite3SkipAccumulatorLoad(sqlite3_context *context){ + context->skipFlag = 1; +} + +/* +** Implementation of the non-aggregate min() and max() functions +*/ +static void minmaxFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + int mask; /* 0 for min() or 0xffffffff for max() */ + int iBest; + CollSeq *pColl; + + assert( argc>1 ); + mask = sqlite3_user_data(context)==0 ? 0 : -1; + pColl = sqlite3GetFuncCollSeq(context); + assert( pColl ); + assert( mask==-1 || mask==0 ); + iBest = 0; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + for(i=1; i=0 ){ + testcase( mask==0 ); + iBest = i; + } + } + sqlite3_result_value(context, argv[iBest]); +} + +/* +** Return the type of the argument. +*/ +static void typeofFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + const char *z = 0; + UNUSED_PARAMETER(NotUsed); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_INTEGER: z = "integer"; break; + case SQLITE_TEXT: z = "text"; break; + case SQLITE_FLOAT: z = "real"; break; + case SQLITE_BLOB: z = "blob"; break; + default: z = "null"; break; + } + sqlite3_result_text(context, z, -1, SQLITE_STATIC); +} + + +/* +** Implementation of the length() function +*/ +static void lengthFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int len; + + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_BLOB: + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + break; + } + case SQLITE_TEXT: { + const unsigned char *z = sqlite3_value_text(argv[0]); + if( z==0 ) return; + len = 0; + while( *z ){ + len++; + SQLITE_SKIP_UTF8(z); + } + sqlite3_result_int(context, len); + break; + } + default: { + sqlite3_result_null(context); + break; + } + } +} + +/* +** Implementation of the abs() function. +** +** IMP: R-23979-26855 The abs(X) function returns the absolute value of +** the numeric argument X. +*/ +static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_INTEGER: { + i64 iVal = sqlite3_value_int64(argv[0]); + if( iVal<0 ){ + if( iVal==SMALLEST_INT64 ){ + /* IMP: R-31676-45509 If X is the integer -9223372036854775808 + ** then abs(X) throws an integer overflow error since there is no + ** equivalent positive 64-bit two complement value. */ + sqlite3_result_error(context, "integer overflow", -1); + return; + } + iVal = -iVal; + } + sqlite3_result_int64(context, iVal); + break; + } + case SQLITE_NULL: { + /* IMP: R-37434-19929 Abs(X) returns NULL if X is NULL. */ + sqlite3_result_null(context); + break; + } + default: { + /* Because sqlite3_value_double() returns 0.0 if the argument is not + ** something that can be converted into a number, we have: + ** IMP: R-57326-31541 Abs(X) return 0.0 if X is a string or blob that + ** cannot be converted to a numeric value. + */ + double rVal = sqlite3_value_double(argv[0]); + if( rVal<0 ) rVal = -rVal; + sqlite3_result_double(context, rVal); + break; + } + } +} + +/* +** Implementation of the instr() function. +** +** instr(haystack,needle) finds the first occurrence of needle +** in haystack and returns the number of previous characters plus 1, +** or 0 if needle does not occur within haystack. +** +** If both haystack and needle are BLOBs, then the result is one more than +** the number of bytes in haystack prior to the first occurrence of needle, +** or 0 if needle never occurs in haystack. +*/ +static void instrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zHaystack; + const unsigned char *zNeedle; + int nHaystack; + int nNeedle; + int typeHaystack, typeNeedle; + int N = 1; + int isText; + + UNUSED_PARAMETER(argc); + typeHaystack = sqlite3_value_type(argv[0]); + typeNeedle = sqlite3_value_type(argv[1]); + if( typeHaystack==SQLITE_NULL || typeNeedle==SQLITE_NULL ) return; + nHaystack = sqlite3_value_bytes(argv[0]); + nNeedle = sqlite3_value_bytes(argv[1]); + if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ + zHaystack = sqlite3_value_blob(argv[0]); + zNeedle = sqlite3_value_blob(argv[1]); + isText = 0; + }else{ + zHaystack = sqlite3_value_text(argv[0]); + zNeedle = sqlite3_value_text(argv[1]); + isText = 1; + } + while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ + N++; + do{ + nHaystack--; + zHaystack++; + }while( isText && (zHaystack[0]&0xc0)==0x80 ); + } + if( nNeedle>nHaystack ) N = 0; + sqlite3_result_int(context, N); +} + +/* +** Implementation of the printf() function. +*/ +static void printfFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + PrintfArguments x; + StrAccum str; + const char *zFormat; + int n; + + if( argc>=1 && (zFormat = (const char*)sqlite3_value_text(argv[0]))!=0 ){ + x.nArg = argc-1; + x.nUsed = 0; + x.apArg = argv+1; + sqlite3StrAccumInit(&str, 0, 0, SQLITE_MAX_LENGTH); + str.db = sqlite3_context_db_handle(context); + sqlite3XPrintf(&str, SQLITE_PRINTF_SQLFUNC, zFormat, &x); + n = str.nChar; + sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, + SQLITE_DYNAMIC); + } +} + +/* +** Implementation of the substr() function. +** +** substr(x,p1,p2) returns p2 characters of x[] beginning with p1. +** p1 is 1-indexed. So substr(x,1,1) returns the first character +** of x. If x is text, then we actually count UTF-8 characters. +** If x is a blob, then we count bytes. +** +** If p1 is negative, then we begin abs(p1) from the end of x[]. +** +** If p2 is negative, return the p2 characters preceding p1. +*/ +static void substrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z; + const unsigned char *z2; + int len; + int p0type; + i64 p1, p2; + int negP2 = 0; + + assert( argc==3 || argc==2 ); + if( sqlite3_value_type(argv[1])==SQLITE_NULL + || (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL) + ){ + return; + } + p0type = sqlite3_value_type(argv[0]); + p1 = sqlite3_value_int(argv[1]); + if( p0type==SQLITE_BLOB ){ + len = sqlite3_value_bytes(argv[0]); + z = sqlite3_value_blob(argv[0]); + if( z==0 ) return; + assert( len==sqlite3_value_bytes(argv[0]) ); + }else{ + z = sqlite3_value_text(argv[0]); + if( z==0 ) return; + len = 0; + if( p1<0 ){ + for(z2=z; *z2; len++){ + SQLITE_SKIP_UTF8(z2); + } + } + } + if( argc==3 ){ + p2 = sqlite3_value_int(argv[2]); + if( p2<0 ){ + p2 = -p2; + negP2 = 1; + } + }else{ + p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; + } + if( p1<0 ){ + p1 += len; + if( p1<0 ){ + p2 += p1; + if( p2<0 ) p2 = 0; + p1 = 0; + } + }else if( p1>0 ){ + p1--; + }else if( p2>0 ){ + p2--; + } + if( negP2 ){ + p1 -= p2; + if( p1<0 ){ + p2 += p1; + p1 = 0; + } + } + assert( p1>=0 && p2>=0 ); + if( p0type!=SQLITE_BLOB ){ + while( *z && p1 ){ + SQLITE_SKIP_UTF8(z); + p1--; + } + for(z2=z; *z2 && p2; p2--){ + SQLITE_SKIP_UTF8(z2); + } + sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT); + }else{ + if( p1+p2>len ){ + p2 = len-p1; + if( p2<0 ) p2 = 0; + } + sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT); + } +} + +/* +** Implementation of the round() function +*/ +#ifndef SQLITE_OMIT_FLOATING_POINT +static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + int n = 0; + double r; + char *zBuf; + assert( argc==1 || argc==2 ); + if( argc==2 ){ + if( SQLITE_NULL==sqlite3_value_type(argv[1]) ) return; + n = sqlite3_value_int(argv[1]); + if( n>30 ) n = 30; + if( n<0 ) n = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + r = sqlite3_value_double(argv[0]); + /* If Y==0 and X will fit in a 64-bit int, + ** handle the rounding directly, + ** otherwise use printf. + */ + if( n==0 && r>=0 && r0 ); + testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + z = 0; + }else{ + z = sqlite3Malloc((int)nByte); + if( !z ){ + sqlite3_result_error_nomem(context); + } + } + return z; +} + +/* +** Implementation of the upper() and lower() SQL functions. +*/ +static void upperFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + char *z1; + const char *z2; + int i, n; + UNUSED_PARAMETER(argc); + z2 = (char*)sqlite3_value_text(argv[0]); + n = sqlite3_value_bytes(argv[0]); + /* Verify that the call to _bytes() does not invalidate the _text() pointer */ + assert( z2==(char*)sqlite3_value_text(argv[0]) ); + if( z2 ){ + z1 = contextMalloc(context, ((i64)n)+1); + if( z1 ){ + for(i=0; imatchOne; + u8 matchAll = pInfo->matchAll; + u8 matchSet = pInfo->matchSet; + u8 noCase = pInfo->noCase; + int prevEscape = 0; /* True if the previous character was 'escape' */ + + while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ + if( c==matchAll && !prevEscape ){ + while( (c=sqlite3Utf8Read(&zPattern)) == matchAll + || c == matchOne ){ + if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ + return 0; + } + } + if( c==0 ){ + return 1; + }else if( c==esc ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ){ + return 0; + } + }else if( c==matchSet ){ + assert( esc==0 ); /* This is GLOB, not LIKE */ + assert( matchSet<0x80 ); /* '[' is a single-byte character */ + while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ + SQLITE_SKIP_UTF8(zString); + } + return *zString!=0; + } + while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ + if( noCase ){ + GlobUpperToLower(c2); + GlobUpperToLower(c); + while( c2 != 0 && c2 != c ){ + c2 = sqlite3Utf8Read(&zString); + GlobUpperToLower(c2); + } + }else{ + while( c2 != 0 && c2 != c ){ + c2 = sqlite3Utf8Read(&zString); + } + } + if( c2==0 ) return 0; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; + } + return 0; + }else if( c==matchOne && !prevEscape ){ + if( sqlite3Utf8Read(&zString)==0 ){ + return 0; + } + }else if( c==matchSet ){ + u32 prior_c = 0; + assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ + seen = 0; + invert = 0; + c = sqlite3Utf8Read(&zString); + if( c==0 ) return 0; + c2 = sqlite3Utf8Read(&zPattern); + if( c2=='^' ){ + invert = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ + c2 = sqlite3Utf8Read(&zPattern); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==0 || (seen ^ invert)==0 ){ + return 0; + } + }else if( esc==c && !prevEscape ){ + prevEscape = 1; + }else{ + c2 = sqlite3Utf8Read(&zString); + if( noCase ){ + GlobUpperToLower(c); + GlobUpperToLower(c2); + } + if( c!=c2 ){ + return 0; + } + prevEscape = 0; + } + } + return *zString==0; +} + +/* +** The sqlite3_strglob() interface. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; +} + +/* +** Count the number of times that the LIKE operator (or GLOB which is +** just a variation of LIKE) gets called. This is used for testing +** only. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_like_count = 0; +#endif + + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B,A). +** +** This same function (with a different compareInfo structure) computes +** the GLOB operator. +*/ +static void likeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA, *zB; + u32 escape = 0; + int nPat; + sqlite3 *db = sqlite3_context_db_handle(context); + + zB = sqlite3_value_text(argv[0]); + zA = sqlite3_value_text(argv[1]); + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + nPat = sqlite3_value_bytes(argv[0]); + testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ); + testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]+1 ); + if( nPat > db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + assert( zB==sqlite3_value_text(argv[0]) ); /* Encoding did not change */ + + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + const unsigned char *zEsc = sqlite3_value_text(argv[2]); + if( zEsc==0 ) return; + if( sqlite3Utf8CharLen((char*)zEsc, -1)!=1 ){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + escape = sqlite3Utf8Read(&zEsc); + } + if( zA && zB ){ + struct compareInfo *pInfo = sqlite3_user_data(context); +#ifdef SQLITE_TEST + sqlite3_like_count++; +#endif + + sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)); + } +} + +/* +** Implementation of the NULLIF(x,y) function. The result is the first +** argument if the arguments are different. The result is NULL if the +** arguments are equal to each other. +*/ +static void nullifFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + CollSeq *pColl = sqlite3GetFuncCollSeq(context); + UNUSED_PARAMETER(NotUsed); + if( sqlite3MemCompare(argv[0], argv[1], pColl)!=0 ){ + sqlite3_result_value(context, argv[0]); + } +} + +/* +** Implementation of the sqlite_version() function. The result is the version +** of the SQLite library that is running. +*/ +static void versionFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + /* IMP: R-48699-48617 This function is an SQL wrapper around the + ** sqlite3_libversion() C-interface. */ + sqlite3_result_text(context, sqlite3_libversion(), -1, SQLITE_STATIC); +} + +/* +** Implementation of the sqlite_source_id() function. The result is a string +** that identifies the particular version of the source code used to build +** SQLite. +*/ +static void sourceidFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + /* IMP: R-24470-31136 This function is an SQL wrapper around the + ** sqlite3_sourceid() C interface. */ + sqlite3_result_text(context, sqlite3_sourceid(), -1, SQLITE_STATIC); +} + +/* +** Implementation of the sqlite_log() function. This is a wrapper around +** sqlite3_log(). The return value is NULL. The function exists purely for +** its side-effects. +*/ +static void errlogFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(context); + sqlite3_log(sqlite3_value_int(argv[0]), "%s", sqlite3_value_text(argv[1])); +} + +/* +** Implementation of the sqlite_compileoption_used() function. +** The result is an integer that identifies if the compiler option +** was used to build SQLite. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +static void compileoptionusedFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zOptName; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + /* IMP: R-39564-36305 The sqlite_compileoption_used() SQL + ** function is a wrapper around the sqlite3_compileoption_used() C/C++ + ** function. + */ + if( (zOptName = (const char*)sqlite3_value_text(argv[0]))!=0 ){ + sqlite3_result_int(context, sqlite3_compileoption_used(zOptName)); + } +} +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/* +** Implementation of the sqlite_compileoption_get() function. +** The result is a string that identifies the compiler options +** used to build SQLite. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +static void compileoptiongetFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + /* IMP: R-04922-24076 The sqlite_compileoption_get() SQL function + ** is a wrapper around the sqlite3_compileoption_get() C/C++ function. + */ + n = sqlite3_value_int(argv[0]); + sqlite3_result_text(context, sqlite3_compileoption_get(n), -1, SQLITE_STATIC); +} +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/* Array for converting from half-bytes (nybbles) into ASCII hex +** digits. */ +static const char hexdigits[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' +}; + +/* +** Implementation of the QUOTE() function. This function takes a single +** argument. If the argument is numeric, the return value is the same as +** the argument. If the argument is NULL, the return value is the string +** "NULL". Otherwise, the argument is enclosed in single quotes with +** single-quote escapes. +*/ +static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_FLOAT: { + double r1, r2; + char zBuf[50]; + r1 = sqlite3_value_double(argv[0]); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1); + sqlite3AtoF(zBuf, &r2, 20, SQLITE_UTF8); + if( r1!=r2 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.20e", r1); + } + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + break; + } + case SQLITE_INTEGER: { + sqlite3_result_value(context, argv[0]); + break; + } + case SQLITE_BLOB: { + char *zText = 0; + char const *zBlob = sqlite3_value_blob(argv[0]); + int nBlob = sqlite3_value_bytes(argv[0]); + assert( zBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ + zText = (char *)contextMalloc(context, (2*(i64)nBlob)+4); + if( zText ){ + int i; + for(i=0; i>4)&0x0F]; + zText[(i*2)+3] = hexdigits[(zBlob[i])&0x0F]; + } + zText[(nBlob*2)+2] = '\''; + zText[(nBlob*2)+3] = '\0'; + zText[0] = 'X'; + zText[1] = '\''; + sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); + sqlite3_free(zText); + } + break; + } + case SQLITE_TEXT: { + int i,j; + u64 n; + const unsigned char *zArg = sqlite3_value_text(argv[0]); + char *z; + + if( zArg==0 ) return; + for(i=0, n=0; zArg[i]; i++){ if( zArg[i]=='\'' ) n++; } + z = contextMalloc(context, ((i64)i)+((i64)n)+3); + if( z ){ + z[0] = '\''; + for(i=0, j=1; zArg[i]; i++){ + z[j++] = zArg[i]; + if( zArg[i]=='\'' ){ + z[j++] = '\''; + } + } + z[j++] = '\''; + z[j] = 0; + sqlite3_result_text(context, z, j, sqlite3_free); + } + break; + } + default: { + assert( sqlite3_value_type(argv[0])==SQLITE_NULL ); + sqlite3_result_text(context, "NULL", 4, SQLITE_STATIC); + break; + } + } +} + +/* +** The unicode() function. Return the integer unicode code-point value +** for the first character of the input string. +*/ +static void unicodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z = sqlite3_value_text(argv[0]); + (void)argc; + if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z)); +} + +/* +** The char() function takes zero or more arguments, each of which is +** an integer. It constructs a string where each character of the string +** is the unicode character for the corresponding integer argument. +*/ +static void charFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *z, *zOut; + int i; + zOut = z = sqlite3_malloc( argc*4+1 ); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + for(i=0; i0x10ffff ) x = 0xfffd; + c = (unsigned)(x & 0x1fffff); + if( c<0x00080 ){ + *zOut++ = (u8)(c&0xFF); + }else if( c<0x00800 ){ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else if( c<0x10000 ){ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else{ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + } \ + } + sqlite3_result_text(context, (char*)z, (int)(zOut-z), sqlite3_free); +} + +/* +** The hex() function. Interpret the argument as a blob. Return +** a hexadecimal rendering as text. +*/ +static void hexFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i, n; + const unsigned char *pBlob; + char *zHex, *z; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + pBlob = sqlite3_value_blob(argv[0]); + n = sqlite3_value_bytes(argv[0]); + assert( pBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ + z = zHex = contextMalloc(context, ((i64)n)*2 + 1); + if( zHex ){ + for(i=0; i>4)&0xf]; + *(z++) = hexdigits[c&0xf]; + } + *z = 0; + sqlite3_result_text(context, zHex, n*2, sqlite3_free); + } +} + +/* +** The zeroblob(N) function returns a zero-filled blob of size N bytes. +*/ +static void zeroblobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + i64 n; + sqlite3 *db = sqlite3_context_db_handle(context); + assert( argc==1 ); + UNUSED_PARAMETER(argc); + n = sqlite3_value_int64(argv[0]); + testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + if( n>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + }else{ + sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */ + } +} + +/* +** The replace() function. Three arguments are all strings: call +** them A, B, and C. The result is also a string which is derived +** from A by replacing every occurrence of B with C. The match +** must be exact. Collating sequences are not used. +*/ +static void replaceFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zStr; /* The input string A */ + const unsigned char *zPattern; /* The pattern string B */ + const unsigned char *zRep; /* The replacement string C */ + unsigned char *zOut; /* The output */ + int nStr; /* Size of zStr */ + int nPattern; /* Size of zPattern */ + int nRep; /* Size of zRep */ + i64 nOut; /* Maximum size of zOut */ + int loopLimit; /* Last zStr[] that might match zPattern[] */ + int i, j; /* Loop counters */ + + assert( argc==3 ); + UNUSED_PARAMETER(argc); + zStr = sqlite3_value_text(argv[0]); + if( zStr==0 ) return; + nStr = sqlite3_value_bytes(argv[0]); + assert( zStr==sqlite3_value_text(argv[0]) ); /* No encoding change */ + zPattern = sqlite3_value_text(argv[1]); + if( zPattern==0 ){ + assert( sqlite3_value_type(argv[1])==SQLITE_NULL + || sqlite3_context_db_handle(context)->mallocFailed ); + return; + } + if( zPattern[0]==0 ){ + assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); + sqlite3_result_value(context, argv[0]); + return; + } + nPattern = sqlite3_value_bytes(argv[1]); + assert( zPattern==sqlite3_value_text(argv[1]) ); /* No encoding change */ + zRep = sqlite3_value_text(argv[2]); + if( zRep==0 ) return; + nRep = sqlite3_value_bytes(argv[2]); + assert( zRep==sqlite3_value_text(argv[2]) ); + nOut = nStr + 1; + assert( nOutaLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); + if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + sqlite3_free(zOut); + return; + } + zOld = zOut; + zOut = sqlite3_realloc(zOut, (int)nOut); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(zOld); + return; + } + memcpy(&zOut[j], zRep, nRep); + j += nRep; + i += nPattern-1; + } + } + assert( j+nStr-i+1==nOut ); + memcpy(&zOut[j], &zStr[i], nStr-i); + j += nStr - i; + assert( j<=nOut ); + zOut[j] = 0; + sqlite3_result_text(context, (char*)zOut, j, sqlite3_free); +} + +/* +** Implementation of the TRIM(), LTRIM(), and RTRIM() functions. +** The userdata is 0x1 for left trim, 0x2 for right trim, 0x3 for both. +*/ +static void trimFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn; /* Input string */ + const unsigned char *zCharSet; /* Set of characters to trim */ + int nIn; /* Number of bytes in input */ + int flags; /* 1: trimleft 2: trimright 3: trim */ + int i; /* Loop counter */ + unsigned char *aLen = 0; /* Length of each character in zCharSet */ + unsigned char **azChar = 0; /* Individual characters in zCharSet */ + int nChar; /* Number of characters in zCharSet */ + + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + return; + } + zIn = sqlite3_value_text(argv[0]); + if( zIn==0 ) return; + nIn = sqlite3_value_bytes(argv[0]); + assert( zIn==sqlite3_value_text(argv[0]) ); + if( argc==1 ){ + static const unsigned char lenOne[] = { 1 }; + static unsigned char * const azOne[] = { (u8*)" " }; + nChar = 1; + aLen = (u8*)lenOne; + azChar = (unsigned char **)azOne; + zCharSet = 0; + }else if( (zCharSet = sqlite3_value_text(argv[1]))==0 ){ + return; + }else{ + const unsigned char *z; + for(z=zCharSet, nChar=0; *z; nChar++){ + SQLITE_SKIP_UTF8(z); + } + if( nChar>0 ){ + azChar = contextMalloc(context, ((i64)nChar)*(sizeof(char*)+1)); + if( azChar==0 ){ + return; + } + aLen = (unsigned char*)&azChar[nChar]; + for(z=zCharSet, nChar=0; *z; nChar++){ + azChar[nChar] = (unsigned char *)z; + SQLITE_SKIP_UTF8(z); + aLen[nChar] = (u8)(z - azChar[nChar]); + } + } + } + if( nChar>0 ){ + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(context)); + if( flags & 1 ){ + while( nIn>0 ){ + int len = 0; + for(i=0; i=nChar ) break; + zIn += len; + nIn -= len; + } + } + if( flags & 2 ){ + while( nIn>0 ){ + int len = 0; + for(i=0; i=nChar ) break; + nIn -= len; + } + } + if( zCharSet ){ + sqlite3_free(azChar); + } + } + sqlite3_result_text(context, (char*)zIn, nIn, SQLITE_TRANSIENT); +} + + +/* IMP: R-25361-16150 This function is omitted from SQLite by default. It +** is only available if the SQLITE_SOUNDEX compile-time option is used +** when SQLite is built. +*/ +#ifdef SQLITE_SOUNDEX +/* +** Compute the soundex encoding of a word. +** +** IMP: R-59782-00072 The soundex(X) function returns a string that is the +** soundex encoding of the string X. +*/ +static void soundexFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + char zResult[8]; + const u8 *zIn; + int i, j; + static const unsigned char iCode[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + }; + assert( argc==1 ); + zIn = (u8*)sqlite3_value_text(argv[0]); + if( zIn==0 ) zIn = (u8*)""; + for(i=0; zIn[i] && !sqlite3Isalpha(zIn[i]); i++){} + if( zIn[i] ){ + u8 prevcode = iCode[zIn[i]&0x7f]; + zResult[0] = sqlite3Toupper(zIn[i]); + for(j=1; j<4 && zIn[i]; i++){ + int code = iCode[zIn[i]&0x7f]; + if( code>0 ){ + if( code!=prevcode ){ + prevcode = code; + zResult[j++] = code + '0'; + } + }else{ + prevcode = 0; + } + } + while( j<4 ){ + zResult[j++] = '0'; + } + zResult[j] = 0; + sqlite3_result_text(context, zResult, 4, SQLITE_TRANSIENT); + }else{ + /* IMP: R-64894-50321 The string "?000" is returned if the argument + ** is NULL or contains no ASCII alphabetic characters. */ + sqlite3_result_text(context, "?000", 4, SQLITE_STATIC); + } +} +#endif /* SQLITE_SOUNDEX */ + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** A function that loads a shared-library extension then returns NULL. +*/ +static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *zFile = (const char *)sqlite3_value_text(argv[0]); + const char *zProc; + sqlite3 *db = sqlite3_context_db_handle(context); + char *zErrMsg = 0; + + if( argc==2 ){ + zProc = (const char *)sqlite3_value_text(argv[1]); + }else{ + zProc = 0; + } + if( zFile && sqlite3_load_extension(db, zFile, zProc, &zErrMsg) ){ + sqlite3_result_error(context, zErrMsg, -1); + sqlite3_free(zErrMsg); + } +} +#endif + + +/* +** An instance of the following structure holds the context of a +** sum() or avg() aggregate computation. +*/ +typedef struct SumCtx SumCtx; +struct SumCtx { + double rSum; /* Floating point sum */ + i64 iSum; /* Integer sum */ + i64 cnt; /* Number of elements summed */ + u8 overflow; /* True if integer overflow seen */ + u8 approx; /* True if non-integer value was input to the sum */ +}; + +/* +** Routines used to compute the sum, average, and total. +** +** The SUM() function follows the (broken) SQL standard which means +** that it returns NULL if it sums over no inputs. TOTAL returns +** 0.0 in that case. In addition, TOTAL always returns a float where +** SUM might return an integer if it never encounters a floating point +** value. TOTAL never fails, but SUM might through an exception if +** it overflows an integer. +*/ +static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + SumCtx *p; + int type; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + type = sqlite3_value_numeric_type(argv[0]); + if( p && type!=SQLITE_NULL ){ + p->cnt++; + if( type==SQLITE_INTEGER ){ + i64 v = sqlite3_value_int64(argv[0]); + p->rSum += v; + if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ + p->overflow = 1; + } + }else{ + p->rSum += sqlite3_value_double(argv[0]); + p->approx = 1; + } + } +} +static void sumFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>0 ){ + if( p->overflow ){ + sqlite3_result_error(context,"integer overflow",-1); + }else if( p->approx ){ + sqlite3_result_double(context, p->rSum); + }else{ + sqlite3_result_int64(context, p->iSum); + } + } +} +static void avgFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>0 ){ + sqlite3_result_double(context, p->rSum/(double)p->cnt); + } +} +static void totalFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + sqlite3_result_double(context, p ? p->rSum : (double)0); +} + +/* +** The following structure keeps track of state information for the +** count() aggregate function. +*/ +typedef struct CountCtx CountCtx; +struct CountCtx { + i64 n; +}; + +/* +** Routines to implement the count() aggregate function. +*/ +static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + CountCtx *p; + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && p ){ + p->n++; + } + +#ifndef SQLITE_OMIT_DEPRECATED + /* The sqlite3_aggregate_count() function is deprecated. But just to make + ** sure it still operates correctly, verify that its count agrees with our + ** internal count when using count(*) and when the total count can be + ** expressed as a 32-bit integer. */ + assert( argc==1 || p==0 || p->n>0x7fffffff + || p->n==sqlite3_aggregate_count(context) ); +#endif +} +static void countFinalize(sqlite3_context *context){ + CountCtx *p; + p = sqlite3_aggregate_context(context, 0); + sqlite3_result_int64(context, p ? p->n : 0); +} + +/* +** Routines to implement min() and max() aggregate functions. +*/ +static void minmaxStep( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + Mem *pArg = (Mem *)argv[0]; + Mem *pBest; + UNUSED_PARAMETER(NotUsed); + + pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); + if( !pBest ) return; + + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + if( pBest->flags ) sqlite3SkipAccumulatorLoad(context); + }else if( pBest->flags ){ + int max; + int cmp; + CollSeq *pColl = sqlite3GetFuncCollSeq(context); + /* This step function is used for both the min() and max() aggregates, + ** the only difference between the two being that the sense of the + ** comparison is inverted. For the max() aggregate, the + ** sqlite3_user_data() function returns (void *)-1. For min() it + ** returns (void *)db, where db is the sqlite3* database pointer. + ** Therefore the next statement sets variable 'max' to 1 for the max() + ** aggregate, or 0 for min(). + */ + max = sqlite3_user_data(context)!=0; + cmp = sqlite3MemCompare(pBest, pArg, pColl); + if( (max && cmp<0) || (!max && cmp>0) ){ + sqlite3VdbeMemCopy(pBest, pArg); + }else{ + sqlite3SkipAccumulatorLoad(context); + } + }else{ + sqlite3VdbeMemCopy(pBest, pArg); + } +} +static void minMaxFinalize(sqlite3_context *context){ + sqlite3_value *pRes; + pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); + if( pRes ){ + if( pRes->flags ){ + sqlite3_result_value(context, pRes); + } + sqlite3VdbeMemRelease(pRes); + } +} + +/* +** group_concat(EXPR, ?SEPARATOR?) +*/ +static void groupConcatStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zVal; + StrAccum *pAccum; + const char *zSep; + int nVal, nSep; + assert( argc==1 || argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); + + if( pAccum ){ + sqlite3 *db = sqlite3_context_db_handle(context); + int firstTerm = pAccum->useMalloc==0; + pAccum->useMalloc = 2; + pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; + if( !firstTerm ){ + if( argc==2 ){ + zSep = (char*)sqlite3_value_text(argv[1]); + nSep = sqlite3_value_bytes(argv[1]); + }else{ + zSep = ","; + nSep = 1; + } + if( nSep ) sqlite3StrAccumAppend(pAccum, zSep, nSep); + } + zVal = (char*)sqlite3_value_text(argv[0]); + nVal = sqlite3_value_bytes(argv[0]); + if( zVal ) sqlite3StrAccumAppend(pAccum, zVal, nVal); + } +} +static void groupConcatFinalize(sqlite3_context *context){ + StrAccum *pAccum; + pAccum = sqlite3_aggregate_context(context, 0); + if( pAccum ){ + if( pAccum->accError==STRACCUM_TOOBIG ){ + sqlite3_result_error_toobig(context); + }else if( pAccum->accError==STRACCUM_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, sqlite3StrAccumFinish(pAccum), -1, + sqlite3_free); + } + } +} + +/* +** This routine does per-connection function registration. Most +** of the built-in functions above are part of the global function set. +** This routine only deals with those that are not global. +*/ +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ + int rc = sqlite3_overload_function(db, "MATCH", 2); +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifndef OMIT_EXPORT + extern void sqlcipher_exportFunc(sqlite3_context *, int, sqlite3_value **); +#endif +#endif +/* END SQLCIPHER */ + assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); + if( rc==SQLITE_NOMEM ){ + db->mallocFailed = 1; + } +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifndef OMIT_EXPORT + sqlite3CreateFunc(db, "sqlcipher_export", 1, SQLITE_TEXT, 0, sqlcipher_exportFunc, 0, 0, 0); +#endif +#endif +/* END SQLCIPHER */ +} + +/* +** Set the LIKEOPT flag on the 2-argument function with the given name. +*/ +static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ + FuncDef *pDef; + pDef = sqlite3FindFunction(db, zName, sqlite3Strlen30(zName), + 2, SQLITE_UTF8, 0); + if( ALWAYS(pDef) ){ + pDef->funcFlags |= flagVal; + } +} + +/* +** Register the built-in LIKE and GLOB functions. The caseSensitive +** parameter determines whether or not the LIKE operator is case +** sensitive. GLOB is always case sensitive. +*/ +SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ + struct compareInfo *pInfo; + if( caseSensitive ){ + pInfo = (struct compareInfo*)&likeInfoAlt; + }else{ + pInfo = (struct compareInfo*)&likeInfoNorm; + } + sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); + sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); + sqlite3CreateFunc(db, "glob", 2, SQLITE_UTF8, + (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0); + setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); + setLikeOptFlag(db, "like", + caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); +} + +/* +** pExpr points to an expression which implements a function. If +** it is appropriate to apply the LIKE optimization to that function +** then set aWc[0] through aWc[2] to the wildcard characters and +** return TRUE. If the function is not a LIKE-style function then +** return FALSE. +*/ +SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ + FuncDef *pDef; + if( pExpr->op!=TK_FUNCTION + || !pExpr->x.pList + || pExpr->x.pList->nExpr!=2 + ){ + return 0; + } + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + pDef = sqlite3FindFunction(db, pExpr->u.zToken, + sqlite3Strlen30(pExpr->u.zToken), + 2, SQLITE_UTF8, 0); + if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ + return 0; + } + + /* The memcpy() statement assumes that the wildcard characters are + ** the first three statements in the compareInfo structure. The + ** asserts() that follow verify that assumption + */ + memcpy(aWc, pDef->pUserData, 3); + assert( (char*)&likeInfoAlt == (char*)&likeInfoAlt.matchAll ); + assert( &((char*)&likeInfoAlt)[1] == (char*)&likeInfoAlt.matchOne ); + assert( &((char*)&likeInfoAlt)[2] == (char*)&likeInfoAlt.matchSet ); + *pIsNocase = (pDef->funcFlags & SQLITE_FUNC_CASE)==0; + return 1; +} + +/* +** All all of the FuncDef structures in the aBuiltinFunc[] array above +** to the global function hash table. This occurs at start-time (as +** a consequence of calling sqlite3_initialize()). +** +** After this routine runs +*/ +SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ + /* + ** The following array holds FuncDef structures for all of the functions + ** defined in this file. + ** + ** The array cannot be constant since changes are made to the + ** FuncDef.pHash elements at start-time. The elements of this array + ** are read-only after initialization is complete. + */ + static SQLITE_WSD FuncDef aBuiltinFunc[] = { + FUNCTION(ltrim, 1, 1, 0, trimFunc ), + FUNCTION(ltrim, 2, 1, 0, trimFunc ), + FUNCTION(rtrim, 1, 2, 0, trimFunc ), + FUNCTION(rtrim, 2, 2, 0, trimFunc ), + FUNCTION(trim, 1, 3, 0, trimFunc ), + FUNCTION(trim, 2, 3, 0, trimFunc ), + FUNCTION(min, -1, 0, 1, minmaxFunc ), + FUNCTION(min, 0, 0, 1, 0 ), + AGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize ), + FUNCTION(max, -1, 1, 1, minmaxFunc ), + FUNCTION(max, 0, 1, 1, 0 ), + AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), + FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), + FUNCTION(instr, 2, 0, 0, instrFunc ), + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(printf, -1, 0, 0, printfFunc ), + FUNCTION(unicode, 1, 0, 0, unicodeFunc ), + FUNCTION(char, -1, 0, 0, charFunc ), + FUNCTION(abs, 1, 0, 0, absFunc ), +#ifndef SQLITE_OMIT_FLOATING_POINT + FUNCTION(round, 1, 0, 0, roundFunc ), + FUNCTION(round, 2, 0, 0, roundFunc ), +#endif + FUNCTION(upper, 1, 0, 0, upperFunc ), + FUNCTION(lower, 1, 0, 0, lowerFunc ), + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, 0, 0, 0, 0 ), + FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), + FUNCTION(hex, 1, 0, 0, hexFunc ), + FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), + FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + VFUNCTION(random, 0, 0, 0, randomFunc ), + VFUNCTION(randomblob, 1, 0, 0, randomBlob ), + FUNCTION(nullif, 2, 0, 1, nullifFunc ), + FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + FUNCTION(quote, 1, 0, 0, quoteFunc ), + VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), + VFUNCTION(changes, 0, 0, 0, changes ), + VFUNCTION(total_changes, 0, 0, 0, total_changes ), + FUNCTION(replace, 3, 0, 0, replaceFunc ), + FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), + #ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), + #endif + #ifndef SQLITE_OMIT_LOAD_EXTENSION + FUNCTION(load_extension, 1, 0, 0, loadExt ), + FUNCTION(load_extension, 2, 0, 0, loadExt ), + #endif + AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), + AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), + AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), + /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ + {0,SQLITE_UTF8|SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, + AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), + AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), + AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), + + LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + #ifdef SQLITE_CASE_SENSITIVE_LIKE + LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + #else + LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), + LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), + #endif + }; + + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aBuiltinFunc); + + for(i=0; idb->mallocFailed flag is set. +*/ +SQLITE_PRIVATE int sqlite3FkLocateIndex( + Parse *pParse, /* Parse context to store any error in */ + Table *pParent, /* Parent table of FK constraint pFKey */ + FKey *pFKey, /* Foreign key to find index for */ + Index **ppIdx, /* OUT: Unique index on parent table */ + int **paiCol /* OUT: Map of index columns in pFKey */ +){ + Index *pIdx = 0; /* Value to return via *ppIdx */ + int *aiCol = 0; /* Value to return via *paiCol */ + int nCol = pFKey->nCol; /* Number of columns in parent key */ + char *zKey = pFKey->aCol[0].zCol; /* Name of left-most parent key column */ + + /* The caller is responsible for zeroing output parameters. */ + assert( ppIdx && *ppIdx==0 ); + assert( !paiCol || *paiCol==0 ); + assert( pParse ); + + /* If this is a non-composite (single column) foreign key, check if it + ** maps to the INTEGER PRIMARY KEY of table pParent. If so, leave *ppIdx + ** and *paiCol set to zero and return early. + ** + ** Otherwise, for a composite foreign key (more than one column), allocate + ** space for the aiCol array (returned via output parameter *paiCol). + ** Non-composite foreign keys do not require the aiCol array. + */ + if( nCol==1 ){ + /* The FK maps to the IPK if any of the following are true: + ** + ** 1) There is an INTEGER PRIMARY KEY column and the FK is implicitly + ** mapped to the primary key of table pParent, or + ** 2) The FK is explicitly mapped to a column declared as INTEGER + ** PRIMARY KEY. + */ + if( pParent->iPKey>=0 ){ + if( !zKey ) return 0; + if( !sqlite3StrICmp(pParent->aCol[pParent->iPKey].zName, zKey) ) return 0; + } + }else if( paiCol ){ + assert( nCol>1 ); + aiCol = (int *)sqlite3DbMallocRaw(pParse->db, nCol*sizeof(int)); + if( !aiCol ) return 1; + *paiCol = aiCol; + } + + for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) ){ + /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number + ** of columns. If each indexed column corresponds to a foreign key + ** column of pFKey, then this index is a winner. */ + + if( zKey==0 ){ + /* If zKey is NULL, then this foreign key is implicitly mapped to + ** the PRIMARY KEY of table pParent. The PRIMARY KEY index may be + ** identified by the test. */ + if( IsPrimaryKeyIndex(pIdx) ){ + if( aiCol ){ + int i; + for(i=0; iaCol[i].iFrom; + } + break; + } + }else{ + /* If zKey is non-NULL, then this foreign key was declared to + ** map to an explicit list of columns in table pParent. Check if this + ** index matches those columns. Also, check that the index uses + ** the default collation sequences for each column. */ + int i, j; + for(i=0; iaiColumn[i]; /* Index of column in parent tbl */ + char *zDfltColl; /* Def. collation for column */ + char *zIdxCol; /* Name of indexed column */ + + /* If the index uses a collation sequence that is different from + ** the default collation sequence for the column, this index is + ** unusable. Bail out early in this case. */ + zDfltColl = pParent->aCol[iCol].zColl; + if( !zDfltColl ){ + zDfltColl = "BINARY"; + } + if( sqlite3StrICmp(pIdx->azColl[i], zDfltColl) ) break; + + zIdxCol = pParent->aCol[iCol].zName; + for(j=0; jaCol[j].zCol, zIdxCol)==0 ){ + if( aiCol ) aiCol[i] = pFKey->aCol[j].iFrom; + break; + } + } + if( j==nCol ) break; + } + if( i==nCol ) break; /* pIdx is usable */ + } + } + } + + if( !pIdx ){ + if( !pParse->disableTriggers ){ + sqlite3ErrorMsg(pParse, + "foreign key mismatch - \"%w\" referencing \"%w\"", + pFKey->pFrom->zName, pFKey->zTo); + } + sqlite3DbFree(pParse->db, aiCol); + return 1; + } + + *ppIdx = pIdx; + return 0; +} + +/* +** This function is called when a row is inserted into or deleted from the +** child table of foreign key constraint pFKey. If an SQL UPDATE is executed +** on the child table of pFKey, this function is invoked twice for each row +** affected - once to "delete" the old row, and then again to "insert" the +** new row. +** +** Each time it is called, this function generates VDBE code to locate the +** row in the parent table that corresponds to the row being inserted into +** or deleted from the child table. If the parent row can be found, no +** special action is taken. Otherwise, if the parent row can *not* be +** found in the parent table: +** +** Operation | FK type | Action taken +** -------------------------------------------------------------------------- +** INSERT immediate Increment the "immediate constraint counter". +** +** DELETE immediate Decrement the "immediate constraint counter". +** +** INSERT deferred Increment the "deferred constraint counter". +** +** DELETE deferred Decrement the "deferred constraint counter". +** +** These operations are identified in the comment at the top of this file +** (fkey.c) as "I.1" and "D.1". +*/ +static void fkLookupParent( + Parse *pParse, /* Parse context */ + int iDb, /* Index of database housing pTab */ + Table *pTab, /* Parent table of FK pFKey */ + Index *pIdx, /* Unique index on parent key columns in pTab */ + FKey *pFKey, /* Foreign key constraint */ + int *aiCol, /* Map from parent key columns to child table columns */ + int regData, /* Address of array containing child table row */ + int nIncr, /* Increment constraint counter by this */ + int isIgnore /* If true, pretend pTab contains all NULL values */ +){ + int i; /* Iterator variable */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Vdbe to add code to */ + int iCur = pParse->nTab - 1; /* Cursor number to use */ + int iOk = sqlite3VdbeMakeLabel(v); /* jump here if parent key found */ + + /* If nIncr is less than zero, then check at runtime if there are any + ** outstanding constraints to resolve. If there are not, there is no need + ** to check if deleting this row resolves any outstanding violations. + ** + ** Check if any of the key columns in the child table row are NULL. If + ** any are, then the constraint is considered satisfied. No need to + ** search for a matching row in the parent table. */ + if( nIncr<0 ){ + sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, iOk); + VdbeCoverage(v); + } + for(i=0; inCol; i++){ + int iReg = aiCol[i] + regData + 1; + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); VdbeCoverage(v); + } + + if( isIgnore==0 ){ + if( pIdx==0 ){ + /* If pIdx is NULL, then the parent key is the INTEGER PRIMARY KEY + ** column of the parent table (table pTab). */ + int iMustBeInt; /* Address of MustBeInt instruction */ + int regTemp = sqlite3GetTempReg(pParse); + + /* Invoke MustBeInt to coerce the child key value to an integer (i.e. + ** apply the affinity of the parent key). If this fails, then there + ** is no matching parent key. Before using MustBeInt, make a copy of + ** the value. Otherwise, the value inserted into the child key column + ** will have INTEGER affinity applied to it, which may not be correct. */ + sqlite3VdbeAddOp2(v, OP_SCopy, aiCol[0]+1+regData, regTemp); + iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); + VdbeCoverage(v); + + /* If the parent table is the same as the child table, and we are about + ** to increment the constraint-counter (i.e. this is an INSERT operation), + ** then check if the row being inserted matches itself. If so, do not + ** increment the constraint-counter. */ + if( pTab==pFKey->pFrom && nIncr==1 ){ + sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + } + + sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + sqlite3VdbeJumpHere(v, iMustBeInt); + sqlite3ReleaseTempReg(pParse, regTemp); + }else{ + int nCol = pFKey->nCol; + int regTemp = sqlite3GetTempRange(pParse, nCol); + int regRec = sqlite3GetTempReg(pParse); + + sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + for(i=0; ipFrom && nIncr==1 ){ + int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; + for(i=0; iaiColumn[i]+1+regData; + assert( aiCol[i]!=pTab->iPKey ); + if( pIdx->aiColumn[i]==pTab->iPKey ){ + /* The parent key is a composite key that includes the IPK column */ + iParent = regData; + } + sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + } + + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, + sqlite3IndexAffinityStr(v,pIdx), nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); + + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempRange(pParse, regTemp, nCol); + } + } + + if( !pFKey->isDeferred && !(pParse->db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel + && !pParse->isMultiWrite + ){ + /* Special case: If this is an INSERT statement that will insert exactly + ** one row into the table, raise a constraint immediately instead of + ** incrementing a counter. This is necessary as the VM code is being + ** generated for will not open a statement transaction. */ + assert( nIncr==1 ); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, 0, P4_STATIC, P5_ConstraintFK); + }else{ + if( nIncr>0 && pFKey->isDeferred==0 ){ + sqlite3ParseToplevel(pParse)->mayAbort = 1; + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + } + + sqlite3VdbeResolveLabel(v, iOk); + sqlite3VdbeAddOp1(v, OP_Close, iCur); +} + + +/* +** Return an Expr object that refers to a memory register corresponding +** to column iCol of table pTab. +** +** regBase is the first of an array of register that contains the data +** for pTab. regBase itself holds the rowid. regBase+1 holds the first +** column. regBase+2 holds the second column, and so forth. +*/ +static Expr *exprTableRegister( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* The table whose content is at r[regBase]... */ + int regBase, /* Contents of table pTab */ + i16 iCol /* Which column of pTab is desired */ +){ + Expr *pExpr; + Column *pCol; + const char *zColl; + sqlite3 *db = pParse->db; + + pExpr = sqlite3Expr(db, TK_REGISTER, 0); + if( pExpr ){ + if( iCol>=0 && iCol!=pTab->iPKey ){ + pCol = &pTab->aCol[iCol]; + pExpr->iTable = regBase + iCol + 1; + pExpr->affinity = pCol->affinity; + zColl = pCol->zColl; + if( zColl==0 ) zColl = db->pDfltColl->zName; + pExpr = sqlite3ExprAddCollateString(pParse, pExpr, zColl); + }else{ + pExpr->iTable = regBase; + pExpr->affinity = SQLITE_AFF_INTEGER; + } + } + return pExpr; +} + +/* +** Return an Expr object that refers to column iCol of table pTab which +** has cursor iCur. +*/ +static Expr *exprTableColumn( + sqlite3 *db, /* The database connection */ + Table *pTab, /* The table whose column is desired */ + int iCursor, /* The open cursor on the table */ + i16 iCol /* The column that is wanted */ +){ + Expr *pExpr = sqlite3Expr(db, TK_COLUMN, 0); + if( pExpr ){ + pExpr->pTab = pTab; + pExpr->iTable = iCursor; + pExpr->iColumn = iCol; + } + return pExpr; +} + +/* +** This function is called to generate code executed when a row is deleted +** from the parent table of foreign key constraint pFKey and, if pFKey is +** deferred, when a row is inserted into the same table. When generating +** code for an SQL UPDATE operation, this function may be called twice - +** once to "delete" the old row and once to "insert" the new row. +** +** The code generated by this function scans through the rows in the child +** table that correspond to the parent table row being deleted or inserted. +** For each child row found, one of the following actions is taken: +** +** Operation | FK type | Action taken +** -------------------------------------------------------------------------- +** DELETE immediate Increment the "immediate constraint counter". +** Or, if the ON (UPDATE|DELETE) action is RESTRICT, +** throw a "FOREIGN KEY constraint failed" exception. +** +** INSERT immediate Decrement the "immediate constraint counter". +** +** DELETE deferred Increment the "deferred constraint counter". +** Or, if the ON (UPDATE|DELETE) action is RESTRICT, +** throw a "FOREIGN KEY constraint failed" exception. +** +** INSERT deferred Decrement the "deferred constraint counter". +** +** These operations are identified in the comment at the top of this file +** (fkey.c) as "I.2" and "D.2". +*/ +static void fkScanChildren( + Parse *pParse, /* Parse context */ + SrcList *pSrc, /* The child table to be scanned */ + Table *pTab, /* The parent table */ + Index *pIdx, /* Index on parent covering the foreign key */ + FKey *pFKey, /* The foreign key linking pSrc to pTab */ + int *aiCol, /* Map from pIdx cols to child table cols */ + int regData, /* Parent row data starts here */ + int nIncr /* Amount to increment deferred counter by */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + int i; /* Iterator variable */ + Expr *pWhere = 0; /* WHERE clause to scan with */ + NameContext sNameContext; /* Context used to resolve WHERE clause */ + WhereInfo *pWInfo; /* Context used by sqlite3WhereXXX() */ + int iFkIfZero = 0; /* Address of OP_FkIfZero */ + Vdbe *v = sqlite3GetVdbe(pParse); + + assert( pIdx==0 || pIdx->pTable==pTab ); + assert( pIdx==0 || pIdx->nKeyCol==pFKey->nCol ); + assert( pIdx!=0 || pFKey->nCol==1 ); + assert( pIdx!=0 || HasRowid(pTab) ); + + if( nIncr<0 ){ + iFkIfZero = sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, 0); + VdbeCoverage(v); + } + + /* Create an Expr object representing an SQL expression like: + ** + ** = AND = ... + ** + ** The collation sequence used for the comparison should be that of + ** the parent key columns. The affinity of the parent key column should + ** be applied to each child key value before the comparison takes place. + */ + for(i=0; inCol; i++){ + Expr *pLeft; /* Value from parent table row */ + Expr *pRight; /* Column ref to child table */ + Expr *pEq; /* Expression (pLeft = pRight) */ + i16 iCol; /* Index of column in child table */ + const char *zCol; /* Name of column in child table */ + + iCol = pIdx ? pIdx->aiColumn[i] : -1; + pLeft = exprTableRegister(pParse, pTab, regData, iCol); + iCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; + assert( iCol>=0 ); + zCol = pFKey->pFrom->aCol[iCol].zName; + pRight = sqlite3Expr(db, TK_ID, zCol); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); + pWhere = sqlite3ExprAnd(db, pWhere, pEq); + } + + /* If the child table is the same as the parent table, then add terms + ** to the WHERE clause that prevent this entry from being scanned. + ** The added WHERE clause terms are like this: + ** + ** $current_rowid!=rowid + ** NOT( $current_a==a AND $current_b==b AND ... ) + ** + ** The first form is used for rowid tables. The second form is used + ** for WITHOUT ROWID tables. In the second form, the primary key is + ** (a,b,...) + */ + if( pTab==pFKey->pFrom && nIncr>0 ){ + Expr *pNe; /* Expression (pLeft != pRight) */ + Expr *pLeft; /* Value from parent table row */ + Expr *pRight; /* Column ref to child table */ + if( HasRowid(pTab) ){ + pLeft = exprTableRegister(pParse, pTab, regData, -1); + pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, -1); + pNe = sqlite3PExpr(pParse, TK_NE, pLeft, pRight, 0); + }else{ + Expr *pEq, *pAll = 0; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pIdx!=0 ); + for(i=0; inKeyCol; i++){ + i16 iCol = pIdx->aiColumn[i]; + pLeft = exprTableRegister(pParse, pTab, regData, iCol); + pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); + pAll = sqlite3ExprAnd(db, pAll, pEq); + } + pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0, 0); + } + pWhere = sqlite3ExprAnd(db, pWhere, pNe); + } + + /* Resolve the references in the WHERE clause. */ + memset(&sNameContext, 0, sizeof(NameContext)); + sNameContext.pSrcList = pSrc; + sNameContext.pParse = pParse; + sqlite3ResolveExprNames(&sNameContext, pWhere); + + /* Create VDBE to loop through the entries in pSrc that match the WHERE + ** clause. If the constraint is not deferred, throw an exception for + ** each row found. Otherwise, for deferred constraints, increment the + ** deferred constraint counter by nIncr for each row selected. */ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); + if( nIncr>0 && pFKey->isDeferred==0 ){ + sqlite3ParseToplevel(pParse)->mayAbort = 1; + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + if( pWInfo ){ + sqlite3WhereEnd(pWInfo); + } + + /* Clean up the WHERE clause constructed above. */ + sqlite3ExprDelete(db, pWhere); + if( iFkIfZero ){ + sqlite3VdbeJumpHere(v, iFkIfZero); + } +} + +/* +** This function returns a linked list of FKey objects (connected by +** FKey.pNextTo) holding all children of table pTab. For example, +** given the following schema: +** +** CREATE TABLE t1(a PRIMARY KEY); +** CREATE TABLE t2(b REFERENCES t1(a); +** +** Calling this function with table "t1" as an argument returns a pointer +** to the FKey structure representing the foreign key constraint on table +** "t2". Calling this function with "t2" as the argument would return a +** NULL pointer (as there are no FK constraints for which t2 is the parent +** table). +*/ +SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *pTab){ + int nName = sqlite3Strlen30(pTab->zName); + return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName, nName); +} + +/* +** The second argument is a Trigger structure allocated by the +** fkActionTrigger() routine. This function deletes the Trigger structure +** and all of its sub-components. +** +** The Trigger structure or any of its sub-components may be allocated from +** the lookaside buffer belonging to database handle dbMem. +*/ +static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ + if( p ){ + TriggerStep *pStep = p->step_list; + sqlite3ExprDelete(dbMem, pStep->pWhere); + sqlite3ExprListDelete(dbMem, pStep->pExprList); + sqlite3SelectDelete(dbMem, pStep->pSelect); + sqlite3ExprDelete(dbMem, p->pWhen); + sqlite3DbFree(dbMem, p); + } +} + +/* +** This function is called to generate code that runs when table pTab is +** being dropped from the database. The SrcList passed as the second argument +** to this function contains a single entry guaranteed to resolve to +** table pTab. +** +** Normally, no code is required. However, if either +** +** (a) The table is the parent table of a FK constraint, or +** (b) The table is the child table of a deferred FK constraint and it is +** determined at runtime that there are outstanding deferred FK +** constraint violations in the database, +** +** then the equivalent of "DELETE FROM " is executed before dropping +** the table from the database. Triggers are disabled while running this +** DELETE, but foreign key actions are not. +*/ +SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ + sqlite3 *db = pParse->db; + if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) && !pTab->pSelect ){ + int iSkip = 0; + Vdbe *v = sqlite3GetVdbe(pParse); + + assert( v ); /* VDBE has already been allocated */ + if( sqlite3FkReferences(pTab)==0 ){ + /* Search for a deferred foreign key constraint for which this table + ** is the child table. If one cannot be found, return without + ** generating any VDBE code. If one can be found, then jump over + ** the entire DELETE if there are no outstanding deferred constraints + ** when this statement is run. */ + FKey *p; + for(p=pTab->pFKey; p; p=p->pNextFrom){ + if( p->isDeferred || (db->flags & SQLITE_DeferFKs) ) break; + } + if( !p ) return; + iSkip = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); VdbeCoverage(v); + } + + pParse->disableTriggers = 1; + sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0); + pParse->disableTriggers = 0; + + /* If the DELETE has generated immediate foreign key constraint + ** violations, halt the VDBE and return an error at this point, before + ** any modifications to the schema are made. This is because statement + ** transactions are not able to rollback schema changes. + ** + ** If the SQLITE_DeferFKs flag is set, then this is not required, as + ** the statement transaction will not be rolled back even if FK + ** constraints are violated. + */ + if( (db->flags & SQLITE_DeferFKs)==0 ){ + sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, 0, P4_STATIC, P5_ConstraintFK); + } + + if( iSkip ){ + sqlite3VdbeResolveLabel(v, iSkip); + } + } +} + + +/* +** The second argument points to an FKey object representing a foreign key +** for which pTab is the child table. An UPDATE statement against pTab +** is currently being processed. For each column of the table that is +** actually updated, the corresponding element in the aChange[] array +** is zero or greater (if a column is unmodified the corresponding element +** is set to -1). If the rowid column is modified by the UPDATE statement +** the bChngRowid argument is non-zero. +** +** This function returns true if any of the columns that are part of the +** child key for FK constraint *p are modified. +*/ +static int fkChildIsModified( + Table *pTab, /* Table being updated */ + FKey *p, /* Foreign key for which pTab is the child */ + int *aChange, /* Array indicating modified columns */ + int bChngRowid /* True if rowid is modified by this update */ +){ + int i; + for(i=0; inCol; i++){ + int iChildKey = p->aCol[i].iFrom; + if( aChange[iChildKey]>=0 ) return 1; + if( iChildKey==pTab->iPKey && bChngRowid ) return 1; + } + return 0; +} + +/* +** The second argument points to an FKey object representing a foreign key +** for which pTab is the parent table. An UPDATE statement against pTab +** is currently being processed. For each column of the table that is +** actually updated, the corresponding element in the aChange[] array +** is zero or greater (if a column is unmodified the corresponding element +** is set to -1). If the rowid column is modified by the UPDATE statement +** the bChngRowid argument is non-zero. +** +** This function returns true if any of the columns that are part of the +** parent key for FK constraint *p are modified. +*/ +static int fkParentIsModified( + Table *pTab, + FKey *p, + int *aChange, + int bChngRowid +){ + int i; + for(i=0; inCol; i++){ + char *zKey = p->aCol[i].zCol; + int iKey; + for(iKey=0; iKeynCol; iKey++){ + if( aChange[iKey]>=0 || (iKey==pTab->iPKey && bChngRowid) ){ + Column *pCol = &pTab->aCol[iKey]; + if( zKey ){ + if( 0==sqlite3StrICmp(pCol->zName, zKey) ) return 1; + }else if( pCol->colFlags & COLFLAG_PRIMKEY ){ + return 1; + } + } + } + } + return 0; +} + +/* +** This function is called when inserting, deleting or updating a row of +** table pTab to generate VDBE code to perform foreign key constraint +** processing for the operation. +** +** For a DELETE operation, parameter regOld is passed the index of the +** first register in an array of (pTab->nCol+1) registers containing the +** rowid of the row being deleted, followed by each of the column values +** of the row being deleted, from left to right. Parameter regNew is passed +** zero in this case. +** +** For an INSERT operation, regOld is passed zero and regNew is passed the +** first register of an array of (pTab->nCol+1) registers containing the new +** row data. +** +** For an UPDATE operation, this function is called twice. Once before +** the original record is deleted from the table using the calling convention +** described for DELETE. Then again after the original record is deleted +** but before the new record is inserted using the INSERT convention. +*/ +SQLITE_PRIVATE void sqlite3FkCheck( + Parse *pParse, /* Parse context */ + Table *pTab, /* Row is being deleted from this table */ + int regOld, /* Previous row data is stored here */ + int regNew, /* New row data is stored here */ + int *aChange, /* Array indicating UPDATEd columns (or 0) */ + int bChngRowid /* True if rowid is UPDATEd */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + FKey *pFKey; /* Used to iterate through FKs */ + int iDb; /* Index of database containing pTab */ + const char *zDb; /* Name of database containing pTab */ + int isIgnoreErrors = pParse->disableTriggers; + + /* Exactly one of regOld and regNew should be non-zero. */ + assert( (regOld==0)!=(regNew==0) ); + + /* If foreign-keys are disabled, this function is a no-op. */ + if( (db->flags&SQLITE_ForeignKeys)==0 ) return; + + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zDb = db->aDb[iDb].zName; + + /* Loop through all the foreign key constraints for which pTab is the + ** child table (the table that the foreign key definition is part of). */ + for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + Table *pTo; /* Parent table of foreign key pFKey */ + Index *pIdx = 0; /* Index on key columns in pTo */ + int *aiFree = 0; + int *aiCol; + int iCol; + int i; + int isIgnore = 0; + + if( aChange + && sqlite3_stricmp(pTab->zName, pFKey->zTo)!=0 + && fkChildIsModified(pTab, pFKey, aChange, bChngRowid)==0 + ){ + continue; + } + + /* Find the parent table of this foreign key. Also find a unique index + ** on the parent key columns in the parent table. If either of these + ** schema items cannot be located, set an error in pParse and return + ** early. */ + if( pParse->disableTriggers ){ + pTo = sqlite3FindTable(db, pFKey->zTo, zDb); + }else{ + pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); + } + if( !pTo || sqlite3FkLocateIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + assert( isIgnoreErrors==0 || (regOld!=0 && regNew==0) ); + if( !isIgnoreErrors || db->mallocFailed ) return; + if( pTo==0 ){ + /* If isIgnoreErrors is true, then a table is being dropped. In this + ** case SQLite runs a "DELETE FROM xxx" on the table being dropped + ** before actually dropping it in order to check FK constraints. + ** If the parent table of an FK constraint on the current table is + ** missing, behave as if it is empty. i.e. decrement the relevant + ** FK counter for each row of the current table with non-NULL keys. + */ + Vdbe *v = sqlite3GetVdbe(pParse); + int iJump = sqlite3VdbeCurrentAddr(v) + pFKey->nCol + 1; + for(i=0; inCol; i++){ + int iReg = pFKey->aCol[i].iFrom + regOld + 1; + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iJump); VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, -1); + } + continue; + } + assert( pFKey->nCol==1 || (aiFree && pIdx) ); + + if( aiFree ){ + aiCol = aiFree; + }else{ + iCol = pFKey->aCol[0].iFrom; + aiCol = &iCol; + } + for(i=0; inCol; i++){ + if( aiCol[i]==pTab->iPKey ){ + aiCol[i] = -1; + } +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Request permission to read the parent key columns. If the + ** authorization callback returns SQLITE_IGNORE, behave as if any + ** values read from the parent table are NULL. */ + if( db->xAuth ){ + int rcauth; + char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zName; + rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); + isIgnore = (rcauth==SQLITE_IGNORE); + } +#endif + } + + /* Take a shared-cache advisory read-lock on the parent table. Allocate + ** a cursor to use to search the unique index on the parent key columns + ** in the parent table. */ + sqlite3TableLock(pParse, iDb, pTo->tnum, 0, pTo->zName); + pParse->nTab++; + + if( regOld!=0 ){ + /* A row is being removed from the child table. Search for the parent. + ** If the parent does not exist, removing the child row resolves an + ** outstanding foreign key constraint violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1,isIgnore); + } + if( regNew!=0 ){ + /* A row is being added to the child table. If a parent row cannot + ** be found, adding the child row has violated the FK constraint. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1,isIgnore); + } + + sqlite3DbFree(db, aiFree); + } + + /* Loop through all the foreign key constraints that refer to this table. + ** (the "child" constraints) */ + for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ + Index *pIdx = 0; /* Foreign key index for pFKey */ + SrcList *pSrc; + int *aiCol = 0; + + if( aChange && fkParentIsModified(pTab, pFKey, aChange, bChngRowid)==0 ){ + continue; + } + + if( !pFKey->isDeferred && !(db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel && !pParse->isMultiWrite + ){ + assert( regOld==0 && regNew!=0 ); + /* Inserting a single row into a parent table cannot cause an immediate + ** foreign key violation. So do nothing in this case. */ + continue; + } + + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ + if( !isIgnoreErrors || db->mallocFailed ) return; + continue; + } + assert( aiCol || pFKey->nCol==1 ); + + /* Create a SrcList structure containing the child table. We need the + ** child table as a SrcList for sqlite3WhereBegin() */ + pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + if( pSrc ){ + struct SrcList_item *pItem = pSrc->a; + pItem->pTab = pFKey->pFrom; + pItem->zName = pFKey->pFrom->zName; + pItem->pTab->nRef++; + pItem->iCursor = pParse->nTab++; + + if( regNew!=0 ){ + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); + } + if( regOld!=0 ){ + /* If there is a RESTRICT action configured for the current operation + ** on the parent table of this FK, then throw an exception + ** immediately if the FK constraint is violated, even if this is a + ** deferred trigger. That's what RESTRICT means. To defer checking + ** the constraint, the FK should specify NO ACTION (represented + ** using OE_None). NO ACTION is the default. */ + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); + } + pItem->zName = 0; + sqlite3SrcListDelete(db, pSrc); + } + sqlite3DbFree(db, aiCol); + } +} + +#define COLUMN_MASK(x) (((x)>31) ? 0xffffffff : ((u32)1<<(x))) + +/* +** This function is called before generating code to update or delete a +** row contained in table pTab. +*/ +SQLITE_PRIVATE u32 sqlite3FkOldmask( + Parse *pParse, /* Parse context */ + Table *pTab /* Table being modified */ +){ + u32 mask = 0; + if( pParse->db->flags&SQLITE_ForeignKeys ){ + FKey *p; + int i; + for(p=pTab->pFKey; p; p=p->pNextFrom){ + for(i=0; inCol; i++) mask |= COLUMN_MASK(p->aCol[i].iFrom); + } + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + Index *pIdx = 0; + sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); + if( pIdx ){ + for(i=0; inKeyCol; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); + } + } + } + return mask; +} + + +/* +** This function is called before generating code to update or delete a +** row contained in table pTab. If the operation is a DELETE, then +** parameter aChange is passed a NULL value. For an UPDATE, aChange points +** to an array of size N, where N is the number of columns in table pTab. +** If the i'th column is not modified by the UPDATE, then the corresponding +** entry in the aChange[] array is set to -1. If the column is modified, +** the value is 0 or greater. Parameter chngRowid is set to true if the +** UPDATE statement modifies the rowid fields of the table. +** +** If any foreign key processing will be required, this function returns +** true. If there is no foreign key related processing, this function +** returns false. +*/ +SQLITE_PRIVATE int sqlite3FkRequired( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being modified */ + int *aChange, /* Non-NULL for UPDATE operations */ + int chngRowid /* True for UPDATE that affects rowid */ +){ + if( pParse->db->flags&SQLITE_ForeignKeys ){ + if( !aChange ){ + /* A DELETE operation. Foreign key processing is required if the + ** table in question is either the child or parent table for any + ** foreign key constraint. */ + return (sqlite3FkReferences(pTab) || pTab->pFKey); + }else{ + /* This is an UPDATE. Foreign key processing is only required if the + ** operation modifies one or more child or parent key columns. */ + FKey *p; + + /* Check if any child key columns are being modified. */ + for(p=pTab->pFKey; p; p=p->pNextFrom){ + if( fkChildIsModified(pTab, p, aChange, chngRowid) ) return 1; + } + + /* Check if any parent key columns are being modified. */ + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + if( fkParentIsModified(pTab, p, aChange, chngRowid) ) return 1; + } + } + } + return 0; +} + +/* +** This function is called when an UPDATE or DELETE operation is being +** compiled on table pTab, which is the parent table of foreign-key pFKey. +** If the current operation is an UPDATE, then the pChanges parameter is +** passed a pointer to the list of columns being modified. If it is a +** DELETE, pChanges is passed a NULL pointer. +** +** It returns a pointer to a Trigger structure containing a trigger +** equivalent to the ON UPDATE or ON DELETE action specified by pFKey. +** If the action is "NO ACTION" or "RESTRICT", then a NULL pointer is +** returned (these actions require no special handling by the triggers +** sub-system, code for them is created by fkScanChildren()). +** +** For example, if pFKey is the foreign key and pTab is table "p" in +** the following schema: +** +** CREATE TABLE p(pk PRIMARY KEY); +** CREATE TABLE c(ck REFERENCES p ON DELETE CASCADE); +** +** then the returned trigger structure is equivalent to: +** +** CREATE TRIGGER ... DELETE ON p BEGIN +** DELETE FROM c WHERE ck = old.pk; +** END; +** +** The returned pointer is cached as part of the foreign key object. It +** is eventually freed along with the rest of the foreign key object by +** sqlite3FkDelete(). +*/ +static Trigger *fkActionTrigger( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated or deleted from */ + FKey *pFKey, /* Foreign key to get action for */ + ExprList *pChanges /* Change-list for UPDATE, NULL for DELETE */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + int action; /* One of OE_None, OE_Cascade etc. */ + Trigger *pTrigger; /* Trigger definition to return */ + int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ + + action = pFKey->aAction[iAction]; + pTrigger = pFKey->apTrigger[iAction]; + + if( action!=OE_None && !pTrigger ){ + u8 enableLookaside; /* Copy of db->lookaside.bEnabled */ + char const *zFrom; /* Name of child table */ + int nFrom; /* Length in bytes of zFrom */ + Index *pIdx = 0; /* Parent key index for this FK */ + int *aiCol = 0; /* child table cols -> parent key cols */ + TriggerStep *pStep = 0; /* First (only) step of trigger program */ + Expr *pWhere = 0; /* WHERE clause of trigger step */ + ExprList *pList = 0; /* Changes list if ON UPDATE CASCADE */ + Select *pSelect = 0; /* If RESTRICT, "SELECT RAISE(...)" */ + int i; /* Iterator variable */ + Expr *pWhen = 0; /* WHEN clause for the trigger */ + + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; + assert( aiCol || pFKey->nCol==1 ); + + for(i=0; inCol; i++){ + Token tOld = { "old", 3 }; /* Literal "old" token */ + Token tNew = { "new", 3 }; /* Literal "new" token */ + Token tFromCol; /* Name of column in child table */ + Token tToCol; /* Name of column in parent table */ + int iFromCol; /* Idx of column in child table */ + Expr *pEq; /* tFromCol = OLD.tToCol */ + + iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; + assert( iFromCol>=0 ); + tToCol.z = pIdx ? pTab->aCol[pIdx->aiColumn[i]].zName : "oid"; + tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; + + tToCol.n = sqlite3Strlen30(tToCol.z); + tFromCol.n = sqlite3Strlen30(tFromCol.z); + + /* Create the expression "OLD.zToCol = zFromCol". It is important + ** that the "OLD.zToCol" term is on the LHS of the = operator, so + ** that the affinity and collation sequence associated with the + ** parent table are used for the comparison. */ + pEq = sqlite3PExpr(pParse, TK_EQ, + sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) + , 0), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tFromCol) + , 0); + pWhere = sqlite3ExprAnd(db, pWhere, pEq); + + /* For ON UPDATE, construct the next term of the WHEN clause. + ** The final WHEN clause will be like this: + ** + ** WHEN NOT(old.col1 IS new.col1 AND ... AND old.colN IS new.colN) + */ + if( pChanges ){ + pEq = sqlite3PExpr(pParse, TK_IS, + sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), + 0), + sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), + 0), + 0); + pWhen = sqlite3ExprAnd(db, pWhen, pEq); + } + + if( action!=OE_Restrict && (action!=OE_Cascade || pChanges) ){ + Expr *pNew; + if( action==OE_Cascade ){ + pNew = sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) + , 0); + }else if( action==OE_SetDflt ){ + Expr *pDflt = pFKey->pFrom->aCol[iFromCol].pDflt; + if( pDflt ){ + pNew = sqlite3ExprDup(db, pDflt, 0); + }else{ + pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + } + }else{ + pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + } + pList = sqlite3ExprListAppend(pParse, pList, pNew); + sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); + } + } + sqlite3DbFree(db, aiCol); + + zFrom = pFKey->pFrom->zName; + nFrom = sqlite3Strlen30(zFrom); + + if( action==OE_Restrict ){ + Token tFrom; + Expr *pRaise; + + tFrom.z = zFrom; + tFrom.n = nFrom; + pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); + if( pRaise ){ + pRaise->affinity = OE_Abort; + } + pSelect = sqlite3SelectNew(pParse, + sqlite3ExprListAppend(pParse, 0, pRaise), + sqlite3SrcListAppend(db, 0, &tFrom, 0), + pWhere, + 0, 0, 0, 0, 0, 0 + ); + pWhere = 0; + } + + /* Disable lookaside memory allocation */ + enableLookaside = db->lookaside.bEnabled; + db->lookaside.bEnabled = 0; + + pTrigger = (Trigger *)sqlite3DbMallocZero(db, + sizeof(Trigger) + /* struct Trigger */ + sizeof(TriggerStep) + /* Single step in trigger program */ + nFrom + 1 /* Space for pStep->target.z */ + ); + if( pTrigger ){ + pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; + pStep->target.z = (char *)&pStep[1]; + pStep->target.n = nFrom; + memcpy((char *)pStep->target.z, zFrom, nFrom); + + pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); + pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + if( pWhen ){ + pWhen = sqlite3PExpr(pParse, TK_NOT, pWhen, 0, 0); + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); + } + } + + /* Re-enable the lookaside buffer, if it was disabled earlier. */ + db->lookaside.bEnabled = enableLookaside; + + sqlite3ExprDelete(db, pWhere); + sqlite3ExprDelete(db, pWhen); + sqlite3ExprListDelete(db, pList); + sqlite3SelectDelete(db, pSelect); + if( db->mallocFailed==1 ){ + fkTriggerDelete(db, pTrigger); + return 0; + } + assert( pStep!=0 ); + + switch( action ){ + case OE_Restrict: + pStep->op = TK_SELECT; + break; + case OE_Cascade: + if( !pChanges ){ + pStep->op = TK_DELETE; + break; + } + default: + pStep->op = TK_UPDATE; + } + pStep->pTrig = pTrigger; + pTrigger->pSchema = pTab->pSchema; + pTrigger->pTabSchema = pTab->pSchema; + pFKey->apTrigger[iAction] = pTrigger; + pTrigger->op = (pChanges ? TK_UPDATE : TK_DELETE); + } + + return pTrigger; +} + +/* +** This function is called when deleting or updating a row to implement +** any required CASCADE, SET NULL or SET DEFAULT actions. +*/ +SQLITE_PRIVATE void sqlite3FkActions( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated or deleted from */ + ExprList *pChanges, /* Change-list for UPDATE, NULL for DELETE */ + int regOld, /* Address of array containing old row */ + int *aChange, /* Array indicating UPDATEd columns (or 0) */ + int bChngRowid /* True if rowid is UPDATEd */ +){ + /* If foreign-key support is enabled, iterate through all FKs that + ** refer to table pTab. If there is an action associated with the FK + ** for this operation (either update or delete), invoke the associated + ** trigger sub-program. */ + if( pParse->db->flags&SQLITE_ForeignKeys ){ + FKey *pFKey; /* Iterator variable */ + for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ + if( aChange==0 || fkParentIsModified(pTab, pFKey, aChange, bChngRowid) ){ + Trigger *pAct = fkActionTrigger(pParse, pTab, pFKey, pChanges); + if( pAct ){ + sqlite3CodeRowTriggerDirect(pParse, pAct, pTab, regOld, OE_Abort, 0); + } + } + } + } +} + +#endif /* ifndef SQLITE_OMIT_TRIGGER */ + +/* +** Free all memory associated with foreign key definitions attached to +** table pTab. Remove the deleted foreign keys from the Schema.fkeyHash +** hash table. +*/ +SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ + FKey *pFKey; /* Iterator variable */ + FKey *pNext; /* Copy of pFKey->pNextFrom */ + + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); + for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ + + /* Remove the FK from the fkeyHash hash table. */ + if( !db || db->pnBytesFreed==0 ){ + if( pFKey->pPrevTo ){ + pFKey->pPrevTo->pNextTo = pFKey->pNextTo; + }else{ + void *p = (void *)pFKey->pNextTo; + const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); + } + if( pFKey->pNextTo ){ + pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; + } + } + + /* EV: R-30323-21917 Each foreign key constraint in SQLite is + ** classified as either immediate or deferred. + */ + assert( pFKey->isDeferred==0 || pFKey->isDeferred==1 ); + + /* Delete any triggers created to implement actions for this FK. */ +#ifndef SQLITE_OMIT_TRIGGER + fkTriggerDelete(db, pFKey->apTrigger[0]); + fkTriggerDelete(db, pFKey->apTrigger[1]); +#endif + + pNext = pFKey->pNextFrom; + sqlite3DbFree(db, pFKey); + } +} +#endif /* ifndef SQLITE_OMIT_FOREIGN_KEY */ + +/************** End of fkey.c ************************************************/ +/************** Begin file insert.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle INSERT statements in SQLite. +*/ + +/* +** Generate code that will +** +** (1) acquire a lock for table pTab then +** (2) open pTab as cursor iCur. +** +** If pTab is a WITHOUT ROWID table, then it is the PRIMARY KEY index +** for that table that is actually opened. +*/ +SQLITE_PRIVATE void sqlite3OpenTable( + Parse *pParse, /* Generate code into this VDBE */ + int iCur, /* The cursor number of the table */ + int iDb, /* The database index in sqlite3.aDb[] */ + Table *pTab, /* The table to be opened */ + int opcode /* OP_OpenRead or OP_OpenWrite */ +){ + Vdbe *v; + assert( !IsVirtual(pTab) ); + v = sqlite3GetVdbe(pParse); + assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); + sqlite3TableLock(pParse, iDb, pTab->tnum, + (opcode==OP_OpenWrite)?1:0, pTab->zName); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nCol); + VdbeComment((v, "%s", pTab->zName)); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->tnum=pTab->tnum ); + sqlite3VdbeAddOp3(v, opcode, iCur, pPk->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + VdbeComment((v, "%s", pTab->zName)); + } +} + +/* +** Return a pointer to the column affinity string associated with index +** pIdx. A column affinity string has one character for each column in +** the table, according to the affinity of the column: +** +** Character Column affinity +** ------------------------------ +** 'a' TEXT +** 'b' NONE +** 'c' NUMERIC +** 'd' INTEGER +** 'e' REAL +** +** An extra 'd' is appended to the end of the string to cover the +** rowid that appears as the last column in every index. +** +** Memory for the buffer containing the column index affinity string +** is managed along with the rest of the Index structure. It will be +** released when sqlite3DeleteIndex() is called. +*/ +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ + if( !pIdx->zColAff ){ + /* The first time a column affinity string for a particular index is + ** required, it is allocated and populated here. It is then stored as + ** a member of the Index structure for subsequent use. + ** + ** The column affinity string will eventually be deleted by + ** sqliteDeleteIndex() when the Index structure itself is cleaned + ** up. + */ + int n; + Table *pTab = pIdx->pTable; + sqlite3 *db = sqlite3VdbeDb(v); + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); + if( !pIdx->zColAff ){ + db->mallocFailed = 1; + return 0; + } + for(n=0; nnColumn; n++){ + i16 x = pIdx->aiColumn[n]; + pIdx->zColAff[n] = x<0 ? SQLITE_AFF_INTEGER : pTab->aCol[x].affinity; + } + pIdx->zColAff[n] = 0; + } + + return pIdx->zColAff; +} + +/* +** Compute the affinity string for table pTab, if it has not already been +** computed. As an optimization, omit trailing SQLITE_AFF_NONE affinities. +** +** If the affinity exists (if it is no entirely SQLITE_AFF_NONE values) and +** if iReg>0 then code an OP_Affinity opcode that will set the affinities +** for register iReg and following. Or if affinities exists and iReg==0, +** then just set the P4 operand of the previous opcode (which should be +** an OP_MakeRecord) to the affinity string. +** +** A column affinity string has one character per column: +** +** Character Column affinity +** ------------------------------ +** 'a' TEXT +** 'b' NONE +** 'c' NUMERIC +** 'd' INTEGER +** 'e' REAL +*/ +SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ + int i; + char *zColAff = pTab->zColAff; + if( zColAff==0 ){ + sqlite3 *db = sqlite3VdbeDb(v); + zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); + if( !zColAff ){ + db->mallocFailed = 1; + return; + } + + for(i=0; inCol; i++){ + zColAff[i] = pTab->aCol[i].affinity; + } + do{ + zColAff[i--] = 0; + }while( i>=0 && zColAff[i]==SQLITE_AFF_NONE ); + pTab->zColAff = zColAff; + } + i = sqlite3Strlen30(zColAff); + if( i ){ + if( iReg ){ + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); + }else{ + sqlite3VdbeChangeP4(v, -1, zColAff, i); + } + } +} + +/* +** Return non-zero if the table pTab in database iDb or any of its indices +** have been opened at any point in the VDBE program. This is used to see if +** a statement of the form "INSERT INTO SELECT ..." can +** run without using a temporary table for the results of the SELECT. +*/ +static int readsTable(Parse *p, int iDb, Table *pTab){ + Vdbe *v = sqlite3GetVdbe(p); + int i; + int iEnd = sqlite3VdbeCurrentAddr(v); +#ifndef SQLITE_OMIT_VIRTUALTABLE + VTable *pVTab = IsVirtual(pTab) ? sqlite3GetVTable(p->db, pTab) : 0; +#endif + + for(i=1; iopcode==OP_OpenRead && pOp->p3==iDb ){ + Index *pIndex; + int tnum = pOp->p2; + if( tnum==pTab->tnum ){ + return 1; + } + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( tnum==pIndex->tnum ){ + return 1; + } + } + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pOp->opcode==OP_VOpen && pOp->p4.pVtab==pVTab ){ + assert( pOp->p4.pVtab!=0 ); + assert( pOp->p4type==P4_VTAB ); + return 1; + } +#endif + } + return 0; +} + +#ifndef SQLITE_OMIT_AUTOINCREMENT +/* +** Locate or create an AutoincInfo structure associated with table pTab +** which is in database iDb. Return the register number for the register +** that holds the maximum rowid. +** +** There is at most one AutoincInfo structure per table even if the +** same table is autoincremented multiple times due to inserts within +** triggers. A new AutoincInfo structure is created if this is the +** first use of table pTab. On 2nd and subsequent uses, the original +** AutoincInfo structure is used. +** +** Three memory locations are allocated: +** +** (1) Register to hold the name of the pTab table. +** (2) Register to hold the maximum ROWID of pTab. +** (3) Register to hold the rowid in sqlite_sequence of pTab +** +** The 2nd register is the one that is returned. That is all the +** insert routine needs to know about. +*/ +static int autoIncBegin( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database holding pTab */ + Table *pTab /* The table we are writing to */ +){ + int memId = 0; /* Register holding maximum rowid */ + if( pTab->tabFlags & TF_Autoincrement ){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + AutoincInfo *pInfo; + + pInfo = pToplevel->pAinc; + while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } + if( pInfo==0 ){ + pInfo = sqlite3DbMallocRaw(pParse->db, sizeof(*pInfo)); + if( pInfo==0 ) return 0; + pInfo->pNext = pToplevel->pAinc; + pToplevel->pAinc = pInfo; + pInfo->pTab = pTab; + pInfo->iDb = iDb; + pToplevel->nMem++; /* Register to hold name of table */ + pInfo->regCtr = ++pToplevel->nMem; /* Max rowid register */ + pToplevel->nMem++; /* Rowid in sqlite_sequence */ + } + memId = pInfo->regCtr; + } + return memId; +} + +/* +** This routine generates code that will initialize all of the +** register used by the autoincrement tracker. +*/ +SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ + AutoincInfo *p; /* Information about an AUTOINCREMENT */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* Database only autoinc table */ + int memId; /* Register holding max rowid */ + int addr; /* A VDBE address */ + Vdbe *v = pParse->pVdbe; /* VDBE under construction */ + + /* This routine is never called during trigger-generation. It is + ** only called from the top-level */ + assert( pParse->pTriggerTab==0 ); + assert( pParse==sqlite3ParseToplevel(pParse) ); + + assert( v ); /* We failed long ago if this is not so */ + for(p = pParse->pAinc; p; p = p->pNext){ + pDb = &db->aDb[p->iDb]; + memId = p->regCtr; + assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); + sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); + addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); + sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); + sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); + sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); + sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); + sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); + sqlite3VdbeAddOp0(v, OP_Close); + } +} + +/* +** Update the maximum rowid for an autoincrement calculation. +** +** This routine should be called when the top of the stack holds a +** new rowid that is about to be inserted. If that new rowid is +** larger than the maximum rowid in the memId memory cell, then the +** memory cell is updated. The stack is unchanged. +*/ +static void autoIncStep(Parse *pParse, int memId, int regRowid){ + if( memId>0 ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MemMax, memId, regRowid); + } +} + +/* +** This routine generates the code needed to write autoincrement +** maximum rowid values back into the sqlite_sequence register. +** Every statement that might do an INSERT into an autoincrement +** table (either directly or through triggers) needs to call this +** routine just before the "exit" code. +*/ +SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ + AutoincInfo *p; + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + + assert( v ); + for(p = pParse->pAinc; p; p = p->pNext){ + Db *pDb = &db->aDb[p->iDb]; + int j1; + int iRec; + int memId = p->regCtr; + + iRec = sqlite3GetTempReg(pParse); + assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); + sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); + sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeAddOp0(v, OP_Close); + sqlite3ReleaseTempReg(pParse, iRec); + } +} +#else +/* +** If SQLITE_OMIT_AUTOINCREMENT is defined, then the three routines +** above are all no-ops +*/ +# define autoIncBegin(A,B,C) (0) +# define autoIncStep(A,B,C) +#endif /* SQLITE_OMIT_AUTOINCREMENT */ + + +/* Forward declaration */ +static int xferOptimization( + Parse *pParse, /* Parser context */ + Table *pDest, /* The table we are inserting into */ + Select *pSelect, /* A SELECT statement to use as the data source */ + int onError, /* How to handle constraint errors */ + int iDbDest /* The database of pDest */ +); + +/* +** This routine is called to handle SQL of the following forms: +** +** insert into TABLE (IDLIST) values(EXPRLIST) +** insert into TABLE (IDLIST) select +** +** The IDLIST following the table name is always optional. If omitted, +** then a list of all columns for the table is substituted. The IDLIST +** appears in the pColumn parameter. pColumn is NULL if IDLIST is omitted. +** +** The pList parameter holds EXPRLIST in the first form of the INSERT +** statement above, and pSelect is NULL. For the second form, pList is +** NULL and pSelect is a pointer to the select statement used to generate +** data for the insert. +** +** The code generated follows one of four templates. For a simple +** insert with data coming from a VALUES clause, the code executes +** once straight down through. Pseudo-code follows (we call this +** the "1st template"): +** +** open write cursor to
    and its indices +** put VALUES clause expressions into registers +** write the resulting record into
    +** cleanup +** +** The three remaining templates assume the statement is of the form +** +** INSERT INTO
    SELECT ... +** +** If the SELECT clause is of the restricted form "SELECT * FROM " - +** in other words if the SELECT pulls all columns from a single table +** and there is no WHERE or LIMIT or GROUP BY or ORDER BY clauses, and +** if and are distinct tables but have identical +** schemas, including all the same indices, then a special optimization +** is invoked that copies raw records from over to . +** See the xferOptimization() function for the implementation of this +** template. This is the 2nd template. +** +** open a write cursor to
    +** open read cursor on +** transfer all records in over to
    +** close cursors +** foreach index on
    +** open a write cursor on the
    index +** open a read cursor on the corresponding index +** transfer all records from the read to the write cursors +** close cursors +** end foreach +** +** The 3rd template is for when the second template does not apply +** and the SELECT clause does not read from
    at any time. +** The generated code follows this template: +** +** X <- A +** goto B +** A: setup for the SELECT +** loop over the rows in the SELECT +** load values into registers R..R+n +** yield X +** end loop +** cleanup after the SELECT +** end-coroutine X +** B: open write cursor to
    and its indices +** C: yield X, at EOF goto D +** insert the select result into
    from R..R+n +** goto C +** D: cleanup +** +** The 4th template is used if the insert statement takes its +** values from a SELECT but the data is being inserted into a table +** that is also read as part of the SELECT. In the third form, +** we have to use a intermediate table to store the results of +** the select. The template is like this: +** +** X <- A +** goto B +** A: setup for the SELECT +** loop over the tables in the SELECT +** load value into register R..R+n +** yield X +** end loop +** cleanup after the SELECT +** end co-routine R +** B: open temp table +** L: yield X, at EOF goto M +** insert row from R..R+n into temp table +** goto L +** M: open write cursor to
    and its indices +** rewind temp table +** C: loop over rows of intermediate table +** transfer values form intermediate table into
    +** end loop +** D: cleanup +*/ +SQLITE_PRIVATE void sqlite3Insert( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* Name of table into which we are inserting */ + Select *pSelect, /* A SELECT statement to use as the data source */ + IdList *pColumn, /* Column names corresponding to IDLIST. */ + int onError /* How to handle constraint errors */ +){ + sqlite3 *db; /* The main database structure */ + Table *pTab; /* The table to insert into. aka TABLE */ + char *zTab; /* Name of the table into which we are inserting */ + const char *zDb; /* Name of the database holding this table */ + int i, j, idx; /* Loop counters */ + Vdbe *v; /* Generate code into this virtual machine */ + Index *pIdx; /* For looping over indices of the table */ + int nColumn; /* Number of columns in the data */ + int nHidden = 0; /* Number of hidden columns if TABLE is virtual */ + int iDataCur = 0; /* VDBE cursor that is the main data repository */ + int iIdxCur = 0; /* First index cursor */ + int ipkColumn = -1; /* Column that is the INTEGER PRIMARY KEY */ + int endOfLoop; /* Label for the end of the insertion loop */ + int srcTab = 0; /* Data comes from this temporary cursor if >=0 */ + int addrInsTop = 0; /* Jump to label "D" */ + int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ + SelectDest dest; /* Destination for SELECT on rhs of INSERT */ + int iDb; /* Index of database holding TABLE */ + Db *pDb; /* The database containing table being inserted into */ + u8 useTempTable = 0; /* Store SELECT results in intermediate table */ + u8 appendFlag = 0; /* True if the insert is likely to be an append */ + u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ + u8 bIdListInOrder = 1; /* True if IDLIST is in table order */ + ExprList *pList = 0; /* List of VALUES() to be inserted */ + + /* Register allocations */ + int regFromSelect = 0;/* Base register for data coming from SELECT */ + int regAutoinc = 0; /* Register holding the AUTOINCREMENT counter */ + int regRowCount = 0; /* Memory cell used for the row counter */ + int regIns; /* Block of regs holding rowid+data being inserted */ + int regRowid; /* registers holding insert rowid */ + int regData; /* register holding first column to insert */ + int *aRegIdx = 0; /* One register allocated to each index */ + +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True if attempting to insert into a view */ + Trigger *pTrigger; /* List of triggers on pTab, if required */ + int tmask; /* Mask of trigger times */ +#endif + + db = pParse->db; + memset(&dest, 0, sizeof(dest)); + if( pParse->nErr || db->mallocFailed ){ + goto insert_cleanup; + } + + /* If the Select object is really just a simple VALUES() list with a + ** single row values (the common case) then keep that one row of values + ** and go ahead and discard the Select object + */ + if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){ + pList = pSelect->pEList; + pSelect->pEList = 0; + sqlite3SelectDelete(db, pSelect); + pSelect = 0; + } + + /* Locate the table into which we will be inserting new information. + */ + assert( pTabList->nSrc==1 ); + zTab = pTabList->a[0].zName; + if( NEVER(zTab==0) ) goto insert_cleanup; + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ){ + goto insert_cleanup; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDbnDb ); + pDb = &db->aDb[iDb]; + zDb = pDb->zName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, zDb) ){ + goto insert_cleanup; + } + withoutRowid = !HasRowid(pTab); + + /* Figure out if we have any triggers and if the table being + ** inserted into is a view + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_INSERT, 0, &tmask); + isView = pTab->pSelect!=0; +#else +# define pTrigger 0 +# define tmask 0 +# define isView 0 +#endif +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + assert( (pTrigger && tmask) || (pTrigger==0 && tmask==0) ); + + /* If pTab is really a view, make sure it has been initialized. + ** ViewGetColumnNames() is a no-op if pTab is not a view. + */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto insert_cleanup; + } + + /* Cannot insert into a read-only table. + */ + if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + goto insert_cleanup; + } + + /* Allocate a VDBE + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto insert_cleanup; + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, pSelect || pTrigger, iDb); + +#ifndef SQLITE_OMIT_XFER_OPT + /* If the statement is of the form + ** + ** INSERT INTO SELECT * FROM ; + ** + ** Then special optimizations can be applied that make the transfer + ** very fast and which reduce fragmentation of indices. + ** + ** This is the 2nd template. + */ + if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + assert( !pTrigger ); + assert( pList==0 ); + goto insert_end; + } +#endif /* SQLITE_OMIT_XFER_OPT */ + + /* If this is an AUTOINCREMENT table, look up the sequence number in the + ** sqlite_sequence table and store it in memory cell regAutoinc. + */ + regAutoinc = autoIncBegin(pParse, iDb, pTab); + + /* Allocate registers for holding the rowid of the new row, + ** the content of the new row, and the assemblied row record. + */ + regRowid = regIns = pParse->nMem+1; + pParse->nMem += pTab->nCol + 1; + if( IsVirtual(pTab) ){ + regRowid++; + pParse->nMem++; + } + regData = regRowid+1; + + /* If the INSERT statement included an IDLIST term, then make sure + ** all elements of the IDLIST really are columns of the table and + ** remember the column indices. + ** + ** If the table has an INTEGER PRIMARY KEY column and that column + ** is named in the IDLIST, then record in the ipkColumn variable + ** the index into IDLIST of the primary key column. ipkColumn is + ** the index of the primary key as it appears in IDLIST, not as + ** is appears in the original table. (The index of the INTEGER + ** PRIMARY KEY in the original table is pTab->iPKey.) + */ + if( pColumn ){ + for(i=0; inId; i++){ + pColumn->a[i].idx = -1; + } + for(i=0; inId; i++){ + for(j=0; jnCol; j++){ + if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zName)==0 ){ + pColumn->a[i].idx = j; + if( i!=j ) bIdListInOrder = 0; + if( j==pTab->iPKey ){ + ipkColumn = i; assert( !withoutRowid ); + } + break; + } + } + if( j>=pTab->nCol ){ + if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ + ipkColumn = i; + bIdListInOrder = 0; + }else{ + sqlite3ErrorMsg(pParse, "table %S has no column named %s", + pTabList, 0, pColumn->a[i].zName); + pParse->checkSchema = 1; + goto insert_cleanup; + } + } + } + } + + /* Figure out how many columns of data are supplied. If the data + ** is coming from a SELECT statement, then generate a co-routine that + ** produces a single row of the SELECT on each invocation. The + ** co-routine is the common header to the 3rd and 4th templates. + */ + if( pSelect ){ + /* Data is coming from a SELECT. Generate a co-routine to run the SELECT */ + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int rc; /* Result code */ + + regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + assert( pParse->nErr==0 || rc ); + if( rc || db->mallocFailed ) goto insert_cleanup; + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + + /* Set useTempTable to TRUE if the result of the SELECT statement + ** should be written into a temporary table (template 4). Set to + ** FALSE if each output row of the SELECT can be written directly into + ** the destination table (template 3). + ** + ** A temp table must be used if the table being updated is also one + ** of the tables being read by the SELECT statement. Also use a + ** temp table in the case of row triggers. + */ + if( pTrigger || readsTable(pParse, iDb, pTab) ){ + useTempTable = 1; + } + + if( useTempTable ){ + /* Invoke the coroutine to extract information from the SELECT + ** and add it to a transient table srcTab. The code generated + ** here is from the 4th template: + ** + ** B: open temp table + ** L: yield X, goto M at EOF + ** insert row from R..R+n into temp table + ** goto L + ** M: ... + */ + int regRec; /* Register to hold packed record */ + int regTempRowid; /* Register to hold temp table ROWID */ + int addrL; /* Label "L" */ + + srcTab = pParse->nTab++; + regRec = sqlite3GetTempReg(pParse); + regTempRowid = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, srcTab, nColumn); + addrL = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); + sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrL); + sqlite3VdbeJumpHere(v, addrL); + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempReg(pParse, regTempRowid); + } + }else{ + /* This is the case if the data for the INSERT is coming from a VALUES + ** clause + */ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + srcTab = -1; + assert( useTempTable==0 ); + nColumn = pList ? pList->nExpr : 0; + for(i=0; ia[i].pExpr) ){ + goto insert_cleanup; + } + } + } + + /* If there is no IDLIST term but the table has an integer primary + ** key, the set the ipkColumn variable to the integer primary key + ** column index in the original table definition. + */ + if( pColumn==0 && nColumn>0 ){ + ipkColumn = pTab->iPKey; + } + + /* Make sure the number of columns in the source data matches the number + ** of columns to be inserted into the table. + */ + if( IsVirtual(pTab) ){ + for(i=0; inCol; i++){ + nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); + } + } + if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ + sqlite3ErrorMsg(pParse, + "table %S has %d columns but %d values were supplied", + pTabList, 0, pTab->nCol-nHidden, nColumn); + goto insert_cleanup; + } + if( pColumn!=0 && nColumn!=pColumn->nId ){ + sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); + goto insert_cleanup; + } + + /* Initialize the count of rows to be inserted + */ + if( db->flags & SQLITE_CountRows ){ + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + } + + /* If this is not a view, open the table and and all indices */ + if( !isView ){ + int nIdx; + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, -1, 0, + &iDataCur, &iIdxCur); + aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); + if( aRegIdx==0 ){ + goto insert_cleanup; + } + for(i=0; inMem; + } + } + + /* This is the top of the main insertion loop */ + if( useTempTable ){ + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 4): + ** + ** rewind temp table, if empty goto D + ** C: loop over rows of intermediate table + ** transfer values form intermediate table into
    + ** end loop + ** D: ... + */ + addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); VdbeCoverage(v); + addrCont = sqlite3VdbeCurrentAddr(v); + }else if( pSelect ){ + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 3): + ** + ** C: yield X, at EOF goto D + ** insert the select result into
    from R..R+n + ** goto C + ** D: ... + */ + addrInsTop = addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); + } + + /* Run the BEFORE and INSTEAD OF triggers, if there are any + */ + endOfLoop = sqlite3VdbeMakeLabel(v); + if( tmask & TRIGGER_BEFORE ){ + int regCols = sqlite3GetTempRange(pParse, pTab->nCol+1); + + /* build the NEW.* reference row. Note that if there is an INTEGER + ** PRIMARY KEY into which a NULL is being inserted, that NULL will be + ** translated into a unique ID for the row. But on a BEFORE trigger, + ** we do not know what the unique ID will be (because the insert has + ** not happened yet) so we substitute a rowid of -1 + */ + if( ipkColumn<0 ){ + sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); + }else{ + int j1; + assert( !withoutRowid ); + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); + }else{ + assert( pSelect==0 ); /* Otherwise useTempTable is true */ + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); + } + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); + } + + /* Cannot have triggers on a virtual table. If it were possible, + ** this block would have to account for hidden column. + */ + assert( !IsVirtual(pTab) ); + + /* Create the new column data + */ + for(i=0; inCol; i++){ + if( pColumn==0 ){ + j = i; + }else{ + for(j=0; jnId; j++){ + if( pColumn->a[j].idx==i ) break; + } + } + if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) ){ + sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1); + }else if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); + }else{ + assert( pSelect==0 ); /* Otherwise useTempTable is true */ + sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1); + } + } + + /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, + ** do not attempt any conversions before assembling the record. + ** If this is a real table, attempt conversions as required by the + ** table column affinities. + */ + if( !isView ){ + sqlite3TableAffinity(v, pTab, regCols+1); + } + + /* Fire BEFORE or INSTEAD OF triggers */ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, + pTab, regCols-pTab->nCol-1, onError, endOfLoop); + + sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); + } + + /* Compute the content of the next row to insert into a range of + ** registers beginning at regIns. + */ + if( !isView ){ + if( IsVirtual(pTab) ){ + /* The row that the VUpdate opcode will delete: none */ + sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); + } + if( ipkColumn>=0 ){ + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regRowid); + }else if( pSelect ){ + sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); + }else{ + VdbeOp *pOp; + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); + pOp = sqlite3VdbeGetOp(v, -1); + if( ALWAYS(pOp) && pOp->opcode==OP_Null && !IsVirtual(pTab) ){ + appendFlag = 1; + pOp->opcode = OP_NewRowid; + pOp->p1 = iDataCur; + pOp->p2 = regRowid; + pOp->p3 = regAutoinc; + } + } + /* If the PRIMARY KEY expression is NULL, then use OP_NewRowid + ** to generate a unique primary key value. + */ + if( !appendFlag ){ + int j1; + if( !IsVirtual(pTab) ){ + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); + sqlite3VdbeJumpHere(v, j1); + }else{ + j1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); VdbeCoverage(v); + } + sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); + } + }else if( IsVirtual(pTab) || withoutRowid ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); + }else{ + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); + appendFlag = 1; + } + autoIncStep(pParse, regAutoinc, regRowid); + + /* Compute data for all columns of the new entry, beginning + ** with the first column. + */ + nHidden = 0; + for(i=0; inCol; i++){ + int iRegStore = regRowid+1+i; + if( i==pTab->iPKey ){ + /* The value of the INTEGER PRIMARY KEY column is always a NULL. + ** Whenever this column is read, the rowid will be substituted + ** in its place. Hence, fill this column with a NULL to avoid + ** taking up data space with information that will never be used. + ** As there may be shallow copies of this value, make it a soft-NULL */ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + continue; + } + if( pColumn==0 ){ + if( IsHiddenColumn(&pTab->aCol[i]) ){ + assert( IsVirtual(pTab) ); + j = -1; + nHidden++; + }else{ + j = i - nHidden; + } + }else{ + for(j=0; jnId; j++){ + if( pColumn->a[j].idx==i ) break; + } + } + if( j<0 || nColumn==0 || (pColumn && j>=pColumn->nId) ){ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + }else if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, iRegStore); + }else if( pSelect ){ + if( regFromSelect!=regData ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); + } + }else{ + sqlite3ExprCode(pParse, pList->a[j].pExpr, iRegStore); + } + } + + /* Generate code to check constraints and generate index keys and + ** do the insertion. + */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 1, pTab->nCol+2, regIns, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); + sqlite3MayAbort(pParse); + }else +#endif + { + int isReplace; /* Set to true if constraints may cause a replace */ + sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, + regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace + ); + sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, + regIns, aRegIdx, 0, appendFlag, isReplace==0); + } + } + + /* Update the count of rows that are inserted + */ + if( (db->flags & SQLITE_CountRows)!=0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); + } + + if( pTrigger ){ + /* Code AFTER triggers */ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, + pTab, regData-2-pTab->nCol, onError, endOfLoop); + } + + /* The bottom of the main insertion loop, if the data source + ** is a SELECT statement. + */ + sqlite3VdbeResolveLabel(v, endOfLoop); + if( useTempTable ){ + sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrInsTop); + sqlite3VdbeAddOp1(v, OP_Close, srcTab); + }else if( pSelect ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrCont); + sqlite3VdbeJumpHere(v, addrInsTop); + } + + if( !IsVirtual(pTab) && !isView ){ + /* Close all tables opened */ + if( iDataCurpIndex; pIdx; pIdx=pIdx->pNext, idx++){ + sqlite3VdbeAddOp1(v, OP_Close, idx+iIdxCur); + } + } + +insert_end: + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. + */ + if( pParse->nested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* + ** Return the number of rows inserted. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. + */ + if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC); + } + +insert_cleanup: + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprListDelete(db, pList); + sqlite3SelectDelete(db, pSelect); + sqlite3IdListDelete(db, pColumn); + sqlite3DbFree(db, aRegIdx); +} + +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** thely may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif +#ifdef tmask + #undef tmask +#endif + +/* +** Generate code to do constraint checks prior to an INSERT or an UPDATE +** on table pTab. +** +** The regNewData parameter is the first register in a range that contains +** the data to be inserted or the data after the update. There will be +** pTab->nCol+1 registers in this range. The first register (the one +** that regNewData points to) will contain the new rowid, or NULL in the +** case of a WITHOUT ROWID table. The second register in the range will +** contain the content of the first table column. The third register will +** contain the content of the second table column. And so forth. +** +** The regOldData parameter is similar to regNewData except that it contains +** the data prior to an UPDATE rather than afterwards. regOldData is zero +** for an INSERT. This routine can distinguish between UPDATE and INSERT by +** checking regOldData for zero. +** +** For an UPDATE, the pkChng boolean is true if the true primary key (the +** rowid for a normal table or the PRIMARY KEY for a WITHOUT ROWID table) +** might be modified by the UPDATE. If pkChng is false, then the key of +** the iDataCur content table is guaranteed to be unchanged by the UPDATE. +** +** For an INSERT, the pkChng boolean indicates whether or not the rowid +** was explicitly specified as part of the INSERT statement. If pkChng +** is zero, it means that the either rowid is computed automatically or +** that the table is a WITHOUT ROWID table and has no rowid. On an INSERT, +** pkChng will only be true if the INSERT statement provides an integer +** value for either the rowid column or its INTEGER PRIMARY KEY alias. +** +** The code generated by this routine will store new index entries into +** registers identified by aRegIdx[]. No index entry is created for +** indices where aRegIdx[i]==0. The order of indices in aRegIdx[] is +** the same as the order of indices on the linked list of indices +** at pTab->pIndex. +** +** The caller must have already opened writeable cursors on the main +** table and all applicable indices (that is to say, all indices for which +** aRegIdx[] is not zero). iDataCur is the cursor for the main table when +** inserting or updating a rowid table, or the cursor for the PRIMARY KEY +** index when operating on a WITHOUT ROWID table. iIdxCur is the cursor +** for the first index in the pTab->pIndex list. Cursors for other indices +** are at iIdxCur+N for the N-th element of the pTab->pIndex list. +** +** This routine also generates code to check constraints. NOT NULL, +** CHECK, and UNIQUE constraints are all checked. If a constraint fails, +** then the appropriate action is performed. There are five possible +** actions: ROLLBACK, ABORT, FAIL, REPLACE, and IGNORE. +** +** Constraint type Action What Happens +** --------------- ---------- ---------------------------------------- +** any ROLLBACK The current transaction is rolled back and +** sqlite3_step() returns immediately with a +** return code of SQLITE_CONSTRAINT. +** +** any ABORT Back out changes from the current command +** only (do not do a complete rollback) then +** cause sqlite3_step() to return immediately +** with SQLITE_CONSTRAINT. +** +** any FAIL Sqlite3_step() returns immediately with a +** return code of SQLITE_CONSTRAINT. The +** transaction is not rolled back and any +** changes to prior rows are retained. +** +** any IGNORE The attempt in insert or update the current +** row is skipped, without throwing an error. +** Processing continues with the next row. +** (There is an immediate jump to ignoreDest.) +** +** NOT NULL REPLACE The NULL value is replace by the default +** value for that column. If the default value +** is NULL, the action is the same as ABORT. +** +** UNIQUE REPLACE The other row that conflicts with the row +** being inserted is removed. +** +** CHECK REPLACE Illegal. The results in an exception. +** +** Which action to take is determined by the overrideError parameter. +** Or if overrideError==OE_Default, then the pParse->onError parameter +** is used. Or if pParse->onError==OE_Default then the onError value +** for the constraint is used. +*/ +SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( + Parse *pParse, /* The parser context */ + Table *pTab, /* The table being inserted or updated */ + int *aRegIdx, /* Use register aRegIdx[i] for index i. 0 for unused */ + int iDataCur, /* Canonical data cursor (main table or PK index) */ + int iIdxCur, /* First index cursor */ + int regNewData, /* First register in a range holding values to insert */ + int regOldData, /* Previous content. 0 for INSERTs */ + u8 pkChng, /* Non-zero if the rowid or PRIMARY KEY changed */ + u8 overrideError, /* Override onError to this if not OE_Default */ + int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ + int *pbMayReplace /* OUT: Set to true if constraint may cause a replace */ +){ + Vdbe *v; /* VDBE under constrution */ + Index *pIdx; /* Pointer to one of the indices */ + Index *pPk = 0; /* The PRIMARY KEY index */ + sqlite3 *db; /* Database connection */ + int i; /* loop counter */ + int ix; /* Index loop counter */ + int nCol; /* Number of columns */ + int onError; /* Conflict resolution strategy */ + int j1; /* Addresss of jump instruction */ + int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ + int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ + int ipkTop = 0; /* Top of the rowid change constraint check */ + int ipkBottom = 0; /* Bottom of the rowid change constraint check */ + u8 isUpdate; /* True if this is an UPDATE operation */ + u8 bAffinityDone = 0; /* True if the OP_Affinity operation has been run */ + int regRowid = -1; /* Register holding ROWID value */ + + isUpdate = regOldData!=0; + db = pParse->db; + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + assert( pTab->pSelect==0 ); /* This table is not a VIEW */ + nCol = pTab->nCol; + + /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for + ** normal rowid tables. nPkField is the number of key fields in the + ** pPk index or 1 for a rowid table. In other words, nPkField is the + ** number of fields in the true primary key of the table. */ + if( HasRowid(pTab) ){ + pPk = 0; + nPkField = 1; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + nPkField = pPk->nKeyCol; + } + + /* Record that this module has started */ + VdbeModuleComment((v, "BEGIN: GenCnstCks(%d,%d,%d,%d,%d)", + iDataCur, iIdxCur, regNewData, regOldData, pkChng)); + + /* Test all NOT NULL constraints. + */ + for(i=0; iiPKey ){ + continue; + } + onError = pTab->aCol[i].notNull; + if( onError==OE_None ) continue; + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + if( onError==OE_Replace && pTab->aCol[i].pDflt==0 ){ + onError = OE_Abort; + } + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace ); + switch( onError ){ + case OE_Abort: + sqlite3MayAbort(pParse); + /* Fall through */ + case OE_Rollback: + case OE_Fail: { + char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, + pTab->aCol[i].zName); + sqlite3VdbeAddOp4(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, + regNewData+1+i, zMsg, P4_DYNAMIC); + sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); + VdbeCoverage(v); + break; + } + case OE_Ignore: { + sqlite3VdbeAddOp2(v, OP_IsNull, regNewData+1+i, ignoreDest); + VdbeCoverage(v); + break; + } + default: { + assert( onError==OE_Replace ); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); VdbeCoverage(v); + sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); + sqlite3VdbeJumpHere(v, j1); + break; + } + } + } + + /* Test all CHECK constraints + */ +#ifndef SQLITE_OMIT_CHECK + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = pTab->pCheck; + pParse->ckBase = regNewData+1; + onError = overrideError!=OE_Default ? overrideError : OE_Abort; + for(i=0; inExpr; i++){ + int allOk = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); + if( onError==OE_Ignore ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + }else{ + char *zName = pCheck->a[i].zName; + if( zName==0 ) zName = pTab->zName; + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, + onError, zName, P4_TRANSIENT, + P5_ConstraintCheck); + } + sqlite3VdbeResolveLabel(v, allOk); + } + } +#endif /* !defined(SQLITE_OMIT_CHECK) */ + + /* If rowid is changing, make sure the new rowid does not previously + ** exist in the table. + */ + if( pkChng && pPk==0 ){ + int addrRowidOk = sqlite3VdbeMakeLabel(v); + + /* Figure out what action to take in case of a rowid collision */ + onError = pTab->keyConf; + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + + if( isUpdate ){ + /* pkChng!=0 does not mean that the rowid has change, only that + ** it might have changed. Skip the conflict logic below if the rowid + ** is unchanged. */ + sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + + /* If the response to a rowid conflict is REPLACE but the response + ** to some other UNIQUE constraint is FAIL or IGNORE, then we need + ** to defer the running of the rowid conflict checking until after + ** the UNIQUE constraints have run. + */ + if( onError==OE_Replace && overrideError!=OE_Replace ){ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_Ignore || pIdx->onError==OE_Fail ){ + ipkTop = sqlite3VdbeAddOp0(v, OP_Goto); + break; + } + } + } + + /* Check to see if the new rowid already exists in the table. Skip + ** the following conflict logic if it does not. */ + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRowidOk, regNewData); + VdbeCoverage(v); + + /* Generate code that deals with a rowid collision */ + switch( onError ){ + default: { + onError = OE_Abort; + /* Fall thru into the next case */ + } + case OE_Rollback: + case OE_Abort: + case OE_Fail: { + sqlite3RowidConstraint(pParse, onError, pTab); + break; + } + case OE_Replace: { + /* If there are DELETE triggers on this table and the + ** recursive-triggers flag is set, call GenerateRowDelete() to + ** remove the conflicting row from the table. This will fire + ** the triggers and remove both the table and index b-tree entries. + ** + ** Otherwise, if there are no triggers or the recursive-triggers + ** flag is not set, but the table has one or more indexes, call + ** GenerateRowIndexDelete(). This removes the index b-tree entries + ** only. The table b-tree entry will be replaced by the new entry + ** when it is inserted. + ** + ** If either GenerateRowDelete() or GenerateRowIndexDelete() is called, + ** also invoke MultiWrite() to indicate that this VDBE may require + ** statement rollback (if the statement is aborted after the delete + ** takes place). Earlier versions called sqlite3MultiWrite() regardless, + ** but being more selective here allows statements like: + ** + ** REPLACE INTO t(rowid) VALUES($newrowid) + ** + ** to run without a statement journal if there are no indexes on the + ** table. + */ + Trigger *pTrigger = 0; + if( db->flags&SQLITE_RecTriggers ){ + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + } + if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + regNewData, 1, 0, OE_Replace, 1); + }else if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + } + seenReplace = 1; + break; + } + case OE_Ignore: { + /*assert( seenReplace==0 );*/ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + break; + } + } + sqlite3VdbeResolveLabel(v, addrRowidOk); + if( ipkTop ){ + ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, ipkTop); + } + } + + /* Test all UNIQUE constraints by creating entries for each UNIQUE + ** index and making sure that duplicate entries do not already exist. + ** Compute the revised record entries for indices as we go. + ** + ** This loop also handles the case of the PRIMARY KEY index for a + ** WITHOUT ROWID table. + */ + for(ix=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, ix++){ + int regIdx; /* Range of registers hold conent for pIdx */ + int regR; /* Range of registers holding conflicting PK */ + int iThisCur; /* Cursor for this UNIQUE index */ + int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ + + if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ + if( bAffinityDone==0 ){ + sqlite3TableAffinity(v, pTab, regNewData+1); + bAffinityDone = 1; + } + iThisCur = iIdxCur+ix; + addrUniqueOk = sqlite3VdbeMakeLabel(v); + + /* Skip partial indices for which the WHERE clause is not true */ + if( pIdx->pPartIdxWhere ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); + pParse->ckBase = regNewData+1; + sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, addrUniqueOk, + SQLITE_JUMPIFNULL); + pParse->ckBase = 0; + } + + /* Create a record for this index entry as it should appear after + ** the insert or update. Store that record in the aRegIdx[ix] register + */ + regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn); + for(i=0; inColumn; i++){ + int iField = pIdx->aiColumn[i]; + int x; + if( iField<0 || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ + x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + }else{ + x = iField + regNewData + 1; + } + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); + VdbeComment((v, "for %s", pIdx->zName)); + sqlite3ExprCacheAffinityChange(pParse, regIdx, pIdx->nColumn); + + /* In an UPDATE operation, if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table and there has been no change the + ** primary key, then no collision is possible. The collision detection + ** logic below can all be skipped. */ + if( isUpdate && pPk==pIdx && pkChng==0 ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; + } + + /* Find out what action to take in case there is a uniqueness conflict */ + onError = pIdx->onError; + if( onError==OE_None ){ + sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn); + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; /* pIdx is not a UNIQUE index */ + } + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + + /* Check to see if the new index entry will be unique */ + sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, + regIdx, pIdx->nKeyCol); VdbeCoverage(v); + + /* Generate code to handle collisions */ + regR = (pIdx==pPk) ? regIdx : sqlite3GetTempRange(pParse, nPkField); + if( isUpdate || onError==OE_Replace ){ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_IdxRowid, iThisCur, regR); + /* Conflict only if the rowid of the existing index entry + ** is different from old-rowid */ + if( isUpdate ){ + sqlite3VdbeAddOp3(v, OP_Eq, regR, addrUniqueOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + }else{ + int x; + /* Extract the PRIMARY KEY from the end of the index entry and + ** store it in registers regR..regR+nPk-1 */ + if( pIdx!=pPk ){ + for(i=0; inKeyCol; i++){ + x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); + sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); + VdbeComment((v, "%s.%s", pTab->zName, + pTab->aCol[pPk->aiColumn[i]].zName)); + } + } + if( isUpdate ){ + /* If currently processing the PRIMARY KEY of a WITHOUT ROWID + ** table, only conflict if the new PRIMARY KEY values are actually + ** different from the old. + ** + ** For a UNIQUE index, only conflict if the PRIMARY KEY values + ** of the matched index row are different from the original PRIMARY + ** KEY values of this row before the update. */ + int addrJump = sqlite3VdbeCurrentAddr(v)+pPk->nKeyCol; + int op = OP_Ne; + int regCmp = (IsPrimaryKeyIndex(pIdx) ? regIdx : regR); + + for(i=0; inKeyCol; i++){ + char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); + x = pPk->aiColumn[i]; + if( i==(pPk->nKeyCol-1) ){ + addrJump = addrUniqueOk; + op = OP_Eq; + } + sqlite3VdbeAddOp4(v, op, + regOldData+1+x, addrJump, regCmp+i, p4, P4_COLLSEQ + ); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverageIf(v, op==OP_Eq); + VdbeCoverageIf(v, op==OP_Ne); + } + } + } + } + + /* Generate code that executes if the new index entry is not unique */ + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace ); + switch( onError ){ + case OE_Rollback: + case OE_Abort: + case OE_Fail: { + sqlite3UniqueConstraint(pParse, onError, pIdx); + break; + } + case OE_Ignore: { + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + break; + } + default: { + Trigger *pTrigger = 0; + assert( onError==OE_Replace ); + sqlite3MultiWrite(pParse); + if( db->flags&SQLITE_RecTriggers ){ + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + } + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + regR, nPkField, 0, OE_Replace, pIdx==pPk); + seenReplace = 1; + break; + } + } + sqlite3VdbeResolveLabel(v, addrUniqueOk); + sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn); + if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); + } + if( ipkTop ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ipkTop+1); + sqlite3VdbeJumpHere(v, ipkBottom); + } + + *pbMayReplace = seenReplace; + VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); +} + +/* +** This routine generates code to finish the INSERT or UPDATE operation +** that was started by a prior call to sqlite3GenerateConstraintChecks. +** A consecutive range of registers starting at regNewData contains the +** rowid and the content to be inserted. +** +** The arguments to this routine should be the same as the first six +** arguments to sqlite3GenerateConstraintChecks. +*/ +SQLITE_PRIVATE void sqlite3CompleteInsertion( + Parse *pParse, /* The parser context */ + Table *pTab, /* the table into which we are inserting */ + int iDataCur, /* Cursor of the canonical data source */ + int iIdxCur, /* First index cursor */ + int regNewData, /* Range of content */ + int *aRegIdx, /* Register used by each index. 0 for unused indices */ + int isUpdate, /* True for UPDATE, False for INSERT */ + int appendBias, /* True if this is likely to be an append */ + int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ +){ + Vdbe *v; /* Prepared statements under construction */ + Index *pIdx; /* An index being inserted or updated */ + u8 pik_flags; /* flag values passed to the btree insert */ + int regData; /* Content registers (after the rowid) */ + int regRec; /* Register holding assemblied record for the table */ + int i; /* Loop counter */ + u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ + + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + assert( pTab->pSelect==0 ); /* This table is not a VIEW */ + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + if( aRegIdx[i]==0 ) continue; + bAffinityDone = 1; + if( pIdx->pPartIdxWhere ){ + sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i]); + pik_flags = 0; + if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT; + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + assert( pParse->nested==0 ); + pik_flags |= OPFLAG_NCHANGE; + } + if( pik_flags ) sqlite3VdbeChangeP5(v, pik_flags); + } + if( !HasRowid(pTab) ) return; + regData = regNewData + 1; + regRec = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); + if( !bAffinityDone ) sqlite3TableAffinity(v, pTab, 0); + sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); + if( pParse->nested ){ + pik_flags = 0; + }else{ + pik_flags = OPFLAG_NCHANGE; + pik_flags |= (isUpdate?OPFLAG_ISUPDATE:OPFLAG_LASTROWID); + } + if( appendBias ){ + pik_flags |= OPFLAG_APPEND; + } + if( useSeekResult ){ + pik_flags |= OPFLAG_USESEEKRESULT; + } + sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, regRec, regNewData); + if( !pParse->nested ){ + sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); + } + sqlite3VdbeChangeP5(v, pik_flags); +} + +/* +** Allocate cursors for the pTab table and all its indices and generate +** code to open and initialized those cursors. +** +** The cursor for the object that contains the complete data (normally +** the table itself, but the PRIMARY KEY index in the case of a WITHOUT +** ROWID table) is returned in *piDataCur. The first index cursor is +** returned in *piIdxCur. The number of indices is returned. +** +** Use iBase as the first cursor (either the *piDataCur for rowid tables +** or the first index for WITHOUT ROWID tables) if it is non-negative. +** If iBase is negative, then allocate the next available cursor. +** +** For a rowid table, *piDataCur will be exactly one less than *piIdxCur. +** For a WITHOUT ROWID table, *piDataCur will be somewhere in the range +** of *piIdxCurs, depending on where the PRIMARY KEY index appears on the +** pTab->pIndex list. +*/ +SQLITE_PRIVATE int sqlite3OpenTableAndIndices( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table to be opened */ + int op, /* OP_OpenRead or OP_OpenWrite */ + int iBase, /* Use this for the table cursor, if there is one */ + u8 *aToOpen, /* If not NULL: boolean for each table and index */ + int *piDataCur, /* Write the database source cursor number here */ + int *piIdxCur /* Write the first index cursor number here */ +){ + int i; + int iDb; + int iDataCur; + Index *pIdx; + Vdbe *v; + + assert( op==OP_OpenRead || op==OP_OpenWrite ); + if( IsVirtual(pTab) ){ + assert( aToOpen==0 ); + *piDataCur = 0; + *piIdxCur = 1; + return 0; + } + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + if( iBase<0 ) iBase = pParse->nTab; + iDataCur = iBase++; + if( piDataCur ) *piDataCur = iDataCur; + if( HasRowid(pTab) && (aToOpen==0 || aToOpen[0]) ){ + sqlite3OpenTable(pParse, iDataCur, iDb, pTab, op); + }else{ + sqlite3TableLock(pParse, iDb, pTab->tnum, op==OP_OpenWrite, pTab->zName); + } + if( piIdxCur ) *piIdxCur = iBase; + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + int iIdxCur = iBase++; + assert( pIdx->pSchema==pTab->pSchema ); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) && piDataCur ){ + *piDataCur = iIdxCur; + } + if( aToOpen==0 || aToOpen[i+1] ){ + sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + } + } + if( iBase>pParse->nTab ) pParse->nTab = iBase; + return i; +} + + +#ifdef SQLITE_TEST +/* +** The following global variable is incremented whenever the +** transfer optimization is used. This is used for testing +** purposes only - to make sure the transfer optimization really +** is happening when it is suppose to. +*/ +SQLITE_API int sqlite3_xferopt_count; +#endif /* SQLITE_TEST */ + + +#ifndef SQLITE_OMIT_XFER_OPT +/* +** Check to collation names to see if they are compatible. +*/ +static int xferCompatibleCollation(const char *z1, const char *z2){ + if( z1==0 ){ + return z2==0; + } + if( z2==0 ){ + return 0; + } + return sqlite3StrICmp(z1, z2)==0; +} + + +/* +** Check to see if index pSrc is compatible as a source of data +** for index pDest in an insert transfer optimization. The rules +** for a compatible index: +** +** * The index is over the same set of columns +** * The same DESC and ASC markings occurs on all columns +** * The same onError processing (OE_Abort, OE_Ignore, etc) +** * The same collating sequence on each column +** * The index has the exact same WHERE clause +*/ +static int xferCompatibleIndex(Index *pDest, Index *pSrc){ + int i; + assert( pDest && pSrc ); + assert( pDest->pTable!=pSrc->pTable ); + if( pDest->nKeyCol!=pSrc->nKeyCol ){ + return 0; /* Different number of columns */ + } + if( pDest->onError!=pSrc->onError ){ + return 0; /* Different conflict resolution strategies */ + } + for(i=0; inKeyCol; i++){ + if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ + return 0; /* Different columns indexed */ + } + if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ + return 0; /* Different sort orders */ + } + if( !xferCompatibleCollation(pSrc->azColl[i],pDest->azColl[i]) ){ + return 0; /* Different collating sequences */ + } + } + if( sqlite3ExprCompare(pSrc->pPartIdxWhere, pDest->pPartIdxWhere, -1) ){ + return 0; /* Different WHERE clauses */ + } + + /* If no test above fails then the indices must be compatible */ + return 1; +} + +/* +** Attempt the transfer optimization on INSERTs of the form +** +** INSERT INTO tab1 SELECT * FROM tab2; +** +** The xfer optimization transfers raw records from tab2 over to tab1. +** Columns are not decoded and reassemblied, which greatly improves +** performance. Raw index records are transferred in the same way. +** +** The xfer optimization is only attempted if tab1 and tab2 are compatible. +** There are lots of rules for determining compatibility - see comments +** embedded in the code for details. +** +** This routine returns TRUE if the optimization is guaranteed to be used. +** Sometimes the xfer optimization will only work if the destination table +** is empty - a factor that can only be determined at run-time. In that +** case, this routine generates code for the xfer optimization but also +** does a test to see if the destination table is empty and jumps over the +** xfer optimization code if the test fails. In that case, this routine +** returns FALSE so that the caller will know to go ahead and generate +** an unoptimized transfer. This routine also returns FALSE if there +** is no chance that the xfer optimization can be applied. +** +** This optimization is particularly useful at making VACUUM run faster. +*/ +static int xferOptimization( + Parse *pParse, /* Parser context */ + Table *pDest, /* The table we are inserting into */ + Select *pSelect, /* A SELECT statement to use as the data source */ + int onError, /* How to handle constraint errors */ + int iDbDest /* The database of pDest */ +){ + ExprList *pEList; /* The result set of the SELECT */ + Table *pSrc; /* The table in the FROM clause of SELECT */ + Index *pSrcIdx, *pDestIdx; /* Source and destination indices */ + struct SrcList_item *pItem; /* An element of pSelect->pSrc */ + int i; /* Loop counter */ + int iDbSrc; /* The database of pSrc */ + int iSrc, iDest; /* Cursors from source and destination */ + int addr1, addr2; /* Loop addresses */ + int emptyDestTest = 0; /* Address of test for empty pDest */ + int emptySrcTest = 0; /* Address of test for empty pSrc */ + Vdbe *v; /* The VDBE we are building */ + int regAutoinc; /* Memory register used by AUTOINC */ + int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ + int regData, regRowid; /* Registers holding data and rowid */ + + if( pSelect==0 ){ + return 0; /* Must be of the form INSERT INTO ... SELECT ... */ + } + if( pParse->pWith || pSelect->pWith ){ + /* Do not attempt to process this query if there are an WITH clauses + ** attached to it. Proceeding may generate a false "no such table: xxx" + ** error if pSelect reads from a CTE named "xxx". */ + return 0; + } + if( sqlite3TriggerList(pParse, pDest) ){ + return 0; /* tab1 must not have triggers */ + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pDest->tabFlags & TF_Virtual ){ + return 0; /* tab1 must not be a virtual table */ + } +#endif + if( onError==OE_Default ){ + if( pDest->iPKey>=0 ) onError = pDest->keyConf; + if( onError==OE_Default ) onError = OE_Abort; + } + assert(pSelect->pSrc); /* allocated even if there is no FROM clause */ + if( pSelect->pSrc->nSrc!=1 ){ + return 0; /* FROM clause must have exactly one term */ + } + if( pSelect->pSrc->a[0].pSelect ){ + return 0; /* FROM clause cannot contain a subquery */ + } + if( pSelect->pWhere ){ + return 0; /* SELECT may not have a WHERE clause */ + } + if( pSelect->pOrderBy ){ + return 0; /* SELECT may not have an ORDER BY clause */ + } + /* Do not need to test for a HAVING clause. If HAVING is present but + ** there is no ORDER BY, we will get an error. */ + if( pSelect->pGroupBy ){ + return 0; /* SELECT may not have a GROUP BY clause */ + } + if( pSelect->pLimit ){ + return 0; /* SELECT may not have a LIMIT clause */ + } + assert( pSelect->pOffset==0 ); /* Must be so if pLimit==0 */ + if( pSelect->pPrior ){ + return 0; /* SELECT may not be a compound query */ + } + if( pSelect->selFlags & SF_Distinct ){ + return 0; /* SELECT may not be DISTINCT */ + } + pEList = pSelect->pEList; + assert( pEList!=0 ); + if( pEList->nExpr!=1 ){ + return 0; /* The result set must have exactly one column */ + } + assert( pEList->a[0].pExpr ); + if( pEList->a[0].pExpr->op!=TK_ALL ){ + return 0; /* The result set must be the special operator "*" */ + } + + /* At this point we have established that the statement is of the + ** correct syntactic form to participate in this optimization. Now + ** we have to check the semantics. + */ + pItem = pSelect->pSrc->a; + pSrc = sqlite3LocateTableItem(pParse, 0, pItem); + if( pSrc==0 ){ + return 0; /* FROM clause does not contain a real table */ + } + if( pSrc==pDest ){ + return 0; /* tab1 and tab2 may not be the same table */ + } + if( HasRowid(pDest)!=HasRowid(pSrc) ){ + return 0; /* source and destination must both be WITHOUT ROWID or not */ + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pSrc->tabFlags & TF_Virtual ){ + return 0; /* tab2 must not be a virtual table */ + } +#endif + if( pSrc->pSelect ){ + return 0; /* tab2 may not be a view */ + } + if( pDest->nCol!=pSrc->nCol ){ + return 0; /* Number of columns must be the same in tab1 and tab2 */ + } + if( pDest->iPKey!=pSrc->iPKey ){ + return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ + } + for(i=0; inCol; i++){ + Column *pDestCol = &pDest->aCol[i]; + Column *pSrcCol = &pSrc->aCol[i]; + if( pDestCol->affinity!=pSrcCol->affinity ){ + return 0; /* Affinity must be the same on all columns */ + } + if( !xferCompatibleCollation(pDestCol->zColl, pSrcCol->zColl) ){ + return 0; /* Collating sequence must be the same on all columns */ + } + if( pDestCol->notNull && !pSrcCol->notNull ){ + return 0; /* tab2 must be NOT NULL if tab1 is */ + } + /* Default values for second and subsequent columns need to match. */ + if( i>0 + && ((pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) + || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)!=0)) + ){ + return 0; /* Default values must be the same for all columns */ + } + } + for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + if( IsUniqueIndex(pDestIdx) ){ + destHasUniqueIdx = 1; + } + for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ + if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + } + if( pSrcIdx==0 ){ + return 0; /* pDestIdx has no corresponding index in pSrc */ + } + } +#ifndef SQLITE_OMIT_CHECK + if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + return 0; /* Tables have different CHECK constraints. Ticket #2252 */ + } +#endif +#ifndef SQLITE_OMIT_FOREIGN_KEY + /* Disallow the transfer optimization if the destination table constains + ** any foreign key constraints. This is more restrictive than necessary. + ** But the main beneficiary of the transfer optimization is the VACUUM + ** command, and the VACUUM command disables foreign key constraints. So + ** the extra complication to make this rule less restrictive is probably + ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] + */ + if( (pParse->db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ + return 0; + } +#endif + if( (pParse->db->flags & SQLITE_CountRows)!=0 ){ + return 0; /* xfer opt does not play well with PRAGMA count_changes */ + } + + /* If we get this far, it means that the xfer optimization is at + ** least a possibility, though it might only work if the destination + ** table (tab1) is initially empty. + */ +#ifdef SQLITE_TEST + sqlite3_xferopt_count++; +#endif + iDbSrc = sqlite3SchemaToIndex(pParse->db, pSrc->pSchema); + v = sqlite3GetVdbe(pParse); + sqlite3CodeVerifySchema(pParse, iDbSrc); + iSrc = pParse->nTab++; + iDest = pParse->nTab++; + regAutoinc = autoIncBegin(pParse, iDbDest, pDest); + regData = sqlite3GetTempReg(pParse); + regRowid = sqlite3GetTempReg(pParse); + sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); + assert( HasRowid(pDest) || destHasUniqueIdx ); + if( (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ + || destHasUniqueIdx /* (2) */ + || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ + ){ + /* In some circumstances, we are able to run the xfer optimization + ** only if the destination table is initially empty. This code makes + ** that determination. Conditions under which the destination must + ** be empty: + ** + ** (1) There is no INTEGER PRIMARY KEY but there are indices. + ** (If the destination is not initially empty, the rowid fields + ** of index entries might need to change.) + ** + ** (2) The destination has a unique index. (The xfer optimization + ** is unable to test uniqueness.) + ** + ** (3) onError is something other than OE_Abort and OE_Rollback. + */ + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); + emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + sqlite3VdbeJumpHere(v, addr1); + } + if( HasRowid(pSrc) ){ + sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); + emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); + if( pDest->iPKey>=0 ){ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); + VdbeCoverage(v); + sqlite3RowidConstraint(pParse, onError, pDest); + sqlite3VdbeJumpHere(v, addr2); + autoIncStep(pParse, regAutoinc, regRowid); + }else if( pDest->pIndex==0 ){ + addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); + }else{ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + assert( (pDest->tabFlags & TF_Autoincrement)==0 ); + } + sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); + sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + }else{ + sqlite3TableLock(pParse, iDbDest, pDest->tnum, 1, pDest->zName); + sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); + } + for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ + if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + } + assert( pSrcIdx ); + sqlite3VdbeAddOp3(v, OP_OpenRead, iSrc, pSrcIdx->tnum, iDbSrc); + sqlite3VdbeSetP4KeyInfo(pParse, pSrcIdx); + VdbeComment((v, "%s", pSrcIdx->zName)); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iDest, pDestIdx->tnum, iDbDest); + sqlite3VdbeSetP4KeyInfo(pParse, pDestIdx); + sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); + VdbeComment((v, "%s", pDestIdx->zName)); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + } + if( emptySrcTest ) sqlite3VdbeJumpHere(v, emptySrcTest); + sqlite3ReleaseTempReg(pParse, regRowid); + sqlite3ReleaseTempReg(pParse, regData); + if( emptyDestTest ){ + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); + sqlite3VdbeJumpHere(v, emptyDestTest); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + return 0; + }else{ + return 1; + } +} +#endif /* SQLITE_OMIT_XFER_OPT */ + +/************** End of insert.c **********************************************/ +/************** Begin file legacy.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ + + +/* +** Execute SQL code. Return one of the SQLITE_ success/failure +** codes. Also write an error message into memory obtained from +** malloc() and make *pzErrMsg point to that message. +** +** If the SQL is a query, then for each row in the query result +** the xCallback() function is called. pArg becomes the first +** argument to xCallback(). If xCallback=NULL then no callback +** is invoked, even for queries. +*/ +SQLITE_API int sqlite3_exec( + sqlite3 *db, /* The database on which the SQL executes */ + const char *zSql, /* The SQL to be executed */ + sqlite3_callback xCallback, /* Invoke this callback routine */ + void *pArg, /* First argument to xCallback() */ + char **pzErrMsg /* Write error messages here */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zLeftover; /* Tail of unprocessed SQL */ + sqlite3_stmt *pStmt = 0; /* The current SQL statement */ + char **azCols = 0; /* Names of result columns */ + int callbackIsInit; /* True if callback data is initialized */ + + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; + if( zSql==0 ) zSql = ""; + + sqlite3_mutex_enter(db->mutex); + sqlite3Error(db, SQLITE_OK, 0); + while( rc==SQLITE_OK && zSql[0] ){ + int nCol; + char **azVals = 0; + + pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); + assert( rc==SQLITE_OK || pStmt==0 ); + if( rc!=SQLITE_OK ){ + continue; + } + if( !pStmt ){ + /* this happens for a comment or white-space */ + zSql = zLeftover; + continue; + } + + callbackIsInit = 0; + nCol = sqlite3_column_count(pStmt); + + while( 1 ){ + int i; + rc = sqlite3_step(pStmt); + + /* Invoke the callback function if required */ + if( xCallback && (SQLITE_ROW==rc || + (SQLITE_DONE==rc && !callbackIsInit + && db->flags&SQLITE_NullCallback)) ){ + if( !callbackIsInit ){ + azCols = sqlite3DbMallocZero(db, 2*nCol*sizeof(const char*) + 1); + if( azCols==0 ){ + goto exec_out; + } + for(i=0; imallocFailed = 1; + goto exec_out; + } + } + } + if( xCallback(pArg, nCol, azVals, azCols) ){ + /* EVIDENCE-OF: R-38229-40159 If the callback function to + ** sqlite3_exec() returns non-zero, then sqlite3_exec() will + ** return SQLITE_ABORT. */ + rc = SQLITE_ABORT; + sqlite3VdbeFinalize((Vdbe *)pStmt); + pStmt = 0; + sqlite3Error(db, SQLITE_ABORT, 0); + goto exec_out; + } + } + + if( rc!=SQLITE_ROW ){ + rc = sqlite3VdbeFinalize((Vdbe *)pStmt); + pStmt = 0; + zSql = zLeftover; + while( sqlite3Isspace(zSql[0]) ) zSql++; + break; + } + } + + sqlite3DbFree(db, azCols); + azCols = 0; + } + +exec_out: + if( pStmt ) sqlite3VdbeFinalize((Vdbe *)pStmt); + sqlite3DbFree(db, azCols); + + rc = sqlite3ApiExit(db, rc); + if( rc!=SQLITE_OK && ALWAYS(rc==sqlite3_errcode(db)) && pzErrMsg ){ + int nErrMsg = 1 + sqlite3Strlen30(sqlite3_errmsg(db)); + *pzErrMsg = sqlite3Malloc(nErrMsg); + if( *pzErrMsg ){ + memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); + }else{ + rc = SQLITE_NOMEM; + sqlite3Error(db, SQLITE_NOMEM, 0); + } + }else if( pzErrMsg ){ + *pzErrMsg = 0; + } + + assert( (rc&db->errMask)==rc ); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/************** End of legacy.c **********************************************/ +/************** Begin file loadext.c *****************************************/ +/* +** 2006 June 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to dynamically load extensions into +** the SQLite library. +*/ + +#ifndef SQLITE_CORE + #define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */ +#endif +/************** Include sqlite3ext.h in the middle of loadext.c **************/ +/************** Begin file sqlite3ext.h **************************************/ +/* +** 2006 June 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the SQLite interface for use by +** shared libraries that want to be imported as extensions into +** an SQLite instance. Shared libraries that intend to be loaded +** as extensions by SQLite should #include this file instead of +** sqlite3.h. +*/ +#ifndef _SQLITE3EXT_H_ +#define _SQLITE3EXT_H_ + +typedef struct sqlite3_api_routines sqlite3_api_routines; + +/* +** The following structure holds pointers to all of the SQLite API +** routines. +** +** WARNING: In order to maintain backwards compatibility, add new +** interfaces to the end of this structure only. If you insert new +** interfaces in the middle of this structure, then older different +** versions of SQLite will not be able to load each others' shared +** libraries! +*/ +struct sqlite3_api_routines { + void * (*aggregate_context)(sqlite3_context*,int nBytes); + int (*aggregate_count)(sqlite3_context*); + int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*)); + int (*bind_double)(sqlite3_stmt*,int,double); + int (*bind_int)(sqlite3_stmt*,int,int); + int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64); + int (*bind_null)(sqlite3_stmt*,int); + int (*bind_parameter_count)(sqlite3_stmt*); + int (*bind_parameter_index)(sqlite3_stmt*,const char*zName); + const char * (*bind_parameter_name)(sqlite3_stmt*,int); + int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*)); + int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*)); + int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*); + int (*busy_handler)(sqlite3*,int(*)(void*,int),void*); + int (*busy_timeout)(sqlite3*,int ms); + int (*changes)(sqlite3*); + int (*close)(sqlite3*); + int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const char*)); + int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const void*)); + const void * (*column_blob)(sqlite3_stmt*,int iCol); + int (*column_bytes)(sqlite3_stmt*,int iCol); + int (*column_bytes16)(sqlite3_stmt*,int iCol); + int (*column_count)(sqlite3_stmt*pStmt); + const char * (*column_database_name)(sqlite3_stmt*,int); + const void * (*column_database_name16)(sqlite3_stmt*,int); + const char * (*column_decltype)(sqlite3_stmt*,int i); + const void * (*column_decltype16)(sqlite3_stmt*,int); + double (*column_double)(sqlite3_stmt*,int iCol); + int (*column_int)(sqlite3_stmt*,int iCol); + sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol); + const char * (*column_name)(sqlite3_stmt*,int); + const void * (*column_name16)(sqlite3_stmt*,int); + const char * (*column_origin_name)(sqlite3_stmt*,int); + const void * (*column_origin_name16)(sqlite3_stmt*,int); + const char * (*column_table_name)(sqlite3_stmt*,int); + const void * (*column_table_name16)(sqlite3_stmt*,int); + const unsigned char * (*column_text)(sqlite3_stmt*,int iCol); + const void * (*column_text16)(sqlite3_stmt*,int iCol); + int (*column_type)(sqlite3_stmt*,int iCol); + sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol); + void * (*commit_hook)(sqlite3*,int(*)(void*),void*); + int (*complete)(const char*sql); + int (*complete16)(const void*sql); + int (*create_collation)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_collation16)(sqlite3*,const void*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_function)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_function16)(sqlite3*,const void*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); + int (*data_count)(sqlite3_stmt*pStmt); + sqlite3 * (*db_handle)(sqlite3_stmt*); + int (*declare_vtab)(sqlite3*,const char*); + int (*enable_shared_cache)(int); + int (*errcode)(sqlite3*db); + const char * (*errmsg)(sqlite3*); + const void * (*errmsg16)(sqlite3*); + int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**); + int (*expired)(sqlite3_stmt*); + int (*finalize)(sqlite3_stmt*pStmt); + void (*free)(void*); + void (*free_table)(char**result); + int (*get_autocommit)(sqlite3*); + void * (*get_auxdata)(sqlite3_context*,int); + int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**); + int (*global_recover)(void); + void (*interruptx)(sqlite3*); + sqlite_int64 (*last_insert_rowid)(sqlite3*); + const char * (*libversion)(void); + int (*libversion_number)(void); + void *(*malloc)(int); + char * (*mprintf)(const char*,...); + int (*open)(const char*,sqlite3**); + int (*open16)(const void*,sqlite3**); + int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*); + void (*progress_handler)(sqlite3*,int,int(*)(void*),void*); + void *(*realloc)(void*,int); + int (*reset)(sqlite3_stmt*pStmt); + void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_double)(sqlite3_context*,double); + void (*result_error)(sqlite3_context*,const char*,int); + void (*result_error16)(sqlite3_context*,const void*,int); + void (*result_int)(sqlite3_context*,int); + void (*result_int64)(sqlite3_context*,sqlite_int64); + void (*result_null)(sqlite3_context*); + void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*)); + void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_value)(sqlite3_context*,sqlite3_value*); + void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); + int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, + const char*,const char*),void*); + void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); + char * (*snprintf)(int,char*,const char*,...); + int (*step)(sqlite3_stmt*); + int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, + char const**,char const**,int*,int*,int*); + void (*thread_cleanup)(void); + int (*total_changes)(sqlite3*); + void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); + int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); + void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*, + sqlite_int64),void*); + void * (*user_data)(sqlite3_context*); + const void * (*value_blob)(sqlite3_value*); + int (*value_bytes)(sqlite3_value*); + int (*value_bytes16)(sqlite3_value*); + double (*value_double)(sqlite3_value*); + int (*value_int)(sqlite3_value*); + sqlite_int64 (*value_int64)(sqlite3_value*); + int (*value_numeric_type)(sqlite3_value*); + const unsigned char * (*value_text)(sqlite3_value*); + const void * (*value_text16)(sqlite3_value*); + const void * (*value_text16be)(sqlite3_value*); + const void * (*value_text16le)(sqlite3_value*); + int (*value_type)(sqlite3_value*); + char *(*vmprintf)(const char*,va_list); + /* Added ??? */ + int (*overload_function)(sqlite3*, const char *zFuncName, int nArg); + /* Added by 3.3.13 */ + int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + int (*clear_bindings)(sqlite3_stmt*); + /* Added by 3.4.1 */ + int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*, + void (*xDestroy)(void *)); + /* Added by 3.5.0 */ + int (*bind_zeroblob)(sqlite3_stmt*,int,int); + int (*blob_bytes)(sqlite3_blob*); + int (*blob_close)(sqlite3_blob*); + int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64, + int,sqlite3_blob**); + int (*blob_read)(sqlite3_blob*,void*,int,int); + int (*blob_write)(sqlite3_blob*,const void*,int,int); + int (*create_collation_v2)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*), + void(*)(void*)); + int (*file_control)(sqlite3*,const char*,int,void*); + sqlite3_int64 (*memory_highwater)(int); + sqlite3_int64 (*memory_used)(void); + sqlite3_mutex *(*mutex_alloc)(int); + void (*mutex_enter)(sqlite3_mutex*); + void (*mutex_free)(sqlite3_mutex*); + void (*mutex_leave)(sqlite3_mutex*); + int (*mutex_try)(sqlite3_mutex*); + int (*open_v2)(const char*,sqlite3**,int,const char*); + int (*release_memory)(int); + void (*result_error_nomem)(sqlite3_context*); + void (*result_error_toobig)(sqlite3_context*); + int (*sleep)(int); + void (*soft_heap_limit)(int); + sqlite3_vfs *(*vfs_find)(const char*); + int (*vfs_register)(sqlite3_vfs*,int); + int (*vfs_unregister)(sqlite3_vfs*); + int (*xthreadsafe)(void); + void (*result_zeroblob)(sqlite3_context*,int); + void (*result_error_code)(sqlite3_context*,int); + int (*test_control)(int, ...); + void (*randomness)(int,void*); + sqlite3 *(*context_db_handle)(sqlite3_context*); + int (*extended_result_codes)(sqlite3*,int); + int (*limit)(sqlite3*,int,int); + sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); + const char *(*sql)(sqlite3_stmt*); + int (*status)(int,int*,int*,int); + int (*backup_finish)(sqlite3_backup*); + sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); + int (*backup_pagecount)(sqlite3_backup*); + int (*backup_remaining)(sqlite3_backup*); + int (*backup_step)(sqlite3_backup*,int); + const char *(*compileoption_get)(int); + int (*compileoption_used)(const char*); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*)); + int (*db_config)(sqlite3*,int,...); + sqlite3_mutex *(*db_mutex)(sqlite3*); + int (*db_status)(sqlite3*,int,int*,int*,int); + int (*extended_errcode)(sqlite3*); + void (*log)(int,const char*,...); + sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); + const char *(*sourceid)(void); + int (*stmt_status)(sqlite3_stmt*,int,int); + int (*strnicmp)(const char*,const char*,int); + int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); + int (*wal_autocheckpoint)(sqlite3*,int); + int (*wal_checkpoint)(sqlite3*,const char*); + void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); + int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); + int (*vtab_config)(sqlite3*,int op,...); + int (*vtab_on_conflict)(sqlite3*); + /* Version 3.7.16 and later */ + int (*close_v2)(sqlite3*); + const char *(*db_filename)(sqlite3*,const char*); + int (*db_readonly)(sqlite3*,const char*); + int (*db_release_memory)(sqlite3*); + const char *(*errstr)(int); + int (*stmt_busy)(sqlite3_stmt*); + int (*stmt_readonly)(sqlite3_stmt*); + int (*stricmp)(const char*,const char*); + int (*uri_boolean)(const char*,const char*,int); + sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); + const char *(*uri_parameter)(const char*,const char*); + char *(*vsnprintf)(int,char*,const char*,va_list); + int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); +}; + +/* +** The following macros redefine the API routines so that they are +** redirected throught the global sqlite3_api structure. +** +** This header file is also used by the loadext.c source file +** (part of the main SQLite library - not an extension) so that +** it can get access to the sqlite3_api_routines structure +** definition. But the main library does not want to redefine +** the API. So the redefinition macros are only valid if the +** SQLITE_CORE macros is undefined. +*/ +#ifndef SQLITE_CORE +#define sqlite3_aggregate_context sqlite3_api->aggregate_context +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_aggregate_count sqlite3_api->aggregate_count +#endif +#define sqlite3_bind_blob sqlite3_api->bind_blob +#define sqlite3_bind_double sqlite3_api->bind_double +#define sqlite3_bind_int sqlite3_api->bind_int +#define sqlite3_bind_int64 sqlite3_api->bind_int64 +#define sqlite3_bind_null sqlite3_api->bind_null +#define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count +#define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index +#define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name +#define sqlite3_bind_text sqlite3_api->bind_text +#define sqlite3_bind_text16 sqlite3_api->bind_text16 +#define sqlite3_bind_value sqlite3_api->bind_value +#define sqlite3_busy_handler sqlite3_api->busy_handler +#define sqlite3_busy_timeout sqlite3_api->busy_timeout +#define sqlite3_changes sqlite3_api->changes +#define sqlite3_close sqlite3_api->close +#define sqlite3_collation_needed sqlite3_api->collation_needed +#define sqlite3_collation_needed16 sqlite3_api->collation_needed16 +#define sqlite3_column_blob sqlite3_api->column_blob +#define sqlite3_column_bytes sqlite3_api->column_bytes +#define sqlite3_column_bytes16 sqlite3_api->column_bytes16 +#define sqlite3_column_count sqlite3_api->column_count +#define sqlite3_column_database_name sqlite3_api->column_database_name +#define sqlite3_column_database_name16 sqlite3_api->column_database_name16 +#define sqlite3_column_decltype sqlite3_api->column_decltype +#define sqlite3_column_decltype16 sqlite3_api->column_decltype16 +#define sqlite3_column_double sqlite3_api->column_double +#define sqlite3_column_int sqlite3_api->column_int +#define sqlite3_column_int64 sqlite3_api->column_int64 +#define sqlite3_column_name sqlite3_api->column_name +#define sqlite3_column_name16 sqlite3_api->column_name16 +#define sqlite3_column_origin_name sqlite3_api->column_origin_name +#define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16 +#define sqlite3_column_table_name sqlite3_api->column_table_name +#define sqlite3_column_table_name16 sqlite3_api->column_table_name16 +#define sqlite3_column_text sqlite3_api->column_text +#define sqlite3_column_text16 sqlite3_api->column_text16 +#define sqlite3_column_type sqlite3_api->column_type +#define sqlite3_column_value sqlite3_api->column_value +#define sqlite3_commit_hook sqlite3_api->commit_hook +#define sqlite3_complete sqlite3_api->complete +#define sqlite3_complete16 sqlite3_api->complete16 +#define sqlite3_create_collation sqlite3_api->create_collation +#define sqlite3_create_collation16 sqlite3_api->create_collation16 +#define sqlite3_create_function sqlite3_api->create_function +#define sqlite3_create_function16 sqlite3_api->create_function16 +#define sqlite3_create_module sqlite3_api->create_module +#define sqlite3_create_module_v2 sqlite3_api->create_module_v2 +#define sqlite3_data_count sqlite3_api->data_count +#define sqlite3_db_handle sqlite3_api->db_handle +#define sqlite3_declare_vtab sqlite3_api->declare_vtab +#define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache +#define sqlite3_errcode sqlite3_api->errcode +#define sqlite3_errmsg sqlite3_api->errmsg +#define sqlite3_errmsg16 sqlite3_api->errmsg16 +#define sqlite3_exec sqlite3_api->exec +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_expired sqlite3_api->expired +#endif +#define sqlite3_finalize sqlite3_api->finalize +#define sqlite3_free sqlite3_api->free +#define sqlite3_free_table sqlite3_api->free_table +#define sqlite3_get_autocommit sqlite3_api->get_autocommit +#define sqlite3_get_auxdata sqlite3_api->get_auxdata +#define sqlite3_get_table sqlite3_api->get_table +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_global_recover sqlite3_api->global_recover +#endif +#define sqlite3_interrupt sqlite3_api->interruptx +#define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid +#define sqlite3_libversion sqlite3_api->libversion +#define sqlite3_libversion_number sqlite3_api->libversion_number +#define sqlite3_malloc sqlite3_api->malloc +#define sqlite3_mprintf sqlite3_api->mprintf +#define sqlite3_open sqlite3_api->open +#define sqlite3_open16 sqlite3_api->open16 +#define sqlite3_prepare sqlite3_api->prepare +#define sqlite3_prepare16 sqlite3_api->prepare16 +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_profile sqlite3_api->profile +#define sqlite3_progress_handler sqlite3_api->progress_handler +#define sqlite3_realloc sqlite3_api->realloc +#define sqlite3_reset sqlite3_api->reset +#define sqlite3_result_blob sqlite3_api->result_blob +#define sqlite3_result_double sqlite3_api->result_double +#define sqlite3_result_error sqlite3_api->result_error +#define sqlite3_result_error16 sqlite3_api->result_error16 +#define sqlite3_result_int sqlite3_api->result_int +#define sqlite3_result_int64 sqlite3_api->result_int64 +#define sqlite3_result_null sqlite3_api->result_null +#define sqlite3_result_text sqlite3_api->result_text +#define sqlite3_result_text16 sqlite3_api->result_text16 +#define sqlite3_result_text16be sqlite3_api->result_text16be +#define sqlite3_result_text16le sqlite3_api->result_text16le +#define sqlite3_result_value sqlite3_api->result_value +#define sqlite3_rollback_hook sqlite3_api->rollback_hook +#define sqlite3_set_authorizer sqlite3_api->set_authorizer +#define sqlite3_set_auxdata sqlite3_api->set_auxdata +#define sqlite3_snprintf sqlite3_api->snprintf +#define sqlite3_step sqlite3_api->step +#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata +#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup +#define sqlite3_total_changes sqlite3_api->total_changes +#define sqlite3_trace sqlite3_api->trace +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_transfer_bindings sqlite3_api->transfer_bindings +#endif +#define sqlite3_update_hook sqlite3_api->update_hook +#define sqlite3_user_data sqlite3_api->user_data +#define sqlite3_value_blob sqlite3_api->value_blob +#define sqlite3_value_bytes sqlite3_api->value_bytes +#define sqlite3_value_bytes16 sqlite3_api->value_bytes16 +#define sqlite3_value_double sqlite3_api->value_double +#define sqlite3_value_int sqlite3_api->value_int +#define sqlite3_value_int64 sqlite3_api->value_int64 +#define sqlite3_value_numeric_type sqlite3_api->value_numeric_type +#define sqlite3_value_text sqlite3_api->value_text +#define sqlite3_value_text16 sqlite3_api->value_text16 +#define sqlite3_value_text16be sqlite3_api->value_text16be +#define sqlite3_value_text16le sqlite3_api->value_text16le +#define sqlite3_value_type sqlite3_api->value_type +#define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_overload_function sqlite3_api->overload_function +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_clear_bindings sqlite3_api->clear_bindings +#define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob +#define sqlite3_blob_bytes sqlite3_api->blob_bytes +#define sqlite3_blob_close sqlite3_api->blob_close +#define sqlite3_blob_open sqlite3_api->blob_open +#define sqlite3_blob_read sqlite3_api->blob_read +#define sqlite3_blob_write sqlite3_api->blob_write +#define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2 +#define sqlite3_file_control sqlite3_api->file_control +#define sqlite3_memory_highwater sqlite3_api->memory_highwater +#define sqlite3_memory_used sqlite3_api->memory_used +#define sqlite3_mutex_alloc sqlite3_api->mutex_alloc +#define sqlite3_mutex_enter sqlite3_api->mutex_enter +#define sqlite3_mutex_free sqlite3_api->mutex_free +#define sqlite3_mutex_leave sqlite3_api->mutex_leave +#define sqlite3_mutex_try sqlite3_api->mutex_try +#define sqlite3_open_v2 sqlite3_api->open_v2 +#define sqlite3_release_memory sqlite3_api->release_memory +#define sqlite3_result_error_nomem sqlite3_api->result_error_nomem +#define sqlite3_result_error_toobig sqlite3_api->result_error_toobig +#define sqlite3_sleep sqlite3_api->sleep +#define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit +#define sqlite3_vfs_find sqlite3_api->vfs_find +#define sqlite3_vfs_register sqlite3_api->vfs_register +#define sqlite3_vfs_unregister sqlite3_api->vfs_unregister +#define sqlite3_threadsafe sqlite3_api->xthreadsafe +#define sqlite3_result_zeroblob sqlite3_api->result_zeroblob +#define sqlite3_result_error_code sqlite3_api->result_error_code +#define sqlite3_test_control sqlite3_api->test_control +#define sqlite3_randomness sqlite3_api->randomness +#define sqlite3_context_db_handle sqlite3_api->context_db_handle +#define sqlite3_extended_result_codes sqlite3_api->extended_result_codes +#define sqlite3_limit sqlite3_api->limit +#define sqlite3_next_stmt sqlite3_api->next_stmt +#define sqlite3_sql sqlite3_api->sql +#define sqlite3_status sqlite3_api->status +#define sqlite3_backup_finish sqlite3_api->backup_finish +#define sqlite3_backup_init sqlite3_api->backup_init +#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount +#define sqlite3_backup_remaining sqlite3_api->backup_remaining +#define sqlite3_backup_step sqlite3_api->backup_step +#define sqlite3_compileoption_get sqlite3_api->compileoption_get +#define sqlite3_compileoption_used sqlite3_api->compileoption_used +#define sqlite3_create_function_v2 sqlite3_api->create_function_v2 +#define sqlite3_db_config sqlite3_api->db_config +#define sqlite3_db_mutex sqlite3_api->db_mutex +#define sqlite3_db_status sqlite3_api->db_status +#define sqlite3_extended_errcode sqlite3_api->extended_errcode +#define sqlite3_log sqlite3_api->log +#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 +#define sqlite3_sourceid sqlite3_api->sourceid +#define sqlite3_stmt_status sqlite3_api->stmt_status +#define sqlite3_strnicmp sqlite3_api->strnicmp +#define sqlite3_unlock_notify sqlite3_api->unlock_notify +#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint +#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint +#define sqlite3_wal_hook sqlite3_api->wal_hook +#define sqlite3_blob_reopen sqlite3_api->blob_reopen +#define sqlite3_vtab_config sqlite3_api->vtab_config +#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict +/* Version 3.7.16 and later */ +#define sqlite3_close_v2 sqlite3_api->close_v2 +#define sqlite3_db_filename sqlite3_api->db_filename +#define sqlite3_db_readonly sqlite3_api->db_readonly +#define sqlite3_db_release_memory sqlite3_api->db_release_memory +#define sqlite3_errstr sqlite3_api->errstr +#define sqlite3_stmt_busy sqlite3_api->stmt_busy +#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly +#define sqlite3_stricmp sqlite3_api->stricmp +#define sqlite3_uri_boolean sqlite3_api->uri_boolean +#define sqlite3_uri_int64 sqlite3_api->uri_int64 +#define sqlite3_uri_parameter sqlite3_api->uri_parameter +#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 +#endif /* SQLITE_CORE */ + +#ifndef SQLITE_CORE + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +# define SQLITE_EXTENSION_INIT3 \ + extern const sqlite3_api_routines *sqlite3_api; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +# define SQLITE_EXTENSION_INIT3 /*no-op*/ +#endif + +#endif /* _SQLITE3EXT_H_ */ + +/************** End of sqlite3ext.h ******************************************/ +/************** Continuing where we left off in loadext.c ********************/ +/* #include */ + +#ifndef SQLITE_OMIT_LOAD_EXTENSION + +/* +** Some API routines are omitted when various features are +** excluded from a build of SQLite. Substitute a NULL pointer +** for any missing APIs. +*/ +#ifndef SQLITE_ENABLE_COLUMN_METADATA +# define sqlite3_column_database_name 0 +# define sqlite3_column_database_name16 0 +# define sqlite3_column_table_name 0 +# define sqlite3_column_table_name16 0 +# define sqlite3_column_origin_name 0 +# define sqlite3_column_origin_name16 0 +# define sqlite3_table_column_metadata 0 +#endif + +#ifdef SQLITE_OMIT_AUTHORIZATION +# define sqlite3_set_authorizer 0 +#endif + +#ifdef SQLITE_OMIT_UTF16 +# define sqlite3_bind_text16 0 +# define sqlite3_collation_needed16 0 +# define sqlite3_column_decltype16 0 +# define sqlite3_column_name16 0 +# define sqlite3_column_text16 0 +# define sqlite3_complete16 0 +# define sqlite3_create_collation16 0 +# define sqlite3_create_function16 0 +# define sqlite3_errmsg16 0 +# define sqlite3_open16 0 +# define sqlite3_prepare16 0 +# define sqlite3_prepare16_v2 0 +# define sqlite3_result_error16 0 +# define sqlite3_result_text16 0 +# define sqlite3_result_text16be 0 +# define sqlite3_result_text16le 0 +# define sqlite3_value_text16 0 +# define sqlite3_value_text16be 0 +# define sqlite3_value_text16le 0 +# define sqlite3_column_database_name16 0 +# define sqlite3_column_table_name16 0 +# define sqlite3_column_origin_name16 0 +#endif + +#ifdef SQLITE_OMIT_COMPLETE +# define sqlite3_complete 0 +# define sqlite3_complete16 0 +#endif + +#ifdef SQLITE_OMIT_DECLTYPE +# define sqlite3_column_decltype16 0 +# define sqlite3_column_decltype 0 +#endif + +#ifdef SQLITE_OMIT_PROGRESS_CALLBACK +# define sqlite3_progress_handler 0 +#endif + +#ifdef SQLITE_OMIT_VIRTUALTABLE +# define sqlite3_create_module 0 +# define sqlite3_create_module_v2 0 +# define sqlite3_declare_vtab 0 +# define sqlite3_vtab_config 0 +# define sqlite3_vtab_on_conflict 0 +#endif + +#ifdef SQLITE_OMIT_SHARED_CACHE +# define sqlite3_enable_shared_cache 0 +#endif + +#ifdef SQLITE_OMIT_TRACE +# define sqlite3_profile 0 +# define sqlite3_trace 0 +#endif + +#ifdef SQLITE_OMIT_GET_TABLE +# define sqlite3_free_table 0 +# define sqlite3_get_table 0 +#endif + +#ifdef SQLITE_OMIT_INCRBLOB +#define sqlite3_bind_zeroblob 0 +#define sqlite3_blob_bytes 0 +#define sqlite3_blob_close 0 +#define sqlite3_blob_open 0 +#define sqlite3_blob_read 0 +#define sqlite3_blob_write 0 +#define sqlite3_blob_reopen 0 +#endif + +/* +** The following structure contains pointers to all SQLite API routines. +** A pointer to this structure is passed into extensions when they are +** loaded so that the extension can make calls back into the SQLite +** library. +** +** When adding new APIs, add them to the bottom of this structure +** in order to preserve backwards compatibility. +** +** Extensions that use newer APIs should first call the +** sqlite3_libversion_number() to make sure that the API they +** intend to use is supported by the library. Extensions should +** also check to make sure that the pointer to the function is +** not NULL before calling it. +*/ +static const sqlite3_api_routines sqlite3Apis = { + sqlite3_aggregate_context, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_aggregate_count, +#else + 0, +#endif + sqlite3_bind_blob, + sqlite3_bind_double, + sqlite3_bind_int, + sqlite3_bind_int64, + sqlite3_bind_null, + sqlite3_bind_parameter_count, + sqlite3_bind_parameter_index, + sqlite3_bind_parameter_name, + sqlite3_bind_text, + sqlite3_bind_text16, + sqlite3_bind_value, + sqlite3_busy_handler, + sqlite3_busy_timeout, + sqlite3_changes, + sqlite3_close, + sqlite3_collation_needed, + sqlite3_collation_needed16, + sqlite3_column_blob, + sqlite3_column_bytes, + sqlite3_column_bytes16, + sqlite3_column_count, + sqlite3_column_database_name, + sqlite3_column_database_name16, + sqlite3_column_decltype, + sqlite3_column_decltype16, + sqlite3_column_double, + sqlite3_column_int, + sqlite3_column_int64, + sqlite3_column_name, + sqlite3_column_name16, + sqlite3_column_origin_name, + sqlite3_column_origin_name16, + sqlite3_column_table_name, + sqlite3_column_table_name16, + sqlite3_column_text, + sqlite3_column_text16, + sqlite3_column_type, + sqlite3_column_value, + sqlite3_commit_hook, + sqlite3_complete, + sqlite3_complete16, + sqlite3_create_collation, + sqlite3_create_collation16, + sqlite3_create_function, + sqlite3_create_function16, + sqlite3_create_module, + sqlite3_data_count, + sqlite3_db_handle, + sqlite3_declare_vtab, + sqlite3_enable_shared_cache, + sqlite3_errcode, + sqlite3_errmsg, + sqlite3_errmsg16, + sqlite3_exec, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_expired, +#else + 0, +#endif + sqlite3_finalize, + sqlite3_free, + sqlite3_free_table, + sqlite3_get_autocommit, + sqlite3_get_auxdata, + sqlite3_get_table, + 0, /* Was sqlite3_global_recover(), but that function is deprecated */ + sqlite3_interrupt, + sqlite3_last_insert_rowid, + sqlite3_libversion, + sqlite3_libversion_number, + sqlite3_malloc, + sqlite3_mprintf, + sqlite3_open, + sqlite3_open16, + sqlite3_prepare, + sqlite3_prepare16, + sqlite3_profile, + sqlite3_progress_handler, + sqlite3_realloc, + sqlite3_reset, + sqlite3_result_blob, + sqlite3_result_double, + sqlite3_result_error, + sqlite3_result_error16, + sqlite3_result_int, + sqlite3_result_int64, + sqlite3_result_null, + sqlite3_result_text, + sqlite3_result_text16, + sqlite3_result_text16be, + sqlite3_result_text16le, + sqlite3_result_value, + sqlite3_rollback_hook, + sqlite3_set_authorizer, + sqlite3_set_auxdata, + sqlite3_snprintf, + sqlite3_step, + sqlite3_table_column_metadata, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_thread_cleanup, +#else + 0, +#endif + sqlite3_total_changes, + sqlite3_trace, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_transfer_bindings, +#else + 0, +#endif + sqlite3_update_hook, + sqlite3_user_data, + sqlite3_value_blob, + sqlite3_value_bytes, + sqlite3_value_bytes16, + sqlite3_value_double, + sqlite3_value_int, + sqlite3_value_int64, + sqlite3_value_numeric_type, + sqlite3_value_text, + sqlite3_value_text16, + sqlite3_value_text16be, + sqlite3_value_text16le, + sqlite3_value_type, + sqlite3_vmprintf, + /* + ** The original API set ends here. All extensions can call any + ** of the APIs above provided that the pointer is not NULL. But + ** before calling APIs that follow, extension should check the + ** sqlite3_libversion_number() to make sure they are dealing with + ** a library that is new enough to support that API. + ************************************************************************* + */ + sqlite3_overload_function, + + /* + ** Added after 3.3.13 + */ + sqlite3_prepare_v2, + sqlite3_prepare16_v2, + sqlite3_clear_bindings, + + /* + ** Added for 3.4.1 + */ + sqlite3_create_module_v2, + + /* + ** Added for 3.5.0 + */ + sqlite3_bind_zeroblob, + sqlite3_blob_bytes, + sqlite3_blob_close, + sqlite3_blob_open, + sqlite3_blob_read, + sqlite3_blob_write, + sqlite3_create_collation_v2, + sqlite3_file_control, + sqlite3_memory_highwater, + sqlite3_memory_used, +#ifdef SQLITE_MUTEX_OMIT + 0, + 0, + 0, + 0, + 0, +#else + sqlite3_mutex_alloc, + sqlite3_mutex_enter, + sqlite3_mutex_free, + sqlite3_mutex_leave, + sqlite3_mutex_try, +#endif + sqlite3_open_v2, + sqlite3_release_memory, + sqlite3_result_error_nomem, + sqlite3_result_error_toobig, + sqlite3_sleep, + sqlite3_soft_heap_limit, + sqlite3_vfs_find, + sqlite3_vfs_register, + sqlite3_vfs_unregister, + + /* + ** Added for 3.5.8 + */ + sqlite3_threadsafe, + sqlite3_result_zeroblob, + sqlite3_result_error_code, + sqlite3_test_control, + sqlite3_randomness, + sqlite3_context_db_handle, + + /* + ** Added for 3.6.0 + */ + sqlite3_extended_result_codes, + sqlite3_limit, + sqlite3_next_stmt, + sqlite3_sql, + sqlite3_status, + + /* + ** Added for 3.7.4 + */ + sqlite3_backup_finish, + sqlite3_backup_init, + sqlite3_backup_pagecount, + sqlite3_backup_remaining, + sqlite3_backup_step, +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + sqlite3_compileoption_get, + sqlite3_compileoption_used, +#else + 0, + 0, +#endif + sqlite3_create_function_v2, + sqlite3_db_config, + sqlite3_db_mutex, + sqlite3_db_status, + sqlite3_extended_errcode, + sqlite3_log, + sqlite3_soft_heap_limit64, + sqlite3_sourceid, + sqlite3_stmt_status, + sqlite3_strnicmp, +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + sqlite3_unlock_notify, +#else + 0, +#endif +#ifndef SQLITE_OMIT_WAL + sqlite3_wal_autocheckpoint, + sqlite3_wal_checkpoint, + sqlite3_wal_hook, +#else + 0, + 0, + 0, +#endif + sqlite3_blob_reopen, + sqlite3_vtab_config, + sqlite3_vtab_on_conflict, + sqlite3_close_v2, + sqlite3_db_filename, + sqlite3_db_readonly, + sqlite3_db_release_memory, + sqlite3_errstr, + sqlite3_stmt_busy, + sqlite3_stmt_readonly, + sqlite3_stricmp, + sqlite3_uri_boolean, + sqlite3_uri_int64, + sqlite3_uri_parameter, + sqlite3_vsnprintf, + sqlite3_wal_checkpoint_v2 +}; + +/* +** Attempt to load an SQLite extension library contained in the file +** zFile. The entry point is zProc. zProc may be 0 in which case a +** default entry point name (sqlite3_extension_init) is used. Use +** of the default name is recommended. +** +** Return SQLITE_OK on success and SQLITE_ERROR if something goes wrong. +** +** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with +** error message text. The calling function should free this memory +** by calling sqlite3DbFree(db, ). +*/ +static int sqlite3LoadExtension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +){ + sqlite3_vfs *pVfs = db->pVfs; + void *handle; + int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + char *zErrmsg = 0; + const char *zEntry; + char *zAltEntry = 0; + void **aHandle; + int nMsg = 300 + sqlite3Strlen30(zFile); + int ii; + + /* Shared library endings to try if zFile cannot be loaded as written */ + static const char *azEndings[] = { +#if SQLITE_OS_WIN + "dll" +#elif defined(__APPLE__) + "dylib" +#else + "so" +#endif + }; + + + if( pzErrMsg ) *pzErrMsg = 0; + + /* Ticket #1863. To avoid a creating security problems for older + ** applications that relink against newer versions of SQLite, the + ** ability to run load_extension is turned off by default. One + ** must call sqlite3_enable_load_extension() to turn on extension + ** loading. Otherwise you get the following error. + */ + if( (db->flags & SQLITE_LoadExtension)==0 ){ + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("not authorized"); + } + return SQLITE_ERROR; + } + + zEntry = zProc ? zProc : "sqlite3_extension_init"; + + handle = sqlite3OsDlOpen(pVfs, zFile); +#if SQLITE_OS_UNIX || SQLITE_OS_WIN + for(ii=0; ii sqlite3_example_init + ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + */ + if( xInit==0 && zProc==0 ){ + int iFile, iEntry, c; + int ncFile = sqlite3Strlen30(zFile); + zAltEntry = sqlite3_malloc(ncFile+30); + if( zAltEntry==0 ){ + sqlite3OsDlClose(pVfs, handle); + return SQLITE_NOMEM; + } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + sqlite3OsDlSym(pVfs, handle, zEntry); + } + if( xInit==0 ){ + if( pzErrMsg ){ + nMsg += sqlite3Strlen30(zEntry); + *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); + if( zErrmsg ){ + sqlite3_snprintf(nMsg, zErrmsg, + "no entry point [%s] in shared library [%s]", zEntry, zFile); + sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); + } + } + sqlite3OsDlClose(pVfs, handle); + sqlite3_free(zAltEntry); + return SQLITE_ERROR; + } + sqlite3_free(zAltEntry); + if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); + } + sqlite3_free(zErrmsg); + sqlite3OsDlClose(pVfs, handle); + return SQLITE_ERROR; + } + + /* Append the new shared library handle to the db->aExtension array. */ + aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); + if( aHandle==0 ){ + return SQLITE_NOMEM; + } + if( db->nExtension>0 ){ + memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); + } + sqlite3DbFree(db, db->aExtension); + db->aExtension = aHandle; + + db->aExtension[db->nExtension++] = handle; + return SQLITE_OK; +} +SQLITE_API int sqlite3_load_extension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +){ + int rc; + sqlite3_mutex_enter(db->mutex); + rc = sqlite3LoadExtension(db, zFile, zProc, pzErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Call this routine when the database connection is closing in order +** to clean up loaded extensions +*/ +SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3 *db){ + int i; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inExtension; i++){ + sqlite3OsDlClose(db->pVfs, db->aExtension[i]); + } + sqlite3DbFree(db, db->aExtension); +} + +/* +** Enable or disable extension loading. Extension loading is disabled by +** default so as not to open security holes in older applications. +*/ +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ + sqlite3_mutex_enter(db->mutex); + if( onoff ){ + db->flags |= SQLITE_LoadExtension; + }else{ + db->flags &= ~SQLITE_LoadExtension; + } + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ + +/* +** The auto-extension code added regardless of whether or not extension +** loading is supported. We need a dummy sqlite3Apis pointer for that +** code if regular extension loading is not available. This is that +** dummy pointer. +*/ +#ifdef SQLITE_OMIT_LOAD_EXTENSION +static const sqlite3_api_routines sqlite3Apis = { 0 }; +#endif + + +/* +** The following object holds the list of automatically loaded +** extensions. +** +** This list is shared across threads. The SQLITE_MUTEX_STATIC_MASTER +** mutex must be held while accessing this list. +*/ +typedef struct sqlite3AutoExtList sqlite3AutoExtList; +static SQLITE_WSD struct sqlite3AutoExtList { + int nExt; /* Number of entries in aExt[] */ + void (**aExt)(void); /* Pointers to the extension init functions */ +} sqlite3Autoext = { 0, 0 }; + +/* The "wsdAutoext" macro will resolve to the autoextension +** state vector. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdStat can refer directly +** to the "sqlite3Autoext" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdAutoextInit \ + sqlite3AutoExtList *x = &GLOBAL(sqlite3AutoExtList,sqlite3Autoext) +# define wsdAutoext x[0] +#else +# define wsdAutoextInit +# define wsdAutoext sqlite3Autoext +#endif + + +/* +** Register a statically linked extension that is automatically +** loaded by every new database connection. +*/ +SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ){ + return rc; + }else +#endif + { + int i; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + wsdAutoextInit; + sqlite3_mutex_enter(mutex); + for(i=0; i=0; i--){ + if( wsdAutoext.aExt[i]==xInit ){ + wsdAutoext.nExt--; + wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt]; + n++; + break; + } + } + sqlite3_mutex_leave(mutex); + return n; +} + +/* +** Reset the automatic extension loading mechanism. +*/ +SQLITE_API void sqlite3_reset_auto_extension(void){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize()==SQLITE_OK ) +#endif + { +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + wsdAutoextInit; + sqlite3_mutex_enter(mutex); + sqlite3_free(wsdAutoext.aExt); + wsdAutoext.aExt = 0; + wsdAutoext.nExt = 0; + sqlite3_mutex_leave(mutex); + } +} + +/* +** Load all automatic extensions. +** +** If anything goes wrong, set an error in the database connection. +*/ +SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ + int i; + int go = 1; + int rc; + int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + + wsdAutoextInit; + if( wsdAutoext.nExt==0 ){ + /* Common case: early out without every having to acquire a mutex */ + return; + } + for(i=0; go; i++){ + char *zErrmsg; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + sqlite3_mutex_enter(mutex); + if( i>=wsdAutoext.nExt ){ + xInit = 0; + go = 0; + }else{ + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + wsdAutoext.aExt[i]; + } + sqlite3_mutex_leave(mutex); + zErrmsg = 0; + if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ + sqlite3Error(db, rc, + "automatic extension loading failed: %s", zErrmsg); + go = 0; + } + sqlite3_free(zErrmsg); + } +} + +/************** End of loadext.c *********************************************/ +/************** Begin file pragma.c ******************************************/ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the PRAGMA command. +*/ + +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif + +/*************************************************************************** +** The next block of code, including the PragTyp_XXXX macro definitions and +** the aPragmaName[] object is composed of generated code. DO NOT EDIT. +** +** To add new pragmas, edit the code in ../tool/mkpragmatab.tcl and rerun +** that script. Then copy/paste the output in place of the following: +*/ +#define PragTyp_HEADER_VALUE 0 +#define PragTyp_AUTO_VACUUM 1 +#define PragTyp_FLAG 2 +#define PragTyp_BUSY_TIMEOUT 3 +#define PragTyp_CACHE_SIZE 4 +#define PragTyp_CASE_SENSITIVE_LIKE 5 +#define PragTyp_COLLATION_LIST 6 +#define PragTyp_COMPILE_OPTIONS 7 +#define PragTyp_DATA_STORE_DIRECTORY 8 +#define PragTyp_DATABASE_LIST 9 +#define PragTyp_DEFAULT_CACHE_SIZE 10 +#define PragTyp_ENCODING 11 +#define PragTyp_FOREIGN_KEY_CHECK 12 +#define PragTyp_FOREIGN_KEY_LIST 13 +#define PragTyp_INCREMENTAL_VACUUM 14 +#define PragTyp_INDEX_INFO 15 +#define PragTyp_INDEX_LIST 16 +#define PragTyp_INTEGRITY_CHECK 17 +#define PragTyp_JOURNAL_MODE 18 +#define PragTyp_JOURNAL_SIZE_LIMIT 19 +#define PragTyp_LOCK_PROXY_FILE 20 +#define PragTyp_LOCKING_MODE 21 +#define PragTyp_PAGE_COUNT 22 +#define PragTyp_MMAP_SIZE 23 +#define PragTyp_PAGE_SIZE 24 +#define PragTyp_SECURE_DELETE 25 +#define PragTyp_SHRINK_MEMORY 26 +#define PragTyp_SOFT_HEAP_LIMIT 27 +#define PragTyp_STATS 28 +#define PragTyp_SYNCHRONOUS 29 +#define PragTyp_TABLE_INFO 30 +#define PragTyp_TEMP_STORE 31 +#define PragTyp_TEMP_STORE_DIRECTORY 32 +#define PragTyp_WAL_AUTOCHECKPOINT 33 +#define PragTyp_WAL_CHECKPOINT 34 +#define PragTyp_ACTIVATE_EXTENSIONS 35 +#define PragTyp_HEXKEY 36 +#define PragTyp_KEY 37 +#define PragTyp_REKEY 38 +#define PragTyp_LOCK_STATUS 39 +#define PragTyp_PARSER_TRACE 40 +#define PragFlag_NeedSchema 0x01 +static const struct sPragmaNames { + const char *const zName; /* Name of pragma */ + u8 ePragTyp; /* PragTyp_XXX value */ + u8 mPragFlag; /* Zero or more PragFlag_XXX values */ + u32 iArg; /* Extra argument */ +} aPragmaNames[] = { +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) + { /* zName: */ "activate_extensions", + /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "application_id", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_AUTOVACUUM) + { /* zName: */ "auto_vacuum", + /* ePragTyp: */ PragTyp_AUTO_VACUUM, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_AUTOMATIC_INDEX) + { /* zName: */ "automatic_index", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_AutoIndex }, +#endif +#endif + { /* zName: */ "busy_timeout", + /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "cache_size", + /* ePragTyp: */ PragTyp_CACHE_SIZE, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "cache_spill", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_CacheSpill }, +#endif + { /* zName: */ "case_sensitive_like", + /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "checkpoint_fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_CkptFullFSync }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "collation_list", + /* ePragTyp: */ PragTyp_COLLATION_LIST, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) + { /* zName: */ "compile_options", + /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "count_changes", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_CountRows }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN + { /* zName: */ "data_store_directory", + /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "database_list", + /* ePragTyp: */ PragTyp_DATABASE_LIST, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) + { /* zName: */ "default_cache_size", + /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + { /* zName: */ "defer_foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_DeferFKs }, +#endif +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "empty_result_callbacks", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_NullCallback }, +#endif +#if !defined(SQLITE_OMIT_UTF16) + { /* zName: */ "encoding", + /* ePragTyp: */ PragTyp_ENCODING, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + { /* zName: */ "foreign_key_check", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FOREIGN_KEY) + { /* zName: */ "foreign_key_list", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + { /* zName: */ "foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_ForeignKeys }, +#endif +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "freelist_count", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "full_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_FullColNames }, + { /* zName: */ "fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_FullFSync }, +#endif +#if defined(SQLITE_HAS_CODEC) + { /* zName: */ "hexkey", + /* ePragTyp: */ PragTyp_HEXKEY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "hexrekey", + /* ePragTyp: */ PragTyp_HEXKEY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_CHECK) + { /* zName: */ "ignore_check_constraints", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_IgnoreChecks }, +#endif +#endif +#if !defined(SQLITE_OMIT_AUTOVACUUM) + { /* zName: */ "incremental_vacuum", + /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "index_info", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, + { /* zName: */ "index_list", + /* ePragTyp: */ PragTyp_INDEX_LIST, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) + { /* zName: */ "integrity_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "journal_mode", + /* ePragTyp: */ PragTyp_JOURNAL_MODE, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, + { /* zName: */ "journal_size_limit", + /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if defined(SQLITE_HAS_CODEC) + { /* zName: */ "key", + /* ePragTyp: */ PragTyp_KEY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "legacy_file_format", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_LegacyFileFmt }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE + { /* zName: */ "lock_proxy_file", + /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + { /* zName: */ "lock_status", + /* ePragTyp: */ PragTyp_LOCK_STATUS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "locking_mode", + /* ePragTyp: */ PragTyp_LOCKING_MODE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "max_page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, + { /* zName: */ "mmap_size", + /* ePragTyp: */ PragTyp_MMAP_SIZE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, + { /* zName: */ "page_size", + /* ePragTyp: */ PragTyp_PAGE_SIZE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if defined(SQLITE_DEBUG) + { /* zName: */ "parser_trace", + /* ePragTyp: */ PragTyp_PARSER_TRACE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "query_only", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_QueryOnly }, +#endif +#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) + { /* zName: */ "quick_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "read_uncommitted", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_ReadUncommitted }, + { /* zName: */ "recursive_triggers", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_RecTriggers }, +#endif +#if defined(SQLITE_HAS_CODEC) + { /* zName: */ "rekey", + /* ePragTyp: */ PragTyp_REKEY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "reverse_unordered_selects", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_ReverseOrder }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "schema_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "secure_delete", + /* ePragTyp: */ PragTyp_SECURE_DELETE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "short_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_ShortColNames }, +#endif + { /* zName: */ "shrink_memory", + /* ePragTyp: */ PragTyp_SHRINK_MEMORY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "soft_heap_limit", + /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + { /* zName: */ "sql_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_SqlTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "stats", + /* ePragTyp: */ PragTyp_STATS, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "synchronous", + /* ePragTyp: */ PragTyp_SYNCHRONOUS, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "table_info", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "temp_store", + /* ePragTyp: */ PragTyp_TEMP_STORE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "temp_store_directory", + /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "user_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + { /* zName: */ "vdbe_addoptrace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_VdbeAddopTrace }, + { /* zName: */ "vdbe_debug", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, + { /* zName: */ "vdbe_eqp", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_VdbeEQP }, + { /* zName: */ "vdbe_listing", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_VdbeListing }, + { /* zName: */ "vdbe_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_VdbeTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_WAL) + { /* zName: */ "wal_autocheckpoint", + /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "wal_checkpoint", + /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "writable_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, +#endif +}; +/* Number of pragmas: 56 on by default, 69 total. */ +/* End of the automatically generated pragma table. +***************************************************************************/ + +/* +** Interpret the given string as a safety level. Return 0 for OFF, +** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or +** unrecognized string argument. The FULL option is disallowed +** if the omitFull parameter it 1. +** +** Note that the values returned are one less that the values that +** should be passed into sqlite3BtreeSetSafetyLevel(). The is done +** to support legacy SQL code. The safety level used to be boolean +** and older scripts may have used numbers 0 for OFF and 1 for ON. +*/ +static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ + /* 123456789 123456789 */ + static const char zText[] = "onoffalseyestruefull"; + static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; + static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 4}; + static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; + int i, n; + if( sqlite3Isdigit(*z) ){ + return (u8)sqlite3Atoi(z); + } + n = sqlite3Strlen30(z); + for(i=0; i=0&&i<=2)?i:0); +} +#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* +** Interpret the given string as a temp db location. Return 1 for file +** backed temporary databases, 2 for the Red-Black tree in memory database +** and 0 to use the compile-time default. +*/ +static int getTempStore(const char *z){ + if( z[0]>='0' && z[0]<='2' ){ + return z[0] - '0'; + }else if( sqlite3StrICmp(z, "file")==0 ){ + return 1; + }else if( sqlite3StrICmp(z, "memory")==0 ){ + return 2; + }else{ + return 0; + } +} +#endif /* SQLITE_PAGER_PRAGMAS */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* +** Invalidate temp storage, either when the temp storage is changed +** from default, or when 'file' and the temp_store_directory has changed +*/ +static int invalidateTempStorage(Parse *pParse){ + sqlite3 *db = pParse->db; + if( db->aDb[1].pBt!=0 ){ + if( !db->autoCommit || sqlite3BtreeIsInReadTrans(db->aDb[1].pBt) ){ + sqlite3ErrorMsg(pParse, "temporary storage cannot be changed " + "from within a transaction"); + return SQLITE_ERROR; + } + sqlite3BtreeClose(db->aDb[1].pBt); + db->aDb[1].pBt = 0; + sqlite3ResetAllSchemasOfConnection(db); + } + return SQLITE_OK; +} +#endif /* SQLITE_PAGER_PRAGMAS */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* +** If the TEMP database is open, close it and mark the database schema +** as needing reloading. This must be done when using the SQLITE_TEMP_STORE +** or DEFAULT_TEMP_STORE pragmas. +*/ +static int changeTempStorage(Parse *pParse, const char *zStorageType){ + int ts = getTempStore(zStorageType); + sqlite3 *db = pParse->db; + if( db->temp_store==ts ) return SQLITE_OK; + if( invalidateTempStorage( pParse ) != SQLITE_OK ){ + return SQLITE_ERROR; + } + db->temp_store = (u8)ts; + return SQLITE_OK; +} +#endif /* SQLITE_PAGER_PRAGMAS */ + +/* +** Generate code to return a single integer value. +*/ +static void returnSingleInt(Parse *pParse, const char *zLabel, i64 value){ + Vdbe *v = sqlite3GetVdbe(pParse); + int mem = ++pParse->nMem; + i64 *pI64 = sqlite3DbMallocRaw(pParse->db, sizeof(value)); + if( pI64 ){ + memcpy(pI64, &value, sizeof(value)); + } + sqlite3VdbeAddOp4(v, OP_Int64, 0, mem, 0, (char*)pI64, P4_INT64); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); +} + + +/* +** Set the safety_level and pager flags for pager iDb. Or if iDb<0 +** set these values for all pagers. +*/ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +static void setAllPagerFlags(sqlite3 *db){ + if( db->autoCommit ){ + Db *pDb = db->aDb; + int n = db->nDb; + assert( SQLITE_FullFSync==PAGER_FULLFSYNC ); + assert( SQLITE_CkptFullFSync==PAGER_CKPT_FULLFSYNC ); + assert( SQLITE_CacheSpill==PAGER_CACHESPILL ); + assert( (PAGER_FULLFSYNC | PAGER_CKPT_FULLFSYNC | PAGER_CACHESPILL) + == PAGER_FLAGS_MASK ); + assert( (pDb->safety_level & PAGER_SYNCHRONOUS_MASK)==pDb->safety_level ); + while( (n--) > 0 ){ + if( pDb->pBt ){ + sqlite3BtreeSetPagerFlags(pDb->pBt, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK) ); + } + pDb++; + } + } +} +#else +# define setAllPagerFlags(X) /* no-op */ +#endif + + +/* +** Return a human-readable name for a constraint resolution action. +*/ +#ifndef SQLITE_OMIT_FOREIGN_KEY +static const char *actionName(u8 action){ + const char *zName; + switch( action ){ + case OE_SetNull: zName = "SET NULL"; break; + case OE_SetDflt: zName = "SET DEFAULT"; break; + case OE_Cascade: zName = "CASCADE"; break; + case OE_Restrict: zName = "RESTRICT"; break; + default: zName = "NO ACTION"; + assert( action==OE_None ); break; + } + return zName; +} +#endif + + +/* +** Parameter eMode must be one of the PAGER_JOURNALMODE_XXX constants +** defined in pager.h. This function returns the associated lowercase +** journal-mode name. +*/ +SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ + static char * const azModeName[] = { + "delete", "persist", "off", "truncate", "memory" +#ifndef SQLITE_OMIT_WAL + , "wal" +#endif + }; + assert( PAGER_JOURNALMODE_DELETE==0 ); + assert( PAGER_JOURNALMODE_PERSIST==1 ); + assert( PAGER_JOURNALMODE_OFF==2 ); + assert( PAGER_JOURNALMODE_TRUNCATE==3 ); + assert( PAGER_JOURNALMODE_MEMORY==4 ); + assert( PAGER_JOURNALMODE_WAL==5 ); + assert( eMode>=0 && eMode<=ArraySize(azModeName) ); + + if( eMode==ArraySize(azModeName) ) return 0; + return azModeName[eMode]; +} + +/* +** Process a pragma statement. +** +** Pragmas are of this form: +** +** PRAGMA [database.]id [= value] +** +** The identifier might also be a string. The value is a string, and +** identifier, or a number. If minusFlag is true, then the value is +** a number that was preceded by a minus sign. +** +** If the left side is "database.id" then pId1 is the database name +** and pId2 is the id. If the left side is just "id" then pId1 is the +** id and pId2 is any empty string. +*/ +SQLITE_PRIVATE void sqlite3Pragma( + Parse *pParse, + Token *pId1, /* First part of [database.]id field */ + Token *pId2, /* Second part of [database.]id field, or NULL */ + Token *pValue, /* Token for , or NULL */ + int minusFlag /* True if a '-' sign preceded */ +){ + char *zLeft = 0; /* Nul-terminated UTF-8 string */ + char *zRight = 0; /* Nul-terminated UTF-8 string , or NULL */ + const char *zDb = 0; /* The database name */ + Token *pId; /* Pointer to token */ + char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ + int iDb; /* Database index for */ + int lwr, upr, mid; /* Binary search bounds */ + int rc; /* return value form SQLITE_FCNTL_PRAGMA */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* The specific database being pragmaed */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC + extern int sqlcipher_codec_pragma(sqlite3*, int, Parse *, const char *, const char *); +#endif +/* END SQLCIPHER */ + + + if( v==0 ) return; + sqlite3VdbeRunOnlyOnce(v); + pParse->nMem = 2; + + /* Interpret the [database.] part of the pragma statement. iDb is the + ** index of the database this pragma is being applied to in db.aDb[]. */ + iDb = sqlite3TwoPartName(pParse, pId1, pId2, &pId); + if( iDb<0 ) return; + pDb = &db->aDb[iDb]; + + /* If the temp database has been explicitly named as part of the + ** pragma, make sure it is open. + */ + if( iDb==1 && sqlite3OpenTempDatabase(pParse) ){ + return; + } + + zLeft = sqlite3NameFromToken(db, pId); + if( !zLeft ) return; + if( minusFlag ){ + zRight = sqlite3MPrintf(db, "-%T", pValue); + }else{ + zRight = sqlite3NameFromToken(db, pValue); + } + + assert( pId2 ); + zDb = pId2->n>0 ? pDb->zName : 0; + if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ + goto pragma_out; + } + + /* Send an SQLITE_FCNTL_PRAGMA file-control to the underlying VFS + ** connection. If it returns SQLITE_OK, then assume that the VFS + ** handled the pragma and generate a no-op prepared statement. + */ + aFcntl[0] = 0; + aFcntl[1] = zLeft; + aFcntl[2] = zRight; + aFcntl[3] = 0; + db->busyHandler.nBusy = 0; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + if( rc==SQLITE_OK ){ + if( aFcntl[0] ){ + int mem = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_String8, 0, mem, 0, aFcntl[0], 0); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); + sqlite3_free(aFcntl[0]); + } + goto pragma_out; + } + if( rc!=SQLITE_NOTFOUND ){ + if( aFcntl[0] ){ + sqlite3ErrorMsg(pParse, "%s", aFcntl[0]); + sqlite3_free(aFcntl[0]); + } + pParse->nErr++; + pParse->rc = rc; + + goto pragma_out; + } + +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC + if(sqlcipher_codec_pragma(db, iDb, pParse, zLeft, zRight)) { + /* sqlcipher_codec_pragma executes internal */ + goto pragma_out; + } +#endif +/* END SQLCIPHER */ + + /* Locate the pragma in the lookup table */ + lwr = 0; + upr = ArraySize(aPragmaNames)-1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + rc = sqlite3_stricmp(zLeft, aPragmaNames[mid].zName); + if( rc==0 ) break; + if( rc<0 ){ + upr = mid - 1; + }else{ + lwr = mid + 1; + } + } + if( lwr>upr ) goto pragma_out; + + /* Make sure the database schema is loaded if the pragma requires that */ + if( (aPragmaNames[mid].mPragFlag & PragFlag_NeedSchema)!=0 ){ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + } + + /* Jump to the appropriate pragma handler */ + switch( aPragmaNames[mid].ePragTyp ){ + +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) + /* + ** PRAGMA [database.]default_cache_size + ** PRAGMA [database.]default_cache_size=N + ** + ** The first form reports the current persistent setting for the + ** page cache size. The value returned is the maximum number of + ** pages in the page cache. The second form sets both the current + ** page cache size value and the persistent page cache size value + ** stored in the database file. + ** + ** Older versions of SQLite would set the default cache size to a + ** negative number to indicate synchronous=OFF. These days, synchronous + ** is always on by default regardless of the sign of the default cache + ** size. But continue to take the absolute value of the default cache + ** size of historical compatibility. + */ + case PragTyp_DEFAULT_CACHE_SIZE: { + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList getCacheSize[] = { + { OP_Transaction, 0, 0, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ + { OP_IfPos, 1, 8, 0}, + { OP_Integer, 0, 2, 0}, + { OP_Subtract, 1, 2, 1}, + { OP_IfPos, 1, 8, 0}, + { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, + { OP_ResultRow, 1, 1, 0}, + }; + int addr; + sqlite3VdbeUsesBtree(v, iDb); + if( !zRight ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); + pParse->nMem += 2; + addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn); + sqlite3VdbeChangeP1(v, addr, iDb); + sqlite3VdbeChangeP1(v, addr+1, iDb); + sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); + }else{ + int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, size, 1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + break; + } +#endif /* !SQLITE_OMIT_PAGER_PRAGMAS && !SQLITE_OMIT_DEPRECATED */ + +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + /* + ** PRAGMA [database.]page_size + ** PRAGMA [database.]page_size=N + ** + ** The first form reports the current setting for the + ** database page size in bytes. The second form sets the + ** database page size value. The value can only be set if + ** the database has not yet been created. + */ + case PragTyp_PAGE_SIZE: { + Btree *pBt = pDb->pBt; + assert( pBt!=0 ); + if( !zRight ){ + int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; + returnSingleInt(pParse, "page_size", size); + }else{ + /* Malloc may fail when setting the page-size, as there is an internal + ** buffer that the pager module resizes using sqlite3_realloc(). + */ + db->nextPagesize = sqlite3Atoi(zRight); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,-1,0) ){ + db->mallocFailed = 1; + } + } + break; + } + + /* + ** PRAGMA [database.]secure_delete + ** PRAGMA [database.]secure_delete=ON/OFF + ** + ** The first form reports the current setting for the + ** secure_delete flag. The second form changes the secure_delete + ** flag setting and reports thenew value. + */ + case PragTyp_SECURE_DELETE: { + Btree *pBt = pDb->pBt; + int b = -1; + assert( pBt!=0 ); + if( zRight ){ + b = sqlite3GetBoolean(zRight, 0); + } + if( pId2->n==0 && b>=0 ){ + int ii; + for(ii=0; iinDb; ii++){ + sqlite3BtreeSecureDelete(db->aDb[ii].pBt, b); + } + } + b = sqlite3BtreeSecureDelete(pBt, b); + returnSingleInt(pParse, "secure_delete", b); + break; + } + + /* + ** PRAGMA [database.]max_page_count + ** PRAGMA [database.]max_page_count=N + ** + ** The first form reports the current setting for the + ** maximum number of pages in the database file. The + ** second form attempts to change this setting. Both + ** forms return the current setting. + ** + ** The absolute value of N is used. This is undocumented and might + ** change. The only purpose is to provide an easy way to test + ** the sqlite3AbsInt32() function. + ** + ** PRAGMA [database.]page_count + ** + ** Return the number of pages in the specified database. + */ + case PragTyp_PAGE_COUNT: { + int iReg; + sqlite3CodeVerifySchema(pParse, iDb); + iReg = ++pParse->nMem; + if( sqlite3Tolower(zLeft[0])=='p' ){ + sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); + }else{ + sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, + sqlite3AbsInt32(sqlite3Atoi(zRight))); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); + break; + } + + /* + ** PRAGMA [database.]locking_mode + ** PRAGMA [database.]locking_mode = (normal|exclusive) + */ + case PragTyp_LOCKING_MODE: { + const char *zRet = "normal"; + int eMode = getLockingMode(zRight); + + if( pId2->n==0 && eMode==PAGER_LOCKINGMODE_QUERY ){ + /* Simple "PRAGMA locking_mode;" statement. This is a query for + ** the current default locking mode (which may be different to + ** the locking-mode of the main database). + */ + eMode = db->dfltLockMode; + }else{ + Pager *pPager; + if( pId2->n==0 ){ + /* This indicates that no database name was specified as part + ** of the PRAGMA command. In this case the locking-mode must be + ** set on all attached databases, as well as the main db file. + ** + ** Also, the sqlite3.dfltLockMode variable is set so that + ** any subsequently attached databases also use the specified + ** locking mode. + */ + int ii; + assert(pDb==&db->aDb[0]); + for(ii=2; iinDb; ii++){ + pPager = sqlite3BtreePager(db->aDb[ii].pBt); + sqlite3PagerLockingMode(pPager, eMode); + } + db->dfltLockMode = (u8)eMode; + } + pPager = sqlite3BtreePager(pDb->pBt); + eMode = sqlite3PagerLockingMode(pPager, eMode); + } + + assert( eMode==PAGER_LOCKINGMODE_NORMAL + || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); + if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ + zRet = "exclusive"; + } + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + break; + } + + /* + ** PRAGMA [database.]journal_mode + ** PRAGMA [database.]journal_mode = + ** (delete|persist|off|truncate|memory|wal|off) + */ + case PragTyp_JOURNAL_MODE: { + int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ + int ii; /* Loop counter */ + + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); + + if( zRight==0 ){ + /* If there is no "=MODE" part of the pragma, do a query for the + ** current mode */ + eMode = PAGER_JOURNALMODE_QUERY; + }else{ + const char *zMode; + int n = sqlite3Strlen30(zRight); + for(eMode=0; (zMode = sqlite3JournalModename(eMode))!=0; eMode++){ + if( sqlite3StrNICmp(zRight, zMode, n)==0 ) break; + } + if( !zMode ){ + /* If the "=MODE" part does not match any known journal mode, + ** then do a query */ + eMode = PAGER_JOURNALMODE_QUERY; + } + } + if( eMode==PAGER_JOURNALMODE_QUERY && pId2->n==0 ){ + /* Convert "PRAGMA journal_mode" into "PRAGMA main.journal_mode" */ + iDb = 0; + pId2->n = 1; + } + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3VdbeUsesBtree(v, ii); + sqlite3VdbeAddOp3(v, OP_JournalMode, ii, 1, eMode); + } + } + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + break; + } + + /* + ** PRAGMA [database.]journal_size_limit + ** PRAGMA [database.]journal_size_limit=N + ** + ** Get or set the size limit on rollback journal files. + */ + case PragTyp_JOURNAL_SIZE_LIMIT: { + Pager *pPager = sqlite3BtreePager(pDb->pBt); + i64 iLimit = -2; + if( zRight ){ + sqlite3DecOrHexToI64(zRight, &iLimit); + if( iLimit<-1 ) iLimit = -1; + } + iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); + returnSingleInt(pParse, "journal_size_limit", iLimit); + break; + } + +#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ + + /* + ** PRAGMA [database.]auto_vacuum + ** PRAGMA [database.]auto_vacuum=N + ** + ** Get or set the value of the database 'auto-vacuum' parameter. + ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + case PragTyp_AUTO_VACUUM: { + Btree *pBt = pDb->pBt; + assert( pBt!=0 ); + if( !zRight ){ + returnSingleInt(pParse, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); + }else{ + int eAuto = getAutoVacuum(zRight); + assert( eAuto>=0 && eAuto<=2 ); + db->nextAutovac = (u8)eAuto; + /* Call SetAutoVacuum() to set initialize the internal auto and + ** incr-vacuum flags. This is required in case this connection + ** creates the database file. It is important that it is created + ** as an auto-vacuum capable db. + */ + rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); + if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ + /* When setting the auto_vacuum mode to either "full" or + ** "incremental", write the value of meta[6] in the database + ** file. Before writing to meta[6], check that meta[3] indicates + ** that this really is an auto-vacuum capable database. + */ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList setMeta6[] = { + { OP_Transaction, 0, 1, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, + { OP_If, 1, 0, 0}, /* 2 */ + { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ + { OP_Integer, 0, 1, 0}, /* 4 */ + { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ + }; + int iAddr; + iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); + sqlite3VdbeChangeP1(v, iAddr, iDb); + sqlite3VdbeChangeP1(v, iAddr+1, iDb); + sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); + sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); + sqlite3VdbeChangeP1(v, iAddr+5, iDb); + sqlite3VdbeUsesBtree(v, iDb); + } + } + break; + } +#endif + + /* + ** PRAGMA [database.]incremental_vacuum(N) + ** + ** Do N steps of incremental vacuuming on a database. + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + case PragTyp_INCREMENTAL_VACUUM: { + int iLimit, addr; + if( zRight==0 || !sqlite3GetInt32(zRight, &iLimit) || iLimit<=0 ){ + iLimit = 0x7fffffff; + } + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, iLimit, 1); + addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_ResultRow, 1); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + break; + } +#endif + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + /* + ** PRAGMA [database.]cache_size + ** PRAGMA [database.]cache_size=N + ** + ** The first form reports the current local setting for the + ** page cache size. The second form sets the local + ** page cache size value. If N is positive then that is the + ** number of pages in the cache. If N is negative, then the + ** number of pages is adjusted so that the cache uses -N kibibytes + ** of memory. + */ + case PragTyp_CACHE_SIZE: { + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); + }else{ + int size = sqlite3Atoi(zRight); + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + break; + } + + /* + ** PRAGMA [database.]mmap_size(N) + ** + ** Used to set mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. + ** The parameter N is measured in bytes. + ** + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. + */ + case PragTyp_MMAP_SIZE: { + sqlite3_int64 sz; +#if SQLITE_MAX_MMAP_SIZE>0 + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( zRight ){ + int ii; + sqlite3DecOrHexToI64(zRight, &sz); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); + } + } + } + sz = -1; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_MMAP_SIZE, &sz); +#else + sz = 0; + rc = SQLITE_OK; +#endif + if( rc==SQLITE_OK ){ + returnSingleInt(pParse, "mmap_size", sz); + }else if( rc!=SQLITE_NOTFOUND ){ + pParse->nErr++; + pParse->rc = rc; + } + break; + } + + /* + ** PRAGMA temp_store + ** PRAGMA temp_store = "default"|"memory"|"file" + ** + ** Return or set the local value of the temp_store flag. Changing + ** the local value does not make changes to the disk file and the default + ** value will be restored the next time the database is opened. + ** + ** Note that it is possible for the library compile-time options to + ** override this setting + */ + case PragTyp_TEMP_STORE: { + if( !zRight ){ + returnSingleInt(pParse, "temp_store", db->temp_store); + }else{ + changeTempStorage(pParse, zRight); + } + break; + } + + /* + ** PRAGMA temp_store_directory + ** PRAGMA temp_store_directory = ""|"directory_name" + ** + ** Return or set the local value of the temp_store_directory flag. Changing + ** the value sets a specific directory to be used for temporary files. + ** Setting to a null string reverts to the default temporary directory search. + ** If temporary directory is changed, then invalidateTempStorage. + ** + */ + case PragTyp_TEMP_STORE_DIRECTORY: { + if( !zRight ){ + if( sqlite3_temp_directory ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "temp_store_directory", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_temp_directory, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + }else{ +#ifndef SQLITE_OMIT_WSD + if( zRight[0] ){ + int res; + rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); + if( rc!=SQLITE_OK || res==0 ){ + sqlite3ErrorMsg(pParse, "not a writable directory"); + goto pragma_out; + } + } + if( SQLITE_TEMP_STORE==0 + || (SQLITE_TEMP_STORE==1 && db->temp_store<=1) + || (SQLITE_TEMP_STORE==2 && db->temp_store==1) + ){ + invalidateTempStorage(pParse); + } + sqlite3_free(sqlite3_temp_directory); + if( zRight[0] ){ + sqlite3_temp_directory = sqlite3_mprintf("%s", zRight); + }else{ + sqlite3_temp_directory = 0; + } +#endif /* SQLITE_OMIT_WSD */ + } + break; + } + +#if SQLITE_OS_WIN + /* + ** PRAGMA data_store_directory + ** PRAGMA data_store_directory = ""|"directory_name" + ** + ** Return or set the local value of the data_store_directory flag. Changing + ** the value sets a specific directory to be used for database files that + ** were specified with a relative pathname. Setting to a null string reverts + ** to the default database directory, which for database files specified with + ** a relative path will probably be based on the current directory for the + ** process. Database file specified with an absolute path are not impacted + ** by this setting, regardless of its value. + ** + */ + case PragTyp_DATA_STORE_DIRECTORY: { + if( !zRight ){ + if( sqlite3_data_directory ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "data_store_directory", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_data_directory, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + }else{ +#ifndef SQLITE_OMIT_WSD + if( zRight[0] ){ + int res; + rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); + if( rc!=SQLITE_OK || res==0 ){ + sqlite3ErrorMsg(pParse, "not a writable directory"); + goto pragma_out; + } + } + sqlite3_free(sqlite3_data_directory); + if( zRight[0] ){ + sqlite3_data_directory = sqlite3_mprintf("%s", zRight); + }else{ + sqlite3_data_directory = 0; + } +#endif /* SQLITE_OMIT_WSD */ + } + break; + } +#endif + +#if SQLITE_ENABLE_LOCKING_STYLE + /* + ** PRAGMA [database.]lock_proxy_file + ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" + ** + ** Return or set the value of the lock_proxy_file flag. Changing + ** the value sets a specific file to be used for database access locks. + ** + */ + case PragTyp_LOCK_PROXY_FILE: { + if( !zRight ){ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + char *proxy_file_path = NULL; + sqlite3_file *pFile = sqlite3PagerFile(pPager); + sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, + &proxy_file_path); + + if( proxy_file_path ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "lock_proxy_file", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, proxy_file_path, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + }else{ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + sqlite3_file *pFile = sqlite3PagerFile(pPager); + int res; + if( zRight[0] ){ + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + zRight); + } else { + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + NULL); + } + if( res!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "failed to set lock proxy file"); + goto pragma_out; + } + } + break; + } +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + + /* + ** PRAGMA [database.]synchronous + ** PRAGMA [database.]synchronous=OFF|ON|NORMAL|FULL + ** + ** Return or set the local value of the synchronous flag. Changing + ** the local value does not make changes to the disk file and the + ** default value will be restored the next time the database is + ** opened. + */ + case PragTyp_SYNCHRONOUS: { + if( !zRight ){ + returnSingleInt(pParse, "synchronous", pDb->safety_level-1); + }else{ + if( !db->autoCommit ){ + sqlite3ErrorMsg(pParse, + "Safety level may not be changed inside a transaction"); + }else{ + pDb->safety_level = getSafetyLevel(zRight,0,1)+1; + setAllPagerFlags(db); + } + } + break; + } +#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ + +#ifndef SQLITE_OMIT_FLAG_PRAGMAS + case PragTyp_FLAG: { + if( zRight==0 ){ + returnSingleInt(pParse, aPragmaNames[mid].zName, + (db->flags & aPragmaNames[mid].iArg)!=0 ); + }else{ + int mask = aPragmaNames[mid].iArg; /* Mask of bits to set or clear. */ + if( db->autoCommit==0 ){ + /* Foreign key support may not be enabled or disabled while not + ** in auto-commit mode. */ + mask &= ~(SQLITE_ForeignKeys); + } + + if( sqlite3GetBoolean(zRight, 0) ){ + db->flags |= mask; + }else{ + db->flags &= ~mask; + if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; + } + + /* Many of the flag-pragmas modify the code generated by the SQL + ** compiler (eg. count_changes). So add an opcode to expire all + ** compiled SQL statements after modifying a pragma value. + */ + sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + setAllPagerFlags(db); + } + break; + } +#endif /* SQLITE_OMIT_FLAG_PRAGMAS */ + +#ifndef SQLITE_OMIT_SCHEMA_PRAGMAS + /* + ** PRAGMA table_info(
    ) + ** + ** Return a single row for each column of the named table. The columns of + ** the returned data set are: + ** + ** cid: Column id (numbered from left to right, starting at 0) + ** name: Column name + ** type: Column declaration type. + ** notnull: True if 'NOT NULL' is part of column declaration + ** dflt_value: The default value for the column, if any. + */ + case PragTyp_TABLE_INFO: if( zRight ){ + Table *pTab; + pTab = sqlite3FindTable(db, zRight, zDb); + if( pTab ){ + int i, k; + int nHidden = 0; + Column *pCol; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + sqlite3VdbeSetNumCols(v, 6); + pParse->nMem = 6; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", SQLITE_STATIC); + sqlite3ViewGetColumnNames(pParse, pTab); + for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + if( IsHiddenColumn(pCol) ){ + nHidden++; + continue; + } + sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + pCol->zType ? pCol->zType : "", 0); + sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4); + if( pCol->zDflt ){ + sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pCol->zDflt, 0); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, 5); + } + if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ + k = 0; + }else if( pPk==0 ){ + k = 1; + }else{ + for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + } + sqlite3VdbeAddOp2(v, OP_Integer, k, 6); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); + } + } + } + break; + + case PragTyp_STATS: { + Index *pIdx; + HashElem *i; + v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 4); + pParse->nMem = 4; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "index", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "width", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "height", SQLITE_STATIC); + for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, pTab->zName, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, 2); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->szTabRow), 3); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + } + } + } + break; + + case PragTyp_INDEX_INFO: if( zRight ){ + Index *pIdx; + Table *pTab; + pIdx = sqlite3FindIndex(db, zRight, zDb); + if( pIdx ){ + int i; + pTab = pIdx->pTable; + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); + for(i=0; inKeyCol; i++){ + i16 cnum = pIdx->aiColumn[i]; + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); + assert( pTab->nCol>cnum ); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pTab->aCol[cnum].zName, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + } + } + } + break; + + case PragTyp_INDEX_LIST: if( zRight ){ + Index *pIdx; + Table *pTab; + int i; + pTab = sqlite3FindTable(db, zRight, zDb); + if( pTab ){ + v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); + for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); + sqlite3VdbeAddOp2(v, OP_Integer, IsUniqueIndex(pIdx), 3); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + } + } + } + break; + + case PragTyp_DATABASE_LIST: { + int i; + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", SQLITE_STATIC); + for(i=0; inDb; i++){ + if( db->aDb[i].pBt==0 ) continue; + assert( db->aDb[i].zName!=0 ); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, db->aDb[i].zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + } + } + break; + + case PragTyp_COLLATION_LIST: { + int i = 0; + HashElem *p; + sqlite3VdbeSetNumCols(v, 2); + pParse->nMem = 2; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(p); + sqlite3VdbeAddOp2(v, OP_Integer, i++, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); + } + } + break; +#endif /* SQLITE_OMIT_SCHEMA_PRAGMAS */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY + case PragTyp_FOREIGN_KEY_LIST: if( zRight ){ + FKey *pFK; + Table *pTab; + pTab = sqlite3FindTable(db, zRight, zDb); + if( pTab ){ + v = sqlite3GetVdbe(pParse); + pFK = pTab->pFKey; + if( pFK ){ + int i = 0; + sqlite3VdbeSetNumCols(v, 8); + pParse->nMem = 8; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "on_update", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 6, COLNAME_NAME, "on_delete", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 7, COLNAME_NAME, "match", SQLITE_STATIC); + while(pFK){ + int j; + for(j=0; jnCol; j++){ + char *zCol = pFK->aCol[j].zCol; + char *zOnDelete = (char *)actionName(pFK->aAction[0]); + char *zOnUpdate = (char *)actionName(pFK->aAction[1]); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp2(v, OP_Integer, j, 2); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pFK->zTo, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + pTab->aCol[pFK->aCol[j].iFrom].zName, 0); + sqlite3VdbeAddOp4(v, zCol ? OP_String8 : OP_Null, 0, 5, 0, zCol, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 6, 0, zOnUpdate, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 7, 0, zOnDelete, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 8, 0, "NONE", 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); + } + ++i; + pFK = pFK->pNextFrom; + } + } + } + } + break; +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + case PragTyp_FOREIGN_KEY_CHECK: { + FKey *pFK; /* A foreign key constraint */ + Table *pTab; /* Child table contain "REFERENCES" keyword */ + Table *pParent; /* Parent table that child points to */ + Index *pIdx; /* Index in the parent table */ + int i; /* Loop counter: Foreign key number for pTab */ + int j; /* Loop counter: Field of the foreign key */ + HashElem *k; /* Loop counter: Next table in schema */ + int x; /* result variable */ + int regResult; /* 3 registers to hold a result row */ + int regKey; /* Register to hold key for checking the FK */ + int regRow; /* Registers to hold a row from pTab */ + int addrTop; /* Top of a loop checking foreign keys */ + int addrOk; /* Jump here if the key is OK */ + int *aiCols; /* child to parent column mapping */ + + regResult = pParse->nMem+1; + pParse->nMem += 4; + regKey = ++pParse->nMem; + regRow = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 4); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + sqlite3CodeVerifySchema(pParse, iDb); + k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); + while( k ){ + if( zRight ){ + pTab = sqlite3LocateTable(pParse, 0, zRight, zDb); + k = 0; + }else{ + pTab = (Table*)sqliteHashData(k); + k = sqliteHashNext(k); + } + if( pTab==0 || pTab->pFKey==0 ) continue; + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, + P4_TRANSIENT); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3FindTable(db, pFK->zTo, zDb); + if( pParent==0 ) continue; + pIdx = 0; + sqlite3TableLock(pParse, iDb, pParent->tnum, 0, pParent->zName); + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, 0); + if( x==0 ){ + if( pIdx==0 ){ + sqlite3OpenTable(pParse, i, iDb, pParent, OP_OpenRead); + }else{ + sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + } + }else{ + k = 0; + break; + } + } + assert( pParse->nErr>0 || pFK==0 ); + if( pFK ) break; + if( pParse->nTabnTab = i; + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); VdbeCoverage(v); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3FindTable(db, pFK->zTo, zDb); + pIdx = 0; + aiCols = 0; + if( pParent ){ + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); + assert( x==0 ); + } + addrOk = sqlite3VdbeMakeLabel(v); + if( pParent && pIdx==0 ){ + int iKey = pFK->aCol[0].iFrom; + assert( iKey>=0 && iKeynCol ); + if( iKey!=pTab->iPKey ){ + sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); + sqlite3ColumnDefault(v, pTab, iKey, regRow); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, + sqlite3VdbeCurrentAddr(v)+3); VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); + } + sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + }else{ + for(j=0; jnCol; j++){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, + aiCols ? aiCols[j] : pFK->aCol[j].iFrom, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v); + } + if( pParent ){ + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, + sqlite3IndexAffinityStr(v,pIdx), pFK->nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + VdbeCoverage(v); + } + } + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, + pFK->zTo, P4_TRANSIENT); + sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); + sqlite3VdbeResolveLabel(v, addrOk); + sqlite3DbFree(db, aiCols); + } + sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrTop); + } + } + break; +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + +#ifndef NDEBUG + case PragTyp_PARSER_TRACE: { + if( zRight ){ + if( sqlite3GetBoolean(zRight, 0) ){ + sqlite3ParserTrace(stderr, "parser: "); + }else{ + sqlite3ParserTrace(0, 0); + } + } + } + break; +#endif + + /* Reinstall the LIKE and GLOB functions. The variant of LIKE + ** used will be case sensitive or not depending on the RHS. + */ + case PragTyp_CASE_SENSITIVE_LIKE: { + if( zRight ){ + sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight, 0)); + } + } + break; + +#ifndef SQLITE_INTEGRITY_CHECK_ERROR_MAX +# define SQLITE_INTEGRITY_CHECK_ERROR_MAX 100 +#endif + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK + /* Pragma "quick_check" is reduced version of + ** integrity_check designed to detect most database corruption + ** without most of the overhead of a full integrity-check. + */ + case PragTyp_INTEGRITY_CHECK: { + int i, j, addr, mxErr; + + /* Code that appears at the end of the integrity check. If no error + ** messages have been generated, output OK. Otherwise output the + ** error message + */ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList endCode[] = { + { OP_IfNeg, 1, 0, 0}, /* 0 */ + { OP_String8, 0, 3, 0}, /* 1 */ + { OP_ResultRow, 3, 1, 0}, + }; + + int isQuick = (sqlite3Tolower(zLeft[0])=='q'); + + /* If the PRAGMA command was of the form "PRAGMA .integrity_check", + ** then iDb is set to the index of the database identified by . + ** In this case, the integrity of database iDb only is verified by + ** the VDBE created below. + ** + ** Otherwise, if the command was simply "PRAGMA integrity_check" (or + ** "PRAGMA quick_check"), then iDb is set to 0. In this case, set iDb + ** to -1 here, to indicate that the VDBE should verify the integrity + ** of all attached databases. */ + assert( iDb>=0 ); + assert( iDb==0 || pId2->z ); + if( pId2->z==0 ) iDb = -1; + + /* Initialize the VDBE program */ + pParse->nMem = 6; + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); + + /* Set the maximum error count */ + mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; + if( zRight ){ + sqlite3GetInt32(zRight, &mxErr); + if( mxErr<=0 ){ + mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; + } + } + sqlite3VdbeAddOp2(v, OP_Integer, mxErr, 1); /* reg[1] holds errors left */ + + /* Do an integrity check on each database file */ + for(i=0; inDb; i++){ + HashElem *x; + Hash *pTbls; + int cnt = 0; + + if( OMIT_TEMPDB && i==1 ) continue; + if( iDb>=0 && i!=iDb ) continue; + + sqlite3CodeVerifySchema(pParse, i); + addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + sqlite3VdbeJumpHere(v, addr); + + /* Do an integrity check of the B-Tree + ** + ** Begin by filling registers 2, 3, ... with the root pages numbers + ** for all tables and indices in the database. + */ + assert( sqlite3SchemaMutexHeld(db, i, 0) ); + pTbls = &db->aDb[i].pSchema->tblHash; + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); + VdbeComment((v, "%s", pTab->zName)); + cnt++; + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); + VdbeComment((v, "%s", pIdx->zName)); + cnt++; + } + } + + /* Make sure sufficient number of registers have been allocated */ + pParse->nMem = MAX( pParse->nMem, cnt+8 ); + + /* Do the b-tree integrity checks */ + sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); + sqlite3VdbeChangeP5(v, (u8)i); + addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), + P4_DYNAMIC); + sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); + sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); + sqlite3VdbeJumpHere(v, addr); + + /* Make sure all the indices are constructed correctly. + */ + for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx, *pPk; + Index *pPrior = 0; + int loopTop; + int iDataCur, iIdxCur; + int r1 = -1; + + if( pTab->pIndex==0 ) continue; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + sqlite3VdbeJumpHere(v, addr); + sqlite3ExprCacheClear(pParse); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, + 1, 0, &iDataCur, &iIdxCur); + sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ + } + pParse->nMem = MAX(pParse->nMem, 8+j); + sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); + loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + /* Verify that all NOT NULL columns really are NOT NULL */ + for(j=0; jnCol; j++){ + char *zErr; + int jmp2, jmp3; + if( j==pTab->iPKey ) continue; + if( pTab->aCol[j].notNull==0 ) continue; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, + pTab->aCol[j].zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + jmp3 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); + sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeJumpHere(v, jmp3); + } + /* Validate index entries for the current row */ + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + int jmp2, jmp3, jmp4, jmp5; + int ckUniq = sqlite3VdbeMakeLabel(v); + if( pPk==pIdx ) continue; + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, + pPrior, r1); + pPrior = pIdx; + sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1); /* increment entry count */ + /* Verify that an index entry exists for the current table row */ + jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, + pIdx->nColumn); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + " missing from index ", P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + jmp5 = sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + pIdx->zName, P4_TRANSIENT); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); + sqlite3VdbeJumpHere(v, jmp2); + /* For UNIQUE indexes, verify that only one entry exists with the + ** current key. The entry is unique if (1) any column is NULL + ** or (2) the next entry has a different key */ + if( IsUniqueIndex(pIdx) ){ + int uniqOk = sqlite3VdbeMakeLabel(v); + int jmp6; + int kk; + for(kk=0; kknKeyCol; kk++){ + int iCol = pIdx->aiColumn[kk]; + assert( iCol>=0 && iColnCol ); + if( pTab->aCol[iCol].notNull ) continue; + sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); + VdbeCoverage(v); + } + jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, uniqOk); + sqlite3VdbeJumpHere(v, jmp6); + sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, + pIdx->nKeyCol); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + "non-unique entry in index ", P4_STATIC); + sqlite3VdbeAddOp2(v, OP_Goto, 0, jmp5); + sqlite3VdbeResolveLabel(v, uniqOk); + } + sqlite3VdbeJumpHere(v, jmp4); + sqlite3ResolvePartIdxLabel(pParse, jmp3); + } + sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, loopTop-1); +#ifndef SQLITE_OMIT_BTREECOUNT + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, + "wrong # of entries in index ", P4_STATIC); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + if( pPk==pIdx ) continue; + addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); + sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pIdx->zName, P4_TRANSIENT); + sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); + sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); + } +#endif /* SQLITE_OMIT_BTREECOUNT */ + } + } + addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); + sqlite3VdbeChangeP3(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); + } + break; +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_UTF16 + /* + ** PRAGMA encoding + ** PRAGMA encoding = "utf-8"|"utf-16"|"utf-16le"|"utf-16be" + ** + ** In its first form, this pragma returns the encoding of the main + ** database. If the database is not initialized, it is initialized now. + ** + ** The second form of this pragma is a no-op if the main database file + ** has not already been initialized. In this case it sets the default + ** encoding that will be used for the main database file if a new file + ** is created. If an existing main database file is opened, then the + ** default text encoding for the existing database is used. + ** + ** In all cases new databases created using the ATTACH command are + ** created to use the same default text encoding as the main database. If + ** the main database has not been initialized and/or created when ATTACH + ** is executed, this is done before the ATTACH operation. + ** + ** In the second form this pragma sets the text encoding to be used in + ** new database files created using this database handle. It is only + ** useful if invoked immediately after the main database i + */ + case PragTyp_ENCODING: { + static const struct EncName { + char *zName; + u8 enc; + } encnames[] = { + { "UTF8", SQLITE_UTF8 }, + { "UTF-8", SQLITE_UTF8 }, /* Must be element [1] */ + { "UTF-16le", SQLITE_UTF16LE }, /* Must be element [2] */ + { "UTF-16be", SQLITE_UTF16BE }, /* Must be element [3] */ + { "UTF16le", SQLITE_UTF16LE }, + { "UTF16be", SQLITE_UTF16BE }, + { "UTF-16", 0 }, /* SQLITE_UTF16NATIVE */ + { "UTF16", 0 }, /* SQLITE_UTF16NATIVE */ + { 0, 0 } + }; + const struct EncName *pEnc; + if( !zRight ){ /* "PRAGMA encoding" */ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_String8, 0, 1); + assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); + assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); + assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); + sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + }else{ /* "PRAGMA encoding = XXX" */ + /* Only change the value of sqlite.enc if the database handle is not + ** initialized. If the main database exists, the new sqlite.enc value + ** will be overwritten when the schema is next loaded. If it does not + ** already exists, it will be created to use the new encoding value. + */ + if( + !(DbHasProperty(db, 0, DB_SchemaLoaded)) || + DbHasProperty(db, 0, DB_Empty) + ){ + for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ + if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ + ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + break; + } + } + if( !pEnc->zName ){ + sqlite3ErrorMsg(pParse, "unsupported encoding: %s", zRight); + } + } + } + } + break; +#endif /* SQLITE_OMIT_UTF16 */ + +#ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + /* + ** PRAGMA [database.]schema_version + ** PRAGMA [database.]schema_version = + ** + ** PRAGMA [database.]user_version + ** PRAGMA [database.]user_version = + ** + ** PRAGMA [database.]freelist_count = + ** + ** PRAGMA [database.]application_id + ** PRAGMA [database.]application_id = + ** + ** The pragma's schema_version and user_version are used to set or get + ** the value of the schema-version and user-version, respectively. Both + ** the schema-version and the user-version are 32-bit signed integers + ** stored in the database header. + ** + ** The schema-cookie is usually only manipulated internally by SQLite. It + ** is incremented by SQLite whenever the database schema is modified (by + ** creating or dropping a table or index). The schema version is used by + ** SQLite each time a query is executed to ensure that the internal cache + ** of the schema used when compiling the SQL query matches the schema of + ** the database against which the compiled query is actually executed. + ** Subverting this mechanism by using "PRAGMA schema_version" to modify + ** the schema-version is potentially dangerous and may lead to program + ** crashes or database corruption. Use with caution! + ** + ** The user-version is not used internally by SQLite. It may be used by + ** applications for any purpose. + */ + case PragTyp_HEADER_VALUE: { + int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ + sqlite3VdbeUsesBtree(v, iDb); + switch( zLeft[0] ){ + case 'a': case 'A': + iCookie = BTREE_APPLICATION_ID; + break; + case 'f': case 'F': + iCookie = BTREE_FREE_PAGE_COUNT; + break; + case 's': case 'S': + iCookie = BTREE_SCHEMA_VERSION; + break; + default: + iCookie = BTREE_USER_VERSION; + break; + } + + if( zRight && iCookie!=BTREE_FREE_PAGE_COUNT ){ + /* Write the specified cookie value */ + static const VdbeOpList setCookie[] = { + { OP_Transaction, 0, 1, 0}, /* 0 */ + { OP_Integer, 0, 1, 0}, /* 1 */ + { OP_SetCookie, 0, 0, 1}, /* 2 */ + }; + int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); + sqlite3VdbeChangeP1(v, addr, iDb); + sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); + sqlite3VdbeChangeP1(v, addr+2, iDb); + sqlite3VdbeChangeP2(v, addr+2, iCookie); + }else{ + /* Read the specified cookie value */ + static const VdbeOpList readCookie[] = { + { OP_Transaction, 0, 0, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, 0}, /* 1 */ + { OP_ResultRow, 1, 1, 0} + }; + int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie, 0); + sqlite3VdbeChangeP1(v, addr, iDb); + sqlite3VdbeChangeP1(v, addr+1, iDb); + sqlite3VdbeChangeP3(v, addr+1, iCookie); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); + } + } + break; +#endif /* SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS */ + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + /* + ** PRAGMA compile_options + ** + ** Return the names of all compile-time options used in this build, + ** one option per row. + */ + case PragTyp_COMPILE_OPTIONS: { + int i = 0; + const char *zOpt; + sqlite3VdbeSetNumCols(v, 1); + pParse->nMem = 1; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE_STATIC); + while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + } + break; +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +#ifndef SQLITE_OMIT_WAL + /* + ** PRAGMA [database.]wal_checkpoint = passive|full|restart + ** + ** Checkpoint the database. + */ + case PragTyp_WAL_CHECKPOINT: { + int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); + int eMode = SQLITE_CHECKPOINT_PASSIVE; + if( zRight ){ + if( sqlite3StrICmp(zRight, "full")==0 ){ + eMode = SQLITE_CHECKPOINT_FULL; + }else if( sqlite3StrICmp(zRight, "restart")==0 ){ + eMode = SQLITE_CHECKPOINT_RESTART; + } + } + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC); + + sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + } + break; + + /* + ** PRAGMA wal_autocheckpoint + ** PRAGMA wal_autocheckpoint = N + ** + ** Configure a database connection to automatically checkpoint a database + ** after accumulating N frames in the log. Or query for the current value + ** of N. + */ + case PragTyp_WAL_AUTOCHECKPOINT: { + if( zRight ){ + sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); + } + returnSingleInt(pParse, "wal_autocheckpoint", + db->xWalCallback==sqlite3WalDefaultHook ? + SQLITE_PTR_TO_INT(db->pWalArg) : 0); + } + break; +#endif + + /* + ** PRAGMA shrink_memory + ** + ** This pragma attempts to free as much memory as possible from the + ** current database connection. + */ + case PragTyp_SHRINK_MEMORY: { + sqlite3_db_release_memory(db); + break; + } + + /* + ** PRAGMA busy_timeout + ** PRAGMA busy_timeout = N + ** + ** Call sqlite3_busy_timeout(db, N). Return the current timeout value + ** if one is set. If no busy handler or a different busy handler is set + ** then 0 is returned. Setting the busy_timeout to 0 or negative + ** disables the timeout. + */ + /*case PragTyp_BUSY_TIMEOUT*/ default: { + assert( aPragmaNames[mid].ePragTyp==PragTyp_BUSY_TIMEOUT ); + if( zRight ){ + sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); + } + returnSingleInt(pParse, "timeout", db->busyTimeout); + break; + } + + /* + ** PRAGMA soft_heap_limit + ** PRAGMA soft_heap_limit = N + ** + ** Call sqlite3_soft_heap_limit64(N). Return the result. If N is omitted, + ** use -1. + */ + case PragTyp_SOFT_HEAP_LIMIT: { + sqlite3_int64 N; + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ + sqlite3_soft_heap_limit64(N); + } + returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + break; + } + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* + ** Report the current state of file logs for all databases + */ + case PragTyp_LOCK_STATUS: { + static const char *const azLockName[] = { + "unlocked", "shared", "reserved", "pending", "exclusive" + }; + int i; + sqlite3VdbeSetNumCols(v, 2); + pParse->nMem = 2; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); + for(i=0; inDb; i++){ + Btree *pBt; + const char *zState = "unknown"; + int j; + if( db->aDb[i].zName==0 ) continue; + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); + pBt = db->aDb[i].pBt; + if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ + zState = "closed"; + }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, + SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ + zState = azLockName[j]; + } + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); + } + break; + } +#endif + +#ifdef SQLITE_HAS_CODEC + case PragTyp_KEY: { + if( zRight ) sqlite3_key_v2(db, zDb, zRight, sqlite3Strlen30(zRight)); + break; + } + case PragTyp_REKEY: { + if( zRight ) sqlite3_rekey_v2(db, zDb, zRight, sqlite3Strlen30(zRight)); + break; + } + case PragTyp_HEXKEY: { + if( zRight ){ + u8 iByte; + int i; + char zKey[40]; + for(i=0, iByte=0; idb; + if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ + if( zObj==0 ) zObj = "?"; + sqlite3SetString(pData->pzErrMsg, db, + "malformed database schema (%s)", zObj); + if( zExtra ){ + *pData->pzErrMsg = sqlite3MAppendf(db, *pData->pzErrMsg, + "%s - %s", *pData->pzErrMsg, zExtra); + } + } + pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT_BKPT; +} + +/* +** This is the callback routine for the code that initializes the +** database. See sqlite3Init() below for additional information. +** This routine is also called from the OP_ParseSchema opcode of the VDBE. +** +** Each callback contains the following information: +** +** argv[0] = name of thing being created +** argv[1] = root page number for table or index. 0 for trigger or view. +** argv[2] = SQL text for the CREATE statement. +** +*/ +SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ + InitData *pData = (InitData*)pInit; + sqlite3 *db = pData->db; + int iDb = pData->iDb; + + assert( argc==3 ); + UNUSED_PARAMETER2(NotUsed, argc); + assert( sqlite3_mutex_held(db->mutex) ); + DbClearProperty(db, iDb, DB_Empty); + if( db->mallocFailed ){ + corruptSchema(pData, argv[0], 0); + return 1; + } + + assert( iDb>=0 && iDbnDb ); + if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ + if( argv[1]==0 ){ + corruptSchema(pData, argv[0], 0); + }else if( argv[2] && argv[2][0] ){ + /* Call the parser to process a CREATE TABLE, INDEX or VIEW. + ** But because db->init.busy is set to 1, no VDBE code is generated + ** or executed. All the parser does is build the internal data + ** structures that describe the table, index, or view. + */ + int rc; + sqlite3_stmt *pStmt; + TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ + + assert( db->init.busy ); + db->init.iDb = iDb; + db->init.newTnum = sqlite3Atoi(argv[1]); + db->init.orphanTrigger = 0; + TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); + rc = db->errCode; + assert( (rc&0xFF)==(rcp&0xFF) ); + db->init.iDb = 0; + if( SQLITE_OK!=rc ){ + if( db->init.orphanTrigger ){ + assert( iDb==1 ); + }else{ + pData->rc = rc; + if( rc==SQLITE_NOMEM ){ + db->mallocFailed = 1; + }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ + corruptSchema(pData, argv[0], sqlite3_errmsg(db)); + } + } + } + sqlite3_finalize(pStmt); + }else if( argv[0]==0 ){ + corruptSchema(pData, 0, 0); + }else{ + /* If the SQL column is blank it means this is an index that + ** was created to be the PRIMARY KEY or to fulfill a UNIQUE + ** constraint for a CREATE TABLE. The index should have already + ** been created when we processed the CREATE TABLE. All we have + ** to do here is record the root page number for that index. + */ + Index *pIndex; + pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zName); + if( pIndex==0 ){ + /* This can occur if there exists an index on a TEMP table which + ** has the same name as another index on a permanent index. Since + ** the permanent table is hidden by the TEMP table, we can also + ** safely ignore the index on the permanent table. + */ + /* Do Nothing */; + }else if( sqlite3GetInt32(argv[1], &pIndex->tnum)==0 ){ + corruptSchema(pData, argv[0], "invalid rootpage"); + } + } + return 0; +} + +/* +** Attempt to read the database schema and initialize internal +** data structures for a single database file. The index of the +** database file is given by iDb. iDb==0 is used for the main +** database. iDb==1 should never be used. iDb>=2 is used for +** auxiliary databases. Return one of the SQLITE_ error codes to +** indicate success or failure. +*/ +static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ + int rc; + int i; +#ifndef SQLITE_OMIT_DEPRECATED + int size; +#endif + Table *pTab; + Db *pDb; + char const *azArg[4]; + int meta[5]; + InitData initData; + char const *zMasterSchema; + char const *zMasterName; + int openedTransaction = 0; + + /* + ** The master database table has a structure like this + */ + static const char master_schema[] = + "CREATE TABLE sqlite_master(\n" + " type text,\n" + " name text,\n" + " tbl_name text,\n" + " rootpage integer,\n" + " sql text\n" + ")" + ; +#ifndef SQLITE_OMIT_TEMPDB + static const char temp_master_schema[] = + "CREATE TEMP TABLE sqlite_temp_master(\n" + " type text,\n" + " name text,\n" + " tbl_name text,\n" + " rootpage integer,\n" + " sql text\n" + ")" + ; +#else + #define temp_master_schema 0 +#endif + + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pSchema ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); + + /* zMasterSchema and zInitScript are set to point at the master schema + ** and initialisation script appropriate for the database being + ** initialized. zMasterName is the name of the master table. + */ + if( !OMIT_TEMPDB && iDb==1 ){ + zMasterSchema = temp_master_schema; + }else{ + zMasterSchema = master_schema; + } + zMasterName = SCHEMA_TABLE(iDb); + + /* Construct the schema tables. */ + azArg[0] = zMasterName; + azArg[1] = "1"; + azArg[2] = zMasterSchema; + azArg[3] = 0; + initData.db = db; + initData.iDb = iDb; + initData.rc = SQLITE_OK; + initData.pzErrMsg = pzErrMsg; + sqlite3InitCallback(&initData, 3, (char **)azArg, 0); + if( initData.rc ){ + rc = initData.rc; + goto error_out; + } + pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName); + if( ALWAYS(pTab) ){ + pTab->tabFlags |= TF_Readonly; + } + + /* Create a cursor to hold the database open + */ + pDb = &db->aDb[iDb]; + if( pDb->pBt==0 ){ + if( !OMIT_TEMPDB && ALWAYS(iDb==1) ){ + DbSetProperty(db, 1, DB_SchemaLoaded); + } + return SQLITE_OK; + } + + /* If there is not already a read-only (or read-write) transaction opened + ** on the b-tree database, open one now. If a transaction is opened, it + ** will be closed before this function returns. */ + sqlite3BtreeEnter(pDb->pBt); + if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ + rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); + if( rc!=SQLITE_OK ){ + sqlite3SetString(pzErrMsg, db, "%s", sqlite3ErrStr(rc)); + goto initone_error_out; + } + openedTransaction = 1; + } + + /* Get the database meta information. + ** + ** Meta values are as follows: + ** meta[0] Schema cookie. Changes with each schema change. + ** meta[1] File format of schema layer. + ** meta[2] Size of the page cache. + ** meta[3] Largest rootpage (auto/incr_vacuum mode) + ** meta[4] Db text encoding. 1:UTF-8 2:UTF-16LE 3:UTF-16BE + ** meta[5] User version + ** meta[6] Incremental vacuum mode + ** meta[7] unused + ** meta[8] unused + ** meta[9] unused + ** + ** Note: The #defined SQLITE_UTF* symbols in sqliteInt.h correspond to + ** the possible values of meta[4]. + */ + for(i=0; ipBt, i+1, (u32 *)&meta[i]); + } + pDb->pSchema->schema_cookie = meta[BTREE_SCHEMA_VERSION-1]; + + /* If opening a non-empty database, check the text encoding. For the + ** main database, set sqlite3.enc to the encoding of the main database. + ** For an attached db, it is an error if the encoding is not the same + ** as sqlite3.enc. + */ + if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ + if( iDb==0 ){ +#ifndef SQLITE_OMIT_UTF16 + u8 encoding; + /* If opening the main database, set ENC(db). */ + encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; + if( encoding==0 ) encoding = SQLITE_UTF8; + ENC(db) = encoding; +#else + ENC(db) = SQLITE_UTF8; +#endif + }else{ + /* If opening an attached database, the encoding much match ENC(db) */ + if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ + sqlite3SetString(pzErrMsg, db, "attached databases must use the same" + " text encoding as main database"); + rc = SQLITE_ERROR; + goto initone_error_out; + } + } + }else{ + DbSetProperty(db, iDb, DB_Empty); + } + pDb->pSchema->enc = ENC(db); + + if( pDb->pSchema->cache_size==0 ){ +#ifndef SQLITE_OMIT_DEPRECATED + size = sqlite3AbsInt32(meta[BTREE_DEFAULT_CACHE_SIZE-1]); + if( size==0 ){ size = SQLITE_DEFAULT_CACHE_SIZE; } + pDb->pSchema->cache_size = size; +#else + pDb->pSchema->cache_size = SQLITE_DEFAULT_CACHE_SIZE; +#endif + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + + /* + ** file_format==1 Version 3.0.0. + ** file_format==2 Version 3.1.3. // ALTER TABLE ADD COLUMN + ** file_format==3 Version 3.1.4. // ditto but with non-NULL defaults + ** file_format==4 Version 3.3.0. // DESC indices. Boolean constants + */ + pDb->pSchema->file_format = (u8)meta[BTREE_FILE_FORMAT-1]; + if( pDb->pSchema->file_format==0 ){ + pDb->pSchema->file_format = 1; + } + if( pDb->pSchema->file_format>SQLITE_MAX_FILE_FORMAT ){ + sqlite3SetString(pzErrMsg, db, "unsupported file format"); + rc = SQLITE_ERROR; + goto initone_error_out; + } + + /* Ticket #2804: When we open a database in the newer file format, + ** clear the legacy_file_format pragma flag so that a VACUUM will + ** not downgrade the database and thus invalidate any descending + ** indices that the user might have created. + */ + if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){ + db->flags &= ~SQLITE_LegacyFileFmt; + } + + /* Read the schema information out of the schema tables + */ + assert( db->init.busy ); + { + char *zSql; + zSql = sqlite3MPrintf(db, + "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid", + db->aDb[iDb].zName, zMasterName); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + xAuth = db->xAuth; + db->xAuth = 0; +#endif + rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; + } +#endif + if( rc==SQLITE_OK ) rc = initData.rc; + sqlite3DbFree(db, zSql); +#ifndef SQLITE_OMIT_ANALYZE + if( rc==SQLITE_OK ){ + sqlite3AnalysisLoad(db, iDb); + } +#endif + } + if( db->mallocFailed ){ + rc = SQLITE_NOMEM; + sqlite3ResetAllSchemasOfConnection(db); + } + if( rc==SQLITE_OK || (db->flags&SQLITE_RecoveryMode)){ + /* Black magic: If the SQLITE_RecoveryMode flag is set, then consider + ** the schema loaded, even if errors occurred. In this situation the + ** current sqlite3_prepare() operation will fail, but the following one + ** will attempt to compile the supplied statement against whatever subset + ** of the schema was loaded before the error occurred. The primary + ** purpose of this is to allow access to the sqlite_master table + ** even when its contents have been corrupted. + */ + DbSetProperty(db, iDb, DB_SchemaLoaded); + rc = SQLITE_OK; + } + + /* Jump here for an error that occurs after successfully allocating + ** curMain and calling sqlite3BtreeEnter(). For an error that occurs + ** before that point, jump to error_out. + */ +initone_error_out: + if( openedTransaction ){ + sqlite3BtreeCommit(pDb->pBt); + } + sqlite3BtreeLeave(pDb->pBt); + +error_out: + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + db->mallocFailed = 1; + } + return rc; +} + +/* +** Initialize all database files - the main database file, the file +** used to store temporary tables, and any additional database files +** created using ATTACH statements. Return a success code. If an +** error occurs, write an error message into *pzErrMsg. +** +** After a database is initialized, the DB_SchemaLoaded bit is set +** bit is set in the flags field of the Db structure. If the database +** file was of zero-length, then the DB_Empty flag is also set. +*/ +SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ + int i, rc; + int commit_internal = !(db->flags&SQLITE_InternChanges); + + assert( sqlite3_mutex_held(db->mutex) ); + rc = SQLITE_OK; + db->init.busy = 1; + for(i=0; rc==SQLITE_OK && inDb; i++){ + if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; + rc = sqlite3InitOne(db, i, pzErrMsg); + if( rc ){ + sqlite3ResetOneSchema(db, i); + } + } + + /* Once all the other databases have been initialized, load the schema + ** for the TEMP database. This is loaded last, as the TEMP database + ** schema may contain references to objects in other databases. + */ +#ifndef SQLITE_OMIT_TEMPDB + if( rc==SQLITE_OK && ALWAYS(db->nDb>1) + && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, 1, pzErrMsg); + if( rc ){ + sqlite3ResetOneSchema(db, 1); + } + } +#endif + + db->init.busy = 0; + if( rc==SQLITE_OK && commit_internal ){ + sqlite3CommitInternalChanges(db); + } + + return rc; +} + +/* +** This routine is a no-op if the database schema is already initialized. +** Otherwise, the schema is loaded. An error code is returned. +*/ +SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse){ + int rc = SQLITE_OK; + sqlite3 *db = pParse->db; + assert( sqlite3_mutex_held(db->mutex) ); + if( !db->init.busy ){ + rc = sqlite3Init(db, &pParse->zErrMsg); + } + if( rc!=SQLITE_OK ){ + pParse->rc = rc; + pParse->nErr++; + } + return rc; +} + + +/* +** Check schema cookies in all databases. If any cookie is out +** of date set pParse->rc to SQLITE_SCHEMA. If all schema cookies +** make no changes to pParse->rc. +*/ +static void schemaIsValid(Parse *pParse){ + sqlite3 *db = pParse->db; + int iDb; + int rc; + int cookie; + + assert( pParse->checkSchema ); + assert( sqlite3_mutex_held(db->mutex) ); + for(iDb=0; iDbnDb; iDb++){ + int openedTransaction = 0; /* True if a transaction is opened */ + Btree *pBt = db->aDb[iDb].pBt; /* Btree database to read cookie from */ + if( pBt==0 ) continue; + + /* If there is not already a read-only (or read-write) transaction opened + ** on the b-tree database, open one now. If a transaction is opened, it + ** will be closed immediately after reading the meta-value. */ + if( !sqlite3BtreeIsInReadTrans(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0); + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + db->mallocFailed = 1; + } + if( rc!=SQLITE_OK ) return; + openedTransaction = 1; + } + + /* Read the schema cookie from the database. If it does not match the + ** value stored as part of the in-memory schema representation, + ** set Parse.rc to SQLITE_SCHEMA. */ + sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + sqlite3ResetOneSchema(db, iDb); + pParse->rc = SQLITE_SCHEMA; + } + + /* Close the transaction, if one was opened. */ + if( openedTransaction ){ + sqlite3BtreeCommit(pBt); + } + } +} + +/* +** Convert a schema pointer into the iDb index that indicates +** which database file in db->aDb[] the schema refers to. +** +** If the same database is attached more than once, the first +** attached database is returned. +*/ +SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ + int i = -1000000; + + /* If pSchema is NULL, then return -1000000. This happens when code in + ** expr.c is trying to resolve a reference to a transient table (i.e. one + ** created by a sub-select). In this case the return value of this + ** function should never be used. + ** + ** We return -1000000 instead of the more usual -1 simply because using + ** -1000000 as the incorrect index into db->aDb[] is much + ** more likely to cause a segfault than -1 (of course there are assert() + ** statements too, but it never hurts to play the odds). + */ + assert( sqlite3_mutex_held(db->mutex) ); + if( pSchema ){ + for(i=0; ALWAYS(inDb); i++){ + if( db->aDb[i].pSchema==pSchema ){ + break; + } + } + assert( i>=0 && inDb ); + } + return i; +} + +/* +** Free all memory allocations in the pParse object +*/ +SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ + if( pParse ){ + sqlite3 *db = pParse->db; + sqlite3DbFree(db, pParse->aLabel); + sqlite3ExprListDelete(db, pParse->pConstExpr); + } +} + +/* +** Compile the UTF-8 encoded SQL statement zSql into a statement handle. +*/ +static int sqlite3Prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + int saveSqlFlag, /* True to copy SQL text into the sqlite3_stmt */ + Vdbe *pReprepare, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + Parse *pParse; /* Parsing context */ + char *zErrMsg = 0; /* Error message */ + int rc = SQLITE_OK; /* Result code */ + int i; /* Loop counter */ + + /* Allocate the parsing context */ + pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); + if( pParse==0 ){ + rc = SQLITE_NOMEM; + goto end_prepare; + } + pParse->pReprepare = pReprepare; + assert( ppStmt && *ppStmt==0 ); + assert( !db->mallocFailed ); + assert( sqlite3_mutex_held(db->mutex) ); + + /* Check to verify that it is possible to get a read lock on all + ** database schemas. The inability to get a read lock indicates that + ** some other database connection is holding a write-lock, which in + ** turn means that the other connection has made uncommitted changes + ** to the schema. + ** + ** Were we to proceed and prepare the statement against the uncommitted + ** schema changes and if those schema changes are subsequently rolled + ** back and different changes are made in their place, then when this + ** prepared statement goes to run the schema cookie would fail to detect + ** the schema change. Disaster would follow. + ** + ** This thread is currently holding mutexes on all Btrees (because + ** of the sqlite3BtreeEnterAll() in sqlite3LockAndPrepare()) so it + ** is not possible for another thread to start a new schema change + ** while this routine is running. Hence, we do not need to hold + ** locks on the schema, we just need to make sure nobody else is + ** holding them. + ** + ** Note that setting READ_UNCOMMITTED overrides most lock detection, + ** but it does *not* override schema lock detection, so this all still + ** works even if READ_UNCOMMITTED is set. + */ + for(i=0; inDb; i++) { + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + assert( sqlite3BtreeHoldsMutex(pBt) ); + rc = sqlite3BtreeSchemaLocked(pBt); + if( rc ){ + const char *zDb = db->aDb[i].zName; + sqlite3Error(db, rc, "database schema is locked: %s", zDb); + testcase( db->flags & SQLITE_ReadUncommitted ); + goto end_prepare; + } + } + } + + sqlite3VtabUnlockList(db); + + pParse->db = db; + pParse->nQueryLoop = 0; /* Logarithmic, so 0 really means 1 */ + if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ + char *zSqlCopy; + int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + testcase( nBytes==mxLen ); + testcase( nBytes==mxLen+1 ); + if( nBytes>mxLen ){ + sqlite3Error(db, SQLITE_TOOBIG, "statement too long"); + rc = sqlite3ApiExit(db, SQLITE_TOOBIG); + goto end_prepare; + } + zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); + if( zSqlCopy ){ + sqlite3RunParser(pParse, zSqlCopy, &zErrMsg); + sqlite3DbFree(db, zSqlCopy); + pParse->zTail = &zSql[pParse->zTail-zSqlCopy]; + }else{ + pParse->zTail = &zSql[nBytes]; + } + }else{ + sqlite3RunParser(pParse, zSql, &zErrMsg); + } + assert( 0==pParse->nQueryLoop ); + + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; + if( pParse->checkSchema ){ + schemaIsValid(pParse); + } + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pzTail ){ + *pzTail = pParse->zTail; + } + rc = pParse->rc; + +#ifndef SQLITE_OMIT_EXPLAIN + if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ + static const char * const azColName[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", + "selectid", "order", "from", "detail" + }; + int iFirst, mx; + if( pParse->explain==2 ){ + sqlite3VdbeSetNumCols(pParse->pVdbe, 4); + iFirst = 8; + mx = 12; + }else{ + sqlite3VdbeSetNumCols(pParse->pVdbe, 8); + iFirst = 0; + mx = 8; + } + for(i=iFirst; ipVdbe, i-iFirst, COLNAME_NAME, + azColName[i], SQLITE_STATIC); + } + } +#endif + + if( db->init.busy==0 ){ + Vdbe *pVdbe = pParse->pVdbe; + sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); + } + if( pParse->pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ + sqlite3VdbeFinalize(pParse->pVdbe); + assert(!(*ppStmt)); + }else{ + *ppStmt = (sqlite3_stmt*)pParse->pVdbe; + } + + if( zErrMsg ){ + sqlite3Error(db, rc, "%s", zErrMsg); + sqlite3DbFree(db, zErrMsg); + }else{ + sqlite3Error(db, rc, 0); + } + + /* Delete any TriggerPrg structures allocated while parsing this statement. */ + while( pParse->pTriggerPrg ){ + TriggerPrg *pT = pParse->pTriggerPrg; + pParse->pTriggerPrg = pT->pNext; + sqlite3DbFree(db, pT); + } + +end_prepare: + + sqlite3ParserReset(pParse); + sqlite3StackFree(db, pParse); + rc = sqlite3ApiExit(db, rc); + assert( (rc&db->errMask)==rc ); + return rc; +} +static int sqlite3LockAndPrepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + int saveSqlFlag, /* True to copy SQL text into the sqlite3_stmt */ + Vdbe *pOld, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + assert( ppStmt!=0 ); + *ppStmt = 0; + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); + if( rc==SQLITE_SCHEMA ){ + sqlite3_finalize(*ppStmt); + rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + assert( rc==SQLITE_OK || *ppStmt==0 ); + return rc; +} + +/* +** Rerun the compilation of a statement after a schema change. +** +** If the statement is successfully recompiled, return SQLITE_OK. Otherwise, +** if the statement cannot be recompiled because another connection has +** locked the sqlite3_master table, return SQLITE_LOCKED. If any other error +** occurs, return SQLITE_SCHEMA. +*/ +SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ + int rc; + sqlite3_stmt *pNew; + const char *zSql; + sqlite3 *db; + + assert( sqlite3_mutex_held(sqlite3VdbeDb(p)->mutex) ); + zSql = sqlite3_sql((sqlite3_stmt *)p); + assert( zSql!=0 ); /* Reprepare only called for prepare_v2() statements */ + db = sqlite3VdbeDb(p); + assert( sqlite3_mutex_held(db->mutex) ); + rc = sqlite3LockAndPrepare(db, zSql, -1, 0, p, &pNew, 0); + if( rc ){ + if( rc==SQLITE_NOMEM ){ + db->mallocFailed = 1; + } + assert( pNew==0 ); + return rc; + }else{ + assert( pNew!=0 ); + } + sqlite3VdbeSwap((Vdbe*)pNew, p); + sqlite3TransferBindings(pNew, (sqlite3_stmt*)p); + sqlite3VdbeResetStepResult((Vdbe*)pNew); + sqlite3VdbeFinalize((Vdbe*)pNew); + return SQLITE_OK; +} + + +/* +** Two versions of the official API. Legacy and new use. In the legacy +** version, the original SQL text is not saved in the prepared statement +** and so if a schema change occurs, SQLITE_SCHEMA is returned by +** sqlite3_step(). In the new version, the original SQL text is retained +** and the statement is automatically recompiled if an schema change +** occurs. +*/ +SQLITE_API int sqlite3_prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3LockAndPrepare(db,zSql,nBytes,0,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +SQLITE_API int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3LockAndPrepare(db,zSql,nBytes,1,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} + + +#ifndef SQLITE_OMIT_UTF16 +/* +** Compile the UTF-16 encoded SQL statement zSql into a statement handle. +*/ +static int sqlite3Prepare16( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + int saveSqlFlag, /* True to save SQL text into the sqlite3_stmt */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + /* This function currently works by first transforming the UTF-16 + ** encoded string to UTF-8, then invoking sqlite3_prepare(). The + ** tricky bit is figuring out the pointer to return in *pzTail. + */ + char *zSql8; + const char *zTail8 = 0; + int rc = SQLITE_OK; + + assert( ppStmt ); + *ppStmt = 0; + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( nBytes>=0 ){ + int sz; + const char *z = (const char*)zSql; + for(sz=0; szmutex); + zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); + if( zSql8 ){ + rc = sqlite3LockAndPrepare(db, zSql8, -1, saveSqlFlag, 0, ppStmt, &zTail8); + } + + if( zTail8 && pzTail ){ + /* If sqlite3_prepare returns a tail pointer, we calculate the + ** equivalent pointer into the UTF-16 string by counting the unicode + ** characters between zSql8 and zTail8, and then returning a pointer + ** the same number of characters into the UTF-16 string. + */ + int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); + *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); + } + sqlite3DbFree(db, zSql8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Two versions of the official API. Legacy and new use. In the legacy +** version, the original SQL text is not saved in the prepared statement +** and so if a schema change occurs, SQLITE_SCHEMA is returned by +** sqlite3_step(). In the new version, the original SQL text is retained +** and the statement is automatically recompiled if an schema change +** occurs. +*/ +SQLITE_API int sqlite3_prepare16( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +SQLITE_API int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,1,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} + +#endif /* SQLITE_OMIT_UTF16 */ + +/************** End of prepare.c *********************************************/ +/************** Begin file select.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle SELECT statements in SQLite. +*/ + +/* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + +/* +** An instance of the following object is used to record information about +** the ORDER BY (or GROUP BY) clause of query is being coded. +*/ +typedef struct SortCtx SortCtx; +struct SortCtx { + ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ + int nOBSat; /* Number of ORDER BY terms satisfied by indices */ + int iECursor; /* Cursor number for the sorter */ + int regReturn; /* Register holding block-output return address */ + int labelBkOut; /* Start label for the block-output subroutine */ + int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + u8 sortFlags; /* Zero or more SORTFLAG_* bits */ +}; +#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ + +/* +** Delete all the content of a Select structure but do not deallocate +** the select structure itself. +*/ +static void clearSelect(sqlite3 *db, Select *p){ + sqlite3ExprListDelete(db, p->pEList); + sqlite3SrcListDelete(db, p->pSrc); + sqlite3ExprDelete(db, p->pWhere); + sqlite3ExprListDelete(db, p->pGroupBy); + sqlite3ExprDelete(db, p->pHaving); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3SelectDelete(db, p->pPrior); + sqlite3ExprDelete(db, p->pLimit); + sqlite3ExprDelete(db, p->pOffset); + sqlite3WithDelete(db, p->pWith); +} + +/* +** Initialize a SelectDest structure. +*/ +SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ + pDest->eDest = (u8)eDest; + pDest->iSDParm = iParm; + pDest->affSdst = 0; + pDest->iSdst = 0; + pDest->nSdst = 0; +} + + +/* +** Allocate a new Select structure and return a pointer to that +** structure. +*/ +SQLITE_PRIVATE Select *sqlite3SelectNew( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* which columns to include in the result */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* the WHERE clause */ + ExprList *pGroupBy, /* the GROUP BY clause */ + Expr *pHaving, /* the HAVING clause */ + ExprList *pOrderBy, /* the ORDER BY clause */ + u16 selFlags, /* Flag parameters, such as SF_Distinct */ + Expr *pLimit, /* LIMIT value. NULL means not used */ + Expr *pOffset /* OFFSET value. NULL means no offset */ +){ + Select *pNew; + Select standin; + sqlite3 *db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + assert( db->mallocFailed || !pOffset || pLimit ); /* OFFSET implies LIMIT */ + if( pNew==0 ){ + assert( db->mallocFailed ); + pNew = &standin; + memset(pNew, 0, sizeof(*pNew)); + } + if( pEList==0 ){ + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0)); + } + pNew->pEList = pEList; + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); + pNew->pSrc = pSrc; + pNew->pWhere = pWhere; + pNew->pGroupBy = pGroupBy; + pNew->pHaving = pHaving; + pNew->pOrderBy = pOrderBy; + pNew->selFlags = selFlags; + pNew->op = TK_SELECT; + pNew->pLimit = pLimit; + pNew->pOffset = pOffset; + assert( pOffset==0 || pLimit!=0 ); + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + if( db->mallocFailed ) { + clearSelect(db, pNew); + if( pNew!=&standin ) sqlite3DbFree(db, pNew); + pNew = 0; + }else{ + assert( pNew->pSrc!=0 || pParse->nErr>0 ); + } + assert( pNew!=&standin ); + return pNew; +} + +/* +** Delete the given Select structure and all of its substructures. +*/ +SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ + if( p ){ + clearSelect(db, p); + sqlite3DbFree(db, p); + } +} + +/* +** Return a pointer to the right-most SELECT statement in a compound. +*/ +static Select *findRightmost(Select *p){ + while( p->pNext ) p = p->pNext; + return p; +} + +/* +** Given 1 to 3 identifiers preceding the JOIN keyword, determine the +** type of join. Return an integer constant that expresses that type +** in terms of the following bit values: +** +** JT_INNER +** JT_CROSS +** JT_OUTER +** JT_NATURAL +** JT_LEFT +** JT_RIGHT +** +** A full outer join is the combination of JT_LEFT and JT_RIGHT. +** +** If an illegal or unsupported join type is seen, then still return +** a join type, but put an error in the pParse structure. +*/ +SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ + int jointype = 0; + Token *apAll[3]; + Token *p; + /* 0123456789 123456789 123456789 123 */ + static const char zKeyText[] = "naturaleftouterightfullinnercross"; + static const struct { + u8 i; /* Beginning of keyword text in zKeyText[] */ + u8 nChar; /* Length of the keyword in characters */ + u8 code; /* Join type mask */ + } aKeyword[] = { + /* natural */ { 0, 7, JT_NATURAL }, + /* left */ { 6, 4, JT_LEFT|JT_OUTER }, + /* outer */ { 10, 5, JT_OUTER }, + /* right */ { 14, 5, JT_RIGHT|JT_OUTER }, + /* full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER }, + /* inner */ { 23, 5, JT_INNER }, + /* cross */ { 28, 5, JT_INNER|JT_CROSS }, + }; + int i, j; + apAll[0] = pA; + apAll[1] = pB; + apAll[2] = pC; + for(i=0; i<3 && apAll[i]; i++){ + p = apAll[i]; + for(j=0; jn==aKeyword[j].nChar + && sqlite3StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ + jointype |= aKeyword[j].code; + break; + } + } + testcase( j==0 || j==1 || j==2 || j==3 || j==4 || j==5 || j==6 ); + if( j>=ArraySize(aKeyword) ){ + jointype |= JT_ERROR; + break; + } + } + if( + (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) || + (jointype & JT_ERROR)!=0 + ){ + const char *zSp = " "; + assert( pB!=0 ); + if( pC==0 ){ zSp++; } + sqlite3ErrorMsg(pParse, "unknown or unsupported join type: " + "%T %T%s%T", pA, pB, zSp, pC); + jointype = JT_INNER; + }else if( (jointype & JT_OUTER)!=0 + && (jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ){ + sqlite3ErrorMsg(pParse, + "RIGHT and FULL OUTER JOINs are not currently supported"); + jointype = JT_INNER; + } + return jointype; +} + +/* +** Return the index of a column in a table. Return -1 if the column +** is not contained in the table. +*/ +static int columnIndex(Table *pTab, const char *zCol){ + int i; + for(i=0; inCol; i++){ + if( sqlite3StrICmp(pTab->aCol[i].zName, zCol)==0 ) return i; + } + return -1; +} + +/* +** Search the first N tables in pSrc, from left to right, looking for a +** table that has a column named zCol. +** +** When found, set *piTab and *piCol to the table index and column index +** of the matching column and return TRUE. +** +** If not found, return FALSE. +*/ +static int tableAndColumnIndex( + SrcList *pSrc, /* Array of tables to search */ + int N, /* Number of tables in pSrc->a[] to search */ + const char *zCol, /* Name of the column we are looking for */ + int *piTab, /* Write index of pSrc->a[] here */ + int *piCol /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ +){ + int i; /* For looping over tables in pSrc */ + int iCol; /* Index of column matching zCol */ + + assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ + for(i=0; ia[i].pTab, zCol); + if( iCol>=0 ){ + if( piTab ){ + *piTab = i; + *piCol = iCol; + } + return 1; + } + } + return 0; +} + +/* +** This function is used to add terms implied by JOIN syntax to the +** WHERE clause expression of a SELECT statement. The new term, which +** is ANDed with the existing WHERE clause, is of the form: +** +** (tab1.col1 = tab2.col2) +** +** where tab1 is the iSrc'th table in SrcList pSrc and tab2 is the +** (iSrc+1)'th. Column col1 is column iColLeft of tab1, and col2 is +** column iColRight of tab2. +*/ +static void addWhereTerm( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* List of tables in FROM clause */ + int iLeft, /* Index of first table to join in pSrc */ + int iColLeft, /* Index of column in first table */ + int iRight, /* Index of second table in pSrc */ + int iColRight, /* Index of column in second table */ + int isOuterJoin, /* True if this is an OUTER join */ + Expr **ppWhere /* IN/OUT: The WHERE clause to add to */ +){ + sqlite3 *db = pParse->db; + Expr *pE1; + Expr *pE2; + Expr *pEq; + + assert( iLeftnSrc>iRight ); + assert( pSrc->a[iLeft].pTab ); + assert( pSrc->a[iRight].pTab ); + + pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iColLeft); + pE2 = sqlite3CreateColumnExpr(db, pSrc, iRight, iColRight); + + pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2, 0); + if( pEq && isOuterJoin ){ + ExprSetProperty(pEq, EP_FromJoin); + assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pEq, EP_NoReduce); + pEq->iRightJoinTable = (i16)pE2->iTable; + } + *ppWhere = sqlite3ExprAnd(db, *ppWhere, pEq); +} + +/* +** Set the EP_FromJoin property on all terms of the given expression. +** And set the Expr.iRightJoinTable to iTable for every term in the +** expression. +** +** The EP_FromJoin property is used on terms of an expression to tell +** the LEFT OUTER JOIN processing logic that this term is part of the +** join restriction specified in the ON or USING clause and not a part +** of the more general WHERE clause. These terms are moved over to the +** WHERE clause during join processing but we need to remember that they +** originated in the ON or USING clause. +** +** The Expr.iRightJoinTable tells the WHERE clause processing that the +** expression depends on table iRightJoinTable even if that table is not +** explicitly mentioned in the expression. That information is needed +** for cases like this: +** +** SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.b AND t1.x=5 +** +** The where clause needs to defer the handling of the t1.x=5 +** term until after the t2 loop of the join. In that way, a +** NULL t2 row will be inserted whenever t1.x!=5. If we do not +** defer the handling of t1.x=5, it will be processed immediately +** after the t1 loop and rows with t1.x!=5 will never appear in +** the output, which is incorrect. +*/ +static void setJoinExpr(Expr *p, int iTable){ + while( p ){ + ExprSetProperty(p, EP_FromJoin); + assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(p, EP_NoReduce); + p->iRightJoinTable = (i16)iTable; + setJoinExpr(p->pLeft, iTable); + p = p->pRight; + } +} + +/* +** This routine processes the join information for a SELECT statement. +** ON and USING clauses are converted into extra terms of the WHERE clause. +** NATURAL joins also create extra WHERE clause terms. +** +** The terms of a FROM clause are contained in the Select.pSrc structure. +** The left most table is the first entry in Select.pSrc. The right-most +** table is the last entry. The join operator is held in the entry to +** the left. Thus entry 0 contains the join operator for the join between +** entries 0 and 1. Any ON or USING clauses associated with the join are +** also attached to the left entry. +** +** This routine returns the number of errors encountered. +*/ +static int sqliteProcessJoin(Parse *pParse, Select *p){ + SrcList *pSrc; /* All tables in the FROM clause */ + int i, j; /* Loop counters */ + struct SrcList_item *pLeft; /* Left table being joined */ + struct SrcList_item *pRight; /* Right table being joined */ + + pSrc = p->pSrc; + pLeft = &pSrc->a[0]; + pRight = &pLeft[1]; + for(i=0; inSrc-1; i++, pRight++, pLeft++){ + Table *pLeftTab = pLeft->pTab; + Table *pRightTab = pRight->pTab; + int isOuter; + + if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; + isOuter = (pRight->jointype & JT_OUTER)!=0; + + /* When the NATURAL keyword is present, add WHERE clause terms for + ** every column that the two tables have in common. + */ + if( pRight->jointype & JT_NATURAL ){ + if( pRight->pOn || pRight->pUsing ){ + sqlite3ErrorMsg(pParse, "a NATURAL join may not have " + "an ON or USING clause", 0); + return 1; + } + for(j=0; jnCol; j++){ + char *zName; /* Name of column in the right table */ + int iLeft; /* Matching left table */ + int iLeftCol; /* Matching column in the left table */ + + zName = pRightTab->aCol[j].zName; + if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) ){ + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, j, + isOuter, &p->pWhere); + } + } + } + + /* Disallow both ON and USING clauses in the same join + */ + if( pRight->pOn && pRight->pUsing ){ + sqlite3ErrorMsg(pParse, "cannot have both ON and USING " + "clauses in the same join"); + return 1; + } + + /* Add the ON clause to the end of the WHERE clause, connected by + ** an AND operator. + */ + if( pRight->pOn ){ + if( isOuter ) setJoinExpr(pRight->pOn, pRight->iCursor); + p->pWhere = sqlite3ExprAnd(pParse->db, p->pWhere, pRight->pOn); + pRight->pOn = 0; + } + + /* Create extra terms on the WHERE clause for each column named + ** in the USING clause. Example: If the two tables to be joined are + ** A and B and the USING clause names X, Y, and Z, then add this + ** to the WHERE clause: A.X=B.X AND A.Y=B.Y AND A.Z=B.Z + ** Report an error if any column mentioned in the USING clause is + ** not contained in both tables to be joined. + */ + if( pRight->pUsing ){ + IdList *pList = pRight->pUsing; + for(j=0; jnId; j++){ + char *zName; /* Name of the term in the USING clause */ + int iLeft; /* Table on the left with matching column name */ + int iLeftCol; /* Column number of matching column on the left */ + int iRightCol; /* Column number of matching column on the right */ + + zName = pList->a[j].zName; + iRightCol = columnIndex(pRightTab, zName); + if( iRightCol<0 + || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) + ){ + sqlite3ErrorMsg(pParse, "cannot join using column %s - column " + "not present in both tables", zName); + return 1; + } + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, iRightCol, + isOuter, &p->pWhere); + } + } + } + return 0; +} + +/* Forward reference */ +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +); + +/* +** Insert code into "v" that will push the record in register regData +** into the sorter. +*/ +static void pushOntoSorter( + Parse *pParse, /* Parser context */ + SortCtx *pSort, /* Information about the ORDER BY clause */ + Select *pSelect, /* The whole SELECT statement */ + int regData /* Register holding data to be sorted */ +){ + Vdbe *v = pParse->pVdbe; + int nExpr = pSort->pOrderBy->nExpr; + int regRecord = ++pParse->nMem; + int regBase = pParse->nMem+1; + int nOBSat = pSort->nOBSat; + int op; + + pParse->nMem += nExpr+2; /* nExpr+2 registers allocated at regBase */ + sqlite3ExprCacheClear(pParse); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); + sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat,regRecord); + if( nOBSat>0 ){ + int regPrevKey; /* The first nOBSat columns of the previous row */ + int addrFirst; /* Address of the OP_IfNot opcode */ + int addrJmp; /* Address of the OP_Jump opcode */ + VdbeOp *pOp; /* Opcode that opens the sorter */ + int nKey; /* Number of sorting key columns, including OP_Sequence */ + KeyInfo *pKI; /* Original KeyInfo on the sorter table */ + + regPrevKey = pParse->nMem+1; + pParse->nMem += pSort->nOBSat; + nKey = nExpr - pSort->nOBSat + 1; + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); + pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + if( pParse->db->mallocFailed ) return; + pOp->p2 = nKey + 1; + pKI = pOp->p4.pKeyInfo; + memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ + sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); + pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); + addrJmp = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); + pSort->labelBkOut = sqlite3VdbeMakeLabel(v); + pSort->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); + sqlite3VdbeJumpHere(v, addrFirst); + sqlite3VdbeAddOp3(v, OP_Move, regBase, regPrevKey, pSort->nOBSat); + sqlite3VdbeJumpHere(v, addrJmp); + } + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + op = OP_SorterInsert; + }else{ + op = OP_IdxInsert; + } + sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); + if( pSelect->iLimit ){ + int addr1, addr2; + int iLimit; + if( pSelect->iOffset ){ + iLimit = pSelect->iOffset+1; + }else{ + iLimit = pSelect->iLimit; + } + addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); + addr2 = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); + sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); + sqlite3VdbeJumpHere(v, addr2); + } +} + +/* +** Add code to implement the OFFSET +*/ +static void codeOffset( + Vdbe *v, /* Generate code into this VM */ + int iOffset, /* Register holding the offset counter */ + int iContinue /* Jump here to skip the current record */ +){ + if( iOffset>0 ){ + int addr; + addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); + VdbeComment((v, "skip OFFSET records")); + sqlite3VdbeJumpHere(v, addr); + } +} + +/* +** Add code that will check to make sure the N registers starting at iMem +** form a distinct entry. iTab is a sorting index that holds previously +** seen combinations of the N values. A new entry is made in iTab +** if the current N values are new. +** +** A jump to addrRepeat is made and the N+1 values are popped from the +** stack if the top N elements are not distinct. +*/ +static void codeDistinct( + Parse *pParse, /* Parsing and code generating context */ + int iTab, /* A sorting index used to test for distinctness */ + int addrRepeat, /* Jump to here if not distinct */ + int N, /* Number of elements */ + int iMem /* First element */ +){ + Vdbe *v; + int r1; + + v = pParse->pVdbe; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); + sqlite3ReleaseTempReg(pParse, r1); +} + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate an error message when a SELECT is used within a subexpression +** (example: "a IN (SELECT * FROM table)") but it has more than 1 result +** column. We do this in a subroutine because the error used to occur +** in multiple places. (The error only occurs in one place now, but we +** retain the subroutine to minimize code disruption.) +*/ +static int checkForMultiColumnSelectError( + Parse *pParse, /* Parse context. */ + SelectDest *pDest, /* Destination of SELECT results */ + int nExpr /* Number of result columns returned by SELECT */ +){ + int eDest = pDest->eDest; + if( nExpr>1 && (eDest==SRT_Mem || eDest==SRT_Set) ){ + sqlite3ErrorMsg(pParse, "only a single result allowed for " + "a SELECT that is part of an expression"); + return 1; + }else{ + return 0; + } +} +#endif + +/* +** This routine generates the code for the inside of the inner loop +** of a SELECT. +** +** If srcTab is negative, then the pEList expressions +** are evaluated in order to get the data for this row. If srcTab is +** zero or more, then data is pulled from srcTab and pEList is used only +** to get number columns and the datatype for each column. +*/ +static void selectInnerLoop( + Parse *pParse, /* The parser context */ + Select *p, /* The complete select statement being coded */ + ExprList *pEList, /* List of values being extracted */ + int srcTab, /* Pull data from this table */ + SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */ + DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ + SelectDest *pDest, /* How to dispose of the results */ + int iContinue, /* Jump here to continue with next row */ + int iBreak /* Jump here to break out of the inner loop */ +){ + Vdbe *v = pParse->pVdbe; + int i; + int hasDistinct; /* True if the DISTINCT keyword is present */ + int regResult; /* Start of memory holding result set */ + int eDest = pDest->eDest; /* How to dispose of results */ + int iParm = pDest->iSDParm; /* First argument to disposal method */ + int nResultCol; /* Number of result columns */ + + assert( v ); + assert( pEList!=0 ); + hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; + if( pSort && pSort->pOrderBy==0 ) pSort = 0; + if( pSort==0 && !hasDistinct ){ + assert( iContinue!=0 ); + codeOffset(v, p->iOffset, iContinue); + } + + /* Pull the requested columns. + */ + nResultCol = pEList->nExpr; + + if( pDest->iSdst==0 ){ + pDest->iSdst = pParse->nMem+1; + pParse->nMem += nResultCol; + }else if( pDest->iSdst+nResultCol > pParse->nMem ){ + /* This is an error condition that can result, for example, when a SELECT + ** on the right-hand side of an INSERT contains more result columns than + ** there are columns in the table on the left. The error will be caught + ** and reported later. But we need to make sure enough memory is allocated + ** to avoid other spurious errors in the meantime. */ + pParse->nMem += nResultCol; + } + pDest->nSdst = nResultCol; + regResult = pDest->iSdst; + if( srcTab>=0 ){ + for(i=0; ia[i].zName)); + } + }else if( eDest!=SRT_Exists ){ + /* If the destination is an EXISTS(...) expression, the actual + ** values returned by the SELECT are not required. + */ + sqlite3ExprCodeExprList(pParse, pEList, regResult, + (eDest==SRT_Output||eDest==SRT_Coroutine)?SQLITE_ECEL_DUP:0); + } + + /* If the DISTINCT keyword was present on the SELECT statement + ** and this row has been seen before, then do not make this row + ** part of the result. + */ + if( hasDistinct ){ + switch( pDistinct->eTnctType ){ + case WHERE_DISTINCT_ORDERED: { + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + int iJump; /* Jump destination */ + int regPrev; /* Previous row content */ + + /* Allocate space for the previous row */ + regPrev = pParse->nMem+1; + pParse->nMem += nResultCol; + + /* Change the OP_OpenEphemeral coded earlier to an OP_Null + ** sets the MEM_Cleared bit on the first register of the + ** previous value. This will cause the OP_Ne below to always + ** fail on the first iteration of the loop even if the first + ** row is all NULLs. + */ + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = regPrev; + + iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; + for(i=0; ia[i].pExpr); + if( idb->mallocFailed ); + sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nResultCol-1); + break; + } + + case WHERE_DISTINCT_UNIQUE: { + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + break; + } + + default: { + assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); + break; + } + } + if( pSort==0 ){ + codeOffset(v, p->iOffset, iContinue); + } + } + + switch( eDest ){ + /* In this mode, write each query result to the key of the temporary + ** table iParm. + */ +#ifndef SQLITE_OMIT_COMPOUND_SELECT + case SRT_Union: { + int r1; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + + /* Construct a record from the query result, but instead of + ** saving that record, use it as a key to delete elements from + ** the temporary table iParm. + */ + case SRT_Except: { + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); + break; + } +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + + /* Store the result as data using a unique key. + */ + case SRT_Fifo: + case SRT_DistFifo: + case SRT_Table: + case SRT_EphemTab: { + int r1 = sqlite3GetTempReg(pParse); + testcase( eDest==SRT_Table ); + testcase( eDest==SRT_EphemTab ); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); +#ifndef SQLITE_OMIT_CTE + if( eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open + ** on an ephemeral index. If the current row is already present + ** in the index, do not write it to the output. If not, add the + ** current row to the index and proceed with writing it to the + ** output table as well. */ + int addr = sqlite3VdbeCurrentAddr(v) + 4; + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); + assert( pSort==0 ); + } +#endif + if( pSort ){ + pushOntoSorter(pParse, pSort, p, r1); + }else{ + int r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); + } + sqlite3ReleaseTempReg(pParse, r1); + break; + } + +#ifndef SQLITE_OMIT_SUBQUERY + /* If we are creating a set for an "expr IN (SELECT ...)" construct, + ** then there should be a single item on the stack. Write this + ** item into the set table with bogus data. + */ + case SRT_Set: { + assert( nResultCol==1 ); + pDest->affSdst = + sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); + if( pSort ){ + /* At first glance you would think we could optimize out the + ** ORDER BY in this case since the order of entries in the set + ** does not matter. But there might be a LIMIT clause, in which + ** case the order does matter */ + pushOntoSorter(pParse, pSort, p, regResult); + }else{ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); + sqlite3ExprCacheAffinityChange(pParse, regResult, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3ReleaseTempReg(pParse, r1); + } + break; + } + + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, iParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } + + /* If this is a scalar select that is part of an expression, then + ** store the results in the appropriate memory cell and break out + ** of the scan loop. + */ + case SRT_Mem: { + assert( nResultCol==1 ); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, regResult); + }else{ + sqlite3ExprCodeMove(pParse, regResult, iParm, 1); + /* The LIMIT clause will jump out of the loop for us */ + } + break; + } +#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ + + case SRT_Coroutine: /* Send data to a co-routine */ + case SRT_Output: { /* Return the results */ + testcase( eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + if( pSort ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + pushOntoSorter(pParse, pSort, p, r1); + sqlite3ReleaseTempReg(pParse, r1); + }else if( eDest==SRT_Coroutine ){ + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); + }else{ + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); + sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); + } + break; + } + +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3; + int addrTest = 0; + ExprList *pSO; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + if( eDest==SRT_DistQueue ){ + /* If the destination is DistQueue, then cursor (iParm+1) is open + ** on a second ephemeral index that holds all values every previously + ** added to the queue. */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, + regResult, nResultCol); + VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + for(i=0; ia[i].u.x.iOrderByCol - 1, + r2+i); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + + +#if !defined(SQLITE_OMIT_TRIGGER) + /* Discard the results. This is used for SELECT statements inside + ** the body of a TRIGGER. The purpose of such selects is to call + ** user-defined functions that have side effects. We do not care + ** about the actual results of the select. + */ + default: { + assert( eDest==SRT_Discard ); + break; + } +#endif + } + + /* Jump to the end of the loop if the LIMIT is reached. Except, if + ** there is a sorter, in which case the sorter has already limited + ** the output for us. + */ + if( pSort==0 && p->iLimit ){ + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); + } +} + +/* +** Allocate a KeyInfo object sufficient for an index of N key columns and +** X extra columns. +*/ +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ + KeyInfo *p = sqlite3DbMallocZero(0, + sizeof(KeyInfo) + (N+X)*(sizeof(CollSeq*)+1)); + if( p ){ + p->aSortOrder = (u8*)&p->aColl[N+X]; + p->nField = (u16)N; + p->nXField = (u16)X; + p->enc = ENC(db); + p->db = db; + p->nRef = 1; + }else{ + db->mallocFailed = 1; + } + return p; +} + +/* +** Deallocate a KeyInfo object +*/ +SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef--; + if( p->nRef==0 ) sqlite3DbFree(0, p); + } +} + +/* +** Make a new pointer to a KeyInfo object +*/ +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef++; + } + return p; +} + +#ifdef SQLITE_DEBUG +/* +** Return TRUE if a KeyInfo object can be change. The KeyInfo object +** can only be changed if this is just a single reference to the object. +** +** This routine is used only inside of assert() statements. +*/ +SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } +#endif /* SQLITE_DEBUG */ + +/* +** Given an expression list, generate a KeyInfo structure that records +** the collating sequence for each expression in that expression list. +** +** If the ExprList is an ORDER BY or GROUP BY clause then the resulting +** KeyInfo structure is appropriate for initializing a virtual index to +** implement that clause. If the ExprList is the result set of a SELECT +** then the KeyInfo structure is appropriate for initializing a virtual +** index to implement a DISTINCT test. +** +** Space to hold the KeyInfo structure is obtain from malloc. The calling +** function is responsible for seeing that this structure is eventually +** freed. +*/ +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +){ + int nExpr; + KeyInfo *pInfo; + struct ExprList_item *pItem; + sqlite3 *db = pParse->db; + int i; + + nExpr = pList->nExpr; + pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra-iStart, 1); + if( pInfo ){ + assert( sqlite3KeyInfoIsWriteable(pInfo) ); + for(i=iStart, pItem=pList->a+iStart; ipExpr); + if( !pColl ) pColl = db->pDfltColl; + pInfo->aColl[i-iStart] = pColl; + pInfo->aSortOrder[i-iStart] = pItem->sortOrder; + } + } + return pInfo; +} + +#ifndef SQLITE_OMIT_COMPOUND_SELECT +/* +** Name of the connection operator, used for error messages. +*/ +static const char *selectOpName(int id){ + char *z; + switch( id ){ + case TK_ALL: z = "UNION ALL"; break; + case TK_INTERSECT: z = "INTERSECT"; break; + case TK_EXCEPT: z = "EXCEPT"; break; + default: z = "UNION"; break; + } + return z; +} +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of the form: +** +** "USE TEMP B-TREE FOR xxx" +** +** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which +** is determined by the zUsage argument. +*/ +static void explainTempTable(Parse *pParse, const char *zUsage){ + if( pParse->explain==2 ){ + Vdbe *v = pParse->pVdbe; + char *zMsg = sqlite3MPrintf(pParse->db, "USE TEMP B-TREE FOR %s", zUsage); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + } +} + +/* +** Assign expression b to lvalue a. A second, no-op, version of this macro +** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code +** in sqlite3Select() to assign values to structure member variables that +** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the +** code with #ifndef directives. +*/ +# define explainSetInteger(a, b) a = b + +#else +/* No-op versions of the explainXXX() functions and macros. */ +# define explainTempTable(y,z) +# define explainSetInteger(y,z) +#endif + +#if !defined(SQLITE_OMIT_EXPLAIN) && !defined(SQLITE_OMIT_COMPOUND_SELECT) +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of one of the two forms: +** +** "COMPOSITE SUBQUERIES iSub1 and iSub2 (op)" +** "COMPOSITE SUBQUERIES iSub1 and iSub2 USING TEMP B-TREE (op)" +** +** where iSub1 and iSub2 are the integers passed as the corresponding +** function parameters, and op is the text representation of the parameter +** of the same name. The parameter "op" must be one of TK_UNION, TK_EXCEPT, +** TK_INTERSECT or TK_ALL. The first form is used if argument bUseTmp is +** false, or the second form if it is true. +*/ +static void explainComposite( + Parse *pParse, /* Parse context */ + int op, /* One of TK_UNION, TK_EXCEPT etc. */ + int iSub1, /* Subquery id 1 */ + int iSub2, /* Subquery id 2 */ + int bUseTmp /* True if a temp table was used */ +){ + assert( op==TK_UNION || op==TK_EXCEPT || op==TK_INTERSECT || op==TK_ALL ); + if( pParse->explain==2 ){ + Vdbe *v = pParse->pVdbe; + char *zMsg = sqlite3MPrintf( + pParse->db, "COMPOUND SUBQUERIES %d AND %d %s(%s)", iSub1, iSub2, + bUseTmp?"USING TEMP B-TREE ":"", selectOpName(op) + ); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + } +} +#else +/* No-op versions of the explainXXX() functions and macros. */ +# define explainComposite(v,w,x,y,z) +#endif + +/* +** If the inner loop was generated using a non-null pOrderBy argument, +** then the results were placed in a sorter. After the loop is terminated +** we need to run the sorter and output the results. The following +** routine generates the code needed to do that. +*/ +static void generateSortTail( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + SortCtx *pSort, /* Information on the ORDER BY clause */ + int nColumn, /* Number of columns of data */ + SelectDest *pDest /* Write the sorted results here */ +){ + Vdbe *v = pParse->pVdbe; /* The prepared statement */ + int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ + int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ + int addr; + int addrOnce = 0; + int iTab; + int pseudoTab = 0; + ExprList *pOrderBy = pSort->pOrderBy; + int eDest = pDest->eDest; + int iParm = pDest->iSDParm; + int regRow; + int regRowid; + int nKey; + + if( pSort->labelBkOut ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); + sqlite3VdbeResolveLabel(v, pSort->labelBkOut); + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } + iTab = pSort->iECursor; + regRow = sqlite3GetTempReg(pParse); + if( eDest==SRT_Output || eDest==SRT_Coroutine ){ + pseudoTab = pParse->nTab++; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); + regRowid = 0; + }else{ + regRowid = sqlite3GetTempReg(pParse); + } + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + int regSortOut = ++pParse->nMem; + int ptab2 = pParse->nTab++; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); + addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); + VdbeCoverage(v); + codeOffset(v, p->iOffset, addrContinue); + sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); + sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow); + sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + }else{ + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); + codeOffset(v, p->iOffset, addrContinue); + sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); + } + switch( eDest ){ + case SRT_Table: + case SRT_EphemTab: { + testcase( eDest==SRT_Table ); + testcase( eDest==SRT_EphemTab ); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case SRT_Set: { + assert( nColumn==1 ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, + &pDest->affSdst, 1); + sqlite3ExprCacheAffinityChange(pParse, regRow, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); + break; + } + case SRT_Mem: { + assert( nColumn==1 ); + sqlite3ExprCodeMove(pParse, regRow, iParm, 1); + /* The LIMIT clause will terminate the loop for us */ + break; + } +#endif + default: { + int i; + assert( eDest==SRT_Output || eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + testcase( eDest==SRT_Coroutine ); + for(i=0; iiSdst+i ); + sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iSdst+i); + if( i==0 ){ + sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + } + } + if( eDest==SRT_Output ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); + sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); + }else{ + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); + } + break; + } + } + sqlite3ReleaseTempReg(pParse, regRow); + sqlite3ReleaseTempReg(pParse, regRowid); + + /* The bottom of the loop + */ + sqlite3VdbeResolveLabel(v, addrContinue); + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); + } + if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); + sqlite3VdbeResolveLabel(v, addrBreak); +} + +/* +** Return a pointer to a string containing the 'declaration type' of the +** expression pExpr. The string may be treated as static by the caller. +** +** Also try to estimate the size of the returned value and return that +** result in *pEstWidth. +** +** The declaration type is the exact datatype definition extracted from the +** original CREATE TABLE statement if the expression is a column. The +** declaration type for a ROWID field is INTEGER. Exactly when an expression +** is considered a column can be complex in the presence of subqueries. The +** result-set expression in all of the following SELECT statements is +** considered a column by this function. +** +** SELECT col FROM tbl; +** SELECT (SELECT col FROM tbl; +** SELECT (SELECT col FROM tbl); +** SELECT abc FROM (SELECT col AS abc FROM tbl); +** +** The declaration type for any expression other than a column is NULL. +** +** This routine has either 3 or 6 parameters depending on whether or not +** the SQLITE_ENABLE_COLUMN_METADATA compile-time option is used. +*/ +#ifdef SQLITE_ENABLE_COLUMN_METADATA +# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,C,D,E,F) +static const char *columnTypeImpl( + NameContext *pNC, + Expr *pExpr, + const char **pzOrigDb, + const char **pzOrigTab, + const char **pzOrigCol, + u8 *pEstWidth +){ + char const *zOrigDb = 0; + char const *zOrigTab = 0; + char const *zOrigCol = 0; +#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ +# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) +static const char *columnTypeImpl( + NameContext *pNC, + Expr *pExpr, + u8 *pEstWidth +){ +#endif /* !defined(SQLITE_ENABLE_COLUMN_METADATA) */ + char const *zType = 0; + int j; + u8 estWidth = 1; + + if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; + switch( pExpr->op ){ + case TK_AGG_COLUMN: + case TK_COLUMN: { + /* The expression is a column. Locate the table the column is being + ** extracted from in NameContext.pSrcList. This table may be real + ** database table or a subquery. + */ + Table *pTab = 0; /* Table structure column is extracted from */ + Select *pS = 0; /* Select the column is extracted from */ + int iCol = pExpr->iColumn; /* Index of column in pTab */ + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_COLUMN ); + while( pNC && !pTab ){ + SrcList *pTabList = pNC->pSrcList; + for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); + if( jnSrc ){ + pTab = pTabList->a[j].pTab; + pS = pTabList->a[j].pSelect; + }else{ + pNC = pNC->pNext; + } + } + + if( pTab==0 ){ + /* At one time, code such as "SELECT new.x" within a trigger would + ** cause this condition to run. Since then, we have restructured how + ** trigger code is generated and so this condition is no longer + ** possible. However, it can still be true for statements like + ** the following: + ** + ** CREATE TABLE t1(col INTEGER); + ** SELECT (SELECT t1.col) FROM FROM t1; + ** + ** when columnType() is called on the expression "t1.col" in the + ** sub-select. In this case, set the column type to NULL, even + ** though it should really be "INTEGER". + ** + ** This is not a problem, as the column type of "t1.col" is never + ** used. When columnType() is called on the expression + ** "(SELECT t1.col)", the correct type is returned (see the TK_SELECT + ** branch below. */ + break; + } + + assert( pTab && pExpr->pTab==pTab ); + if( pS ){ + /* The "table" is actually a sub-select or a view in the FROM clause + ** of the SELECT statement. Return the declaration type and origin + ** data for the result-set column of the sub-select. + */ + if( iCol>=0 && ALWAYS(iColpEList->nExpr) ){ + /* If iCol is less than zero, then the expression requests the + ** rowid of the sub-select or view. This expression is legal (see + ** test case misc2.2.2) - it always evaluates to NULL. + */ + NameContext sNC; + Expr *p = pS->pEList->a[iCol].pExpr; + sNC.pSrcList = pS->pSrc; + sNC.pNext = pNC; + sNC.pParse = pNC->pParse; + zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol, &estWidth); + } + }else if( pTab->pSchema ){ + /* A real table */ + assert( !pS ); + if( iCol<0 ) iCol = pTab->iPKey; + assert( iCol==-1 || (iCol>=0 && iColnCol) ); +#ifdef SQLITE_ENABLE_COLUMN_METADATA + if( iCol<0 ){ + zType = "INTEGER"; + zOrigCol = "rowid"; + }else{ + zType = pTab->aCol[iCol].zType; + zOrigCol = pTab->aCol[iCol].zName; + estWidth = pTab->aCol[iCol].szEst; + } + zOrigTab = pTab->zName; + if( pNC->pParse ){ + int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); + zOrigDb = pNC->pParse->db->aDb[iDb].zName; + } +#else + if( iCol<0 ){ + zType = "INTEGER"; + }else{ + zType = pTab->aCol[iCol].zType; + estWidth = pTab->aCol[iCol].szEst; + } +#endif + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_SELECT: { + /* The expression is a sub-select. Return the declaration type and + ** origin info for the single column in the result set of the SELECT + ** statement. + */ + NameContext sNC; + Select *pS = pExpr->x.pSelect; + Expr *p = pS->pEList->a[0].pExpr; + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + sNC.pSrcList = pS->pSrc; + sNC.pNext = pNC; + sNC.pParse = pNC->pParse; + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol, &estWidth); + break; + } +#endif + } + +#ifdef SQLITE_ENABLE_COLUMN_METADATA + if( pzOrigDb ){ + assert( pzOrigTab && pzOrigCol ); + *pzOrigDb = zOrigDb; + *pzOrigTab = zOrigTab; + *pzOrigCol = zOrigCol; + } +#endif + if( pEstWidth ) *pEstWidth = estWidth; + return zType; +} + +/* +** Generate code that will tell the VDBE the declaration types of columns +** in the result set. +*/ +static void generateColumnTypes( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* List of tables */ + ExprList *pEList /* Expressions defining the result set */ +){ +#ifndef SQLITE_OMIT_DECLTYPE + Vdbe *v = pParse->pVdbe; + int i; + NameContext sNC; + sNC.pSrcList = pTabList; + sNC.pParse = pParse; + for(i=0; inExpr; i++){ + Expr *p = pEList->a[i].pExpr; + const char *zType; +#ifdef SQLITE_ENABLE_COLUMN_METADATA + const char *zOrigDb = 0; + const char *zOrigTab = 0; + const char *zOrigCol = 0; + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol, 0); + + /* The vdbe must make its own copy of the column-type and other + ** column specific strings, in case the schema is reset before this + ** virtual machine is deleted. + */ + sqlite3VdbeSetColName(v, i, COLNAME_DATABASE, zOrigDb, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT); +#else + zType = columnType(&sNC, p, 0, 0, 0, 0); +#endif + sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); + } +#endif /* !defined(SQLITE_OMIT_DECLTYPE) */ +} + +/* +** Generate code that will tell the VDBE the names of columns +** in the result set. This information is used to provide the +** azCol[] values in the callback. +*/ +static void generateColumnNames( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* List of tables */ + ExprList *pEList /* Expressions defining the result set */ +){ + Vdbe *v = pParse->pVdbe; + int i, j; + sqlite3 *db = pParse->db; + int fullNames, shortNames; + +#ifndef SQLITE_OMIT_EXPLAIN + /* If this is an EXPLAIN, skip this step */ + if( pParse->explain ){ + return; + } +#endif + + if( pParse->colNamesSet || NEVER(v==0) || db->mallocFailed ) return; + pParse->colNamesSet = 1; + fullNames = (db->flags & SQLITE_FullColNames)!=0; + shortNames = (db->flags & SQLITE_ShortColNames)!=0; + sqlite3VdbeSetNumCols(v, pEList->nExpr); + for(i=0; inExpr; i++){ + Expr *p; + p = pEList->a[i].pExpr; + if( NEVER(p==0) ) continue; + if( pEList->a[i].zName ){ + char *zName = pEList->a[i].zName; + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); + }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && pTabList ){ + Table *pTab; + char *zCol; + int iCol = p->iColumn; + for(j=0; ALWAYS(jnSrc); j++){ + if( pTabList->a[j].iCursor==p->iTable ) break; + } + assert( jnSrc ); + pTab = pTabList->a[j].pTab; + if( iCol<0 ) iCol = pTab->iPKey; + assert( iCol==-1 || (iCol>=0 && iColnCol) ); + if( iCol<0 ){ + zCol = "rowid"; + }else{ + zCol = pTab->aCol[iCol].zName; + } + if( !shortNames && !fullNames ){ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, + sqlite3DbStrDup(db, pEList->a[i].zSpan), SQLITE_DYNAMIC); + }else if( fullNames ){ + char *zName = 0; + zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC); + }else{ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); + } + }else{ + const char *z = pEList->a[i].zSpan; + z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC); + } + } + generateColumnTypes(pParse, pTabList, pEList); +} + +/* +** Given a an expression list (which is really the list of expressions +** that form the result set of a SELECT statement) compute appropriate +** column names for a table that would hold the expression list. +** +** All column names will be unique. +** +** Only the column names are computed. Column.zType, Column.zColl, +** and other fields of Column are zeroed. +** +** Return SQLITE_OK on success. If a memory allocation error occurs, +** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. +*/ +static int selectColumnsFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* Expr list from which to derive column names */ + i16 *pnCol, /* Write the number of columns here */ + Column **paCol /* Write the new column list here */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + int i, j; /* Loop counters */ + int cnt; /* Index added to make the name unique */ + Column *aCol, *pCol; /* For looping over result columns */ + int nCol; /* Number of columns in the result set */ + Expr *p; /* Expression for a single result column */ + char *zName; /* Column name */ + int nName; /* Size of name in zName[] */ + + if( pEList ){ + nCol = pEList->nExpr; + aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); + testcase( aCol==0 ); + }else{ + nCol = 0; + aCol = 0; + } + *pnCol = nCol; + *paCol = aCol; + + for(i=0, pCol=aCol; ia[i].pExpr); + if( (zName = pEList->a[i].zName)!=0 ){ + /* If the column contains an "AS " phrase, use as the name */ + zName = sqlite3DbStrDup(db, zName); + }else{ + Expr *pColExpr = p; /* The expression that is the result column name */ + Table *pTab; /* Table associated with this expression */ + while( pColExpr->op==TK_DOT ){ + pColExpr = pColExpr->pRight; + assert( pColExpr!=0 ); + } + if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){ + /* For columns use the column name name */ + int iCol = pColExpr->iColumn; + pTab = pColExpr->pTab; + if( iCol<0 ) iCol = pTab->iPKey; + zName = sqlite3MPrintf(db, "%s", + iCol>=0 ? pTab->aCol[iCol].zName : "rowid"); + }else if( pColExpr->op==TK_ID ){ + assert( !ExprHasProperty(pColExpr, EP_IntValue) ); + zName = sqlite3MPrintf(db, "%s", pColExpr->u.zToken); + }else{ + /* Use the original text of the column expression as its name */ + zName = sqlite3MPrintf(db, "%s", pEList->a[i].zSpan); + } + } + if( db->mallocFailed ){ + sqlite3DbFree(db, zName); + break; + } + + /* Make sure the column name is unique. If the name is not unique, + ** append a integer to the name so that it becomes unique. + */ + nName = sqlite3Strlen30(zName); + for(j=cnt=0; j1 && sqlite3Isdigit(zName[k]); k--){} + if( k>=0 && zName[k]==':' ) nName = k; + zName[nName] = 0; + zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); + sqlite3DbFree(db, zName); + zName = zNewName; + j = -1; + if( zName==0 ) break; + } + } + pCol->zName = zName; + } + if( db->mallocFailed ){ + for(j=0; jdb; + NameContext sNC; + Column *pCol; + CollSeq *pColl; + int i; + Expr *p; + struct ExprList_item *a; + u64 szAll = 0; + + assert( pSelect!=0 ); + assert( (pSelect->selFlags & SF_Resolved)!=0 ); + assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); + if( db->mallocFailed ) return; + memset(&sNC, 0, sizeof(sNC)); + sNC.pSrcList = pSelect->pSrc; + a = pSelect->pEList->a; + for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + p = a[i].pExpr; + pCol->zType = sqlite3DbStrDup(db, columnType(&sNC, p,0,0,0, &pCol->szEst)); + szAll += pCol->szEst; + pCol->affinity = sqlite3ExprAffinity(p); + if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_NONE; + pColl = sqlite3ExprCollSeq(pParse, p); + if( pColl ){ + pCol->zColl = sqlite3DbStrDup(db, pColl->zName); + } + } + pTab->szTabRow = sqlite3LogEst(szAll*4); +} + +/* +** Given a SELECT statement, generate a Table structure that describes +** the result set of that SELECT. +*/ +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ + Table *pTab; + sqlite3 *db = pParse->db; + int savedFlags; + + savedFlags = db->flags; + db->flags &= ~SQLITE_FullColNames; + db->flags |= SQLITE_ShortColNames; + sqlite3SelectPrep(pParse, pSelect, 0); + if( pParse->nErr ) return 0; + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + db->flags = savedFlags; + pTab = sqlite3DbMallocZero(db, sizeof(Table) ); + if( pTab==0 ){ + return 0; + } + /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside + ** is disabled */ + assert( db->lookaside.bEnabled==0 ); + pTab->nRef = 1; + pTab->zName = 0; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + selectAddColumnTypeAndCollation(pParse, pTab, pSelect); + pTab->iPKey = -1; + if( db->mallocFailed ){ + sqlite3DeleteTable(db, pTab); + return 0; + } + return pTab; +} + +/* +** Get a VDBE for the given parser context. Create a new one if necessary. +** If an error occurs, return NULL and leave a message in pParse. +*/ +SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ + Vdbe *v = pParse->pVdbe; + if( v==0 ){ + v = pParse->pVdbe = sqlite3VdbeCreate(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Init); + if( pParse->pToplevel==0 + && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) + ){ + pParse->okConstFactor = 1; + } + + } + return v; +} + + +/* +** Compute the iLimit and iOffset fields of the SELECT based on the +** pLimit and pOffset expressions. pLimit and pOffset hold the expressions +** that appear in the original SQL statement after the LIMIT and OFFSET +** keywords. Or NULL if those keywords are omitted. iLimit and iOffset +** are the integer memory register numbers for counters used to compute +** the limit and offset. If there is no limit and/or offset, then +** iLimit and iOffset are negative. +** +** This routine changes the values of iLimit and iOffset only if +** a limit or offset is defined by pLimit and pOffset. iLimit and +** iOffset should have been preset to appropriate default values (zero) +** prior to calling this routine. +** +** The iOffset register (if it exists) is initialized to the value +** of the OFFSET. The iLimit register is initialized to LIMIT. Register +** iOffset+1 is initialized to LIMIT+OFFSET. +** +** Only if pLimit!=0 or pOffset!=0 do the limit registers get +** redefined. The UNION ALL operator uses this property to force +** the reuse of the same limit and offset registers across multiple +** SELECT statements. +*/ +static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ + Vdbe *v = 0; + int iLimit = 0; + int iOffset; + int addr1, n; + if( p->iLimit ) return; + + /* + ** "LIMIT -1" always shows all rows. There is some + ** controversy about what the correct behavior should be. + ** The current implementation interprets "LIMIT 0" to mean + ** no rows. + */ + sqlite3ExprCacheClear(pParse); + assert( p->pOffset==0 || p->pLimit!=0 ); + if( p->pLimit ){ + p->iLimit = iLimit = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + if( sqlite3ExprIsInteger(p->pLimit, &n) ){ + sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); + VdbeComment((v, "LIMIT counter")); + if( n==0 ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + }else if( n>=0 && p->nSelectRow>(u64)n ){ + p->nSelectRow = n; + } + }else{ + sqlite3ExprCode(pParse, p->pLimit, iLimit); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); + VdbeComment((v, "LIMIT counter")); + sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); VdbeCoverage(v); + } + if( p->pOffset ){ + p->iOffset = iOffset = ++pParse->nMem; + pParse->nMem++; /* Allocate an extra register for limit+offset */ + sqlite3ExprCode(pParse, p->pOffset, iOffset); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); + VdbeComment((v, "OFFSET counter")); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); + VdbeComment((v, "LIMIT+OFFSET")); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); + sqlite3VdbeJumpHere(v, addr1); + } + } +} + +#ifndef SQLITE_OMIT_COMPOUND_SELECT +/* +** Return the appropriate collating sequence for the iCol-th column of +** the result set for the compound-select statement "p". Return NULL if +** the column has no default collating sequence. +** +** The collating sequence for the compound select is taken from the +** left-most term of the select that has a collating sequence. +*/ +static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ + CollSeq *pRet; + if( p->pPrior ){ + pRet = multiSelectCollSeq(pParse, p->pPrior, iCol); + }else{ + pRet = 0; + } + assert( iCol>=0 ); + if( pRet==0 && iColpEList->nExpr ){ + pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr); + } + return pRet; +} + +/* +** The select statement passed as the second parameter is a compound SELECT +** with an ORDER BY clause. This function allocates and returns a KeyInfo +** structure suitable for implementing the ORDER BY. +** +** Space to hold the KeyInfo structure is obtained from malloc. The calling +** function is responsible for ensuring that this structure is eventually +** freed. +*/ +static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ + ExprList *pOrderBy = p->pOrderBy; + int nOrderBy = p->pOrderBy->nExpr; + sqlite3 *db = pParse->db; + KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1); + if( pRet ){ + int i; + for(i=0; ia[i]; + Expr *pTerm = pItem->pExpr; + CollSeq *pColl; + + if( pTerm->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pTerm); + }else{ + pColl = multiSelectCollSeq(pParse, p, pItem->u.x.iOrderByCol-1); + if( pColl==0 ) pColl = db->pDfltColl; + pOrderBy->a[i].pExpr = + sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName); + } + assert( sqlite3KeyInfoIsWriteable(pRet) ); + pRet->aColl[i] = pColl; + pRet->aSortOrder[i] = pOrderBy->a[i].sortOrder; + } + } + + return pRet; +} + +#ifndef SQLITE_OMIT_CTE +/* +** This routine generates VDBE code to compute the content of a WITH RECURSIVE +** query of the form: +** +** AS ( UNION [ALL] ) +** \___________/ \_______________/ +** p->pPrior p +** +** +** There is exactly one reference to the recursive-table in the FROM clause +** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** +** The setup-query runs once to generate an initial set of rows that go +** into a Queue table. Rows are extracted from the Queue table one by +** one. Each row extracted from Queue is output to pDest. Then the single +** extracted row (now in the iCurrent table) becomes the content of the +** recursive-table for a recursive-query run. The output of the recursive-query +** is added back into the Queue table. Then another row is extracted from Queue +** and the iteration continues until the Queue table is empty. +** +** If the compound query operator is UNION then no duplicate rows are ever +** inserted into the Queue table. The iDistinct table keeps a copy of all rows +** that have ever been inserted into Queue and causes duplicates to be +** discarded. If the operator is UNION ALL, then duplicates are allowed. +** +** If the query has an ORDER BY, then entries in the Queue table are kept in +** ORDER BY order and the first entry is extracted for each cycle. Without +** an ORDER BY, the Queue table is just a FIFO. +** +** If a LIMIT clause is provided, then the iteration stops after LIMIT rows +** have been output to pDest. A LIMIT of zero means to output no rows and a +** negative LIMIT means to output all rows. If there is also an OFFSET clause +** with a positive value, then the first OFFSET outputs are discarded rather +** than being sent to pDest. The LIMIT count does not begin until after OFFSET +** rows have been skipped. +*/ +static void generateWithRecursiveQuery( + Parse *pParse, /* Parsing context */ + Select *p, /* The recursive SELECT to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ + int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Select *pSetup = p->pPrior; /* The setup query */ + int addrTop; /* Top of the loop */ + int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ + int iCurrent = 0; /* The Current table */ + int regCurrent; /* Register holding Current table */ + int iQueue; /* The Queue table */ + int iDistinct = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Fifo; /* How to write to Queue */ + SelectDest destQueue; /* SelectDest targetting the Queue table */ + int i; /* Loop counter */ + int rc; /* Result code */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Expr *pLimit, *pOffset; /* Saved LIMIT and OFFSET */ + int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ + + /* Obtain authorization to do a recursive query */ + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; + + /* Process the LIMIT and OFFSET clauses, if they exist */ + addrBreak = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, addrBreak); + pLimit = p->pLimit; + pOffset = p->pOffset; + regLimit = p->iLimit; + regOffset = p->iOffset; + p->pLimit = p->pOffset = 0; + p->iLimit = p->iOffset = 0; + pOrderBy = p->pOrderBy; + + /* Locate the cursor number of the Current table */ + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].isRecursive ){ + iCurrent = pSrc->a[i].iCursor; + break; + } + } + + /* Allocate cursors numbers for Queue and Distinct. The cursor number for + ** the Distinct table must be exactly one greater than Queue in order + ** for the SRT_DistFifo and SRT_DistQueue destinations to work. */ + iQueue = pParse->nTab++; + if( p->op==TK_UNION ){ + eDest = pOrderBy ? SRT_DistQueue : SRT_DistFifo; + iDistinct = pParse->nTab++; + }else{ + eDest = pOrderBy ? SRT_Queue : SRT_Fifo; + } + sqlite3SelectDestInit(&destQueue, eDest, iQueue); + + /* Allocate cursors for Current, Queue, and Distinct. */ + regCurrent = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); + if( pOrderBy ){ + KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO); + destQueue.pOrderBy = pOrderBy; + }else{ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + } + VdbeComment((v, "Queue table")); + if( iDistinct ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Detach the ORDER BY clause from the compound SELECT */ + p->pOrderBy = 0; + + /* Store the results of the setup-query in Queue. */ + pSetup->pNext = 0; + rc = sqlite3Select(pParse, pSetup, &destQueue); + pSetup->pNext = p; + if( rc ) goto end_of_recursive_query; + + /* Find the next row in the Queue and output that row */ + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); VdbeCoverage(v); + + /* Transfer the next row in Queue over to Current */ + sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ + if( pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + } + sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + + /* Output the single row in Current */ + addrCont = sqlite3VdbeMakeLabel(v); + codeOffset(v, regOffset, addrCont); + selectInnerLoop(pParse, p, p->pEList, iCurrent, + 0, 0, pDest, addrCont, addrBreak); + if( regLimit ){ + sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); + VdbeCoverage(v); + } + sqlite3VdbeResolveLabel(v, addrCont); + + /* Execute the recursive SELECT taking the single row in Current as + ** the value for the recursive-table. Store the results in the Queue. + */ + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; + + /* Keep running the loop until the Queue is empty */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeResolveLabel(v, addrBreak); + +end_of_recursive_query: + sqlite3ExprListDelete(pParse->db, p->pOrderBy); + p->pOrderBy = pOrderBy; + p->pLimit = pLimit; + p->pOffset = pOffset; + return; +} +#endif /* SQLITE_OMIT_CTE */ + +/* Forward references */ +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +); + + +/* +** This routine is called to process a compound query form from +** two or more separate queries using UNION, UNION ALL, EXCEPT, or +** INTERSECT +** +** "p" points to the right-most of the two queries. the query on the +** left is p->pPrior. The left query could also be a compound query +** in which case this routine will be called recursively. +** +** The results of the total query are to be written into a destination +** of type eDest with parameter iParm. +** +** Example 1: Consider a three-way compound SQL statement. +** +** SELECT a FROM t1 UNION SELECT b FROM t2 UNION SELECT c FROM t3 +** +** This statement is parsed up as follows: +** +** SELECT c FROM t3 +** | +** `-----> SELECT b FROM t2 +** | +** `------> SELECT a FROM t1 +** +** The arrows in the diagram above represent the Select.pPrior pointer. +** So if this routine is called with p equal to the t3 query, then +** pPrior will be the t2 query. p->op will be TK_UNION in this case. +** +** Notice that because of the way SQLite parses compound SELECTs, the +** individual selects always group from left to right. +*/ +static int multiSelect( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int rc = SQLITE_OK; /* Success code from a subroutine */ + Select *pPrior; /* Another SELECT immediately to our left */ + Vdbe *v; /* Generate code to this VDBE */ + SelectDest dest; /* Alternative data destination */ + Select *pDelete = 0; /* Chain of simple selects to delete */ + sqlite3 *db; /* Database connection */ +#ifndef SQLITE_OMIT_EXPLAIN + int iSub1 = 0; /* EQP id of left-hand query */ + int iSub2 = 0; /* EQP id of right-hand query */ +#endif + + /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only + ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. + */ + assert( p && p->pPrior ); /* Calling function guarantees this much */ + assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); + db = pParse->db; + pPrior = p->pPrior; + dest = *pDest; + if( pPrior->pOrderBy ){ + sqlite3ErrorMsg(pParse,"ORDER BY clause should come after %s not before", + selectOpName(p->op)); + rc = 1; + goto multi_select_end; + } + if( pPrior->pLimit ){ + sqlite3ErrorMsg(pParse,"LIMIT clause should come after %s not before", + selectOpName(p->op)); + rc = 1; + goto multi_select_end; + } + + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); /* The VDBE already created by calling function */ + + /* Create the destination temporary table if necessary + */ + if( dest.eDest==SRT_EphemTab ){ + assert( p->pEList ); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + dest.eDest = SRT_Table; + } + + /* Make sure all SELECTs in the statement have the same number of elements + ** in their result sets. + */ + assert( p->pEList && pPrior->pEList ); + if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ + if( p->selFlags & SF_Values ){ + sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); + } + rc = 1; + goto multi_select_end; + } + +#ifndef SQLITE_OMIT_CTE + if( p->selFlags & SF_Recursive ){ + generateWithRecursiveQuery(pParse, p, &dest); + }else +#endif + + /* Compound SELECTs that have an ORDER BY clause are handled separately. + */ + if( p->pOrderBy ){ + return multiSelectOrderBy(pParse, p, pDest); + }else + + /* Generate code for the left and right SELECT statements. + */ + switch( p->op ){ + case TK_ALL: { + int addr = 0; + int nLimit; + assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; + pPrior->pLimit = p->pLimit; + pPrior->pOffset = p->pOffset; + explainSetInteger(iSub1, pParse->iNextSelectId); + rc = sqlite3Select(pParse, pPrior, &dest); + p->pLimit = 0; + p->pOffset = 0; + if( rc ){ + goto multi_select_end; + } + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); VdbeCoverage(v); + VdbeComment((v, "Jump ahead if LIMIT reached")); + } + explainSetInteger(iSub2, pParse->iNextSelectId); + rc = sqlite3Select(pParse, p, &dest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->nSelectRow += pPrior->nSelectRow; + if( pPrior->pLimit + && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) + && nLimit>0 && p->nSelectRow > (u64)nLimit + ){ + p->nSelectRow = nLimit; + } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + } + break; + } + case TK_EXCEPT: + case TK_UNION: { + int unionTab; /* Cursor number of the temporary table holding result */ + u8 op = 0; /* One of the SRT_ operations to apply to self */ + int priorOp; /* The SRT_ operation to apply to prior selects */ + Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */ + int addr; + SelectDest uniondest; + + testcase( p->op==TK_EXCEPT ); + testcase( p->op==TK_UNION ); + priorOp = SRT_Union; + if( dest.eDest==priorOp ){ + /* We can reuse a temporary table generated by a SELECT to our + ** right. + */ + assert( p->pLimit==0 ); /* Not allowed on leftward elements */ + assert( p->pOffset==0 ); /* Not allowed on leftward elements */ + unionTab = dest.iSDParm; + }else{ + /* We will need to create our own temporary table to hold the + ** intermediate results. + */ + unionTab = pParse->nTab++; + assert( p->pOrderBy==0 ); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + findRightmost(p)->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + } + + /* Code the SELECT statements to our left + */ + assert( !pPrior->pOrderBy ); + sqlite3SelectDestInit(&uniondest, priorOp, unionTab); + explainSetInteger(iSub1, pParse->iNextSelectId); + rc = sqlite3Select(pParse, pPrior, &uniondest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT statement + */ + if( p->op==TK_EXCEPT ){ + op = SRT_Except; + }else{ + assert( p->op==TK_UNION ); + op = SRT_Union; + } + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + pOffset = p->pOffset; + p->pOffset = 0; + uniondest.eDest = op; + explainSetInteger(iSub2, pParse->iNextSelectId); + rc = sqlite3Select(pParse, p, &uniondest); + testcase( rc!=SQLITE_OK ); + /* Query flattening in sqlite3Select() might refill p->pOrderBy. + ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */ + sqlite3ExprListDelete(db, p->pOrderBy); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->pOrderBy = 0; + if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->pOffset = pOffset; + p->iLimit = 0; + p->iOffset = 0; + + /* Convert the data in the temporary table into whatever form + ** it is that we currently need. + */ + assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); + if( dest.eDest!=priorOp ){ + int iCont, iBreak, iStart; + assert( p->pEList ); + if( dest.eDest==SRT_Output ){ + Select *pFirst = p; + while( pFirst->pPrior ) pFirst = pFirst->pPrior; + generateColumnNames(pParse, 0, pFirst->pEList); + } + iBreak = sqlite3VdbeMakeLabel(v); + iCont = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); + iStart = sqlite3VdbeCurrentAddr(v); + selectInnerLoop(pParse, p, p->pEList, unionTab, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); + } + break; + } + default: assert( p->op==TK_INTERSECT ); { + int tab1, tab2; + int iCont, iBreak, iStart; + Expr *pLimit, *pOffset; + int addr; + SelectDest intersectdest; + int r1; + + /* INTERSECT is different from the others since it requires + ** two temporary tables. Hence it has its own case. Begin + ** by allocating the tables we will need. + */ + tab1 = pParse->nTab++; + tab2 = pParse->nTab++; + assert( p->pOrderBy==0 ); + + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + findRightmost(p)->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + + /* Code the SELECTs to our left into temporary table "tab1". + */ + sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); + explainSetInteger(iSub1, pParse->iNextSelectId); + rc = sqlite3Select(pParse, pPrior, &intersectdest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT into temporary table "tab2" + */ + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); + assert( p->addrOpenEphm[1] == -1 ); + p->addrOpenEphm[1] = addr; + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + pOffset = p->pOffset; + p->pOffset = 0; + intersectdest.iSDParm = tab2; + explainSetInteger(iSub2, pParse->iNextSelectId); + rc = sqlite3Select(pParse, p, &intersectdest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->pOffset = pOffset; + + /* Generate code to take the intersection of the two temporary + ** tables. + */ + assert( p->pEList ); + if( dest.eDest==SRT_Output ){ + Select *pFirst = p; + while( pFirst->pPrior ) pFirst = pFirst->pPrior; + generateColumnNames(pParse, 0, pFirst->pEList); + } + iBreak = sqlite3VdbeMakeLabel(v); + iCont = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); + r1 = sqlite3GetTempReg(pParse); + iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); + sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, r1); + selectInnerLoop(pParse, p, p->pEList, tab1, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); + break; + } + } + + explainComposite(pParse, p->op, iSub1, iSub2, p->op!=TK_ALL); + + /* Compute collating sequences used by + ** temporary tables needed to implement the compound select. + ** Attach the KeyInfo structure to all temporary tables. + ** + ** This section is run by the right-most SELECT statement only. + ** SELECT statements to the left always skip this part. The right-most + ** SELECT might also skip this part if it has no ORDER BY clause and + ** no temp tables are required. + */ + if( p->selFlags & SF_UsesEphemeral ){ + int i; /* Loop counter */ + KeyInfo *pKeyInfo; /* Collating sequence for the result set */ + Select *pLoop; /* For looping through SELECT statements */ + CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ + int nCol; /* Number of columns in result set */ + + assert( p->pNext==0 ); + nCol = p->pEList->nExpr; + pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); + if( !pKeyInfo ){ + rc = SQLITE_NOMEM; + goto multi_select_end; + } + for(i=0, apColl=pKeyInfo->aColl; ipDfltColl; + } + } + + for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ + for(i=0; i<2; i++){ + int addr = pLoop->addrOpenEphm[i]; + if( addr<0 ){ + /* If [0] is unused then [1] is also unused. So we can + ** always safely abort as soon as the first unused slot is found */ + assert( pLoop->addrOpenEphm[1]<0 ); + break; + } + sqlite3VdbeChangeP2(v, addr, nCol); + sqlite3VdbeChangeP4(v, addr, (char*)sqlite3KeyInfoRef(pKeyInfo), + P4_KEYINFO); + pLoop->addrOpenEphm[i] = -1; + } + } + sqlite3KeyInfoUnref(pKeyInfo); + } + +multi_select_end: + pDest->iSdst = dest.iSdst; + pDest->nSdst = dest.nSdst; + sqlite3SelectDelete(db, pDelete); + return rc; +} +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + +/* +** Code an output subroutine for a coroutine implementation of a +** SELECT statment. +** +** The data to be output is contained in pIn->iSdst. There are +** pIn->nSdst columns to be output. pDest is where the output should +** be sent. +** +** regReturn is the number of the register holding the subroutine +** return address. +** +** If regPrev>0 then it is the first register in a vector that +** records the previous output. mem[regPrev] is a flag that is false +** if there has been no previous output. If regPrev>0 then code is +** generated to suppress duplicates. pKeyInfo is used for comparing +** keys. +** +** If the LIMIT found in p->iLimit is reached, jump immediately to +** iBreak. +*/ +static int generateOutputSubroutine( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + SelectDest *pIn, /* Coroutine supplying data */ + SelectDest *pDest, /* Where to send the data */ + int regReturn, /* The return address register */ + int regPrev, /* Previous result register. No uniqueness if 0 */ + KeyInfo *pKeyInfo, /* For comparing with previous entry */ + int iBreak /* Jump here if we hit the LIMIT */ +){ + Vdbe *v = pParse->pVdbe; + int iContinue; + int addr; + + addr = sqlite3VdbeCurrentAddr(v); + iContinue = sqlite3VdbeMakeLabel(v); + + /* Suppress duplicates for UNION, EXCEPT, and INTERSECT + */ + if( regPrev ){ + int j1, j2; + j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); + } + if( pParse->db->mallocFailed ) return 0; + + /* Suppress the first OFFSET entries if there is an OFFSET clause + */ + codeOffset(v, p->iOffset, iContinue); + + switch( pDest->eDest ){ + /* Store the result as data using a unique key. + */ + case SRT_Table: + case SRT_EphemTab: { + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3GetTempReg(pParse); + testcase( pDest->eDest==SRT_Table ); + testcase( pDest->eDest==SRT_EphemTab ); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); + sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + +#ifndef SQLITE_OMIT_SUBQUERY + /* If we are creating a set for an "expr IN (SELECT ...)" construct, + ** then there should be a single item on the stack. Write this + ** item into the set table with bogus data. + */ + case SRT_Set: { + int r1; + assert( pIn->nSdst==1 ); + pDest->affSdst = + sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + +#if 0 /* Never occurs on an ORDER BY query */ + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } +#endif + + /* If this is a scalar select that is part of an expression, then + ** store the results in the appropriate memory cell and break out + ** of the scan loop. + */ + case SRT_Mem: { + assert( pIn->nSdst==1 ); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, 1); + /* The LIMIT clause will jump out of the loop for us */ + break; + } +#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ + + /* The results are stored in a sequence of registers + ** starting at pDest->iSdst. Then the co-routine yields. + */ + case SRT_Coroutine: { + if( pDest->iSdst==0 ){ + pDest->iSdst = sqlite3GetTempRange(pParse, pIn->nSdst); + pDest->nSdst = pIn->nSdst; + } + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pDest->nSdst); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); + break; + } + + /* If none of the above, then the result destination must be + ** SRT_Output. This routine is never called with any other + ** destination other than the ones handled above or SRT_Output. + ** + ** For SRT_Output, results are stored in a sequence of registers. + ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to + ** return the next row of result. + */ + default: { + assert( pDest->eDest==SRT_Output ); + sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); + break; + } + } + + /* Jump to the end of the loop if the LIMIT is reached. + */ + if( p->iLimit ){ + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); + } + + /* Generate the subroutine return + */ + sqlite3VdbeResolveLabel(v, iContinue); + sqlite3VdbeAddOp1(v, OP_Return, regReturn); + + return addr; +} + +/* +** Alternative compound select code generator for cases when there +** is an ORDER BY clause. +** +** We assume a query of the following form: +** +** ORDER BY +** +** is one of UNION ALL, UNION, EXCEPT, or INTERSECT. The idea +** is to code both and with the ORDER BY clause as +** co-routines. Then run the co-routines in parallel and merge the results +** into the output. In addition to the two coroutines (called selectA and +** selectB) there are 7 subroutines: +** +** outA: Move the output of the selectA coroutine into the output +** of the compound query. +** +** outB: Move the output of the selectB coroutine into the output +** of the compound query. (Only generated for UNION and +** UNION ALL. EXCEPT and INSERTSECT never output a row that +** appears only in B.) +** +** AltB: Called when there is data from both coroutines and AB. +** +** EofA: Called when data is exhausted from selectA. +** +** EofB: Called when data is exhausted from selectB. +** +** The implementation of the latter five subroutines depend on which +** is used: +** +** +** UNION ALL UNION EXCEPT INTERSECT +** ------------- ----------------- -------------- ----------------- +** AltB: outA, nextA outA, nextA outA, nextA nextA +** +** AeqB: outA, nextA nextA nextA outA, nextA +** +** AgtB: outB, nextB outB, nextB nextB nextB +** +** EofA: outB, nextB outB, nextB halt halt +** +** EofB: outA, nextA outA, nextA outA, nextA halt +** +** In the AltB, AeqB, and AgtB subroutines, an EOF on A following nextA +** causes an immediate jump to EofA and an EOF on B following nextB causes +** an immediate jump to EofB. Within EofA and EofB, and EOF on entry or +** following nextX causes a jump to the end of the select processing. +** +** Duplicate removal in the UNION, EXCEPT, and INTERSECT cases is handled +** within the output subroutine. The regPrev register set holds the previously +** output value. A comparison is made against this value and the output +** is skipped if the next results would be the same as the previous. +** +** The implementation plan is to implement the two coroutines and seven +** subroutines first, then put the control logic at the bottom. Like this: +** +** goto Init +** coA: coroutine for left query (A) +** coB: coroutine for right query (B) +** outA: output one row of A +** outB: output one row of B (UNION and UNION ALL only) +** EofA: ... +** EofB: ... +** AltB: ... +** AeqB: ... +** AgtB: ... +** Init: initialize coroutine registers +** yield coA +** if eof(A) goto EofA +** yield coB +** if eof(B) goto EofB +** Cmpr: Compare A, B +** Jump AltB, AeqB, AgtB +** End: ... +** +** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not +** actually called using Gosub and they do not Return. EofA and EofB loop +** until all data is exhausted then jump to the "end" labe. AltB, AeqB, +** and AgtB jump to either L2 or to one of EofA or EofB. +*/ +#ifndef SQLITE_OMIT_COMPOUND_SELECT +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int i, j; /* Loop counters */ + Select *pPrior; /* Another SELECT immediately to our left */ + Vdbe *v; /* Generate code to this VDBE */ + SelectDest destA; /* Destination for coroutine A */ + SelectDest destB; /* Destination for coroutine B */ + int regAddrA; /* Address register for select-A coroutine */ + int regAddrB; /* Address register for select-B coroutine */ + int addrSelectA; /* Address of the select-A coroutine */ + int addrSelectB; /* Address of the select-B coroutine */ + int regOutA; /* Address register for the output-A subroutine */ + int regOutB; /* Address register for the output-B subroutine */ + int addrOutA; /* Address of the output-A subroutine */ + int addrOutB = 0; /* Address of the output-B subroutine */ + int addrEofA; /* Address of the select-A-exhausted subroutine */ + int addrEofA_noB; /* Alternate addrEofA if B is uninitialized */ + int addrEofB; /* Address of the select-B-exhausted subroutine */ + int addrAltB; /* Address of the AB subroutine */ + int regLimitA; /* Limit register for select-A */ + int regLimitB; /* Limit register for select-A */ + int regPrev; /* A range of registers to hold previous output */ + int savedLimit; /* Saved value of p->iLimit */ + int savedOffset; /* Saved value of p->iOffset */ + int labelCmpr; /* Label for the start of the merge algorithm */ + int labelEnd; /* Label for the end of the overall SELECT stmt */ + int j1; /* Jump instructions that get retargetted */ + int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ + KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ + KeyInfo *pKeyMerge; /* Comparison information for merging rows */ + sqlite3 *db; /* Database connection */ + ExprList *pOrderBy; /* The ORDER BY clause */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + int *aPermute; /* Mapping from ORDER BY terms to result set columns */ +#ifndef SQLITE_OMIT_EXPLAIN + int iSub1; /* EQP id of left-hand query */ + int iSub2; /* EQP id of right-hand query */ +#endif + + assert( p->pOrderBy!=0 ); + assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */ + db = pParse->db; + v = pParse->pVdbe; + assert( v!=0 ); /* Already thrown the error if VDBE alloc failed */ + labelEnd = sqlite3VdbeMakeLabel(v); + labelCmpr = sqlite3VdbeMakeLabel(v); + + + /* Patch up the ORDER BY clause + */ + op = p->op; + pPrior = p->pPrior; + assert( pPrior->pOrderBy==0 ); + pOrderBy = p->pOrderBy; + assert( pOrderBy ); + nOrderBy = pOrderBy->nExpr; + + /* For operators other than UNION ALL we have to make sure that + ** the ORDER BY clause covers every term of the result set. Add + ** terms to the ORDER BY clause as necessary. + */ + if( op!=TK_ALL ){ + for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){ + struct ExprList_item *pItem; + for(j=0, pItem=pOrderBy->a; ju.x.iOrderByCol>0 ); + if( pItem->u.x.iOrderByCol==i ) break; + } + if( j==nOrderBy ){ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->flags |= EP_IntValue; + pNew->u.iValue = i; + pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); + if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i; + } + } + } + + /* Compute the comparison permutation and keyinfo that is used with + ** the permutation used to determine if the next + ** row of results comes from selectA or selectB. Also add explicit + ** collations to the ORDER BY clause terms so that when the subqueries + ** to the right and the left are evaluated, they use the correct + ** collation. + */ + aPermute = sqlite3DbMallocRaw(db, sizeof(int)*nOrderBy); + if( aPermute ){ + struct ExprList_item *pItem; + for(i=0, pItem=pOrderBy->a; iu.x.iOrderByCol>0 + && pItem->u.x.iOrderByCol<=p->pEList->nExpr ); + aPermute[i] = pItem->u.x.iOrderByCol - 1; + } + pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); + }else{ + pKeyMerge = 0; + } + + /* Reattach the ORDER BY clause to the query. + */ + p->pOrderBy = pOrderBy; + pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0); + + /* Allocate a range of temporary registers and the KeyInfo needed + ** for the logic that removes duplicate result rows when the + ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL). + */ + if( op==TK_ALL ){ + regPrev = 0; + }else{ + int nExpr = p->pEList->nExpr; + assert( nOrderBy>=nExpr || db->mallocFailed ); + regPrev = pParse->nMem+1; + pParse->nMem += nExpr+1; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); + pKeyDup = sqlite3KeyInfoAlloc(db, nExpr, 1); + if( pKeyDup ){ + assert( sqlite3KeyInfoIsWriteable(pKeyDup) ); + for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); + pKeyDup->aSortOrder[i] = 0; + } + } + } + + /* Separate the left and the right query from one another + */ + p->pPrior = 0; + pPrior->pNext = 0; + sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); + if( pPrior->pPrior==0 ){ + sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); + } + + /* Compute the limit registers */ + computeLimitRegisters(pParse, p, labelEnd); + if( p->iLimit && op==TK_ALL ){ + regLimitA = ++pParse->nMem; + regLimitB = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, p->iOffset ? p->iOffset+1 : p->iLimit, + regLimitA); + sqlite3VdbeAddOp2(v, OP_Copy, regLimitA, regLimitB); + }else{ + regLimitA = regLimitB = 0; + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = 0; + sqlite3ExprDelete(db, p->pOffset); + p->pOffset = 0; + + regAddrA = ++pParse->nMem; + regAddrB = ++pParse->nMem; + regOutA = ++pParse->nMem; + regOutB = ++pParse->nMem; + sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA); + sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB); + + /* Generate a coroutine to evaluate the SELECT statement to the + ** left of the compound operator - the "A" select. + */ + addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; + j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + VdbeComment((v, "left SELECT")); + pPrior->iLimit = regLimitA; + explainSetInteger(iSub1, pParse->iNextSelectId); + sqlite3Select(pParse, pPrior, &destA); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); + sqlite3VdbeJumpHere(v, j1); + + /* Generate a coroutine to evaluate the SELECT statement on + ** the right - the "B" select + */ + addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; + j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + VdbeComment((v, "right SELECT")); + savedLimit = p->iLimit; + savedOffset = p->iOffset; + p->iLimit = regLimitB; + p->iOffset = 0; + explainSetInteger(iSub2, pParse->iNextSelectId); + sqlite3Select(pParse, p, &destB); + p->iLimit = savedLimit; + p->iOffset = savedOffset; + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrB); + + /* Generate a subroutine that outputs the current row of the A + ** select as the next output row of the compound select. + */ + VdbeNoopComment((v, "Output routine for A")); + addrOutA = generateOutputSubroutine(pParse, + p, &destA, pDest, regOutA, + regPrev, pKeyDup, labelEnd); + + /* Generate a subroutine that outputs the current row of the B + ** select as the next output row of the compound select. + */ + if( op==TK_ALL || op==TK_UNION ){ + VdbeNoopComment((v, "Output routine for B")); + addrOutB = generateOutputSubroutine(pParse, + p, &destB, pDest, regOutB, + regPrev, pKeyDup, labelEnd); + } + sqlite3KeyInfoUnref(pKeyDup); + + /* Generate a subroutine to run when the results from select A + ** are exhausted and only data in select B remains. + */ + if( op==TK_EXCEPT || op==TK_INTERSECT ){ + addrEofA_noB = addrEofA = labelEnd; + }else{ + VdbeNoopComment((v, "eof-A subroutine")); + addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); + p->nSelectRow += pPrior->nSelectRow; + } + + /* Generate a subroutine to run when the results from select B + ** are exhausted and only data in select A remains. + */ + if( op==TK_INTERSECT ){ + addrEofB = addrEofA; + if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; + }else{ + VdbeNoopComment((v, "eof-B subroutine")); + addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); + } + + /* Generate code to handle the case of AB + */ + VdbeNoopComment((v, "A-gt-B subroutine")); + addrAgtB = sqlite3VdbeCurrentAddr(v); + if( op==TK_ALL || op==TK_UNION ){ + sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + } + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); + + /* This code runs once to initialize everything. + */ + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + + /* Implement the main merge loop + */ + sqlite3VdbeResolveLabel(v, labelCmpr); + sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, + (char*)pKeyMerge, P4_KEYINFO); + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v); + + /* Jump to the this point in order to terminate the query. + */ + sqlite3VdbeResolveLabel(v, labelEnd); + + /* Set the number of output columns + */ + if( pDest->eDest==SRT_Output ){ + Select *pFirst = pPrior; + while( pFirst->pPrior ) pFirst = pFirst->pPrior; + generateColumnNames(pParse, 0, pFirst->pEList); + } + + /* Reassembly the compound query so that it will be freed correctly + ** by the calling function */ + if( p->pPrior ){ + sqlite3SelectDelete(db, p->pPrior); + } + p->pPrior = pPrior; + pPrior->pNext = p; + + /*** TBD: Insert subroutine calls to close cursors on incomplete + **** subqueries ****/ + explainComposite(pParse, p->op, iSub1, iSub2, 0); + return SQLITE_OK; +} +#endif + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* Forward Declarations */ +static void substExprList(sqlite3*, ExprList*, int, ExprList*); +static void substSelect(sqlite3*, Select *, int, ExprList *); + +/* +** Scan through the expression pExpr. Replace every reference to +** a column in table number iTable with a copy of the iColumn-th +** entry in pEList. (But leave references to the ROWID column +** unchanged.) +** +** This routine is part of the flattening procedure. A subquery +** whose result set is defined by pEList appears as entry in the +** FROM clause of a SELECT such that the VDBE cursor assigned to that +** FORM clause entry is iTable. This routine make the necessary +** changes to pExpr so that it refers directly to the source table +** of the subquery rather the result set of the subquery. +*/ +static Expr *substExpr( + sqlite3 *db, /* Report malloc errors to this connection */ + Expr *pExpr, /* Expr in which substitution occurs */ + int iTable, /* Table to be substituted */ + ExprList *pEList /* Substitute expressions */ +){ + if( pExpr==0 ) return 0; + if( pExpr->op==TK_COLUMN && pExpr->iTable==iTable ){ + if( pExpr->iColumn<0 ){ + pExpr->op = TK_NULL; + }else{ + Expr *pNew; + assert( pEList!=0 && pExpr->iColumnnExpr ); + assert( pExpr->pLeft==0 && pExpr->pRight==0 ); + pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0); + sqlite3ExprDelete(db, pExpr); + pExpr = pNew; + } + }else{ + pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); + pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + substSelect(db, pExpr->x.pSelect, iTable, pEList); + }else{ + substExprList(db, pExpr->x.pList, iTable, pEList); + } + } + return pExpr; +} +static void substExprList( + sqlite3 *db, /* Report malloc errors here */ + ExprList *pList, /* List to scan and in which to make substitutes */ + int iTable, /* Table to be substituted */ + ExprList *pEList /* Substitute values */ +){ + int i; + if( pList==0 ) return; + for(i=0; inExpr; i++){ + pList->a[i].pExpr = substExpr(db, pList->a[i].pExpr, iTable, pEList); + } +} +static void substSelect( + sqlite3 *db, /* Report malloc errors here */ + Select *p, /* SELECT statement in which to make substitutions */ + int iTable, /* Table to be replaced */ + ExprList *pEList /* Substitute values */ +){ + SrcList *pSrc; + struct SrcList_item *pItem; + int i; + if( !p ) return; + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(db, p->pHaving, iTable, pEList); + p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + substSelect(db, p->pPrior, iTable, pEList); + pSrc = p->pSrc; + assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ + if( ALWAYS(pSrc) ){ + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + substSelect(db, pItem->pSelect, iTable, pEList); + } + } +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** This routine attempts to flatten subqueries as a performance optimization. +** This routine returns 1 if it makes changes and 0 if no flattening occurs. +** +** To understand the concept of flattening, consider the following +** query: +** +** SELECT a FROM (SELECT x+y AS a FROM t1 WHERE z<100) WHERE a>5 +** +** The default way of implementing this query is to execute the +** subquery first and store the results in a temporary table, then +** run the outer query on that temporary table. This requires two +** passes over the data. Furthermore, because the temporary table +** has no indices, the WHERE clause on the outer query cannot be +** optimized. +** +** This routine attempts to rewrite queries such as the above into +** a single flat select, like this: +** +** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5 +** +** The code generated for this simpification gives the same result +** but only has to scan the data once. And because indices might +** exist on the table t1, a complete scan of the data might be +** avoided. +** +** Flattening is only attempted if all of the following are true: +** +** (1) The subquery and the outer query do not both use aggregates. +** +** (2) The subquery is not an aggregate or the outer query is not a join. +** +** (3) The subquery is not the right operand of a left outer join +** (Originally ticket #306. Strengthened by ticket #3300) +** +** (4) The subquery is not DISTINCT. +** +** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT +** sub-queries that were excluded from this optimization. Restriction +** (4) has since been expanded to exclude all DISTINCT subqueries. +** +** (6) The subquery does not use aggregates or the outer query is not +** DISTINCT. +** +** (7) The subquery has a FROM clause. TODO: For subqueries without +** A FROM clause, consider adding a FROM close with the special +** table sqlite_once that consists of a single row containing a +** single NULL. +** +** (8) The subquery does not use LIMIT or the outer query is not a join. +** +** (9) The subquery does not use LIMIT or the outer query does not use +** aggregates. +** +** (10) The subquery does not use aggregates or the outer query does not +** use LIMIT. +** +** (11) The subquery and the outer query do not both have ORDER BY clauses. +** +** (**) Not implemented. Subsumed into restriction (3). Was previously +** a separate restriction deriving from ticket #350. +** +** (13) The subquery and outer query do not both use LIMIT. +** +** (14) The subquery does not use OFFSET. +** +** (15) The outer query is not part of a compound select or the +** subquery does not have a LIMIT clause. +** (See ticket #2339 and ticket [02a8e81d44]). +** +** (16) The outer query is not an aggregate or the subquery does +** not contain ORDER BY. (Ticket #2942) This used to not matter +** until we introduced the group_concat() function. +** +** (17) The sub-query is not a compound select, or it is a UNION ALL +** compound clause made up entirely of non-aggregate queries, and +** the parent query: +** +** * is not itself part of a compound select, +** * is not an aggregate or DISTINCT query, and +** * is not a join +** +** The parent and sub-query may contain WHERE clauses. Subject to +** rules (11), (13) and (14), they may also contain ORDER BY, +** LIMIT and OFFSET clauses. The subquery cannot use any compound +** operator other than UNION ALL because all the other compound +** operators have an implied DISTINCT which is disallowed by +** restriction (4). +** +** Also, each component of the sub-query must return the same number +** of result columns. This is actually a requirement for any compound +** SELECT statement, but all the code here does is make sure that no +** such (illegal) sub-query is flattened. The caller will detect the +** syntax error and return a detailed message. +** +** (18) If the sub-query is a compound select, then all terms of the +** ORDER by clause of the parent must be simple references to +** columns of the sub-query. +** +** (19) The subquery does not use LIMIT or the outer query does not +** have a WHERE clause. +** +** (20) If the sub-query is a compound select, then it must not use +** an ORDER BY clause. Ticket #3773. We could relax this constraint +** somewhat by saying that the terms of the ORDER BY clause must +** appear as unmodified result columns in the outer query. But we +** have other optimizations in mind to deal with that case. +** +** (21) The subquery does not use LIMIT or the outer query is not +** DISTINCT. (See ticket [752e1646fc]). +** +** (22) The subquery is not a recursive CTE. +** +** (23) The parent is not a recursive CTE, or the sub-query is not a +** compound query. This restriction is because transforming the +** parent to a compound query confuses the code that handles +** recursive queries in multiSelect(). +** +** +** In this routine, the "p" parameter is a pointer to the outer query. +** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query +** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. +** +** If flattening is not attempted, this routine is a no-op and returns 0. +** If flattening is attempted this routine returns 1. +** +** All of the expression analysis must occur on both the outer query and +** the subquery before this routine runs. +*/ +static int flattenSubquery( + Parse *pParse, /* Parsing context */ + Select *p, /* The parent or outer SELECT statement */ + int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ + int isAgg, /* True if outer SELECT uses aggregate functions */ + int subqueryIsAgg /* True if the subquery uses aggregate functions */ +){ + const char *zSavedAuthContext = pParse->zAuthContext; + Select *pParent; + Select *pSub; /* The inner query or "subquery" */ + Select *pSub1; /* Pointer to the rightmost select in sub-query */ + SrcList *pSrc; /* The FROM clause of the outer query */ + SrcList *pSubSrc; /* The FROM clause of the subquery */ + ExprList *pList; /* The result set of the outer query */ + int iParent; /* VDBE cursor number of the pSub result set temp table */ + int i; /* Loop counter */ + Expr *pWhere; /* The WHERE clause */ + struct SrcList_item *pSubitem; /* The subquery */ + sqlite3 *db = pParse->db; + + /* Check to see if flattening is permitted. Return 0 if not. + */ + assert( p!=0 ); + assert( p->pPrior==0 ); /* Unable to flatten compound queries */ + if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0; + pSrc = p->pSrc; + assert( pSrc && iFrom>=0 && iFromnSrc ); + pSubitem = &pSrc->a[iFrom]; + iParent = pSubitem->iCursor; + pSub = pSubitem->pSelect; + assert( pSub!=0 ); + if( isAgg && subqueryIsAgg ) return 0; /* Restriction (1) */ + if( subqueryIsAgg && pSrc->nSrc>1 ) return 0; /* Restriction (2) */ + pSubSrc = pSub->pSrc; + assert( pSubSrc ); + /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, + ** not arbitrary expresssions, we allowed some combining of LIMIT and OFFSET + ** because they could be computed at compile-time. But when LIMIT and OFFSET + ** became arbitrary expressions, we were forced to add restrictions (13) + ** and (14). */ + if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */ + if( pSub->pOffset ) return 0; /* Restriction (14) */ + if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){ + return 0; /* Restriction (15) */ + } + if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ + if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (5) */ + if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ + return 0; /* Restrictions (8)(9) */ + } + if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ + return 0; /* Restriction (6) */ + } + if( p->pOrderBy && pSub->pOrderBy ){ + return 0; /* Restriction (11) */ + } + if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */ + if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */ + if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ + return 0; /* Restriction (21) */ + } + if( pSub->selFlags & SF_Recursive ) return 0; /* Restriction (22) */ + if( (p->selFlags & SF_Recursive) && pSub->pPrior ) return 0; /* (23) */ + + /* OBSOLETE COMMENT 1: + ** Restriction 3: If the subquery is a join, make sure the subquery is + ** not used as the right operand of an outer join. Examples of why this + ** is not allowed: + ** + ** t1 LEFT OUTER JOIN (t2 JOIN t3) + ** + ** If we flatten the above, we would get + ** + ** (t1 LEFT OUTER JOIN t2) JOIN t3 + ** + ** which is not at all the same thing. + ** + ** OBSOLETE COMMENT 2: + ** Restriction 12: If the subquery is the right operand of a left outer + ** join, make sure the subquery has no WHERE clause. + ** An examples of why this is not allowed: + ** + ** t1 LEFT OUTER JOIN (SELECT * FROM t2 WHERE t2.x>0) + ** + ** If we flatten the above, we would get + ** + ** (t1 LEFT OUTER JOIN t2) WHERE t2.x>0 + ** + ** But the t2.x>0 test will always fail on a NULL row of t2, which + ** effectively converts the OUTER JOIN into an INNER JOIN. + ** + ** THIS OVERRIDES OBSOLETE COMMENTS 1 AND 2 ABOVE: + ** Ticket #3300 shows that flattening the right term of a LEFT JOIN + ** is fraught with danger. Best to avoid the whole thing. If the + ** subquery is the right term of a LEFT JOIN, then do not flatten. + */ + if( (pSubitem->jointype & JT_OUTER)!=0 ){ + return 0; + } + + /* Restriction 17: If the sub-query is a compound SELECT, then it must + ** use only the UNION ALL operator. And none of the simple select queries + ** that make up the compound SELECT are allowed to be aggregate or distinct + ** queries. + */ + if( pSub->pPrior ){ + if( pSub->pOrderBy ){ + return 0; /* Restriction 20 */ + } + if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){ + return 0; + } + for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ + testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); + testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + assert( pSub->pSrc!=0 ); + if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 + || (pSub1->pPrior && pSub1->op!=TK_ALL) + || pSub1->pSrc->nSrc<1 + || pSub->pEList->nExpr!=pSub1->pEList->nExpr + ){ + return 0; + } + testcase( pSub1->pSrc->nSrc>1 ); + } + + /* Restriction 18. */ + if( p->pOrderBy ){ + int ii; + for(ii=0; iipOrderBy->nExpr; ii++){ + if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0; + } + } + } + + /***** If we reach this point, flattening is permitted. *****/ + + /* Authorize the subquery */ + pParse->zAuthContext = pSubitem->zName; + TESTONLY(i =) sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + testcase( i==SQLITE_DENY ); + pParse->zAuthContext = zSavedAuthContext; + + /* If the sub-query is a compound SELECT statement, then (by restrictions + ** 17 and 18 above) it must be a UNION ALL and the parent query must + ** be of the form: + ** + ** SELECT FROM () + ** + ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block + ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or + ** OFFSET clauses and joins them to the left-hand-side of the original + ** using UNION ALL operators. In this case N is the number of simple + ** select statements in the compound sub-query. + ** + ** Example: + ** + ** SELECT a+1 FROM ( + ** SELECT x FROM tab + ** UNION ALL + ** SELECT y FROM tab + ** UNION ALL + ** SELECT abs(z*2) FROM tab2 + ** ) WHERE a!=5 ORDER BY 1 + ** + ** Transformed into: + ** + ** SELECT x+1 FROM tab WHERE x+1!=5 + ** UNION ALL + ** SELECT y+1 FROM tab WHERE y+1!=5 + ** UNION ALL + ** SELECT abs(z*2)+1 FROM tab2 WHERE abs(z*2)+1!=5 + ** ORDER BY 1 + ** + ** We call this the "compound-subquery flattening". + */ + for(pSub=pSub->pPrior; pSub; pSub=pSub->pPrior){ + Select *pNew; + ExprList *pOrderBy = p->pOrderBy; + Expr *pLimit = p->pLimit; + Expr *pOffset = p->pOffset; + Select *pPrior = p->pPrior; + p->pOrderBy = 0; + p->pSrc = 0; + p->pPrior = 0; + p->pLimit = 0; + p->pOffset = 0; + pNew = sqlite3SelectDup(db, p, 0); + p->pOffset = pOffset; + p->pLimit = pLimit; + p->pOrderBy = pOrderBy; + p->pSrc = pSrc; + p->op = TK_ALL; + if( pNew==0 ){ + p->pPrior = pPrior; + }else{ + pNew->pPrior = pPrior; + if( pPrior ) pPrior->pNext = pNew; + pNew->pNext = p; + p->pPrior = pNew; + } + if( db->mallocFailed ) return 1; + } + + /* Begin flattening the iFrom-th entry of the FROM clause + ** in the outer query. + */ + pSub = pSub1 = pSubitem->pSelect; + + /* Delete the transient table structure associated with the + ** subquery + */ + sqlite3DbFree(db, pSubitem->zDatabase); + sqlite3DbFree(db, pSubitem->zName); + sqlite3DbFree(db, pSubitem->zAlias); + pSubitem->zDatabase = 0; + pSubitem->zName = 0; + pSubitem->zAlias = 0; + pSubitem->pSelect = 0; + + /* Defer deleting the Table object associated with the + ** subquery until code generation is + ** complete, since there may still exist Expr.pTab entries that + ** refer to the subquery even after flattening. Ticket #3346. + ** + ** pSubitem->pTab is always non-NULL by test restrictions and tests above. + */ + if( ALWAYS(pSubitem->pTab!=0) ){ + Table *pTabToDel = pSubitem->pTab; + if( pTabToDel->nRef==1 ){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pTabToDel->pNextZombie = pToplevel->pZombieTab; + pToplevel->pZombieTab = pTabToDel; + }else{ + pTabToDel->nRef--; + } + pSubitem->pTab = 0; + } + + /* The following loop runs once for each term in a compound-subquery + ** flattening (as described above). If we are doing a different kind + ** of flattening - a flattening other than a compound-subquery flattening - + ** then this loop only runs once. + ** + ** This loop moves all of the FROM elements of the subquery into the + ** the FROM clause of the outer query. Before doing this, remember + ** the cursor number for the original outer query FROM element in + ** iParent. The iParent cursor will never be used. Subsequent code + ** will scan expressions looking for iParent references and replace + ** those references with expressions that resolve to the subquery FROM + ** elements we are now copying in. + */ + for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ + int nSubSrc; + u8 jointype = 0; + pSubSrc = pSub->pSrc; /* FROM clause of subquery */ + nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ + pSrc = pParent->pSrc; /* FROM clause of the outer query */ + + if( pSrc ){ + assert( pParent==p ); /* First time through the loop */ + jointype = pSubitem->jointype; + }else{ + assert( pParent!=p ); /* 2nd and subsequent times through the loop */ + pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + if( pSrc==0 ){ + assert( db->mallocFailed ); + break; + } + } + + /* The subquery uses a single slot of the FROM clause of the outer + ** query. If the subquery has more than one element in its FROM clause, + ** then expand the outer query to make space for it to hold all elements + ** of the subquery. + ** + ** Example: + ** + ** SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB; + ** + ** The outer query has 3 slots in its FROM clause. One slot of the + ** outer query (the middle slot) is used by the subquery. The next + ** block of code will expand the out query to 4 slots. The middle + ** slot is expanded to two slots in order to make space for the + ** two elements in the FROM clause of the subquery. + */ + if( nSubSrc>1 ){ + pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); + if( db->mallocFailed ){ + break; + } + } + + /* Transfer the FROM clause terms from the subquery into the + ** outer query. + */ + for(i=0; ia[i+iFrom].pUsing); + pSrc->a[i+iFrom] = pSubSrc->a[i]; + memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); + } + pSrc->a[iFrom].jointype = jointype; + + /* Now begin substituting subquery result set expressions for + ** references to the iParent in the outer query. + ** + ** Example: + ** + ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b; + ** \ \_____________ subquery __________/ / + ** \_____________________ outer query ______________________________/ + ** + ** We look at every expression in the outer query and every place we see + ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". + */ + pList = pParent->pEList; + for(i=0; inExpr; i++){ + if( pList->a[i].zName==0 ){ + char *zName = sqlite3DbStrDup(db, pList->a[i].zSpan); + sqlite3Dequote(zName); + pList->a[i].zName = zName; + } + } + substExprList(db, pParent->pEList, iParent, pSub->pEList); + if( isAgg ){ + substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); + pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); + } + if( pSub->pOrderBy ){ + assert( pParent->pOrderBy==0 ); + pParent->pOrderBy = pSub->pOrderBy; + pSub->pOrderBy = 0; + }else if( pParent->pOrderBy ){ + substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); + } + if( pSub->pWhere ){ + pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); + }else{ + pWhere = 0; + } + if( subqueryIsAgg ){ + assert( pParent->pHaving==0 ); + pParent->pHaving = pParent->pWhere; + pParent->pWhere = pWhere; + pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); + pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, + sqlite3ExprDup(db, pSub->pHaving, 0)); + assert( pParent->pGroupBy==0 ); + pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); + }else{ + pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); + pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); + } + + /* The flattened query is distinct if either the inner or the + ** outer query is distinct. + */ + pParent->selFlags |= pSub->selFlags & SF_Distinct; + + /* + ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; + ** + ** One is tempted to try to add a and b to combine the limits. But this + ** does not work if either limit is negative. + */ + if( pSub->pLimit ){ + pParent->pLimit = pSub->pLimit; + pSub->pLimit = 0; + } + } + + /* Finially, delete what is left of the subquery and return + ** success. + */ + sqlite3SelectDelete(db, pSub1); + + return 1; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +/* +** Based on the contents of the AggInfo structure indicated by the first +** argument, this function checks if the following are true: +** +** * the query contains just a single aggregate function, +** * the aggregate function is either min() or max(), and +** * the argument to the aggregate function is a column value. +** +** If all of the above are true, then WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX +** is returned as appropriate. Also, *ppMinMax is set to point to the +** list of arguments passed to the aggregate before returning. +** +** Or, if the conditions above are not met, *ppMinMax is set to 0 and +** WHERE_ORDERBY_NORMAL is returned. +*/ +static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ + + *ppMinMax = 0; + if( pAggInfo->nFunc==1 ){ + Expr *pExpr = pAggInfo->aFunc[0].pExpr; /* Aggregate function */ + ExprList *pEList = pExpr->x.pList; /* Arguments to agg function */ + + assert( pExpr->op==TK_AGG_FUNCTION ); + if( pEList && pEList->nExpr==1 && pEList->a[0].pExpr->op==TK_AGG_COLUMN ){ + const char *zFunc = pExpr->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + *ppMinMax = pEList; + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + *ppMinMax = pEList; + } + } + } + + assert( *ppMinMax==0 || (*ppMinMax)->nExpr==1 ); + return eRet; +} + +/* +** The select statement passed as the first argument is an aggregate query. +** The second argment is the associated aggregate-info object. This +** function tests if the SELECT is of the form: +** +** SELECT count(*) FROM +** +** where table is a database table, not a sub-select or view. If the query +** does match this pattern, then a pointer to the Table object representing +** is returned. Otherwise, 0 is returned. +*/ +static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ + Table *pTab; + Expr *pExpr; + + assert( !p->pGroupBy ); + + if( p->pWhere || p->pEList->nExpr!=1 + || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect + ){ + return 0; + } + pTab = p->pSrc->a[0].pTab; + pExpr = p->pEList->a[0].pExpr; + assert( pTab && !pTab->pSelect && pExpr ); + + if( IsVirtual(pTab) ) return 0; + if( pExpr->op!=TK_AGG_FUNCTION ) return 0; + if( NEVER(pAggInfo->nFunc==0) ) return 0; + if( (pAggInfo->aFunc[0].pFunc->funcFlags&SQLITE_FUNC_COUNT)==0 ) return 0; + if( pExpr->flags&EP_Distinct ) return 0; + + return pTab; +} + +/* +** If the source-list item passed as an argument was augmented with an +** INDEXED BY clause, then try to locate the specified index. If there +** was such a clause and the named index cannot be found, return +** SQLITE_ERROR and leave an error in pParse. Otherwise, populate +** pFrom->pIndex and return SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ + if( pFrom->pTab && pFrom->zIndex ){ + Table *pTab = pFrom->pTab; + char *zIndex = pFrom->zIndex; + Index *pIdx; + for(pIdx=pTab->pIndex; + pIdx && sqlite3StrICmp(pIdx->zName, zIndex); + pIdx=pIdx->pNext + ); + if( !pIdx ){ + sqlite3ErrorMsg(pParse, "no such index: %s", zIndex, 0); + pParse->checkSchema = 1; + return SQLITE_ERROR; + } + pFrom->pIndex = pIdx; + } + return SQLITE_OK; +} +/* +** Detect compound SELECT statements that use an ORDER BY clause with +** an alternative collating sequence. +** +** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... +** +** These are rewritten as a subquery: +** +** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) +** ORDER BY ... COLLATE ... +** +** This transformation is necessary because the multiSelectOrderBy() routine +** above that generates the code for a compound SELECT with an ORDER BY clause +** uses a merge algorithm that requires the same collating sequence on the +** result columns as on the ORDER BY clause. See ticket +** http://www.sqlite.org/src/info/6709574d2a +** +** This transformation is only needed for EXCEPT, INTERSECT, and UNION. +** The UNION ALL operator works fine with multiSelectOrderBy() even when +** there are COLLATE terms in the ORDER BY. +*/ +static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ + int i; + Select *pNew; + Select *pX; + sqlite3 *db; + struct ExprList_item *a; + SrcList *pNewSrc; + Parse *pParse; + Token dummy; + + if( p->pPrior==0 ) return WRC_Continue; + if( p->pOrderBy==0 ) return WRC_Continue; + for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} + if( pX==0 ) return WRC_Continue; + a = p->pOrderBy->a; + for(i=p->pOrderBy->nExpr-1; i>=0; i--){ + if( a[i].pExpr->flags & EP_Collate ) break; + } + if( i<0 ) return WRC_Continue; + + /* If we reach this point, that means the transformation is required. */ + + pParse = pWalker->pParse; + db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + if( pNew==0 ) return WRC_Abort; + memset(&dummy, 0, sizeof(dummy)); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + if( pNewSrc==0 ) return WRC_Abort; + *pNew = *p; + p->pSrc = pNewSrc; + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); + p->op = TK_SELECT; + p->pWhere = 0; + pNew->pGroupBy = 0; + pNew->pHaving = 0; + pNew->pOrderBy = 0; + p->pPrior = 0; + p->pNext = 0; + p->selFlags &= ~SF_Compound; + assert( pNew->pPrior!=0 ); + pNew->pPrior->pNext = pNew; + pNew->pLimit = 0; + pNew->pOffset = 0; + return WRC_Continue; +} + +#ifndef SQLITE_OMIT_CTE +/* +** Argument pWith (which may be NULL) points to a linked list of nested +** WITH contexts, from inner to outermost. If the table identified by +** FROM clause element pItem is really a common-table-expression (CTE) +** then return a pointer to the CTE definition for that table. Otherwise +** return NULL. +** +** If a non-NULL value is returned, set *ppContext to point to the With +** object that the returned CTE belongs to. +*/ +static struct Cte *searchWith( + With *pWith, /* Current outermost WITH clause */ + struct SrcList_item *pItem, /* FROM clause element to resolve */ + With **ppContext /* OUT: WITH clause return value belongs to */ +){ + const char *zName; + if( pItem->zDatabase==0 && (zName = pItem->zName)!=0 ){ + With *p; + for(p=pWith; p; p=p->pOuter){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ + *ppContext = p; + return &p->a[i]; + } + } + } + } + return 0; +} + +/* The code generator maintains a stack of active WITH clauses +** with the inner-most WITH clause being at the top of the stack. +** +** This routine pushes the WITH clause passed as the second argument +** onto the top of the stack. If argument bFree is true, then this +** WITH clause will never be popped from the stack. In this case it +** should be freed along with the Parse object. In other cases, when +** bFree==0, the With object will be freed along with the SELECT +** statement with which it is associated. +*/ +SQLITE_PRIVATE void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ + assert( bFree==0 || pParse->pWith==0 ); + if( pWith ){ + pWith->pOuter = pParse->pWith; + pParse->pWith = pWith; + pParse->bFreeWith = bFree; + } +} + +/* +** This function checks if argument pFrom refers to a CTE declared by +** a WITH clause on the stack currently maintained by the parser. And, +** if currently processing a CTE expression, if it is a recursive +** reference to the current CTE. +** +** If pFrom falls into either of the two categories above, pFrom->pTab +** and other fields are populated accordingly. The caller should check +** (pFrom->pTab!=0) to determine whether or not a successful match +** was found. +** +** Whether or not a match is found, SQLITE_OK is returned if no error +** occurs. If an error does occur, an error message is stored in the +** parser and some error code other than SQLITE_OK returned. +*/ +static int withExpand( + Walker *pWalker, + struct SrcList_item *pFrom +){ + Parse *pParse = pWalker->pParse; + sqlite3 *db = pParse->db; + struct Cte *pCte; /* Matched CTE (or NULL if no match) */ + With *pWith; /* WITH clause that pCte belongs to */ + + assert( pFrom->pTab==0 ); + + pCte = searchWith(pParse->pWith, pFrom, &pWith); + if( pCte ){ + Table *pTab; + ExprList *pEList; + Select *pSel; + Select *pLeft; /* Left-most SELECT statement */ + int bMayRecursive; /* True if compound joined by UNION [ALL] */ + With *pSavedWith; /* Initial value of pParse->pWith */ + + /* If pCte->zErr is non-NULL at this point, then this is an illegal + ** recursive reference to CTE pCte. Leave an error in pParse and return + ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** In this case, proceed. */ + if( pCte->zErr ){ + sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + return SQLITE_ERROR; + } + + assert( pFrom->pTab==0 ); + pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return WRC_Abort; + pTab->nRef = 1; + pTab->zName = sqlite3DbStrDup(db, pCte->zName); + pTab->iPKey = -1; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + pTab->tabFlags |= TF_Ephemeral; + pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + if( db->mallocFailed ) return SQLITE_NOMEM; + assert( pFrom->pSelect ); + + /* Check if this is a recursive CTE. */ + pSel = pFrom->pSelect; + bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); + if( bMayRecursive ){ + int i; + SrcList *pSrc = pFrom->pSelect->pSrc; + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pSrc->a[i]; + if( pItem->zDatabase==0 + && pItem->zName!=0 + && 0==sqlite3StrICmp(pItem->zName, pCte->zName) + ){ + pItem->pTab = pTab; + pItem->isRecursive = 1; + pTab->nRef++; + pSel->selFlags |= SF_Recursive; + } + } + } + + /* Only one recursive reference is permitted. */ + if( pTab->nRef>2 ){ + sqlite3ErrorMsg( + pParse, "multiple references to recursive table: %s", pCte->zName + ); + return SQLITE_ERROR; + } + assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); + + pCte->zErr = "circular reference: %s"; + pSavedWith = pParse->pWith; + pParse->pWith = pWith; + sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + + for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); + pEList = pLeft->pEList; + if( pCte->pCols ){ + if( pEList->nExpr!=pCte->pCols->nExpr ){ + sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", + pCte->zName, pEList->nExpr, pCte->pCols->nExpr + ); + pParse->pWith = pSavedWith; + return SQLITE_ERROR; + } + pEList = pCte->pCols; + } + + selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + if( bMayRecursive ){ + if( pSel->selFlags & SF_Recursive ){ + pCte->zErr = "multiple recursive references: %s"; + }else{ + pCte->zErr = "recursive reference in a subquery: %s"; + } + sqlite3WalkSelect(pWalker, pSel); + } + pCte->zErr = 0; + pParse->pWith = pSavedWith; + } + + return SQLITE_OK; +} +#endif + +#ifndef SQLITE_OMIT_CTE +/* +** If the SELECT passed as the second argument has an associated WITH +** clause, pop it from the stack stored as part of the Parse object. +** +** This function is used as the xSelectCallback2() callback by +** sqlite3SelectExpand() when walking a SELECT tree to resolve table +** names and other FROM clause elements. +*/ +static void selectPopWith(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + With *pWith = findRightmost(p)->pWith; + if( pWith!=0 ){ + assert( pParse->pWith==pWith ); + pParse->pWith = pWith->pOuter; + } +} +#else +#define selectPopWith 0 +#endif + +/* +** This routine is a Walker callback for "expanding" a SELECT statement. +** "Expanding" means to do the following: +** +** (1) Make sure VDBE cursor numbers have been assigned to every +** element of the FROM clause. +** +** (2) Fill in the pTabList->a[].pTab fields in the SrcList that +** defines FROM clause. When views appear in the FROM clause, +** fill pTabList->a[].pSelect with a copy of the SELECT statement +** that implements the view. A copy is made of the view's SELECT +** statement so that we can freely modify or delete that statement +** without worrying about messing up the presistent representation +** of the view. +** +** (3) Add terms to the WHERE clause to accomodate the NATURAL keyword +** on joins and the ON and USING clause of joins. +** +** (4) Scan the list of columns in the result set (pEList) looking +** for instances of the "*" operator or the TABLE.* operator. +** If found, expand each "*" to be every column in every table +** and TABLE.* to be every column in TABLE. +** +*/ +static int selectExpander(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i, j, k; + SrcList *pTabList; + ExprList *pEList; + struct SrcList_item *pFrom; + sqlite3 *db = pParse->db; + Expr *pE, *pRight, *pExpr; + u16 selFlags = p->selFlags; + + p->selFlags |= SF_Expanded; + if( db->mallocFailed ){ + return WRC_Abort; + } + if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ + return WRC_Prune; + } + pTabList = p->pSrc; + pEList = p->pEList; + sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); + + /* Make sure cursor numbers have been assigned to all entries in + ** the FROM clause of the SELECT statement. + */ + sqlite3SrcListAssignCursors(pParse, pTabList); + + /* Look up every table named in the FROM clause of the select. If + ** an entry of the FROM clause is a subquery instead of a table or view, + ** then create a transient table structure to describe the subquery. + */ + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab; + assert( pFrom->isRecursive==0 || pFrom->pTab ); + if( pFrom->isRecursive ) continue; + if( pFrom->pTab!=0 ){ + /* This statement has already been prepared. There is no need + ** to go further. */ + assert( i==0 ); +#ifndef SQLITE_OMIT_CTE + selectPopWith(pWalker, p); +#endif + return WRC_Prune; + } +#ifndef SQLITE_OMIT_CTE + if( withExpand(pWalker, pFrom) ) return WRC_Abort; + if( pFrom->pTab ) {} else +#endif + if( pFrom->zName==0 ){ +#ifndef SQLITE_OMIT_SUBQUERY + Select *pSel = pFrom->pSelect; + /* A sub-query in the FROM clause of a SELECT */ + assert( pSel!=0 ); + assert( pFrom->pTab==0 ); + sqlite3WalkSelect(pWalker, pSel); + pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return WRC_Abort; + pTab->nRef = 1; + pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); + while( pSel->pPrior ){ pSel = pSel->pPrior; } + selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); + pTab->iPKey = -1; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + pTab->tabFlags |= TF_Ephemeral; +#endif + }else{ + /* An ordinary table or view name in the FROM clause */ + assert( pFrom->pTab==0 ); + pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); + if( pTab==0 ) return WRC_Abort; + if( pTab->nRef==0xffff ){ + sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", + pTab->zName); + pFrom->pTab = 0; + return WRC_Abort; + } + pTab->nRef++; +#if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) + if( pTab->pSelect || IsVirtual(pTab) ){ + /* We reach here if the named table is a really a view */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; + assert( pFrom->pSelect==0 ); + pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); + sqlite3WalkSelect(pWalker, pFrom->pSelect); + } +#endif + } + + /* Locate the index named by the INDEXED BY clause, if any. */ + if( sqlite3IndexedByLookup(pParse, pFrom) ){ + return WRC_Abort; + } + } + + /* Process NATURAL keywords, and ON and USING clauses of joins. + */ + if( db->mallocFailed || sqliteProcessJoin(pParse, p) ){ + return WRC_Abort; + } + + /* For every "*" that occurs in the column list, insert the names of + ** all columns in all tables. And for every TABLE.* insert the names + ** of all columns in TABLE. The parser inserted a special expression + ** with the TK_ALL operator for each "*" that it found in the column list. + ** The following code just has to locate the TK_ALL expressions and expand + ** each one to the list of all columns in all tables. + ** + ** The first loop just checks to see if there are any "*" operators + ** that need expanding. + */ + for(k=0; knExpr; k++){ + pE = pEList->a[k].pExpr; + if( pE->op==TK_ALL ) break; + assert( pE->op!=TK_DOT || pE->pRight!=0 ); + assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); + if( pE->op==TK_DOT && pE->pRight->op==TK_ALL ) break; + } + if( knExpr ){ + /* + ** If we get here it means the result set contains one or more "*" + ** operators that need to be expanded. Loop through each expression + ** in the result set and expand them one by one. + */ + struct ExprList_item *a = pEList->a; + ExprList *pNew = 0; + int flags = pParse->db->flags; + int longNames = (flags & SQLITE_FullColNames)!=0 + && (flags & SQLITE_ShortColNames)==0; + + /* When processing FROM-clause subqueries, it is always the case + ** that full_column_names=OFF and short_column_names=ON. The + ** sqlite3ResultSetOfSelect() routine makes it so. */ + assert( (p->selFlags & SF_NestedFrom)==0 + || ((flags & SQLITE_FullColNames)==0 && + (flags & SQLITE_ShortColNames)!=0) ); + + for(k=0; knExpr; k++){ + pE = a[k].pExpr; + pRight = pE->pRight; + assert( pE->op!=TK_DOT || pRight!=0 ); + if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){ + /* This particular expression does not need to be expanded. + */ + pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); + if( pNew ){ + pNew->a[pNew->nExpr-1].zName = a[k].zName; + pNew->a[pNew->nExpr-1].zSpan = a[k].zSpan; + a[k].zName = 0; + a[k].zSpan = 0; + } + a[k].pExpr = 0; + }else{ + /* This expression is a "*" or a "TABLE.*" and needs to be + ** expanded. */ + int tableSeen = 0; /* Set to 1 when TABLE matches */ + char *zTName = 0; /* text of name of TABLE */ + if( pE->op==TK_DOT ){ + assert( pE->pLeft!=0 ); + assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); + zTName = pE->pLeft->u.zToken; + } + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + Select *pSub = pFrom->pSelect; + char *zTabName = pFrom->zAlias; + const char *zSchemaName = 0; + int iDb; + if( zTabName==0 ){ + zTabName = pTab->zName; + } + if( db->mallocFailed ) break; + if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ + pSub = 0; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; + } + for(j=0; jnCol; j++){ + char *zName = pTab->aCol[j].zName; + char *zColname; /* The computed column name */ + char *zToFree; /* Malloced string that needs to be freed */ + Token sColname; /* Computed column name as a token */ + + assert( zName ); + if( zTName && pSub + && sqlite3MatchSpanName(pSub->pEList->a[j].zSpan, 0, zTName, 0)==0 + ){ + continue; + } + + /* If a column is marked as 'hidden' (currently only possible + ** for virtual tables), do not include it in the expanded + ** result-set list. + */ + if( IsHiddenColumn(&pTab->aCol[j]) ){ + assert(IsVirtual(pTab)); + continue; + } + tableSeen = 1; + + if( i>0 && zTName==0 ){ + if( (pFrom->jointype & JT_NATURAL)!=0 + && tableAndColumnIndex(pTabList, i, zName, 0, 0) + ){ + /* In a NATURAL join, omit the join columns from the + ** table to the right of the join */ + continue; + } + if( sqlite3IdListIndex(pFrom->pUsing, zName)>=0 ){ + /* In a join with a USING clause, omit columns in the + ** using clause from the table on the right. */ + continue; + } + } + pRight = sqlite3Expr(db, TK_ID, zName); + zColname = zName; + zToFree = 0; + if( longNames || pTabList->nSrc>1 ){ + Expr *pLeft; + pLeft = sqlite3Expr(db, TK_ID, zTabName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + if( zSchemaName ){ + pLeft = sqlite3Expr(db, TK_ID, zSchemaName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr, 0); + } + if( longNames ){ + zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); + zToFree = zColname; + } + }else{ + pExpr = pRight; + } + pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); + sColname.z = zColname; + sColname.n = sqlite3Strlen30(zColname); + sqlite3ExprListSetName(pParse, pNew, &sColname, 0); + if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + if( pSub ){ + pX->zSpan = sqlite3DbStrDup(db, pSub->pEList->a[j].zSpan); + testcase( pX->zSpan==0 ); + }else{ + pX->zSpan = sqlite3MPrintf(db, "%s.%s.%s", + zSchemaName, zTabName, zColname); + testcase( pX->zSpan==0 ); + } + pX->bSpanIsTab = 1; + } + sqlite3DbFree(db, zToFree); + } + } + if( !tableSeen ){ + if( zTName ){ + sqlite3ErrorMsg(pParse, "no such table: %s", zTName); + }else{ + sqlite3ErrorMsg(pParse, "no tables specified"); + } + } + } + } + sqlite3ExprListDelete(db, pEList); + p->pEList = pNew; + } +#if SQLITE_MAX_COLUMN + if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns in result set"); + } +#endif + return WRC_Continue; +} + +/* +** No-op routine for the parse-tree walker. +** +** When this routine is the Walker.xExprCallback then expression trees +** are walked without any actions being taken at each node. Presumably, +** when this routine is used for Walker.xExprCallback then +** Walker.xSelectCallback is set to do something useful for every +** subquery in the parser tree. +*/ +static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return WRC_Continue; +} + +/* +** This routine "expands" a SELECT statement and all of its subqueries. +** For additional information on what it means to "expand" a SELECT +** statement, see the comment on the selectExpand worker callback above. +** +** Expanding a SELECT statement is the first step in processing a +** SELECT statement. The SELECT statement must be expanded before +** name resolution is performed. +** +** If anything goes wrong, an error message is written into pParse. +** The calling function can detect the problem by looking at pParse->nErr +** and/or pParse->db->mallocFailed. +*/ +static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = exprWalkNoop; + w.pParse = pParse; + if( pParse->hasCompound ){ + w.xSelectCallback = convertCompoundSelectToSubquery; + sqlite3WalkSelect(&w, pSelect); + } + w.xSelectCallback = selectExpander; + w.xSelectCallback2 = selectPopWith; + sqlite3WalkSelect(&w, pSelect); +} + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() +** interface. +** +** For each FROM-clause subquery, add Column.zType and Column.zColl +** information to the Table structure that represents the result set +** of that subquery. +** +** The Table structure that represents the result set was constructed +** by selectExpander() but the type and collation information was omitted +** at that point because identifiers had not yet been resolved. This +** routine is called after identifier resolution. +*/ +static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ + Parse *pParse; + int i; + SrcList *pTabList; + struct SrcList_item *pFrom; + + assert( p->selFlags & SF_Resolved ); + if( (p->selFlags & SF_HasTypeInfo)==0 ){ + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab, pSel); + } + } + } + } +} +#endif + + +/* +** This routine adds datatype and collating sequence information to +** the Table structures of all FROM-clause subqueries in a +** SELECT statement. +** +** Use this routine after name resolution. +*/ +static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ +#ifndef SQLITE_OMIT_SUBQUERY + Walker w; + memset(&w, 0, sizeof(w)); + w.xSelectCallback2 = selectAddSubqueryTypeInfo; + w.xExprCallback = exprWalkNoop; + w.pParse = pParse; + sqlite3WalkSelect(&w, pSelect); +#endif +} + + +/* +** This routine sets up a SELECT statement for processing. The +** following is accomplished: +** +** * VDBE Cursor numbers are assigned to all FROM-clause terms. +** * Ephemeral Table objects are created for all FROM-clause subqueries. +** * ON and USING clauses are shifted into WHERE statements +** * Wildcards "*" and "TABLE.*" in result sets are expanded. +** * Identifiers in expression are matched to tables. +** +** This routine acts recursively on all subqueries within the SELECT. +*/ +SQLITE_PRIVATE void sqlite3SelectPrep( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + NameContext *pOuterNC /* Name context for container */ +){ + sqlite3 *db; + if( NEVER(p==0) ) return; + db = pParse->db; + if( db->mallocFailed ) return; + if( p->selFlags & SF_HasTypeInfo ) return; + sqlite3SelectExpand(pParse, p); + if( pParse->nErr || db->mallocFailed ) return; + sqlite3ResolveSelectNames(pParse, p, pOuterNC); + if( pParse->nErr || db->mallocFailed ) return; + sqlite3SelectAddTypeInfo(pParse, p); +} + +/* +** Reset the aggregate accumulator. +** +** The aggregate accumulator is a set of memory cells that hold +** intermediate results while calculating an aggregate. This +** routine generates code that stores NULLs in all of those memory +** cells. +*/ +static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + struct AggInfo_func *pFunc; + int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + if( nReg==0 ) return; +#ifdef SQLITE_DEBUG + /* Verify that all AggInfo registers are within the range specified by + ** AggInfo.mnReg..AggInfo.mxReg */ + assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); + for(i=0; inColumn; i++){ + assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg + && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); + } + for(i=0; inFunc; i++){ + assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg + && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); + } +#endif + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ + if( pFunc->iDistinct>=0 ){ + Expr *pE = pFunc->pExpr; + assert( !ExprHasProperty(pE, EP_xIsSelect) ); + if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){ + sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one " + "argument"); + pFunc->iDistinct = -1; + }else{ + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0, 0); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO); + } + } + } +} + +/* +** Invoke the OP_AggFinalize opcode for every aggregate function +** in the AggInfo structure. +*/ +static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + struct AggInfo_func *pF; + for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ + ExprList *pList = pF->pExpr->x.pList; + assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); + sqlite3VdbeAddOp4(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0, 0, + (void*)pF->pFunc, P4_FUNCDEF); + } +} + +/* +** Update the accumulator memory cells for an aggregate based on +** the current cursor position. +*/ +static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + int regHit = 0; + int addrHitTest = 0; + struct AggInfo_func *pF; + struct AggInfo_col *pC; + + pAggInfo->directMode = 1; + for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ + int nArg; + int addrNext = 0; + int regAgg; + ExprList *pList = pF->pExpr->x.pList; + assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); + if( pList ){ + nArg = pList->nExpr; + regAgg = sqlite3GetTempRange(pParse, nArg); + sqlite3ExprCodeExprList(pParse, pList, regAgg, SQLITE_ECEL_DUP); + }else{ + nArg = 0; + regAgg = 0; + } + if( pF->iDistinct>=0 ){ + addrNext = sqlite3VdbeMakeLabel(v); + assert( nArg==1 ); + codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); + } + if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl = 0; + struct ExprList_item *pItem; + int j; + assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ + for(j=0, pItem=pList->a; !pColl && jpExpr); + } + if( !pColl ){ + pColl = pParse->db->pDfltColl; + } + if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, + (void*)pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); + if( addrNext ){ + sqlite3VdbeResolveLabel(v, addrNext); + sqlite3ExprCacheClear(pParse); + } + } + + /* Before populating the accumulator registers, clear the column cache. + ** Otherwise, if any of the required column values are already present + ** in registers, sqlite3ExprCode() may use OP_SCopy to copy the value + ** to pC->iMem. But by the time the value is used, the original register + ** may have been used, invalidating the underlying buffer holding the + ** text or blob value. See ticket [883034dcb5]. + ** + ** Another solution would be to change the OP_SCopy used to copy cached + ** values to an OP_Copy. + */ + if( regHit ){ + addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); + } + sqlite3ExprCacheClear(pParse); + for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ + sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); + } + pAggInfo->directMode = 0; + sqlite3ExprCacheClear(pParse); + if( addrHitTest ){ + sqlite3VdbeJumpHere(v, addrHitTest); + } +} + +/* +** Add a single OP_Explain instruction to the VDBE to explain a simple +** count(*) query ("SELECT count(*) FROM pTab"). +*/ +#ifndef SQLITE_OMIT_EXPLAIN +static void explainSimpleCount( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being queried */ + Index *pIdx /* Index used to optimize scan, or NULL */ +){ + if( pParse->explain==2 ){ + int bCover = (pIdx!=0 && (HasRowid(pTab) || !IsPrimaryKeyIndex(pIdx))); + char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s%s%s", + pTab->zName, + bCover ? " USING COVERING INDEX " : "", + bCover ? pIdx->zName : "" + ); + sqlite3VdbeAddOp4( + pParse->pVdbe, OP_Explain, pParse->iSelectId, 0, 0, zEqp, P4_DYNAMIC + ); + } +} +#else +# define explainSimpleCount(a,b,c) +#endif + +/* +** Generate code for the SELECT statement given in the p argument. +** +** The results are returned according to the SelectDest structure. +** See comments in sqliteInt.h for further information. +** +** This routine returns the number of errors. If any errors are +** encountered, then an appropriate error message is left in +** pParse->zErrMsg. +** +** This routine does NOT free the Select structure passed in. The +** calling function needs to do that. +*/ +SQLITE_PRIVATE int sqlite3Select( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + SelectDest *pDest /* What to do with the query results */ +){ + int i, j; /* Loop counters */ + WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ + Vdbe *v; /* The virtual machine under construction */ + int isAgg; /* True for select lists like "count(*)" */ + ExprList *pEList; /* List of columns to extract. */ + SrcList *pTabList; /* List of tables to select from */ + Expr *pWhere; /* The WHERE clause. May be NULL */ + ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ + Expr *pHaving; /* The HAVING clause. May be NULL */ + int rc = 1; /* Value to return from this function */ + DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ + SortCtx sSort; /* Info on how to code the ORDER BY clause */ + AggInfo sAggInfo; /* Information used by aggregate queries */ + int iEnd; /* Address of the end of the query */ + sqlite3 *db; /* The database connection */ + +#ifndef SQLITE_OMIT_EXPLAIN + int iRestoreSelectId = pParse->iSelectId; + pParse->iSelectId = pParse->iNextSelectId++; +#endif + + db = pParse->db; + if( p==0 || db->mallocFailed || pParse->nErr ){ + return 1; + } + if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; + memset(&sAggInfo, 0, sizeof(sAggInfo)); + + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); + if( IgnorableOrderby(pDest) ){ + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistFifo || + pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_Fifo); + /* If ORDER BY makes no difference in the output then neither does + ** DISTINCT so it can be removed too. */ + sqlite3ExprListDelete(db, p->pOrderBy); + p->pOrderBy = 0; + p->selFlags &= ~SF_Distinct; + } + sqlite3SelectPrep(pParse, p, 0); + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; + pTabList = p->pSrc; + pEList = p->pEList; + if( pParse->nErr || db->mallocFailed ){ + goto select_end; + } + isAgg = (p->selFlags & SF_Aggregate)!=0; + assert( pEList!=0 ); + + /* Begin generating code. + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto select_end; + + /* If writing to memory or generating a set + ** only a single column may be output. + */ +#ifndef SQLITE_OMIT_SUBQUERY + if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + goto select_end; + } +#endif + + /* Generate code for all sub-queries in the FROM clause + */ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + for(i=0; !p->pPrior && inSrc; i++){ + struct SrcList_item *pItem = &pTabList->a[i]; + SelectDest dest; + Select *pSub = pItem->pSelect; + int isAggSub; + + if( pSub==0 ) continue; + + /* Sometimes the code for a subquery will be generated more than + ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, + ** for example. In that case, do not regenerate the code to manifest + ** a view or the co-routine to implement a view. The first instance + ** is sufficient, though the subroutine to manifest the view does need + ** to be invoked again. */ + if( pItem->addrFillSub ){ + if( pItem->viaCoroutine==0 ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + } + continue; + } + + /* Increment Parse.nHeight by the height of the largest expression + ** tree referred to by this, the parent select. The child select + ** may contain expression trees of at most + ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit + ** more conservative than necessary, but much easier than enforcing + ** an exact limit. + */ + pParse->nHeight += sqlite3SelectExprHeight(p); + + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; + if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + /* This subquery can be absorbed into its parent. */ + if( isAggSub ){ + isAgg = 1; + p->selFlags |= SF_Aggregate; + } + i = -1; + }else if( pTabList->nSrc==1 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) + ){ + /* Implement a co-routine that will return a single row of the result + ** set on each invocation. + */ + int addrTop = sqlite3VdbeCurrentAddr(v)+1; + pItem->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + VdbeComment((v, "%s", pItem->pTab->zName)); + pItem->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->viaCoroutine = 1; + pItem->regResult = dest.iSdst; + sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); + sqlite3VdbeJumpHere(v, addrTop-1); + sqlite3ClearTempRegCache(pParse); + }else{ + /* Generate a subroutine that will fill an ephemeral table with + ** the content of this subquery. pItem->addrFillSub will point + ** to the address of the generated subroutine. pItem->regReturn + ** is a register allocated to hold the subroutine return address + */ + int topAddr; + int onceAddr = 0; + int retAddr; + assert( pItem->addrFillSub==0 ); + pItem->regReturn = ++pParse->nMem; + topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); + pItem->addrFillSub = topAddr+1; + if( pItem->isCorrelated==0 ){ + /* If the subquery is not correlated and if we are not inside of + ** a trigger, then we only need to compute the value of the subquery + ** once. */ + onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); + }else{ + VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); + } + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); + retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); + VdbeComment((v, "end %s", pItem->pTab->zName)); + sqlite3VdbeChangeP1(v, topAddr, retAddr); + sqlite3ClearTempRegCache(pParse); + } + if( /*pParse->nErr ||*/ db->mallocFailed ){ + goto select_end; + } + pParse->nHeight -= sqlite3SelectExprHeight(p); + pTabList = p->pSrc; + if( !IgnorableOrderby(pDest) ){ + sSort.pOrderBy = p->pOrderBy; + } + } + pEList = p->pEList; +#endif + pWhere = p->pWhere; + pGroupBy = p->pGroupBy; + pHaving = p->pHaving; + sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; + +#ifndef SQLITE_OMIT_COMPOUND_SELECT + /* If there is are a sequence of queries, do the earlier ones first. + */ + if( p->pPrior ){ + rc = multiSelect(pParse, p, pDest); + explainSetInteger(pParse->iSelectId, iRestoreSelectId); + return rc; + } +#endif + + /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + ** if the select-list is the same as the ORDER BY list, then this query + ** can be rewritten as a GROUP BY. In other words, this: + ** + ** SELECT DISTINCT xyz FROM ... ORDER BY xyz + ** + ** is transformed to: + ** + ** SELECT xyz FROM ... GROUP BY xyz + ** + ** The second form is preferred as a single index (or temp-table) may be + ** used for both the ORDER BY and DISTINCT processing. As originally + ** written the query must use a temp-table for at least one of the ORDER + ** BY and DISTINCT, and an index or separate temp-table for the other. + */ + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct + && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 + ){ + p->selFlags &= ~SF_Distinct; + p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); + pGroupBy = p->pGroupBy; + sSort.pOrderBy = 0; + /* Notice that even thought SF_Distinct has been cleared from p->selFlags, + ** the sDistinct.isTnct is still set. Hence, isTnct represents the + ** original setting of the SF_Distinct flag, not the current setting */ + assert( sDistinct.isTnct ); + } + + /* If there is an ORDER BY clause, then this sorting + ** index might end up being unused if the data can be + ** extracted in pre-sorted order. If that is the case, then the + ** OP_OpenEphemeral instruction will be changed to an OP_Noop once + ** we figure out that the sorting index is not needed. The addrSortIndex + ** variable is used to facilitate that change. + */ + if( sSort.pOrderBy ){ + KeyInfo *pKeyInfo; + pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, 0); + sSort.iECursor = pParse->nTab++; + sSort.addrSortIndex = + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sSort.iECursor, sSort.pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO); + }else{ + sSort.addrSortIndex = -1; + } + + /* If the output is destined for a temporary table, open that table. + */ + if( pDest->eDest==SRT_EphemTab ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); + } + + /* Set the limiter. + */ + iEnd = sqlite3VdbeMakeLabel(v); + p->nSelectRow = LARGEST_INT64; + computeLimitRegisters(pParse, p, iEnd); + if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ + sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; + sSort.sortFlags |= SORTFLAG_UseSorter; + } + + /* Open a virtual index to use for the distinct set. + */ + if( p->selFlags & SF_Distinct ){ + sDistinct.tabTnct = pParse->nTab++; + sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sDistinct.tabTnct, 0, 0, + (char*)keyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; + }else{ + sDistinct.eTnctType = WHERE_DISTINCT_NOOP; + } + + if( !isAgg && pGroupBy==0 ){ + /* No aggregate functions and no GROUP BY clause */ + u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + + /* Begin the database scan. */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, + p->pEList, wctrlFlags, 0); + if( pWInfo==0 ) goto select_end; + if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ + p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); + } + if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ + sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); + } + if( sSort.pOrderBy ){ + sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ + sSort.pOrderBy = 0; + } + } + + /* If sorting index that was created by a prior OP_OpenEphemeral + ** instruction ended up not being needed, then change the OP_OpenEphemeral + ** into an OP_Noop. + */ + if( sSort.addrSortIndex>=0 && sSort.pOrderBy==0 ){ + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); + } + + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, pEList, -1, &sSort, &sDistinct, pDest, + sqlite3WhereContinueLabel(pWInfo), + sqlite3WhereBreakLabel(pWInfo)); + + /* End the database scan loop. + */ + sqlite3WhereEnd(pWInfo); + }else{ + /* This case when there exist aggregate functions or a GROUP BY clause + ** or both */ + NameContext sNC; /* Name context for processing aggregate information */ + int iAMem; /* First Mem address for storing current GROUP BY */ + int iBMem; /* First Mem address for previous GROUP BY */ + int iUseFlag; /* Mem address holding flag indicating that at least + ** one row of the input to the aggregator has been + ** processed */ + int iAbortFlag; /* Mem address which causes query abort if positive */ + int groupBySort; /* Rows come from source in GROUP BY order */ + int addrEnd; /* End of processing for this SELECT */ + int sortPTab = 0; /* Pseudotable used to decode sorting results */ + int sortOut = 0; /* Output register from the sorter */ + int orderByGrp = 0; /* True if the GROUP BY and ORDER BY are the same */ + + /* Remove any and all aliases between the result set and the + ** GROUP BY clause. + */ + if( pGroupBy ){ + int k; /* Loop counter */ + struct ExprList_item *pItem; /* For looping over expression in a list */ + + for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){ + pItem->u.x.iAlias = 0; + } + for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ + pItem->u.x.iAlias = 0; + } + if( p->nSelectRow>100 ) p->nSelectRow = 100; + }else{ + p->nSelectRow = 1; + } + + + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out + ** in the correct order. It also may not - the GROUP BY may use a + ** database index that causes rows to be grouped together as required + ** but not actually sorted. Either way, record the fact that the + ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp + ** variable. */ + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } + + /* Create a label to jump to when we want to abort the query */ + addrEnd = sqlite3VdbeMakeLabel(v); + + /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in + ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the + ** SELECT statement. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + sNC.pAggInfo = &sAggInfo; + sAggInfo.mnReg = pParse->nMem+1; + sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; + sAggInfo.pGroupBy = pGroupBy; + sqlite3ExprAnalyzeAggList(&sNC, pEList); + sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); + if( pHaving ){ + sqlite3ExprAnalyzeAggregates(&sNC, pHaving); + } + sAggInfo.nAccumulator = sAggInfo.nColumn; + for(i=0; ix.pList); + sNC.ncFlags &= ~NC_InAggFunc; + } + sAggInfo.mxReg = pParse->nMem; + if( db->mallocFailed ) goto select_end; + + /* Processing for aggregates with GROUP BY is very different and + ** much more complex than aggregates without a GROUP BY. + */ + if( pGroupBy ){ + KeyInfo *pKeyInfo; /* Keying information for the group by clause */ + int j1; /* A-vs-B comparision jump */ + int addrOutputRow; /* Start of subroutine that outputs a result row */ + int regOutputRow; /* Return address register for output subroutine */ + int addrSetAbort; /* Set the abort flag and return */ + int addrTopOfLoop; /* Top of the input loop */ + int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ + int addrReset; /* Subroutine for resetting the accumulator */ + int regReset; /* Return address register for reset subroutine */ + + /* If there is a GROUP BY clause we might need a sorting index to + ** implement it. Allocate that sorting index now. If it turns out + ** that we do not need it after all, the OP_SorterOpen instruction + ** will be converted into a Noop. + */ + sAggInfo.sortingIdx = pParse->nTab++; + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, 0); + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, + sAggInfo.sortingIdx, sAggInfo.nSortingColumn, + 0, (char*)pKeyInfo, P4_KEYINFO); + + /* Initialize memory locations used by GROUP BY aggregate processing + */ + iUseFlag = ++pParse->nMem; + iAbortFlag = ++pParse->nMem; + regOutputRow = ++pParse->nMem; + addrOutputRow = sqlite3VdbeMakeLabel(v); + regReset = ++pParse->nMem; + addrReset = sqlite3VdbeMakeLabel(v); + iAMem = pParse->nMem + 1; + pParse->nMem += pGroupBy->nExpr; + iBMem = pParse->nMem + 1; + pParse->nMem += pGroupBy->nExpr; + sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); + VdbeComment((v, "clear abort flag")); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); + VdbeComment((v, "indicate accumulator empty")); + sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); + + /* Begin a loop that will extract all source rows in GROUP BY order. + ** This might involve two separate loops with an OP_Sort in between, or + ** it might be a single loop that uses an index to extract information + ** in the right order to begin with. + */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, + WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 + ); + if( pWInfo==0 ) goto select_end; + if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ + /* The optimizer is able to deliver rows in group by order so + ** we do not have to sort. The OP_OpenEphemeral table will be + ** cancelled later because we still need to use the pKeyInfo + */ + groupBySort = 0; + }else{ + /* Rows are coming out in undetermined order. We have to push + ** each row into a sorting index, terminate the first loop, + ** then loop over the sorting index in order to get the output + ** in sorted order + */ + int regBase; + int regRecord; + int nCol; + int nGroupBy; + + explainTempTable(pParse, + (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? + "DISTINCT" : "GROUP BY"); + + groupBySort = 1; + nGroupBy = pGroupBy->nExpr; + nCol = nGroupBy + 1; + j = nGroupBy+1; + for(i=0; i=j ){ + nCol++; + j++; + } + } + regBase = sqlite3GetTempRange(pParse, nCol); + sqlite3ExprCacheClear(pParse); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); + sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); + j = nGroupBy+1; + for(i=0; iiSorterColumn>=j ){ + int r1 = j + regBase; + int r2; + + r2 = sqlite3ExprCodeGetColumn(pParse, + pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); + if( r1!=r2 ){ + sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); + } + j++; + } + } + regRecord = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, sAggInfo.sortingIdx, regRecord); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ReleaseTempRange(pParse, regBase, nCol); + sqlite3WhereEnd(pWInfo); + sAggInfo.sortingIdxPTab = sortPTab = pParse->nTab++; + sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); + sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); + VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); + sAggInfo.useSortingIdx = 1; + sqlite3ExprCacheClear(pParse); + + } + + /* If the index or temporary table used by the GROUP BY sort + ** will naturally deliver rows in the order required by the ORDER BY + ** clause, cancel the ephemeral table open coded earlier. + ** + ** This is an optimization - the correct answer should result regardless. + ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to + ** disable this optimization for testing purposes. */ + if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) + && (groupBySort || sqlite3WhereIsSorted(pWInfo)) + ){ + sSort.pOrderBy = 0; + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); + } + + /* Evaluate the current GROUP BY terms and store in b0, b1, b2... + ** (b0 is memory location iBMem+0, b1 is iBMem+1, and so forth) + ** Then compare the current GROUP BY terms against the GROUP BY terms + ** from the previous row currently stored in a0, a1, a2... + */ + addrTopOfLoop = sqlite3VdbeCurrentAddr(v); + sqlite3ExprCacheClear(pParse); + if( groupBySort ){ + sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut); + } + for(j=0; jnExpr; j++){ + if( groupBySort ){ + sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); + if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + }else{ + sAggInfo.directMode = 1; + sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); + } + } + sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, + (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); + j1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); VdbeCoverage(v); + + /* Generate code that runs whenever the GROUP BY changes. + ** Changes in the GROUP BY are detected by the previous code + ** block. If there were no changes, this block is skipped. + ** + ** This code copies current group by terms in b0,b1,b2,... + ** over to a0,a1,a2. It then calls the output subroutine + ** and resets the aggregate accumulator registers in preparation + ** for the next GROUP BY batch. + */ + sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output one row")); + sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); + VdbeComment((v, "check abort flag")); + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + VdbeComment((v, "reset accumulator")); + + /* Update the aggregate accumulators based on the content of + ** the current row + */ + sqlite3VdbeJumpHere(v, j1); + updateAccumulator(pParse, &sAggInfo); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); + VdbeComment((v, "indicate data in accumulator")); + + /* End of the loop + */ + if( groupBySort ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, sAggInfo.sortingIdx, addrTopOfLoop); + VdbeCoverage(v); + }else{ + sqlite3WhereEnd(pWInfo); + sqlite3VdbeChangeToNoop(v, addrSortingIdx); + } + + /* Output the final row of result + */ + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output final row")); + + /* Jump over the subroutines + */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEnd); + + /* Generate a subroutine that outputs a single row of the result + ** set. This subroutine first looks at the iUseFlag. If iUseFlag + ** is less than or equal to zero, the subroutine is a no-op. If + ** the processing calls for the query to abort, this subroutine + ** increments the iAbortFlag memory location before returning in + ** order to signal the caller to abort. + */ + addrSetAbort = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iAbortFlag); + VdbeComment((v, "set abort flag")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + sqlite3VdbeResolveLabel(v, addrOutputRow); + addrOutputRow = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); + VdbeComment((v, "Groupby result generator entry point")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + finalizeAggFunctions(pParse, &sAggInfo); + sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); + selectInnerLoop(pParse, p, p->pEList, -1, &sSort, + &sDistinct, pDest, + addrOutputRow+1, addrSetAbort); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + VdbeComment((v, "end groupby result generator")); + + /* Generate a subroutine that will reset the group-by accumulator + */ + sqlite3VdbeResolveLabel(v, addrReset); + resetAccumulator(pParse, &sAggInfo); + sqlite3VdbeAddOp1(v, OP_Return, regReset); + + } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ + else { + ExprList *pDel = 0; +#ifndef SQLITE_OMIT_BTREECOUNT + Table *pTab; + if( (pTab = isSimpleCount(p, &sAggInfo))!=0 ){ + /* If isSimpleCount() returns a pointer to a Table structure, then + ** the SQL statement is of the form: + ** + ** SELECT count(*) FROM + ** + ** where the Table structure returned represents table . + ** + ** This statement is so common that it is optimized specially. The + ** OP_Count instruction is executed either on the intkey table that + ** contains the data for table or on one of its indexes. It + ** is better to execute the op on an index, as indexes are almost + ** always spread across less pages than their corresponding tables. + */ + const int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + const int iCsr = pParse->nTab++; /* Cursor to scan b-tree */ + Index *pIdx; /* Iterator variable */ + KeyInfo *pKeyInfo = 0; /* Keyinfo for scanned index */ + Index *pBest = 0; /* Best index found so far */ + int iRoot = pTab->tnum; /* Root page of scanned b-tree */ + + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + + /* Search for the index that has the lowest scan cost. + ** + ** (2011-04-15) Do not do a full scan of an unordered index. + ** + ** (2013-10-03) Do not count the entries in a partial index. + ** + ** In practice the KeyInfo structure will not be used. It is only + ** passed to keep OP_OpenRead happy. + */ + if( !HasRowid(pTab) ) pBest = sqlite3PrimaryKeyIndex(pTab); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->bUnordered==0 + && pIdx->szIdxRowszTabRow + && pIdx->pPartIdxWhere==0 + && (!pBest || pIdx->szIdxRowszIdxRow) + ){ + pBest = pIdx; + } + } + if( pBest ){ + iRoot = pBest->tnum; + pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pBest); + } + + /* Open a read-only cursor, execute the OP_Count, close the cursor. */ + sqlite3VdbeAddOp4Int(v, OP_OpenRead, iCsr, iRoot, iDb, 1); + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); + } + sqlite3VdbeAddOp2(v, OP_Count, iCsr, sAggInfo.aFunc[0].iMem); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); + explainSimpleCount(pParse, pTab, pBest); + }else +#endif /* SQLITE_OMIT_BTREECOUNT */ + { + /* Check if the query is of one of the following forms: + ** + ** SELECT min(x) FROM ... + ** SELECT max(x) FROM ... + ** + ** If it is, then ask the code in where.c to attempt to sort results + ** as if there was an "ORDER ON x" or "ORDER ON x DESC" clause. + ** If where.c is able to produce results sorted in this order, then + ** add vdbe code to break out of the processing loop after the + ** first iteration (since the first iteration of the loop is + ** guaranteed to operate on the row with the minimum or maximum + ** value of x, the only row required). + ** + ** A special flag must be passed to sqlite3WhereBegin() to slightly + ** modify behavior as follows: + ** + ** + If the query is a "SELECT min(x)", then the loop coded by + ** where.c should not iterate over any values with a NULL value + ** for x. + ** + ** + The optimizer code in where.c (the thing that decides which + ** index or indices to use) should place a different priority on + ** satisfying the 'ORDER BY' clause than it does in other cases. + ** Refer to code and comments in where.c for details. + */ + ExprList *pMinMax = 0; + u8 flag = WHERE_ORDERBY_NORMAL; + + assert( p->pGroupBy==0 ); + assert( flag==0 ); + if( p->pHaving==0 ){ + flag = minMaxQuery(&sAggInfo, &pMinMax); + } + assert( flag==0 || (pMinMax!=0 && pMinMax->nExpr==1) ); + + if( flag ){ + pMinMax = sqlite3ExprListDup(db, pMinMax, 0); + pDel = pMinMax; + if( pMinMax && !db->mallocFailed ){ + pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; + pMinMax->a[0].pExpr->op = TK_COLUMN; + } + } + + /* This case runs if the aggregate has no GROUP BY clause. The + ** processing is much simpler since there is only a single row + ** of output. + */ + resetAccumulator(pParse, &sAggInfo); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); + if( pWInfo==0 ){ + sqlite3ExprListDelete(db, pDel); + goto select_end; + } + updateAccumulator(pParse, &sAggInfo); + assert( pMinMax==0 || pMinMax->nExpr==1 ); + if( sqlite3WhereIsOrdered(pWInfo)>0 ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); + VdbeComment((v, "%s() by index", + (flag==WHERE_ORDERBY_MIN?"min":"max"))); + } + sqlite3WhereEnd(pWInfo); + finalizeAggFunctions(pParse, &sAggInfo); + } + + sSort.pOrderBy = 0; + sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, + pDest, addrEnd, addrEnd); + sqlite3ExprListDelete(db, pDel); + } + sqlite3VdbeResolveLabel(v, addrEnd); + + } /* endif aggregate query */ + + if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){ + explainTempTable(pParse, "DISTINCT"); + } + + /* If there is an ORDER BY clause, then we need to sort the results + ** and send them to the callback one by one. + */ + if( sSort.pOrderBy ){ + explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); + } + + /* Jump here to skip this query + */ + sqlite3VdbeResolveLabel(v, iEnd); + + /* The SELECT was successfully coded. Set the return code to 0 + ** to indicate no errors. + */ + rc = 0; + + /* Control jumps to here if an error is encountered above, or upon + ** successful coding of the SELECT. + */ +select_end: + explainSetInteger(pParse->iSelectId, iRestoreSelectId); + + /* Identify column names if results of the SELECT are to be output. + */ + if( rc==SQLITE_OK && pDest->eDest==SRT_Output ){ + generateColumnNames(pParse, pTabList, pEList); + } + + sqlite3DbFree(db, sAggInfo.aCol); + sqlite3DbFree(db, sAggInfo.aFunc); + return rc; +} + +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +/* +** Generate a human-readable description of a the Select object. +*/ +static void explainOneSelect(Vdbe *pVdbe, Select *p){ + sqlite3ExplainPrintf(pVdbe, "SELECT "); + if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ + if( p->selFlags & SF_Distinct ){ + sqlite3ExplainPrintf(pVdbe, "DISTINCT "); + } + if( p->selFlags & SF_Aggregate ){ + sqlite3ExplainPrintf(pVdbe, "agg_flag "); + } + sqlite3ExplainNL(pVdbe); + sqlite3ExplainPrintf(pVdbe, " "); + } + sqlite3ExplainExprList(pVdbe, p->pEList); + sqlite3ExplainNL(pVdbe); + if( p->pSrc && p->pSrc->nSrc ){ + int i; + sqlite3ExplainPrintf(pVdbe, "FROM "); + sqlite3ExplainPush(pVdbe); + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + sqlite3ExplainPrintf(pVdbe, "{%d,*} = ", pItem->iCursor); + if( pItem->pSelect ){ + sqlite3ExplainSelect(pVdbe, pItem->pSelect); + if( pItem->pTab ){ + sqlite3ExplainPrintf(pVdbe, " (tabname=%s)", pItem->pTab->zName); + } + }else if( pItem->zName ){ + sqlite3ExplainPrintf(pVdbe, "%s", pItem->zName); + } + if( pItem->zAlias ){ + sqlite3ExplainPrintf(pVdbe, " (AS %s)", pItem->zAlias); + } + if( pItem->jointype & JT_LEFT ){ + sqlite3ExplainPrintf(pVdbe, " LEFT-JOIN"); + } + sqlite3ExplainNL(pVdbe); + } + sqlite3ExplainPop(pVdbe); + } + if( p->pWhere ){ + sqlite3ExplainPrintf(pVdbe, "WHERE "); + sqlite3ExplainExpr(pVdbe, p->pWhere); + sqlite3ExplainNL(pVdbe); + } + if( p->pGroupBy ){ + sqlite3ExplainPrintf(pVdbe, "GROUPBY "); + sqlite3ExplainExprList(pVdbe, p->pGroupBy); + sqlite3ExplainNL(pVdbe); + } + if( p->pHaving ){ + sqlite3ExplainPrintf(pVdbe, "HAVING "); + sqlite3ExplainExpr(pVdbe, p->pHaving); + sqlite3ExplainNL(pVdbe); + } + if( p->pOrderBy ){ + sqlite3ExplainPrintf(pVdbe, "ORDERBY "); + sqlite3ExplainExprList(pVdbe, p->pOrderBy); + sqlite3ExplainNL(pVdbe); + } + if( p->pLimit ){ + sqlite3ExplainPrintf(pVdbe, "LIMIT "); + sqlite3ExplainExpr(pVdbe, p->pLimit); + sqlite3ExplainNL(pVdbe); + } + if( p->pOffset ){ + sqlite3ExplainPrintf(pVdbe, "OFFSET "); + sqlite3ExplainExpr(pVdbe, p->pOffset); + sqlite3ExplainNL(pVdbe); + } +} +SQLITE_PRIVATE void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ + if( p==0 ){ + sqlite3ExplainPrintf(pVdbe, "(null-select)"); + return; + } + sqlite3ExplainPush(pVdbe); + while( p ){ + explainOneSelect(pVdbe, p); + p = p->pNext; + if( p==0 ) break; + sqlite3ExplainNL(pVdbe); + sqlite3ExplainPrintf(pVdbe, "%s\n", selectOpName(p->op)); + } + sqlite3ExplainPrintf(pVdbe, "END"); + sqlite3ExplainPop(pVdbe); +} + +/* End of the structure debug printing code +*****************************************************************************/ +#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ + +/************** End of select.c **********************************************/ +/************** Begin file table.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the sqlite3_get_table() and sqlite3_free_table() +** interface routines. These are just wrappers around the main +** interface routine of sqlite3_exec(). +** +** These routines are in a separate files so that they will not be linked +** if they are not used. +*/ +/* #include */ +/* #include */ + +#ifndef SQLITE_OMIT_GET_TABLE + +/* +** This structure is used to pass data from sqlite3_get_table() through +** to the callback function is uses to build the result. +*/ +typedef struct TabResult { + char **azResult; /* Accumulated output */ + char *zErrMsg; /* Error message text, if an error occurs */ + int nAlloc; /* Slots allocated for azResult[] */ + int nRow; /* Number of rows in the result */ + int nColumn; /* Number of columns in the result */ + int nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ + int rc; /* Return code from sqlite3_exec() */ +} TabResult; + +/* +** This routine is called once for each row in the result table. Its job +** is to fill in the TabResult structure appropriately, allocating new +** memory as necessary. +*/ +static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ + TabResult *p = (TabResult*)pArg; /* Result accumulator */ + int need; /* Slots needed in p->azResult[] */ + int i; /* Loop counter */ + char *z; /* A single column of result */ + + /* Make sure there is enough space in p->azResult to hold everything + ** we need to remember from this invocation of the callback. + */ + if( p->nRow==0 && argv!=0 ){ + need = nCol*2; + }else{ + need = nCol; + } + if( p->nData + need > p->nAlloc ){ + char **azNew; + p->nAlloc = p->nAlloc*2 + need; + azNew = sqlite3_realloc( p->azResult, sizeof(char*)*p->nAlloc ); + if( azNew==0 ) goto malloc_failed; + p->azResult = azNew; + } + + /* If this is the first row, then generate an extra row containing + ** the names of all columns. + */ + if( p->nRow==0 ){ + p->nColumn = nCol; + for(i=0; iazResult[p->nData++] = z; + } + }else if( p->nColumn!=nCol ){ + sqlite3_free(p->zErrMsg); + p->zErrMsg = sqlite3_mprintf( + "sqlite3_get_table() called with two or more incompatible queries" + ); + p->rc = SQLITE_ERROR; + return 1; + } + + /* Copy over the row data + */ + if( argv!=0 ){ + for(i=0; iazResult[p->nData++] = z; + } + p->nRow++; + } + return 0; + +malloc_failed: + p->rc = SQLITE_NOMEM; + return 1; +} + +/* +** Query the database. But instead of invoking a callback for each row, +** malloc() for space to hold the result and return the entire results +** at the conclusion of the call. +** +** The result that is written to ***pazResult is held in memory obtained +** from malloc(). But the caller cannot free this memory directly. +** Instead, the entire table should be passed to sqlite3_free_table() when +** the calling procedure is finished using it. +*/ +SQLITE_API int sqlite3_get_table( + sqlite3 *db, /* The database on which the SQL executes */ + const char *zSql, /* The SQL to be executed */ + char ***pazResult, /* Write the result table here */ + int *pnRow, /* Write the number of rows in the result here */ + int *pnColumn, /* Write the number of columns of result here */ + char **pzErrMsg /* Write error messages here */ +){ + int rc; + TabResult res; + + *pazResult = 0; + if( pnColumn ) *pnColumn = 0; + if( pnRow ) *pnRow = 0; + if( pzErrMsg ) *pzErrMsg = 0; + res.zErrMsg = 0; + res.nRow = 0; + res.nColumn = 0; + res.nData = 1; + res.nAlloc = 20; + res.rc = SQLITE_OK; + res.azResult = sqlite3_malloc(sizeof(char*)*res.nAlloc ); + if( res.azResult==0 ){ + db->errCode = SQLITE_NOMEM; + return SQLITE_NOMEM; + } + res.azResult[0] = 0; + rc = sqlite3_exec(db, zSql, sqlite3_get_table_cb, &res, pzErrMsg); + assert( sizeof(res.azResult[0])>= sizeof(res.nData) ); + res.azResult[0] = SQLITE_INT_TO_PTR(res.nData); + if( (rc&0xff)==SQLITE_ABORT ){ + sqlite3_free_table(&res.azResult[1]); + if( res.zErrMsg ){ + if( pzErrMsg ){ + sqlite3_free(*pzErrMsg); + *pzErrMsg = sqlite3_mprintf("%s",res.zErrMsg); + } + sqlite3_free(res.zErrMsg); + } + db->errCode = res.rc; /* Assume 32-bit assignment is atomic */ + return res.rc; + } + sqlite3_free(res.zErrMsg); + if( rc!=SQLITE_OK ){ + sqlite3_free_table(&res.azResult[1]); + return rc; + } + if( res.nAlloc>res.nData ){ + char **azNew; + azNew = sqlite3_realloc( res.azResult, sizeof(char*)*res.nData ); + if( azNew==0 ){ + sqlite3_free_table(&res.azResult[1]); + db->errCode = SQLITE_NOMEM; + return SQLITE_NOMEM; + } + res.azResult = azNew; + } + *pazResult = &res.azResult[1]; + if( pnColumn ) *pnColumn = res.nColumn; + if( pnRow ) *pnRow = res.nRow; + return rc; +} + +/* +** This routine frees the space the sqlite3_get_table() malloced. +*/ +SQLITE_API void sqlite3_free_table( + char **azResult /* Result returned from from sqlite3_get_table() */ +){ + if( azResult ){ + int i, n; + azResult--; + assert( azResult!=0 ); + n = SQLITE_PTR_TO_INT(azResult[0]); + for(i=1; ipNext; + + sqlite3ExprDelete(db, pTmp->pWhere); + sqlite3ExprListDelete(db, pTmp->pExprList); + sqlite3SelectDelete(db, pTmp->pSelect); + sqlite3IdListDelete(db, pTmp->pIdList); + + sqlite3DbFree(db, pTmp); + } +} + +/* +** Given table pTab, return a list of all the triggers attached to +** the table. The list is connected by Trigger.pNext pointers. +** +** All of the triggers on pTab that are in the same database as pTab +** are already attached to pTab->pTrigger. But there might be additional +** triggers on pTab in the TEMP schema. This routine prepends all +** TEMP triggers on pTab to the beginning of the pTab->pTrigger list +** and returns the combined list. +** +** To state it another way: This routine returns a list of all triggers +** that fire off of pTab. The list will include any TEMP triggers on +** pTab as well as the triggers lised in pTab->pTrigger. +*/ +SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ + Schema * const pTmpSchema = pParse->db->aDb[1].pSchema; + Trigger *pList = 0; /* List of triggers to return */ + + if( pParse->disableTriggers ){ + return 0; + } + + if( pTmpSchema!=pTab->pSchema ){ + HashElem *p; + assert( sqlite3SchemaMutexHeld(pParse->db, 0, pTmpSchema) ); + for(p=sqliteHashFirst(&pTmpSchema->trigHash); p; p=sqliteHashNext(p)){ + Trigger *pTrig = (Trigger *)sqliteHashData(p); + if( pTrig->pTabSchema==pTab->pSchema + && 0==sqlite3StrICmp(pTrig->table, pTab->zName) + ){ + pTrig->pNext = (pList ? pList : pTab->pTrigger); + pList = pTrig; + } + } + } + + return (pList ? pList : pTab->pTrigger); +} + +/* +** This is called by the parser when it sees a CREATE TRIGGER statement +** up to the point of the BEGIN before the trigger actions. A Trigger +** structure is generated based on the information available and stored +** in pParse->pNewTrigger. After the trigger actions have been parsed, the +** sqlite3FinishTrigger() function is called to complete the trigger +** construction process. +*/ +SQLITE_PRIVATE void sqlite3BeginTrigger( + Parse *pParse, /* The parse context of the CREATE TRIGGER statement */ + Token *pName1, /* The name of the trigger */ + Token *pName2, /* The name of the trigger */ + int tr_tm, /* One of TK_BEFORE, TK_AFTER, TK_INSTEAD */ + int op, /* One of TK_INSERT, TK_UPDATE, TK_DELETE */ + IdList *pColumns, /* column list if this is an UPDATE OF trigger */ + SrcList *pTableName,/* The name of the table/view the trigger applies to */ + Expr *pWhen, /* WHEN clause */ + int isTemp, /* True if the TEMPORARY keyword is present */ + int noErr /* Suppress errors if the trigger already exists */ +){ + Trigger *pTrigger = 0; /* The new trigger */ + Table *pTab; /* Table that the trigger fires off of */ + char *zName = 0; /* Name of the trigger */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb; /* The database to store the trigger in */ + Token *pName; /* The unqualified db name */ + DbFixer sFix; /* State vector for the DB fixer */ + int iTabDb; /* Index of the database holding pTab */ + + assert( pName1!=0 ); /* pName1->z might be NULL, but not pName1 itself */ + assert( pName2!=0 ); + assert( op==TK_INSERT || op==TK_UPDATE || op==TK_DELETE ); + assert( op>0 && op<0xff ); + if( isTemp ){ + /* If TEMP was specified, then the trigger name may not be qualified. */ + if( pName2->n>0 ){ + sqlite3ErrorMsg(pParse, "temporary trigger may not have qualified name"); + goto trigger_cleanup; + } + iDb = 1; + pName = pName1; + }else{ + /* Figure out the db that the trigger will be created in */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ){ + goto trigger_cleanup; + } + } + if( !pTableName || db->mallocFailed ){ + goto trigger_cleanup; + } + + /* A long-standing parser bug is that this syntax was allowed: + ** + ** CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab .... + ** ^^^^^^^^ + ** + ** To maintain backwards compatibility, ignore the database + ** name on pTableName if we are reparsing our of SQLITE_MASTER. + */ + if( db->init.busy && iDb!=1 ){ + sqlite3DbFree(db, pTableName->a[0].zDatabase); + pTableName->a[0].zDatabase = 0; + } + + /* If the trigger name was unqualified, and the table is a temp table, + ** then set iDb to 1 to create the trigger in the temporary database. + ** If sqlite3SrcListLookup() returns 0, indicating the table does not + ** exist, the error is caught by the block below. + */ + pTab = sqlite3SrcListLookup(pParse, pTableName); + if( db->init.busy==0 && pName2->n==0 && pTab + && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } + + /* Ensure the table name matches database name and that the table exists */ + if( db->mallocFailed ) goto trigger_cleanup; + assert( pTableName->nSrc==1 ); + sqlite3FixInit(&sFix, pParse, iDb, "trigger", pName); + if( sqlite3FixSrcList(&sFix, pTableName) ){ + goto trigger_cleanup; + } + pTab = sqlite3SrcListLookup(pParse, pTableName); + if( !pTab ){ + /* The table does not exist. */ + if( db->init.iDb==1 ){ + /* Ticket #3810. + ** Normally, whenever a table is dropped, all associated triggers are + ** dropped too. But if a TEMP trigger is created on a non-TEMP table + ** and the table is dropped by a different database connection, the + ** trigger is not visible to the database connection that does the + ** drop so the trigger cannot be dropped. This results in an + ** "orphaned trigger" - a trigger whose associated table is missing. + */ + db->init.orphanTrigger = 1; + } + goto trigger_cleanup; + } + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "cannot create triggers on virtual tables"); + goto trigger_cleanup; + } + + /* Check that the trigger name is not reserved and that no trigger of the + ** specified name exists */ + zName = sqlite3NameFromToken(db, pName); + if( !zName || SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + goto trigger_cleanup; + } + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash), + zName, sqlite3Strlen30(zName)) ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + } + goto trigger_cleanup; + } + + /* Do not create a trigger on a system table */ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); + pParse->nErr++; + goto trigger_cleanup; + } + + /* INSTEAD of triggers are only for views and views only support INSTEAD + ** of triggers. + */ + if( pTab->pSelect && tr_tm!=TK_INSTEAD ){ + sqlite3ErrorMsg(pParse, "cannot create %s trigger on view: %S", + (tr_tm == TK_BEFORE)?"BEFORE":"AFTER", pTableName, 0); + goto trigger_cleanup; + } + if( !pTab->pSelect && tr_tm==TK_INSTEAD ){ + sqlite3ErrorMsg(pParse, "cannot create INSTEAD OF" + " trigger on table: %S", pTableName, 0); + goto trigger_cleanup; + } + iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); + +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code = SQLITE_CREATE_TRIGGER; + const char *zDb = db->aDb[iTabDb].zName; + const char *zDbTrig = isTemp ? db->aDb[1].zName : zDb; + if( iTabDb==1 || isTemp ) code = SQLITE_CREATE_TEMP_TRIGGER; + if( sqlite3AuthCheck(pParse, code, zName, pTab->zName, zDbTrig) ){ + goto trigger_cleanup; + } + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iTabDb),0,zDb)){ + goto trigger_cleanup; + } + } +#endif + + /* INSTEAD OF triggers can only appear on views and BEFORE triggers + ** cannot appear on views. So we might as well translate every + ** INSTEAD OF trigger into a BEFORE trigger. It simplifies code + ** elsewhere. + */ + if (tr_tm == TK_INSTEAD){ + tr_tm = TK_BEFORE; + } + + /* Build the Trigger object */ + pTrigger = (Trigger*)sqlite3DbMallocZero(db, sizeof(Trigger)); + if( pTrigger==0 ) goto trigger_cleanup; + pTrigger->zName = zName; + zName = 0; + pTrigger->table = sqlite3DbStrDup(db, pTableName->a[0].zName); + pTrigger->pSchema = db->aDb[iDb].pSchema; + pTrigger->pTabSchema = pTab->pSchema; + pTrigger->op = (u8)op; + pTrigger->tr_tm = tr_tm==TK_BEFORE ? TRIGGER_BEFORE : TRIGGER_AFTER; + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); + pTrigger->pColumns = sqlite3IdListDup(db, pColumns); + assert( pParse->pNewTrigger==0 ); + pParse->pNewTrigger = pTrigger; + +trigger_cleanup: + sqlite3DbFree(db, zName); + sqlite3SrcListDelete(db, pTableName); + sqlite3IdListDelete(db, pColumns); + sqlite3ExprDelete(db, pWhen); + if( !pParse->pNewTrigger ){ + sqlite3DeleteTrigger(db, pTrigger); + }else{ + assert( pParse->pNewTrigger==pTrigger ); + } +} + +/* +** This routine is called after all of the trigger actions have been parsed +** in order to complete the process of building the trigger. +*/ +SQLITE_PRIVATE void sqlite3FinishTrigger( + Parse *pParse, /* Parser context */ + TriggerStep *pStepList, /* The triggered program */ + Token *pAll /* Token that describes the complete CREATE TRIGGER */ +){ + Trigger *pTrig = pParse->pNewTrigger; /* Trigger being finished */ + char *zName; /* Name of trigger */ + sqlite3 *db = pParse->db; /* The database */ + DbFixer sFix; /* Fixer object */ + int iDb; /* Database containing the trigger */ + Token nameToken; /* Trigger name for error reporting */ + + pParse->pNewTrigger = 0; + if( NEVER(pParse->nErr) || !pTrig ) goto triggerfinish_cleanup; + zName = pTrig->zName; + iDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); + pTrig->step_list = pStepList; + while( pStepList ){ + pStepList->pTrig = pTrig; + pStepList = pStepList->pNext; + } + nameToken.z = pTrig->zName; + nameToken.n = sqlite3Strlen30(nameToken.z); + sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken); + if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) + || sqlite3FixExpr(&sFix, pTrig->pWhen) + ){ + goto triggerfinish_cleanup; + } + + /* if we are not initializing, + ** build the sqlite_master entry + */ + if( !db->init.busy ){ + Vdbe *v; + char *z; + + /* Make an entry in the sqlite_master table */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto triggerfinish_cleanup; + sqlite3BeginWriteOperation(pParse, 0, iDb); + z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n); + sqlite3NestedParse(pParse, + "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName, + pTrig->table, z); + sqlite3DbFree(db, z); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName)); + } + + if( db->init.busy ){ + Trigger *pLink = pTrig; + Hash *pHash = &db->aDb[iDb].pSchema->trigHash; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pTrig = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), pTrig); + if( pTrig ){ + db->mallocFailed = 1; + }else if( pLink->pSchema==pLink->pTabSchema ){ + Table *pTab; + int n = sqlite3Strlen30(pLink->table); + pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table, n); + assert( pTab!=0 ); + pLink->pNext = pTab->pTrigger; + pTab->pTrigger = pLink; + } + } + +triggerfinish_cleanup: + sqlite3DeleteTrigger(db, pTrig); + assert( !pParse->pNewTrigger ); + sqlite3DeleteTriggerStep(db, pStepList); +} + +/* +** Turn a SELECT statement (that the pSelect parameter points to) into +** a trigger step. Return a pointer to a TriggerStep structure. +** +** The parser calls this routine when it finds a SELECT statement in +** body of a TRIGGER. +*/ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3 *db, Select *pSelect){ + TriggerStep *pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep)); + if( pTriggerStep==0 ) { + sqlite3SelectDelete(db, pSelect); + return 0; + } + pTriggerStep->op = TK_SELECT; + pTriggerStep->pSelect = pSelect; + pTriggerStep->orconf = OE_Default; + return pTriggerStep; +} + +/* +** Allocate space to hold a new trigger step. The allocated space +** holds both the TriggerStep object and the TriggerStep.target.z string. +** +** If an OOM error occurs, NULL is returned and db->mallocFailed is set. +*/ +static TriggerStep *triggerStepAllocate( + sqlite3 *db, /* Database connection */ + u8 op, /* Trigger opcode */ + Token *pName /* The target name */ +){ + TriggerStep *pTriggerStep; + + pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n); + if( pTriggerStep ){ + char *z = (char*)&pTriggerStep[1]; + memcpy(z, pName->z, pName->n); + pTriggerStep->target.z = z; + pTriggerStep->target.n = pName->n; + pTriggerStep->op = op; + } + return pTriggerStep; +} + +/* +** Build a trigger step out of an INSERT statement. Return a pointer +** to the new trigger step. +** +** The parser calls this routine when it sees an INSERT inside the +** body of a trigger. +*/ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( + sqlite3 *db, /* The database connection */ + Token *pTableName, /* Name of the table into which we insert */ + IdList *pColumn, /* List of columns in pTableName to insert into */ + Select *pSelect, /* A SELECT statement that supplies values */ + u8 orconf /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ +){ + TriggerStep *pTriggerStep; + + assert(pSelect != 0 || db->mallocFailed); + + pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName); + if( pTriggerStep ){ + pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + pTriggerStep->pIdList = pColumn; + pTriggerStep->orconf = orconf; + }else{ + sqlite3IdListDelete(db, pColumn); + } + sqlite3SelectDelete(db, pSelect); + + return pTriggerStep; +} + +/* +** Construct a trigger step that implements an UPDATE statement and return +** a pointer to that trigger step. The parser calls this routine when it +** sees an UPDATE statement inside the body of a CREATE TRIGGER. +*/ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( + sqlite3 *db, /* The database connection */ + Token *pTableName, /* Name of the table to be updated */ + ExprList *pEList, /* The SET clause: list of column and new values */ + Expr *pWhere, /* The WHERE clause */ + u8 orconf /* The conflict algorithm. (OE_Abort, OE_Ignore, etc) */ +){ + TriggerStep *pTriggerStep; + + pTriggerStep = triggerStepAllocate(db, TK_UPDATE, pTableName); + if( pTriggerStep ){ + pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pTriggerStep->orconf = orconf; + } + sqlite3ExprListDelete(db, pEList); + sqlite3ExprDelete(db, pWhere); + return pTriggerStep; +} + +/* +** Construct a trigger step that implements a DELETE statement and return +** a pointer to that trigger step. The parser calls this routine when it +** sees a DELETE statement inside the body of a CREATE TRIGGER. +*/ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep( + sqlite3 *db, /* Database connection */ + Token *pTableName, /* The table from which rows are deleted */ + Expr *pWhere /* The WHERE clause */ +){ + TriggerStep *pTriggerStep; + + pTriggerStep = triggerStepAllocate(db, TK_DELETE, pTableName); + if( pTriggerStep ){ + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pTriggerStep->orconf = OE_Default; + } + sqlite3ExprDelete(db, pWhere); + return pTriggerStep; +} + +/* +** Recursively delete a Trigger structure +*/ +SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3 *db, Trigger *pTrigger){ + if( pTrigger==0 ) return; + sqlite3DeleteTriggerStep(db, pTrigger->step_list); + sqlite3DbFree(db, pTrigger->zName); + sqlite3DbFree(db, pTrigger->table); + sqlite3ExprDelete(db, pTrigger->pWhen); + sqlite3IdListDelete(db, pTrigger->pColumns); + sqlite3DbFree(db, pTrigger); +} + +/* +** This function is called to drop a trigger from the database schema. +** +** This may be called directly from the parser and therefore identifies +** the trigger by name. The sqlite3DropTriggerPtr() routine does the +** same job as this routine except it takes a pointer to the trigger +** instead of the trigger name. +**/ +SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ + Trigger *pTrigger = 0; + int i; + const char *zDb; + const char *zName; + int nName; + sqlite3 *db = pParse->db; + + if( db->mallocFailed ) goto drop_trigger_cleanup; + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto drop_trigger_cleanup; + } + + assert( pName->nSrc==1 ); + zDb = pName->a[0].zDatabase; + zName = pName->a[0].zName; + nName = sqlite3Strlen30(zName); + assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName, nName); + if( pTrigger ) break; + } + if( !pTrigger ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "no such trigger: %S", pName, 0); + }else{ + sqlite3CodeVerifyNamedSchema(pParse, zDb); + } + pParse->checkSchema = 1; + goto drop_trigger_cleanup; + } + sqlite3DropTriggerPtr(pParse, pTrigger); + +drop_trigger_cleanup: + sqlite3SrcListDelete(db, pName); +} + +/* +** Return a pointer to the Table structure for the table that a trigger +** is set on. +*/ +static Table *tableOfTrigger(Trigger *pTrigger){ + int n = sqlite3Strlen30(pTrigger->table); + return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table, n); +} + + +/* +** Drop a trigger given a pointer to that trigger. +*/ +SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ + Table *pTable; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; + + iDb = sqlite3SchemaToIndex(pParse->db, pTrigger->pSchema); + assert( iDb>=0 && iDbnDb ); + pTable = tableOfTrigger(pTrigger); + assert( pTable ); + assert( pTable->pSchema==pTrigger->pSchema || iDb==1 ); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code = SQLITE_DROP_TRIGGER; + const char *zDb = db->aDb[iDb].zName; + const char *zTab = SCHEMA_TABLE(iDb); + if( iDb==1 ) code = SQLITE_DROP_TEMP_TRIGGER; + if( sqlite3AuthCheck(pParse, code, pTrigger->zName, pTable->zName, zDb) || + sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ + return; + } + } +#endif + + /* Generate code to destroy the database record of the trigger. + */ + assert( pTable!=0 ); + if( (v = sqlite3GetVdbe(pParse))!=0 ){ + int base; + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList dropTrigger[] = { + { OP_Rewind, 0, ADDR(9), 0}, + { OP_String8, 0, 1, 0}, /* 1 */ + { OP_Column, 0, 1, 2}, + { OP_Ne, 2, ADDR(8), 1}, + { OP_String8, 0, 1, 0}, /* 4: "trigger" */ + { OP_Column, 0, 0, 2}, + { OP_Ne, 2, ADDR(8), 1}, + { OP_Delete, 0, 0, 0}, + { OP_Next, 0, ADDR(1), 0}, /* 8 */ + }; + + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3OpenMasterTable(pParse, iDb); + base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger, iLn); + sqlite3VdbeChangeP4(v, base+1, pTrigger->zName, P4_TRANSIENT); + sqlite3VdbeChangeP4(v, base+4, "trigger", P4_STATIC); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddOp2(v, OP_Close, 0, 0); + sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0); + if( pParse->nMem<3 ){ + pParse->nMem = 3; + } + } +} + +/* +** Remove a trigger from the hash tables of the sqlite* pointer. +*/ +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){ + Trigger *pTrigger; + Hash *pHash; + + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pHash = &(db->aDb[iDb].pSchema->trigHash); + pTrigger = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), 0); + if( ALWAYS(pTrigger) ){ + if( pTrigger->pSchema==pTrigger->pTabSchema ){ + Table *pTab = tableOfTrigger(pTrigger); + Trigger **pp; + for(pp=&pTab->pTrigger; *pp!=pTrigger; pp=&((*pp)->pNext)); + *pp = (*pp)->pNext; + } + sqlite3DeleteTrigger(db, pTrigger); + db->flags |= SQLITE_InternChanges; + } +} + +/* +** pEList is the SET clause of an UPDATE statement. Each entry +** in pEList is of the format =. If any of the entries +** in pEList have an which matches an identifier in pIdList, +** then return TRUE. If pIdList==NULL, then it is considered a +** wildcard that matches anything. Likewise if pEList==NULL then +** it matches anything so always return true. Return false only +** if there is no match. +*/ +static int checkColumnOverlap(IdList *pIdList, ExprList *pEList){ + int e; + if( pIdList==0 || NEVER(pEList==0) ) return 1; + for(e=0; enExpr; e++){ + if( sqlite3IdListIndex(pIdList, pEList->a[e].zName)>=0 ) return 1; + } + return 0; +} + +/* +** Return a list of all triggers on table pTab if there exists at least +** one trigger that must be fired when an operation of type 'op' is +** performed on the table, and, if that operation is an UPDATE, if at +** least one of the columns in pChanges is being modified. +*/ +SQLITE_PRIVATE Trigger *sqlite3TriggersExist( + Parse *pParse, /* Parse context */ + Table *pTab, /* The table the contains the triggers */ + int op, /* one of TK_DELETE, TK_INSERT, TK_UPDATE */ + ExprList *pChanges, /* Columns that change in an UPDATE statement */ + int *pMask /* OUT: Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ +){ + int mask = 0; + Trigger *pList = 0; + Trigger *p; + + if( (pParse->db->flags & SQLITE_EnableTrigger)!=0 ){ + pList = sqlite3TriggerList(pParse, pTab); + } + assert( pList==0 || IsVirtual(pTab)==0 ); + for(p=pList; p; p=p->pNext){ + if( p->op==op && checkColumnOverlap(p->pColumns, pChanges) ){ + mask |= p->tr_tm; + } + } + if( pMask ){ + *pMask = mask; + } + return (mask ? pList : 0); +} + +/* +** Convert the pStep->target token into a SrcList and return a pointer +** to that SrcList. +** +** This routine adds a specific database name, if needed, to the target when +** forming the SrcList. This prevents a trigger in one database from +** referring to a target in another database. An exception is when the +** trigger is in TEMP in which case it can refer to any other database it +** wants. +*/ +static SrcList *targetSrcList( + Parse *pParse, /* The parsing context */ + TriggerStep *pStep /* The trigger containing the target token */ +){ + int iDb; /* Index of the database to use */ + SrcList *pSrc; /* SrcList to be returned */ + + pSrc = sqlite3SrcListAppend(pParse->db, 0, &pStep->target, 0); + if( pSrc ){ + assert( pSrc->nSrc>0 ); + assert( pSrc->a!=0 ); + iDb = sqlite3SchemaToIndex(pParse->db, pStep->pTrig->pSchema); + if( iDb==0 || iDb>=2 ){ + sqlite3 *db = pParse->db; + assert( iDbdb->nDb ); + pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + } + } + return pSrc; +} + +/* +** Generate VDBE code for the statements inside the body of a single +** trigger. +*/ +static int codeTriggerProgram( + Parse *pParse, /* The parser context */ + TriggerStep *pStepList, /* List of statements inside the trigger body */ + int orconf /* Conflict algorithm. (OE_Abort, etc) */ +){ + TriggerStep *pStep; + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + + assert( pParse->pTriggerTab && pParse->pToplevel ); + assert( pStepList ); + assert( v!=0 ); + for(pStep=pStepList; pStep; pStep=pStep->pNext){ + /* Figure out the ON CONFLICT policy that will be used for this step + ** of the trigger program. If the statement that caused this trigger + ** to fire had an explicit ON CONFLICT, then use it. Otherwise, use + ** the ON CONFLICT policy that was specified as part of the trigger + ** step statement. Example: + ** + ** CREATE TRIGGER AFTER INSERT ON t1 BEGIN; + ** INSERT OR REPLACE INTO t2 VALUES(new.a, new.b); + ** END; + ** + ** INSERT INTO t1 ... ; -- insert into t2 uses REPLACE policy + ** INSERT OR IGNORE INTO t1 ... ; -- insert into t2 uses IGNORE policy + */ + pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf; + assert( pParse->okConstFactor==0 ); + + switch( pStep->op ){ + case TK_UPDATE: { + sqlite3Update(pParse, + targetSrcList(pParse, pStep), + sqlite3ExprListDup(db, pStep->pExprList, 0), + sqlite3ExprDup(db, pStep->pWhere, 0), + pParse->eOrconf + ); + break; + } + case TK_INSERT: { + sqlite3Insert(pParse, + targetSrcList(pParse, pStep), + sqlite3SelectDup(db, pStep->pSelect, 0), + sqlite3IdListDup(db, pStep->pIdList), + pParse->eOrconf + ); + break; + } + case TK_DELETE: { + sqlite3DeleteFrom(pParse, + targetSrcList(pParse, pStep), + sqlite3ExprDup(db, pStep->pWhere, 0) + ); + break; + } + default: assert( pStep->op==TK_SELECT ); { + SelectDest sDest; + Select *pSelect = sqlite3SelectDup(db, pStep->pSelect, 0); + sqlite3SelectDestInit(&sDest, SRT_Discard, 0); + sqlite3Select(pParse, pSelect, &sDest); + sqlite3SelectDelete(db, pSelect); + break; + } + } + if( pStep->op!=TK_SELECT ){ + sqlite3VdbeAddOp0(v, OP_ResetCount); + } + } + + return 0; +} + +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +/* +** This function is used to add VdbeComment() annotations to a VDBE +** program. It is not used in production code, only for debugging. +*/ +static const char *onErrorText(int onError){ + switch( onError ){ + case OE_Abort: return "abort"; + case OE_Rollback: return "rollback"; + case OE_Fail: return "fail"; + case OE_Replace: return "replace"; + case OE_Ignore: return "ignore"; + case OE_Default: return "default"; + } + return "n/a"; +} +#endif + +/* +** Parse context structure pFrom has just been used to create a sub-vdbe +** (trigger program). If an error has occurred, transfer error information +** from pFrom to pTo. +*/ +static void transferParseError(Parse *pTo, Parse *pFrom){ + assert( pFrom->zErrMsg==0 || pFrom->nErr ); + assert( pTo->zErrMsg==0 || pTo->nErr ); + if( pTo->nErr==0 ){ + pTo->zErrMsg = pFrom->zErrMsg; + pTo->nErr = pFrom->nErr; + }else{ + sqlite3DbFree(pFrom->db, pFrom->zErrMsg); + } +} + +/* +** Create and populate a new TriggerPrg object with a sub-program +** implementing trigger pTrigger with ON CONFLICT policy orconf. +*/ +static TriggerPrg *codeRowTrigger( + Parse *pParse, /* Current parse context */ + Trigger *pTrigger, /* Trigger to code */ + Table *pTab, /* The table pTrigger is attached to */ + int orconf /* ON CONFLICT policy to code trigger program with */ +){ + Parse *pTop = sqlite3ParseToplevel(pParse); + sqlite3 *db = pParse->db; /* Database handle */ + TriggerPrg *pPrg; /* Value to return */ + Expr *pWhen = 0; /* Duplicate of trigger WHEN expression */ + Vdbe *v; /* Temporary VM */ + NameContext sNC; /* Name context for sub-vdbe */ + SubProgram *pProgram = 0; /* Sub-vdbe for trigger program */ + Parse *pSubParse; /* Parse context for sub-vdbe */ + int iEndTrigger = 0; /* Label to jump to if WHEN is false */ + + assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); + assert( pTop->pVdbe ); + + /* Allocate the TriggerPrg and SubProgram objects. To ensure that they + ** are freed if an error occurs, link them into the Parse.pTriggerPrg + ** list of the top-level Parse object sooner rather than later. */ + pPrg = sqlite3DbMallocZero(db, sizeof(TriggerPrg)); + if( !pPrg ) return 0; + pPrg->pNext = pTop->pTriggerPrg; + pTop->pTriggerPrg = pPrg; + pPrg->pProgram = pProgram = sqlite3DbMallocZero(db, sizeof(SubProgram)); + if( !pProgram ) return 0; + sqlite3VdbeLinkSubProgram(pTop->pVdbe, pProgram); + pPrg->pTrigger = pTrigger; + pPrg->orconf = orconf; + pPrg->aColmask[0] = 0xffffffff; + pPrg->aColmask[1] = 0xffffffff; + + /* Allocate and populate a new Parse context to use for coding the + ** trigger sub-program. */ + pSubParse = sqlite3StackAllocZero(db, sizeof(Parse)); + if( !pSubParse ) return 0; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pSubParse; + pSubParse->db = db; + pSubParse->pTriggerTab = pTab; + pSubParse->pToplevel = pTop; + pSubParse->zAuthContext = pTrigger->zName; + pSubParse->eTriggerOp = pTrigger->op; + pSubParse->nQueryLoop = pParse->nQueryLoop; + + v = sqlite3GetVdbe(pSubParse); + if( v ){ + VdbeComment((v, "Start: %s.%s (%s %s%s%s ON %s)", + pTrigger->zName, onErrorText(orconf), + (pTrigger->tr_tm==TRIGGER_BEFORE ? "BEFORE" : "AFTER"), + (pTrigger->op==TK_UPDATE ? "UPDATE" : ""), + (pTrigger->op==TK_INSERT ? "INSERT" : ""), + (pTrigger->op==TK_DELETE ? "DELETE" : ""), + pTab->zName + )); +#ifndef SQLITE_OMIT_TRACE + sqlite3VdbeChangeP4(v, -1, + sqlite3MPrintf(db, "-- TRIGGER %s", pTrigger->zName), P4_DYNAMIC + ); +#endif + + /* If one was specified, code the WHEN clause. If it evaluates to false + ** (or NULL) the sub-vdbe is immediately halted by jumping to the + ** OP_Halt inserted at the end of the program. */ + if( pTrigger->pWhen ){ + pWhen = sqlite3ExprDup(db, pTrigger->pWhen, 0); + if( SQLITE_OK==sqlite3ResolveExprNames(&sNC, pWhen) + && db->mallocFailed==0 + ){ + iEndTrigger = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pSubParse, pWhen, iEndTrigger, SQLITE_JUMPIFNULL); + } + sqlite3ExprDelete(db, pWhen); + } + + /* Code the trigger program into the sub-vdbe. */ + codeTriggerProgram(pSubParse, pTrigger->step_list, orconf); + + /* Insert an OP_Halt at the end of the sub-program. */ + if( iEndTrigger ){ + sqlite3VdbeResolveLabel(v, iEndTrigger); + } + sqlite3VdbeAddOp0(v, OP_Halt); + VdbeComment((v, "End: %s.%s", pTrigger->zName, onErrorText(orconf))); + + transferParseError(pParse, pSubParse); + if( db->mallocFailed==0 ){ + pProgram->aOp = sqlite3VdbeTakeOpArray(v, &pProgram->nOp, &pTop->nMaxArg); + } + pProgram->nMem = pSubParse->nMem; + pProgram->nCsr = pSubParse->nTab; + pProgram->nOnce = pSubParse->nOnce; + pProgram->token = (void *)pTrigger; + pPrg->aColmask[0] = pSubParse->oldmask; + pPrg->aColmask[1] = pSubParse->newmask; + sqlite3VdbeDelete(v); + } + + assert( !pSubParse->pAinc && !pSubParse->pZombieTab ); + assert( !pSubParse->pTriggerPrg && !pSubParse->nMaxArg ); + sqlite3ParserReset(pSubParse); + sqlite3StackFree(db, pSubParse); + + return pPrg; +} + +/* +** Return a pointer to a TriggerPrg object containing the sub-program for +** trigger pTrigger with default ON CONFLICT algorithm orconf. If no such +** TriggerPrg object exists, a new object is allocated and populated before +** being returned. +*/ +static TriggerPrg *getRowTrigger( + Parse *pParse, /* Current parse context */ + Trigger *pTrigger, /* Trigger to code */ + Table *pTab, /* The table trigger pTrigger is attached to */ + int orconf /* ON CONFLICT algorithm. */ +){ + Parse *pRoot = sqlite3ParseToplevel(pParse); + TriggerPrg *pPrg; + + assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); + + /* It may be that this trigger has already been coded (or is in the + ** process of being coded). If this is the case, then an entry with + ** a matching TriggerPrg.pTrigger field will be present somewhere + ** in the Parse.pTriggerPrg list. Search for such an entry. */ + for(pPrg=pRoot->pTriggerPrg; + pPrg && (pPrg->pTrigger!=pTrigger || pPrg->orconf!=orconf); + pPrg=pPrg->pNext + ); + + /* If an existing TriggerPrg could not be located, create a new one. */ + if( !pPrg ){ + pPrg = codeRowTrigger(pParse, pTrigger, pTab, orconf); + } + + return pPrg; +} + +/* +** Generate code for the trigger program associated with trigger p on +** table pTab. The reg, orconf and ignoreJump parameters passed to this +** function are the same as those described in the header function for +** sqlite3CodeRowTrigger() +*/ +SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( + Parse *pParse, /* Parse context */ + Trigger *p, /* Trigger to code */ + Table *pTab, /* The table to code triggers from */ + int reg, /* Reg array containing OLD.* and NEW.* values */ + int orconf, /* ON CONFLICT policy */ + int ignoreJump /* Instruction to jump to for RAISE(IGNORE) */ +){ + Vdbe *v = sqlite3GetVdbe(pParse); /* Main VM */ + TriggerPrg *pPrg; + pPrg = getRowTrigger(pParse, p, pTab, orconf); + assert( pPrg || pParse->nErr || pParse->db->mallocFailed ); + + /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program + ** is a pointer to the sub-vdbe containing the trigger program. */ + if( pPrg ){ + int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); + + sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); + sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); + VdbeComment( + (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); + + /* Set the P5 operand of the OP_Program instruction to non-zero if + ** recursive invocation of this trigger program is disallowed. Recursive + ** invocation is disallowed if (a) the sub-program is really a trigger, + ** not a foreign key action, and (b) the flag to enable recursive triggers + ** is clear. */ + sqlite3VdbeChangeP5(v, (u8)bRecursive); + } +} + +/* +** This is called to code the required FOR EACH ROW triggers for an operation +** on table pTab. The operation to code triggers for (INSERT, UPDATE or DELETE) +** is given by the op parameter. The tr_tm parameter determines whether the +** BEFORE or AFTER triggers are coded. If the operation is an UPDATE, then +** parameter pChanges is passed the list of columns being modified. +** +** If there are no triggers that fire at the specified time for the specified +** operation on pTab, this function is a no-op. +** +** The reg argument is the address of the first in an array of registers +** that contain the values substituted for the new.* and old.* references +** in the trigger program. If N is the number of columns in table pTab +** (a copy of pTab->nCol), then registers are populated as follows: +** +** Register Contains +** ------------------------------------------------------ +** reg+0 OLD.rowid +** reg+1 OLD.* value of left-most column of pTab +** ... ... +** reg+N OLD.* value of right-most column of pTab +** reg+N+1 NEW.rowid +** reg+N+2 OLD.* value of left-most column of pTab +** ... ... +** reg+N+N+1 NEW.* value of right-most column of pTab +** +** For ON DELETE triggers, the registers containing the NEW.* values will +** never be accessed by the trigger program, so they are not allocated or +** populated by the caller (there is no data to populate them with anyway). +** Similarly, for ON INSERT triggers the values stored in the OLD.* registers +** are never accessed, and so are not allocated by the caller. So, for an +** ON INSERT trigger, the value passed to this function as parameter reg +** is not a readable register, although registers (reg+N) through +** (reg+N+N+1) are. +** +** Parameter orconf is the default conflict resolution algorithm for the +** trigger program to use (REPLACE, IGNORE etc.). Parameter ignoreJump +** is the instruction that control should jump to if a trigger program +** raises an IGNORE exception. +*/ +SQLITE_PRIVATE void sqlite3CodeRowTrigger( + Parse *pParse, /* Parse context */ + Trigger *pTrigger, /* List of triggers on table pTab */ + int op, /* One of TK_UPDATE, TK_INSERT, TK_DELETE */ + ExprList *pChanges, /* Changes list for any UPDATE OF triggers */ + int tr_tm, /* One of TRIGGER_BEFORE, TRIGGER_AFTER */ + Table *pTab, /* The table to code triggers from */ + int reg, /* The first in an array of registers (see above) */ + int orconf, /* ON CONFLICT policy */ + int ignoreJump /* Instruction to jump to for RAISE(IGNORE) */ +){ + Trigger *p; /* Used to iterate through pTrigger list */ + + assert( op==TK_UPDATE || op==TK_INSERT || op==TK_DELETE ); + assert( tr_tm==TRIGGER_BEFORE || tr_tm==TRIGGER_AFTER ); + assert( (op==TK_UPDATE)==(pChanges!=0) ); + + for(p=pTrigger; p; p=p->pNext){ + + /* Sanity checking: The schema for the trigger and for the table are + ** always defined. The trigger must be in the same schema as the table + ** or else it must be a TEMP trigger. */ + assert( p->pSchema!=0 ); + assert( p->pTabSchema!=0 ); + assert( p->pSchema==p->pTabSchema + || p->pSchema==pParse->db->aDb[1].pSchema ); + + /* Determine whether we should code this trigger */ + if( p->op==op + && p->tr_tm==tr_tm + && checkColumnOverlap(p->pColumns, pChanges) + ){ + sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump); + } + } +} + +/* +** Triggers may access values stored in the old.* or new.* pseudo-table. +** This function returns a 32-bit bitmask indicating which columns of the +** old.* or new.* tables actually are used by triggers. This information +** may be used by the caller, for example, to avoid having to load the entire +** old.* record into memory when executing an UPDATE or DELETE command. +** +** Bit 0 of the returned mask is set if the left-most column of the +** table may be accessed using an [old|new].reference. Bit 1 is set if +** the second leftmost column value is required, and so on. If there +** are more than 32 columns in the table, and at least one of the columns +** with an index greater than 32 may be accessed, 0xffffffff is returned. +** +** It is not possible to determine if the old.rowid or new.rowid column is +** accessed by triggers. The caller must always assume that it is. +** +** Parameter isNew must be either 1 or 0. If it is 0, then the mask returned +** applies to the old.* table. If 1, the new.* table. +** +** Parameter tr_tm must be a mask with one or both of the TRIGGER_BEFORE +** and TRIGGER_AFTER bits set. Values accessed by BEFORE triggers are only +** included in the returned mask if the TRIGGER_BEFORE bit is set in the +** tr_tm parameter. Similarly, values accessed by AFTER triggers are only +** included in the returned mask if the TRIGGER_AFTER bit is set in tr_tm. +*/ +SQLITE_PRIVATE u32 sqlite3TriggerColmask( + Parse *pParse, /* Parse context */ + Trigger *pTrigger, /* List of triggers on table pTab */ + ExprList *pChanges, /* Changes list for any UPDATE OF triggers */ + int isNew, /* 1 for new.* ref mask, 0 for old.* ref mask */ + int tr_tm, /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ + Table *pTab, /* The table to code triggers from */ + int orconf /* Default ON CONFLICT policy for trigger steps */ +){ + const int op = pChanges ? TK_UPDATE : TK_DELETE; + u32 mask = 0; + Trigger *p; + + assert( isNew==1 || isNew==0 ); + for(p=pTrigger; p; p=p->pNext){ + if( p->op==op && (tr_tm&p->tr_tm) + && checkColumnOverlap(p->pColumns,pChanges) + ){ + TriggerPrg *pPrg; + pPrg = getRowTrigger(pParse, p, pTab, orconf); + if( pPrg ){ + mask |= pPrg->aColmask[isNew]; + } + } + } + + return mask; +} + +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ + +/************** End of trigger.c *********************************************/ +/************** Begin file update.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle UPDATE statements. +*/ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Forward declaration */ +static void updateVirtualTable( + Parse *pParse, /* The parsing context */ + SrcList *pSrc, /* The virtual table to be modified */ + Table *pTab, /* The virtual table */ + ExprList *pChanges, /* The columns to change in the UPDATE statement */ + Expr *pRowidExpr, /* Expression used to recompute the rowid */ + int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ +); +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** The most recently coded instruction was an OP_Column to retrieve the +** i-th column of table pTab. This routine sets the P4 parameter of the +** OP_Column to the default value, if any. +** +** The default value of a column is specified by a DEFAULT clause in the +** column definition. This was either supplied by the user when the table +** was created, or added later to the table definition by an ALTER TABLE +** command. If the latter, then the row-records in the table btree on disk +** may not contain a value for the column and the default value, taken +** from the P4 parameter of the OP_Column instruction, is returned instead. +** If the former, then all row-records are guaranteed to include a value +** for the column and the P4 value is not required. +** +** Column definitions created by an ALTER TABLE command may only have +** literal default values specified: a number, null or a string. (If a more +** complicated default expression value was provided, it is evaluated +** when the ALTER TABLE is executed and one of the literal values written +** into the sqlite_master table.) +** +** Therefore, the P4 parameter is only required if the default value for +** the column is a literal number, string or null. The sqlite3ValueFromExpr() +** function is capable of transforming these types of expressions into +** sqlite3_value objects. +** +** If parameter iReg is not negative, code an OP_RealAffinity instruction +** on register iReg. This is used when an equivalent integer value is +** stored in place of an 8-byte floating point value in order to save +** space. +*/ +SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ + assert( pTab!=0 ); + if( !pTab->pSelect ){ + sqlite3_value *pValue = 0; + u8 enc = ENC(sqlite3VdbeDb(v)); + Column *pCol = &pTab->aCol[i]; + VdbeComment((v, "%s.%s", pTab->zName, pCol->zName)); + assert( inCol ); + sqlite3ValueFromExpr(sqlite3VdbeDb(v), pCol->pDflt, enc, + pCol->affinity, &pValue); + if( pValue ){ + sqlite3VdbeChangeP4(v, -1, (const char *)pValue, P4_MEM); + } +#ifndef SQLITE_OMIT_FLOATING_POINT + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + } +#endif + } +} + +/* +** Process an UPDATE statement. +** +** UPDATE OR IGNORE table_wxyz SET a=b, c=d WHERE e<5 AND f NOT NULL; +** \_______/ \________/ \______/ \________________/ +* onError pTabList pChanges pWhere +*/ +SQLITE_PRIVATE void sqlite3Update( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* The table in which we should change things */ + ExprList *pChanges, /* Things to be changed */ + Expr *pWhere, /* The WHERE clause. May be null */ + int onError /* How to handle constraint errors */ +){ + int i, j; /* Loop counters */ + Table *pTab; /* The table to be updated */ + int addrTop = 0; /* VDBE instruction address of the start of the loop */ + WhereInfo *pWInfo; /* Information about the WHERE clause */ + Vdbe *v; /* The virtual database engine */ + Index *pIdx; /* For looping over indices */ + Index *pPk; /* The PRIMARY KEY index for WITHOUT ROWID tables */ + int nIdx; /* Number of indices that need updating */ + int iBaseCur; /* Base cursor number */ + int iDataCur; /* Cursor for the canonical data btree */ + int iIdxCur; /* Cursor for the first index */ + sqlite3 *db; /* The database structure */ + int *aRegIdx = 0; /* One register assigned to each index to be updated */ + int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the + ** an expression for the i-th column of the table. + ** aXRef[i]==-1 if the i-th column is not changed. */ + u8 *aToOpen; /* 1 for tables and indices to be opened */ + u8 chngPk; /* PRIMARY KEY changed in a WITHOUT ROWID table */ + u8 chngRowid; /* Rowid changed in a normal table */ + u8 chngKey; /* Either chngPk or chngRowid */ + Expr *pRowidExpr = 0; /* Expression defining the new record number */ + AuthContext sContext; /* The authorization context */ + NameContext sNC; /* The name-context to resolve expressions in */ + int iDb; /* Database containing the table being updated */ + int okOnePass; /* True for one-pass algorithm without the FIFO */ + int hasFK; /* True if foreign key processing is required */ + int labelBreak; /* Jump here to break out of UPDATE loop */ + int labelContinue; /* Jump here to continue next step of UPDATE loop */ + +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True when updating a view (INSTEAD OF trigger) */ + Trigger *pTrigger; /* List of triggers on pTab, if required */ + int tmask; /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ +#endif + int newmask; /* Mask of NEW.* columns accessed by BEFORE triggers */ + int iEph = 0; /* Ephemeral table holding all primary key values */ + int nKey = 0; /* Number of elements in regKey for WITHOUT ROWID */ + int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ + + /* Register Allocations */ + int regRowCount = 0; /* A count of rows changed */ + int regOldRowid; /* The old rowid */ + int regNewRowid; /* The new rowid */ + int regNew; /* Content of the NEW.* table in triggers */ + int regOld = 0; /* Content of OLD.* table in triggers */ + int regRowSet = 0; /* Rowset of rows to be updated */ + int regKey = 0; /* composite PRIMARY KEY value */ + + memset(&sContext, 0, sizeof(sContext)); + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ + goto update_cleanup; + } + assert( pTabList->nSrc==1 ); + + /* Locate the table which we want to update. + */ + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ) goto update_cleanup; + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + + /* Figure out if we have any triggers and if the table being + ** updated is a view. + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_UPDATE, pChanges, &tmask); + isView = pTab->pSelect!=0; + assert( pTrigger || tmask==0 ); +#else +# define pTrigger 0 +# define isView 0 +# define tmask 0 +#endif +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto update_cleanup; + } + if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + goto update_cleanup; + } + + /* Allocate a cursors for the main database table and for all indices. + ** The index cursors might not be used, but if they are used they + ** need to occur right after the database cursor. So go ahead and + ** allocate enough space, just in case. + */ + pTabList->a[0].iCursor = iBaseCur = iDataCur = pParse->nTab++; + iIdxCur = iDataCur+1; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ + if( IsPrimaryKeyIndex(pIdx) && pPk!=0 ){ + iDataCur = pParse->nTab; + pTabList->a[0].iCursor = iDataCur; + } + pParse->nTab++; + } + + /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. + ** Initialize aXRef[] and aToOpen[] to their default values. + */ + aXRef = sqlite3DbMallocRaw(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); + if( aXRef==0 ) goto update_cleanup; + aRegIdx = aXRef+pTab->nCol; + aToOpen = (u8*)(aRegIdx+nIdx); + memset(aToOpen, 1, nIdx+1); + aToOpen[nIdx+1] = 0; + for(i=0; inCol; i++) aXRef[i] = -1; + + /* Initialize the name-context */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + + /* Resolve the column names in all the expressions of the + ** of the UPDATE statement. Also find the column index + ** for each column to be updated in the pChanges array. For each + ** column to be updated, make sure we have authorization to change + ** that column. + */ + chngRowid = chngPk = 0; + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ + goto update_cleanup; + } + for(j=0; jnCol; j++){ + if( sqlite3StrICmp(pTab->aCol[j].zName, pChanges->a[i].zName)==0 ){ + if( j==pTab->iPKey ){ + chngRowid = 1; + pRowidExpr = pChanges->a[i].pExpr; + }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ + chngPk = 1; + } + aXRef[j] = i; + break; + } + } + if( j>=pTab->nCol ){ + if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zName) ){ + j = -1; + chngRowid = 1; + pRowidExpr = pChanges->a[i].pExpr; + }else{ + sqlite3ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zName); + pParse->checkSchema = 1; + goto update_cleanup; + } + } +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int rc; + rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, + j<0 ? "ROWID" : pTab->aCol[j].zName, + db->aDb[iDb].zName); + if( rc==SQLITE_DENY ){ + goto update_cleanup; + }else if( rc==SQLITE_IGNORE ){ + aXRef[j] = -1; + } + } +#endif + } + assert( (chngRowid & chngPk)==0 ); + assert( chngRowid==0 || chngRowid==1 ); + assert( chngPk==0 || chngPk==1 ); + chngKey = chngRowid + chngPk; + + /* The SET expressions are not actually used inside the WHERE loop. + ** So reset the colUsed mask + */ + pTabList->a[0].colUsed = 0; + + hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngKey); + + /* There is one entry in the aRegIdx[] array for each index on the table + ** being updated. Fill in aRegIdx[] with a register number that will hold + ** the key for accessing each index. + */ + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + int reg; + if( chngKey || hasFK || pIdx->pPartIdxWhere || pIdx==pPk ){ + reg = ++pParse->nMem; + }else{ + reg = 0; + for(i=0; inKeyCol; i++){ + if( aXRef[pIdx->aiColumn[i]]>=0 ){ + reg = ++pParse->nMem; + break; + } + } + } + if( reg==0 ) aToOpen[j+1] = 0; + aRegIdx[j] = reg; + } + + /* Begin generating code. */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto update_cleanup; + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Virtual tables must be handled separately */ + if( IsVirtual(pTab) ){ + updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, + pWhere, onError); + pWhere = 0; + pTabList = 0; + goto update_cleanup; + } +#endif + + /* Allocate required registers. */ + regRowSet = ++pParse->nMem; + regOldRowid = regNewRowid = ++pParse->nMem; + if( chngPk || pTrigger || hasFK ){ + regOld = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } + if( chngKey || pTrigger || hasFK ){ + regNewRowid = ++pParse->nMem; + } + regNew = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + + /* Start the view context. */ + if( isView ){ + sqlite3AuthContextPush(pParse, &sContext, pTab->zName); + } + + /* If we are trying to update a view, realize that view into + ** a ephemeral table. + */ +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) + if( isView ){ + sqlite3MaterializeView(pParse, pTab, pWhere, iDataCur); + } +#endif + + /* Resolve the column names in all the expressions in the + ** WHERE clause. + */ + if( sqlite3ResolveExprNames(&sNC, pWhere) ){ + goto update_cleanup; + } + + /* Begin the database scan + */ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur + ); + if( pWInfo==0 ) goto update_cleanup; + okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + + /* Remember the rowid of every item to be updated. + */ + sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); + if( !okOnePass ){ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); + } + + /* End the database scan loop. + */ + sqlite3WhereEnd(pWInfo); + }else{ + int iPk; /* First of nPk memory cells holding PRIMARY KEY value */ + i16 nPk; /* Number of components of the PRIMARY KEY */ + int addrOpen; /* Address of the OpenEphemeral instruction */ + + assert( pPk!=0 ); + nPk = pPk->nKeyCol; + iPk = pParse->nMem+1; + pParse->nMem += nPk; + regKey = ++pParse->nMem; + iEph = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_Null, 0, iPk); + addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, + WHERE_ONEPASS_DESIRED, iIdxCur); + if( pWInfo==0 ) goto update_cleanup; + okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + for(i=0; iaiColumn[i], + iPk+i); + } + if( okOnePass ){ + sqlite3VdbeChangeToNoop(v, addrOpen); + nKey = nPk; + regKey = iPk; + }else{ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, + sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); + } + sqlite3WhereEnd(pWInfo); + } + + /* Initialize the count of updated rows + */ + if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){ + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + } + + labelBreak = sqlite3VdbeMakeLabel(v); + if( !isView ){ + /* + ** Open every index that needs updating. Note that if any + ** index could potentially invoke a REPLACE conflict resolution + ** action, then we need to open all indices because we might need + ** to be deleting some records. + */ + if( onError==OE_Replace ){ + memset(aToOpen, 1, nIdx+1); + }else{ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_Replace ){ + memset(aToOpen, 1, nIdx+1); + break; + } + } + } + if( okOnePass ){ + if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; + if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; + } + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iBaseCur, aToOpen, + 0, 0); + } + + /* Top of the update loop */ + if( okOnePass ){ + if( aToOpen[iDataCur-iBaseCur] ){ + assert( pPk!=0 ); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); + VdbeCoverageNeverTaken(v); + } + labelContinue = labelBreak; + sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); + VdbeCoverageIf(v, pPk==0); + VdbeCoverageIf(v, pPk!=0); + }else if( pPk ){ + labelContinue = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v); + addrTop = sqlite3VdbeAddOp2(v, OP_RowKey, iEph, regKey); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0); + VdbeCoverage(v); + }else{ + labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet, labelBreak, + regOldRowid); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid); + VdbeCoverage(v); + } + + /* If the record number will change, set register regNewRowid to + ** contain the new value. If the record number is not being modified, + ** then regNewRowid is the same register as regOldRowid, which is + ** already populated. */ + assert( chngKey || pTrigger || hasFK || regOldRowid==regNewRowid ); + if( chngRowid ){ + sqlite3ExprCode(pParse, pRowidExpr, regNewRowid); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); VdbeCoverage(v); + } + + /* Compute the old pre-UPDATE content of the row being changed, if that + ** information is needed */ + if( chngPk || hasFK || pTrigger ){ + u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0); + oldmask |= sqlite3TriggerColmask(pParse, + pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError + ); + for(i=0; inCol; i++){ + if( oldmask==0xffffffff + || (i<32 && (oldmask & MASKBIT32(i))!=0) + || (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 + ){ + testcase( oldmask!=0xffffffff && i==31 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regOld+i); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, regOld+i); + } + } + if( chngRowid==0 && pPk==0 ){ + sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); + } + } + + /* Populate the array of registers beginning at regNew with the new + ** row data. This array is used to check constaints, create the new + ** table and index records, and as the values for any new.* references + ** made by triggers. + ** + ** If there are one or more BEFORE triggers, then do not populate the + ** registers associated with columns that are (a) not modified by + ** this UPDATE statement and (b) not accessed by new.* references. The + ** values for registers not modified by the UPDATE must be reloaded from + ** the database after the BEFORE triggers are fired anyway (as the trigger + ** may have modified them). So not loading those that are not going to + ** be used eliminates some redundant opcodes. + */ + newmask = sqlite3TriggerColmask( + pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError + ); + /*sqlite3VdbeAddOp3(v, OP_Null, 0, regNew, regNew+pTab->nCol-1);*/ + for(i=0; inCol; i++){ + if( i==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); + }else{ + j = aXRef[i]; + if( j>=0 ){ + sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i); + }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask & MASKBIT32(i)) ){ + /* This branch loads the value of a column that will not be changed + ** into a register. This is done if there are no BEFORE triggers, or + ** if there are one or more BEFORE triggers that use this value via + ** a new.* reference in a trigger program. + */ + testcase( i==31 ); + testcase( i==32 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); + } + } + } + + /* Fire any BEFORE UPDATE triggers. This happens before constraints are + ** verified. One could argue that this is wrong. + */ + if( tmask&TRIGGER_BEFORE ){ + sqlite3TableAffinity(v, pTab, regNew); + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + TRIGGER_BEFORE, pTab, regOldRowid, onError, labelContinue); + + /* The row-trigger may have deleted the row being updated. In this + ** case, jump to the next row. No updates or AFTER triggers are + ** required. This behavior - what happens when the row being updated + ** is deleted or renamed by a BEFORE trigger - is left undefined in the + ** documentation. + */ + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue,regKey,nKey); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid); + VdbeCoverage(v); + } + + /* If it did not delete it, the row-trigger may still have modified + ** some of the columns of the row being updated. Load the values for + ** all columns not modified by the update statement into their + ** registers in case this has happened. + */ + for(i=0; inCol; i++){ + if( aXRef[i]<0 && i!=pTab->iPKey ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + } + } + } + + if( !isView ){ + int j1 = 0; /* Address of jump instruction */ + int bReplace = 0; /* True if REPLACE conflict resolution might happen */ + + /* Do constraint checks. */ + assert( regOldRowid>0 ); + sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, + regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace); + + /* Do FK constraint checks. */ + if( hasFK ){ + sqlite3FkCheck(pParse, pTab, regOldRowid, 0, aXRef, chngKey); + } + + /* Delete the index entries associated with the current record. */ + if( bReplace || chngKey ){ + if( pPk ){ + j1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + }else{ + j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + } + VdbeCoverageNeverTaken(v); + } + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx); + + /* If changing the record number, delete the old record. */ + if( hasFK || chngKey || pPk!=0 ){ + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); + } + if( bReplace || chngKey ){ + sqlite3VdbeJumpHere(v, j1); + } + + if( hasFK ){ + sqlite3FkCheck(pParse, pTab, 0, regNewRowid, aXRef, chngKey); + } + + /* Insert the new index entries and the new record. */ + sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, + regNewRowid, aRegIdx, 1, 0, 0); + + /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to + ** handle rows (possibly in other tables) that refer via a foreign key + ** to the row just updated. */ + if( hasFK ){ + sqlite3FkActions(pParse, pTab, pChanges, regOldRowid, aXRef, chngKey); + } + } + + /* Increment the row counter + */ + if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab){ + sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); + } + + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue); + + /* Repeat the above with the next record to be updated, until + ** all record selected by the WHERE clause have been updated. + */ + if( okOnePass ){ + /* Nothing to do at end-of-loop for a single-pass */ + }else if( pPk ){ + sqlite3VdbeResolveLabel(v, labelContinue); + sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, labelContinue); + } + sqlite3VdbeResolveLabel(v, labelBreak); + + /* Close all tables */ + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + assert( aRegIdx ); + if( aToOpen[i+1] ){ + sqlite3VdbeAddOp2(v, OP_Close, iIdxCur+i, 0); + } + } + if( iDataCurnested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* + ** Return the number of rows that were changed. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. + */ + if( (db->flags&SQLITE_CountRows) && !pParse->pTriggerTab && !pParse->nested ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows updated", SQLITE_STATIC); + } + +update_cleanup: + sqlite3AuthContextPop(&sContext); + sqlite3DbFree(db, aXRef); /* Also frees aRegIdx[] and aToOpen[] */ + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprListDelete(db, pChanges); + sqlite3ExprDelete(db, pWhere); + return; +} +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** thely may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Generate code for an UPDATE of a virtual table. +** +** The strategy is that we create an ephemerial table that contains +** for each row to be changed: +** +** (A) The original rowid of that row. +** (B) The revised rowid for the row. (note1) +** (C) The content of every column in the row. +** +** Then we loop over this ephemeral table and for each row in +** the ephermeral table call VUpdate. +** +** When finished, drop the ephemeral table. +** +** (note1) Actually, if we know in advance that (A) is always the same +** as (B) we only store (A), then duplicate (A) when pulling +** it out of the ephemeral table before calling VUpdate. +*/ +static void updateVirtualTable( + Parse *pParse, /* The parsing context */ + SrcList *pSrc, /* The virtual table to be modified */ + Table *pTab, /* The virtual table */ + ExprList *pChanges, /* The columns to change in the UPDATE statement */ + Expr *pRowid, /* Expression used to recompute the rowid */ + int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ +){ + Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ + ExprList *pEList = 0; /* The result set of the SELECT statement */ + Select *pSelect = 0; /* The SELECT statement */ + Expr *pExpr; /* Temporary expression */ + int ephemTab; /* Table holding the result of the SELECT */ + int i; /* Loop counter */ + int addr; /* Address of top of loop */ + int iReg; /* First register in set passed to OP_VUpdate */ + sqlite3 *db = pParse->db; /* Database connection */ + const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); + SelectDest dest; + + /* Construct the SELECT statement that will find the new values for + ** all updated rows. + */ + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, "_rowid_")); + if( pRowid ){ + pEList = sqlite3ExprListAppend(pParse, pEList, + sqlite3ExprDup(db, pRowid, 0)); + } + assert( pTab->iPKey<0 ); + for(i=0; inCol; i++){ + if( aXRef[i]>=0 ){ + pExpr = sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0); + }else{ + pExpr = sqlite3Expr(db, TK_ID, pTab->aCol[i].zName); + } + pEList = sqlite3ExprListAppend(pParse, pEList, pExpr); + } + pSelect = sqlite3SelectNew(pParse, pEList, pSrc, pWhere, 0, 0, 0, 0, 0, 0); + + /* Create the ephemeral table into which the update results will + ** be stored. + */ + assert( v ); + ephemTab = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + + /* fill the ephemeral table + */ + sqlite3SelectDestInit(&dest, SRT_Table, ephemTab); + sqlite3Select(pParse, pSelect, &dest); + + /* Generate code to scan the ephemeral table and call VUpdate. */ + iReg = ++pParse->nMem; + pParse->nMem += pTab->nCol+1; + addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); + for(i=0; inCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); + } + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); + sqlite3MayAbort(pParse); + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); + + /* Cleanup */ + sqlite3SelectDelete(db, pSelect); +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/************** End of update.c **********************************************/ +/************** Begin file vacuum.c ******************************************/ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the VACUUM command. +** +** Most of the code in this file may be omitted by defining the +** SQLITE_OMIT_VACUUM macro. +*/ + +#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) +/* +** Finalize a prepared statement. If there was an error, store the +** text of the error message in *pzErrMsg. Return the result code. +*/ +static int vacuumFinalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ + int rc; + rc = sqlite3VdbeFinalize((Vdbe*)pStmt); + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + return rc; +} + +/* +** Execute zSql on database db. Return an error code. +*/ +static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + VVA_ONLY( int rc; ) + if( !zSql ){ + return SQLITE_NOMEM; + } + if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + return sqlite3_errcode(db); + } + VVA_ONLY( rc = ) sqlite3_step(pStmt); + assert( rc!=SQLITE_ROW || (db->flags&SQLITE_CountRows) ); + return vacuumFinalize(db, pStmt, pzErrMsg); +} + +/* +** Execute zSql on database db. The statement returns exactly +** one column. Execute this as SQL on the same database. +*/ +static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + int rc; + + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + rc = execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); + if( rc!=SQLITE_OK ){ + vacuumFinalize(db, pStmt, pzErrMsg); + return rc; + } + } + + return vacuumFinalize(db, pStmt, pzErrMsg); +} + +/* +** The VACUUM command is used to clean up the database, +** collapse free space, etc. It is modelled after the VACUUM command +** in PostgreSQL. The VACUUM command works as follows: +** +** (1) Create a new transient database file +** (2) Copy all content from the database being vacuumed into +** the new transient database file +** (3) Copy content from the transient database back into the +** original database. +** +** The transient database requires temporary disk space approximately +** equal to the size of the original database. The copy operation of +** step (3) requires additional temporary disk space approximately equal +** to the size of the original database for the rollback journal. +** Hence, temporary disk space that is approximately 2x the size of the +** orginal database is required. Every page of the database is written +** approximately 3 times: Once for step (2) and twice for step (3). +** Two writes per page are required in step (3) because the original +** database content must be written into the rollback journal prior to +** overwriting the database with the vacuumed content. +** +** Only 1x temporary space and only 1x writes would be required if +** the copy of step (3) were replace by deleting the original database +** and renaming the transient database as the original. But that will +** not work if other processes are attached to the original database. +** And a power loss in between deleting the original and renaming the +** transient would cause the database file to appear to be deleted +** following reboot. +*/ +SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0); + sqlite3VdbeUsesBtree(v, 0); + } + return; +} + +/* +** This routine implements the OP_Vacuum opcode of the VDBE. +*/ +SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ + int rc = SQLITE_OK; /* Return code from service routines */ + Btree *pMain; /* The database being vacuumed */ + Btree *pTemp; /* The temporary database we vacuum into */ + char *zSql = 0; /* SQL statements */ + int saved_flags; /* Saved value of the db->flags */ + int saved_nChange; /* Saved value of db->nChange */ + int saved_nTotalChange; /* Saved value of db->nTotalChange */ + void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ + Db *pDb = 0; /* Database to detach at end of vacuum */ + int isMemDb; /* True if vacuuming a :memory: database */ + int nRes; /* Bytes of reserved space at the end of each page */ + int nDb; /* Number of attached databases */ + + if( !db->autoCommit ){ + sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); + return SQLITE_ERROR; + } + if( db->nVdbeActive>1 ){ + sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); + return SQLITE_ERROR; + } + + /* Save the current value of the database flags so that it can be + ** restored before returning. Then set the writable-schema flag, and + ** disable CHECK and foreign key constraints. */ + saved_flags = db->flags; + saved_nChange = db->nChange; + saved_nTotalChange = db->nTotalChange; + saved_xTrace = db->xTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); + db->xTrace = 0; + + pMain = db->aDb[0].pBt; + isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); + + /* Attach the temporary database as 'vacuum_db'. The synchronous pragma + ** can be set to 'off' for this file, as it is not recovered if a crash + ** occurs anyway. The integrity of the database is maintained by a + ** (possibly synchronous) transaction opened on the main database before + ** sqlite3BtreeCopyFile() is called. + ** + ** An optimisation would be to use a non-journaled pager. + ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but + ** that actually made the VACUUM run slower. Very little journalling + ** actually occurs when doing a vacuum since the vacuum_db is initially + ** empty. Only the journal header is written. Apparently it takes more + ** time to parse and run the PRAGMA to turn journalling off than it does + ** to write the journal header file. + */ + nDb = db->nDb; + if( sqlite3TempInMemory(db) ){ + zSql = "ATTACH ':memory:' AS vacuum_db;"; + }else{ + zSql = "ATTACH '' AS vacuum_db;"; + } + rc = execSql(db, pzErrMsg, zSql); + if( db->nDb>nDb ){ + pDb = &db->aDb[db->nDb-1]; + assert( strcmp(pDb->zName,"vacuum_db")==0 ); + } + if( rc!=SQLITE_OK ) goto end_of_vacuum; + pTemp = db->aDb[db->nDb-1].pBt; + + /* The call to execSql() to attach the temp database has left the file + ** locked (as there was more than one active statement when the transaction + ** to read the schema was concluded. Unlock it here so that this doesn't + ** cause problems for the call to BtreeSetPageSize() below. */ + sqlite3BtreeCommit(pTemp); + + nRes = sqlite3BtreeGetReserve(pMain); + + /* A VACUUM cannot change the pagesize of an encrypted database. */ +#ifdef SQLITE_HAS_CODEC + if( db->nextPagesize ){ + extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); + int nKey; + char *zKey; + sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); + if( nKey ) db->nextPagesize = 0; + } +#endif + + rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Begin a transaction and take an exclusive lock on the main database + ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, + ** to ensure that we do not try to change the page-size on a WAL database. + */ + rc = execSql(db, pzErrMsg, "BEGIN;"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = sqlite3BtreeBeginTrans(pMain, 2); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Do not attempt to change the page size for a WAL database */ + if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) + ==PAGER_JOURNALMODE_WAL ){ + db->nextPagesize = 0; + } + + if( sqlite3BtreeSetPageSize(pTemp, sqlite3BtreeGetPageSize(pMain), nRes, 0) + || (!isMemDb && sqlite3BtreeSetPageSize(pTemp, db->nextPagesize, nRes, 0)) + || NEVER(db->mallocFailed) + ){ + rc = SQLITE_NOMEM; + goto end_of_vacuum; + } + +#ifndef SQLITE_OMIT_AUTOVACUUM + sqlite3BtreeSetAutoVacuum(pTemp, db->nextAutovac>=0 ? db->nextAutovac : + sqlite3BtreeGetAutoVacuum(pMain)); +#endif + + /* Query the schema of the main database. Create a mirror schema + ** in the temporary database. + */ + rc = execExecSql(db, pzErrMsg, + "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) " + " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" + " AND coalesce(rootpage,1)>0" + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = execExecSql(db, pzErrMsg, + "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)" + " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %' "); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = execExecSql(db, pzErrMsg, + "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) " + " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %'"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Loop through the tables in the main database. For each, do + ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy + ** the contents to the temporary database. + */ + rc = execExecSql(db, pzErrMsg, + "SELECT 'INSERT INTO vacuum_db.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';'" + "FROM main.sqlite_master " + "WHERE type = 'table' AND name!='sqlite_sequence' " + " AND coalesce(rootpage,1)>0" + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Copy over the sequence table + */ + rc = execExecSql(db, pzErrMsg, + "SELECT 'DELETE FROM vacuum_db.' || quote(name) || ';' " + "FROM vacuum_db.sqlite_master WHERE name='sqlite_sequence' " + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = execExecSql(db, pzErrMsg, + "SELECT 'INSERT INTO vacuum_db.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';' " + "FROM vacuum_db.sqlite_master WHERE name=='sqlite_sequence';" + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + + /* Copy the triggers, views, and virtual tables from the main database + ** over to the temporary database. None of these objects has any + ** associated storage, so all we have to do is copy their entries + ** from the SQLITE_MASTER table. + */ + rc = execSql(db, pzErrMsg, + "INSERT INTO vacuum_db.sqlite_master " + " SELECT type, name, tbl_name, rootpage, sql" + " FROM main.sqlite_master" + " WHERE type='view' OR type='trigger'" + " OR (type='table' AND rootpage=0)" + ); + if( rc ) goto end_of_vacuum; + + /* At this point, there is a write transaction open on both the + ** vacuum database and the main database. Assuming no error occurs, + ** both transactions are closed by this block - the main database + ** transaction by sqlite3BtreeCopyFile() and the other by an explicit + ** call to sqlite3BtreeCommit(). + */ + { + u32 meta; + int i; + + /* This array determines which meta meta values are preserved in the + ** vacuum. Even entries are the meta value number and odd entries + ** are an increment to apply to the meta value after the vacuum. + ** The increment is used to increase the schema cookie so that other + ** connections to the same database will know to reread the schema. + */ + static const unsigned char aCopy[] = { + BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ + BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ + BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ + BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ + }; + + assert( 1==sqlite3BtreeIsInTrans(pTemp) ); + assert( 1==sqlite3BtreeIsInTrans(pMain) ); + + /* Copy Btree meta values */ + for(i=0; iflags */ + db->flags = saved_flags; + db->nChange = saved_nChange; + db->nTotalChange = saved_nTotalChange; + db->xTrace = saved_xTrace; + sqlite3BtreeSetPageSize(pMain, -1, -1, 1); + + /* Currently there is an SQL level transaction open on the vacuum + ** database. No locks are held on any other files (since the main file + ** was committed at the btree level). So it safe to end the transaction + ** by manually setting the autoCommit flag to true and detaching the + ** vacuum database. The vacuum_db journal file is deleted when the pager + ** is closed by the DETACH. + */ + db->autoCommit = 1; + + if( pDb ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; + } + + /* This both clears the schemas and reduces the size of the db->aDb[] + ** array. */ + sqlite3ResetAllSchemasOfConnection(db); + + return rc; +} + +#endif /* SQLITE_OMIT_VACUUM && SQLITE_OMIT_ATTACH */ + +/************** End of vacuum.c **********************************************/ +/************** Begin file vtab.c ********************************************/ +/* +** 2006 June 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to help implement virtual tables. +*/ +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Before a virtual table xCreate() or xConnect() method is invoked, the +** sqlite3.pVtabCtx member variable is set to point to an instance of +** this struct allocated on the stack. It is used by the implementation of +** the sqlite3_declare_vtab() and sqlite3_vtab_config() APIs, both of which +** are invoked only from within xCreate and xConnect methods. +*/ +struct VtabCtx { + VTable *pVTable; /* The virtual table being constructed */ + Table *pTab; /* The Table object to which the virtual table belongs */ +}; + +/* +** The actual function that does the work of creating a new module. +** This function implements the sqlite3_create_module() and +** sqlite3_create_module_v2() interfaces. +*/ +static int createModule( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux, /* Context pointer for xCreate/xConnect */ + void (*xDestroy)(void *) /* Module destructor function */ +){ + int rc = SQLITE_OK; + int nName; + + sqlite3_mutex_enter(db->mutex); + nName = sqlite3Strlen30(zName); + if( sqlite3HashFind(&db->aModule, zName, nName) ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + Module *pMod; + pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); + if( pMod ){ + Module *pDel; + char *zCopy = (char *)(&pMod[1]); + memcpy(zCopy, zName, nName+1); + pMod->zName = zCopy; + pMod->pModule = pModule; + pMod->pAux = pAux; + pMod->xDestroy = xDestroy; + pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,nName,(void*)pMod); + assert( pDel==0 || pDel==pMod ); + if( pDel ){ + db->mallocFailed = 1; + sqlite3DbFree(db, pDel); + } + } + } + rc = sqlite3ApiExit(db, rc); + if( rc!=SQLITE_OK && xDestroy ) xDestroy(pAux); + + sqlite3_mutex_leave(db->mutex); + return rc; +} + + +/* +** External API function used to create a new virtual-table module. +*/ +SQLITE_API int sqlite3_create_module( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux /* Context pointer for xCreate/xConnect */ +){ + return createModule(db, zName, pModule, pAux, 0); +} + +/* +** External API function used to create a new virtual-table module. +*/ +SQLITE_API int sqlite3_create_module_v2( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux, /* Context pointer for xCreate/xConnect */ + void (*xDestroy)(void *) /* Module destructor function */ +){ + return createModule(db, zName, pModule, pAux, xDestroy); +} + +/* +** Lock the virtual table so that it cannot be disconnected. +** Locks nest. Every lock should have a corresponding unlock. +** If an unlock is omitted, resources leaks will occur. +** +** If a disconnect is attempted while a virtual table is locked, +** the disconnect is deferred until all locks have been removed. +*/ +SQLITE_PRIVATE void sqlite3VtabLock(VTable *pVTab){ + pVTab->nRef++; +} + + +/* +** pTab is a pointer to a Table structure representing a virtual-table. +** Return a pointer to the VTable object used by connection db to access +** this virtual-table, if one has been created, or NULL otherwise. +*/ +SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3 *db, Table *pTab){ + VTable *pVtab; + assert( IsVirtual(pTab) ); + for(pVtab=pTab->pVTable; pVtab && pVtab->db!=db; pVtab=pVtab->pNext); + return pVtab; +} + +/* +** Decrement the ref-count on a virtual table object. When the ref-count +** reaches zero, call the xDisconnect() method to delete the object. +*/ +SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ + sqlite3 *db = pVTab->db; + + assert( db ); + assert( pVTab->nRef>0 ); + assert( db->magic==SQLITE_MAGIC_OPEN || db->magic==SQLITE_MAGIC_ZOMBIE ); + + pVTab->nRef--; + if( pVTab->nRef==0 ){ + sqlite3_vtab *p = pVTab->pVtab; + if( p ){ + p->pModule->xDisconnect(p); + } + sqlite3DbFree(db, pVTab); + } +} + +/* +** Table p is a virtual table. This function moves all elements in the +** p->pVTable list to the sqlite3.pDisconnect lists of their associated +** database connections to be disconnected at the next opportunity. +** Except, if argument db is not NULL, then the entry associated with +** connection db is left in the p->pVTable list. +*/ +static VTable *vtabDisconnectAll(sqlite3 *db, Table *p){ + VTable *pRet = 0; + VTable *pVTable = p->pVTable; + p->pVTable = 0; + + /* Assert that the mutex (if any) associated with the BtShared database + ** that contains table p is held by the caller. See header comments + ** above function sqlite3VtabUnlockList() for an explanation of why + ** this makes it safe to access the sqlite3.pDisconnect list of any + ** database connection that may have an entry in the p->pVTable list. + */ + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); + + while( pVTable ){ + sqlite3 *db2 = pVTable->db; + VTable *pNext = pVTable->pNext; + assert( db2 ); + if( db2==db ){ + pRet = pVTable; + p->pVTable = pRet; + pRet->pNext = 0; + }else{ + pVTable->pNext = db2->pDisconnect; + db2->pDisconnect = pVTable; + } + pVTable = pNext; + } + + assert( !db || pRet ); + return pRet; +} + +/* +** Table *p is a virtual table. This function removes the VTable object +** for table *p associated with database connection db from the linked +** list in p->pVTab. It also decrements the VTable ref count. This is +** used when closing database connection db to free all of its VTable +** objects without disturbing the rest of the Schema object (which may +** be being used by other shared-cache connections). +*/ +SQLITE_PRIVATE void sqlite3VtabDisconnect(sqlite3 *db, Table *p){ + VTable **ppVTab; + + assert( IsVirtual(p) ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3_mutex_held(db->mutex) ); + + for(ppVTab=&p->pVTable; *ppVTab; ppVTab=&(*ppVTab)->pNext){ + if( (*ppVTab)->db==db ){ + VTable *pVTab = *ppVTab; + *ppVTab = pVTab->pNext; + sqlite3VtabUnlock(pVTab); + break; + } + } +} + + +/* +** Disconnect all the virtual table objects in the sqlite3.pDisconnect list. +** +** This function may only be called when the mutexes associated with all +** shared b-tree databases opened using connection db are held by the +** caller. This is done to protect the sqlite3.pDisconnect list. The +** sqlite3.pDisconnect list is accessed only as follows: +** +** 1) By this function. In this case, all BtShared mutexes and the mutex +** associated with the database handle itself must be held. +** +** 2) By function vtabDisconnectAll(), when it adds a VTable entry to +** the sqlite3.pDisconnect list. In this case either the BtShared mutex +** associated with the database the virtual table is stored in is held +** or, if the virtual table is stored in a non-sharable database, then +** the database handle mutex is held. +** +** As a result, a sqlite3.pDisconnect cannot be accessed simultaneously +** by multiple threads. It is thread-safe. +*/ +SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ + VTable *p = db->pDisconnect; + db->pDisconnect = 0; + + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3_mutex_held(db->mutex) ); + + if( p ){ + sqlite3ExpirePreparedStatements(db); + do { + VTable *pNext = p->pNext; + sqlite3VtabUnlock(p); + p = pNext; + }while( p ); + } +} + +/* +** Clear any and all virtual-table information from the Table record. +** This routine is called, for example, just before deleting the Table +** record. +** +** Since it is a virtual-table, the Table structure contains a pointer +** to the head of a linked list of VTable structures. Each VTable +** structure is associated with a single sqlite3* user of the schema. +** The reference count of the VTable structure associated with database +** connection db is decremented immediately (which may lead to the +** structure being xDisconnected and free). Any other VTable structures +** in the list are moved to the sqlite3.pDisconnect list of the associated +** database connection. +*/ +SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ + if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); + if( p->azModuleArg ){ + int i; + for(i=0; inModuleArg; i++){ + if( i!=1 ) sqlite3DbFree(db, p->azModuleArg[i]); + } + sqlite3DbFree(db, p->azModuleArg); + } +} + +/* +** Add a new module argument to pTable->azModuleArg[]. +** The string is not copied - the pointer is stored. The +** string will be freed automatically when the table is +** deleted. +*/ +static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ + int i = pTable->nModuleArg++; + int nBytes = sizeof(char *)*(1+pTable->nModuleArg); + char **azModuleArg; + azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); + if( azModuleArg==0 ){ + int j; + for(j=0; jazModuleArg[j]); + } + sqlite3DbFree(db, zArg); + sqlite3DbFree(db, pTable->azModuleArg); + pTable->nModuleArg = 0; + }else{ + azModuleArg[i] = zArg; + azModuleArg[i+1] = 0; + } + pTable->azModuleArg = azModuleArg; +} + +/* +** The parser calls this routine when it first sees a CREATE VIRTUAL TABLE +** statement. The module name has been parsed, but the optional list +** of parameters that follow the module name are still pending. +*/ +SQLITE_PRIVATE void sqlite3VtabBeginParse( + Parse *pParse, /* Parsing context */ + Token *pName1, /* Name of new table, or database name */ + Token *pName2, /* Name of new table or NULL */ + Token *pModuleName, /* Name of the module for the virtual table */ + int ifNotExists /* No error if the table already exists */ +){ + int iDb; /* The database the table is being created in */ + Table *pTable; /* The new virtual table */ + sqlite3 *db; /* Database connection */ + + sqlite3StartTable(pParse, pName1, pName2, 0, 0, 1, ifNotExists); + pTable = pParse->pNewTable; + if( pTable==0 ) return; + assert( 0==pTable->pIndex ); + + db = pParse->db; + iDb = sqlite3SchemaToIndex(db, pTable->pSchema); + assert( iDb>=0 ); + + pTable->tabFlags |= TF_Virtual; + pTable->nModuleArg = 0; + addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); + addModuleArgument(db, pTable, 0); + addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); + pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Creating a virtual table invokes the authorization callback twice. + ** The first invocation, to obtain permission to INSERT a row into the + ** sqlite_master table, has already been made by sqlite3StartTable(). + ** The second call, to obtain permission to create the table, is made now. + */ + if( pTable->azModuleArg ){ + sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, + pTable->azModuleArg[0], pParse->db->aDb[iDb].zName); + } +#endif +} + +/* +** This routine takes the module argument that has been accumulating +** in pParse->zArg[] and appends it to the list of arguments on the +** virtual table currently under construction in pParse->pTable. +*/ +static void addArgumentToVtab(Parse *pParse){ + if( pParse->sArg.z && pParse->pNewTable ){ + const char *z = (const char*)pParse->sArg.z; + int n = pParse->sArg.n; + sqlite3 *db = pParse->db; + addModuleArgument(db, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); + } +} + +/* +** The parser calls this routine after the CREATE VIRTUAL TABLE statement +** has been completely parsed. +*/ +SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ + Table *pTab = pParse->pNewTable; /* The table being constructed */ + sqlite3 *db = pParse->db; /* The database connection */ + + if( pTab==0 ) return; + addArgumentToVtab(pParse); + pParse->sArg.z = 0; + if( pTab->nModuleArg<1 ) return; + + /* If the CREATE VIRTUAL TABLE statement is being entered for the + ** first time (in other words if the virtual table is actually being + ** created now instead of just being read out of sqlite_master) then + ** do additional initialization work and store the statement text + ** in the sqlite_master table. + */ + if( !db->init.busy ){ + char *zStmt; + char *zWhere; + int iDb; + Vdbe *v; + + /* Compute the complete text of the CREATE VIRTUAL TABLE statement */ + if( pEnd ){ + pParse->sNameToken.n = (int)(pEnd->z - pParse->sNameToken.z) + pEnd->n; + } + zStmt = sqlite3MPrintf(db, "CREATE VIRTUAL TABLE %T", &pParse->sNameToken); + + /* A slot for the record has already been allocated in the + ** SQLITE_MASTER table. We just need to update that slot with all + ** the information we've collected. + ** + ** The VM register number pParse->regRowid holds the rowid of an + ** entry in the sqlite_master table tht was created for this vtab + ** by sqlite3StartTable(). + */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3NestedParse(pParse, + "UPDATE %Q.%s " + "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q " + "WHERE rowid=#%d", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + pTab->zName, + pTab->zName, + zStmt, + pParse->regRowid + ); + sqlite3DbFree(db, zStmt); + v = sqlite3GetVdbe(pParse); + sqlite3ChangeCookie(pParse, iDb); + + sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); + sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, + pTab->zName, sqlite3Strlen30(pTab->zName) + 1); + } + + /* If we are rereading the sqlite_master table create the in-memory + ** record of the table. The xConnect() method is not called until + ** the first time the virtual table is used in an SQL statement. This + ** allows a schema that contains virtual tables to be loaded before + ** the required virtual table implementations are registered. */ + else { + Table *pOld; + Schema *pSchema = pTab->pSchema; + const char *zName = pTab->zName; + int nName = sqlite3Strlen30(zName); + assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); + pOld = sqlite3HashInsert(&pSchema->tblHash, zName, nName, pTab); + if( pOld ){ + db->mallocFailed = 1; + assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ + return; + } + pParse->pNewTable = 0; + } +} + +/* +** The parser calls this routine when it sees the first token +** of an argument to the module name in a CREATE VIRTUAL TABLE statement. +*/ +SQLITE_PRIVATE void sqlite3VtabArgInit(Parse *pParse){ + addArgumentToVtab(pParse); + pParse->sArg.z = 0; + pParse->sArg.n = 0; +} + +/* +** The parser calls this routine for each token after the first token +** in an argument to the module name in a CREATE VIRTUAL TABLE statement. +*/ +SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){ + Token *pArg = &pParse->sArg; + if( pArg->z==0 ){ + pArg->z = p->z; + pArg->n = p->n; + }else{ + assert(pArg->z < p->z); + pArg->n = (int)(&p->z[p->n] - pArg->z); + } +} + +/* +** Invoke a virtual table constructor (either xCreate or xConnect). The +** pointer to the function to invoke is passed as the fourth parameter +** to this procedure. +*/ +static int vtabCallConstructor( + sqlite3 *db, + Table *pTab, + Module *pMod, + int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), + char **pzErr +){ + VtabCtx sCtx, *pPriorCtx; + VTable *pVTable; + int rc; + const char *const*azArg = (const char *const*)pTab->azModuleArg; + int nArg = pTab->nModuleArg; + char *zErr = 0; + char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + int iDb; + + if( !zModuleName ){ + return SQLITE_NOMEM; + } + + pVTable = sqlite3DbMallocZero(db, sizeof(VTable)); + if( !pVTable ){ + sqlite3DbFree(db, zModuleName); + return SQLITE_NOMEM; + } + pVTable->db = db; + pVTable->pMod = pMod; + + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pTab->azModuleArg[1] = db->aDb[iDb].zName; + + /* Invoke the virtual table constructor */ + assert( &db->pVtabCtx ); + assert( xConstruct ); + sCtx.pTab = pTab; + sCtx.pVTable = pVTable; + pPriorCtx = db->pVtabCtx; + db->pVtabCtx = &sCtx; + rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + db->pVtabCtx = pPriorCtx; + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + + if( SQLITE_OK!=rc ){ + if( zErr==0 ){ + *pzErr = sqlite3MPrintf(db, "vtable constructor failed: %s", zModuleName); + }else { + *pzErr = sqlite3MPrintf(db, "%s", zErr); + sqlite3_free(zErr); + } + sqlite3DbFree(db, pVTable); + }else if( ALWAYS(pVTable->pVtab) ){ + /* Justification of ALWAYS(): A correct vtab constructor must allocate + ** the sqlite3_vtab object if successful. */ + pVTable->pVtab->pModule = pMod->pModule; + pVTable->nRef = 1; + if( sCtx.pTab ){ + const char *zFormat = "vtable constructor did not declare schema: %s"; + *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); + sqlite3VtabUnlock(pVTable); + rc = SQLITE_ERROR; + }else{ + int iCol; + /* If everything went according to plan, link the new VTable structure + ** into the linked list headed by pTab->pVTable. Then loop through the + ** columns of the table to see if any of them contain the token "hidden". + ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from + ** the type string. */ + pVTable->pNext = pTab->pVTable; + pTab->pVTable = pVTable; + + for(iCol=0; iColnCol; iCol++){ + char *zType = pTab->aCol[iCol].zType; + int nType; + int i = 0; + if( !zType ) continue; + nType = sqlite3Strlen30(zType); + if( sqlite3StrNICmp("hidden", zType, 6)||(zType[6] && zType[6]!=' ') ){ + for(i=0; i0 ){ + assert(zType[i-1]==' '); + zType[i-1] = '\0'; + } + pTab->aCol[iCol].colFlags |= COLFLAG_HIDDEN; + } + } + } + } + + sqlite3DbFree(db, zModuleName); + return rc; +} + +/* +** This function is invoked by the parser to call the xConnect() method +** of the virtual table pTab. If an error occurs, an error code is returned +** and an error left in pParse. +** +** This call is a no-op if table pTab is not a virtual table. +*/ +SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ + sqlite3 *db = pParse->db; + const char *zMod; + Module *pMod; + int rc; + + assert( pTab ); + if( (pTab->tabFlags & TF_Virtual)==0 || sqlite3GetVTable(db, pTab) ){ + return SQLITE_OK; + } + + /* Locate the required virtual table module */ + zMod = pTab->azModuleArg[0]; + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + + if( !pMod ){ + const char *zModule = pTab->azModuleArg[0]; + sqlite3ErrorMsg(pParse, "no such module: %s", zModule); + rc = SQLITE_ERROR; + }else{ + char *zErr = 0; + rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xConnect, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "%s", zErr); + } + sqlite3DbFree(db, zErr); + } + + return rc; +} +/* +** Grow the db->aVTrans[] array so that there is room for at least one +** more v-table. Return SQLITE_NOMEM if a malloc fails, or SQLITE_OK otherwise. +*/ +static int growVTrans(sqlite3 *db){ + const int ARRAY_INCR = 5; + + /* Grow the sqlite3.aVTrans array if required */ + if( (db->nVTrans%ARRAY_INCR)==0 ){ + VTable **aVTrans; + int nBytes = sizeof(sqlite3_vtab *) * (db->nVTrans + ARRAY_INCR); + aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); + if( !aVTrans ){ + return SQLITE_NOMEM; + } + memset(&aVTrans[db->nVTrans], 0, sizeof(sqlite3_vtab *)*ARRAY_INCR); + db->aVTrans = aVTrans; + } + + return SQLITE_OK; +} + +/* +** Add the virtual table pVTab to the array sqlite3.aVTrans[]. Space should +** have already been reserved using growVTrans(). +*/ +static void addToVTrans(sqlite3 *db, VTable *pVTab){ + /* Add pVtab to the end of sqlite3.aVTrans */ + db->aVTrans[db->nVTrans++] = pVTab; + sqlite3VtabLock(pVTab); +} + +/* +** This function is invoked by the vdbe to call the xCreate method +** of the virtual table named zTab in database iDb. +** +** If an error occurs, *pzErr is set to point an an English language +** description of the error and an SQLITE_XXX error code is returned. +** In this case the caller must call sqlite3DbFree(db, ) on *pzErr. +*/ +SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ + int rc = SQLITE_OK; + Table *pTab; + Module *pMod; + const char *zMod; + + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); + assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable ); + + /* Locate the required virtual table module */ + zMod = pTab->azModuleArg[0]; + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + + /* If the module has been registered and includes a Create method, + ** invoke it now. If the module has not been registered, return an + ** error. Otherwise, do nothing. + */ + if( !pMod ){ + *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); + rc = SQLITE_ERROR; + }else{ + rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xCreate, pzErr); + } + + /* Justification of ALWAYS(): The xConstructor method is required to + ** create a valid sqlite3_vtab if it returns SQLITE_OK. */ + if( rc==SQLITE_OK && ALWAYS(sqlite3GetVTable(db, pTab)) ){ + rc = growVTrans(db); + if( rc==SQLITE_OK ){ + addToVTrans(db, sqlite3GetVTable(db, pTab)); + } + } + + return rc; +} + +/* +** This function is used to set the schema of a virtual table. It is only +** valid to call this function from within the xCreate() or xConnect() of a +** virtual table module. +*/ +SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ + Parse *pParse; + + int rc = SQLITE_OK; + Table *pTab; + char *zErr = 0; + + sqlite3_mutex_enter(db->mutex); + if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ + sqlite3Error(db, SQLITE_MISUSE, 0); + sqlite3_mutex_leave(db->mutex); + return SQLITE_MISUSE_BKPT; + } + assert( (pTab->tabFlags & TF_Virtual)!=0 ); + + pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); + if( pParse==0 ){ + rc = SQLITE_NOMEM; + }else{ + pParse->declareVtab = 1; + pParse->db = db; + pParse->nQueryLoop = 1; + + if( SQLITE_OK==sqlite3RunParser(pParse, zCreateTable, &zErr) + && pParse->pNewTable + && !db->mallocFailed + && !pParse->pNewTable->pSelect + && (pParse->pNewTable->tabFlags & TF_Virtual)==0 + ){ + if( !pTab->aCol ){ + pTab->aCol = pParse->pNewTable->aCol; + pTab->nCol = pParse->pNewTable->nCol; + pParse->pNewTable->nCol = 0; + pParse->pNewTable->aCol = 0; + } + db->pVtabCtx->pTab = 0; + }else{ + sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + rc = SQLITE_ERROR; + } + pParse->declareVtab = 0; + + if( pParse->pVdbe ){ + sqlite3VdbeFinalize(pParse->pVdbe); + } + sqlite3DeleteTable(db, pParse->pNewTable); + sqlite3ParserReset(pParse); + sqlite3StackFree(db, pParse); + } + + assert( (rc&0xff)==rc ); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This function is invoked by the vdbe to call the xDestroy method +** of the virtual table named zTab in database iDb. This occurs +** when a DROP TABLE is mentioned. +** +** This call is a no-op if zTab is not a virtual table. +*/ +SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ + int rc = SQLITE_OK; + Table *pTab; + + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); + if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ + VTable *p = vtabDisconnectAll(db, pTab); + + assert( rc==SQLITE_OK ); + rc = p->pMod->pModule->xDestroy(p->pVtab); + + /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ + if( rc==SQLITE_OK ){ + assert( pTab->pVTable==p && p->pNext==0 ); + p->pVtab = 0; + pTab->pVTable = 0; + sqlite3VtabUnlock(p); + } + } + + return rc; +} + +/* +** This function invokes either the xRollback or xCommit method +** of each of the virtual tables in the sqlite3.aVTrans array. The method +** called is identified by the second argument, "offset", which is +** the offset of the method to call in the sqlite3_module structure. +** +** The array is cleared after invoking the callbacks. +*/ +static void callFinaliser(sqlite3 *db, int offset){ + int i; + if( db->aVTrans ){ + for(i=0; inVTrans; i++){ + VTable *pVTab = db->aVTrans[i]; + sqlite3_vtab *p = pVTab->pVtab; + if( p ){ + int (*x)(sqlite3_vtab *); + x = *(int (**)(sqlite3_vtab *))((char *)p->pModule + offset); + if( x ) x(p); + } + pVTab->iSavepoint = 0; + sqlite3VtabUnlock(pVTab); + } + sqlite3DbFree(db, db->aVTrans); + db->nVTrans = 0; + db->aVTrans = 0; + } +} + +/* +** Invoke the xSync method of all virtual tables in the sqlite3.aVTrans +** array. Return the error code for the first error that occurs, or +** SQLITE_OK if all xSync operations are successful. +** +** If an error message is available, leave it in p->zErrMsg. +*/ +SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, Vdbe *p){ + int i; + int rc = SQLITE_OK; + VTable **aVTrans = db->aVTrans; + + db->aVTrans = 0; + for(i=0; rc==SQLITE_OK && inVTrans; i++){ + int (*x)(sqlite3_vtab *); + sqlite3_vtab *pVtab = aVTrans[i]->pVtab; + if( pVtab && (x = pVtab->pModule->xSync)!=0 ){ + rc = x(pVtab); + sqlite3VtabImportErrmsg(p, pVtab); + } + } + db->aVTrans = aVTrans; + return rc; +} + +/* +** Invoke the xRollback method of all virtual tables in the +** sqlite3.aVTrans array. Then clear the array itself. +*/ +SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db){ + callFinaliser(db, offsetof(sqlite3_module,xRollback)); + return SQLITE_OK; +} + +/* +** Invoke the xCommit method of all virtual tables in the +** sqlite3.aVTrans array. Then clear the array itself. +*/ +SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db){ + callFinaliser(db, offsetof(sqlite3_module,xCommit)); + return SQLITE_OK; +} + +/* +** If the virtual table pVtab supports the transaction interface +** (xBegin/xRollback/xCommit and optionally xSync) and a transaction is +** not currently open, invoke the xBegin method now. +** +** If the xBegin call is successful, place the sqlite3_vtab pointer +** in the sqlite3.aVTrans array. +*/ +SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ + int rc = SQLITE_OK; + const sqlite3_module *pModule; + + /* Special case: If db->aVTrans is NULL and db->nVTrans is greater + ** than zero, then this function is being called from within a + ** virtual module xSync() callback. It is illegal to write to + ** virtual module tables in this case, so return SQLITE_LOCKED. + */ + if( sqlite3VtabInSync(db) ){ + return SQLITE_LOCKED; + } + if( !pVTab ){ + return SQLITE_OK; + } + pModule = pVTab->pVtab->pModule; + + if( pModule->xBegin ){ + int i; + + /* If pVtab is already in the aVTrans array, return early */ + for(i=0; inVTrans; i++){ + if( db->aVTrans[i]==pVTab ){ + return SQLITE_OK; + } + } + + /* Invoke the xBegin method. If successful, add the vtab to the + ** sqlite3.aVTrans[] array. */ + rc = growVTrans(db); + if( rc==SQLITE_OK ){ + rc = pModule->xBegin(pVTab->pVtab); + if( rc==SQLITE_OK ){ + addToVTrans(db, pVTab); + } + } + } + return rc; +} + +/* +** Invoke either the xSavepoint, xRollbackTo or xRelease method of all +** virtual tables that currently have an open transaction. Pass iSavepoint +** as the second argument to the virtual table method invoked. +** +** If op is SAVEPOINT_BEGIN, the xSavepoint method is invoked. If it is +** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is +** SAVEPOINT_RELEASE, then the xRelease method of each virtual table with +** an open transaction is invoked. +** +** If any virtual table method returns an error code other than SQLITE_OK, +** processing is abandoned and the error returned to the caller of this +** function immediately. If all calls to virtual table methods are successful, +** SQLITE_OK is returned. +*/ +SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ + int rc = SQLITE_OK; + + assert( op==SAVEPOINT_RELEASE||op==SAVEPOINT_ROLLBACK||op==SAVEPOINT_BEGIN ); + assert( iSavepoint>=0 ); + if( db->aVTrans ){ + int i; + for(i=0; rc==SQLITE_OK && inVTrans; i++){ + VTable *pVTab = db->aVTrans[i]; + const sqlite3_module *pMod = pVTab->pMod->pModule; + if( pVTab->pVtab && pMod->iVersion>=2 ){ + int (*xMethod)(sqlite3_vtab *, int); + switch( op ){ + case SAVEPOINT_BEGIN: + xMethod = pMod->xSavepoint; + pVTab->iSavepoint = iSavepoint+1; + break; + case SAVEPOINT_ROLLBACK: + xMethod = pMod->xRollbackTo; + break; + default: + xMethod = pMod->xRelease; + break; + } + if( xMethod && pVTab->iSavepoint>iSavepoint ){ + rc = xMethod(pVTab->pVtab, iSavepoint); + } + } + } + } + return rc; +} + +/* +** The first parameter (pDef) is a function implementation. The +** second parameter (pExpr) is the first argument to this function. +** If pExpr is a column in a virtual table, then let the virtual +** table implementation have an opportunity to overload the function. +** +** This routine is used to allow virtual table implementations to +** overload MATCH, LIKE, GLOB, and REGEXP operators. +** +** Return either the pDef argument (indicating no change) or a +** new FuncDef structure that is marked as ephemeral using the +** SQLITE_FUNC_EPHEM flag. +*/ +SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( + sqlite3 *db, /* Database connection for reporting malloc problems */ + FuncDef *pDef, /* Function to possibly overload */ + int nArg, /* Number of arguments to the function */ + Expr *pExpr /* First argument to the function */ +){ + Table *pTab; + sqlite3_vtab *pVtab; + sqlite3_module *pMod; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**) = 0; + void *pArg = 0; + FuncDef *pNew; + int rc = 0; + char *zLowerName; + unsigned char *z; + + + /* Check to see the left operand is a column in a virtual table */ + if( NEVER(pExpr==0) ) return pDef; + if( pExpr->op!=TK_COLUMN ) return pDef; + pTab = pExpr->pTab; + if( NEVER(pTab==0) ) return pDef; + if( (pTab->tabFlags & TF_Virtual)==0 ) return pDef; + pVtab = sqlite3GetVTable(db, pTab)->pVtab; + assert( pVtab!=0 ); + assert( pVtab->pModule!=0 ); + pMod = (sqlite3_module *)pVtab->pModule; + if( pMod->xFindFunction==0 ) return pDef; + + /* Call the xFindFunction method on the virtual table implementation + ** to see if the implementation wants to overload this function + */ + zLowerName = sqlite3DbStrDup(db, pDef->zName); + if( zLowerName ){ + for(z=(unsigned char*)zLowerName; *z; z++){ + *z = sqlite3UpperToLower[*z]; + } + rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xFunc, &pArg); + sqlite3DbFree(db, zLowerName); + } + if( rc==0 ){ + return pDef; + } + + /* Create a new ephemeral function definition for the overloaded + ** function */ + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) + + sqlite3Strlen30(pDef->zName) + 1); + if( pNew==0 ){ + return pDef; + } + *pNew = *pDef; + pNew->zName = (char *)&pNew[1]; + memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1); + pNew->xFunc = xFunc; + pNew->pUserData = pArg; + pNew->funcFlags |= SQLITE_FUNC_EPHEM; + return pNew; +} + +/* +** Make sure virtual table pTab is contained in the pParse->apVirtualLock[] +** array so that an OP_VBegin will get generated for it. Add pTab to the +** array if it is missing. If pTab is already in the array, this routine +** is a no-op. +*/ +SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + int i, n; + Table **apVtabLock; + + assert( IsVirtual(pTab) ); + for(i=0; inVtabLock; i++){ + if( pTab==pToplevel->apVtabLock[i] ) return; + } + n = (pToplevel->nVtabLock+1)*sizeof(pToplevel->apVtabLock[0]); + apVtabLock = sqlite3_realloc(pToplevel->apVtabLock, n); + if( apVtabLock ){ + pToplevel->apVtabLock = apVtabLock; + pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; + }else{ + pToplevel->db->mallocFailed = 1; + } +} + +/* +** Return the ON CONFLICT resolution mode in effect for the virtual +** table update operation currently in progress. +** +** The results of this routine are undefined unless it is called from +** within an xUpdate method. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ + static const unsigned char aMap[] = { + SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE + }; + assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); + assert( OE_Ignore==4 && OE_Replace==5 ); + assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); + return (int)aMap[db->vtabOnConflict-1]; +} + +/* +** Call from within the xCreate() or xConnect() methods to provide +** the SQLite core with additional information about the behavior +** of the virtual table being implemented. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + + va_start(ap, op); + switch( op ){ + case SQLITE_VTAB_CONSTRAINT_SUPPORT: { + VtabCtx *p = db->pVtabCtx; + if( !p ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + assert( p->pTab==0 || (p->pTab->tabFlags & TF_Virtual)!=0 ); + p->pVTable->bConstraint = (u8)va_arg(ap, int); + } + break; + } + default: + rc = SQLITE_MISUSE_BKPT; + break; + } + va_end(ap); + + if( rc!=SQLITE_OK ) sqlite3Error(db, rc, 0); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/************** End of vtab.c ************************************************/ +/************** Begin file where.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. This module is responsible for +** generating the code that loops through a table looking for applicable +** rows. Indices are selected and used to speed the search when doing +** so is applicable. Because this module is responsible for selecting +** indices, you might also think of this module as the "query optimizer". +*/ +/************** Include whereInt.h in the middle of where.c ******************/ +/************** Begin file whereInt.h ****************************************/ +/* +** 2013-11-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains structure and macro definitions for the query +** planner logic in "where.c". These definitions are broken out into +** a separate source file for easier editing. +*/ + +/* +** Trace output macros +*/ +#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) +/***/ int sqlite3WhereTrace = 0; +#endif +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) +# define WHERETRACE(K,X) if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X +# define WHERETRACE_ENABLED 1 +#else +# define WHERETRACE(K,X) +#endif + +/* Forward references +*/ +typedef struct WhereClause WhereClause; +typedef struct WhereMaskSet WhereMaskSet; +typedef struct WhereOrInfo WhereOrInfo; +typedef struct WhereAndInfo WhereAndInfo; +typedef struct WhereLevel WhereLevel; +typedef struct WhereLoop WhereLoop; +typedef struct WherePath WherePath; +typedef struct WhereTerm WhereTerm; +typedef struct WhereLoopBuilder WhereLoopBuilder; +typedef struct WhereScan WhereScan; +typedef struct WhereOrCost WhereOrCost; +typedef struct WhereOrSet WhereOrSet; + +/* +** This object contains information needed to implement a single nested +** loop in WHERE clause. +** +** Contrast this object with WhereLoop. This object describes the +** implementation of the loop. WhereLoop describes the algorithm. +** This object contains a pointer to the WhereLoop algorithm as one of +** its elements. +** +** The WhereInfo object contains a single instance of this object for +** each term in the FROM clause (which is to say, for each of the +** nested loops as implemented). The order of WhereLevel objects determines +** the loop nested order, with WhereInfo.a[0] being the outer loop and +** WhereInfo.a[WhereInfo.nLevel-1] being the inner loop. +*/ +struct WhereLevel { + int iLeftJoin; /* Memory cell used to implement LEFT OUTER JOIN */ + int iTabCur; /* The VDBE cursor used to access the table */ + int iIdxCur; /* The VDBE cursor used to access pIdx */ + int addrBrk; /* Jump here to break out of the loop */ + int addrNxt; /* Jump here to start the next IN combination */ + int addrSkip; /* Jump here for next iteration of skip-scan */ + int addrCont; /* Jump here to continue with the next loop cycle */ + int addrFirst; /* First instruction of interior of the loop */ + int addrBody; /* Beginning of the body of this loop */ + u8 iFrom; /* Which entry in the FROM clause */ + u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ + int p1, p2; /* Operands of the opcode used to ends the loop */ + union { /* Information that depends on pWLoop->wsFlags */ + struct { + int nIn; /* Number of entries in aInLoop[] */ + struct InLoop { + int iCur; /* The VDBE cursor used by this IN operator */ + int addrInTop; /* Top of the IN loop */ + u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ + } *aInLoop; /* Information about each nested IN operator */ + } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ + Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ + } u; + struct WhereLoop *pWLoop; /* The selected WhereLoop object */ + Bitmask notReady; /* FROM entries not usable at this level */ +}; + +/* +** Each instance of this object represents an algorithm for evaluating one +** term of a join. Every term of the FROM clause will have at least +** one corresponding WhereLoop object (unless INDEXED BY constraints +** prevent a query solution - which is an error) and many terms of the +** FROM clause will have multiple WhereLoop objects, each describing a +** potential way of implementing that FROM-clause term, together with +** dependencies and cost estimates for using the chosen algorithm. +** +** Query planning consists of building up a collection of these WhereLoop +** objects, then computing a particular sequence of WhereLoop objects, with +** one WhereLoop object per FROM clause term, that satisfy all dependencies +** and that minimize the overall cost. +*/ +struct WhereLoop { + Bitmask prereq; /* Bitmask of other loops that must run first */ + Bitmask maskSelf; /* Bitmask identifying table iTab */ +#ifdef SQLITE_DEBUG + char cId; /* Symbolic ID of this loop for debugging use */ +#endif + u8 iTab; /* Position in FROM clause of table for this loop */ + u8 iSortIdx; /* Sorting index number. 0==None */ + LogEst rSetup; /* One-time setup cost (ex: create transient index) */ + LogEst rRun; /* Cost of running each loop */ + LogEst nOut; /* Estimated number of output rows */ + union { + struct { /* Information for internal btree tables */ + u16 nEq; /* Number of equality constraints */ + u16 nSkip; /* Number of initial index columns to skip */ + Index *pIndex; /* Index used, or NULL */ + } btree; + struct { /* Information for virtual tables */ + int idxNum; /* Index number */ + u8 needFree; /* True if sqlite3_free(idxStr) is needed */ + i8 isOrdered; /* True if satisfies ORDER BY */ + u16 omitMask; /* Terms that may be omitted */ + char *idxStr; /* Index identifier string */ + } vtab; + } u; + u32 wsFlags; /* WHERE_* flags describing the plan */ + u16 nLTerm; /* Number of entries in aLTerm[] */ + /**** whereLoopXfer() copies fields above ***********************/ +# define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) + u16 nLSlot; /* Number of slots allocated for aLTerm[] */ + WhereTerm **aLTerm; /* WhereTerms used */ + WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ + WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ +}; + +/* This object holds the prerequisites and the cost of running a +** subquery on one operand of an OR operator in the WHERE clause. +** See WhereOrSet for additional information +*/ +struct WhereOrCost { + Bitmask prereq; /* Prerequisites */ + LogEst rRun; /* Cost of running this subquery */ + LogEst nOut; /* Number of outputs for this subquery */ +}; + +/* The WhereOrSet object holds a set of possible WhereOrCosts that +** correspond to the subquery(s) of OR-clause processing. Only the +** best N_OR_COST elements are retained. +*/ +#define N_OR_COST 3 +struct WhereOrSet { + u16 n; /* Number of valid a[] entries */ + WhereOrCost a[N_OR_COST]; /* Set of best costs */ +}; + + +/* Forward declaration of methods */ +static int whereLoopResize(sqlite3*, WhereLoop*, int); + +/* +** Each instance of this object holds a sequence of WhereLoop objects +** that implement some or all of a query plan. +** +** Think of each WhereLoop object as a node in a graph with arcs +** showing dependencies and costs for travelling between nodes. (That is +** not a completely accurate description because WhereLoop costs are a +** vector, not a scalar, and because dependencies are many-to-one, not +** one-to-one as are graph nodes. But it is a useful visualization aid.) +** Then a WherePath object is a path through the graph that visits some +** or all of the WhereLoop objects once. +** +** The "solver" works by creating the N best WherePath objects of length +** 1. Then using those as a basis to compute the N best WherePath objects +** of length 2. And so forth until the length of WherePaths equals the +** number of nodes in the FROM clause. The best (lowest cost) WherePath +** at the end is the choosen query plan. +*/ +struct WherePath { + Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ + Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ + LogEst nRow; /* Estimated number of rows generated by this path */ + LogEst rCost; /* Total cost of this path */ + LogEst rUnsorted; /* Total cost of this path ignoring sorting costs */ + i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */ + WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ +}; + +/* +** The query generator uses an array of instances of this structure to +** help it analyze the subexpressions of the WHERE clause. Each WHERE +** clause subexpression is separated from the others by AND operators, +** usually, or sometimes subexpressions separated by OR. +** +** All WhereTerms are collected into a single WhereClause structure. +** The following identity holds: +** +** WhereTerm.pWC->a[WhereTerm.idx] == WhereTerm +** +** When a term is of the form: +** +** X +** +** where X is a column name and is one of certain operators, +** then WhereTerm.leftCursor and WhereTerm.u.leftColumn record the +** cursor number and column number for X. WhereTerm.eOperator records +** the using a bitmask encoding defined by WO_xxx below. The +** use of a bitmask encoding for the operator allows us to search +** quickly for terms that match any of several different operators. +** +** A WhereTerm might also be two or more subterms connected by OR: +** +** (t1.X ) OR (t1.Y ) OR .... +** +** In this second case, wtFlag has the TERM_ORINFO bit set and eOperator==WO_OR +** and the WhereTerm.u.pOrInfo field points to auxiliary information that +** is collected about the OR clause. +** +** If a term in the WHERE clause does not match either of the two previous +** categories, then eOperator==0. The WhereTerm.pExpr field is still set +** to the original subexpression content and wtFlags is set up appropriately +** but no other fields in the WhereTerm object are meaningful. +** +** When eOperator!=0, prereqRight and prereqAll record sets of cursor numbers, +** but they do so indirectly. A single WhereMaskSet structure translates +** cursor number into bits and the translated bit is stored in the prereq +** fields. The translation is used in order to maximize the number of +** bits that will fit in a Bitmask. The VDBE cursor numbers might be +** spread out over the non-negative integers. For example, the cursor +** numbers might be 3, 8, 9, 10, 20, 23, 41, and 45. The WhereMaskSet +** translates these sparse cursor numbers into consecutive integers +** beginning with 0 in order to make the best possible use of the available +** bits in the Bitmask. So, in the example above, the cursor numbers +** would be mapped into integers 0 through 7. +** +** The number of terms in a join is limited by the number of bits +** in prereqRight and prereqAll. The default is 64 bits, hence SQLite +** is only able to process joins with 64 or fewer tables. +*/ +struct WhereTerm { + Expr *pExpr; /* Pointer to the subexpression that is this term */ + int iParent; /* Disable pWC->a[iParent] when this term disabled */ + int leftCursor; /* Cursor number of X in "X " */ + union { + int leftColumn; /* Column number of X in "X " */ + WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ + WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ + } u; + LogEst truthProb; /* Probability of truth for this expression */ + u16 eOperator; /* A WO_xx value describing */ + u8 wtFlags; /* TERM_xxx bit flags. See below */ + u8 nChild; /* Number of children that must disable us */ + WhereClause *pWC; /* The clause this term is part of */ + Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ + Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ +}; + +/* +** Allowed values of WhereTerm.wtFlags +*/ +#define TERM_DYNAMIC 0x01 /* Need to call sqlite3ExprDelete(db, pExpr) */ +#define TERM_VIRTUAL 0x02 /* Added by the optimizer. Do not code */ +#define TERM_CODED 0x04 /* This term is already coded */ +#define TERM_COPIED 0x08 /* Has a child */ +#define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ +#define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ +#define TERM_OR_OK 0x40 /* Used during OR-clause processing */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +# define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ +#else +# define TERM_VNULL 0x00 /* Disabled if not using stat3 */ +#endif + +/* +** An instance of the WhereScan object is used as an iterator for locating +** terms in the WHERE clause that are useful to the query planner. +*/ +struct WhereScan { + WhereClause *pOrigWC; /* Original, innermost WhereClause */ + WhereClause *pWC; /* WhereClause currently being scanned */ + char *zCollName; /* Required collating sequence, if not NULL */ + char idxaff; /* Must match this affinity, if zCollName!=NULL */ + unsigned char nEquiv; /* Number of entries in aEquiv[] */ + unsigned char iEquiv; /* Next unused slot in aEquiv[] */ + u32 opMask; /* Acceptable operators */ + int k; /* Resume scanning at this->pWC->a[this->k] */ + int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ +}; + +/* +** An instance of the following structure holds all information about a +** WHERE clause. Mostly this is a container for one or more WhereTerms. +** +** Explanation of pOuter: For a WHERE clause of the form +** +** a AND ((b AND c) OR (d AND e)) AND f +** +** There are separate WhereClause objects for the whole clause and for +** the subclauses "(b AND c)" and "(d AND e)". The pOuter field of the +** subclauses points to the WhereClause object for the whole clause. +*/ +struct WhereClause { + WhereInfo *pWInfo; /* WHERE clause processing context */ + WhereClause *pOuter; /* Outer conjunction */ + u8 op; /* Split operator. TK_AND or TK_OR */ + int nTerm; /* Number of terms */ + int nSlot; /* Number of entries in a[] */ + WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ +#if defined(SQLITE_SMALL_STACK) + WhereTerm aStatic[1]; /* Initial static space for a[] */ +#else + WhereTerm aStatic[8]; /* Initial static space for a[] */ +#endif +}; + +/* +** A WhereTerm with eOperator==WO_OR has its u.pOrInfo pointer set to +** a dynamically allocated instance of the following structure. +*/ +struct WhereOrInfo { + WhereClause wc; /* Decomposition into subterms */ + Bitmask indexable; /* Bitmask of all indexable tables in the clause */ +}; + +/* +** A WhereTerm with eOperator==WO_AND has its u.pAndInfo pointer set to +** a dynamically allocated instance of the following structure. +*/ +struct WhereAndInfo { + WhereClause wc; /* The subexpression broken out */ +}; + +/* +** An instance of the following structure keeps track of a mapping +** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. +** +** The VDBE cursor numbers are small integers contained in +** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** clause, the cursor numbers might not begin with 0 and they might +** contain gaps in the numbering sequence. But we want to make maximum +** use of the bits in our bitmasks. This structure provides a mapping +** from the sparse cursor numbers into consecutive integers beginning +** with 0. +** +** If WhereMaskSet.ix[A]==B it means that The A-th bit of a Bitmask +** corresponds VDBE cursor number B. The A-th bit of a bitmask is 1<3, 5->1, 8->2, 29->0, +** 57->5, 73->4. Or one of 719 other combinations might be used. It +** does not really matter. What is important is that sparse cursor +** numbers all get mapped into bit numbers that begin with 0 and contain +** no gaps. +*/ +struct WhereMaskSet { + int n; /* Number of assigned cursor values */ + int ix[BMS]; /* Cursor assigned to each bit */ +}; + +/* +** This object is a convenience wrapper holding all information needed +** to construct WhereLoop objects for a particular query. +*/ +struct WhereLoopBuilder { + WhereInfo *pWInfo; /* Information about this WHERE */ + WhereClause *pWC; /* WHERE clause terms */ + ExprList *pOrderBy; /* ORDER BY clause */ + WhereLoop *pNew; /* Template WhereLoop */ + WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + UnpackedRecord *pRec; /* Probe for stat4 (if required) */ + int nRecValid; /* Number of valid fields currently in pRec */ +#endif +}; + +/* +** The WHERE clause processing routine has two halves. The +** first part does the start of the WHERE loop and the second +** half does the tail of the WHERE loop. An instance of +** this structure is returned by the first half and passed +** into the second half to give some continuity. +** +** An instance of this object holds the complete state of the query +** planner. +*/ +struct WhereInfo { + Parse *pParse; /* Parsing and code generating context */ + SrcList *pTabList; /* List of tables in the join */ + ExprList *pOrderBy; /* The ORDER BY clause or NULL */ + ExprList *pResultSet; /* Result set. DISTINCT operates on these */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ + LogEst nRowOut; /* Estimated number of output rows */ + u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u8 sorted; /* True if really sorted (not just grouped) */ + u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ + u8 nLevel; /* Number of nested loop */ + int iTop; /* The very beginning of the WHERE loop */ + int iContinue; /* Jump here to continue with next record */ + int iBreak; /* Jump here to break out of the loop */ + int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ + WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ + WhereClause sWC; /* Decomposition of the WHERE clause */ + WhereLevel a[1]; /* Information about each nest loop in WHERE */ +}; + +/* +** Bitmasks for the operators on WhereTerm objects. These are all +** operators that are of interest to the query planner. An +** OR-ed combination of these values can be used when searching for +** particular WhereTerms within a WhereClause. +*/ +#define WO_IN 0x001 +#define WO_EQ 0x002 +#define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) +#define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) +#define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) +#define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) +#define WO_MATCH 0x040 +#define WO_ISNULL 0x080 +#define WO_OR 0x100 /* Two or more OR-connected terms */ +#define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ +#define WO_NOOP 0x800 /* This term does not restrict search space */ + +#define WO_ALL 0xfff /* Mask of all possible WO_* values */ +#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ + +/* +** These are definitions of bits in the WhereLoop.wsFlags field. +** The particular combination of bits in each WhereLoop help to +** determine the algorithm that WhereLoop represents. +*/ +#define WHERE_COLUMN_EQ 0x00000001 /* x=EXPR */ +#define WHERE_COLUMN_RANGE 0x00000002 /* xEXPR */ +#define WHERE_COLUMN_IN 0x00000004 /* x IN (...) */ +#define WHERE_COLUMN_NULL 0x00000008 /* x IS NULL */ +#define WHERE_CONSTRAINT 0x0000000f /* Any of the WHERE_COLUMN_xxx values */ +#define WHERE_TOP_LIMIT 0x00000010 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00000030 /* Both x>EXPR and xnRowOut); +} + +/* +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this +** WHERE clause returns outputs for DISTINCT processing. +*/ +SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ + return pWInfo->eDistinct; +} + +/* +** Return TRUE if the WHERE clause returns rows in ORDER BY order. +** Return FALSE if the output needs to be sorted. +*/ +SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ + return pWInfo->nOBSat; +} + +/* +** Return the VDBE address or label to jump to in order to continue +** immediately with the next row of a WHERE clause. +*/ +SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); + return pWInfo->iContinue; +} + +/* +** Return the VDBE address or label to jump to in order to break +** out of a WHERE loop. +*/ +SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ + return pWInfo->iBreak; +} + +/* +** Return TRUE if an UPDATE or DELETE statement can operate directly on +** the rowids returned by a WHERE clause. Return FALSE if doing an +** UPDATE or DELETE might change subsequent WHERE clause results. +** +** If the ONEPASS optimization is used (if this routine returns true) +** then also write the indices of open cursors used by ONEPASS +** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data +** table and iaCur[1] gets the cursor used by an auxiliary index. +** Either value may be -1, indicating that cursor is not used. +** Any cursors returned will have been opened for writing. +** +** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is +** unable to use the ONEPASS optimization. +*/ +SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ + memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); + return pWInfo->okOnePass; +} + +/* +** Move the content of pSrc into pDest +*/ +static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ + pDest->n = pSrc->n; + memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); +} + +/* +** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. +** +** The new entry might overwrite an existing entry, or it might be +** appended, or it might be discarded. Do whatever is the right thing +** so that pSet keeps the N_OR_COST best entries seen so far. +*/ +static int whereOrInsert( + WhereOrSet *pSet, /* The WhereOrSet to be updated */ + Bitmask prereq, /* Prerequisites of the new entry */ + LogEst rRun, /* Run-cost of the new entry */ + LogEst nOut /* Number of outputs for the new entry */ +){ + u16 i; + WhereOrCost *p; + for(i=pSet->n, p=pSet->a; i>0; i--, p++){ + if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ + goto whereOrInsert_done; + } + if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ + return 0; + } + } + if( pSet->na[pSet->n++]; + p->nOut = nOut; + }else{ + p = pSet->a; + for(i=1; in; i++){ + if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; + } + if( p->rRun<=rRun ) return 0; + } +whereOrInsert_done: + p->prereq = prereq; + p->rRun = rRun; + if( p->nOut>nOut ) p->nOut = nOut; + return 1; +} + +/* +** Initialize a preallocated WhereClause structure. +*/ +static void whereClauseInit( + WhereClause *pWC, /* The WhereClause to be initialized */ + WhereInfo *pWInfo /* The WHERE processing context */ +){ + pWC->pWInfo = pWInfo; + pWC->pOuter = 0; + pWC->nTerm = 0; + pWC->nSlot = ArraySize(pWC->aStatic); + pWC->a = pWC->aStatic; +} + +/* Forward reference */ +static void whereClauseClear(WhereClause*); + +/* +** Deallocate all memory associated with a WhereOrInfo object. +*/ +static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ + whereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} + +/* +** Deallocate all memory associated with a WhereAndInfo object. +*/ +static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ + whereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} + +/* +** Deallocate a WhereClause structure. The WhereClause structure +** itself is not freed. This routine is the inverse of whereClauseInit(). +*/ +static void whereClauseClear(WhereClause *pWC){ + int i; + WhereTerm *a; + sqlite3 *db = pWC->pWInfo->pParse->db; + for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ + if( a->wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, a->pExpr); + } + if( a->wtFlags & TERM_ORINFO ){ + whereOrInfoDelete(db, a->u.pOrInfo); + }else if( a->wtFlags & TERM_ANDINFO ){ + whereAndInfoDelete(db, a->u.pAndInfo); + } + } + if( pWC->a!=pWC->aStatic ){ + sqlite3DbFree(db, pWC->a); + } +} + +/* +** Add a single new WhereTerm entry to the WhereClause object pWC. +** The new WhereTerm object is constructed from Expr p and with wtFlags. +** The index in pWC->a[] of the new WhereTerm is returned on success. +** 0 is returned if the new WhereTerm could not be added due to a memory +** allocation error. The memory allocation failure will be recorded in +** the db->mallocFailed flag so that higher-level functions can detect it. +** +** This routine will increase the size of the pWC->a[] array as necessary. +** +** If the wtFlags argument includes TERM_DYNAMIC, then responsibility +** for freeing the expression p is assumed by the WhereClause object pWC. +** This is true even if this routine fails to allocate a new WhereTerm. +** +** WARNING: This routine might reallocate the space used to store +** WhereTerms. All pointers to WhereTerms should be invalidated after +** calling this routine. Such pointers may be reinitialized by referencing +** the pWC->a[] array. +*/ +static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ + WhereTerm *pTerm; + int idx; + testcase( wtFlags & TERM_VIRTUAL ); + if( pWC->nTerm>=pWC->nSlot ){ + WhereTerm *pOld = pWC->a; + sqlite3 *db = pWC->pWInfo->pParse->db; + pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + if( pWC->a==0 ){ + if( wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, p); + } + pWC->a = pOld; + return 0; + } + memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); + if( pOld!=pWC->aStatic ){ + sqlite3DbFree(db, pOld); + } + pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + } + pTerm = &pWC->a[idx = pWC->nTerm++]; + if( p && ExprHasProperty(p, EP_Unlikely) ){ + pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; + }else{ + pTerm->truthProb = 1; + } + pTerm->pExpr = sqlite3ExprSkipCollate(p); + pTerm->wtFlags = wtFlags; + pTerm->pWC = pWC; + pTerm->iParent = -1; + return idx; +} + +/* +** This routine identifies subexpressions in the WHERE clause where +** each subexpression is separated by the AND operator or some other +** operator specified in the op parameter. The WhereClause structure +** is filled with pointers to subexpressions. For example: +** +** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) +** \________/ \_______________/ \________________/ +** slot[0] slot[1] slot[2] +** +** The original WHERE clause in pExpr is unaltered. All this routine +** does is make slot[] entries point to substructure within pExpr. +** +** In the previous sentence and in the diagram, "slot[]" refers to +** the WhereClause.a[] array. The slot[] array grows as needed to contain +** all terms of the WHERE clause. +*/ +static void whereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ + pWC->op = op; + if( pExpr==0 ) return; + if( pExpr->op!=op ){ + whereClauseInsert(pWC, pExpr, 0); + }else{ + whereSplit(pWC, pExpr->pLeft, op); + whereSplit(pWC, pExpr->pRight, op); + } +} + +/* +** Initialize a WhereMaskSet object +*/ +#define initMaskSet(P) (P)->n=0 + +/* +** Return the bitmask for the given cursor number. Return 0 if +** iCursor is not in the set. +*/ +static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ + int i; + assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); + for(i=0; in; i++){ + if( pMaskSet->ix[i]==iCursor ){ + return MASKBIT(i); + } + } + return 0; +} + +/* +** Create a new mask for cursor iCursor. +** +** There is one cursor per table in the FROM clause. The number of +** tables in the FROM clause is limited by a test early in the +** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] +** array will never overflow. +*/ +static void createMask(WhereMaskSet *pMaskSet, int iCursor){ + assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); + pMaskSet->ix[pMaskSet->n++] = iCursor; +} + +/* +** These routines walk (recursively) an expression tree and generate +** a bitmask indicating which tables are used in that expression +** tree. +*/ +static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*); +static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*); +static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){ + Bitmask mask = 0; + if( p==0 ) return 0; + if( p->op==TK_COLUMN ){ + mask = getMask(pMaskSet, p->iTable); + return mask; + } + mask = exprTableUsage(pMaskSet, p->pRight); + mask |= exprTableUsage(pMaskSet, p->pLeft); + if( ExprHasProperty(p, EP_xIsSelect) ){ + mask |= exprSelectTableUsage(pMaskSet, p->x.pSelect); + }else{ + mask |= exprListTableUsage(pMaskSet, p->x.pList); + } + return mask; +} +static Bitmask exprListTableUsage(WhereMaskSet *pMaskSet, ExprList *pList){ + int i; + Bitmask mask = 0; + if( pList ){ + for(i=0; inExpr; i++){ + mask |= exprTableUsage(pMaskSet, pList->a[i].pExpr); + } + } + return mask; +} +static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ + Bitmask mask = 0; + while( pS ){ + SrcList *pSrc = pS->pSrc; + mask |= exprListTableUsage(pMaskSet, pS->pEList); + mask |= exprListTableUsage(pMaskSet, pS->pGroupBy); + mask |= exprListTableUsage(pMaskSet, pS->pOrderBy); + mask |= exprTableUsage(pMaskSet, pS->pWhere); + mask |= exprTableUsage(pMaskSet, pS->pHaving); + if( ALWAYS(pSrc!=0) ){ + int i; + for(i=0; inSrc; i++){ + mask |= exprSelectTableUsage(pMaskSet, pSrc->a[i].pSelect); + mask |= exprTableUsage(pMaskSet, pSrc->a[i].pOn); + } + } + pS = pS->pPrior; + } + return mask; +} + +/* +** Return TRUE if the given operator is one of the operators that is +** allowed for an indexable WHERE clause term. The allowed operators are +** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" +*/ +static int allowedOp(int op){ + assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL; +} + +/* +** Swap two objects of type TYPE. +*/ +#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} + +/* +** Commute a comparison operator. Expressions of the form "X op Y" +** are converted into "Y op X". +** +** If left/right precedence rules come into play when determining the +** collating sequence, then COLLATE operators are adjusted to ensure +** that the collating sequence does not change. For example: +** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on +** the left hand side of a comparison overrides any collation sequence +** attached to the right. For the same reason the EP_Collate flag +** is not commuted. +*/ +static void exprCommute(Parse *pParse, Expr *pExpr){ + u16 expRight = (pExpr->pRight->flags & EP_Collate); + u16 expLeft = (pExpr->pLeft->flags & EP_Collate); + assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); + if( expRight==expLeft ){ + /* Either X and Y both have COLLATE operator or neither do */ + if( expRight ){ + /* Both X and Y have COLLATE operators. Make sure X is always + ** used by clearing the EP_Collate flag from Y. */ + pExpr->pRight->flags &= ~EP_Collate; + }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ + /* Neither X nor Y have COLLATE operators, but X has a non-default + ** collating sequence. So add the EP_Collate marker on X to cause + ** it to be searched first. */ + pExpr->pLeft->flags |= EP_Collate; + } + } + SWAP(Expr*,pExpr->pRight,pExpr->pLeft); + if( pExpr->op>=TK_GT ){ + assert( TK_LT==TK_GT+2 ); + assert( TK_GE==TK_LE+2 ); + assert( TK_GT>TK_EQ ); + assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); + pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; + } +} + +/* +** Translate from TK_xx operator to WO_xx bitmask. +*/ +static u16 operatorMask(int op){ + u16 c; + assert( allowedOp(op) ); + if( op==TK_IN ){ + c = WO_IN; + }else if( op==TK_ISNULL ){ + c = WO_ISNULL; + }else{ + assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); + c = (u16)(WO_EQ<<(op-TK_EQ)); + } + assert( op!=TK_ISNULL || c==WO_ISNULL ); + assert( op!=TK_IN || c==WO_IN ); + assert( op!=TK_EQ || c==WO_EQ ); + assert( op!=TK_LT || c==WO_LT ); + assert( op!=TK_LE || c==WO_LE ); + assert( op!=TK_GT || c==WO_GT ); + assert( op!=TK_GE || c==WO_GE ); + return c; +} + +/* +** Advance to the next WhereTerm that matches according to the criteria +** established when the pScan object was initialized by whereScanInit(). +** Return NULL if there are no more matching WhereTerms. +*/ +static WhereTerm *whereScanNext(WhereScan *pScan){ + int iCur; /* The cursor on the LHS of the term */ + int iColumn; /* The column on the LHS of the term. -1 for IPK */ + Expr *pX; /* An expression being tested */ + WhereClause *pWC; /* Shorthand for pScan->pWC */ + WhereTerm *pTerm; /* The term being tested */ + int k = pScan->k; /* Where to start scanning */ + + while( pScan->iEquiv<=pScan->nEquiv ){ + iCur = pScan->aEquiv[pScan->iEquiv-2]; + iColumn = pScan->aEquiv[pScan->iEquiv-1]; + while( (pWC = pScan->pWC)!=0 ){ + for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + && (pScan->iEquiv<=2 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + ){ + if( (pTerm->eOperator & WO_EQUIV)!=0 + && pScan->nEquivaEquiv) + ){ + int j; + pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); + assert( pX->op==TK_COLUMN ); + for(j=0; jnEquiv; j+=2){ + if( pScan->aEquiv[j]==pX->iTable + && pScan->aEquiv[j+1]==pX->iColumn ){ + break; + } + } + if( j==pScan->nEquiv ){ + pScan->aEquiv[j] = pX->iTable; + pScan->aEquiv[j+1] = pX->iColumn; + pScan->nEquiv += 2; + } + } + if( (pTerm->eOperator & pScan->opMask)!=0 ){ + /* Verify the affinity and collating sequence match */ + if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + Parse *pParse = pWC->pWInfo->pParse; + pX = pTerm->pExpr; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse, + pX->pLeft, pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + continue; + } + } + if( (pTerm->eOperator & WO_EQ)!=0 + && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && pX->iTable==pScan->aEquiv[0] + && pX->iColumn==pScan->aEquiv[1] + ){ + continue; + } + pScan->k = k+1; + return pTerm; + } + } + } + pScan->pWC = pScan->pWC->pOuter; + k = 0; + } + pScan->pWC = pScan->pOrigWC; + k = 0; + pScan->iEquiv += 2; + } + return 0; +} + +/* +** Initialize a WHERE clause scanner object. Return a pointer to the +** first match. Return NULL if there are no matches. +** +** The scanner will be searching the WHERE clause pWC. It will look +** for terms of the form "X " where X is column iColumn of table +** iCur. The must be one of the operators described by opMask. +** +** If the search is for X and the WHERE clause contains terms of the +** form X=Y then this routine might also return terms of the form +** "Y ". The number of levels of transitivity is limited, +** but is enough to handle most commonly occurring SQL statements. +** +** If X is not the INTEGER PRIMARY KEY then X must be compatible with +** index pIdx. +*/ +static WhereTerm *whereScanInit( + WhereScan *pScan, /* The WhereScan object being initialized */ + WhereClause *pWC, /* The WHERE clause to be scanned */ + int iCur, /* Cursor to scan for */ + int iColumn, /* Column to scan for */ + u32 opMask, /* Operator(s) to scan for */ + Index *pIdx /* Must be compatible with this index */ +){ + int j; + + /* memset(pScan, 0, sizeof(*pScan)); */ + pScan->pOrigWC = pWC; + pScan->pWC = pWC; + if( pIdx && iColumn>=0 ){ + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ + if( NEVER(j>pIdx->nColumn) ) return 0; + } + pScan->zCollName = pIdx->azColl[j]; + }else{ + pScan->idxaff = 0; + pScan->zCollName = 0; + } + pScan->opMask = opMask; + pScan->k = 0; + pScan->aEquiv[0] = iCur; + pScan->aEquiv[1] = iColumn; + pScan->nEquiv = 2; + pScan->iEquiv = 2; + return whereScanNext(pScan); +} + +/* +** Search for a term in the WHERE clause that is of the form "X " +** where X is a reference to the iColumn of table iCur and is one of +** the WO_xx operator codes specified by the op parameter. +** Return a pointer to the term. Return 0 if not found. +** +** The term returned might by Y= if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aEquiv[] array holds X and all its equivalents, with each SQL variable +** taking up two slots in aEquiv[]. The first slot is for the cursor number +** and the second is for the column number. There are 22 slots in aEquiv[] +** so that means we can look for X plus up to 10 other equivalent values. +** Hence a search for X will return if X=A1 and A1=A2 and A2=A3 +** and ... and A9=A10 and A10=. +** +** If there are multiple terms in the WHERE clause of the form "X " +** then try for the one with no dependencies on - in other words where +** is a constant expression of some kind. Only return entries of +** the form "X Y" where Y is a column in another table if no terms of +** the form "X " exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. +*/ +static WhereTerm *findTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ +){ + WhereTerm *pResult = 0; + WhereTerm *p; + WhereScan scan; + + p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); + while( p ){ + if( (p->prereqRight & notReady)==0 ){ + if( p->prereqRight==0 && (p->eOperator&WO_EQ)!=0 ){ + return p; + } + if( pResult==0 ) pResult = p; + } + p = whereScanNext(&scan); + } + return pResult; +} + +/* Forward reference */ +static void exprAnalyze(SrcList*, WhereClause*, int); + +/* +** Call exprAnalyze on all terms in a WHERE clause. +*/ +static void exprAnalyzeAll( + SrcList *pTabList, /* the FROM clause */ + WhereClause *pWC /* the WHERE clause to be analyzed */ +){ + int i; + for(i=pWC->nTerm-1; i>=0; i--){ + exprAnalyze(pTabList, pWC, i); + } +} + +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION +/* +** Check to see if the given expression is a LIKE or GLOB operator that +** can be optimized using inequality constraints. Return TRUE if it is +** so and false if not. +** +** In order for the operator to be optimizible, the RHS must be a string +** literal that does not begin with a wildcard. +*/ +static int isLikeOrGlob( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* Test this expression */ + Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ + int *pisComplete, /* True if the only wildcard is % in the last character */ + int *pnoCase /* True if uppercase is equivalent to lowercase */ +){ + const char *z = 0; /* String on RHS of LIKE operator */ + Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ + ExprList *pList; /* List of operands to the LIKE operator */ + int c; /* One character in z[] */ + int cnt; /* Number of non-wildcard prefix characters */ + char wc[3]; /* Wildcard characters */ + sqlite3 *db = pParse->db; /* Database connection */ + sqlite3_value *pVal = 0; + int op; /* Opcode of pRight */ + + if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ + return 0; + } +#ifdef SQLITE_EBCDIC + if( *pnoCase ) return 0; +#endif + pList = pExpr->x.pList; + pLeft = pList->a[1].pExpr; + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) + ){ + /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must + ** be the name of an indexed column with TEXT affinity. */ + return 0; + } + assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ + + pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); + op = pRight->op; + if( op==TK_VARIABLE ){ + Vdbe *pReprepare = pParse->pReprepare; + int iCol = pRight->iColumn; + pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_NONE); + if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ + z = (char *)sqlite3_value_text(pVal); + } + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); + assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); + }else if( op==TK_STRING ){ + z = pRight->u.zToken; + } + if( z ){ + cnt = 0; + while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ + cnt++; + } + if( cnt!=0 && 255!=(u8)z[cnt-1] ){ + Expr *pPrefix; + *pisComplete = c==wc[0] && z[cnt+1]==0; + pPrefix = sqlite3Expr(db, TK_STRING, z); + if( pPrefix ) pPrefix->u.zToken[cnt] = 0; + *ppPrefix = pPrefix; + if( op==TK_VARIABLE ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeSetVarmask(v, pRight->iColumn); + if( *pisComplete && pRight->u.zToken[1] ){ + /* If the rhs of the LIKE expression is a variable, and the current + ** value of the variable means there is no need to invoke the LIKE + ** function, then no OP_Variable will be added to the program. + ** This causes problems for the sqlite3_bind_parameter_name() + ** API. To workaround them, add a dummy OP_Variable here. + */ + int r1 = sqlite3GetTempReg(pParse); + sqlite3ExprCodeTarget(pParse, pRight, r1); + sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); + sqlite3ReleaseTempReg(pParse, r1); + } + } + }else{ + z = 0; + } + } + + sqlite3ValueFree(pVal); + return (z!=0); +} +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Check to see if the given expression is of the form +** +** column MATCH expr +** +** If it is then return TRUE. If not, return FALSE. +*/ +static int isMatchOfColumn( + Expr *pExpr /* Test this expression */ +){ + ExprList *pList; + + if( pExpr->op!=TK_FUNCTION ){ + return 0; + } + if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ + return 0; + } + pList = pExpr->x.pList; + if( pList->nExpr!=2 ){ + return 0; + } + if( pList->a[1].pExpr->op != TK_COLUMN ){ + return 0; + } + return 1; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** If the pBase expression originated in the ON or USING clause of +** a join, then transfer the appropriate markings over to derived. +*/ +static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ + if( pDerived ){ + pDerived->flags |= pBase->flags & EP_FromJoin; + pDerived->iRightJoinTable = pBase->iRightJoinTable; + } +} + +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) +/* +** Analyze a term that consists of two or more OR-connected +** subterms. So in: +** +** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) +** ^^^^^^^^^^^^^^^^^^^^ +** +** This routine analyzes terms such as the middle term in the above example. +** A WhereOrTerm object is computed and attached to the term under +** analysis, regardless of the outcome of the analysis. Hence: +** +** WhereTerm.wtFlags |= TERM_ORINFO +** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object +** +** The term being analyzed must have two or more of OR-connected subterms. +** A single subterm might be a set of AND-connected sub-subterms. +** Examples of terms under analysis: +** +** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 +** (B) x=expr1 OR expr2=x OR x=expr3 +** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) +** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') +** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) +** +** CASE 1: +** +** If all subterms are of the form T.C=expr for some single column of C and +** a single table T (as shown in example B above) then create a new virtual +** term that is an equivalent IN expression. In other words, if the term +** being analyzed is: +** +** x = expr1 OR expr2 = x OR x = expr3 +** +** then create a new virtual term like this: +** +** x IN (expr1,expr2,expr3) +** +** CASE 2: +** +** If all subterms are indexable by a single table T, then set +** +** WhereTerm.eOperator = WO_OR +** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T +** +** A subterm is "indexable" if it is of the form +** "T.C " where C is any column of table T and +** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". +** A subterm is also indexable if it is an AND of two or more +** subsubterms at least one of which is indexable. Indexable AND +** subterms have their eOperator set to WO_AND and they have +** u.pAndInfo set to a dynamically allocated WhereAndTerm object. +** +** From another point of view, "indexable" means that the subterm could +** potentially be used with an index if an appropriate index exists. +** This analysis does not consider whether or not the index exists; that +** is decided elsewhere. This analysis only looks at whether subterms +** appropriate for indexing exist. +** +** All examples A through E above satisfy case 2. But if a term +** also statisfies case 1 (such as B) we know that the optimizer will +** always prefer case 1, so in that case we pretend that case 2 is not +** satisfied. +** +** It might be the case that multiple tables are indexable. For example, +** (E) above is indexable on tables P, Q, and R. +** +** Terms that satisfy case 2 are candidates for lookup by using +** separate indices to find rowids for each subterm and composing +** the union of all rowids using a RowSet object. This is similar +** to "bitmap indices" in other database engines. +** +** OTHERWISE: +** +** If neither case 1 nor case 2 apply, then leave the eOperator set to +** zero. This term is not useful for search. +*/ +static void exprAnalyzeOrTerm( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the complete WHERE clause */ + int idxTerm /* Index of the OR-term to be analyzed */ +){ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + Parse *pParse = pWInfo->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ + Expr *pExpr = pTerm->pExpr; /* The expression of the term */ + int i; /* Loop counters */ + WhereClause *pOrWc; /* Breakup of pTerm into subterms */ + WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ + WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ + Bitmask chngToIN; /* Tables that might satisfy case 1 */ + Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ + + /* + ** Break the OR clause into its separate subterms. The subterms are + ** stored in a WhereClause structure containing within the WhereOrInfo + ** object that is attached to the original OR clause term. + */ + assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); + assert( pExpr->op==TK_OR ); + pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); + if( pOrInfo==0 ) return; + pTerm->wtFlags |= TERM_ORINFO; + pOrWc = &pOrInfo->wc; + whereClauseInit(pOrWc, pWInfo); + whereSplit(pOrWc, pExpr, TK_OR); + exprAnalyzeAll(pSrc, pOrWc); + if( db->mallocFailed ) return; + assert( pOrWc->nTerm>=2 ); + + /* + ** Compute the set of tables that might satisfy cases 1 or 2. + */ + indexable = ~(Bitmask)0; + chngToIN = ~(Bitmask)0; + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ + if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ + WhereAndInfo *pAndInfo; + assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); + chngToIN = 0; + pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); + if( pAndInfo ){ + WhereClause *pAndWC; + WhereTerm *pAndTerm; + int j; + Bitmask b = 0; + pOrTerm->u.pAndInfo = pAndInfo; + pOrTerm->wtFlags |= TERM_ANDINFO; + pOrTerm->eOperator = WO_AND; + pAndWC = &pAndInfo->wc; + whereClauseInit(pAndWC, pWC->pWInfo); + whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); + exprAnalyzeAll(pSrc, pAndWC); + pAndWC->pOuter = pWC; + testcase( db->mallocFailed ); + if( !db->mallocFailed ){ + for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ + assert( pAndTerm->pExpr ); + if( allowedOp(pAndTerm->pExpr->op) ){ + b |= getMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); + } + } + } + indexable &= b; + } + }else if( pOrTerm->wtFlags & TERM_COPIED ){ + /* Skip this term for now. We revisit it when we process the + ** corresponding TERM_VIRTUAL term */ + }else{ + Bitmask b; + b = getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); + if( pOrTerm->wtFlags & TERM_VIRTUAL ){ + WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; + b |= getMask(&pWInfo->sMaskSet, pOther->leftCursor); + } + indexable &= b; + if( (pOrTerm->eOperator & WO_EQ)==0 ){ + chngToIN = 0; + }else{ + chngToIN &= b; + } + } + } + + /* + ** Record the set of tables that satisfy case 2. The set might be + ** empty. + */ + pOrInfo->indexable = indexable; + pTerm->eOperator = indexable==0 ? 0 : WO_OR; + + /* + ** chngToIN holds a set of tables that *might* satisfy case 1. But + ** we have to do some additional checking to see if case 1 really + ** is satisfied. + ** + ** chngToIN will hold either 0, 1, or 2 bits. The 0-bit case means + ** that there is no possibility of transforming the OR clause into an + ** IN operator because one or more terms in the OR clause contain + ** something other than == on a column in the single table. The 1-bit + ** case means that every term of the OR clause is of the form + ** "table.column=expr" for some single table. The one bit that is set + ** will correspond to the common table. We still need to check to make + ** sure the same column is used on all terms. The 2-bit case is when + ** the all terms are of the form "table1.column=table2.column". It + ** might be possible to form an IN operator with either table1.column + ** or table2.column as the LHS if either is common to every term of + ** the OR clause. + ** + ** Note that terms of the form "table.column1=table.column2" (the + ** same table on both sizes of the ==) cannot be optimized. + */ + if( chngToIN ){ + int okToChngToIN = 0; /* True if the conversion to IN is valid */ + int iColumn = -1; /* Column index on lhs of IN operator */ + int iCursor = -1; /* Table cursor common to all terms */ + int j = 0; /* Loop counter */ + + /* Search for a table and column that appears on one side or the + ** other of the == operator in every subterm. That table and column + ** will be recorded in iCursor and iColumn. There might not be any + ** such table and column. Set okToChngToIN if an appropriate table + ** and column is found but leave okToChngToIN false if not found. + */ + for(j=0; j<2 && !okToChngToIN; j++){ + pOrTerm = pOrWc->a; + for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + pOrTerm->wtFlags &= ~TERM_OR_OK; + if( pOrTerm->leftCursor==iCursor ){ + /* This is the 2-bit case and we are on the second iteration and + ** current term is from the first iteration. So skip this term. */ + assert( j==1 ); + continue; + } + if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ + /* This term must be of the form t1.a==t2.b where t2 is in the + ** chngToIN set but t1 is not. This term will be either preceeded + ** or follwed by an inverted copy (t2.b==t1.a). Skip this term + ** and use its inversion. */ + testcase( pOrTerm->wtFlags & TERM_COPIED ); + testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); + assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); + continue; + } + iColumn = pOrTerm->u.leftColumn; + iCursor = pOrTerm->leftCursor; + break; + } + if( i<0 ){ + /* No candidate table+column was found. This can only occur + ** on the second iteration */ + assert( j==1 ); + assert( IsPowerOfTwo(chngToIN) ); + assert( chngToIN==getMask(&pWInfo->sMaskSet, iCursor) ); + break; + } + testcase( j==1 ); + + /* We have found a candidate table and column. Check to see if that + ** table and column is common to every term in the OR clause */ + okToChngToIN = 1; + for(; i>=0 && okToChngToIN; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + if( pOrTerm->leftCursor!=iCursor ){ + pOrTerm->wtFlags &= ~TERM_OR_OK; + }else if( pOrTerm->u.leftColumn!=iColumn ){ + okToChngToIN = 0; + }else{ + int affLeft, affRight; + /* If the right-hand side is also a column, then the affinities + ** of both right and left sides must be such that no type + ** conversions are required on the right. (Ticket #2249) + */ + affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight); + affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); + if( affRight!=0 && affRight!=affLeft ){ + okToChngToIN = 0; + }else{ + pOrTerm->wtFlags |= TERM_OR_OK; + } + } + } + } + + /* At this point, okToChngToIN is true if original pTerm satisfies + ** case 1. In that case, construct a new virtual term that is + ** pTerm converted into an IN operator. + */ + if( okToChngToIN ){ + Expr *pDup; /* A transient duplicate expression */ + ExprList *pList = 0; /* The RHS of the IN operator */ + Expr *pLeft = 0; /* The LHS of the IN operator */ + Expr *pNew; /* The complete IN operator */ + + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ + if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; + assert( pOrTerm->eOperator & WO_EQ ); + assert( pOrTerm->leftCursor==iCursor ); + assert( pOrTerm->u.leftColumn==iColumn ); + pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); + pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); + pLeft = pOrTerm->pExpr->pLeft; + } + assert( pLeft!=0 ); + pDup = sqlite3ExprDup(db, pLeft, 0); + pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); + if( pNew ){ + int idxNew; + transferJoinMarkings(pNew, pExpr); + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + pNew->x.pList = pList; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + pWC->a[idxNew].iParent = idxTerm; + pTerm->nChild = 1; + }else{ + sqlite3ExprListDelete(db, pList); + } + pTerm->eOperator = WO_NOOP; /* case 1 trumps case 2 */ + } + } +} +#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ + +/* +** The input to this routine is an WhereTerm structure with only the +** "pExpr" field filled in. The job of this routine is to analyze the +** subexpression and populate all the other fields of the WhereTerm +** structure. +** +** If the expression is of the form " X" it gets commuted +** to the standard form of "X ". +** +** If the expression is of the form "X Y" where both X and Y are +** columns, then the original expression is unchanged and a new virtual +** term of the form "Y X" is added to the WHERE clause and +** analyzed separately. The original term is marked with TERM_COPIED +** and the new term is marked with TERM_DYNAMIC (because it's pExpr +** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it +** is a commuted copy of a prior term.) The original term has nChild=1 +** and the copy has idxParent set to the index of the original term. +*/ +static void exprAnalyze( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the WHERE clause */ + int idxTerm /* Index of the term to be analyzed */ +){ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + WhereTerm *pTerm; /* The term to be analyzed */ + WhereMaskSet *pMaskSet; /* Set of table index masks */ + Expr *pExpr; /* The expression to be analyzed */ + Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ + Bitmask prereqAll; /* Prerequesites of pExpr */ + Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ + Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ + int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ + int noCase = 0; /* LIKE/GLOB distinguishes case */ + int op; /* Top-level operator. pExpr->op */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection */ + + if( db->mallocFailed ){ + return; + } + pTerm = &pWC->a[idxTerm]; + pMaskSet = &pWInfo->sMaskSet; + pExpr = pTerm->pExpr; + assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); + prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); + op = pExpr->op; + if( op==TK_IN ){ + assert( pExpr->pRight==0 ); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + pTerm->prereqRight = exprSelectTableUsage(pMaskSet, pExpr->x.pSelect); + }else{ + pTerm->prereqRight = exprListTableUsage(pMaskSet, pExpr->x.pList); + } + }else if( op==TK_ISNULL ){ + pTerm->prereqRight = 0; + }else{ + pTerm->prereqRight = exprTableUsage(pMaskSet, pExpr->pRight); + } + prereqAll = exprTableUsage(pMaskSet, pExpr); + if( ExprHasProperty(pExpr, EP_FromJoin) ){ + Bitmask x = getMask(pMaskSet, pExpr->iRightJoinTable); + prereqAll |= x; + extraRight = x-1; /* ON clause terms may not be used with an index + ** on left table of a LEFT JOIN. Ticket #3015 */ + } + pTerm->prereqAll = prereqAll; + pTerm->leftCursor = -1; + pTerm->iParent = -1; + pTerm->eOperator = 0; + if( allowedOp(op) ){ + Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); + Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; + if( pLeft->op==TK_COLUMN ){ + pTerm->leftCursor = pLeft->iTable; + pTerm->u.leftColumn = pLeft->iColumn; + pTerm->eOperator = operatorMask(op) & opMask; + } + if( pRight && pRight->op==TK_COLUMN ){ + WhereTerm *pNew; + Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ + if( pTerm->leftCursor>=0 ){ + int idxNew; + pDup = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + return; + } + idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); + if( idxNew==0 ) return; + pNew = &pWC->a[idxNew]; + pNew->iParent = idxTerm; + pTerm = &pWC->a[idxTerm]; + pTerm->nChild = 1; + pTerm->wtFlags |= TERM_COPIED; + if( pExpr->op==TK_EQ + && !ExprHasProperty(pExpr, EP_FromJoin) + && OptimizationEnabled(db, SQLITE_Transitive) + ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } + }else{ + pDup = pExpr; + pNew = pTerm; + } + exprCommute(pParse, pDup); + pLeft = sqlite3ExprSkipCollate(pDup->pLeft); + pNew->leftCursor = pLeft->iTable; + pNew->u.leftColumn = pLeft->iColumn; + testcase( (prereqLeft | extraRight) != prereqLeft ); + pNew->prereqRight = prereqLeft | extraRight; + pNew->prereqAll = prereqAll; + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; + } + } + +#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION + /* If a term is the BETWEEN operator, create two new virtual terms + ** that define the range that the BETWEEN implements. For example: + ** + ** a BETWEEN b AND c + ** + ** is converted into: + ** + ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) + ** + ** The two new terms are added onto the end of the WhereClause object. + ** The new terms are "dynamic" and are children of the original BETWEEN + ** term. That means that if the BETWEEN term is coded, the children are + ** skipped. Or, if the children are satisfied by an index, the original + ** BETWEEN term is skipped. + */ + else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ + ExprList *pList = pExpr->x.pList; + int i; + static const u8 ops[] = {TK_GE, TK_LE}; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + for(i=0; i<2; i++){ + Expr *pNewExpr; + int idxNew; + pNewExpr = sqlite3PExpr(pParse, ops[i], + sqlite3ExprDup(db, pExpr->pLeft, 0), + sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); + transferJoinMarkings(pNewExpr, pExpr); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + pWC->a[idxNew].iParent = idxTerm; + } + pTerm->nChild = 2; + } +#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ + +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) + /* Analyze a term that is composed of two or more subterms connected by + ** an OR operator. + */ + else if( pExpr->op==TK_OR ){ + assert( pWC->op==TK_AND ); + exprAnalyzeOrTerm(pSrc, pWC, idxTerm); + pTerm = &pWC->a[idxTerm]; + } +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION + /* Add constraints to reduce the search space on a LIKE or GLOB + ** operator. + ** + ** A like pattern of the form "x LIKE 'abc%'" is changed into constraints + ** + ** x>='abc' AND x<'abd' AND x LIKE 'abc%' + ** + ** The last character of the prefix "abc" is incremented to form the + ** termination condition "abd". + */ + if( pWC->op==TK_AND + && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) + ){ + Expr *pLeft; /* LHS of LIKE/GLOB operator */ + Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ + Expr *pNewExpr1; + Expr *pNewExpr2; + int idxNew1; + int idxNew2; + Token sCollSeqName; /* Name of collating sequence */ + + pLeft = pExpr->x.pList->a[1].pExpr; + pStr2 = sqlite3ExprDup(db, pStr1, 0); + if( !db->mallocFailed ){ + u8 c, *pC; /* Last character before the first wildcard */ + pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; + c = *pC; + if( noCase ){ + /* The point is to increment the last character before the first + ** wildcard. But if we increment '@', that will push it into the + ** alphabetic range where case conversions will mess up the + ** inequality. To avoid this, make sure to also run the full + ** LIKE on all candidate expressions by clearing the isComplete flag + */ + if( c=='A'-1 ) isComplete = 0; + c = sqlite3UpperToLower[c]; + } + *pC = c + 1; + } + sCollSeqName.z = noCase ? "NOCASE" : "BINARY"; + sCollSeqName.n = 6; + pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprAddCollateToken(pParse,pNewExpr1,&sCollSeqName), + pStr1, 0); + transferJoinMarkings(pNewExpr1, pExpr); + idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew1==0 ); + exprAnalyze(pSrc, pWC, idxNew1); + pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr2 = sqlite3PExpr(pParse, TK_LT, + sqlite3ExprAddCollateToken(pParse,pNewExpr2,&sCollSeqName), + pStr2, 0); + transferJoinMarkings(pNewExpr2, pExpr); + idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew2==0 ); + exprAnalyze(pSrc, pWC, idxNew2); + pTerm = &pWC->a[idxTerm]; + if( isComplete ){ + pWC->a[idxNew1].iParent = idxTerm; + pWC->a[idxNew2].iParent = idxTerm; + pTerm->nChild = 2; + } + } +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Add a WO_MATCH auxiliary term to the constraint set if the + ** current expression is of the form: column MATCH expr. + ** This information is used by the xBestIndex methods of + ** virtual tables. The native query optimizer does not attempt + ** to do anything with MATCH functions. + */ + if( isMatchOfColumn(pExpr) ){ + int idxNew; + Expr *pRight, *pLeft; + WhereTerm *pNewTerm; + Bitmask prereqColumn, prereqExpr; + + pRight = pExpr->x.pList->a[0].pExpr; + pLeft = pExpr->x.pList->a[1].pExpr; + prereqExpr = exprTableUsage(pMaskSet, pRight); + prereqColumn = exprTableUsage(pMaskSet, pLeft); + if( (prereqExpr & prereqColumn)==0 ){ + Expr *pNewExpr; + pNewExpr = sqlite3PExpr(pParse, TK_MATCH, + 0, sqlite3ExprDup(db, pRight, 0), 0); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = prereqExpr; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_MATCH; + pNewTerm->iParent = idxTerm; + pTerm = &pWC->a[idxTerm]; + pTerm->nChild = 1; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } + } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + /* When sqlite_stat3 histogram data is available an operator of the + ** form "x IS NOT NULL" can sometimes be evaluated more efficiently + ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a + ** virtual term of that form. + ** + ** Note that the virtual term must be tagged with TERM_VNULL. This + ** TERM_VNULL tag will suppress the not-null check at the beginning + ** of the loop. Without the TERM_VNULL flag, the not-null check at + ** the start of the loop will prevent any results from being returned. + */ + if( pExpr->op==TK_NOTNULL + && pExpr->pLeft->op==TK_COLUMN + && pExpr->pLeft->iColumn>=0 + && OptimizationEnabled(db, SQLITE_Stat3) + ){ + Expr *pNewExpr; + Expr *pLeft = pExpr->pLeft; + int idxNew; + WhereTerm *pNewTerm; + + pNewExpr = sqlite3PExpr(pParse, TK_GT, + sqlite3ExprDup(db, pLeft, 0), + sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0); + + idxNew = whereClauseInsert(pWC, pNewExpr, + TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); + if( idxNew ){ + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = 0; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_GT; + pNewTerm->iParent = idxTerm; + pTerm = &pWC->a[idxTerm]; + pTerm->nChild = 1; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } + } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + + /* Prevent ON clause terms of a LEFT JOIN from being used to drive + ** an index for tables to the left of the join. + */ + pTerm->prereqRight |= extraRight; +} + +/* +** This function searches pList for a entry that matches the iCol-th column +** of index pIdx. +** +** If such an expression is found, its index in pList->a[] is returned. If +** no expression is found, -1 is returned. +*/ +static int findIndexCol( + Parse *pParse, /* Parse context */ + ExprList *pList, /* Expression list to search */ + int iBase, /* Cursor for table associated with pIdx */ + Index *pIdx, /* Index to match column of */ + int iCol /* Column of index to match */ +){ + int i; + const char *zColl = pIdx->azColl[iCol]; + + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); + if( p->op==TK_COLUMN + && p->iColumn==pIdx->aiColumn[iCol] + && p->iTable==iBase + ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); + if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + return i; + } + } + } + + return -1; +} + +/* +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. +** +** A DISTINCT list is redundant if the database contains some subset of +** columns that are unique and non-null. +*/ +static int isDistinctRedundant( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* The FROM clause */ + WhereClause *pWC, /* The WHERE clause */ + ExprList *pDistinct /* The result set that needs to be DISTINCT */ +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; + + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; + + /* If any of the expressions is an IPK column on table iBase, then return + ** true. Note: The (p->iTable==iBase) part of this test may be false if the + ** current SELECT is a correlated sub-query. + */ + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); + if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; + } + + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + ** + ** 3. All of those index columns for which the WHERE clause does not + ** contain a "col=X" term are subject to a NOT NULL constraint. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !IsUniqueIndex(pIdx) ) continue; + for(i=0; inKeyCol; i++){ + i16 iCol = pIdx->aiColumn[i]; + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ + int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); + if( iIdxCol<0 || pTab->aCol[iCol].notNull==0 ){ + break; + } + } + } + if( i==pIdx->nKeyCol ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; + } + } + + return 0; +} + + +/* +** Estimate the logarithm of the input value to base 2. +*/ +static LogEst estLog(LogEst N){ + return N<=10 ? 0 : sqlite3LogEst(N) - 33; +} + +/* +** Two routines for printing the content of an sqlite3_index_info +** structure. Used for testing and debugging only. If neither +** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines +** are no-ops. +*/ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) +static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", + i, + p->aConstraint[i].iColumn, + p->aConstraint[i].iTermOffset, + p->aConstraint[i].op, + p->aConstraint[i].usable); + } + for(i=0; inOrderBy; i++){ + sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); + } +} +static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", + i, + p->aConstraintUsage[i].argvIndex, + p->aConstraintUsage[i].omit); + } + sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); + sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); + sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); + sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); + sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); +} +#else +#define TRACE_IDX_INPUTS(A) +#define TRACE_IDX_OUTPUTS(A) +#endif + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Return TRUE if the WHERE clause term pTerm is of a form where it +** could be used with an index to access pSrc, assuming an appropriate +** index existed. +*/ +static int termCanDriveIndex( + WhereTerm *pTerm, /* WHERE clause term to check */ + struct SrcList_item *pSrc, /* Table we are trying to access */ + Bitmask notReady /* Tables in outer loops of the join */ +){ + char aff; + if( pTerm->leftCursor!=pSrc->iCursor ) return 0; + if( (pTerm->eOperator & WO_EQ)==0 ) return 0; + if( (pTerm->prereqRight & notReady)!=0 ) return 0; + if( pTerm->u.leftColumn<0 ) return 0; + aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; + if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; + return 1; +} +#endif + + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Generate code to construct the Index object for an automatic index +** and to set up the WhereLevel object pLevel so that the code generator +** makes use of the automatic index. +*/ +static void constructAutomaticIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ + Bitmask notReady, /* Mask of cursors that are not available */ + WhereLevel *pLevel /* Write new index here */ +){ + int nKeyCol; /* Number of columns in the constructed index */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pWCEnd; /* End of pWC->a[] */ + Index *pIdx; /* Object describing the transient index */ + Vdbe *v; /* Prepared statement under construction */ + int addrInit; /* Address of the initialization bypass jump */ + Table *pTable; /* The table being indexed */ + int addrTop; /* Top of the index fill loop */ + int regRecord; /* Register holding an index record */ + int n; /* Column counter */ + int i; /* Loop counter */ + int mxBitCol; /* Maximum column in pSrc->colUsed */ + CollSeq *pColl; /* Collating sequence to on a column */ + WhereLoop *pLoop; /* The Loop object */ + char *zNotUsed; /* Extra space on the end of pIdx */ + Bitmask idxCols; /* Bitmap of columns used for indexing */ + Bitmask extraCols; /* Bitmap of additional columns */ + u8 sentWarning = 0; /* True if a warnning has been issued */ + + /* Generate code to skip over the creation and initialization of the + ** transient index on 2nd and subsequent iterations of the loop. */ + v = pParse->pVdbe; + assert( v!=0 ); + addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); + + /* Count the number of columns that will be added to the index + ** and used to match WHERE clause constraints */ + nKeyCol = 0; + pTable = pSrc->pTab; + pWCEnd = &pWC->a[pWC->nTerm]; + pLoop = pLevel->pWLoop; + idxCols = 0; + for(pTerm=pWC->a; pTermu.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + if( !sentWarning ){ + sqlite3_log(SQLITE_WARNING_AUTOINDEX, + "automatic index on %s(%s)", pTable->zName, + pTable->aCol[iCol].zName); + sentWarning = 1; + } + if( (idxCols & cMask)==0 ){ + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return; + pLoop->aLTerm[nKeyCol++] = pTerm; + idxCols |= cMask; + } + } + } + assert( nKeyCol>0 ); + pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; + pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED + | WHERE_AUTO_INDEX; + + /* Count the number of additional columns needed to create a + ** covering index. A "covering index" is an index that contains all + ** columns that are needed by the query. With a covering index, the + ** original table never needs to be accessed. Automatic indices must + ** be a covering index because the index will not be updated if the + ** original table changes and the index and table cannot both be used + ** if they go out of sync. + */ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; + testcase( pTable->nCol==BMS-1 ); + testcase( pTable->nCol==BMS-2 ); + for(i=0; icolUsed & MASKBIT(BMS-1) ){ + nKeyCol += pTable->nCol - BMS + 1; + } + pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; + + /* Construct the Index object to describe this index */ + pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); + if( pIdx==0 ) return; + pLoop->u.btree.pIndex = pIdx; + pIdx->zName = "auto-index"; + pIdx->pTable = pTable; + n = 0; + idxCols = 0; + for(pTerm=pWC->a; pTermu.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS-1 ); + testcase( iCol==BMS ); + if( (idxCols & cMask)==0 ){ + Expr *pX = pTerm->pExpr; + idxCols |= cMask; + pIdx->aiColumn[n] = pTerm->u.leftColumn; + pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + pIdx->azColl[n] = ALWAYS(pColl) ? pColl->zName : "BINARY"; + n++; + } + } + } + assert( (u32)n==pLoop->u.btree.nEq ); + + /* Add additional columns needed to make the automatic index into + ** a covering index */ + for(i=0; iaiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; + } + } + if( pSrc->colUsed & MASKBIT(BMS-1) ){ + for(i=BMS-1; inCol; i++){ + pIdx->aiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; + } + } + assert( n==nKeyCol ); + pIdx->aiColumn[n] = -1; + pIdx->azColl[n] = "BINARY"; + + /* Create the automatic index */ + assert( pLevel->iIdxCur>=0 ); + pLevel->iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "for %s", pTable->zName)); + + /* Fill the automatic index with content */ + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + regRecord = sqlite3GetTempReg(pParse); + sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + sqlite3VdbeJumpHere(v, addrTop); + sqlite3ReleaseTempReg(pParse, regRecord); + + /* Jump here when skipping the initialization */ + sqlite3VdbeJumpHere(v, addrInit); +} +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Allocate and populate an sqlite3_index_info structure. It is the +** responsibility of the caller to eventually release the structure +** by passing the pointer returned by this function to sqlite3_free(). +*/ +static sqlite3_index_info *allocateIndexInfo( + Parse *pParse, + WhereClause *pWC, + struct SrcList_item *pSrc, + ExprList *pOrderBy +){ + int i, j; + int nTerm; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_orderby *pIdxOrderBy; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int nOrderBy; + sqlite3_index_info *pIdxInfo; + + /* Count the number of possible WHERE clause constraints referring + ** to this virtual table */ + for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + if( pTerm->leftCursor != pSrc->iCursor ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + nTerm++; + } + + /* If the ORDER BY clause contains only columns in the current + ** virtual table then allocate space for the aOrderBy part of + ** the sqlite3_index_info structure. + */ + nOrderBy = 0; + if( pOrderBy ){ + int n = pOrderBy->nExpr; + for(i=0; ia[i].pExpr; + if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; + } + if( i==n){ + nOrderBy = n; + } + } + + /* Allocate the sqlite3_index_info structure + */ + pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm + + sizeof(*pIdxOrderBy)*nOrderBy ); + if( pIdxInfo==0 ){ + sqlite3ErrorMsg(pParse, "out of memory"); + return 0; + } + + /* Initialize the structure. The sqlite3_index_info structure contains + ** many fields that are declared "const" to prevent xBestIndex from + ** changing them. We have to do some funky casting in order to + ** initialize those fields. + */ + pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; + pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; + pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; + *(int*)&pIdxInfo->nConstraint = nTerm; + *(int*)&pIdxInfo->nOrderBy = nOrderBy; + *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; + *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; + *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = + pUsage; + + for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + u8 op; + if( pTerm->leftCursor != pSrc->iCursor ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + pIdxCons[j].iColumn = pTerm->u.leftColumn; + pIdxCons[j].iTermOffset = i; + op = (u8)pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + pIdxCons[j].op = op; + /* The direct assignment in the previous line is possible only because + ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The + ** following asserts verify this fact. */ + assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); + assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); + assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); + assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); + assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); + assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); + assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + j++; + } + for(i=0; ia[i].pExpr; + pIdxOrderBy[i].iColumn = pExpr->iColumn; + pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; + } + + return pIdxInfo; +} + +/* +** The table object reference passed as the second argument to this function +** must represent a virtual table. This function invokes the xBestIndex() +** method of the virtual table with the sqlite3_index_info object that +** comes in as the 3rd argument to this function. +** +** If an error occurs, pParse is populated with an error message and a +** non-zero value is returned. Otherwise, 0 is returned and the output +** part of the sqlite3_index_info structure is left populated. +** +** Whether or not an error is returned, it is the responsibility of the +** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates +** that this is required. +*/ +static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ + sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; + int i; + int rc; + + TRACE_IDX_INPUTS(p); + rc = pVtab->pModule->xBestIndex(pVtab, p); + TRACE_IDX_OUTPUTS(p); + + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + pParse->db->mallocFailed = 1; + }else if( !pVtab->zErrMsg ){ + sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); + }else{ + sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); + } + } + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; + + for(i=0; inConstraint; i++){ + if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ + sqlite3ErrorMsg(pParse, + "table %s: xBestIndex returned an invalid plan", pTab->zName); + } + } + + return pParse->nErr; +} +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ + + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Estimate the location of a particular key among all keys in an +** index. Store the results in aStat as follows: +** +** aStat[0] Est. number of rows less than pVal +** aStat[1] Est. number of rows equal to pVal +** +** Return SQLITE_OK on success. +*/ +static void whereKeyStats( + Parse *pParse, /* Database connection */ + Index *pIdx, /* Index to consider domain of */ + UnpackedRecord *pRec, /* Vector of values to consider */ + int roundUp, /* Round up if true. Round down if false */ + tRowcnt *aStat /* OUT: stats written here */ +){ + IndexSample *aSample = pIdx->aSample; + int iCol; /* Index of required stats in anEq[] etc. */ + int iMin = 0; /* Smallest sample not yet tested */ + int i = pIdx->nSample; /* Smallest sample larger than or equal to pRec */ + int iTest; /* Next sample to test */ + int res; /* Result of comparison operation */ + +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( pParse ); +#endif + assert( pRec!=0 ); + iCol = pRec->nField - 1; + assert( pIdx->nSample>0 ); + assert( pRec->nField>0 && iColnSampleCol ); + do{ + iTest = (iMin+i)/2; + res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec, 0); + if( res<0 ){ + iMin = iTest+1; + }else{ + i = iTest; + } + }while( res && iMinnSample ); + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0) + || pParse->db->mallocFailed ); + }else{ + /* Otherwise, pRec must be smaller than sample $i and larger than + ** sample ($i-1). */ + assert( i==pIdx->nSample + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0)>0 + || pParse->db->mallocFailed ); + assert( i==0 + || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec, 0)<0 + || pParse->db->mallocFailed ); + } +#endif /* ifdef SQLITE_DEBUG */ + + /* At this point, aSample[i] is the first sample that is greater than + ** or equal to pVal. Or if i==pIdx->nSample, then all samples are less + ** than pVal. If aSample[i]==pVal, then res==0. + */ + if( res==0 ){ + aStat[0] = aSample[i].anLt[iCol]; + aStat[1] = aSample[i].anEq[iCol]; + }else{ + tRowcnt iLower, iUpper, iGap; + if( i==0 ){ + iLower = 0; + iUpper = aSample[0].anLt[iCol]; + }else{ + i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; + iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; + } + aStat[1] = pIdx->aAvgEq[iCol]; + if( iLower>=iUpper ){ + iGap = 0; + }else{ + iGap = iUpper - iLower; + } + if( roundUp ){ + iGap = (iGap*2)/3; + }else{ + iGap = iGap/3; + } + aStat[0] = iLower + iGap; + } +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** This function is called to estimate the number of rows visited by a +** range-scan on a skip-scan index. For example: +** +** CREATE INDEX i1 ON t1(a, b, c); +** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; +** +** Value pLoop->nOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate +** by some factor to account for the (c BETWEEN ? AND ?) expression based +** on the stat4 data for the index. this scan will be peformed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with +** by the caller. +** +** It does this by scanning through all stat4 samples, comparing values +** extracted from pLower and pUpper with the corresponding column in each +** sample. If L and U are the number of samples found to be less than or +** equal to the values extracted from pLower and pUpper respectively, and +** N is the total number of samples, the pLoop->nOut value is adjusted +** as follows: +** +** nOut = nOut * ( min(U - L, 1) / N ) +** +** If pLower is NULL, or a value cannot be extracted from the term, L is +** set to zero. If pUpper is NULL, or a value cannot be extracted from it, +** U is set to N. +** +** Normally, this function sets *pbDone to 1 before returning. However, +** if no value can be extracted from either pLower or pUpper (and so the +** estimate of the number of rows delivered remains unchanged), *pbDone +** is left as is. +** +** If an error occurs, an SQLite error code is returned. Otherwise, +** SQLITE_OK. +*/ +static int whereRangeSkipScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop, /* Update the .nOut value of this loop */ + int *pbDone /* Set to true if at least one expr. value extracted */ +){ + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + sqlite3 *db = pParse->db; + int nLower = -1; + int nUpper = p->nSample+1; + int rc = SQLITE_OK; + int iCol = p->aiColumn[nEq]; + u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; + CollSeq *pColl; + + sqlite3_value *p1 = 0; /* Value extracted from pLower */ + sqlite3_value *p2 = 0; /* Value extracted from pUpper */ + sqlite3_value *pVal = 0; /* Value extracted from record */ + + pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); + if( pLower ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + nLower = 0; + } + if( pUpper && rc==SQLITE_OK ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + nUpper = p2 ? 0 : p->nSample; + } + + if( p1 || p2 ){ + int i; + int nDiff; + for(i=0; rc==SQLITE_OK && inSample; i++){ + rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); + if( rc==SQLITE_OK && p1 ){ + int res = sqlite3MemCompare(p1, pVal, pColl); + if( res>=0 ) nLower++; + } + if( rc==SQLITE_OK && p2 ){ + int res = sqlite3MemCompare(p2, pVal, pColl); + if( res>=0 ) nUpper++; + } + } + nDiff = (nUpper - nLower); + if( nDiff<=0 ) nDiff = 1; + + /* If there is both an upper and lower bound specified, and the + ** comparisons indicate that they are close together, use the fallback + ** method (assume that the scan visits 1/64 of the rows) for estimating + ** the number of rows visited. Otherwise, estimate the number of rows + ** using the method described in the header comment for this function. */ + if( nDiff!=1 || pUpper==0 || pLower==0 ){ + int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + pLoop->nOut -= nAdjust; + *pbDone = 1; + WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + nLower, nUpper, nAdjust*-1, pLoop->nOut)); + } + + }else{ + assert( *pbDone==0 ); + } + + sqlite3ValueFree(p1); + sqlite3ValueFree(p2); + sqlite3ValueFree(pVal); + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** This function is used to estimate the number of rows that will be visited +** by scanning an index for a range of values. The range may have an upper +** bound, a lower bound, or both. The WHERE clause terms that set the upper +** and lower bounds are represented by pLower and pUpper respectively. For +** example, assuming that index p is on t1(a): +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** |_____| |_____| +** | | +** pLower pUpper +** +** If either of the upper or lower bound is not present, then NULL is passed in +** place of the corresponding WhereTerm. +** +** The value in (pBuilder->pNew->u.btree.nEq) is the index of the index +** column subject to the range constraint. Or, equivalently, the number of +** equality constraints optimized by the proposed index scan. For example, +** assuming index p is on t1(a, b), and the SQL query is: +** +** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... +** +** then nEq is set to 1 (as the range restricted column, b, is the second +** left-most column of the index). Or, if the query is: +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** +** then nEq is set to 0. +** +** When this function is called, *pnOut is set to the sqlite3LogEst() of the +** number of rows that the index scan is expected to visit without +** considering the range constraints. If nEq is 0, this is the number of +** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) +** to account for the range contraints pLower and pUpper. +** +** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be +** used, a single range inequality reduces the search space by a factor of 4. +** and a pair of constraints (x>? AND x123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ +){ + int rc = SQLITE_OK; + int nOut = pLoop->nOut; + LogEst nNew; + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + + if( p->nSample>0 + && nEqnSampleCol + && OptimizationEnabled(pParse->db, SQLITE_Stat3) + ){ + if( nEq==pBuilder->nRecValid ){ + UnpackedRecord *pRec = pBuilder->pRec; + tRowcnt a[2]; + u8 aff; + + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + */ + tRowcnt iLower; + tRowcnt iUpper; + + if( nEq==p->nKeyCol ){ + aff = SQLITE_AFF_INTEGER; + }else{ + aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; + } + /* Determine iLower and iUpper using ($P) only. */ + if( nEq==0 ){ + iLower = 0; + iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); + }else{ + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ + whereKeyStats(pParse, p, pRec, 0, a); + iLower = a[0]; + iUpper = a[0] + a[1]; + } + + /* If possible, improve on the iLower estimate using ($P:$L). */ + if( pLower ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pLower->pExpr->pRight; + assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; + nOut--; + } + } + + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pUpper->pExpr->pRight; + assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); + if( iNewpRec = pRec; + if( rc==SQLITE_OK ){ + if( iUpper>iLower ){ + nNew = sqlite3LogEst(iUpper - iLower); + }else{ + nNew = 10; assert( 10==sqlite3LogEst(2) ); + } + if( nNewnOut = (LogEst)nOut; + WHERETRACE(0x10, ("range scan regions: %u..%u est=%d\n", + (u32)iLower, (u32)iUpper, nOut)); + return SQLITE_OK; + } + }else{ + int bDone = 0; + rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone); + if( bDone ) return rc; + } + } +#else + UNUSED_PARAMETER(pParse); + UNUSED_PARAMETER(pBuilder); +#endif + assert( pLower || pUpper ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + + /* TUNING: If there is both an upper and lower limit, assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pUpper ) nNew -= 20; + + nOut -= (pLower!=0) + (pUpper!=0); + if( nNew<10 ) nNew = 10; + if( nNewnOut = (LogEst)nOut; + return rc; +} + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Estimate the number of rows that will be returned based on +** an equality constraint x=VALUE and where that VALUE occurs in +** the histogram data. This only works when x is the left-most +** column of an index and sqlite_stat3 histogram data is available +** for that index. When pExpr==NULL that means the constraint is +** "x IS NULL" instead of "x=VALUE". +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereEqualScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ + tRowcnt *pnRow /* Write the revised row estimate here */ +){ + Index *p = pBuilder->pNew->u.btree.pIndex; + int nEq = pBuilder->pNew->u.btree.nEq; + UnpackedRecord *pRec = pBuilder->pRec; + u8 aff; /* Column affinity */ + int rc; /* Subfunction return code */ + tRowcnt a[2]; /* Statistics */ + int bOk; + + assert( nEq>=1 ); + assert( nEq<=p->nColumn ); + assert( p->aSample!=0 ); + assert( p->nSample>0 ); + assert( pBuilder->nRecValidnRecValid<(nEq-1) ){ + return SQLITE_NOTFOUND; + } + + /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() + ** below would return the same value. */ + if( nEq>=p->nColumn ){ + *pnRow = 1; + return SQLITE_OK; + } + + aff = p->pTable->aCol[p->aiColumn[nEq-1]].affinity; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); + pBuilder->pRec = pRec; + if( rc!=SQLITE_OK ) return rc; + if( bOk==0 ) return SQLITE_NOTFOUND; + pBuilder->nRecValid = nEq; + + whereKeyStats(pParse, p, pRec, 0, a); + WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1])); + *pnRow = a[1]; + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Estimate the number of rows that will be returned based on +** an IN constraint where the right-hand side of the IN operator +** is a list of values. Example: +** +** WHERE x IN (1,2,3,4) +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereInScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ + tRowcnt *pnRow /* Write the revised row estimate here */ +){ + Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); + int nRecValid = pBuilder->nRecValid; + int rc = SQLITE_OK; /* Subfunction return code */ + tRowcnt nEst; /* Number of rows for a single term */ + tRowcnt nRowEst = 0; /* New estimate of the number of rows */ + int i; /* Loop counter */ + + assert( p->aSample!=0 ); + for(i=0; rc==SQLITE_OK && inExpr; i++){ + nEst = nRow0; + rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); + nRowEst += nEst; + pBuilder->nRecValid = nRecValid; + } + + if( rc==SQLITE_OK ){ + if( nRowEst > nRow0 ) nRowEst = nRow0; + *pnRow = nRowEst; + WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); + } + assert( pBuilder->nRecValid==nRecValid ); + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** Disable a term in the WHERE clause. Except, do not disable the term +** if it controls a LEFT OUTER JOIN and it did not originate in the ON +** or USING clause of that join. +** +** Consider the term t2.z='ok' in the following queries: +** +** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' +** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' +** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' +** +** The t2.z='ok' is disabled in the in (2) because it originates +** in the ON clause. The term is disabled in (3) because it is not part +** of a LEFT OUTER JOIN. In (1), the term is not disabled. +** +** Disabling a term causes that term to not be tested in the inner loop +** of the join. Disabling is an optimization. When terms are satisfied +** by indices, we disable them to prevent redundant tests in the inner +** loop. We would get the correct results if nothing were ever disabled, +** but joins might run a little slower. The trick is to disable as much +** as we can without disabling too much. If we disabled in (1), we'd get +** the wrong answer. See ticket #813. +*/ +static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ + if( pTerm + && (pTerm->wtFlags & TERM_CODED)==0 + && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pLevel->notReady & pTerm->prereqAll)==0 + ){ + pTerm->wtFlags |= TERM_CODED; + if( pTerm->iParent>=0 ){ + WhereTerm *pOther = &pTerm->pWC->a[pTerm->iParent]; + if( (--pOther->nChild)==0 ){ + disableTerm(pLevel, pOther); + } + } + } +} + +/* +** Code an OP_Affinity opcode to apply the column affinity string zAff +** to the n registers starting at base. +** +** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the +** beginning and end of zAff are ignored. If all entries in zAff are +** SQLITE_AFF_NONE, then no code gets generated. +** +** This routine makes its own copy of zAff so that the caller is free +** to modify zAff after this routine returns. +*/ +static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ + Vdbe *v = pParse->pVdbe; + if( zAff==0 ){ + assert( pParse->db->mallocFailed ); + return; + } + assert( v!=0 ); + + /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning + ** and end of the affinity string. + */ + while( n>0 && zAff[0]==SQLITE_AFF_NONE ){ + n--; + base++; + zAff++; + } + while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){ + n--; + } + + /* Code the OP_Affinity opcode if there is anything left to do. */ + if( n>0 ){ + sqlite3VdbeAddOp2(v, OP_Affinity, base, n); + sqlite3VdbeChangeP4(v, -1, zAff, n); + sqlite3ExprCacheAffinityChange(pParse, base, n); + } +} + + +/* +** Generate code for a single equality term of the WHERE clause. An equality +** term can be either X=expr or X IN (...). pTerm is the term to be +** coded. +** +** The current value for the constraint is left in register iReg. +** +** For a constraint of the form X=expr, the expression is evaluated and its +** result is left on the stack. For constraints of the form X IN (...) +** this routine sets up a loop that will iterate over all values of X. +*/ +static int codeEqualityTerm( + Parse *pParse, /* The parsing context */ + WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ + int iTarget /* Attempt to leave results in this register */ +){ + Expr *pX = pTerm->pExpr; + Vdbe *v = pParse->pVdbe; + int iReg; /* Register holding results */ + + assert( iTarget>0 ); + if( pX->op==TK_EQ ){ + iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); + }else if( pX->op==TK_ISNULL ){ + iReg = iTarget; + sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); +#ifndef SQLITE_OMIT_SUBQUERY + }else{ + int eType; + int iTab; + struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } + assert( pX->op==TK_IN ); + iReg = iTarget; + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + iTab = pX->iTable; + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + } + pLevel->u.in.nIn++; + pLevel->u.in.aInLoop = + sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + pIn += pLevel->u.in.nIn - 1; + pIn->iCur = iTab; + if( eType==IN_INDEX_ROWID ){ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); + }else{ + pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); + } + pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; + sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); + }else{ + pLevel->u.in.nIn = 0; + } +#endif + } + disableTerm(pLevel, pTerm); + return iReg; +} + +/* +** Generate code that will evaluate all == and IN constraints for an +** index scan. +** +** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). +** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 +** The index has as many as three equality constraints, but in this +** example, the third "c" value is an inequality. So only two +** constraints are coded. This routine will generate code to evaluate +** a==5 and b IN (1,2,3). The current values for a and b will be stored +** in consecutive registers and the index of the first register is returned. +** +** In the example above nEq==2. But this subroutine works for any value +** of nEq including 0. If nEq==0, this routine is nearly a no-op. +** The only thing it does is allocate the pLevel->iMem memory cell and +** compute the affinity string. +** +** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints +** are == or IN and are covered by the nEq. nExtraReg is 1 if there is +** an inequality constraint (such as the "c>=5 AND c<10" in the example) that +** occurs after the nEq quality constraints. +** +** This routine allocates a range of nEq+nExtraReg memory cells and returns +** the index of the first memory cell in that range. The code that +** calls this routine will use that memory range to store keys for +** start and termination conditions of the loop. +** key value of the loop. If one or more IN operators appear, then +** this routine allocates an additional nEq memory cells for internal +** use. +** +** Before returning, *pzAff is set to point to a buffer containing a +** copy of the column affinity string of the index allocated using +** sqlite3DbMalloc(). Except, entries in the copy of the string associated +** with equality constraints that use NONE affinity are set to +** SQLITE_AFF_NONE. This is to deal with SQL such as the following: +** +** CREATE TABLE t1(a TEXT PRIMARY KEY, b); +** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; +** +** In the example above, the index on t1(a) has TEXT affinity. But since +** the right hand side of the equality constraint (t2.b) has NONE affinity, +** no conversion should be attempted before using a t2.b value as part of +** a key to search the index. Hence the first byte in the returned affinity +** string in this example would be set to SQLITE_AFF_NONE. +*/ +static int codeAllEqualityTerms( + Parse *pParse, /* Parsing context */ + WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ + int bRev, /* Reverse the order of IN operators */ + int nExtraReg, /* Number of extra registers to allocate */ + char **pzAff /* OUT: Set to point to affinity string */ +){ + u16 nEq; /* The number of == or IN constraints to code */ + u16 nSkip; /* Number of left-most columns to skip */ + Vdbe *v = pParse->pVdbe; /* The vm under construction */ + Index *pIdx; /* The index being used for this loop */ + WhereTerm *pTerm; /* A single constraint term */ + WhereLoop *pLoop; /* The WhereLoop object */ + int j; /* Loop counter */ + int regBase; /* Base register */ + int nReg; /* Number of registers to allocate */ + char *zAff; /* Affinity string to return */ + + /* This module is only called on query plans that use an index. */ + pLoop = pLevel->pWLoop; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + nEq = pLoop->u.btree.nEq; + nSkip = pLoop->u.btree.nSkip; + pIdx = pLoop->u.btree.pIndex; + assert( pIdx!=0 ); + + /* Figure out how many memory cells we will need then allocate them. + */ + regBase = pParse->nMem + 1; + nReg = pLoop->u.btree.nEq + nExtraReg; + pParse->nMem += nReg; + + zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); + if( !zAff ){ + pParse->db->mallocFailed = 1; + } + + if( nSkip ){ + int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); + j = sqlite3VdbeAddOp0(v, OP_Goto); + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), + iIdxCur, 0, regBase, nSkip); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + sqlite3VdbeJumpHere(v, j); + for(j=0; jaiColumn[j]>=0 ); + VdbeComment((v, "%s", pIdx->pTable->aCol[pIdx->aiColumn[j]].zName)); + } + } + + /* Evaluate the equality constraints + */ + assert( zAff==0 || (int)strlen(zAff)>=nEq ); + for(j=nSkip; jaLTerm[j]; + assert( pTerm!=0 ); + /* The following testcase is true for indices with redundant columns. + ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ + testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); + if( r1!=regBase+j ){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, regBase); + regBase = r1; + }else{ + sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); + } + } + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IN ); + if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ + Expr *pRight = pTerm->pExpr->pRight; + if( sqlite3ExprCanBeNull(pRight) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); + VdbeCoverage(v); + } + if( zAff ){ + if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ + zAff[j] = SQLITE_AFF_NONE; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ + zAff[j] = SQLITE_AFF_NONE; + } + } + } + } + *pzAff = zAff; + return regBase; +} + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** This routine is a helper for explainIndexRange() below +** +** pStr holds the text of an expression that we are building up one term +** at a time. This routine adds a new term to the end of the expression. +** Terms are separated by AND so add the "AND" text for second and subsequent +** terms only. +*/ +static void explainAppendTerm( + StrAccum *pStr, /* The text expression being built */ + int iTerm, /* Index of this term. First is zero */ + const char *zColumn, /* Name of the column */ + const char *zOp /* Name of the operator */ +){ + if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3StrAccumAppendAll(pStr, zColumn); + sqlite3StrAccumAppend(pStr, zOp, 1); + sqlite3StrAccumAppend(pStr, "?", 1); +} + +/* +** Argument pLevel describes a strategy for scanning table pTab. This +** function returns a pointer to a string buffer containing a description +** of the subset of table rows scanned by the strategy in the form of an +** SQL expression. Or, if all rows are scanned, NULL is returned. +** +** For example, if the query: +** +** SELECT * FROM t1 WHERE a=1 AND b>2; +** +** is run and there is an index on (a, b), then this function returns a +** string similar to: +** +** "a=? AND b>?" +** +** The returned pointer points to memory obtained from sqlite3DbMalloc(). +** It is the responsibility of the caller to free the buffer when it is +** no longer required. +*/ +static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ + Index *pIndex = pLoop->u.btree.pIndex; + u16 nEq = pLoop->u.btree.nEq; + u16 nSkip = pLoop->u.btree.nSkip; + int i, j; + Column *aCol = pTab->aCol; + i16 *aiColumn = pIndex->aiColumn; + StrAccum txt; + + if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ + return 0; + } + sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); + txt.db = db; + sqlite3StrAccumAppend(&txt, " (", 2); + for(i=0; i=nSkip ){ + explainAppendTerm(&txt, i, z, "="); + }else{ + if( i ) sqlite3StrAccumAppend(&txt, " AND ", 5); + sqlite3StrAccumAppend(&txt, "ANY(", 4); + sqlite3StrAccumAppendAll(&txt, z); + sqlite3StrAccumAppend(&txt, ")", 1); + } + } + + j = i; + if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(&txt, i++, z, ">"); + } + if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(&txt, i, z, "<"); + } + sqlite3StrAccumAppend(&txt, ")", 1); + return sqlite3StrAccumFinish(&txt); +} + +/* +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single +** record is added to the output to describe the table scan strategy in +** pLevel. +*/ +static void explainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +){ +#ifndef SQLITE_DEBUG + if( pParse->explain==2 ) +#endif + { + struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zMsg; /* Text to add to EQP output */ + int iId = pParse->iSelectId; /* Select id (left-most output column) */ + int isSearch; /* True for a SEARCH. False for SCAN. */ + WhereLoop *pLoop; /* The controlling WhereLoop object */ + u32 flags; /* Flags that describe this loop */ + + pLoop = pLevel->pWLoop; + flags = pLoop->wsFlags; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + + isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + + zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); + if( pItem->pSelect ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); + }else{ + zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName); + } + + if( pItem->zAlias ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); + } + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 + && ALWAYS(pLoop->u.btree.pIndex!=0) + ){ + const char *zFmt; + Index *pIdx = pLoop->u.btree.pIndex; + char *zWhere = explainIndexRange(db, pLoop, pItem->pTab); + assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); + if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + zFmt = zWhere ? "%s USING PRIMARY KEY%.0s%s" : "%s%.0s%s"; + }else if( flags & WHERE_AUTO_INDEX ){ + zFmt = "%s USING AUTOMATIC COVERING INDEX%.0s%s"; + }else if( flags & WHERE_IDX_ONLY ){ + zFmt = "%s USING COVERING INDEX %s%s"; + }else{ + zFmt = "%s USING INDEX %s%s"; + } + zMsg = sqlite3MAppendf(db, zMsg, zFmt, zMsg, pIdx->zName, zWhere); + sqlite3DbFree(db, zWhere); + }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); + + if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid?)", zMsg); + }else if( ALWAYS(flags&WHERE_TOP_LIMIT) ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowidu.vtab.idxNum, pLoop->u.vtab.idxStr); + } +#endif + zMsg = sqlite3MAppendf(db, zMsg, "%s", zMsg); + sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); + } +} +#else +# define explainOneScan(u,v,w,x,y,z) +#endif /* SQLITE_OMIT_EXPLAIN */ + + +/* +** Generate code for the start of the iLevel-th loop in the WHERE clause +** implementation described by pWInfo. +*/ +static Bitmask codeOneLoopStart( + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + Bitmask notReady /* Which tables are currently available */ +){ + int j, k; /* Loop counters */ + int iCur; /* The VDBE cursor for the table */ + int addrNxt; /* Where to jump to continue with the next IN case */ + int omitTable; /* True if we use the index only */ + int bRev; /* True if we need to scan in reverse order */ + WhereLevel *pLevel; /* The where level to be coded */ + WhereLoop *pLoop; /* The WhereLoop object being coded */ + WhereClause *pWC; /* Decomposition of the entire WHERE clause */ + WhereTerm *pTerm; /* A WHERE clause term */ + Parse *pParse; /* Parsing context */ + sqlite3 *db; /* Database connection */ + Vdbe *v; /* The prepared stmt under constructions */ + struct SrcList_item *pTabItem; /* FROM clause term being coded */ + int addrBrk; /* Jump here to break out of the loop */ + int addrCont; /* Jump here to continue with next cycle */ + int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ + int iReleaseReg = 0; /* Temp register to free before returning */ + + pParse = pWInfo->pParse; + v = pParse->pVdbe; + pWC = &pWInfo->sWC; + db = pParse->db; + pLevel = &pWInfo->a[iLevel]; + pLoop = pLevel->pWLoop; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + iCur = pTabItem->iCursor; + pLevel->notReady = notReady & ~getMask(&pWInfo->sMaskSet, iCur); + bRev = (pWInfo->revMask>>iLevel)&1; + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); + + /* Create labels for the "break" and "continue" instructions + ** for the current loop. Jump to addrBrk to break out of a loop. + ** Jump to cont to go immediately to the next iteration of the + ** loop. + ** + ** When there is an IN operator, we also have a "addrNxt" label that + ** means to continue with the next IN value combination. When + ** there are no IN operators in the constraints, the "addrNxt" label + ** is the same as "addrBrk". + */ + addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + + /* If this is the right table of a LEFT OUTER JOIN, allocate and + ** initialize a memory cell that records if this table matches any + ** row of the left table of the join. + */ + if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ + pLevel->iLeftJoin = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); + VdbeComment((v, "init LEFT JOIN no-match flag")); + } + + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + pLevel->op = OP_Goto; + }else + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 1: The table is a virtual-table. Use the VFilter and VNext + ** to access the data. + */ + int iReg; /* P3 Value for OP_VFilter */ + int addrNotFound; + int nConstraint = pLoop->nLTerm; + + sqlite3ExprCachePush(pParse); + iReg = sqlite3GetTempRange(pParse, nConstraint+2); + addrNotFound = pLevel->addrBrk; + for(j=0; jaLTerm[j]; + if( pTerm==0 ) continue; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } + } + sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); + sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, + pLoop->u.vtab.idxStr, + pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + VdbeCoverage(v); + pLoop->u.vtab.needFree = 0; + for(j=0; ju.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pLoop->aLTerm[j]); + } + } + pLevel->op = OP_VNext; + pLevel->p1 = iCur; + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); + sqlite3ExprCachePop(pParse); + }else +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 + ){ + /* Case 2: We can directly reference a single row using an + ** equality comparison against the ROWID field. Or + ** we reference multiple rows using a "rowid IN (...)" + ** construct. + */ + assert( pLoop->u.btree.nEq==1 ); + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + iReleaseReg = ++pParse->nMem; + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); + addrNxt = pLevel->addrNxt; + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + VdbeCoverage(v); + sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + VdbeComment((v, "pk")); + pLevel->op = OP_Noop; + }else if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 + ){ + /* Case 3: We have an inequality comparison against the ROWID field. + */ + int testOp = OP_Noop; + int start; + int memEndValue = 0; + WhereTerm *pStart, *pEnd; + + assert( omitTable==0 ); + j = 0; + pStart = pEnd = 0; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; + assert( pStart!=0 || pEnd!=0 ); + if( bRev ){ + pTerm = pStart; + pStart = pEnd; + pEnd = pTerm; + } + if( pStart ){ + Expr *pX; /* The expression that defines the start bound */ + int r1, rTemp; /* Registers for holding the start boundary */ + + /* The following constant maps TK_xx codes into corresponding + ** seek opcodes. It depends on a particular ordering of TK_xx + */ + const u8 aMoveOp[] = { + /* TK_GT */ OP_SeekGT, + /* TK_LE */ OP_SeekLE, + /* TK_LT */ OP_SeekLT, + /* TK_GE */ OP_SeekGE + }; + assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ + assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ + assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + + assert( (pStart->wtFlags & TERM_VNULL)==0 ); + testcase( pStart->wtFlags & TERM_VIRTUAL ); + pX = pStart->pExpr; + assert( pX!=0 ); + testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ + r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); + sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); + VdbeComment((v, "pk")); + VdbeCoverageIf(v, pX->op==TK_GT); + VdbeCoverageIf(v, pX->op==TK_LE); + VdbeCoverageIf(v, pX->op==TK_LT); + VdbeCoverageIf(v, pX->op==TK_GE); + sqlite3ExprCacheAffinityChange(pParse, r1, 1); + sqlite3ReleaseTempReg(pParse, rTemp); + disableTerm(pLevel, pStart); + }else{ + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + } + if( pEnd ){ + Expr *pX; + pX = pEnd->pExpr; + assert( pX!=0 ); + assert( (pEnd->wtFlags & TERM_VNULL)==0 ); + testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ + testcase( pEnd->wtFlags & TERM_VIRTUAL ); + memEndValue = ++pParse->nMem; + sqlite3ExprCode(pParse, pX->pRight, memEndValue); + if( pX->op==TK_LT || pX->op==TK_GT ){ + testOp = bRev ? OP_Le : OP_Ge; + }else{ + testOp = bRev ? OP_Lt : OP_Gt; + } + disableTerm(pLevel, pEnd); + } + start = sqlite3VdbeCurrentAddr(v); + pLevel->op = bRev ? OP_Prev : OP_Next; + pLevel->p1 = iCur; + pLevel->p2 = start; + assert( pLevel->p5==0 ); + if( testOp!=OP_Noop ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + VdbeCoverageIf(v, testOp==OP_Le); + VdbeCoverageIf(v, testOp==OP_Lt); + VdbeCoverageIf(v, testOp==OP_Ge); + VdbeCoverageIf(v, testOp==OP_Gt); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); + } + }else if( pLoop->wsFlags & WHERE_INDEXED ){ + /* Case 4: A scan using an index. + ** + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==" and "IN" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: + ** + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 + ** + ** The z<10 term of the following cannot be used, only + ** the x=5 term: + ** + ** x=5 AND z<10 + ** + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. + ** + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. + */ + static const u8 aStartOp[] = { + 0, + 0, + OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ + OP_Last, /* 3: (!start_constraints && startEq && bRev) */ + OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ + OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ + OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ + OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ + }; + static const u8 aEndOp[] = { + OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ + OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ + OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ + OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ + }; + u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + int regBase; /* Base register holding constraint values */ + WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ + WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ + int startEq; /* True if range start uses ==, >= or <= */ + int endEq; /* True if range end uses ==, >= or <= */ + int start_constraints; /* Start of range is constrained */ + int nConstraint; /* Number of constraint terms */ + Index *pIdx; /* The index we will be using */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ + char *zStartAff; /* Affinity for start of range constraint */ + char cEndAff = 0; /* Affinity for end of range constraint */ + u8 bSeekPastNull = 0; /* True to seek past initial nulls */ + u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ + + pIdx = pLoop->u.btree.pIndex; + iIdxCur = pLevel->iIdxCur; + assert( nEq>=pLoop->u.btree.nSkip ); + + /* If this loop satisfies a sort order (pOrderBy) request that + ** was passed to this function to implement a "SELECT min(x) ..." + ** query, then the caller will only allow the loop to run for + ** a single iteration. This means that the first row returned + ** should not have a NULL value stored in 'x'. If column 'x' is + ** the first one after the nEq equality constraints in the index, + ** this requires some special handling. + */ + assert( pWInfo->pOrderBy==0 + || pWInfo->pOrderBy->nExpr==1 + || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); + if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 + && pWInfo->nOBSat>0 + && (pIdx->nKeyCol>nEq) + ){ + assert( pLoop->u.btree.nSkip==0 ); + bSeekPastNull = 1; + nExtraReg = 1; + } + + /* Find any inequality constraint terms for the start and end + ** of the range. + */ + j = nEq; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = pLoop->aLTerm[j++]; + nExtraReg = 1; + } + if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = pLoop->aLTerm[j++]; + nExtraReg = 1; + if( pRangeStart==0 + && (j = pIdx->aiColumn[nEq])>=0 + && pIdx->pTable->aCol[j].notNull==0 + ){ + bSeekPastNull = 1; + } + } + assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); + + /* Generate code to evaluate all constraint terms using == or IN + ** and store the values of those terms in an array of registers + ** starting at regBase. + */ + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); + assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); + if( zStartAff ) cEndAff = zStartAff[nEq]; + addrNxt = pLevel->addrNxt; + + /* If we are doing a reverse order scan on an ascending index, or + ** a forward order scan on a descending index, interchange the + ** start and end terms (pRangeStart and pRangeEnd). + */ + if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) + || (bRev && pIdx->nKeyCol==nEq) + ){ + SWAP(WhereTerm *, pRangeEnd, pRangeStart); + SWAP(u8, bSeekPastNull, bStopAtNull); + } + + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); + startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); + endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); + start_constraints = pRangeStart || nEq>0; + + /* Seek the index cursor to the start of the range. */ + nConstraint = nEq; + if( pRangeStart ){ + Expr *pRight = pRangeStart->pExpr->pRight; + sqlite3ExprCode(pParse, pRight, regBase+nEq); + if( (pRangeStart->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); + } + if( zStartAff ){ + if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){ + /* Since the comparison is to be performed with no conversions + ** applied to the operands, set the affinity to apply to pRight to + ** SQLITE_AFF_NONE. */ + zStartAff[nEq] = SQLITE_AFF_NONE; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ + zStartAff[nEq] = SQLITE_AFF_NONE; + } + } + nConstraint++; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); + }else if( bSeekPastNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + nConstraint++; + startEq = 0; + start_constraints = 1; + } + codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; + assert( op!=0 ); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + + /* Load the value for the inequality constraint at the end of the + ** range (if any). + */ + nConstraint = nEq; + if( pRangeEnd ){ + Expr *pRight = pRangeEnd->pExpr->pRight; + sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); + sqlite3ExprCode(pParse, pRight, regBase+nEq); + if( (pRangeEnd->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); + } + if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_NONE + && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) + ){ + codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff); + } + nConstraint++; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + }else if( bStopAtNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + nConstraint++; + } + sqlite3DbFree(db, zStartAff); + + /* Top of the loop body */ + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + + /* Check if the index cursor is past the end of the range. */ + if( nConstraint ){ + op = aEndOp[bRev*2 + endEq]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); + } + + /* Seek the table cursor, if required */ + disableTerm(pLevel, pRangeStart); + disableTerm(pLevel, pRangeEnd); + if( omitTable ){ + /* pIdx is a covering index. No need to access the main table. */ + }else if( HasRowid(pIdx->pTable) ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ + }else if( iCur!=iIdxCur ){ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); + } + sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, + iRowidReg, pPk->nKeyCol); VdbeCoverage(v); + } + + /* Record the instruction used to terminate the loop. Disable + ** WHERE clause terms made redundant by the index range scan. + */ + if( pLoop->wsFlags & WHERE_ONEROW ){ + pLevel->op = OP_Noop; + }else if( bRev ){ + pLevel->op = OP_Prev; + }else{ + pLevel->op = OP_Next; + } + pLevel->p1 = iIdxCur; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; + if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } + }else + +#ifndef SQLITE_OMIT_OR_OPTIMIZATION + if( pLoop->wsFlags & WHERE_MULTI_OR ){ + /* Case 5: Two or more separately indexed terms connected by OR + ** + ** Example: + ** + ** CREATE TABLE t1(a,b,c,d); + ** CREATE INDEX i1 ON t1(a); + ** CREATE INDEX i2 ON t1(b); + ** CREATE INDEX i3 ON t1(c); + ** + ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) + ** + ** In the example, there are three indexed terms connected by OR. + ** The top of the loop looks like this: + ** + ** Null 1 # Zero the rowset in reg 1 + ** + ** Then, for each indexed term, the following. The arguments to + ** RowSetTest are such that the rowid of the current row is inserted + ** into the RowSet. If it is already present, control skips the + ** Gosub opcode and jumps straight to the code generated by WhereEnd(). + ** + ** sqlite3WhereBegin() + ** RowSetTest # Insert rowid into rowset + ** Gosub 2 A + ** sqlite3WhereEnd() + ** + ** Following the above, code to terminate the loop. Label A, the target + ** of the Gosub above, jumps to the instruction right after the Goto. + ** + ** Null 1 # Zero the rowset in reg 1 + ** Goto B # The loop is finished. + ** + ** A: # Return data, whatever. + ** + ** Return 2 # Jump back to the Gosub + ** + ** B: + ** + ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then + ** use an ephermeral index instead of a RowSet to record the primary + ** keys of the rows we have already seen. + ** + */ + WhereClause *pOrWc; /* The OR-clause broken out into subterms */ + SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + Index *pCov = 0; /* Potential covering index (or NULL) */ + int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ + + int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ + int regRowset = 0; /* Register for RowSet object */ + int regRowid = 0; /* Register holding rowid */ + int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ + int iRetInit; /* Address of regReturn init */ + int untestedTerms = 0; /* Some terms not completely tested */ + int ii; /* Loop counter */ + u16 wctrlFlags; /* Flags for sub-WHERE clause */ + Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ + Table *pTab = pTabItem->pTab; + + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->eOperator & WO_OR ); + assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); + pOrWc = &pTerm->u.pOrInfo->wc; + pLevel->op = OP_Return; + pLevel->p1 = regReturn; + + /* Set up a new SrcList in pOrTab containing the table being scanned + ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. + ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). + */ + if( pWInfo->nLevel>1 ){ + int nNotReady; /* The number of notReady tables */ + struct SrcList_item *origSrc; /* Original list of tables */ + nNotReady = pWInfo->nLevel - iLevel - 1; + pOrTab = sqlite3StackAllocRaw(db, + sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + if( pOrTab==0 ) return notReady; + pOrTab->nAlloc = (u8)(nNotReady + 1); + pOrTab->nSrc = pOrTab->nAlloc; + memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); + origSrc = pWInfo->pTabList->a; + for(k=1; k<=nNotReady; k++){ + memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); + } + }else{ + pOrTab = pWInfo->pTabList; + } + + /* Initialize the rowset register to contain NULL. An SQL NULL is + ** equivalent to an empty rowset. Or, create an ephermeral index + ** capable of holding primary keys in the case of a WITHOUT ROWID. + ** + ** Also initialize regReturn to contain the address of the instruction + ** immediately following the OP_Return at the bottom of the loop. This + ** is required in a few obscure LEFT JOIN cases where control jumps + ** over the top of the loop into the body of it. In this case the + ** correct response for the end-of-loop code (the OP_Return) is to + ** fall through to the next instruction, just as an OP_Next does if + ** called on an uninitialized cursor. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( HasRowid(pTab) ){ + regRowset = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + regRowset = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + regRowid = ++pParse->nMem; + } + iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); + + /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y + ** Then for every term xN, evaluate as the subexpression: xN AND z + ** That way, terms in y that are factored into the disjunction will + ** be picked up by the recursive calls to sqlite3WhereBegin() below. + ** + ** Actually, each subexpression is converted to "xN AND w" where w is + ** the "interesting" terms of z - terms that did not originate in the + ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 + */ + if( pWC->nTerm>1 ){ + int iTerm; + for(iTerm=0; iTermnTerm; iTerm++){ + Expr *pExpr = pWC->a[iTerm].pExpr; + if( &pWC->a[iTerm] == pTerm ) continue; + if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); + if( pWC->a[iTerm].wtFlags & (TERM_ORINFO|TERM_VIRTUAL) ) continue; + if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + pExpr = sqlite3ExprDup(db, pExpr, 0); + pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); + } + if( pAndExpr ){ + pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); + } + } + + /* Run a separate WHERE clause for each term of the OR clause. After + ** eliminating duplicates from other WHERE clauses, the action for each + ** sub-WHERE clause is to to invoke the main loop body as a subroutine. + */ + wctrlFlags = WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | + WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY; + for(ii=0; iinTerm; ii++){ + WhereTerm *pOrTerm = &pOrWc->a[ii]; + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ + WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ + int j1 = 0; /* Address of jump operation */ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ + pAndExpr->pLeft = pOrExpr; + pOrExpr = pAndExpr; + } + /* Loop through table entries that match term pOrTerm. */ + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + wctrlFlags, iCovCur); + assert( pSubWInfo || pParse->nErr || db->mallocFailed ); + if( pSubWInfo ){ + WhereLoop *pSubLoop; + explainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); + /* This is the sub-WHERE clause body. First skip over + ** duplicate rows from prior sub-WHERE clauses, and record the + ** rowid (or PRIMARY KEY) for the current row so that the same + ** row will be skipped in subsequent sub-WHERE clauses. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + int r; + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + if( HasRowid(pTab) ){ + r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + j1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, r,iSet); + VdbeCoverage(v); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk; + + /* Read the PK into an array of temp registers. */ + r = sqlite3GetTempRange(pParse, nPk); + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur, r+iPk, 0); + } + + /* Check if the temp table already contains this key. If so, + ** the row has already been included in the result set and + ** can be ignored (by jumping past the Gosub below). Otherwise, + ** insert the key into the temp table and proceed with processing + ** the row. + ** + ** Use some of the same optimizations as OP_RowSetTest: If iSet + ** is zero, assume that the key cannot already be present in + ** the temp table. And if iSet is -1, assume that there is no + ** need to insert the key into the temp table, as it will never + ** be tested for. */ + if( iSet ){ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + VdbeCoverage(v); + } + if( iSet>=0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); + sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); + if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + + /* Release the array of temp registers */ + sqlite3ReleaseTempRange(pParse, r, nPk); + } + } + + /* Invoke the main loop body as a subroutine */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + + /* Jump here (skipping the main loop body subroutine) if the + ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ + if( j1 ) sqlite3VdbeJumpHere(v, j1); + + /* The pSubWInfo->untestedTerms flag means that this OR term + ** contained one or more AND term from a notReady table. The + ** terms from the notReady table could not be tested and will + ** need to be tested later. + */ + if( pSubWInfo->untestedTerms ) untestedTerms = 1; + + /* If all of the OR-connected terms are optimized using the same + ** index, and the index is opened using the same cursor number + ** by each call to sqlite3WhereBegin() made by this loop, it may + ** be possible to use that index as a covering index. + ** + ** If the call to sqlite3WhereBegin() above resulted in a scan that + ** uses an index, and this is either the first OR-connected term + ** processed or the index is the same as that used by all previous + ** terms, set pCov to the candidate covering index. Otherwise, set + ** pCov to NULL to indicate that no candidate covering index will + ** be available. + */ + pSubLoop = pSubWInfo->a[0].pWLoop; + assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 + && (ii==0 || pSubLoop->u.btree.pIndex==pCov) + && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) + ){ + assert( pSubWInfo->a[0].iIdxCur==iCovCur ); + pCov = pSubLoop->u.btree.pIndex; + wctrlFlags |= WHERE_REOPEN_IDX; + }else{ + pCov = 0; + } + + /* Finish the loop through table entries that match term pOrTerm. */ + sqlite3WhereEnd(pSubWInfo); + } + } + } + pLevel->u.pCovidx = pCov; + if( pCov ) pLevel->iIdxCur = iCovCur; + if( pAndExpr ){ + pAndExpr->pLeft = 0; + sqlite3ExprDelete(db, pAndExpr); + } + sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); + sqlite3VdbeResolveLabel(v, iLoopBody); + + if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); + if( !untestedTerms ) disableTerm(pLevel, pTerm); + }else +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + + { + /* Case 6: There is no usable index. We must do a complete + ** scan of the entire table. + */ + static const u8 aStep[] = { OP_Next, OP_Prev }; + static const u8 aStart[] = { OP_Rewind, OP_Last }; + assert( bRev==0 || bRev==1 ); + if( pTabItem->isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } + } + + /* Insert code to test every subexpression that can be completely + ** computed using the current set of tables. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + continue; + } + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + pTerm->wtFlags |= TERM_CODED; + } + + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE, *pEAlt; + WhereTerm *pAlt; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; + if( pTerm->leftCursor!=iCur ) continue; + if( pLevel->iLeftJoin ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); + pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + testcase( pAlt->eOperator & WO_EQ ); + testcase( pAlt->eOperator & WO_IN ); + VdbeModuleComment((v, "begin transitive constraint")); + pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt)); + if( pEAlt ){ + *pEAlt = *pAlt->pExpr; + pEAlt->pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL); + sqlite3StackFree(db, pEAlt); + } + } + + /* For a LEFT OUTER JOIN, generate code that will record the fact that + ** at least one row of the right table has matched the left table. + */ + if( pLevel->iLeftJoin ){ + pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); + VdbeComment((v, "record LEFT JOIN hit")); + sqlite3ExprCacheClear(pParse); + for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + assert( pWInfo->untestedTerms ); + continue; + } + assert( pTerm->pExpr ); + sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); + pTerm->wtFlags |= TERM_CODED; + } + } + + return pLevel->notReady; +} + +#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN) +/* +** Generate "Explanation" text for a WhereTerm. +*/ +static void whereExplainTerm(Vdbe *v, WhereTerm *pTerm){ + char zType[4]; + memcpy(zType, "...", 4); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + sqlite3ExplainPrintf(v, "%s ", zType); + sqlite3ExplainExpr(v, pTerm->pExpr); +} +#endif /* WHERETRACE_ENABLED && SQLITE_ENABLE_TREE_EXPLAIN */ + + +#ifdef WHERETRACE_ENABLED +/* +** Print a WhereLoop object for debugging purposes +*/ +static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+7)/8; + struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + const char *zName; + if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ + if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ + int i = sqlite3Strlen30(zName) - 1; + while( zName[i]!='_' ) i--; + zName += i; + } + sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); + }else{ + sqlite3DebugPrintf("%20s",""); + } + }else{ + char *z; + if( p->u.vtab.idxStr ){ + z = sqlite3_mprintf("(%d,\"%s\",%x)", + p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); + }else{ + z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); + } + sqlite3DebugPrintf(" %-19s", z); + sqlite3_free(z); + } + sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); +#ifdef SQLITE_ENABLE_TREE_EXPLAIN + /* If the 0x100 bit of wheretracing is set, then show all of the constraint + ** expressions in the WhereLoop.aLTerm[] array. + */ + if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ /* WHERETRACE 0x100 */ + int i; + Vdbe *v = pWInfo->pParse->pVdbe; + sqlite3ExplainBegin(v); + for(i=0; inLTerm; i++){ + WhereTerm *pTerm = p->aLTerm[i]; + if( pTerm==0 ) continue; + sqlite3ExplainPrintf(v, " (%d) #%-2d ", i+1, (int)(pTerm-pWC->a)); + sqlite3ExplainPush(v); + whereExplainTerm(v, pTerm); + sqlite3ExplainPop(v); + sqlite3ExplainNL(v); + } + sqlite3ExplainFinish(v); + sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v)); + } +#endif +} +#endif + +/* +** Convert bulk memory into a valid WhereLoop that can be passed +** to whereLoopClear harmlessly. +*/ +static void whereLoopInit(WhereLoop *p){ + p->aLTerm = p->aLTermSpace; + p->nLTerm = 0; + p->nLSlot = ArraySize(p->aLTermSpace); + p->wsFlags = 0; +} + +/* +** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. +*/ +static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ + if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ + sqlite3_free(p->u.vtab.idxStr); + p->u.vtab.needFree = 0; + p->u.vtab.idxStr = 0; + }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ + sqlite3DbFree(db, p->u.btree.pIndex->zColAff); + sqlite3KeyInfoUnref(p->u.btree.pIndex->pKeyInfo); + sqlite3DbFree(db, p->u.btree.pIndex); + p->u.btree.pIndex = 0; + } + } +} + +/* +** Deallocate internal memory used by a WhereLoop object +*/ +static void whereLoopClear(sqlite3 *db, WhereLoop *p){ + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + whereLoopClearUnion(db, p); + whereLoopInit(p); +} + +/* +** Increase the memory allocation for pLoop->aLTerm[] to be at least n. +*/ +static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ + WhereTerm **paNew; + if( p->nLSlot>=n ) return SQLITE_OK; + n = (n+7)&~7; + paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n); + if( paNew==0 ) return SQLITE_NOMEM; + memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + p->aLTerm = paNew; + p->nLSlot = n; + return SQLITE_OK; +} + +/* +** Transfer content from the second pLoop into the first. +*/ +static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ + whereLoopClearUnion(db, pTo); + if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ + memset(&pTo->u, 0, sizeof(pTo->u)); + return SQLITE_NOMEM; + } + memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); + memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); + if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ + pFrom->u.vtab.needFree = 0; + }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + pFrom->u.btree.pIndex = 0; + } + return SQLITE_OK; +} + +/* +** Delete a WhereLoop object +*/ +static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + whereLoopClear(db, p); + sqlite3DbFree(db, p); +} + +/* +** Free a WhereInfo structure +*/ +static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ + if( ALWAYS(pWInfo) ){ + whereClauseClear(&pWInfo->sWC); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); + } + sqlite3DbFree(db, pWInfo); + } +} + +/* +** Return TRUE if both of the following are true: +** +** (1) X has the same or lower cost that Y +** (2) X is a proper subset of Y +** +** By "proper subset" we mean that X uses fewer WHERE clause terms +** than Y and that every WHERE clause term used by X is also used +** by Y. +** +** If X is a proper subset of Y then Y is a better choice and ought +** to have a lower cost. This routine returns TRUE when that cost +** relationship is inverted and needs to be adjusted. +*/ +static int whereLoopCheaperProperSubset( + const WhereLoop *pX, /* First WhereLoop to compare */ + const WhereLoop *pY /* Compare against this WhereLoop */ +){ + int i, j; + if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->rRun >= pY->rRun ){ + if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ + if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ + } + for(i=pX->nLTerm-1; i>=0; i--){ + for(j=pY->nLTerm-1; j>=0; j--){ + if( pY->aLTerm[j]==pX->aLTerm[i] ) break; + } + if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + } + return 1; /* All conditions meet */ +} + +/* +** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so +** that: +** +** (1) pTemplate costs less than any other WhereLoops that are a proper +** subset of pTemplate +** +** (2) pTemplate costs more than any other WhereLoops for which pTemplate +** is a proper subset. +** +** To say "WhereLoop X is a proper subset of Y" means that X uses fewer +** WHERE clause terms than Y and that every WHERE clause term used by X is +** also used by Y. +** +** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the +** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE +** clause terms covered, since some of the first nLTerm entries in aLTerm[] +** will be NULL (because they are skipped). That makes it more difficult +** to compare the loops. We could add extra code to do the comparison, and +** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this +** adjustment is sufficient minor, that it is very difficult to construct +** a test case where the extra code would improve the query plan. Better +** to avoid the added complexity and just omit cost adjustments to SKIPSCAN +** loops. +*/ +static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ + if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; + for(; p; p=p->pNextLoop){ + if( p->iTab!=pTemplate->iTab ) continue; + if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; + if( whereLoopCheaperProperSubset(p, pTemplate) ){ + /* Adjust pTemplate cost downward so that it is cheaper than its + ** subset p */ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut - 1; + }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ + /* Adjust pTemplate cost upward so that it is costlier than p since + ** pTemplate is a proper subset of p */ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut + 1; + } + } +} + +/* +** Search the list of WhereLoops in *ppPrev looking for one that can be +** supplanted by pTemplate. +** +** Return NULL if the WhereLoop list contains an entry that can supplant +** pTemplate, in other words if pTemplate does not belong on the list. +** +** If pX is a WhereLoop that pTemplate can supplant, then return the +** link that points to pX. +** +** If pTemplate cannot supplant any existing element of the list but needs +** to be added to the list, then return a pointer to the tail of the list. +*/ +static WhereLoop **whereLoopFindLesser( + WhereLoop **ppPrev, + const WhereLoop *pTemplate +){ + WhereLoop *p; + for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ + /* If either the iTab or iSortIdx values for two WhereLoop are different + ** then those WhereLoops need to be considered separately. Neither is + ** a candidate to replace the other. */ + continue; + } + /* In the current implementation, the rSetup value is either zero + ** or the cost of building an automatic index (NlogN) and the NlogN + ** is the same for compatible WhereLoops. */ + assert( p->rSetup==0 || pTemplate->rSetup==0 + || p->rSetup==pTemplate->rSetup ); + + /* whereLoopAddBtree() always generates and inserts the automatic index + ** case first. Hence compatible candidate WhereLoops never have a larger + ** rSetup. Call this SETUP-INVARIANT */ + assert( p->rSetup>=pTemplate->rSetup ); + + /* Any loop using an appliation-defined index (or PRIMARY KEY or + ** UNIQUE constraint) with one or more == constraints is better + ** than an automatic index. */ + if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->wsFlags & WHERE_INDEXED)!=0 + && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 + && (p->prereq & pTemplate->prereq)==pTemplate->prereq + ){ + break; + } + + /* If existing WhereLoop p is better than pTemplate, pTemplate can be + ** discarded. WhereLoop p is better if: + ** (1) p has no more dependencies than pTemplate, and + ** (2) p has an equal or lower cost than pTemplate + */ + if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ + && p->rSetup<=pTemplate->rSetup /* (2a) */ + && p->rRun<=pTemplate->rRun /* (2b) */ + && p->nOut<=pTemplate->nOut /* (2c) */ + ){ + return 0; /* Discard pTemplate */ + } + + /* If pTemplate is always better than p, then cause p to be overwritten + ** with pTemplate. pTemplate is better than p if: + ** (1) pTemplate has no more dependences than p, and + ** (2) pTemplate has an equal or lower cost than p. + */ + if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ + && p->rRun>=pTemplate->rRun /* (2a) */ + && p->nOut>=pTemplate->nOut /* (2b) */ + ){ + assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ + break; /* Cause p to be overwritten by pTemplate */ + } + } + return ppPrev; +} + +/* +** Insert or replace a WhereLoop entry using the template supplied. +** +** An existing WhereLoop entry might be overwritten if the new template +** is better and has fewer dependencies. Or the template will be ignored +** and no insert will occur if an existing WhereLoop is faster and has +** fewer dependencies than the template. Otherwise a new WhereLoop is +** added based on the template. +** +** If pBuilder->pOrSet is not NULL then we care about only the +** prerequisites and rRun and nOut costs of the N best loops. That +** information is gathered in the pBuilder->pOrSet object. This special +** processing mode is used only for OR clause processing. +** +** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we +** still might overwrite similar loops with the new template if the +** new template is better. Loops may be overwritten if the following +** conditions are met: +** +** (1) They have the same iTab. +** (2) They have the same iSortIdx. +** (3) The template has same or fewer dependencies than the current loop +** (4) The template has the same or lower cost than the current loop +*/ +static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ + WhereLoop **ppPrev, *p; + WhereInfo *pWInfo = pBuilder->pWInfo; + sqlite3 *db = pWInfo->pParse->db; + + /* If pBuilder->pOrSet is defined, then only keep track of the costs + ** and prereqs. + */ + if( pBuilder->pOrSet!=0 ){ +#if WHERETRACE_ENABLED + u16 n = pBuilder->pOrSet->n; + int x = +#endif + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + pTemplate->nOut); +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + } + + /* Look for an existing WhereLoop to replace with pTemplate + */ + whereLoopAdjustCost(pWInfo->pLoops, pTemplate); + ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); + + if( ppPrev==0 ){ + /* There already exists a WhereLoop on the list that is better + ** than pTemplate, so just ignore pTemplate */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-noop: "); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + }else{ + p = *ppPrev; + } + + /* If we reach this point it means that either p[] should be overwritten + ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new + ** WhereLoop and insert it. + */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + if( p!=0 ){ + sqlite3DebugPrintf("ins-del: "); + whereLoopPrint(p, pBuilder->pWC); + } + sqlite3DebugPrintf("ins-new: "); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + if( p==0 ){ + /* Allocate a new WhereLoop to add to the end of the list */ + *ppPrev = p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + if( p==0 ) return SQLITE_NOMEM; + whereLoopInit(p); + p->pNextLoop = 0; + }else{ + /* We will be overwriting WhereLoop p[]. But before we do, first + ** go through the rest of the list and delete any other entries besides + ** p[] that are also supplated by pTemplate */ + WhereLoop **ppTail = &p->pNextLoop; + WhereLoop *pToDel; + while( *ppTail ){ + ppTail = whereLoopFindLesser(ppTail, pTemplate); + if( ppTail==0 ) break; + pToDel = *ppTail; + if( pToDel==0 ) break; + *ppTail = pToDel->pNextLoop; +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-del: "); + whereLoopPrint(pToDel, pBuilder->pWC); + } +#endif + whereLoopDelete(db, pToDel); + } + } + whereLoopXfer(db, p, pTemplate); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + Index *pIndex = p->u.btree.pIndex; + if( pIndex && pIndex->tnum==0 ){ + p->u.btree.pIndex = 0; + } + } + return SQLITE_OK; +} + +/* +** Adjust the WhereLoop.nOut value downward to account for terms of the +** WHERE clause that reference the loop but which are not used by an +** index. +** +** In the current implementation, the first extra WHERE clause term reduces +** the number of output rows by a factor of 10 and each additional term +** reduces the number of output rows by sqrt(2). +*/ +static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ + WhereTerm *pTerm, *pX; + Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); + int i, j; + + if( !OptimizationEnabled(pWC->pWInfo->pParse->db, SQLITE_AdjustOutEst) ){ + return; + } + for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ + if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; + if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; + if( (pTerm->prereqAll & notAllowed)!=0 ) continue; + for(j=pLoop->nLTerm-1; j>=0; j--){ + pX = pLoop->aLTerm[j]; + if( pX==0 ) continue; + if( pX==pTerm ) break; + if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; + } + if( j<0 ){ + pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + } + } +} + +/* +** Adjust the cost C by the costMult facter T. This only occurs if +** compiled with -DSQLITE_ENABLE_COSTMULT +*/ +#ifdef SQLITE_ENABLE_COSTMULT +# define ApplyCostMultiplier(C,T) C += T +#else +# define ApplyCostMultiplier(C,T) +#endif + +/* +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. +** +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. +** +** If pProbe->tnum==0, that means pIndex is a fake index used for the +** INTEGER PRIMARY KEY. +*/ +static int whereLoopAddBtreeIndex( + WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ + struct SrcList_item *pSrc, /* FROM clause term being analyzed */ + Index *pProbe, /* An index on pSrc */ + LogEst nInMul /* log(Number of iterations due to IN) */ +){ + WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection malloc context */ + WhereLoop *pNew; /* Template WhereLoop under construction */ + WhereTerm *pTerm; /* A WhereTerm under consideration */ + int opMask; /* Valid operators for constraints */ + WhereScan scan; /* Iterator for WHERE terms */ + Bitmask saved_prereq; /* Original value of pNew->prereq */ + u16 saved_nLTerm; /* Original value of pNew->nLTerm */ + u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ + u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ + u32 saved_wsFlags; /* Original value of pNew->wsFlags */ + LogEst saved_nOut; /* Original value of pNew->nOut */ + int iCol; /* Index of the column in the table */ + int rc = SQLITE_OK; /* Return code */ + LogEst rLogSize; /* Logarithm of table size */ + WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ + + pNew = pBuilder->pNew; + if( db->mallocFailed ) return SQLITE_NOMEM; + + assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); + if( pNew->wsFlags & WHERE_BTM_LIMIT ){ + opMask = WO_LT|WO_LE; + }else if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ + opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; + }else{ + opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; + } + if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + + assert( pNew->u.btree.nEqnColumn ); + iCol = pProbe->aiColumn[pNew->u.btree.nEq]; + + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, + opMask, pProbe); + saved_nEq = pNew->u.btree.nEq; + saved_nSkip = pNew->u.btree.nSkip; + saved_nLTerm = pNew->nLTerm; + saved_wsFlags = pNew->wsFlags; + saved_prereq = pNew->prereq; + saved_nOut = pNew->nOut; + pNew->rSetup = 0; + rLogSize = estLog(pProbe->aiRowLogEst[0]); + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( pTerm==0 + && saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->u.btree.nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + } + for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ + LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ + int nIn = 0; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int nRecValid = pBuilder->nRecValid; +#endif + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) + ){ + continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ + } + if( pTerm->prereqRight & pNew->maskSelf ) continue; + + pNew->wsFlags = saved_wsFlags; + pNew->u.btree.nEq = saved_nEq; + pNew->nLTerm = saved_nLTerm; + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTerm; + pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; + + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); + + if( eOp & WO_IN ){ + Expr *pExpr = pTerm->pExpr; + pNew->wsFlags |= WHERE_COLUMN_IN; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ + nIn = 46; assert( 46==sqlite3LogEst(25) ); + }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ + /* "x IN (value, value, ...)" */ + nIn = sqlite3LogEst(pExpr->x.pList->nExpr); + } + assert( nIn>0 ); /* RHS always has 2 or more terms... The parser + ** changes "x IN (?)" into "x=?". */ + + }else if( eOp & (WO_EQ) ){ + pNew->wsFlags |= WHERE_COLUMN_EQ; + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ + if( iCol>=0 && !IsUniqueIndex(pProbe) ){ + pNew->wsFlags |= WHERE_UNQ_WANTED; + }else{ + pNew->wsFlags |= WHERE_ONEROW; + } + } + }else if( eOp & WO_ISNULL ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pBtm = pTerm; + pTop = 0; + }else{ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } + + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + ** data, using some other estimate. */ + whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 && iCol>=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + testcase( eOp & WO_IN ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + }else{ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && OptimizationEnabled(db, SQLITE_Stat3) + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) +#endif + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } + } + } + + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); + } + ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); + + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; + whereLoopOutputAdjust(pBuilder->pWC, pNew); + rc = whereLoopInsert(pBuilder, pNew); + + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } + + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 + && pNew->u.btree.nEqnColumn + ){ + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); + } + pNew->nOut = saved_nOut; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + pBuilder->nRecValid = nRecValid; +#endif + } + pNew->prereq = saved_prereq; + pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + pNew->nOut = saved_nOut; + pNew->nLTerm = saved_nLTerm; + return rc; +} + +/* +** Return True if it is possible that pIndex might be useful in +** implementing the ORDER BY clause in pBuilder. +** +** Return False if pBuilder does not contain an ORDER BY clause or +** if there is no way for pIndex to be useful in implementing that +** ORDER BY clause. +*/ +static int indexMightHelpWithOrderBy( + WhereLoopBuilder *pBuilder, + Index *pIndex, + int iCursor +){ + ExprList *pOB; + int ii, jj; + + if( pIndex->bUnordered ) return 0; + if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); + if( pExpr->op!=TK_COLUMN ) return 0; + if( pExpr->iTable==iCursor ){ + for(jj=0; jjnKeyCol; jj++){ + if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; + } + } + } + return 0; +} + +/* +** Return a bitmask where 1s indicate that the corresponding column of +** the table is used by an index. Only the first 63 columns are considered. +*/ +static Bitmask columnsInIndex(Index *pIdx){ + Bitmask m = 0; + int j; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( x>=0 ){ + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xa; inTerm; i++, pTerm++){ + if( sqlite3ExprImpliesExpr(pTerm->pExpr, pWhere, iTab) ) return 1; + } + return 0; +} + +/* +** Add all WhereLoop objects for a single table of the join where the table +** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be +** a b-tree table, not a virtual table. +** +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. +** +** The estimated values (nRow, nVisit, nSeek) often contain a large amount +** of uncertainty. For this reason, scoring is designed to pick plans that +** "do the least harm" if the estimates are inaccurate. For example, a +** log(nRow) factor is omitted from a non-covering index scan in order to +** bias the scoring in favor of using an index, since the worst-case +** performance of using an index is far better than the worst-case performance +** of a full table scan. +*/ +static int whereLoopAddBtree( + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + Bitmask mExtra /* Extra prerequesites for using this table */ +){ + WhereInfo *pWInfo; /* WHERE analysis context */ + Index *pProbe; /* An index we are evaluating */ + Index sPk; /* A fake index object for the primary key */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ + i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ + SrcList *pTabList; /* The FROM clause */ + struct SrcList_item *pSrc; /* The FROM clause btree term to add */ + WhereLoop *pNew; /* Template WhereLoop object */ + int rc = SQLITE_OK; /* Return code */ + int iSortIdx = 1; /* Index number */ + int b; /* A boolean value */ + LogEst rSize; /* number of rows in the table */ + LogEst rLogSize; /* Logarithm of the number of rows in the table */ + WhereClause *pWC; /* The parsed WHERE clause */ + Table *pTab; /* Table being queried */ + + pNew = pBuilder->pNew; + pWInfo = pBuilder->pWInfo; + pTabList = pWInfo->pTabList; + pSrc = pTabList->a + pNew->iTab; + pTab = pSrc->pTab; + pWC = pBuilder->pWC; + assert( !IsVirtual(pSrc->pTab) ); + + if( pSrc->pIndex ){ + /* An INDEXED BY clause specifies a particular index to use */ + pProbe = pSrc->pIndex; + }else if( !HasRowid(pTab) ){ + pProbe = pTab->pIndex; + }else{ + /* There is no INDEXED BY clause. Create a fake Index object in local + ** variable sPk to represent the rowid primary key index. Make this + ** fake index the first in a chain of Index objects with all of the real + ** indices to follow */ + Index *pFirst; /* First of real indices on the table */ + memset(&sPk, 0, sizeof(Index)); + sPk.nKeyCol = 1; + sPk.nColumn = 1; + sPk.aiColumn = &aiColumnPk; + sPk.aiRowLogEst = aiRowEstPk; + sPk.onError = OE_Replace; + sPk.pTable = pTab; + sPk.szIdxRow = pTab->szTabRow; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; + pFirst = pSrc->pTab->pIndex; + if( pSrc->notIndexed==0 ){ + /* The real indices of the table are only considered if the + ** NOT INDEXED qualifier is omitted from the FROM clause */ + sPk.pNext = pFirst; + } + pProbe = &sPk; + } + rSize = pTab->nRowLogEst; + rLogSize = estLog(rSize); + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + /* Automatic indexes */ + if( !pBuilder->pOrSet + && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 + && pSrc->pIndex==0 + && !pSrc->viaCoroutine + && !pSrc->notIndexed + && HasRowid(pTab) + && !pSrc->isCorrelated + && !pSrc->isRecursive + ){ + /* Generate auto-index WhereLoops */ + WhereTerm *pTerm; + WhereTerm *pWCEnd = pWC->a + pWC->nTerm; + for(pTerm=pWC->a; rc==SQLITE_OK && pTermprereqRight & pNew->maskSelf ) continue; + if( termCanDriveIndex(pTerm, pSrc, 0) ){ + pNew->u.btree.nEq = 1; + pNew->u.btree.nSkip = 0; + pNew->u.btree.pIndex = 0; + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; + /* TUNING: One-time cost for computing the automatic index is + ** approximately 7*N*log2(N) where N is the number of rows in + ** the table being indexed. */ + pNew->rSetup = rLogSize + rSize + 28; assert( 28==sqlite3LogEst(7) ); + ApplyCostMultiplier(pNew->rSetup, pTab->costMult); + /* TUNING: Each index lookup yields 20 rows in the table. This + ** is more than the usual guess of 10 rows, since we have no way + ** of knowning how selective the index will ultimately be. It would + ** not be unreasonable to make this value much larger. */ + pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); + pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); + pNew->wsFlags = WHERE_AUTO_INDEX; + pNew->prereq = mExtra | pTerm->prereqRight; + rc = whereLoopInsert(pBuilder, pNew); + } + } + } +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + + /* Loop over all indices + */ + for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ + if( pProbe->pPartIdxWhere!=0 + && !whereUsablePartialIndex(pNew->iTab, pWC, pProbe->pPartIdxWhere) ){ + continue; /* Partial index inappropriate for this query */ + } + rSize = pProbe->aiRowLogEst[0]; + pNew->u.btree.nEq = 0; + pNew->u.btree.nSkip = 0; + pNew->nLTerm = 0; + pNew->iSortIdx = 0; + pNew->rSetup = 0; + pNew->prereq = mExtra; + pNew->nOut = rSize; + pNew->u.btree.pIndex = pProbe; + b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); + /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ + assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); + if( pProbe->tnum<=0 ){ + /* Integer primary key index */ + pNew->wsFlags = WHERE_IPK; + + /* Full table scan */ + pNew->iSortIdx = b ? iSortIdx : 0; + /* TUNING: Cost of full table scan is (N*3.0). */ + pNew->rRun = rSize + 16; + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew); + rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; + if( rc ) break; + }else{ + Bitmask m; + if( pProbe->isCovering ){ + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; + m = 0; + }else{ + m = pSrc->colUsed & ~columnsInIndex(pProbe); + pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; + } + + /* Full scan via index */ + if( b + || !HasRowid(pTab) + || ( m==0 + && pProbe->bUnordered==0 + && (pProbe->szIdxRowszTabRow) + && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 + && sqlite3GlobalConfig.bUseCis + && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) + ) + ){ + pNew->iSortIdx = b ? iSortIdx : 0; + + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. If this is a non-covering index scan, + ** also add the cost of visiting table rows (N*3.0). */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); + } + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew); + rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; + if( rc ) break; + } + } + + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3Stat4ProbeFree(pBuilder->pRec); + pBuilder->nRecValid = 0; + pBuilder->pRec = 0; +#endif + + /* If there was an INDEXED BY clause, then only that one index is + ** considered. */ + if( pSrc->pIndex ) break; + } + return rc; +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Add all WhereLoop objects for a table of the join identified by +** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. +*/ +static int whereLoopAddVirtual( + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + Bitmask mExtra +){ + WhereInfo *pWInfo; /* WHERE analysis context */ + Parse *pParse; /* The parsing context */ + WhereClause *pWC; /* The WHERE clause */ + struct SrcList_item *pSrc; /* The FROM clause term to search */ + Table *pTab; + sqlite3 *db; + sqlite3_index_info *pIdxInfo; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int i, j; + int iTerm, mxTerm; + int nConstraint; + int seenIn = 0; /* True if an IN operator is seen */ + int seenVar = 0; /* True if a non-constant constraint is seen */ + int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ + WhereLoop *pNew; + int rc = SQLITE_OK; + + pWInfo = pBuilder->pWInfo; + pParse = pWInfo->pParse; + db = pParse->db; + pWC = pBuilder->pWC; + pNew = pBuilder->pNew; + pSrc = &pWInfo->pTabList->a[pNew->iTab]; + pTab = pSrc->pTab; + assert( IsVirtual(pTab) ); + pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); + if( pIdxInfo==0 ) return SQLITE_NOMEM; + pNew->prereq = 0; + pNew->rSetup = 0; + pNew->wsFlags = WHERE_VIRTUALTABLE; + pNew->nLTerm = 0; + pNew->u.vtab.needFree = 0; + pUsage = pIdxInfo->aConstraintUsage; + nConstraint = pIdxInfo->nConstraint; + if( whereLoopResize(db, pNew, nConstraint) ){ + sqlite3DbFree(db, pIdxInfo); + return SQLITE_NOMEM; + } + + for(iPhase=0; iPhase<=3; iPhase++){ + if( !seenIn && (iPhase&1)!=0 ){ + iPhase++; + if( iPhase>3 ) break; + } + if( !seenVar && iPhase>1 ) break; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pIdxCons++){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + switch( iPhase ){ + case 0: /* Constants without IN operator */ + pIdxCons->usable = 0; + if( (pTerm->eOperator & WO_IN)!=0 ){ + seenIn = 1; + } + if( pTerm->prereqRight!=0 ){ + seenVar = 1; + }else if( (pTerm->eOperator & WO_IN)==0 ){ + pIdxCons->usable = 1; + } + break; + case 1: /* Constants with IN operators */ + assert( seenIn ); + pIdxCons->usable = (pTerm->prereqRight==0); + break; + case 2: /* Variables without IN */ + assert( seenVar ); + pIdxCons->usable = (pTerm->eOperator & WO_IN)==0; + break; + default: /* Variables with IN */ + assert( seenVar && seenIn ); + pIdxCons->usable = 1; + break; + } + } + memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); + if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->needToFreeIdxStr = 0; + pIdxInfo->orderByConsumed = 0; + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; + pIdxInfo->estimatedRows = 25; + rc = vtabBestIndex(pParse, pTab, pIdxInfo); + if( rc ) goto whereLoopAddVtab_exit; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + pNew->prereq = mExtra; + mxTerm = -1; + assert( pNew->nLSlot>=nConstraint ); + for(i=0; iaLTerm[i] = 0; + pNew->u.vtab.omitMask = 0; + for(i=0; i=0 ){ + j = pIdxCons->iTermOffset; + if( iTerm>=nConstraint + || j<0 + || j>=pWC->nTerm + || pNew->aLTerm[iTerm]!=0 + ){ + rc = SQLITE_ERROR; + sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName); + goto whereLoopAddVtab_exit; + } + testcase( iTerm==nConstraint-1 ); + testcase( j==0 ); + testcase( j==pWC->nTerm-1 ); + pTerm = &pWC->a[j]; + pNew->prereq |= pTerm->prereqRight; + assert( iTermnLSlot ); + pNew->aLTerm[iTerm] = pTerm; + if( iTerm>mxTerm ) mxTerm = iTerm; + testcase( iTerm==15 ); + testcase( iTerm==16 ); + if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit==0 ){ + /* Do not attempt to use an IN constraint if the virtual table + ** says that the equivalent EQ constraint cannot be safely omitted. + ** If we do attempt to use such a constraint, some rows might be + ** repeated in the output. */ + break; + } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + } + } + } + if( i>=nConstraint ){ + pNew->nLTerm = mxTerm+1; + assert( pNew->nLTerm<=pNew->nLSlot ); + pNew->u.vtab.idxNum = pIdxInfo->idxNum; + pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; + pIdxInfo->needToFreeIdxStr = 0; + pNew->u.vtab.idxStr = pIdxInfo->idxStr; + pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? + pIdxInfo->nOrderBy : 0); + pNew->rSetup = 0; + pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); + pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); + whereLoopInsert(pBuilder, pNew); + if( pNew->u.vtab.needFree ){ + sqlite3_free(pNew->u.vtab.idxStr); + pNew->u.vtab.needFree = 0; + } + } + } + +whereLoopAddVtab_exit: + if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); + sqlite3DbFree(db, pIdxInfo); + return rc; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Add WhereLoop entries to handle OR terms. This works for either +** btrees or virtual tables. +*/ +static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ + WhereInfo *pWInfo = pBuilder->pWInfo; + WhereClause *pWC; + WhereLoop *pNew; + WhereTerm *pTerm, *pWCEnd; + int rc = SQLITE_OK; + int iCur; + WhereClause tempWC; + WhereLoopBuilder sSubBuild; + WhereOrSet sSum, sCur; + struct SrcList_item *pItem; + + pWC = pBuilder->pWC; + if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; + pWCEnd = pWC->a + pWC->nTerm; + pNew = pBuilder->pNew; + memset(&sSum, 0, sizeof(sSum)); + pItem = pWInfo->pTabList->a + pNew->iTab; + iCur = pItem->iCursor; + + for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 + && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 + ){ + WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; + WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; + WhereTerm *pOrTerm; + int once = 1; + int i, j; + + sSubBuild = *pBuilder; + sSubBuild.pOrderBy = 0; + sSubBuild.pOrSet = &sCur; + + for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ + sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; + }else if( pOrTerm->leftCursor==iCur ){ + tempWC.pWInfo = pWC->pWInfo; + tempWC.pOuter = pWC; + tempWC.op = TK_AND; + tempWC.nTerm = 1; + tempWC.a = pOrTerm; + sSubBuild.pWC = &tempWC; + }else{ + continue; + } + sCur.n = 0; +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(&sSubBuild, mExtra); + }else +#endif + { + rc = whereLoopAddBtree(&sSubBuild, mExtra); + } + assert( rc==SQLITE_OK || sCur.n==0 ); + if( sCur.n==0 ){ + sSum.n = 0; + break; + }else if( once ){ + whereOrMove(&sSum, &sCur); + once = 0; + }else{ + WhereOrSet sPrev; + whereOrMove(&sPrev, &sSum); + sSum.n = 0; + for(i=0; inLTerm = 1; + pNew->aLTerm[0] = pTerm; + pNew->wsFlags = WHERE_MULTI_OR; + pNew->rSetup = 0; + pNew->iSortIdx = 0; + memset(&pNew->u, 0, sizeof(pNew->u)); + for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 1; + pNew->nOut = sSum.a[i].nOut; + pNew->prereq = sSum.a[i].prereq; + rc = whereLoopInsert(pBuilder, pNew); + } + } + } + return rc; +} + +/* +** Add all WhereLoop objects for all tables +*/ +static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo = pBuilder->pWInfo; + Bitmask mExtra = 0; + Bitmask mPrior = 0; + int iTab; + SrcList *pTabList = pWInfo->pTabList; + struct SrcList_item *pItem; + sqlite3 *db = pWInfo->pParse->db; + int nTabList = pWInfo->nLevel; + int rc = SQLITE_OK; + u8 priorJoinType = 0; + WhereLoop *pNew; + + /* Loop over the tables in the join, from left to right */ + pNew = pBuilder->pNew; + whereLoopInit(pNew); + for(iTab=0, pItem=pTabList->a; iTabiTab = iTab; + pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); + if( ((pItem->jointype|priorJoinType) & (JT_LEFT|JT_CROSS))!=0 ){ + mExtra = mPrior; + } + priorJoinType = pItem->jointype; + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(pBuilder, mExtra); + }else{ + rc = whereLoopAddBtree(pBuilder, mExtra); + } + if( rc==SQLITE_OK ){ + rc = whereLoopAddOr(pBuilder, mExtra); + } + mPrior |= pNew->maskSelf; + if( rc || db->mallocFailed ) break; + } + whereLoopClear(db, pNew); + return rc; +} + +/* +** Examine a WherePath (with the addition of the extra WhereLoop of the 5th +** parameters) to see if it outputs rows in the requested ORDER BY +** (or GROUP BY) without requiring a separate sort operation. Return N: +** +** N>0: N terms of the ORDER BY clause are satisfied +** N==0: No terms of the ORDER BY clause are satisfied +** N<0: Unknown yet how many terms of ORDER BY might be satisfied. +** +** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as +** strict. With GROUP BY and DISTINCT the only requirement is that +** equivalent rows appear immediately adjacent to one another. GROUP BY +** and DISTINCT do not require rows to appear in any particular order as long +** as equivelent rows are grouped together. Thus for GROUP BY and DISTINCT +** the pOrderBy terms can be matched in any order. With ORDER BY, the +** pOrderBy terms must be matched in strict left-to-right order. +*/ +static i8 wherePathSatisfiesOrderBy( + WhereInfo *pWInfo, /* The WHERE clause */ + ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ + WherePath *pPath, /* The WherePath to check */ + u16 wctrlFlags, /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */ + u16 nLoop, /* Number of entries in pPath->aLoop[] */ + WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ + Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ +){ + u8 revSet; /* True if rev is known */ + u8 rev; /* Composite sort order */ + u8 revIdx; /* Index sort order */ + u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ + u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ + u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ + u16 nKeyCol; /* Number of key columns in pIndex */ + u16 nColumn; /* Total number of ordered columns in the index */ + u16 nOrderBy; /* Number terms in the ORDER BY clause */ + int iLoop; /* Index of WhereLoop in pPath being processed */ + int i, j; /* Loop counters */ + int iCur; /* Cursor number for current WhereLoop */ + int iColumn; /* A column number within table iCur */ + WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + Expr *pOBExpr; /* An expression from the ORDER BY clause */ + CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ + Index *pIndex; /* The index associated with pLoop */ + sqlite3 *db = pWInfo->pParse->db; /* Database connection */ + Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ + Bitmask obDone; /* Mask of all ORDER BY terms */ + Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ + Bitmask ready; /* Mask of inner loops */ + + /* + ** We say the WhereLoop is "one-row" if it generates no more than one + ** row of output. A WhereLoop is one-row if all of the following are true: + ** (a) All index columns match with WHERE_COLUMN_EQ. + ** (b) The index is unique + ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. + ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. + ** + ** We say the WhereLoop is "order-distinct" if the set of columns from + ** that WhereLoop that are in the ORDER BY clause are different for every + ** row of the WhereLoop. Every one-row WhereLoop is automatically + ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause + ** is not order-distinct. To be order-distinct is not quite the same as being + ** UNIQUE since a UNIQUE column or index can have multiple rows that + ** are NULL and NULL values are equivalent for the purpose of order-distinct. + ** To be order-distinct, the columns must be UNIQUE and NOT NULL. + ** + ** The rowid for a table is always UNIQUE and NOT NULL so whenever the + ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is + ** automatically order-distinct. + */ + + assert( pOrderBy!=0 ); + if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; + + nOrderBy = pOrderBy->nExpr; + testcase( nOrderBy==BMS-1 ); + if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ + isOrderDistinct = 1; + obDone = MASKBIT(nOrderBy)-1; + orderDistinctMask = 0; + ready = 0; + for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; + pLoop = iLoopaLoop[iLoop] : pLast; + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + if( pLoop->u.vtab.isOrdered ) obSat = obDone; + break; + } + iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; + + /* Mark off any ORDER BY term X that is a column in the table of + ** the current loop for which there is term in the WHERE + ** clause of the form X IS NULL or X=? that reference only outer + ** loops. + */ + for(i=0; ia[i].pExpr); + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, + ~ready, WO_EQ|WO_ISNULL, 0); + if( pTerm==0 ) continue; + if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){ + const char *z1, *z2; + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + z1 = pColl->zName; + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr); + if( !pColl ) pColl = db->pDfltColl; + z2 = pColl->zName; + if( sqlite3StrICmp(z1, z2)!=0 ) continue; + } + obSat |= MASKBIT(i); + } + + if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ + if( pLoop->wsFlags & WHERE_IPK ){ + pIndex = 0; + nKeyCol = 0; + nColumn = 1; + }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ + return 0; + }else{ + nKeyCol = pIndex->nKeyCol; + nColumn = pIndex->nColumn; + assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); + assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); + isOrderDistinct = IsUniqueIndex(pIndex); + } + + /* Loop through all columns of the index and deal with the ones + ** that are not constrained by == or IN. + */ + rev = revSet = 0; + distinctColumns = 0; + for(j=0; ju.btree.nEq + && pLoop->u.btree.nSkip==0 + && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 + ){ + if( i & WO_ISNULL ){ + testcase( isOrderDistinct ); + isOrderDistinct = 0; + } + continue; + } + + /* Get the column number in the table (iColumn) and sort order + ** (revIdx) for the j-th column of the index. + */ + if( pIndex ){ + iColumn = pIndex->aiColumn[j]; + revIdx = pIndex->aSortOrder[j]; + if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; + }else{ + iColumn = -1; + revIdx = 0; + } + + /* An unconstrained column that might be NULL means that this + ** WhereLoop is not well-ordered + */ + if( isOrderDistinct + && iColumn>=0 + && j>=pLoop->u.btree.nEq + && pIndex->pTable->aCol[iColumn].notNull==0 + ){ + isOrderDistinct = 0; + } + + /* Find the ORDER BY term that corresponds to the j-th column + ** of the index and mark that ORDER BY term off + */ + bOnce = 1; + isMatch = 0; + for(i=0; bOnce && ia[i].pExpr); + testcase( wctrlFlags & WHERE_GROUPBY ); + testcase( wctrlFlags & WHERE_DISTINCTBY ); + if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=0 ){ + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; + } + isMatch = 1; + break; + } + if( isMatch && (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + /* Make sure the sort order is compatible in an ORDER BY clause. + ** Sort order is irrelevant for a GROUP BY clause. */ + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; + }else{ + rev = revIdx ^ pOrderBy->a[i].sortOrder; + if( rev ) *pRevMask |= MASKBIT(iLoop); + revSet = 1; + } + } + if( isMatch ){ + if( iColumn<0 ){ + testcase( distinctColumns==0 ); + distinctColumns = 1; + } + obSat |= MASKBIT(i); + }else{ + /* No match found */ + if( j==0 || jmaskSelf; + for(i=0; ia[i].pExpr; + mTerm = exprTableUsage(&pWInfo->sMaskSet,p); + if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( (mTerm&~orderDistinctMask)==0 ){ + obSat |= MASKBIT(i); + } + } + } + } /* End the loop over all WhereLoops from outer-most down to inner-most */ + if( obSat==obDone ) return (i8)nOrderBy; + if( !isOrderDistinct ){ + for(i=nOrderBy-1; i>0; i--){ + Bitmask m = MASKBIT(i) - 1; + if( (obSat&m)==m ) return i; + } + return 0; + } + return -1; +} + + +/* +** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(), +** the planner assumes that the specified pOrderBy list is actually a GROUP +** BY clause - and so any order that groups rows as required satisfies the +** request. +** +** Normally, in this case it is not possible for the caller to determine +** whether or not the rows are really being delivered in sorted order, or +** just in some other order that provides the required grouping. However, +** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then +** this function may be called on the returned WhereInfo object. It returns +** true if the rows really will be sorted in the specified order, or false +** otherwise. +** +** For example, assuming: +** +** CREATE INDEX i1 ON t1(x, Y); +** +** then +** +** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1 +** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 +*/ +SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo *pWInfo){ + assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); + assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); + return pWInfo->sorted; +} + +#ifdef WHERETRACE_ENABLED +/* For debugging use only: */ +static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ + static char zName[65]; + int i; + for(i=0; iaLoop[i]->cId; } + if( pLast ) zName[i++] = pLast->cId; + zName[i] = 0; + return zName; +} +#endif + +/* +** Return the cost of sorting nRow rows, assuming that the keys have +** nOrderby columns and that the first nSorted columns are already in +** order. +*/ +static LogEst whereSortingCost( + WhereInfo *pWInfo, + LogEst nRow, + int nOrderBy, + int nSorted +){ + /* TUNING: Estimated cost of a full external sort, where N is + ** the number of rows to sort is: + ** + ** cost = (3.0 * N * log(N)). + ** + ** Or, if the order-by clause has X terms but only the last Y + ** terms are out of order, then block-sorting will reduce the + ** sorting cost to: + ** + ** cost = (3.0 * N * log(N)) * (Y/X) + ** + ** The (Y/X) term is implemented using stack variable rScale + ** below. */ + LogEst rScale, rSortCost; + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); + rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + rSortCost = nRow + estLog(nRow) + rScale + 16; + + /* TUNING: The cost of implementing DISTINCT using a B-TREE is + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + rSortCost += 16; + } + + return rSortCost; +} + +/* +** Given the list of WhereLoop objects at pWInfo->pLoops, this routine +** attempts to find the lowest cost path that visits each WhereLoop +** once. This path is then loaded into the pWInfo->a[].pWLoop fields. +** +** Assume that the total number of output rows that will need to be sorted +** will be nRowEst (in the 10*log2 representation). Or, ignore sorting +** costs if nRowEst==0. +** +** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation +** error occurs. +*/ +static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ + int mxChoice; /* Maximum number of simultaneous paths tracked */ + int nLoop; /* Number of terms in the join */ + Parse *pParse; /* Parsing context */ + sqlite3 *db; /* The database connection */ + int iLoop; /* Loop counter over the terms of the join */ + int ii, jj; /* Loop counters */ + int mxI = 0; /* Index of next entry to replace */ + int nOrderBy; /* Number of ORDER BY clause terms */ + LogEst mxCost = 0; /* Maximum cost of a set of paths */ + LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ + int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ + WherePath *aFrom; /* All nFrom paths at the previous level */ + WherePath *aTo; /* The nTo best paths at the current level */ + WherePath *pFrom; /* An element of aFrom[] that we are working on */ + WherePath *pTo; /* An element of aTo[] that we are working on */ + WhereLoop *pWLoop; /* One of the WhereLoop objects */ + WhereLoop **pX; /* Used to divy up the pSpace memory */ + LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ + char *pSpace; /* Temporary memory used by this routine */ + int nSpace; /* Bytes of space allocated at pSpace */ + + pParse = pWInfo->pParse; + db = pParse->db; + nLoop = pWInfo->nLevel; + /* TUNING: For simple queries, only the best path is tracked. + ** For 2-way joins, the 5 best paths are followed. + ** For joins of 3 or more tables, track the 10 best paths */ + mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); + assert( nLoop<=pWInfo->pTabList->nSrc ); + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); + + /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this + ** case the purpose of this call is to estimate the number of rows returned + ** by the overall query. Once this estimate has been obtained, the caller + ** will invoke this function a second time, passing the estimate as the + ** nRowEst parameter. */ + if( pWInfo->pOrderBy==0 || nRowEst==0 ){ + nOrderBy = 0; + }else{ + nOrderBy = pWInfo->pOrderBy->nExpr; + } + + /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ + nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + nSpace += sizeof(LogEst) * nOrderBy; + pSpace = sqlite3DbMallocRaw(db, nSpace); + if( pSpace==0 ) return SQLITE_NOMEM; + aTo = (WherePath*)pSpace; + aFrom = aTo+mxChoice; + memset(aFrom, 0, sizeof(aFrom[0])); + pX = (WhereLoop**)(aFrom+mxChoice); + for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ + pFrom->aLoop = pX; + } + if( nOrderBy ){ + /* If there is an ORDER BY clause and it is not being ignored, set up + ** space for the aSortCost[] array. Each element of the aSortCost array + ** is either zero - meaning it has not yet been initialized - or the + ** cost of sorting nRowEst rows of data where the first X terms of + ** the ORDER BY clause are already in order, where X is the array + ** index. */ + aSortCost = (LogEst*)pX; + memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); + } + assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] ); + assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX ); + + /* Seed the search with a single WherePath containing zero WhereLoops. + ** + ** TUNING: Do not let the number of iterations go above 25. If the cost + ** of computing an automatic index is not paid back within the first 25 + ** rows, then do not use the automatic index. */ + aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==sqlite3LogEst(25) ); + nFrom = 1; + assert( aFrom[0].isOrdered==0 ); + if( nOrderBy ){ + /* If nLoop is zero, then there are no FROM terms in the query. Since + ** in this case the query may return a maximum of one row, the results + ** are already in the requested order. Set isOrdered to nOrderBy to + ** indicate this. Or, if nLoop is greater than zero, set isOrdered to + ** -1, indicating that the result set may or may not be ordered, + ** depending on the loops added to the current plan. */ + aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; + } + + /* Compute successively longer WherePaths using the previous generation + ** of WherePaths as the basis for the next. Keep track of the mxChoice + ** best paths at each generation */ + for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ + LogEst rCost; /* Cost of path (pFrom+pWLoop) */ + LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ + i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + Bitmask maskNew; /* Mask of src visited by (..) */ + Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + + if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + /* At this point, pWLoop is a candidate to be the next loop. + ** Compute its cost */ + rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); + nOut = pFrom->nRow + pWLoop->nOut; + maskNew = pFrom->maskLoop | pWLoop->maskSelf; + if( isOrdered<0 ){ + isOrdered = wherePathSatisfiesOrderBy(pWInfo, + pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, + iLoop, pWLoop, &revMask); + }else{ + revMask = pFrom->revLoop; + } + if( isOrdered>=0 && isOrderedisOrdered^isOrdered)&0x80)==0" is equivalent + ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range + ** of legal values for isOrdered, -1..64. + */ + for(jj=0, pTo=aTo; jjmaskLoop==maskNew + && ((pTo->isOrdered^isOrdered)&0x80)==0 + ){ + testcase( jj==nTo-1 ); + break; + } + } + if( jj>=nTo ){ + /* None of the existing best-so-far paths match the candidate. */ + if( nTo>=mxChoice + && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) + ){ + /* The current candidate is no better than any of the mxChoice + ** paths currently in the best-so-far buffer. So discard + ** this candidate as not viable. */ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + isOrdered>=0 ? isOrdered+'0' : '?'); + } +#endif + continue; + } + /* If we reach this points it means that the new candidate path + ** needs to be added to the set of best-so-far paths. */ + if( nTo=0 ? isOrdered+'0' : '?'); + } +#endif + }else{ + /* Control reaches here if best-so-far path pTo=aTo[jj] covers the + ** same set of loops and has the sam isOrdered setting as the + ** candidate path. Check to see if the candidate should replace + ** pTo or if the candidate should be skipped */ + if( pTo->rCostrCost==rCost && pTo->nRow<=nOut) ){ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf( + "Skip %s cost=%-3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + isOrdered>=0 ? isOrdered+'0' : '?'); + sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + } +#endif + /* Discard the candidate path from further consideration */ + testcase( pTo->rCost==rCost ); + continue; + } + testcase( pTo->rCost==rCost+1 ); + /* Control reaches here if the candidate path is better than the + ** pTo path. Replace pTo with the candidate. */ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf( + "Update %s cost=%-3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + isOrdered>=0 ? isOrdered+'0' : '?'); + sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + } +#endif + } + /* pWLoop is a winner. Add it to the set of best so far */ + pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; + pTo->revLoop = revMask; + pTo->nRow = nOut; + pTo->rCost = rCost; + pTo->rUnsorted = rUnsorted; + pTo->isOrdered = isOrdered; + memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); + pTo->aLoop[iLoop] = pWLoop; + if( nTo>=mxChoice ){ + mxI = 0; + mxCost = aTo[0].rCost; + mxUnsorted = aTo[0].nRow; + for(jj=1, pTo=&aTo[1]; jjrCost>mxCost + || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) + ){ + mxCost = pTo->rCost; + mxUnsorted = pTo->rUnsorted; + mxI = jj; + } + } + } + } + } + +#ifdef WHERETRACE_ENABLED /* >=2 */ + if( sqlite3WhereTrace>=2 ){ + sqlite3DebugPrintf("---- after round %d ----\n", iLoop); + for(ii=0, pTo=aTo; iirCost, pTo->nRow, + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ + sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); + }else{ + sqlite3DebugPrintf("\n"); + } + } + } +#endif + + /* Swap the roles of aFrom and aTo for the next generation */ + pFrom = aTo; + aTo = aFrom; + aFrom = pFrom; + nFrom = nTo; + } + + if( nFrom==0 ){ + sqlite3ErrorMsg(pParse, "no query solution"); + sqlite3DbFree(db, pSpace); + return SQLITE_ERROR; + } + + /* Find the lowest cost path. pFrom will be left pointing to that path */ + pFrom = aFrom; + for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; + } + assert( pWInfo->nLevel==nLoop ); + /* Load the lowest cost path into pWInfo */ + for(iLoop=0; iLoopa + iLoop; + pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; + pLevel->iFrom = pWLoop->iTab; + pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; + } + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 + && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 + && pWInfo->eDistinct==WHERE_DISTINCT_NOOP + && nRowEst + ){ + Bitmask notUsed; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, + WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); + if( rc==pWInfo->pResultSet->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } + } + if( pWInfo->pOrderBy ){ + if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ + if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } + }else{ + pWInfo->nOBSat = pFrom->isOrdered; + if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; + pWInfo->revMask = pFrom->revLoop; + } + if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) + && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr + ){ + Bitmask notUsed = 0; + int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, + pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used + ); + assert( pWInfo->sorted==0 ); + pWInfo->sorted = (nOrder==pWInfo->pOrderBy->nExpr); + } + } + + + pWInfo->nRowOut = pFrom->nRow; + + /* Free temporary memory and return success */ + sqlite3DbFree(db, pSpace); + return SQLITE_OK; +} + +/* +** Most queries use only a single table (they are not joins) and have +** simple == constraints against indexed fields. This routine attempts +** to plan those simple cases using much less ceremony than the +** general-purpose query planner, and thereby yield faster sqlite3_prepare() +** times for the common case. +** +** Return non-zero on success, if this query can be handled by this +** no-frills query planner. Return zero if this query needs the +** general-purpose query planner. +*/ +static int whereShortCut(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo; + struct SrcList_item *pItem; + WhereClause *pWC; + WhereTerm *pTerm; + WhereLoop *pLoop; + int iCur; + int j; + Table *pTab; + Index *pIdx; + + pWInfo = pBuilder->pWInfo; + if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0; + assert( pWInfo->pTabList->nSrc>=1 ); + pItem = pWInfo->pTabList->a; + pTab = pItem->pTab; + if( IsVirtual(pTab) ) return 0; + if( pItem->zIndex ) return 0; + iCur = pItem->iCursor; + pWC = &pWInfo->sWC; + pLoop = pBuilder->pNew; + pLoop->wsFlags = 0; + pLoop->u.btree.nSkip = 0; + pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); + if( pTerm ){ + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; + pLoop->aLTerm[0] = pTerm; + pLoop->nLTerm = 1; + pLoop->u.btree.nEq = 1; + /* TUNING: Cost of a rowid lookup is 10 */ + pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ + }else{ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pLoop->aLTermSpace==pLoop->aLTerm ); + assert( ArraySize(pLoop->aLTermSpace)==4 ); + if( !IsUniqueIndex(pIdx) + || pIdx->pPartIdxWhere!=0 + || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) + ) continue; + for(j=0; jnKeyCol; j++){ + pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx); + if( pTerm==0 ) break; + pLoop->aLTerm[j] = pTerm; + } + if( j!=pIdx->nKeyCol ) continue; + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; + if( pIdx->isCovering || (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ + pLoop->wsFlags |= WHERE_IDX_ONLY; + } + pLoop->nLTerm = j; + pLoop->u.btree.nEq = j; + pLoop->u.btree.pIndex = pIdx; + /* TUNING: Cost of a unique index lookup is 15 */ + pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */ + break; + } + } + if( pLoop->wsFlags ){ + pLoop->nOut = (LogEst)1; + pWInfo->a[0].pWLoop = pLoop; + pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); + pWInfo->a[0].iTabCur = iCur; + pWInfo->nRowOut = 1; + if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } +#ifdef SQLITE_DEBUG + pLoop->cId = '0'; +#endif + return 1; + } + return 0; +} + +/* +** Generate the beginning of the loop used for WHERE clause processing. +** The return value is a pointer to an opaque structure that contains +** information needed to terminate the loop. Later, the calling routine +** should invoke sqlite3WhereEnd() with the return value of this function +** in order to complete the WHERE clause processing. +** +** If an error occurs, this routine returns NULL. +** +** The basic idea is to do a nested loop, one loop for each table in +** the FROM clause of a select. (INSERT and UPDATE statements are the +** same as a SELECT with only a single table in the FROM clause.) For +** example, if the SQL is this: +** +** SELECT * FROM t1, t2, t3 WHERE ...; +** +** Then the code generated is conceptually like the following: +** +** foreach row1 in t1 do \ Code generated +** foreach row2 in t2 do |-- by sqlite3WhereBegin() +** foreach row3 in t3 do / +** ... +** end \ Code generated +** end |-- by sqlite3WhereEnd() +** end / +** +** Note that the loops might not be nested in the order in which they +** appear in the FROM clause if a different order is better able to make +** use of indices. Note also that when the IN operator appears in +** the WHERE clause, it might result in additional nested loops for +** scanning through all values on the right-hand side of the IN. +** +** There are Btree cursors associated with each table. t1 uses cursor +** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor. +** And so forth. This routine generates code to open those VDBE cursors +** and sqlite3WhereEnd() generates the code to close them. +** +** The code that sqlite3WhereBegin() generates leaves the cursors named +** in pTabList pointing at their appropriate entries. The [...] code +** can use OP_Column and OP_Rowid opcodes on these cursors to extract +** data from the various tables of the loop. +** +** If the WHERE clause is empty, the foreach loops must each scan their +** entire tables. Thus a three-way join is an O(N^3) operation. But if +** the tables have indices and there are terms in the WHERE clause that +** refer to those indices, a complete table scan can be avoided and the +** code will run much faster. Most of the work of this routine is checking +** to see if there are indices that can be used to speed up the loop. +** +** Terms of the WHERE clause are also used to limit which rows actually +** make it to the "..." in the middle of the loop. After each "foreach", +** terms of the WHERE clause that use only terms in that loop and outer +** loops are evaluated and if false a jump is made around all subsequent +** inner loops (or around the "..." if the test occurs within the inner- +** most loop) +** +** OUTER JOINS +** +** An outer join of tables t1 and t2 is conceptally coded as follows: +** +** foreach row1 in t1 do +** flag = 0 +** foreach row2 in t2 do +** start: +** ... +** flag = 1 +** end +** if flag==0 then +** move the row2 cursor to a null row +** goto start +** fi +** end +** +** ORDER BY CLAUSE PROCESSING +** +** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause +** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement +** if there is one. If there is no ORDER BY clause or if this routine +** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. +** +** The iIdxCur parameter is the cursor number of an index. If +** WHERE_ONETABLE_ONLY is set, iIdxCur is the cursor number of an index +** to use for OR clause processing. The WHERE clause should use this +** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is +** the first cursor in an array of cursors for all indices. iIdxCur should +** be used to compute the appropriate cursor depending on which index is +** used. +*/ +SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ + Expr *pWhere, /* The WHERE clause */ + ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ + ExprList *pResultSet, /* Result set of the query */ + u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ + int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ +){ + int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ + int nTabList; /* Number of elements in pTabList */ + WhereInfo *pWInfo; /* Will become the return value of this function */ + Vdbe *v = pParse->pVdbe; /* The virtual database engine */ + Bitmask notReady; /* Cursors that are not yet positioned */ + WhereLoopBuilder sWLB; /* The WhereLoop builder */ + WhereMaskSet *pMaskSet; /* The expression mask set */ + WhereLevel *pLevel; /* A single level in pWInfo->a[] */ + WhereLoop *pLoop; /* Pointer to a single WhereLoop object */ + int ii; /* Loop counter */ + sqlite3 *db; /* Database connection */ + int rc; /* Return code */ + + + /* Variable initialization */ + db = pParse->db; + memset(&sWLB, 0, sizeof(sWLB)); + + /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ + testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); + if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; + sWLB.pOrderBy = pOrderBy; + + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + wctrlFlags &= ~WHERE_WANT_DISTINCT; + } + + /* The number of tables in the FROM clause is limited by the number of + ** bits in a Bitmask + */ + testcase( pTabList->nSrc==BMS ); + if( pTabList->nSrc>BMS ){ + sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS); + return 0; + } + + /* This function normally generates a nested loop for all tables in + ** pTabList. But if the WHERE_ONETABLE_ONLY flag is set, then we should + ** only generate code for the first table in pTabList and assume that + ** any cursors associated with subsequent tables are uninitialized. + */ + nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc; + + /* Allocate and initialize the WhereInfo structure that will become the + ** return value. A single allocation is used to store the WhereInfo + ** struct, the contents of WhereInfo.a[], the WhereClause structure + ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte + ** field (type Bitmask) it must be aligned on an 8-byte boundary on + ** some architectures. Hence the ROUND8() below. + */ + nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); + pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereLoop)); + if( db->mallocFailed ){ + sqlite3DbFree(db, pWInfo); + pWInfo = 0; + goto whereBeginError; + } + pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; + pWInfo->nLevel = nTabList; + pWInfo->pParse = pParse; + pWInfo->pTabList = pTabList; + pWInfo->pOrderBy = pOrderBy; + pWInfo->pResultSet = pResultSet; + pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); + pWInfo->wctrlFlags = wctrlFlags; + pWInfo->savedNQueryLoop = pParse->nQueryLoop; + pMaskSet = &pWInfo->sMaskSet; + sWLB.pWInfo = pWInfo; + sWLB.pWC = &pWInfo->sWC; + sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo); + assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) ); + whereLoopInit(sWLB.pNew); +#ifdef SQLITE_DEBUG + sWLB.pNew->cId = '*'; +#endif + + /* Split the WHERE clause into separate subexpressions where each + ** subexpression is separated by an AND operator. + */ + initMaskSet(pMaskSet); + whereClauseInit(&pWInfo->sWC, pWInfo); + whereSplit(&pWInfo->sWC, pWhere, TK_AND); + + /* Special case: a WHERE clause that is constant. Evaluate the + ** expression and either jump over all of the code or fall thru. + */ + for(ii=0; iinTerm; ii++){ + if( nTabList==0 || sqlite3ExprIsConstantNotJoin(sWLB.pWC->a[ii].pExpr) ){ + sqlite3ExprIfFalse(pParse, sWLB.pWC->a[ii].pExpr, pWInfo->iBreak, + SQLITE_JUMPIFNULL); + sWLB.pWC->a[ii].wtFlags |= TERM_CODED; + } + } + + /* Special case: No FROM clause + */ + if( nTabList==0 ){ + if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } + } + + /* Assign a bit from the bitmask to every term in the FROM clause. + ** + ** When assigning bitmask values to FROM clause cursors, it must be + ** the case that if X is the bitmask for the N-th FROM clause term then + ** the bitmask for all FROM clause terms to the left of the N-th term + ** is (X-1). An expression from the ON clause of a LEFT JOIN can use + ** its Expr.iRightJoinTable value to find the bitmask of the right table + ** of the join. Subtracting one from the right table bitmask gives a + ** bitmask for all tables to the left of the join. Knowing the bitmask + ** for all tables to the left of a left join is important. Ticket #3015. + ** + ** Note that bitmasks are created for all pTabList->nSrc tables in + ** pTabList, not just the first nTabList tables. nTabList is normally + ** equal to pTabList->nSrc but might be shortened to 1 if the + ** WHERE_ONETABLE_ONLY flag is set. + */ + for(ii=0; iinSrc; ii++){ + createMask(pMaskSet, pTabList->a[ii].iCursor); + } +#ifndef NDEBUG + { + Bitmask toTheLeft = 0; + for(ii=0; iinSrc; ii++){ + Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); + assert( (m-1)==toTheLeft ); + toTheLeft |= m; + } + } +#endif + + /* Analyze all of the subexpressions. Note that exprAnalyze() might + ** add new virtual terms onto the end of the WHERE clause. We do not + ** want to analyze these virtual terms, so start analyzing at the end + ** and work forward so that the added virtual terms are never processed. + */ + exprAnalyzeAll(pTabList, &pWInfo->sWC); + if( db->mallocFailed ){ + goto whereBeginError; + } + + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ + /* The DISTINCT marking is pointless. Ignore it. */ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + }else if( pOrderBy==0 ){ + /* Try to ORDER BY the result set to make distinct processing easier */ + pWInfo->wctrlFlags |= WHERE_DISTINCTBY; + pWInfo->pOrderBy = pResultSet; + } + } + + /* Construct the WhereLoop objects */ + WHERETRACE(0xffff,("*** Optimizer Start ***\n")); + /* Display all terms of the WHERE clause */ +#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN) + if( sqlite3WhereTrace & 0x100 ){ + int i; + Vdbe *v = pParse->pVdbe; + sqlite3ExplainBegin(v); + for(i=0; inTerm; i++){ + sqlite3ExplainPrintf(v, "#%-2d ", i); + sqlite3ExplainPush(v); + whereExplainTerm(v, &sWLB.pWC->a[i]); + sqlite3ExplainPop(v); + sqlite3ExplainNL(v); + } + sqlite3ExplainFinish(v); + sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v)); + } +#endif + if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; + + /* Display all of the WhereLoop objects if wheretrace is enabled */ +#ifdef WHERETRACE_ENABLED /* !=0 */ + if( sqlite3WhereTrace ){ + WhereLoop *p; + int i; + static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ + p->cId = zLabel[i%sizeof(zLabel)]; + whereLoopPrint(p, sWLB.pWC); + } + } +#endif + + wherePathSolver(pWInfo, 0); + if( db->mallocFailed ) goto whereBeginError; + if( pWInfo->pOrderBy ){ + wherePathSolver(pWInfo, pWInfo->nRowOut+1); + if( db->mallocFailed ) goto whereBeginError; + } + } + if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ + pWInfo->revMask = (Bitmask)(-1); + } + if( pParse->nErr || NEVER(db->mallocFailed) ){ + goto whereBeginError; + } +#ifdef WHERETRACE_ENABLED /* !=0 */ + if( sqlite3WhereTrace ){ + int ii; + sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); + if( pWInfo->nOBSat>0 ){ + sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); + } + switch( pWInfo->eDistinct ){ + case WHERE_DISTINCT_UNIQUE: { + sqlite3DebugPrintf(" DISTINCT=unique"); + break; + } + case WHERE_DISTINCT_ORDERED: { + sqlite3DebugPrintf(" DISTINCT=ordered"); + break; + } + case WHERE_DISTINCT_UNORDERED: { + sqlite3DebugPrintf(" DISTINCT=unordered"); + break; + } + } + sqlite3DebugPrintf("\n"); + for(ii=0; iinLevel; ii++){ + whereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); + } + } +#endif + /* Attempt to omit tables from the join that do not effect the result */ + if( pWInfo->nLevel>=2 + && pResultSet!=0 + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + ){ + Bitmask tabUsed = exprListTableUsage(pMaskSet, pResultSet); + if( sWLB.pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, sWLB.pOrderBy); + while( pWInfo->nLevel>=2 ){ + WhereTerm *pTerm, *pEnd; + pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; + if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 + && (pLoop->wsFlags & WHERE_ONEROW)==0 + ){ + break; + } + if( (tabUsed & pLoop->maskSelf)!=0 ) break; + pEnd = sWLB.pWC->a + sWLB.pWC->nTerm; + for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + ){ + break; + } + } + if( pTerm drop loop %c not used\n", pLoop->cId)); + pWInfo->nLevel--; + nTabList--; + } + } + WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); + pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; + + /* If the caller is an UPDATE or DELETE statement that is requesting + ** to use a one-pass algorithm, determine if this is appropriate. + ** The one-pass algorithm only works if the WHERE clause constrains + ** the statement to update a single row. + */ + assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ + pWInfo->okOnePass = 1; + if( HasRowid(pTabList->a[0].pTab) ){ + pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + } + } + + /* Open all tables in the pTabList and any indices selected for + ** searching those tables. + */ + notReady = ~(Bitmask)0; + for(ii=0, pLevel=pWInfo->a; iia[pLevel->iFrom]; + pTab = pTabItem->pTab; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pLoop = pLevel->pWLoop; + if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ + /* Do nothing */ + }else +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + int iCur = pTabItem->iCursor; + sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); + }else if( IsVirtual(pTab) ){ + /* noop */ + }else +#endif + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ + int op = OP_OpenRead; + if( pWInfo->okOnePass ){ + op = OP_OpenWrite; + pWInfo->aiCurOnePass[0] = pTabItem->iCursor; + }; + sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); + assert( pTabItem->iCursor==pLevel->iTabCur ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS ); + if( !pWInfo->okOnePass && pTab->nColcolUsed; + int n = 0; + for(; b; b=b>>1, n++){} + sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, + SQLITE_INT_TO_PTR(n), P4_INT32); + assert( n<=pTab->nCol ); + } + }else{ + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + } + if( pLoop->wsFlags & WHERE_INDEXED ){ + Index *pIx = pLoop->u.btree.pIndex; + int iIndexCur; + int op = OP_OpenRead; + /* iIdxCur is always set if to a positive value if ONEPASS is possible */ + assert( iIdxCur!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) + && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 + ){ + /* This is one term of an OR-optimization using the PRIMARY KEY of a + ** WITHOUT ROWID table. No need for a separate index */ + iIndexCur = pLevel->iTabCur; + op = 0; + }else if( pWInfo->okOnePass ){ + Index *pJ = pTabItem->pTab->pIndex; + iIndexCur = iIdxCur; + assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); + while( ALWAYS(pJ) && pJ!=pIx ){ + iIndexCur++; + pJ = pJ->pNext; + } + op = OP_OpenWrite; + pWInfo->aiCurOnePass[1] = iIndexCur; + }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ + iIndexCur = iIdxCur; + if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; + }else{ + iIndexCur = pParse->nTab++; + } + pLevel->iIdxCur = iIndexCur; + assert( pIx->pSchema==pTab->pSchema ); + assert( iIndexCur>=0 ); + if( op ){ + sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIx); + VdbeComment((v, "%s", pIx->zName)); + } + } + if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); + notReady &= ~getMask(&pWInfo->sMaskSet, pTabItem->iCursor); + } + pWInfo->iTop = sqlite3VdbeCurrentAddr(v); + if( db->mallocFailed ) goto whereBeginError; + + /* Generate the code to do the search. Each iteration of the for + ** loop below generates code for a single nested loop of the VM + ** program. + */ + notReady = ~(Bitmask)0; + for(ii=0; iia[ii]; +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + constructAutomaticIndex(pParse, &pWInfo->sWC, + &pTabList->a[pLevel->iFrom], notReady, pLevel); + if( db->mallocFailed ) goto whereBeginError; + } +#endif + explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + pLevel->addrBody = sqlite3VdbeCurrentAddr(v); + notReady = codeOneLoopStart(pWInfo, ii, notReady); + pWInfo->iContinue = pLevel->addrCont; + } + + /* Done. */ + VdbeModuleComment((v, "Begin WHERE-core")); + return pWInfo; + + /* Jump here if malloc fails */ +whereBeginError: + if( pWInfo ){ + pParse->nQueryLoop = pWInfo->savedNQueryLoop; + whereInfoFree(db, pWInfo); + } + return 0; +} + +/* +** Generate the end of the WHERE loop. See comments on +** sqlite3WhereBegin() for additional information. +*/ +SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ + Parse *pParse = pWInfo->pParse; + Vdbe *v = pParse->pVdbe; + int i; + WhereLevel *pLevel; + WhereLoop *pLoop; + SrcList *pTabList = pWInfo->pTabList; + sqlite3 *db = pParse->db; + + /* Generate loop termination code. + */ + VdbeModuleComment((v, "End WHERE-core")); + sqlite3ExprCacheClear(pParse); + for(i=pWInfo->nLevel-1; i>=0; i--){ + int addr; + pLevel = &pWInfo->a[i]; + pLoop = pLevel->pWLoop; + sqlite3VdbeResolveLabel(v, pLevel->addrCont); + if( pLevel->op!=OP_Noop ){ + sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); + sqlite3VdbeChangeP5(v, pLevel->p5); + VdbeCoverage(v); + VdbeCoverageIf(v, pLevel->op==OP_Next); + VdbeCoverageIf(v, pLevel->op==OP_Prev); + VdbeCoverageIf(v, pLevel->op==OP_VNext); + } + if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ + struct InLoop *pIn; + int j; + sqlite3VdbeResolveLabel(v, pLevel->addrNxt); + for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ + sqlite3VdbeJumpHere(v, pIn->addrInTop+1); + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); + VdbeCoverage(v); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); + sqlite3VdbeJumpHere(v, pIn->addrInTop-1); + } + sqlite3DbFree(db, pLevel->u.in.aInLoop); + } + sqlite3VdbeResolveLabel(v, pLevel->addrBrk); + if( pLevel->addrSkip ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); + sqlite3VdbeJumpHere(v, pLevel->addrSkip); + sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); + } + if( pLevel->iLeftJoin ){ + addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){ + sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); + } + if( pLoop->wsFlags & WHERE_INDEXED ){ + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); + } + if( pLevel->op==OP_Return ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrFirst); + } + sqlite3VdbeJumpHere(v, addr); + } + VdbeModuleComment((v, "End WHERE-loop%d: %s", i, + pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); + } + + /* The "break" point is here, just past the end of the outer loop. + ** Set it. + */ + sqlite3VdbeResolveLabel(v, pWInfo->iBreak); + + assert( pWInfo->nLevel<=pTabList->nSrc ); + for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ + int k, last; + VdbeOp *pOp; + Index *pIdx = 0; + struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; + Table *pTab = pTabItem->pTab; + assert( pTab!=0 ); + pLoop = pLevel->pWLoop; + + /* For a co-routine, change all OP_Column references to the table of + ** the co-routine into OP_SCopy of result contained in a register. + ** OP_Rowid becomes OP_Null. + */ + if( pTabItem->viaCoroutine && !db->mallocFailed ){ + last = sqlite3VdbeCurrentAddr(v); + k = pLevel->addrBody; + pOp = sqlite3VdbeGetOp(v, k); + for(; kp1!=pLevel->iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + pOp->opcode = OP_Copy; + pOp->p1 = pOp->p2 + pTabItem->regResult; + pOp->p2 = pOp->p3; + pOp->p3 = 0; + }else if( pOp->opcode==OP_Rowid ){ + pOp->opcode = OP_Null; + pOp->p1 = 0; + pOp->p3 = 0; + } + } + continue; + } + + /* Close all of the cursors that were opened by sqlite3WhereBegin. + ** Except, do not close cursors that will be reused by the OR optimization + ** (WHERE_OMIT_OPEN_CLOSE). And do not close the OP_OpenWrite cursors + ** created for the ONEPASS optimization. + */ + if( (pTab->tabFlags & TF_Ephemeral)==0 + && pTab->pSelect==0 + && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 + ){ + int ws = pLoop->wsFlags; + if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ + sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); + } + if( (ws & WHERE_INDEXED)!=0 + && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 + && pLevel->iIdxCur!=pWInfo->aiCurOnePass[1] + ){ + sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); + } + } + + /* If this scan uses an index, make VDBE code substitutions to read data + ** from the index instead of from the table where possible. In some cases + ** this optimization prevents the table from ever being read, which can + ** yield a significant performance boost. + ** + ** Calls to the code generator in between sqlite3WhereBegin and + ** sqlite3WhereEnd will have created code that references the table + ** directly. This loop scans all that code looking for opcodes + ** that reference the table and converts them into opcodes that + ** reference the index. + */ + if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ + pIdx = pLoop->u.btree.pIndex; + }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ + pIdx = pLevel->u.pCovidx; + } + if( pIdx && !db->mallocFailed ){ + last = sqlite3VdbeCurrentAddr(v); + k = pLevel->addrBody; + pOp = sqlite3VdbeGetOp(v, k); + for(; kp1!=pLevel->iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + int x = pOp->p2; + assert( pIdx->pTable==pTab ); + if( !HasRowid(pTab) ){ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + x = pPk->aiColumn[x]; + } + x = sqlite3ColumnOfIndex(pIdx, x); + if( x>=0 ){ + pOp->p2 = x; + pOp->p1 = pLevel->iIdxCur; + } + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 ); + }else if( pOp->opcode==OP_Rowid ){ + pOp->p1 = pLevel->iIdxCur; + pOp->opcode = OP_IdxRowid; + } + } + } + } + + /* Final cleanup + */ + pParse->nQueryLoop = pWInfo->savedNQueryLoop; + whereInfoFree(db, pWInfo); + return; +} + +/************** End of where.c ***********************************************/ +/************** Begin file parse.c *******************************************/ +/* Driver template for the LEMON parser generator. +** The author disclaims copyright to this source code. +** +** This version of "lempar.c" is modified, slightly, for use by SQLite. +** The only modifications are the addition of a couple of NEVER() +** macros to disable tests that are needed in the case of a general +** LALR(1) grammar but which are always false in the +** specific grammar used by SQLite. +*/ +/* First off, code is included that follows the "include" declaration +** in the input grammar file. */ +/* #include */ + + +/* +** Disable all error recovery processing in the parser push-down +** automaton. +*/ +#define YYNOERRORRECOVERY 1 + +/* +** Make yytestcase() the same as testcase() +*/ +#define yytestcase(X) testcase(X) + +/* +** An instance of this structure holds information about the +** LIMIT clause of a SELECT statement. +*/ +struct LimitVal { + Expr *pLimit; /* The LIMIT expression. NULL if there is no limit */ + Expr *pOffset; /* The OFFSET expression. NULL if there is none */ +}; + +/* +** An instance of this structure is used to store the LIKE, +** GLOB, NOT LIKE, and NOT GLOB operators. +*/ +struct LikeOp { + Token eOperator; /* "like" or "glob" or "regexp" */ + int bNot; /* True if the NOT keyword is present */ +}; + +/* +** An instance of the following structure describes the event of a +** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, +** TK_DELETE, or TK_INSTEAD. If the event is of the form +** +** UPDATE ON (a,b,c) +** +** Then the "b" IdList records the list "a,b,c". +*/ +struct TrigEvent { int a; IdList * b; }; + +/* +** An instance of this structure holds the ATTACH key and the key type. +*/ +struct AttachKey { int type; Token key; }; + + + /* This is a utility routine used to set the ExprSpan.zStart and + ** ExprSpan.zEnd values of pOut so that the span covers the complete + ** range of text beginning with pStart and going to the end of pEnd. + */ + static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){ + pOut->zStart = pStart->z; + pOut->zEnd = &pEnd->z[pEnd->n]; + } + + /* Construct a new Expr object from a single identifier. Use the + ** new Expr to populate pOut. Set the span of pOut to be the identifier + ** that created the expression. + */ + static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){ + pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, pValue); + pOut->zStart = pValue->z; + pOut->zEnd = &pValue->z[pValue->n]; + } + + /* This routine constructs a binary expression node out of two ExprSpan + ** objects and uses the result to populate a new ExprSpan object. + */ + static void spanBinaryExpr( + ExprSpan *pOut, /* Write the result here */ + Parse *pParse, /* The parsing context. Errors accumulate here */ + int op, /* The binary operation */ + ExprSpan *pLeft, /* The left operand */ + ExprSpan *pRight /* The right operand */ + ){ + pOut->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); + pOut->zStart = pLeft->zStart; + pOut->zEnd = pRight->zEnd; + } + + /* Construct an expression node for a unary postfix operator + */ + static void spanUnaryPostfix( + ExprSpan *pOut, /* Write the new expression node here */ + Parse *pParse, /* Parsing context to record errors */ + int op, /* The operator */ + ExprSpan *pOperand, /* The operand */ + Token *pPostOp /* The operand token for setting the span */ + ){ + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOut->zStart = pOperand->zStart; + pOut->zEnd = &pPostOp->z[pPostOp->n]; + } + + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a + ** unary TK_ISNULL or TK_NOTNULL expression. */ + static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ + sqlite3 *db = pParse->db; + if( db->mallocFailed==0 && pY->op==TK_NULL ){ + pA->op = (u8)op; + sqlite3ExprDelete(db, pA->pRight); + pA->pRight = 0; + } + } + + /* Construct an expression node for a unary prefix operator + */ + static void spanUnaryPrefix( + ExprSpan *pOut, /* Write the new expression node here */ + Parse *pParse, /* Parsing context to record errors */ + int op, /* The operator */ + ExprSpan *pOperand, /* The operand */ + Token *pPreOp /* The operand token for setting the span */ + ){ + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOut->zStart = pPreOp->z; + pOut->zEnd = pOperand->zEnd; + } +/* Next is all token values, in a form suitable for use by makeheaders. +** This section will be null unless lemon is run with the -m switch. +*/ +/* +** These constants (all generated automatically by the parser generator) +** specify the various kinds of tokens (terminals) that the parser +** understands. +** +** Each symbol here is a terminal symbol in the grammar. +*/ +/* Make sure the INTERFACE macro is defined. +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/* The next thing included is series of defines which control +** various aspects of the generated parser. +** YYCODETYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 terminals +** and nonterminals. "int" is used otherwise. +** YYNOCODE is a number of type YYCODETYPE which corresponds +** to no legal terminal or nonterminal number. This +** number is used to fill in empty slots of the hash +** table. +** YYFALLBACK If defined, this indicates that one or more tokens +** have fall-back values which should be used if the +** original value of the token will not parse. +** YYACTIONTYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 rules and +** states combined. "int" is used otherwise. +** sqlite3ParserTOKENTYPE is the data type used for minor tokens given +** directly to the parser from the tokenizer. +** YYMINORTYPE is the data type used for all minor tokens. +** This is typically a union of many types, one of +** which is sqlite3ParserTOKENTYPE. The entry in the union +** for base tokens is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument +** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser +** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +*/ +#define YYCODETYPE unsigned char +#define YYNOCODE 254 +#define YYACTIONTYPE unsigned short int +#define YYWILDCARD 70 +#define sqlite3ParserTOKENTYPE Token +typedef union { + int yyinit; + sqlite3ParserTOKENTYPE yy0; + Select* yy3; + ExprList* yy14; + With* yy59; + SrcList* yy65; + struct LikeOp yy96; + Expr* yy132; + u8 yy186; + int yy328; + ExprSpan yy346; + struct TrigEvent yy378; + u16 yy381; + IdList* yy408; + struct {int value; int mask;} yy429; + TriggerStep* yy473; + struct LimitVal yy476; +} YYMINORTYPE; +#ifndef YYSTACKDEPTH +#define YYSTACKDEPTH 100 +#endif +#define sqlite3ParserARG_SDECL Parse *pParse; +#define sqlite3ParserARG_PDECL ,Parse *pParse +#define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse +#define sqlite3ParserARG_STORE yypParser->pParse = pParse +#define YYNSTATE 642 +#define YYNRULE 327 +#define YYFALLBACK 1 +#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) +#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) +#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) + +/* The yyzerominor constant is used to initialize instances of +** YYMINORTYPE objects to zero. */ +static const YYMINORTYPE yyzerominor = { 0 }; + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** +** N == YYNSTATE+YYNRULE A syntax error has occurred. +** +** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** +** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** slots in the yy_action[] table. +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as +** +** yy_action[ yy_shift_ofst[S] + X ] +** +** If the index value yy_shift_ofst[S]+X is out of range or if the value +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] +** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table +** and that yy_default[S] should be used instead. +** +** The formula above is for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of +** YY_SHIFT_USE_DFLT. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +*/ +#define YY_ACTTAB_COUNT (1497) +static const YYACTIONTYPE yy_action[] = { + /* 0 */ 306, 212, 432, 955, 639, 191, 955, 295, 559, 88, + /* 10 */ 88, 88, 88, 81, 86, 86, 86, 86, 85, 85, + /* 20 */ 84, 84, 84, 83, 330, 185, 184, 183, 635, 635, + /* 30 */ 292, 606, 606, 88, 88, 88, 88, 683, 86, 86, + /* 40 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 16, + /* 50 */ 436, 597, 89, 90, 80, 600, 599, 601, 601, 87, + /* 60 */ 87, 88, 88, 88, 88, 684, 86, 86, 86, 86, + /* 70 */ 85, 85, 84, 84, 84, 83, 330, 306, 559, 84, + /* 80 */ 84, 84, 83, 330, 65, 86, 86, 86, 86, 85, + /* 90 */ 85, 84, 84, 84, 83, 330, 635, 635, 634, 633, + /* 100 */ 182, 682, 550, 379, 376, 375, 17, 322, 606, 606, + /* 110 */ 371, 198, 479, 91, 374, 82, 79, 165, 85, 85, + /* 120 */ 84, 84, 84, 83, 330, 598, 635, 635, 107, 89, + /* 130 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, + /* 140 */ 88, 88, 186, 86, 86, 86, 86, 85, 85, 84, + /* 150 */ 84, 84, 83, 330, 306, 594, 594, 142, 328, 327, + /* 160 */ 484, 249, 344, 238, 635, 635, 634, 633, 585, 448, + /* 170 */ 526, 525, 229, 388, 1, 394, 450, 584, 449, 635, + /* 180 */ 635, 635, 635, 319, 395, 606, 606, 199, 157, 273, + /* 190 */ 382, 268, 381, 187, 635, 635, 634, 633, 311, 555, + /* 200 */ 266, 593, 593, 266, 347, 588, 89, 90, 80, 600, + /* 210 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 478, + /* 220 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, + /* 230 */ 330, 306, 272, 536, 634, 633, 146, 610, 197, 310, + /* 240 */ 575, 182, 482, 271, 379, 376, 375, 506, 21, 634, + /* 250 */ 633, 634, 633, 635, 635, 374, 611, 574, 548, 440, + /* 260 */ 111, 563, 606, 606, 634, 633, 324, 479, 608, 608, + /* 270 */ 608, 300, 435, 573, 119, 407, 210, 162, 562, 883, + /* 280 */ 592, 592, 306, 89, 90, 80, 600, 599, 601, 601, + /* 290 */ 87, 87, 88, 88, 88, 88, 506, 86, 86, 86, + /* 300 */ 86, 85, 85, 84, 84, 84, 83, 330, 620, 111, + /* 310 */ 635, 635, 361, 606, 606, 358, 249, 349, 248, 433, + /* 320 */ 243, 479, 586, 634, 633, 195, 611, 93, 119, 221, + /* 330 */ 575, 497, 534, 534, 89, 90, 80, 600, 599, 601, + /* 340 */ 601, 87, 87, 88, 88, 88, 88, 574, 86, 86, + /* 350 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 306, + /* 360 */ 77, 429, 638, 573, 589, 530, 240, 230, 242, 105, + /* 370 */ 249, 349, 248, 515, 588, 208, 460, 529, 564, 173, + /* 380 */ 634, 633, 970, 144, 430, 2, 424, 228, 380, 557, + /* 390 */ 606, 606, 190, 153, 159, 158, 514, 51, 632, 631, + /* 400 */ 630, 71, 536, 432, 954, 196, 610, 954, 614, 45, + /* 410 */ 18, 89, 90, 80, 600, 599, 601, 601, 87, 87, + /* 420 */ 88, 88, 88, 88, 261, 86, 86, 86, 86, 85, + /* 430 */ 85, 84, 84, 84, 83, 330, 306, 608, 608, 608, + /* 440 */ 542, 424, 402, 385, 241, 506, 451, 320, 211, 543, + /* 450 */ 164, 436, 386, 293, 451, 587, 108, 496, 111, 334, + /* 460 */ 391, 591, 424, 614, 27, 452, 453, 606, 606, 72, + /* 470 */ 257, 70, 259, 452, 339, 342, 564, 582, 68, 415, + /* 480 */ 469, 328, 327, 62, 614, 45, 110, 393, 89, 90, + /* 490 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, + /* 500 */ 88, 152, 86, 86, 86, 86, 85, 85, 84, 84, + /* 510 */ 84, 83, 330, 306, 110, 499, 520, 538, 402, 389, + /* 520 */ 424, 110, 566, 500, 593, 593, 454, 82, 79, 165, + /* 530 */ 424, 591, 384, 564, 340, 615, 188, 162, 424, 350, + /* 540 */ 616, 424, 614, 44, 606, 606, 445, 582, 300, 434, + /* 550 */ 151, 19, 614, 9, 568, 580, 348, 615, 469, 567, + /* 560 */ 614, 26, 616, 614, 45, 89, 90, 80, 600, 599, + /* 570 */ 601, 601, 87, 87, 88, 88, 88, 88, 411, 86, + /* 580 */ 86, 86, 86, 85, 85, 84, 84, 84, 83, 330, + /* 590 */ 306, 579, 110, 578, 521, 282, 433, 398, 400, 255, + /* 600 */ 486, 82, 79, 165, 487, 164, 82, 79, 165, 488, + /* 610 */ 488, 364, 387, 424, 544, 544, 509, 350, 362, 155, + /* 620 */ 191, 606, 606, 559, 642, 640, 333, 82, 79, 165, + /* 630 */ 305, 564, 507, 312, 357, 614, 45, 329, 596, 595, + /* 640 */ 194, 337, 89, 90, 80, 600, 599, 601, 601, 87, + /* 650 */ 87, 88, 88, 88, 88, 424, 86, 86, 86, 86, + /* 660 */ 85, 85, 84, 84, 84, 83, 330, 306, 20, 323, + /* 670 */ 150, 263, 211, 543, 421, 596, 595, 614, 22, 424, + /* 680 */ 193, 424, 284, 424, 391, 424, 509, 424, 577, 424, + /* 690 */ 186, 335, 424, 559, 424, 313, 120, 546, 606, 606, + /* 700 */ 67, 614, 47, 614, 50, 614, 48, 614, 100, 614, + /* 710 */ 99, 614, 101, 576, 614, 102, 614, 109, 326, 89, + /* 720 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, + /* 730 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, + /* 740 */ 84, 84, 83, 330, 306, 424, 311, 424, 585, 54, + /* 750 */ 424, 516, 517, 590, 614, 112, 424, 584, 424, 572, + /* 760 */ 424, 195, 424, 571, 424, 67, 424, 614, 94, 614, + /* 770 */ 98, 424, 614, 97, 264, 606, 606, 195, 614, 46, + /* 780 */ 614, 96, 614, 30, 614, 49, 614, 115, 614, 114, + /* 790 */ 418, 229, 388, 614, 113, 306, 89, 90, 80, 600, + /* 800 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 424, + /* 810 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, + /* 820 */ 330, 119, 424, 590, 110, 372, 606, 606, 195, 53, + /* 830 */ 250, 614, 29, 195, 472, 438, 729, 190, 302, 498, + /* 840 */ 14, 523, 641, 2, 614, 43, 306, 89, 90, 80, + /* 850 */ 600, 599, 601, 601, 87, 87, 88, 88, 88, 88, + /* 860 */ 424, 86, 86, 86, 86, 85, 85, 84, 84, 84, + /* 870 */ 83, 330, 424, 613, 964, 964, 354, 606, 606, 420, + /* 880 */ 312, 64, 614, 42, 391, 355, 283, 437, 301, 255, + /* 890 */ 414, 410, 495, 492, 614, 28, 471, 306, 89, 90, + /* 900 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, + /* 910 */ 88, 424, 86, 86, 86, 86, 85, 85, 84, 84, + /* 920 */ 84, 83, 330, 424, 110, 110, 110, 110, 606, 606, + /* 930 */ 110, 254, 13, 614, 41, 532, 531, 283, 481, 531, + /* 940 */ 457, 284, 119, 561, 356, 614, 40, 284, 306, 89, + /* 950 */ 78, 80, 600, 599, 601, 601, 87, 87, 88, 88, + /* 960 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, + /* 970 */ 84, 84, 83, 330, 110, 424, 341, 220, 555, 606, + /* 980 */ 606, 351, 555, 318, 614, 95, 413, 255, 83, 330, + /* 990 */ 284, 284, 255, 640, 333, 356, 255, 614, 39, 306, + /* 1000 */ 356, 90, 80, 600, 599, 601, 601, 87, 87, 88, + /* 1010 */ 88, 88, 88, 424, 86, 86, 86, 86, 85, 85, + /* 1020 */ 84, 84, 84, 83, 330, 424, 317, 316, 141, 465, + /* 1030 */ 606, 606, 219, 619, 463, 614, 10, 417, 462, 255, + /* 1040 */ 189, 510, 553, 351, 207, 363, 161, 614, 38, 315, + /* 1050 */ 218, 255, 255, 80, 600, 599, 601, 601, 87, 87, + /* 1060 */ 88, 88, 88, 88, 424, 86, 86, 86, 86, 85, + /* 1070 */ 85, 84, 84, 84, 83, 330, 76, 419, 255, 3, + /* 1080 */ 878, 461, 424, 247, 331, 331, 614, 37, 217, 76, + /* 1090 */ 419, 390, 3, 216, 215, 422, 4, 331, 331, 424, + /* 1100 */ 547, 12, 424, 545, 614, 36, 424, 541, 422, 424, + /* 1110 */ 540, 424, 214, 424, 408, 424, 539, 403, 605, 605, + /* 1120 */ 237, 614, 25, 119, 614, 24, 588, 408, 614, 45, + /* 1130 */ 118, 614, 35, 614, 34, 614, 33, 614, 23, 588, + /* 1140 */ 60, 223, 603, 602, 513, 378, 73, 74, 140, 139, + /* 1150 */ 424, 110, 265, 75, 426, 425, 59, 424, 610, 73, + /* 1160 */ 74, 549, 402, 404, 424, 373, 75, 426, 425, 604, + /* 1170 */ 138, 610, 614, 11, 392, 76, 419, 181, 3, 614, + /* 1180 */ 32, 271, 369, 331, 331, 493, 614, 31, 149, 608, + /* 1190 */ 608, 608, 607, 15, 422, 365, 614, 8, 137, 489, + /* 1200 */ 136, 190, 608, 608, 608, 607, 15, 485, 176, 135, + /* 1210 */ 7, 252, 477, 408, 174, 133, 175, 474, 57, 56, + /* 1220 */ 132, 130, 119, 76, 419, 588, 3, 468, 245, 464, + /* 1230 */ 171, 331, 331, 125, 123, 456, 447, 122, 446, 104, + /* 1240 */ 336, 231, 422, 166, 154, 73, 74, 332, 116, 431, + /* 1250 */ 121, 309, 75, 426, 425, 222, 106, 610, 308, 637, + /* 1260 */ 204, 408, 629, 627, 628, 6, 200, 428, 427, 290, + /* 1270 */ 203, 622, 201, 588, 62, 63, 289, 66, 419, 399, + /* 1280 */ 3, 401, 288, 92, 143, 331, 331, 287, 608, 608, + /* 1290 */ 608, 607, 15, 73, 74, 227, 422, 325, 69, 416, + /* 1300 */ 75, 426, 425, 612, 412, 610, 192, 61, 569, 209, + /* 1310 */ 396, 226, 278, 225, 383, 408, 527, 558, 276, 533, + /* 1320 */ 552, 528, 321, 523, 370, 508, 180, 588, 494, 179, + /* 1330 */ 366, 117, 253, 269, 522, 503, 608, 608, 608, 607, + /* 1340 */ 15, 551, 502, 58, 274, 524, 178, 73, 74, 304, + /* 1350 */ 501, 368, 303, 206, 75, 426, 425, 491, 360, 610, + /* 1360 */ 213, 177, 483, 131, 345, 298, 297, 296, 202, 294, + /* 1370 */ 480, 490, 466, 134, 172, 129, 444, 346, 470, 128, + /* 1380 */ 314, 459, 103, 127, 126, 148, 124, 167, 443, 235, + /* 1390 */ 608, 608, 608, 607, 15, 442, 439, 623, 234, 299, + /* 1400 */ 145, 583, 291, 377, 581, 160, 119, 156, 270, 636, + /* 1410 */ 971, 169, 279, 626, 520, 625, 473, 624, 170, 621, + /* 1420 */ 618, 119, 168, 55, 409, 423, 537, 609, 286, 285, + /* 1430 */ 405, 570, 560, 556, 5, 52, 458, 554, 147, 267, + /* 1440 */ 519, 504, 518, 406, 262, 239, 260, 512, 343, 511, + /* 1450 */ 258, 353, 565, 256, 224, 251, 359, 277, 275, 476, + /* 1460 */ 475, 246, 352, 244, 467, 455, 236, 233, 232, 307, + /* 1470 */ 441, 281, 205, 163, 397, 280, 535, 505, 330, 617, + /* 1480 */ 971, 971, 971, 971, 367, 971, 971, 971, 971, 971, + /* 1490 */ 971, 971, 971, 971, 971, 971, 338, +}; +static const YYCODETYPE yy_lookahead[] = { + /* 0 */ 19, 22, 22, 23, 1, 24, 26, 15, 27, 80, + /* 10 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + /* 20 */ 91, 92, 93, 94, 95, 108, 109, 110, 27, 28, + /* 30 */ 23, 50, 51, 80, 81, 82, 83, 122, 85, 86, + /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 22, + /* 50 */ 70, 23, 71, 72, 73, 74, 75, 76, 77, 78, + /* 60 */ 79, 80, 81, 82, 83, 122, 85, 86, 87, 88, + /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 91, + /* 80 */ 92, 93, 94, 95, 26, 85, 86, 87, 88, 89, + /* 90 */ 90, 91, 92, 93, 94, 95, 27, 28, 97, 98, + /* 100 */ 99, 122, 211, 102, 103, 104, 79, 19, 50, 51, + /* 110 */ 19, 122, 59, 55, 113, 224, 225, 226, 89, 90, + /* 120 */ 91, 92, 93, 94, 95, 23, 27, 28, 26, 71, + /* 130 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + /* 140 */ 82, 83, 51, 85, 86, 87, 88, 89, 90, 91, + /* 150 */ 92, 93, 94, 95, 19, 132, 133, 58, 89, 90, + /* 160 */ 21, 108, 109, 110, 27, 28, 97, 98, 33, 100, + /* 170 */ 7, 8, 119, 120, 22, 19, 107, 42, 109, 27, + /* 180 */ 28, 27, 28, 95, 28, 50, 51, 99, 100, 101, + /* 190 */ 102, 103, 104, 105, 27, 28, 97, 98, 107, 152, + /* 200 */ 112, 132, 133, 112, 65, 69, 71, 72, 73, 74, + /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 11, + /* 220 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + /* 230 */ 95, 19, 101, 97, 97, 98, 24, 101, 122, 157, + /* 240 */ 12, 99, 103, 112, 102, 103, 104, 152, 22, 97, + /* 250 */ 98, 97, 98, 27, 28, 113, 27, 29, 91, 164, + /* 260 */ 165, 124, 50, 51, 97, 98, 219, 59, 132, 133, + /* 270 */ 134, 22, 23, 45, 66, 47, 212, 213, 124, 140, + /* 280 */ 132, 133, 19, 71, 72, 73, 74, 75, 76, 77, + /* 290 */ 78, 79, 80, 81, 82, 83, 152, 85, 86, 87, + /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 164, 165, + /* 310 */ 27, 28, 230, 50, 51, 233, 108, 109, 110, 70, + /* 320 */ 16, 59, 23, 97, 98, 26, 97, 22, 66, 185, + /* 330 */ 12, 187, 27, 28, 71, 72, 73, 74, 75, 76, + /* 340 */ 77, 78, 79, 80, 81, 82, 83, 29, 85, 86, + /* 350 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 19, + /* 360 */ 22, 148, 149, 45, 23, 47, 62, 154, 64, 156, + /* 370 */ 108, 109, 110, 37, 69, 23, 163, 59, 26, 26, + /* 380 */ 97, 98, 144, 145, 146, 147, 152, 200, 52, 23, + /* 390 */ 50, 51, 26, 22, 89, 90, 60, 210, 7, 8, + /* 400 */ 9, 138, 97, 22, 23, 26, 101, 26, 174, 175, + /* 410 */ 197, 71, 72, 73, 74, 75, 76, 77, 78, 79, + /* 420 */ 80, 81, 82, 83, 16, 85, 86, 87, 88, 89, + /* 430 */ 90, 91, 92, 93, 94, 95, 19, 132, 133, 134, + /* 440 */ 23, 152, 208, 209, 140, 152, 152, 111, 195, 196, + /* 450 */ 98, 70, 163, 160, 152, 23, 22, 164, 165, 246, + /* 460 */ 207, 27, 152, 174, 175, 171, 172, 50, 51, 137, + /* 470 */ 62, 139, 64, 171, 172, 222, 124, 27, 138, 24, + /* 480 */ 163, 89, 90, 130, 174, 175, 197, 163, 71, 72, + /* 490 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + /* 500 */ 83, 22, 85, 86, 87, 88, 89, 90, 91, 92, + /* 510 */ 93, 94, 95, 19, 197, 181, 182, 23, 208, 209, + /* 520 */ 152, 197, 26, 189, 132, 133, 232, 224, 225, 226, + /* 530 */ 152, 97, 91, 26, 232, 116, 212, 213, 152, 222, + /* 540 */ 121, 152, 174, 175, 50, 51, 243, 97, 22, 23, + /* 550 */ 22, 234, 174, 175, 177, 23, 239, 116, 163, 177, + /* 560 */ 174, 175, 121, 174, 175, 71, 72, 73, 74, 75, + /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 24, 85, + /* 580 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 590 */ 19, 23, 197, 11, 23, 227, 70, 208, 220, 152, + /* 600 */ 31, 224, 225, 226, 35, 98, 224, 225, 226, 108, + /* 610 */ 109, 110, 115, 152, 117, 118, 27, 222, 49, 123, + /* 620 */ 24, 50, 51, 27, 0, 1, 2, 224, 225, 226, + /* 630 */ 166, 124, 168, 169, 239, 174, 175, 170, 171, 172, + /* 640 */ 22, 194, 71, 72, 73, 74, 75, 76, 77, 78, + /* 650 */ 79, 80, 81, 82, 83, 152, 85, 86, 87, 88, + /* 660 */ 89, 90, 91, 92, 93, 94, 95, 19, 22, 208, + /* 670 */ 24, 23, 195, 196, 170, 171, 172, 174, 175, 152, + /* 680 */ 26, 152, 152, 152, 207, 152, 97, 152, 23, 152, + /* 690 */ 51, 244, 152, 97, 152, 247, 248, 23, 50, 51, + /* 700 */ 26, 174, 175, 174, 175, 174, 175, 174, 175, 174, + /* 710 */ 175, 174, 175, 23, 174, 175, 174, 175, 188, 71, + /* 720 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + /* 730 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, + /* 740 */ 92, 93, 94, 95, 19, 152, 107, 152, 33, 24, + /* 750 */ 152, 100, 101, 27, 174, 175, 152, 42, 152, 23, + /* 760 */ 152, 26, 152, 23, 152, 26, 152, 174, 175, 174, + /* 770 */ 175, 152, 174, 175, 23, 50, 51, 26, 174, 175, + /* 780 */ 174, 175, 174, 175, 174, 175, 174, 175, 174, 175, + /* 790 */ 163, 119, 120, 174, 175, 19, 71, 72, 73, 74, + /* 800 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 152, + /* 810 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + /* 820 */ 95, 66, 152, 97, 197, 23, 50, 51, 26, 53, + /* 830 */ 23, 174, 175, 26, 23, 23, 23, 26, 26, 26, + /* 840 */ 36, 106, 146, 147, 174, 175, 19, 71, 72, 73, + /* 850 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + /* 860 */ 152, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 870 */ 94, 95, 152, 196, 119, 120, 19, 50, 51, 168, + /* 880 */ 169, 26, 174, 175, 207, 28, 152, 249, 250, 152, + /* 890 */ 163, 163, 163, 163, 174, 175, 163, 19, 71, 72, + /* 900 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + /* 910 */ 83, 152, 85, 86, 87, 88, 89, 90, 91, 92, + /* 920 */ 93, 94, 95, 152, 197, 197, 197, 197, 50, 51, + /* 930 */ 197, 194, 36, 174, 175, 191, 192, 152, 191, 192, + /* 940 */ 163, 152, 66, 124, 152, 174, 175, 152, 19, 71, + /* 950 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + /* 960 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, + /* 970 */ 92, 93, 94, 95, 197, 152, 100, 188, 152, 50, + /* 980 */ 51, 152, 152, 188, 174, 175, 252, 152, 94, 95, + /* 990 */ 152, 152, 152, 1, 2, 152, 152, 174, 175, 19, + /* 1000 */ 152, 72, 73, 74, 75, 76, 77, 78, 79, 80, + /* 1010 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, + /* 1020 */ 91, 92, 93, 94, 95, 152, 188, 188, 22, 194, + /* 1030 */ 50, 51, 240, 173, 194, 174, 175, 252, 194, 152, + /* 1040 */ 36, 181, 28, 152, 23, 219, 122, 174, 175, 219, + /* 1050 */ 221, 152, 152, 73, 74, 75, 76, 77, 78, 79, + /* 1060 */ 80, 81, 82, 83, 152, 85, 86, 87, 88, 89, + /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 152, 22, + /* 1080 */ 23, 194, 152, 240, 27, 28, 174, 175, 240, 19, + /* 1090 */ 20, 26, 22, 194, 194, 38, 22, 27, 28, 152, + /* 1100 */ 23, 22, 152, 116, 174, 175, 152, 23, 38, 152, + /* 1110 */ 23, 152, 221, 152, 57, 152, 23, 163, 50, 51, + /* 1120 */ 194, 174, 175, 66, 174, 175, 69, 57, 174, 175, + /* 1130 */ 40, 174, 175, 174, 175, 174, 175, 174, 175, 69, + /* 1140 */ 22, 53, 74, 75, 30, 53, 89, 90, 22, 22, + /* 1150 */ 152, 197, 23, 96, 97, 98, 22, 152, 101, 89, + /* 1160 */ 90, 91, 208, 209, 152, 53, 96, 97, 98, 101, + /* 1170 */ 22, 101, 174, 175, 152, 19, 20, 105, 22, 174, + /* 1180 */ 175, 112, 19, 27, 28, 20, 174, 175, 24, 132, + /* 1190 */ 133, 134, 135, 136, 38, 44, 174, 175, 107, 61, + /* 1200 */ 54, 26, 132, 133, 134, 135, 136, 54, 107, 22, + /* 1210 */ 5, 140, 1, 57, 36, 111, 122, 28, 79, 79, + /* 1220 */ 131, 123, 66, 19, 20, 69, 22, 1, 16, 20, + /* 1230 */ 125, 27, 28, 123, 111, 120, 23, 131, 23, 16, + /* 1240 */ 68, 142, 38, 15, 22, 89, 90, 3, 167, 4, + /* 1250 */ 248, 251, 96, 97, 98, 180, 180, 101, 251, 151, + /* 1260 */ 6, 57, 151, 13, 151, 26, 25, 151, 161, 202, + /* 1270 */ 153, 162, 153, 69, 130, 128, 203, 19, 20, 127, + /* 1280 */ 22, 126, 204, 129, 22, 27, 28, 205, 132, 133, + /* 1290 */ 134, 135, 136, 89, 90, 231, 38, 95, 137, 179, + /* 1300 */ 96, 97, 98, 206, 179, 101, 122, 107, 159, 159, + /* 1310 */ 125, 231, 216, 228, 107, 57, 184, 217, 216, 176, + /* 1320 */ 217, 176, 48, 106, 18, 184, 158, 69, 159, 158, + /* 1330 */ 46, 71, 237, 176, 176, 176, 132, 133, 134, 135, + /* 1340 */ 136, 217, 176, 137, 216, 178, 158, 89, 90, 179, + /* 1350 */ 176, 159, 179, 159, 96, 97, 98, 159, 159, 101, + /* 1360 */ 5, 158, 202, 22, 18, 10, 11, 12, 13, 14, + /* 1370 */ 190, 238, 17, 190, 158, 193, 41, 159, 202, 193, + /* 1380 */ 159, 202, 245, 193, 193, 223, 190, 32, 159, 34, + /* 1390 */ 132, 133, 134, 135, 136, 159, 39, 155, 43, 150, + /* 1400 */ 223, 177, 201, 178, 177, 186, 66, 199, 177, 152, + /* 1410 */ 253, 56, 215, 152, 182, 152, 202, 152, 63, 152, + /* 1420 */ 152, 66, 67, 242, 229, 152, 174, 152, 152, 152, + /* 1430 */ 152, 152, 152, 152, 199, 242, 202, 152, 198, 152, + /* 1440 */ 152, 152, 183, 192, 152, 215, 152, 183, 215, 183, + /* 1450 */ 152, 241, 214, 152, 211, 152, 152, 211, 211, 152, + /* 1460 */ 152, 241, 152, 152, 152, 152, 152, 152, 152, 114, + /* 1470 */ 152, 152, 235, 152, 152, 152, 174, 187, 95, 174, + /* 1480 */ 253, 253, 253, 253, 236, 253, 253, 253, 253, 253, + /* 1490 */ 253, 253, 253, 253, 253, 253, 141, +}; +#define YY_SHIFT_USE_DFLT (-86) +#define YY_SHIFT_COUNT (429) +#define YY_SHIFT_MIN (-85) +#define YY_SHIFT_MAX (1383) +static const short yy_shift_ofst[] = { + /* 0 */ 992, 1057, 1355, 1156, 1204, 1204, 1, 262, -19, 135, + /* 10 */ 135, 776, 1204, 1204, 1204, 1204, 69, 69, 53, 208, + /* 20 */ 283, 755, 58, 725, 648, 571, 494, 417, 340, 263, + /* 30 */ 212, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 40 */ 827, 827, 827, 827, 827, 827, 878, 827, 929, 980, + /* 50 */ 980, 1070, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 60 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 70 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 80 */ 1258, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 90 */ 1204, 1204, 1204, 1204, -71, -47, -47, -47, -47, -47, + /* 100 */ 0, 29, -12, 283, 283, 139, 91, 392, 392, 894, + /* 110 */ 672, 726, 1383, -86, -86, -86, 88, 318, 318, 99, + /* 120 */ 381, -20, 283, 283, 283, 283, 283, 283, 283, 283, + /* 130 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + /* 140 */ 283, 283, 283, 283, 624, 876, 726, 672, 1340, 1340, + /* 150 */ 1340, 1340, 1340, 1340, -86, -86, -86, 305, 136, 136, + /* 160 */ 142, 167, 226, 154, 137, 152, 283, 283, 283, 283, + /* 170 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + /* 180 */ 283, 283, 283, 336, 336, 336, 283, 283, 352, 283, + /* 190 */ 283, 283, 283, 283, 228, 283, 283, 283, 283, 283, + /* 200 */ 283, 283, 283, 283, 283, 501, 569, 596, 596, 596, + /* 210 */ 507, 497, 441, 391, 353, 156, 156, 857, 353, 857, + /* 220 */ 735, 813, 639, 715, 156, 332, 715, 715, 496, 419, + /* 230 */ 646, 1357, 1184, 1184, 1335, 1335, 1184, 1341, 1260, 1144, + /* 240 */ 1346, 1346, 1346, 1346, 1184, 1306, 1144, 1341, 1260, 1260, + /* 250 */ 1144, 1184, 1306, 1206, 1284, 1184, 1184, 1306, 1184, 1306, + /* 260 */ 1184, 1306, 1262, 1207, 1207, 1207, 1274, 1262, 1207, 1217, + /* 270 */ 1207, 1274, 1207, 1207, 1185, 1200, 1185, 1200, 1185, 1200, + /* 280 */ 1184, 1184, 1161, 1262, 1202, 1202, 1262, 1154, 1155, 1147, + /* 290 */ 1152, 1144, 1241, 1239, 1250, 1250, 1254, 1254, 1254, 1254, + /* 300 */ -86, -86, -86, -86, -86, -86, 1068, 304, 526, 249, + /* 310 */ 408, -83, 434, 812, 27, 811, 807, 802, 751, 589, + /* 320 */ 651, 163, 131, 674, 366, 450, 299, 148, 23, 102, + /* 330 */ 229, -21, 1245, 1244, 1222, 1099, 1228, 1172, 1223, 1215, + /* 340 */ 1213, 1115, 1106, 1123, 1110, 1209, 1105, 1212, 1226, 1098, + /* 350 */ 1089, 1140, 1139, 1104, 1189, 1178, 1094, 1211, 1205, 1187, + /* 360 */ 1101, 1071, 1153, 1175, 1146, 1138, 1151, 1091, 1164, 1165, + /* 370 */ 1163, 1069, 1072, 1148, 1112, 1134, 1127, 1129, 1126, 1092, + /* 380 */ 1114, 1118, 1088, 1090, 1093, 1087, 1084, 987, 1079, 1077, + /* 390 */ 1074, 1065, 924, 1021, 1014, 1004, 1006, 819, 739, 896, + /* 400 */ 855, 804, 739, 740, 736, 690, 654, 665, 618, 582, + /* 410 */ 568, 528, 554, 379, 532, 479, 455, 379, 432, 371, + /* 420 */ 341, 28, 338, 116, -11, -57, -85, 7, -8, 3, +}; +#define YY_REDUCE_USE_DFLT (-110) +#define YY_REDUCE_COUNT (305) +#define YY_REDUCE_MIN (-109) +#define YY_REDUCE_MAX (1323) +static const short yy_reduce_ofst[] = { + /* 0 */ 238, 954, 213, 289, 310, 234, 144, 317, -109, 382, + /* 10 */ 377, 303, 461, 389, 378, 368, 302, 294, 253, 395, + /* 20 */ 293, 324, 403, 403, 403, 403, 403, 403, 403, 403, + /* 30 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, + /* 40 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, + /* 50 */ 403, 1022, 1012, 1005, 998, 963, 961, 959, 957, 950, + /* 60 */ 947, 930, 912, 873, 861, 823, 810, 771, 759, 720, + /* 70 */ 708, 670, 657, 619, 614, 612, 610, 608, 606, 604, + /* 80 */ 598, 595, 593, 580, 542, 540, 537, 535, 533, 531, + /* 90 */ 529, 527, 503, 386, 403, 403, 403, 403, 403, 403, + /* 100 */ 403, 403, 403, 95, 447, 82, 334, 504, 467, 403, + /* 110 */ 477, 464, 403, 403, 403, 403, 860, 747, 744, 785, + /* 120 */ 638, 638, 926, 891, 900, 899, 887, 844, 840, 835, + /* 130 */ 848, 830, 843, 829, 792, 839, 826, 737, 838, 795, + /* 140 */ 789, 47, 734, 530, 696, 777, 711, 677, 733, 730, + /* 150 */ 729, 728, 727, 627, 448, 64, 187, 1305, 1302, 1252, + /* 160 */ 1290, 1273, 1323, 1322, 1321, 1319, 1318, 1316, 1315, 1314, + /* 170 */ 1313, 1312, 1311, 1310, 1308, 1307, 1304, 1303, 1301, 1298, + /* 180 */ 1294, 1292, 1289, 1266, 1264, 1259, 1288, 1287, 1238, 1285, + /* 190 */ 1281, 1280, 1279, 1278, 1251, 1277, 1276, 1275, 1273, 1268, + /* 200 */ 1267, 1265, 1263, 1261, 1257, 1248, 1237, 1247, 1246, 1243, + /* 210 */ 1238, 1240, 1235, 1249, 1234, 1233, 1230, 1220, 1214, 1210, + /* 220 */ 1225, 1219, 1232, 1231, 1197, 1195, 1227, 1224, 1201, 1208, + /* 230 */ 1242, 1137, 1236, 1229, 1193, 1181, 1221, 1177, 1196, 1179, + /* 240 */ 1191, 1190, 1186, 1182, 1218, 1216, 1176, 1162, 1183, 1180, + /* 250 */ 1160, 1199, 1203, 1133, 1095, 1198, 1194, 1188, 1192, 1171, + /* 260 */ 1169, 1168, 1173, 1174, 1166, 1159, 1141, 1170, 1158, 1167, + /* 270 */ 1157, 1132, 1145, 1143, 1124, 1128, 1103, 1102, 1100, 1096, + /* 280 */ 1150, 1149, 1085, 1125, 1080, 1064, 1120, 1097, 1082, 1078, + /* 290 */ 1073, 1067, 1109, 1107, 1119, 1117, 1116, 1113, 1111, 1108, + /* 300 */ 1007, 1000, 1002, 1076, 1075, 1081, +}; +static const YYACTIONTYPE yy_default[] = { + /* 0 */ 647, 964, 964, 964, 878, 878, 969, 964, 774, 802, + /* 10 */ 802, 938, 969, 969, 969, 876, 969, 969, 969, 964, + /* 20 */ 969, 778, 808, 969, 969, 969, 969, 969, 969, 969, + /* 30 */ 969, 937, 939, 816, 815, 918, 789, 813, 806, 810, + /* 40 */ 879, 872, 873, 871, 875, 880, 969, 809, 841, 856, + /* 50 */ 840, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 60 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 70 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 80 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 90 */ 969, 969, 969, 969, 850, 855, 862, 854, 851, 843, + /* 100 */ 842, 844, 845, 969, 969, 673, 739, 969, 969, 846, + /* 110 */ 969, 685, 847, 859, 858, 857, 680, 969, 969, 969, + /* 120 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 130 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 140 */ 969, 969, 969, 969, 647, 964, 969, 969, 964, 964, + /* 150 */ 964, 964, 964, 964, 956, 778, 768, 969, 969, 969, + /* 160 */ 969, 969, 969, 969, 969, 969, 969, 944, 942, 969, + /* 170 */ 891, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 180 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 190 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 200 */ 969, 969, 969, 969, 653, 969, 911, 774, 774, 774, + /* 210 */ 776, 754, 766, 655, 812, 791, 791, 923, 812, 923, + /* 220 */ 710, 733, 707, 802, 791, 874, 802, 802, 775, 766, + /* 230 */ 969, 949, 782, 782, 941, 941, 782, 821, 743, 812, + /* 240 */ 750, 750, 750, 750, 782, 670, 812, 821, 743, 743, + /* 250 */ 812, 782, 670, 917, 915, 782, 782, 670, 782, 670, + /* 260 */ 782, 670, 884, 741, 741, 741, 725, 884, 741, 710, + /* 270 */ 741, 725, 741, 741, 795, 790, 795, 790, 795, 790, + /* 280 */ 782, 782, 969, 884, 888, 888, 884, 807, 796, 805, + /* 290 */ 803, 812, 676, 728, 663, 663, 652, 652, 652, 652, + /* 300 */ 961, 961, 956, 712, 712, 695, 969, 969, 969, 969, + /* 310 */ 969, 969, 687, 969, 893, 969, 969, 969, 969, 969, + /* 320 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 330 */ 969, 828, 969, 648, 951, 969, 969, 948, 969, 969, + /* 340 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 350 */ 969, 969, 969, 969, 969, 969, 921, 969, 969, 969, + /* 360 */ 969, 969, 969, 914, 913, 969, 969, 969, 969, 969, + /* 370 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 380 */ 969, 969, 969, 969, 969, 969, 969, 757, 969, 969, + /* 390 */ 969, 761, 969, 969, 969, 969, 969, 969, 804, 969, + /* 400 */ 797, 969, 877, 969, 969, 969, 969, 969, 969, 969, + /* 410 */ 969, 969, 969, 966, 969, 969, 969, 965, 969, 969, + /* 420 */ 969, 969, 969, 830, 969, 829, 833, 969, 661, 969, + /* 430 */ 644, 649, 960, 963, 962, 959, 958, 957, 952, 950, + /* 440 */ 947, 946, 945, 943, 940, 936, 897, 895, 902, 901, + /* 450 */ 900, 899, 898, 896, 894, 892, 818, 817, 814, 811, + /* 460 */ 753, 935, 890, 752, 749, 748, 669, 953, 920, 929, + /* 470 */ 928, 927, 822, 926, 925, 924, 922, 919, 906, 820, + /* 480 */ 819, 744, 882, 881, 672, 910, 909, 908, 912, 916, + /* 490 */ 907, 784, 751, 671, 668, 675, 679, 731, 732, 740, + /* 500 */ 738, 737, 736, 735, 734, 730, 681, 686, 724, 709, + /* 510 */ 708, 717, 716, 722, 721, 720, 719, 718, 715, 714, + /* 520 */ 713, 706, 705, 711, 704, 727, 726, 723, 703, 747, + /* 530 */ 746, 745, 742, 702, 701, 700, 833, 699, 698, 838, + /* 540 */ 837, 866, 826, 755, 759, 758, 762, 763, 771, 770, + /* 550 */ 769, 780, 781, 793, 792, 824, 823, 794, 779, 773, + /* 560 */ 772, 788, 787, 786, 785, 777, 767, 799, 798, 868, + /* 570 */ 783, 867, 865, 934, 933, 932, 931, 930, 870, 967, + /* 580 */ 968, 887, 889, 886, 801, 800, 885, 869, 839, 836, + /* 590 */ 690, 691, 905, 904, 903, 693, 692, 689, 688, 863, + /* 600 */ 860, 852, 864, 861, 853, 849, 848, 834, 832, 831, + /* 610 */ 827, 835, 760, 756, 825, 765, 764, 697, 696, 694, + /* 620 */ 678, 677, 674, 667, 665, 664, 666, 662, 660, 659, + /* 630 */ 658, 657, 656, 684, 683, 682, 654, 651, 650, 646, + /* 640 */ 645, 643, +}; + +/* The next table maps tokens into fallback tokens. If a construct +** like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +*/ +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { + 0, /* $ => nothing */ + 0, /* SEMI => nothing */ + 27, /* EXPLAIN => ID */ + 27, /* QUERY => ID */ + 27, /* PLAN => ID */ + 27, /* BEGIN => ID */ + 0, /* TRANSACTION => nothing */ + 27, /* DEFERRED => ID */ + 27, /* IMMEDIATE => ID */ + 27, /* EXCLUSIVE => ID */ + 0, /* COMMIT => nothing */ + 27, /* END => ID */ + 27, /* ROLLBACK => ID */ + 27, /* SAVEPOINT => ID */ + 27, /* RELEASE => ID */ + 0, /* TO => nothing */ + 0, /* TABLE => nothing */ + 0, /* CREATE => nothing */ + 27, /* IF => ID */ + 0, /* NOT => nothing */ + 0, /* EXISTS => nothing */ + 27, /* TEMP => ID */ + 0, /* LP => nothing */ + 0, /* RP => nothing */ + 0, /* AS => nothing */ + 27, /* WITHOUT => ID */ + 0, /* COMMA => nothing */ + 0, /* ID => nothing */ + 0, /* INDEXED => nothing */ + 27, /* ABORT => ID */ + 27, /* ACTION => ID */ + 27, /* AFTER => ID */ + 27, /* ANALYZE => ID */ + 27, /* ASC => ID */ + 27, /* ATTACH => ID */ + 27, /* BEFORE => ID */ + 27, /* BY => ID */ + 27, /* CASCADE => ID */ + 27, /* CAST => ID */ + 27, /* COLUMNKW => ID */ + 27, /* CONFLICT => ID */ + 27, /* DATABASE => ID */ + 27, /* DESC => ID */ + 27, /* DETACH => ID */ + 27, /* EACH => ID */ + 27, /* FAIL => ID */ + 27, /* FOR => ID */ + 27, /* IGNORE => ID */ + 27, /* INITIALLY => ID */ + 27, /* INSTEAD => ID */ + 27, /* LIKE_KW => ID */ + 27, /* MATCH => ID */ + 27, /* NO => ID */ + 27, /* KEY => ID */ + 27, /* OF => ID */ + 27, /* OFFSET => ID */ + 27, /* PRAGMA => ID */ + 27, /* RAISE => ID */ + 27, /* RECURSIVE => ID */ + 27, /* REPLACE => ID */ + 27, /* RESTRICT => ID */ + 27, /* ROW => ID */ + 27, /* TRIGGER => ID */ + 27, /* VACUUM => ID */ + 27, /* VIEW => ID */ + 27, /* VIRTUAL => ID */ + 27, /* WITH => ID */ + 27, /* REINDEX => ID */ + 27, /* RENAME => ID */ + 27, /* CTIME_KW => ID */ +}; +#endif /* YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + int yyidx; /* Index of top element in stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyidxMax; /* Maximum value of yyidx */ +#endif + int yyerrcnt; /* Shifts left before out of the error */ + sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ +#if YYSTACKDEPTH<=0 + int yystksz; /* Current side of the stack */ + yyStackEntry *yystack; /* The parser's stack */ +#else + yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ +#endif +}; +typedef struct yyParser yyParser; + +#ifndef NDEBUG +/* #include */ +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
      +**
    • A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +**
    • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
    +** +** Outputs: +** None. +*/ +SQLITE_PRIVATE void sqlite3ParserTrace(FILE *TraceFILE, char *zTracePrompt){ + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if( yyTraceFILE==0 ) yyTracePrompt = 0; + else if( yyTracePrompt==0 ) yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { + "$", "SEMI", "EXPLAIN", "QUERY", + "PLAN", "BEGIN", "TRANSACTION", "DEFERRED", + "IMMEDIATE", "EXCLUSIVE", "COMMIT", "END", + "ROLLBACK", "SAVEPOINT", "RELEASE", "TO", + "TABLE", "CREATE", "IF", "NOT", + "EXISTS", "TEMP", "LP", "RP", + "AS", "WITHOUT", "COMMA", "ID", + "INDEXED", "ABORT", "ACTION", "AFTER", + "ANALYZE", "ASC", "ATTACH", "BEFORE", + "BY", "CASCADE", "CAST", "COLUMNKW", + "CONFLICT", "DATABASE", "DESC", "DETACH", + "EACH", "FAIL", "FOR", "IGNORE", + "INITIALLY", "INSTEAD", "LIKE_KW", "MATCH", + "NO", "KEY", "OF", "OFFSET", + "PRAGMA", "RAISE", "RECURSIVE", "REPLACE", + "RESTRICT", "ROW", "TRIGGER", "VACUUM", + "VIEW", "VIRTUAL", "WITH", "REINDEX", + "RENAME", "CTIME_KW", "ANY", "OR", + "AND", "IS", "BETWEEN", "IN", + "ISNULL", "NOTNULL", "NE", "EQ", + "GT", "LE", "LT", "GE", + "ESCAPE", "BITAND", "BITOR", "LSHIFT", + "RSHIFT", "PLUS", "MINUS", "STAR", + "SLASH", "REM", "CONCAT", "COLLATE", + "BITNOT", "STRING", "JOIN_KW", "CONSTRAINT", + "DEFAULT", "NULL", "PRIMARY", "UNIQUE", + "CHECK", "REFERENCES", "AUTOINCR", "ON", + "INSERT", "DELETE", "UPDATE", "SET", + "DEFERRABLE", "FOREIGN", "DROP", "UNION", + "ALL", "EXCEPT", "INTERSECT", "SELECT", + "VALUES", "DISTINCT", "DOT", "FROM", + "JOIN", "USING", "ORDER", "GROUP", + "HAVING", "LIMIT", "WHERE", "INTO", + "INTEGER", "FLOAT", "BLOB", "VARIABLE", + "CASE", "WHEN", "THEN", "ELSE", + "INDEX", "ALTER", "ADD", "error", + "input", "cmdlist", "ecmd", "explain", + "cmdx", "cmd", "transtype", "trans_opt", + "nm", "savepoint_opt", "create_table", "create_table_args", + "createkw", "temp", "ifnotexists", "dbnm", + "columnlist", "conslist_opt", "table_options", "select", + "column", "columnid", "type", "carglist", + "typetoken", "typename", "signed", "plus_num", + "minus_num", "ccons", "term", "expr", + "onconf", "sortorder", "autoinc", "idxlist_opt", + "refargs", "defer_subclause", "refarg", "refact", + "init_deferred_pred_opt", "conslist", "tconscomma", "tcons", + "idxlist", "defer_subclause_opt", "orconf", "resolvetype", + "raisetype", "ifexists", "fullname", "selectnowith", + "oneselect", "with", "multiselect_op", "distinct", + "selcollist", "from", "where_opt", "groupby_opt", + "having_opt", "orderby_opt", "limit_opt", "values", + "nexprlist", "exprlist", "sclp", "as", + "seltablist", "stl_prefix", "joinop", "indexed_opt", + "on_opt", "using_opt", "joinop2", "idlist", + "sortlist", "setlist", "insert_cmd", "inscollist_opt", + "likeop", "between_op", "in_op", "case_operand", + "case_exprlist", "case_else", "uniqueflag", "collate", + "nmnum", "trigger_decl", "trigger_cmd_list", "trigger_time", + "trigger_event", "foreach_clause", "when_clause", "trigger_cmd", + "trnm", "tridxby", "database_kw_opt", "key_opt", + "add_column_fullname", "kwcolumn_opt", "create_vtab", "vtabarglist", + "vtabarg", "vtabargtoken", "lp", "anylist", + "wqlist", +}; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { + /* 0 */ "input ::= cmdlist", + /* 1 */ "cmdlist ::= cmdlist ecmd", + /* 2 */ "cmdlist ::= ecmd", + /* 3 */ "ecmd ::= SEMI", + /* 4 */ "ecmd ::= explain cmdx SEMI", + /* 5 */ "explain ::=", + /* 6 */ "explain ::= EXPLAIN", + /* 7 */ "explain ::= EXPLAIN QUERY PLAN", + /* 8 */ "cmdx ::= cmd", + /* 9 */ "cmd ::= BEGIN transtype trans_opt", + /* 10 */ "trans_opt ::=", + /* 11 */ "trans_opt ::= TRANSACTION", + /* 12 */ "trans_opt ::= TRANSACTION nm", + /* 13 */ "transtype ::=", + /* 14 */ "transtype ::= DEFERRED", + /* 15 */ "transtype ::= IMMEDIATE", + /* 16 */ "transtype ::= EXCLUSIVE", + /* 17 */ "cmd ::= COMMIT trans_opt", + /* 18 */ "cmd ::= END trans_opt", + /* 19 */ "cmd ::= ROLLBACK trans_opt", + /* 20 */ "savepoint_opt ::= SAVEPOINT", + /* 21 */ "savepoint_opt ::=", + /* 22 */ "cmd ::= SAVEPOINT nm", + /* 23 */ "cmd ::= RELEASE savepoint_opt nm", + /* 24 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", + /* 25 */ "cmd ::= create_table create_table_args", + /* 26 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", + /* 27 */ "createkw ::= CREATE", + /* 28 */ "ifnotexists ::=", + /* 29 */ "ifnotexists ::= IF NOT EXISTS", + /* 30 */ "temp ::= TEMP", + /* 31 */ "temp ::=", + /* 32 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", + /* 33 */ "create_table_args ::= AS select", + /* 34 */ "table_options ::=", + /* 35 */ "table_options ::= WITHOUT nm", + /* 36 */ "columnlist ::= columnlist COMMA column", + /* 37 */ "columnlist ::= column", + /* 38 */ "column ::= columnid type carglist", + /* 39 */ "columnid ::= nm", + /* 40 */ "nm ::= ID|INDEXED", + /* 41 */ "nm ::= STRING", + /* 42 */ "nm ::= JOIN_KW", + /* 43 */ "type ::=", + /* 44 */ "type ::= typetoken", + /* 45 */ "typetoken ::= typename", + /* 46 */ "typetoken ::= typename LP signed RP", + /* 47 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 48 */ "typename ::= ID|STRING", + /* 49 */ "typename ::= typename ID|STRING", + /* 50 */ "signed ::= plus_num", + /* 51 */ "signed ::= minus_num", + /* 52 */ "carglist ::= carglist ccons", + /* 53 */ "carglist ::=", + /* 54 */ "ccons ::= CONSTRAINT nm", + /* 55 */ "ccons ::= DEFAULT term", + /* 56 */ "ccons ::= DEFAULT LP expr RP", + /* 57 */ "ccons ::= DEFAULT PLUS term", + /* 58 */ "ccons ::= DEFAULT MINUS term", + /* 59 */ "ccons ::= DEFAULT ID|INDEXED", + /* 60 */ "ccons ::= NULL onconf", + /* 61 */ "ccons ::= NOT NULL onconf", + /* 62 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 63 */ "ccons ::= UNIQUE onconf", + /* 64 */ "ccons ::= CHECK LP expr RP", + /* 65 */ "ccons ::= REFERENCES nm idxlist_opt refargs", + /* 66 */ "ccons ::= defer_subclause", + /* 67 */ "ccons ::= COLLATE ID|STRING", + /* 68 */ "autoinc ::=", + /* 69 */ "autoinc ::= AUTOINCR", + /* 70 */ "refargs ::=", + /* 71 */ "refargs ::= refargs refarg", + /* 72 */ "refarg ::= MATCH nm", + /* 73 */ "refarg ::= ON INSERT refact", + /* 74 */ "refarg ::= ON DELETE refact", + /* 75 */ "refarg ::= ON UPDATE refact", + /* 76 */ "refact ::= SET NULL", + /* 77 */ "refact ::= SET DEFAULT", + /* 78 */ "refact ::= CASCADE", + /* 79 */ "refact ::= RESTRICT", + /* 80 */ "refact ::= NO ACTION", + /* 81 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 82 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 83 */ "init_deferred_pred_opt ::=", + /* 84 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 85 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 86 */ "conslist_opt ::=", + /* 87 */ "conslist_opt ::= COMMA conslist", + /* 88 */ "conslist ::= conslist tconscomma tcons", + /* 89 */ "conslist ::= tcons", + /* 90 */ "tconscomma ::= COMMA", + /* 91 */ "tconscomma ::=", + /* 92 */ "tcons ::= CONSTRAINT nm", + /* 93 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", + /* 94 */ "tcons ::= UNIQUE LP idxlist RP onconf", + /* 95 */ "tcons ::= CHECK LP expr RP onconf", + /* 96 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", + /* 97 */ "defer_subclause_opt ::=", + /* 98 */ "defer_subclause_opt ::= defer_subclause", + /* 99 */ "onconf ::=", + /* 100 */ "onconf ::= ON CONFLICT resolvetype", + /* 101 */ "orconf ::=", + /* 102 */ "orconf ::= OR resolvetype", + /* 103 */ "resolvetype ::= raisetype", + /* 104 */ "resolvetype ::= IGNORE", + /* 105 */ "resolvetype ::= REPLACE", + /* 106 */ "cmd ::= DROP TABLE ifexists fullname", + /* 107 */ "ifexists ::= IF EXISTS", + /* 108 */ "ifexists ::=", + /* 109 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select", + /* 110 */ "cmd ::= DROP VIEW ifexists fullname", + /* 111 */ "cmd ::= select", + /* 112 */ "select ::= with selectnowith", + /* 113 */ "selectnowith ::= oneselect", + /* 114 */ "selectnowith ::= selectnowith multiselect_op oneselect", + /* 115 */ "multiselect_op ::= UNION", + /* 116 */ "multiselect_op ::= UNION ALL", + /* 117 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 118 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 119 */ "oneselect ::= values", + /* 120 */ "values ::= VALUES LP nexprlist RP", + /* 121 */ "values ::= values COMMA LP exprlist RP", + /* 122 */ "distinct ::= DISTINCT", + /* 123 */ "distinct ::= ALL", + /* 124 */ "distinct ::=", + /* 125 */ "sclp ::= selcollist COMMA", + /* 126 */ "sclp ::=", + /* 127 */ "selcollist ::= sclp expr as", + /* 128 */ "selcollist ::= sclp STAR", + /* 129 */ "selcollist ::= sclp nm DOT STAR", + /* 130 */ "as ::= AS nm", + /* 131 */ "as ::= ID|STRING", + /* 132 */ "as ::=", + /* 133 */ "from ::=", + /* 134 */ "from ::= FROM seltablist", + /* 135 */ "stl_prefix ::= seltablist joinop", + /* 136 */ "stl_prefix ::=", + /* 137 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 138 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 139 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 140 */ "dbnm ::=", + /* 141 */ "dbnm ::= DOT nm", + /* 142 */ "fullname ::= nm dbnm", + /* 143 */ "joinop ::= COMMA|JOIN", + /* 144 */ "joinop ::= JOIN_KW JOIN", + /* 145 */ "joinop ::= JOIN_KW nm JOIN", + /* 146 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 147 */ "on_opt ::= ON expr", + /* 148 */ "on_opt ::=", + /* 149 */ "indexed_opt ::=", + /* 150 */ "indexed_opt ::= INDEXED BY nm", + /* 151 */ "indexed_opt ::= NOT INDEXED", + /* 152 */ "using_opt ::= USING LP idlist RP", + /* 153 */ "using_opt ::=", + /* 154 */ "orderby_opt ::=", + /* 155 */ "orderby_opt ::= ORDER BY sortlist", + /* 156 */ "sortlist ::= sortlist COMMA expr sortorder", + /* 157 */ "sortlist ::= expr sortorder", + /* 158 */ "sortorder ::= ASC", + /* 159 */ "sortorder ::= DESC", + /* 160 */ "sortorder ::=", + /* 161 */ "groupby_opt ::=", + /* 162 */ "groupby_opt ::= GROUP BY nexprlist", + /* 163 */ "having_opt ::=", + /* 164 */ "having_opt ::= HAVING expr", + /* 165 */ "limit_opt ::=", + /* 166 */ "limit_opt ::= LIMIT expr", + /* 167 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 168 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 169 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", + /* 170 */ "where_opt ::=", + /* 171 */ "where_opt ::= WHERE expr", + /* 172 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", + /* 173 */ "setlist ::= setlist COMMA nm EQ expr", + /* 174 */ "setlist ::= nm EQ expr", + /* 175 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt select", + /* 176 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", + /* 177 */ "insert_cmd ::= INSERT orconf", + /* 178 */ "insert_cmd ::= REPLACE", + /* 179 */ "inscollist_opt ::=", + /* 180 */ "inscollist_opt ::= LP idlist RP", + /* 181 */ "idlist ::= idlist COMMA nm", + /* 182 */ "idlist ::= nm", + /* 183 */ "expr ::= term", + /* 184 */ "expr ::= LP expr RP", + /* 185 */ "term ::= NULL", + /* 186 */ "expr ::= ID|INDEXED", + /* 187 */ "expr ::= JOIN_KW", + /* 188 */ "expr ::= nm DOT nm", + /* 189 */ "expr ::= nm DOT nm DOT nm", + /* 190 */ "term ::= INTEGER|FLOAT|BLOB", + /* 191 */ "term ::= STRING", + /* 192 */ "expr ::= VARIABLE", + /* 193 */ "expr ::= expr COLLATE ID|STRING", + /* 194 */ "expr ::= CAST LP expr AS typetoken RP", + /* 195 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 196 */ "expr ::= ID|INDEXED LP STAR RP", + /* 197 */ "term ::= CTIME_KW", + /* 198 */ "expr ::= expr AND expr", + /* 199 */ "expr ::= expr OR expr", + /* 200 */ "expr ::= expr LT|GT|GE|LE expr", + /* 201 */ "expr ::= expr EQ|NE expr", + /* 202 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 203 */ "expr ::= expr PLUS|MINUS expr", + /* 204 */ "expr ::= expr STAR|SLASH|REM expr", + /* 205 */ "expr ::= expr CONCAT expr", + /* 206 */ "likeop ::= LIKE_KW|MATCH", + /* 207 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 208 */ "expr ::= expr likeop expr", + /* 209 */ "expr ::= expr likeop expr ESCAPE expr", + /* 210 */ "expr ::= expr ISNULL|NOTNULL", + /* 211 */ "expr ::= expr NOT NULL", + /* 212 */ "expr ::= expr IS expr", + /* 213 */ "expr ::= expr IS NOT expr", + /* 214 */ "expr ::= NOT expr", + /* 215 */ "expr ::= BITNOT expr", + /* 216 */ "expr ::= MINUS expr", + /* 217 */ "expr ::= PLUS expr", + /* 218 */ "between_op ::= BETWEEN", + /* 219 */ "between_op ::= NOT BETWEEN", + /* 220 */ "expr ::= expr between_op expr AND expr", + /* 221 */ "in_op ::= IN", + /* 222 */ "in_op ::= NOT IN", + /* 223 */ "expr ::= expr in_op LP exprlist RP", + /* 224 */ "expr ::= LP select RP", + /* 225 */ "expr ::= expr in_op LP select RP", + /* 226 */ "expr ::= expr in_op nm dbnm", + /* 227 */ "expr ::= EXISTS LP select RP", + /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 230 */ "case_exprlist ::= WHEN expr THEN expr", + /* 231 */ "case_else ::= ELSE expr", + /* 232 */ "case_else ::=", + /* 233 */ "case_operand ::= expr", + /* 234 */ "case_operand ::=", + /* 235 */ "exprlist ::= nexprlist", + /* 236 */ "exprlist ::=", + /* 237 */ "nexprlist ::= nexprlist COMMA expr", + /* 238 */ "nexprlist ::= expr", + /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt", + /* 240 */ "uniqueflag ::= UNIQUE", + /* 241 */ "uniqueflag ::=", + /* 242 */ "idxlist_opt ::=", + /* 243 */ "idxlist_opt ::= LP idxlist RP", + /* 244 */ "idxlist ::= idxlist COMMA nm collate sortorder", + /* 245 */ "idxlist ::= nm collate sortorder", + /* 246 */ "collate ::=", + /* 247 */ "collate ::= COLLATE ID|STRING", + /* 248 */ "cmd ::= DROP INDEX ifexists fullname", + /* 249 */ "cmd ::= VACUUM", + /* 250 */ "cmd ::= VACUUM nm", + /* 251 */ "cmd ::= PRAGMA nm dbnm", + /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 256 */ "nmnum ::= plus_num", + /* 257 */ "nmnum ::= nm", + /* 258 */ "nmnum ::= ON", + /* 259 */ "nmnum ::= DELETE", + /* 260 */ "nmnum ::= DEFAULT", + /* 261 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 262 */ "plus_num ::= INTEGER|FLOAT", + /* 263 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 264 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 265 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 266 */ "trigger_time ::= BEFORE", + /* 267 */ "trigger_time ::= AFTER", + /* 268 */ "trigger_time ::= INSTEAD OF", + /* 269 */ "trigger_time ::=", + /* 270 */ "trigger_event ::= DELETE|INSERT", + /* 271 */ "trigger_event ::= UPDATE", + /* 272 */ "trigger_event ::= UPDATE OF idlist", + /* 273 */ "foreach_clause ::=", + /* 274 */ "foreach_clause ::= FOR EACH ROW", + /* 275 */ "when_clause ::=", + /* 276 */ "when_clause ::= WHEN expr", + /* 277 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 278 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 279 */ "trnm ::= nm", + /* 280 */ "trnm ::= nm DOT nm", + /* 281 */ "tridxby ::=", + /* 282 */ "tridxby ::= INDEXED BY nm", + /* 283 */ "tridxby ::= NOT INDEXED", + /* 284 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", + /* 285 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", + /* 286 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", + /* 287 */ "trigger_cmd ::= select", + /* 288 */ "expr ::= RAISE LP IGNORE RP", + /* 289 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 290 */ "raisetype ::= ROLLBACK", + /* 291 */ "raisetype ::= ABORT", + /* 292 */ "raisetype ::= FAIL", + /* 293 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 294 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 295 */ "cmd ::= DETACH database_kw_opt expr", + /* 296 */ "key_opt ::=", + /* 297 */ "key_opt ::= KEY expr", + /* 298 */ "database_kw_opt ::= DATABASE", + /* 299 */ "database_kw_opt ::=", + /* 300 */ "cmd ::= REINDEX", + /* 301 */ "cmd ::= REINDEX nm dbnm", + /* 302 */ "cmd ::= ANALYZE", + /* 303 */ "cmd ::= ANALYZE nm dbnm", + /* 304 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 305 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", + /* 306 */ "add_column_fullname ::= fullname", + /* 307 */ "kwcolumn_opt ::=", + /* 308 */ "kwcolumn_opt ::= COLUMNKW", + /* 309 */ "cmd ::= create_vtab", + /* 310 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 311 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 312 */ "vtabarglist ::= vtabarg", + /* 313 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 314 */ "vtabarg ::=", + /* 315 */ "vtabarg ::= vtabarg vtabargtoken", + /* 316 */ "vtabargtoken ::= ANY", + /* 317 */ "vtabargtoken ::= lp anylist RP", + /* 318 */ "lp ::= LP", + /* 319 */ "anylist ::=", + /* 320 */ "anylist ::= anylist LP anylist RP", + /* 321 */ "anylist ::= anylist ANY", + /* 322 */ "with ::=", + /* 323 */ "with ::= WITH wqlist", + /* 324 */ "with ::= WITH RECURSIVE wqlist", + /* 325 */ "wqlist ::= nm idxlist_opt AS LP select RP", + /* 326 */ "wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP", +}; +#endif /* NDEBUG */ + + +#if YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. +*/ +static void yyGrowStack(yyParser *p){ + int newSize; + yyStackEntry *pNew; + + newSize = p->yystksz*2 + 100; + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + if( pNew ){ + p->yystack = pNew; + p->yystksz = newSize; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows to %d entries!\n", + yyTracePrompt, p->yystksz); + } +#endif + } +} +#endif + +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to sqlite3Parser and sqlite3ParserFree. +*/ +SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(size_t)){ + yyParser *pParser; + pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); + if( pParser ){ + pParser->yyidx = -1; +#ifdef YYTRACKMAXSTACKDEPTH + pParser->yyidxMax = 0; +#endif +#if YYSTACKDEPTH<=0 + pParser->yystack = NULL; + pParser->yystksz = 0; + yyGrowStack(pParser); +#endif + } + return pParser; +} + +/* The following function deletes the value associated with a +** symbol. The symbol can be either a terminal or nonterminal. +** "yymajor" is the symbol code, and "yypminor" is a pointer to +** the value. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + sqlite3ParserARG_FETCH; + switch( yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are not used + ** inside the C code. + */ + case 163: /* select */ + case 195: /* selectnowith */ + case 196: /* oneselect */ + case 207: /* values */ +{ +sqlite3SelectDelete(pParse->db, (yypminor->yy3)); +} + break; + case 174: /* term */ + case 175: /* expr */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy346).pExpr); +} + break; + case 179: /* idxlist_opt */ + case 188: /* idxlist */ + case 200: /* selcollist */ + case 203: /* groupby_opt */ + case 205: /* orderby_opt */ + case 208: /* nexprlist */ + case 209: /* exprlist */ + case 210: /* sclp */ + case 220: /* sortlist */ + case 221: /* setlist */ + case 228: /* case_exprlist */ +{ +sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); +} + break; + case 194: /* fullname */ + case 201: /* from */ + case 212: /* seltablist */ + case 213: /* stl_prefix */ +{ +sqlite3SrcListDelete(pParse->db, (yypminor->yy65)); +} + break; + case 197: /* with */ + case 252: /* wqlist */ +{ +sqlite3WithDelete(pParse->db, (yypminor->yy59)); +} + break; + case 202: /* where_opt */ + case 204: /* having_opt */ + case 216: /* on_opt */ + case 227: /* case_operand */ + case 229: /* case_else */ + case 238: /* when_clause */ + case 243: /* key_opt */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy132)); +} + break; + case 217: /* using_opt */ + case 219: /* idlist */ + case 223: /* inscollist_opt */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy408)); +} + break; + case 234: /* trigger_cmd_list */ + case 239: /* trigger_cmd */ +{ +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy473)); +} + break; + case 236: /* trigger_event */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy378).b); +} + break; + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +** +** Return the major token number for the symbol popped. +*/ +static int yy_pop_parser_stack(yyParser *pParser){ + YYCODETYPE yymajor; + yyStackEntry *yytos = &pParser->yystack[pParser->yyidx]; + + /* There is no mechanism by which the parser stack can be popped below + ** empty in SQLite. */ + if( NEVER(pParser->yyidx<0) ) return 0; +#ifndef NDEBUG + if( yyTraceFILE && pParser->yyidx>=0 ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yymajor = yytos->major; + yy_destructor(pParser, yymajor, &yytos->minor); + pParser->yyidx--; + return yymajor; +} + +/* +** Deallocate and destroy a parser. Destructors are all called for +** all stack elements before shutting the parser down. +** +** Inputs: +**
      +**
    • A pointer to the parser. This should be a pointer +** obtained from sqlite3ParserAlloc. +**
    • A pointer to a function used to reclaim memory obtained +** from malloc. +**
    +*/ +SQLITE_PRIVATE void sqlite3ParserFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ + yyParser *pParser = (yyParser*)p; + /* In SQLite, we never try to destroy a parser that was not successfully + ** created in the first place. */ + if( NEVER(pParser==0) ) return; + while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + free(pParser->yystack); +#endif + (*freeProc)((void*)pParser); +} + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +SQLITE_PRIVATE int sqlite3ParserStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyidxMax; +} +#endif + +/* +** Find the appropriate action for a parser given the terminal +** look-ahead token iLookAhead. +** +** If the look-ahead token is YYNOCODE, then check to see if the action is +** independent of the look-ahead. If it is, return the action, otherwise +** return YY_NO_ACTION. +*/ +static int yy_find_shift_action( + yyParser *pParser, /* The parser */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; + int stateno = pParser->yystack[pParser->yyidx].stateno; + + if( stateno>YY_SHIFT_COUNT + || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ + return yy_default[stateno]; + } + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + if( iLookAhead>0 ){ +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + if( iLookAhead %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + return yy_find_shift_action(pParser, iFallback); + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + } + return yy_default[stateno]; + }else{ + return yy_action[i]; + } +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +** +** If the look-ahead token is YYNOCODE, then check to see if the action is +** independent of the look-ahead. If it is, return the action, otherwise +** return YY_NO_ACTION. +*/ +static int yy_find_reduce_action( + int stateno, /* Current state number */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_COUNT ){ + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT ); +#endif + i = yy_reduce_ofst[stateno]; + assert( i!=YY_REDUCE_USE_DFLT ); + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + return yy_default[stateno]; + } +#else + assert( i>=0 && iyyidx--; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); + } +#endif + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ + + UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ + sqlite3ErrorMsg(pParse, "parser stack overflow"); + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ +} + +/* +** Perform a shift action. +*/ +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + int yyNewState, /* The new state to shift in */ + int yyMajor, /* The major token to shift in */ + YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ +){ + yyStackEntry *yytos; + yypParser->yyidx++; +#ifdef YYTRACKMAXSTACKDEPTH + if( yypParser->yyidx>yypParser->yyidxMax ){ + yypParser->yyidxMax = yypParser->yyidx; + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yyidx>=YYSTACKDEPTH ){ + yyStackOverflow(yypParser, yypMinor); + return; + } +#else + if( yypParser->yyidx>=yypParser->yystksz ){ + yyGrowStack(yypParser); + if( yypParser->yyidx>=yypParser->yystksz ){ + yyStackOverflow(yypParser, yypMinor); + return; + } + } +#endif + yytos = &yypParser->yystack[yypParser->yyidx]; + yytos->stateno = (YYACTIONTYPE)yyNewState; + yytos->major = (YYCODETYPE)yyMajor; + yytos->minor = *yypMinor; +#ifndef NDEBUG + if( yyTraceFILE && yypParser->yyidx>0 ){ + int i; + fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); + fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); + for(i=1; i<=yypParser->yyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + } +#endif +} + +/* The following table contains information about every rule that +** is used during the reduce. +*/ +static const struct { + YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ + unsigned char nrhs; /* Number of right-hand side symbols in the rule */ +} yyRuleInfo[] = { + { 144, 1 }, + { 145, 2 }, + { 145, 1 }, + { 146, 1 }, + { 146, 3 }, + { 147, 0 }, + { 147, 1 }, + { 147, 3 }, + { 148, 1 }, + { 149, 3 }, + { 151, 0 }, + { 151, 1 }, + { 151, 2 }, + { 150, 0 }, + { 150, 1 }, + { 150, 1 }, + { 150, 1 }, + { 149, 2 }, + { 149, 2 }, + { 149, 2 }, + { 153, 1 }, + { 153, 0 }, + { 149, 2 }, + { 149, 3 }, + { 149, 5 }, + { 149, 2 }, + { 154, 6 }, + { 156, 1 }, + { 158, 0 }, + { 158, 3 }, + { 157, 1 }, + { 157, 0 }, + { 155, 5 }, + { 155, 2 }, + { 162, 0 }, + { 162, 2 }, + { 160, 3 }, + { 160, 1 }, + { 164, 3 }, + { 165, 1 }, + { 152, 1 }, + { 152, 1 }, + { 152, 1 }, + { 166, 0 }, + { 166, 1 }, + { 168, 1 }, + { 168, 4 }, + { 168, 6 }, + { 169, 1 }, + { 169, 2 }, + { 170, 1 }, + { 170, 1 }, + { 167, 2 }, + { 167, 0 }, + { 173, 2 }, + { 173, 2 }, + { 173, 4 }, + { 173, 3 }, + { 173, 3 }, + { 173, 2 }, + { 173, 2 }, + { 173, 3 }, + { 173, 5 }, + { 173, 2 }, + { 173, 4 }, + { 173, 4 }, + { 173, 1 }, + { 173, 2 }, + { 178, 0 }, + { 178, 1 }, + { 180, 0 }, + { 180, 2 }, + { 182, 2 }, + { 182, 3 }, + { 182, 3 }, + { 182, 3 }, + { 183, 2 }, + { 183, 2 }, + { 183, 1 }, + { 183, 1 }, + { 183, 2 }, + { 181, 3 }, + { 181, 2 }, + { 184, 0 }, + { 184, 2 }, + { 184, 2 }, + { 161, 0 }, + { 161, 2 }, + { 185, 3 }, + { 185, 1 }, + { 186, 1 }, + { 186, 0 }, + { 187, 2 }, + { 187, 7 }, + { 187, 5 }, + { 187, 5 }, + { 187, 10 }, + { 189, 0 }, + { 189, 1 }, + { 176, 0 }, + { 176, 3 }, + { 190, 0 }, + { 190, 2 }, + { 191, 1 }, + { 191, 1 }, + { 191, 1 }, + { 149, 4 }, + { 193, 2 }, + { 193, 0 }, + { 149, 8 }, + { 149, 4 }, + { 149, 1 }, + { 163, 2 }, + { 195, 1 }, + { 195, 3 }, + { 198, 1 }, + { 198, 2 }, + { 198, 1 }, + { 196, 9 }, + { 196, 1 }, + { 207, 4 }, + { 207, 5 }, + { 199, 1 }, + { 199, 1 }, + { 199, 0 }, + { 210, 2 }, + { 210, 0 }, + { 200, 3 }, + { 200, 2 }, + { 200, 4 }, + { 211, 2 }, + { 211, 1 }, + { 211, 0 }, + { 201, 0 }, + { 201, 2 }, + { 213, 2 }, + { 213, 0 }, + { 212, 7 }, + { 212, 7 }, + { 212, 7 }, + { 159, 0 }, + { 159, 2 }, + { 194, 2 }, + { 214, 1 }, + { 214, 2 }, + { 214, 3 }, + { 214, 4 }, + { 216, 2 }, + { 216, 0 }, + { 215, 0 }, + { 215, 3 }, + { 215, 2 }, + { 217, 4 }, + { 217, 0 }, + { 205, 0 }, + { 205, 3 }, + { 220, 4 }, + { 220, 2 }, + { 177, 1 }, + { 177, 1 }, + { 177, 0 }, + { 203, 0 }, + { 203, 3 }, + { 204, 0 }, + { 204, 2 }, + { 206, 0 }, + { 206, 2 }, + { 206, 4 }, + { 206, 4 }, + { 149, 6 }, + { 202, 0 }, + { 202, 2 }, + { 149, 8 }, + { 221, 5 }, + { 221, 3 }, + { 149, 6 }, + { 149, 7 }, + { 222, 2 }, + { 222, 1 }, + { 223, 0 }, + { 223, 3 }, + { 219, 3 }, + { 219, 1 }, + { 175, 1 }, + { 175, 3 }, + { 174, 1 }, + { 175, 1 }, + { 175, 1 }, + { 175, 3 }, + { 175, 5 }, + { 174, 1 }, + { 174, 1 }, + { 175, 1 }, + { 175, 3 }, + { 175, 6 }, + { 175, 5 }, + { 175, 4 }, + { 174, 1 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 224, 1 }, + { 224, 2 }, + { 175, 3 }, + { 175, 5 }, + { 175, 2 }, + { 175, 3 }, + { 175, 3 }, + { 175, 4 }, + { 175, 2 }, + { 175, 2 }, + { 175, 2 }, + { 175, 2 }, + { 225, 1 }, + { 225, 2 }, + { 175, 5 }, + { 226, 1 }, + { 226, 2 }, + { 175, 5 }, + { 175, 3 }, + { 175, 5 }, + { 175, 4 }, + { 175, 4 }, + { 175, 5 }, + { 228, 5 }, + { 228, 4 }, + { 229, 2 }, + { 229, 0 }, + { 227, 1 }, + { 227, 0 }, + { 209, 1 }, + { 209, 0 }, + { 208, 3 }, + { 208, 1 }, + { 149, 12 }, + { 230, 1 }, + { 230, 0 }, + { 179, 0 }, + { 179, 3 }, + { 188, 5 }, + { 188, 3 }, + { 231, 0 }, + { 231, 2 }, + { 149, 4 }, + { 149, 1 }, + { 149, 2 }, + { 149, 3 }, + { 149, 5 }, + { 149, 6 }, + { 149, 5 }, + { 149, 6 }, + { 232, 1 }, + { 232, 1 }, + { 232, 1 }, + { 232, 1 }, + { 232, 1 }, + { 171, 2 }, + { 171, 1 }, + { 172, 2 }, + { 149, 5 }, + { 233, 11 }, + { 235, 1 }, + { 235, 1 }, + { 235, 2 }, + { 235, 0 }, + { 236, 1 }, + { 236, 1 }, + { 236, 3 }, + { 237, 0 }, + { 237, 3 }, + { 238, 0 }, + { 238, 2 }, + { 234, 3 }, + { 234, 2 }, + { 240, 1 }, + { 240, 3 }, + { 241, 0 }, + { 241, 3 }, + { 241, 2 }, + { 239, 7 }, + { 239, 5 }, + { 239, 5 }, + { 239, 1 }, + { 175, 4 }, + { 175, 6 }, + { 192, 1 }, + { 192, 1 }, + { 192, 1 }, + { 149, 4 }, + { 149, 6 }, + { 149, 3 }, + { 243, 0 }, + { 243, 2 }, + { 242, 1 }, + { 242, 0 }, + { 149, 1 }, + { 149, 3 }, + { 149, 1 }, + { 149, 3 }, + { 149, 6 }, + { 149, 6 }, + { 244, 1 }, + { 245, 0 }, + { 245, 1 }, + { 149, 1 }, + { 149, 4 }, + { 246, 8 }, + { 247, 1 }, + { 247, 3 }, + { 248, 0 }, + { 248, 2 }, + { 249, 1 }, + { 249, 3 }, + { 250, 1 }, + { 251, 0 }, + { 251, 4 }, + { 251, 2 }, + { 197, 0 }, + { 197, 2 }, + { 197, 3 }, + { 252, 6 }, + { 252, 8 }, +}; + +static void yy_accept(yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +*/ +static void yy_reduce( + yyParser *yypParser, /* The parser */ + int yyruleno /* Number of the rule by which to reduce */ +){ + int yygoto; /* The next state */ + int yyact; /* The next action */ + YYMINORTYPE yygotominor; /* The LHS of the rule reduced */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + sqlite3ParserARG_FETCH; + yymsp = &yypParser->yystack[yypParser->yyidx]; +#ifndef NDEBUG + if( yyTraceFILE && yyruleno>=0 + && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, + yyRuleName[yyruleno]); + } +#endif /* NDEBUG */ + + /* Silence complaints from purify about yygotominor being uninitialized + ** in some cases when it is copied into the stack after the following + ** switch. yygotominor is uninitialized when a rule reduces that does + ** not set the value of its left-hand side nonterminal. Leaving the + ** value of the nonterminal uninitialized is utterly harmless as long + ** as the value is never used. So really the only thing this code + ** accomplishes is to quieten purify. + ** + ** 2007-01-16: The wireshark project (www.wireshark.org) reports that + ** without this code, their parser segfaults. I'm not sure what there + ** parser is doing to make this happen. This is the second bug report + ** from wireshark this week. Clearly they are stressing Lemon in ways + ** that it has not been previously stressed... (SQLite ticket #2172) + */ + /*memset(&yygotominor, 0, sizeof(yygotominor));*/ + yygotominor = yyzerominor; + + + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ + case 5: /* explain ::= */ +{ sqlite3BeginParse(pParse, 0); } + break; + case 6: /* explain ::= EXPLAIN */ +{ sqlite3BeginParse(pParse, 1); } + break; + case 7: /* explain ::= EXPLAIN QUERY PLAN */ +{ sqlite3BeginParse(pParse, 2); } + break; + case 8: /* cmdx ::= cmd */ +{ sqlite3FinishCoding(pParse); } + break; + case 9: /* cmd ::= BEGIN transtype trans_opt */ +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy328);} + break; + case 13: /* transtype ::= */ +{yygotominor.yy328 = TK_DEFERRED;} + break; + case 14: /* transtype ::= DEFERRED */ + case 15: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==15); + case 16: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==16); + case 115: /* multiselect_op ::= UNION */ yytestcase(yyruleno==115); + case 117: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==117); +{yygotominor.yy328 = yymsp[0].major;} + break; + case 17: /* cmd ::= COMMIT trans_opt */ + case 18: /* cmd ::= END trans_opt */ yytestcase(yyruleno==18); +{sqlite3CommitTransaction(pParse);} + break; + case 19: /* cmd ::= ROLLBACK trans_opt */ +{sqlite3RollbackTransaction(pParse);} + break; + case 22: /* cmd ::= SAVEPOINT nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); +} + break; + case 23: /* cmd ::= RELEASE savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); +} + break; + case 24: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); +} + break; + case 26: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ +{ + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy328,0,0,yymsp[-2].minor.yy328); +} + break; + case 27: /* createkw ::= CREATE */ +{ + pParse->db->lookaside.bEnabled = 0; + yygotominor.yy0 = yymsp[0].minor.yy0; +} + break; + case 28: /* ifnotexists ::= */ + case 31: /* temp ::= */ yytestcase(yyruleno==31); + case 68: /* autoinc ::= */ yytestcase(yyruleno==68); + case 81: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ yytestcase(yyruleno==81); + case 83: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==83); + case 85: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==85); + case 97: /* defer_subclause_opt ::= */ yytestcase(yyruleno==97); + case 108: /* ifexists ::= */ yytestcase(yyruleno==108); + case 218: /* between_op ::= BETWEEN */ yytestcase(yyruleno==218); + case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); +{yygotominor.yy328 = 0;} + break; + case 29: /* ifnotexists ::= IF NOT EXISTS */ + case 30: /* temp ::= TEMP */ yytestcase(yyruleno==30); + case 69: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==69); + case 84: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==84); + case 107: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==107); + case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); +{yygotominor.yy328 = 1;} + break; + case 32: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ +{ + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy186,0); +} + break; + case 33: /* create_table_args ::= AS select */ +{ + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy3); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); +} + break; + case 34: /* table_options ::= */ +{yygotominor.yy186 = 0;} + break; + case 35: /* table_options ::= WITHOUT nm */ +{ + if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ + yygotominor.yy186 = TF_WithoutRowid; + }else{ + yygotominor.yy186 = 0; + sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); + } +} + break; + case 38: /* column ::= columnid type carglist */ +{ + yygotominor.yy0.z = yymsp[-2].minor.yy0.z; + yygotominor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-2].minor.yy0.z) + pParse->sLastToken.n; +} + break; + case 39: /* columnid ::= nm */ +{ + sqlite3AddColumn(pParse,&yymsp[0].minor.yy0); + yygotominor.yy0 = yymsp[0].minor.yy0; + pParse->constraintName.n = 0; +} + break; + case 40: /* nm ::= ID|INDEXED */ + case 41: /* nm ::= STRING */ yytestcase(yyruleno==41); + case 42: /* nm ::= JOIN_KW */ yytestcase(yyruleno==42); + case 45: /* typetoken ::= typename */ yytestcase(yyruleno==45); + case 48: /* typename ::= ID|STRING */ yytestcase(yyruleno==48); + case 130: /* as ::= AS nm */ yytestcase(yyruleno==130); + case 131: /* as ::= ID|STRING */ yytestcase(yyruleno==131); + case 141: /* dbnm ::= DOT nm */ yytestcase(yyruleno==141); + case 150: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==150); + case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); + case 256: /* nmnum ::= plus_num */ yytestcase(yyruleno==256); + case 257: /* nmnum ::= nm */ yytestcase(yyruleno==257); + case 258: /* nmnum ::= ON */ yytestcase(yyruleno==258); + case 259: /* nmnum ::= DELETE */ yytestcase(yyruleno==259); + case 260: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==260); + case 261: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==261); + case 262: /* plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==262); + case 263: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==263); + case 279: /* trnm ::= nm */ yytestcase(yyruleno==279); +{yygotominor.yy0 = yymsp[0].minor.yy0;} + break; + case 44: /* type ::= typetoken */ +{sqlite3AddColumnType(pParse,&yymsp[0].minor.yy0);} + break; + case 46: /* typetoken ::= typename LP signed RP */ +{ + yygotominor.yy0.z = yymsp[-3].minor.yy0.z; + yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); +} + break; + case 47: /* typetoken ::= typename LP signed COMMA signed RP */ +{ + yygotominor.yy0.z = yymsp[-5].minor.yy0.z; + yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); +} + break; + case 49: /* typename ::= typename ID|STRING */ +{yygotominor.yy0.z=yymsp[-1].minor.yy0.z; yygotominor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} + break; + case 54: /* ccons ::= CONSTRAINT nm */ + case 92: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==92); +{pParse->constraintName = yymsp[0].minor.yy0;} + break; + case 55: /* ccons ::= DEFAULT term */ + case 57: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==57); +{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy346);} + break; + case 56: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy346);} + break; + case 58: /* ccons ::= DEFAULT MINUS term */ +{ + ExprSpan v; + v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy346.pExpr, 0, 0); + v.zStart = yymsp[-1].minor.yy0.z; + v.zEnd = yymsp[0].minor.yy346.zEnd; + sqlite3AddDefaultValue(pParse,&v); +} + break; + case 59: /* ccons ::= DEFAULT ID|INDEXED */ +{ + ExprSpan v; + spanExpr(&v, pParse, TK_STRING, &yymsp[0].minor.yy0); + sqlite3AddDefaultValue(pParse,&v); +} + break; + case 61: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy328);} + break; + case 62: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy328,yymsp[0].minor.yy328,yymsp[-2].minor.yy328);} + break; + case 63: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy328,0,0,0,0);} + break; + case 64: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy346.pExpr);} + break; + case 65: /* ccons ::= REFERENCES nm idxlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy328);} + break; + case 66: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy328);} + break; + case 67: /* ccons ::= COLLATE ID|STRING */ +{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} + break; + case 70: /* refargs ::= */ +{ yygotominor.yy328 = OE_None*0x0101; /* EV: R-19803-45884 */} + break; + case 71: /* refargs ::= refargs refarg */ +{ yygotominor.yy328 = (yymsp[-1].minor.yy328 & ~yymsp[0].minor.yy429.mask) | yymsp[0].minor.yy429.value; } + break; + case 72: /* refarg ::= MATCH nm */ + case 73: /* refarg ::= ON INSERT refact */ yytestcase(yyruleno==73); +{ yygotominor.yy429.value = 0; yygotominor.yy429.mask = 0x000000; } + break; + case 74: /* refarg ::= ON DELETE refact */ +{ yygotominor.yy429.value = yymsp[0].minor.yy328; yygotominor.yy429.mask = 0x0000ff; } + break; + case 75: /* refarg ::= ON UPDATE refact */ +{ yygotominor.yy429.value = yymsp[0].minor.yy328<<8; yygotominor.yy429.mask = 0x00ff00; } + break; + case 76: /* refact ::= SET NULL */ +{ yygotominor.yy328 = OE_SetNull; /* EV: R-33326-45252 */} + break; + case 77: /* refact ::= SET DEFAULT */ +{ yygotominor.yy328 = OE_SetDflt; /* EV: R-33326-45252 */} + break; + case 78: /* refact ::= CASCADE */ +{ yygotominor.yy328 = OE_Cascade; /* EV: R-33326-45252 */} + break; + case 79: /* refact ::= RESTRICT */ +{ yygotominor.yy328 = OE_Restrict; /* EV: R-33326-45252 */} + break; + case 80: /* refact ::= NO ACTION */ +{ yygotominor.yy328 = OE_None; /* EV: R-33326-45252 */} + break; + case 82: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 98: /* defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==98); + case 100: /* onconf ::= ON CONFLICT resolvetype */ yytestcase(yyruleno==100); + case 103: /* resolvetype ::= raisetype */ yytestcase(yyruleno==103); +{yygotominor.yy328 = yymsp[0].minor.yy328;} + break; + case 86: /* conslist_opt ::= */ +{yygotominor.yy0.n = 0; yygotominor.yy0.z = 0;} + break; + case 87: /* conslist_opt ::= COMMA conslist */ +{yygotominor.yy0 = yymsp[-1].minor.yy0;} + break; + case 90: /* tconscomma ::= COMMA */ +{pParse->constraintName.n = 0;} + break; + case 93: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy328,yymsp[-2].minor.yy328,0);} + break; + case 94: /* tcons ::= UNIQUE LP idxlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy328,0,0,0,0);} + break; + case 95: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy346.pExpr);} + break; + case 96: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ +{ + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy328); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy328); +} + break; + case 99: /* onconf ::= */ +{yygotominor.yy328 = OE_Default;} + break; + case 101: /* orconf ::= */ +{yygotominor.yy186 = OE_Default;} + break; + case 102: /* orconf ::= OR resolvetype */ +{yygotominor.yy186 = (u8)yymsp[0].minor.yy328;} + break; + case 104: /* resolvetype ::= IGNORE */ +{yygotominor.yy328 = OE_Ignore;} + break; + case 105: /* resolvetype ::= REPLACE */ +{yygotominor.yy328 = OE_Replace;} + break; + case 106: /* cmd ::= DROP TABLE ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy65, 0, yymsp[-1].minor.yy328); +} + break; + case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ +{ + sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy3, yymsp[-6].minor.yy328, yymsp[-4].minor.yy328); +} + break; + case 110: /* cmd ::= DROP VIEW ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy65, 1, yymsp[-1].minor.yy328); +} + break; + case 111: /* cmd ::= select */ +{ + SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; + sqlite3Select(pParse, yymsp[0].minor.yy3, &dest); + sqlite3ExplainBegin(pParse->pVdbe); + sqlite3ExplainSelect(pParse->pVdbe, yymsp[0].minor.yy3); + sqlite3ExplainFinish(pParse->pVdbe); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); +} + break; + case 112: /* select ::= with selectnowith */ +{ + Select *p = yymsp[0].minor.yy3, *pNext, *pLoop; + if( p ){ + int cnt = 0, mxSelect; + p->pWith = yymsp[-1].minor.yy59; + if( p->pPrior ){ + pNext = 0; + for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ + pLoop->pNext = pNext; + pLoop->selFlags |= SF_Compound; + } + mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; + if( mxSelect && cnt>mxSelect ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); + } + } + }else{ + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy59); + } + yygotominor.yy3 = p; +} + break; + case 113: /* selectnowith ::= oneselect */ + case 119: /* oneselect ::= values */ yytestcase(yyruleno==119); +{yygotominor.yy3 = yymsp[0].minor.yy3;} + break; + case 114: /* selectnowith ::= selectnowith multiselect_op oneselect */ +{ + Select *pRhs = yymsp[0].minor.yy3; + if( pRhs && pRhs->pPrior ){ + SrcList *pFrom; + Token x; + x.n = 0; + pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); + pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); + } + if( pRhs ){ + pRhs->op = (u8)yymsp[-1].minor.yy328; + pRhs->pPrior = yymsp[-2].minor.yy3; + if( yymsp[-1].minor.yy328!=TK_ALL ) pParse->hasCompound = 1; + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy3); + } + yygotominor.yy3 = pRhs; +} + break; + case 116: /* multiselect_op ::= UNION ALL */ +{yygotominor.yy328 = TK_ALL;} + break; + case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ +{ + yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy65,yymsp[-4].minor.yy132,yymsp[-3].minor.yy14,yymsp[-2].minor.yy132,yymsp[-1].minor.yy14,yymsp[-7].minor.yy381,yymsp[0].minor.yy476.pLimit,yymsp[0].minor.yy476.pOffset); +} + break; + case 120: /* values ::= VALUES LP nexprlist RP */ +{ + yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0); +} + break; + case 121: /* values ::= values COMMA LP exprlist RP */ +{ + Select *pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0); + if( pRight ){ + pRight->op = TK_ALL; + pRight->pPrior = yymsp[-4].minor.yy3; + yygotominor.yy3 = pRight; + }else{ + yygotominor.yy3 = yymsp[-4].minor.yy3; + } +} + break; + case 122: /* distinct ::= DISTINCT */ +{yygotominor.yy381 = SF_Distinct;} + break; + case 123: /* distinct ::= ALL */ + case 124: /* distinct ::= */ yytestcase(yyruleno==124); +{yygotominor.yy381 = 0;} + break; + case 125: /* sclp ::= selcollist COMMA */ + case 243: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==243); +{yygotominor.yy14 = yymsp[-1].minor.yy14;} + break; + case 126: /* sclp ::= */ + case 154: /* orderby_opt ::= */ yytestcase(yyruleno==154); + case 161: /* groupby_opt ::= */ yytestcase(yyruleno==161); + case 236: /* exprlist ::= */ yytestcase(yyruleno==236); + case 242: /* idxlist_opt ::= */ yytestcase(yyruleno==242); +{yygotominor.yy14 = 0;} + break; + case 127: /* selcollist ::= sclp expr as */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, yymsp[-1].minor.yy346.pExpr); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yygotominor.yy14,&yymsp[-1].minor.yy346); +} + break; + case 128: /* selcollist ::= sclp STAR */ +{ + Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0); + yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy14, p); +} + break; + case 129: /* selcollist ::= sclp nm DOT STAR */ +{ + Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0); + Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, pDot); +} + break; + case 132: /* as ::= */ +{yygotominor.yy0.n = 0;} + break; + case 133: /* from ::= */ +{yygotominor.yy65 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy65));} + break; + case 134: /* from ::= FROM seltablist */ +{ + yygotominor.yy65 = yymsp[0].minor.yy65; + sqlite3SrcListShiftJoinType(yygotominor.yy65); +} + break; + case 135: /* stl_prefix ::= seltablist joinop */ +{ + yygotominor.yy65 = yymsp[-1].minor.yy65; + if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].jointype = (u8)yymsp[0].minor.yy328; +} + break; + case 136: /* stl_prefix ::= */ +{yygotominor.yy65 = 0;} + break; + case 137: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ +{ + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + sqlite3SrcListIndexedBy(pParse, yygotominor.yy65, &yymsp[-2].minor.yy0); +} + break; + case 138: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ +{ + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy3,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + } + break; + case 139: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ +{ + if( yymsp[-6].minor.yy65==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy132==0 && yymsp[0].minor.yy408==0 ){ + yygotominor.yy65 = yymsp[-4].minor.yy65; + }else if( yymsp[-4].minor.yy65->nSrc==1 ){ + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + if( yygotominor.yy65 ){ + struct SrcList_item *pNew = &yygotominor.yy65->a[yygotominor.yy65->nSrc-1]; + struct SrcList_item *pOld = yymsp[-4].minor.yy65->a; + pNew->zName = pOld->zName; + pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; + pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; + } + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy65); + }else{ + Select *pSubquery; + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy65); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy65,0,0,0,0,SF_NestedFrom,0,0); + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + } + } + break; + case 140: /* dbnm ::= */ + case 149: /* indexed_opt ::= */ yytestcase(yyruleno==149); +{yygotominor.yy0.z=0; yygotominor.yy0.n=0;} + break; + case 142: /* fullname ::= nm dbnm */ +{yygotominor.yy65 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} + break; + case 143: /* joinop ::= COMMA|JOIN */ +{ yygotominor.yy328 = JT_INNER; } + break; + case 144: /* joinop ::= JOIN_KW JOIN */ +{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } + break; + case 145: /* joinop ::= JOIN_KW nm JOIN */ +{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } + break; + case 146: /* joinop ::= JOIN_KW nm nm JOIN */ +{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } + break; + case 147: /* on_opt ::= ON expr */ + case 164: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==164); + case 171: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==171); + case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); + case 233: /* case_operand ::= expr */ yytestcase(yyruleno==233); +{yygotominor.yy132 = yymsp[0].minor.yy346.pExpr;} + break; + case 148: /* on_opt ::= */ + case 163: /* having_opt ::= */ yytestcase(yyruleno==163); + case 170: /* where_opt ::= */ yytestcase(yyruleno==170); + case 232: /* case_else ::= */ yytestcase(yyruleno==232); + case 234: /* case_operand ::= */ yytestcase(yyruleno==234); +{yygotominor.yy132 = 0;} + break; + case 151: /* indexed_opt ::= NOT INDEXED */ +{yygotominor.yy0.z=0; yygotominor.yy0.n=1;} + break; + case 152: /* using_opt ::= USING LP idlist RP */ + case 180: /* inscollist_opt ::= LP idlist RP */ yytestcase(yyruleno==180); +{yygotominor.yy408 = yymsp[-1].minor.yy408;} + break; + case 153: /* using_opt ::= */ + case 179: /* inscollist_opt ::= */ yytestcase(yyruleno==179); +{yygotominor.yy408 = 0;} + break; + case 155: /* orderby_opt ::= ORDER BY sortlist */ + case 162: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==162); + case 235: /* exprlist ::= nexprlist */ yytestcase(yyruleno==235); +{yygotominor.yy14 = yymsp[0].minor.yy14;} + break; + case 156: /* sortlist ::= sortlist COMMA expr sortorder */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14,yymsp[-1].minor.yy346.pExpr); + if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; +} + break; + case 157: /* sortlist ::= expr sortorder */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy346.pExpr); + if( yygotominor.yy14 && ALWAYS(yygotominor.yy14->a) ) yygotominor.yy14->a[0].sortOrder = (u8)yymsp[0].minor.yy328; +} + break; + case 158: /* sortorder ::= ASC */ + case 160: /* sortorder ::= */ yytestcase(yyruleno==160); +{yygotominor.yy328 = SQLITE_SO_ASC;} + break; + case 159: /* sortorder ::= DESC */ +{yygotominor.yy328 = SQLITE_SO_DESC;} + break; + case 165: /* limit_opt ::= */ +{yygotominor.yy476.pLimit = 0; yygotominor.yy476.pOffset = 0;} + break; + case 166: /* limit_opt ::= LIMIT expr */ +{yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr; yygotominor.yy476.pOffset = 0;} + break; + case 167: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yygotominor.yy476.pLimit = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pOffset = yymsp[0].minor.yy346.pExpr;} + break; + case 168: /* limit_opt ::= LIMIT expr COMMA expr */ +{yygotominor.yy476.pOffset = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr;} + break; + case 169: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ +{ + sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy65, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy65,yymsp[0].minor.yy132); +} + break; + case 172: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ +{ + sqlite3WithPush(pParse, yymsp[-7].minor.yy59, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy65, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy14,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy65,yymsp[-1].minor.yy14,yymsp[0].minor.yy132,yymsp[-5].minor.yy186); +} + break; + case 173: /* setlist ::= setlist COMMA nm EQ expr */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy346.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); +} + break; + case 174: /* setlist ::= nm EQ expr */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy346.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); +} + break; + case 175: /* cmd ::= with insert_cmd INTO fullname inscollist_opt select */ +{ + sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); + sqlite3Insert(pParse, yymsp[-2].minor.yy65, yymsp[0].minor.yy3, yymsp[-1].minor.yy408, yymsp[-4].minor.yy186); +} + break; + case 176: /* cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ +{ + sqlite3WithPush(pParse, yymsp[-6].minor.yy59, 1); + sqlite3Insert(pParse, yymsp[-3].minor.yy65, 0, yymsp[-2].minor.yy408, yymsp[-5].minor.yy186); +} + break; + case 177: /* insert_cmd ::= INSERT orconf */ +{yygotominor.yy186 = yymsp[0].minor.yy186;} + break; + case 178: /* insert_cmd ::= REPLACE */ +{yygotominor.yy186 = OE_Replace;} + break; + case 181: /* idlist ::= idlist COMMA nm */ +{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy408,&yymsp[0].minor.yy0);} + break; + case 182: /* idlist ::= nm */ +{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} + break; + case 183: /* expr ::= term */ +{yygotominor.yy346 = yymsp[0].minor.yy346;} + break; + case 184: /* expr ::= LP expr RP */ +{yygotominor.yy346.pExpr = yymsp[-1].minor.yy346.pExpr; spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} + break; + case 185: /* term ::= NULL */ + case 190: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==190); + case 191: /* term ::= STRING */ yytestcase(yyruleno==191); +{spanExpr(&yygotominor.yy346, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} + break; + case 186: /* expr ::= ID|INDEXED */ + case 187: /* expr ::= JOIN_KW */ yytestcase(yyruleno==187); +{spanExpr(&yygotominor.yy346, pParse, TK_ID, &yymsp[0].minor.yy0);} + break; + case 188: /* expr ::= nm DOT nm */ +{ + Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); + spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 189: /* expr ::= nm DOT nm DOT nm */ +{ + Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); + Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); + Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); + spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 192: /* expr ::= VARIABLE */ +{ + if( yymsp[0].minor.yy0.n>=2 && yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1]) ){ + /* When doing a nested parse, one can include terms in an expression + ** that look like this: #1 #2 ... These terms refer to registers + ** in the virtual machine. #N is the N-th register. */ + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = 0; + }else{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); + if( yygotominor.yy346.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy346.pExpr->iTable); + } + }else{ + spanExpr(&yygotominor.yy346, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yygotominor.yy346.pExpr); + } + spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 193: /* expr ::= expr COLLATE ID|STRING */ +{ + yygotominor.yy346.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy346.pExpr, &yymsp[0].minor.yy0); + yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 194: /* expr ::= CAST LP expr AS typetoken RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy346.pExpr, 0, &yymsp[-1].minor.yy0); + spanSet(&yygotominor.yy346,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 195: /* expr ::= ID|INDEXED LP distinct exprlist RP */ +{ + if( yymsp[-1].minor.yy14 && yymsp[-1].minor.yy14->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); + } + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); + spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + if( yymsp[-2].minor.yy381 && yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->flags |= EP_Distinct; + } +} + break; + case 196: /* expr ::= ID|INDEXED LP STAR RP */ +{ + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); + spanSet(&yygotominor.yy346,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 197: /* term ::= CTIME_KW */ +{ + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); + spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 198: /* expr ::= expr AND expr */ + case 199: /* expr ::= expr OR expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==204); + case 205: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==205); +{spanBinaryExpr(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);} + break; + case 206: /* likeop ::= LIKE_KW|MATCH */ +{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 0;} + break; + case 207: /* likeop ::= NOT LIKE_KW|MATCH */ +{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 1;} + break; + case 208: /* expr ::= expr likeop expr */ +{ + ExprList *pList; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy346.pExpr); + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy96.eOperator); + if( yymsp[-1].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; + if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; +} + break; + case 209: /* expr ::= expr likeop expr ESCAPE expr */ +{ + ExprList *pList; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy96.eOperator); + if( yymsp[-3].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; + if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; +} + break; + case 210: /* expr ::= expr ISNULL|NOTNULL */ +{spanUnaryPostfix(&yygotominor.yy346,pParse,yymsp[0].major,&yymsp[-1].minor.yy346,&yymsp[0].minor.yy0);} + break; + case 211: /* expr ::= expr NOT NULL */ +{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);} + break; + case 212: /* expr ::= expr IS expr */ +{ + spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL); +} + break; + case 213: /* expr ::= expr IS NOT expr */ +{ + spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL); +} + break; + case 214: /* expr ::= NOT expr */ + case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); +{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + break; + case 216: /* expr ::= MINUS expr */ +{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UMINUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + break; + case 217: /* expr ::= PLUS expr */ +{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UPLUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + break; + case 220: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pList = pList; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } + if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; +} + break; + case 223: /* expr ::= expr in_op LP exprlist RP */ +{ + if( yymsp[-1].minor.yy14==0 ){ + /* Expressions of the form + ** + ** expr1 IN () + ** expr1 NOT IN () + ** + ** simplify to constants 0 (false) and 1 (true), respectively, + ** regardless of the value of expr1. + */ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy328]); + sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy346.pExpr); + }else if( yymsp[-1].minor.yy14->nExpr==1 ){ + /* Expressions of the form: + ** + ** expr1 IN (?1) + ** expr1 NOT IN (?2) + ** + ** with exactly one value on the RHS can be simplified to something + ** like this: + ** + ** expr1 == ?1 + ** expr1 <> ?2 + ** + ** But, the RHS of the == or <> is marked with the EP_Generic flag + ** so that it may not contribute to the computation of comparison + ** affinity or the collating sequence to use for comparison. Otherwise, + ** the semantics would be subtly different from IN or NOT IN. + */ + Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; + yymsp[-1].minor.yy14->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + /* pRHS cannot be NULL because a malloc error would have been detected + ** before now and control would have never reached this point */ + if( ALWAYS(pRHS) ){ + pRHS->flags &= ~EP_Collate; + pRHS->flags |= EP_Generic; + } + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy328 ? TK_NE : TK_EQ, yymsp[-4].minor.yy346.pExpr, pRHS, 0); + }else{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy14; + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + } + if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + } + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 224: /* expr ::= LP select RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); + } + yygotominor.yy346.zStart = yymsp[-2].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 225: /* expr ::= expr in_op LP select RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); + } + if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 226: /* expr ::= expr in_op nm dbnm */ +{ + SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3SrcListDelete(pParse->db, pSrc); + } + if( yymsp[-2].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-3].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; + } + break; + case 227: /* expr ::= EXISTS LP select RP */ +{ + Expr *p = yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); + if( p ){ + p->x.pSelect = yymsp[-1].minor.yy3; + ExprSetProperty(p, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, p); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); + } + yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy132, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy132 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy132) : yymsp[-2].minor.yy14; + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy132); + } + yygotominor.yy346.zStart = yymsp[-4].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy346.pExpr); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); +} + break; + case 230: /* case_exprlist ::= WHEN expr THEN expr */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); +} + break; + case 237: /* nexprlist ::= nexprlist COMMA expr */ +{yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy346.pExpr);} + break; + case 238: /* nexprlist ::= expr */ +{yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy346.pExpr);} + break; + case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt */ +{ + sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, + sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy328, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy132, SQLITE_SO_ASC, yymsp[-8].minor.yy328); +} + break; + case 240: /* uniqueflag ::= UNIQUE */ + case 291: /* raisetype ::= ABORT */ yytestcase(yyruleno==291); +{yygotominor.yy328 = OE_Abort;} + break; + case 241: /* uniqueflag ::= */ +{yygotominor.yy328 = OE_None;} + break; + case 244: /* idxlist ::= idxlist COMMA nm collate sortorder */ +{ + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, p); + sqlite3ExprListSetName(pParse,yygotominor.yy14,&yymsp[-2].minor.yy0,1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); + if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; +} + break; + case 245: /* idxlist ::= nm collate sortorder */ +{ + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, p); + sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); + if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; +} + break; + case 246: /* collate ::= */ +{yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} + break; + case 248: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy65, yymsp[-1].minor.yy328);} + break; + case 249: /* cmd ::= VACUUM */ + case 250: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==250); +{sqlite3Vacuum(pParse);} + break; + case 251: /* cmd ::= PRAGMA nm dbnm */ +{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} + break; + case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} + break; + case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} + break; + case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} + break; + case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} + break; + case 264: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ +{ + Token all; + all.z = yymsp[-3].minor.yy0.z; + all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy473, &all); +} + break; + case 265: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ +{ + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy328, yymsp[-4].minor.yy378.a, yymsp[-4].minor.yy378.b, yymsp[-2].minor.yy65, yymsp[0].minor.yy132, yymsp[-10].minor.yy328, yymsp[-8].minor.yy328); + yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); +} + break; + case 266: /* trigger_time ::= BEFORE */ + case 269: /* trigger_time ::= */ yytestcase(yyruleno==269); +{ yygotominor.yy328 = TK_BEFORE; } + break; + case 267: /* trigger_time ::= AFTER */ +{ yygotominor.yy328 = TK_AFTER; } + break; + case 268: /* trigger_time ::= INSTEAD OF */ +{ yygotominor.yy328 = TK_INSTEAD;} + break; + case 270: /* trigger_event ::= DELETE|INSERT */ + case 271: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==271); +{yygotominor.yy378.a = yymsp[0].major; yygotominor.yy378.b = 0;} + break; + case 272: /* trigger_event ::= UPDATE OF idlist */ +{yygotominor.yy378.a = TK_UPDATE; yygotominor.yy378.b = yymsp[0].minor.yy408;} + break; + case 275: /* when_clause ::= */ + case 296: /* key_opt ::= */ yytestcase(yyruleno==296); +{ yygotominor.yy132 = 0; } + break; + case 276: /* when_clause ::= WHEN expr */ + case 297: /* key_opt ::= KEY expr */ yytestcase(yyruleno==297); +{ yygotominor.yy132 = yymsp[0].minor.yy346.pExpr; } + break; + case 277: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ +{ + assert( yymsp[-2].minor.yy473!=0 ); + yymsp[-2].minor.yy473->pLast->pNext = yymsp[-1].minor.yy473; + yymsp[-2].minor.yy473->pLast = yymsp[-1].minor.yy473; + yygotominor.yy473 = yymsp[-2].minor.yy473; +} + break; + case 278: /* trigger_cmd_list ::= trigger_cmd SEMI */ +{ + assert( yymsp[-1].minor.yy473!=0 ); + yymsp[-1].minor.yy473->pLast = yymsp[-1].minor.yy473; + yygotominor.yy473 = yymsp[-1].minor.yy473; +} + break; + case 280: /* trnm ::= nm DOT nm */ +{ + yygotominor.yy0 = yymsp[0].minor.yy0; + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); +} + break; + case 282: /* tridxby ::= INDEXED BY nm */ +{ + sqlite3ErrorMsg(pParse, + "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 283: /* tridxby ::= NOT INDEXED */ +{ + sqlite3ErrorMsg(pParse, + "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 284: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ +{ yygotominor.yy473 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy14, yymsp[0].minor.yy132, yymsp[-5].minor.yy186); } + break; + case 285: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ +{yygotominor.yy473 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy408, yymsp[0].minor.yy3, yymsp[-4].minor.yy186);} + break; + case 286: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ +{yygotominor.yy473 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy132);} + break; + case 287: /* trigger_cmd ::= select */ +{yygotominor.yy473 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy3); } + break; + case 288: /* expr ::= RAISE LP IGNORE RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->affinity = OE_Ignore; + } + yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 289: /* expr ::= RAISE LP raisetype COMMA nm RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); + if( yygotominor.yy346.pExpr ) { + yygotominor.yy346.pExpr->affinity = (char)yymsp[-3].minor.yy328; + } + yygotominor.yy346.zStart = yymsp[-5].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 290: /* raisetype ::= ROLLBACK */ +{yygotominor.yy328 = OE_Rollback;} + break; + case 292: /* raisetype ::= FAIL */ +{yygotominor.yy328 = OE_Fail;} + break; + case 293: /* cmd ::= DROP TRIGGER ifexists fullname */ +{ + sqlite3DropTrigger(pParse,yymsp[0].minor.yy65,yymsp[-1].minor.yy328); +} + break; + case 294: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ +{ + sqlite3Attach(pParse, yymsp[-3].minor.yy346.pExpr, yymsp[-1].minor.yy346.pExpr, yymsp[0].minor.yy132); +} + break; + case 295: /* cmd ::= DETACH database_kw_opt expr */ +{ + sqlite3Detach(pParse, yymsp[0].minor.yy346.pExpr); +} + break; + case 300: /* cmd ::= REINDEX */ +{sqlite3Reindex(pParse, 0, 0);} + break; + case 301: /* cmd ::= REINDEX nm dbnm */ +{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 302: /* cmd ::= ANALYZE */ +{sqlite3Analyze(pParse, 0, 0);} + break; + case 303: /* cmd ::= ANALYZE nm dbnm */ +{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 304: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ +{ + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy65,&yymsp[0].minor.yy0); +} + break; + case 305: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ +{ + sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); +} + break; + case 306: /* add_column_fullname ::= fullname */ +{ + pParse->db->lookaside.bEnabled = 0; + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy65); +} + break; + case 309: /* cmd ::= create_vtab */ +{sqlite3VtabFinishParse(pParse,0);} + break; + case 310: /* cmd ::= create_vtab LP vtabarglist RP */ +{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} + break; + case 311: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ +{ + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy328); +} + break; + case 314: /* vtabarg ::= */ +{sqlite3VtabArgInit(pParse);} + break; + case 316: /* vtabargtoken ::= ANY */ + case 317: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==317); + case 318: /* lp ::= LP */ yytestcase(yyruleno==318); +{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} + break; + case 322: /* with ::= */ +{yygotominor.yy59 = 0;} + break; + case 323: /* with ::= WITH wqlist */ + case 324: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==324); +{ yygotominor.yy59 = yymsp[0].minor.yy59; } + break; + case 325: /* wqlist ::= nm idxlist_opt AS LP select RP */ +{ + yygotominor.yy59 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); +} + break; + case 326: /* wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP */ +{ + yygotominor.yy59 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy59, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); +} + break; + default: + /* (0) input ::= cmdlist */ yytestcase(yyruleno==0); + /* (1) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==1); + /* (2) cmdlist ::= ecmd */ yytestcase(yyruleno==2); + /* (3) ecmd ::= SEMI */ yytestcase(yyruleno==3); + /* (4) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==4); + /* (10) trans_opt ::= */ yytestcase(yyruleno==10); + /* (11) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==11); + /* (12) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==12); + /* (20) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==20); + /* (21) savepoint_opt ::= */ yytestcase(yyruleno==21); + /* (25) cmd ::= create_table create_table_args */ yytestcase(yyruleno==25); + /* (36) columnlist ::= columnlist COMMA column */ yytestcase(yyruleno==36); + /* (37) columnlist ::= column */ yytestcase(yyruleno==37); + /* (43) type ::= */ yytestcase(yyruleno==43); + /* (50) signed ::= plus_num */ yytestcase(yyruleno==50); + /* (51) signed ::= minus_num */ yytestcase(yyruleno==51); + /* (52) carglist ::= carglist ccons */ yytestcase(yyruleno==52); + /* (53) carglist ::= */ yytestcase(yyruleno==53); + /* (60) ccons ::= NULL onconf */ yytestcase(yyruleno==60); + /* (88) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==88); + /* (89) conslist ::= tcons */ yytestcase(yyruleno==89); + /* (91) tconscomma ::= */ yytestcase(yyruleno==91); + /* (273) foreach_clause ::= */ yytestcase(yyruleno==273); + /* (274) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==274); + /* (281) tridxby ::= */ yytestcase(yyruleno==281); + /* (298) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==298); + /* (299) database_kw_opt ::= */ yytestcase(yyruleno==299); + /* (307) kwcolumn_opt ::= */ yytestcase(yyruleno==307); + /* (308) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==308); + /* (312) vtabarglist ::= vtabarg */ yytestcase(yyruleno==312); + /* (313) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==313); + /* (315) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==315); + /* (319) anylist ::= */ yytestcase(yyruleno==319); + /* (320) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==320); + /* (321) anylist ::= anylist ANY */ yytestcase(yyruleno==321); + break; + }; + assert( yyruleno>=0 && yyrulenoyyidx -= yysize; + yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); + if( yyact < YYNSTATE ){ +#ifdef NDEBUG + /* If we are not debugging and the reduce action popped at least + ** one element off the stack, then we can push the new element back + ** onto the stack here, and skip the stack overflow test in yy_shift(). + ** That gives a significant speed improvement. */ + if( yysize ){ + yypParser->yyidx++; + yymsp -= yysize-1; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yymsp->minor = yygotominor; + }else +#endif + { + yy_shift(yypParser,yyact,yygoto,&yygotominor); + } + }else{ + assert( yyact == YYNSTATE + YYNRULE + 1 ); + yy_accept(yypParser); + } +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + YYMINORTYPE yyminor /* The minor type of the error token */ +){ + sqlite3ParserARG_FETCH; +#define TOKEN (yyminor.yy0) + + UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ + assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); + } +#endif + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "sqlite3ParserAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
      +**
    • A pointer to the parser (an opaque structure.) +**
    • The major token number. +**
    • The minor token number. +**
    • An option argument of a grammar-specified type. +**
    +** +** Outputs: +** None. +*/ +SQLITE_PRIVATE void sqlite3Parser( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + sqlite3ParserTOKENTYPE yyminor /* The value for the token */ + sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ +){ + YYMINORTYPE yyminorunion; + int yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser; /* The parser */ + + /* (re)initialize the parser, if necessary */ + yypParser = (yyParser*)yyp; + if( yypParser->yyidx<0 ){ +#if YYSTACKDEPTH<=0 + if( yypParser->yystksz <=0 ){ + /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ + yyminorunion = yyzerominor; + yyStackOverflow(yypParser, &yyminorunion); + return; + } +#endif + yypParser->yyidx = 0; + yypParser->yyerrcnt = -1; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; + } + yyminorunion.yy0 = yyminor; +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor==0); +#endif + sqlite3ParserARG_STORE; + +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); + } +#endif + + do{ + yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); + if( yyactyyerrcnt--; + yymajor = YYNOCODE; + }else if( yyact < YYNSTATE + YYNRULE ){ + yy_reduce(yypParser,yyact-YYNSTATE); + }else{ + assert( yyact == YY_ERROR_ACTION ); +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminorunion); + } + yymx = yypParser->yystack[yypParser->yyidx].major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); + yymajor = YYNOCODE; + }else{ + while( + yypParser->yyidx >= 0 && + yymx != YYERRORSYMBOL && + (yyact = yy_find_reduce_action( + yypParser->yystack[yypParser->yyidx].stateno, + YYERRORSYMBOL)) >= YYNSTATE + ){ + yy_pop_parser_stack(yypParser); + } + if( yypParser->yyidx < 0 || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + YYMINORTYPE u2; + u2.YYERRSYMDT = 0; + yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yymajor = YYNOCODE; + +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor,yyminorunion); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); + } + yymajor = YYNOCODE; +#endif + } + }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); + return; +} + +/************** End of parse.c ***********************************************/ +/************** Begin file tokenize.c ****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that splits an SQL input string up into +** individual tokens and sends those tokens one-by-one over to the +** parser for analysis. +*/ +/* #include */ + +/* +** The charMap() macro maps alphabetic characters into their +** lower-case ASCII equivalent. On ASCII machines, this is just +** an upper-to-lower case map. On EBCDIC machines we also need +** to adjust the encoding. Only alphabetic characters and underscores +** need to be translated. +*/ +#ifdef SQLITE_ASCII +# define charMap(X) sqlite3UpperToLower[(unsigned char)X] +#endif +#ifdef SQLITE_EBCDIC +# define charMap(X) ebcdicToAscii[(unsigned char)X] +const unsigned char ebcdicToAscii[] = { +/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ +}; +#endif + +/* +** The sqlite3KeywordCode function looks up an identifier to determine if +** it is a keyword. If it is a keyword, the token code of that keyword is +** returned. If the input is not a keyword, TK_ID is returned. +** +** The implementation of this routine was generated by a program, +** mkkeywordhash.h, located in the tool subdirectory of the distribution. +** The output of the mkkeywordhash.c program is written into a file +** named keywordhash.h and then included into this source file by +** the #include below. +*/ +/************** Include keywordhash.h in the middle of tokenize.c ************/ +/************** Begin file keywordhash.h *************************************/ +/***** This file contains automatically generated code ****** +** +** The code in this file has been automatically generated by +** +** sqlite/tool/mkkeywordhash.c +** +** The code in this file implements a function that determines whether +** or not a given identifier is really an SQL keyword. The same thing +** might be implemented more directly using a hand-written hash table. +** But by using this automatically generated code, the size of the code +** is substantially reduced. This is important for embedded applications +** on platforms with limited memory. +*/ +/* Hash score: 182 */ +static int keywordCode(const char *z, int n){ + /* zText[] encodes 834 bytes of keywords in 554 bytes */ + /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ + /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ + /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ + /* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERECURSIVE */ + /* BETWEENOTNULLIKECASCADELETECASECOLLATECREATECURRENT_DATEDETACH */ + /* IMMEDIATEJOINSERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHEN */ + /* WHERENAMEAFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMIT */ + /* CONFLICTCROSSCURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAIL */ + /* FROMFULLGLOBYIFISNULLORDERESTRICTRIGHTROLLBACKROWUNIONUSING */ + /* VACUUMVIEWINITIALLY */ + static const char zText[553] = { + 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', + 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', + 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', + 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', + 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N', + 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I', + 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E', + 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', + 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', + 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', + 'U','E','R','Y','W','I','T','H','O','U','T','E','R','E','L','E','A','S', + 'E','A','T','T','A','C','H','A','V','I','N','G','R','O','U','P','D','A', + 'T','E','B','E','G','I','N','N','E','R','E','C','U','R','S','I','V','E', + 'B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C','A', + 'S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L','A', + 'T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D','A', + 'T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E','J', + 'O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A','L', + 'Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U','E', + 'S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W','H', + 'E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C','E', + 'A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R','E', + 'M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M','M', + 'I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U','R', + 'R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M','A', + 'R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T','D', + 'R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L','O', + 'B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S','T', + 'R','I','C','T','R','I','G','H','T','R','O','L','L','B','A','C','K','R', + 'O','W','U','N','I','O','N','U','S','I','N','G','V','A','C','U','U','M', + 'V','I','E','W','I','N','I','T','I','A','L','L','Y', + }; + static const unsigned char aHash[127] = { + 76, 105, 117, 74, 0, 45, 0, 0, 82, 0, 77, 0, 0, + 42, 12, 78, 15, 0, 116, 85, 54, 112, 0, 19, 0, 0, + 121, 0, 119, 115, 0, 22, 93, 0, 9, 0, 0, 70, 71, + 0, 69, 6, 0, 48, 90, 102, 0, 118, 101, 0, 0, 44, + 0, 103, 24, 0, 17, 0, 122, 53, 23, 0, 5, 110, 25, + 96, 0, 0, 124, 106, 60, 123, 57, 28, 55, 0, 91, 0, + 100, 26, 0, 99, 0, 0, 0, 95, 92, 97, 88, 109, 14, + 39, 108, 0, 81, 0, 18, 89, 111, 32, 0, 120, 80, 113, + 62, 46, 84, 0, 0, 94, 40, 59, 114, 0, 36, 0, 0, + 29, 0, 86, 63, 64, 0, 20, 61, 0, 56, + }; + static const unsigned char aNext[124] = { + 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, + 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 0, 0, 50, + 0, 43, 3, 47, 0, 0, 0, 0, 30, 0, 58, 0, 38, + 0, 0, 0, 1, 66, 0, 0, 67, 0, 41, 0, 0, 0, + 0, 0, 0, 49, 65, 0, 0, 0, 0, 31, 52, 16, 34, + 10, 0, 0, 0, 0, 0, 0, 0, 11, 72, 79, 0, 8, + 0, 104, 98, 0, 107, 0, 87, 0, 75, 51, 0, 27, 37, + 73, 83, 0, 35, 68, 0, 0, + }; + static const unsigned char aLen[124] = { + 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, + 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, + 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, + 4, 6, 2, 3, 9, 4, 2, 6, 5, 7, 4, 5, 7, + 6, 6, 5, 6, 5, 5, 9, 7, 7, 3, 2, 4, 4, + 7, 3, 6, 4, 7, 6, 12, 6, 9, 4, 6, 5, 4, + 7, 6, 5, 6, 7, 5, 4, 5, 6, 5, 7, 3, 7, + 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, 7, 8, 8, + 2, 4, 4, 4, 4, 4, 2, 2, 6, 5, 8, 5, 8, + 3, 5, 5, 6, 4, 9, 3, + }; + static const unsigned short int aOffset[124] = { + 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, + 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, + 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, + 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 184, 188, 192, + 199, 204, 209, 212, 218, 221, 225, 234, 240, 240, 240, 243, 246, + 250, 251, 255, 261, 265, 272, 278, 290, 296, 305, 307, 313, 318, + 320, 327, 332, 337, 343, 349, 354, 358, 361, 367, 371, 378, 380, + 387, 389, 391, 400, 404, 410, 416, 424, 429, 429, 445, 452, 459, + 460, 467, 471, 475, 479, 483, 486, 488, 490, 496, 500, 508, 513, + 521, 524, 529, 534, 540, 544, 549, + }; + static const unsigned char aCode[124] = { + TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, + TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, + TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, + TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, + TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, + TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, + TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, + TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, + TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, + TK_OR, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH, + TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_GROUP, + TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RECURSIVE, TK_BETWEEN, + TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW, + TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, TK_COLLATE, + TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, TK_JOIN, + TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA, + TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN, + TK_WHERE, TK_RENAME, TK_AFTER, TK_REPLACE, TK_AND, + TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, TK_CAST, + TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, + TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, TK_IS, + TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, TK_LIKE_KW, + TK_BY, TK_IF, TK_ISNULL, TK_ORDER, TK_RESTRICT, + TK_JOIN_KW, TK_ROLLBACK, TK_ROW, TK_UNION, TK_USING, + TK_VACUUM, TK_VIEW, TK_INITIALLY, TK_ALL, + }; + int h, i; + if( n<2 ) return TK_ID; + h = ((charMap(z[0])*4) ^ + (charMap(z[n-1])*3) ^ + n) % 127; + for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){ + if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){ + testcase( i==0 ); /* REINDEX */ + testcase( i==1 ); /* INDEXED */ + testcase( i==2 ); /* INDEX */ + testcase( i==3 ); /* DESC */ + testcase( i==4 ); /* ESCAPE */ + testcase( i==5 ); /* EACH */ + testcase( i==6 ); /* CHECK */ + testcase( i==7 ); /* KEY */ + testcase( i==8 ); /* BEFORE */ + testcase( i==9 ); /* FOREIGN */ + testcase( i==10 ); /* FOR */ + testcase( i==11 ); /* IGNORE */ + testcase( i==12 ); /* REGEXP */ + testcase( i==13 ); /* EXPLAIN */ + testcase( i==14 ); /* INSTEAD */ + testcase( i==15 ); /* ADD */ + testcase( i==16 ); /* DATABASE */ + testcase( i==17 ); /* AS */ + testcase( i==18 ); /* SELECT */ + testcase( i==19 ); /* TABLE */ + testcase( i==20 ); /* LEFT */ + testcase( i==21 ); /* THEN */ + testcase( i==22 ); /* END */ + testcase( i==23 ); /* DEFERRABLE */ + testcase( i==24 ); /* ELSE */ + testcase( i==25 ); /* EXCEPT */ + testcase( i==26 ); /* TRANSACTION */ + testcase( i==27 ); /* ACTION */ + testcase( i==28 ); /* ON */ + testcase( i==29 ); /* NATURAL */ + testcase( i==30 ); /* ALTER */ + testcase( i==31 ); /* RAISE */ + testcase( i==32 ); /* EXCLUSIVE */ + testcase( i==33 ); /* EXISTS */ + testcase( i==34 ); /* SAVEPOINT */ + testcase( i==35 ); /* INTERSECT */ + testcase( i==36 ); /* TRIGGER */ + testcase( i==37 ); /* REFERENCES */ + testcase( i==38 ); /* CONSTRAINT */ + testcase( i==39 ); /* INTO */ + testcase( i==40 ); /* OFFSET */ + testcase( i==41 ); /* OF */ + testcase( i==42 ); /* SET */ + testcase( i==43 ); /* TEMPORARY */ + testcase( i==44 ); /* TEMP */ + testcase( i==45 ); /* OR */ + testcase( i==46 ); /* UNIQUE */ + testcase( i==47 ); /* QUERY */ + testcase( i==48 ); /* WITHOUT */ + testcase( i==49 ); /* WITH */ + testcase( i==50 ); /* OUTER */ + testcase( i==51 ); /* RELEASE */ + testcase( i==52 ); /* ATTACH */ + testcase( i==53 ); /* HAVING */ + testcase( i==54 ); /* GROUP */ + testcase( i==55 ); /* UPDATE */ + testcase( i==56 ); /* BEGIN */ + testcase( i==57 ); /* INNER */ + testcase( i==58 ); /* RECURSIVE */ + testcase( i==59 ); /* BETWEEN */ + testcase( i==60 ); /* NOTNULL */ + testcase( i==61 ); /* NOT */ + testcase( i==62 ); /* NO */ + testcase( i==63 ); /* NULL */ + testcase( i==64 ); /* LIKE */ + testcase( i==65 ); /* CASCADE */ + testcase( i==66 ); /* ASC */ + testcase( i==67 ); /* DELETE */ + testcase( i==68 ); /* CASE */ + testcase( i==69 ); /* COLLATE */ + testcase( i==70 ); /* CREATE */ + testcase( i==71 ); /* CURRENT_DATE */ + testcase( i==72 ); /* DETACH */ + testcase( i==73 ); /* IMMEDIATE */ + testcase( i==74 ); /* JOIN */ + testcase( i==75 ); /* INSERT */ + testcase( i==76 ); /* MATCH */ + testcase( i==77 ); /* PLAN */ + testcase( i==78 ); /* ANALYZE */ + testcase( i==79 ); /* PRAGMA */ + testcase( i==80 ); /* ABORT */ + testcase( i==81 ); /* VALUES */ + testcase( i==82 ); /* VIRTUAL */ + testcase( i==83 ); /* LIMIT */ + testcase( i==84 ); /* WHEN */ + testcase( i==85 ); /* WHERE */ + testcase( i==86 ); /* RENAME */ + testcase( i==87 ); /* AFTER */ + testcase( i==88 ); /* REPLACE */ + testcase( i==89 ); /* AND */ + testcase( i==90 ); /* DEFAULT */ + testcase( i==91 ); /* AUTOINCREMENT */ + testcase( i==92 ); /* TO */ + testcase( i==93 ); /* IN */ + testcase( i==94 ); /* CAST */ + testcase( i==95 ); /* COLUMN */ + testcase( i==96 ); /* COMMIT */ + testcase( i==97 ); /* CONFLICT */ + testcase( i==98 ); /* CROSS */ + testcase( i==99 ); /* CURRENT_TIMESTAMP */ + testcase( i==100 ); /* CURRENT_TIME */ + testcase( i==101 ); /* PRIMARY */ + testcase( i==102 ); /* DEFERRED */ + testcase( i==103 ); /* DISTINCT */ + testcase( i==104 ); /* IS */ + testcase( i==105 ); /* DROP */ + testcase( i==106 ); /* FAIL */ + testcase( i==107 ); /* FROM */ + testcase( i==108 ); /* FULL */ + testcase( i==109 ); /* GLOB */ + testcase( i==110 ); /* BY */ + testcase( i==111 ); /* IF */ + testcase( i==112 ); /* ISNULL */ + testcase( i==113 ); /* ORDER */ + testcase( i==114 ); /* RESTRICT */ + testcase( i==115 ); /* RIGHT */ + testcase( i==116 ); /* ROLLBACK */ + testcase( i==117 ); /* ROW */ + testcase( i==118 ); /* UNION */ + testcase( i==119 ); /* USING */ + testcase( i==120 ); /* VACUUM */ + testcase( i==121 ); /* VIEW */ + testcase( i==122 ); /* INITIALLY */ + testcase( i==123 ); /* ALL */ + return aCode[i]; + } + } + return TK_ID; +} +SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ + return keywordCode((char*)z, n); +} +#define SQLITE_N_KEYWORD 124 + +/************** End of keywordhash.h *****************************************/ +/************** Continuing where we left off in tokenize.c *******************/ + + +/* +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** For EBCDIC, the rules are more complex but have the same +** end result. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identfiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. +*/ +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ + 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ +}; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif + + +/* +** Return the length of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( *z ){ + case ' ': case '\t': case '\n': case '\f': case '\r': { + testcase( z[0]==' ' ); + testcase( z[0]=='\t' ); + testcase( z[0]=='\n' ); + testcase( z[0]=='\f' ); + testcase( z[0]=='\r' ); + for(i=1; sqlite3Isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case '-': { + if( z[1]=='-' ){ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; + } + *tokenType = TK_MINUS; + return 1; + } + case '(': { + *tokenType = TK_LP; + return 1; + } + case ')': { + *tokenType = TK_RP; + return 1; + } + case ';': { + *tokenType = TK_SEMI; + return 1; + } + case '+': { + *tokenType = TK_PLUS; + return 1; + } + case '*': { + *tokenType = TK_STAR; + return 1; + } + case '/': { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; + } + case '%': { + *tokenType = TK_REM; + return 1; + } + case '=': { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case '<': { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case '>': { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case '!': { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 2; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case '|': { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case ',': { + *tokenType = TK_COMMA; + return 1; + } + case '&': { + *tokenType = TK_BITAND; + return 1; + } + case '~': { + *tokenType = TK_BITNOT; + return 1; + } + case '`': + case '\'': + case '"': { + int delim = z[0]; + testcase( delim=='`' ); + testcase( delim=='\'' ); + testcase( delim=='"' ); + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + if( c=='\'' ){ + *tokenType = TK_STRING; + return i+1; + }else if( c!=0 ){ + *tokenType = TK_ID; + return i+1; + }else{ + *tokenType = TK_ILLEGAL; + return i; + } + } + case '.': { +#ifndef SQLITE_OMIT_FLOATING_POINT + if( !sqlite3Isdigit(z[1]) ) +#endif + { + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ + } + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': { + testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); + testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); + testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); + testcase( z[0]=='9' ); + *tokenType = TK_INTEGER; +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } +#endif + for(i=0; sqlite3Isdigit(z[i]); i++){} +#ifndef SQLITE_OMIT_FLOATING_POINT + if( z[i]=='.' ){ + i++; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + i += 2; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } +#endif + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; + } + case '[': { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = c==']' ? TK_ID : TK_ILLEGAL; + return i; + } + case '?': { + *tokenType = TK_VARIABLE; + for(i=1; sqlite3Isdigit(z[i]); i++){} + return i; + } +#ifndef SQLITE_OMIT_TCL_VARIABLE + case '$': +#endif + case '@': /* For compatibility with MS SQL Server */ + case '#': + case ':': { + int n = 0; + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; +#ifndef SQLITE_OMIT_TCL_VARIABLE + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; + } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; +#endif + }else{ + break; + } + } + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case 'x': case 'X': { + testcase( z[0]=='x' ); testcase( z[0]=='X' ); + if( z[1]=='\'' ){ + *tokenType = TK_BLOB; + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } + } + if( z[i] ) i++; + return i; + } + /* Otherwise fall through to the next case */ + } +#endif + default: { + if( !IdChar(*z) ){ + break; + } + for(i=1; IdChar(z[i]); i++){} + *tokenType = keywordCode((char*)z, i); + return i; + } + } + *tokenType = TK_ILLEGAL; + return 1; +} + +/* +** Run the parser on the given SQL string. The parser structure is +** passed in. An SQLITE_ status code is returned. If an error occurs +** then an and attempt is made to write an error message into +** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that +** error message. +*/ +SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ + int nErr = 0; /* Number of errors encountered */ + int i; /* Loop counter */ + void *pEngine; /* The LEMON-generated LALR(1) parser */ + int tokenType; /* type of the next token */ + int lastTokenParsed = -1; /* type of the previous token */ + u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ + sqlite3 *db = pParse->db; /* The database connection */ + int mxSqlLen; /* Max length of an SQL string */ + + + mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + if( db->nVdbeActive==0 ){ + db->u1.isInterrupted = 0; + } + pParse->rc = SQLITE_OK; + pParse->zTail = zSql; + i = 0; + assert( pzErrMsg!=0 ); + pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc); + if( pEngine==0 ){ + db->mallocFailed = 1; + return SQLITE_NOMEM; + } + assert( pParse->pNewTable==0 ); + assert( pParse->pNewTrigger==0 ); + assert( pParse->nVar==0 ); + assert( pParse->nzVar==0 ); + assert( pParse->azVar==0 ); + enableLookaside = db->lookaside.bEnabled; + if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; + while( !db->mallocFailed && zSql[i]!=0 ){ + assert( i>=0 ); + pParse->sLastToken.z = &zSql[i]; + pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); + i += pParse->sLastToken.n; + if( i>mxSqlLen ){ + pParse->rc = SQLITE_TOOBIG; + break; + } + switch( tokenType ){ + case TK_SPACE: { + if( db->u1.isInterrupted ){ + sqlite3ErrorMsg(pParse, "interrupt"); + pParse->rc = SQLITE_INTERRUPT; + goto abort_parse; + } + break; + } + case TK_ILLEGAL: { + sqlite3DbFree(db, *pzErrMsg); + *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"", + &pParse->sLastToken); + nErr++; + goto abort_parse; + } + case TK_SEMI: { + pParse->zTail = &zSql[i]; + /* Fall thru into the default case */ + } + default: { + sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); + lastTokenParsed = tokenType; + if( pParse->rc!=SQLITE_OK ){ + goto abort_parse; + } + break; + } + } + } +abort_parse: + if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ + if( lastTokenParsed!=TK_SEMI ){ + sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); + pParse->zTail = &zSql[i]; + } + sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); + } +#ifdef YYTRACKMAXSTACKDEPTH + sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, + sqlite3ParserStackPeak(pEngine) + ); +#endif /* YYDEBUG */ + sqlite3ParserFree(pEngine, sqlite3_free); + db->lookaside.bEnabled = enableLookaside; + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ + sqlite3SetString(&pParse->zErrMsg, db, "%s", sqlite3ErrStr(pParse->rc)); + } + assert( pzErrMsg!=0 ); + if( pParse->zErrMsg ){ + *pzErrMsg = pParse->zErrMsg; + sqlite3_log(pParse->rc, "%s", *pzErrMsg); + pParse->zErrMsg = 0; + nErr++; + } + if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ + sqlite3VdbeDelete(pParse->pVdbe); + pParse->pVdbe = 0; + } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->nested==0 ){ + sqlite3DbFree(db, pParse->aTableLock); + pParse->aTableLock = 0; + pParse->nTableLock = 0; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_free(pParse->apVtabLock); +#endif + + if( !IN_DECLARE_VTAB ){ + /* If the pParse->declareVtab flag is set, do not delete any table + ** structure built up in pParse->pNewTable. The calling code (see vtab.c) + ** will take responsibility for freeing the Table structure. + */ + sqlite3DeleteTable(db, pParse->pNewTable); + } + + if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); + sqlite3DbFree(db, pParse->azVar); + while( pParse->pAinc ){ + AutoincInfo *p = pParse->pAinc; + pParse->pAinc = p->pNext; + sqlite3DbFree(db, p); + } + while( pParse->pZombieTab ){ + Table *p = pParse->pZombieTab; + pParse->pZombieTab = p->pNextZombie; + sqlite3DeleteTable(db, p); + } + if( nErr>0 && pParse->rc==SQLITE_OK ){ + pParse->rc = SQLITE_ERROR; + } + return nErr; +} + +/************** End of tokenize.c ********************************************/ +/************** Begin file complete.c ****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that implements the sqlite3_complete() API. +** This code used to be part of the tokenizer.c source file. But by +** separating it out, the code will be automatically omitted from +** static links that do not use it. +*/ +#ifndef SQLITE_OMIT_COMPLETE + +/* +** This is defined in tokenize.c. We just have to import the definition. +*/ +#ifndef SQLITE_AMALGAMATION +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +#endif /* SQLITE_AMALGAMATION */ + + +/* +** Token types used by the sqlite3_complete() routine. See the header +** comments on that procedure for additional information. +*/ +#define tkSEMI 0 +#define tkWS 1 +#define tkOTHER 2 +#ifndef SQLITE_OMIT_TRIGGER +#define tkEXPLAIN 3 +#define tkCREATE 4 +#define tkTEMP 5 +#define tkTRIGGER 6 +#define tkEND 7 +#endif + +/* +** Return TRUE if the given SQL string ends in a semicolon. +** +** Special handling is require for CREATE TRIGGER statements. +** Whenever the CREATE TRIGGER keywords are seen, the statement +** must end with ";END;". +** +** This implementation uses a state machine with 8 states: +** +** (0) INVALID We have not yet seen a non-whitespace character. +** +** (1) START At the beginning or end of an SQL statement. This routine +** returns 1 if it ends in the START state and 0 if it ends +** in any other state. +** +** (2) NORMAL We are in the middle of statement which ends with a single +** semicolon. +** +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** a statement. +** +** (4) CREATE The keyword CREATE has been seen at the beginning of a +** statement, possibly preceeded by EXPLAIN and/or followed by +** TEMP or TEMPORARY +** +** (5) TRIGGER We are in the middle of a trigger definition that must be +** ended by a semicolon, the keyword END, and another semicolon. +** +** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at +** the end of a trigger definition. +** +** (7) END We've seen the ";END" of the ";END;" that occurs at the end +** of a trigger difinition. +** +** Transitions between states above are determined by tokens extracted +** from the input. The following tokens are significant: +** +** (0) tkSEMI A semicolon. +** (1) tkWS Whitespace. +** (2) tkOTHER Any other SQL token. +** (3) tkEXPLAIN The "explain" keyword. +** (4) tkCREATE The "create" keyword. +** (5) tkTEMP The "temp" or "temporary" keyword. +** (6) tkTRIGGER The "trigger" keyword. +** (7) tkEND The "end" keyword. +** +** Whitespace never causes a state transition and is always ignored. +** This means that a SQL string of all whitespace is invalid. +** +** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed +** to recognize the end of a trigger can be omitted. All we have to do +** is look for a semicolon that is not part of an string or comment. +*/ +SQLITE_API int sqlite3_complete(const char *zSql){ + u8 state = 0; /* Current state, using numbers defined in header comment */ + u8 token; /* Value of the next token */ + +#ifndef SQLITE_OMIT_TRIGGER + /* A complex statement machine used to detect the end of a CREATE TRIGGER + ** statement. This is the normal case. + */ + static const u8 trans[8][8] = { + /* Token: */ + /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ + /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, + /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, + /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, + /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, + /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, + /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, + }; +#else + /* If triggers are not supported by this compile then the statement machine + ** used to detect the end of a statement is much simplier + */ + static const u8 trans[3][3] = { + /* Token: */ + /* State: ** SEMI WS OTHER */ + /* 0 INVALID: */ { 1, 0, 2, }, + /* 1 START: */ { 1, 1, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, }, + }; +#endif /* SQLITE_OMIT_TRIGGER */ + + while( *zSql ){ + switch( *zSql ){ + case ';': { /* A semicolon */ + token = tkSEMI; + break; + } + case ' ': + case '\r': + case '\t': + case '\n': + case '\f': { /* White space is ignored */ + token = tkWS; + break; + } + case '/': { /* C-style comments */ + if( zSql[1]!='*' ){ + token = tkOTHER; + break; + } + zSql += 2; + while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } + if( zSql[0]==0 ) return 0; + zSql++; + token = tkWS; + break; + } + case '-': { /* SQL-style comments from "--" to end of line */ + if( zSql[1]!='-' ){ + token = tkOTHER; + break; + } + while( *zSql && *zSql!='\n' ){ zSql++; } + if( *zSql==0 ) return state==1; + token = tkWS; + break; + } + case '[': { /* Microsoft-style identifiers in [...] */ + zSql++; + while( *zSql && *zSql!=']' ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + case '`': /* Grave-accent quoted symbols used by MySQL */ + case '"': /* single- and double-quoted strings */ + case '\'': { + int c = *zSql; + zSql++; + while( *zSql && *zSql!=c ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + default: { +#ifdef SQLITE_EBCDIC + unsigned char c; +#endif + if( IdChar((u8)*zSql) ){ + /* Keywords and unquoted identifiers */ + int nId; + for(nId=1; IdChar(zSql[nId]); nId++){} +#ifdef SQLITE_OMIT_TRIGGER + token = tkOTHER; +#else + switch( *zSql ){ + case 'c': case 'C': { + if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ + token = tkCREATE; + }else{ + token = tkOTHER; + } + break; + } + case 't': case 'T': { + if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ + token = tkTRIGGER; + }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ + token = tkTEMP; + }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ + token = tkTEMP; + }else{ + token = tkOTHER; + } + break; + } + case 'e': case 'E': { + if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ + token = tkEND; + }else +#ifndef SQLITE_OMIT_EXPLAIN + if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ + token = tkEXPLAIN; + }else +#endif + { + token = tkOTHER; + } + break; + } + default: { + token = tkOTHER; + break; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ + zSql += nId-1; + }else{ + /* Operators and special symbols */ + token = tkOTHER; + } + break; + } + } + state = trans[state][token]; + zSql++; + } + return state==1; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine is the same as the sqlite3_complete() routine described +** above, except that the parameter is required to be UTF-16 encoded, not +** UTF-8. +*/ +SQLITE_API int sqlite3_complete16(const void *zSql){ + sqlite3_value *pVal; + char const *zSql8; + int rc = SQLITE_NOMEM; + +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zSql8 ){ + rc = sqlite3_complete(zSql8); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3ValueFree(pVal); + return sqlite3ApiExit(0, rc); +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_COMPLETE */ + +/************** End of complete.c ********************************************/ +/************** Begin file main.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ + +#ifdef SQLITE_ENABLE_FTS3 +/************** Include fts3.h in the middle of main.c ***********************/ +/************** Begin file fts3.h ********************************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +*/ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of fts3.h ************************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_RTREE +/************** Include rtree.h in the middle of main.c **********************/ +/************** Begin file rtree.h *******************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** RTREE library. All it does is declare the sqlite3RtreeInit() interface. +*/ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of rtree.h ***********************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_ICU +/************** Include sqliteicu.h in the middle of main.c ******************/ +/************** Begin file sqliteicu.h ***************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** ICU extension. All it does is declare the sqlite3IcuInit() interface. +*/ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + + +/************** End of sqliteicu.h *******************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif + +#ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ +SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +#endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ +SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. +*/ +SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ +SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled with mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ +SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } + +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) +/* +** If the following function pointer is not NULL and if +** SQLITE_ENABLE_IOTRACE is enabled, then messages describing +** I/O active are written using this function. These messages +** are intended for debugging activity only. +*/ +SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*, ...) = 0; +#endif + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** temporary files. +** +** See also the "PRAGMA temp_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_temp_directory = 0; + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** all database files specified with a relative pathname. +** +** See also the "PRAGMA data_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_data_directory = 0; + +/* +** Initialize SQLite. +** +** This routine must be called to initialize the memory allocation, +** VFS, and mutex subsystems prior to doing any serious work with +** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite3_open(). +** +** This routine is a no-op except on its very first call for the process, +** or for the first call after a call to sqlite3_shutdown. +** +** The first thread to call this routine runs the initialization to +** completion. If subsequent threads call this routine before the first +** thread has finished the initialization process, then the subsequent +** threads must block until the first thread finishes with the initialization. +** +** The first thread might call this routine recursively. Recursive +** calls to this routine should not block, of course. Otherwise the +** initialization process would never complete. +** +** Let X be the first thread to enter this routine. Let Y be some other +** thread. Then while the initial invocation of this routine by X is +** incomplete, it is required that: +** +** * Calls to this routine from Y must block until the outer-most +** call by X completes. +** +** * Recursive calls to this routine from thread X return immediately +** without blocking. +*/ +SQLITE_API int sqlite3_initialize(void){ + MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ + int rc; /* Result code */ +#ifdef SQLITE_EXTRA_INIT + int bRunExtraInit = 0; /* Extra initialization needed */ +#endif + +#ifdef SQLITE_OMIT_WSD + rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + /* If SQLite is already completely initialized, then this call + ** to sqlite3_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; + + /* Make sure the mutex subsystem is initialized. If unable to + ** initialize the mutex subsystem, return early with the error. + ** If the system is so sick that we are unable to allocate a mutex, + ** there is not much SQLite is going to be able to do. + ** + ** The mutex subsystem must take care of serializing its own + ** initialization. + */ + rc = sqlite3MutexInit(); + if( rc ) return rc; + + /* Initialize the malloc() system and the recursive pInitMutex mutex. + ** This operation is protected by the STATIC_MASTER mutex. Note that + ** MutexAlloc() is called for a static mutex prior to initializing the + ** malloc subsystem - this implies that the allocation of a static + ** mutex must not require support from the malloc subsystem. + */ + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.isMutexInit = 1; + if( !sqlite3GlobalConfig.isMallocInit ){ + rc = sqlite3MallocInit(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isMallocInit = 1; + if( !sqlite3GlobalConfig.pInitMutex ){ + sqlite3GlobalConfig.pInitMutex = + sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ + rc = SQLITE_NOMEM; + } + } + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.nRefInitMutex++; + } + sqlite3_mutex_leave(pMaster); + + /* If rc is not SQLITE_OK at this point, then either the malloc + ** subsystem could not be initialized or the system failed to allocate + ** the pInitMutex mutex. Return an error in either case. */ + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Do the rest of the initialization under the recursive mutex so + ** that we will be able to handle recursive calls into + ** sqlite3_initialize(). The recursive calls normally come through + ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other + ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). + */ + sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); + if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + sqlite3GlobalConfig.inProgress = 1; + memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); + sqlite3RegisterGlobalFunctions(); + if( sqlite3GlobalConfig.isPCacheInit==0 ){ + rc = sqlite3PcacheInitialize(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isPCacheInit = 1; + rc = sqlite3OsInit(); + } + if( rc==SQLITE_OK ){ + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + sqlite3GlobalConfig.isInit = 1; +#ifdef SQLITE_EXTRA_INIT + bRunExtraInit = 1; +#endif + } + sqlite3GlobalConfig.inProgress = 0; + } + sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); + + /* Go back under the static mutex and clean up the recursive + ** mutex to prevent a resource leak. + */ + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.nRefInitMutex--; + if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ + assert( sqlite3GlobalConfig.nRefInitMutex==0 ); + sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); + sqlite3GlobalConfig.pInitMutex = 0; + } + sqlite3_mutex_leave(pMaster); + + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG +#ifndef SQLITE_OMIT_FLOATING_POINT + /* This section of code's only "output" is via assert() statements. */ + if ( rc==SQLITE_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite3IsNaN(y) ); + } +#endif +#endif + + /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT + ** compile-time option. + */ +#ifdef SQLITE_EXTRA_INIT + if( bRunExtraInit ){ + int SQLITE_EXTRA_INIT(const char*); + rc = SQLITE_EXTRA_INIT(0); + } +#endif + + return rc; +} + +/* +** Undo the effects of sqlite3_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. But it is safe to invoke this routine +** on when SQLite is already shut down. If SQLite is already shut down +** when this routine is invoked, then this routine is a harmless no-op. +*/ +SQLITE_API int sqlite3_shutdown(void){ + if( sqlite3GlobalConfig.isInit ){ +#ifdef SQLITE_EXTRA_SHUTDOWN + void SQLITE_EXTRA_SHUTDOWN(void); + SQLITE_EXTRA_SHUTDOWN(); +#endif + sqlite3_os_end(); + sqlite3_reset_auto_extension(); + sqlite3GlobalConfig.isInit = 0; + } + if( sqlite3GlobalConfig.isPCacheInit ){ + sqlite3PcacheShutdown(); + sqlite3GlobalConfig.isPCacheInit = 0; + } + if( sqlite3GlobalConfig.isMallocInit ){ + sqlite3MallocEnd(); + sqlite3GlobalConfig.isMallocInit = 0; + +#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + /* The heap subsystem has now been shutdown and these values are supposed + ** to be NULL or point to memory that was obtained from sqlite3_malloc(), + ** which would rely on that heap subsystem; therefore, make sure these + ** values cannot refer to heap memory that was just invalidated when the + ** heap subsystem was shutdown. This is only done if the current call to + ** this function resulted in the heap subsystem actually being shutdown. + */ + sqlite3_data_directory = 0; + sqlite3_temp_directory = 0; +#endif + } + if( sqlite3GlobalConfig.isMutexInit ){ + sqlite3MutexEnd(); + sqlite3GlobalConfig.isMutexInit = 0; + } + + return SQLITE_OK; +} + +/* +** This API allows applications to modify the global configuration of +** the SQLite library at run-time. +** +** This routine should only be called when there are no outstanding +** database connections or memory allocations. This routine is not +** threadsafe. Failure to heed these warnings can lead to unpredictable +** behavior. +*/ +SQLITE_API int sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + + va_start(ap, op); + switch( op ){ + + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 + case SQLITE_CONFIG_SINGLETHREAD: { + /* Disable all mutexing */ + sqlite3GlobalConfig.bCoreMutex = 0; + sqlite3GlobalConfig.bFullMutex = 0; + break; + } + case SQLITE_CONFIG_MULTITHREAD: { + /* Disable mutexing of database connections */ + /* Enable mutexing of core data structures */ + sqlite3GlobalConfig.bCoreMutex = 1; + sqlite3GlobalConfig.bFullMutex = 0; + break; + } + case SQLITE_CONFIG_SERIALIZED: { + /* Enable all mutexing */ + sqlite3GlobalConfig.bCoreMutex = 1; + sqlite3GlobalConfig.bFullMutex = 1; + break; + } + case SQLITE_CONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); + break; + } + case SQLITE_CONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + break; + } +#endif + + + case SQLITE_CONFIG_MALLOC: { + /* Specify an alternative malloc implementation */ + sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); + break; + } + case SQLITE_CONFIG_GETMALLOC: { + /* Retrieve the current malloc() implementation */ + if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); + *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; + break; + } + case SQLITE_CONFIG_MEMSTATUS: { + /* Enable or disable the malloc status collection */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_SCRATCH: { + /* Designate a buffer for scratch memory space */ + sqlite3GlobalConfig.pScratch = va_arg(ap, void*); + sqlite3GlobalConfig.szScratch = va_arg(ap, int); + sqlite3GlobalConfig.nScratch = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PAGECACHE: { + /* Designate a buffer for page cache memory space */ + sqlite3GlobalConfig.pPage = va_arg(ap, void*); + sqlite3GlobalConfig.szPage = va_arg(ap, int); + sqlite3GlobalConfig.nPage = va_arg(ap, int); + break; + } + + case SQLITE_CONFIG_PCACHE: { + /* no-op */ + break; + } + case SQLITE_CONFIG_GETPCACHE: { + /* now an error */ + rc = SQLITE_ERROR; + break; + } + + case SQLITE_CONFIG_PCACHE2: { + /* Specify an alternative page cache implementation */ + sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); + break; + } + case SQLITE_CONFIG_GETPCACHE2: { + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + sqlite3PCacheSetDefault(); + } + *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; + break; + } + +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + case SQLITE_CONFIG_HEAP: { + /* Designate a buffer for heap memory space */ + sqlite3GlobalConfig.pHeap = va_arg(ap, void*); + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + sqlite3GlobalConfig.mnReq = va_arg(ap, int); + + if( sqlite3GlobalConfig.mnReq<1 ){ + sqlite3GlobalConfig.mnReq = 1; + }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ + /* cap min request size at 2^12 */ + sqlite3GlobalConfig.mnReq = (1<<12); + } + + if( sqlite3GlobalConfig.pHeap==0 ){ + /* If the heap pointer is NULL, then restore the malloc implementation + ** back to NULL pointers too. This will cause the malloc to go + ** back to its default implementation when sqlite3_initialize() is + ** run. + */ + memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); + }else{ + /* The heap pointer is not NULL, then install one of the + ** mem5.c/mem3.c methods. The enclosing #if guarantees at + ** least one of these methods is currently enabled. + */ +#ifdef SQLITE_ENABLE_MEMSYS3 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); +#endif + } + break; + } +#endif + + case SQLITE_CONFIG_LOOKASIDE: { + sqlite3GlobalConfig.szLookaside = va_arg(ap, int); + sqlite3GlobalConfig.nLookaside = va_arg(ap, int); + break; + } + + /* Record a pointer to the logger function and its first argument. + ** The default is NULL. Logging is disabled if the function pointer is + ** NULL. + */ + case SQLITE_CONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); + sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + break; + } + + case SQLITE_CONFIG_URI: { + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } + + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + + case SQLITE_CONFIG_MMAP_SIZE: { + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + sqlite3GlobalConfig.mxMmap = mxMmap; + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) + case SQLITE_CONFIG_WIN32_HEAPSIZE: { + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + break; + } +#endif + + default: { + rc = SQLITE_ERROR; + break; + } + } + va_end(ap); + return rc; +} + +/* +** Set up the lookaside buffers for a database connection. +** Return SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** The sz parameter is the number of bytes in each lookaside slot. +** The cnt parameter is the number of slots. If pStart is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc(). +** If pStart is not NULL then it is sz*cnt bytes of memory to use for +** the lookaside memory. +*/ +static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ + void *pStart; + if( db->lookaside.nOut ){ + return SQLITE_BUSY; + } + /* Free any existing lookaside buffer for this handle before + ** allocating a new one so we don't have to have space for + ** both at the same time. + */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than a pointer to be useful. + */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; + if( cnt<0 ) cnt = 0; + if( sz==0 || cnt==0 ){ + sz = 0; + pStart = 0; + }else if( pBuf==0 ){ + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ + sqlite3EndBenignMalloc(); + if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; + }else{ + pStart = pBuf; + } + db->lookaside.pStart = pStart; + db->lookaside.pFree = 0; + db->lookaside.sz = (u16)sz; + if( pStart ){ + int i; + LookasideSlot *p; + assert( sz > (int)sizeof(LookasideSlot*) ); + p = (LookasideSlot*)pStart; + for(i=cnt-1; i>=0; i--){ + p->pNext = db->lookaside.pFree; + db->lookaside.pFree = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } + db->lookaside.pEnd = p; + db->lookaside.bEnabled = 1; + db->lookaside.bMalloced = pBuf==0 ?1:0; + }else{ + db->lookaside.pStart = db; + db->lookaside.pEnd = db; + db->lookaside.bEnabled = 0; + db->lookaside.bMalloced = 0; + } + return SQLITE_OK; +} + +/* +** Return the mutex associated with a database connection. +*/ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ + return db->mutex; +} + +/* +** Free up as much memory as we can from the given database +** connection. +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3 *db){ + int i; + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerShrink(pPager); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Configuration settings for an individual database connection +*/ +SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc; + va_start(ap, op); + switch( op ){ + case SQLITE_DBCONFIG_LOOKASIDE: { + void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ + rc = setupLookaside(db, pBuf, sz, cnt); + break; + } + default: { + static const struct { + int op; /* The opcode */ + u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + } aFlagOp[] = { + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + }; + unsigned int i; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ + for(i=0; iflags; + if( onoff>0 ){ + db->flags |= aFlagOp[i].mask; + }else if( onoff==0 ){ + db->flags &= ~aFlagOp[i].mask; + } + if( oldFlags!=db->flags ){ + sqlite3ExpirePreparedStatements(db); + } + if( pRes ){ + *pRes = (db->flags & aFlagOp[i].mask)!=0; + } + rc = SQLITE_OK; + break; + } + } + break; + } + } + va_end(ap); + return rc; +} + + +/* +** Return true if the buffer z[0..n-1] contains all spaces. +*/ +static int allSpaces(const char *z, int n){ + while( n>0 && z[n-1]==' ' ){ n--; } + return n==0; +} + +/* +** This is the default collating function named "BINARY" which is always +** available. +** +** If the padFlag argument is not NULL then space padding at the end +** of strings is ignored. This implements the RTRIM collation. +*/ +static int binCollFunc( + void *padFlag, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + n = nKey1lastRowid; +} + +/* +** Return the number of changes in the most recent call to sqlite3_exec(). +*/ +SQLITE_API int sqlite3_changes(sqlite3 *db){ + return db->nChange; +} + +/* +** Return the number of changes since the database handle was opened. +*/ +SQLITE_API int sqlite3_total_changes(sqlite3 *db){ + return db->nTotalChange; +} + +/* +** Close all open savepoints. This function only manipulates fields of the +** database handle object, it does not close any savepoints that may be open +** at the b-tree/pager level. +*/ +SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ + while( db->pSavepoint ){ + Savepoint *pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + } + db->nSavepoint = 0; + db->nStatement = 0; + db->isTransactionSavepoint = 0; +} + +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } + } +} + +/* +** Disconnect all sqlite3_vtab objects that belong to database connection +** db. This is called when db is being closed. +*/ +static void disconnectAllVtab(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int i; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( db->aDb[i].pSchema ){ + HashElem *p; + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + Table *pTab = (Table *)sqliteHashData(p); + if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); + } + } + } + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); +#else + UNUSED_PARAMETER(db); +#endif +} + +/* +** Return TRUE if database connection db has unfinalized prepared +** statements or unfinished sqlite3_backup objects. +*/ +static int connectionIsBusy(sqlite3 *db){ + int j; + assert( sqlite3_mutex_held(db->mutex) ); + if( db->pVdbe ) return 1; + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; + } + return 0; +} + +/* +** Close an existing SQLite database +*/ +static int sqlite3Close(sqlite3 *db, int forceZombie){ + if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ + return SQLITE_OK; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + + /* Force xDisconnect calls on all virtual tables */ + disconnectAllVtab(db); + + /* If a transaction is open, the disconnectAllVtab() call above + ** will not have called the xDisconnect() method on any virtual + ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() + ** call will do so. We need to do this before the check for active + ** SQL statements below, as the v-table implementation may be storing + ** some prepared statements internally. + */ + sqlite3VtabRollback(db); + + /* Legacy behavior (sqlite3_close() behavior) is to return + ** SQLITE_BUSY if the connection can not be closed immediately. + */ + if( !forceZombie && connectionIsBusy(db) ){ + sqlite3Error(db, SQLITE_BUSY, "unable to close due to unfinalized " + "statements or unfinished backups"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + + /* Convert the connection into a zombie and then close it. + */ + db->magic = SQLITE_MAGIC_ZOMBIE; + sqlite3LeaveMutexAndCloseZombie(db); + return SQLITE_OK; +} + +/* +** Two variations on the public interface for closing a database +** connection. The sqlite3_close() version returns SQLITE_BUSY and +** leaves the connection option if there are unfinalized prepared +** statements or unfinished sqlite3_backups. The sqlite3_close_v2() +** version forces the connection to become a zombie if there are +** unclosed resources, and arranges for deallocation when the last +** prepare statement or sqlite3_backup closes. +*/ +SQLITE_API int sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } +SQLITE_API int sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } + + +/* +** Close the mutex on database connection db. +** +** Furthermore, if database connection db is a zombie (meaning that there +** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and +** every sqlite3_stmt has now been finalized and every sqlite3_backup has +** finished, then free all resources. +*/ +SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ + HashElem *i; /* Hash table iterator */ + int j; + + /* If there are outstanding sqlite3_stmt or sqlite3_backup objects + ** or if the connection has not yet been closed by sqlite3_close_v2(), + ** then just leave the mutex and return. + */ + if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ + sqlite3_mutex_leave(db->mutex); + return; + } + + /* If we reach this point, it means that the database connection has + ** closed all sqlite3_stmt and sqlite3_backup objects and has been + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, + ** go ahead and free all resources. + */ + + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + + /* Free any outstanding Savepoint structures. */ + sqlite3CloseSavepoints(db); + + /* Close all database connections */ + for(j=0; jnDb; j++){ + struct Db *pDb = &db->aDb[j]; + if( pDb->pBt ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + if( j!=1 ){ + pDb->pSchema = 0; + } + } + } + /* Clear the TEMP schema separately and last */ + if( db->aDb[1].pSchema ){ + sqlite3SchemaClear(db->aDb[1].pSchema); + } + sqlite3VtabUnlockList(db); + + /* Free up the array of auxiliary databases */ + sqlite3CollapseDatabaseArray(db); + assert( db->nDb<=2 ); + assert( db->aDb==db->aDbStatic ); + + /* Tell the code in notify.c that the connection no longer holds any + ** locks and does not require any further unlock-notify callbacks. + */ + sqlite3ConnectionClosed(db); + + for(j=0; jaFunc.a); j++){ + FuncDef *pNext, *pHash, *p; + for(p=db->aFunc.a[j]; p; p=pHash){ + pHash = p->pHash; + while( p ){ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + } + } + } + for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(i); + /* Invoke any destructors registered for collation sequence user data. */ + for(j=0; j<3; j++){ + if( pColl[j].xDel ){ + pColl[j].xDel(pColl[j].pUser); + } + } + sqlite3DbFree(db, pColl); + } + sqlite3HashClear(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ + Module *pMod = (Module *)sqliteHashData(i); + if( pMod->xDestroy ){ + pMod->xDestroy(pMod->pAux); + } + sqlite3DbFree(db, pMod); + } + sqlite3HashClear(&db->aModule); +#endif + + sqlite3Error(db, SQLITE_OK, 0); /* Deallocates any cached error strings. */ + sqlite3ValueFree(db->pErr); + sqlite3CloseExtensions(db); + + db->magic = SQLITE_MAGIC_ERROR; + + /* The temp-database schema is allocated differently from the other schema + ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). + ** So it needs to be freed here. Todo: Why not roll the temp schema into + ** the same sqliteMalloc() as the one that allocates the database + ** structure? + */ + sqlite3DbFree(db, db->aDb[1].pSchema); + sqlite3_mutex_leave(db->mutex); + db->magic = SQLITE_MAGIC_CLOSED; + sqlite3_mutex_free(db->mutex); + assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + sqlite3_free(db); +} + +/* +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. +*/ +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ + int i; + int inTrans = 0; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeIsInTrans(p) ){ + inTrans = 1; + } + sqlite3BtreeRollback(p, tripCode); + } + } + sqlite3VtabRollback(db); + sqlite3EndBenignMalloc(); + + if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ + sqlite3ExpirePreparedStatements(db); + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3BtreeLeaveAll(db); + + /* Any deferred constraint violations have now been resolved. */ + db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~SQLITE_DeferFKs; + + /* If one has been configured, invoke the rollback-hook callback */ + if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ + db->xRollbackCallback(db->pRollbackArg); + } +} + +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) || defined(SQLITE_TEST) +SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + +/* +** Return a static string that describes the kind of error specified in the +** argument. +*/ +SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ + static const char* const aMsg[] = { + /* SQLITE_OK */ "not an error", + /* SQLITE_ERROR */ "SQL logic error or missing database", + /* SQLITE_INTERNAL */ 0, + /* SQLITE_PERM */ "access permission denied", + /* SQLITE_ABORT */ "callback requested query abort", + /* SQLITE_BUSY */ "database is locked", + /* SQLITE_LOCKED */ "database table is locked", + /* SQLITE_NOMEM */ "out of memory", + /* SQLITE_READONLY */ "attempt to write a readonly database", + /* SQLITE_INTERRUPT */ "interrupted", + /* SQLITE_IOERR */ "disk I/O error", + /* SQLITE_CORRUPT */ "database disk image is malformed", + /* SQLITE_NOTFOUND */ "unknown operation", + /* SQLITE_FULL */ "database or disk is full", + /* SQLITE_CANTOPEN */ "unable to open database file", + /* SQLITE_PROTOCOL */ "locking protocol", + /* SQLITE_EMPTY */ "table contains no data", + /* SQLITE_SCHEMA */ "database schema has changed", + /* SQLITE_TOOBIG */ "string or blob too big", + /* SQLITE_CONSTRAINT */ "constraint failed", + /* SQLITE_MISMATCH */ "datatype mismatch", + /* SQLITE_MISUSE */ "library routine called out of sequence", + /* SQLITE_NOLFS */ "large file support is disabled", + /* SQLITE_AUTH */ "authorization denied", + /* SQLITE_FORMAT */ "auxiliary database format error", + /* SQLITE_RANGE */ "bind or column index out of range", + /* SQLITE_NOTADB */ "file is encrypted or is not a database", + }; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rcbusyTimeout; + int delay, prior; + + assert( count>=0 ); + if( count < NDELAY ){ + delay = delays[count]; + prior = totals[count]; + }else{ + delay = delays[NDELAY-1]; + prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); + } + if( prior + delay > timeout ){ + delay = timeout - prior; + if( delay<=0 ) return 0; + } + sqlite3OsSleep(db->pVfs, delay*1000); + return 1; +#else + sqlite3 *db = (sqlite3 *)ptr; + int timeout = ((sqlite3 *)ptr)->busyTimeout; + if( (count+1)*1000 > timeout ){ + return 0; + } + sqlite3OsSleep(db->pVfs, 1000000); + return 1; +#endif +} + +/* +** Invoke the given busy handler. +** +** This routine is called when an operation failed with a lock. +** If this routine returns non-zero, the lock is retried. If it +** returns 0, the operation aborts with an SQLITE_BUSY error. +*/ +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ + int rc; + if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; + rc = p->xFunc(p->pArg, p->nBusy); + if( rc==0 ){ + p->nBusy = -1; + }else{ + p->nBusy++; + } + return rc; +} + +/* +** This routine sets the busy callback for an Sqlite database to the +** given callback function with the given argument. +*/ +SQLITE_API int sqlite3_busy_handler( + sqlite3 *db, + int (*xBusy)(void*,int), + void *pArg +){ + sqlite3_mutex_enter(db->mutex); + db->busyHandler.xFunc = xBusy; + db->busyHandler.pArg = pArg; + db->busyHandler.nBusy = 0; + db->busyTimeout = 0; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This routine sets the progress callback for an Sqlite database to the +** given callback function with the given argument. The progress callback will +** be invoked every nOps opcodes. +*/ +SQLITE_API void sqlite3_progress_handler( + sqlite3 *db, + int nOps, + int (*xProgress)(void*), + void *pArg +){ + sqlite3_mutex_enter(db->mutex); + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = (unsigned)nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; + } + sqlite3_mutex_leave(db->mutex); +} +#endif + + +/* +** This routine installs a default busy handler that waits for the +** specified number of milliseconds before returning 0. +*/ +SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ + if( ms>0 ){ + sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + db->busyTimeout = ms; + }else{ + sqlite3_busy_handler(db, 0, 0); + } + return SQLITE_OK; +} + +/* +** Cause any pending operation to stop at its earliest opportunity. +*/ +SQLITE_API void sqlite3_interrupt(sqlite3 *db){ + db->u1.isInterrupted = 1; +} + + +/* +** This function is exactly the same as sqlite3_create_function(), except +** that it is designed to be called by internal code. The difference is +** that if a malloc() fails in sqlite3_create_function(), an error code +** is returned and the mallocFailed flag cleared. +*/ +SQLITE_PRIVATE int sqlite3CreateFunc( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int enc, + void *pUserData, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + FuncDestructor *pDestructor +){ + FuncDef *p; + int nName; + int extraFlags; + + assert( sqlite3_mutex_held(db->mutex) ); + if( zFunctionName==0 || + (xFunc && (xFinal || xStep)) || + (!xFunc && (xFinal && !xStep)) || + (!xFunc && (!xFinal && xStep)) || + (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || + (255<(nName = sqlite3Strlen30( zFunctionName))) ){ + return SQLITE_MISUSE_BKPT; + } + + assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); + extraFlags = enc & SQLITE_DETERMINISTIC; + enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); + +#ifndef SQLITE_OMIT_UTF16 + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + ** + ** If SQLITE_ANY is specified, add three versions of the function + ** to the hash table. + */ + if( enc==SQLITE_UTF16 ){ + enc = SQLITE_UTF16NATIVE; + }else if( enc==SQLITE_ANY ){ + int rc; + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, + pUserData, xFunc, xStep, xFinal, pDestructor); + if( rc==SQLITE_OK ){ + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, + pUserData, xFunc, xStep, xFinal, pDestructor); + } + if( rc!=SQLITE_OK ){ + return rc; + } + enc = SQLITE_UTF16BE; + } +#else + enc = SQLITE_UTF8; +#endif + + /* Check if an existing function is being overridden or deleted. If so, + ** and there are active VMs, then return SQLITE_BUSY. If a function + ** is being overridden/deleted but there are no active VMs, allow the + ** operation to continue but invalidate all precompiled statements. + */ + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ + if( db->nVdbeActive ){ + sqlite3Error(db, SQLITE_BUSY, + "unable to delete/modify user-function due to active statements"); + assert( !db->mallocFailed ); + return SQLITE_BUSY; + }else{ + sqlite3ExpirePreparedStatements(db); + } + } + + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); + assert(p || db->mallocFailed); + if( !p ){ + return SQLITE_NOMEM; + } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->pDestructor = pDestructor; + p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; + testcase( p->funcFlags & SQLITE_DETERMINISTIC ); + p->xFunc = xFunc; + p->xStep = xStep; + p->xFinalize = xFinal; + p->pUserData = pUserData; + p->nArg = (u16)nArg; + return SQLITE_OK; +} + +/* +** Create new user functions. +*/ +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + xFinal, 0); +} + +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; + sqlite3_mutex_enter(db->mutex); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); + if( !pArg ){ + xDestroy(p); + goto out; + } + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3DbFree(db, pArg); + } + + out: + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +){ + int rc; + char *zFunc8; + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); + sqlite3DbFree(db, zFunc8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif + + +/* +** Declare that a function has been overloaded by a virtual table. +** +** If the function already exists as a regular global function, then +** this routine is a no-op. If the function does not exist, then create +** a new one that always throws a run-time error. +** +** When virtual tables intend to provide an overloaded function, they +** should call this routine to make sure the global function exists. +** A global function must exist in order for name resolution to work +** properly. +*/ +SQLITE_API int sqlite3_overload_function( + sqlite3 *db, + const char *zName, + int nArg +){ + int nName = sqlite3Strlen30(zName); + int rc = SQLITE_OK; + sqlite3_mutex_enter(db->mutex); + if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ + rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, + 0, sqlite3InvalidFunction, 0, 0, 0); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_TRACE +/* +** Register a trace function. The pArg from the previously registered trace +** is returned. +** +** A NULL trace function means that no tracing is executes. A non-NULL +** trace is a pointer to a function that is invoked at the start of each +** SQL statement. +*/ +SQLITE_API void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ + void *pOld; + sqlite3_mutex_enter(db->mutex); + pOld = db->pTraceArg; + db->xTrace = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +/* +** Register a profile function. The pArg from the previously registered +** profile function is returned. +** +** A NULL profile function means that no profiling is executes. A non-NULL +** profile is a pointer to a function that is invoked at the conclusion of +** each SQL statement that is run. +*/ +SQLITE_API void *sqlite3_profile( + sqlite3 *db, + void (*xProfile)(void*,const char*,sqlite_uint64), + void *pArg +){ + void *pOld; + sqlite3_mutex_enter(db->mutex); + pOld = db->pProfileArg; + db->xProfile = xProfile; + db->pProfileArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_TRACE */ + +/* +** Register a function to be invoked when a transaction commits. +** If the invoked function returns non-zero, then the commit becomes a +** rollback. +*/ +SQLITE_API void *sqlite3_commit_hook( + sqlite3 *db, /* Attach the hook to this database */ + int (*xCallback)(void*), /* Function to invoke on each commit */ + void *pArg /* Argument to the function */ +){ + void *pOld; + sqlite3_mutex_enter(db->mutex); + pOld = db->pCommitArg; + db->xCommitCallback = xCallback; + db->pCommitArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} + +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +SQLITE_API void *sqlite3_update_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), + void *pArg /* Argument to the function */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pUpdateArg; + db->xUpdateCallback = xCallback; + db->pUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +/* +** Register a callback to be invoked each time a transaction is rolled +** back by this database connection. +*/ +SQLITE_API void *sqlite3_rollback_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*), /* Callback function */ + void *pArg /* Argument to the function */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pRollbackArg; + db->xRollbackCallback = xCallback; + db->pRollbackArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +#ifndef SQLITE_OMIT_WAL +/* +** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). +** Invoke sqlite3_wal_checkpoint if the number of frames in the log file +** is greater than sqlite3.pWalArg cast to an integer (the value configured by +** wal_autocheckpoint()). +*/ +SQLITE_PRIVATE int sqlite3WalDefaultHook( + void *pClientData, /* Argument */ + sqlite3 *db, /* Connection */ + const char *zDb, /* Database */ + int nFrame /* Size of WAL */ +){ + if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ + sqlite3BeginBenignMalloc(); + sqlite3_wal_checkpoint(db, zDb); + sqlite3EndBenignMalloc(); + } + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** Configure an sqlite3_wal_hook() callback to automatically checkpoint +** a database after committing a transaction if there are nFrame or +** more frames in the log file. Passing zero or a negative value as the +** nFrame parameter disables automatic checkpoints entirely. +** +** The callback registered by this function replaces any existing callback +** registered using sqlite3_wal_hook(). Likewise, registering a callback +** using sqlite3_wal_hook() disables the automatic checkpoint mechanism +** configured by this function. +*/ +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else + if( nFrame>0 ){ + sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); + }else{ + sqlite3_wal_hook(db, 0, 0); + } +#endif + return SQLITE_OK; +} + +/* +** Register a callback to be invoked each time a transaction is written +** into the write-ahead-log by this database connection. +*/ +SQLITE_API void *sqlite3_wal_hook( + sqlite3 *db, /* Attach the hook to this db handle */ + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg /* First argument passed to xCallback() */ +){ +#ifndef SQLITE_OMIT_WAL + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pWalArg; + db->xWalCallback = xCallback; + db->pWalArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +#else + return 0; +#endif +} + +/* +** Checkpoint database zDb. +*/ +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +){ +#ifdef SQLITE_OMIT_WAL + return SQLITE_OK; +#else + int rc; /* Return code */ + int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ + + /* Initialize the output variables to -1 in case an error occurs. */ + if( pnLog ) *pnLog = -1; + if( pnCkpt ) *pnCkpt = -1; + + assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE ); + assert( SQLITE_CHECKPOINT_FULLSQLITE_CHECKPOINT_RESTART ){ + return SQLITE_MISUSE; + } + + sqlite3_mutex_enter(db->mutex); + if( zDb && zDb[0] ){ + iDb = sqlite3FindDbName(db, zDb); + } + if( iDb<0 ){ + rc = SQLITE_ERROR; + sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); + }else{ + rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); + sqlite3Error(db, rc, 0); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +#endif +} + + +/* +** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points +** to contains a zero-length string, all attached databases are +** checkpointed. +*/ +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0); +} + +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on database iDb. This is a no-op if database iDb is +** not currently open in WAL mode. +** +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is +** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** +** The mutex on database handle db should be held by the caller. The mutex +** associated with the specific b-tree being checkpointed is taken by +** this function while the checkpoint is running. +** +** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are +** checkpointed. If an error is encountered it is returned immediately - +** no attempt is made to checkpoint any remaining databases. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ + + assert( sqlite3_mutex_held(db->mutex) ); + assert( !pnLog || *pnLog==-1 ); + assert( !pnCkpt || *pnCkpt==-1 ); + + for(i=0; inDb && rc==SQLITE_OK; i++){ + if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; + } + } + } + + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** This function returns true if main-memory should be used instead of +** a temporary file for transient pager files and statement journals. +** The value returned depends on the value of db->temp_store (runtime +** parameter) and the compile time value of SQLITE_TEMP_STORE. The +** following table describes the relationship between these two values +** and this functions return value. +** +** SQLITE_TEMP_STORE db->temp_store Location of temporary database +** ----------------- -------------- ------------------------------ +** 0 any file (return 0) +** 1 1 file (return 0) +** 1 2 memory (return 1) +** 1 0 file (return 0) +** 2 1 file (return 0) +** 2 2 memory (return 1) +** 2 0 memory (return 1) +** 3 any memory (return 1) +*/ +SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ +#if SQLITE_TEMP_STORE==1 + return ( db->temp_store==2 ); +#endif +#if SQLITE_TEMP_STORE==2 + return ( db->temp_store!=1 ); +#endif +#if SQLITE_TEMP_STORE==3 + return 1; +#endif +#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 + return 0; +#endif +} + +/* +** Return UTF-8 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ + const char *z; + if( !db ){ + return sqlite3ErrStr(SQLITE_NOMEM); + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return sqlite3ErrStr(SQLITE_MISUSE_BKPT); + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = sqlite3ErrStr(SQLITE_NOMEM); + }else{ + testcase( db->pErr==0 ); + z = (char*)sqlite3_value_text(db->pErr); + assert( !db->mallocFailed ); + if( z==0 ){ + z = sqlite3ErrStr(db->errCode); + } + } + sqlite3_mutex_leave(db->mutex); + return z; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Return UTF-16 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ + static const u16 outOfMem[] = { + 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 + }; + static const u16 misuse[] = { + 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', + 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', + 'c', 'a', 'l', 'l', 'e', 'd', ' ', + 'o', 'u', 't', ' ', + 'o', 'f', ' ', + 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 + }; + + const void *z; + if( !db ){ + return (void *)outOfMem; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return (void *)misuse; + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = (void *)outOfMem; + }else{ + z = sqlite3_value_text16(db->pErr); + if( z==0 ){ + sqlite3Error(db, db->errCode, sqlite3ErrStr(db->errCode)); + z = sqlite3_value_text16(db->pErr); + } + /* A malloc() may have failed within the call to sqlite3_value_text16() + ** above. If this is the case, then the db->mallocFailed flag needs to + ** be cleared before returning. Do this directly, instead of via + ** sqlite3ApiExit(), to avoid setting the database handle error message. + */ + db->mallocFailed = 0; + } + sqlite3_mutex_leave(db->mutex); + return z; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the most recent error code generated by an SQLite routine. If NULL is +** passed to this function, we assume a malloc() failed during sqlite3_open(). +*/ +SQLITE_API int sqlite3_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } + return db->errCode & db->errMask; +} +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } + return db->errCode; +} + +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +SQLITE_API const char *sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + +/* +** Invalidate all cached KeyInfo objects for database connection "db" +*/ +static void invalidateCachedKeyInfo(sqlite3 *db){ + Db *pDb; /* A single database */ + int iDb; /* The database index number */ + HashElem *k; /* For looping over tables in pDb */ + Table *pTab; /* A table in the database */ + Index *pIdx; /* Each index */ + + for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ + if( pDb->pBt==0 ) continue; + sqlite3BtreeEnter(pDb->pBt); + for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->pKeyInfo && pIdx->pKeyInfo->db==db ){ + sqlite3KeyInfoUnref(pIdx->pKeyInfo); + pIdx->pKeyInfo = 0; + } + } + } + sqlite3BtreeLeave(pDb->pBt); + } +} + +/* +** Create a new collating function for database "db". The name is zName +** and the encoding is enc. +*/ +static int createCollation( + sqlite3* db, + const char *zName, + u8 enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + CollSeq *pColl; + int enc2; + int nName = sqlite3Strlen30(zName); + + assert( sqlite3_mutex_held(db->mutex) ); + + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + */ + enc2 = enc; + testcase( enc2==SQLITE_UTF16 ); + testcase( enc2==SQLITE_UTF16_ALIGNED ); + if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ + enc2 = SQLITE_UTF16NATIVE; + } + if( enc2SQLITE_UTF16BE ){ + return SQLITE_MISUSE_BKPT; + } + + /* Check if this call is removing or replacing an existing collation + ** sequence. If so, and there are active VMs, return busy. If there + ** are no active VMs, invalidate any pre-compiled statements. + */ + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); + if( pColl && pColl->xCmp ){ + if( db->nVdbeActive ){ + sqlite3Error(db, SQLITE_BUSY, + "unable to delete/modify collation sequence due to active statements"); + return SQLITE_BUSY; + } + sqlite3ExpirePreparedStatements(db); + invalidateCachedKeyInfo(db); + + /* If collation sequence pColl was created directly by a call to + ** sqlite3_create_collation, and not generated by synthCollSeq(), + ** then any copies made by synthCollSeq() need to be invalidated. + ** Also, collation destructor - CollSeq.xDel() - function may need + ** to be called. + */ + if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + int j; + for(j=0; j<3; j++){ + CollSeq *p = &aColl[j]; + if( p->enc==pColl->enc ){ + if( p->xDel ){ + p->xDel(p->pUser); + } + p->xCmp = 0; + } + } + } + } + + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); + if( pColl==0 ) return SQLITE_NOMEM; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + sqlite3Error(db, SQLITE_OK, 0); + return SQLITE_OK; +} + + +/* +** This array defines hard upper bounds on limit values. The +** initializer must be kept in sync with the SQLITE_LIMIT_* +** #defines in sqlite3.h. +*/ +static const int aHardLimit[] = { + SQLITE_MAX_LENGTH, + SQLITE_MAX_SQL_LENGTH, + SQLITE_MAX_COLUMN, + SQLITE_MAX_EXPR_DEPTH, + SQLITE_MAX_COMPOUND_SELECT, + SQLITE_MAX_VDBE_OP, + SQLITE_MAX_FUNCTION_ARG, + SQLITE_MAX_ATTACHED, + SQLITE_MAX_LIKE_PATTERN_LENGTH, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ + SQLITE_MAX_TRIGGER_DEPTH, +}; + +/* +** Make sure the hard limits are set to reasonable values +*/ +#if SQLITE_MAX_LENGTH<100 +# error SQLITE_MAX_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH<100 +# error SQLITE_MAX_SQL_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH +# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH +#endif +#if SQLITE_MAX_COMPOUND_SELECT<2 +# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 +#endif +#if SQLITE_MAX_VDBE_OP<40 +# error SQLITE_MAX_VDBE_OP must be at least 40 +#endif +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 +#endif +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 +#endif +#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 +# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 +#endif +#if SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN must not exceed 32767 +#endif +#if SQLITE_MAX_TRIGGER_DEPTH<1 +# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 +#endif + + +/* +** Change the value of a limit. Report the old value. +** If an invalid limit index is supplied, report -1. +** Make no changes but still report the old value if the +** new limit is negative. +** +** A new lower limit does not shrink existing constructs. +** It merely prevents new constructs that exceed the limit +** from forming. +*/ +SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ + int oldLimit; + + + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); + + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ + return -1; + } + oldLimit = db->aLimit[limitId]; + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ + if( newLimit>aHardLimit[limitId] ){ + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ + } + db->aLimit[limitId] = newLimit; + } + return oldLimit; /* IMP: R-53341-35419 */ +} + +/* +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. +** +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. +** +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. It is the +** responsibility of the caller to eventually call sqlite3_free() to release +** this buffer. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). +*/ +SQLITE_PRIVATE int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ +){ + int rc = SQLITE_OK; + unsigned int flags = *pFlags; + const char *zVfs = zDefaultVfs; + char *zFile; + char c; + int nUri = sqlite3Strlen30(zUri); + + assert( *pzErrMsg==0 ); + + if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) + && nUri>=5 && memcmp(zUri, "file:", 5)==0 + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + int nByte = nUri+2; /* Bytes of space to allocate */ + + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; + + for(iIn=0; iIn=0 && octet<256 ); + if( octet==0 ){ + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; + } + zFile[iOut++] = c; + } + if( eState==1 ) zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); + + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; + + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { "memory", SQLITE_OPEN_MEMORY }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } + + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } + + zOpt = &zVal[nVal+1]; + } + + }else{ + zFile = sqlite3_malloc(nUri+2); + if( !zFile ) return SQLITE_NOMEM; + memcpy(zFile, zUri, nUri); + zFile[nUri] = '\0'; + zFile[nUri+1] = '\0'; + flags &= ~SQLITE_OPEN_URI; + } + + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; + } + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free(zFile); + zFile = 0; + } + *pFlags = flags; + *pzFile = zFile; + return rc; +} + + +/* +** This routine does the work of opening a database on behalf of +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** is UTF-8 encoded. +*/ +static int openDatabase( + const char *zFilename, /* Database filename UTF-8 encoded */ + sqlite3 **ppDb, /* OUT: Returned database handle */ + unsigned int flags, /* Operational flags */ + const char *zVfs /* Name of the VFS to use */ +){ + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + + /* Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE_BKPT; + + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_NOMUTEX ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_FULLMUTEX ){ + isThreadsafe = 1; + }else{ + isThreadsafe = sqlite3GlobalConfig.bFullMutex; + } + if( flags & SQLITE_OPEN_PRIVATECACHE ){ + flags &= ~SQLITE_OPEN_SHAREDCACHE; + }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ + flags |= SQLITE_OPEN_SHAREDCACHE; + } + + /* Remove harmful bits from the flags parameter + ** + ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were + ** dealt with in the previous code block. Besides these, the only + ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, + ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, + ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask + ** off all other flags. + */ + flags &= ~( SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_MAIN_DB | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_MASTER_JOURNAL | + SQLITE_OPEN_NOMUTEX | + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL + ); + + /* Allocate the sqlite data structure */ + db = sqlite3MallocZero( sizeof(sqlite3) ); + if( db==0 ) goto opendb_out; + if( isThreadsafe ){ + db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( db->mutex==0 ){ + sqlite3_free(db); + db = 0; + goto opendb_out; + } + } + sqlite3_mutex_enter(db->mutex); + db->errMask = 0xff; + db->nDb = 2; + db->magic = SQLITE_MAGIC_BUSY; + db->aDb = db->aDbStatic; + + assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); + memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->autoCommit = 1; + db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; + db->nextPagesize = 0; + db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill +#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX + | SQLITE_AutoIndex +#endif +#if SQLITE_DEFAULT_FILE_FORMAT<4 + | SQLITE_LegacyFileFmt +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + | SQLITE_LoadExtension +#endif +#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS + | SQLITE_RecTriggers +#endif +#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS + | SQLITE_ForeignKeys +#endif + ; + sqlite3HashInit(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3HashInit(&db->aModule); +#endif + + /* Add the default collation sequence BINARY. BINARY works for both UTF-8 + ** and UTF-16, so add a version for each to avoid any unnecessary + ** conversions. The only error that can occur here is a malloc() failure. + */ + createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); + if( db->mallocFailed ){ + goto opendb_out; + } + db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); + assert( db->pDfltColl!=0 ); + + /* Also add a UTF-8 case-insensitive collation sequence. */ + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); + + /* Parse the filename/URI argument. */ + db->openFlags = flags; + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; + } + + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM; + } + sqlite3Error(db, rc, 0); + goto opendb_out; + } + db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); + + + /* The default safety_level for the main database is 'full'; for the temp + ** database it is 'NONE'. This matches the pager layer defaults. + */ + db->aDb[0].zName = "main"; + db->aDb[0].safety_level = 3; + db->aDb[1].zName = "temp"; + db->aDb[1].safety_level = 1; + + db->magic = SQLITE_MAGIC_OPEN; + if( db->mallocFailed ){ + goto opendb_out; + } + + /* Register all built-in functions, but do not attempt to read the + ** database schema yet. This is delayed until the first time the database + ** is accessed. + */ + sqlite3Error(db, SQLITE_OK, 0); + sqlite3RegisterBuiltinFunctions(db); + + /* Load automatic extensions - extensions that have been registered + ** using the sqlite3_automatic_extension() API. + */ + rc = sqlite3_errcode(db); + if( rc==SQLITE_OK ){ + sqlite3AutoLoadExtensions(db); + rc = sqlite3_errcode(db); + if( rc!=SQLITE_OK ){ + goto opendb_out; + } + } + +#ifdef SQLITE_ENABLE_FTS1 + if( !db->mallocFailed ){ + extern int sqlite3Fts1Init(sqlite3*); + rc = sqlite3Fts1Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_FTS2 + if( !db->mallocFailed && rc==SQLITE_OK ){ + extern int sqlite3Fts2Init(sqlite3*); + rc = sqlite3Fts2Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_FTS3 + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts3Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_ICU + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3IcuInit(db); + } +#endif + +#ifdef SQLITE_ENABLE_RTREE + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3RtreeInit(db); + } +#endif + + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ +#ifdef SQLITE_DEFAULT_LOCKING_MODE + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); +#endif + + if( rc ) sqlite3Error(db, rc, 0); + + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + +opendb_out: + sqlite3_free(zOpen); + if( db ){ + assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); + sqlite3_mutex_leave(db->mutex); + } + rc = sqlite3_errcode(db); + assert( db!=0 || rc==SQLITE_NOMEM ); + if( rc==SQLITE_NOMEM ){ + sqlite3_close(db); + db = 0; + }else if( rc!=SQLITE_OK ){ + db->magic = SQLITE_MAGIC_SICK; + } + *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif + return sqlite3ApiExit(0, rc); +} + +/* +** Open a new database handle. +*/ +SQLITE_API int sqlite3_open( + const char *zFilename, + sqlite3 **ppDb +){ + return openDatabase(zFilename, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); +} +SQLITE_API int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +){ + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Open a new database handle. +*/ +SQLITE_API int sqlite3_open16( + const void *zFilename, + sqlite3 **ppDb +){ + char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ + sqlite3_value *pVal; + int rc; + + assert( zFilename ); + assert( ppDb ); + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zFilename8 ){ + rc = openDatabase(zFilename8, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); + assert( *ppDb || rc==SQLITE_NOMEM ); + if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ + ENC(*ppDb) = SQLITE_UTF16NATIVE; + } + }else{ + rc = SQLITE_NOMEM; + } + sqlite3ValueFree(pVal); + + return sqlite3ApiExit(0, rc); +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int sqlite3_create_collation( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + int rc; + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int sqlite3_create_collation_v2( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + int rc; + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int sqlite3_create_collation16( + sqlite3* db, + const void *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + int rc = SQLITE_OK; + char *zName8; + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); + if( zName8 ){ + rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); + sqlite3DbFree(db, zName8); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +SQLITE_API int sqlite3_collation_needed( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) +){ + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = xCollNeeded; + db->xCollNeeded16 = 0; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +SQLITE_API int sqlite3_collation_needed16( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) +){ + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = 0; + db->xCollNeeded16 = xCollNeeded16; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_UTF16 */ + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This function is now an anachronism. It used to be used to recover from a +** malloc() failure, but SQLite now does this automatically. +*/ +SQLITE_API int sqlite3_global_recover(void){ + return SQLITE_OK; +} +#endif + +/* +** Test to see whether or not the database connection is in autocommit +** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on +** by default. Autocommit is disabled by a BEGIN statement and reenabled +** by the next COMMIT or ROLLBACK. +*/ +SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ + return db->autoCommit; +} + +/* +** The following routines are subtitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error +** constants. They server two purposes: +** +** 1. Serve as a convenient place to set a breakpoint in a debugger +** to detect when version error conditions occurs. +** +** 2. Invoke sqlite3_log() to provide the source code location where +** a low-level error is first detected. +*/ +SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CORRUPT, + "database corruption at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CORRUPT; +} +SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_MISUSE, + "misuse at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_MISUSE; +} +SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CANTOPEN, + "cannot open file at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CANTOPEN; +} + + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This is a convenience routine that makes sure that all thread-specific +** data for this thread has been deallocated. +** +** SQLite no longer uses thread-specific data so this routine is now a +** no-op. It is retained for historical compatibility. +*/ +SQLITE_API void sqlite3_thread_cleanup(void){ +} +#endif + +/* +** Return meta information about a specific column of a database table. +** See comment in sqlite3.h (sqlite.h.in) for details. +*/ +#ifdef SQLITE_ENABLE_COLUMN_METADATA +SQLITE_API int sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +){ + int rc; + char *zErrMsg = 0; + Table *pTab = 0; + Column *pCol = 0; + int iCol; + + char const *zDataType = 0; + char const *zCollSeq = 0; + int notnull = 0; + int primarykey = 0; + int autoinc = 0; + + /* Ensure the database schema has been loaded */ + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrMsg); + if( SQLITE_OK!=rc ){ + goto error_out; + } + + /* Locate the table in question */ + pTab = sqlite3FindTable(db, zTableName, zDbName); + if( !pTab || pTab->pSelect ){ + pTab = 0; + goto error_out; + } + + /* Find the column for which info is requested */ + if( sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + if( iCol>=0 ){ + pCol = &pTab->aCol[iCol]; + } + }else{ + for(iCol=0; iColnCol; iCol++){ + pCol = &pTab->aCol[iCol]; + if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ + break; + } + } + if( iCol==pTab->nCol ){ + pTab = 0; + goto error_out; + } + } + + /* The following block stores the meta information that will be returned + ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey + ** and autoinc. At this point there are two possibilities: + ** + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an + ** explicitly declared column. Copy meta information from *pCol. + */ + if( pCol ){ + zDataType = pCol->zType; + zCollSeq = pCol->zColl; + notnull = pCol->notNull!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; + autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; + }else{ + zDataType = "INTEGER"; + primarykey = 1; + } + if( !zCollSeq ){ + zCollSeq = "BINARY"; + } + +error_out: + sqlite3BtreeLeaveAll(db); + + /* Whether the function call succeeded or failed, set the output parameters + ** to whatever their local counterparts contain. If an error did occur, + ** this has the effect of zeroing all output parameters. + */ + if( pzDataType ) *pzDataType = zDataType; + if( pzCollSeq ) *pzCollSeq = zCollSeq; + if( pNotNull ) *pNotNull = notnull; + if( pPrimaryKey ) *pPrimaryKey = primarykey; + if( pAutoinc ) *pAutoinc = autoinc; + + if( SQLITE_OK==rc && !pTab ){ + sqlite3DbFree(db, zErrMsg); + zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, + zColumnName); + rc = SQLITE_ERROR; + } + sqlite3Error(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3DbFree(db, zErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif + +/* +** Sleep for a little while. Return the amount of time slept. +*/ +SQLITE_API int sqlite3_sleep(int ms){ + sqlite3_vfs *pVfs; + int rc; + pVfs = sqlite3_vfs_find(0); + if( pVfs==0 ) return 0; + + /* This function works in milliseconds, but the underlying OsSleep() + ** API uses microseconds. Hence the 1000's. + */ + rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + return rc; +} + +/* +** Enable or disable the extended result codes. +*/ +SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ + sqlite3_mutex_enter(db->mutex); + db->errMask = onoff ? 0xffffffff : 0xff; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Invoke the xFileControl method on a particular database. +*/ +SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ + int rc = SQLITE_ERROR; + Btree *pBtree; + + sqlite3_mutex_enter(db->mutex); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( fd->pMethods ){ + rc = sqlite3OsFileControl(fd, op, pArg); + }else{ + rc = SQLITE_NOTFOUND; + } + sqlite3BtreeLeave(pBtree); + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Interface to the testing logic. +*/ +SQLITE_API int sqlite3_test_control(int op, ...){ + int rc = 0; +#ifndef SQLITE_OMIT_BUILTIN_TEST + va_list ap; + va_start(ap, op); + switch( op ){ + + /* + ** Save the current state of the PRNG. + */ + case SQLITE_TESTCTRL_PRNG_SAVE: { + sqlite3PrngSaveState(); + break; + } + + /* + ** Restore the state of the PRNG to the last state saved using + ** PRNG_SAVE. If PRNG_SAVE has never before been called, then + ** this verb acts like PRNG_RESET. + */ + case SQLITE_TESTCTRL_PRNG_RESTORE: { + sqlite3PrngRestoreState(); + break; + } + + /* + ** Reset the PRNG back to its uninitialized state. The next call + ** to sqlite3_randomness() will reseed the PRNG using a single call + ** to the xRandomness method of the default VFS. + */ + case SQLITE_TESTCTRL_PRNG_RESET: { + sqlite3_randomness(0,0); + break; + } + + /* + ** sqlite3_test_control(BITVEC_TEST, size, program) + ** + ** Run a test against a Bitvec object of size. The program argument + ** is an array of integers that defines the test. Return -1 on a + ** memory allocation error, 0 on success, or non-zero for an error. + ** See the sqlite3BitvecBuiltinTest() for additional information. + */ + case SQLITE_TESTCTRL_BITVEC_TEST: { + int sz = va_arg(ap, int); + int *aProg = va_arg(ap, int*); + rc = sqlite3BitvecBuiltinTest(sz, aProg); + break; + } + + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + rc = sqlite3FaultSim(0); + break; + } + + /* + ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) + ** + ** Register hooks to call to indicate which malloc() failures + ** are benign. + */ + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { + typedef void (*void_function)(void); + void_function xBenignBegin; + void_function xBenignEnd; + xBenignBegin = va_arg(ap, void_function); + xBenignEnd = va_arg(ap, void_function); + sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) + ** + ** Set the PENDING byte to the value in the argument, if X>0. + ** Make no changes if X==0. Return the value of the pending byte + ** as it existing before this routine was called. + ** + ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in + ** an incompatible database file format. Changing the PENDING byte + ** while any database connection is open results in undefined and + ** dileterious behavior. + */ + case SQLITE_TESTCTRL_PENDING_BYTE: { + rc = PENDING_BYTE; +#ifndef SQLITE_OMIT_WSD + { + unsigned int newVal = va_arg(ap, unsigned int); + if( newVal ) sqlite3PendingByte = newVal; + } +#endif + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) + ** + ** This action provides a run-time test to see whether or not + ** assert() was enabled at compile-time. If X is true and assert() + ** is enabled, then the return value is true. If X is true and + ** assert() is disabled, then the return value is zero. If X is + ** false and assert() is enabled, then the assertion fires and the + ** process aborts. If X is false and assert() is disabled, then the + ** return value is zero. + */ + case SQLITE_TESTCTRL_ASSERT: { + volatile int x = 0; + assert( (x = va_arg(ap,int))!=0 ); + rc = x; + break; + } + + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) + ** + ** This action provides a run-time test to see how the ALWAYS and + ** NEVER macros were defined at compile-time. + ** + ** The return value is ALWAYS(X). + ** + ** The recommended test is X==2. If the return value is 2, that means + ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the + ** default setting. If the return value is 1, then ALWAYS() is either + ** hard-coded to true or else it asserts if its argument is false. + ** The first behavior (hard-coded to true) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second + ** behavior (assert if the argument to ALWAYS() is false) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. + ** + ** The run-time test procedure might look something like this: + ** + ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ + ** // ALWAYS() and NEVER() are no-op pass-through macros + ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ + ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. + ** }else{ + ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. + ** } + */ + case SQLITE_TESTCTRL_ALWAYS: { + int x = va_arg(ap,int); + rc = ALWAYS(x); + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) + ** + ** Set the nReserve size to N for the main database on the database + ** connection db. + */ + case SQLITE_TESTCTRL_RESERVE: { + sqlite3 *db = va_arg(ap, sqlite3*); + int x = va_arg(ap,int); + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); + sqlite3_mutex_leave(db->mutex); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) + ** + ** Enable or disable various optimizations for testing purposes. The + ** argument N is a bitmask of optimizations to be disabled. For normal + ** operation N should be 0. The idea is that a test program (like the + ** SQL Logic Test or SLT test module) can run the same SQL multiple times + ** with various optimizations disabled to verify that the same answer + ** is obtained in every case. + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); + break; + } + +#ifdef SQLITE_N_KEYWORD + /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) + ** + ** If zWord is a keyword recognized by the parser, then return the + ** number of keywords. Or if zWord is not a keyword, return 0. + ** + ** This test feature is only available in the amalgamation since + ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite + ** is built using separate source files. + */ + case SQLITE_TESTCTRL_ISKEYWORD: { + const char *zWord = va_arg(ap, const char*); + int n = sqlite3Strlen30(zWord); + rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; + break; + } +#endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); + ** + ** Pass pFree into sqlite3ScratchFree(). + ** If sz>0 then allocate a scratch buffer into pNew. + */ + case SQLITE_TESTCTRL_SCRATCHMALLOC: { + void *pFree, **ppNew; + int sz; + sz = va_arg(ap, int); + ppNew = va_arg(ap, void**); + pFree = va_arg(ap, void*); + if( sz ) *ppNew = sqlite3ScratchMalloc(sz); + sqlite3ScratchFree(pFree); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, configure the wrappers so that all + ** subsequent calls to localtime() and variants fail. If onoff is zero, + ** undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } + +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) + /* sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT, + ** sqlite3_stmt*,const char**); + ** + ** If compiled with SQLITE_ENABLE_TREE_EXPLAIN, each sqlite3_stmt holds + ** a string that describes the optimized parse tree. This test-control + ** returns a pointer to that string. + */ + case SQLITE_TESTCTRL_EXPLAIN_STMT: { + sqlite3_stmt *pStmt = va_arg(ap, sqlite3_stmt*); + const char **pzRet = va_arg(ap, const char**); + *pzRet = sqlite3VdbeExplanation((Vdbe*)pStmt); + break; + } +#endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); + ** + ** Set or clear a flag that indicates that the database file is always well- + ** formed and never corrupt. This flag is clear by default, indicating that + ** database files might have arbitrary corruption. Setting the flag during + ** testing causes certain assert() statements in the code to be activated + ** that demonstrat invariants on well-formed database files. + */ + case SQLITE_TESTCTRL_NEVER_CORRUPT: { + sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); + break; + } + + + /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); + ** + ** Set the VDBE coverage callback function to xCallback with context + ** pointer ptr. + */ + case SQLITE_TESTCTRL_VDBE_COVERAGE: { +#ifdef SQLITE_VDBE_COVERAGE + typedef void (*branch_callback)(void*,int,u8,u8); + sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); + sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); +#endif + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; + break; + } + + } + va_end(ap); +#endif /* SQLITE_OMIT_BUILTIN_TEST */ + return rc; +} + +/* +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. +*/ +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + if( zFilename==0 ) return 0; + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return 0; +} + +/* +** Return a boolean value for a query parameter. +*/ +SQLITE_API int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + bDflt = bDflt!=0; + return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +} + +/* +** Return a 64-bit integer value for a query parameter. +*/ +SQLITE_API sqlite3_int64 sqlite3_uri_int64( + const char *zFilename, /* Filename as passed to xOpen */ + const char *zParam, /* URI parameter sought */ + sqlite3_int64 bDflt /* return if parameter is missing */ +){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + sqlite3_int64 v; + if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ + bDflt = v; + } + return bDflt; +} + +/* +** Return the Btree pointer identified by zDbName. Return NULL if not found. +*/ +SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ + int i; + for(i=0; inDb; i++){ + if( db->aDb[i].pBt + && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) + ){ + return db->aDb[i].pBt; + } + } + return 0; +} + +/* +** Return the filename of the database associated with a database +** connection. +*/ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeGetFilename(pBt) : 0; +} + +/* +** Return 1 if database is read-only or 0 if read/write. Return -1 if +** no such database exists. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ + Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; +} + +/************** End of main.c ************************************************/ +/************** Begin file notify.c ******************************************/ +/* +** 2009 March 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of the sqlite3_unlock_notify() +** API method and its associated functionality. +*/ + +/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + +/* +** Public interfaces: +** +** sqlite3ConnectionBlocked() +** sqlite3ConnectionUnlocked() +** sqlite3ConnectionClosed() +** sqlite3_unlock_notify() +*/ + +#define assertMutexHeld() \ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) + +/* +** Head of a linked list of all sqlite3 objects created by this process +** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection +** is not NULL. This variable may only accessed while the STATIC_MASTER +** mutex is held. +*/ +static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; + +#ifndef NDEBUG +/* +** This function is a complex assert() that verifies the following +** properties of the blocked connections list: +** +** 1) Each entry in the list has a non-NULL value for either +** pUnlockConnection or pBlockingConnection, or both. +** +** 2) All entries in the list that share a common value for +** xUnlockNotify are grouped together. +** +** 3) If the argument db is not NULL, then none of the entries in the +** blocked connections list have pUnlockConnection or pBlockingConnection +** set to db. This is used when closing connection db. +*/ +static void checkListProperties(sqlite3 *db){ + sqlite3 *p; + for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ + int seen = 0; + sqlite3 *p2; + + /* Verify property (1) */ + assert( p->pUnlockConnection || p->pBlockingConnection ); + + /* Verify property (2) */ + for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ + if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; + assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); + assert( db==0 || p->pUnlockConnection!=db ); + assert( db==0 || p->pBlockingConnection!=db ); + } + } +} +#else +# define checkListProperties(x) +#endif + +/* +** Remove connection db from the blocked connections list. If connection +** db is not currently a part of the list, this function is a no-op. +*/ +static void removeFromBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ + if( *pp==db ){ + *pp = (*pp)->pNextBlocked; + break; + } + } +} + +/* +** Add connection db to the blocked connections list. It is assumed +** that it is not already a part of the list. +*/ +static void addToBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for( + pp=&sqlite3BlockedList; + *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; + pp=&(*pp)->pNextBlocked + ); + db->pNextBlocked = *pp; + *pp = db; +} + +/* +** Obtain the STATIC_MASTER mutex. +*/ +static void enterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + checkListProperties(0); +} + +/* +** Release the STATIC_MASTER mutex. +*/ +static void leaveMutex(void){ + assertMutexHeld(); + checkListProperties(0); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} + +/* +** Register an unlock-notify callback. +** +** This is called after connection "db" has attempted some operation +** but has received an SQLITE_LOCKED error because another connection +** (call it pOther) in the same process was busy using the same shared +** cache. pOther is found by looking at db->pBlockingConnection. +** +** If there is no blocking connection, the callback is invoked immediately, +** before this routine returns. +** +** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate +** a deadlock. +** +** Otherwise, make arrangements to invoke xNotify when pOther drops +** its locks. +** +** Each call to this routine overrides any prior callbacks registered +** on the same "db". If xNotify==0 then any prior callbacks are immediately +** cancelled. +*/ +SQLITE_API int sqlite3_unlock_notify( + sqlite3 *db, + void (*xNotify)(void **, int), + void *pArg +){ + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + enterMutex(); + + if( xNotify==0 ){ + removeFromBlockedList(db); + db->pBlockingConnection = 0; + db->pUnlockConnection = 0; + db->xUnlockNotify = 0; + db->pUnlockArg = 0; + }else if( 0==db->pBlockingConnection ){ + /* The blocking transaction has been concluded. Or there never was a + ** blocking transaction. In either case, invoke the notify callback + ** immediately. + */ + xNotify(&pArg, 1); + }else{ + sqlite3 *p; + + for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} + if( p ){ + rc = SQLITE_LOCKED; /* Deadlock detected. */ + }else{ + db->pUnlockConnection = db->pBlockingConnection; + db->xUnlockNotify = xNotify; + db->pUnlockArg = pArg; + removeFromBlockedList(db); + addToBlockedList(db); + } + } + + leaveMutex(); + assert( !db->mallocFailed ); + sqlite3Error(db, rc, (rc?"database is deadlocked":0)); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This function is called while stepping or preparing a statement +** associated with connection db. The operation will return SQLITE_LOCKED +** to the user because it requires a lock that will not be available +** until connection pBlocker concludes its current transaction. +*/ +SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ + enterMutex(); + if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ + addToBlockedList(db); + } + db->pBlockingConnection = pBlocker; + leaveMutex(); +} + +/* +** This function is called when +** the transaction opened by database db has just finished. Locks held +** by database connection db have been released. +** +** This function loops through each entry in the blocked connections +** list and does the following: +** +** 1) If the sqlite3.pBlockingConnection member of a list entry is +** set to db, then set pBlockingConnection=0. +** +** 2) If the sqlite3.pUnlockConnection member of a list entry is +** set to db, then invoke the configured unlock-notify callback and +** set pUnlockConnection=0. +** +** 3) If the two steps above mean that pBlockingConnection==0 and +** pUnlockConnection==0, remove the entry from the blocked connections +** list. +*/ +SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ + void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ + int nArg = 0; /* Number of entries in aArg[] */ + sqlite3 **pp; /* Iterator variable */ + void **aArg; /* Arguments to the unlock callback */ + void **aDyn = 0; /* Dynamically allocated space for aArg[] */ + void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ + + aArg = aStatic; + enterMutex(); /* Enter STATIC_MASTER mutex */ + + /* This loop runs once for each entry in the blocked-connections list. */ + for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ + sqlite3 *p = *pp; + + /* Step 1. */ + if( p->pBlockingConnection==db ){ + p->pBlockingConnection = 0; + } + + /* Step 2. */ + if( p->pUnlockConnection==db ){ + assert( p->xUnlockNotify ); + if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + + sqlite3BeginBenignMalloc(); + assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); + assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); + if( (!aDyn && nArg==(int)ArraySize(aStatic)) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) + ){ + /* The aArg[] array needs to grow. */ + void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); + if( pNew ){ + memcpy(pNew, aArg, nArg*sizeof(void *)); + sqlite3_free(aDyn); + aDyn = aArg = pNew; + }else{ + /* This occurs when the array of context pointers that need to + ** be passed to the unlock-notify callback is larger than the + ** aStatic[] array allocated on the stack and the attempt to + ** allocate a larger array from the heap has failed. + ** + ** This is a difficult situation to handle. Returning an error + ** code to the caller is insufficient, as even if an error code + ** is returned the transaction on connection db will still be + ** closed and the unlock-notify callbacks on blocked connections + ** will go unissued. This might cause the application to wait + ** indefinitely for an unlock-notify callback that will never + ** arrive. + ** + ** Instead, invoke the unlock-notify callback with the context + ** array already accumulated. We can then clear the array and + ** begin accumulating any further context pointers without + ** requiring any dynamic allocation. This is sub-optimal because + ** it means that instead of one callback with a large array of + ** context pointers the application will receive two or more + ** callbacks with smaller arrays of context pointers, which will + ** reduce the applications ability to prioritize multiple + ** connections. But it is the best that can be done under the + ** circumstances. + */ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + } + sqlite3EndBenignMalloc(); + + aArg[nArg++] = p->pUnlockArg; + xUnlockNotify = p->xUnlockNotify; + p->pUnlockConnection = 0; + p->xUnlockNotify = 0; + p->pUnlockArg = 0; + } + + /* Step 3. */ + if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ + /* Remove connection p from the blocked connections list. */ + *pp = p->pNextBlocked; + p->pNextBlocked = 0; + }else{ + pp = &p->pNextBlocked; + } + } + + if( nArg!=0 ){ + xUnlockNotify(aArg, nArg); + } + sqlite3_free(aDyn); + leaveMutex(); /* Leave STATIC_MASTER mutex */ +} + +/* +** This is called when the database connection passed as an argument is +** being closed. The connection is removed from the blocked list. +*/ +SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ + sqlite3ConnectionUnlocked(db); + enterMutex(); + removeFromBlockedList(db); + checkListProperties(db); + leaveMutex(); +} +#endif + +/************** End of notify.c **********************************************/ +/************** Begin file fts3.c ********************************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ + +/* The full-text index is stored in a series of b+tree (-like) +** structures called segments which map terms to doclists. The +** structures are like b+trees in layout, but are constructed from the +** bottom up in optimal fashion and are not updatable. Since trees +** are built from the bottom up, things will be described from the +** bottom up. +** +** +**** Varints **** +** The basic unit of encoding is a variable-length integer called a +** varint. We encode variable-length integers in little-endian order +** using seven bits * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +** +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be up to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 +** +** +**** Document lists **** +** A doclist (document list) holds a docid-sorted list of hits for a +** given term. Doclists hold docids and associated token positions. +** A docid is the unique integer identifier for a single document. +** A position is the index of a word within the document. The first +** word of the document has a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. +** +** A doclist is stored like this: +** +** array { +** varint docid; (delta from previous doclist) +** array { (position list for column 0) +** varint position; (2 more than the delta from previous position) +** } +** array { +** varint POS_COLUMN; (marks start of position list for new column) +** varint column; (index of new column) +** array { +** varint position; (2 more than the delta from previous position) +** } +** } +** varint POS_END; (marks end of positions for this document. +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. A "position" is an index of a token in the token stream +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** than the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. +** +** A "position-list" is the list of positions for multiple columns for +** a single docid. A "column-list" is the set of positions for a single +** column. Hence, a position-list consists of one or more column-lists, +** a document record consists of a docid followed by a position-list and +** a doclist consists of one or more document records. +** +** A bare doclist omits the position information, becoming an +** array of varint-encoded docids. +** +**** Segment leaf nodes **** +** Segment leaf nodes store terms and doclists, ordered by term. Leaf +** nodes are written using LeafWriter, and read using LeafReader (to +** iterate through a single leaf node's data) and LeavesReader (to +** iterate through a segment's entire leaf layer). Leaf nodes have +** the format: +** +** varint iHeight; (height from leaf level, always 0) +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of prefix shared with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix];(unshared suffix of next term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. +** +** Leaf nodes are broken into blocks which are stored contiguously in +** the %_segments table in sorted order. This means that when the end +** of a node is reached, the next term is in the node with the next +** greater node id. +** +** New data is spilled to a new leaf node when the current node +** exceeds LEAF_MAX bytes (default 2048). New data which itself is +** larger than STANDALONE_MIN (default 1024) is placed in a standalone +** node (a leaf node with a single term and doclist). The goal of +** these settings is to pack together groups of small doclists while +** making it efficient to directly access large doclists. The +** assumption is that large doclists represent terms which are more +** likely to be query targets. +** +** TODO(shess) It may be useful for blocking decisions to be more +** dynamic. For instance, it may make more sense to have a 2.5k leaf +** node rather than splitting into 2k and .5k nodes. My intuition is +** that this might extend through 2x or 4x the pagesize. +** +** +**** Segment interior nodes **** +** Segment interior nodes store blockids for subtree nodes and terms +** to describe what data is stored by the each subtree. Interior +** nodes are written using InteriorWriter, and read using +** InteriorReader. InteriorWriters are created as needed when +** SegmentWriter creates new leaf nodes, or when an interior node +** itself grows too big and must be split. The format of interior +** nodes: +** +** varint iHeight; (height from leaf level, always >0) +** varint iBlockid; (block id of node's leftmost subtree) +** optional { +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of shared prefix with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** } +** } +** +** Here, optional { X } means an optional element, while array { X } +** means zero or more occurrences of X, adjacent in memory. +** +** An interior node encodes n terms separating n+1 subtrees. The +** subtree blocks are contiguous, so only the first subtree's blockid +** is encoded. The subtree at iBlockid will contain all terms less +** than the first term encoded (or all terms if no term is encoded). +** Otherwise, for terms greater than or equal to pTerm[i] but less +** than pTerm[i+1], the subtree for that term will be rooted at +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). +** +** New data is spilled to a new interior node at the same height when +** the current node exceeds INTERIOR_MAX bytes (default 2048). +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing +** interior nodes and making the tree too skinny. The interior nodes +** at a given height are naturally tracked by interior nodes at +** height+1, and so on. +** +** +**** Segment directory **** +** The segment directory in table %_segdir stores meta-information for +** merging and deleting segments, and also the root node of the +** segment's tree. +** +** The root node is the top node of the segment's tree after encoding +** the entire segment, restricted to ROOT_MAX bytes (default 1024). +** This could be either a leaf node or an interior node. If the top +** node requires more than ROOT_MAX bytes, it is flushed to %_segments +** and a new root interior node is generated (which should always fit +** within ROOT_MAX because it only needs space for 2 varints, the +** height and the blockid of the previous root). +** +** The meta-information in the segment directory is: +** level - segment level (see below) +** idx - index within level +** - (level,idx uniquely identify a segment) +** start_block - first leaf node +** leaves_end_block - last leaf node +** end_block - last block (including interior nodes) +** root - contents of root node +** +** If the root node is a leaf node, then start_block, +** leaves_end_block, and end_block are all 0. +** +** +**** Segment merging **** +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially +** more documents. +** +** New documents (actually, document updates) are tokenized and +** written individually (using LeafWriter) to a level 0 segment, with +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all +** level 0 segments are merged into a single level 1 segment. Level 1 +** is populated like level 0, and eventually MERGE_COUNT level 1 +** segments are merged to a single level 2 segment (representing +** MERGE_COUNT^2 updates), and so on. +** +** A segment merge traverses all segments at a given level in +** parallel, performing a straightforward sorted merge. Since segment +** leaf nodes are written in to the %_segments table in order, this +** merge traverses the underlying sqlite disk structures efficiently. +** After the merge, all segment blocks from the merged level are +** deleted. +** +** MERGE_COUNT controls how often we merge segments. 16 seems to be +** somewhat of a sweet spot for insertion performance. 32 and 64 show +** very similar performance numbers to 16 on insertion, though they're +** a tiny bit slower (perhaps due to more overhead in merge-time +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than +** 16, 2 about 66% slower than 16. +** +** At query time, high MERGE_COUNT increases the number of segments +** which need to be scanned and merged. For instance, with 100k docs +** inserted: +** +** MERGE_COUNT segments +** 16 25 +** 8 12 +** 4 10 +** 2 6 +** +** This appears to have only a moderate impact on queries for very +** frequent terms (which are somewhat dominated by segment merge +** costs), and infrequent and non-existent terms still seem to be fast +** even with many segments. +** +** TODO(shess) That said, it would be nice to have a better query-side +** argument for MERGE_COUNT of 16. Also, it is possible/likely that +** optimizations to things like doclist merging will swing the sweet +** spot around. +** +** +** +**** Handling of deletions and updates **** +** Since we're using a segmented structure, with no docid-oriented +** index into the term index, we clearly cannot simply update the term +** index when a document is deleted or updated. For deletions, we +** write an empty doclist (varint(docid) varint(POS_END)), for updates +** we simply write the new doclist. Segment merges overwrite older +** data for a particular docid with newer data, so deletes or updates +** will eventually overtake the earlier data and knock it out. The +** query logic likewise merges doclists so that newer data knocks out +** older data. +*/ + +/************** Include fts3Int.h in the middle of fts3.c ********************/ +/************** Begin file fts3Int.h *****************************************/ +/* +** 2009 Nov 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTSINT_H +#define _FTSINT_H + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT3 +#endif + +/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ +/************** Begin file fts3_tokenizer.h **********************************/ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ + +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + + /* + ** Structure version. Should always be set to 0 or 1. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; + +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); + + +#endif /* _FTS3_TOKENIZER_H_ */ + +/************** End of fts3_tokenizer.h **************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ +/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ +/************** Begin file fts3_hash.h ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. We've modified it slightly to serve as a standalone +** hash table implementation for the full-text indexing module. +** +*/ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ + +/* Forward declarations of structures. */ +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; + +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, many of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +*/ +struct Fts3Hash { + char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ + char copyKey; /* True if copy of key made on insert */ + int count; /* Number of entries in this table */ + Fts3HashElem *first; /* The first element of the array */ + int htsize; /* Number of buckets in the hash table */ + struct _fts3ht { /* the hash table */ + int count; /* Number of entries with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; + +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + void *pKey; int nKey; /* Key associated with this element */ +}; + +/* +** There are 2 different modes of operation for a hash table: +** +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long +** (including the null-terminator, if any). Case +** is respected in comparisons. +** +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** memcmp() is used to compare keys. +** +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +*/ +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 + +/* +** Access routines. To delete, insert a NULL pointer. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); + +/* +** Shorthand for the functions above +*/ +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem + +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Fts3Hash h; +** Fts3HashElem *p; +** ... +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); +** // do something with pData +** } +*/ +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) + +/* +** Number of entries in a hash table +*/ +#define fts3HashCount(H) ((H)->count) + +#endif /* _FTS3_HASH_H_ */ + +/************** End of fts3_hash.h *******************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ + +/* +** This constant determines the maximum depth of an FTS expression tree +** that the library will create and use. FTS uses recursion to perform +** various operations on the query tree, so the disadvantage of a large +** limit is that it may allow very large queries to use large amounts +** of stack space (perhaps causing a stack overflow). +*/ +#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH +# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 +#endif + + +/* +** This constant controls how often segments are merged. Once there are +** FTS3_MERGE_COUNT segments of level N, they are merged into a single +** segment of level N+1. +*/ +#define FTS3_MERGE_COUNT 16 + +/* +** This is the maximum amount of data (in bytes) to store in the +** Fts3Table.pendingTerms hash table. Normally, the hash table is +** populated as documents are inserted/updated/deleted in a transaction +** and used to create a new segment when the transaction is committed. +** However if this limit is reached midway through a transaction, a new +** segment is created and the hash table cleared immediately. +*/ +#define FTS3_MAX_PENDING_DATA (1*1024*1024) + +/* +** Macro to return the number of elements in an array. SQLite has a +** similar macro called ArraySize(). Use a different name to avoid +** a collision when building an amalgamation with built-in FTS3. +*/ +#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif + +/* +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. +*/ +#define FTS3_VARINT_MAX 10 + +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +*/ +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" + +/* +** The testcase() macro is only used by the amalgamation. If undefined, +** make it a no-op. +*/ +#ifndef testcase +# define testcase(X) +#endif + +/* +** Terminator values for position-lists and column-lists. +*/ +#define POS_COLUMN (1) /* Column-list terminator */ +#define POS_END (0) /* Position-list terminator */ + +/* +** This section provides definitions to allow the +** FTS3 extension to be compiled outside of the +** amalgamation. +*/ +#ifndef SQLITE_AMALGAMATION +/* +** Macros indicating that conditional expressions are always true or +** false. +*/ +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#else +# define ALWAYS(x) (x) +# define NEVER(x) (x) +#endif + +/* +** Internal types used by SQLite. +*/ +typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ +typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ + +/* +** Macro used to suppress compiler warnings for unused parameters. +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +#endif /* SQLITE_AMALGAMATION */ + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB +#endif + +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3PhraseToken Fts3PhraseToken; + +typedef struct Fts3Doclist Fts3Doclist; +typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; + +/* +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. +*/ +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of named columns in virtual table */ + char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ + int nAutoincrmerge; /* Value configured by 'automerge' */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ + + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[40]; + + char *zReadExprlist; + char *zWriteExprlist; + + int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ + u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + + /* + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. + */ + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif + +#ifdef SQLITE_TEST + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; +#endif +}; + +/* +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. +*/ +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + i16 eSearch; /* Search strategy (see below) */ + u8 isEof; /* True if at End Of Results */ + u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + u32 *aMatchinfo; /* Information about most recent match */ + int nMatchinfo; /* Number of elements in aMatchinfo[] */ + char *zMatchinfo; /* Matchinfo specification */ +}; + +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 + +/* +** The Fts3Cursor.eSearch member is always set to one of the following. +** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index +** of the column to be searched. For example, in +** +** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); +** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; +** +** Because the LHS of the MATCH operator is 2nd column "b", +** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** indicating that all columns should be searched, +** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. +*/ +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +/* +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. +*/ +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +}; + +/* +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. +*/ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ +}; + +struct Fts3Phrase { + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ +}; + +/* +** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. +** +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. +*/ +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ + + u32 *aMI; +}; + +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 + + +/* fts3_write.c */ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); + +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif + +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); + +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); + +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 +#define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +struct Fts3MultiSegReader { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ + + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ + + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; + +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + +/* fts3.c */ +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); + +/* fts3_tokenizer.c */ +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** +); +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); + +/* fts3_snippet.c */ +SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int +); +SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); + +/* fts3_expr.c */ +SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, + char **, int, int, int, const char *, int, Fts3Expr **, char ** +); +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); +#endif + +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); + +/* fts3_aux.c */ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); + +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +/* fts3_tokenize_vtab.c */ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); + +/* fts3_unicode2.c (functions generated by parsing unicode text files) */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); +#endif + +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ +#endif /* _FTSINT_H */ + +/************** End of fts3Int.h *********************************************/ +/************** Continuing where we left off in fts3.c ***********************/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif + +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +#ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT1 +#endif + +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. +*/ +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); + return (int) (q - (unsigned char *)p); +} + +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*ptr++) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ + const char *pStart = p; + u32 a; + u64 b; + int shift; + + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); + b = (a & 0x0FFFFFFF ); + + for(shift=28; shift<=63; shift+=7){ + u64 c = *p++; + b += (c&0x7F) << shift; + if( (c & 0x80)==0 ) break; + } + *v = b; + return (int)(p - pStart); +} + +/* +** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a +** 32-bit integer before it is returned. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){ + u32 a; + +#ifndef fts3GetVarint32 + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1); +#else + a = (*p++); + assert( a & 0x80 ); +#endif + + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); + a = (a & 0x0FFFFFFF ); + *pi = (int)(a | ((u32)(*p & 0x0F) << 28)); + return 5; +} + +/* +** Return the number of bytes required to encode v as a varint +*/ +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 ); + return i; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +** +*/ +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ + char quote; /* Quote character (if any ) */ + + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ + + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; + + while( ALWAYS(z[iIn]) ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; + }else{ + z[iOut++] = z[iIn++]; + } + } + z[iOut] = '\0'; + } +} + +/* +** Read a single varint from the doclist at *pp and advance *pp to point +** to the first byte past the end of the varint. Add the value of the varint +** to *pVal. +*/ +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; +} + +/* +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. +*/ +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; + + assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); + + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. +*/ +static void fts3DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts3DestroyMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ + + /* Drop the shadow tables */ + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + } + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); + + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} + + +/* +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + const char *zLanguageid; + + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); + } + + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); + } + + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; + } +} + +/* +** Create the %_stat table if it does not already exist. +*/ +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + +/* +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +** +** If the p->bHasDocsize boolean is true (indicating that this is an +** FTS4 table, not an FTS3 table) then also create the %_docsize and +** %_stat tables required by FTS4. +*/ +static int fts3CreateTables(Fts3Table *p){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3 *db = p->db; /* The database connection */ + + if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } + + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName + ); + } + assert( p->bHasStat==p->bFts4 ); + if( p->bHasStat ){ + sqlite3Fts3CreateStatTable(&rc, p); + } + return rc; +} + +/* +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } +} + +/* +** "Special" FTS4 arguments are column specifications of the following form: +** +** = +** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. +*/ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue +){ + char *zValue; + const char *zCsr = z; + + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } + + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; +} + +/* +** Append the output of a printf() style formatting to an existing string. +*/ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; + } + if( z==0 ) *pRc = SQLITE_NOMEM; + sqlite3_free(*pz); + *pz = z; + } +} + +/* +** Return a copy of input string zInput enclosed in double-quotes (") and +** with all double quote characters escaped. For example: +** +** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" +** +** The pointer returned points to memory obtained from sqlite3_malloc(). It +** is the callers responsibility to call sqlite3_free() to release this +** memory. +*/ +static char *fts3QuoteId(char const *zInput){ + int nRet; + char *zRet; + nRet = 2 + (int)strlen(zInput)*2 + 1; + zRet = sqlite3_malloc(nRet); + if( zRet ){ + int i; + char *z = zRet; + *(z++) = '"'; + for(i=0; zInput[i]; i++){ + if( zInput[i]=='"' ) *(z++) = '"'; + *(z++) = zInput[i]; + } + *(z++) = '"'; + *(z++) = '\0'; + } + return zRet; +} + +/* +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: +** +** SELECT FROM %_content AS x ... +** +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: +** +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } + sqlite3_free(zFree); + }else{ + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } + } + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); + return zRet; +} + +/* +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). +** +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: +** +** "?, zip(?), zip(?), zip(?)" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } + sqlite3_free(zFree); + return zRet; +} + +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ +static int fts3GobbleInt(const char **pp, int *pnOut){ + const char *p; /* Iterator pointer */ + int nInt = 0; /* Output value */ + + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} + +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +){ + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = aIndex; + *pnIndex = nIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i module name ("fts3" or "fts4") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + int nByte; /* Size of allocation used for *p */ + int iCol; /* Column index */ + int nString = 0; /* Bytes required to hold all column names */ + int nCol = 0; /* Number of columns in the FTS table */ + char *zCsr; /* Space for holding column names */ + int nDb; /* Bytes required to hold database name */ + int nName; /* Bytes required to hold table name */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + const char **aCol; /* Array of column names */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ + + int nIndex; /* Size of aIndex[] array */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ + + /* The results of parsing supported FTS4 key=value options: */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ + char **azNotindexed = 0; /* The set of notindexed= columns */ + int nNotindexed = 0; /* Size of azNotindexed[] array */ + + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); + + nDb = (int)strlen(argv[1]) + 1; + nName = (int)strlen(argv[2]) + 1; + + nByte = sizeof(const char *) * (argc-2); + aCol = (const char **)sqlite3_malloc(nByte); + if( aCol ){ + memset((void*)aCol, 0, nByte); + azNotindexed = (char **)sqlite3_malloc(nByte); + } + if( azNotindexed ){ + memset(azNotindexed, 0, nByte); + } + if( !aCol || !azNotindexed ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + + /* Loop through all of the arguments passed by the user to the FTS3/4 + ** module (i.e. all the column names and special arguments). This loop + ** does the following: + ** + ** + Figures out the number of columns the FTSX table will have, and + ** the number of bytes of space that must be allocated to store copies + ** of the column names. + ** + ** + If there is a tokenizer specification included in the arguments, + ** initializes the tokenizer pTokenizer. + */ + for(i=3; rc==SQLITE_OK && i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + } + + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + } aFts4Opt[] = { + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ + }; + + int iOpt; + if( !zVal ){ + rc = SQLITE_NOMEM; + }else{ + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + if( iOpt==SizeofArray(aFts4Opt) ){ + *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + } + } + sqlite3_free(zVal); + } + } + + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; + } + } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; jdb = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = isFts4; + p->bFts4 = isFts4; + p->bDescIdx = bDescIdx; + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; + zContent = 0; + zLanguageid = 0; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; + + /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->abNotindexed[nCol]; + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; + + /* Fill in the azColumn array */ + for(iCol=0; iColazColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); + } + + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; + + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); + } + + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + p->bHasStat = 2; + } + + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database. */ + fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; + + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); + +fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aIndex); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + sqlite3_free(zContent); + sqlite3_free(zLanguageid); + for(i=0; ipModule->xDestroy(pTokenizer); + } + }else{ + assert( p->pSegments==0 ); + *ppVTab = &p->base; + } + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). +*/ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: +** +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. +*/ +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ + + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; + + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 5000000; + for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } + + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); + + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + } + + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; + } + + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; + + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } + } + } + + iIdx = 1; + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; + pInfo->aConstraintUsage[iCons].omit = 1; + } + if( iLangidCons>=0 ){ + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; + } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } + + assert( p->pSegments==0 ); + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ + + UNUSED_PARAMETER(pVTab); + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_finalize(pCsr->pStmt); + sqlite3Fts3ExprFree(pCsr->pExpr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3_free(pCsr->aMatchinfo); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +** +** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + *ppStmt = pCsr->pStmt; + return rc; +} + +/* +** Position the pCsr->pStmt statement so that it is on the row +** of the %_content table that contains the last match. Return +** SQLITE_OK on success. +*/ +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->isRequireSeek ){ + sqlite3_stmt *pStmt = 0; + + rc = fts3CursorSeekStmt(pCsr, &pStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + return SQLITE_OK; + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occurred, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } + } + } + } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; +} + +/* +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. +** +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ + + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + if( zCsr>zEnd ){ + return FTS_CORRUPT_VTAB; + } + + while( zCsrzEnd ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + if( nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + assert( zBuffer ); + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; + } + + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; + } + + iChild++; + }; + + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; + + finish_scan: + sqlite3_free(zBuffer); + return rc; +} + + +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. +** +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. +** +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ +){ + int rc; /* Return code */ + int iHeight; /* Height of this node in tree */ + + assert( piLeaf || piLeaf2 ); + + fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob; /* Size of zBlob in bytes */ + + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + sqlite3_free(zBlob); + } + + return rc; +} + +/* +** This function is used to create delta-encoded serialized lists of FTS3 +** varints. Each call to this function appends a single varint to a list. +*/ +static void fts3PutDeltaVarint( + char **pp, /* IN/OUT: Output pointer */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. +** +** A position list is list of positions (delta encoded) and columns for +** a single document record of a doclist. So, in other words, this +** routine advances *ppPoslist so that it points to the next docid in +** the doclist, or to the first byte past the end of the doclist. +** +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. +*/ +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following while-loop moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. + */ + while( *pEnd | c ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && (*pEnd)==0 ); + } + pEnd++; /* Advance past the POS_END terminator byte */ + + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a column-list. After it returns, *ppPoslist points to the +** to the terminator (POS_COLUMN or POS_END) byte of the column-list. +** +** A column-list is list of delta-encoded positions for a single column +** within a single document within a doclist. +** +** The column-list is terminated either by a POS_COLUMN varint (1) or +** a POS_END varint (0). This routine leaves *ppPoslist pointing to +** the POS_COLUMN or POS_END that terminates the column-list. +** +** If pp is not NULL, then the contents of the column-list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. The POS_COLUMN or POS_END terminator +** is not copied into *pp. +*/ +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* A column-list is terminated by either a 0x01 or 0x00 byte that is + ** not part of a multi-byte varint. + */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && ((*pEnd)&0xfe)==0 ); + } + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** Value used to signify the end of an position-list. This is safe because +** it is not possible to have a document with 2^31 terms. +*/ +#define POSITION_LIST_END 0x7fffffff + +/* +** This function is used to help parse position-lists. When this function is +** called, *pp may point to the start of the next varint in the position-list +** being parsed, or it may point to 1 byte past the end of the position-list +** (in which case **pp will be a terminator bytes POS_END (0) or +** (1)). +** +** If *pp points past the end of the current position-list, set *pi to +** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, +** increment the current value of *pi by the value read, and set *pp to +** point to the next value before returning. +** +** Before calling this routine *pi must be initialized to the value of +** the previous position, or zero if we are reading the first position +** in the position-list. Because positions are delta-encoded, the value +** of the previous position is needed in order to compute the value of +** the next position. +*/ +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into position-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +){ + if( (**pp)&0xFE ){ + fts3GetDeltaVarint(pp, pi); + *pi -= 2; + }else{ + *pi = POSITION_LIST_END; + } +} + +/* +** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by +** the value of iCol encoded as a varint to *pp. This will start a new +** column list. +** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). +*/ +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; + } + return n; +} + +/* +** Compute the union of two position lists. The output written +** into *pp contains all positions of both *pp1 and *pp2 in sorted +** order and with any duplicates removed. All pointers are +** updated appropriately. The caller is responsible for insuring +** that there is enough space in *pp to hold the complete output. +*/ +static void fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + + while( *p1 || *p2 ){ + int iCol1; /* The current column index in pp1 */ + int iCol2; /* The current column index in pp2 */ + + if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); + else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; + else iCol1 = 0; + + if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); + else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; + else iCol2 = 0; + + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; /* Last position from pp1 */ + sqlite3_int64 i2 = 0; /* Last position from pp2 */ + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; + + /* At this point, both p1 and p2 point to the start of column-lists + ** for the same column (the column with index iCol1 and iCol2). + ** A column-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a + ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (POS_END or POS_COLUMN) is + ** written to the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + do { + fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. +*/ +static int fts3PoslistPhraseMerge( + char **pp, /* IN/OUT: Preallocated output buffer */ + int nToken, /* Maximum difference in token positions */ + int isSaveLeft, /* Save the left position */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + int iCol1 = 0; + int iCol2 = 0; + + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); + + assert( p!=0 && *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + } + if( *p2==POS_COLUMN ){ + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + while( 1 ){ + if( iCol1==iCol2 ){ + char *pSave = p; + sqlite3_int64 iPrev = 0; + sqlite3_int64 iPos1 = 0; + sqlite3_int64 iPos2 = 0; + + if( iCol1 ){ + *p++ = POS_COLUMN; + p += sqlite3Fts3PutVarint(p, iCol1); + } + + assert( *p1!=POS_END && *p1!=POS_COLUMN ); + assert( *p2!=POS_END && *p2!=POS_COLUMN ); + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + + while( 1 ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ + sqlite3_int64 iSave; + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + assert( p ); + } + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + } + + if( pSave ){ + assert( pp && p ); + p = pSave; + } + + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; + + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; + }else{ + *pVal += iVal; + } + } +} + +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + + +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ +static int fts3DoclistOrMerge( + int bDescDoclist, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + + /* Allocate space for the output. Both the input and output doclists + ** are delta encoded. If they are in ascending order (bDescDoclist==0), + ** then the first docid in each list is simply encoded as a varint. For + ** each subsequent docid, the varint stored is the difference between the + ** current and previous docid (a positive number - since the list is in + ** ascending order). + ** + ** The first docid written to the output is therefore encoded using the + ** same number of bytes as it is in whichever of the input lists it is + ** read from. And each subsequent docid read from the same input list + ** consumes either the same or less bytes as it did in the input (since + ** the difference between it and the previous value in the output must + ** be a positive value less than or equal to the delta value read from + ** the input list). The same argument applies to all but the first docid + ** read from the 'other' list. And to the contents of all position lists + ** that will be copied and merged from the input to the output. + ** + ** However, if the first docid copied to the output is a negative number, + ** then the encoding of the first docid from the 'other' input list may + ** be larger in the output than it was in the input (since the delta value + ** may be a larger positive integer than the actual docid). + ** + ** The space required to store the output is therefore the sum of the + ** sizes of the two inputs, plus enough space for exactly one of the input + ** docids to grow. + ** + ** A symetric argument may be made if the doclists are in descending + ** order. + */ + aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + } + + *paOut = aOut; + *pnOut = (int)(p-aOut); + assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); + return SQLITE_OK; +} + +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ +static void fts3DoclistPhraseMerge( + int bDescDoclist, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut = aRight; + + assert( nDist>0 ); + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + } + + *pnRight = (int)(p - aOut); +} + +/* +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). +*/ +SQLITE_PRIVATE int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ +){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; + + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( paaOutput); i++){ + if( pTS->aaOutput[i] ){ + if( !aOut ){ + aOut = pTS->aaOutput[i]; + nOut = pTS->anOutput[i]; + pTS->aaOutput[i] = 0; + }else{ + int nNew; + char *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + + sqlite3_free(pTS->aaOutput[i]); + sqlite3_free(aOut); + pTS->aaOutput[i] = 0; + aOut = aNew; + nOut = nNew; + } + } + } + + pTS->aaOutput[0] = aOut; + pTS->anOutput[0] = nOut; + return SQLITE_OK; +} + +/* +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). +** +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. +*/ +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ +){ + if( pTS->aaOutput[0]==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist); + pTS->anOutput[0] = nDoclist; + if( pTS->aaOutput[0] ){ + memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + }else{ + return SQLITE_NOMEM; + } + }else{ + char *aMerge = aDoclist; + int nMerge = nDoclist; + int iOut; + + for(iOut=0; iOutaaOutput); iOut++){ + if( pTS->aaOutput[iOut]==0 ){ + assert( iOut>0 ); + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + break; + }else{ + char *aNew; + int nNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; + } + + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } + } + } + } + return SQLITE_OK; +} + +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; + } + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; +} + +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +static int fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ + + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); + } + + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; + + /* Read the values returned by the SELECT into local variables. */ + sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); + sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + + /* If zTerm is not NULL, and this segment is not stored entirely on its + ** root node, the range of leaves scanned can be reduced. Do this. */ + if( iStartBlock && zTerm ){ + sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); + if( rc!=SQLITE_OK ) goto finished; + if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + } + + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + (isPrefix==0 && isScan==0), + iStartBlock, iLeavesEndBlock, + iEndBlock, zRoot, nRoot, &pSeg + ); + if( rc!=SQLITE_OK ) goto finished; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + finished: + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_DONE ) rc = rc2; + + return rc; +} + +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language-id to search */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelbase.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; + } + } + + *ppSegcsr = pSegcsr; + return rc; +} + +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ + sqlite3Fts3SegReaderFinish(pSegcsr); + sqlite3_free(pSegcsr); +} + +/* +** This function retrieves the doclist for the specified term (or term +** prefix) from the database. +*/ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ + int iColumn, /* Column to query (or -ve for all columns) */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int rc; /* Return code */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ + Fts3SegFilter filter; /* Segment term filter configuration */ + + pSegcsr = pTok->pSegcsr; + memset(&tsc, 0, sizeof(TermSelect)); + + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; + + rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); + while( SQLITE_OK==rc + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + ){ + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); + } + + if( rc==SQLITE_OK ){ + rc = fts3TermSelectFinishMerge(p, &tsc); + } + if( rc==SQLITE_OK ){ + *ppOut = tsc.aaOutput[0]; + *pnOut = tsc.anOutput[0]; + }else{ + int i; + for(i=0; ipSegcsr = 0; + return rc; +} + +/* +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. +*/ +static int fts3DoclistCountDocids(char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + while( peSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; + } + }else{ + rc = fts3EvalNext((Fts3Cursor *)pCursor); + } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** The following are copied from sqliteInt.h. +** +** Constants for the largest and smallest possible 64-bit signed integers. +** These macros are designed to work correctly on both 32-bit and 64-bit +** compilers. +*/ +#ifndef SQLITE_AMALGAMATION +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** If the numeric type of argument pVal is "integer", then return it +** converted to a 64-bit signed integer. Otherwise, return a copy of +** the second parameter, iDefault. +*/ +static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against +** the %_content table. +** +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry +** in the %_content table. +** +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The +** column on the left-hand side of the MATCH operator is column +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand +** side of the MATCH operator. +*/ +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc; + char *zSql; /* SQL statement used to access %_content */ + int eSearch; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ + sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ + sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ + sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ + int iIdx; + + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + eSearch = (idxNum & 0x0000FFFF); + assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( p->pSegments==0 ); + + /* Collect arguments into local variables */ + iIdx = 0; + if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; + assert( iIdx==nVal ); + + /* In case the cursor has been used before, clear it now. */ + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3ExprFree(pCsr->pExpr); + memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + + /* Set the lower and upper bounds on docids to return */ + pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); + pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)eSearch; + + if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ + int iCol = eSearch-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(pCons); + + if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + + pCsr->iLangid = 0; + if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); + + assert( p->base.zErrMsg==0 ); + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg + ); + if( rc!=SQLITE_OK ){ + return rc; + } + + rc = fts3EvalStart(pCsr); + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; + } + + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + if( eSearch==FTS3_FULLSCAN_SEARCH ){ + zSql = sqlite3_mprintf( + "SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + if( zSql ){ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( eSearch==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); + } + } + if( rc!=SQLITE_OK ) return rc; + + return fts3NextMethod(pCursor); +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + return ((Fts3Cursor *)pCursor)->isEof; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + *pRowid = pCsr->iPrevId; + return SQLITE_OK; +} + +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column +*/ +static int fts3ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + + /* The column value supplied by SQLite must be in range. */ + assert( iCol>=0 && iCol<=p->nColumn+2 ); + + if( iCol==p->nColumn+1 ){ + /* This call is a request for the "docid" column. Since "docid" is an + ** alias for "rowid", use the xRowid() method to obtain the value. + */ + sqlite3_result_int64(pCtx, pCsr->iPrevId); + }else if( iCol==p->nColumn ){ + /* The extra column whose name is the same as the table. + ** Return a blob which is a pointer to the cursor. */ + sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); + }else{ + /* The requested column is either a user column (one that contains + ** indexed data), or the language-id column. */ + rc = fts3CursorSeek(0, pCsr); + + if( rc==SQLITE_OK ){ + if( iCol==p->nColumn+2 ){ + int iLangid = 0; + if( p->zLanguageid ){ + iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); + } + sqlite3_result_int(pCtx, iLangid); + }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } + } + } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +*/ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +} + +/* +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. +*/ +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + + Fts3Table *p = (Fts3Table*)pVtab; + int rc = sqlite3Fts3PendingTermsFlush(p); + + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + +/* +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; + char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + } + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. +*/ +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(pVtab); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + TESTONLY( p->inTransaction = 1 ); + TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; + return fts3SetHasStat(p); +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). +*/ +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(pVtab); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. +*/ +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c = 0; + + while( p>pStart && (c=*p--)==0 ); + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; + } + if( p>pStart ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; +} + +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. +** +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. +*/ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +){ + Fts3Cursor *pRet; + if( sqlite3_value_type(pVal)!=SQLITE_BLOB + || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) + ){ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + return SQLITE_ERROR; + } + memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); + *ppCsr = pRet; + return SQLITE_OK; +} + +/* +** Implementation of the snippet() function for FTS3 +*/ +static void fts3SnippetFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of apVal[] array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + int iCol = -1; + int nToken = 15; /* Default number of tokens in snippet */ + + /* There must be at least one argument passed to this function (otherwise + ** the non-overloaded version would have been called instead of this one). + */ + assert( nVal>=1 ); + + if( nVal>6 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function snippet()", -1); + return; + } + if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; + + switch( nVal ){ + case 6: nToken = sqlite3_value_int(apVal[5]); + case 5: iCol = sqlite3_value_int(apVal[4]); + case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + } + if( !zEllipsis || !zEnd || !zStart ){ + sqlite3_result_error_nomem(pContext); + }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + } +} + +/* +** Implementation of the offsets() function for FTS3 +*/ +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Offsets(pContext, pCsr); + } +} + +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: +** +** SELECT optimize(t) FROM t LIMIT 1; +** +** where 't' is the name of an FTS3 table. +*/ +static void fts3OptimizeFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + int rc; /* Return code */ + Fts3Table *p; /* Virtual table handle */ + Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; + p = (Fts3Table *)pCursor->base.pVtab; + assert( p ); + + rc = sqlite3Fts3Optimize(p); + + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; + } +} + +/* +** Implementation of the matchinfo() function for FTS3 +*/ +static void fts3MatchinfoFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + assert( nVal==1 || nVal==2 ); + if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); + } +} + +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ +){ + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + { "matchinfo", fts3MatchinfoFunc }, + }; + int i; /* Iterator variable */ + + UNUSED_PARAMETER(pVtab); + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(ppArg); + + for(i=0; idb; /* Database connection */ + int rc; /* Return Code */ + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); + } + + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", + p->zDb, p->zName, zName + ); + } + if( p->bHasStat ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", + p->zDb, p->zName, zName + ); + } + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", + p->zDb, p->zName, zName + ); + return rc; +} + +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(iSavepoint); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); + } + return rc; +} + +/* +** The xRelease() method. +** +** This is a no-op. +*/ +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(iSavepoint); + UNUSED_PARAMETER(pVtab); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint-1 ); + return SQLITE_OK; +} + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint ); + sqlite3Fts3PendingTermsClear(p); + return SQLITE_OK; +} + +static const sqlite3_module fts3Module = { + /* iVersion */ 2, + /* xCreate */ fts3CreateMethod, + /* xConnect */ fts3ConnectMethod, + /* xBestIndex */ fts3BestIndexMethod, + /* xDisconnect */ fts3DisconnectMethod, + /* xDestroy */ fts3DestroyMethod, + /* xOpen */ fts3OpenMethod, + /* xClose */ fts3CloseMethod, + /* xFilter */ fts3FilterMethod, + /* xNext */ fts3NextMethod, + /* xEof */ fts3EofMethod, + /* xColumn */ fts3ColumnMethod, + /* xRowid */ fts3RowidMethod, + /* xUpdate */ fts3UpdateMethod, + /* xBegin */ fts3BeginMethod, + /* xSync */ fts3SyncMethod, + /* xCommit */ fts3CommitMethod, + /* xRollback */ fts3RollbackMethod, + /* xFindFunction */ fts3FindFunctionMethod, + /* xRename */ fts3RenameMethod, + /* xSavepoint */ fts3SavepointMethod, + /* xRelease */ fts3ReleaseMethod, + /* xRollbackTo */ fts3RollbackToMethod, +}; + +/* +** This function is registered as the module destructor (called when an +** FTS3 enabled database connection is closed). It frees the memory +** allocated for the tokenizer hash table. +*/ +static void hashDestroy(void *p){ + Fts3Hash *pHash = (Fts3Hash *)p; + sqlite3Fts3HashClear(pHash); + sqlite3_free(pHash); +} + +/* +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c +** respectively. The following three forward declarations are for functions +** declared in these files used to retrieve the respective implementations. +** +** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed +** to by the argument to point to the "simple" tokenizer implementation. +** And so on. +*/ +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); +#endif +#ifdef SQLITE_ENABLE_ICU +SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif + +/* +** Initialize the fts3 extension. If this extension is built as part +** of the sqlite library, then this function is called directly by +** SQLite. If fts3 is built as a dynamically loadable extension, this +** function is called by the sqlite3_extension_init() entry point. +*/ +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ + int rc = SQLITE_OK; + Fts3Hash *pHash = 0; + const sqlite3_tokenizer_module *pSimple = 0; + const sqlite3_tokenizer_module *pPorter = 0; +#ifndef SQLITE_DISABLE_FTS3_UNICODE + const sqlite3_tokenizer_module *pUnicode = 0; +#endif + +#ifdef SQLITE_ENABLE_ICU + const sqlite3_tokenizer_module *pIcu = 0; + sqlite3Fts3IcuTokenizerModule(&pIcu); +#endif + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + sqlite3Fts3UnicodeTokenizer(&pUnicode); +#endif + +#ifdef SQLITE_TEST + rc = sqlite3Fts3InitTerm(db); + if( rc!=SQLITE_OK ) return rc; +#endif + + rc = sqlite3Fts3InitAux(db); + if( rc!=SQLITE_OK ) return rc; + + sqlite3Fts3SimpleTokenizerModule(&pSimple); + sqlite3Fts3PorterTokenizerModule(&pPorter); + + /* Allocate and initialize the hash-table used to store tokenizers. */ + pHash = sqlite3_malloc(sizeof(Fts3Hash)); + if( !pHash ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); + } + + /* Load the built-in tokenizers into the hash table */ + if( rc==SQLITE_OK ){ + if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) +#endif +#ifdef SQLITE_ENABLE_ICU + || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) +#endif + ){ + rc = SQLITE_NOMEM; + } + } + +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ExprInitTestInterface(db); + } +#endif + + /* Create the virtual table wrapper around the hash-table and overload + ** the two scalar functions. If this is successful, register the + ** module with sqlite. + */ + if( SQLITE_OK==rc + && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) + ){ + rc = sqlite3_create_module_v2( + db, "fts3", &fts3Module, (void *)pHash, hashDestroy + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module_v2( + db, "fts4", &fts3Module, (void *)pHash, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3InitTok(db, (void *)pHash); + } + return rc; + } + + + /* An error has occurred. Delete the hash table and return the error code. */ + assert( rc!=SQLITE_OK ); + if( pHash ){ + sqlite3Fts3HashClear(pHash); + sqlite3_free(pHash); + } + return rc; +} + +/* +** Allocate an Fts3MultiSegReader for each token in the expression headed +** by pExpr. +** +** An Fts3SegReader object is a cursor that can seek or scan a range of +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple +** Fts3SegReader objects internally to provide an interface to seek or scan +** within the union of all segments of a b-tree. Hence the name. +** +** If the allocated Fts3MultiSegReader just seeks to a single entry in a +** segment b-tree (if the term is not a prefix or it is a prefix for which +** there exists prefix b-tree of the right length) then it may be traversed +** and merged incrementally. Otherwise, it has to be merged into an in-memory +** doclist and then traversed. +*/ +static void fts3EvalAllocateReaders( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Allocate readers for this expression */ + int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} + +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +*/ +static void fts3EvalPhraseMergeToken( + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ +){ + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; +} + +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + + if( pToken->pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); + if( rc==SQLITE_OK ){ + fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } + } + assert( pToken->pSegcsr==0 ); + } + + return rc; +} + +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; /* Used to iterate through phrase tokens */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ + + assert( pPhrase->doclist.bFreeList==0 ); + + for(iToken=0; iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = (int)(aOut - aPoslist); + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } + + return SQLITE_OK; +} + +/* +** Maximum number of tokens a phrase may have to be considered for the +** incremental doclists strategy. +*/ +#define MAX_INCR_PHRASE_TOKENS 4 + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Error code */ + int i; + + /* Determine if doclists may be loaded from disk incrementally. This is + ** possible if the bOptOk argument is true, the FTS doclists will be + ** scanned in forward order, and the phrase consists of + ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" + ** tokens or prefix tokens that cannot use a prefix-index. */ + int bHaveIncr = 0; + int bIncrOk = (bOptOk + && pCsr->bDesc==pTab->bDescIdx + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 +#ifdef SQLITE_TEST + && pTab->bNoIncrDoclist==0 +#endif + ); + for(i=0; bIncrOk==1 && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ + bIncrOk = 0; + } + if( pToken->pSegcsr ) bHaveIncr = 1; + } + + if( bIncrOk && bHaveIncr ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + for(i=0; rc==SQLITE_OK && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; + if( pSegcsr ){ + rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + } + } + p->bIncr = 1; + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. +*/ +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDocid0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + + if( p==0 ){ + p = aDoclist; + p += sqlite3Fts3GetVarint(p, piDocid); + }else{ + fts3PoslistCopy(0, &p); + if( p>=&aDoclist[nDoclist] ){ + *pbEof = 1; + }else{ + sqlite3_int64 iVar; + p += sqlite3Fts3GetVarint(p, &iVar); + *piDocid += ((bDescIdx ? -1 : 1) * iVar); + } + } + + *ppIter = p; +} + +/* +** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof +** to true if EOF is reached. +*/ +static void fts3EvalDlPhraseNext( + Fts3Table *pTab, + Fts3Doclist *pDL, + u8 *pbEof +){ + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=pEnd ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (int)(pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } +} + +/* +** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +*/ +typedef struct TokenDoclist TokenDoclist; +struct TokenDoclist { + int bIgnore; + sqlite3_int64 iDocid; + char *pList; + int nList; +}; + +/* +** Token pToken is an incrementally loaded token that is part of a +** multi-token phrase. Advance it to the next matching document in the +** database and populate output variable *p with the details of the new +** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int incrPhraseTokenNext( + Fts3Table *pTab, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to advance token of */ + int iToken, /* Specific token to advance */ + TokenDoclist *p, /* OUT: Docid and doclist for new entry */ + u8 *pbEof /* OUT: True if iterator is at EOF */ +){ + int rc = SQLITE_OK; + + if( pPhrase->iDoclistToken==iToken ){ + assert( p->bIgnore==0 ); + assert( pPhrase->aToken[iToken].pSegcsr==0 ); + fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); + p->pList = pPhrase->doclist.pList; + p->nList = pPhrase->doclist.nList; + p->iDocid = pPhrase->doclist.iDocid; + }else{ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + assert( pToken->pDeferred==0 ); + assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); + if( pToken->pSegcsr ){ + assert( p->bIgnore==0 ); + rc = sqlite3Fts3MsrIncrNext( + pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList + ); + if( p->pList==0 ) *pbEof = 1; + }else{ + p->bIgnore = 1; + } + } + + return rc; +} + + +/* +** The phrase iterator passed as the second argument: +** +** * features at least one token that uses an incremental doclist, and +** +** * does not contain any deferred tokens. +** +** Advance it to the next matching documnent in the database and populate +** the Fts3Doclist.pList and nList fields. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int fts3EvalIncrPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + u8 bEof = 0; + + /* This is only called if it is guaranteed that the phrase has at least + ** one incremental token. In which case the bIncr flag is set. */ + assert( p->bIncr==1 ); + + if( p->nToken==1 && p->bIncr ){ + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( pDL->pList==0 ) bEof = 1; + }else{ + int bDescDoclist = pCsr->bDesc; + struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; + + memset(a, 0, sizeof(a)); + assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); + assert( p->iDoclistTokennToken && bEof==0; i++){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ + iMax = a[i].iDocid; + bMaxSet = 1; + } + } + assert( rc!=SQLITE_OK || a[p->nToken-1].bIgnore==0 ); + assert( rc!=SQLITE_OK || bMaxSet ); + + /* Keep advancing iterators until they all point to the same document */ + for(i=0; inToken; i++){ + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + ){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ + iMax = a[i].iDocid; + i = 0; + } + } + } + + /* Check if the current entries really are a phrase match */ + if( bEof==0 ){ + int nList = 0; + int nByte = a[p->nToken-1].nList; + char *aDoclist = sqlite3_malloc(nByte+1); + if( !aDoclist ) return SQLITE_NOMEM; + memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + + for(i=0; i<(p->nToken-1); i++){ + if( a[i].bIgnore==0 ){ + char *pL = a[i].pList; + char *pR = aDoclist; + char *pOut = aDoclist; + int nDist = p->nToken-1-i; + int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); + if( res==0 ) break; + nList = (int)(pOut - aDoclist); + } + } + if( i==(p->nToken-1) ){ + pDL->iDocid = iMax; + pDL->pList = aDoclist; + pDL->nList = nList; + pDL->bFreeList = 1; + break; + } + sqlite3_free(aDoclist); + } + } + } + + *pbEof = bEof; + return rc; +} + +/* +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +*/ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + fts3EvalDlPhraseNext(pTab, pDL, pbEof); + } + + return rc; +} + +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ + int iCol; /* The column the token must match */ +}; + +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + int rc; + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + assert( a ); + + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + + *pnPage = pCsr->nRowAvg; + return SQLITE_OK; +} + +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ + + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ + for(ii=0; ii0 ); + + + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) + ){ + pTC = &aTC[iTC]; + } + } + assert( pTC ); + + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + + if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + int nCount; + fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountpToken = 0; + } + + return rc; +} + +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); + + /* Determine which, if any, tokens in the expression should be deferred. */ +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ + Fts3TokenAndCost *aTC; + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + apOr = (Fts3Expr **)&aTC[nToken]; + + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; + + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); + + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iipExpr, &rc); + return rc; +} + +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ + if( pPhrase->doclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} + +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in, the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; + + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } + + return res; +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See fts3EvalTestDeferredAndNear() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this - it's the way the parser in fts3_expr.c works. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && pExpr->bEof==0 + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + int nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + if( nTmp==0 ){ + res = 0; + }else{ + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + } + + return res; +} + +/* +** This function is a helper function for fts3EvalTestDeferredAndNear(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. +*/ +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ +){ + int bHit = 1; /* Return value */ + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); + + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pPhrase); + } + } + + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalInvalidatePoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else +#endif + { + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + } + break; + } + } + } + return bHit; +} + +/* +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does *not* match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. +*/ +static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. +*/ +static int fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNextRow(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); + } + + /* Check if the cursor is past the end of the docid range specified + ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ + if( rc==SQLITE_OK && ( + (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) + || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) + )){ + pCsr->isEof = 1; + } + + return rc; +} + +/* +** Restart interation for expression pExpr so that the next call to +** fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. +*/ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase ){ + fts3EvalInvalidatePoslist(pPhrase); + if( pPhrase->bIncr ){ + int i; + for(i=0; inToken; i++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[i]; + assert( pToken->pDeferred==0 ); + if( pToken->pSegcsr ){ + sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); + } + } + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + } + + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; + + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + assert( *p ); + while( 1 ){ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += fts3GetVarint32(p, &iCol); + } + } + + fts3EvalUpdateCounts(pExpr->pLeft); + fts3EvalUpdateCounts(pExpr->pRight); + } +} + +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNextRow(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && fts3EvalTestDeferredAndNear(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + fts3EvalTestDeferredAndNear(pCsr, &rc); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } + } + } + + return rc; +} + +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurrence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol, /* Column to return position list for */ + char **ppOut /* OUT: Pointer to position list */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter; + int iThis; + sqlite3_int64 iDocid; + + /* If this phrase is applies specifically to some column other than + ** column iCol, return a NULL pointer. */ + *ppOut = 0; + assert( iCol>=0 && iColnColumn ); + if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ + return SQLITE_OK; + } + + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int iMul; /* +1 if csr dir matches index dir, else -1 */ + int bOr = 0; + u8 bEof = 0; + u8 bTreeEof = 0; + Fts3Expr *p; /* Used to iterate from pExpr to root */ + Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid + ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the + ** tree that the node is part of has been marked as EOF, but the node + ** itself is not EOF, then it may point to an earlier entry. */ + pNear = pExpr; + for(p=pExpr->pParent; p; p=p->pParent){ + if( p->eType==FTSQUERY_OR ) bOr = 1; + if( p->eType==FTSQUERY_NEAR ) pNear = p; + if( p->bEof ) bTreeEof = 1; + } + if( bOr==0 ) return SQLITE_OK; + + /* This is the descendent of an OR node. In this case we cannot use + ** an incremental phrase. Load the entire doclist for the phrase + ** into memory in this case. */ + if( pPhrase->bIncr ){ + int rc = SQLITE_OK; + int bEofSave = pExpr->bEof; + fts3EvalRestart(pCsr, pExpr, &rc); + while( rc==SQLITE_OK && !pExpr->bEof ){ + fts3EvalNextRow(pCsr, pExpr, &rc); + if( bEofSave==0 && pExpr->iDocid==iDocid ) break; + } + pIter = pPhrase->doclist.pList; + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + if( rc!=SQLITE_OK ) return rc; + } + + iMul = ((pCsr->bDesc==bDescDoclist) ? 1 : -1); + while( bTreeEof==1 + && pNear->bEof==0 + && (DOCID_CMP(pNear->iDocid, pCsr->iPrevId) * iMul)<0 + ){ + int rc = SQLITE_OK; + fts3EvalNextRow(pCsr, pExpr, &rc); + if( rc!=SQLITE_OK ) return rc; + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + } + + bEof = (pPhrase->doclist.nAll==0); + assert( bDescDoclist==0 || bDescDoclist==1 ); + assert( pCsr->bDesc==0 || pCsr->bDesc==1 ); + + if( bEof==0 ){ + if( pCsr->bDesc==bDescDoclist ){ + int dummy; + if( pNear->bEof ){ + /* This expression is already at EOF. So position it to point to the + ** last entry in the doclist at pPhrase->doclist.aAll[]. Variable + ** iDocid is already set for this entry, so all that is required is + ** to set pIter to point to the first byte of the last position-list + ** in the doclist. + ** + ** It would also be correct to set pIter and iDocid to zero. In + ** this case, the first call to sqltie3Fts4DoclistPrev() below + ** would also move the iterator to point to the last entry in the + ** doclist. However, this is expensive, as to do so it has to + ** iterate through the entire doclist from start to finish (since + ** it does not know the docid for the last entry). */ + pIter = &pPhrase->doclist.aAll[pPhrase->doclist.nAll-1]; + fts3ReversePoslist(pPhrase->doclist.aAll, &pIter); + } + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + sqlite3Fts3DoclistPrev( + bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + }else{ + if( pNear->bEof ){ + pIter = 0; + iDocid = 0; + } + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } + } + } + + if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0; + } + if( pIter==0 ) return SQLITE_OK; + + if( *pIter==0x01 ){ + pIter++; + pIter += fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. +*/ +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalInvalidatePoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} + + +/* +** Return SQLITE_CORRUPT_VTAB. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ + return SQLITE_CORRUPT_VTAB; +} +#endif + +#if !SQLITE_CORE +/* +** Initialize API pointer table, if required. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_fts3_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif + +#endif + +/************** End of fts3.c ************************************************/ +/************** Begin file fts3_aux.c ****************************************/ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3auxTable Fts3auxTable; +typedef struct Fts3auxCursor Fts3auxCursor; + +struct Fts3auxTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; + +struct Fts3auxCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + char *zStop; + int nStop; /* Byte-length of string zStop */ + int iLangid; /* Language id to query */ + int isEof; /* True if cursor is at EOF */ + sqlite3_int64 iRowid; /* Current rowid */ + + int iCol; /* Current value of 'col' column */ + int nStat; /* Size of aStat[] array */ + struct Fts3auxColstats { + sqlite3_int64 nDoc; /* 'documents' values for current csr row */ + sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ + } *aStat; +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_AUX_SCHEMA \ + "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3auxConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + int nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3auxTable *p; /* Virtual table object to return */ + + UNUSED_PARAMETER(pUnused); + + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; + + zDb = argv[1]; + nDb = (int)strlen(zDb); + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } + nFts3 = (int)strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3auxTable *)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; + + bad_args: + *pzErr = sqlite3_mprintf("invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3auxTable *p = (Fts3auxTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; +} + +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3auxBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + int iEq = -1; + int iGe = -1; + int iLe = -1; + int iLangid = -1; + int iNext = 1; /* Next free argvIndex value */ + + UNUSED_PARAMETER(pVTab); + + /* This vtab delivers always results in "ORDER BY term ASC" order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + /* Search for equality and range constraints on the "term" column. + ** And equality constraints on the hidden "languageid" column. */ + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable ){ + int op = pInfo->aConstraint[i].op; + int iCol = pInfo->aConstraint[i].iColumn; + + if( iCol==0 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + } + if( iCol==4 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; + } + } + } + + if( iEq>=0 ){ + pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; + pInfo->aConstraintUsage[iEq].argvIndex = iNext++; + pInfo->estimatedCost = 5; + }else{ + pInfo->idxNum = 0; + pInfo->estimatedCost = 20000; + if( iGe>=0 ){ + pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; + pInfo->aConstraintUsage[iGe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + if( iLe>=0 ){ + pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; + pInfo->aConstraintUsage[iLe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + } + if( iLangid>=0 ){ + pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; + pInfo->estimatedCost--; + } + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ + + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3auxCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->zStop); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ + if( nSize>pCsr->nStat ){ + struct Fts3auxColstats *aNew; + aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, + sizeof(struct Fts3auxColstats) * nSize + ); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nStat], 0, + sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) + ); + pCsr->aStat = aNew; + pCsr->nStat = nSize; + } + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ + if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; + } + + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc==SQLITE_ROW ){ + int i = 0; + int nDoclist = pCsr->csr.nDoclist; + char *aDoclist = pCsr->csr.aDoclist; + int iCol; + + int eState = 0; + + if( pCsr->zStop ){ + int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; + int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); + if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ + pCsr->isEof = 1; + return SQLITE_OK; + } + } + + if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; + memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); + iCol = 0; + + while( iaStat[0].nDoc++; + eState = 1; + iCol = 0; + break; + + /* State 1. In this state we are expecting either a 1, indicating + ** that the following integer will be a column number, or the + ** start of a position list for column 0. + ** + ** The only difference between state 1 and state 2 is that if the + ** integer encountered in state 1 is not 0 or 1, then we need to + ** increment the column 0 "nDoc" count for this term. + */ + case 1: + assert( iCol==0 ); + if( v>1 ){ + pCsr->aStat[1].nDoc++; + } + eState = 2; + /* fall through */ + + case 2: + if( v==0 ){ /* 0x00. Next integer will be a docid. */ + eState = 0; + }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ + eState = 3; + }else{ /* 2 or greater. A position. */ + pCsr->aStat[iCol+1].nOcc++; + pCsr->aStat[0].nOcc++; + } + break; + + /* State 3. The integer just read is a column number. */ + default: assert( eState==3 ); + iCol = (int)v; + if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; + pCsr->aStat[iCol+1].nDoc++; + eState = 2; + break; + } + } + + pCsr->iCol = 0; + rc = SQLITE_OK; + }else{ + pCsr->isEof = 1; + } + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3auxFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + int isScan = 0; + int iLangVal = 0; /* Language id to query */ + + int iEq = -1; /* Index of term=? value in apVal */ + int iGe = -1; /* Index of term>=? value in apVal */ + int iLe = -1; /* Index of term<=? value in apVal */ + int iLangid = -1; /* Index of languageid=? value in apVal */ + int iNext = 0; + + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); + + assert( idxStr==0 ); + assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 + || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT + || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) + ); + + if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ + iEq = iNext++; + }else{ + isScan = 1; + if( idxNum & FTS4AUX_GE_CONSTRAINT ){ + iGe = iNext++; + } + if( idxNum & FTS4AUX_LE_CONSTRAINT ){ + iLe = iNext++; + } + } + if( iNextfilter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->aStat); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + + if( iEq>=0 || iGe>=0 ){ + const unsigned char *zStr = sqlite3_value_text(apVal[0]); + assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); + if( zStr ){ + pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); + pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); + if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; + } + } + + if( iLe>=0 ){ + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); + pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); + if( pCsr->zStop==0 ) return SQLITE_NOMEM; + } + + if( iLangid>=0 ){ + iLangVal = sqlite3_value_int(apVal[iLangid]); + + /* If the user specified a negative value for the languageid, use zero + ** instead. This works, as the "languageid=?" constraint will also + ** be tested by the VDBE layer. The test will always be false (since + ** this module will not return a row with a negative languageid), and + ** so the overall query will return zero rows. */ + if( iLangVal<0 ) iLangVal = 0; + } + pCsr->iLangid = iLangVal; + + rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + + if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3auxColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3auxCursor *p = (Fts3auxCursor *)pCursor; + + assert( p->isEof==0 ); + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + + case 1: /* col */ + if( p->iCol ){ + sqlite3_result_int(pCtx, p->iCol-1); + }else{ + sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); + } + break; + + case 2: /* documents */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); + break; + + case 3: /* occurrences */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); + break; + + default: /* languageid */ + assert( iCol==4 ); + sqlite3_result_int(pCtx, p->iLangid); + break; + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3auxRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3aux module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ + static const sqlite3_module fts3aux_module = { + 0, /* iVersion */ + fts3auxConnectMethod, /* xCreate */ + fts3auxConnectMethod, /* xConnect */ + fts3auxBestIndexMethod, /* xBestIndex */ + fts3auxDisconnectMethod, /* xDisconnect */ + fts3auxDisconnectMethod, /* xDestroy */ + fts3auxOpenMethod, /* xOpen */ + fts3auxCloseMethod, /* xClose */ + fts3auxFilterMethod, /* xFilter */ + fts3auxNextMethod, /* xNext */ + fts3auxEofMethod, /* xEof */ + fts3auxColumnMethod, /* xColumn */ + fts3auxRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_aux.c ********************************************/ +/************** Begin file fts3_expr.c ***************************************/ +/* +** 2008 Nov 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This module contains code that implements a parser for fts3 query strings +** (the right-hand argument to the MATCH operator). Because the supported +** syntax is relatively simple, the whole tokenizer/parser system is +** hand-coded. +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* +** By default, this module parses the legacy syntax that has been +** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined, then it uses the new syntax. The differences between +** the new and the old syntaxes are: +** +** a) The new syntax supports parenthesis. The old does not. +** +** b) The new syntax supports the AND and NOT operators. The old does not. +** +** c) The old syntax supports the "-" token qualifier. This is not +** supported by the new syntax (it is replaced by the NOT operator). +** +** d) When using the old syntax, the OR operator has a greater precedence +** than an implicit AND. When using the new, both implicity and explicit +** AND operators have a higher precedence than OR. +** +** If compiled with SQLITE_TEST defined, then this module exports the +** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable +** to zero causes the module to use the old syntax. If it is set to +** non-zero the new syntax is activated. This is so both syntaxes can +** be tested using a single build of testfixture. +** +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". +*/ + +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_fts3_enable_parentheses = 0; +#else +# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +# define sqlite3_fts3_enable_parentheses 1 +# else +# define sqlite3_fts3_enable_parentheses 0 +# endif +#endif + +/* +** Default span for NEAR operators. +*/ +#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 + +/* #include */ +/* #include */ + +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ +typedef struct ParseContext ParseContext; +struct ParseContext { + sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ + const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ + int nCol; /* Number of entries in azCol[] */ + int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ + sqlite3_context *pCtx; /* Write error message here */ + int nNest; /* Number of nested brackets */ +}; + +/* +** This function is equivalent to the standard isspace() function. +** +** The standard isspace() can be awkward to use safely, because although it +** is defined to accept an argument of type int, its behavior when passed +** an integer that falls outside of the range of the unsigned char type +** is undefined (and sometimes, "undefined" means segfault). This wrapper +** is defined to accept an argument of type char, and always returns 0 for +** any values that fall outside of the range of the unsigned char type (i.e. +** negative values). +*/ +static int fts3isspace(char c){ + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +} + +/* +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, +** zero the memory before returning a pointer to it. If unsuccessful, +** return NULL. +*/ +static void *fts3MallocZero(int nByte){ + void *pRet = sqlite3_malloc(nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr +){ + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; + } + } + } + *ppCsr = pCsr; + return rc; +} + +/* +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. +*/ +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); + +/* +** Extract the next token from buffer z (length n) using the tokenizer +** and other information (column names etc.) in pParse. Create an Fts3Expr +** structure of type FTSQUERY_PHRASE containing a phrase consisting of this +** single token and set *ppExpr to point to it. If the end of the buffer is +** reached before a token is found, set *ppExpr to zero. It is the +** responsibility of the caller to eventually deallocate the allocated +** Fts3Expr structure (if any) by passing it to sqlite3_free(). +** +** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation +** fails. +*/ +static int getNextToken( + ParseContext *pParse, /* fts3 query parse context */ + int iCol, /* Value for Fts3Phrase.iColumn */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + sqlite3_tokenizer_cursor *pCursor; + Fts3Expr *pRet = 0; + int i = 0; + + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); + if( rc==SQLITE_OK ){ + const char *zToken; + int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; + int nByte; /* total space to allocate */ + + rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + pRet = (Fts3Expr *)fts3MallocZero(nByte); + if( !pRet ){ + rc = SQLITE_NOMEM; + }else{ + pRet->eType = FTSQUERY_PHRASE; + pRet->pPhrase = (Fts3Phrase *)&pRet[1]; + pRet->pPhrase->nToken = 1; + pRet->pPhrase->iColumn = iCol; + pRet->pPhrase->aToken[0].n = nToken; + pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); + + if( iEndpPhrase->aToken[0].isPrefix = 1; + iEnd++; + } + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } + } + + } + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pModule->xClose(pCursor); + } + + *ppExpr = pRet; + return rc; +} + + +/* +** Enlarge a memory allocation. If an out-of-memory allocation occurs, +** then free the old allocation. +*/ +static void *fts3ReallocOrFree(void *pOrig, int nNew){ + void *pRet = sqlite3_realloc(pOrig, nNew); + if( !pRet ){ + sqlite3_free(pOrig); + } + return pRet; +} + +/* +** Buffer zInput, length nInput, contains the contents of a quoted string +** that appeared as part of an fts3 query expression. Neither quote character +** is included in the buffer. This function attempts to tokenize the entire +** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE +** containing the results. +** +** If successful, SQLITE_OK is returned and *ppExpr set to point at the +** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory +** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set +** to 0. +*/ +static int getNextString( + ParseContext *pParse, /* fts3 query parse context */ + const char *zInput, int nInput, /* Input string */ + Fts3Expr **ppExpr /* OUT: expression */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + Fts3Expr *p = 0; + sqlite3_tokenizer_cursor *pCursor = 0; + char *zTemp = 0; + int nTemp = 0; + + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ + rc = sqlite3Fts3OpenTokenizer( + pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); + if( rc==SQLITE_OK ){ + int ii; + for(ii=0; rc==SQLITE_OK; ii++){ + const char *zByte; + int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); + if( rc==SQLITE_OK ){ + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); + nToken = ii+1; + } + } + + pModule->xClose(pCursor); + pCursor = 0; + } + + if( rc==SQLITE_DONE ){ + int jj; + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); + p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; + p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + if( zTemp ){ + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + }else{ + assert( nTemp==0 ); + } + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } + rc = SQLITE_OK; + } + + *ppExpr = p; + return rc; +no_mem: + + if( pCursor ){ + pModule->xClose(pCursor); + } + sqlite3_free(zTemp); + sqlite3_free(p); + *ppExpr = 0; + return SQLITE_NOMEM; +} + +/* +** The output variable *ppExpr is populated with an allocated Fts3Expr +** structure, or set to 0 if the end of the input buffer is reached. +** +** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM +** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. +** If SQLITE_ERROR is returned, pContext is populated with an error message. +*/ +static int getNextNode( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + static const struct Fts3Keyword { + char *z; /* Keyword text */ + unsigned char n; /* Length of the keyword */ + unsigned char parenOnly; /* Only valid in paren mode */ + unsigned char eType; /* Keyword code */ + } aKeyword[] = { + { "OR" , 2, 0, FTSQUERY_OR }, + { "AND", 3, 1, FTSQUERY_AND }, + { "NOT", 3, 1, FTSQUERY_NOT }, + { "NEAR", 4, 0, FTSQUERY_NEAR } + }; + int ii; + int iCol; + int iColLen; + int rc; + Fts3Expr *pRet = 0; + + const char *zInput = z; + int nInput = n; + + pParse->isNot = 0; + + /* Skip over any whitespace before checking for a keyword, an open or + ** close bracket, or a quoted string. + */ + while( nInput>0 && fts3isspace(*zInput) ){ + nInput--; + zInput++; + } + if( nInput==0 ){ + return SQLITE_DONE; + } + + /* See if we are dealing with a keyword. */ + for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ + const struct Fts3Keyword *pKey = &aKeyword[ii]; + + if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ + continue; + } + + if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ + int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; + int nKey = pKey->n; + char cNext; + + /* If this is a "NEAR" keyword, check for an explicit nearness. */ + if( pKey->eType==FTSQUERY_NEAR ){ + assert( nKey==4 ); + if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ + nNear = 0; + for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ + nNear = nNear * 10 + (zInput[nKey] - '0'); + } + } + } + + /* At this point this is probably a keyword. But for that to be true, + ** the next byte must contain either whitespace, an open or close + ** parenthesis, a quote character, or EOF. + */ + cNext = zInput[nKey]; + if( fts3isspace(cNext) + || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 + ){ + pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); + if( !pRet ){ + return SQLITE_NOMEM; + } + pRet->eType = pKey->eType; + pRet->nNear = nNear; + *ppExpr = pRet; + *pnConsumed = (int)((zInput - z) + nKey); + return SQLITE_OK; + } + + /* Turns out that wasn't a keyword after all. This happens if the + ** user has supplied a token such as "ORacle". Continue. + */ + } + } + + /* See if we are dealing with a quoted phrase. If this is the case, then + ** search for the closing quote and pass the whole string to getNextString() + ** for processing. This is easy to do, as fts3 has no syntax for escaping + ** a quote character embedded in a string. + */ + if( *zInput=='"' ){ + for(ii=1; iinNest++; + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } + + /* If control flows to this point, this must be a regular token, or + ** the end of the input. Read a regular token using the sqlite3_tokenizer + ** interface. Before doing so, figure out if there is an explicit + ** column specifier for the token. + ** + ** TODO: Strangely, it is not possible to associate a column specifier + ** with a quoted phrase, only with a single token. Not sure if this was + ** an implementation artifact or an intentional decision when fts3 was + ** first implemented. Whichever it was, this module duplicates the + ** limitation. + */ + iCol = pParse->iDefaultCol; + iColLen = 0; + for(ii=0; iinCol; ii++){ + const char *zStr = pParse->azCol[ii]; + int nStr = (int)strlen(zStr); + if( nInput>nStr && zInput[nStr]==':' + && sqlite3_strnicmp(zStr, zInput, nStr)==0 + ){ + iCol = ii; + iColLen = (int)((zInput - z) + nStr + 1); + break; + } + } + rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); + *pnConsumed += iColLen; + return rc; +} + +/* +** The argument is an Fts3Expr structure for a binary operator (any type +** except an FTSQUERY_PHRASE). Return an integer value representing the +** precedence of the operator. Lower values have a higher precedence (i.e. +** group more tightly). For example, in the C language, the == operator +** groups more tightly than ||, and would therefore have a higher precedence. +** +** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined), the order of the operators in precedence from highest to +** lowest is: +** +** NEAR +** NOT +** AND (including implicit ANDs) +** OR +** +** Note that when using the old query syntax, the OR operator has a higher +** precedence than the AND operator. +*/ +static int opPrecedence(Fts3Expr *p){ + assert( p->eType!=FTSQUERY_PHRASE ); + if( sqlite3_fts3_enable_parentheses ){ + return p->eType; + }else if( p->eType==FTSQUERY_NEAR ){ + return 1; + }else if( p->eType==FTSQUERY_OR ){ + return 2; + } + assert( p->eType==FTSQUERY_AND ); + return 3; +} + +/* +** Argument ppHead contains a pointer to the current head of a query +** expression tree being parsed. pPrev is the expression node most recently +** inserted into the tree. This function adds pNew, which is always a binary +** operator node, into the expression tree based on the relative precedence +** of pNew and the existing nodes of the tree. This may result in the head +** of the tree changing, in which case *ppHead is set to the new root node. +*/ +static void insertBinaryOperator( + Fts3Expr **ppHead, /* Pointer to the root node of a tree */ + Fts3Expr *pPrev, /* Node most recently inserted into the tree */ + Fts3Expr *pNew /* New binary node to insert into expression tree */ +){ + Fts3Expr *pSplit = pPrev; + while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ + pSplit = pSplit->pParent; + } + + if( pSplit->pParent ){ + assert( pSplit->pParent->pRight==pSplit ); + pSplit->pParent->pRight = pNew; + pNew->pParent = pSplit->pParent; + }else{ + *ppHead = pNew; + } + pNew->pLeft = pSplit; + pSplit->pParent = pNew; +} + +/* +** Parse the fts3 query expression found in buffer z, length n. This function +** returns either when the end of the buffer is reached or an unmatched +** closing bracket - ')' - is encountered. +** +** If successful, SQLITE_OK is returned, *ppExpr is set to point to the +** parsed form of the expression and *pnConsumed is set to the number of +** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM +** (out of memory error) or SQLITE_ERROR (parse error) is returned. +*/ +static int fts3ExprParse( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + Fts3Expr *pRet = 0; + Fts3Expr *pPrev = 0; + Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ + int nIn = n; + const char *zIn = z; + int rc = SQLITE_OK; + int isRequirePhrase = 1; + + while( rc==SQLITE_OK ){ + Fts3Expr *p = 0; + int nByte = 0; + + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); + if( rc==SQLITE_OK ){ + if( p ){ + int isPhrase; + + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( + (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) + || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; + } + pPrev = p; + } + assert( nByte>0 ); + } + assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); + nIn -= nByte; + zIn += nByte; + } + + if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ + if( !pRet ){ + rc = SQLITE_ERROR; + }else{ + Fts3Expr *pIter = pNotBranch; + while( pIter->pLeft ){ + pIter = pIter->pLeft; + } + pIter->pLeft = pRet; + pRet->pParent = pIter; + pRet = pNotBranch; + } + } + } + *pnConsumed = n - nIn; + +exprparse_out: + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRet); + sqlite3Fts3ExprFree(pNotBranch); + pRet = 0; + } + *ppExpr = pRet; + return rc; +} + +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth<0 ){ + rc = SQLITE_TOOBIG; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Parameters z and n contain a pointer to and length of a buffer containing +** an fts3 query expression, respectively. This function attempts to parse the +** query expression and create a tree of Fts3Expr structures representing the +** parsed expression. If successful, *ppExpr is set to point to the head +** of the parsed expression tree and SQLITE_OK is returned. If an error +** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse +** error) is returned and *ppExpr is set to 0. +** +** If parameter n is a negative number, then z is assumed to point to a +** nul-terminated string and the length is determined using strlen(). +** +** The first parameter, pTokenizer, is passed the fts3 tokenizer module to +** use to normalize query tokens while parsing the expression. The azCol[] +** array, which is assumed to contain nCol entries, should contain the names +** of each column in the target fts3 table, in order from left to right. +** Column names must be nul-terminated strings. +** +** The iDefaultCol parameter should be passed the index of the table column +** that appears on the left-hand-side of the MATCH operator (the default +** column to match against for tokens for which a column name is not explicitly +** specified as part of the query string), or -1 if tokens may by default +** match any table column. +*/ +SQLITE_PRIVATE int sqlite3Fts3ExprParse( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ +){ + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(*ppExpr); + *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + *pzErr = sqlite3_mprintf( + "FTS expression tree is too large (maximum depth %d)", + SQLITE_FTS3_MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + *pzErr = sqlite3_mprintf("malformed MATCH expression: [%s]", z); + } + } + + return rc; +} + +/* +** Free a single node of an expression tree. +*/ +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } + } +} + +/**************************************************************************** +***************************************************************************** +** Everything after this point is just test code. +*/ + +#ifdef SQLITE_TEST + +/* #include */ + +/* +** Function to query the hash-table of tokenizers (see README.tokenizers). +*/ +static int queryTestTokenizer( + sqlite3 *db, + const char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +/* +** Return a pointer to a buffer containing a text representation of the +** expression passed as the first argument. The buffer is obtained from +** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_free() to release the memory. If an OOM condition is encountered, +** NULL is returned. +** +** If the second argument is not NULL, then its contents are prepended to +** the returned expression text and then freed using sqlite3_free(). +*/ +static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + zBuf = sqlite3_mprintf( + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); + for(i=0; zBuf && inToken; i++){ + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + pPhrase->aToken[i].n, pPhrase->aToken[i].z, + (pPhrase->aToken[i].isPrefix?"+":"") + ); + } + return zBuf; + } + + case FTSQUERY_NEAR: + zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); + break; + case FTSQUERY_NOT: + zBuf = sqlite3_mprintf("%zNOT ", zBuf); + break; + case FTSQUERY_AND: + zBuf = sqlite3_mprintf("%zAND ", zBuf); + break; + case FTSQUERY_OR: + zBuf = sqlite3_mprintf("%zOR ", zBuf); + break; + } + + if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); + if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); + + if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); + + return zBuf; +} + +/* +** This is the implementation of a scalar SQL function used to test the +** expression parser. It should be called as follows: +** +** fts3_exprtest(, , , ...); +** +** The first argument, , is the name of the fts3 tokenizer used +** to parse the query expression (see README.tokenizers). The second argument +** is the query expression to parse. Each subsequent argument is the name +** of a column of the fts3 table that the query expression may refer to. +** For example: +** +** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); +*/ +static void fts3ExprTest( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_tokenizer_module const *pModule = 0; + sqlite3_tokenizer *pTokenizer = 0; + int rc; + char **azCol = 0; + const char *zExpr; + int nExpr; + int nCol; + int ii; + Fts3Expr *pExpr; + char *zBuf = 0; + sqlite3 *db = sqlite3_context_db_handle(context); + + if( argc<3 ){ + sqlite3_result_error(context, + "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 + ); + return; + } + + rc = queryTestTokenizer(db, + (const char *)sqlite3_value_text(argv[0]), &pModule); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + }else if( !pModule ){ + sqlite3_result_error(context, "No such tokenizer module", -1); + goto exprtest_out; + } + + rc = pModule->xCreate(0, 0, &pTokenizer); + assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + pTokenizer->pModule = pModule; + + zExpr = (const char *)sqlite3_value_text(argv[1]); + nExpr = sqlite3_value_bytes(argv[1]); + nCol = argc-2; + azCol = (char **)sqlite3_malloc(nCol*sizeof(char *)); + if( !azCol ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + for(ii=0; iixDestroy(pTokenizer); + } + sqlite3_free(azCol); +} + +/* +** Register the query expression parser test function fts3_exprtest() +** with database connection db. +*/ +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ + int rc = sqlite3_create_function( + db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 + ); + } + return rc; +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_expr.c *******************************************/ +/************** Begin file fts3_hash.c ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of generic hash-tables used in SQLite. +** We've modified it slightly to serve as a standalone hash table +** implementation for the full-text indexing module. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ + + +/* +** Malloc and Free functions +*/ +static void *fts3HashMalloc(int n){ + void *p = sqlite3_malloc(n); + if( p ){ + memset(p, 0, n); + } + return p; +} +static void fts3HashFree(void *p){ + sqlite3_free(p); +} + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. +** +** "pNew" is a pointer to the hash table that is to be initialized. +** keyClass is one of the constants +** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass +** determines what kind of key the hash table will use. "copyKey" is +** true if the hash table should make its own private copy of keys and +** false if it should just use the supplied pointer. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ + assert( pNew!=0 ); + assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); + pNew->keyClass = keyClass; + pNew->copyKey = copyKey; + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} + +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ + Fts3HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + fts3HashFree(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + Fts3HashElem *next_elem = elem->next; + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree(elem); + elem = next_elem; + } + pH->count = 0; +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_STRING +*/ +static int fts3StrHash(const void *pKey, int nKey){ + const char *z = (const char *)pKey; + unsigned h = 0; + if( nKey<=0 ) nKey = (int) strlen(z); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ *z++; + nKey--; + } + return (int)(h & 0x7fffffff); +} +static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return strncmp((const char*)pKey1,(const char*)pKey2,n1); +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_BINARY +*/ +static int fts3BinHash(const void *pKey, int nKey){ + int h = 0; + const char *z = (const char *)pKey; + while( nKey-- > 0 ){ + h = (h<<3) ^ h ^ *(z++); + } + return h & 0x7fffffff; +} +static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return memcmp(pKey1,pKey2,n1); +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** The C syntax in this function definition may be unfamilar to some +** programmers, so we provide the following additional explanation: +** +** The name of the function is "ftsHashFunction". The function takes a +** single parameter "keyClass". The return value of ftsHashFunction() +** is a pointer to another function. Specifically, the return value +** of ftsHashFunction() is a pointer to a function that takes two parameters +** with types "const void*" and "int" and returns an "int". +*/ +static int (*ftsHashFunction(int keyClass))(const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrHash; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinHash; + } +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** For help in interpreted the obscure C code in the function definition, +** see the header comment on the previous function. +*/ +static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrCompare; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinCompare; + } +} + +/* Link an element into the hash table +*/ +static void fts3HashInsertElement( + Fts3Hash *pH, /* The complete hash table */ + struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ + Fts3HashElem *pNew /* The element to be inserted */ +){ + Fts3HashElem *pHead; /* First element already in pEntry */ + pHead = pEntry->chain; + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; + } + pEntry->count++; + pEntry->chain = pNew; +} + + +/* Resize the hash table so that it cantains "new_size" buckets. +** "new_size" must be a power of 2. The hash table might fail +** to resize if sqliteMalloc() fails. +** +** Return non-zero if a memory allocation error occurs. +*/ +static int fts3Rehash(Fts3Hash *pH, int new_size){ + struct _fts3ht *new_ht; /* The new hash table */ + Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( (new_size & (new_size-1))==0 ); + new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); + if( new_ht==0 ) return 1; + fts3HashFree(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size; + xHash = ftsHashFunction(pH->keyClass); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); + next_elem = elem->next; + fts3HashInsertElement(pH, &new_ht[h], elem); + } + return 0; +} + +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. The hash for this key has +** already been computed and is passed as the 4th parameter. +*/ +static Fts3HashElem *fts3FindElementByHash( + const Fts3Hash *pH, /* The pH to be searched */ + const void *pKey, /* The key we are searching for */ + int nKey, + int h /* The hash for this key. */ +){ + Fts3HashElem *elem; /* Used to loop thru the element list */ + int count; /* Number of elements left to test */ + int (*xCompare)(const void*,int,const void*,int); /* comparison function */ + + if( pH->ht ){ + struct _fts3ht *pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + xCompare = ftsCompareFunction(pH->keyClass); + while( count-- && elem ){ + if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ + return elem; + } + elem = elem->next; + } + } + return 0; +} + +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void fts3RemoveElementByHash( + Fts3Hash *pH, /* The pH containing "elem" */ + Fts3HashElem* elem, /* The element to be removed from the pH */ + int h /* Hash value for the element */ +){ + struct _fts3ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; + }else{ + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + pEntry->count--; + if( pEntry->count<=0 ){ + pEntry->chain = 0; + } + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree( elem ); + pH->count--; + if( pH->count<=0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + fts3HashClear(pH); + } +} + +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( + const Fts3Hash *pH, + const void *pKey, + int nKey +){ + int h; /* A hash on key */ + int (*xHash)(const void*,int); /* The hash function */ + + if( pH==0 || pH->ht==0 ) return 0; + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + h = (*xHash)(pKey,nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); +} + +/* +** Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ + Fts3HashElem *pElem; /* The element that matches key (if any) */ + + pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); + return pElem ? pElem->data : 0; +} + +/* Insert an element into the hash table pH. The key is pKey,nKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created. A copy of the key is made if the copyKey +** flag is set. NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. +*/ +SQLITE_PRIVATE void *sqlite3Fts3HashInsert( + Fts3Hash *pH, /* The hash table to insert into */ + const void *pKey, /* The key */ + int nKey, /* Number of bytes in the key */ + void *data /* The data */ +){ + int hraw; /* Raw hash value of the key */ + int h; /* the hash of the key modulo hash table size */ + Fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *new_elem; /* New element added to the pH */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( pH!=0 ); + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + hraw = (*xHash)(pKey, nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + elem = fts3FindElementByHash(pH,pKey,nKey,h); + if( elem ){ + void *old_data = elem->data; + if( data==0 ){ + fts3RemoveElementByHash(pH,elem,h); + }else{ + elem->data = data; + } + return old_data; + } + if( data==0 ) return 0; + if( (pH->htsize==0 && fts3Rehash(pH,8)) + || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) + ){ + pH->count = 0; + return data; + } + assert( pH->htsize>0 ); + new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); + if( new_elem==0 ) return data; + if( pH->copyKey && pKey!=0 ){ + new_elem->pKey = fts3HashMalloc( nKey ); + if( new_elem->pKey==0 ){ + fts3HashFree(new_elem); + return data; + } + memcpy((void*)new_elem->pKey, pKey, nKey); + }else{ + new_elem->pKey = (void*)pKey; + } + new_elem->nKey = nKey; + pH->count++; + assert( pH->htsize>0 ); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + fts3HashInsertElement(pH, &pH->ht[h], new_elem); + new_elem->data = data; + return 0; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_hash.c *******************************************/ +/************** Begin file fts3_porter.c *************************************/ +/* +** 2006 September 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the full-text-search tokenizer that implements +** a Porter stemmer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + + +/* +** Class derived from sqlite3_tokenizer +*/ +typedef struct porter_tokenizer { + sqlite3_tokenizer base; /* Base class */ +} porter_tokenizer; + +/* +** Class derived from sqlite3_tokenizer_cursor +*/ +typedef struct porter_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ +} porter_tokenizer_cursor; + + +/* +** Create a new tokenizer instance. +*/ +static int porterCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + porter_tokenizer *t; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int porterDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is zInput[0..nInput-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int porterOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + porter_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->zInput = zInput; + if( zInput==0 ){ + c->nInput = 0; + }else if( nInput<0 ){ + c->nInput = (int)strlen(zInput); + }else{ + c->nInput = nInput; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->zToken = NULL; /* no space allocated, yet. */ + c->nAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** porterOpen() above. +*/ +static int porterClose(sqlite3_tokenizer_cursor *pCursor){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + sqlite3_free(c->zToken); + sqlite3_free(c); + return SQLITE_OK; +} +/* +** Vowel or consonant +*/ +static const char cType[] = { + 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 2, 1 +}; + +/* +** isConsonant() and isVowel() determine if their first character in +** the string they point to is a consonant or a vowel, according +** to Porter ruls. +** +** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. +** 'Y' is a consonant unless it follows another consonant, +** in which case it is a vowel. +** +** In these routine, the letters are in reverse order. So the 'y' rule +** is that 'y' is a consonant unless it is followed by another +** consonent. +*/ +static int isVowel(const char*); +static int isConsonant(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return j; + return z[1]==0 || isVowel(z + 1); +} +static int isVowel(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return 1-j; + return isConsonant(z + 1); +} + +/* +** Let any sequence of one or more vowels be represented by V and let +** C be sequence of one or more consonants. Then every word can be +** represented as: +** +** [C] (VC){m} [V] +** +** In prose: A word is an optional consonant followed by zero or +** vowel-consonant pairs followed by an optional vowel. "m" is the +** number of vowel consonant pairs. This routine computes the value +** of m for the first i bytes of a word. +** +** Return true if the m-value for z is 1 or more. In other words, +** return true if z contains at least one vowel that is followed +** by a consonant. +** +** In this routine z[] is in reverse order. So we are really looking +** for an instance of of a consonant followed by a vowel. +*/ +static int m_gt_0(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* Like mgt0 above except we are looking for a value of m which is +** exactly 1 +*/ +static int m_eq_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 1; + while( isConsonant(z) ){ z++; } + return *z==0; +} + +/* Like mgt0 above except we are looking for a value of m>1 instead +** or m>0 +*/ +static int m_gt_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if there is a vowel anywhere within z[0..n-1] +*/ +static int hasVowel(const char *z){ + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if the word ends in a double consonant. +** +** The text is reversed here. So we are really looking at +** the first two characters of z[]. +*/ +static int doubleConsonant(const char *z){ + return isConsonant(z) && z[0]==z[1]; +} + +/* +** Return TRUE if the word ends with three letters which +** are consonant-vowel-consonent and where the final consonant +** is not 'w', 'x', or 'y'. +** +** The word is reversed here. So we are really checking the +** first three letters and the first one cannot be in [wxy]. +*/ +static int star_oh(const char *z){ + return + isConsonant(z) && + z[0]!='w' && z[0]!='x' && z[0]!='y' && + isVowel(z+1) && + isConsonant(z+2); +} + +/* +** If the word ends with zFrom and xCond() is true for the stem +** of the word that preceeds the zFrom ending, then change the +** ending to zTo. +** +** The input word *pz and zFrom are both in reverse order. zTo +** is in normal order. +** +** Return TRUE if zFrom matches. Return FALSE if zFrom does not +** match. Not that TRUE is returned even if xCond() fails and +** no substitution occurs. +*/ +static int stem( + char **pz, /* The word being stemmed (Reversed) */ + const char *zFrom, /* If the ending matches this... (Reversed) */ + const char *zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char*) /* Condition that must be true */ +){ + char *z = *pz; + while( *zFrom && *zFrom==*z ){ z++; zFrom++; } + if( *zFrom!=0 ) return 0; + if( xCond && !xCond(z) ) return 1; + while( *zTo ){ + *(--z) = *(zTo++); + } + *pz = z; + return 1; +} + +/* +** This is the fallback stemmer used when the porter stemmer is +** inappropriate. The input word is copied into the output with +** US-ASCII case folding. If the input word is too long (more +** than 20 bytes if it contains no digits or more than 6 bytes if +** it contains digits) then word is truncated to 20 or 6 bytes +** by taking 10 or 3 bytes from the beginning and end. +*/ +static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ + int i, mx, j; + int hasDigit = 0; + for(i=0; i='A' && c<='Z' ){ + zOut[i] = c - 'A' + 'a'; + }else{ + if( c>='0' && c<='9' ) hasDigit = 1; + zOut[i] = c; + } + } + mx = hasDigit ? 3 : 10; + if( nIn>mx*2 ){ + for(j=mx, i=nIn-mx; i=(int)sizeof(zReverse)-7 ){ + /* The word is too big or too small for the porter stemmer. + ** Fallback to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ + zReverse[j] = c + 'a' - 'A'; + }else if( c>='a' && c<='z' ){ + zReverse[j] = c; + }else{ + /* The use of a character not in [a-zA-Z] means that we fallback + ** to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + } + memset(&zReverse[sizeof(zReverse)-5], 0, 5); + z = &zReverse[j+1]; + + + /* Step 1a */ + if( z[0]=='s' ){ + if( + !stem(&z, "sess", "ss", 0) && + !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0) + ){ + z++; + } + } + + /* Step 1b */ + z2 = z; + if( stem(&z, "dee", "ee", m_gt_0) ){ + /* Do nothing. The work was all in the test */ + }else if( + (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) + && z!=z2 + ){ + if( stem(&z, "ta", "ate", 0) || + stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0) ){ + /* Do nothing. The work was all in the test */ + }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ + z++; + }else if( m_eq_1(z) && star_oh(z) ){ + *(--z) = 'e'; + } + } + + /* Step 1c */ + if( z[0]=='y' && hasVowel(z+1) ){ + z[0] = 'i'; + } + + /* Step 2 */ + switch( z[1] ){ + case 'a': + if( !stem(&z, "lanoita", "ate", m_gt_0) ){ + stem(&z, "lanoit", "tion", m_gt_0); + } + break; + case 'c': + if( !stem(&z, "icne", "ence", m_gt_0) ){ + stem(&z, "icna", "ance", m_gt_0); + } + break; + case 'e': + stem(&z, "rezi", "ize", m_gt_0); + break; + case 'g': + stem(&z, "igol", "log", m_gt_0); + break; + case 'l': + if( !stem(&z, "ilb", "ble", m_gt_0) + && !stem(&z, "illa", "al", m_gt_0) + && !stem(&z, "iltne", "ent", m_gt_0) + && !stem(&z, "ile", "e", m_gt_0) + ){ + stem(&z, "ilsuo", "ous", m_gt_0); + } + break; + case 'o': + if( !stem(&z, "noitazi", "ize", m_gt_0) + && !stem(&z, "noita", "ate", m_gt_0) + ){ + stem(&z, "rota", "ate", m_gt_0); + } + break; + case 's': + if( !stem(&z, "msila", "al", m_gt_0) + && !stem(&z, "ssenevi", "ive", m_gt_0) + && !stem(&z, "ssenluf", "ful", m_gt_0) + ){ + stem(&z, "ssensuo", "ous", m_gt_0); + } + break; + case 't': + if( !stem(&z, "itila", "al", m_gt_0) + && !stem(&z, "itivi", "ive", m_gt_0) + ){ + stem(&z, "itilib", "ble", m_gt_0); + } + break; + } + + /* Step 3 */ + switch( z[0] ){ + case 'e': + if( !stem(&z, "etaci", "ic", m_gt_0) + && !stem(&z, "evita", "", m_gt_0) + ){ + stem(&z, "ezila", "al", m_gt_0); + } + break; + case 'i': + stem(&z, "itici", "ic", m_gt_0); + break; + case 'l': + if( !stem(&z, "laci", "ic", m_gt_0) ){ + stem(&z, "luf", "", m_gt_0); + } + break; + case 's': + stem(&z, "ssen", "", m_gt_0); + break; + } + + /* Step 4 */ + switch( z[1] ){ + case 'a': + if( z[0]=='l' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'c': + if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'e': + if( z[0]=='r' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'i': + if( z[0]=='c' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'l': + if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'n': + if( z[0]=='t' ){ + if( z[2]=='a' ){ + if( m_gt_1(z+3) ){ + z += 3; + } + }else if( z[2]=='e' ){ + if( !stem(&z, "tneme", "", m_gt_1) + && !stem(&z, "tnem", "", m_gt_1) + ){ + stem(&z, "tne", "", m_gt_1); + } + } + } + break; + case 'o': + if( z[0]=='u' ){ + if( m_gt_1(z+2) ){ + z += 2; + } + }else if( z[3]=='s' || z[3]=='t' ){ + stem(&z, "noi", "", m_gt_1); + } + break; + case 's': + if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 't': + if( !stem(&z, "eta", "", m_gt_1) ){ + stem(&z, "iti", "", m_gt_1); + } + break; + case 'u': + if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 'v': + case 'z': + if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + } + + /* Step 5a */ + if( z[0]=='e' ){ + if( m_gt_1(z+1) ){ + z++; + }else if( m_eq_1(z+1) && !star_oh(z+1) ){ + z++; + } + } + + /* Step 5b */ + if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ + z++; + } + + /* z[] is now the stemmed word in reverse order. Flip it back + ** around into forward order and return. + */ + *pnOut = i = (int)strlen(z); + zOut[i] = 0; + while( *z ){ + zOut[--i] = *(z++); + } +} + +/* +** Characters that can be part of a token. We assume any character +** whose value is greater than 0x80 (any UTF character) can be +** part of a token. In other words, delimiters all must have +** values of 0x7f or lower. +*/ +static const char porterIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; +#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to porterOpen(). +*/ +static int porterNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ + const char **pzToken, /* OUT: *pzToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + const char *z = c->zInput; + + while( c->iOffsetnInput ){ + int iStartOffset, ch; + + /* Scan past delimiter characters */ + while( c->iOffsetnInput && isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnInput && !isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int n = c->iOffset-iStartOffset; + if( n>c->nAllocated ){ + char *pNew; + c->nAllocated = n+20; + pNew = sqlite3_realloc(c->zToken, c->nAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->zToken = pNew; + } + porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + *pzToken = c->zToken; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the porter-stemmer tokenizer +*/ +static const sqlite3_tokenizer_module porterTokenizerModule = { + 0, + porterCreate, + porterDestroy, + porterOpen, + porterClose, + porterNext, + 0 +}; + +/* +** Allocate a new porter tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &porterTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_porter.c *****************************************/ +/************** Begin file fts3_tokenizer.c **********************************/ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is part of an SQLite module implementing full-text search. +** This particular file implements the generic tokenizer interface. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +/* +** Implementation of the SQL scalar function for accessing the underlying +** hash table. This function may be called as follows: +** +** SELECT (); +** SELECT (, ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). +** +** If the argument is specified, it must be a blob value +** containing a pointer to be stored as the hash data corresponding +** to the string . If is not specified, then +** the string must already exist in the has table. Otherwise, +** an error is returned. +** +** Whether or not the argument is specified, the value returned +** is a blob containing the pointer stored as the hash data corresponding +** to string (after the hash-table is updated, if applicable). +*/ +static void scalarFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + void *pPtr = 0; + const unsigned char *zName; + int nName; + + assert( argc==1 || argc==2 ); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + + zName = sqlite3_value_text(argv[0]); + nName = sqlite3_value_bytes(argv[0])+1; + + if( argc==2 ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + return; + } + }else{ + pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + if( !pPtr ){ + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + } + + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); +} + +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ + static const char isFtsIdChar[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + }; + return (c&0x80 || isFtsIdChar[(int)(c)]); +} + +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ + const char *z1; + const char *z2 = 0; + + /* Find the start of the next token. */ + z1 = zStr; + while( z2==0 ){ + char c = *z1; + switch( c ){ + case '\0': return 0; /* No more tokens here */ + case '\'': + case '"': + case '`': { + z2 = z1; + while( *++z2 && (*z2!=c || *++z2==c) ); + break; + } + case '[': + z2 = &z1[1]; + while( *z2 && z2[0]!=']' ) z2++; + if( *z2 ) z2++; + break; + + default: + if( sqlite3Fts3IsIdChar(*z1) ){ + z2 = &z1[1]; + while( sqlite3Fts3IsIdChar(*z2) ) z2++; + }else{ + z1++; + } + } + } + + *pn = (int)(z2-z1); + return z1; +} + +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( + Fts3Hash *pHash, /* Tokenizer hash table */ + const char *zArg, /* Tokenizer name */ + sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ + char **pzErr /* OUT: Set to malloced error message */ +){ + int rc; + char *z = (char *)zArg; + int n = 0; + char *zCopy; + char *zEnd; /* Pointer to nul-term of zCopy */ + sqlite3_tokenizer_module *m; + + zCopy = sqlite3_mprintf("%s", zArg); + if( !zCopy ) return SQLITE_NOMEM; + zEnd = &zCopy[strlen(zCopy)]; + + z = (char *)sqlite3Fts3NextToken(zCopy, &n); + z[n] = '\0'; + sqlite3Fts3Dequote(z); + + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); + if( !m ){ + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); + rc = SQLITE_ERROR; + }else{ + char const **aArg = 0; + int iArg = 0; + z = &z[n+1]; + while( zxCreate(iArg, aArg, ppTok); + assert( rc!=SQLITE_OK || *ppTok ); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("unknown tokenizer"); + }else{ + (*ppTok)->pModule = m; + } + sqlite3_free((void *)aArg); + } + + sqlite3_free(zCopy); + return rc; +} + + +#ifdef SQLITE_TEST + +#include +/* #include */ + +/* +** Implementation of a special SQL scalar function for testing tokenizers +** designed to be used in concert with the Tcl testing framework. This +** function must be called with two or more arguments: +** +** SELECT (, ..., ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') +** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). +** +** The return value is a string that may be interpreted as a Tcl +** list. For each token in the , three elements are +** added to the returned list. The first is the token position, the +** second is the token text (folded, stemmed, etc.) and the third is the +** substring of associated with the token. For example, +** using the built-in "simple" tokenizer: +** +** SELECT fts_tokenizer_test('simple', 'I don't see how'); +** +** will return the string: +** +** "{0 i I 1 dont don't 2 see see 3 how how}" +** +*/ +static void testFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + sqlite3_tokenizer_module *p; + sqlite3_tokenizer *pTokenizer = 0; + sqlite3_tokenizer_cursor *pCsr = 0; + + const char *zErr = 0; + + const char *zName; + int nName; + const char *zInput; + int nInput; + + const char *azArg[64]; + + const char *zToken; + int nToken = 0; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int i; + + Tcl_Obj *pRet; + + if( argc<2 ){ + sqlite3_result_error(context, "insufficient arguments", -1); + return; + } + + nName = sqlite3_value_bytes(argv[0]); + zName = (const char *)sqlite3_value_text(argv[0]); + nInput = sqlite3_value_bytes(argv[argc-1]); + zInput = (const char *)sqlite3_value_text(argv[argc-1]); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + + if( !p ){ + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + for(i=1; ixCreate(argc-2, azArg, &pTokenizer) ){ + zErr = "error in xCreate()"; + goto finish; + } + pTokenizer->pModule = p; + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ + zErr = "error in xOpen()"; + goto finish; + } + + while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ + Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + zToken = &zInput[iStart]; + nToken = iEnd-iStart; + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + } + + if( SQLITE_OK!=p->xClose(pCsr) ){ + zErr = "error in xClose()"; + goto finish; + } + if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ + zErr = "error in xDestroy()"; + goto finish; + } + +finish: + if( zErr ){ + sqlite3_result_error(context, zErr, -1); + }else{ + sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); + } + Tcl_DecrRefCount(pRet); +} + +static +int registerTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module *p +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); + sqlite3_step(pStmt); + + return sqlite3_finalize(pStmt); +} + +static +int queryTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); + +/* +** Implementation of the scalar function fts3_tokenizer_internal_test(). +** This function is used for testing only, it is not included in the +** build unless SQLITE_TEST is defined. +** +** The purpose of this is to test that the fts3_tokenizer() function +** can be used as designed by the C-code in the queryTokenizer and +** registerTokenizer() functions above. These two functions are repeated +** in the README.tokenizer file as an example, so it is important to +** test them. +** +** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar +** function with no arguments. An assert() will fail if a problem is +** detected. i.e.: +** +** SELECT fts3_tokenizer_internal_test(); +** +*/ +static void intTestFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int rc; + const sqlite3_tokenizer_module *p1; + const sqlite3_tokenizer_module *p2; + sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + /* Test the query function */ + sqlite3Fts3SimpleTokenizerModule(&p1); + rc = queryTokenizer(db, "simple", &p2); + assert( rc==SQLITE_OK ); + assert( p1==p2 ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_ERROR ); + assert( p2==0 ); + assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); + + /* Test the storage function */ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); + + sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); +} + +#endif + +/* +** Set up SQL objects in database db used to access the contents of +** the hash table pointed to by argument pHash. The hash table must +** been initialized to use string keys, and to take a private copy +** of the key when a value is inserted. i.e. by a call similar to: +** +** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); +** +** This function adds a scalar function (see header comment above +** scalarFunc() in this file for details) and, if ENABLE_TABLE is +** defined at compilation time, a temporary virtual table (see header +** comment above struct HashTableVtab) to the database schema. Both +** provide read/write access to the contents of *pHash. +** +** The third argument to this function, zName, is used as the name +** of both the scalar and, if created, the virtual table. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitHashTable( + sqlite3 *db, + Fts3Hash *pHash, + const char *zName +){ + int rc = SQLITE_OK; + void *p = (void *)pHash; + const int any = SQLITE_ANY; + +#ifdef SQLITE_TEST + char *zTest = 0; + char *zTest2 = 0; + void *pdb = (void *)db; + zTest = sqlite3_mprintf("%s_test", zName); + zTest2 = sqlite3_mprintf("%s_internal_test", zName); + if( !zTest || !zTest2 ){ + rc = SQLITE_NOMEM; + } +#endif + + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); + } +#ifdef SQLITE_TEST + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); + } +#endif + +#ifdef SQLITE_TEST + sqlite3_free(zTest); + sqlite3_free(zTest2); +#endif + + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenizer.c **************************************/ +/************** Begin file fts3_tokenizer1.c *********************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "simple" full-text-search tokenizer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + + +typedef struct simple_tokenizer { + sqlite3_tokenizer base; + char delim[128]; /* flag ASCII delimiters */ +} simple_tokenizer; + +typedef struct simple_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *pInput; /* input we are tokenizing */ + int nBytes; /* size of the input */ + int iOffset; /* current position in pInput */ + int iToken; /* index of next token to be returned */ + char *pToken; /* storage for current token */ + int nTokenAllocated; /* space allocated to zToken buffer */ +} simple_tokenizer_cursor; + + +static int simpleDelim(simple_tokenizer *t, unsigned char c){ + return c<0x80 && t->delim[c]; +} +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} + +/* +** Create a new tokenizer instance. +*/ +static int simpleCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + simple_tokenizer *t; + + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + + /* TODO(shess) Delimiters need to remain the same from run to run, + ** else we need to reindex. One solution would be a meta-table to + ** track such information in the database, then we'd only want this + ** information on the initial create. + */ + if( argc>1 ){ + int i, n = (int)strlen(argv[1]); + for(i=0; i=0x80 ){ + sqlite3_free(t); + return SQLITE_ERROR; + } + t->delim[ch] = 1; + } + } else { + /* Mark non-alphanumeric ASCII characters as delimiters */ + int i; + for(i=1; i<0x80; i++){ + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; + } + } + + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int simpleOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + simple_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->pInput = pInput; + if( pInput==0 ){ + c->nBytes = 0; + }else if( nBytes<0 ){ + c->nBytes = (int)strlen(pInput); + }else{ + c->nBytes = nBytes; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->pToken = NULL; /* no space allocated, yet. */ + c->nTokenAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + sqlite3_free(c->pToken); + sqlite3_free(c); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int simpleNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; + unsigned char *p = (unsigned char *)c->pInput; + + while( c->iOffsetnBytes ){ + int iStartOffset; + + /* Scan past delimiter characters */ + while( c->iOffsetnBytes && simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnBytes && !simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int i, n = c->iOffset-iStartOffset; + if( n>c->nTokenAllocated ){ + char *pNew; + c->nTokenAllocated = n+20; + pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->pToken = pNew; + } + for(i=0; ipToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); + } + *ppToken = c->pToken; + *pnBytes = n; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module simpleTokenizerModule = { + 0, + simpleCreate, + simpleDestroy, + simpleOpen, + simpleClose, + simpleNext, + 0, +}; + +/* +** Allocate a new simple tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &simpleTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenizer1.c *************************************/ +/************** Begin file fts3_tokenize_vtab.c ******************************/ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + Fts3Hash *pHash, + const char *zName, + const sqlite3_tokenizer_module **pp, + char **pzErr +){ + sqlite3_tokenizer_module *p; + int nName = (int)strlen(zName); + + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + return SQLITE_ERROR; + } + + *pp = p; + return SQLITE_OK; +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + UNUSED_PARAMETER(pVTab); + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenize_vtab.c **********************************/ +/************** Begin file fts3_write.c **************************************/ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is part of the SQLite FTS3 extension module. Specifically, +** this file contains code to insert, update and delete rows from FTS3 +** tables. It also contains code to merge FTS3 b-tree segments. Some +** of the sub-routines used to merge segments are also used by the query +** code in fts3.c. +*/ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ + + +#define FTS_MAX_APPENDABLE_HEIGHT 16 + +/* +** When full-text index nodes are loaded from disk, the buffer that they +** are loaded into has the following number of bytes of padding at the end +** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer +** of 920 bytes is allocated for it. +** +** This means that if we have a pointer into a buffer containing node data, +** it is always safe to read up to two varints from it without risking an +** overread, even if the node data is corrupted. +*/ +#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) + +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + +/* +** The two values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + +typedef struct PendingList PendingList; +typedef struct SegmentNode SegmentNode; +typedef struct SegmentWriter SegmentWriter; + +/* +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. +*/ +struct PendingList { + int nData; + char *aData; + int nSpace; + sqlite3_int64 iLastDocid; + sqlite3_int64 iLastCol; + sqlite3_int64 iLastPos; +}; + + +/* +** Each cursor has a (possibly empty) linked list of the following objects. +*/ +struct Fts3DeferredToken { + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ + int iCol; /* Column token must occur in */ + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ + PendingList *pList; /* Doclist is assembled here */ +}; + +/* +** An instance of this structure is used to iterate through the terms on +** a contiguous set of segment b-tree leaf nodes. Although the details of +** this structure are only manipulated by code in this file, opaque handles +** of type Fts3SegReader* are also used by code in fts3.c to iterate through +** terms when querying the full-text index. See functions: +** +** sqlite3Fts3SegReaderNew() +** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderIterate() +** +** Methods used to manipulate Fts3SegReader structures: +** +** fts3SegReaderNext() +** fts3SegReaderFirstDocid() +** fts3SegReaderNextDocid() +*/ +struct Fts3SegReader { + int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ + u8 bLookup; /* True for a lookup only */ + u8 rootOnly; /* True for a root-only reader */ + + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + + char *aNode; /* Pointer to node data (or NULL) */ + int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + + Fts3HashElem **ppNextElem; + + /* Variables set by fts3SegReaderNext(). These may be read directly + ** by the caller. They are valid from the time SegmentReaderNew() returns + ** until SegmentReaderNext() returns something other than SQLITE_OK + ** (i.e. SQLITE_DONE). + */ + int nTerm; /* Number of bytes in current term */ + char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ + char *aDoclist; /* Pointer to doclist of current entry */ + int nDoclist; /* Size of doclist in current entry */ + + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ + char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ + sqlite3_int64 iDocid; +}; + +#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) +#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) + +/* +** An instance of this structure is used to create a segment b-tree in the +** database. The internal details of this type are only accessed by the +** following functions: +** +** fts3SegWriterAdd() +** fts3SegWriterFlush() +** fts3SegWriterFree() +*/ +struct SegmentWriter { + SegmentNode *pTree; /* Pointer to interior tree structure */ + sqlite3_int64 iFirst; /* First slot in %_segments written */ + sqlite3_int64 iFree; /* Next free slot in %_segments */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nSize; /* Size of allocation at aData */ + int nData; /* Bytes of data in aData */ + char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ +}; + +/* +** Type SegmentNode is used by the following three functions to create +** the interior part of the segment b+-tree structures (everything except +** the leaf nodes). These functions and type are only ever used by code +** within the fts3SegWriterXXX() family of functions described above. +** +** fts3NodeAddTerm() +** fts3NodeWrite() +** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. +*/ +struct SegmentNode { + SegmentNode *pParent; /* Parent node (or NULL for root node) */ + SegmentNode *pRight; /* Pointer to right-sibling */ + SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ + int nEntry; /* Number of terms written to node so far */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nData; /* Bytes of valid data so far */ + char *aData; /* Node data */ +}; + +/* +** Valid values for the second argument to fts3SqlStmt(). +*/ +#define SQL_DELETE_CONTENT 0 +#define SQL_IS_EMPTY 1 +#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_SEGMENTS 3 +#define SQL_DELETE_ALL_SEGDIR 4 +#define SQL_DELETE_ALL_DOCSIZE 5 +#define SQL_DELETE_ALL_STAT 6 +#define SQL_SELECT_CONTENT_BY_ROWID 7 +#define SQL_NEXT_SEGMENT_INDEX 8 +#define SQL_INSERT_SEGMENTS 9 +#define SQL_NEXT_SEGMENTS_ID 10 +#define SQL_INSERT_SEGDIR 11 +#define SQL_SELECT_LEVEL 12 +#define SQL_SELECT_LEVEL_RANGE 13 +#define SQL_SELECT_LEVEL_COUNT 14 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 +#define SQL_DELETE_SEGMENTS_RANGE 17 +#define SQL_CONTENT_INSERT 18 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 + +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 +#define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 + +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + +/* +** This function is used to obtain an SQLite prepared statement handle +** for the statement identified by the second argument. If successful, +** *pp is set to the requested statement handle and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned and *pp is set to 0. +** +** If argument apVal is not NULL, then it must point to an array with +** at least as many entries as the requested statement has bound +** parameters. The values are bound to the statements parameters before +** returning. +*/ +static int fts3SqlStmt( + Fts3Table *p, /* Virtual table handle */ + int eStmt, /* One of the SQL_XXX constants above */ + sqlite3_stmt **pp, /* OUT: Statement handle */ + sqlite3_value **apVal /* Values to bind to statement */ +){ + const char *azSql[] = { +/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", +/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", +/* 2 */ "DELETE FROM %Q.'%q_content'", +/* 3 */ "DELETE FROM %Q.'%q_segments'", +/* 4 */ "DELETE FROM %Q.'%q_segdir'", +/* 5 */ "DELETE FROM %Q.'%q_docsize'", +/* 6 */ "DELETE FROM %Q.'%q_stat'", +/* 7 */ "SELECT %s WHERE rowid=?", +/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", + + /* Return segments in order from oldest to newest.*/ +/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", +/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", + +/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + +/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'", + +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" + " ORDER BY (level %% 1024) ASC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + + }; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt; + + assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); + assert( eStmt=0 ); + + pStmt = p->aStmt[eStmt]; + if( !pStmt ){ + char *zSql; + if( eStmt==SQL_CONTENT_INSERT ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); + }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); + }else{ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + p->aStmt[eStmt] = pStmt; + } + } + if( apVal ){ + int i; + int nParam = sqlite3_bind_parameter_count(pStmt); + for(i=0; rc==SQLITE_OK && inPendingData==0 ){ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + + return rc; +} + +/* +** FTS maintains a separate indexes for each language-id (a 32-bit integer). +** Within each language id, a separate index is maintained to store the +** document terms, and each configured prefix size (configured the FTS +** "prefix=" option). And each index consists of multiple levels ("relative +** levels"). +** +** All three of these values (the language id, the specific index and the +** level within the index) are encoded in 64-bit integer values stored +** in the %_segdir table on disk. This function is used to convert three +** separate component values into the single 64-bit integer value that +** can be used to query the %_segdir table. +** +** Specifically, each language-id/index combination is allocated 1024 +** 64-bit integer level values ("absolute levels"). The main terms index +** for language-id 0 is allocate values 0-1023. The first prefix index +** (if any) for language-id 0 is allocated values 1024-2047. And so on. +** Language 1 indexes are allocated immediately following language 0. +** +** So, for a system with nPrefix prefix indexes configured, the block of +** absolute levels that corresponds to language-id iLangid and index +** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). +*/ +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ +){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ + assert( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; +} + +/* +** Set *ppStmt to a statement handle that may be used to iterate through +** all rows in the %_segdir table, from oldest to newest. If successful, +** return SQLITE_OK. If an error occurs while preparing the statement, +** return an SQLite error code. +** +** There is only ever one instance of this SQL statement compiled for +** each FTS3 table. +** +** The statement returns the following columns from the %_segdir table: +** +** 0: idx +** 1: start_block +** 2: leaves_end_block +** 3: end_block +** 4: root +*/ +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select (relative level) */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel<0 ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + } + } + *ppStmt = pStmt; + return rc; +} + + +/* +** Append a single varint to a PendingList buffer. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. +** +** This function also serves to allocate the PendingList structure itself. +** For example, to create a new PendingList structure containing two +** varints: +** +** PendingList *p = 0; +** fts3PendingListAppendVarint(&p, 1); +** fts3PendingListAppendVarint(&p, 2); +*/ +static int fts3PendingListAppendVarint( + PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ + sqlite3_int64 i /* Value to append to data */ +){ + PendingList *p = *pp; + + /* Allocate or grow the PendingList as required. */ + if( !p ){ + p = sqlite3_malloc(sizeof(*p) + 100); + if( !p ){ + return SQLITE_NOMEM; + } + p->nSpace = 100; + p->aData = (char *)&p[1]; + p->nData = 0; + } + else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ + int nNew = p->nSpace * 2; + p = sqlite3_realloc(p, sizeof(*p) + nNew); + if( !p ){ + sqlite3_free(*pp); + *pp = 0; + return SQLITE_NOMEM; + } + p->nSpace = nNew; + p->aData = (char *)&p[1]; + } + + /* Append the new serialized varint to the end of the list. */ + p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); + p->aData[p->nData] = '\0'; + *pp = p; + return SQLITE_OK; +} + +/* +** Add a docid/column/position entry to a PendingList structure. Non-zero +** is returned if the structure is sqlite3_realloced as part of adding +** the entry. Otherwise, zero. +** +** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. +** Zero is always returned in this case. Otherwise, if no OOM error occurs, +** it is set to SQLITE_OK. +*/ +static int fts3PendingListAppend( + PendingList **pp, /* IN/OUT: PendingList structure */ + sqlite3_int64 iDocid, /* Docid for entry to add */ + sqlite3_int64 iCol, /* Column for entry to add */ + sqlite3_int64 iPos, /* Position of term for entry to add */ + int *pRc /* OUT: Return code */ +){ + PendingList *p = *pp; + int rc = SQLITE_OK; + + assert( !p || p->iLastDocid<=iDocid ); + + if( !p || p->iLastDocid!=iDocid ){ + sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); + if( p ){ + assert( p->nDatanSpace ); + assert( p->aData[p->nData]==0 ); + p->nData++; + } + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ + goto pendinglistappend_out; + } + p->iLastCol = -1; + p->iLastPos = 0; + p->iLastDocid = iDocid; + } + if( iCol>0 && p->iLastCol!=iCol ){ + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) + || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) + ){ + goto pendinglistappend_out; + } + p->iLastCol = iCol; + p->iLastPos = 0; + } + if( iCol>=0 ){ + assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); + rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + if( rc==SQLITE_OK ){ + p->iLastPos = iPos; + } + } + + pendinglistappend_out: + *pRc = rc; + if( p!=*pp ){ + *pp = p; + return 1; + } + return 0; +} + +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + +/* +** Tokenize the nul-terminated string zText and add all tokens to the +** pending-terms hash-table. The docid used is that currently stored in +** p->iPrevDocid, and the column is specified by argument iCol. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3PendingTermsAdd( + Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ + const char *zText, /* Text of document to be inserted */ + int iCol, /* Column into which text is being inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ +){ + int rc; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int nWord = 0; + + char const *zToken; + int nToken = 0; + + sqlite3_tokenizer *pTokenizer = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr; + int (*xNext)(sqlite3_tokenizer_cursor *pCursor, + const char**,int*,int*,int*,int*); + + assert( pTokenizer && pModule ); + + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + + xNext = pModule->xNext; + while( SQLITE_OK==rc + && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) + ){ + int i; + if( iPos>=nWord ) nWord = iPos+1; + + /* Positions cannot be negative; we use -1 as a terminator internally. + ** Tokens must have a non-zero length. + */ + if( iPos<0 || !zToken || nToken<=0 ){ + rc = SQLITE_ERROR; + break; + } + + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); + } + } + + pModule->xClose(pCsr); + *pnWord += nWord; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Calling this function indicates that subsequent calls to +** fts3PendingTermsAdd() are to add term/position-list pairs for the +** contents of the document with docid iDocid. +*/ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ +){ + assert( iLangid>=0 ); + + /* TODO(shess) Explore whether partially flushing the buffer on + ** forced-flush would provide better performance. I suspect that if + ** we ordered the doclists by size and flushed the largest until the + ** buffer was half empty, that would let the less frequent terms + ** generate longer doclists. + */ + if( iDocid<=p->iPrevDocid + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + } + p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; + return SQLITE_OK; +} + +/* +** Discard the contents of the pending-terms hash tables. +*/ +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); + } + p->nPendingData = 0; +} + +/* +** This function is called by the xUpdate() method as part of an INSERT +** operation. It adds entries for each term in the new record to the +** pendingTerms hash table. +** +** Argument apVal is the same as the similarly named argument passed to +** fts3InsertData(). Parameter iDocid is the docid of the new row. +*/ +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ + int i; /* Iterator variable */ + for(i=2; inColumn+2; i++){ + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); + } + } + return SQLITE_OK; +} + +/* +** This function is called by the xUpdate() method for an INSERT operation. +** The apVal parameter is passed a copy of the apVal argument passed by +** SQLite to the xUpdate() method. i.e: +** +** apVal[0] Not used for INSERT. +** apVal[1] rowid +** apVal[2] Left-most user-defined column +** ... +** apVal[p->nColumn+1] Right-most user-defined column +** apVal[p->nColumn+2] Hidden column with same name as table +** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column +*/ +static int fts3InsertData( + Fts3Table *p, /* Full-text table */ + sqlite3_value **apVal, /* Array of values to insert */ + sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ +){ + int rc; /* Return code */ + sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + + /* Locate the statement handle used to insert data into the %_content + ** table. The SQL for this statement is: + ** + ** INSERT INTO %_content VALUES(?, ?, ?, ...) + ** + ** The statement features N '?' variables, where N is the number of user + ** defined columns in the FTS3 table, plus one for the docid field. + */ + rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); + } + if( rc!=SQLITE_OK ) return rc; + + /* There is a quirk here. The users INSERT statement may have specified + ** a value for the "rowid" field, for the "docid" field, or for both. + ** Which is a problem, since "rowid" and "docid" are aliases for the + ** same value. For example: + ** + ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); + ** + ** In FTS3, this is an error. It is an error to specify non-NULL values + ** for both docid and some other rowid alias. + */ + if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + if( SQLITE_NULL==sqlite3_value_type(apVal[0]) + && SQLITE_NULL!=sqlite3_value_type(apVal[1]) + ){ + /* A rowid/docid conflict. */ + return SQLITE_ERROR; + } + rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); + if( rc!=SQLITE_OK ) return rc; + } + + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. + */ + sqlite3_step(pContentInsert); + rc = sqlite3_reset(pContentInsert); + + *piDocid = sqlite3_last_insert_rowid(p->db); + return rc; +} + + + +/* +** Remove all data from the FTS3 table. Clear the hash table containing +** pending terms. +*/ +static int fts3DeleteAll(Fts3Table *p, int bContent){ + int rc = SQLITE_OK; /* Return code */ + + /* Discard the contents of the pending-terms hash table. */ + sqlite3Fts3PendingTermsClear(p); + + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + } + if( p->bHasStat ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); + } + return rc; +} + +/* +** +*/ +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; +} + +/* +** The first element in the apVal[] array is assumed to contain the docid +** (an integer) of a row about to be deleted. Remove all terms from the +** full-text index. +*/ +static void fts3DeleteTerms( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS table to delete from */ + sqlite3_value *pRowid, /* The docid to be deleted */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ +){ + int rc; + sqlite3_stmt *pSelect; + + assert( *pbFound==0 ); + if( *pRC ) return; + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + int i; + int iLangid = langidFromSelect(p, pSelect); + rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + } + } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } + *pbFound = 1; + } + rc = sqlite3_reset(pSelect); + }else{ + sqlite3_reset(pSelect); + } + *pRC = rc; +} + +/* +** Forward declaration to account for the circular dependency between +** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). +*/ +static int fts3SegmentMerge(Fts3Table *, int, int, int); + +/* +** This function allocates a new level iLevel index in the segdir table. +** Usually, indexes are allocated within a level sequentially starting +** with 0, so the allocated index is one greater than the value returned +** by: +** +** SELECT max(idx) FROM %_segdir WHERE level = :iLevel +** +** However, if there are already FTS3_MERGE_COUNT indexes at the requested +** level, they are merged into a single level (iLevel+1) segment and the +** allocated index is 0. +** +** If successful, *piIdx is set to the allocated index slot and SQLITE_OK +** returned. Otherwise, an SQLite error code is returned. +*/ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ + int rc; /* Return Code */ + sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ + int iNext = 0; /* Result of query pNextIdx */ + + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + + /* Set variable iNext to the next available segdir index at level iLevel. */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ + iNext = sqlite3_column_int(pNextIdx, 0); + } + rc = sqlite3_reset(pNextIdx); + } + + if( rc==SQLITE_OK ){ + /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already + ** full, merge all segments in level iLevel into a single iLevel+1 + ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, + ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. + */ + if( iNext>=FTS3_MERGE_COUNT ){ + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); + *piIdx = 0; + }else{ + *piIdx = iNext; + } + } + + return rc; +} + +/* +** The %_segments table is declared as follows: +** +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. +** +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ +){ + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob ); + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments + ); + } + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; + if( paBlob ){ + char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + memset(&aByte[nByte], 0, FTS3_NODE_PADDING); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; + } + } + + return rc; +} + +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; +} + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} + +/* +** Set an Fts3SegReader cursor to point at EOF. +*/ +static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ + if( !fts3SegReaderIsRootOnly(pSeg) ){ + sqlite3_free(pSeg->aNode); + sqlite3_blob_close(pSeg->pBlob); + pSeg->pBlob = 0; + } + pSeg->aNode = 0; +} + +/* +** Move the iterator passed as the first argument to the next term in the +** segment. If successful, SQLITE_OK is returned. If there is no next term, +** SQLITE_DONE. Otherwise, an SQLite error code. +*/ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ + char *pNext; /* Cursor variable */ + int nPrefix; /* Number of bytes in term prefix */ + int nSuffix; /* Number of bytes in term suffix */ + + if( !pReader->aDoclist ){ + pNext = pReader->aNode; + }else{ + pNext = &pReader->aDoclist[pReader->nDoclist]; + } + + if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ + + if( fts3SegReaderIsPending(pReader) ){ + Fts3HashElem *pElem = *(pReader->ppNextElem); + if( pElem==0 ){ + pReader->aNode = 0; + }else{ + PendingList *pList = (PendingList *)fts3HashData(pElem); + pReader->zTerm = (char *)fts3HashKey(pElem); + pReader->nTerm = fts3HashKeysize(pElem); + pReader->nNode = pReader->nDoclist = pList->nData + 1; + pReader->aNode = pReader->aDoclist = pList->aData; + pReader->ppNextElem++; + assert( pReader->aNode ); + } + return SQLITE_OK; + } + + fts3SegReaderSetEof(pReader); + + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + ** blocks have already been traversed. */ + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ + return SQLITE_OK; + } + + rc = sqlite3Fts3ReadBlock( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) + ); + if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } + pNext = pReader->aNode; + } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; + + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + ** safe (no risk of overread) even if the node data is corrupted. */ + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); + if( nPrefix<0 || nSuffix<=0 + || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] + ){ + return FTS_CORRUPT_VTAB; + } + + if( nPrefix+nSuffix>pReader->nTermAlloc ){ + int nNew = (nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc(pReader->zTerm, nNew); + if( !zNew ){ + return SQLITE_NOMEM; + } + pReader->zTerm = zNew; + pReader->nTermAlloc = nNew; + } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); + pReader->nTerm = nPrefix+nSuffix; + pNext += nSuffix; + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); + pReader->aDoclist = pNext; + pReader->pOffsetList = 0; + + /* Check that the doclist does not appear to extend past the end of the + ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** of these statements is untrue, then the data structure is corrupt. + */ + if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + ){ + return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} + +/* +** Set the SegReader to point to the first docid in the doclist associated +** with the current term. +*/ +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; + assert( pReader->aDoclist ); + assert( !pReader->pOffsetList ); + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; +} + +/* +** Advance the SegReader to point to the next docid in the doclist +** associated with the current term. +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then +** *ppOffsetList is set to point to the first column-offset list +** in the doclist entry (i.e. immediately past the docid varint). +** *pnOffsetList is set to the length of the set of column-offset +** lists, not including the nul-terminator byte. For example: +*/ +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ +){ + int rc = SQLITE_OK; + char *p = pReader->pOffsetList; + char c = 0; + + assert( p ); + + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } + }else{ + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte past + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* List may have been edited in place by fts3EvalNearTrim() */ + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } + } + } + + return SQLITE_OK; +} + + +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; + + assert( p->bFts4 ); + assert( pgsz>0 ); + + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } + } + } + } + *pnOvfl = nOvfl; + return rc; +} + +/* +** Free all allocations associated with the iterator passed as the +** second argument. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader && !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); + } + } + sqlite3_free(pReader); +} + +/* +** Allocate a new SegReader object. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( + int iAge, /* Segment "age". */ + int bLookup, /* True for a lookup only */ + sqlite3_int64 iStartLeaf, /* First leaf to traverse */ + sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ + sqlite3_int64 iEndBlock, /* Final block of segment */ + const char *zRoot, /* Buffer containing root node */ + int nRoot, /* Size of buffer containing root node */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + Fts3SegReader *pReader; /* Newly allocated SegReader object */ + int nExtra = 0; /* Bytes to allocate segment root node */ + + assert( iStartLeaf<=iEndLeaf ); + if( iStartLeaf==0 ){ + nExtra = nRoot + FTS3_NODE_PADDING; + } + + pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + if( !pReader ){ + return SQLITE_NOMEM; + } + memset(pReader, 0, sizeof(Fts3SegReader)); + pReader->iIdx = iAge; + pReader->bLookup = bLookup!=0; + pReader->iStartBlock = iStartLeaf; + pReader->iLeafEndBlock = iEndLeaf; + pReader->iEndBlock = iEndBlock; + + if( nExtra ){ + /* The entire segment is stored in the root node. */ + pReader->aNode = (char *)&pReader[1]; + pReader->rootOnly = 1; + pReader->nNode = nRoot; + memcpy(pReader->aNode, zRoot, nRoot); + memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); + }else{ + pReader->iCurrentBlock = iStartLeaf-1; + } + *ppReader = pReader; + return SQLITE_OK; +} + +/* +** This is a comparison function used as a qsort() callback when sorting +** an array of pending terms by term. This occurs as part of flushing +** the contents of the pending-terms hash table to the database. +*/ +static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ + char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); + char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); + int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); + int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); + + int n = (n1aIndex */ + const char *zTerm, /* Term to search for */ + int nTerm, /* Size of buffer zTerm */ + int bPrefix, /* True for a prefix iterator */ + Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ +){ + Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ + Fts3HashElem *pE; /* Iterator variable */ + Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ + int nElem = 0; /* Size of array at aElem */ + int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; + + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ + int nAlloc = 0; /* Size of allocated array at aElem */ + + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ + char *zKey = (char *)fts3HashKey(pE); + int nKey = fts3HashKeysize(pE); + if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ + if( nElem==nAlloc ){ + Fts3HashElem **aElem2; + nAlloc += 16; + aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem, nAlloc*sizeof(Fts3HashElem *) + ); + if( !aElem2 ){ + rc = SQLITE_NOMEM; + nElem = 0; + break; + } + aElem = aElem2; + } + + aElem[nElem++] = pE; + } + } + + /* If more than one term matches the prefix, sort the Fts3HashElem + ** objects in term order using qsort(). This uses the same comparison + ** callback as is used when flushing terms to disk. + */ + if( nElem>1 ){ + qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); + } + + }else{ + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. + ** + ** Because the stack address of pE may be accessed via the aElem pointer + ** below, the "Fts3HashElem *pE" must be declared so that it is valid + ** within this entire function, not just this "else{...}" block. + */ + pE = fts3HashFindElem(pHash, zTerm, nTerm); + if( pE ){ + aElem = &pE; + nElem = 1; + } + } + + if( nElem>0 ){ + int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc(nByte); + if( !pReader ){ + rc = SQLITE_NOMEM; + }else{ + memset(pReader, 0, nByte); + pReader->iIdx = 0x7FFFFFFF; + pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; + memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); + } + } + + if( bPrefix ){ + sqlite3_free(aElem); + } + *ppReader = pReader; + return rc; +} + +/* +** Compare the entries pointed to by two Fts3SegReader structures. +** Comparison is as follows: +** +** 1) EOF is greater than not EOF. +** +** 2) The current terms (if any) are compared using memcmp(). If one +** term is a prefix of another, the longer term is considered the +** larger. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc; + if( pLhs->aNode && pRhs->aNode ){ + int rc2 = pLhs->nTerm - pRhs->nTerm; + if( rc2<0 ){ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); + }else{ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); + } + if( rc==0 ){ + rc = rc2; + } + }else{ + rc = (pLhs->aNode==0) - (pRhs->aNode==0); + } + if( rc==0 ){ + rc = pRhs->iIdx - pLhs->iIdx; + } + assert( rc!=0 ); + return rc; +} + +/* +** A different comparison function for SegReader structures. In this +** version, it is assumed that each SegReader points to an entry in +** a doclist for identical terms. Comparison is made as follows: +** +** 1) EOF (end of doclist in this case) is greater than not EOF. +** +** 2) By current docid. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} + +/* +** Compare the term that the Fts3SegReader object passed as the first argument +** points to with the term specified by arguments zTerm and nTerm. +** +** If the pSeg iterator is already at EOF, return 0. Otherwise, return +** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are +** equal, or +ve if the pSeg term is greater than zTerm/nTerm. +*/ +static int fts3SegReaderTermCmp( + Fts3SegReader *pSeg, /* Segment reader object */ + const char *zTerm, /* Term to compare to */ + int nTerm /* Size of term zTerm in bytes */ +){ + int res = 0; + if( pSeg->aNode ){ + if( pSeg->nTerm>nTerm ){ + res = memcmp(pSeg->zTerm, zTerm, nTerm); + }else{ + res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); + } + if( res==0 ){ + res = pSeg->nTerm-nTerm; + } + } + return res; +} + +/* +** Argument apSegment is an array of nSegment elements. It is known that +** the final (nSegment-nSuspect) members are already in sorted order +** (according to the comparison function provided). This function shuffles +** the array around until all entries are in sorted order. +*/ +static void fts3SegReaderSort( + Fts3SegReader **apSegment, /* Array to sort entries of */ + int nSegment, /* Size of apSegment array */ + int nSuspect, /* Unsorted entry count */ + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ +){ + int i; /* Iterator variable */ + + assert( nSuspect<=nSegment ); + + if( nSuspect==nSegment ) nSuspect--; + for(i=nSuspect-1; i>=0; i--){ + int j; + for(j=i; j<(nSegment-1); j++){ + Fts3SegReader *pTmp; + if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; + pTmp = apSegment[j+1]; + apSegment[j+1] = apSegment[j]; + apSegment[j] = pTmp; + } + } + +#ifndef NDEBUG + /* Check that the list really is sorted now. */ + for(i=0; i<(nSuspect-1); i++){ + assert( xCmp(apSegment[i], apSegment[i+1])<0 ); + } +#endif +} + +/* +** Insert a record into the %_segments table. +*/ +static int fts3WriteSegment( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iBlock, /* Block id for new block */ + char *z, /* Pointer to buffer containing block data */ + int n /* Size of buffer z in bytes */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iBlock); + sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + return rc; +} + +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + +/* +** Insert a record into the %_segdir table. +*/ +static int fts3WriteSegdir( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ + int iIdx, /* Value for "idx" field */ + sqlite3_int64 iStartBlock, /* Value for "start_block" field */ + sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ + sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ + char *zRoot, /* Blob value for "root" field */ + int nRoot /* Number of bytes in buffer zRoot */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iLevel); + sqlite3_bind_int(pStmt, 2, iIdx); + sqlite3_bind_int64(pStmt, 3, iStartBlock); + sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } + sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + return rc; +} + +/* +** Return the size of the common prefix (if any) shared by zPrev and +** zNext, in bytes. For example, +** +** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 +** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 +** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 +*/ +static int fts3PrefixCompress( + const char *zPrev, /* Buffer containing previous term */ + int nPrev, /* Size of buffer zPrev in bytes */ + const char *zNext, /* Buffer containing next term */ + int nNext /* Size of buffer zNext in bytes */ +){ + int n; + UNUSED_PARAMETER(nNext); + for(n=0; nnData; /* Current size of node in bytes */ + int nReq = nData; /* Required space after adding zTerm */ + int nPrefix; /* Number of bytes of prefix compression */ + int nSuffix; /* Suffix length */ + + nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; + if( nReq<=p->nNodeSize || !pTree->zTerm ){ + + if( nReq>p->nNodeSize ){ + /* An unusual case: this is the first term to be added to the node + ** and the static node buffer (p->nNodeSize bytes) is not large + ** enough. Use a separately malloced buffer instead This wastes + ** p->nNodeSize bytes, but since this scenario only comes about when + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. + */ + assert( pTree->aData==(char *)&pTree[1] ); + pTree->aData = (char *)sqlite3_malloc(nReq); + if( !pTree->aData ){ + return SQLITE_NOMEM; + } + } + + if( pTree->zTerm ){ + /* There is no prefix-length field for first term in a node */ + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + } + + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); + memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); + pTree->nData = nData + nSuffix; + pTree->nEntry++; + + if( isCopyTerm ){ + if( pTree->nMalloczMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pTree->nMalloc = nTerm*2; + pTree->zMalloc = zNew; + } + pTree->zTerm = pTree->zMalloc; + memcpy(pTree->zTerm, zTerm, nTerm); + pTree->nTerm = nTerm; + }else{ + pTree->zTerm = (char *)zTerm; + pTree->nTerm = nTerm; + } + return SQLITE_OK; + } + } + + /* If control flows to here, it was not possible to append zTerm to the + ** current node. Create a new node (a right-sibling of the current node). + ** If this is the first node in the tree, the term is added to it. + ** + ** Otherwise, the term is not added to the new node, it is left empty for + ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** has no parent, one is created here. + */ + pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SegmentNode)); + pNew->nData = 1 + FTS3_VARINT_MAX; + pNew->aData = (char *)&pNew[1]; + + if( pTree ){ + SegmentNode *pParent = pTree->pParent; + rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); + if( pTree->pParent==0 ){ + pTree->pParent = pParent; + } + pTree->pRight = pNew; + pNew->pLeftmost = pTree->pLeftmost; + pNew->pParent = pParent; + pNew->zMalloc = pTree->zMalloc; + pNew->nMalloc = pTree->nMalloc; + pTree->zMalloc = 0; + }else{ + pNew->pLeftmost = pNew; + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + } + + *ppTree = pNew; + return rc; +} + +/* +** Helper function for fts3NodeWrite(). +*/ +static int fts3TreeFinishNode( + SegmentNode *pTree, + int iHeight, + sqlite3_int64 iLeftChild +){ + int nStart; + assert( iHeight>=1 && iHeight<128 ); + nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); + pTree->aData[nStart] = (char)iHeight; + sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); + return nStart; +} + +/* +** Write the buffer for the segment node pTree and all of its peers to the +** database. Then call this function recursively to write the parent of +** pTree and its peers to the database. +** +** Except, if pTree is a root node, do not write it to the database. Instead, +** set output variables *paRoot and *pnRoot to contain the root node. +** +** If successful, SQLITE_OK is returned and output variable *piLast is +** set to the largest blockid written to the database (or zero if no +** blocks were written to the db). Otherwise, an SQLite error code is +** returned. +*/ +static int fts3NodeWrite( + Fts3Table *p, /* Virtual table handle */ + SegmentNode *pTree, /* SegmentNode handle */ + int iHeight, /* Height of this node in tree */ + sqlite3_int64 iLeaf, /* Block id of first leaf node */ + sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ + sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ + char **paRoot, /* OUT: Data for root node */ + int *pnRoot /* OUT: Size of root node in bytes */ +){ + int rc = SQLITE_OK; + + if( !pTree->pParent ){ + /* Root node of the tree. */ + int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); + *piLast = iFree-1; + *pnRoot = pTree->nData - nStart; + *paRoot = &pTree->aData[nStart]; + }else{ + SegmentNode *pIter; + sqlite3_int64 iNextFree = iFree; + sqlite3_int64 iNextLeaf = iLeaf; + for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ + int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); + int nWrite = pIter->nData - nStart; + + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); + iNextFree++; + iNextLeaf += (pIter->nEntry+1); + } + if( rc==SQLITE_OK ){ + assert( iNextLeaf==iFree ); + rc = fts3NodeWrite( + p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot + ); + } + } + + return rc; +} + +/* +** Free all memory allocations associated with the tree pTree. +*/ +static void fts3NodeFree(SegmentNode *pTree){ + if( pTree ){ + SegmentNode *p = pTree->pLeftmost; + fts3NodeFree(p->pParent); + while( p ){ + SegmentNode *pRight = p->pRight; + if( p->aData!=(char *)&p[1] ){ + sqlite3_free(p->aData); + } + assert( pRight==0 || p->zMalloc==0 ); + sqlite3_free(p->zMalloc); + sqlite3_free(p); + p = pRight; + } + } +} + +/* +** Add a term to the segment being constructed by the SegmentWriter object +** *ppWriter. When adding the first term to a segment, *ppWriter should +** be passed NULL. This function will allocate a new SegmentWriter object +** and return it via the input/output variable *ppWriter in this case. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterAdd( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + int isCopyTerm, /* True if buffer zTerm must be copied */ + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of term in bytes */ + const char *aDoclist, /* Pointer to buffer containing doclist */ + int nDoclist /* Size of doclist in bytes */ +){ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + int nReq; /* Number of bytes required on leaf page */ + int nData; + SegmentWriter *pWriter = *ppWriter; + + if( !pWriter ){ + int rc; + sqlite3_stmt *pStmt; + + /* Allocate the SegmentWriter structure */ + pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, sizeof(SegmentWriter)); + *ppWriter = pWriter; + + /* Allocate a buffer in which to accumulate data */ + pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + if( !pWriter->aData ) return SQLITE_NOMEM; + pWriter->nSize = p->nNodeSize; + + /* Find the next free blockid in the %_segments table */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + pWriter->iFree = sqlite3_column_int64(pStmt, 0); + pWriter->iFirst = pWriter->iFree; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + nData = pWriter->nData; + + nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* Figure out how many bytes are required by this new entry */ + nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ + nSuffix + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + + if( nData>0 && nData+nReq>p->nNodeSize ){ + int rc; + + /* The current leaf node is full. Write it out to the database. */ + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); + if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; + + /* Add the current term to the interior node tree. The term added to + ** the interior tree must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pWriter->zTerm), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); + if( rc!=SQLITE_OK ) return rc; + + nData = 0; + pWriter->nTerm = 0; + + nPrefix = 0; + nSuffix = nTerm; + nReq = 1 + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ + nTerm + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + } + + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + + /* If the buffer currently allocated is too small for this entry, realloc + ** the buffer to make it large enough. + */ + if( nReq>pWriter->nSize ){ + char *aNew = sqlite3_realloc(pWriter->aData, nReq); + if( !aNew ) return SQLITE_NOMEM; + pWriter->aData = aNew; + pWriter->nSize = nReq; + } + assert( nData+nReq<=pWriter->nSize ); + + /* Append the prefix-compressed term and doclist to the buffer. */ + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); + nData += nSuffix; + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + memcpy(&pWriter->aData[nData], aDoclist, nDoclist); + pWriter->nData = nData + nDoclist; + + /* Save the current term so that it can be used to prefix-compress the next. + ** If the isCopyTerm parameter is true, then the buffer pointed to by + ** zTerm is transient, so take a copy of the term data. Otherwise, just + ** store a copy of the pointer. + */ + if( isCopyTerm ){ + if( nTerm>pWriter->nMalloc ){ + char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pWriter->nMalloc = nTerm*2; + pWriter->zMalloc = zNew; + pWriter->zTerm = zNew; + } + assert( pWriter->zTerm==pWriter->zMalloc ); + memcpy(pWriter->zTerm, zTerm, nTerm); + }else{ + pWriter->zTerm = (char *)zTerm; + } + pWriter->nTerm = nTerm; + + return SQLITE_OK; +} + +/* +** Flush all data associated with the SegmentWriter object pWriter to the +** database. This function must be called after all terms have been added +** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is +** returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterFlush( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ + int iIdx /* Value for 'idx' column of %_segdir */ +){ + int rc; /* Return code */ + if( pWriter->pTree ){ + sqlite3_int64 iLast = 0; /* Largest block id written to database */ + sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ + char *zRoot = NULL; /* Pointer to buffer containing root node */ + int nRoot = 0; /* Size of buffer zRoot */ + + iLastLeaf = pWriter->iFree; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); + if( rc==SQLITE_OK ){ + rc = fts3NodeWrite(p, pWriter->pTree, 1, + pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); + } + }else{ + /* The entire tree fits on the root node. Write it to the segdir table. */ + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); + } + p->nLeafAdd++; + return rc; +} + +/* +** Release all memory held by the SegmentWriter object passed as the +** first argument. +*/ +static void fts3SegWriterFree(SegmentWriter *pWriter){ + if( pWriter ){ + sqlite3_free(pWriter->aData); + sqlite3_free(pWriter->zMalloc); + fts3NodeFree(pWriter->pTree); + sqlite3_free(pWriter); + } +} + +/* +** The first value in the apVal[] array is assumed to contain an integer. +** This function tests if there exist any documents with docid values that +** are different from that integer. i.e. if deleting the document with docid +** pRowid would mean the FTS3 table were empty. +** +** If successful, *pisEmpty is set to true if the table is empty except for +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an +** error occurs, an SQLite error code is returned. +*/ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ + sqlite3_stmt *pStmt; + int rc; + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Set *pnMax to the largest segment level in the database for the index +** iIndex. +** +** Segment levels are stored in the 'level' column of the %_segdir table. +** +** Return SQLITE_OK if successful, or an SQLite error code if not. +*/ +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + sqlite3_int64 *pnMax +){ + sqlite3_stmt *pStmt; + int rc; + assert( iIndex>=0 && iIndexnIndex ); + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnMax = sqlite3_column_int64(pStmt, 0); + } + return sqlite3_reset(pStmt); +} + +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + +/* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; +} + +/* +** This function is used after merging multiple segments into a single large +** segment to delete the old, now redundant, segment b-trees. Specifically, +** it: +** +** 1) Deletes all %_segments entries for the segments associated with +** each of the SegReader objects in the array passed as the third +** argument, and +** +** 2) deletes all %_segdir entries with level iLevel, or all %_segdir +** entries regardless of level if (iLevel<0). +** +** SQLITE_OK is returned if successful, otherwise an SQLite error code. +*/ +static int fts3DeleteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, /* Level of %_segdir entries to delete */ + Fts3SegReader **apSegment, /* Array of SegReader objects */ + int nReader /* Size of array apSegment */ +){ + int rc = SQLITE_OK; /* Return Code */ + int i; /* Iterator variable */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ + + for(i=0; rc==SQLITE_OK && i=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** When this function is called, buffer *ppList (size *pnList bytes) contains +** a position list that may (or may not) feature multiple columns. This +** function adjusts the pointer *ppList and the length *pnList so that they +** identify the subset of the position list that corresponds to column iCol. +** +** If there are no entries in the input position list for column iCol, then +** *pnList is set to zero before returning. +** +** If parameter bZero is non-zero, then any part of the input list following +** the end of the output list is zeroed before returning. +*/ +static void fts3ColumnFilter( + int iCol, /* Column to filter on */ + int bZero, /* Zero out anything following *ppList */ + char **ppList, /* IN/OUT: Pointer to position list */ + int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ +){ + char *pList = *ppList; + int nList = *pnList; + char *pEnd = &pList[nList]; + int iCurrent = 0; + char *p = pList; + + assert( iCol>=0 ); + while( 1 ){ + char c = 0; + while( ppMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + } + + memcpy(pMsr->aBuffer, pList, nList); + return SQLITE_OK; +} + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; + } + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); + } + + if( nList>0 ){ + *paPoslist = pList; + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } + } + + return SQLITE_OK; +} + +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ +){ + int i; + int nSeg = pCsr->nSegment; + + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + for(i=0; pCsr->bRestart==0 && inSegment; i++){ + int res = 0; + Fts3SegReader *pSeg = pCsr->apSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg, 0); + if( rc!=SQLITE_OK ) return rc; + }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); + + if( pSeg->bLookup && res!=0 ){ + fts3SegReaderSetEof(pSeg); + } + } + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); + + return SQLITE_OK; +} + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + int rc = SQLITE_OK; + + int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); + int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); + + Fts3SegReader **apSegment = pCsr->apSegment; + int nSegment = pCsr->nSegment; + Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( pCsr->nSegment==0 ) return SQLITE_OK; + + do { + int nMerge; + int i; + + /* Advance the first pCsr->nAdvance entries in the apSegment[] array + ** forward. Then sort the list in order of current term again. + */ + for(i=0; inAdvance; i++){ + Fts3SegReader *pSeg = apSegment[i]; + if( pSeg->bLookup ){ + fts3SegReaderSetEof(pSeg); + }else{ + rc = fts3SegReaderNext(p, pSeg, 0); + } + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); + pCsr->nAdvance = 0; + + /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ + assert( rc==SQLITE_OK ); + if( apSegment[0]->aNode==0 ) break; + + pCsr->nTerm = apSegment[0]->nTerm; + pCsr->zTerm = apSegment[0]->zTerm; + + /* If this is a prefix-search, and if the term that apSegment[0] points + ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** required callbacks have been made. In this case exit early. + ** + ** Similarly, if this is a search for an exact match, and the first term + ** of segment apSegment[0] is not a match, exit early. + */ + if( pFilter->zTerm && !isScan ){ + if( pCsr->nTermnTerm + || (!isPrefix && pCsr->nTerm>pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + break; + } + } + + nMerge = 1; + while( nMergeaNode + && apSegment[nMerge]->nTerm==pCsr->nTerm + && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) + ){ + nMerge++; + } + + assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); + if( nMerge==1 + && !isIgnoreEmpty + && !isFirst + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ + pCsr->nDoclist = apSegment[0]->nDoclist; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; + }else{ + int nDoclist = 0; /* Size of doclist */ + sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ + + /* The current term of the first nMerge entries in the array + ** of Fts3SegReader objects is the same. The doclists must be merged + ** and a single term returned with the merged doclist. + */ + for(i=0; ipOffsetList ){ + int j; /* Number of segments that share a docid */ + char *pList = 0; + int nList = 0; + int nByte; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + + if( isColFilter ){ + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); + } + + if( !isIgnoreEmpty || nList>0 ){ + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); + if( nDoclist+nByte>pCsr->nBuffer ){ + char *aNew; + pCsr->nBuffer = (nDoclist+nByte)*2; + aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + if( !aNew ){ + return SQLITE_NOMEM; + } + pCsr->aBuffer = aNew; + } + + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } + } + } + + fts3SegReaderSort(apSegment, nMerge, j, xCmp); + } + if( nDoclist>0 ){ + pCsr->aDoclist = pCsr->aBuffer; + pCsr->nDoclist = nDoclist; + rc = SQLITE_ROW; + } + } + pCsr->nAdvance = nMerge; + }while( rc==SQLITE_OK ); + + return rc; +} + + +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + if( pCsr ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); + } + sqlite3_free(pCsr->apSegment); + sqlite3_free(pCsr->aBuffer); + + pCsr->nSegment = 0; + pCsr->apSegment = 0; + pCsr->aBuffer = 0; + } +} + +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = (iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1; + sqlite3_stmt *pUpdate2; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + +/* +** Merge all level iLevel segments in the database into a single +** iLevel+1 segment. Or, if iLevel<0, merge all segments into a +** single segment with a level equal to the numerically largest level +** currently present in the database. +** +** If this function is called with iLevel<0, but there is only one +** segment in the database, SQLITE_DONE is returned immediately. +** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, +** an SQLite error code is returned. +*/ +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ + int rc; /* Return code */ + int iIdx = 0; /* Index of new segment */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ + SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ + Fts3SegFilter filter; /* Segment term filter condition */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ + + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); + if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + + if( iLevel==FTS3_SEGCURSOR_ALL ){ + /* This call is to merge all segments in the database to a single + ** segment. The level of the new segment is equal to the numerically + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ + if( csr.nSegment==1 ){ + rc = SQLITE_DONE; + goto finished; + } + iNewLevel = iMaxLevel; + bIgnoreEmpty = 1; + + }else{ + /* This call is to merge all segments at level iLevel. find the next + ** available segment index at level iLevel+1. The call to + ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to + ** a single iLevel+2 segment if necessary. */ + assert( FTS3_SEGCURSOR_PENDING==-1 ); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); + } + if( rc!=SQLITE_OK ) goto finished; + + assert( csr.nSegment>0 ); + assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert( iNewLevelnLeafData); + } + } + } + + finished: + fts3SegWriterFree(pWriter); + sqlite3Fts3SegReaderFinish(&csr); + return rc; +} + + +/* +** Flush the contents of pendingTerms to level 0 segments. +*/ +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Encode N integers as varints into a blob. +*/ +static void fts3EncodeIntArray( + int N, /* The number of integers to encode */ + u32 *a, /* The integer values */ + char *zBuf, /* Write the BLOB here */ + int *pNBuf /* Write number of bytes if zBuf[] used here */ +){ + int i, j; + for(i=j=0; iiPrevDocid. The sizes are encoded as +** a blob of varints. +*/ +static void fts3InsertDocsize( + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ +){ + char *pBlob; /* The BLOB encoding of the document size */ + int nBlob; /* Number of bytes in the BLOB */ + sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ + int rc; /* Result code from subfunctions */ + + if( *pRC ) return; + pBlob = sqlite3_malloc( 10*p->nColumn ); + if( pBlob==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); + if( rc ){ + sqlite3_free(pBlob); + *pRC = rc; + return; + } + sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); +} + +/* +** Record 0 of the %_stat table contains a blob consisting of N varints, +** where N is the number of user defined columns in the fts3 table plus +** two. If nCol is the number of user defined columns, then values of the +** varints are set as follows: +** +** Varint 0: Total number of rows in the table. +** +** Varint 1..nCol: For each column, the total number of tokens stored in +** the column for all rows of the table. +** +** Varint 1+nCol: The total size, in bytes, of all text values in all +** columns of all rows of the table. +** +*/ +static void fts3UpdateDocTotals( + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ +){ + char *pBlob; /* Storage for BLOB written into %_stat */ + int nBlob; /* Size of BLOB written into %_stat */ + u32 *a; /* Array of integers that becomes the BLOB */ + sqlite3_stmt *pStmt; /* Statement for reading and writing */ + int i; /* Loop counter */ + int rc; /* Result code from subfunctions */ + + const int nStat = p->nColumn+2; + + if( *pRC ) return; + a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); + if( a==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + pBlob = (char*)&a[nStat]; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + fts3DecodeIntArray(nStat, a, + sqlite3_column_blob(pStmt, 0), + sqlite3_column_bytes(pStmt, 0)); + }else{ + memset(a, 0, sizeof(u32)*(nStat) ); + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + sqlite3_free(a); + *pRC = rc; + return; + } + if( nChng<0 && a[0]<(u32)(-nChng) ){ + a[0] = 0; + }else{ + a[0] += nChng; + } + for(i=0; inColumn+1; i++){ + u32 x = a[i+1]; + if( x+aSzIns[i] < aSzDel[i] ){ + x = 0; + }else{ + x = x + aSzIns[i] - aSzDel[i]; + } + a[i+1] = x; + } + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); + sqlite3_free(a); +} + +/* +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. +*/ +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int bSeenDone = 0; + int rc; + sqlite3_stmt *pAllLangid = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + +/* +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. +*/ +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ + + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns = 0; + u32 *aSzDel = 0; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + int nByte = sizeof(u32) * (p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); + + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + return rc; +} + + +/* +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. +*/ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + int nByte; /* Bytes allocated at pCsr->apSegment[] */ + + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] + ); + pCsr->nSegment++; + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ +struct Blob { + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ +}; + +/* +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). +*/ +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; + +/* +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. +*/ +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; + +/* +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() +*/ +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ + + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. +*/ +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } + } + + assert( p->iOff<=p->nNode ); + + return rc; +} + +/* +** Release all dynamic resources held by node-reader object *p. +*/ +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); +} + +/* +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. +*/ +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; + + /* Figure out if this is a leaf or an internal node. */ + if( p->aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; + } + + return nodeReaderNext(p); +} + +/* +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. +** +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. +*/ +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; + + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; + + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; + } + }else{ + /* Otherwise, flush the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); + + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; + } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; + } + + assert( 0 ); + return 0; +} + +/* +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. +*/ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; +} + +/* +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ + + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + } + + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + } + + pWriter->nLeafData += nSpace; + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; + } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); + } + + return rc; +} + +/* +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. +*/ +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ +){ + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ + + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); + } + iRoot = 1; + } + pRoot = &pWriter->aNodeWriter[iRoot]; + + /* Flush all currently outstanding nodes to disk. */ + for(i=0; iaNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + } + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); + } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); + + *pRc = rc; +} + +/* +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. +*/ +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; + + res = memcmp(zLhs, zRhs, nCmp); + if( res==0 ) res = nLhs - nRhs; + + return res; +} + + +/* +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. +*/ +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); + } + + *pbRes = bRes; + return rc; +} + +/* +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. +** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. +*/ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ + + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + }else{ + return sqlite3_reset(pSelect); + } + + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); + + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; + + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; + } + nodeReaderRelease(&reader); + } + sqlite3_free(aLeaf); + } + + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; + + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; + + for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; + } + + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; + + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); + blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + } + sqlite3_free(aBlock); + } + } + nodeReaderRelease(&reader); + } + } + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. +*/ +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); + } + + return rc; +} + +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. +** +** In the %_segdir table, a segment is defined by the values in three +** columns: +** +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. +** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. +** +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +*/ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ + + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); + } + if( rc!=SQLITE_OK ) return rc; + + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); + } + if( rc!=SQLITE_OK ) return rc; + + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + + /* Set up the array of NodeWriter objects */ + for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + return SQLITE_OK; +} + +/* +** Remove an entry from the %_segdir table. This involves running the +** following two statements: +** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. +*/ +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ +){ + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ + + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. +*/ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ +){ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; + + sqlite3_free(aIdx); + return rc; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; + } +} + +/* +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +*/ +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ + + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; + } + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; +} + +/* +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. +*/ +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; + } + + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + } + } + + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; +} + +/* +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. +*/ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ +){ + int i; + int nRem = 0; + int rc = SQLITE_OK; + + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; + + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; + } + } + + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); + } + + *pnRem = nRem; + return rc; +} + +/* +** Store an incr-merge hint in the database. +*/ +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + + return rc; +} + +/* +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. +*/ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; + int rc; + + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-2; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += fts3GetVarint32(&pHint->a[i], pnInput); + if( i!=nHint ) return SQLITE_CORRUPT_VTAB; + + return SQLITE_OK; +} + + +/* +** Attempt an incremental merge that writes nMerge leaf blocks. +** +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. +*/ +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = nMin; + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); + + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + iAbsLevel = iHintAbsLevel; + nSeg = nHintSeg; + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<0 ) break; + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) + ){ + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); + } + + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } + } + } + + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } + } + + sqlite3Fts3SegReaderFinish(pCsr); + } + + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); + } + + sqlite3_free(pWriter); + sqlite3_free(hint.a); + return rc; +} + +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. +*/ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ + int rc; + int nMin = (FTS3_MERGE_COUNT / 2); + int nMerge = 0; + const char *z = zParam; + + /* Read the first integer value */ + nMerge = fts3Getint(&z); + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); + } + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + } + sqlite3Fts3SegmentsClose(p); + } + return rc; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; + } + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; + } + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + +/* +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. +*/ +static u64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + u64 ret = (u64)iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; inIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; + + assert( *pRc==SQLITE_OK ); + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; + + i64 iDocid = 0; + i64 iCol = 0; + i64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrnIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + int nText = sqlite3_column_bytes(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + } + + sqlite3_finalize(pStmt); + } + + *pbOk = (cksum1==cksum2); + return rc; +} + +/* +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. +** +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. +*/ +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ +){ + int rc; + int bOk = 0; + rc = fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB; + return rc; +} + +/* +** Handle a 'special' INSERT of the form: +** +** "INSERT INTO tbl(tbl) VALUES()" +** +** Argument pVal contains the result of . Currently the only +** meaningful value to insert is the text 'optimize'. +*/ +static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ + int rc; /* Return Code */ + const char *zVal = (const char *)sqlite3_value_text(pVal); + int nVal = sqlite3_value_bytes(pVal); + + if( !zVal ){ + return SQLITE_NOMEM; + }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ + rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); +#ifdef SQLITE_TEST + }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ + p->nNodeSize = atoi(&zVal[9]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + p->nMaxPendingData = atoi(&zVal[11]); + rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; +#endif + }else{ + rc = SQLITE_ERROR; + } + + return rc; +} + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +/* +** Delete all cached deferred doclists. Deferred doclists are cached +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ + fts3PendingListDelete(pDef->pList); + pDef->pList = 0; + } +} + +/* +** Free all entries in the pCsr->pDeffered list. Entries are added to +** this list using sqlite3Fts3DeferToken(). +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + Fts3DeferredToken *pNext; + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ + pNext = pDef->pNext; + fts3PendingListDelete(pDef->pList); + sqlite3_free(pDef); + } + pCsr->pDeferred = 0; +} + +/* +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list +** based on the row that pCsr currently points to. +** +** A deferred-doclist is like any other doclist with position information +** included, except that it only contains entries for a single row of the +** table, not for all rows. +*/ +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + if( pCsr->pDeferred ){ + int i; /* Used to iterate through table columns */ + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ + + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer *pT = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pT->pModule; + + assert( pCsr->isRequireSeek==0 ); + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); + + for(i=0; inColumn && rc==SQLITE_OK; i++){ + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } + } + } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); + } + } + } + + return rc; +} + +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + +/* +** Add an entry for token pToken to the pCsr->pDeferred list. +*/ +SQLITE_PRIVATE int sqlite3Fts3DeferToken( + Fts3Cursor *pCsr, /* Fts3 table cursor */ + Fts3PhraseToken *pToken, /* Token to defer */ + int iCol /* Column that token must appear in (or -1) */ +){ + Fts3DeferredToken *pDeferred; + pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + if( !pDeferred ){ + return SQLITE_NOMEM; + } + memset(pDeferred, 0, sizeof(*pDeferred)); + pDeferred->pToken = pToken; + pDeferred->pNext = pCsr->pDeferred; + pDeferred->iCol = iCol; + pCsr->pDeferred = pDeferred; + + assert( pToken->pDeferred==0 ); + pToken->pDeferred = pDeferred; + + return SQLITE_OK; +} +#endif + +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ + u32 *aSzDel +){ + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ + + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); + }else{ + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } + } + + return rc; +} + +/* +** This function does the work for the xUpdate method of FTS3 virtual +** tables. The schema of the virtual table being: +** +** CREATE TABLE
    ( +** , +**
    HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** +*/ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( + sqlite3_vtab *pVtab, /* FTS3 vtab object */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return Code */ + int isRemove = 0; /* True for an UPDATE or DELETE */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ + u32 *aSzDel = 0; /* Sizes of deleted documents */ + int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); + + assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); + + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; + } + + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; + } + + /* Allocate space to hold the change in document sizes */ + aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + if( aSzDel==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); + + rc = fts3Writelock(p); + if( rc!=SQLITE_OK ) goto update_out; + + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. + */ + if( nArg>1 && p->zContentTbl==0 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } + + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); + }else{ + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; + } + } + } + if( rc!=SQLITE_OK ){ + goto update_out; + } + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + isRemove = 1; + } + + /* If this is an INSERT or UPDATE operation, insert the new record. */ + if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ + rc = fts3PendingTermsDocid(p, iLangid, *pRowid); + } + if( rc==SQLITE_OK ){ + assert( p->iPrevDocid==*pRowid ); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSzIns); + } + nChng++; + } + + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); + } + + update_out: + sqlite3_free(aSzDel); + sqlite3Fts3SegmentsClose(p); + return rc; +} + +/* +** Flush any data in the pending-terms hash table to disk. If successful, +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. +*/ +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ + int rc; + rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; + }else{ + sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + } + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + +#endif + +/************** End of fts3_write.c ******************************************/ +/************** Begin file fts3_snippet.c ************************************/ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +/* +** Characters that may appear in the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ +#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ +#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ +#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ +#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ +#define FTS3_MATCHINFO_LCS 's' /* nCol values */ +#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ + +/* +** The default value for the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_DEFAULT "pcx" + + +/* +** Used as an fts3ExprIterate() context when loading phrase doclists to +** Fts3Expr.aDoclist[]/nDoclist. +*/ +typedef struct LoadDoclistCtx LoadDoclistCtx; +struct LoadDoclistCtx { + Fts3Cursor *pCsr; /* FTS3 Cursor */ + int nPhrase; /* Number of phrases seen so far */ + int nToken; /* Number of tokens seen so far */ +}; + +/* +** The following types are used as part of the implementation of the +** fts3BestSnippet() routine. +*/ +typedef struct SnippetIter SnippetIter; +typedef struct SnippetPhrase SnippetPhrase; +typedef struct SnippetFragment SnippetFragment; + +struct SnippetIter { + Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ + int iCol; /* Extract snippet from this column */ + int nSnippet; /* Requested snippet length (in tokens) */ + int nPhrase; /* Number of phrases in query */ + SnippetPhrase *aPhrase; /* Array of size nPhrase */ + int iCurrent; /* First token of current snippet */ +}; + +struct SnippetPhrase { + int nToken; /* Number of tokens in phrase */ + char *pList; /* Pointer to start of phrase position list */ + int iHead; /* Next value in position list */ + char *pHead; /* Position list data following iHead */ + int iTail; /* Next value in trailing position list */ + char *pTail; /* Position list data following iTail */ +}; + +struct SnippetFragment { + int iCol; /* Column snippet is extracted from */ + int iPos; /* Index of first token in snippet */ + u64 covered; /* Mask of query phrases covered */ + u64 hlmask; /* Mask of snippet terms to highlight */ +}; + +/* +** This type is used as an fts3ExprIterate() context object while +** accumulating the data returned by the matchinfo() function. +*/ +typedef struct MatchInfo MatchInfo; +struct MatchInfo { + Fts3Cursor *pCursor; /* FTS3 Cursor */ + int nCol; /* Number of columns in table */ + int nPhrase; /* Number of matchable phrases in query */ + sqlite3_int64 nDoc; /* Number of docs in database */ + u32 *aMatchinfo; /* Pre-allocated buffer */ +}; + + + +/* +** The snippet() and offsets() functions both return text values. An instance +** of the following structure is used to accumulate those values while the +** functions are running. See fts3StringAppend() for details. +*/ +typedef struct StrBuffer StrBuffer; +struct StrBuffer { + char *z; /* Pointer to buffer containing string */ + int n; /* Length of z in bytes (excl. nul-term) */ + int nAlloc; /* Allocated size of buffer z in bytes */ +}; + + +/* +** This function is used to help iterate through a position-list. A position +** list is a list of unique integers, sorted from smallest to largest. Each +** element of the list is represented by an FTS3 varint that takes the value +** of the difference between the current element and the previous one plus +** two. For example, to store the position-list: +** +** 4 9 113 +** +** the three varints: +** +** 6 7 106 +** +** are encoded. +** +** When this function is called, *pp points to the start of an element of +** the list. *piPos contains the value of the previous entry in the list. +** After it returns, *piPos contains the value of the next element of the +** list and *pp is advanced to the following varint. +*/ +static void fts3GetDeltaPosition(char **pp, int *piPos){ + int iVal; + *pp += fts3GetVarint32(*pp, &iVal); + *piPos += (iVal-2); +} + +/* +** Helper function for fts3ExprIterate() (see below). +*/ +static int fts3ExprIterate2( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int *piPhrase, /* Pointer to phrase counter */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int rc; /* Return code */ + int eType = pExpr->eType; /* Type of expression node pExpr */ + + if( eType!=FTSQUERY_PHRASE ){ + assert( pExpr->pLeft && pExpr->pRight ); + rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ + rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); + } + }else{ + rc = x(pExpr, *piPhrase, pCtx); + (*piPhrase)++; + } + return rc; +} + +/* +** Iterate through all phrase nodes in an FTS3 query, except those that +** are part of a sub-tree that is the right-hand-side of a NOT operator. +** For each phrase node found, the supplied callback function is invoked. +** +** If the callback function returns anything other than SQLITE_OK, +** the iteration is abandoned and the error code returned immediately. +** Otherwise, SQLITE_OK is returned after a callback has been made for +** all eligible phrase nodes. +*/ +static int fts3ExprIterate( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int iPhrase = 0; /* Variable used as the phrase counter */ + return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); +} + +/* +** This is an fts3ExprIterate() callback used while loading the doclists +** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** fts3ExprLoadDoclists(). +*/ +static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; + LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; + + UNUSED_PARAMETER(iPhrase); + + p->nPhrase++; + p->nToken += pPhrase->nToken; + + return rc; +} + +/* +** Load the doclists for each phrase in the query associated with FTS3 cursor +** pCsr. +** +** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable +** phrases in the expression (all phrases except those directly or +** indirectly descended from the right-hand-side of a NOT operator). If +** pnToken is not NULL, then it is set to the number of tokens in all +** matchable phrases of the expression. +*/ +static int fts3ExprLoadDoclists( + Fts3Cursor *pCsr, /* Fts3 cursor for current query */ + int *pnPhrase, /* OUT: Number of phrases in query */ + int *pnToken /* OUT: Number of tokens in query */ +){ + int rc; /* Return Code */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + sCtx.pCsr = pCsr; + rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + if( pnPhrase ) *pnPhrase = sCtx.nPhrase; + if( pnToken ) *pnToken = sCtx.nToken; + return rc; +} + +static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + (*(int *)ctx)++; + UNUSED_PARAMETER(pExpr); + UNUSED_PARAMETER(iPhrase); + return SQLITE_OK; +} +static int fts3ExprPhraseCount(Fts3Expr *pExpr){ + int nPhrase = 0; + (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + return nPhrase; +} + +/* +** Advance the position list iterator specified by the first two +** arguments so that it points to the first element with a value greater +** than or equal to parameter iNext. +*/ +static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ + char *pIter = *ppIter; + if( pIter ){ + int iIter = *piIter; + + while( iIteriCurrent<0 ){ + /* The SnippetIter object has just been initialized. The first snippet + ** candidate always starts at offset 0 (even if this candidate has a + ** score of 0.0). + */ + pIter->iCurrent = 0; + + /* Advance the 'head' iterator of each phrase to the first offset that + ** is greater than or equal to (iNext+nSnippet). + */ + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); + } + }else{ + int iStart; + int iEnd = 0x7FFFFFFF; + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pHead && pPhrase->iHeadiHead; + } + } + if( iEnd==0x7FFFFFFF ){ + return 1; + } + + pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); + fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); + } + } + + return 0; +} + +/* +** Retrieve information about the current candidate snippet of snippet +** iterator pIter. +*/ +static void fts3SnippetDetails( + SnippetIter *pIter, /* Snippet iterator */ + u64 mCovered, /* Bitmask of phrases already covered */ + int *piToken, /* OUT: First token of proposed snippet */ + int *piScore, /* OUT: "Score" for this snippet */ + u64 *pmCover, /* OUT: Bitmask of phrases covered */ + u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ +){ + int iStart = pIter->iCurrent; /* First token of snippet */ + int iScore = 0; /* Score of this snippet */ + int i; /* Loop counter */ + u64 mCover = 0; /* Mask of phrases covered by this snippet */ + u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pTail ){ + char *pCsr = pPhrase->pTail; + int iCsr = pPhrase->iTail; + + while( iCsr<(iStart+pIter->nSnippet) ){ + int j; + u64 mPhrase = (u64)1 << i; + u64 mPos = (u64)1 << (iCsr - iStart); + assert( iCsr>=iStart ); + if( (mCover|mCovered)&mPhrase ){ + iScore++; + }else{ + iScore += 1000; + } + mCover |= mPhrase; + + for(j=0; jnToken; j++){ + mHighlight |= (mPos>>j); + } + + if( 0==(*pCsr & 0x0FE) ) break; + fts3GetDeltaPosition(&pCsr, &iCsr); + } + } + } + + /* Set the output variables before returning. */ + *piToken = iStart; + *piScore = iScore; + *pmCover = mCover; + *pmHighlight = mHighlight; +} + +/* +** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). +** Each invocation populates an element of the SnippetIter.aPhrase[] array. +*/ +static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ + SnippetIter *p = (SnippetIter *)ctx; + SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; + char *pCsr; + int rc; + + pPhrase->nToken = pExpr->pPhrase->nToken; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( pCsr ){ + int iFirst = 0; + pPhrase->pList = pCsr; + fts3GetDeltaPosition(&pCsr, &iFirst); + assert( iFirst>=0 ); + pPhrase->pHead = pCsr; + pPhrase->pTail = pCsr; + pPhrase->iHead = iFirst; + pPhrase->iTail = iFirst; + }else{ + assert( rc!=SQLITE_OK || ( + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + )); + } + + return rc; +} + +/* +** Select the fragment of text consisting of nFragment contiguous tokens +** from column iCol that represent the "best" snippet. The best snippet +** is the snippet with the highest score, where scores are calculated +** by adding: +** +** (a) +1 point for each occurrence of a matchable phrase in the snippet. +** +** (b) +1000 points for the first occurrence of each matchable phrase in +** the snippet for which the corresponding mCovered bit is not set. +** +** The selected snippet parameters are stored in structure *pFragment before +** returning. The score of the selected snippet is stored in *piScore +** before returning. +*/ +static int fts3BestSnippet( + int nSnippet, /* Desired snippet length */ + Fts3Cursor *pCsr, /* Cursor to create snippet for */ + int iCol, /* Index of column to create snippet from */ + u64 mCovered, /* Mask of phrases already covered */ + u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ + SnippetFragment *pFragment, /* OUT: Best snippet found */ + int *piScore /* OUT: Score of snippet pFragment */ +){ + int rc; /* Return Code */ + int nList; /* Number of phrases in expression */ + SnippetIter sIter; /* Iterates through snippet candidates */ + int nByte; /* Number of bytes of space to allocate */ + int iBestScore = -1; /* Best snippet score found so far */ + int i; /* Loop counter */ + + memset(&sIter, 0, sizeof(sIter)); + + /* Iterate through the phrases in the expression to count them. The same + ** callback makes sure the doclists are loaded for each phrase. + */ + rc = fts3ExprLoadDoclists(pCsr, &nList, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Now that it is known how many phrases there are, allocate and zero + ** the required space using malloc(). + */ + nByte = sizeof(SnippetPhrase) * nList; + sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte); + if( !sIter.aPhrase ){ + return SQLITE_NOMEM; + } + memset(sIter.aPhrase, 0, nByte); + + /* Initialize the contents of the SnippetIter object. Then iterate through + ** the set of phrases in the expression to populate the aPhrase[] array. + */ + sIter.pCsr = pCsr; + sIter.iCol = iCol; + sIter.nSnippet = nSnippet; + sIter.nPhrase = nList; + sIter.iCurrent = -1; + (void)fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter); + + /* Set the *pmSeen output variable. */ + for(i=0; iiCol = iCol; + while( !fts3SnippetNextCandidate(&sIter) ){ + int iPos; + int iScore; + u64 mCover; + u64 mHighlight; + fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover, &mHighlight); + assert( iScore>=0 ); + if( iScore>iBestScore ){ + pFragment->iPos = iPos; + pFragment->hlmask = mHighlight; + pFragment->covered = mCover; + iBestScore = iScore; + } + } + + sqlite3_free(sIter.aPhrase); + *piScore = iBestScore; + return SQLITE_OK; +} + + +/* +** Append a string to the string-buffer passed as the first argument. +** +** If nAppend is negative, then the length of the string zAppend is +** determined using strlen(). +*/ +static int fts3StringAppend( + StrBuffer *pStr, /* Buffer to append to */ + const char *zAppend, /* Pointer to data to append to buffer */ + int nAppend /* Size of zAppend in bytes (or -1) */ +){ + if( nAppend<0 ){ + nAppend = (int)strlen(zAppend); + } + + /* If there is insufficient space allocated at StrBuffer.z, use realloc() + ** to grow the buffer until so that it is big enough to accomadate the + ** appended data. + */ + if( pStr->n+nAppend+1>=pStr->nAlloc ){ + int nAlloc = pStr->nAlloc+nAppend+100; + char *zNew = sqlite3_realloc(pStr->z, nAlloc); + if( !zNew ){ + return SQLITE_NOMEM; + } + pStr->z = zNew; + pStr->nAlloc = nAlloc; + } + assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); + + /* Append the data to the string buffer. */ + memcpy(&pStr->z[pStr->n], zAppend, nAppend); + pStr->n += nAppend; + pStr->z[pStr->n] = '\0'; + + return SQLITE_OK; +} + +/* +** The fts3BestSnippet() function often selects snippets that end with a +** query term. That is, the final term of the snippet is always a term +** that requires highlighting. For example, if 'X' is a highlighted term +** and '.' is a non-highlighted term, BestSnippet() may select: +** +** ........X.....X +** +** This function "shifts" the beginning of the snippet forward in the +** document so that there are approximately the same number of +** non-highlighted terms to the right of the final highlighted term as there +** are to the left of the first highlighted term. For example, to this: +** +** ....X.....X.... +** +** This is done as part of extracting the snippet text, not when selecting +** the snippet. Snippet selection is done based on doclists only, so there +** is no way for fts3BestSnippet() to know whether or not the document +** actually contains terms that follow the final highlighted term. +*/ +static int fts3SnippetShift( + Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ + int nSnippet, /* Number of tokens desired for snippet */ + const char *zDoc, /* Document text to extract snippet from */ + int nDoc, /* Size of buffer zDoc in bytes */ + int *piPos, /* IN/OUT: First token of snippet */ + u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ +){ + u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ + + if( hlmask ){ + int nLeft; /* Tokens to the left of first highlight */ + int nRight; /* Tokens to the right of last highlight */ + int nDesired; /* Ideal number of tokens to shift forward */ + + for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); + for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); + nDesired = (nLeft-nRight)/2; + + /* Ideally, the start of the snippet should be pushed forward in the + ** document nDesired tokens. This block checks if there are actually + ** nDesired tokens to the right of the snippet. If so, *piPos and + ** *pHlMask are updated to shift the snippet nDesired tokens to the + ** right. Otherwise, the snippet is shifted by the number of tokens + ** available. + */ + if( nDesired>0 ){ + int nShift; /* Number of tokens to shift snippet by */ + int iCurrent = 0; /* Token counter */ + int rc; /* Return Code */ + sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer_cursor *pC; + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + + /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) + ** or more tokens in zDoc/nDoc. + */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); + if( rc!=SQLITE_OK ){ + return rc; + } + while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ + const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); + } + pMod->xClose(pC); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } + + nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; + assert( nShift<=nDesired ); + if( nShift>0 ){ + *piPos += nShift; + *pHlmask = hlmask >> nShift; + } + } + } + return SQLITE_OK; +} + +/* +** Extract the snippet text for fragment pFragment from cursor pCsr and +** append it to string buffer pOut. +*/ +static int fts3SnippetText( + Fts3Cursor *pCsr, /* FTS3 Cursor */ + SnippetFragment *pFragment, /* Snippet to extract */ + int iFragment, /* Fragment number */ + int isLast, /* True for final fragment in snippet */ + int nSnippet, /* Number of tokens in extracted snippet */ + const char *zOpen, /* String inserted before highlighted term */ + const char *zClose, /* String inserted after highlighted term */ + const char *zEllipsis, /* String inserted between snippets */ + StrBuffer *pOut /* Write output here */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc; /* Return code */ + const char *zDoc; /* Document text to extract snippet from */ + int nDoc; /* Size of zDoc in bytes */ + int iCurrent = 0; /* Current token number of document */ + int iEnd = 0; /* Byte offset of end of current token */ + int isShiftDone = 0; /* True after snippet is shifted */ + int iPos = pFragment->iPos; /* First token of snippet */ + u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ + int iCol = pFragment->iCol+1; /* Query column to extract text from */ + sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ + + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + return SQLITE_OK; + } + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); + + /* Open a token cursor on the document. */ + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); + if( rc!=SQLITE_OK ){ + return rc; + } + + while( rc==SQLITE_OK ){ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1 = -1; /* Dummy argument used with tokenizer */ + int iBegin = 0; /* Offset in zDoc of start of token */ + int iFin = 0; /* Offset in zDoc of end of token */ + int isHighlight = 0; /* True for highlighted terms */ + + /* Variable DUMMY1 is initialized to a negative value above. Elsewhere + ** in the FTS code the variable that the third argument to xNext points to + ** is initialized to zero before the first (*but not necessarily + ** subsequent*) call to xNext(). This is done for a particular application + ** that needs to know whether or not the tokenizer is being used for + ** snippet generation or for some other purpose. + ** + ** Extreme care is required when writing code to depend on this + ** initialization. It is not a documented part of the tokenizer interface. + ** If a tokenizer is used directly by any code outside of FTS, this + ** convention might not be respected. */ + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + /* Special case - the last token of the snippet is also the last token + ** of the column. Append any punctuation that occurred between the end + ** of the previous token and the end of the document to the output. + ** Then break out of the loop. */ + rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + } + break; + } + if( iCurrentiLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); + isShiftDone = 1; + + /* Now that the shift has been done, check if the initial "..." are + ** required. They are required if (a) this is not the first fragment, + ** or (b) this fragment does not begin at position 0 of its column. + */ + if( rc==SQLITE_OK && (iPos>0 || iFragment>0) ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + } + if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){ + if( isLast ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + } + break; + } + + /* Set isHighlight to true if this term should be highlighted. */ + isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; + + if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); + if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); + + iEnd = iFin; + } + + pMod->xClose(pC); + return rc; +} + + +/* +** This function is used to count the entries in a column-list (a +** delta-encoded list of term offsets within a single column of a single +** row). When this function is called, *ppCollist should point to the +** beginning of the first varint in the column-list (the varint that +** contains the position of the first matching term in the column data). +** Before returning, *ppCollist is set to point to the first byte after +** the last varint in the column-list (either the 0x00 signifying the end +** of the position-list, or the 0x01 that precedes the column number of +** the next column in the position-list). +** +** The number of elements in the column-list is returned. +*/ +static int fts3ColumnlistCount(char **ppCollist){ + char *pEnd = *ppCollist; + char c = 0; + int nEntry = 0; + + /* A column-list is terminated by either a 0x01 or 0x00. */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + if( !c ) nEntry++; + } + + *ppCollist = pEnd; + return nEntry; +} + +/* +** fts3ExprIterate() callback used to collect the "global" matchinfo stats +** for a single query. +** +** fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the +** current query. +** +** Argument pCtx is actually a pointer to a struct of type MatchInfo. This +** function populates Matchinfo.aMatchinfo[] as follows: +** +** for(iCol=0; iColpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); +} + +/* +** fts3ExprIterate() callback used to collect the "local" part of the +** FTS3_MATCHINFO_HITS array. The local stats are those elements of the +** array that are different for each row returned by the query. +*/ +static int fts3ExprLocalHitsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + int rc = SQLITE_OK; + MatchInfo *p = (MatchInfo *)pCtx; + int iStart = iPhrase * p->nCol * 3; + int i; + + for(i=0; inCol && rc==SQLITE_OK; i++){ + char *pCsr; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); + if( pCsr ){ + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; + } + } + + return rc; +} + +static int fts3MatchinfoCheck( + Fts3Table *pTab, + char cArg, + char **pzErr +){ + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) + ){ + return SQLITE_OK; + } + *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg); + return SQLITE_ERROR; +} + +static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + int nVal; /* Number of integers output by cArg */ + + switch( cArg ){ + case FTS3_MATCHINFO_NDOC: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: + nVal = 1; + break; + + case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_LENGTH: + case FTS3_MATCHINFO_LCS: + nVal = pInfo->nCol; + break; + + default: + assert( cArg==FTS3_MATCHINFO_HITS ); + nVal = pInfo->nCol * pInfo->nPhrase * 3; + break; + } + + return nVal; +} + +static int fts3MatchinfoSelectDoctotal( + Fts3Table *pTab, + sqlite3_stmt **ppStmt, + sqlite3_int64 *pnDoc, + const char **paLen +){ + sqlite3_stmt *pStmt; + const char *a; + sqlite3_int64 nDoc; + + if( !*ppStmt ){ + int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); + if( rc!=SQLITE_OK ) return rc; + } + pStmt = *ppStmt; + assert( sqlite3_data_count(pStmt)==1 ); + + a = sqlite3_column_blob(pStmt, 0); + a += sqlite3Fts3GetVarint(a, &nDoc); + if( nDoc==0 ) return FTS_CORRUPT_VTAB; + *pnDoc = (u32)nDoc; + + if( paLen ) *paLen = a; + return SQLITE_OK; +} + +/* +** An instance of the following structure is used to store state while +** iterating through a multi-column position-list corresponding to the +** hits for a single phrase on a single row in order to calculate the +** values for a matchinfo() FTS3_MATCHINFO_LCS request. +*/ +typedef struct LcsIterator LcsIterator; +struct LcsIterator { + Fts3Expr *pExpr; /* Pointer to phrase expression */ + int iPosOffset; /* Tokens count up to end of this phrase */ + char *pRead; /* Cursor used to iterate through aDoclist */ + int iPos; /* Current position */ +}; + +/* +** If LcsIterator.iCol is set to the following value, the iterator has +** finished iterating through all offsets for all columns. +*/ +#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; + +static int fts3MatchinfoLcsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + LcsIterator *aIter = (LcsIterator *)pCtx; + aIter[iPhrase].pExpr = pExpr; + return SQLITE_OK; +} + +/* +** Advance the iterator passed as an argument to the next position. Return +** 1 if the iterator is at EOF or if it now points to the start of the +** position list for the next column. +*/ +static int fts3LcsIteratorAdvance(LcsIterator *pIter){ + char *pRead = pIter->pRead; + sqlite3_int64 iRead; + int rc = 0; + + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + if( iRead==0 || iRead==1 ){ + pRead = 0; + rc = 1; + }else{ + pIter->iPos += (int)(iRead-2); + } + + pIter->pRead = pRead; + return rc; +} + +/* +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. +** +** If the call is successful, the longest-common-substring lengths for each +** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** array before returning. SQLITE_OK is returned in this case. +** +** Otherwise, if an error occurs, an SQLite error code is returned and the +** data written to the first nCol elements of pInfo->aMatchinfo[] is +** undefined. +*/ +static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ + LcsIterator *aIter; + int i; + int iCol; + int nToken = 0; + + /* Allocate and populate the array of LcsIterator objects. The array + ** contains one element for each matchable phrase in the query. + **/ + aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); + if( !aIter ) return SQLITE_NOMEM; + memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); + (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + nToken -= pIter->pExpr->pPhrase->nToken; + pIter->iPosOffset = nToken; + } + + for(iCol=0; iColnCol; iCol++){ + int nLcs = 0; /* LCS value for this column */ + int nLive = 0; /* Number of iterators in aIter not at EOF */ + + for(i=0; inPhrase; i++){ + int rc; + LcsIterator *pIt = &aIter[i]; + rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); + if( rc!=SQLITE_OK ) return rc; + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + nLive++; + } + } + + while( nLive>0 ){ + LcsIterator *pAdv = 0; /* The iterator to advance by one position */ + int nThisLcs = 0; /* LCS for the current iterator positions */ + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + if( pIter->pRead==0 ){ + /* This iterator is already at EOF for this column. */ + nThisLcs = 0; + }else{ + if( pAdv==0 || pIter->iPosiPos ){ + pAdv = pIter; + } + if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ + nThisLcs++; + }else{ + nThisLcs = 1; + } + if( nThisLcs>nLcs ) nLcs = nThisLcs; + } + } + if( fts3LcsIteratorAdvance(pAdv) ) nLive--; + } + + pInfo->aMatchinfo[iCol] = nLcs; + } + + sqlite3_free(aIter); + return SQLITE_OK; +} + +/* +** Populate the buffer pInfo->aMatchinfo[] with an array of integers to +** be returned by the matchinfo() function. Argument zArg contains the +** format string passed as the second argument to matchinfo (or the +** default value "pcx" if no second argument was specified). The format +** string has already been validated and the pInfo->aMatchinfo[] array +** is guaranteed to be large enough for the output. +** +** If bGlobal is true, then populate all fields of the matchinfo() output. +** If it is false, then assume that those fields that do not change between +** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) +** have already been populated. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. If a value other than SQLITE_OK is returned, the state the +** pInfo->aMatchinfo[] buffer is left in is undefined. +*/ +static int fts3MatchinfoValues( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + int bGlobal, /* True to grab the global stats */ + MatchInfo *pInfo, /* Matchinfo context object */ + const char *zArg /* Matchinfo format string */ +){ + int rc = SQLITE_OK; + int i; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_stmt *pSelect = 0; + + for(i=0; rc==SQLITE_OK && zArg[i]; i++){ + + switch( zArg[i] ){ + case FTS3_MATCHINFO_NPHRASE: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_NCOL: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; + break; + + case FTS3_MATCHINFO_NDOC: + if( bGlobal ){ + sqlite3_int64 nDoc = 0; + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); + pInfo->aMatchinfo[0] = (u32)nDoc; + } + break; + + case FTS3_MATCHINFO_AVGLENGTH: + if( bGlobal ){ + sqlite3_int64 nDoc; /* Number of rows in table */ + const char *a; /* Aggregate column length array */ + + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); + if( rc==SQLITE_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + u32 iVal; + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + pInfo->aMatchinfo[iCol] = iVal; + } + } + } + break; + + case FTS3_MATCHINFO_LENGTH: { + sqlite3_stmt *pSelectDocsize = 0; + rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); + if( rc==SQLITE_OK ){ + int iCol; + const char *a = sqlite3_column_blob(pSelectDocsize, 0); + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + pInfo->aMatchinfo[iCol] = (u32)nToken; + } + } + sqlite3_reset(pSelectDocsize); + break; + } + + case FTS3_MATCHINFO_LCS: + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3MatchinfoLcs(pCsr, pInfo); + } + break; + + default: { + Fts3Expr *pExpr; + assert( zArg[i]==FTS3_MATCHINFO_HITS ); + pExpr = pCsr->pExpr; + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc!=SQLITE_OK ) break; + if( bGlobal ){ + if( pCsr->pDeferred ){ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); + if( rc!=SQLITE_OK ) break; + } + rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + if( rc!=SQLITE_OK ) break; + } + (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + break; + } + } + + pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); + } + + sqlite3_reset(pSelect); + return rc; +} + + +/* +** Populate pCsr->aMatchinfo[] with data for the current row. The +** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). +*/ +static int fts3GetMatchinfo( + Fts3Cursor *pCsr, /* FTS3 Cursor object */ + const char *zArg /* Second argument to matchinfo() function */ +){ + MatchInfo sInfo; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int bGlobal = 0; /* Collect 'global' stats as well as local */ + + memset(&sInfo, 0, sizeof(MatchInfo)); + sInfo.pCursor = pCsr; + sInfo.nCol = pTab->nColumn; + + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard + ** the cached data. */ + if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){ + assert( pCsr->aMatchinfo ); + sqlite3_free(pCsr->aMatchinfo); + pCsr->zMatchinfo = 0; + pCsr->aMatchinfo = 0; + } + + /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. + */ + if( pCsr->aMatchinfo==0 ){ + int nMatchinfo = 0; /* Number of u32 elements in match-info */ + int nArg; /* Bytes in zArg */ + int i; /* Used to iterate through zArg */ + + /* Determine the number of phrases in the query */ + pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); + sInfo.nPhrase = pCsr->nPhrase; + + /* Determine the number of integers in the buffer returned by this call. */ + for(i=0; zArg[i]; i++){ + nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); + } + + /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ + nArg = (int)strlen(zArg); + pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1); + if( !pCsr->aMatchinfo ) return SQLITE_NOMEM; + + pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo]; + pCsr->nMatchinfo = nMatchinfo; + memcpy(pCsr->zMatchinfo, zArg, nArg+1); + memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo); + pCsr->isMatchinfoNeeded = 1; + bGlobal = 1; + } + + sInfo.aMatchinfo = pCsr->aMatchinfo; + sInfo.nPhrase = pCsr->nPhrase; + if( pCsr->isMatchinfoNeeded ){ + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); + pCsr->isMatchinfoNeeded = 0; + } + + return rc; +} + +/* +** Implementation of snippet() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Snippet( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr, /* Cursor object */ + const char *zStart, /* Snippet start text - "" */ + const char *zEnd, /* Snippet end text - "" */ + const char *zEllipsis, /* Snippet ellipsis text - "..." */ + int iCol, /* Extract snippet from this column */ + int nToken /* Approximate number of tokens in snippet */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int i; + StrBuffer res = {0, 0, 0}; + + /* The returned text includes up to four fragments of text extracted from + ** the data in the current row. The first iteration of the for(...) loop + ** below attempts to locate a single fragment of text nToken tokens in + ** size that contains at least one instance of all phrases in the query + ** expression that appear in the current row. If such a fragment of text + ** cannot be found, the second iteration of the loop attempts to locate + ** a pair of fragments, and so on. + */ + int nSnippet = 0; /* Number of fragments in this snippet */ + SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ + int nFToken = -1; /* Number of tokens in each fragment */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + for(nSnippet=1; 1; nSnippet++){ + + int iSnip; /* Loop counter 0..nSnippet-1 */ + u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ + u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ + + if( nToken>=0 ){ + nFToken = (nToken+nSnippet-1) / nSnippet; + }else{ + nFToken = -1 * nToken; + } + + for(iSnip=0; iSnipnColumn; iRead++){ + SnippetFragment sF = {0, 0, 0, 0}; + int iS; + if( iCol>=0 && iRead!=iCol ) continue; + + /* Find the best snippet of nFToken tokens in column iRead. */ + rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); + if( rc!=SQLITE_OK ){ + goto snippet_out; + } + if( iS>iBestScore ){ + *pFragment = sF; + iBestScore = iS; + } + } + + mCovered |= pFragment->covered; + } + + /* If all query phrases seen by fts3BestSnippet() are present in at least + ** one of the nSnippet snippet fragments, break out of the loop. + */ + assert( (mCovered&mSeen)==mCovered ); + if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; + } + + assert( nFToken>0 ); + + for(i=0; ipCsr, pExpr, p->iCol, &pList); + nTerm = pExpr->pPhrase->nToken; + if( pList ){ + fts3GetDeltaPosition(&pList, &iPos); + assert( iPos>=0 ); + } + + for(iTerm=0; iTermaTerm[p->iTerm++]; + pT->iOff = nTerm-iTerm-1; + pT->pList = pList; + pT->iPos = iPos; + } + + return rc; +} + +/* +** Implementation of offsets() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; + int rc; /* Return Code */ + int nToken; /* Number of tokens in query */ + int iCol; /* Column currently being processed */ + StrBuffer res = {0, 0, 0}; /* Result string */ + TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + memset(&sCtx, 0, sizeof(sCtx)); + assert( pCsr->isRequireSeek==0 ); + + /* Count the number of terms in the query */ + rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); + if( rc!=SQLITE_OK ) goto offsets_out; + + /* Allocate the array of TermOffset iterators. */ + sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken); + if( 0==sCtx.aTerm ){ + rc = SQLITE_NOMEM; + goto offsets_out; + } + sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; + + /* Loop through the table columns, appending offset information to + ** string-buffer res for each column. + */ + for(iCol=0; iColnColumn; iCol++){ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY = 0; /* Dummy argument used with xNext() */ + int iStart = 0; + int iEnd = 0; + int iCurrent = 0; + const char *zDoc; + int nDoc; + + /* Initialize the contents of sCtx.aTerm[] for column iCol. There is + ** no way that this operation can fail, so the return code from + ** fts3ExprIterate() can be discarded. + */ + sCtx.iCol = iCol; + sCtx.iTerm = 0; + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx); + + /* Retreive the text stored in column iCol. If an SQL NULL is stored + ** in column iCol, jump immediately to the next iteration of the loop. + ** If an OOM occurs while retrieving the data (this can happen if SQLite + ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM + ** to the caller. + */ + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ + continue; + } + rc = SQLITE_NOMEM; + goto offsets_out; + } + + /* Initialize a tokenizer iterator to iterate through column iCol. */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); + if( rc!=SQLITE_OK ) goto offsets_out; + + rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + while( rc==SQLITE_OK ){ + int i; /* Used to loop through terms */ + int iMinPos = 0x7FFFFFFF; /* Position of next token */ + TermOffset *pTerm = 0; /* TermOffset associated with next token */ + + for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; + pTerm = pT; + } + } + + if( !pTerm ){ + /* All offsets for this column have been gathered. */ + rc = SQLITE_DONE; + }else{ + assert( iCurrent<=iMinPos ); + if( 0==(0xFE&*pTerm->pList) ){ + pTerm->pList = 0; + }else{ + fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); + } + while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + } + if( rc==SQLITE_OK ){ + char aBuffer[64]; + sqlite3_snprintf(sizeof(aBuffer), aBuffer, + "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart + ); + rc = fts3StringAppend(&res, aBuffer, -1); + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pMod->xClose(pC); + if( rc!=SQLITE_OK ) goto offsets_out; + } + + offsets_out: + sqlite3_free(sCtx.aTerm); + assert( rc!=SQLITE_DONE ); + sqlite3Fts3SegmentsClose(pTab); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(res.z); + }else{ + sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); + } + return; +} + +/* +** Implementation of matchinfo() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Matchinfo( + sqlite3_context *pContext, /* Function call context */ + Fts3Cursor *pCsr, /* FTS3 table cursor */ + const char *zArg /* Second arg to matchinfo() function */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc; + int i; + const char *zFormat; + + if( zArg ){ + for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + return; + } + } + zFormat = zArg; + }else{ + zFormat = FTS3_MATCHINFO_DEFAULT; + } + + if( !pCsr->pExpr ){ + sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); + return; + } + + /* Retrieve matchinfo() data. */ + rc = fts3GetMatchinfo(pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pContext, rc); + }else{ + int n = pCsr->nMatchinfo * sizeof(u32); + sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); + } +} + +#endif + +/************** End of fts3_snippet.c ****************************************/ +/************** Begin file fts3_unicode.c ************************************/ +/* +** 2012 May 24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "unicode" full-text-search tokenizer. +*/ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + + +/* +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. +*/ +#ifndef SQLITE_AMALGAMATION + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} + +#endif /* ifndef SQLITE_AMALGAMATION */ + +typedef struct unicode_tokenizer unicode_tokenizer; +typedef struct unicode_cursor unicode_cursor; + +struct unicode_tokenizer { + sqlite3_tokenizer base; + int bRemoveDiacritic; + int nException; + int *aiException; +}; + +struct unicode_cursor { + sqlite3_tokenizer_cursor base; + const unsigned char *aInput; /* Input text being tokenized */ + int nInput; /* Size of aInput[] in bytes */ + int iOff; /* Current offset within aInput[] */ + int iToken; /* Index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAlloc; /* space allocated at zToken */ +}; + + +/* +** Destroy a tokenizer allocated by unicodeCreate(). +*/ +static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ + if( pTokenizer ){ + unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; + sqlite3_free(p->aiException); + sqlite3_free(p); + } + return SQLITE_OK; +} + +/* +** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE +** statement has specified that the tokenizer for this table shall consider +** all characters in string zIn/nIn to be separators (if bAlnum==0) or +** token characters (if bAlnum==1). +** +** For each codepoint in the zIn/nIn string, this function checks if the +** sqlite3FtsUnicodeIsalnum() function already returns the desired result. +** If so, no action is taken. Otherwise, the codepoint is added to the +** unicode_tokenizer.aiException[] array. For the purposes of tokenization, +** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all +** codepoints in the aiException[] array. +** +** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() +** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. +** It is not possible to change the behavior of the tokenizer with respect +** to these codepoints. +*/ +static int unicodeAddExceptions( + unicode_tokenizer *p, /* Tokenizer to add exceptions to */ + int bAlnum, /* Replace Isalnum() return value with this */ + const char *zIn, /* Array of characters to make exceptions */ + int nIn /* Length of z in bytes */ +){ + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zTerm = &z[nIn]; + int iCode; + int nEntry = 0; + + assert( bAlnum==0 || bAlnum==1 ); + + while( zaiException, (p->nException+nEntry)*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + nNew = p->nException; + + z = (const unsigned char *)zIn; + while( zi; j--) aNew[j] = aNew[j-1]; + aNew[i] = iCode; + nNew++; + } + } + p->aiException = aNew; + p->nException = nNew; + } + + return SQLITE_OK; +} + +/* +** Return true if the p->aiException[] array contains the value iCode. +*/ +static int unicodeIsException(unicode_tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + } + + return 0; +} + +/* +** Return true if, for the purposes of tokenization, codepoint iCode is +** considered a token character (not a separator). +*/ +static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ + assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); + return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); +} + +/* +** Create a new tokenizer instance. +*/ +static int unicodeCreate( + int nArg, /* Size of array argv[] */ + const char * const *azArg, /* Tokenizer creation arguments */ + sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ +){ + unicode_tokenizer *pNew; /* New tokenizer object */ + int i; + int rc = SQLITE_OK; + + pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); + if( pNew==NULL ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(unicode_tokenizer)); + pNew->bRemoveDiacritic = 1; + + for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; + } + else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ + pNew->bRemoveDiacritic = 0; + } + else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); + } + else if( n>=11 && memcmp("separators=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); + } + else{ + /* Unrecognized argument */ + rc = SQLITE_ERROR; + } + } + + if( rc!=SQLITE_OK ){ + unicodeDestroy((sqlite3_tokenizer *)pNew); + pNew = 0; + } + *pp = (sqlite3_tokenizer *)pNew; + return rc; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int unicodeOpen( + sqlite3_tokenizer *p, /* The tokenizer */ + const char *aInput, /* Input string */ + int nInput, /* Size of string aInput in bytes */ + sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ +){ + unicode_cursor *pCsr; + + pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(unicode_cursor)); + + pCsr->aInput = (const unsigned char *)aInput; + if( aInput==0 ){ + pCsr->nInput = 0; + }else if( nInput<0 ){ + pCsr->nInput = (int)strlen(aInput); + }else{ + pCsr->nInput = nInput; + } + + *pp = &pCsr->base; + UNUSED_PARAMETER(p); + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ + unicode_cursor *pCsr = (unicode_cursor *) pCursor; + sqlite3_free(pCsr->zToken); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int unicodeNext( + sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ + const char **paToken, /* OUT: Token text */ + int *pnToken, /* OUT: Number of bytes at *paToken */ + int *piStart, /* OUT: Starting offset of token */ + int *piEnd, /* OUT: Ending offset of token */ + int *piPos /* OUT: Position integer of token */ +){ + unicode_cursor *pCsr = (unicode_cursor *)pC; + unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); + int iCode = 0; + char *zOut; + const unsigned char *z = &pCsr->aInput[pCsr->iOff]; + const unsigned char *zStart = z; + const unsigned char *zEnd; + const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; + + /* Scan past any delimiter characters before the start of the next token. + ** Return SQLITE_DONE early if this takes us all the way to the end of + ** the input. */ + while( z=zTerm ) return SQLITE_DONE; + + zOut = pCsr->zToken; + do { + int iOut; + + /* Grow the output buffer if required. */ + if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ + char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64); + if( !zNew ) return SQLITE_NOMEM; + zOut = &zNew[zOut - pCsr->zToken]; + pCsr->zToken = zNew; + pCsr->nAlloc += 64; + } + + /* Write the folded case of the last character read to the output */ + zEnd = z; + iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); + if( iOut ){ + WRITE_UTF8(zOut, iOut); + } + + /* If the cursor is not at EOF, read the next character */ + if( z>=zTerm ) break; + READ_UTF8(z, zTerm, iCode); + }while( unicodeIsAlnum(p, iCode) + || sqlite3FtsUnicodeIsdiacritic(iCode) + ); + + /* Set the output variables and return. */ + pCsr->iOff = (int)(z - pCsr->aInput); + *paToken = pCsr->zToken; + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); + *piPos = pCsr->iToken++; + return SQLITE_OK; +} + +/* +** Set *ppModule to a pointer to the sqlite3_tokenizer_module +** structure for the unicode tokenizer. +*/ +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ + static const sqlite3_tokenizer_module module = { + 0, + unicodeCreate, + unicodeDestroy, + unicodeOpen, + unicodeClose, + unicodeNext, + 0, + }; + *ppModule = &module; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ + +/************** End of fts3_unicode.c ****************************************/ +/************** Begin file fts3_unicode2.c ***********************************/ +/* +** 2012 May 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* +** DO NOT EDIT THIS MACHINE GENERATED FILE. +*/ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + +/* #include */ + +/* +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + static const unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; + + if( c<128 ){ + return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); + }else if( c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); + } + return 1; +} + + +/* +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. +*/ +static int remove_diacritic(int c){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, + 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, + 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, + 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, + 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, + 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, + 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, + 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, + 62924, 63050, 63082, 63274, 63390, + }; + char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', + 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', + 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', + 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', + 'e', 'i', 'o', 'u', 'y', + }; + + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( key>=aDia[iRes] ); + return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); +} + + +/* +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & (1 << (c-768))) : + (mask1 & (1 << (c-768-32))); +} + + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; + + int ret = c; + + assert( c>=0 ); + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( iRes<0 || c>=aEntry[iRes].iCode ); + + if( iRes>=0 ){ + const struct TableEntry *p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + } + + if( bRemoveDiacritic ) ret = remove_diacritic(ret); + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; + } + + return ret; +} +#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ + +/************** End of fts3_unicode2.c ***************************************/ +/************** Begin file rtree.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code for implementations of the r-tree and r*-tree +** algorithms packaged as an SQLite virtual table module. +*/ + +/* +** Database Format of R-Tree Tables +** -------------------------------- +** +** The data structure for a single virtual r-tree table is stored in three +** native SQLite tables declared as follows. In each case, the '%' character +** in the table name is replaced with the user-supplied name of the r-tree +** table. +** +** CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB) +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER) +** +** The data for each node of the r-tree structure is stored in the %_node +** table. For each node that is not the root node of the r-tree, there is +** an entry in the %_parent table associating the node with its parent. +** And for each row of data in the table, there is an entry in the %_rowid +** table that maps from the entries rowid to the id of the node that it +** is stored on. +** +** The root node of an r-tree always exists, even if the r-tree table is +** empty. The nodeno of the root node is always 1. All other nodes in the +** table must be the same size as the root node. The content of each node +** is formatted as follows: +** +** 1. If the node is the root node (node 1), then the first 2 bytes +** of the node contain the tree depth as a big-endian integer. +** For non-root nodes, the first 2 bytes are left unused. +** +** 2. The next 2 bytes contain the number of entries currently +** stored in the node. +** +** 3. The remainder of the node contains the node entries. Each entry +** consists of a single 8-byte integer followed by an even number +** of 4-byte coordinates. For leaf nodes the integer is the rowid +** of a record. For internal nodes it is the node number of a +** child page. +*/ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) + +#ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT1 +#else +#endif + +/* #include */ +/* #include */ +/* #include */ + +#ifndef SQLITE_AMALGAMATION +#include "sqlite3rtree.h" +typedef sqlite3_int64 i64; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +#endif + +/* The following macro is used to suppress compiler warnings. +*/ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(x) (void)(x) +#endif + +typedef struct Rtree Rtree; +typedef struct RtreeCursor RtreeCursor; +typedef struct RtreeNode RtreeNode; +typedef struct RtreeCell RtreeCell; +typedef struct RtreeConstraint RtreeConstraint; +typedef struct RtreeMatchArg RtreeMatchArg; +typedef struct RtreeGeomCallback RtreeGeomCallback; +typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; + +/* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ +#define RTREE_MAX_DIMENSIONS 5 + +/* Size of hash table Rtree.aHash. This hash table is not expected to +** ever contain very many entries, so a fixed number of buckets is +** used. +*/ +#define HASHSIZE 97 + +/* The xBestIndex method of this virtual table requires an estimate of +** the number of rows in the virtual table to calculate the costs of +** various strategies. If possible, this estimate is loaded from the +** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). +** Otherwise, if no sqlite_stat1 entry is available, use +** RTREE_DEFAULT_ROWEST. +*/ +#define RTREE_DEFAULT_ROWEST 1048576 +#define RTREE_MIN_ROWEST 100 + +/* +** An rtree virtual-table object. +*/ +struct Rtree { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* Host database connection */ + int iNodeSize; /* Size in bytes of each node in the node table */ + u8 nDim; /* Number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ + int iDepth; /* Current depth of the r-tree structure */ + char *zDb; /* Name of database containing r-tree table */ + char *zName; /* Name of r-tree table */ + int nBusy; /* Current number of users of this structure */ + i64 nRowEst; /* Estimated number of rows in this table */ + + /* List of nodes removed during a CondenseTree operation. List is + ** linked together via the pointer normally used for hash chains - + ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree + ** headed by the node (leaf nodes have RtreeNode.iNode==0). + */ + RtreeNode *pDeleted; + int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ + + /* Statements to read/write/delete a record from xxx_node */ + sqlite3_stmt *pReadNode; + sqlite3_stmt *pWriteNode; + sqlite3_stmt *pDeleteNode; + + /* Statements to read/write/delete a record from xxx_rowid */ + sqlite3_stmt *pReadRowid; + sqlite3_stmt *pWriteRowid; + sqlite3_stmt *pDeleteRowid; + + /* Statements to read/write/delete a record from xxx_parent */ + sqlite3_stmt *pReadParent; + sqlite3_stmt *pWriteParent; + sqlite3_stmt *pDeleteParent; + + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ +}; + +/* Possible values for Rtree.eCoordType: */ +#define RTREE_COORD_REAL32 0 +#define RTREE_COORD_INT32 1 + +/* +** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will +** only deal with integer coordinates. No floating point operations +** will be done. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ + typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 +#else + typedef double RtreeDValue; /* High accuracy coordinate */ + typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 +#endif + +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. +** +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. +*/ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ +}; + +/* +** The minimum number of cells allowed for a node is a third of the +** maximum. In Gutman's notation: +** +** m = M/3 +** +** If an R*-tree "Reinsert" operation is required, the same number of +** cells are removed from the overfull node and reinserted into the tree. +*/ +#define RTREE_MINCELLS(p) ((((p)->iNodeSize-4)/(p)->nBytesPerCell)/3) +#define RTREE_REINSERT(p) RTREE_MINCELLS(p) +#define RTREE_MAXCELLS 51 + +/* +** The smallest possible node-size is (512-64)==448 bytes. And the largest +** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). +** Therefore all non-root nodes must contain at least 3 entries. Since +** 2^40 is greater than 2^64, an r-tree structure always has a depth of +** 40 or less. +*/ +#define RTREE_MAX_DEPTH 40 + + +/* +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. +*/ +#define RTREE_CACHE_SZ 5 + +/* +** An rtree cursor object. +*/ +struct RtreeCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ + int iStrategy; /* Copy of idxNum search parameter */ + int nConstraint; /* Number of entries in aConstraint */ + RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ +}; + +/* Return the Rtree of a RtreeCursor */ +#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) + +/* +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. +*/ +union RtreeCoord { + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ +}; + +/* +** The argument is an RtreeCoord. Return the value stored within the RtreeCoord +** formatted as a RtreeDValue (double or int64). This macro assumes that local +** variable pRtree points to the Rtree structure associated with the +** RtreeCoord. +*/ +#ifdef SQLITE_RTREE_INT_ONLY +# define DCOORD(coord) ((RtreeDValue)coord.i) +#else +# define DCOORD(coord) ( \ + (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ + ((double)coord.f) : \ + ((double)coord.i) \ + ) +#endif + +/* +** A search constraint. +*/ +struct RtreeConstraint { + int iCoord; /* Index of constrained coordinate */ + int op; /* Constraining operation */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ +}; + +/* Possible values for RtreeConstraint.op */ +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ + + +/* +** An rtree structure node. +*/ +struct RtreeNode { + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ +}; + +/* Return the number of cells in a node */ +#define NCELL(pNode) readInt16(&(pNode)->zData[2]) + +/* +** A single cell from a node, deserialized +*/ +struct RtreeCell { + i64 iRowid; /* Node or entry ID */ + RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ +}; + + +/* +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. +** +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; +}; + + +/* +** Value for the first field of every RtreeMatchArg object. The MATCH +** operator tests that the first field of a blob operand matches this +** value to avoid operating on invalid blobs (which could cause a segfault). +*/ +#define RTREE_GEOMETRY_MAGIC 0x891245AB + +/* +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. +*/ +struct RtreeMatchArg { + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ +}; + +#ifndef MAX +# define MAX(x,y) ((x) < (y) ? (y) : (x)) +#endif +#ifndef MIN +# define MIN(x,y) ((x) > (y) ? (y) : (x)) +#endif + +/* +** Functions to deserialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The deserialized value is returned. +*/ +static int readInt16(u8 *p){ + return (p[0]<<8) + p[1]; +} +static void readCoord(u8 *p, RtreeCoord *pCoord){ + u32 i = ( + (((u32)p[0]) << 24) + + (((u32)p[1]) << 16) + + (((u32)p[2]) << 8) + + (((u32)p[3]) << 0) + ); + *(u32 *)pCoord = i; +} +static i64 readInt64(u8 *p){ + return ( + (((i64)p[0]) << 56) + + (((i64)p[1]) << 48) + + (((i64)p[2]) << 40) + + (((i64)p[3]) << 32) + + (((i64)p[4]) << 24) + + (((i64)p[5]) << 16) + + (((i64)p[6]) << 8) + + (((i64)p[7]) << 0) + ); +} + +/* +** Functions to serialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The value returned is the number of bytes written +** to the argument buffer (always 2, 4 and 8 respectively). +*/ +static int writeInt16(u8 *p, int i){ + p[0] = (i>> 8)&0xFF; + p[1] = (i>> 0)&0xFF; + return 2; +} +static int writeCoord(u8 *p, RtreeCoord *pCoord){ + u32 i; + assert( sizeof(RtreeCoord)==4 ); + assert( sizeof(u32)==4 ); + i = *(u32 *)pCoord; + p[0] = (i>>24)&0xFF; + p[1] = (i>>16)&0xFF; + p[2] = (i>> 8)&0xFF; + p[3] = (i>> 0)&0xFF; + return 4; +} +static int writeInt64(u8 *p, i64 i){ + p[0] = (i>>56)&0xFF; + p[1] = (i>>48)&0xFF; + p[2] = (i>>40)&0xFF; + p[3] = (i>>32)&0xFF; + p[4] = (i>>24)&0xFF; + p[5] = (i>>16)&0xFF; + p[6] = (i>> 8)&0xFF; + p[7] = (i>> 0)&0xFF; + return 8; +} + +/* +** Increment the reference count of node p. +*/ +static void nodeReference(RtreeNode *p){ + if( p ){ + p->nRef++; + } +} + +/* +** Clear the content of node p (set all bytes to 0x00). +*/ +static void nodeZero(Rtree *pRtree, RtreeNode *p){ + memset(&p->zData[2], 0, pRtree->iNodeSize-2); + p->isDirty = 1; +} + +/* +** Given a node number iNode, return the corresponding key to use +** in the Rtree.aHash table. +*/ +static int nodeHash(i64 iNode){ + return iNode % HASHSIZE; +} + +/* +** Search the node hash table for node iNode. If found, return a pointer +** to it. Otherwise, return 0. +*/ +static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ + RtreeNode *p; + for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); + return p; +} + +/* +** Add node pNode to the node hash table. +*/ +static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ + int iHash; + assert( pNode->pNext==0 ); + iHash = nodeHash(pNode->iNode); + pNode->pNext = pRtree->aHash[iHash]; + pRtree->aHash[iHash] = pNode; +} + +/* +** Remove node pNode from the node hash table. +*/ +static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode **pp; + if( pNode->iNode!=0 ){ + pp = &pRtree->aHash[nodeHash(pNode->iNode)]; + for( ; (*pp)!=pNode; pp = &(*pp)->pNext){ assert(*pp); } + *pp = pNode->pNext; + pNode->pNext = 0; + } +} + +/* +** Allocate and return new r-tree node. Initially, (RtreeNode.iNode==0), +** indicating that node has not yet been assigned a node number. It is +** assigned a node number when nodeWrite() is called to write the +** node contents out to the database. +*/ +static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ + RtreeNode *pNode; + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); + if( pNode ){ + memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->pParent = pParent; + pNode->isDirty = 1; + nodeReference(pParent); + } + return pNode; +} + +/* +** Obtain a reference to an r-tree node. +*/ +static int nodeAcquire( + Rtree *pRtree, /* R-tree structure */ + i64 iNode, /* Node number to load */ + RtreeNode *pParent, /* Either the parent node or NULL */ + RtreeNode **ppNode /* OUT: Acquired node */ +){ + int rc; + int rc2 = SQLITE_OK; + RtreeNode *pNode; + + /* Check if the requested node is already in the hash table. If so, + ** increase its reference count and return it. + */ + if( (pNode = nodeHashLookup(pRtree, iNode)) ){ + assert( !pParent || !pNode->pParent || pNode->pParent==pParent ); + if( pParent && !pNode->pParent ){ + nodeReference(pParent); + pNode->pParent = pParent; + } + pNode->nRef++; + *ppNode = pNode; + return SQLITE_OK; + } + + sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); + rc = sqlite3_step(pRtree->pReadNode); + if( rc==SQLITE_ROW ){ + const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); + if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc2 = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + memcpy(pNode->zData, zBlob, pRtree->iNodeSize); + nodeReference(pParent); + } + } + } + rc = sqlite3_reset(pRtree->pReadNode); + if( rc==SQLITE_OK ) rc = rc2; + + /* If the root node was just loaded, set pRtree->iDepth to the height + ** of the r-tree structure. A height of zero means all data is stored on + ** the root node. A height of one means the children of the root node + ** are the leaves, and so on. If the depth as specified on the root node + ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. + */ + if( pNode && iNode==1 ){ + pRtree->iDepth = readInt16(pNode->zData); + if( pRtree->iDepth>RTREE_MAX_DEPTH ){ + rc = SQLITE_CORRUPT_VTAB; + } + } + + /* If no error has occurred so far, check if the "number of entries" + ** field on the node is too large. If so, set the return code to + ** SQLITE_CORRUPT_VTAB. + */ + if( pNode && rc==SQLITE_OK ){ + if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ + rc = SQLITE_CORRUPT_VTAB; + } + } + + if( rc==SQLITE_OK ){ + if( pNode!=0 ){ + nodeHashInsert(pRtree, pNode); + }else{ + rc = SQLITE_CORRUPT_VTAB; + } + *ppNode = pNode; + }else{ + sqlite3_free(pNode); + *ppNode = 0; + } + + return rc; +} + +/* +** Overwrite cell iCell of node pNode with the contents of pCell. +*/ +static void nodeOverwriteCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ +){ + int ii; + u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + p += writeInt64(p, pCell->iRowid); + for(ii=0; ii<(pRtree->nDim*2); ii++){ + p += writeCoord(p, &pCell->aCoord[ii]); + } + pNode->isDirty = 1; +} + +/* +** Remove the cell with index iCell from node pNode. +*/ +static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ + u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + u8 *pSrc = &pDst[pRtree->nBytesPerCell]; + int nByte = (NCELL(pNode) - iCell - 1) * pRtree->nBytesPerCell; + memmove(pDst, pSrc, nByte); + writeInt16(&pNode->zData[2], NCELL(pNode)-1); + pNode->isDirty = 1; +} + +/* +** Insert the contents of cell pCell into node pNode. If the insert +** is successful, return SQLITE_OK. +** +** If there is not enough free space in pNode, return SQLITE_FULL. +*/ +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ +){ + int nCell; /* Current number of cells in pNode */ + int nMaxCell; /* Maximum number of cells for pNode */ + + nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; + nCell = NCELL(pNode); + + assert( nCell<=nMaxCell ); + if( nCellzData[2], nCell+1); + pNode->isDirty = 1; + } + + return (nCell==nMaxCell); +} + +/* +** If the node is dirty, write it out to the database. +*/ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode->isDirty ){ + sqlite3_stmt *p = pRtree->pWriteNode; + if( pNode->iNode ){ + sqlite3_bind_int64(p, 1, pNode->iNode); + }else{ + sqlite3_bind_null(p, 1); + } + sqlite3_bind_blob(p, 2, pNode->zData, pRtree->iNodeSize, SQLITE_STATIC); + sqlite3_step(p); + pNode->isDirty = 0; + rc = sqlite3_reset(p); + if( pNode->iNode==0 && rc==SQLITE_OK ){ + pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); + nodeHashInsert(pRtree, pNode); + } + } + return rc; +} + +/* +** Release a reference to a node. If the node is dirty and the reference +** count drops to zero, the node data is written to the database. +*/ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode ){ + assert( pNode->nRef>0 ); + pNode->nRef--; + if( pNode->nRef==0 ){ + if( pNode->iNode==1 ){ + pRtree->iDepth = -1; + } + if( pNode->pParent ){ + rc = nodeRelease(pRtree, pNode->pParent); + } + if( rc==SQLITE_OK ){ + rc = nodeWrite(pRtree, pNode); + } + nodeHashDelete(pRtree, pNode); + sqlite3_free(pNode); + } + } + return rc; +} + +/* +** Return the 64-bit integer value associated with cell iCell of +** node pNode. If pNode is a leaf node, this is a rowid. If it is +** an internal node, then the 64-bit integer is a child page number. +*/ +static i64 nodeGetRowid( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ +){ + assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); +} + +/* +** Return coordinate iCoord from cell iCell in node pNode. +*/ +static void nodeGetCoord( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ +){ + readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); +} + +/* +** Deserialize cell iCell of node pNode. Populate the structure pointed +** to by pCell with the results. +*/ +static void nodeGetCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ +){ + u8 *pData; + u8 *pEnd; + RtreeCoord *pCoord; + pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pEnd = pData + pRtree->nDim*8; + pCoord = pCell->aCoord; + for(; pDatanBusy++; +} + +/* +** Decrement the r-tree reference count. When the reference count reaches +** zero the structure is deleted. +*/ +static void rtreeRelease(Rtree *pRtree){ + pRtree->nBusy--; + if( pRtree->nBusy==0 ){ + sqlite3_finalize(pRtree->pReadNode); + sqlite3_finalize(pRtree->pWriteNode); + sqlite3_finalize(pRtree->pDeleteNode); + sqlite3_finalize(pRtree->pReadRowid); + sqlite3_finalize(pRtree->pWriteRowid); + sqlite3_finalize(pRtree->pDeleteRowid); + sqlite3_finalize(pRtree->pReadParent); + sqlite3_finalize(pRtree->pWriteParent); + sqlite3_finalize(pRtree->pDeleteParent); + sqlite3_free(pRtree); + } +} + +/* +** Rtree virtual table module xDisconnect method. +*/ +static int rtreeDisconnect(sqlite3_vtab *pVtab){ + rtreeRelease((Rtree *)pVtab); + return SQLITE_OK; +} + +/* +** Rtree virtual table module xDestroy method. +*/ +static int rtreeDestroy(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + int rc; + char *zCreate = sqlite3_mprintf( + "DROP TABLE '%q'.'%q_node';" + "DROP TABLE '%q'.'%q_rowid';" + "DROP TABLE '%q'.'%q_parent';", + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName + ); + if( !zCreate ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + } + if( rc==SQLITE_OK ){ + rtreeRelease(pRtree); + } + + return rc; +} + +/* +** Rtree virtual table module xOpen method. +*/ +static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + int rc = SQLITE_NOMEM; + RtreeCursor *pCsr; + + pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); + if( pCsr ){ + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = pVTab; + rc = SQLITE_OK; + } + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + + return rc; +} + + +/* +** Free the RtreeCursor.aConstraint[] array and its contents. +*/ +static void freeCursorConstraints(RtreeCursor *pCsr){ + if( pCsr->aConstraint ){ + int i; /* Used to iterate through constraint array */ + for(i=0; inConstraint; i++){ + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); + } + } + sqlite3_free(pCsr->aConstraint); + pCsr->aConstraint = 0; + } +} + +/* +** Rtree virtual table module xClose method. +*/ +static int rtreeClose(sqlite3_vtab_cursor *cur){ + Rtree *pRtree = (Rtree *)(cur->pVtab); + int ii; + RtreeCursor *pCsr = (RtreeCursor *)cur; + freeCursorConstraints(pCsr); + sqlite3_free(pCsr->aPoint); + for(ii=0; iiaNode[ii]); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Rtree virtual table module xEof method. +** +** Return non-zero if the cursor does not currently point to a valid +** record (i.e if the scan has finished), or zero otherwise. +*/ +static int rtreeEof(sqlite3_vtab_cursor *cur){ + RtreeCursor *pCsr = (RtreeCursor *)cur; + return pCsr->atEOF; +} + +/* +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". +** +** There are three versions of this macro, one each for little-endian and +** big-endian processors and a third generic implementation. The endian- +** specific implementations are much faster and are preferred if the +** processor endianness is known at compile-time. The SQLITE_BYTEORDER +** macro is part of sqliteInt.h and hence the endian-specific +** implementation will only be used if this module is compiled as part +** of the amalgamation. +*/ +#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#endif + +/* +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. +*/ +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ +){ + int i; /* Loop counter */ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ + + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); + + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); + } + pCellData += 8; + for(i=0; iop==RTREE_MATCH ){ + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &i); + if( i==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; + }else{ + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; + } + } + return rc; +} + +/* +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. +*/ +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ + + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + switch( p->op ){ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ + + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; + } + *peWithin = NOT_WITHIN; +} + +/* +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. +** +** The constraint is of the form: xN op $val +** +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. +*/ +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + RtreeDValue xN; /* Coordinate value converted to a double */ + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + pCellData += 8 + p->iCoord*4; + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; + } + *peWithin = NOT_WITHIN; +} + +/* +** One of the cells in node pNode is guaranteed to have a 64-bit +** integer value equal to iRowid. Return the index of this cell. +*/ +static int nodeRowidIndex( + Rtree *pRtree, + RtreeNode *pNode, + i64 iRowid, + int *piIndex +){ + int ii; + int nCell = NCELL(pNode); + assert( nCell<200 ); + for(ii=0; iipParent; + if( pParent ){ + return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); + } + *piIndex = -1; + return SQLITE_OK; +} + +/* +** Compare two search points. Return negative, zero, or positive if the first +** is less than, equal to, or greater than the second. +** +** The rScore is the primary key. Smaller rScore values come first. +** If the rScore is a tie, then use iLevel as the tie breaker with smaller +** iLevel values coming first. In this way, if rScore is the same for all +** SearchPoints, then iLevel becomes the deciding factor and the result +** is a depth-first search, which is the desired default behavior. +*/ +static int rtreeSearchPointCompare( + const RtreeSearchPoint *pA, + const RtreeSearchPoint *pB +){ + if( pA->rScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} + +/* +** Interchange to search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + }else{ + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; + } + } +} + +/* +** Return the search point with the lowest current score. +*/ +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} + +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); + } + return pCur->aNode[ii]; +} + +/* +** Push a new element onto the priority queue +*/ +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; + } + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel>=0 && iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; + } + return pNew; +} + +/* +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. +*/ +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + if( iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; + } + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; + }else{ + return rtreeEnqueue(pCur, rScore, iLevel); + } +} + +#if 0 +/* Tracing routines for the RtreeSearchPoint queue */ +static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ + if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } + printf(" %d.%05lld.%02d %g %d", + p->iLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); + } +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); + } +} +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif + +/* Remove the search point with the lowest current score. +*/ +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( p->nPoint ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } + }else{ + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; + } + } + } + } +} + + +/* +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. +*/ +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; + + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); + eWithin = FULLY_WITHIN; + for(ii=0; iiaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + } + if( eWithin==NOT_WITHIN ) break; + } + p->iCell++; + if( eWithin==NOT_WITHIN ) continue; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); + } + if( rScoreeWithin = eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); + } + } + pCur->atEOF = p==0; + return SQLITE_OK; +} + +/* +** Rtree virtual table module xNext method. +*/ +static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + int rc = SQLITE_OK; + + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); + return rc; +} + +/* +** Rtree virtual table module xRowid method. +*/ +static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && p ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } + return rc; +} + +/* +** Rtree virtual table module xColumn method. +*/ +static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; + if( i==0 ){ + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); + }else{ + if( rc ) return rc; + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + sqlite3_result_double(ctx, c.f); + }else +#endif + { + assert( pRtree->eCoordType==RTREE_COORD_INT32 ); + sqlite3_result_int(ctx, c.i); + } + } + return SQLITE_OK; +} + +/* +** Use nodeAcquire() to obtain the leaf node containing the record with +** rowid iRowid. If successful, set *ppLeaf to point to the node and +** return SQLITE_OK. If there is no such record in the table, set +** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf +** to zero and return an SQLite error code. +*/ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ +){ + int rc; + *ppLeaf = 0; + sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); + if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ + i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; + rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); + sqlite3_reset(pRtree->pReadRowid); + }else{ + rc = sqlite3_reset(pRtree->pReadRowid); + } + return rc; +} + +/* +** This function is called to configure the RtreeConstraint object passed +** as the second argument for a MATCH constraint. The value passed as the +** first argument to this function is the right-hand operand to the MATCH +** operator. +*/ +static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ + RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + int nBlob; /* Size of the geometry function blob */ + int nExpected; /* Expected size of the BLOB */ + + /* Check that value is actually a blob. */ + if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; + + /* Check that the blob is roughly the right size. */ + nBlob = sqlite3_value_bytes(pValue); + if( nBlob<(int)sizeof(RtreeMatchArg) + || ((nBlob-sizeof(RtreeMatchArg))%sizeof(RtreeDValue))!=0 + ){ + return SQLITE_ERROR; + } + + pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; + + memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); + nExpected = (int)(sizeof(RtreeMatchArg) + + (pBlob->nParam-1)*sizeof(RtreeDValue)); + if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ + sqlite3_free(pInfo); + return SQLITE_ERROR; + } + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; + + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; + return SQLITE_OK; +} + +/* +** Rtree virtual table module xFilter method. +*/ +static int rtreeFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int ii; + int rc = SQLITE_OK; + int iCell = 0; + + rtreeReference(pRtree); + + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + freeCursorConstraints(pCsr); + sqlite3_free(pCsr->aPoint); + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ + pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); + pCsr->nConstraint = argc; + if( !pCsr->aConstraint ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + assert( (idxStr==0 && argc==0) + || (idxStr && (int)strlen(idxStr)==argc*2) ); + for(ii=0; iiaConstraint[ii]; + p->op = idxStr[ii*2]; + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ + /* A MATCH operator. The right-hand-side must be a blob that + ** can be cast into an RtreeMatchArg object. One created using + ** an sqlite3_rtree_geometry_callback() SQL user function. + */ + rc = deserializeGeometry(argv[ii], p); + if( rc!=SQLITE_OK ){ + break; + } + p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; + }else{ +#ifdef SQLITE_RTREE_INT_ONLY + p->u.rValue = sqlite3_value_int64(argv[ii]); +#else + p->u.rValue = sqlite3_value_double(argv[ii]); +#endif + } + } + } + } + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); + } + } + + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); + return rc; +} + +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + +/* +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): +** +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 Unused Direct lookup by rowid. +** 2 See below R-tree query or full-table scan. +** ------------------------------------------------ +** +** If strategy 1 is used, then idxStr is not meaningful. If strategy +** 2 is used, idxStr is formatted to contain 2 bytes for each +** constraint used. The first two bytes of idxStr correspond to +** the constraint in sqlite3_index_info.aConstraintUsage[] with +** (argvIndex==1) etc. +** +** The first of each pair of bytes in idxStr identifies the constraint +** operator as follows: +** +** Operator Byte Value +** ---------------------- +** = 0x41 ('A') +** <= 0x42 ('B') +** < 0x43 ('C') +** >= 0x44 ('D') +** > 0x45 ('E') +** MATCH 0x46 ('F') +** ---------------------- +** +** The second of each pair of bytes identifies the coordinate column +** to which the constraint applies. The leftmost coordinate column +** is 'a', the second from the left 'b' etc. +*/ +static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + Rtree *pRtree = (Rtree*)tab; + int rc = SQLITE_OK; + int ii; + i64 nRow; /* Estimated rows returned by this scan */ + + int iIdx = 0; + char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; + memset(zIdxStr, 0, sizeof(zIdxStr)); + + assert( pIdxInfo->idxStr==0 ); + for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; + + if( p->usable && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + /* We have an equality constraint on the rowid. Use strategy 1. */ + int jj; + for(jj=0; jjaConstraintUsage[jj].argvIndex = 0; + pIdxInfo->aConstraintUsage[jj].omit = 0; + } + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[ii].argvIndex = 1; + pIdxInfo->aConstraintUsage[jj].omit = 1; + + /* This strategy involves a two rowid lookups on an B-Tree structures + ** and then a linear search of an R-Tree node. This should be + ** considered almost as quick as a direct rowid lookup (for which + ** sqlite uses an internal cost of 0.0). It is expected to return + ** a single row. + */ + pIdxInfo->estimatedCost = 30.0; + setEstimatedRows(pIdxInfo, 1); + return SQLITE_OK; + } + + if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ + u8 op; + switch( p->op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; + case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; + case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; + case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; + case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; + default: + assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH ); + op = RTREE_MATCH; + break; + } + zIdxStr[iIdx++] = op; + zIdxStr[iIdx++] = p->iColumn - 1 + '0'; + pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); + pIdxInfo->aConstraintUsage[ii].omit = 1; + } + } + + pIdxInfo->idxNum = 2; + pIdxInfo->needToFreeIdxStr = 1; + if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ + return SQLITE_NOMEM; + } + + nRow = pRtree->nRowEst / (iIdx + 1); + pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; + setEstimatedRows(pIdxInfo, nRow); + + return rc; +} + +/* +** Return the N-dimensional volumn of the cell stored in *p. +*/ +static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ + RtreeDValue area = (RtreeDValue)1; + int ii; + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); + } + return area; +} + +/* +** Return the margin length of cell p. The margin length is the sum +** of the objects size in each dimension. +*/ +static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ + RtreeDValue margin = (RtreeDValue)0; + int ii; + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + } + return margin; +} + +/* +** Store the union of cells p1 and p2 in p1. +*/ +static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii; + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); + p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); + } + }else{ + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); + p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); + } + } +} + +/* +** Return true if the area covered by p2 is a subset of the area covered +** by p1. False otherwise. +*/ +static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii; + int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( (!isInt && (a2[0].fa1[1].f)) + || ( isInt && (a2[0].ia1[1].i)) + ){ + return 0; + } + } + return 1; +} + +/* +** Return the amount cell p would grow by if it were unioned with pCell. +*/ +static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ + RtreeDValue area; + RtreeCell cell; + memcpy(&cell, p, sizeof(RtreeCell)); + area = cellArea(pRtree, &cell); + cellUnion(pRtree, &cell, pCell); + return (cellArea(pRtree, &cell)-area); +} + +static RtreeDValue cellOverlap( + Rtree *pRtree, + RtreeCell *p, + RtreeCell *aCell, + int nCell +){ + int ii; + RtreeDValue overlap = RTREE_ZERO; + for(ii=0; iinDim*2); jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2iDepth-iHeight); ii++){ + int iCell; + sqlite3_int64 iBest = 0; + + RtreeDValue fMinGrowth = RTREE_ZERO; + RtreeDValue fMinArea = RTREE_ZERO; + + int nCell = NCELL(pNode); + RtreeCell cell; + RtreeNode *pChild; + + RtreeCell *aCell = 0; + + /* Select the child node which will be enlarged the least if pCell + ** is inserted into it. Resolve ties by choosing the entry with + ** the smallest area. + */ + for(iCell=0; iCellpParent ){ + RtreeNode *pParent = p->pParent; + RtreeCell cell; + int iCell; + + if( nodeParentIndex(pRtree, p, &iCell) ){ + return SQLITE_CORRUPT_VTAB; + } + + nodeGetCell(pRtree, pParent, iCell, &cell); + if( !cellContains(pRtree, &cell, pCell) ){ + cellUnion(pRtree, &cell, pCell); + nodeOverwriteCell(pRtree, pParent, &cell, iCell); + } + + p = pParent; + } + return SQLITE_OK; +} + +/* +** Write mapping (iRowid->iNode) to the _rowid table. +*/ +static int rowidWrite(Rtree *pRtree, sqlite3_int64 iRowid, sqlite3_int64 iNode){ + sqlite3_bind_int64(pRtree->pWriteRowid, 1, iRowid); + sqlite3_bind_int64(pRtree->pWriteRowid, 2, iNode); + sqlite3_step(pRtree->pWriteRowid); + return sqlite3_reset(pRtree->pWriteRowid); +} + +/* +** Write mapping (iNode->iPar) to the _parent table. +*/ +static int parentWrite(Rtree *pRtree, sqlite3_int64 iNode, sqlite3_int64 iPar){ + sqlite3_bind_int64(pRtree->pWriteParent, 1, iNode); + sqlite3_bind_int64(pRtree->pWriteParent, 2, iPar); + sqlite3_step(pRtree->pWriteParent); + return sqlite3_reset(pRtree->pWriteParent); +} + +static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int); + + +/* +** Arguments aIdx, aDistance and aSpare all point to arrays of size +** nIdx. The aIdx array contains the set of integers from 0 to +** (nIdx-1) in no particular order. This function sorts the values +** in aIdx according to the indexed values in aDistance. For +** example, assuming the inputs: +** +** aIdx = { 0, 1, 2, 3 } +** aDistance = { 5.0, 2.0, 7.0, 6.0 } +** +** this function sets the aIdx array to contain: +** +** aIdx = { 0, 1, 2, 3 } +** +** The aSpare array is used as temporary working space by the +** sorting algorithm. +*/ +static void SortByDistance( + int *aIdx, + int nIdx, + RtreeDValue *aDistance, + int *aSpare +){ + if( nIdx>1 ){ + int iLeft = 0; + int iRight = 0; + + int nLeft = nIdx/2; + int nRight = nIdx-nLeft; + int *aLeft = aIdx; + int *aRight = &aIdx[nLeft]; + + SortByDistance(aLeft, nLeft, aDistance, aSpare); + SortByDistance(aRight, nRight, aDistance, aSpare); + + memcpy(aSpare, aLeft, sizeof(int)*nLeft); + aLeft = aSpare; + + while( iLeft1 ){ + + int iLeft = 0; + int iRight = 0; + + int nLeft = nIdx/2; + int nRight = nIdx-nLeft; + int *aLeft = aIdx; + int *aRight = &aIdx[nLeft]; + + SortByDimension(pRtree, aLeft, nLeft, iDim, aCell, aSpare); + SortByDimension(pRtree, aRight, nRight, iDim, aCell, aSpare); + + memcpy(aSpare, aLeft, sizeof(int)*nLeft); + aLeft = aSpare; + while( iLeftnDim+1)*(sizeof(int*)+nCell*sizeof(int)); + + aaSorted = (int **)sqlite3_malloc(nByte); + if( !aaSorted ){ + return SQLITE_NOMEM; + } + + aSpare = &((int *)&aaSorted[pRtree->nDim])[pRtree->nDim*nCell]; + memset(aaSorted, 0, nByte); + for(ii=0; iinDim; ii++){ + int jj; + aaSorted[ii] = &((int *)&aaSorted[pRtree->nDim])[ii*nCell]; + for(jj=0; jjnDim; ii++){ + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; + int iBestLeft = 0; + int nLeft; + + for( + nLeft=RTREE_MINCELLS(pRtree); + nLeft<=(nCell-RTREE_MINCELLS(pRtree)); + nLeft++ + ){ + RtreeCell left; + RtreeCell right; + int kk; + RtreeDValue overlap; + RtreeDValue area; + + memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell)); + memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell)); + for(kk=1; kk<(nCell-1); kk++){ + if( kk0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, iRowid); + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; + } + } + return xSetMapping(pRtree, iRowid, pNode->iNode); +} + +static int SplitNode( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight +){ + int i; + int newCellIsRight = 0; + + int rc = SQLITE_OK; + int nCell = NCELL(pNode); + RtreeCell *aCell; + int *aiUsed; + + RtreeNode *pLeft = 0; + RtreeNode *pRight = 0; + + RtreeCell leftbbox; + RtreeCell rightbbox; + + /* Allocate an array and populate it with a copy of pCell and + ** all cells from node pLeft. Then zero the original node. + */ + aCell = sqlite3_malloc((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); + if( !aCell ){ + rc = SQLITE_NOMEM; + goto splitnode_out; + } + aiUsed = (int *)&aCell[nCell+1]; + memset(aiUsed, 0, sizeof(int)*(nCell+1)); + for(i=0; iiNode==1 ){ + pRight = nodeNew(pRtree, pNode); + pLeft = nodeNew(pRtree, pNode); + pRtree->iDepth++; + pNode->isDirty = 1; + writeInt16(pNode->zData, pRtree->iDepth); + }else{ + pLeft = pNode; + pRight = nodeNew(pRtree, pLeft->pParent); + nodeReference(pLeft); + } + + if( !pLeft || !pRight ){ + rc = SQLITE_NOMEM; + goto splitnode_out; + } + + memset(pLeft->zData, 0, pRtree->iNodeSize); + memset(pRight->zData, 0, pRtree->iNodeSize); + + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + + /* Ensure both child nodes have node numbers assigned to them by calling + ** nodeWrite(). Node pRight always needs a node number, as it was created + ** by nodeNew() above. But node pLeft sometimes already has a node number. + ** In this case avoid the all to nodeWrite(). + */ + if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) + || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) + ){ + goto splitnode_out; + } + + rightbbox.iRowid = pRight->iNode; + leftbbox.iRowid = pLeft->iNode; + + if( pNode->iNode==1 ){ + rc = rtreeInsertCell(pRtree, pLeft->pParent, &leftbbox, iHeight+1); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + }else{ + RtreeNode *pParent = pLeft->pParent; + int iCell; + rc = nodeParentIndex(pRtree, pLeft, &iCell); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); + rc = AdjustTree(pRtree, pParent, &leftbbox); + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + } + if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ + goto splitnode_out; + } + + for(i=0; iiRowid ){ + newCellIsRight = 1; + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + } + if( pNode->iNode==1 ){ + for(i=0; iiRowid, pLeft, iHeight); + } + + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRight); + pRight = 0; + } + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pLeft); + pLeft = 0; + } + +splitnode_out: + nodeRelease(pRtree, pRight); + nodeRelease(pRtree, pLeft); + sqlite3_free(aCell); + return rc; +} + +/* +** If node pLeaf is not the root of the r-tree and its pParent pointer is +** still NULL, load all ancestor nodes of pLeaf into memory and populate +** the pLeaf->pParent chain all the way up to the root node. +** +** This operation is required when a row is deleted (or updated - an update +** is implemented as a delete followed by an insert). SQLite provides the +** rowid of the row to delete, which can be used to find the leaf on which +** the entry resides (argument pLeaf). Once the leaf is located, this +** function is called to determine its ancestry. +*/ +static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ + int rc = SQLITE_OK; + RtreeNode *pChild = pLeaf; + while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ + int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); + rc = sqlite3_step(pRtree->pReadParent); + if( rc==SQLITE_ROW ){ + RtreeNode *pTest; /* Used to test for reference loops */ + i64 iNode; /* Node number of parent node */ + + /* Before setting pChild->pParent, test that we are not creating a + ** loop of references (as we would if, say, pChild==pParent). We don't + ** want to do this as it leads to a memory leak when trying to delete + ** the referenced counted node structures. + */ + iNode = sqlite3_column_int64(pRtree->pReadParent, 0); + for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); + if( !pTest ){ + rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); + } + } + rc = sqlite3_reset(pRtree->pReadParent); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; + pChild = pChild->pParent; + } + return rc; +} + +static int deleteCell(Rtree *, RtreeNode *, int, int); + +static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ + int rc; + int rc2; + RtreeNode *pParent = 0; + int iCell; + + assert( pNode->nRef==1 ); + + /* Remove the entry in the parent cell. */ + rc = nodeParentIndex(pRtree, pNode, &iCell); + if( rc==SQLITE_OK ){ + pParent = pNode->pParent; + pNode->pParent = 0; + rc = deleteCell(pRtree, pParent, iCell, iHeight+1); + } + rc2 = nodeRelease(pRtree, pParent); + if( rc==SQLITE_OK ){ + rc = rc2; + } + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Remove the xxx_node entry. */ + sqlite3_bind_int64(pRtree->pDeleteNode, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteNode); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteNode)) ){ + return rc; + } + + /* Remove the xxx_parent entry. */ + sqlite3_bind_int64(pRtree->pDeleteParent, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteParent); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteParent)) ){ + return rc; + } + + /* Remove the node from the in-memory hash table and link it into + ** the Rtree.pDeleted list. Its contents will be re-inserted later on. + */ + nodeHashDelete(pRtree, pNode); + pNode->iNode = iHeight; + pNode->pNext = pRtree->pDeleted; + pNode->nRef++; + pRtree->pDeleted = pNode; + + return SQLITE_OK; +} + +static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode *pParent = pNode->pParent; + int rc = SQLITE_OK; + if( pParent ){ + int ii; + int nCell = NCELL(pNode); + RtreeCell box; /* Bounding box for pNode */ + nodeGetCell(pRtree, pNode, 0, &box); + for(ii=1; iiiNode; + rc = nodeParentIndex(pRtree, pNode, &ii); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &box, ii); + rc = fixBoundingBox(pRtree, pParent); + } + } + return rc; +} + +/* +** Delete the cell at index iCell of node pNode. After removing the +** cell, adjust the r-tree data structure if required. +*/ +static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ + RtreeNode *pParent; + int rc; + + if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ + return rc; + } + + /* Remove the cell from the node. This call just moves bytes around + ** the in-memory node image, so it cannot fail. + */ + nodeDeleteCell(pRtree, pNode, iCell); + + /* If the node is not the tree root and now has less than the minimum + ** number of cells, remove it from the tree. Otherwise, update the + ** cell in the parent node so that it tightly contains the updated + ** node. + */ + pParent = pNode->pParent; + assert( pParent || pNode->iNode==1 ); + if( pParent ){ + if( NCELL(pNode)nDim; iDim++){ + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); + } + } + for(iDim=0; iDimnDim; iDim++){ + aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); + } + + for(ii=0; iinDim; iDim++){ + RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); + aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); + } + } + + SortByDistance(aOrder, nCell, aDistance, aSpare); + nodeZero(pRtree, pNode); + + for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ + RtreeCell *p = &aCell[aOrder[ii]]; + nodeInsertCell(pRtree, pNode, p); + if( p->iRowid==pCell->iRowid ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, p->iRowid, pNode->iNode); + } + } + } + if( rc==SQLITE_OK ){ + rc = fixBoundingBox(pRtree, pNode); + } + for(; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + RtreeNode *pInsert; + RtreeCell *p = &aCell[aOrder[ii]]; + rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); + rc2 = nodeRelease(pRtree, pInsert); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + sqlite3_free(aCell); + return rc; +} + +/* +** Insert cell pCell into node pNode. Node pNode is the head of a +** subtree iHeight high (leaf nodes have iHeight==0). +*/ +static int rtreeInsertCell( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight +){ + int rc = SQLITE_OK; + if( iHeight>0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, pCell->iRowid); + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; + } + } + if( nodeInsertCell(pRtree, pNode, pCell) ){ + if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ + rc = SplitNode(pRtree, pNode, pCell, iHeight); + }else{ + pRtree->iReinsertHeight = iHeight; + rc = Reinsert(pRtree, pNode, pCell, iHeight); + } + }else{ + rc = AdjustTree(pRtree, pNode, pCell); + if( rc==SQLITE_OK ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + } + } + } + return rc; +} + +static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ + int ii; + int rc = SQLITE_OK; + int nCell = NCELL(pNode); + + for(ii=0; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); + rc2 = nodeRelease(pRtree, pInsert); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + return rc; +} + +/* +** Select a currently unused rowid for a new r-tree record. +*/ +static int newRowid(Rtree *pRtree, i64 *piRowid){ + int rc; + sqlite3_bind_null(pRtree->pWriteRowid, 1); + sqlite3_bind_null(pRtree->pWriteRowid, 2); + sqlite3_step(pRtree->pWriteRowid); + rc = sqlite3_reset(pRtree->pWriteRowid); + *piRowid = sqlite3_last_insert_rowid(pRtree->db); + return rc; +} + +/* +** Remove the entry with rowid=iDelete from the r-tree structure. +*/ +static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ + int rc; /* Return code */ + RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ + int iCell; /* Index of iDelete cell in pLeaf */ + RtreeNode *pRoot; /* Root node of rtree structure */ + + + /* Obtain a reference to the root node to initialize Rtree.iDepth */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. + */ + if( rc==SQLITE_OK ){ + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); + } + + /* Delete the cell in question from the leaf node. */ + if( rc==SQLITE_OK ){ + int rc2; + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + + /* Delete the corresponding entry in the _rowid table. */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); + sqlite3_step(pRtree->pDeleteRowid); + rc = sqlite3_reset(pRtree->pDeleteRowid); + } + + /* Check if the root node now has exactly one child. If so, remove + ** it, schedule the contents of the child for reinsertion and + ** reduce the tree height by one. + ** + ** This is equivalent to copying the contents of the child into + ** the root node (the operation that Gutman's paper says to perform + ** in this scenario). + */ + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; + } + } + + /* Re-insert the contents of any underfull nodes removed from the tree. */ + for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ + if( rc==SQLITE_OK ){ + rc = reinsertNodeContent(pRtree, pLeaf); + } + pRtree->pDeleted = pLeaf->pNext; + sqlite3_free(pLeaf); + } + + /* Release the reference to the root node. */ + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRoot); + }else{ + nodeRelease(pRtree, pRoot); + } + + return rc; +} + +/* +** Rounding constants for float->double conversion. +*/ +#define RNDTOWARDS (1.0 - 1.0/8388608.0) /* Round towards zero */ +#define RNDAWAY (1.0 + 1.0/8388608.0) /* Round away from zero */ + +#if !defined(SQLITE_RTREE_INT_ONLY) +/* +** Convert an sqlite3_value into an RtreeValue (presumably a float) +** while taking care to round toward negative or positive, respectively. +*/ +static RtreeValue rtreeValueDown(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( f>d ){ + f = (float)(d*(d<0 ? RNDAWAY : RNDTOWARDS)); + } + return f; +} +static RtreeValue rtreeValueUp(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( f1 */ + int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ + + rtreeReference(pRtree); + assert(nData>=1); + + /* Constraint handling. A write operation on an r-tree table may return + ** SQLITE_CONSTRAINT for two reasons: + ** + ** 1. A duplicate rowid value, or + ** 2. The supplied data violates the "x2>=x1" constraint. + ** + ** In the first case, if the conflict-handling mode is REPLACE, then + ** the conflicting row can be removed before proceeding. In the second + ** case, SQLITE_CONSTRAINT must be returned regardless of the + ** conflict-handling mode specified by the user. + */ + if( nData>1 ){ + int ii; + + /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */ + assert( nData==(pRtree->nDim*2 + 3) ); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + cell.aCoord[ii].f = rtreeValueDown(azData[ii+3]); + cell.aCoord[ii+1].f = rtreeValueUp(azData[ii+4]); + if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } + }else +#endif + { + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]); + cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]); + if( cell.aCoord[ii].i>cell.aCoord[ii+1].i ){ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } + } + + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){ + cell.iRowid = sqlite3_value_int64(azData[2]); + if( sqlite3_value_type(azData[0])==SQLITE_NULL + || sqlite3_value_int64(azData[0])!=cell.iRowid + ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } + } + bHaveRowid = 1; + } + } + + /* If azData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); + } + + /* If the azData[] array contains more than one element, elements + ** (azData[2]..azData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; + + /* Figure out the rowid of the new row. */ + if( bHaveRowid==0 ){ + rc = newRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; + + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + pRtree->iReinsertHeight = -1; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + +constraint: + rtreeRelease(pRtree); + return rc; +} + +/* +** The xRename method for rtree module virtual tables. +*/ +static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_NOMEM; + char *zSql = sqlite3_mprintf( + "ALTER TABLE %Q.'%q_node' RENAME TO \"%w_node\";" + "ALTER TABLE %Q.'%q_parent' RENAME TO \"%w_parent\";" + "ALTER TABLE %Q.'%q_rowid' RENAME TO \"%w_rowid\";" + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + ); + if( zSql ){ + rc = sqlite3_exec(pRtree->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + return rc; +} + +/* +** This function populates the pRtree->nRowEst variable with an estimate +** of the number of rows in the virtual table. If possible, this is based +** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. +*/ +static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; + sqlite3_stmt *p; + int rc; + i64 nRow = 0; + + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; + } + + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + } + } + sqlite3_free(zSql); + } + + return rc; +} + +static sqlite3_module rtreeModule = { + 0, /* iVersion */ + rtreeCreate, /* xCreate - create a table */ + rtreeConnect, /* xConnect - connect to an existing table */ + rtreeBestIndex, /* xBestIndex - Determine search strategy */ + rtreeDisconnect, /* xDisconnect - Disconnect from a table */ + rtreeDestroy, /* xDestroy - Drop a table */ + rtreeOpen, /* xOpen - open a cursor */ + rtreeClose, /* xClose - close a cursor */ + rtreeFilter, /* xFilter - configure scan constraints */ + rtreeNext, /* xNext - advance a cursor */ + rtreeEof, /* xEof */ + rtreeColumn, /* xColumn - read data */ + rtreeRowid, /* xRowid - read data */ + rtreeUpdate, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + rtreeRename, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; + +static int rtreeSqlInit( + Rtree *pRtree, + sqlite3 *db, + const char *zDb, + const char *zPrefix, + int isCreate +){ + int rc = SQLITE_OK; + + #define N_STATEMENT 9 + static const char *azSql[N_STATEMENT] = { + /* Read and write the xxx_node table */ + "SELECT data FROM '%q'.'%q_node' WHERE nodeno = :1", + "INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)", + "DELETE FROM '%q'.'%q_node' WHERE nodeno = :1", + + /* Read and write the xxx_rowid table */ + "SELECT nodeno FROM '%q'.'%q_rowid' WHERE rowid = :1", + "INSERT OR REPLACE INTO '%q'.'%q_rowid' VALUES(:1, :2)", + "DELETE FROM '%q'.'%q_rowid' WHERE rowid = :1", + + /* Read and write the xxx_parent table */ + "SELECT parentnode FROM '%q'.'%q_parent' WHERE nodeno = :1", + "INSERT OR REPLACE INTO '%q'.'%q_parent' VALUES(:1, :2)", + "DELETE FROM '%q'.'%q_parent' WHERE nodeno = :1" + }; + sqlite3_stmt **appStmt[N_STATEMENT]; + int i; + + pRtree->db = db; + + if( isCreate ){ + char *zCreate = sqlite3_mprintf( +"CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" +"CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" +"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," + " parentnode INTEGER);" +"INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", + zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize + ); + if( !zCreate ){ + return SQLITE_NOMEM; + } + rc = sqlite3_exec(db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + if( rc!=SQLITE_OK ){ + return rc; + } + } + + appStmt[0] = &pRtree->pReadNode; + appStmt[1] = &pRtree->pWriteNode; + appStmt[2] = &pRtree->pDeleteNode; + appStmt[3] = &pRtree->pReadRowid; + appStmt[4] = &pRtree->pWriteRowid; + appStmt[5] = &pRtree->pDeleteRowid; + appStmt[6] = &pRtree->pReadParent; + appStmt[7] = &pRtree->pWriteParent; + appStmt[8] = &pRtree->pDeleteParent; + + rc = rtreeQueryStat1(db, pRtree); + for(i=0; iiNodeSize is populated and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned. +** +** If this function is being called as part of an xConnect(), then the rtree +** table already exists. In this case the node-size is determined by inspecting +** the root node of the tree. +** +** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. +** This ensures that each node is stored on a single database page. If the +** database page-size is so large that more than RTREE_MAXCELLS entries +** would fit in a single node, use a smaller node-size. +*/ +static int getNodeSize( + sqlite3 *db, /* Database handle */ + Rtree *pRtree, /* Rtree handle */ + int isCreate, /* True for xCreate, false for xConnect */ + char **pzErr /* OUT: Error message, if any */ +){ + int rc; + char *zSql; + if( isCreate ){ + int iPageSize = 0; + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); + rc = getIntFromStmt(db, zSql, &iPageSize); + if( rc==SQLITE_OK ){ + pRtree->iNodeSize = iPageSize-64; + if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ + pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; + } + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + }else{ + zSql = sqlite3_mprintf( + "SELECT length(data) FROM '%q'.'%q_node' WHERE nodeno = 1", + pRtree->zDb, pRtree->zName + ); + rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + } + + sqlite3_free(zSql); + return rc; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the r-tree virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int rtreeInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ +){ + int rc = SQLITE_OK; + Rtree *pRtree; + int nDb; /* Length of string argv[1] */ + int nName; /* Length of string argv[2] */ + int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); + + const char *aErrMsg[] = { + 0, /* 0 */ + "Wrong number of columns for an rtree table", /* 1 */ + "Too few columns for an rtree table", /* 2 */ + "Too many columns for an rtree table" /* 3 */ + }; + + int iErr = (argc<6) ? 2 : argc>(RTREE_MAX_DIMENSIONS*2+4) ? 3 : argc%2; + if( aErrMsg[iErr] ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); + return SQLITE_ERROR; + } + + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Allocate the sqlite3_vtab structure */ + nDb = (int)strlen(argv[1]); + nName = (int)strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); + if( !pRtree ){ + return SQLITE_NOMEM; + } + memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + pRtree->nBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->nDim = (argc-4)/2; + pRtree->nBytesPerCell = 8 + pRtree->nDim*4*2; + pRtree->eCoordType = eCoordType; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); + + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); + + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + if( rc==SQLITE_OK ){ + if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + }else{ + char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]); + char *zTmp; + int ii; + for(ii=4; zSql && iinBusy==1 ); + rtreeRelease(pRtree); + } + return rc; +} + + +/* +** Implementation of a scalar function that decodes r-tree nodes to +** human readable strings. This can be used for debugging and analysis. +** +** The scalar function takes two arguments: (1) the number of dimensions +** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing +** an r-tree node. For a two-dimensional r-tree structure called "rt", to +** deserialize all nodes, a statement like: +** +** SELECT rtreenode(2, data) FROM rt_node; +** +** The human readable string takes the form of a Tcl list with one +** entry for each cell in the r-tree node. Each entry is itself a +** list, containing the 8-byte rowid/pageno followed by the +** *2 coordinates. +*/ +static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ + char *zText = 0; + RtreeNode node; + Rtree tree; + int ii; + + UNUSED_PARAMETER(nArg); + memset(&node, 0, sizeof(RtreeNode)); + memset(&tree, 0, sizeof(Rtree)); + tree.nDim = sqlite3_value_int(apArg[0]); + tree.nBytesPerCell = 8 + 8 * tree.nDim; + node.zData = (u8 *)sqlite3_value_blob(apArg[1]); + + for(ii=0; iixDestructor ) pInfo->xDestructor(pInfo->pContext); + sqlite3_free(p); +} + +/* +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. +** +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. +** +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. +*/ +static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ + RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); + RtreeMatchArg *pBlob; + int nBlob; + + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue); + pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); + if( !pBlob ){ + sqlite3_result_error_nomem(ctx); + }else{ + int i; + pBlob->magic = RTREE_GEOMETRY_MAGIC; + pBlob->cb = pGeomCtx[0]; + pBlob->nParam = nArg; + for(i=0; iaParam[i] = sqlite3_value_int64(aArg[i]); +#else + pBlob->aParam[i] = sqlite3_value_double(aArg[i]); +#endif + } + sqlite3_result_blob(ctx, pBlob, nBlob, sqlite3_free); + } +} + +/* +** Register a new geometry function for use with the r-tree MATCH operator. +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +/* +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_rtree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3RtreeInit(db); +} +#endif + +#endif + +/************** End of rtree.c ***********************************************/ +/************** Begin file icu.c *********************************************/ +/* +** 2007 May 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ +** +** This file implements an integration between the ICU library +** ("International Components for Unicode", an open-source library +** for handling unicode data) and SQLite. The integration uses +** ICU to provide the following to SQLite: +** +** * An implementation of the SQL regexp() function (and hence REGEXP +** operator) using the ICU uregex_XX() APIs. +** +** * Implementations of the SQL scalar upper() and lower() functions +** for case mapping. +** +** * Integration of ICU and SQLite collation sequences. +** +** * An implementation of the LIKE operator that uses ICU to +** provide case-independent matching. +*/ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + +/* Include ICU headers */ +#include +#include +#include +#include + +/* #include */ + +#ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT1 +#else +#endif + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Version of sqlite3_free() that is always a function, never a macro. +*/ +static void xFree(void *p){ + sqlite3_free(p); +} + +/* +** Compare two UTF-8 strings for equality where the first string is +** a "LIKE" expression. Return true (1) if they are the same and +** false (0) if they are different. +*/ +static int icuLikeCompare( + const uint8_t *zPattern, /* LIKE pattern */ + const uint8_t *zString, /* The UTF-8 string to compare against */ + const UChar32 uEsc /* The escape character */ +){ + static const int MATCH_ONE = (UChar32)'_'; + static const int MATCH_ALL = (UChar32)'%'; + + int iPattern = 0; /* Current byte index in zPattern */ + int iString = 0; /* Current byte index in zString */ + + int prevEscape = 0; /* True if the previous character was uEsc */ + + while( zPattern[iPattern]!=0 ){ + + /* Read (and consume) the next character from the input pattern. */ + UChar32 uPattern; + U8_NEXT_UNSAFE(zPattern, iPattern, uPattern); + assert(uPattern!=0); + + /* There are now 4 possibilities: + ** + ** 1. uPattern is an unescaped match-all character "%", + ** 2. uPattern is an unescaped match-one character "_", + ** 3. uPattern is an unescaped escape character, or + ** 4. uPattern is to be handled as an ordinary character + */ + if( !prevEscape && uPattern==MATCH_ALL ){ + /* Case 1. */ + uint8_t c; + + /* Skip any MATCH_ALL or MATCH_ONE characters that follow a + ** MATCH_ALL. For each MATCH_ONE, skip one character in the + ** test string. + */ + while( (c=zPattern[iPattern]) == MATCH_ALL || c == MATCH_ONE ){ + if( c==MATCH_ONE ){ + if( zString[iString]==0 ) return 0; + U8_FWD_1_UNSAFE(zString, iString); + } + iPattern++; + } + + if( zPattern[iPattern]==0 ) return 1; + + while( zString[iString] ){ + if( icuLikeCompare(&zPattern[iPattern], &zString[iString], uEsc) ){ + return 1; + } + U8_FWD_1_UNSAFE(zString, iString); + } + return 0; + + }else if( !prevEscape && uPattern==MATCH_ONE ){ + /* Case 2. */ + if( zString[iString]==0 ) return 0; + U8_FWD_1_UNSAFE(zString, iString); + + }else if( !prevEscape && uPattern==uEsc){ + /* Case 3. */ + prevEscape = 1; + + }else{ + /* Case 4. */ + UChar32 uString; + U8_NEXT_UNSAFE(zString, iString, uString); + uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT); + uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT); + if( uString!=uPattern ){ + return 0; + } + prevEscape = 0; + } + } + + return zString[iString]==0; +} + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B, A). If there is an escape character E, +** +** A LIKE B ESCAPE E +** +** is mapped to like(B, A, E). +*/ +static void icuLikeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA = sqlite3_value_text(argv[0]); + const unsigned char *zB = sqlite3_value_text(argv[1]); + UChar32 uEsc = 0; + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + + + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + int nE= sqlite3_value_bytes(argv[2]); + const unsigned char *zE = sqlite3_value_text(argv[2]); + int i = 0; + if( zE==0 ) return; + U8_NEXT(zE, i, nE, uEsc); + if( i!=nE){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + } + + if( zA && zB ){ + sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); + } +} + +/* +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. +** +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. +*/ +static void icuFunctionError( + sqlite3_context *pCtx, /* SQLite scalar function context */ + const char *zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ +){ + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); +} + +/* +** Function to delete compiled regexp objects. Registered as +** a destructor function with sqlite3_set_auxdata(). +*/ +static void icuRegexpDelete(void *p){ + URegularExpression *pExpr = (URegularExpression *)p; + uregex_close(pExpr); +} + +/* +** Implementation of SQLite REGEXP operator. This scalar function takes +** two arguments. The first is a regular expression pattern to compile +** the second is a string to match against that pattern. If either +** argument is an SQL NULL, then NULL Is returned. Otherwise, the result +** is 1 if the string matches the pattern, or 0 otherwise. +** +** SQLite maps the regexp() function to the regexp() operator such +** that the following two are equivalent: +** +** zString REGEXP zPattern +** regexp(zPattern, zString) +** +** Uses the following ICU regexp APIs: +** +** uregex_open() +** uregex_matches() +** uregex_close() +*/ +static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + UErrorCode status = U_ZERO_ERROR; + URegularExpression *pExpr; + UBool res; + const UChar *zString = sqlite3_value_text16(apArg[1]); + + (void)nArg; /* Unused parameter */ + + /* If the left hand side of the regexp operator is NULL, + ** then the result is also NULL. + */ + if( !zString ){ + return; + } + + pExpr = sqlite3_get_auxdata(p, 0); + if( !pExpr ){ + const UChar *zPattern = sqlite3_value_text16(apArg[0]); + if( !zPattern ){ + return; + } + pExpr = uregex_open(zPattern, -1, 0, 0, &status); + + if( U_SUCCESS(status) ){ + sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); + }else{ + assert(!pExpr); + icuFunctionError(p, "uregex_open", status); + return; + } + } + + /* Configure the text that the regular expression operates on. */ + uregex_setText(pExpr, zString, -1, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_setText", status); + return; + } + + /* Attempt the match */ + res = uregex_matches(pExpr, 0, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_matches", status); + return; + } + + /* Set the text that the regular expression operates on to a NULL + ** pointer. This is not really necessary, but it is tidier than + ** leaving the regular expression object configured with an invalid + ** pointer after this function returns. + */ + uregex_setText(pExpr, 0, 0, &status); + + /* Return 1 or 0. */ + sqlite3_result_int(p, res ? 1 : 0); +} + +/* +** Implementations of scalar functions for case mapping - upper() and +** lower(). Function upper() converts its input to upper-case (ABC). +** Function lower() converts to lower-case (abc). +** +** ICU provides two types of case mapping, "general" case mapping and +** "language specific". Refer to ICU documentation for the differences +** between the two. +** +** To utilise "general" case mapping, the upper() or lower() scalar +** functions are invoked with one argument: +** +** upper('ABC') -> 'abc' +** lower('abc') -> 'ABC' +** +** To access ICU "language specific" case mapping, upper() or lower() +** should be invoked with two arguments. The second argument is the name +** of the locale to use. Passing an empty string ("") or SQL NULL value +** as the second argument is the same as invoking the 1 argument version +** of upper() or lower(). +** +** lower('I', 'en_us') -> 'i' +** lower('I', 'tr_tr') -> 'ı' (small dotless i) +** +** http://www.icu-project.org/userguide/posix.html#case_mappings +*/ +static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + const UChar *zInput; + UChar *zOutput; + int nInput; + int nOutput; + + UErrorCode status = U_ZERO_ERROR; + const char *zLocale = 0; + + assert(nArg==1 || nArg==2); + if( nArg==2 ){ + zLocale = (const char *)sqlite3_value_text(apArg[1]); + } + + zInput = sqlite3_value_text16(apArg[0]); + if( !zInput ){ + return; + } + nInput = sqlite3_value_bytes16(apArg[0]); + + nOutput = nInput * 2 + 2; + zOutput = sqlite3_malloc(nOutput); + if( !zOutput ){ + return; + } + + if( sqlite3_user_data(p) ){ + u_strToUpper(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); + }else{ + u_strToLower(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); + } + + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "u_strToLower()/u_strToUpper", status); + return; + } + + sqlite3_result_text16(p, zOutput, -1, xFree); +} + +/* +** Collation sequence destructor function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static void icuCollationDel(void *pCtx){ + UCollator *p = (UCollator *)pCtx; + ucol_close(p); +} + +/* +** Collation sequence comparison function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static int icuCollationColl( + void *pCtx, + int nLeft, + const void *zLeft, + int nRight, + const void *zRight +){ + UCollationResult res; + UCollator *p = (UCollator *)pCtx; + res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); + switch( res ){ + case UCOL_LESS: return -1; + case UCOL_GREATER: return +1; + case UCOL_EQUAL: return 0; + } + assert(!"Unexpected return value from ucol_strcoll()"); + return 0; +} + +/* +** Implementation of the scalar function icu_load_collation(). +** +** This scalar function is used to add ICU collation based collation +** types to an SQLite database connection. It is intended to be called +** as follows: +** +** SELECT icu_load_collation(, ); +** +** Where is a string containing an ICU locale identifier (i.e. +** "en_AU", "tr_TR" etc.) and is the name of the +** collation sequence to create. +*/ +static void icuLoadCollation( + sqlite3_context *p, + int nArg, + sqlite3_value **apArg +){ + sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); + UErrorCode status = U_ZERO_ERROR; + const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ + const char *zName; /* SQL Collation sequence name (eg. "japanese") */ + UCollator *pUCollator; /* ICU library collation object */ + int rc; /* Return code from sqlite3_create_collation_x() */ + + assert(nArg==2); + zLocale = (const char *)sqlite3_value_text(apArg[0]); + zName = (const char *)sqlite3_value_text(apArg[1]); + + if( !zLocale || !zName ){ + return; + } + + pUCollator = ucol_open(zLocale, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "ucol_open", status); + return; + } + assert(p); + + rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, + icuCollationColl, icuCollationDel + ); + if( rc!=SQLITE_OK ){ + ucol_close(pUCollator); + sqlite3_result_error(p, "Error registering collation function", -1); + } +} + +/* +** Register the ICU extension functions with database db. +*/ +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ + struct IcuScalar { + const char *zName; /* Function name */ + int nArg; /* Number of arguments */ + int enc; /* Optimal text encoding */ + void *pContext; /* sqlite3_user_data() context */ + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } scalars[] = { + {"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc}, + + {"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16, (void*)1, icuCaseFunc16}, + + {"lower", 1, SQLITE_UTF8, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8, (void*)1, icuCaseFunc16}, + + {"like", 2, SQLITE_UTF8, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8, 0, icuLikeFunc}, + + {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, + }; + + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ + struct IcuScalar *p = &scalars[i]; + rc = sqlite3_create_function( + db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 + ); + } + + return rc; +} + +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_icu_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3IcuInit(db); +} +#endif + +#endif + +/************** End of icu.c *************************************************/ +/************** Begin file fts3_icu.c ****************************************/ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a tokenizer for fts3 based on the ICU library. +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#ifdef SQLITE_ENABLE_ICU + +/* #include */ +/* #include */ + +#include +/* #include */ +/* #include */ +#include + +typedef struct IcuTokenizer IcuTokenizer; +typedef struct IcuCursor IcuCursor; + +struct IcuTokenizer { + sqlite3_tokenizer base; + char *zLocale; +}; + +struct IcuCursor { + sqlite3_tokenizer_cursor base; + + UBreakIterator *pIter; /* ICU break-iterator object */ + int nChar; /* Number of UChar elements in pInput */ + UChar *aChar; /* Copy of input using utf-16 encoding */ + int *aOffset; /* Offsets of each character in utf-8 input */ + + int nBuffer; + char *zBuffer; + + int iToken; +}; + +/* +** Create a new tokenizer instance. +*/ +static int icuCreate( + int argc, /* Number of entries in argv[] */ + const char * const *argv, /* Tokenizer creation arguments */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ +){ + IcuTokenizer *p; + int n = 0; + + if( argc>0 ){ + n = strlen(argv[0])+1; + } + p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n); + if( !p ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(IcuTokenizer)); + + if( n ){ + p->zLocale = (char *)&p[1]; + memcpy(p->zLocale, argv[0], n); + } + + *ppTokenizer = (sqlite3_tokenizer *)p; + + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int icuDestroy(sqlite3_tokenizer *pTokenizer){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int icuOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, /* Input string */ + int nInput, /* Length of zInput in bytes */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + IcuCursor *pCsr; + + const int32_t opt = U_FOLD_CASE_DEFAULT; + UErrorCode status = U_ZERO_ERROR; + int nChar; + + UChar32 c; + int iInput = 0; + int iOut = 0; + + *ppCursor = 0; + + if( zInput==0 ){ + nInput = 0; + zInput = ""; + }else if( nInput<0 ){ + nInput = strlen(zInput); + } + nChar = nInput+1; + pCsr = (IcuCursor *)sqlite3_malloc( + sizeof(IcuCursor) + /* IcuCursor */ + ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ + (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ + ); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(IcuCursor)); + pCsr->aChar = (UChar *)&pCsr[1]; + pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; + + pCsr->aOffset[iOut] = iInput; + U8_NEXT(zInput, iInput, nInput, c); + while( c>0 ){ + int isError = 0; + c = u_foldCase(c, opt); + U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); + if( isError ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->aOffset[iOut] = iInput; + + if( iInputpIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); + if( !U_SUCCESS(status) ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->nChar = iOut; + + ubrk_first(pCsr->pIter); + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to icuOpen(). +*/ +static int icuClose(sqlite3_tokenizer_cursor *pCursor){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + ubrk_close(pCsr->pIter); + sqlite3_free(pCsr->zBuffer); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. +*/ +static int icuNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + + int iStart = 0; + int iEnd = 0; + int nByte = 0; + + while( iStart==iEnd ){ + UChar32 c; + + iStart = ubrk_current(pCsr->pIter); + iEnd = ubrk_next(pCsr->pIter); + if( iEnd==UBRK_DONE ){ + return SQLITE_DONE; + } + + while( iStartaChar, iWhite, pCsr->nChar, c); + if( u_isspace(c) ){ + iStart = iWhite; + }else{ + break; + } + } + assert(iStart<=iEnd); + } + + do { + UErrorCode status = U_ZERO_ERROR; + if( nByte ){ + char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); + if( !zNew ){ + return SQLITE_NOMEM; + } + pCsr->zBuffer = zNew; + pCsr->nBuffer = nByte; + } + + u_strToUTF8( + pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ + &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ + &status /* Output success/failure */ + ); + } while( nByte>pCsr->nBuffer ); + + *ppToken = pCsr->zBuffer; + *pnBytes = nByte; + *piStartOffset = pCsr->aOffset[iStart]; + *piEndOffset = pCsr->aOffset[iEnd]; + *piPosition = pCsr->iToken++; + + return SQLITE_OK; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module icuTokenizerModule = { + 0, /* iVersion */ + icuCreate, /* xCreate */ + icuDestroy, /* xCreate */ + icuOpen, /* xOpen */ + icuClose, /* xClose */ + icuNext, /* xNext */ +}; + +/* +** Set *ppModule to point at the implementation of the ICU tokenizer. +*/ +SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &icuTokenizerModule; +} + +#endif /* defined(SQLITE_ENABLE_ICU) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_icu.c ********************************************/ diff --git a/Postbox/PostboxPrivate/sqlcipher/sqlite3.h b/Postbox/PostboxPrivate/sqlcipher/sqlite3.h new file mode 100644 index 0000000000..9879f80d40 --- /dev/null +++ b/Postbox/PostboxPrivate/sqlcipher/sqlite3.h @@ -0,0 +1,7494 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the SQLite library +** presents to client programs. If a C-function, structure, datatype, +** or constant definition does not appear in this file, then it is +** not a published API of SQLite, is subject to change without +** notice, and should not be referenced by programs that use SQLite. +** +** Some of the definitions that are in this file are marked as +** "experimental". Experimental interfaces are normally new +** features recently added to SQLite. We do not anticipate changes +** to experimental interfaces but reserve the right to make minor changes +** if experience from use "in the wild" suggest such changes are prudent. +** +** The official C-language API documentation for SQLite is derived +** from comments in this file. This file is the authoritative source +** on how SQLite interfaces are suppose to operate. +** +** The name of this file under configuration management is "sqlite.h.in". +** The makefile makes some minor changes to this file (such as inserting +** the version number) and changes its name to "sqlite3.h" as +** part of the build process. +*/ +#ifndef _SQLITE3_H_ +#define _SQLITE3_H_ +#include /* Needed for the definition of va_list */ + +/* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +extern "C" { +#endif + + +/* +** Add the ability to override 'extern' +*/ +#ifndef SQLITE_EXTERN +# define SQLITE_EXTERN extern +#endif + +#ifndef SQLITE_API +# define SQLITE_API +#endif + + +/* +** These no-op macros are used in front of interfaces to mark those +** interfaces as either deprecated or experimental. New applications +** should not use deprecated interfaces - they are support for backwards +** compatibility only. Application writers should be aware that +** experimental interfaces are subject to change in point releases. +** +** These macros used to resolve to various kinds of compiler magic that +** would generate warning messages when they were used. But that +** compiler magic ended up generating such a flurry of bug reports +** that we have taken it all out and gone back to using simple +** noop macros. +*/ +#define SQLITE_DEPRECATED +#define SQLITE_EXPERIMENTAL + +/* +** Ensure these symbols were not defined by some previous header file. +*/ +#ifdef SQLITE_VERSION +# undef SQLITE_VERSION +#endif +#ifdef SQLITE_VERSION_NUMBER +# undef SQLITE_VERSION_NUMBER +#endif + +/* +** CAPI3REF: Compile-Time Library Version Numbers +** +** ^(The [SQLITE_VERSION] C preprocessor macro in the sqlite3.h header +** evaluates to a string literal that is the SQLite version in the +** format "X.Y.Z" where X is the major version number (always 3 for +** SQLite3) and Y is the minor version number and Z is the release number.)^ +** ^(The [SQLITE_VERSION_NUMBER] C preprocessor macro resolves to an integer +** with the value (X*1000000 + Y*1000 + Z) where X, Y, and Z are the same +** numbers used in [SQLITE_VERSION].)^ +** The SQLITE_VERSION_NUMBER for any given release of SQLite will also +** be larger than the release from which it is derived. Either Y will +** be held constant and Z will be incremented or else Y will be incremented +** and Z will be reset to zero. +** +** Since version 3.6.18, SQLite source code has been stored in the +** Fossil configuration management +** system. ^The SQLITE_SOURCE_ID macro evaluates to +** a string which identifies a particular check-in of SQLite +** within its configuration management system. ^The SQLITE_SOURCE_ID +** string contains the date and time of the check-in (UTC) and an SHA1 +** hash of the entire source tree. +** +** See also: [sqlite3_libversion()], +** [sqlite3_libversion_number()], [sqlite3_sourceid()], +** [sqlite_version()] and [sqlite_source_id()]. +*/ +#define SQLITE_VERSION "3.8.6" +#define SQLITE_VERSION_NUMBER 3008006 +#define SQLITE_SOURCE_ID "2014-08-15 11:46:33 9491ba7d738528f168657adb43a198238abde19e" + +/* +** CAPI3REF: Run-Time Library Version Numbers +** KEYWORDS: sqlite3_version, sqlite3_sourceid +** +** These interfaces provide the same information as the [SQLITE_VERSION], +** [SQLITE_VERSION_NUMBER], and [SQLITE_SOURCE_ID] C preprocessor macros +** but are associated with the library instead of the header file. ^(Cautious +** programmers might include assert() statements in their application to +** verify that values returned by these interfaces match the macros in +** the header, and thus insure that the application is +** compiled with matching library and header files. +** +** 
    +** assert( sqlite3_libversion_number()==SQLITE_VERSION_NUMBER );
+** assert( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)==0 );
+** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
+**
    )^ +** +** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] +** macro. ^The sqlite3_libversion() function returns a pointer to the +** to the sqlite3_version[] string constant. The sqlite3_libversion() +** function is provided for use in DLLs since DLL users usually do not have +** direct access to string constants within the DLL. ^The +** sqlite3_libversion_number() function returns an integer equal to +** [SQLITE_VERSION_NUMBER]. ^The sqlite3_sourceid() function returns +** a pointer to a string constant whose value is the same as the +** [SQLITE_SOURCE_ID] C preprocessor macro. +** +** See also: [sqlite_version()] and [sqlite_source_id()]. +*/ +SQLITE_API SQLITE_EXTERN const char sqlite3_version[]; +SQLITE_API const char *sqlite3_libversion(void); +SQLITE_API const char *sqlite3_sourceid(void); +SQLITE_API int sqlite3_libversion_number(void); + +/* +** CAPI3REF: Run-Time Library Compilation Options Diagnostics +** +** ^The sqlite3_compileoption_used() function returns 0 or 1 +** indicating whether the specified option was defined at +** compile time. ^The SQLITE_ prefix may be omitted from the +** option name passed to sqlite3_compileoption_used(). +** +** ^The sqlite3_compileoption_get() function allows iterating +** over the list of options that were defined at compile time by +** returning the N-th compile time option string. ^If N is out of range, +** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ +** prefix is omitted from any strings returned by +** sqlite3_compileoption_get(). +** +** ^Support for the diagnostic functions sqlite3_compileoption_used() +** and sqlite3_compileoption_get() may be omitted by specifying the +** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. +** +** See also: SQL functions [sqlite_compileoption_used()] and +** [sqlite_compileoption_get()] and the [compile_options pragma]. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +SQLITE_API int sqlite3_compileoption_used(const char *zOptName); +SQLITE_API const char *sqlite3_compileoption_get(int N); +#endif + +/* +** CAPI3REF: Test To See If The Library Is Threadsafe +** +** ^The sqlite3_threadsafe() function returns zero if and only if +** SQLite was compiled with mutexing code omitted due to the +** [SQLITE_THREADSAFE] compile-time option being set to 0. +** +** SQLite can be compiled with or without mutexes. When +** the [SQLITE_THREADSAFE] C preprocessor macro is 1 or 2, mutexes +** are enabled and SQLite is threadsafe. When the +** [SQLITE_THREADSAFE] macro is 0, +** the mutexes are omitted. Without the mutexes, it is not safe +** to use SQLite concurrently from more than one thread. +** +** Enabling mutexes incurs a measurable performance penalty. +** So if speed is of utmost importance, it makes sense to disable +** the mutexes. But for maximum safety, mutexes should be enabled. +** ^The default behavior is for mutexes to be enabled. +** +** This interface can be used by an application to make sure that the +** version of SQLite that it is linking against was compiled with +** the desired setting of the [SQLITE_THREADSAFE] macro. +** +** This interface only reports on the compile-time mutex setting +** of the [SQLITE_THREADSAFE] flag. If SQLite is compiled with +** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but +** can be fully or partially disabled using a call to [sqlite3_config()] +** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD], +** or [SQLITE_CONFIG_MUTEX]. ^(The return value of the +** sqlite3_threadsafe() function shows only the compile-time setting of +** thread safety, not any run-time changes to that setting made by +** sqlite3_config(). In other words, the return value from sqlite3_threadsafe() +** is unchanged by calls to sqlite3_config().)^ +** +** See the [threading mode] documentation for additional information. +*/ +SQLITE_API int sqlite3_threadsafe(void); + +/* +** CAPI3REF: Database Connection Handle +** KEYWORDS: {database connection} {database connections} +** +** Each open SQLite database is represented by a pointer to an instance of +** the opaque structure named "sqlite3". It is useful to think of an sqlite3 +** pointer as an object. The [sqlite3_open()], [sqlite3_open16()], and +** [sqlite3_open_v2()] interfaces are its constructors, and [sqlite3_close()] +** and [sqlite3_close_v2()] are its destructors. There are many other +** interfaces (such as +** [sqlite3_prepare_v2()], [sqlite3_create_function()], and +** [sqlite3_busy_timeout()] to name but three) that are methods on an +** sqlite3 object. +*/ +typedef struct sqlite3 sqlite3; + +/* +** CAPI3REF: 64-Bit Integer Types +** KEYWORDS: sqlite_int64 sqlite_uint64 +** +** Because there is no cross-platform way to specify 64-bit integer types +** SQLite includes typedefs for 64-bit signed and unsigned integers. +** +** The sqlite3_int64 and sqlite3_uint64 are the preferred type definitions. +** The sqlite_int64 and sqlite_uint64 types are supported for backwards +** compatibility only. +** +** ^The sqlite3_int64 and sqlite_int64 types can store integer values +** between -9223372036854775808 and +9223372036854775807 inclusive. ^The +** sqlite3_uint64 and sqlite_uint64 types can store integer values +** between 0 and +18446744073709551615 inclusive. +*/ +#ifdef SQLITE_INT64_TYPE + typedef SQLITE_INT64_TYPE sqlite_int64; + typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +#elif defined(_MSC_VER) || defined(__BORLANDC__) + typedef __int64 sqlite_int64; + typedef unsigned __int64 sqlite_uint64; +#else + typedef long long int sqlite_int64; + typedef unsigned long long int sqlite_uint64; +#endif +typedef sqlite_int64 sqlite3_int64; +typedef sqlite_uint64 sqlite3_uint64; + +/* +** If compiling for a processor that lacks floating point support, +** substitute integer for floating-point. +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# define double sqlite3_int64 +#endif + +/* +** CAPI3REF: Closing A Database Connection +** +** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors +** for the [sqlite3] object. +** ^Calls to sqlite3_close() and sqlite3_close_v2() return [SQLITE_OK] if +** the [sqlite3] object is successfully destroyed and all associated +** resources are deallocated. +** +** ^If the database connection is associated with unfinalized prepared +** statements or unfinished sqlite3_backup objects then sqlite3_close() +** will leave the database connection open and return [SQLITE_BUSY]. +** ^If sqlite3_close_v2() is called with unfinalized prepared statements +** and/or unfinished sqlite3_backups, then the database connection becomes +** an unusable "zombie" which will automatically be deallocated when the +** last prepared statement is finalized or the last sqlite3_backup is +** finished. The sqlite3_close_v2() interface is intended for use with +** host languages that are garbage collected, and where the order in which +** destructors are called is arbitrary. +** +** Applications should [sqlite3_finalize | finalize] all [prepared statements], +** [sqlite3_blob_close | close] all [BLOB handles], and +** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated +** with the [sqlite3] object prior to attempting to close the object. ^If +** sqlite3_close_v2() is called on a [database connection] that still has +** outstanding [prepared statements], [BLOB handles], and/or +** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation +** of resources is deferred until all [prepared statements], [BLOB handles], +** and [sqlite3_backup] objects are also destroyed. +** +** ^If an [sqlite3] object is destroyed while a transaction is open, +** the transaction is automatically rolled back. +** +** The C parameter to [sqlite3_close(C)] and [sqlite3_close_v2(C)] +** must be either a NULL +** pointer or an [sqlite3] object pointer obtained +** from [sqlite3_open()], [sqlite3_open16()], or +** [sqlite3_open_v2()], and not previously closed. +** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer +** argument is a harmless no-op. +*/ +SQLITE_API int sqlite3_close(sqlite3*); +SQLITE_API int sqlite3_close_v2(sqlite3*); + +/* +** The type for a callback function. +** This is legacy and deprecated. It is included for historical +** compatibility and is not documented. +*/ +typedef int (*sqlite3_callback)(void*,int,char**, char**); + +/* +** CAPI3REF: One-Step Query Execution Interface +** +** The sqlite3_exec() interface is a convenience wrapper around +** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()], +** that allows an application to run multiple statements of SQL +** without having to use a lot of C code. +** +** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, +** semicolon-separate SQL statements passed into its 2nd argument, +** in the context of the [database connection] passed in as its 1st +** argument. ^If the callback function of the 3rd argument to +** sqlite3_exec() is not NULL, then it is invoked for each result row +** coming out of the evaluated SQL statements. ^The 4th argument to +** sqlite3_exec() is relayed through to the 1st argument of each +** callback invocation. ^If the callback pointer to sqlite3_exec() +** is NULL, then no callback is ever invoked and result rows are +** ignored. +** +** ^If an error occurs while evaluating the SQL statements passed into +** sqlite3_exec(), then execution of the current statement stops and +** subsequent statements are skipped. ^If the 5th parameter to sqlite3_exec() +** is not NULL then any error message is written into memory obtained +** from [sqlite3_malloc()] and passed back through the 5th parameter. +** To avoid memory leaks, the application should invoke [sqlite3_free()] +** on error message strings returned through the 5th parameter of +** of sqlite3_exec() after the error message string is no longer needed. +** ^If the 5th parameter to sqlite3_exec() is not NULL and no errors +** occur, then sqlite3_exec() sets the pointer in its 5th parameter to +** NULL before returning. +** +** ^If an sqlite3_exec() callback returns non-zero, the sqlite3_exec() +** routine returns SQLITE_ABORT without invoking the callback again and +** without running any subsequent SQL statements. +** +** ^The 2nd argument to the sqlite3_exec() callback function is the +** number of columns in the result. ^The 3rd argument to the sqlite3_exec() +** callback is an array of pointers to strings obtained as if from +** [sqlite3_column_text()], one for each column. ^If an element of a +** result row is NULL then the corresponding string pointer for the +** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the +** sqlite3_exec() callback is an array of pointers to strings where each +** entry represents the name of corresponding result column as obtained +** from [sqlite3_column_name()]. +** +** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer +** to an empty string, or a pointer that contains only whitespace and/or +** SQL comments, then no SQL statements are evaluated and the database +** is not changed. +** +** Restrictions: +** +**
      +**
    • The application must insure that the 1st parameter to sqlite3_exec() +** is a valid and open [database connection]. +**
    • The application must not close the [database connection] specified by +** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. +**
    • The application must not modify the SQL statement text passed into +** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
    +*/ +SQLITE_API int sqlite3_exec( + sqlite3*, /* An open database */ + const char *sql, /* SQL to be evaluated */ + int (*callback)(void*,int,char**,char**), /* Callback function */ + void *, /* 1st argument to callback */ + char **errmsg /* Error msg written here */ +); + +/* +** CAPI3REF: Result Codes +** KEYWORDS: {result code definitions} +** +** Many SQLite functions return an integer result code from the set shown +** here in order to indicate success or failure. +** +** New error codes may be added in future versions of SQLite. +** +** See also: [extended result code definitions] +*/ +#define SQLITE_OK 0 /* Successful result */ +/* beginning-of-error-codes */ +#define SQLITE_ERROR 1 /* SQL error or missing database */ +#define SQLITE_INTERNAL 2 /* Internal logic error in SQLite */ +#define SQLITE_PERM 3 /* Access permission denied */ +#define SQLITE_ABORT 4 /* Callback routine requested an abort */ +#define SQLITE_BUSY 5 /* The database file is locked */ +#define SQLITE_LOCKED 6 /* A table in the database is locked */ +#define SQLITE_NOMEM 7 /* A malloc() failed */ +#define SQLITE_READONLY 8 /* Attempt to write a readonly database */ +#define SQLITE_INTERRUPT 9 /* Operation terminated by sqlite3_interrupt()*/ +#define SQLITE_IOERR 10 /* Some kind of disk I/O error occurred */ +#define SQLITE_CORRUPT 11 /* The database disk image is malformed */ +#define SQLITE_NOTFOUND 12 /* Unknown opcode in sqlite3_file_control() */ +#define SQLITE_FULL 13 /* Insertion failed because database is full */ +#define SQLITE_CANTOPEN 14 /* Unable to open the database file */ +#define SQLITE_PROTOCOL 15 /* Database lock protocol error */ +#define SQLITE_EMPTY 16 /* Database is empty */ +#define SQLITE_SCHEMA 17 /* The database schema changed */ +#define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ +#define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ +#define SQLITE_MISMATCH 20 /* Data type mismatch */ +#define SQLITE_MISUSE 21 /* Library used incorrectly */ +#define SQLITE_NOLFS 22 /* Uses OS features not supported on host */ +#define SQLITE_AUTH 23 /* Authorization denied */ +#define SQLITE_FORMAT 24 /* Auxiliary database format error */ +#define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ +#define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ +#define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ +#define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ +/* end-of-error-codes */ + +/* +** CAPI3REF: Extended Result Codes +** KEYWORDS: {extended result code definitions} +** +** In its default configuration, SQLite API routines return one of 30 integer +** [result codes]. However, experience has shown that many of +** these result codes are too coarse-grained. They do not provide as +** much information about problems as programmers might like. In an effort to +** address this, newer versions of SQLite (version 3.3.8 and later) include +** support for additional result codes that provide more detailed information +** about errors. These [extended result codes] are enabled or disabled +** on a per database connection basis using the +** [sqlite3_extended_result_codes()] API. Or, the extended code for +** the most recent error can be obtained using +** [sqlite3_extended_errcode()]. +*/ +#define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) +#define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) +#define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) +#define SQLITE_IOERR_FSYNC (SQLITE_IOERR | (4<<8)) +#define SQLITE_IOERR_DIR_FSYNC (SQLITE_IOERR | (5<<8)) +#define SQLITE_IOERR_TRUNCATE (SQLITE_IOERR | (6<<8)) +#define SQLITE_IOERR_FSTAT (SQLITE_IOERR | (7<<8)) +#define SQLITE_IOERR_UNLOCK (SQLITE_IOERR | (8<<8)) +#define SQLITE_IOERR_RDLOCK (SQLITE_IOERR | (9<<8)) +#define SQLITE_IOERR_DELETE (SQLITE_IOERR | (10<<8)) +#define SQLITE_IOERR_BLOCKED (SQLITE_IOERR | (11<<8)) +#define SQLITE_IOERR_NOMEM (SQLITE_IOERR | (12<<8)) +#define SQLITE_IOERR_ACCESS (SQLITE_IOERR | (13<<8)) +#define SQLITE_IOERR_CHECKRESERVEDLOCK (SQLITE_IOERR | (14<<8)) +#define SQLITE_IOERR_LOCK (SQLITE_IOERR | (15<<8)) +#define SQLITE_IOERR_CLOSE (SQLITE_IOERR | (16<<8)) +#define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8)) +#define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) +#define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) +#define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) +#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) +#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) +#define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) +#define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) +#define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) +#define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) +#define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) +#define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) +#define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) + +/* +** CAPI3REF: Flags For File Open Operations +** +** These bit values are intended for use in the +** 3rd parameter to the [sqlite3_open_v2()] interface and +** in the 4th parameter to the [sqlite3_vfs.xOpen] method. +*/ +#define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_CREATE 0x00000004 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ +#define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ +#define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ +#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MEMORY 0x00000080 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ +#define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ +#define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ +#define SQLITE_OPEN_MAIN_JOURNAL 0x00000800 /* VFS only */ +#define SQLITE_OPEN_TEMP_JOURNAL 0x00001000 /* VFS only */ +#define SQLITE_OPEN_SUBJOURNAL 0x00002000 /* VFS only */ +#define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 /* VFS only */ +#define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ + +/* Reserved: 0x00F00000 */ + +/* +** CAPI3REF: Device Characteristics +** +** The xDeviceCharacteristics method of the [sqlite3_io_methods] +** object returns an integer which is a vector of these +** bit values expressing I/O characteristics of the mass storage +** device that holds the file that the [sqlite3_io_methods] +** refers to. +** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). The SQLITE_IOCAP_POWERSAFE_OVERWRITE property means that +** after reboot following a crash or power loss, the only bytes in a +** file that were written at the application level might have changed +** and that adjacent bytes, even bytes within the same sector are +** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN +** flag indicate that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. +*/ +#define SQLITE_IOCAP_ATOMIC 0x00000001 +#define SQLITE_IOCAP_ATOMIC512 0x00000002 +#define SQLITE_IOCAP_ATOMIC1K 0x00000004 +#define SQLITE_IOCAP_ATOMIC2K 0x00000008 +#define SQLITE_IOCAP_ATOMIC4K 0x00000010 +#define SQLITE_IOCAP_ATOMIC8K 0x00000020 +#define SQLITE_IOCAP_ATOMIC16K 0x00000040 +#define SQLITE_IOCAP_ATOMIC32K 0x00000080 +#define SQLITE_IOCAP_ATOMIC64K 0x00000100 +#define SQLITE_IOCAP_SAFE_APPEND 0x00000200 +#define SQLITE_IOCAP_SEQUENTIAL 0x00000400 +#define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 +#define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 + +/* +** CAPI3REF: File Locking Levels +** +** SQLite uses one of these integer values as the second +** argument to calls it makes to the xLock() and xUnlock() methods +** of an [sqlite3_io_methods] object. +*/ +#define SQLITE_LOCK_NONE 0 +#define SQLITE_LOCK_SHARED 1 +#define SQLITE_LOCK_RESERVED 2 +#define SQLITE_LOCK_PENDING 3 +#define SQLITE_LOCK_EXCLUSIVE 4 + +/* +** CAPI3REF: Synchronization Type Flags +** +** When SQLite invokes the xSync() method of an +** [sqlite3_io_methods] object it uses a combination of +** these integer values as the second argument. +** +** When the SQLITE_SYNC_DATAONLY flag is used, it means that the +** sync operation only needs to flush data to mass storage. Inode +** information need not be flushed. If the lower four bits of the flag +** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics. +** If the lower four bits equal SQLITE_SYNC_FULL, that means +** to use Mac OS X style fullsync instead of fsync(). +** +** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags +** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL +** settings. The [synchronous pragma] determines when calls to the +** xSync VFS method occur and applies uniformly across all platforms. +** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how +** energetic or rigorous or forceful the sync operations are and +** only make a difference on Mac OSX for the default SQLite code. +** (Third-party VFS implementations might also make the distinction +** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the +** operating systems natively supported by SQLite, only Mac OSX +** cares about the difference.) +*/ +#define SQLITE_SYNC_NORMAL 0x00002 +#define SQLITE_SYNC_FULL 0x00003 +#define SQLITE_SYNC_DATAONLY 0x00010 + +/* +** CAPI3REF: OS Interface Open File Handle +** +** An [sqlite3_file] object represents an open file in the +** [sqlite3_vfs | OS interface layer]. Individual OS interface +** implementations will +** want to subclass this object by appending additional fields +** for their own use. The pMethods entry is a pointer to an +** [sqlite3_io_methods] object that defines methods for performing +** I/O operations on the open file. +*/ +typedef struct sqlite3_file sqlite3_file; +struct sqlite3_file { + const struct sqlite3_io_methods *pMethods; /* Methods for an open file */ +}; + +/* +** CAPI3REF: OS Interface File Virtual Methods Object +** +** Every file opened by the [sqlite3_vfs.xOpen] method populates an +** [sqlite3_file] object (or, more commonly, a subclass of the +** [sqlite3_file] object) with a pointer to an instance of this object. +** This object defines the methods used to perform various operations +** against the open file represented by the [sqlite3_file] object. +** +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element +** to a non-NULL pointer, then the sqlite3_io_methods.xClose method +** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The +** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] +** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element +** to NULL. +** +** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or +** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). +** The second choice is a Mac OS X style fullsync. The [SQLITE_SYNC_DATAONLY] +** flag may be ORed in to indicate that only the data of the file +** and not its inode needs to be synced. +** +** The integer values to xLock() and xUnlock() are one of +**
      +**
    • [SQLITE_LOCK_NONE], +**
    • [SQLITE_LOCK_SHARED], +**
    • [SQLITE_LOCK_RESERVED], +**
    • [SQLITE_LOCK_PENDING], or +**
    • [SQLITE_LOCK_EXCLUSIVE]. +**
    +** xLock() increases the lock. xUnlock() decreases the lock. +** The xCheckReservedLock() method checks whether any database connection, +** either in this process or in some other process, is holding a RESERVED, +** PENDING, or EXCLUSIVE lock on the file. It returns true +** if such a lock exists and false otherwise. +** +** The xFileControl() method is a generic interface that allows custom +** VFS implementations to directly control an open file using the +** [sqlite3_file_control()] interface. The second "op" argument is an +** integer opcode. The third argument is a generic pointer intended to +** point to a structure that may contain arguments or space in which to +** write return values. Potential uses for xFileControl() might be +** functions to enable blocking locks with timeouts, to change the +** locking strategy (for example to use dot-file locks), to inquire +** about the status of a lock, or to break stale locks. The SQLite +** core reserves all opcodes less than 100 for its own use. +** A [file control opcodes | list of opcodes] less than 100 is available. +** Applications that define a custom xFileControl method should use opcodes +** greater than 100 to avoid conflicts. VFS implementations should +** return [SQLITE_NOTFOUND] for file control opcodes that they do not +** recognize. +** +** The xSectorSize() method returns the sector size of the +** device that underlies the file. The sector size is the +** minimum write that can be performed without disturbing +** other bytes in the file. The xDeviceCharacteristics() +** method returns a bit vector describing behaviors of the +** underlying device: +** +**
      +**
    • [SQLITE_IOCAP_ATOMIC] +**
    • [SQLITE_IOCAP_ATOMIC512] +**
    • [SQLITE_IOCAP_ATOMIC1K] +**
    • [SQLITE_IOCAP_ATOMIC2K] +**
    • [SQLITE_IOCAP_ATOMIC4K] +**
    • [SQLITE_IOCAP_ATOMIC8K] +**
    • [SQLITE_IOCAP_ATOMIC16K] +**
    • [SQLITE_IOCAP_ATOMIC32K] +**
    • [SQLITE_IOCAP_ATOMIC64K] +**
    • [SQLITE_IOCAP_SAFE_APPEND] +**
    • [SQLITE_IOCAP_SEQUENTIAL] +**
    +** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). +** +** If xRead() returns SQLITE_IOERR_SHORT_READ it must also fill +** in the unread portions of the buffer with zeros. A VFS that +** fails to zero-fill short reads might seem to work. However, +** failure to zero-fill short reads will eventually lead to +** database corruption. +*/ +typedef struct sqlite3_io_methods sqlite3_io_methods; +struct sqlite3_io_methods { + int iVersion; + int (*xClose)(sqlite3_file*); + int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); + int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst); + int (*xTruncate)(sqlite3_file*, sqlite3_int64 size); + int (*xSync)(sqlite3_file*, int flags); + int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize); + int (*xLock)(sqlite3_file*, int); + int (*xUnlock)(sqlite3_file*, int); + int (*xCheckReservedLock)(sqlite3_file*, int *pResOut); + int (*xFileControl)(sqlite3_file*, int op, void *pArg); + int (*xSectorSize)(sqlite3_file*); + int (*xDeviceCharacteristics)(sqlite3_file*); + /* Methods above are valid for version 1 */ + int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); + int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); + void (*xShmBarrier)(sqlite3_file*); + int (*xShmUnmap)(sqlite3_file*, int deleteFlag); + /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ + /* Additional methods may be added in future releases */ +}; + +/* +** CAPI3REF: Standard File Control Opcodes +** KEYWORDS: {file control opcodes} {file control opcode} +** +** These integer constants are opcodes for the xFileControl method +** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] +** interface. +** +** The [SQLITE_FCNTL_LOCKSTATE] opcode is used for debugging. This +** opcode causes the xFileControl method to write the current state of +** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], +** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) +** into an integer that the pArg argument points to. This capability +** is used during testing and only needs to be supported when SQLITE_TEST +** is defined. +**
      +**
    • [[SQLITE_FCNTL_SIZE_HINT]] +** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS +** layer a hint of how large the database file will grow to be during the +** current transaction. This hint is not guaranteed to be accurate but it +** is often close. The underlying VFS might choose to preallocate database +** file space based on this hint in order to help writes to the database +** file run faster. +** +**
    • [[SQLITE_FCNTL_CHUNK_SIZE]] +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. +** +**
    • [[SQLITE_FCNTL_FILE_POINTER]] +** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with a particular database +** connection. See the [sqlite3_file_control()] documentation for +** additional information. +** +**
    • [[SQLITE_FCNTL_SYNC_OMITTED]] +** No longer in use. +** +**
    • [[SQLITE_FCNTL_SYNC]] +** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and +** sent to the VFS immediately before the xSync method is invoked on a +** database file descriptor. Or, if the xSync method is not invoked +** because the user has configured SQLite with +** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place +** of the xSync method. In most cases, the pointer argument passed with +** this file-control is NULL. However, if the database file is being synced +** as part of a multi-database commit, the argument points to a nul-terminated +** string containing the transactions master-journal file name. VFSes that +** do not need this signal should silently ignore this opcode. Applications +** should not call [sqlite3_file_control()] with this opcode as doing so may +** disrupt the operation of the specialized VFSes that do require it. +** +**
    • [[SQLITE_FCNTL_COMMIT_PHASETWO]] +** The [SQLITE_FCNTL_COMMIT_PHASETWO] opcode is generated internally by SQLite +** and sent to the VFS after a transaction has been committed immediately +** but before the database is unlocked. VFSes that do not need this signal +** should silently ignore this opcode. Applications should not call +** [sqlite3_file_control()] with this opcode as doing so may disrupt the +** operation of the specialized VFSes that do require it. +** +**
    • [[SQLITE_FCNTL_WIN32_AV_RETRY]] +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to provide robustness in the presence of +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete operations up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows these two values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer i the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** +**
    • [[SQLITE_FCNTL_PERSIST_WAL]] +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary +** write ahead log and shared memory files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. +** +**
    • [[SQLITE_FCNTL_POWERSAFE_OVERWRITE]] +** ^The [SQLITE_FCNTL_POWERSAFE_OVERWRITE] opcode is used to set or query the +** persistent "powersafe-overwrite" or "PSOW" setting. The PSOW setting +** determines the [SQLITE_IOCAP_POWERSAFE_OVERWRITE] bit of the +** xDeviceCharacteristics methods. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable zero-damage mode or 1 to enable zero-damage +** mode. If the integer is -1, then it is overwritten with the current +** zero-damage mode setting. +** +**
    • [[SQLITE_FCNTL_OVERWRITE]] +** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening +** a write transaction to indicate that, unless it is rolled back for some +** reason, the entire database file will be overwritten by the current +** transaction. This is used by VACUUM operations. +** +**
    • [[SQLITE_FCNTL_VFSNAME]] +** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of +** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** final bottom-level VFS are written into memory obtained from +** [sqlite3_malloc()] and the result is stored in the char* variable +** that the fourth parameter of [sqlite3_file_control()] points to. +** The caller is responsible for freeing the memory when done. As with +** all file-control actions, there is no guarantee that this will actually +** do anything. Callers should initialize the char* variable to a NULL +** pointer in case this file-control is not implemented. This file-control +** is intended for diagnostic use only. +** +**
    • [[SQLITE_FCNTL_PRAGMA]] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^The argument +** to the [SQLITE_FCNTL_PRAGMA] file control is an array of +** pointers to strings (char**) in which the second element of the array +** is the name of the pragma and the third element is the argument to the +** pragma or NULL if the pragma has no argument. ^The handler for an +** [SQLITE_FCNTL_PRAGMA] file control can optionally make the first element +** of the char** argument point to a string obtained from [sqlite3_mprintf()] +** or the equivalent and that string will become the result of the pragma or +** the error message if the pragma fails. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the PRAGMA itself and the parser generates a no-op +** prepared statement. ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means +** that the VFS encountered an error while handling the [PRAGMA] and the +** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] +** file control occurs at the beginning of pragma statement analysis and so +** it is able to override built-in [PRAGMA] statements. +** +**
    • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connections busy-handler callback. The argument is of type (void **) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connections +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
    • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** +**
    • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** +**
    • [[SQLITE_FCNTL_TRACE]] +** The [SQLITE_FCNTL_TRACE] file control provides advisory information +** to the VFS about what the higher layers of the SQLite stack are doing. +** This file control is used by some VFS activity tracing [shims]. +** The argument is a zero-terminated string. Higher layers in the +** SQLite stack may generate instances of this file control if +** the [SQLITE_USE_FCNTL_TRACE] compile-time option is enabled. +** +**
    • [[SQLITE_FCNTL_HAS_MOVED]] +** The [SQLITE_FCNTL_HAS_MOVED] file control interprets its argument as a +** pointer to an integer and it writes a boolean into that integer depending +** on whether or not the file has been renamed, moved, or deleted since it +** was first opened. +** +**
    • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This +** opcode causes the xFileControl method to swap the file handle with the one +** pointed to by the pArg argument. This capability is used during testing +** and only needs to be supported when SQLITE_TEST is defined. +** +**
    +*/ +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_GET_LOCKPROXYFILE 2 +#define SQLITE_SET_LOCKPROXYFILE 3 +#define SQLITE_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 +#define SQLITE_FCNTL_PERSIST_WAL 10 +#define SQLITE_FCNTL_OVERWRITE 11 +#define SQLITE_FCNTL_VFSNAME 12 +#define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_TRACE 19 +#define SQLITE_FCNTL_HAS_MOVED 20 +#define SQLITE_FCNTL_SYNC 21 +#define SQLITE_FCNTL_COMMIT_PHASETWO 22 +#define SQLITE_FCNTL_WIN32_SET_HANDLE 23 + +/* +** CAPI3REF: Mutex Handle +** +** The mutex module within SQLite defines [sqlite3_mutex] to be an +** abstract type for a mutex object. The SQLite core never looks +** at the internal representation of an [sqlite3_mutex]. It only +** deals with pointers to the [sqlite3_mutex] object. +** +** Mutexes are created using [sqlite3_mutex_alloc()]. +*/ +typedef struct sqlite3_mutex sqlite3_mutex; + +/* +** CAPI3REF: OS Interface Object +** +** An instance of the sqlite3_vfs object defines the interface between +** the SQLite core and the underlying operating system. The "vfs" +** in the name of the object stands for "virtual file system". See +** the [VFS | VFS documentation] for further information. +** +** The value of the iVersion field is initially 1 but may be larger in +** future versions of SQLite. Additional fields may be appended to this +** object when the iVersion value is increased. Note that the structure +** of the sqlite3_vfs object changes in the transaction between +** SQLite version 3.5.9 and 3.6.0 and yet the iVersion field was not +** modified. +** +** The szOsFile field is the size of the subclassed [sqlite3_file] +** structure used by this VFS. mxPathname is the maximum length of +** a pathname in this VFS. +** +** Registered sqlite3_vfs objects are kept on a linked list formed by +** the pNext pointer. The [sqlite3_vfs_register()] +** and [sqlite3_vfs_unregister()] interfaces manage this list +** in a thread-safe way. The [sqlite3_vfs_find()] interface +** searches the list. Neither the application code nor the VFS +** implementation should use the pNext pointer. +** +** The pNext field is the only field in the sqlite3_vfs +** structure that SQLite will ever modify. SQLite will only access +** or modify this field while holding a particular static mutex. +** The application should never modify anything within the sqlite3_vfs +** object once the object has been registered. +** +** The zName field holds the name of the VFS module. The name must +** be unique across all VFS modules. +** +** [[sqlite3_vfs.xOpen]] +** ^SQLite guarantees that the zFilename parameter to xOpen +** is either a NULL pointer or string obtained +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 11 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that +** the string will be valid and unchanged until xClose() is +** called. Because of the previous sentence, +** the [sqlite3_file] can safely store a pointer to the +** filename if it needs to remember the filename for some reason. +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the +** xFilename parameter is NULL it will also be the case that the +** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. +** +** The flags argument to xOpen() includes all bits set in +** the flags argument to [sqlite3_open_v2()]. Or if [sqlite3_open()] +** or [sqlite3_open16()] is used, then flags includes at least +** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. +** If xOpen() opens a file read-only then it sets *pOutFlags to +** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. +** +** ^(SQLite will also add one of the following flags to the xOpen() +** call, depending on the object being opened: +** +**
      +**
    • [SQLITE_OPEN_MAIN_DB] +**
    • [SQLITE_OPEN_MAIN_JOURNAL] +**
    • [SQLITE_OPEN_TEMP_DB] +**
    • [SQLITE_OPEN_TEMP_JOURNAL] +**
    • [SQLITE_OPEN_TRANSIENT_DB] +**
    • [SQLITE_OPEN_SUBJOURNAL] +**
    • [SQLITE_OPEN_MASTER_JOURNAL] +**
    • [SQLITE_OPEN_WAL] +**
    )^ +** +** The file I/O implementation can use the object type flags to +** change the way it deals with files. For example, an application +** that does not care about crash recovery or rollback might make +** the open of a journal file a no-op. Writes to this journal would +** also be no-ops, and any attempt to read the journal would return +** SQLITE_IOERR. Or the implementation might recognize that a database +** file will be doing page-aligned sector reads and writes in a random +** order and set up its I/O subsystem accordingly. +** +** SQLite might also add one of the following flags to the xOpen method: +** +**
      +**
    • [SQLITE_OPEN_DELETEONCLOSE] +**
    • [SQLITE_OPEN_EXCLUSIVE] +**
    +** +** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. +** +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** with the [SQLITE_OPEN_CREATE] flag, which are both directly +** analogous to the O_EXCL and O_CREAT flags of the POSIX open() +** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the +** SQLITE_OPEN_CREATE, is used to indicate that file should always +** be created, and that it is an error if it already exists. +** It is not used to indicate the file should be opened +** for exclusive access. +** +** ^At least szOsFile bytes of memory are allocated by SQLite +** to hold the [sqlite3_file] structure passed as the third +** argument to xOpen. The xOpen method does not have to +** allocate the structure; it should just fill it in. Note that +** the xOpen method must set the sqlite3_file.pMethods to either +** a valid [sqlite3_io_methods] object or to NULL. xOpen must do +** this even if the open fails. SQLite expects that the sqlite3_file.pMethods +** element will be valid after xOpen returns regardless of the success +** or failure of the xOpen call. +** +** [[sqlite3_vfs.xAccess]] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to +** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] +** to test whether a file is at least readable. The file can be a +** directory. +** +** ^SQLite will always allocate at least mxPathname+1 bytes for the +** output buffer xFullPathname. The exact size of the output buffer +** is also passed as a parameter to both methods. If the output buffer +** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is +** handled as a fatal error by SQLite, vfs implementations should endeavor +** to prevent this by setting mxPathname to a sufficiently large value. +** +** The xRandomness(), xSleep(), xCurrentTime(), and xCurrentTimeInt64() +** interfaces are not strictly a part of the filesystem, but they are +** included in the VFS structure for completeness. +** The xRandomness() function attempts to return nBytes bytes +** of good-quality randomness into zOut. The return value is +** the actual number of bytes of randomness obtained. +** The xSleep() method causes the calling thread to sleep for at +** least the number of microseconds given. ^The xCurrentTime() +** method returns a Julian Day Number for the current date and time as +** a floating point value. +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian +** Day Number multiplied by 86400000 (the number of milliseconds in +** a 24-hour day). +** ^SQLite will use the xCurrentTimeInt64() method to get the current +** date and time if that method is available (if iVersion is 2 or +** greater and the function pointer is not NULL) and will fall back +** to xCurrentTime() if xCurrentTimeInt64() is unavailable. +** +** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** are not used by the SQLite core. These optional interfaces are provided +** by some VFSes to facilitate testing of the VFS code. By overriding +** system calls with functions under its control, a test program can +** simulate faults and error conditions that would otherwise be difficult +** or impossible to induce. The set of system calls that can be overridden +** varies from one VFS to another, and from one version of the same VFS to the +** next. Applications that use these interfaces must be prepared for any +** or all of these interfaces to be NULL or for their behavior to change +** from one release to the next. Applications must not attempt to access +** any of these methods if the iVersion of the VFS is less than 3. +*/ +typedef struct sqlite3_vfs sqlite3_vfs; +typedef void (*sqlite3_syscall_ptr)(void); +struct sqlite3_vfs { + int iVersion; /* Structure version number (currently 3) */ + int szOsFile; /* Size of subclassed sqlite3_file */ + int mxPathname; /* Maximum file pathname length */ + sqlite3_vfs *pNext; /* Next registered VFS */ + const char *zName; /* Name of this virtual file system */ + void *pAppData; /* Pointer to application-specific data */ + int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int flags, int *pOutFlags); + int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); + int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); + int (*xFullPathname)(sqlite3_vfs*, const char *zName, int nOut, char *zOut); + void *(*xDlOpen)(sqlite3_vfs*, const char *zFilename); + void (*xDlError)(sqlite3_vfs*, int nByte, char *zErrMsg); + void (*(*xDlSym)(sqlite3_vfs*,void*, const char *zSymbol))(void); + void (*xDlClose)(sqlite3_vfs*, void*); + int (*xRandomness)(sqlite3_vfs*, int nByte, char *zOut); + int (*xSleep)(sqlite3_vfs*, int microseconds); + int (*xCurrentTime)(sqlite3_vfs*, double*); + int (*xGetLastError)(sqlite3_vfs*, int, char *); + /* + ** The methods above are in version 1 of the sqlite_vfs object + ** definition. Those that follow are added in version 2 or later + */ + int (*xCurrentTimeInt64)(sqlite3_vfs*, sqlite3_int64*); + /* + ** The methods above are in versions 1 and 2 of the sqlite_vfs object. + ** Those below are for version 3 and greater. + */ + int (*xSetSystemCall)(sqlite3_vfs*, const char *zName, sqlite3_syscall_ptr); + sqlite3_syscall_ptr (*xGetSystemCall)(sqlite3_vfs*, const char *zName); + const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName); + /* + ** The methods above are in versions 1 through 3 of the sqlite_vfs object. + ** New fields may be appended in figure versions. The iVersion + ** value will increment whenever this happens. + */ +}; + +/* +** CAPI3REF: Flags for the xAccess VFS method +** +** These integer constants can be used as the third parameter to +** the xAccess method of an [sqlite3_vfs] object. They determine +** what kind of permissions the xAccess method is looking for. +** With SQLITE_ACCESS_EXISTS, the xAccess method +** simply checks whether the file exists. +** With SQLITE_ACCESS_READWRITE, the xAccess method +** checks whether the named directory is both readable and writable +** (in other words, if files can be added, removed, and renamed within +** the directory). +** The SQLITE_ACCESS_READWRITE constant is currently used only by the +** [temp_store_directory pragma], though this could change in a future +** release of SQLite. +** With SQLITE_ACCESS_READ, the xAccess method +** checks whether the file is readable. The SQLITE_ACCESS_READ constant is +** currently unused, though it might be used in a future release of +** SQLite. +*/ +#define SQLITE_ACCESS_EXISTS 0 +#define SQLITE_ACCESS_READWRITE 1 /* Used by PRAGMA temp_store_directory */ +#define SQLITE_ACCESS_READ 2 /* Unused */ + +/* +** CAPI3REF: Flags for the xShmLock VFS method +** +** These integer constants define the various locking operations +** allowed by the xShmLock method of [sqlite3_io_methods]. The +** following are the only legal combinations of flags to the +** xShmLock method: +** +**
      +**
    • SQLITE_SHM_LOCK | SQLITE_SHM_SHARED +**
    • SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE +**
    • SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED +**
    • SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE +**
    +** +** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as +** was given no the corresponding lock. +** +** The xShmLock method can transition between unlocked and SHARED or +** between unlocked and EXCLUSIVE. It cannot transition between SHARED +** and EXCLUSIVE. +*/ +#define SQLITE_SHM_UNLOCK 1 +#define SQLITE_SHM_LOCK 2 +#define SQLITE_SHM_SHARED 4 +#define SQLITE_SHM_EXCLUSIVE 8 + +/* +** CAPI3REF: Maximum xShmLock index +** +** The xShmLock method on [sqlite3_io_methods] may use values +** between 0 and this upper bound as its "offset" argument. +** The SQLite core will never attempt to acquire or release a +** lock outside of this range +*/ +#define SQLITE_SHM_NLOCK 8 + + +/* +** CAPI3REF: Initialize The SQLite Library +** +** ^The sqlite3_initialize() routine initializes the +** SQLite library. ^The sqlite3_shutdown() routine +** deallocates any resources that were allocated by sqlite3_initialize(). +** These routines are designed to aid in process initialization and +** shutdown on embedded systems. Workstation applications using +** SQLite normally do not need to invoke either of these routines. +** +** A call to sqlite3_initialize() is an "effective" call if it is +** the first time sqlite3_initialize() is invoked during the lifetime of +** the process, or if it is the first time sqlite3_initialize() is invoked +** following a call to sqlite3_shutdown(). ^(Only an effective call +** of sqlite3_initialize() does any initialization. All other calls +** are harmless no-ops.)^ +** +** A call to sqlite3_shutdown() is an "effective" call if it is the first +** call to sqlite3_shutdown() since the last sqlite3_initialize(). ^(Only +** an effective call to sqlite3_shutdown() does any deinitialization. +** All other valid calls to sqlite3_shutdown() are harmless no-ops.)^ +** +** The sqlite3_initialize() interface is threadsafe, but sqlite3_shutdown() +** is not. The sqlite3_shutdown() interface must only be called from a +** single thread. All open [database connections] must be closed and all +** other SQLite resources must be deallocated prior to invoking +** sqlite3_shutdown(). +** +** Among other things, ^sqlite3_initialize() will invoke +** sqlite3_os_init(). Similarly, ^sqlite3_shutdown() +** will invoke sqlite3_os_end(). +** +** ^The sqlite3_initialize() routine returns [SQLITE_OK] on success. +** ^If for some reason, sqlite3_initialize() is unable to initialize +** the library (perhaps it is unable to allocate a needed resource such +** as a mutex) it returns an [error code] other than [SQLITE_OK]. +** +** ^The sqlite3_initialize() routine is called internally by many other +** SQLite interfaces so that an application usually does not need to +** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] +** calls sqlite3_initialize() so the SQLite library will be automatically +** initialized when [sqlite3_open()] is called if it has not be initialized +** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] +** compile-time option, then the automatic calls to sqlite3_initialize() +** are omitted and the application must call sqlite3_initialize() directly +** prior to using any other SQLite interface. For maximum portability, +** it is recommended that applications always invoke sqlite3_initialize() +** directly prior to using any other SQLite interface. Future releases +** of SQLite may require this. In other words, the behavior exhibited +** when SQLite is compiled with [SQLITE_OMIT_AUTOINIT] might become the +** default behavior in some future release of SQLite. +** +** The sqlite3_os_init() routine does operating-system specific +** initialization of the SQLite library. The sqlite3_os_end() +** routine undoes the effect of sqlite3_os_init(). Typical tasks +** performed by these routines include allocation or deallocation +** of static resources, initialization of global variables, +** setting up a default [sqlite3_vfs] module, or setting up +** a default configuration using [sqlite3_config()]. +** +** The application should never invoke either sqlite3_os_init() +** or sqlite3_os_end() directly. The application should only invoke +** sqlite3_initialize() and sqlite3_shutdown(). The sqlite3_os_init() +** interface is called automatically by sqlite3_initialize() and +** sqlite3_os_end() is called by sqlite3_shutdown(). Appropriate +** implementations for sqlite3_os_init() and sqlite3_os_end() +** are built into SQLite when it is compiled for Unix, Windows, or OS/2. +** When [custom builds | built for other platforms] +** (using the [SQLITE_OS_OTHER=1] compile-time +** option) the application must supply a suitable implementation for +** sqlite3_os_init() and sqlite3_os_end(). An application-supplied +** implementation of sqlite3_os_init() or sqlite3_os_end() +** must return [SQLITE_OK] on success and some other [error code] upon +** failure. +*/ +SQLITE_API int sqlite3_initialize(void); +SQLITE_API int sqlite3_shutdown(void); +SQLITE_API int sqlite3_os_init(void); +SQLITE_API int sqlite3_os_end(void); + +/* +** CAPI3REF: Configuring The SQLite Library +** +** The sqlite3_config() interface is used to make global configuration +** changes to SQLite in order to tune SQLite to the specific needs of +** the application. The default configuration is recommended for most +** applications and so this routine is usually not necessary. It is +** provided to support rare applications with unusual needs. +** +** The sqlite3_config() interface is not threadsafe. The application +** must insure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running. Furthermore, sqlite3_config() +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** +** The first argument to sqlite3_config() is an integer +** [configuration option] that determines +** what property of SQLite is to be configured. Subsequent arguments +** vary depending on the [configuration option] +** in the first argument. +** +** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. +** ^If the option is unknown or SQLite is unable to set the option +** then this routine returns a non-zero [error code]. +*/ +SQLITE_API int sqlite3_config(int, ...); + +/* +** CAPI3REF: Configure database connections +** +** The sqlite3_db_config() interface is used to make configuration +** changes to a [database connection]. The interface is similar to +** [sqlite3_config()] except that the changes apply to a single +** [database connection] (specified in the first argument). +** +** The second argument to sqlite3_db_config(D,V,...) is the +** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code +** that indicates what aspect of the [database connection] is being configured. +** Subsequent arguments vary depending on the configuration verb. +** +** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if +** the call is considered successful. +*/ +SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Memory Allocation Routines +** +** An instance of this object defines the interface between SQLite +** and low-level memory allocation routines. +** +** This object is used in only one place in the SQLite interface. +** A pointer to an instance of this object is the argument to +** [sqlite3_config()] when the configuration option is +** [SQLITE_CONFIG_MALLOC] or [SQLITE_CONFIG_GETMALLOC]. +** By creating an instance of this object +** and passing it to [sqlite3_config]([SQLITE_CONFIG_MALLOC]) +** during configuration, an application can specify an alternative +** memory allocation subsystem for SQLite to use for all of its +** dynamic memory needs. +** +** Note that SQLite comes with several [built-in memory allocators] +** that are perfectly adequate for the overwhelming majority of applications +** and that this object is only useful to a tiny minority of applications +** with specialized memory allocation requirements. This object is +** also used during testing of SQLite in order to specify an alternative +** memory allocator that simulates memory out-of-memory conditions in +** order to verify that SQLite recovers gracefully from such +** conditions. +** +** The xMalloc, xRealloc, and xFree methods must work like the +** malloc(), realloc() and free() functions from the standard C library. +** ^SQLite guarantees that the second argument to +** xRealloc is always a value returned by a prior call to xRoundup. +** +** xSize should return the allocated size of a memory allocation +** previously obtained from xMalloc or xRealloc. The allocated size +** is always at least as big as the requested size but may be larger. +** +** The xRoundup method returns what would be the allocated size of +** a memory allocation given a particular requested size. Most memory +** allocators round up memory allocations at least to the next multiple +** of 8. Some allocators round up to a larger multiple or to a power of 2. +** Every memory allocation request coming in through [sqlite3_malloc()] +** or [sqlite3_realloc()] first calls xRoundup. If xRoundup returns 0, +** that causes the corresponding memory allocation to fail. +** +** The xInit method initializes the memory allocator. For example, +** it might allocate any require mutexes or initialize internal data +** structures. The xShutdown method is invoked (indirectly) by +** [sqlite3_shutdown()] and should deallocate any resources acquired +** by xInit. The pAppData pointer is used as the only parameter to +** xInit and xShutdown. +** +** SQLite holds the [SQLITE_MUTEX_STATIC_MASTER] mutex when it invokes +** the xInit method, so the xInit method need not be threadsafe. The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. For all other methods, SQLite +** holds the [SQLITE_MUTEX_STATIC_MEM] mutex as long as the +** [SQLITE_CONFIG_MEMSTATUS] configuration option is turned on (which +** it is by default) and so the methods are automatically serialized. +** However, if [SQLITE_CONFIG_MEMSTATUS] is disabled, then the other +** methods must be threadsafe or else make their own arrangements for +** serialization. +** +** SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +*/ +typedef struct sqlite3_mem_methods sqlite3_mem_methods; +struct sqlite3_mem_methods { + void *(*xMalloc)(int); /* Memory allocation function */ + void (*xFree)(void*); /* Free a prior allocation */ + void *(*xRealloc)(void*,int); /* Resize an allocation */ + int (*xSize)(void*); /* Return the size of an allocation */ + int (*xRoundup)(int); /* Round up request size to allocation size */ + int (*xInit)(void*); /* Initialize the memory allocator */ + void (*xShutdown)(void*); /* Deinitialize the memory allocator */ + void *pAppData; /* Argument to xInit() and xShutdown() */ +}; + +/* +** CAPI3REF: Configuration Options +** KEYWORDS: {configuration option} +** +** These constants are the available integer configuration options that +** can be passed as the first argument to the [sqlite3_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_config()] to make sure that +** the call worked. The [sqlite3_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
    +** [[SQLITE_CONFIG_SINGLETHREAD]]
    SQLITE_CONFIG_SINGLETHREAD
    +**
    There are no arguments to this option. ^This option sets the +** [threading mode] to Single-thread. In other words, it disables +** all mutexing and puts SQLite into a mode where it can only be used +** by a single thread. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to change the [threading mode] from its default +** value of Single-thread and so [sqlite3_config()] will return +** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD +** configuration option.
    +** +** [[SQLITE_CONFIG_MULTITHREAD]]
    SQLITE_CONFIG_MULTITHREAD
    +**
    There are no arguments to this option. ^This option sets the +** [threading mode] to Multi-thread. In other words, it disables +** mutexing on [database connection] and [prepared statement] objects. +** The application is responsible for serializing access to +** [database connections] and [prepared statements]. But other mutexes +** are enabled so that SQLite will be safe to use in a multi-threaded +** environment as long as no two threads attempt to use the same +** [database connection] at the same time. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Multi-thread [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_MULTITHREAD configuration option.
    +** +** [[SQLITE_CONFIG_SERIALIZED]]
    SQLITE_CONFIG_SERIALIZED
    +**
    There are no arguments to this option. ^This option sets the +** [threading mode] to Serialized. In other words, this option enables +** all mutexes including the recursive +** mutexes on [database connection] and [prepared statement] objects. +** In this mode (which is the default when SQLite is compiled with +** [SQLITE_THREADSAFE=1]) the SQLite library will itself serialize access +** to [database connections] and [prepared statements] so that the +** application is free to use the same [database connection] or the +** same [prepared statement] in different threads at the same time. +** ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Serialized [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_SERIALIZED configuration option.
    +** +** [[SQLITE_CONFIG_MALLOC]]
    SQLITE_CONFIG_MALLOC
    +**
    ^(This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mem_methods] structure. The argument specifies +** alternative low-level memory allocation routines to be used in place of +** the memory allocation routines built into SQLite.)^ ^SQLite makes +** its own private copy of the content of the [sqlite3_mem_methods] structure +** before the [sqlite3_config()] call returns.
    +** +** [[SQLITE_CONFIG_GETMALLOC]]
    SQLITE_CONFIG_GETMALLOC
    +**
    ^(This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] +** structure is filled with the currently defined memory allocation routines.)^ +** This option can be used to overload the default memory allocation +** routines with a wrapper that simulations memory allocation failure or +** tracks memory usage, for example.
    +** +** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    +**
    ^This option takes single argument of type int, interpreted as a +** boolean, which enables or disables the collection of memory allocation +** statistics. ^(When memory allocation statistics are disabled, the +** following SQLite interfaces become non-operational: +**
      +**
    • [sqlite3_memory_used()] +**
    • [sqlite3_memory_highwater()] +**
    • [sqlite3_soft_heap_limit64()] +**
    • [sqlite3_status()] +**
    )^ +** ^Memory allocation statistics are enabled by default unless SQLite is +** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory +** allocation statistics are disabled by default. +**
    +** +** [[SQLITE_CONFIG_SCRATCH]]
    SQLITE_CONFIG_SCRATCH
    +**
    ^This option specifies a static memory buffer that SQLite can use for +** scratch memory. There are three arguments: A pointer an 8-byte +** aligned memory buffer from which the scratch allocations will be +** drawn, the size of each scratch allocation (sz), +** and the maximum number of scratch allocations (N). The sz +** argument must be a multiple of 16. +** The first argument must be a pointer to an 8-byte aligned buffer +** of at least sz*N bytes of memory. +** ^SQLite will use no more than two scratch buffers per thread. So +** N should be set to twice the expected maximum number of threads. +** ^SQLite will never require a scratch buffer that is more than 6 +** times the database page size. ^If SQLite needs needs additional +** scratch memory beyond what is provided by this configuration option, then +** [sqlite3_malloc()] will be used to obtain the memory needed.
    +** +** [[SQLITE_CONFIG_PAGECACHE]]
    SQLITE_CONFIG_PAGECACHE
    +**
    ^This option specifies a static memory buffer that SQLite can use for +** the database page cache with the default page cache implementation. +** This configuration should not be used if an application-define page +** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. +** There are three arguments to this option: A pointer to 8-byte aligned +** memory, the size of each page buffer (sz), and the number of pages (N). +** The sz argument should be the size of the largest database page +** (a power of two between 512 and 32768) plus a little extra for each +** page header. ^The page header size is 20 to 40 bytes depending on +** the host architecture. ^It is harmless, apart from the wasted memory, +** to make sz a little too large. The first +** argument should point to an allocation of at least sz*N bytes of memory. +** ^SQLite will use the memory provided by the first argument to satisfy its +** memory needs for the first N pages that it adds to cache. ^If additional +** page cache memory is needed beyond what is provided by this option, then +** SQLite goes to [sqlite3_malloc()] for the additional storage space. +** The pointer in the first argument must +** be aligned to an 8-byte boundary or subsequent behavior of SQLite +** will be undefined.
    +** +** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    +**
    ^This option specifies a static memory buffer that SQLite will use +** for all of its dynamic memory allocation needs beyond those provided +** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** There are three arguments: An 8-byte aligned pointer to the memory, +** the number of bytes in the memory buffer, and the minimum allocation size. +** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts +** to using its default memory allocator (the system malloc() implementation), +** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the +** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or +** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory +** allocator is engaged to handle all of SQLites memory allocation needs. +** The first pointer (the memory pointer) must be aligned to an 8-byte +** boundary or subsequent behavior of SQLite will be undefined. +** The minimum allocation size is capped at 2**12. Reasonable values +** for the minimum allocation size are 2**5 through 2**8.
    +** +** [[SQLITE_CONFIG_MUTEX]]
    SQLITE_CONFIG_MUTEX
    +**
    ^(This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mutex_methods] structure. The argument specifies +** alternative low-level mutex routines to be used in place +** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the +** content of the [sqlite3_mutex_methods] structure before the call to +** [sqlite3_config()] returns. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will +** return [SQLITE_ERROR].
    +** +** [[SQLITE_CONFIG_GETMUTEX]]
    SQLITE_CONFIG_GETMUTEX
    +**
    ^(This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mutex_methods] structure. The +** [sqlite3_mutex_methods] +** structure is filled with the currently defined mutex routines.)^ +** This option can be used to overload the default mutex allocation +** routines with a wrapper used to track mutex usage for performance +** profiling or testing, for example. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will +** return [SQLITE_ERROR].
    +** +** [[SQLITE_CONFIG_LOOKASIDE]]
    SQLITE_CONFIG_LOOKASIDE
    +**
    ^(This option takes two arguments that determine the default +** memory allocation for the lookaside memory allocator on each +** [database connection]. The first argument is the +** size of each lookaside buffer slot and the second is the number of +** slots allocated to each database connection.)^ ^(This option sets the +** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** verb to [sqlite3_db_config()] can be used to change the lookaside +** configuration on individual connections.)^
    +** +** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    +**
    ^(This option takes a single argument which is a pointer to +** an [sqlite3_pcache_methods2] object. This object specifies the interface +** to a custom page cache implementation.)^ ^SQLite makes a copy of the +** object and uses it for page cache memory allocations.
    +** +** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    +**
    ^(This option takes a single argument which is a pointer to an +** [sqlite3_pcache_methods2] object. SQLite copies of the current +** page cache implementation into that object.)^
    +** +** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    +**
    The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +** function with a call signature of void(*)(void*,int,const char*), +** and a pointer to void. ^If the function pointer is not NULL, it is +** invoked by [sqlite3_log()] to process each logging event. ^If the +** function pointer is NULL, the [sqlite3_log()] interface becomes a no-op. +** ^The void pointer that is the second argument to SQLITE_CONFIG_LOG is +** passed through as the first parameter to the application-defined logger +** function whenever that function is invoked. ^The second parameter to +** the logger function is a copy of the first parameter to the corresponding +** [sqlite3_log()] call and is intended to be a [result code] or an +** [extended result code]. ^The third parameter passed to the logger is +** log message after formatting via [sqlite3_snprintf()]. +** The SQLite logging interface is not reentrant; the logger function +** supplied by the application must not invoke any SQLite interface. +** In a multi-threaded application, the application-defined logger +** function must be threadsafe.
    +** +** [[SQLITE_CONFIG_URI]]
    SQLITE_CONFIG_URI +**
    ^(This option takes a single argument of type int. If non-zero, then +** URI handling is globally enabled. If the parameter is zero, then URI handling +** is globally disabled.)^ ^If URI handling is globally enabled, all filenames +** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +** specified as part of [ATTACH] commands are interpreted as URIs, regardless +** of whether or not the [SQLITE_OPEN_URI] flag is set when the database +** connection is opened. ^If it is globally disabled, filenames are +** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the +** database connection is opened. ^(By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** [SQLITE_USE_URI] symbol defined.)^ +** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN +**
    ^This option takes a single integer argument which is interpreted as +** a boolean in order to enable or disable the use of covering indices for +** full table scans in the query optimizer. ^The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** +** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] +**
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE +**
    These options are obsolete and should not be used by new code. +** They are retained for backwards compatibility but are now no-ops. +**
    +** +** [[SQLITE_CONFIG_SQLLOG]] +**
    SQLITE_CONFIG_SQLLOG +**
    This option is only available if sqlite is compiled with the +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
    +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
    SQLITE_CONFIG_MMAP_SIZE +**
    ^SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** ^The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ +** ^If either argument to this option is negative, then that argument is +** changed to its compile-time default. +** +** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] +**
    SQLITE_CONFIG_WIN32_HEAPSIZE +**
    ^This option is only available if SQLite is compiled for Windows +** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. +** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** that specifies the maximum size of the created heap. +**
    +*/ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* void*, int sz, int N */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ + +/* +** CAPI3REF: Database Connection Configuration Options +** +** These constants are the available integer configuration options that +** can be passed as the second argument to the [sqlite3_db_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_db_config()] to make sure that +** the call worked. ^The [sqlite3_db_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
    +**
    SQLITE_DBCONFIG_LOOKASIDE
    +**
    ^This option takes three additional arguments that determine the +** [lookaside memory allocator] configuration for the [database connection]. +** ^The first argument (the third parameter to [sqlite3_db_config()] is a +** pointer to a memory buffer to use for lookaside memory. +** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb +** may be NULL in which case SQLite will allocate the +** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the +** size of each lookaside buffer slot. ^The third argument is the number of +** slots. The size of the buffer in the first argument must be greater than +** or equal to the product of the second and third arguments. The buffer +** must be aligned to an 8-byte boundary. ^If the second argument to +** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^
    +** +**
    SQLITE_DBCONFIG_ENABLE_FKEY
    +**
    ^This option is used to enable or disable the enforcement of +** [foreign key constraints]. There should be two additional arguments. +** The first argument is an integer which is 0 to disable FK enforcement, +** positive to enable FK enforcement or negative to leave FK enforcement +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether FK enforcement is off or on +** following this call. The second parameter may be a NULL pointer, in +** which case the FK enforcement setting is not reported back.
    +** +**
    SQLITE_DBCONFIG_ENABLE_TRIGGER
    +**
    ^This option is used to enable or disable [CREATE TRIGGER | triggers]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable triggers, +** positive to enable triggers or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether triggers are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the trigger setting is not reported back.
    +** +**
    +*/ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ + + +/* +** CAPI3REF: Enable Or Disable Extended Result Codes +** +** ^The sqlite3_extended_result_codes() routine enables or disables the +** [extended result codes] feature of SQLite. ^The extended result +** codes are disabled by default for historical compatibility. +*/ +SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); + +/* +** CAPI3REF: Last Insert Rowid +** +** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables) +** has a unique 64-bit signed +** integer key called the [ROWID | "rowid"]. ^The rowid is always available +** as an undeclared column named ROWID, OID, or _ROWID_ as long as those +** names are not also used by explicitly declared columns. ^If +** the table has a column of type [INTEGER PRIMARY KEY] then that column +** is another alias for the rowid. +** +** ^The sqlite3_last_insert_rowid(D) interface returns the [rowid] of the +** most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. +** ^Inserts into [WITHOUT ROWID] tables are not recorded. +** ^If no successful [INSERT]s into rowid tables +** have ever occurred on the database connection D, +** then sqlite3_last_insert_rowid(D) returns zero. +** +** ^(If an [INSERT] occurs within a trigger or within a [virtual table] +** method, then this routine will return the [rowid] of the inserted +** row as long as the trigger or virtual table method is running. +** But once the trigger or virtual table method ends, the value returned +** by this routine reverts to what it was before the trigger or virtual +** table method began.)^ +** +** ^An [INSERT] that fails due to a constraint violation is not a +** successful [INSERT] and does not change the value returned by this +** routine. ^Thus INSERT OR FAIL, INSERT OR IGNORE, INSERT OR ROLLBACK, +** and INSERT OR ABORT make no changes to the return value of this +** routine when their insertion fails. ^(When INSERT OR REPLACE +** encounters a constraint violation, it does not fail. The +** INSERT continues to completion after deleting rows that caused +** the constraint problem so INSERT OR REPLACE will always change +** the return value of this interface.)^ +** +** ^For the purposes of this routine, an [INSERT] is considered to +** be successful even if it is subsequently rolled back. +** +** This function is accessible to SQL statements via the +** [last_insert_rowid() SQL function]. +** +** If a separate thread performs a new [INSERT] on the same +** database connection while the [sqlite3_last_insert_rowid()] +** function is running and thus changes the last insert [rowid], +** then the value returned by [sqlite3_last_insert_rowid()] is +** unpredictable and might not equal either the old or the new +** last insert [rowid]. +*/ +SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); + +/* +** CAPI3REF: Count The Number Of Rows Modified +** +** ^This function returns the number of database rows that were changed +** or inserted or deleted by the most recently completed SQL statement +** on the [database connection] specified by the first parameter. +** ^(Only changes that are directly specified by the [INSERT], [UPDATE], +** or [DELETE] statement are counted. Auxiliary changes caused by +** triggers or [foreign key actions] are not counted.)^ Use the +** [sqlite3_total_changes()] function to find the total number of changes +** including changes caused by triggers and foreign key actions. +** +** ^Changes to a view that are simulated by an [INSTEAD OF trigger] +** are not counted. Only real table changes are counted. +** +** ^(A "row change" is a change to a single row of a single table +** caused by an INSERT, DELETE, or UPDATE statement. Rows that +** are changed as side effects of [REPLACE] constraint resolution, +** rollback, ABORT processing, [DROP TABLE], or by any other +** mechanisms do not count as direct row changes.)^ +** +** A "trigger context" is a scope of execution that begins and +** ends with the script of a [CREATE TRIGGER | trigger]. +** Most SQL statements are +** evaluated outside of any trigger. This is the "top level" +** trigger context. If a trigger fires from the top level, a +** new trigger context is entered for the duration of that one +** trigger. Subtriggers create subcontexts for their duration. +** +** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does +** not create a new trigger context. +** +** ^This function returns the number of direct row changes in the +** most recent INSERT, UPDATE, or DELETE statement within the same +** trigger context. +** +** ^Thus, when called from the top level, this function returns the +** number of changes in the most recent INSERT, UPDATE, or DELETE +** that also occurred at the top level. ^(Within the body of a trigger, +** the sqlite3_changes() interface can be called to find the number of +** changes in the most recently completed INSERT, UPDATE, or DELETE +** statement within the body of the same trigger. +** However, the number returned does not include changes +** caused by subtriggers since those have their own context.)^ +** +** See also the [sqlite3_total_changes()] interface, the +** [count_changes pragma], and the [changes() SQL function]. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_changes()] is running then the value returned +** is unpredictable and not meaningful. +*/ +SQLITE_API int sqlite3_changes(sqlite3*); + +/* +** CAPI3REF: Total Number Of Rows Modified +** +** ^This function returns the number of row changes caused by [INSERT], +** [UPDATE] or [DELETE] statements since the [database connection] was opened. +** ^(The count returned by sqlite3_total_changes() includes all changes +** from all [CREATE TRIGGER | trigger] contexts and changes made by +** [foreign key actions]. However, +** the count does not include changes used to implement [REPLACE] constraints, +** do rollbacks or ABORT processing, or [DROP TABLE] processing. The +** count does not include rows of views that fire an [INSTEAD OF trigger], +** though if the INSTEAD OF trigger makes changes of its own, those changes +** are counted.)^ +** ^The sqlite3_total_changes() function counts the changes as soon as +** the statement that makes them is completed (when the statement handle +** is passed to [sqlite3_reset()] or [sqlite3_finalize()]). +** +** See also the [sqlite3_changes()] interface, the +** [count_changes pragma], and the [total_changes() SQL function]. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_total_changes()] is running then the value +** returned is unpredictable and not meaningful. +*/ +SQLITE_API int sqlite3_total_changes(sqlite3*); + +/* +** CAPI3REF: Interrupt A Long-Running Query +** +** ^This function causes any pending database operation to abort and +** return at its earliest opportunity. This routine is typically +** called in response to a user action such as pressing "Cancel" +** or Ctrl-C where the user wants a long query operation to halt +** immediately. +** +** ^It is safe to call this routine from a thread different from the +** thread that is currently running the database operation. But it +** is not safe to call this routine with a [database connection] that +** is closed or might close before sqlite3_interrupt() returns. +** +** ^If an SQL operation is very nearly finished at the time when +** sqlite3_interrupt() is called, then it might not have an opportunity +** to be interrupted and might continue to completion. +** +** ^An SQL operation that is interrupted will return [SQLITE_INTERRUPT]. +** ^If the interrupted SQL operation is an INSERT, UPDATE, or DELETE +** that is inside an explicit transaction, then the entire transaction +** will be rolled back automatically. +** +** ^The sqlite3_interrupt(D) call is in effect until all currently running +** SQL statements on [database connection] D complete. ^Any new SQL statements +** that are started after the sqlite3_interrupt() call and before the +** running statements reaches zero are interrupted as if they had been +** running prior to the sqlite3_interrupt() call. ^New SQL statements +** that are started after the running statement count reaches zero are +** not effected by the sqlite3_interrupt(). +** ^A call to sqlite3_interrupt(D) that occurs when there are no running +** SQL statements is a no-op and has no effect on SQL statements +** that are started after the sqlite3_interrupt() call returns. +** +** If the database connection closes while [sqlite3_interrupt()] +** is running then bad things will likely happen. +*/ +SQLITE_API void sqlite3_interrupt(sqlite3*); + +/* +** CAPI3REF: Determine If An SQL Statement Is Complete +** +** These routines are useful during command-line input to determine if the +** currently entered text seems to form a complete SQL statement or +** if additional input is needed before sending the text into +** SQLite for parsing. ^These routines return 1 if the input string +** appears to be a complete SQL statement. ^A statement is judged to be +** complete if it ends with a semicolon token and is not a prefix of a +** well-formed CREATE TRIGGER statement. ^Semicolons that are embedded within +** string literals or quoted identifier names or comments are not +** independent tokens (they are part of the token in which they are +** embedded) and thus do not count as a statement terminator. ^Whitespace +** and comments that follow the final semicolon are ignored. +** +** ^These routines return 0 if the statement is incomplete. ^If a +** memory allocation fails, then SQLITE_NOMEM is returned. +** +** ^These routines do not parse the SQL statements thus +** will not detect syntactically incorrect SQL. +** +** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior +** to invoking sqlite3_complete16() then sqlite3_initialize() is invoked +** automatically by sqlite3_complete16(). If that initialization fails, +** then the return value from sqlite3_complete16() will be non-zero +** regardless of whether or not the input SQL is complete.)^ +** +** The input to [sqlite3_complete()] must be a zero-terminated +** UTF-8 string. +** +** The input to [sqlite3_complete16()] must be a zero-terminated +** UTF-16 string in native byte order. +*/ +SQLITE_API int sqlite3_complete(const char *sql); +SQLITE_API int sqlite3_complete16(const void *sql); + +/* +** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. +** +** ^If the busy callback is NULL, then [SQLITE_BUSY] +** is returned immediately upon encountering the lock. ^If the busy callback +** is not NULL, then the callback might be invoked with two arguments. +** +** ^The first argument to the busy handler is a copy of the void* pointer which +** is the third argument to sqlite3_busy_handler(). ^The second argument to +** the busy handler callback is the number of times that the busy handler has +** been invoked for the same locking event. ^If the +** busy callback returns 0, then no additional attempts are made to +** access the database and [SQLITE_BUSY] is returned +** to the application. +** ^If the callback returns non-zero, then another attempt +** is made to access the database and the cycle repeats. +** +** The presence of a busy handler does not guarantee that it will be invoked +** when there is lock contention. ^If SQLite determines that invoking the busy +** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] +** to the application instead of invoking the +** busy handler. +** Consider a scenario where one process is holding a read lock that +** it is trying to promote to a reserved lock and +** a second process is holding a reserved lock that it is trying +** to promote to an exclusive lock. The first process cannot proceed +** because it is blocked by the second and the second process cannot +** proceed because it is blocked by the first. If both processes +** invoke the busy handlers, neither will make any progress. Therefore, +** SQLite returns [SQLITE_BUSY] for the first process, hoping that this +** will induce the first process to release its read lock and allow +** the second process to proceed. +** +** ^The default busy callback is NULL. +** +** ^(There can only be a single busy handler defined for each +** [database connection]. Setting a new busy handler clears any +** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. +** +** The busy callback should not take any actions which modify the +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions +** result in undefined behavior. +** +** A busy handler must not close the database connection +** or [prepared statement] that invoked the busy handler. +*/ +SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); + +/* +** CAPI3REF: Set A Busy Timeout +** +** ^This routine sets a [sqlite3_busy_handler | busy handler] that sleeps +** for a specified amount of time when a table is locked. ^The handler +** will sleep multiple times until at least "ms" milliseconds of sleeping +** have accumulated. ^After at least "ms" milliseconds of sleeping, +** the handler returns 0 which causes [sqlite3_step()] to return +** [SQLITE_BUSY]. +** +** ^Calling this routine with an argument less than or equal to zero +** turns off all busy handlers. +** +** ^(There can only be a single busy handler for a particular +** [database connection] any any given moment. If another busy handler +** was defined (using [sqlite3_busy_handler()]) prior to calling +** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] +*/ +SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); + +/* +** CAPI3REF: Convenience Routines For Running Queries +** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. +** +** Definition: A result table is memory data structure created by the +** [sqlite3_get_table()] interface. A result table records the +** complete query results from one or more queries. +** +** The table conceptually has a number of rows and columns. But +** these numbers are not part of the result table itself. These +** numbers are obtained separately. Let N be the number of rows +** and M be the number of columns. +** +** A result table is an array of pointers to zero-terminated UTF-8 strings. +** There are (N+1)*M elements in the array. The first M pointers point +** to zero-terminated strings that contain the names of the columns. +** The remaining entries all point to query results. NULL values result +** in NULL pointers. All other values are in their UTF-8 zero-terminated +** string representation as returned by [sqlite3_column_text()]. +** +** A result table might consist of one or more memory allocations. +** It is not safe to pass a result table directly to [sqlite3_free()]. +** A result table should be deallocated using [sqlite3_free_table()]. +** +** ^(As an example of the result table format, suppose a query result +** is as follows: +** +** 
    +**        Name        | Age
+**        -----------------------
+**        Alice       | 43
+**        Bob         | 28
+**        Cindy       | 21
+**
    +** +** There are two column (M==2) and three rows (N==3). Thus the +** result table has 8 entries. Suppose the result table is stored +** in an array names azResult. Then azResult holds this content: +** +** 
    +**        azResult[0] = "Name";
+**        azResult[1] = "Age";
+**        azResult[2] = "Alice";
+**        azResult[3] = "43";
+**        azResult[4] = "Bob";
+**        azResult[5] = "28";
+**        azResult[6] = "Cindy";
+**        azResult[7] = "21";
+**
    )^ +** +** ^The sqlite3_get_table() function evaluates one or more +** semicolon-separated SQL statements in the zero-terminated UTF-8 +** string of its 2nd parameter and returns a result table to the +** pointer given in its 3rd parameter. +** +** After the application has finished with the result from sqlite3_get_table(), +** it must pass the result table pointer to sqlite3_free_table() in order to +** release the memory that was malloced. Because of the way the +** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling +** function must not try to call [sqlite3_free()] directly. Only +** [sqlite3_free_table()] is able to release the memory properly and safely. +** +** The sqlite3_get_table() interface is implemented as a wrapper around +** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access +** to any internal data structures of SQLite. It uses only the public +** interface defined here. As a consequence, errors that occur in the +** wrapper layer outside of the internal [sqlite3_exec()] call are not +** reflected in subsequent calls to [sqlite3_errcode()] or +** [sqlite3_errmsg()]. +*/ +SQLITE_API int sqlite3_get_table( + sqlite3 *db, /* An open database */ + const char *zSql, /* SQL to be evaluated */ + char ***pazResult, /* Results of the query */ + int *pnRow, /* Number of result rows written here */ + int *pnColumn, /* Number of result columns written here */ + char **pzErrmsg /* Error msg written here */ +); +SQLITE_API void sqlite3_free_table(char **result); + +/* +** CAPI3REF: Formatted String Printing Functions +** +** These routines are work-alikes of the "printf()" family of functions +** from the standard C library. +** +** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their +** results into memory obtained from [sqlite3_malloc()]. +** The strings returned by these two routines should be +** released by [sqlite3_free()]. ^Both routines return a +** NULL pointer if [sqlite3_malloc()] is unable to allocate enough +** memory to hold the resulting string. +** +** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from +** the standard C library. The result is written into the +** buffer supplied as the second parameter whose size is given by +** the first parameter. Note that the order of the +** first two parameters is reversed from snprintf().)^ This is an +** historical accident that cannot be fixed without breaking +** backwards compatibility. ^(Note also that sqlite3_snprintf() +** returns a pointer to its buffer instead of the number of +** characters actually written into the buffer.)^ We admit that +** the number of characters written would be a more useful return +** value but we cannot change the implementation of sqlite3_snprintf() +** now without breaking compatibility. +** +** ^As long as the buffer size is greater than zero, sqlite3_snprintf() +** guarantees that the buffer is always zero-terminated. ^The first +** parameter "n" is the total size of the buffer, including space for +** the zero terminator. So the longest string that can be completely +** written will be n-1 characters. +** +** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf(). +** +** These routines all implement some additional formatting +** options that are useful for constructing SQL statements. +** All of the usual printf() formatting options apply. In addition, there +** is are "%q", "%Q", and "%z" options. +** +** ^(The %q option works like %s in that it substitutes a nul-terminated +** string from the argument list. But %q also doubles every '\'' character. +** %q is designed for use inside a string literal.)^ By doubling each '\'' +** character it escapes that character and allows it to be inserted into +** the string. +** +** For example, assume the string variable zText contains text as follows: +** +** 
    +**  char *zText = "It's a happy day!";
+**
    +** +** One can use this text in an SQL statement as follows: +** +** 
    +**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES('%q')", zText);
+**  sqlite3_exec(db, zSQL, 0, 0, 0);
+**  sqlite3_free(zSQL);
+**
    +** +** Because the %q format string is used, the '\'' character in zText +** is escaped and the SQL generated is as follows: +** +** 
    +**  INSERT INTO table1 VALUES('It''s a happy day!')
+**
    +** +** This is correct. Had we used %s instead of %q, the generated SQL +** would have looked like this: +** +** 
    +**  INSERT INTO table1 VALUES('It's a happy day!');
+**
    +** +** This second example is an SQL syntax error. As a general rule you should +** always use %q instead of %s when inserting text into a string literal. +** +** ^(The %Q option works like %q except it also adds single quotes around +** the outside of the total string. Additionally, if the parameter in the +** argument list is a NULL pointer, %Q substitutes the text "NULL" (without +** single quotes).)^ So, for example, one could say: +** +** 
    +**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
+**  sqlite3_exec(db, zSQL, 0, 0, 0);
+**  sqlite3_free(zSQL);
+**
    +** +** The code above will render a correct SQL statement in the zSQL +** variable even if the zText variable is a NULL pointer. +** +** ^(The "%z" formatting option works like "%s" but with the +** addition that after the string has been read and copied into +** the result, [sqlite3_free()] is called on the input string.)^ +*/ +SQLITE_API char *sqlite3_mprintf(const char*,...); +SQLITE_API char *sqlite3_vmprintf(const char*, va_list); +SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); + +/* +** CAPI3REF: Memory Allocation Subsystem +** +** The SQLite core uses these three routines for all of its own +** internal memory allocation needs. "Core" in the previous sentence +** does not include operating-system specific VFS implementation. The +** Windows VFS uses native malloc() and free() for some operations. +** +** ^The sqlite3_malloc() routine returns a pointer to a block +** of memory at least N bytes in length, where N is the parameter. +** ^If sqlite3_malloc() is unable to obtain sufficient free +** memory, it returns a NULL pointer. ^If the parameter N to +** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns +** a NULL pointer. +** +** ^Calling sqlite3_free() with a pointer previously returned +** by sqlite3_malloc() or sqlite3_realloc() releases that memory so +** that it might be reused. ^The sqlite3_free() routine is +** a no-op if is called with a NULL pointer. Passing a NULL pointer +** to sqlite3_free() is harmless. After being freed, memory +** should neither be read nor written. Even reading previously freed +** memory might result in a segmentation fault or other severe error. +** Memory corruption, a segmentation fault, or other severe error +** might result if sqlite3_free() is called with a non-NULL pointer that +** was not obtained from sqlite3_malloc() or sqlite3_realloc(). +** +** ^(The sqlite3_realloc() interface attempts to resize a +** prior memory allocation to be at least N bytes, where N is the +** second parameter. The memory allocation to be resized is the first +** parameter.)^ ^ If the first parameter to sqlite3_realloc() +** is a NULL pointer then its behavior is identical to calling +** sqlite3_malloc(N) where N is the second parameter to sqlite3_realloc(). +** ^If the second parameter to sqlite3_realloc() is zero or +** negative then the behavior is exactly the same as calling +** sqlite3_free(P) where P is the first parameter to sqlite3_realloc(). +** ^sqlite3_realloc() returns a pointer to a memory allocation +** of at least N bytes in size or NULL if sufficient memory is unavailable. +** ^If M is the size of the prior allocation, then min(N,M) bytes +** of the prior allocation are copied into the beginning of buffer returned +** by sqlite3_realloc() and the prior allocation is freed. +** ^If sqlite3_realloc() returns NULL, then the prior allocation +** is not freed. +** +** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. +** +** In SQLite version 3.5.0 and 3.5.1, it was possible to define +** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in +** implementation of these routines to be omitted. That capability +** is no longer provided. Only built-in memory allocators can be used. +** +** Prior to SQLite version 3.7.10, the Windows OS interface layer called +** the system malloc() and free() directly when converting +** filenames between the UTF-8 encoding used by SQLite +** and whatever filename encoding is used by the particular Windows +** installation. Memory allocation errors were detected, but +** they were reported back as [SQLITE_CANTOPEN] or +** [SQLITE_IOERR] rather than [SQLITE_NOMEM]. +** +** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()] +** must be either NULL or else pointers obtained from a prior +** invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have +** not yet been released. +** +** The application must not read or write any part of +** a block of memory after it has been released using +** [sqlite3_free()] or [sqlite3_realloc()]. +*/ +SQLITE_API void *sqlite3_malloc(int); +SQLITE_API void *sqlite3_realloc(void*, int); +SQLITE_API void sqlite3_free(void*); + +/* +** CAPI3REF: Memory Allocator Statistics +** +** SQLite provides these two interfaces for reporting on the status +** of the [sqlite3_malloc()], [sqlite3_free()], and [sqlite3_realloc()] +** routines, which form the built-in memory allocation subsystem. +** +** ^The [sqlite3_memory_used()] routine returns the number of bytes +** of memory currently outstanding (malloced but not freed). +** ^The [sqlite3_memory_highwater()] routine returns the maximum +** value of [sqlite3_memory_used()] since the high-water mark +** was last reset. ^The values returned by [sqlite3_memory_used()] and +** [sqlite3_memory_highwater()] include any overhead +** added by SQLite in its implementation of [sqlite3_malloc()], +** but not overhead added by the any underlying system library +** routines that [sqlite3_malloc()] may call. +** +** ^The memory high-water mark is reset to the current value of +** [sqlite3_memory_used()] if and only if the parameter to +** [sqlite3_memory_highwater()] is true. ^The value returned +** by [sqlite3_memory_highwater(1)] is the high-water mark +** prior to the reset. +*/ +SQLITE_API sqlite3_int64 sqlite3_memory_used(void); +SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); + +/* +** CAPI3REF: Pseudo-Random Number Generator +** +** SQLite contains a high-quality pseudo-random number generator (PRNG) used to +** select random [ROWID | ROWIDs] when inserting new records into a table that +** already uses the largest possible [ROWID]. The PRNG is also used for +** the build-in random() and randomblob() SQL functions. This interface allows +** applications to access the same PRNG for other purposes. +** +** ^A call to this routine stores N bytes of randomness into buffer P. +** ^If N is less than one, then P can be a NULL pointer. +** +** ^If this routine has not been previously called or if the previous +** call had N less than one, then the PRNG is seeded using randomness +** obtained from the xRandomness method of the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more then +** the pseudo-randomness is generated +** internally and without recourse to the [sqlite3_vfs] xRandomness +** method. +*/ +SQLITE_API void sqlite3_randomness(int N, void *P); + +/* +** CAPI3REF: Compile-Time Authorization Callbacks +** +** ^This routine registers an authorizer callback with a particular +** [database connection], supplied in the first argument. +** ^The authorizer callback is invoked as SQL statements are being compiled +** by [sqlite3_prepare()] or its variants [sqlite3_prepare_v2()], +** [sqlite3_prepare16()] and [sqlite3_prepare16_v2()]. ^At various +** points during the compilation process, as logic is being created +** to perform various actions, the authorizer callback is invoked to +** see if those actions are allowed. ^The authorizer callback should +** return [SQLITE_OK] to allow the action, [SQLITE_IGNORE] to disallow the +** specific action but allow the SQL statement to continue to be +** compiled, or [SQLITE_DENY] to cause the entire SQL statement to be +** rejected with an error. ^If the authorizer callback returns +** any value other than [SQLITE_IGNORE], [SQLITE_OK], or [SQLITE_DENY] +** then the [sqlite3_prepare_v2()] or equivalent call that triggered +** the authorizer will fail with an error message. +** +** When the callback returns [SQLITE_OK], that means the operation +** requested is ok. ^When the callback returns [SQLITE_DENY], the +** [sqlite3_prepare_v2()] or equivalent call that triggered the +** authorizer will fail with an error message explaining that +** access is denied. +** +** ^The first parameter to the authorizer callback is a copy of the third +** parameter to the sqlite3_set_authorizer() interface. ^The second parameter +** to the callback is an integer [SQLITE_COPY | action code] that specifies +** the particular action to be authorized. ^The third through sixth parameters +** to the callback are zero-terminated strings that contain additional +** details about the action to be authorized. +** +** ^If the action code is [SQLITE_READ] +** and the callback returns [SQLITE_IGNORE] then the +** [prepared statement] statement is constructed to substitute +** a NULL value in place of the table column that would have +** been read if [SQLITE_OK] had been returned. The [SQLITE_IGNORE] +** return can be used to deny an untrusted user access to individual +** columns of a table. +** ^If the action code is [SQLITE_DELETE] and the callback returns +** [SQLITE_IGNORE] then the [DELETE] operation proceeds but the +** [truncate optimization] is disabled and all rows are deleted individually. +** +** An authorizer is used when [sqlite3_prepare | preparing] +** SQL statements from an untrusted source, to ensure that the SQL statements +** do not try to access data they are not allowed to see, or that they do not +** try to execute malicious statements that damage the database. For +** example, an application may allow a user to enter arbitrary +** SQL queries for evaluation by a database. But the application does +** not want the user to be able to make arbitrary changes to the +** database. An authorizer could then be put in place while the +** user-entered SQL is being [sqlite3_prepare | prepared] that +** disallows everything except [SELECT] statements. +** +** Applications that need to process SQL from untrusted sources +** might also consider lowering resource limits using [sqlite3_limit()] +** and limiting database size using the [max_page_count] [PRAGMA] +** in addition to using an authorizer. +** +** ^(Only a single authorizer can be in place on a database connection +** at a time. Each call to sqlite3_set_authorizer overrides the +** previous call.)^ ^Disable the authorizer by installing a NULL callback. +** The authorizer is disabled by default. +** +** The authorizer callback must not do anything that will modify +** the database connection that invoked the authorizer callback. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^When [sqlite3_prepare_v2()] is used to prepare a statement, the +** statement might be re-prepared during [sqlite3_step()] due to a +** schema change. Hence, the application should ensure that the +** correct authorizer callback remains in place during the [sqlite3_step()]. +** +** ^Note that the authorizer callback is invoked only during +** [sqlite3_prepare()] or its variants. Authorization is not +** performed during statement evaluation in [sqlite3_step()], unless +** as stated in the previous paragraph, sqlite3_step() invokes +** sqlite3_prepare_v2() to reprepare a statement after a schema change. +*/ +SQLITE_API int sqlite3_set_authorizer( + sqlite3*, + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), + void *pUserData +); + +/* +** CAPI3REF: Authorizer Return Codes +** +** The [sqlite3_set_authorizer | authorizer callback function] must +** return either [SQLITE_OK] or one of these two constants in order +** to signal SQLite whether or not the action is permitted. See the +** [sqlite3_set_authorizer | authorizer documentation] for additional +** information. +** +** Note that SQLITE_IGNORE is also used as a [conflict resolution mode] +** returned from the [sqlite3_vtab_on_conflict()] interface. +*/ +#define SQLITE_DENY 1 /* Abort the SQL statement with an error */ +#define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ + +/* +** CAPI3REF: Authorizer Action Codes +** +** The [sqlite3_set_authorizer()] interface registers a callback function +** that is invoked to authorize certain SQL statement actions. The +** second parameter to the callback is an integer code that specifies +** what action is being authorized. These are the integer action codes that +** the authorizer callback may be passed. +** +** These action code values signify what kind of operation is to be +** authorized. The 3rd and 4th parameters to the authorization +** callback function will be parameters or NULL depending on which of these +** codes is used as the second parameter. ^(The 5th parameter to the +** authorizer callback is the name of the database ("main", "temp", +** etc.) if applicable.)^ ^The 6th parameter to the authorizer callback +** is the name of the inner-most trigger or view that is responsible for +** the access attempt or NULL if this access attempt is directly from +** top-level SQL code. +*/ +/******************************************* 3rd ************ 4th ***********/ +#define SQLITE_CREATE_INDEX 1 /* Index Name Table Name */ +#define SQLITE_CREATE_TABLE 2 /* Table Name NULL */ +#define SQLITE_CREATE_TEMP_INDEX 3 /* Index Name Table Name */ +#define SQLITE_CREATE_TEMP_TABLE 4 /* Table Name NULL */ +#define SQLITE_CREATE_TEMP_TRIGGER 5 /* Trigger Name Table Name */ +#define SQLITE_CREATE_TEMP_VIEW 6 /* View Name NULL */ +#define SQLITE_CREATE_TRIGGER 7 /* Trigger Name Table Name */ +#define SQLITE_CREATE_VIEW 8 /* View Name NULL */ +#define SQLITE_DELETE 9 /* Table Name NULL */ +#define SQLITE_DROP_INDEX 10 /* Index Name Table Name */ +#define SQLITE_DROP_TABLE 11 /* Table Name NULL */ +#define SQLITE_DROP_TEMP_INDEX 12 /* Index Name Table Name */ +#define SQLITE_DROP_TEMP_TABLE 13 /* Table Name NULL */ +#define SQLITE_DROP_TEMP_TRIGGER 14 /* Trigger Name Table Name */ +#define SQLITE_DROP_TEMP_VIEW 15 /* View Name NULL */ +#define SQLITE_DROP_TRIGGER 16 /* Trigger Name Table Name */ +#define SQLITE_DROP_VIEW 17 /* View Name NULL */ +#define SQLITE_INSERT 18 /* Table Name NULL */ +#define SQLITE_PRAGMA 19 /* Pragma Name 1st arg or NULL */ +#define SQLITE_READ 20 /* Table Name Column Name */ +#define SQLITE_SELECT 21 /* NULL NULL */ +#define SQLITE_TRANSACTION 22 /* Operation NULL */ +#define SQLITE_UPDATE 23 /* Table Name Column Name */ +#define SQLITE_ATTACH 24 /* Filename NULL */ +#define SQLITE_DETACH 25 /* Database Name NULL */ +#define SQLITE_ALTER_TABLE 26 /* Database Name Table Name */ +#define SQLITE_REINDEX 27 /* Index Name NULL */ +#define SQLITE_ANALYZE 28 /* Table Name NULL */ +#define SQLITE_CREATE_VTABLE 29 /* Table Name Module Name */ +#define SQLITE_DROP_VTABLE 30 /* Table Name Module Name */ +#define SQLITE_FUNCTION 31 /* NULL Function Name */ +#define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ +#define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ + +/* +** CAPI3REF: Tracing And Profiling Functions +** +** These routines register callback functions that can be used for +** tracing and profiling the execution of SQL statements. +** +** ^The callback function registered by sqlite3_trace() is invoked at +** various times when an SQL statement is being run by [sqlite3_step()]. +** ^The sqlite3_trace() callback is invoked with a UTF-8 rendering of the +** SQL statement text as the statement first begins executing. +** ^(Additional sqlite3_trace() callbacks might occur +** as each triggered subprogram is entered. The callbacks for triggers +** contain a UTF-8 SQL comment that identifies the trigger.)^ +** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** +** ^The callback function registered by sqlite3_profile() is invoked +** as each SQL statement finishes. ^The profile callback contains +** the original statement text and an estimate of wall-clock time +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. The +** sqlite3_profile() function is considered experimental and is +** subject to change in future versions of SQLite. +*/ +SQLITE_API void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); +SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, + void(*xProfile)(void*,const char*,sqlite3_uint64), void*); + +/* +** CAPI3REF: Query Progress Callbacks +** +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this +** interface is to keep a GUI updated during a large query. +** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the approximate number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. ^If N is less than one then the progress +** handler is disabled. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. +** +** ^If the progress callback returns non-zero, the operation is +** interrupted. This feature can be used to implement a +** "Cancel" button on a GUI progress dialog box. +** +** The progress handler callback must not do anything that will modify +** the database connection that invoked the progress handler. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +*/ +SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); + +/* +** CAPI3REF: Opening A New Database Connection +** +** ^These routines open an SQLite database file as specified by the +** filename argument. ^The filename argument is interpreted as UTF-8 for +** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte +** order for sqlite3_open16(). ^(A [database connection] handle is usually +** returned in *ppDb, even if an error occurs. The only exception is that +** if SQLite is unable to allocate memory to hold the [sqlite3] object, +** a NULL will be written into *ppDb instead of a pointer to the [sqlite3] +** object.)^ ^(If the database is opened (and/or created) successfully, then +** [SQLITE_OK] is returned. Otherwise an [error code] is returned.)^ ^The +** [sqlite3_errmsg()] or [sqlite3_errmsg16()] routines can be used to obtain +** an English language description of the error following a failure of any +** of the sqlite3_open() routines. +** +** ^The default encoding for the database will be UTF-8 if +** sqlite3_open() or sqlite3_open_v2() is called and +** UTF-16 in the native byte order if sqlite3_open16() is used. +** +** Whether or not an error occurs when it is opened, resources +** associated with the [database connection] handle should be released by +** passing it to [sqlite3_close()] when it is no longer required. +** +** The sqlite3_open_v2() interface works like sqlite3_open() +** except that it accepts two additional parameters for additional control +** over the new database connection. ^(The flags parameter to +** sqlite3_open_v2() can take one of +** the following three values, optionally combined with the +** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE], +** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^ +** +**
    +** ^(
    [SQLITE_OPEN_READONLY]
    +**
    The database is opened in read-only mode. If the database does not +** already exist, an error is returned.
    )^ +** +** ^(
    [SQLITE_OPEN_READWRITE]
    +**
    The database is opened for reading and writing if possible, or reading +** only if the file is write protected by the operating system. In either +** case the database must already exist, otherwise an error is returned.
    )^ +** +** ^(
    [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]
    +**
    The database is opened for reading and writing, and is created if +** it does not already exist. This is the behavior that is always used for +** sqlite3_open() and sqlite3_open16().
    )^ +**
    +** +** If the 3rd parameter to sqlite3_open_v2() is not one of the +** combinations shown above optionally combined with other +** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] +** then the behavior is undefined. +** +** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection +** opens in the multi-thread [threading mode] as long as the single-thread +** mode has not been set at compile-time or start-time. ^If the +** [SQLITE_OPEN_FULLMUTEX] flag is set then the database connection opens +** in the serialized [threading mode] unless single-thread was +** previously selected at compile-time or start-time. +** ^The [SQLITE_OPEN_SHAREDCACHE] flag causes the database connection to be +** eligible to use [shared cache mode], regardless of whether or not shared +** cache is enabled using [sqlite3_enable_shared_cache()]. ^The +** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not +** participate in [shared cache mode] even if it is enabled. +** +** ^The fourth parameter to sqlite3_open_v2() is the name of the +** [sqlite3_vfs] object that defines the operating system interface that +** the new database connection should use. ^If the fourth parameter is +** a NULL pointer then the default [sqlite3_vfs] object is used. +** +** ^If the filename is ":memory:", then a private, temporary in-memory database +** is created for the connection. ^This in-memory database will vanish when +** the database connection is closed. Future versions of SQLite might +** make use of additional special filenames that begin with the ":" character. +** It is recommended that when a database filename actually does begin with +** a ":" character you should prefix the filename with a pathname such as +** "./" to avoid ambiguity. +** +** ^If the filename is an empty string, then a private, temporary +** on-disk database will be created. ^This private database will be +** automatically deleted as soon as the database connection is closed. +** +** [[URI filenames in sqlite3_open()]]

    URI Filenames

    +** +** ^If [URI filename] interpretation is enabled, and the filename argument +** begins with "file:", then the filename is interpreted as a URI. ^URI +** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is +** set in the fourth argument to sqlite3_open_v2(), or if it has +** been enabled globally using the [SQLITE_CONFIG_URI] option with the +** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. +** As of SQLite version 3.7.7, URI filename interpretation is turned off +** by default, but future releases of SQLite might enable URI filename +** interpretation by default. See "[URI filenames]" for additional +** information. +** +** URI filenames are parsed according to RFC 3986. ^If the URI contains an +** authority, then it must be either an empty string or the string +** "localhost". ^If the authority is not an empty string or "localhost", an +** error is returned to the caller. ^The fragment component of a URI, if +** present, is ignored. +** +** ^SQLite uses the path component of the URI as the name of the disk file +** which contains the database. ^If the path begins with a '/' character, +** then it is interpreted as an absolute path. ^If the path does not begin +** with a '/' (meaning that the authority section is omitted from the URI) +** then the path is interpreted as a relative path. +** ^On windows, the first component of an absolute path +** is a drive specification (e.g. "C:"). +** +** [[core URI query parameters]] +** The query component of a URI may contain parameters that are interpreted +** either by SQLite itself, or by a [VFS | custom VFS implementation]. +** SQLite interprets the following three query parameters: +** +**
      +**
    • vfs: ^The "vfs" parameter may be used to specify the name of +** a VFS object that provides the operating system interface that should +** be used to access the database file on disk. ^If this option is set to +** an empty string the default VFS object is used. ^Specifying an unknown +** VFS is an error. ^If sqlite3_open_v2() is used and the vfs option is +** present, then the VFS specified by the option takes precedence over +** the value passed as the fourth parameter to sqlite3_open_v2(). +** +**
    • mode: ^(The mode parameter may be set to either "ro", "rw", +** "rwc", or "memory". Attempting to set it to any other value is +** an error)^. +** ^If "ro" is specified, then the database is opened for read-only +** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the +** third argument to sqlite3_open_v2(). ^If the mode option is set to +** "rw", then the database is opened for read-write (but not create) +** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had +** been set. ^Value "rwc" is equivalent to setting both +** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If the mode option is +** set to "memory" then a pure [in-memory database] that never reads +** or writes from disk is used. ^It is an error to specify a value for +** the mode parameter that is less restrictive than that specified by +** the flags passed in the third parameter to sqlite3_open_v2(). +** +**
    • cache: ^The cache parameter may be set to either "shared" or +** "private". ^Setting it to "shared" is equivalent to setting the +** SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to +** sqlite3_open_v2(). ^Setting the cache parameter to "private" is +** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. +** ^If sqlite3_open_v2() is used and the "cache" parameter is present in +** a URI filename, its value overrides any behavior requested by setting +** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
    • psow: ^The psow parameter may be "true" (or "on" or "yes" or +** "1") or "false" (or "off" or "no" or "0") to indicate that the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. ^The psow query +** parameter only works for the built-in unix and Windows VFSes. +** +**
    • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
    • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** +**
    +** +** ^Specifying an unknown parameter in the query component of a URI is not an +** error. Future versions of SQLite might understand additional query +** parameters. See "[query parameters with special meaning to SQLite]" for +** additional information. +** +** [[URI filename examples]]

    URI filename examples

    +** +**
    +**
    URI filenames Results +**
    file:data.db +** Open the file "data.db" in the current directory. +**
    file:/home/fred/data.db
    +** file:///home/fred/data.db
    +** file://localhost/home/fred/data.db
    		+** Open the database file "/home/fred/data.db". +**
    file://darkstar/home/fred/data.db +** An error. "darkstar" is not a recognized authority. +**
    +** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db +** Windows only: Open the file "data.db" on fred's desktop on drive +** C:. Note that the %20 escaping in this example is not strictly +** necessary - space characters can be used literally +** in URI filenames. +**
    file:data.db?mode=ro&cache=private +** Open file "data.db" in the current directory for read-only access. +** Regardless of whether or not shared-cache mode is enabled by +** default, use a private cache. +**
    file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. +**
    file:data.db?mode=readonly +** An error. "readonly" is not a valid option for the "mode" parameter. +**
    +** +** ^URI hexadecimal escape sequences (%HH) are supported within the path and +** query components of a URI. A hexadecimal escape sequence consists of a +** percent sign - "%" - followed by exactly two hexadecimal digits +** specifying an octet value. ^Before the path or query components of a +** URI filename are interpreted, they are encoded using UTF-8 and all +** hexadecimal escape sequences replaced by a single byte containing the +** corresponding octet. If this process generates an invalid UTF-8 encoding, +** the results are undefined. +** +** Note to Windows users: The encoding used for the filename argument +** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever +** codepage is currently defined. Filenames containing international +** characters must be converted to UTF-8 prior to passing them into +** sqlite3_open() or sqlite3_open_v2(). +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling sqlite3_open() or sqlite3_open_v2(). Otherwise, various +** features that require the use of temporary files may fail. +** +** See also: [sqlite3_temp_directory] +*/ +SQLITE_API int sqlite3_open( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ +); +SQLITE_API int sqlite3_open16( + const void *filename, /* Database filename (UTF-16) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ +); +SQLITE_API int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +); + +/* +** CAPI3REF: Obtain Values For URI Parameters +** +** These are utility routines, useful to VFS implementations, that check +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of that query parameter. +** +** If F is the database filename pointer passed into the xOpen() method of +** a VFS implementation when the flags parameter to xOpen() has one or +** more of the [SQLITE_OPEN_URI] or [SQLITE_OPEN_MAIN_DB] bits set and +** P is the name of the query parameter, then +** sqlite3_uri_parameter(F,P) returns the value of the P +** parameter if it exists or a NULL pointer if P does not appear as a +** query parameter on F. If P is a query parameter of F +** has no explicit value, then sqlite3_uri_parameter(F,P) returns +** a pointer to an empty string. +** +** The sqlite3_uri_boolean(F,P,B) routine assumes that P is a boolean +** parameter and returns true (1) or false (0) according to the value +** of P. The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the +** value of query parameter P is one of "yes", "true", or "on" in any +** case or if the value begins with a non-zero number. The +** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of +** query parameter P is one of "no", "false", or "off" in any case or +** if the value begins with a numeric zero. If P is not a query +** parameter on F or if the value of P is does not match any of the +** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). +** +** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a +** 64-bit signed integer and returns that integer, or D if P does not +** exist. If the value of P is something other than an integer, then +** zero is returned. +** +** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and +** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and +** is not a database file pathname pointer that SQLite passed into the xOpen +** VFS method, then the behavior of this routine is undefined and probably +** undesirable. +*/ +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); + + +/* +** CAPI3REF: Error Codes And Messages +** +** ^The sqlite3_errcode() interface returns the numeric [result code] or +** [extended result code] for the most recent failed sqlite3_* API call +** associated with a [database connection]. If a prior API call failed +** but the most recent API call succeeded, the return value from +** sqlite3_errcode() is undefined. ^The sqlite3_extended_errcode() +** interface is the same except that it always returns the +** [extended result code] even when extended result codes are +** disabled. +** +** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language +** text that describes the error, as either UTF-8 or UTF-16 respectively. +** ^(Memory to hold the error message string is managed internally. +** The application does not need to worry about freeing the result. +** However, the error string might be overwritten or deallocated by +** subsequent calls to other SQLite interface functions.)^ +** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** +** When the serialized [threading mode] is in use, it might be the +** case that a second error occurs on a separate thread in between +** the time of the first error and the call to these interfaces. +** When that happens, the second error will be reported since these +** interfaces always report the most recent result. To avoid +** this, each thread can obtain exclusive use of the [database connection] D +** by invoking [sqlite3_mutex_enter]([sqlite3_db_mutex](D)) before beginning +** to use D and invoking [sqlite3_mutex_leave]([sqlite3_db_mutex](D)) after +** all calls to the interfaces listed here are completed. +** +** If an interface fails with SQLITE_MISUSE, that means the interface +** was invoked incorrectly by the application. In that case, the +** error code and message may or may not be set. +*/ +SQLITE_API int sqlite3_errcode(sqlite3 *db); +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); +SQLITE_API const char *sqlite3_errmsg(sqlite3*); +SQLITE_API const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *sqlite3_errstr(int); + +/* +** CAPI3REF: SQL Statement Object +** KEYWORDS: {prepared statement} {prepared statements} +** +** An instance of this object represents a single SQL statement. +** This object is variously known as a "prepared statement" or a +** "compiled SQL statement" or simply as a "statement". +** +** The life of a statement object goes something like this: +** +**
      +**
    1. Create the object using [sqlite3_prepare_v2()] or a related +** function. +**
    2. Bind values to [host parameters] using the sqlite3_bind_*() +** interfaces. +**
    3. Run the SQL by calling [sqlite3_step()] one or more times. +**
    4. Reset the statement using [sqlite3_reset()] then go back +** to step 2. Do this zero or more times. +**
    5. Destroy the object using [sqlite3_finalize()]. +**
    +** +** Refer to documentation on individual methods above for additional +** information. +*/ +typedef struct sqlite3_stmt sqlite3_stmt; + +/* +** CAPI3REF: Run-time Limits +** +** ^(This interface allows the size of various constructs to be limited +** on a connection by connection basis. The first parameter is the +** [database connection] whose limit is to be set or queried. The +** second parameter is one of the [limit categories] that define a +** class of constructs to be size limited. The third parameter is the +** new limit for that construct.)^ +** +** ^If the new limit is a negative number, the limit is unchanged. +** ^(For each limit category SQLITE_LIMIT_NAME there is a +** [limits | hard upper bound] +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. +** (The "_LIMIT_" in the name is changed to "_MAX_".))^ +** ^Attempts to increase a limit above its hard upper bound are +** silently truncated to the hard upper bound. +** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** +** Run-time limits are intended for use in applications that manage +** both their own internal database and also databases that are controlled +** by untrusted external sources. An example application might be a +** web browser that has its own databases for storing history and +** separate databases controlled by JavaScript applications downloaded +** off the Internet. The internal databases can be given the +** large, default limits. Databases managed by external sources can +** be given much smaller limits designed to prevent a denial of service +** attack. Developers might also want to use the [sqlite3_set_authorizer()] +** interface to further control untrusted SQL. The size of the database +** created by an untrusted script can be contained using the +** [max_page_count] [PRAGMA]. +** +** New run-time limit categories may be added in future releases. +*/ +SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); + +/* +** CAPI3REF: Run-Time Limit Categories +** KEYWORDS: {limit category} {*limit categories} +** +** These constants define various performance limits +** that can be lowered at run-time using [sqlite3_limit()]. +** The synopsis of the meanings of the various limits is shown below. +** Additional information is available at [limits | Limits in SQLite]. +** +**
    +** [[SQLITE_LIMIT_LENGTH]] ^(
    SQLITE_LIMIT_LENGTH
    +**
    The maximum size of any string or BLOB or table row, in bytes.
    )^ +** +** [[SQLITE_LIMIT_SQL_LENGTH]] ^(
    SQLITE_LIMIT_SQL_LENGTH
    +**
    The maximum length of an SQL statement, in bytes.
    )^ +** +** [[SQLITE_LIMIT_COLUMN]] ^(
    SQLITE_LIMIT_COLUMN
    +**
    The maximum number of columns in a table definition or in the +** result set of a [SELECT] or the maximum number of columns in an index +** or in an ORDER BY or GROUP BY clause.
    )^ +** +** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(
    SQLITE_LIMIT_EXPR_DEPTH
    +**
    The maximum depth of the parse tree on any expression.
    )^ +** +** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(
    SQLITE_LIMIT_COMPOUND_SELECT
    +**
    The maximum number of terms in a compound SELECT statement.
    )^ +** +** [[SQLITE_LIMIT_VDBE_OP]] ^(
    SQLITE_LIMIT_VDBE_OP
    +**
    The maximum number of instructions in a virtual machine program +** used to implement an SQL statement. This limit is not currently +** enforced, though that might be added in some future release of +** SQLite.
    )^ +** +** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
    SQLITE_LIMIT_FUNCTION_ARG
    +**
    The maximum number of arguments on a function.
    )^ +** +** [[SQLITE_LIMIT_ATTACHED]] ^(
    SQLITE_LIMIT_ATTACHED
    +**
    The maximum number of [ATTACH | attached databases].)^
    +** +** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]] +** ^(
    SQLITE_LIMIT_LIKE_PATTERN_LENGTH
    +**
    The maximum length of the pattern argument to the [LIKE] or +** [GLOB] operators.
    )^ +** +** [[SQLITE_LIMIT_VARIABLE_NUMBER]] +** ^(
    SQLITE_LIMIT_VARIABLE_NUMBER
    +**
    The maximum index number of any [parameter] in an SQL statement.)^ +** +** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    +**
    The maximum depth of recursion for triggers.
    )^ +**
    +*/ +#define SQLITE_LIMIT_LENGTH 0 +#define SQLITE_LIMIT_SQL_LENGTH 1 +#define SQLITE_LIMIT_COLUMN 2 +#define SQLITE_LIMIT_EXPR_DEPTH 3 +#define SQLITE_LIMIT_COMPOUND_SELECT 4 +#define SQLITE_LIMIT_VDBE_OP 5 +#define SQLITE_LIMIT_FUNCTION_ARG 6 +#define SQLITE_LIMIT_ATTACHED 7 +#define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 +#define SQLITE_LIMIT_VARIABLE_NUMBER 9 +#define SQLITE_LIMIT_TRIGGER_DEPTH 10 + +/* +** CAPI3REF: Compiling An SQL Statement +** KEYWORDS: {SQL statement compiler} +** +** To execute an SQL query, it must first be compiled into a byte-code +** program using one of these routines. +** +** The first argument, "db", is a [database connection] obtained from a +** prior successful call to [sqlite3_open()], [sqlite3_open_v2()] or +** [sqlite3_open16()]. The database connection must not have been closed. +** +** The second argument, "zSql", is the statement to be compiled, encoded +** as either UTF-8 or UTF-16. The sqlite3_prepare() and sqlite3_prepare_v2() +** interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2() +** use UTF-16. +** +** ^If the nByte argument is less than zero, then zSql is read up to the +** first zero terminator. ^If nByte is non-negative, then it is the maximum +** number of bytes read from zSql. ^When nByte is non-negative, the +** zSql string ends at either the first '\000' or '\u0000' character or +** the nByte-th byte, whichever comes first. If the caller knows +** that the supplied string is nul-terminated, then there is a small +** performance advantage to be gained by passing an nByte parameter that +** is equal to the number of bytes in the input string including +** the nul-terminator bytes as this saves SQLite from having to +** make a copy of the input string. +** +** ^If pzTail is not NULL then *pzTail is made to point to the first byte +** past the end of the first SQL statement in zSql. These routines only +** compile the first statement in zSql, so *pzTail is left pointing to +** what remains uncompiled. +** +** ^*ppStmt is left pointing to a compiled [prepared statement] that can be +** executed using [sqlite3_step()]. ^If there is an error, *ppStmt is set +** to NULL. ^If the input text contains no SQL (if the input is an empty +** string or a comment) then *ppStmt is set to NULL. +** The calling procedure is responsible for deleting the compiled +** SQL statement using [sqlite3_finalize()] after it has finished with it. +** ppStmt may not be NULL. +** +** ^On success, the sqlite3_prepare() family of routines return [SQLITE_OK]; +** otherwise an [error code] is returned. +** +** The sqlite3_prepare_v2() and sqlite3_prepare16_v2() interfaces are +** recommended for all new programs. The two older interfaces are retained +** for backwards compatibility, but their use is discouraged. +** ^In the "v2" interfaces, the prepared statement +** that is returned (the [sqlite3_stmt] object) contains a copy of the +** original SQL text. This causes the [sqlite3_step()] interface to +** behave differently in three ways: +** +**
      +**
    1. +** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it +** always used to do, [sqlite3_step()] will automatically recompile the SQL +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. +**
      2. +** +**
    2. +** ^When an error occurs, [sqlite3_step()] will return one of the detailed +** [error codes] or [extended error codes]. ^The legacy behavior was that +** [sqlite3_step()] would only return a generic [SQLITE_ERROR] result code +** and the application would have to make a second call to [sqlite3_reset()] +** in order to find the underlying cause of the problem. With the "v2" prepare +** interfaces, the underlying reason for the error is returned immediately. +**
      3. +** +**
    3. +** ^If the specific value bound to [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. +**
      4. +**
    +*/ +SQLITE_API int sqlite3_prepare( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); + +/* +** CAPI3REF: Retrieving Statement SQL +** +** ^This interface can be used to retrieve a saved copy of the original +** SQL text used to create a [prepared statement] if that statement was +** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. +*/ +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Determine If An SQL Statement Writes The Database +** +** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if +** and only if the [prepared statement] X makes no direct changes to +** the content of the database file. +** +** Note that [application-defined SQL functions] or +** [virtual tables] might change the database indirectly as a side effect. +** ^(For example, if an application defines a function "eval()" that +** calls [sqlite3_exec()], then the following SQL statement would +** change the database file through side-effects: +** +** 
    +**    SELECT eval('DELETE FROM t1') FROM t2;
+**
    +** +** But because the [SELECT] statement does not change the database file +** directly, sqlite3_stmt_readonly() would still return true.)^ +** +** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK], +** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true, +** since the statements themselves do not actually modify the database but +** rather they control the timing of when other statements modify the +** database. ^The [ATTACH] and [DETACH] statements also cause +** sqlite3_stmt_readonly() to return true since, while those statements +** change the configuration of a database connection, they do not make +** changes to the content of the database files on disk. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** +** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the +** [prepared statement] S has been stepped at least once using +** [sqlite3_step(S)] but has not run to completion and/or has not +** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) +** interface returns false if S is a NULL pointer. If S is not a +** NULL pointer and is not a pointer to a valid [prepared statement] +** object, then the behavior is undefined and probably undesirable. +** +** This interface can be used in combination [sqlite3_next_stmt()] +** to locate all prepared statements associated with a database +** connection that are in need of being reset. This can be used, +** for example, in diagnostic routines to search for prepared +** statements that are holding a transaction open. +*/ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); + +/* +** CAPI3REF: Dynamically Typed Value Object +** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} +** +** SQLite uses the sqlite3_value object to represent all values +** that can be stored in a database table. SQLite uses dynamic typing +** for the values it stores. ^Values stored in sqlite3_value objects +** can be integers, floating point values, strings, BLOBs, or NULL. +** +** An sqlite3_value object may be either "protected" or "unprotected". +** Some interfaces require a protected sqlite3_value. Other interfaces +** will accept either a protected or an unprotected sqlite3_value. +** Every interface that accepts sqlite3_value arguments specifies +** whether or not it requires a protected sqlite3_value. +** +** The terms "protected" and "unprotected" refer to whether or not +** a mutex is held. An internal mutex is held for a protected +** sqlite3_value object but no mutex is held for an unprotected +** sqlite3_value object. If SQLite is compiled to be single-threaded +** (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0) +** or if SQLite is run in one of reduced mutex modes +** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD] +** then there is no distinction between protected and unprotected +** sqlite3_value objects and they can be used interchangeably. However, +** for maximum code portability it is recommended that applications +** still make the distinction between protected and unprotected +** sqlite3_value objects even when not strictly required. +** +** ^The sqlite3_value objects that are passed as parameters into the +** implementation of [application-defined SQL functions] are protected. +** ^The sqlite3_value object returned by +** [sqlite3_column_value()] is unprotected. +** Unprotected sqlite3_value objects may only be used with +** [sqlite3_result_value()] and [sqlite3_bind_value()]. +** The [sqlite3_value_blob | sqlite3_value_type()] family of +** interfaces require protected sqlite3_value objects. +*/ +typedef struct Mem sqlite3_value; + +/* +** CAPI3REF: SQL Function Context Object +** +** The context in which an SQL function executes is stored in an +** sqlite3_context object. ^A pointer to an sqlite3_context object +** is always first parameter to [application-defined SQL functions]. +** The application-defined SQL function implementation will pass this +** pointer through into calls to [sqlite3_result_int | sqlite3_result()], +** [sqlite3_aggregate_context()], [sqlite3_user_data()], +** [sqlite3_context_db_handle()], [sqlite3_get_auxdata()], +** and/or [sqlite3_set_auxdata()]. +*/ +typedef struct sqlite3_context sqlite3_context; + +/* +** CAPI3REF: Binding Values To Prepared Statements +** KEYWORDS: {host parameter} {host parameters} {host parameter name} +** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding} +** +** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, +** literals may be replaced by a [parameter] that matches one of following +** templates: +** +**
      +**
    • ? +**
    • ?NNN +**
    • :VVV +**
    • @VVV +**
    • $VVV +**
    +** +** In the templates above, NNN represents an integer literal, +** and VVV represents an alphanumeric identifier.)^ ^The values of these +** parameters (also called "host parameter names" or "SQL parameters") +** can be set using the sqlite3_bind_*() routines defined here. +** +** ^The first argument to the sqlite3_bind_*() routines is always +** a pointer to the [sqlite3_stmt] object returned from +** [sqlite3_prepare_v2()] or its variants. +** +** ^The second argument is the index of the SQL parameter to be set. +** ^The leftmost SQL parameter has an index of 1. ^When the same named +** SQL parameter is used more than once, second and subsequent +** occurrences have the same index as the first occurrence. +** ^The index for named parameters can be looked up using the +** [sqlite3_bind_parameter_index()] API if desired. ^The index +** for "?NNN" parameters is the value of NNN. +** ^The NNN value must be between 1 and the [sqlite3_limit()] +** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). +** +** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). +** +** ^(In those routines that have a fourth argument, its value is the +** number of bytes in the parameter. To be clear: the value is the +** number of bytes in the value, not the number of characters.)^ +** ^If the fourth parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** is negative, then the length of the string is +** the number of bytes up to the first zero terminator. +** If the fourth parameter to sqlite3_bind_blob() is negative, then +** the behavior is undefined. +** If a non-negative fourth parameter is provided to sqlite3_bind_text() +** or sqlite3_bind_text16() then that parameter must be the byte offset +** where the NUL terminator would occur assuming the string were NUL +** terminated. If any NUL characters occur at byte offsets less than +** the value of the fourth parameter then the resulting string value will +** contain embedded NULs. The result of expressions involving strings +** with embedded NULs is undefined. +** +** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and +** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or +** string after SQLite has finished with it. ^The destructor is called +** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), +** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** ^If the fifth argument is +** the special value [SQLITE_STATIC], then SQLite assumes that the +** information is in static, unmanaged space and does not need to be freed. +** ^If the fifth argument has the value [SQLITE_TRANSIENT], then +** SQLite makes its own private copy of the data immediately, before +** the sqlite3_bind_*() routine returns. +** +** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that +** is filled with zeroes. ^A zeroblob uses a fixed amount of memory +** (just an integer to hold its size) while it is being processed. +** Zeroblobs are intended to serve as placeholders for BLOBs whose +** content is later written using +** [sqlite3_blob_open | incremental BLOB I/O] routines. +** ^A negative value for the zeroblob results in a zero-length BLOB. +** +** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer +** for the [prepared statement] or with a prepared statement for which +** [sqlite3_step()] has been called more recently than [sqlite3_reset()], +** then the call will return [SQLITE_MISUSE]. If any sqlite3_bind_() +** routine is passed a [prepared statement] that has been finalized, the +** result is undefined and probably harmful. +** +** ^Bindings are not cleared by the [sqlite3_reset()] routine. +** ^Unbound parameters are interpreted as NULL. +** +** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an +** [error code] if anything goes wrong. +** ^[SQLITE_RANGE] is returned if the parameter +** index is out of range. ^[SQLITE_NOMEM] is returned if malloc() fails. +** +** See also: [sqlite3_bind_parameter_count()], +** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); +SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); +SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); +SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); +SQLITE_API int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); +SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); + +/* +** CAPI3REF: Number Of SQL Parameters +** +** ^This routine can be used to find the number of [SQL parameters] +** in a [prepared statement]. SQL parameters are tokens of the +** form "?", "?NNN", ":AAA", "$AAA", or "@AAA" that serve as +** placeholders for values that are [sqlite3_bind_blob | bound] +** to the parameters at a later time. +** +** ^(This routine actually returns the index of the largest (rightmost) +** parameter. For all forms except ?NNN, this will correspond to the +** number of unique parameters. If parameters of the ?NNN form are used, +** there may be gaps in the list.)^ +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_name()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); + +/* +** CAPI3REF: Name Of A Host Parameter +** +** ^The sqlite3_bind_parameter_name(P,N) interface returns +** the name of the N-th [SQL parameter] in the [prepared statement] P. +** ^(SQL parameters of the form "?NNN" or ":AAA" or "@AAA" or "$AAA" +** have a name which is the string "?NNN" or ":AAA" or "@AAA" or "$AAA" +** respectively. +** In other words, the initial ":" or "$" or "@" or "?" +** is included as part of the name.)^ +** ^Parameters of the form "?" without a following integer have no name +** and are referred to as "nameless" or "anonymous parameters". +** +** ^The first host parameter has an index of 1, not 0. +** +** ^If the value N is out of range or if the N-th parameter is +** nameless, then NULL is returned. ^The returned string is +** always in UTF-8 encoding even if the named parameter was +** originally specified as UTF-16 in [sqlite3_prepare16()] or +** [sqlite3_prepare16_v2()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); + +/* +** CAPI3REF: Index Of A Parameter With A Given Name +** +** ^Return the index of an SQL parameter given its name. ^The +** index value returned is suitable for use as the second +** parameter to [sqlite3_bind_blob|sqlite3_bind()]. ^A zero +** is returned if no matching parameter is found. ^The parameter +** name must be given in UTF-8 even if the original statement +** was prepared from UTF-16 text using [sqlite3_prepare16_v2()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); + +/* +** CAPI3REF: Reset All Bindings On A Prepared Statement +** +** ^Contrary to the intuition of many, [sqlite3_reset()] does not reset +** the [sqlite3_bind_blob | bindings] on a [prepared statement]. +** ^Use this routine to reset all host parameters to NULL. +*/ +SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); + +/* +** CAPI3REF: Number Of Columns In A Result Set +** +** ^Return the number of columns in the result set returned by the +** [prepared statement]. ^This routine returns 0 if pStmt is an SQL +** statement that does not return data (for example an [UPDATE]). +** +** See also: [sqlite3_data_count()] +*/ +SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Column Names In A Result Set +** +** ^These routines return the name assigned to a particular column +** in the result set of a [SELECT] statement. ^The sqlite3_column_name() +** interface returns a pointer to a zero-terminated UTF-8 string +** and sqlite3_column_name16() returns a pointer to a zero-terminated +** UTF-16 string. ^The first parameter is the [prepared statement] +** that implements the [SELECT] statement. ^The second parameter is the +** column number. ^The leftmost column is number 0. +** +** ^The returned string pointer is valid until either the [prepared statement] +** is destroyed by [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the next call to +** sqlite3_column_name() or sqlite3_column_name16() on the same column. +** +** ^If sqlite3_malloc() fails during the processing of either routine +** (for example during a conversion from UTF-8 to UTF-16) then a +** NULL pointer is returned. +** +** ^The name of a result column is the value of the "AS" clause for +** that column, if there is an AS clause. If there is no AS clause +** then the name of the column is unspecified and may change from +** one release of SQLite to the next. +*/ +SQLITE_API const char *sqlite3_column_name(sqlite3_stmt*, int N); +SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); + +/* +** CAPI3REF: Source Of Data In A Query Result +** +** ^These routines provide a means to determine the database, table, and +** table column that is the origin of a particular result column in +** [SELECT] statement. +** ^The name of the database or table or column can be returned as +** either a UTF-8 or UTF-16 string. ^The _database_ routines return +** the database name, the _table_ routines return the table name, and +** the origin_ routines return the column name. +** ^The returned string is valid until the [prepared statement] is destroyed +** using [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the same information is requested +** again in a different encoding. +** +** ^The names returned are the original un-aliased names of the +** database, table, and column. +** +** ^The first argument to these interfaces is a [prepared statement]. +** ^These functions return information about the Nth result column returned by +** the statement, where N is the second function argument. +** ^The left-most column is column 0 for these routines. +** +** ^If the Nth column returned by the statement is an expression or +** subquery and is not a column value, then all of these functions return +** NULL. ^These routine might also return NULL if a memory allocation error +** occurs. ^Otherwise, they return the name of the attached database, table, +** or column that query result column was extracted from. +** +** ^As with all other SQLite APIs, those whose names end with "16" return +** UTF-16 encoded strings and the other functions return UTF-8. +** +** ^These APIs are only available if the library was compiled with the +** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol. +** +** If two or more threads call one or more of these routines against the same +** prepared statement and column at the same time then the results are +** undefined. +** +** If two or more threads call one or more +** [sqlite3_column_database_name | column metadata interfaces] +** for the same [prepared statement] and result column +** at the same time then the results are undefined. +*/ +SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); + +/* +** CAPI3REF: Declared Datatype Of A Query Result +** +** ^(The first parameter is a [prepared statement]. +** If this statement is a [SELECT] statement and the Nth column of the +** returned result set of that [SELECT] is a table column (not an +** expression or subquery) then the declared type of the table +** column is returned.)^ ^If the Nth column of the result set is an +** expression or subquery, then a NULL pointer is returned. +** ^The returned string is always UTF-8 encoded. +** +** ^(For example, given the database schema: +** +** CREATE TABLE t1(c1 VARIANT); +** +** and the following statement to be compiled: +** +** SELECT c1 + 1, c1 FROM t1; +** +** this routine would return the string "VARIANT" for the second result +** column (i==1), and a NULL pointer for the first result column (i==0).)^ +** +** ^SQLite uses dynamic run-time typing. ^So just because a column +** is declared to contain a particular type does not mean that the +** data stored in that column is of the declared type. SQLite is +** strongly typed, but the typing is dynamic not static. ^Type +** is associated with individual values, not with the containers +** used to hold those values. +*/ +SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); + +/* +** CAPI3REF: Evaluate An SQL Statement +** +** After a [prepared statement] has been prepared using either +** [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or one of the legacy +** interfaces [sqlite3_prepare()] or [sqlite3_prepare16()], this function +** must be called one or more times to evaluate the statement. +** +** The details of the behavior of the sqlite3_step() interface depend +** on whether the statement was prepared using the newer "v2" interface +** [sqlite3_prepare_v2()] and [sqlite3_prepare16_v2()] or the older legacy +** interface [sqlite3_prepare()] and [sqlite3_prepare16()]. The use of the +** new "v2" interface is recommended for new applications but the legacy +** interface will continue to be supported. +** +** ^In the legacy interface, the return value will be either [SQLITE_BUSY], +** [SQLITE_DONE], [SQLITE_ROW], [SQLITE_ERROR], or [SQLITE_MISUSE]. +** ^With the "v2" interface, any of the other [result codes] or +** [extended result codes] might be returned as well. +** +** ^[SQLITE_BUSY] means that the database engine was unable to acquire the +** database locks it needs to do its job. ^If the statement is a [COMMIT] +** or occurs outside of an explicit transaction, then you can retry the +** statement. If the statement is not a [COMMIT] and occurs within an +** explicit transaction then you should rollback the transaction before +** continuing. +** +** ^[SQLITE_DONE] means that the statement has finished executing +** successfully. sqlite3_step() should not be called again on this virtual +** machine without first calling [sqlite3_reset()] to reset the virtual +** machine back to its initial state. +** +** ^If the SQL statement being executed returns any data, then [SQLITE_ROW] +** is returned each time a new row of data is ready for processing by the +** caller. The values may be accessed using the [column access functions]. +** sqlite3_step() is called again to retrieve the next row of data. +** +** ^[SQLITE_ERROR] means that a run-time error (such as a constraint +** violation) has occurred. sqlite3_step() should not be called again on +** the VM. More information may be found by calling [sqlite3_errmsg()]. +** ^With the legacy interface, a more specific error code (for example, +** [SQLITE_INTERRUPT], [SQLITE_SCHEMA], [SQLITE_CORRUPT], and so forth) +** can be obtained by calling [sqlite3_reset()] on the +** [prepared statement]. ^In the "v2" interface, +** the more specific error code is returned directly by sqlite3_step(). +** +** [SQLITE_MISUSE] means that the this routine was called inappropriately. +** Perhaps it was called on a [prepared statement] that has +** already been [sqlite3_finalize | finalized] or on one that had +** previously returned [SQLITE_ERROR] or [SQLITE_DONE]. Or it could +** be the case that the same database connection is being used by two or +** more threads at the same moment in time. +** +** For all versions of SQLite up to and including 3.6.23.1, a call to +** [sqlite3_reset()] was required after sqlite3_step() returned anything +** other than [SQLITE_ROW] before any subsequent invocation of +** sqlite3_step(). Failure to reset the prepared statement using +** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from +** sqlite3_step(). But after version 3.6.23.1, sqlite3_step() began +** calling [sqlite3_reset()] automatically in this circumstance rather +** than returning [SQLITE_MISUSE]. This is not considered a compatibility +** break because any application that ever receives an SQLITE_MISUSE error +** is broken by definition. The [SQLITE_OMIT_AUTORESET] compile-time option +** can be used to restore the legacy behavior. +** +** Goofy Interface Alert: In the legacy interface, the sqlite3_step() +** API always returns a generic error code, [SQLITE_ERROR], following any +** error other than [SQLITE_BUSY] and [SQLITE_MISUSE]. You must call +** [sqlite3_reset()] or [sqlite3_finalize()] in order to find one of the +** specific [error codes] that better describes the error. +** We admit that this is a goofy design. The problem has been fixed +** with the "v2" interface. If you prepare all of your SQL statements +** using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] instead +** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces, +** then the more specific [error codes] are returned directly +** by sqlite3_step(). The use of the "v2" interface is recommended. +*/ +SQLITE_API int sqlite3_step(sqlite3_stmt*); + +/* +** CAPI3REF: Number of columns in a result set +** +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. +** +** See also: [sqlite3_column_count()] +*/ +SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Fundamental Datatypes +** KEYWORDS: SQLITE_TEXT +** +** ^(Every value in SQLite has one of five fundamental datatypes: +** +**
      +**
    • 64-bit signed integer +**
    • 64-bit IEEE floating point number +**
    • string +**
    • BLOB +**
    • NULL +**
    )^ +** +** These constants are codes for each of those types. +** +** Note that the SQLITE_TEXT constant was also used in SQLite version 2 +** for a completely different meaning. Software that links against both +** SQLite version 2 and SQLite version 3 should use SQLITE3_TEXT, not +** SQLITE_TEXT. +*/ +#define SQLITE_INTEGER 1 +#define SQLITE_FLOAT 2 +#define SQLITE_BLOB 4 +#define SQLITE_NULL 5 +#ifdef SQLITE_TEXT +# undef SQLITE_TEXT +#else +# define SQLITE_TEXT 3 +#endif +#define SQLITE3_TEXT 3 + +/* +** CAPI3REF: Result Values From A Query +** KEYWORDS: {column access functions} +** +** These routines form the "result set" interface. +** +** ^These routines return information about a single column of the current +** result row of a query. ^In every case the first argument is a pointer +** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*] +** that was returned from [sqlite3_prepare_v2()] or one of its variants) +** and the second argument is the index of the column for which information +** should be returned. ^The leftmost column of the result set has the index 0. +** ^The number of columns in the result can be determined using +** [sqlite3_column_count()]. +** +** If the SQL statement does not currently point to a valid row, or if the +** column index is out of range, the result is undefined. +** These routines may only be called when the most recent call to +** [sqlite3_step()] has returned [SQLITE_ROW] and neither +** [sqlite3_reset()] nor [sqlite3_finalize()] have been called subsequently. +** If any of these routines are called after [sqlite3_reset()] or +** [sqlite3_finalize()] or after [sqlite3_step()] has returned +** something other than [SQLITE_ROW], the results are undefined. +** If [sqlite3_step()] or [sqlite3_reset()] or [sqlite3_finalize()] +** are called from a different thread while any of these routines +** are pending, then the results are undefined. +** +** ^The sqlite3_column_type() routine returns the +** [SQLITE_INTEGER | datatype code] for the initial data type +** of the result column. ^The returned value is one of [SQLITE_INTEGER], +** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL]. The value +** returned by sqlite3_column_type() is only meaningful if no type +** conversions have occurred as described below. After a type conversion, +** the value returned by sqlite3_column_type() is undefined. Future +** versions of SQLite may change the behavior of sqlite3_column_type() +** following a type conversion. +** +** ^If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-16 string, then sqlite3_column_bytes() converts +** the string to UTF-8 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes() uses +** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of +** bytes in the string, not the number of characters. +** +** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), +** even empty strings, are always zero-terminated. ^The return +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. +** +** ^The object returned by [sqlite3_column_value()] is an +** [unprotected sqlite3_value] object. An unprotected sqlite3_value object +** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()]. +** If the [unprotected sqlite3_value] object returned by +** [sqlite3_column_value()] is used in any other way, including calls +** to routines like [sqlite3_value_int()], [sqlite3_value_text()], +** or [sqlite3_value_bytes()], then the behavior is undefined. +** +** These routines attempt to convert the value where appropriate. ^For +** example, if the internal representation is FLOAT and a text result +** is requested, [sqlite3_snprintf()] is used internally to perform the +** conversion automatically. ^(The following table details the conversions +** that are applied: +** +**
    +** +**
    Internal
    Type     		Requested
    Type     		Conversion +** +**
    NULL INTEGER Result is 0 +**
    NULL FLOAT Result is 0.0 +**
    NULL TEXT Result is a NULL pointer +**
    NULL BLOB Result is a NULL pointer +**
    INTEGER FLOAT Convert from integer to float +**
    INTEGER TEXT ASCII rendering of the integer +**
    INTEGER BLOB Same as INTEGER->TEXT +**
    FLOAT INTEGER [CAST] to INTEGER +**
    FLOAT TEXT ASCII rendering of the float +**
    FLOAT BLOB [CAST] to BLOB +**
    TEXT INTEGER [CAST] to INTEGER +**
    TEXT FLOAT [CAST] to REAL +**
    TEXT BLOB No change +**
    BLOB INTEGER [CAST] to INTEGER +**
    BLOB FLOAT [CAST] to REAL +**
    BLOB TEXT Add a zero terminator if needed +**
    +**
    )^ +** +** The table above makes reference to standard C library functions atoi() +** and atof(). SQLite does not really use these functions. It has its +** own equivalent internal routines. The atoi() and atof() names are +** used in the table for brevity and because they are familiar to most +** C programmers. +** +** Note that when type conversions occur, pointers returned by prior +** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or +** sqlite3_column_text16() may be invalidated. +** Type conversions and pointer invalidations might occur +** in the following cases: +** +**
      +**
    • The initial content is a BLOB and sqlite3_column_text() or +** sqlite3_column_text16() is called. A zero-terminator might +** need to be added to the string.
      • +**
    • The initial content is UTF-8 text and sqlite3_column_bytes16() or +** sqlite3_column_text16() is called. The content must be converted +** to UTF-16.
      • +**
    • The initial content is UTF-16 text and sqlite3_column_bytes() or +** sqlite3_column_text() is called. The content must be converted +** to UTF-8.
      • +**
    +** +** ^Conversions between UTF-16be and UTF-16le are always done in place and do +** not invalidate a prior pointer, though of course the content of the buffer +** that the prior pointer references will have been modified. Other kinds +** of conversion are done in place when it is possible, but sometimes they +** are not possible and in those cases prior pointers are invalidated. +** +** The safest and easiest to remember policy is to invoke these routines +** in one of the following ways: +** +**
      +**
    • sqlite3_column_text() followed by sqlite3_column_bytes()
      • +**
    • sqlite3_column_blob() followed by sqlite3_column_bytes()
      • +**
    • sqlite3_column_text16() followed by sqlite3_column_bytes16()
      • +**
    +** +** In other words, you should call sqlite3_column_text(), +** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result +** into the desired format, then invoke sqlite3_column_bytes() or +** sqlite3_column_bytes16() to find the size of the result. Do not mix calls +** to sqlite3_column_text() or sqlite3_column_blob() with calls to +** sqlite3_column_bytes16(), and do not mix calls to sqlite3_column_text16() +** with calls to sqlite3_column_bytes(). +** +** ^The pointers returned are valid until a type conversion occurs as +** described above, or until [sqlite3_step()] or [sqlite3_reset()] or +** [sqlite3_finalize()] is called. ^The memory space used to hold strings +** and BLOBs is freed automatically. Do not pass the pointers returned +** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into +** [sqlite3_free()]. +** +** ^(If a memory allocation error occurs during the evaluation of any +** of these routines, a default value is returned. The default value +** is either the integer 0, the floating point number 0.0, or a NULL +** pointer. Subsequent calls to [sqlite3_errcode()] will return +** [SQLITE_NOMEM].)^ +*/ +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); +SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_int(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); + +/* +** CAPI3REF: Destroy A Prepared Statement Object +** +** ^The sqlite3_finalize() function is called to delete a [prepared statement]. +** ^If the most recent evaluation of the statement encountered no errors +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. +** +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. +*/ +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Reset A Prepared Statement Object +** +** The sqlite3_reset() function is called to reset a [prepared statement] +** object back to its initial state, ready to be re-executed. +** ^Any SQL statement variables that had values bound to them using +** the [sqlite3_bind_blob | sqlite3_bind_*() API] retain their values. +** Use [sqlite3_clear_bindings()] to reset the bindings. +** +** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S +** back to the beginning of its program. +** +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], +** or if [sqlite3_step(S)] has never before been called on S, +** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S indicated an error, then +** [sqlite3_reset(S)] returns an appropriate [error code]. +** +** ^The [sqlite3_reset(S)] interface does not change the values +** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. +*/ +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Create Or Redefine SQL Functions +** KEYWORDS: {function creation routines} +** KEYWORDS: {application-defined SQL function} +** KEYWORDS: {application-defined SQL functions} +** +** ^These functions (collectively known as "function creation routines") +** are used to add SQL functions or aggregates or to redefine the behavior +** of existing SQL functions or aggregates. The only differences between +** these routines are the text encoding expected for +** the second parameter (the name of the function being created) +** and the presence or absence of a destructor callback for +** the application data pointer. +** +** ^The first parameter is the [database connection] to which the SQL +** function is to be added. ^If an application uses more than one database +** connection then application-defined SQL functions must be added +** to each database connection separately. +** +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. +** +** ^The third parameter (nArg) +** is the number of arguments that the SQL function or +** aggregate takes. ^If this parameter is -1, then the SQL function or +** aggregate may take any number of arguments between 0 and the limit +** set by [sqlite3_limit]([SQLITE_LIMIT_FUNCTION_ARG]). If the third +** parameter is less than -1 or greater than 127 then the behavior is +** undefined. +** +** ^The fourth parameter, eTextRep, specifies what +** [SQLITE_UTF8 | text encoding] this SQL function prefers for +** its parameters. The application should set this parameter to +** [SQLITE_UTF16LE] if the function implementation invokes +** [sqlite3_value_text16le()] on an input, or [SQLITE_UTF16BE] if the +** implementation invokes [sqlite3_value_text16be()] on an input, or +** [SQLITE_UTF16] if [sqlite3_value_text16()] is used, or [SQLITE_UTF8] +** otherwise. ^The same SQL function may be registered multiple times using +** different preferred text encodings, with different implementations for +** each encoding. +** ^When multiple implementations of the same function are available, SQLite +** will pick the one that involves the least amount of data conversion. +** +** ^The fourth parameter may optionally be ORed with [SQLITE_DETERMINISTIC] +** to signal that the function will always return the same result given +** the same inputs within a single SQL statement. Most SQL functions are +** deterministic. The built-in [random()] SQL function is an example of a +** function that is not deterministic. The SQLite query planner is able to +** perform additional optimizations on deterministic functions, so use +** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. +** +** ^(The fifth parameter is an arbitrary pointer. The implementation of the +** function can gain access to this pointer using [sqlite3_user_data()].)^ +** +** ^The sixth, seventh and eighth parameters, xFunc, xStep and xFinal, are +** pointers to C-language functions that implement the SQL function or +** aggregate. ^A scalar SQL function requires an implementation of the xFunc +** callback only; NULL pointers must be passed as the xStep and xFinal +** parameters. ^An aggregate SQL function requires an implementation of xStep +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL pointers for all three function +** callbacks. +** +** ^(If the ninth parameter to sqlite3_create_function_v2() is not NULL, +** then it is destructor for the application data pointer. +** The destructor is invoked when the function is deleted, either by being +** overloaded or when the database connection closes.)^ +** ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. +** ^When the destructor callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the application data +** pointer which was the fifth parameter to sqlite3_create_function_v2(). +** +** ^It is permitted to register multiple implementations of the same +** functions with the same name but with either differing numbers of +** arguments or differing preferred text encodings. ^SQLite will use +** the implementation that most closely matches the way in which the +** SQL function is used. ^A function implementation with a non-negative +** nArg parameter is a better match than a function implementation with +** a negative nArg. ^A function where the preferred text encoding +** matches the database encoding is a better +** match than a function where the encoding is different. +** ^A function where the encoding difference is between UTF16le and UTF16be +** is a closer match than a function where the encoding difference is +** between UTF8 and UTF16. +** +** ^Built-in functions may be overloaded by new application-defined functions. +** +** ^An application-defined function is permitted to call other +** SQLite interfaces. However, such calls must not +** close the database connection nor finalize or reset the prepared +** statement in which the function is running. +*/ +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) +); + +/* +** CAPI3REF: Text Encodings +** +** These constant define integer codes that represent the various +** text encodings supported by SQLite. +*/ +#define SQLITE_UTF8 1 +#define SQLITE_UTF16LE 2 +#define SQLITE_UTF16BE 3 +#define SQLITE_UTF16 4 /* Use native byte order */ +#define SQLITE_ANY 5 /* Deprecated */ +#define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ + +/* +** CAPI3REF: Function Flags +** +** These constants may be ORed together with the +** [SQLITE_UTF8 | preferred text encoding] as the fourth argument +** to [sqlite3_create_function()], [sqlite3_create_function16()], or +** [sqlite3_create_function_v2()]. +*/ +#define SQLITE_DETERMINISTIC 0x800 + +/* +** CAPI3REF: Deprecated Functions +** DEPRECATED +** +** These functions are [deprecated]. In order to maintain +** backwards compatibility with older code, these functions continue +** to be supported. However, new applications should avoid +** the use of these functions. To help encourage people to avoid +** using these functions, we are not going to tell you what they do. +*/ +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); +SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); +#endif + +/* +** CAPI3REF: Obtaining SQL Function Parameter Values +** +** The C-language implementation of SQL functions and aggregates uses +** this set of interface routines to access the parameter values on +** the function or aggregate. +** +** The xFunc (for scalar functions) or xStep (for aggregates) parameters +** to [sqlite3_create_function()] and [sqlite3_create_function16()] +** define callbacks that implement the SQL functions and aggregates. +** The 3rd parameter to these callbacks is an array of pointers to +** [protected sqlite3_value] objects. There is one [sqlite3_value] object for +** each parameter to the SQL function. These routines are used to +** extract values from the [sqlite3_value] objects. +** +** These routines work only with [protected sqlite3_value] objects. +** Any attempt to use these routines on an [unprotected sqlite3_value] +** object results in undefined behavior. +** +** ^These routines work just like the corresponding [column access functions] +** except that these routines take a single [protected sqlite3_value] object +** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. +** +** ^The sqlite3_value_text16() interface extracts a UTF-16 string +** in the native byte-order of the host machine. ^The +** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces +** extract UTF-16 strings as big-endian and little-endian respectively. +** +** ^(The sqlite3_value_numeric_type() interface attempts to apply +** numeric affinity to the value. This means that an attempt is +** made to convert the value to an integer or floating point. If +** such a conversion is possible without loss of information (in other +** words, if the value is a string that looks like a number) +** then the conversion is performed. Otherwise no conversion occurs. +** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ +** +** Please pay particular attention to the fact that the pointer returned +** from [sqlite3_value_blob()], [sqlite3_value_text()], or +** [sqlite3_value_text16()] can be invalidated by a subsequent call to +** [sqlite3_value_bytes()], [sqlite3_value_bytes16()], [sqlite3_value_text()], +** or [sqlite3_value_text16()]. +** +** These routines must be called from the same thread as +** the SQL function that supplied the [sqlite3_value*] parameters. +*/ +SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); +SQLITE_API double sqlite3_value_double(sqlite3_value*); +SQLITE_API int sqlite3_value_int(sqlite3_value*); +SQLITE_API sqlite3_int64 sqlite3_value_int64(sqlite3_value*); +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); +SQLITE_API int sqlite3_value_type(sqlite3_value*); +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); + +/* +** CAPI3REF: Obtain Aggregate Function Context +** +** Implementations of aggregate SQL functions use this +** routine to allocate memory for storing their state. +** +** ^The first time the sqlite3_aggregate_context(C,N) routine is called +** for a particular aggregate function, SQLite +** allocates N of memory, zeroes out that memory, and returns a pointer +** to the new memory. ^On second and subsequent calls to +** sqlite3_aggregate_context() for the same aggregate function instance, +** the same buffer is returned. Sqlite3_aggregate_context() is normally +** called once for each invocation of the xStep callback and then one +** last time when the xFinal callback is invoked. ^(When no rows match +** an aggregate query, the xStep() callback of the aggregate function +** implementation is never called and xFinal() is called exactly once. +** In those cases, sqlite3_aggregate_context() might be called for the +** first time from within xFinal().)^ +** +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. +** +** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is +** determined by the N parameter on first successful call. Changing the +** value of N in subsequent call to sqlite3_aggregate_context() within +** the same aggregate function instance will not resize the memory +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. +** +** ^SQLite automatically frees the memory allocated by +** sqlite3_aggregate_context() when the aggregate query concludes. +** +** The first parameter must be a copy of the +** [sqlite3_context | SQL function context] that is the first parameter +** to the xStep or xFinal callback routine that implements the aggregate +** function. +** +** This routine must be called from the same thread in which +** the aggregate SQL function is running. +*/ +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); + +/* +** CAPI3REF: User Data For Functions +** +** ^The sqlite3_user_data() interface returns a copy of +** the pointer that was the pUserData parameter (the 5th parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +** +** This routine must be called from the same thread in which +** the application-defined function is running. +*/ +SQLITE_API void *sqlite3_user_data(sqlite3_context*); + +/* +** CAPI3REF: Database Connection For Functions +** +** ^The sqlite3_context_db_handle() interface returns a copy of +** the pointer to the [database connection] (the 1st parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +*/ +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); + +/* +** CAPI3REF: Function Auxiliary Data +** +** These functions may be used by (non-aggregate) SQL functions to +** associate metadata with argument values. If the same value is passed to +** multiple invocations of the same SQL function during query execution, under +** some circumstances the associated metadata may be preserved. An example +** of where this might be useful is in a regular-expression matching +** function. The compiled version of the regular expression can be stored as +** metadata associated with the pattern string. +** Then as long as the pattern string remains the same, +** the compiled regular expression can be reused on multiple +** invocations of the same function. +** +** ^The sqlite3_get_auxdata() interface returns a pointer to the metadata +** associated by the sqlite3_set_auxdata() function with the Nth argument +** value to the application-defined function. ^If there is no metadata +** associated with the function argument, this sqlite3_get_auxdata() interface +** returns a NULL pointer. +** +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th +** argument of the application-defined function. ^Subsequent +** calls to sqlite3_get_auxdata(C,N) return P from the most recent +** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or +** NULL if the metadata has been discarded. +** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, +** SQLite will invoke the destructor function X with parameter P exactly +** once, when the metadata is discarded. +** SQLite is free to discard the metadata at any time, including:
      +**
    • when the corresponding function parameter changes, or +**
    • when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement, or +**
    • when sqlite3_set_auxdata() is invoked again on the same parameter, or +**
    • during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.
    )^ +** +** Note the last bullet in particular. The destructor X in +** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the +** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() +** should be called near the end of the function implementation and the +** function implementation should not make any use of P after +** sqlite3_set_auxdata() has been called. +** +** ^(In practice, metadata is preserved between function calls for +** function parameters that are compile-time constants, including literal +** values and [parameters] and expressions composed from the same.)^ +** +** These routines must be called from the same thread in which +** the SQL function is running. +*/ +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); +SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); + + +/* +** CAPI3REF: Constants Defining Special Destructor Behavior +** +** These are special values for the destructor that is passed in as the +** final argument to routines like [sqlite3_result_blob()]. ^If the destructor +** argument is SQLITE_STATIC, it means that the content pointer is constant +** and will never change. It does not need to be destroyed. ^The +** SQLITE_TRANSIENT value means that the content will likely change in +** the near future and that SQLite should make its own private copy of +** the content before returning. +** +** The typedef is necessary to work around problems in certain +** C++ compilers. +*/ +typedef void (*sqlite3_destructor_type)(void*); +#define SQLITE_STATIC ((sqlite3_destructor_type)0) +#define SQLITE_TRANSIENT ((sqlite3_destructor_type)-1) + +/* +** CAPI3REF: Setting The Result Of An SQL Function +** +** These routines are used by the xFunc or xFinal callbacks that +** implement SQL functions and aggregates. See +** [sqlite3_create_function()] and [sqlite3_create_function16()] +** for additional information. +** +** These functions work very much like the [parameter binding] family of +** functions used to bind values to host parameters in prepared statements. +** Refer to the [SQL parameter] documentation for additional information. +** +** ^The sqlite3_result_blob() interface sets the result from +** an application-defined function to be the BLOB whose content is pointed +** to by the second parameter and which is N bytes long where N is the +** third parameter. +** +** ^The sqlite3_result_zeroblob() interfaces set the result of +** the application-defined function to be a BLOB containing all zero +** bytes and N bytes in size, where N is the value of the 2nd parameter. +** +** ^The sqlite3_result_double() interface sets the result from +** an application-defined function to be a floating point value specified +** by its 2nd argument. +** +** ^The sqlite3_result_error() and sqlite3_result_error16() functions +** cause the implemented SQL function to throw an exception. +** ^SQLite uses the string pointed to by the +** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() +** as the text of an error message. ^SQLite interprets the error +** message string from sqlite3_result_error() as UTF-8. ^SQLite +** interprets the string from sqlite3_result_error16() as UTF-16 in native +** byte order. ^If the third parameter to sqlite3_result_error() +** or sqlite3_result_error16() is negative then SQLite takes as the error +** message all text up through the first zero character. +** ^If the third parameter to sqlite3_result_error() or +** sqlite3_result_error16() is non-negative then SQLite takes that many +** bytes (not characters) from the 2nd parameter as the error message. +** ^The sqlite3_result_error() and sqlite3_result_error16() +** routines make a private copy of the error message text before +** they return. Hence, the calling function can deallocate or +** modify the text after they return without harm. +** ^The sqlite3_result_error_code() function changes the error code +** returned by SQLite as a result of an error in a function. ^By default, +** the error code is SQLITE_ERROR. ^A subsequent call to sqlite3_result_error() +** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. +** +** ^The sqlite3_result_error_toobig() interface causes SQLite to throw an +** error indicating that a string or BLOB is too long to represent. +** +** ^The sqlite3_result_error_nomem() interface causes SQLite to throw an +** error indicating that a memory allocation failed. +** +** ^The sqlite3_result_int() interface sets the return value +** of the application-defined function to be the 32-bit signed integer +** value given in the 2nd argument. +** ^The sqlite3_result_int64() interface sets the return value +** of the application-defined function to be the 64-bit signed integer +** value given in the 2nd argument. +** +** ^The sqlite3_result_null() interface sets the return value +** of the application-defined function to be NULL. +** +** ^The sqlite3_result_text(), sqlite3_result_text16(), +** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces +** set the return value of the application-defined function to be +** a text string which is represented as UTF-8, UTF-16 native byte order, +** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^SQLite takes the text result from the application from +** the 2nd parameter of the sqlite3_result_text* interfaces. +** ^If the 3rd parameter to the sqlite3_result_text* interfaces +** is negative, then SQLite takes result text from the 2nd parameter +** through the first zero character. +** ^If the 3rd parameter to the sqlite3_result_text* interfaces +** is non-negative, then as many bytes (not characters) of the text +** pointed to by the 2nd parameter are taken as the application-defined +** function result. If the 3rd parameter is non-negative, then it +** must be the byte offset into the string where the NUL terminator would +** appear if the string where NUL terminated. If any NUL characters occur +** in the string at a byte offset that is less than the value of the 3rd +** parameter, then the resulting string will contain embedded NULs and the +** result of expressions operating on strings with embedded NULs is undefined. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that +** function as the destructor on the text or BLOB result when it has +** finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces or to +** sqlite3_result_blob is the special constant SQLITE_STATIC, then SQLite +** assumes that the text or BLOB result is in constant space and does not +** copy the content of the parameter nor call a destructor on the content +** when it has finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is the special constant SQLITE_TRANSIENT +** then SQLite makes a copy of the result into space obtained from +** from [sqlite3_malloc()] before it returns. +** +** ^The sqlite3_result_value() interface sets the result of +** the application-defined function to be a copy the +** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The +** sqlite3_result_value() interface makes a copy of the [sqlite3_value] +** so that the [sqlite3_value] specified in the parameter may change or +** be deallocated after sqlite3_result_value() returns without harm. +** ^A [protected sqlite3_value] object may always be used where an +** [unprotected sqlite3_value] object is required, so either +** kind of [sqlite3_value] object can be used with this interface. +** +** If these routines are called from within the different thread +** than the one containing the application-defined function that received +** the [sqlite3_context] pointer, the results are undefined. +*/ +SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_double(sqlite3_context*, double); +SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); +SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context*); +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context*); +SQLITE_API void sqlite3_result_error_code(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); +SQLITE_API void sqlite3_result_null(sqlite3_context*); +SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); + +/* +** CAPI3REF: Define New Collating Sequences +** +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. +** +** ^The name of the collation is a UTF-8 string +** for sqlite3_create_collation() and sqlite3_create_collation_v2() +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. +** +** ^(The third argument (eTextRep) must be one of the constants: +**
      +**
    • [SQLITE_UTF8], +**
    • [SQLITE_UTF16LE], +**
    • [SQLITE_UTF16BE], +**
    • [SQLITE_UTF16], or +**
    • [SQLITE_UTF16_ALIGNED]. +**
    )^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCallback. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. +** +** ^The fourth argument, pArg, is an application data pointer that is passed +** through as the first argument to the collating function callback. +** +** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCallback argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The collating function must return an +** integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must always return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
      +**
    1. If A==B then B==A. +**
    2. If A==B and B==C then A==C. +**
    3. If A<B THEN B>A. +**
    4. If A<B and B<C then A<C. +**
    +** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. +** +** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. +** +** ^The xDestroy callback is not called if the +** sqlite3_create_collation_v2() function fails. Applications that invoke +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** check the return code and dispose of the application data pointer +** themselves rather than expecting SQLite to deal with it for them. +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards +** compatibility. +** +** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. +*/ +SQLITE_API int sqlite3_create_collation( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*) +); +SQLITE_API int sqlite3_create_collation_v2( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDestroy)(void*) +); +SQLITE_API int sqlite3_create_collation16( + sqlite3*, + const void *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*) +); + +/* +** CAPI3REF: Collation Needed Callbacks +** +** ^To avoid having to register all collation sequences before a database +** can be used, a single callback function may be registered with the +** [database connection] to be invoked whenever an undefined collation +** sequence is required. +** +** ^If the function is registered using the sqlite3_collation_needed() API, +** then it is passed the names of undefined collation sequences as strings +** encoded in UTF-8. ^If sqlite3_collation_needed16() is used, +** the names are passed as UTF-16 in machine native byte order. +** ^A call to either function replaces the existing collation-needed callback. +** +** ^(When the callback is invoked, the first argument passed is a copy +** of the second argument to sqlite3_collation_needed() or +** sqlite3_collation_needed16(). The second argument is the database +** connection. The third argument is one of [SQLITE_UTF8], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE], indicating the most desirable form of the collation +** sequence function required. The fourth parameter is the name of the +** required collation sequence.)^ +** +** The callback function should register the desired collation using +** [sqlite3_create_collation()], [sqlite3_create_collation16()], or +** [sqlite3_create_collation_v2()]. +*/ +SQLITE_API int sqlite3_collation_needed( + sqlite3*, + void*, + void(*)(void*,sqlite3*,int eTextRep,const char*) +); +SQLITE_API int sqlite3_collation_needed16( + sqlite3*, + void*, + void(*)(void*,sqlite3*,int eTextRep,const void*) +); + +#ifdef SQLITE_HAS_CODEC +/* +** Specify the key for an encrypted database. This routine should be +** called right after sqlite3_open(). +** +** The code to implement this API is not available in the public release +** of SQLite. +*/ +SQLITE_API int sqlite3_key( + sqlite3 *db, /* Database to be rekeyed */ + const void *pKey, int nKey /* The key */ +); +SQLITE_API int sqlite3_key_v2( + sqlite3 *db, /* Database to be rekeyed */ + const char *zDbName, /* Name of the database */ + const void *pKey, int nKey /* The key */ +); + +/* +** Change the key on an open database. If the current database is not +** encrypted, this routine will encrypt it. If pNew==0 or nNew==0, the +** database is decrypted. +** +** The code to implement this API is not available in the public release +** of SQLite. +*/ +SQLITE_API int sqlite3_rekey( + sqlite3 *db, /* Database to be rekeyed */ + const void *pKey, int nKey /* The new key */ +); +SQLITE_API int sqlite3_rekey_v2( + sqlite3 *db, /* Database to be rekeyed */ + const char *zDbName, /* Name of the database */ + const void *pKey, int nKey /* The new key */ +); + +/* +** Specify the activation key for a SEE database. Unless +** activated, none of the SEE routines will work. +*/ +SQLITE_API void sqlite3_activate_see( + const char *zPassPhrase /* Activation phrase */ +); +#endif + +#ifdef SQLITE_ENABLE_CEROD +/* +** Specify the activation key for a CEROD database. Unless +** activated, none of the CEROD routines will work. +*/ +SQLITE_API void sqlite3_activate_cerod( + const char *zPassPhrase /* Activation phrase */ +); +#endif + +/* +** CAPI3REF: Suspend Execution For A Short Time +** +** The sqlite3_sleep() function causes the current thread to suspend execution +** for at least a number of milliseconds specified in its parameter. +** +** If the operating system does not support sleep requests with +** millisecond time resolution, then the time will be rounded up to +** the nearest second. The number of milliseconds of sleep actually +** requested from the operating system is returned. +** +** ^SQLite implements this interface by calling the xSleep() +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. +*/ +SQLITE_API int sqlite3_sleep(int); + +/* +** CAPI3REF: Name Of The Folder Holding Temporary Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all temporary files +** created by SQLite when using a built-in [sqlite3_vfs | VFS] +** will be placed in that directory.)^ ^If this variable +** is a NULL pointer, then SQLite performs a search for an appropriate +** temporary file directory. +** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [temp_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [temp_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various +** features that require the use of temporary files may fail. Here is an +** example of how to do this using C++ with the Windows Runtime: +** +** 
    +** LPCWSTR zPath = Windows::Storage::ApplicationData::Current->
+**       TemporaryFolder->Path->Data();
+** char zPathBuf[MAX_PATH + 1];
+** memset(zPathBuf, 0, sizeof(zPathBuf));
+** WideCharToMultiByte(CP_UTF8, 0, zPath, -1, zPathBuf, sizeof(zPathBuf),
+**       NULL, NULL);
+** sqlite3_temp_directory = sqlite3_mprintf("%s", zPathBuf);
+**
    +*/ +SQLITE_API SQLITE_EXTERN char *sqlite3_temp_directory; + +/* +** CAPI3REF: Name Of The Folder Holding Database Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all database files +** specified with a relative pathname and created or accessed by +** SQLite when using a built-in windows [sqlite3_vfs | VFS] will be assumed +** to be relative to that directory.)^ ^If this variable is a NULL +** pointer, then SQLite assumes that all database files specified +** with a relative pathname are relative to the current directory +** for the process. Only the windows VFS makes use of this global +** variable; it is ignored by the unix VFS. +** +** Changing the value of this variable while a database connection is +** open can result in a corrupt database. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [data_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [data_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [data_store_directory pragma] should be avoided. +*/ +SQLITE_API SQLITE_EXTERN char *sqlite3_data_directory; + +/* +** CAPI3REF: Test For Auto-Commit Mode +** KEYWORDS: {autocommit mode} +** +** ^The sqlite3_get_autocommit() interface returns non-zero or +** zero if the given database connection is or is not in autocommit mode, +** respectively. ^Autocommit mode is on by default. +** ^Autocommit mode is disabled by a [BEGIN] statement. +** ^Autocommit mode is re-enabled by a [COMMIT] or [ROLLBACK]. +** +** If certain kinds of errors occur on a statement within a multi-statement +** transaction (errors including [SQLITE_FULL], [SQLITE_IOERR], +** [SQLITE_NOMEM], [SQLITE_BUSY], and [SQLITE_INTERRUPT]) then the +** transaction might be rolled back automatically. The only way to +** find out whether SQLite automatically rolled back the transaction after +** an error is to use this function. +** +** If another thread changes the autocommit status of the database +** connection while this routine is running, then the return value +** is undefined. +*/ +SQLITE_API int sqlite3_get_autocommit(sqlite3*); + +/* +** CAPI3REF: Find The Database Handle Of A Prepared Statement +** +** ^The sqlite3_db_handle interface returns the [database connection] handle +** to which a [prepared statement] belongs. ^The [database connection] +** returned by sqlite3_db_handle is the same [database connection] +** that was the first argument +** to the [sqlite3_prepare_v2()] call (or its variants) that was used to +** create the statement in the first place. +*/ +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); + +/* +** CAPI3REF: Return The Filename For A Database Connection +** +** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename +** associated with database N of connection D. ^The main database file +** has the name "main". If there is no attached database N on the database +** connection D, or if database N is a temporary or in-memory database, then +** a NULL pointer is returned. +** +** ^The filename returned by this function is the output of the +** xFullPathname method of the [VFS]. ^In other words, the filename +** will be an absolute pathname, even if the filename used +** to open the database originally was a URI or relative pathname. +*/ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine if a database is read-only +** +** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N +** of connection D is read-only, 0 if it is read/write, or -1 if N is not +** the name of a database on connection D. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Find the next prepared statement +** +** ^This interface returns a pointer to the next [prepared statement] after +** pStmt associated with the [database connection] pDb. ^If pStmt is NULL +** then this interface returns a pointer to the first prepared statement +** associated with the database connection pDb. ^If no prepared statement +** satisfies the conditions of this routine, it returns NULL. +** +** The [database connection] pointer D in a call to +** [sqlite3_next_stmt(D,S)] must refer to an open database +** connection and in particular must not be a NULL pointer. +*/ +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Commit And Rollback Notification Callbacks +** +** ^The sqlite3_commit_hook() interface registers a callback +** function to be invoked whenever a transaction is [COMMIT | committed]. +** ^Any callback set by a previous call to sqlite3_commit_hook() +** for the same database connection is overridden. +** ^The sqlite3_rollback_hook() interface registers a callback +** function to be invoked whenever a transaction is [ROLLBACK | rolled back]. +** ^Any callback set by a previous call to sqlite3_rollback_hook() +** for the same database connection is overridden. +** ^The pArg argument is passed through to the callback. +** ^If the callback on a commit hook function returns non-zero, +** then the commit is converted into a rollback. +** +** ^The sqlite3_commit_hook(D,C,P) and sqlite3_rollback_hook(D,C,P) functions +** return the P argument from the previous call of the same function +** on the same [database connection] D, or NULL for +** the first call for each function on D. +** +** The commit and rollback hook callbacks are not reentrant. +** The callback implementation must not do anything that will modify +** the database connection that invoked the callback. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the commit +** or rollback hook in the first place. +** Note that running any other SQL statements, including SELECT statements, +** or merely calling [sqlite3_prepare_v2()] and [sqlite3_step()] will modify +** the database connections for the meaning of "modify" in this paragraph. +** +** ^Registering a NULL function disables the callback. +** +** ^When the commit hook callback routine returns zero, the [COMMIT] +** operation is allowed to continue normally. ^If the commit hook +** returns non-zero, then the [COMMIT] is converted into a [ROLLBACK]. +** ^The rollback hook is invoked on a rollback that results from a commit +** hook returning non-zero, just as it would be with any other rollback. +** +** ^For the purposes of this API, a transaction is said to have been +** rolled back if an explicit "ROLLBACK" statement is executed, or +** an error or constraint causes an implicit rollback to occur. +** ^The rollback callback is not invoked if a transaction is +** automatically rolled back because the database connection is closed. +** +** See also the [sqlite3_update_hook()] interface. +*/ +SQLITE_API void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); +SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); + +/* +** CAPI3REF: Data Change Notification Callbacks +** +** ^The sqlite3_update_hook() interface registers a callback function +** with the [database connection] identified by the first argument +** to be invoked whenever a row is updated, inserted or deleted in +** a rowid table. +** ^Any callback set by a previous call to this function +** for the same database connection is overridden. +** +** ^The second argument is a pointer to the function to invoke when a +** row is updated, inserted or deleted in a rowid table. +** ^The first argument to the callback is a copy of the third argument +** to sqlite3_update_hook(). +** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], +** or [SQLITE_UPDATE], depending on the operation that caused the callback +** to be invoked. +** ^The third and fourth arguments to the callback contain pointers to the +** database and table name containing the affected row. +** ^The final callback parameter is the [rowid] of the row. +** ^In the case of an update, this is the [rowid] after the update takes place. +** +** ^(The update hook is not invoked when internal system tables are +** modified (i.e. sqlite_master and sqlite_sequence).)^ +** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. +** +** ^In the current implementation, the update hook +** is not invoked when duplication rows are deleted because of an +** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook +** invoked when rows are deleted using the [truncate optimization]. +** The exceptions defined in this paragraph might change in a future +** release of SQLite. +** +** The update hook implementation must not do anything that will modify +** the database connection that invoked the update hook. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the update hook. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^The sqlite3_update_hook(D,C,P) function +** returns the P argument from the previous call +** on the same [database connection] D, or NULL for +** the first call on D. +** +** See also the [sqlite3_commit_hook()] and [sqlite3_rollback_hook()] +** interfaces. +*/ +SQLITE_API void *sqlite3_update_hook( + sqlite3*, + void(*)(void *,int ,char const *,char const *,sqlite3_int64), + void* +); + +/* +** CAPI3REF: Enable Or Disable Shared Pager Cache +** +** ^(This routine enables or disables the sharing of the database cache +** and schema data structures between [database connection | connections] +** to the same database. Sharing is enabled if the argument is true +** and disabled if the argument is false.)^ +** +** ^Cache sharing is enabled and disabled for an entire process. +** This is a change as of SQLite version 3.5.0. In prior versions of SQLite, +** sharing was enabled or disabled for each thread separately. +** +** ^(The cache sharing mode set by this interface effects all subsequent +** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. +** Existing database connections continue use the sharing mode +** that was in effect at the time they were opened.)^ +** +** ^(This routine returns [SQLITE_OK] if shared cache was enabled or disabled +** successfully. An [error code] is returned otherwise.)^ +** +** ^Shared cache is disabled by default. But this might change in +** future releases of SQLite. Applications that care about shared +** cache setting should set it explicitly. +** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** +** See Also: [SQLite Shared-Cache Mode] +*/ +SQLITE_API int sqlite3_enable_shared_cache(int); + +/* +** CAPI3REF: Attempt To Free Heap Memory +** +** ^The sqlite3_release_memory() interface attempts to free N bytes +** of heap memory by deallocating non-essential memory allocations +** held by the database library. Memory used to cache database +** pages to improve performance is an example of non-essential memory. +** ^sqlite3_release_memory() returns the number of bytes actually freed, +** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** See also: [sqlite3_db_release_memory()] +*/ +SQLITE_API int sqlite3_release_memory(int); + +/* +** CAPI3REF: Free Memory Used By A Database Connection +** +** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap +** memory as possible from database connection D. Unlike the +** [sqlite3_release_memory()] interface, this interface is in effect even +** when the [SQLITE_ENABLE_MEMORY_MANAGEMENT] compile-time option is +** omitted. +** +** See also: [sqlite3_release_memory()] +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3*); + +/* +** CAPI3REF: Impose A Limit On Heap Size +** +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. +** +** ^The return value from sqlite3_soft_heap_limit64() is the size of +** the soft heap limit prior to the call, or negative in the case of an +** error. ^If the argument N is negative +** then no change is made to the soft heap limit. Hence, the current +** size of the soft heap limit can be determined by invoking +** sqlite3_soft_heap_limit64() with a negative argument. +** +** ^If the argument N is zero then the soft heap limit is disabled. +** +** ^(The soft heap limit is not enforced in the current implementation +** if one or more of following conditions are true: +** +**
      +**
    • The soft heap limit is set to zero. +**
    • Memory accounting is disabled using a combination of the +** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and +** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. +**
    • An alternative page cache implementation is specified using +** [sqlite3_config]([SQLITE_CONFIG_PCACHE2],...). +**
    • The page cache allocates from its own memory pool supplied +** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than +** from the heap. +**
    )^ +** +** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] +** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], +** the soft heap limit is enforced on every memory allocation. Without +** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced +** when memory is allocated by the page cache. Testing suggests that because +** the page cache is the predominate memory user in SQLite, most +** applications will achieve adequate soft heap limit enforcement without +** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** The circumstances under which SQLite will enforce the soft heap limit may +** changes in future releases of SQLite. +*/ +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + + +/* +** CAPI3REF: Extract Metadata About A Column Of A Table +** +** ^This routine returns metadata about a specific column of a specific +** database table accessible using the [database connection] handle +** passed as the first function argument. +** +** ^The column is identified by the second, third and fourth parameters to +** this function. ^The second parameter is either the name of the database +** (i.e. "main", "temp", or an attached database) containing the specified +** table or NULL. ^If it is NULL, then all attached databases are searched +** for the table using the same algorithm used by the database engine to +** resolve unqualified table references. +** +** ^The third and fourth parameters to this function are the table and column +** name of the desired column, respectively. Neither of these parameters +** may be NULL. +** +** ^Metadata is returned by writing to the memory locations passed as the 5th +** and subsequent parameters to this function. ^Any of these arguments may be +** NULL, in which case the corresponding element of metadata is omitted. +** +** ^(
    +** +**
    Parameter Output
    Type     		Description +** +**
    5th const char* Data type +**
    6th const char* Name of default collation sequence +**
    7th int True if column has a NOT NULL constraint +**
    8th int True if column is part of the PRIMARY KEY +**
    9th int True if column is [AUTOINCREMENT] +**
    +**
    )^ +** +** ^The memory pointed to by the character pointers returned for the +** declaration type and collation sequence is valid only until the next +** call to any SQLite API function. +** +** ^If the specified table is actually a view, an [error code] is returned. +** +** ^If the specified column is "rowid", "oid" or "_rowid_" and an +** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output +** parameters are set for the explicitly declared column. ^(If there is no +** explicitly declared [INTEGER PRIMARY KEY] column, then the output +** parameters are set as follows: +** +** 
    +**     data type: "INTEGER"
+**     collation sequence: "BINARY"
+**     not null: 0
+**     primary key: 1
+**     auto increment: 0
+**
    )^ +** +** ^(This function may load one or more schemas from database files. If an +** error occurs during this process, or if the requested table or column +** cannot be found, an [error code] is returned and an error message left +** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^ +** +** ^This API is only available if the library was compiled with the +** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined. +*/ +SQLITE_API int sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +); + +/* +** CAPI3REF: Load An Extension +** +** ^This interface loads an SQLite extension library from the named file. +** +** ^The sqlite3_load_extension() interface attempts to load an +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. +** +** ^The entry point is zProc. +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ +** ^The sqlite3_load_extension() interface returns +** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. +** ^If an error occurs and pzErrMsg is not 0, then the +** [sqlite3_load_extension()] interface shall attempt to +** fill *pzErrMsg with error message text stored in memory +** obtained from [sqlite3_malloc()]. The calling function +** should free this memory by calling [sqlite3_free()]. +** +** ^Extension loading must be enabled using +** [sqlite3_enable_load_extension()] prior to calling this API, +** otherwise an error will be returned. +** +** See also the [load_extension() SQL function]. +*/ +SQLITE_API int sqlite3_load_extension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Derived from zFile if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +); + +/* +** CAPI3REF: Enable Or Disable Extension Loading +** +** ^So as not to open security holes in older applications that are +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API +** is provided to turn the [sqlite3_load_extension()] mechanism on and off. +** +** ^Extension loading is off by default. +** ^Call the sqlite3_enable_load_extension() routine with onoff==1 +** to turn extension loading on and call it with onoff==0 to turn +** it back off again. +*/ +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); + +/* +** CAPI3REF: Automatically Load Statically Linked Extensions +** +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked [SQLite extension] +** that is to be automatically loaded into all new database connections. +** +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects and integer result as if the signature of the +** entry point where as follows: +** +** 
    +**    int xEntryPoint(
+**      sqlite3 *db,
+**      const char **pzErrMsg,
+**      const struct sqlite3_api_routines *pThunk
+**    );
+**
    )^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()] +** and [sqlite3_cancel_auto_extension()] +*/ +SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); + +/* +** CAPI3REF: Cancel Automatic Extension Loading +** +** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the +** initialization routine X that was registered using a prior call to +** [sqlite3_auto_extension(X)]. ^The [sqlite3_cancel_auto_extension(X)] +** routine returns 1 if initialization routine X was successfully +** unregistered and it returns 0 if X was not on the list of initialization +** routines. +*/ +SQLITE_API int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); + +/* +** CAPI3REF: Reset Automatic Extension Loading +** +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. +*/ +SQLITE_API void sqlite3_reset_auto_extension(void); + +/* +** The interface to the virtual-table mechanism is currently considered +** to be experimental. The interface might change in incompatible ways. +** If this is a problem for you, do not use the interface at this time. +** +** When the virtual-table mechanism stabilizes, we will declare the +** interface fixed, support it indefinitely, and remove this comment. +*/ + +/* +** Structures used by the virtual table interface +*/ +typedef struct sqlite3_vtab sqlite3_vtab; +typedef struct sqlite3_index_info sqlite3_index_info; +typedef struct sqlite3_vtab_cursor sqlite3_vtab_cursor; +typedef struct sqlite3_module sqlite3_module; + +/* +** CAPI3REF: Virtual Table Object +** KEYWORDS: sqlite3_module {virtual table module} +** +** This structure, sometimes called a "virtual table module", +** defines the implementation of a [virtual tables]. +** This structure consists mostly of methods for the module. +** +** ^A virtual table module is created by filling in a persistent +** instance of this structure and passing a pointer to that instance +** to [sqlite3_create_module()] or [sqlite3_create_module_v2()]. +** ^The registration remains valid until it is replaced by a different +** module or until the [database connection] closes. The content +** of this structure must not change while it is registered with +** any database connection. +*/ +struct sqlite3_module { + int iVersion; + int (*xCreate)(sqlite3*, void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, char**); + int (*xConnect)(sqlite3*, void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, char**); + int (*xBestIndex)(sqlite3_vtab *pVTab, sqlite3_index_info*); + int (*xDisconnect)(sqlite3_vtab *pVTab); + int (*xDestroy)(sqlite3_vtab *pVTab); + int (*xOpen)(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor); + int (*xClose)(sqlite3_vtab_cursor*); + int (*xFilter)(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); + int (*xNext)(sqlite3_vtab_cursor*); + int (*xEof)(sqlite3_vtab_cursor*); + int (*xColumn)(sqlite3_vtab_cursor*, sqlite3_context*, int); + int (*xRowid)(sqlite3_vtab_cursor*, sqlite3_int64 *pRowid); + int (*xUpdate)(sqlite3_vtab *, int, sqlite3_value **, sqlite3_int64 *); + int (*xBegin)(sqlite3_vtab *pVTab); + int (*xSync)(sqlite3_vtab *pVTab); + int (*xCommit)(sqlite3_vtab *pVTab); + int (*xRollback)(sqlite3_vtab *pVTab); + int (*xFindFunction)(sqlite3_vtab *pVtab, int nArg, const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg); + int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 1 of the sqlite_module object. Those + ** below are for version 2 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); +}; + +/* +** CAPI3REF: Virtual Table Indexing Information +** KEYWORDS: sqlite3_index_info +** +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to +** pass information into and receive the reply from the [xBestIndex] +** method of a [virtual table module]. The fields under **Inputs** are the +** inputs to xBestIndex and are read-only. xBestIndex inserts its +** results into the **Outputs** fields. +** +** ^(The aConstraint[] array records WHERE clause constraints of the form: +** +**
    column OP expr
    +** +** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in +** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the +** expr on the right-hand side can be evaluated (and thus the constraint +** is usable) and false if it cannot.)^ +** +** ^The optimizer automatically inverts terms of the form "expr OP column" +** and makes other simplifications to the WHERE clause in an attempt to +** get as many WHERE clause terms into the form shown above as possible. +** ^The aConstraint[] array only reports WHERE clause terms that are +** relevant to the particular virtual table being queried. +** +** ^Information about the ORDER BY clause is stored in aOrderBy[]. +** ^Each term of aOrderBy records a column of the ORDER BY clause. +** +** The [xBestIndex] method must fill aConstraintUsage[] with information +** about what parameters to pass to xFilter. ^If argvIndex>0 then +** the right-hand side of the corresponding aConstraint[] is evaluated +** and becomes the argvIndex-th entry in argv. ^(If aConstraintUsage[].omit +** is true, then the constraint is assumed to be fully handled by the +** virtual table and is not checked again by SQLite.)^ +** +** ^The idxNum and idxPtr values are recorded and passed into the +** [xFilter] method. +** ^[sqlite3_free()] is used to free idxPtr if and only if +** needToFreeIdxPtr is true. +** +** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in +** the correct order to satisfy the ORDER BY clause so that no separate +** sorting step is required. +** +** ^The estimatedCost value is an estimate of the cost of a particular +** strategy. A cost of N indicates that the cost of the strategy is similar +** to a linear scan of an SQLite table with N rows. A cost of log(N) +** indicates that the expense of the operation is similar to that of a +** binary search on a unique indexed field of an SQLite table with N rows. +** +** ^The estimatedRows value is an estimate of the number of rows that +** will be returned by the strategy. +** +** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info +** structure for SQLite version 3.8.2. If a virtual table extension is +** used with an SQLite version earlier than 3.8.2, the results of attempting +** to read or write the estimatedRows field are undefined (but are likely +** to included crashing the application). The estimatedRows field should +** therefore only be used if [sqlite3_libversion_number()] returns a +** value greater than or equal to 3008002. +*/ +struct sqlite3_index_info { + /* Inputs */ + int nConstraint; /* Number of entries in aConstraint */ + struct sqlite3_index_constraint { + int iColumn; /* Column on left-hand side of constraint */ + unsigned char op; /* Constraint operator */ + unsigned char usable; /* True if this constraint is usable */ + int iTermOffset; /* Used internally - xBestIndex should ignore */ + } *aConstraint; /* Table of WHERE clause constraints */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + struct sqlite3_index_orderby { + int iColumn; /* Column number */ + unsigned char desc; /* True for DESC. False for ASC. */ + } *aOrderBy; /* The ORDER BY clause */ + /* Outputs */ + struct sqlite3_index_constraint_usage { + int argvIndex; /* if >0, constraint is part of argv to xFilter */ + unsigned char omit; /* Do not code a test for this constraint */ + } *aConstraintUsage; + int idxNum; /* Number used to identify the index */ + char *idxStr; /* String, possibly obtained from sqlite3_malloc */ + int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ + int orderByConsumed; /* True if output is already ordered */ + double estimatedCost; /* Estimated cost of using this index */ + /* Fields below are only available in SQLite 3.8.2 and later */ + sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ +}; + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros defined the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the wHERE clause of +** a query that uses a [virtual table]. +*/ +#define SQLITE_INDEX_CONSTRAINT_EQ 2 +#define SQLITE_INDEX_CONSTRAINT_GT 4 +#define SQLITE_INDEX_CONSTRAINT_LE 8 +#define SQLITE_INDEX_CONSTRAINT_LT 16 +#define SQLITE_INDEX_CONSTRAINT_GE 32 +#define SQLITE_INDEX_CONSTRAINT_MATCH 64 + +/* +** CAPI3REF: Register A Virtual Table Implementation +** +** ^These routines are used to register a new [virtual table module] name. +** ^Module names must be registered before +** creating a new [virtual table] using the module and before using a +** preexisting [virtual table] for the module. +** +** ^The module name is registered on the [database connection] specified +** by the first parameter. ^The name of the module is given by the +** second parameter. ^The third parameter is a pointer to +** the implementation of the [virtual table module]. ^The fourth +** parameter is an arbitrary client data pointer that is passed through +** into the [xCreate] and [xConnect] methods of the virtual table module +** when a new virtual table is be being created or reinitialized. +** +** ^The sqlite3_create_module_v2() interface has a fifth parameter which +** is a pointer to a destructor for the pClientData. ^SQLite will +** invoke the destructor function (if it is not NULL) when SQLite +** no longer needs the pClientData pointer. ^The destructor will also +** be invoked if the call to sqlite3_create_module_v2() fails. +** ^The sqlite3_create_module() +** interface is equivalent to sqlite3_create_module_v2() with a NULL +** destructor. +*/ +SQLITE_API int sqlite3_create_module( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData /* Client data for xCreate/xConnect */ +); +SQLITE_API int sqlite3_create_module_v2( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData, /* Client data for xCreate/xConnect */ + void(*xDestroy)(void*) /* Module destructor function */ +); + +/* +** CAPI3REF: Virtual Table Instance Object +** KEYWORDS: sqlite3_vtab +** +** Every [virtual table module] implementation uses a subclass +** of this object to describe a particular instance +** of the [virtual table]. Each subclass will +** be tailored to the specific needs of the module implementation. +** The purpose of this superclass is to define certain fields that are +** common to all module implementations. +** +** ^Virtual tables methods can set an error message by assigning a +** string obtained from [sqlite3_mprintf()] to zErrMsg. The method should +** take care that any prior string is freed by a call to [sqlite3_free()] +** prior to assigning a new string to zErrMsg. ^After the error message +** is delivered up to the client application, the string will be automatically +** freed by sqlite3_free() and the zErrMsg field will be zeroed. +*/ +struct sqlite3_vtab { + const sqlite3_module *pModule; /* The module for this virtual table */ + int nRef; /* NO LONGER USED */ + char *zErrMsg; /* Error message from sqlite3_mprintf() */ + /* Virtual table implementations will typically add additional fields */ +}; + +/* +** CAPI3REF: Virtual Table Cursor Object +** KEYWORDS: sqlite3_vtab_cursor {virtual table cursor} +** +** Every [virtual table module] implementation uses a subclass of the +** following structure to describe cursors that point into the +** [virtual table] and are used +** to loop through the virtual table. Cursors are created using the +** [sqlite3_module.xOpen | xOpen] method of the module and are destroyed +** by the [sqlite3_module.xClose | xClose] method. Cursors are used +** by the [xFilter], [xNext], [xEof], [xColumn], and [xRowid] methods +** of the module. Each module implementation will define +** the content of a cursor structure to suit its own needs. +** +** This superclass exists in order to define fields of the cursor that +** are common to all implementations. +*/ +struct sqlite3_vtab_cursor { + sqlite3_vtab *pVtab; /* Virtual table of this cursor */ + /* Virtual table implementations will typically add additional fields */ +}; + +/* +** CAPI3REF: Declare The Schema Of A Virtual Table +** +** ^The [xCreate] and [xConnect] methods of a +** [virtual table module] call this interface +** to declare the format (the names and datatypes of the columns) of +** the virtual tables they implement. +*/ +SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); + +/* +** CAPI3REF: Overload A Function For A Virtual Table +** +** ^(Virtual tables can provide alternative implementations of functions +** using the [xFindFunction] method of the [virtual table module]. +** But global versions of those functions +** must exist in order to be overloaded.)^ +** +** ^(This API makes sure a global version of a function with a particular +** name and number of parameters exists. If no such function exists +** before this API is called, a new function is created.)^ ^The implementation +** of the new function always causes an exception to be thrown. So +** the new function is not good for anything by itself. Its only +** purpose is to be a placeholder function that can be overloaded +** by a [virtual table]. +*/ +SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); + +/* +** The interface to the virtual-table mechanism defined above (back up +** to a comment remarkably similar to this one) is currently considered +** to be experimental. The interface might change in incompatible ways. +** If this is a problem for you, do not use the interface at this time. +** +** When the virtual-table mechanism stabilizes, we will declare the +** interface fixed, support it indefinitely, and remove this comment. +*/ + +/* +** CAPI3REF: A Handle To An Open BLOB +** KEYWORDS: {BLOB handle} {BLOB handles} +** +** An instance of this object represents an open BLOB on which +** [sqlite3_blob_open | incremental BLOB I/O] can be performed. +** ^Objects of this type are created by [sqlite3_blob_open()] +** and destroyed by [sqlite3_blob_close()]. +** ^The [sqlite3_blob_read()] and [sqlite3_blob_write()] interfaces +** can be used to read or write small subsections of the BLOB. +** ^The [sqlite3_blob_bytes()] interface returns the size of the BLOB in bytes. +*/ +typedef struct sqlite3_blob sqlite3_blob; + +/* +** CAPI3REF: Open A BLOB For Incremental I/O +** +** ^(This interfaces opens a [BLOB handle | handle] to the BLOB located +** in row iRow, column zColumn, table zTable in database zDb; +** in other words, the same BLOB that would be selected by: +** +** 
    +**     SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow;
+**
    )^ +** +** ^If the flags parameter is non-zero, then the BLOB is opened for read +** and write access. ^If it is zero, the BLOB is opened for read access. +** ^It is not possible to open a column that is part of an index or primary +** key for writing. ^If [foreign key constraints] are enabled, it is +** not possible to open a column that is part of a [child key] for writing. +** +** ^Note that the database name is not the filename that contains +** the database but rather the symbolic name of the database that +** appears after the AS keyword when the database is connected using [ATTACH]. +** ^For the main database file, the database name is "main". +** ^For TEMP tables, the database name is "temp". +** +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is written +** to *ppBlob. Otherwise an [error code] is returned and *ppBlob is set +** to be a null pointer.)^ +** ^This function sets the [database connection] error code and message +** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()] and related +** functions. ^Note that the *ppBlob variable is always initialized in a +** way that makes it safe to invoke [sqlite3_blob_close()] on *ppBlob +** regardless of the success or failure of this routine. +** +** ^(If the row that a BLOB handle points to is modified by an +** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects +** then the BLOB handle is marked as "expired". +** This is true if any column of the row is changed, even a column +** other than the one the BLOB handle is open on.)^ +** ^Calls to [sqlite3_blob_read()] and [sqlite3_blob_write()] for +** an expired BLOB handle fail with a return code of [SQLITE_ABORT]. +** ^(Changes written into a BLOB prior to the BLOB expiring are not +** rolled back by the expiration of the BLOB. Such changes will eventually +** commit if the transaction continues to completion.)^ +** +** ^Use the [sqlite3_blob_bytes()] interface to determine the size of +** the opened blob. ^The size of a blob may not be changed by this +** interface. Use the [UPDATE] SQL command to change the size of a +** blob. +** +** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] +** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. +** +** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces +** and the built-in [zeroblob] SQL function can be used, if desired, +** to create an empty, zero-filled blob in which to read or write using +** this interface. +** +** To avoid a resource leak, every open [BLOB handle] should eventually +** be released by a call to [sqlite3_blob_close()]. +*/ +SQLITE_API int sqlite3_blob_open( + sqlite3*, + const char *zDb, + const char *zTable, + const char *zColumn, + sqlite3_int64 iRow, + int flags, + sqlite3_blob **ppBlob +); + +/* +** CAPI3REF: Move a BLOB Handle to a New Row +** +** ^This function is used to move an existing blob handle so that it points +** to a different row of the same database table. ^The new row is identified +** by the rowid value passed as the second argument. Only the row can be +** changed. ^The database, table and column on which the blob handle is open +** remain the same. Moving an existing blob handle to a new row can be +** faster than closing the existing handle and opening a new one. +** +** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - +** it must exist and there must be either a blob or text value stored in +** the nominated column.)^ ^If the new row is not present in the table, or if +** it does not contain a blob or text value, or if another error occurs, an +** SQLite error code is returned and the blob handle is considered aborted. +** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return +** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle +** always returns zero. +** +** ^This function sets the database handle error code and message. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); + +/* +** CAPI3REF: Close A BLOB Handle +** +** ^Closes an open [BLOB handle]. +** +** ^Closing a BLOB shall cause the current transaction to commit +** if there are no other BLOBs, no pending prepared statements, and the +** database connection is in [autocommit mode]. +** ^If any writes were made to the BLOB, they might be held in cache +** until the close operation if they will fit. +** +** ^(Closing the BLOB often forces the changes +** out to disk and so if any I/O errors occur, they will likely occur +** at the time when the BLOB is closed. Any errors that occur during +** closing are reported as a non-zero return value.)^ +** +** ^(The BLOB is closed unconditionally. Even if this routine returns +** an error code, the BLOB is still closed.)^ +** +** ^Calling this routine with a null pointer (such as would be returned +** by a failed call to [sqlite3_blob_open()]) is a harmless no-op. +*/ +SQLITE_API int sqlite3_blob_close(sqlite3_blob *); + +/* +** CAPI3REF: Return The Size Of An Open BLOB +** +** ^Returns the size in bytes of the BLOB accessible via the +** successfully opened [BLOB handle] in its only argument. ^The +** incremental blob I/O routines can only read or overwriting existing +** blob content; they cannot change the size of a blob. +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +*/ +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); + +/* +** CAPI3REF: Read Data From A BLOB Incrementally +** +** ^(This function is used to read data from an open [BLOB handle] into a +** caller-supplied buffer. N bytes of data are copied into buffer Z +** from the open BLOB, starting at offset iOffset.)^ +** +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is read. ^If N or iOffset is +** less than zero, [SQLITE_ERROR] is returned and no data is read. +** ^The size of the blob (and hence the maximum value of N+iOffset) +** can be determined using the [sqlite3_blob_bytes()] interface. +** +** ^An attempt to read from an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. +** +** ^(On success, sqlite3_blob_read() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_write()]. +*/ +SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); + +/* +** CAPI3REF: Write Data Into A BLOB Incrementally +** +** ^This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. ^N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset. +** +** ^If the [BLOB handle] passed as the first argument was not opened for +** writing (the flags parameter to [sqlite3_blob_open()] was zero), +** this function returns [SQLITE_READONLY]. +** +** ^This function may only modify the contents of the BLOB; it is +** not possible to increase the size of a BLOB using this API. +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is written. ^If N is +** less than zero [SQLITE_ERROR] is returned and no data is written. +** The size of the BLOB (and hence the maximum value of N+iOffset) +** can be determined using the [sqlite3_blob_bytes()] interface. +** +** ^An attempt to write to an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred +** before the [BLOB handle] expired are not rolled back by the +** expiration of the handle, though of course those changes might +** have been overwritten by the statement that expired the BLOB handle +** or by other independent statements. +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_read()]. +*/ +SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); + +/* +** CAPI3REF: Virtual File System Objects +** +** A virtual filesystem (VFS) is an [sqlite3_vfs] object +** that SQLite uses to interact +** with the underlying operating system. Most SQLite builds come with a +** single default VFS that is appropriate for the host computer. +** New VFSes can be registered and existing VFSes can be unregistered. +** The following interfaces are provided. +** +** ^The sqlite3_vfs_find() interface returns a pointer to a VFS given its name. +** ^Names are case sensitive. +** ^Names are zero-terminated UTF-8 strings. +** ^If there is no match, a NULL pointer is returned. +** ^If zVfsName is NULL then the default VFS is returned. +** +** ^New VFSes are registered with sqlite3_vfs_register(). +** ^Each new VFS becomes the default VFS if the makeDflt flag is set. +** ^The same VFS can be registered multiple times without injury. +** ^To make an existing VFS into the default VFS, register it again +** with the makeDflt flag set. If two different VFSes with the +** same name are registered, the behavior is undefined. If a +** VFS is registered with a name that is NULL or an empty string, +** then the behavior is undefined. +** +** ^Unregister a VFS with the sqlite3_vfs_unregister() interface. +** ^(If the default VFS is unregistered, another VFS is chosen as +** the default. The choice for the new VFS is arbitrary.)^ +*/ +SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName); +SQLITE_API int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); +SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); + +/* +** CAPI3REF: Mutexes +** +** The SQLite core uses these routines for thread +** synchronization. Though they are intended for internal +** use by SQLite, code that links against SQLite is +** permitted to use any of these routines. +** +** The SQLite source code contains multiple implementations +** of these mutex routines. An appropriate implementation +** is selected automatically at compile-time. ^(The following +** implementations are available in the SQLite core: +** +**
      +**
    • SQLITE_MUTEX_PTHREADS +**
    • SQLITE_MUTEX_W32 +**
    • SQLITE_MUTEX_NOOP +**
    )^ +** +** ^The SQLITE_MUTEX_NOOP implementation is a set of routines +** that does no real locking and is appropriate for use in +** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix +** and Windows. +** +** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex +** implementation is included with the library. In this case the +** application must supply a custom mutex implementation using the +** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function +** before calling sqlite3_initialize() or any other public sqlite3_ +** function that calls sqlite3_initialize().)^ +** +** ^The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. ^If it returns NULL +** that means that a mutex could not be allocated. ^SQLite +** will unwind its stack and return an error. ^(The argument +** to sqlite3_mutex_alloc() is one of these integer constants: +** +**
      +**
    • SQLITE_MUTEX_FAST +**
    • SQLITE_MUTEX_RECURSIVE +**
    • SQLITE_MUTEX_STATIC_MASTER +**
    • SQLITE_MUTEX_STATIC_MEM +**
    • SQLITE_MUTEX_STATIC_OPEN +**
    • SQLITE_MUTEX_STATIC_PRNG +**
    • SQLITE_MUTEX_STATIC_LRU +**
    • SQLITE_MUTEX_STATIC_PMEM +**
    • SQLITE_MUTEX_STATIC_APP1 +**
    • SQLITE_MUTEX_STATIC_APP2 +**
    )^ +** +** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) +** cause sqlite3_mutex_alloc() to create +** a new mutex. ^The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. ^SQLite will only request a recursive mutex in +** cases where it really needs one. ^If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other +** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return +** a pointer to a static preexisting mutex. ^Six static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. ^But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +** +** ^The sqlite3_mutex_free() routine deallocates a previously +** allocated dynamic mutex. ^SQLite is careful to deallocate every +** dynamic mutex that it allocates. The dynamic mutexes must not be in +** use when they are deallocated. Attempting to deallocate a static +** mutex results in undefined behavior. ^SQLite never deallocates +** a static mutex. +** +** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. ^If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] +** upon successful entry. ^(Mutexes created using +** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. +** In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter.)^ ^(If the same thread tries to enter any other +** kind of mutex more than once, the behavior is undefined. +** SQLite will never exhibit +** such behavior in its own use of mutexes.)^ +** +** ^(Some systems (for example, Windows 95) do not support the operation +** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable behavior.)^ +** +** ^The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. ^(The behavior +** is undefined if the mutex is not currently entered by the +** calling thread or is not currently allocated. SQLite will +** never do either.)^ +** +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or +** sqlite3_mutex_leave() is a NULL pointer, then all three routines +** behave as no-ops. +** +** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. +*/ +SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int); +SQLITE_API void sqlite3_mutex_free(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_try(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); + +/* +** CAPI3REF: Mutex Methods Object +** +** An instance of this structure defines the low-level routines +** used to allocate and use mutexes. +** +** Usually, the default mutex implementations provided by SQLite are +** sufficient, however the user has the option of substituting a custom +** implementation for specialized deployments or systems for which SQLite +** does not provide a suitable implementation. In this case, the user +** creates and populates an instance of this structure to pass +** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. +** Additionally, an instance of this structure can be used as an +** output variable when querying the system for the current mutex +** implementation, using the [SQLITE_CONFIG_GETMUTEX] option. +** +** ^The xMutexInit method defined by this structure is invoked as +** part of system initialization by the sqlite3_initialize() function. +** ^The xMutexInit routine is called by SQLite exactly once for each +** effective call to [sqlite3_initialize()]. +** +** ^The xMutexEnd method defined by this structure is invoked as +** part of system shutdown by the sqlite3_shutdown() function. The +** implementation of this method is expected to release all outstanding +** resources obtained by the mutex methods implementation, especially +** those obtained by the xMutexInit method. ^The xMutexEnd() +** interface is invoked exactly once for each call to [sqlite3_shutdown()]. +** +** ^(The remaining seven methods defined by this structure (xMutexAlloc, +** xMutexFree, xMutexEnter, xMutexTry, xMutexLeave, xMutexHeld and +** xMutexNotheld) implement the following interfaces (respectively): +** +**
      +**
    • [sqlite3_mutex_alloc()]
      • +**
    • [sqlite3_mutex_free()]
      • +**
    • [sqlite3_mutex_enter()]
      • +**
    • [sqlite3_mutex_try()]
      • +**
    • [sqlite3_mutex_leave()]
      • +**
    • [sqlite3_mutex_held()]
      • +**
    • [sqlite3_mutex_notheld()]
      • +**
    )^ +** +** The only difference is that the public sqlite3_XXX functions enumerated +** above silently ignore any invocations that pass a NULL pointer instead +** of a valid mutex handle. The implementations of the methods defined +** by this structure are not required to handle this case, the results +** of passing a NULL pointer instead of a valid mutex handle are undefined +** (i.e. it is acceptable to provide an implementation that segfaults if +** it is passed a NULL pointer). +** +** The xMutexInit() method must be threadsafe. ^It must be harmless to +** invoke xMutexInit() multiple times within the same process and without +** intervening calls to xMutexEnd(). Second and subsequent calls to +** xMutexInit() must be no-ops. +** +** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory +** allocation for a static mutex. ^However xMutexAlloc() may use SQLite +** memory allocation for a fast or recursive mutex. +** +** ^SQLite will invoke the xMutexEnd() method when [sqlite3_shutdown()] is +** called, but only if the prior call to xMutexInit returned SQLITE_OK. +** If xMutexInit fails in any way, it is expected to clean up after itself +** prior to returning. +*/ +typedef struct sqlite3_mutex_methods sqlite3_mutex_methods; +struct sqlite3_mutex_methods { + int (*xMutexInit)(void); + int (*xMutexEnd)(void); + sqlite3_mutex *(*xMutexAlloc)(int); + void (*xMutexFree)(sqlite3_mutex *); + void (*xMutexEnter)(sqlite3_mutex *); + int (*xMutexTry)(sqlite3_mutex *); + void (*xMutexLeave)(sqlite3_mutex *); + int (*xMutexHeld)(sqlite3_mutex *); + int (*xMutexNotheld)(sqlite3_mutex *); +}; + +/* +** CAPI3REF: Mutex Verification Routines +** +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines +** are intended for use inside assert() statements. ^The SQLite core +** never uses these routines except inside an assert() and applications +** are advised to follow the lead of the core. ^The SQLite core only +** provides implementations for these routines when it is compiled +** with the SQLITE_DEBUG flag. ^External mutex implementations +** are only required to provide these routines if SQLITE_DEBUG is +** defined and if NDEBUG is not defined. +** +** ^These routines should return true if the mutex in their argument +** is held or not held, respectively, by the calling thread. +** +** ^The implementation is not required to provide versions of these +** routines that actually work. If the implementation does not provide working +** versions of these routines, it should at least provide stubs that always +** return true so that one does not get spurious assertion failures. +** +** ^If the argument to sqlite3_mutex_held() is a NULL pointer then +** the routine should return 1. This seems counter-intuitive since +** clearly the mutex cannot be held if it does not exist. But +** the reason the mutex does not exist is because the build is not +** using mutexes. And we do not want the assert() containing the +** call to sqlite3_mutex_held() to fail, so a non-zero return is +** the appropriate thing to do. ^The sqlite3_mutex_notheld() +** interface should also return 1 when given a NULL pointer. +*/ +#ifndef NDEBUG +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); +#endif + +/* +** CAPI3REF: Mutex Types +** +** The [sqlite3_mutex_alloc()] interface takes a single argument +** which is one of these integer constants. +** +** The set of static mutexes may change from one SQLite release to the +** next. Applications that override the built-in mutex logic must be +** prepared to accommodate additional static mutexes. +*/ +#define SQLITE_MUTEX_FAST 0 +#define SQLITE_MUTEX_RECURSIVE 1 +#define SQLITE_MUTEX_STATIC_MASTER 2 +#define SQLITE_MUTEX_STATIC_MEM 3 /* sqlite3_malloc() */ +#define SQLITE_MUTEX_STATIC_MEM2 4 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */ +#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_random() */ +#define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ +#define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ + +/* +** CAPI3REF: Retrieve the mutex for a database connection +** +** ^This interface returns a pointer the [sqlite3_mutex] object that +** serializes access to the [database connection] given in the argument +** when the [threading mode] is Serialized. +** ^If the [threading mode] is Single-thread or Multi-thread then this +** routine returns a NULL pointer. +*/ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); + +/* +** CAPI3REF: Low-Level Control Of Database Files +** +** ^The [sqlite3_file_control()] interface makes a direct call to the +** xFileControl method for the [sqlite3_io_methods] object associated +** with a particular database identified by the second argument. ^The +** name of the database is "main" for the main database or "temp" for the +** TEMP database, or the name that appears after the AS keyword for +** databases that are added using the [ATTACH] SQL command. +** ^A NULL pointer can be used in place of "main" to refer to the +** main database file. +** ^The third and fourth parameters to this routine +** are passed directly through to the second and third parameters of +** the xFileControl method. ^The return value of the xFileControl +** method becomes the return value of this routine. +** +** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes +** a pointer to the underlying [sqlite3_file] object to be written into +** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER +** case is a short-circuit path which does not actually invoke the +** underlying sqlite3_io_methods.xFileControl method. +** +** ^If the second parameter (zDbName) does not match the name of any +** open database file, then SQLITE_ERROR is returned. ^This error +** code is not remembered and will not be recalled by [sqlite3_errcode()] +** or [sqlite3_errmsg()]. The underlying xFileControl method might +** also return SQLITE_ERROR. There is no way to distinguish between +** an incorrect zDbName and an SQLITE_ERROR return from the underlying +** xFileControl method. +** +** See also: [SQLITE_FCNTL_LOCKSTATE] +*/ +SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); + +/* +** CAPI3REF: Testing Interface +** +** ^The sqlite3_test_control() interface is used to read out internal +** state of SQLite and to inject faults into SQLite for testing +** purposes. ^The first parameter is an operation code that determines +** the number, meaning, and operation of all subsequent parameters. +** +** This interface is not for use by applications. It exists solely +** for verifying the correct operation of the SQLite library. Depending +** on how the SQLite library is compiled, this interface might not exist. +** +** The details of the operation codes, their meanings, the parameters +** they take, and what they do are all subject to change without notice. +** Unlike most of the SQLite API, this function is not guaranteed to +** operate consistently from one release to the next. +*/ +SQLITE_API int sqlite3_test_control(int op, ...); + +/* +** CAPI3REF: Testing Interface Operation Codes +** +** These constants are the valid operation code parameters used +** as the first argument to [sqlite3_test_control()]. +** +** These parameters and their meanings are subject to change +** without notice. These values are for testing purposes only. +** Applications should not use any of these parameters or the +** [sqlite3_test_control()] interface. +*/ +#define SQLITE_TESTCTRL_FIRST 5 +#define SQLITE_TESTCTRL_PRNG_SAVE 5 +#define SQLITE_TESTCTRL_PRNG_RESTORE 6 +#define SQLITE_TESTCTRL_PRNG_RESET 7 +#define SQLITE_TESTCTRL_BITVEC_TEST 8 +#define SQLITE_TESTCTRL_FAULT_INSTALL 9 +#define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 +#define SQLITE_TESTCTRL_PENDING_BYTE 11 +#define SQLITE_TESTCTRL_ASSERT 12 +#define SQLITE_TESTCTRL_ALWAYS 13 +#define SQLITE_TESTCTRL_RESERVE 14 +#define SQLITE_TESTCTRL_OPTIMIZATIONS 15 +#define SQLITE_TESTCTRL_ISKEYWORD 16 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 +#define SQLITE_TESTCTRL_NEVER_CORRUPT 20 +#define SQLITE_TESTCTRL_VDBE_COVERAGE 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_ISINIT 23 +#define SQLITE_TESTCTRL_LAST 23 + +/* +** CAPI3REF: SQLite Runtime Status +** +** ^This interface is used to retrieve runtime status information +** about the performance of SQLite, and optionally to reset various +** highwater marks. ^The first argument is an integer code for +** the specific parameter to measure. ^(Recognized integer codes +** are of the form [status parameters | SQLITE_STATUS_...].)^ +** ^The current value of the parameter is returned into *pCurrent. +** ^The highest recorded value is returned in *pHighwater. ^If the +** resetFlag is true, then the highest record value is reset after +** *pHighwater is written. ^(Some parameters do not record the highest +** value. For those parameters +** nothing is written into *pHighwater and the resetFlag is ignored.)^ +** ^(Other parameters record only the highwater mark and not the current +** value. For these latter parameters nothing is written into *pCurrent.)^ +** +** ^The sqlite3_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** +** This routine is threadsafe but is not atomic. This routine can be +** called while other threads are running the same or different SQLite +** interfaces. However the values returned in *pCurrent and +** *pHighwater reflect the status of SQLite at different points in time +** and it is possible that another thread might change the parameter +** in between the times when *pCurrent and *pHighwater are written. +** +** See also: [sqlite3_db_status()] +*/ +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); + + +/* +** CAPI3REF: Status Parameters +** KEYWORDS: {status parameters} +** +** These integer constants designate various run-time status parameters +** that can be returned by [sqlite3_status()]. +** +**
    +** [[SQLITE_STATUS_MEMORY_USED]] ^(
    SQLITE_STATUS_MEMORY_USED
    +**
    This parameter is the current amount of memory checked out +** using [sqlite3_malloc()], either directly or indirectly. The +** figure includes calls made to [sqlite3_malloc()] by the application +** and internal memory usage by the SQLite library. Scratch memory +** controlled by [SQLITE_CONFIG_SCRATCH] and auxiliary page-cache +** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in +** this parameter. The amount returned is the sum of the allocation +** sizes as reported by the xSize method in [sqlite3_mem_methods].
    )^ +** +** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
    SQLITE_STATUS_MALLOC_SIZE
    +**
    This parameter records the largest memory allocation request +** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their +** internal equivalents). Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
    )^ +** +** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
    SQLITE_STATUS_MALLOC_COUNT
    +**
    This parameter records the number of separate memory allocations +** currently checked out.
    )^ +** +** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
    SQLITE_STATUS_PAGECACHE_USED
    +**
    This parameter returns the number of pages used out of the +** [pagecache memory allocator] that was configured using +** [SQLITE_CONFIG_PAGECACHE]. The +** value returned is in pages, not in bytes.
    )^ +** +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] +** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    +**
    This parameter returns the number of bytes of page cache +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] +** buffer and where forced to overflow to [sqlite3_malloc()]. The +** returned value includes allocations that overflowed because they +** where too large (they were larger than the "sz" parameter to +** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because +** no space was left in the page cache.
    )^ +** +** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
    SQLITE_STATUS_PAGECACHE_SIZE
    +**
    This parameter records the largest memory allocation request +** handed to [pagecache memory allocator]. Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
    )^ +** +** [[SQLITE_STATUS_SCRATCH_USED]] ^(
    SQLITE_STATUS_SCRATCH_USED
    +**
    This parameter returns the number of allocations used out of the +** [scratch memory allocator] configured using +** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not +** in bytes. Since a single thread may only have one scratch allocation +** outstanding at time, this parameter also reports the number of threads +** using scratch memory at the same time.
    )^ +** +** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    +**
    This parameter returns the number of bytes of scratch memory +** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] +** buffer and where forced to overflow to [sqlite3_malloc()]. The values +** returned include overflows because the requested allocation was too +** larger (that is, because the requested allocation was larger than the +** "sz" parameter to [SQLITE_CONFIG_SCRATCH]) and because no scratch buffer +** slots were available. +**
    )^ +** +** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(
    SQLITE_STATUS_SCRATCH_SIZE
    +**
    This parameter records the largest memory allocation request +** handed to [scratch memory allocator]. Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
    )^ +** +** [[SQLITE_STATUS_PARSER_STACK]] ^(
    SQLITE_STATUS_PARSER_STACK
    +**
    This parameter records the deepest parser stack. It is only +** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
    )^ +**
    +** +** New status parameters may be added from time to time. +*/ +#define SQLITE_STATUS_MEMORY_USED 0 +#define SQLITE_STATUS_PAGECACHE_USED 1 +#define SQLITE_STATUS_PAGECACHE_OVERFLOW 2 +#define SQLITE_STATUS_SCRATCH_USED 3 +#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 +#define SQLITE_STATUS_MALLOC_SIZE 5 +#define SQLITE_STATUS_PARSER_STACK 6 +#define SQLITE_STATUS_PAGECACHE_SIZE 7 +#define SQLITE_STATUS_SCRATCH_SIZE 8 +#define SQLITE_STATUS_MALLOC_COUNT 9 + +/* +** CAPI3REF: Database Connection Status +** +** ^This interface is used to retrieve runtime status information +** about a single [database connection]. ^The first argument is the +** database connection object to be interrogated. ^The second argument +** is an integer constant, taken from the set of +** [SQLITE_DBSTATUS options], that +** determines the parameter to interrogate. The set of +** [SQLITE_DBSTATUS options] is likely +** to grow in future releases of SQLite. +** +** ^The current value of the requested parameter is written into *pCur +** and the highest instantaneous value is written into *pHiwtr. ^If +** the resetFlg is true, then the highest instantaneous value is +** reset back down to the current value. +** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** +** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. +*/ +SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); + +/* +** CAPI3REF: Status Parameters for database connections +** KEYWORDS: {SQLITE_DBSTATUS options} +** +** These constants are the available integer "verbs" that can be passed as +** the second argument to the [sqlite3_db_status()] interface. +** +** New verbs may be added in future releases of SQLite. Existing verbs +** might be discontinued. Applications should check the return code from +** [sqlite3_db_status()] to make sure that the call worked. +** The [sqlite3_db_status()] interface will return a non-zero error code +** if a discontinued or unsupported verb is invoked. +** +**
    +** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(
    SQLITE_DBSTATUS_LOOKASIDE_USED
    +**
    This parameter returns the number of lookaside memory slots currently +** checked out.
    )^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
    SQLITE_DBSTATUS_LOOKASIDE_HIT
    +**
    This parameter returns the number malloc attempts that were +** satisfied using lookaside memory. Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] +** ^(
    SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
    +**
    This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to the amount of +** memory requested being larger than the lookaside slot size. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] +** ^(
    SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
    +**
    This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to all lookaside +** memory already being in use. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
    SQLITE_DBSTATUS_CACHE_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used by all pager caches associated with the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +** +** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** [[SQLITE_DBSTATUS_STMT_USED]] ^(
    SQLITE_DBSTATUS_STMT_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. +**
    +** +** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
    SQLITE_DBSTATUS_CACHE_HIT
    +**
    This parameter returns the number of pager cache hits that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** is always 0. +**
    +** +** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
    SQLITE_DBSTATUS_CACHE_MISS
    +**
    This parameter returns the number of pager cache misses that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** is always 0. +**
    +** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
    SQLITE_DBSTATUS_CACHE_WRITE
    +**
    This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
    +** +** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(
    SQLITE_DBSTATUS_DEFERRED_FKS
    +**
    This parameter returns zero for the current value if and only if +** all foreign key constraints (deferred or immediate) have been +** resolved.)^ ^The highwater mark is always 0. +**
    +**
    +*/ +#define SQLITE_DBSTATUS_LOOKASIDE_USED 0 +#define SQLITE_DBSTATUS_CACHE_USED 1 +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 +#define SQLITE_DBSTATUS_CACHE_HIT 7 +#define SQLITE_DBSTATUS_CACHE_MISS 8 +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_DEFERRED_FKS 10 +#define SQLITE_DBSTATUS_MAX 10 /* Largest defined DBSTATUS */ + + +/* +** CAPI3REF: Prepared Statement Status +** +** ^(Each prepared statement maintains various +** [SQLITE_STMTSTATUS counters] that measure the number +** of times it has performed specific operations.)^ These counters can +** be used to monitor the performance characteristics of the prepared +** statements. For example, if the number of table steps greatly exceeds +** the number of table searches or result rows, that would tend to indicate +** that the prepared statement is using a full table scan rather than +** an index. +** +** ^(This interface is used to retrieve and reset counter values from +** a [prepared statement]. The first argument is the prepared statement +** object to be interrogated. The second argument +** is an integer code for a specific [SQLITE_STMTSTATUS counter] +** to be interrogated.)^ +** ^The current value of the requested counter is returned. +** ^If the resetFlg is true, then the counter is reset to zero after this +** interface call returns. +** +** See also: [sqlite3_status()] and [sqlite3_db_status()]. +*/ +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); + +/* +** CAPI3REF: Status Parameters for prepared statements +** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters} +** +** These preprocessor macros define integer codes that name counter +** values associated with the [sqlite3_stmt_status()] interface. +** The meanings of the various counters are as follows: +** +**
    +** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
    SQLITE_STMTSTATUS_FULLSCAN_STEP
    +**
    ^This is the number of times that SQLite has stepped forward in +** a table as part of a full table scan. Large numbers for this counter +** may indicate opportunities for performance improvement through +** careful use of indices.
    +** +** [[SQLITE_STMTSTATUS_SORT]]
    SQLITE_STMTSTATUS_SORT
    +**
    ^This is the number of sort operations that have occurred. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance through careful use of indices.
    +** +** [[SQLITE_STMTSTATUS_AUTOINDEX]]
    SQLITE_STMTSTATUS_AUTOINDEX
    +**
    ^This is the number of rows inserted into transient indices that +** were created automatically in order to help joins run faster. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance by adding permanent indices that do not +** need to be reinitialized each time the statement is run.
    +** +** [[SQLITE_STMTSTATUS_VM_STEP]]
    SQLITE_STMTSTATUS_VM_STEP
    +**
    ^This is the number of virtual machine operations executed +** by the prepared statement if that number is less than or equal +** to 2147483647. The number of virtual machine operations can be +** used as a proxy for the total work done by the prepared statement. +** If the number of virtual machine operations exceeds 2147483647 +** then the value returned by this statement status code is undefined. +**
    +**
    +*/ +#define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 +#define SQLITE_STMTSTATUS_SORT 2 +#define SQLITE_STMTSTATUS_AUTOINDEX 3 +#define SQLITE_STMTSTATUS_VM_STEP 4 + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache type is opaque. It is implemented by +** the pluggable module. The SQLite core has no knowledge of +** its size or internal structure and never deals with the +** sqlite3_pcache object except by holding and passing pointers +** to the object. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache sqlite3_pcache; + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache_page object represents a single page in the +** page cache. The page cache will allocate instances of this +** object. Various methods of the page cache use pointers to instances +** of this object as parameters or as their return value. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache_page sqlite3_pcache_page; +struct sqlite3_pcache_page { + void *pBuf; /* The content of the page */ + void *pExtra; /* Extra information associated with the page */ +}; + +/* +** CAPI3REF: Application Defined Page Cache. +** KEYWORDS: {page cache} +** +** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can +** register an alternative page cache implementation by passing in an +** instance of the sqlite3_pcache_methods2 structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which +** that memory is allocated and released, and the policies used to +** determine exactly which parts of a database file are cached and for +** how long. +** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** +** ^(The contents of the sqlite3_pcache_methods2 structure are copied to an +** internal buffer by SQLite within the call to [sqlite3_config]. Hence +** the application may discard the parameter after the call to +** [sqlite3_config()] returns.)^ +** +** [[the xInit() page cache method]] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ +** (usually only once during the lifetime of the process). ^(The xInit() +** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^ +** The intent of the xInit() method is to set up global data structures +** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ +** +** [[the xShutdown() page cache method]] +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up +** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. +** +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. All other methods must be threadsafe +** in multithreaded applications. +** +** ^SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +** +** [[the xCreate() page cache methods]] +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, +** though this is not guaranteed. ^The +** first parameter, szPage, is the size in bytes of the pages that must +** be allocated by the cache. ^szPage will always a power of two. ^The +** second parameter szExtra is a number of bytes of extra storage +** associated with each page cache entry. ^The szExtra parameter will +** a number less than 250. SQLite will use the +** extra szExtra bytes on each page to store metadata about the underlying +** database page on disk. The value passed into szExtra depends +** on the SQLite version, the target platform, and how SQLite was compiled. +** ^The third argument to xCreate(), bPurgeable, is true if the cache being +** created will be used to cache database pages of a file stored on disk, or +** false if it is used for an in-memory database. The cache implementation +** does not have to do anything special based with the value of bPurgeable; +** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will +** never invoke xUnpin() except to deliberately delete a page. +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will +** never contain any unpinned pages. +** +** [[the xCachesize() page cache method]] +** ^(The xCachesize() method may be called at any time by SQLite to set the +** suggested maximum cache-size (number of pages stored by) the cache +** instance passed as the first argument. This is the value configured using +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable +** parameter, the implementation is not required to do anything with this +** value; it is advisory only. +** +** [[the xPagecount() page cache methods]] +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. +** +** [[the xFetch() page cache methods]] +** The xFetch() method locates a page in the cache and returns a pointer to +** an sqlite3_pcache_page object associated with that page, or a NULL pointer. +** The pBuf element of the returned sqlite3_pcache_page object will be a +** pointer to a buffer of szPage bytes used to store the content of a +** single database page. The pExtra element of sqlite3_pcache_page will be +** a pointer to the szExtra bytes of extra storage that SQLite has requested +** for each entry in the page cache. +** +** The page to be fetched is determined by the key. ^The minimum key value +** is 1. After it has been retrieved using xFetch, the page is considered +** to be "pinned". +** +** If the requested page is already in the page cache, then the page cache +** implementation must return a pointer to the page buffer with its content +** intact. If the requested page is not already in the cache, then the +** cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: +** +** +**
    createFlag Behavior when page is not already in cache +**
    0 Do not allocate a new page. Return NULL. +**
    1 Allocate a new page if it easy and convenient to do so. +** Otherwise return NULL. +**
    2 Make every effort to allocate a new page. Only return +** NULL if allocating a new page is effectively impossible. +**
    +** +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the to xFetch() calls, SQLite may +** attempt to unpin one or more cache pages by spilling the content of +** pinned pages to disk and synching the operating system disk cache. +** +** [[the xUnpin() page cache method]] +** ^xUnpin() is called by SQLite with a pointer to a currently pinned page +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation +** may choose to evict unpinned pages at any time. +** +** The cache must not perform any reference counting. A single +** call to xUnpin() unpins the page regardless of the number of prior calls +** to xFetch(). +** +** [[the xRekey() page cache methods]] +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be +** discarded. ^Any prior cache entry associated with newKey is guaranteed not +** to be pinned. +** +** When SQLite calls the xTruncate() method, the cache must discard all +** existing cache entries with page numbers (keys) greater than or equal +** to the value of the iLimit parameter passed to xTruncate(). If any +** of these pages are pinned, they are implicitly unpinned, meaning that +** they can be safely discarded. +** +** [[the xDestroy() page cache method]] +** ^The xDestroy() method is used to delete a cache allocated by xCreate(). +** All resources associated with the specified cache should be freed. ^After +** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] +** handle invalid, and will not use it with any other sqlite3_pcache_methods2 +** functions. +** +** [[the xShrink() page cache method]] +** ^SQLite invokes the xShrink() method when it wants the page cache to +** free up as much of heap memory as possible. The page cache implementation +** is not obligated to free any memory, but well-behaved implementations should +** do their best. +*/ +typedef struct sqlite3_pcache_methods2 sqlite3_pcache_methods2; +struct sqlite3_pcache_methods2 { + int iVersion; + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int szExtra, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard); + void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, + unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); + void (*xShrink)(sqlite3_pcache*); +}; + +/* +** This is the obsolete pcache_methods object that has now been replaced +** by sqlite3_pcache_methods2. This object is not used by SQLite. It is +** retained in the header file for backwards compatibility only. +*/ +typedef struct sqlite3_pcache_methods sqlite3_pcache_methods; +struct sqlite3_pcache_methods { + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + void *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, void*, int discard); + void (*xRekey)(sqlite3_pcache*, void*, unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); +}; + + +/* +** CAPI3REF: Online Backup Object +** +** The sqlite3_backup object records state information about an ongoing +** online backup operation. ^The sqlite3_backup object is created by +** a call to [sqlite3_backup_init()] and is destroyed by a call to +** [sqlite3_backup_finish()]. +** +** See Also: [Using the SQLite Online Backup API] +*/ +typedef struct sqlite3_backup sqlite3_backup; + +/* +** CAPI3REF: Online Backup API. +** +** The backup API copies the content of one database into another. +** It is useful either for creating backups of databases or +** for copying in-memory databases to or from persistent files. +** +** See Also: [Using the SQLite Online Backup API] +** +** ^SQLite holds a write transaction open on the destination database file +** for the duration of the backup operation. +** ^The source database is read-locked only while it is being read; +** it is not locked continuously for the entire backup operation. +** ^Thus, the backup may be performed on a live source database without +** preventing other database connections from +** reading or writing to the source database while the backup is underway. +** +** ^(To perform a backup operation: +**
      +**
    1. sqlite3_backup_init() is called once to initialize the +** backup, +**
    2. sqlite3_backup_step() is called one or more times to transfer +** the data between the two databases, and finally +**
    3. sqlite3_backup_finish() is called to release all resources +** associated with the backup operation. +**
    )^ +** There should be exactly one call to sqlite3_backup_finish() for each +** successful call to sqlite3_backup_init(). +** +** [[sqlite3_backup_init()]] sqlite3_backup_init() +** +** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the +** [database connection] associated with the destination database +** and the database name, respectively. +** ^The database name is "main" for the main database, "temp" for the +** temporary database, or the name specified after the AS keyword in +** an [ATTACH] statement for an attached database. +** ^The S and M arguments passed to +** sqlite3_backup_init(D,N,S,M) identify the [database connection] +** and database name of the source database, respectively. +** ^The source and destination [database connections] (parameters S and D) +** must be different or else sqlite3_backup_init(D,N,S,M) will fail with +** an error. +** +** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is +** returned and an error code and error message are stored in the +** destination [database connection] D. +** ^The error code and message for the failed call to sqlite3_backup_init() +** can be retrieved using the [sqlite3_errcode()], [sqlite3_errmsg()], and/or +** [sqlite3_errmsg16()] functions. +** ^A successful call to sqlite3_backup_init() returns a pointer to an +** [sqlite3_backup] object. +** ^The [sqlite3_backup] object may be used with the sqlite3_backup_step() and +** sqlite3_backup_finish() functions to perform the specified backup +** operation. +** +** [[sqlite3_backup_step()]] sqlite3_backup_step() +** +** ^Function sqlite3_backup_step(B,N) will copy up to N pages between +** the source and destination databases specified by [sqlite3_backup] object B. +** ^If N is negative, all remaining source pages are copied. +** ^If sqlite3_backup_step(B,N) successfully copies N pages and there +** are still more pages to be copied, then the function returns [SQLITE_OK]. +** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages +** from source to destination, then it returns [SQLITE_DONE]. +** ^If an error occurs while running sqlite3_backup_step(B,N), +** then an [error code] is returned. ^As well as [SQLITE_OK] and +** [SQLITE_DONE], a call to sqlite3_backup_step() may return [SQLITE_READONLY], +** [SQLITE_NOMEM], [SQLITE_BUSY], [SQLITE_LOCKED], or an +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX] extended error code. +** +** ^(The sqlite3_backup_step() might return [SQLITE_READONLY] if +**
      +**
    1. the destination database was opened read-only, or +**
    2. the destination database is using write-ahead-log journaling +** and the destination and source page sizes differ, or +**
    3. the destination database is an in-memory database and the +** destination and source page sizes differ. +**
    )^ +** +** ^If sqlite3_backup_step() cannot obtain a required file-system lock, then +** the [sqlite3_busy_handler | busy-handler function] +** is invoked (if one is specified). ^If the +** busy-handler returns non-zero before the lock is available, then +** [SQLITE_BUSY] is returned to the caller. ^In this case the call to +** sqlite3_backup_step() can be retried later. ^If the source +** [database connection] +** is being used to write to the source database when sqlite3_backup_step() +** is called, then [SQLITE_LOCKED] is returned immediately. ^Again, in this +** case the call to sqlite3_backup_step() can be retried later on. ^(If +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX], [SQLITE_NOMEM], or +** [SQLITE_READONLY] is returned, then +** there is no point in retrying the call to sqlite3_backup_step(). These +** errors are considered fatal.)^ The application must accept +** that the backup operation has failed and pass the backup operation handle +** to the sqlite3_backup_finish() to release associated resources. +** +** ^The first call to sqlite3_backup_step() obtains an exclusive lock +** on the destination file. ^The exclusive lock is not released until either +** sqlite3_backup_finish() is called or the backup operation is complete +** and sqlite3_backup_step() returns [SQLITE_DONE]. ^Every call to +** sqlite3_backup_step() obtains a [shared lock] on the source database that +** lasts for the duration of the sqlite3_backup_step() call. +** ^Because the source database is not locked between calls to +** sqlite3_backup_step(), the source database may be modified mid-way +** through the backup process. ^If the source database is modified by an +** external process or via a database connection other than the one being +** used by the backup operation, then the backup will be automatically +** restarted by the next call to sqlite3_backup_step(). ^If the source +** database is modified by the using the same database connection as is used +** by the backup operation, then the backup database is automatically +** updated at the same time. +** +** [[sqlite3_backup_finish()]] sqlite3_backup_finish() +** +** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the +** application wishes to abandon the backup operation, the application +** should destroy the [sqlite3_backup] by passing it to sqlite3_backup_finish(). +** ^The sqlite3_backup_finish() interfaces releases all +** resources associated with the [sqlite3_backup] object. +** ^If sqlite3_backup_step() has not yet returned [SQLITE_DONE], then any +** active write-transaction on the destination database is rolled back. +** The [sqlite3_backup] object is invalid +** and may not be used following a call to sqlite3_backup_finish(). +** +** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no +** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() completed. +** ^If an out-of-memory condition or IO error occurred during any prior +** sqlite3_backup_step() call on the same [sqlite3_backup] object, then +** sqlite3_backup_finish() returns the corresponding [error code]. +** +** ^A return of [SQLITE_BUSY] or [SQLITE_LOCKED] from sqlite3_backup_step() +** is not a permanent error and does not affect the return value of +** sqlite3_backup_finish(). +** +** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]] +** sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** +** ^Each call to sqlite3_backup_step() sets two values inside +** the [sqlite3_backup] object: the number of pages still to be backed +** up and the total number of pages in the source database file. +** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces +** retrieve these two values, respectively. +** +** ^The values returned by these functions are only updated by +** sqlite3_backup_step(). ^If the source database is modified during a backup +** operation, then the values are not updated to account for any extra +** pages that need to be updated or the size of the source database file +** changing. +** +** Concurrent Usage of Database Handles +** +** ^The source [database connection] may be used by the application for other +** purposes while a backup operation is underway or being initialized. +** ^If SQLite is compiled and configured to support threadsafe database +** connections, then the source database connection may be used concurrently +** from within other threads. +** +** However, the application must guarantee that the destination +** [database connection] is not passed to any other API (by any thread) after +** sqlite3_backup_init() is called and before the corresponding call to +** sqlite3_backup_finish(). SQLite does not currently check to see +** if the application incorrectly accesses the destination [database connection] +** and so no error code is reported, but the operations may malfunction +** nevertheless. Use of the destination database connection while a +** backup is in progress might also also cause a mutex deadlock. +** +** If running in [shared cache mode], the application must +** guarantee that the shared cache used by the destination database +** is not accessed while the backup is running. In practice this means +** that the application must guarantee that the disk file being +** backed up to is not accessed by any connection within the process, +** not just the specific connection that was passed to sqlite3_backup_init(). +** +** The [sqlite3_backup] object itself is partially threadsafe. Multiple +** threads may safely make multiple concurrent calls to sqlite3_backup_step(). +** However, the sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** APIs are not strictly speaking threadsafe. If they are invoked at the +** same time as another thread is invoking sqlite3_backup_step() it is +** possible that they return invalid values. +*/ +SQLITE_API sqlite3_backup *sqlite3_backup_init( + sqlite3 *pDest, /* Destination database handle */ + const char *zDestName, /* Destination database name */ + sqlite3 *pSource, /* Source database handle */ + const char *zSourceName /* Source database name */ +); +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage); +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); + +/* +** CAPI3REF: Unlock Notification +** +** ^When running in shared-cache mode, a database operation may fail with +** an [SQLITE_LOCKED] error if the required locks on the shared-cache or +** individual tables within the shared-cache cannot be obtained. See +** [SQLite Shared-Cache Mode] for a description of shared-cache locking. +** ^This API may be used to register a callback that SQLite will invoke +** when the connection currently holding the required lock relinquishes it. +** ^This API is only available if the library was compiled with the +** [SQLITE_ENABLE_UNLOCK_NOTIFY] C-preprocessor symbol defined. +** +** See Also: [Using the SQLite Unlock Notification Feature]. +** +** ^Shared-cache locks are released when a database connection concludes +** its current transaction, either by committing it or rolling it back. +** +** ^When a connection (known as the blocked connection) fails to obtain a +** shared-cache lock and SQLITE_LOCKED is returned to the caller, the +** identity of the database connection (the blocking connection) that +** has locked the required resource is stored internally. ^After an +** application receives an SQLITE_LOCKED error, it may call the +** sqlite3_unlock_notify() method with the blocked connection handle as +** the first argument to register for a callback that will be invoked +** when the blocking connections current transaction is concluded. ^The +** callback is invoked from within the [sqlite3_step] or [sqlite3_close] +** call that concludes the blocking connections transaction. +** +** ^(If sqlite3_unlock_notify() is called in a multi-threaded application, +** there is a chance that the blocking connection will have already +** concluded its transaction by the time sqlite3_unlock_notify() is invoked. +** If this happens, then the specified callback is invoked immediately, +** from within the call to sqlite3_unlock_notify().)^ +** +** ^If the blocked connection is attempting to obtain a write-lock on a +** shared-cache table, and more than one other connection currently holds +** a read-lock on the same table, then SQLite arbitrarily selects one of +** the other connections to use as the blocking connection. +** +** ^(There may be at most one unlock-notify callback registered by a +** blocked connection. If sqlite3_unlock_notify() is called when the +** blocked connection already has a registered unlock-notify callback, +** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is +** called with a NULL pointer as its second argument, then any existing +** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback may also be canceled by closing the blocked +** connection using [sqlite3_close()]. +** +** The unlock-notify callback is not reentrant. If an application invokes +** any sqlite3_xxx API functions from within an unlock-notify callback, a +** crash or deadlock may be the result. +** +** ^Unless deadlock is detected (see below), sqlite3_unlock_notify() always +** returns SQLITE_OK. +** +** Callback Invocation Details +** +** When an unlock-notify callback is registered, the application provides a +** single void* pointer that is passed to the callback when it is invoked. +** However, the signature of the callback function allows SQLite to pass +** it an array of void* context pointers. The first argument passed to +** an unlock-notify callback is a pointer to an array of void* pointers, +** and the second is the number of entries in the array. +** +** When a blocking connections transaction is concluded, there may be +** more than one blocked connection that has registered for an unlock-notify +** callback. ^If two or more such blocked connections have specified the +** same callback function, then instead of invoking the callback function +** multiple times, it is invoked once with the set of void* context pointers +** specified by the blocked connections bundled together into an array. +** This gives the application an opportunity to prioritize any actions +** related to the set of unblocked database connections. +** +** Deadlock Detection +** +** Assuming that after registering for an unlock-notify callback a +** database waits for the callback to be issued before taking any further +** action (a reasonable assumption), then using this API may cause the +** application to deadlock. For example, if connection X is waiting for +** connection Y's transaction to be concluded, and similarly connection +** Y is waiting on connection X's transaction, then neither connection +** will proceed and the system may remain deadlocked indefinitely. +** +** To avoid this scenario, the sqlite3_unlock_notify() performs deadlock +** detection. ^If a given call to sqlite3_unlock_notify() would put the +** system in a deadlocked state, then SQLITE_LOCKED is returned and no +** unlock-notify callback is registered. The system is said to be in +** a deadlocked state if connection A has registered for an unlock-notify +** callback on the conclusion of connection B's transaction, and connection +** B has itself registered for an unlock-notify callback when connection +** A's transaction is concluded. ^Indirect deadlock is also detected, so +** the system is also considered to be deadlocked if connection B has +** registered for an unlock-notify callback on the conclusion of connection +** C's transaction, where connection C is waiting on connection A. ^Any +** number of levels of indirection are allowed. +** +** The "DROP TABLE" Exception +** +** When a call to [sqlite3_step()] returns SQLITE_LOCKED, it is almost +** always appropriate to call sqlite3_unlock_notify(). There is however, +** one exception. When executing a "DROP TABLE" or "DROP INDEX" statement, +** SQLite checks if there are any currently executing SELECT statements +** that belong to the same connection. If there are, SQLITE_LOCKED is +** returned. In this case there is no "blocking connection", so invoking +** sqlite3_unlock_notify() results in the unlock-notify callback being +** invoked immediately. If the application then re-attempts the "DROP TABLE" +** or "DROP INDEX" query, an infinite loop might be the result. +** +** One way around this problem is to check the extended error code returned +** by an sqlite3_step() call. ^(If there is a blocking connection, then the +** extended error code is set to SQLITE_LOCKED_SHAREDCACHE. Otherwise, in +** the special "DROP TABLE/INDEX" case, the extended error code is just +** SQLITE_LOCKED.)^ +*/ +SQLITE_API int sqlite3_unlock_notify( + sqlite3 *pBlocked, /* Waiting connection */ + void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ + void *pNotifyArg /* Argument to pass to xNotify */ +); + + +/* +** CAPI3REF: String Comparison +** +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. +*/ +SQLITE_API int sqlite3_stricmp(const char *, const char *); +SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); + +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); + +/* +** CAPI3REF: Error Logging Interface +** +** ^The [sqlite3_log()] interface writes a message into the [error log] +** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. +** ^If logging is enabled, the zFormat string and subsequent arguments are +** used with [sqlite3_snprintf()] to generate the final output string. +** +** The sqlite3_log() interface is intended for use by extensions such as +** virtual tables, collating functions, and SQL functions. While there is +** nothing to prevent an application from calling sqlite3_log(), doing so +** is considered bad form. +** +** The zFormat string must not be NULL. +** +** To avoid deadlocks and other threading problems, the sqlite3_log() routine +** will not use dynamically allocated memory. The log message is stored in +** a fixed-length buffer on the stack. If the log message is longer than +** a few hundred characters, it will be truncated to the length of the +** buffer. +*/ +SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); + +/* +** CAPI3REF: Write-Ahead Log Commit Hook +** +** ^The [sqlite3_wal_hook()] function is used to register a callback that +** will be invoked each time a database connection commits data to a +** [write-ahead log] (i.e. whenever a transaction is committed in +** [journal_mode | journal_mode=WAL mode]). +** +** ^The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released, so the implementation +** may read, write or [checkpoint] the database as required. +** +** ^The first parameter passed to the callback function when it is invoked +** is a copy of the third parameter passed to sqlite3_wal_hook() when +** registering the callback. ^The second is a copy of the database handle. +** ^The third parameter is the name of the database that was written to - +** either "main" or the name of an [ATTACH]-ed database. ^The fourth parameter +** is the number of pages currently in the write-ahead log file, +** including those that were just committed. +** +** The callback function should normally return [SQLITE_OK]. ^If an error +** code is returned, that error will propagate back up through the +** SQLite code base to cause the statement that provoked the callback +** to report an error, though the commit will have still occurred. If the +** callback returns [SQLITE_ROW] or [SQLITE_DONE], or if it returns a value +** that does not correspond to any valid SQLite error code, the results +** are undefined. +** +** A single database handle may have at most a single write-ahead log callback +** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any +** previously registered write-ahead log callback. ^Note that the +** [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will +** those overwrite any prior [sqlite3_wal_hook()] settings. +*/ +SQLITE_API void *sqlite3_wal_hook( + sqlite3*, + int(*)(void *,sqlite3*,const char*,int), + void* +); + +/* +** CAPI3REF: Configure an auto-checkpoint +** +** ^The [sqlite3_wal_autocheckpoint(D,N)] is a wrapper around +** [sqlite3_wal_hook()] that causes any database on [database connection] D +** to automatically [checkpoint] +** after committing a transaction if there are N or +** more frames in the [write-ahead log] file. ^Passing zero or +** a negative value as the nFrame parameter disables automatic +** checkpoints entirely. +** +** ^The callback registered by this function replaces any existing callback +** registered using [sqlite3_wal_hook()]. ^Likewise, registering a callback +** using [sqlite3_wal_hook()] disables the automatic checkpoint mechanism +** configured by this function. +** +** ^The [wal_autocheckpoint pragma] can be used to invoke this interface +** from SQL. +** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** +** ^Every new [database connection] defaults to having the auto-checkpoint +** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] +** pages. The use of this interface +** is only necessary if the default setting is found to be suboptimal +** for a particular application. +*/ +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); + +/* +** CAPI3REF: Checkpoint a database +** +** ^The [sqlite3_wal_checkpoint(D,X)] interface causes database named X +** on [database connection] D to be [checkpointed]. ^If X is NULL or an +** empty string, then a checkpoint is run on all databases of +** connection D. ^If the database connection D is not in +** [WAL | write-ahead log mode] then this interface is a harmless no-op. +** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a +** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. +** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL +** or RESET checkpoint. +** +** ^The [wal_checkpoint pragma] can be used to invoke this interface +** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] can be used to cause this interface to be +** run whenever the WAL reaches a certain size threshold. +** +** See also: [sqlite3_wal_checkpoint_v2()] +*/ +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Checkpoint a database +** +** Run a checkpoint operation on WAL database zDb attached to database +** handle db. The specific operation is determined by the value of the +** eMode parameter: +** +**
    +**
    SQLITE_CHECKPOINT_PASSIVE
    +** Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish. Sync the db file if all frames in the log +** are checkpointed. This mode is the same as calling +** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] +** is never invoked. +** +**
    SQLITE_CHECKPOINT_FULL
    +** This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no +** database writer and all readers are reading from the most recent database +** snapshot. It then checkpoints all frames in the log file and syncs the +** database file. This call blocks database writers while it is running, +** but not database readers. +** +**
    SQLITE_CHECKPOINT_RESTART
    +** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after +** checkpointing the log file it blocks (calls the +** [sqlite3_busy_handler|busy-handler callback]) +** until all readers are reading from the database file only. This ensures +** that the next client to write to the database file restarts the log file +** from the beginning. This call blocks database writers while it is running, +** but not database readers. +**
    +** +** If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to +** the total number of checkpointed frames (including any that were already +** checkpointed when this function is called). *pnLog and *pnCkpt may be +** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. +** If no values are available because of an error, they are both set to -1 +** before returning to communicate this to the caller. +** +** All calls obtain an exclusive "checkpoint" lock on the database file. If +** any other process is running a checkpoint operation at the same time, the +** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** busy-handler configured, it will not be invoked in this case. +** +** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive +** "writer" lock on the database file. If the writer lock cannot be obtained +** immediately, and a busy-handler is configured, it is invoked and the writer +** lock retried until either the busy-handler returns 0 or the lock is +** successfully obtained. The busy-handler is also invoked while waiting for +** database readers as described above. If the busy-handler returns 0 before +** the writer lock is obtained or while waiting for database readers, the +** checkpoint operation proceeds from that point in the same way as +** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible +** without blocking any further. SQLITE_BUSY is returned in this case. +** +** If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. If +** an SQLITE_BUSY error is encountered when processing one or more of the +** attached WAL databases, the operation is still attempted on any remaining +** attached databases and SQLITE_BUSY is returned to the caller. If any other +** error occurs while processing an attached database, processing is abandoned +** and the error code returned to the caller immediately. If no error +** (SQLITE_BUSY or otherwise) is encountered while processing the attached +** databases, SQLITE_OK is returned. +** +** If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** zDb is not NULL (or a zero length string) and is not the name of any +** attached database, SQLITE_ERROR is returned to the caller. +*/ +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +); + +/* +** CAPI3REF: Checkpoint operation parameters +** +** These constants can be used as the 3rd parameter to +** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] +** documentation for additional information about the meaning and use of +** each of these values. +*/ +#define SQLITE_CHECKPOINT_PASSIVE 0 +#define SQLITE_CHECKPOINT_FULL 1 +#define SQLITE_CHECKPOINT_RESTART 2 + +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function may be called by either the [xConnect] or [xCreate] method +** of a [virtual table] implementation to configure +** various facets of the virtual table interface. +** +** If this interface is invoked outside the context of an xConnect or +** xCreate virtual table method then the behavior is undefined. +** +** At present, there is only one option that may be configured using +** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options +** may be added in the future. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Virtual Table Configuration Options +** +** These macros define the various options to the +** [sqlite3_vtab_config()] interface that [virtual table] implementations +** can use to customize and optimize their behavior. +** +**
    +**
    SQLITE_VTAB_CONSTRAINT_SUPPORT +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, +** where X is an integer. If X is zero, then the [virtual table] whose +** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not +** support constraints. In this configuration (which is the default) if +** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If X is non-zero, then the virtual table implementation guarantees +** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before +** any modifications to internal or persistent data structures have been made. +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** is able to roll back a statement or database transaction, and abandon +** or continue processing the current SQL statement as appropriate. +** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns +** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode +** had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
    +*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the [xUpdate] method +** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The +** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL], +** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode +** of the SQL statement that triggered the call to the [xUpdate] method of the +** [virtual table]. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Conflict resolution modes +** KEYWORDS: {conflict resolution mode} +** +** These constants are returned by [sqlite3_vtab_on_conflict()] to +** inform a [virtual table] implementation what the [ON CONFLICT] mode +** is for the SQL statement being evaluated. +** +** Note that the [SQLITE_IGNORE] constant is also used as a potential +** return value from the [sqlite3_set_authorizer()] callback and that +** [SQLITE_ABORT] is also a [result code]. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 // Also an error code */ +#define SQLITE_REPLACE 5 + + + +/* +** Undo the hack that converts floating point types to integer for +** builds on processors without floating point support. +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# undef double +#endif + +#ifdef __cplusplus +} /* End of the 'extern "C"' block */ +#endif +#endif /* _SQLITE3_H_ */ + +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visiblity */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ + diff --git a/Postbox/PostboxPrivate/sqlcipher/sqlite3ext.h b/Postbox/PostboxPrivate/sqlcipher/sqlite3ext.h new file mode 100644 index 0000000000..ecf93f62f6 --- /dev/null +++ b/Postbox/PostboxPrivate/sqlcipher/sqlite3ext.h @@ -0,0 +1,487 @@ +/* +** 2006 June 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the SQLite interface for use by +** shared libraries that want to be imported as extensions into +** an SQLite instance. Shared libraries that intend to be loaded +** as extensions by SQLite should #include this file instead of +** sqlite3.h. +*/ +#ifndef _SQLITE3EXT_H_ +#define _SQLITE3EXT_H_ +#include "sqlite3.h" + +typedef struct sqlite3_api_routines sqlite3_api_routines; + +/* +** The following structure holds pointers to all of the SQLite API +** routines. +** +** WARNING: In order to maintain backwards compatibility, add new +** interfaces to the end of this structure only. If you insert new +** interfaces in the middle of this structure, then older different +** versions of SQLite will not be able to load each others' shared +** libraries! +*/ +struct sqlite3_api_routines { + void * (*aggregate_context)(sqlite3_context*,int nBytes); + int (*aggregate_count)(sqlite3_context*); + int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*)); + int (*bind_double)(sqlite3_stmt*,int,double); + int (*bind_int)(sqlite3_stmt*,int,int); + int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64); + int (*bind_null)(sqlite3_stmt*,int); + int (*bind_parameter_count)(sqlite3_stmt*); + int (*bind_parameter_index)(sqlite3_stmt*,const char*zName); + const char * (*bind_parameter_name)(sqlite3_stmt*,int); + int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*)); + int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*)); + int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*); + int (*busy_handler)(sqlite3*,int(*)(void*,int),void*); + int (*busy_timeout)(sqlite3*,int ms); + int (*changes)(sqlite3*); + int (*close)(sqlite3*); + int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const char*)); + int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const void*)); + const void * (*column_blob)(sqlite3_stmt*,int iCol); + int (*column_bytes)(sqlite3_stmt*,int iCol); + int (*column_bytes16)(sqlite3_stmt*,int iCol); + int (*column_count)(sqlite3_stmt*pStmt); + const char * (*column_database_name)(sqlite3_stmt*,int); + const void * (*column_database_name16)(sqlite3_stmt*,int); + const char * (*column_decltype)(sqlite3_stmt*,int i); + const void * (*column_decltype16)(sqlite3_stmt*,int); + double (*column_double)(sqlite3_stmt*,int iCol); + int (*column_int)(sqlite3_stmt*,int iCol); + sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol); + const char * (*column_name)(sqlite3_stmt*,int); + const void * (*column_name16)(sqlite3_stmt*,int); + const char * (*column_origin_name)(sqlite3_stmt*,int); + const void * (*column_origin_name16)(sqlite3_stmt*,int); + const char * (*column_table_name)(sqlite3_stmt*,int); + const void * (*column_table_name16)(sqlite3_stmt*,int); + const unsigned char * (*column_text)(sqlite3_stmt*,int iCol); + const void * (*column_text16)(sqlite3_stmt*,int iCol); + int (*column_type)(sqlite3_stmt*,int iCol); + sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol); + void * (*commit_hook)(sqlite3*,int(*)(void*),void*); + int (*complete)(const char*sql); + int (*complete16)(const void*sql); + int (*create_collation)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_collation16)(sqlite3*,const void*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_function)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_function16)(sqlite3*,const void*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); + int (*data_count)(sqlite3_stmt*pStmt); + sqlite3 * (*db_handle)(sqlite3_stmt*); + int (*declare_vtab)(sqlite3*,const char*); + int (*enable_shared_cache)(int); + int (*errcode)(sqlite3*db); + const char * (*errmsg)(sqlite3*); + const void * (*errmsg16)(sqlite3*); + int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**); + int (*expired)(sqlite3_stmt*); + int (*finalize)(sqlite3_stmt*pStmt); + void (*free)(void*); + void (*free_table)(char**result); + int (*get_autocommit)(sqlite3*); + void * (*get_auxdata)(sqlite3_context*,int); + int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**); + int (*global_recover)(void); + void (*interruptx)(sqlite3*); + sqlite_int64 (*last_insert_rowid)(sqlite3*); + const char * (*libversion)(void); + int (*libversion_number)(void); + void *(*malloc)(int); + char * (*mprintf)(const char*,...); + int (*open)(const char*,sqlite3**); + int (*open16)(const void*,sqlite3**); + int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*); + void (*progress_handler)(sqlite3*,int,int(*)(void*),void*); + void *(*realloc)(void*,int); + int (*reset)(sqlite3_stmt*pStmt); + void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_double)(sqlite3_context*,double); + void (*result_error)(sqlite3_context*,const char*,int); + void (*result_error16)(sqlite3_context*,const void*,int); + void (*result_int)(sqlite3_context*,int); + void (*result_int64)(sqlite3_context*,sqlite_int64); + void (*result_null)(sqlite3_context*); + void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*)); + void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_value)(sqlite3_context*,sqlite3_value*); + void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); + int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, + const char*,const char*),void*); + void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); + char * (*snprintf)(int,char*,const char*,...); + int (*step)(sqlite3_stmt*); + int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, + char const**,char const**,int*,int*,int*); + void (*thread_cleanup)(void); + int (*total_changes)(sqlite3*); + void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); + int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); + void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*, + sqlite_int64),void*); + void * (*user_data)(sqlite3_context*); + const void * (*value_blob)(sqlite3_value*); + int (*value_bytes)(sqlite3_value*); + int (*value_bytes16)(sqlite3_value*); + double (*value_double)(sqlite3_value*); + int (*value_int)(sqlite3_value*); + sqlite_int64 (*value_int64)(sqlite3_value*); + int (*value_numeric_type)(sqlite3_value*); + const unsigned char * (*value_text)(sqlite3_value*); + const void * (*value_text16)(sqlite3_value*); + const void * (*value_text16be)(sqlite3_value*); + const void * (*value_text16le)(sqlite3_value*); + int (*value_type)(sqlite3_value*); + char *(*vmprintf)(const char*,va_list); + /* Added ??? */ + int (*overload_function)(sqlite3*, const char *zFuncName, int nArg); + /* Added by 3.3.13 */ + int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + int (*clear_bindings)(sqlite3_stmt*); + /* Added by 3.4.1 */ + int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*, + void (*xDestroy)(void *)); + /* Added by 3.5.0 */ + int (*bind_zeroblob)(sqlite3_stmt*,int,int); + int (*blob_bytes)(sqlite3_blob*); + int (*blob_close)(sqlite3_blob*); + int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64, + int,sqlite3_blob**); + int (*blob_read)(sqlite3_blob*,void*,int,int); + int (*blob_write)(sqlite3_blob*,const void*,int,int); + int (*create_collation_v2)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*), + void(*)(void*)); + int (*file_control)(sqlite3*,const char*,int,void*); + sqlite3_int64 (*memory_highwater)(int); + sqlite3_int64 (*memory_used)(void); + sqlite3_mutex *(*mutex_alloc)(int); + void (*mutex_enter)(sqlite3_mutex*); + void (*mutex_free)(sqlite3_mutex*); + void (*mutex_leave)(sqlite3_mutex*); + int (*mutex_try)(sqlite3_mutex*); + int (*open_v2)(const char*,sqlite3**,int,const char*); + int (*release_memory)(int); + void (*result_error_nomem)(sqlite3_context*); + void (*result_error_toobig)(sqlite3_context*); + int (*sleep)(int); + void (*soft_heap_limit)(int); + sqlite3_vfs *(*vfs_find)(const char*); + int (*vfs_register)(sqlite3_vfs*,int); + int (*vfs_unregister)(sqlite3_vfs*); + int (*xthreadsafe)(void); + void (*result_zeroblob)(sqlite3_context*,int); + void (*result_error_code)(sqlite3_context*,int); + int (*test_control)(int, ...); + void (*randomness)(int,void*); + sqlite3 *(*context_db_handle)(sqlite3_context*); + int (*extended_result_codes)(sqlite3*,int); + int (*limit)(sqlite3*,int,int); + sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); + const char *(*sql)(sqlite3_stmt*); + int (*status)(int,int*,int*,int); + int (*backup_finish)(sqlite3_backup*); + sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); + int (*backup_pagecount)(sqlite3_backup*); + int (*backup_remaining)(sqlite3_backup*); + int (*backup_step)(sqlite3_backup*,int); + const char *(*compileoption_get)(int); + int (*compileoption_used)(const char*); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*)); + int (*db_config)(sqlite3*,int,...); + sqlite3_mutex *(*db_mutex)(sqlite3*); + int (*db_status)(sqlite3*,int,int*,int*,int); + int (*extended_errcode)(sqlite3*); + void (*log)(int,const char*,...); + sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); + const char *(*sourceid)(void); + int (*stmt_status)(sqlite3_stmt*,int,int); + int (*strnicmp)(const char*,const char*,int); + int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); + int (*wal_autocheckpoint)(sqlite3*,int); + int (*wal_checkpoint)(sqlite3*,const char*); + void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); + int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); + int (*vtab_config)(sqlite3*,int op,...); + int (*vtab_on_conflict)(sqlite3*); + /* Version 3.7.16 and later */ + int (*close_v2)(sqlite3*); + const char *(*db_filename)(sqlite3*,const char*); + int (*db_readonly)(sqlite3*,const char*); + int (*db_release_memory)(sqlite3*); + const char *(*errstr)(int); + int (*stmt_busy)(sqlite3_stmt*); + int (*stmt_readonly)(sqlite3_stmt*); + int (*stricmp)(const char*,const char*); + int (*uri_boolean)(const char*,const char*,int); + sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); + const char *(*uri_parameter)(const char*,const char*); + char *(*vsnprintf)(int,char*,const char*,va_list); + int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); +}; + +/* +** The following macros redefine the API routines so that they are +** redirected throught the global sqlite3_api structure. +** +** This header file is also used by the loadext.c source file +** (part of the main SQLite library - not an extension) so that +** it can get access to the sqlite3_api_routines structure +** definition. But the main library does not want to redefine +** the API. So the redefinition macros are only valid if the +** SQLITE_CORE macros is undefined. +*/ +#ifndef SQLITE_CORE +#define sqlite3_aggregate_context sqlite3_api->aggregate_context +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_aggregate_count sqlite3_api->aggregate_count +#endif +#define sqlite3_bind_blob sqlite3_api->bind_blob +#define sqlite3_bind_double sqlite3_api->bind_double +#define sqlite3_bind_int sqlite3_api->bind_int +#define sqlite3_bind_int64 sqlite3_api->bind_int64 +#define sqlite3_bind_null sqlite3_api->bind_null +#define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count +#define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index +#define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name +#define sqlite3_bind_text sqlite3_api->bind_text +#define sqlite3_bind_text16 sqlite3_api->bind_text16 +#define sqlite3_bind_value sqlite3_api->bind_value +#define sqlite3_busy_handler sqlite3_api->busy_handler +#define sqlite3_busy_timeout sqlite3_api->busy_timeout +#define sqlite3_changes sqlite3_api->changes +#define sqlite3_close sqlite3_api->close +#define sqlite3_collation_needed sqlite3_api->collation_needed +#define sqlite3_collation_needed16 sqlite3_api->collation_needed16 +#define sqlite3_column_blob sqlite3_api->column_blob +#define sqlite3_column_bytes sqlite3_api->column_bytes +#define sqlite3_column_bytes16 sqlite3_api->column_bytes16 +#define sqlite3_column_count sqlite3_api->column_count +#define sqlite3_column_database_name sqlite3_api->column_database_name +#define sqlite3_column_database_name16 sqlite3_api->column_database_name16 +#define sqlite3_column_decltype sqlite3_api->column_decltype +#define sqlite3_column_decltype16 sqlite3_api->column_decltype16 +#define sqlite3_column_double sqlite3_api->column_double +#define sqlite3_column_int sqlite3_api->column_int +#define sqlite3_column_int64 sqlite3_api->column_int64 +#define sqlite3_column_name sqlite3_api->column_name +#define sqlite3_column_name16 sqlite3_api->column_name16 +#define sqlite3_column_origin_name sqlite3_api->column_origin_name +#define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16 +#define sqlite3_column_table_name sqlite3_api->column_table_name +#define sqlite3_column_table_name16 sqlite3_api->column_table_name16 +#define sqlite3_column_text sqlite3_api->column_text +#define sqlite3_column_text16 sqlite3_api->column_text16 +#define sqlite3_column_type sqlite3_api->column_type +#define sqlite3_column_value sqlite3_api->column_value +#define sqlite3_commit_hook sqlite3_api->commit_hook +#define sqlite3_complete sqlite3_api->complete +#define sqlite3_complete16 sqlite3_api->complete16 +#define sqlite3_create_collation sqlite3_api->create_collation +#define sqlite3_create_collation16 sqlite3_api->create_collation16 +#define sqlite3_create_function sqlite3_api->create_function +#define sqlite3_create_function16 sqlite3_api->create_function16 +#define sqlite3_create_module sqlite3_api->create_module +#define sqlite3_create_module_v2 sqlite3_api->create_module_v2 +#define sqlite3_data_count sqlite3_api->data_count +#define sqlite3_db_handle sqlite3_api->db_handle +#define sqlite3_declare_vtab sqlite3_api->declare_vtab +#define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache +#define sqlite3_errcode sqlite3_api->errcode +#define sqlite3_errmsg sqlite3_api->errmsg +#define sqlite3_errmsg16 sqlite3_api->errmsg16 +#define sqlite3_exec sqlite3_api->exec +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_expired sqlite3_api->expired +#endif +#define sqlite3_finalize sqlite3_api->finalize +#define sqlite3_free sqlite3_api->free +#define sqlite3_free_table sqlite3_api->free_table +#define sqlite3_get_autocommit sqlite3_api->get_autocommit +#define sqlite3_get_auxdata sqlite3_api->get_auxdata +#define sqlite3_get_table sqlite3_api->get_table +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_global_recover sqlite3_api->global_recover +#endif +#define sqlite3_interrupt sqlite3_api->interruptx +#define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid +#define sqlite3_libversion sqlite3_api->libversion +#define sqlite3_libversion_number sqlite3_api->libversion_number +#define sqlite3_malloc sqlite3_api->malloc +#define sqlite3_mprintf sqlite3_api->mprintf +#define sqlite3_open sqlite3_api->open +#define sqlite3_open16 sqlite3_api->open16 +#define sqlite3_prepare sqlite3_api->prepare +#define sqlite3_prepare16 sqlite3_api->prepare16 +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_profile sqlite3_api->profile +#define sqlite3_progress_handler sqlite3_api->progress_handler +#define sqlite3_realloc sqlite3_api->realloc +#define sqlite3_reset sqlite3_api->reset +#define sqlite3_result_blob sqlite3_api->result_blob +#define sqlite3_result_double sqlite3_api->result_double +#define sqlite3_result_error sqlite3_api->result_error +#define sqlite3_result_error16 sqlite3_api->result_error16 +#define sqlite3_result_int sqlite3_api->result_int +#define sqlite3_result_int64 sqlite3_api->result_int64 +#define sqlite3_result_null sqlite3_api->result_null +#define sqlite3_result_text sqlite3_api->result_text +#define sqlite3_result_text16 sqlite3_api->result_text16 +#define sqlite3_result_text16be sqlite3_api->result_text16be +#define sqlite3_result_text16le sqlite3_api->result_text16le +#define sqlite3_result_value sqlite3_api->result_value +#define sqlite3_rollback_hook sqlite3_api->rollback_hook +#define sqlite3_set_authorizer sqlite3_api->set_authorizer +#define sqlite3_set_auxdata sqlite3_api->set_auxdata +#define sqlite3_snprintf sqlite3_api->snprintf +#define sqlite3_step sqlite3_api->step +#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata +#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup +#define sqlite3_total_changes sqlite3_api->total_changes +#define sqlite3_trace sqlite3_api->trace +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_transfer_bindings sqlite3_api->transfer_bindings +#endif +#define sqlite3_update_hook sqlite3_api->update_hook +#define sqlite3_user_data sqlite3_api->user_data +#define sqlite3_value_blob sqlite3_api->value_blob +#define sqlite3_value_bytes sqlite3_api->value_bytes +#define sqlite3_value_bytes16 sqlite3_api->value_bytes16 +#define sqlite3_value_double sqlite3_api->value_double +#define sqlite3_value_int sqlite3_api->value_int +#define sqlite3_value_int64 sqlite3_api->value_int64 +#define sqlite3_value_numeric_type sqlite3_api->value_numeric_type +#define sqlite3_value_text sqlite3_api->value_text +#define sqlite3_value_text16 sqlite3_api->value_text16 +#define sqlite3_value_text16be sqlite3_api->value_text16be +#define sqlite3_value_text16le sqlite3_api->value_text16le +#define sqlite3_value_type sqlite3_api->value_type +#define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_overload_function sqlite3_api->overload_function +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_clear_bindings sqlite3_api->clear_bindings +#define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob +#define sqlite3_blob_bytes sqlite3_api->blob_bytes +#define sqlite3_blob_close sqlite3_api->blob_close +#define sqlite3_blob_open sqlite3_api->blob_open +#define sqlite3_blob_read sqlite3_api->blob_read +#define sqlite3_blob_write sqlite3_api->blob_write +#define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2 +#define sqlite3_file_control sqlite3_api->file_control +#define sqlite3_memory_highwater sqlite3_api->memory_highwater +#define sqlite3_memory_used sqlite3_api->memory_used +#define sqlite3_mutex_alloc sqlite3_api->mutex_alloc +#define sqlite3_mutex_enter sqlite3_api->mutex_enter +#define sqlite3_mutex_free sqlite3_api->mutex_free +#define sqlite3_mutex_leave sqlite3_api->mutex_leave +#define sqlite3_mutex_try sqlite3_api->mutex_try +#define sqlite3_open_v2 sqlite3_api->open_v2 +#define sqlite3_release_memory sqlite3_api->release_memory +#define sqlite3_result_error_nomem sqlite3_api->result_error_nomem +#define sqlite3_result_error_toobig sqlite3_api->result_error_toobig +#define sqlite3_sleep sqlite3_api->sleep +#define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit +#define sqlite3_vfs_find sqlite3_api->vfs_find +#define sqlite3_vfs_register sqlite3_api->vfs_register +#define sqlite3_vfs_unregister sqlite3_api->vfs_unregister +#define sqlite3_threadsafe sqlite3_api->xthreadsafe +#define sqlite3_result_zeroblob sqlite3_api->result_zeroblob +#define sqlite3_result_error_code sqlite3_api->result_error_code +#define sqlite3_test_control sqlite3_api->test_control +#define sqlite3_randomness sqlite3_api->randomness +#define sqlite3_context_db_handle sqlite3_api->context_db_handle +#define sqlite3_extended_result_codes sqlite3_api->extended_result_codes +#define sqlite3_limit sqlite3_api->limit +#define sqlite3_next_stmt sqlite3_api->next_stmt +#define sqlite3_sql sqlite3_api->sql +#define sqlite3_status sqlite3_api->status +#define sqlite3_backup_finish sqlite3_api->backup_finish +#define sqlite3_backup_init sqlite3_api->backup_init +#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount +#define sqlite3_backup_remaining sqlite3_api->backup_remaining +#define sqlite3_backup_step sqlite3_api->backup_step +#define sqlite3_compileoption_get sqlite3_api->compileoption_get +#define sqlite3_compileoption_used sqlite3_api->compileoption_used +#define sqlite3_create_function_v2 sqlite3_api->create_function_v2 +#define sqlite3_db_config sqlite3_api->db_config +#define sqlite3_db_mutex sqlite3_api->db_mutex +#define sqlite3_db_status sqlite3_api->db_status +#define sqlite3_extended_errcode sqlite3_api->extended_errcode +#define sqlite3_log sqlite3_api->log +#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 +#define sqlite3_sourceid sqlite3_api->sourceid +#define sqlite3_stmt_status sqlite3_api->stmt_status +#define sqlite3_strnicmp sqlite3_api->strnicmp +#define sqlite3_unlock_notify sqlite3_api->unlock_notify +#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint +#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint +#define sqlite3_wal_hook sqlite3_api->wal_hook +#define sqlite3_blob_reopen sqlite3_api->blob_reopen +#define sqlite3_vtab_config sqlite3_api->vtab_config +#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict +/* Version 3.7.16 and later */ +#define sqlite3_close_v2 sqlite3_api->close_v2 +#define sqlite3_db_filename sqlite3_api->db_filename +#define sqlite3_db_readonly sqlite3_api->db_readonly +#define sqlite3_db_release_memory sqlite3_api->db_release_memory +#define sqlite3_errstr sqlite3_api->errstr +#define sqlite3_stmt_busy sqlite3_api->stmt_busy +#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly +#define sqlite3_stricmp sqlite3_api->stricmp +#define sqlite3_uri_boolean sqlite3_api->uri_boolean +#define sqlite3_uri_int64 sqlite3_api->uri_int64 +#define sqlite3_uri_parameter sqlite3_api->uri_parameter +#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 +#endif /* SQLITE_CORE */ + +#ifndef SQLITE_CORE + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +# define SQLITE_EXTENSION_INIT3 \ + extern const sqlite3_api_routines *sqlite3_api; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +# define SQLITE_EXTENSION_INIT3 /*no-op*/ +#endif + +#endif /* _SQLITE3EXT_H_ */ diff --git a/Postbox/module.private.modulemap b/Postbox/module.private.modulemap new file mode 100644 index 0000000000..44e6fd2c50 --- /dev/null +++ b/Postbox/module.private.modulemap @@ -0,0 +1,3 @@ +module PostboxPrivate { + export * +} diff --git a/PostboxTests/CodingTests.swift b/PostboxTests/CodingTests.swift new file mode 100644 index 0000000000..8a8f69011a --- /dev/null +++ b/PostboxTests/CodingTests.swift @@ -0,0 +1,159 @@ +import UIKit +import XCTest + +import Postbox + +class TestParent: Coding, Equatable { + var parentInt32: Int32 + + required init(decoder: Decoder) { + self.parentInt32 = decoder.decodeInt32ForKey("parentInt32") + } + + init(parentInt32: Int32) { + self.parentInt32 = parentInt32 + } + + func encode(encoder: Encoder) { + encoder.encodeInt32(self.parentInt32, forKey: "parentInt32") + } +} + +class TestObject: TestParent { + var int32: Int32 + var int64: Int64 + var double: Double + var string: String + var int32Array: [Int32] + var int64Array: [Int64] + + required init(decoder: Decoder) { + self.int32 = decoder.decodeInt32ForKey("int32") + self.int64 = decoder.decodeInt64ForKey("int64") + self.double = decoder.decodeDoubleForKey("double") + self.string = decoder.decodeStringForKey("string") + self.int32Array = decoder.decodeInt32ArrayForKey("int32Array") + self.int64Array = decoder.decodeInt64ArrayForKey("int64Array") + super.init(decoder: decoder) + } + + init(parentInt32: Int32, int32: Int32, int64: Int64, double: Double, string: String, int32Array: [Int32], int64Array: [Int64]) { + self.int32 = int32 + self.int64 = int64 + self.double = double + self.string = string + self.int32Array = int32Array + self.int64Array = int64Array + super.init(parentInt32: parentInt32) + } + + override func encode(encoder: Encoder) { + encoder.encodeInt32(self.int32, forKey: "int32") + encoder.encodeInt64(self.int64, forKey: "int64") + encoder.encodeDouble(self.double, forKey: "double") + encoder.encodeString(self.string, forKey: "string") + encoder.encodeInt32Array(self.int32Array, forKey: "int32Array") + encoder.encodeInt64Array(self.int64Array, forKey: "int64Array") + super.encode(encoder) + } +} + +class TestKey: Coding, Hashable { + let value: Int + required init(decoder: Decoder) { + self.value = Int(decoder.decodeInt32ForKey("value")) + } + + init(value: Int) { + self.value = value + } + + func encode(encoder: Encoder) { + encoder.encodeInt32(Int32(self.value), forKey: "value") + } + + var hashValue: Int { + get { + return self.value + } + } +} + +func ==(lhs: TestObject, rhs: TestObject) -> Bool { + return lhs.int32 == rhs.int32 && + lhs.int64 == rhs.int64 && + lhs.double == rhs.double && + lhs.string == rhs.string && + lhs.int32Array == rhs.int32Array && + lhs.int64Array == rhs.int64Array && + lhs.parentInt32 == rhs.parentInt32 +} + +func ==(lhs: TestParent, rhs: TestParent) -> Bool { + return lhs.parentInt32 == rhs.parentInt32 +} + +func ==(lhs: TestKey, rhs: TestKey) -> Bool { + return lhs.value == rhs.value +} + +class SerializationTests: XCTestCase { + override func setUp() { + super.setUp() + } + + override func tearDown() { + super.tearDown() + } + + func testExample() { + declareEncodable(TestParent.self, { TestParent(decoder: $0) }) + declareEncodable(TestObject.self, { TestObject(decoder: $0) }) + declareEncodable(TestKey.self, { TestKey(decoder: $0) }) + + let encoder = Encoder() + encoder.encodeInt32(12345, forKey: "a") + encoder.encodeInt64(Int64(12345), forKey: "b") + encoder.encodeBool(true, forKey: "c") + encoder.encodeString("test", forKey: "d") + + let before = TestObject(parentInt32: 100, int32: 12345, int64: 67890, double: 1.23456, string: "test", int32Array: [1, 2, 3, 4, 5], int64Array: [6, 7, 8, 9, 0]) + encoder.encodeObject(before, forKey: "e") + + encoder.encodeInt32Array([1, 2, 3, 4], forKey: "f") + encoder.encodeInt64Array([1, 2, 3, 4], forKey: "g") + + let beforeArray: [TestParent] = [TestObject(parentInt32: 1000, int32: 12345, int64: 67890, double: 1.23456, string: "test", int32Array: [1, 2, 3, 4, 5], int64Array: [6, 7, 8, 9, 0]), TestParent(parentInt32: 2000)] + + encoder.encodeObjectArray(beforeArray, forKey: "h") + + let beforeDictionary: [TestKey : TestParent] = [ + TestKey(value: 1): TestObject(parentInt32: 1000, int32: 12345, int64: 67890, double: 1.23456, string: "test", int32Array: [1, 2, 3, 4, 5], int64Array: [6, 7, 8, 9, 0]), + TestKey(value: 2): TestParent(parentInt32: 2000) + ] + + encoder.encodeObjectDictionary(beforeDictionary, forKey: "i") + + let decoder = Decoder(buffer: encoder.makeReadBufferAndReset()) + + let afterDictionary = decoder.decodeObjectDictionaryForKey("i") as [TestKey : TestParent] + XCTAssert(afterDictionary == beforeDictionary, "object dictionary failed") + + let afterArray = decoder.decodeObjectArrayForKey("h") as [TestParent] + XCTAssert(afterArray == beforeArray, "object array failed") + + XCTAssert(decoder.decodeInt64ArrayForKey("g") == [1, 2, 3, 4], "int64 array failed") + XCTAssert(decoder.decodeInt32ArrayForKey("f") == [1, 2, 3, 4], "int32 array failed") + + if let after = decoder.decodeObjectForKey("e") as? TestObject { + XCTAssert(after == before, "object failed") + } else { + XCTFail("object failed") + } + + XCTAssert(decoder.decodeStringForKey("d") == "test", "string failed") + XCTAssert(decoder.decodeBoolForKey("c"), "bool failed") + XCTAssert(decoder.decodeInt64ForKey("b") == Int64(12345), "int64 failed") + XCTAssert(decoder.decodeInt32ForKey("a") == 12345, "int32 failed") + } +} diff --git a/PostboxTests/PostboxTests.swift b/PostboxTests/PostboxTests.swift index d2545f03f8..e7369d36cc 100644 --- a/PostboxTests/PostboxTests.swift +++ b/PostboxTests/PostboxTests.swift @@ -1,36 +1,663 @@ -// -// PostboxTests.swift -// PostboxTests -// -// Created by Peter on 10/06/15. -// Copyright (c) 2015 Telegram. All rights reserved. -// - import UIKit import XCTest +import Postbox + +enum TestPeerNamespace: PeerId.Namespace { + case User = 0 +} + +enum TestMessageNamespace: MessageId.Namespace { + case Cloud = 0 + case Local = 1 +} + +enum TestMediaNamespace: MediaId.Namespace { + case Test = 0 +} + +class TestMessage: Message { + var id: MessageId + var authorId: PeerId + var date: Int32 + var text: String + var referencedMediaIds: [MediaId] + + var timestamp: Int32 { + return date + } + + init(id: MessageId, authorId: PeerId, date: Int32, text: String, referencedMediaIds: [MediaId]) { + self.id = id + self.authorId = authorId + self.date = date + self.text = text + self.referencedMediaIds = referencedMediaIds + } + + required init(decoder: Decoder) { + self.id = MessageId(decoder.decodeBytesForKeyNoCopy("id")) + self.authorId = PeerId(decoder.decodeInt64ForKey("authorId")) + self.date = decoder.decodeInt32ForKey("date") + self.text = decoder.decodeStringForKey("text") + self.referencedMediaIds = MediaId.decodeArrayFromBuffer(decoder.decodeBytesForKeyNoCopy("mediaIds")) + } + + func encode(encoder: Encoder) { + let buffer = WriteBuffer() + self.id.encodeToBuffer(buffer) + encoder.encodeBytes(buffer, forKey: "id") + buffer.reset() + + encoder.encodeInt64(self.authorId.toInt64(), forKey: "authorId") + encoder.encodeInt32(self.date, forKey: "date") + encoder.encodeString(self.text, forKey: "text") + + MediaId.encodeArrayToBuffer(self.referencedMediaIds, buffer: buffer) + encoder.encodeBytes(buffer, forKey: "mediaIds") + buffer.reset() + } +} + +class TestMedia: Media { + var id: MediaId + + init(id: MediaId) { + self.id = id + } + + required init(decoder: Decoder) { + self.id = MediaId(decoder.decodeBytesForKeyNoCopy("id")) + } + + func encode(encoder: Encoder) { + let buffer = WriteBuffer() + self.id.encodeToBuffer(buffer) + encoder.encodeBytes(buffer, forKey: "id") + } +} + class PostboxTests: XCTestCase { override func setUp() { super.setUp() - // Put setup code here. This method is called before the invocation of each test method in the class. } override func tearDown() { - // Put teardown code here. This method is called after the invocation of each test method in the class. super.tearDown() } - func testExample() { - // This is an example of a functional test case. - XCTAssert(true, "Pass") - } - - func testPerformanceExample() { - // This is an example of a performance test case. - self.measureBlock() { - // Put the code you want to measure the time of here. + func testAddMessages() { + declareEncodable(TestMessage.self, { TestMessage(decoder: $0) }) + declareEncodable(TestMedia.self, { TestMedia(decoder: $0) }) + + let ownerId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 1000) + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + let messageNamespace = TestMessageNamespace.Cloud.rawValue + + let basePath = "/tmp/postboxtest" + NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) + let postbox = Postbox(basePath: basePath, ownerId: ownerId, messageNamespaces: [messageNamespace]) + postbox.modify { state in + let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) + for i in 0 ..< 10 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) + let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) + state.addMessages([message, message], medias: [testMedia]) + } + return } + postbox._dumpTables() + + postbox.modify { state in + var messageIds: [MessageId] = [] + for i in 0 ..< 5 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) + messageIds.append(messageId) + } + state.deleteMessagesWithIds(messageIds) + } + postbox._dumpTables(); + + postbox.modify { state in + var messageIds: [MessageId] = [] + for i in 0 ..< 10 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) + messageIds.append(messageId) + } + state.deleteMessagesWithIds(messageIds) + } + postbox._dumpTables(); } + func testMessageNamespaceViewAddTail() { + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + + let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [:], messages: [], later: [:]) + + func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { + let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) + } + + func add(message: Message) { + view.add(message) + println("\(view)\n") + } + + /*func assertIds(earlier: Int32?, ids: [Int32], later: Int32?) { + let otherMessages = ids.reverse().map({ id -> Message in + let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(id)) + let message = TestMessage(id: messageId, authorId: otherId, date: Int32(0), text: "", referencedMediaIds: []) + return message + }) + let otherView = MessageNamespaceView(namespace: messageNamespace, messages: otherMessages, earlierId: earlier, laterId: later, count: 2) + + XCTAssert(view == otherView, "\(view) != \(otherView)") + }*/ + + add(message(1, 90, 90)) + add(message(2, 70, 70)) + + add(message(1, 70, 70)) + + add(message(1, 80, 80)) + add(message(2, 100, 100)) + + add(message(1, 75, 75)) + + add(message(1, 60, 60)) + } + + func testMessageNamespaceViewAddMiddle1() { + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + + func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { + let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) + } + + let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [1: message(1, 90, 90)], messages: [message(1, 100, 100), message(1, 120, 120), message(1, 140, 140)], later: [1: message(1, 200, 200)]) + + func add(message: Message) { + view.add(message) + println("\(view)\n") + } + + add(message(2, 105, 105)) + add(message(2, 150, 150)) + add(message(2, 250, 250)) + add(message(2, 180, 180)) + } + + func testMessageNamespaceRemoveTail() { + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + + var messages: [Message] = [] + + func print(messages: [Message]) { + var string = "" + string += "[" + var first = true + for message in messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" + } + string += "]" + println(string) + } + + let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [:], messages: [], later: [:]) + + func id(namespace: Int32, id: Int32) -> MessageId { + return MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + } + + func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { + let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) + } + + func add(message: Message) { + view.add(message) + println("\(view)") + + messages.append(message) + messages.sort({MessageIndex($0) < MessageIndex($1)}) + + print(messages) + println() + } + + func remove(ids: Set) -> MutableMessageView.RemoveContext { + let context = view.remove(ids) + println("\(view)") + + messages = messages.filter { !ids.contains($0.id) } + print(messages) + + return context + } + + func fetchEarlier(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + var filtered: [Message] = [] + var i = messages.count - 1 + while i >= 0 && filtered.count < count { + if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id < id!) { + filtered.append(messages[i]) + } + i-- + } + + return filtered + } + + func fetchLater(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + var filtered: [Message] = [] + var i = 0 + while i < messages.count && filtered.count < count { + if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id > id!) { + filtered.append(messages[i]) + } + i++ + } + + return filtered + } + + func complete(context: MutableMessageView.RemoveContext) { + view.complete(context, fetchEarlier: fetchEarlier(messages), fetchLater: fetchLater(messages)) + println("\(view)\n") + } + + add(message(1, 90, 90)) + add(message(2, 70, 70)) + add(message(1, 70, 70)) + add(message(1, 80, 80)) + add(message(2, 100, 100)) + add(message(1, 75, 75)) + add(message(1, 60, 60)) + + println("remove 1:90, 2:100") + complete(remove([id(1, 90), id(2, 100)])) + + println("remove 1:60, 2:70") + complete(remove([id(1, 60), id(2, 70)])) + + println("remove 1:80, 2:100") + complete(remove([id(1, 80), id(2, 100)])) + } + + func testMessageNamespaceRemoveAllInside() { + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + + var messages: [Message] = [] + + func print(messages: [Message]) { + var string = "" + string += "[" + var first = true + for message in messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" + } + string += "]" + println(string) + } + + let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [:], messages: [], later: [:]) + + func id(namespace: Int32, id: Int32) -> MessageId { + return MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + } + + func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { + let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) + } + + func add(message: Message) { + view.add(message) + println("\(view)") + + messages.append(message) + messages.sort({MessageIndex($0) < MessageIndex($1)}) + + print(messages) + println() + } + + func remove(ids: Set) -> MutableMessageView.RemoveContext { + let context = view.remove(ids) + println("\(view)") + + messages = messages.filter { !ids.contains($0.id) } + print(messages) + + return context + } + + func fetchEarlier(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + var filtered: [Message] = [] + var i = messages.count - 1 + while i >= 0 && filtered.count < count { + if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id < id!) { + filtered.append(messages[i]) + } + i-- + } + + return filtered + } + + func fetchLater(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + var filtered: [Message] = [] + var i = 0 + while i < messages.count && filtered.count < count { + if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id > id!) { + filtered.append(messages[i]) + } + i++ + } + + return filtered + } + + func complete(context: MutableMessageView.RemoveContext) { + view.complete(context, fetchEarlier: fetchEarlier(messages), fetchLater: fetchLater(messages)) + println("\(view)\n") + } + + add(message(2, 10, 10)) + add(message(2, 20, 20)) + add(message(1, 90, 90)) + add(message(2, 70, 70)) + add(message(1, 70, 70)) + add(message(1, 80, 80)) + add(message(2, 100, 100)) + add(message(1, 75, 75)) + add(message(1, 60, 60)) + + println("remove 2:20, 1:80, 1:90, 2:100") + complete(remove([id(2, 20), id(1, 80), id(1, 90), id(2, 100)])) + } + + func testMessageNamespaceRemoveMiddleSome() { + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + + var messages: [Message] = [] + + func print(messages: [Message]) { + var string = "" + string += "[" + var first = true + for message in messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" + } + string += "]" + println(string) + } + + func id(namespace: Int32, id: Int32) -> MessageId { + return MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + } + + func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { + let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) + } + + messages = [message(2, 1, 1), message(2, 2, 2), message(1, 98, 98), message(1, 99, 99), message(1, 100, 100), message(2, 101, 101), message(1, 102, 102), message(1, 103, 103), message(1, 104, 104), message(1, 105, 105), message(2, 200, 200), message(2, 300, 300)] + + let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [1: message(1, 99, 99), 2: message(2, 2, 2)], messages: [message(1, 100, 100), message(2, 101, 101), message(1, 102, 102)], later: [1: message(1, 103, 103), 2: message(2, 200, 200)]) + + func add(message: Message) { + view.add(message) + println("\(view)") + + messages.append(message) + messages.sort({MessageIndex($0) < MessageIndex($1)}) + + print(messages) + println() + } + + func remove(ids: Set) -> MutableMessageView.RemoveContext { + let context = view.remove(ids) + println("\(view)") + + messages = messages.filter { !ids.contains($0.id) } + print(messages) + + return context + } + + func fetchEarlier(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + var filtered: [Message] = [] + var i = messages.count - 1 + while i >= 0 && filtered.count < count { + if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id < id!) { + filtered.append(messages[i]) + } + i-- + } + + return filtered + } + + func fetchLater(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + var filtered: [Message] = [] + var i = 0 + while i < messages.count && filtered.count < count { + if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id > id!) { + filtered.append(messages[i]) + } + i++ + } + + return filtered + } + + func complete(context: MutableMessageView.RemoveContext) { + view.complete(context, fetchEarlier: fetchEarlier(messages), fetchLater: fetchLater(messages)) + println("\(view)\n") + } + + print(messages) + println("\(view)") + + println("remove 1:100, 2:101") + complete(remove([id(1, 100), id(2, 101)])) + } + + func testMessageNamespaceRemoveMiddleAllInside() { + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + + var messages: [Message] = [] + + func print(messages: [Message]) { + var string = "" + string += "[" + var first = true + for message in messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" + } + string += "]" + println(string) + } + + func id(namespace: Int32, id: Int32) -> MessageId { + return MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + } + + func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { + let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) + return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) + } + + messages = [message(2, 1, 1), message(2, 2, 2), message(1, 98, 98), message(1, 99, 99), message(1, 100, 100), message(2, 101, 101), message(1, 102, 102), message(1, 103, 103), message(1, 104, 104), message(1, 105, 105), message(2, 200, 200), message(2, 300, 300)] + + let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [1: message(1, 99, 99), 2: message(2, 2, 2)], messages: [message(1, 100, 100), message(2, 101, 101), message(1, 102, 102)], later: [1: message(1, 103, 103), 2: message(2, 200, 200)]) + + func add(message: Message) { + view.add(message) + println("\(view)") + + messages.append(message) + messages.sort({MessageIndex($0) < MessageIndex($1)}) + + print(messages) + println() + } + + func remove(ids: Set) -> MutableMessageView.RemoveContext { + let context = view.remove(ids) + println("\(view)") + + messages = messages.filter { !ids.contains($0.id) } + print(messages) + + return context + } + + func fetchEarlier(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + var filtered: [Message] = [] + var i = messages.count - 1 + while i >= 0 && filtered.count < count { + if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id < id!) { + filtered.append(messages[i]) + } + i-- + } + + return filtered + } + + func fetchLater(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + var filtered: [Message] = [] + var i = 0 + while i < messages.count && filtered.count < count { + if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id > id!) { + filtered.append(messages[i]) + } + i++ + } + + return filtered + } + + func complete(context: MutableMessageView.RemoveContext) { + view.complete(context, fetchEarlier: fetchEarlier(messages), fetchLater: fetchLater(messages)) + println("\(view)\n") + } + + print(messages) + println("\(view)\n") + + println("remove 1:100, 2:101, 1: 102") + complete(remove([id(1, 100), id(2, 101), id(1, 102)])) + } + + func testViewTail() { + declareEncodable(TestMessage.self, { TestMessage(decoder: $0) }) + declareEncodable(TestMedia.self, { TestMedia(decoder: $0) }) + + let ownerId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 1000) + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + let messageNamespace = TestMessageNamespace.Cloud.rawValue + + let basePath = "/tmp/postboxtest" + NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) + let postbox = Postbox(basePath: basePath, ownerId: ownerId, messageNamespaces: [messageNamespace]) + + postbox.modify { state in + let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) + for i in 0 ..< 10 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) + let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) + state.addMessages([message, message], medias: [testMedia]) + } + return + } + + postbox.tailMessageViewForPeerId(otherId, count: 4).start(next: { next in + println(next) + }) + + postbox.modify { state in + let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) + for i in 10 ..< 15 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) + let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) + state.addMessages([message, message], medias: [testMedia]) + } + return + } + + postbox._sync() + } + + func testViewAround() { + declareEncodable(TestMessage.self, { TestMessage(decoder: $0) }) + declareEncodable(TestMedia.self, { TestMedia(decoder: $0) }) + + let ownerId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 1000) + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + let messageNamespaceCloud = TestMessageNamespace.Cloud.rawValue + let messageNamespaceLocal = TestMessageNamespace.Local.rawValue + + let basePath = "/tmp/postboxtest" + NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) + let postbox = Postbox(basePath: basePath, ownerId: ownerId, messageNamespaces: [messageNamespaceCloud, messageNamespaceLocal]) + + postbox.modify { state in + let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) + for i in 0 ..< 10 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i + 1)) + let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) + state.addMessages([message, message], medias: [testMedia]) + } + for i in 10 ..< 12 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespaceLocal, id: Int32(i + 1)) + let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) + state.addMessages([message, message], medias: [testMedia]) + } + return + } + + var i = 1000 + postbox.aroundMessageViewForPeerId(otherId, id: MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i + 1)), count: 3).start(next: { next in + println(next) + }) + + postbox.modify { state in + let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) + for i in 10 ..< 15 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i + 1)) + let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) + state.addMessages([message, message], medias: [testMedia]) + } + return + } + + postbox._sync() + } } diff --git a/submodules/sqlite.swift b/submodules/sqlite.swift new file mode 160000 index 0000000000..fde8c3391b --- /dev/null +++ b/submodules/sqlite.swift @@ -0,0 +1 @@ +Subproject commit fde8c3391b265eb34b6407e3665160fb981b6857 From f687c0120b8dfd196edc5b3bea1f59b6100a2522 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 8 Jul 2015 17:18:27 +0300 Subject: [PATCH 003/427] no message --- .../xcschemes/xcschememanagement.plist | 27 + Postbox/Coding.swift | 603 ++++++++++ Postbox/Media.swift | 72 ++ Postbox/Message.swift | 116 ++ Postbox/Peer.swift | 58 + Postbox/Postbox.swift | 1050 +++++++++++++++++ 6 files changed, 1926 insertions(+) create mode 100644 Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist create mode 100644 Postbox/Coding.swift create mode 100644 Postbox/Media.swift create mode 100644 Postbox/Message.swift create mode 100644 Postbox/Peer.swift create mode 100644 Postbox/Postbox.swift diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 0000000000..db91d6f463 --- /dev/null +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,27 @@ + + + + + SchemeUserState + + Postbox.xcscheme + + orderHint + 1 + + + SuppressBuildableAutocreation + + D0E3A7491B28A7E300A402D9 + + primary + + + D0E3A7541B28A7E300A402D9 + + primary + + + + + diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift new file mode 100644 index 0000000000..6900a84ea7 --- /dev/null +++ b/Postbox/Coding.swift @@ -0,0 +1,603 @@ +import Foundation + +public protocol Coding { + init(decoder: Decoder) + func encode(encoder: Encoder) +} + +private struct EncodableTypeStore { + var dict: [Int32 : Decoder -> Coding] = [:] + + func decode(typeHash: Int32, decoder: Decoder) -> Coding? { + if let typeDecoder = self.dict[typeHash] { + return typeDecoder(decoder) + } else { + return nil + } + } +} + +private var typeStore = EncodableTypeStore() + +public func declareEncodable(type: Any.Type, f: Decoder -> Coding) { + let hash = murMurHashString32("\(type)") + if typeStore.dict[hash] != nil { + assertionFailure("Encodable type hash collision for \(type)") + } + typeStore.dict[murMurHashString32("\(type)")] = f +} + +public final class WriteBuffer { + var memory: UnsafeMutablePointer = nil + var capacity: Int + var offset: Int + + public init() { + self.memory = malloc(32) + self.capacity = 32 + self.offset = 0 + } + + deinit { + free(self.memory) + } + + func makeReadBufferAndReset() -> ReadBuffer { + let buffer = ReadBuffer(memory: self.memory, length: self.offset, freeWhenDone: true) + self.memory = malloc(32) + self.capacity = 32 + self.offset = 0 + return buffer + } + + func makeData() -> NSData { + return NSData(bytes: self.memory, length: self.offset) + } + + public func reset() { + self.offset = 0 + } + + func write(data: UnsafePointer, offset: Int, length: Int) { + if self.offset + length > self.capacity { + self.capacity = self.offset + length + 256 + self.memory = realloc(self.memory, self.capacity) + } + memcpy(self.memory + self.offset, data + offset, length) + self.offset += length + } +} + +public final class ReadBuffer { + var memory: UnsafeMutablePointer + var length: Int + var offset : Int + let freeWhenDone: Bool + + init(memory: UnsafeMutablePointer, length: Int, freeWhenDone: Bool) { + self.memory = memory + self.length = length + self.offset = 0 + self.freeWhenDone = freeWhenDone + } + + deinit { + if self.freeWhenDone { + free(self.memory) + } + } + + func read(data: UnsafeMutablePointer, offset: Int, length: Int) { + memcpy(data + offset, self.memory + self.offset, length) + self.offset += length + } + + func reset() { + self.offset = 0 + } +} + +private enum ValueType: Int8 { + case Int32 = 0 + case Int64 = 1 + case Bool = 2 + case Double = 3 + case String = 4 + case Object = 5 + case Int32Array = 6 + case Int64Array = 7 + case ObjectArray = 8 + case ObjectDictionary = 9 + case Bytes = 10 +} + +public final class Encoder { + private let buffer = WriteBuffer() + + public init() { + } + + public func makeReadBufferAndReset() -> ReadBuffer { + return self.buffer.makeReadBufferAndReset() + } + + public func makeData() -> NSData { + return self.buffer.makeData() + } + + public func reset() { + self.buffer.reset() + } + + public func encodeKey(key: UnsafePointer) { + var length: Int8 = Int8(strlen(key)) + self.buffer.write(&length, offset: 0, length: 1) + self.buffer.write(UnsafePointer(key), offset: 0, length: Int(length)) + } + + public func encodeInt32(value: Int32, forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.Int32.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var v = value + self.buffer.write(&v, offset: 0, length: 4) + } + + public func encodeInt64(value: Int64, forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.Int64.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var v = value + self.buffer.write(&v, offset: 0, length: 8) + } + + public func encodeBool(value: Bool, forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.Bool.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var v: Int8 = value ? 1 : 0 + self.buffer.write(&v, offset: 0, length: 1) + } + + public func encodeDouble(value: Double, forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.Double.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var v = value + self.buffer.write(&v, offset: 0, length: 8) + } + + public func encodeString(value: String, forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.String.rawValue + self.buffer.write(&type, offset: 0, length: 1) + let data = value.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! + var length: Int32 = Int32(data.length) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(data.bytes, offset: 0, length: Int(length)) + } + + public func encodeRootObject(value: Coding) { + self.encodeObject(value, forKey: "_") + } + + public func encodeObject(value: Coding, forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.Object.rawValue + self.buffer.write(&type, offset: 0, length: 1) + + var typeHash: Int32 = murMurHashString32(_stdlib_getDemangledTypeName(value)) + self.buffer.write(&typeHash, offset: 0, length: 4) + + let innerEncoder = Encoder() + value.encode(innerEncoder) + + var length: Int32 = Int32(innerEncoder.buffer.offset) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(length)) + } + + public func encodeInt32Array(value: [Int32], forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.Int32Array.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var length: Int32 = Int32(value.count) + self.buffer.write(&length, offset: 0, length: 4) + value.withUnsafeBufferPointer { (data: UnsafeBufferPointer) -> Void in + self.buffer.write(UnsafePointer(data.baseAddress), offset: 0, length: Int(length) * 4) + return + } + } + + public func encodeInt64Array(value: [Int64], forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.Int64Array.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var length: Int32 = Int32(value.count) + self.buffer.write(&length, offset: 0, length: 4) + value.withUnsafeBufferPointer { (data: UnsafeBufferPointer) -> Void in + self.buffer.write(UnsafePointer(data.baseAddress), offset: 0, length: Int(length) * 8) + return + } + } + + public func encodeObjectArray(value: [T], forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.ObjectArray.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var length: Int32 = Int32(value.count) + self.buffer.write(&length, offset: 0, length: 4) + let innerEncoder = Encoder() + for object in value { + var typeHash: Int32 = murMurHashString32("\(object.dynamicType)") + self.buffer.write(&typeHash, offset: 0, length: 4) + + innerEncoder.reset() + object.encode(innerEncoder) + + var length: Int32 = Int32(innerEncoder.buffer.offset) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(length)) + } + } + + public func encodeObjectDictionary(value: [K : V], forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.ObjectDictionary.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var length: Int32 = Int32(value.count) + self.buffer.write(&length, offset: 0, length: 4) + + let innerEncoder = Encoder() + for record in value { + var keyTypeHash: Int32 = murMurHashString32("\(record.0.dynamicType)") + self.buffer.write(&keyTypeHash, offset: 0, length: 4) + innerEncoder.reset() + record.0.encode(innerEncoder) + var keyLength: Int32 = Int32(innerEncoder.buffer.offset) + self.buffer.write(&keyLength, offset: 0, length: 4) + self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(keyLength)) + + var valueTypeHash: Int32 = murMurHashString32("\(record.1.dynamicType)") + self.buffer.write(&valueTypeHash, offset: 0, length: 4) + innerEncoder.reset() + record.1.encode(innerEncoder) + var valueLength: Int32 = Int32(innerEncoder.buffer.offset) + self.buffer.write(&valueLength, offset: 0, length: 4) + self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(valueLength)) + } + } + + public func encodeBytes(bytes: WriteBuffer, forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.Bytes.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var bytesLength: Int32 = Int32(bytes.offset) + self.buffer.write(&bytesLength, offset: 0, length: 4) + self.buffer.write(bytes.memory, offset: 0, length: bytes.offset) + } +} + +public final class Decoder { + private let buffer: ReadBuffer + + public init(buffer: ReadBuffer) { + self.buffer = buffer + } + + private class func skipValue(bytes: UnsafePointer, inout offset: Int, length: Int, valueType: ValueType) { + switch valueType { + case .Int32: + offset += 4 + case .Int64: + offset += 8 + case .Bool: + offset += 1 + case .Double: + offset += 8 + case .String: + var length: Int32 = 0 + memcpy(&length, bytes + offset, 4) + offset += 4 + Int(length) + case .Object: + var length: Int32 = 0 + memcpy(&length, bytes + (offset + 4), 4) + offset += 8 + Int(length) + case .Int32Array: + var length: Int32 = 0 + memcpy(&length, bytes + offset, 4) + offset += 4 + Int(length) * 4 + case .Int64Array: + var length: Int32 = 0 + memcpy(&length, bytes + offset, 4) + offset += 4 + Int(length) * 8 + case .ObjectArray: + var length: Int32 = 0 + memcpy(&length, bytes + offset, 4) + offset += 4 + var i: Int32 = 0 + while i < length { + var objectLength: Int32 = 0 + memcpy(&objectLength, bytes + (offset + 4), 4) + offset += 8 + Int(objectLength) + i++ + } + case .ObjectDictionary: + var length: Int32 = 0 + memcpy(&length, bytes + offset, 4) + offset += 4 + var i: Int32 = 0 + while i < length { + var keyLength: Int32 = 0 + memcpy(&keyLength, bytes + (offset + 4), 4) + offset += 8 + Int(keyLength) + + var valueLength: Int32 = 0 + memcpy(&valueLength, bytes + (offset + 4), 4) + offset += 8 + Int(valueLength) + i++ + } + case .Bytes: + var length: Int32 = 0 + memcpy(&length, bytes + offset, 4) + offset += 4 + Int(length) + } + } + + private class func positionOnKey(bytes: UnsafePointer, inout offset: Int, maxOffset: Int, length: Int, key: UnsafePointer, valueType: ValueType) -> Bool + { + var startOffset = offset + + let keyLength: Int = Int(strlen(key)) + while (offset < maxOffset) + { + let readKeyLength = bytes[offset] + offset += 1 + offset += Int(readKeyLength) + + let readValueType = bytes[offset] + offset += 1 + + if readValueType != valueType.rawValue || keyLength != Int(readKeyLength) || memcmp(bytes + (offset - Int(readKeyLength) - 1), key, keyLength) != 0 { + skipValue(bytes, offset: &offset, length: length, valueType: ValueType(rawValue: readValueType)!) + } else { + return true + } + } + + if (startOffset != 0) + { + offset = 0 + return positionOnKey(bytes, offset: &offset, maxOffset: startOffset, length: length, key: key, valueType: valueType) + } + + return false + } + + public func decodeInt32ForKey(key: UnsafePointer) -> Int32 { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { + var value: Int32 = 0 + memcpy(&value, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + return value + } else { + return 0 + } + } + + public func decodeInt64ForKey(key: UnsafePointer) -> Int64 { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { + var value: Int64 = 0 + memcpy(&value, self.buffer.memory + self.buffer.offset, 8) + self.buffer.offset += 8 + return value + } else { + return 0 + } + } + + public func decodeBoolForKey(key: UnsafePointer) -> Bool { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bool) { + var value: Int8 = 0 + memcpy(&value, self.buffer.memory + self.buffer.offset, 1) + self.buffer.offset += 1 + return value != 0 + } else { + return false + } + } + + public func decodeDoubleForKey(key: UnsafePointer) -> Double { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Double) { + var value: Double = 0 + memcpy(&value, self.buffer.memory + self.buffer.offset, 8) + self.buffer.offset += 8 + return value + } else { + return 0 + } + } + + public func decodeStringForKey(key: UnsafePointer) -> String { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + let data = NSData(bytes: self.buffer.memory + (self.buffer.offset + 4), length: Int(length)) + self.buffer.offset += 4 + Int(length) + let value = NSString(data: data, encoding: NSUTF8StringEncoding) + return (value as? String) ?? "" + } else { + return "" + } + } + + public func decodeRootObject() -> Coding? { + return self.decodeObjectForKey("_") + } + + public func decodeObjectForKey(key: UnsafePointer) -> Coding? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { + var typeHash: Int32 = 0 + memcpy(&typeHash, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + + var innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(length), freeWhenDone: false)) + self.buffer.offset += 4 + Int(length) + + return typeStore.decode(typeHash, decoder: innerDecoder) + } else { + return nil + } + } + + public func decodeInt32ArrayForKey(key: UnsafePointer) -> [Int32] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32Array) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + var array: [Int32] = [] + array.reserveCapacity(Int(length)) + var i: Int32 = 0 + while i < length { + var element: Int32 = 0 + memcpy(&element, self.buffer.memory + (self.buffer.offset + 4 + 4 * Int(i)), 4) + array.append(element) + i++ + } + self.buffer.offset += 4 + Int(length) * 4 + return array + } else { + return [] + } + } + + public func decodeInt64ArrayForKey(key: UnsafePointer) -> [Int64] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64Array) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + var array: [Int64] = [] + array.reserveCapacity(Int(length)) + var i: Int32 = 0 + while i < length { + var element: Int64 = 0 + memcpy(&element, self.buffer.memory + (self.buffer.offset + 4 + 8 * Int(i)), 8) + array.append(element) + i++ + } + self.buffer.offset += 4 + Int(length) * 8 + return array + } else { + return [] + } + } + + public func decodeObjectArrayForKey(key: UnsafePointer) -> [T] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + + var array: [T] = [] + array.reserveCapacity(Int(length)) + + var failed = false + var i: Int32 = 0 + while i < length { + var typeHash: Int32 = 0 + memcpy(&typeHash, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + + var objectLength: Int32 = 0 + memcpy(&objectLength, self.buffer.memory + self.buffer.offset, 4) + + var innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(objectLength), freeWhenDone: false)) + self.buffer.offset += 4 + Int(objectLength) + + if !failed { + if let object = typeStore.decode(typeHash, decoder: innerDecoder) as? T { + array.append(object) + } else { + failed = true + } + } + + i++ + } + + if failed { + return [] + } else { + return array + } + } else { + return [] + } + } + + public func decodeObjectDictionaryForKey(key: UnsafePointer) -> [K : V] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + + var dictionary: [K : V] = [:] + + var failed = false + var i: Int32 = 0 + while i < length { + var keyHash: Int32 = 0 + memcpy(&keyHash, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + + var keyLength: Int32 = 0 + memcpy(&keyLength, self.buffer.memory + self.buffer.offset, 4) + + var innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(keyLength), freeWhenDone: false)) + self.buffer.offset += 4 + Int(keyLength) + + let key = failed ? nil : (typeStore.decode(keyHash, decoder: innerDecoder) as? K) + + var valueHash: Int32 = 0 + memcpy(&valueHash, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + + var valueLength: Int32 = 0 + memcpy(&valueLength, self.buffer.memory + self.buffer.offset, 4) + + innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(valueLength), freeWhenDone: false)) + self.buffer.offset += 4 + Int(valueLength) + + let value = failed ? nil : (typeStore.decode(valueHash, decoder: innerDecoder) as? V) + + if let key = key, value = value { + dictionary[key] = value + } else { + failed = true + } + + i++ + } + + if failed { + return [:] + } else { + return dictionary + } + } else { + return [:] + } + } + + public func decodeBytesForKeyNoCopy(key: UnsafePointer) -> ReadBuffer! { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bytes) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + Int(length) + return ReadBuffer(memory: UnsafeMutablePointer(self.buffer.memory + (self.buffer.offset - Int(length))), length: Int(length), freeWhenDone: false) + } else { + return nil + } + } +} diff --git a/Postbox/Media.swift b/Postbox/Media.swift new file mode 100644 index 0000000000..e49661547b --- /dev/null +++ b/Postbox/Media.swift @@ -0,0 +1,72 @@ +import Foundation + +public struct MediaId: Hashable, Printable { + public typealias Namespace = Int32 + public typealias Id = Int64 + + let namespace: Namespace + let id: Id + + public var hashValue: Int { + get { + return Int(self.id) + } + } + + public var description: String { + get { + return "\(namespace):\(id)" + } + } + + public init(namespace: Namespace, id: Id) { + self.namespace = namespace + self.id = id + } + + public init(_ buffer: ReadBuffer) { + self.namespace = 0 + self.id = 0 + + memcpy(&self.namespace, buffer.memory + buffer.offset, 4) + memcpy(&self.id, buffer.memory + (buffer.offset + 4), 8) + buffer.offset += 12 + } + + public func encodeToBuffer(buffer: WriteBuffer) { + var namespace = self.namespace + var id = self.id + buffer.write(&namespace, offset: 0, length: 4); + buffer.write(&id, offset: 0, length: 8); + } + + public static func encodeArrayToBuffer(array: [MediaId], buffer: WriteBuffer) { + var length: Int32 = Int32(array.count) + buffer.write(&length, offset: 0, length: 4) + for id in array { + id.encodeToBuffer(buffer) + } + } + + public static func decodeArrayFromBuffer(buffer: ReadBuffer) -> [MediaId] { + var length: Int32 = 0 + var offset = 0 + memcpy(&length, buffer.memory, 4) + buffer.offset += 4 + var i = 0 + var array: [MediaId] = [] + while i < Int(length) { + array.append(MediaId(buffer)) + i++ + } + return array + } +} + +public func ==(lhs: MediaId, rhs: MediaId) -> Bool { + return lhs.id == rhs.id && lhs.namespace == rhs.namespace +} + +public protocol Media: Coding { + var id: MediaId { get } +} diff --git a/Postbox/Message.swift b/Postbox/Message.swift new file mode 100644 index 0000000000..3a62008174 --- /dev/null +++ b/Postbox/Message.swift @@ -0,0 +1,116 @@ +import Foundation + +public struct MessageId: Hashable, Printable { + public typealias Namespace = Int32 + public typealias Id = Int32 + + public let peerId: PeerId + public let namespace: Namespace + public let id: Id + + public var hashValue: Int { + get { + return Int(self.id) + } + } + + public var description: String { + get { + return "\(namespace):\(id)" + } + } + + public init(peerId: PeerId, namespace: Namespace, id: Id) { + self.peerId = peerId + self.namespace = namespace + self.id = id + } + + public init(_ buffer: ReadBuffer) { + self.peerId = PeerId(namespace: 0, id: 0) + self.namespace = 0 + self.id = 0 + + memcpy(&self.peerId.namespace, buffer.memory + buffer.offset, 4) + memcpy(&self.peerId.id, buffer.memory + (buffer.offset + 4), 4) + memcpy(&self.namespace, buffer.memory + (buffer.offset + 8), 4) + memcpy(&self.id, buffer.memory + (buffer.offset + 12), 4) + buffer.offset += 16 + } + + public func encodeToBuffer(buffer: WriteBuffer) { + var peerIdNamespace = self.peerId.namespace + var peerIdId = self.peerId.id + var namespace = self.namespace + var id = self.id + buffer.write(&peerIdNamespace, offset: 0, length: 4); + buffer.write(&peerIdId, offset: 0, length: 4); + buffer.write(&namespace, offset: 0, length: 4); + buffer.write(&id, offset: 0, length: 4); + } + + public static func encodeArrayToBuffer(array: [MessageId], buffer: WriteBuffer) { + var length: Int32 = Int32(array.count) + buffer.write(&length, offset: 0, length: 4) + for id in array { + id.encodeToBuffer(buffer) + } + } + + public static func decodeArrayFromBuffer(buffer: ReadBuffer) -> [MessageId] { + var length: Int32 = 0 + var offset = 0 + memcpy(&length, buffer.memory, 4) + buffer.offset += 4 + var i = 0 + var array: [MessageId] = [] + while i < Int(length) { + array[i] = MessageId(buffer) + i++ + } + return array + } +} + +public func ==(lhs: MessageId, rhs: MessageId) -> Bool { + return lhs.id == rhs.id && lhs.namespace == rhs.namespace +} + +public struct MessageIndex: Equatable, Comparable { + public let id: MessageId + public let timestamp: Int32 + + public init(_ message: Message) { + self.id = message.id + self.timestamp = message.timestamp + } + + public init(id: MessageId, timestamp: Int32) { + self.id = id + self.timestamp = timestamp + } +} + +public func ==(lhs: MessageIndex, rhs: MessageIndex) -> Bool { + return lhs.id == rhs.id && lhs.timestamp == rhs.timestamp +} + +public func <(lhs: MessageIndex, rhs: MessageIndex) -> Bool { + if lhs.timestamp != rhs.timestamp { + return lhs.timestamp < rhs.timestamp + } + + if lhs.id.namespace != rhs.id.namespace { + return lhs.id.namespace < rhs.id.namespace + } + + return lhs.id.id < rhs.id.id +} + +public protocol Message: Coding { + var id: MessageId { get } + var authorId: PeerId { get } + var timestamp: Int32 { get } + var text: String { get } + var referencedMediaIds: [MediaId] { get } +} diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift new file mode 100644 index 0000000000..2d1ab17b51 --- /dev/null +++ b/Postbox/Peer.swift @@ -0,0 +1,58 @@ +import Foundation + +public struct PeerId: Hashable, Printable { + public typealias Namespace = Int32 + public typealias Id = Int32 + + let namespace: Namespace + let id: Id + + public init(namespace: Namespace, id: Id) { + self.namespace = namespace + self.id = id + } + + public init(_ n: Int64) { + self.namespace = Int32((n >> 32) & 0xffffffff) + self.id = Int32(n & 0xffffffff) + } + + public func toInt64() -> Int64 { + return (Int64(self.namespace) << 32) | Int64(self.id) + } + + public var hashValue: Int { + get { + return Int(self.id) + } + } + + public var description: String { + get { + return "\(namespace):\(id)" + } + } + + public init(_ buffer: ReadBuffer) { + self.namespace = 0 + self.id = 0 + + memcpy(&self.namespace, buffer.memory, 4) + memcpy(&self.id, buffer.memory + 4, 4) + } + + public func encodeToBuffer(buffer: WriteBuffer) { + var namespace = self.namespace + var id = self.id + buffer.write(&namespace, offset: 0, length: 4); + buffer.write(&id, offset: 0, length: 4); + } +} + +public func ==(lhs: PeerId, rhs: PeerId) -> Bool { + return lhs.id == rhs.id && lhs.namespace == rhs.namespace +} + +public protocol Peer: Coding { + var id: PeerId { get } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift new file mode 100644 index 0000000000..4572304d04 --- /dev/null +++ b/Postbox/Postbox.swift @@ -0,0 +1,1050 @@ +import Foundation + +import SwiftSignalKit + +public final class Postbox { + public class Modifier { + private unowned var postbox: Postbox + + private init(postbox: Postbox) { + self.postbox = postbox + } + + public func addMessages(messages: [Message], medias: [Media]) { + self.postbox.addMessages(messages, medias: medias) + } + + public func deleteMessagesWithIds(ids: [MessageId]) { + self.postbox.deleteMessagesWithIds(ids) + } + } + + private let basePath: String + private let ownerId: PeerId + private let messageNamespaces: [MessageId.Namespace] + + private let queue = SwiftSignalKit.Queue() + private var database: Database! + + private var peerMessageViews: [PeerId : Bag<(MutableMessageView, Pipe)>] = [:] + + public init(basePath: String, ownerId: PeerId, messageNamespaces: [MessageId.Namespace]) { + self.basePath = basePath + self.ownerId = ownerId + self.messageNamespaces = messageNamespaces + self.openDatabase() + } + + private func openDatabase() { + NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil, error: nil) + self.database = Database(basePath.stringByAppendingPathComponent("db")) + + self.queue.dispatch { + let result = self.database.userVersion + if result == 1 { + println("(Postbox schema version \(result))") + } else { + println("(Postbox creating schema)") + self.createSchema() + } + } + } + + private func createSchema() { + self.database.transaction() + + //state + self.database.execute("CREATE TABLE state (key INTEGER PRIMARY KEY, data BLOB)") + + //peer_messages + self.database.execute("CREATE TABLE peer_messages (peerId INTEGER, namespace INTEGER, id INTEGER, data BLOB, associatedMediaIds BLOB, timestamp INTEGER, PRIMARY KEY(peerId, namespace, id))") + + //peer_maxread + self.database.execute("CREATE TABLE peer_maxread (peerId INTEGER, namespace INTEGER, id INTEGER)") + + //peer_media + self.database.execute("CREATE TABLE peer_media (peerId INTEGER, mediaNamespace INTEGER, messageNamespace INTEGER, messageId INTEGER, PRIMARY KEY (peerId, mediaNamespace, messageNamespace, messageId))") + self.database.execute("CREATE INDEX peer_media_peerId_messageNamespace_messageId ON peer_media (peerId, messageNamespace, messageId)") + + //media + self.database.execute("CREATE TABLE media (namespace INTEGER, id INTEGER, data BLOB, associatedMessageIds BLOB, PRIMARY KEY (namespace, id))") + + //media_cleanup + self.database.execute("CREATE TABLE media_cleanup (namespace INTEGER, id INTEGER, data BLOB, PRIMARY KEY(namespace, id))") + + //messages_readstate + self.database.execute("CREATE TABLE messages_readstate (peerId INTEGER, namespace INTEGER, maxUnreadIncomingId INTEGER, maxUnreadOutgoingId INTEGER, incomingUnreadCount INTEGER, PRIMARY KEY (peerId, namespace))") + + //top_messages + self.database.execute("CREATE TABLE top_messages (peerId INTEGER PRIMARY KEY, timestamp INTEGER)") + + //peers + self.database.execute("CREATE TABLE peers (peerId INTEGER PRIMARY KEY, data BLOB)") + + //user_statuses + self.database.execute("CREATE TABLE user_statuses (id INTEGER PRIMARY KEY, data BLOB)") + + self.database.commit() + } + + private class func messageIdsGroupedByNamespace(ids: [MessageId]) -> [MessageId.Namespace : [MessageId]] { + var grouped: [MessageId.Namespace : [MessageId]] = [:] + + for id in ids { + if grouped[id.namespace] != nil { + grouped[id.namespace]!.append(id) + } else { + grouped[id.namespace] = [id] + } + } + + return grouped + } + + private class func mediaIdsGroupedByNamespaceFromMediaArray(mediaArray: [Media]) -> [MediaId.Namespace : [MediaId]] { + var grouped: [MediaId.Namespace : [MediaId]] = [:] + var seenMediaIds = Set() + + for media in mediaArray { + if !seenMediaIds.contains(media.id) { + seenMediaIds.insert(media.id) + if grouped[media.id.namespace] != nil { + grouped[media.id.namespace]!.append(media.id) + } else { + grouped[media.id.namespace] = [media.id] + } + } + } + + return grouped + } + + private class func mediaIdsGroupedByNamespaceFromSet(ids: Set) -> [MediaId.Namespace : [MediaId]] { + var grouped: [MediaId.Namespace : [MediaId]] = [:] + + for id in ids { + if var groupedIds = grouped[id.namespace] { + grouped[id.namespace]!.append(id) + } else { + grouped[id.namespace] = [id] + } + } + + return grouped + } + + private class func mediaIdsGroupedByNamespaceFromDictionaryKeys(dict: [MediaId : T]) -> [MediaId.Namespace : [MediaId]] { + var grouped: [MediaId.Namespace : [MediaId]] = [:] + + for (id, _) in dict { + if grouped[id.namespace] != nil { + grouped[id.namespace]!.append(id) + } else { + grouped[id.namespace] = [id] + } + } + + return grouped + } + + private class func messagesGroupedByPeerId(messages: [Message]) -> [PeerId : [Message]] { + var grouped: [PeerId : [Message]] = [:] + + for message in messages { + if grouped[message.id.peerId] != nil { + grouped[message.id.peerId]!.append(message) + } else { + grouped[message.id.peerId] = [message] + } + } + + return grouped + } + + private class func messageIdsGroupedByPeerId(messageIds: [MessageId]) -> [PeerId : [MessageId]] { + var grouped: [PeerId : [MessageId]] = [:] + + for id in messageIds { + if grouped[id.peerId] != nil { + grouped[id.peerId]!.append(id) + } else { + grouped[id.peerId] = [id] + } + } + + return grouped + } + + private class func blobForMediaIds(ids: [MediaId]) -> Blob { + let data = NSMutableData() + var version: Int8 = 1 + data.appendBytes(&version, length: 1) + + var count = Int32(ids.count) + data.appendBytes(&count, length:4) + + for id in ids { + var mNamespace = id.namespace + var mId = id.id + data.appendBytes(&mNamespace, length: 4) + data.appendBytes(&mId, length: 8) + } + + return Blob(data: data) + } + + private static func mediaIdsForBlob(blob: Blob) -> [MediaId] { + var ids: [MediaId] = [] + + var offset: Int = 0 + var version = 0 + blob.data.getBytes(&version, range: NSMakeRange(offset, 1)) + offset += 1 + + if version == 1 { + var count: Int32 = 0 + blob.data.getBytes(&count, range: NSMakeRange(offset, 4)) + offset += 4 + + var i = 0 + while i < Int(count) { + var mNamespace: Int32 = 0 + var mId: Int64 = 0 + blob.data.getBytes(&mNamespace, range: NSMakeRange(offset, 4)) + blob.data.getBytes(&mId, range: NSMakeRange(offset + 4, 8)) + ids.append(MediaId(namespace: mNamespace, id: mId)) + offset += 12 + i++ + } + } + + return ids + } + + private class func blobForMessageIds(ids: [MessageId]) -> Blob { + let data = NSMutableData() + var version: Int8 = 1 + data.appendBytes(&version, length: 1) + + var count = Int32(ids.count) + data.appendBytes(&count, length:4) + + for id in ids { + var mPeerNamespace = id.peerId.namespace + var mPeerId = id.peerId.id + var mNamespace = id.namespace + var mId = id.id + data.appendBytes(&mPeerNamespace, length: 4) + data.appendBytes(&mPeerId, length: 4) + data.appendBytes(&mNamespace, length: 4) + data.appendBytes(&mId, length: 4) + } + + return Blob(data: data) + } + + private static func messageIdsForBlob(blob: Blob) -> [MessageId] { + var ids: [MessageId] = [] + + var offset: Int = 0 + var version = 0 + blob.data.getBytes(&version, range: NSMakeRange(offset, 1)) + offset += 1 + + if version == 1 { + var count: Int32 = 0 + blob.data.getBytes(&count, range: NSMakeRange(offset, 4)) + offset += 4 + + var i = 0 + while i < Int(count) { + var mPeerNamespace: Int32 = 0 + var mPeerId: Int32 = 0 + var mNamespace: Int32 = 0 + var mId: Int32 = 0 + blob.data.getBytes(&mPeerNamespace, range: NSMakeRange(offset, 4)) + blob.data.getBytes(&mPeerId, range: NSMakeRange(offset + 4, 4)) + blob.data.getBytes(&mNamespace, range: NSMakeRange(offset + 8, 4)) + blob.data.getBytes(&mId, range: NSMakeRange(offset + 12, 4)) + ids.append(MessageId(peerId: PeerId(namespace: mPeerNamespace, id: mPeerId), namespace: mNamespace, id: mId)) + offset += 16 + i++ + } + } + + return ids + } + + private func addMessages(messages: [Message], medias: [Media]) { + let messageInsertStatement = self.database.prepare("INSERT INTO peer_messages (peerId, namespace, id, data, associatedMediaIds, timestamp) VALUES (?, ?, ?, ?, ?, ?)") + let peerMediaInsertStatement = self.database.prepare("INSERT INTO peer_media (peerId, mediaNamespace, messageNamespace, messageId) VALUES (?, ?, ?, ?)") + let mediaInsertStatement = self.database.prepare("INSERT INTO media (namespace, id, data, associatedMessageIds) VALUES (?, ?, ?, ?)") + let referencedMessageIdsStatement = self.database.prepare("SELECT associatedMessageIds FROM media WHERE namespace = ? AND id = ?") + let updateReferencedMessageIdsStatement = self.database.prepare("UPDATE media SET associatedMessageIds = ? WHERE namespace = ? AND id = ?") + + let encoder = Encoder() + + var messageIdsByMediaId: [MediaId : [MessageId]] = [:] + for (peerId, peerMessages) in Postbox.messagesGroupedByPeerId(messages) { + var maxTimestamp: Int32 = Int32.min + + var messageIds: [MessageId] = [] + var seenMessageIds = Set() + for message in peerMessages { + if !seenMessageIds.contains(message.id) { + seenMessageIds.insert(message.id) + messageIds.append(message.id) + } + } + + var existingMessageIds = Set() + + let messageIdsByNamespace = Postbox.messageIdsGroupedByNamespace(messageIds) + for (namespace, ids) in messageIdsByNamespace { + var queryString = "SELECT id FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (" + var first = true + for id in ids { + if first { + first = false + } else { + queryString += "," + } + queryString += "\(id.id)" + } + queryString += ")" + + let statement = self.database.prepare(queryString) + for row in statement.run(peerId.toInt64(), Int64(namespace)) { + existingMessageIds.insert(MessageId(peerId: peerId, namespace: namespace, id: Int32(row[0] as! Int64))) + } + } + + var relatedViews = self.peerMessageViews[peerId] ?? Bag() + + for message in peerMessages { + if existingMessageIds.contains(message.id) { + continue + } + existingMessageIds.insert(message.id) + + for record in relatedViews.copyItems() { + record.0.add(message) + } + + if maxTimestamp == Int32.min || message.timestamp > maxTimestamp { + maxTimestamp = message.timestamp + } + + encoder.reset() + encoder.encodeRootObject(message) + let messageBlob = Blob(data: encoder.makeData()) + + let referencedMediaIdsMediaIdsBlob = Postbox.blobForMediaIds(message.referencedMediaIds) + for id in message.referencedMediaIds { + if messageIdsByMediaId[id] != nil { + messageIdsByMediaId[id]!.append(message.id) + } else { + messageIdsByMediaId[id] = [message.id] + } + } + + messageInsertStatement.run(peerId.toInt64(), Int64(message.id.namespace), Int64(message.id.id), messageBlob, referencedMediaIdsMediaIdsBlob, Int64(message.timestamp)) + + for id in message.referencedMediaIds { + peerMediaInsertStatement.run(peerId.toInt64(), Int64(id.namespace), Int64(message.id.namespace), Int64(message.id.id)) + } + } + + for record in relatedViews.copyItems() { + record.1.putNext(MessageView(record.0)) + } + + var currentMaxTimestamp: Int32? + for row in self.database.prepare("SELECT timestamp FROM top_messages WHERE peerId = ?").run(peerId.toInt64()) { + currentMaxTimestamp = Int32(row[0] as! Int64) + } + + if let currentMaxTimestamp = currentMaxTimestamp { + if maxTimestamp > currentMaxTimestamp { + self.database.prepare("UPDATE top_messages SET timestamp = ? WHERE peerId = ?").run(Int64(maxTimestamp), peerId.toInt64()) + } + } else { + self.database.prepare("INSERT INTO top_messages(peerId, timestamp) VALUES(?, ?)").run(peerId.toInt64(), Int64(maxTimestamp)) + } + } + + var existingMediaIds = Set() + + for (namespace, ids) in Postbox.mediaIdsGroupedByNamespaceFromMediaArray(medias) { + var queryString = "SELECT id FROM media WHERE namespace = ? AND id IN ("; + var first = true + for id in ids { + if first { + first = false + } else { + queryString += "," + } + queryString += "\(id.id)" + } + queryString += ")" + for row in self.database.prepare(queryString).run(Int64(namespace)) { + existingMediaIds.insert(MediaId(namespace: namespace, id: row[0] as! Int64)) + } + } + + var processedMediaIdsForMessageIds = Set() + for media in medias { + if existingMediaIds.contains(media.id) { + continue + } + existingMediaIds.insert(media.id) + + encoder.reset() + encoder.encodeRootObject(media) + let mediaBlob = Blob(data: encoder.makeData()) + + processedMediaIdsForMessageIds.insert(media.id) + if let messageIds = messageIdsByMediaId[media.id] { + let referencedMessageIdsBlob = Postbox.blobForMessageIds(messageIds) + mediaInsertStatement.run(Int64(media.id.namespace), media.id.id, mediaBlob, referencedMessageIdsBlob) + } + } + + var updatedMessageIdsBlobByMediaId: [MediaId : Blob] = [:] + for (mediaId, messageIds) in messageIdsByMediaId { + if !processedMediaIdsForMessageIds.contains(mediaId) { + for row in referencedMessageIdsStatement.run(Int64(mediaId.namespace), mediaId.id) { + var currentMessageIds = Postbox.messageIdsForBlob(row[0] as! Blob) + currentMessageIds += messageIds + updatedMessageIdsBlobByMediaId[mediaId] = Postbox.blobForMessageIds(currentMessageIds) + } + } + } + + for (mediaId, messageIds) in updatedMessageIdsBlobByMediaId { + updateReferencedMessageIdsStatement.run(messageIds, Int64(mediaId.namespace), mediaId.id) + } + } + + private func loadMessageIdsByMediaIdForPeerId(peerId: PeerId, idsByNamespace: [MessageId.Namespace : [MessageId]]) -> [MediaId : [MessageId]] { + var grouped: [MediaId : [MessageId]] = [:] + + for (namespace, ids) in idsByNamespace { + var messageIdByNamespaceAndIdQuery = "SELECT messageId FROM peer_media WHERE peerId = ? AND messageNamespace = ? AND messageId IN (" + var first = true + for id in ids { + if first { + first = false + } else { + messageIdByNamespaceAndIdQuery += "," + } + messageIdByNamespaceAndIdQuery += "\(id.id)" + } + messageIdByNamespaceAndIdQuery += ")" + + var messageIdsWithMedia: [MessageId] = [] + for row in self.database.prepare(messageIdByNamespaceAndIdQuery).run(peerId.toInt64(), Int64(namespace)) { + messageIdsWithMedia.append(MessageId(peerId: peerId, namespace: namespace, id:Int32(row[0] as! Int64))) + } + + var associatedMediaIdsQueryString = "SELECT id, associatedMediaIds FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (" + first = true + for id in messageIdsWithMedia { + if first { + first = false + } else { + associatedMediaIdsQueryString += "," + } + associatedMediaIdsQueryString += "\(id.id)" + } + associatedMediaIdsQueryString += ")" + + for row in self.database.prepare(associatedMediaIdsQueryString).run(peerId.toInt64(), Int64(namespace)) { + let id = MessageId(peerId: peerId, namespace: namespace, id: Int32(row[0] as! Int64)) + let referencedMediaIds = Postbox.mediaIdsForBlob(row[1] as! Blob) + for mediaId in referencedMediaIds { + if grouped[mediaId] != nil { + grouped[mediaId]!.append(id) + } else { + grouped[mediaId] = [id] + } + } + } + } + + return grouped + } + + private func updateTopMessageTimestampForPeerId(peerId: PeerId) { + let selectIdStatement = self.database.prepare("SELECT MAX(id) FROM peer_messages WHERE peerId = ? AND namespace = ?") + let selectTimestampStatement = self.database.prepare("SELECT timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id = ?") + let updateTimestampStatement = self.database.prepare("UPDATE top_messages SET timestamp = ? WHERE peerId = ?") + + var maxTimestamp = Int32.min + for namespace in self.messageNamespaces { + for rowId in selectIdStatement.run(peerId.toInt64(), Int64(namespace)) { + for rowTimestamp in selectTimestampStatement.run(peerId.toInt64(), Int64(namespace), rowId[0]) { + let timestamp = Int32(rowTimestamp[0] as! Int64) + if timestamp > maxTimestamp { + maxTimestamp = timestamp + } + } + } + } + + if maxTimestamp != Int32.min { + updateTimestampStatement.run(Int64(maxTimestamp), peerId.toInt64()) + } + } + + private func deleteMessagesWithIds(ids: [MessageId]) { + for (peerId, messageIds) in Postbox.messageIdsGroupedByPeerId(ids) { + let messageIdsByNamespace = Postbox.messageIdsGroupedByNamespace(messageIds) + let messageIdsByMediaId = self.loadMessageIdsByMediaIdForPeerId(peerId, idsByNamespace: messageIdsByNamespace) + + for (namespace, messageIds) in messageIdsByNamespace { + var queryString = "DELETE FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (" + var first = true + for id in messageIds { + if first { + first = false + } else { + queryString += "," + } + queryString += "\(id.id)" + } + queryString += ")" + self.database.prepare(queryString).run(peerId.toInt64(), Int64(namespace)) + } + + for (namespace, messageIds) in messageIdsByNamespace { + var queryString = "DELETE FROM peer_media WHERE peerId = ? AND messageNamespace = ? AND messageId IN (" + var first = true + for id in messageIds { + if first { + first = false + } else { + queryString += "," + } + queryString += "\(id.id)" + } + queryString += ")" + self.database.prepare(queryString).run(peerId.toInt64(), Int64(namespace)) + } + + let mediaIdsByNamespace = Postbox.mediaIdsGroupedByNamespaceFromDictionaryKeys(messageIdsByMediaId) + var updatedMessageIdsByMediaId: [MediaId : [MessageId]] = [:] + + for (namespace, mediaIds) in mediaIdsByNamespace { + var queryString = "SELECT id, data, associatedMessageIds FROM media WHERE namespace = ? AND id in (" + var first = true + for id in mediaIds { + if first { + first = false + } else { + queryString += "," + } + queryString += "\(id.id)" + } + queryString += ")" + for row in self.database.prepare(queryString).run(Int64(namespace)) { + let mediaId = MediaId(namespace: namespace, id: row[0] as! Int64) + if let removedMessageIds = messageIdsByMediaId[mediaId] { + var messageIds = Postbox.messageIdsForBlob(row[2] as! Blob).filter { + !contains(removedMessageIds, $0) + } + updatedMessageIdsByMediaId[mediaId] = messageIds + + if messageIds.count == 0 { + self.database.prepare("INSERT OR IGNORE INTO media_cleanup (namespace, id, data) VALUES (?, ?, ?)").run(Int64(namespace), mediaId.id, row[1] as! Blob) + } + } + } + + for (mediaId, messageIds) in updatedMessageIdsByMediaId { + if messageIds.count == 0 { + self.database.prepare("DELETE FROM media WHERE namespace = ? AND id = ?").run(Int64(mediaId.namespace), mediaId.id) + } else { + self.database.prepare("UPDATE media SET associatedMessageIds = ? WHERE namespace = ? AND id = ?").run(Postbox.blobForMessageIds(messageIds), Int64(mediaId.namespace), mediaId.id) + } + } + } + + self.updateTopMessageTimestampForPeerId(peerId) + } + } + + public func modify(f: Modifier -> Void) { + self.queue.dispatch { + self.database.transaction() + f(Modifier(postbox: self)) + self.database.commit() + } + } + + private struct FetchedMessage { + let id: MessageId + let timestamp: Int32 + let data: NSData + } + + private func findAdjacentMessageIds(peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex) -> (MessageId.Id?, MessageId.Id?) { + var minId: MessageId.Id? + var maxId: MessageId.Id? + for row in self.database.prepare("SELECT MIN(id), MAX(id) FROM peer_messages WHERE peerId = ? AND namespace = ?").run(peerId.toInt64(), Int64(namespace)) { + minId = MessageId.Id(row[0] as! Int64) + maxId = MessageId.Id(row[1] as! Int64) + } + + if let minId = minId, maxId = maxId { + var minTimestamp: Int32! + var maxTimestamp: Int32! + for row in self.database.prepare("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (?, ?)").run(peerId.toInt64(), Int64(namespace), Int64(minId), Int64(maxId)) { + let id = Int32(row[0] as! Int64) + let timestamp = Int32(row[1] as! Int64) + if id == minId { + minTimestamp = timestamp + } else { + maxTimestamp = timestamp + } + } + + let earlierMidStatement = self.database.prepare("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id <= ? LIMIT 1") + let laterMidStatement = self.database.prepare("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id >= ? LIMIT 1") + + func lowerBound(timestamp: Int32) -> MessageId.Id? { + var leftId = minId + var leftTimestamp = minTimestamp + var rightId = maxId + var rightTimestamp = maxTimestamp + + while leftTimestamp <= timestamp && rightTimestamp >= timestamp { + let approximateMiddleId = leftId + (rightId - leftId) / 2 + if approximateMiddleId == leftId { + return rightId + } + var middleId: MessageId.Id? + var middleTimestamp: Int32? + for row in earlierMidStatement.run(peerId.toInt64(), Int64(namespace), Int64(approximateMiddleId)) { + middleId = MessageId.Id(row[0] as! Int64) + middleTimestamp = Int32(row[1] as! Int64) + break + } + if middleId == leftId { + return rightId + } + + if let middleId = middleId, middleTimestamp = middleTimestamp { + if middleTimestamp >= timestamp { + rightId = middleId + rightTimestamp = middleTimestamp + } else { + leftId = middleId + leftTimestamp = middleTimestamp + } + } else { + return nil + } + } + + return leftId + } + + func upperBound(timestamp: Int32) -> MessageId.Id? { + var leftId = minId + var leftTimestamp = minTimestamp + var rightId = maxId + var rightTimestamp = maxTimestamp + + while leftTimestamp <= timestamp && rightTimestamp >= timestamp { + let approximateMiddleId = leftId + (rightId - leftId) / 2 + if approximateMiddleId == leftId { + return leftId + } + var middleId: MessageId.Id? + var middleTimestamp: Int32? + for row in earlierMidStatement.run(peerId.toInt64(), Int64(namespace), Int64(approximateMiddleId)) { + middleId = MessageId.Id(row[0] as! Int64) + middleTimestamp = Int32(row[1] as! Int64) + break + } + if middleId == leftId { + return leftId + } + + if let middleId = middleId, middleTimestamp = middleTimestamp { + if middleTimestamp <= timestamp { + rightId = middleId + rightTimestamp = middleTimestamp + } else { + leftId = middleId + leftTimestamp = middleTimestamp + } + } else { + return nil + } + } + + return rightTimestamp + } + + if index.id.namespace < namespace { + let left = upperBound(index.timestamp - 1) + let right = lowerBound(index.timestamp) + return (left, right) + } else { + let left = upperBound(index.timestamp) + let right = lowerBound(index.timestamp + 1) + return (left, right) + } + } else { + return (nil, nil) + } + } + + private func fetchMessagesAround(peerId: PeerId, anchorId: MessageId, count: Int) -> ([Message], [MessageId.Namespace : Message], [MessageId.Namespace : Message]) { + var messages: [Message] = [] + + messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: anchorId.namespace, id: anchorId.id, count: count + 1) + messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: anchorId.namespace, id: anchorId.id - 1, count: count + 1) + + messages.sort({ MessageIndex($0) < MessageIndex($1) }) + var i = messages.count - 1 + while i >= 1 { + if messages[i].id == messages[i - 1].id { + messages.removeAtIndex(i) + } + i-- + } + + if messages.count == 0 { + return ([], [:], [:]) + } else { + var index: MessageIndex! + for message in messages { + if message.id == anchorId { + index = MessageIndex(message) + break + } + } + if index == nil { + var closestId: MessageId.Id = messages[0].id.id + var closestDistance = abs(closestId - anchorId.id) + var closestTimestamp: Int32 = messages[0].timestamp + for message in messages { + if abs(message.id.id - anchorId.id) < closestDistance { + closestId = message.id.id + closestDistance = abs(message.id.id - anchorId.id) + } + } + index = MessageIndex(id: MessageId(peerId: peerId, namespace: anchorId.namespace, id: closestId), timestamp: closestTimestamp) + } + + for namespace in self.messageNamespaces { + if namespace != anchorId.namespace { + let (left, right) = self.findAdjacentMessageIds(peerId, namespace: namespace, index: index) + if let left = left { + messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: left + 1, count: count + 1) + } + if let right = right { + messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: namespace, id: right - 1, count: count + 1) + } + } + } + + messages.sort({ MessageIndex($0) < MessageIndex($1) }) + var i = messages.count - 1 + while i >= 1 { + if messages[i].id == messages[i - 1].id { + messages.removeAtIndex(i) + } + i-- + } + + var anchorIndex = messages.count / 2 + i = 0 + while i < messages.count { + if messages[i].id == index.id { + anchorIndex = i + break + } + i++ + } + + var filteredMessages: [Message] = [] + var earlier: [MessageId.Namespace : Message] = [:] + var later: [MessageId.Namespace : Message] = [:] + + i = anchorIndex + var j = anchorIndex - 1 + var leftIndex = j + var rightIndex = i + + while i < messages.count || j >= 0 { + if i < messages.count && filteredMessages.count < count { + filteredMessages.append(messages[i]) + rightIndex = i + } + if j >= 0 && filteredMessages.count < count { + filteredMessages.append(messages[j]) + leftIndex = j + } + + i++ + j-- + } + + i = leftIndex - 1 + while i >= 0 { + if earlier[messages[i].id.namespace] == nil { + earlier[messages[i].id.namespace] = messages[i] + } + i-- + } + + i = rightIndex + 1 + while i < messages.count { + if later[messages[i].id.namespace] == nil { + later[messages[i].id.namespace] = messages[i] + } + i++ + } + + filteredMessages.sort({ MessageIndex($0) < MessageIndex($1) }) + + return (filteredMessages, earlier, later) + } + } + + private func fetchMessagesRelative(peerId: PeerId, earlier: Bool)(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + var messages: [Message] = [] + + let sign = earlier ? "<" : ">" + let order = earlier ? "DESC" : "ASC" + let statement = self.database.prepare("SELECT data, associatedMediaIds FROM peer_messages WHERE peerId = ? AND namespace = ? AND id \(sign) ? ORDER BY id \(order) LIMIT \(count)") + let bound: Int64 + if let id = id { + bound = Int64(id) + } else if earlier { + bound = Int64(Int32.max) + } else { + bound = Int64(Int32.min) + } + + for row in statement.run(Int64(peerId.toInt64()), Int64(namespace), bound) { + let data = (row[0] as! Blob).data + let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) + if let message = decoder.decodeRootObject() as? Message { + messages.append(message) + } else { + println("(PostBox: can't decode message)") + } + } + + return messages + } + + private func fetchMessagesTail(peerId: PeerId, count: Int) -> [Message] { + var messages: [Message] = [] + + for namespace in self.messageNamespaces { + messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: nil, count: count) + } + + messages.sort({ MessageIndex($0) < MessageIndex($1)}) + + return messages + } + + public func tailMessageViewForPeerId(peerId: PeerId, count: Int) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.queue.dispatch { + let tail = self.fetchMessagesTail(peerId, count: count + 1) + + var messages: [Message] = [] + var i = tail.count - 1 + while i >= 0 && i >= tail.count - count { + messages.insert(tail[i], atIndex: 0) + i-- + } + + var earlier: [MessageId.Namespace : Message] = [:] + + for namespace in self.messageNamespaces { + var i = tail.count - count - 1 + while i >= 0 { + if tail[i].id.namespace == namespace { + earlier[namespace] = tail[i] + break + } + i-- + } + } + + let mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: earlier, messages: messages, later: [:]) + let record = (mutableView, Pipe()) + + let index: Bag<(MutableMessageView, Pipe)>.Index + if let bag = self.peerMessageViews[peerId] { + index = bag.add(record) + } else { + let bag = Bag<(MutableMessageView, Pipe)>() + index = bag.add(record) + self.peerMessageViews[peerId] = bag + } + + subscriber.putNext(MessageView(mutableView)) + + let pipeDisposable = record.1.signal().start(next: { next in + subscriber.putNext(next) + }) + + disposable.set(ActionDisposable { [weak self] in + pipeDisposable.dispose() + + if let strongSelf = self { + strongSelf.queue.dispatch { + if let bag = strongSelf.peerMessageViews[peerId] { + bag.remove(index) + } + } + } + return + }) + } + + return disposable + } + } + + public func aroundMessageViewForPeerId(peerId: PeerId, id: MessageId, count: Int) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.queue.dispatch { + let mutableView: MutableMessageView + + let around = self.fetchMessagesAround(peerId, anchorId: id, count: count) + if around.0.count == 0 { + let tail = self.fetchMessagesTail(peerId, count: count + 1) + + var messages: [Message] = [] + var i = tail.count - 1 + while i >= 0 && i >= tail.count - count { + messages.insert(tail[i], atIndex: 0) + i-- + } + + var earlier: [MessageId.Namespace : Message] = [:] + + for namespace in self.messageNamespaces { + var i = tail.count - count - 1 + while i >= 0 { + if tail[i].id.namespace == namespace { + earlier[namespace] = tail[i] + break + } + i-- + } + } + + mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: earlier, messages: messages, later: [:]) + } else { + mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: around.1, messages: around.0, later: around.2) + } + + let record = (mutableView, Pipe()) + + let index: Bag<(MutableMessageView, Pipe)>.Index + if let bag = self.peerMessageViews[peerId] { + index = bag.add(record) + } else { + let bag = Bag<(MutableMessageView, Pipe)>() + index = bag.add(record) + self.peerMessageViews[peerId] = bag + } + + subscriber.putNext(MessageView(mutableView)) + + let pipeDisposable = record.1.signal().start(next: { next in + subscriber.putNext(next) + }) + + disposable.set(ActionDisposable { [weak self] in + pipeDisposable.dispose() + + if let strongSelf = self { + strongSelf.queue.dispatch { + if let bag = strongSelf.peerMessageViews[peerId] { + bag.remove(index) + } + } + } + return + }) + } + + return disposable + } + } + + func printMessages(messages: [Message]) { + var string = "" + string += "[" + var first = true + for message in messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" + } + string += "]" + println(string) + } + + public func _dumpTables() { + println("\n-----------") + println("top_messages") + println("------------") + for row in self.database.prepare("SELECT peerId, timestamp FROM top_messages ORDER BY timestamp DESC").run() { + println("\(PeerId(row[0] as! Int64)): \(row[1] as! Int64)") + } + + println("\n------------") + println("peer_messages") + println("-------------") + for row in self.database.prepare("SELECT peerId, namespace, id, timestamp, associatedMediaIds FROM peer_messages").run() { + println("peer(\(PeerId(row[0] as! Int64))) id(\(MessageId(peerId: PeerId(row[0] as! Int64), namespace: Int32(row[1] as! Int64), id:(Int32(row[2] as! Int64))))) timestamp(\(row[3] as! Int64)) media(\(Postbox.mediaIdsForBlob(row[4] as! Blob)))") + } + + println("\n---------") + println("peer_media") + println("----------") + for row in self.database.prepare("SELECT peerId, mediaNamespace, messageNamespace, messageId FROM peer_media").run() { + println("peer(\(PeerId(row[0] as! Int64))) namespace(\(row[1] as! Int64)) id(\(MessageId(peerId: PeerId(row[0] as! Int64), namespace: Int32(row[2] as! Int64), id:Int32(row[3] as! Int64))))") + } + + println("\n----") + println("media") + println("-----") + for row in self.database.prepare("SELECT namespace, id, associatedMessageIds FROM media").run() { + println("id(\(MediaId(namespace: Int32(row[0] as! Int64), id:(row[1] as! Int64)))) messages(\(Postbox.messageIdsForBlob(row[2] as! Blob)))") + } + + println("\n------------") + println("media_cleanup") + println("-------------") + for row in self.database.prepare("SELECT namespace, id FROM media_cleanup").run() { + println("id(\(MediaId(namespace: Int32(row[0] as! Int64), id:(row[1] as! Int64))))") + } + } + + public func _sync() { + self.queue.sync { + } + } +} From e2c1cbfcaf3f191ea18c1687793a6870bc7971ee Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 8 Jul 2015 18:21:15 +0300 Subject: [PATCH 004/427] no message --- .gitmodules | 6 +++++ Postbox.xcodeproj/project.pbxproj | 40 +++++++++++++++---------------- submodules/sqlite.swift | 1 - 3 files changed, 26 insertions(+), 21 deletions(-) delete mode 160000 submodules/sqlite.swift diff --git a/.gitmodules b/.gitmodules index 98e5887e06..64416a3e4c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "submodules/sqlite.swift"] path = submodules/sqlite.swift url = https://github.com/stephencelis/SQLite.swift.git +[submodule "submodules/SQlite.swift"] + path = submodules/SQlite.swift + url = git@github.com:peter-iakovlev/SQLite.swift.git +[submodule "submodules/SQLite.swift"] + path = submodules/SQLite.swift + url = git@github.com:peter-iakovlev/SQLite.swift.git diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index cb95fe5cc6..ded107159b 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -17,17 +17,17 @@ D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */ = {isa = PBXBuildFile; fileRef = D075165B1B2EC5B000AE42E0 /* module.private.modulemap */; }; D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */ = {isa = PBXBuildFile; fileRef = D075165D1B2EC5B500AE42E0 /* module.modulemap */; }; D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516611B2EC7FE00AE42E0 /* Database.swift */; }; - D075166B1B2EC7FE00AE42E0 /* Expression.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516621B2EC7FE00AE42E0 /* Expression.swift */; }; - D075166C1B2EC7FE00AE42E0 /* FTS.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516631B2EC7FE00AE42E0 /* FTS.swift */; }; - D075166E1B2EC7FE00AE42E0 /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516651B2EC7FE00AE42E0 /* Functions.swift */; }; - D075166F1B2EC7FE00AE42E0 /* Query.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516661B2EC7FE00AE42E0 /* Query.swift */; }; - D07516701B2EC7FE00AE42E0 /* Schema.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516671B2EC7FE00AE42E0 /* Schema.swift */; }; D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516681B2EC7FE00AE42E0 /* Statement.swift */; }; D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516691B2EC7FE00AE42E0 /* Value.swift */; }; D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; D07FC7D31B4A3D6B0010B3F7 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */; }; + D0D224F01B4D6ABD0085E26D /* Expression.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EB1B4D6ABD0085E26D /* Expression.swift */; }; + D0D224F11B4D6ABD0085E26D /* FTS.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EC1B4D6ABD0085E26D /* FTS.swift */; }; + D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; + D0D224F31B4D6ABD0085E26D /* Query.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EE1B4D6ABD0085E26D /* Query.swift */; }; + D0D224F41B4D6ABD0085E26D /* Schema.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EF1B4D6ABD0085E26D /* Schema.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; D0E3A75D1B28A7E300A402D9 /* PostboxTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */; }; @@ -60,17 +60,17 @@ D075165B1B2EC5B000AE42E0 /* module.private.modulemap */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = "sourcecode.module-map"; path = module.private.modulemap; sourceTree = ""; }; D075165D1B2EC5B500AE42E0 /* module.modulemap */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = "sourcecode.module-map"; path = module.modulemap; sourceTree = ""; }; D07516611B2EC7FE00AE42E0 /* Database.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Database.swift; path = submodules/sqlite.swift/SQLite/Database.swift; sourceTree = SOURCE_ROOT; }; - D07516621B2EC7FE00AE42E0 /* Expression.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Expression.swift; path = submodules/sqlite.swift/SQLite/Expression.swift; sourceTree = SOURCE_ROOT; }; - D07516631B2EC7FE00AE42E0 /* FTS.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = FTS.swift; path = submodules/sqlite.swift/SQLite/FTS.swift; sourceTree = SOURCE_ROOT; }; - D07516651B2EC7FE00AE42E0 /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; - D07516661B2EC7FE00AE42E0 /* Query.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Query.swift; path = submodules/sqlite.swift/SQLite/Query.swift; sourceTree = SOURCE_ROOT; }; - D07516671B2EC7FE00AE42E0 /* Schema.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Schema.swift; path = submodules/sqlite.swift/SQLite/Schema.swift; sourceTree = SOURCE_ROOT; }; D07516681B2EC7FE00AE42E0 /* Statement.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Statement.swift; path = submodules/sqlite.swift/SQLite/Statement.swift; sourceTree = SOURCE_ROOT; }; D07516691B2EC7FE00AE42E0 /* Value.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Value.swift; path = submodules/sqlite.swift/SQLite/Value.swift; sourceTree = SOURCE_ROOT; }; D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../SSignalKit/build/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; + D0D224EB1B4D6ABD0085E26D /* Expression.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Expression.swift; path = submodules/sqlite.swift/SQLite/Expression.swift; sourceTree = SOURCE_ROOT; }; + D0D224EC1B4D6ABD0085E26D /* FTS.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = FTS.swift; path = submodules/sqlite.swift/SQLite/FTS.swift; sourceTree = SOURCE_ROOT; }; + D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; + D0D224EE1B4D6ABD0085E26D /* Query.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Query.swift; path = submodules/sqlite.swift/SQLite/Query.swift; sourceTree = SOURCE_ROOT; }; + D0D224EF1B4D6ABD0085E26D /* Schema.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Schema.swift; path = submodules/sqlite.swift/SQLite/Schema.swift; sourceTree = SOURCE_ROOT; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; D0E3A74F1B28A7E300A402D9 /* Postbox.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Postbox.h; sourceTree = ""; }; @@ -137,12 +137,12 @@ D07516731B2EC8C700AE42E0 /* sqlite.swift */ = { isa = PBXGroup; children = ( + D0D224EB1B4D6ABD0085E26D /* Expression.swift */, + D0D224EC1B4D6ABD0085E26D /* FTS.swift */, + D0D224ED1B4D6ABD0085E26D /* Functions.swift */, + D0D224EE1B4D6ABD0085E26D /* Query.swift */, + D0D224EF1B4D6ABD0085E26D /* Schema.swift */, D07516611B2EC7FE00AE42E0 /* Database.swift */, - D07516621B2EC7FE00AE42E0 /* Expression.swift */, - D07516631B2EC7FE00AE42E0 /* FTS.swift */, - D07516651B2EC7FE00AE42E0 /* Functions.swift */, - D07516661B2EC7FE00AE42E0 /* Query.swift */, - D07516671B2EC7FE00AE42E0 /* Schema.swift */, D07516681B2EC7FE00AE42E0 /* Statement.swift */, D07516691B2EC7FE00AE42E0 /* Value.swift */, ); @@ -328,25 +328,25 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - D07516701B2EC7FE00AE42E0 /* Schema.swift in Sources */, D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */, + D0D224F11B4D6ABD0085E26D /* FTS.swift in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, - D075166E1B2EC7FE00AE42E0 /* Functions.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */, - D075166B1B2EC7FE00AE42E0 /* Expression.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, - D075166C1B2EC7FE00AE42E0 /* FTS.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */, D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */, + D0D224F31B4D6ABD0085E26D /* Query.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, + D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */, D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, - D075166F1B2EC7FE00AE42E0 /* Query.swift in Sources */, + D0D224F41B4D6ABD0085E26D /* Schema.swift in Sources */, D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, + D0D224F01B4D6ABD0085E26D /* Expression.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/submodules/sqlite.swift b/submodules/sqlite.swift deleted file mode 160000 index fde8c3391b..0000000000 --- a/submodules/sqlite.swift +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fde8c3391b265eb34b6407e3665160fb981b6857 From 457109bca459dcca5a02fef0fc21c3b361ba7ca0 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 8 Jul 2015 18:21:56 +0300 Subject: [PATCH 005/427] no message --- .gitmodules | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitmodules b/.gitmodules index 64416a3e4c..a075144d43 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,3 @@ -[submodule "submodules/sqlite.swift"] - path = submodules/sqlite.swift - url = https://github.com/stephencelis/SQLite.swift.git -[submodule "submodules/SQlite.swift"] - path = submodules/SQlite.swift - url = git@github.com:peter-iakovlev/SQLite.swift.git [submodule "submodules/SQLite.swift"] path = submodules/SQLite.swift url = git@github.com:peter-iakovlev/SQLite.swift.git From 6fdee4c6b395afd90ed5372a95880c234ba97f6b Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 8 Jul 2015 18:24:13 +0300 Subject: [PATCH 006/427] no message --- submodules/SQLite.swift | 1 + 1 file changed, 1 insertion(+) create mode 160000 submodules/SQLite.swift diff --git a/submodules/SQLite.swift b/submodules/SQLite.swift new file mode 160000 index 0000000000..b3a75ca484 --- /dev/null +++ b/submodules/SQLite.swift @@ -0,0 +1 @@ +Subproject commit b3a75ca4840bd8a541825cfd6ce14b5185985ac6 From 13f3951ddd60ce659d34dbc553e82bca6c0ec38b Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 16 Jul 2015 02:20:03 +0300 Subject: [PATCH 007/427] no message --- Postbox.xcodeproj/project.pbxproj | 4 + Postbox/MessageView.swift | 1 - Postbox/Peer.swift | 18 +- Postbox/PeerView.swift | 278 ++++++++++++++++++++++++++++++ Postbox/Postbox.swift | 101 +++++++++-- PostboxTests/PostboxTests.swift | 119 ++++++++++++- 6 files changed, 499 insertions(+), 22 deletions(-) create mode 100644 Postbox/PeerView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index ded107159b..ff36d79283 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -28,6 +28,7 @@ D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; D0D224F31B4D6ABD0085E26D /* Query.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EE1B4D6ABD0085E26D /* Query.swift */; }; D0D224F41B4D6ABD0085E26D /* Schema.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EF1B4D6ABD0085E26D /* Schema.swift */; }; + D0D225261B4D84930085E26D /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D225251B4D84930085E26D /* PeerView.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; D0E3A75D1B28A7E300A402D9 /* PostboxTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */; }; @@ -71,6 +72,7 @@ D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; D0D224EE1B4D6ABD0085E26D /* Query.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Query.swift; path = submodules/sqlite.swift/SQLite/Query.swift; sourceTree = SOURCE_ROOT; }; D0D224EF1B4D6ABD0085E26D /* Schema.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Schema.swift; path = submodules/sqlite.swift/SQLite/Schema.swift; sourceTree = SOURCE_ROOT; }; + D0D225251B4D84930085E26D /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; D0E3A74F1B28A7E300A402D9 /* Postbox.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Postbox.h; sourceTree = ""; }; @@ -180,6 +182,7 @@ D0E3A79D1B28B50400A402D9 /* Message.swift */, D0E3A7A11B28B7DC00A402D9 /* Media.swift */, D003E4E51B38DBDB00C22CBC /* MessageView.swift */, + D0D225251B4D84930085E26D /* PeerView.swift */, D0E3A7811B28ADD000A402D9 /* Postbox.swift */, D0E3A74D1B28A7E300A402D9 /* Supporting Files */, ); @@ -342,6 +345,7 @@ D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */, D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, + D0D225261B4D84930085E26D /* PeerView.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, D0D224F41B4D6ABD0085E26D /* Schema.swift in Sources */, diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift index 173d734d2e..20c9fa6fee 100644 --- a/Postbox/MessageView.swift +++ b/Postbox/MessageView.swift @@ -175,7 +175,6 @@ public final class MutableMessageView: Printable { } self.later.removeAll(keepCapacity: true) - if anchorIndex + 1 < addedMessages.count { for namespace in self.namespaces { var i = anchorIndex + 1 diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index 2d1ab17b51..49491b027c 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -1,11 +1,11 @@ import Foundation -public struct PeerId: Hashable, Printable { +public struct PeerId: Hashable, Printable, Comparable { public typealias Namespace = Int32 public typealias Id = Int32 - let namespace: Namespace - let id: Id + public let namespace: Namespace + public let id: Id public init(namespace: Namespace, id: Id) { self.namespace = namespace @@ -53,6 +53,18 @@ public func ==(lhs: PeerId, rhs: PeerId) -> Bool { return lhs.id == rhs.id && lhs.namespace == rhs.namespace } +public func <(lhs: PeerId, rhs: PeerId) -> Bool { + if lhs.namespace != rhs.namespace { + return lhs.namespace < rhs.namespace + } + + if lhs.id != rhs.id { + return lhs.id < rhs.id + } + + return false +} + public protocol Peer: Coding { var id: PeerId { get } } diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift new file mode 100644 index 0000000000..328c57497e --- /dev/null +++ b/Postbox/PeerView.swift @@ -0,0 +1,278 @@ +import Foundation + +public class PeerViewEntry { + public let peer: Peer + public let message: Message + + public init(peer: Peer, message: Message) { + self.peer = peer + self.message = message + } +} + +public struct PeerViewEntryIndex: Equatable, Comparable { + public let peerId: PeerId + public let messageIndex: MessageIndex + + public init(_ entry: PeerViewEntry) { + self.peerId = entry.peer.id + self.messageIndex = MessageIndex(entry.message) + } + + public init(peerId: PeerId, messageIndex: MessageIndex) { + self.peerId = peerId + self.messageIndex = messageIndex + } + + public func earlier() -> PeerViewEntryIndex { + return PeerViewEntryIndex(peerId: self.peerId, messageIndex: MessageIndex(id: MessageId(peerId: self.messageIndex.id.peerId, namespace: self.messageIndex.id.namespace, id: self.messageIndex.id.id - 1), timestamp: self.messageIndex.timestamp)) + } + + public func later() -> PeerViewEntryIndex { + return PeerViewEntryIndex(peerId: self.peerId, messageIndex: MessageIndex(id: MessageId(peerId: self.messageIndex.id.peerId, namespace: self.messageIndex.id.namespace, id: self.messageIndex.id.id + 1), timestamp: self.messageIndex.timestamp)) + } +} + +public func ==(lhs: PeerViewEntryIndex, rhs: PeerViewEntryIndex) -> Bool { + return lhs.peerId == rhs.peerId && lhs.messageIndex == rhs.messageIndex +} + +public func <(lhs: PeerViewEntryIndex, rhs: PeerViewEntryIndex) -> Bool { + if lhs.messageIndex != rhs.messageIndex { + return lhs.messageIndex < rhs.messageIndex + } + + return lhs.peerId < rhs.peerId +} + +public final class MutablePeerView: Printable { + public struct RemoveContext { + var invalidEarlier = false + var invalidLater = false + var removedEntries = false + } + + let tags: [Int32] + let count: Int + var earlier: PeerViewEntry? + var later: PeerViewEntry? + var entries: [PeerViewEntry] + + public init(tags: [Int32], count: Int, earlier: PeerViewEntry?, entries: [PeerViewEntry], later: PeerViewEntry?) { + self.tags = tags + self.count = count + self.earlier = earlier + self.entries = entries + self.later = later + } + + public func removeEntry(context: RemoveContext?, peerId: PeerId) -> RemoveContext { + var invalidationContext = context ?? RemoveContext() + + if let earlier = self.earlier { + if peerId == earlier.peer.id { + invalidationContext.invalidEarlier = true + } + } + + if let later = self.later { + if peerId == later.peer.id { + invalidationContext.invalidLater = true + } + } + + var i = 0 + while i < self.entries.count { + if self.entries[i].peer.id == peerId { + self.entries.removeAtIndex(i) + invalidationContext.removedEntries = true + break + } + i++ + } + + return invalidationContext + } + + public func addEntry(entry: PeerViewEntry) { + if self.entries.count == 0 { + self.entries.append(entry) + } else { + var first = PeerViewEntryIndex(self.entries[self.entries.count - 1]) + var last = PeerViewEntryIndex(self.entries[0]) + + let index = PeerViewEntryIndex(entry) + + var next: PeerViewEntryIndex? + if let later = self.later { + next = PeerViewEntryIndex(later) + } + + if index < last { + let earlierEntry = self.earlier + if earlierEntry == nil || PeerViewEntryIndex(earlierEntry!) < index { + if self.entries.count < self.count { + self.entries.insert(entry, atIndex: 0) + } else { + self.earlier = entry + } + } + } else if index > first { + if next != nil && index > next! { + let laterEntry = self.later + if laterEntry == nil || PeerViewEntryIndex(laterEntry!) > index { + if self.entries.count < self.count { + self.entries.append(entry) + } else { + self.later = entry + } + } + } else { + self.entries.append(entry) + if self.entries.count > self.count { + let earliest = self.entries[0] + self.earlier = earliest + self.entries.removeAtIndex(0) + } + } + } else if index != last && index != first { + var i = self.entries.count + while i >= 1 { + if PeerViewEntryIndex(self.entries[i - 1]) < index { + break + } + i-- + } + self.entries.insert(entry, atIndex: i) + if self.entries.count > self.count { + let earliest = self.entries[0] + self.earlier = earliest + self.entries.removeAtIndex(0) + } + } + } + } + + public func complete(context: RemoveContext, fetchEarlier: (PeerViewEntryIndex?, Int) -> [PeerViewEntry], fetchLater: (PeerViewEntryIndex?, Int) -> [PeerViewEntry]) { + if context.removedEntries && self.entries.count != self.count { + var addedEntries: [PeerViewEntry] = [] + + var latestAnchor: PeerViewEntryIndex? + + if self.entries.count != 0 { + latestAnchor = PeerViewEntryIndex(self.entries[self.entries.count - 1]) + } else if let later = self.later { + latestAnchor = PeerViewEntryIndex(later) + } + + if let later = self.later { + addedEntries += fetchLater(PeerViewEntryIndex(later).earlier(), self.count) + } + if let earlier = self.earlier { + addedEntries += fetchEarlier(PeerViewEntryIndex(earlier).later(), self.count) + } + + addedEntries += self.entries + addedEntries.sort({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) + + var i = addedEntries.count - 1 + while i >= 1 { + if PeerViewEntryIndex(addedEntries[i]) == PeerViewEntryIndex(addedEntries[i - 1]) { + addedEntries.removeAtIndex(i) + } + i-- + } + self.entries = [] + + var anchorIndex = addedEntries.count - 1 + if let latestAnchor = latestAnchor { + var i = addedEntries.count - 1 + while i >= 0 { + if PeerViewEntryIndex(addedEntries[i]) <= latestAnchor { + anchorIndex = i + break + } + i-- + } + } + + self.later = nil + if anchorIndex + 1 < addedEntries.count { + var i = anchorIndex + 1 + while i < addedEntries.count { + self.later = addedEntries[i] + break + } + } + + i = anchorIndex + while i >= 0 && i > anchorIndex - self.count { + self.entries.insert(addedEntries[i], atIndex: 0) + i-- + } + + self.earlier = nil + if anchorIndex - self.count >= 0 { + i = anchorIndex - self.count + while i >= 0 { + self.earlier = addedEntries[i] + break + } + } + } else { + var earlyIndex: PeerViewEntryIndex? + if self.entries.count != 0 { + earlyIndex = PeerViewEntryIndex(self.entries[0]) + } + + let earlierEntries = fetchEarlier(earlyIndex, 1) + if earlierEntries.count == 0 { + self.earlier = nil + } else { + self.earlier = earlierEntries[0] + } + + var lateIndex: PeerViewEntryIndex? + if self.entries.count != 0 { + lateIndex = PeerViewEntryIndex(self.entries[self.entries.count - 1]) + } + + let laterEntries = fetchLater(lateIndex, 1) + if laterEntries.count == 0 { + self.later = nil + } else { + self.later = laterEntries[0] + } + } + } + + public var description: String { + var string = "" + + if let earlier = self.earlier { + string += "more(" + string += "(p \(earlier.peer.id.namespace):\(earlier.peer.id.id), m \(earlier.message.id.namespace):\(earlier.message.id.id)—\(earlier.message.timestamp)" + string += ") " + } + + string += "[" + var first = true + for entry in self.entries { + if first { + first = false + } else { + string += ", " + } + string += "(p \(entry.peer.id.namespace):\(entry.peer.id.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" + } + string += "]" + + if let later = self.later { + string += " more(" + string += "(p \(later.peer.id.namespace):\(later.peer.id.id), m \(later.message.id.namespace):\(later.message.id.id)—\(later.message.timestamp)" + string += ")" + } + + return string + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 4572304d04..6018c1d352 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -3,6 +3,8 @@ import Foundation import SwiftSignalKit public final class Postbox { + static let PeerTagGeneral = 0 + public class Modifier { private unowned var postbox: Postbox @@ -20,7 +22,6 @@ public final class Postbox { } private let basePath: String - private let ownerId: PeerId private let messageNamespaces: [MessageId.Namespace] private let queue = SwiftSignalKit.Queue() @@ -28,9 +29,8 @@ public final class Postbox { private var peerMessageViews: [PeerId : Bag<(MutableMessageView, Pipe)>] = [:] - public init(basePath: String, ownerId: PeerId, messageNamespaces: [MessageId.Namespace]) { + public init(basePath: String, messageNamespaces: [MessageId.Namespace]) { self.basePath = basePath - self.ownerId = ownerId self.messageNamespaces = messageNamespaces self.openDatabase() } @@ -51,17 +51,12 @@ public final class Postbox { } private func createSchema() { - self.database.transaction() - //state self.database.execute("CREATE TABLE state (key INTEGER PRIMARY KEY, data BLOB)") //peer_messages self.database.execute("CREATE TABLE peer_messages (peerId INTEGER, namespace INTEGER, id INTEGER, data BLOB, associatedMediaIds BLOB, timestamp INTEGER, PRIMARY KEY(peerId, namespace, id))") - //peer_maxread - self.database.execute("CREATE TABLE peer_maxread (peerId INTEGER, namespace INTEGER, id INTEGER)") - //peer_media self.database.execute("CREATE TABLE peer_media (peerId INTEGER, mediaNamespace INTEGER, messageNamespace INTEGER, messageId INTEGER, PRIMARY KEY (peerId, mediaNamespace, messageNamespace, messageId))") self.database.execute("CREATE INDEX peer_media_peerId_messageNamespace_messageId ON peer_media (peerId, messageNamespace, messageId)") @@ -72,21 +67,97 @@ public final class Postbox { //media_cleanup self.database.execute("CREATE TABLE media_cleanup (namespace INTEGER, id INTEGER, data BLOB, PRIMARY KEY(namespace, id))") - //messages_readstate - self.database.execute("CREATE TABLE messages_readstate (peerId INTEGER, namespace INTEGER, maxUnreadIncomingId INTEGER, maxUnreadOutgoingId INTEGER, incomingUnreadCount INTEGER, PRIMARY KEY (peerId, namespace))") - - //top_messages - self.database.execute("CREATE TABLE top_messages (peerId INTEGER PRIMARY KEY, timestamp INTEGER)") + //peer_entries + self.database.execute("CREATE TABLE peer_entries (entry BLOB)") + self.database.execute("CREATE INDEX peer_entries_entry on peer_entries (entry)") //peers self.database.execute("CREATE TABLE peers (peerId INTEGER PRIMARY KEY, data BLOB)") + } + + public func _testBlobsPrepare(range: Int) { + self.database.execute("CREATE TABLE test_blobs (number INTEGER, data BLOB)") + self.database.execute("CREATE INDEX test_blobs_index_data ON test_blobs (data)") - //user_statuses - self.database.execute("CREATE TABLE user_statuses (id INTEGER PRIMARY KEY, data BLOB)") + self.database.transaction() + let insert = self.database.prepare("INSERT INTO test_blobs (number, data) VALUES (?, ?)") + for i in 0 ..< range { + var value = Int32(bigEndian: Int32(i)) + let blob = Blob(bytes: &value, length: 4) + insert.run(Int64(i), blob) + } self.database.commit() } + public func _testBlobsTest(range: Int) { + let select = self.database.prepare("SELECT number FROM test_blobs WHERE data < ? ORDER BY data DESC LIMIT 1") + + for i in 1 ..< 1000 { + var value = 1 + Int32(arc4random_uniform(UInt32(range - 1))) + var binary = Int32(bigEndian: value) + var found = false + for row in select.run(Blob(bytes: &binary, length: 4)) { + found = true + let selected = Int32(row[0] as! Int64) + if selected != value - 1 { + assertionFailure("invalid value \(selected) != \(value - 1)") + } + break + } + if !found { + assertionFailure("value not found for \(value - 1)") + } + } + } + + private class func peerViewEntryIndexForBlob(blob: Blob) -> PeerViewEntryIndex { + let buffer = ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false) + var offset: Int = 0 + + var timestamp: Int32 = 0 + buffer.read(×tamp, offset: offset, length: 4) + timestamp = Int32(bigEndian: timestamp) + offset += 4 + + var namespace: Int32 = 0 + buffer.read(&namespace, offset: offset, length: 4) + namespace = Int32(bigEndian: namespace) + offset += 4 + + var id: Int32 = 0 + buffer.read(&id, offset: offset, length: 4) + id = Int32(bigEndian: id) + offset += 4 + + var peerIdRepresentation: Int64 = 0 + buffer.read(&peerIdRepresentation, offset: offset, length: 8) + peerIdRepresentation = Int64(bigEndian: peerIdRepresentation) + offset += 8 + + let peerId = PeerId(peerIdRepresentation) + + return PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(id: MessageId(peerId:peerId, namespace: namespace, id: id), timestamp: timestamp)) + } + + private class func blobForPeerViewEntryIndex(index: PeerViewEntryIndex) -> Blob { + let buffer = WriteBuffer() + + var timestamp = Int32(bigEndian: index.messageIndex.timestamp) + buffer.write(×tamp, offset: 0, length: 4) + + var namespace = Int32(bigEndian: index.messageIndex.id.namespace) + buffer.write(&namespace, offset: 0, length: 4) + + var id = Int32(bigEndian: index.messageIndex.id.id) + buffer.write(&id, offset: 0, length: 4) + + var peerIdRepresentation = Int64(bigEndian: index.peerId.toInt64()) + buffer.write(&peerIdRepresentation, offset: 0, length: 8) + + return Blob(data: buffer.makeData()) + } + private class func messageIdsGroupedByNamespace(ids: [MessageId]) -> [MessageId.Namespace : [MessageId]] { var grouped: [MessageId.Namespace : [MessageId]] = [:] diff --git a/PostboxTests/PostboxTests.swift b/PostboxTests/PostboxTests.swift index e7369d36cc..957984c544 100644 --- a/PostboxTests/PostboxTests.swift +++ b/PostboxTests/PostboxTests.swift @@ -16,6 +16,22 @@ enum TestMediaNamespace: MediaId.Namespace { case Test = 0 } +class TestPeer: Peer { + var id: PeerId + + init(id: PeerId) { + self.id = id + } + + required init(decoder: Decoder) { + self.id = PeerId(decoder.decodeInt64ForKey("id")) + } + + func encode(encoder: Encoder) { + encoder.encodeInt64(self.id.toInt64(), forKey: "id") + } +} + class TestMessage: Message { var id: MessageId var authorId: PeerId @@ -97,7 +113,7 @@ class PostboxTests: XCTestCase { let basePath = "/tmp/postboxtest" NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, ownerId: ownerId, messageNamespaces: [messageNamespace]) + let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) postbox.modify { state in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) for i in 0 ..< 10 { @@ -586,7 +602,7 @@ class PostboxTests: XCTestCase { let basePath = "/tmp/postboxtest" NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, ownerId: ownerId, messageNamespaces: [messageNamespace]) + let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) postbox.modify { state in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) @@ -626,7 +642,7 @@ class PostboxTests: XCTestCase { let basePath = "/tmp/postboxtest" NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, ownerId: ownerId, messageNamespaces: [messageNamespaceCloud, messageNamespaceLocal]) + let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespaceCloud, messageNamespaceLocal]) postbox.modify { state in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) @@ -660,4 +676,101 @@ class PostboxTests: XCTestCase { postbox._sync() } + + func testPeerView() { + let view = MutablePeerView(tags: [], count: 3, earlier: nil, entries: [], later: nil) + let messageNamespaceCloud = TestMessageNamespace.Cloud.rawValue + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + + var entries: [PeerViewEntry] = [] + + func print(entries: [PeerViewEntry]) { + var string = "" + string += "[" + var first = true + for entry in entries { + if first { + first = false + } else { + string += ", " + } + string += "(p \(entry.peer.id.namespace):\(entry.peer.id.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" + } + string += "]" + println("\(string)") + } + + func add(entry: PeerViewEntry) { + entries.append(entry) + entries.sort({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) + + view.addEntry(entry) + + println("\n\(view)") + print(entries) + } + + func remove(peerId: PeerId, context: MutablePeerView.RemoveContext? = nil) -> MutablePeerView.RemoveContext { + entries = entries.filter({ $0.peer.id != peerId }) + return view.removeEntry(context, peerId: peerId) + } + + func fetchEarlier(entries: [PeerViewEntry])(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntry] { + var filtered: [PeerViewEntry] = [] + var i = entries.count - 1 + while i >= 0 && filtered.count < count { + if index == nil || PeerViewEntryIndex(entries[i]) < index! { + filtered.append(entries[i]) + } + i-- + } + + return filtered + } + + func fetchLater(entries: [PeerViewEntry])(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntry] { + var filtered: [PeerViewEntry] = [] + var i = 0 + while i < entries.count && filtered.count < count { + if index == nil || PeerViewEntryIndex(entries[i]) > index! { + filtered.append(entries[i]) + } + i++ + } + + return filtered + } + + func complete(context: MutablePeerView.RemoveContext) { + view.complete(context, fetchEarlier: fetchEarlier(entries), fetchLater: fetchLater(entries)) + } + + println("\(view)") + + for i in 1 ..< 10 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i * 2 * 100)) + let message = TestMessage(id: messageId, authorId: otherId, date: Int32(i * 2 * 100), text: "\(i)", referencedMediaIds: []) + + add(PeerViewEntry(peer: TestPeer(id: PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(i * 2))), message: message)) + } + + if true { + let i = 15 + let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i * 100)) + let message = TestMessage(id: messageId, authorId: otherId, date: Int32(i * 100), text: "\(i)", referencedMediaIds: []) + + add(PeerViewEntry(peer: TestPeer(id: PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(i))), message: message)) + } + + if true { + var context = remove(PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(15))) + context = remove(PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(14)), context: context) + context = remove(PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(16)), context: context) + context = remove(PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(18)), context: context) + println("\n\(view)") + print(entries) + complete(context) + println("\(view)") + } + } } From 52b9f7291a93e84a07384558003c09ee8a1e4b87 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 16 Jul 2015 15:39:12 +0300 Subject: [PATCH 008/427] no message --- Postbox/MessageView.swift | 4 + Postbox/PeerView.swift | 80 +++++++-- Postbox/Postbox.swift | 293 +++++++++++++++++++++++--------- PostboxTests/PostboxTests.swift | 48 +++++- 4 files changed, 329 insertions(+), 96 deletions(-) diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift index 20c9fa6fee..786aa15576 100644 --- a/Postbox/MessageView.swift +++ b/Postbox/MessageView.swift @@ -11,6 +11,10 @@ public final class MutableMessageView: Printable { self.invalidLater = [] self.removedMessages = false } + + func empty() -> Bool { + return !self.removedMessages && self.invalidEarlier.count == 0 && self.invalidLater.count == 0 + } } let namespaces: [MessageId.Namespace] diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index 328c57497e..cf396590ce 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -1,13 +1,21 @@ import Foundation public class PeerViewEntry { - public let peer: Peer + public let peerId: PeerId + public let peer: Peer? public let message: Message public init(peer: Peer, message: Message) { + self.peerId = peer.id self.peer = peer self.message = message } + + public init(peerId: PeerId, message: Message) { + self.peerId = peerId + self.peer = nil + self.message = message + } } public struct PeerViewEntryIndex: Equatable, Comparable { @@ -15,7 +23,7 @@ public struct PeerViewEntryIndex: Equatable, Comparable { public let messageIndex: MessageIndex public init(_ entry: PeerViewEntry) { - self.peerId = entry.peer.id + self.peerId = entry.peerId self.messageIndex = MessageIndex(entry.message) } @@ -52,14 +60,12 @@ public final class MutablePeerView: Printable { var removedEntries = false } - let tags: [Int32] let count: Int var earlier: PeerViewEntry? var later: PeerViewEntry? var entries: [PeerViewEntry] - public init(tags: [Int32], count: Int, earlier: PeerViewEntry?, entries: [PeerViewEntry], later: PeerViewEntry?) { - self.tags = tags + public init(count: Int, earlier: PeerViewEntry?, entries: [PeerViewEntry], later: PeerViewEntry?) { self.count = count self.earlier = earlier self.entries = entries @@ -70,20 +76,20 @@ public final class MutablePeerView: Printable { var invalidationContext = context ?? RemoveContext() if let earlier = self.earlier { - if peerId == earlier.peer.id { + if peerId == earlier.peerId { invalidationContext.invalidEarlier = true } } if let later = self.later { - if peerId == later.peer.id { + if peerId == later.peerId { invalidationContext.invalidLater = true } } var i = 0 while i < self.entries.count { - if self.entries[i].peer.id == peerId { + if self.entries[i].peerId == peerId { self.entries.removeAtIndex(i) invalidationContext.removedEntries = true break @@ -251,7 +257,7 @@ public final class MutablePeerView: Printable { if let earlier = self.earlier { string += "more(" - string += "(p \(earlier.peer.id.namespace):\(earlier.peer.id.id), m \(earlier.message.id.namespace):\(earlier.message.id.id)—\(earlier.message.timestamp)" + string += "(p \(earlier.peerId.namespace):\(earlier.peerId.id), m \(earlier.message.id.namespace):\(earlier.message.id.id)—\(earlier.message.timestamp)" string += ") " } @@ -263,13 +269,65 @@ public final class MutablePeerView: Printable { } else { string += ", " } - string += "(p \(entry.peer.id.namespace):\(entry.peer.id.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" + string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" } string += "]" if let later = self.later { string += " more(" - string += "(p \(later.peer.id.namespace):\(later.peer.id.id), m \(later.message.id.namespace):\(later.message.id.id)—\(later.message.timestamp)" + string += "(p \(later.peerId.namespace):\(later.peerId), m \(later.message.id.namespace):\(later.message.id.id)—\(later.message.timestamp)" + string += ")" + } + + return string + } +} + +public final class PeerView: Printable { + let earlier: PeerViewEntryIndex? + let later: PeerViewEntryIndex? + let entries: [PeerViewEntry] + + init(_ mutableView: MutablePeerView) { + if let earlier = mutableView.earlier { + self.earlier = PeerViewEntryIndex(earlier) + } else { + self.earlier = nil + } + + if let later = mutableView.later { + self.later = PeerViewEntryIndex(later) + } else { + self.later = nil + } + + self.entries = mutableView.entries + } + + public var description: String { + var string = "" + + if let earlier = self.earlier { + string += "more(" + string += "(p \(earlier.peerId.namespace):\(earlier.peerId.id), m \(earlier.messageIndex.id.namespace):\(earlier.messageIndex.id.id)—\(earlier.messageIndex.timestamp)" + string += ") " + } + + string += "[" + var first = true + for entry in self.entries { + if first { + first = false + } else { + string += ", " + } + string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" + } + string += "]" + + if let later = self.later { + string += " more(" + string += "(p \(later.peerId.namespace):\(later.peerId), m \(later.messageIndex.id.namespace):\(later.messageIndex.id.id)—\(later.messageIndex.timestamp)" string += ")" } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 6018c1d352..b5a6f87549 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -28,6 +28,7 @@ public final class Postbox { private var database: Database! private var peerMessageViews: [PeerId : Bag<(MutableMessageView, Pipe)>] = [:] + private var peerViews: Bag<(MutablePeerView, Pipe)> = Bag() public init(basePath: String, messageNamespaces: [MessageId.Namespace]) { self.basePath = basePath @@ -52,7 +53,7 @@ public final class Postbox { private func createSchema() { //state - self.database.execute("CREATE TABLE state (key INTEGER PRIMARY KEY, data BLOB)") + self.database.execute("CREATE TABLE state (data BLOB)") //peer_messages self.database.execute("CREATE TABLE peer_messages (peerId INTEGER, namespace INTEGER, id INTEGER, data BLOB, associatedMediaIds BLOB, timestamp INTEGER, PRIMARY KEY(peerId, namespace, id))") @@ -68,49 +69,13 @@ public final class Postbox { self.database.execute("CREATE TABLE media_cleanup (namespace INTEGER, id INTEGER, data BLOB, PRIMARY KEY(namespace, id))") //peer_entries - self.database.execute("CREATE TABLE peer_entries (entry BLOB)") + self.database.execute("CREATE TABLE peer_entries (peerId INTEGER PRIMARY KEY, entry BLOB)") self.database.execute("CREATE INDEX peer_entries_entry on peer_entries (entry)") //peers self.database.execute("CREATE TABLE peers (peerId INTEGER PRIMARY KEY, data BLOB)") } - public func _testBlobsPrepare(range: Int) { - self.database.execute("CREATE TABLE test_blobs (number INTEGER, data BLOB)") - self.database.execute("CREATE INDEX test_blobs_index_data ON test_blobs (data)") - - self.database.transaction() - let insert = self.database.prepare("INSERT INTO test_blobs (number, data) VALUES (?, ?)") - - for i in 0 ..< range { - var value = Int32(bigEndian: Int32(i)) - let blob = Blob(bytes: &value, length: 4) - insert.run(Int64(i), blob) - } - self.database.commit() - } - - public func _testBlobsTest(range: Int) { - let select = self.database.prepare("SELECT number FROM test_blobs WHERE data < ? ORDER BY data DESC LIMIT 1") - - for i in 1 ..< 1000 { - var value = 1 + Int32(arc4random_uniform(UInt32(range - 1))) - var binary = Int32(bigEndian: value) - var found = false - for row in select.run(Blob(bytes: &binary, length: 4)) { - found = true - let selected = Int32(row[0] as! Int64) - if selected != value - 1 { - assertionFailure("invalid value \(selected) != \(value - 1)") - } - break - } - if !found { - assertionFailure("value not found for \(value - 1)") - } - } - } - private class func peerViewEntryIndexForBlob(blob: Blob) -> PeerViewEntryIndex { let buffer = ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false) var offset: Int = 0 @@ -357,7 +322,7 @@ public final class Postbox { var messageIdsByMediaId: [MediaId : [MessageId]] = [:] for (peerId, peerMessages) in Postbox.messagesGroupedByPeerId(messages) { - var maxTimestamp: Int32 = Int32.min + var maxMessage: (MessageIndex, Message)? var messageIds: [MessageId] = [] var seenMessageIds = Set() @@ -402,8 +367,9 @@ public final class Postbox { record.0.add(message) } - if maxTimestamp == Int32.min || message.timestamp > maxTimestamp { - maxTimestamp = message.timestamp + let index = MessageIndex(message) + if maxMessage == nil || index > maxMessage!.0 { + maxMessage = (index, message) } encoder.reset() @@ -430,17 +396,8 @@ public final class Postbox { record.1.putNext(MessageView(record.0)) } - var currentMaxTimestamp: Int32? - for row in self.database.prepare("SELECT timestamp FROM top_messages WHERE peerId = ?").run(peerId.toInt64()) { - currentMaxTimestamp = Int32(row[0] as! Int64) - } - - if let currentMaxTimestamp = currentMaxTimestamp { - if maxTimestamp > currentMaxTimestamp { - self.database.prepare("UPDATE top_messages SET timestamp = ? WHERE peerId = ?").run(Int64(maxTimestamp), peerId.toInt64()) - } - } else { - self.database.prepare("INSERT INTO top_messages(peerId, timestamp) VALUES(?, ?)").run(peerId.toInt64(), Int64(maxTimestamp)) + if let maxMessage = maxMessage { + self.updatePeerEntry(peerId, message: maxMessage.1) } } @@ -546,25 +503,79 @@ public final class Postbox { return grouped } - private func updateTopMessageTimestampForPeerId(peerId: PeerId) { - let selectIdStatement = self.database.prepare("SELECT MAX(id) FROM peer_messages WHERE peerId = ? AND namespace = ?") - let selectTimestampStatement = self.database.prepare("SELECT timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id = ?") - let updateTimestampStatement = self.database.prepare("UPDATE top_messages SET timestamp = ? WHERE peerId = ?") - - var maxTimestamp = Int32.min - for namespace in self.messageNamespaces { - for rowId in selectIdStatement.run(peerId.toInt64(), Int64(namespace)) { - for rowTimestamp in selectTimestampStatement.run(peerId.toInt64(), Int64(namespace), rowId[0]) { - let timestamp = Int32(rowTimestamp[0] as! Int64) - if timestamp > maxTimestamp { - maxTimestamp = timestamp - } - } + private func peerWithId(peerId: PeerId) -> Peer? { + for row in self.database.prepare("SELECT data FROM peers WHERE peerId = ?").run(peerId.toInt64()) { + let data = (row[0] as! Blob).data + let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) + if let peer = decoder.decodeRootObject() as? Peer { + return peer + } else { + println("(PostBox: can't decode peer)") } + + break } - if maxTimestamp != Int32.min { - updateTimestampStatement.run(Int64(maxTimestamp), peerId.toInt64()) + return nil + } + + private func updatePeerEntry(peerId: PeerId, message: Message?, replace: Bool = false) { + var currentIndex: PeerViewEntryIndex? + for row in self.database.prepare("SELECT entry FROM peer_entries WHERE peerId = ?").run(peerId.toInt64()) { + currentIndex = Postbox.peerViewEntryIndexForBlob(row[0] as! Blob) + break + } + + var updatedPeerMessage: Message? + + if let currentIndex = currentIndex { + if let message = message { + let messageIndex = MessageIndex(message) + if replace || currentIndex.messageIndex < messageIndex { + let updatedIndex = PeerViewEntryIndex(peerId: peerId, messageIndex: messageIndex) + updatedPeerMessage = message + let updatedBlob = Postbox.blobForPeerViewEntryIndex(updatedIndex) + self.database.prepare("UPDATE peer_entries SET entry = ? WHERE peerId = ?").run(updatedBlob, peerId.toInt64()) + } + } else if replace { + //TODO: remove? + } + } else if let message = message { + updatedPeerMessage = message + let updatedIndex = PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(message)) + let updatedBlob = Postbox.blobForPeerViewEntryIndex(updatedIndex) + self.database.prepare("INSERT INTO peer_entries (peerId, entry) VALUES (?, ?)").run(peerId.toInt64(), updatedBlob) + } + + if let updatedPeerMessage = updatedPeerMessage { + var peer: Peer? + for (view, sink) in self.peerViews.copyItems() { + + if peer == nil { + for entry in view.entries { + if entry.peerId == peerId { + peer = entry.peer + break + } + } + + if peer == nil { + peer = self.peerWithId(peerId) + } + } + + let entry: PeerViewEntry + if let peer = peer { + entry = PeerViewEntry(peer: peer, message: updatedPeerMessage) + } else { + entry = PeerViewEntry(peerId: peerId, message: updatedPeerMessage) + } + + let context = view.removeEntry(nil, peerId: peerId) + view.addEntry(entry) + + sink.putNext(PeerView(view)) + } } } @@ -573,6 +584,19 @@ public final class Postbox { let messageIdsByNamespace = Postbox.messageIdsGroupedByNamespace(messageIds) let messageIdsByMediaId = self.loadMessageIdsByMediaIdForPeerId(peerId, idsByNamespace: messageIdsByNamespace) + for (peerId, messageIds) in Postbox.messageIdsGroupedByPeerId(ids) { + if let relatedViews = self.peerMessageViews[peerId] { + for (view, sink) in relatedViews.copyItems() { + let context = view.remove(Set(messageIds)) + if !context.empty() { + view.complete(context, fetchEarlier: self.fetchMessagesRelative(peerId, earlier: true), fetchLater: self.fetchMessagesRelative(peerId, earlier: false)) + sink.putNext(MessageView(view)) + } + } + } + } + + for (namespace, messageIds) in messageIdsByNamespace { var queryString = "DELETE FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (" var first = true @@ -641,7 +665,9 @@ public final class Postbox { } } - self.updateTopMessageTimestampForPeerId(peerId) + let tail = self.fetchMessagesTail(peerId, count: 1) + + self.updatePeerEntry(peerId, message: tail.first, replace: true) } } @@ -653,12 +679,6 @@ public final class Postbox { } } - private struct FetchedMessage { - let id: MessageId - let timestamp: Int32 - let data: NSData - } - private func findAdjacentMessageIds(peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex) -> (MessageId.Id?, MessageId.Id?) { var minId: MessageId.Id? var maxId: MessageId.Id? @@ -915,6 +935,71 @@ public final class Postbox { return messages } + private func fetchPeerEntryIndicesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntryIndex] { + var entries: [PeerViewEntryIndex] = [] + + let rows: Statement + + if let index = index { + let bound = Postbox.blobForPeerViewEntryIndex(index) + let sign = earlier ? "<" : ">" + let order = earlier ? "DESC" : "ASC" + let statement = self.database.prepareCached("SELECT entry FROM peer_entries WHERE entry \(sign) ? ORDER BY entry \(order) LIMIT ?") + rows = statement.run(bound, Int64(count)) + } else { + let order = earlier ? "DESC" : "ASC" + let statement = self.database.prepareCached("SELECT entry FROM peer_entries ORDER BY entry \(order) LIMIT ?") + rows = statement.run(Int64(count)) + } + + return entries + } + + private func messageForPeer(peerId: PeerId, id: MessageId) -> Message? { + for row in self.database.prepareCached("SELECT data, associatedMediaIds FROM peer_messages WHERE peerId = ? AND namespace = ? AND id = ?").run(peerId.toInt64(), Int64(id.namespace), Int64(id.id)) { + let data = (row[0] as! Blob).data + let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) + if let message = decoder.decodeRootObject() as? Message { + return message + } else { + println("(PostBox: can't decode message)") + } + break + } + + return nil + } + + private func fetchPeerEntriesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntry] { + var entries: [PeerViewEntry] = [] + var peers: [PeerId : Peer] = [:] + for entryIndex in self.fetchPeerEntryIndicesRelative(earlier)(index: index, count: count) { + var peer: Peer? + + if let cachedPeer = peers[entryIndex.peerId] { + peer = cachedPeer + } else { + if let fetchedPeer = self.peerWithId(entryIndex.peerId) { + peer = fetchedPeer + peers[fetchedPeer.id] = fetchedPeer + } + } + + if let message = self.messageForPeer(entryIndex.peerId, id: entryIndex.messageIndex.id) { + let entry: PeerViewEntry + if let peer = peer { + entry = PeerViewEntry(peer: peer, message: message) + } else { + entry = PeerViewEntry(peerId: entryIndex.peerId, message: message) + } + } else { + println("(PostBox: missing message for peer entry)") + } + } + + return entries + } + private func fetchMessagesTail(peerId: PeerId, count: Int) -> [Message] { var messages: [Message] = [] @@ -1061,6 +1146,57 @@ public final class Postbox { } } + public func tailPeerView(count: Int) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.queue.dispatch { + let tail = self.fetchPeerEntriesRelative(true)(index: nil, count: count + 1) + + var entries: [PeerViewEntry] = [] + var i = tail.count - 1 + while i >= 0 && i >= tail.count - count { + entries.insert(tail[i], atIndex: 0) + i-- + } + + var earlier: PeerViewEntry? + + for namespace in self.messageNamespaces { + var i = tail.count - count - 1 + while i >= 0 { + earlier = tail[i] + break + } + } + + let mutableView = MutablePeerView(count: count, earlier: earlier, entries: entries, later: nil) + let record = (mutableView, Pipe()) + + let index = self.peerViews.add(record) + + subscriber.putNext(PeerView(mutableView)) + + let pipeDisposable = record.1.signal().start(next: { next in + subscriber.putNext(next) + }) + + disposable.set(ActionDisposable { [weak self] in + pipeDisposable.dispose() + + if let strongSelf = self { + strongSelf.queue.dispatch { + strongSelf.peerViews.remove(index) + } + } + return + }) + } + + return disposable + } + } + func printMessages(messages: [Message]) { var string = "" string += "[" @@ -1078,13 +1214,6 @@ public final class Postbox { } public func _dumpTables() { - println("\n-----------") - println("top_messages") - println("------------") - for row in self.database.prepare("SELECT peerId, timestamp FROM top_messages ORDER BY timestamp DESC").run() { - println("\(PeerId(row[0] as! Int64)): \(row[1] as! Int64)") - } - println("\n------------") println("peer_messages") println("-------------") diff --git a/PostboxTests/PostboxTests.swift b/PostboxTests/PostboxTests.swift index 957984c544..6b1fca9f4c 100644 --- a/PostboxTests/PostboxTests.swift +++ b/PostboxTests/PostboxTests.swift @@ -678,7 +678,7 @@ class PostboxTests: XCTestCase { } func testPeerView() { - let view = MutablePeerView(tags: [], count: 3, earlier: nil, entries: [], later: nil) + let view = MutablePeerView(count: 3, earlier: nil, entries: [], later: nil) let messageNamespaceCloud = TestMessageNamespace.Cloud.rawValue let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) @@ -694,7 +694,7 @@ class PostboxTests: XCTestCase { } else { string += ", " } - string += "(p \(entry.peer.id.namespace):\(entry.peer.id.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" + string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" } string += "]" println("\(string)") @@ -711,7 +711,7 @@ class PostboxTests: XCTestCase { } func remove(peerId: PeerId, context: MutablePeerView.RemoveContext? = nil) -> MutablePeerView.RemoveContext { - entries = entries.filter({ $0.peer.id != peerId }) + entries = entries.filter({ $0.peerId != peerId }) return view.removeEntry(context, peerId: peerId) } @@ -773,4 +773,46 @@ class PostboxTests: XCTestCase { println("\(view)") } } + + func testPeerViewTail() { + declareEncodable(TestMessage.self, { TestMessage(decoder: $0) }) + declareEncodable(TestMedia.self, { TestMedia(decoder: $0) }) + + let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) + let messageNamespace = TestMessageNamespace.Cloud.rawValue + + let basePath = "/tmp/postboxtest" + NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) + let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) + + postbox.modify { state in + let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) + for i in 0 ..< 10 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) + let message = TestMessage(id: messageId, authorId: otherId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) + state.addMessages([message, message], medias: [testMedia]) + } + return + } + + postbox.tailPeerView(3).start(next: { next in + println(next) + }) + + postbox.tailMessageViewForPeerId(otherId, count: 4).start(next: { next in + println(next) + }) + + postbox.modify { state in + let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) + for i in 10 ..< 15 { + let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) + let message = TestMessage(id: messageId, authorId: otherId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) + state.addMessages([message, message], medias: [testMedia]) + } + return + } + + postbox._sync() + } } From f74a350697f0fc8ac71edaf585ad363014ad7973 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 16 Jul 2015 15:45:45 +0300 Subject: [PATCH 009/427] no message --- submodules/SQLite.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/SQLite.swift b/submodules/SQLite.swift index b3a75ca484..e53267d739 160000 --- a/submodules/SQLite.swift +++ b/submodules/SQLite.swift @@ -1 +1 @@ -Subproject commit b3a75ca4840bd8a541825cfd6ce14b5185985ac6 +Subproject commit e53267d7394c6df648d588bebc4db10dc853c75d From 95e2d53dd90d3e88c632fe00fc429e6dd13bff23 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 16 Jul 2015 18:27:59 +0300 Subject: [PATCH 010/427] no message --- Postbox/Postbox.swift | 82 ++++++++++++++++++++++++--------- PostboxTests/PostboxTests.swift | 52 ++++++++++++--------- 2 files changed, 89 insertions(+), 45 deletions(-) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index b5a6f87549..4fd0ae69a8 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -2,25 +2,27 @@ import Foundation import SwiftSignalKit -public final class Postbox { - static let PeerTagGeneral = 0 +public class Modifier { + private unowned var postbox: Postbox - public class Modifier { - private unowned var postbox: Postbox - - private init(postbox: Postbox) { - self.postbox = postbox - } - - public func addMessages(messages: [Message], medias: [Media]) { - self.postbox.addMessages(messages, medias: medias) - } - - public func deleteMessagesWithIds(ids: [MessageId]) { - self.postbox.deleteMessagesWithIds(ids) - } + private init(postbox: Postbox) { + self.postbox = postbox } + public func addMessages(messages: [Message], medias: [Media]) { + self.postbox.addMessages(messages, medias: medias) + } + + public func deleteMessagesWithIds(ids: [MessageId]) { + self.postbox.deleteMessagesWithIds(ids) + } + + public func setState(state: Coding) { + self.postbox.setState(state) + } +} + +public final class Postbox { private let basePath: String private let messageNamespaces: [MessageId.Namespace] @@ -29,6 +31,7 @@ public final class Postbox { private var peerMessageViews: [PeerId : Bag<(MutableMessageView, Pipe)>] = [:] private var peerViews: Bag<(MutablePeerView, Pipe)> = Bag() + private var statePipe: Pipe = Pipe() public init(basePath: String, messageNamespaces: [MessageId.Namespace]) { self.basePath = basePath @@ -53,7 +56,7 @@ public final class Postbox { private func createSchema() { //state - self.database.execute("CREATE TABLE state (data BLOB)") + self.database.execute("CREATE TABLE state (id INTEGER, data BLOB)") //peer_messages self.database.execute("CREATE TABLE peer_messages (peerId INTEGER, namespace INTEGER, id INTEGER, data BLOB, associatedMediaIds BLOB, timestamp INTEGER, PRIMARY KEY(peerId, namespace, id))") @@ -311,6 +314,34 @@ public final class Postbox { return ids } + private func setState(state: Coding) { + let encoder = Encoder() + encoder.encodeRootObject(state) + let blob = Blob(data: encoder.makeData()) + self.database.prepareCached("INSERT OR REPLACE INTO state (id, data) VALUES (?, ?)").run(Int64(0), blob) + } + + public func state() -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.dispatch { + for row in self.database.prepareCached("SELECT data FROM state WHERE id = ?").run(Int64(0)) { + let data = (row[0] as! Blob).data + let buffer = ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false) + let decoder = Decoder(buffer: buffer) + if let state = decoder.decodeRootObject() as? State { + subscriber.putNext(state) + } + break + } + disposable.set(self.statePipe.signal().start(next: { next in + subscriber.putNext(next) + })) + } + return disposable + } + } + private func addMessages(messages: [Message], medias: [Media]) { let messageInsertStatement = self.database.prepare("INSERT INTO peer_messages (peerId, namespace, id, data, associatedMediaIds, timestamp) VALUES (?, ?, ?, ?, ?, ?)") let peerMediaInsertStatement = self.database.prepare("INSERT INTO peer_media (peerId, mediaNamespace, messageNamespace, messageId) VALUES (?, ?, ?, ?)") @@ -596,7 +627,6 @@ public final class Postbox { } } - for (namespace, messageIds) in messageIdsByNamespace { var queryString = "DELETE FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (" var first = true @@ -671,11 +701,17 @@ public final class Postbox { } } - public func modify(f: Modifier -> Void) { - self.queue.dispatch { - self.database.transaction() - f(Modifier(postbox: self)) - self.database.commit() + public func modify(f: Modifier -> T) -> Signal { + return Signal { subscriber in + self.queue.dispatch { + self.database.transaction() + let result = f(Modifier(postbox: self)) + self.database.commit() + + subscriber.putNext(result) + subscriber.putCompletion() + } + return EmptyDisposable } } diff --git a/PostboxTests/PostboxTests.swift b/PostboxTests/PostboxTests.swift index 6b1fca9f4c..effe0caeaa 100644 --- a/PostboxTests/PostboxTests.swift +++ b/PostboxTests/PostboxTests.swift @@ -93,6 +93,14 @@ class TestMedia: Media { } } +class EmptyState: Coding { + required init(decoder: Decoder) { + } + + func encode(encoder: Encoder) { + } +} + class PostboxTests: XCTestCase { override func setUp() { @@ -113,8 +121,8 @@ class PostboxTests: XCTestCase { let basePath = "/tmp/postboxtest" NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) - postbox.modify { state in + let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) + (postbox.modify { state -> Void in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) for i in 0 ..< 10 { let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) @@ -122,27 +130,27 @@ class PostboxTests: XCTestCase { state.addMessages([message, message], medias: [testMedia]) } return - } + }).start() postbox._dumpTables() - postbox.modify { state in + (postbox.modify { state -> Void in var messageIds: [MessageId] = [] for i in 0 ..< 5 { let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) messageIds.append(messageId) } state.deleteMessagesWithIds(messageIds) - } + }).start() postbox._dumpTables(); - postbox.modify { state in + (postbox.modify { state -> Void in var messageIds: [MessageId] = [] for i in 0 ..< 10 { let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) messageIds.append(messageId) } state.deleteMessagesWithIds(messageIds) - } + }).start() postbox._dumpTables(); } @@ -602,9 +610,9 @@ class PostboxTests: XCTestCase { let basePath = "/tmp/postboxtest" NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) + let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) - postbox.modify { state in + (postbox.modify { state -> Void in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) for i in 0 ..< 10 { let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) @@ -612,13 +620,13 @@ class PostboxTests: XCTestCase { state.addMessages([message, message], medias: [testMedia]) } return - } + }).start() postbox.tailMessageViewForPeerId(otherId, count: 4).start(next: { next in println(next) }) - postbox.modify { state in + (postbox.modify { state -> Void in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) for i in 10 ..< 15 { let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) @@ -626,7 +634,7 @@ class PostboxTests: XCTestCase { state.addMessages([message, message], medias: [testMedia]) } return - } + }).start() postbox._sync() } @@ -642,9 +650,9 @@ class PostboxTests: XCTestCase { let basePath = "/tmp/postboxtest" NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespaceCloud, messageNamespaceLocal]) + let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespaceCloud, messageNamespaceLocal]) - postbox.modify { state in + (postbox.modify { state -> Void in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) for i in 0 ..< 10 { let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i + 1)) @@ -657,14 +665,14 @@ class PostboxTests: XCTestCase { state.addMessages([message, message], medias: [testMedia]) } return - } + }).start() var i = 1000 postbox.aroundMessageViewForPeerId(otherId, id: MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i + 1)), count: 3).start(next: { next in println(next) }) - postbox.modify { state in + (postbox.modify { state -> Void in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) for i in 10 ..< 15 { let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i + 1)) @@ -672,7 +680,7 @@ class PostboxTests: XCTestCase { state.addMessages([message, message], medias: [testMedia]) } return - } + }).start() postbox._sync() } @@ -783,9 +791,9 @@ class PostboxTests: XCTestCase { let basePath = "/tmp/postboxtest" NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) + let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) - postbox.modify { state in + (postbox.modify { state -> Void in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) for i in 0 ..< 10 { let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) @@ -793,7 +801,7 @@ class PostboxTests: XCTestCase { state.addMessages([message, message], medias: [testMedia]) } return - } + }).start() postbox.tailPeerView(3).start(next: { next in println(next) @@ -803,7 +811,7 @@ class PostboxTests: XCTestCase { println(next) }) - postbox.modify { state in + (postbox.modify { state -> Void in let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) for i in 10 ..< 15 { let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) @@ -811,7 +819,7 @@ class PostboxTests: XCTestCase { state.addMessages([message, message], medias: [testMedia]) } return - } + }).start() postbox._sync() } From 126d711c0fc1c77f866d015c4a7124d1e420e07a Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 16 Jul 2015 21:19:12 +0300 Subject: [PATCH 011/427] no message --- Postbox/Postbox.swift | 30 +++++++++++++++++++++--------- PostboxTests/PostboxTests.swift | 2 +- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 4fd0ae69a8..21d5fbb882 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -2,7 +2,10 @@ import Foundation import SwiftSignalKit -public class Modifier { +public protocol PostboxState: Coding { +} + +public class Modifier { private unowned var postbox: Postbox private init(postbox: Postbox) { @@ -17,12 +20,12 @@ public class Modifier { self.postbox.deleteMessagesWithIds(ids) } - public func setState(state: Coding) { + public func setState(state: State) { self.postbox.setState(state) } } -public final class Postbox { +public final class Postbox { private let basePath: String private let messageNamespaces: [MessageId.Namespace] @@ -314,26 +317,35 @@ public final class Postbox { return ids } - private func setState(state: Coding) { - let encoder = Encoder() - encoder.encodeRootObject(state) - let blob = Blob(data: encoder.makeData()) - self.database.prepareCached("INSERT OR REPLACE INTO state (id, data) VALUES (?, ?)").run(Int64(0), blob) + private func setState(state: State) { + self.queue.dispatch { + let encoder = Encoder() + encoder.encodeRootObject(state) + let blob = Blob(data: encoder.makeData()) + self.database.prepareCached("INSERT OR REPLACE INTO state (id, data) VALUES (?, ?)").run(Int64(0), blob) + + self.statePipe.putNext(state) + } } - public func state() -> Signal { + public func state() -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.queue.dispatch { + var found = false for row in self.database.prepareCached("SELECT data FROM state WHERE id = ?").run(Int64(0)) { let data = (row[0] as! Blob).data let buffer = ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false) let decoder = Decoder(buffer: buffer) if let state = decoder.decodeRootObject() as? State { + found = true subscriber.putNext(state) } break } + if !found { + subscriber.putNext(nil) + } disposable.set(self.statePipe.signal().start(next: { next in subscriber.putNext(next) })) diff --git a/PostboxTests/PostboxTests.swift b/PostboxTests/PostboxTests.swift index effe0caeaa..8b905a676d 100644 --- a/PostboxTests/PostboxTests.swift +++ b/PostboxTests/PostboxTests.swift @@ -93,7 +93,7 @@ class TestMedia: Media { } } -class EmptyState: Coding { +class EmptyState: PostboxState { required init(decoder: Decoder) { } From d3a85be396d550470e89251eec6be7928e3b0e2a Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 16 Jul 2015 23:25:26 +0300 Subject: [PATCH 012/427] no message --- Postbox/Postbox.swift | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 21d5fbb882..91ccd421b5 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -3,25 +3,26 @@ import Foundation import SwiftSignalKit public protocol PostboxState: Coding { + } public class Modifier { - private unowned var postbox: Postbox + private weak var postbox: Postbox? private init(postbox: Postbox) { self.postbox = postbox } public func addMessages(messages: [Message], medias: [Media]) { - self.postbox.addMessages(messages, medias: medias) + self.postbox?.addMessages(messages, medias: medias) } public func deleteMessagesWithIds(ids: [MessageId]) { - self.postbox.deleteMessagesWithIds(ids) + self.postbox?.deleteMessagesWithIds(ids) } public func setState(state: State) { - self.postbox.setState(state) + self.postbox?.setState(state) } } @@ -61,6 +62,9 @@ public final class Postbox { //state self.database.execute("CREATE TABLE state (id INTEGER, data BLOB)") + //keychain + self.database.execute("CREATE TABLE keychain (key BLOB, data BLOB)") + //peer_messages self.database.execute("CREATE TABLE peer_messages (peerId INTEGER, namespace INTEGER, id INTEGER, data BLOB, associatedMediaIds BLOB, timestamp INTEGER, PRIMARY KEY(peerId, namespace, id))") @@ -354,6 +358,32 @@ public final class Postbox { } } + public func keychainEntryForKey(key: String) -> Signal { + return Signal { subscriber in + self.queue.dispatch { + let blob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) + var buffer: ReadBuffer? + for row in self.database.prepareCached("SELECT data FROM keychain WHERE key = ?").run(blob) { + let data = (row[0] as! Blob).data + let memory = malloc(data.length) + memcpy(memory, data.bytes, data.length) + buffer = ReadBuffer(memory: UnsafeMutablePointer(memory), length: data.length, freeWhenDone: true) + break + } + subscriber.putNext(buffer) + subscriber.putCompletion() + } + return EmptyDisposable + } + } + + public func setKeychainEntry(key: String, value: NSData) { + self.queue.dispatch { + let keyBlob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) + self.database.prepareCached("INSERT OR REPLACE INTO keychain (key, data) VALUES (?, ?)").run(keyBlob, Blob(data: value)) + } + } + private func addMessages(messages: [Message], medias: [Media]) { let messageInsertStatement = self.database.prepare("INSERT INTO peer_messages (peerId, namespace, id, data, associatedMediaIds, timestamp) VALUES (?, ?, ?, ?, ?, ?)") let peerMediaInsertStatement = self.database.prepare("INSERT INTO peer_media (peerId, mediaNamespace, messageNamespace, messageId) VALUES (?, ?, ?, ?)") From 00a69da9a321f6234604331e4912a85bea44d37d Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 16 Jul 2015 23:55:19 +0300 Subject: [PATCH 013/427] no message --- Postbox/Postbox.swift | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 91ccd421b5..8367aa60dc 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -358,23 +358,17 @@ public final class Postbox { } } - public func keychainEntryForKey(key: String) -> Signal { - return Signal { subscriber in - self.queue.dispatch { - let blob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) - var buffer: ReadBuffer? - for row in self.database.prepareCached("SELECT data FROM keychain WHERE key = ?").run(blob) { - let data = (row[0] as! Blob).data - let memory = malloc(data.length) - memcpy(memory, data.bytes, data.length) - buffer = ReadBuffer(memory: UnsafeMutablePointer(memory), length: data.length, freeWhenDone: true) - break - } - subscriber.putNext(buffer) - subscriber.putCompletion() + public func keychainEntryForKey(key: String) -> NSData? { + var result: NSData? + self.queue.sync { + let blob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) + var buffer: ReadBuffer? + for row in self.database.prepareCached("SELECT data FROM keychain WHERE key = ?").run(blob) { + result = (row[0] as! Blob).data + break } - return EmptyDisposable } + return result } public func setKeychainEntry(key: String, value: NSData) { From 8f23c15be30c56cebc36387753bc16a4184bde3a Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 17 Jul 2015 00:06:33 +0300 Subject: [PATCH 014/427] no message --- Postbox/Postbox.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 8367aa60dc..0a96e09511 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -359,6 +359,8 @@ public final class Postbox { } public func keychainEntryForKey(key: String) -> NSData? { + //TODO: load keychain on first request then sync to disk + var result: NSData? self.queue.sync { let blob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) From ac1cc0b5ccfe15b6b700fcef2e6be3a70d73b9a3 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 17 Jul 2015 00:16:18 +0300 Subject: [PATCH 015/427] no message --- Postbox/Postbox.swift | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 0a96e09511..6fbc9a620d 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -373,13 +373,19 @@ public final class Postbox { return result } - public func setKeychainEntry(key: String, value: NSData) { + public func setKeychainEntryForKey(key: String, value: NSData) { self.queue.dispatch { let keyBlob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) self.database.prepareCached("INSERT OR REPLACE INTO keychain (key, data) VALUES (?, ?)").run(keyBlob, Blob(data: value)) } } + public func removeKeychainEntryForKey(key: String) { + self.queue.dispatch { + self.database.prepareCached("DELETE FROM keychain WHERE key = ?").run(keyBlob) + } + } + private func addMessages(messages: [Message], medias: [Media]) { let messageInsertStatement = self.database.prepare("INSERT INTO peer_messages (peerId, namespace, id, data, associatedMediaIds, timestamp) VALUES (?, ?, ?, ?, ?, ?)") let peerMediaInsertStatement = self.database.prepare("INSERT INTO peer_media (peerId, mediaNamespace, messageNamespace, messageId) VALUES (?, ?, ?, ?)") From d95f61a82a32b3ba1158c3faaaa3e10156b968f7 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 17 Jul 2015 00:18:08 +0300 Subject: [PATCH 016/427] no message --- Postbox/Postbox.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 6fbc9a620d..ac11f60828 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -382,6 +382,7 @@ public final class Postbox { public func removeKeychainEntryForKey(key: String) { self.queue.dispatch { + let keyBlob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) self.database.prepareCached("DELETE FROM keychain WHERE key = ?").run(keyBlob) } } From 6889f59981e8bab450cc4a57a1bc5afe5f9ff601 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 27 Jul 2015 15:02:47 +0300 Subject: [PATCH 017/427] no message --- Postbox.xcodeproj/project.pbxproj | 17 +-- Postbox/Coding.swift | 30 +++++- Postbox/Media.swift | 3 +- Postbox/Message.swift | 4 +- Postbox/MessageView.swift | 10 +- Postbox/Peer.swift | 2 +- Postbox/PeerView.swift | 14 +-- Postbox/Postbox.swift | 167 ++++++++++++++++-------------- 8 files changed, 132 insertions(+), 115 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index ff36d79283..4533d9cee2 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -23,11 +23,7 @@ D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; D07FC7D31B4A3D6B0010B3F7 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */; }; - D0D224F01B4D6ABD0085E26D /* Expression.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EB1B4D6ABD0085E26D /* Expression.swift */; }; - D0D224F11B4D6ABD0085E26D /* FTS.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EC1B4D6ABD0085E26D /* FTS.swift */; }; D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; - D0D224F31B4D6ABD0085E26D /* Query.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EE1B4D6ABD0085E26D /* Query.swift */; }; - D0D224F41B4D6ABD0085E26D /* Schema.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224EF1B4D6ABD0085E26D /* Schema.swift */; }; D0D225261B4D84930085E26D /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D225251B4D84930085E26D /* PeerView.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; @@ -67,11 +63,7 @@ D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../SSignalKit/build/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; - D0D224EB1B4D6ABD0085E26D /* Expression.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Expression.swift; path = submodules/sqlite.swift/SQLite/Expression.swift; sourceTree = SOURCE_ROOT; }; - D0D224EC1B4D6ABD0085E26D /* FTS.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = FTS.swift; path = submodules/sqlite.swift/SQLite/FTS.swift; sourceTree = SOURCE_ROOT; }; D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; - D0D224EE1B4D6ABD0085E26D /* Query.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Query.swift; path = submodules/sqlite.swift/SQLite/Query.swift; sourceTree = SOURCE_ROOT; }; - D0D224EF1B4D6ABD0085E26D /* Schema.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Schema.swift; path = submodules/sqlite.swift/SQLite/Schema.swift; sourceTree = SOURCE_ROOT; }; D0D225251B4D84930085E26D /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -139,11 +131,7 @@ D07516731B2EC8C700AE42E0 /* sqlite.swift */ = { isa = PBXGroup; children = ( - D0D224EB1B4D6ABD0085E26D /* Expression.swift */, - D0D224EC1B4D6ABD0085E26D /* FTS.swift */, D0D224ED1B4D6ABD0085E26D /* Functions.swift */, - D0D224EE1B4D6ABD0085E26D /* Query.swift */, - D0D224EF1B4D6ABD0085E26D /* Schema.swift */, D07516611B2EC7FE00AE42E0 /* Database.swift */, D07516681B2EC7FE00AE42E0 /* Statement.swift */, D07516691B2EC7FE00AE42E0 /* Value.swift */, @@ -279,6 +267,7 @@ D0E3A7411B28A7E300A402D9 /* Project object */ = { isa = PBXProject; attributes = { + LastSwiftMigration = 0700; LastSwiftUpdateCheck = 0700; LastUpgradeCheck = 0630; ORGANIZATIONNAME = Telegram; @@ -332,7 +321,6 @@ buildActionMask = 2147483647; files = ( D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */, - D0D224F11B4D6ABD0085E26D /* FTS.swift in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, @@ -341,16 +329,13 @@ D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */, D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */, - D0D224F31B4D6ABD0085E26D /* Query.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */, D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, D0D225261B4D84930085E26D /* PeerView.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, - D0D224F41B4D6ABD0085E26D /* Schema.swift in Sources */, D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, - D0D224F01B4D6ABD0085E26D /* Expression.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 6900a84ea7..51f53915b0 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -20,7 +20,8 @@ private struct EncodableTypeStore { private var typeStore = EncodableTypeStore() public func declareEncodable(type: Any.Type, f: Decoder -> Coding) { - let hash = murMurHashString32("\(type)") + let string = "\(type)" + let hash = murMurHashString32(string) if typeStore.dict[hash] != nil { assertionFailure("Encodable type hash collision for \(type)") } @@ -186,7 +187,8 @@ public final class Encoder { var type: Int8 = ValueType.Object.rawValue self.buffer.write(&type, offset: 0, length: 1) - var typeHash: Int32 = murMurHashString32(_stdlib_getDemangledTypeName(value)) + let string = "\(value.dynamicType)" + var typeHash: Int32 = murMurHashString32(string) self.buffer.write(&typeHash, offset: 0, length: 4) let innerEncoder = Encoder() @@ -346,7 +348,7 @@ public final class Decoder { private class func positionOnKey(bytes: UnsafePointer, inout offset: Int, maxOffset: Int, length: Int, key: UnsafePointer, valueType: ValueType) -> Bool { - var startOffset = offset + let startOffset = offset let keyLength: Int = Int(strlen(key)) while (offset < maxOffset) @@ -444,7 +446,7 @@ public final class Decoder { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.buffer.offset, 4) - var innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(length), freeWhenDone: false)) + let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(length), freeWhenDone: false)) self.buffer.offset += 4 + Int(length) return typeStore.decode(typeHash, decoder: innerDecoder) @@ -453,6 +455,24 @@ public final class Decoder { } } + public func decodeObjectForKey(key: UnsafePointer, decoder: Decoder -> Coding) -> Coding? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { + var typeHash: Int32 = 0 + memcpy(&typeHash, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + + let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(length), freeWhenDone: false)) + self.buffer.offset += 4 + Int(length) + + return decoder(innerDecoder) + } else { + return nil + } + } + public func decodeInt32ArrayForKey(key: UnsafePointer) -> [Int32] { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32Array) { var length: Int32 = 0 @@ -512,7 +532,7 @@ public final class Decoder { var objectLength: Int32 = 0 memcpy(&objectLength, self.buffer.memory + self.buffer.offset, 4) - var innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(objectLength), freeWhenDone: false)) + let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(objectLength), freeWhenDone: false)) self.buffer.offset += 4 + Int(objectLength) if !failed { diff --git a/Postbox/Media.swift b/Postbox/Media.swift index e49661547b..a4f54c2540 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -1,6 +1,6 @@ import Foundation -public struct MediaId: Hashable, Printable { +public struct MediaId: Hashable, CustomStringConvertible { public typealias Namespace = Int32 public typealias Id = Int64 @@ -50,7 +50,6 @@ public struct MediaId: Hashable, Printable { public static func decodeArrayFromBuffer(buffer: ReadBuffer) -> [MediaId] { var length: Int32 = 0 - var offset = 0 memcpy(&length, buffer.memory, 4) buffer.offset += 4 var i = 0 diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 3a62008174..6db0eda143 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -1,6 +1,6 @@ import Foundation -public struct MessageId: Hashable, Printable { +public struct MessageId: Hashable, CustomStringConvertible { public typealias Namespace = Int32 public typealias Id = Int32 @@ -59,7 +59,6 @@ public struct MessageId: Hashable, Printable { public static func decodeArrayFromBuffer(buffer: ReadBuffer) -> [MessageId] { var length: Int32 = 0 - var offset = 0 memcpy(&length, buffer.memory, 4) buffer.offset += 4 var i = 0 @@ -109,7 +108,6 @@ public func <(lhs: MessageIndex, rhs: MessageIndex) -> Bool { public protocol Message: Coding { var id: MessageId { get } - var authorId: PeerId { get } var timestamp: Int32 { get } var text: String { get } var referencedMediaIds: [MediaId] { get } diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift index 786aa15576..a7b4eda4d9 100644 --- a/Postbox/MessageView.swift +++ b/Postbox/MessageView.swift @@ -1,6 +1,6 @@ import Foundation -public final class MutableMessageView: Printable { +public final class MutableMessageView: CustomStringConvertible { public struct RemoveContext { var invalidEarlier: Set var invalidLater: Set @@ -35,8 +35,8 @@ public final class MutableMessageView: Printable { if self.messages.count == 0 { self.messages.append(message) } else { - var first = MessageIndex(self.messages[self.messages.count - 1]) - var last = MessageIndex(self.messages[0]) + let first = MessageIndex(self.messages[self.messages.count - 1]) + let last = MessageIndex(self.messages[0]) var next: MessageIndex? for namespace in self.namespaces { @@ -156,7 +156,7 @@ public final class MutableMessageView: Printable { } addedMessages += self.messages - addedMessages.sort({ MessageIndex($0) < MessageIndex($1) }) + addedMessages.sortInPlace({ MessageIndex($0) < MessageIndex($1) }) var i = addedMessages.count - 1 while i >= 1 { if addedMessages[i].id == addedMessages[i - 1].id { @@ -299,7 +299,7 @@ public final class MutableMessageView: Printable { } } -public final class MessageView: Printable { +public final class MessageView: CustomStringConvertible { public let hasEarlier: Bool private let earlierIds: [MessageIndex] public let hasLater: Bool diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index 49491b027c..190b7bf41a 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -1,6 +1,6 @@ import Foundation -public struct PeerId: Hashable, Printable, Comparable { +public struct PeerId: Hashable, CustomStringConvertible, Comparable { public typealias Namespace = Int32 public typealias Id = Int32 diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index cf396590ce..5f77d90ded 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -53,7 +53,7 @@ public func <(lhs: PeerViewEntryIndex, rhs: PeerViewEntryIndex) -> Bool { return lhs.peerId < rhs.peerId } -public final class MutablePeerView: Printable { +public final class MutablePeerView: CustomStringConvertible { public struct RemoveContext { var invalidEarlier = false var invalidLater = false @@ -104,8 +104,8 @@ public final class MutablePeerView: Printable { if self.entries.count == 0 { self.entries.append(entry) } else { - var first = PeerViewEntryIndex(self.entries[self.entries.count - 1]) - var last = PeerViewEntryIndex(self.entries[0]) + let first = PeerViewEntryIndex(self.entries[self.entries.count - 1]) + let last = PeerViewEntryIndex(self.entries[0]) let index = PeerViewEntryIndex(entry) @@ -179,7 +179,7 @@ public final class MutablePeerView: Printable { } addedEntries += self.entries - addedEntries.sort({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) + addedEntries.sortInPlace({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) var i = addedEntries.count - 1 while i >= 1 { @@ -204,7 +204,7 @@ public final class MutablePeerView: Printable { self.later = nil if anchorIndex + 1 < addedEntries.count { - var i = anchorIndex + 1 + let i = anchorIndex + 1 while i < addedEntries.count { self.later = addedEntries[i] break @@ -283,10 +283,10 @@ public final class MutablePeerView: Printable { } } -public final class PeerView: Printable { +public final class PeerView: CustomStringConvertible { let earlier: PeerViewEntryIndex? let later: PeerViewEntryIndex? - let entries: [PeerViewEntry] + public let entries: [PeerViewEntry] init(_ mutableView: MutablePeerView) { if let earlier = mutableView.earlier { diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index ac11f60828..5c8e08ec7f 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -21,6 +21,10 @@ public class Modifier { self.postbox?.deleteMessagesWithIds(ids) } + public func getState() -> State? { + return self.postbox?.getState() + } + public func setState(state: State) { self.postbox?.setState(state) } @@ -44,26 +48,31 @@ public final class Postbox { } private func openDatabase() { - NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil, error: nil) + do { + try NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil) + } catch _ { + } self.database = Database(basePath.stringByAppendingPathComponent("db")) self.queue.dispatch { - let result = self.database.userVersion + let result = self.database.scalar("PRAGMA user_version") as! Int64 if result == 1 { - println("(Postbox schema version \(result))") + print("(Postbox schema version \(result))") } else { - println("(Postbox creating schema)") + print("(Postbox creating schema)") self.createSchema() + self.database.execute("PRAGMA user_version = 1") } } } private func createSchema() { //state - self.database.execute("CREATE TABLE state (id INTEGER, data BLOB)") + self.database.execute("CREATE TABLE state (id INTEGER PRIMARY KEY, data BLOB)") //keychain self.database.execute("CREATE TABLE keychain (key BLOB, data BLOB)") + self.database.execute("CREATE INDEX keychain_key ON keychain (key)") //peer_messages self.database.execute("CREATE TABLE peer_messages (peerId INTEGER, namespace INTEGER, id INTEGER, data BLOB, associatedMediaIds BLOB, timestamp INTEGER, PRIMARY KEY(peerId, namespace, id))") @@ -88,27 +97,22 @@ public final class Postbox { private class func peerViewEntryIndexForBlob(blob: Blob) -> PeerViewEntryIndex { let buffer = ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false) - var offset: Int = 0 - + var timestamp: Int32 = 0 - buffer.read(×tamp, offset: offset, length: 4) + buffer.read(×tamp, offset: 0, length: 4) timestamp = Int32(bigEndian: timestamp) - offset += 4 var namespace: Int32 = 0 - buffer.read(&namespace, offset: offset, length: 4) + buffer.read(&namespace, offset: 0, length: 4) namespace = Int32(bigEndian: namespace) - offset += 4 var id: Int32 = 0 - buffer.read(&id, offset: offset, length: 4) + buffer.read(&id, offset: 0, length: 4) id = Int32(bigEndian: id) - offset += 4 var peerIdRepresentation: Int64 = 0 - buffer.read(&peerIdRepresentation, offset: offset, length: 8) + buffer.read(&peerIdRepresentation, offset: 0, length: 8) peerIdRepresentation = Int64(bigEndian: peerIdRepresentation) - offset += 8 let peerId = PeerId(peerIdRepresentation) @@ -169,7 +173,7 @@ public final class Postbox { var grouped: [MediaId.Namespace : [MediaId]] = [:] for id in ids { - if var groupedIds = grouped[id.namespace] { + if let _ = grouped[id.namespace] { grouped[id.namespace]!.append(id) } else { grouped[id.namespace] = [id] @@ -332,24 +336,24 @@ public final class Postbox { } } + private func getState() -> State? { + for row in self.database.prepareCached("SELECT data FROM state WHERE id = ?").run(Int64(0)) { + let data = (row[0] as! Blob).data + let buffer = ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false) + let decoder = Decoder(buffer: buffer) + if let state = decoder.decodeRootObject() as? State { + return state + } + break + } + return nil + } + public func state() -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.queue.dispatch { - var found = false - for row in self.database.prepareCached("SELECT data FROM state WHERE id = ?").run(Int64(0)) { - let data = (row[0] as! Blob).data - let buffer = ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false) - let decoder = Decoder(buffer: buffer) - if let state = decoder.decodeRootObject() as? State { - found = true - subscriber.putNext(state) - } - break - } - if !found { - subscriber.putNext(nil) - } + subscriber.putNext(self.getState()) disposable.set(self.statePipe.signal().start(next: { next in subscriber.putNext(next) })) @@ -361,22 +365,24 @@ public final class Postbox { public func keychainEntryForKey(key: String) -> NSData? { //TODO: load keychain on first request then sync to disk - var result: NSData? - self.queue.sync { - let blob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) - var buffer: ReadBuffer? - for row in self.database.prepareCached("SELECT data FROM keychain WHERE key = ?").run(blob) { - result = (row[0] as! Blob).data - break - } + let blob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) + for row in self.database.prepareCached("SELECT data FROM keychain WHERE key = ?").run(blob) { + return (row[0] as! Blob).data } - return result + return nil } public func setKeychainEntryForKey(key: String, value: NSData) { - self.queue.dispatch { - let keyBlob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) - self.database.prepareCached("INSERT OR REPLACE INTO keychain (key, data) VALUES (?, ?)").run(keyBlob, Blob(data: value)) + let keyBlob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) + var rowId: Int64? + for row in self.database.prepareCached("SELECT rowid FROM keychain WHERE key = ? LIMIT 1").run(keyBlob) { + rowId = row[0] as? Int64 + break + } + if let rowId = rowId { + self.database.prepareCached("UPDATE keychain SET data = ? WHERE rowid = ?").run(Blob(data: value), rowId) + } else { + self.database.prepareCached("INSERT INTO keychain (key, data) VALUES (?, ?)").run(keyBlob, Blob(data: value)) } } @@ -431,7 +437,7 @@ public final class Postbox { } } - var relatedViews = self.peerMessageViews[peerId] ?? Bag() + let relatedViews = self.peerMessageViews[peerId] ?? Bag() for message in peerMessages { if existingMessageIds.contains(message.id) { @@ -586,7 +592,7 @@ public final class Postbox { if let peer = decoder.decodeRootObject() as? Peer { return peer } else { - println("(PostBox: can't decode peer)") + print("(PostBox: can't decode peer)") } break @@ -595,6 +601,8 @@ public final class Postbox { return nil } + + private func updatePeerEntry(peerId: PeerId, message: Message?, replace: Bool = false) { var currentIndex: PeerViewEntryIndex? for row in self.database.prepare("SELECT entry FROM peer_entries WHERE peerId = ?").run(peerId.toInt64()) { @@ -649,6 +657,7 @@ public final class Postbox { let context = view.removeEntry(nil, peerId: peerId) view.addEntry(entry) + view.complete(context, fetchEarlier: self.fetchPeerEntriesRelative(true), fetchLater: self.fetchPeerEntriesRelative(false)) sink.putNext(PeerView(view)) } @@ -721,7 +730,7 @@ public final class Postbox { let mediaId = MediaId(namespace: namespace, id: row[0] as! Int64) if let removedMessageIds = messageIdsByMediaId[mediaId] { var messageIds = Postbox.messageIdsForBlob(row[2] as! Blob).filter { - !contains(removedMessageIds, $0) + !removedMessageIds.contains($0) } updatedMessageIdsByMediaId[mediaId] = messageIds @@ -880,7 +889,7 @@ public final class Postbox { messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: anchorId.namespace, id: anchorId.id, count: count + 1) messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: anchorId.namespace, id: anchorId.id - 1, count: count + 1) - messages.sort({ MessageIndex($0) < MessageIndex($1) }) + messages.sortInPlace({ MessageIndex($0) < MessageIndex($1) }) var i = messages.count - 1 while i >= 1 { if messages[i].id == messages[i - 1].id { @@ -902,7 +911,7 @@ public final class Postbox { if index == nil { var closestId: MessageId.Id = messages[0].id.id var closestDistance = abs(closestId - anchorId.id) - var closestTimestamp: Int32 = messages[0].timestamp + let closestTimestamp: Int32 = messages[0].timestamp for message in messages { if abs(message.id.id - anchorId.id) < closestDistance { closestId = message.id.id @@ -924,7 +933,7 @@ public final class Postbox { } } - messages.sort({ MessageIndex($0) < MessageIndex($1) }) + messages.sortInPlace({ MessageIndex($0) < MessageIndex($1) }) var i = messages.count - 1 while i >= 1 { if messages[i].id == messages[i - 1].id { @@ -982,7 +991,7 @@ public final class Postbox { i++ } - filteredMessages.sort({ MessageIndex($0) < MessageIndex($1) }) + filteredMessages.sortInPlace({ MessageIndex($0) < MessageIndex($1) }) return (filteredMessages, earlier, later) } @@ -1009,7 +1018,7 @@ public final class Postbox { if let message = decoder.decodeRootObject() as? Message { messages.append(message) } else { - println("(PostBox: can't decode message)") + print("(PostBox: can't decode message)") } } @@ -1033,6 +1042,10 @@ public final class Postbox { rows = statement.run(Int64(count)) } + for row in rows { + entries.append(Postbox.peerViewEntryIndexForBlob(row[0] as! Blob)) + } + return entries } @@ -1043,7 +1056,7 @@ public final class Postbox { if let message = decoder.decodeRootObject() as? Message { return message } else { - println("(PostBox: can't decode message)") + print("(PostBox: can't decode message)") } break } @@ -1073,11 +1086,15 @@ public final class Postbox { } else { entry = PeerViewEntry(peerId: entryIndex.peerId, message: message) } + + entries.append(entry) } else { - println("(PostBox: missing message for peer entry)") + print("(PostBox: missing message for peer entry)") } } + entries.sortInPlace({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) + return entries } @@ -1088,7 +1105,7 @@ public final class Postbox { messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: nil, count: count) } - messages.sort({ MessageIndex($0) < MessageIndex($1)}) + messages.sortInPlace({ MessageIndex($0) < MessageIndex($1)}) return messages } @@ -1243,12 +1260,10 @@ public final class Postbox { var earlier: PeerViewEntry? - for namespace in self.messageNamespaces { - var i = tail.count - count - 1 - while i >= 0 { - earlier = tail[i] - break - } + i = tail.count - count - 1 + while i >= 0 { + earlier = tail[i] + break } let mutableView = MutablePeerView(count: count, earlier: earlier, entries: entries, later: nil) @@ -1291,36 +1306,36 @@ public final class Postbox { string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" } string += "]" - println(string) + print(string) } public func _dumpTables() { - println("\n------------") - println("peer_messages") - println("-------------") + print("\n------------") + print("peer_messages") + print("-------------") for row in self.database.prepare("SELECT peerId, namespace, id, timestamp, associatedMediaIds FROM peer_messages").run() { - println("peer(\(PeerId(row[0] as! Int64))) id(\(MessageId(peerId: PeerId(row[0] as! Int64), namespace: Int32(row[1] as! Int64), id:(Int32(row[2] as! Int64))))) timestamp(\(row[3] as! Int64)) media(\(Postbox.mediaIdsForBlob(row[4] as! Blob)))") + print("peer(\(PeerId(row[0] as! Int64))) id(\(MessageId(peerId: PeerId(row[0] as! Int64), namespace: Int32(row[1] as! Int64), id:(Int32(row[2] as! Int64))))) timestamp(\(row[3] as! Int64)) media(\(Postbox.mediaIdsForBlob(row[4] as! Blob)))") } - println("\n---------") - println("peer_media") - println("----------") + print("\n---------") + print("peer_media") + print("----------") for row in self.database.prepare("SELECT peerId, mediaNamespace, messageNamespace, messageId FROM peer_media").run() { - println("peer(\(PeerId(row[0] as! Int64))) namespace(\(row[1] as! Int64)) id(\(MessageId(peerId: PeerId(row[0] as! Int64), namespace: Int32(row[2] as! Int64), id:Int32(row[3] as! Int64))))") + print("peer(\(PeerId(row[0] as! Int64))) namespace(\(row[1] as! Int64)) id(\(MessageId(peerId: PeerId(row[0] as! Int64), namespace: Int32(row[2] as! Int64), id:Int32(row[3] as! Int64))))") } - println("\n----") - println("media") - println("-----") + print("\n----") + print("media") + print("-----") for row in self.database.prepare("SELECT namespace, id, associatedMessageIds FROM media").run() { - println("id(\(MediaId(namespace: Int32(row[0] as! Int64), id:(row[1] as! Int64)))) messages(\(Postbox.messageIdsForBlob(row[2] as! Blob)))") + print("id(\(MediaId(namespace: Int32(row[0] as! Int64), id:(row[1] as! Int64)))) messages(\(Postbox.messageIdsForBlob(row[2] as! Blob)))") } - println("\n------------") - println("media_cleanup") - println("-------------") + print("\n------------") + print("media_cleanup") + print("-------------") for row in self.database.prepare("SELECT namespace, id FROM media_cleanup").run() { - println("id(\(MediaId(namespace: Int32(row[0] as! Int64), id:(row[1] as! Int64))))") + print("id(\(MediaId(namespace: Int32(row[0] as! Int64), id:(row[1] as! Int64))))") } } From 7f5890f45994d9a97ad1c42c48bd26fb6f7eabd2 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 29 Jul 2015 18:02:56 +0300 Subject: [PATCH 018/427] no message --- Postbox.xcodeproj/project.pbxproj | 15 +- Postbox/Coding.swift | 129 +++++++ Postbox/DeferredString.swift | 124 +++++++ Postbox/Message.swift | 25 +- Postbox/MessageView.swift | 148 +++++--- Postbox/Peer.swift | 26 ++ Postbox/PeerView.swift | 97 +++++- Postbox/Postbox.swift | 544 ++++++++++++++++++++++++------ submodules/SQLite.swift | 2 +- 9 files changed, 940 insertions(+), 170 deletions(-) create mode 100644 Postbox/DeferredString.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 4533d9cee2..5b3a1ed5a8 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -22,7 +22,8 @@ D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; - D07FC7D31B4A3D6B0010B3F7 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */; }; + D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B76BE61B66639F0095CF45 /* DeferredString.swift */; }; + D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; D0D225261B4D84930085E26D /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D225251B4D84930085E26D /* PeerView.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -62,7 +63,8 @@ D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; - D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../SSignalKit/build/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; + D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = ""; }; + D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; D0D225251B4D84930085E26D /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -83,6 +85,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -90,7 +93,6 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - D07FC7D31B4A3D6B0010B3F7 /* SwiftSignalKit.framework in Frameworks */, D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -142,7 +144,7 @@ D0E3A7401B28A7E300A402D9 = { isa = PBXGroup; children = ( - D07FC7D21B4A3D6B0010B3F7 /* SwiftSignalKit.framework */, + D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */, D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */, D0E3A74C1B28A7E300A402D9 /* Postbox */, D0E3A7591B28A7E300A402D9 /* PostboxTests */, @@ -166,6 +168,7 @@ D075163D1B2D9CEF00AE42E0 /* PostboxPrivate */, D07515FC1B2C44A200AE42E0 /* thirdparty */, D0E3A7871B28AE9C00A402D9 /* Coding.swift */, + D0B76BE61B66639F0095CF45 /* DeferredString.swift */, D0E3A7831B28AE0900A402D9 /* Peer.swift */, D0E3A79D1B28B50400A402D9 /* Message.swift */, D0E3A7A11B28B7DC00A402D9 /* Media.swift */, @@ -334,6 +337,7 @@ D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, D0D225261B4D84930085E26D /* PeerView.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, + D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */, D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, ); @@ -459,6 +463,7 @@ DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; DYLIB_INSTALL_NAME_BASE = "@rpath"; + ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", @@ -487,6 +492,7 @@ DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; DYLIB_INSTALL_NAME_BASE = "@rpath"; + ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", @@ -503,6 +509,7 @@ ); PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; }; name = Release; }; diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 51f53915b0..864a6449fa 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -136,6 +136,13 @@ public final class Encoder { self.buffer.write(UnsafePointer(key), offset: 0, length: Int(length)) } + public func encodeKey(key: Int8) { + var length: Int8 = 1 + self.buffer.write(&length, offset: 0, length: 1) + var keyValue = key + self.buffer.write(&keyValue, offset: 0, length: Int(length)) + } + public func encodeInt32(value: Int32, forKey key: UnsafePointer) { self.encodeKey(key) var type: Int8 = ValueType.Int32.rawValue @@ -144,6 +151,14 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 4) } + public func encodeInt32(value: Int32, forKey key: Int8) { + self.encodeKey(key) + var type: Int8 = ValueType.Int32.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var v = value + self.buffer.write(&v, offset: 0, length: 4) + } + public func encodeInt64(value: Int64, forKey key: UnsafePointer) { self.encodeKey(key) var type: Int8 = ValueType.Int64.rawValue @@ -152,6 +167,14 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 8) } + public func encodeInt64(value: Int64, forKey key: Int8) { + self.encodeKey(key) + var type: Int8 = ValueType.Int64.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var v = value + self.buffer.write(&v, offset: 0, length: 8) + } + public func encodeBool(value: Bool, forKey key: UnsafePointer) { self.encodeKey(key) var type: Int8 = ValueType.Bool.rawValue @@ -160,6 +183,14 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 1) } + public func encodeBool(value: Bool, forKey key: Int8) { + self.encodeKey(key) + var type: Int8 = ValueType.Bool.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var v: Int8 = value ? 1 : 0 + self.buffer.write(&v, offset: 0, length: 1) + } + public func encodeDouble(value: Double, forKey key: UnsafePointer) { self.encodeKey(key) var type: Int8 = ValueType.Double.rawValue @@ -168,6 +199,14 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 8) } + public func encodeDouble(value: Double, forKey key: Int8) { + self.encodeKey(key) + var type: Int8 = ValueType.Double.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var v = value + self.buffer.write(&v, offset: 0, length: 8) + } + public func encodeString(value: String, forKey key: UnsafePointer) { self.encodeKey(key) var type: Int8 = ValueType.String.rawValue @@ -178,6 +217,26 @@ public final class Encoder { self.buffer.write(data.bytes, offset: 0, length: Int(length)) } + public func encodeString(value: DeferredString, forKey key: UnsafePointer) { + self.encodeKey(key) + var type: Int8 = ValueType.String.rawValue + self.buffer.write(&type, offset: 0, length: 1) + let data = value.data + var length: Int32 = Int32(data.length) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(data.bytes, offset: 0, length: Int(length)) + } + + public func encodeString(value: String, forKey key: Int8) { + self.encodeKey(key) + var type: Int8 = ValueType.String.rawValue + self.buffer.write(&type, offset: 0, length: 1) + let data = value.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! + var length: Int32 = Int32(data.length) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(data.bytes, offset: 0, length: Int(length)) + } + public func encodeRootObject(value: Coding) { self.encodeObject(value, forKey: "_") } @@ -278,6 +337,8 @@ public final class Encoder { self.buffer.write(&bytesLength, offset: 0, length: 4) self.buffer.write(bytes.memory, offset: 0, length: bytes.offset) } + + public let sharedWriteBuffer = WriteBuffer() } public final class Decoder { @@ -376,6 +437,37 @@ public final class Decoder { return false } + private class func positionOnKey(bytes: UnsafePointer, inout offset: Int, maxOffset: Int, length: Int, key: Int16, valueType: ValueType) -> Bool + { + var keyValue = key + let startOffset = offset + + let keyLength: Int = 2 + while (offset < maxOffset) + { + let readKeyLength = bytes[offset] + offset += 1 + offset += Int(readKeyLength) + + let readValueType = bytes[offset] + offset += 1 + + if readValueType != valueType.rawValue || keyLength != Int(readKeyLength) || memcmp(bytes + (offset - Int(readKeyLength) - 1), &keyValue, keyLength) != 0 { + skipValue(bytes, offset: &offset, length: length, valueType: ValueType(rawValue: readValueType)!) + } else { + return true + } + } + + if (startOffset != 0) + { + offset = 0 + return positionOnKey(bytes, offset: &offset, maxOffset: startOffset, length: length, key: key, valueType: valueType) + } + + return false + } + public func decodeInt32ForKey(key: UnsafePointer) -> Int32 { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 @@ -433,6 +525,43 @@ public final class Decoder { } } + public func decodeStringForKey(key: UnsafePointer) -> String? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + let data = NSData(bytes: self.buffer.memory + (self.buffer.offset + 4), length: Int(length)) + self.buffer.offset += 4 + Int(length) + let value = NSString(data: data, encoding: NSUTF8StringEncoding) + return value as? String + } else { + return nil + } + } + + public func decodeStringForKey(key: UnsafePointer) -> DeferredString { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + let data = NSData(bytes: self.buffer.memory + (self.buffer.offset + 4), length: Int(length)) + self.buffer.offset += 4 + Int(length) + return DeferredStringValue(data) + } else { + return DeferredStringValue("") + } + } + + public func decodeStringForKey(key: UnsafePointer) -> DeferredString? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + let data = NSData(bytes: self.buffer.memory + (self.buffer.offset + 4), length: Int(length)) + self.buffer.offset += 4 + Int(length) + return DeferredStringValue(data) + } else { + return nil + } + } + public func decodeRootObject() -> Coding? { return self.decodeObjectForKey("_") } diff --git a/Postbox/DeferredString.swift b/Postbox/DeferredString.swift new file mode 100644 index 0000000000..adce77284f --- /dev/null +++ b/Postbox/DeferredString.swift @@ -0,0 +1,124 @@ +import Foundation + +public protocol DeferredString { + var data: NSData { get } + var string: String { get } +} + +private final class DeferredStringImpl { + var string: String! + var data: NSData! + var lock: OSSpinLock = 0 + + init(string: String!, data: NSData!) { + self.string = string + self.data = data + } +} + +public final class DeferredStringValue: DeferredString, CustomStringConvertible, StringLiteralConvertible { + private var impl: DeferredStringImpl + + public typealias ExtendedGraphemeClusterLiteralType = StringLiteralType + public typealias UnicodeScalarLiteralType = StringLiteralType + + public init(unicodeScalarLiteral value: UnicodeScalarLiteralType) { + self.impl = DeferredStringImpl(string: "\(value)", data: nil) + } + + public init(extendedGraphemeClusterLiteral value: ExtendedGraphemeClusterLiteralType) { + self.impl = DeferredStringImpl(string: value, data: nil) + } + + public init(stringLiteral value: StringLiteralType) { + self.impl = DeferredStringImpl(string: value, data: nil) + } + + public init(_ string: String) { + self.impl = DeferredStringImpl(string: string, data: nil) + } + + public init(_ data: NSData) { + self.impl = DeferredStringImpl(string: nil, data: data) + } + + public var description: String { + return self.string + } + + public var data: NSData { + if isUniquelyReferencedNonObjC(&self.impl) { + let value: NSData + + if impl.data != nil { + value = impl.data + } else if impl.string != nil { + value = impl.string.dataUsingEncoding(NSUTF8StringEncoding) ?? NSData() + impl.data = value + } else { + value = NSData() + } + + return value + } else { + let value: NSData + + OSSpinLockLock(&impl.lock) + if impl.data != nil { + value = impl.data + } else if impl.string != nil { + value = impl.string.dataUsingEncoding(NSUTF8StringEncoding) ?? NSData() + impl.data = value + } else { + value = NSData() + } + OSSpinLockUnlock(&impl.lock) + + return value + } + } + + public var string: String { + if isUniquelyReferencedNonObjC(&self.impl) { + let value: String + + if impl.string != nil { + value = impl.string + } else if impl.data != nil { + let fromData = NSString(data: impl.data, encoding: NSUTF8StringEncoding) + if fromData == nil { + value = "" + impl.string = value + } else { + value = fromData as! String + impl.string = value + } + } else { + value = "" + } + + return value + } else { + let value: String + + OSSpinLockLock(&impl.lock) + if impl.string != nil { + value = impl.string + } else if impl.data != nil { + let fromData = NSString(data: impl.data, encoding: NSUTF8StringEncoding) + if fromData == nil { + value = "" + impl.string = value + } else { + value = fromData as! String + impl.string = value + } + } else { + value = "" + } + OSSpinLockUnlock(&impl.lock) + + return value + } + } +} diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 6db0eda143..a2b25de31e 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -110,5 +110,28 @@ public protocol Message: Coding { var id: MessageId { get } var timestamp: Int32 { get } var text: String { get } - var referencedMediaIds: [MediaId] { get } + var mediaIds: [MediaId] { get } + var peerIds: [PeerId] { get } +} + +public struct RenderedMessage { + public let message: Message + + internal let incomplete: Bool + public let peers: [Peer] + public let media: [Media] + + internal init(message: Message) { + self.message = message + self.peers = [] + self.media = [] + self.incomplete = true + } + + internal init(message: Message, peers: [Peer], media: [Media]) { + self.message = message + self.peers = peers + self.media = media + self.incomplete = false + } } diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift index a7b4eda4d9..3050f7cf2a 100644 --- a/Postbox/MessageView.swift +++ b/Postbox/MessageView.swift @@ -19,11 +19,11 @@ public final class MutableMessageView: CustomStringConvertible { let namespaces: [MessageId.Namespace] let count: Int - var earlier: [MessageId.Namespace : Message] = [:] - var later: [MessageId.Namespace : Message] = [:] - var messages: [Message] + var earlier: [MessageId.Namespace : RenderedMessage] = [:] + var later: [MessageId.Namespace : RenderedMessage] = [:] + var messages: [RenderedMessage] - public init(namespaces: [MessageId.Namespace], count: Int, earlier: [MessageId.Namespace : Message], messages: [Message], later: [MessageId.Namespace : Message]) { + public init(namespaces: [MessageId.Namespace], count: Int, earlier: [MessageId.Namespace : RenderedMessage], messages: [RenderedMessage], later: [MessageId.Namespace : RenderedMessage]) { self.namespaces = namespaces self.count = count self.earlier = earlier @@ -31,56 +31,60 @@ public final class MutableMessageView: CustomStringConvertible { self.messages = messages } - public func add(message: Message) { + public func add(message: RenderedMessage) -> Bool { if self.messages.count == 0 { self.messages.append(message) + return true } else { - let first = MessageIndex(self.messages[self.messages.count - 1]) - let last = MessageIndex(self.messages[0]) + let first = MessageIndex(self.messages[self.messages.count - 1].message) + let last = MessageIndex(self.messages[0].message) var next: MessageIndex? for namespace in self.namespaces { if let message = later[namespace] { - let messageIndex = MessageIndex(message) + let messageIndex = MessageIndex(message.message) if next == nil || messageIndex < next! { next = messageIndex } } } - let index = MessageIndex(message) + let index = MessageIndex(message.message) if index < last { - let earlierMessage = self.earlier[message.id.namespace] - if earlierMessage == nil || earlierMessage!.id.id < message.id.id { + let earlierMessage = self.earlier[message.message.id.namespace] + if earlierMessage == nil || earlierMessage!.message.id.id < message.message.id.id { if self.messages.count < self.count { self.messages.insert(message, atIndex: 0) } else { - self.earlier[message.id.namespace] = message + self.earlier[message.message.id.namespace] = message } } + + return true } else if index > first { if next != nil && index > next! { - let laterMessage = self.later[message.id.namespace] - if laterMessage == nil || laterMessage!.id.id > message.id.id { + let laterMessage = self.later[message.message.id.namespace] + if laterMessage == nil || laterMessage!.message.id.id > message.message.id.id { if self.messages.count < self.count { self.messages.append(message) } else { - self.later[message.id.namespace] = message + self.later[message.message.id.namespace] = message } } } else { self.messages.append(message) if self.messages.count > self.count { let earliest = self.messages[0] - self.earlier[earliest.id.namespace] = earliest + self.earlier[earliest.message.id.namespace] = earliest self.messages.removeAtIndex(0) } } + return true } else if index != last && index != first { var i = self.messages.count while i >= 1 { - if MessageIndex(self.messages[i - 1]) < index { + if MessageIndex(self.messages[i - 1].message) < index { break } i-- @@ -88,9 +92,12 @@ public final class MutableMessageView: CustomStringConvertible { self.messages.insert(message, atIndex: i) if self.messages.count > self.count { let earliest = self.messages[0] - self.earlier[earliest.id.namespace] = earliest + self.earlier[earliest.message.id.namespace] = earliest self.messages.removeAtIndex(0) } + return true + } else { + return false } } } @@ -102,21 +109,21 @@ public final class MutableMessageView: CustomStringConvertible { } for (_, message) in self.earlier { - if ids.contains(message.id) { - updatedContext.invalidEarlier.insert(message.id.namespace) + if ids.contains(message.message.id) { + updatedContext.invalidEarlier.insert(message.message.id.namespace) } } for (_, message) in self.later { - if ids.contains(message.id) { - updatedContext.invalidLater.insert(message.id.namespace) + if ids.contains(message.message.id) { + updatedContext.invalidLater.insert(message.message.id.namespace) } } if self.messages.count != 0 { var i = self.messages.count - 1 while i >= 0 { - if ids.contains(self.messages[i].id) { + if ids.contains(self.messages[i].message.id) { self.messages.removeAtIndex(i) updatedContext.removedMessages = true } @@ -127,19 +134,18 @@ public final class MutableMessageView: CustomStringConvertible { return updatedContext } - public func complete(context: RemoveContext, fetchEarlier: (MessageId.Namespace, MessageId.Id?, Int) -> [Message], fetchLater: (MessageId.Namespace, MessageId.Id?, Int) -> [Message]) { + public func complete(context: RemoveContext, fetchEarlier: (MessageId.Namespace, MessageId.Id?, Int) -> [RenderedMessage], fetchLater: (MessageId.Namespace, MessageId.Id?, Int) -> [RenderedMessage]) { if context.removedMessages { - var addedMessages: [Message] = [] + var addedMessages: [RenderedMessage] = [] var latestAnchor: MessageIndex? if let lastMessage = self.messages.last { - latestAnchor = MessageIndex(lastMessage) + latestAnchor = MessageIndex(lastMessage.message) } if latestAnchor == nil { - var laterMessages: [Message] = [] for (_, message) in self.later { - let messageIndex = MessageIndex(message) + let messageIndex = MessageIndex(message.message) if latestAnchor == nil || latestAnchor! > messageIndex { latestAnchor = messageIndex } @@ -148,18 +154,18 @@ public final class MutableMessageView: CustomStringConvertible { for namespace in self.namespaces { if let later = self.later[namespace] { - addedMessages += fetchLater(namespace, later.id.id - 1, self.count) + addedMessages += fetchLater(namespace, later.message.id.id - 1, self.count) } if let earlier = self.earlier[namespace] { - addedMessages += fetchEarlier(namespace, earlier.id.id + 1, self.count) + addedMessages += fetchEarlier(namespace, earlier.message.id.id + 1, self.count) } } addedMessages += self.messages - addedMessages.sortInPlace({ MessageIndex($0) < MessageIndex($1) }) + addedMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) var i = addedMessages.count - 1 while i >= 1 { - if addedMessages[i].id == addedMessages[i - 1].id { + if addedMessages[i].message.id == addedMessages[i - 1].message.id { addedMessages.removeAtIndex(i) } i-- @@ -170,7 +176,7 @@ public final class MutableMessageView: CustomStringConvertible { if let latestAnchor = latestAnchor { var i = addedMessages.count - 1 while i >= 0 { - if MessageIndex(addedMessages[i]) <= latestAnchor { + if MessageIndex(addedMessages[i].message) <= latestAnchor { anchorIndex = i break } @@ -183,7 +189,7 @@ public final class MutableMessageView: CustomStringConvertible { for namespace in self.namespaces { var i = anchorIndex + 1 while i < addedMessages.count { - if addedMessages[i].id.namespace == namespace { + if addedMessages[i].message.id.namespace == namespace { self.later[namespace] = addedMessages[i] break } @@ -203,7 +209,7 @@ public final class MutableMessageView: CustomStringConvertible { for namespace in self.namespaces { i = anchorIndex - self.count while i >= 0 { - if addedMessages[i].id.namespace == namespace { + if addedMessages[i].message.id.namespace == namespace { self.earlier[namespace] = addedMessages[i] break } @@ -217,8 +223,8 @@ public final class MutableMessageView: CustomStringConvertible { var earlyId: MessageId.Id? var i = 0 while i < self.messages.count { - if self.messages[i].id.namespace == namespace { - earlyId = self.messages[i].id.id + if self.messages[i].message.id.namespace == namespace { + earlyId = self.messages[i].message.id.id break } i++ @@ -236,8 +242,8 @@ public final class MutableMessageView: CustomStringConvertible { var lateId: MessageId.Id? var i = self.messages.count - 1 while i >= 0 { - if self.messages[i].id.namespace == namespace { - lateId = self.messages[i].id.id + if self.messages[i].message.id.namespace == namespace { + lateId = self.messages[i].message.id.id break } i-- @@ -253,6 +259,56 @@ public final class MutableMessageView: CustomStringConvertible { } } + + public func incompleteMessages() -> [Message] { + var result: [Message] = [] + + for (_, message) in self.earlier { + if message.incomplete { + result.append(message.message) + } + } + for (_, message) in self.later { + if message.incomplete { + result.append(message.message) + } + } + + for message in self.messages { + if message.incomplete { + result.append(message.message) + } + } + + return result + } + + public func completeMessages(messages: [MessageId : RenderedMessage]) { + var earlier = self.earlier + for (namespace, message) in self.earlier { + if let message = messages[message.message.id] { + earlier[namespace] = message + } + } + self.earlier = earlier + + var later = self.later + for (namespace, message) in self.later { + if let message = messages[message.message.id] { + later[namespace] = message + } + } + self.later = later + + var i = 0 + while i < self.messages.count { + if let message = messages[self.messages[i].message.id] { + self.messages[i] = message + } + i++ + } + } + public var description: String { var string = "" string += "...(" @@ -264,7 +320,7 @@ public final class MutableMessageView: CustomStringConvertible { } else { string += ", " } - string += "\(namespace): \(value.id.id)—\(value.timestamp)" + string += "\(namespace): \(value.message.id.id)—\(value.message.timestamp)" } } string += ") —— " @@ -277,7 +333,7 @@ public final class MutableMessageView: CustomStringConvertible { } else { string += ", " } - string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" + string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" } string += "]" @@ -290,7 +346,7 @@ public final class MutableMessageView: CustomStringConvertible { } else { string += ", " } - string += "\(namespace): \(value.id.id)—\(value.timestamp)" + string += "\(namespace): \(value.message.id.id)—\(value.message.timestamp)" } } string += ")..." @@ -304,7 +360,7 @@ public final class MessageView: CustomStringConvertible { private let earlierIds: [MessageIndex] public let hasLater: Bool private let laterIds: [MessageIndex] - public let messages: [Message] + public let messages: [RenderedMessage] init(_ mutableView: MutableMessageView) { self.hasEarlier = mutableView.earlier.count != 0 @@ -313,13 +369,13 @@ public final class MessageView: CustomStringConvertible { var earlierIds: [MessageIndex] = [] for (_, message) in mutableView.earlier { - earlierIds.append(MessageIndex(message)) + earlierIds.append(MessageIndex(message.message)) } self.earlierIds = earlierIds var laterIds: [MessageIndex] = [] for (_, message) in mutableView.later { - laterIds.append(MessageIndex(message)) + laterIds.append(MessageIndex(message.message)) } self.laterIds = laterIds } @@ -347,7 +403,7 @@ public final class MessageView: CustomStringConvertible { } else { string += ", " } - string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" + string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" } string += "]" if self.hasLater { diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index 190b7bf41a..03ca43228a 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -21,6 +21,30 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { return (Int64(self.namespace) << 32) | Int64(self.id) } + public static func encodeArrayToBuffer(array: [PeerId], buffer: WriteBuffer) { + var length: Int32 = Int32(array.count) + buffer.write(&length, offset: 0, length: 4) + for id in array { + var value = id.toInt64() + buffer.write(&value, offset: 0, length: 8) + } + } + + public static func decodeArrayFromBuffer(buffer: ReadBuffer) -> [PeerId] { + var length: Int32 = 0 + memcpy(&length, buffer.memory, 4) + buffer.offset += 4 + var i = 0 + var array: [PeerId] = [] + while i < Int(length) { + var value: Int64 = 0 + buffer.read(&value, offset: 0, length: 8) + array.append(PeerId(value)) + i++ + } + return array + } + public var hashValue: Int { get { return Int(self.id) @@ -67,4 +91,6 @@ public func <(lhs: PeerId, rhs: PeerId) -> Bool { public protocol Peer: Coding { var id: PeerId { get } + + func equalsTo(other: Peer) -> Bool } diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index 5f77d90ded..82085302e8 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -1,21 +1,27 @@ import Foundation -public class PeerViewEntry { +public final class PeerViewEntry { public let peerId: PeerId public let peer: Peer? - public let message: Message + public let message: RenderedMessage - public init(peer: Peer, message: Message) { + public init(peer: Peer, message: RenderedMessage) { self.peerId = peer.id self.peer = peer self.message = message } - public init(peerId: PeerId, message: Message) { + public init(peerId: PeerId, message: RenderedMessage) { self.peerId = peerId self.peer = nil self.message = message } + + private init(peer: Peer?, peerId: PeerId, message: RenderedMessage) { + self.peer = peer + self.peerId = peerId + self.message = message + } } public struct PeerViewEntryIndex: Equatable, Comparable { @@ -24,7 +30,7 @@ public struct PeerViewEntryIndex: Equatable, Comparable { public init(_ entry: PeerViewEntry) { self.peerId = entry.peerId - self.messageIndex = MessageIndex(entry.message) + self.messageIndex = MessageIndex(entry.message.message) } public init(peerId: PeerId, messageIndex: MessageIndex) { @@ -252,12 +258,85 @@ public final class MutablePeerView: CustomStringConvertible { } } + public func updatePeers(peers: [PeerId : Peer]) -> Bool { + var updated = false + + if let earlier = self.earlier { + if let peer = peers[earlier.peerId] { + self.earlier = PeerViewEntry(peer: peer, message: earlier.message) + updated = true + } + } + + if let later = self.later { + if let peer = peers[later.peerId] { + self.later = PeerViewEntry(peer: peer, message: later.message) + updated = true + } + } + + var i = 0 + while i < self.entries.count { + if let peer = peers[self.entries[i].peerId] { + self.entries[i] = PeerViewEntry(peer: peer, message: self.entries[i].message) + updated = true + } + i++ + } + + return updated + } + + public func incompleteMessages() -> [Message] { + var result: [Message] = [] + + if let earlier = self.earlier { + if earlier.message.incomplete { + result.append(earlier.message.message) + } + } + if let later = self.later { + if later.message.incomplete { + result.append(later.message.message) + } + } + + for entry in self.entries { + if entry.message.incomplete { + result.append(entry.message.message) + } + } + + return result + } + + public func completeMessages(messages: [MessageId : RenderedMessage]) { + if let earlier = self.earlier { + if let message = messages[earlier.message.message.id] { + self.earlier = PeerViewEntry(peer: earlier.peer, peerId: earlier.peerId, message: message) + } + } + if let later = self.later { + if let message = messages[later.message.message.id] { + self.later = PeerViewEntry(peer: later.peer, peerId: later.peerId, message: message) + } + } + + var i = 0 + while i < self.entries.count { + if let message = messages[self.entries[i].message.message.id] { + self.entries[i] = PeerViewEntry(peer: self.entries[i].peer, peerId: self.entries[i].peerId, message: message) + } + i++ + } + } + public var description: String { var string = "" if let earlier = self.earlier { string += "more(" - string += "(p \(earlier.peerId.namespace):\(earlier.peerId.id), m \(earlier.message.id.namespace):\(earlier.message.id.id)—\(earlier.message.timestamp)" + string += "(p \(earlier.peerId.namespace):\(earlier.peerId.id), m \(earlier.message.message.id.namespace):\(earlier.message.message.id.id)—\(earlier.message.message.timestamp)" string += ") " } @@ -269,13 +348,13 @@ public final class MutablePeerView: CustomStringConvertible { } else { string += ", " } - string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" + string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.message.id.namespace):\(entry.message.message.id.id)—\(entry.message.message.timestamp))" } string += "]" if let later = self.later { string += " more(" - string += "(p \(later.peerId.namespace):\(later.peerId), m \(later.message.id.namespace):\(later.message.id.id)—\(later.message.timestamp)" + string += "(p \(later.peerId.namespace):\(later.peerId), m \(later.message.message.id.namespace):\(later.message.message.id.id)—\(later.message.message.timestamp)" string += ")" } @@ -321,7 +400,7 @@ public final class PeerView: CustomStringConvertible { } else { string += ", " } - string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" + string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.message.id.namespace):\(entry.message.message.id.id)—\(entry.message.message.timestamp))" } string += "]" diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 5c8e08ec7f..5b0a1c8875 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -6,7 +6,7 @@ public protocol PostboxState: Coding { } -public class Modifier { +public final class Modifier { private weak var postbox: Postbox? private init(postbox: Postbox) { @@ -28,6 +28,10 @@ public class Modifier { public func setState(state: State) { self.postbox?.setState(state) } + + public func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer) { + self.postbox?.updatePeers(peers, update: update) + } } public final class Postbox { @@ -38,7 +42,10 @@ public final class Postbox { private var database: Database! private var peerMessageViews: [PeerId : Bag<(MutableMessageView, Pipe)>] = [:] + private var deferredMessageViewsToUpdate: [(MutableMessageView, Pipe)] = [] private var peerViews: Bag<(MutablePeerView, Pipe)> = Bag() + private var deferredPeerViewsToUpdate: [(MutablePeerView, Pipe)] = [] + private var statePipe: Pipe = Pipe() public init(basePath: String, messageNamespaces: [MessageId.Namespace]) { @@ -48,21 +55,45 @@ public final class Postbox { } private func openDatabase() { - do { - try NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil) - } catch _ { - } - self.database = Database(basePath.stringByAppendingPathComponent("db")) - self.queue.dispatch { + let startTime = CFAbsoluteTimeGetCurrent() + + do { + try NSFileManager.defaultManager().createDirectoryAtPath(self.basePath, withIntermediateDirectories: true, attributes: nil) + } catch _ { + } + self.database = Database(self.basePath.stringByAppendingPathComponent("db")) + let result = self.database.scalar("PRAGMA user_version") as! Int64 - if result == 1 { + let version: Int64 = 7 + if result == version { print("(Postbox schema version \(result))") } else { + if result != 0 { + print("(Postbox migrating to version \(version))") + do { + try NSFileManager.defaultManager().removeItemAtPath(self.basePath) + try NSFileManager.defaultManager().createDirectoryAtPath(self.basePath, withIntermediateDirectories: true, attributes: nil) + } catch (_) { + } + + self.database = Database(self.basePath.stringByAppendingPathComponent("db")) + } print("(Postbox creating schema)") self.createSchema() - self.database.execute("PRAGMA user_version = 1") + self.database.execute("PRAGMA user_version = \(version)") } + + self.database.adjustChunkSize() + self.database.execute("PRAGMA page_size=1024") + self.database.execute("PRAGMA cache_size=-2097152") + self.database.execute("PRAGMA synchronous=NORMAL") + self.database.execute("PRAGMA journal_mode=truncate") + self.database.execute("PRAGMA temp_store=MEMORY") + //self.database.execute("PRAGMA wal_autocheckpoint=32") + //self.database.execute("PRAGMA journal_size_limit=1536") + + print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") } } @@ -92,7 +123,7 @@ public final class Postbox { self.database.execute("CREATE INDEX peer_entries_entry on peer_entries (entry)") //peers - self.database.execute("CREATE TABLE peers (peerId INTEGER PRIMARY KEY, data BLOB)") + self.database.execute("CREATE TABLE peers (id INTEGER PRIMARY KEY, data BLOB)") } private class func peerViewEntryIndexForBlob(blob: Blob) -> PeerViewEntryIndex { @@ -197,14 +228,23 @@ public final class Postbox { return grouped } - private class func messagesGroupedByPeerId(messages: [Message]) -> [PeerId : [Message]] { - var grouped: [PeerId : [Message]] = [:] + private class func messagesGroupedByPeerId(messages: [Message]) -> [(PeerId, [Message])] { + var grouped: [(PeerId, [Message])] = [] for message in messages { - if grouped[message.id.peerId] != nil { - grouped[message.id.peerId]!.append(message) - } else { - grouped[message.id.peerId] = [message] + var i = 0 + let count = grouped.count + var found = false + while i < count { + if grouped[i].0 == message.id.peerId { + grouped[i].1.append(message) + found = true + break + } + i++ + } + if !found { + grouped.append((message.id.peerId, [message])) } } @@ -325,8 +365,12 @@ public final class Postbox { return ids } + private var cachedState: State? + private func setState(state: State) { self.queue.dispatch { + self.cachedState = state + let encoder = Encoder() encoder.encodeRootObject(state) let blob = Blob(data: encoder.makeData()) @@ -337,16 +381,21 @@ public final class Postbox { } private func getState() -> State? { - for row in self.database.prepareCached("SELECT data FROM state WHERE id = ?").run(Int64(0)) { - let data = (row[0] as! Blob).data - let buffer = ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false) - let decoder = Decoder(buffer: buffer) - if let state = decoder.decodeRootObject() as? State { - return state + if let cachedState = self.cachedState { + return cachedState + } else { + for row in self.database.prepareCached("SELECT data FROM state WHERE id = ?").run(Int64(0)) { + let data = (row[0] as! Blob).data + let buffer = ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false) + let decoder = Decoder(buffer: buffer) + if let state = decoder.decodeRootObject() as? State { + self.cachedState = state + return state + } + break } - break + return nil } - return nil } public func state() -> Signal { @@ -394,11 +443,11 @@ public final class Postbox { } private func addMessages(messages: [Message], medias: [Media]) { - let messageInsertStatement = self.database.prepare("INSERT INTO peer_messages (peerId, namespace, id, data, associatedMediaIds, timestamp) VALUES (?, ?, ?, ?, ?, ?)") - let peerMediaInsertStatement = self.database.prepare("INSERT INTO peer_media (peerId, mediaNamespace, messageNamespace, messageId) VALUES (?, ?, ?, ?)") - let mediaInsertStatement = self.database.prepare("INSERT INTO media (namespace, id, data, associatedMessageIds) VALUES (?, ?, ?, ?)") - let referencedMessageIdsStatement = self.database.prepare("SELECT associatedMessageIds FROM media WHERE namespace = ? AND id = ?") - let updateReferencedMessageIdsStatement = self.database.prepare("UPDATE media SET associatedMessageIds = ? WHERE namespace = ? AND id = ?") + let messageInsertStatement = self.database.prepareCached("INSERT INTO peer_messages (peerId, namespace, id, data, associatedMediaIds, timestamp) VALUES (?, ?, ?, ?, ?, ?)") + let peerMediaInsertStatement = self.database.prepareCached("INSERT INTO peer_media (peerId, mediaNamespace, messageNamespace, messageId) VALUES (?, ?, ?, ?)") + let mediaInsertStatement = self.database.prepareCached("INSERT INTO media (namespace, id, data, associatedMessageIds) VALUES (?, ?, ?, ?)") + let referencedMessageIdsStatement = self.database.prepareCached("SELECT associatedMessageIds FROM media WHERE namespace = ? AND id = ?") + let updateReferencedMessageIdsStatement = self.database.prepareCached("UPDATE media SET associatedMessageIds = ? WHERE namespace = ? AND id = ?") let encoder = Encoder() @@ -437,18 +486,12 @@ public final class Postbox { } } - let relatedViews = self.peerMessageViews[peerId] ?? Bag() - for message in peerMessages { if existingMessageIds.contains(message.id) { continue } existingMessageIds.insert(message.id) - for record in relatedViews.copyItems() { - record.0.add(message) - } - let index = MessageIndex(message) if maxMessage == nil || index > maxMessage!.0 { maxMessage = (index, message) @@ -458,8 +501,8 @@ public final class Postbox { encoder.encodeRootObject(message) let messageBlob = Blob(data: encoder.makeData()) - let referencedMediaIdsMediaIdsBlob = Postbox.blobForMediaIds(message.referencedMediaIds) - for id in message.referencedMediaIds { + let referencedMediaIdsMediaIdsBlob = Postbox.blobForMediaIds(message.mediaIds) + for id in message.mediaIds { if messageIdsByMediaId[id] != nil { messageIdsByMediaId[id]!.append(message.id) } else { @@ -469,17 +512,28 @@ public final class Postbox { messageInsertStatement.run(peerId.toInt64(), Int64(message.id.namespace), Int64(message.id.id), messageBlob, referencedMediaIdsMediaIdsBlob, Int64(message.timestamp)) - for id in message.referencedMediaIds { + for id in message.mediaIds { peerMediaInsertStatement.run(peerId.toInt64(), Int64(id.namespace), Int64(message.id.namespace), Int64(message.id.id)) } } - for record in relatedViews.copyItems() { - record.1.putNext(MessageView(record.0)) + if let relatedViews = self.peerMessageViews[peerId] { + for record in relatedViews.copyItems() { + var updated = false + for message in peerMessages { + if record.0.add(RenderedMessage(message: message)) { + updated = true + } + } + + if updated { + self.deferMessageViewUpdate(record.0, pipe: record.1) + } + } } if let maxMessage = maxMessage { - self.updatePeerEntry(peerId, message: maxMessage.1) + self.updatePeerEntry(peerId, message: RenderedMessage(message: maxMessage.1)) } } @@ -585,56 +639,197 @@ public final class Postbox { return grouped } - private func peerWithId(peerId: PeerId) -> Peer? { - for row in self.database.prepare("SELECT data FROM peers WHERE peerId = ?").run(peerId.toInt64()) { - let data = (row[0] as! Blob).data - let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) - if let peer = decoder.decodeRootObject() as? Peer { - return peer - } else { - print("(PostBox: can't decode peer)") + private func mediaWithIds(ids: [MediaId]) -> [MediaId : Media] { + if ids.count == 0 { + return [:] + } else { + let select = self.database.prepareCached("SELECT data FROM media WHERE namespace = ? AND id = ?") + var result: [MediaId : Media] = [:] + + for id in ids { + for row in select.run(Int64(id.namespace), id.id) { + let blob = row[0] as! Blob + if let media = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false)) as? Media { + result[media.id] = media + } + break + } } - break + return result } - - return nil } - + var cachedPeers: [PeerId : Peer] = [:] - private func updatePeerEntry(peerId: PeerId, message: Message?, replace: Bool = false) { + private func peerWithId(peerId: PeerId) -> Peer? { + if let cachedPeer = cachedPeers[peerId] { + return cachedPeer + } else { + for row in self.database.prepareCached("SELECT data FROM peers WHERE id = ?").run(peerId.toInt64()) { + let data = (row[0] as! Blob).data + let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) + if let peer = decoder.decodeRootObject() as? Peer { + cachedPeers[peer.id] = peer + return peer + } else { + print("(PostBox: can't decode peer)") + } + + break + } + + return nil + } + } + + private func peersWithIds(ids: [PeerId]) -> [PeerId : Peer] { + if ids.count == 0 { + return [:] + } else { + var remainingIds: [PeerId] = [] + + var peers: [PeerId : Peer] = [:] + + for id in ids { + if let cachedPeer = cachedPeers[id] { + peers[id] = cachedPeer + } else { + remainingIds.append(id) + } + } + + if remainingIds.count != 0 { + let rows: Statement + if ids.count == 1 { + rows = self.database.prepareCached("SELECT data FROM peers WHERE id = ?").run(ids[0].toInt64()) + } else if ids.count == 2 { + rows = self.database.prepareCached("SELECT data FROM peers WHERE id IN (?, ?)").run(ids[0].toInt64(), ids[1].toInt64()) + } else { + var query = "SELECT data FROM peers WHERE id IN (" + var first = true + for id in ids { + if first { + first = false + query += "\(id.toInt64())" + } else { + query += ",\(id.toInt64())" + } + } + query += ")" + rows = self.database.prepare(query).run() + } + + for row in rows { + let blob = row[0] as! Blob + if let peer = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false)).decodeRootObject() as? Peer { + self.cachedPeers[peer.id] = peer + peers[peer.id] = peer + } + } + } + + return peers + } + } + + private func deferPeerViewUpdate(view: MutablePeerView, pipe: Pipe) { + var i = 0 + var found = false + while i < self.deferredPeerViewsToUpdate.count { + if self.deferredPeerViewsToUpdate[i].1 === pipe { + self.deferredPeerViewsToUpdate[i] = (view, pipe) + found = true + break + } + i++ + } + if !found { + self.deferredPeerViewsToUpdate.append((view, pipe)) + } + } + + private func deferMessageViewUpdate(view: MutableMessageView, pipe: Pipe) { + var i = 0 + var found = false + while i < self.deferredPeerViewsToUpdate.count { + if self.deferredMessageViewsToUpdate[i].1 === pipe { + self.deferredMessageViewsToUpdate[i] = (view, pipe) + found = true + break + } + i++ + } + if !found { + self.deferredMessageViewsToUpdate.append((view, pipe)) + } + } + + private func performDeferredUpdates() { + let deferredPeerViewsToUpdate = self.deferredPeerViewsToUpdate + self.deferredPeerViewsToUpdate.removeAll() + + for entry in deferredPeerViewsToUpdate { + let viewRenderedMessages = self.renderedMessages(entry.0.incompleteMessages()) + if viewRenderedMessages.count != 0 { + var viewRenderedMessagesDict: [MessageId : RenderedMessage] = [:] + for message in viewRenderedMessages { + viewRenderedMessagesDict[message.message.id] = message + } + entry.0.completeMessages(viewRenderedMessagesDict) + } + + entry.1.putNext(PeerView(entry.0)) + } + + let deferredMessageViewsToUpdate = self.deferredMessageViewsToUpdate + self.deferredMessageViewsToUpdate.removeAll() + + for entry in deferredMessageViewsToUpdate { + let viewRenderedMessages = self.renderedMessages(entry.0.incompleteMessages()) + if viewRenderedMessages.count != 0 { + var viewRenderedMessagesDict: [MessageId : RenderedMessage] = [:] + for message in viewRenderedMessages { + viewRenderedMessagesDict[message.message.id] = message + } + entry.0.completeMessages(viewRenderedMessagesDict) + } + + entry.1.putNext(MessageView(entry.0)) + } + } + + private func updatePeerEntry(peerId: PeerId, message: RenderedMessage?, replace: Bool = false) { var currentIndex: PeerViewEntryIndex? - for row in self.database.prepare("SELECT entry FROM peer_entries WHERE peerId = ?").run(peerId.toInt64()) { + for row in self.database.prepareCached("SELECT entry FROM peer_entries WHERE peerId = ?").run(peerId.toInt64()) { currentIndex = Postbox.peerViewEntryIndexForBlob(row[0] as! Blob) break } - var updatedPeerMessage: Message? + var updatedPeerMessage: RenderedMessage? if let currentIndex = currentIndex { if let message = message { - let messageIndex = MessageIndex(message) + let messageIndex = MessageIndex(message.message) if replace || currentIndex.messageIndex < messageIndex { let updatedIndex = PeerViewEntryIndex(peerId: peerId, messageIndex: messageIndex) updatedPeerMessage = message let updatedBlob = Postbox.blobForPeerViewEntryIndex(updatedIndex) - self.database.prepare("UPDATE peer_entries SET entry = ? WHERE peerId = ?").run(updatedBlob, peerId.toInt64()) + self.database.prepareCached("UPDATE peer_entries SET entry = ? WHERE peerId = ?").run(updatedBlob, peerId.toInt64()) } } else if replace { //TODO: remove? } } else if let message = message { updatedPeerMessage = message - let updatedIndex = PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(message)) + let updatedIndex = PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(message.message)) let updatedBlob = Postbox.blobForPeerViewEntryIndex(updatedIndex) - self.database.prepare("INSERT INTO peer_entries (peerId, entry) VALUES (?, ?)").run(peerId.toInt64(), updatedBlob) + self.database.prepareCached("INSERT INTO peer_entries (peerId, entry) VALUES (?, ?)").run(peerId.toInt64(), updatedBlob) } if let updatedPeerMessage = updatedPeerMessage { var peer: Peer? - for (view, sink) in self.peerViews.copyItems() { - + for (view, pipe) in self.peerViews.copyItems() { if peer == nil { for entry in view.entries { if entry.peerId == peerId { @@ -659,7 +854,7 @@ public final class Postbox { view.addEntry(entry) view.complete(context, fetchEarlier: self.fetchPeerEntriesRelative(true), fetchLater: self.fetchPeerEntriesRelative(false)) - sink.putNext(PeerView(view)) + self.deferPeerViewUpdate(view, pipe: pipe) } } } @@ -671,11 +866,11 @@ public final class Postbox { for (peerId, messageIds) in Postbox.messageIdsGroupedByPeerId(ids) { if let relatedViews = self.peerMessageViews[peerId] { - for (view, sink) in relatedViews.copyItems() { + for (view, pipe) in relatedViews.copyItems() { let context = view.remove(Set(messageIds)) if !context.empty() { view.complete(context, fetchEarlier: self.fetchMessagesRelative(peerId, earlier: true), fetchLater: self.fetchMessagesRelative(peerId, earlier: false)) - sink.putNext(MessageView(view)) + self.deferMessageViewUpdate(view, pipe: pipe) } } } @@ -735,16 +930,16 @@ public final class Postbox { updatedMessageIdsByMediaId[mediaId] = messageIds if messageIds.count == 0 { - self.database.prepare("INSERT OR IGNORE INTO media_cleanup (namespace, id, data) VALUES (?, ?, ?)").run(Int64(namespace), mediaId.id, row[1] as! Blob) + self.database.prepareCached("INSERT OR IGNORE INTO media_cleanup (namespace, id, data) VALUES (?, ?, ?)").run(Int64(namespace), mediaId.id, row[1] as! Blob) } } } for (mediaId, messageIds) in updatedMessageIdsByMediaId { if messageIds.count == 0 { - self.database.prepare("DELETE FROM media WHERE namespace = ? AND id = ?").run(Int64(mediaId.namespace), mediaId.id) + self.database.prepareCached("DELETE FROM media WHERE namespace = ? AND id = ?").run(Int64(mediaId.namespace), mediaId.id) } else { - self.database.prepare("UPDATE media SET associatedMessageIds = ? WHERE namespace = ? AND id = ?").run(Postbox.blobForMessageIds(messageIds), Int64(mediaId.namespace), mediaId.id) + self.database.prepareCached("UPDATE media SET associatedMessageIds = ? WHERE namespace = ? AND id = ?").run(Postbox.blobForMessageIds(messageIds), Int64(mediaId.namespace), mediaId.id) } } } @@ -755,13 +950,79 @@ public final class Postbox { } } + private func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer) { + if peers.count == 0 { + return + } + + if peers.count == -1 { + + } else { + var peerIds: [PeerId] = [] + for peer in peers { + peerIds.append(peer.id) + } + + let currentPeers = self.peersWithIds(peerIds) + + let updatePeer = self.database.prepareCached("UPDATE peers SET data = ? WHERE id = ?") + let insertPeer = self.database.prepareCached("INSERT INTO peers (id, data) VALUES (?, ?)") + let encoder = Encoder() + + var updatedPeers: [PeerId : Peer] = [:] + + for updatedPeer in peers { + let currentPeer = currentPeers[updatedPeer.id] + + var finalPeer = updatedPeer + if let currentPeer = currentPeer { + finalPeer = update(currentPeer, updatedPeer) + } + + if currentPeer == nil || !finalPeer.equalsTo(currentPeer!) { + updatedPeers[finalPeer.id] = finalPeer + self.cachedPeers[finalPeer.id] = finalPeer + + encoder.reset() + encoder.encodeRootObject(finalPeer) + + if currentPeer != nil { + updatePeer.run(Blob(data: encoder.makeData()), finalPeer.id.toInt64()) + } else { + insertPeer.run(finalPeer.id.toInt64(), Blob(data: encoder.makeData())) + } + } + } + + for record in self.peerViews.copyItems() { + if record.0.updatePeers(updatedPeers) { + deferPeerViewUpdate(record.0, pipe: record.1) + } + } + } + } + public func modify(f: Modifier -> T) -> Signal { return Signal { subscriber in self.queue.dispatch { + //#if DEBUG + let startTime = CFAbsoluteTimeGetCurrent() + //#endif + self.database.transaction() let result = f(Modifier(postbox: self)) + //print("(Postbox modify took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") + //#if DEBUG + //startTime = CFAbsoluteTimeGetCurrent() + //#endif self.database.commit() + //#if DEBUG + print("(Postbox commit took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") + //#endif + + self.performDeferredUpdates() + subscriber.putNext(result) subscriber.putCompletion() } @@ -772,7 +1033,7 @@ public final class Postbox { private func findAdjacentMessageIds(peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex) -> (MessageId.Id?, MessageId.Id?) { var minId: MessageId.Id? var maxId: MessageId.Id? - for row in self.database.prepare("SELECT MIN(id), MAX(id) FROM peer_messages WHERE peerId = ? AND namespace = ?").run(peerId.toInt64(), Int64(namespace)) { + for row in self.database.prepareCached("SELECT MIN(id), MAX(id) FROM peer_messages WHERE peerId = ? AND namespace = ?").run(peerId.toInt64(), Int64(namespace)) { minId = MessageId.Id(row[0] as! Int64) maxId = MessageId.Id(row[1] as! Int64) } @@ -780,7 +1041,7 @@ public final class Postbox { if let minId = minId, maxId = maxId { var minTimestamp: Int32! var maxTimestamp: Int32! - for row in self.database.prepare("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (?, ?)").run(peerId.toInt64(), Int64(namespace), Int64(minId), Int64(maxId)) { + for row in self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (?, ?)").run(peerId.toInt64(), Int64(namespace), Int64(minId), Int64(maxId)) { let id = Int32(row[0] as! Int64) let timestamp = Int32(row[1] as! Int64) if id == minId { @@ -790,8 +1051,8 @@ public final class Postbox { } } - let earlierMidStatement = self.database.prepare("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id <= ? LIMIT 1") - let laterMidStatement = self.database.prepare("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id >= ? LIMIT 1") + let earlierMidStatement = self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id <= ? LIMIT 1") + let laterMidStatement = self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id >= ? LIMIT 1") func lowerBound(timestamp: Int32) -> MessageId.Id? { var leftId = minId @@ -883,16 +1144,16 @@ public final class Postbox { } } - private func fetchMessagesAround(peerId: PeerId, anchorId: MessageId, count: Int) -> ([Message], [MessageId.Namespace : Message], [MessageId.Namespace : Message]) { - var messages: [Message] = [] + private func fetchMessagesAround(peerId: PeerId, anchorId: MessageId, count: Int) -> ([RenderedMessage], [MessageId.Namespace : RenderedMessage], [MessageId.Namespace : RenderedMessage]) { + var messages: [RenderedMessage] = [] messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: anchorId.namespace, id: anchorId.id, count: count + 1) messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: anchorId.namespace, id: anchorId.id - 1, count: count + 1) - messages.sortInPlace({ MessageIndex($0) < MessageIndex($1) }) + messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) var i = messages.count - 1 while i >= 1 { - if messages[i].id == messages[i - 1].id { + if messages[i].message.id == messages[i - 1].message.id { messages.removeAtIndex(i) } i-- @@ -903,19 +1164,19 @@ public final class Postbox { } else { var index: MessageIndex! for message in messages { - if message.id == anchorId { - index = MessageIndex(message) + if message.message.id == anchorId { + index = MessageIndex(message.message) break } } if index == nil { - var closestId: MessageId.Id = messages[0].id.id + var closestId: MessageId.Id = messages[0].message.id.id var closestDistance = abs(closestId - anchorId.id) - let closestTimestamp: Int32 = messages[0].timestamp + let closestTimestamp: Int32 = messages[0].message.timestamp for message in messages { - if abs(message.id.id - anchorId.id) < closestDistance { - closestId = message.id.id - closestDistance = abs(message.id.id - anchorId.id) + if abs(message.message.id.id - anchorId.id) < closestDistance { + closestId = message.message.id.id + closestDistance = abs(message.message.id.id - anchorId.id) } } index = MessageIndex(id: MessageId(peerId: peerId, namespace: anchorId.namespace, id: closestId), timestamp: closestTimestamp) @@ -933,10 +1194,10 @@ public final class Postbox { } } - messages.sortInPlace({ MessageIndex($0) < MessageIndex($1) }) + messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) var i = messages.count - 1 while i >= 1 { - if messages[i].id == messages[i - 1].id { + if messages[i].message.id == messages[i - 1].message.id { messages.removeAtIndex(i) } i-- @@ -945,16 +1206,16 @@ public final class Postbox { var anchorIndex = messages.count / 2 i = 0 while i < messages.count { - if messages[i].id == index.id { + if messages[i].message.id == index.id { anchorIndex = i break } i++ } - var filteredMessages: [Message] = [] - var earlier: [MessageId.Namespace : Message] = [:] - var later: [MessageId.Namespace : Message] = [:] + var filteredMessages: [RenderedMessage] = [] + var earlier: [MessageId.Namespace : RenderedMessage] = [:] + var later: [MessageId.Namespace : RenderedMessage] = [:] i = anchorIndex var j = anchorIndex - 1 @@ -977,32 +1238,32 @@ public final class Postbox { i = leftIndex - 1 while i >= 0 { - if earlier[messages[i].id.namespace] == nil { - earlier[messages[i].id.namespace] = messages[i] + if earlier[messages[i].message.id.namespace] == nil { + earlier[messages[i].message.id.namespace] = messages[i] } i-- } i = rightIndex + 1 while i < messages.count { - if later[messages[i].id.namespace] == nil { - later[messages[i].id.namespace] = messages[i] + if later[messages[i].message.id.namespace] == nil { + later[messages[i].message.id.namespace] = messages[i] } i++ } - filteredMessages.sortInPlace({ MessageIndex($0) < MessageIndex($1) }) + filteredMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) return (filteredMessages, earlier, later) } } - private func fetchMessagesRelative(peerId: PeerId, earlier: Bool)(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { + private func fetchMessagesRelative(peerId: PeerId, earlier: Bool)(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [RenderedMessage] { var messages: [Message] = [] let sign = earlier ? "<" : ">" let order = earlier ? "DESC" : "ASC" - let statement = self.database.prepare("SELECT data, associatedMediaIds FROM peer_messages WHERE peerId = ? AND namespace = ? AND id \(sign) ? ORDER BY id \(order) LIMIT \(count)") + let statement = self.database.prepareCached("SELECT data, associatedMediaIds FROM peer_messages WHERE peerId = ? AND namespace = ? AND id \(sign) ? ORDER BY id \(order) LIMIT ?") let bound: Int64 if let id = id { bound = Int64(id) @@ -1012,7 +1273,7 @@ public final class Postbox { bound = Int64(Int32.min) } - for row in statement.run(Int64(peerId.toInt64()), Int64(namespace), bound) { + for row in statement.run(Int64(peerId.toInt64()), Int64(namespace), bound, Int64(count)) { let data = (row[0] as! Blob).data let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) if let message = decoder.decodeRootObject() as? Message { @@ -1022,7 +1283,7 @@ public final class Postbox { } } - return messages + return self.renderedMessages(messages) } private func fetchPeerEntryIndicesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntryIndex] { @@ -1049,12 +1310,12 @@ public final class Postbox { return entries } - private func messageForPeer(peerId: PeerId, id: MessageId) -> Message? { + private func messageForPeer(peerId: PeerId, id: MessageId) -> RenderedMessage? { for row in self.database.prepareCached("SELECT data, associatedMediaIds FROM peer_messages WHERE peerId = ? AND namespace = ? AND id = ?").run(peerId.toInt64(), Int64(id.namespace), Int64(id.id)) { let data = (row[0] as! Blob).data let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) if let message = decoder.decodeRootObject() as? Message { - return message + return self.renderedMessages([message]).first } else { print("(PostBox: can't decode message)") } @@ -1098,38 +1359,98 @@ public final class Postbox { return entries } - private func fetchMessagesTail(peerId: PeerId, count: Int) -> [Message] { - var messages: [Message] = [] + private func renderedMessages(messages: [Message]) -> [RenderedMessage] { + if messages.count == 0 { + return [] + } + + var peerIds = Set() + var mediaIds = Set() + + for message in messages { + for peerId in message.peerIds { + peerIds.insert(peerId) + } + for mediaId in message.mediaIds { + mediaIds.insert(mediaId) + } + } + + var arrayPeerIds: [PeerId] = [] + for id in peerIds { + arrayPeerIds.append(id) + } + let peers = self.peersWithIds(arrayPeerIds) + + var arrayMediaIds: [MediaId] = [] + for id in mediaIds { + arrayMediaIds.append(id) + } + let medias = self.mediaWithIds(arrayMediaIds) + + var result: [RenderedMessage] = [] + + for message in messages { + if message.peerIds.count == 0 && message.mediaIds.count == 0 { + result.append(RenderedMessage(message: message, peers: [], media: [])) + } else { + var messagePeers: [Peer] = [] + for id in message.peerIds { + if let peer = peers[id] { + messagePeers.append(peer) + } + } + + var messageMedia: [Media] = [] + for id in message.mediaIds { + if let media = medias[id] { + messageMedia.append(media) + } + } + + result.append(RenderedMessage(message: message, peers: messagePeers, media: messageMedia)) + } + } + + return result + } + + private func fetchMessagesTail(peerId: PeerId, count: Int) -> [RenderedMessage] { + var messages: [RenderedMessage] = [] for namespace in self.messageNamespaces { messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: nil, count: count) } - messages.sortInPlace({ MessageIndex($0) < MessageIndex($1)}) + messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message)}) return messages } public func tailMessageViewForPeerId(peerId: PeerId, count: Int) -> Signal { return Signal { subscriber in + let startTime = CFAbsoluteTimeGetCurrent() + let disposable = MetaDisposable() self.queue.dispatch { let tail = self.fetchMessagesTail(peerId, count: count + 1) - var messages: [Message] = [] + print("tailMessageViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") + + var messages: [RenderedMessage] = [] var i = tail.count - 1 while i >= 0 && i >= tail.count - count { messages.insert(tail[i], atIndex: 0) i-- } - var earlier: [MessageId.Namespace : Message] = [:] + var earlier: [MessageId.Namespace : RenderedMessage] = [:] for namespace in self.messageNamespaces { var i = tail.count - count - 1 while i >= 0 { - if tail[i].id.namespace == namespace { + if tail[i].message.id.namespace == namespace { earlier[namespace] = tail[i] break } @@ -1184,19 +1505,19 @@ public final class Postbox { if around.0.count == 0 { let tail = self.fetchMessagesTail(peerId, count: count + 1) - var messages: [Message] = [] + var messages: [RenderedMessage] = [] var i = tail.count - 1 while i >= 0 && i >= tail.count - count { messages.insert(tail[i], atIndex: 0) i-- } - var earlier: [MessageId.Namespace : Message] = [:] + var earlier: [MessageId.Namespace : RenderedMessage] = [:] for namespace in self.messageNamespaces { var i = tail.count - count - 1 while i >= 0 { - if tail[i].id.namespace == namespace { + if tail[i].message.id.namespace == namespace { earlier[namespace] = tail[i] break } @@ -1249,7 +1570,12 @@ public final class Postbox { let disposable = MetaDisposable() self.queue.dispatch { + let startTime = CFAbsoluteTimeGetCurrent() + let tail = self.fetchPeerEntriesRelative(true)(index: nil, count: count + 1) + self.fetchPeerEntriesRelative(true)(index: nil, count: count + 1) + + print("(Postbox fetchPeerEntriesRelative took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") var entries: [PeerViewEntry] = [] var i = tail.count - 1 diff --git a/submodules/SQLite.swift b/submodules/SQLite.swift index e53267d739..b0b93692fd 160000 --- a/submodules/SQLite.swift +++ b/submodules/SQLite.swift @@ -1 +1 @@ -Subproject commit e53267d7394c6df648d588bebc4db10dc853c75d +Subproject commit b0b93692fdac68e8ffa71e11b23c95b3e0f8b256 From 65d12e60c636679790c9bc2ec2c74e237097df42 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 30 Jul 2015 01:10:02 +0300 Subject: [PATCH 019/427] no message --- Postbox/Postbox.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 5b0a1c8875..ab23743c6e 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -752,7 +752,7 @@ public final class Postbox { private func deferMessageViewUpdate(view: MutableMessageView, pipe: Pipe) { var i = 0 var found = false - while i < self.deferredPeerViewsToUpdate.count { + while i < self.deferredMessageViewsToUpdate.count { if self.deferredMessageViewsToUpdate[i].1 === pipe { self.deferredMessageViewsToUpdate[i] = (view, pipe) found = true From 3b82299f950e4affa4531595eb0f4b12e9fd4489 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 10 Aug 2015 12:02:20 +0200 Subject: [PATCH 020/427] no message --- Postbox/Coding.swift | 11 +++ Postbox/Media.swift | 2 +- Postbox/Message.swift | 30 +++++-- Postbox/MessageView.swift | 10 ++- Postbox/Peer.swift | 1 + Postbox/PeerView.swift | 83 ++++++++++++++----- Postbox/Postbox.swift | 165 ++++++++++++++++++++++++++++++++------ 7 files changed, 242 insertions(+), 60 deletions(-) diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 864a6449fa..55c423660e 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -479,6 +479,17 @@ public final class Decoder { } } + public func decodeInt32ForKey(key: UnsafePointer) -> Int32? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { + var value: Int32 = 0 + memcpy(&value, self.buffer.memory + self.buffer.offset, 4) + self.buffer.offset += 4 + return value + } else { + return nil + } + } + public func decodeInt64ForKey(key: UnsafePointer) -> Int64 { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { var value: Int64 = 0 diff --git a/Postbox/Media.swift b/Postbox/Media.swift index a4f54c2540..5b53c468b2 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -9,7 +9,7 @@ public struct MediaId: Hashable, CustomStringConvertible { public var hashValue: Int { get { - return Int(self.id) + return Int((self.id & 0xffffffff) ^ ((self.id >> 32) & 0xffffffff)) } } diff --git a/Postbox/Message.swift b/Postbox/Message.swift index a2b25de31e..62e0829108 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -27,14 +27,20 @@ public struct MessageId: Hashable, CustomStringConvertible { } public init(_ buffer: ReadBuffer) { - self.peerId = PeerId(namespace: 0, id: 0) - self.namespace = 0 - self.id = 0 - memcpy(&self.peerId.namespace, buffer.memory + buffer.offset, 4) - memcpy(&self.peerId.id, buffer.memory + (buffer.offset + 4), 4) - memcpy(&self.namespace, buffer.memory + (buffer.offset + 8), 4) - memcpy(&self.id, buffer.memory + (buffer.offset + 12), 4) + var peerIdNamespaceValue: Int32 = 0 + memcpy(&peerIdNamespaceValue, buffer.memory + buffer.offset, 4) + var peerIdIdValue: Int32 = 0 + memcpy(&peerIdIdValue, buffer.memory + (buffer.offset + 4), 4) + self.peerId = PeerId(namespace: peerIdNamespaceValue, id: peerIdIdValue) + + var namespaceValue: Int32 = 0 + memcpy(&namespaceValue, buffer.memory + (buffer.offset + 8), 4) + self.namespace = namespaceValue + var idValue: Int32 = 0 + memcpy(&idValue, buffer.memory + (buffer.offset + 12), 4) + self.id = idValue + buffer.offset += 16 } @@ -114,7 +120,7 @@ public protocol Message: Coding { var peerIds: [PeerId] { get } } -public struct RenderedMessage { +public struct RenderedMessage: Equatable, Comparable { public let message: Message internal let incomplete: Bool @@ -135,3 +141,11 @@ public struct RenderedMessage { self.incomplete = false } } + +public func ==(lhs: RenderedMessage, rhs: RenderedMessage) -> Bool { + return lhs.message.id == rhs.message.id +} + +public func <(lhs: RenderedMessage, rhs: RenderedMessage) -> Bool { + return MessageIndex(lhs.message) < MessageIndex(rhs.message) +} diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift index 3050f7cf2a..aea71cd29b 100644 --- a/Postbox/MessageView.swift +++ b/Postbox/MessageView.swift @@ -59,9 +59,10 @@ public final class MutableMessageView: CustomStringConvertible { } else { self.earlier[message.message.id.namespace] = message } + return true + } else { + return false } - - return true } else if index > first { if next != nil && index > next! { let laterMessage = self.later[message.message.id.namespace] @@ -71,6 +72,9 @@ public final class MutableMessageView: CustomStringConvertible { } else { self.later[message.message.id.namespace] = message } + return true + } else { + return false } } else { self.messages.append(message) @@ -79,8 +83,8 @@ public final class MutableMessageView: CustomStringConvertible { self.earlier[earliest.message.id.namespace] = earliest self.messages.removeAtIndex(0) } + return true } - return true } else if index != last && index != first { var i = self.messages.count while i >= 1 { diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index 03ca43228a..a6f7fac52c 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -36,6 +36,7 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { buffer.offset += 4 var i = 0 var array: [PeerId] = [] + array.reserveCapacity(Int(length)) while i < Int(length) { var value: Int64 = 0 buffer.read(&value, offset: 0, length: 8) diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index 82085302e8..74b96c743f 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -1,27 +1,53 @@ import Foundation -public final class PeerViewEntry { +public final class PeerViewEntry: Equatable, Comparable { public let peerId: PeerId public let peer: Peer? - public let message: RenderedMessage + public let messageIndex: MessageIndex + public let message: RenderedMessage? public init(peer: Peer, message: RenderedMessage) { self.peerId = peer.id self.peer = peer self.message = message + self.messageIndex = MessageIndex(message.message) } public init(peerId: PeerId, message: RenderedMessage) { self.peerId = peerId self.peer = nil self.message = message + self.messageIndex = MessageIndex(message.message) } private init(peer: Peer?, peerId: PeerId, message: RenderedMessage) { self.peer = peer self.peerId = peerId self.message = message + self.messageIndex = MessageIndex(message.message) } + + private init(peer: Peer?, peerId: PeerId, message: RenderedMessage?, messageIndex: MessageIndex) { + self.peer = peer + self.peerId = peerId + self.message = message + self.messageIndex = messageIndex + } + + init(peer: Peer?, peerId: PeerId, messageIndex: MessageIndex) { + self.peer = peer + self.peerId = peerId + self.messageIndex = messageIndex + self.message = nil + } +} + +public func ==(lhs: PeerViewEntry, rhs: PeerViewEntry) -> Bool { + return PeerViewEntryIndex(lhs) == PeerViewEntryIndex(rhs) +} + +public func <(lhs: PeerViewEntry, rhs: PeerViewEntry) -> Bool { + return PeerViewEntryIndex(lhs) < PeerViewEntryIndex(rhs) } public struct PeerViewEntryIndex: Equatable, Comparable { @@ -30,7 +56,7 @@ public struct PeerViewEntryIndex: Equatable, Comparable { public init(_ entry: PeerViewEntry) { self.peerId = entry.peerId - self.messageIndex = MessageIndex(entry.message.message) + self.messageIndex = entry.messageIndex } public init(peerId: PeerId, messageIndex: MessageIndex) { @@ -263,14 +289,14 @@ public final class MutablePeerView: CustomStringConvertible { if let earlier = self.earlier { if let peer = peers[earlier.peerId] { - self.earlier = PeerViewEntry(peer: peer, message: earlier.message) + self.earlier = PeerViewEntry(peer: peer, peerId: earlier.peerId, message: earlier.message, messageIndex: earlier.messageIndex) updated = true } } if let later = self.later { if let peer = peers[later.peerId] { - self.later = PeerViewEntry(peer: peer, message: later.message) + self.later = PeerViewEntry(peer: peer, peerId: later.peerId, message: later.message, messageIndex: later.messageIndex) updated = true } } @@ -278,7 +304,8 @@ public final class MutablePeerView: CustomStringConvertible { var i = 0 while i < self.entries.count { if let peer = peers[self.entries[i].peerId] { - self.entries[i] = PeerViewEntry(peer: peer, message: self.entries[i].message) + let entry = self.entries[i] + self.entries[i] = PeerViewEntry(peer: peer, peerId: entry.peerId, message: entry.message, messageIndex: entry.messageIndex) updated = true } i++ @@ -291,19 +318,25 @@ public final class MutablePeerView: CustomStringConvertible { var result: [Message] = [] if let earlier = self.earlier { - if earlier.message.incomplete { - result.append(earlier.message.message) + if let message = earlier.message { + if message.incomplete { + result.append(message.message) + } } } if let later = self.later { - if later.message.incomplete { - result.append(later.message.message) + if let message = later.message { + if message.incomplete { + result.append(message.message) + } } } for entry in self.entries { - if entry.message.incomplete { - result.append(entry.message.message) + if let message = entry.message { + if message.incomplete { + result.append(message.message) + } } } @@ -312,20 +345,26 @@ public final class MutablePeerView: CustomStringConvertible { public func completeMessages(messages: [MessageId : RenderedMessage]) { if let earlier = self.earlier { - if let message = messages[earlier.message.message.id] { - self.earlier = PeerViewEntry(peer: earlier.peer, peerId: earlier.peerId, message: message) + if let earlierMessage = earlier.message { + if let message = messages[earlierMessage.message.id] { + self.earlier = PeerViewEntry(peer: earlier.peer, peerId: earlier.peerId, message: message) + } } } if let later = self.later { - if let message = messages[later.message.message.id] { - self.later = PeerViewEntry(peer: later.peer, peerId: later.peerId, message: message) + if let laterMessage = later.message { + if let message = messages[laterMessage.message.id] { + self.later = PeerViewEntry(peer: later.peer, peerId: later.peerId, message: message) + } } } var i = 0 while i < self.entries.count { - if let message = messages[self.entries[i].message.message.id] { - self.entries[i] = PeerViewEntry(peer: self.entries[i].peer, peerId: self.entries[i].peerId, message: message) + if let entryMessage = self.entries[i].message { + if let message = messages[entryMessage.message.id] { + self.entries[i] = PeerViewEntry(peer: self.entries[i].peer, peerId: self.entries[i].peerId, message: message) + } } i++ } @@ -336,7 +375,7 @@ public final class MutablePeerView: CustomStringConvertible { if let earlier = self.earlier { string += "more(" - string += "(p \(earlier.peerId.namespace):\(earlier.peerId.id), m \(earlier.message.message.id.namespace):\(earlier.message.message.id.id)—\(earlier.message.message.timestamp)" + string += "(p \(earlier.peerId.namespace):\(earlier.peerId.id), m \(earlier.message?.message.id.namespace ?? 0):\(earlier.message?.message.id.id ?? 0)—\(earlier.message?.message.timestamp ?? 0)" string += ") " } @@ -348,13 +387,13 @@ public final class MutablePeerView: CustomStringConvertible { } else { string += ", " } - string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.message.id.namespace):\(entry.message.message.id.id)—\(entry.message.message.timestamp))" + string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message?.message.id.namespace ?? 0):\(entry.message?.message.id.id ?? 0)—\(entry.message?.message.timestamp ?? 0))" } string += "]" if let later = self.later { string += " more(" - string += "(p \(later.peerId.namespace):\(later.peerId), m \(later.message.message.id.namespace):\(later.message.message.id.id)—\(later.message.message.timestamp)" + string += "(p \(later.peerId.namespace):\(later.peerId), m \(later.message?.message.id.namespace ?? 0):\(later.message?.message.id.id ?? 0)—\(later.message?.message.timestamp ?? 0)" string += ")" } @@ -400,7 +439,7 @@ public final class PeerView: CustomStringConvertible { } else { string += ", " } - string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.message.id.namespace):\(entry.message.message.id.id)—\(entry.message.message.timestamp))" + string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message?.message.id.namespace ?? 0):\(entry.message?.message.id.id ?? 0)—\(entry.message?.message.timestamp ?? 0))" } string += "]" diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index ab23743c6e..3e4aea8711 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -1,6 +1,6 @@ import Foundation - import SwiftSignalKit +import sqlcipher public protocol PostboxState: Coding { @@ -21,6 +21,13 @@ public final class Modifier { self.postbox?.deleteMessagesWithIds(ids) } + public func deleteMessagesWithAbsoluteIndexedIds(ids: [Int32]) { + if let postbox = self.postbox { + let messageIds = postbox.messageIdsForAbsoluteIndexedIds(ids) + postbox.deleteMessagesWithIds(messageIds) + } + } + public func getState() -> State? { return self.postbox?.getState() } @@ -32,11 +39,16 @@ public final class Modifier { public func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer) { self.postbox?.updatePeers(peers, update: update) } + + public func peersWithIds(ids: [PeerId]) -> [PeerId : Peer] { + return self.postbox?.peersWithIds(ids) ?? [:] + } } public final class Postbox { private let basePath: String private let messageNamespaces: [MessageId.Namespace] + private let absoluteIndexedMessageNamespaces: [MessageId.Namespace] private let queue = SwiftSignalKit.Queue() private var database: Database! @@ -48,9 +60,10 @@ public final class Postbox { private var statePipe: Pipe = Pipe() - public init(basePath: String, messageNamespaces: [MessageId.Namespace]) { + public init(basePath: String, messageNamespaces: [MessageId.Namespace], absoluteIndexedMessageNamespaces: [MessageId.Namespace]) { self.basePath = basePath self.messageNamespaces = messageNamespaces + self.absoluteIndexedMessageNamespaces = absoluteIndexedMessageNamespaces self.openDatabase() } @@ -62,10 +75,10 @@ public final class Postbox { try NSFileManager.defaultManager().createDirectoryAtPath(self.basePath, withIntermediateDirectories: true, attributes: nil) } catch _ { } - self.database = Database(self.basePath.stringByAppendingPathComponent("db")) + self.database = Database(self.basePath + "/db") let result = self.database.scalar("PRAGMA user_version") as! Int64 - let version: Int64 = 7 + let version: Int64 = 11 if result == version { print("(Postbox schema version \(result))") } else { @@ -77,7 +90,7 @@ public final class Postbox { } catch (_) { } - self.database = Database(self.basePath.stringByAppendingPathComponent("db")) + self.database = Database(self.basePath + "/db") } print("(Postbox creating schema)") self.createSchema() @@ -108,6 +121,9 @@ public final class Postbox { //peer_messages self.database.execute("CREATE TABLE peer_messages (peerId INTEGER, namespace INTEGER, id INTEGER, data BLOB, associatedMediaIds BLOB, timestamp INTEGER, PRIMARY KEY(peerId, namespace, id))") + //peer_absolute_indexed_message_ids + self.database.execute("CREATE TABLE peer_absolute_indexed_message_ids (id INTEGER PRIMARY KEY, completeId BLOB)") + //peer_media self.database.execute("CREATE TABLE peer_media (peerId INTEGER, mediaNamespace INTEGER, messageNamespace INTEGER, messageId INTEGER, PRIMARY KEY (peerId, mediaNamespace, messageNamespace, messageId))") self.database.execute("CREATE INDEX peer_media_peerId_messageNamespace_messageId ON peer_media (peerId, messageNamespace, messageId)") @@ -126,9 +142,7 @@ public final class Postbox { self.database.execute("CREATE TABLE peers (id INTEGER PRIMARY KEY, data BLOB)") } - private class func peerViewEntryIndexForBlob(blob: Blob) -> PeerViewEntryIndex { - let buffer = ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false) - + private class func peerViewEntryIndexForBuffer(buffer: ReadBuffer) -> PeerViewEntryIndex { var timestamp: Int32 = 0 buffer.read(×tamp, offset: 0, length: 4) timestamp = Int32(bigEndian: timestamp) @@ -373,6 +387,9 @@ public final class Postbox { let encoder = Encoder() encoder.encodeRootObject(state) + + + let blob = Blob(data: encoder.makeData()) self.database.prepareCached("INSERT OR REPLACE INTO state (id, data) VALUES (?, ?)").run(Int64(0), blob) @@ -444,6 +461,7 @@ public final class Postbox { private func addMessages(messages: [Message], medias: [Media]) { let messageInsertStatement = self.database.prepareCached("INSERT INTO peer_messages (peerId, namespace, id, data, associatedMediaIds, timestamp) VALUES (?, ?, ?, ?, ?, ?)") + let absoluteMessageIdsInsertStatement = self.database.prepareCached("INSERT INTO peer_absolute_indexed_message_ids (id, completeId) VALUES (?, ?)") let peerMediaInsertStatement = self.database.prepareCached("INSERT INTO peer_media (peerId, mediaNamespace, messageNamespace, messageId) VALUES (?, ?, ?, ?)") let mediaInsertStatement = self.database.prepareCached("INSERT INTO media (namespace, id, data, associatedMessageIds) VALUES (?, ?, ?, ?)") let referencedMessageIdsStatement = self.database.prepareCached("SELECT associatedMessageIds FROM media WHERE namespace = ? AND id = ?") @@ -486,6 +504,13 @@ public final class Postbox { } } + let completeIdData = NSMutableData(capacity: 8 + 4 + 4)! + var zero: Int64 = 0 + completeIdData.appendBytes(&zero, length: 8) + completeIdData.appendBytes(&zero, length: 8) + let completeIdDataMutableBytes = completeIdData.mutableBytes + let completeIdBlob = Blob(data: completeIdData) + for message in peerMessages { if existingMessageIds.contains(message.id) { continue @@ -512,6 +537,16 @@ public final class Postbox { messageInsertStatement.run(peerId.toInt64(), Int64(message.id.namespace), Int64(message.id.id), messageBlob, referencedMediaIdsMediaIdsBlob, Int64(message.timestamp)) + if self.absoluteIndexedMessageNamespaces.contains(message.id.namespace) { + var peerIdValue = peerId.toInt64() + memcpy(completeIdDataMutableBytes, &peerIdValue, 8) + var messageIdNamespaceValue: Int32 = message.id.namespace + memcpy(completeIdDataMutableBytes + 8, &messageIdNamespaceValue, 4) + var messageIdIdValue: Int32 = message.id.id + memcpy(completeIdDataMutableBytes + 12, &messageIdIdValue, 4) + absoluteMessageIdsInsertStatement.run(Int64(message.id.id), completeIdBlob) + } + for id in message.mediaIds { peerMediaInsertStatement.run(peerId.toInt64(), Int64(id.namespace), Int64(message.id.namespace), Int64(message.id.id)) } @@ -649,7 +684,7 @@ public final class Postbox { for id in ids { for row in select.run(Int64(id.namespace), id.id) { let blob = row[0] as! Blob - if let media = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false)) as? Media { + if let media = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false)).decodeRootObject() as? Media { result[media.id] = media } break @@ -802,7 +837,8 @@ public final class Postbox { private func updatePeerEntry(peerId: PeerId, message: RenderedMessage?, replace: Bool = false) { var currentIndex: PeerViewEntryIndex? for row in self.database.prepareCached("SELECT entry FROM peer_entries WHERE peerId = ?").run(peerId.toInt64()) { - currentIndex = Postbox.peerViewEntryIndexForBlob(row[0] as! Blob) + let blob = row[0] as! Blob + currentIndex = Postbox.peerViewEntryIndexForBuffer(ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false)) break } @@ -859,6 +895,40 @@ public final class Postbox { } } + private func messageIdsForAbsoluteIndexedIds(ids: [Int32]) -> [MessageId] { + if ids.count == 0 { + return [] + } + + var queryString = "SELECT completeId FROM peer_absolute_indexed_message_ids WHERE id IN (" + var first = true + for id in ids { + if first { + first = false + } else { + queryString += "," + } + queryString += "\(id)" + } + queryString += ")" + + var result: [MessageId] = [] + for row in self.database.prepare(queryString).run() { + let blob = row[0] as! Blob + let blobBytes = blob.data.bytes + + var peerIdValue: Int64 = 0 + memcpy(&peerIdValue, blobBytes, 8) + var messageIdNamespaceValue: Int32 = 0 + memcpy(&messageIdNamespaceValue, blobBytes + 8, 4) + var messageIdIdValue: Int32 = 0 + memcpy(&messageIdIdValue, blobBytes + 12, 4) + result.append(MessageId(peerId: PeerId(peerIdValue), namespace: messageIdNamespaceValue, id: messageIdIdValue)) + } + + return result + } + private func deleteMessagesWithIds(ids: [MessageId]) { for (peerId, messageIds) in Postbox.messageIdsGroupedByPeerId(ids) { let messageIdsByNamespace = Postbox.messageIdsGroupedByNamespace(messageIds) @@ -889,6 +959,21 @@ public final class Postbox { } queryString += ")" self.database.prepare(queryString).run(peerId.toInt64(), Int64(namespace)) + + if self.absoluteIndexedMessageNamespaces.contains(namespace) { + var queryString = "DELETE FROM peer_absolute_indexed_message_ids WHERE id IN (" + var first = true + for id in messageIds { + if first { + first = false + } else { + queryString += "," + } + queryString += "\(id.id)" + } + queryString += ")" + self.database.prepare(queryString).run() + } } for (namespace, messageIds) in messageIdsByNamespace { @@ -1263,7 +1348,6 @@ public final class Postbox { let sign = earlier ? "<" : ">" let order = earlier ? "DESC" : "ASC" - let statement = self.database.prepareCached("SELECT data, associatedMediaIds FROM peer_messages WHERE peerId = ? AND namespace = ? AND id \(sign) ? ORDER BY id \(order) LIMIT ?") let bound: Int64 if let id = id { bound = Int64(id) @@ -1273,15 +1357,25 @@ public final class Postbox { bound = Int64(Int32.min) } - for row in statement.run(Int64(peerId.toInt64()), Int64(namespace), bound, Int64(count)) { - let data = (row[0] as! Blob).data - let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) - if let message = decoder.decodeRootObject() as? Message { - messages.append(message) + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT data, associatedMediaIds FROM peer_messages WHERE peerId=\(peerId.toInt64()) AND namespace=\(namespace) AND id \(sign) \(bound) ORDER BY id \(order) LIMIT \(count)", -1, &statement, nil) + + while true { + let result = sqlite3_step(statement) + if result == SQLITE_ROW { + let length = sqlite3_column_bytes(statement, 0) + let data = sqlite3_column_blob(statement, 0) + let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data), length: Int(length), freeWhenDone: false)) + if let message = decoder.decodeRootObject() as? Message { + messages.append(message) + } else { + print("(PostBox: can't decode message)") + } } else { - print("(PostBox: can't decode message)") + break } } + sqlite3_finalize(statement) return self.renderedMessages(messages) } @@ -1289,23 +1383,32 @@ public final class Postbox { private func fetchPeerEntryIndicesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntryIndex] { var entries: [PeerViewEntryIndex] = [] - let rows: Statement + var statement: COpaquePointer = nil if let index = index { let bound = Postbox.blobForPeerViewEntryIndex(index) let sign = earlier ? "<" : ">" let order = earlier ? "DESC" : "ASC" - let statement = self.database.prepareCached("SELECT entry FROM peer_entries WHERE entry \(sign) ? ORDER BY entry \(order) LIMIT ?") - rows = statement.run(bound, Int64(count)) + + sqlite3_prepare_v2(self.database.handle, "SELECT entry FROM peer_entries WHERE entry \(sign) ? ORDER BY entry \(order) LIMIT \(count)", -1, &statement, nil) + sqlite3_bind_blob(statement, 0, bound.data.bytes, Int32(bound.data.length), nil) + } else { let order = earlier ? "DESC" : "ASC" - let statement = self.database.prepareCached("SELECT entry FROM peer_entries ORDER BY entry \(order) LIMIT ?") - rows = statement.run(Int64(count)) + sqlite3_prepare_v2(self.database.handle, "SELECT entry FROM peer_entries ORDER BY entry \(order) LIMIT \(count)", -1, &statement, nil) } - for row in rows { - entries.append(Postbox.peerViewEntryIndexForBlob(row[0] as! Blob)) + while true { + let result = sqlite3_step(statement) + if result == SQLITE_ROW { + let length = sqlite3_column_bytes(statement, 0) + let data = sqlite3_column_blob(statement, 0) + entries.append(Postbox.peerViewEntryIndexForBuffer(ReadBuffer(memory: UnsafeMutablePointer(data), length: Int(length), freeWhenDone: false))) + } else { + break + } } + sqlite3_finalize(statement) return entries } @@ -1350,7 +1453,14 @@ public final class Postbox { entries.append(entry) } else { - print("(PostBox: missing message for peer entry)") + let entry: PeerViewEntry + if let peer = peer { + entry = PeerViewEntry(peer: peer, peerId: entryIndex.peerId, messageIndex: entryIndex.messageIndex) + } else { + entry = PeerViewEntry(peer: nil, peerId: entryIndex.peerId, messageIndex: entryIndex.messageIndex) + } + + entries.append(entry) } } @@ -1501,6 +1611,8 @@ public final class Postbox { self.queue.dispatch { let mutableView: MutableMessageView + let startTime = CFAbsoluteTimeGetCurrent() + let around = self.fetchMessagesAround(peerId, anchorId: id, count: count) if around.0.count == 0 { let tail = self.fetchMessagesTail(peerId, count: count + 1) @@ -1530,6 +1642,8 @@ public final class Postbox { mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: around.1, messages: around.0, later: around.2) } + print("aroundMessageViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") + let record = (mutableView, Pipe()) let index: Bag<(MutableMessageView, Pipe)>.Index @@ -1573,7 +1687,6 @@ public final class Postbox { let startTime = CFAbsoluteTimeGetCurrent() let tail = self.fetchPeerEntriesRelative(true)(index: nil, count: count + 1) - self.fetchPeerEntriesRelative(true)(index: nil, count: count + 1) print("(Postbox fetchPeerEntriesRelative took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") From 341e9a6609119e72b1cd3902b0f0edfc5aa258b1 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 10 Aug 2015 12:02:33 +0200 Subject: [PATCH 021/427] no message --- submodules/SQLite.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/SQLite.swift b/submodules/SQLite.swift index b0b93692fd..dca58041a5 160000 --- a/submodules/SQLite.swift +++ b/submodules/SQLite.swift @@ -1 +1 @@ -Subproject commit b0b93692fdac68e8ffa71e11b23c95b3e0f8b256 +Subproject commit dca58041a5b02d508d967d72a3b3e48b825692e4 From ab426e64c20bd8911bc236cc0e125a045ca5bf72 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 22 Sep 2015 15:24:13 +0300 Subject: [PATCH 022/427] no message --- .gitmodules | 3 + Postbox.xcodeproj/project.pbxproj | 60 +- .../xcschemes/PostboxTests.xcscheme | 56 + .../xcschemes/xcschememanagement.plist | 5 + Postbox/Coding.swift | 459 +- Postbox/LmdbValueBox.swift | 395 + Postbox/Media.swift | 4 +- Postbox/MediaBox.swift | 145 + Postbox/Postbox.swift | 952 +- Postbox/PostboxCodingUtils.swift | 233 + Postbox/PostboxPrivate/module.modulemap | 1 + Postbox/PostboxTables.swift | 274 + Postbox/SqliteValueBox.swift | 574 + Postbox/ValueBox.swift | 14 + Postbox/ValueBoxKey.swift | 144 + PostboxTests/CodingTests.swift | 108 +- PostboxTests/PostboxTests.swift | 826 -- submodules/lmdb/libraries/liblmdb/.gitignore | 20 + submodules/lmdb/libraries/liblmdb/CHANGES | 112 + submodules/lmdb/libraries/liblmdb/COPYRIGHT | 20 + submodules/lmdb/libraries/liblmdb/Doxyfile | 1631 +++ submodules/lmdb/libraries/liblmdb/LICENSE | 47 + submodules/lmdb/libraries/liblmdb/Makefile | 97 + submodules/lmdb/libraries/liblmdb/lmdb.h | 1565 +++ submodules/lmdb/libraries/liblmdb/mdb.c | 9519 +++++++++++++++++ submodules/lmdb/libraries/liblmdb/mdb_copy.1 | 54 + submodules/lmdb/libraries/liblmdb/mdb_copy.c | 82 + submodules/lmdb/libraries/liblmdb/mdb_dump.1 | 75 + submodules/lmdb/libraries/liblmdb/mdb_dump.c | 317 + submodules/lmdb/libraries/liblmdb/mdb_load.1 | 77 + submodules/lmdb/libraries/liblmdb/mdb_load.c | 452 + submodules/lmdb/libraries/liblmdb/mdb_stat.1 | 64 + submodules/lmdb/libraries/liblmdb/mdb_stat.c | 263 + submodules/lmdb/libraries/liblmdb/midl.c | 360 + submodules/lmdb/libraries/liblmdb/midl.h | 186 + submodules/lmdb/libraries/liblmdb/mtest.c | 173 + submodules/lmdb/libraries/liblmdb/mtest2.c | 123 + submodules/lmdb/libraries/liblmdb/mtest3.c | 133 + submodules/lmdb/libraries/liblmdb/mtest4.c | 168 + submodules/lmdb/libraries/liblmdb/mtest5.c | 135 + submodules/lmdb/libraries/liblmdb/mtest6.c | 137 + .../lmdb/libraries/liblmdb/sample-bdb.txt | 73 + .../lmdb/libraries/liblmdb/sample-mdb.txt | 62 + submodules/lmdb/libraries/liblmdb/tooltag | 22 + 44 files changed, 18441 insertions(+), 1779 deletions(-) create mode 100644 Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme create mode 100644 Postbox/LmdbValueBox.swift create mode 100644 Postbox/MediaBox.swift create mode 100644 Postbox/PostboxCodingUtils.swift create mode 100644 Postbox/PostboxTables.swift create mode 100644 Postbox/SqliteValueBox.swift create mode 100644 Postbox/ValueBox.swift create mode 100644 Postbox/ValueBoxKey.swift delete mode 100644 PostboxTests/PostboxTests.swift create mode 100644 submodules/lmdb/libraries/liblmdb/.gitignore create mode 100644 submodules/lmdb/libraries/liblmdb/CHANGES create mode 100644 submodules/lmdb/libraries/liblmdb/COPYRIGHT create mode 100644 submodules/lmdb/libraries/liblmdb/Doxyfile create mode 100644 submodules/lmdb/libraries/liblmdb/LICENSE create mode 100644 submodules/lmdb/libraries/liblmdb/Makefile create mode 100644 submodules/lmdb/libraries/liblmdb/lmdb.h create mode 100644 submodules/lmdb/libraries/liblmdb/mdb.c create mode 100644 submodules/lmdb/libraries/liblmdb/mdb_copy.1 create mode 100644 submodules/lmdb/libraries/liblmdb/mdb_copy.c create mode 100644 submodules/lmdb/libraries/liblmdb/mdb_dump.1 create mode 100644 submodules/lmdb/libraries/liblmdb/mdb_dump.c create mode 100644 submodules/lmdb/libraries/liblmdb/mdb_load.1 create mode 100644 submodules/lmdb/libraries/liblmdb/mdb_load.c create mode 100644 submodules/lmdb/libraries/liblmdb/mdb_stat.1 create mode 100644 submodules/lmdb/libraries/liblmdb/mdb_stat.c create mode 100644 submodules/lmdb/libraries/liblmdb/midl.c create mode 100644 submodules/lmdb/libraries/liblmdb/midl.h create mode 100644 submodules/lmdb/libraries/liblmdb/mtest.c create mode 100644 submodules/lmdb/libraries/liblmdb/mtest2.c create mode 100644 submodules/lmdb/libraries/liblmdb/mtest3.c create mode 100644 submodules/lmdb/libraries/liblmdb/mtest4.c create mode 100644 submodules/lmdb/libraries/liblmdb/mtest5.c create mode 100644 submodules/lmdb/libraries/liblmdb/mtest6.c create mode 100644 submodules/lmdb/libraries/liblmdb/sample-bdb.txt create mode 100644 submodules/lmdb/libraries/liblmdb/sample-mdb.txt create mode 100644 submodules/lmdb/libraries/liblmdb/tooltag diff --git a/.gitmodules b/.gitmodules index a075144d43..fe42ab59be 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "submodules/SQLite.swift"] path = submodules/SQLite.swift url = git@github.com:peter-iakovlev/SQLite.swift.git +[submodule "submodules/lmdb"] + path = submodules/lmdb + url = https://github.com/LMDB/lmdb.git diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 5b3a1ed5a8..fb5836c807 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -8,9 +8,16 @@ /* Begin PBXBuildFile section */ D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageView.swift */; }; + D00E0FB31B84CEDA002E4EB5 /* Display.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D00E0FB21B84CEDA002E4EB5 /* Display.framework */; }; + D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB41B85D192002E4EB5 /* lmdb.h */; }; + D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; }; + D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB61B85D192002E4EB5 /* midl.c */; }; + D00E0FBB1B85D192002E4EB5 /* midl.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB71B85D192002E4EB5 /* midl.h */; }; + D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */; }; D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044E15D1B2ACB9C001EE087 /* CodingTests.swift */; }; D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */ = {isa = PBXBuildFile; fileRef = D044E1621B2AD677001EE087 /* MurMurHash32.m */; }; D044E1641B2AD718001EE087 /* MurMurHash32.h in Headers */ = {isa = PBXBuildFile; fileRef = D044E1611B2AD667001EE087 /* MurMurHash32.h */; settings = {ATTRIBUTES = (Public, ); }; }; + D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; }; D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516411B2D9CEF00AE42E0 /* sqlite3.h */; }; D07516461B2D9CEF00AE42E0 /* sqlite3ext.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */; }; @@ -22,13 +29,17 @@ D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; + D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; + D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; + D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; + D0977FA21B82930C009994B2 /* PostboxCodingUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */; }; + D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA31B829F87009994B2 /* PostboxTables.swift */; }; D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B76BE61B66639F0095CF45 /* DeferredString.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; D0D225261B4D84930085E26D /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D225251B4D84930085E26D /* PeerView.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; - D0E3A75D1B28A7E300A402D9 /* PostboxTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */; }; D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7811B28ADD000A402D9 /* Postbox.swift */; }; D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; @@ -48,9 +59,16 @@ /* Begin PBXFileReference section */ D003E4E51B38DBDB00C22CBC /* MessageView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageView.swift; sourceTree = ""; }; + D00E0FB21B84CEDA002E4EB5 /* Display.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Display.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/Display.framework"; sourceTree = ""; }; + D00E0FB41B85D192002E4EB5 /* lmdb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lmdb.h; path = submodules/lmdb/libraries/liblmdb/lmdb.h; sourceTree = SOURCE_ROOT; }; + D00E0FB51B85D192002E4EB5 /* mdb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = mdb.c; path = submodules/lmdb/libraries/liblmdb/mdb.c; sourceTree = SOURCE_ROOT; }; + D00E0FB61B85D192002E4EB5 /* midl.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = midl.c; path = submodules/lmdb/libraries/liblmdb/midl.c; sourceTree = SOURCE_ROOT; }; + D00E0FB71B85D192002E4EB5 /* midl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = midl.h; path = submodules/lmdb/libraries/liblmdb/midl.h; sourceTree = SOURCE_ROOT; }; + D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LmdbValueBox.swift; sourceTree = ""; }; D044E15D1B2ACB9C001EE087 /* CodingTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CodingTests.swift; sourceTree = ""; }; D044E1611B2AD667001EE087 /* MurMurHash32.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MurMurHash32.h; sourceTree = ""; }; D044E1621B2AD677001EE087 /* MurMurHash32.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MurMurHash32.m; sourceTree = ""; }; + D055BD321B7D3D2D00F06C0A /* MediaBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaBox.swift; sourceTree = ""; }; D07516401B2D9CEF00AE42E0 /* sqlite3.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = sqlite3.c; sourceTree = ""; }; D07516411B2D9CEF00AE42E0 /* sqlite3.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3.h; sourceTree = ""; }; D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3ext.h; sourceTree = ""; }; @@ -63,6 +81,11 @@ D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; + D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = ""; }; + D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; + D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; + D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxCodingUtils.swift; sourceTree = ""; }; + D0977FA31B829F87009994B2 /* PostboxTables.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxTables.swift; sourceTree = ""; }; D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; @@ -72,7 +95,6 @@ D0E3A74F1B28A7E300A402D9 /* Postbox.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Postbox.h; sourceTree = ""; }; D0E3A7551B28A7E300A402D9 /* PostboxTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = PostboxTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A75B1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; - D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostboxTests.swift; sourceTree = ""; }; D0E3A7811B28ADD000A402D9 /* Postbox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Postbox.swift; sourceTree = ""; }; D0E3A7831B28AE0900A402D9 /* Peer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Peer.swift; sourceTree = ""; }; D0E3A7871B28AE9C00A402D9 /* Coding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Coding.swift; sourceTree = ""; }; @@ -85,6 +107,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + D00E0FB31B84CEDA002E4EB5 /* Display.framework in Frameworks */, D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -100,9 +123,21 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + D00E0FBC1B85D194002E4EB5 /* lmdb */ = { + isa = PBXGroup; + children = ( + D00E0FB41B85D192002E4EB5 /* lmdb.h */, + D00E0FB51B85D192002E4EB5 /* mdb.c */, + D00E0FB61B85D192002E4EB5 /* midl.c */, + D00E0FB71B85D192002E4EB5 /* midl.h */, + ); + name = lmdb; + sourceTree = ""; + }; D07515FC1B2C44A200AE42E0 /* thirdparty */ = { isa = PBXGroup; children = ( + D00E0FBC1B85D194002E4EB5 /* lmdb */, D07516731B2EC8C700AE42E0 /* sqlite.swift */, ); name = thirdparty; @@ -144,6 +179,7 @@ D0E3A7401B28A7E300A402D9 = { isa = PBXGroup; children = ( + D00E0FB21B84CEDA002E4EB5 /* Display.framework */, D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */, D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */, D0E3A74C1B28A7E300A402D9 /* Postbox */, @@ -174,7 +210,14 @@ D0E3A7A11B28B7DC00A402D9 /* Media.swift */, D003E4E51B38DBDB00C22CBC /* MessageView.swift */, D0D225251B4D84930085E26D /* PeerView.swift */, + D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */, D0E3A7811B28ADD000A402D9 /* Postbox.swift */, + D0977FA31B829F87009994B2 /* PostboxTables.swift */, + D055BD321B7D3D2D00F06C0A /* MediaBox.swift */, + D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */, + D0977F9B1B822DB4009994B2 /* ValueBox.swift */, + D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */, + D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */, D0E3A74D1B28A7E300A402D9 /* Supporting Files */, ); path = Postbox; @@ -194,7 +237,6 @@ D0E3A7591B28A7E300A402D9 /* PostboxTests */ = { isa = PBXGroup; children = ( - D0E3A75C1B28A7E300A402D9 /* PostboxTests.swift */, D0E3A75A1B28A7E300A402D9 /* Supporting Files */, D044E15D1B2ACB9C001EE087 /* CodingTests.swift */, ); @@ -216,7 +258,9 @@ isa = PBXHeadersBuildPhase; buildActionMask = 2147483647; files = ( + D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */, D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */, + D00E0FBB1B85D192002E4EB5 /* midl.h in Headers */, D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */, D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */, D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */, @@ -327,16 +371,25 @@ D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, + D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */, D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */, + D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, + D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, + D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */, D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */, D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, + D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */, + D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */, D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, D0D225261B4D84930085E26D /* PeerView.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, + D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, + D0977FA21B82930C009994B2 /* PostboxCodingUtils.swift in Sources */, + D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */, D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */, D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, @@ -348,7 +401,6 @@ buildActionMask = 2147483647; files = ( D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */, - D0E3A75D1B28A7E300A402D9 /* PostboxTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme new file mode 100644 index 0000000000..2a374db351 --- /dev/null +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist index db91d6f463..b20f3c66cb 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist @@ -9,6 +9,11 @@ orderHint 1 + PostboxTests.xcscheme + + orderHint + 12 + SuppressBuildableAutocreation diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 55c423660e..a90fbc0d63 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -5,7 +5,7 @@ public protocol Coding { func encode(encoder: Encoder) } -private struct EncodableTypeStore { +private final class EncodableTypeStore { var dict: [Int32 : Decoder -> Coding] = [:] func decode(typeHash: Int32, decoder: Decoder) -> Coding? { @@ -17,7 +17,10 @@ private struct EncodableTypeStore { } } -private var typeStore = EncodableTypeStore() +private let _typeStore = EncodableTypeStore() +private let typeStore = { () -> EncodableTypeStore in + return _typeStore +}() public func declareEncodable(type: Any.Type, f: Decoder -> Coding) { let string = "\(type)" @@ -28,22 +31,62 @@ public func declareEncodable(type: Any.Type, f: Decoder -> Coding) { typeStore.dict[murMurHashString32("\(type)")] = f } -public final class WriteBuffer { - var memory: UnsafeMutablePointer = nil +public class MemoryBuffer: Equatable, CustomStringConvertible { + var memory: UnsafeMutablePointer var capacity: Int - var offset: Int + var length: Int + var freeWhenDone: Bool + + public init(memory: UnsafeMutablePointer, capacity: Int, length: Int, freeWhenDone: Bool) { + self.memory = memory + self.capacity = capacity + self.length = length + self.freeWhenDone = freeWhenDone + } + + public init(data: NSData) { + self.memory = UnsafeMutablePointer(data.bytes) + self.capacity = data.length + self.length = data.length + self.freeWhenDone = false + } public init() { - self.memory = malloc(32) - self.capacity = 32 - self.offset = 0 + self.memory = nil + self.capacity = 0 + self.length = 0 + self.freeWhenDone = false } deinit { - free(self.memory) + if self.freeWhenDone { + free(self.memory) + } } - func makeReadBufferAndReset() -> ReadBuffer { + public var description: String { + let hexString = NSMutableString() + let bytes = UnsafeMutablePointer(self.memory) + for i in 0 ..< self.length { + hexString.appendFormat("%02x", UInt(bytes[i])) + } + + return hexString as String + } +} + +public func ==(lhs: MemoryBuffer, rhs: MemoryBuffer) -> Bool { + return lhs.length == rhs.length && memcmp(lhs.memory, rhs.memory, lhs.length) == 0 +} + +public final class WriteBuffer: MemoryBuffer { + var offset = 0 + + public override init() { + super.init(memory: malloc(32), capacity: 32, length: 0, freeWhenDone: true) + } + + public func makeReadBufferAndReset() -> ReadBuffer { let buffer = ReadBuffer(memory: self.memory, length: self.offset, freeWhenDone: true) self.memory = malloc(32) self.capacity = 32 @@ -51,7 +94,11 @@ public final class WriteBuffer { return buffer } - func makeData() -> NSData { + public func readBufferNoCopy() -> ReadBuffer { + return ReadBuffer(memory: self.memory, length: self.offset, freeWhenDone: false) + } + + public func makeData() -> NSData { return NSData(bytes: self.memory, length: self.offset) } @@ -59,33 +106,30 @@ public final class WriteBuffer { self.offset = 0 } - func write(data: UnsafePointer, offset: Int, length: Int) { + public func write(data: UnsafePointer, offset: Int, length: Int) { if self.offset + length > self.capacity { self.capacity = self.offset + length + 256 self.memory = realloc(self.memory, self.capacity) } memcpy(self.memory + self.offset, data + offset, length) self.offset += length + self.length = self.offset } } -public final class ReadBuffer { - var memory: UnsafeMutablePointer - var length: Int - var offset : Int - let freeWhenDone: Bool +public final class ReadBuffer: MemoryBuffer { + var offset = 0 - init(memory: UnsafeMutablePointer, length: Int, freeWhenDone: Bool) { - self.memory = memory - self.length = length - self.offset = 0 - self.freeWhenDone = freeWhenDone + public init(memory: UnsafeMutablePointer, length: Int, freeWhenDone: Bool) { + super.init(memory: memory, capacity: length, length: length, freeWhenDone: freeWhenDone) } - deinit { - if self.freeWhenDone { - free(self.memory) - } + public init(memoryBufferNoCopy: MemoryBuffer) { + super.init(memory: memoryBufferNoCopy.memory, capacity: memoryBufferNoCopy.length, length: memoryBufferNoCopy.length, freeWhenDone: false) + } + + func dataNoCopy() -> NSData { + return NSData(bytesNoCopy: self.memory, length: self.length, freeWhenDone: false) } func read(data: UnsafeMutablePointer, offset: Int, length: Int) { @@ -118,6 +162,10 @@ public final class Encoder { public init() { } + public func memoryBuffer() -> MemoryBuffer { + return self.buffer + } + public func makeReadBufferAndReset() -> ReadBuffer { return self.buffer.makeReadBufferAndReset() } @@ -130,20 +178,13 @@ public final class Encoder { self.buffer.reset() } - public func encodeKey(key: UnsafePointer) { - var length: Int8 = Int8(strlen(key)) + public func encodeKey(key: StaticString) { + var length: Int8 = Int8(key.byteSize) self.buffer.write(&length, offset: 0, length: 1) - self.buffer.write(UnsafePointer(key), offset: 0, length: Int(length)) + self.buffer.write(key.utf8Start, offset: 0, length: Int(length)) } - public func encodeKey(key: Int8) { - var length: Int8 = 1 - self.buffer.write(&length, offset: 0, length: 1) - var keyValue = key - self.buffer.write(&keyValue, offset: 0, length: Int(length)) - } - - public func encodeInt32(value: Int32, forKey key: UnsafePointer) { + public func encodeInt32(value: Int32, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int32.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -151,15 +192,7 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 4) } - public func encodeInt32(value: Int32, forKey key: Int8) { - self.encodeKey(key) - var type: Int8 = ValueType.Int32.rawValue - self.buffer.write(&type, offset: 0, length: 1) - var v = value - self.buffer.write(&v, offset: 0, length: 4) - } - - public func encodeInt64(value: Int64, forKey key: UnsafePointer) { + public func encodeInt64(value: Int64, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int64.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -167,15 +200,7 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 8) } - public func encodeInt64(value: Int64, forKey key: Int8) { - self.encodeKey(key) - var type: Int8 = ValueType.Int64.rawValue - self.buffer.write(&type, offset: 0, length: 1) - var v = value - self.buffer.write(&v, offset: 0, length: 8) - } - - public func encodeBool(value: Bool, forKey key: UnsafePointer) { + public func encodeBool(value: Bool, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Bool.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -183,15 +208,7 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 1) } - public func encodeBool(value: Bool, forKey key: Int8) { - self.encodeKey(key) - var type: Int8 = ValueType.Bool.rawValue - self.buffer.write(&type, offset: 0, length: 1) - var v: Int8 = value ? 1 : 0 - self.buffer.write(&v, offset: 0, length: 1) - } - - public func encodeDouble(value: Double, forKey key: UnsafePointer) { + public func encodeDouble(value: Double, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Double.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -199,15 +216,7 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 8) } - public func encodeDouble(value: Double, forKey key: Int8) { - self.encodeKey(key) - var type: Int8 = ValueType.Double.rawValue - self.buffer.write(&type, offset: 0, length: 1) - var v = value - self.buffer.write(&v, offset: 0, length: 8) - } - - public func encodeString(value: String, forKey key: UnsafePointer) { + public func encodeString(value: String, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.String.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -217,7 +226,7 @@ public final class Encoder { self.buffer.write(data.bytes, offset: 0, length: Int(length)) } - public func encodeString(value: DeferredString, forKey key: UnsafePointer) { + public func encodeString(value: DeferredString, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.String.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -227,21 +236,11 @@ public final class Encoder { self.buffer.write(data.bytes, offset: 0, length: Int(length)) } - public func encodeString(value: String, forKey key: Int8) { - self.encodeKey(key) - var type: Int8 = ValueType.String.rawValue - self.buffer.write(&type, offset: 0, length: 1) - let data = value.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! - var length: Int32 = Int32(data.length) - self.buffer.write(&length, offset: 0, length: 4) - self.buffer.write(data.bytes, offset: 0, length: Int(length)) - } - public func encodeRootObject(value: Coding) { self.encodeObject(value, forKey: "_") } - public func encodeObject(value: Coding, forKey key: UnsafePointer) { + public func encodeObject(value: Coding, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Object.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -258,7 +257,7 @@ public final class Encoder { self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(length)) } - public func encodeInt32Array(value: [Int32], forKey key: UnsafePointer) { + public func encodeInt32Array(value: [Int32], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int32Array.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -270,7 +269,7 @@ public final class Encoder { } } - public func encodeInt64Array(value: [Int64], forKey key: UnsafePointer) { + public func encodeInt64Array(value: [Int64], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int64Array.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -282,7 +281,7 @@ public final class Encoder { } } - public func encodeObjectArray(value: [T], forKey key: UnsafePointer) { + public func encodeObjectArray(value: [T], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.ObjectArray.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -302,7 +301,27 @@ public final class Encoder { } } - public func encodeObjectDictionary(value: [K : V], forKey key: UnsafePointer) { + public func encodeObjectArray(value: [Coding], forKey key: StaticString) { + self.encodeKey(key) + var type: Int8 = ValueType.ObjectArray.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var length: Int32 = Int32(value.count) + self.buffer.write(&length, offset: 0, length: 4) + let innerEncoder = Encoder() + for object in value { + var typeHash: Int32 = murMurHashString32("\(object.dynamicType)") + self.buffer.write(&typeHash, offset: 0, length: 4) + + innerEncoder.reset() + object.encode(innerEncoder) + + var length: Int32 = Int32(innerEncoder.buffer.offset) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(length)) + } + } + + public func encodeObjectDictionary(value: [K : V], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.ObjectDictionary.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -329,7 +348,16 @@ public final class Encoder { } } - public func encodeBytes(bytes: WriteBuffer, forKey key: UnsafePointer) { + public func encodeBytes(bytes: WriteBuffer, forKey key: StaticString) { + self.encodeKey(key) + var type: Int8 = ValueType.Bytes.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var bytesLength: Int32 = Int32(bytes.offset) + self.buffer.write(&bytesLength, offset: 0, length: 4) + self.buffer.write(bytes.memory, offset: 0, length: bytes.offset) + } + + public func encodeBytes(bytes: ReadBuffer, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Bytes.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -342,9 +370,10 @@ public final class Encoder { } public final class Decoder { - private let buffer: ReadBuffer + private let buffer: MemoryBuffer + private var offset: Int = 0 - public init(buffer: ReadBuffer) { + public init(buffer: MemoryBuffer) { self.buffer = buffer } @@ -407,11 +436,11 @@ public final class Decoder { } } - private class func positionOnKey(bytes: UnsafePointer, inout offset: Int, maxOffset: Int, length: Int, key: UnsafePointer, valueType: ValueType) -> Bool + private class func positionOnKey(bytes: UnsafePointer, inout offset: Int, maxOffset: Int, length: Int, key: StaticString, valueType: ValueType) -> Bool { let startOffset = offset - let keyLength: Int = Int(strlen(key)) + let keyLength: Int = key.byteSize while (offset < maxOffset) { let readKeyLength = bytes[offset] @@ -421,7 +450,7 @@ public final class Decoder { let readValueType = bytes[offset] offset += 1 - if readValueType != valueType.rawValue || keyLength != Int(readKeyLength) || memcmp(bytes + (offset - Int(readKeyLength) - 1), key, keyLength) != 0 { + if readValueType != valueType.rawValue || keyLength != Int(readKeyLength) || memcmp(bytes + (offset - Int(readKeyLength) - 1), key.utf8Start, keyLength) != 0 { skipValue(bytes, offset: &offset, length: length, valueType: ValueType(rawValue: readValueType)!) } else { return true @@ -468,67 +497,78 @@ public final class Decoder { return false } - public func decodeInt32ForKey(key: UnsafePointer) -> Int32 { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { + public func decodeInt32ForKey(key: StaticString) -> Int32 { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 - memcpy(&value, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + memcpy(&value, self.buffer.memory + self.offset, 4) + self.offset += 4 return value } else { return 0 } } - public func decodeInt32ForKey(key: UnsafePointer) -> Int32? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { + public func decodeInt32ForKey(key: StaticString) -> Int32? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 - memcpy(&value, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + memcpy(&value, self.buffer.memory + self.offset, 4) + self.offset += 4 return value } else { return nil } } - public func decodeInt64ForKey(key: UnsafePointer) -> Int64 { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { + public func decodeInt64ForKey(key: StaticString) -> Int64 { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { var value: Int64 = 0 - memcpy(&value, self.buffer.memory + self.buffer.offset, 8) - self.buffer.offset += 8 + memcpy(&value, self.buffer.memory + self.offset, 8) + self.offset += 8 return value } else { return 0 } } - public func decodeBoolForKey(key: UnsafePointer) -> Bool { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bool) { + public func decodeInt64ForKey(key: StaticString) -> Int64? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { + var value: Int64 = 0 + memcpy(&value, self.buffer.memory + self.offset, 8) + self.offset += 8 + return value + } else { + return nil + } + } + + public func decodeBoolForKey(key: StaticString) -> Bool { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bool) { var value: Int8 = 0 - memcpy(&value, self.buffer.memory + self.buffer.offset, 1) - self.buffer.offset += 1 + memcpy(&value, self.buffer.memory + self.offset, 1) + self.offset += 1 return value != 0 } else { return false } } - public func decodeDoubleForKey(key: UnsafePointer) -> Double { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Double) { + public func decodeDoubleForKey(key: StaticString) -> Double { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Double) { var value: Double = 0 - memcpy(&value, self.buffer.memory + self.buffer.offset, 8) - self.buffer.offset += 8 + memcpy(&value, self.buffer.memory + self.offset, 8) + self.offset += 8 return value } else { return 0 } } - public func decodeStringForKey(key: UnsafePointer) -> String { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + public func decodeStringForKey(key: StaticString) -> String { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) - let data = NSData(bytes: self.buffer.memory + (self.buffer.offset + 4), length: Int(length)) - self.buffer.offset += 4 + Int(length) + memcpy(&length, self.buffer.memory + self.offset, 4) + let data = NSData(bytes: self.buffer.memory + (self.offset + 4), length: Int(length)) + self.offset += 4 + Int(length) let value = NSString(data: data, encoding: NSUTF8StringEncoding) return (value as? String) ?? "" } else { @@ -536,12 +576,12 @@ public final class Decoder { } } - public func decodeStringForKey(key: UnsafePointer) -> String? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + public func decodeStringForKey(key: StaticString) -> String? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) - let data = NSData(bytes: self.buffer.memory + (self.buffer.offset + 4), length: Int(length)) - self.buffer.offset += 4 + Int(length) + memcpy(&length, self.buffer.memory + self.offset, 4) + let data = NSData(bytes: self.buffer.memory + (self.offset + 4), length: Int(length)) + self.offset += 4 + Int(length) let value = NSString(data: data, encoding: NSUTF8StringEncoding) return value as? String } else { @@ -549,24 +589,24 @@ public final class Decoder { } } - public func decodeStringForKey(key: UnsafePointer) -> DeferredString { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + public func decodeStringForKey(key: StaticString) -> DeferredString { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) - let data = NSData(bytes: self.buffer.memory + (self.buffer.offset + 4), length: Int(length)) - self.buffer.offset += 4 + Int(length) + memcpy(&length, self.buffer.memory + self.offset, 4) + let data = NSData(bytes: self.buffer.memory + (self.offset + 4), length: Int(length)) + self.offset += 4 + Int(length) return DeferredStringValue(data) } else { return DeferredStringValue("") } } - public func decodeStringForKey(key: UnsafePointer) -> DeferredString? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + public func decodeStringForKey(key: StaticString) -> DeferredString? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) - let data = NSData(bytes: self.buffer.memory + (self.buffer.offset + 4), length: Int(length)) - self.buffer.offset += 4 + Int(length) + memcpy(&length, self.buffer.memory + self.offset, 4) + let data = NSData(bytes: self.buffer.memory + (self.offset + 4), length: Int(length)) + self.offset += 4 + Int(length) return DeferredStringValue(data) } else { return nil @@ -577,17 +617,17 @@ public final class Decoder { return self.decodeObjectForKey("_") } - public func decodeObjectForKey(key: UnsafePointer) -> Coding? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { + public func decodeObjectForKey(key: StaticString) -> Coding? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { var typeHash: Int32 = 0 - memcpy(&typeHash, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + memcpy(&typeHash, self.buffer.memory + self.offset, 4) + self.offset += 4 var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + memcpy(&length, self.buffer.memory + self.offset, 4) - let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(length), freeWhenDone: false)) - self.buffer.offset += 4 + Int(length) + let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(length), freeWhenDone: false)) + self.offset += 4 + Int(length) return typeStore.decode(typeHash, decoder: innerDecoder) } else { @@ -595,17 +635,17 @@ public final class Decoder { } } - public func decodeObjectForKey(key: UnsafePointer, decoder: Decoder -> Coding) -> Coding? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { + public func decodeObjectForKey(key: StaticString, decoder: Decoder -> Coding) -> Coding? { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { var typeHash: Int32 = 0 - memcpy(&typeHash, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + memcpy(&typeHash, self.buffer.memory + self.offset, 4) + self.offset += 4 var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + memcpy(&length, self.buffer.memory + self.offset, 4) - let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(length), freeWhenDone: false)) - self.buffer.offset += 4 + Int(length) + let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(length), freeWhenDone: false)) + self.offset += 4 + Int(length) return decoder(innerDecoder) } else { @@ -613,51 +653,51 @@ public final class Decoder { } } - public func decodeInt32ArrayForKey(key: UnsafePointer) -> [Int32] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32Array) { + public func decodeInt32ArrayForKey(key: StaticString) -> [Int32] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32Array) { var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + memcpy(&length, self.buffer.memory + self.offset, 4) var array: [Int32] = [] array.reserveCapacity(Int(length)) var i: Int32 = 0 while i < length { var element: Int32 = 0 - memcpy(&element, self.buffer.memory + (self.buffer.offset + 4 + 4 * Int(i)), 4) + memcpy(&element, self.buffer.memory + (self.offset + 4 + 4 * Int(i)), 4) array.append(element) i++ } - self.buffer.offset += 4 + Int(length) * 4 + self.offset += 4 + Int(length) * 4 return array } else { return [] } } - public func decodeInt64ArrayForKey(key: UnsafePointer) -> [Int64] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64Array) { + public func decodeInt64ArrayForKey(key: StaticString) -> [Int64] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64Array) { var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) + memcpy(&length, self.buffer.memory + self.offset, 4) var array: [Int64] = [] array.reserveCapacity(Int(length)) var i: Int32 = 0 while i < length { var element: Int64 = 0 - memcpy(&element, self.buffer.memory + (self.buffer.offset + 4 + 8 * Int(i)), 8) + memcpy(&element, self.buffer.memory + (self.offset + 4 + 8 * Int(i)), 8) array.append(element) i++ } - self.buffer.offset += 4 + Int(length) * 8 + self.offset += 4 + Int(length) * 8 return array } else { return [] } } - public func decodeObjectArrayForKey(key: UnsafePointer) -> [T] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { + public func decodeObjectArrayForKey(key: StaticString) -> [T] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + memcpy(&length, self.buffer.memory + self.offset, 4) + self.offset += 4 var array: [T] = [] array.reserveCapacity(Int(length)) @@ -666,14 +706,14 @@ public final class Decoder { var i: Int32 = 0 while i < length { var typeHash: Int32 = 0 - memcpy(&typeHash, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + memcpy(&typeHash, self.buffer.memory + self.offset, 4) + self.offset += 4 var objectLength: Int32 = 0 - memcpy(&objectLength, self.buffer.memory + self.buffer.offset, 4) + memcpy(&objectLength, self.buffer.memory + self.offset, 4) - let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(objectLength), freeWhenDone: false)) - self.buffer.offset += 4 + Int(objectLength) + let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(objectLength), freeWhenDone: false)) + self.offset += 4 + Int(objectLength) if !failed { if let object = typeStore.decode(typeHash, decoder: innerDecoder) as? T { @@ -695,12 +735,55 @@ public final class Decoder { return [] } } - - public func decodeObjectDictionaryForKey(key: UnsafePointer) -> [K : V] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { + + public func decodeObjectArrayForKey(key: StaticString) -> [Coding] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + memcpy(&length, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var array: [Coding] = [] + array.reserveCapacity(Int(length)) + + var failed = false + var i: Int32 = 0 + while i < length { + var typeHash: Int32 = 0 + memcpy(&typeHash, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var objectLength: Int32 = 0 + memcpy(&objectLength, self.buffer.memory + self.offset, 4) + + let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(objectLength), freeWhenDone: false)) + self.offset += 4 + Int(objectLength) + + if !failed { + if let object = typeStore.decode(typeHash, decoder: innerDecoder) { + array.append(object) + } else { + failed = true + } + } + + i++ + } + + if failed { + return [] + } else { + return array + } + } else { + return [] + } + } + + public func decodeObjectDictionaryForKey(key: StaticString) -> [K : V] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.offset, 4) + self.offset += 4 var dictionary: [K : V] = [:] @@ -708,26 +791,26 @@ public final class Decoder { var i: Int32 = 0 while i < length { var keyHash: Int32 = 0 - memcpy(&keyHash, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + memcpy(&keyHash, self.buffer.memory + self.offset, 4) + self.offset += 4 var keyLength: Int32 = 0 - memcpy(&keyLength, self.buffer.memory + self.buffer.offset, 4) + memcpy(&keyLength, self.buffer.memory + self.offset, 4) - var innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(keyLength), freeWhenDone: false)) - self.buffer.offset += 4 + Int(keyLength) + var innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(keyLength), freeWhenDone: false)) + self.offset += 4 + Int(keyLength) let key = failed ? nil : (typeStore.decode(keyHash, decoder: innerDecoder) as? K) var valueHash: Int32 = 0 - memcpy(&valueHash, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + memcpy(&valueHash, self.buffer.memory + self.offset, 4) + self.offset += 4 var valueLength: Int32 = 0 - memcpy(&valueLength, self.buffer.memory + self.buffer.offset, 4) + memcpy(&valueLength, self.buffer.memory + self.offset, 4) - innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.buffer.offset + 4), length: Int(valueLength), freeWhenDone: false)) - self.buffer.offset += 4 + Int(valueLength) + innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(valueLength), freeWhenDone: false)) + self.offset += 4 + Int(valueLength) let value = failed ? nil : (typeStore.decode(valueHash, decoder: innerDecoder) as? V) @@ -750,12 +833,12 @@ public final class Decoder { } } - public func decodeBytesForKeyNoCopy(key: UnsafePointer) -> ReadBuffer! { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.buffer.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bytes) { + public func decodeBytesForKeyNoCopy(key: StaticString) -> ReadBuffer! { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bytes) { var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.buffer.offset, 4) - self.buffer.offset += 4 + Int(length) - return ReadBuffer(memory: UnsafeMutablePointer(self.buffer.memory + (self.buffer.offset - Int(length))), length: Int(length), freeWhenDone: false) + memcpy(&length, self.buffer.memory + self.offset, 4) + self.offset += 4 + Int(length) + return ReadBuffer(memory: UnsafeMutablePointer(self.buffer.memory + (self.offset - Int(length))), length: Int(length), freeWhenDone: false) } else { return nil } diff --git a/Postbox/LmdbValueBox.swift b/Postbox/LmdbValueBox.swift new file mode 100644 index 0000000000..f433362f47 --- /dev/null +++ b/Postbox/LmdbValueBox.swift @@ -0,0 +1,395 @@ +import Foundation +import sqlcipher + +private struct LmdbTable { + var dbi: MDB_dbi +} + +private struct LmdbCursor { + var cursor: COpaquePointer + + func seekTo(key: ValueBoxKey, forward: Bool) -> (ValueBoxKey, ReadBuffer)? { + var mdbKey = MDB_val() + var mdbData = MDB_val() + + mdbKey.mv_data = key.memory + mdbKey.mv_size = key.length + mdbData.mv_data = nil + mdbData.mv_size = 0 + + let result = mdb_cursor_get(self.cursor, &mdbKey, &mdbData, forward ? MDB_SET_RANGE : MDB_SET_KEY) + + if result == MDB_SUCCESS { + let actualKey = ValueBoxKey(length: mdbKey.mv_size) + memcpy(actualKey.memory, mdbKey.mv_data, mdbKey.mv_size) + + let value = malloc(mdbData.mv_size) + memcpy(value, mdbData.mv_data, mdbData.mv_size) + + return (actualKey, ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true)) + } else if result == MDB_NOTFOUND { + if !forward { + return self.previous() + } else { + return nil + } + } else { + print("(LmdbValueBox mdb_cursor_get failed with \(result))") + return nil + } + } + + func previous() -> (ValueBoxKey, ReadBuffer)? { + var mdbKey = MDB_val() + var mdbData = MDB_val() + + mdbKey.mv_data = nil + mdbKey.mv_size = 0 + mdbData.mv_data = nil + mdbData.mv_size = 0 + + let result = mdb_cursor_get(self.cursor, &mdbKey, &mdbData, MDB_PREV) + if result == MDB_SUCCESS { + let actualKey = ValueBoxKey(length: mdbKey.mv_size) + memcpy(actualKey.memory, mdbKey.mv_data, mdbKey.mv_size) + + let value = malloc(mdbData.mv_size) + memcpy(value, mdbData.mv_data, mdbData.mv_size) + + return (actualKey, ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true)) + } else if result == MDB_NOTFOUND { + return nil + } else { + print("(LmdbValueBox mdb_cursor_get failed with \(result))") + return nil + } + } + + func next() -> (ValueBoxKey, ReadBuffer)? { + var mdbKey = MDB_val() + var mdbData = MDB_val() + + mdbKey.mv_data = nil + mdbKey.mv_size = 0 + mdbData.mv_data = nil + mdbData.mv_size = 0 + + let result = mdb_cursor_get(self.cursor, &mdbKey, &mdbData, MDB_NEXT) + if result == MDB_SUCCESS { + let actualKey = ValueBoxKey(length: mdbKey.mv_size) + memcpy(actualKey.memory, mdbKey.mv_data, mdbKey.mv_size) + + let value = malloc(mdbData.mv_size) + memcpy(value, mdbData.mv_data, mdbData.mv_size) + + return (actualKey, ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true)) + } else if result == MDB_NOTFOUND { + return nil + } else { + print("(LmdbValueBox mdb_cursor_get failed with \(result))") + return nil + } + } +} + +public final class LmdbValueBox: ValueBox { + private var env: COpaquePointer = nil + private var tables: [Int32 : LmdbTable] = [:] + + private var sharedTxn: COpaquePointer = nil + + public init?(basePath: String) { + var result = mdb_env_create(&self.env) + if result != MDB_SUCCESS { + print("(LmdbValueBox mdb_env_create failed with \(result))") + return nil + } + + let path = basePath + "/lmdb" + + var createDirectory = false + var isDirectory: ObjCBool = false as ObjCBool + if NSFileManager.defaultManager().fileExistsAtPath(path, isDirectory: &isDirectory) { + if !isDirectory { + do { + try NSFileManager.defaultManager().removeItemAtPath(path) + } catch _ { } + createDirectory = true + } + } + else { + createDirectory = true + } + + if createDirectory { + do { + try NSFileManager.defaultManager().createDirectoryAtPath(path, withIntermediateDirectories: true, attributes: nil) + } catch _ { } + } + + mdb_env_set_mapsize(self.env, 500 * 1024 * 1024); + mdb_env_set_maxdbs(self.env, 64) + + path.withCString { string in + result = mdb_env_open(self.env, string, UInt32(MDB_NOSYNC), 0o664) + } + if result != MDB_SUCCESS { + print("(LmdbValueBox mdb_env_open failed with \(result))") + return nil + } + + var removedReaders: Int32 = 0 + result = mdb_reader_check(self.env, &removedReaders) + + if removedReaders != 0 { + print("(LmdbValueBox removed \(removedReaders) stale readers)") + } + } + + deinit { + mdb_env_close(self.env) + } + + private func createTableWithName(name: Int32) -> LmdbTable? { + var dbi = MDB_dbi() + let result = mdb_dbi_open(self.sharedTxn, "\(name)", UInt32(MDB_CREATE), &dbi) + + if result != MDB_SUCCESS { + print("(LmdbValueBox mdb_dbi_open failed with \(result))") + + return nil + } + + return LmdbTable(dbi: dbi) + } + + public func begin() { + if self.sharedTxn != nil { + print("(LmdbValueBox already in transaction)") + } else { + let result = mdb_txn_begin(self.env, nil, 0, &sharedTxn) + if result != MDB_SUCCESS { + print("(LmdbValueBox txn_begin failed with \(result))") + return + } + } + } + + public func commit() { + if self.sharedTxn == nil { + print("(LmdbValueBox already no current transaction)") + } else { + let result = mdb_txn_commit(self.sharedTxn) + self.sharedTxn = nil + if result != MDB_SUCCESS { + print("(LmdbValueBox txn_commit failed with \(result))") + return + } + } + } + + public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { + if start == end || limit == 0 { + return + } + + var commit = false + if self.sharedTxn == nil { + self.begin() + commit = true + } + + var nativeTable: LmdbTable? + if let existingTable = self.tables[table] { + nativeTable = existingTable + } else if let createdTable = self.createTableWithName(table) { + nativeTable = createdTable + self.tables[table] = createdTable + } + + if let nativeTable = nativeTable { + var cursorPtr: COpaquePointer = nil + let result = mdb_cursor_open(self.sharedTxn, nativeTable.dbi, &cursorPtr) + if result != MDB_SUCCESS { + print("(LmdbValueBox mdb_cursor_open failed with \(result))") + } else { + let cursor = LmdbCursor(cursor: cursorPtr) + + if start < end { + var value = cursor.seekTo(start, forward: true) + if value != nil { + if value!.0 == start { + value = cursor.next() + } + } + + var count = 0 + if value != nil && value!.0 < end { + count++ + values(value!.0, value!.1) + } + + while value != nil && value!.0 < end && count < limit { + value = cursor.next() + if value != nil && value!.0 < end { + count++ + values(value!.0, value!.1) + } + } + } else { + var value = cursor.seekTo(start, forward: false) + if value != nil { + if value!.0 == start { + value = cursor.previous() + } + } + + var count = 0 + if value != nil && value!.0 > end { + count++ + values(value!.0, value!.1) + } + + while value != nil && value!.0 > end && count < limit { + value = cursor.previous() + if value != nil && value!.0 > end { + count++ + values(value!.0, value!.1) + } + } + } + + mdb_cursor_close(cursorPtr) + } + } + + if commit { + self.commit() + } + } + + public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: ValueBoxKey -> Bool, limit: Int) { + self.range(table, start: start, end: end, values: { key, _ in + return keys(key) + }, limit: limit) + } + + public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? { + var commit = false + if self.sharedTxn == nil { + self.begin() + commit = true + } + + var nativeTable: LmdbTable? + if let existingTable = self.tables[table] { + nativeTable = existingTable + } else if let createdTable = self.createTableWithName(table) { + nativeTable = createdTable + self.tables[table] = createdTable + } + + var resultValue: ReadBuffer? + + if let nativeTable = nativeTable { + var mdbKey = MDB_val() + var mdbData = MDB_val() + + mdbKey.mv_data = key.memory + mdbKey.mv_size = key.length + + let result = mdb_get(self.sharedTxn, nativeTable.dbi, &mdbKey, &mdbData) + + if result == MDB_SUCCESS { + let value = malloc(mdbData.mv_size) + memcpy(value, mdbData.mv_data, mdbData.mv_size) + resultValue = ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true) + } else { + if result != MDB_NOTFOUND { + print("(LmdbValueBox mdb_get failed with \(result))") + } + } + } + + if commit { + self.commit() + } + + return resultValue + } + + public func exists(table: Int32, key: ValueBoxKey) -> Bool { + return self.get(table, key: key) != nil + } + + public func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) { + var commit = false + if self.sharedTxn == nil { + self.begin() + commit = true + } + + var nativeTable: LmdbTable? + if let existingTable = self.tables[table] { + nativeTable = existingTable + } else if let createdTable = self.createTableWithName(table) { + nativeTable = createdTable + self.tables[table] = createdTable + } + + if let nativeTable = nativeTable { + var mdbKey = MDB_val() + var mdbData = MDB_val() + + mdbKey.mv_data = key.memory + mdbKey.mv_size = key.length + mdbData.mv_data = value.memory + mdbData.mv_size = value.length + + let result = mdb_put(self.sharedTxn, nativeTable.dbi, &mdbKey, &mdbData, 0) + if result != MDB_SUCCESS { + print("(LmdbValueBox mdb_set failed with \(result))") + } + } + + if commit { + self.commit() + } + } + + public func remove(table: Int32, key: ValueBoxKey) { + var commit = false + if self.sharedTxn == nil { + self.begin() + commit = true + } + + var nativeTable: LmdbTable? + if let existingTable = self.tables[table] { + nativeTable = existingTable + } else if let createdTable = self.createTableWithName(table) { + nativeTable = createdTable + self.tables[table] = createdTable + } + + if let nativeTable = nativeTable { + var mdbKey = MDB_val() + + mdbKey.mv_data = key.memory + mdbKey.mv_size = key.length + + let result = mdb_del(self.sharedTxn, nativeTable.dbi, &mdbKey, nil) + if result != MDB_SUCCESS { + print("(LmdbValueBox mdb_set failed with \(result))") + } + } + + if commit { + self.commit() + } + } + + public func drop() { + + } +} diff --git a/Postbox/Media.swift b/Postbox/Media.swift index 5b53c468b2..a62a74759d 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -9,7 +9,7 @@ public struct MediaId: Hashable, CustomStringConvertible { public var hashValue: Int { get { - return Int((self.id & 0xffffffff) ^ ((self.id >> 32) & 0xffffffff)) + return Int((self.id & 0x7fffffff) ^ ((self.id >> 32) & 0x7fffffff)) } } @@ -67,5 +67,5 @@ public func ==(lhs: MediaId, rhs: MediaId) -> Bool { } public protocol Media: Coding { - var id: MediaId { get } + var id: MediaId? { get } } diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift new file mode 100644 index 0000000000..6d3ecce762 --- /dev/null +++ b/Postbox/MediaBox.swift @@ -0,0 +1,145 @@ +import Foundation +import SwiftSignalKit +import Display + +public struct MediaKeyStatus { + let size: Int +} + +private final class MediaBoxMutexEntry { + var mutex = pthread_mutex_t() + + init() { + var attr = pthread_mutexattr_t() + pthread_mutexattr_init(&attr) + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) + pthread_mutex_init(&mutex, &attr) + } + + deinit { + pthread_mutex_destroy(&mutex) + } + + func with(@noescape f: Void -> Void) { + pthread_mutex_lock(&self.mutex) + f() + pthread_mutex_unlock(&self.mutex) + } +} + +public final class MediaBox { + let basePath: String + let buffer = WriteBuffer() + + private var mutexEntriesLock = OSSpinLock() + private var mutexEntries: [String : MediaBoxMutexEntry] = [:] + + lazy var ensureDirectoryCreated: Void = { + try! NSFileManager.defaultManager().createDirectoryAtPath(self.basePath, withIntermediateDirectories: true, attributes: nil) + }() + + public init(basePath: String) { + self.basePath = basePath + } + + private func keyForId(id: MediaId, key: MemoryBuffer) -> String { + let string = NSMutableString() + string.appendFormat("%d", Int(id.namespace)) + string.appendFormat("_%lld", Int64(id.id)) + string.appendString("_\(key)") + return string as String + } + + private func pathForId(id: MediaId, key: MemoryBuffer) -> String { + return "\(self.basePath)/\(self.keyForId(id, key: key))" + } + + private func mutexForKey(key: String) -> MediaBoxMutexEntry { + let entry: MediaBoxMutexEntry + + OSSpinLockLock(&mutexEntriesLock) + if let existingEntry = self.mutexEntries[key] { + entry = existingEntry + } else { + entry = MediaBoxMutexEntry() + self.mutexEntries[key] = entry + } + OSSpinLockUnlock(&mutexEntriesLock) + + return entry + } + + func writeId(id: MediaId, key: ReadBuffer, value: NSData) { + assertNotOnMainThread() + self.mutexForKey(self.keyForId(id, key: key)).with { + let _ = self.ensureDirectoryCreated + value.writeToFile(self.pathForId(id, key: key), atomically: false) + } + } + + private func status(id: MediaId, key: ReadBuffer) -> MediaKeyStatus? { + assertNotOnMainThread() + var value = stat() + stat(self.pathForId(id, key: key), &value) + return MediaKeyStatus(size: Int(value.st_size)) + } + + private func read(id: MediaId, key: ReadBuffer) -> NSData? { + assertNotOnMainThread() + var data: NSData? + do { + data = try NSData(contentsOfFile: self.pathForId(id, key: key), options: NSDataReadingOptions.DataReadingMappedIfSafe) + } catch _ { + } + return data + } + + public func data(id: MediaId, key: ReadBuffer, size: Int, fetch: ((NSData, Int) -> Signal)?) -> Signal { + return Signal { subscriber in + var cancelled = false + let disposable = MetaDisposable() + disposable.set(ActionDisposable { + cancelled = true + }) + + self.mutexForKey(self.keyForId(id, key: key)).with { + if cancelled { + return + } + + let status = self.status(id, key: key) ?? MediaKeyStatus(size: 0) + + var currentData: NSData? + if status.size > 0 { + if let data = self.read(id, key: key) { + currentData = data + subscriber.putNext(data) + } + } + + if status.size < size { + if let fetch = fetch { + disposable.set(fetch(currentData ?? NSData(), status.size).start(next: { [weak self] next in + if let strongSelf = self { + strongSelf.mutexForKey(strongSelf.keyForId(id, key: key)).with { + strongSelf.writeId(id, key: key, value: next) + } + } + subscriber.putNext(next) + }, error: { error in + subscriber.putError(error) + }, completed: { + subscriber.putCompletion() + })) + } else { + subscriber.putError(NoError()) + } + } else { + subscriber.putCompletion() + } + } + + return disposable + } + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 3e4aea8711..e5a8ed5528 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -50,20 +50,24 @@ public final class Postbox { private let messageNamespaces: [MessageId.Namespace] private let absoluteIndexedMessageNamespaces: [MessageId.Namespace] - private let queue = SwiftSignalKit.Queue() - private var database: Database! + private let queue = Queue(name: "org.telegram.postbox.Postbox") + private var valueBox: ValueBox! private var peerMessageViews: [PeerId : Bag<(MutableMessageView, Pipe)>] = [:] private var deferredMessageViewsToUpdate: [(MutableMessageView, Pipe)] = [] private var peerViews: Bag<(MutablePeerView, Pipe)> = Bag() private var deferredPeerViewsToUpdate: [(MutablePeerView, Pipe)] = [] + private var peerPipes: [PeerId : Pipe] = [:] private var statePipe: Pipe = Pipe() + public let mediaBox: MediaBox + public init(basePath: String, messageNamespaces: [MessageId.Namespace], absoluteIndexedMessageNamespaces: [MessageId.Namespace]) { self.basePath = basePath self.messageNamespaces = messageNamespaces self.absoluteIndexedMessageNamespaces = absoluteIndexedMessageNamespaces + self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.openDatabase() } @@ -75,310 +79,28 @@ public final class Postbox { try NSFileManager.defaultManager().createDirectoryAtPath(self.basePath, withIntermediateDirectories: true, attributes: nil) } catch _ { } - self.database = Database(self.basePath + "/db") - let result = self.database.scalar("PRAGMA user_version") as! Int64 - let version: Int64 = 11 - if result == version { - print("(Postbox schema version \(result))") - } else { - if result != 0 { - print("(Postbox migrating to version \(version))") - do { - try NSFileManager.defaultManager().removeItemAtPath(self.basePath) - try NSFileManager.defaultManager().createDirectoryAtPath(self.basePath, withIntermediateDirectories: true, attributes: nil) - } catch (_) { - } - - self.database = Database(self.basePath + "/db") - } - print("(Postbox creating schema)") - self.createSchema() - self.database.execute("PRAGMA user_version = \(version)") + self.valueBox = SqliteValueBox(basePath: self.basePath + "/db") + //self.valueBox = LmdbValueBox(basePath: self.basePath + "/db") + var userVersion: Int32 = 0 + let currentUserVersion: Int32 = 3 + + if let value = self.valueBox.get(Table_Meta.id, key: Table_Meta.key()) { + value.read(&userVersion, offset: 0, length: 4) } - self.database.adjustChunkSize() - self.database.execute("PRAGMA page_size=1024") - self.database.execute("PRAGMA cache_size=-2097152") - self.database.execute("PRAGMA synchronous=NORMAL") - self.database.execute("PRAGMA journal_mode=truncate") - self.database.execute("PRAGMA temp_store=MEMORY") - //self.database.execute("PRAGMA wal_autocheckpoint=32") - //self.database.execute("PRAGMA journal_size_limit=1536") + if userVersion != currentUserVersion { + self.valueBox.drop() + let buffer = WriteBuffer() + var currentVersion: Int32 = currentUserVersion + buffer.write(¤tVersion, offset: 0, length: 4) + self.valueBox.set(Table_Meta.id, key: Table_Meta.key(), value: buffer) + } print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") } } - private func createSchema() { - //state - self.database.execute("CREATE TABLE state (id INTEGER PRIMARY KEY, data BLOB)") - - //keychain - self.database.execute("CREATE TABLE keychain (key BLOB, data BLOB)") - self.database.execute("CREATE INDEX keychain_key ON keychain (key)") - - //peer_messages - self.database.execute("CREATE TABLE peer_messages (peerId INTEGER, namespace INTEGER, id INTEGER, data BLOB, associatedMediaIds BLOB, timestamp INTEGER, PRIMARY KEY(peerId, namespace, id))") - - //peer_absolute_indexed_message_ids - self.database.execute("CREATE TABLE peer_absolute_indexed_message_ids (id INTEGER PRIMARY KEY, completeId BLOB)") - - //peer_media - self.database.execute("CREATE TABLE peer_media (peerId INTEGER, mediaNamespace INTEGER, messageNamespace INTEGER, messageId INTEGER, PRIMARY KEY (peerId, mediaNamespace, messageNamespace, messageId))") - self.database.execute("CREATE INDEX peer_media_peerId_messageNamespace_messageId ON peer_media (peerId, messageNamespace, messageId)") - - //media - self.database.execute("CREATE TABLE media (namespace INTEGER, id INTEGER, data BLOB, associatedMessageIds BLOB, PRIMARY KEY (namespace, id))") - - //media_cleanup - self.database.execute("CREATE TABLE media_cleanup (namespace INTEGER, id INTEGER, data BLOB, PRIMARY KEY(namespace, id))") - - //peer_entries - self.database.execute("CREATE TABLE peer_entries (peerId INTEGER PRIMARY KEY, entry BLOB)") - self.database.execute("CREATE INDEX peer_entries_entry on peer_entries (entry)") - - //peers - self.database.execute("CREATE TABLE peers (id INTEGER PRIMARY KEY, data BLOB)") - } - - private class func peerViewEntryIndexForBuffer(buffer: ReadBuffer) -> PeerViewEntryIndex { - var timestamp: Int32 = 0 - buffer.read(×tamp, offset: 0, length: 4) - timestamp = Int32(bigEndian: timestamp) - - var namespace: Int32 = 0 - buffer.read(&namespace, offset: 0, length: 4) - namespace = Int32(bigEndian: namespace) - - var id: Int32 = 0 - buffer.read(&id, offset: 0, length: 4) - id = Int32(bigEndian: id) - - var peerIdRepresentation: Int64 = 0 - buffer.read(&peerIdRepresentation, offset: 0, length: 8) - peerIdRepresentation = Int64(bigEndian: peerIdRepresentation) - - let peerId = PeerId(peerIdRepresentation) - - return PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(id: MessageId(peerId:peerId, namespace: namespace, id: id), timestamp: timestamp)) - } - - private class func blobForPeerViewEntryIndex(index: PeerViewEntryIndex) -> Blob { - let buffer = WriteBuffer() - - var timestamp = Int32(bigEndian: index.messageIndex.timestamp) - buffer.write(×tamp, offset: 0, length: 4) - - var namespace = Int32(bigEndian: index.messageIndex.id.namespace) - buffer.write(&namespace, offset: 0, length: 4) - - var id = Int32(bigEndian: index.messageIndex.id.id) - buffer.write(&id, offset: 0, length: 4) - - var peerIdRepresentation = Int64(bigEndian: index.peerId.toInt64()) - buffer.write(&peerIdRepresentation, offset: 0, length: 8) - - return Blob(data: buffer.makeData()) - } - - private class func messageIdsGroupedByNamespace(ids: [MessageId]) -> [MessageId.Namespace : [MessageId]] { - var grouped: [MessageId.Namespace : [MessageId]] = [:] - - for id in ids { - if grouped[id.namespace] != nil { - grouped[id.namespace]!.append(id) - } else { - grouped[id.namespace] = [id] - } - } - - return grouped - } - - private class func mediaIdsGroupedByNamespaceFromMediaArray(mediaArray: [Media]) -> [MediaId.Namespace : [MediaId]] { - var grouped: [MediaId.Namespace : [MediaId]] = [:] - var seenMediaIds = Set() - - for media in mediaArray { - if !seenMediaIds.contains(media.id) { - seenMediaIds.insert(media.id) - if grouped[media.id.namespace] != nil { - grouped[media.id.namespace]!.append(media.id) - } else { - grouped[media.id.namespace] = [media.id] - } - } - } - - return grouped - } - - private class func mediaIdsGroupedByNamespaceFromSet(ids: Set) -> [MediaId.Namespace : [MediaId]] { - var grouped: [MediaId.Namespace : [MediaId]] = [:] - - for id in ids { - if let _ = grouped[id.namespace] { - grouped[id.namespace]!.append(id) - } else { - grouped[id.namespace] = [id] - } - } - - return grouped - } - - private class func mediaIdsGroupedByNamespaceFromDictionaryKeys(dict: [MediaId : T]) -> [MediaId.Namespace : [MediaId]] { - var grouped: [MediaId.Namespace : [MediaId]] = [:] - - for (id, _) in dict { - if grouped[id.namespace] != nil { - grouped[id.namespace]!.append(id) - } else { - grouped[id.namespace] = [id] - } - } - - return grouped - } - - private class func messagesGroupedByPeerId(messages: [Message]) -> [(PeerId, [Message])] { - var grouped: [(PeerId, [Message])] = [] - - for message in messages { - var i = 0 - let count = grouped.count - var found = false - while i < count { - if grouped[i].0 == message.id.peerId { - grouped[i].1.append(message) - found = true - break - } - i++ - } - if !found { - grouped.append((message.id.peerId, [message])) - } - } - - return grouped - } - - private class func messageIdsGroupedByPeerId(messageIds: [MessageId]) -> [PeerId : [MessageId]] { - var grouped: [PeerId : [MessageId]] = [:] - - for id in messageIds { - if grouped[id.peerId] != nil { - grouped[id.peerId]!.append(id) - } else { - grouped[id.peerId] = [id] - } - } - - return grouped - } - - private class func blobForMediaIds(ids: [MediaId]) -> Blob { - let data = NSMutableData() - var version: Int8 = 1 - data.appendBytes(&version, length: 1) - - var count = Int32(ids.count) - data.appendBytes(&count, length:4) - - for id in ids { - var mNamespace = id.namespace - var mId = id.id - data.appendBytes(&mNamespace, length: 4) - data.appendBytes(&mId, length: 8) - } - - return Blob(data: data) - } - - private static func mediaIdsForBlob(blob: Blob) -> [MediaId] { - var ids: [MediaId] = [] - - var offset: Int = 0 - var version = 0 - blob.data.getBytes(&version, range: NSMakeRange(offset, 1)) - offset += 1 - - if version == 1 { - var count: Int32 = 0 - blob.data.getBytes(&count, range: NSMakeRange(offset, 4)) - offset += 4 - - var i = 0 - while i < Int(count) { - var mNamespace: Int32 = 0 - var mId: Int64 = 0 - blob.data.getBytes(&mNamespace, range: NSMakeRange(offset, 4)) - blob.data.getBytes(&mId, range: NSMakeRange(offset + 4, 8)) - ids.append(MediaId(namespace: mNamespace, id: mId)) - offset += 12 - i++ - } - } - - return ids - } - - private class func blobForMessageIds(ids: [MessageId]) -> Blob { - let data = NSMutableData() - var version: Int8 = 1 - data.appendBytes(&version, length: 1) - - var count = Int32(ids.count) - data.appendBytes(&count, length:4) - - for id in ids { - var mPeerNamespace = id.peerId.namespace - var mPeerId = id.peerId.id - var mNamespace = id.namespace - var mId = id.id - data.appendBytes(&mPeerNamespace, length: 4) - data.appendBytes(&mPeerId, length: 4) - data.appendBytes(&mNamespace, length: 4) - data.appendBytes(&mId, length: 4) - } - - return Blob(data: data) - } - - private static func messageIdsForBlob(blob: Blob) -> [MessageId] { - var ids: [MessageId] = [] - - var offset: Int = 0 - var version = 0 - blob.data.getBytes(&version, range: NSMakeRange(offset, 1)) - offset += 1 - - if version == 1 { - var count: Int32 = 0 - blob.data.getBytes(&count, range: NSMakeRange(offset, 4)) - offset += 4 - - var i = 0 - while i < Int(count) { - var mPeerNamespace: Int32 = 0 - var mPeerId: Int32 = 0 - var mNamespace: Int32 = 0 - var mId: Int32 = 0 - blob.data.getBytes(&mPeerNamespace, range: NSMakeRange(offset, 4)) - blob.data.getBytes(&mPeerId, range: NSMakeRange(offset + 4, 4)) - blob.data.getBytes(&mNamespace, range: NSMakeRange(offset + 8, 4)) - blob.data.getBytes(&mId, range: NSMakeRange(offset + 12, 4)) - ids.append(MessageId(peerId: PeerId(namespace: mPeerNamespace, id: mPeerId), namespace: mNamespace, id: mId)) - offset += 16 - i++ - } - } - - return ids - } - private var cachedState: State? private func setState(state: State) { @@ -388,10 +110,7 @@ public final class Postbox { let encoder = Encoder() encoder.encodeRootObject(state) - - - let blob = Blob(data: encoder.makeData()) - self.database.prepareCached("INSERT OR REPLACE INTO state (id, data) VALUES (?, ?)").run(Int64(0), blob) + self.valueBox.set(Table_State.id, key: Table_State.key(), value: encoder.memoryBuffer()) self.statePipe.putNext(state) } @@ -401,16 +120,14 @@ public final class Postbox { if let cachedState = self.cachedState { return cachedState } else { - for row in self.database.prepareCached("SELECT data FROM state WHERE id = ?").run(Int64(0)) { - let data = (row[0] as! Blob).data - let buffer = ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false) - let decoder = Decoder(buffer: buffer) + if let value = self.valueBox.get(Table_State.id, key: Table_State.key()) { + let decoder = Decoder(buffer: value) if let state = decoder.decodeRootObject() as? State { self.cachedState = state return state } - break } + return nil } } @@ -424,53 +141,34 @@ public final class Postbox { subscriber.putNext(next) })) } + return disposable } } public func keychainEntryForKey(key: String) -> NSData? { - //TODO: load keychain on first request then sync to disk - - let blob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) - for row in self.database.prepareCached("SELECT data FROM keychain WHERE key = ?").run(blob) { - return (row[0] as! Blob).data + if let value = self.valueBox.get(Table_Keychain.id, key: Table_Keychain.key(key)) { + print("get \(key) -> \(value.length) bytes") + return NSData(bytes: value.memory, length: value.length) } + print("get \(key) -> nil") + return nil } public func setKeychainEntryForKey(key: String, value: NSData) { - let keyBlob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) - var rowId: Int64? - for row in self.database.prepareCached("SELECT rowid FROM keychain WHERE key = ? LIMIT 1").run(keyBlob) { - rowId = row[0] as? Int64 - break - } - if let rowId = rowId { - self.database.prepareCached("UPDATE keychain SET data = ? WHERE rowid = ?").run(Blob(data: value), rowId) - } else { - self.database.prepareCached("INSERT INTO keychain (key, data) VALUES (?, ?)").run(keyBlob, Blob(data: value)) - } + print("set \(key) -> \(value.length) bytes") + self.valueBox.set(Table_Keychain.id, key: Table_Keychain.key(key), value: MemoryBuffer(data: value)) } public func removeKeychainEntryForKey(key: String) { - self.queue.dispatch { - let keyBlob = Blob(data: key.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!) - self.database.prepareCached("DELETE FROM keychain WHERE key = ?").run(keyBlob) - } + self.valueBox.remove(Table_Keychain.id, key: Table_Keychain.key(key)) } private func addMessages(messages: [Message], medias: [Media]) { - let messageInsertStatement = self.database.prepareCached("INSERT INTO peer_messages (peerId, namespace, id, data, associatedMediaIds, timestamp) VALUES (?, ?, ?, ?, ?, ?)") - let absoluteMessageIdsInsertStatement = self.database.prepareCached("INSERT INTO peer_absolute_indexed_message_ids (id, completeId) VALUES (?, ?)") - let peerMediaInsertStatement = self.database.prepareCached("INSERT INTO peer_media (peerId, mediaNamespace, messageNamespace, messageId) VALUES (?, ?, ?, ?)") - let mediaInsertStatement = self.database.prepareCached("INSERT INTO media (namespace, id, data, associatedMessageIds) VALUES (?, ?, ?, ?)") - let referencedMessageIdsStatement = self.database.prepareCached("SELECT associatedMessageIds FROM media WHERE namespace = ? AND id = ?") - let updateReferencedMessageIdsStatement = self.database.prepareCached("UPDATE media SET associatedMessageIds = ? WHERE namespace = ? AND id = ?") - let encoder = Encoder() - var messageIdsByMediaId: [MediaId : [MessageId]] = [:] - for (peerId, peerMessages) in Postbox.messagesGroupedByPeerId(messages) { + for (peerId, peerMessages) in messagesGroupedByPeerId(messages) { var maxMessage: (MessageIndex, Message)? var messageIds: [MessageId] = [] @@ -484,38 +182,23 @@ public final class Postbox { var existingMessageIds = Set() - let messageIdsByNamespace = Postbox.messageIdsGroupedByNamespace(messageIds) - for (namespace, ids) in messageIdsByNamespace { - var queryString = "SELECT id FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (" - var first = true - for id in ids { - if first { - first = false - } else { - queryString += "," - } - queryString += "\(id.id)" - } - queryString += ")" - - let statement = self.database.prepare(queryString) - for row in statement.run(peerId.toInt64(), Int64(namespace)) { - existingMessageIds.insert(MessageId(peerId: peerId, namespace: namespace, id: Int32(row[0] as! Int64))) + let existingMessageKey = Table_Message.emptyKey() + existingMessageKey.setInt64(0, value: peerId.toInt64()) + for id in messageIds { + if self.valueBox.exists(Table_Message.id, key: Table_Message.key(id, key: existingMessageKey)) { + existingMessageIds.insert(id) } } - let completeIdData = NSMutableData(capacity: 8 + 4 + 4)! - var zero: Int64 = 0 - completeIdData.appendBytes(&zero, length: 8) - completeIdData.appendBytes(&zero, length: 8) - let completeIdDataMutableBytes = completeIdData.mutableBytes - let completeIdBlob = Blob(data: completeIdData) + var addedMessages: [Message] = [] + let mediaMessageIdKey = Table_Media_MessageIds.emptyKey() for message in peerMessages { if existingMessageIds.contains(message.id) { continue } existingMessageIds.insert(message.id) + addedMessages.append(message) let index = MessageIndex(message) if maxMessage == nil || index > maxMessage!.0 { @@ -524,38 +207,23 @@ public final class Postbox { encoder.reset() encoder.encodeRootObject(message) - let messageBlob = Blob(data: encoder.makeData()) - let referencedMediaIdsMediaIdsBlob = Postbox.blobForMediaIds(message.mediaIds) for id in message.mediaIds { - if messageIdsByMediaId[id] != nil { - messageIdsByMediaId[id]!.append(message.id) - } else { - messageIdsByMediaId[id] = [message.id] - } + self.valueBox.set(Table_Media_MessageIds.id, key: Table_Media_MessageIds.key(id, messageId: message.id, key: mediaMessageIdKey), value: MemoryBuffer()) } - messageInsertStatement.run(peerId.toInt64(), Int64(message.id.namespace), Int64(message.id.id), messageBlob, referencedMediaIdsMediaIdsBlob, Int64(message.timestamp)) + self.valueBox.set(Table_Message.id, key: Table_Message.key(message.id), value: Table_Message.set(message)) + let absoluteKey = Table_AbsoluteMessageId.emptyKey() if self.absoluteIndexedMessageNamespaces.contains(message.id.namespace) { - var peerIdValue = peerId.toInt64() - memcpy(completeIdDataMutableBytes, &peerIdValue, 8) - var messageIdNamespaceValue: Int32 = message.id.namespace - memcpy(completeIdDataMutableBytes + 8, &messageIdNamespaceValue, 4) - var messageIdIdValue: Int32 = message.id.id - memcpy(completeIdDataMutableBytes + 12, &messageIdIdValue, 4) - absoluteMessageIdsInsertStatement.run(Int64(message.id.id), completeIdBlob) - } - - for id in message.mediaIds { - peerMediaInsertStatement.run(peerId.toInt64(), Int64(id.namespace), Int64(message.id.namespace), Int64(message.id.id)) + self.valueBox.set(Table_AbsoluteMessageId.id, key: Table_AbsoluteMessageId.key(message.id.id, key: absoluteKey), value: Table_AbsoluteMessageId.set(message.id)) } } if let relatedViews = self.peerMessageViews[peerId] { for record in relatedViews.copyItems() { var updated = false - for message in peerMessages { + for message in addedMessages { if record.0.add(RenderedMessage(message: message)) { updated = true } @@ -574,120 +242,42 @@ public final class Postbox { var existingMediaIds = Set() - for (namespace, ids) in Postbox.mediaIdsGroupedByNamespaceFromMediaArray(medias) { - var queryString = "SELECT id FROM media WHERE namespace = ? AND id IN ("; - var first = true - for id in ids { - if first { - first = false - } else { - queryString += "," - } - queryString += "\(id.id)" - } - queryString += ")" - for row in self.database.prepare(queryString).run(Int64(namespace)) { - existingMediaIds.insert(MediaId(namespace: namespace, id: row[0] as! Int64)) - } - } - - var processedMediaIdsForMessageIds = Set() + let existingMediaKey = Table_Media.emptyKey() for media in medias { - if existingMediaIds.contains(media.id) { - continue - } - existingMediaIds.insert(media.id) - - encoder.reset() - encoder.encodeRootObject(media) - let mediaBlob = Blob(data: encoder.makeData()) - - processedMediaIdsForMessageIds.insert(media.id) - if let messageIds = messageIdsByMediaId[media.id] { - let referencedMessageIdsBlob = Postbox.blobForMessageIds(messageIds) - mediaInsertStatement.run(Int64(media.id.namespace), media.id.id, mediaBlob, referencedMessageIdsBlob) - } - } - - var updatedMessageIdsBlobByMediaId: [MediaId : Blob] = [:] - for (mediaId, messageIds) in messageIdsByMediaId { - if !processedMediaIdsForMessageIds.contains(mediaId) { - for row in referencedMessageIdsStatement.run(Int64(mediaId.namespace), mediaId.id) { - var currentMessageIds = Postbox.messageIdsForBlob(row[0] as! Blob) - currentMessageIds += messageIds - updatedMessageIdsBlobByMediaId[mediaId] = Postbox.blobForMessageIds(currentMessageIds) + if let id = media.id { + if self.valueBox.exists(Table_Media.id, key: Table_Media.key(id, key: existingMediaKey)) { + existingMediaIds.insert(id) } } } - for (mediaId, messageIds) in updatedMessageIdsBlobByMediaId { - updateReferencedMessageIdsStatement.run(messageIds, Int64(mediaId.namespace), mediaId.id) - } - } - - private func loadMessageIdsByMediaIdForPeerId(peerId: PeerId, idsByNamespace: [MessageId.Namespace : [MessageId]]) -> [MediaId : [MessageId]] { - var grouped: [MediaId : [MessageId]] = [:] - - for (namespace, ids) in idsByNamespace { - var messageIdByNamespaceAndIdQuery = "SELECT messageId FROM peer_media WHERE peerId = ? AND messageNamespace = ? AND messageId IN (" - var first = true - for id in ids { - if first { - first = false - } else { - messageIdByNamespaceAndIdQuery += "," - } - messageIdByNamespaceAndIdQuery += "\(id.id)" - } - messageIdByNamespaceAndIdQuery += ")" - - var messageIdsWithMedia: [MessageId] = [] - for row in self.database.prepare(messageIdByNamespaceAndIdQuery).run(peerId.toInt64(), Int64(namespace)) { - messageIdsWithMedia.append(MessageId(peerId: peerId, namespace: namespace, id:Int32(row[0] as! Int64))) - } - - var associatedMediaIdsQueryString = "SELECT id, associatedMediaIds FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (" - first = true - for id in messageIdsWithMedia { - if first { - first = false - } else { - associatedMediaIdsQueryString += "," - } - associatedMediaIdsQueryString += "\(id.id)" - } - associatedMediaIdsQueryString += ")" - - for row in self.database.prepare(associatedMediaIdsQueryString).run(peerId.toInt64(), Int64(namespace)) { - let id = MessageId(peerId: peerId, namespace: namespace, id: Int32(row[0] as! Int64)) - let referencedMediaIds = Postbox.mediaIdsForBlob(row[1] as! Blob) - for mediaId in referencedMediaIds { - if grouped[mediaId] != nil { - grouped[mediaId]!.append(id) - } else { - grouped[mediaId] = [id] - } + let mediaKey = Table_Media.emptyKey() + for media in medias { + if let id = media.id { + if existingMediaIds.contains(id) { + continue } + existingMediaIds.insert(id) + + self.valueBox.set(Table_Media.id, key: Table_Media.key(id, key: mediaKey), value: Table_Media.set(media)) } } - - return grouped } private func mediaWithIds(ids: [MediaId]) -> [MediaId : Media] { if ids.count == 0 { return [:] } else { - let select = self.database.prepareCached("SELECT data FROM media WHERE namespace = ? AND id = ?") var result: [MediaId : Media] = [:] + let mediaKey = Table_Media.emptyKey() for id in ids { - for row in select.run(Int64(id.namespace), id.id) { - let blob = row[0] as! Blob - if let media = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false)).decodeRootObject() as? Media { - result[media.id] = media + if let value = self.valueBox.get(Table_Media.id, key: Table_Media.key(id, key: mediaKey)) { + if let media = Table_Media.get(value) { + result[id] = media + } else { + print("can't parse media") } - break } } @@ -701,17 +291,15 @@ public final class Postbox { if let cachedPeer = cachedPeers[peerId] { return cachedPeer } else { - for row in self.database.prepareCached("SELECT data FROM peers WHERE id = ?").run(peerId.toInt64()) { - let data = (row[0] as! Blob).data - let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) + let peerKey = Table_Peer.emptyKey() + if let value = self.valueBox.get(Table_Peer.id, key: Table_Peer.key(peerId, key: peerKey)) { + let decoder = Decoder(buffer: value) if let peer = decoder.decodeRootObject() as? Peer { cachedPeers[peer.id] = peer return peer } else { print("(PostBox: can't decode peer)") } - - break } return nil @@ -722,45 +310,11 @@ public final class Postbox { if ids.count == 0 { return [:] } else { - var remainingIds: [PeerId] = [] - var peers: [PeerId : Peer] = [:] for id in ids { - if let cachedPeer = cachedPeers[id] { - peers[id] = cachedPeer - } else { - remainingIds.append(id) - } - } - - if remainingIds.count != 0 { - let rows: Statement - if ids.count == 1 { - rows = self.database.prepareCached("SELECT data FROM peers WHERE id = ?").run(ids[0].toInt64()) - } else if ids.count == 2 { - rows = self.database.prepareCached("SELECT data FROM peers WHERE id IN (?, ?)").run(ids[0].toInt64(), ids[1].toInt64()) - } else { - var query = "SELECT data FROM peers WHERE id IN (" - var first = true - for id in ids { - if first { - first = false - query += "\(id.toInt64())" - } else { - query += ",\(id.toInt64())" - } - } - query += ")" - rows = self.database.prepare(query).run() - } - - for row in rows { - let blob = row[0] as! Blob - if let peer = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false)).decodeRootObject() as? Peer { - self.cachedPeers[peer.id] = peer - peers[peer.id] = peer - } + if let peer: Peer = self.peerWithId(id) { + peers[id] = peer } } @@ -836,10 +390,9 @@ public final class Postbox { private func updatePeerEntry(peerId: PeerId, message: RenderedMessage?, replace: Bool = false) { var currentIndex: PeerViewEntryIndex? - for row in self.database.prepareCached("SELECT entry FROM peer_entries WHERE peerId = ?").run(peerId.toInt64()) { - let blob = row[0] as! Blob - currentIndex = Postbox.peerViewEntryIndexForBuffer(ReadBuffer(memory: UnsafeMutablePointer(blob.data.bytes), length: blob.data.length, freeWhenDone: false)) - break + + if let value = self.valueBox.get(Table_PeerEntry.id, key: Table_PeerEntry.key(peerId)) { + currentIndex = Table_PeerEntry.get(peerId, value: value) } var updatedPeerMessage: RenderedMessage? @@ -850,8 +403,10 @@ public final class Postbox { if replace || currentIndex.messageIndex < messageIndex { let updatedIndex = PeerViewEntryIndex(peerId: peerId, messageIndex: messageIndex) updatedPeerMessage = message - let updatedBlob = Postbox.blobForPeerViewEntryIndex(updatedIndex) - self.database.prepareCached("UPDATE peer_entries SET entry = ? WHERE peerId = ?").run(updatedBlob, peerId.toInt64()) + + self.valueBox.remove(Table_PeerEntry_Sorted.id, key: Table_PeerEntry_Sorted.key(currentIndex)) + self.valueBox.set(Table_PeerEntry_Sorted.id, key: Table_PeerEntry_Sorted.key(updatedIndex), value: MemoryBuffer()) + self.valueBox.set(Table_PeerEntry.id, key: Table_PeerEntry.key(peerId), value: Table_PeerEntry.set(updatedIndex)) } } else if replace { //TODO: remove? @@ -859,8 +414,9 @@ public final class Postbox { } else if let message = message { updatedPeerMessage = message let updatedIndex = PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(message.message)) - let updatedBlob = Postbox.blobForPeerViewEntryIndex(updatedIndex) - self.database.prepareCached("INSERT INTO peer_entries (peerId, entry) VALUES (?, ?)").run(peerId.toInt64(), updatedBlob) + + self.valueBox.set(Table_PeerEntry_Sorted.id, key: Table_PeerEntry_Sorted.key(updatedIndex), value: MemoryBuffer()) + self.valueBox.set(Table_PeerEntry.id, key: Table_PeerEntry.key(peerId), value: Table_PeerEntry.set(updatedIndex)) } if let updatedPeerMessage = updatedPeerMessage { @@ -900,132 +456,61 @@ public final class Postbox { return [] } - var queryString = "SELECT completeId FROM peer_absolute_indexed_message_ids WHERE id IN (" - var first = true - for id in ids { - if first { - first = false - } else { - queryString += "," - } - queryString += "\(id)" - } - queryString += ")" - var result: [MessageId] = [] - for row in self.database.prepare(queryString).run() { - let blob = row[0] as! Blob - let blobBytes = blob.data.bytes - - var peerIdValue: Int64 = 0 - memcpy(&peerIdValue, blobBytes, 8) - var messageIdNamespaceValue: Int32 = 0 - memcpy(&messageIdNamespaceValue, blobBytes + 8, 4) - var messageIdIdValue: Int32 = 0 - memcpy(&messageIdIdValue, blobBytes + 12, 4) - result.append(MessageId(peerId: PeerId(peerIdValue), namespace: messageIdNamespaceValue, id: messageIdIdValue)) + + let key = Table_AbsoluteMessageId.emptyKey() + for id in ids { + if let value = self.valueBox.get(Table_AbsoluteMessageId.id, key: Table_AbsoluteMessageId.key(id, key: key)) { + result.append(Table_AbsoluteMessageId.get(id, value: value)) + } } return result } private func deleteMessagesWithIds(ids: [MessageId]) { - for (peerId, messageIds) in Postbox.messageIdsGroupedByPeerId(ids) { - let messageIdsByNamespace = Postbox.messageIdsGroupedByNamespace(messageIds) - let messageIdsByMediaId = self.loadMessageIdsByMediaIdForPeerId(peerId, idsByNamespace: messageIdsByNamespace) - - for (peerId, messageIds) in Postbox.messageIdsGroupedByPeerId(ids) { - if let relatedViews = self.peerMessageViews[peerId] { - for (view, pipe) in relatedViews.copyItems() { - let context = view.remove(Set(messageIds)) - if !context.empty() { - view.complete(context, fetchEarlier: self.fetchMessagesRelative(peerId, earlier: true), fetchLater: self.fetchMessagesRelative(peerId, earlier: false)) - self.deferMessageViewUpdate(view, pipe: pipe) + for (peerId, messageIds) in messageIdsGroupedByPeerId(ids) { + if let relatedViews = self.peerMessageViews[peerId] { + for (view, pipe) in relatedViews.copyItems() { + let context = view.remove(Set(messageIds)) + if !context.empty() { + view.complete(context, fetchEarlier: self.fetchMessagesRelative(peerId, earlier: true), fetchLater: self.fetchMessagesRelative(peerId, earlier: false)) + self.deferMessageViewUpdate(view, pipe: pipe) + } + } + } + } + + var touchedMediaIds = Set() + + let removeMediaMessageIdKey = Table_Media_MessageIds.emptyKey() + for (peerId, messageIds) in messageIdsGroupedByPeerId(ids) { + let messageKey = Table_Message.emptyKey() + for id in messageIds { + if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(id, key: messageKey)) { + if let message = Table_Message.get(value) { + for mediaId in message.mediaIds { + touchedMediaIds.insert(mediaId) + self.valueBox.remove(Table_Media_MessageIds.id, key: Table_Media_MessageIds.key(mediaId, messageId: message.id, key: removeMediaMessageIdKey)) } } } } + + for id in messageIds { + self.valueBox.remove(Table_Message.id, key: Table_Message.key(id, key: messageKey)) + } - for (namespace, messageIds) in messageIdsByNamespace { - var queryString = "DELETE FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (" - var first = true - for id in messageIds { - if first { - first = false - } else { - queryString += "," - } - queryString += "\(id.id)" - } - queryString += ")" - self.database.prepare(queryString).run(peerId.toInt64(), Int64(namespace)) + for mediaId in touchedMediaIds { + var referenced = false + self.valueBox.range(Table_Media_MessageIds.id, start: Table_Media_MessageIds.lowerBoundKey(mediaId), end: Table_Media_MessageIds.upperBoundKey(mediaId), keys: { key in + referenced = true + return false + }, limit: 1) - if self.absoluteIndexedMessageNamespaces.contains(namespace) { - var queryString = "DELETE FROM peer_absolute_indexed_message_ids WHERE id IN (" - var first = true - for id in messageIds { - if first { - first = false - } else { - queryString += "," - } - queryString += "\(id.id)" - } - queryString += ")" - self.database.prepare(queryString).run() - } - } - - for (namespace, messageIds) in messageIdsByNamespace { - var queryString = "DELETE FROM peer_media WHERE peerId = ? AND messageNamespace = ? AND messageId IN (" - var first = true - for id in messageIds { - if first { - first = false - } else { - queryString += "," - } - queryString += "\(id.id)" - } - queryString += ")" - self.database.prepare(queryString).run(peerId.toInt64(), Int64(namespace)) - } - - let mediaIdsByNamespace = Postbox.mediaIdsGroupedByNamespaceFromDictionaryKeys(messageIdsByMediaId) - var updatedMessageIdsByMediaId: [MediaId : [MessageId]] = [:] - - for (namespace, mediaIds) in mediaIdsByNamespace { - var queryString = "SELECT id, data, associatedMessageIds FROM media WHERE namespace = ? AND id in (" - var first = true - for id in mediaIds { - if first { - first = false - } else { - queryString += "," - } - queryString += "\(id.id)" - } - queryString += ")" - for row in self.database.prepare(queryString).run(Int64(namespace)) { - let mediaId = MediaId(namespace: namespace, id: row[0] as! Int64) - if let removedMessageIds = messageIdsByMediaId[mediaId] { - var messageIds = Postbox.messageIdsForBlob(row[2] as! Blob).filter { - !removedMessageIds.contains($0) - } - updatedMessageIdsByMediaId[mediaId] = messageIds - - if messageIds.count == 0 { - self.database.prepareCached("INSERT OR IGNORE INTO media_cleanup (namespace, id, data) VALUES (?, ?, ?)").run(Int64(namespace), mediaId.id, row[1] as! Blob) - } - } - } - - for (mediaId, messageIds) in updatedMessageIdsByMediaId { - if messageIds.count == 0 { - self.database.prepareCached("DELETE FROM media WHERE namespace = ? AND id = ?").run(Int64(mediaId.namespace), mediaId.id) - } else { - self.database.prepareCached("UPDATE media SET associatedMessageIds = ? WHERE namespace = ? AND id = ?").run(Postbox.blobForMessageIds(messageIds), Int64(mediaId.namespace), mediaId.id) - } + if !referenced { + //TODO write to cleanup queue + self.valueBox.remove(Table_Media.id, key: Table_Media.key(mediaId)) } } @@ -1050,12 +535,11 @@ public final class Postbox { let currentPeers = self.peersWithIds(peerIds) - let updatePeer = self.database.prepareCached("UPDATE peers SET data = ? WHERE id = ?") - let insertPeer = self.database.prepareCached("INSERT INTO peers (id, data) VALUES (?, ?)") let encoder = Encoder() var updatedPeers: [PeerId : Peer] = [:] + let peerKey = Table_Peer.emptyKey() for updatedPeer in peers { let currentPeer = currentPeers[updatedPeer.id] @@ -1071,11 +555,8 @@ public final class Postbox { encoder.reset() encoder.encodeRootObject(finalPeer) - if currentPeer != nil { - updatePeer.run(Blob(data: encoder.makeData()), finalPeer.id.toInt64()) - } else { - insertPeer.run(finalPeer.id.toInt64(), Blob(data: encoder.makeData())) - } + peerKey.setInt64(0, value: updatedPeer.id.toInt64()) + self.valueBox.set(Table_Peer.id, key: Table_Peer.key(finalPeer.id, key: peerKey), value: encoder.memoryBuffer()) } } @@ -1094,13 +575,13 @@ public final class Postbox { let startTime = CFAbsoluteTimeGetCurrent() //#endif - self.database.transaction() + self.valueBox.begin() let result = f(Modifier(postbox: self)) //print("(Postbox modify took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") //#if DEBUG //startTime = CFAbsoluteTimeGetCurrent() //#endif - self.database.commit() + self.valueBox.commit() //#if DEBUG print("(Postbox commit took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") @@ -1116,8 +597,21 @@ public final class Postbox { } private func findAdjacentMessageIds(peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex) -> (MessageId.Id?, MessageId.Id?) { - var minId: MessageId.Id? + /*var minId: MessageId.Id? var maxId: MessageId.Id? + + let lowerBoundKey = ValueBoxKey(length: 8 + 4) + lowerBoundKey.setInt64(0, value: peerId.toInt64()) + lowerBoundKey.setInt32(8, value: namespace) + + let upperBoundKey = ValueBoxKey(length: 8 + 4) + upperBoundKey.setInt64(0, value: peerId.toInt64()) + upperBoundKey.setInt32(8, value: namespace) + + self.valueBox.range("peer_messages", start: lowerBoundKey, end: upperBoundKey.successor, keys: { key in + + }, limit: 1) + for row in self.database.prepareCached("SELECT MIN(id), MAX(id) FROM peer_messages WHERE peerId = ? AND namespace = ?").run(peerId.toInt64(), Int64(namespace)) { minId = MessageId.Id(row[0] as! Int64) maxId = MessageId.Id(row[1] as! Int64) @@ -1226,7 +720,9 @@ public final class Postbox { } } else { return (nil, nil) - } + }*/ + + return (nil, nil) } private func fetchMessagesAround(peerId: PeerId, anchorId: MessageId, count: Int) -> ([RenderedMessage], [MessageId.Namespace : RenderedMessage], [MessageId.Namespace : RenderedMessage]) { @@ -1346,36 +842,32 @@ public final class Postbox { private func fetchMessagesRelative(peerId: PeerId, earlier: Bool)(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [RenderedMessage] { var messages: [Message] = [] - let sign = earlier ? "<" : ">" - let order = earlier ? "DESC" : "ASC" - let bound: Int64 + let lowerBound = Table_Message.lowerBoundKey(peerId, namespace: namespace) + let upperBound = Table_Message.upperBoundKey(peerId, namespace: namespace) + + let bound: ValueBoxKey if let id = id { - bound = Int64(id) + bound = Table_Message.key(MessageId(peerId: peerId, namespace: namespace, id: id)) } else if earlier { - bound = Int64(Int32.max) + bound = upperBound } else { - bound = Int64(Int32.min) + bound = lowerBound } - var statement: COpaquePointer = nil - sqlite3_prepare_v2(self.database.handle, "SELECT data, associatedMediaIds FROM peer_messages WHERE peerId=\(peerId.toInt64()) AND namespace=\(namespace) AND id \(sign) \(bound) ORDER BY id \(order) LIMIT \(count)", -1, &statement, nil) - - while true { - let result = sqlite3_step(statement) - if result == SQLITE_ROW { - let length = sqlite3_column_bytes(statement, 0) - let data = sqlite3_column_blob(statement, 0) - let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data), length: Int(length), freeWhenDone: false)) - if let message = decoder.decodeRootObject() as? Message { - messages.append(message) - } else { - print("(PostBox: can't decode message)") - } + let values: (ValueBoxKey, ReadBuffer) -> Bool = { _, value in + if let message = Table_Message.get(value) { + messages.append(message) } else { - break + print("can't parse message") } + return true + } + + if earlier { + self.valueBox.range(Table_Message.id, start: bound, end: lowerBound, values: values, limit: count) + } else { + self.valueBox.range(Table_Message.id, start: bound, end: upperBound, values: values, limit: count) } - sqlite3_finalize(statement) return self.renderedMessages(messages) } @@ -1383,51 +875,33 @@ public final class Postbox { private func fetchPeerEntryIndicesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntryIndex] { var entries: [PeerViewEntryIndex] = [] - var statement: COpaquePointer = nil + let lowerBound = Table_PeerEntry_Sorted.lowerBoundKey() + let upperBound = Table_PeerEntry_Sorted.upperBoundKey() + let bound: ValueBoxKey if let index = index { - let bound = Postbox.blobForPeerViewEntryIndex(index) - let sign = earlier ? "<" : ">" - let order = earlier ? "DESC" : "ASC" - - sqlite3_prepare_v2(self.database.handle, "SELECT entry FROM peer_entries WHERE entry \(sign) ? ORDER BY entry \(order) LIMIT \(count)", -1, &statement, nil) - sqlite3_bind_blob(statement, 0, bound.data.bytes, Int32(bound.data.length), nil) - + bound = Table_PeerEntry_Sorted.key(index) + } else if earlier { + bound = upperBound } else { - let order = earlier ? "DESC" : "ASC" - sqlite3_prepare_v2(self.database.handle, "SELECT entry FROM peer_entries ORDER BY entry \(order) LIMIT \(count)", -1, &statement, nil) + bound = lowerBound } - while true { - let result = sqlite3_step(statement) - if result == SQLITE_ROW { - let length = sqlite3_column_bytes(statement, 0) - let data = sqlite3_column_blob(statement, 0) - entries.append(Postbox.peerViewEntryIndexForBuffer(ReadBuffer(memory: UnsafeMutablePointer(data), length: Int(length), freeWhenDone: false))) - } else { - break - } + let keys: ValueBoxKey -> Bool = { key in + entries.append(Table_PeerEntry_Sorted.get(key)) + + return true + } + + if earlier { + self.valueBox.range(Table_PeerEntry_Sorted.id, start: bound, end: lowerBound, keys: keys, limit: count) + } else { + self.valueBox.range(Table_PeerEntry_Sorted.id, start: bound, end: upperBound, keys: keys, limit: count) } - sqlite3_finalize(statement) return entries } - private func messageForPeer(peerId: PeerId, id: MessageId) -> RenderedMessage? { - for row in self.database.prepareCached("SELECT data, associatedMediaIds FROM peer_messages WHERE peerId = ? AND namespace = ? AND id = ?").run(peerId.toInt64(), Int64(id.namespace), Int64(id.id)) { - let data = (row[0] as! Blob).data - let decoder = Decoder(buffer: ReadBuffer(memory: UnsafeMutablePointer(data.bytes), length: data.length, freeWhenDone: false)) - if let message = decoder.decodeRootObject() as? Message { - return self.renderedMessages([message]).first - } else { - print("(PostBox: can't decode message)") - } - break - } - - return nil - } - private func fetchPeerEntriesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntry] { var entries: [PeerViewEntry] = [] var peers: [PeerId : Peer] = [:] @@ -1437,18 +911,23 @@ public final class Postbox { if let cachedPeer = peers[entryIndex.peerId] { peer = cachedPeer } else { - if let fetchedPeer = self.peerWithId(entryIndex.peerId) { + if let fetchedPeer: Peer = self.peerWithId(entryIndex.peerId) { peer = fetchedPeer peers[fetchedPeer.id] = fetchedPeer } } - if let message = self.messageForPeer(entryIndex.peerId, id: entryIndex.messageIndex.id) { + var message: Message? + if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(entryIndex.messageIndex.id)) { + message = Table_Message.get(value) + } + + if let message = message, renderedMessage = self.renderedMessages([message]).first { let entry: PeerViewEntry if let peer = peer { - entry = PeerViewEntry(peer: peer, message: message) + entry = PeerViewEntry(peer: peer, message: renderedMessage) } else { - entry = PeerViewEntry(peerId: entryIndex.peerId, message: message) + entry = PeerViewEntry(peerId: entryIndex.peerId, message: renderedMessage) } entries.append(entry) @@ -1732,54 +1211,15 @@ public final class Postbox { } } - func printMessages(messages: [Message]) { - var string = "" - string += "[" - var first = true - for message in messages { - if first { - first = false - } else { - string += ", " + public func peerWithId(id: PeerId) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.dispatch { + if let peer: Peer = self.peerWithId(id) { + subscriber.putNext(peer) + } } - string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" - } - string += "]" - print(string) - } - - public func _dumpTables() { - print("\n------------") - print("peer_messages") - print("-------------") - for row in self.database.prepare("SELECT peerId, namespace, id, timestamp, associatedMediaIds FROM peer_messages").run() { - print("peer(\(PeerId(row[0] as! Int64))) id(\(MessageId(peerId: PeerId(row[0] as! Int64), namespace: Int32(row[1] as! Int64), id:(Int32(row[2] as! Int64))))) timestamp(\(row[3] as! Int64)) media(\(Postbox.mediaIdsForBlob(row[4] as! Blob)))") - } - - print("\n---------") - print("peer_media") - print("----------") - for row in self.database.prepare("SELECT peerId, mediaNamespace, messageNamespace, messageId FROM peer_media").run() { - print("peer(\(PeerId(row[0] as! Int64))) namespace(\(row[1] as! Int64)) id(\(MessageId(peerId: PeerId(row[0] as! Int64), namespace: Int32(row[2] as! Int64), id:Int32(row[3] as! Int64))))") - } - - print("\n----") - print("media") - print("-----") - for row in self.database.prepare("SELECT namespace, id, associatedMessageIds FROM media").run() { - print("id(\(MediaId(namespace: Int32(row[0] as! Int64), id:(row[1] as! Int64)))) messages(\(Postbox.messageIdsForBlob(row[2] as! Blob)))") - } - - print("\n------------") - print("media_cleanup") - print("-------------") - for row in self.database.prepare("SELECT namespace, id FROM media_cleanup").run() { - print("id(\(MediaId(namespace: Int32(row[0] as! Int64), id:(row[1] as! Int64))))") - } - } - - public func _sync() { - self.queue.sync { + return disposable } } } diff --git a/Postbox/PostboxCodingUtils.swift b/Postbox/PostboxCodingUtils.swift new file mode 100644 index 0000000000..189cecc38e --- /dev/null +++ b/Postbox/PostboxCodingUtils.swift @@ -0,0 +1,233 @@ +import Foundation + +func peerViewEntryIndexForBuffer(buffer: ReadBuffer) -> PeerViewEntryIndex { + var timestamp: Int32 = 0 + buffer.read(×tamp, offset: 0, length: 4) + timestamp = Int32(bigEndian: timestamp) + + var namespace: Int32 = 0 + buffer.read(&namespace, offset: 0, length: 4) + namespace = Int32(bigEndian: namespace) + + var id: Int32 = 0 + buffer.read(&id, offset: 0, length: 4) + id = Int32(bigEndian: id) + + var peerIdRepresentation: Int64 = 0 + buffer.read(&peerIdRepresentation, offset: 0, length: 8) + peerIdRepresentation = Int64(bigEndian: peerIdRepresentation) + + let peerId = PeerId(peerIdRepresentation) + + return PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(id: MessageId(peerId:peerId, namespace: namespace, id: id), timestamp: timestamp)) +} + +func bufferForPeerViewEntryIndex(index: PeerViewEntryIndex) -> MemoryBuffer { + let buffer = WriteBuffer() + + var timestamp = Int32(bigEndian: index.messageIndex.timestamp) + buffer.write(×tamp, offset: 0, length: 4) + + var namespace = Int32(bigEndian: index.messageIndex.id.namespace) + buffer.write(&namespace, offset: 0, length: 4) + + var id = Int32(bigEndian: index.messageIndex.id.id) + buffer.write(&id, offset: 0, length: 4) + + var peerIdRepresentation = Int64(bigEndian: index.peerId.toInt64()) + buffer.write(&peerIdRepresentation, offset: 0, length: 8) + + return buffer +} + +func messageIdsGroupedByNamespace(ids: [MessageId]) -> [MessageId.Namespace : [MessageId]] { + var grouped: [MessageId.Namespace : [MessageId]] = [:] + + for id in ids { + if grouped[id.namespace] != nil { + grouped[id.namespace]!.append(id) + } else { + grouped[id.namespace] = [id] + } + } + + return grouped +} + +func mediaIdsGroupedByNamespaceFromMediaArray(mediaArray: [Media]) -> [MediaId.Namespace : [MediaId]] { + var grouped: [MediaId.Namespace : [MediaId]] = [:] + var seenMediaIds = Set() + + for media in mediaArray { + if let id = media.id { + if !seenMediaIds.contains(id) { + seenMediaIds.insert(id) + if grouped[id.namespace] != nil { + grouped[id.namespace]!.append(id) + } else { + grouped[id.namespace] = [id] + } + } + } + } + + return grouped +} + +func mediaIdsGroupedByNamespaceFromSet(ids: Set) -> [MediaId.Namespace : [MediaId]] { + var grouped: [MediaId.Namespace : [MediaId]] = [:] + + for id in ids { + if let _ = grouped[id.namespace] { + grouped[id.namespace]!.append(id) + } else { + grouped[id.namespace] = [id] + } + } + + return grouped +} + +func mediaIdsGroupedByNamespaceFromDictionaryKeys(dict: [MediaId : T]) -> [MediaId.Namespace : [MediaId]] { + var grouped: [MediaId.Namespace : [MediaId]] = [:] + + for (id, _) in dict { + if grouped[id.namespace] != nil { + grouped[id.namespace]!.append(id) + } else { + grouped[id.namespace] = [id] + } + } + + return grouped +} + +func messagesGroupedByPeerId(messages: [Message]) -> [(PeerId, [Message])] { + var grouped: [(PeerId, [Message])] = [] + + for message in messages { + var i = 0 + let count = grouped.count + var found = false + while i < count { + if grouped[i].0 == message.id.peerId { + grouped[i].1.append(message) + found = true + break + } + i++ + } + if !found { + grouped.append((message.id.peerId, [message])) + } + } + + return grouped +} + +func messageIdsGroupedByPeerId(messageIds: [MessageId]) -> [PeerId : [MessageId]] { + var grouped: [PeerId : [MessageId]] = [:] + + for id in messageIds { + if grouped[id.peerId] != nil { + grouped[id.peerId]!.append(id) + } else { + grouped[id.peerId] = [id] + } + } + + return grouped +} + +func blobForMediaIds(ids: [MediaId]) -> Blob { + let data = NSMutableData() + var version: Int8 = 1 + data.appendBytes(&version, length: 1) + + var count = Int32(ids.count) + data.appendBytes(&count, length:4) + + for id in ids { + var mNamespace = id.namespace + var mId = id.id + data.appendBytes(&mNamespace, length: 4) + data.appendBytes(&mId, length: 8) + } + + return Blob(data: data) +} + +func mediaIdsForBlob(blob: Blob) -> [MediaId] { + var ids: [MediaId] = [] + + var offset: Int = 0 + var version = 0 + blob.data.getBytes(&version, range: NSMakeRange(offset, 1)) + offset += 1 + + if version == 1 { + var count: Int32 = 0 + blob.data.getBytes(&count, range: NSMakeRange(offset, 4)) + offset += 4 + + var i = 0 + while i < Int(count) { + var mNamespace: Int32 = 0 + var mId: Int64 = 0 + blob.data.getBytes(&mNamespace, range: NSMakeRange(offset, 4)) + blob.data.getBytes(&mId, range: NSMakeRange(offset + 4, 8)) + ids.append(MediaId(namespace: mNamespace, id: mId)) + offset += 12 + i++ + } + } + + return ids +} + +func memoryBufferForMessageIds(ids: [MessageId]) -> MemoryBuffer { + let data = NSMutableData() + var version: Int8 = 1 + data.appendBytes(&version, length: 1) + + var count = Int32(ids.count) + data.appendBytes(&count, length:4) + + for id in ids { + var mPeerNamespace = id.peerId.namespace + var mPeerId = id.peerId.id + var mNamespace = id.namespace + var mId = id.id + data.appendBytes(&mPeerNamespace, length: 4) + data.appendBytes(&mPeerId, length: 4) + data.appendBytes(&mNamespace, length: 4) + data.appendBytes(&mId, length: 4) + } + + return MemoryBuffer(data: data) +} + +func messageIdsForMemoryBuffer(buffer: MemoryBuffer) -> [MessageId] { + var ids: [MessageId] = [] + + let readBuffer = ReadBuffer(memoryBufferNoCopy: buffer) + + var count: Int32 = 0 + readBuffer.read(&count, offset: 0, length: 4) + + var i = 0 + while i < Int(count) { + var mPeerNamespace: Int32 = 0 + var mPeerId: Int32 = 0 + var mNamespace: Int32 = 0 + var mId: Int32 = 0 + readBuffer.read(&mPeerNamespace, offset: 0, length: 4) + readBuffer.read(&mPeerId, offset: 0, length: 4) + readBuffer.read(&mNamespace, offset: 0, length: 4) + readBuffer.read(&mId, offset: 0, length: 4) + ids.append(MessageId(peerId: PeerId(namespace: mPeerNamespace, id: mPeerId), namespace: mNamespace, id: mId)) + i++ + } + + return ids +} diff --git a/Postbox/PostboxPrivate/module.modulemap b/Postbox/PostboxPrivate/module.modulemap index b4494ba2ae..52083fa5cf 100644 --- a/Postbox/PostboxPrivate/module.modulemap +++ b/Postbox/PostboxPrivate/module.modulemap @@ -3,5 +3,6 @@ module sqlcipher { header "sqlcipher/sqlite3ext.h" header "sqlcipher/SQLite-Bridging.h" header "sqlcipher/fts3_tokenizer.h" + header "../../submodules/lmdb/libraries/liblmdb/lmdb.h" export * } diff --git a/Postbox/PostboxTables.swift b/Postbox/PostboxTables.swift new file mode 100644 index 0000000000..c17f466265 --- /dev/null +++ b/Postbox/PostboxTables.swift @@ -0,0 +1,274 @@ +import Foundation + +struct Table_Meta { + static let id: Int32 = 0 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 4) + } + + static func key(key: ValueBoxKey = Table_Meta.emptyKey()) -> ValueBoxKey { + key.setInt32(0, value: 0) + return key + } +} + +struct Table_State { + static let id: Int32 = 1 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 4) + } + + static func key(key: ValueBoxKey = Table_State.emptyKey()) -> ValueBoxKey { + key.setInt32(0, value: 0) + return key + } +} + +struct Table_Keychain { + static let id: Int32 = 2 + + static func key(string: String) -> ValueBoxKey { + let data = string.dataUsingEncoding(NSUTF8StringEncoding) ?? NSData() + let key = ValueBoxKey(length: data.length) + memcpy(key.memory, data.bytes, data.length) + return key + } +} + +struct Table_Message { + static let id: Int32 = 4 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 8 + 4 + 4) + } + + static func lowerBoundKey(peerId: PeerId, namespace: Int32) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setInt32(8, value: namespace) + return key + } + + static func upperBoundKey(peerId: PeerId, namespace: Int32) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setInt32(8, value: namespace) + return key.successor + } + + static func key(messageId: MessageId, key: ValueBoxKey = Table_Message.emptyKey()) -> ValueBoxKey { + key.setInt64(0, value: messageId.peerId.toInt64()) + key.setInt32(8, value: messageId.namespace) + key.setInt32(8 + 4, value: messageId.id) + return key + } + + static func set(message: Message, encoder: Encoder = Encoder()) -> MemoryBuffer { + encoder.reset() + encoder.encodeRootObject(message) + return encoder.memoryBuffer() + } + + static func get(value: ReadBuffer) -> Message? { + if let message = Decoder(buffer: value).decodeRootObject() as? Message { + return message + } + return nil + } +} + +struct Table_AbsoluteMessageId { + static let id: Int32 = 5 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 4) + } + + static func key(id: Int32, key: ValueBoxKey = Table_AbsoluteMessageId.emptyKey()) -> ValueBoxKey { + key.setInt32(0, value: id) + return key + } + + static func set(messageId: MessageId) -> MemoryBuffer { + let buffer = WriteBuffer() + + var peerId: Int64 = messageId.peerId.toInt64() + buffer.write(&peerId, offset: 0, length: 8) + var id_namespace: Int32 = messageId.namespace + buffer.write(&id_namespace, offset: 0, length: 4) + + return buffer + } + + static func get(id: Int32, value: ReadBuffer) -> MessageId { + let offset = value.offset + + var peerId: Int64 = 0 + var id_namespace: Int32 = 0 + value.read(&peerId, offset: 0, length: 8) + value.read(&id_namespace, offset: 0, length: 4) + value.offset = offset + + return MessageId(peerId: PeerId(peerId), namespace: id_namespace, id: id) + } +} + +struct Table_Media { + static let id: Int32 = 6 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 4 + 8) + } + + static func key(id: MediaId, key: ValueBoxKey = Table_Media.emptyKey()) -> ValueBoxKey { + key.setInt32(0, value: id.namespace) + key.setInt64(4, value: id.id) + return key + } + + static func set(media: Media, encoder: Encoder = Encoder()) -> MemoryBuffer { + encoder.reset() + encoder.encodeRootObject(media) + return encoder.memoryBuffer() + } + + static func get(value: ReadBuffer) -> Media? { + let decoder = Decoder(buffer: value) + if let media = decoder.decodeRootObject() as? Media { + return media + } + + return nil + } +} + +struct Table_Media_MessageIds { + static let id: Int32 = 3 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 4 + 8 + 8 + 4 + 4) + } + + static func key(id: MediaId, messageId: MessageId, key: ValueBoxKey = Table_Media_MessageIds.emptyKey()) -> ValueBoxKey { + key.setInt32(0, value: id.namespace) + key.setInt64(4, value: id.id) + key.setInt64(4 + 8, value: messageId.peerId.toInt64()) + key.setInt32(4 + 8 + 8, value: messageId.namespace) + key.setInt32(4 + 8 + 8 + 4, value: messageId.id) + return key + } + + static func lowerBoundKey(id: MediaId) -> ValueBoxKey { + let key = ValueBoxKey(length: 4 + 8) + key.setInt32(0, value: id.namespace) + key.setInt64(4, value: id.id) + return key + } + + static func upperBoundKey(id: MediaId) -> ValueBoxKey { + let key = ValueBoxKey(length: 4 + 8) + key.setInt32(0, value: id.namespace) + key.setInt64(4, value: id.id) + return key.successor + } + + static func getMessageId(key: ValueBoxKey) -> MessageId { + let peerId = key.getInt64(4 + 8) + let messageId_namespace: Int32 = key.getInt32(4 + 8 + 8) + let messageId_id: Int32 = key.getInt32(4 + 8 + 8 + 4) + return MessageId(peerId: PeerId(peerId), namespace: messageId_namespace, id: messageId_id) + } +} + +struct Table_Peer { + static let id: Int32 = 7 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 4 + 4) + } + + static func key(id: PeerId, key: ValueBoxKey = Table_Peer.emptyKey()) -> ValueBoxKey { + key.setInt32(0, value: id.namespace) + key.setInt32(4, value: id.id) + return key + } +} + +struct Table_PeerEntry_Sorted { + static let id: Int32 = 8 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 4 + 4 + 4 + 8) + } + + static func lowerBoundKey() -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setInt32(0, value: 0) + return key + } + + static func upperBoundKey() -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setInt32(0, value: Int32.max) + return key + } + + static func key(index: PeerViewEntryIndex, key: ValueBoxKey = Table_PeerEntry_Sorted.emptyKey()) -> ValueBoxKey { + key.setInt32(0, value: index.messageIndex.timestamp) + key.setInt32(4, value: index.messageIndex.id.namespace) + key.setInt32(4 + 4, value: index.messageIndex.id.id) + key.setInt64(4 + 4 + 4, value: index.peerId.toInt64()) + return key + } + + static func get(key: ValueBoxKey) -> PeerViewEntryIndex { + let messageIndex_timestamp = key.getInt32(0) + let messageIndex_id_namespace = key.getInt32(4) + let messageIndex_id_id = key.getInt32(4 + 4) + let messageIndex_peerId = key.getInt64(4 + 4 + 4) + return PeerViewEntryIndex(peerId: PeerId(messageIndex_peerId), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(messageIndex_peerId), namespace: messageIndex_id_namespace, id: messageIndex_id_id), timestamp: messageIndex_timestamp)) + } +} + +struct Table_PeerEntry { + static let id: Int32 = 9 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 8) + } + + static func key(peerId: PeerId, key: ValueBoxKey = Table_PeerEntry.emptyKey()) -> ValueBoxKey { + key.setInt64(0, value: peerId.toInt64()) + return key + } + + static func set(index: PeerViewEntryIndex) -> MemoryBuffer { + let buffer = WriteBuffer() + + var messageId_namespace: Int32 = index.messageIndex.id.namespace + var messageId_id: Int32 = index.messageIndex.id.id + var timestamp: Int32 = index.messageIndex.timestamp + buffer.write(&messageId_namespace, offset: 0, length: 4) + buffer.write(&messageId_id, offset: 0, length: 4) + buffer.write(×tamp, offset: 0, length: 4) + + return buffer + } + + static func get(peerId: PeerId, value: ReadBuffer) -> PeerViewEntryIndex { + let offset = value.offset + + var messageId_namespace: Int32 = 0 + var messageId_id: Int32 = 0 + var timestamp: Int32 = 0 + value.read(&messageId_namespace, offset: 0, length: 4) + value.read(&messageId_id, offset: 0, length: 4) + value.read(×tamp, offset: 0, length: 4) + let index = PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: messageId_namespace, id: messageId_id), timestamp: timestamp)) + value.offset = offset + + return index + } +} diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift new file mode 100644 index 0000000000..4eb9b4d862 --- /dev/null +++ b/Postbox/SqliteValueBox.swift @@ -0,0 +1,574 @@ +import Foundation +import sqlcipher + +private struct SqlitePreparedStatement { + let statement: COpaquePointer + + func bind(index: Int, data: UnsafePointer, length: Int) { + sqlite3_bind_blob(statement, Int32(index), data, Int32(length), nil) + } + + func bind(index: Int, number: Int32) { + sqlite3_bind_int(statement, Int32(index), number) + } + + func reset() { + sqlite3_reset(statement) + sqlite3_clear_bindings(statement) + } + + func step() -> Bool { + let result = sqlite3_step(statement) + if result != SQLITE_ROW && result != SQLITE_DONE { + assertionFailure("Sqlite error \(result)") + } + return result == SQLITE_ROW + } + + func valueAt(index: Int) -> ReadBuffer { + let valueLength = sqlite3_column_bytes(statement, Int32(index)) + let valueData = sqlite3_column_blob(statement, Int32(index)) + + let valueMemory = malloc(Int(valueLength)) + memcpy(valueMemory, valueData, Int(valueLength)) + return ReadBuffer(memory: valueMemory, length: Int(valueLength), freeWhenDone: true) + } + + func keyAt(index: Int) -> ValueBoxKey { + let valueLength = sqlite3_column_bytes(statement, Int32(index)) + let valueData = sqlite3_column_blob(statement, Int32(index)) + + let key = ValueBoxKey(length: Int(valueLength)) + memcpy(key.memory, valueData, Int(valueLength)) + return key + } + + func destroy() { + sqlite3_finalize(statement) + } +} + +public final class SqliteValueBox: ValueBox { + private let database: Database + private var tables = Set() + private var getStatements: [Int32 : SqlitePreparedStatement] = [:] + private var rangeKeyAscStatementsLimit: [Int32 : SqlitePreparedStatement] = [:] + private var rangeKeyAscStatementsNoLimit: [Int32 : SqlitePreparedStatement] = [:] + private var rangeKeyDescStatementsLimit: [Int32 : SqlitePreparedStatement] = [:] + private var rangeKeyDescStatementsNoLimit: [Int32 : SqlitePreparedStatement] = [:] + private var rangeValueAscStatementsLimit: [Int32 : SqlitePreparedStatement] = [:] + private var rangeValueAscStatementsNoLimit: [Int32 : SqlitePreparedStatement] = [:] + private var rangeValueDescStatementsLimit: [Int32 : SqlitePreparedStatement] = [:] + private var rangeValueDescStatementsNoLimit: [Int32 : SqlitePreparedStatement] = [:] + private var existsStatements: [Int32 : SqlitePreparedStatement] = [:] + private var updateStatements: [Int32 : SqlitePreparedStatement] = [:] + private var insertStatements: [Int32 : SqlitePreparedStatement] = [:] + private var deleteStatements: [Int32 : SqlitePreparedStatement] = [:] + + public init(basePath: String) { + do { + try NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil) + } catch _ { } + let path = basePath + "/db_sqlite" + self.database = Database(path) + + self.database.adjustChunkSize() + self.database.execute("PRAGMA page_size=1024") + self.database.execute("PRAGMA cache_size=-2097152") + self.database.execute("PRAGMA synchronous=NORMAL") + self.database.execute("PRAGMA journal_mode=truncate") + self.database.execute("PRAGMA temp_store=MEMORY") + //self.database.execute("PRAGMA wal_autocheckpoint=32") + //self.database.execute("PRAGMA journal_size_limit=1536") + + let result = self.database.scalar("PRAGMA user_version") as! Int64 + if result != 1 { + self.database.execute("PRAGMA user_version=1") + self.database.execute("CREATE TABLE __meta_tables (name INTEGER)") + } + for row in self.database.prepare("SELECT name FROM __meta_tables").run() { + self.tables.insert(Int32(row[0] as! Int64)) + } + } + + public func begin() { + self.database.transaction() + } + + public func commit() { + self.database.commit() + } + + private func getStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.getStatements[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT value FROM t\(table) WHERE key=?", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.getStatements[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: key.memory, length: key.length) + + return resultStatement + } + + private func rangeKeyAscStatementLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.rangeKeyAscStatementsLimit[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.rangeKeyAscStatementsLimit[table] = preparedStatement + resultStatement = preparedStatement + } + resultStatement.reset() + + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + resultStatement.bind(3, number: Int32(limit)) + + return resultStatement + } + + private func rangeKeyAscStatementNoLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.rangeKeyAscStatementsNoLimit[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.rangeKeyAscStatementsNoLimit[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + + return resultStatement + } + + private func rangeKeyDescStatementLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.rangeKeyDescStatementsLimit[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.rangeKeyDescStatementsLimit[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + resultStatement.bind(3, number: Int32(limit)) + + return resultStatement + } + + private func rangeKeyDescStatementNoLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.rangeKeyDescStatementsNoLimit[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.rangeKeyDescStatementsNoLimit[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + + return resultStatement + } + + private func rangeValueAscStatementLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.rangeValueAscStatementsLimit[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.rangeValueAscStatementsLimit[table] = preparedStatement + resultStatement = preparedStatement + } + resultStatement.reset() + + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + resultStatement.bind(3, number: Int32(limit)) + + return resultStatement + } + + private func rangeValueAscStatementNoLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.rangeValueAscStatementsNoLimit[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.rangeValueAscStatementsNoLimit[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + + return resultStatement + } + + private func rangeValueDescStatementLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.rangeValueDescStatementsLimit[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.rangeValueDescStatementsLimit[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + resultStatement.bind(3, number: Int32(limit)) + + return resultStatement + } + + private func rangeValueDescStatementNoLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.rangeKeyDescStatementsNoLimit[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.rangeValueDescStatementsNoLimit[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + + return resultStatement + } + + private func existsStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.existsStatements[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key=?", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.existsStatements[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: key.memory, length: key.length) + + return resultStatement + } + + private func updateStatement(table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.updateStatements[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "UPDATE t\(table) SET value=? WHERE key=?", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.updateStatements[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: value.memory, length: value.length) + resultStatement.bind(2, data: key.memory, length: key.length) + + return resultStatement + } + + private func insertStatement(table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.insertStatements[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table) (key, value) VALUES(?, ?)", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.insertStatements[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: key.memory, length: key.length) + resultStatement.bind(2, data: value.memory, length: value.length) + + return resultStatement + } + + private func deleteStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + let resultStatement: SqlitePreparedStatement + + if let statement = self.deleteStatements[table] { + resultStatement = statement + } else { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(self.database.handle, "DELETE FROM t\(table) WHERE key=?", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.deleteStatements[table] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + resultStatement.bind(1, data: key.memory, length: key.length) + + return resultStatement + } + + public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? { + if self.tables.contains(table) { + let statement = self.getStatement(table, key: key) + + var buffer: ReadBuffer? + + while statement.step() { + buffer = statement.valueAt(0) + break + } + + statement.reset() + + return buffer + } + + return nil + } + + public func exists(table: Int32, key: ValueBoxKey) -> Bool { + if let _ = self.get(table, key: key) { + return true + } + return false + } + + public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { + if start == end { + return + } + + if self.tables.contains(table) { + let statement: SqlitePreparedStatement + + if start < end { + if limit <= 0 { + statement = self.rangeValueAscStatementNoLimit(table, start: start, end: end) + } else { + statement = self.rangeValueAscStatementLimit(table, start: start, end: end, limit: limit) + } + } else { + if limit <= 0 { + statement = self.rangeValueDescStatementNoLimit(table, start: end, end: start) + } else { + statement = self.rangeValueDescStatementLimit(table, start: end, end: start, limit: limit) + } + } + + while statement.step() { + let key = statement.keyAt(0) + let value = statement.valueAt(1) + + if !values(key, value) { + break + } + } + + statement.reset() + } + } + + public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: ValueBoxKey -> Bool, limit: Int) { + if self.tables.contains(table) { + let statement: SqlitePreparedStatement + + if start < end { + if limit <= 0 { + statement = self.rangeKeyAscStatementNoLimit(table, start: start, end: end) + } else { + statement = self.rangeKeyAscStatementLimit(table, start: start, end: end, limit: limit) + } + } else { + if limit <= 0 { + statement = self.rangeKeyDescStatementNoLimit(table, start: end, end: start) + } else { + statement = self.rangeKeyDescStatementLimit(table, start: end, end: start, limit: limit) + } + } + + while statement.step() { + let key = statement.keyAt(0) + + if !keys(key) { + break + } + } + + statement.reset() + } + } + + public func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) { + if !self.tables.contains(table) { + self.database.execute("CREATE TABLE t\(table) (key BLOB, value BLOB)") + self.database.execute("CREATE INDEX t\(table)_key ON t\(table) (key)") + self.tables.insert(table) + self.database.execute("INSERT INTO __meta_tables(name) VALUES (\(table))") + } + + var exists = false + let existsStatement = self.existsStatement(table, key: key) + if existsStatement.step() { + exists = true + } + existsStatement.reset() + + if exists { + let statement = self.updateStatement(table, key: key, value: value) + while statement.step() { + } + statement.reset() + } else { + let statement = self.insertStatement(table, key: key, value: value) + while statement.step() { + } + statement.reset() + } + } + + public func remove(table: Int32, key: ValueBoxKey) { + if self.tables.contains(table) { + + let statement = self.deleteStatement(table, key: key) + while statement.step() { + } + statement.reset() + } + } + + public func drop() { + for (_, statement) in self.getStatements { + statement.destroy() + } + self.getStatements.removeAll() + + for (_, statement) in self.rangeKeyAscStatementsLimit { + statement.destroy() + } + self.rangeKeyAscStatementsLimit.removeAll() + + for (_, statement) in self.rangeKeyAscStatementsNoLimit { + statement.destroy() + } + self.rangeKeyAscStatementsNoLimit.removeAll() + + for (_, statement) in self.rangeKeyDescStatementsLimit { + statement.destroy() + } + self.rangeKeyDescStatementsLimit.removeAll() + + for (_, statement) in self.rangeKeyDescStatementsNoLimit { + statement.destroy() + } + self.rangeKeyDescStatementsNoLimit.removeAll() + + for (_, statement) in self.rangeValueAscStatementsLimit { + statement.destroy() + } + self.rangeValueAscStatementsLimit.removeAll() + + for (_, statement) in self.rangeValueAscStatementsNoLimit { + statement.destroy() + } + self.rangeValueAscStatementsNoLimit.removeAll() + + for (_, statement) in self.rangeValueDescStatementsLimit { + statement.destroy() + } + self.rangeValueDescStatementsLimit.removeAll() + + for (_, statement) in self.rangeValueDescStatementsNoLimit { + statement.destroy() + } + self.rangeValueDescStatementsNoLimit.removeAll() + + for (_, statement) in self.existsStatements { + statement.destroy() + } + self.existsStatements.removeAll() + + for (_, statement) in self.updateStatements { + statement.destroy() + } + self.updateStatements.removeAll() + + for (_, statement) in self.insertStatements { + statement.destroy() + } + self.insertStatements.removeAll() + + for (_, statement) in self.deleteStatements { + statement.destroy() + } + self.deleteStatements.removeAll() + + for table in self.tables { + self.database.execute("DROP TABLE IF EXISTS t\(table)") + } + self.database.execute("DELETE FROM __meta_tables") + } +} diff --git a/Postbox/ValueBox.swift b/Postbox/ValueBox.swift new file mode 100644 index 0000000000..7549d15ea1 --- /dev/null +++ b/Postbox/ValueBox.swift @@ -0,0 +1,14 @@ +import Foundation + +public protocol ValueBox { + func begin() + func commit() + + func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) + func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: ValueBoxKey -> Bool, limit: Int) + func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? + func exists(table: Int32, key: ValueBoxKey) -> Bool + func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) + func remove(table: Int32, key: ValueBoxKey) + func drop() +} diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift new file mode 100644 index 0000000000..f017b8719a --- /dev/null +++ b/Postbox/ValueBoxKey.swift @@ -0,0 +1,144 @@ +import Foundation + +private final class ValueBoxKeyImpl { + let memory: UnsafeMutablePointer + + init(memory: UnsafeMutablePointer) { + self.memory = memory + } + + deinit { + free(self.memory) + } +} + +public struct ValueBoxKey: Comparable, CustomStringConvertible { + public let memory: UnsafeMutablePointer + public let length: Int + private let impl: ValueBoxKeyImpl + + public init(length: Int) { + self.memory = malloc(length) + self.length = length + self.impl = ValueBoxKeyImpl(memory: self.memory) + } + + public init(_ value: String) { + let data = value.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! + self.memory = malloc(data.length) + self.length = data.length + self.impl = ValueBoxKeyImpl(memory: self.memory) + memcpy(self.memory, data.bytes, data.length) + } + + public init(_ buffer: MemoryBuffer) { + self.memory = malloc(buffer.length) + self.length = buffer.length + self.impl = ValueBoxKeyImpl(memory: self.memory) + memcpy(self.memory, buffer.memory, buffer.length) + } + + public func setInt32(offset: Int, value: Int32) { + var bigEndianValue = Int32(bigEndian: value) + memcpy(self.memory + offset, &bigEndianValue, 4) + } + + public func setInt64(offset: Int, value: Int64) { + var bigEndianValue = Int64(bigEndian: value) + memcpy(self.memory + offset, &bigEndianValue, 8) + } + + public func getInt32(offset: Int) -> Int32 { + var value: Int32 = 0 + memcpy(&value, self.memory + offset, 4) + return Int32(bigEndian: value) + } + + public func getInt64(offset: Int) -> Int64 { + var value: Int64 = 0 + memcpy(&value, self.memory + offset, 8) + return Int64(bigEndian: value) + } + + public func prefix(length: Int) -> ValueBoxKey { + assert(length <= self.length, "length <= self.length") + let key = ValueBoxKey(length: length) + memcpy(key.memory, self.memory, length) + return key + } + + public var successor: ValueBoxKey { + let key = ValueBoxKey(length: self.length) + memcpy(key.memory, self.memory, self.length) + let memory = UnsafeMutablePointer(key.memory) + var i = self.length - 1 + while i >= 0 { + var byte = memory[i] + if byte != 0xff { + byte += 1 + memory[i] = byte + break + } else { + byte = 0 + memory[i] = byte + } + i-- + } + return key + } + + public var predecessor: ValueBoxKey { + let key = ValueBoxKey(length: self.length) + memcpy(key.memory, self.memory, self.length) + let memory = UnsafeMutablePointer(key.memory) + var i = self.length - 1 + while i >= 0 { + var byte = memory[i] + if byte != 0x00 { + byte -= 1 + memory[i] = byte + break + } else { + byte = 0xff + memory[i] = byte + } + i-- + } + return key + } + + public var description: String { + let string = NSMutableString() + let memory = UnsafeMutablePointer(self.memory) + for i in 0 ..< self.length { + let byte: Int = Int(memory[i]) + string.appendFormat("%02x", byte) + } + return string as String + } +} + +public func ==(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { + return lhs.length == rhs.length && memcmp(lhs.memory, rhs.memory, lhs.length) == 0 +} + +private func mdb_cmp_memn(a_memory: UnsafeMutablePointer, _ a_length: Int, _ b_memory: UnsafeMutablePointer, _ b_length: Int) -> Int +{ + var diff: Int = 0 + var len_diff: Int = 0 + var len: Int = 0 + + len = a_length + len_diff = a_length - b_length + if len_diff > 0 { + len = b_length + len_diff = 1 + } + + diff = Int(memcmp(a_memory, b_memory, len)) + return diff != 0 ? diff : len_diff < 0 ? -1 : len_diff +} + +public func <(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { + return mdb_cmp_memn(lhs.memory, lhs.length, rhs.memory, rhs.length) < 0 +} diff --git a/PostboxTests/CodingTests.swift b/PostboxTests/CodingTests.swift index 8a8f69011a..f8f70bdaec 100644 --- a/PostboxTests/CodingTests.swift +++ b/PostboxTests/CodingTests.swift @@ -97,6 +97,14 @@ func ==(lhs: TestKey, rhs: TestKey) -> Bool { return lhs.value == rhs.value } +class EmptyState: PostboxState { + required init(decoder: Decoder) { + } + + func encode(encoder: Encoder) { + } +} + class SerializationTests: XCTestCase { override func setUp() { super.setUp() @@ -107,9 +115,9 @@ class SerializationTests: XCTestCase { } func testExample() { - declareEncodable(TestParent.self, { TestParent(decoder: $0) }) - declareEncodable(TestObject.self, { TestObject(decoder: $0) }) - declareEncodable(TestKey.self, { TestKey(decoder: $0) }) + declareEncodable(TestParent.self, f: { TestParent(decoder: $0) }) + declareEncodable(TestObject.self, f: { TestObject(decoder: $0) }) + declareEncodable(TestKey.self, f: { TestKey(decoder: $0) }) let encoder = Encoder() encoder.encodeInt32(12345, forKey: "a") @@ -156,4 +164,98 @@ class SerializationTests: XCTestCase { XCTAssert(decoder.decodeInt64ForKey("b") == Int64(12345), "int64 failed") XCTAssert(decoder.decodeInt32ForKey("a") == 12345, "int32 failed") } + + func testIndexBaselinePerformance() { + let basePath = "/tmp/postboxtest" + do { + try NSFileManager.defaultManager().removeItemAtPath(basePath) + } catch _ { } + let postbox = Postbox(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: []) + postbox._prepareBaselineIndexPerformance() + + measureBlock { + postbox._measureBaselineIndexPerformance() + } + } + + func testBlobReadPerformance() { + let basePath = "/tmp/postboxtest" + do { + try NSFileManager.defaultManager().removeItemAtPath(basePath) + } catch _ { } + let postbox = Postbox(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: []) + postbox._prepareBlobIndexPerformance() + + measureBlock { + postbox._measureBlobReadPerformance() + } + } + + func testBlobIndexPerformance() { + let basePath = "/tmp/postboxtest" + do { + try NSFileManager.defaultManager().removeItemAtPath(basePath) + } catch _ { } + let postbox = Postbox(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: []) + postbox._prepareBlobIndexPerformance() + + measureBlock { + postbox._measureBlobIndexPerformance() + } + } + + func testKeys() { + let key1 = ValueBoxKey(length: 8) + let key2 = ValueBoxKey(length: 8) + + key1.setInt32(0, value: 1) + key1.setInt32(4, value: 2) + + key2.setInt32(0, value: 1) + key2.setInt32(4, value: 3) + + let lowerBound = ValueBoxKey(length: 4) + lowerBound.setInt32(0, value: 0) + let upperBound = ValueBoxKey(length: 4) + upperBound.setInt32(0, value: 2) + + XCTAssert(key1 > lowerBound, "key1 <= lowerBound") + XCTAssert(key1 < upperBound, "key1 >= upperBound") + XCTAssert(key2 > lowerBound, "key2 <= lowerBound") + XCTAssert(key2 < upperBound, "key2 >= upperBound") + + XCTAssert(key1 < key2, "key1 >= key2") + XCTAssert(key1.successor == key2, "key1.next != key2") + XCTAssert(key2.predecessor == key1, "key2.previous != key1") + } + + func testKeyValue() { + let basePath = "/tmp/postboxtest" + do { + try NSFileManager.defaultManager().removeItemAtPath(basePath) + } catch _ { } + + let box = SqliteValueBox(basePath: basePath) + box.transaction { transaction in + let key = ValueBoxKey(length: 4) + let value = WriteBuffer() + for i in 1 ... 100 { + key.setInt32(0, value: Int32(i)) + transaction.set("test", key: key, value: value) + } + } + + do { + box.transaction { transaction in + let lowerBound = ValueBoxKey(length: 4) + lowerBound.setInt32(0, value: 2) + let upperBound = ValueBoxKey(length: 4) + upperBound.setInt32(0, value: 99) + transaction.range("test", start: upperBound, end: lowerBound, values: { key, value in + print("\(key.getInt32(0))") + return true + }, limit: 10) + } + } + } } diff --git a/PostboxTests/PostboxTests.swift b/PostboxTests/PostboxTests.swift deleted file mode 100644 index 8b905a676d..0000000000 --- a/PostboxTests/PostboxTests.swift +++ /dev/null @@ -1,826 +0,0 @@ -import UIKit -import XCTest - -import Postbox - -enum TestPeerNamespace: PeerId.Namespace { - case User = 0 -} - -enum TestMessageNamespace: MessageId.Namespace { - case Cloud = 0 - case Local = 1 -} - -enum TestMediaNamespace: MediaId.Namespace { - case Test = 0 -} - -class TestPeer: Peer { - var id: PeerId - - init(id: PeerId) { - self.id = id - } - - required init(decoder: Decoder) { - self.id = PeerId(decoder.decodeInt64ForKey("id")) - } - - func encode(encoder: Encoder) { - encoder.encodeInt64(self.id.toInt64(), forKey: "id") - } -} - -class TestMessage: Message { - var id: MessageId - var authorId: PeerId - var date: Int32 - var text: String - var referencedMediaIds: [MediaId] - - var timestamp: Int32 { - return date - } - - init(id: MessageId, authorId: PeerId, date: Int32, text: String, referencedMediaIds: [MediaId]) { - self.id = id - self.authorId = authorId - self.date = date - self.text = text - self.referencedMediaIds = referencedMediaIds - } - - required init(decoder: Decoder) { - self.id = MessageId(decoder.decodeBytesForKeyNoCopy("id")) - self.authorId = PeerId(decoder.decodeInt64ForKey("authorId")) - self.date = decoder.decodeInt32ForKey("date") - self.text = decoder.decodeStringForKey("text") - self.referencedMediaIds = MediaId.decodeArrayFromBuffer(decoder.decodeBytesForKeyNoCopy("mediaIds")) - } - - func encode(encoder: Encoder) { - let buffer = WriteBuffer() - self.id.encodeToBuffer(buffer) - encoder.encodeBytes(buffer, forKey: "id") - buffer.reset() - - encoder.encodeInt64(self.authorId.toInt64(), forKey: "authorId") - encoder.encodeInt32(self.date, forKey: "date") - encoder.encodeString(self.text, forKey: "text") - - MediaId.encodeArrayToBuffer(self.referencedMediaIds, buffer: buffer) - encoder.encodeBytes(buffer, forKey: "mediaIds") - buffer.reset() - } -} - -class TestMedia: Media { - var id: MediaId - - init(id: MediaId) { - self.id = id - } - - required init(decoder: Decoder) { - self.id = MediaId(decoder.decodeBytesForKeyNoCopy("id")) - } - - func encode(encoder: Encoder) { - let buffer = WriteBuffer() - self.id.encodeToBuffer(buffer) - encoder.encodeBytes(buffer, forKey: "id") - } -} - -class EmptyState: PostboxState { - required init(decoder: Decoder) { - } - - func encode(encoder: Encoder) { - } -} - -class PostboxTests: XCTestCase { - - override func setUp() { - super.setUp() - } - - override func tearDown() { - super.tearDown() - } - - func testAddMessages() { - declareEncodable(TestMessage.self, { TestMessage(decoder: $0) }) - declareEncodable(TestMedia.self, { TestMedia(decoder: $0) }) - - let ownerId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 1000) - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - let messageNamespace = TestMessageNamespace.Cloud.rawValue - - let basePath = "/tmp/postboxtest" - NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) - (postbox.modify { state -> Void in - let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) - for i in 0 ..< 10 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) - let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) - state.addMessages([message, message], medias: [testMedia]) - } - return - }).start() - postbox._dumpTables() - - (postbox.modify { state -> Void in - var messageIds: [MessageId] = [] - for i in 0 ..< 5 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) - messageIds.append(messageId) - } - state.deleteMessagesWithIds(messageIds) - }).start() - postbox._dumpTables(); - - (postbox.modify { state -> Void in - var messageIds: [MessageId] = [] - for i in 0 ..< 10 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) - messageIds.append(messageId) - } - state.deleteMessagesWithIds(messageIds) - }).start() - postbox._dumpTables(); - } - - func testMessageNamespaceViewAddTail() { - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - - let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [:], messages: [], later: [:]) - - func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { - let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) - } - - func add(message: Message) { - view.add(message) - println("\(view)\n") - } - - /*func assertIds(earlier: Int32?, ids: [Int32], later: Int32?) { - let otherMessages = ids.reverse().map({ id -> Message in - let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(id)) - let message = TestMessage(id: messageId, authorId: otherId, date: Int32(0), text: "", referencedMediaIds: []) - return message - }) - let otherView = MessageNamespaceView(namespace: messageNamespace, messages: otherMessages, earlierId: earlier, laterId: later, count: 2) - - XCTAssert(view == otherView, "\(view) != \(otherView)") - }*/ - - add(message(1, 90, 90)) - add(message(2, 70, 70)) - - add(message(1, 70, 70)) - - add(message(1, 80, 80)) - add(message(2, 100, 100)) - - add(message(1, 75, 75)) - - add(message(1, 60, 60)) - } - - func testMessageNamespaceViewAddMiddle1() { - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - - func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { - let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) - } - - let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [1: message(1, 90, 90)], messages: [message(1, 100, 100), message(1, 120, 120), message(1, 140, 140)], later: [1: message(1, 200, 200)]) - - func add(message: Message) { - view.add(message) - println("\(view)\n") - } - - add(message(2, 105, 105)) - add(message(2, 150, 150)) - add(message(2, 250, 250)) - add(message(2, 180, 180)) - } - - func testMessageNamespaceRemoveTail() { - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - - var messages: [Message] = [] - - func print(messages: [Message]) { - var string = "" - string += "[" - var first = true - for message in messages { - if first { - first = false - } else { - string += ", " - } - string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" - } - string += "]" - println(string) - } - - let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [:], messages: [], later: [:]) - - func id(namespace: Int32, id: Int32) -> MessageId { - return MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - } - - func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { - let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) - } - - func add(message: Message) { - view.add(message) - println("\(view)") - - messages.append(message) - messages.sort({MessageIndex($0) < MessageIndex($1)}) - - print(messages) - println() - } - - func remove(ids: Set) -> MutableMessageView.RemoveContext { - let context = view.remove(ids) - println("\(view)") - - messages = messages.filter { !ids.contains($0.id) } - print(messages) - - return context - } - - func fetchEarlier(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { - var filtered: [Message] = [] - var i = messages.count - 1 - while i >= 0 && filtered.count < count { - if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id < id!) { - filtered.append(messages[i]) - } - i-- - } - - return filtered - } - - func fetchLater(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { - var filtered: [Message] = [] - var i = 0 - while i < messages.count && filtered.count < count { - if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id > id!) { - filtered.append(messages[i]) - } - i++ - } - - return filtered - } - - func complete(context: MutableMessageView.RemoveContext) { - view.complete(context, fetchEarlier: fetchEarlier(messages), fetchLater: fetchLater(messages)) - println("\(view)\n") - } - - add(message(1, 90, 90)) - add(message(2, 70, 70)) - add(message(1, 70, 70)) - add(message(1, 80, 80)) - add(message(2, 100, 100)) - add(message(1, 75, 75)) - add(message(1, 60, 60)) - - println("remove 1:90, 2:100") - complete(remove([id(1, 90), id(2, 100)])) - - println("remove 1:60, 2:70") - complete(remove([id(1, 60), id(2, 70)])) - - println("remove 1:80, 2:100") - complete(remove([id(1, 80), id(2, 100)])) - } - - func testMessageNamespaceRemoveAllInside() { - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - - var messages: [Message] = [] - - func print(messages: [Message]) { - var string = "" - string += "[" - var first = true - for message in messages { - if first { - first = false - } else { - string += ", " - } - string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" - } - string += "]" - println(string) - } - - let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [:], messages: [], later: [:]) - - func id(namespace: Int32, id: Int32) -> MessageId { - return MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - } - - func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { - let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) - } - - func add(message: Message) { - view.add(message) - println("\(view)") - - messages.append(message) - messages.sort({MessageIndex($0) < MessageIndex($1)}) - - print(messages) - println() - } - - func remove(ids: Set) -> MutableMessageView.RemoveContext { - let context = view.remove(ids) - println("\(view)") - - messages = messages.filter { !ids.contains($0.id) } - print(messages) - - return context - } - - func fetchEarlier(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { - var filtered: [Message] = [] - var i = messages.count - 1 - while i >= 0 && filtered.count < count { - if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id < id!) { - filtered.append(messages[i]) - } - i-- - } - - return filtered - } - - func fetchLater(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { - var filtered: [Message] = [] - var i = 0 - while i < messages.count && filtered.count < count { - if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id > id!) { - filtered.append(messages[i]) - } - i++ - } - - return filtered - } - - func complete(context: MutableMessageView.RemoveContext) { - view.complete(context, fetchEarlier: fetchEarlier(messages), fetchLater: fetchLater(messages)) - println("\(view)\n") - } - - add(message(2, 10, 10)) - add(message(2, 20, 20)) - add(message(1, 90, 90)) - add(message(2, 70, 70)) - add(message(1, 70, 70)) - add(message(1, 80, 80)) - add(message(2, 100, 100)) - add(message(1, 75, 75)) - add(message(1, 60, 60)) - - println("remove 2:20, 1:80, 1:90, 2:100") - complete(remove([id(2, 20), id(1, 80), id(1, 90), id(2, 100)])) - } - - func testMessageNamespaceRemoveMiddleSome() { - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - - var messages: [Message] = [] - - func print(messages: [Message]) { - var string = "" - string += "[" - var first = true - for message in messages { - if first { - first = false - } else { - string += ", " - } - string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" - } - string += "]" - println(string) - } - - func id(namespace: Int32, id: Int32) -> MessageId { - return MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - } - - func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { - let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) - } - - messages = [message(2, 1, 1), message(2, 2, 2), message(1, 98, 98), message(1, 99, 99), message(1, 100, 100), message(2, 101, 101), message(1, 102, 102), message(1, 103, 103), message(1, 104, 104), message(1, 105, 105), message(2, 200, 200), message(2, 300, 300)] - - let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [1: message(1, 99, 99), 2: message(2, 2, 2)], messages: [message(1, 100, 100), message(2, 101, 101), message(1, 102, 102)], later: [1: message(1, 103, 103), 2: message(2, 200, 200)]) - - func add(message: Message) { - view.add(message) - println("\(view)") - - messages.append(message) - messages.sort({MessageIndex($0) < MessageIndex($1)}) - - print(messages) - println() - } - - func remove(ids: Set) -> MutableMessageView.RemoveContext { - let context = view.remove(ids) - println("\(view)") - - messages = messages.filter { !ids.contains($0.id) } - print(messages) - - return context - } - - func fetchEarlier(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { - var filtered: [Message] = [] - var i = messages.count - 1 - while i >= 0 && filtered.count < count { - if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id < id!) { - filtered.append(messages[i]) - } - i-- - } - - return filtered - } - - func fetchLater(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { - var filtered: [Message] = [] - var i = 0 - while i < messages.count && filtered.count < count { - if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id > id!) { - filtered.append(messages[i]) - } - i++ - } - - return filtered - } - - func complete(context: MutableMessageView.RemoveContext) { - view.complete(context, fetchEarlier: fetchEarlier(messages), fetchLater: fetchLater(messages)) - println("\(view)\n") - } - - print(messages) - println("\(view)") - - println("remove 1:100, 2:101") - complete(remove([id(1, 100), id(2, 101)])) - } - - func testMessageNamespaceRemoveMiddleAllInside() { - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - - var messages: [Message] = [] - - func print(messages: [Message]) { - var string = "" - string += "[" - var first = true - for message in messages { - if first { - first = false - } else { - string += ", " - } - string += "\(message.id.namespace): \(message.id.id)—\(message.timestamp)" - } - string += "]" - println(string) - } - - func id(namespace: Int32, id: Int32) -> MessageId { - return MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - } - - func message(namespace: Int32, id: Int32, timestamp: Int32) -> Message { - let messageId = MessageId(peerId: otherId, namespace: namespace, id: Int32(id)) - return TestMessage(id: messageId, authorId: otherId, date: Int32(timestamp), text: "", referencedMediaIds: []) - } - - messages = [message(2, 1, 1), message(2, 2, 2), message(1, 98, 98), message(1, 99, 99), message(1, 100, 100), message(2, 101, 101), message(1, 102, 102), message(1, 103, 103), message(1, 104, 104), message(1, 105, 105), message(2, 200, 200), message(2, 300, 300)] - - let view = MutableMessageView(namespaces: [1, 2], count: 3, earlier: [1: message(1, 99, 99), 2: message(2, 2, 2)], messages: [message(1, 100, 100), message(2, 101, 101), message(1, 102, 102)], later: [1: message(1, 103, 103), 2: message(2, 200, 200)]) - - func add(message: Message) { - view.add(message) - println("\(view)") - - messages.append(message) - messages.sort({MessageIndex($0) < MessageIndex($1)}) - - print(messages) - println() - } - - func remove(ids: Set) -> MutableMessageView.RemoveContext { - let context = view.remove(ids) - println("\(view)") - - messages = messages.filter { !ids.contains($0.id) } - print(messages) - - return context - } - - func fetchEarlier(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { - var filtered: [Message] = [] - var i = messages.count - 1 - while i >= 0 && filtered.count < count { - if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id < id!) { - filtered.append(messages[i]) - } - i-- - } - - return filtered - } - - func fetchLater(messages: [Message])(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [Message] { - var filtered: [Message] = [] - var i = 0 - while i < messages.count && filtered.count < count { - if messages[i].id.namespace == namespace && (id == nil || messages[i].id.id > id!) { - filtered.append(messages[i]) - } - i++ - } - - return filtered - } - - func complete(context: MutableMessageView.RemoveContext) { - view.complete(context, fetchEarlier: fetchEarlier(messages), fetchLater: fetchLater(messages)) - println("\(view)\n") - } - - print(messages) - println("\(view)\n") - - println("remove 1:100, 2:101, 1: 102") - complete(remove([id(1, 100), id(2, 101), id(1, 102)])) - } - - func testViewTail() { - declareEncodable(TestMessage.self, { TestMessage(decoder: $0) }) - declareEncodable(TestMedia.self, { TestMedia(decoder: $0) }) - - let ownerId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 1000) - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - let messageNamespace = TestMessageNamespace.Cloud.rawValue - - let basePath = "/tmp/postboxtest" - NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) - - (postbox.modify { state -> Void in - let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) - for i in 0 ..< 10 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) - let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) - state.addMessages([message, message], medias: [testMedia]) - } - return - }).start() - - postbox.tailMessageViewForPeerId(otherId, count: 4).start(next: { next in - println(next) - }) - - (postbox.modify { state -> Void in - let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) - for i in 10 ..< 15 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) - let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) - state.addMessages([message, message], medias: [testMedia]) - } - return - }).start() - - postbox._sync() - } - - func testViewAround() { - declareEncodable(TestMessage.self, { TestMessage(decoder: $0) }) - declareEncodable(TestMedia.self, { TestMedia(decoder: $0) }) - - let ownerId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 1000) - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - let messageNamespaceCloud = TestMessageNamespace.Cloud.rawValue - let messageNamespaceLocal = TestMessageNamespace.Local.rawValue - - let basePath = "/tmp/postboxtest" - NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespaceCloud, messageNamespaceLocal]) - - (postbox.modify { state -> Void in - let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) - for i in 0 ..< 10 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i + 1)) - let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) - state.addMessages([message, message], medias: [testMedia]) - } - for i in 10 ..< 12 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespaceLocal, id: Int32(i + 1)) - let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) - state.addMessages([message, message], medias: [testMedia]) - } - return - }).start() - - var i = 1000 - postbox.aroundMessageViewForPeerId(otherId, id: MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i + 1)), count: 3).start(next: { next in - println(next) - }) - - (postbox.modify { state -> Void in - let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) - for i in 10 ..< 15 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i + 1)) - let message = TestMessage(id: messageId, authorId: ownerId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) - state.addMessages([message, message], medias: [testMedia]) - } - return - }).start() - - postbox._sync() - } - - func testPeerView() { - let view = MutablePeerView(count: 3, earlier: nil, entries: [], later: nil) - let messageNamespaceCloud = TestMessageNamespace.Cloud.rawValue - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - - var entries: [PeerViewEntry] = [] - - func print(entries: [PeerViewEntry]) { - var string = "" - string += "[" - var first = true - for entry in entries { - if first { - first = false - } else { - string += ", " - } - string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message.id.namespace):\(entry.message.id.id)—\(entry.message.timestamp))" - } - string += "]" - println("\(string)") - } - - func add(entry: PeerViewEntry) { - entries.append(entry) - entries.sort({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) - - view.addEntry(entry) - - println("\n\(view)") - print(entries) - } - - func remove(peerId: PeerId, context: MutablePeerView.RemoveContext? = nil) -> MutablePeerView.RemoveContext { - entries = entries.filter({ $0.peerId != peerId }) - return view.removeEntry(context, peerId: peerId) - } - - func fetchEarlier(entries: [PeerViewEntry])(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntry] { - var filtered: [PeerViewEntry] = [] - var i = entries.count - 1 - while i >= 0 && filtered.count < count { - if index == nil || PeerViewEntryIndex(entries[i]) < index! { - filtered.append(entries[i]) - } - i-- - } - - return filtered - } - - func fetchLater(entries: [PeerViewEntry])(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntry] { - var filtered: [PeerViewEntry] = [] - var i = 0 - while i < entries.count && filtered.count < count { - if index == nil || PeerViewEntryIndex(entries[i]) > index! { - filtered.append(entries[i]) - } - i++ - } - - return filtered - } - - func complete(context: MutablePeerView.RemoveContext) { - view.complete(context, fetchEarlier: fetchEarlier(entries), fetchLater: fetchLater(entries)) - } - - println("\(view)") - - for i in 1 ..< 10 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i * 2 * 100)) - let message = TestMessage(id: messageId, authorId: otherId, date: Int32(i * 2 * 100), text: "\(i)", referencedMediaIds: []) - - add(PeerViewEntry(peer: TestPeer(id: PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(i * 2))), message: message)) - } - - if true { - let i = 15 - let messageId = MessageId(peerId: otherId, namespace: messageNamespaceCloud, id: Int32(i * 100)) - let message = TestMessage(id: messageId, authorId: otherId, date: Int32(i * 100), text: "\(i)", referencedMediaIds: []) - - add(PeerViewEntry(peer: TestPeer(id: PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(i))), message: message)) - } - - if true { - var context = remove(PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(15))) - context = remove(PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(14)), context: context) - context = remove(PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(16)), context: context) - context = remove(PeerId(namespace: TestPeerNamespace.User.rawValue, id: Int32(18)), context: context) - println("\n\(view)") - print(entries) - complete(context) - println("\(view)") - } - } - - func testPeerViewTail() { - declareEncodable(TestMessage.self, { TestMessage(decoder: $0) }) - declareEncodable(TestMedia.self, { TestMedia(decoder: $0) }) - - let otherId = PeerId(namespace: TestPeerNamespace.User.rawValue, id: 2000) - let messageNamespace = TestMessageNamespace.Cloud.rawValue - - let basePath = "/tmp/postboxtest" - NSFileManager.defaultManager().removeItemAtPath(basePath, error: nil) - let postbox = Postbox(basePath: basePath, messageNamespaces: [messageNamespace]) - - (postbox.modify { state -> Void in - let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) - for i in 0 ..< 10 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) - let message = TestMessage(id: messageId, authorId: otherId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) - state.addMessages([message, message], medias: [testMedia]) - } - return - }).start() - - postbox.tailPeerView(3).start(next: { next in - println(next) - }) - - postbox.tailMessageViewForPeerId(otherId, count: 4).start(next: { next in - println(next) - }) - - (postbox.modify { state -> Void in - let testMedia = TestMedia(id: MediaId(namespace: TestMediaNamespace.Test.rawValue, id: 1)) - for i in 10 ..< 15 { - let messageId = MessageId(peerId: otherId, namespace: messageNamespace, id: Int32(i + 1)) - let message = TestMessage(id: messageId, authorId: otherId, date: Int32(i + 100), text: "\(i)", referencedMediaIds: [testMedia.id]) - state.addMessages([message, message], medias: [testMedia]) - } - return - }).start() - - postbox._sync() - } -} diff --git a/submodules/lmdb/libraries/liblmdb/.gitignore b/submodules/lmdb/libraries/liblmdb/.gitignore new file mode 100644 index 0000000000..f3277afe42 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/.gitignore @@ -0,0 +1,20 @@ +mtest +mtest[23456] +testdb +mdb_copy +mdb_stat +*.[ao] +*.so +*.exe +*[~#] +*.bak +*.orig +*.rej +*.gcov +*.gcda +*.gcno +core +core.* +valgrind.* +man/ +html/ diff --git a/submodules/lmdb/libraries/liblmdb/CHANGES b/submodules/lmdb/libraries/liblmdb/CHANGES new file mode 100644 index 0000000000..34049ba7ac --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/CHANGES @@ -0,0 +1,112 @@ +LMDB 0.9 Change Log + +LMDB 0.9.14 Release (2014/09/20) + Fix to support 64K page size (ITS#7713) + Fix to persist decreased as well as increased mapsizes (ITS#7789) + Fix cursor bug when deleting last node of a DUPSORT key + Fix mdb_env_info to return FIXEDMAP address + Fix ambiguous error code from writing to closed DBI (ITS#7825) + Fix mdb_copy copying past end of file (ITS#7886) + Fix cursor bugs from page_merge/rebalance + Fix to dirty fewer pages in deletes (mdb_page_loose()) + Fix mdb_dbi_open creating subDBs (ITS#7917) + Fix mdb_cursor_get(_DUP) with single value (ITS#7913) + Fix Windows compat issues in mtests (ITS#7879) + Add compacting variant of mdb_copy + Add BigEndian integer key compare code + Add mdb_dump/mdb_load utilities + +LMDB 0.9.13 Release (2014/06/18) + Fix mdb_page_alloc unlimited overflow page search + Documentation + Re-fix MDB_CURRENT doc (ITS#7793) + Fix MDB_GET_MULTIPLE/MDB_NEXT_MULTIPLE doc + +LMDB 0.9.12 Release (2014/06/13) + Fix MDB_GET_BOTH regression (ITS#7875,#7681) + Fix MDB_MULTIPLE writing multiple keys (ITS#7834) + Fix mdb_rebalance (ITS#7829) + Fix mdb_page_split (ITS#7815) + Fix md_entries count (ITS#7861,#7828,#7793) + Fix MDB_CURRENT (ITS#7793) + Fix possible crash on Windows DLL detach + Misc code cleanup + Documentation + mdb_cursor_put: cursor moves on error (ITS#7771) + + +LMDB 0.9.11 Release (2014/01/15) + Add mdb_env_set_assert() (ITS#7775) + Fix: invalidate txn on page allocation errors (ITS#7377) + Fix xcursor tracking in mdb_cursor_del0() (ITS#7771) + Fix corruption from deletes (ITS#7756) + Fix Windows/MSVC build issues + Raise safe limit of max MDB_MAXKEYSIZE + Misc code cleanup + Documentation + Remove spurious note about non-overlapping flags (ITS#7665) + +LMDB 0.9.10 Release (2013/11/12) + Add MDB_NOMEMINIT option + Fix mdb_page_split() again (ITS#7589) + Fix MDB_NORDAHEAD definition (ITS#7734) + Fix mdb_cursor_del() positioning (ITS#7733) + Partial fix for larger page sizes (ITS#7713) + Fix Windows64/MSVC build issues + +LMDB 0.9.9 Release (2013/10/24) + Add mdb_env_get_fd() + Add MDB_NORDAHEAD option + Add MDB_NOLOCK option + Avoid wasting space in mdb_page_split() (ITS#7589) + Fix mdb_page_merge() cursor fixup (ITS#7722) + Fix mdb_cursor_del() on last delete (ITS#7718) + Fix adding WRITEMAP on existing env (ITS#7715) + Fix nested txns (ITS#7515) + Fix mdb_env_copy() O_DIRECT bug (ITS#7682) + Fix mdb_cursor_set(SET_RANGE) return code (ITS#7681) + Fix mdb_rebalance() cursor fixup (ITS#7701) + Misc code cleanup + Documentation + Note that by default, readers need write access + + +LMDB 0.9.8 Release (2013/09/09) + Allow mdb_env_set_mapsize() on an open environment + Fix mdb_dbi_flags() (ITS#7672) + Fix mdb_page_unspill() in nested txns + Fix mdb_cursor_get(CURRENT|NEXT) after a delete + Fix mdb_cursor_get(DUP) to always return key (ITS#7671) + Fix mdb_cursor_del() to always advance to next item (ITS#7670) + Fix mdb_cursor_set(SET_RANGE) for tree with single page (ITS#7681) + Fix mdb_env_copy() retry open if O_DIRECT fails (ITS#7682) + Tweak mdb_page_spill() to be less aggressive + Documentation + Update caveats since mdb_reader_check() added in 0.9.7 + +LMDB 0.9.7 Release (2013/08/17) + Don't leave stale lockfile on failed RDONLY open (ITS#7664) + Fix mdb_page_split() ref beyond cursor depth + Fix read txn data race (ITS#7635) + Fix mdb_rebalance (ITS#7536, #7538) + Fix mdb_drop() (ITS#7561) + Misc DEBUG macro fixes + Add MDB_NOTLS envflag + Add mdb_env_copyfd() + Add mdb_txn_env() (ITS#7660) + Add mdb_dbi_flags() (ITS#7661) + Add mdb_env_get_maxkeysize() + Add mdb_env_reader_list()/mdb_env_reader_check() + Add mdb_page_spill/unspill, remove hard txn size limit + Use shorter names for semaphores (ITS#7615) + Build + Fix install target (ITS#7656) + Documentation + Misc updates for cursors, DB handles, data lifetime + +LMDB 0.9.6 Release (2013/02/25) + Many fixes/enhancements + +LMDB 0.9.5 Release (2012/11/30) + Renamed from libmdb to liblmdb + Many fixes/enhancements diff --git a/submodules/lmdb/libraries/liblmdb/COPYRIGHT b/submodules/lmdb/libraries/liblmdb/COPYRIGHT new file mode 100644 index 0000000000..e0eb484c7a --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/COPYRIGHT @@ -0,0 +1,20 @@ +Copyright 2011-2014 Howard Chu, Symas Corp. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted only as authorized by the OpenLDAP +Public License. + +A copy of this license is available in the file LICENSE in the +top-level directory of the distribution or, alternatively, at +. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Individual files and/or contributed packages may be copyright by +other parties and/or subject to additional restrictions. + +This work also contains materials derived from public sources. + +Additional information about OpenLDAP can be obtained at +. diff --git a/submodules/lmdb/libraries/liblmdb/Doxyfile b/submodules/lmdb/libraries/liblmdb/Doxyfile new file mode 100644 index 0000000000..92d17b09eb --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/Doxyfile @@ -0,0 +1,1631 @@ +# Doxyfile 1.7.1 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = LMDB + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, +# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English +# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, +# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak, +# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = YES + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this +# tag. The format is ext=language, where ext is a file extension, and language +# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, +# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make +# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C +# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions +# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen to replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +INLINE_GROUPED_CLASSES = YES +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. + +TYPEDEF_HIDES_STRUCT = YES + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penality. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will rougly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols + +SYMBOL_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespace are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen +# will list include files with double quotes in the documentation +# rather than with sharp brackets. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = NO + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen +# will sort the (brief and detailed) documentation of class members so that +# constructors and destructors are listed first. If set to NO (the default) +# the constructors will appear in the respective orders defined by +# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. +# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO +# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = NO + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. +# This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. The create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. +# You can optionally specify a file name after the option, if omitted +# DoxygenLayout.xml will be used as the name of the layout file. + +LAYOUT_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = lmdb.h midl.h mdb.c midl.c + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. +# If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. +# Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. +# The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. +# Otherwise they will link to the documentation. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = YES + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. +# Doxygen will adjust the colors in the stylesheet and background images +# according to this color. Hue is specified as an angle on a colorwheel, +# see http://en.wikipedia.org/wiki/Hue for more information. +# For instance the value 0 represents red, 60 is yellow, 120 is green, +# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. +# The allowed range is 0 to 359. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of +# the colors in the HTML output. For a value of 0 the output will use +# grayscales only. A value of 255 will produce the most vivid colors. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to +# the luminance component of the colors in the HTML output. Values below +# 100 gradually make the output lighter, whereas values above 100 make +# the output darker. The value divided by 100 is the actual gamma applied, +# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, +# and 100 does not change the gamma. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting +# this to NO can help when comparing the output of multiple runs. + +HTML_TIMESTAMP = YES + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated +# that can be used as input for Qt's qhelpgenerator to generate a +# Qt Compressed Help (.qch) of the generated HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to +# add. For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see +# +# Qt Help Project / Custom Filters. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's +# filter section matches. +# +# Qt Help Project / Filter Attributes. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files +# will be generated, which together with the HTML files, form an Eclipse help +# plugin. To install this plugin and make it available under the help contents +# menu in Eclipse, the contents of the directory containing the HTML and XML +# files needs to be copied into the plugins directory of eclipse. The name of +# the directory within the plugins directory should be the same as +# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before +# the help appears. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have +# this name. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to YES, a side panel will be generated +# containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). +# Windows users are probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, +# and Class Hierarchy pages using a tree view instead of an ordered list. + +USE_INLINE_TREES = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open +# links to external symbols imported via tag files in a separate window. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are +# not supported properly for IE 6.0, but are supported on all modern browsers. +# Note that when changing this option you need to delete any form_*.png files +# in the HTML output before the changes have effect. + +FORMULA_TRANSPARENT = YES + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box +# for the HTML output. The underlying search engine uses javascript +# and DHTML and should work on any modern browser. Note that when using +# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets +# (GENERATE_DOCSET) there is already a search function so this one should +# typically be disabled. For large projects the javascript based search engine +# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a PHP enabled web server instead of at the web client +# using Javascript. Doxygen will generate the search PHP script and index +# file to put on the web server. The advantage of the server +# based approach is that it scales better to large projects and allows +# full text search. The disadvances is that it is more difficult to setup +# and does not have live searching capabilities. + +SERVER_BASED_SEARCH = NO + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. +# Note that when enabling USE_PDFLATEX this option is only used for +# generating bitmaps for formulas in the HTML output, but not in the +# Makefile that is written to the output directory. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +# If LATEX_SOURCE_CODE is set to YES then doxygen will include +# source code with syntax highlighting in the LaTeX output. +# Note that which sources are shown also depends on other settings +# such as SOURCE_BROWSER. + +LATEX_SOURCE_CODE = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = YES + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. +# This is useful +# if you want to understand what is going on. +# On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = DEBUG=2 __GNUC__=1 + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = tooltag=./man1 + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is +# allowed to run in parallel. When set to 0 (the default) doxygen will +# base this on the number of processors available in the system. You can set it +# explicitly to a value larger than 0 to get control over the balance +# between CPU load and processing speed. + +DOT_NUM_THREADS = 0 + +# By default doxygen will write a font called FreeSans.ttf to the output +# directory and reference it in all dot files that doxygen generates. This +# font does not include all possible unicode characters however, so when you need +# these (or just want a differently looking font) you can specify the font name +# using DOT_FONTNAME. You need need to make sure dot is able to find the font, +# which can be done by putting it in a standard location or by setting the +# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory +# containing the font. + +DOT_FONTNAME = FreeSans.ttf + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the output directory to look for the +# FreeSans.ttf font (which doxygen will put there itself). If you specify a +# different font using DOT_FONTNAME you can set the path where dot +# can find it using this tag. + +DOT_FONTPATH = + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = YES + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES diff --git a/submodules/lmdb/libraries/liblmdb/LICENSE b/submodules/lmdb/libraries/liblmdb/LICENSE new file mode 100644 index 0000000000..05ad7571e4 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/LICENSE @@ -0,0 +1,47 @@ +The OpenLDAP Public License + Version 2.8, 17 August 2003 + +Redistribution and use of this software and associated documentation +("Software"), with or without modification, are permitted provided +that the following conditions are met: + +1. Redistributions in source form must retain copyright statements + and notices, + +2. Redistributions in binary form must reproduce applicable copyright + statements and notices, this list of conditions, and the following + disclaimer in the documentation and/or other materials provided + with the distribution, and + +3. Redistributions must contain a verbatim copy of this document. + +The OpenLDAP Foundation may revise this license from time to time. +Each revision is distinguished by a version number. You may use +this Software under terms of this license revision or under the +terms of any subsequent revision of the license. + +THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS +CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) +OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +The names of the authors and copyright holders must not be used in +advertising or otherwise to promote the sale, use or other dealing +in this Software without specific, written prior permission. Title +to copyright in this Software shall at all times remain with copyright +holders. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, +California, USA. All Rights Reserved. Permission to copy and +distribute verbatim copies of this document is granted. diff --git a/submodules/lmdb/libraries/liblmdb/Makefile b/submodules/lmdb/libraries/liblmdb/Makefile new file mode 100644 index 0000000000..25c1095466 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/Makefile @@ -0,0 +1,97 @@ +# Makefile for liblmdb (Lightning memory-mapped database library). + +######################################################################## +# Configuration. The compiler options must enable threaded compilation. +# +# Preprocessor macros (for CPPFLAGS) of interest... +# Note that the defaults should already be correct for most +# platforms; you should not need to change any of these. +# Read their descriptions in mdb.c if you do: +# +# - MDB_USE_POSIX_SEM +# - MDB_DSYNC +# - MDB_FDATASYNC +# - MDB_USE_PWRITEV +# +# There may be other macros in mdb.c of interest. You should +# read mdb.c before changing any of them. +# +CC = gcc +W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized +THREADS = -pthread +OPT = -O2 -g +CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS) +LDLIBS = +SOLIBS = +prefix = /usr/local + +######################################################################## + +IHDRS = lmdb.h +ILIBS = liblmdb.a liblmdb.so +IPROGS = mdb_stat mdb_copy mdb_dump mdb_load +IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 +PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 +all: $(ILIBS) $(PROGS) + +install: $(ILIBS) $(IPROGS) $(IHDRS) + for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done + for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done + for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done + for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done + +clean: + rm -rf $(PROGS) *.[ao] *.so *~ testdb + +test: all + mkdir testdb + ./mtest && ./mdb_stat testdb + +liblmdb.a: mdb.o midl.o + ar rs $@ mdb.o midl.o + +liblmdb.so: mdb.o midl.o +# $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS) + $(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.o midl.o $(SOLIBS) + +mdb_stat: mdb_stat.o liblmdb.a +mdb_copy: mdb_copy.o liblmdb.a +mdb_dump: mdb_dump.o liblmdb.a +mdb_load: mdb_load.o liblmdb.a +mtest: mtest.o liblmdb.a +mtest2: mtest2.o liblmdb.a +mtest3: mtest3.o liblmdb.a +mtest4: mtest4.o liblmdb.a +mtest5: mtest5.o liblmdb.a +mtest6: mtest6.o liblmdb.a + +mdb.o: mdb.c lmdb.h midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c + +midl.o: midl.c midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c + +%: %.o + $(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@ + +%.o: %.c lmdb.h + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< + +COV_FLAGS=-fprofile-arcs -ftest-coverage +COV_OBJS=xmdb.o xmidl.o + +coverage: xmtest + for i in mtest*.c [0-9]*.c; do j=`basename \$$i .c`; $(MAKE) $$j.o; \ + gcc -o x$$j $$j.o $(COV_OBJS) -pthread $(COV_FLAGS); \ + rm -rf testdb; mkdir testdb; ./x$$j; done + gcov xmdb.c + gcov xmidl.c + +xmtest: mtest.o xmdb.o xmidl.o + gcc -o xmtest mtest.o xmdb.o xmidl.o -pthread $(COV_FLAGS) + +xmdb.o: mdb.c lmdb.h midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c mdb.c -o $@ + +xmidl.o: midl.c midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c midl.c -o $@ diff --git a/submodules/lmdb/libraries/liblmdb/lmdb.h b/submodules/lmdb/libraries/liblmdb/lmdb.h new file mode 100644 index 0000000000..82a5410ab1 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/lmdb.h @@ -0,0 +1,1565 @@ +/** @file lmdb.h + * @brief Lightning memory-mapped database library + * + * @mainpage Lightning Memory-Mapped Database Manager (LMDB) + * + * @section intro_sec Introduction + * LMDB is a Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. The entire database is exposed + * in a memory map, and all data fetches return data directly + * from the mapped memory, so no malloc's or memcpy's occur during + * data fetches. As such, the library is extremely simple because it + * requires no page caching layer of its own, and it is extremely high + * performance and memory-efficient. It is also fully transactional with + * full ACID semantics, and when the memory map is read-only, the + * database integrity cannot be corrupted by stray pointer writes from + * application code. + * + * The library is fully thread-aware and supports concurrent read/write + * access from multiple processes and threads. Data pages use a copy-on- + * write strategy so no active data pages are ever overwritten, which + * also provides resistance to corruption and eliminates the need of any + * special recovery procedures after a system crash. Writes are fully + * serialized; only one write transaction may be active at a time, which + * guarantees that writers can never deadlock. The database structure is + * multi-versioned so readers run with no locks; writers cannot block + * readers, and readers don't block writers. + * + * Unlike other well-known database mechanisms which use either write-ahead + * transaction logs or append-only data writes, LMDB requires no maintenance + * during operation. Both write-ahead loggers and append-only databases + * require periodic checkpointing and/or compaction of their log or database + * files otherwise they grow without bound. LMDB tracks free pages within + * the database and re-uses them for new write operations, so the database + * size does not grow without bound in normal use. + * + * The memory map can be used as a read-only or read-write map. It is + * read-only by default as this provides total immunity to corruption. + * Using read-write mode offers much higher write performance, but adds + * the possibility for stray application writes thru pointers to silently + * corrupt the database. Of course if your application code is known to + * be bug-free (...) then this is not an issue. + * + * @section caveats_sec Caveats + * Troubleshooting the lock file, plus semaphores on BSD systems: + * + * - A broken lockfile can cause sync issues. + * Stale reader transactions left behind by an aborted program + * cause further writes to grow the database quickly, and + * stale locks can block further operation. + * + * Fix: Check for stale readers periodically, using the + * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. + * Catch stale + * locks with option MDB_ROBUST if supported (non-BSD). Or just + * make all programs using the database close it; the lockfile + * is always reset on first open of the environment. + * + * - On BSD systems or others configured with MDB_USE_POSIX_SEM, + * startup can fail due to semaphores owned by another userid. + * + * Fix: Open and close the database as the user which owns the + * semaphores (likely last user) or as root, while no other + * process is using the database. + * + * Restrictions/caveats (in addition to those listed for some functions): + * + * - Only the database owner should normally use the database on + * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. + * Multiple users can cause startup to fail later, as noted above. + * + * - There is normally no pure read-only mode, since readers need write + * access to locks and lock file. Exceptions: On read-only filesystems + * or with the #MDB_NOLOCK flag described under #mdb_env_open(). + * + * - By default, in versions before 0.9.10, unused portions of the data + * file might receive garbage data from memory freed by other code. + * (This does not happen when using the #MDB_WRITEMAP flag.) As of + * 0.9.10 the default behavior is to initialize such memory before + * writing to the data file. Since there may be a slight performance + * cost due to this initialization, applications may disable it using + * the #MDB_NOMEMINIT flag. Applications handling sensitive data + * which must not be written should not use this flag. This flag is + * irrelevant when using #MDB_WRITEMAP. + * + * - A thread can only use one transaction at a time, plus any child + * transactions. Each transaction belongs to one thread. See below. + * The #MDB_NOTLS flag changes this for read-only transactions. + * + * - Use an MDB_env* in the process which opened it, without fork()ing. + * + * - Do not have open an LMDB database twice in the same process at + * the same time. Not even from a plain open() call - close()ing it + * breaks flock() advisory locking. + * + * - Avoid long-lived transactions. Read transactions prevent + * reuse of pages freed by newer write transactions, thus the + * database can grow quickly. Write transactions prevent + * other write transactions, since writes are serialized. + * + * - Avoid suspending a process with active transactions. These + * would then be "long-lived" as above. Also read transactions + * suspended when writers commit could sometimes see wrong data. + * + * ...when several processes can use a database concurrently: + * + * - Avoid aborting a process with an active transaction. + * The transaction becomes "long-lived" as above until a check + * for stale readers is performed or the lockfile is reset, + * since the process may not remove it from the lockfile. + * Except write-transactions on Unix with MDB_ROBUST or on Windows. + * + * - If you do that anyway, do a periodic check for stale readers. Or + * close the environment once in a while, so the lockfile can get reset. + * + * - Do not use LMDB databases on remote filesystems, even between + * processes on the same host. This breaks flock() on some OSes, + * possibly memory map sync, and certainly sync between programs + * on different hosts. + * + * - Opening a database can fail if another process is opening or + * closing it at exactly the same time. + * + * @author Howard Chu, Symas Corporation. + * + * @copyright Copyright 2011-2014 Howard Chu, Symas Corp. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * @par Derived From: + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _LMDB_H_ +#define _LMDB_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Unix permissions for creating files, or dummy definition for Windows */ +#ifdef _MSC_VER +typedef int mdb_mode_t; +#else +typedef mode_t mdb_mode_t; +#endif + +/** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#ifdef _WIN32 +typedef void *mdb_filehandle_t; +#else +typedef int mdb_filehandle_t; +#endif + +/** @defgroup mdb LMDB API + * @{ + * @brief OpenLDAP Lightning Memory-Mapped Database Manager + */ +/** @defgroup Version Version Macros + * @{ + */ +/** Library major version */ +#define MDB_VERSION_MAJOR 0 +/** Library minor version */ +#define MDB_VERSION_MINOR 9 +/** Library patch version */ +#define MDB_VERSION_PATCH 14 + +/** Combine args a,b,c into a single integer for easy version comparisons */ +#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) + +/** The full library version as a single integer */ +#define MDB_VERSION_FULL \ + MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) + +/** The release date of this library version */ +#define MDB_VERSION_DATE "September 20, 2014" + +/** A stringifier for the version info */ +#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" + +/** A helper for the stringifier macro */ +#define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) + +/** The full library version as a C string */ +#define MDB_VERSION_STRING \ + MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) +/** @} */ + +/** @brief Opaque structure for a database environment. + * + * A DB environment supports multiple databases, all residing in the same + * shared-memory map. + */ +typedef struct MDB_env MDB_env; + +/** @brief Opaque structure for a transaction handle. + * + * All database operations require a transaction handle. Transactions may be + * read-only or read-write. + */ +typedef struct MDB_txn MDB_txn; + +/** @brief A handle for an individual database in the DB environment. */ +typedef unsigned int MDB_dbi; + +/** @brief Opaque structure for navigating through a database */ +typedef struct MDB_cursor MDB_cursor; + +/** @brief Generic structure used for passing keys and data in and out + * of the database. + * + * Values returned from the database are valid only until a subsequent + * update operation, or the end of the transaction. Do not modify or + * free them, they commonly point into the database itself. + * + * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. + * The same applies to data sizes in databases with the #MDB_DUPSORT flag. + * Other data items can in theory be from 0 to 0xffffffff bytes long. + */ +typedef struct MDB_val { + size_t mv_size; /**< size of the data item */ + void *mv_data; /**< address of the data item */ +} MDB_val; + +/** @brief A callback function used to compare two keys in a database */ +typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); + +/** @brief A callback function used to relocate a position-dependent data item + * in a fixed-address database. + * + * The \b newptr gives the item's desired address in + * the memory map, and \b oldptr gives its previous address. The item's actual + * data resides at the address in \b item. This callback is expected to walk + * through the fields of the record in \b item and modify any + * values based at the \b oldptr address to be relative to the \b newptr address. + * @param[in,out] item The item that is to be relocated. + * @param[in] oldptr The previous address. + * @param[in] newptr The new address to relocate to. + * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). + * @todo This feature is currently unimplemented. + */ +typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); + +/** @defgroup mdb_env Environment Flags + * @{ + */ + /** mmap at a fixed address (experimental) */ +#define MDB_FIXEDMAP 0x01 + /** no environment directory */ +#define MDB_NOSUBDIR 0x4000 + /** don't fsync after commit */ +#define MDB_NOSYNC 0x10000 + /** read only */ +#define MDB_RDONLY 0x20000 + /** don't fsync metapage after commit */ +#define MDB_NOMETASYNC 0x40000 + /** use writable mmap */ +#define MDB_WRITEMAP 0x80000 + /** use asynchronous msync when #MDB_WRITEMAP is used */ +#define MDB_MAPASYNC 0x100000 + /** tie reader locktable slots to #MDB_txn objects instead of to threads */ +#define MDB_NOTLS 0x200000 + /** don't do any locking, caller must manage their own locks */ +#define MDB_NOLOCK 0x400000 + /** don't do readahead (no effect on Windows) */ +#define MDB_NORDAHEAD 0x800000 + /** don't initialize malloc'd memory before writing to datafile */ +#define MDB_NOMEMINIT 0x1000000 + /** catch stale locks if supported (not on BSD, needs robust mutexes) */ +#define MDB_ROBUST 0x2000000 +/** @} */ + +/** @defgroup mdb_dbi_open Database Flags + * @{ + */ + /** use reverse string keys */ +#define MDB_REVERSEKEY 0x02 + /** use sorted duplicates */ +#define MDB_DUPSORT 0x04 + /** numeric keys in native byte order. + * The keys must all be of the same size. */ +#define MDB_INTEGERKEY 0x08 + /** with #MDB_DUPSORT, sorted dup items have fixed size */ +#define MDB_DUPFIXED 0x10 + /** with #MDB_DUPSORT, dups are numeric in native byte order */ +#define MDB_INTEGERDUP 0x20 + /** with #MDB_DUPSORT, use reverse string dups */ +#define MDB_REVERSEDUP 0x40 + /** create DB if not already existing */ +#define MDB_CREATE 0x40000 +/** @} */ + +/** @defgroup mdb_put Write Flags + * @{ + */ +/** For put: Don't write if the key already exists. */ +#define MDB_NOOVERWRITE 0x10 +/** Only for #MDB_DUPSORT
    + * For put: don't write if the key and data pair already exist.
    + * For mdb_cursor_del: remove all duplicate data items. + */ +#define MDB_NODUPDATA 0x20 +/** For mdb_cursor_put: overwrite the current key/data pair */ +#define MDB_CURRENT 0x40 +/** For put: Just reserve space for data, don't copy it. Return a + * pointer to the reserved space. + */ +#define MDB_RESERVE 0x10000 +/** Data is being appended, don't split full pages. */ +#define MDB_APPEND 0x20000 +/** Duplicate data is being appended, don't split full pages. */ +#define MDB_APPENDDUP 0x40000 +/** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ +#define MDB_MULTIPLE 0x80000 +/* @} */ + +/** @defgroup mdb_copy Copy Flags + * @{ + */ +/** Compacting copy: Omit free space from copy, and renumber all + * pages sequentially. + */ +#define MDB_CP_COMPACT 0x01 +/* @} */ + +/** @brief Cursor Get operations. + * + * This is the set of all operations for retrieving data + * using a cursor. + */ +typedef enum MDB_cursor_op { + MDB_FIRST, /**< Position at first key/data item */ + MDB_FIRST_DUP, /**< Position at first data item of current key. + Only for #MDB_DUPSORT */ + MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ + MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ + MDB_GET_CURRENT, /**< Return key/data at current cursor position */ + MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items + from current cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_LAST, /**< Position at last key/data item */ + MDB_LAST_DUP, /**< Position at last data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT, /**< Position at next data item */ + MDB_NEXT_DUP, /**< Position at next data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items + from next cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_NEXT_NODUP, /**< Position at first data item of next key */ + MDB_PREV, /**< Position at previous data item */ + MDB_PREV_DUP, /**< Position at previous data item of current key. + Only for #MDB_DUPSORT */ + MDB_PREV_NODUP, /**< Position at last data item of previous key */ + MDB_SET, /**< Position at specified key */ + MDB_SET_KEY, /**< Position at specified key, return key + data */ + MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */ +} MDB_cursor_op; + +/** @defgroup errors Return Codes + * + * BerkeleyDB uses -30800 to -30999, we'll go under them + * @{ + */ + /** Successful result */ +#define MDB_SUCCESS 0 + /** key/data pair already exists */ +#define MDB_KEYEXIST (-30799) + /** key/data pair not found (EOF) */ +#define MDB_NOTFOUND (-30798) + /** Requested page not found - this usually indicates corruption */ +#define MDB_PAGE_NOTFOUND (-30797) + /** Located page was wrong type */ +#define MDB_CORRUPTED (-30796) + /** Update of meta page failed or environment had fatal error */ +#define MDB_PANIC (-30795) + /** Environment version mismatch */ +#define MDB_VERSION_MISMATCH (-30794) + /** File is not a valid LMDB file */ +#define MDB_INVALID (-30793) + /** Environment mapsize reached */ +#define MDB_MAP_FULL (-30792) + /** Environment maxdbs reached */ +#define MDB_DBS_FULL (-30791) + /** Environment maxreaders reached */ +#define MDB_READERS_FULL (-30790) + /** Too many TLS keys in use - Windows only */ +#define MDB_TLS_FULL (-30789) + /** Txn has too many dirty pages */ +#define MDB_TXN_FULL (-30788) + /** Cursor stack too deep - internal error */ +#define MDB_CURSOR_FULL (-30787) + /** Page has not enough space - internal error */ +#define MDB_PAGE_FULL (-30786) + /** Database contents grew beyond environment mapsize */ +#define MDB_MAP_RESIZED (-30785) + /** MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed */ +#define MDB_INCOMPATIBLE (-30784) + /** Invalid reuse of reader locktable slot */ +#define MDB_BAD_RSLOT (-30783) + /** Transaction cannot recover - it must be aborted */ +#define MDB_BAD_TXN (-30782) + /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ +#define MDB_BAD_VALSIZE (-30781) + /** The specified DBI was changed unexpectedly */ +#define MDB_BAD_DBI (-30780) + /** The last defined error code */ +#define MDB_LAST_ERRCODE MDB_BAD_DBI +/** @} */ + +/** @brief Statistics for a database in the environment */ +typedef struct MDB_stat { + unsigned int ms_psize; /**< Size of a database page. + This is currently the same for all databases. */ + unsigned int ms_depth; /**< Depth (height) of the B-tree */ + size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ + size_t ms_leaf_pages; /**< Number of leaf pages */ + size_t ms_overflow_pages; /**< Number of overflow pages */ + size_t ms_entries; /**< Number of data items */ +} MDB_stat; + +/** @brief Information about the environment */ +typedef struct MDB_envinfo { + void *me_mapaddr; /**< Address of map, if fixed */ + size_t me_mapsize; /**< Size of the data memory map */ + size_t me_last_pgno; /**< ID of the last used page */ + size_t me_last_txnid; /**< ID of the last committed transaction */ + unsigned int me_maxreaders; /**< max reader slots in the environment */ + unsigned int me_numreaders; /**< max reader slots used in the environment */ +} MDB_envinfo; + + /** @brief Return the LMDB library version information. + * + * @param[out] major if non-NULL, the library major version number is copied here + * @param[out] minor if non-NULL, the library minor version number is copied here + * @param[out] patch if non-NULL, the library patch version number is copied here + * @retval "version string" The library version as a string + */ +char *mdb_version(int *major, int *minor, int *patch); + + /** @brief Return a string describing a given error code. + * + * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) + * function. If the error code is greater than or equal to 0, then the string + * returned by the system function strerror(3) is returned. If the error code + * is less than 0, an error string corresponding to the LMDB library error is + * returned. See @ref errors for a list of LMDB-specific error codes. + * @param[in] err The error code + * @retval "error message" The description of the error + */ +char *mdb_strerror(int err); + + /** @brief Create an LMDB environment handle. + * + * This function allocates memory for a #MDB_env structure. To release + * the allocated memory and discard the handle, call #mdb_env_close(). + * Before the handle may be used, it must be opened using #mdb_env_open(). + * Various other options may also need to be set before opening the handle, + * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), + * depending on usage requirements. + * @param[out] env The address where the new handle will be stored + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_create(MDB_env **env); + + /** @brief Open an environment handle. + * + * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] path The directory in which the database files reside. This + * directory must already exist and be writable. + * @param[in] flags Special options for this environment. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * Flags set by mdb_env_set_flags() are also used. + *
      + *
    • #MDB_FIXEDMAP + * use a fixed address for the mmap region. This flag must be specified + * when creating the environment, and is stored persistently in the environment. + * If successful, the memory map will always reside at the same virtual address + * and pointers used to reference data items in the database will be constant + * across multiple invocations. This option may not always work, depending on + * how the operating system has allocated memory to shared libraries and other uses. + * The feature is highly experimental. + *
    • #MDB_NOSUBDIR + * By default, LMDB creates its environment in a directory whose + * pathname is given in \b path, and creates its data and lock files + * under that directory. With this option, \b path is used as-is for + * the database main data file. The database lock file is the \b path + * with "-lock" appended. + *
    • #MDB_RDONLY + * Open the environment in read-only mode. No write operations will be + * allowed. LMDB will still modify the lock file - except on read-only + * filesystems, where LMDB does not use locks. + *
    • #MDB_ROBUST + * Initialize the lockfile to catch stale locks if robust mutexes + * are supported, so aborted processes will not block others. + * Ignored when another process has the environment open. Unsupported + * by liblmdb built with MDB_USE_POSIX_SEM (such as BSD systems). + * Enabled by default on Windows. Some locking slowdown on Unix. + *
    • #MDB_WRITEMAP + * Use a writeable memory map unless MDB_RDONLY is set. This is faster + * and uses fewer mallocs, but loses protection from application bugs + * like wild pointer writes and other bad updates into the database. + * Incompatible with nested transactions. + * Processes with and without MDB_WRITEMAP on the same environment do + * not cooperate well. + *
    • #MDB_NOMETASYNC + * Flush system buffers to disk only once per transaction, omit the + * metadata flush. Defer that until the system flushes files to disk, + * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization + * maintains database integrity, but a system crash may undo the last + * committed transaction. I.e. it preserves the ACI (atomicity, + * consistency, isolation) but not D (durability) database property. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
    • #MDB_NOSYNC + * Don't flush system buffers to disk when committing a transaction. + * This optimization means a system crash can corrupt the database or + * lose the last transactions if buffers are not yet flushed to disk. + * The risk is governed by how often the system flushes dirty buffers + * to disk and how often #mdb_env_sync() is called. However, if the + * filesystem preserves write order and the #MDB_WRITEMAP flag is not + * used, transactions exhibit ACI (atomicity, consistency, isolation) + * properties and only lose D (durability). I.e. database integrity + * is maintained, but a system crash may undo the final transactions. + * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no + * hint for when to write transactions to disk, unless #mdb_env_sync() + * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
    • #MDB_MAPASYNC + * When using #MDB_WRITEMAP, use asynchronous flushes to disk. + * As with #MDB_NOSYNC, a system crash can then corrupt the + * database or lose the last transactions. Calling #mdb_env_sync() + * ensures on-disk database integrity until next commit. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
    • #MDB_NOTLS + * Don't use Thread-Local Storage. Tie reader locktable slots to + * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps + * the slot reseved for the #MDB_txn object. A thread may use parallel + * read-only transactions. A read-only transaction may span threads if + * the user synchronizes its use. Applications that multiplex many + * user threads over individual OS threads need this option. Such an + * application must also serialize the write transactions in an OS + * thread, since LMDB's write locking is unaware of the user threads. + *
    • #MDB_NOLOCK + * Don't do any locking. If concurrent access is anticipated, the + * caller must manage all concurrency itself. For proper operation + * the caller must enforce single-writer semantics, and must ensure + * that no readers are using old transactions while a writer is + * active. The simplest approach is to use an exclusive lock so that + * no readers may be active at all when a writer begins. + *
    • #MDB_NORDAHEAD + * Turn off readahead. Most operating systems perform readahead on + * read requests by default. This option turns it off if the OS + * supports it. Turning it off may help random read performance + * when the DB is larger than RAM and system RAM is full. + * The option is not implemented on Windows. + *
    • #MDB_NOMEMINIT + * Don't initialize malloc'd memory before writing to unused spaces + * in the data file. By default, memory for pages written to the data + * file is obtained using malloc. While these pages may be reused in + * subsequent transactions, freshly malloc'd pages will be initialized + * to zeroes before use. This avoids persisting leftover data from other + * code (that used the heap and subsequently freed the memory) into the + * data file. Note that many other system libraries may allocate + * and free memory from the heap for arbitrary uses. E.g., stdio may + * use the heap for file I/O buffers. This initialization step has a + * modest performance cost so some applications may want to disable + * it using this flag. This option can be a problem for applications + * which handle sensitive data like passwords, and it makes memory + * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, + * which writes directly to the mmap instead of using malloc for pages. The + * initialization is also skipped if #MDB_RESERVE is used; the + * caller is expected to overwrite all of the memory that was + * reserved in that case. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
    + * @param[in] mode The UNIX permissions to set on created files. This parameter + * is ignored on Windows. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • #MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the + * version that created the database environment. + *
    • #MDB_INVALID - the environment file headers are corrupted. + *
    • ENOENT - the directory specified by the path parameter doesn't exist. + *
    • EACCES - the user didn't have permission to access the environment files. + *
    • EAGAIN - the environment was locked by another process. + *
    + */ +int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); + + /** @brief Copy an LMDB environment to the specified path. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy(MDB_env *env, const char *path); + + /** @brief Copy an LMDB environment to the specified file descriptor. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); + + /** @brief Copy an LMDB environment to the specified path, with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
      + *
    • #MDB_CP_COMPACT - Perform compaction while copying: omit free + * pages and sequentially renumber all pages in output. This option + * consumes more CPU and runs more slowly than the default. + *
    + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); + + /** @brief Copy an LMDB environment to the specified file descriptor, + * with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. See + * #mdb_env_copy2() for further details. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @param[in] flags Special options for this operation. + * See #mdb_env_copy2() for options. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); + + /** @brief Return statistics about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + */ +int mdb_env_stat(MDB_env *env, MDB_stat *stat); + + /** @brief Return information about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_envinfo structure + * where the information will be copied + */ +int mdb_env_info(MDB_env *env, MDB_envinfo *stat); + + /** @brief Flush the data buffers to disk. + * + * Data is always written to disk when #mdb_txn_commit() is called, + * but the operating system may keep it buffered. LMDB always flushes + * the OS buffers upon commit as well, unless the environment was + * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] force If non-zero, force a synchronous flush. Otherwise + * if the environment has the #MDB_NOSYNC flag set the flushes + * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    • EIO - an error occurred during synchronization. + *
    + */ +int mdb_env_sync(MDB_env *env, int force); + + /** @brief Close the environment and release the memory map. + * + * Only a single thread may call this function. All transactions, databases, + * and cursors must already be closed before calling this function. Attempts to + * use any such handles after calling this function will cause a SIGSEGV. + * The environment handle will be freed and must not be used again after this call. + * @param[in] env An environment handle returned by #mdb_env_create() + */ +void mdb_env_close(MDB_env *env); + + /** @brief Set environment flags. + * + * This may be used to set some flags in addition to those from + * #mdb_env_open(), or to unset these flags. If several threads + * change the flags at the same time, the result is undefined. + * Most flags cannot be changed after #mdb_env_open(). + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] flags The flags to change, bitwise OR'ed together + * @param[in] onoff A non-zero value sets the flags, zero clears them. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); + + /** @brief Get environment flags. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] flags The address of an integer to store the flags + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_env_get_flags(MDB_env *env, unsigned int *flags); + + /** @brief Return the path that was used in #mdb_env_open(). + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] path Address of a string pointer to contain the path. This + * is the actual string in the environment, not a copy. It should not be + * altered in any way. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_env_get_path(MDB_env *env, const char **path); + + /** @brief Return the filedescriptor for the given environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); + + /** @brief Set the size of the memory map to use for this environment. + * + * The size should be a multiple of the OS page size. The default is + * 10485760 bytes. The size of the memory map is also the maximum size + * of the database. The value should be chosen as large as possible, + * to accommodate future growth of the database. + * This function should be called after #mdb_env_create() and before #mdb_env_open(). + * It may be called at later times if no transactions are active in + * this process. Note that the library does not check for this condition, + * the caller must ensure it explicitly. + * + * The new size takes effect immediately for the current process but + * will not be persisted to any others until a write transaction has been + * committed by the current process. Also, only mapsize increases are + * persisted into the environment. + * + * If the mapsize is increased by another process, and data has grown + * beyond the range of the current mapsize, #mdb_txn_begin() will + * return #MDB_MAP_RESIZED. This function may be called with a size + * of zero to adopt the new size. + * + * Any attempt to set a size smaller than the space already consumed + * by the environment will be silently changed to the current size of the used space. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] size The size in bytes + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified, or the environment has + * an active write transaction. + *
    + */ +int mdb_env_set_mapsize(MDB_env *env, size_t size); + + /** @brief Set the maximum number of threads/reader slots for the environment. + * + * This defines the number of slots in the lock table that is used to track readers in the + * the environment. The default is 126. + * Starting a read-only transaction normally ties a lock table slot to the + * current thread until the environment closes or the thread exits. If + * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the + * MDB_txn object until it or the #MDB_env object is destroyed. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] readers The maximum number of reader lock table slots + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified, or the environment is already open. + *
    + */ +int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); + + /** @brief Get the maximum number of threads/reader slots for the environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] readers Address of an integer to store the number of readers + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); + + /** @brief Set the maximum number of named databases for the environment. + * + * This function is only needed if multiple databases will be used in the + * environment. Simpler applications that use the environment as a single + * unnamed database can ignore this option. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * + * Currently a moderate number of slots are cheap but a huge number gets + * expensive: 7-120 words per transaction, and every #mdb_dbi_open() + * does a linear search of the opened slots. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbs The maximum number of databases + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified, or the environment is already open. + *
    + */ +int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); + + /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. + * + * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. + * See @ref MDB_val. + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The maximum size of a key we can write + */ +int mdb_env_get_maxkeysize(MDB_env *env); + + /** @brief Set application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_userctx(MDB_env *env, void *ctx); + + /** @brief Get the application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The pointer set by #mdb_env_set_userctx(). + */ +void *mdb_env_get_userctx(MDB_env *env); + + /** @brief A callback function for most LMDB assert() failures, + * called before printing the message and aborting. + * + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] msg The assertion message, not including newline. + */ +typedef void MDB_assert_func(MDB_env *env, const char *msg); + + /** Set or reset the assert() callback of the environment. + * Disabled if liblmdb is buillt with NDEBUG. + * @note This hack should become obsolete as lmdb's error handling matures. + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] func An #MDB_assert_func function, or 0. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); + + /** @brief Create a transaction for use with the environment. + * + * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). + * @note A transaction and its cursors must only be used by a single + * thread, and a thread may only have a single transaction at a time. + * If #MDB_NOTLS is in use, this does not apply to read-only transactions. + * @note Cursors may not span transactions. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] parent If this parameter is non-NULL, the new transaction + * will be a nested transaction, with the transaction indicated by \b parent + * as its parent. Transactions may be nested to any level. A parent + * transaction and its cursors may not issue any other operations than + * mdb_txn_commit and mdb_txn_abort while it has active child transactions. + * @param[in] flags Special options for this transaction. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
      + *
    • #MDB_RDONLY + * This transaction will not perform any write operations. + *
    + * @param[out] txn Address where the new #MDB_txn handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • #MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + *
    • #MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's + * mapsize and this environment's map must be resized as well. + * See #mdb_env_set_mapsize(). + *
    • #MDB_READERS_FULL - a read-only transaction was requested and + * the reader lock table is full. See #mdb_env_set_maxreaders(). + *
    • ENOMEM - out of memory. + *
    + */ +int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); + + /** @brief Returns the transaction's #MDB_env + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +MDB_env *mdb_txn_env(MDB_txn *txn); + + /** @brief Commit all the operations of a transaction into the database. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    • ENOSPC - no more disk space. + *
    • EIO - a low-level I/O error occurred while writing. + *
    • ENOMEM - out of memory. + *
    + */ +int mdb_txn_commit(MDB_txn *txn); + + /** @brief Abandon all the operations of the transaction instead of saving them. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_abort(MDB_txn *txn); + + /** @brief Reset a read-only transaction. + * + * Abort the transaction like #mdb_txn_abort(), but keep the transaction + * handle. #mdb_txn_renew() may reuse the handle. This saves allocation + * overhead if the process will start a new read-only transaction soon, + * and also locking overhead if #MDB_NOTLS is in use. The reader table + * lock is released, but the table slot stays tied to its thread or + * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free + * its lock table slot if MDB_NOTLS is in use. + * Cursors opened within the transaction must not be used + * again after this call, except with #mdb_cursor_renew(). + * Reader locks generally don't interfere with writers, but they keep old + * versions of database pages allocated. Thus they prevent the old pages + * from being reused when writers commit new data, and so under heavy load + * the database size may grow much more rapidly than otherwise. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_reset(MDB_txn *txn); + + /** @brief Renew a read-only transaction. + * + * This acquires a new reader lock for a transaction handle that had been + * released by #mdb_txn_reset(). It must be called before a reset transaction + * may be used again. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • #MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_txn_renew(MDB_txn *txn); + +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_close(env,dbi) mdb_dbi_close(env,dbi) + + /** @brief Open a database in the environment. + * + * A database handle denotes the name and parameters of a database, + * independently of whether such a database exists. + * The database handle may be discarded by calling #mdb_dbi_close(). + * The old database handle is returned if the database was already open. + * The handle may only be closed once. + * The database handle will be private to the current transaction until + * the transaction is successfully committed. If the transaction is + * aborted the handle will be closed automatically. + * After a successful commit the + * handle will reside in the shared environment, and may be used + * by other transactions. This function must not be called from + * multiple concurrent transactions. A transaction that uses this function + * must finish (either commit or abort) before any other transaction may + * use this function. + * + * To use named databases (with name != NULL), #mdb_env_set_maxdbs() + * must be called before opening the environment. Database names + * are kept as keys in the unnamed database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] name The name of the database to open. If only a single + * database is needed in the environment, this value may be NULL. + * @param[in] flags Special options for this database. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
      + *
    • #MDB_REVERSEKEY + * Keys are strings to be compared in reverse order, from the end + * of the strings to the beginning. By default, Keys are treated as strings and + * compared from beginning to end. + *
    • #MDB_DUPSORT + * Duplicate keys may be used in the database. (Or, from another perspective, + * keys may have multiple data items, stored in sorted order.) By default + * keys must be unique and may have only a single data item. + *
    • #MDB_INTEGERKEY + * Keys are binary integers in native byte order. Setting this option + * requires all keys to be the same size, typically sizeof(int) + * or sizeof(size_t). + *
    • #MDB_DUPFIXED + * This flag may only be used in combination with #MDB_DUPSORT. This option + * tells the library that the data items for this database are all the same + * size, which allows further optimizations in storage and retrieval. When + * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE + * cursor operations may be used to retrieve multiple items at once. + *
    • #MDB_INTEGERDUP + * This option specifies that duplicate data items are also integers, and + * should be sorted as such. + *
    • #MDB_REVERSEDUP + * This option specifies that duplicate data items should be compared as + * strings in reverse order. + *
    • #MDB_CREATE + * Create the named database if it doesn't exist. This option is not + * allowed in a read-only transaction or a read-only environment. + *
    + * @param[out] dbi Address where the new #MDB_dbi handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • #MDB_NOTFOUND - the specified database doesn't exist in the environment + * and #MDB_CREATE was not specified. + *
    • #MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). + *
    + */ +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); + + /** @brief Retrieve statistics for a database. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); + + /** @brief Retrieve the DB flags for a database handle. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] flags Address where the flags will be returned. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); + + /** @brief Close a database handle. Normally unnecessary. Use with care: + * + * This call is not mutex protected. Handles should only be closed by + * a single thread, and only if no other threads are going to reference + * the database handle or one of its cursors any further. Do not close + * a handle if an existing transaction has modified its database. + * Doing so can cause misbehavior from database corruption to errors + * like MDB_BAD_VALSIZE (since the DB name is gone). + * + * Closing a database handle is not necessary, but lets #mdb_dbi_open() + * reuse the handle value. Usually it's better to set a bigger + * #mdb_env_set_maxdbs(), unless that value would be large. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); + + /** @brief Empty or delete+close a database. + * + * See #mdb_dbi_close() for restrictions about closing the DB handle. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] del 0 to empty the DB, 1 to delete it from the + * environment and close the DB handle. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); + + /** @brief Set a custom key comparison function for a database. + * + * The comparison function is called whenever it is necessary to compare a + * key specified by the application with a key currently stored in the database. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating + * before longer keys. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. + * + * This comparison function is called whenever it is necessary to compare a data + * item specified by the application with a data item currently stored in the database. + * This function only takes effect if the database was opened with the #MDB_DUPSORT + * flag. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating + * before longer items. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a relocation function for a #MDB_FIXEDMAP database. + * + * @todo The relocation function is called whenever it is necessary to move the data + * of an item to a different position in the database (e.g. through tree + * balancing operations, shifts as a result of adds or deletes, etc.). It is + * intended to allow address/position-dependent data items to be stored in + * a database in an environment opened with the #MDB_FIXEDMAP option. + * Currently the relocation feature is unimplemented and setting + * this function has no effect. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] rel A #MDB_rel_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); + + /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. + * + * See #mdb_set_relfunc and #MDB_rel_func for more details. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * It will be passed to the callback function set by #mdb_set_relfunc + * as its \b relctx parameter whenever the callback is invoked. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); + + /** @brief Get items from a database. + * + * This function retrieves key/data pairs from the database. The address + * and length of the data associated with the specified \b key are returned + * in the structure to which \b data refers. + * If the database supports duplicate keys (#MDB_DUPSORT) then the + * first data item for the key will be returned. Retrieval of other + * items requires the use of #mdb_cursor_get(). + * + * @note The memory pointed to by the returned values is owned by the + * database. The caller need not dispose of the memory, and may not + * modify it in any way. For values returned in a read-only transaction + * any modification attempts will cause a SIGSEGV. + * @note Values returned from the database are valid only until a + * subsequent update operation, or the end of the transaction. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to search for in the database + * @param[out] data The data corresponding to the key + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • #MDB_NOTFOUND - the key was not in the database. + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Store items into a database. + * + * This function stores key/data pairs in the database. The default behavior + * is to enter the new key/data pair, replacing any previously existing key + * if duplicates are disallowed, or adding a duplicate data item if + * duplicates are allowed (#MDB_DUPSORT). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to store in the database + * @param[in,out] data The data to store + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
      + *
    • #MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + *
    • #MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). The \b data + * parameter will be set to point to the existing item. + *
    • #MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later - before + * the next update operation or the transaction ends. This saves + * an extra memcpy if the data is being generated later. + * LMDB does nothing else with this memory, the caller is expected + * to modify all of the space requested. + *
    • #MDB_APPEND - append the given key/data pair to the end of the + * database. No key comparisons are performed. This option allows + * fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * data corruption. + *
    • #MDB_APPENDDUP - as above, but for sorted dup data. + *
    + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + *
    • #MDB_TXN_FULL - the transaction has too many dirty pages. + *
    • EACCES - an attempt was made to write in a read-only transaction. + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete items from a database. + * + * This function removes key/data pairs from the database. + * If the database does not support sorted duplicate data items + * (#MDB_DUPSORT) the data parameter is ignored. + * If the database supports sorted duplicates and the data parameter + * is NULL, all of the duplicate data items for the key will be + * deleted. Otherwise, if the data parameter is non-NULL + * only the matching data item will be deleted. + * This function will return #MDB_NOTFOUND if the specified key/data + * pair is not in the database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to delete from the database + * @param[in] data The data to delete + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EACCES - an attempt was made to write in a read-only transaction. + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Create a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * A cursor cannot be used when its database handle is closed. Nor + * when its transaction has ended, except with #mdb_cursor_renew(). + * It can be discarded with #mdb_cursor_close(). + * A cursor in a write-transaction can be closed before its transaction + * ends, and will otherwise be closed when its transaction ends. + * A cursor in a read-only transaction must be closed explicitly, before + * or after its transaction ends. It can be reused with + * #mdb_cursor_renew() before finally closing it. + * @note Earlier documentation said that cursors in every transaction + * were closed when the transaction committed or aborted. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] cursor Address where the new #MDB_cursor handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); + + /** @brief Close a cursor handle. + * + * The cursor handle will be freed and must not be used again after this call. + * Its transaction must still be live if it is a write-transaction. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +void mdb_cursor_close(MDB_cursor *cursor); + + /** @brief Renew a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * Cursors that are only used in read-only + * transactions may be re-used, to avoid unnecessary malloc/free overhead. + * The cursor may be associated with a new read-only transaction, and + * referencing the same database handle as it was created with. + * This may be done whether the previous transaction is live or dead. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); + + /** @brief Return the cursor's transaction handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); + + /** @brief Return the cursor's database handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); + + /** @brief Retrieve by cursor. + * + * This function retrieves key/data pairs from the database. The address and length + * of the key are returned in the object to which \b key refers (except for the + * case of the #MDB_SET option, in which the \b key object is unchanged), and + * the address and length of the data are returned in the object to which \b data + * refers. + * See #mdb_get() for restrictions on using the output values. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in,out] key The key for a retrieved item + * @param[in,out] data The data of a retrieved item + * @param[in] op A cursor operation #MDB_cursor_op + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • #MDB_NOTFOUND - no matching key found. + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + MDB_cursor_op op); + + /** @brief Store by cursor. + * + * This function stores key/data pairs into the database. + * The cursor is positioned at the new item, or on failure usually near it. + * @note Earlier documentation incorrectly said errors would leave the + * state of the cursor unchanged. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] key The key operated on. + * @param[in] data The data operated on. + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + *
      + *
    • #MDB_CURRENT - replace the item at the current cursor position. + * The \b key parameter must still be provided, and must match it. + * If using sorted duplicates (#MDB_DUPSORT) the data item must still + * sort into the same place. This is intended to be used when the + * new data is the same size as the old. Otherwise it will simply + * perform a delete of the old record followed by an insert. + *
    • #MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + *
    • #MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). + *
    • #MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later. This saves + * an extra memcpy if the data is being generated later. + *
    • #MDB_APPEND - append the given key/data pair to the end of the + * database. No key comparisons are performed. This option allows + * fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * data corruption. + *
    • #MDB_APPENDDUP - as above, but for sorted dup data. + *
    • #MDB_MULTIPLE - store multiple contiguous data elements in a + * single request. This flag may only be specified if the database + * was opened with #MDB_DUPFIXED. The \b data argument must be an + * array of two MDB_vals. The mv_size of the first MDB_val must be + * the size of a single data element. The mv_data of the first MDB_val + * must point to the beginning of the array of contiguous data elements. + * The mv_size of the second MDB_val must be the count of the number + * of data elements to store. On return this field will be set to + * the count of the number of elements actually written. The mv_data + * of the second MDB_val is unused. + *
    + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + *
    • #MDB_TXN_FULL - the transaction has too many dirty pages. + *
    • EACCES - an attempt was made to modify a read-only database. + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete current key/data pair + * + * This function deletes the key/data pair to which the cursor refers. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + *
      + *
    • #MDB_NODUPDATA - delete all of the data items for the current key. + * This flag may only be specified if the database was opened with #MDB_DUPSORT. + *
    + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EACCES - an attempt was made to modify a read-only database. + *
    • EINVAL - an invalid parameter was specified. + *
    + */ +int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); + + /** @brief Return count of duplicates for current key. + * + * This call is only valid on databases that support sorted duplicate + * data items #MDB_DUPSORT. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[out] countp Address where the count will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
      + *
    • EINVAL - cursor is not initialized, or an invalid parameter was specified. + *
    + */ +int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two data items were keys in the + * specified database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two items were data items of + * the specified database. The database must have the #MDB_DUPSORT flag. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief A callback function used to print a message from the library. + * + * @param[in] msg The string to be printed. + * @param[in] ctx An arbitrary context pointer for the callback. + * @return < 0 on failure, >= 0 on success. + */ +typedef int (MDB_msg_func)(const char *msg, void *ctx); + + /** @brief Dump the entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] func A #MDB_msg_func function + * @param[in] ctx Anything the message function needs + * @return < 0 on failure, >= 0 on success. + */ +int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); + + /** @brief Check for stale entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] dead Number of stale slots that were cleared + * @return 0 on success, non-zero on failure. + */ +int mdb_reader_check(MDB_env *env, int *dead); +/** @} */ + +#ifdef __cplusplus +} +#endif +/** @page tools LMDB Command Line Tools + The following describes the command line tools that are available for LMDB. + \li \ref mdb_copy_1 + \li \ref mdb_dump_1 + \li \ref mdb_load_1 + \li \ref mdb_stat_1 +*/ + +#endif /* _LMDB_H_ */ diff --git a/submodules/lmdb/libraries/liblmdb/mdb.c b/submodules/lmdb/libraries/liblmdb/mdb.c new file mode 100644 index 0000000000..6179b57868 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mdb.c @@ -0,0 +1,9519 @@ +/** @file mdb.c + * @brief Lightning memory-mapped database library + * + * A Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. + */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#ifdef _WIN32 +#include +#include +/** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it + * as int64 which is wrong. MSVC doesn't define it at all, so just + * don't use it. + */ +#define MDB_PID_T int +#define MDB_THR_T DWORD +#include +#include +#ifdef __GNUC__ +# include +#else +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# define BYTE_ORDER LITTLE_ENDIAN +# ifndef SSIZE_MAX +# define SSIZE_MAX INT_MAX +# endif +#endif +#else +#include +#include +#define MDB_PID_T pid_t +#define MDB_THR_T pthread_t +#include +#include +#include +#ifdef HAVE_SYS_FILE_H +#include +#endif +#include +#endif + +#if defined(__mips) && defined(__linux) +/* MIPS has cache coherency issues, requires explicit cache control */ +#include +extern int cacheflush(char *addr, int nbytes, int cache); +#define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache) +#else +#define CACHEFLUSH(addr, bytes, cache) +#endif + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__sun) || defined(ANDROID) +/* Most platforms have posix_memalign, older may only have memalign */ +#define HAVE_MEMALIGN 1 +#include +#endif + +#if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) +#include +#include /* defines BYTE_ORDER on HPUX and Solaris */ +#endif + +#if defined(__APPLE__) || defined (BSD) +# define MDB_USE_POSIX_SEM 1 +# define MDB_FDATASYNC fsync +#elif defined(ANDROID) +# define MDB_FDATASYNC fsync +#endif + +#ifndef _WIN32 +#include +#ifdef MDB_USE_POSIX_SEM +# define MDB_USE_HASH 1 +#include +#endif +#endif + +#ifdef USE_VALGRIND +#include +#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) +#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) +#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) +#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) +#else +#define VGMEMP_CREATE(h,r,z) +#define VGMEMP_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) +#define VGMEMP_DESTROY(h) +#define VGMEMP_DEFINED(a,s) +#endif + +#ifndef BYTE_ORDER +# if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) +/* Solaris just defines one or the other */ +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# ifdef _LITTLE_ENDIAN +# define BYTE_ORDER LITTLE_ENDIAN +# else +# define BYTE_ORDER BIG_ENDIAN +# endif +# else +# define BYTE_ORDER __BYTE_ORDER +# endif +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN __LITTLE_ENDIAN +#endif +#ifndef BIG_ENDIAN +#define BIG_ENDIAN __BIG_ENDIAN +#endif + +#if defined(__i386) || defined(__x86_64) || defined(_M_IX86) +#define MISALIGNED_OK 1 +#endif + +#include "lmdb.h" +#include "midl.h" + +#if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) +# error "Unknown or unsupported endianness (BYTE_ORDER)" +#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +# error "Two's complement, reasonably sized integer types, please" +#endif + +#ifdef __GNUC__ +/** Put infrequently used env functions in separate section */ +# ifdef __APPLE__ +# define ESECT __attribute__ ((section("__TEXT,text_env"))) +# else +# define ESECT __attribute__ ((section("text_env"))) +# endif +#else +#define ESECT +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup compat Compatibility Macros + * A bunch of macros to minimize the amount of platform-specific ifdefs + * needed throughout the rest of the code. When the features this library + * needs are similar enough to POSIX to be hidden in a one-or-two line + * replacement, this macro approach is used. + * @{ + */ + + /** Features under development */ +#ifndef MDB_DEVEL +#define MDB_DEVEL 0 +#endif + +#if MDB_DEVEL && (defined(_WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_POSIX_SEM))) +#define MDB_ROBUST_SUPPORTED 1 +#endif + + /** Wrapper around __func__, which is a C99 feature */ +#if __STDC_VERSION__ >= 199901L +# define mdb_func_ __func__ +#elif __GNUC__ >= 2 || _MSC_VER >= 1300 +# define mdb_func_ __FUNCTION__ +#else +/* If a debug message says (), update the #if statements above */ +# define mdb_func_ "" +#endif + +#ifdef _WIN32 +#define MDB_USE_HASH 1 +#define MDB_PIDLOCK 0 +#define THREAD_RET DWORD +#define pthread_t HANDLE +#define pthread_mutex_t HANDLE +#define pthread_cond_t HANDLE +typedef HANDLE mdb_mutex_t; +#define pthread_key_t DWORD +#define pthread_self() GetCurrentThreadId() +#define pthread_key_create(x,y) \ + ((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0) +#define pthread_key_delete(x) TlsFree(x) +#define pthread_getspecific(x) TlsGetValue(x) +#define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) +#define pthread_mutex_consistent(mutex) 0 +#define pthread_mutex_unlock(x) ReleaseMutex(*x) +#define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) +#define pthread_cond_signal(x) SetEvent(*x) +#define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) +#define THREAD_CREATE(thr,start,arg) thr=CreateThread(NULL,0,start,arg,0,NULL) +#define THREAD_FINISH(thr) WaitForSingleObject(thr, INFINITE) +#define MDB_MUTEX(env, rw) ((env)->me_##rw##mutex) +#define LOCK_MUTEX0(mutex) WaitForSingleObject(mutex, INFINITE) +#define UNLOCK_MUTEX(mutex) ReleaseMutex(mutex) +#define getpid() GetCurrentProcessId() +#define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) +#define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) +#define ErrCode() GetLastError() +#define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;} +#define close(fd) (CloseHandle(fd) ? 0 : -1) +#define munmap(ptr,len) UnmapViewOfFile(ptr) +#ifdef PROCESS_QUERY_LIMITED_INFORMATION +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION PROCESS_QUERY_LIMITED_INFORMATION +#else +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 +#endif +#define Z "I" +#else +#define THREAD_RET void * +#define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) +#define THREAD_FINISH(thr) pthread_join(thr,NULL) +#define Z "z" /**< printf format modifier for size_t */ + + /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ +#define MDB_PIDLOCK 1 + +#ifdef MDB_USE_POSIX_SEM + +typedef sem_t *mdb_mutex_t; +#define MDB_MUTEX(env, rw) ((env)->me_##rw##mutex) +#define LOCK_MUTEX0(mutex) mdb_sem_wait(mutex) +#define UNLOCK_MUTEX(mutex) sem_post(mutex) + +static int +mdb_sem_wait(sem_t *sem) +{ + int rc; + while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ; + return rc; +} + +#else + /** Pointer/HANDLE type of shared mutex/semaphore. + */ +typedef pthread_mutex_t *mdb_mutex_t; + /** Mutex for the reader table (rw = r) or write transaction (rw = w). + */ +#define MDB_MUTEX(env, rw) (&(env)->me_txns->mti_##rw##mutex) + /** Lock the reader or writer mutex. + * Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX(). + */ +#define LOCK_MUTEX0(mutex) pthread_mutex_lock(mutex) + /** Unlock the reader or writer mutex. + */ +#define UNLOCK_MUTEX(mutex) pthread_mutex_unlock(mutex) +#endif /* MDB_USE_POSIX_SEM */ + + /** Get the error code for the last failed system function. + */ +#define ErrCode() errno + + /** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#define HANDLE int + + /** A value for an invalid file handle. + * Mainly used to initialize file variables and signify that they are + * unused. + */ +#define INVALID_HANDLE_VALUE (-1) + + /** Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. + */ +#define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) +#endif + +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) +#define MNAME_LEN 32 +#else +#define MNAME_LEN (sizeof(pthread_mutex_t)) +#endif + +/** @} */ + +#ifdef MDB_ROBUST_SUPPORTED + /** Lock mutex, handle any error, set rc = result. + * Return 0 on success, nonzero (not rc) on error. + */ +#define LOCK_MUTEX(rc, env, mutex) \ + (((rc) = LOCK_MUTEX0(mutex)) && \ + ((rc) = mdb_mutex_failed(env, mutex, rc))) +static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t mutex, int rc); +#else +#define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex)) +#define mdb_mutex_failed(env, mutex, rc) (rc) +#endif + +#ifndef _WIN32 +/** A flag for opening a file and requesting synchronous data writes. + * This is only used when writing a meta page. It's not strictly needed; + * we could just do a normal write and then immediately perform a flush. + * But if this flag is available it saves us an extra system call. + * + * @note If O_DSYNC is undefined but exists in /usr/include, + * preferably set some compiler flag to get the definition. + * Otherwise compile with the less efficient -DMDB_DSYNC=O_SYNC. + */ +#ifndef MDB_DSYNC +# define MDB_DSYNC O_DSYNC +#endif +#endif + +/** Function for flushing the data of a file. Define this to fsync + * if fdatasync() is not supported. + */ +#ifndef MDB_FDATASYNC +# define MDB_FDATASYNC fdatasync +#endif + +#ifndef MDB_MSYNC +# define MDB_MSYNC(addr,len,flags) msync(addr,len,flags) +#endif + +#ifndef MS_SYNC +#define MS_SYNC 1 +#endif + +#ifndef MS_ASYNC +#define MS_ASYNC 0 +#endif + + /** A page number in the database. + * Note that 64 bit page numbers are overkill, since pages themselves + * already represent 12-13 bits of addressable memory, and the OS will + * always limit applications to a maximum of 63 bits of address space. + * + * @note In the #MDB_node structure, we only store 48 bits of this value, + * which thus limits us to only 60 bits of addressable data. + */ +typedef MDB_ID pgno_t; + + /** A transaction ID. + * See struct MDB_txn.mt_txnid for details. + */ +typedef MDB_ID txnid_t; + +/** @defgroup debug Debug Macros + * @{ + */ +#ifndef MDB_DEBUG + /** Enable debug output. Needs variable argument macros (a C99 feature). + * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs + * read from and written to the database (used for free space management). + */ +#define MDB_DEBUG 0 +#endif + +#if MDB_DEBUG +static int mdb_debug; +static txnid_t mdb_debug_start; + + /** Print a debug message with printf formatting. + * Requires double parenthesis around 2 or more args. + */ +# define DPRINTF(args) ((void) ((mdb_debug) && DPRINTF0 args)) +# define DPRINTF0(fmt, ...) \ + fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__) +#else +# define DPRINTF(args) ((void) 0) +#endif + /** Print a debug string. + * The string is printed literally, with no format processing. + */ +#define DPUTS(arg) DPRINTF(("%s", arg)) + /** Debuging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +/** @} */ + + /** @brief The maximum size of a database page. + * + * It is 32k or 64k, since value-PAGEBASE must fit in + * #MDB_page.%mp_upper. + * + * LMDB will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. + */ +#define MAX_PAGESIZE (PAGEBASE ? 0x10000 : 0x8000) + + /** The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. + */ +#define MDB_MINKEYS 2 + + /** A stamp that identifies a file as an LMDB file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. + */ +#define MDB_MAGIC 0xBEEFC0DE + + /** The version number for a database's datafile format. */ +#define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 1) + /** The version number for a database's lockfile format. */ +#define MDB_LOCK_VERSION ((MDB_DEVEL) ? 999 : 1) + + /** @brief The max size of a key we can write, or 0 for dynamic max. + * + * Define this as 0 to compute the max from the page size. 511 + * is default for backwards compat: liblmdb <= 0.9.10 can break + * when modifying a DB with keys/dupsort data bigger than its max. + * #MDB_DEVEL sets the default to 0. + * + * Data items in an #MDB_DUPSORT database are also limited to + * this size, since they're actually keys of a sub-DB. Keys and + * #MDB_DUPSORT data items must fit on a node in a regular page. + */ +#ifndef MDB_MAXKEYSIZE +#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) +#endif + + /** The maximum size of a key we can write to the environment. */ +#if MDB_MAXKEYSIZE +#define ENV_MAXKEY(env) (MDB_MAXKEYSIZE) +#else +#define ENV_MAXKEY(env) ((env)->me_maxkey) +#endif + + /** @brief The maximum size of a data item. + * + * We only store a 32 bit value for node sizes. + */ +#define MAXDATASIZE 0xffffffffUL + +#if MDB_DEBUG + /** Key size which fits in a #DKBUF. + * @ingroup debug + */ +#define DKBUF_MAXKEYSIZE ((MDB_MAXKEYSIZE) > 0 ? (MDB_MAXKEYSIZE) : 511) + /** A key buffer. + * @ingroup debug + * This is used for printing a hex dump of a key's contents. + */ +#define DKBUF char kbuf[DKBUF_MAXKEYSIZE*2+1] + /** Display a key in hex. + * @ingroup debug + * Invoke a function to display a key in hex. + */ +#define DKEY(x) mdb_dkey(x, kbuf) +#else +#define DKBUF +#define DKEY(x) 0 +#endif + + /** An invalid page number. + * Mainly used to denote an empty tree. + */ +#define P_INVALID (~(pgno_t)0) + + /** Test if the flags \b f are set in a flag word \b w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + + /** Round \b n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + + /** Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. + */ +typedef uint16_t indx_t; + + /** Default size of memory map. + * This is certainly too small for any actual applications. Apps should always set + * the size explicitly using #mdb_env_set_mapsize(). + */ +#define DEFAULT_MAPSIZE 1048576 + +/** @defgroup readers Reader Lock Table + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent read + * transactions started by the same thread need no further locking to proceed. + * + * If #MDB_NOTLS is set, the slot address is not saved in thread-specific data. + * + * No reader table is used if the database is on a read-only filesystem, or + * if #MDB_NOLOCK is set. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. + * @{ + */ + /** Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. 126 readers plus a + * couple mutexes fit exactly into 8KB on my development machine. + * Applications should set the table size using #mdb_env_set_maxreaders(). + */ +#define DEFAULT_READERS 126 + + /** The size of a CPU cache line in bytes. We want our lock structures + * aligned to this size to avoid false cache line sharing in the + * lock table. + * This value works for most CPUs. For Itanium this should be 128. + */ +#ifndef CACHELINE +#define CACHELINE 64 +#endif + + /** The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * @note We currently don't check for stale records. We simply re-init + * the table when we know that we're the only process opening the + * lock file. + */ +typedef struct MDB_rxbody { + /** Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. + */ + txnid_t mrb_txnid; + /** The process ID of the process owning this reader txn. */ + MDB_PID_T mrb_pid; + /** The thread ID of the thread owning this txn. */ + MDB_THR_T mrb_tid; +} MDB_rxbody; + + /** The actual reader record, with cacheline padding. */ +typedef struct MDB_reader { + union { + MDB_rxbody mrx; + /** shorthand for mrb_txnid */ +#define mr_txnid mru.mrx.mrb_txnid +#define mr_pid mru.mrx.mrb_pid +#define mr_tid mru.mrx.mrb_tid + /** cache line alignment */ + char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mru; +} MDB_reader; + + /** The header for the reader table. + * The table resides in a memory-mapped file. (This is a different file + * than is used for the main database.) + * + * For POSIX the actual mutexes reside in the shared memory of this + * mapped file. On Windows, mutexes are named objects allocated by the + * kernel; we store the mutex names in this mapped file so that other + * processes can grab them. This same approach is also used on + * MacOSX/Darwin (using named semaphores) since MacOSX doesn't support + * process-shared POSIX mutexes. For these cases where a named object + * is used, the object name is derived from a 64 bit FNV hash of the + * environment pathname. As such, naming collisions are extremely + * unlikely. If a collision occurs, the results are unpredictable. + */ +typedef struct MDB_txbody { + /** Stamp identifying this as an LMDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mtb_magic; + /** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */ + uint32_t mtb_format; +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mtb_rmname[MNAME_LEN]; +#else + /** Mutex protecting access to this table. + * This is the #MDB_MUTEX(env,r) reader table lock. + */ + pthread_mutex_t mtb_rmutex; +#endif + /** The ID of the last transaction committed to the database. + * This is recorded here only for convenience; the value can always + * be determined by reading the main database meta pages. + */ + txnid_t mtb_txnid; + /** The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. + */ + unsigned mtb_numreaders; + /** Flags which the lock file was initialized with. */ + unsigned mtb_flags; +} MDB_txbody; + + /** The actual reader table definition. */ +typedef struct MDB_txninfo { + union { + MDB_txbody mtb; +#define mti_magic mt1.mtb.mtb_magic +#define mti_format mt1.mtb.mtb_format +#define mti_rmutex mt1.mtb.mtb_rmutex +#define mti_rmname mt1.mtb.mtb_rmname +#define mti_txnid mt1.mtb.mtb_txnid +#define mti_numreaders mt1.mtb.mtb_numreaders +#define mti_flags mt1.mtb.mtb_flags + char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mt1; + union { +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mt2_wmname[MNAME_LEN]; +#define mti_wmname mt2.mt2_wmname +#else + pthread_mutex_t mt2_wmutex; +#define mti_wmutex mt2.mt2_wmutex +#endif + char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)]; + } mt2; + MDB_reader mti_readers[1]; +} MDB_txninfo; + + /** Lockfile format signature: version, features and field layout */ +#define MDB_LOCK_FORMAT \ + ((uint32_t) \ + ((MDB_LOCK_VERSION) \ + /* Flags which describe functionality */ \ + + (((MDB_PIDLOCK) != 0) << 16))) +/** @} */ + +/** Common header for all page types. + * Overflow records occupy a number of contiguous pages with no + * headers on any page after the first. + */ +typedef struct MDB_page { +#define mp_pgno mp_p.p_pgno +#define mp_next mp_p.p_next + union { + pgno_t p_pgno; /**< page number */ + struct MDB_page *p_next; /**< for in-memory list of freed pages */ + } mp_p; + uint16_t mp_pad; +/** @defgroup mdb_page Page Flags + * @ingroup internal + * Flags for the page headers. + * @{ + */ +#define P_BRANCH 0x01 /**< branch page */ +#define P_LEAF 0x02 /**< leaf page */ +#define P_OVERFLOW 0x04 /**< overflow page */ +#define P_META 0x08 /**< meta page */ +#define P_DIRTY 0x10 /**< dirty page, also set for #P_SUBP pages */ +#define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ +#define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /**< page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /**< leave this page alone during spill */ +/** @} */ + uint16_t mp_flags; /**< @ref mdb_page */ +#define mp_lower mp_pb.pb.pb_lower +#define mp_upper mp_pb.pb.pb_upper +#define mp_pages mp_pb.pb_pages + union { + struct { + indx_t pb_lower; /**< lower bound of free space */ + indx_t pb_upper; /**< upper bound of free space */ + } pb; + uint32_t pb_pages; /**< number of overflow pages */ + } mp_pb; + indx_t mp_ptrs[1]; /**< dynamic size */ +} MDB_page; + + /** Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned) offsetof(MDB_page, mp_ptrs)) + + /** Address of first usable data byte in a page, after the header */ +#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ)) + + /** ITS#7713, change PAGEBASE to handle 65536 byte pages */ +#define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0) + + /** Number of nodes on a page */ +#define NUMKEYS(p) (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1) + + /** The amount of space remaining in the page */ +#define SIZELEFT(p) (indx_t)((p)->mp_upper - (p)->mp_lower) + + /** The percentage of space used in the page, in tenths of a percent. */ +#define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \ + ((env)->me_psize - PAGEHDRSZ)) + /** The minimum page fill factor, in tenths of a percent. + * Pages emptier than this are candidates for merging. + */ +#define FILL_THRESHOLD 250 + + /** Test if a page is a leaf page */ +#define IS_LEAF(p) F_ISSET((p)->mp_flags, P_LEAF) + /** Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) F_ISSET((p)->mp_flags, P_LEAF2) + /** Test if a page is a branch page */ +#define IS_BRANCH(p) F_ISSET((p)->mp_flags, P_BRANCH) + /** Test if a page is an overflow page */ +#define IS_OVERFLOW(p) F_ISSET((p)->mp_flags, P_OVERFLOW) + /** Test if a page is a sub page */ +#define IS_SUBP(p) F_ISSET((p)->mp_flags, P_SUBP) + + /** The number of overflow pages needed to store the given size. */ +#define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) + + /** Link in #MDB_txn.%mt_loose_pgs list */ +#define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) + + /** Header for a single key/data pair within a page. + * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. + * We guarantee 2-byte alignment for 'MDB_node's. + */ +typedef struct MDB_node { + /** lo and hi are used for data size on leaf nodes and for + * child pgno on branch nodes. On 64 bit platforms, flags + * is also used for pgno. (Branch nodes have no flags). + * They are in host byte order in case that lets some + * accesses be optimized into a 32-bit word access. + */ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short mn_lo, mn_hi; /**< part of data size or pgno */ +#else + unsigned short mn_hi, mn_lo; +#endif +/** @defgroup mdb_node Node Flags + * @ingroup internal + * Flags for node headers. + * @{ + */ +#define F_BIGDATA 0x01 /**< data put on overflow page */ +#define F_SUBDATA 0x02 /**< data is a sub-database */ +#define F_DUPDATA 0x04 /**< data has duplicates */ + +/** valid flags for #mdb_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND) + +/** @} */ + unsigned short mn_flags; /**< @ref mdb_node */ + unsigned short mn_ksize; /**< key size */ + char mn_data[1]; /**< key and data are appended here */ +} MDB_node; + + /** Size of the node header, excluding dynamic data at the end */ +#define NODESIZE offsetof(MDB_node, mn_data) + + /** Bit position of top word in page number, for shifting mn_flags */ +#define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0) + + /** Size of a node in a branch page with a given key. + * This is just the node header plus the key, there is no data. + */ +#define INDXSIZE(k) (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size)) + + /** Size of a node in a leaf page with a given key and data. + * This is node header plus key plus data size. + */ +#define LEAFSIZE(k, d) (NODESIZE + (k)->mv_size + (d)->mv_size) + + /** Address of node \b i in page \b p */ +#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + (p)->mp_ptrs[i] + PAGEBASE)) + + /** Address of the key for the node */ +#define NODEKEY(node) (void *)((node)->mn_data) + + /** Address of the data for a node */ +#define NODEDATA(node) (void *)((char *)(node)->mn_data + (node)->mn_ksize) + + /** Get the page number pointed to by a branch node */ +#define NODEPGNO(node) \ + ((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \ + (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0)) + /** Set the page number in a branch node */ +#define SETPGNO(node,pgno) do { \ + (node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \ + if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0) + + /** Get the size of the data in a leaf node */ +#define NODEDSZ(node) ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16)) + /** Set the size of the data for a leaf node */ +#define SETDSZ(node,size) do { \ + (node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0) + /** The size of a key in a node */ +#define NODEKSZ(node) ((node)->mn_ksize) + + /** Copy a page number from src to dst */ +#ifdef MISALIGNED_OK +#define COPY_PGNO(dst,src) dst = src +#else +#if SIZE_MAX > 4294967295UL +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d++ = *s++; \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#else +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#endif +#endif + /** The address of a key in a LEAF2 page. + * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs. + * There are no node headers, keys are stored contiguously. + */ +#define LEAF2KEY(p, i, ks) ((char *)(p) + PAGEHDRSZ + ((i)*(ks))) + + /** Set the \b node's key into \b keyptr, if requested. */ +#define MDB_GET_KEY(node, keyptr) { if ((keyptr) != NULL) { \ + (keyptr)->mv_size = NODEKSZ(node); (keyptr)->mv_data = NODEKEY(node); } } + + /** Set the \b node's key into \b key. */ +#define MDB_GET_KEY2(node, key) { key.mv_size = NODEKSZ(node); key.mv_data = NODEKEY(node); } + + /** Information about a single database in the environment. */ +typedef struct MDB_db { + uint32_t md_pad; /**< also ksize for LEAF2 pages */ + uint16_t md_flags; /**< @ref mdb_dbi_open */ + uint16_t md_depth; /**< depth of this tree */ + pgno_t md_branch_pages; /**< number of internal pages */ + pgno_t md_leaf_pages; /**< number of leaf pages */ + pgno_t md_overflow_pages; /**< number of overflow pages */ + size_t md_entries; /**< number of data items */ + pgno_t md_root; /**< the root page of this tree */ +} MDB_db; + + /** mdb_dbi_open flags */ +#define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ +#define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) +#define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ + MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) + + /** Handle for the DB used to track free pages. */ +#define FREE_DBI 0 + /** Handle for the default DB. */ +#define MAIN_DBI 1 + + /** Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2). + */ +typedef struct MDB_meta { + /** Stamp identifying this as an LMDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mm_magic; + /** Version number of this lock file. Must be set to #MDB_DATA_VERSION. */ + uint32_t mm_version; + void *mm_address; /**< address for fixed mapping */ + size_t mm_mapsize; /**< size of mmap region */ + MDB_db mm_dbs[2]; /**< first is free space, 2nd is main db */ + /** The size of pages used in this DB */ +#define mm_psize mm_dbs[0].md_pad + /** Any persistent environment flags. @ref mdb_env */ +#define mm_flags mm_dbs[0].md_flags + pgno_t mm_last_pg; /**< last used page in file */ + txnid_t mm_txnid; /**< txnid that committed this page */ +} MDB_meta; + + /** Buffer for a stack-allocated meta page. + * The members define size and alignment, and silence type + * aliasing warnings. They are not used directly; that could + * mean incorrectly using several union members in parallel. + */ +typedef union MDB_metabuf { + MDB_page mb_page; + struct { + char mm_pad[PAGEHDRSZ]; + MDB_meta mm_meta; + } mb_metabuf; +} MDB_metabuf; + + /** Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. + */ +typedef struct MDB_dbx { + MDB_val md_name; /**< name of the database */ + MDB_cmp_func *md_cmp; /**< function for comparing keys */ + MDB_cmp_func *md_dcmp; /**< function for comparing data items */ + MDB_rel_func *md_rel; /**< user relocate function */ + void *md_relctx; /**< user-provided context for md_rel */ +} MDB_dbx; + + /** A database transaction. + * Every operation requires a transaction handle. + */ +struct MDB_txn { + MDB_txn *mt_parent; /**< parent of a nested txn */ + MDB_txn *mt_child; /**< nested txn under this txn */ + pgno_t mt_next_pgno; /**< next unallocated page */ + /** The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. + */ + txnid_t mt_txnid; + MDB_env *mt_env; /**< the DB environment */ + /** The list of pages that became unused during this transaction. + */ + MDB_IDL mt_free_pgs; + /** The list of loose pages that became unused and may be reused + * in this transaction, linked through #NEXT_LOOSE_PAGE(page). + */ + MDB_page *mt_loose_pgs; + /* #Number of loose pages (#mt_loose_pgs) */ + int mt_loose_count; + /** The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. + */ + MDB_IDL mt_spill_pgs; + union { + /** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */ + MDB_ID2L dirty_list; + /** For read txns: This thread/txn's reader table slot, or NULL. */ + MDB_reader *reader; + } mt_u; + /** Array of records for each DB known in the environment. */ + MDB_dbx *mt_dbxs; + /** Array of MDB_db records for each known DB */ + MDB_db *mt_dbs; + /** Array of sequence numbers for each DB handle */ + unsigned int *mt_dbiseqs; +/** @defgroup mt_dbflag Transaction DB Flags + * @ingroup internal + * @{ + */ +#define DB_DIRTY 0x01 /**< DB was modified or is DUPSORT data */ +#define DB_STALE 0x02 /**< Named-DB record is older than txnID */ +#define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ +#define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ +/** @} */ + /** In write txns, array of cursors for each DB */ + MDB_cursor **mt_cursors; + /** Array of flags for each DB */ + unsigned char *mt_dbflags; + /** Number of DB records in use. This number only ever increments; + * we don't decrement it when individual DB handles are closed. + */ + MDB_dbi mt_numdbs; + +/** @defgroup mdb_txn Transaction Flags + * @ingroup internal + * @{ + */ +#define MDB_TXN_RDONLY 0x01 /**< read-only transaction */ +#define MDB_TXN_ERROR 0x02 /**< txn is unusable after an error */ +#define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ +#define MDB_TXN_SPILLS 0x08 /**< txn or a parent has spilled pages */ +/** @} */ + unsigned int mt_flags; /**< @ref mdb_txn */ + /** #dirty_list room: Array size - \#dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirty_list into mt_parent after freeing hidden mt_parent pages. + */ + unsigned int mt_dirty_room; +}; + +/** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. + */ +#define CURSOR_STACK 32 + +struct MDB_xcursor; + + /** Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. #MDB_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a #P_SUBP page can be stale. + * (A node with #F_DUPDATA but no #F_SUBDATA contains a subpage). + */ +struct MDB_cursor { + /** Next cursor on this DB in this txn */ + MDB_cursor *mc_next; + /** Backup of the original cursor if this cursor is a shadow */ + MDB_cursor *mc_backup; + /** Context used for databases with #MDB_DUPSORT, otherwise NULL */ + struct MDB_xcursor *mc_xcursor; + /** The transaction that owns this cursor */ + MDB_txn *mc_txn; + /** The database handle this cursor operates on */ + MDB_dbi mc_dbi; + /** The database record for this cursor */ + MDB_db *mc_db; + /** The database auxiliary record for this cursor */ + MDB_dbx *mc_dbx; + /** The @ref mt_dbflag for this database */ + unsigned char *mc_dbflag; + unsigned short mc_snum; /**< number of pushed pages */ + unsigned short mc_top; /**< index of top page, normally mc_snum-1 */ +/** @defgroup mdb_cursor Cursor Flags + * @ingroup internal + * Cursor state flags. + * @{ + */ +#define C_INITIALIZED 0x01 /**< cursor has been initialized and is valid */ +#define C_EOF 0x02 /**< No more data */ +#define C_SUB 0x04 /**< Cursor is a sub-cursor */ +#define C_DEL 0x08 /**< last op was a cursor_del */ +#define C_SPLITTING 0x20 /**< Cursor is in page_split */ +#define C_UNTRACK 0x40 /**< Un-track cursor when closing */ +/** @} */ + unsigned int mc_flags; /**< @ref mdb_cursor */ + MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ +}; + + /** Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. + */ +typedef struct MDB_xcursor { + /** A sub-cursor for traversing the Dup DB */ + MDB_cursor mx_cursor; + /** The database record for this Dup DB */ + MDB_db mx_db; + /** The auxiliary DB record for this Dup DB */ + MDB_dbx mx_dbx; + /** The @ref mt_dbflag for this Dup DB */ + unsigned char mx_dbflag; +} MDB_xcursor; + + /** State of FreeDB old pages, stored in the MDB_env */ +typedef struct MDB_pgstate { + pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ + txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ +} MDB_pgstate; + + /** The database environment. */ +struct MDB_env { + HANDLE me_fd; /**< The main data file */ + HANDLE me_lfd; /**< The lock file */ + HANDLE me_mfd; /**< just for writing the meta pages */ + /** Failed to update the meta page. Probably an I/O error. */ +#define MDB_FATAL_ERROR 0x80000000U + /** Some fields are initialized. */ +#define MDB_ENV_ACTIVE 0x20000000U + /** me_txkey is set */ +#define MDB_ENV_TXKEY 0x10000000U + uint32_t me_flags; /**< @ref mdb_env */ + unsigned int me_psize; /**< DB page size, inited from me_os_psize */ + unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ + unsigned int me_maxreaders; /**< size of the reader table */ + unsigned int me_numreaders; /**< max numreaders set by this env */ + MDB_dbi me_numdbs; /**< number of DBs opened */ + MDB_dbi me_maxdbs; /**< size of the DB table */ + MDB_PID_T me_pid; /**< process ID of this env */ + char *me_path; /**< path to the DB files */ + char *me_map; /**< the memory map of the data file */ + MDB_txninfo *me_txns; /**< the memory map of the lock file or NULL */ + MDB_meta *me_metas[2]; /**< pointers to the two meta pages */ + void *me_pbuf; /**< scratch area for DUPSORT put() */ + MDB_txn *me_txn; /**< current write transaction */ + MDB_txn *me_txn0; /**< prealloc'd write transaction */ + size_t me_mapsize; /**< size of the data memory map */ + off_t me_size; /**< current file size */ + pgno_t me_maxpg; /**< me_mapsize / me_psize */ + MDB_dbx *me_dbxs; /**< array of static DB info */ + uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ + unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ + pthread_key_t me_txkey; /**< thread-key for readers */ + txnid_t me_pgoldest; /**< ID of oldest reader last time we looked */ + MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ +# define me_pglast me_pgstate.mf_pglast +# define me_pghead me_pgstate.mf_pghead + MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ + /** IDL of pages that became unused in a write txn */ + MDB_IDL me_free_pgs; + /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */ + MDB_ID2L me_dirty_list; + /** Max number of freelist items that can fit in a single overflow page */ + int me_maxfree_1pg; + /** Max size of a node on a page */ + unsigned int me_nodemax; +#if !(MDB_MAXKEYSIZE) + unsigned int me_maxkey; /**< max size of a key */ +#endif + int me_live_reader; /**< have liveness lock in reader table */ +#ifdef _WIN32 + int me_pidquery; /**< Used in OpenProcess */ +#endif +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + /* Windows mutexes/POSIX semaphores do not reside in shared mem */ + mdb_mutex_t me_rmutex; + mdb_mutex_t me_wmutex; +#endif + void *me_userctx; /**< User-settable context */ + MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ +}; + + /** Nested transaction */ +typedef struct MDB_ntxn { + MDB_txn mnt_txn; /**< the transaction */ + MDB_pgstate mnt_pgstate; /**< parent transaction's saved freestate */ +} MDB_ntxn; + + /** max number of pages to commit in one writev() call */ +#define MDB_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES +#undef MDB_COMMIT_PAGES +#define MDB_COMMIT_PAGES IOV_MAX +#endif + + /** max bytes to write in one call */ +#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4)) + + /** Check \b txn and \b dbi arguments to a function */ +#define TXN_DBI_EXIST(txn, dbi) \ + ((txn) && (dbi) < (txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & DB_VALID)) + + /** Check for misused \b dbi handles */ +#define TXN_DBI_CHANGED(txn, dbi) \ + ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) + +static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); +static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); +static int mdb_page_touch(MDB_cursor *mc); + +static int mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **mp, int *lvl); +static int mdb_page_search_root(MDB_cursor *mc, + MDB_val *key, int modify); +#define MDB_PS_MODIFY 1 +#define MDB_PS_ROOTONLY 2 +#define MDB_PS_FIRST 4 +#define MDB_PS_LAST 8 +static int mdb_page_search(MDB_cursor *mc, + MDB_val *key, int flags); +static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); + +#define MDB_SPLIT_REPLACE MDB_APPENDDUP /**< newkey is not new */ +static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, + pgno_t newpgno, unsigned int nflags); + +static int mdb_env_read_header(MDB_env *env, MDB_meta *meta); +static int mdb_env_pick_meta(const MDB_env *env); +static int mdb_env_write_meta(MDB_txn *txn); +#if !(defined(_WIN32) || defined(MDB_USE_POSIX_SEM)) /* Drop unused excl arg */ +# define mdb_env_close0(env, excl) mdb_env_close1(env) +#endif +static void mdb_env_close0(MDB_env *env, int excl); + +static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp); +static int mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags); +static void mdb_node_del(MDB_cursor *mc, int ksize); +static void mdb_node_shrink(MDB_page *mp, indx_t indx); +static int mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst); +static int mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data); +static size_t mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data); +static size_t mdb_branch_size(MDB_env *env, MDB_val *key); + +static int mdb_rebalance(MDB_cursor *mc); +static int mdb_update_key(MDB_cursor *mc, MDB_val *key); + +static void mdb_cursor_pop(MDB_cursor *mc); +static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp); + +static int mdb_cursor_del0(MDB_cursor *mc); +static int mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags); +static int mdb_cursor_sibling(MDB_cursor *mc, int move_right); +static int mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op, + int *exactp); +static int mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data); +static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); + +static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); +static void mdb_xcursor_init0(MDB_cursor *mc); +static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); + +static int mdb_drop0(MDB_cursor *mc, int subs); +static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); +static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead); + +/** @cond */ +static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; +/** @endcond */ + +#ifdef _WIN32 +static SECURITY_DESCRIPTOR mdb_null_sd; +static SECURITY_ATTRIBUTES mdb_all_sa; +static int mdb_sec_inited; +#endif + +/** Return the library version info. */ +char * +mdb_version(int *major, int *minor, int *patch) +{ + if (major) *major = MDB_VERSION_MAJOR; + if (minor) *minor = MDB_VERSION_MINOR; + if (patch) *patch = MDB_VERSION_PATCH; + return MDB_VERSION_STRING; +} + +/** Table of descriptions for LMDB @ref errors */ +static char *const mdb_errstr[] = { + "MDB_KEYEXIST: Key/data pair already exists", + "MDB_NOTFOUND: No matching key/data pair found", + "MDB_PAGE_NOTFOUND: Requested page not found", + "MDB_CORRUPTED: Located page was wrong type", + "MDB_PANIC: Update of meta page failed or environment had fatal error", + "MDB_VERSION_MISMATCH: Database environment version mismatch", + "MDB_INVALID: File is not an LMDB file", + "MDB_MAP_FULL: Environment mapsize limit reached", + "MDB_DBS_FULL: Environment maxdbs limit reached", + "MDB_READERS_FULL: Environment maxreaders limit reached", + "MDB_TLS_FULL: Thread-local storage keys full - too many environments open", + "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", + "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", + "MDB_PAGE_FULL: Internal error - page has no more space", + "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", + "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", + "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", + "MDB_BAD_TXN: Transaction cannot recover - it must be aborted", + "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", + "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", +}; + +char * +mdb_strerror(int err) +{ +#ifdef _WIN32 + /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. + * This works as long as no function between the call to mdb_strerror + * and the actual use of the message uses more than 4K of stack. + */ + char pad[4096]; + char buf[1024], *ptr = buf; +#endif + int i; + if (!err) + return ("Successful return: 0"); + + if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { + i = err - MDB_KEYEXIST; + return mdb_errstr[i]; + } + +#ifdef _WIN32 + /* These are the C-runtime error codes we use. The comment indicates + * their numeric value, and the Win32 error they would correspond to + * if the error actually came from a Win32 API. A major mess, we should + * have used LMDB-specific error codes for everything. + */ + switch(err) { + case ENOENT: /* 2, FILE_NOT_FOUND */ + case EIO: /* 5, ACCESS_DENIED */ + case ENOMEM: /* 12, INVALID_ACCESS */ + case EACCES: /* 13, INVALID_DATA */ + case EBUSY: /* 16, CURRENT_DIRECTORY */ + case EINVAL: /* 22, BAD_COMMAND */ + case ENOSPC: /* 28, OUT_OF_PAPER */ + return strerror(err); + default: + ; + } + buf[0] = 0; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, 0, ptr, sizeof(buf), pad); + return ptr; +#else + return strerror(err); +#endif +} + +/** assert(3) variant in cursor context */ +#define mdb_cassert(mc, expr) mdb_assert0((mc)->mc_txn->mt_env, expr, #expr) +/** assert(3) variant in transaction context */ +#define mdb_tassert(mc, expr) mdb_assert0((txn)->mt_env, expr, #expr) +/** assert(3) variant in environment context */ +#define mdb_eassert(env, expr) mdb_assert0(env, expr, #expr) + +#ifndef NDEBUG +# define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \ + mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__)) + +static void +mdb_assert_fail(MDB_env *env, const char *expr_txt, + const char *func, const char *file, int line) +{ + char buf[400]; + sprintf(buf, "%.100s:%d: Assertion '%.200s' failed in %.40s()", + file, line, expr_txt, func); + if (env->me_assert_func) + env->me_assert_func(env, buf); + fprintf(stderr, "%s\n", buf); + abort(); +} +#else +# define mdb_assert0(env, expr, expr_txt) ((void) 0) +#endif /* NDEBUG */ + +#if MDB_DEBUG +/** Return the page number of \b mp which may be sub-page, for debug output */ +static pgno_t +mdb_dbg_pgno(MDB_page *mp) +{ + pgno_t ret; + COPY_PGNO(ret, mp->mp_pgno); + return ret; +} + +/** Display a key in hexadecimal and return the address of the result. + * @param[in] key the key to display + * @param[in] buf the buffer to write into. Should always be #DKBUF. + * @return The key in hexadecimal form. + */ +char * +mdb_dkey(MDB_val *key, char *buf) +{ + char *ptr = buf; + unsigned char *c = key->mv_data; + unsigned int i; + + if (!key) + return ""; + + if (key->mv_size > DKBUF_MAXKEYSIZE) + return "MDB_MAXKEYSIZE"; + /* may want to make this a dynamic check: if the key is mostly + * printable characters, print it as-is instead of converting to hex. + */ +#if 1 + buf[0] = '\0'; + for (i=0; imv_size; i++) + ptr += sprintf(ptr, "%02x", *c++); +#else + sprintf(buf, "%.*s", key->mv_size, key->mv_data); +#endif + return buf; +} + +static const char * +mdb_leafnode_type(MDB_node *n) +{ + static char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; + return F_ISSET(n->mn_flags, F_BIGDATA) ? ": overflow page" : + tp[F_ISSET(n->mn_flags, F_DUPDATA)][F_ISSET(n->mn_flags, F_SUBDATA)]; +} + +/** Display all the keys in the page. */ +void +mdb_page_list(MDB_page *mp) +{ + pgno_t pgno = mdb_dbg_pgno(mp); + const char *type, *state = (mp->mp_flags & P_DIRTY) ? ", dirty" : ""; + MDB_node *node; + unsigned int i, nkeys, nsize, total = 0; + MDB_val key; + DKBUF; + + switch (mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { + case P_BRANCH: type = "Branch page"; break; + case P_LEAF: type = "Leaf page"; break; + case P_LEAF|P_SUBP: type = "Sub-page"; break; + case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; + case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; + case P_OVERFLOW: + fprintf(stderr, "Overflow page %"Z"u pages %u%s\n", + pgno, mp->mp_pages, state); + return; + case P_META: + fprintf(stderr, "Meta-page %"Z"u txnid %"Z"u\n", + pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); + return; + default: + fprintf(stderr, "Bad page %"Z"u flags 0x%u\n", pgno, mp->mp_flags); + return; + } + + nkeys = NUMKEYS(mp); + fprintf(stderr, "%s %"Z"u numkeys %d%s\n", type, pgno, nkeys, state); + + for (i=0; imp_pad; + key.mv_data = LEAF2KEY(mp, i, nsize); + total += nsize; + fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key)); + continue; + } + node = NODEPTR(mp, i); + key.mv_size = node->mn_ksize; + key.mv_data = node->mn_data; + nsize = NODESIZE + key.mv_size; + if (IS_BRANCH(mp)) { + fprintf(stderr, "key %d: page %"Z"u, %s\n", i, NODEPGNO(node), + DKEY(&key)); + total += nsize; + } else { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + nsize += sizeof(pgno_t); + else + nsize += NODEDSZ(node); + total += nsize; + nsize += sizeof(indx_t); + fprintf(stderr, "key %d: nsize %d, %s%s\n", + i, nsize, DKEY(&key), mdb_leafnode_type(node)); + } + total = EVEN(total); + } + fprintf(stderr, "Total: header %d + contents %d + unused %d\n", + IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + mp->mp_lower, total, SIZELEFT(mp)); +} + +void +mdb_cursor_chk(MDB_cursor *mc) +{ + unsigned int i; + MDB_node *node; + MDB_page *mp; + + if (!mc->mc_snum && !(mc->mc_flags & C_INITIALIZED)) return; + for (i=0; imc_top; i++) { + mp = mc->mc_pg[i]; + node = NODEPTR(mp, mc->mc_ki[i]); + if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno) + printf("oops!\n"); + } + if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) + printf("ack!\n"); +} +#endif + +#if (MDB_DEBUG) > 2 +/** Count all the pages in each DB and in the freelist + * and make sure it matches the actual number of pages + * being used. + * All named DBs must be open for a correct count. + */ +static void mdb_audit(MDB_txn *txn) +{ + MDB_cursor mc; + MDB_val key, data; + MDB_ID freecount, count; + MDB_dbi i; + int rc; + + freecount = 0; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + mdb_tassert(txn, rc == MDB_NOTFOUND); + + count = 0; + for (i = 0; imt_numdbs; i++) { + MDB_xcursor mx; + if (!(txn->mt_dbflags[i] & DB_VALID)) + continue; + mdb_cursor_init(&mc, txn, i, &mx); + if (txn->mt_dbs[i].md_root == P_INVALID) + continue; + count += txn->mt_dbs[i].md_branch_pages + + txn->mt_dbs[i].md_leaf_pages + + txn->mt_dbs[i].md_overflow_pages; + if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) { + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST); + for (; rc == MDB_SUCCESS; rc = mdb_cursor_sibling(&mc, 1)) { + unsigned j; + MDB_page *mp; + mp = mc.mc_pg[mc.mc_top]; + for (j=0; jmn_flags & F_SUBDATA) { + MDB_db db; + memcpy(&db, NODEDATA(leaf), sizeof(db)); + count += db.md_branch_pages + db.md_leaf_pages + + db.md_overflow_pages; + } + } + } + mdb_tassert(txn, rc == MDB_NOTFOUND); + } + } + if (freecount + count + 2 /* metapages */ != txn->mt_next_pgno) { + fprintf(stderr, "audit: %lu freecount: %lu count: %lu total: %lu next_pgno: %lu\n", + txn->mt_txnid, freecount, count+2, freecount+count+2, txn->mt_next_pgno); + } +} +#endif + +int +mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_cmp(a, b); +} + +int +mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_dcmp(a, b); +} + +/** Allocate memory for a page. + * Re-use old malloc'd pages first for singletons, otherwise just malloc. + */ +static MDB_page * +mdb_page_malloc(MDB_txn *txn, unsigned num) +{ + MDB_env *env = txn->mt_env; + MDB_page *ret = env->me_dpages; + size_t psize = env->me_psize, sz = psize, off; + /* For ! #MDB_NOMEMINIT, psize counts how much to init. + * For a single page alloc, we init everything after the page header. + * For multi-page, we init the final page; if the caller needed that + * many pages they will be filling in at least up to the last page. + */ + if (num == 1) { + if (ret) { + VGMEMP_ALLOC(env, ret, sz); + VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); + env->me_dpages = ret->mp_next; + return ret; + } + psize -= off = PAGEHDRSZ; + } else { + sz *= num; + off = sz - psize; + } + if ((ret = malloc(sz)) != NULL) { + VGMEMP_ALLOC(env, ret, sz); + if (!(env->me_flags & MDB_NOMEMINIT)) { + memset((char *)ret + off, 0, psize); + ret->mp_pad = 0; + } + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + return ret; +} +/** Free a single page. + * Saves single pages to a list, for future reuse. + * (This is not used for multi-page overflow pages.) + */ +static void +mdb_page_free(MDB_env *env, MDB_page *mp) +{ + mp->mp_next = env->me_dpages; + VGMEMP_FREE(env, mp); + env->me_dpages = mp; +} + +/** Free a dirty page */ +static void +mdb_dpage_free(MDB_env *env, MDB_page *dp) +{ + if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { + mdb_page_free(env, dp); + } else { + /* large pages just get freed directly */ + VGMEMP_FREE(env, dp); + free(dp); + } +} + +/** Return all dirty pages to dpage list */ +static void +mdb_dlist_free(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned i, n = dl[0].mid; + + for (i = 1; i <= n; i++) { + mdb_dpage_free(env, dl[i].mptr); + } + dl[0].mid = 0; +} + +/** Loosen or free a single page. + * Saves single pages to a list for future reuse + * in this same txn. It has been pulled from the freeDB + * and already resides on the dirty list, but has been + * deleted. Use these pages first before pulling again + * from the freeDB. + * + * If the page wasn't dirtied in this txn, just add it + * to this txn's free list. + */ +static int +mdb_page_loose(MDB_cursor *mc, MDB_page *mp) +{ + int loose = 0; + pgno_t pgno = mp->mp_pgno; + MDB_txn *txn = mc->mc_txn; + + if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) { + if (txn->mt_parent) { + MDB_ID2 *dl = txn->mt_u.dirty_list; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + /* ok, it's ours */ + loose = 1; + } + } + } else { + /* no parent txn, so it's just ours */ + loose = 1; + } + } + if (loose) { + DPRINTF(("loosen db %d page %"Z"u", DDBI(mc), + mp->mp_pgno)); + NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; + txn->mt_loose_pgs = mp; + txn->mt_loose_count++; + mp->mp_flags |= P_LOOSE; + } else { + int rc = mdb_midl_append(&txn->mt_free_pgs, pgno); + if (rc) + return rc; + } + + return MDB_SUCCESS; +} + +/** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. + * @param[in] mc A cursor handle for the current operation. + * @param[in] pflags Flags of the pages to update: + * P_DIRTY to set P_KEEP, P_DIRTY|P_KEEP to clear it. + * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush(). + * @return 0 on success, non-zero on failure. + */ +static int +mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) +{ + enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP }; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m3; + MDB_xcursor *mx; + MDB_page *dp, *mp; + MDB_node *leaf; + unsigned i, j; + int rc = MDB_SUCCESS, level; + + /* Mark pages seen by cursors */ + if (mc->mc_flags & C_UNTRACK) + mc = NULL; /* will find mc in mt_cursors */ + for (i = txn->mt_numdbs;; mc = txn->mt_cursors[--i]) { + for (; mc; mc=mc->mc_next) { + if (!(mc->mc_flags & C_INITIALIZED)) + continue; + for (m3 = mc;; m3 = &mx->mx_cursor) { + mp = NULL; + for (j=0; jmc_snum; j++) { + mp = m3->mc_pg[j]; + if ((mp->mp_flags & Mask) == pflags) + mp->mp_flags ^= P_KEEP; + } + mx = m3->mc_xcursor; + /* Proceed to mx if it is at a sub-database */ + if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED))) + break; + if (! (mp && (mp->mp_flags & P_LEAF))) + break; + leaf = NODEPTR(mp, m3->mc_ki[j-1]); + if (!(leaf->mn_flags & F_SUBDATA)) + break; + } + } + if (i == 0) + break; + } + + if (all) { + /* Mark dirty root pages */ + for (i=0; imt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + pgno_t pgno = txn->mt_dbs[i].md_root; + if (pgno == P_INVALID) + continue; + if ((rc = mdb_page_get(txn, pgno, &dp, &level)) != MDB_SUCCESS) + break; + if ((dp->mp_flags & Mask) == pflags && level <= 1) + dp->mp_flags ^= P_KEEP; + } + } + } + + return rc; +} + +static int mdb_page_flush(MDB_txn *txn, int keep); + +/** Spill pages from the dirty list back to disk. + * This is intended to prevent running into #MDB_TXN_FULL situations, + * but note that they may still occur in a few cases: + * 1) our estimate of the txn size could be too small. Currently this + * seems unlikely, except with a large number of #MDB_MULTIPLE items. + * 2) child txns may run out of space if their parents dirtied a + * lot of pages and never spilled them. TODO: we probably should do + * a preemptive spill during #mdb_txn_begin() of a child txn, if + * the parent's dirty_room is below a given threshold. + * + * Otherwise, if not using nested txns, it is expected that apps will + * not run into #MDB_TXN_FULL any more. The pages are flushed to disk + * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared. + * If the txn never references them again, they can be left alone. + * If the txn only reads them, they can be used without any fuss. + * If the txn writes them again, they can be dirtied immediately without + * going thru all of the work of #mdb_page_touch(). Such references are + * handled by #mdb_page_unspill(). + * + * Also note, we never spill DB root pages, nor pages of active cursors, + * because we'll need these back again soon anyway. And in nested txns, + * we can't spill a page in a child txn if it was already spilled in a + * parent txn. That would alter the parent txns' data even though + * the child hasn't committed yet, and we'd have no way to undo it if + * the child aborted. + * + * @param[in] m0 cursor A cursor handle identifying the transaction and + * database for which we are checking space. + * @param[in] key For a put operation, the key being stored. + * @param[in] data For a put operation, the data being stored. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data) +{ + MDB_txn *txn = m0->mc_txn; + MDB_page *dp; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned int i, j, need; + int rc; + + if (m0->mc_flags & C_SUB) + return MDB_SUCCESS; + + /* Estimate how much space this op will take */ + i = m0->mc_db->md_depth; + /* Named DBs also dirty the main DB */ + if (m0->mc_dbi > MAIN_DBI) + i += txn->mt_dbs[MAIN_DBI].md_depth; + /* For puts, roughly factor in the key+data size */ + if (key) + i += (LEAFSIZE(key, data) + txn->mt_env->me_psize) / txn->mt_env->me_psize; + i += i; /* double it for good measure */ + need = i; + + if (txn->mt_dirty_room > i) + return MDB_SUCCESS; + + if (!txn->mt_spill_pgs) { + txn->mt_spill_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX); + if (!txn->mt_spill_pgs) + return ENOMEM; + } else { + /* purge deleted slots */ + MDB_IDL sl = txn->mt_spill_pgs; + unsigned int num = sl[0]; + j=0; + for (i=1; i<=num; i++) { + if (!(sl[i] & 1)) + sl[++j] = sl[i]; + } + sl[0] = j; + } + + /* Preserve pages which may soon be dirtied again */ + if ((rc = mdb_pages_xkeep(m0, P_DIRTY, 1)) != MDB_SUCCESS) + goto done; + + /* Less aggressive spill - we originally spilled the entire dirty list, + * with a few exceptions for cursor pages and DB root pages. But this + * turns out to be a lot of wasted effort because in a large txn many + * of those pages will need to be used again. So now we spill only 1/8th + * of the dirty pages. Testing revealed this to be a good tradeoff, + * better than 1/2, 1/4, or 1/10. + */ + if (need < MDB_IDL_UM_MAX / 8) + need = MDB_IDL_UM_MAX / 8; + + /* Save the page IDs of all the pages we're flushing */ + /* flush from the tail forward, this saves a lot of shifting later on. */ + for (i=dl[0].mid; i && need; i--) { + MDB_ID pn = dl[i].mid << 1; + dp = dl[i].mptr; + if (dp->mp_flags & (P_LOOSE|P_KEEP)) + continue; + /* Can't spill twice, make sure it's not already in a parent's + * spill list. + */ + if (txn->mt_parent) { + MDB_txn *tx2; + for (tx2 = txn->mt_parent; tx2; tx2 = tx2->mt_parent) { + if (tx2->mt_spill_pgs) { + j = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (j <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[j] == pn) { + dp->mp_flags |= P_KEEP; + break; + } + } + } + if (tx2) + continue; + } + if ((rc = mdb_midl_append(&txn->mt_spill_pgs, pn))) + goto done; + need--; + } + mdb_midl_sort(txn->mt_spill_pgs); + + /* Flush the spilled part of dirty list */ + if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS) + goto done; + + /* Reset any dirty pages we kept that page_flush didn't see */ + rc = mdb_pages_xkeep(m0, P_DIRTY|P_KEEP, i); + +done: + txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS; + return rc; +} + +/** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */ +static txnid_t +mdb_find_oldest(MDB_txn *txn) +{ + int i; + txnid_t mr, oldest = txn->mt_txnid - 1; + if (txn->mt_env->me_txns) { + MDB_reader *r = txn->mt_env->me_txns->mti_readers; + for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) { + if (r[i].mr_pid) { + mr = r[i].mr_txnid; + if (oldest > mr) + oldest = mr; + } + } + } + return oldest; +} + +/** Add a page to the txn's dirty list */ +static void +mdb_page_dirty(MDB_txn *txn, MDB_page *mp) +{ + MDB_ID2 mid; + int rc, (*insert)(MDB_ID2L, MDB_ID2 *); + + if (txn->mt_env->me_flags & MDB_WRITEMAP) { + insert = mdb_mid2l_append; + } else { + insert = mdb_mid2l_insert; + } + mid.mid = mp->mp_pgno; + mid.mptr = mp; + rc = insert(txn->mt_u.dirty_list, &mid); + mdb_tassert(txn, rc == 0); + txn->mt_dirty_room--; +} + +/** Allocate page numbers and memory for writing. Maintain me_pglast, + * me_pghead and mt_next_pgno. + * + * If there are free pages available from older transactions, they + * are re-used first. Otherwise allocate a new page at mt_next_pgno. + * Do not modify the freedB, just merge freeDB records into me_pghead[] + * and move me_pglast to say which records were consumed. Only this + * function can create me_pghead and move me_pglast/mt_next_pgno. + * @param[in] mc cursor A cursor handle identifying the transaction and + * database for which we are allocating. + * @param[in] num the number of pages to allocate. + * @param[out] mp Address of the allocated page(s). Requests for multiple pages + * will always be satisfied by a single contiguous chunk of memory. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) +{ +#ifdef MDB_PARANOID /* Seems like we can ignore this now */ + /* Get at most more freeDB records once me_pghead + * has enough pages. If not enough, use new pages from the map. + * If and mc is updating the freeDB, only get new + * records if me_pghead is empty. Then the freelist cannot play + * catch-up with itself by growing while trying to save it. + */ + enum { Paranoid = 1, Max_retries = 500 }; +#else + enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; +#endif + int rc, retry = num * 60; + MDB_txn *txn = mc->mc_txn; + MDB_env *env = txn->mt_env; + pgno_t pgno, *mop = env->me_pghead; + unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1; + MDB_page *np; + txnid_t oldest = 0, last; + MDB_cursor_op op; + MDB_cursor m2; + int found_old = 0; + + /* If there are any loose pages, just use them */ + if (num == 1 && txn->mt_loose_pgs) { + np = txn->mt_loose_pgs; + txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); + txn->mt_loose_count--; + DPRINTF(("db %d use loose page %"Z"u", DDBI(mc), + np->mp_pgno)); + *mp = np; + return MDB_SUCCESS; + } + + *mp = NULL; + + /* If our dirty list is already full, we can't do anything */ + if (txn->mt_dirty_room == 0) { + rc = MDB_TXN_FULL; + goto fail; + } + + for (op = MDB_FIRST;; op = MDB_NEXT) { + MDB_val key, data; + MDB_node *leaf; + pgno_t *idl; + + /* Seek a big enough contiguous page range. Prefer + * pages at the tail, just truncating the list. + */ + if (mop_len > n2) { + i = mop_len; + do { + pgno = mop[i]; + if (mop[i-n2] == pgno+n2) + goto search_done; + } while (--i > n2); + if (--retry < 0) + break; + } + + if (op == MDB_FIRST) { /* 1st iteration */ + /* Prepare to fetch more and coalesce */ + last = env->me_pglast; + oldest = env->me_pgoldest; + mdb_cursor_init(&m2, txn, FREE_DBI, NULL); + if (last) { + op = MDB_SET_RANGE; + key.mv_data = &last; /* will look up last+1 */ + key.mv_size = sizeof(last); + } + if (Paranoid && mc->mc_dbi == FREE_DBI) + retry = -1; + } + if (Paranoid && retry < 0 && mop_len) + break; + + last++; + /* Do not fetch more if the record will be too recent */ + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + rc = mdb_cursor_get(&m2, &key, NULL, op); + if (rc) { + if (rc == MDB_NOTFOUND) + break; + goto fail; + } + last = *(txnid_t*)key.mv_data; + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + np = m2.mc_pg[m2.mc_top]; + leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); + if ((rc = mdb_node_read(txn, leaf, &data)) != MDB_SUCCESS) + return rc; + + idl = (MDB_ID *) data.mv_data; + i = idl[0]; + if (!mop) { + if (!(env->me_pghead = mop = mdb_midl_alloc(i))) { + rc = ENOMEM; + goto fail; + } + } else { + if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) + goto fail; + mop = env->me_pghead; + } + env->me_pglast = last; +#if (MDB_DEBUG) > 1 + DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u", + last, txn->mt_dbs[FREE_DBI].md_root, i)); + for (j = i; j; j--) + DPRINTF(("IDL %"Z"u", idl[j])); +#endif + /* Merge in descending sorted order */ + mdb_midl_xmerge(mop, idl); + mop_len = mop[0]; + } + + /* Use new pages from the map when nothing suitable in the freeDB */ + i = 0; + pgno = txn->mt_next_pgno; + if (pgno + num >= env->me_maxpg) { + DPUTS("DB size maxed out"); + rc = MDB_MAP_FULL; + goto fail; + } + +search_done: + if (env->me_flags & MDB_WRITEMAP) { + np = (MDB_page *)(env->me_map + env->me_psize * pgno); + } else { + if (!(np = mdb_page_malloc(txn, num))) { + rc = ENOMEM; + goto fail; + } + } + if (i) { + mop[0] = mop_len -= num; + /* Move any stragglers down */ + for (j = i-num; j < mop_len; ) + mop[++j] = mop[++i]; + } else { + txn->mt_next_pgno = pgno + num; + } + np->mp_pgno = pgno; + mdb_page_dirty(txn, np); + *mp = np; + + return MDB_SUCCESS; + +fail: + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +/** Copy the used portions of a non-overflow page. + * @param[in] dst page to copy into + * @param[in] src page to copy from + * @param[in] psize size of a page + */ +static void +mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize) +{ + enum { Align = sizeof(pgno_t) }; + indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower; + + /* If page isn't full, just copy the used portion. Adjust + * alignment so memcpy may copy words instead of bytes. + */ + if ((unused &= -Align) && !IS_LEAF2(src)) { + upper = (upper + PAGEBASE) & -Align; + memcpy(dst, src, (lower + PAGEBASE + (Align-1)) & -Align); + memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper), + psize - upper); + } else { + memcpy(dst, src, psize - unused); + } +} + +/** Pull a page off the txn's spill list, if present. + * If a page being referenced was spilled to disk in this txn, bring + * it back and make it dirty/writable again. + * @param[in] txn the transaction handle. + * @param[in] mp the page being referenced. It must not be dirty. + * @param[out] ret the writable page, if any. ret is unchanged if + * mp wasn't spilled. + */ +static int +mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) +{ + MDB_env *env = txn->mt_env; + const MDB_txn *tx2; + unsigned x; + pgno_t pgno = mp->mp_pgno, pn = pgno << 1; + + for (tx2 = txn; tx2; tx2=tx2->mt_parent) { + if (!tx2->mt_spill_pgs) + continue; + x = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { + MDB_page *np; + int num; + if (txn->mt_dirty_room == 0) + return MDB_TXN_FULL; + if (IS_OVERFLOW(mp)) + num = mp->mp_pages; + else + num = 1; + if (env->me_flags & MDB_WRITEMAP) { + np = mp; + } else { + np = mdb_page_malloc(txn, num); + if (!np) + return ENOMEM; + if (num > 1) + memcpy(np, mp, num * env->me_psize); + else + mdb_page_copy(np, mp, env->me_psize); + } + if (tx2 == txn) { + /* If in current txn, this page is no longer spilled. + * If it happens to be the last page, truncate the spill list. + * Otherwise mark it as deleted by setting the LSB. + */ + if (x == txn->mt_spill_pgs[0]) + txn->mt_spill_pgs[0]--; + else + txn->mt_spill_pgs[x] |= 1; + } /* otherwise, if belonging to a parent txn, the + * page remains spilled until child commits + */ + + mdb_page_dirty(txn, np); + np->mp_flags |= P_DIRTY; + *ret = np; + break; + } + } + return MDB_SUCCESS; +} + +/** Touch a page: make it dirty and re-insert into tree with updated pgno. + * @param[in] mc cursor pointing to the page to be touched + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_touch(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top], *np; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m2, *m3; + pgno_t pgno; + int rc; + + if (!F_ISSET(mp->mp_flags, P_DIRTY)) { + if (txn->mt_flags & MDB_TXN_SPILLS) { + np = NULL; + rc = mdb_page_unspill(txn, mp, &np); + if (rc) + goto fail; + if (np) + goto done; + } + if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || + (rc = mdb_page_alloc(mc, 1, &np))) + goto fail; + pgno = np->mp_pgno; + DPRINTF(("touched db %d page %"Z"u -> %"Z"u", DDBI(mc), + mp->mp_pgno, pgno)); + mdb_cassert(mc, mp->mp_pgno != pgno); + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* Update the parent page, if any, to point to the new page */ + if (mc->mc_top) { + MDB_page *parent = mc->mc_pg[mc->mc_top-1]; + MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]); + SETPGNO(node, pgno); + } else { + mc->mc_db->md_root = pgno; + } + } else if (txn->mt_parent && !IS_SUBP(mp)) { + MDB_ID2 mid, *dl = txn->mt_u.dirty_list; + pgno = mp->mp_pgno; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + return 0; + } + } + mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX); + /* No - copy it */ + np = mdb_page_malloc(txn, 1); + if (!np) + return ENOMEM; + mid.mid = pgno; + mid.mptr = np; + rc = mdb_mid2l_insert(dl, &mid); + mdb_cassert(mc, rc == 0); + } else { + return 0; + } + + mdb_page_copy(np, mp, txn->mt_env->me_psize); + np->mp_pgno = pgno; + np->mp_flags |= P_DIRTY; + +done: + /* Adjust cursors pointing to mp */ + mc->mc_pg[mc->mc_top] = np; + m2 = txn->mt_cursors[mc->mc_dbi]; + if (mc->mc_flags & C_SUB) { + for (; m2; m2=m2->mc_next) { + m3 = &m2->mc_xcursor->mx_cursor; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[mc->mc_top] == mp) + m3->mc_pg[mc->mc_top] = np; + } + } else { + for (; m2; m2=m2->mc_next) { + if (m2->mc_snum < mc->mc_snum) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + m2->mc_pg[mc->mc_top] = np; + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + IS_LEAF(np) && + m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) + { + MDB_node *leaf = NODEPTR(np, mc->mc_ki[mc->mc_top]); + if (!(leaf->mn_flags & F_SUBDATA)) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + } + } + } + return 0; + +fail: + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_env_sync(MDB_env *env, int force) +{ + int rc = 0; + if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { + if (env->me_flags & MDB_WRITEMAP) { + int flags = ((env->me_flags & MDB_MAPASYNC) && !force) + ? MS_ASYNC : MS_SYNC; + if (MDB_MSYNC(env->me_map, env->me_mapsize, flags)) + rc = ErrCode(); +#ifdef _WIN32 + else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); +#endif + } else { + if (MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); + } + } + return rc; +} + +/** Back up parent txn's cursors, then grab the originals for tracking */ +static int +mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) +{ + MDB_cursor *mc, *bk; + MDB_xcursor *mx; + size_t size; + int i; + + for (i = src->mt_numdbs; --i >= 0; ) { + if ((mc = src->mt_cursors[i]) != NULL) { + size = sizeof(MDB_cursor); + if (mc->mc_xcursor) + size += sizeof(MDB_xcursor); + for (; mc; mc = bk->mc_next) { + bk = malloc(size); + if (!bk) + return ENOMEM; + *bk = *mc; + mc->mc_backup = bk; + mc->mc_db = &dst->mt_dbs[i]; + /* Kill pointers into src - and dst to reduce abuse: The + * user may not use mc until dst ends. Otherwise we'd... + */ + mc->mc_txn = NULL; /* ...set this to dst */ + mc->mc_dbflag = NULL; /* ...and &dst->mt_dbflags[i] */ + if ((mx = mc->mc_xcursor) != NULL) { + *(MDB_xcursor *)(bk+1) = *mx; + mx->mx_cursor.mc_txn = NULL; /* ...and dst. */ + } + mc->mc_next = dst->mt_cursors[i]; + dst->mt_cursors[i] = mc; + } + } + } + return MDB_SUCCESS; +} + +/** Close this write txn's cursors, give parent txn's cursors back to parent. + * @param[in] txn the transaction handle. + * @param[in] merge true to keep changes to parent cursors, false to revert. + * @return 0 on success, non-zero on failure. + */ +static void +mdb_cursors_close(MDB_txn *txn, unsigned merge) +{ + MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; + MDB_xcursor *mx; + int i; + + for (i = txn->mt_numdbs; --i >= 0; ) { + for (mc = cursors[i]; mc; mc = next) { + next = mc->mc_next; + if ((bk = mc->mc_backup) != NULL) { + if (merge) { + /* Commit changes to parent txn */ + mc->mc_next = bk->mc_next; + mc->mc_backup = bk->mc_backup; + mc->mc_txn = bk->mc_txn; + mc->mc_db = bk->mc_db; + mc->mc_dbflag = bk->mc_dbflag; + if ((mx = mc->mc_xcursor) != NULL) + mx->mx_cursor.mc_txn = bk->mc_txn; + } else { + /* Abort nested txn */ + *mc = *bk; + if ((mx = mc->mc_xcursor) != NULL) + *mx = *(MDB_xcursor *)(bk+1); + } + mc = bk; + } + /* Only malloced cursors are permanently tracked. */ + free(mc); + } + cursors[i] = NULL; + } +} + +#if !(MDB_DEBUG) +#define mdb_txn_reset0(txn, act) mdb_txn_reset0(txn) +#endif +static void +mdb_txn_reset0(MDB_txn *txn, const char *act); + +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ +enum Pidlock_op { + Pidset, Pidcheck +}; +#else +enum Pidlock_op { + Pidset = F_SETLK, Pidcheck = F_GETLK +}; +#endif + +/** Set or check a pid lock. Set returns 0 on success. + * Check returns 0 if the process is certainly dead, nonzero if it may + * be alive (the lock exists or an error happened so we do not know). + * + * On Windows Pidset is a no-op, we merely check for the existence + * of the process with the given pid. On POSIX we use a single byte + * lock on the lockfile, set at an offset equal to the pid. + */ +static int +mdb_reader_pid(MDB_env *env, enum Pidlock_op op, MDB_PID_T pid) +{ +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ + int ret = 0; + HANDLE h; + if (op == Pidcheck) { + h = OpenProcess(env->me_pidquery, FALSE, pid); + /* No documented "no such process" code, but other program use this: */ + if (!h) + return ErrCode() != ERROR_INVALID_PARAMETER; + /* A process exists until all handles to it close. Has it exited? */ + ret = WaitForSingleObject(h, 0) != 0; + CloseHandle(h); + } + return ret; +#else + for (;;) { + int rc; + struct flock lock_info; + memset(&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = pid; + lock_info.l_len = 1; + if ((rc = fcntl(env->me_lfd, op, &lock_info)) == 0) { + if (op == F_GETLK && lock_info.l_type != F_UNLCK) + rc = -1; + } else if ((rc = ErrCode()) == EINTR) { + continue; + } + return rc; + } +#endif +} + +/** Common code for #mdb_txn_begin() and #mdb_txn_renew(). + * @param[in] txn the transaction handle to initialize + * @return 0 on success, non-zero on failure. + */ +static int +mdb_txn_renew0(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_txninfo *ti = env->me_txns; + MDB_meta *meta; + unsigned int i, nr; + uint16_t x; + int rc, new_notls = 0; + + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; + txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ + + if (txn->mt_flags & MDB_TXN_RDONLY) { + if (!ti) { + meta = env->me_metas[ mdb_env_pick_meta(env) ]; + txn->mt_txnid = meta->mm_txnid; + txn->mt_u.reader = NULL; + } else { + MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader : + pthread_getspecific(env->me_txkey); + if (r) { + if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1) + return MDB_BAD_RSLOT; + } else { + MDB_PID_T pid = env->me_pid; + MDB_THR_T tid = pthread_self(); + mdb_mutex_t rmutex = MDB_MUTEX(env, r); + + if (!env->me_live_reader) { + rc = mdb_reader_pid(env, Pidset, pid); + if (rc) + return rc; + env->me_live_reader = 1; + } + + if (LOCK_MUTEX(rc, env, rmutex)) + return rc; + nr = ti->mti_numreaders; + for (i=0; imti_readers[i].mr_pid == 0) + break; + if (i == env->me_maxreaders) { + UNLOCK_MUTEX(rmutex); + return MDB_READERS_FULL; + } + ti->mti_readers[i].mr_pid = pid; + ti->mti_readers[i].mr_tid = tid; + if (i == nr) + ti->mti_numreaders = ++nr; + /* Save numreaders for un-mutexed mdb_env_close() */ + env->me_numreaders = nr; + UNLOCK_MUTEX(rmutex); + + r = &ti->mti_readers[i]; + new_notls = (env->me_flags & MDB_NOTLS); + if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { + r->mr_pid = 0; + return rc; + } + } + txn->mt_txnid = r->mr_txnid = ti->mti_txnid; + txn->mt_u.reader = r; + meta = env->me_metas[txn->mt_txnid & 1]; + } + } else { + if (ti) { + if (LOCK_MUTEX(rc, env, MDB_MUTEX(env, w))) + return rc; + + txn->mt_txnid = ti->mti_txnid; + meta = env->me_metas[txn->mt_txnid & 1]; + } else { + meta = env->me_metas[ mdb_env_pick_meta(env) ]; + txn->mt_txnid = meta->mm_txnid; + } + txn->mt_txnid++; +#if MDB_DEBUG + if (txn->mt_txnid == mdb_debug_start) + mdb_debug = 1; +#endif + txn->mt_dirty_room = MDB_IDL_UM_MAX; + txn->mt_u.dirty_list = env->me_dirty_list; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_free_pgs = env->me_free_pgs; + txn->mt_free_pgs[0] = 0; + txn->mt_spill_pgs = NULL; + env->me_txn = txn; + memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); + } + + /* Copy the DB info and flags */ + memcpy(txn->mt_dbs, meta->mm_dbs, 2 * sizeof(MDB_db)); + + /* Moved to here to avoid a data race in read TXNs */ + txn->mt_next_pgno = meta->mm_last_pg+1; + + for (i=2; imt_numdbs; i++) { + x = env->me_dbflags[i]; + txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; + txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_STALE : 0; + } + txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; + + if (env->me_maxpg < txn->mt_next_pgno) { + mdb_txn_reset0(txn, "renew0-mapfail"); + if (new_notls) { + txn->mt_u.reader->mr_pid = 0; + txn->mt_u.reader = NULL; + } + return MDB_MAP_RESIZED; + } + + return MDB_SUCCESS; +} + +int +mdb_txn_renew(MDB_txn *txn) +{ + int rc; + + if (!txn || txn->mt_dbxs) /* A reset txn has mt_dbxs==NULL */ + return EINVAL; + + if (txn->mt_env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + return MDB_PANIC; + } + + rc = mdb_txn_renew0(txn); + if (rc == MDB_SUCCESS) { + DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); + } + return rc; +} + +int +mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) +{ + MDB_txn *txn; + MDB_ntxn *ntxn; + int rc, size, tsize = sizeof(MDB_txn); + + if (env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + return MDB_PANIC; + } + if ((env->me_flags & MDB_RDONLY) && !(flags & MDB_RDONLY)) + return EACCES; + if (parent) { + /* Nested transactions: Max 1 child, write txns only, no writemap */ + if (parent->mt_child || + (flags & MDB_RDONLY) || + (parent->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) || + (env->me_flags & MDB_WRITEMAP)) + { + return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN; + } + tsize = sizeof(MDB_ntxn); + } + size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1); + if (!(flags & MDB_RDONLY)) { + if (!parent) { + txn = env->me_txn0; + txn->mt_flags = 0; + goto ok; + } + size += env->me_maxdbs * sizeof(MDB_cursor *); + /* child txns use parent's dbiseqs */ + if (!parent) + size += env->me_maxdbs * sizeof(unsigned int); + } + + if ((txn = calloc(1, size)) == NULL) { + DPRINTF(("calloc: %s", strerror(errno))); + return ENOMEM; + } + txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); + if (flags & MDB_RDONLY) { + txn->mt_flags |= MDB_TXN_RDONLY; + txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = env->me_dbiseqs; + } else { + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + if (parent) { + txn->mt_dbiseqs = parent->mt_dbiseqs; + txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); + } else { + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + } + } + txn->mt_env = env; + +ok: + if (parent) { + unsigned int i; + txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); + if (!txn->mt_u.dirty_list || + !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) + { + free(txn->mt_u.dirty_list); + free(txn); + return ENOMEM; + } + txn->mt_txnid = parent->mt_txnid; + txn->mt_dirty_room = parent->mt_dirty_room; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_spill_pgs = NULL; + txn->mt_next_pgno = parent->mt_next_pgno; + parent->mt_child = txn; + txn->mt_parent = parent; + txn->mt_numdbs = parent->mt_numdbs; + txn->mt_flags = parent->mt_flags; + txn->mt_dbxs = parent->mt_dbxs; + memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + /* Copy parent's mt_dbflags, but clear DB_NEW */ + for (i=0; imt_numdbs; i++) + txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW; + rc = 0; + ntxn = (MDB_ntxn *)txn; + ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */ + if (env->me_pghead) { + size = MDB_IDL_SIZEOF(env->me_pghead); + env->me_pghead = mdb_midl_alloc(env->me_pghead[0]); + if (env->me_pghead) + memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size); + else + rc = ENOMEM; + } + if (!rc) + rc = mdb_cursor_shadow(parent, txn); + if (rc) + mdb_txn_reset0(txn, "beginchild-fail"); + } else { + rc = mdb_txn_renew0(txn); + } + if (rc) { + if (txn != env->me_txn0) + free(txn); + } else { + *ret = txn; + DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); + } + + return rc; +} + +MDB_env * +mdb_txn_env(MDB_txn *txn) +{ + if(!txn) return NULL; + return txn->mt_env; +} + +/** Export or close DBI handles opened in this txn. */ +static void +mdb_dbis_update(MDB_txn *txn, int keep) +{ + int i; + MDB_dbi n = txn->mt_numdbs; + MDB_env *env = txn->mt_env; + unsigned char *tdbflags = txn->mt_dbflags; + + for (i = n; --i >= 2;) { + if (tdbflags[i] & DB_NEW) { + if (keep) { + env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; + } else { + char *ptr = env->me_dbxs[i].md_name.mv_data; + if (ptr) { + env->me_dbxs[i].md_name.mv_data = NULL; + env->me_dbxs[i].md_name.mv_size = 0; + env->me_dbflags[i] = 0; + env->me_dbiseqs[i]++; + free(ptr); + } + } + } + } + if (keep && env->me_numdbs < n) + env->me_numdbs = n; +} + +/** Common code for #mdb_txn_reset() and #mdb_txn_abort(). + * May be called twice for readonly txns: First reset it, then abort. + * @param[in] txn the transaction handle to reset + * @param[in] act why the transaction is being reset + */ +static void +mdb_txn_reset0(MDB_txn *txn, const char *act) +{ + MDB_env *env = txn->mt_env; + + /* Close any DBI handles opened in this txn */ + mdb_dbis_update(txn, 0); + + DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + act, txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + if (txn->mt_u.reader) { + txn->mt_u.reader->mr_txnid = (txnid_t)-1; + if (!(env->me_flags & MDB_NOTLS)) + txn->mt_u.reader = NULL; /* txn does not own reader */ + } + txn->mt_numdbs = 0; /* close nothing if called again */ + txn->mt_dbxs = NULL; /* mark txn as reset */ + } else { + pgno_t *pghead = env->me_pghead; + env->me_pghead = NULL; + env->me_pglast = 0; + + if (!(env->me_flags & MDB_WRITEMAP)) { + mdb_dlist_free(txn); + } + + if (!txn->mt_parent) { + if (mdb_midl_shrink(&txn->mt_free_pgs)) + env->me_free_pgs = txn->mt_free_pgs; + + env->me_txn = NULL; + /* The writer mutex was locked in mdb_txn_begin. */ + if (env->me_txns) + UNLOCK_MUTEX(MDB_MUTEX(env, w)); + } + + mdb_cursors_close(txn, 0); + + mdb_midl_free(pghead); + + if (txn->mt_parent) { + txn->mt_parent->mt_child = NULL; + env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; + mdb_midl_free(txn->mt_free_pgs); + mdb_midl_free(txn->mt_spill_pgs); + free(txn->mt_u.dirty_list); + } + } +} + +void +mdb_txn_reset(MDB_txn *txn) +{ + if (txn == NULL) + return; + + /* This call is only valid for read-only txns */ + if (!(txn->mt_flags & MDB_TXN_RDONLY)) + return; + + mdb_txn_reset0(txn, "reset"); +} + +void +mdb_txn_abort(MDB_txn *txn) +{ + if (txn == NULL) + return; + + if (txn->mt_child) + mdb_txn_abort(txn->mt_child); + + mdb_txn_reset0(txn, "abort"); + /* Free reader slot tied to this txn (if MDB_NOTLS && writable FS) */ + if ((txn->mt_flags & MDB_TXN_RDONLY) && txn->mt_u.reader) + txn->mt_u.reader->mr_pid = 0; + + if (txn != txn->mt_env->me_txn0) + free(txn); +} + +/** Save the freelist as of this transaction to the freeDB. + * This changes the freelist. Keep trying until it stabilizes. + */ +static int +mdb_freelist_save(MDB_txn *txn) +{ + /* env->me_pghead[] can grow and shrink during this call. + * env->me_pglast and txn->mt_free_pgs[] can only grow. + * Page numbers cannot disappear from txn->mt_free_pgs[]. + */ + MDB_cursor mc; + MDB_env *env = txn->mt_env; + int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; + txnid_t pglast = 0, head_id = 0; + pgno_t freecnt = 0, *free_pgs, *mop; + ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; + + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + + if (env->me_pghead) { + /* Make sure first page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (!env->me_pghead && txn->mt_loose_pgs) { + /* Put loose page numbers in mt_free_pgs, since + * we may be unable to return them to me_pghead. + */ + MDB_page *mp = txn->mt_loose_pgs; + if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) + return rc; + for (; mp; mp = NEXT_LOOSE_PAGE(mp)) + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + } + + /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ + clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) + ? SSIZE_MAX : maxfree_1pg; + + for (;;) { + /* Come back here after each Put() in case freelist changed */ + MDB_val key, data; + pgno_t *pgs; + ssize_t j; + + /* If using records from freeDB which we have not yet + * deleted, delete them and any we reserved for me_pghead. + */ + while (pglast < env->me_pglast) { + rc = mdb_cursor_first(&mc, &key, NULL); + if (rc) + return rc; + pglast = head_id = *(txnid_t *)key.mv_data; + total_room = head_room = 0; + mdb_tassert(txn, pglast <= env->me_pglast); + rc = mdb_cursor_del(&mc, 0); + if (rc) + return rc; + } + + /* Save the IDL of pages freed by this txn, to a single record */ + if (freecnt < txn->mt_free_pgs[0]) { + if (!freecnt) { + /* Make sure last page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_LAST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + free_pgs = txn->mt_free_pgs; + /* Write to last page of freeDB */ + key.mv_size = sizeof(txn->mt_txnid); + key.mv_data = &txn->mt_txnid; + do { + freecnt = free_pgs[0]; + data.mv_size = MDB_IDL_SIZEOF(free_pgs); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* Retry if mt_free_pgs[] grew during the Put() */ + free_pgs = txn->mt_free_pgs; + } while (freecnt < free_pgs[0]); + mdb_midl_sort(free_pgs); + memcpy(data.mv_data, free_pgs, data.mv_size); +#if (MDB_DEBUG) > 1 + { + unsigned int i = free_pgs[0]; + DPRINTF(("IDL write txn %"Z"u root %"Z"u num %u", + txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); + for (; i; i--) + DPRINTF(("IDL %"Z"u", free_pgs[i])); + } +#endif + continue; + } + + mop = env->me_pghead; + mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; + + /* Reserve records for me_pghead[]. Split it if multi-page, + * to avoid searching freeDB for a page range. Use keys in + * range [1,me_pglast]: Smaller than txnid of oldest reader. + */ + if (total_room >= mop_len) { + if (total_room == mop_len || --more < 0) + break; + } else if (head_room >= maxfree_1pg && head_id > 1) { + /* Keep current record (overflow page), add a new one */ + head_id--; + head_room = 0; + } + /* (Re)write {key = head_id, IDL length = head_room} */ + total_room -= head_room; + head_room = mop_len - total_room; + if (head_room > maxfree_1pg && head_id > 1) { + /* Overflow multi-page for part of me_pghead */ + head_room /= head_id; /* amortize page sizes */ + head_room += maxfree_1pg - head_room % (maxfree_1pg + 1); + } else if (head_room < 0) { + /* Rare case, not bothering to delete this record */ + head_room = 0; + } + key.mv_size = sizeof(head_id); + key.mv_data = &head_id; + data.mv_size = (head_room + 1) * sizeof(pgno_t); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* IDL is initially empty, zero out at least the length */ + pgs = (pgno_t *)data.mv_data; + j = head_room > clean_limit ? head_room : 0; + do { + pgs[j] = 0; + } while (--j >= 0); + total_room += head_room; + } + + /* Return loose page numbers to me_pghead, though usually none are + * left at this point. The pages themselves remain in dirty_list. + */ + if (txn->mt_loose_pgs) { + MDB_page *mp = txn->mt_loose_pgs; + unsigned count = txn->mt_loose_count; + MDB_IDL loose; + /* Room for loose pages + temp IDL with same */ + if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) + return rc; + mop = env->me_pghead; + loose = mop + MDB_IDL_ALLOCLEN(mop) - count; + for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) + loose[ ++count ] = mp->mp_pgno; + loose[0] = count; + mdb_midl_sort(loose); + mdb_midl_xmerge(mop, loose); + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + mop_len = mop[0]; + } + + /* Fill in the reserved me_pghead records */ + rc = MDB_SUCCESS; + if (mop_len) { + MDB_val key, data; + + mop += mop_len; + rc = mdb_cursor_first(&mc, &key, &data); + for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { + txnid_t id = *(txnid_t *)key.mv_data; + ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; + MDB_ID save; + + mdb_tassert(txn, len >= 0 && id <= env->me_pglast); + key.mv_data = &id; + if (len > mop_len) { + len = mop_len; + data.mv_size = (len + 1) * sizeof(MDB_ID); + } + data.mv_data = mop -= len; + save = mop[0]; + mop[0] = len; + rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT); + mop[0] = save; + if (rc || !(mop_len -= len)) + break; + } + } + return rc; +} + +/** Flush (some) dirty pages to the map, after clearing their dirty flag. + * @param[in] txn the transaction that's being committed + * @param[in] keep number of initial pages in dirty_list to keep dirty. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_flush(MDB_txn *txn, int keep) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned psize = env->me_psize, j; + int i, pagecount = dl[0].mid, rc; + size_t size = 0, pos = 0; + pgno_t pgno = 0; + MDB_page *dp = NULL; +#ifdef _WIN32 + OVERLAPPED ov; +#else + struct iovec iov[MDB_COMMIT_PAGES]; + ssize_t wpos = 0, wsize = 0, wres; + size_t next_pos = 1; /* impossible pos, so pos != next_pos */ + int n = 0; +#endif + + j = i = keep; + + if (env->me_flags & MDB_WRITEMAP) { + /* Clear dirty flags */ + while (++i <= pagecount) { + dp = dl[i].mptr; + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE|P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[++j] = dl[i]; + continue; + } + dp->mp_flags &= ~P_DIRTY; + } + goto done; + } + + /* Write the pages */ + for (;;) { + if (++i <= pagecount) { + dp = dl[i].mptr; + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE|P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[i].mid = 0; + continue; + } + pgno = dl[i].mid; + /* clear dirty flag */ + dp->mp_flags &= ~P_DIRTY; + pos = pgno * psize; + size = psize; + if (IS_OVERFLOW(dp)) size *= dp->mp_pages; + } +#ifdef _WIN32 + else break; + + /* Windows actually supports scatter/gather I/O, but only on + * unbuffered file handles. Since we're relying on the OS page + * cache for all our data, that's self-defeating. So we just + * write pages one at a time. We use the ov structure to set + * the write offset, to at least save the overhead of a Seek + * system call. + */ + DPRINTF(("committing page %"Z"u", pgno)); + memset(&ov, 0, sizeof(ov)); + ov.Offset = pos & 0xffffffff; + ov.OffsetHigh = pos >> 16 >> 16; + if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) { + rc = ErrCode(); + DPRINTF(("WriteFile: %d", rc)); + return rc; + } +#else + /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ + if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { + if (n) { + /* Write previous page(s) */ +#ifdef MDB_USE_PWRITEV + wres = pwritev(env->me_fd, iov, n, wpos); +#else + if (n == 1) { + wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos); + } else { + if (lseek(env->me_fd, wpos, SEEK_SET) == -1) { + rc = ErrCode(); + DPRINTF(("lseek: %s", strerror(rc))); + return rc; + } + wres = writev(env->me_fd, iov, n); + } +#endif + if (wres != wsize) { + if (wres < 0) { + rc = ErrCode(); + DPRINTF(("Write error: %s", strerror(rc))); + } else { + rc = EIO; /* TODO: Use which error code? */ + DPUTS("short write, filesystem full?"); + } + return rc; + } + n = 0; + } + if (i > pagecount) + break; + wpos = pos; + wsize = 0; + } + DPRINTF(("committing page %"Z"u", pgno)); + next_pos = pos + size; + iov[n].iov_len = size; + iov[n].iov_base = (char *)dp; + wsize += size; + n++; +#endif /* _WIN32 */ + } + + /* MIPS has cache coherency issues, this is a no-op everywhere else + * Note: for any size >= on-chip cache size, entire on-chip cache is + * flushed. + */ + CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE); + + for (i = keep; ++i <= pagecount; ) { + dp = dl[i].mptr; + /* This is a page we skipped above */ + if (!dl[i].mid) { + dl[++j] = dl[i]; + dl[j].mid = dp->mp_pgno; + continue; + } + mdb_dpage_free(env, dp); + } + +done: + i--; + txn->mt_dirty_room += i - j; + dl[0].mid = j; + return MDB_SUCCESS; +} + +int +mdb_txn_commit(MDB_txn *txn) +{ + int rc; + unsigned int i; + MDB_env *env; + + if (txn == NULL || txn->mt_env == NULL) + return EINVAL; + + if (txn->mt_child) { + rc = mdb_txn_commit(txn->mt_child); + txn->mt_child = NULL; + if (rc) + goto fail; + } + + env = txn->mt_env; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + mdb_dbis_update(txn, 1); + txn->mt_numdbs = 2; /* so txn_abort() doesn't close any new handles */ + mdb_txn_abort(txn); + return MDB_SUCCESS; + } + + if (F_ISSET(txn->mt_flags, MDB_TXN_ERROR)) { + DPUTS("error flag is set, can't commit"); + if (txn->mt_parent) + txn->mt_parent->mt_flags |= MDB_TXN_ERROR; + rc = MDB_BAD_TXN; + goto fail; + } + + if (txn->mt_parent) { + MDB_txn *parent = txn->mt_parent; + MDB_page **lp; + MDB_ID2L dst, src; + MDB_IDL pspill; + unsigned x, y, len, ps_len; + + /* Append our free list to parent's */ + rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs); + if (rc) + goto fail; + mdb_midl_free(txn->mt_free_pgs); + /* Failures after this must either undo the changes + * to the parent or set MDB_TXN_ERROR in the parent. + */ + + parent->mt_next_pgno = txn->mt_next_pgno; + parent->mt_flags = txn->mt_flags; + + /* Merge our cursors into parent's and close them */ + mdb_cursors_close(txn, 1); + + /* Update parent's DB table. */ + memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + parent->mt_numdbs = txn->mt_numdbs; + parent->mt_dbflags[0] = txn->mt_dbflags[0]; + parent->mt_dbflags[1] = txn->mt_dbflags[1]; + for (i=2; imt_numdbs; i++) { + /* preserve parent's DB_NEW status */ + x = parent->mt_dbflags[i] & DB_NEW; + parent->mt_dbflags[i] = txn->mt_dbflags[i] | x; + } + + dst = parent->mt_u.dirty_list; + src = txn->mt_u.dirty_list; + /* Remove anything in our dirty list from parent's spill list */ + if ((pspill = parent->mt_spill_pgs) && (ps_len = pspill[0])) { + x = y = ps_len; + pspill[0] = (pgno_t)-1; + /* Mark our dirty pages as deleted in parent spill list */ + for (i=0, len=src[0].mid; ++i <= len; ) { + MDB_ID pn = src[i].mid << 1; + while (pn > pspill[x]) + x--; + if (pn == pspill[x]) { + pspill[x] = 1; + y = --x; + } + } + /* Squash deleted pagenums if we deleted any */ + for (x=y; ++x <= ps_len; ) + if (!(pspill[x] & 1)) + pspill[++y] = pspill[x]; + pspill[0] = y; + } + + /* Find len = length of merging our dirty list with parent's */ + x = dst[0].mid; + dst[0].mid = 0; /* simplify loops */ + if (parent->mt_parent) { + len = x + src[0].mid; + y = mdb_mid2l_search(src, dst[x].mid + 1) - 1; + for (i = x; y && i; y--) { + pgno_t yp = src[y].mid; + while (yp < dst[i].mid) + i--; + if (yp == dst[i].mid) { + i--; + len--; + } + } + } else { /* Simplify the above for single-ancestor case */ + len = MDB_IDL_UM_MAX - txn->mt_dirty_room; + } + /* Merge our dirty list with parent's */ + y = src[0].mid; + for (i = len; y; dst[i--] = src[y--]) { + pgno_t yp = src[y].mid; + while (yp < dst[x].mid) + dst[i--] = dst[x--]; + if (yp == dst[x].mid) + free(dst[x--].mptr); + } + mdb_tassert(txn, i == x); + dst[0].mid = len; + free(txn->mt_u.dirty_list); + parent->mt_dirty_room = txn->mt_dirty_room; + if (txn->mt_spill_pgs) { + if (parent->mt_spill_pgs) { + /* TODO: Prevent failure here, so parent does not fail */ + rc = mdb_midl_append_list(&parent->mt_spill_pgs, txn->mt_spill_pgs); + if (rc) + parent->mt_flags |= MDB_TXN_ERROR; + mdb_midl_free(txn->mt_spill_pgs); + mdb_midl_sort(parent->mt_spill_pgs); + } else { + parent->mt_spill_pgs = txn->mt_spill_pgs; + } + } + + /* Append our loose page list to parent's */ + for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(lp)) + ; + *lp = txn->mt_loose_pgs; + parent->mt_loose_count += txn->mt_loose_count; + + parent->mt_child = NULL; + mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); + free(txn); + return rc; + } + + if (txn != env->me_txn) { + DPUTS("attempt to commit unknown transaction"); + rc = EINVAL; + goto fail; + } + + mdb_cursors_close(txn, 0); + + if (!txn->mt_u.dirty_list[0].mid && + !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) + goto done; + + DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); + + /* Update DB root pointers */ + if (txn->mt_numdbs > 2) { + MDB_cursor mc; + MDB_dbi i; + MDB_val data; + data.mv_size = sizeof(MDB_db); + + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + for (i = 2; i < txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + if (TXN_DBI_CHANGED(txn, i)) { + rc = MDB_BAD_DBI; + goto fail; + } + data.mv_data = &txn->mt_dbs[i]; + rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, 0); + if (rc) + goto fail; + } + } + } + + rc = mdb_freelist_save(txn); + if (rc) + goto fail; + + mdb_midl_free(env->me_pghead); + env->me_pghead = NULL; + if (mdb_midl_shrink(&txn->mt_free_pgs)) + env->me_free_pgs = txn->mt_free_pgs; + +#if (MDB_DEBUG) > 2 + mdb_audit(txn); +#endif + + if ((rc = mdb_page_flush(txn, 0)) || + (rc = mdb_env_sync(env, 0)) || + (rc = mdb_env_write_meta(txn))) + goto fail; + + /* Free P_LOOSE pages left behind in dirty_list */ + if (!(env->me_flags & MDB_WRITEMAP)) + mdb_dlist_free(txn); + +done: + env->me_pglast = 0; + env->me_txn = NULL; + mdb_dbis_update(txn, 1); + + if (env->me_txns) + UNLOCK_MUTEX(MDB_MUTEX(env, w)); + if (txn != env->me_txn0) + free(txn); + + return MDB_SUCCESS; + +fail: + mdb_txn_abort(txn); + return rc; +} + +/** Read the environment parameters of a DB environment before + * mapping it into memory. + * @param[in] env the environment handle + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_read_header(MDB_env *env, MDB_meta *meta) +{ + MDB_metabuf pbuf; + MDB_page *p; + MDB_meta *m; + int i, rc, off; + enum { Size = sizeof(pbuf) }; + + /* We don't know the page size yet, so use a minimum value. + * Read both meta pages so we can use the latest one. + */ + + for (i=off=0; i<2; i++, off = meta->mm_psize) { +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + rc = ReadFile(env->me_fd, &pbuf, Size, &len, &ov) ? (int)len : -1; + if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) + rc = 0; +#else + rc = pread(env->me_fd, &pbuf, Size, off); +#endif + if (rc != Size) { + if (rc == 0 && off == 0) + return ENOENT; + rc = rc < 0 ? (int) ErrCode() : MDB_INVALID; + DPRINTF(("read: %s", mdb_strerror(rc))); + return rc; + } + + p = (MDB_page *)&pbuf; + + if (!F_ISSET(p->mp_flags, P_META)) { + DPRINTF(("page %"Z"u not a meta page", p->mp_pgno)); + return MDB_INVALID; + } + + m = METADATA(p); + if (m->mm_magic != MDB_MAGIC) { + DPUTS("meta has invalid magic"); + return MDB_INVALID; + } + + if (m->mm_version != MDB_DATA_VERSION) { + DPRINTF(("database is version %u, expected version %u", + m->mm_version, MDB_DATA_VERSION)); + return MDB_VERSION_MISMATCH; + } + + if (off == 0 || m->mm_txnid > meta->mm_txnid) + *meta = *m; + } + return 0; +} + +static void ESECT +mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) +{ + meta->mm_magic = MDB_MAGIC; + meta->mm_version = MDB_DATA_VERSION; + meta->mm_mapsize = env->me_mapsize; + meta->mm_psize = env->me_psize; + meta->mm_last_pg = 1; + meta->mm_flags = env->me_flags & 0xffff; + meta->mm_flags |= MDB_INTEGERKEY; + meta->mm_dbs[0].md_root = P_INVALID; + meta->mm_dbs[1].md_root = P_INVALID; +} + +/** Write the environment parameters of a freshly created DB environment. + * @param[in] env the environment handle + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_init_meta(MDB_env *env, MDB_meta *meta) +{ + MDB_page *p, *q; + int rc; + unsigned int psize; +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + ov.Offset = pos; \ + rc = WriteFile(fd, ptr, size, &len, &ov); } while(0) +#else + int len; +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + len = pwrite(fd, ptr, size, pos); \ + rc = (len >= 0); } while(0) +#endif + + DPUTS("writing new meta page"); + + psize = env->me_psize; + + mdb_env_init_meta0(env, meta); + + p = calloc(2, psize); + p->mp_pgno = 0; + p->mp_flags = P_META; + *(MDB_meta *)METADATA(p) = *meta; + + q = (MDB_page *)((char *)p + psize); + q->mp_pgno = 1; + q->mp_flags = P_META; + *(MDB_meta *)METADATA(q) = *meta; + + DO_PWRITE(rc, env->me_fd, p, psize * 2, len, 0); + if (!rc) + rc = ErrCode(); + else if ((unsigned) len == psize * 2) + rc = MDB_SUCCESS; + else + rc = ENOSPC; + free(p); + return rc; +} + +/** Update the environment info to commit a transaction. + * @param[in] txn the transaction that's being committed + * @return 0 on success, non-zero on failure. + */ +static int +mdb_env_write_meta(MDB_txn *txn) +{ + MDB_env *env; + MDB_meta meta, metab, *mp; + size_t mapsize; + off_t off; + int rc, len, toggle; + char *ptr; + HANDLE mfd; +#ifdef _WIN32 + OVERLAPPED ov; +#else + int r2; +#endif + + toggle = txn->mt_txnid & 1; + DPRINTF(("writing meta page %d for root page %"Z"u", + toggle, txn->mt_dbs[MAIN_DBI].md_root)); + + env = txn->mt_env; + mp = env->me_metas[toggle]; + mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; + /* Persist any increases of mapsize config */ + if (mapsize < env->me_mapsize) + mapsize = env->me_mapsize; + + if (env->me_flags & MDB_WRITEMAP) { + mp->mm_mapsize = mapsize; + mp->mm_dbs[0] = txn->mt_dbs[0]; + mp->mm_dbs[1] = txn->mt_dbs[1]; + mp->mm_last_pg = txn->mt_next_pgno - 1; + mp->mm_txnid = txn->mt_txnid; + if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { + unsigned meta_size = env->me_psize; + rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; + ptr = env->me_map; + if (toggle) { +#ifndef _WIN32 /* POSIX msync() requires ptr = start of OS page */ + if (meta_size < env->me_os_psize) + meta_size += meta_size; + else +#endif + ptr += meta_size; + } + if (MDB_MSYNC(ptr, meta_size, rc)) { + rc = ErrCode(); + goto fail; + } + } + goto done; + } + metab.mm_txnid = env->me_metas[toggle]->mm_txnid; + metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg; + + meta.mm_mapsize = mapsize; + meta.mm_dbs[0] = txn->mt_dbs[0]; + meta.mm_dbs[1] = txn->mt_dbs[1]; + meta.mm_last_pg = txn->mt_next_pgno - 1; + meta.mm_txnid = txn->mt_txnid; + + off = offsetof(MDB_meta, mm_mapsize); + ptr = (char *)&meta + off; + len = sizeof(MDB_meta) - off; + if (toggle) + off += env->me_psize; + off += PAGEHDRSZ; + + /* Write to the SYNC fd */ + mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ? + env->me_fd : env->me_mfd; +#ifdef _WIN32 + { + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov)) + rc = -1; + } +#else + rc = pwrite(mfd, ptr, len, off); +#endif + if (rc != len) { + rc = rc < 0 ? ErrCode() : EIO; + DPUTS("write failed, disk error?"); + /* On a failure, the pagecache still contains the new data. + * Write some old data back, to prevent it from being used. + * Use the non-SYNC fd; we know it will fail anyway. + */ + meta.mm_last_pg = metab.mm_last_pg; + meta.mm_txnid = metab.mm_txnid; +#ifdef _WIN32 + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + WriteFile(env->me_fd, ptr, len, NULL, &ov); +#else + r2 = pwrite(env->me_fd, ptr, len, off); + (void)r2; /* Silence warnings. We don't care about pwrite's return value */ +#endif +fail: + env->me_flags |= MDB_FATAL_ERROR; + return rc; + } + /* MIPS has cache coherency issues, this is a no-op everywhere else */ + CACHEFLUSH(env->me_map + off, len, DCACHE); +done: + /* Memory ordering issues are irrelevant; since the entire writer + * is wrapped by wmutex, all of these changes will become visible + * after the wmutex is unlocked. Since the DB is multi-version, + * readers will get consistent data regardless of how fresh or + * how stale their view of these values is. + */ + if (env->me_txns) + env->me_txns->mti_txnid = txn->mt_txnid; + + return MDB_SUCCESS; +} + +/** Check both meta pages to see which one is newer. + * @param[in] env the environment handle + * @return meta toggle (0 or 1). + */ +static int +mdb_env_pick_meta(const MDB_env *env) +{ + return (env->me_metas[0]->mm_txnid < env->me_metas[1]->mm_txnid); +} + +int ESECT +mdb_env_create(MDB_env **env) +{ + MDB_env *e; + + e = calloc(1, sizeof(MDB_env)); + if (!e) + return ENOMEM; + + e->me_maxreaders = DEFAULT_READERS; + e->me_maxdbs = e->me_numdbs = 2; + e->me_fd = INVALID_HANDLE_VALUE; + e->me_lfd = INVALID_HANDLE_VALUE; + e->me_mfd = INVALID_HANDLE_VALUE; +#ifdef MDB_USE_POSIX_SEM + e->me_rmutex = SEM_FAILED; + e->me_wmutex = SEM_FAILED; +#endif + e->me_pid = getpid(); + GET_PAGESIZE(e->me_os_psize); + VGMEMP_CREATE(e,0,0); + *env = e; + return MDB_SUCCESS; +} + +static int ESECT +mdb_env_map(MDB_env *env, void *addr) +{ + MDB_page *p; + unsigned int flags = env->me_flags; +#ifdef _WIN32 + int rc; + HANDLE mh; + LONG sizelo, sizehi; + size_t msize; + + if (flags & MDB_RDONLY) { + /* Don't set explicit map size, use whatever exists */ + msize = 0; + sizelo = 0; + sizehi = 0; + } else { + msize = env->me_mapsize; + sizelo = msize & 0xffffffff; + sizehi = msize >> 16 >> 16; /* only needed on Win64 */ + + /* Windows won't create mappings for zero length files. + * and won't map more than the file size. + * Just set the maxsize right now. + */ + if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo + || !SetEndOfFile(env->me_fd) + || SetFilePointer(env->me_fd, 0, NULL, 0) != 0) + return ErrCode(); + } + + mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ? + PAGE_READWRITE : PAGE_READONLY, + sizehi, sizelo, NULL); + if (!mh) + return ErrCode(); + env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ? + FILE_MAP_WRITE : FILE_MAP_READ, + 0, 0, msize, addr); + rc = env->me_map ? 0 : ErrCode(); + CloseHandle(mh); + if (rc) + return rc; +#else + int prot = PROT_READ; + if (flags & MDB_WRITEMAP) { + prot |= PROT_WRITE; + if (ftruncate(env->me_fd, env->me_mapsize) < 0) + return ErrCode(); + } + env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED, + env->me_fd, 0); + if (env->me_map == MAP_FAILED) { + env->me_map = NULL; + return ErrCode(); + } + + if (flags & MDB_NORDAHEAD) { + /* Turn off readahead. It's harmful when the DB is larger than RAM. */ +#ifdef MADV_RANDOM + madvise(env->me_map, env->me_mapsize, MADV_RANDOM); +#else +#ifdef POSIX_MADV_RANDOM + posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM); +#endif /* POSIX_MADV_RANDOM */ +#endif /* MADV_RANDOM */ + } +#endif /* _WIN32 */ + + /* Can happen because the address argument to mmap() is just a + * hint. mmap() can pick another, e.g. if the range is in use. + * The MAP_FIXED flag would prevent that, but then mmap could + * instead unmap existing pages to make room for the new map. + */ + if (addr && env->me_map != addr) + return EBUSY; /* TODO: Make a new MDB_* error code? */ + + p = (MDB_page *)env->me_map; + env->me_metas[0] = METADATA(p); + env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize); + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_mapsize(MDB_env *env, size_t size) +{ + /* If env is already open, caller is responsible for making + * sure there are no active txns. + */ + if (env->me_map) { + int rc; + void *old; + if (env->me_txn) + return EINVAL; + if (!size) + size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize; + else if (size < env->me_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; + if (size < minsize) + size = minsize; + } + munmap(env->me_map, env->me_mapsize); + env->me_mapsize = size; + old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; + rc = mdb_env_map(env, old); + if (rc) + return rc; + } + env->me_mapsize = size; + if (env->me_psize) + env->me_maxpg = env->me_mapsize / env->me_psize; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs) +{ + if (env->me_map) + return EINVAL; + env->me_maxdbs = dbs + 2; /* Named databases + main and free DB */ + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) +{ + if (env->me_map || readers < 1) + return EINVAL; + env->me_maxreaders = readers; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) +{ + if (!env || !readers) + return EINVAL; + *readers = env->me_maxreaders; + return MDB_SUCCESS; +} + +/** Further setup required for opening an LMDB environment + */ +static int ESECT +mdb_env_open2(MDB_env *env) +{ + unsigned int flags = env->me_flags; + int i, newenv = 0, rc; + MDB_meta meta; + +#ifdef _WIN32 + /* See if we should use QueryLimited */ + rc = GetVersion(); + if ((rc & 0xff) > 5) + env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION; + else + env->me_pidquery = PROCESS_QUERY_INFORMATION; +#endif /* _WIN32 */ + + memset(&meta, 0, sizeof(meta)); + + if ((i = mdb_env_read_header(env, &meta)) != 0) { + if (i != ENOENT) + return i; + DPUTS("new mdbenv"); + newenv = 1; + env->me_psize = env->me_os_psize; + if (env->me_psize > MAX_PAGESIZE) + env->me_psize = MAX_PAGESIZE; + } else { + env->me_psize = meta.mm_psize; + } + + /* Was a mapsize configured? */ + if (!env->me_mapsize) { + /* If this is a new environment, take the default, + * else use the size recorded in the existing env. + */ + env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize; + } else if (env->me_mapsize < meta.mm_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; + if (env->me_mapsize < minsize) + env->me_mapsize = minsize; + } + + rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); + if (rc) + return rc; + + if (newenv) { + if (flags & MDB_FIXEDMAP) + meta.mm_address = env->me_map; + i = mdb_env_init_meta(env, &meta); + if (i != MDB_SUCCESS) { + return i; + } + } + + env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; + env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2) + - sizeof(indx_t); +#if !(MDB_MAXKEYSIZE) + env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db)); +#endif + env->me_maxpg = env->me_mapsize / env->me_psize; + +#if MDB_DEBUG + { + int toggle = mdb_env_pick_meta(env); + MDB_db *db = &env->me_metas[toggle]->mm_dbs[MAIN_DBI]; + + DPRINTF(("opened database version %u, pagesize %u", + env->me_metas[0]->mm_version, env->me_psize)); + DPRINTF(("using meta page %d", toggle)); + DPRINTF(("depth: %u", db->md_depth)); + DPRINTF(("entries: %"Z"u", db->md_entries)); + DPRINTF(("branch pages: %"Z"u", db->md_branch_pages)); + DPRINTF(("leaf pages: %"Z"u", db->md_leaf_pages)); + DPRINTF(("overflow pages: %"Z"u", db->md_overflow_pages)); + DPRINTF(("root: %"Z"u", db->md_root)); + } +#endif + + return MDB_SUCCESS; +} + + +/** Release a reader thread's slot in the reader lock table. + * This function is called automatically when a thread exits. + * @param[in] ptr This points to the slot in the reader lock table. + */ +static void +mdb_env_reader_dest(void *ptr) +{ + MDB_reader *reader = ptr; + + reader->mr_pid = 0; +} + +#ifdef _WIN32 +/** Junk for arranging thread-specific callbacks on Windows. This is + * necessarily platform and compiler-specific. Windows supports up + * to 1088 keys. Let's assume nobody opens more than 64 environments + * in a single process, for now. They can override this if needed. + */ +#ifndef MAX_TLS_KEYS +#define MAX_TLS_KEYS 64 +#endif +static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS]; +static int mdb_tls_nkeys; + +static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr) +{ + int i; + switch(reason) { + case DLL_PROCESS_ATTACH: break; + case DLL_THREAD_ATTACH: break; + case DLL_THREAD_DETACH: + for (i=0; ime_txns->mti_txnid = env->me_metas[toggle]->mm_txnid; + +#ifdef _WIN32 + { + OVERLAPPED ov; + /* First acquire a shared lock. The Unlock will + * then release the existing exclusive lock. + */ + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + rc = ErrCode(); + } else { + UnlockFile(env->me_lfd, 0, 0, 1, 0); + *excl = 0; + } + } +#else + { + struct flock lock_info; + /* The shared lock replaces the existing lock */ + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_RDLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + *excl = rc ? -1 : 0; /* error may mean we lost the lock */ + } +#endif + + return rc; +} + +/** Try to get exlusive lock, otherwise shared. + * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. + */ +static int ESECT +mdb_env_excl_lock(MDB_env *env, int *excl) +{ + int rc = 0; +#ifdef _WIN32 + if (LockFile(env->me_lfd, 0, 0, 1, 0)) { + *excl = 1; + } else { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + *excl = 0; + } else { + rc = ErrCode(); + } + } +#else + struct flock lock_info; + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (!rc) { + *excl = 1; + } else +# ifdef MDB_USE_POSIX_SEM + if (*excl < 0) /* always true when !MDB_USE_POSIX_SEM */ +# endif + { + lock_info.l_type = F_RDLCK; + while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (rc == 0) + *excl = 0; + } +#endif + return rc; +} + +#ifdef MDB_USE_HASH +/* + * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code + * + * @(#) $Revision: 5.1 $ + * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ + * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ + * + * http://www.isthe.com/chongo/tech/comp/fnv/index.html + * + *** + * + * Please do not copyright this code. This code is in the public domain. + * + * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO + * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + * By: + * chongo /\oo/\ + * http://www.isthe.com/chongo/ + * + * Share and Enjoy! :-) + */ + +typedef unsigned long long mdb_hash_t; +#define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL) + +/** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer + * @param[in] val value to hash + * @param[in] hval initial value for hash + * @return 64 bit hash + * + * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the + * hval arg on the first call. + */ +static mdb_hash_t +mdb_hash_val(MDB_val *val, mdb_hash_t hval) +{ + unsigned char *s = (unsigned char *)val->mv_data; /* unsigned string */ + unsigned char *end = s + val->mv_size; + /* + * FNV-1a hash each octet of the string + */ + while (s < end) { + /* xor the bottom with the current octet */ + hval ^= (mdb_hash_t)*s++; + + /* multiply by the 64 bit FNV magic prime mod 2^64 */ + hval += (hval << 1) + (hval << 4) + (hval << 5) + + (hval << 7) + (hval << 8) + (hval << 40); + } + /* return our new hash value */ + return hval; +} + +/** Hash the string and output the encoded hash. + * This uses modified RFC1924 Ascii85 encoding to accommodate systems with + * very short name limits. We don't care about the encoding being reversible, + * we just want to preserve as many bits of the input as possible in a + * small printable string. + * @param[in] str string to hash + * @param[out] encbuf an array of 11 chars to hold the hash + */ +static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; + +static void +mdb_pack85(unsigned long l, char *out) +{ + int i; + + for (i=0; i<5; i++) { + *out++ = mdb_a85[l % 85]; + l /= 85; + } +} + +static void +mdb_hash_enc(MDB_val *val, char *encbuf) +{ + mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT); + + mdb_pack85(h, encbuf); + mdb_pack85(h>>32, encbuf+5); + encbuf[10] = '\0'; +} +#endif + +/** Open and/or initialize the lock region for the environment. + * @param[in] env The LMDB environment. + * @param[in] lpath The pathname of the file used for the lock region. + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[out] excl Resulting file lock type: -1 none, 0 shared, 1 exclusive + * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) +{ +#ifdef _WIN32 +# define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT +#else +# define MDB_ERRCODE_ROFS EROFS +#ifdef O_CLOEXEC /* Linux: Open file and set FD_CLOEXEC atomically */ +# define MDB_CLOEXEC O_CLOEXEC +#else + int fdflags; +# define MDB_CLOEXEC 0 +#endif +#endif + int rc; + off_t size, rsize; + +#ifdef _WIN32 + env->me_lfd = CreateFile(lpath, GENERIC_READ|GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, NULL); +#else + env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode); +#endif + if (env->me_lfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { + return MDB_SUCCESS; + } + goto fail_errno; + } +#if ! ((MDB_CLOEXEC) || defined(_WIN32)) + /* Lose record locks when exec*() */ + if ((fdflags = fcntl(env->me_lfd, F_GETFD) | FD_CLOEXEC) >= 0) + fcntl(env->me_lfd, F_SETFD, fdflags); +#endif + + if (!(env->me_flags & MDB_NOTLS)) { + rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); + if (rc) + goto fail; + env->me_flags |= MDB_ENV_TXKEY; +#ifdef _WIN32 + /* Windows TLS callbacks need help finding their TLS info. */ + if (mdb_tls_nkeys >= MAX_TLS_KEYS) { + rc = MDB_TLS_FULL; + goto fail; + } + mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey; +#endif + } + + /* Try to get exclusive lock. If we succeed, then + * nobody is using the lock region and we should initialize it. + */ + if ((rc = mdb_env_excl_lock(env, excl))) goto fail; + +#ifdef _WIN32 + size = GetFileSize(env->me_lfd, NULL); +#else + size = lseek(env->me_lfd, 0, SEEK_END); + if (size == -1) goto fail_errno; +#endif + rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo); + if (size < rsize && *excl > 0) { +#ifdef _WIN32 + if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != (DWORD)rsize + || !SetEndOfFile(env->me_lfd)) + goto fail_errno; +#else + if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno; +#endif + } else { + rsize = size; + size = rsize - sizeof(MDB_txninfo); + env->me_maxreaders = size/sizeof(MDB_reader) + 1; + } + { +#ifdef _WIN32 + HANDLE mh; + mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE, + 0, 0, NULL); + if (!mh) goto fail_errno; + env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL); + CloseHandle(mh); + if (!env->me_txns) goto fail_errno; +#else + void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED, + env->me_lfd, 0); + if (m == MAP_FAILED) goto fail_errno; + env->me_txns = m; +#endif + } + if (*excl > 0) { +#ifdef _WIN32 + BY_HANDLE_FILE_INFORMATION stbuf; + struct { + DWORD volume; + DWORD nhigh; + DWORD nlow; + } idbuf; + MDB_val val; + char encbuf[11]; + + if (!mdb_sec_inited) { + InitializeSecurityDescriptor(&mdb_null_sd, + SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE); + mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES); + mdb_all_sa.bInheritHandle = FALSE; + mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd; + mdb_sec_inited = 1; + } + if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.volume = stbuf.dwVolumeSerialNumber; + idbuf.nhigh = stbuf.nFileIndexHigh; + idbuf.nlow = stbuf.nFileIndexLow; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_enc(&val, encbuf); + sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf); + sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf); + env->me_rmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; + env->me_flags |= MDB_ROBUST; +#elif defined(MDB_USE_POSIX_SEM) + struct stat stbuf; + struct { + dev_t dev; + ino_t ino; + } idbuf; + MDB_val val; + char encbuf[11]; + +#if defined(__NetBSD__) +#define MDB_SHORT_SEMNAMES 1 /* limited to 14 chars */ +#endif + if (fstat(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.dev = stbuf.st_dev; + idbuf.ino = stbuf.st_ino; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_enc(&val, encbuf); +#ifdef MDB_SHORT_SEMNAMES + encbuf[9] = '\0'; /* drop name from 15 chars to 14 chars */ +#endif + sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf); + sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf); + /* Clean up after a previous run, if needed: Try to + * remove both semaphores before doing anything else. + */ + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + env->me_rmutex = sem_open(env->me_txns->mti_rmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#else /* MDB_USE_POSIX_SEM */ + pthread_mutexattr_t mattr; + + if ((rc = pthread_mutexattr_init(&mattr)) + || (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) +#ifdef MDB_ROBUST_SUPPORTED + || ((env->me_flags & MDB_ROBUST) && + (rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST))) +#endif + || (rc = pthread_mutex_init(&env->me_txns->mti_rmutex, &mattr)) + || (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr))) + goto fail; + pthread_mutexattr_destroy(&mattr); +#endif /* _WIN32 || MDB_USE_POSIX_SEM */ +#ifndef MDB_ROBUST_SUPPORTED + env->me_flags &= ~MDB_ROBUST; +#endif + + env->me_txns->mti_magic = MDB_MAGIC; + env->me_txns->mti_format = MDB_LOCK_FORMAT; + env->me_txns->mti_txnid = 0; + env->me_txns->mti_numreaders = 0; + env->me_txns->mti_flags = env->me_flags; + + } else { + if (env->me_txns->mti_magic != MDB_MAGIC) { + DPUTS("lock region has invalid magic"); + rc = MDB_INVALID; + goto fail; + } + if (env->me_txns->mti_format != MDB_LOCK_FORMAT) { + DPRINTF(("lock region has format+version 0x%x, expected 0x%x", + env->me_txns->mti_format, MDB_LOCK_FORMAT)); + rc = MDB_VERSION_MISMATCH; + goto fail; + } + rc = ErrCode(); + if (rc && rc != EACCES && rc != EAGAIN) { + goto fail; + } + env->me_flags = (env->me_flags & ~MDB_ROBUST) | + (env->me_txns->mti_flags & MDB_ROBUST); +#ifdef _WIN32 + env->me_rmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#endif + } + return MDB_SUCCESS; + +fail_errno: + rc = ErrCode(); +fail: + return rc; +} + + /** The name of the lock file in the DB environment */ +#define LOCKNAME "/lock.mdb" + /** The name of the data file in the DB environment */ +#define DATANAME "/data.mdb" + /** The suffix of the lock file when no subdir is used */ +#define LOCKSUFF "-lock" + /** Only a subset of the @ref mdb_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. + */ +#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) +#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|ROBUST_FLAG| \ + MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) +#ifdef MDB_ROBUST_SUPPORTED +#define ROBUST_FLAG MDB_ROBUST +#else +#define ROBUST_FLAG 0 +#endif + +#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) +# error "Persistent DB flags & env flags overlap, but both go in mm_flags" +#endif + +int ESECT +mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) +{ + int oflags, rc, len, excl = -1; + char *lpath, *dpath; + + if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) + return EINVAL; + + len = strlen(path); + if (flags & MDB_NOSUBDIR) { + rc = len + sizeof(LOCKSUFF) + len + 1; + } else { + rc = len + sizeof(LOCKNAME) + len + sizeof(DATANAME); + } + lpath = malloc(rc); + if (!lpath) + return ENOMEM; + if (flags & MDB_NOSUBDIR) { + dpath = lpath + len + sizeof(LOCKSUFF); + sprintf(lpath, "%s" LOCKSUFF, path); + strcpy(dpath, path); + } else { + dpath = lpath + len + sizeof(LOCKNAME); + sprintf(lpath, "%s" LOCKNAME, path); + sprintf(dpath, "%s" DATANAME, path); + } + + rc = MDB_SUCCESS; + flags |= env->me_flags; + if (flags & MDB_RDONLY) { + /* silently ignore WRITEMAP when we're only getting read access */ + flags &= ~MDB_WRITEMAP; + } else { + if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) && + (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) + rc = ENOMEM; + } + env->me_flags = flags |= MDB_ENV_ACTIVE; + if (rc) + goto leave; + + env->me_path = strdup(path); + env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); + env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); + env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int)); + if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { + rc = ENOMEM; + goto leave; + } + + /* For RDONLY, get lockfile after we know datafile exists */ + if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { + rc = mdb_env_setup_locks(env, lpath, mode, &excl); + if (rc) + goto leave; + } + +#ifdef _WIN32 + if (F_ISSET(flags, MDB_RDONLY)) { + oflags = GENERIC_READ; + len = OPEN_EXISTING; + } else { + oflags = GENERIC_READ|GENERIC_WRITE; + len = OPEN_ALWAYS; + } + mode = FILE_ATTRIBUTE_NORMAL; + env->me_fd = CreateFile(dpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, + NULL, len, mode, NULL); +#else + if (F_ISSET(flags, MDB_RDONLY)) + oflags = O_RDONLY; + else + oflags = O_RDWR | O_CREAT; + + env->me_fd = open(dpath, oflags, mode); +#endif + if (env->me_fd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + + if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { + rc = mdb_env_setup_locks(env, lpath, mode, &excl); + if (rc) + goto leave; + } + + if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) { + if (flags & (MDB_RDONLY|MDB_WRITEMAP)) { + env->me_mfd = env->me_fd; + } else { + /* Synchronous fd for meta writes. Needed even with + * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. + */ +#ifdef _WIN32 + len = OPEN_EXISTING; + env->me_mfd = CreateFile(dpath, oflags, + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, + mode | FILE_FLAG_WRITE_THROUGH, NULL); +#else + oflags &= ~O_CREAT; + env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode); +#endif + if (env->me_mfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + } + DPRINTF(("opened dbenv %p", (void *) env)); + if (excl > 0) { + rc = mdb_env_share_locks(env, &excl); + if (rc) + goto leave; + } + if (!((flags & MDB_RDONLY) || + (env->me_pbuf = calloc(1, env->me_psize)))) + rc = ENOMEM; + if (!(flags & MDB_RDONLY)) { + MDB_txn *txn; + int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs * + (sizeof(MDB_db)+sizeof(MDB_cursor)+sizeof(unsigned int)+1); + txn = calloc(1, size); + if (txn) { + txn->mt_dbs = (MDB_db *)((char *)txn + tsize); + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + txn->mt_env = env; + env->me_txn0 = txn; + } else { + rc = ENOMEM; + } + } + } + +leave: + if (rc) { + mdb_env_close0(env, excl); + } + free(lpath); + return rc; +} + +/** Destroy resources from mdb_env_open(), clear our readers & DBIs */ +static void ESECT +mdb_env_close0(MDB_env *env, int excl) +{ + int i; + + if (!(env->me_flags & MDB_ENV_ACTIVE)) + return; + + /* Doing this here since me_dbxs may not exist during mdb_env_close */ + for (i = env->me_maxdbs; --i > MAIN_DBI; ) + free(env->me_dbxs[i].md_name.mv_data); + + free(env->me_pbuf); + free(env->me_dbiseqs); + free(env->me_dbflags); + free(env->me_dbxs); + free(env->me_path); + free(env->me_dirty_list); + free(env->me_txn0); + mdb_midl_free(env->me_free_pgs); + + if (env->me_flags & MDB_ENV_TXKEY) { + pthread_key_delete(env->me_txkey); +#ifdef _WIN32 + /* Delete our key from the global list */ + for (i=0; ime_txkey) { + mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1]; + mdb_tls_nkeys--; + break; + } +#endif + } + + if (env->me_map) { + munmap(env->me_map, env->me_mapsize); + } + if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE) + (void) close(env->me_mfd); + if (env->me_fd != INVALID_HANDLE_VALUE) + (void) close(env->me_fd); + if (env->me_txns) { + MDB_PID_T pid = env->me_pid; + /* Clearing readers is done in this function because + * me_txkey with its destructor must be disabled first. + */ + for (i = env->me_numreaders; --i >= 0; ) + if (env->me_txns->mti_readers[i].mr_pid == pid) + env->me_txns->mti_readers[i].mr_pid = 0; +#ifdef _WIN32 + if (env->me_rmutex) { + CloseHandle(env->me_rmutex); + if (env->me_wmutex) CloseHandle(env->me_wmutex); + } + /* Windows automatically destroys the mutexes when + * the last handle closes. + */ +#elif defined(MDB_USE_POSIX_SEM) + if (env->me_rmutex != SEM_FAILED) { + sem_close(env->me_rmutex); + if (env->me_wmutex != SEM_FAILED) + sem_close(env->me_wmutex); + /* If we have the filelock: If we are the + * only remaining user, clean up semaphores. + */ + if (excl == 0) + mdb_env_excl_lock(env, &excl); + if (excl > 0) { + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + } + } +#endif + munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); + } + if (env->me_lfd != INVALID_HANDLE_VALUE) { +#ifdef _WIN32 + if (excl >= 0) { + /* Unlock the lockfile. Windows would have unlocked it + * after closing anyway, but not necessarily at once. + */ + UnlockFile(env->me_lfd, 0, 0, 1, 0); + } +#endif + (void) close(env->me_lfd); + } + + env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); +} + +void ESECT +mdb_env_close(MDB_env *env) +{ + MDB_page *dp; + + if (env == NULL) + return; + + VGMEMP_DESTROY(env); + while ((dp = env->me_dpages) != NULL) { + VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); + env->me_dpages = dp->mp_next; + free(dp); + } + + mdb_env_close0(env, 0); + free(env); +} + +/** Compare two items pointing at aligned size_t's */ +static int +mdb_cmp_long(const MDB_val *a, const MDB_val *b) +{ + return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 : + *(size_t *)a->mv_data > *(size_t *)b->mv_data; +} + +/** Compare two items pointing at aligned unsigned int's */ +static int +mdb_cmp_int(const MDB_val *a, const MDB_val *b) +{ + return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 : + *(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data; +} + +/** Compare two items pointing at unsigned ints of unknown alignment. + * Nodes and keys are guaranteed to be 2-byte aligned. + */ +static int +mdb_cmp_cint(const MDB_val *a, const MDB_val *b) +{ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short *u, *c; + int x; + + u = (unsigned short *) ((char *) a->mv_data + a->mv_size); + c = (unsigned short *) ((char *) b->mv_data + a->mv_size); + do { + x = *--u - *--c; + } while(!x && u > (unsigned short *)a->mv_data); + return x; +#else + unsigned short *u, *c, *end; + int x; + + end = (unsigned short *) ((char *) a->mv_data + a->mv_size); + u = (unsigned short *)a->mv_data; + c = (unsigned short *)b->mv_data; + do { + x = *u++ - *c++; + } while(!x && u < end); + return x; +#endif +} + +/** Compare two items pointing at size_t's of unknown alignment. */ +#ifdef MISALIGNED_OK +# define mdb_cmp_clong mdb_cmp_long +#else +# define mdb_cmp_clong mdb_cmp_cint +#endif + +/** Compare two items lexically */ +static int +mdb_cmp_memn(const MDB_val *a, const MDB_val *b) +{ + int diff; + ssize_t len_diff; + unsigned int len; + + len = a->mv_size; + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + len = b->mv_size; + len_diff = 1; + } + + diff = memcmp(a->mv_data, b->mv_data, len); + return diff ? diff : len_diff<0 ? -1 : len_diff; +} + +/** Compare two items in reverse byte order */ +static int +mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) +{ + const unsigned char *p1, *p2, *p1_lim; + ssize_t len_diff; + int diff; + + p1_lim = (const unsigned char *)a->mv_data; + p1 = (const unsigned char *)a->mv_data + a->mv_size; + p2 = (const unsigned char *)b->mv_data + b->mv_size; + + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + p1_lim += len_diff; + len_diff = 1; + } + + while (p1 > p1_lim) { + diff = *--p1 - *--p2; + if (diff) + return diff; + } + return len_diff<0 ? -1 : len_diff; +} + +/** Search for key within a page, using binary search. + * Returns the smallest entry larger or equal to the key. + * If exactp is non-null, stores whether the found entry was an exact match + * in *exactp (1 or 0). + * Updates the cursor index with the index of the found entry. + * If no entry larger or equal to the key is found, returns NULL. + */ +static MDB_node * +mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) +{ + unsigned int i = 0, nkeys; + int low, high; + int rc = 0; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NULL; + MDB_val nodekey; + MDB_cmp_func *cmp; + DKBUF; + + nkeys = NUMKEYS(mp); + + DPRINTF(("searching %u keys in %s %spage %"Z"u", + nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp))); + + low = IS_LEAF(mp) ? 0 : 1; + high = nkeys - 1; + cmp = mc->mc_dbx->md_cmp; + + /* Branch pages have no data, so if using integer keys, + * alignment is guaranteed. Use faster mdb_cmp_int. + */ + if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { + if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t)) + cmp = mdb_cmp_long; + else + cmp = mdb_cmp_int; + } + + if (IS_LEAF2(mp)) { + nodekey.mv_size = mc->mc_db->md_pad; + node = NODEPTR(mp, 0); /* fake */ + while (low <= high) { + i = (low + high) >> 1; + nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size); + rc = cmp(key, &nodekey); + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } else { + while (low <= high) { + i = (low + high) >> 1; + + node = NODEPTR(mp, i); + nodekey.mv_size = NODEKSZ(node); + nodekey.mv_data = NODEKEY(node); + + rc = cmp(key, &nodekey); +#if MDB_DEBUG + if (IS_LEAF(mp)) + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + else + DPRINTF(("found branch index %u [%s -> %"Z"u], rc = %i", + i, DKEY(&nodekey), NODEPGNO(node), rc)); +#endif + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } + + if (rc > 0) { /* Found entry is less than the key. */ + i++; /* Skip to get the smallest entry larger than key. */ + if (!IS_LEAF2(mp)) + node = NODEPTR(mp, i); + } + if (exactp) + *exactp = (rc == 0 && nkeys > 0); + /* store the key index */ + mc->mc_ki[mc->mc_top] = i; + if (i >= nkeys) + /* There is no entry larger or equal to the key. */ + return NULL; + + /* nodeptr is fake for LEAF2 */ + return node; +} + +#if 0 +static void +mdb_cursor_adjust(MDB_cursor *mc, func) +{ + MDB_cursor *m2; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { + func(mc, m2); + } + } +} +#endif + +/** Pop a page off the top of the cursor's stack. */ +static void +mdb_cursor_pop(MDB_cursor *mc) +{ + if (mc->mc_snum) { +#if MDB_DEBUG + MDB_page *top = mc->mc_pg[mc->mc_top]; +#endif + mc->mc_snum--; + if (mc->mc_snum) + mc->mc_top--; + + DPRINTF(("popped page %"Z"u off db %d cursor %p", top->mp_pgno, + DDBI(mc), (void *) mc)); + } +} + +/** Push a page onto the top of the cursor's stack. */ +static int +mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) +{ + DPRINTF(("pushing page %"Z"u on db %d cursor %p", mp->mp_pgno, + DDBI(mc), (void *) mc)); + + if (mc->mc_snum >= CURSOR_STACK) { + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CURSOR_FULL; + } + + mc->mc_top = mc->mc_snum++; + mc->mc_pg[mc->mc_top] = mp; + mc->mc_ki[mc->mc_top] = 0; + + return MDB_SUCCESS; +} + +/** Find the address of the page corresponding to a given page number. + * @param[in] txn the transaction for this access. + * @param[in] pgno the page number for the page to retrieve. + * @param[out] ret address of a pointer where the page's address will be stored. + * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl) +{ + MDB_env *env = txn->mt_env; + MDB_page *p = NULL; + int level; + + if (!((txn->mt_flags & MDB_TXN_RDONLY) | (env->me_flags & MDB_WRITEMAP))) { + MDB_txn *tx2 = txn; + level = 1; + do { + MDB_ID2L dl = tx2->mt_u.dirty_list; + unsigned x; + /* Spilled pages were dirtied in this txn and flushed + * because the dirty list got full. Bring this page + * back in from the map (but don't unspill it here, + * leave that unless page_touch happens again). + */ + if (tx2->mt_spill_pgs) { + MDB_ID pn = pgno << 1; + x = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { + p = (MDB_page *)(env->me_map + env->me_psize * pgno); + goto done; + } + } + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + p = dl[x].mptr; + goto done; + } + } + level++; + } while ((tx2 = tx2->mt_parent) != NULL); + } + + if (pgno < txn->mt_next_pgno) { + level = 0; + p = (MDB_page *)(env->me_map + env->me_psize * pgno); + } else { + DPRINTF(("page %"Z"u not found", pgno)); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_NOTFOUND; + } + +done: + *ret = p; + if (lvl) + *lvl = level; + return MDB_SUCCESS; +} + +/** Finish #mdb_page_search() / #mdb_page_search_lowest(). + * The cursor is at the root page, set up the rest of it. + */ +static int +mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int rc; + DKBUF; + + while (IS_BRANCH(mp)) { + MDB_node *node; + indx_t i; + + DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); + mdb_cassert(mc, NUMKEYS(mp) > 1); + DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0)))); + + if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { + i = 0; + if (flags & MDB_PS_LAST) + i = NUMKEYS(mp) - 1; + } else { + int exact; + node = mdb_node_search(mc, key, &exact); + if (node == NULL) + i = NUMKEYS(mp) - 1; + else { + i = mc->mc_ki[mc->mc_top]; + if (!exact) { + mdb_cassert(mc, i > 0); + i--; + } + } + DPRINTF(("following index %u for key [%s]", i, DKEY(key))); + } + + mdb_cassert(mc, i < NUMKEYS(mp)); + node = NODEPTR(mp, i); + + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = i; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc)) != 0) + return rc; + mp = mc->mc_pg[mc->mc_top]; + } + } + + if (!IS_LEAF(mp)) { + DPRINTF(("internal error, index points to a %02X page!?", + mp->mp_flags)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + + DPRINTF(("found leaf page %"Z"u for key [%s]", mp->mp_pgno, + key ? DKEY(key) : "null")); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + return MDB_SUCCESS; +} + +/** Search for the lowest key under the current branch page. + * This just bypasses a NUMKEYS check in the current page + * before calling mdb_page_search_root(), because the callers + * are all in situations where the current page is known to + * be underfilled. + */ +static int +mdb_page_search_lowest(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NODEPTR(mp, 0); + int rc; + + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = 0; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + return mdb_page_search_root(mc, NULL, MDB_PS_FIRST); +} + +/** Search for the page a given key should be in. + * Push it and its parent pages on the cursor stack. + * @param[in,out] mc the cursor for this operation. + * @param[in] key the key to search for, or NULL for first/last page. + * @param[in] flags If MDB_PS_MODIFY is set, visited pages in the DB + * are touched (updated with new page numbers). + * If MDB_PS_FIRST or MDB_PS_LAST is set, find first or last leaf. + * This is used by #mdb_cursor_first() and #mdb_cursor_last(). + * If MDB_PS_ROOTONLY set, just fetch root node, no further lookups. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) +{ + int rc; + pgno_t root; + + /* Make sure the txn is still viable, then find the root from + * the txn's db table and set it as the root of the cursor's stack. + */ + if (F_ISSET(mc->mc_txn->mt_flags, MDB_TXN_ERROR)) { + DPUTS("transaction has failed, must abort"); + return MDB_BAD_TXN; + } else { + /* Make sure we're using an up-to-date root */ + if (*mc->mc_dbflag & DB_STALE) { + MDB_cursor mc2; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); + if (rc) + return rc; + { + MDB_val data; + int exact = 0; + uint16_t flags; + MDB_node *leaf = mdb_node_search(&mc2, + &mc->mc_dbx->md_name, &exact); + if (!exact) + return MDB_NOTFOUND; + rc = mdb_node_read(mc->mc_txn, leaf, &data); + if (rc) + return rc; + memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)), + sizeof(uint16_t)); + /* The txn may not know this DBI, or another process may + * have dropped and recreated the DB with other flags. + */ + if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags) + return MDB_INCOMPATIBLE; + memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db)); + } + *mc->mc_dbflag &= ~DB_STALE; + } + root = mc->mc_db->md_root; + + if (root == P_INVALID) { /* Tree is empty. */ + DPUTS("tree is empty"); + return MDB_NOTFOUND; + } + } + + mdb_cassert(mc, root > 1); + if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) + if ((rc = mdb_page_get(mc->mc_txn, root, &mc->mc_pg[0], NULL)) != 0) + return rc; + + mc->mc_snum = 1; + mc->mc_top = 0; + + DPRINTF(("db %d root page %"Z"u has flags 0x%X", + DDBI(mc), root, mc->mc_pg[0]->mp_flags)); + + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc))) + return rc; + } + + if (flags & MDB_PS_ROOTONLY) + return MDB_SUCCESS; + + return mdb_page_search_root(mc, key, flags); +} + +static int +mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) +{ + MDB_txn *txn = mc->mc_txn; + pgno_t pg = mp->mp_pgno; + unsigned x = 0, ovpages = mp->mp_pages; + MDB_env *env = txn->mt_env; + MDB_IDL sl = txn->mt_spill_pgs; + MDB_ID pn = pg << 1; + int rc; + + DPRINTF(("free ov page %"Z"u (%d)", pg, ovpages)); + /* If the page is dirty or on the spill list we just acquired it, + * so we should give it back to our current free list, if any. + * Otherwise put it onto the list of pages we freed in this txn. + * + * Won't create me_pghead: me_pglast must be inited along with it. + * Unsupported in nested txns: They would need to hide the page + * range in ancestor txns' dirty and spilled lists. + */ + if (env->me_pghead && + !txn->mt_parent && + ((mp->mp_flags & P_DIRTY) || + (sl && (x = mdb_midl_search(sl, pn)) <= sl[0] && sl[x] == pn))) + { + unsigned i, j; + pgno_t *mop; + MDB_ID2 *dl, ix, iy; + rc = mdb_midl_need(&env->me_pghead, ovpages); + if (rc) + return rc; + if (!(mp->mp_flags & P_DIRTY)) { + /* This page is no longer spilled */ + if (x == sl[0]) + sl[0]--; + else + sl[x] |= 1; + goto release; + } + /* Remove from dirty list */ + dl = txn->mt_u.dirty_list; + x = dl[0].mid--; + for (ix = dl[x]; ix.mptr != mp; ix = iy) { + if (x > 1) { + x--; + iy = dl[x]; + dl[x] = ix; + } else { + mdb_cassert(mc, x > 1); + j = ++(dl[0].mid); + dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + } + if (!(env->me_flags & MDB_WRITEMAP)) + mdb_dpage_free(env, mp); +release: + /* Insert in me_pghead */ + mop = env->me_pghead; + j = mop[0] + ovpages; + for (i = mop[0]; i && mop[i] < pg; i--) + mop[j--] = mop[i]; + while (j>i) + mop[j--] = pg++; + mop[0] += ovpages; + } else { + rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); + if (rc) + return rc; + } + mc->mc_db->md_overflow_pages -= ovpages; + return 0; +} + +/** Return the data associated with a given node. + * @param[in] txn The transaction for this operation. + * @param[in] leaf The node being read. + * @param[out] data Updated to point to the node's data. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data) +{ + MDB_page *omp; /* overflow page */ + pgno_t pgno; + int rc; + + if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { + data->mv_size = NODEDSZ(leaf); + data->mv_data = NODEDATA(leaf); + return MDB_SUCCESS; + } + + /* Read overflow data. + */ + data->mv_size = NODEDSZ(leaf); + memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); + if ((rc = mdb_page_get(txn, pgno, &omp, NULL)) != 0) { + DPRINTF(("read overflow page %"Z"u failed", pgno)); + return rc; + } + data->mv_data = METADATA(omp); + + return MDB_SUCCESS; +} + +int +mdb_get(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + MDB_cursor mc; + MDB_xcursor mx; + int exact = 0; + DKBUF; + + DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); + + if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + mdb_cursor_init(&mc, txn, dbi, &mx); + return mdb_cursor_set(&mc, key, data, MDB_SET, &exact); +} + +/** Find a sibling for a page. + * Replaces the page at the top of the cursor's stack with the + * specified sibling, if one exists. + * @param[in] mc The cursor for this operation. + * @param[in] move_right Non-zero if the right sibling is requested, + * otherwise the left sibling. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_cursor_sibling(MDB_cursor *mc, int move_right) +{ + int rc; + MDB_node *indx; + MDB_page *mp; + + if (mc->mc_snum < 2) { + return MDB_NOTFOUND; /* root has no siblings */ + } + + mdb_cursor_pop(mc); + DPRINTF(("parent page is page %"Z"u, index %u", + mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); + + if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) + : (mc->mc_ki[mc->mc_top] == 0)) { + DPRINTF(("no more keys left, moving to %s sibling", + move_right ? "right" : "left")); + if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) { + /* undo cursor_pop before returning */ + mc->mc_top++; + mc->mc_snum++; + return rc; + } + } else { + if (move_right) + mc->mc_ki[mc->mc_top]++; + else + mc->mc_ki[mc->mc_top]--; + DPRINTF(("just moving to %s index key %u", + move_right ? "right" : "left", mc->mc_ki[mc->mc_top])); + } + mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); + + indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(indx), &mp, NULL)) != 0) { + /* mc will be inconsistent if caller does mc_snum++ as above */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + return rc; + } + + mdb_cursor_push(mc, mp); + if (!move_right) + mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1; + + return MDB_SUCCESS; +} + +/** Move the cursor to the next data item. */ +static int +mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + if (mc->mc_flags & C_EOF) { + return MDB_NOTFOUND; + } + + mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_NEXT || op == MDB_NEXT_DUP) { + rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT); + if (op != MDB_NEXT || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) + MDB_GET_KEY(leaf, key); + return rc; + } + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_NEXT_DUP) + return MDB_NOTFOUND; + } + } + + DPRINTF(("cursor_next: top page is %"Z"u in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + if (mc->mc_flags & C_DEL) + goto skip; + + if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { + DPUTS("=====> move to next sibling page"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { + mc->mc_flags |= C_EOF; + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + DPRINTF(("next page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]++; + +skip: + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the previous data item. */ +static int +mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_PREV || op == MDB_PREV_DUP) { + rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV); + if (op != MDB_PREV || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) { + MDB_GET_KEY(leaf, key); + mc->mc_flags &= ~C_EOF; + } + return rc; + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_PREV_DUP) + return MDB_NOTFOUND; + } + } + } + + DPRINTF(("cursor_prev: top page is %"Z"u in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + + if (mc->mc_ki[mc->mc_top] == 0) { + DPUTS("=====> move to prev sibling page"); + if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; + DPRINTF(("prev page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]--; + + mc->mc_flags &= ~C_EOF; + + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Set the cursor on a specific data item. */ +static int +mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op, int *exactp) +{ + int rc; + MDB_page *mp; + MDB_node *leaf = NULL; + DKBUF; + + if (key->mv_size == 0) + return MDB_BAD_VALSIZE; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + /* See if we're already on the right page */ + if (mc->mc_flags & C_INITIALIZED) { + MDB_val nodekey; + + mp = mc->mc_pg[mc->mc_top]; + if (!NUMKEYS(mp)) { + mc->mc_ki[mc->mc_top] = 0; + return MDB_NOTFOUND; + } + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_size = mc->mc_db->md_pad; + nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, 0); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* Probably happens rarely, but first node on the page + * was the one we wanted. + */ + mc->mc_ki[mc->mc_top] = 0; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc > 0) { + unsigned int i; + unsigned int nkeys = NUMKEYS(mp); + if (nkeys > 1) { + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + nkeys-1, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, nkeys-1); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* last node was the one we wanted */ + mc->mc_ki[mc->mc_top] = nkeys-1; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc < 0) { + if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { + /* This is definitely the right page, skip search_page */ + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + mc->mc_ki[mc->mc_top], nodekey.mv_size); + } else { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* current node was the one we wanted */ + if (exactp) + *exactp = 1; + goto set1; + } + } + rc = 0; + goto set2; + } + } + /* If any parents have right-sibs, search. + * Otherwise, there's nothing further. + */ + for (i=0; imc_top; i++) + if (mc->mc_ki[i] < + NUMKEYS(mc->mc_pg[i])-1) + break; + if (i == mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = nkeys; + return MDB_NOTFOUND; + } + } + if (!mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = 0; + if (op == MDB_SET_RANGE && !exactp) { + rc = 0; + goto set1; + } else + return MDB_NOTFOUND; + } + } + + rc = mdb_page_search(mc, key, 0); + if (rc != MDB_SUCCESS) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + +set2: + leaf = mdb_node_search(mc, key, exactp); + if (exactp != NULL && !*exactp) { + /* MDB_SET specified and not an exact match. */ + return MDB_NOTFOUND; + } + + if (leaf == NULL) { + DPUTS("===> inexact leaf not found, goto sibling"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) + return rc; /* no entries matched */ + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, 0); + } + +set1: + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + if (IS_LEAF2(mp)) { + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + } else { + int ex2, *ex2p; + if (op == MDB_GET_BOTH) { + ex2p = &ex2; + ex2 = 0; + } else { + ex2p = NULL; + } + rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p); + if (rc != MDB_SUCCESS) + return rc; + } + } else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { + MDB_val d2; + if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS) + return rc; + rc = mc->mc_dbx->md_dcmp(data, &d2); + if (rc) { + if (op == MDB_GET_BOTH || rc > 0) + return MDB_NOTFOUND; + rc = 0; + *data = d2; + } + + } else { + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + /* The key already matches in all other cases */ + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) + MDB_GET_KEY(leaf, key); + DPRINTF(("==> cursor placed on key [%s]", DKEY(key))); + + return rc; +} + +/** Move the cursor to the first item in the database. */ +static int +mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + mc->mc_ki[mc->mc_top] = 0; + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size); + return MDB_SUCCESS; + } + + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the last item in the database. */ +static int +mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + if (!(mc->mc_flags & C_EOF)) { + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_LAST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + } + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; + mc->mc_flags |= C_INITIALIZED|C_EOF; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +int +mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op) +{ + int rc; + int exact = 0; + int (*mfunc)(MDB_cursor *mc, MDB_val *key, MDB_val *data); + + if (mc == NULL) + return EINVAL; + + if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + switch (op) { + case MDB_GET_CURRENT: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + } else { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int nkeys = NUMKEYS(mp); + if (!nkeys || mc->mc_ki[mc->mc_top] >= nkeys) { + mc->mc_ki[mc->mc_top] = nkeys; + rc = MDB_NOTFOUND; + break; + } + rc = MDB_SUCCESS; + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } else { + MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY(leaf, key); + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (mc->mc_flags & C_DEL) + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); + } else { + rc = mdb_node_read(mc->mc_txn, leaf, data); + } + } + } + } + break; + case MDB_GET_BOTH: + case MDB_GET_BOTH_RANGE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + /* FALLTHRU */ + case MDB_SET: + case MDB_SET_KEY: + case MDB_SET_RANGE: + if (key == NULL) { + rc = EINVAL; + } else { + rc = mdb_cursor_set(mc, key, data, op, + op == MDB_SET_RANGE ? NULL : &exact); + } + break; + case MDB_GET_MULTIPLE: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + rc = MDB_SUCCESS; + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || + (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) + break; + goto fetchm; + case MDB_NEXT_MULTIPLE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_first(mc, key, data); + else + rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); + if (rc == MDB_SUCCESS) { + if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + MDB_cursor *mx; +fetchm: + mx = &mc->mc_xcursor->mx_cursor; + data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) * + mx->mc_db->md_pad; + data->mv_data = METADATA(mx->mc_pg[mx->mc_top]); + mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1; + } else { + rc = MDB_NOTFOUND; + } + } + break; + case MDB_NEXT: + case MDB_NEXT_DUP: + case MDB_NEXT_NODUP: + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_first(mc, key, data); + else + rc = mdb_cursor_next(mc, key, data, op); + break; + case MDB_PREV: + case MDB_PREV_DUP: + case MDB_PREV_NODUP: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = mdb_cursor_last(mc, key, data); + if (rc) + break; + mc->mc_flags |= C_INITIALIZED; + mc->mc_ki[mc->mc_top]++; + } + rc = mdb_cursor_prev(mc, key, data, op); + break; + case MDB_FIRST: + rc = mdb_cursor_first(mc, key, data); + break; + case MDB_FIRST_DUP: + mfunc = mdb_cursor_first; + mmove: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + { + MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + MDB_GET_KEY(leaf, key); + rc = mdb_node_read(mc->mc_txn, leaf, data); + break; + } + } + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); + break; + case MDB_LAST: + rc = mdb_cursor_last(mc, key, data); + break; + case MDB_LAST_DUP: + mfunc = mdb_cursor_last; + goto mmove; + default: + DPRINTF(("unhandled/unimplemented cursor operation %u", op)); + rc = EINVAL; + break; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + return rc; +} + +/** Touch all the pages in the cursor stack. Set mc_top. + * Makes sure all the pages are writable, before attempting a write operation. + * @param[in] mc The cursor to operate on. + */ +static int +mdb_cursor_touch(MDB_cursor *mc) +{ + int rc = MDB_SUCCESS; + + if (mc->mc_dbi > MAIN_DBI && !(*mc->mc_dbflag & DB_DIRTY)) { + MDB_cursor mc2; + MDB_xcursor mcx; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); + if (rc) + return rc; + *mc->mc_dbflag |= DB_DIRTY; + } + mc->mc_top = 0; + if (mc->mc_snum) { + do { + rc = mdb_page_touch(mc); + } while (!rc && ++(mc->mc_top) < mc->mc_snum); + mc->mc_top = mc->mc_snum-1; + } + return rc; +} + +/** Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDB_NOSPILL 0x8000 + +int +mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, + unsigned int flags) +{ + enum { MDB_NO_ROOT = MDB_LAST_ERRCODE+10 }; /* internal code */ + MDB_env *env; + MDB_node *leaf = NULL; + MDB_page *fp, *mp; + uint16_t fp_flags; + MDB_val xdata, *rdata, dkey, olddata; + MDB_db dummy; + int do_sub = 0, insert_key, insert_data; + unsigned int mcount = 0, dcount = 0, nospill; + size_t nsize; + int rc, rc2; + unsigned int nflags; + DKBUF; + + if (mc == NULL || key == NULL) + return EINVAL; + + env = mc->mc_txn->mt_env; + + /* Check this first so counter will always be zero on any + * early failures. + */ + if (flags & MDB_MULTIPLE) { + dcount = data[1].mv_size; + data[1].mv_size = 0; + if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) + return MDB_INCOMPATIBLE; + } + + nospill = flags & MDB_NOSPILL; + flags &= ~MDB_NOSPILL; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (key->mv_size-1 >= ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; + +#if SIZE_MAX > MAXDATASIZE + if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE)) + return MDB_BAD_VALSIZE; +#else + if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; +#endif + + DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u", + DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size)); + + dkey.mv_size = 0; + + if (flags == MDB_CURRENT) { + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + rc = MDB_SUCCESS; + } else if (mc->mc_db->md_root == P_INVALID) { + /* new database, cursor has nothing to point to */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + rc = MDB_NO_ROOT; + } else { + int exact = 0; + MDB_val d2; + if (flags & MDB_APPEND) { + MDB_val k2; + rc = mdb_cursor_last(mc, &k2, &d2); + if (rc == 0) { + rc = mc->mc_dbx->md_cmp(key, &k2); + if (rc > 0) { + rc = MDB_NOTFOUND; + mc->mc_ki[mc->mc_top]++; + } else { + /* new key is <= last key */ + rc = MDB_KEYEXIST; + } + } + } else { + rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); + } + if ((flags & MDB_NOOVERWRITE) && rc == 0) { + DPRINTF(("duplicate key [%s]", DKEY(key))); + *data = d2; + return MDB_KEYEXIST; + } + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + /* Cursor is positioned, check for room in the dirty list */ + if (!nospill) { + if (flags & MDB_MULTIPLE) { + rdata = &xdata; + xdata.mv_size = data->mv_size * dcount; + } else { + rdata = data; + } + if ((rc2 = mdb_page_spill(mc, key, rdata))) + return rc2; + } + + if (rc == MDB_NO_ROOT) { + MDB_page *np; + /* new database, write a root leaf page */ + DPUTS("allocating new root leaf page"); + if ((rc2 = mdb_page_new(mc, P_LEAF, 1, &np))) { + return rc2; + } + mdb_cursor_push(mc, np); + mc->mc_db->md_root = np->mp_pgno; + mc->mc_db->md_depth++; + *mc->mc_dbflag |= DB_DIRTY; + if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) + == MDB_DUPFIXED) + np->mp_flags |= P_LEAF2; + mc->mc_flags |= C_INITIALIZED; + } else { + /* make sure all cursor pages are writable */ + rc2 = mdb_cursor_touch(mc); + if (rc2) + return rc2; + } + + insert_key = insert_data = rc; + if (insert_key) { + /* The key does not exist */ + DPRINTF(("inserting key at index %i", mc->mc_ki[mc->mc_top])); + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + LEAFSIZE(key, data) > env->me_nodemax) + { + /* Too big for a node, insert in sub-DB. Set up an empty + * "old sub-page" for prep_subDB to expand to a full page. + */ + fp_flags = P_LEAF|P_DIRTY; + fp = env->me_pbuf; + fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */ + fp->mp_lower = fp->mp_upper = (PAGEHDRSZ-PAGEBASE); + olddata.mv_size = PAGEHDRSZ; + goto prep_subDB; + } + } else { + /* there's only a key anyway, so this is a no-op */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + char *ptr; + unsigned int ksize = mc->mc_db->md_pad; + if (key->mv_size != ksize) + return MDB_BAD_VALSIZE; + ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); + memcpy(ptr, key->mv_data, ksize); +fix_parent: + /* if overwriting slot 0 of leaf, need to + * update branch key if there is a parent page + */ + if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + unsigned short top = mc->mc_top; + mc->mc_top--; + /* slot 0 is always an empty key, find real slot */ + while (mc->mc_top && !mc->mc_ki[mc->mc_top]) + mc->mc_top--; + if (mc->mc_ki[mc->mc_top]) + rc2 = mdb_update_key(mc, key); + else + rc2 = MDB_SUCCESS; + mc->mc_top = top; + if (rc2) + return rc2; + } + return MDB_SUCCESS; + } + +more: + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + olddata.mv_size = NODEDSZ(leaf); + olddata.mv_data = NODEDATA(leaf); + + /* DB has dups? */ + if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) { + /* Prepare (sub-)page/sub-DB to accept the new item, + * if needed. fp: old sub-page or a header faking + * it. mp: new (sub-)page. offset: growth in page + * size. xdata: node data with new page or DB. + */ + unsigned i, offset = 0; + mp = fp = xdata.mv_data = env->me_pbuf; + mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; + + /* Was a single item before, must convert now */ + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + /* Just overwrite the current item */ + if (flags == MDB_CURRENT) + goto current; + +#if UINT_MAX < SIZE_MAX + if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) + mc->mc_dbx->md_dcmp = mdb_cmp_clong; +#endif + /* does data match? */ + if (!mc->mc_dbx->md_dcmp(data, &olddata)) { + if (flags & MDB_NODUPDATA) + return MDB_KEYEXIST; + /* overwrite it */ + goto current; + } + + /* Back up original data item */ + dkey.mv_size = olddata.mv_size; + dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size); + + /* Make sub-page header for the dup items, with dummy body */ + fp->mp_flags = P_LEAF|P_DIRTY|P_SUBP; + fp->mp_lower = (PAGEHDRSZ-PAGEBASE); + xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + fp->mp_flags |= P_LEAF2; + fp->mp_pad = data->mv_size; + xdata.mv_size += 2 * data->mv_size; /* leave space for 2 more */ + } else { + xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) + + (dkey.mv_size & 1) + (data->mv_size & 1); + } + fp->mp_upper = xdata.mv_size - PAGEBASE; + olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */ + } else if (leaf->mn_flags & F_SUBDATA) { + /* Data is on sub-DB, just store it */ + flags |= F_DUPDATA|F_SUBDATA; + goto put_sub; + } else { + /* Data is on sub-page */ + fp = olddata.mv_data; + switch (flags) { + default: + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + offset = EVEN(NODESIZE + sizeof(indx_t) + + data->mv_size); + break; + } + offset = fp->mp_pad; + if (SIZELEFT(fp) < offset) { + offset *= 4; /* space for 4 more */ + break; + } + /* FALLTHRU: Big enough MDB_DUPFIXED sub-page */ + case MDB_CURRENT: + fp->mp_flags |= P_DIRTY; + COPY_PGNO(fp->mp_pgno, mp->mp_pgno); + mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; + flags |= F_DUPDATA; + goto put_sub; + } + xdata.mv_size = olddata.mv_size + offset; + } + + fp_flags = fp->mp_flags; + if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) { + /* Too big for a sub-page, convert to sub-DB */ + fp_flags &= ~P_SUBP; +prep_subDB: + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + fp_flags |= P_LEAF2; + dummy.md_pad = fp->mp_pad; + dummy.md_flags = MDB_DUPFIXED; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + dummy.md_flags |= MDB_INTEGERKEY; + } else { + dummy.md_pad = 0; + dummy.md_flags = 0; + } + dummy.md_depth = 1; + dummy.md_branch_pages = 0; + dummy.md_leaf_pages = 1; + dummy.md_overflow_pages = 0; + dummy.md_entries = NUMKEYS(fp); + xdata.mv_size = sizeof(MDB_db); + xdata.mv_data = &dummy; + if ((rc = mdb_page_alloc(mc, 1, &mp))) + return rc; + offset = env->me_psize - olddata.mv_size; + flags |= F_DUPDATA|F_SUBDATA; + dummy.md_root = mp->mp_pgno; + } + if (mp != fp) { + mp->mp_flags = fp_flags | P_DIRTY; + mp->mp_pad = fp->mp_pad; + mp->mp_lower = fp->mp_lower; + mp->mp_upper = fp->mp_upper + offset; + if (fp_flags & P_LEAF2) { + memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); + } else { + memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE, + olddata.mv_size - fp->mp_upper - PAGEBASE); + for (i=0; imp_ptrs[i] = fp->mp_ptrs[i] + offset; + } + } + + rdata = &xdata; + flags |= F_DUPDATA; + do_sub = 1; + if (!insert_key) + mdb_node_del(mc, 0); + goto new_sub; + } +current: + /* overflow page overwrites need special handling */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize); + + memcpy(&pg, olddata.mv_data, sizeof(pg)); + if ((rc2 = mdb_page_get(mc->mc_txn, pg, &omp, &level)) != 0) + return rc2; + ovpages = omp->mp_pages; + + /* Is the ov page large enough? */ + if (ovpages >= dpages) { + if (!(omp->mp_flags & P_DIRTY) && + (level || (env->me_flags & MDB_WRITEMAP))) + { + rc = mdb_page_unspill(mc->mc_txn, omp, &omp); + if (rc) + return rc; + level = 0; /* dirty in this txn or clean */ + } + /* Is it dirty? */ + if (omp->mp_flags & P_DIRTY) { + /* yes, overwrite it. Note in this case we don't + * bother to try shrinking the page if the new data + * is smaller than the overflow threshold. + */ + if (level > 1) { + /* It is writable only in a parent txn */ + size_t sz = (size_t) env->me_psize * ovpages, off; + MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages); + MDB_ID2 id2; + if (!np) + return ENOMEM; + id2.mid = pg; + id2.mptr = np; + rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); + mdb_cassert(mc, rc2 == 0); + if (!(flags & MDB_RESERVE)) { + /* Copy end of page, adjusting alignment so + * compiler may copy words instead of bytes. + */ + off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); + memcpy((size_t *)((char *)np + off), + (size_t *)((char *)omp + off), sz - off); + sz = PAGEHDRSZ; + } + memcpy(np, omp, sz); /* Copy beginning of page */ + omp = np; + } + SETDSZ(leaf, data->mv_size); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(omp); + else + memcpy(METADATA(omp), data->mv_data, data->mv_size); + return MDB_SUCCESS; + } + } + if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS) + return rc2; + } else if (data->mv_size == olddata.mv_size) { + /* same size, just replace it. Note that we could + * also reuse this node if the new data is smaller, + * but instead we opt to shrink the node in that case. + */ + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = olddata.mv_data; + else if (!(mc->mc_flags & C_SUB)) + memcpy(olddata.mv_data, data->mv_data, data->mv_size); + else { + memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); + goto fix_parent; + } + return MDB_SUCCESS; + } + mdb_node_del(mc, 0); + } + + rdata = data; + +new_sub: + nflags = flags & NODE_ADD_FLAGS; + nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(env, key, rdata); + if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) { + if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA ) + nflags &= ~MDB_APPEND; /* sub-page may need room to grow */ + if (!insert_key) + nflags |= MDB_SPLIT_REPLACE; + rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags); + } else { + /* There is room already in this leaf page. */ + rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); + if (rc == 0 && insert_key) { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[i] == mp && m3->mc_ki[i] >= mc->mc_ki[i]) { + m3->mc_ki[i]++; + } + } + } + } + + if (rc == MDB_SUCCESS) { + /* Now store the actual data in the child DB. Note that we're + * storing the user data in the keys field, so there are strict + * size limits on dupdata. The actual data fields of the child + * DB are all zero size. + */ + if (do_sub) { + int xflags; + size_t ecount; +put_sub: + xdata.mv_size = 0; + xdata.mv_data = ""; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (flags & MDB_CURRENT) { + xflags = MDB_CURRENT|MDB_NOSPILL; + } else { + mdb_xcursor_init1(mc, leaf); + xflags = (flags & MDB_NODUPDATA) ? + MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; + } + /* converted, write the original data first */ + if (dkey.mv_size) { + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); + if (rc) + goto bad_sub; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { + mdb_xcursor_init1(m2, leaf); + } + } + } + /* we've done our job */ + dkey.mv_size = 0; + } + ecount = mc->mc_xcursor->mx_db.md_entries; + if (flags & MDB_APPENDDUP) + xflags |= MDB_APPEND; + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); + if (flags & F_SUBDATA) { + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } + insert_data = mc->mc_xcursor->mx_db.md_entries - ecount; + } + /* Increment count unless we just replaced an existing item. */ + if (insert_data) + mc->mc_db->md_entries++; + if (insert_key) { + /* Invalidate txn if we created an empty sub-DB */ + if (rc) + goto bad_sub; + /* If we succeeded and the key didn't exist before, + * make sure the cursor is marked valid. + */ + mc->mc_flags |= C_INITIALIZED; + } + if (flags & MDB_MULTIPLE) { + if (!rc) { + mcount++; + /* let caller know how many succeeded, if any */ + data[1].mv_size = mcount; + if (mcount < dcount) { + data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size; + insert_key = insert_data = 0; + goto more; + } + } + } + return rc; +bad_sub: + if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ + rc = MDB_CORRUPTED; + } + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_cursor_del(MDB_cursor *mc, unsigned int flags) +{ + MDB_node *leaf; + MDB_page *mp; + int rc; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + return MDB_NOTFOUND; + + if (!(flags & MDB_NOSPILL) && (rc = mdb_page_spill(mc, NULL, NULL))) + return rc; + + rc = mdb_cursor_touch(mc); + if (rc) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + if (IS_LEAF2(mp)) + goto del_key; + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (flags & MDB_NODUPDATA) { + /* mdb_cursor_del0() will subtract the final entry */ + mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; + } else { + if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL); + if (rc) + return rc; + /* If sub-DB still has entries, we're done */ + if (mc->mc_xcursor->mx_db.md_entries) { + if (leaf->mn_flags & F_SUBDATA) { + /* update subDB info */ + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } else { + MDB_cursor *m2; + /* shrink fake page */ + mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + /* fix other sub-DB cursors pointed at this fake page */ + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (m2->mc_pg[mc->mc_top] == mp && + m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + } + mc->mc_db->md_entries--; + mc->mc_flags |= C_DEL; + return rc; + } + /* otherwise fall thru and delete the sub-DB */ + } + + if (leaf->mn_flags & F_SUBDATA) { + /* add all the child DB's pages to the free list */ + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto fail; + } + } + + /* add overflow pages to free list */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + + memcpy(&pg, NODEDATA(leaf), sizeof(pg)); + if ((rc = mdb_page_get(mc->mc_txn, pg, &omp, NULL)) || + (rc = mdb_ovpage_free(mc, omp))) + goto fail; + } + +del_key: + return mdb_cursor_del0(mc); + +fail: + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +/** Allocate and initialize new pages for a database. + * @param[in] mc a cursor on the database being added to. + * @param[in] flags flags defining what type of page is being allocated. + * @param[in] num the number of pages to allocate. This is usually 1, + * unless allocating overflow pages for a large record. + * @param[out] mp Address of a page, or NULL on failure. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) +{ + MDB_page *np; + int rc; + + if ((rc = mdb_page_alloc(mc, num, &np))) + return rc; + DPRINTF(("allocated new mpage %"Z"u, page size %u", + np->mp_pgno, mc->mc_txn->mt_env->me_psize)); + np->mp_flags = flags | P_DIRTY; + np->mp_lower = (PAGEHDRSZ-PAGEBASE); + np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE; + + if (IS_BRANCH(np)) + mc->mc_db->md_branch_pages++; + else if (IS_LEAF(np)) + mc->mc_db->md_leaf_pages++; + else if (IS_OVERFLOW(np)) { + mc->mc_db->md_overflow_pages += num; + np->mp_pages = num; + } + *mp = np; + + return 0; +} + +/** Calculate the size of a leaf node. + * The size depends on the environment's page size; if a data item + * is too large it will be put onto an overflow page and the node + * size will only include the key and not the data. Sizes are always + * rounded up to an even number of bytes, to guarantee 2-byte alignment + * of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @param[in] data The data for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) +{ + size_t sz; + + sz = LEAFSIZE(key, data); + if (sz > env->me_nodemax) { + /* put on overflow page */ + sz -= data->mv_size - sizeof(pgno_t); + } + + return EVEN(sz + sizeof(indx_t)); +} + +/** Calculate the size of a branch node. + * The size should depend on the environment's page size but since + * we currently don't support spilling large keys onto overflow + * pages, it's simply the size of the #MDB_node header plus the + * size of the key. Sizes are always rounded up to an even number + * of bytes, to guarantee 2-byte alignment of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_branch_size(MDB_env *env, MDB_val *key) +{ + size_t sz; + + sz = INDXSIZE(key); + if (sz > env->me_nodemax) { + /* put on overflow page */ + /* not implemented */ + /* sz -= key->size - sizeof(pgno_t); */ + } + + return sz + sizeof(indx_t); +} + +/** Add a node to the page pointed to by the cursor. + * @param[in] mc The cursor for this operation. + * @param[in] indx The index on the page where the new node should be added. + * @param[in] key The key for the new node. + * @param[in] data The data for the new node, if any. + * @param[in] pgno The page number, if adding a branch node. + * @param[in] flags Flags for the node. + * @return 0 on success, non-zero on failure. Possible errors are: + *
      + *
    • ENOMEM - failed to allocate overflow pages for the node. + *
    • MDB_PAGE_FULL - there is insufficient room in the page. This error + * should never happen since all callers already calculate the + * page's free space before calling this function. + *
    + */ +static int +mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags) +{ + unsigned int i; + size_t node_size = NODESIZE; + ssize_t room; + indx_t ofs; + MDB_node *node; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_page *ofp = NULL; /* overflow page */ + DKBUF; + + mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); + + DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]", + IS_LEAF(mp) ? "leaf" : "branch", + IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, + key ? key->mv_size : 0, key ? DKEY(key) : "null")); + + if (IS_LEAF2(mp)) { + /* Move higher keys up one slot. */ + int ksize = mc->mc_db->md_pad, dif; + char *ptr = LEAF2KEY(mp, indx, ksize); + dif = NUMKEYS(mp) - indx; + if (dif > 0) + memmove(ptr+ksize, ptr, dif*ksize); + /* insert new key */ + memcpy(ptr, key->mv_data, ksize); + + /* Just using these for counting */ + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + return MDB_SUCCESS; + } + + room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t); + if (key != NULL) + node_size += key->mv_size; + if (IS_LEAF(mp)) { + mdb_cassert(mc, data); + if (F_ISSET(flags, F_BIGDATA)) { + /* Data already on overflow page. */ + node_size += sizeof(pgno_t); + } else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) { + int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize); + int rc; + /* Put data on overflow page. */ + DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page", + data->mv_size, node_size+data->mv_size)); + node_size = EVEN(node_size + sizeof(pgno_t)); + if ((ssize_t)node_size > room) + goto full; + if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) + return rc; + DPRINTF(("allocated overflow page %"Z"u", ofp->mp_pgno)); + flags |= F_BIGDATA; + goto update; + } else { + node_size += data->mv_size; + } + } + node_size = EVEN(node_size); + if ((ssize_t)node_size > room) + goto full; + +update: + /* Move higher pointers up one slot. */ + for (i = NUMKEYS(mp); i > indx; i--) + mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; + + /* Adjust free space offsets. */ + ofs = mp->mp_upper - node_size; + mdb_cassert(mc, ofs >= mp->mp_lower + sizeof(indx_t)); + mp->mp_ptrs[indx] = ofs; + mp->mp_upper = ofs; + mp->mp_lower += sizeof(indx_t); + + /* Write the node data. */ + node = NODEPTR(mp, indx); + node->mn_ksize = (key == NULL) ? 0 : key->mv_size; + node->mn_flags = flags; + if (IS_LEAF(mp)) + SETDSZ(node,data->mv_size); + else + SETPGNO(node,pgno); + + if (key) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + if (IS_LEAF(mp)) { + mdb_cassert(mc, key); + if (ofp == NULL) { + if (F_ISSET(flags, F_BIGDATA)) + memcpy(node->mn_data + key->mv_size, data->mv_data, + sizeof(pgno_t)); + else if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = node->mn_data + key->mv_size; + else + memcpy(node->mn_data + key->mv_size, data->mv_data, + data->mv_size); + } else { + memcpy(node->mn_data + key->mv_size, &ofp->mp_pgno, + sizeof(pgno_t)); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(ofp); + else + memcpy(METADATA(ofp), data->mv_data, data->mv_size); + } + } + + return MDB_SUCCESS; + +full: + DPRINTF(("not enough room in page %"Z"u, got %u ptrs", + mdb_dbg_pgno(mp), NUMKEYS(mp))); + DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); + DPRINTF(("node size = %"Z"u", node_size)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_FULL; +} + +/** Delete the specified node from a page. + * @param[in] mc Cursor pointing to the node to delete. + * @param[in] ksize The size of a node. Only used if the page is + * part of a #MDB_DUPFIXED database. + */ +static void +mdb_node_del(MDB_cursor *mc, int ksize) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + indx_t indx = mc->mc_ki[mc->mc_top]; + unsigned int sz; + indx_t i, j, numkeys, ptr; + MDB_node *node; + char *base; + + DPRINTF(("delete node %u on %s page %"Z"u", indx, + IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); + numkeys = NUMKEYS(mp); + mdb_cassert(mc, indx < numkeys); + + if (IS_LEAF2(mp)) { + int x = numkeys - 1 - indx; + base = LEAF2KEY(mp, indx, ksize); + if (x) + memmove(base, base + ksize, x * ksize); + mp->mp_lower -= sizeof(indx_t); + mp->mp_upper += ksize - sizeof(indx_t); + return; + } + + node = NODEPTR(mp, indx); + sz = NODESIZE + node->mn_ksize; + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + sz += sizeof(pgno_t); + else + sz += NODEDSZ(node); + } + sz = EVEN(sz); + + ptr = mp->mp_ptrs[indx]; + for (i = j = 0; i < numkeys; i++) { + if (i != indx) { + mp->mp_ptrs[j] = mp->mp_ptrs[i]; + if (mp->mp_ptrs[i] < ptr) + mp->mp_ptrs[j] += sz; + j++; + } + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + memmove(base + sz, base, ptr - mp->mp_upper); + + mp->mp_lower -= sizeof(indx_t); + mp->mp_upper += sz; +} + +/** Compact the main page after deleting a node on a subpage. + * @param[in] mp The main page to operate on. + * @param[in] indx The index of the subpage on the main page. + */ +static void +mdb_node_shrink(MDB_page *mp, indx_t indx) +{ + MDB_node *node; + MDB_page *sp, *xp; + char *base; + int nsize, delta; + indx_t i, numkeys, ptr; + + node = NODEPTR(mp, indx); + sp = (MDB_page *)NODEDATA(node); + delta = SIZELEFT(sp); + xp = (MDB_page *)((char *)sp + delta); + + /* shift subpage upward */ + if (IS_LEAF2(sp)) { + nsize = NUMKEYS(sp) * sp->mp_pad; + if (nsize & 1) + return; /* do not make the node uneven-sized */ + memmove(METADATA(xp), METADATA(sp), nsize); + } else { + int i; + numkeys = NUMKEYS(sp); + for (i=numkeys-1; i>=0; i--) + xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta; + } + xp->mp_upper = sp->mp_lower; + xp->mp_lower = sp->mp_lower; + xp->mp_flags = sp->mp_flags; + xp->mp_pad = sp->mp_pad; + COPY_PGNO(xp->mp_pgno, mp->mp_pgno); + + nsize = NODEDSZ(node) - delta; + SETDSZ(node, nsize); + + /* shift lower nodes upward */ + ptr = mp->mp_ptrs[indx]; + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] += delta; + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + memmove(base + delta, base, ptr - mp->mp_upper + NODESIZE + NODEKSZ(node)); + mp->mp_upper += delta; +} + +/** Initial setup of a sorted-dups cursor. + * Sorted duplicates are implemented as a sub-database for the given key. + * The duplicate data items are actually keys of the sub-database. + * Operations on the duplicate data items are performed using a sub-cursor + * initialized when the sub-database is first accessed. This function does + * the preliminary setup of the sub-cursor, filling in the fields that + * depend only on the parent DB. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + */ +static void +mdb_xcursor_init0(MDB_cursor *mc) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + mx->mx_cursor.mc_xcursor = NULL; + mx->mx_cursor.mc_txn = mc->mc_txn; + mx->mx_cursor.mc_db = &mx->mx_db; + mx->mx_cursor.mc_dbx = &mx->mx_dbx; + mx->mx_cursor.mc_dbi = mc->mc_dbi; + mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + mx->mx_dbx.md_name.mv_size = 0; + mx->mx_dbx.md_name.mv_data = NULL; + mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; + mx->mx_dbx.md_dcmp = NULL; + mx->mx_dbx.md_rel = mc->mc_dbx->md_rel; +} + +/** Final setup of a sorted-dups cursor. + * Sets up the fields that depend on the data from the main cursor. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + * @param[in] node The data containing the #MDB_db record for the + * sorted-dup database. + */ +static void +mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + if (node->mn_flags & F_SUBDATA) { + memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); + mx->mx_cursor.mc_pg[0] = 0; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + } else { + MDB_page *fp = NODEDATA(node); + mx->mx_db.md_pad = mc->mc_pg[mc->mc_top]->mp_pad; + mx->mx_db.md_flags = 0; + mx->mx_db.md_depth = 1; + mx->mx_db.md_branch_pages = 0; + mx->mx_db.md_leaf_pages = 1; + mx->mx_db.md_overflow_pages = 0; + mx->mx_db.md_entries = NUMKEYS(fp); + COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB; + mx->mx_cursor.mc_pg[0] = fp; + mx->mx_cursor.mc_ki[0] = 0; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + mx->mx_db.md_flags = MDB_DUPFIXED; + mx->mx_db.md_pad = fp->mp_pad; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + mx->mx_db.md_flags |= MDB_INTEGERKEY; + } + } + DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, + mx->mx_db.md_root)); + mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ +#if UINT_MAX < SIZE_MAX + if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) + mx->mx_dbx.md_cmp = mdb_cmp_clong; +#endif +} + +/** Initialize a cursor for a given transaction and database. */ +static void +mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) +{ + mc->mc_next = NULL; + mc->mc_backup = NULL; + mc->mc_dbi = dbi; + mc->mc_txn = txn; + mc->mc_db = &txn->mt_dbs[dbi]; + mc->mc_dbx = &txn->mt_dbxs[dbi]; + mc->mc_dbflag = &txn->mt_dbflags[dbi]; + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_pg[0] = 0; + mc->mc_flags = 0; + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { + mdb_tassert(txn, mx != NULL); + mc->mc_xcursor = mx; + mdb_xcursor_init0(mc); + } else { + mc->mc_xcursor = NULL; + } + if (*mc->mc_dbflag & DB_STALE) { + mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); + } +} + +int +mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) +{ + MDB_cursor *mc; + size_t size = sizeof(MDB_cursor); + + if (!ret || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + /* Allow read access to the freelist */ + if (!dbi && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EINVAL; + + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) + size += sizeof(MDB_xcursor); + + if ((mc = malloc(size)) != NULL) { + mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1)); + if (txn->mt_cursors) { + mc->mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = mc; + mc->mc_flags |= C_UNTRACK; + } + } else { + return ENOMEM; + } + + *ret = mc; + + return MDB_SUCCESS; +} + +int +mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) +{ + if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi)) + return EINVAL; + + if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor); + return MDB_SUCCESS; +} + +/* Return the count of duplicate data items for the current key */ +int +mdb_cursor_count(MDB_cursor *mc, size_t *countp) +{ + MDB_node *leaf; + + if (mc == NULL || countp == NULL) + return EINVAL; + + if (mc->mc_xcursor == NULL) + return MDB_INCOMPATIBLE; + + if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (!mc->mc_snum || (mc->mc_flags & C_EOF)) + return MDB_NOTFOUND; + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + *countp = 1; + } else { + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + return EINVAL; + + *countp = mc->mc_xcursor->mx_db.md_entries; + } + return MDB_SUCCESS; +} + +void +mdb_cursor_close(MDB_cursor *mc) +{ + if (mc && !mc->mc_backup) { + /* remove from txn, if tracked */ + if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { + MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; + while (*prev && *prev != mc) prev = &(*prev)->mc_next; + if (*prev == mc) + *prev = mc->mc_next; + } + free(mc); + } +} + +MDB_txn * +mdb_cursor_txn(MDB_cursor *mc) +{ + if (!mc) return NULL; + return mc->mc_txn; +} + +MDB_dbi +mdb_cursor_dbi(MDB_cursor *mc) +{ + return mc->mc_dbi; +} + +/** Replace the key for a branch node with a new key. + * @param[in] mc Cursor pointing to the node to operate on. + * @param[in] key The new key to use. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_update_key(MDB_cursor *mc, MDB_val *key) +{ + MDB_page *mp; + MDB_node *node; + char *base; + size_t len; + int delta, ksize, oksize; + indx_t ptr, i, numkeys, indx; + DKBUF; + + indx = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + node = NODEPTR(mp, indx); + ptr = mp->mp_ptrs[indx]; +#if MDB_DEBUG + { + MDB_val k2; + char kbuf2[DKBUF_MAXKEYSIZE*2+1]; + k2.mv_data = NODEKEY(node); + k2.mv_size = node->mn_ksize; + DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Z"u", + indx, ptr, + mdb_dkey(&k2, kbuf2), + DKEY(key), + mp->mp_pgno)); + } +#endif + + /* Sizes must be 2-byte aligned. */ + ksize = EVEN(key->mv_size); + oksize = EVEN(node->mn_ksize); + delta = ksize - oksize; + + /* Shift node contents if EVEN(key length) changed. */ + if (delta) { + if (delta > 0 && SIZELEFT(mp) < delta) { + pgno_t pgno; + /* not enough space left, do a delete and split */ + DPRINTF(("Not enough room, delta = %d, splitting...", delta)); + pgno = NODEPGNO(node); + mdb_node_del(mc, 0); + return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE); + } + + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] -= delta; + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + len = ptr - mp->mp_upper + NODESIZE; + memmove(base - delta, base, len); + mp->mp_upper -= delta; + + node = NODEPTR(mp, indx); + } + + /* But even if no shift was needed, update ksize */ + if (node->mn_ksize != key->mv_size) + node->mn_ksize = key->mv_size; + + if (key->mv_size) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + return MDB_SUCCESS; +} + +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); + +/** Move a node from csrc to cdst. + */ +static int +mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) +{ + MDB_node *srcnode; + MDB_val key, data; + pgno_t srcpg; + MDB_cursor mn; + int rc; + unsigned short flags; + + DKBUF; + + /* Mark src and dst as dirty. */ + if ((rc = mdb_page_touch(csrc)) || + (rc = mdb_page_touch(cdst))) + return rc; + + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); + data.mv_size = 0; + data.mv_data = NULL; + srcpg = 0; + flags = 0; + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]); + mdb_cassert(csrc, !((size_t)srcnode & 1)); + srcpg = NODEPGNO(srcnode); + flags = srcnode->mn_flags; + if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + unsigned int snum = csrc->mc_snum; + MDB_node *s2; + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(csrc); + if (rc) + return rc; + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + csrc->mc_snum = snum--; + csrc->mc_top = snum; + } else { + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { + unsigned int snum = cdst->mc_snum; + MDB_node *s2; + MDB_val bkey; + /* must find the lowest key below dst */ + mdb_cursor_copy(cdst, &mn); + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + bkey.mv_size = mn.mc_db->md_pad; + bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + bkey.mv_size = NODEKSZ(s2); + bkey.mv_data = NODEKEY(s2); + } + mn.mc_snum = snum--; + mn.mc_top = snum; + mn.mc_ki[snum] = 0; + rc = mdb_update_key(&mn, &bkey); + if (rc) + return rc; + } + + DPRINTF(("moving %s node %u [%s] on page %"Z"u to node %u on page %"Z"u", + IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", + csrc->mc_ki[csrc->mc_top], + DKEY(&key), + csrc->mc_pg[csrc->mc_top]->mp_pgno, + cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno)); + + /* Add the node to the destination page. + */ + rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags); + if (rc != MDB_SUCCESS) + return rc; + + /* Delete the node from the source page. + */ + mdb_node_del(csrc, key.mv_size); + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + MDB_page *mp = csrc->mc_pg[csrc->mc_top]; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] == + csrc->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + } + } + } + + /* Update the parent separators. + */ + if (csrc->mc_ki[csrc->mc_top] == 0) { + if (csrc->mc_ki[csrc->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for source page %"Z"u to [%s]", + csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(csrc, &mn); + mn.mc_snum--; + mn.mc_top--; + if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) + return rc; + } + if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + MDB_val nullkey; + indx_t ix = csrc->mc_ki[csrc->mc_top]; + nullkey.mv_size = 0; + csrc->mc_ki[csrc->mc_top] = 0; + rc = mdb_update_key(csrc, &nullkey); + csrc->mc_ki[csrc->mc_top] = ix; + mdb_cassert(csrc, rc == MDB_SUCCESS); + } + } + + if (cdst->mc_ki[cdst->mc_top] == 0) { + if (cdst->mc_ki[cdst->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for destination page %"Z"u to [%s]", + cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(cdst, &mn); + mn.mc_snum--; + mn.mc_top--; + if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) + return rc; + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) { + MDB_val nullkey; + indx_t ix = cdst->mc_ki[cdst->mc_top]; + nullkey.mv_size = 0; + cdst->mc_ki[cdst->mc_top] = 0; + rc = mdb_update_key(cdst, &nullkey); + cdst->mc_ki[cdst->mc_top] = ix; + mdb_cassert(csrc, rc == MDB_SUCCESS); + } + } + + return MDB_SUCCESS; +} + +/** Merge one page into another. + * The nodes from the page pointed to by \b csrc will + * be copied to the page pointed to by \b cdst and then + * the \b csrc page will be freed. + * @param[in] csrc Cursor pointing to the source page. + * @param[in] cdst Cursor pointing to the destination page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) +{ + MDB_page *psrc, *pdst; + MDB_node *srcnode; + MDB_val key, data; + unsigned nkeys; + int rc; + indx_t i, j; + + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + + DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno)); + + mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ + mdb_cassert(csrc, cdst->mc_snum > 1); + + /* Mark dst as dirty. */ + if ((rc = mdb_page_touch(cdst))) + return rc; + + /* Move all nodes from src to dst. + */ + j = nkeys = NUMKEYS(pdst); + if (IS_LEAF2(psrc)) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = METADATA(psrc); + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); + if (rc != MDB_SUCCESS) + return rc; + key.mv_data = (char *)key.mv_data + key.mv_size; + } + } else { + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + srcnode = NODEPTR(psrc, i); + if (i == 0 && IS_BRANCH(psrc)) { + MDB_cursor mn; + MDB_node *s2; + mdb_cursor_copy(csrc, &mn); + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + key.mv_size = mn.mc_db->md_pad; + key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + } else { + key.mv_size = srcnode->mn_ksize; + key.mv_data = NODEKEY(srcnode); + } + + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags); + if (rc != MDB_SUCCESS) + return rc; + } + } + + DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)", + pdst->mp_pgno, NUMKEYS(pdst), + (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); + + /* Unlink the src page from parent and add to free list. + */ + csrc->mc_top--; + mdb_node_del(csrc, 0); + if (csrc->mc_ki[csrc->mc_top] == 0) { + key.mv_size = 0; + rc = mdb_update_key(csrc, &key); + if (rc) { + csrc->mc_top++; + return rc; + } + } + csrc->mc_top++; + + psrc = csrc->mc_pg[csrc->mc_top]; + /* If not operating on FreeDB, allow this page to be reused + * in this txn. Otherwise just add to free list. + */ + rc = mdb_page_loose(csrc, psrc); + if (rc) + return rc; + if (IS_LEAF(psrc)) + csrc->mc_db->md_leaf_pages--; + else + csrc->mc_db->md_branch_pages--; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_snum < csrc->mc_snum) continue; + if (m3->mc_pg[csrc->mc_top] == psrc) { + m3->mc_pg[csrc->mc_top] = pdst; + m3->mc_ki[csrc->mc_top] += nkeys; + } + } + } + { + unsigned int snum = cdst->mc_snum; + uint16_t depth = cdst->mc_db->md_depth; + mdb_cursor_pop(cdst); + rc = mdb_rebalance(cdst); + /* Did the tree shrink? */ + if (depth > cdst->mc_db->md_depth) + snum--; + cdst->mc_snum = snum; + cdst->mc_top = snum-1; + } + return rc; +} + +/** Copy the contents of a cursor. + * @param[in] csrc The cursor to copy from. + * @param[out] cdst The cursor to copy to. + */ +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) +{ + unsigned int i; + + cdst->mc_txn = csrc->mc_txn; + cdst->mc_dbi = csrc->mc_dbi; + cdst->mc_db = csrc->mc_db; + cdst->mc_dbx = csrc->mc_dbx; + cdst->mc_snum = csrc->mc_snum; + cdst->mc_top = csrc->mc_top; + cdst->mc_flags = csrc->mc_flags; + + for (i=0; imc_snum; i++) { + cdst->mc_pg[i] = csrc->mc_pg[i]; + cdst->mc_ki[i] = csrc->mc_ki[i]; + } +} + +/** Rebalance the tree after a delete operation. + * @param[in] mc Cursor pointing to the page where rebalancing + * should begin. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_rebalance(MDB_cursor *mc) +{ + MDB_node *node; + int rc; + unsigned int ptop, minkeys; + MDB_cursor mn; + indx_t oldki; + + minkeys = 1 + (IS_BRANCH(mc->mc_pg[mc->mc_top])); + DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)", + IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); + + if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= FILL_THRESHOLD && + NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { + DPRINTF(("no need to rebalance page %"Z"u, above fill threshold", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); + return MDB_SUCCESS; + } + + if (mc->mc_snum < 2) { + MDB_page *mp = mc->mc_pg[0]; + if (IS_SUBP(mp)) { + DPUTS("Can't rebalance a subpage, ignoring"); + return MDB_SUCCESS; + } + if (NUMKEYS(mp) == 0) { + DPUTS("tree is completely empty"); + mc->mc_db->md_root = P_INVALID; + mc->mc_db->md_depth = 0; + mc->mc_db->md_leaf_pages = 0; + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + /* Adjust cursors pointing to mp */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + { + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum = 0; + m3->mc_top = 0; + m3->mc_flags &= ~C_INITIALIZED; + } + } + } + } else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) { + int i; + DPUTS("collapsing root page!"); + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0)); + rc = mdb_page_get(mc->mc_txn,mc->mc_db->md_root,&mc->mc_pg[0],NULL); + if (rc) + return rc; + mc->mc_db->md_depth--; + mc->mc_db->md_branch_pages--; + mc->mc_ki[0] = mc->mc_ki[1]; + for (i = 1; imc_db->md_depth; i++) { + mc->mc_pg[i] = mc->mc_pg[i+1]; + mc->mc_ki[i] = mc->mc_ki[i+1]; + } + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum--; + m3->mc_top--; + for (i=0; imc_snum; i++) { + m3->mc_pg[i] = m3->mc_pg[i+1]; + m3->mc_ki[i] = m3->mc_ki[i+1]; + } + } + } + } + } else + DPUTS("root page doesn't need rebalancing"); + return MDB_SUCCESS; + } + + /* The parent (branch page) must have at least 2 pointers, + * otherwise the tree is invalid. + */ + ptop = mc->mc_top-1; + mdb_cassert(mc, NUMKEYS(mc->mc_pg[ptop]) > 1); + + /* Leaf page fill factor is below the threshold. + * Try to move keys from left or right neighbor, or + * merge with a neighbor page. + */ + + /* Find neighbors. + */ + mdb_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + + oldki = mc->mc_ki[mc->mc_top]; + if (mc->mc_ki[ptop] == 0) { + /* We're the leftmost leaf in our parent. + */ + DPUTS("reading right neighbor"); + mn.mc_ki[ptop]++; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = 0; + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + } else { + /* There is at least one neighbor to the left. + */ + DPUTS("reading left neighbor"); + mn.mc_ki[ptop]--; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1; + mc->mc_ki[mc->mc_top] = 0; + } + + DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)", + mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); + + /* If the neighbor page is above threshold and has enough keys, + * move one key from it. Otherwise we should try to merge them. + * (A branch page must never have less than 2 keys.) + */ + minkeys = 1 + (IS_BRANCH(mn.mc_pg[mn.mc_top])); + if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= FILL_THRESHOLD && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { + rc = mdb_node_move(&mn, mc); + if (mc->mc_ki[ptop]) { + oldki++; + } + } else { + if (mc->mc_ki[ptop] == 0) { + rc = mdb_page_merge(&mn, mc); + } else { + oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); + mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; + rc = mdb_page_merge(mc, &mn); + mdb_cursor_copy(&mn, mc); + } + mc->mc_flags &= ~C_EOF; + } + mc->mc_ki[mc->mc_top] = oldki; + return rc; +} + +/** Complete a delete operation started by #mdb_cursor_del(). */ +static int +mdb_cursor_del0(MDB_cursor *mc) +{ + int rc; + MDB_page *mp; + indx_t ki; + unsigned int nkeys; + + ki = mc->mc_ki[mc->mc_top]; + mdb_node_del(mc, mc->mc_db->md_pad); + mc->mc_db->md_entries--; + rc = mdb_rebalance(mc); + + if (rc == MDB_SUCCESS) { + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + mp = mc->mc_pg[mc->mc_top]; + nkeys = NUMKEYS(mp); + + /* if mc points past last node in page, find next sibling */ + if (mc->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(mc, 1); + if (rc == MDB_NOTFOUND) { + mc->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + } + } + + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3 == mc || m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= ki) { + m3->mc_flags |= C_DEL; + if (m3->mc_ki[mc->mc_top] > ki) + m3->mc_ki[mc->mc_top]--; + else if (mc->mc_db->md_flags & MDB_DUPSORT) + m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF; + } + if (m3->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(m3, 1); + if (rc == MDB_NOTFOUND) { + m3->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + } + } + } + } + mc->mc_flags |= C_DEL; + } + + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_del(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + if (!key || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) + return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) { + /* must ignore any data */ + data = NULL; + } + + return mdb_del0(txn, dbi, key, data, 0); +} + +static int +mdb_del0(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + MDB_cursor_op op; + MDB_val rdata, *xdata; + int rc, exact = 0; + DKBUF; + + DPRINTF(("====> delete db %u key [%s]", dbi, DKEY(key))); + + mdb_cursor_init(&mc, txn, dbi, &mx); + + if (data) { + op = MDB_GET_BOTH; + rdata = *data; + xdata = &rdata; + } else { + op = MDB_SET; + xdata = NULL; + flags |= MDB_NODUPDATA; + } + rc = mdb_cursor_set(&mc, key, xdata, op, &exact); + if (rc == 0) { + /* let mdb_page_split know about this cursor if needed: + * delete will trigger a rebalance; if it needs to move + * a node from one page to another, it will have to + * update the parent's separator key(s). If the new sepkey + * is larger than the current one, the parent page may + * run out of space, triggering a split. We need this + * cursor to be consistent until the end of the rebalance. + */ + mc.mc_flags |= C_UNTRACK; + mc.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &mc; + rc = mdb_cursor_del(&mc, flags); + txn->mt_cursors[dbi] = mc.mc_next; + } + return rc; +} + +/** Split a page and insert a new node. + * @param[in,out] mc Cursor pointing to the page and desired insertion index. + * The cursor will be updated to point to the actual page and index where + * the node got inserted after the split. + * @param[in] newkey The key for the newly inserted node. + * @param[in] newdata The data for the newly inserted node. + * @param[in] newpgno The page number, if the new node is a branch node. + * @param[in] nflags The #NODE_ADD_FLAGS for the new node. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, + unsigned int nflags) +{ + unsigned int flags; + int rc = MDB_SUCCESS, new_root = 0, did_split = 0; + indx_t newindx; + pgno_t pgno = 0; + int i, j, split_indx, nkeys, pmax; + MDB_env *env = mc->mc_txn->mt_env; + MDB_node *node; + MDB_val sepkey, rkey, xdata, *rdata = &xdata; + MDB_page *copy = NULL; + MDB_page *mp, *rp, *pp; + int ptop; + MDB_cursor mn; + DKBUF; + + mp = mc->mc_pg[mc->mc_top]; + newindx = mc->mc_ki[mc->mc_top]; + nkeys = NUMKEYS(mp); + + DPRINTF(("-----> splitting %s page %"Z"u and adding [%s] at index %i/%i", + IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, + DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); + + /* Create a right sibling. */ + if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) + return rc; + DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno)); + + if (mc->mc_snum < 2) { + if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp))) + goto done; + /* shift current top to make room for new parent */ + mc->mc_pg[1] = mc->mc_pg[0]; + mc->mc_ki[1] = mc->mc_ki[0]; + mc->mc_pg[0] = pp; + mc->mc_ki[0] = 0; + mc->mc_db->md_root = pp->mp_pgno; + DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno)); + mc->mc_db->md_depth++; + new_root = 1; + + /* Add left (implicit) pointer. */ + if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { + /* undo the pre-push */ + mc->mc_pg[0] = mc->mc_pg[1]; + mc->mc_ki[0] = mc->mc_ki[1]; + mc->mc_db->md_root = mp->mp_pgno; + mc->mc_db->md_depth--; + goto done; + } + mc->mc_snum = 2; + mc->mc_top = 1; + ptop = 0; + } else { + ptop = mc->mc_top-1; + DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno)); + } + + mc->mc_flags |= C_SPLITTING; + mdb_cursor_copy(mc, &mn); + mn.mc_pg[mn.mc_top] = rp; + mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; + + if (nflags & MDB_APPEND) { + mn.mc_ki[mn.mc_top] = 0; + sepkey = *newkey; + split_indx = newindx; + nkeys = 0; + } else { + + split_indx = (nkeys+1) / 2; + + if (IS_LEAF2(rp)) { + char *split, *ins; + int x; + unsigned int lsize, rsize, ksize; + /* Move half of the keys to the right sibling */ + x = mc->mc_ki[mc->mc_top] - split_indx; + ksize = mc->mc_db->md_pad; + split = LEAF2KEY(mp, split_indx, ksize); + rsize = (nkeys - split_indx) * ksize; + lsize = (nkeys - split_indx) * sizeof(indx_t); + mp->mp_lower -= lsize; + rp->mp_lower += lsize; + mp->mp_upper += rsize - lsize; + rp->mp_upper -= rsize - lsize; + sepkey.mv_size = ksize; + if (newindx == split_indx) { + sepkey.mv_data = newkey->mv_data; + } else { + sepkey.mv_data = split; + } + if (x<0) { + ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize); + memcpy(rp->mp_ptrs, split, rsize); + sepkey.mv_data = rp->mp_ptrs; + memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); + memcpy(ins, newkey->mv_data, ksize); + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + } else { + if (x) + memcpy(rp->mp_ptrs, split, x * ksize); + ins = LEAF2KEY(rp, x, ksize); + memcpy(ins, newkey->mv_data, ksize); + memcpy(ins+ksize, split + x * ksize, rsize - x * ksize); + rp->mp_lower += sizeof(indx_t); + rp->mp_upper -= ksize - sizeof(indx_t); + mc->mc_ki[mc->mc_top] = x; + mc->mc_pg[mc->mc_top] = rp; + } + } else { + int psize, nsize, k; + /* Maximum free space in an empty page */ + pmax = env->me_psize - PAGEHDRSZ; + if (IS_LEAF(mp)) + nsize = mdb_leaf_size(env, newkey, newdata); + else + nsize = mdb_branch_size(env, newkey); + nsize = EVEN(nsize); + + /* grab a page to hold a temporary copy */ + copy = mdb_page_malloc(mc->mc_txn, 1); + if (copy == NULL) { + rc = ENOMEM; + goto done; + } + copy->mp_pgno = mp->mp_pgno; + copy->mp_flags = mp->mp_flags; + copy->mp_lower = (PAGEHDRSZ-PAGEBASE); + copy->mp_upper = env->me_psize - PAGEBASE; + + /* prepare to insert */ + for (i=0, j=0; imp_ptrs[j++] = 0; + } + copy->mp_ptrs[j++] = mp->mp_ptrs[i]; + } + + /* When items are relatively large the split point needs + * to be checked, because being off-by-one will make the + * difference between success or failure in mdb_node_add. + * + * It's also relevant if a page happens to be laid out + * such that one half of its nodes are all "small" and + * the other half of its nodes are "large." If the new + * item is also "large" and falls on the half with + * "large" nodes, it also may not fit. + * + * As a final tweak, if the new item goes on the last + * spot on the page (and thus, onto the new page), bias + * the split so the new page is emptier than the old page. + * This yields better packing during sequential inserts. + */ + if (nkeys < 20 || nsize > pmax/16 || newindx >= nkeys) { + /* Find split point */ + psize = 0; + if (newindx <= split_indx || newindx >= nkeys) { + i = 0; j = 1; + k = newindx >= nkeys ? nkeys : split_indx+2; + } else { + i = nkeys; j = -1; + k = split_indx-1; + } + for (; i!=k; i+=j) { + if (i == newindx) { + psize += nsize; + node = NULL; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t); + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + psize += sizeof(pgno_t); + else + psize += NODEDSZ(node); + } + psize = EVEN(psize); + } + if (psize > pmax || i == k-j) { + split_indx = i + (j<0); + break; + } + } + } + if (split_indx == newindx) { + sepkey.mv_size = newkey->mv_size; + sepkey.mv_data = newkey->mv_data; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEBASE); + sepkey.mv_size = node->mn_ksize; + sepkey.mv_data = NODEKEY(node); + } + } + } + + DPRINTF(("separator is %d [%s]", split_indx, DKEY(&sepkey))); + + /* Copy separator key to the parent. + */ + if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) { + mn.mc_snum--; + mn.mc_top--; + did_split = 1; + rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0); + if (rc) + goto done; + + /* root split? */ + if (mn.mc_snum == mc->mc_snum) { + mc->mc_pg[mc->mc_snum] = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_snum] = mc->mc_ki[mc->mc_top]; + mc->mc_pg[mc->mc_top] = mc->mc_pg[ptop]; + mc->mc_ki[mc->mc_top] = mc->mc_ki[ptop]; + mc->mc_snum++; + mc->mc_top++; + ptop++; + } + /* Right page might now have changed parent. + * Check if left page also changed parent. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; imc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + mc->mc_pg[ptop] = mn.mc_pg[ptop]; + if (mn.mc_ki[ptop]) { + mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; + } else { + /* find right page's left sibling */ + mc->mc_ki[ptop] = mn.mc_ki[ptop]; + mdb_cursor_sibling(mc, 0); + } + } + } else { + mn.mc_top--; + rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0); + mn.mc_top++; + } + mc->mc_flags ^= C_SPLITTING; + if (rc != MDB_SUCCESS) { + goto done; + } + if (nflags & MDB_APPEND) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[mc->mc_top] = 0; + rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags); + if (rc) + goto done; + for (i=0; imc_top; i++) + mc->mc_ki[i] = mn.mc_ki[i]; + } else if (!IS_LEAF2(mp)) { + /* Move nodes */ + mc->mc_pg[mc->mc_top] = rp; + i = split_indx; + j = 0; + do { + if (i == newindx) { + rkey.mv_data = newkey->mv_data; + rkey.mv_size = newkey->mv_size; + if (IS_LEAF(mp)) { + rdata = newdata; + } else + pgno = newpgno; + flags = nflags; + /* Update index for the new key. */ + mc->mc_ki[mc->mc_top] = j; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + rkey.mv_data = NODEKEY(node); + rkey.mv_size = node->mn_ksize; + if (IS_LEAF(mp)) { + xdata.mv_data = NODEDATA(node); + xdata.mv_size = NODEDSZ(node); + rdata = &xdata; + } else + pgno = NODEPGNO(node); + flags = node->mn_flags; + } + + if (!IS_LEAF(mp) && j == 0) { + /* First branch index doesn't need key data. */ + rkey.mv_size = 0; + } + + rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags); + if (rc) + goto done; + if (i == nkeys) { + i = 0; + j = 0; + mc->mc_pg[mc->mc_top] = copy; + } else { + i++; + j++; + } + } while (i != split_indx); + + nkeys = NUMKEYS(copy); + for (i=0; imp_ptrs[i] = copy->mp_ptrs[i]; + mp->mp_lower = copy->mp_lower; + mp->mp_upper = copy->mp_upper; + memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1), + env->me_psize - copy->mp_upper - PAGEBASE); + + /* reset back to original page */ + if (newindx < split_indx) { + mc->mc_pg[mc->mc_top] = mp; + if (nflags & MDB_RESERVE) { + node = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (!(node->mn_flags & F_BIGDATA)) + newdata->mv_data = NODEDATA(node); + } + } else { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; i<=ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + } + } + } + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + int fixup = NUMKEYS(mp); + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc) + continue; + if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_flags & C_SPLITTING) + continue; + if (new_root) { + int k; + /* root split */ + for (k=m3->mc_top; k>=0; k--) { + m3->mc_ki[k+1] = m3->mc_ki[k]; + m3->mc_pg[k+1] = m3->mc_pg[k]; + } + if (m3->mc_ki[0] >= split_indx) { + m3->mc_ki[0] = 1; + } else { + m3->mc_ki[0] = 0; + } + m3->mc_pg[0] = mc->mc_pg[0]; + m3->mc_snum++; + m3->mc_top++; + } + if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE)) + m3->mc_ki[mc->mc_top]++; + if (m3->mc_ki[mc->mc_top] >= fixup) { + m3->mc_pg[mc->mc_top] = rp; + m3->mc_ki[mc->mc_top] -= fixup; + m3->mc_ki[ptop] = mn.mc_ki[ptop]; + } + } else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] && + m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { + m3->mc_ki[ptop]++; + } + } + } + DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); + +done: + if (copy) /* tmp page */ + mdb_page_free(env, copy); + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_put(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned int flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + + if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if ((flags & (MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) != flags) + return EINVAL; + + mdb_cursor_init(&mc, txn, dbi, &mx); + return mdb_cursor_put(&mc, key, data, flags); +} + +#ifndef MDB_WBUF +#define MDB_WBUF (1024*1024) +#endif + + /** State needed for a compacting copy. */ +typedef struct mdb_copy { + pthread_mutex_t mc_mutex; + pthread_cond_t mc_cond; + char *mc_wbuf[2]; + char *mc_over[2]; + MDB_env *mc_env; + MDB_txn *mc_txn; + int mc_wlen[2]; + int mc_olen[2]; + pgno_t mc_next_pgno; + HANDLE mc_fd; + int mc_status; + volatile int mc_new; + int mc_toggle; + +} mdb_copy; + + /** Dedicated writer thread for compacting copy. */ +static THREAD_RET ESECT +mdb_env_copythr(void *arg) +{ + mdb_copy *my = arg; + char *ptr; + int toggle = 0, wsize, rc; +#ifdef _WIN32 + DWORD len; +#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) +#else + int len; +#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#endif + + pthread_mutex_lock(&my->mc_mutex); + my->mc_new = 0; + pthread_cond_signal(&my->mc_cond); + for(;;) { + while (!my->mc_new) + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + if (my->mc_new < 0) { + my->mc_new = 0; + break; + } + my->mc_new = 0; + wsize = my->mc_wlen[toggle]; + ptr = my->mc_wbuf[toggle]; +again: + while (wsize > 0) { + DO_WRITE(rc, my->mc_fd, ptr, wsize, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; + continue; + } else { + rc = EIO; + break; + } + } + if (rc) { + my->mc_status = rc; + break; + } + /* If there's an overflow page tail, write it too */ + if (my->mc_olen[toggle]) { + wsize = my->mc_olen[toggle]; + ptr = my->mc_over[toggle]; + my->mc_olen[toggle] = 0; + goto again; + } + my->mc_wlen[toggle] = 0; + toggle ^= 1; + pthread_cond_signal(&my->mc_cond); + } + pthread_cond_signal(&my->mc_cond); + pthread_mutex_unlock(&my->mc_mutex); + return (THREAD_RET)0; +#undef DO_WRITE +} + + /** Tell the writer thread there's a buffer ready to write */ +static int ESECT +mdb_env_cthr_toggle(mdb_copy *my, int st) +{ + int toggle = my->mc_toggle ^ 1; + pthread_mutex_lock(&my->mc_mutex); + if (my->mc_status) { + pthread_mutex_unlock(&my->mc_mutex); + return my->mc_status; + } + while (my->mc_new == 1) + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + my->mc_new = st; + my->mc_toggle = toggle; + pthread_cond_signal(&my->mc_cond); + pthread_mutex_unlock(&my->mc_mutex); + return 0; +} + + /** Depth-first tree traversal for compacting copy. */ +static int ESECT +mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) +{ + MDB_cursor mc; + MDB_txn *txn = my->mc_txn; + MDB_node *ni; + MDB_page *mo, *mp, *leaf; + char *buf, *ptr; + int rc, toggle; + unsigned int i; + + /* Empty DB, nothing to do */ + if (*pg == P_INVALID) + return MDB_SUCCESS; + + mc.mc_snum = 1; + mc.mc_top = 0; + mc.mc_txn = txn; + + rc = mdb_page_get(my->mc_txn, *pg, &mc.mc_pg[0], NULL); + if (rc) + return rc; + rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST); + if (rc) + return rc; + + /* Make cursor pages writable */ + buf = ptr = malloc(my->mc_env->me_psize * mc.mc_snum); + if (buf == NULL) + return ENOMEM; + + for (i=0; imc_env->me_psize); + mc.mc_pg[i] = (MDB_page *)ptr; + ptr += my->mc_env->me_psize; + } + + /* This is writable space for a leaf page. Usually not needed. */ + leaf = (MDB_page *)ptr; + + toggle = my->mc_toggle; + while (mc.mc_snum > 0) { + unsigned n; + mp = mc.mc_pg[mc.mc_top]; + n = NUMKEYS(mp); + + if (IS_LEAF(mp)) { + if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) { + for (i=0; imn_flags & F_BIGDATA) { + MDB_page *omp; + pgno_t pg; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&pg, NODEDATA(ni), sizeof(pg)); + rc = mdb_page_get(txn, pg, &omp, NULL); + if (rc) + goto done; + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + memcpy(mo, omp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno; + my->mc_next_pgno += omp->mp_pages; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (omp->mp_pages > 1) { + my->mc_olen[toggle] = my->mc_env->me_psize * (omp->mp_pages - 1); + my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + memcpy(NODEDATA(ni), &mo->mp_pgno, sizeof(pgno_t)); + } else if (ni->mn_flags & F_SUBDATA) { + MDB_db db; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&db, NODEDATA(ni), sizeof(db)); + my->mc_toggle = toggle; + rc = mdb_env_cwalk(my, &db.md_root, ni->mn_flags & F_DUPDATA); + if (rc) + goto done; + toggle = my->mc_toggle; + memcpy(NODEDATA(ni), &db, sizeof(db)); + } + } + } + } else { + mc.mc_ki[mc.mc_top]++; + if (mc.mc_ki[mc.mc_top] < n) { + pgno_t pg; +again: + ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]); + pg = NODEPGNO(ni); + rc = mdb_page_get(txn, pg, &mp, NULL); + if (rc) + goto done; + mc.mc_top++; + mc.mc_snum++; + mc.mc_ki[mc.mc_top] = 0; + if (IS_BRANCH(mp)) { + /* Whenever we advance to a sibling branch page, + * we must proceed all the way down to its first leaf. + */ + mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); + goto again; + } else + mc.mc_pg[mc.mc_top] = mp; + continue; + } + } + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + mdb_page_copy(mo, mp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno++; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (mc.mc_top) { + /* Update parent if there is one */ + ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); + SETPGNO(ni, mo->mp_pgno); + mdb_cursor_pop(&mc); + } else { + /* Otherwise we're done */ + *pg = mo->mp_pgno; + break; + } + } +done: + free(buf); + return rc; +} + + /** Copy environment with compaction. */ +static int ESECT +mdb_env_copyfd1(MDB_env *env, HANDLE fd) +{ + MDB_meta *mm; + MDB_page *mp; + mdb_copy my; + MDB_txn *txn = NULL; + pthread_t thr; + int rc; + +#ifdef _WIN32 + my.mc_mutex = CreateMutex(NULL, FALSE, NULL); + my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL); + my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); + if (my.mc_wbuf[0] == NULL) + return errno; +#else + pthread_mutex_init(&my.mc_mutex, NULL); + pthread_cond_init(&my.mc_cond, NULL); +#ifdef HAVE_MEMALIGN + my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); + if (my.mc_wbuf[0] == NULL) + return errno; +#else + rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); + if (rc) + return rc; +#endif +#endif + memset(my.mc_wbuf[0], 0, MDB_WBUF*2); + my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; + my.mc_wlen[0] = 0; + my.mc_wlen[1] = 0; + my.mc_olen[0] = 0; + my.mc_olen[1] = 0; + my.mc_next_pgno = 2; + my.mc_status = 0; + my.mc_new = 1; + my.mc_toggle = 0; + my.mc_env = env; + my.mc_fd = fd; + THREAD_CREATE(thr, mdb_env_copythr, &my); + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + return rc; + + mp = (MDB_page *)my.mc_wbuf[0]; + memset(mp, 0, 2*env->me_psize); + mp->mp_pgno = 0; + mp->mp_flags = P_META; + mm = (MDB_meta *)METADATA(mp); + mdb_env_init_meta0(env, mm); + mm->mm_address = env->me_metas[0]->mm_address; + + mp = (MDB_page *)(my.mc_wbuf[0] + env->me_psize); + mp->mp_pgno = 1; + mp->mp_flags = P_META; + *(MDB_meta *)METADATA(mp) = *mm; + mm = (MDB_meta *)METADATA(mp); + + /* Count the number of free pages, subtract from lastpg to find + * number of active pages + */ + { + MDB_ID freecount = 0; + MDB_cursor mc; + MDB_val key, data; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + freecount += txn->mt_dbs[0].md_branch_pages + + txn->mt_dbs[0].md_leaf_pages + + txn->mt_dbs[0].md_overflow_pages; + + /* Set metapage 1 */ + mm->mm_last_pg = txn->mt_next_pgno - freecount - 1; + mm->mm_dbs[1] = txn->mt_dbs[1]; + mm->mm_dbs[1].md_root = mm->mm_last_pg; + mm->mm_txnid = 1; + } + my.mc_wlen[0] = env->me_psize * 2; + my.mc_txn = txn; + pthread_mutex_lock(&my.mc_mutex); + while(my.mc_new) + pthread_cond_wait(&my.mc_cond, &my.mc_mutex); + pthread_mutex_unlock(&my.mc_mutex); + rc = mdb_env_cwalk(&my, &txn->mt_dbs[1].md_root, 0); + if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle]) + rc = mdb_env_cthr_toggle(&my, 1); + mdb_env_cthr_toggle(&my, -1); + pthread_mutex_lock(&my.mc_mutex); + while(my.mc_new) + pthread_cond_wait(&my.mc_cond, &my.mc_mutex); + pthread_mutex_unlock(&my.mc_mutex); + THREAD_FINISH(thr); + + mdb_txn_abort(txn); +#ifdef _WIN32 + CloseHandle(my.mc_cond); + CloseHandle(my.mc_mutex); + _aligned_free(my.mc_wbuf[0]); +#else + pthread_cond_destroy(&my.mc_cond); + pthread_mutex_destroy(&my.mc_mutex); + free(my.mc_wbuf[0]); +#endif + return rc; +} + + /** Copy environment as-is. */ +static int ESECT +mdb_env_copyfd0(MDB_env *env, HANDLE fd) +{ + MDB_txn *txn = NULL; + mdb_mutex_t wmutex = NULL; + int rc; + size_t wsize; + char *ptr; +#ifdef _WIN32 + DWORD len, w2; +#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) +#else + ssize_t len; + size_t w2; +#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#endif + + /* Do the lock/unlock of the reader mutex before starting the + * write txn. Otherwise other read txns could block writers. + */ + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + return rc; + + if (env->me_txns) { + /* We must start the actual read txn after blocking writers */ + mdb_txn_reset0(txn, "reset-stage1"); + + /* Temporarily block writers until we snapshot the meta pages */ + wmutex = MDB_MUTEX(env, w); + if (LOCK_MUTEX(rc, env, wmutex)) + goto leave; + + rc = mdb_txn_renew0(txn); + if (rc) { + UNLOCK_MUTEX(wmutex); + goto leave; + } + } + + wsize = env->me_psize * 2; + ptr = env->me_map; + w2 = wsize; + while (w2 > 0) { + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + w2 -= len; + continue; + } else { + /* Non-blocking or async handles are not supported */ + rc = EIO; + break; + } + } + if (wmutex) + UNLOCK_MUTEX(wmutex); + + if (rc) + goto leave; + + w2 = txn->mt_next_pgno * env->me_psize; +#ifdef WIN32 + { + LARGE_INTEGER fsize; + GetFileSizeEx(env->me_fd, &fsize); + if (w2 > fsize.QuadPart) + w2 = fsize.QuadPart; + } +#else + { + struct stat st; + fstat(env->me_fd, &st); + if (w2 > (size_t)st.st_size) + w2 = st.st_size; + } +#endif + wsize = w2 - wsize; + while (wsize > 0) { + if (wsize > MAX_WRITE) + w2 = MAX_WRITE; + else + w2 = wsize; + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; + continue; + } else { + rc = EIO; + break; + } + } + +leave: + mdb_txn_abort(txn); + return rc; +} + +int ESECT +mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags) +{ + if (flags & MDB_CP_COMPACT) + return mdb_env_copyfd1(env, fd); + else + return mdb_env_copyfd0(env, fd); +} + +int ESECT +mdb_env_copyfd(MDB_env *env, HANDLE fd) +{ + return mdb_env_copyfd2(env, fd, 0); +} + +int ESECT +mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) +{ + int rc, len; + char *lpath; + HANDLE newfd = INVALID_HANDLE_VALUE; + + if (env->me_flags & MDB_NOSUBDIR) { + lpath = (char *)path; + } else { + len = strlen(path); + len += sizeof(DATANAME); + lpath = malloc(len); + if (!lpath) + return ENOMEM; + sprintf(lpath, "%s" DATANAME, path); + } + + /* The destination path must exist, but the destination file must not. + * We don't want the OS to cache the writes, since the source data is + * already in the OS cache. + */ +#ifdef _WIN32 + newfd = CreateFile(lpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, + FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); +#else + newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666); +#endif + if (newfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + + if (env->me_psize >= env->me_os_psize) { +#ifdef O_DIRECT + /* Set O_DIRECT if the file system supports it */ + if ((rc = fcntl(newfd, F_GETFL)) != -1) + (void) fcntl(newfd, F_SETFL, rc | O_DIRECT); +#endif +#ifdef F_NOCACHE /* __APPLE__ */ + rc = fcntl(newfd, F_NOCACHE, 1); + if (rc) { + rc = ErrCode(); + goto leave; + } +#endif + } + + rc = mdb_env_copyfd2(env, newfd, flags); + +leave: + if (!(env->me_flags & MDB_NOSUBDIR)) + free(lpath); + if (newfd != INVALID_HANDLE_VALUE) + if (close(newfd) < 0 && rc == MDB_SUCCESS) + rc = ErrCode(); + + return rc; +} + +int ESECT +mdb_env_copy(MDB_env *env, const char *path) +{ + return mdb_env_copy2(env, path, 0); +} + +int ESECT +mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) +{ + if (flag & (env->me_map ? ~CHANGEABLE : ~(CHANGEABLE|CHANGELESS))) + return EINVAL; + if (onoff) + env->me_flags |= flag; + else + env->me_flags &= ~flag; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_flags(MDB_env *env, unsigned int *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_flags; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_userctx(MDB_env *env, void *ctx) +{ + if (!env) + return EINVAL; + env->me_userctx = ctx; + return MDB_SUCCESS; +} + +void * ESECT +mdb_env_get_userctx(MDB_env *env) +{ + return env ? env->me_userctx : NULL; +} + +int ESECT +mdb_env_set_assert(MDB_env *env, MDB_assert_func *func) +{ + if (!env) + return EINVAL; +#ifndef NDEBUG + env->me_assert_func = func; +#endif + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_path(MDB_env *env, const char **arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_path; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_fd; + return MDB_SUCCESS; +} + +/** Common code for #mdb_stat() and #mdb_env_stat(). + * @param[in] env the environment to operate in. + * @param[in] db the #MDB_db record containing the stats to return. + * @param[out] arg the address of an #MDB_stat structure to receive the stats. + * @return 0, this function always succeeds. + */ +static int ESECT +mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg) +{ + arg->ms_psize = env->me_psize; + arg->ms_depth = db->md_depth; + arg->ms_branch_pages = db->md_branch_pages; + arg->ms_leaf_pages = db->md_leaf_pages; + arg->ms_overflow_pages = db->md_overflow_pages; + arg->ms_entries = db->md_entries; + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_stat(MDB_env *env, MDB_stat *arg) +{ + int toggle; + + if (env == NULL || arg == NULL) + return EINVAL; + + toggle = mdb_env_pick_meta(env); + + return mdb_stat0(env, &env->me_metas[toggle]->mm_dbs[MAIN_DBI], arg); +} + +int ESECT +mdb_env_info(MDB_env *env, MDB_envinfo *arg) +{ + int toggle; + + if (env == NULL || arg == NULL) + return EINVAL; + + toggle = mdb_env_pick_meta(env); + arg->me_mapaddr = env->me_metas[toggle]->mm_address; + arg->me_mapsize = env->me_mapsize; + arg->me_maxreaders = env->me_maxreaders; + + /* me_numreaders may be zero if this process never used any readers. Use + * the shared numreader count if it exists. + */ + arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : env->me_numreaders; + + arg->me_last_pgno = env->me_metas[toggle]->mm_last_pg; + arg->me_last_txnid = env->me_metas[toggle]->mm_txnid; + return MDB_SUCCESS; +} + +/** Set the default comparison functions for a database. + * Called immediately after a database is opened to set the defaults. + * The user can then override them with #mdb_set_compare() or + * #mdb_set_dupsort(). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +static void +mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi) +{ + uint16_t f = txn->mt_dbs[dbi].md_flags; + + txn->mt_dbxs[dbi].md_cmp = + (f & MDB_REVERSEKEY) ? mdb_cmp_memnr : + (f & MDB_INTEGERKEY) ? mdb_cmp_cint : mdb_cmp_memn; + + txn->mt_dbxs[dbi].md_dcmp = + !(f & MDB_DUPSORT) ? 0 : + ((f & MDB_INTEGERDUP) + ? ((f & MDB_DUPFIXED) ? mdb_cmp_int : mdb_cmp_cint) + : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn)); +} + +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi) +{ + MDB_val key, data; + MDB_dbi i; + MDB_cursor mc; + MDB_db dummy; + int rc, dbflag, exact; + unsigned int unused = 0, seq; + size_t len; + + if (txn->mt_dbxs[FREE_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, FREE_DBI); + } + + if ((flags & VALID_FLAGS) != flags) + return EINVAL; + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + /* main DB? */ + if (!name) { + *dbi = MAIN_DBI; + if (flags & PERSISTENT_FLAGS) { + uint16_t f2 = flags & PERSISTENT_FLAGS; + /* make sure flag changes get committed */ + if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) { + txn->mt_dbs[MAIN_DBI].md_flags |= f2; + txn->mt_flags |= MDB_TXN_DIRTY; + } + } + mdb_default_cmp(txn, MAIN_DBI); + return MDB_SUCCESS; + } + + if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, MAIN_DBI); + } + + /* Is the DB already open? */ + len = strlen(name); + for (i=2; imt_numdbs; i++) { + if (!txn->mt_dbxs[i].md_name.mv_size) { + /* Remember this free slot */ + if (!unused) unused = i; + continue; + } + if (len == txn->mt_dbxs[i].md_name.mv_size && + !strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) { + *dbi = i; + return MDB_SUCCESS; + } + } + + /* If no free slot and max hit, fail */ + if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs) + return MDB_DBS_FULL; + + /* Cannot mix named databases with some mainDB flags */ + if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY)) + return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; + + /* Find the DB info */ + dbflag = DB_NEW|DB_VALID; + exact = 0; + key.mv_size = len; + key.mv_data = (void *)name; + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact); + if (rc == MDB_SUCCESS) { + /* make sure this is actually a DB */ + MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); + if (!(node->mn_flags & F_SUBDATA)) + return MDB_INCOMPATIBLE; + } else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) { + /* Create if requested */ + data.mv_size = sizeof(MDB_db); + data.mv_data = &dummy; + memset(&dummy, 0, sizeof(dummy)); + dummy.md_root = P_INVALID; + dummy.md_flags = flags & PERSISTENT_FLAGS; + rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA); + dbflag |= DB_DIRTY; + } + + /* OK, got info, add to table */ + if (rc == MDB_SUCCESS) { + unsigned int slot = unused ? unused : txn->mt_numdbs; + txn->mt_dbxs[slot].md_name.mv_data = strdup(name); + txn->mt_dbxs[slot].md_name.mv_size = len; + txn->mt_dbxs[slot].md_rel = NULL; + txn->mt_dbflags[slot] = dbflag; + /* txn-> and env-> are the same in read txns, use + * tmp variable to avoid undefined assignment + */ + seq = ++txn->mt_env->me_dbiseqs[slot]; + txn->mt_dbiseqs[slot] = seq; + + memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); + *dbi = slot; + mdb_default_cmp(txn, slot); + if (!unused) { + txn->mt_numdbs++; + } + } + + return rc; +} + +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) +{ + if (!arg || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + if (txn->mt_dbflags[dbi] & DB_STALE) { + MDB_cursor mc; + MDB_xcursor mx; + /* Stale, must read the DB's root. cursor_init does it for us. */ + mdb_cursor_init(&mc, txn, dbi, &mx); + } + return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg); +} + +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) +{ + char *ptr; + if (dbi <= MAIN_DBI || dbi >= env->me_maxdbs) + return; + ptr = env->me_dbxs[dbi].md_name.mv_data; + /* If there was no name, this was already closed */ + if (ptr) { + env->me_dbxs[dbi].md_name.mv_data = NULL; + env->me_dbxs[dbi].md_name.mv_size = 0; + env->me_dbflags[dbi] = 0; + env->me_dbiseqs[dbi]++; + free(ptr); + } +} + +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) +{ + /* We could return the flags for the FREE_DBI too but what's the point? */ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; + return MDB_SUCCESS; +} + +/** Add all the DB's pages to the free list. + * @param[in] mc Cursor on the DB to free. + * @param[in] subs non-Zero to check for sub-DBs in this DB. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_drop0(MDB_cursor *mc, int subs) +{ + int rc; + + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc == MDB_SUCCESS) { + MDB_txn *txn = mc->mc_txn; + MDB_node *ni; + MDB_cursor mx; + unsigned int i; + + /* LEAF2 pages have no nodes, cannot have sub-DBs */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) + mdb_cursor_pop(mc); + + mdb_cursor_copy(mc, &mx); + while (mc->mc_snum > 0) { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + unsigned n = NUMKEYS(mp); + if (IS_LEAF(mp)) { + for (i=0; imn_flags & F_BIGDATA) { + MDB_page *omp; + pgno_t pg; + memcpy(&pg, NODEDATA(ni), sizeof(pg)); + rc = mdb_page_get(txn, pg, &omp, NULL); + if (rc != 0) + goto done; + mdb_cassert(mc, IS_OVERFLOW(omp)); + rc = mdb_midl_append_range(&txn->mt_free_pgs, + pg, omp->mp_pages); + if (rc) + goto done; + } else if (subs && (ni->mn_flags & F_SUBDATA)) { + mdb_xcursor_init1(mc, ni); + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto done; + } + } + } else { + if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) + goto done; + for (i=0; imt_free_pgs, pg); + } + } + if (!mc->mc_top) + break; + mc->mc_ki[mc->mc_top] = i; + rc = mdb_cursor_sibling(mc, 1); + if (rc) { + if (rc != MDB_NOTFOUND) + goto done; + /* no more siblings, go back to beginning + * of previous level. + */ + mdb_cursor_pop(mc); + mc->mc_ki[0] = 0; + for (i=1; imc_snum; i++) { + mc->mc_ki[i] = 0; + mc->mc_pg[i] = mx.mc_pg[i]; + } + } + } + /* free it */ + rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root); +done: + if (rc) + txn->mt_flags |= MDB_TXN_ERROR; + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + return rc; +} + +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) +{ + MDB_cursor *mc, *m2; + int rc; + + if ((unsigned)del > 1 || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; + + if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, dbi)) + return MDB_BAD_DBI; + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) + return rc; + + rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT); + /* Invalidate the dropped DB's cursors */ + for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) + m2->mc_flags &= ~(C_INITIALIZED|C_EOF); + if (rc) + goto leave; + + /* Can't delete the main DB */ + if (del && dbi > MAIN_DBI) { + rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, 0); + if (!rc) { + txn->mt_dbflags[dbi] = DB_STALE; + mdb_dbi_close(txn->mt_env, dbi); + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + } else { + /* reset the DB record, mark it dirty */ + txn->mt_dbflags[dbi] |= DB_DIRTY; + txn->mt_dbs[dbi].md_depth = 0; + txn->mt_dbs[dbi].md_branch_pages = 0; + txn->mt_dbs[dbi].md_leaf_pages = 0; + txn->mt_dbs[dbi].md_overflow_pages = 0; + txn->mt_dbs[dbi].md_entries = 0; + txn->mt_dbs[dbi].md_root = P_INVALID; + + txn->mt_flags |= MDB_TXN_DIRTY; + } +leave: + mdb_cursor_close(mc); + return rc; +} + +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_cmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_dcmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_rel = rel; + return MDB_SUCCESS; +} + +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_relctx = ctx; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxkeysize(MDB_env *env) +{ + return ENV_MAXKEY(env); +} + +int ESECT +mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) +{ + unsigned int i, rdrs; + MDB_reader *mr; + char buf[64]; + int rc = 0, first = 1; + + if (!env || !func) + return -1; + if (!env->me_txns) { + return func("(no reader locks)\n", ctx); + } + rdrs = env->me_txns->mti_numreaders; + mr = env->me_txns->mti_readers; + for (i=0; i> 1; + cursor = base + pivot + 1; + val = pid - ids[cursor]; + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + /* found, so it's a duplicate */ + return -1; + } + } + + if( val > 0 ) { + ++cursor; + } + ids[0]++; + for (n = ids[0]; n > cursor; n--) + ids[n] = ids[n-1]; + ids[n] = pid; + return 0; +} + +int ESECT +mdb_reader_check(MDB_env *env, int *dead) +{ + if (!env) + return EINVAL; + if (dead) + *dead = 0; + return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS; +} + +/** As #mdb_reader_check(). rlocked = . */ +static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead) +{ + mdb_mutex_t rmutex = rlocked ? NULL : MDB_MUTEX(env, r); + unsigned int i, j, rdrs; + MDB_reader *mr; + MDB_PID_T *pids, pid; + int rc = MDB_SUCCESS, count = 0; + + rdrs = env->me_txns->mti_numreaders; + pids = malloc((rdrs+1) * sizeof(MDB_PID_T)); + if (!pids) + return ENOMEM; + pids[0] = 0; + mr = env->me_txns->mti_readers; + for (i=0; ime_pid) { + if (mdb_pid_insert(pids, pid) == 0) { + if (!mdb_reader_pid(env, Pidcheck, pid)) { + /* Stale reader found */ + j = i; + if (rmutex) { + if ((rc = LOCK_MUTEX0(rmutex)) != 0) { + if ((rc = mdb_mutex_failed(env, rmutex, rc))) + break; + rdrs = 0; /* the above checked all readers */ + } else { + /* Recheck, a new process may have reused pid */ + if (mdb_reader_pid(env, Pidcheck, pid)) + j = rdrs; + } + } + for (; jme_txns->mti_txnid = env->me_metas[toggle]->mm_txnid; + /* env is hosed if the dead thread was ours */ + if (env->me_txn) { + env->me_flags |= MDB_FATAL_ERROR; + env->me_txn = NULL; + rc = MDB_PANIC; + } + } + DPRINTF(("%cmutex owner died, %s", (rlocked ? 'r' : 'w'), + (rc ? "this process' env is hosed" : "recovering"))); + rc2 = mdb_reader_check0(env, rlocked, NULL); + if (rc2 == 0) + rc2 = pthread_mutex_consistent(mutex); + if (rc || (rc = rc2)) { + DPRINTF(("LOCK_MUTEX recovery failed, %s", mdb_strerror(rc))); + UNLOCK_MUTEX(mutex); + } + } else { +#ifdef _WIN32 + rc = ErrCode(); +#endif + DPRINTF(("LOCK_MUTEX failed, %s", mdb_strerror(rc))); + } + + return rc; +} +#endif /* MDB_ROBUST_SUPPORTED */ +/** @} */ diff --git a/submodules/lmdb/libraries/liblmdb/mdb_copy.1 b/submodules/lmdb/libraries/liblmdb/mdb_copy.1 new file mode 100644 index 0000000000..094b260563 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mdb_copy.1 @@ -0,0 +1,54 @@ +.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_copy \- LMDB environment copy tool +.SH SYNOPSIS +.B mdb_copy +[\c +.BR \-V ] +[\c +.BR \-c ] +[\c +.BR \-n ] +.B srcpath +[\c +.BR dstpath ] +.SH DESCRIPTION +The +.B mdb_copy +utility copies an LMDB environment. The environment can +be copied regardless of whether it is currently in use. +No lockfile is created, since it gets recreated at need. + +If +.I dstpath +is specified it must be the path of an empty directory +for storing the backup. Otherwise, the backup will be +written to stdout. + +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-c +Compact while copying. Only current data pages will be copied; freed +or unused pages will be omitted from the copy. This option will +slow down the backup process as it is more CPU-intensive. +.TP +.BR \-n +Open LDMB environment(s) which do not use subdirectories. + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH CAVEATS +This utility can trigger significant file size growth if run +in parallel with write transactions, because pages which they +free during copying cannot be reused until the copy is done. +.SH "SEE ALSO" +.BR mdb_stat (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/submodules/lmdb/libraries/liblmdb/mdb_copy.c b/submodules/lmdb/libraries/liblmdb/mdb_copy.c new file mode 100644 index 0000000000..e7f965c03a --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mdb_copy.c @@ -0,0 +1,82 @@ +/* mdb_copy.c - memory-mapped database backup tool */ +/* + * Copyright 2012 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#ifdef _WIN32 +#include +#define MDB_STDOUT GetStdHandle(STD_OUTPUT_HANDLE) +#else +#define MDB_STDOUT 1 +#endif +#include +#include +#include +#include "lmdb.h" + +static void +sighandle(int sig) +{ +} + +int main(int argc,char * argv[]) +{ + int rc; + MDB_env *env; + const char *progname = argv[0], *act; + unsigned flags = MDB_RDONLY; + unsigned cpflags = 0; + + for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { + if (argv[1][1] == 'n' && argv[1][2] == '\0') + flags |= MDB_NOSUBDIR; + else if (argv[1][1] == 'c' && argv[1][2] == '\0') + cpflags |= MDB_CP_COMPACT; + else if (argv[1][1] == 'V' && argv[1][2] == '\0') { + printf("%s\n", MDB_VERSION_STRING); + exit(0); + } else + argc = 0; + } + + if (argc<2 || argc>3) { + fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname); + exit(EXIT_FAILURE); + } + +#ifdef SIGPIPE + signal(SIGPIPE, sighandle); +#endif +#ifdef SIGHUP + signal(SIGHUP, sighandle); +#endif + signal(SIGINT, sighandle); + signal(SIGTERM, sighandle); + + act = "opening environment"; + rc = mdb_env_create(&env); + if (rc == MDB_SUCCESS) { + rc = mdb_env_open(env, argv[1], flags, 0664); + } + if (rc == MDB_SUCCESS) { + act = "copying"; + if (argc == 2) + rc = mdb_env_copyfd2(env, MDB_STDOUT, cpflags); + else + rc = mdb_env_copy2(env, argv[2], cpflags); + } + if (rc) + fprintf(stderr, "%s: %s failed, error %d (%s)\n", + progname, act, rc, mdb_strerror(rc)); + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/submodules/lmdb/libraries/liblmdb/mdb_dump.1 b/submodules/lmdb/libraries/liblmdb/mdb_dump.1 new file mode 100644 index 0000000000..6fcc930811 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mdb_dump.1 @@ -0,0 +1,75 @@ +.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_dump \- LMDB environment export tool +.SH SYNOPSIS +.B mdb_dump +.BR \ envpath +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-l ] +[\c +.BR \-n ] +[\c +.BR \-p ] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.SH DESCRIPTION +The +.B mdb_dump +utility reads a database and writes its contents to the +standard output using a portable flat-text format +understood by the +.BR mdb_load (1) +utility. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Write to the specified file instead of to the standard output. +.TP +.BR \-l +List the databases stored in the environment. Just the +names will be listed, no data will be output. +.TP +.BR \-n +Dump an LMDB database which does not use subdirectories. +.TP +.BR \-p +If characters in either the key or data items are printing characters (as +defined by isprint(3)), output them directly. This option permits users to +use standard text editors and tools to modify the contents of databases. + +Note: different systems may have different notions about what characters +are considered printing characters, and databases dumped in this manner may +be less portable to external systems. +.TP +.BR \-a +Dump all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Dump a specific subdatabase. If no database is specified, only the main database is dumped. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +Dumping and reloading databases that use user-defined comparison functions +will result in new databases that use the default comparison functions. +\fBIn this case it is quite likely that the reloaded database will be +damaged beyond repair permitting neither record storage nor retrieval.\fP + +The only available workaround is to modify the source for the +.BR mdb_load (1) +utility to load the database using the correct comparison functions. +.SH "SEE ALSO" +.BR mdb_load (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/submodules/lmdb/libraries/liblmdb/mdb_dump.c b/submodules/lmdb/libraries/liblmdb/mdb_dump.c new file mode 100644 index 0000000000..0eb85fd20e --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mdb_dump.c @@ -0,0 +1,317 @@ +/* mdb_dump.c - memory-mapped database dump tool */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include +#include +#include +#include +#include +#include +#include "lmdb.h" + +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + +#define PRINT 1 +static int mode; + +typedef struct flagbit { + int bit; + char *name; +} flagbit; + +flagbit dbflags[] = { + { MDB_REVERSEKEY, "reversekey" }, + { MDB_DUPSORT, "dupsort" }, + { MDB_INTEGERKEY, "integerkey" }, + { MDB_DUPFIXED, "dupfixed" }, + { MDB_INTEGERDUP, "integerdup" }, + { MDB_REVERSEDUP, "reversedup" }, + { 0, NULL } +}; + +static volatile sig_atomic_t gotsig; + +static void dumpsig( int sig ) +{ + gotsig=1; +} + +static const char hexc[] = "0123456789abcdef"; + +static void hex(unsigned char c) +{ + putchar(hexc[c >> 4]); + putchar(hexc[c & 0xf]); +} + +static void text(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + if (isprint(*c)) { + putchar(*c); + } else { + putchar('\\'); + hex(*c); + } + c++; + } + putchar('\n'); +} + +static void byte(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + hex(*c++); + } + putchar('\n'); +} + +/* Dump in BDB-compatible format */ +static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) +{ + MDB_cursor *mc; + MDB_stat ms; + MDB_val key, data; + MDB_envinfo info; + unsigned int flags; + int rc, i; + + rc = mdb_dbi_flags(txn, dbi, &flags); + if (rc) return rc; + + rc = mdb_stat(txn, dbi, &ms); + if (rc) return rc; + + rc = mdb_env_info(mdb_txn_env(txn), &info); + if (rc) return rc; + + printf("VERSION=3\n"); + printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); + if (name) + printf("database=%s\n", name); + printf("type=btree\n"); + printf("mapsize=%" Z "u\n", info.me_mapsize); + if (info.me_mapaddr) + printf("mapaddr=%p\n", info.me_mapaddr); + printf("maxreaders=%u\n", info.me_maxreaders); + + if (flags & MDB_DUPSORT) + printf("duplicates=1\n"); + + for (i=0; dbflags[i].bit; i++) + if (flags & dbflags[i].bit) + printf("%s=1\n", dbflags[i].name); + + printf("db_pagesize=%d\n", ms.ms_psize); + printf("HEADER=END\n"); + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) return rc; + + while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT) == MDB_SUCCESS)) { + if (gotsig) { + rc = EINTR; + break; + } + if (mode & PRINT) { + text(&key); + text(&data); + } else { + byte(&key); + byte(&data); + } + } + printf("DATA=END\n"); + if (rc == MDB_NOTFOUND) + rc = MDB_SUCCESS; + + return rc; +} + +static void usage(char *prog) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envflags = 0, list = 0; + + if (argc < 2) { + usage(prog); + } + + /* -a: dump main DB and all subDBs + * -s: dump only the named subDB + * -n: use NOSUBDIR flag on env_open + * -p: use printable characters + * -f: write to file instead of stdout + * -V: print version and exit + * (default) dump only the main DB + */ + while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'l': + list = 1; + /*FALLTHROUGH*/; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'f': + if (freopen(optarg, "w", stdout) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + prog, optarg, strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 'p': + mode |= PRINT; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + +#ifdef SIGPIPE + signal(SIGPIPE, dumpsig); +#endif +#ifdef SIGHUP + signal(SIGHUP, dumpsig); +#endif + signal(SIGINT, dumpsig); + signal(SIGTERM, dumpsig); + + envname = argv[optind]; + rc = mdb_env_create(&env); + if (rc) { + fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); + return EXIT_FAILURE; + } + + if (alldbs || subname) { + mdb_env_set_maxdbs(env, 2); + } + + rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); + if (rc) { + fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_open(txn, subname, 0, &dbi); + if (rc) { + fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + if (alldbs) { + MDB_cursor *cursor; + MDB_val key; + int count = 0; + + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { + char *str; + MDB_dbi db2; + if (memchr(key.mv_data, '\0', key.mv_size)) + continue; + count++; + str = malloc(key.mv_size+1); + memcpy(str, key.mv_data, key.mv_size); + str[key.mv_size] = '\0'; + rc = mdb_open(txn, str, 0, &db2); + if (rc == MDB_SUCCESS) { + if (list) { + printf("%s\n", str); + list++; + } else { + rc = dumpit(txn, db2, str); + if (rc) + break; + } + mdb_close(env, db2); + } + free(str); + if (rc) continue; + } + mdb_cursor_close(cursor); + if (!count) { + fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname); + rc = MDB_NOTFOUND; + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + } else { + rc = dumpit(txn, dbi, subname); + } + if (rc && rc != MDB_NOTFOUND) + fprintf(stderr, "%s: %s: %s\n", prog, envname, mdb_strerror(rc)); + + mdb_close(env, dbi); +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/submodules/lmdb/libraries/liblmdb/mdb_load.1 b/submodules/lmdb/libraries/liblmdb/mdb_load.1 new file mode 100644 index 0000000000..7280240103 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mdb_load.1 @@ -0,0 +1,77 @@ +.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_load \- LMDB environment import tool +.SH SYNOPSIS +.B mdb_load +.BR \ envpath +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-n ] +[\c +.BI \-s \ subdb\fR] +[\c +.BR \-N ] +[\c +.BR \-T ] +.SH DESCRIPTION +The +.B mdb_load +utility reads from the standard input and loads it into the +LMDB environment +.BR envpath . + +The input to +.B mdb_load +must be in the output format specified by the +.BR mdb_dump (1) +utility or as specified by the +.B -T +option below. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Read from the specified file instead of from the standard input. +.TP +.BR \-n +Load an LMDB database which does not use subdirectories. +.TP +.BR \-s \ subdb +Load a specific subdatabase. If no database is specified, data is loaded into the main database. +.TP +.BR \-N +Don't overwrite existing records when loading into an already existing database; just skip them. +.TP +.BR \-T +Load data from simple text files. The input must be paired lines of text, where the first +line of the pair is the key item, and the second line of the pair is its corresponding +data item. + +A simple escape mechanism, where newline and backslash (\\) characters are special, is +applied to the text input. Newline characters are interpreted as record separators. +Backslash characters in the text will be interpreted in one of two ways: If the backslash +character precedes another backslash character, the pair will be interpreted as a literal +backslash. If the backslash character precedes any other character, the two characters +following the backslash will be interpreted as a hexadecimal specification of a single +character; for example, \\0a is a newline character in the ASCII character set. + +For this reason, any backslash or newline characters that naturally occur in the text +input must be escaped to avoid misinterpretation by +.BR mdb_load . + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +.SH "SEE ALSO" +.BR mdb_dump (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/submodules/lmdb/libraries/liblmdb/mdb_load.c b/submodules/lmdb/libraries/liblmdb/mdb_load.c new file mode 100644 index 0000000000..e0d95e13c3 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mdb_load.c @@ -0,0 +1,452 @@ +/* mdb_load.c - memory-mapped database load tool */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include +#include +#include +#include +#include +#include "lmdb.h" + +#define PRINT 1 +#define NOHDR 2 +static int mode; + +static char *subname = NULL; + +static size_t lineno; +static int version; + +static int flags; + +static char *prog; + +static int Eof; + +static MDB_envinfo info; + +static MDB_val kbuf, dbuf; + +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + +#define STRLENOF(s) (sizeof(s)-1) + +typedef struct flagbit { + int bit; + char *name; + int len; +} flagbit; + +#define S(s) s, STRLENOF(s) + +flagbit dbflags[] = { + { MDB_REVERSEKEY, S("reversekey") }, + { MDB_DUPSORT, S("dupsort") }, + { MDB_INTEGERKEY, S("integerkey") }, + { MDB_DUPFIXED, S("dupfixed") }, + { MDB_INTEGERDUP, S("integerdup") }, + { MDB_REVERSEDUP, S("reversedup") }, + { 0, NULL, 0 } +}; + +static void readhdr(void) +{ + char *ptr; + + while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) { + lineno++; + if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { + version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION=")); + if (version > 3) { + fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n", + prog, lineno, version); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) { + break; + } else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) { + if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) + mode |= PRINT; + else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { + fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) { + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + if (subname) free(subname); + subname = strdup((char *)dbuf.mv_data+STRLENOF("database=")); + } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) { + if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { + fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("type=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) { + int i; + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr); + if (i != 1) { + fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) { + int i; + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize); + if (i != 1) { + fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) { + int i; + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders); + if (i != 1) { + fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders=")); + exit(EXIT_FAILURE); + } + } else { + int i; + for (i=0; dbflags[i].bit; i++) { + if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) && + ((char *)dbuf.mv_data)[dbflags[i].len] == '=') { + flags |= dbflags[i].bit; + break; + } + } + if (!dbflags[i].bit) { + ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size); + if (!ptr) { + fprintf(stderr, "%s: line %" Z "d: unexpected format\n", + prog, lineno); + exit(EXIT_FAILURE); + } else { + *ptr = '\0'; + fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n", + prog, lineno, (char *)dbuf.mv_data); + } + } + } + } +} + +static void badend(void) +{ + fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n", + prog, lineno); +} + +static int unhex(unsigned char *c2) +{ + int x, c; + x = *c2++ & 0x4f; + if (x & 0x40) + x -= 55; + c = x << 4; + x = *c2 & 0x4f; + if (x & 0x40) + x -= 55; + c |= x; + return c; +} + +static int readline(MDB_val *out, MDB_val *buf) +{ + unsigned char *c1, *c2, *end; + size_t len; + int c; + + if (!(mode & NOHDR)) { + c = fgetc(stdin); + if (c == EOF) { + Eof = 1; + return EOF; + } + if (c != ' ') { + lineno++; + if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { +badend: + Eof = 1; + badend(); + return EOF; + } + if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END"))) + return EOF; + goto badend; + } + } + if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { + Eof = 1; + return EOF; + } + lineno++; + + c1 = buf->mv_data; + len = strlen((char *)c1); + + /* Is buffer too short? */ + while (c1[len-1] != '\n') { + buf->mv_data = realloc(buf->mv_data, buf->mv_size*2); + if (!buf->mv_data) { + Eof = 1; + fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n", + prog, lineno); + return EOF; + } + c1 = buf->mv_data; + c1 += buf->mv_size; + if (fgets((char *)c1, buf->mv_size, stdin) == NULL) { + Eof = 1; + badend(); + return EOF; + } + buf->mv_size *= 2; + len = strlen((char *)c1); + } + c1 = c2 = buf->mv_data; + len = strlen((char *)c1); + c1[--len] = '\0'; + end = c1 + len; + + if (mode & PRINT) { + while (c2 < end) { + if (*c2 == '\\') { + if (c2[1] == '\\') { + c1++; c2 += 2; + } else { + if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { + Eof = 1; + badend(); + return EOF; + } + *c1++ = unhex(++c2); + c2 += 2; + } + } else { + c1++; c2++; + } + } + } else { + /* odd length not allowed */ + if (len & 1) { + Eof = 1; + badend(); + return EOF; + } + while (c2 < end) { + if (!isxdigit(*c2) || !isxdigit(c2[1])) { + Eof = 1; + badend(); + return EOF; + } + *c1++ = unhex(c2); + c2 += 2; + } + } + c2 = out->mv_data = buf->mv_data; + out->mv_size = c1 - c2; + + return 0; +} + +static void usage(void) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-f input] [-n] [-s name] [-N] [-T]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_cursor *mc; + MDB_dbi dbi; + char *envname; + int envflags = 0, putflags = 0; + int dohdr = 0; + + prog = argv[0]; + + if (argc < 2) { + usage(); + } + + /* -f: load file instead of stdin + * -n: use NOSUBDIR flag on env_open + * -s: load into named subDB + * -N: use NOOVERWRITE on puts + * -T: read plaintext + * -V: print version and exit + */ + while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'f': + if (freopen(optarg, "r", stdin) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + prog, optarg, strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 's': + subname = strdup(optarg); + break; + case 'N': + putflags = MDB_NOOVERWRITE|MDB_NODUPDATA; + break; + case 'T': + mode |= NOHDR; + break; + default: + usage(); + } + } + + if (optind != argc - 1) + usage(); + + dbuf.mv_size = 4096; + dbuf.mv_data = malloc(dbuf.mv_size); + + if (!(mode & NOHDR)) + readhdr(); + + envname = argv[optind]; + rc = mdb_env_create(&env); + if (rc) { + fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); + return EXIT_FAILURE; + } + + mdb_env_set_maxdbs(env, 2); + + if (info.me_maxreaders) + mdb_env_set_maxreaders(env, info.me_maxreaders); + + if (info.me_mapsize) + mdb_env_set_mapsize(env, info.me_mapsize); + + if (info.me_mapaddr) + envflags |= MDB_FIXEDMAP; + + rc = mdb_env_open(env, envname, envflags, 0664); + if (rc) { + fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2; + kbuf.mv_data = malloc(kbuf.mv_size); + + while(!Eof) { + MDB_val key, data; + int batch = 0; + flags = 0; + + if (!dohdr) { + dohdr = 1; + } else if (!(mode & NOHDR)) + readhdr(); + + rc = mdb_txn_begin(env, NULL, 0, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi); + if (rc) { + fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + while(1) { + rc = readline(&key, &kbuf); + if (rc == EOF) + break; + if (rc) + goto txn_abort; + + rc = readline(&data, &dbuf); + if (rc) + goto txn_abort; + + rc = mdb_cursor_put(mc, &key, &data, putflags); + if (rc == MDB_KEYEXIST && putflags) + continue; + if (rc) + goto txn_abort; + batch++; + if (batch == 100) { + rc = mdb_txn_commit(txn); + if (rc) { + fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + prog, lineno, mdb_strerror(rc)); + goto env_close; + } + rc = mdb_txn_begin(env, NULL, 0, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + batch = 0; + } + } + rc = mdb_txn_commit(txn); + txn = NULL; + if (rc) { + fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + prog, lineno, mdb_strerror(rc)); + goto env_close; + } + mdb_dbi_close(env, dbi); + } + +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/submodules/lmdb/libraries/liblmdb/mdb_stat.1 b/submodules/lmdb/libraries/liblmdb/mdb_stat.1 new file mode 100644 index 0000000000..3d8d461d99 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mdb_stat.1 @@ -0,0 +1,64 @@ +.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_stat \- LMDB environment status tool +.SH SYNOPSIS +.B mdb_stat +.BR \ envpath +[\c +.BR \-V ] +[\c +.BR \-e ] +[\c +.BR \-f [ f [ f ]]] +[\c +.BR \-n ] +[\c +.BR \-r [ r ]] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.SH DESCRIPTION +The +.B mdb_stat +utility displays the status of an LMDB environment. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-e +Display information about the database environment. +.TP +.BR \-f +Display information about the environment freelist. +If \fB\-ff\fP is given, summarize each freelist entry. +If \fB\-fff\fP is given, display the full list of page IDs in the freelist. +.TP +.BR \-n +Display the status of an LMDB database which does not use subdirectories. +.TP +.BR \-r +Display information about the environment reader table. +Shows the process ID, thread ID, and transaction ID for each active +reader slot. The process ID and transaction ID are in decimal, the +thread ID is in hexadecimal. The transaction ID is displayed as "-" +if the reader does not currently have a read transaction open. +If \fB\-rr\fP is given, check for stale entries in the reader +table and clear them. The reader table will be printed again +after the check is performed. +.TP +.BR \-a +Display the status of all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Display the status of a specific subdatabase. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH "SEE ALSO" +.BR mdb_copy (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/submodules/lmdb/libraries/liblmdb/mdb_stat.c b/submodules/lmdb/libraries/liblmdb/mdb_stat.c new file mode 100644 index 0000000000..1e9229296f --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mdb_stat.c @@ -0,0 +1,263 @@ +/* mdb_stat.c - memory-mapped database status tool */ +/* + * Copyright 2011-2013 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include +#include +#include +#include "lmdb.h" + +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + +static void prstat(MDB_stat *ms) +{ +#if 0 + printf(" Page size: %u\n", ms->ms_psize); +#endif + printf(" Tree depth: %u\n", ms->ms_depth); + printf(" Branch pages: %"Z"u\n", ms->ms_branch_pages); + printf(" Leaf pages: %"Z"u\n", ms->ms_leaf_pages); + printf(" Overflow pages: %"Z"u\n", ms->ms_overflow_pages); + printf(" Entries: %"Z"u\n", ms->ms_entries); +} + +static void usage(char *prog) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + MDB_stat mst; + MDB_envinfo mei; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0; + + if (argc < 2) { + usage(prog); + } + + /* -a: print stat of main DB and all subDBs + * -s: print stat of only the named subDB + * -e: print env info + * -f: print freelist info + * -r: print reader info + * -n: use NOSUBDIR flag on env_open + * -V: print version and exit + * (default) print stat of only the main DB + */ + while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'e': + envinfo++; + break; + case 'f': + freinfo++; + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 'r': + rdrinfo++; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + + envname = argv[optind]; + rc = mdb_env_create(&env); + if (rc) { + fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); + return EXIT_FAILURE; + } + + if (alldbs || subname) { + mdb_env_set_maxdbs(env, 4); + } + + rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); + if (rc) { + fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + if (envinfo) { + (void)mdb_env_stat(env, &mst); + (void)mdb_env_info(env, &mei); + printf("Environment Info\n"); + printf(" Map address: %p\n", mei.me_mapaddr); + printf(" Map size: %"Z"u\n", mei.me_mapsize); + printf(" Page size: %u\n", mst.ms_psize); + printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize); + printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1); + printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid); + printf(" Max readers: %u\n", mei.me_maxreaders); + printf(" Number of readers used: %u\n", mei.me_numreaders); + } + + if (rdrinfo) { + printf("Reader Table Status\n"); + rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); + if (rdrinfo > 1) { + int dead; + mdb_reader_check(env, &dead); + printf(" %d stale readers cleared.\n", dead); + rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); + } + if (!(subname || alldbs || freinfo)) + goto env_close; + } + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + if (freinfo) { + MDB_cursor *cursor; + MDB_val key, data; + size_t pages = 0, *iptr; + + printf("Freelist Status\n"); + dbi = 0; + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + rc = mdb_stat(txn, dbi, &mst); + if (rc) { + fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + iptr = data.mv_data; + pages += *iptr; + if (freinfo > 1) { + char *bad = ""; + size_t pg, prev; + ssize_t i, j, span = 0; + j = *iptr++; + for (i = j, prev = 1; --i >= 0; ) { + pg = iptr[i]; + if (pg <= prev) + bad = " [bad sequence]"; + prev = pg; + pg += span; + for (; i >= span && iptr[i-span] == pg; span++, pg++) ; + } + printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n", + *(size_t *)key.mv_data, j, span, bad); + if (freinfo > 2) { + for (--j; j >= 0; ) { + pg = iptr[j]; + for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ; + printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n", + pg, span); + } + } + } + } + mdb_cursor_close(cursor); + printf(" Free pages: %"Z"u\n", pages); + } + + rc = mdb_open(txn, subname, 0, &dbi); + if (rc) { + fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + rc = mdb_stat(txn, dbi, &mst); + if (rc) { + fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + printf("Status of %s\n", subname ? subname : "Main DB"); + prstat(&mst); + + if (alldbs) { + MDB_cursor *cursor; + MDB_val key; + + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { + char *str; + MDB_dbi db2; + if (memchr(key.mv_data, '\0', key.mv_size)) + continue; + str = malloc(key.mv_size+1); + memcpy(str, key.mv_data, key.mv_size); + str[key.mv_size] = '\0'; + rc = mdb_open(txn, str, 0, &db2); + if (rc == MDB_SUCCESS) + printf("Status of %s\n", str); + free(str); + if (rc) continue; + rc = mdb_stat(txn, db2, &mst); + if (rc) { + fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + mdb_close(env, db2); + } + mdb_cursor_close(cursor); + } + + if (rc == MDB_NOTFOUND) + rc = MDB_SUCCESS; + + mdb_close(env, dbi); +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/submodules/lmdb/libraries/liblmdb/midl.c b/submodules/lmdb/libraries/liblmdb/midl.c new file mode 100644 index 0000000000..88a3aff10c --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/midl.c @@ -0,0 +1,360 @@ +/** @file midl.c + * @brief ldap bdb back-end ID List functions */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 2000-2014 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include +#include +#include +#include +#include +#include "midl.h" + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup idls ID List Management + * @{ + */ +#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) + +unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( ids[cursor], id ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +#if 0 /* superseded by append/sort */ +int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) +{ + unsigned x, i; + + x = mdb_midl_search( ids, id ); + assert( x > 0 ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0] && ids[x] == id ) { + /* duplicate */ + assert(0); + return -1; + } + + if ( ++ids[0] >= MDB_IDL_DB_MAX ) { + /* no room */ + --ids[0]; + return -2; + + } else { + /* insert id */ + for (i=ids[0]; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = id; + } + + return 0; +} +#endif + +MDB_IDL mdb_midl_alloc(int num) +{ + MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); + if (ids) { + *ids++ = num; + *ids = 0; + } + return ids; +} + +void mdb_midl_free(MDB_IDL ids) +{ + if (ids) + free(ids-1); +} + +int mdb_midl_shrink( MDB_IDL *idp ) +{ + MDB_IDL ids = *idp; + if (*(--ids) > MDB_IDL_UM_MAX && + (ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID)))) + { + *ids++ = MDB_IDL_UM_MAX; + *idp = ids; + return 1; + } + return 0; +} + +static int mdb_midl_grow( MDB_IDL *idp, int num ) +{ + MDB_IDL idn = *idp-1; + /* grow it */ + idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); + if (!idn) + return ENOMEM; + *idn++ += num; + *idp = idn; + return 0; +} + +int mdb_midl_need( MDB_IDL *idp, unsigned num ) +{ + MDB_IDL ids = *idp; + num += ids[0]; + if (num > ids[-1]) { + num = (num + num/4 + (256 + 2)) & -256; + if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) + return ENOMEM; + *ids++ = num - 2; + *idp = ids; + } + return 0; +} + +int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] >= ids[-1]) { + if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0]++; + ids[ids[0]] = id; + return 0; +} + +int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] + app[0] >= ids[-1]) { + if (mdb_midl_grow(idp, app[0])) + return ENOMEM; + ids = *idp; + } + memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); + ids[0] += app[0]; + return 0; +} + +int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) +{ + MDB_ID *ids = *idp, len = ids[0]; + /* Too big? */ + if (len + n > ids[-1]) { + if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0] = len + n; + ids += len; + while (n) + ids[n--] = id++; + return 0; +} + +void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) +{ + MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; + idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ + old_id = idl[j]; + while (i) { + merge_id = merge[i--]; + for (; old_id < merge_id; old_id = idl[--j]) + idl[k--] = old_id; + idl[k--] = merge_id; + } + idl[0] = total; +} + +/* Quicksort + Insertion sort for small arrays */ + +#define SMALL 8 +#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } + +void +mdb_midl_sort( MDB_IDL ids ) +{ + /* Max possible depth of int-indexed tree * 2 items/level */ + int istack[sizeof(int)*CHAR_BIT * 2]; + int i,j,k,l,ir,jstack; + MDB_ID a, itmp; + + ir = (int)ids[0]; + l = 1; + jstack = 0; + for(;;) { + if (ir - l < SMALL) { /* Insertion sort */ + for (j=l+1;j<=ir;j++) { + a = ids[j]; + for (i=j-1;i>=1;i--) { + if (ids[i] >= a) break; + ids[i+1] = ids[i]; + } + ids[i+1] = a; + } + if (jstack == 0) break; + ir = istack[jstack--]; + l = istack[jstack--]; + } else { + k = (l + ir) >> 1; /* Choose median of left, center, right */ + MIDL_SWAP(ids[k], ids[l+1]); + if (ids[l] < ids[ir]) { + MIDL_SWAP(ids[l], ids[ir]); + } + if (ids[l+1] < ids[ir]) { + MIDL_SWAP(ids[l+1], ids[ir]); + } + if (ids[l] < ids[l+1]) { + MIDL_SWAP(ids[l], ids[l+1]); + } + i = l+1; + j = ir; + a = ids[l+1]; + for(;;) { + do i++; while(ids[i] > a); + do j--; while(ids[j] < a); + if (j < i) break; + MIDL_SWAP(ids[i],ids[j]); + } + ids[l+1] = ids[j]; + ids[j] = a; + jstack += 2; + if (ir-i+1 >= j-l) { + istack[jstack] = ir; + istack[jstack-1] = i; + ir = j-1; + } else { + istack[jstack] = j-1; + istack[jstack-1] = l; + l = i; + } + } + } +} + +unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = (unsigned)ids[0].mid; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( id, ids[cursor].mid ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) +{ + unsigned x, i; + + x = mdb_mid2l_search( ids, id->mid ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0].mid && ids[x].mid == id->mid ) { + /* duplicate */ + return -1; + } + + if ( ids[0].mid >= MDB_IDL_UM_MAX ) { + /* too big */ + return -2; + + } else { + /* insert id */ + ids[0].mid++; + for (i=(unsigned)ids[0].mid; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = *id; + } + + return 0; +} + +int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) +{ + /* Too big? */ + if (ids[0].mid >= MDB_IDL_UM_MAX) { + return -2; + } + ids[0].mid++; + ids[ids[0].mid] = *id; + return 0; +} + +/** @} */ +/** @} */ diff --git a/submodules/lmdb/libraries/liblmdb/midl.h b/submodules/lmdb/libraries/liblmdb/midl.h new file mode 100644 index 0000000000..a7f25026ce --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/midl.h @@ -0,0 +1,186 @@ +/** @file midl.h + * @brief LMDB ID List header file. + * + * This file was originally part of back-bdb but has been + * modified for use in libmdb. Most of the macros defined + * in this file are unused, just left over from the original. + * + * This file is only used internally in libmdb and its definitions + * are not exposed publicly. + */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 2000-2014 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _MDB_MIDL_H_ +#define _MDB_MIDL_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ + +/** @defgroup idls ID List Management + * @{ + */ + /** A generic unsigned ID number. These were entryIDs in back-bdb. + * Preferably it should have the same size as a pointer. + */ +typedef size_t MDB_ID; + + /** An IDL is an ID List, a sorted array of IDs. The first + * element of the array is a counter for how many actual + * IDs are in the list. In the original back-bdb code, IDLs are + * sorted in ascending order. For libmdb IDLs are sorted in + * descending order. + */ +typedef MDB_ID *MDB_IDL; + +/* IDL sizes - likely should be even bigger + * limiting factors: sizeof(ID), thread stack size + */ +#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ +#define MDB_IDL_DB_SIZE (1<. + */ +#include +#include +#include +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor, *cur2; + MDB_cursor_op op; + int count; + int *values; + char sval[32] = ""; + + srand(time(NULL)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(sval, "%03x ", values[i]); + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor last\n"); + E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor last/prev\n"); + E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + E(mdb_cursor_get(cursor, &key, &data, MDB_PREV)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + + mdb_txn_abort(txn); + + printf("Deleting with cursor\n"); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cur2)); + for (i=0; i<50; i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + E(mdb_del(txn, dbi, &key, NULL)); + } + + printf("Restarting cursor in txn\n"); + for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cur2); + E(mdb_txn_commit(txn)); + + printf("Restarting cursor outside txn\n"); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/submodules/lmdb/libraries/liblmdb/mtest2.c b/submodules/lmdb/libraries/liblmdb/mtest2.c new file mode 100644 index 0000000000..f1a3dbd6c8 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mtest2.c @@ -0,0 +1,123 @@ +/* mtest2.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Just like mtest.c, but using a subDB instead of the main DB */ + +#include +#include +#include +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32] = ""; + + srand(time(NULL)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(sval, "%03x ", values[i]); + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/submodules/lmdb/libraries/liblmdb/mtest3.c b/submodules/lmdb/libraries/liblmdb/mtest3.c new file mode 100644 index 0000000000..f705c52dec --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mtest3.c @@ -0,0 +1,133 @@ +/* mtest3.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Tests for sorted duplicate DBs */ +#include +#include +#include +#include +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32]; + char kval[sizeof(int)]; + + srand(time(NULL)); + + memset(sval, 0, sizeof(sval)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/submodules/lmdb/libraries/liblmdb/mtest4.c b/submodules/lmdb/libraries/liblmdb/mtest4.c new file mode 100644 index 0000000000..da5a953044 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mtest4.c @@ -0,0 +1,168 @@ +/* mtest4.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Tests for sorted duplicate DBs with fixed-size keys */ +#include +#include +#include +#include +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[8]; + char kval[sizeof(int)]; + + memset(sval, 0, sizeof(sval)); + + count = 510; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (rand()%3)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(sval, "%07x", values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/submodules/lmdb/libraries/liblmdb/mtest5.c b/submodules/lmdb/libraries/liblmdb/mtest5.c new file mode 100644 index 0000000000..39a8c728ac --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mtest5.c @@ -0,0 +1,135 @@ +/* mtest5.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Tests for sorted duplicate DBs using cursor_put */ +#include +#include +#include +#include +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32]; + char kval[sizeof(int)]; + + srand(time(NULL)); + + memset(sval, 0, sizeof(sval)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/submodules/lmdb/libraries/liblmdb/mtest6.c b/submodules/lmdb/libraries/liblmdb/mtest6.c new file mode 100644 index 0000000000..07d5758932 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/mtest6.c @@ -0,0 +1,137 @@ +/* mtest6.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* Tests for DB splits and merges */ +#include +#include +#include +#include +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +char dkbuf[1024]; + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + long kval; + char *sval; + + srand(time(NULL)); + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi)); + E(mdb_cursor_open(txn, dbi, &cursor)); + E(mdb_stat(txn, dbi, &mst)); + + sval = calloc(1, mst.ms_psize / 4); + key.mv_size = sizeof(long); + key.mv_data = &kval; + data.mv_size = mst.ms_psize / 4 - 30; + data.mv_data = sval; + + printf("Adding 12 values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5; + sprintf(sval, "%08x", kval); + (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); + } + printf("Adding 12 more values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5+4; + sprintf(sval, "%08x", kval); + (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); + } + printf("Adding 12 more values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5+1; + sprintf(sval, "%08x", kval); + (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); + } + E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST)); + + do { + printf("key: %p %s, data: %p %.*s\n", + key.mv_data, mdb_dkey(&key, dkbuf), + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0); + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_commit(txn); + +#if 0 + j=0; + + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); +#endif + mdb_env_close(env); + + return 0; +} diff --git a/submodules/lmdb/libraries/liblmdb/sample-bdb.txt b/submodules/lmdb/libraries/liblmdb/sample-bdb.txt new file mode 100644 index 0000000000..6a959bd685 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/sample-bdb.txt @@ -0,0 +1,73 @@ +/* sample-bdb.txt - BerkeleyDB toy/sample + * + * Do a line-by-line comparison of this and sample-mdb.txt + */ +/* + * Copyright 2012 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include +#include + +int main(int argc,char * argv[]) +{ + int rc; + DB_ENV *env; + DB *dbi; + DBT key, data; + DB_TXN *txn; + DBC *cursor; + char sval[32], kval[32]; + + /* Note: Most error checking omitted for simplicity */ + +#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD) + rc = db_env_create(&env, 0); + rc = env->open(env, "./testdb", FLAGS, 0664); + rc = db_create(&dbi, env, 0); + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664); + + memset(&key, 0, sizeof(DBT)); + memset(&data, 0, sizeof(DBT)); + key.size = sizeof(int); + key.data = sval; + data.size = sizeof(sval); + data.data = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = dbi->put(dbi, txn, &key, &data, 0); + rc = txn->commit(txn, 0); + if (rc) { + fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc)); + goto leave; + } + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->cursor(dbi, txn, &cursor, 0); + key.flags = DB_DBT_USERMEM; + key.data = kval; + key.ulen = sizeof(kval); + data.flags = DB_DBT_USERMEM; + data.data = sval; + data.ulen = sizeof(sval); + while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.data, (int) key.size, (char *) key.data, + data.data, (int) data.size, (char *) data.data); + } + rc = cursor->c_close(cursor); + rc = txn->abort(txn); +leave: + rc = dbi->close(dbi, 0); + rc = env->close(env, 0); + return rc; +} diff --git a/submodules/lmdb/libraries/liblmdb/sample-mdb.txt b/submodules/lmdb/libraries/liblmdb/sample-mdb.txt new file mode 100644 index 0000000000..a233ec5dd3 --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/sample-mdb.txt @@ -0,0 +1,62 @@ +/* sample-mdb.txt - MDB toy/sample + * + * Do a line-by-line comparison of this and sample-bdb.txt + */ +/* + * Copyright 2012 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include "lmdb.h" + +int main(int argc,char * argv[]) +{ + int rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_cursor *cursor; + char sval[32]; + + /* Note: Most error checking omitted for simplicity */ + + rc = mdb_env_create(&env); + rc = mdb_env_open(env, "./testdb", 0, 0664); + rc = mdb_txn_begin(env, NULL, 0, &txn); + rc = mdb_open(txn, NULL, 0, &dbi); + + key.mv_size = sizeof(int); + key.mv_data = sval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = mdb_put(txn, dbi, &key, &data, 0); + rc = mdb_txn_commit(txn); + if (rc) { + fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc)); + goto leave; + } + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_txn_abort(txn); +leave: + mdb_close(env, dbi); + mdb_env_close(env); + return 0; +} diff --git a/submodules/lmdb/libraries/liblmdb/tooltag b/submodules/lmdb/libraries/liblmdb/tooltag new file mode 100644 index 0000000000..229bf16baa --- /dev/null +++ b/submodules/lmdb/libraries/liblmdb/tooltag @@ -0,0 +1,22 @@ + + + mdb_copy_1 + mdb_copy - environment copy tool + mdb_copy.1 + + + mdb_dump_1 + mdb_dump - environment export tool + mdb_dump.1 + + + mdb_load_1 + mdb_load - environment import tool + mdb_load.1 + + + mdb_stat_1 + mdb_stat - environment status tool + mdb_stat.1 + + From ef90efbd29f9ced7fdd9a68bd49c111079b749bb Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 22 Sep 2015 15:24:29 +0300 Subject: [PATCH 023/427] no message --- submodules/SQLite.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/SQLite.swift b/submodules/SQLite.swift index dca58041a5..9fe0d2df4b 160000 --- a/submodules/SQLite.swift +++ b/submodules/SQLite.swift @@ -1 +1 @@ -Subproject commit dca58041a5b02d508d967d72a3b3e48b825692e4 +Subproject commit 9fe0d2df4beac6ceea4b208bcc9e9b05db2e6b61 From e783a814f3bbded444d0f29d00fa04ae09e89f96 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 22 Sep 2015 20:58:49 +0300 Subject: [PATCH 024/427] no message --- Postbox.xcodeproj/project.pbxproj | 7 ++++++- .../peter.xcuserdatad/xcschemes/PostboxTests.xcscheme | 2 +- Postbox/Info.plist | 2 +- PostboxTests/Info.plist | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index fb5836c807..9c8cd5f2b6 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -316,7 +316,7 @@ attributes = { LastSwiftMigration = 0700; LastSwiftUpdateCheck = 0700; - LastUpgradeCheck = 0630; + LastUpgradeCheck = 0710; ORGANIZATIONNAME = Telegram; TargetAttributes = { D0E3A7491B28A7E300A402D9 = { @@ -438,6 +438,7 @@ CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; GCC_C_LANGUAGE_STANDARD = gnu99; GCC_DYNAMIC_NO_PIC = NO; GCC_NO_COMMON_BLOCKS = YES; @@ -530,6 +531,7 @@ "-DSQLCIPHER_CRYPTO_CC=1", "-DSQLITE_ENABLE_FTS3", ); + PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; @@ -559,6 +561,7 @@ "-DSQLCIPHER_CRYPTO_CC=1", "-DSQLITE_ENABLE_FTS3", ); + PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; @@ -580,6 +583,7 @@ ); INFOPLIST_FILE = PostboxTests/Info.plist; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; }; name = Debug; @@ -594,6 +598,7 @@ ); INFOPLIST_FILE = PostboxTests/Info.plist; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; }; name = Release; diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme index 2a374db351..35ff5f38d6 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme @@ -1,6 +1,6 @@ CFBundleExecutable $(EXECUTABLE_NAME) CFBundleIdentifier - org.telegram.$(PRODUCT_NAME:rfc1034identifier) + $(PRODUCT_BUNDLE_IDENTIFIER) CFBundleInfoDictionaryVersion 6.0 CFBundleName diff --git a/PostboxTests/Info.plist b/PostboxTests/Info.plist index 29dacae6b5..ba72822e87 100644 --- a/PostboxTests/Info.plist +++ b/PostboxTests/Info.plist @@ -7,7 +7,7 @@ CFBundleExecutable $(EXECUTABLE_NAME) CFBundleIdentifier - org.telegram.$(PRODUCT_NAME:rfc1034identifier) + $(PRODUCT_BUNDLE_IDENTIFIER) CFBundleInfoDictionaryVersion 6.0 CFBundleName From b34244cae86d99fc081945970a03d44bea1d65d4 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 26 Jan 2016 16:16:40 +0300 Subject: [PATCH 025/427] no message --- Postbox.xcodeproj/project.pbxproj | 36 +- .../xcschemes/PostboxTests.xcscheme | 3 +- Postbox/Coding.swift | 4 + Postbox/LmdbValueBox.swift | 56 ++ Postbox/Media.swift | 12 +- Postbox/MediaCleanupTable.swift | 21 + Postbox/Message.swift | 77 ++- Postbox/MessageHistoryHole.swift | 14 + Postbox/MessageHistoryIndexTable.swift | 322 +++++++++ Postbox/MessageHistoryTable.swift | 359 ++++++++++ Postbox/MessageHistoryView.swift | 341 +++++++++ Postbox/MessageMediaTable.swift | 210 ++++++ Postbox/MessageView.swift | 428 ------------ Postbox/Peer.swift | 10 +- Postbox/Postbox.swift | 646 ++---------------- Postbox/PostboxTables.swift | 54 +- Postbox/SqliteValueBox.swift | 61 +- Postbox/ValueBox.swift | 3 + PostboxTests/CodingTests.swift | 43 +- .../MessageHistoryIndexTableTests.swift | 427 ++++++++++++ PostboxTests/MessageHistoryTableTests.swift | 306 +++++++++ 21 files changed, 2335 insertions(+), 1098 deletions(-) create mode 100644 Postbox/MediaCleanupTable.swift create mode 100644 Postbox/MessageHistoryHole.swift create mode 100644 Postbox/MessageHistoryIndexTable.swift create mode 100644 Postbox/MessageHistoryTable.swift create mode 100644 Postbox/MessageHistoryView.swift create mode 100644 Postbox/MessageMediaTable.swift delete mode 100644 Postbox/MessageView.swift create mode 100644 PostboxTests/MessageHistoryIndexTableTests.swift create mode 100644 PostboxTests/MessageHistoryTableTests.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 9c8cd5f2b6..dec81ee69a 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -7,7 +7,7 @@ objects = { /* Begin PBXBuildFile section */ - D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageView.swift */; }; + D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; D00E0FB31B84CEDA002E4EB5 /* Display.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D00E0FB21B84CEDA002E4EB5 /* Display.framework */; }; D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB41B85D192002E4EB5 /* lmdb.h */; }; D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; }; @@ -29,11 +29,15 @@ D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; + D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; + D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; + D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; D0977FA21B82930C009994B2 /* PostboxCodingUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */; }; D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA31B829F87009994B2 /* PostboxTables.swift */; }; + D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B76BE61B66639F0095CF45 /* DeferredString.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; @@ -45,6 +49,9 @@ D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; + D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; + D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */; }; + D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -58,7 +65,7 @@ /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ - D003E4E51B38DBDB00C22CBC /* MessageView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageView.swift; sourceTree = ""; }; + D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryView.swift; sourceTree = ""; }; D00E0FB21B84CEDA002E4EB5 /* Display.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Display.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/Display.framework"; sourceTree = ""; }; D00E0FB41B85D192002E4EB5 /* lmdb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lmdb.h; path = submodules/lmdb/libraries/liblmdb/lmdb.h; sourceTree = SOURCE_ROOT; }; D00E0FB51B85D192002E4EB5 /* mdb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = mdb.c; path = submodules/lmdb/libraries/liblmdb/mdb.c; sourceTree = SOURCE_ROOT; }; @@ -81,11 +88,15 @@ D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; + D08C71391C501F0700779C0F /* MessageHistoryHole.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHole.swift; sourceTree = ""; }; + D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTable.swift; sourceTree = ""; }; + D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTable.swift; sourceTree = ""; }; D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = ""; }; D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxCodingUtils.swift; sourceTree = ""; }; D0977FA31B829F87009994B2 /* PostboxTables.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxTables.swift; sourceTree = ""; }; + D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; @@ -100,6 +111,9 @@ D0E3A7871B28AE9C00A402D9 /* Coding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Coding.swift; sourceTree = ""; }; D0E3A79D1B28B50400A402D9 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = ""; }; D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = ""; }; + D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageMediaTable.swift; sourceTree = ""; }; + D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTableTests.swift; sourceTree = ""; }; + D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaCleanupTable.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -207,8 +221,9 @@ D0B76BE61B66639F0095CF45 /* DeferredString.swift */, D0E3A7831B28AE0900A402D9 /* Peer.swift */, D0E3A79D1B28B50400A402D9 /* Message.swift */, + D08C71391C501F0700779C0F /* MessageHistoryHole.swift */, D0E3A7A11B28B7DC00A402D9 /* Media.swift */, - D003E4E51B38DBDB00C22CBC /* MessageView.swift */, + D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */, D0D225251B4D84930085E26D /* PeerView.swift */, D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */, D0E3A7811B28ADD000A402D9 /* Postbox.swift */, @@ -218,6 +233,10 @@ D0977F9B1B822DB4009994B2 /* ValueBox.swift */, D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */, D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */, + D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */, + D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, + D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */, + D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */, D0E3A74D1B28A7E300A402D9 /* Supporting Files */, ); path = Postbox; @@ -239,6 +258,8 @@ children = ( D0E3A75A1B28A7E300A402D9 /* Supporting Files */, D044E15D1B2ACB9C001EE087 /* CodingTests.swift */, + D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */, + D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */, ); path = PostboxTests; sourceTree = ""; @@ -367,24 +388,29 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */, D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */, + D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, + D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, + D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */, - D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */, + D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */, D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */, D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, + D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D0D225261B4D84930085E26D /* PeerView.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, @@ -401,6 +427,8 @@ buildActionMask = 2147483647; files = ( D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */, + D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */, + D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme index 35ff5f38d6..fdd99d4b63 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme @@ -10,7 +10,8 @@ buildConfiguration = "Debug" selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" - shouldUseLaunchSchemeArgsEnv = "YES"> + shouldUseLaunchSchemeArgsEnv = "YES" + codeCoverageEnabled = "YES"> diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index a90fbc0d63..e084d99829 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -137,6 +137,10 @@ public final class ReadBuffer: MemoryBuffer { self.offset += length } + func skip(length: Int) { + self.offset += length + } + func reset() { self.offset = 0 } diff --git a/Postbox/LmdbValueBox.swift b/Postbox/LmdbValueBox.swift index f433362f47..6ad6b4b304 100644 --- a/Postbox/LmdbValueBox.swift +++ b/Postbox/LmdbValueBox.swift @@ -98,6 +98,10 @@ public final class LmdbValueBox: ValueBox { private var sharedTxn: COpaquePointer = nil + private var readQueryTime: CFAbsoluteTime = 0.0 + private var writeQueryTime: CFAbsoluteTime = 0.0 + private var commitTime: CFAbsoluteTime = 0.0 + public init?(basePath: String) { var result = mdb_env_create(&self.env) if result != MDB_SUCCESS { @@ -163,6 +167,16 @@ public final class LmdbValueBox: ValueBox { return LmdbTable(dbi: dbi) } + public func beginStats() { + self.readQueryTime = 0.0 + self.writeQueryTime = 0.0 + self.commitTime = 0.0 + } + + public func endStats() { + print("(LmdbValueBox stats read: \(self.readQueryTime * 1000.0) ms, write: \(self.writeQueryTime * 1000.0) ms, commit: \(self.commitTime * 1000.0) ms") + } + public func begin() { if self.sharedTxn != nil { print("(LmdbValueBox already in transaction)") @@ -176,11 +190,13 @@ public final class LmdbValueBox: ValueBox { } public func commit() { + let startTime = CFAbsoluteTimeGetCurrent() if self.sharedTxn == nil { print("(LmdbValueBox already no current transaction)") } else { let result = mdb_txn_commit(self.sharedTxn) self.sharedTxn = nil + self.commitTime += CFAbsoluteTimeGetCurrent() - startTime if result != MDB_SUCCESS { print("(LmdbValueBox txn_commit failed with \(result))") return @@ -208,6 +224,7 @@ public final class LmdbValueBox: ValueBox { } if let nativeTable = nativeTable { + var startTime = CFAbsoluteTimeGetCurrent() var cursorPtr: COpaquePointer = nil let result = mdb_cursor_open(self.sharedTxn, nativeTable.dbi, &cursorPtr) if result != MDB_SUCCESS { @@ -223,6 +240,10 @@ public final class LmdbValueBox: ValueBox { } } + var currentTime = CFAbsoluteTimeGetCurrent() + readQueryTime += currentTime - startTime + startTime = currentTime + var count = 0 if value != nil && value!.0 < end { count++ @@ -230,13 +251,21 @@ public final class LmdbValueBox: ValueBox { } while value != nil && value!.0 < end && count < limit { + startTime = CFAbsoluteTimeGetCurrent() + value = cursor.next() + + currentTime = CFAbsoluteTimeGetCurrent() + readQueryTime += currentTime - startTime + startTime = currentTime + if value != nil && value!.0 < end { count++ values(value!.0, value!.1) } } } else { + var startTime = CFAbsoluteTimeGetCurrent() var value = cursor.seekTo(start, forward: false) if value != nil { if value!.0 == start { @@ -244,6 +273,10 @@ public final class LmdbValueBox: ValueBox { } } + var currentTime = CFAbsoluteTimeGetCurrent() + readQueryTime += currentTime - startTime + startTime = currentTime + var count = 0 if value != nil && value!.0 > end { count++ @@ -251,7 +284,14 @@ public final class LmdbValueBox: ValueBox { } while value != nil && value!.0 > end && count < limit { + startTime = CFAbsoluteTimeGetCurrent() + value = cursor.previous() + + currentTime = CFAbsoluteTimeGetCurrent() + readQueryTime += currentTime - startTime + startTime = currentTime + if value != nil && value!.0 > end { count++ values(value!.0, value!.1) @@ -264,7 +304,11 @@ public final class LmdbValueBox: ValueBox { } if commit { + let startTime = CFAbsoluteTimeGetCurrent() + self.commit() + + readQueryTime += CFAbsoluteTimeGetCurrent() - startTime } } @@ -275,6 +319,8 @@ public final class LmdbValueBox: ValueBox { } public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? { + let startTime = CFAbsoluteTimeGetCurrent() + var commit = false if self.sharedTxn == nil { self.begin() @@ -315,6 +361,8 @@ public final class LmdbValueBox: ValueBox { self.commit() } + readQueryTime += CFAbsoluteTimeGetCurrent() - startTime + return resultValue } @@ -323,6 +371,8 @@ public final class LmdbValueBox: ValueBox { } public func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) { + let startTime = CFAbsoluteTimeGetCurrent() + var commit = false if self.sharedTxn == nil { self.begin() @@ -355,9 +405,13 @@ public final class LmdbValueBox: ValueBox { if commit { self.commit() } + + writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } public func remove(table: Int32, key: ValueBoxKey) { + let startTime = CFAbsoluteTimeGetCurrent() + var commit = false if self.sharedTxn == nil { self.begin() @@ -387,6 +441,8 @@ public final class LmdbValueBox: ValueBox { if commit { self.commit() } + + writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } public func drop() { diff --git a/Postbox/Media.swift b/Postbox/Media.swift index a62a74759d..4cd9848945 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -25,11 +25,13 @@ public struct MediaId: Hashable, CustomStringConvertible { } public init(_ buffer: ReadBuffer) { - self.namespace = 0 - self.id = 0 + var namespace: Int32 = 0 + var id: Int64 = 0 - memcpy(&self.namespace, buffer.memory + buffer.offset, 4) - memcpy(&self.id, buffer.memory + (buffer.offset + 4), 8) + memcpy(&namespace, buffer.memory + buffer.offset, 4) + self.namespace = namespace + memcpy(&id, buffer.memory + (buffer.offset + 4), 8) + self.id = id buffer.offset += 12 } @@ -68,4 +70,6 @@ public func ==(lhs: MediaId, rhs: MediaId) -> Bool { public protocol Media: Coding { var id: MediaId? { get } + + func isEqual(other: Media) -> Bool } diff --git a/Postbox/MediaCleanupTable.swift b/Postbox/MediaCleanupTable.swift new file mode 100644 index 0000000000..d86e3c91a7 --- /dev/null +++ b/Postbox/MediaCleanupTable.swift @@ -0,0 +1,21 @@ +import Foundation + +final class MediaCleanupTable { + let valueBox: ValueBox + let tableId: Int32 + + var debugMedia: [Media] = [] + + init(valueBox: ValueBox, tableId: Int32) { + self.valueBox = valueBox + self.tableId = tableId + } + + func add(media: Media, sharedEncoder: Encoder = Encoder()) { + debugMedia.append(media) + } + + func debugList() -> [Media] { + return self.debugMedia + } +} \ No newline at end of file diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 62e0829108..54ac816893 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -1,6 +1,6 @@ import Foundation -public struct MessageId: Hashable, CustomStringConvertible { +public struct MessageId: Hashable, Comparable, CustomStringConvertible { public typealias Namespace = Int32 public typealias Id = Int32 @@ -81,6 +81,14 @@ public func ==(lhs: MessageId, rhs: MessageId) -> Bool { return lhs.id == rhs.id && lhs.namespace == rhs.namespace } +public func <(lhs: MessageId, rhs: MessageId) -> Bool { + if lhs.namespace == rhs.namespace { + return lhs.id < rhs.id + } else { + return lhs.namespace < rhs.namespace + } +} + public struct MessageIndex: Equatable, Comparable { public let id: MessageId public let timestamp: Int32 @@ -90,10 +98,23 @@ public struct MessageIndex: Equatable, Comparable { self.timestamp = message.timestamp } + init(_ message: StoreMessage) { + self.id = message.id + self.timestamp = message.timestamp + } + public init(id: MessageId, timestamp: Int32) { self.id = id self.timestamp = timestamp } + + public func predecessor() -> MessageIndex { + return MessageIndex(id: MessageId(peerId: self.id.peerId, namespace: self.id.namespace, id: self.id.id - 1), timestamp: self.timestamp) + } + + public func successor() -> MessageIndex { + return MessageIndex(id: MessageId(peerId: self.id.peerId, namespace: self.id.namespace, id: self.id.id + 1), timestamp: self.timestamp) + } } public func ==(lhs: MessageIndex, rhs: MessageIndex) -> Bool { @@ -112,12 +133,54 @@ public func <(lhs: MessageIndex, rhs: MessageIndex) -> Bool { return lhs.id.id < rhs.id.id } -public protocol Message: Coding { - var id: MessageId { get } - var timestamp: Int32 { get } - var text: String { get } - var mediaIds: [MediaId] { get } - var peerIds: [PeerId] { get } +public class Message { + let id: MessageId + let timestamp: Int32 + let text: String + let attributes: [Coding] + let media: [Media] + + init(id: MessageId, timestamp: Int32, text: String, attributes: [Coding], media: [Media]) { + self.id = id + self.timestamp = timestamp + self.text = text + self.attributes = attributes + self.media = media + } +} + +class StoreMessage { + let id: MessageId + let timestamp: Int32 + let text: String + let attributes: [Coding] + let media: [Media] + + init(id: MessageId, timestamp: Int32, text: String, attributes: [Coding], media: [Media]) { + self.id = id + self.timestamp = timestamp + self.text = text + self.attributes = attributes + self.media = media + } +} + +class IntermediateMessage { + let id: MessageId + let timestamp: Int32 + let text: String + let attributesData: ReadBuffer + let embeddedMediaData: ReadBuffer + let referencedMedia: [MediaId] + + init(id: MessageId, timestamp: Int32, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { + self.id = id + self.timestamp = timestamp + self.text = text + self.attributesData = attributesData + self.embeddedMediaData = embeddedMediaData + self.referencedMedia = referencedMedia + } } public struct RenderedMessage: Equatable, Comparable { diff --git a/Postbox/MessageHistoryHole.swift b/Postbox/MessageHistoryHole.swift new file mode 100644 index 0000000000..e43822bba9 --- /dev/null +++ b/Postbox/MessageHistoryHole.swift @@ -0,0 +1,14 @@ +import Foundation + +public struct MessageHistoryHole: Equatable { + let maxIndex: MessageIndex + let min: MessageId.Id + + var id: MessageId { + return maxIndex.id + } +} + +public func ==(lhs: MessageHistoryHole, rhs: MessageHistoryHole) -> Bool { + return lhs.maxIndex == rhs.maxIndex && lhs.min == rhs.min +} diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift new file mode 100644 index 0000000000..feb548012c --- /dev/null +++ b/Postbox/MessageHistoryIndexTable.swift @@ -0,0 +1,322 @@ +import Foundation + +enum HistoryIndexEntry { + case Message(MessageIndex) + case Hole(MessageHistoryHole) +} + +enum HoleFillType { + case UpperToLower + case LowerToUpper + case Complete +} + +private func readHistoryIndexEntry(peerId: PeerId, namespace: MessageId.Namespace, key: ValueBoxKey, value: ReadBuffer) -> HistoryIndexEntry { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + var timestamp: Int32 = 0 + value.read(×tamp, offset: 0, length: 4) + let index = MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: key.getInt32(8 + 4)), timestamp: timestamp) + + if type == 0 { + return .Message(index) + } else { + var min: Int32 = 0 + value.read(&min, offset: 0, length: 4) + return .Hole(MessageHistoryHole(maxIndex: index, min: min)) + } +} + +final class MessageHistoryIndexTable { + let valueBox: ValueBox + let tableId: Int32 + + init(valueBox: ValueBox, tableId: Int32) { + self.valueBox = valueBox + self.tableId = tableId + } + + func key(id: MessageId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4 + 4) + key.setInt64(0, value: id.peerId.toInt64()) + key.setInt32(8, value: id.namespace) + key.setInt32(8 + 4, value: id.id) + return key + } + + func lowerBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setInt32(8, value: namespace) + return key + } + + func upperBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setInt32(8, value: namespace) + return key.successor + } + + func addHole(id: MessageId) { + let adjacent = self.adjacentItems(id) + + /* + + 1. [x] nothing + 2. [x] message - * - nothing + 3. [x] nothing - * - message + 4. [x] nothing - * - hole + 5. [x] message - * - message + 6. [x] hole - * - message + 7. [x] message - * - hole + + */ + + if let lowerItem = adjacent.lower, upperItem = adjacent.upper { + switch lowerItem { + case .Hole: + break + case let .Message(lowerMessage): + switch upperItem { + case .Hole: + break + case let .Message(upperMessage): + if lowerMessage.id.id < upperMessage.id.id - 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerMessage.id.id + 1)) + } + break + } + } + } else if let lowerItem = adjacent.lower { + switch lowerItem { + case let .Message(lowerMessage): + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerMessage.id.id + 1)) + case .Hole: + break + } + } else if let upperItem = adjacent.upper { + switch upperItem { + case let .Message(upperMessage): + if upperMessage.id.id > 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: 1)) + } + case .Hole: + break + } + } else { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: 1)) + } + } + + func addMessage(index: MessageIndex) { + var upperItem: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: self.key(index.id).predecessor, end: self.upperBound(index.id.peerId, namespace: index.id.namespace), values: { key, value in + upperItem = readHistoryIndexEntry(index.id.peerId, namespace: index.id.namespace, key: key, value: value) + return true + }, limit: 1) + + if let upperItem = upperItem { + switch upperItem { + case let .Hole(upperHole): + self.justRemove(upperHole.id) + if upperHole.maxIndex.id.id > index.id.id + 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: index.id.id + 1)) + } + if upperHole.min <= index.id.id - 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: index.id.peerId, namespace: index.id.namespace, id: index.id.id - 1), timestamp: index.timestamp), min: upperHole.min)) + } + break + case .Message: + break + } + } + + self.justInsertMessage(index) + } + + func removeMessage(id: MessageId) { + let key = self.key(id) + if self.valueBox.exists(self.tableId, key: key) { + self.justRemove(id) + + let adjacent = self.adjacentItems(id) + + if let lowerItem = adjacent.lower, upperItem = adjacent.upper { + switch lowerItem { + case let .Message(lowerMessage): + switch upperItem { + case let .Hole(upperHole): + self.justRemove(upperHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerMessage.id.id + 1)) + case .Message: + break + } + case let .Hole(lowerHole): + switch upperItem { + case let .Hole(upperHole): + self.justRemove(lowerHole.id) + self.justRemove(upperHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerHole.min)) + case let .Message(upperMessage): + self.justRemove(lowerHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerHole.min)) + } + } + } else if let lowerItem = adjacent.lower { + switch lowerItem { + case let .Hole(lowerHole): + self.justRemove(lowerHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerHole.min)) + break + case .Message: + break + } + } else if let upperItem = adjacent.upper { + switch upperItem { + case let .Hole(upperHole): + self.justRemove(upperHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: 1)) + break + case .Message: + break + } + } + } + } + + func fillHole(id: MessageId, fillType: HoleFillType, indices: [MessageIndex]) { + var upperItem: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: self.key(id).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in + upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) + return true + }, limit: 1) + + let sortedByIdIndices = indices.sort({$0.id < $1.id}) + var remainingIndices = sortedByIdIndices + + if let upperItem = upperItem { + switch upperItem { + case let .Hole(upperHole): + var i = 0 + var minIndexInRange: MessageIndex? + var maxIndexInRange: MessageIndex? + var removedHole = false + while i < remainingIndices.count { + let index = remainingIndices[i] + if index.id.id >= upperHole.min && index.id.id <= upperHole.maxIndex.id.id { + if (fillType == .UpperToLower || fillType == .Complete) && (minIndexInRange == nil || minIndexInRange!.id > index.id) { + minIndexInRange = index + if !removedHole { + removedHole = true + self.justRemove(upperHole.id) + } + } + if (fillType == .LowerToUpper || fillType == .Complete) && (maxIndexInRange == nil || maxIndexInRange!.id < index.id) { + maxIndexInRange = index + if !removedHole { + removedHole = true + self.justRemove(upperHole.id) + } + } + self.justInsertMessage(index) + remainingIndices.removeAtIndex(i) + } else { + i++ + } + } + switch fillType { + case .Complete: + if !removedHole { + self.justRemove(upperHole.id) + } + case .LowerToUpper: + if let maxIndexInRange = maxIndexInRange where maxIndexInRange.id.id != Int32.max && maxIndexInRange.id.id + 1 <= upperHole.maxIndex.id.id { + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: maxIndexInRange.id.id + 1)) + } + case .UpperToLower: + if let minIndexInRange = minIndexInRange where minIndexInRange.id.id - 1 >= upperHole.min { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minIndexInRange.id.id - 1), timestamp: minIndexInRange.timestamp), min: upperHole.min)) + } + } + break + case .Message: + break + } + } + + for index in remainingIndices { + self.addMessage(index) + } + } + + func justInsertHole(hole: MessageHistoryHole) { + let value = WriteBuffer() + var type: Int8 = 1 + var timestamp: Int32 = hole.maxIndex.timestamp + var min: Int32 = hole.min + value.write(&type, offset: 0, length: 1) + value.write(×tamp, offset: 0, length: 4) + value.write(&min, offset: 0, length: 4) + self.valueBox.set(self.tableId, key: self.key(hole.id), value: value) + } + + func justInsertMessage(index: MessageIndex) { + let value = WriteBuffer() + var type: Int8 = 0 + var timestamp: Int32 = index.timestamp + value.write(&type, offset: 0, length: 1) + value.write(×tamp, offset: 0, length: 4) + self.valueBox.set(self.tableId, key: self.key(index.id), value: value) + } + + func justRemove(id: MessageId) { + self.valueBox.remove(self.tableId, key: self.key(id)) + } + + func adjacentItems(id: MessageId) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) { + let key = self.key(id) + + var lowerItem: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: key, end: self.lowerBound(id.peerId, namespace: id.namespace), values: { key, value in + lowerItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) + return true + }, limit: 1) + + var upperItem: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: key.predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in + upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) + return true + }, limit: 1) + + return (lower: lowerItem, upper: upperItem) + } + + func get(id: MessageId) -> HistoryIndexEntry? { + let key = self.key(id) + if let value = self.valueBox.get(self.tableId, key: key) { + return readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) + } + return nil + } + + func messageExists(id: MessageId) -> Bool { + if let entry = self.get(id) { + if case .Message = entry { + return true + } + } + + return false + } + + func debugList(peerId: PeerId, namespace: MessageId.Namespace) -> [HistoryIndexEntry] { + var list: [HistoryIndexEntry] = [] + self.valueBox.range(self.tableId, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in + list.append(readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value)) + + return true + }, limit: 0) + return list + } +} diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift new file mode 100644 index 0000000000..9600d5705f --- /dev/null +++ b/Postbox/MessageHistoryTable.swift @@ -0,0 +1,359 @@ +import Foundation + +enum MessageHistoryEntry { + case Msg(Message) + case Hole(MessageHistoryHole) + + var index: MessageIndex { + switch self { + case let .Msg(message): + return MessageIndex(message) + case let .Hole(hole): + return hole.maxIndex + } + } +} + +final class MessageHistoryTable { + let valueBox: ValueBox + let tableId: Int32 + + let messageHistoryIndexTable: MessageHistoryIndexTable + let messageMediaTable: MessageMediaTable + + init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable) { + self.valueBox = valueBox + self.tableId = tableId + self.messageHistoryIndexTable = messageHistoryIndexTable + self.messageMediaTable = messageMediaTable + } + + private func key(index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4)) -> ValueBoxKey { + key.setInt64(0, value: index.id.peerId.toInt64()) + key.setInt32(8, value: index.timestamp) + key.setInt32(8 + 4, value: index.id.namespace) + key.setInt32(8 + 4 + 4, value: index.id.id) + return key + } + + private func lowerBound(peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8) + key.setInt64(0, value: peerId.toInt64()) + return key + } + + private func upperBound(peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8) + key.setInt64(0, value: peerId.toInt64()) + return key.successor + } + + private func messagesByPeerId(messages: [StoreMessage]) -> [PeerId: [StoreMessage]] { + var dict: [PeerId: [StoreMessage]] = [:] + + for message in messages { + let peerId = message.id.peerId + if dict[peerId] == nil { + dict[peerId] = [message] + } else { + dict[peerId]!.append(message) + } + } + + return dict + } + + func addMessages(messages: [StoreMessage]) { + let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0)) + let sharedBuffer = WriteBuffer() + let sharedEncoder = Encoder() + + let messagesByPeerId = self.messagesByPeerId(messages) + for (_, peerMessages) in messagesByPeerId { + for message in peerMessages { + if self.messageHistoryIndexTable.messageExists(message.id) { + continue + } + + self.messageHistoryIndexTable.addMessage(MessageIndex(message)) + self.justInsert(message, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder) + } + } + } + + func removeMessages(messageIds: [MessageId]) { + for messageId in messageIds { + if let entry = self.messageHistoryIndexTable.get(messageId) { + if case let .Message(index) = entry { + if let message = self.get(index) { + let embeddedMediaData = message.embeddedMediaData + if embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + self.messageMediaTable.removeEmbeddedMedia(media) + } + embeddedMediaData.skip(Int(mediaLength)) + } + } + + for mediaId in message.referencedMedia { + self.messageMediaTable.removeReference(mediaId) + } + } + + self.messageHistoryIndexTable.removeMessage(messageId) + self.valueBox.remove(self.tableId, key: self.key(index)) + } + } + } + } + + private func justInsert(message: StoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) { + sharedBuffer.reset() + + let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! + var length: Int32 = Int32(data.length) + sharedBuffer.write(&length, offset: 0, length: 4) + sharedBuffer.write(data.bytes, offset: 0, length: Int(length)) + + var attributeCount: Int32 = Int32(message.attributes.count) + sharedBuffer.write(&attributeCount, offset: 0, length: 4) + for attribute in message.attributes { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(attribute) + let attributeBuffer = sharedEncoder.memoryBuffer() + var attributeBufferLength = Int32(attributeBuffer.length) + sharedBuffer.write(&attributeBufferLength, offset: 0, length: 4) + sharedBuffer.write(attributeBuffer.memory, offset: 0, length: attributeBuffer.length) + } + + var embeddedMedia: [Media] = [] + var referencedMedia: [MediaId] = [] + for media in message.media { + if let mediaId = media.id { + let mediaInsertResult = self.messageMediaTable.set(media, index: MessageIndex(message), messageHistoryTable: self) + switch mediaInsertResult { + case let .Embed(media): + embeddedMedia.append(media) + case .Reference: + referencedMedia.append(mediaId) + } + } else { + embeddedMedia.append(media) + } + } + + var embeddedMediaCount: Int32 = Int32(embeddedMedia.count) + sharedBuffer.write(&embeddedMediaCount, offset: 0, length: 4) + for media in embeddedMedia { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(media) + let mediaBuffer = sharedEncoder.memoryBuffer() + var mediaBufferLength = Int32(mediaBuffer.length) + sharedBuffer.write(&mediaBufferLength, offset: 0, length: 4) + sharedBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length) + } + + var referencedMediaCount: Int32 = Int32(referencedMedia.count) + sharedBuffer.write(&referencedMediaCount, offset: 0, length: 4) + for mediaId in referencedMedia { + var idNamespace: Int32 = mediaId.namespace + var idId: Int64 = mediaId.id + sharedBuffer.write(&idNamespace, offset: 0, length: 4) + sharedBuffer.write(&idId, offset: 0, length: 8) + } + + self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) + } + + func unembedMedia(index: MessageIndex, id: MediaId) -> Media? { + if let message = self.get(index) where message.embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + + let updatedEmbeddedMediaBuffer = WriteBuffer() + var updatedEmbeddedMediaCount = embeddedMediaCount - 1 + updatedEmbeddedMediaBuffer.write(&updatedEmbeddedMediaCount, offset: 0, length: 4) + + var extractedMedia: Media? + + for _ in 0 ..< embeddedMediaCount { + let mediaOffset = message.embeddedMediaData.offset + var mediaLength: Int32 = 0 + var copyMedia = true + message.embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: message.embeddedMediaData.memory + message.embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + + if let mediaId = media.id where mediaId == id { + copyMedia = false + extractedMedia = media + } + } + + if copyMedia { + updatedEmbeddedMediaBuffer.write(message.embeddedMediaData.memory + mediaOffset, offset: 0, length: message.embeddedMediaData.offset - mediaOffset) + } + } + + if let extractedMedia = extractedMedia { + var updatedReferencedMedia = message.referencedMedia + updatedReferencedMedia.append(extractedMedia.id!) + self.storeIntermediateMessage(IntermediateMessage(id: message.id, timestamp: message.timestamp, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) + + return extractedMedia + } + } + return nil + } + + func storeIntermediateMessage(message: IntermediateMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer = WriteBuffer()) { + sharedBuffer.reset() + + let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! + var length: Int32 = Int32(data.length) + sharedBuffer.write(&length, offset: 0, length: 4) + sharedBuffer.write(data.bytes, offset: 0, length: Int(length)) + + sharedBuffer.write(message.attributesData.memory, offset: 0, length: message.attributesData.length) + sharedBuffer.write(message.embeddedMediaData.memory, offset: 0, length: message.embeddedMediaData.length) + + var referencedMediaCount: Int32 = Int32(message.referencedMedia.count) + sharedBuffer.write(&referencedMediaCount, offset: 0, length: 4) + for mediaId in message.referencedMedia { + var idNamespace: Int32 = mediaId.namespace + var idId: Int64 = mediaId.id + sharedBuffer.write(&idNamespace, offset: 0, length: 4) + sharedBuffer.write(&idId, offset: 0, length: 8) + } + + self.valueBox.set(self.tableId, key: self.key(MessageIndex(id: message.id, timestamp: message.timestamp), key: sharedKey), value: sharedBuffer) + } + + private func readIntermediateMessage(key: ValueBoxKey, value: ReadBuffer) -> IntermediateMessage { + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4), id: key.getInt32(8 + 4 + 4)), timestamp: key.getInt32(8)) + + var textLength: Int32 = 0 + value.read(&textLength, offset: 0, length: 4) + let text = String(data: NSData(bytes: value.memory + value.offset, length: Int(textLength)), encoding: NSUTF8StringEncoding) ?? "" + value.skip(Int(textLength)) + + let attributesOffset = value.offset + var attributeCount: Int32 = 0 + value.read(&attributeCount, offset: 0, length: 4) + for _ in 0 ..< attributeCount { + var attributeLength: Int32 = 0 + value.read(&attributeLength, offset: 0, length: 4) + value.skip(Int(attributeLength)) + } + let attributesLength = value.offset - attributesOffset + let attributesBytes = malloc(attributesLength) + memcpy(attributesBytes, value.memory + attributesOffset, attributesLength) + let attributesData = ReadBuffer(memory: attributesBytes, length: attributesLength, freeWhenDone: true) + + let embeddedMediaOffset = value.offset + var embeddedMediaCount: Int32 = 0 + value.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + value.read(&mediaLength, offset: 0, length: 4) + value.skip(Int(mediaLength)) + } + let embeddedMediaLength = value.offset - embeddedMediaOffset + let embeddedMediaBytes = malloc(embeddedMediaLength) + memcpy(embeddedMediaBytes, value.memory + embeddedMediaOffset, embeddedMediaLength) + let embeddedMediaData = ReadBuffer(memory: embeddedMediaBytes, length: embeddedMediaLength, freeWhenDone: true) + + var referencedMediaIds: [MediaId] = [] + var referencedMediaIdsCount: Int32 = 0 + value.read(&referencedMediaIdsCount, offset: 0, length: 4) + for _ in 0 ..< referencedMediaIdsCount { + var idNamespace: Int32 = 0 + var idId: Int64 = 0 + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 8) + referencedMediaIds.append(MediaId(namespace: idNamespace, id: idId)) + } + + return IntermediateMessage(id: index.id, timestamp: index.timestamp, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds) + } + + func get(index: MessageIndex) -> IntermediateMessage? { + let key = self.key(index) + if let value = self.valueBox.get(self.tableId, key: key) { + return self.readIntermediateMessage(key, value: value) + } + return nil + } + + func renderMessage(message: IntermediateMessage) -> Message { + var parsedAttributes: [Coding] = [] + var parsedMedia: [Media] = [] + + let attributesData = message.attributesData + if attributesData.length > 4 { + var attributeCount: Int32 = 0 + attributesData.read(&attributeCount, offset: 0, length: 4) + for _ in 0 ..< attributeCount { + var attributeLength: Int32 = 0 + attributesData.read(&attributeLength, offset: 0, length: 4) + if let attribute = Decoder(buffer: MemoryBuffer(memory: attributesData.memory + attributesData.offset, capacity: Int(attributeLength), length: Int(attributeLength), freeWhenDone: false)).decodeRootObject() { + parsedAttributes.append(attribute) + } + attributesData.skip(Int(attributeLength)) + } + } + + let embeddedMediaData = message.embeddedMediaData + if embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + parsedMedia.append(media) + } + embeddedMediaData.skip(Int(mediaLength)) + } + } + + for mediaId in message.referencedMedia { + if let media = self.messageMediaTable.get(mediaId) { + parsedMedia.append(media) + } + } + + return Message(id: message.id, timestamp: message.timestamp, text: message.text, attributes: parsedAttributes, media: parsedMedia) + } + + func messagesAround(index: MessageIndex, count: Int) -> [IntermediateMessage] { + var lowerMessages: [IntermediateMessage] = [] + var upperMessages: [IntermediateMessage] = [] + + self.valueBox.range(self.tableId, start: self.key(index), end: self.lowerBound(index.id.peerId), values: { key, value in + lowerMessages.append(self.readIntermediateMessage(key, value: value)) + return true + }, limit: count) + + self.valueBox.range(self.tableId, start: self.key(index).predecessor, end: self.upperBound(index.id.peerId), values: { key, value in + upperMessages.append(self.readIntermediateMessage(key, value: value)) + return true + }, limit: count) + + var messages: [IntermediateMessage] = [] + for message in lowerMessages.reverse() { + messages.append(message) + } + messages.appendContentsOf(upperMessages) + + return messages + } + + func debugList(peerId: PeerId) -> [Message] { + return self.messagesAround(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: 0), count: 1000).map({self.renderMessage($0)}) + } +} diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift new file mode 100644 index 0000000000..0efad9cde0 --- /dev/null +++ b/Postbox/MessageHistoryView.swift @@ -0,0 +1,341 @@ +import Foundation + +public final class MutableMessageHistoryView: CustomStringConvertible { + public struct RemoveContext { + var invalidEarlier: Bool = false + var invalidLater: Bool = false + var invalidEarlierHole: Bool = false + var invalidLaterHole: Bool = false + var removedMessages: Bool = false + var removedHole: Bool = false + + func empty() -> Bool { + return !self.removedMessages && !invalidEarlier && !invalidLater + } + } + + let count: Int + var earlierMessage: RenderedMessage? + var laterMessage: RenderedMessage? + var messages: [RenderedMessage] + + public init(count: Int, earlierMessage: RenderedMessage?, messages: [RenderedMessage], laterMessage: RenderedMessage?) { + self.count = count + self.earlierMessage = earlierMessage + self.laterMessage = laterMessage + self.messages = messages + } + + public func add(message: RenderedMessage) -> Bool { + if self.messages.count == 0 { + self.messages.append(message) + return true + } else { + let first = MessageIndex(self.messages[self.messages.count - 1].message) + let last = MessageIndex(self.messages[0].message) + + var next: MessageIndex? + if let message = laterMessage { + let messageIndex = MessageIndex(message.message) + next = messageIndex + } + + let index = MessageIndex(message.message) + + if index < last { + let earlierMessage = self.earlierMessage + if earlierMessage == nil || MessageIndex(earlierMessage!.message) < index { + if self.messages.count < self.count { + self.messages.insert(message, atIndex: 0) + } else { + self.earlierMessage = message + } + return true + } else { + return false + } + } else if index > first { + if next != nil && index > next! { + let laterMessage = self.laterMessage + if laterMessage == nil || MessageIndex(laterMessage!.message) > index { + if self.messages.count < self.count { + self.messages.append(message) + } else { + self.laterMessage = message + } + return true + } else { + return false + } + } else { + self.messages.append(message) + if self.messages.count > self.count { + let earliest = self.messages[0] + self.earlierMessage = earliest + self.messages.removeAtIndex(0) + } + return true + } + } else if index != last && index != first { + var i = self.messages.count + while i >= 1 { + if MessageIndex(self.messages[i - 1].message) < index { + break + } + i-- + } + self.messages.insert(message, atIndex: i) + if self.messages.count > self.count { + let earliest = self.messages[0] + self.earlierMessage = earliest + self.messages.removeAtIndex(0) + } + return true + } else { + return false + } + } + } + + public func remove(ids: Set, context: RemoveContext? = nil) -> RemoveContext { + var updatedContext = RemoveContext() + if let context = context { + updatedContext = context + } + + if let earlierMessage = self.earlierMessage where ids.contains(earlierMessage.message.id) { + updatedContext.invalidEarlier = true + } + + if let laterMessage = self.laterMessage where ids.contains(laterMessage.message.id) { + updatedContext.invalidLater = true + } + + if self.messages.count != 0 { + var i = self.messages.count - 1 + while i >= 0 { + if ids.contains(self.messages[i].message.id) { + self.messages.removeAtIndex(i) + updatedContext.removedMessages = true + } + i-- + } + } + + return updatedContext + } + + public func complete(context: RemoveContext, fetchEarlier: (MessageIndex?, Int) -> [RenderedMessage], fetchLater: (MessageIndex?, Int) -> [RenderedMessage]) { + if context.removedMessages { + var addedMessages: [RenderedMessage] = [] + + var latestAnchor: MessageIndex? + if let lastMessage = self.messages.last { + latestAnchor = MessageIndex(lastMessage.message) + } + + if latestAnchor == nil { + if let laterMessage = self.laterMessage { + latestAnchor = MessageIndex(laterMessage.message) + } + } + + if let laterMessage = self.laterMessage { + addedMessages += fetchLater(MessageIndex(laterMessage.message).predecessor(), self.count) + } + if let earlierMessage = self.earlierMessage { + addedMessages += fetchEarlier(MessageIndex(earlierMessage.message).successor(), self.count) + } + + addedMessages += self.messages + addedMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) + var i = addedMessages.count - 1 + while i >= 1 { + if addedMessages[i].message.id == addedMessages[i - 1].message.id { + addedMessages.removeAtIndex(i) + } + i-- + } + self.messages = [] + + var anchorIndex = addedMessages.count - 1 + if let latestAnchor = latestAnchor { + var i = addedMessages.count - 1 + while i >= 0 { + if MessageIndex(addedMessages[i].message) <= latestAnchor { + anchorIndex = i + break + } + i-- + } + } + + self.laterMessage = nil + if anchorIndex + 1 < addedMessages.count { + self.laterMessage = addedMessages[anchorIndex + 1] + } + + i = anchorIndex + while i >= 0 && i > anchorIndex - self.count { + self.messages.insert(addedMessages[i], atIndex: 0) + i-- + } + + self.earlierMessage = nil + if anchorIndex - self.count >= 0 { + self.earlierMessage = addedMessages[anchorIndex - self.count] + } + } + else { + if context.invalidEarlier { + var earlyId: MessageIndex? + let i = 0 + if i < self.messages.count { + earlyId = MessageIndex(self.messages[i].message) + } + + let earlierMessages = fetchEarlier(earlyId, 1) + self.earlierMessage = earlierMessages.first + } + + if context.invalidLater { + var laterId: MessageIndex? + let i = self.messages.count - 1 + if i >= 0 { + laterId = MessageIndex(self.messages[i].message) + } + + let laterMessages = fetchLater(laterId, 1) + self.laterMessage = laterMessages.first + } + } + } + + + public func incompleteMessages() -> [Message] { + var result: [Message] = [] + + if let earlierMessage = self.earlierMessage where earlierMessage.incomplete { + result.append(earlierMessage.message) + } + + if let laterMessage = self.laterMessage where laterMessage.incomplete { + result.append(laterMessage.message) + } + + for message in self.messages { + if message.incomplete { + result.append(message.message) + } + } + + return result + } + + public func completeMessages(messages: [MessageId : RenderedMessage]) { + if let earlierMessage = self.earlierMessage { + if let renderedMessage = messages[earlierMessage.message.id] { + self.earlierMessage = renderedMessage + } + } + + if let laterMessage = self.laterMessage { + if let renderedMessage = messages[laterMessage.message.id] { + self.laterMessage = renderedMessage + } + } + + var i = 0 + while i < self.messages.count { + if let message = messages[self.messages[i].message.id] { + self.messages[i] = message + } + i++ + } + } + + public var description: String { + var string = "" + string += "...(" + if let value = self.earlierMessage { + string += "\(value.message.id.namespace): \(value.message.id.id)—\(value.message.timestamp)" + } + string += ") —— " + + string += "[" + var first = true + for message in self.messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" + } + string += "]" + + string += " —— (" + if let value = self.laterMessage { + string += "\(value.message.id.namespace): \(value.message.id.id)—\(value.message.timestamp)" + } + string += ")..." + + return string + } +} + +public final class MessageHistoryView: CustomStringConvertible { + public let hasEarlier: Bool + private let earlierId: MessageIndex? + public let hasLater: Bool + private let laterId: MessageIndex? + public let messages: [RenderedMessage] + + init(_ mutableView: MutableMessageHistoryView) { + self.hasEarlier = mutableView.earlierMessage != nil + self.hasLater = mutableView.laterMessage != nil + self.messages = mutableView.messages + + if let earlierMessage = mutableView.earlierMessage { + self.earlierId = MessageIndex(earlierMessage.message) + } else { + self.earlierId = nil + } + + if let laterMessage = mutableView.laterMessage { + self.laterId = MessageIndex(laterMessage.message) + } else { + self.laterId = nil + } + } + + public var description: String { + var string = "" + if self.hasEarlier { + string += "more(" + if let earlierId = self.earlierId { + string += "\(earlierId.id.namespace): \(earlierId.id.id)—\(earlierId.timestamp)" + } + string += ") " + } + string += "[" + var first = true + for message in self.messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" + } + string += "]" + if self.hasLater { + string += " more(" + if let laterId = self.laterId { + string += "\(laterId.id.namespace): \(laterId.id.id)—\(laterId.timestamp)" + } + string += ")" + } + return string + } +} diff --git a/Postbox/MessageMediaTable.swift b/Postbox/MessageMediaTable.swift new file mode 100644 index 0000000000..0fd6794974 --- /dev/null +++ b/Postbox/MessageMediaTable.swift @@ -0,0 +1,210 @@ +import Foundation + +private enum MediaEntryType: Int8 { + case Direct + case MessageReference +} + +enum InsertMediaResult { + case Reference + case Embed(Media) +} + +enum DebugMediaEntry { + case Direct(Media, Int) + case MessageReference(MessageIndex) +} + +final class MessageMediaTable { + let valueBox: ValueBox + let tableId: Int32 + + let mediaCleanupTable: MediaCleanupTable + + init(valueBox: ValueBox, tableId: Int32, mediaCleanupTable: MediaCleanupTable) { + self.valueBox = valueBox + self.tableId = tableId + self.mediaCleanupTable = mediaCleanupTable + } + + func key(id: MediaId, key: ValueBoxKey = ValueBoxKey(length: 4 + 8)) -> ValueBoxKey { + key.setInt32(0, value: id.namespace) + key.setInt64(4, value: id.id) + return key + } + + func get(id: MediaId) -> Media? { + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == MediaEntryType.Direct.rawValue { + var dataLength: Int32 = 0 + value.read(&dataLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + value.offset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media { + return media + } + } + } + return nil + } + + func set(media: Media, index: MessageIndex, messageHistoryTable: MessageHistoryTable, sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) -> InsertMediaResult { + if let id = media.id { + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == MediaEntryType.Direct.rawValue { + var dataLength: Int32 = 0 + value.read(&dataLength, offset: 0, length: 4) + value.skip(Int(dataLength)) + + sharedWriteBuffer.reset() + sharedWriteBuffer.write(value.memory, offset: 0, length: value.offset) + + var messageReferenceCount: Int32 = 0 + value.read(&messageReferenceCount, offset: 0, length: 4) + messageReferenceCount++ + sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) + + self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + + return .Reference + } else if type == MediaEntryType.MessageReference.rawValue { + var idPeerId: Int64 = 0 + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idPeerId, offset: 0, length: 8) + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + + let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp) + if referencedMessageIndex == index { + return .Embed(media) + } + + if let media = messageHistoryTable.unembedMedia(referencedMessageIndex, id: id) { + sharedWriteBuffer.reset() + var directType: Int8 = MediaEntryType.Direct.rawValue + sharedWriteBuffer.write(&directType, offset: 0, length: 1) + + sharedEncoder.reset() + sharedEncoder.encodeRootObject(media) + let mediaBuffer = sharedEncoder.memoryBuffer() + var mediaBufferLength = Int32(mediaBuffer.length) + sharedWriteBuffer.write(&mediaBufferLength, offset: 0, length: 4) + sharedWriteBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length) + + var messageReferenceCount: Int32 = 2 + sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) + + self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + } + + return .Reference + } else { + return .Embed(media) + } + } else { + sharedWriteBuffer.reset() + var type: Int8 = MediaEntryType.MessageReference.rawValue + sharedWriteBuffer.write(&type, offset: 0, length: 1) + var idPeerId: Int64 = index.id.peerId.toInt64() + var idNamespace: Int32 = index.id.namespace + var idId: Int32 = index.id.id + var idTimestamp: Int32 = index.timestamp + sharedWriteBuffer.write(&idPeerId, offset: 0, length: 8) + sharedWriteBuffer.write(&idNamespace, offset: 0, length: 4) + sharedWriteBuffer.write(&idId, offset: 0, length: 4) + sharedWriteBuffer.write(&idTimestamp, offset: 0, length: 4) + + self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + + return .Embed(media) + } + } else { + return .Embed(media) + } + } + + func removeReference(id: MediaId, sharedWriteBuffer: WriteBuffer = WriteBuffer()) { + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == MediaEntryType.Direct.rawValue { + var dataLength: Int32 = 0 + value.read(&dataLength, offset: 0, length: 4) + let mediaOffset = value.offset + value.skip(Int(dataLength)) + + sharedWriteBuffer.reset() + sharedWriteBuffer.write(value.memory, offset: 0, length: value.offset) + + var messageReferenceCount: Int32 = 0 + value.read(&messageReferenceCount, offset: 0, length: 4) + messageReferenceCount -= 1 + sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) + + if messageReferenceCount <= 0 { + if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + mediaOffset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media { + self.mediaCleanupTable.add(media) + } + self.valueBox.remove(self.tableId, key: self.key(id)) + } else { + self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + } + } else if type == MediaEntryType.MessageReference.rawValue { + self.valueBox.remove(self.tableId, key: self.key(id)) + } + } + } + + func removeEmbeddedMedia(media: Media) { + if let id = media.id { + self.valueBox.remove(self.tableId, key: self.key(id)) + } + self.mediaCleanupTable.add(media) + } + + func debugList() -> [DebugMediaEntry] { + var entries: [DebugMediaEntry] = [] + + let upperBoundKey = ValueBoxKey(length: 8 + 4) + memset(upperBoundKey.memory, 0xff, 8 + 4) + self.valueBox.range(self.tableId, start: ValueBoxKey(length: 0), end: upperBoundKey, values: { key, value in + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == MediaEntryType.Direct.rawValue { + var dataLength: Int32 = 0 + value.read(&dataLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + value.offset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media { + + value.skip(Int(dataLength)) + + var messageReferenceCount: Int32 = 0 + value.read(&messageReferenceCount, offset: 0, length: 4) + + entries.append(.Direct(media, Int(messageReferenceCount))) + } + } else if type == MediaEntryType.MessageReference.rawValue { + var idPeerId: Int64 = 0 + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idPeerId, offset: 0, length: 8) + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + + let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp) + + entries.append(.MessageReference(referencedMessageIndex)) + } + + return true + }, limit: 1000) + + return entries + } +} \ No newline at end of file diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift deleted file mode 100644 index aea71cd29b..0000000000 --- a/Postbox/MessageView.swift +++ /dev/null @@ -1,428 +0,0 @@ -import Foundation - -public final class MutableMessageView: CustomStringConvertible { - public struct RemoveContext { - var invalidEarlier: Set - var invalidLater: Set - var removedMessages: Bool - - init() { - self.invalidEarlier = [] - self.invalidLater = [] - self.removedMessages = false - } - - func empty() -> Bool { - return !self.removedMessages && self.invalidEarlier.count == 0 && self.invalidLater.count == 0 - } - } - - let namespaces: [MessageId.Namespace] - let count: Int - var earlier: [MessageId.Namespace : RenderedMessage] = [:] - var later: [MessageId.Namespace : RenderedMessage] = [:] - var messages: [RenderedMessage] - - public init(namespaces: [MessageId.Namespace], count: Int, earlier: [MessageId.Namespace : RenderedMessage], messages: [RenderedMessage], later: [MessageId.Namespace : RenderedMessage]) { - self.namespaces = namespaces - self.count = count - self.earlier = earlier - self.later = later - self.messages = messages - } - - public func add(message: RenderedMessage) -> Bool { - if self.messages.count == 0 { - self.messages.append(message) - return true - } else { - let first = MessageIndex(self.messages[self.messages.count - 1].message) - let last = MessageIndex(self.messages[0].message) - - var next: MessageIndex? - for namespace in self.namespaces { - if let message = later[namespace] { - let messageIndex = MessageIndex(message.message) - if next == nil || messageIndex < next! { - next = messageIndex - } - } - } - - let index = MessageIndex(message.message) - - if index < last { - let earlierMessage = self.earlier[message.message.id.namespace] - if earlierMessage == nil || earlierMessage!.message.id.id < message.message.id.id { - if self.messages.count < self.count { - self.messages.insert(message, atIndex: 0) - } else { - self.earlier[message.message.id.namespace] = message - } - return true - } else { - return false - } - } else if index > first { - if next != nil && index > next! { - let laterMessage = self.later[message.message.id.namespace] - if laterMessage == nil || laterMessage!.message.id.id > message.message.id.id { - if self.messages.count < self.count { - self.messages.append(message) - } else { - self.later[message.message.id.namespace] = message - } - return true - } else { - return false - } - } else { - self.messages.append(message) - if self.messages.count > self.count { - let earliest = self.messages[0] - self.earlier[earliest.message.id.namespace] = earliest - self.messages.removeAtIndex(0) - } - return true - } - } else if index != last && index != first { - var i = self.messages.count - while i >= 1 { - if MessageIndex(self.messages[i - 1].message) < index { - break - } - i-- - } - self.messages.insert(message, atIndex: i) - if self.messages.count > self.count { - let earliest = self.messages[0] - self.earlier[earliest.message.id.namespace] = earliest - self.messages.removeAtIndex(0) - } - return true - } else { - return false - } - } - } - - public func remove(ids: Set, context: RemoveContext? = nil) -> RemoveContext { - var updatedContext = RemoveContext() - if let context = context { - updatedContext = context - } - - for (_, message) in self.earlier { - if ids.contains(message.message.id) { - updatedContext.invalidEarlier.insert(message.message.id.namespace) - } - } - - for (_, message) in self.later { - if ids.contains(message.message.id) { - updatedContext.invalidLater.insert(message.message.id.namespace) - } - } - - if self.messages.count != 0 { - var i = self.messages.count - 1 - while i >= 0 { - if ids.contains(self.messages[i].message.id) { - self.messages.removeAtIndex(i) - updatedContext.removedMessages = true - } - i-- - } - } - - return updatedContext - } - - public func complete(context: RemoveContext, fetchEarlier: (MessageId.Namespace, MessageId.Id?, Int) -> [RenderedMessage], fetchLater: (MessageId.Namespace, MessageId.Id?, Int) -> [RenderedMessage]) { - if context.removedMessages { - var addedMessages: [RenderedMessage] = [] - - var latestAnchor: MessageIndex? - if let lastMessage = self.messages.last { - latestAnchor = MessageIndex(lastMessage.message) - } - - if latestAnchor == nil { - for (_, message) in self.later { - let messageIndex = MessageIndex(message.message) - if latestAnchor == nil || latestAnchor! > messageIndex { - latestAnchor = messageIndex - } - } - } - - for namespace in self.namespaces { - if let later = self.later[namespace] { - addedMessages += fetchLater(namespace, later.message.id.id - 1, self.count) - } - if let earlier = self.earlier[namespace] { - addedMessages += fetchEarlier(namespace, earlier.message.id.id + 1, self.count) - } - } - - addedMessages += self.messages - addedMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) - var i = addedMessages.count - 1 - while i >= 1 { - if addedMessages[i].message.id == addedMessages[i - 1].message.id { - addedMessages.removeAtIndex(i) - } - i-- - } - self.messages = [] - - var anchorIndex = addedMessages.count - 1 - if let latestAnchor = latestAnchor { - var i = addedMessages.count - 1 - while i >= 0 { - if MessageIndex(addedMessages[i].message) <= latestAnchor { - anchorIndex = i - break - } - i-- - } - } - - self.later.removeAll(keepCapacity: true) - if anchorIndex + 1 < addedMessages.count { - for namespace in self.namespaces { - var i = anchorIndex + 1 - while i < addedMessages.count { - if addedMessages[i].message.id.namespace == namespace { - self.later[namespace] = addedMessages[i] - break - } - i++ - } - } - } - - i = anchorIndex - while i >= 0 && i > anchorIndex - self.count { - self.messages.insert(addedMessages[i], atIndex: 0) - i-- - } - - self.earlier.removeAll(keepCapacity: true) - if anchorIndex - self.count >= 0 { - for namespace in self.namespaces { - i = anchorIndex - self.count - while i >= 0 { - if addedMessages[i].message.id.namespace == namespace { - self.earlier[namespace] = addedMessages[i] - break - } - i-- - } - } - } - } - else { - for namespace in context.invalidEarlier { - var earlyId: MessageId.Id? - var i = 0 - while i < self.messages.count { - if self.messages[i].message.id.namespace == namespace { - earlyId = self.messages[i].message.id.id - break - } - i++ - } - - let earlierMessages = fetchEarlier(namespace, earlyId, 1) - if earlierMessages.count == 0 { - self.earlier.removeValueForKey(namespace) - } else { - self.earlier[namespace] = earlierMessages[0] - } - } - - for namespace in context.invalidLater { - var lateId: MessageId.Id? - var i = self.messages.count - 1 - while i >= 0 { - if self.messages[i].message.id.namespace == namespace { - lateId = self.messages[i].message.id.id - break - } - i-- - } - - let laterMessages = fetchLater(namespace, lateId, 1) - if laterMessages.count == 0 { - self.later.removeValueForKey(namespace) - } else { - self.later[namespace] = laterMessages[0] - } - } - } - } - - - public func incompleteMessages() -> [Message] { - var result: [Message] = [] - - for (_, message) in self.earlier { - if message.incomplete { - result.append(message.message) - } - } - for (_, message) in self.later { - if message.incomplete { - result.append(message.message) - } - } - - for message in self.messages { - if message.incomplete { - result.append(message.message) - } - } - - return result - } - - public func completeMessages(messages: [MessageId : RenderedMessage]) { - var earlier = self.earlier - for (namespace, message) in self.earlier { - if let message = messages[message.message.id] { - earlier[namespace] = message - } - } - self.earlier = earlier - - var later = self.later - for (namespace, message) in self.later { - if let message = messages[message.message.id] { - later[namespace] = message - } - } - self.later = later - - var i = 0 - while i < self.messages.count { - if let message = messages[self.messages[i].message.id] { - self.messages[i] = message - } - i++ - } - } - - public var description: String { - var string = "" - string += "...(" - var first = true - for namespace in self.namespaces { - if let value = self.earlier[namespace] { - if first { - first = false - } else { - string += ", " - } - string += "\(namespace): \(value.message.id.id)—\(value.message.timestamp)" - } - } - string += ") —— " - - string += "[" - first = true - for message in self.messages { - if first { - first = false - } else { - string += ", " - } - string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" - } - string += "]" - - string += " —— (" - first = true - for namespace in self.namespaces { - if let value = self.later[namespace] { - if first { - first = false - } else { - string += ", " - } - string += "\(namespace): \(value.message.id.id)—\(value.message.timestamp)" - } - } - string += ")..." - - return string - } -} - -public final class MessageView: CustomStringConvertible { - public let hasEarlier: Bool - private let earlierIds: [MessageIndex] - public let hasLater: Bool - private let laterIds: [MessageIndex] - public let messages: [RenderedMessage] - - init(_ mutableView: MutableMessageView) { - self.hasEarlier = mutableView.earlier.count != 0 - self.hasLater = mutableView.later.count != 0 - self.messages = mutableView.messages - - var earlierIds: [MessageIndex] = [] - for (_, message) in mutableView.earlier { - earlierIds.append(MessageIndex(message.message)) - } - self.earlierIds = earlierIds - - var laterIds: [MessageIndex] = [] - for (_, message) in mutableView.later { - laterIds.append(MessageIndex(message.message)) - } - self.laterIds = laterIds - } - - public var description: String { - var string = "" - if self.hasEarlier { - string += "more(" - var first = true - for id in self.earlierIds { - if first { - first = false - } else { - string += ", " - } - string += "\(id.id.namespace): \(id.id.id)—\(id.timestamp)" - } - string += ") " - } - string += "[" - var first = true - for message in self.messages { - if first { - first = false - } else { - string += ", " - } - string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" - } - string += "]" - if self.hasLater { - string += " more(" - var first = true - for id in self.laterIds { - if first { - first = false - } else { - string += ", " - } - string += "\(id.id.namespace): \(id.id.id)—\(id.timestamp)" - } - string += ")" - } - return string - } -} diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index a6f7fac52c..04bf16f2d9 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -59,11 +59,13 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { } public init(_ buffer: ReadBuffer) { - self.namespace = 0 - self.id = 0 - memcpy(&self.namespace, buffer.memory, 4) - memcpy(&self.id, buffer.memory + 4, 4) + var namespace: Int32 = 0 + var id: Int32 = 0 + memcpy(&namespace, buffer.memory, 4) + self.namespace = namespace + memcpy(&id, buffer.memory + 4, 4) + self.id = id } public func encodeToBuffer(buffer: WriteBuffer) { diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index e5a8ed5528..9ddd12ec63 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -48,13 +48,13 @@ public final class Modifier { public final class Postbox { private let basePath: String private let messageNamespaces: [MessageId.Namespace] - private let absoluteIndexedMessageNamespaces: [MessageId.Namespace] + private let absoluteIndexedMessageNamespace: MessageId.Namespace private let queue = Queue(name: "org.telegram.postbox.Postbox") private var valueBox: ValueBox! - private var peerMessageViews: [PeerId : Bag<(MutableMessageView, Pipe)>] = [:] - private var deferredMessageViewsToUpdate: [(MutableMessageView, Pipe)] = [] + private var peerMessageHistoryViews: [PeerId : Bag<(MutableMessageHistoryView, Pipe)>] = [:] + private var deferredMessageHistoryViewsToUpdate: [(MutableMessageHistoryView, Pipe)] = [] private var peerViews: Bag<(MutablePeerView, Pipe)> = Bag() private var deferredPeerViewsToUpdate: [(MutablePeerView, Pipe)] = [] private var peerPipes: [PeerId : Pipe] = [:] @@ -63,10 +63,14 @@ public final class Postbox { public let mediaBox: MediaBox - public init(basePath: String, messageNamespaces: [MessageId.Namespace], absoluteIndexedMessageNamespaces: [MessageId.Namespace]) { + public init(basePath: String, messageNamespaces: [MessageId.Namespace], absoluteIndexedMessageNamespace: MessageId.Namespace?) { self.basePath = basePath self.messageNamespaces = messageNamespaces - self.absoluteIndexedMessageNamespaces = absoluteIndexedMessageNamespaces + if let absoluteIndexedMessageNamespace = absoluteIndexedMessageNamespace { + self.absoluteIndexedMessageNamespace = absoluteIndexedMessageNamespace + } else { + self.absoluteIndexedMessageNamespace = MessageId.Namespace.max + } self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.openDatabase() } @@ -83,7 +87,7 @@ public final class Postbox { self.valueBox = SqliteValueBox(basePath: self.basePath + "/db") //self.valueBox = LmdbValueBox(basePath: self.basePath + "/db") var userVersion: Int32 = 0 - let currentUserVersion: Int32 = 3 + let currentUserVersion: Int32 = 4 if let value = self.valueBox.get(Table_Meta.id, key: Table_Meta.key()) { value.read(&userVersion, offset: 0, length: 4) @@ -166,123 +170,11 @@ public final class Postbox { } private func addMessages(messages: [Message], medias: [Media]) { - let encoder = Encoder() - for (peerId, peerMessages) in messagesGroupedByPeerId(messages) { - var maxMessage: (MessageIndex, Message)? - - var messageIds: [MessageId] = [] - var seenMessageIds = Set() - for message in peerMessages { - if !seenMessageIds.contains(message.id) { - seenMessageIds.insert(message.id) - messageIds.append(message.id) - } - } - - var existingMessageIds = Set() - - let existingMessageKey = Table_Message.emptyKey() - existingMessageKey.setInt64(0, value: peerId.toInt64()) - for id in messageIds { - if self.valueBox.exists(Table_Message.id, key: Table_Message.key(id, key: existingMessageKey)) { - existingMessageIds.insert(id) - } - } - - var addedMessages: [Message] = [] - - let mediaMessageIdKey = Table_Media_MessageIds.emptyKey() - for message in peerMessages { - if existingMessageIds.contains(message.id) { - continue - } - existingMessageIds.insert(message.id) - addedMessages.append(message) - - let index = MessageIndex(message) - if maxMessage == nil || index > maxMessage!.0 { - maxMessage = (index, message) - } - - encoder.reset() - encoder.encodeRootObject(message) - - for id in message.mediaIds { - self.valueBox.set(Table_Media_MessageIds.id, key: Table_Media_MessageIds.key(id, messageId: message.id, key: mediaMessageIdKey), value: MemoryBuffer()) - } - - self.valueBox.set(Table_Message.id, key: Table_Message.key(message.id), value: Table_Message.set(message)) - - let absoluteKey = Table_AbsoluteMessageId.emptyKey() - if self.absoluteIndexedMessageNamespaces.contains(message.id.namespace) { - self.valueBox.set(Table_AbsoluteMessageId.id, key: Table_AbsoluteMessageId.key(message.id.id, key: absoluteKey), value: Table_AbsoluteMessageId.set(message.id)) - } - } - - if let relatedViews = self.peerMessageViews[peerId] { - for record in relatedViews.copyItems() { - var updated = false - for message in addedMessages { - if record.0.add(RenderedMessage(message: message)) { - updated = true - } - } - - if updated { - self.deferMessageViewUpdate(record.0, pipe: record.1) - } - } - } - - if let maxMessage = maxMessage { - self.updatePeerEntry(peerId, message: RenderedMessage(message: maxMessage.1)) - } - } - - var existingMediaIds = Set() - - let existingMediaKey = Table_Media.emptyKey() - for media in medias { - if let id = media.id { - if self.valueBox.exists(Table_Media.id, key: Table_Media.key(id, key: existingMediaKey)) { - existingMediaIds.insert(id) - } - } - } - - let mediaKey = Table_Media.emptyKey() - for media in medias { - if let id = media.id { - if existingMediaIds.contains(id) { - continue - } - existingMediaIds.insert(id) - - self.valueBox.set(Table_Media.id, key: Table_Media.key(id, key: mediaKey), value: Table_Media.set(media)) - } - } } private func mediaWithIds(ids: [MediaId]) -> [MediaId : Media] { - if ids.count == 0 { - return [:] - } else { - var result: [MediaId : Media] = [:] - - let mediaKey = Table_Media.emptyKey() - for id in ids { - if let value = self.valueBox.get(Table_Media.id, key: Table_Media.key(id, key: mediaKey)) { - if let media = Table_Media.get(value) { - result[id] = media - } else { - print("can't parse media") - } - } - } - - return result - } + return [:] } var cachedPeers: [PeerId : Peer] = [:] @@ -338,19 +230,19 @@ public final class Postbox { } } - private func deferMessageViewUpdate(view: MutableMessageView, pipe: Pipe) { + private func deferMessageHistoryViewUpdate(view: MutableMessageHistoryView, pipe: Pipe) { var i = 0 var found = false - while i < self.deferredMessageViewsToUpdate.count { - if self.deferredMessageViewsToUpdate[i].1 === pipe { - self.deferredMessageViewsToUpdate[i] = (view, pipe) + while i < self.deferredMessageHistoryViewsToUpdate.count { + if self.deferredMessageHistoryViewsToUpdate[i].1 === pipe { + self.deferredMessageHistoryViewsToUpdate[i] = (view, pipe) found = true break } i++ } if !found { - self.deferredMessageViewsToUpdate.append((view, pipe)) + self.deferredMessageHistoryViewsToUpdate.append((view, pipe)) } } @@ -371,10 +263,10 @@ public final class Postbox { entry.1.putNext(PeerView(entry.0)) } - let deferredMessageViewsToUpdate = self.deferredMessageViewsToUpdate - self.deferredMessageViewsToUpdate.removeAll() + let deferredMessageHistoryViewsToUpdate = self.deferredMessageHistoryViewsToUpdate + self.deferredMessageHistoryViewsToUpdate.removeAll() - for entry in deferredMessageViewsToUpdate { + for entry in deferredMessageHistoryViewsToUpdate { let viewRenderedMessages = self.renderedMessages(entry.0.incompleteMessages()) if viewRenderedMessages.count != 0 { var viewRenderedMessagesDict: [MessageId : RenderedMessage] = [:] @@ -384,7 +276,7 @@ public final class Postbox { entry.0.completeMessages(viewRenderedMessagesDict) } - entry.1.putNext(MessageView(entry.0)) + entry.1.putNext(MessageHistoryView(entry.0)) } } @@ -458,10 +350,10 @@ public final class Postbox { var result: [MessageId] = [] - let key = Table_AbsoluteMessageId.emptyKey() + let key = Table_GlobalMessageId.emptyKey() for id in ids { - if let value = self.valueBox.get(Table_AbsoluteMessageId.id, key: Table_AbsoluteMessageId.key(id, key: key)) { - result.append(Table_AbsoluteMessageId.get(id, value: value)) + if let value = self.valueBox.get(Table_GlobalMessageId.id, key: Table_GlobalMessageId.key(id, key: key)) { + result.append(Table_GlobalMessageId.get(id, value: value)) } } @@ -469,55 +361,7 @@ public final class Postbox { } private func deleteMessagesWithIds(ids: [MessageId]) { - for (peerId, messageIds) in messageIdsGroupedByPeerId(ids) { - if let relatedViews = self.peerMessageViews[peerId] { - for (view, pipe) in relatedViews.copyItems() { - let context = view.remove(Set(messageIds)) - if !context.empty() { - view.complete(context, fetchEarlier: self.fetchMessagesRelative(peerId, earlier: true), fetchLater: self.fetchMessagesRelative(peerId, earlier: false)) - self.deferMessageViewUpdate(view, pipe: pipe) - } - } - } - } - var touchedMediaIds = Set() - - let removeMediaMessageIdKey = Table_Media_MessageIds.emptyKey() - for (peerId, messageIds) in messageIdsGroupedByPeerId(ids) { - let messageKey = Table_Message.emptyKey() - for id in messageIds { - if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(id, key: messageKey)) { - if let message = Table_Message.get(value) { - for mediaId in message.mediaIds { - touchedMediaIds.insert(mediaId) - self.valueBox.remove(Table_Media_MessageIds.id, key: Table_Media_MessageIds.key(mediaId, messageId: message.id, key: removeMediaMessageIdKey)) - } - } - } - } - - for id in messageIds { - self.valueBox.remove(Table_Message.id, key: Table_Message.key(id, key: messageKey)) - } - - for mediaId in touchedMediaIds { - var referenced = false - self.valueBox.range(Table_Media_MessageIds.id, start: Table_Media_MessageIds.lowerBoundKey(mediaId), end: Table_Media_MessageIds.upperBoundKey(mediaId), keys: { key in - referenced = true - return false - }, limit: 1) - - if !referenced { - //TODO write to cleanup queue - self.valueBox.remove(Table_Media.id, key: Table_Media.key(mediaId)) - } - } - - let tail = self.fetchMessagesTail(peerId, count: 1) - - self.updatePeerEntry(peerId, message: tail.first, replace: true) - } } private func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer) { @@ -572,19 +416,21 @@ public final class Postbox { return Signal { subscriber in self.queue.dispatch { //#if DEBUG - let startTime = CFAbsoluteTimeGetCurrent() + //let startTime = CFAbsoluteTimeGetCurrent() //#endif + //self.valueBox.beginStats() self.valueBox.begin() let result = f(Modifier(postbox: self)) - //print("(Postbox modify took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") + //print("(Postbox modify took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") //#if DEBUG //startTime = CFAbsoluteTimeGetCurrent() //#endif self.valueBox.commit() //#if DEBUG - print("(Postbox commit took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") + //print("(Postbox commit took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") + //self.valueBox.endStats() //#endif self.performDeferredUpdates() @@ -596,282 +442,6 @@ public final class Postbox { } } - private func findAdjacentMessageIds(peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex) -> (MessageId.Id?, MessageId.Id?) { - /*var minId: MessageId.Id? - var maxId: MessageId.Id? - - let lowerBoundKey = ValueBoxKey(length: 8 + 4) - lowerBoundKey.setInt64(0, value: peerId.toInt64()) - lowerBoundKey.setInt32(8, value: namespace) - - let upperBoundKey = ValueBoxKey(length: 8 + 4) - upperBoundKey.setInt64(0, value: peerId.toInt64()) - upperBoundKey.setInt32(8, value: namespace) - - self.valueBox.range("peer_messages", start: lowerBoundKey, end: upperBoundKey.successor, keys: { key in - - }, limit: 1) - - for row in self.database.prepareCached("SELECT MIN(id), MAX(id) FROM peer_messages WHERE peerId = ? AND namespace = ?").run(peerId.toInt64(), Int64(namespace)) { - minId = MessageId.Id(row[0] as! Int64) - maxId = MessageId.Id(row[1] as! Int64) - } - - if let minId = minId, maxId = maxId { - var minTimestamp: Int32! - var maxTimestamp: Int32! - for row in self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (?, ?)").run(peerId.toInt64(), Int64(namespace), Int64(minId), Int64(maxId)) { - let id = Int32(row[0] as! Int64) - let timestamp = Int32(row[1] as! Int64) - if id == minId { - minTimestamp = timestamp - } else { - maxTimestamp = timestamp - } - } - - let earlierMidStatement = self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id <= ? LIMIT 1") - let laterMidStatement = self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id >= ? LIMIT 1") - - func lowerBound(timestamp: Int32) -> MessageId.Id? { - var leftId = minId - var leftTimestamp = minTimestamp - var rightId = maxId - var rightTimestamp = maxTimestamp - - while leftTimestamp <= timestamp && rightTimestamp >= timestamp { - let approximateMiddleId = leftId + (rightId - leftId) / 2 - if approximateMiddleId == leftId { - return rightId - } - var middleId: MessageId.Id? - var middleTimestamp: Int32? - for row in earlierMidStatement.run(peerId.toInt64(), Int64(namespace), Int64(approximateMiddleId)) { - middleId = MessageId.Id(row[0] as! Int64) - middleTimestamp = Int32(row[1] as! Int64) - break - } - if middleId == leftId { - return rightId - } - - if let middleId = middleId, middleTimestamp = middleTimestamp { - if middleTimestamp >= timestamp { - rightId = middleId - rightTimestamp = middleTimestamp - } else { - leftId = middleId - leftTimestamp = middleTimestamp - } - } else { - return nil - } - } - - return leftId - } - - func upperBound(timestamp: Int32) -> MessageId.Id? { - var leftId = minId - var leftTimestamp = minTimestamp - var rightId = maxId - var rightTimestamp = maxTimestamp - - while leftTimestamp <= timestamp && rightTimestamp >= timestamp { - let approximateMiddleId = leftId + (rightId - leftId) / 2 - if approximateMiddleId == leftId { - return leftId - } - var middleId: MessageId.Id? - var middleTimestamp: Int32? - for row in earlierMidStatement.run(peerId.toInt64(), Int64(namespace), Int64(approximateMiddleId)) { - middleId = MessageId.Id(row[0] as! Int64) - middleTimestamp = Int32(row[1] as! Int64) - break - } - if middleId == leftId { - return leftId - } - - if let middleId = middleId, middleTimestamp = middleTimestamp { - if middleTimestamp <= timestamp { - rightId = middleId - rightTimestamp = middleTimestamp - } else { - leftId = middleId - leftTimestamp = middleTimestamp - } - } else { - return nil - } - } - - return rightTimestamp - } - - if index.id.namespace < namespace { - let left = upperBound(index.timestamp - 1) - let right = lowerBound(index.timestamp) - return (left, right) - } else { - let left = upperBound(index.timestamp) - let right = lowerBound(index.timestamp + 1) - return (left, right) - } - } else { - return (nil, nil) - }*/ - - return (nil, nil) - } - - private func fetchMessagesAround(peerId: PeerId, anchorId: MessageId, count: Int) -> ([RenderedMessage], [MessageId.Namespace : RenderedMessage], [MessageId.Namespace : RenderedMessage]) { - var messages: [RenderedMessage] = [] - - messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: anchorId.namespace, id: anchorId.id, count: count + 1) - messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: anchorId.namespace, id: anchorId.id - 1, count: count + 1) - - messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) - var i = messages.count - 1 - while i >= 1 { - if messages[i].message.id == messages[i - 1].message.id { - messages.removeAtIndex(i) - } - i-- - } - - if messages.count == 0 { - return ([], [:], [:]) - } else { - var index: MessageIndex! - for message in messages { - if message.message.id == anchorId { - index = MessageIndex(message.message) - break - } - } - if index == nil { - var closestId: MessageId.Id = messages[0].message.id.id - var closestDistance = abs(closestId - anchorId.id) - let closestTimestamp: Int32 = messages[0].message.timestamp - for message in messages { - if abs(message.message.id.id - anchorId.id) < closestDistance { - closestId = message.message.id.id - closestDistance = abs(message.message.id.id - anchorId.id) - } - } - index = MessageIndex(id: MessageId(peerId: peerId, namespace: anchorId.namespace, id: closestId), timestamp: closestTimestamp) - } - - for namespace in self.messageNamespaces { - if namespace != anchorId.namespace { - let (left, right) = self.findAdjacentMessageIds(peerId, namespace: namespace, index: index) - if let left = left { - messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: left + 1, count: count + 1) - } - if let right = right { - messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: namespace, id: right - 1, count: count + 1) - } - } - } - - messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) - var i = messages.count - 1 - while i >= 1 { - if messages[i].message.id == messages[i - 1].message.id { - messages.removeAtIndex(i) - } - i-- - } - - var anchorIndex = messages.count / 2 - i = 0 - while i < messages.count { - if messages[i].message.id == index.id { - anchorIndex = i - break - } - i++ - } - - var filteredMessages: [RenderedMessage] = [] - var earlier: [MessageId.Namespace : RenderedMessage] = [:] - var later: [MessageId.Namespace : RenderedMessage] = [:] - - i = anchorIndex - var j = anchorIndex - 1 - var leftIndex = j - var rightIndex = i - - while i < messages.count || j >= 0 { - if i < messages.count && filteredMessages.count < count { - filteredMessages.append(messages[i]) - rightIndex = i - } - if j >= 0 && filteredMessages.count < count { - filteredMessages.append(messages[j]) - leftIndex = j - } - - i++ - j-- - } - - i = leftIndex - 1 - while i >= 0 { - if earlier[messages[i].message.id.namespace] == nil { - earlier[messages[i].message.id.namespace] = messages[i] - } - i-- - } - - i = rightIndex + 1 - while i < messages.count { - if later[messages[i].message.id.namespace] == nil { - later[messages[i].message.id.namespace] = messages[i] - } - i++ - } - - filteredMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) - - return (filteredMessages, earlier, later) - } - } - - private func fetchMessagesRelative(peerId: PeerId, earlier: Bool)(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [RenderedMessage] { - var messages: [Message] = [] - - let lowerBound = Table_Message.lowerBoundKey(peerId, namespace: namespace) - let upperBound = Table_Message.upperBoundKey(peerId, namespace: namespace) - - let bound: ValueBoxKey - if let id = id { - bound = Table_Message.key(MessageId(peerId: peerId, namespace: namespace, id: id)) - } else if earlier { - bound = upperBound - } else { - bound = lowerBound - } - - let values: (ValueBoxKey, ReadBuffer) -> Bool = { _, value in - if let message = Table_Message.get(value) { - messages.append(message) - } else { - print("can't parse message") - } - return true - } - - if earlier { - self.valueBox.range(Table_Message.id, start: bound, end: lowerBound, values: values, limit: count) - } else { - self.valueBox.range(Table_Message.id, start: bound, end: upperBound, values: values, limit: count) - } - - return self.renderedMessages(messages) - } - private func fetchPeerEntryIndicesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntryIndex] { var entries: [PeerViewEntryIndex] = [] @@ -918,7 +488,7 @@ public final class Postbox { } var message: Message? - if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(entryIndex.messageIndex.id)) { + if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(entryIndex.messageIndex)) { message = Table_Message.get(value) } @@ -943,123 +513,38 @@ public final class Postbox { } } - entries.sortInPlace({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) + //entries.sortInPlace() - return entries + return entries.sort({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) } private func renderedMessages(messages: [Message]) -> [RenderedMessage] { - if messages.count == 0 { - return [] - } - - var peerIds = Set() - var mediaIds = Set() - - for message in messages { - for peerId in message.peerIds { - peerIds.insert(peerId) - } - for mediaId in message.mediaIds { - mediaIds.insert(mediaId) - } - } - - var arrayPeerIds: [PeerId] = [] - for id in peerIds { - arrayPeerIds.append(id) - } - let peers = self.peersWithIds(arrayPeerIds) - - var arrayMediaIds: [MediaId] = [] - for id in mediaIds { - arrayMediaIds.append(id) - } - let medias = self.mediaWithIds(arrayMediaIds) - - var result: [RenderedMessage] = [] - - for message in messages { - if message.peerIds.count == 0 && message.mediaIds.count == 0 { - result.append(RenderedMessage(message: message, peers: [], media: [])) - } else { - var messagePeers: [Peer] = [] - for id in message.peerIds { - if let peer = peers[id] { - messagePeers.append(peer) - } - } - - var messageMedia: [Media] = [] - for id in message.mediaIds { - if let media = medias[id] { - messageMedia.append(media) - } - } - - result.append(RenderedMessage(message: message, peers: messagePeers, media: messageMedia)) - } - } - - return result + return [] } - private func fetchMessagesTail(peerId: PeerId, count: Int) -> [RenderedMessage] { - var messages: [RenderedMessage] = [] - - for namespace in self.messageNamespaces { - messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: nil, count: count) - } - - messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message)}) - - return messages - } - - public func tailMessageViewForPeerId(peerId: PeerId, count: Int) -> Signal { + public func tailMessageHistoryViewForPeerId(peerId: PeerId, count: Int) -> Signal { return Signal { subscriber in let startTime = CFAbsoluteTimeGetCurrent() let disposable = MetaDisposable() self.queue.dispatch { - let tail = self.fetchMessagesTail(peerId, count: count + 1) - print("tailMessageViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") + print("tailMessageHistoryViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - var messages: [RenderedMessage] = [] - var i = tail.count - 1 - while i >= 0 && i >= tail.count - count { - messages.insert(tail[i], atIndex: 0) - i-- - } + let mutableView = MutableMessageHistoryView(count: count, earlierMessage: nil, messages: [], laterMessage: nil) + let record = (mutableView, Pipe()) - var earlier: [MessageId.Namespace : RenderedMessage] = [:] - - for namespace in self.messageNamespaces { - var i = tail.count - count - 1 - while i >= 0 { - if tail[i].message.id.namespace == namespace { - earlier[namespace] = tail[i] - break - } - i-- - } - } - - let mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: earlier, messages: messages, later: [:]) - let record = (mutableView, Pipe()) - - let index: Bag<(MutableMessageView, Pipe)>.Index - if let bag = self.peerMessageViews[peerId] { + let index: Bag<(MutableMessageHistoryView, Pipe)>.Index + if let bag = self.peerMessageHistoryViews[peerId] { index = bag.add(record) } else { - let bag = Bag<(MutableMessageView, Pipe)>() + let bag = Bag<(MutableMessageHistoryView, Pipe)>() index = bag.add(record) - self.peerMessageViews[peerId] = bag + self.peerMessageHistoryViews[peerId] = bag } - subscriber.putNext(MessageView(mutableView)) + subscriber.putNext(MessageHistoryView(mutableView)) let pipeDisposable = record.1.signal().start(next: { next in subscriber.putNext(next) @@ -1070,7 +555,7 @@ public final class Postbox { if let strongSelf = self { strongSelf.queue.dispatch { - if let bag = strongSelf.peerMessageViews[peerId] { + if let bag = strongSelf.peerMessageHistoryViews[peerId] { bag.remove(index) } } @@ -1083,58 +568,31 @@ public final class Postbox { } } - public func aroundMessageViewForPeerId(peerId: PeerId, id: MessageId, count: Int) -> Signal { + public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.queue.dispatch { - let mutableView: MutableMessageView + let mutableView: MutableMessageHistoryView let startTime = CFAbsoluteTimeGetCurrent() - let around = self.fetchMessagesAround(peerId, anchorId: id, count: count) - if around.0.count == 0 { - let tail = self.fetchMessagesTail(peerId, count: count + 1) - - var messages: [RenderedMessage] = [] - var i = tail.count - 1 - while i >= 0 && i >= tail.count - count { - messages.insert(tail[i], atIndex: 0) - i-- - } - - var earlier: [MessageId.Namespace : RenderedMessage] = [:] - - for namespace in self.messageNamespaces { - var i = tail.count - count - 1 - while i >= 0 { - if tail[i].message.id.namespace == namespace { - earlier[namespace] = tail[i] - break - } - i-- - } - } - - mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: earlier, messages: messages, later: [:]) - } else { - mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: around.1, messages: around.0, later: around.2) - } + mutableView = MutableMessageHistoryView(count: count, earlierMessage: nil, messages: [], laterMessage: nil) print("aroundMessageViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - let record = (mutableView, Pipe()) + let record = (mutableView, Pipe()) - let index: Bag<(MutableMessageView, Pipe)>.Index - if let bag = self.peerMessageViews[peerId] { + let index: Bag<(MutableMessageHistoryView, Pipe)>.Index + if let bag = self.peerMessageHistoryViews[peerId] { index = bag.add(record) } else { - let bag = Bag<(MutableMessageView, Pipe)>() + let bag = Bag<(MutableMessageHistoryView, Pipe)>() index = bag.add(record) - self.peerMessageViews[peerId] = bag + self.peerMessageHistoryViews[peerId] = bag } - subscriber.putNext(MessageView(mutableView)) + subscriber.putNext(MessageHistoryView(mutableView)) let pipeDisposable = record.1.signal().start(next: { next in subscriber.putNext(next) @@ -1145,7 +603,7 @@ public final class Postbox { if let strongSelf = self { strongSelf.queue.dispatch { - if let bag = strongSelf.peerMessageViews[peerId] { + if let bag = strongSelf.peerMessageHistoryViews[peerId] { bag.remove(index) } } diff --git a/Postbox/PostboxTables.swift b/Postbox/PostboxTables.swift index c17f466265..8f048b3912 100644 --- a/Postbox/PostboxTables.swift +++ b/Postbox/PostboxTables.swift @@ -41,33 +41,31 @@ struct Table_Message { static let id: Int32 = 4 static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 8 + 4 + 4) + return ValueBoxKey(length: 8 + 4 + 4 + 4) } - static func lowerBoundKey(peerId: PeerId, namespace: Int32) -> ValueBoxKey { - let key = ValueBoxKey(length: 8 + 4) + static func lowerBoundKey(peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8) key.setInt64(0, value: peerId.toInt64()) - key.setInt32(8, value: namespace) return key } - static func upperBoundKey(peerId: PeerId, namespace: Int32) -> ValueBoxKey { - let key = ValueBoxKey(length: 8 + 4) + static func upperBoundKey(peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8) key.setInt64(0, value: peerId.toInt64()) - key.setInt32(8, value: namespace) return key.successor } - static func key(messageId: MessageId, key: ValueBoxKey = Table_Message.emptyKey()) -> ValueBoxKey { - key.setInt64(0, value: messageId.peerId.toInt64()) - key.setInt32(8, value: messageId.namespace) - key.setInt32(8 + 4, value: messageId.id) + static func key(index: MessageIndex, key: ValueBoxKey = Table_Message.emptyKey()) -> ValueBoxKey { + key.setInt64(0, value: index.id.peerId.toInt64()) + key.setInt32(8, value: index.timestamp) + key.setInt32(8 + 4, value: index.id.namespace) + key.setInt32(8 + 4 + 4, value: index.id.id) return key } static func set(message: Message, encoder: Encoder = Encoder()) -> MemoryBuffer { encoder.reset() - encoder.encodeRootObject(message) return encoder.memoryBuffer() } @@ -79,14 +77,14 @@ struct Table_Message { } } -struct Table_AbsoluteMessageId { +struct Table_GlobalMessageId { static let id: Int32 = 5 static func emptyKey() -> ValueBoxKey { return ValueBoxKey(length: 4) } - static func key(id: Int32, key: ValueBoxKey = Table_AbsoluteMessageId.emptyKey()) -> ValueBoxKey { + static func key(id: Int32, key: ValueBoxKey = Table_GlobalMessageId.emptyKey()) -> ValueBoxKey { key.setInt32(0, value: id) return key } @@ -272,3 +270,31 @@ struct Table_PeerEntry { return index } } + +struct Table_MessageIndex { + static let id: Int32 = 11 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 8 + 4 + 4) + } + + static func key(id: MessageId, key: ValueBoxKey = Table_MessageIndex.emptyKey()) -> ValueBoxKey { + key.setInt64(0, value: id.peerId.toInt64()) + key.setInt32(8, value: id.namespace) + key.setInt32(8 + 4, value: id.id) + return key + } + + static func set(index: MessageIndex) -> MemoryBuffer { + let buffer = WriteBuffer() + var timestamp = index.timestamp + buffer.write(×tamp, offset: 0, length: 4) + return buffer.readBufferNoCopy() + } + + static func get(id: MessageId, value: ReadBuffer) -> MessageIndex { + var timestamp: Int32 = 0 + value.read(×tamp, offset: 0, length: 4) + return MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: id.id), timestamp: timestamp) + } +} diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 4eb9b4d862..e5a954326f 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -65,6 +65,10 @@ public final class SqliteValueBox: ValueBox { private var insertStatements: [Int32 : SqlitePreparedStatement] = [:] private var deleteStatements: [Int32 : SqlitePreparedStatement] = [:] + private var readQueryTime: CFAbsoluteTime = 0.0 + private var writeQueryTime: CFAbsoluteTime = 0.0 + private var commitTime: CFAbsoluteTime = 0.0 + public init(basePath: String) { do { try NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil) @@ -91,12 +95,28 @@ public final class SqliteValueBox: ValueBox { } } + deinit { + self.clearStatements() + } + + public func beginStats() { + self.readQueryTime = 0.0 + self.writeQueryTime = 0.0 + self.commitTime = 0.0 + } + + public func endStats() { + print("(SqliteValueBox stats read: \(self.readQueryTime * 1000.0) ms, write: \(self.writeQueryTime * 1000.0) ms, commit: \(self.commitTime * 1000.0) ms") + } + public func begin() { self.database.transaction() } public func commit() { + let startTime = CFAbsoluteTimeGetCurrent() self.database.commit() + self.commitTime += CFAbsoluteTimeGetCurrent() - startTime } private func getStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { @@ -372,6 +392,7 @@ public final class SqliteValueBox: ValueBox { } public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? { + let startTime = CFAbsoluteTimeGetCurrent() if self.tables.contains(table) { let statement = self.getStatement(table, key: key) @@ -384,6 +405,8 @@ public final class SqliteValueBox: ValueBox { statement.reset() + self.readQueryTime += CFAbsoluteTimeGetCurrent() - startTime + return buffer } @@ -405,6 +428,8 @@ public final class SqliteValueBox: ValueBox { if self.tables.contains(table) { let statement: SqlitePreparedStatement + var startTime = CFAbsoluteTimeGetCurrent() + if start < end { if limit <= 0 { statement = self.rangeValueAscStatementNoLimit(table, start: start, end: end) @@ -419,10 +444,20 @@ public final class SqliteValueBox: ValueBox { } } + var currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + startTime = currentTime + while statement.step() { + startTime = CFAbsoluteTimeGetCurrent() + let key = statement.keyAt(0) let value = statement.valueAt(1) + currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + if !values(key, value) { break } @@ -436,6 +471,8 @@ public final class SqliteValueBox: ValueBox { if self.tables.contains(table) { let statement: SqlitePreparedStatement + var startTime = CFAbsoluteTimeGetCurrent() + if start < end { if limit <= 0 { statement = self.rangeKeyAscStatementNoLimit(table, start: start, end: end) @@ -450,9 +487,19 @@ public final class SqliteValueBox: ValueBox { } } + var currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + startTime = currentTime + while statement.step() { + startTime = CFAbsoluteTimeGetCurrent() + let key = statement.keyAt(0) + currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + if !keys(key) { break } @@ -470,6 +517,8 @@ public final class SqliteValueBox: ValueBox { self.database.execute("INSERT INTO __meta_tables(name) VALUES (\(table))") } + let startTime = CFAbsoluteTimeGetCurrent() + var exists = false let existsStatement = self.existsStatement(table, key: key) if existsStatement.step() { @@ -488,19 +537,25 @@ public final class SqliteValueBox: ValueBox { } statement.reset() } + + self.writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } public func remove(table: Int32, key: ValueBoxKey) { if self.tables.contains(table) { + let startTime = CFAbsoluteTimeGetCurrent() + let statement = self.deleteStatement(table, key: key) while statement.step() { } statement.reset() + + self.writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } } - public func drop() { + private func clearStatements() { for (_, statement) in self.getStatements { statement.destroy() } @@ -565,6 +620,10 @@ public final class SqliteValueBox: ValueBox { statement.destroy() } self.deleteStatements.removeAll() + } + + public func drop() { + self.clearStatements() for table in self.tables { self.database.execute("DROP TABLE IF EXISTS t\(table)") diff --git a/Postbox/ValueBox.swift b/Postbox/ValueBox.swift index 7549d15ea1..83973eeb8f 100644 --- a/Postbox/ValueBox.swift +++ b/Postbox/ValueBox.swift @@ -4,6 +4,9 @@ public protocol ValueBox { func begin() func commit() + func beginStats() + func endStats() + func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: ValueBoxKey -> Bool, limit: Int) func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? diff --git a/PostboxTests/CodingTests.swift b/PostboxTests/CodingTests.swift index f8f70bdaec..00c75ab465 100644 --- a/PostboxTests/CodingTests.swift +++ b/PostboxTests/CodingTests.swift @@ -165,45 +165,6 @@ class SerializationTests: XCTestCase { XCTAssert(decoder.decodeInt32ForKey("a") == 12345, "int32 failed") } - func testIndexBaselinePerformance() { - let basePath = "/tmp/postboxtest" - do { - try NSFileManager.defaultManager().removeItemAtPath(basePath) - } catch _ { } - let postbox = Postbox(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: []) - postbox._prepareBaselineIndexPerformance() - - measureBlock { - postbox._measureBaselineIndexPerformance() - } - } - - func testBlobReadPerformance() { - let basePath = "/tmp/postboxtest" - do { - try NSFileManager.defaultManager().removeItemAtPath(basePath) - } catch _ { } - let postbox = Postbox(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: []) - postbox._prepareBlobIndexPerformance() - - measureBlock { - postbox._measureBlobReadPerformance() - } - } - - func testBlobIndexPerformance() { - let basePath = "/tmp/postboxtest" - do { - try NSFileManager.defaultManager().removeItemAtPath(basePath) - } catch _ { } - let postbox = Postbox(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: []) - postbox._prepareBlobIndexPerformance() - - measureBlock { - postbox._measureBlobIndexPerformance() - } - } - func testKeys() { let key1 = ValueBoxKey(length: 8) let key2 = ValueBoxKey(length: 8) @@ -230,7 +191,7 @@ class SerializationTests: XCTestCase { } func testKeyValue() { - let basePath = "/tmp/postboxtest" + /*let basePath = "/tmp/postboxtest" do { try NSFileManager.defaultManager().removeItemAtPath(basePath) } catch _ { } @@ -256,6 +217,6 @@ class SerializationTests: XCTestCase { return true }, limit: 10) } - } + }*/ } } diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift new file mode 100644 index 0000000000..9c742da934 --- /dev/null +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -0,0 +1,427 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +private let peerId = PeerId(namespace: 1, id: 1) +private let namespace: Int32 = 1 + +private enum Item: Equatable, CustomStringConvertible { + case Message(Int32, Int32) + case Hole(Int32, Int32, Int32) + + init(_ item: HistoryIndexEntry) { + switch item { + case let .Message(index): + self = .Message(index.id.id, index.timestamp) + case let .Hole(hole): + self = .Hole(hole.min, hole.maxIndex.id.id, hole.maxIndex.timestamp) + } + } + + var description: String { + switch self { + case let .Message(id, timestamp): + return "Message(\(id), \(timestamp))" + case let .Hole(minId, maxId, maxTimestamp): + return "Hole(\(minId), \(maxId), \(maxTimestamp))" + } + } +} + +private func ==(lhs: Item, rhs: Item) -> Bool { + switch lhs { + case let .Message(id, timestamp): + switch rhs { + case let .Message(rId, rTimestamp): + return id == rId && timestamp == rTimestamp + case .Hole: + return false + } + case let .Hole(minId, maxId, maxTimestamp): + switch rhs { + case .Message: + return false + case let .Hole(rMinId, rMaxId, rMaxTimestamp): + return minId == rMinId && maxId == rMaxId && maxTimestamp == rMaxTimestamp + } + } +} + +@testable import Postbox + +class MessageHistoryIndexTableTests: XCTestCase { + var valueBox: ValueBox? + var path: String? + + var indexTable: MessageHistoryIndexTable? + + override func setUp() { + super.setUp() + + var randomId: Int64 = 0 + arc4random_buf(&randomId, 8) + path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + self.valueBox = SqliteValueBox(basePath: path!) + + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1) + } + + override func tearDown() { + super.tearDown() + + self.indexTable = nil + + self.valueBox = nil + let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + self.path = nil + } + + func addHole(id: Int32) { + self.indexTable!.addHole(MessageId(peerId: peerId, namespace: namespace, id: id)) + } + + func addMessage(id: Int32, _ timestamp: Int32) { + self.indexTable!.addMessage(MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp)) + } + + func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32)]) { + self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, indices: messages.map({MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1)})) + } + + func removeMessage(id: Int32) { + self.indexTable!.removeMessage(MessageId(peerId: peerId, namespace: namespace, id: id)) + } + + private func expect(items: [Item]) { + let actualItems = self.indexTable!.debugList(peerId, namespace: namespace).map { return Item($0) } + if items != actualItems { + XCTFail("Expected\n\(items)\nGot\n\(actualItems)") + } + } + + func testEmpty() { + expect([]) + } + + func testAddMessageToEmpty() { + addMessage(100, 100) + expect([.Message(100, 100)]) + + addMessage(110, 110) + expect([.Message(100, 100), .Message(110, 110)]) + + addMessage(90, 90) + expect([.Message(90, 90), .Message(100, 100), .Message(110, 110)]) + } + + func testAddHoleToEmpty() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testAddHoleToFullHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + addHole(110) + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testAddMessageToFullHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + } + + func testAddMessageDividingUpperHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + addMessage(100, 100) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, 99, 100), .Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + } + + func testAddMessageDividingLowerHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + addMessage(80, 80) + expect([.Hole(1, 79, 80), .Message(80, 80), .Hole(81, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + } + + func testAddMessageOffsettingUpperHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + addMessage(91, 91) + expect([.Hole(1, 89, 90), .Message(90, 90), .Message(91, 91), .Hole(92, Int32.max, Int32.max)]) + } + + func testAddMessageOffsettingLowerHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + addMessage(89, 89) + expect([.Hole(1, 88, 89), .Message(89, 89), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + } + + func testAddMessageOffsettingLeftmostHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + + addMessage(1, 1) + + expect([.Message(1, 1), .Hole(2, Int32.max, Int32.max)]) + } + + func testAddMessageRemovingLefmostHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + + addMessage(2, 2) + expect([.Hole(1, 1, 2), .Message(2, 2), .Hole(3, Int32.max, Int32.max)]) + + addMessage(1, 1) + expect([.Message(1, 1), .Message(2, 2), .Hole(3, Int32.max, Int32.max)]) + } + + func testAddHoleLowerThanMessage() { + addMessage(100, 100) + addHole(1) + + expect([.Hole(1, 99, 100), .Message(100, 100)]) + } + + func testAddHoleHigherThanMessage() { + addMessage(100, 100) + addHole(200) + + expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + } + + func testIgnoreHigherHole() { + addHole(200) + expect([.Hole(1, Int32.max, Int32.max)]) + addHole(400) + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testIgnoreHigherHoleAfterMessage() { + addMessage(100, 100) + addHole(200) + expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + addHole(400) + expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + } + + func testAddHoleBetweenMessages() { + addMessage(100, 100) + addMessage(200, 200) + addHole(150) + + expect([.Message(100, 100), .Hole(101, 199, 200), .Message(200, 200)]) + } + + func testFillHoleEmpty() { + fillHole(1, .Complete, []) + expect([]) + } + + func testFillHoleComplete() { + addHole(100) + + fillHole(1, .Complete, [(100, 100), (200, 200)]) + expect([.Message(100, 100), .Message(200, 200)]) + } + + func testFillHoleUpperToLowerPartial() { + addHole(100) + + fillHole(1, .UpperToLower, [(100, 100), (200, 200)]) + expect([.Hole(1, 99, 100), .Message(100, 100), .Message(200, 200)]) + } + + func testFillHoleUpperToLowerToBounds() { + addHole(100) + + fillHole(1, .UpperToLower, [(1, 1), (200, 200)]) + expect([.Message(1, 1), .Message(200, 200)]) + } + + func testFillHoleLowerToUpperToBounds() { + addHole(100) + + fillHole(1, .LowerToUpper, [(100, 100), (Int32.max, 200)]) + expect([.Message(100, 100), .Message(Int32.max, 200)]) + } + + func testFillHoleLowerToUpperPartial() { + addHole(100) + + fillHole(1, .LowerToUpper, [(100, 100), (200, 200)]) + expect([.Message(100, 100), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesUpperToLower() { + addHole(1) + + addMessage(100, 100) + addMessage(200, 200) + + fillHole(199, .UpperToLower, [(150, 150)]) + + expect([.Hole(1, 99, 100), .Message(100, 100), .Hole(101, 149, 150), .Message(150, 150), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesLowerToUpper() { + addHole(1) + + addMessage(100, 100) + addMessage(200, 200) + + fillHole(199, .LowerToUpper, [(150, 150)]) + + expect([.Hole(1, 99, 100), .Message(100, 100), .Message(150, 150), .Hole(151, 199, 200), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesComplete() { + addHole(1) + + addMessage(100, 100) + addMessage(200, 200) + + fillHole(199, .Complete, [(150, 150)]) + + expect([.Hole(1, 99, 100), .Message(100, 100), .Message(150, 150), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesWithMessage() { + addMessage(200, 200) + addMessage(202, 202) + addHole(201) + addMessage(201, 201) + + expect([.Message(200, 200), .Message(201, 201), .Message(202, 202)]) + } + + func testFillHoleWithNoMessagesComplete() { + addMessage(100, 100) + addHole(1) + + fillHole(99, .Complete, []) + + expect([.Message(100, 100)]) + } + + func testFillHoleIgnoreOverMessage() { + addMessage(100, 100) + addMessage(101, 101) + + fillHole(100, .Complete, [(90, 90)]) + + expect([.Message(90, 90), .Message(100, 100), .Message(101, 101)]) + } + + func testFillHoleWithOverflow() { + addMessage(100, 100) + addMessage(200, 200) + addHole(150) + + fillHole(199, .UpperToLower, [(150, 150), (300, 300)]) + + expect([.Message(100, 100), .Hole(101, 149, 150), .Message(150, 150), .Message(200, 200), .Message(300, 300)]) + } + + func testIgnoreHoleOverMessageBetweenMessages() { + addMessage(199, 199) + addMessage(200, 200) + addHole(200) + + expect([.Message(199, 199), .Message(200, 200)]) + } + + func testMergeHoleAfterDeletingMessage() { + addMessage(100, 100) + addHole(1) + addHole(200) + + expect([.Hole(1, 99, 100), .Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + + removeMessage(100) + + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testMergeHoleLowerAfterDeletingMessage() { + addMessage(100, 100) + addHole(1) + addMessage(200, 200) + + removeMessage(100) + + expect([.Hole(1, 199, 200), .Message(200, 200)]) + } + + func testMergeHoleUpperAfterDeletingMessage() { + addMessage(100, 100) + addMessage(200, 200) + addHole(300) + + removeMessage(200) + + expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + } + + func testExtendLowerHoleAfterDeletingMessage() { + addMessage(100, 100) + addHole(100) + + removeMessage(100) + + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testExtendUpperHoleAfterDeletingMessage() { + addMessage(100, 100) + addHole(101) + + removeMessage(100) + + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testDeleteMessageBelowMessage() { + addMessage(100, 100) + addMessage(200, 200) + removeMessage(100) + + expect([.Message(200, 200)]) + } + + func testDeleteMessageAboveMessage() { + addMessage(100, 100) + addMessage(200, 200) + removeMessage(200) + + expect([.Message(100, 100)]) + } + + func testDeleteMessageBetweenMessages() { + addMessage(100, 100) + addMessage(200, 200) + addMessage(300, 300) + removeMessage(200) + + expect([.Message(100, 100), .Message(300, 300)]) + } +} diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift new file mode 100644 index 0000000000..925f1acb8c --- /dev/null +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -0,0 +1,306 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +private let peerId = PeerId(namespace: 1, id: 1) +private let namespace: Int32 = 1 + +private func ==(lhs: (Int32, Int32, String, [Media]), rhs: (Int32, Int32, String, [Media])) -> Bool { + if lhs.3.count != rhs.3.count { + return false + } + for i in 0 ..< lhs.3.count { + if !lhs.3[i].isEqual(rhs.3[i]) { + return false + } + } + return lhs.0 == rhs.0 && lhs.1 == rhs.1 && lhs.2 == rhs.2 +} + +private class TestEmbeddedMedia: Media, CustomStringConvertible { + var id: MediaId? { return nil } + let data: String + + init(data: String) { + self.data = data + } + + required init(decoder: Decoder) { + self.data = decoder.decodeStringForKey("s") + } + + func encode(encoder: Encoder) { + encoder.encodeString(self.data, forKey: "s") + } + + func isEqual(other: Media) -> Bool { + if let other = other as? TestEmbeddedMedia { + return self.data == other.data + } + return false + } + + var description: String { + return "TestEmbeddedMedia(\(self.data))" + } +} + +private class TestExternalMedia: Media { + let id: MediaId? + let data: String + + init(id: Int64, data: String) { + self.id = MediaId(namespace: namespace, id: id) + self.data = data + } + + required init(decoder: Decoder) { + self.id = MediaId(namespace: decoder.decodeInt32ForKey("i.n"), id: decoder.decodeInt64ForKey("i.i")) + self.data = decoder.decodeStringForKey("s") + } + + func encode(encoder: Encoder) { + encoder.encodeInt32(self.id!.namespace, forKey: "i.n") + encoder.encodeInt64(self.id!.id, forKey: "i.i") + encoder.encodeString(self.data, forKey: "s") + } + + func isEqual(other: Media) -> Bool { + if let other = other as? TestExternalMedia { + return self.id == other.id && self.data == other.data + } + return false + } + + var description: String { + return "TestExternalMedia(\(self.id!.id), \(self.data))" + } +} + +private enum MediaEntry: Equatable { + case Direct(Media, Int) + case MessageReference(Int32) + + init(_ entry: DebugMediaEntry) { + switch entry { + case let .Direct(media, referenceCount): + self = .Direct(media, referenceCount) + case let .MessageReference(index): + self = MessageReference(index.id.id) + } + } +} + +private func ==(lhs: MediaEntry, rhs: MediaEntry) -> Bool { + switch lhs { + case let .Direct(lhsMedia, lhsReferenceCount): + switch rhs { + case let .Direct(rhsMedia, rhsReferenceCount): + return lhsMedia.isEqual(rhsMedia) && lhsReferenceCount == rhsReferenceCount + case .MessageReference: + return false + } + case let .MessageReference(lhsId): + switch rhs { + case .Direct: + return false + case let .MessageReference(rhsId): + return lhsId == rhsId + } + } +} + +class MessageHistoryTableTests: XCTestCase { + var valueBox: ValueBox? + var path: String? + + var indexTable: MessageHistoryIndexTable? + var mediaTable: MessageMediaTable? + var mediaCleanupTable: MediaCleanupTable? + var historyTable: MessageHistoryTable? + + override class func setUp() { + super.setUp() + + declareEncodable(TestEmbeddedMedia.self, f: {TestEmbeddedMedia(decoder: $0)}) + declareEncodable(TestExternalMedia.self, f: {TestExternalMedia(decoder: $0)}) + } + + override func setUp() { + super.setUp() + + var randomId: Int64 = 0 + arc4random_buf(&randomId, 8) + path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + self.valueBox = SqliteValueBox(basePath: path!) + + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1) + self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!) + } + + override func tearDown() { + super.tearDown() + + self.historyTable = nil + self.indexTable = nil + self.mediaTable = nil + self.mediaCleanupTable = nil + + self.valueBox = nil + let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + self.path = nil + } + + private func addMessage(id: Int32, _ timestamp: Int32, _ text: String, _ media: [Media] = []) { + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, text: text, attributes: [], media: media)]) + } + + private func removeMessages(ids: [Int32]) { + self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) })) + } + + func expectMessages(messages: [(Int32, Int32, String, [Media])]) { + let actualMessages = self.historyTable!.debugList(peerId).map({ ($0.id.id, $0.timestamp, $0.text, $0.media) }) + var equal = true + if messages.count != actualMessages.count { + equal = false + } else { + for i in 0 ..< messages.count { + if !(messages[i] == actualMessages[i]) { + equal = false + break + } + } + } + + if !equal { + XCTFail("Expected\n\(messages)\nActual\n\(actualMessages)") + } + } + + private func expectMedia(media: [MediaEntry]) { + let actualMedia = self.mediaTable!.debugList().map({MediaEntry($0)}) + if media != actualMedia { + XCTFail("Expected\n\(media)\nActual\n\(actualMedia)") + } + } + + private func expectCleanupMedia(media: [Media]) { + let actualMedia = self.mediaCleanupTable!.debugList() + var equal = true + if media.count != actualMedia.count { + equal = false + } else { + for i in 0 ..< media.count { + if !media[i].isEqual(actualMedia[i]) { + equal = false + break + } + } + } + + if !equal { + XCTFail("Expected\n\(media)\nActual\n\(actualMedia)") + } + } + + func testInsertMessageIntoEmpty() { + addMessage(100, 100, "t100") + addMessage(200, 200, "t200") + + expectMessages([(100, 100, "t100", []), (200, 200, "t200", [])]) + } + + func testInsertMessageIgnoreOverwrite() { + addMessage(100, 100, "t100") + addMessage(100, 200, "t200") + + expectMessages([(100, 100, "t100", [])]) + } + + func testInsertMessageWithEmbeddedMedia() { + addMessage(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")]) + + expectMessages([(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")])]) + expectMedia([]) + } + + func testInsertMessageWithExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + addMessage(100, 100, "t100", [media]) + + expectMessages([(100, 100, "t100", [media])]) + expectMedia([.MessageReference(100)]) + } + + func testUnembedExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + addMessage(100, 100, "t100", [media]) + addMessage(200, 200, "t200", [media]) + + expectMessages([(100, 100, "t100", [media]), (200, 200, "t200", [media])]) + expectMedia([.Direct(media, 2)]) + } + + func testIgnoreOverrideExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + let media1 = TestExternalMedia(id: 10, data: "abc2") + addMessage(100, 100, "t100", [media]) + addMessage(200, 200, "t200", [media1]) + + expectMessages([(100, 100, "t100", [media]), (200, 200, "t200", [media])]) + expectMedia([.Direct(media, 2)]) + } + + func testRemoveSingleMessage() { + addMessage(100, 100, "t100", []) + + removeMessages([100]) + + expectMessages([]) + expectMedia([]) + } + + func testRemoveMessageWithEmbeddedMedia() { + let media = TestEmbeddedMedia(data: "abc1") + addMessage(100, 100, "t100", [media]) + self.removeMessages([100]) + + expectMessages([]) + expectMedia([]) + expectCleanupMedia([media]) + } + + func testRemoveOnlyReferenceToExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + addMessage(100, 100, "t100", [media]) + removeMessages([100]) + + expectMessages([]) + expectMedia([]) + expectCleanupMedia([media]) + } + + func testRemoveReferenceToExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + addMessage(100, 100, "t100", [media]) + addMessage(200, 200, "t200", [media]) + removeMessages([100]) + + expectMessages([(200, 200, "t200", [media])]) + expectMedia([.Direct(media, 1)]) + expectCleanupMedia([]) + + removeMessages([200]) + + expectMessages([]) + expectMedia([]) + expectCleanupMedia([media]) + } +} \ No newline at end of file From b70f657183d23ac02552c37876173cff766a166d Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 1 Feb 2016 03:06:14 +0300 Subject: [PATCH 026/427] no message --- Postbox.xcodeproj/project.pbxproj | 126 ++- Postbox/ChatListIndexTable.swift | 45 ++ Postbox/ChatListTable.swift | 156 ++++ Postbox/ChatListView.swift | 312 ++++++++ Postbox/Coding.swift | 4 + Postbox/GlobalMessageIdsTable.swift | 45 ++ Postbox/KeychainTable.swift | 30 + Postbox/Media.swift | 1 + Postbox/Message.swift | 76 +- Postbox/MessageHistoryHole.swift | 10 +- Postbox/MessageHistoryIndexTable.swift | 260 ++++--- Postbox/MessageHistoryTable.swift | 463 +++++++++--- Postbox/MessageHistoryView.swift | 399 +++++----- Postbox/MetadataTable.swift | 53 ++ Postbox/Peer.swift | 2 +- Postbox/PeerTable.swift | 40 + Postbox/PeerView.swift | 454 ----------- Postbox/Postbox.swift | 715 ++++++------------ Postbox/PostboxCodingUtils.swift | 233 ------ Postbox/PostboxTables.swift | 300 -------- Postbox/SimpleDictionary.swift | 42 + Postbox/SqliteValueBox.swift | 10 +- Postbox/ViewTracker.swift | 199 +++++ PostboxTests/ChatListTableTests.swift | 185 +++++ PostboxTests/CodingTests.swift | 2 +- .../MessageHistoryIndexTableTests.swift | 86 ++- PostboxTests/MessageHistoryTableTests.swift | 470 +++++++++++- PostboxTests/MessageHistoryViewTests.swift | 3 + 28 files changed, 2726 insertions(+), 1995 deletions(-) create mode 100644 Postbox/ChatListIndexTable.swift create mode 100644 Postbox/ChatListTable.swift create mode 100644 Postbox/ChatListView.swift create mode 100644 Postbox/GlobalMessageIdsTable.swift create mode 100644 Postbox/KeychainTable.swift create mode 100644 Postbox/MetadataTable.swift create mode 100644 Postbox/PeerTable.swift delete mode 100644 Postbox/PeerView.swift delete mode 100644 Postbox/PostboxCodingUtils.swift delete mode 100644 Postbox/PostboxTables.swift create mode 100644 Postbox/SimpleDictionary.swift create mode 100644 Postbox/ViewTracker.swift create mode 100644 PostboxTests/ChatListTableTests.swift create mode 100644 PostboxTests/MessageHistoryViewTests.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index dec81ee69a..dd12187c67 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -35,13 +35,12 @@ D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; - D0977FA21B82930C009994B2 /* PostboxCodingUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */; }; - D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA31B829F87009994B2 /* PostboxTables.swift */; }; D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B76BE61B66639F0095CF45 /* DeferredString.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; + D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */; }; D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; - D0D225261B4D84930085E26D /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D225251B4D84930085E26D /* PeerView.swift */; }; + D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7811B28ADD000A402D9 /* Postbox.swift */; }; @@ -52,6 +51,15 @@ D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */; }; D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; + D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8641C58CB7F00037222 /* ChatListTable.swift */; }; + D0F9E8671C58D08900037222 /* ChatListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */; }; + D0F9E8691C58FA9300037222 /* ChatListTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8681C58FA9300037222 /* ChatListTableTests.swift */; }; + D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86A1C59719800037222 /* ChatListView.swift */; }; + D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */; }; + D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */; }; + D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; + D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; + D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8741C5A334100037222 /* SimpleDictionary.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -94,13 +102,12 @@ D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = ""; }; D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; - D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxCodingUtils.swift; sourceTree = ""; }; - D0977FA31B829F87009994B2 /* PostboxTables.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxTables.swift; sourceTree = ""; }; D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; + D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryViewTests.swift; sourceTree = ""; }; D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; - D0D225251B4D84930085E26D /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; + D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; D0E3A74F1B28A7E300A402D9 /* Postbox.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Postbox.h; sourceTree = ""; }; @@ -114,6 +121,15 @@ D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageMediaTable.swift; sourceTree = ""; }; D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTableTests.swift; sourceTree = ""; }; D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaCleanupTable.swift; sourceTree = ""; }; + D0F9E8641C58CB7F00037222 /* ChatListTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatListTable.swift; sourceTree = ""; }; + D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatListIndexTable.swift; sourceTree = ""; }; + D0F9E8681C58FA9300037222 /* ChatListTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatListTableTests.swift; sourceTree = ""; }; + D0F9E86A1C59719800037222 /* ChatListView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatListView.swift; sourceTree = ""; }; + D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MetadataTable.swift; sourceTree = ""; }; + D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = KeychainTable.swift; sourceTree = ""; }; + D0F9E8701C5A0E9B00037222 /* PeerTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerTable.swift; sourceTree = ""; }; + D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GlobalMessageIdsTable.swift; sourceTree = ""; }; + D0F9E8741C5A334100037222 /* SimpleDictionary.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SimpleDictionary.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -190,6 +206,64 @@ name = sqlite.swift; sourceTree = ""; }; + D0E1DE161C5EB06000C7826E /* Tables */ = { + isa = PBXGroup; + children = ( + D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */, + D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */, + D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, + D0F9E8641C58CB7F00037222 /* ChatListTable.swift */, + D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */, + D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */, + D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */, + D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */, + D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */, + D0F9E8701C5A0E9B00037222 /* PeerTable.swift */, + ); + name = Tables; + sourceTree = ""; + }; + D0E1DE171C5EB06B00C7826E /* Objects */ = { + isa = PBXGroup; + children = ( + D0E3A7831B28AE0900A402D9 /* Peer.swift */, + D0E3A79D1B28B50400A402D9 /* Message.swift */, + D0E3A7A11B28B7DC00A402D9 /* Media.swift */, + D08C71391C501F0700779C0F /* MessageHistoryHole.swift */, + ); + name = Objects; + sourceTree = ""; + }; + D0E1DE181C5EB09300C7826E /* Utils */ = { + isa = PBXGroup; + children = ( + D0E3A7871B28AE9C00A402D9 /* Coding.swift */, + D0B76BE61B66639F0095CF45 /* DeferredString.swift */, + D0F9E8741C5A334100037222 /* SimpleDictionary.swift */, + ); + name = Utils; + sourceTree = ""; + }; + D0E1DE191C5EB0B500C7826E /* Value Box */ = { + isa = PBXGroup; + children = ( + D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */, + D0977F9B1B822DB4009994B2 /* ValueBox.swift */, + D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */, + D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */, + ); + name = "Value Box"; + sourceTree = ""; + }; + D0E1DE1A1C5EB0DB00C7826E /* Views */ = { + isa = PBXGroup; + children = ( + D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */, + D0F9E86A1C59719800037222 /* ChatListView.swift */, + ); + name = Views; + sourceTree = ""; + }; D0E3A7401B28A7E300A402D9 = { isa = PBXGroup; children = ( @@ -217,26 +291,14 @@ D075165B1B2EC5B000AE42E0 /* module.private.modulemap */, D075163D1B2D9CEF00AE42E0 /* PostboxPrivate */, D07515FC1B2C44A200AE42E0 /* thirdparty */, - D0E3A7871B28AE9C00A402D9 /* Coding.swift */, - D0B76BE61B66639F0095CF45 /* DeferredString.swift */, - D0E3A7831B28AE0900A402D9 /* Peer.swift */, - D0E3A79D1B28B50400A402D9 /* Message.swift */, - D08C71391C501F0700779C0F /* MessageHistoryHole.swift */, - D0E3A7A11B28B7DC00A402D9 /* Media.swift */, - D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */, - D0D225251B4D84930085E26D /* PeerView.swift */, - D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */, + D0E1DE181C5EB09300C7826E /* Utils */, + D0E1DE171C5EB06B00C7826E /* Objects */, + D0E1DE191C5EB0B500C7826E /* Value Box */, + D0E1DE161C5EB06000C7826E /* Tables */, + D0E1DE1A1C5EB0DB00C7826E /* Views */, D0E3A7811B28ADD000A402D9 /* Postbox.swift */, - D0977FA31B829F87009994B2 /* PostboxTables.swift */, + D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */, D055BD321B7D3D2D00F06C0A /* MediaBox.swift */, - D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */, - D0977F9B1B822DB4009994B2 /* ValueBox.swift */, - D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */, - D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */, - D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */, - D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, - D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */, - D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */, D0E3A74D1B28A7E300A402D9 /* Supporting Files */, ); path = Postbox; @@ -260,6 +322,8 @@ D044E15D1B2ACB9C001EE087 /* CodingTests.swift */, D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */, D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */, + D0F9E8681C58FA9300037222 /* ChatListTableTests.swift */, + D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */, ); path = PostboxTests; sourceTree = ""; @@ -390,6 +454,7 @@ files = ( D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */, + D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, @@ -397,24 +462,29 @@ D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */, D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, + D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, + D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, - D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */, + D0F9E8671C58D08900037222 /* ChatListIndexTable.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */, + D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */, D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */, + D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, + D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, - D0D225261B4D84930085E26D /* PeerView.swift in Sources */, + D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, + D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */, D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, - D0977FA21B82930C009994B2 /* PostboxCodingUtils.swift in Sources */, D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */, D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */, D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, @@ -428,6 +498,8 @@ files = ( D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */, D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */, + D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */, + D0F9E8691C58FA9300037222 /* ChatListTableTests.swift in Sources */, D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift new file mode 100644 index 0000000000..f25db86b63 --- /dev/null +++ b/Postbox/ChatListIndexTable.swift @@ -0,0 +1,45 @@ +import Foundation + +final class ChatListIndexTable { + let valueBox: ValueBox + let tableId: Int32 + + init(valueBox: ValueBox, tableId: Int32) { + self.valueBox = valueBox + self.tableId = tableId + } + + private func key(peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8) + key.setInt64(0, value: peerId.toInt64()) + return key + } + + func set(index: MessageIndex) { + let writeBuffer = WriteBuffer() + var idNamespace: Int32 = index.id.namespace + var idId: Int32 = index.id.id + var idTimestamp: Int32 = index.timestamp + writeBuffer.write(&idNamespace, offset: 0, length: 4) + writeBuffer.write(&idId, offset: 0, length: 4) + writeBuffer.write(&idTimestamp, offset: 0, length: 4) + self.valueBox.set(self.tableId, key: self.key(index.id.peerId), value: writeBuffer.readBufferNoCopy()) + } + + func remove(peerId: PeerId) { + self.valueBox.remove(self.tableId, key: self.key(peerId)) + } + + func get(peerId: PeerId) -> MessageIndex? { + if let value = self.valueBox.get(self.tableId, key: self.key(peerId)) { + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + return MessageIndex(id: MessageId(peerId: peerId, namespace: idNamespace, id: idId), timestamp: idTimestamp) + } + return nil + } +} diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift new file mode 100644 index 0000000000..59734c80d1 --- /dev/null +++ b/Postbox/ChatListTable.swift @@ -0,0 +1,156 @@ +import Foundation + +enum ChatListOperation { + case InsertMessage(IntermediateMessage) + case InsertNothing(MessageIndex) + case Remove([MessageIndex]) +} + +enum ChatListIntermediateEntry { + case Message(IntermediateMessage) + case Nothing(MessageIndex) +} + +final class ChatListTable { + let valueBox: ValueBox + let tableId: Int32 + let indexTable: ChatListIndexTable + + init(valueBox: ValueBox, tableId: Int32, indexTable: ChatListIndexTable) { + self.valueBox = valueBox + self.tableId = tableId + self.indexTable = indexTable + } + + private func key(index: MessageIndex) -> ValueBoxKey { + let key = ValueBoxKey(length: 4 + 4 + 4 + 8) + key.setInt32(0, value: index.timestamp) + key.setInt32(4, value: index.id.namespace) + key.setInt32(4 + 4, value: index.id.id) + key.setInt64(4 + 4 + 4, value: index.id.peerId.toInt64()) + return key + } + + private func lowerBound() -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setInt32(0, value: 0) + return key + } + + private func upperBound() -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setInt32(0, value: Int32.max) + return key + } + + func replay(historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]], messageHistoryTable: MessageHistoryTable, inout operations: [ChatListOperation]) { + for (peerId, _) in historyOperationsByPeerId { + let currentIndex: MessageIndex? = self.indexTable.get(peerId) + if let topMessage = messageHistoryTable.topMessage(peerId) { + let index = MessageIndex(id: topMessage.id, timestamp: topMessage.timestamp) + + if let currentIndex = currentIndex where currentIndex != index { + self.justRemove(currentIndex) + } + if let currentIndex = currentIndex { + operations.append(.Remove([currentIndex])) + } + self.indexTable.set(index) + self.justInsert(index) + operations.append(.InsertMessage(topMessage)) + } else { + if let currentIndex = currentIndex { + operations.append(.Remove([currentIndex])) + operations.append(.InsertNothing(currentIndex)) + } + } + } + } + + func justInsert(index: MessageIndex) { + self.valueBox.set(self.tableId, key: self.key(index), value: MemoryBuffer(memory: nil, capacity: 0, length: 0, freeWhenDone: false)) + } + + func justRemove(index: MessageIndex) { + self.valueBox.remove(self.tableId, key: self.key(index)) + } + + func entriesAround(index: MessageIndex, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + var lowerEntries: [ChatListIntermediateEntry] = [] + + self.valueBox.range(self.tableId, start: self.key(index).successor, end: self.lowerBound(), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) + if let message = messageHistoryTable.getMessage(index) { + lowerEntries.append(.Message(message)) + } else { + lowerEntries.append(.Nothing(index)) + } + return true + }, limit: count) + + var upperEntries: [ChatListIntermediateEntry] = [] + self.valueBox.range(self.tableId, start: self.key(index), end: self.upperBound(), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) + if let message = messageHistoryTable.getMessage(index) { + upperEntries.append(.Message(message)) + } else { + upperEntries.append(.Nothing(index)) + } + return true + }, limit: count) + + var entries: [ChatListIntermediateEntry] = [] + for entry in lowerEntries.reverse() { + entries.append(entry) + } + entries.appendContentsOf(upperEntries) + + return entries + } + + func earlierEntries(index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + var entries: [ChatListIntermediateEntry] = [] + let key: ValueBoxKey + if let index = index { + key = self.key(index) + } else { + key = self.upperBound() + } + + self.valueBox.range(self.tableId, start: key, end: self.lowerBound(), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) + if let message = messageHistoryTable.getMessage(index) { + entries.append(.Message(message)) + } else { + entries.append(.Nothing(index)) + } + return true + }, limit: count) + return entries + } + + func laterEntries(index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + var entries: [ChatListIntermediateEntry] = [] + let key: ValueBoxKey + if let index = index { + key = self.key(index) + } else { + key = self.lowerBound() + } + + self.valueBox.range(self.tableId, start: key, end: self.upperBound(), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) + if let message = messageHistoryTable.getMessage(index) { + entries.append(.Message(message)) + } else { + entries.append(.Nothing(index)) + } + return true + }, limit: count) + return entries + } + + func debugList(messageHistoryTable: MessageHistoryTable) -> [ChatListIntermediateEntry] { + return self.entriesAround(MessageIndex(id: MessageId(peerId: PeerId(0), namespace: 0, id: 0), timestamp: 0), messageHistoryTable: messageHistoryTable, count: 1000) + } +} diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift new file mode 100644 index 0000000000..f8a197bb38 --- /dev/null +++ b/Postbox/ChatListView.swift @@ -0,0 +1,312 @@ +import Foundation + +public enum ChatListEntry: Comparable { + case MessageEntry(Message) + case Nothing(MessageIndex) + + public var index: MessageIndex { + switch self { + case let .MessageEntry(message): + return MessageIndex(message) + case let .Nothing(index): + return index + } + } +} + +public func ==(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { + return lhs.index == rhs.index +} + +public func <(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { + return lhs.index < rhs.index +} + +enum MutableChatListEntry { + case IntermediateMessageEntry(IntermediateMessage) + case MessageEntry(Message) + case Nothing(MessageIndex) + + var index: MessageIndex { + switch self { + case let .IntermediateMessageEntry(message): + return MessageIndex(id: message.id, timestamp: message.timestamp) + case let .MessageEntry(message): + return MessageIndex(message) + case let .Nothing(index): + return index + } + } +} + +final class MutableChatListViewReplayContext { + var invalidEarlier: Bool = false + var invalidLater: Bool = false + var removedEntries: Bool = false + + func empty() -> Bool { + return !self.removedEntries && !invalidEarlier && !invalidLater + } +} + +final class MutableChatListView { + private var earlier: MutableChatListEntry? + private var later: MutableChatListEntry? + private var entries: [MutableChatListEntry] + private var count: Int + + init(earlier: MutableChatListEntry?, entries: [MutableChatListEntry], later: MutableChatListEntry?, count: Int) { + self.earlier = earlier + self.entries = entries + self.later = later + self.count = count + } + + func replay(operations: [ChatListOperation], context: MutableChatListViewReplayContext) -> Bool { + var hasChanges = false + for operation in operations { + switch operation { + case let .InsertMessage(message): + if self.add(.IntermediateMessageEntry(message)) { + hasChanges = true + } + case let .InsertNothing(index): + if self.add(.Nothing(index)) { + hasChanges = true + } + case let .Remove(indices): + if self.remove(Set(indices), context: context) { + hasChanges = true + } + } + } + return hasChanges + } + + func add(entry: MutableChatListEntry) -> Bool { + if self.entries.count == 0 { + self.entries.append(entry) + return true + } else { + let first = self.entries[self.entries.count - 1].index + let last = self.entries[0].index + + var next: MessageIndex? + if let later = self.later { + next = later.index + } + + let index = entry.index + + if index < last { + if self.earlier == nil || self.earlier!.index < index { + if self.entries.count < self.count { + self.entries.insert(entry, atIndex: 0) + } else { + self.earlier = entry + } + return true + } else { + return false + } + } else if index > first { + if next != nil && index > next! { + if self.later == nil || self.later!.index > index { + if self.entries.count < self.count { + self.entries.append(entry) + } else { + self.later = entry + } + return true + } else { + return false + } + } else { + self.entries.append(entry) + if self.entries.count > self.count { + self.earlier = self.entries[0] + self.entries.removeAtIndex(0) + } + return true + } + } else if index != last && index != first { + var i = self.entries.count + while i >= 1 { + if self.entries[i - 1].index < index { + break + } + i-- + } + self.entries.insert(entry, atIndex: i) + if self.entries.count > self.count { + self.earlier = self.entries[0] + self.entries.removeAtIndex(0) + } + return true + } else { + return false + } + } + } + + func remove(indices: Set, context: MutableChatListViewReplayContext) -> Bool { + var hasChanges = false + if let earlier = self.earlier where indices.contains(earlier.index) { + context.invalidEarlier = true + hasChanges = true + } + + if let later = self.later where indices.contains(later.index) { + context.invalidLater = true + hasChanges = true + } + + if self.entries.count != 0 { + var i = self.entries.count - 1 + while i >= 0 { + if indices.contains(self.entries[i].index) { + self.entries.removeAtIndex(i) + context.removedEntries = true + hasChanges = true + } + i-- + } + } + + return hasChanges + } + + func complete(context: MutableChatListViewReplayContext, fetchEarlier: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLater: (MessageIndex?, Int) -> [MutableChatListEntry]) { + if context.removedEntries { + var addedEntries: [MutableChatListEntry] = [] + + var latestAnchor: MessageIndex? + if let last = self.entries.last { + latestAnchor = last.index + } + + if latestAnchor == nil { + if let later = self.later { + latestAnchor = later.index + } + } + + if let later = self.later { + addedEntries += fetchLater(later.index.predecessor(), self.count) + } + if let earlier = self.earlier { + addedEntries += fetchEarlier(earlier.index.successor(), self.count) + } + + addedEntries += self.entries + addedEntries.sortInPlace({ $0.index < $1.index }) + var i = addedEntries.count - 1 + while i >= 1 { + if addedEntries[i].index.id == addedEntries[i - 1].index.id { + addedEntries.removeAtIndex(i) + } + i-- + } + self.entries = [] + + var anchorIndex = addedEntries.count - 1 + if let latestAnchor = latestAnchor { + var i = addedEntries.count - 1 + while i >= 0 { + if addedEntries[i].index <= latestAnchor { + anchorIndex = i + break + } + i-- + } + } + + self.later = nil + if anchorIndex + 1 < addedEntries.count { + self.later = addedEntries[anchorIndex + 1] + } + + i = anchorIndex + while i >= 0 && i > anchorIndex - self.count { + self.entries.insert(addedEntries[i], atIndex: 0) + i-- + } + + self.earlier = nil + if anchorIndex - self.count >= 0 { + self.earlier = addedEntries[anchorIndex - self.count] + } + } else { + if context.invalidEarlier { + var earlyId: MessageIndex? + let i = 0 + if i < self.entries.count { + earlyId = self.entries[i].index + } + + let earlierEntries = fetchEarlier(earlyId, 1) + self.earlier = earlierEntries.first + } + + if context.invalidLater { + var laterId: MessageIndex? + let i = self.entries.count - 1 + if i >= 0 { + laterId = self.entries[i].index + } + + let laterEntries = fetchLater(laterId, 1) + self.later = laterEntries.first + } + } + } + + func updatePeers(peers: [PeerId: Peer]) -> Bool { + var hasChanges = false + /*for i in 0 ..< self.entries.count { + switch self.entries[i] { + case let .MessageEntry(message): + var updatedAuthor: Peer? + if let author = message.author, let peer = peers[author.id] { + updatedAuthor = peer + } + + for peer in message.peers { + + } + + break + default: + break + } + }*/ + return hasChanges + } + + func render(renderMessage: IntermediateMessage -> Message) { + for i in 0 ..< self.entries.count { + if case let .IntermediateMessageEntry(message) = self.entries[i] { + self.entries[i] = .MessageEntry(renderMessage(message)) + } + } + } +} + +public final class ChatListView { + public let entries: [ChatListEntry] + + init(_ mutableView: MutableChatListView) { + var entries: [ChatListEntry] = [] + for entry in mutableView.entries { + switch entry { + case let .MessageEntry(message): + entries.append(.MessageEntry(message)) + case let .Nothing(index): + entries.append(.Nothing(index)) + case .IntermediateMessageEntry: + assertionFailure() + } + } + self.entries = entries + } +} diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index e084d99829..96db1c7b43 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -174,6 +174,10 @@ public final class Encoder { return self.buffer.makeReadBufferAndReset() } + public func readBufferNoCopy() -> ReadBuffer { + return self.buffer.readBufferNoCopy() + } + public func makeData() -> NSData { return self.buffer.makeData() } diff --git a/Postbox/GlobalMessageIdsTable.swift b/Postbox/GlobalMessageIdsTable.swift new file mode 100644 index 0000000000..a95d66f88e --- /dev/null +++ b/Postbox/GlobalMessageIdsTable.swift @@ -0,0 +1,45 @@ +import Foundation + +final class GlobalMessageIdsTable { + let valueBox: ValueBox + let tableId: Int32 + let namespace: Int32 + + let sharedKey = ValueBoxKey(length: 4) + let sharedBuffer = WriteBuffer() + + init(valueBox: ValueBox, tableId: Int32, namespace: Int32) { + self.valueBox = valueBox + self.tableId = tableId + self.namespace = namespace + } + + private func key(id: Int32) -> ValueBoxKey { + self.sharedKey.setInt32(0, value: id) + return self.sharedKey + } + + func set(globalId: Int32, id: MessageId) { + self.sharedBuffer.reset() + var idPeerId: Int64 = id.peerId.toInt64() + var idNamespace: Int32 = id.namespace + self.sharedBuffer.write(&idPeerId, offset: 0, length: 8) + self.sharedBuffer.write(&idNamespace, offset: 0, length: 4) + self.valueBox.set(self.tableId, key: self.key(globalId), value: self.sharedBuffer) + } + + func get(globalId: Int32) -> MessageId? { + if let value = self.valueBox.get(self.tableId, key: self.key(globalId)) { + var idPeerId: Int64 = 0 + var idNamespace: Int32 = 0 + value.read(&idPeerId, offset: 0, length: 8) + value.read(&idNamespace, offset: 0, length: 4) + return MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: globalId) + } + return nil + } + + func remove(globalId: Int32) { + self.valueBox.remove(self.tableId, key: self.key(globalId)) + } +} \ No newline at end of file diff --git a/Postbox/KeychainTable.swift b/Postbox/KeychainTable.swift new file mode 100644 index 0000000000..34da757a82 --- /dev/null +++ b/Postbox/KeychainTable.swift @@ -0,0 +1,30 @@ +import Foundation + +final class KeychainTable { + let valueBox: ValueBox + let tableId: Int32 + + init(valueBox: ValueBox, tableId: Int32) { + self.valueBox = valueBox + self.tableId = tableId + } + + private func key(string: String) -> ValueBoxKey { + return ValueBoxKey(string) + } + + func get(key: String) -> NSData? { + if let value = self.valueBox.get(self.tableId, key: self.key(key)) { + return NSData(bytes: value.memory, length: value.length) + } + return nil + } + + func set(key: String, value: NSData) { + self.valueBox.set(self.tableId, key: self.key(key), value: MemoryBuffer(data: value)) + } + + func remove(key: String) { + self.valueBox.remove(self.tableId, key: self.key(key)) + } +} diff --git a/Postbox/Media.swift b/Postbox/Media.swift index 4cd9848945..4d7d771546 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -70,6 +70,7 @@ public func ==(lhs: MediaId, rhs: MediaId) -> Bool { public protocol Media: Coding { var id: MediaId? { get } + var peerIds: [PeerId] { get } func isEqual(other: Media) -> Bool } diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 54ac816893..472182614a 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -89,7 +89,7 @@ public func <(lhs: MessageId, rhs: MessageId) -> Bool { } } -public struct MessageIndex: Equatable, Comparable { +public struct MessageIndex: Equatable, Comparable, Hashable { public let id: MessageId public let timestamp: Int32 @@ -115,6 +115,14 @@ public struct MessageIndex: Equatable, Comparable { public func successor() -> MessageIndex { return MessageIndex(id: MessageId(peerId: self.id.peerId, namespace: self.id.namespace, id: self.id.id + 1), timestamp: self.timestamp) } + + public var hashValue: Int { + return self.id.hashValue + } + + static func upperBound(peerId: PeerId) -> MessageIndex { + return MessageIndex(id: MessageId(peerId: peerId, namespace: Int32.max, id: Int32.max), timestamp: Int32.max) + } } public func ==(lhs: MessageIndex, rhs: MessageIndex) -> Bool { @@ -134,30 +142,36 @@ public func <(lhs: MessageIndex, rhs: MessageIndex) -> Bool { } public class Message { - let id: MessageId - let timestamp: Int32 - let text: String - let attributes: [Coding] - let media: [Media] + public let id: MessageId + public let timestamp: Int32 + public let author: Peer? + public let text: String + public let attributes: [Coding] + public let media: [Media] + public let peers: SimpleDictionary - init(id: MessageId, timestamp: Int32, text: String, attributes: [Coding], media: [Media]) { + init(id: MessageId, timestamp: Int32, author: Peer?, text: String, attributes: [Coding], media: [Media], peers: SimpleDictionary) { self.id = id self.timestamp = timestamp + self.author = author self.text = text self.attributes = attributes self.media = media + self.peers = peers } } -class StoreMessage { - let id: MessageId - let timestamp: Int32 - let text: String - let attributes: [Coding] - let media: [Media] +public final class StoreMessage { + public let id: MessageId + public let timestamp: Int32 + public let authorId: PeerId? + public let text: String + public let attributes: [Coding] + public let media: [Media] - init(id: MessageId, timestamp: Int32, text: String, attributes: [Coding], media: [Media]) { + public init(id: MessageId, timestamp: Int32, authorId: PeerId?, text: String, attributes: [Coding], media: [Media]) { self.id = id + self.authorId = authorId self.timestamp = timestamp self.text = text self.attributes = attributes @@ -168,47 +182,19 @@ class StoreMessage { class IntermediateMessage { let id: MessageId let timestamp: Int32 + let authorId: PeerId? let text: String let attributesData: ReadBuffer let embeddedMediaData: ReadBuffer let referencedMedia: [MediaId] - init(id: MessageId, timestamp: Int32, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { + init(id: MessageId, timestamp: Int32, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { self.id = id self.timestamp = timestamp + self.authorId = authorId self.text = text self.attributesData = attributesData self.embeddedMediaData = embeddedMediaData self.referencedMedia = referencedMedia } } - -public struct RenderedMessage: Equatable, Comparable { - public let message: Message - - internal let incomplete: Bool - public let peers: [Peer] - public let media: [Media] - - internal init(message: Message) { - self.message = message - self.peers = [] - self.media = [] - self.incomplete = true - } - - internal init(message: Message, peers: [Peer], media: [Media]) { - self.message = message - self.peers = peers - self.media = media - self.incomplete = false - } -} - -public func ==(lhs: RenderedMessage, rhs: RenderedMessage) -> Bool { - return lhs.message.id == rhs.message.id -} - -public func <(lhs: RenderedMessage, rhs: RenderedMessage) -> Bool { - return MessageIndex(lhs.message) < MessageIndex(rhs.message) -} diff --git a/Postbox/MessageHistoryHole.swift b/Postbox/MessageHistoryHole.swift index e43822bba9..b360771793 100644 --- a/Postbox/MessageHistoryHole.swift +++ b/Postbox/MessageHistoryHole.swift @@ -1,12 +1,16 @@ import Foundation -public struct MessageHistoryHole: Equatable { - let maxIndex: MessageIndex - let min: MessageId.Id +public struct MessageHistoryHole: Equatable, CustomStringConvertible { + public let maxIndex: MessageIndex + public let min: MessageId.Id var id: MessageId { return maxIndex.id } + + public var description: String { + return "MessageHistoryHole(peerId: \(self.maxIndex.id.peerId), min: \(self.min), max: \(self.maxIndex.id.id))" + } } public func ==(lhs: MessageHistoryHole, rhs: MessageHistoryHole) -> Bool { diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index feb548012c..228e1f7d84 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -3,14 +3,34 @@ import Foundation enum HistoryIndexEntry { case Message(MessageIndex) case Hole(MessageHistoryHole) + + var index: MessageIndex { + switch self { + case let .Message(index): + return index + case let .Hole(hole): + return hole.maxIndex + } + } } -enum HoleFillType { +public enum HoleFillType { case UpperToLower case LowerToUpper case Complete } +public enum AddMessagesLocation { + case Random + case UpperHistoryBlock +} + +enum MessageHistoryIndexOperation { + case InsertMessage(StoreMessage) + case InsertHole(MessageHistoryHole) + case Remove(MessageIndex) +} + private func readHistoryIndexEntry(peerId: PeerId, namespace: MessageId.Namespace, key: ValueBoxKey, value: ReadBuffer) -> HistoryIndexEntry { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) @@ -30,13 +50,17 @@ private func readHistoryIndexEntry(peerId: PeerId, namespace: MessageId.Namespac final class MessageHistoryIndexTable { let valueBox: ValueBox let tableId: Int32 + let globalMessageIdsNamespace: Int32 + let globalMessageIdsTable: GlobalMessageIdsTable - init(valueBox: ValueBox, tableId: Int32) { + init(valueBox: ValueBox, tableId: Int32, globalMessageIdsTable: GlobalMessageIdsTable) { self.valueBox = valueBox self.tableId = tableId + self.globalMessageIdsTable = globalMessageIdsTable + self.globalMessageIdsNamespace = globalMessageIdsTable.namespace } - func key(id: MessageId) -> ValueBoxKey { + private func key(id: MessageId) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 4 + 4) key.setInt64(0, value: id.peerId.toInt64()) key.setInt32(8, value: id.namespace) @@ -44,35 +68,23 @@ final class MessageHistoryIndexTable { return key } - func lowerBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + private func lowerBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 4) key.setInt64(0, value: peerId.toInt64()) key.setInt32(8, value: namespace) return key } - func upperBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + private func upperBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 4) key.setInt64(0, value: peerId.toInt64()) key.setInt32(8, value: namespace) return key.successor } - func addHole(id: MessageId) { + func addHole(id: MessageId, inout operations: [MessageHistoryIndexOperation]) { let adjacent = self.adjacentItems(id) - /* - - 1. [x] nothing - 2. [x] message - * - nothing - 3. [x] nothing - * - message - 4. [x] nothing - * - hole - 5. [x] message - * - message - 6. [x] hole - * - message - 7. [x] message - * - hole - - */ - if let lowerItem = adjacent.lower, upperItem = adjacent.upper { switch lowerItem { case .Hole: @@ -83,7 +95,7 @@ final class MessageHistoryIndexTable { break case let .Message(upperMessage): if lowerMessage.id.id < upperMessage.id.id - 1 { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerMessage.id.id + 1)) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerMessage.id.id + 1), operations: &operations) } break } @@ -91,7 +103,7 @@ final class MessageHistoryIndexTable { } else if let lowerItem = adjacent.lower { switch lowerItem { case let .Message(lowerMessage): - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerMessage.id.id + 1)) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerMessage.id.id + 1), operations: &operations) case .Hole: break } @@ -99,46 +111,104 @@ final class MessageHistoryIndexTable { switch upperItem { case let .Message(upperMessage): if upperMessage.id.id > 1 { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: 1)) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: 1), operations: &operations) } case .Hole: break } } else { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: 1)) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: 1), operations: &operations) } } - func addMessage(index: MessageIndex) { - var upperItem: HistoryIndexEntry? - self.valueBox.range(self.tableId, start: self.key(index.id).predecessor, end: self.upperBound(index.id.peerId, namespace: index.id.namespace), values: { key, value in - upperItem = readHistoryIndexEntry(index.id.peerId, namespace: index.id.namespace, key: key, value: value) - return true - }, limit: 1) + func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operations: [MessageHistoryIndexOperation]) { + if messages.count == 0 { + return + } - if let upperItem = upperItem { - switch upperItem { - case let .Hole(upperHole): - self.justRemove(upperHole.id) - if upperHole.maxIndex.id.id > index.id.id + 1 { - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: index.id.id + 1)) + switch location { + case .UpperHistoryBlock: + var lowerIds = SimpleDictionary>() + for message in messages { + if lowerIds[message.id.peerId] == nil { + var dict = SimpleDictionary() + dict[message.id.namespace] = MessageIndex(id: message.id, timestamp: message.timestamp) + lowerIds[message.id.peerId] = dict + } else { + let lowerIndex = lowerIds[message.id.peerId]![message.id.namespace] + if lowerIndex == nil || lowerIndex!.id.id > message.id.id { + lowerIds[message.id.peerId]![message.id.namespace] = MessageIndex(id: message.id, timestamp: message.timestamp) + } } - if upperHole.min <= index.id.id - 1 { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: index.id.peerId, namespace: index.id.namespace, id: index.id.id - 1), timestamp: index.timestamp), min: upperHole.min)) + } + + for (peerId, lowerIdsByNamespace) in lowerIds { + for (namespace, lowerIndex) in lowerIdsByNamespace { + var removeHoles: [MessageIndex] = [] + var modifyHole: (MessageIndex, MessageHistoryHole)? + self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: lowerIndex.id.id)), end: self.upperBound(peerId, namespace: namespace), values: { key, value in + let entry = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value) + if case let .Hole(hole) = entry { + if lowerIndex.id.id <= hole.min { + removeHoles.append(hole.maxIndex) + } else { + modifyHole = (hole.maxIndex, MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: lowerIndex.id.id - 1), timestamp: lowerIndex.timestamp), min: hole.min)) + } + } + return true + }, limit: 0) + + for index in removeHoles { + self.justRemove(index, operations: &operations) + } + + if let modifyHole = modifyHole { + self.justRemove(modifyHole.0, operations: &operations) + self.justInsertHole(modifyHole.1, operations: &operations) + } } - break - case .Message: - break + } + case .Random: + break + } + + for message in messages { + let index = MessageIndex(id: message.id, timestamp: message.timestamp) + + var upperItem: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: self.key(index.id).predecessor, end: self.upperBound(index.id.peerId, namespace: index.id.namespace), values: { key, value in + upperItem = readHistoryIndexEntry(index.id.peerId, namespace: index.id.namespace, key: key, value: value) + return true + }, limit: 1) + + var exists = false + + if let upperItem = upperItem { + switch upperItem { + case let .Hole(upperHole): + self.justRemove(upperHole.maxIndex, operations: &operations) + if upperHole.maxIndex.id.id > index.id.id + 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: index.id.id + 1), operations: &operations) + } + if upperHole.min <= index.id.id - 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: index.id.peerId, namespace: index.id.namespace, id: index.id.id - 1), timestamp: index.timestamp), min: upperHole.min), operations: &operations) + } + case let .Message(messageIndex): + if messageIndex.id == index.id { + exists = true + } + } + } + + if !exists { + self.justInsertMessage(message, operations: &operations) } } - - self.justInsertMessage(index) } - func removeMessage(id: MessageId) { - let key = self.key(id) - if self.valueBox.exists(self.tableId, key: key) { - self.justRemove(id) + func removeMessage(id: MessageId, inout operations: [MessageHistoryIndexOperation]) { + if let existingEntry = self.get(id) { + self.justRemove(existingEntry.index, operations: &operations) let adjacent = self.adjacentItems(id) @@ -147,27 +217,27 @@ final class MessageHistoryIndexTable { case let .Message(lowerMessage): switch upperItem { case let .Hole(upperHole): - self.justRemove(upperHole.id) - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerMessage.id.id + 1)) + self.justRemove(upperHole.maxIndex, operations: &operations) + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerMessage.id.id + 1), operations: &operations) case .Message: break } case let .Hole(lowerHole): switch upperItem { case let .Hole(upperHole): - self.justRemove(lowerHole.id) - self.justRemove(upperHole.id) - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerHole.min)) + self.justRemove(lowerHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, operations: &operations) + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerHole.min), operations: &operations) case let .Message(upperMessage): - self.justRemove(lowerHole.id) - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerHole.min)) + self.justRemove(lowerHole.maxIndex, operations: &operations) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerHole.min), operations: &operations) } } } else if let lowerItem = adjacent.lower { switch lowerItem { case let .Hole(lowerHole): - self.justRemove(lowerHole.id) - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerHole.min)) + self.justRemove(lowerHole.maxIndex, operations: &operations) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerHole.min), operations: &operations) break case .Message: break @@ -175,8 +245,8 @@ final class MessageHistoryIndexTable { } else if let upperItem = adjacent.upper { switch upperItem { case let .Hole(upperHole): - self.justRemove(upperHole.id) - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: 1)) + self.justRemove(upperHole.maxIndex, operations: &operations) + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: 1), operations: &operations) break case .Message: break @@ -185,42 +255,42 @@ final class MessageHistoryIndexTable { } } - func fillHole(id: MessageId, fillType: HoleFillType, indices: [MessageIndex]) { + func fillHole(id: MessageId, fillType: HoleFillType, messages: [StoreMessage], inout operations: [MessageHistoryIndexOperation]) { var upperItem: HistoryIndexEntry? self.valueBox.range(self.tableId, start: self.key(id).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) return true }, limit: 1) - let sortedByIdIndices = indices.sort({$0.id < $1.id}) - var remainingIndices = sortedByIdIndices + let sortedByIdMessages = messages.sort({$0.id < $1.id}) + var remainingMessages = sortedByIdMessages if let upperItem = upperItem { switch upperItem { case let .Hole(upperHole): var i = 0 - var minIndexInRange: MessageIndex? - var maxIndexInRange: MessageIndex? + var minMessageInRange: StoreMessage? + var maxMessageInRange: StoreMessage? var removedHole = false - while i < remainingIndices.count { - let index = remainingIndices[i] - if index.id.id >= upperHole.min && index.id.id <= upperHole.maxIndex.id.id { - if (fillType == .UpperToLower || fillType == .Complete) && (minIndexInRange == nil || minIndexInRange!.id > index.id) { - minIndexInRange = index + while i < remainingMessages.count { + let message = remainingMessages[i] + if message.id.id >= upperHole.min && message.id.id <= upperHole.maxIndex.id.id { + if (fillType == .UpperToLower || fillType == .Complete) && (minMessageInRange == nil || minMessageInRange!.id > message.id) { + minMessageInRange = message if !removedHole { removedHole = true - self.justRemove(upperHole.id) + self.justRemove(upperHole.maxIndex, operations: &operations) } } - if (fillType == .LowerToUpper || fillType == .Complete) && (maxIndexInRange == nil || maxIndexInRange!.id < index.id) { - maxIndexInRange = index + if (fillType == .LowerToUpper || fillType == .Complete) && (maxMessageInRange == nil || maxMessageInRange!.id < message.id) { + maxMessageInRange = message if !removedHole { removedHole = true - self.justRemove(upperHole.id) + self.justRemove(upperHole.maxIndex, operations: &operations) } } - self.justInsertMessage(index) - remainingIndices.removeAtIndex(i) + self.justInsertMessage(message, operations: &operations) + remainingMessages.removeAtIndex(i) } else { i++ } @@ -228,15 +298,15 @@ final class MessageHistoryIndexTable { switch fillType { case .Complete: if !removedHole { - self.justRemove(upperHole.id) + self.justRemove(upperHole.maxIndex, operations: &operations) } case .LowerToUpper: - if let maxIndexInRange = maxIndexInRange where maxIndexInRange.id.id != Int32.max && maxIndexInRange.id.id + 1 <= upperHole.maxIndex.id.id { - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: maxIndexInRange.id.id + 1)) + if let maxMessageInRange = maxMessageInRange where maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: maxMessageInRange.id.id + 1), operations: &operations) } case .UpperToLower: - if let minIndexInRange = minIndexInRange where minIndexInRange.id.id - 1 >= upperHole.min { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minIndexInRange.id.id - 1), timestamp: minIndexInRange.timestamp), min: upperHole.min)) + if let minMessageInRange = minMessageInRange where minMessageInRange.id.id - 1 >= upperHole.min { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minMessageInRange.id.id - 1), timestamp: minMessageInRange.timestamp), min: upperHole.min), operations: &operations) } } break @@ -245,12 +315,12 @@ final class MessageHistoryIndexTable { } } - for index in remainingIndices { - self.addMessage(index) + for message in remainingMessages { + self.addMessages([message], location: .Random, operations: &operations) } } - func justInsertHole(hole: MessageHistoryHole) { + func justInsertHole(hole: MessageHistoryHole, inout operations: [MessageHistoryIndexOperation]) { let value = WriteBuffer() var type: Int8 = 1 var timestamp: Int32 = hole.maxIndex.timestamp @@ -259,19 +329,33 @@ final class MessageHistoryIndexTable { value.write(×tamp, offset: 0, length: 4) value.write(&min, offset: 0, length: 4) self.valueBox.set(self.tableId, key: self.key(hole.id), value: value) + + operations.append(.InsertHole(hole)) } - func justInsertMessage(index: MessageIndex) { + func justInsertMessage(message: StoreMessage, inout operations: [MessageHistoryIndexOperation]) { + let index = MessageIndex(id: message.id, timestamp: message.timestamp) + let value = WriteBuffer() var type: Int8 = 0 var timestamp: Int32 = index.timestamp value.write(&type, offset: 0, length: 1) value.write(×tamp, offset: 0, length: 4) self.valueBox.set(self.tableId, key: self.key(index.id), value: value) + + operations.append(.InsertMessage(message)) + if index.id.namespace == self.globalMessageIdsNamespace { + self.globalMessageIdsTable.set(index.id.id, id: index.id) + } } - func justRemove(id: MessageId) { - self.valueBox.remove(self.tableId, key: self.key(id)) + func justRemove(index: MessageIndex, inout operations: [MessageHistoryIndexOperation]) { + self.valueBox.remove(self.tableId, key: self.key(index.id)) + + operations.append(.Remove(index)) + if index.id.namespace == self.globalMessageIdsNamespace { + self.globalMessageIdsTable.remove(index.id.id) + } } func adjacentItems(id: MessageId) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) { @@ -300,16 +384,6 @@ final class MessageHistoryIndexTable { return nil } - func messageExists(id: MessageId) -> Bool { - if let entry = self.get(id) { - if case .Message = entry { - return true - } - } - - return false - } - func debugList(peerId: PeerId, namespace: MessageId.Namespace) -> [HistoryIndexEntry] { var list: [HistoryIndexEntry] = [] self.valueBox.range(self.tableId, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 9600d5705f..f649976f91 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -1,19 +1,39 @@ import Foundation -enum MessageHistoryEntry { - case Msg(Message) +enum MessageHistoryOperation { + case InsertMessage(IntermediateMessage) + case InsertHole(MessageHistoryHole) + case Remove([MessageIndex]) +} + +enum IntermediateMessageHistoryEntry { + case Message(IntermediateMessage) case Hole(MessageHistoryHole) var index: MessageIndex { switch self { - case let .Msg(message): - return MessageIndex(message) + case let .Message(message): + return MessageIndex(id: message.id, timestamp: message.timestamp) case let .Hole(hole): return hole.maxIndex } } } +enum RenderedMessageHistoryEntry { + case RenderedMessage(Message) + case Hole(MessageHistoryHole) + + var index: MessageIndex { + switch self { + case let .RenderedMessage(message): + return MessageIndex(id: message.id, timestamp: message.timestamp) + case let .Hole(hole): + return hole.maxIndex + } + } +} + final class MessageHistoryTable { let valueBox: ValueBox let tableId: Int32 @@ -48,7 +68,7 @@ final class MessageHistoryTable { return key.successor } - private func messagesByPeerId(messages: [StoreMessage]) -> [PeerId: [StoreMessage]] { + private func messagesGroupedByPeerId(messages: [StoreMessage]) -> [PeerId: [StoreMessage]] { var dict: [PeerId: [StoreMessage]] = [:] for message in messages { @@ -63,74 +83,154 @@ final class MessageHistoryTable { return dict } - func addMessages(messages: [StoreMessage]) { + private func messageIdsByPeerId(ids: [MessageId]) -> [PeerId: [MessageId]] { + var dict: [PeerId: [MessageId]] = [:] + + for id in ids { + let peerId = id.peerId + if dict[peerId] == nil { + dict[peerId] = [id] + } else { + dict[peerId]!.append(id) + } + } + + return dict + } + + private func processIndexOperations(peerId: PeerId, operations: [MessageHistoryIndexOperation], inout processedOperationsByPeerId: [PeerId: [MessageHistoryOperation]]) { let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0)) let sharedBuffer = WriteBuffer() let sharedEncoder = Encoder() - let messagesByPeerId = self.messagesByPeerId(messages) - for (_, peerMessages) in messagesByPeerId { - for message in peerMessages { - if self.messageHistoryIndexTable.messageExists(message.id) { - continue + var outputOperations: [MessageHistoryOperation] = [] + var accumulatedRemoveIndices: [MessageIndex] = [] + for operation in operations { + switch operation { + case let .InsertHole(hole): + if accumulatedRemoveIndices.count != 0 { + outputOperations.append(.Remove(accumulatedRemoveIndices)) + accumulatedRemoveIndices.removeAll() } - - self.messageHistoryIndexTable.addMessage(MessageIndex(message)) - self.justInsert(message, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder) + self.justInsertHole(hole) + outputOperations.append(.InsertHole(hole)) + case let .InsertMessage(storeMessage): + if accumulatedRemoveIndices.count != 0 { + outputOperations.append(.Remove(accumulatedRemoveIndices)) + accumulatedRemoveIndices.removeAll() + } + let message = self.justInsertMessage(storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder) + outputOperations.append(.InsertMessage(message)) + case let .Remove(index): + self.justRemove(index) + accumulatedRemoveIndices.append(index) } } + if accumulatedRemoveIndices.count != 0 { + outputOperations.append(.Remove(accumulatedRemoveIndices)) + } + + if processedOperationsByPeerId[peerId] == nil { + processedOperationsByPeerId[peerId] = outputOperations + } else { + processedOperationsByPeerId[peerId]!.appendContentsOf(outputOperations) + } } - func removeMessages(messageIds: [MessageId]) { - for messageId in messageIds { - if let entry = self.messageHistoryIndexTable.get(messageId) { - if case let .Message(index) = entry { - if let message = self.get(index) { - let embeddedMediaData = message.embeddedMediaData - if embeddedMediaData.length > 4 { - var embeddedMediaCount: Int32 = 0 - embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) - for _ in 0 ..< embeddedMediaCount { - var mediaLength: Int32 = 0 - embeddedMediaData.read(&mediaLength, offset: 0, length: 4) - if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { - self.messageMediaTable.removeEmbeddedMedia(media) - } - embeddedMediaData.skip(Int(mediaLength)) - } - } - - for mediaId in message.referencedMedia { - self.messageMediaTable.removeReference(mediaId) - } - } - - self.messageHistoryIndexTable.removeMessage(messageId) - self.valueBox.remove(self.tableId, key: self.key(index)) - } - } + func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + let messagesByPeerId = self.messagesGroupedByPeerId(messages) + for (peerId, peerMessages) in messagesByPeerId { + var operations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.addMessages(peerMessages, location: location, operations: &operations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId) } } - private func justInsert(message: StoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) { + func addHoles(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { + var operations: [MessageHistoryIndexOperation] = [] + for id in messageIds { + self.messageHistoryIndexTable.addHole(id, operations: &operations) + } + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId) + } + } + + func removeMessages(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { + var operations: [MessageHistoryIndexOperation] = [] + for id in messageIds { + self.messageHistoryIndexTable.removeMessage(id, operations: &operations) + } + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId) + } + } + + func fillHole(id: MessageId, fillType: HoleFillType, messages: [StoreMessage], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + var operations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.fillHole(id, fillType: fillType, messages: messages, operations: &operations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId) + } + + func topMessage(peerId: PeerId) -> IntermediateMessage? { + var currentKey = self.lowerBound(peerId) + while true { + var entry: IntermediateMessageHistoryEntry? + self.valueBox.range(self.tableId, start: self.upperBound(peerId), end: currentKey, values: { key, value in + entry = self.readIntermediateEntry(key, value: value) + return true + }, limit: 1) + + if let entry = entry { + switch entry { + case .Hole: + currentKey = self.key(entry.index).predecessor + case let .Message(message): + return message + } + } else { + break + } + } + return nil + } + + private func justInsertMessage(message: StoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) -> IntermediateMessage { sharedBuffer.reset() + + var type: Int8 = 0 + sharedBuffer.write(&type, offset: 0, length: 1) + + if let authorId = message.authorId { + var varAuthorId: Int64 = authorId.toInt64() + var hasAuthor: Int8 = 1 + sharedBuffer.write(&hasAuthor, offset: 0, length: 1) + sharedBuffer.write(&varAuthorId, offset: 0, length: 8) + } else { + var hasAuthor: Int8 = 0 + sharedBuffer.write(&hasAuthor, offset: 0, length: 1) + } let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! var length: Int32 = Int32(data.length) sharedBuffer.write(&length, offset: 0, length: 4) sharedBuffer.write(data.bytes, offset: 0, length: Int(length)) + let attributesBuffer = WriteBuffer() + var attributeCount: Int32 = Int32(message.attributes.count) - sharedBuffer.write(&attributeCount, offset: 0, length: 4) + attributesBuffer.write(&attributeCount, offset: 0, length: 4) for attribute in message.attributes { sharedEncoder.reset() sharedEncoder.encodeRootObject(attribute) let attributeBuffer = sharedEncoder.memoryBuffer() var attributeBufferLength = Int32(attributeBuffer.length) - sharedBuffer.write(&attributeBufferLength, offset: 0, length: 4) - sharedBuffer.write(attributeBuffer.memory, offset: 0, length: attributeBuffer.length) + attributesBuffer.write(&attributeBufferLength, offset: 0, length: 4) + attributesBuffer.write(attributeBuffer.memory, offset: 0, length: attributeBuffer.length) } + sharedBuffer.write(attributesBuffer.memory, offset: 0, length: attributesBuffer.length) + var embeddedMedia: [Media] = [] var referencedMedia: [MediaId] = [] for media in message.media { @@ -147,17 +247,20 @@ final class MessageHistoryTable { } } + let embeddedMediaBuffer = WriteBuffer() var embeddedMediaCount: Int32 = Int32(embeddedMedia.count) - sharedBuffer.write(&embeddedMediaCount, offset: 0, length: 4) + embeddedMediaBuffer.write(&embeddedMediaCount, offset: 0, length: 4) for media in embeddedMedia { sharedEncoder.reset() sharedEncoder.encodeRootObject(media) let mediaBuffer = sharedEncoder.memoryBuffer() var mediaBufferLength = Int32(mediaBuffer.length) - sharedBuffer.write(&mediaBufferLength, offset: 0, length: 4) - sharedBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length) + embeddedMediaBuffer.write(&mediaBufferLength, offset: 0, length: 4) + embeddedMediaBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length) } + sharedBuffer.write(embeddedMediaBuffer.memory, offset: 0, length: embeddedMediaBuffer.length) + var referencedMediaCount: Int32 = Int32(referencedMedia.count) sharedBuffer.write(&referencedMediaCount, offset: 0, length: 4) for mediaId in referencedMedia { @@ -168,10 +271,51 @@ final class MessageHistoryTable { } self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) + + return IntermediateMessage(id: message.id, timestamp: message.timestamp, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) + } + + private func justInsertHole(hole: MessageHistoryHole, sharedBuffer: WriteBuffer = WriteBuffer()) { + sharedBuffer.reset() + var type: Int8 = 1 + sharedBuffer.write(&type, offset: 0, length: 1) + var minId: Int32 = hole.min + sharedBuffer.write(&minId, offset: 0, length: 4) + self.valueBox.set(self.tableId, key: self.key(hole.maxIndex), value: sharedBuffer.readBufferNoCopy()) + } + + private func justRemove(index: MessageIndex) { + let key = self.key(index) + if let value = self.valueBox.get(self.tableId, key: key) { + switch self.readIntermediateEntry(key, value: value) { + case let .Message(message): + let embeddedMediaData = message.embeddedMediaData + if embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + self.messageMediaTable.removeEmbeddedMedia(media) + } + embeddedMediaData.skip(Int(mediaLength)) + } + } + + for mediaId in message.referencedMedia { + self.messageMediaTable.removeReference(mediaId) + } + case .Hole: + break + } + + self.valueBox.remove(self.tableId, key: key) + } } func unembedMedia(index: MessageIndex, id: MediaId) -> Media? { - if let message = self.get(index) where message.embeddedMediaData.length > 4 { + if let message = self.getMessage(index) where message.embeddedMediaData.length > 4 { var embeddedMediaCount: Int32 = 0 message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) @@ -202,7 +346,7 @@ final class MessageHistoryTable { if let extractedMedia = extractedMedia { var updatedReferencedMedia = message.referencedMedia updatedReferencedMedia.append(extractedMedia.id!) - self.storeIntermediateMessage(IntermediateMessage(id: message.id, timestamp: message.timestamp, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) + self.storeIntermediateMessage(IntermediateMessage(id: message.id, timestamp: message.timestamp, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) return extractedMedia } @@ -213,6 +357,19 @@ final class MessageHistoryTable { func storeIntermediateMessage(message: IntermediateMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer = WriteBuffer()) { sharedBuffer.reset() + var type: Int8 = 0 + sharedBuffer.write(&type, offset: 0, length: 1) + + if let authorId = message.authorId { + var varAuthorId: Int64 = authorId.toInt64() + var hasAuthor: Int8 = 1 + sharedBuffer.write(&hasAuthor, offset: 0, length: 1) + sharedBuffer.write(&varAuthorId, offset: 0, length: 8) + } else { + var hasAuthor: Int8 = 0 + sharedBuffer.write(&hasAuthor, offset: 0, length: 1) + } + let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! var length: Int32 = Int32(data.length) sharedBuffer.write(&length, offset: 0, length: 4) @@ -233,63 +390,83 @@ final class MessageHistoryTable { self.valueBox.set(self.tableId, key: self.key(MessageIndex(id: message.id, timestamp: message.timestamp), key: sharedKey), value: sharedBuffer) } - private func readIntermediateMessage(key: ValueBoxKey, value: ReadBuffer) -> IntermediateMessage { + private func readIntermediateEntry(key: ValueBoxKey, value: ReadBuffer) -> IntermediateMessageHistoryEntry { let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4), id: key.getInt32(8 + 4 + 4)), timestamp: key.getInt32(8)) - var textLength: Int32 = 0 - value.read(&textLength, offset: 0, length: 4) - let text = String(data: NSData(bytes: value.memory + value.offset, length: Int(textLength)), encoding: NSUTF8StringEncoding) ?? "" - value.skip(Int(textLength)) - - let attributesOffset = value.offset - var attributeCount: Int32 = 0 - value.read(&attributeCount, offset: 0, length: 4) - for _ in 0 ..< attributeCount { - var attributeLength: Int32 = 0 - value.read(&attributeLength, offset: 0, length: 4) - value.skip(Int(attributeLength)) + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == 0 { + var hasAuthor: Int8 = 0 + value.read(&hasAuthor, offset: 0, length: 1) + var authorId: PeerId? + if hasAuthor == 1 { + var varAuthorId: Int64 = 0 + value.read(&varAuthorId, offset: 0, length: 8) + authorId = PeerId(varAuthorId) + } + + var textLength: Int32 = 0 + value.read(&textLength, offset: 0, length: 4) + let text = String(data: NSData(bytes: value.memory + value.offset, length: Int(textLength)), encoding: NSUTF8StringEncoding) ?? "" + value.skip(Int(textLength)) + + let attributesOffset = value.offset + var attributeCount: Int32 = 0 + value.read(&attributeCount, offset: 0, length: 4) + for _ in 0 ..< attributeCount { + var attributeLength: Int32 = 0 + value.read(&attributeLength, offset: 0, length: 4) + value.skip(Int(attributeLength)) + } + let attributesLength = value.offset - attributesOffset + let attributesBytes = malloc(attributesLength) + memcpy(attributesBytes, value.memory + attributesOffset, attributesLength) + let attributesData = ReadBuffer(memory: attributesBytes, length: attributesLength, freeWhenDone: true) + + let embeddedMediaOffset = value.offset + var embeddedMediaCount: Int32 = 0 + value.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + value.read(&mediaLength, offset: 0, length: 4) + value.skip(Int(mediaLength)) + } + let embeddedMediaLength = value.offset - embeddedMediaOffset + let embeddedMediaBytes = malloc(embeddedMediaLength) + memcpy(embeddedMediaBytes, value.memory + embeddedMediaOffset, embeddedMediaLength) + let embeddedMediaData = ReadBuffer(memory: embeddedMediaBytes, length: embeddedMediaLength, freeWhenDone: true) + + var referencedMediaIds: [MediaId] = [] + var referencedMediaIdsCount: Int32 = 0 + value.read(&referencedMediaIdsCount, offset: 0, length: 4) + for _ in 0 ..< referencedMediaIdsCount { + var idNamespace: Int32 = 0 + var idId: Int64 = 0 + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 8) + referencedMediaIds.append(MediaId(namespace: idNamespace, id: idId)) + } + + return .Message(IntermediateMessage(id: index.id, timestamp: index.timestamp, authorId: authorId, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)) + } else { + var minId: Int32 = 0 + value.read(&minId, offset: 0, length: 4) + + return .Hole(MessageHistoryHole(maxIndex: index, min: minId)) } - let attributesLength = value.offset - attributesOffset - let attributesBytes = malloc(attributesLength) - memcpy(attributesBytes, value.memory + attributesOffset, attributesLength) - let attributesData = ReadBuffer(memory: attributesBytes, length: attributesLength, freeWhenDone: true) - - let embeddedMediaOffset = value.offset - var embeddedMediaCount: Int32 = 0 - value.read(&embeddedMediaCount, offset: 0, length: 4) - for _ in 0 ..< embeddedMediaCount { - var mediaLength: Int32 = 0 - value.read(&mediaLength, offset: 0, length: 4) - value.skip(Int(mediaLength)) - } - let embeddedMediaLength = value.offset - embeddedMediaOffset - let embeddedMediaBytes = malloc(embeddedMediaLength) - memcpy(embeddedMediaBytes, value.memory + embeddedMediaOffset, embeddedMediaLength) - let embeddedMediaData = ReadBuffer(memory: embeddedMediaBytes, length: embeddedMediaLength, freeWhenDone: true) - - var referencedMediaIds: [MediaId] = [] - var referencedMediaIdsCount: Int32 = 0 - value.read(&referencedMediaIdsCount, offset: 0, length: 4) - for _ in 0 ..< referencedMediaIdsCount { - var idNamespace: Int32 = 0 - var idId: Int64 = 0 - value.read(&idNamespace, offset: 0, length: 4) - value.read(&idId, offset: 0, length: 8) - referencedMediaIds.append(MediaId(namespace: idNamespace, id: idId)) - } - - return IntermediateMessage(id: index.id, timestamp: index.timestamp, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds) } - func get(index: MessageIndex) -> IntermediateMessage? { + func getMessage(index: MessageIndex) -> IntermediateMessage? { let key = self.key(index) if let value = self.valueBox.get(self.tableId, key: key) { - return self.readIntermediateMessage(key, value: value) + if case let .Message(message) = self.readIntermediateEntry(key, value: value) { + return message + } } return nil } - func renderMessage(message: IntermediateMessage) -> Message { + func renderMessage(message: IntermediateMessage, peerTable: PeerTable) -> Message { var parsedAttributes: [Coding] = [] var parsedMedia: [Media] = [] @@ -327,33 +504,95 @@ final class MessageHistoryTable { } } - return Message(id: message.id, timestamp: message.timestamp, text: message.text, attributes: parsedAttributes, media: parsedMedia) + var author: Peer? + if let authorId = message.authorId { + author = peerTable.get(authorId) + } + + var peers = SimpleDictionary() + if let chatPeer = peerTable.get(message.id.peerId) { + peers[chatPeer.id] = chatPeer + } + + for media in parsedMedia { + for peerId in media.peerIds { + if let peer = peerTable.get(peerId) { + peers[peer.id] = peer + } + } + } + + return Message(id: message.id, timestamp: message.timestamp, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers) } - func messagesAround(index: MessageIndex, count: Int) -> [IntermediateMessage] { - var lowerMessages: [IntermediateMessage] = [] - var upperMessages: [IntermediateMessage] = [] + func entriesAround(index: MessageIndex, count: Int) -> [IntermediateMessageHistoryEntry] { + var lowerEntries: [IntermediateMessageHistoryEntry] = [] + var upperEntries: [IntermediateMessageHistoryEntry] = [] self.valueBox.range(self.tableId, start: self.key(index), end: self.lowerBound(index.id.peerId), values: { key, value in - lowerMessages.append(self.readIntermediateMessage(key, value: value)) + lowerEntries.append(self.readIntermediateEntry(key, value: value)) return true - }, limit: count) + }, limit: count / 2) self.valueBox.range(self.tableId, start: self.key(index).predecessor, end: self.upperBound(index.id.peerId), values: { key, value in - upperMessages.append(self.readIntermediateMessage(key, value: value)) + upperEntries.append(self.readIntermediateEntry(key, value: value)) return true - }, limit: count) + }, limit: count - lowerEntries.count) - var messages: [IntermediateMessage] = [] - for message in lowerMessages.reverse() { - messages.append(message) + if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count { + self.valueBox.range(self.tableId, start: self.key(lowerEntries.last!.index), end: self.lowerBound(index.id.peerId), values: { key, value in + lowerEntries.append(self.readIntermediateEntry(key, value: value)) + return true + }, limit: count - (lowerEntries.count + upperEntries.count)) } - messages.appendContentsOf(upperMessages) - return messages + var entries: [IntermediateMessageHistoryEntry] = [] + for entry in lowerEntries.reverse() { + entries.append(entry) + } + entries.appendContentsOf(upperEntries) + + return entries } - func debugList(peerId: PeerId) -> [Message] { - return self.messagesAround(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: 0), count: 1000).map({self.renderMessage($0)}) + func earlierEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + var entries: [IntermediateMessageHistoryEntry] = [] + let key: ValueBoxKey + if let index = index { + key = self.key(index) + } else { + key = self.upperBound(peerId) + } + self.valueBox.range(self.tableId, start: key, end: self.lowerBound(peerId), values: { key, value in + entries.append(self.readIntermediateEntry(key, value: value)) + return true + }, limit: count) + return entries + } + + func laterEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + var entries: [IntermediateMessageHistoryEntry] = [] + let key: ValueBoxKey + if let index = index { + key = self.key(index) + } else { + key = self.lowerBound(peerId) + } + self.valueBox.range(self.tableId, start: key, end: self.upperBound(peerId), values: { key, value in + entries.append(self.readIntermediateEntry(key, value: value)) + return true + }, limit: count) + return entries + } + + func debugList(peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { + return self.entriesAround(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: 0), count: 1000).map({ entry -> RenderedMessageHistoryEntry in + switch entry { + case let .Hole(hole): + return .Hole(hole) + case let .Message(message): + return .RenderedMessage(self.renderMessage(message, peerTable: peerTable)) + } + }) } } diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 0efad9cde0..e3dabce69f 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -1,54 +1,109 @@ import Foundation -public final class MutableMessageHistoryView: CustomStringConvertible { - public struct RemoveContext { - var invalidEarlier: Bool = false - var invalidLater: Bool = false - var invalidEarlierHole: Bool = false - var invalidLaterHole: Bool = false - var removedMessages: Bool = false - var removedHole: Bool = false - - func empty() -> Bool { - return !self.removedMessages && !invalidEarlier && !invalidLater +enum MutableMessageHistoryEntry { + case IntermediateMessageEntry(IntermediateMessage) + case MessageEntry(Message) + case HoleEntry(MessageHistoryHole) + + var index: MessageIndex { + switch self { + case let .IntermediateMessageEntry(message): + return MessageIndex(id: message.id, timestamp: message.timestamp) + case let .MessageEntry(message): + return MessageIndex(id: message.id, timestamp: message.timestamp) + case let .HoleEntry(hole): + return hole.maxIndex } } +} + +public enum MessageHistoryEntry: Comparable { + case MessageEntry(Message) + case HoleEntry(MessageHistoryHole) - let count: Int - var earlierMessage: RenderedMessage? - var laterMessage: RenderedMessage? - var messages: [RenderedMessage] + public var index: MessageIndex { + switch self { + case let .MessageEntry(message): + return MessageIndex(id: message.id, timestamp: message.timestamp) + case let .HoleEntry(hole): + return hole.maxIndex + } + } +} + +public func ==(lhs: MessageHistoryEntry, rhs: MessageHistoryEntry) -> Bool { + return lhs.index == rhs.index +} + +public func <(lhs: MessageHistoryEntry, rhs: MessageHistoryEntry) -> Bool { + return lhs.index < rhs.index +} + +final class MutableMessageHistoryViewReplayContext { + var invalidEarlier: Bool = false + var invalidLater: Bool = false + var removedEntries: Bool = false - public init(count: Int, earlierMessage: RenderedMessage?, messages: [RenderedMessage], laterMessage: RenderedMessage?) { + func empty() -> Bool { + return !self.removedEntries && !invalidEarlier && !invalidLater + } +} + +final class MutableMessageHistoryView { + private let count: Int + private var earlier: MutableMessageHistoryEntry? + private var later: MutableMessageHistoryEntry? + private var entries: [MutableMessageHistoryEntry] + + init(earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, count: Int) { + self.earlier = earlier + self.entries = entries + self.later = later self.count = count - self.earlierMessage = earlierMessage - self.laterMessage = laterMessage - self.messages = messages } - public func add(message: RenderedMessage) -> Bool { - if self.messages.count == 0 { - self.messages.append(message) + func replay(operations: [MessageHistoryOperation], context: MutableMessageHistoryViewReplayContext) -> Bool { + var hasChanges = false + for operation in operations { + switch operation { + case let .InsertHole(hole): + if self.add(.HoleEntry(hole)) { + hasChanges = true + } + case let .InsertMessage(intermediateMessage): + if self.add(.IntermediateMessageEntry(intermediateMessage)) { + hasChanges = true + } + case let .Remove(indices): + if self.remove(Set(indices), context: context) { + hasChanges = true + } + } + } + return hasChanges + } + + private func add(entry: MutableMessageHistoryEntry) -> Bool { + if self.entries.count == 0 { + self.entries.append(entry) return true } else { - let first = MessageIndex(self.messages[self.messages.count - 1].message) - let last = MessageIndex(self.messages[0].message) + let first = self.entries[self.entries.count - 1].index + let last = self.entries[0].index var next: MessageIndex? - if let message = laterMessage { - let messageIndex = MessageIndex(message.message) - next = messageIndex + if let later = self.later { + next = later.index } - let index = MessageIndex(message.message) + let index = entry.index if index < last { - let earlierMessage = self.earlierMessage - if earlierMessage == nil || MessageIndex(earlierMessage!.message) < index { - if self.messages.count < self.count { - self.messages.insert(message, atIndex: 0) + if self.earlier == nil || self.earlier!.index < index { + if self.entries.count < self.count { + self.entries.insert(entry, atIndex: 0) } else { - self.earlierMessage = message + self.earlier = entry } return true } else { @@ -56,39 +111,36 @@ public final class MutableMessageHistoryView: CustomStringConvertible { } } else if index > first { if next != nil && index > next! { - let laterMessage = self.laterMessage - if laterMessage == nil || MessageIndex(laterMessage!.message) > index { - if self.messages.count < self.count { - self.messages.append(message) + if self.later == nil || self.later!.index > index { + if self.entries.count < self.count { + self.entries.append(entry) } else { - self.laterMessage = message + self.later = entry } return true } else { return false } } else { - self.messages.append(message) - if self.messages.count > self.count { - let earliest = self.messages[0] - self.earlierMessage = earliest - self.messages.removeAtIndex(0) + self.entries.append(entry) + if self.entries.count > self.count { + self.earlier = self.entries[0] + self.entries.removeAtIndex(0) } return true } } else if index != last && index != first { - var i = self.messages.count + var i = self.entries.count while i >= 1 { - if MessageIndex(self.messages[i - 1].message) < index { + if self.entries[i - 1].index < index { break } i-- } - self.messages.insert(message, atIndex: i) - if self.messages.count > self.count { - let earliest = self.messages[0] - self.earlierMessage = earliest - self.messages.removeAtIndex(0) + self.entries.insert(entry, atIndex: i) + if self.entries.count > self.count { + self.earlier = self.entries[0] + self.entries.removeAtIndex(0) } return true } else { @@ -97,72 +149,75 @@ public final class MutableMessageHistoryView: CustomStringConvertible { } } - public func remove(ids: Set, context: RemoveContext? = nil) -> RemoveContext { - var updatedContext = RemoveContext() - if let context = context { - updatedContext = context + private func remove(indices: Set, context: MutableMessageHistoryViewReplayContext) -> Bool { + var hasChanges = false + if let earlier = self.earlier where indices.contains(earlier.index) { + context.invalidEarlier = true + hasChanges = true } - if let earlierMessage = self.earlierMessage where ids.contains(earlierMessage.message.id) { - updatedContext.invalidEarlier = true + if let later = self.later where indices.contains(later.index) { + context.invalidLater = true + hasChanges = true } - if let laterMessage = self.laterMessage where ids.contains(laterMessage.message.id) { - updatedContext.invalidLater = true - } - - if self.messages.count != 0 { - var i = self.messages.count - 1 + if self.entries.count != 0 { + var i = self.entries.count - 1 while i >= 0 { - if ids.contains(self.messages[i].message.id) { - self.messages.removeAtIndex(i) - updatedContext.removedMessages = true + if indices.contains(self.entries[i].index) { + self.entries.removeAtIndex(i) + context.removedEntries = true + hasChanges = true } i-- } } - return updatedContext + return hasChanges } - public func complete(context: RemoveContext, fetchEarlier: (MessageIndex?, Int) -> [RenderedMessage], fetchLater: (MessageIndex?, Int) -> [RenderedMessage]) { - if context.removedMessages { - var addedMessages: [RenderedMessage] = [] + func updatePeers(peers: [PeerId: Peer]) -> Bool { + return false + } + + func complete(context: MutableMessageHistoryViewReplayContext, fetchEarlier: (MessageIndex?, Int) -> [MutableMessageHistoryEntry], fetchLater: (MessageIndex?, Int) -> [MutableMessageHistoryEntry]) { + if context.removedEntries { + var addedEntries: [MutableMessageHistoryEntry] = [] var latestAnchor: MessageIndex? - if let lastMessage = self.messages.last { - latestAnchor = MessageIndex(lastMessage.message) + if let last = self.entries.last { + latestAnchor = last.index } if latestAnchor == nil { - if let laterMessage = self.laterMessage { - latestAnchor = MessageIndex(laterMessage.message) + if let later = self.later { + latestAnchor = later.index } } - if let laterMessage = self.laterMessage { - addedMessages += fetchLater(MessageIndex(laterMessage.message).predecessor(), self.count) + if let later = self.later { + addedEntries += fetchLater(later.index.predecessor(), self.count) } - if let earlierMessage = self.earlierMessage { - addedMessages += fetchEarlier(MessageIndex(earlierMessage.message).successor(), self.count) + if let earlier = self.earlier { + addedEntries += fetchEarlier(earlier.index.successor(), self.count) } - addedMessages += self.messages - addedMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) - var i = addedMessages.count - 1 + addedEntries += self.entries + addedEntries.sortInPlace({ $0.index < $1.index }) + var i = addedEntries.count - 1 while i >= 1 { - if addedMessages[i].message.id == addedMessages[i - 1].message.id { - addedMessages.removeAtIndex(i) + if addedEntries[i].index.id == addedEntries[i - 1].index.id { + addedEntries.removeAtIndex(i) } i-- } - self.messages = [] + self.entries = [] - var anchorIndex = addedMessages.count - 1 + var anchorIndex = addedEntries.count - 1 if let latestAnchor = latestAnchor { - var i = addedMessages.count - 1 + var i = addedEntries.count - 1 while i >= 0 { - if MessageIndex(addedMessages[i].message) <= latestAnchor { + if addedEntries[i].index <= latestAnchor { anchorIndex = i break } @@ -170,172 +225,92 @@ public final class MutableMessageHistoryView: CustomStringConvertible { } } - self.laterMessage = nil - if anchorIndex + 1 < addedMessages.count { - self.laterMessage = addedMessages[anchorIndex + 1] + self.later = nil + if anchorIndex + 1 < addedEntries.count { + self.later = addedEntries[anchorIndex + 1] } i = anchorIndex while i >= 0 && i > anchorIndex - self.count { - self.messages.insert(addedMessages[i], atIndex: 0) + self.entries.insert(addedEntries[i], atIndex: 0) i-- } - self.earlierMessage = nil + self.earlier = nil if anchorIndex - self.count >= 0 { - self.earlierMessage = addedMessages[anchorIndex - self.count] + self.earlier = addedEntries[anchorIndex - self.count] } - } - else { + } else { if context.invalidEarlier { var earlyId: MessageIndex? let i = 0 - if i < self.messages.count { - earlyId = MessageIndex(self.messages[i].message) + if i < self.entries.count { + earlyId = self.entries[i].index } - let earlierMessages = fetchEarlier(earlyId, 1) - self.earlierMessage = earlierMessages.first + let earlierEntries = fetchEarlier(earlyId, 1) + self.earlier = earlierEntries.first } if context.invalidLater { var laterId: MessageIndex? - let i = self.messages.count - 1 + let i = self.entries.count - 1 if i >= 0 { - laterId = MessageIndex(self.messages[i].message) + laterId = self.entries[i].index } - let laterMessages = fetchLater(laterId, 1) - self.laterMessage = laterMessages.first + let laterEntries = fetchLater(laterId, 1) + self.later = laterEntries.first } } } - - public func incompleteMessages() -> [Message] { - var result: [Message] = [] - - if let earlierMessage = self.earlierMessage where earlierMessage.incomplete { - result.append(earlierMessage.message) + func render(renderIntermediateMessage: IntermediateMessage -> Message) { + if let earlier = self.earlier, case let .IntermediateMessageEntry(intermediateMessage) = earlier { + self.earlier = .MessageEntry(renderIntermediateMessage(intermediateMessage)) } - - if let laterMessage = self.laterMessage where laterMessage.incomplete { - result.append(laterMessage.message) + if let later = self.later, case let .IntermediateMessageEntry(intermediateMessage) = later { + self.later = .MessageEntry(renderIntermediateMessage(intermediateMessage)) } - for message in self.messages { - if message.incomplete { - result.append(message.message) + for i in 0 ..< self.entries.count { + if case let .IntermediateMessageEntry(intermediateMessage) = self.entries[i] { + self.entries[i] = .MessageEntry(renderIntermediateMessage(intermediateMessage)) } } - - return result - } - - public func completeMessages(messages: [MessageId : RenderedMessage]) { - if let earlierMessage = self.earlierMessage { - if let renderedMessage = messages[earlierMessage.message.id] { - self.earlierMessage = renderedMessage - } - } - - if let laterMessage = self.laterMessage { - if let renderedMessage = messages[laterMessage.message.id] { - self.laterMessage = renderedMessage - } - } - - var i = 0 - while i < self.messages.count { - if let message = messages[self.messages[i].message.id] { - self.messages[i] = message - } - i++ - } } - public var description: String { - var string = "" - string += "...(" - if let value = self.earlierMessage { - string += "\(value.message.id.namespace): \(value.message.id.id)—\(value.message.timestamp)" - } - string += ") —— " - - string += "[" - var first = true - for message in self.messages { - if first { - first = false - } else { - string += ", " + func firstHole() -> MessageHistoryHole? { + for entry in self.entries { + if case let .HoleEntry(hole) = entry { + return hole } - string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" } - string += "]" - string += " —— (" - if let value = self.laterMessage { - string += "\(value.message.id.namespace): \(value.message.id.id)—\(value.message.timestamp)" - } - string += ")..." - - return string + return nil } } -public final class MessageHistoryView: CustomStringConvertible { - public let hasEarlier: Bool - private let earlierId: MessageIndex? - public let hasLater: Bool - private let laterId: MessageIndex? - public let messages: [RenderedMessage] +public final class MessageHistoryView { + public let earlierId: MessageIndex? + public let laterId: MessageIndex? + public let entries: [MessageHistoryEntry] init(_ mutableView: MutableMessageHistoryView) { - self.hasEarlier = mutableView.earlierMessage != nil - self.hasLater = mutableView.laterMessage != nil - self.messages = mutableView.messages + var entries: [MessageHistoryEntry] = [] + for entry in mutableView.entries { + switch entry { + case let .HoleEntry(hole): + entries.append(.HoleEntry(hole)) + case let .MessageEntry(message): + entries.append(.MessageEntry(message)) + case .IntermediateMessageEntry: + assertionFailure("got IntermediateMessageEntry") + } + } + self.entries = entries - if let earlierMessage = mutableView.earlierMessage { - self.earlierId = MessageIndex(earlierMessage.message) - } else { - self.earlierId = nil - } - - if let laterMessage = mutableView.laterMessage { - self.laterId = MessageIndex(laterMessage.message) - } else { - self.laterId = nil - } - } - - public var description: String { - var string = "" - if self.hasEarlier { - string += "more(" - if let earlierId = self.earlierId { - string += "\(earlierId.id.namespace): \(earlierId.id.id)—\(earlierId.timestamp)" - } - string += ") " - } - string += "[" - var first = true - for message in self.messages { - if first { - first = false - } else { - string += ", " - } - string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" - } - string += "]" - if self.hasLater { - string += " more(" - if let laterId = self.laterId { - string += "\(laterId.id.namespace): \(laterId.id.id)—\(laterId.timestamp)" - } - string += ")" - } - return string + self.earlierId = mutableView.earlier?.index + self.laterId = mutableView.later?.index } } diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift new file mode 100644 index 0000000000..5dfb6f4e0f --- /dev/null +++ b/Postbox/MetadataTable.swift @@ -0,0 +1,53 @@ +import Foundation + +private enum MetadataKey: Int32 { + case UserVersion = 1 + case State = 2 +} + +final class MetadataTable { + let valueBox: ValueBox + let tableId: Int32 + + init(valueBox: ValueBox, tableId: Int32) { + self.valueBox = valueBox + self.tableId = tableId + } + + private func key(key: MetadataKey) -> ValueBoxKey { + let valueBoxKey = ValueBoxKey(length: 4) + valueBoxKey.setInt32(0, value: key.rawValue) + return valueBoxKey + } + + func userVersion() -> Int32? { + if let value = self.valueBox.get(self.tableId, key: self.key(.UserVersion)) { + var version: Int32 = 0 + value.read(&version, offset: 0, length: 4) + return version + } + return nil + } + + func setUserVersion(version: Int32) { + let buffer = WriteBuffer() + var varVersion: Int32 = version + buffer.write(&varVersion, offset: 0, length: 4) + self.valueBox.set(self.tableId, key: self.key(.UserVersion), value: buffer) + } + + func state() -> Coding? { + if let value = self.valueBox.get(self.tableId, key: self.key(.State)) { + if let state = Decoder(buffer: value).decodeRootObject() { + return state + } + } + return nil + } + + func setState(state: Coding) { + let encoder = Encoder() + encoder.encodeRootObject(state) + self.valueBox.set(self.tableId, key: self.key(.State), value: encoder.readBufferNoCopy()) + } +} \ No newline at end of file diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index 04bf16f2d9..f9d741f6f2 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -95,5 +95,5 @@ public func <(lhs: PeerId, rhs: PeerId) -> Bool { public protocol Peer: Coding { var id: PeerId { get } - func equalsTo(other: Peer) -> Bool + func isEqual(other: Peer) -> Bool } diff --git a/Postbox/PeerTable.swift b/Postbox/PeerTable.swift new file mode 100644 index 0000000000..6f3aee5056 --- /dev/null +++ b/Postbox/PeerTable.swift @@ -0,0 +1,40 @@ +import Foundation + +final class PeerTable { + let valueBox: ValueBox + let tableId: Int32 + + private let sharedEncoder = Encoder() + private let sharedKey = ValueBoxKey(length: 8) + private var cachedPeers: [PeerId: Peer] = [:] + + init(valueBox: ValueBox, tableId: Int32) { + self.valueBox = valueBox + self.tableId = tableId + } + + private func key(id: PeerId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: id.toInt64()) + return self.sharedKey + } + + func set(peer: Peer) { + self.sharedEncoder.reset() + self.sharedEncoder.encodeRootObject(peer) + + self.valueBox.set(self.tableId, key: self.key(peer.id), value: self.sharedEncoder.readBufferNoCopy()) + } + + func get(id: PeerId) -> Peer? { + if let peer = self.cachedPeers[id] { + return peer + } + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let peer = Decoder(buffer: value).decodeRootObject() as? Peer { + self.cachedPeers[id] = peer + return peer + } + } + return nil + } +} \ No newline at end of file diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift deleted file mode 100644 index 74b96c743f..0000000000 --- a/Postbox/PeerView.swift +++ /dev/null @@ -1,454 +0,0 @@ -import Foundation - -public final class PeerViewEntry: Equatable, Comparable { - public let peerId: PeerId - public let peer: Peer? - public let messageIndex: MessageIndex - public let message: RenderedMessage? - - public init(peer: Peer, message: RenderedMessage) { - self.peerId = peer.id - self.peer = peer - self.message = message - self.messageIndex = MessageIndex(message.message) - } - - public init(peerId: PeerId, message: RenderedMessage) { - self.peerId = peerId - self.peer = nil - self.message = message - self.messageIndex = MessageIndex(message.message) - } - - private init(peer: Peer?, peerId: PeerId, message: RenderedMessage) { - self.peer = peer - self.peerId = peerId - self.message = message - self.messageIndex = MessageIndex(message.message) - } - - private init(peer: Peer?, peerId: PeerId, message: RenderedMessage?, messageIndex: MessageIndex) { - self.peer = peer - self.peerId = peerId - self.message = message - self.messageIndex = messageIndex - } - - init(peer: Peer?, peerId: PeerId, messageIndex: MessageIndex) { - self.peer = peer - self.peerId = peerId - self.messageIndex = messageIndex - self.message = nil - } -} - -public func ==(lhs: PeerViewEntry, rhs: PeerViewEntry) -> Bool { - return PeerViewEntryIndex(lhs) == PeerViewEntryIndex(rhs) -} - -public func <(lhs: PeerViewEntry, rhs: PeerViewEntry) -> Bool { - return PeerViewEntryIndex(lhs) < PeerViewEntryIndex(rhs) -} - -public struct PeerViewEntryIndex: Equatable, Comparable { - public let peerId: PeerId - public let messageIndex: MessageIndex - - public init(_ entry: PeerViewEntry) { - self.peerId = entry.peerId - self.messageIndex = entry.messageIndex - } - - public init(peerId: PeerId, messageIndex: MessageIndex) { - self.peerId = peerId - self.messageIndex = messageIndex - } - - public func earlier() -> PeerViewEntryIndex { - return PeerViewEntryIndex(peerId: self.peerId, messageIndex: MessageIndex(id: MessageId(peerId: self.messageIndex.id.peerId, namespace: self.messageIndex.id.namespace, id: self.messageIndex.id.id - 1), timestamp: self.messageIndex.timestamp)) - } - - public func later() -> PeerViewEntryIndex { - return PeerViewEntryIndex(peerId: self.peerId, messageIndex: MessageIndex(id: MessageId(peerId: self.messageIndex.id.peerId, namespace: self.messageIndex.id.namespace, id: self.messageIndex.id.id + 1), timestamp: self.messageIndex.timestamp)) - } -} - -public func ==(lhs: PeerViewEntryIndex, rhs: PeerViewEntryIndex) -> Bool { - return lhs.peerId == rhs.peerId && lhs.messageIndex == rhs.messageIndex -} - -public func <(lhs: PeerViewEntryIndex, rhs: PeerViewEntryIndex) -> Bool { - if lhs.messageIndex != rhs.messageIndex { - return lhs.messageIndex < rhs.messageIndex - } - - return lhs.peerId < rhs.peerId -} - -public final class MutablePeerView: CustomStringConvertible { - public struct RemoveContext { - var invalidEarlier = false - var invalidLater = false - var removedEntries = false - } - - let count: Int - var earlier: PeerViewEntry? - var later: PeerViewEntry? - var entries: [PeerViewEntry] - - public init(count: Int, earlier: PeerViewEntry?, entries: [PeerViewEntry], later: PeerViewEntry?) { - self.count = count - self.earlier = earlier - self.entries = entries - self.later = later - } - - public func removeEntry(context: RemoveContext?, peerId: PeerId) -> RemoveContext { - var invalidationContext = context ?? RemoveContext() - - if let earlier = self.earlier { - if peerId == earlier.peerId { - invalidationContext.invalidEarlier = true - } - } - - if let later = self.later { - if peerId == later.peerId { - invalidationContext.invalidLater = true - } - } - - var i = 0 - while i < self.entries.count { - if self.entries[i].peerId == peerId { - self.entries.removeAtIndex(i) - invalidationContext.removedEntries = true - break - } - i++ - } - - return invalidationContext - } - - public func addEntry(entry: PeerViewEntry) { - if self.entries.count == 0 { - self.entries.append(entry) - } else { - let first = PeerViewEntryIndex(self.entries[self.entries.count - 1]) - let last = PeerViewEntryIndex(self.entries[0]) - - let index = PeerViewEntryIndex(entry) - - var next: PeerViewEntryIndex? - if let later = self.later { - next = PeerViewEntryIndex(later) - } - - if index < last { - let earlierEntry = self.earlier - if earlierEntry == nil || PeerViewEntryIndex(earlierEntry!) < index { - if self.entries.count < self.count { - self.entries.insert(entry, atIndex: 0) - } else { - self.earlier = entry - } - } - } else if index > first { - if next != nil && index > next! { - let laterEntry = self.later - if laterEntry == nil || PeerViewEntryIndex(laterEntry!) > index { - if self.entries.count < self.count { - self.entries.append(entry) - } else { - self.later = entry - } - } - } else { - self.entries.append(entry) - if self.entries.count > self.count { - let earliest = self.entries[0] - self.earlier = earliest - self.entries.removeAtIndex(0) - } - } - } else if index != last && index != first { - var i = self.entries.count - while i >= 1 { - if PeerViewEntryIndex(self.entries[i - 1]) < index { - break - } - i-- - } - self.entries.insert(entry, atIndex: i) - if self.entries.count > self.count { - let earliest = self.entries[0] - self.earlier = earliest - self.entries.removeAtIndex(0) - } - } - } - } - - public func complete(context: RemoveContext, fetchEarlier: (PeerViewEntryIndex?, Int) -> [PeerViewEntry], fetchLater: (PeerViewEntryIndex?, Int) -> [PeerViewEntry]) { - if context.removedEntries && self.entries.count != self.count { - var addedEntries: [PeerViewEntry] = [] - - var latestAnchor: PeerViewEntryIndex? - - if self.entries.count != 0 { - latestAnchor = PeerViewEntryIndex(self.entries[self.entries.count - 1]) - } else if let later = self.later { - latestAnchor = PeerViewEntryIndex(later) - } - - if let later = self.later { - addedEntries += fetchLater(PeerViewEntryIndex(later).earlier(), self.count) - } - if let earlier = self.earlier { - addedEntries += fetchEarlier(PeerViewEntryIndex(earlier).later(), self.count) - } - - addedEntries += self.entries - addedEntries.sortInPlace({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) - - var i = addedEntries.count - 1 - while i >= 1 { - if PeerViewEntryIndex(addedEntries[i]) == PeerViewEntryIndex(addedEntries[i - 1]) { - addedEntries.removeAtIndex(i) - } - i-- - } - self.entries = [] - - var anchorIndex = addedEntries.count - 1 - if let latestAnchor = latestAnchor { - var i = addedEntries.count - 1 - while i >= 0 { - if PeerViewEntryIndex(addedEntries[i]) <= latestAnchor { - anchorIndex = i - break - } - i-- - } - } - - self.later = nil - if anchorIndex + 1 < addedEntries.count { - let i = anchorIndex + 1 - while i < addedEntries.count { - self.later = addedEntries[i] - break - } - } - - i = anchorIndex - while i >= 0 && i > anchorIndex - self.count { - self.entries.insert(addedEntries[i], atIndex: 0) - i-- - } - - self.earlier = nil - if anchorIndex - self.count >= 0 { - i = anchorIndex - self.count - while i >= 0 { - self.earlier = addedEntries[i] - break - } - } - } else { - var earlyIndex: PeerViewEntryIndex? - if self.entries.count != 0 { - earlyIndex = PeerViewEntryIndex(self.entries[0]) - } - - let earlierEntries = fetchEarlier(earlyIndex, 1) - if earlierEntries.count == 0 { - self.earlier = nil - } else { - self.earlier = earlierEntries[0] - } - - var lateIndex: PeerViewEntryIndex? - if self.entries.count != 0 { - lateIndex = PeerViewEntryIndex(self.entries[self.entries.count - 1]) - } - - let laterEntries = fetchLater(lateIndex, 1) - if laterEntries.count == 0 { - self.later = nil - } else { - self.later = laterEntries[0] - } - } - } - - public func updatePeers(peers: [PeerId : Peer]) -> Bool { - var updated = false - - if let earlier = self.earlier { - if let peer = peers[earlier.peerId] { - self.earlier = PeerViewEntry(peer: peer, peerId: earlier.peerId, message: earlier.message, messageIndex: earlier.messageIndex) - updated = true - } - } - - if let later = self.later { - if let peer = peers[later.peerId] { - self.later = PeerViewEntry(peer: peer, peerId: later.peerId, message: later.message, messageIndex: later.messageIndex) - updated = true - } - } - - var i = 0 - while i < self.entries.count { - if let peer = peers[self.entries[i].peerId] { - let entry = self.entries[i] - self.entries[i] = PeerViewEntry(peer: peer, peerId: entry.peerId, message: entry.message, messageIndex: entry.messageIndex) - updated = true - } - i++ - } - - return updated - } - - public func incompleteMessages() -> [Message] { - var result: [Message] = [] - - if let earlier = self.earlier { - if let message = earlier.message { - if message.incomplete { - result.append(message.message) - } - } - } - if let later = self.later { - if let message = later.message { - if message.incomplete { - result.append(message.message) - } - } - } - - for entry in self.entries { - if let message = entry.message { - if message.incomplete { - result.append(message.message) - } - } - } - - return result - } - - public func completeMessages(messages: [MessageId : RenderedMessage]) { - if let earlier = self.earlier { - if let earlierMessage = earlier.message { - if let message = messages[earlierMessage.message.id] { - self.earlier = PeerViewEntry(peer: earlier.peer, peerId: earlier.peerId, message: message) - } - } - } - if let later = self.later { - if let laterMessage = later.message { - if let message = messages[laterMessage.message.id] { - self.later = PeerViewEntry(peer: later.peer, peerId: later.peerId, message: message) - } - } - } - - var i = 0 - while i < self.entries.count { - if let entryMessage = self.entries[i].message { - if let message = messages[entryMessage.message.id] { - self.entries[i] = PeerViewEntry(peer: self.entries[i].peer, peerId: self.entries[i].peerId, message: message) - } - } - i++ - } - } - - public var description: String { - var string = "" - - if let earlier = self.earlier { - string += "more(" - string += "(p \(earlier.peerId.namespace):\(earlier.peerId.id), m \(earlier.message?.message.id.namespace ?? 0):\(earlier.message?.message.id.id ?? 0)—\(earlier.message?.message.timestamp ?? 0)" - string += ") " - } - - string += "[" - var first = true - for entry in self.entries { - if first { - first = false - } else { - string += ", " - } - string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message?.message.id.namespace ?? 0):\(entry.message?.message.id.id ?? 0)—\(entry.message?.message.timestamp ?? 0))" - } - string += "]" - - if let later = self.later { - string += " more(" - string += "(p \(later.peerId.namespace):\(later.peerId), m \(later.message?.message.id.namespace ?? 0):\(later.message?.message.id.id ?? 0)—\(later.message?.message.timestamp ?? 0)" - string += ")" - } - - return string - } -} - -public final class PeerView: CustomStringConvertible { - let earlier: PeerViewEntryIndex? - let later: PeerViewEntryIndex? - public let entries: [PeerViewEntry] - - init(_ mutableView: MutablePeerView) { - if let earlier = mutableView.earlier { - self.earlier = PeerViewEntryIndex(earlier) - } else { - self.earlier = nil - } - - if let later = mutableView.later { - self.later = PeerViewEntryIndex(later) - } else { - self.later = nil - } - - self.entries = mutableView.entries - } - - public var description: String { - var string = "" - - if let earlier = self.earlier { - string += "more(" - string += "(p \(earlier.peerId.namespace):\(earlier.peerId.id), m \(earlier.messageIndex.id.namespace):\(earlier.messageIndex.id.id)—\(earlier.messageIndex.timestamp)" - string += ") " - } - - string += "[" - var first = true - for entry in self.entries { - if first { - first = false - } else { - string += ", " - } - string += "(p \(entry.peerId.namespace):\(entry.peerId.id), m \(entry.message?.message.id.namespace ?? 0):\(entry.message?.message.id.id ?? 0)—\(entry.message?.message.timestamp ?? 0))" - } - string += "]" - - if let later = self.later { - string += " more(" - string += "(p \(later.peerId.namespace):\(later.peerId), m \(later.messageIndex.id.namespace):\(later.messageIndex.id.id)—\(later.messageIndex.timestamp)" - string += ")" - } - - return string - } -} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 9ddd12ec63..f2dea6d270 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -2,75 +2,91 @@ import Foundation import SwiftSignalKit import sqlcipher -public protocol PostboxState: Coding { +public final class Modifier { + private weak var postbox: Postbox? -} - -public final class Modifier { - private weak var postbox: Postbox? - - private init(postbox: Postbox) { + private init(postbox: Postbox) { self.postbox = postbox } - public func addMessages(messages: [Message], medias: [Media]) { - self.postbox?.addMessages(messages, medias: medias) + public func addMessages(messages: [StoreMessage], location: AddMessagesLocation) { + self.postbox?.addMessages(messages, location: location) } - public func deleteMessagesWithIds(ids: [MessageId]) { - self.postbox?.deleteMessagesWithIds(ids) + public func initializeHole(peerId: PeerId, namespace: MessageId.Namespace) { + self.postbox?.addHole(MessageId(peerId: peerId, namespace: namespace, id: 1)) } - public func deleteMessagesWithAbsoluteIndexedIds(ids: [Int32]) { + public func fillHole(hole: MessageHistoryHole, fillType: HoleFillType, messages: [StoreMessage]) { + self.postbox?.fillHole(hole, fillType: fillType, messages: messages) + } + + public func deleteMessages(messageIds: [MessageId]) { + self.postbox?.deleteMessages(messageIds) + } + + public func deleteMessagesWithGlobalIds(ids: [Int32]) { if let postbox = self.postbox { - let messageIds = postbox.messageIdsForAbsoluteIndexedIds(ids) - postbox.deleteMessagesWithIds(messageIds) + let messageIds = postbox.messageIdsForGlobalIds(ids) + postbox.deleteMessages(messageIds) } } - public func getState() -> State? { + public func getState() -> Coding? { return self.postbox?.getState() } - public func setState(state: State) { + public func setState(state: Coding) { self.postbox?.setState(state) } - public func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer) { + public func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer?) { self.postbox?.updatePeers(peers, update: update) } - public func peersWithIds(ids: [PeerId]) -> [PeerId : Peer] { - return self.postbox?.peersWithIds(ids) ?? [:] + public func knownPeerIds(ids: Set) -> Set { + return self.postbox?.knownPeerIds(ids) ?? Set() } } -public final class Postbox { +public final class Postbox { private let basePath: String - private let messageNamespaces: [MessageId.Namespace] - private let absoluteIndexedMessageNamespace: MessageId.Namespace + private let globalMessageIdsNamespace: MessageId.Namespace private let queue = Queue(name: "org.telegram.postbox.Postbox") private var valueBox: ValueBox! - private var peerMessageHistoryViews: [PeerId : Bag<(MutableMessageHistoryView, Pipe)>] = [:] - private var deferredMessageHistoryViewsToUpdate: [(MutableMessageHistoryView, Pipe)] = [] - private var peerViews: Bag<(MutablePeerView, Pipe)> = Bag() - private var deferredPeerViewsToUpdate: [(MutablePeerView, Pipe)] = [] - private var peerPipes: [PeerId : Pipe] = [:] + private var viewTracker: ViewTracker! - private var statePipe: Pipe = Pipe() + private var peerPipes: [PeerId: Pipe] = [:] + + private var currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + private var currentFilledHolesByPeerId = Set() + private var currentUpdatedPeers: [PeerId: Peer] = [:] + + private var statePipe: Pipe = Pipe() + + private let fetchMessageHistoryHoleImpl = Promise Signal>() + public func setFetchMessageHistoryHole(fetch: MessageHistoryHole -> Signal) { + self.fetchMessageHistoryHoleImpl.set(single(fetch, NoError.self)) + } public let mediaBox: MediaBox - public init(basePath: String, messageNamespaces: [MessageId.Namespace], absoluteIndexedMessageNamespace: MessageId.Namespace?) { + var metadataTable: MetadataTable! + var keychainTable: KeychainTable! + var peerTable: PeerTable! + var globalMessageIdsTable: GlobalMessageIdsTable! + var messageHistoryIndexTable: MessageHistoryIndexTable! + var messageHistoryTable: MessageHistoryTable! + var mediaTable: MessageMediaTable! + var mediaCleanupTable: MediaCleanupTable! + var chatListIndexTable: ChatListIndexTable! + var chatListTable: ChatListTable! + + public init(basePath: String, globalMessageIdsNamespace: MessageId.Namespace) { self.basePath = basePath - self.messageNamespaces = messageNamespaces - if let absoluteIndexedMessageNamespace = absoluteIndexedMessageNamespace { - self.absoluteIndexedMessageNamespace = absoluteIndexedMessageNamespace - } else { - self.absoluteIndexedMessageNamespace = MessageId.Namespace.max - } + self.globalMessageIdsNamespace = globalMessageIdsNamespace self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.openDatabase() } @@ -85,58 +101,58 @@ public final class Postbox { } self.valueBox = SqliteValueBox(basePath: self.basePath + "/db") - //self.valueBox = LmdbValueBox(basePath: self.basePath + "/db") - var userVersion: Int32 = 0 - let currentUserVersion: Int32 = 4 + self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) - if let value = self.valueBox.get(Table_Meta.id, key: Table_Meta.key()) { - value.read(&userVersion, offset: 0, length: 4) - } + let userVersion: Int32? = self.metadataTable.userVersion() + let currentUserVersion: Int32 = 5 if userVersion != currentUserVersion { self.valueBox.drop() - let buffer = WriteBuffer() - var currentVersion: Int32 = currentUserVersion - buffer.write(¤tVersion, offset: 0, length: 4) - self.valueBox.set(Table_Meta.id, key: Table_Meta.key(), value: buffer) + self.metadataTable.setUserVersion(currentUserVersion) } + self.keychainTable = KeychainTable(valueBox: self.valueBox, tableId: 1) + self.peerTable = PeerTable(valueBox: self.valueBox, tableId: 2) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox, tableId: 3, namespace: self.globalMessageIdsNamespace) + self.messageHistoryIndexTable = MessageHistoryIndexTable(valueBox: self.valueBox, tableId: 4, globalMessageIdsTable: self.globalMessageIdsTable) + self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox, tableId: 5) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox, tableId: 6, mediaCleanupTable: self.mediaCleanupTable) + self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, tableId: 7, messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable) + self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, tableId: 8) + self.chatListTable = ChatListTable(valueBox: self.valueBox, tableId: 9, indexTable: self.chatListIndexTable) + + self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, renderMessage: self.renderIntermediateMessage, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper) + print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") } } - private var cachedState: State? + private var cachedState: Coding? - private func setState(state: State) { + private func setState(state: Coding) { self.queue.dispatch { self.cachedState = state - let encoder = Encoder() - encoder.encodeRootObject(state) - - self.valueBox.set(Table_State.id, key: Table_State.key(), value: encoder.memoryBuffer()) + self.metadataTable.setState(state) self.statePipe.putNext(state) } } - private func getState() -> State? { + private func getState() -> Coding? { if let cachedState = self.cachedState { return cachedState } else { - if let value = self.valueBox.get(Table_State.id, key: Table_State.key()) { - let decoder = Decoder(buffer: value) - if let state = decoder.decodeRootObject() as? State { - self.cachedState = state - return state - } + if let state = self.metadataTable.state() { + self.cachedState = state + return state } return nil } } - public func state() -> Signal { + public func state() -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.queue.dispatch { @@ -151,290 +167,169 @@ public final class Postbox { } public func keychainEntryForKey(key: String) -> NSData? { - if let value = self.valueBox.get(Table_Keychain.id, key: Table_Keychain.key(key)) { - print("get \(key) -> \(value.length) bytes") - return NSData(bytes: value.memory, length: value.length) - } - print("get \(key) -> nil") - - return nil + return self.keychainTable.get(key) } public func setKeychainEntryForKey(key: String, value: NSData) { - print("set \(key) -> \(value.length) bytes") - self.valueBox.set(Table_Keychain.id, key: Table_Keychain.key(key), value: MemoryBuffer(data: value)) + self.keychainTable.set(key, value: value) } public func removeKeychainEntryForKey(key: String) { - self.valueBox.remove(Table_Keychain.id, key: Table_Keychain.key(key)) + self.keychainTable.remove(key) } - private func addMessages(messages: [Message], medias: [Media]) { - + private func addMessages(messages: [StoreMessage], location: AddMessagesLocation) { + self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId) } - private func mediaWithIds(ids: [MediaId]) -> [MediaId : Media] { - return [:] + private func addHole(id: MessageId) { + self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId) } - var cachedPeers: [PeerId : Peer] = [:] - - private func peerWithId(peerId: PeerId) -> Peer? { - if let cachedPeer = cachedPeers[peerId] { - return cachedPeer - } else { - let peerKey = Table_Peer.emptyKey() - if let value = self.valueBox.get(Table_Peer.id, key: Table_Peer.key(peerId, key: peerKey)) { - let decoder = Decoder(buffer: value) - if let peer = decoder.decodeRootObject() as? Peer { - cachedPeers[peer.id] = peer - return peer - } else { - print("(PostBox: can't decode peer)") - } - } - - return nil - } + private func fillHole(hole: MessageHistoryHole, fillType: HoleFillType, messages: [StoreMessage]) { + self.messageHistoryTable.fillHole(hole.id, fillType: fillType, messages: messages, operationsByPeerId: &self.currentOperationsByPeerId) + self.currentFilledHolesByPeerId.insert(hole.id.peerId) } - private func peersWithIds(ids: [PeerId]) -> [PeerId : Peer] { - if ids.count == 0 { - return [:] - } else { - var peers: [PeerId : Peer] = [:] - - for id in ids { - if let peer: Peer = self.peerWithId(id) { - peers[id] = peer - } - } - - return peers - } + private func deleteMessages(messageIds: [MessageId]) { + self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId) } - private func deferPeerViewUpdate(view: MutablePeerView, pipe: Pipe) { - var i = 0 - var found = false - while i < self.deferredPeerViewsToUpdate.count { - if self.deferredPeerViewsToUpdate[i].1 === pipe { - self.deferredPeerViewsToUpdate[i] = (view, pipe) - found = true - break - } - i++ - } - if !found { - self.deferredPeerViewsToUpdate.append((view, pipe)) - } - } - - private func deferMessageHistoryViewUpdate(view: MutableMessageHistoryView, pipe: Pipe) { - var i = 0 - var found = false - while i < self.deferredMessageHistoryViewsToUpdate.count { - if self.deferredMessageHistoryViewsToUpdate[i].1 === pipe { - self.deferredMessageHistoryViewsToUpdate[i] = (view, pipe) - found = true - break - } - i++ - } - if !found { - self.deferredMessageHistoryViewsToUpdate.append((view, pipe)) - } - } - - private func performDeferredUpdates() { - let deferredPeerViewsToUpdate = self.deferredPeerViewsToUpdate - self.deferredPeerViewsToUpdate.removeAll() + private func knownPeerIds(ids: Set) -> Set { + var result = Set() - for entry in deferredPeerViewsToUpdate { - let viewRenderedMessages = self.renderedMessages(entry.0.incompleteMessages()) - if viewRenderedMessages.count != 0 { - var viewRenderedMessagesDict: [MessageId : RenderedMessage] = [:] - for message in viewRenderedMessages { - viewRenderedMessagesDict[message.message.id] = message - } - entry.0.completeMessages(viewRenderedMessagesDict) - } - - entry.1.putNext(PeerView(entry.0)) - } - - let deferredMessageHistoryViewsToUpdate = self.deferredMessageHistoryViewsToUpdate - self.deferredMessageHistoryViewsToUpdate.removeAll() - - for entry in deferredMessageHistoryViewsToUpdate { - let viewRenderedMessages = self.renderedMessages(entry.0.incompleteMessages()) - if viewRenderedMessages.count != 0 { - var viewRenderedMessagesDict: [MessageId : RenderedMessage] = [:] - for message in viewRenderedMessages { - viewRenderedMessagesDict[message.message.id] = message - } - entry.0.completeMessages(viewRenderedMessagesDict) - } - - entry.1.putNext(MessageHistoryView(entry.0)) - } - } - - private func updatePeerEntry(peerId: PeerId, message: RenderedMessage?, replace: Bool = false) { - var currentIndex: PeerViewEntryIndex? - - if let value = self.valueBox.get(Table_PeerEntry.id, key: Table_PeerEntry.key(peerId)) { - currentIndex = Table_PeerEntry.get(peerId, value: value) - } - - var updatedPeerMessage: RenderedMessage? - - if let currentIndex = currentIndex { - if let message = message { - let messageIndex = MessageIndex(message.message) - if replace || currentIndex.messageIndex < messageIndex { - let updatedIndex = PeerViewEntryIndex(peerId: peerId, messageIndex: messageIndex) - updatedPeerMessage = message - - self.valueBox.remove(Table_PeerEntry_Sorted.id, key: Table_PeerEntry_Sorted.key(currentIndex)) - self.valueBox.set(Table_PeerEntry_Sorted.id, key: Table_PeerEntry_Sorted.key(updatedIndex), value: MemoryBuffer()) - self.valueBox.set(Table_PeerEntry.id, key: Table_PeerEntry.key(peerId), value: Table_PeerEntry.set(updatedIndex)) - } - } else if replace { - //TODO: remove? - } - } else if let message = message { - updatedPeerMessage = message - let updatedIndex = PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(message.message)) - - self.valueBox.set(Table_PeerEntry_Sorted.id, key: Table_PeerEntry_Sorted.key(updatedIndex), value: MemoryBuffer()) - self.valueBox.set(Table_PeerEntry.id, key: Table_PeerEntry.key(peerId), value: Table_PeerEntry.set(updatedIndex)) - } - - if let updatedPeerMessage = updatedPeerMessage { - var peer: Peer? - for (view, pipe) in self.peerViews.copyItems() { - if peer == nil { - for entry in view.entries { - if entry.peerId == peerId { - peer = entry.peer - break - } - } - - if peer == nil { - peer = self.peerWithId(peerId) - } - } - - let entry: PeerViewEntry - if let peer = peer { - entry = PeerViewEntry(peer: peer, message: updatedPeerMessage) - } else { - entry = PeerViewEntry(peerId: peerId, message: updatedPeerMessage) - } - - let context = view.removeEntry(nil, peerId: peerId) - view.addEntry(entry) - view.complete(context, fetchEarlier: self.fetchPeerEntriesRelative(true), fetchLater: self.fetchPeerEntriesRelative(false)) - - self.deferPeerViewUpdate(view, pipe: pipe) - } - } - } - - private func messageIdsForAbsoluteIndexedIds(ids: [Int32]) -> [MessageId] { - if ids.count == 0 { - return [] - } - - var result: [MessageId] = [] - - let key = Table_GlobalMessageId.emptyKey() for id in ids { - if let value = self.valueBox.get(Table_GlobalMessageId.id, key: Table_GlobalMessageId.key(id, key: key)) { - result.append(Table_GlobalMessageId.get(id, value: value)) + if let _ = self.peerTable.get(id) { + result.insert(id) } } return result } - private func deleteMessagesWithIds(ids: [MessageId]) { - + private func fetchEarlierHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [MutableMessageHistoryEntry] { + let intermediateEntries = self.messageHistoryTable.earlierEntries(peerId, index: index, count: count) + var entries: [MutableMessageHistoryEntry] = [] + for entry in intermediateEntries { + switch entry { + case let .Message(message): + entries.append(.IntermediateMessageEntry(message)) + case let .Hole(index): + entries.append(.HoleEntry(index)) + } + } + return entries } - private func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer) { - if peers.count == 0 { - return + private func fetchAroundHistoryEntries(index: MessageIndex, count: Int) -> [MutableMessageHistoryEntry] { + let intermediateEntries = self.messageHistoryTable.entriesAround(index, count: count) + var entries: [MutableMessageHistoryEntry] = [] + for entry in intermediateEntries { + switch entry { + case let .Message(message): + entries.append(.IntermediateMessageEntry(message)) + case let .Hole(index): + entries.append(.HoleEntry(index)) + } } + return entries + } + + private func fetchLaterHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [MutableMessageHistoryEntry] { + let intermediateEntries = self.messageHistoryTable.laterEntries(peerId, index: index, count: count) + var entries: [MutableMessageHistoryEntry] = [] + for entry in intermediateEntries { + switch entry { + case let .Message(message): + entries.append(.IntermediateMessageEntry(message)) + case let .Hole(index): + entries.append(.HoleEntry(index)) + } + } + return entries + } + + private func fetchEarlierChatEntries(index: MessageIndex?, count: Int) -> [MutableChatListEntry] { + let intermediateEntries = self.chatListTable.earlierEntries(index, messageHistoryTable: self.messageHistoryTable, count: count) + var entries: [MutableChatListEntry] = [] + for entry in intermediateEntries { + switch entry { + case let .Message(message): + entries.append(.IntermediateMessageEntry(message)) + case let .Nothing(index): + entries.append(.Nothing(index)) + } + } + return entries + } + + private func fetchLaterChatEntries(index: MessageIndex?, count: Int) -> [MutableChatListEntry] { + let intermediateEntries = self.chatListTable.laterEntries(index, messageHistoryTable: self.messageHistoryTable, count: count) + var entries: [MutableChatListEntry] = [] + for entry in intermediateEntries { + switch entry { + case let .Message(message): + entries.append(.IntermediateMessageEntry(message)) + case let .Nothing(index): + entries.append(.Nothing(index)) + } + } + return entries + } + + private func renderIntermediateMessage(message: IntermediateMessage) -> Message { + return self.messageHistoryTable.renderMessage(message, peerTable: self.peerTable) + } + + private func fetchMessageHistoryHoleWrapper(hole: MessageHistoryHole) -> Disposable { + return (self.fetchMessageHistoryHoleImpl.get() |> mapToSignal { fetch in + return fetch(hole) + }).start() + } + + private func beforeCommit() { + var chatListOperations: [ChatListOperation] = [] + self.chatListTable.replay(self.currentOperationsByPeerId, messageHistoryTable: self.messageHistoryTable, operations: &chatListOperations) - if peers.count == -1 { - - } else { - var peerIds: [PeerId] = [] - for peer in peers { - peerIds.append(peer.id) + self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers) + + self.currentOperationsByPeerId.removeAll() + self.currentFilledHolesByPeerId.removeAll() + self.currentUpdatedPeers.removeAll() + } + + private func messageIdsForGlobalIds(ids: [Int32]) -> [MessageId] { + var result: [MessageId] = [] + for globalId in ids { + if let id = self.globalMessageIdsTable.get(globalId) { + result.append(id) } - - let currentPeers = self.peersWithIds(peerIds) - - let encoder = Encoder() - - var updatedPeers: [PeerId : Peer] = [:] - - let peerKey = Table_Peer.emptyKey() - for updatedPeer in peers { - let currentPeer = currentPeers[updatedPeer.id] - - var finalPeer = updatedPeer - if let currentPeer = currentPeer { - finalPeer = update(currentPeer, updatedPeer) - } - - if currentPeer == nil || !finalPeer.equalsTo(currentPeer!) { - updatedPeers[finalPeer.id] = finalPeer - self.cachedPeers[finalPeer.id] = finalPeer - - encoder.reset() - encoder.encodeRootObject(finalPeer) - - peerKey.setInt64(0, value: updatedPeer.id.toInt64()) - self.valueBox.set(Table_Peer.id, key: Table_Peer.key(finalPeer.id, key: peerKey), value: encoder.memoryBuffer()) - } - } - - for record in self.peerViews.copyItems() { - if record.0.updatePeers(updatedPeers) { - deferPeerViewUpdate(record.0, pipe: record.1) + } + return result + } + + private func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer?) { + for peer in peers { + if let currentPeer = self.peerTable.get(peer.id) { + if let updatedPeer = update(currentPeer, peer) { + self.peerTable.set(updatedPeer) + self.currentUpdatedPeers[updatedPeer.id] = updatedPeer } + } else { + self.peerTable.set(peer) } } } - public func modify(f: Modifier -> T) -> Signal { + public func modify(f: Modifier -> T) -> Signal { return Signal { subscriber in self.queue.dispatch { - //#if DEBUG - //let startTime = CFAbsoluteTimeGetCurrent() - //#endif - //self.valueBox.beginStats() self.valueBox.begin() let result = f(Modifier(postbox: self)) - //print("(Postbox modify took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") - //#if DEBUG - //startTime = CFAbsoluteTimeGetCurrent() - //#endif + self.beforeCommit() self.valueBox.commit() - //#if DEBUG - //print("(Postbox commit took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") - //self.valueBox.endStats() - //#endif - - self.performDeferredUpdates() - subscriber.putNext(result) subscriber.putCompletion() } @@ -442,111 +337,43 @@ public final class Postbox { } } - private func fetchPeerEntryIndicesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntryIndex] { - var entries: [PeerViewEntryIndex] = [] - - let lowerBound = Table_PeerEntry_Sorted.lowerBoundKey() - let upperBound = Table_PeerEntry_Sorted.upperBoundKey() - - let bound: ValueBoxKey - if let index = index { - bound = Table_PeerEntry_Sorted.key(index) - } else if earlier { - bound = upperBound - } else { - bound = lowerBound - } - - let keys: ValueBoxKey -> Bool = { key in - entries.append(Table_PeerEntry_Sorted.get(key)) - - return true - } - - if earlier { - self.valueBox.range(Table_PeerEntry_Sorted.id, start: bound, end: lowerBound, keys: keys, limit: count) - } else { - self.valueBox.range(Table_PeerEntry_Sorted.id, start: bound, end: upperBound, keys: keys, limit: count) - } - - return entries + public func tailMessageHistoryViewForPeerId(peerId: PeerId, count: Int) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + return self.aroundMessageHistoryViewForPeerId(peerId, index: MessageIndex.upperBound(peerId), count: count) } - private func fetchPeerEntriesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntry] { - var entries: [PeerViewEntry] = [] - var peers: [PeerId : Peer] = [:] - for entryIndex in self.fetchPeerEntryIndicesRelative(earlier)(index: index, count: count) { - var peer: Peer? - - if let cachedPeer = peers[entryIndex.peerId] { - peer = cachedPeer - } else { - if let fetchedPeer: Peer = self.peerWithId(entryIndex.peerId) { - peer = fetchedPeer - peers[fetchedPeer.id] = fetchedPeer - } - } - - var message: Message? - if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(entryIndex.messageIndex)) { - message = Table_Message.get(value) - } - - if let message = message, renderedMessage = self.renderedMessages([message]).first { - let entry: PeerViewEntry - if let peer = peer { - entry = PeerViewEntry(peer: peer, message: renderedMessage) - } else { - entry = PeerViewEntry(peerId: entryIndex.peerId, message: renderedMessage) - } - - entries.append(entry) - } else { - let entry: PeerViewEntry - if let peer = peer { - entry = PeerViewEntry(peer: peer, peerId: entryIndex.peerId, messageIndex: entryIndex.messageIndex) - } else { - entry = PeerViewEntry(peer: nil, peerId: entryIndex.peerId, messageIndex: entryIndex.messageIndex) - } - - entries.append(entry) - } - } - - //entries.sortInPlace() - - return entries.sort({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) - } - - private func renderedMessages(messages: [Message]) -> [RenderedMessage] { - return [] - } - - public func tailMessageHistoryViewForPeerId(peerId: PeerId, count: Int) -> Signal { + public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return Signal { subscriber in - let startTime = CFAbsoluteTimeGetCurrent() - let disposable = MetaDisposable() self.queue.dispatch { + let list = self.fetchAroundHistoryEntries(index, count: count + 2) - print("tailMessageHistoryViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") + var entries: [MutableMessageHistoryEntry] = [] + var earlier: MutableMessageHistoryEntry? + var later: MutableMessageHistoryEntry? - let mutableView = MutableMessageHistoryView(count: count, earlierMessage: nil, messages: [], laterMessage: nil) - let record = (mutableView, Pipe()) - - let index: Bag<(MutableMessageHistoryView, Pipe)>.Index - if let bag = self.peerMessageHistoryViews[peerId] { - index = bag.add(record) + if list.count >= count + 2 { + earlier = list[0] + for i in 1 ..< count + 1 { + entries.append(list[i]) + } + later = list[count + 1] + } else if list.count >= count + 1 { + for i in 0 ..< count { + entries.append(list[i]) + } + later = list[count] } else { - let bag = Bag<(MutableMessageHistoryView, Pipe)>() - index = bag.add(record) - self.peerMessageHistoryViews[peerId] = bag + entries = list } - subscriber.putNext(MessageHistoryView(mutableView)) + let mutableView = MutableMessageHistoryView(earlier: earlier, entries: entries, later: later, count: count) + mutableView.render(self.renderIntermediateMessage) + subscriber.putNext((MessageHistoryView(mutableView), .Generic)) - let pipeDisposable = record.1.signal().start(next: { next in + let (index, signal) = self.viewTracker.addMessageHistoryView(peerId, view: mutableView) + + let pipeDisposable = signal.start(next: { next in subscriber.putNext(next) }) @@ -555,9 +382,7 @@ public final class Postbox { if let strongSelf = self { strongSelf.queue.dispatch { - if let bag = strongSelf.peerMessageHistoryViews[peerId] { - bag.remove(index) - } + strongSelf.viewTracker.removeMessageHistoryView(peerId, index: index) } } return @@ -568,88 +393,34 @@ public final class Postbox { } } - public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int) -> Signal { + public func tailChatListView(count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { return Signal { subscriber in let disposable = MetaDisposable() self.queue.dispatch { - let mutableView: MutableMessageHistoryView + let tail = self.fetchEarlierChatEntries(nil, count: count + 1).reverse() + var entries: [MutableChatListEntry] = [] + var earlier: MutableChatListEntry? - let startTime = CFAbsoluteTimeGetCurrent() - - mutableView = MutableMessageHistoryView(count: count, earlierMessage: nil, messages: [], laterMessage: nil) - - print("aroundMessageViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - - let record = (mutableView, Pipe()) - - let index: Bag<(MutableMessageHistoryView, Pipe)>.Index - if let bag = self.peerMessageHistoryViews[peerId] { - index = bag.add(record) - } else { - let bag = Bag<(MutableMessageHistoryView, Pipe)>() - index = bag.add(record) - self.peerMessageHistoryViews[peerId] = bag - } - - subscriber.putNext(MessageHistoryView(mutableView)) - - let pipeDisposable = record.1.signal().start(next: { next in - subscriber.putNext(next) - }) - - disposable.set(ActionDisposable { [weak self] in - pipeDisposable.dispose() - - if let strongSelf = self { - strongSelf.queue.dispatch { - if let bag = strongSelf.peerMessageHistoryViews[peerId] { - bag.remove(index) - } - } + var i = 0 + for entry in tail { + if i < count { + entries.append(entry) + } else if i < count + 1 { + earlier = entry + } else { + break } - return - }) - } - - return disposable - } - } - - public func tailPeerView(count: Int) -> Signal { - return Signal { subscriber in - let disposable = MetaDisposable() - - self.queue.dispatch { - let startTime = CFAbsoluteTimeGetCurrent() - - let tail = self.fetchPeerEntriesRelative(true)(index: nil, count: count + 1) - - print("(Postbox fetchPeerEntriesRelative took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") - - var entries: [PeerViewEntry] = [] - var i = tail.count - 1 - while i >= 0 && i >= tail.count - count { - entries.insert(tail[i], atIndex: 0) - i-- + i++ } - var earlier: PeerViewEntry? + let mutableView = MutableChatListView(earlier: earlier, entries: entries, later: nil, count: count) + mutableView.render(self.renderIntermediateMessage) + subscriber.putNext((ChatListView(mutableView), .Generic)) - i = tail.count - count - 1 - while i >= 0 { - earlier = tail[i] - break - } + let (index, signal) = self.viewTracker.addChatListView(mutableView) - let mutableView = MutablePeerView(count: count, earlier: earlier, entries: entries, later: nil) - let record = (mutableView, Pipe()) - - let index = self.peerViews.add(record) - - subscriber.putNext(PeerView(mutableView)) - - let pipeDisposable = record.1.signal().start(next: { next in + let pipeDisposable = signal.start(next: { next in subscriber.putNext(next) }) @@ -658,7 +429,7 @@ public final class Postbox { if let strongSelf = self { strongSelf.queue.dispatch { - strongSelf.peerViews.remove(index) + strongSelf.viewTracker.removeChatListView(index) } } return @@ -673,7 +444,7 @@ public final class Postbox { return Signal { subscriber in let disposable = MetaDisposable() self.queue.dispatch { - if let peer: Peer = self.peerWithId(id) { + if let peer: Peer = self.peerTable.get(id) { subscriber.putNext(peer) } } diff --git a/Postbox/PostboxCodingUtils.swift b/Postbox/PostboxCodingUtils.swift deleted file mode 100644 index 189cecc38e..0000000000 --- a/Postbox/PostboxCodingUtils.swift +++ /dev/null @@ -1,233 +0,0 @@ -import Foundation - -func peerViewEntryIndexForBuffer(buffer: ReadBuffer) -> PeerViewEntryIndex { - var timestamp: Int32 = 0 - buffer.read(×tamp, offset: 0, length: 4) - timestamp = Int32(bigEndian: timestamp) - - var namespace: Int32 = 0 - buffer.read(&namespace, offset: 0, length: 4) - namespace = Int32(bigEndian: namespace) - - var id: Int32 = 0 - buffer.read(&id, offset: 0, length: 4) - id = Int32(bigEndian: id) - - var peerIdRepresentation: Int64 = 0 - buffer.read(&peerIdRepresentation, offset: 0, length: 8) - peerIdRepresentation = Int64(bigEndian: peerIdRepresentation) - - let peerId = PeerId(peerIdRepresentation) - - return PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(id: MessageId(peerId:peerId, namespace: namespace, id: id), timestamp: timestamp)) -} - -func bufferForPeerViewEntryIndex(index: PeerViewEntryIndex) -> MemoryBuffer { - let buffer = WriteBuffer() - - var timestamp = Int32(bigEndian: index.messageIndex.timestamp) - buffer.write(×tamp, offset: 0, length: 4) - - var namespace = Int32(bigEndian: index.messageIndex.id.namespace) - buffer.write(&namespace, offset: 0, length: 4) - - var id = Int32(bigEndian: index.messageIndex.id.id) - buffer.write(&id, offset: 0, length: 4) - - var peerIdRepresentation = Int64(bigEndian: index.peerId.toInt64()) - buffer.write(&peerIdRepresentation, offset: 0, length: 8) - - return buffer -} - -func messageIdsGroupedByNamespace(ids: [MessageId]) -> [MessageId.Namespace : [MessageId]] { - var grouped: [MessageId.Namespace : [MessageId]] = [:] - - for id in ids { - if grouped[id.namespace] != nil { - grouped[id.namespace]!.append(id) - } else { - grouped[id.namespace] = [id] - } - } - - return grouped -} - -func mediaIdsGroupedByNamespaceFromMediaArray(mediaArray: [Media]) -> [MediaId.Namespace : [MediaId]] { - var grouped: [MediaId.Namespace : [MediaId]] = [:] - var seenMediaIds = Set() - - for media in mediaArray { - if let id = media.id { - if !seenMediaIds.contains(id) { - seenMediaIds.insert(id) - if grouped[id.namespace] != nil { - grouped[id.namespace]!.append(id) - } else { - grouped[id.namespace] = [id] - } - } - } - } - - return grouped -} - -func mediaIdsGroupedByNamespaceFromSet(ids: Set) -> [MediaId.Namespace : [MediaId]] { - var grouped: [MediaId.Namespace : [MediaId]] = [:] - - for id in ids { - if let _ = grouped[id.namespace] { - grouped[id.namespace]!.append(id) - } else { - grouped[id.namespace] = [id] - } - } - - return grouped -} - -func mediaIdsGroupedByNamespaceFromDictionaryKeys(dict: [MediaId : T]) -> [MediaId.Namespace : [MediaId]] { - var grouped: [MediaId.Namespace : [MediaId]] = [:] - - for (id, _) in dict { - if grouped[id.namespace] != nil { - grouped[id.namespace]!.append(id) - } else { - grouped[id.namespace] = [id] - } - } - - return grouped -} - -func messagesGroupedByPeerId(messages: [Message]) -> [(PeerId, [Message])] { - var grouped: [(PeerId, [Message])] = [] - - for message in messages { - var i = 0 - let count = grouped.count - var found = false - while i < count { - if grouped[i].0 == message.id.peerId { - grouped[i].1.append(message) - found = true - break - } - i++ - } - if !found { - grouped.append((message.id.peerId, [message])) - } - } - - return grouped -} - -func messageIdsGroupedByPeerId(messageIds: [MessageId]) -> [PeerId : [MessageId]] { - var grouped: [PeerId : [MessageId]] = [:] - - for id in messageIds { - if grouped[id.peerId] != nil { - grouped[id.peerId]!.append(id) - } else { - grouped[id.peerId] = [id] - } - } - - return grouped -} - -func blobForMediaIds(ids: [MediaId]) -> Blob { - let data = NSMutableData() - var version: Int8 = 1 - data.appendBytes(&version, length: 1) - - var count = Int32(ids.count) - data.appendBytes(&count, length:4) - - for id in ids { - var mNamespace = id.namespace - var mId = id.id - data.appendBytes(&mNamespace, length: 4) - data.appendBytes(&mId, length: 8) - } - - return Blob(data: data) -} - -func mediaIdsForBlob(blob: Blob) -> [MediaId] { - var ids: [MediaId] = [] - - var offset: Int = 0 - var version = 0 - blob.data.getBytes(&version, range: NSMakeRange(offset, 1)) - offset += 1 - - if version == 1 { - var count: Int32 = 0 - blob.data.getBytes(&count, range: NSMakeRange(offset, 4)) - offset += 4 - - var i = 0 - while i < Int(count) { - var mNamespace: Int32 = 0 - var mId: Int64 = 0 - blob.data.getBytes(&mNamespace, range: NSMakeRange(offset, 4)) - blob.data.getBytes(&mId, range: NSMakeRange(offset + 4, 8)) - ids.append(MediaId(namespace: mNamespace, id: mId)) - offset += 12 - i++ - } - } - - return ids -} - -func memoryBufferForMessageIds(ids: [MessageId]) -> MemoryBuffer { - let data = NSMutableData() - var version: Int8 = 1 - data.appendBytes(&version, length: 1) - - var count = Int32(ids.count) - data.appendBytes(&count, length:4) - - for id in ids { - var mPeerNamespace = id.peerId.namespace - var mPeerId = id.peerId.id - var mNamespace = id.namespace - var mId = id.id - data.appendBytes(&mPeerNamespace, length: 4) - data.appendBytes(&mPeerId, length: 4) - data.appendBytes(&mNamespace, length: 4) - data.appendBytes(&mId, length: 4) - } - - return MemoryBuffer(data: data) -} - -func messageIdsForMemoryBuffer(buffer: MemoryBuffer) -> [MessageId] { - var ids: [MessageId] = [] - - let readBuffer = ReadBuffer(memoryBufferNoCopy: buffer) - - var count: Int32 = 0 - readBuffer.read(&count, offset: 0, length: 4) - - var i = 0 - while i < Int(count) { - var mPeerNamespace: Int32 = 0 - var mPeerId: Int32 = 0 - var mNamespace: Int32 = 0 - var mId: Int32 = 0 - readBuffer.read(&mPeerNamespace, offset: 0, length: 4) - readBuffer.read(&mPeerId, offset: 0, length: 4) - readBuffer.read(&mNamespace, offset: 0, length: 4) - readBuffer.read(&mId, offset: 0, length: 4) - ids.append(MessageId(peerId: PeerId(namespace: mPeerNamespace, id: mPeerId), namespace: mNamespace, id: mId)) - i++ - } - - return ids -} diff --git a/Postbox/PostboxTables.swift b/Postbox/PostboxTables.swift deleted file mode 100644 index 8f048b3912..0000000000 --- a/Postbox/PostboxTables.swift +++ /dev/null @@ -1,300 +0,0 @@ -import Foundation - -struct Table_Meta { - static let id: Int32 = 0 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 4) - } - - static func key(key: ValueBoxKey = Table_Meta.emptyKey()) -> ValueBoxKey { - key.setInt32(0, value: 0) - return key - } -} - -struct Table_State { - static let id: Int32 = 1 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 4) - } - - static func key(key: ValueBoxKey = Table_State.emptyKey()) -> ValueBoxKey { - key.setInt32(0, value: 0) - return key - } -} - -struct Table_Keychain { - static let id: Int32 = 2 - - static func key(string: String) -> ValueBoxKey { - let data = string.dataUsingEncoding(NSUTF8StringEncoding) ?? NSData() - let key = ValueBoxKey(length: data.length) - memcpy(key.memory, data.bytes, data.length) - return key - } -} - -struct Table_Message { - static let id: Int32 = 4 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 8 + 4 + 4 + 4) - } - - static func lowerBoundKey(peerId: PeerId) -> ValueBoxKey { - let key = ValueBoxKey(length: 8) - key.setInt64(0, value: peerId.toInt64()) - return key - } - - static func upperBoundKey(peerId: PeerId) -> ValueBoxKey { - let key = ValueBoxKey(length: 8) - key.setInt64(0, value: peerId.toInt64()) - return key.successor - } - - static func key(index: MessageIndex, key: ValueBoxKey = Table_Message.emptyKey()) -> ValueBoxKey { - key.setInt64(0, value: index.id.peerId.toInt64()) - key.setInt32(8, value: index.timestamp) - key.setInt32(8 + 4, value: index.id.namespace) - key.setInt32(8 + 4 + 4, value: index.id.id) - return key - } - - static func set(message: Message, encoder: Encoder = Encoder()) -> MemoryBuffer { - encoder.reset() - return encoder.memoryBuffer() - } - - static func get(value: ReadBuffer) -> Message? { - if let message = Decoder(buffer: value).decodeRootObject() as? Message { - return message - } - return nil - } -} - -struct Table_GlobalMessageId { - static let id: Int32 = 5 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 4) - } - - static func key(id: Int32, key: ValueBoxKey = Table_GlobalMessageId.emptyKey()) -> ValueBoxKey { - key.setInt32(0, value: id) - return key - } - - static func set(messageId: MessageId) -> MemoryBuffer { - let buffer = WriteBuffer() - - var peerId: Int64 = messageId.peerId.toInt64() - buffer.write(&peerId, offset: 0, length: 8) - var id_namespace: Int32 = messageId.namespace - buffer.write(&id_namespace, offset: 0, length: 4) - - return buffer - } - - static func get(id: Int32, value: ReadBuffer) -> MessageId { - let offset = value.offset - - var peerId: Int64 = 0 - var id_namespace: Int32 = 0 - value.read(&peerId, offset: 0, length: 8) - value.read(&id_namespace, offset: 0, length: 4) - value.offset = offset - - return MessageId(peerId: PeerId(peerId), namespace: id_namespace, id: id) - } -} - -struct Table_Media { - static let id: Int32 = 6 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 4 + 8) - } - - static func key(id: MediaId, key: ValueBoxKey = Table_Media.emptyKey()) -> ValueBoxKey { - key.setInt32(0, value: id.namespace) - key.setInt64(4, value: id.id) - return key - } - - static func set(media: Media, encoder: Encoder = Encoder()) -> MemoryBuffer { - encoder.reset() - encoder.encodeRootObject(media) - return encoder.memoryBuffer() - } - - static func get(value: ReadBuffer) -> Media? { - let decoder = Decoder(buffer: value) - if let media = decoder.decodeRootObject() as? Media { - return media - } - - return nil - } -} - -struct Table_Media_MessageIds { - static let id: Int32 = 3 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 4 + 8 + 8 + 4 + 4) - } - - static func key(id: MediaId, messageId: MessageId, key: ValueBoxKey = Table_Media_MessageIds.emptyKey()) -> ValueBoxKey { - key.setInt32(0, value: id.namespace) - key.setInt64(4, value: id.id) - key.setInt64(4 + 8, value: messageId.peerId.toInt64()) - key.setInt32(4 + 8 + 8, value: messageId.namespace) - key.setInt32(4 + 8 + 8 + 4, value: messageId.id) - return key - } - - static func lowerBoundKey(id: MediaId) -> ValueBoxKey { - let key = ValueBoxKey(length: 4 + 8) - key.setInt32(0, value: id.namespace) - key.setInt64(4, value: id.id) - return key - } - - static func upperBoundKey(id: MediaId) -> ValueBoxKey { - let key = ValueBoxKey(length: 4 + 8) - key.setInt32(0, value: id.namespace) - key.setInt64(4, value: id.id) - return key.successor - } - - static func getMessageId(key: ValueBoxKey) -> MessageId { - let peerId = key.getInt64(4 + 8) - let messageId_namespace: Int32 = key.getInt32(4 + 8 + 8) - let messageId_id: Int32 = key.getInt32(4 + 8 + 8 + 4) - return MessageId(peerId: PeerId(peerId), namespace: messageId_namespace, id: messageId_id) - } -} - -struct Table_Peer { - static let id: Int32 = 7 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 4 + 4) - } - - static func key(id: PeerId, key: ValueBoxKey = Table_Peer.emptyKey()) -> ValueBoxKey { - key.setInt32(0, value: id.namespace) - key.setInt32(4, value: id.id) - return key - } -} - -struct Table_PeerEntry_Sorted { - static let id: Int32 = 8 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 4 + 4 + 4 + 8) - } - - static func lowerBoundKey() -> ValueBoxKey { - let key = ValueBoxKey(length: 4) - key.setInt32(0, value: 0) - return key - } - - static func upperBoundKey() -> ValueBoxKey { - let key = ValueBoxKey(length: 4) - key.setInt32(0, value: Int32.max) - return key - } - - static func key(index: PeerViewEntryIndex, key: ValueBoxKey = Table_PeerEntry_Sorted.emptyKey()) -> ValueBoxKey { - key.setInt32(0, value: index.messageIndex.timestamp) - key.setInt32(4, value: index.messageIndex.id.namespace) - key.setInt32(4 + 4, value: index.messageIndex.id.id) - key.setInt64(4 + 4 + 4, value: index.peerId.toInt64()) - return key - } - - static func get(key: ValueBoxKey) -> PeerViewEntryIndex { - let messageIndex_timestamp = key.getInt32(0) - let messageIndex_id_namespace = key.getInt32(4) - let messageIndex_id_id = key.getInt32(4 + 4) - let messageIndex_peerId = key.getInt64(4 + 4 + 4) - return PeerViewEntryIndex(peerId: PeerId(messageIndex_peerId), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(messageIndex_peerId), namespace: messageIndex_id_namespace, id: messageIndex_id_id), timestamp: messageIndex_timestamp)) - } -} - -struct Table_PeerEntry { - static let id: Int32 = 9 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 8) - } - - static func key(peerId: PeerId, key: ValueBoxKey = Table_PeerEntry.emptyKey()) -> ValueBoxKey { - key.setInt64(0, value: peerId.toInt64()) - return key - } - - static func set(index: PeerViewEntryIndex) -> MemoryBuffer { - let buffer = WriteBuffer() - - var messageId_namespace: Int32 = index.messageIndex.id.namespace - var messageId_id: Int32 = index.messageIndex.id.id - var timestamp: Int32 = index.messageIndex.timestamp - buffer.write(&messageId_namespace, offset: 0, length: 4) - buffer.write(&messageId_id, offset: 0, length: 4) - buffer.write(×tamp, offset: 0, length: 4) - - return buffer - } - - static func get(peerId: PeerId, value: ReadBuffer) -> PeerViewEntryIndex { - let offset = value.offset - - var messageId_namespace: Int32 = 0 - var messageId_id: Int32 = 0 - var timestamp: Int32 = 0 - value.read(&messageId_namespace, offset: 0, length: 4) - value.read(&messageId_id, offset: 0, length: 4) - value.read(×tamp, offset: 0, length: 4) - let index = PeerViewEntryIndex(peerId: peerId, messageIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: messageId_namespace, id: messageId_id), timestamp: timestamp)) - value.offset = offset - - return index - } -} - -struct Table_MessageIndex { - static let id: Int32 = 11 - - static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 8 + 4 + 4) - } - - static func key(id: MessageId, key: ValueBoxKey = Table_MessageIndex.emptyKey()) -> ValueBoxKey { - key.setInt64(0, value: id.peerId.toInt64()) - key.setInt32(8, value: id.namespace) - key.setInt32(8 + 4, value: id.id) - return key - } - - static func set(index: MessageIndex) -> MemoryBuffer { - let buffer = WriteBuffer() - var timestamp = index.timestamp - buffer.write(×tamp, offset: 0, length: 4) - return buffer.readBufferNoCopy() - } - - static func get(id: MessageId, value: ReadBuffer) -> MessageIndex { - var timestamp: Int32 = 0 - value.read(×tamp, offset: 0, length: 4) - return MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: id.id), timestamp: timestamp) - } -} diff --git a/Postbox/SimpleDictionary.swift b/Postbox/SimpleDictionary.swift new file mode 100644 index 0000000000..3ab962a23a --- /dev/null +++ b/Postbox/SimpleDictionary.swift @@ -0,0 +1,42 @@ +import Foundation + +public struct SimpleDictionary: SequenceType { + private var items: [(K, V)] = [] + + public subscript(key: K) -> V? { + get { + for (k, value) in self.items { + if k == key { + return value + } + } + return nil + } set(value) { + var index = 0 + for (k, _) in self.items { + if k == key { + if let value = value { + self.items[index] = (k, value) + } else { + self.items.removeAtIndex(index) + } + return + } + index++ + } + if let value = value { + self.items.append((key, value)) + } + } + } + + public func generate() -> AnyGenerator<(K, V)> { + var index = 0 + return anyGenerator { () -> (K, V)? in + if index < self.items.count { + return self.items[index++] + } + return nil + } + } +} \ No newline at end of file diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index e5a954326f..3e1be6dbdf 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -8,6 +8,10 @@ private struct SqlitePreparedStatement { sqlite3_bind_blob(statement, Int32(index), data, Int32(length), nil) } + func bindNull(index: Int) { + sqlite3_bind_null(statement, Int32(index)) + } + func bind(index: Int, number: Int32) { sqlite3_bind_int(statement, Int32(index), number) } @@ -366,7 +370,11 @@ public final class SqliteValueBox: ValueBox { resultStatement.reset() resultStatement.bind(1, data: key.memory, length: key.length) - resultStatement.bind(2, data: value.memory, length: value.length) + if value.memory == nil { + resultStatement.bindNull(2) + } else { + resultStatement.bind(2, data: value.memory, length: value.length) + } return resultStatement } diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift new file mode 100644 index 0000000000..6ffcbfcf9b --- /dev/null +++ b/Postbox/ViewTracker.swift @@ -0,0 +1,199 @@ +import Foundation +import SwiftSignalKit + +public enum ViewUpdateType { + case Generic + case FillHole +} + +final class ViewTracker { + private let queue: Queue + private let fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int) -> [MutableMessageHistoryEntry] + private let fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int) -> [MutableMessageHistoryEntry] + private let fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] + private let fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] + private let renderMessage: IntermediateMessage -> Message + private let fetchMessageHistoryHole: MessageHistoryHole -> Disposable + + private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] + private var chatListViews = Bag<(MutableChatListView, Pipe<(ChatListView, ViewUpdateType)>)>() + + private var holeDisposablesByPeerId: [PeerId: [(MessageHistoryHole, Disposable)]] = [:] + + init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], renderMessage: IntermediateMessage -> Message, fetchMessageHistoryHole: MessageHistoryHole -> Disposable) { + self.queue = queue + self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries + self.fetchLaterHistoryEntries = fetchLaterHistoryEntries + self.fetchEarlierChatEntries = fetchEarlierChatEntries + self.fetchLaterChatEntries = fetchLaterChatEntries + self.renderMessage = renderMessage + self.fetchMessageHistoryHole = fetchMessageHistoryHole + } + + deinit { + for (_, indexToDisposable) in holeDisposablesByPeerId { + for (_, disposable) in indexToDisposable { + disposable.dispose() + } + } + } + + func addMessageHistoryView(peerId: PeerId, view: MutableMessageHistoryView) -> (Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>.Index, Signal<(MessageHistoryView, ViewUpdateType), NoError>) { + let record = (view, Pipe<(MessageHistoryView, ViewUpdateType)>()) + + let index: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>.Index + if let bag = self.messageHistoryViews[peerId] { + index = bag.add(record) + } else { + let bag = Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>() + index = bag.add(record) + self.messageHistoryViews[peerId] = bag + } + + self.updateTrackedHoles(peerId) + + return (index, record.1.signal()) + } + + func removeMessageHistoryView(peerId: PeerId, index: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>.Index) { + if let bag = self.messageHistoryViews[peerId] { + bag.remove(index) + + self.updateTrackedHoles(peerId) + } + } + + func addChatListView(view: MutableChatListView) -> (Bag<(MutableChatListView, Pipe<(ChatListView, ViewUpdateType)>)>.Index, Signal<(ChatListView, ViewUpdateType), NoError>) { + let record = (view, Pipe<(ChatListView, ViewUpdateType)>()) + + let index = self.chatListViews.add(record) + return (index, record.1.signal()) + } + + func removeChatListView(index: Bag<(MutableChatListView, Pipe)>.Index) { + self.chatListViews.remove(index) + } + + func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: Set, chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer]) { + var updateTrackedHolesPeerIds: [PeerId] = [] + + for (peerId, bag) in self.messageHistoryViews { + var updateHoles = false + if let operations = currentOperationsByPeerId[peerId] { + updateHoles = true + for (mutableView, pipe) in bag.copyItems() { + let context = MutableMessageHistoryViewReplayContext() + var updated = false + if mutableView.replay(operations, context: context) { + mutableView.complete(context, fetchEarlier: { index, count in + return self.fetchEarlierHistoryEntries(peerId, index, count) + }, fetchLater: { index, count in + return self.fetchLaterHistoryEntries(peerId, index, count) + }) + updated = true + } + + if mutableView.updatePeers(currentUpdatedPeers) { + updated = true + } + + if updated { + mutableView.render(self.renderMessage) + pipe.putNext((MessageHistoryView(mutableView), peerIdsWithFilledHoles.contains(peerId) ? .FillHole : .Generic)) + } + } + } + + if updateHoles { + updateTrackedHolesPeerIds.append(peerId) + } + } + + if chatListOperations.count != 0 { + for (mutableView, pipe) in self.chatListViews.copyItems() { + let context = MutableChatListViewReplayContext() + if mutableView.replay(chatListOperations, context: context) { + mutableView.complete(context, fetchEarlier: self.fetchEarlierChatEntries, fetchLater: self.fetchLaterChatEntries) + mutableView.render(self.renderMessage) + pipe.putNext((ChatListView(mutableView), .Generic)) + } + } + } + + for peerId in updateTrackedHolesPeerIds { + self.updateTrackedHoles(peerId) + } + } + + func updateTrackedHoles(peerId: PeerId) { + if let bag = self.messageHistoryViews[peerId] { + var disposeHoles: [Disposable] = [] + + var firstHoles: [MessageHistoryHole] = [] + + for (view, _) in bag.copyItems() { + if let hole = view.firstHole() { + var exists = false + for existingHole in firstHoles { + if existingHole == hole { + exists = true + break + } + } + if !exists { + firstHoles.append(hole) + } + } + } + + if let holes = self.holeDisposablesByPeerId[peerId] { + var i = 0 + for (hole, disposable) in holes { + var exists = false + for firstHole in firstHoles { + if hole == firstHole { + exists = true + break + } + } + if !exists { + disposeHoles.append(disposable) + self.holeDisposablesByPeerId[peerId]!.removeAtIndex(i) + } else { + i++ + } + } + } + + for disposable in disposeHoles { + disposable.dispose() + } + + for hole in firstHoles { + var exists = false + if let existingHoles = self.holeDisposablesByPeerId[hole.id.peerId] { + for (existingHole, _) in existingHoles { + if existingHole == hole { + exists = true + break + } + } + } + + if !exists { + if self.holeDisposablesByPeerId[hole.id.peerId] == nil { + self.holeDisposablesByPeerId[hole.id.peerId] = [] + } + + self.holeDisposablesByPeerId[hole.id.peerId]!.append((hole, self.fetchMessageHistoryHole(hole))) + } + } + } else if let holes = self.holeDisposablesByPeerId[peerId] { + self.holeDisposablesByPeerId[peerId]?.removeAll() + + for (_, disposable) in holes { + disposable.dispose() + } + } + } +} diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift new file mode 100644 index 0000000000..f8a7e39f76 --- /dev/null +++ b/PostboxTests/ChatListTableTests.swift @@ -0,0 +1,185 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +private let namespace: Int32 = 1 + +private let authorPeerId = PeerId(namespace: 2, id: 3) + +private enum Entry: Equatable, CustomStringConvertible { + case Message(Int32, Int32, Int32) + case Nothing(Int32, Int32, Int32) + + var description: String { + switch self { + case let .Message(peerId, id, timestamp): + return "Message(\(peerId), \(id), \(timestamp))" + case let .Nothing(peerId, id, timestamp): + return "Nothing(\(peerId), \(id), \(timestamp))" + } + } +} + +private func ==(lhs: Entry, rhs: Entry) -> Bool { + switch lhs { + case let .Message(lhsPeerId, lhsId, lhsTimestamp): + switch rhs { + case let .Message(rhsPeerId, rhsId, rhsTimestamp): + return lhsPeerId == rhsPeerId && lhsId == rhsId && lhsTimestamp == rhsTimestamp + case .Nothing: + return false + } + case let .Nothing(lhsPeerId, lhsId, lhsTimestamp): + switch rhs { + case .Message: + return false + case let .Nothing(rhsPeerId, rhsId, rhsTimestamp): + return lhsPeerId == rhsPeerId && lhsId == rhsId && lhsTimestamp == rhsTimestamp + } + } +} + +class ChatListTableTests: XCTestCase { + var valueBox: ValueBox? + var path: String? + + var globalMessageIdsTable: GlobalMessageIdsTable? + var indexTable: MessageHistoryIndexTable? + var mediaTable: MessageMediaTable? + var mediaCleanupTable: MediaCleanupTable? + var historyTable: MessageHistoryTable? + var chatListIndexTable: ChatListIndexTable? + var chatListTable: ChatListTable? + + override class func setUp() { + super.setUp() + } + + override func setUp() { + super.setUp() + + var randomId: Int64 = 0 + arc4random_buf(&randomId, 8) + path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + self.valueBox = SqliteValueBox(basePath: path!) + + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 7, namespace: namespace) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!) + self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!) + self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, tableId: 5) + self.chatListTable = ChatListTable(valueBox: self.valueBox!, tableId: 6, indexTable: self.chatListIndexTable!) + } + + override func tearDown() { + super.tearDown() + + self.historyTable = nil + self.indexTable = nil + self.mediaTable = nil + self.mediaCleanupTable = nil + self.chatListIndexTable = nil + self.chatListTable = nil + + self.valueBox = nil + let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + self.path = nil + } + + private func addMessage(peerId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = []) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId) + var operations: [ChatListOperation] = [] + self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) + } + + private func addHole(peerId: Int32, _ id: Int32) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.historyTable!.addHoles([MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId) + var operations: [ChatListOperation] = [] + self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) + } + + private func removeMessages(peerId: Int32, _ ids: [Int32]) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.historyTable!.removeMessages(ids.map({ MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId) + var operations: [ChatListOperation] = [] + self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) + } + + private func fillHole(peerId: Int32, _ id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32, String, [Media])]) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.historyTable!.fillHole(MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), fillType: fillType, messages: messages.map({ StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0.0), timestamp: $0.1, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId) + var operations: [ChatListOperation] = [] + self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) + } + + private func expectEntries(entries: [Entry]) { + let actualEntries = self.chatListTable!.debugList(self.historyTable!).map({ entry -> Entry in + switch entry { + case let .Message(message): + if message.authorId != authorPeerId { + XCTFail("Expected authorId \(authorPeerId), actual \(message.authorId)") + } + return .Message(message.id.peerId.id, message.id.id, message.timestamp) + case let .Nothing(index): + return .Nothing(index.id.peerId.id, index.id.id, index.timestamp) + } + }) + if entries != actualEntries { + XCTFail("Expected\n\(entries)\nActual\n\(actualEntries)") + } + } + + func testEmpty() { + expectEntries([]) + } + + func testAddSingleMessage() { + addMessage(1, 100, 100) + expectEntries([.Message(1, 100, 100)]) + } + + func testInsertLaterMessage() { + addMessage(1, 100, 100) + addMessage(1, 200, 200) + expectEntries([.Message(1, 200, 200)]) + } + + func testInsertEarlierMessage() { + addMessage(1, 100, 100) + addMessage(1, 10, 20) + expectEntries([.Message(1, 100, 100)]) + } + + func testInsertTwoChatMessages() { + addMessage(1, 100, 100) + addMessage(2, 10, 20) + expectEntries([.Message(2, 10, 20), .Message(1, 100, 100)]) + } + + func testMoveChatUpper() { + addMessage(1, 100, 100) + addMessage(2, 10, 20) + addMessage(2, 120, 120) + expectEntries([.Message(1, 100, 100), .Message(2, 120, 120)]) + } + + func testRemoveSingleMessage() { + addMessage(1, 100, 100) + removeMessages(1, [100]) + expectEntries([.Nothing(1, 100, 100)]) + } + + func testOverrideNothing() { + addMessage(1, 100, 100) + removeMessages(1, [100]) + addMessage(1, 100, 100) + expectEntries([.Message(1, 100, 100)]) + } +} \ No newline at end of file diff --git a/PostboxTests/CodingTests.swift b/PostboxTests/CodingTests.swift index 00c75ab465..51963433c8 100644 --- a/PostboxTests/CodingTests.swift +++ b/PostboxTests/CodingTests.swift @@ -97,7 +97,7 @@ func ==(lhs: TestKey, rhs: TestKey) -> Bool { return lhs.value == rhs.value } -class EmptyState: PostboxState { +class EmptyState: Coding { required init(decoder: Decoder) { } diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index 9c742da934..ca5b7bb9db 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -58,6 +58,7 @@ class MessageHistoryIndexTableTests: XCTestCase { var path: String? var indexTable: MessageHistoryIndexTable? + var globalMessageIdsTable: GlobalMessageIdsTable? override func setUp() { super.setUp() @@ -67,7 +68,8 @@ class MessageHistoryIndexTableTests: XCTestCase { path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") self.valueBox = SqliteValueBox(basePath: path!) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 2, namespace: namespace) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!) } override func tearDown() { @@ -81,23 +83,41 @@ class MessageHistoryIndexTableTests: XCTestCase { } func addHole(id: Int32) { - self.indexTable!.addHole(MessageId(peerId: peerId, namespace: namespace, id: id)) + var operations: [MessageHistoryIndexOperation] = [] + self.indexTable!.addHole(MessageId(peerId: peerId, namespace: namespace, id: id), operations: &operations) } func addMessage(id: Int32, _ timestamp: Int32) { - self.indexTable!.addMessage(MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp)) + var operations: [MessageHistoryIndexOperation] = [] + self.indexTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, authorId: peerId, text: "", attributes: [], media: [])], location: .Random, operations: &operations) + } + + func addMessagesUpperBlock(messages: [(Int32, Int32)]) { + var operations: [MessageHistoryIndexOperation] = [] + self.indexTable!.addMessages(messages.map { (id, timestamp) in + return StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, authorId: peerId, text: "", attributes: [], media: []) + }, location: .UpperHistoryBlock, operations: &operations) } func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32)]) { - self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, indices: messages.map({MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1)})) + var operations: [MessageHistoryIndexOperation] = [] + let zeroData = WriteBuffer() + var zero: Int32 = 0 + zeroData.write(&zero, offset: 0, length: 4) + + self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, messages: messages.map({ + return StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, authorId: peerId, text: "", attributes: [], media: []) + }), operations: &operations) } func removeMessage(id: Int32) { - self.indexTable!.removeMessage(MessageId(peerId: peerId, namespace: namespace, id: id)) + var operations: [MessageHistoryIndexOperation] = [] + self.indexTable!.removeMessage(MessageId(peerId: peerId, namespace: namespace, id: id), operations: &operations) } private func expect(items: [Item]) { - let actualItems = self.indexTable!.debugList(peerId, namespace: namespace).map { return Item($0) } + let actualList = self.indexTable!.debugList(peerId, namespace: namespace) + let actualItems = actualList.map { return Item($0) } if items != actualItems { XCTFail("Expected\n\(items)\nGot\n\(actualItems)") } @@ -118,6 +138,14 @@ class MessageHistoryIndexTableTests: XCTestCase { expect([.Message(90, 90), .Message(100, 100), .Message(110, 110)]) } + func testAddMessageIgnoreOverwrite() { + addMessage(100, 100) + expect([.Message(100, 100)]) + + addMessage(100, 110) + expect([.Message(100, 100)]) + } + func testAddHoleToEmpty() { addHole(100) expect([.Hole(1, Int32.max, Int32.max)]) @@ -424,4 +452,50 @@ class MessageHistoryIndexTableTests: XCTestCase { expect([.Message(100, 100), .Message(300, 300)]) } + + func testAddMessageCutInitialHole() { + addHole(1) + expect([.Hole(1, Int32.max, Int32.max)]) + addMessagesUpperBlock([(100, 100)]) + expect([.Hole(1, 99, 100), .Message(100, 100)]) + } + + func testAddMessageRemoveInitialHole() { + addHole(1) + expect([.Hole(1, Int32.max, Int32.max)]) + addMessagesUpperBlock([(1, 100)]) + expect([.Message(1, 100)]) + } + + func testAddMessageCutHoleAfterMessage1() { + addMessage(10, 10) + addHole(11) + expect([.Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + addMessagesUpperBlock([(100, 100)]) + expect([.Message(10, 10), .Hole(11, 99, 100), .Message(100, 100)]) + } + + func testAddMessageCutHoleAfterMessage2() { + addMessage(10, 10) + addHole(11) + expect([.Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + addMessagesUpperBlock([(12, 12)]) + expect([.Message(10, 10), .Hole(11, 11, 12), .Message(12, 12)]) + } + + func testAddMessageRemoveHoleAfterMessage() { + addMessage(10, 10) + addHole(11) + expect([.Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + addMessagesUpperBlock([(11, 11)]) + expect([.Message(10, 10), .Message(11, 11)]) + } + + func testAddMessageRemoveHoleIgnoreOverwriteMessage() { + addMessage(10, 10) + addHole(11) + expect([.Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + addMessagesUpperBlock([(10, 11)]) + expect([.Message(10, 10)]) + } } diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index 925f1acb8c..a9f7e3d6f9 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -8,21 +8,58 @@ import Postbox private let peerId = PeerId(namespace: 1, id: 1) private let namespace: Int32 = 1 +private let authorPeerId = PeerId(namespace: 1, id: 6) +private let peer = TestPeer(id: 6, data: "abc") -private func ==(lhs: (Int32, Int32, String, [Media]), rhs: (Int32, Int32, String, [Media])) -> Bool { - if lhs.3.count != rhs.3.count { +private func ==(lhs: [Media], rhs: [Media]) -> Bool { + if lhs.count != rhs.count { return false } - for i in 0 ..< lhs.3.count { - if !lhs.3[i].isEqual(rhs.3[i]) { + + for i in 0 ..< lhs.count { + if !lhs[i].isEqual(rhs[i]) { return false } } - return lhs.0 == rhs.0 && lhs.1 == rhs.1 && lhs.2 == rhs.2 + return true +} + +private enum Entry: Equatable, CustomStringConvertible { + case Message(Int32, Int32, String, [Media]) + case Hole(Int32, Int32, Int32) + + var description: String { + switch self { + case let .Message(id, timestamp, text, media): + return "Message(\(id), \(timestamp), \(text), \(media))" + case let .Hole(min, max, timestamp): + return "Hole(\(min), \(max), \(timestamp))" + } + } +} + +private func ==(lhs: Entry, rhs: Entry) -> Bool { + switch lhs { + case let .Message(lhsId, lhsTimestamp, lhsText, lhsMedia): + switch rhs { + case let .Message(rhsId, rhsTimestamp, rhsText, rhsMedia): + return lhsId == rhsId && lhsTimestamp == rhsTimestamp && lhsText == rhsText && lhsMedia == rhsMedia + case .Hole: + return false + } + case let .Hole(lhsMin, lhsMax, lhsMaxTimestamp): + switch rhs { + case .Message: + return false + case let .Hole(rhsMin, rhsMax, rhsMaxTimestamp): + return lhsMin == rhsMin && lhsMax == rhsMax && lhsMaxTimestamp == rhsMaxTimestamp + } + } } private class TestEmbeddedMedia: Media, CustomStringConvertible { var id: MediaId? { return nil } + var peerIds: [PeerId] = [] let data: String init(data: String) { @@ -51,6 +88,7 @@ private class TestEmbeddedMedia: Media, CustomStringConvertible { private class TestExternalMedia: Media { let id: MediaId? + var peerIds: [PeerId] = [] let data: String init(id: Int64, data: String) { @@ -81,6 +119,38 @@ private class TestExternalMedia: Media { } } +private class TestPeer: Peer { + let id: PeerId + let data: String + + init(id: Int32, data: String) { + self.id = PeerId(namespace: namespace, id: id) + self.data = data + } + + required init(decoder: Decoder) { + self.id = PeerId(namespace: decoder.decodeInt32ForKey("i.n"), id: decoder.decodeInt32ForKey("i.i")) + self.data = decoder.decodeStringForKey("s") + } + + func encode(encoder: Encoder) { + encoder.encodeInt32(self.id.namespace, forKey: "i.n") + encoder.encodeInt32(self.id.id, forKey: "i.i") + encoder.encodeString(self.data, forKey: "s") + } + + func isEqual(other: Peer) -> Bool { + if let other = other as? TestPeer { + return self.id == other.id && self.data == other.data + } + return false + } + + var description: String { + return "TestPeer(\(self.id.id), \(self.data))" + } +} + private enum MediaEntry: Equatable { case Direct(Media, Int) case MessageReference(Int32) @@ -118,6 +188,8 @@ class MessageHistoryTableTests: XCTestCase { var valueBox: ValueBox? var path: String? + var peerTable: PeerTable? + var globalMessageIdsTable: GlobalMessageIdsTable? var indexTable: MessageHistoryIndexTable? var mediaTable: MessageMediaTable? var mediaCleanupTable: MediaCleanupTable? @@ -128,6 +200,7 @@ class MessageHistoryTableTests: XCTestCase { declareEncodable(TestEmbeddedMedia.self, f: {TestEmbeddedMedia(decoder: $0)}) declareEncodable(TestExternalMedia.self, f: {TestExternalMedia(decoder: $0)}) + declareEncodable(TestPeer.self, f: {TestPeer(decoder: $0)}) } override func setUp() { @@ -138,10 +211,13 @@ class MessageHistoryTableTests: XCTestCase { path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") self.valueBox = SqliteValueBox(basePath: path!) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 5, namespace: namespace) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!) self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!) + self.peerTable = PeerTable(valueBox: self.valueBox!, tableId: 6) + self.peerTable!.set(peer) } override func tearDown() { @@ -151,36 +227,53 @@ class MessageHistoryTableTests: XCTestCase { self.indexTable = nil self.mediaTable = nil self.mediaCleanupTable = nil + self.peerTable = nil self.valueBox = nil let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) self.path = nil } - private func addMessage(id: Int32, _ timestamp: Int32, _ text: String, _ media: [Media] = []) { - self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, text: text, attributes: [], media: media)]) + private func addMessage(id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = []) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId) + //print("\(operationsByPeerId[peerId]!)") + } + + private func addHole(id: Int32) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.historyTable!.addHoles([MessageId(peerId: peerId, namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId) } private func removeMessages(ids: [Int32]) { - self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) })) + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId) + //print("\(operationsByPeerId[peerId]!)") } - func expectMessages(messages: [(Int32, Int32, String, [Media])]) { - let actualMessages = self.historyTable!.debugList(peerId).map({ ($0.id.id, $0.timestamp, $0.text, $0.media) }) - var equal = true - if messages.count != actualMessages.count { - equal = false - } else { - for i in 0 ..< messages.count { - if !(messages[i] == actualMessages[i]) { - equal = false - break - } + private func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32, String, [Media])]) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.historyTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId) + } + + private func expectEntries(entries: [Entry]) { + let actualEntries = self.historyTable!.debugList(peerId, peerTable: self.peerTable!).map({ entry -> Entry in + switch entry { + case let .RenderedMessage(message): + if let messagePeer = message.author { + if !peer.isEqual(messagePeer) { + XCTFail("Expected peer \(peer), actual: \(messagePeer)") + } + } else { + XCTFail("Expected peer \(peer), actual: nil") + } + return .Message(message.id.id, message.timestamp, message.text, message.media) + case let .Hole(hole): + return .Hole(hole.min, hole.maxIndex.id.id, hole.maxIndex.timestamp) } - } - - if !equal { - XCTFail("Expected\n\(messages)\nActual\n\(actualMessages)") + }) + if actualEntries != entries { + XCTFail("Expected\n\(entries)\nActual\n\(actualEntries)") } } @@ -214,20 +307,20 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "t100") addMessage(200, 200, "t200") - expectMessages([(100, 100, "t100", []), (200, 200, "t200", [])]) + expectEntries([.Message(100, 100, "t100", []), .Message(200, 200, "t200", [])]) } func testInsertMessageIgnoreOverwrite() { addMessage(100, 100, "t100") addMessage(100, 200, "t200") - expectMessages([(100, 100, "t100", [])]) + expectEntries([.Message(100, 100, "t100", [])]) } func testInsertMessageWithEmbeddedMedia() { addMessage(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")]) - expectMessages([(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")])]) + expectEntries([.Message(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")])]) expectMedia([]) } @@ -235,7 +328,7 @@ class MessageHistoryTableTests: XCTestCase { let media = TestExternalMedia(id: 10, data: "abc1") addMessage(100, 100, "t100", [media]) - expectMessages([(100, 100, "t100", [media])]) + expectEntries([.Message(100, 100, "t100", [media])]) expectMedia([.MessageReference(100)]) } @@ -244,7 +337,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "t100", [media]) addMessage(200, 200, "t200", [media]) - expectMessages([(100, 100, "t100", [media]), (200, 200, "t200", [media])]) + expectEntries([.Message(100, 100, "t100", [media]), .Message(200, 200, "t200", [media])]) expectMedia([.Direct(media, 2)]) } @@ -254,7 +347,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "t100", [media]) addMessage(200, 200, "t200", [media1]) - expectMessages([(100, 100, "t100", [media]), (200, 200, "t200", [media])]) + expectEntries([.Message(100, 100, "t100", [media]), .Message(200, 200, "t200", [media])]) expectMedia([.Direct(media, 2)]) } @@ -263,7 +356,7 @@ class MessageHistoryTableTests: XCTestCase { removeMessages([100]) - expectMessages([]) + expectEntries([]) expectMedia([]) } @@ -272,7 +365,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "t100", [media]) self.removeMessages([100]) - expectMessages([]) + expectEntries([]) expectMedia([]) expectCleanupMedia([media]) } @@ -282,7 +375,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "t100", [media]) removeMessages([100]) - expectMessages([]) + expectEntries([]) expectMedia([]) expectCleanupMedia([media]) } @@ -293,14 +386,321 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "t200", [media]) removeMessages([100]) - expectMessages([(200, 200, "t200", [media])]) + expectEntries([.Message(200, 200, "t200", [media])]) expectMedia([.Direct(media, 1)]) expectCleanupMedia([]) removeMessages([200]) - expectMessages([]) + expectEntries([]) expectMedia([]) expectCleanupMedia([media]) } + + func testAddHoleToEmpty() { + addHole(100) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + } + + func testAddHoleToFullHole() { + addHole(100) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + addHole(110) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + } + + func testAddMessageToFullHole() { + addHole(100) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + addMessage(90, 90, "m90") + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + } + + func testAddMessageDividingUpperHole() { + addHole(100) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + addMessage(90, 90, "m90") + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + addMessage(100, 100, "m100") + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, 99, 100), .Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + } + + func testAddMessageDividingLowerHole() { + addHole(100) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + addMessage(90, 90, "m90") + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + addMessage(80, 80, "m80") + expectEntries([.Hole(1, 79, 80), .Message(80, 80, "m80", []), .Hole(81, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + } + + func testAddMessageOffsettingUpperHole() { + addHole(100) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + + addMessage(90, 90, "m90") + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + addMessage(91, 91, "m91") + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Message(91, 91, "m91", []), .Hole(92, Int32.max, Int32.max)]) + } + + func testAddMessageOffsettingLowerHole() { + addHole(100) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + + addMessage(90, 90, "m90") + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + addMessage(89, 89, "m89") + expectEntries([.Hole(1, 88, 89), .Message(89, 89, "m89", []), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + } + + func testAddMessageOffsettingLeftmostHole() { + addHole(100) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + + addMessage(1, 1, "m1") + + expectEntries([.Message(1, 1, "m1", []), .Hole(2, Int32.max, Int32.max)]) + } + + func testAddMessageRemovingLefmostHole() { + addHole(100) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + + addMessage(2, 2, "m2") + expectEntries([.Hole(1, 1, 2), .Message(2, 2, "m2", []), .Hole(3, Int32.max, Int32.max)]) + + addMessage(1, 1, "m1") + expectEntries([.Message(1, 1, "m1", []), .Message(2, 2, "m2", []), .Hole(3, Int32.max, Int32.max)]) + } + + func testAddHoleLowerThanMessage() { + addMessage(100, 100, "m100") + addHole(1) + + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [])]) + } + + func testAddHoleHigherThanMessage() { + addMessage(100, 100, "m100") + addHole(200) + + expectEntries([.Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + } + + func testIgnoreHigherHole() { + addHole(200) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + addHole(400) + expectEntries([.Hole(1, Int32.max, Int32.max)]) + } + + func testIgnoreHigherHoleAfterMessage() { + addMessage(100, 100, "m100") + addHole(200) + expectEntries([.Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + addHole(400) + expectEntries([.Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + } + + func testAddHoleBetweenMessages() { + addMessage(100, 100, "m100") + addMessage(200, 200, "m200") + addHole(150) + + expectEntries([.Message(100, 100, "m100", []), .Hole(101, 199, 200), .Message(200, 200, "m200", [])]) + } + + func testFillHoleEmpty() { + fillHole(1, .Complete, []) + expectEntries([]) + } + + func testFillHoleComplete() { + addHole(100) + + fillHole(1, .Complete, [(100, 100, "m100", []), (200, 200, "m200", [])]) + expectEntries([.Message(100, 100, "m100", []), .Message(200, 200, "m200", [])]) + } + + func testFillHoleUpperToLowerPartial() { + addHole(100) + + fillHole(1, .UpperToLower, [(100, 100, "m100", []), (200, 200, "m200", [])]) + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Message(200, 200, "m200", [])]) + } + + func testFillHoleUpperToLowerToBounds() { + addHole(100) + + fillHole(1, .UpperToLower, [(1, 1, "m1", []), (200, 200, "m200", [])]) + expectEntries([.Message(1, 1, "m1", []), .Message(200, 200, "m200", [])]) + } + + func testFillHoleLowerToUpperToBounds() { + addHole(100) + + fillHole(1, .LowerToUpper, [(100, 100, "m100", []), (Int32.max, 200, "m200", [])]) + expectEntries([.Message(100, 100, "m100", []), .Message(Int32.max, 200, "m200", [])]) + } + + func testFillHoleLowerToUpperPartial() { + addHole(100) + + fillHole(1, .LowerToUpper, [(100, 100, "m100", []), (200, 200, "m200", [])]) + expectEntries([.Message(100, 100, "m100", []), .Message(200, 200, "m200", []), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesUpperToLower() { + addHole(1) + + addMessage(100, 100, "m100") + addMessage(200, 200, "m200") + + fillHole(199, .UpperToLower, [(150, 150, "m150", [])]) + + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Hole(101, 149, 150), .Message(150, 150, "m150", []), .Message(200, 200, "m200", []), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesLowerToUpper() { + addHole(1) + + addMessage(100, 100, "m100") + addMessage(200, 200, "m200") + + fillHole(199, .LowerToUpper, [(150, 150, "m150", [])]) + + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Message(150, 150, "m150", []), .Hole(151, 199, 200), .Message(200, 200, "m200", []), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesComplete() { + addHole(1) + + addMessage(100, 100, "m100") + addMessage(200, 200, "m200") + + fillHole(199, .Complete, [(150, 150, "m150", [])]) + + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Message(150, 150, "m150", []), .Message(200, 200, "m200", []), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesWithMessage() { + addMessage(200, 200, "m200") + addMessage(202, 202, "m202") + addHole(201) + addMessage(201, 201, "m201") + + expectEntries([.Message(200, 200, "m200", []), .Message(201, 201, "m201", []), .Message(202, 202, "m202", [])]) + } + + func testFillHoleWithNoMessagesComplete() { + addMessage(100, 100, "m100") + addHole(1) + + fillHole(99, .Complete, []) + + expectEntries([.Message(100, 100, "m100", [])]) + } + + func testFillHoleIgnoreOverMessage() { + addMessage(100, 100, "m100") + addMessage(101, 101, "m101") + + fillHole(100, .Complete, [(90, 90, "m90", [])]) + + expectEntries([.Message(90, 90, "m90", []), .Message(100, 100, "m100", []), .Message(101, 101, "m101", [])]) + } + + func testFillHoleWithOverflow() { + addMessage(100, 100, "m100") + addMessage(200, 200, "m200") + addHole(150) + + fillHole(199, .UpperToLower, [(150, 150, "m150", []), (300, 300, "m300", [])]) + + expectEntries([.Message(100, 100, "m100", []), .Hole(101, 149, 150), .Message(150, 150, "m150", []), .Message(200, 200, "m200", []), .Message(300, 300, "m300", [])]) + } + + func testIgnoreHoleOverMessageBetweenMessages() { + addMessage(199, 199, "m199") + addMessage(200, 200, "m200") + addHole(200) + + expectEntries([.Message(199, 199, "m199", []), .Message(200, 200, "m200", [])]) + } + + func testMergeHoleAfterDeletingMessage() { + addMessage(100, 100, "m100") + addHole(1) + addHole(200) + + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + + removeMessages([100]) + + expectEntries([.Hole(1, Int32.max, Int32.max)]) + } + + func testMergeHoleLowerAfterDeletingMessage() { + addMessage(100, 100, "m100") + addHole(1) + addMessage(200, 200, "m200") + + removeMessages([100]) + + expectEntries([.Hole(1, 199, 200), .Message(200, 200, "m200", [])]) + } + + func testMergeHoleUpperAfterDeletingMessage() { + addMessage(100, 100, "m100") + addMessage(200, 200, "m200") + addHole(300) + + removeMessages([200]) + + expectEntries([.Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + } + + func testExtendLowerHoleAfterDeletingMessage() { + addMessage(100, 100, "m100") + addHole(100) + + removeMessages([100]) + + expectEntries([.Hole(1, Int32.max, Int32.max)]) + } + + func testExtendUpperHoleAfterDeletingMessage() { + addMessage(100, 100, "m100") + addHole(101) + + removeMessages([100]) + + expectEntries([.Hole(1, Int32.max, Int32.max)]) + } + + func testDeleteMessageBelowMessage() { + addMessage(100, 100, "m100") + addMessage(200, 200, "m200") + removeMessages([100]) + + expectEntries([.Message(200, 200, "m200", [])]) + } + + func testDeleteMessageAboveMessage() { + addMessage(100, 100, "m100") + addMessage(200, 200, "m200") + removeMessages([200]) + + expectEntries([.Message(100, 100, "m100", [])]) + } + + func testDeleteMessageBetweenMessages() { + addMessage(100, 100, "m100") + addMessage(200, 200, "m200") + addMessage(300, 300, "m300") + removeMessages([200]) + + expectEntries([.Message(100, 100, "m100", []), .Message(300, 300, "m300", [])]) + } } \ No newline at end of file diff --git a/PostboxTests/MessageHistoryViewTests.swift b/PostboxTests/MessageHistoryViewTests.swift new file mode 100644 index 0000000000..9a30829567 --- /dev/null +++ b/PostboxTests/MessageHistoryViewTests.swift @@ -0,0 +1,3 @@ +import Foundation + + From 86f8f633b1225b3bd1501e52c6f4d99638e5b852 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 28 Mar 2016 17:23:33 +0300 Subject: [PATCH 027/427] no message --- Postbox.xcodeproj/project.pbxproj | 58 +- Postbox/ChatListHole.swift | 21 + Postbox/ChatListIndexTable.swift | 10 +- Postbox/ChatListTable.swift | 228 ++++-- Postbox/ChatListView.swift | 151 +++- Postbox/Coding.swift | 18 +- Postbox/GlobalMessageIdsTable.swift | 8 +- Postbox/KeychainTable.swift | 10 +- Postbox/LmdbValueBox.swift | 8 +- Postbox/Media.swift | 2 +- Postbox/MediaBox.swift | 381 ++++++--- Postbox/MediaCleanupTable.swift | 10 +- Postbox/MediaResource.swift | 5 + Postbox/MediaResourceData.swift | 6 + Postbox/MediaResourceStatus.swift | 34 + Postbox/Message.swift | 230 +++++- Postbox/MessageHistoryHole.swift | 11 +- Postbox/MessageHistoryIndexTable.swift | 313 ++++++-- Postbox/MessageHistoryMetadataTable.swift | 169 ++++ Postbox/MessageHistoryTable.swift | 721 ++++++++++++++++-- Postbox/MessageHistoryTagsTable.swift | 119 +++ Postbox/MessageHistoryUnsentTable.swift | 58 ++ Postbox/MessageHistoryView.swift | 51 +- Postbox/MessageMediaTable.swift | 11 +- Postbox/MetadataTable.swift | 10 +- Postbox/Peer.swift | 2 +- Postbox/PeerChatStateTable.swift | 46 ++ Postbox/PeerTable.swift | 31 +- Postbox/Postbox.swift | 367 ++++++--- Postbox/SeedConfiguration.swift | 13 + Postbox/SimpleDictionary.swift | 8 +- Postbox/SimpleSet.swift | 23 + Postbox/SqliteValueBox.swift | 88 ++- Postbox/Table.swift | 14 + Postbox/UnsentMessageHistoryView.swift | 40 + Postbox/ValueBoxKey.swift | 26 +- Postbox/ViewTracker.swift | 182 ++++- PostboxTests/ChatListTableTests.swift | 100 ++- .../MessageHistoryIndexTableTests.swift | 16 +- PostboxTests/MessageHistoryTableTests.swift | 395 ++++++++-- 40 files changed, 3334 insertions(+), 660 deletions(-) create mode 100644 Postbox/ChatListHole.swift create mode 100644 Postbox/MediaResource.swift create mode 100644 Postbox/MediaResourceData.swift create mode 100644 Postbox/MediaResourceStatus.swift create mode 100644 Postbox/MessageHistoryMetadataTable.swift create mode 100644 Postbox/MessageHistoryTagsTable.swift create mode 100644 Postbox/MessageHistoryUnsentTable.swift create mode 100644 Postbox/PeerChatStateTable.swift create mode 100644 Postbox/SeedConfiguration.swift create mode 100644 Postbox/SimpleSet.swift create mode 100644 Postbox/Table.swift create mode 100644 Postbox/UnsentMessageHistoryView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index dd12187c67..56baf99ef5 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -14,10 +14,18 @@ D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB61B85D192002E4EB5 /* midl.c */; }; D00E0FBB1B85D192002E4EB5 /* midl.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB71B85D192002E4EB5 /* midl.h */; }; D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */; }; + D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; + D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; + D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; + D03BCCF81C73561C0097A291 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; + D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA291C617D39002160FF /* SeedConfiguration.swift */; }; + D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; + D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044E15D1B2ACB9C001EE087 /* CodingTests.swift */; }; D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */ = {isa = PBXBuildFile; fileRef = D044E1621B2AD677001EE087 /* MurMurHash32.m */; }; D044E1641B2AD718001EE087 /* MurMurHash32.h in Headers */ = {isa = PBXBuildFile; fileRef = D044E1611B2AD667001EE087 /* MurMurHash32.h */; settings = {ATTRIBUTES = (Public, ); }; }; D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; + D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; }; D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516411B2D9CEF00AE42E0 /* sqlite3.h */; }; D07516461B2D9CEF00AE42E0 /* sqlite3ext.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */; }; @@ -38,7 +46,10 @@ D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B76BE61B66639F0095CF45 /* DeferredString.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; + D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */; }; + D0C9DA391C65782500855278 /* SimpleSet.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C9DA381C65782500855278 /* SimpleSet.swift */; }; + D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -60,6 +71,7 @@ D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8741C5A334100037222 /* SimpleDictionary.swift */; }; + D0FC1A3C1CA3FB310056AE9A /* MediaResourceData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -80,10 +92,18 @@ D00E0FB61B85D192002E4EB5 /* midl.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = midl.c; path = submodules/lmdb/libraries/liblmdb/midl.c; sourceTree = SOURCE_ROOT; }; D00E0FB71B85D192002E4EB5 /* midl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = midl.h; path = submodules/lmdb/libraries/liblmdb/midl.h; sourceTree = SOURCE_ROOT; }; D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LmdbValueBox.swift; sourceTree = ""; }; + D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; + D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryUnsentTable.swift; sourceTree = ""; }; + D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageHistoryView.swift; sourceTree = ""; }; + D03BCCF71C73561C0097A291 /* Table.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Table.swift; sourceTree = ""; }; + D044CA291C617D39002160FF /* SeedConfiguration.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SeedConfiguration.swift; sourceTree = ""; }; + D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryMetadataTable.swift; sourceTree = ""; }; + D044CA2D1C618373002160FF /* ChatListHole.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatListHole.swift; sourceTree = ""; }; D044E15D1B2ACB9C001EE087 /* CodingTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CodingTests.swift; sourceTree = ""; }; D044E1611B2AD667001EE087 /* MurMurHash32.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MurMurHash32.h; sourceTree = ""; }; D044E1621B2AD677001EE087 /* MurMurHash32.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MurMurHash32.m; sourceTree = ""; }; D055BD321B7D3D2D00F06C0A /* MediaBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaBox.swift; sourceTree = ""; }; + D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResourceStatus.swift; sourceTree = ""; }; D07516401B2D9CEF00AE42E0 /* sqlite3.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = sqlite3.c; sourceTree = ""; }; D07516411B2D9CEF00AE42E0 /* sqlite3.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3.h; sourceTree = ""; }; D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3ext.h; sourceTree = ""; }; @@ -105,7 +125,10 @@ D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; + D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatStateTable.swift; sourceTree = ""; }; D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryViewTests.swift; sourceTree = ""; }; + D0C9DA381C65782500855278 /* SimpleSet.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SimpleSet.swift; sourceTree = ""; }; + D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResource.swift; sourceTree = ""; }; D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -130,6 +153,7 @@ D0F9E8701C5A0E9B00037222 /* PeerTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerTable.swift; sourceTree = ""; }; D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GlobalMessageIdsTable.swift; sourceTree = ""; }; D0F9E8741C5A334100037222 /* SimpleDictionary.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SimpleDictionary.swift; sourceTree = ""; }; + D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResourceData.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -164,6 +188,17 @@ name = lmdb; sourceTree = ""; }; + D05F09981C9CAC1100BB6F96 /* Media Box */ = { + isa = PBXGroup; + children = ( + D055BD321B7D3D2D00F06C0A /* MediaBox.swift */, + D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */, + D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */, + D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */, + ); + name = "Media Box"; + sourceTree = ""; + }; D07515FC1B2C44A200AE42E0 /* thirdparty */ = { isa = PBXGroup; children = ( @@ -209,9 +244,13 @@ D0E1DE161C5EB06000C7826E /* Tables */ = { isa = PBXGroup; children = ( + D03BCCF71C73561C0097A291 /* Table.swift */, D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */, + D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */, D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */, D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, + D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */, + D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */, D0F9E8641C58CB7F00037222 /* ChatListTable.swift */, D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */, D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */, @@ -219,6 +258,7 @@ D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */, D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */, D0F9E8701C5A0E9B00037222 /* PeerTable.swift */, + D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */, ); name = Tables; sourceTree = ""; @@ -230,6 +270,7 @@ D0E3A79D1B28B50400A402D9 /* Message.swift */, D0E3A7A11B28B7DC00A402D9 /* Media.swift */, D08C71391C501F0700779C0F /* MessageHistoryHole.swift */, + D044CA2D1C618373002160FF /* ChatListHole.swift */, ); name = Objects; sourceTree = ""; @@ -240,6 +281,7 @@ D0E3A7871B28AE9C00A402D9 /* Coding.swift */, D0B76BE61B66639F0095CF45 /* DeferredString.swift */, D0F9E8741C5A334100037222 /* SimpleDictionary.swift */, + D0C9DA381C65782500855278 /* SimpleSet.swift */, ); name = Utils; sourceTree = ""; @@ -260,6 +302,7 @@ children = ( D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */, D0F9E86A1C59719800037222 /* ChatListView.swift */, + D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */, ); name = Views; sourceTree = ""; @@ -296,9 +339,10 @@ D0E1DE191C5EB0B500C7826E /* Value Box */, D0E1DE161C5EB06000C7826E /* Tables */, D0E1DE1A1C5EB0DB00C7826E /* Views */, + D05F09981C9CAC1100BB6F96 /* Media Box */, D0E3A7811B28ADD000A402D9 /* Postbox.swift */, + D044CA291C617D39002160FF /* SeedConfiguration.swift */, D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */, - D055BD321B7D3D2D00F06C0A /* MediaBox.swift */, D0E3A74D1B28A7E300A402D9 /* Supporting Files */, ); path = Postbox; @@ -457,10 +501,12 @@ D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, + D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */, D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */, D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, + D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, @@ -468,8 +514,11 @@ D0F9E8671C58D08900037222 /* ChatListIndexTable.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, + D0C9DA391C65782500855278 /* SimpleSet.swift in Sources */, D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, + D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */, D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */, + D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */, @@ -479,11 +528,18 @@ D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */, D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, + D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, + D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, + D0FC1A3C1CA3FB310056AE9A /* MediaResourceData.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */, + D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */, + D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */, + D03BCCF81C73561C0097A291 /* Table.swift in Sources */, + D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */, D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */, diff --git a/Postbox/ChatListHole.swift b/Postbox/ChatListHole.swift new file mode 100644 index 0000000000..741dcc62b3 --- /dev/null +++ b/Postbox/ChatListHole.swift @@ -0,0 +1,21 @@ +import Foundation + +public struct ChatListHole: Comparable, CustomStringConvertible { + public let index: MessageIndex + + public init(index: MessageIndex) { + self.index = index + } + + public var description: String { + return "ChatListHole(\(self.index.id), \(self.index.timestamp))" + } +} + +public func ==(lhs: ChatListHole, rhs: ChatListHole) -> Bool { + return lhs.index == rhs.index +} + +public func <(lhs: ChatListHole, rhs: ChatListHole) -> Bool { + return lhs.index < rhs.index +} diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift index f25db86b63..6e6451d3a1 100644 --- a/Postbox/ChatListIndexTable.swift +++ b/Postbox/ChatListIndexTable.swift @@ -1,12 +1,8 @@ import Foundation -final class ChatListIndexTable { - let valueBox: ValueBox - let tableId: Int32 - - init(valueBox: ValueBox, tableId: Int32) { - self.valueBox = valueBox - self.tableId = tableId +final class ChatListIndexTable: Table { + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) } private func key(peerId: PeerId) -> ValueBoxKey { diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index 59734c80d1..8267731e8b 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -2,32 +2,55 @@ import Foundation enum ChatListOperation { case InsertMessage(IntermediateMessage) + case InsertHole(ChatListHole) case InsertNothing(MessageIndex) - case Remove([MessageIndex]) + case RemoveMessage([MessageIndex]) + case RemoveHoles([MessageIndex]) } enum ChatListIntermediateEntry { case Message(IntermediateMessage) + case Hole(ChatListHole) case Nothing(MessageIndex) + + var index: MessageIndex { + switch self { + case let .Message(message): + return MessageIndex(id: message.id, timestamp: message.timestamp) + case let .Hole(hole): + return hole.index + case let .Nothing(index): + return index + } + } } -final class ChatListTable { - let valueBox: ValueBox - let tableId: Int32 +private enum ChatListEntryType: Int8 { + case Message = 1 + case Hole = 2 +} + +final class ChatListTable: Table { let indexTable: ChatListIndexTable + let emptyMemoryBuffer = MemoryBuffer(memory: nil, capacity: 0, length: 0, freeWhenDone: false) + let metadataTable: MessageHistoryMetadataTable + let seedConfiguration: SeedConfiguration - init(valueBox: ValueBox, tableId: Int32, indexTable: ChatListIndexTable) { - self.valueBox = valueBox - self.tableId = tableId + init(valueBox: ValueBox, tableId: Int32, indexTable: ChatListIndexTable, metadataTable: MessageHistoryMetadataTable, seedConfiguration: SeedConfiguration) { self.indexTable = indexTable + self.metadataTable = metadataTable + self.seedConfiguration = seedConfiguration + + super.init(valueBox: valueBox, tableId: tableId) } - private func key(index: MessageIndex) -> ValueBoxKey { - let key = ValueBoxKey(length: 4 + 4 + 4 + 8) + private func key(index: MessageIndex, type: ChatListEntryType) -> ValueBoxKey { + let key = ValueBoxKey(length: 4 + 4 + 4 + 8 + 1) key.setInt32(0, value: index.timestamp) key.setInt32(4, value: index.id.namespace) key.setInt32(4 + 4, value: index.id.id) key.setInt64(4 + 4 + 4, value: index.id.peerId.toInt64()) + key.setInt8(4 + 4 + 4 + 8, value: type.rawValue) return key } @@ -43,86 +66,188 @@ final class ChatListTable { return key } + private func ensureInitialized() { + if !self.metadataTable.isInitializedChatList() { + for hole in self.seedConfiguration.initializeChatListWithHoles { + self.justInsertHole(hole) + } + self.metadataTable.setInitializedChatList() + } + } + func replay(historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]], messageHistoryTable: MessageHistoryTable, inout operations: [ChatListOperation]) { + self.ensureInitialized() + for (peerId, _) in historyOperationsByPeerId { let currentIndex: MessageIndex? = self.indexTable.get(peerId) if let topMessage = messageHistoryTable.topMessage(peerId) { let index = MessageIndex(id: topMessage.id, timestamp: topMessage.timestamp) if let currentIndex = currentIndex where currentIndex != index { - self.justRemove(currentIndex) + self.justRemoveMessage(currentIndex) } if let currentIndex = currentIndex { - operations.append(.Remove([currentIndex])) + operations.append(.RemoveMessage([currentIndex])) } self.indexTable.set(index) - self.justInsert(index) + self.justInsertMessage(index) operations.append(.InsertMessage(topMessage)) } else { if let currentIndex = currentIndex { - operations.append(.Remove([currentIndex])) + operations.append(.RemoveMessage([currentIndex])) operations.append(.InsertNothing(currentIndex)) } } } } - func justInsert(index: MessageIndex) { - self.valueBox.set(self.tableId, key: self.key(index), value: MemoryBuffer(memory: nil, capacity: 0, length: 0, freeWhenDone: false)) + func addHole(hole: ChatListHole, inout operations: [ChatListOperation]) { + self.ensureInitialized() + + if self.valueBox.get(self.tableId, key: self.key(hole.index, type: .Hole)) == nil { + self.justInsertHole(hole) + operations.append(.InsertHole(hole)) + } } - func justRemove(index: MessageIndex) { - self.valueBox.remove(self.tableId, key: self.key(index)) + func replaceHole(index: MessageIndex, hole: ChatListHole?, inout operations: [ChatListOperation]) { + self.ensureInitialized() + + if self.valueBox.get(self.tableId, key: self.key(index, type: .Hole)) != nil { + if let hole = hole { + if hole.index != index { + self.justRemoveHole(index) + self.justInsertHole(hole) + operations.append(.RemoveHoles([index])) + operations.append(.InsertHole(hole)) + } + } else{ + self.justRemoveHole(index) + operations.append(.RemoveHoles([index])) + } + } } - func entriesAround(index: MessageIndex, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + private func justInsertMessage(index: MessageIndex) { + self.valueBox.set(self.tableId, key: self.key(index, type: .Message), value: self.emptyMemoryBuffer) + } + + private func justRemoveMessage(index: MessageIndex) { + self.valueBox.remove(self.tableId, key: self.key(index, type: .Message)) + } + + private func justInsertHole(hole: ChatListHole) { + self.valueBox.set(self.tableId, key: self.key(hole.index, type: .Hole), value: self.emptyMemoryBuffer) + } + + private func justRemoveHole(index: MessageIndex) { + self.valueBox.remove(self.tableId, key: self.key(index, type: .Hole)) + } + + func entriesAround(index: MessageIndex, messageHistoryTable: MessageHistoryTable, count: Int) -> (entries: [ChatListIntermediateEntry], lower: ChatListIntermediateEntry?, upper: ChatListIntermediateEntry?) { + self.ensureInitialized() + var lowerEntries: [ChatListIntermediateEntry] = [] - - self.valueBox.range(self.tableId, start: self.key(index).successor, end: self.lowerBound(), keys: { key in - let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) - if let message = messageHistoryTable.getMessage(index) { - lowerEntries.append(.Message(message)) - } else { - lowerEntries.append(.Nothing(index)) - } - return true - }, limit: count) - var upperEntries: [ChatListIntermediateEntry] = [] - self.valueBox.range(self.tableId, start: self.key(index), end: self.upperBound(), keys: { key in + var lower: ChatListIntermediateEntry? + var upper: ChatListIntermediateEntry? + + self.valueBox.range(self.tableId, start: self.key(index, type: .Message), end: self.lowerBound(), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) - if let message = messageHistoryTable.getMessage(index) { - upperEntries.append(.Message(message)) + let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + if type == ChatListEntryType.Message.rawValue { + if let message = messageHistoryTable.getMessage(index) { + lowerEntries.append(.Message(message)) + } else { + lowerEntries.append(.Nothing(index)) + } } else { - upperEntries.append(.Nothing(index)) + lowerEntries.append(.Hole(ChatListHole(index: index))) } return true - }, limit: count) + }, limit: count / 2 + 1) + if lowerEntries.count >= count / 2 + 1 { + lower = lowerEntries.last + lowerEntries.removeLast() + } + + self.valueBox.range(self.tableId, start: self.key(index, type: .Message).predecessor, end: self.upperBound(), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) + let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + if type == ChatListEntryType.Message.rawValue { + if let message = messageHistoryTable.getMessage(index) { + upperEntries.append(.Message(message)) + } else { + upperEntries.append(.Nothing(index)) + } + } else { + upperEntries.append(.Hole(ChatListHole(index: index))) + } + return true + }, limit: count - lowerEntries.count + 1) + if upperEntries.count >= count - lowerEntries.count + 1 { + upper = upperEntries.last + upperEntries.removeLast() + } + + if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count { + var additionalLowerEntries: [ChatListIntermediateEntry] = [] + let startEntryType: ChatListEntryType + switch lowerEntries.last! { + case .Message, .Nothing: + startEntryType = .Message + case .Hole: + startEntryType = .Hole + } + self.valueBox.range(self.tableId, start: self.key(lowerEntries.last!.index, type: startEntryType), end: self.lowerBound(), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) + let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + if type == ChatListEntryType.Message.rawValue { + if let message = messageHistoryTable.getMessage(index) { + additionalLowerEntries.append(.Message(message)) + } else { + additionalLowerEntries.append(.Nothing(index)) + } + } else { + additionalLowerEntries.append(.Hole(ChatListHole(index: index))) + } + return true + }, limit: count - lowerEntries.count - upperEntries.count + 1) + if additionalLowerEntries.count >= count - lowerEntries.count + upperEntries.count + 1 { + lower = additionalLowerEntries.last + additionalLowerEntries.removeLast() + } + lowerEntries.appendContentsOf(additionalLowerEntries) + } var entries: [ChatListIntermediateEntry] = [] - for entry in lowerEntries.reverse() { - entries.append(entry) - } + entries.appendContentsOf(lowerEntries.reverse()) entries.appendContentsOf(upperEntries) - - return entries + return (entries: entries, lower: lower, upper: upper) } func earlierEntries(index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + self.ensureInitialized() + var entries: [ChatListIntermediateEntry] = [] let key: ValueBoxKey if let index = index { - key = self.key(index) + key = self.key(index, type: .Message) } else { key = self.upperBound() } self.valueBox.range(self.tableId, start: key, end: self.lowerBound(), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) - if let message = messageHistoryTable.getMessage(index) { - entries.append(.Message(message)) + let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + if type == ChatListEntryType.Message.rawValue { + if let message = messageHistoryTable.getMessage(index) { + entries.append(.Message(message)) + } else { + entries.append(.Nothing(index)) + } } else { - entries.append(.Nothing(index)) + entries.append(.Hole(ChatListHole(index: index))) } return true }, limit: count) @@ -130,20 +255,27 @@ final class ChatListTable { } func laterEntries(index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + self.ensureInitialized() + var entries: [ChatListIntermediateEntry] = [] let key: ValueBoxKey if let index = index { - key = self.key(index) + key = self.key(index, type: .Message) } else { key = self.lowerBound() } self.valueBox.range(self.tableId, start: key, end: self.upperBound(), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) - if let message = messageHistoryTable.getMessage(index) { - entries.append(.Message(message)) + let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + if type == ChatListEntryType.Message.rawValue { + if let message = messageHistoryTable.getMessage(index) { + entries.append(.Message(message)) + } else { + entries.append(.Nothing(index)) + } } else { - entries.append(.Nothing(index)) + entries.append(.Hole(ChatListHole(index: index))) } return true }, limit: count) @@ -151,6 +283,6 @@ final class ChatListTable { } func debugList(messageHistoryTable: MessageHistoryTable) -> [ChatListIntermediateEntry] { - return self.entriesAround(MessageIndex(id: MessageId(peerId: PeerId(0), namespace: 0, id: 0), timestamp: 0), messageHistoryTable: messageHistoryTable, count: 1000) + return self.laterEntries(MessageIndex.absoluteLowerBound(), messageHistoryTable: messageHistoryTable, count: 1000) } } diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index f8a197bb38..ad76989417 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -2,12 +2,15 @@ import Foundation public enum ChatListEntry: Comparable { case MessageEntry(Message) + case HoleEntry(ChatListHole) case Nothing(MessageIndex) public var index: MessageIndex { switch self { case let .MessageEntry(message): return MessageIndex(message) + case let .HoleEntry(hole): + return hole.index case let .Nothing(index): return index } @@ -22,9 +25,10 @@ public func <(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { return lhs.index < rhs.index } -enum MutableChatListEntry { +enum MutableChatListEntry: Equatable { case IntermediateMessageEntry(IntermediateMessage) case MessageEntry(Message) + case HoleEntry(ChatListHole) case Nothing(MessageIndex) var index: MessageIndex { @@ -33,12 +37,51 @@ enum MutableChatListEntry { return MessageIndex(id: message.id, timestamp: message.timestamp) case let .MessageEntry(message): return MessageIndex(message) + case let .HoleEntry(hole): + return hole.index case let .Nothing(index): return index } } } +func ==(lhs: MutableChatListEntry, rhs: MutableChatListEntry) -> Bool { + if lhs.index != rhs.index { + return false + } + + switch lhs { + case .IntermediateMessageEntry: + switch rhs { + case .IntermediateMessageEntry: + return true + default: + return false + } + case .MessageEntry: + switch rhs { + case .MessageEntry: + return true + default: + return false + } + case .HoleEntry: + switch rhs { + case .HoleEntry: + return true + default: + return false + } + case .Nothing: + switch rhs { + case .Nothing: + return true + default: + return false + } + } +} + final class MutableChatListViewReplayContext { var invalidEarlier: Bool = false var invalidLater: Bool = false @@ -74,8 +117,16 @@ final class MutableChatListView { if self.add(.Nothing(index)) { hasChanges = true } - case let .Remove(indices): - if self.remove(Set(indices), context: context) { + case let .InsertHole(index): + if self.add(.HoleEntry(index)) { + hasChanges = true + } + case let .RemoveMessage(indices): + if self.remove(Set(indices), holes: false, context: context) { + hasChanges = true + } + case let .RemoveHoles(indices): + if self.remove(Set(indices), holes: true, context: context) { hasChanges = true } } @@ -88,18 +139,13 @@ final class MutableChatListView { self.entries.append(entry) return true } else { - let first = self.entries[self.entries.count - 1].index - let last = self.entries[0].index + let first = self.entries[self.entries.count - 1] + let last = self.entries[0] - var next: MessageIndex? - if let later = self.later { - next = later.index - } + let next = self.later - let index = entry.index - - if index < last { - if self.earlier == nil || self.earlier!.index < index { + if entry.index < last.index { + if self.earlier == nil || self.earlier!.index < entry.index { if self.entries.count < self.count { self.entries.insert(entry, atIndex: 0) } else { @@ -109,9 +155,9 @@ final class MutableChatListView { } else { return false } - } else if index > first { - if next != nil && index > next! { - if self.later == nil || self.later!.index > index { + } else if entry.index > first.index { + if next != nil && entry.index > next!.index { + if self.later == nil || self.later!.index > entry.index { if self.entries.count < self.count { self.entries.append(entry) } else { @@ -129,13 +175,13 @@ final class MutableChatListView { } return true } - } else if index != last && index != first { + } else if entry != last && entry != first { var i = self.entries.count while i >= 1 { - if self.entries[i - 1].index < index { + if self.entries[i - 1].index < entry.index { break } - i-- + i -= 1 } self.entries.insert(entry, atIndex: i) if self.entries.count > self.count { @@ -149,27 +195,54 @@ final class MutableChatListView { } } - func remove(indices: Set, context: MutableChatListViewReplayContext) -> Bool { + func remove(indices: Set, holes: Bool, context: MutableChatListViewReplayContext) -> Bool { var hasChanges = false if let earlier = self.earlier where indices.contains(earlier.index) { - context.invalidEarlier = true - hasChanges = true + var match = false + switch earlier { + case .HoleEntry: + match = holes + case .IntermediateMessageEntry, .MessageEntry, .Nothing: + match = !holes + } + if match { + context.invalidEarlier = true + hasChanges = true + } } if let later = self.later where indices.contains(later.index) { - context.invalidLater = true - hasChanges = true + var match = false + switch later { + case .HoleEntry: + match = holes + case .IntermediateMessageEntry, .MessageEntry, .Nothing: + match = !holes + } + if match { + context.invalidLater = true + hasChanges = true + } } if self.entries.count != 0 { var i = self.entries.count - 1 while i >= 0 { if indices.contains(self.entries[i].index) { - self.entries.removeAtIndex(i) - context.removedEntries = true - hasChanges = true + var match = false + switch self.entries[i] { + case .HoleEntry: + match = holes + case .IntermediateMessageEntry, .MessageEntry, .Nothing: + match = !holes + } + if match { + self.entries.removeAtIndex(i) + context.removedEntries = true + hasChanges = true + } } - i-- + i -= 1 } } @@ -205,7 +278,7 @@ final class MutableChatListView { if addedEntries[i].index.id == addedEntries[i - 1].index.id { addedEntries.removeAtIndex(i) } - i-- + i -= 1 } self.entries = [] @@ -217,7 +290,7 @@ final class MutableChatListView { anchorIndex = i break } - i-- + i -= 1 } } @@ -229,7 +302,7 @@ final class MutableChatListView { i = anchorIndex while i >= 0 && i > anchorIndex - self.count { self.entries.insert(addedEntries[i], atIndex: 0) - i-- + i -= 1 } self.earlier = nil @@ -261,6 +334,16 @@ final class MutableChatListView { } } + func firstHole() -> ChatListHole? { + for entry in self.entries { + if case let .HoleEntry(hole) = entry { + return hole + } + } + + return nil + } + func updatePeers(peers: [PeerId: Peer]) -> Bool { var hasChanges = false /*for i in 0 ..< self.entries.count { @@ -294,6 +377,8 @@ final class MutableChatListView { public final class ChatListView { public let entries: [ChatListEntry] + public let earlierIndex: MessageIndex? + public let laterIndex: MessageIndex? init(_ mutableView: MutableChatListView) { var entries: [ChatListEntry] = [] @@ -303,10 +388,14 @@ public final class ChatListView { entries.append(.MessageEntry(message)) case let .Nothing(index): entries.append(.Nothing(index)) + case let .HoleEntry(hole): + entries.append(.HoleEntry(hole)) case .IntermediateMessageEntry: assertionFailure() } } self.entries = entries + self.earlierIndex = mutableView.earlier?.index + self.laterIndex = mutableView.later?.index } } diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 96db1c7b43..1a2b69c812 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -144,6 +144,10 @@ public final class ReadBuffer: MemoryBuffer { func reset() { self.offset = 0 } + + func sharedBufferNoCopy() -> ReadBuffer { + return ReadBuffer(memory: memory, length: length, freeWhenDone: false) + } } private enum ValueType: Int8 { @@ -420,7 +424,7 @@ public final class Decoder { var objectLength: Int32 = 0 memcpy(&objectLength, bytes + (offset + 4), 4) offset += 8 + Int(objectLength) - i++ + i += 1 } case .ObjectDictionary: var length: Int32 = 0 @@ -435,7 +439,7 @@ public final class Decoder { var valueLength: Int32 = 0 memcpy(&valueLength, bytes + (offset + 4), 4) offset += 8 + Int(valueLength) - i++ + i += 1 } case .Bytes: var length: Int32 = 0 @@ -672,7 +676,7 @@ public final class Decoder { var element: Int32 = 0 memcpy(&element, self.buffer.memory + (self.offset + 4 + 4 * Int(i)), 4) array.append(element) - i++ + i += 1 } self.offset += 4 + Int(length) * 4 return array @@ -692,7 +696,7 @@ public final class Decoder { var element: Int64 = 0 memcpy(&element, self.buffer.memory + (self.offset + 4 + 8 * Int(i)), 8) array.append(element) - i++ + i += 1 } self.offset += 4 + Int(length) * 8 return array @@ -731,7 +735,7 @@ public final class Decoder { } } - i++ + i += 1 } if failed { @@ -774,7 +778,7 @@ public final class Decoder { } } - i++ + i += 1 } if failed { @@ -828,7 +832,7 @@ public final class Decoder { failed = true } - i++ + i += 1 } if failed { diff --git a/Postbox/GlobalMessageIdsTable.swift b/Postbox/GlobalMessageIdsTable.swift index a95d66f88e..6b9fec219a 100644 --- a/Postbox/GlobalMessageIdsTable.swift +++ b/Postbox/GlobalMessageIdsTable.swift @@ -1,17 +1,15 @@ import Foundation -final class GlobalMessageIdsTable { - let valueBox: ValueBox - let tableId: Int32 +final class GlobalMessageIdsTable: Table { let namespace: Int32 let sharedKey = ValueBoxKey(length: 4) let sharedBuffer = WriteBuffer() init(valueBox: ValueBox, tableId: Int32, namespace: Int32) { - self.valueBox = valueBox - self.tableId = tableId self.namespace = namespace + + super.init(valueBox: valueBox, tableId: tableId) } private func key(id: Int32) -> ValueBoxKey { diff --git a/Postbox/KeychainTable.swift b/Postbox/KeychainTable.swift index 34da757a82..00208e6459 100644 --- a/Postbox/KeychainTable.swift +++ b/Postbox/KeychainTable.swift @@ -1,12 +1,8 @@ import Foundation -final class KeychainTable { - let valueBox: ValueBox - let tableId: Int32 - - init(valueBox: ValueBox, tableId: Int32) { - self.valueBox = valueBox - self.tableId = tableId +final class KeychainTable: Table { + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) } private func key(string: String) -> ValueBoxKey { diff --git a/Postbox/LmdbValueBox.swift b/Postbox/LmdbValueBox.swift index 6ad6b4b304..1922a4b9d9 100644 --- a/Postbox/LmdbValueBox.swift +++ b/Postbox/LmdbValueBox.swift @@ -246,7 +246,7 @@ public final class LmdbValueBox: ValueBox { var count = 0 if value != nil && value!.0 < end { - count++ + count += 1 values(value!.0, value!.1) } @@ -260,7 +260,7 @@ public final class LmdbValueBox: ValueBox { startTime = currentTime if value != nil && value!.0 < end { - count++ + count += 1 values(value!.0, value!.1) } } @@ -279,7 +279,7 @@ public final class LmdbValueBox: ValueBox { var count = 0 if value != nil && value!.0 > end { - count++ + count += 1 values(value!.0, value!.1) } @@ -293,7 +293,7 @@ public final class LmdbValueBox: ValueBox { startTime = currentTime if value != nil && value!.0 > end { - count++ + count += 1 values(value!.0, value!.1) } } diff --git a/Postbox/Media.swift b/Postbox/Media.swift index 4d7d771546..4b0dfe8fd8 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -58,7 +58,7 @@ public struct MediaId: Hashable, CustomStringConvertible { var array: [MediaId] = [] while i < Int(length) { array.append(MediaId(buffer)) - i++ + i += 1 } return array } diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 6d3ecce762..2e12b545c2 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -2,37 +2,49 @@ import Foundation import SwiftSignalKit import Display -public struct MediaKeyStatus { - let size: Int +private final class ResourceStatusContext { + var status: MediaResourceStatus? + let subscribers = Bag Void>() } -private final class MediaBoxMutexEntry { - var mutex = pthread_mutex_t() +private final class ResourceDataContext { + var data: MediaResourceData + let dataSubscribers = Bag Void>() - init() { - var attr = pthread_mutexattr_t() - pthread_mutexattr_init(&attr) - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) - pthread_mutex_init(&mutex, &attr) - } + var fetchDisposable: Disposable? + let fetchSubscribers = Bag() - deinit { - pthread_mutex_destroy(&mutex) - } - - func with(@noescape f: Void -> Void) { - pthread_mutex_lock(&self.mutex) - f() - pthread_mutex_unlock(&self.mutex) + init(data: MediaResourceData) { + self.data = data } } +private func fileSize(path: String) -> Int { + var value = stat() + stat(path, &value) + return Int(value.st_size) +} + public final class MediaBox { let basePath: String let buffer = WriteBuffer() - private var mutexEntriesLock = OSSpinLock() - private var mutexEntries: [String : MediaBoxMutexEntry] = [:] + private let statusQueue = Queue() + private let dataQueue = Queue() + + private var statusContexts: [String: ResourceStatusContext] = [:] + private var dataContexts: [String: ResourceDataContext] = [:] + + private var wrappedFetchResource = Promise<(MediaResource, Int) -> Signal>() + public var fetchResource: ((MediaResource, Int) -> Signal)? { + didSet { + if let fetchResource = self.fetchResource { + wrappedFetchResource.set(.single(fetchResource)) + } else { + wrappedFetchResource.set(.never()) + } + } + } lazy var ensureDirectoryCreated: Void = { try! NSFileManager.defaultManager().createDirectoryAtPath(self.basePath, withIntermediateDirectories: true, attributes: nil) @@ -42,104 +54,269 @@ public final class MediaBox { self.basePath = basePath } - private func keyForId(id: MediaId, key: MemoryBuffer) -> String { - let string = NSMutableString() - string.appendFormat("%d", Int(id.namespace)) - string.appendFormat("_%lld", Int64(id.id)) - string.appendString("_\(key)") - return string as String + private func pathForId(id: String) -> String { + return "\(self.basePath)/\(id)" } - private func pathForId(id: MediaId, key: MemoryBuffer) -> String { - return "\(self.basePath)/\(self.keyForId(id, key: key))" - } - - private func mutexForKey(key: String) -> MediaBoxMutexEntry { - let entry: MediaBoxMutexEntry - - OSSpinLockLock(&mutexEntriesLock) - if let existingEntry = self.mutexEntries[key] { - entry = existingEntry - } else { - entry = MediaBoxMutexEntry() - self.mutexEntries[key] = entry - } - OSSpinLockUnlock(&mutexEntriesLock) - - return entry - } - - func writeId(id: MediaId, key: ReadBuffer, value: NSData) { - assertNotOnMainThread() - self.mutexForKey(self.keyForId(id, key: key)).with { - let _ = self.ensureDirectoryCreated - value.writeToFile(self.pathForId(id, key: key), atomically: false) - } - } - - private func status(id: MediaId, key: ReadBuffer) -> MediaKeyStatus? { - assertNotOnMainThread() - var value = stat() - stat(self.pathForId(id, key: key), &value) - return MediaKeyStatus(size: Int(value.st_size)) - } - - private func read(id: MediaId, key: ReadBuffer) -> NSData? { - assertNotOnMainThread() - var data: NSData? - do { - data = try NSData(contentsOfFile: self.pathForId(id, key: key), options: NSDataReadingOptions.DataReadingMappedIfSafe) - } catch _ { - } - return data - } - - public func data(id: MediaId, key: ReadBuffer, size: Int, fetch: ((NSData, Int) -> Signal)?) -> Signal { + public func resourceStatus(resource: MediaResource) -> Signal { return Signal { subscriber in - var cancelled = false let disposable = MetaDisposable() - disposable.set(ActionDisposable { - cancelled = true - }) - self.mutexForKey(self.keyForId(id, key: key)).with { - if cancelled { - return + self.statusQueue.dispatch { + let statusContext: ResourceStatusContext + if let current = self.statusContexts[resource.id] { + statusContext = current + } else { + statusContext = ResourceStatusContext() + self.statusContexts[resource.id] = statusContext } - let status = self.status(id, key: key) ?? MediaKeyStatus(size: 0) + let index = statusContext.subscribers.add({ status in + subscriber.putNext(status) + }) - var currentData: NSData? - if status.size > 0 { - if let data = self.read(id, key: key) { - currentData = data - subscriber.putNext(data) - } - } - - if status.size < size { - if let fetch = fetch { - disposable.set(fetch(currentData ?? NSData(), status.size).start(next: { [weak self] next in - if let strongSelf = self { - strongSelf.mutexForKey(strongSelf.keyForId(id, key: key)).with { - strongSelf.writeId(id, key: key, value: next) + if let status = statusContext.status { + subscriber.putNext(status) + } else { + self.dataQueue.dispatch { + let status: MediaResourceStatus + + let path = self.pathForId(resource.id) + let currentSize = fileSize(path) + if currentSize >= resource.size { + status = .Local + } else { + var fetchingData = false + if let dataContext = self.dataContexts[resource.id] { + fetchingData = dataContext.fetchDisposable != nil + } + + if fetchingData { + status = .Fetching(progress: Float(currentSize) / Float(resource.size)) + } else { + status = .Remote + } + } + + self.statusQueue.dispatch { + if let statusContext = self.statusContexts[resource.id] where statusContext.status == nil { + statusContext.status = status + + for subscriber in statusContext.subscribers.copyItems() { + subscriber(status) } } - subscriber.putNext(next) - }, error: { error in - subscriber.putError(error) - }, completed: { - subscriber.putCompletion() - })) - } else { - subscriber.putError(NoError()) + } } - } else { + } + + disposable.set(ActionDisposable { + self.statusQueue.dispatch { + if let current = self.statusContexts[resource.id] { + current.subscribers.remove(index) + if current.subscribers.isEmpty { + self.statusContexts.removeValueForKey(resource.id) + } + } + } + }) + } + + return disposable + } + } + + public func resourceData(resource: MediaResource) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.dataQueue.dispatch { + let path = self.pathForId(resource.id) + let currentSize = fileSize(path) + + if currentSize >= resource.size { + subscriber.putNext(MediaResourceData(path: path, size: currentSize)) subscriber.putCompletion() + } else { + let dataContext: ResourceDataContext + if let current = self.dataContexts[resource.id] { + dataContext = current + } else { + dataContext = ResourceDataContext(data: MediaResourceData(path: path, size: currentSize)) + self.dataContexts[resource.id] = dataContext + } + + let index = dataContext.dataSubscribers.add { data in + subscriber.putNext(data) + if data.size >= resource.size { + subscriber.putCompletion() + } + } + + subscriber.putNext(dataContext.data) + + disposable.set(ActionDisposable { + self.dataQueue.dispatch { + if let dataContext = self.dataContexts[resource.id] { + dataContext.dataSubscribers.remove(index) + + if dataContext.dataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { + self.dataContexts.removeValueForKey(resource.id) + } + } + } + }) } } return disposable } } + + public func fetchedResource(resource: MediaResource, interactive: Bool) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.dataQueue.dispatch { + let path = self.pathForId(resource.id) + let currentSize = fileSize(path) + + if currentSize >= resource.size { + subscriber.putCompletion() + } else { + let dataContext: ResourceDataContext + if let current = self.dataContexts[resource.id] { + dataContext = current + } else { + dataContext = ResourceDataContext(data: MediaResourceData(path: path, size: currentSize)) + self.dataContexts[resource.id] = dataContext + } + + let index: Bag.Index = dataContext.fetchSubscribers.add(Void()) + + if dataContext.fetchDisposable == nil { + let status: MediaResourceStatus = .Fetching(progress: Float(currentSize) / Float(resource.size)) + self.statusQueue.dispatch { + if let statusContext = self.statusContexts[resource.id] { + statusContext.status = status + for subscriber in statusContext.subscribers.copyItems() { + subscriber(status) + } + } + } + + var offset = currentSize + var fd: Int32? + dataContext.fetchDisposable = (self.wrappedFetchResource.get() |> mapToSignal { fetch -> Signal in + return fetch(resource, offset) + } |> afterDisposed { + if let fd = fd { + close(fd) + } + }).start(next: { data in + self.dataQueue.dispatch { + let _ = self.ensureDirectoryCreated + + if fd == nil { + let handle = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) + if handle >= 0 { + fd = handle + } + } + + if let fd = fd { + write(fd, data.bytes, data.length) + + offset += data.length + let updatedSize = offset + + for subscriber in dataContext.dataSubscribers.copyItems() { + subscriber(MediaResourceData(path: path, size: updatedSize)) + } + + let status: MediaResourceStatus + if updatedSize >= resource.size { + status = .Local + } else { + status = .Fetching(progress: Float(updatedSize) / Float(resource.size)) + } + + self.statusQueue.dispatch { + if let statusContext = self.statusContexts[resource.id] { + statusContext.status = status + for subscriber in statusContext.subscribers.copyItems() { + subscriber(status) + } + } + } + } + } + }) + } + + disposable.set(ActionDisposable { + self.dataQueue.dispatch { + if let dataContext = self.dataContexts[resource.id] { + dataContext.fetchSubscribers.remove(index) + + if dataContext.fetchSubscribers.isEmpty { + dataContext.fetchDisposable?.dispose() + dataContext.fetchDisposable = nil + + let currentSize = fileSize(path) + let status: MediaResourceStatus + if currentSize >= resource.size { + status = .Local + } else { + status = .Remote + } + + self.statusQueue.dispatch { + if let statusContext = self.statusContexts[resource.id] where statusContext.status != status { + statusContext.status = status + for subscriber in statusContext.subscribers.copyItems() { + subscriber(status) + } + } + } + } + + if dataContext.dataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { + self.dataContexts.removeValueForKey(resource.id) + } + } + } + }) + } + } + + return disposable + } + } + + public func cancelInteractiveResourceFetch(resource: MediaResource) { + self.dataQueue.dispatch { + if let dataContext = self.dataContexts[resource.id] where dataContext.fetchDisposable != nil { + dataContext.fetchDisposable?.dispose() + dataContext.fetchDisposable = nil + + let currentSize = fileSize(self.pathForId(resource.id)) + let status: MediaResourceStatus + if currentSize >= resource.size { + status = .Local + } else { + status = .Remote + } + + self.statusQueue.dispatch { + if let statusContext = self.statusContexts[resource.id] where statusContext.status != status { + statusContext.status = status + for subscriber in statusContext.subscribers.copyItems() { + subscriber(status) + } + } + } + } + } + } } diff --git a/Postbox/MediaCleanupTable.swift b/Postbox/MediaCleanupTable.swift index d86e3c91a7..6fde12ccca 100644 --- a/Postbox/MediaCleanupTable.swift +++ b/Postbox/MediaCleanupTable.swift @@ -1,14 +1,10 @@ import Foundation -final class MediaCleanupTable { - let valueBox: ValueBox - let tableId: Int32 - +final class MediaCleanupTable: Table { var debugMedia: [Media] = [] - init(valueBox: ValueBox, tableId: Int32) { - self.valueBox = valueBox - self.tableId = tableId + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) } func add(media: Media, sharedEncoder: Encoder = Encoder()) { diff --git a/Postbox/MediaResource.swift b/Postbox/MediaResource.swift new file mode 100644 index 0000000000..3d83936ca7 --- /dev/null +++ b/Postbox/MediaResource.swift @@ -0,0 +1,5 @@ + +public protocol MediaResource { + var id: String { get } + var size: Int { get } +} diff --git a/Postbox/MediaResourceData.swift b/Postbox/MediaResourceData.swift new file mode 100644 index 0000000000..7542471cdc --- /dev/null +++ b/Postbox/MediaResourceData.swift @@ -0,0 +1,6 @@ +import Foundation + +public struct MediaResourceData { + public let path: String + public let size: Int +} diff --git a/Postbox/MediaResourceStatus.swift b/Postbox/MediaResourceStatus.swift new file mode 100644 index 0000000000..c73659ea09 --- /dev/null +++ b/Postbox/MediaResourceStatus.swift @@ -0,0 +1,34 @@ +import Foundation +import SwiftSignalKit + +public enum MediaResourceStatus: Equatable { + case Remote + case Local + case Fetching(progress: Float) +} + +public func ==(lhs: MediaResourceStatus, rhs: MediaResourceStatus) -> Bool { + switch lhs { + case .Remote: + switch rhs { + case .Remote: + return true + default: + return false + } + case .Local: + switch rhs { + case .Local: + return true + default: + return false + } + case let .Fetching(lhsProgress): + switch rhs { + case let .Fetching(rhsProgress): + return abs(lhsProgress - rhsProgress) < FLT_EPSILON + default: + return false + } + } +} diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 472182614a..6862f00a30 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -71,7 +71,7 @@ public struct MessageId: Hashable, Comparable, CustomStringConvertible { var array: [MessageId] = [] while i < Int(length) { array[i] = MessageId(buffer) - i++ + i += 1 } return array } @@ -98,7 +98,12 @@ public struct MessageIndex: Equatable, Comparable, Hashable { self.timestamp = message.timestamp } - init(_ message: StoreMessage) { + init(_ message: InternalStoreMessage) { + self.id = message.id + self.timestamp = message.timestamp + } + + init (_ message: IntermediateMessage) { self.id = message.id self.timestamp = message.timestamp } @@ -120,6 +125,14 @@ public struct MessageIndex: Equatable, Comparable, Hashable { return self.id.hashValue } + static func absoluteUpperBound() -> MessageIndex { + return MessageIndex(id: MessageId(peerId: PeerId(namespace: Int32.max, id: Int32.max), namespace: Int32.max, id: Int32.max), timestamp: Int32.max) + } + + static func absoluteLowerBound() -> MessageIndex { + return MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0) + } + static func upperBound(peerId: PeerId) -> MessageIndex { return MessageIndex(id: MessageId(peerId: peerId, namespace: Int32.max, id: Int32.max), timestamp: Int32.max) } @@ -141,56 +154,251 @@ public func <(lhs: MessageIndex, rhs: MessageIndex) -> Bool { return lhs.id.id < rhs.id.id } +public struct MessageTags: OptionSetType { + public var rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public init() { + self.rawValue = 0 + } + + public static let All = MessageTags(rawValue: 0xffffffff) +} + +public struct MessageFlags: OptionSetType { + public var rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public init() { + self.rawValue = 0 + } + + init(_ flags: StoreMessageFlags) { + var rawValue: UInt32 = 0 + + if flags.contains(StoreMessageFlags.Unsent) { + rawValue |= MessageFlags.Unsent.rawValue + } + + if flags.contains(StoreMessageFlags.Failed) { + rawValue |= MessageFlags.Failed.rawValue + } + + if flags.contains(StoreMessageFlags.Incoming) { + rawValue |= MessageFlags.Incoming.rawValue + } + + self.rawValue = rawValue + } + + public static let Unsent = MessageFlags(rawValue: 1) + public static let Failed = MessageFlags(rawValue: 2) + public static let Incoming = MessageFlags(rawValue: 4) +} + +public struct StoreMessageForwardInfo { + public let authorId: PeerId + public let sourceId: PeerId? + public let sourceMessageId: MessageId? + public let date: Int32 + + public init(authorId: PeerId, sourceId: PeerId?, sourceMessageId: MessageId?, date: Int32) { + self.authorId = authorId + self.sourceId = sourceId + self.sourceMessageId = sourceMessageId + self.date = date + } +} + +public struct MessageForwardInfo { + public let author: Peer + public let source: Peer? + public let sourceMessageId: MessageId? + public let date: Int32 +} + +public protocol MessageAttribute: Coding { + var associatedPeerIds: [PeerId] { get } + var associatedMessageIds: [MessageId] { get } +} + +public extension MessageAttribute { + var associatedPeerIds: [PeerId] { + return [] + } + + var associatedMessageIds: [MessageId] { + return [] + } +} + public class Message { + public let stableId: UInt32 public let id: MessageId public let timestamp: Int32 + public let flags: MessageFlags + public let tags: MessageTags + public let forwardInfo: MessageForwardInfo? public let author: Peer? public let text: String - public let attributes: [Coding] + public let attributes: [MessageAttribute] public let media: [Media] public let peers: SimpleDictionary + public let associatedMessages: SimpleDictionary - init(id: MessageId, timestamp: Int32, author: Peer?, text: String, attributes: [Coding], media: [Media], peers: SimpleDictionary) { + init(stableId: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: MessageForwardInfo?, author: Peer?, text: String, attributes: [MessageAttribute], media: [Media], peers: SimpleDictionary, associatedMessages: SimpleDictionary) { + self.stableId = stableId self.id = id self.timestamp = timestamp + self.flags = flags + self.tags = tags + self.forwardInfo = forwardInfo self.author = author self.text = text self.attributes = attributes self.media = media self.peers = peers + self.associatedMessages = associatedMessages + } +} + +public struct StoreMessageFlags: OptionSetType { + public var rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public init() { + self.rawValue = 0 + } + + public static let Unsent = StoreMessageFlags(rawValue: 1) + public static let Failed = StoreMessageFlags(rawValue: 2) + public static let Incoming = StoreMessageFlags(rawValue: 4) +} + +public enum StoreMessageId { + case Id(MessageId) + case Partial(PeerId, MessageId.Namespace) + + public var peerId: PeerId { + switch self { + case let .Id(id): + return id.peerId + case let .Partial(peerId, _): + return peerId + } } } public final class StoreMessage { - public let id: MessageId + public let id: StoreMessageId public let timestamp: Int32 + public let flags: StoreMessageFlags + public let tags: MessageTags + public let forwardInfo: StoreMessageForwardInfo? public let authorId: PeerId? public let text: String - public let attributes: [Coding] + public let attributes: [MessageAttribute] public let media: [Media] - public init(id: MessageId, timestamp: Int32, authorId: PeerId?, text: String, attributes: [Coding], media: [Media]) { - self.id = id - self.authorId = authorId + public init(id: MessageId, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + self.id = .Id(id) self.timestamp = timestamp + self.flags = flags + self.tags = tags + self.forwardInfo = forwardInfo + self.authorId = authorId + self.text = text + self.attributes = attributes + self.media = media + } + + public init(peerId: PeerId, namespace: MessageId.Namespace, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + self.id = .Partial(peerId, namespace) + self.timestamp = timestamp + self.flags = flags + self.tags = tags + self.forwardInfo = forwardInfo + self.authorId = authorId self.text = text self.attributes = attributes self.media = media } } -class IntermediateMessage { +final class InternalStoreMessage { let id: MessageId let timestamp: Int32 + let flags: StoreMessageFlags + let tags: MessageTags + let forwardInfo: StoreMessageForwardInfo? + let authorId: PeerId? + let text: String + let attributes: [MessageAttribute] + let media: [Media] + + init(id: MessageId, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + self.id = id + self.timestamp = timestamp + self.flags = flags + self.tags = tags + self.forwardInfo = forwardInfo + self.authorId = authorId + self.text = text + self.attributes = attributes + self.media = media + } +} + +struct IntermediateMessageForwardInfo { + let authorId: PeerId + let sourceId: PeerId? + let sourceMessageId: MessageId? + let date: Int32 + + init(authorId: PeerId, sourceId: PeerId?, sourceMessageId: MessageId?, date: Int32) { + self.authorId = authorId + self.sourceId = sourceId + self.sourceMessageId = sourceMessageId + self.date = date + } + + init(_ storeInfo: StoreMessageForwardInfo) { + self.authorId = storeInfo.authorId + self.sourceId = storeInfo.sourceId + self.sourceMessageId = storeInfo.sourceMessageId + self.date = storeInfo.date + } +} + +class IntermediateMessage { + let stableId: UInt32 + let id: MessageId + let timestamp: Int32 + let flags: MessageFlags + let tags: MessageTags + let forwardInfo: IntermediateMessageForwardInfo? let authorId: PeerId? let text: String let attributesData: ReadBuffer let embeddedMediaData: ReadBuffer let referencedMedia: [MediaId] - init(id: MessageId, timestamp: Int32, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { + init(stableId: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { + self.stableId = stableId self.id = id self.timestamp = timestamp + self.flags = flags + self.tags = tags + self.forwardInfo = forwardInfo self.authorId = authorId self.text = text self.attributesData = attributesData diff --git a/Postbox/MessageHistoryHole.swift b/Postbox/MessageHistoryHole.swift index b360771793..03e6fd7867 100644 --- a/Postbox/MessageHistoryHole.swift +++ b/Postbox/MessageHistoryHole.swift @@ -1,8 +1,17 @@ import Foundation public struct MessageHistoryHole: Equatable, CustomStringConvertible { + public let stableId: UInt32 public let maxIndex: MessageIndex public let min: MessageId.Id + let tags: UInt32 + + init(stableId: UInt32, maxIndex: MessageIndex, min: MessageId.Id, tags: UInt32) { + self.stableId = stableId + self.maxIndex = maxIndex + self.min = min + self.tags = tags + } var id: MessageId { return maxIndex.id @@ -14,5 +23,5 @@ public struct MessageHistoryHole: Equatable, CustomStringConvertible { } public func ==(lhs: MessageHistoryHole, rhs: MessageHistoryHole) -> Bool { - return lhs.maxIndex == rhs.maxIndex && lhs.min == rhs.min + return lhs.maxIndex == rhs.maxIndex && lhs.min == rhs.min && lhs.tags == rhs.tags } diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 228e1f7d84..c3b467e79a 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -26,9 +26,10 @@ public enum AddMessagesLocation { } enum MessageHistoryIndexOperation { - case InsertMessage(StoreMessage) + case InsertMessage(InternalStoreMessage) case InsertHole(MessageHistoryHole) case Remove(MessageIndex) + case Update(MessageIndex, InternalStoreMessage) } private func readHistoryIndexEntry(peerId: PeerId, namespace: MessageId.Namespace, key: ValueBoxKey, value: ReadBuffer) -> HistoryIndexEntry { @@ -41,23 +42,34 @@ private func readHistoryIndexEntry(peerId: PeerId, namespace: MessageId.Namespac if type == 0 { return .Message(index) } else { + var stableId: UInt32 = 0 + value.read(&stableId, offset: 0, length: 4) + var min: Int32 = 0 value.read(&min, offset: 0, length: 4) - return .Hole(MessageHistoryHole(maxIndex: index, min: min)) + + var tags: UInt32 = 0 + value.read(&tags, offset: 0, length: 4) + + return .Hole(MessageHistoryHole(stableId: stableId, maxIndex: index, min: min, tags: tags)) } } -final class MessageHistoryIndexTable { - let valueBox: ValueBox - let tableId: Int32 +final class MessageHistoryIndexTable: Table { let globalMessageIdsNamespace: Int32 let globalMessageIdsTable: GlobalMessageIdsTable + let metadataTable: MessageHistoryMetadataTable + let seedConfiguration: SeedConfiguration - init(valueBox: ValueBox, tableId: Int32, globalMessageIdsTable: GlobalMessageIdsTable) { - self.valueBox = valueBox - self.tableId = tableId + var cachedMaxEntryByPeerId: [PeerId: [MessageId.Namespace: ValueBoxKey]] = [:] + + init(valueBox: ValueBox, tableId: Int32, globalMessageIdsTable: GlobalMessageIdsTable, metadataTable: MessageHistoryMetadataTable, seedConfiguration: SeedConfiguration) { self.globalMessageIdsTable = globalMessageIdsTable self.globalMessageIdsNamespace = globalMessageIdsTable.namespace + self.seedConfiguration = seedConfiguration + self.metadataTable = metadataTable + + super.init(valueBox: valueBox, tableId: tableId) } private func key(id: MessageId) -> ValueBoxKey { @@ -82,50 +94,70 @@ final class MessageHistoryIndexTable { return key.successor } + private func ensureInitialized(peerId: PeerId, inout operations: [MessageHistoryIndexOperation]) { + if !self.metadataTable.isInitialized(peerId) { + for namespace in self.seedConfiguration.initializeMessageNamespacesWithHoles { + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: Int32.max), timestamp: Int32.max), min: 1, tags: MessageTags.All.rawValue), operations: &operations) + } + + self.metadataTable.setInitialized(peerId) + } + } + func addHole(id: MessageId, inout operations: [MessageHistoryIndexOperation]) { + self.ensureInitialized(id.peerId, operations: &operations) + let adjacent = self.adjacentItems(id) - if let lowerItem = adjacent.lower, upperItem = adjacent.upper { + if let lowerItem = adjacent.lower { switch lowerItem { - case .Hole: - break - case let .Message(lowerMessage): - switch upperItem { - case .Hole: - break - case let .Message(upperMessage): - if lowerMessage.id.id < upperMessage.id.id - 1 { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerMessage.id.id + 1), operations: &operations) - } - break + case let .Hole(lowerHole): + if lowerHole.tags != MessageTags.All.rawValue { + self.justRemove(lowerHole.maxIndex, operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: lowerHole.stableId, maxIndex: lowerHole.maxIndex, min: lowerHole.min, tags: MessageTags.All.rawValue), operations: &operations) } - } - } else if let lowerItem = adjacent.lower { - switch lowerItem { case let .Message(lowerMessage): - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerMessage.id.id + 1), operations: &operations) - case .Hole: - break + if let upperItem = adjacent.upper { + switch upperItem { + case .Hole: + break + case let .Message(upperMessage): + if lowerMessage.id.id < upperMessage.id.id - 1 { + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerMessage.id.id + 1, tags: MessageTags.All.rawValue), operations: &operations) + } + break + } + } else { + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerMessage.id.id + 1, tags: MessageTags.All.rawValue), operations: &operations) + } } } else if let upperItem = adjacent.upper { switch upperItem { case let .Message(upperMessage): if upperMessage.id.id > 1 { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: 1), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: 1, tags: MessageTags.All.rawValue), operations: &operations) } case .Hole: break } } else { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: 1), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: 1, tags: MessageTags.All.rawValue), operations: &operations) } } - func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operations: [MessageHistoryIndexOperation]) { + func addMessages(messages: [InternalStoreMessage], location: AddMessagesLocation, inout operations: [MessageHistoryIndexOperation]) { if messages.count == 0 { return } + var seenPeerIds = Set() + for message in messages { + if !seenPeerIds.contains(message.id.peerId) { + seenPeerIds.insert(message.id.peerId) + self.ensureInitialized(message.id.peerId, operations: &operations) + } + } + switch location { case .UpperHistoryBlock: var lowerIds = SimpleDictionary>() @@ -146,13 +178,15 @@ final class MessageHistoryIndexTable { for (namespace, lowerIndex) in lowerIdsByNamespace { var removeHoles: [MessageIndex] = [] var modifyHole: (MessageIndex, MessageHistoryHole)? - self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: lowerIndex.id.id)), end: self.upperBound(peerId, namespace: namespace), values: { key, value in + let startKey = self.key(MessageId(peerId: peerId, namespace: namespace, id: lowerIndex.id.id)) + + self.valueBox.range(self.tableId, start: startKey, end: self.upperBound(peerId, namespace: namespace), values: { key, value in let entry = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value) if case let .Hole(hole) = entry { if lowerIndex.id.id <= hole.min { removeHoles.append(hole.maxIndex) } else { - modifyHole = (hole.maxIndex, MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: lowerIndex.id.id - 1), timestamp: lowerIndex.timestamp), min: hole.min)) + modifyHole = (hole.maxIndex, MessageHistoryHole(stableId: hole.stableId, maxIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: lowerIndex.id.id - 1), timestamp: lowerIndex.timestamp), min: hole.min, tags: hole.tags)) } } return true @@ -188,10 +222,10 @@ final class MessageHistoryIndexTable { case let .Hole(upperHole): self.justRemove(upperHole.maxIndex, operations: &operations) if upperHole.maxIndex.id.id > index.id.id + 1 { - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: index.id.id + 1), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: upperHole.maxIndex, min: index.id.id + 1, tags: upperHole.tags), operations: &operations) } if upperHole.min <= index.id.id - 1 { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: index.id.peerId, namespace: index.id.namespace, id: index.id.id - 1), timestamp: index.timestamp), min: upperHole.min), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: index.id.peerId, namespace: index.id.namespace, id: index.id.id - 1), timestamp: index.timestamp), min: upperHole.min, tags: upperHole.tags), operations: &operations) } case let .Message(messageIndex): if messageIndex.id == index.id { @@ -207,6 +241,8 @@ final class MessageHistoryIndexTable { } func removeMessage(id: MessageId, inout operations: [MessageHistoryIndexOperation]) { + self.ensureInitialized(id.peerId, operations: &operations) + if let existingEntry = self.get(id) { self.justRemove(existingEntry.index, operations: &operations) @@ -218,7 +254,7 @@ final class MessageHistoryIndexTable { switch upperItem { case let .Hole(upperHole): self.justRemove(upperHole.maxIndex, operations: &operations) - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerMessage.id.id + 1), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: upperHole.stableId, maxIndex: upperHole.maxIndex, min: lowerMessage.id.id + 1, tags: upperHole.tags), operations: &operations) case .Message: break } @@ -227,17 +263,17 @@ final class MessageHistoryIndexTable { case let .Hole(upperHole): self.justRemove(lowerHole.maxIndex, operations: &operations) self.justRemove(upperHole.maxIndex, operations: &operations) - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerHole.min), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: upperHole.stableId, maxIndex: upperHole.maxIndex, min: lowerHole.min, tags: upperHole.tags | lowerHole.tags), operations: &operations) case let .Message(upperMessage): self.justRemove(lowerHole.maxIndex, operations: &operations) - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerHole.min), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: lowerHole.stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerHole.min, tags: lowerHole.tags), operations: &operations) } } } else if let lowerItem = adjacent.lower { switch lowerItem { case let .Hole(lowerHole): self.justRemove(lowerHole.maxIndex, operations: &operations) - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerHole.min), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: lowerHole.stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerHole.min, tags: lowerHole.tags), operations: &operations) break case .Message: break @@ -246,7 +282,7 @@ final class MessageHistoryIndexTable { switch upperItem { case let .Hole(upperHole): self.justRemove(upperHole.maxIndex, operations: &operations) - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: 1), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: upperHole.stableId, maxIndex: upperHole.maxIndex, min: 1, tags: upperHole.tags), operations: &operations) break case .Message: break @@ -255,7 +291,32 @@ final class MessageHistoryIndexTable { } } - func fillHole(id: MessageId, fillType: HoleFillType, messages: [StoreMessage], inout operations: [MessageHistoryIndexOperation]) { + func updateMessage(id: MessageId, message: InternalStoreMessage, inout operations: [MessageHistoryIndexOperation]) { + if let previousEntry = self.get(id), case let .Message(previousIndex) = previousEntry { + if previousIndex != MessageIndex(message) { + var intermediateOperations: [MessageHistoryIndexOperation] = [] + self.removeMessage(id, operations: &intermediateOperations) + self.addMessages([message], location: .Random, operations: &intermediateOperations) + + for operation in intermediateOperations { + switch operation { + case let .Remove(index) where index == previousIndex: + operations.append(.Update(previousIndex, message)) + case let .InsertMessage(insertMessage) where MessageIndex(insertMessage) == MessageIndex(message): + break + default: + operations.append(operation) + } + } + } else { + operations.append(.Update(previousIndex, message)) + } + } + } + + func fillHole(id: MessageId, fillType: HoleFillType, tagMask: MessageTags?, messages: [InternalStoreMessage], inout operations: [MessageHistoryIndexOperation]) { + self.ensureInitialized(id.peerId, operations: &operations) + var upperItem: HistoryIndexEntry? self.valueBox.range(self.tableId, start: self.key(id).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) @@ -263,53 +324,132 @@ final class MessageHistoryIndexTable { }, limit: 1) let sortedByIdMessages = messages.sort({$0.id < $1.id}) + var remainingMessages = sortedByIdMessages if let upperItem = upperItem { switch upperItem { case let .Hole(upperHole): - var i = 0 - var minMessageInRange: StoreMessage? - var maxMessageInRange: StoreMessage? - var removedHole = false - while i < remainingMessages.count { - let message = remainingMessages[i] - if message.id.id >= upperHole.min && message.id.id <= upperHole.maxIndex.id.id { - if (fillType == .UpperToLower || fillType == .Complete) && (minMessageInRange == nil || minMessageInRange!.id > message.id) { - minMessageInRange = message - if !removedHole { - removedHole = true - self.justRemove(upperHole.maxIndex, operations: &operations) - } + if let tagMask = tagMask { + var messagesInRange: [InternalStoreMessage] = [] + var i = 0 + while i < remainingMessages.count { + let message = remainingMessages[i] + if message.id.id >= upperHole.min && message.id.id <= upperHole.maxIndex.id.id { + messagesInRange.append(message) + remainingMessages.removeAtIndex(i) + } else { + i += 1 } - if (fillType == .LowerToUpper || fillType == .Complete) && (maxMessageInRange == nil || maxMessageInRange!.id < message.id) { - maxMessageInRange = message - if !removedHole { - removedHole = true - self.justRemove(upperHole.maxIndex, operations: &operations) - } - } - self.justInsertMessage(message, operations: &operations) - remainingMessages.removeAtIndex(i) - } else { - i++ } - } - switch fillType { - case .Complete: - if !removedHole { - self.justRemove(upperHole.maxIndex, operations: &operations) + + if messagesInRange.isEmpty { + self.justRemove(upperHole.maxIndex, operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: upperHole.stableId, maxIndex: upperHole.maxIndex, min: upperHole.min, tags: upperHole.tags & ~tagMask.rawValue), operations: &operations) + } else { + self.justRemove(upperHole.maxIndex, operations: &operations) + + let clearedTags = upperHole.tags & ~tagMask.rawValue + + for i in 0 ..< messagesInRange.count { + let message = messagesInRange[i] + + if i == 0 { + if upperHole.min < message.id.id { + let holeTags: UInt32 + switch fillType { + case .LowerToUpper, .Complete: + holeTags = clearedTags + case .UpperToLower: + holeTags = upperHole.tags + } + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: message.id.id - 1), timestamp: message.timestamp), min: upperHole.min, tags: holeTags), operations: &operations) + } + } else { + let previousMessageId = messagesInRange[i - 1].id.id + if previousMessageId + 1 < message.id.id { + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: message.id.id - 1), timestamp: message.timestamp), min: previousMessageId + 1, tags: clearedTags), operations: &operations) + } + } + + if i == messagesInRange.count - 1 { + if upperHole.maxIndex.id.id > message.id.id { + let holeTags: UInt32 + switch fillType { + case .LowerToUpper: + holeTags = upperHole.tags + case .UpperToLower, .Complete: + holeTags = clearedTags + } + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: upperHole.maxIndex, min: message.id.id + 1, tags: holeTags), operations: &operations) + } + } + + self.justInsertMessage(message, operations: &operations) } - case .LowerToUpper: - if let maxMessageInRange = maxMessageInRange where maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { - self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: maxMessageInRange.id.id + 1), operations: &operations) + } + } else { + var i = 0 + var minMessageInRange: InternalStoreMessage? + var maxMessageInRange: InternalStoreMessage? + var removedHole = false + while i < remainingMessages.count { + let message = remainingMessages[i] + if message.id.id >= upperHole.min && message.id.id <= upperHole.maxIndex.id.id { + if (fillType == .UpperToLower || fillType == .Complete) && (minMessageInRange == nil || minMessageInRange!.id > message.id) { + minMessageInRange = message + if !removedHole { + removedHole = true + self.justRemove(upperHole.maxIndex, operations: &operations) + } + } + if (fillType == .LowerToUpper || fillType == .Complete) && (maxMessageInRange == nil || maxMessageInRange!.id < message.id) { + maxMessageInRange = message + if !removedHole { + removedHole = true + self.justRemove(upperHole.maxIndex, operations: &operations) + } + } + self.justInsertMessage(message, operations: &operations) + remainingMessages.removeAtIndex(i) + } else { + i += 1 } - case .UpperToLower: - if let minMessageInRange = minMessageInRange where minMessageInRange.id.id - 1 >= upperHole.min { - self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minMessageInRange.id.id - 1), timestamp: minMessageInRange.timestamp), min: upperHole.min), operations: &operations) - } - } - break + } + switch fillType { + case .Complete: + if !removedHole { + removedHole = true + self.justRemove(upperHole.maxIndex, operations: &operations) + } + case .LowerToUpper: + if let maxMessageInRange = maxMessageInRange where maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { + let stableId: UInt32 + let tags: UInt32 + if removedHole { + stableId = upperHole.stableId + tags = upperHole.tags + } else { + stableId = self.metadataTable.getNextStableMessageIndexId() + tags = MessageTags.All.rawValue + } + self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: upperHole.maxIndex, min: maxMessageInRange.id.id + 1, tags: tags), operations: &operations) + } + case .UpperToLower: + if let minMessageInRange = minMessageInRange where minMessageInRange.id.id - 1 >= upperHole.min { + let stableId: UInt32 + let tags: UInt32 + if removedHole { + stableId = upperHole.stableId + tags = upperHole.tags + } else { + stableId = self.metadataTable.getNextStableMessageIndexId() + tags = MessageTags.All.rawValue + } + self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minMessageInRange.id.id - 1), timestamp: minMessageInRange.timestamp), min: upperHole.min, tags: tags), operations: &operations) + } + } + } case .Message: break } @@ -320,20 +460,24 @@ final class MessageHistoryIndexTable { } } - func justInsertHole(hole: MessageHistoryHole, inout operations: [MessageHistoryIndexOperation]) { + private func justInsertHole(hole: MessageHistoryHole, inout operations: [MessageHistoryIndexOperation]) { let value = WriteBuffer() var type: Int8 = 1 var timestamp: Int32 = hole.maxIndex.timestamp var min: Int32 = hole.min + var tags: UInt32 = hole.tags value.write(&type, offset: 0, length: 1) value.write(×tamp, offset: 0, length: 4) + var stableId: UInt32 = hole.stableId + value.write(&stableId, offset: 0, length: 4) value.write(&min, offset: 0, length: 4) + value.write(&tags, offset: 0, length: 4) self.valueBox.set(self.tableId, key: self.key(hole.id), value: value) operations.append(.InsertHole(hole)) } - func justInsertMessage(message: StoreMessage, inout operations: [MessageHistoryIndexOperation]) { + private func justInsertMessage(message: InternalStoreMessage, inout operations: [MessageHistoryIndexOperation]) { let index = MessageIndex(id: message.id, timestamp: message.timestamp) let value = WriteBuffer() @@ -349,7 +493,7 @@ final class MessageHistoryIndexTable { } } - func justRemove(index: MessageIndex, inout operations: [MessageHistoryIndexOperation]) { + private func justRemove(index: MessageIndex, inout operations: [MessageHistoryIndexOperation]) { self.valueBox.remove(self.tableId, key: self.key(index.id)) operations.append(.Remove(index)) @@ -358,7 +502,7 @@ final class MessageHistoryIndexTable { } } - func adjacentItems(id: MessageId) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) { + private func adjacentItems(id: MessageId) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) { let key = self.key(id) var lowerItem: HistoryIndexEntry? @@ -377,6 +521,9 @@ final class MessageHistoryIndexTable { } func get(id: MessageId) -> HistoryIndexEntry? { + var operations: [MessageHistoryIndexOperation] = [] + self.ensureInitialized(id.peerId, operations: &operations) + let key = self.key(id) if let value = self.valueBox.get(self.tableId, key: key) { return readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) @@ -384,6 +531,10 @@ final class MessageHistoryIndexTable { return nil } + func exists(id: MessageId) -> Bool { + return self.valueBox.exists(self.tableId, key: self.key(id)) + } + func debugList(peerId: PeerId, namespace: MessageId.Namespace) -> [HistoryIndexEntry] { var list: [HistoryIndexEntry] = [] self.valueBox.range(self.tableId, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in diff --git a/Postbox/MessageHistoryMetadataTable.swift b/Postbox/MessageHistoryMetadataTable.swift new file mode 100644 index 0000000000..ef0368eda1 --- /dev/null +++ b/Postbox/MessageHistoryMetadataTable.swift @@ -0,0 +1,169 @@ +import Foundation + +private enum MetadataPrefix: Int8 { + case ChatListInitialized = 0 + case PeerHistoryInitialized = 1 + case PeerNextMessageIdByNamespace = 2 + case NextStableMessageId = 3 +} + +final class MessageHistoryMetadataTable: Table { + let sharedPeerHistoryInitializedKey = ValueBoxKey(length: 8 + 1) + let sharedPeerNextMessageIdByNamespaceKey = ValueBoxKey(length: 8 + 1 + 4) + let sharedBuffer = WriteBuffer() + + private var initializedChatList: Bool? + private var initializedHistoryPeerIds = Set() + + private var peerNextMessageIdByNamespace: [PeerId: [MessageId.Namespace: MessageId.Id]] = [:] + private var updatedPeerNextMessageIdByNamespace: [PeerId: Set] = [:] + + private var nextMessageStableId: UInt32? + private var nextMessageStableIdUpdated = false + + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) + } + + private func peerHistoryInitializedKey(id: PeerId) -> ValueBoxKey { + self.sharedPeerHistoryInitializedKey.setInt64(0, value: id.toInt64()) + self.sharedPeerHistoryInitializedKey.setInt8(8, value: MetadataPrefix.PeerHistoryInitialized.rawValue) + return self.sharedPeerHistoryInitializedKey + } + + private func peerNextMessageIdByNamespaceKey(id: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + self.sharedPeerNextMessageIdByNamespaceKey.setInt64(0, value: id.toInt64()) + self.sharedPeerNextMessageIdByNamespaceKey.setInt8(8, value: MetadataPrefix.PeerNextMessageIdByNamespace.rawValue) + self.sharedPeerNextMessageIdByNamespaceKey.setInt32(8 + 1, value: namespace) + + return self.sharedPeerNextMessageIdByNamespaceKey + } + + private func key(prefix: MetadataPrefix) -> ValueBoxKey { + let key = ValueBoxKey(length: 1) + key.setInt8(0, value: prefix.rawValue) + return key + } + + func setInitializedChatList() { + self.initializedChatList = true + self.valueBox.set(self.tableId, key: self.key(MetadataPrefix.ChatListInitialized), value: MemoryBuffer(memory: nil, capacity: 0, length: 0, freeWhenDone: false)) + } + + func isInitializedChatList() -> Bool { + if let initializedChatList = self.initializedChatList { + return initializedChatList + } + + if self.valueBox.get(self.tableId, key: self.key(MetadataPrefix.ChatListInitialized)) != nil { + self.initializedChatList = true + return true + } + + return false + } + + func setInitialized(peerId: PeerId) { + self.initializedHistoryPeerIds.insert(peerId) + self.sharedBuffer.reset() + self.valueBox.set(self.tableId, key: self.peerHistoryInitializedKey(peerId), value: self.sharedBuffer) + } + + func isInitialized(peerId: PeerId) -> Bool { + if self.initializedHistoryPeerIds.contains(peerId) { + return true + } else { + if self.valueBox.exists(self.tableId, key: self.peerHistoryInitializedKey(peerId)) { + self.initializedHistoryPeerIds.insert(peerId) + return true + } else { + return false + } + } + } + + func getNextMessageIdAndIncrement(peerId: PeerId, namespace: MessageId.Namespace) -> MessageId { + if let messageIdByNamespace = self.peerNextMessageIdByNamespace[peerId] { + if let nextId = messageIdByNamespace[namespace] { + self.peerNextMessageIdByNamespace[peerId]![namespace] = nextId + 1 + if updatedPeerNextMessageIdByNamespace[peerId] != nil { + updatedPeerNextMessageIdByNamespace[peerId]!.insert(namespace) + } else { + updatedPeerNextMessageIdByNamespace[peerId] = Set([namespace]) + } + return MessageId(peerId: peerId, namespace: namespace, id: nextId) + } else { + var nextId: Int32 = 1 + if let value = self.valueBox.get(self.tableId, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) { + value.read(&nextId, offset: 0, length: 4) + } + self.peerNextMessageIdByNamespace[peerId]![namespace] = nextId + 1 + if updatedPeerNextMessageIdByNamespace[peerId] != nil { + updatedPeerNextMessageIdByNamespace[peerId]!.insert(namespace) + } else { + updatedPeerNextMessageIdByNamespace[peerId] = Set([namespace]) + } + return MessageId(peerId: peerId, namespace: namespace, id: nextId) + } + } else { + var nextId: Int32 = 1 + if let value = self.valueBox.get(self.tableId, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) { + value.read(&nextId, offset: 0, length: 4) + } + + self.peerNextMessageIdByNamespace[peerId] = [namespace: nextId + 1] + if updatedPeerNextMessageIdByNamespace[peerId] != nil { + updatedPeerNextMessageIdByNamespace[peerId]!.insert(namespace) + } else { + updatedPeerNextMessageIdByNamespace[peerId] = Set([namespace]) + } + return MessageId(peerId: peerId, namespace: namespace, id: nextId) + } + } + + func getNextStableMessageIndexId() -> UInt32 { + if let nextId = self.nextMessageStableId { + self.nextMessageStableId = nextId + 1 + self.nextMessageStableIdUpdated = true + return nextId + } else { + if let value = self.valueBox.get(self.tableId, key: self.key(.NextStableMessageId)) { + var nextId: UInt32 = 0 + value.read(&nextId, offset: 0, length: 4) + self.nextMessageStableId = nextId + 1 + self.nextMessageStableIdUpdated = true + return nextId + } else { + let nextId: UInt32 = 1 + self.nextMessageStableId = nextId + 1 + self.nextMessageStableIdUpdated = true + return nextId + } + } + } + + override func beforeCommit() { + let sharedBuffer = WriteBuffer() + for (peerId, namespaces) in self.updatedPeerNextMessageIdByNamespace { + for namespace in namespaces { + if let messageIdByNamespace = self.peerNextMessageIdByNamespace[peerId], maxId = messageIdByNamespace[namespace] { + sharedBuffer.reset() + var mutableMaxId = maxId + sharedBuffer.write(&mutableMaxId, offset: 0, length: 4) + self.valueBox.set(self.tableId, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace), value: sharedBuffer) + } else { + self.valueBox.remove(self.tableId, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) + } + } + } + self.updatedPeerNextMessageIdByNamespace.removeAll() + + if self.nextMessageStableIdUpdated { + if let nextMessageStableId = self.nextMessageStableId { + var nextId: UInt32 = nextMessageStableId + self.valueBox.set(self.tableId, key: self.key(.NextStableMessageId), value: MemoryBuffer(memory: &nextId, capacity: 4, length: 4, freeWhenDone: false)) + self.nextMessageStableIdUpdated = false + } + } + } +} \ No newline at end of file diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index f649976f91..08c74c737d 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -34,18 +34,21 @@ enum RenderedMessageHistoryEntry { } } -final class MessageHistoryTable { - let valueBox: ValueBox - let tableId: Int32 - +final class MessageHistoryTable: Table { let messageHistoryIndexTable: MessageHistoryIndexTable let messageMediaTable: MessageMediaTable + let historyMetadataTable: MessageHistoryMetadataTable + let unsentTable: MessageHistoryUnsentTable + let tagsTable: MessageHistoryTagsTable - init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable) { - self.valueBox = valueBox - self.tableId = tableId + init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable) { self.messageHistoryIndexTable = messageHistoryIndexTable self.messageMediaTable = messageMediaTable + self.historyMetadataTable = historyMetadataTable + self.unsentTable = unsentTable + self.tagsTable = tagsTable + + super.init(valueBox: valueBox, tableId: tableId) } private func key(index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4)) -> ValueBoxKey { @@ -98,7 +101,7 @@ final class MessageHistoryTable { return dict } - private func processIndexOperations(peerId: PeerId, operations: [MessageHistoryIndexOperation], inout processedOperationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + private func processIndexOperations(peerId: PeerId, operations: [MessageHistoryIndexOperation], inout processedOperationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0)) let sharedBuffer = WriteBuffer() let sharedEncoder = Encoder() @@ -107,23 +110,62 @@ final class MessageHistoryTable { var accumulatedRemoveIndices: [MessageIndex] = [] for operation in operations { switch operation { - case let .InsertHole(hole): - if accumulatedRemoveIndices.count != 0 { - outputOperations.append(.Remove(accumulatedRemoveIndices)) - accumulatedRemoveIndices.removeAll() - } - self.justInsertHole(hole) - outputOperations.append(.InsertHole(hole)) - case let .InsertMessage(storeMessage): - if accumulatedRemoveIndices.count != 0 { - outputOperations.append(.Remove(accumulatedRemoveIndices)) - accumulatedRemoveIndices.removeAll() - } - let message = self.justInsertMessage(storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder) - outputOperations.append(.InsertMessage(message)) - case let .Remove(index): - self.justRemove(index) - accumulatedRemoveIndices.append(index) + case let .InsertHole(hole): + if accumulatedRemoveIndices.count != 0 { + outputOperations.append(.Remove(accumulatedRemoveIndices)) + accumulatedRemoveIndices.removeAll() + } + self.justInsertHole(hole) + outputOperations.append(.InsertHole(hole)) + + let tags = self.messageHistoryIndexTable.seedConfiguration.existingMessageTags.rawValue & hole.tags + for i in 0 ..< 32 { + let currentTags = tags >> UInt32(i) + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + let tag = MessageTags(rawValue: 1 << UInt32(i)) + self.tagsTable.add(tag, index: hole.maxIndex) + } + } + case let .InsertMessage(storeMessage): + if accumulatedRemoveIndices.count != 0 { + outputOperations.append(.Remove(accumulatedRemoveIndices)) + accumulatedRemoveIndices.removeAll() + } + let message = self.justInsertMessage(storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder) + outputOperations.append(.InsertMessage(message)) + if message.flags.contains(.Unsent) && !message.flags.contains(.Failed) { + self.unsentTable.add(MessageIndex(message), operations: &unsentMessageOperations) + } + let tags = message.tags.rawValue + if tags != 0 { + for i in 0 ..< 32 { + let currentTags = tags >> UInt32(i) + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + let tag = MessageTags(rawValue: 1 << UInt32(i)) + self.tagsTable.add(tag, index: MessageIndex(message)) + } + } + } + case let .Remove(index): + self.justRemove(index, unsentMessageOperations: &unsentMessageOperations) + accumulatedRemoveIndices.append(index) + case let .Update(index, storeMessage): + accumulatedRemoveIndices.append(index) + if accumulatedRemoveIndices.count != 0 { + outputOperations.append(.Remove(accumulatedRemoveIndices)) + accumulatedRemoveIndices.removeAll() + } + if let message = self.justUpdate(index, message: storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder, unsentMessageOperations: &unsentMessageOperations) { + outputOperations.append(.InsertMessage(message)) + } } } if accumulatedRemoveIndices.count != 0 { @@ -137,46 +179,66 @@ final class MessageHistoryTable { } } - func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + private func internalStoreMessages(messages: [StoreMessage]) -> [InternalStoreMessage] { + var internalStoreMessages: [InternalStoreMessage] = [] + for message in messages { + switch message.id { + case let .Id(id): + internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) + case let .Partial(peerId, namespace): + let id = self.historyMetadataTable.getNextMessageIdAndIncrement(peerId, namespace: namespace) + internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) + } + } + return internalStoreMessages + } + + func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { let messagesByPeerId = self.messagesGroupedByPeerId(messages) for (peerId, peerMessages) in messagesByPeerId { var operations: [MessageHistoryIndexOperation] = [] - self.messageHistoryIndexTable.addMessages(peerMessages, location: location, operations: &operations) - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId) + self.messageHistoryIndexTable.addMessages(self.internalStoreMessages(peerMessages), location: location, operations: &operations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) } } - func addHoles(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + func addHoles(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] for id in messageIds { self.messageHistoryIndexTable.addHole(id, operations: &operations) } - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) } } - func removeMessages(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + func removeMessages(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] for id in messageIds { self.messageHistoryIndexTable.removeMessage(id, operations: &operations) } - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) } } - func fillHole(id: MessageId, fillType: HoleFillType, messages: [StoreMessage], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + func fillHole(id: MessageId, fillType: HoleFillType, tagMask: MessageTags?, messages: [StoreMessage], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { var operations: [MessageHistoryIndexOperation] = [] - self.messageHistoryIndexTable.fillHole(id, fillType: fillType, messages: messages, operations: &operations) - self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId) + self.messageHistoryIndexTable.fillHole(id, fillType: fillType, tagMask: tagMask, messages: self.internalStoreMessages(messages), operations: &operations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + } + + func updateMessage(id: MessageId, message: StoreMessage, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { + var operations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.updateMessage(id, message: self.internalStoreMessages([message]).first!, operations: &operations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) } func topMessage(peerId: PeerId) -> IntermediateMessage? { - var currentKey = self.lowerBound(peerId) + var currentKey = self.upperBound(peerId) while true { var entry: IntermediateMessageHistoryEntry? - self.valueBox.range(self.tableId, start: self.upperBound(peerId), end: currentKey, values: { key, value in + self.valueBox.range(self.tableId, start: currentKey, end: self.lowerBound(peerId), values: { key, value in entry = self.readIntermediateEntry(key, value: value) return true }, limit: 1) @@ -195,12 +257,67 @@ final class MessageHistoryTable { return nil } - private func justInsertMessage(message: StoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) -> IntermediateMessage { + func getMessage(index: MessageIndex) -> IntermediateMessage? { + let key = self.key(index) + if let value = self.valueBox.get(self.tableId, key: key) { + let entry = self.readIntermediateEntry(key, value: value) + if case let .Message(message) = entry { + return message + } + } + return nil + } + + private func justInsertMessage(message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) -> IntermediateMessage { sharedBuffer.reset() var type: Int8 = 0 sharedBuffer.write(&type, offset: 0, length: 1) + var stableId: UInt32 = self.historyMetadataTable.getNextStableMessageIndexId() + sharedBuffer.write(&stableId, offset: 0, length: 4) + + var flags = MessageFlags(message.flags) + sharedBuffer.write(&flags.rawValue, offset: 0, length: 4) + + var tags = message.tags + sharedBuffer.write(&tags.rawValue, offset: 0, length: 4) + + var intermediateForwardInfo: IntermediateMessageForwardInfo? + if let forwardInfo = message.forwardInfo { + intermediateForwardInfo = IntermediateMessageForwardInfo(forwardInfo) + + var forwardInfoFlags: Int8 = 1 + if forwardInfo.sourceId != nil { + forwardInfoFlags |= 2 + } + if forwardInfo.sourceMessageId != nil { + forwardInfoFlags |= 4 + } + sharedBuffer.write(&forwardInfoFlags, offset: 0, length: 1) + var forwardAuthorId: Int64 = forwardInfo.authorId.toInt64() + var forwardDate: Int32 = forwardInfo.date + sharedBuffer.write(&forwardAuthorId, offset: 0, length: 8) + sharedBuffer.write(&forwardDate, offset: 0, length: 4) + + if let sourceId = forwardInfo.sourceId { + var sourceIdValue: Int64 = sourceId.toInt64() + sharedBuffer.write(&sourceIdValue, offset: 0, length: 8) + } + + if let sourceMessageId = forwardInfo.sourceMessageId { + var sourceMessageIdPeerId: Int64 = sourceMessageId.peerId.toInt64() + var sourceMessageIdNamespace: Int32 = sourceMessageId.namespace + var sourceMessageIdId: Int32 = sourceMessageId.id + sharedBuffer.write(&sourceMessageIdPeerId, offset: 0, length: 8) + sharedBuffer.write(&sourceMessageIdNamespace, offset: 0, length: 4) + sharedBuffer.write(&sourceMessageIdId, offset: 0, length: 4) + } + } else { + var forwardInfoFlags: Int8 = 0 + sharedBuffer.write(&forwardInfoFlags, offset: 0, length: 1) + } + if let authorId = message.authorId { var varAuthorId: Int64 = authorId.toInt64() var hasAuthor: Int8 = 1 @@ -272,19 +389,23 @@ final class MessageHistoryTable { self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) - return IntermediateMessage(id: message.id, timestamp: message.timestamp, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) + return IntermediateMessage(stableId: stableId, id: message.id, timestamp: message.timestamp, flags: flags, tags: message.tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) } private func justInsertHole(hole: MessageHistoryHole, sharedBuffer: WriteBuffer = WriteBuffer()) { sharedBuffer.reset() var type: Int8 = 1 sharedBuffer.write(&type, offset: 0, length: 1) + var stableId: UInt32 = hole.stableId + sharedBuffer.write(&stableId, offset: 0, length: 4) var minId: Int32 = hole.min sharedBuffer.write(&minId, offset: 0, length: 4) + var tags: UInt32 = hole.tags + sharedBuffer.write(&tags, offset: 0, length: 4) self.valueBox.set(self.tableId, key: self.key(hole.maxIndex), value: sharedBuffer.readBufferNoCopy()) } - private func justRemove(index: MessageIndex) { + private func justRemove(index: MessageIndex, inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { let key = self.key(index) if let value = self.valueBox.get(self.tableId, key: key) { switch self.readIntermediateEntry(key, value: value) { @@ -303,17 +424,273 @@ final class MessageHistoryTable { } } + if message.flags.contains(.Unsent) && !message.flags.contains(.Failed) { + self.unsentTable.remove(index, operations: &unsentMessageOperations) + } + + let tags = message.tags.rawValue + if tags != 0 { + for i in 0 ..< 32 { + let currentTags = tags >> UInt32(i) + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + let tag = MessageTags(rawValue: 1 << UInt32(i)) + self.tagsTable.remove(tag, index: index) + } + } + } + for mediaId in message.referencedMedia { self.messageMediaTable.removeReference(mediaId) - } - case .Hole: - break + } + case let .Hole(hole): + let tags = self.messageHistoryIndexTable.seedConfiguration.existingMessageTags.rawValue & hole.tags + if tags != 0 { + for i in 0 ..< 32 { + let currentTags = tags >> UInt32(i) + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + let tag = MessageTags(rawValue: 1 << UInt32(i)) + self.tagsTable.remove(tag, index: hole.maxIndex) + } + } + } } self.valueBox.remove(self.tableId, key: key) } } + private func justUpdate(index: MessageIndex, message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder, inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) -> IntermediateMessage? { + if let previousMessage = self.getMessage(index) { + self.valueBox.remove(self.tableId, key: self.key(index)) + + switch (previousMessage.flags.contains(.Unsent) && !previousMessage.flags.contains(.Failed), message.flags.contains(.Unsent) && !message.flags.contains(.Failed)) { + case (true, false): + self.unsentTable.remove(index, operations: &unsentMessageOperations) + case (false, true): + self.unsentTable.add(MessageIndex(message), operations: &unsentMessageOperations) + case (true, true): + if index != MessageIndex(message) { + self.unsentTable.remove(index, operations: &unsentMessageOperations) + self.unsentTable.add(MessageIndex(message), operations: &unsentMessageOperations) + } + case (false, false): + break + } + + if previousMessage.tags != message.tags { + assertionFailure() + } + + let previousEmbeddedMediaData = previousMessage.embeddedMediaData + + var previousMedia: [Media] = [] + if previousEmbeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + previousEmbeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + previousEmbeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: previousEmbeddedMediaData.memory + previousEmbeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + previousMedia.append(media) + } + previousEmbeddedMediaData.skip(Int(mediaLength)) + } + } + + for mediaId in previousMessage.referencedMedia { + if let media = self.messageMediaTable.get(mediaId) { + previousMedia.append(media) + } + } + + let updatedMedia = message.media + + var removedMediaIds: [MediaId] = [] + + var updatedEmbeddedMedia: [Media] = [] + var updatedReferencedMediaIds: [MediaId] = [] + + for media in updatedMedia { + assertionFailure() + if let mediaId = media.id { + var found = false + for previous in previousMedia { + if let previousId = previous.id { + if previousId == mediaId { + found = true + break + } + } + } + if !found { + let result = self.messageMediaTable.set(media, index: MessageIndex(message), messageHistoryTable: self) + switch result { + case let .Embed(embedded): + updatedEmbeddedMedia.append(embedded) + case .Reference: + updatedReferencedMediaIds.append(mediaId) + } + } else { + + } + } else { + updatedEmbeddedMedia.append(media) + } + } + for media in previousMedia { + assertionFailure() + if let mediaId = media.id { + var found = false + for updated in updatedMedia { + if let updatedId = updated.id { + if updatedId == mediaId { + found = true + break + } + } + } + if !found { + assertionFailure() + } + } else { + self.messageMediaTable.removeEmbeddedMedia(media) + } + } + + sharedBuffer.reset() + + var type: Int8 = 0 + sharedBuffer.write(&type, offset: 0, length: 1) + + var stableId: UInt32 = previousMessage.stableId + sharedBuffer.write(&stableId, offset: 0, length: 4) + + var flags = MessageFlags(message.flags) + sharedBuffer.write(&flags.rawValue, offset: 0, length: 4) + + var tags = message.tags + sharedBuffer.write(&tags.rawValue, offset: 0, length: 4) + + var intermediateForwardInfo: IntermediateMessageForwardInfo? + if let forwardInfo = message.forwardInfo { + intermediateForwardInfo = IntermediateMessageForwardInfo(forwardInfo) + + var forwardInfoFlags: Int8 = 1 + if forwardInfo.sourceId != nil { + forwardInfoFlags |= 2 + } + if forwardInfo.sourceMessageId != nil { + forwardInfoFlags |= 4 + } + sharedBuffer.write(&forwardInfoFlags, offset: 0, length: 1) + var forwardAuthorId: Int64 = forwardInfo.authorId.toInt64() + var forwardDate: Int32 = forwardInfo.date + sharedBuffer.write(&forwardAuthorId, offset: 0, length: 8) + sharedBuffer.write(&forwardDate, offset: 0, length: 4) + + if let sourceId = forwardInfo.sourceId { + var sourceIdValue: Int64 = sourceId.toInt64() + sharedBuffer.write(&sourceIdValue, offset: 0, length: 8) + } + + if let sourceMessageId = forwardInfo.sourceMessageId { + var sourceMessageIdPeerId: Int64 = sourceMessageId.peerId.toInt64() + var sourceMessageIdNamespace: Int32 = sourceMessageId.namespace + var sourceMessageIdId: Int32 = sourceMessageId.id + sharedBuffer.write(&sourceMessageIdPeerId, offset: 0, length: 8) + sharedBuffer.write(&sourceMessageIdNamespace, offset: 0, length: 4) + sharedBuffer.write(&sourceMessageIdId, offset: 0, length: 4) + } + } else { + var forwardInfoFlags: Int8 = 0 + sharedBuffer.write(&forwardInfoFlags, offset: 0, length: 1) + } + + if let authorId = message.authorId { + var varAuthorId: Int64 = authorId.toInt64() + var hasAuthor: Int8 = 1 + sharedBuffer.write(&hasAuthor, offset: 0, length: 1) + sharedBuffer.write(&varAuthorId, offset: 0, length: 8) + } else { + var hasAuthor: Int8 = 0 + sharedBuffer.write(&hasAuthor, offset: 0, length: 1) + } + + let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! + var length: Int32 = Int32(data.length) + sharedBuffer.write(&length, offset: 0, length: 4) + sharedBuffer.write(data.bytes, offset: 0, length: Int(length)) + + let attributesBuffer = WriteBuffer() + + var attributeCount: Int32 = Int32(message.attributes.count) + attributesBuffer.write(&attributeCount, offset: 0, length: 4) + for attribute in message.attributes { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(attribute) + let attributeBuffer = sharedEncoder.memoryBuffer() + var attributeBufferLength = Int32(attributeBuffer.length) + attributesBuffer.write(&attributeBufferLength, offset: 0, length: 4) + attributesBuffer.write(attributeBuffer.memory, offset: 0, length: attributeBuffer.length) + } + + sharedBuffer.write(attributesBuffer.memory, offset: 0, length: attributesBuffer.length) + + var embeddedMedia: [Media] = [] + var referencedMedia: [MediaId] = [] + for media in message.media { + if let mediaId = media.id { + let mediaInsertResult = self.messageMediaTable.set(media, index: MessageIndex(message), messageHistoryTable: self) + switch mediaInsertResult { + case let .Embed(media): + embeddedMedia.append(media) + case .Reference: + referencedMedia.append(mediaId) + } + } else { + embeddedMedia.append(media) + } + } + + let embeddedMediaBuffer = WriteBuffer() + var embeddedMediaCount: Int32 = Int32(embeddedMedia.count) + embeddedMediaBuffer.write(&embeddedMediaCount, offset: 0, length: 4) + for media in embeddedMedia { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(media) + let mediaBuffer = sharedEncoder.memoryBuffer() + var mediaBufferLength = Int32(mediaBuffer.length) + embeddedMediaBuffer.write(&mediaBufferLength, offset: 0, length: 4) + embeddedMediaBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length) + } + + sharedBuffer.write(embeddedMediaBuffer.memory, offset: 0, length: embeddedMediaBuffer.length) + + var referencedMediaCount: Int32 = Int32(referencedMedia.count) + sharedBuffer.write(&referencedMediaCount, offset: 0, length: 4) + for mediaId in referencedMedia { + var idNamespace: Int32 = mediaId.namespace + var idId: Int64 = mediaId.id + sharedBuffer.write(&idNamespace, offset: 0, length: 4) + sharedBuffer.write(&idId, offset: 0, length: 8) + } + + self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) + + return IntermediateMessage(stableId: stableId, id: message.id, timestamp: message.timestamp, flags: flags, tags: tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) + } else { + return nil + } + } + func unembedMedia(index: MessageIndex, id: MediaId) -> Media? { if let message = self.getMessage(index) where message.embeddedMediaData.length > 4 { var embeddedMediaCount: Int32 = 0 @@ -346,7 +723,7 @@ final class MessageHistoryTable { if let extractedMedia = extractedMedia { var updatedReferencedMedia = message.referencedMedia updatedReferencedMedia.append(extractedMedia.id!) - self.storeIntermediateMessage(IntermediateMessage(id: message.id, timestamp: message.timestamp, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) return extractedMedia } @@ -359,6 +736,47 @@ final class MessageHistoryTable { var type: Int8 = 0 sharedBuffer.write(&type, offset: 0, length: 1) + + var stableId: UInt32 = message.stableId + sharedBuffer.write(&stableId, offset: 0, length: 4) + + var flagsValue: UInt32 = message.flags.rawValue + sharedBuffer.write(&flagsValue, offset: 0, length: 4) + + var tagsValue: UInt32 = message.tags.rawValue + sharedBuffer.write(&tagsValue, offset: 0, length: 4) + + if let forwardInfo = message.forwardInfo { + var forwardInfoFlags: Int8 = 1 + if forwardInfo.sourceId != nil { + forwardInfoFlags |= 2 + } + if forwardInfo.sourceMessageId != nil { + forwardInfoFlags |= 4 + } + sharedBuffer.write(&forwardInfoFlags, offset: 0, length: 1) + var forwardAuthorId: Int64 = forwardInfo.authorId.toInt64() + var forwardDate: Int32 = forwardInfo.date + sharedBuffer.write(&forwardAuthorId, offset: 0, length: 8) + sharedBuffer.write(&forwardDate, offset: 0, length: 4) + + if let sourceId = forwardInfo.sourceId { + var sourceIdValue: Int64 = sourceId.toInt64() + sharedBuffer.write(&sourceIdValue, offset: 0, length: 8) + } + + if let sourceMessageId = forwardInfo.sourceMessageId { + var sourceMessageIdPeerId: Int64 = sourceMessageId.peerId.toInt64() + var sourceMessageIdNamespace: Int32 = sourceMessageId.namespace + var sourceMessageIdId: Int32 = sourceMessageId.id + sharedBuffer.write(&sourceMessageIdPeerId, offset: 0, length: 8) + sharedBuffer.write(&sourceMessageIdNamespace, offset: 0, length: 4) + sharedBuffer.write(&sourceMessageIdId, offset: 0, length: 4) + } + } else { + var forwardInfoFlags: Int8 = 0 + sharedBuffer.write(&forwardInfoFlags, offset: 0, length: 1) + } if let authorId = message.authorId { var varAuthorId: Int64 = authorId.toInt64() @@ -396,6 +814,48 @@ final class MessageHistoryTable { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) if type == 0 { + var stableId: UInt32 = 0 + value.read(&stableId, offset: 0, length: 4) + + var flagsValue: UInt32 = 0 + value.read(&flagsValue, offset: 0, length: 4) + let flags = MessageFlags(rawValue: flagsValue) + + var tagsValue: UInt32 = 0 + value.read(&tagsValue, offset: 0, length: 4) + let tags = MessageTags(rawValue: tagsValue) + + var forwardInfoFlags: Int8 = 0 + value.read(&forwardInfoFlags, offset: 0, length: 1) + var forwardInfo: IntermediateMessageForwardInfo? + if forwardInfoFlags != 0 { + var forwardAuthorId: Int64 = 0 + var forwardDate: Int32 = 0 + var forwardSourceId: PeerId? + var forwardSourceMessageId: MessageId? + + value.read(&forwardAuthorId, offset: 0, length: 8) + value.read(&forwardDate, offset: 0, length: 4) + + if (forwardInfoFlags & 2) != 0 { + var forwardSourceIdValue: Int64 = 0 + value.read(&forwardSourceIdValue, offset: 0, length: 8) + forwardSourceId = PeerId(forwardSourceIdValue) + } + + if (forwardInfoFlags & 4) != 0 { + var forwardSourceMessagePeerId: Int64 = 0 + var forwardSourceMessageNamespace: Int32 = 0 + var forwardSourceMessageIdId: Int32 = 0 + value.read(&forwardSourceMessagePeerId, offset: 0, length: 8) + value.read(&forwardSourceMessageNamespace, offset: 0, length: 4) + value.read(&forwardSourceMessageIdId, offset: 0, length: 4) + forwardSourceMessageId = MessageId(peerId: PeerId(forwardSourceMessagePeerId), namespace: forwardSourceMessageNamespace, id: forwardSourceMessageIdId) + } + + forwardInfo = IntermediateMessageForwardInfo(authorId: PeerId(forwardAuthorId), sourceId: forwardSourceId, sourceMessageId: forwardSourceMessageId, date: forwardDate) + } + var hasAuthor: Int8 = 0 value.read(&hasAuthor, offset: 0, length: 1) var authorId: PeerId? @@ -408,6 +868,7 @@ final class MessageHistoryTable { var textLength: Int32 = 0 value.read(&textLength, offset: 0, length: 4) let text = String(data: NSData(bytes: value.memory + value.offset, length: Int(textLength)), encoding: NSUTF8StringEncoding) ?? "" + //let text = NSString(bytes: value.memory + value.offset, length: Int(textLength), encoding: NSUTF8StringEncoding) ?? "" value.skip(Int(textLength)) let attributesOffset = value.offset @@ -447,44 +908,39 @@ final class MessageHistoryTable { referencedMediaIds.append(MediaId(namespace: idNamespace, id: idId)) } - return .Message(IntermediateMessage(id: index.id, timestamp: index.timestamp, authorId: authorId, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)) + return .Message(IntermediateMessage(stableId: stableId, id: index.id, timestamp: index.timestamp, flags: flags, tags: tags, forwardInfo: forwardInfo, authorId: authorId, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)) } else { + var stableId: UInt32 = 0 + value.read(&stableId, offset: 0, length: 4) + var minId: Int32 = 0 value.read(&minId, offset: 0, length: 4) + var tags: UInt32 = 0 + value.read(&tags, offset: 0, length: 4) - return .Hole(MessageHistoryHole(maxIndex: index, min: minId)) + return .Hole(MessageHistoryHole(stableId: stableId, maxIndex: index, min: minId, tags: tags)) } } - func getMessage(index: MessageIndex) -> IntermediateMessage? { - let key = self.key(index) - if let value = self.valueBox.get(self.tableId, key: key) { - if case let .Message(message) = self.readIntermediateEntry(key, value: value) { - return message - } - } - return nil - } - - func renderMessage(message: IntermediateMessage, peerTable: PeerTable) -> Message { - var parsedAttributes: [Coding] = [] + func renderMessage(message: IntermediateMessage, peerTable: PeerTable, addAssociatedMessages: Bool = true) -> Message { + var parsedAttributes: [MessageAttribute] = [] var parsedMedia: [Media] = [] - let attributesData = message.attributesData + let attributesData = message.attributesData.sharedBufferNoCopy() if attributesData.length > 4 { var attributeCount: Int32 = 0 attributesData.read(&attributeCount, offset: 0, length: 4) for _ in 0 ..< attributeCount { var attributeLength: Int32 = 0 attributesData.read(&attributeLength, offset: 0, length: 4) - if let attribute = Decoder(buffer: MemoryBuffer(memory: attributesData.memory + attributesData.offset, capacity: Int(attributeLength), length: Int(attributeLength), freeWhenDone: false)).decodeRootObject() { + if let attribute = Decoder(buffer: MemoryBuffer(memory: attributesData.memory + attributesData.offset, capacity: Int(attributeLength), length: Int(attributeLength), freeWhenDone: false)).decodeRootObject() as? MessageAttribute { parsedAttributes.append(attribute) } attributesData.skip(Int(attributeLength)) } } - let embeddedMediaData = message.embeddedMediaData + let embeddedMediaData = message.embeddedMediaData.sharedBufferNoCopy() if embeddedMediaData.length > 4 { var embeddedMediaCount: Int32 = 0 embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) @@ -504,6 +960,15 @@ final class MessageHistoryTable { } } + var forwardInfo: MessageForwardInfo? + if let internalForwardInfo = message.forwardInfo, forwardAuthor = peerTable.get(internalForwardInfo.authorId) { + var source: Peer? + if let sourceId = internalForwardInfo.sourceId { + source = peerTable.get(sourceId) + } + forwardInfo = MessageForwardInfo(author: forwardAuthor, source: source, sourceMessageId: internalForwardInfo.sourceMessageId, date: internalForwardInfo.date) + } + var author: Peer? if let authorId = message.authorId { author = peerTable.get(authorId) @@ -522,37 +987,106 @@ final class MessageHistoryTable { } } - return Message(id: message.id, timestamp: message.timestamp, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers) + var associatedMessages = SimpleDictionary() + for attribute in parsedAttributes { + for peerId in attribute.associatedPeerIds { + if let peer = peerTable.get(peerId) { + peers[peer.id] = peer + } + } + for messageId in attribute.associatedMessageIds { + if let entry = self.messageHistoryIndexTable.get(messageId) { + if case let .Message(index) = entry { + if let message = self.getMessage(index) { + associatedMessages[messageId] = self.renderMessage(message, peerTable: peerTable, addAssociatedMessages: false) + } + } + } + } + } + + return Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages) } - func entriesAround(index: MessageIndex, count: Int) -> [IntermediateMessageHistoryEntry] { + func entriesAround(index: MessageIndex, count: Int) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { var lowerEntries: [IntermediateMessageHistoryEntry] = [] var upperEntries: [IntermediateMessageHistoryEntry] = [] + var lower: IntermediateMessageHistoryEntry? + var upper: IntermediateMessageHistoryEntry? self.valueBox.range(self.tableId, start: self.key(index), end: self.lowerBound(index.id.peerId), values: { key, value in lowerEntries.append(self.readIntermediateEntry(key, value: value)) return true - }, limit: count / 2) + }, limit: count / 2 + 1) + + if lowerEntries.count >= count / 2 + 1 { + lower = lowerEntries.last + lowerEntries.removeLast() + } self.valueBox.range(self.tableId, start: self.key(index).predecessor, end: self.upperBound(index.id.peerId), values: { key, value in upperEntries.append(self.readIntermediateEntry(key, value: value)) return true - }, limit: count - lowerEntries.count) + }, limit: count - lowerEntries.count + 1) + if upperEntries.count >= count - lowerEntries.count + 1 { + upper = upperEntries.last + upperEntries.removeLast() + } if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count { + var additionalLowerEntries: [IntermediateMessageHistoryEntry] = [] self.valueBox.range(self.tableId, start: self.key(lowerEntries.last!.index), end: self.lowerBound(index.id.peerId), values: { key, value in - lowerEntries.append(self.readIntermediateEntry(key, value: value)) + additionalLowerEntries.append(self.readIntermediateEntry(key, value: value)) return true - }, limit: count - (lowerEntries.count + upperEntries.count)) + }, limit: count - lowerEntries.count - upperEntries.count + 1) + if additionalLowerEntries.count >= count - lowerEntries.count + upperEntries.count + 1 { + lower = additionalLowerEntries.last + additionalLowerEntries.removeLast() + } + lowerEntries.appendContentsOf(additionalLowerEntries) } var entries: [IntermediateMessageHistoryEntry] = [] - for entry in lowerEntries.reverse() { - entries.append(entry) - } + entries.appendContentsOf(lowerEntries.reverse()) entries.appendContentsOf(upperEntries) + return (entries: entries, lower: lower, upper: upper) + } + + func entriesAround(tagMask: MessageTags, index: MessageIndex, count: Int) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { + let (indices, lower, upper) = self.tagsTable.indicesAround(tagMask, index: index, count: count) - return entries + var entries: [IntermediateMessageHistoryEntry] = [] + for index in indices { + let key = self.key(index) + if let value = self.valueBox.get(self.tableId, key: key) { + entries.append(readIntermediateEntry(key, value: value)) + } else { + assertionFailure() + } + } + + var lowerEntry: IntermediateMessageHistoryEntry? + var upperEntry: IntermediateMessageHistoryEntry? + + if let lowerIndex = lower { + let key = self.key(lowerIndex) + if let value = self.valueBox.get(self.tableId, key: key) { + lowerEntry = readIntermediateEntry(key, value: value) + } else { + assertionFailure() + } + } + + if let upperIndex = upper { + let key = self.key(upperIndex) + if let value = self.valueBox.get(self.tableId, key: key) { + upperEntry = readIntermediateEntry(key, value: value) + } else { + assertionFailure() + } + } + + return (entries, lowerEntry, upperEntry) } func earlierEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { @@ -570,6 +1104,22 @@ final class MessageHistoryTable { return entries } + func earlierEntries(tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + let indices = self.tagsTable.earlierIndices(tagMask, peerId: peerId, index: index, count: count) + + var entries: [IntermediateMessageHistoryEntry] = [] + for index in indices { + let key = self.key(index) + if let value = self.valueBox.get(self.tableId, key: key) { + entries.append(readIntermediateEntry(key, value: value)) + } else { + assertionFailure() + } + } + + return entries + } + func laterEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { var entries: [IntermediateMessageHistoryEntry] = [] let key: ValueBoxKey @@ -585,8 +1135,24 @@ final class MessageHistoryTable { return entries } + func laterEntries(tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + let indices = self.tagsTable.laterIndices(tagMask, peerId: peerId, index: index, count: count) + + var entries: [IntermediateMessageHistoryEntry] = [] + for index in indices { + let key = self.key(index) + if let value = self.valueBox.get(self.tableId, key: key) { + entries.append(readIntermediateEntry(key, value: value)) + } else { + assertionFailure() + } + } + + return entries + } + func debugList(peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { - return self.entriesAround(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: 0), count: 1000).map({ entry -> RenderedMessageHistoryEntry in + return self.laterEntries(peerId, index: nil, count: 1000).map({ entry -> RenderedMessageHistoryEntry in switch entry { case let .Hole(hole): return .Hole(hole) @@ -595,4 +1161,15 @@ final class MessageHistoryTable { } }) } + + func debugList(tagMask: MessageTags, peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { + return self.laterEntries(tagMask, peerId: peerId, index: nil, count: 1000).map({ entry -> RenderedMessageHistoryEntry in + switch entry { + case let .Hole(hole): + return .Hole(hole) + case let .Message(message): + return .RenderedMessage(self.renderMessage(message, peerTable: peerTable)) + } + }) + } } diff --git a/Postbox/MessageHistoryTagsTable.swift b/Postbox/MessageHistoryTagsTable.swift new file mode 100644 index 0000000000..908b36b014 --- /dev/null +++ b/Postbox/MessageHistoryTagsTable.swift @@ -0,0 +1,119 @@ +import Foundation + +class MessageHistoryTagsTable: Table { + private let sharedKey = ValueBoxKey(length: 8 + 4 + 4 + 4 + 4) + + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) + } + + private func key(tagMask: MessageTags, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4 + 4)) -> ValueBoxKey { + key.setInt64(0, value: index.id.peerId.toInt64()) + key.setUInt32(8, value: tagMask.rawValue) + key.setInt32(8 + 4, value: index.timestamp) + key.setInt32(8 + 4 + 4, value: index.id.namespace) + key.setInt32(8 + 4 + 4 + 4, value: index.id.id) + return key + } + + private func lowerBound(tagMask: MessageTags, peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setUInt32(8, value: tagMask.rawValue) + return key + } + + private func upperBound(tagMask: MessageTags, peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setUInt32(8, value: tagMask.rawValue) + return key.successor + } + + func add(tagMask: MessageTags, index: MessageIndex) { + self.valueBox.set(self.tableId, key: self.key(tagMask, index: index, key: self.sharedKey), value: MemoryBuffer()) + } + + func remove(tagMask: MessageTags, index: MessageIndex) { + self.valueBox.remove(self.tableId, key: self.key(tagMask, index: index, key: self.sharedKey)) + } + + func indicesAround(tagMask: MessageTags, index: MessageIndex, count: Int) -> (indices: [MessageIndex], lower: MessageIndex?, upper: MessageIndex?) { + var lowerEntries: [MessageIndex] = [] + var upperEntries: [MessageIndex] = [] + var lower: MessageIndex? + var upper: MessageIndex? + + self.valueBox.range(self.tableId, start: self.key(tagMask, index: index), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) + lowerEntries.append(index) + return true + }, limit: count / 2 + 1) + + if lowerEntries.count >= count / 2 + 1 { + lower = lowerEntries.last + lowerEntries.removeLast() + } + + self.valueBox.range(self.tableId, start: self.key(tagMask, index: index).predecessor, end: self.upperBound(tagMask, peerId: index.id.peerId), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) + upperEntries.append(index) + return true + }, limit: count - lowerEntries.count + 1) + if upperEntries.count >= count - lowerEntries.count + 1 { + upper = upperEntries.last + upperEntries.removeLast() + } + + if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count { + var additionalLowerEntries: [MessageIndex] = [] + self.valueBox.range(self.tableId, start: self.key(tagMask, index: lowerEntries.last!), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) + additionalLowerEntries.append(index) + return true + }, limit: count - lowerEntries.count - upperEntries.count + 1) + if additionalLowerEntries.count >= count - lowerEntries.count + upperEntries.count + 1 { + lower = additionalLowerEntries.last + additionalLowerEntries.removeLast() + } + lowerEntries.appendContentsOf(additionalLowerEntries) + } + + var entries: [MessageIndex] = [] + entries.appendContentsOf(lowerEntries.reverse()) + entries.appendContentsOf(upperEntries) + return (indices: entries, lower: lower, upper: upper) + } + + func earlierIndices(tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [MessageIndex] { + var indices: [MessageIndex] = [] + let key: ValueBoxKey + if let index = index { + key = self.key(tagMask, index: index) + } else { + key = self.upperBound(tagMask, peerId: peerId) + } + self.valueBox.range(self.tableId, start: key, end: self.lowerBound(tagMask, peerId: peerId), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) + indices.append(index) + return true + }, limit: count) + return indices + } + + func laterIndices(tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [MessageIndex] { + var indices: [MessageIndex] = [] + let key: ValueBoxKey + if let index = index { + key = self.key(tagMask, index: index) + } else { + key = self.lowerBound(tagMask, peerId: peerId) + } + self.valueBox.range(self.tableId, start: key, end: self.upperBound(tagMask, peerId: peerId), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) + indices.append(index) + return true + }, limit: count) + return indices + } +} diff --git a/Postbox/MessageHistoryUnsentTable.swift b/Postbox/MessageHistoryUnsentTable.swift new file mode 100644 index 0000000000..1e7fd9773d --- /dev/null +++ b/Postbox/MessageHistoryUnsentTable.swift @@ -0,0 +1,58 @@ +import Foundation + +enum IntermediateMessageHistoryUnsentOperation { + case Insert(MessageIndex) + case Remove(MessageIndex) +} + +final class MessageHistoryUnsentTable: Table { + private let sharedKey = ValueBoxKey(length: 4 + 4 + 4 + 8) + + private func key(index: MessageIndex) -> ValueBoxKey { + self.sharedKey.setInt32(0, value: index.timestamp) + self.sharedKey.setInt32(4, value: index.id.namespace) + self.sharedKey.setInt32(4 + 4, value: index.id.id) + self.sharedKey.setInt64(4 + 4 + 4, value: index.id.peerId.toInt64()) + + return self.sharedKey + } + + private func lowerBound() -> ValueBoxKey { + let key = ValueBoxKey(length: 1) + key.setInt8(0, value: 0) + return key + } + + private func upperBound() -> ValueBoxKey { + let key = ValueBoxKey(length: 4 + 4 + 4 + 8) + memset(key.memory, 0xff, key.length) + return key + } + + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) + } + + func add(index: MessageIndex, inout operations: [IntermediateMessageHistoryUnsentOperation]) { + self.valueBox.set(self.tableId, key: self.key(index), value: MemoryBuffer()) + operations.append(.Insert(index)) + } + + func remove(index: MessageIndex, inout operations: [IntermediateMessageHistoryUnsentOperation]) { + self.valueBox.remove(self.tableId, key: self.key(index)) + operations.append(.Remove(index)) + } + + func get() -> [MessageIndex] { + var indices: [MessageIndex] = [] + self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), keys: { key in + indices.append(MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0))) + return true + }, limit: 0) + return indices + } + + override func beforeCommit() { + + } +} \ No newline at end of file diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index e3dabce69f..ddc3a1d804 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -32,7 +32,25 @@ public enum MessageHistoryEntry: Comparable { } public func ==(lhs: MessageHistoryEntry, rhs: MessageHistoryEntry) -> Bool { - return lhs.index == rhs.index + switch lhs { + case let .MessageEntry(lhsMessage): + switch rhs { + case .HoleEntry: + return false + case let .MessageEntry(rhsMessage): + if MessageIndex(lhsMessage) == MessageIndex(rhsMessage) && lhsMessage.flags == rhsMessage.flags { + return true + } + return false + } + case let .HoleEntry(lhsHole): + switch rhs { + case let .HoleEntry(rhsHole): + return lhsHole == rhsHole + case .MessageEntry: + return false + } + } } public func <(lhs: MessageHistoryEntry, rhs: MessageHistoryEntry) -> Bool { @@ -50,29 +68,38 @@ final class MutableMessageHistoryViewReplayContext { } final class MutableMessageHistoryView { + let tagMask: MessageTags? private let count: Int private var earlier: MutableMessageHistoryEntry? private var later: MutableMessageHistoryEntry? private var entries: [MutableMessageHistoryEntry] - init(earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, count: Int) { + init(earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int) { self.earlier = earlier self.entries = entries self.later = later + self.tagMask = tagMask self.count = count } func replay(operations: [MessageHistoryOperation], context: MutableMessageHistoryViewReplayContext) -> Bool { + let tagMask = self.tagMask + let unwrappedTagMask: UInt32 = tagMask?.rawValue ?? 0 + var hasChanges = false for operation in operations { switch operation { case let .InsertHole(hole): - if self.add(.HoleEntry(hole)) { - hasChanges = true + if tagMask == nil || (hole.tags & unwrappedTagMask) != 0 { + if self.add(.HoleEntry(hole)) { + hasChanges = true + } } case let .InsertMessage(intermediateMessage): - if self.add(.IntermediateMessageEntry(intermediateMessage)) { - hasChanges = true + if tagMask == nil || (intermediateMessage.tags.rawValue & unwrappedTagMask) != 0 { + if self.add(.IntermediateMessageEntry(intermediateMessage)) { + hasChanges = true + } } case let .Remove(indices): if self.remove(Set(indices), context: context) { @@ -135,7 +162,7 @@ final class MutableMessageHistoryView { if self.entries[i - 1].index < index { break } - i-- + i -= 1 } self.entries.insert(entry, atIndex: i) if self.entries.count > self.count { @@ -169,7 +196,7 @@ final class MutableMessageHistoryView { context.removedEntries = true hasChanges = true } - i-- + i -= 1 } } @@ -209,7 +236,7 @@ final class MutableMessageHistoryView { if addedEntries[i].index.id == addedEntries[i - 1].index.id { addedEntries.removeAtIndex(i) } - i-- + i -= 1 } self.entries = [] @@ -221,7 +248,7 @@ final class MutableMessageHistoryView { anchorIndex = i break } - i-- + i -= 1 } } @@ -233,7 +260,7 @@ final class MutableMessageHistoryView { i = anchorIndex while i >= 0 && i > anchorIndex - self.count { self.entries.insert(addedEntries[i], atIndex: 0) - i-- + i -= 1 } self.earlier = nil @@ -281,7 +308,7 @@ final class MutableMessageHistoryView { } func firstHole() -> MessageHistoryHole? { - for entry in self.entries { + for entry in self.entries.reverse() as ReverseCollection { if case let .HoleEntry(hole) = entry { return hole } diff --git a/Postbox/MessageMediaTable.swift b/Postbox/MessageMediaTable.swift index 0fd6794974..18b8973907 100644 --- a/Postbox/MessageMediaTable.swift +++ b/Postbox/MessageMediaTable.swift @@ -15,16 +15,13 @@ enum DebugMediaEntry { case MessageReference(MessageIndex) } -final class MessageMediaTable { - let valueBox: ValueBox - let tableId: Int32 - +final class MessageMediaTable: Table { let mediaCleanupTable: MediaCleanupTable init(valueBox: ValueBox, tableId: Int32, mediaCleanupTable: MediaCleanupTable) { - self.valueBox = valueBox - self.tableId = tableId self.mediaCleanupTable = mediaCleanupTable + + super.init(valueBox: valueBox, tableId: tableId) } func key(id: MediaId, key: ValueBoxKey = ValueBoxKey(length: 4 + 8)) -> ValueBoxKey { @@ -63,7 +60,7 @@ final class MessageMediaTable { var messageReferenceCount: Int32 = 0 value.read(&messageReferenceCount, offset: 0, length: 4) - messageReferenceCount++ + messageReferenceCount += 1 sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index 5dfb6f4e0f..4815cfb0b8 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -5,13 +5,9 @@ private enum MetadataKey: Int32 { case State = 2 } -final class MetadataTable { - let valueBox: ValueBox - let tableId: Int32 - - init(valueBox: ValueBox, tableId: Int32) { - self.valueBox = valueBox - self.tableId = tableId +final class MetadataTable: Table { + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) } private func key(key: MetadataKey) -> ValueBoxKey { diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index f9d741f6f2..d4610b3d8a 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -41,7 +41,7 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { var value: Int64 = 0 buffer.read(&value, offset: 0, length: 8) array.append(PeerId(value)) - i++ + i += 1 } return array } diff --git a/Postbox/PeerChatStateTable.swift b/Postbox/PeerChatStateTable.swift new file mode 100644 index 0000000000..721d889461 --- /dev/null +++ b/Postbox/PeerChatStateTable.swift @@ -0,0 +1,46 @@ +import Foundation + +final class PeerChatStateTable: Table { + private var cachedPeerChatStates: [PeerId: Coding?] = [:] + private var updatedPeerIds = Set() + + private let sharedKey = ValueBoxKey(length: 8) + + private func key(id: PeerId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: id.toInt64()) + return self.sharedKey + } + + func get(id: PeerId) -> Coding? { + if let state = self.cachedPeerChatStates[id] { + return state + } else { + if let value = self.valueBox.get(self.tableId, key: self.key(id)), state = Decoder(buffer: value).decodeRootObject() { + self.cachedPeerChatStates[id] = state + return state + } else { + self.cachedPeerChatStates[id] = nil + return nil + } + } + } + + func set(id: PeerId, state: Coding?) { + self.cachedPeerChatStates[id] = state + self.updatedPeerIds.insert(id) + } + + override func beforeCommit() { + let sharedEncoder = Encoder() + for id in self.updatedPeerIds { + if let wrappedState = self.cachedPeerChatStates[id], state = wrappedState { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(state) + self.valueBox.set(self.tableId, key: self.key(id), value: sharedEncoder.readBufferNoCopy()) + } else { + self.valueBox.remove(self.tableId, key: self.key(id)) + } + } + self.updatedPeerIds.removeAll() + } +} \ No newline at end of file diff --git a/Postbox/PeerTable.swift b/Postbox/PeerTable.swift index 6f3aee5056..d1c318e547 100644 --- a/Postbox/PeerTable.swift +++ b/Postbox/PeerTable.swift @@ -1,16 +1,14 @@ import Foundation -final class PeerTable { - let valueBox: ValueBox - let tableId: Int32 - +final class PeerTable: Table { private let sharedEncoder = Encoder() private let sharedKey = ValueBoxKey(length: 8) private var cachedPeers: [PeerId: Peer] = [:] - init(valueBox: ValueBox, tableId: Int32) { - self.valueBox = valueBox - self.tableId = tableId + private var updatedPeerIds = Set() + + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) } private func key(id: PeerId) -> ValueBoxKey { @@ -19,10 +17,8 @@ final class PeerTable { } func set(peer: Peer) { - self.sharedEncoder.reset() - self.sharedEncoder.encodeRootObject(peer) - - self.valueBox.set(self.tableId, key: self.key(peer.id), value: self.sharedEncoder.readBufferNoCopy()) + self.cachedPeers[peer.id] = peer + self.updatedPeerIds.insert(peer.id) } func get(id: PeerId) -> Peer? { @@ -37,4 +33,17 @@ final class PeerTable { } return nil } + + override func beforeCommit() { + for peerId in self.updatedPeerIds { + if let peer = self.cachedPeers[peerId] { + self.sharedEncoder.reset() + self.sharedEncoder.encodeRootObject(peer) + + self.valueBox.set(self.tableId, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + } + } + + self.updatedPeerIds.removeAll() + } } \ No newline at end of file diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index f2dea6d270..4169d232eb 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -2,6 +2,10 @@ import Foundation import SwiftSignalKit import sqlcipher +public protocol PeerChatState: Coding { + func equals(other: PeerChatState) -> Bool +} + public final class Modifier { private weak var postbox: Postbox? @@ -13,12 +17,16 @@ public final class Modifier { self.postbox?.addMessages(messages, location: location) } - public func initializeHole(peerId: PeerId, namespace: MessageId.Namespace) { - self.postbox?.addHole(MessageId(peerId: peerId, namespace: namespace, id: 1)) + public func addHole(messageId: MessageId) { + self.postbox?.addHole(messageId) } - public func fillHole(hole: MessageHistoryHole, fillType: HoleFillType, messages: [StoreMessage]) { - self.postbox?.fillHole(hole, fillType: fillType, messages: messages) + public func fillHole(hole: MessageHistoryHole, fillType: HoleFillType, tagMask: MessageTags?, messages: [StoreMessage]) { + self.postbox?.fillHole(hole, fillType: fillType, tagMask: tagMask, messages: messages) + } + + public func replaceChatListHole(index: MessageIndex, hole: ChatListHole?) { + self.postbox?.replaceChatListHole(index, hole: hole) } public func deleteMessages(messageIds: [MessageId]) { @@ -40,16 +48,36 @@ public final class Modifier { self.postbox?.setState(state) } + public func getPeerChatState(id: PeerId) -> PeerChatState? { + return self.postbox?.peerChatStateTable.get(id) as? PeerChatState + } + + public func setPeerChatState(id: PeerId, state: PeerChatState) { + self.postbox?.peerChatStateTable.set(id, state: state) + } + + public func getPeer(id: PeerId) -> Peer? { + return self.postbox?.peerTable.get(id) + } + public func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer?) { self.postbox?.updatePeers(peers, update: update) } - public func knownPeerIds(ids: Set) -> Set { - return self.postbox?.knownPeerIds(ids) ?? Set() + public func updateMessage(index: MessageIndex, update: Message -> StoreMessage) { + self.postbox?.updateMessage(index, update: update) + } + + public func filterStoredMessageIds(messageIds: Set) -> Set { + if let postbox = self.postbox { + return postbox.filterStoredMessageIds(messageIds) + } + return Set() } } public final class Postbox { + private let seedConfiguration: SeedConfiguration private let basePath: String private let globalMessageIdsNamespace: MessageId.Namespace @@ -61,18 +89,33 @@ public final class Postbox { private var peerPipes: [PeerId: Pipe] = [:] private var currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + private var currentUnsentOperations: [IntermediateMessageHistoryUnsentOperation] = [] + private var currentFilledHolesByPeerId = Set() private var currentUpdatedPeers: [PeerId: Peer] = [:] + private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var statePipe: Pipe = Pipe() + + private let fetchChatListHoleImpl = Promise Signal>() + public func setFetchChatListHole(fetch: ChatListHole -> Signal) { + self.fetchChatListHoleImpl.set(single(fetch, NoError.self)) + } - private let fetchMessageHistoryHoleImpl = Promise Signal>() - public func setFetchMessageHistoryHole(fetch: MessageHistoryHole -> Signal) { + private let fetchMessageHistoryHoleImpl = Promise<(MessageHistoryHole, MessageTags?) -> Signal>() + public func setFetchMessageHistoryHole(fetch: (MessageHistoryHole, MessageTags?) -> Signal) { self.fetchMessageHistoryHoleImpl.set(single(fetch, NoError.self)) } + private let sendUnsentMessageImpl = Promise Signal>() + public func setSendUnsentMessage(sendUnsentMessage: Message -> Signal) { + self.sendUnsentMessageImpl.set(single(sendUnsentMessage, NoError.self)) + } + public let mediaBox: MediaBox + var tables: [Table] = [] + var metadataTable: MetadataTable! var keychainTable: KeychainTable! var peerTable: PeerTable! @@ -83,14 +126,43 @@ public final class Postbox { var mediaCleanupTable: MediaCleanupTable! var chatListIndexTable: ChatListIndexTable! var chatListTable: ChatListTable! + var messageHistoryMetadataTable: MessageHistoryMetadataTable! + var messageHistoryUnsentTable: MessageHistoryUnsentTable! + var messageHistoryTagsTable: MessageHistoryTagsTable! + var peerChatStateTable: PeerChatStateTable! - public init(basePath: String, globalMessageIdsNamespace: MessageId.Namespace) { + public init(basePath: String, globalMessageIdsNamespace: MessageId.Namespace, seedConfiguration: SeedConfiguration) { self.basePath = basePath self.globalMessageIdsNamespace = globalMessageIdsNamespace + self.seedConfiguration = seedConfiguration self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.openDatabase() } + private func debugSaveState(name: String) { + self.queue.dispatch { + let path = self.basePath + name + let _ = try? NSFileManager.defaultManager().removeItemAtPath(path) + do { + try NSFileManager.defaultManager().copyItemAtPath(self.basePath, toPath: path) + } catch (let e) { + print("(Postbox debugSaveState: error \(e))") + } + } + } + + private func debugRestoreState(name: String) { + self.queue.dispatch { + let path = self.basePath + name + let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath) + do { + try NSFileManager.defaultManager().copyItemAtPath(path, toPath: self.basePath) + } catch (let e) { + print("(Postbox debugRestoreState: error \(e))") + } + } + } + private func openDatabase() { self.queue.dispatch { let startTime = CFAbsoluteTimeGetCurrent() @@ -100,11 +172,15 @@ public final class Postbox { } catch _ { } + //let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath + "/media") + //self.debugSaveState("beforeGetDiff") + //self.debugRestoreState("beforeGetDiff") + self.valueBox = SqliteValueBox(basePath: self.basePath + "/db") self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 5 + let currentUserVersion: Int32 = 18 if userVersion != currentUserVersion { self.valueBox.drop() @@ -114,14 +190,30 @@ public final class Postbox { self.keychainTable = KeychainTable(valueBox: self.valueBox, tableId: 1) self.peerTable = PeerTable(valueBox: self.valueBox, tableId: 2) self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox, tableId: 3, namespace: self.globalMessageIdsNamespace) - self.messageHistoryIndexTable = MessageHistoryIndexTable(valueBox: self.valueBox, tableId: 4, globalMessageIdsTable: self.globalMessageIdsTable) + self.messageHistoryMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox, tableId: 10) + self.messageHistoryUnsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox, tableId: 11) + self.messageHistoryTagsTable = MessageHistoryTagsTable(valueBox: self.valueBox, tableId: 12) + self.messageHistoryIndexTable = MessageHistoryIndexTable(valueBox: self.valueBox, tableId: 4, globalMessageIdsTable: self.globalMessageIdsTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox, tableId: 5) self.mediaTable = MessageMediaTable(valueBox: self.valueBox, tableId: 6, mediaCleanupTable: self.mediaCleanupTable) - self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, tableId: 7, messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable) + self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, tableId: 7, messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, tableId: 8) - self.chatListTable = ChatListTable(valueBox: self.valueBox, tableId: 9, indexTable: self.chatListIndexTable) + self.chatListTable = ChatListTable(valueBox: self.valueBox, tableId: 9, indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) + self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, tableId: 13) - self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, renderMessage: self.renderIntermediateMessage, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper) + self.tables.append(self.keychainTable) + self.tables.append(self.peerTable) + self.tables.append(self.globalMessageIdsTable) + self.tables.append(self.messageHistoryMetadataTable) + self.tables.append(self.messageHistoryIndexTable) + self.tables.append(self.mediaCleanupTable) + self.tables.append(self.mediaTable) + self.tables.append(self.messageHistoryTable) + self.tables.append(self.chatListIndexTable) + self.tables.append(self.chatListTable) + self.tables.append(self.peerChatStateTable) + + self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, renderMessage: self.renderIntermediateMessage, fetchChatListHole: self.fetchChatListHoleWrapper, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper, sendUnsentMessage: self.sendUnsentMessageWrapper, unsentMessageIndices: self.messageHistoryUnsentTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") } @@ -179,36 +271,33 @@ public final class Postbox { } private func addMessages(messages: [StoreMessage], location: AddMessagesLocation) { - self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId) + self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations) } private func addHole(id: MessageId) { - self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId) + self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations) } - private func fillHole(hole: MessageHistoryHole, fillType: HoleFillType, messages: [StoreMessage]) { - self.messageHistoryTable.fillHole(hole.id, fillType: fillType, messages: messages, operationsByPeerId: &self.currentOperationsByPeerId) + private func fillHole(hole: MessageHistoryHole, fillType: HoleFillType, tagMask: MessageTags?, messages: [StoreMessage]) { + self.messageHistoryTable.fillHole(hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations) self.currentFilledHolesByPeerId.insert(hole.id.peerId) } + private func replaceChatListHole(index: MessageIndex, hole: ChatListHole?) { + self.currentReplaceChatListHoles.append((index, hole)) + } + private func deleteMessages(messageIds: [MessageId]) { - self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId) + self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations) } - private func knownPeerIds(ids: Set) -> Set { - var result = Set() - - for id in ids { - if let _ = self.peerTable.get(id) { - result.insert(id) - } + private func fetchEarlierHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { + let intermediateEntries: [IntermediateMessageHistoryEntry] + if let tagMask = tagMask { + intermediateEntries = self.messageHistoryTable.earlierEntries(tagMask, peerId: peerId, index: index, count: count) + } else { + intermediateEntries = self.messageHistoryTable.earlierEntries(peerId, index: index, count: count) } - - return result - } - - private func fetchEarlierHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [MutableMessageHistoryEntry] { - let intermediateEntries = self.messageHistoryTable.earlierEntries(peerId, index: index, count: count) var entries: [MutableMessageHistoryEntry] = [] for entry in intermediateEntries { switch entry { @@ -221,8 +310,58 @@ public final class Postbox { return entries } - private func fetchAroundHistoryEntries(index: MessageIndex, count: Int) -> [MutableMessageHistoryEntry] { - let intermediateEntries = self.messageHistoryTable.entriesAround(index, count: count) + private func fetchAroundHistoryEntries(index: MessageIndex, count: Int, tagMask: MessageTags? = nil) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?) { + + let intermediateEntries: [IntermediateMessageHistoryEntry] + let intermediateLower: IntermediateMessageHistoryEntry? + let intermediateUpper: IntermediateMessageHistoryEntry? + + if let tagMask = tagMask { + (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(tagMask, index: index, count: count) + } else { + (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(index, count: count) + } + + var entries: [MutableMessageHistoryEntry] = [] + for entry in intermediateEntries { + switch entry { + case let .Message(message): + entries.append(.IntermediateMessageEntry(message)) + case let .Hole(index): + entries.append(.HoleEntry(index)) + } + } + + var lower: MutableMessageHistoryEntry? + if let intermediateLower = intermediateLower { + switch intermediateLower { + case let .Message(message): + lower = .IntermediateMessageEntry(message) + case let .Hole(index): + lower = .HoleEntry(index) + } + } + + var upper: MutableMessageHistoryEntry? + if let intermediateUpper = intermediateUpper { + switch intermediateUpper { + case let .Message(message): + upper = .IntermediateMessageEntry(message) + case let .Hole(index): + upper = .HoleEntry(index) + } + } + + return (entries: entries, lower: lower, upper: upper) + } + + private func fetchLaterHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { + let intermediateEntries: [IntermediateMessageHistoryEntry] + if let tagMask = tagMask { + intermediateEntries = self.messageHistoryTable.laterEntries(tagMask, peerId: peerId, index: index, count: count) + } else { + intermediateEntries = self.messageHistoryTable.laterEntries(peerId, index: index, count: count) + } var entries: [MutableMessageHistoryEntry] = [] for entry in intermediateEntries { switch entry { @@ -235,18 +374,46 @@ public final class Postbox { return entries } - private func fetchLaterHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [MutableMessageHistoryEntry] { - let intermediateEntries = self.messageHistoryTable.laterEntries(peerId, index: index, count: count) - var entries: [MutableMessageHistoryEntry] = [] + private func fetchAroundChatEntries(index: MessageIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?) { + let (intermediateEntries, intermediateLower, intermediateUpper) = self.chatListTable.entriesAround(index, messageHistoryTable: self.messageHistoryTable, count: count) + var entries: [MutableChatListEntry] = [] + var lower: MutableChatListEntry? + var upper: MutableChatListEntry? + for entry in intermediateEntries { switch entry { - case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) - case let .Hole(index): - entries.append(.HoleEntry(index)) + case let .Message(message): + entries.append(.IntermediateMessageEntry(message)) + case let .Hole(hole): + entries.append(.HoleEntry(hole)) + case let .Nothing(index): + entries.append(.Nothing(index)) } } - return entries + + if let intermediateLower = intermediateLower { + switch intermediateLower { + case let .Message(message): + lower = .IntermediateMessageEntry(message) + case let .Hole(hole): + lower = .HoleEntry(hole) + case let .Nothing(index): + lower = .Nothing(index) + } + } + + if let intermediateUpper = intermediateUpper { + switch intermediateUpper { + case let .Message(message): + upper = .IntermediateMessageEntry(message) + case let .Hole(hole): + upper = .HoleEntry(hole) + case let .Nothing(index): + upper = .Nothing(index) + } + } + + return (entries, lower, upper) } private func fetchEarlierChatEntries(index: MessageIndex?, count: Int) -> [MutableChatListEntry] { @@ -256,6 +423,8 @@ public final class Postbox { switch entry { case let .Message(message): entries.append(.IntermediateMessageEntry(message)) + case let .Hole(hole): + entries.append(.HoleEntry(hole)) case let .Nothing(index): entries.append(.Nothing(index)) } @@ -268,10 +437,12 @@ public final class Postbox { var entries: [MutableChatListEntry] = [] for entry in intermediateEntries { switch entry { - case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) - case let .Nothing(index): - entries.append(.Nothing(index)) + case let .Message(message): + entries.append(.IntermediateMessageEntry(message)) + case let .Nothing(index): + entries.append(.Nothing(index)) + case let .Hole(index): + entries.append(.HoleEntry(index)) } } return entries @@ -280,22 +451,48 @@ public final class Postbox { private func renderIntermediateMessage(message: IntermediateMessage) -> Message { return self.messageHistoryTable.renderMessage(message, peerTable: self.peerTable) } - - private func fetchMessageHistoryHoleWrapper(hole: MessageHistoryHole) -> Disposable { - return (self.fetchMessageHistoryHoleImpl.get() |> mapToSignal { fetch in + + private func fetchChatListHoleWrapper(hole: ChatListHole) -> Disposable { + return (self.fetchChatListHoleImpl.get() |> mapToSignal { fetch in return fetch(hole) }).start() } + private func fetchMessageHistoryHoleWrapper(hole: MessageHistoryHole, tagMask: MessageTags?) -> Disposable { + return (self.fetchMessageHistoryHoleImpl.get() |> mapToSignal { fetch in + return fetch(hole, tagMask) + }).start() + } + + private func sendUnsentMessageWrapper(index: MessageIndex) -> Disposable { + return (self.sendUnsentMessageImpl.get() |> deliverOn(self.queue) |> mapToSignal { send -> Signal in + if let intermediateMessage = self.messageHistoryTable.getMessage(index) { + let message = self.renderIntermediateMessage(intermediateMessage) + return send(message) + } else { + return never(Void.self, NoError.self) + } + }).start() + } + private func beforeCommit() { var chatListOperations: [ChatListOperation] = [] self.chatListTable.replay(self.currentOperationsByPeerId, messageHistoryTable: self.messageHistoryTable, operations: &chatListOperations) + for (index, hole) in self.currentReplaceChatListHoles { + self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) + } - self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers) + self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations) self.currentOperationsByPeerId.removeAll() self.currentFilledHolesByPeerId.removeAll() self.currentUpdatedPeers.removeAll() + self.currentReplaceChatListHoles.removeAll() + self.currentUnsentOperations.removeAll() + + for table in self.tables { + table.beforeCommit() + } } private func messageIdsForGlobalIds(ids: [Int32]) -> [MessageId] { @@ -321,6 +518,25 @@ public final class Postbox { } } + private func updateMessage(index: MessageIndex, update: Message -> StoreMessage) { + if let intermediateMessage = self.messageHistoryTable.getMessage(index) { + let message = self.renderIntermediateMessage(intermediateMessage) + self.messageHistoryTable.updateMessage(index.id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations) + } + } + + private func filterStoredMessageIds(messageIds: Set) -> Set { + var filteredIds = Set() + + for id in messageIds { + if self.messageHistoryIndexTable.exists(id) { + filteredIds.insert(id) + } + } + + return filteredIds + } + public func modify(f: Modifier -> T) -> Signal { return Signal { subscriber in self.queue.dispatch { @@ -337,37 +553,18 @@ public final class Postbox { } } - public func tailMessageHistoryViewForPeerId(peerId: PeerId, count: Int) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return self.aroundMessageHistoryViewForPeerId(peerId, index: MessageIndex.upperBound(peerId), count: count) + public func tailMessageHistoryViewForPeerId(peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + return self.aroundMessageHistoryViewForPeerId(peerId, index: MessageIndex.upperBound(peerId), count: count, tagMask: tagMask) } - public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return Signal { subscriber in let disposable = MetaDisposable() self.queue.dispatch { - let list = self.fetchAroundHistoryEntries(index, count: count + 2) + let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) - var entries: [MutableMessageHistoryEntry] = [] - var earlier: MutableMessageHistoryEntry? - var later: MutableMessageHistoryEntry? - - if list.count >= count + 2 { - earlier = list[0] - for i in 1 ..< count + 1 { - entries.append(list[i]) - } - later = list[count + 1] - } else if list.count >= count + 1 { - for i in 0 ..< count { - entries.append(list[i]) - } - later = list[count] - } else { - entries = list - } - - let mutableView = MutableMessageHistoryView(earlier: earlier, entries: entries, later: later, count: count) + let mutableView = MutableMessageHistoryView(earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) mutableView.render(self.renderIntermediateMessage) subscriber.putNext((MessageHistoryView(mutableView), .Generic)) @@ -394,27 +591,17 @@ public final class Postbox { } public func tailChatListView(count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { + return self.aroundChatListView(MessageIndex.absoluteUpperBound(), count: count) + } + + public func aroundChatListView(index: MessageIndex, count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { return Signal { subscriber in let disposable = MetaDisposable() self.queue.dispatch { - let tail = self.fetchEarlierChatEntries(nil, count: count + 1).reverse() - var entries: [MutableChatListEntry] = [] - var earlier: MutableChatListEntry? + let (entries, earlier, later) = self.fetchAroundChatEntries(index, count: count) - var i = 0 - for entry in tail { - if i < count { - entries.append(entry) - } else if i < count + 1 { - earlier = entry - } else { - break - } - i++ - } - - let mutableView = MutableChatListView(earlier: earlier, entries: entries, later: nil, count: count) + let mutableView = MutableChatListView(earlier: earlier, entries: entries, later: later, count: count) mutableView.render(self.renderIntermediateMessage) subscriber.putNext((ChatListView(mutableView), .Generic)) diff --git a/Postbox/SeedConfiguration.swift b/Postbox/SeedConfiguration.swift new file mode 100644 index 0000000000..cd0f8320be --- /dev/null +++ b/Postbox/SeedConfiguration.swift @@ -0,0 +1,13 @@ +import Foundation + +public final class SeedConfiguration { + let initializeChatListWithHoles: [ChatListHole] + let initializeMessageNamespacesWithHoles: [MessageId.Namespace] + let existingMessageTags: MessageTags + + public init(initializeChatListWithHoles: [ChatListHole], initializeMessageNamespacesWithHoles: [MessageId.Namespace], existingMessageTags: MessageTags) { + self.initializeChatListWithHoles = initializeChatListWithHoles + self.initializeMessageNamespacesWithHoles = initializeMessageNamespacesWithHoles + self.existingMessageTags = existingMessageTags + } +} diff --git a/Postbox/SimpleDictionary.swift b/Postbox/SimpleDictionary.swift index 3ab962a23a..48d8af5a1f 100644 --- a/Postbox/SimpleDictionary.swift +++ b/Postbox/SimpleDictionary.swift @@ -22,7 +22,7 @@ public struct SimpleDictionary: SequenceType { } return } - index++ + index += 1 } if let value = value { self.items.append((key, value)) @@ -32,9 +32,11 @@ public struct SimpleDictionary: SequenceType { public func generate() -> AnyGenerator<(K, V)> { var index = 0 - return anyGenerator { () -> (K, V)? in + return AnyGenerator { () -> (K, V)? in if index < self.items.count { - return self.items[index++] + let currentIndex = index + index += 1 + return self.items[currentIndex] } return nil } diff --git a/Postbox/SimpleSet.swift b/Postbox/SimpleSet.swift new file mode 100644 index 0000000000..74e0e659a5 --- /dev/null +++ b/Postbox/SimpleSet.swift @@ -0,0 +1,23 @@ +import Foundation + +public struct SimpleSet { + private var items: [T] = [] + + public init() { + } + + public mutating func insert(item: T) { + if !self.contains(item) { + self.items.append(item) + } + } + + public func contains(item: T) -> Bool { + for currentItem in self.items { + if currentItem == item { + return true + } + } + return false + } +} \ No newline at end of file diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 3e1be6dbdf..33631777b2 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -1,5 +1,6 @@ import Foundation import sqlcipher +import SwiftSignalKit private struct SqlitePreparedStatement { let statement: COpaquePointer @@ -53,7 +54,10 @@ private struct SqlitePreparedStatement { } public final class SqliteValueBox: ValueBox { - private let database: Database + private let lock = NSRecursiveLock() + + private let basePath: String + private var database: Database! private var tables = Set() private var getStatements: [Int32 : SqlitePreparedStatement] = [:] private var rangeKeyAscStatementsLimit: [Int32 : SqlitePreparedStatement] = [:] @@ -73,34 +77,58 @@ public final class SqliteValueBox: ValueBox { private var writeQueryTime: CFAbsoluteTime = 0.0 private var commitTime: CFAbsoluteTime = 0.0 + private let checkpoints = MetaDisposable() + public init(basePath: String) { - do { - try NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil) - } catch _ { } - let path = basePath + "/db_sqlite" - self.database = Database(path) - - self.database.adjustChunkSize() - self.database.execute("PRAGMA page_size=1024") - self.database.execute("PRAGMA cache_size=-2097152") - self.database.execute("PRAGMA synchronous=NORMAL") - self.database.execute("PRAGMA journal_mode=truncate") - self.database.execute("PRAGMA temp_store=MEMORY") - //self.database.execute("PRAGMA wal_autocheckpoint=32") - //self.database.execute("PRAGMA journal_size_limit=1536") - - let result = self.database.scalar("PRAGMA user_version") as! Int64 - if result != 1 { - self.database.execute("PRAGMA user_version=1") - self.database.execute("CREATE TABLE __meta_tables (name INTEGER)") - } - for row in self.database.prepare("SELECT name FROM __meta_tables").run() { - self.tables.insert(Int32(row[0] as! Int64)) - } + self.basePath = basePath + self.database = self.openDatabase() } deinit { self.clearStatements() + checkpoints.dispose() + } + + private func openDatabase() -> Database { + checkpoints.set(nil) + lock.lock() + + do { + try NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil) + } catch _ { } + let path = basePath + "/db_sqlite" + let database = Database(path) + + database.adjustChunkSize() + database.execute("PRAGMA page_size=1024") + database.execute("PRAGMA cache_size=-2097152") + database.execute("PRAGMA synchronous=NORMAL") + database.execute("PRAGMA journal_mode=WAL") + database.execute("PRAGMA temp_store=MEMORY") + database.execute("PRAGMA wal_autocheckpoint=200") + database.execute("PRAGMA journal_size_limit=1536") + + let result = database.scalar("PRAGMA user_version") as! Int64 + if result != 1 { + database.execute("PRAGMA user_version=1") + database.execute("CREATE TABLE __meta_tables (name INTEGER)") + } + for row in database.prepare("SELECT name FROM __meta_tables").run() { + self.tables.insert(Int32(row[0] as! Int64)) + } + lock.unlock() + + checkpoints.set((Signal.single(Void()) |> delay(10.0, queue: Queue.concurrentDefaultQueue()) |> restart).start(next: { [weak self] _ in + if let strongSelf = self where strongSelf.database != nil { + strongSelf.lock.lock() + var nLog: Int32 = 0 + var nFrames: Int32 = 0 + sqlite3_wal_checkpoint_v2(strongSelf.database.handle, nil, SQLITE_CHECKPOINT_PASSIVE, &nLog, &nFrames) + strongSelf.lock.unlock() + //print("(SQLite WAL size \(nLog) removed \(nFrames))") + } + })) + return database } public func beginStats() { @@ -633,9 +661,13 @@ public final class SqliteValueBox: ValueBox { public func drop() { self.clearStatements() - for table in self.tables { - self.database.execute("DROP TABLE IF EXISTS t\(table)") - } - self.database.execute("DELETE FROM __meta_tables") + self.lock.lock() + self.database = nil + self.lock.unlock() + + let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath) + self.database = self.openDatabase() + + tables.removeAll() } } diff --git a/Postbox/Table.swift b/Postbox/Table.swift new file mode 100644 index 0000000000..7ec87e866d --- /dev/null +++ b/Postbox/Table.swift @@ -0,0 +1,14 @@ +import Foundation + +class Table { + final let valueBox: ValueBox + final let tableId: Int32 + + init(valueBox: ValueBox, tableId: Int32) { + self.valueBox = valueBox + self.tableId = tableId + } + + func beforeCommit() { + } +} diff --git a/Postbox/UnsentMessageHistoryView.swift b/Postbox/UnsentMessageHistoryView.swift new file mode 100644 index 0000000000..39e76768a8 --- /dev/null +++ b/Postbox/UnsentMessageHistoryView.swift @@ -0,0 +1,40 @@ +import Foundation + +final class UnsentMessageHistoryView { + var indices: [MessageIndex] + + init(indices: [MessageIndex]) { + self.indices = indices + } + + func replay(operations: [IntermediateMessageHistoryUnsentOperation]) -> Bool { + var updated = false + for operation in operations { + switch operation { + case let .Insert(index): + var inserted = false + for i in 0 ..< self.indices.count { + if self.indices[i] > index { + self.indices.insert(index, atIndex: i) + inserted = true + break + } + } + if !inserted { + self.indices.append(index) + } + updated = true + case let .Remove(index): + for i in 0 ..< self.indices.count { + if self.indices[i] == index { + self.indices.removeAtIndex(i) + updated = true + break + } + } + } + } + + return updated + } +} diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index f017b8719a..643b5a39fd 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -43,23 +43,45 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { memcpy(self.memory + offset, &bigEndianValue, 4) } + public func setUInt32(offset: Int, value: UInt32) { + var bigEndianValue = UInt32(bigEndian: value) + memcpy(self.memory + offset, &bigEndianValue, 4) + } + public func setInt64(offset: Int, value: Int64) { var bigEndianValue = Int64(bigEndian: value) memcpy(self.memory + offset, &bigEndianValue, 8) } + public func setInt8(offset: Int, value: Int8) { + var varValue = value + memcpy(self.memory + offset, &varValue, 1) + } + public func getInt32(offset: Int) -> Int32 { var value: Int32 = 0 memcpy(&value, self.memory + offset, 4) return Int32(bigEndian: value) } + public func getUInt32(offset: Int) -> UInt32 { + var value: UInt32 = 0 + memcpy(&value, self.memory + offset, 4) + return UInt32(bigEndian: value) + } + public func getInt64(offset: Int) -> Int64 { var value: Int64 = 0 memcpy(&value, self.memory + offset, 8) return Int64(bigEndian: value) } + public func getInt8(offset: Int) -> Int8 { + var value: Int8 = 0 + memcpy(&value, self.memory + offset, 1) + return value + } + public func prefix(length: Int) -> ValueBoxKey { assert(length <= self.length, "length <= self.length") let key = ValueBoxKey(length: length) @@ -82,7 +104,7 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { byte = 0 memory[i] = byte } - i-- + i -= 1 } return key } @@ -102,7 +124,7 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { byte = 0xff memory[i] = byte } - i-- + i -= 1 } return key } diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 6ffcbfcf9b..276b655c86 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -8,29 +8,43 @@ public enum ViewUpdateType { final class ViewTracker { private let queue: Queue - private let fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int) -> [MutableMessageHistoryEntry] - private let fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int) -> [MutableMessageHistoryEntry] + private let fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry] + private let fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry] private let fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] private let fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] private let renderMessage: IntermediateMessage -> Message - private let fetchMessageHistoryHole: MessageHistoryHole -> Disposable + private let fetchChatListHole: ChatListHole -> Disposable + private let fetchMessageHistoryHole: (MessageHistoryHole, MessageTags?) -> Disposable + private let sendUnsentMessage: MessageIndex -> Disposable - private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] private var chatListViews = Bag<(MutableChatListView, Pipe<(ChatListView, ViewUpdateType)>)>() + private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] + private var unsentMessageView: UnsentMessageHistoryView + private var chatListHoleDisposables: [(ChatListHole, Disposable)] = [] private var holeDisposablesByPeerId: [PeerId: [(MessageHistoryHole, Disposable)]] = [:] + private var unsentMessageDisposables: [MessageIndex: Disposable] = [:] - init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], renderMessage: IntermediateMessage -> Message, fetchMessageHistoryHole: MessageHistoryHole -> Disposable) { + init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], renderMessage: IntermediateMessage -> Message, fetchChatListHole: ChatListHole -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, MessageTags?) -> Disposable, sendUnsentMessage: MessageIndex -> Disposable, unsentMessageIndices: [MessageIndex]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries self.fetchEarlierChatEntries = fetchEarlierChatEntries self.fetchLaterChatEntries = fetchLaterChatEntries self.renderMessage = renderMessage + self.fetchChatListHole = fetchChatListHole self.fetchMessageHistoryHole = fetchMessageHistoryHole + self.sendUnsentMessage = sendUnsentMessage + + self.unsentMessageView = UnsentMessageHistoryView(indices: unsentMessageIndices) + self.unsentViewUpdated() } deinit { + for (_, disposable) in chatListHoleDisposables { + disposable.dispose() + } + for (_, indexToDisposable) in holeDisposablesByPeerId { for (_, disposable) in indexToDisposable { disposable.dispose() @@ -65,16 +79,19 @@ final class ViewTracker { func addChatListView(view: MutableChatListView) -> (Bag<(MutableChatListView, Pipe<(ChatListView, ViewUpdateType)>)>.Index, Signal<(ChatListView, ViewUpdateType), NoError>) { let record = (view, Pipe<(ChatListView, ViewUpdateType)>()) - let index = self.chatListViews.add(record) + + self.updateTrackedChatListHoles() + return (index, record.1.signal()) } func removeChatListView(index: Bag<(MutableChatListView, Pipe)>.Index) { self.chatListViews.remove(index) + self.updateTrackedChatListHoles() } - func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: Set, chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer]) { + func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: Set, chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { @@ -84,11 +101,12 @@ final class ViewTracker { for (mutableView, pipe) in bag.copyItems() { let context = MutableMessageHistoryViewReplayContext() var updated = false + if mutableView.replay(operations, context: context) { mutableView.complete(context, fetchEarlier: { index, count in - return self.fetchEarlierHistoryEntries(peerId, index, count) + return self.fetchEarlierHistoryEntries(peerId, index, count, mutableView.tagMask) }, fetchLater: { index, count in - return self.fetchLaterHistoryEntries(peerId, index, count) + return self.fetchLaterHistoryEntries(peerId, index, count, mutableView.tagMask) }) updated = true } @@ -118,31 +136,82 @@ final class ViewTracker { pipe.putNext((ChatListView(mutableView), .Generic)) } } + + self.updateTrackedChatListHoles() } for peerId in updateTrackedHolesPeerIds { self.updateTrackedHoles(peerId) } + + if self.unsentMessageView.replay(unsentMessageOperations) { + self.unsentViewUpdated() + } } - func updateTrackedHoles(peerId: PeerId) { + private func updateTrackedChatListHoles() { + var disposeHoles: [Disposable] = [] + var firstHoles: [ChatListHole] = [] + + for (view, _) in self.chatListViews.copyItems() { + if let hole = view.firstHole() { + var exists = false + for existingHole in firstHoles { + if existingHole == hole { + exists = true + break + } + } + if !exists { + firstHoles.append(hole) + } + } + } + + var i = 0 + for (hole, disposable) in self.chatListHoleDisposables { + var exists = false + for firstHole in firstHoles { + if hole == firstHole { + exists = true + break + } + } + if !exists { + disposeHoles.append(disposable) + self.chatListHoleDisposables.removeAtIndex(i) + } else { + i += 1 + } + } + + for disposable in disposeHoles { + disposable.dispose() + } + + for hole in firstHoles { + var exists = false + for (existingHole, _) in self.chatListHoleDisposables { + if existingHole == hole { + exists = true + break + } + } + + if !exists { + self.chatListHoleDisposables.append((hole, self.fetchChatListHole(hole))) + } + } + } + + private func updateTrackedHoles(peerId: PeerId) { if let bag = self.messageHistoryViews[peerId] { var disposeHoles: [Disposable] = [] - - var firstHoles: [MessageHistoryHole] = [] + var firstHolesAndTags: [(MessageHistoryHole, MessageTags?)] = [] for (view, _) in bag.copyItems() { if let hole = view.firstHole() { - var exists = false - for existingHole in firstHoles { - if existingHole == hole { - exists = true - break - } - } - if !exists { - firstHoles.append(hole) - } + firstHolesAndTags.append((hole, view.tagMask)) } } @@ -150,7 +219,7 @@ final class ViewTracker { var i = 0 for (hole, disposable) in holes { var exists = false - for firstHole in firstHoles { + for (firstHole, _) in firstHolesAndTags { if hole == firstHole { exists = true break @@ -160,7 +229,7 @@ final class ViewTracker { disposeHoles.append(disposable) self.holeDisposablesByPeerId[peerId]!.removeAtIndex(i) } else { - i++ + i += 1 } } } @@ -169,23 +238,25 @@ final class ViewTracker { disposable.dispose() } - for hole in firstHoles { - var exists = false - if let existingHoles = self.holeDisposablesByPeerId[hole.id.peerId] { - for (existingHole, _) in existingHoles { - if existingHole == hole { - exists = true - break + if let anyHoleAndTag = firstHolesAndTags.first { + if self.holeDisposablesByPeerId[peerId] == nil || self.holeDisposablesByPeerId[peerId]!.count == 0 { + var exists = false + if let existingHoles = self.holeDisposablesByPeerId[peerId] { + for (existingHole, _) in existingHoles { + if existingHole == anyHoleAndTag.0 { + exists = true + break + } } } - } - - if !exists { - if self.holeDisposablesByPeerId[hole.id.peerId] == nil { - self.holeDisposablesByPeerId[hole.id.peerId] = [] - } - self.holeDisposablesByPeerId[hole.id.peerId]!.append((hole, self.fetchMessageHistoryHole(hole))) + if !exists { + if self.holeDisposablesByPeerId[peerId] == nil { + self.holeDisposablesByPeerId[peerId] = [] + } + + self.holeDisposablesByPeerId[peerId]!.append((anyHoleAndTag.0, self.fetchMessageHistoryHole(anyHoleAndTag.0, anyHoleAndTag.1))) + } } } } else if let holes = self.holeDisposablesByPeerId[peerId] { @@ -196,4 +267,39 @@ final class ViewTracker { } } } + + private func unsentViewUpdated() { + var removeIndices: [MessageIndex] = [] + for (index, _) in unsentMessageDisposables { + var found = false + for currentIndex in self.unsentMessageView.indices { + if currentIndex == index { + found = true + break + } + } + + if !found { + removeIndices.append(index) + } + } + + for index in removeIndices { + self.unsentMessageDisposables.removeValueForKey(index)?.dispose() + } + + for index in self.unsentMessageView.indices { + var found = false + for (currentIndex, _) in unsentMessageDisposables { + if index == currentIndex { + found = true + break + } + } + + if !found { + self.unsentMessageDisposables[index] = self.sendUnsentMessage(index) + } + } + } } diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index f8a7e39f76..1e0c17ff18 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -13,6 +13,7 @@ private let authorPeerId = PeerId(namespace: 2, id: 3) private enum Entry: Equatable, CustomStringConvertible { case Message(Int32, Int32, Int32) case Nothing(Int32, Int32, Int32) + case Hole(Int32, Int32, Int32) var description: String { switch self { @@ -20,6 +21,8 @@ private enum Entry: Equatable, CustomStringConvertible { return "Message(\(peerId), \(id), \(timestamp))" case let .Nothing(peerId, id, timestamp): return "Nothing(\(peerId), \(id), \(timestamp))" + case let .Hole(peerId, id, timestamp): + return "Hole(\(peerId), \(id), \(timestamp))" } } } @@ -32,6 +35,8 @@ private func ==(lhs: Entry, rhs: Entry) -> Bool { return lhsPeerId == rhsPeerId && lhsId == rhsId && lhsTimestamp == rhsTimestamp case .Nothing: return false + case .Hole: + return false } case let .Nothing(lhsPeerId, lhsId, lhsTimestamp): switch rhs { @@ -39,6 +44,17 @@ private func ==(lhs: Entry, rhs: Entry) -> Bool { return false case let .Nothing(rhsPeerId, rhsId, rhsTimestamp): return lhsPeerId == rhsPeerId && lhsId == rhsId && lhsTimestamp == rhsTimestamp + case .Hole: + return false + } + case let .Hole(lhsPeerId, lhsId, lhsTimestamp): + switch rhs { + case .Message: + return false + case .Nothing: + return false + case let .Hole(rhsPeerId, rhsId, rhsTimestamp): + return lhsPeerId == rhsPeerId && lhsId == rhsId && lhsTimestamp == rhsTimestamp } } } @@ -54,6 +70,9 @@ class ChatListTableTests: XCTestCase { var historyTable: MessageHistoryTable? var chatListIndexTable: ChatListIndexTable? var chatListTable: ChatListTable? + var historyMetadataTable: MessageHistoryMetadataTable? + var unsentTable: MessageHistoryUnsentTable? + var tagsTable: MessageHistoryTagsTable? override class func setUp() { super.setUp() @@ -67,13 +86,18 @@ class ChatListTableTests: XCTestCase { path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") self.valueBox = SqliteValueBox(basePath: path!) + let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: []) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 7, namespace: namespace) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 8) + self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, tableId: 9) + self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, tableId: 10) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, tableId: 5) - self.chatListTable = ChatListTable(valueBox: self.valueBox!, tableId: 6, indexTable: self.chatListIndexTable!) + self.chatListTable = ChatListTable(valueBox: self.valueBox!, tableId: 6, indexTable: self.chatListIndexTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) } override func tearDown() { @@ -93,28 +117,47 @@ class ChatListTableTests: XCTestCase { private func addMessage(peerId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = []) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId) + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } private func addHole(peerId: Int32, _ id: Int32) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.historyTable!.addHoles([MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId) + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + self.historyTable!.addHoles([MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } + private func addChatListHole(peerId: Int32, _ id: Int32, _ timestamp: Int32) { + var operations: [ChatListOperation] = [] + self.chatListTable!.addHole(ChatListHole(index: MessageIndex(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp)), operations: &operations) + } + + private func replaceChatListHole(peerId: Int32, _ id: Int32, _ timestamp: Int32, _ otherPeerId: Int32, _ otherId: Int32, _ otherTimestamp: Int32) { + var operations: [ChatListOperation] = [] + self.chatListTable!.replaceHole(MessageIndex(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp), hole: ChatListHole(index: MessageIndex(id: MessageId(peerId: PeerId(namespace: namespace, id: otherPeerId), namespace: namespace, id: otherId), timestamp: otherTimestamp)), operations: &operations) + } + + private func removeChatListHole(peerId: Int32, _ id: Int32, _ timestamp: Int32) { + var operations: [ChatListOperation] = [] + self.chatListTable!.replaceHole(MessageIndex(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp), hole: nil, operations: &operations) + } + private func removeMessages(peerId: Int32, _ ids: [Int32]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.historyTable!.removeMessages(ids.map({ MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId) + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + self.historyTable!.removeMessages(ids.map({ MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } private func fillHole(peerId: Int32, _ id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32, String, [Media])]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.historyTable!.fillHole(MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), fillType: fillType, messages: messages.map({ StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0.0), timestamp: $0.1, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId) + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + self.historyTable!.fillHole(MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), fillType: fillType, tagMask: nil, messages: messages.map({ StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } @@ -127,6 +170,8 @@ class ChatListTableTests: XCTestCase { XCTFail("Expected authorId \(authorPeerId), actual \(message.authorId)") } return .Message(message.id.peerId.id, message.id.id, message.timestamp) + case let .Hole(hole): + return .Hole(hole.index.id.peerId.id, hole.index.id.id, hole.index.timestamp) case let .Nothing(index): return .Nothing(index.id.peerId.id, index.id.id, index.timestamp) } @@ -182,4 +227,45 @@ class ChatListTableTests: XCTestCase { addMessage(1, 100, 100) expectEntries([.Message(1, 100, 100)]) } + + func testInsertHoleIntoEmpty() { + addChatListHole(1, 10, 10) + expectEntries([.Hole(1, 10, 10)]) + } + + func testInsertHoleLower() { + addMessage(1, 100, 100) + addChatListHole(1, 10, 10) + expectEntries([.Hole(1, 10, 10), .Message(1, 100, 100)]) + } + + func testInsertHoleUpper() { + addMessage(1, 100, 100) + addChatListHole(1, 200, 200) + expectEntries([.Message(1, 100, 100), .Hole(1, 200, 200)]) + } + + func testIgnoreRemoveHole() { + addChatListHole(1, 100, 100) + removeMessages(1, [100]) + expectEntries([.Hole(1, 100, 100)]) + + addMessage(1, 100, 100) + expectEntries([.Message(1, 100, 100), .Hole(1, 100, 100)]) + + removeMessages(1, [100]) + expectEntries([.Nothing(1, 100, 100), .Hole(1, 100, 100)]) + } + + func testReplaceHoleWithHole() { + addChatListHole(1, 100, 100) + replaceChatListHole(1, 100, 100, 2, 200, 200) + expectEntries([.Hole(2, 200, 200)]) + } + + func testReplaceHoleWithNone() { + addChatListHole(1, 100, 100) + removeChatListHole(1, 100, 100) + expectEntries([]) + } } \ No newline at end of file diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index ca5b7bb9db..17d46e1a54 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -59,6 +59,7 @@ class MessageHistoryIndexTableTests: XCTestCase { var indexTable: MessageHistoryIndexTable? var globalMessageIdsTable: GlobalMessageIdsTable? + var historyMetadataTable: MessageHistoryMetadataTable? override func setUp() { super.setUp() @@ -68,8 +69,11 @@ class MessageHistoryIndexTableTests: XCTestCase { path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") self.valueBox = SqliteValueBox(basePath: path!) + let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: []) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 2, namespace: namespace) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 8) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) } override func tearDown() { @@ -89,24 +93,24 @@ class MessageHistoryIndexTableTests: XCTestCase { func addMessage(id: Int32, _ timestamp: Int32) { var operations: [MessageHistoryIndexOperation] = [] - self.indexTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, authorId: peerId, text: "", attributes: [], media: [])], location: .Random, operations: &operations) + self.indexTable!.addMessages([InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: [])], location: .Random, operations: &operations) } func addMessagesUpperBlock(messages: [(Int32, Int32)]) { var operations: [MessageHistoryIndexOperation] = [] self.indexTable!.addMessages(messages.map { (id, timestamp) in - return StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, authorId: peerId, text: "", attributes: [], media: []) + return InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: []) }, location: .UpperHistoryBlock, operations: &operations) } - func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32)]) { + func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32)], _ tagMask: MessageTags? = nil) { var operations: [MessageHistoryIndexOperation] = [] let zeroData = WriteBuffer() var zero: Int32 = 0 zeroData.write(&zero, offset: 0, length: 4) - self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, messages: messages.map({ - return StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, authorId: peerId, text: "", attributes: [], media: []) + self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ + return InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: []) }), operations: &operations) } diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index a9f7e3d6f9..bacbc988f7 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -25,13 +25,13 @@ private func ==(lhs: [Media], rhs: [Media]) -> Bool { } private enum Entry: Equatable, CustomStringConvertible { - case Message(Int32, Int32, String, [Media]) + case Message(Int32, Int32, String, [Media], MessageFlags) case Hole(Int32, Int32, Int32) var description: String { switch self { - case let .Message(id, timestamp, text, media): - return "Message(\(id), \(timestamp), \(text), \(media))" + case let .Message(id, timestamp, text, media, flags): + return "Message(\(id), \(timestamp), \(text), \(media), \(flags))" case let .Hole(min, max, timestamp): return "Hole(\(min), \(max), \(timestamp))" } @@ -40,10 +40,10 @@ private enum Entry: Equatable, CustomStringConvertible { private func ==(lhs: Entry, rhs: Entry) -> Bool { switch lhs { - case let .Message(lhsId, lhsTimestamp, lhsText, lhsMedia): + case let .Message(lhsId, lhsTimestamp, lhsText, lhsMedia, lhsFlags): switch rhs { - case let .Message(rhsId, rhsTimestamp, rhsText, rhsMedia): - return lhsId == rhsId && lhsTimestamp == rhsTimestamp && lhsText == rhsText && lhsMedia == rhsMedia + case let .Message(rhsId, rhsTimestamp, rhsText, rhsMedia, rhsFlags): + return lhsId == rhsId && lhsTimestamp == rhsTimestamp && lhsText == rhsText && lhsMedia == rhsMedia && lhsFlags == rhsFlags case .Hole: return false } @@ -184,6 +184,11 @@ private func ==(lhs: MediaEntry, rhs: MediaEntry) -> Bool { } } +private extension MessageTags { + static let First = MessageTags(rawValue: 1 << 0) + static let Second = MessageTags(rawValue: 1 << 1) +} + class MessageHistoryTableTests: XCTestCase { var valueBox: ValueBox? var path: String? @@ -194,6 +199,9 @@ class MessageHistoryTableTests: XCTestCase { var mediaTable: MessageMediaTable? var mediaCleanupTable: MediaCleanupTable? var historyTable: MessageHistoryTable? + var historyMetadataTable: MessageHistoryMetadataTable? + var unsentTable: MessageHistoryUnsentTable? + var tagsTable: MessageHistoryTagsTable? override class func setUp() { super.setUp() @@ -211,11 +219,16 @@ class MessageHistoryTableTests: XCTestCase { path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") self.valueBox = SqliteValueBox(basePath: path!) + let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 5, namespace: namespace) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 7) + self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, tableId: 8) + self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, tableId: 9) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!) self.peerTable = PeerTable(valueBox: self.valueBox!, tableId: 6) self.peerTable!.set(peer) } @@ -228,38 +241,58 @@ class MessageHistoryTableTests: XCTestCase { self.mediaTable = nil self.mediaCleanupTable = nil self.peerTable = nil + self.historyMetadataTable = nil self.valueBox = nil let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) self.path = nil } - private func addMessage(id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = []) { + private func addMessage(id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags = [], _ tags: MessageTags = []) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId) - //print("\(operationsByPeerId[peerId]!)") + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + } + + private func updateMessage(previousId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags, _ tags: MessageTags) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) } private func addHole(id: Int32) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.historyTable!.addHoles([MessageId(peerId: peerId, namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId) + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + self.historyTable!.addHoles([MessageId(peerId: peerId, namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) } private func removeMessages(ids: [Int32]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId) - //print("\(operationsByPeerId[peerId]!)") + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) } - private func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32, String, [Media])]) { + private func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32, String, [Media])], _ tagMask: MessageTags? = nil) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.historyTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId) + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + self.historyTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) } - private func expectEntries(entries: [Entry]) { - let actualEntries = self.historyTable!.debugList(peerId, peerTable: self.peerTable!).map({ entry -> Entry in + private func expectEntries(entries: [Entry], tagMask: MessageTags? = nil) { + var stableIds = Set() + + let list: [RenderedMessageHistoryEntry] + if let tagMask = tagMask { + list = self.historyTable!.debugList(tagMask, peerId: peerId, peerTable: self.peerTable!) + } else { + list = self.historyTable!.debugList(peerId, peerTable: self.peerTable!) + } + + let actualEntries = list.map({ entry -> Entry in + let stableId: UInt32 switch entry { case let .RenderedMessage(message): + stableId = message.stableId if let messagePeer = message.author { if !peer.isEqual(messagePeer) { XCTFail("Expected peer \(peer), actual: \(messagePeer)") @@ -267,8 +300,19 @@ class MessageHistoryTableTests: XCTestCase { } else { XCTFail("Expected peer \(peer), actual: nil") } - return .Message(message.id.id, message.timestamp, message.text, message.media) + if stableIds.contains(stableId) { + XCTFail("Stable id not unique \(stableId)") + } else { + stableIds.insert(stableId) + } + return .Message(message.id.id, message.timestamp, message.text, message.media, message.flags) case let .Hole(hole): + stableId = hole.stableId + if stableIds.contains(stableId) { + XCTFail("Stable id not unique \(stableId)") + } else { + stableIds.insert(stableId) + } return .Hole(hole.min, hole.maxIndex.id.id, hole.maxIndex.timestamp) } }) @@ -277,6 +321,24 @@ class MessageHistoryTableTests: XCTestCase { } } + private func expectUnsent(indices: [(Int32, Int32)]) { + let actualUnsent = self.unsentTable!.get().map({ ($0.id.id, $0.timestamp) }) + var match = true + if actualUnsent.count == indices.count { + for i in 0 ..< indices.count { + if indices[i].0 != actualUnsent[i].0 || indices[i].1 != actualUnsent[i].1 { + match = false + break + } + } + } else { + match = false + } + if !match { + XCTFail("Expected\n\(indices)\nActual\n\(actualUnsent)") + } + } + private func expectMedia(media: [MediaEntry]) { let actualMedia = self.mediaTable!.debugList().map({MediaEntry($0)}) if media != actualMedia { @@ -307,20 +369,20 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "t100") addMessage(200, 200, "t200") - expectEntries([.Message(100, 100, "t100", []), .Message(200, 200, "t200", [])]) + expectEntries([.Message(100, 100, "t100", [], []), .Message(200, 200, "t200", [], [])]) } func testInsertMessageIgnoreOverwrite() { addMessage(100, 100, "t100") addMessage(100, 200, "t200") - expectEntries([.Message(100, 100, "t100", [])]) + expectEntries([.Message(100, 100, "t100", [], [])]) } func testInsertMessageWithEmbeddedMedia() { addMessage(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")]) - expectEntries([.Message(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")])]) + expectEntries([.Message(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")], [])]) expectMedia([]) } @@ -328,7 +390,7 @@ class MessageHistoryTableTests: XCTestCase { let media = TestExternalMedia(id: 10, data: "abc1") addMessage(100, 100, "t100", [media]) - expectEntries([.Message(100, 100, "t100", [media])]) + expectEntries([.Message(100, 100, "t100", [media], [])]) expectMedia([.MessageReference(100)]) } @@ -337,7 +399,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "t100", [media]) addMessage(200, 200, "t200", [media]) - expectEntries([.Message(100, 100, "t100", [media]), .Message(200, 200, "t200", [media])]) + expectEntries([.Message(100, 100, "t100", [media], []), .Message(200, 200, "t200", [media], [])]) expectMedia([.Direct(media, 2)]) } @@ -347,7 +409,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "t100", [media]) addMessage(200, 200, "t200", [media1]) - expectEntries([.Message(100, 100, "t100", [media]), .Message(200, 200, "t200", [media])]) + expectEntries([.Message(100, 100, "t100", [media], []), .Message(200, 200, "t200", [media], [])]) expectMedia([.Direct(media, 2)]) } @@ -386,7 +448,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "t200", [media]) removeMessages([100]) - expectEntries([.Message(200, 200, "t200", [media])]) + expectEntries([.Message(200, 200, "t200", [media], [])]) expectMedia([.Direct(media, 1)]) expectCleanupMedia([]) @@ -413,25 +475,25 @@ class MessageHistoryTableTests: XCTestCase { addHole(100) expectEntries([.Hole(1, Int32.max, Int32.max)]) addMessage(90, 90, "m90") - expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", [], []), .Hole(91, Int32.max, Int32.max)]) } func testAddMessageDividingUpperHole() { addHole(100) expectEntries([.Hole(1, Int32.max, Int32.max)]) addMessage(90, 90, "m90") - expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", [], []), .Hole(91, Int32.max, Int32.max)]) addMessage(100, 100, "m100") - expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, 99, 100), .Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", [], []), .Hole(91, 99, 100), .Message(100, 100, "m100", [], []), .Hole(101, Int32.max, Int32.max)]) } func testAddMessageDividingLowerHole() { addHole(100) expectEntries([.Hole(1, Int32.max, Int32.max)]) addMessage(90, 90, "m90") - expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", [], []), .Hole(91, Int32.max, Int32.max)]) addMessage(80, 80, "m80") - expectEntries([.Hole(1, 79, 80), .Message(80, 80, "m80", []), .Hole(81, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 79, 80), .Message(80, 80, "m80", [], []), .Hole(81, 89, 90), .Message(90, 90, "m90", [], []), .Hole(91, Int32.max, Int32.max)]) } func testAddMessageOffsettingUpperHole() { @@ -439,9 +501,9 @@ class MessageHistoryTableTests: XCTestCase { expectEntries([.Hole(1, Int32.max, Int32.max)]) addMessage(90, 90, "m90") - expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", [], []), .Hole(91, Int32.max, Int32.max)]) addMessage(91, 91, "m91") - expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Message(91, 91, "m91", []), .Hole(92, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", [], []), .Message(91, 91, "m91", [], []), .Hole(92, Int32.max, Int32.max)]) } func testAddMessageOffsettingLowerHole() { @@ -449,9 +511,9 @@ class MessageHistoryTableTests: XCTestCase { expectEntries([.Hole(1, Int32.max, Int32.max)]) addMessage(90, 90, "m90") - expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 89, 90), .Message(90, 90, "m90", [], []), .Hole(91, Int32.max, Int32.max)]) addMessage(89, 89, "m89") - expectEntries([.Hole(1, 88, 89), .Message(89, 89, "m89", []), .Message(90, 90, "m90", []), .Hole(91, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 88, 89), .Message(89, 89, "m89", [], []), .Message(90, 90, "m90", [], []), .Hole(91, Int32.max, Int32.max)]) } func testAddMessageOffsettingLeftmostHole() { @@ -460,7 +522,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(1, 1, "m1") - expectEntries([.Message(1, 1, "m1", []), .Hole(2, Int32.max, Int32.max)]) + expectEntries([.Message(1, 1, "m1", [], []), .Hole(2, Int32.max, Int32.max)]) } func testAddMessageRemovingLefmostHole() { @@ -468,24 +530,24 @@ class MessageHistoryTableTests: XCTestCase { expectEntries([.Hole(1, Int32.max, Int32.max)]) addMessage(2, 2, "m2") - expectEntries([.Hole(1, 1, 2), .Message(2, 2, "m2", []), .Hole(3, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 1, 2), .Message(2, 2, "m2", [], []), .Hole(3, Int32.max, Int32.max)]) addMessage(1, 1, "m1") - expectEntries([.Message(1, 1, "m1", []), .Message(2, 2, "m2", []), .Hole(3, Int32.max, Int32.max)]) + expectEntries([.Message(1, 1, "m1", [], []), .Message(2, 2, "m2", [], []), .Hole(3, Int32.max, Int32.max)]) } func testAddHoleLowerThanMessage() { addMessage(100, 100, "m100") addHole(1) - expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [])]) + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], [])]) } func testAddHoleHigherThanMessage() { addMessage(100, 100, "m100") addHole(200) - expectEntries([.Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, Int32.max, Int32.max)]) } func testIgnoreHigherHole() { @@ -498,9 +560,9 @@ class MessageHistoryTableTests: XCTestCase { func testIgnoreHigherHoleAfterMessage() { addMessage(100, 100, "m100") addHole(200) - expectEntries([.Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, Int32.max, Int32.max)]) addHole(400) - expectEntries([.Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, Int32.max, Int32.max)]) } func testAddHoleBetweenMessages() { @@ -508,7 +570,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200") addHole(150) - expectEntries([.Message(100, 100, "m100", []), .Hole(101, 199, 200), .Message(200, 200, "m200", [])]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 199, 200), .Message(200, 200, "m200", [], [])]) } func testFillHoleEmpty() { @@ -520,35 +582,35 @@ class MessageHistoryTableTests: XCTestCase { addHole(100) fillHole(1, .Complete, [(100, 100, "m100", []), (200, 200, "m200", [])]) - expectEntries([.Message(100, 100, "m100", []), .Message(200, 200, "m200", [])]) + expectEntries([.Message(100, 100, "m100", [], []), .Message(200, 200, "m200", [], [])]) } func testFillHoleUpperToLowerPartial() { addHole(100) fillHole(1, .UpperToLower, [(100, 100, "m100", []), (200, 200, "m200", [])]) - expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Message(200, 200, "m200", [])]) + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Message(200, 200, "m200", [], [])]) } func testFillHoleUpperToLowerToBounds() { addHole(100) fillHole(1, .UpperToLower, [(1, 1, "m1", []), (200, 200, "m200", [])]) - expectEntries([.Message(1, 1, "m1", []), .Message(200, 200, "m200", [])]) + expectEntries([.Message(1, 1, "m1", [], []), .Message(200, 200, "m200", [], [])]) } func testFillHoleLowerToUpperToBounds() { addHole(100) fillHole(1, .LowerToUpper, [(100, 100, "m100", []), (Int32.max, 200, "m200", [])]) - expectEntries([.Message(100, 100, "m100", []), .Message(Int32.max, 200, "m200", [])]) + expectEntries([.Message(100, 100, "m100", [], []), .Message(Int32.max, 200, "m200", [], [])]) } func testFillHoleLowerToUpperPartial() { addHole(100) fillHole(1, .LowerToUpper, [(100, 100, "m100", []), (200, 200, "m200", [])]) - expectEntries([.Message(100, 100, "m100", []), .Message(200, 200, "m200", []), .Hole(201, Int32.max, Int32.max)]) + expectEntries([.Message(100, 100, "m100", [], []), .Message(200, 200, "m200", [], []), .Hole(201, Int32.max, Int32.max)]) } func testFillHoleBetweenMessagesUpperToLower() { @@ -559,7 +621,7 @@ class MessageHistoryTableTests: XCTestCase { fillHole(199, .UpperToLower, [(150, 150, "m150", [])]) - expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Hole(101, 149, 150), .Message(150, 150, "m150", []), .Message(200, 200, "m200", []), .Hole(201, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Hole(101, 149, 150), .Message(150, 150, "m150", [], []), .Message(200, 200, "m200", [], []), .Hole(201, Int32.max, Int32.max)]) } func testFillHoleBetweenMessagesLowerToUpper() { @@ -570,7 +632,7 @@ class MessageHistoryTableTests: XCTestCase { fillHole(199, .LowerToUpper, [(150, 150, "m150", [])]) - expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Message(150, 150, "m150", []), .Hole(151, 199, 200), .Message(200, 200, "m200", []), .Hole(201, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Message(150, 150, "m150", [], []), .Hole(151, 199, 200), .Message(200, 200, "m200", [], []), .Hole(201, Int32.max, Int32.max)]) } func testFillHoleBetweenMessagesComplete() { @@ -581,7 +643,7 @@ class MessageHistoryTableTests: XCTestCase { fillHole(199, .Complete, [(150, 150, "m150", [])]) - expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Message(150, 150, "m150", []), .Message(200, 200, "m200", []), .Hole(201, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Message(150, 150, "m150", [], []), .Message(200, 200, "m200", [], []), .Hole(201, Int32.max, Int32.max)]) } func testFillHoleBetweenMessagesWithMessage() { @@ -590,7 +652,7 @@ class MessageHistoryTableTests: XCTestCase { addHole(201) addMessage(201, 201, "m201") - expectEntries([.Message(200, 200, "m200", []), .Message(201, 201, "m201", []), .Message(202, 202, "m202", [])]) + expectEntries([.Message(200, 200, "m200", [], []), .Message(201, 201, "m201", [], []), .Message(202, 202, "m202", [], [])]) } func testFillHoleWithNoMessagesComplete() { @@ -599,7 +661,7 @@ class MessageHistoryTableTests: XCTestCase { fillHole(99, .Complete, []) - expectEntries([.Message(100, 100, "m100", [])]) + expectEntries([.Message(100, 100, "m100", [], [])]) } func testFillHoleIgnoreOverMessage() { @@ -608,7 +670,7 @@ class MessageHistoryTableTests: XCTestCase { fillHole(100, .Complete, [(90, 90, "m90", [])]) - expectEntries([.Message(90, 90, "m90", []), .Message(100, 100, "m100", []), .Message(101, 101, "m101", [])]) + expectEntries([.Message(90, 90, "m90", [], []), .Message(100, 100, "m100", [], []), .Message(101, 101, "m101", [], [])]) } func testFillHoleWithOverflow() { @@ -618,7 +680,7 @@ class MessageHistoryTableTests: XCTestCase { fillHole(199, .UpperToLower, [(150, 150, "m150", []), (300, 300, "m300", [])]) - expectEntries([.Message(100, 100, "m100", []), .Hole(101, 149, 150), .Message(150, 150, "m150", []), .Message(200, 200, "m200", []), .Message(300, 300, "m300", [])]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 149, 150), .Message(150, 150, "m150", [], []), .Message(200, 200, "m200", [], []), .Message(300, 300, "m300", [], [])]) } func testIgnoreHoleOverMessageBetweenMessages() { @@ -626,7 +688,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200") addHole(200) - expectEntries([.Message(199, 199, "m199", []), .Message(200, 200, "m200", [])]) + expectEntries([.Message(199, 199, "m199", [], []), .Message(200, 200, "m200", [], [])]) } func testMergeHoleAfterDeletingMessage() { @@ -634,7 +696,7 @@ class MessageHistoryTableTests: XCTestCase { addHole(1) addHole(200) - expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Hole(101, Int32.max, Int32.max)]) removeMessages([100]) @@ -648,7 +710,7 @@ class MessageHistoryTableTests: XCTestCase { removeMessages([100]) - expectEntries([.Hole(1, 199, 200), .Message(200, 200, "m200", [])]) + expectEntries([.Hole(1, 199, 200), .Message(200, 200, "m200", [], [])]) } func testMergeHoleUpperAfterDeletingMessage() { @@ -658,7 +720,7 @@ class MessageHistoryTableTests: XCTestCase { removeMessages([200]) - expectEntries([.Message(100, 100, "m100", []), .Hole(101, Int32.max, Int32.max)]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, Int32.max, Int32.max)]) } func testExtendLowerHoleAfterDeletingMessage() { @@ -684,7 +746,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200") removeMessages([100]) - expectEntries([.Message(200, 200, "m200", [])]) + expectEntries([.Message(200, 200, "m200", [], [])]) } func testDeleteMessageAboveMessage() { @@ -692,7 +754,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200") removeMessages([200]) - expectEntries([.Message(100, 100, "m100", [])]) + expectEntries([.Message(100, 100, "m100", [], [])]) } func testDeleteMessageBetweenMessages() { @@ -701,6 +763,213 @@ class MessageHistoryTableTests: XCTestCase { addMessage(300, 300, "m300") removeMessages([200]) - expectEntries([.Message(100, 100, "m100", []), .Message(300, 300, "m300", [])]) + expectEntries([.Message(100, 100, "m100", [], []), .Message(300, 300, "m300", [], [])]) } -} \ No newline at end of file + + func testAddUnsent() { + addMessage(100, 100, "m100", [], [.Unsent]) + expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) + expectUnsent([(100, 100)]) + } + + func testRemoveUnsent() { + addMessage(100, 100, "m100", [], [.Unsent]) + expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) + expectUnsent([(100, 100)]) + + removeMessages([100]) + expectEntries([]) + expectUnsent([]) + } + + func testUpdateUnsentToSentSameIndex() { + addMessage(100, 100, "m100", [], [.Unsent]) + expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) + expectUnsent([(100, 100)]) + + updateMessage(100, 100, 100, "m100", [], [], []) + expectEntries([.Message(100, 100, "m100", [], [])]) + expectUnsent([]) + } + + func testUpdateUnsentToFailed() { + addMessage(100, 100, "m100", [], [.Unsent]) + expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) + expectUnsent([(100, 100)]) + + updateMessage(100, 100, 100, "m100", [], [.Unsent, .Failed], []) + expectEntries([.Message(100, 100, "m100", [], [.Unsent, .Failed])]) + expectUnsent([]) + } + + func testUpdateDifferentIndex() { + addMessage(100, 100, "m100", [], [.Unsent]) + expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) + expectUnsent([(100, 100)]) + + updateMessage(100, 200, 200, "m100", [], [], []) + expectEntries([.Message(200, 200, "m100", [], [])]) + expectUnsent([]) + } + + func testUpdateDifferentIndexBreakHole() { + addHole(1) + + addMessage(100, 100, "m100", [], [.Unsent]) + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], [.Unsent]), .Hole(101, Int32.max, Int32.max)]) + expectUnsent([(100, 100)]) + + updateMessage(100, 200, 200, "m100", [], [], []) + expectEntries([.Hole(1, 199, 200), .Message(200, 200, "m100", [], []), .Hole(201, Int32.max, Int32.max)]) + expectUnsent([]) + } + + func testInsertTaggedIntoEmpty() { + addMessage(100, 100, "m100", [], [], MessageTags(rawValue: 1)) + expectEntries([.Message(100, 100, "m100", [], [])], tagMask: MessageTags(rawValue: 1)) + } + + func testInsertMultipleTagsIntoEmpty() { + addMessage(200, 200, "m200", [], [], MessageTags(rawValue: 2)) + addMessage(100, 100, "m100", [], [], MessageTags(rawValue: 1 | 2)) + expectEntries([.Message(100, 100, "m100", [], [])], tagMask: MessageTags(rawValue: 1)) + expectEntries([.Message(100, 100, "m100", [], []), .Message(200, 200, "m200", [], [])], tagMask: MessageTags(rawValue: 2)) + } + + func testRemoveSingleTagged() { + addMessage(100, 100, "m100", [], [], MessageTags(rawValue: 1)) + removeMessages([100]) + + expectEntries([], tagMask: MessageTags(rawValue: 1)) + } + + func testRemoveMultipleTagged() { + addMessage(200, 200, "m200", [], [], MessageTags(rawValue: 2)) + addMessage(100, 100, "m100", [], [], MessageTags(rawValue: 1 | 2)) + removeMessages([100]) + expectEntries([], tagMask: MessageTags(rawValue: 1)) + expectEntries([.Message(200, 200, "m200", [], [])], tagMask: MessageTags(rawValue: 2)) + } + + func testTagsInsertHoleIntoEmpty() { + addHole(1) + expectEntries([.Hole(1, Int32.max, Int32.max)], tagMask: [.First]) + expectEntries([.Hole(1, Int32.max, Int32.max)], tagMask: [.Second]) + } + + func testTagsBreakHoleWithMessage() { + addHole(1) + addMessage(100, 100, "m100", [], [], [.First]) + + expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Hole(101, Int32.max, Int32.max)], tagMask: [.First]) + expectEntries([.Hole(1, 99, 100), .Hole(101, Int32.max, Int32.max)], tagMask: [.Second]) + } + + func testTagsFillHoleUpperToLowerAllTags() { + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.Second]) + addHole(150) + + fillHole(199, .UpperToLower, [(180, 180, "m180", [])]) + + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 179, 180)], tagMask: [.First]) + expectEntries([.Hole(101, 179, 180), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 179, 180), .Message(180, 180, "m180", [], []), .Message(200, 200, "m200", [], [])]) + } + + func testTagsFillHoleLowerToUpperAllTags() { + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.Second]) + addHole(150) + + fillHole(199, .LowerToUpper, [(180, 180, "m180", [])]) + + expectEntries([.Message(100, 100, "m100", [], []), .Hole(181, 199, 200)], tagMask: [.First]) + expectEntries([.Hole(181, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) + expectEntries([.Message(100, 100, "m100", [], []), .Message(180, 180, "m180", [], []), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])]) + } + + func testTagsFillHoleCompleteAllTags() { + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.Second]) + addHole(150) + + fillHole(199, .Complete, [(180, 180, "m180", [])]) + + expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) + expectEntries([.Message(200, 200, "m200", [], [])], tagMask: [.Second]) + expectEntries([.Message(100, 100, "m100", [], []), .Message(180, 180, "m180", [], []), .Message(200, 200, "m200", [], [])]) + } + + func testTagsFillHoleUpperToLowerSingleTagWithMessages() { + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.Second]) + addHole(150) + + fillHole(199, .UpperToLower, [(180, 180, "m180", [])], [.First]) + + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 179, 180)], tagMask: [.First]) + expectEntries([.Hole(101, 179, 180), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 179, 180), .Message(180, 180, "m180", [], []), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])]) + } + + func testTagsFillHoleLowerToUpperSingleTagWithMessages() { + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.Second]) + addHole(150) + + fillHole(199, .LowerToUpper, [(180, 180, "m180", [])], [.First]) + + expectEntries([.Message(100, 100, "m100", [], []), .Hole(181, 199, 200)], tagMask: [.First]) + expectEntries([.Hole(101, 179, 180), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 179, 180), .Message(180, 180, "m180", [], []), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])]) + } + + func testTagsFillHoleCompleteSingleTagWithMessages() { + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.Second]) + addHole(150) + + fillHole(199, .Complete, [(180, 180, "m180", [])], [.First]) + + expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) + expectEntries([.Hole(101, 179, 180), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 179, 180), .Message(180, 180, "m180", [], []), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])]) + } + + func testTagsFillHoleUpperToLowerSingleTagWithEmpty() { + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.Second]) + addHole(150) + + fillHole(199, .UpperToLower, [], [.First]) + + expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) + expectEntries([.Hole(101, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 199, 200), .Message(200, 200, "m200", [], [])]) + } + + func testTagsFillHoleLowerToUpperSingleTagWithEmpty() { + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.Second]) + addHole(150) + + fillHole(199, .LowerToUpper, [], [.First]) + + expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) + expectEntries([.Hole(101, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 199, 200), .Message(200, 200, "m200", [], [])]) + } + + func testTagsFillHoleCompleteSingleTagWithEmpty() { + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.Second]) + addHole(150) + + fillHole(199, .Complete, [], [.First]) + + expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) + expectEntries([.Hole(101, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) + expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 199, 200), .Message(200, 200, "m200", [], [])]) + } +} From 6b448b809bae5f3324668f8743ac9fa089a216c1 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 28 Mar 2016 17:23:55 +0300 Subject: [PATCH 028/427] no message --- submodules/SQLite.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/SQLite.swift b/submodules/SQLite.swift index 9fe0d2df4b..3816cb553f 160000 --- a/submodules/SQLite.swift +++ b/submodules/SQLite.swift @@ -1 +1 @@ -Subproject commit 9fe0d2df4beac6ceea4b208bcc9e9b05db2e6b61 +Subproject commit 3816cb553ff2862804fb9081f7822153d84e2c7a From 93b8ccac9d6ec7c7506e7a40787a3e5e17e0ba1a Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 18 Apr 2016 01:03:09 +0300 Subject: [PATCH 029/427] no message --- Postbox.xcodeproj/project.pbxproj | 147 ++++++++--- Postbox/ChatListTable.swift | 4 +- Postbox/ChatListView.swift | 40 ++- Postbox/InvalidatedPeerReadStatesView.swift | 30 +++ Postbox/LmdbValueBox.swift | 2 +- Postbox/MediaBox.swift | 50 +++- Postbox/Message.swift | 1 - Postbox/MessageHistoryIndexTable.swift | 71 ++++- ...sageHistoryInvalidatedReadStateTable.swift | 66 +++++ Postbox/MessageHistoryReadStateTable.swift | 242 ++++++++++++++++++ Postbox/MessageHistoryTable.swift | 95 ++++++- Postbox/MessageHistoryView.swift | 25 +- Postbox/PeerChatStateTable.swift | 2 +- Postbox/PeerReadState.swift | 20 ++ Postbox/Postbox.swift | 77 ++++-- Postbox/SqliteValueBox.swift | 44 +++- Postbox/ValueBox.swift | 2 +- Postbox/ViewTracker.swift | 44 +++- PostboxTests/ChatListTableTests.swift | 6 +- PostboxTests/MessageHistoryTableTests.swift | 6 +- 20 files changed, 859 insertions(+), 115 deletions(-) create mode 100644 Postbox/InvalidatedPeerReadStatesView.swift create mode 100644 Postbox/MessageHistoryInvalidatedReadStateTable.swift create mode 100644 Postbox/MessageHistoryReadStateTable.swift create mode 100644 Postbox/PeerReadState.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 56baf99ef5..751e452158 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -8,13 +8,14 @@ /* Begin PBXBuildFile section */ D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; - D00E0FB31B84CEDA002E4EB5 /* Display.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D00E0FB21B84CEDA002E4EB5 /* Display.framework */; }; D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB41B85D192002E4EB5 /* lmdb.h */; }; - D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; }; - D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB61B85D192002E4EB5 /* midl.c */; }; + D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-unreachable-code -Wno-conditional-uninitialized -Wno-format-extra-args -Wno-macro-redefined"; }; }; + D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB61B85D192002E4EB5 /* midl.c */; settings = {COMPILER_FLAGS = "-Wno-conversion"; }; }; D00E0FBB1B85D192002E4EB5 /* midl.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB71B85D192002E4EB5 /* midl.h */; }; D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */; }; D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; + D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; + D01F7D9D1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift */; }; D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; D03BCCF81C73561C0097A291 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; @@ -26,14 +27,10 @@ D044E1641B2AD718001EE087 /* MurMurHash32.h in Headers */ = {isa = PBXBuildFile; fileRef = D044E1611B2AD667001EE087 /* MurMurHash32.h */; settings = {ATTRIBUTES = (Public, ); }; }; D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; - D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; }; + D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-ambiguous-macro -Wno-conditional-uninitialized -Wno-unused-const-variable -Wno-unused-function -Wno-unreachable-code"; }; }; D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516411B2D9CEF00AE42E0 /* sqlite3.h */; }; D07516461B2D9CEF00AE42E0 /* sqlite3ext.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */; }; - D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */ = {isa = PBXBuildFile; fileRef = D075165B1B2EC5B000AE42E0 /* module.private.modulemap */; }; - D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */ = {isa = PBXBuildFile; fileRef = D075165D1B2EC5B500AE42E0 /* module.modulemap */; }; D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516611B2EC7FE00AE42E0 /* Database.swift */; }; - D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516681B2EC7FE00AE42E0 /* Statement.swift */; }; - D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516691B2EC7FE00AE42E0 /* Value.swift */; }; D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; @@ -46,11 +43,12 @@ D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B76BE61B66639F0095CF45 /* DeferredString.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; + D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; + D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */; }; D0C9DA391C65782500855278 /* SimpleSet.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C9DA381C65782500855278 /* SimpleSet.swift */; }; D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; - D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; @@ -86,13 +84,14 @@ /* Begin PBXFileReference section */ D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryView.swift; sourceTree = ""; }; - D00E0FB21B84CEDA002E4EB5 /* Display.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Display.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/Display.framework"; sourceTree = ""; }; D00E0FB41B85D192002E4EB5 /* lmdb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lmdb.h; path = submodules/lmdb/libraries/liblmdb/lmdb.h; sourceTree = SOURCE_ROOT; }; D00E0FB51B85D192002E4EB5 /* mdb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = mdb.c; path = submodules/lmdb/libraries/liblmdb/mdb.c; sourceTree = SOURCE_ROOT; }; D00E0FB61B85D192002E4EB5 /* midl.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = midl.c; path = submodules/lmdb/libraries/liblmdb/midl.c; sourceTree = SOURCE_ROOT; }; D00E0FB71B85D192002E4EB5 /* midl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = midl.h; path = submodules/lmdb/libraries/liblmdb/midl.h; sourceTree = SOURCE_ROOT; }; D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LmdbValueBox.swift; sourceTree = ""; }; D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; + D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; + D01F7D9C1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = InvalidatedPeerReadStatesView.swift; sourceTree = ""; }; D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryUnsentTable.swift; sourceTree = ""; }; D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageHistoryView.swift; sourceTree = ""; }; D03BCCF71C73561C0097A291 /* Table.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Table.swift; sourceTree = ""; }; @@ -108,11 +107,7 @@ D07516411B2D9CEF00AE42E0 /* sqlite3.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3.h; sourceTree = ""; }; D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3ext.h; sourceTree = ""; }; D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Postbox.xcconfig; path = Postbox/Config/Postbox.xcconfig; sourceTree = ""; }; - D075165B1B2EC5B000AE42E0 /* module.private.modulemap */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = "sourcecode.module-map"; path = module.private.modulemap; sourceTree = ""; }; - D075165D1B2EC5B500AE42E0 /* module.modulemap */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = "sourcecode.module-map"; path = module.modulemap; sourceTree = ""; }; D07516611B2EC7FE00AE42E0 /* Database.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Database.swift; path = submodules/sqlite.swift/SQLite/Database.swift; sourceTree = SOURCE_ROOT; }; - D07516681B2EC7FE00AE42E0 /* Statement.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Statement.swift; path = submodules/sqlite.swift/SQLite/Statement.swift; sourceTree = SOURCE_ROOT; }; - D07516691B2EC7FE00AE42E0 /* Value.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Value.swift; path = submodules/sqlite.swift/SQLite/Value.swift; sourceTree = SOURCE_ROOT; }; D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; @@ -125,11 +120,12 @@ D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; + D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; + D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatStateTable.swift; sourceTree = ""; }; D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryViewTests.swift; sourceTree = ""; }; D0C9DA381C65782500855278 /* SimpleSet.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SimpleSet.swift; sourceTree = ""; }; D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResource.swift; sourceTree = ""; }; - D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -161,7 +157,6 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - D00E0FB31B84CEDA002E4EB5 /* Display.framework in Frameworks */, D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -211,7 +206,6 @@ D075163D1B2D9CEF00AE42E0 /* PostboxPrivate */ = { isa = PBXGroup; children = ( - D075165D1B2EC5B500AE42E0 /* module.modulemap */, D075163F1B2D9CEF00AE42E0 /* sqlcipher */, ); path = PostboxPrivate; @@ -233,10 +227,7 @@ D07516731B2EC8C700AE42E0 /* sqlite.swift */ = { isa = PBXGroup; children = ( - D0D224ED1B4D6ABD0085E26D /* Functions.swift */, D07516611B2EC7FE00AE42E0 /* Database.swift */, - D07516681B2EC7FE00AE42E0 /* Statement.swift */, - D07516691B2EC7FE00AE42E0 /* Value.swift */, ); name = sqlite.swift; sourceTree = ""; @@ -259,6 +250,8 @@ D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */, D0F9E8701C5A0E9B00037222 /* PeerTable.swift */, D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */, + D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */, + D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */, ); name = Tables; sourceTree = ""; @@ -271,6 +264,7 @@ D0E3A7A11B28B7DC00A402D9 /* Media.swift */, D08C71391C501F0700779C0F /* MessageHistoryHole.swift */, D044CA2D1C618373002160FF /* ChatListHole.swift */, + D0C674CB1CBB14A700183765 /* PeerReadState.swift */, ); name = Objects; sourceTree = ""; @@ -303,6 +297,7 @@ D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */, D0F9E86A1C59719800037222 /* ChatListView.swift */, D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */, + D01F7D9C1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift */, ); name = Views; sourceTree = ""; @@ -310,7 +305,6 @@ D0E3A7401B28A7E300A402D9 = { isa = PBXGroup; children = ( - D00E0FB21B84CEDA002E4EB5 /* Display.framework */, D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */, D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */, D0E3A74C1B28A7E300A402D9 /* Postbox */, @@ -331,7 +325,6 @@ D0E3A74C1B28A7E300A402D9 /* Postbox */ = { isa = PBXGroup; children = ( - D075165B1B2EC5B000AE42E0 /* module.private.modulemap */, D075163D1B2D9CEF00AE42E0 /* PostboxPrivate */, D07515FC1B2C44A200AE42E0 /* thirdparty */, D0E1DE181C5EB09300C7826E /* Utils */, @@ -497,16 +490,16 @@ buildActionMask = 2147483647; files = ( D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, - D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */, D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, + D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */, - D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */, D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */, + D01F7D9D1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, @@ -521,16 +514,13 @@ D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, - D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, - D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */, D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */, D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, - D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D0FC1A3C1CA3FB310056AE9A /* MediaResourceData.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, @@ -540,7 +530,9 @@ D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */, D03BCCF81C73561C0097A291 /* Table.swift in Sources */, D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, + D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, + D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */, D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */, D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */, D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, @@ -571,6 +563,90 @@ /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ + D086A5711CC0116A00F08284 /* Hockeyapp */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 1; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 8.3; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + VALIDATE_PRODUCT = YES; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = Hockeyapp; + }; + D086A5721CC0116A00F08284 /* Hockeyapp */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + DEFINES_MODULE = YES; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + ENABLE_BITCODE = YES; + FRAMEWORK_SEARCH_PATHS = "$(inherited)"; + INFOPLIST_FILE = Postbox/Info.plist; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + OTHER_CFLAGS = ( + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_HAS_CODEC=1", + "-DSQLCIPHER_CRYPTO_CC=1", + "-DSQLITE_ENABLE_FTS3", + ); + PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + }; + name = Hockeyapp; + }; + D086A5731CC0116A00F08284 /* Hockeyapp */ = { + isa = XCBuildConfiguration; + buildSettings = { + FRAMEWORK_SEARCH_PATHS = ( + "$(SDKROOT)/Developer/Library/Frameworks", + "$(inherited)", + "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", + ); + INFOPLIST_FILE = PostboxTests/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Hockeyapp; + }; D0E3A75E1B28A7E300A402D9 /* Debug */ = { isa = XCBuildConfiguration; baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; @@ -672,11 +748,8 @@ DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; DYLIB_INSTALL_NAME_BASE = "@rpath"; - ENABLE_BITCODE = NO; - FRAMEWORK_SEARCH_PATHS = ( - "$(inherited)", - "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", - ); + ENABLE_BITCODE = YES; + FRAMEWORK_SEARCH_PATHS = "$(inherited)"; INFOPLIST_FILE = Postbox/Info.plist; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; IPHONEOS_DEPLOYMENT_TARGET = 8.0; @@ -702,11 +775,8 @@ DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; DYLIB_INSTALL_NAME_BASE = "@rpath"; - ENABLE_BITCODE = NO; - FRAMEWORK_SEARCH_PATHS = ( - "$(inherited)", - "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", - ); + ENABLE_BITCODE = YES; + FRAMEWORK_SEARCH_PATHS = "$(inherited)"; INFOPLIST_FILE = Postbox/Info.plist; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; IPHONEOS_DEPLOYMENT_TARGET = 8.0; @@ -767,6 +837,7 @@ buildConfigurations = ( D0E3A75E1B28A7E300A402D9 /* Debug */, D0E3A75F1B28A7E300A402D9 /* Release */, + D086A5711CC0116A00F08284 /* Hockeyapp */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; @@ -776,6 +847,7 @@ buildConfigurations = ( D0E3A7611B28A7E300A402D9 /* Debug */, D0E3A7621B28A7E300A402D9 /* Release */, + D086A5721CC0116A00F08284 /* Hockeyapp */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; @@ -785,6 +857,7 @@ buildConfigurations = ( D0E3A7641B28A7E300A402D9 /* Debug */, D0E3A7651B28A7E300A402D9 /* Release */, + D086A5731CC0116A00F08284 /* Hockeyapp */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index 8267731e8b..578a2d6079 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -1,7 +1,7 @@ import Foundation enum ChatListOperation { - case InsertMessage(IntermediateMessage) + case InsertMessage(IntermediateMessage, CombinedPeerReadState?) case InsertHole(ChatListHole) case InsertNothing(MessageIndex) case RemoveMessage([MessageIndex]) @@ -91,7 +91,7 @@ final class ChatListTable: Table { } self.indexTable.set(index) self.justInsertMessage(index) - operations.append(.InsertMessage(topMessage)) + operations.append(.InsertMessage(topMessage, messageHistoryTable.readStateTable.getCombinedState(peerId))) } else { if let currentIndex = currentIndex { operations.append(.RemoveMessage([currentIndex])) diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index ad76989417..45acfb3662 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -1,13 +1,13 @@ import Foundation public enum ChatListEntry: Comparable { - case MessageEntry(Message) + case MessageEntry(Message, Int) case HoleEntry(ChatListHole) case Nothing(MessageIndex) public var index: MessageIndex { switch self { - case let .MessageEntry(message): + case let .MessageEntry(message, _): return MessageIndex(message) case let .HoleEntry(hole): return hole.index @@ -18,7 +18,15 @@ public enum ChatListEntry: Comparable { } public func ==(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { - return lhs.index == rhs.index + if lhs.index == rhs.index { + if case let .MessageEntry(_, lhsCount) = lhs, case let .MessageEntry(_, rhsCount) = rhs { + if lhsCount != rhsCount { + return false + } + } + return true + } + return false } public func <(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { @@ -26,16 +34,16 @@ public func <(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { } enum MutableChatListEntry: Equatable { - case IntermediateMessageEntry(IntermediateMessage) - case MessageEntry(Message) + case IntermediateMessageEntry(IntermediateMessage, CombinedPeerReadState?) + case MessageEntry(Message, CombinedPeerReadState?) case HoleEntry(ChatListHole) case Nothing(MessageIndex) var index: MessageIndex { switch self { - case let .IntermediateMessageEntry(message): + case let .IntermediateMessageEntry(message, _): return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .MessageEntry(message): + case let .MessageEntry(message, _): return MessageIndex(message) case let .HoleEntry(hole): return hole.index @@ -109,8 +117,8 @@ final class MutableChatListView { var hasChanges = false for operation in operations { switch operation { - case let .InsertMessage(message): - if self.add(.IntermediateMessageEntry(message)) { + case let .InsertMessage(message, combinedReadState): + if self.add(.IntermediateMessageEntry(message, combinedReadState)) { hasChanges = true } case let .InsertNothing(index): @@ -368,8 +376,8 @@ final class MutableChatListView { func render(renderMessage: IntermediateMessage -> Message) { for i in 0 ..< self.entries.count { - if case let .IntermediateMessageEntry(message) = self.entries[i] { - self.entries[i] = .MessageEntry(renderMessage(message)) + if case let .IntermediateMessageEntry(message, combinedReadState) = self.entries[i] { + self.entries[i] = .MessageEntry(renderMessage(message), combinedReadState) } } } @@ -384,8 +392,14 @@ public final class ChatListView { var entries: [ChatListEntry] = [] for entry in mutableView.entries { switch entry { - case let .MessageEntry(message): - entries.append(.MessageEntry(message)) + case let .MessageEntry(message, combinedReadState): + var unreadCount: Int32 = 0 + if let combinedReadState = combinedReadState { + for (_, state) in combinedReadState.states { + unreadCount += state.count + } + } + entries.append(.MessageEntry(message, Int(unreadCount))) case let .Nothing(index): entries.append(.Nothing(index)) case let .HoleEntry(hole): diff --git a/Postbox/InvalidatedPeerReadStatesView.swift b/Postbox/InvalidatedPeerReadStatesView.swift new file mode 100644 index 0000000000..ee44dea37f --- /dev/null +++ b/Postbox/InvalidatedPeerReadStatesView.swift @@ -0,0 +1,30 @@ +import Foundation + +final class InvalidatedPeerReadStatesView { + var peerIds = Set() + + init(peerIds: [PeerId]) { + for peerId in peerIds { + self.peerIds.insert(peerId) + } + } + + func replay(operations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) -> Bool { + var updated = false + for operation in operations { + switch operation { + case let .Insert(peerId): + if !self.peerIds.contains(peerId) { + self.peerIds.insert(peerId) + updated = true + } + case let .Remove(peerId): + if let _ = self.peerIds.remove(peerId) { + updated = true + } + } + } + + return updated + } +} diff --git a/Postbox/LmdbValueBox.swift b/Postbox/LmdbValueBox.swift index 1922a4b9d9..ec578ff300 100644 --- a/Postbox/LmdbValueBox.swift +++ b/Postbox/LmdbValueBox.swift @@ -312,7 +312,7 @@ public final class LmdbValueBox: ValueBox { } } - public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: ValueBoxKey -> Bool, limit: Int) { + public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape keys: ValueBoxKey -> Bool, limit: Int) { self.range(table, start: start, end: end, values: { key, _ in return keys(key) }, limit: limit) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 2e12b545c2..52addd013e 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -1,6 +1,5 @@ import Foundation import SwiftSignalKit -import Display private final class ResourceStatusContext { var status: MediaResourceStatus? @@ -9,7 +8,8 @@ private final class ResourceStatusContext { private final class ResourceDataContext { var data: MediaResourceData - let dataSubscribers = Bag Void>() + let progresiveDataSubscribers = Bag Void>() + let completeDataSubscribers = Bag Void>() var fetchDisposable: Disposable? let fetchSubscribers = Bag() @@ -126,7 +126,7 @@ public final class MediaBox { } } - public func resourceData(resource: MediaResource) -> Signal { + public func resourceData(resource: MediaResource, complete: Bool = true) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -146,21 +146,39 @@ public final class MediaBox { self.dataContexts[resource.id] = dataContext } - let index = dataContext.dataSubscribers.add { data in - subscriber.putNext(data) - if data.size >= resource.size { - subscriber.putCompletion() + let index: Bag Void>.Index + if complete { + index = dataContext.completeDataSubscribers.add { data in + subscriber.putNext(data) + if data.size >= resource.size { + subscriber.putCompletion() + } } + if dataContext.data.size >= resource.size { + subscriber.putNext(dataContext.data) + } else { + subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0)) + } + } else { + index = dataContext.progresiveDataSubscribers.add { data in + subscriber.putNext(data) + if data.size >= resource.size { + subscriber.putCompletion() + } + } + subscriber.putNext(dataContext.data) } - subscriber.putNext(dataContext.data) - disposable.set(ActionDisposable { self.dataQueue.dispatch { if let dataContext = self.dataContexts[resource.id] { - dataContext.dataSubscribers.remove(index) + if complete { + dataContext.completeDataSubscribers.remove(index) + } else { + dataContext.progresiveDataSubscribers.remove(index) + } - if dataContext.dataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { + if dataContext.progresiveDataSubscribers.isEmpty && dataContext.completeDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { self.dataContexts.removeValueForKey(resource.id) } } @@ -230,10 +248,16 @@ public final class MediaBox { offset += data.length let updatedSize = offset - for subscriber in dataContext.dataSubscribers.copyItems() { + for subscriber in dataContext.progresiveDataSubscribers.copyItems() { subscriber(MediaResourceData(path: path, size: updatedSize)) } + if updatedSize >= resource.size { + for subscriber in dataContext.completeDataSubscribers.copyItems() { + subscriber(MediaResourceData(path: path, size: updatedSize)) + } + } + let status: MediaResourceStatus if updatedSize >= resource.size { status = .Local @@ -281,7 +305,7 @@ public final class MediaBox { } } - if dataContext.dataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { + if dataContext.completeDataSubscribers.isEmpty && dataContext.progresiveDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { self.dataContexts.removeValueForKey(resource.id) } } diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 6862f00a30..c3745d7b00 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -27,7 +27,6 @@ public struct MessageId: Hashable, Comparable, CustomStringConvertible { } public init(_ buffer: ReadBuffer) { - var peerIdNamespaceValue: Int32 = 0 memcpy(&peerIdNamespaceValue, buffer.memory + buffer.offset, 4) var peerIdIdValue: Int32 = 0 diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index c3b467e79a..c7e4b030a8 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -32,14 +32,19 @@ enum MessageHistoryIndexOperation { case Update(MessageIndex, InternalStoreMessage) } +private let HistoryEntryTypeMask: Int8 = 1 +private let HistoryEntryTypeMessage: Int8 = 0 +private let HistoryEntryTypeHole: Int8 = 1 +private let HistoryEntryMessageFlagIncoming: Int8 = 1 << 1 + private func readHistoryIndexEntry(peerId: PeerId, namespace: MessageId.Namespace, key: ValueBoxKey, value: ReadBuffer) -> HistoryIndexEntry { - var type: Int8 = 0 - value.read(&type, offset: 0, length: 1) + var flags: Int8 = 0 + value.read(&flags, offset: 0, length: 1) var timestamp: Int32 = 0 value.read(×tamp, offset: 0, length: 4) let index = MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: key.getInt32(8 + 4)), timestamp: timestamp) - if type == 0 { + if (flags & HistoryEntryTypeMask) == 0 { return .Message(index) } else { var stableId: UInt32 = 0 @@ -462,11 +467,11 @@ final class MessageHistoryIndexTable: Table { private func justInsertHole(hole: MessageHistoryHole, inout operations: [MessageHistoryIndexOperation]) { let value = WriteBuffer() - var type: Int8 = 1 + var flags: Int8 = HistoryEntryTypeHole var timestamp: Int32 = hole.maxIndex.timestamp var min: Int32 = hole.min var tags: UInt32 = hole.tags - value.write(&type, offset: 0, length: 1) + value.write(&flags, offset: 0, length: 1) value.write(×tamp, offset: 0, length: 4) var stableId: UInt32 = hole.stableId value.write(&stableId, offset: 0, length: 4) @@ -481,9 +486,12 @@ final class MessageHistoryIndexTable: Table { let index = MessageIndex(id: message.id, timestamp: message.timestamp) let value = WriteBuffer() - var type: Int8 = 0 + var flags: Int8 = HistoryEntryTypeMessage + if message.flags.contains(.Incoming) { + flags |= HistoryEntryMessageFlagIncoming + } var timestamp: Int32 = index.timestamp - value.write(&type, offset: 0, length: 1) + value.write(&flags, offset: 0, length: 1) value.write(×tamp, offset: 0, length: 4) self.valueBox.set(self.tableId, key: self.key(index.id), value: value) @@ -535,6 +543,55 @@ final class MessageHistoryIndexTable: Table { return self.valueBox.exists(self.tableId, key: self.key(id)) } + func incomingMessageCountInRange(peerId: PeerId, namespace: MessageId.Namespace, minId: MessageId.Id, maxId: MessageId.Id) -> (Int, Bool) { + var count = 0 + var holes = false + + self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: minId)).predecessor, end: self.key(MessageId(peerId: peerId, namespace: namespace, id: maxId)).successor, values: { _, value in + var flags: Int8 = 0 + value.read(&flags, offset: 0, length: 1) + if (flags & HistoryEntryTypeMask) == HistoryEntryTypeMessage { + if (flags & HistoryEntryMessageFlagIncoming) != 0 { + count += 1 + } + } else { + holes = true + } + return true + }, limit: 0) + + return (count, holes) + } + + func incomingMessageCountInIds(peerId: PeerId, namespace: MessageId.Namespace, ids: [MessageId.Id]) -> (Int, Bool) { + var count = 0 + var holes = false + + for id in ids { + self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: id)).predecessor, end: self.upperBound(peerId, namespace: namespace), values: { key, value in + let entryId = key.getInt32(8 + 4) + var flags: Int8 = 0 + value.read(&flags, offset: 0, length: 1) + + if entryId == id { + if (flags & HistoryEntryTypeMask) == HistoryEntryTypeMessage { + if (flags & HistoryEntryMessageFlagIncoming) != 0 { + count += 1 + } + } else { + holes = true + } + } else if (flags & HistoryEntryTypeMask) == HistoryEntryTypeHole { + holes = true + } + + return true + }, limit: 1) + } + + return (count, holes) + } + func debugList(peerId: PeerId, namespace: MessageId.Namespace) -> [HistoryIndexEntry] { var list: [HistoryIndexEntry] = [] self.valueBox.range(self.tableId, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in diff --git a/Postbox/MessageHistoryInvalidatedReadStateTable.swift b/Postbox/MessageHistoryInvalidatedReadStateTable.swift new file mode 100644 index 0000000000..cd5b1be7a1 --- /dev/null +++ b/Postbox/MessageHistoryInvalidatedReadStateTable.swift @@ -0,0 +1,66 @@ +import Foundation + +enum IntermediateMessageHistoryInvalidatedReadStateOperation { + case Insert(PeerId) + case Remove(PeerId) +} + +final class MessageHistoryInvalidatedReadStateTable: Table { + private let sharedKey = ValueBoxKey(length: 8) + + private func key(peerId: PeerId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: peerId.toInt64()) + return self.sharedKey + } + + private var updatedPeerIds: [PeerId: Bool] = [:] + + private func lowerBound() -> ValueBoxKey { + let key = ValueBoxKey(length: 1) + key.setInt8(0, value: 0) + return key + } + + private func upperBound() -> ValueBoxKey { + let key = ValueBoxKey(length: 8) + memset(key.memory, 0xff, key.length) + return key + } + + func add(peerId: PeerId, inout operations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + if !self.valueBox.exists(self.tableId, key: self.key(peerId)) { + self.valueBox.set(self.tableId, key: self.key(peerId), value: MemoryBuffer()) + operations.append(.Insert(peerId)) + } + } + + func remove(peerId: PeerId, inout operations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + if self.valueBox.exists(self.tableId, key: self.key(peerId)) { + self.valueBox.remove(self.tableId, key: self.key(peerId)) + operations.append(.Remove(peerId)) + } + } + + func get() -> [PeerId] { + var peerIds: [PeerId] = [] + self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), keys: { key in + peerIds.append(PeerId(key.getInt64(0))) + return true + }, limit: 0) + return peerIds + } + + override func beforeCommit() { + let key = ValueBoxKey(length: 8) + let buffer = MemoryBuffer() + for (peerId, invalidated) in self.updatedPeerIds { + key.setInt64(0, value: peerId.toInt64()) + if invalidated { + self.valueBox.set(self.tableId, key: key, value: buffer) + } else { + self.valueBox.remove(self.tableId, key: key) + } + } + self.updatedPeerIds.removeAll() + } +} diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift new file mode 100644 index 0000000000..3793491890 --- /dev/null +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -0,0 +1,242 @@ +import Foundation + +private final class InternalPeerReadStates { + var namespaces: [MessageId.Namespace: PeerReadState] + + init(namespaces: [MessageId.Namespace: PeerReadState]) { + self.namespaces = namespaces + } +} + +final class MessageHistoryReadStateTable: Table { + private var cachedPeerReadStates: [PeerId: InternalPeerReadStates?] = [:] + private var updatedPeerIds = Set() + + private let sharedKey = ValueBoxKey(length: 8) + + private func key(id: PeerId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: id.toInt64()) + return self.sharedKey + } + + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) + } + + private func get(id: PeerId) -> InternalPeerReadStates? { + if let states = self.cachedPeerReadStates[id] { + return states + } else { + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + var count: Int32 = 0 + value.read(&count, offset: 0, length: 4) + var stateByNamespace: [MessageId.Namespace: PeerReadState] = [:] + for _ in 0 ..< count { + var namespaceId: Int32 = 0 + var maxReadId: Int32 = 0 + var maxKnownId: Int32 = 0 + var count: Int32 = 0 + value.read(&namespaceId, offset: 0, length: 4) + value.read(&maxReadId, offset: 0, length: 4) + value.read(&maxKnownId, offset: 0, length: 4) + value.read(&count, offset: 0, length: 4) + let state = PeerReadState(maxReadId: maxReadId, maxKnownId: maxKnownId, count: count) + stateByNamespace[namespaceId] = state + } + let states = InternalPeerReadStates(namespaces: stateByNamespace) + self.cachedPeerReadStates[id] = states + return states + } else { + self.cachedPeerReadStates[id] = nil + return nil + } + } + } + + func getCombinedState(peerId: PeerId) -> CombinedPeerReadState? { + if let states = self.get(peerId) { + return CombinedPeerReadState(states: states.namespaces.map({$0})) + } + return nil + } + + func resetStates(peerId: PeerId, namespaces: [MessageId.Namespace: PeerReadState]) -> CombinedPeerReadState? { + self.updatedPeerIds.insert(peerId) + + if let states = self.get(peerId) { + var updated = false + for (namespace, state) in namespaces { + if states.namespaces[namespace] == nil || states.namespaces[namespace]! != state { + updated = true + } + states.namespaces[namespace] = state + } + if updated { + self.updatedPeerIds.insert(peerId) + return CombinedPeerReadState(states: states.namespaces.map({$0})) + } else { + return nil + } + } else { + self.updatedPeerIds.insert(peerId) + let states = InternalPeerReadStates(namespaces: namespaces) + self.cachedPeerReadStates[peerId] = states + return CombinedPeerReadState(states: states.namespaces.map({$0})) + } + } + + func addIncomingMessages(peerId: PeerId, ids: [MessageId]) -> (CombinedPeerReadState?, Bool) { + var idsByNamespace: [MessageId.Namespace: [MessageId.Id]] = [:] + for id in ids { + if idsByNamespace[id.namespace] != nil { + idsByNamespace[id.namespace]!.append(id.id) + } else { + idsByNamespace[id.namespace] = [id.id] + } + } + + if let states = self.get(peerId) { + var updated = false + var invalidated = false + for (namespace, ids) in idsByNamespace { + if let currentState = states.namespaces[namespace] { + var addedUnreadCount: Int32 = 0 + var maxIncomingId: Int32 = 0 + for id in ids { + if id > currentState.maxKnownId { + addedUnreadCount += 1 + maxIncomingId = max(id, maxIncomingId) + } + } + + if addedUnreadCount != 0 { + states.namespaces[namespace] = PeerReadState(maxReadId: currentState.maxReadId, maxKnownId: currentState.maxKnownId, count: currentState.count + addedUnreadCount) + updated = true + } + } + } + + if updated { + self.updatedPeerIds.insert(peerId) + } + + return (updated ? CombinedPeerReadState(states: states.namespaces.map({$0})) : nil, invalidated) + } else { + return (nil, true) + } + + return (nil, false) + } + + func deleteMessages(peerId: PeerId, ids: [MessageId], incomingStatsInIds: (PeerId, MessageId.Namespace, [MessageId.Id]) -> (Int, Bool)) -> (CombinedPeerReadState?, Bool) { + var idsByNamespace: [MessageId.Namespace: [MessageId.Id]] = [:] + for id in ids { + if idsByNamespace[id.namespace] != nil { + idsByNamespace[id.namespace]!.append(id.id) + } else { + idsByNamespace[id.namespace] = [id.id] + } + } + + if let states = self.get(peerId) { + var updated = false + var invalidate = false + for (namespace, ids) in idsByNamespace { + if let currentState = states.namespaces[namespace] { + var unreadIds: [MessageId.Id] = [] + for id in ids { + if id > currentState.maxReadId { + unreadIds.append(id) + } + } + + let (knownCount, holes) = incomingStatsInIds(peerId, namespace, unreadIds) + if holes { + invalidate = true + } + + states.namespaces[namespace] = PeerReadState(maxReadId: currentState.maxReadId, maxKnownId: currentState.maxKnownId, count: currentState.count - knownCount) + updated = true + } else { + invalidate = true + } + } + + if updated { + self.updatedPeerIds.insert(peerId) + } + + return (updated ? CombinedPeerReadState(states: states.namespaces.map({$0})) : nil, invalidate) + } else { + return (nil, true) + } + + return (nil, false) + } + + func applyMaxReadId(peerId: PeerId, namespace: MessageId.Namespace, maxReadId: MessageId.Id, maxKnownId: MessageId.Id, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool)) -> (CombinedPeerReadState?, Bool) { + if let states = self.get(peerId), state = states.namespaces[namespace] { + if state.maxReadId < maxReadId { + let (deltaCount, holes) = incomingStatsInRange(state.maxReadId + 1, maxReadId) + + states.namespaces[namespace] = PeerReadState(maxReadId: maxReadId, maxKnownId: max(state.maxKnownId, maxReadId), count: state.count - Int32(deltaCount)) + self.updatedPeerIds.insert(peerId) + return (CombinedPeerReadState(states: states.namespaces.map({$0})), holes) + } + } else { + return (nil, true) + } + + return (nil, false) + } + + func clearUnreadLocally(peerId: PeerId, topId: (PeerId, MessageId.Namespace) -> MessageId.Id?) -> CombinedPeerReadState? { + if let states = self.get(peerId) { + var updatedNamespaces: [MessageId.Namespace: PeerReadState] = [:] + var updated = false + for (namespace, state) in states.namespaces { + if let topMessageId = topId(peerId, namespace) { + let updatedState = PeerReadState(maxReadId: topMessageId, maxKnownId: topMessageId, count: 0) + if updatedState != state { + updated = true + } + updatedNamespaces[namespace] = updatedState + } else { + let updatedState = PeerReadState(maxReadId: state.maxReadId, maxKnownId: state.maxKnownId, count: 0) + updated = true + } + } + if updated { + self.updatedPeerIds.insert(peerId) + return CombinedPeerReadState(states: states.namespaces.map({$0})) + } + } + + return nil + } + + override func beforeCommit() { + let sharedBuffer = WriteBuffer() + for id in self.updatedPeerIds { + if let wrappedStates = self.cachedPeerReadStates[id], states = wrappedStates { + sharedBuffer.reset() + var count: Int32 = Int32(states.namespaces.count) + sharedBuffer.write(&count, offset: 0, length: 4) + for (namespace, state) in states.namespaces { + var namespaceId: Int32 = namespace + var maxReadId: Int32 = state.maxReadId + var maxKnownId: Int32 = state.maxKnownId + var count: Int32 = state.count + sharedBuffer.write(&namespaceId, offset: 0, length: 4) + sharedBuffer.write(&maxReadId, offset: 0, length: 4) + sharedBuffer.write(&maxKnownId, offset: 0, length: 4) + sharedBuffer.write(&count, offset: 0, length: 4) + } + self.valueBox.set(self.tableId, key: self.key(id), value: sharedBuffer) + } else { + self.valueBox.remove(self.tableId, key: self.key(id)) + } + } + self.updatedPeerIds.removeAll() + } +} diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 08c74c737d..4cc285612c 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -4,6 +4,7 @@ enum MessageHistoryOperation { case InsertMessage(IntermediateMessage) case InsertHole(MessageHistoryHole) case Remove([MessageIndex]) + case UpdateReadState(CombinedPeerReadState) } enum IntermediateMessageHistoryEntry { @@ -40,13 +41,17 @@ final class MessageHistoryTable: Table { let historyMetadataTable: MessageHistoryMetadataTable let unsentTable: MessageHistoryUnsentTable let tagsTable: MessageHistoryTagsTable + let readStateTable: MessageHistoryReadStateTable + let invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable - init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable) { + init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable) { self.messageHistoryIndexTable = messageHistoryIndexTable self.messageMediaTable = messageMediaTable self.historyMetadataTable = historyMetadataTable self.unsentTable = unsentTable self.tagsTable = tagsTable + self.readStateTable = readStateTable + self.invalidatedReadStateTable = invalidatedReadStateTable super.init(valueBox: valueBox, tableId: tableId) } @@ -101,13 +106,15 @@ final class MessageHistoryTable: Table { return dict } - private func processIndexOperations(peerId: PeerId, operations: [MessageHistoryIndexOperation], inout processedOperationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { + private func processIndexOperations(peerId: PeerId, operations: [MessageHistoryIndexOperation], inout processedOperationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0)) let sharedBuffer = WriteBuffer() let sharedEncoder = Encoder() var outputOperations: [MessageHistoryOperation] = [] var accumulatedRemoveIndices: [MessageIndex] = [] + var addedIncomingMessageIds: [MessageId] = [] + var removedMessageIds: [MessageId] = [] for operation in operations { switch operation { case let .InsertHole(hole): @@ -154,6 +161,9 @@ final class MessageHistoryTable: Table { } } } + if message.flags.contains(.Incoming) { + addedIncomingMessageIds.append(message.id) + } case let .Remove(index): self.justRemove(index, unsentMessageOperations: &unsentMessageOperations) accumulatedRemoveIndices.append(index) @@ -172,6 +182,16 @@ final class MessageHistoryTable: Table { outputOperations.append(.Remove(accumulatedRemoveIndices)) } + if !addedIncomingMessageIds.isEmpty { + let (combinedState, invalidate) = self.readStateTable.addIncomingMessages(peerId, ids: addedIncomingMessageIds) + if let combinedState = combinedState { + outputOperations.append(.UpdateReadState(combinedState)) + } + if invalidate { + self.invalidatedReadStateTable.add(peerId, operations: &invalidatedReadStateOperations) + } + } + if processedOperationsByPeerId[peerId] == nil { processedOperationsByPeerId[peerId] = outputOperations } else { @@ -193,45 +213,96 @@ final class MessageHistoryTable: Table { return internalStoreMessages } - func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { + func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { let messagesByPeerId = self.messagesGroupedByPeerId(messages) for (peerId, peerMessages) in messagesByPeerId { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.addMessages(self.internalStoreMessages(peerMessages), location: location, operations: &operations) - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) } } - func addHoles(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { + func addHoles(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] for id in messageIds { self.messageHistoryIndexTable.addHole(id, operations: &operations) } - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) } } - func removeMessages(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { + func removeMessages(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] + + let (combinedState, invalidate) = self.readStateTable.deleteMessages(peerId, ids: messageIds, incomingStatsInIds: { peerId, namespace, ids in + return self.messageHistoryIndexTable.incomingMessageCountInIds(peerId, namespace: namespace, ids: ids) + }) + for id in messageIds { self.messageHistoryIndexTable.removeMessage(id, operations: &operations) } - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) + + if let combinedState = combinedState { + var outputOperations: [MessageHistoryOperation] = [] + outputOperations.append(.UpdateReadState(combinedState)) + + if operationsByPeerId[peerId] == nil { + operationsByPeerId[peerId] = outputOperations + } else { + operationsByPeerId[peerId]!.appendContentsOf(outputOperations) + } + } + + if invalidate { + self.invalidatedReadStateTable.add(peerId, operations: &invalidatedReadStateOperations) + } } } - func fillHole(id: MessageId, fillType: HoleFillType, tagMask: MessageTags?, messages: [StoreMessage], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { + func fillHole(id: MessageId, fillType: HoleFillType, tagMask: MessageTags?, messages: [StoreMessage], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.fillHole(id, fillType: fillType, tagMask: tagMask, messages: self.internalStoreMessages(messages), operations: &operations) - self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) } - func updateMessage(id: MessageId, message: StoreMessage, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { + func updateMessage(id: MessageId, message: StoreMessage, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.updateMessage(id, message: self.internalStoreMessages([message]).first!, operations: &operations) - self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) + } + + func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + for (peerId, namespaces) in states { + if let combinedState = self.readStateTable.resetStates(peerId, namespaces: namespaces) { + if operationsByPeerId[peerId] == nil { + operationsByPeerId[peerId] = [.UpdateReadState(combinedState)] + } else { + operationsByPeerId[peerId]!.append(.UpdateReadState(combinedState)) + } + } + self.invalidatedReadStateTable.remove(peerId, operations: &invalidatedReadStateOperations) + } + } + + func applyIncomingReadMaxId(messageId: MessageId, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + let (combinedState, invalidated) = self.readStateTable.applyMaxReadId(messageId.peerId, namespace: messageId.namespace, maxReadId: messageId.id, maxKnownId: messageId.id, incomingStatsInRange: { fromId, toId in + return self.messageHistoryIndexTable.incomingMessageCountInRange(messageId.peerId, namespace: messageId.namespace, minId: fromId, maxId: toId) + }) + + if let combinedState = combinedState { + if operationsByPeerId[messageId.peerId] == nil { + operationsByPeerId[messageId.peerId] = [.UpdateReadState(combinedState)] + } else { + operationsByPeerId[messageId.peerId]!.append(.UpdateReadState(combinedState)) + } + } + + if invalidated { + self.invalidatedReadStateTable.add(messageId.peerId, operations: &invalidatedReadStateOperations) + } } func topMessage(peerId: PeerId) -> IntermediateMessage? { diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index ddc3a1d804..12e2b49ff9 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -69,12 +69,14 @@ final class MutableMessageHistoryViewReplayContext { final class MutableMessageHistoryView { let tagMask: MessageTags? + private var combinedReadState: CombinedPeerReadState? private let count: Int private var earlier: MutableMessageHistoryEntry? private var later: MutableMessageHistoryEntry? private var entries: [MutableMessageHistoryEntry] - init(earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int) { + init(combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int) { + self.combinedReadState = combinedReadState self.earlier = earlier self.entries = entries self.later = later @@ -105,8 +107,12 @@ final class MutableMessageHistoryView { if self.remove(Set(indices), context: context) { hasChanges = true } + case let .UpdateReadState(combinedReadState): + hasChanges = true + self.combinedReadState = combinedReadState } } + return hasChanges } @@ -322,6 +328,7 @@ public final class MessageHistoryView { public let earlierId: MessageIndex? public let laterId: MessageIndex? public let entries: [MessageHistoryEntry] + public let maxReadIndex: MessageIndex? init(_ mutableView: MutableMessageHistoryView) { var entries: [MessageHistoryEntry] = [] @@ -339,5 +346,21 @@ public final class MessageHistoryView { self.earlierId = mutableView.earlier?.index self.laterId = mutableView.later?.index + + if let combinedReadState = mutableView.combinedReadState { + var maxIndex: MessageIndex? + for (namespace, state) in combinedReadState.states { + for entry in entries { + if entry.index.id.namespace == namespace { + if maxIndex == nil || maxIndex! < entry.index { + maxIndex = entry.index + } + } + } + } + self.maxReadIndex = maxIndex + } else { + self.maxReadIndex = nil + } } } diff --git a/Postbox/PeerChatStateTable.swift b/Postbox/PeerChatStateTable.swift index 721d889461..8354705477 100644 --- a/Postbox/PeerChatStateTable.swift +++ b/Postbox/PeerChatStateTable.swift @@ -43,4 +43,4 @@ final class PeerChatStateTable: Table { } self.updatedPeerIds.removeAll() } -} \ No newline at end of file +} diff --git a/Postbox/PeerReadState.swift b/Postbox/PeerReadState.swift new file mode 100644 index 0000000000..a8a66588c4 --- /dev/null +++ b/Postbox/PeerReadState.swift @@ -0,0 +1,20 @@ + +public struct PeerReadState: Equatable { + public let maxReadId: MessageId.Id + public let maxKnownId: MessageId.Id + public let count: Int32 + + public init(maxReadId: MessageId.Id, maxKnownId: MessageId.Id, count: Int32) { + self.maxReadId = maxReadId + self.maxKnownId = maxKnownId + self.count = count + } +} + +public func ==(lhs: PeerReadState, rhs: PeerReadState) -> Bool { + return lhs.maxReadId == rhs.maxReadId && lhs.maxKnownId == rhs.maxKnownId && lhs.count == rhs.count +} + +struct CombinedPeerReadState { + let states: [(MessageId.Namespace, PeerReadState)] +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 4169d232eb..190a3a50db 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -40,6 +40,18 @@ public final class Modifier { } } + public func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]]) { + self.postbox?.resetIncomingReadStates(states) + } + + public func applyIncomingReadMaxId(messageId: MessageId) { + self.postbox?.applyIncomingReadMaxId(messageId) + } + + public func validateIncomingReadState(peerId: PeerId) { + self.postbox?.validateIncomingReadState(peerId) + } + public func getState() -> Coding? { return self.postbox?.getState() } @@ -90,6 +102,7 @@ public final class Postbox { private var currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] private var currentUnsentOperations: [IntermediateMessageHistoryUnsentOperation] = [] + private var currentInvalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation] = [] private var currentFilledHolesByPeerId = Set() private var currentUpdatedPeers: [PeerId: Peer] = [:] @@ -112,6 +125,11 @@ public final class Postbox { self.sendUnsentMessageImpl.set(single(sendUnsentMessage, NoError.self)) } + private let validatePeerReadStateImpl = Promise Signal>() + public func setValidatePeerReadState(validatePeerReadState: PeerId -> Signal) { + self.validatePeerReadStateImpl.set(.single(validatePeerReadState)) + } + public let mediaBox: MediaBox var tables: [Table] = [] @@ -130,11 +148,16 @@ public final class Postbox { var messageHistoryUnsentTable: MessageHistoryUnsentTable! var messageHistoryTagsTable: MessageHistoryTagsTable! var peerChatStateTable: PeerChatStateTable! + var readStateTable: MessageHistoryReadStateTable! + var invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable! public init(basePath: String, globalMessageIdsNamespace: MessageId.Namespace, seedConfiguration: SeedConfiguration) { self.basePath = basePath self.globalMessageIdsNamespace = globalMessageIdsNamespace self.seedConfiguration = seedConfiguration + + let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath + "/media") + self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.openDatabase() } @@ -175,12 +198,13 @@ public final class Postbox { //let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath + "/media") //self.debugSaveState("beforeGetDiff") //self.debugRestoreState("beforeGetDiff") + //self.debugRestoreState("clean") self.valueBox = SqliteValueBox(basePath: self.basePath + "/db") self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 18 + let currentUserVersion: Int32 = 1 if userVersion != currentUserVersion { self.valueBox.drop() @@ -196,7 +220,9 @@ public final class Postbox { self.messageHistoryIndexTable = MessageHistoryIndexTable(valueBox: self.valueBox, tableId: 4, globalMessageIdsTable: self.globalMessageIdsTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox, tableId: 5) self.mediaTable = MessageMediaTable(valueBox: self.valueBox, tableId: 6, mediaCleanupTable: self.mediaCleanupTable) - self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, tableId: 7, messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox, tableId: 14) + self.invalidatedReadStateTable = MessageHistoryInvalidatedReadStateTable(valueBox: self.valueBox, tableId: 15) + self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, tableId: 7, messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, invalidatedReadStateTable: self.invalidatedReadStateTable!) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, tableId: 8) self.chatListTable = ChatListTable(valueBox: self.valueBox, tableId: 9, indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, tableId: 13) @@ -212,8 +238,10 @@ public final class Postbox { self.tables.append(self.chatListIndexTable) self.tables.append(self.chatListTable) self.tables.append(self.peerChatStateTable) + self.tables.append(self.readStateTable) + self.tables.append(self.invalidatedReadStateTable) - self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, renderMessage: self.renderIntermediateMessage, fetchChatListHole: self.fetchChatListHoleWrapper, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper, sendUnsentMessage: self.sendUnsentMessageWrapper, unsentMessageIndices: self.messageHistoryUnsentTable!.get()) + self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, renderMessage: self.renderIntermediateMessage, fetchChatListHole: self.fetchChatListHoleWrapper, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper, sendUnsentMessage: self.sendUnsentMessageWrapper, unsentMessageIndices: self.messageHistoryUnsentTable!.get(), validateReadState: self.validatePeerReadStateWrapper, invalidatedReadStatePeerIds: self.invalidatedReadStateTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") } @@ -271,15 +299,15 @@ public final class Postbox { } private func addMessages(messages: [StoreMessage], location: AddMessagesLocation) { - self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations) + self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) } private func addHole(id: MessageId) { - self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations) + self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) } private func fillHole(hole: MessageHistoryHole, fillType: HoleFillType, tagMask: MessageTags?, messages: [StoreMessage]) { - self.messageHistoryTable.fillHole(hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations) + self.messageHistoryTable.fillHole(hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) self.currentFilledHolesByPeerId.insert(hole.id.peerId) } @@ -288,7 +316,19 @@ public final class Postbox { } private func deleteMessages(messageIds: [MessageId]) { - self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations) + self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) + } + + private func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]]) { + self.messageHistoryTable.resetIncomingReadStates(states, operationsByPeerId: &self.currentOperationsByPeerId, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) + } + + private func applyIncomingReadMaxId(messageId: MessageId) { + self.messageHistoryTable.applyIncomingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) + } + + private func validateIncomingReadState(peerId: PeerId) { + self.invalidatedReadStateTable.remove(peerId, operations: &self.currentInvalidatedReadStateOperations) } private func fetchEarlierHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { @@ -383,7 +423,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) + entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId))) case let .Hole(hole): entries.append(.HoleEntry(hole)) case let .Nothing(index): @@ -394,7 +434,7 @@ public final class Postbox { if let intermediateLower = intermediateLower { switch intermediateLower { case let .Message(message): - lower = .IntermediateMessageEntry(message) + lower = .IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId)) case let .Hole(hole): lower = .HoleEntry(hole) case let .Nothing(index): @@ -405,7 +445,7 @@ public final class Postbox { if let intermediateUpper = intermediateUpper { switch intermediateUpper { case let .Message(message): - upper = .IntermediateMessageEntry(message) + upper = .IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId)) case let .Hole(hole): upper = .HoleEntry(hole) case let .Nothing(index): @@ -422,7 +462,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) + entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId))) case let .Hole(hole): entries.append(.HoleEntry(hole)) case let .Nothing(index): @@ -438,7 +478,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) + entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId))) case let .Nothing(index): entries.append(.Nothing(index)) case let .Hole(index): @@ -475,6 +515,12 @@ public final class Postbox { }).start() } + private func validatePeerReadStateWrapper(peerId: PeerId) -> Disposable { + return (self.validatePeerReadStateImpl.get() |> mapToSignal { validate -> Signal in + return validate(peerId) + }).start() + } + private func beforeCommit() { var chatListOperations: [ChatListOperation] = [] self.chatListTable.replay(self.currentOperationsByPeerId, messageHistoryTable: self.messageHistoryTable, operations: &chatListOperations) @@ -482,13 +528,14 @@ public final class Postbox { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } - self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations) + self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, invalidatedReadStateOperations: self.currentInvalidatedReadStateOperations) self.currentOperationsByPeerId.removeAll() self.currentFilledHolesByPeerId.removeAll() self.currentUpdatedPeers.removeAll() self.currentReplaceChatListHoles.removeAll() self.currentUnsentOperations.removeAll() + self.currentInvalidatedReadStateOperations.removeAll() for table in self.tables { table.beforeCommit() @@ -521,7 +568,7 @@ public final class Postbox { private func updateMessage(index: MessageIndex, update: Message -> StoreMessage) { if let intermediateMessage = self.messageHistoryTable.getMessage(index) { let message = self.renderIntermediateMessage(intermediateMessage) - self.messageHistoryTable.updateMessage(index.id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations) + self.messageHistoryTable.updateMessage(index.id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) } } @@ -564,7 +611,7 @@ public final class Postbox { self.queue.dispatch { let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) - let mutableView = MutableMessageHistoryView(earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) + let mutableView = MutableMessageHistoryView(combinedReadState: self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) mutableView.render(self.renderIntermediateMessage) subscriber.putNext((MessageHistoryView(mutableView), .Generic)) diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 33631777b2..8185d75c6f 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -17,6 +17,10 @@ private struct SqlitePreparedStatement { sqlite3_bind_int(statement, Int32(index), number) } + func bind(index: Int, number: Int64) { + sqlite3_bind_int64(statement, Int32(index), number) + } + func reset() { sqlite3_reset(statement) sqlite3_clear_bindings(statement) @@ -30,6 +34,10 @@ private struct SqlitePreparedStatement { return result == SQLITE_ROW } + func int64At(index: Int) -> Int64 { + return sqlite3_column_int64(statement, Int32(index)) + } + func valueAt(index: Int) -> ReadBuffer { let valueLength = sqlite3_column_bytes(statement, Int32(index)) let valueData = sqlite3_column_blob(statement, Int32(index)) @@ -108,13 +116,14 @@ public final class SqliteValueBox: ValueBox { database.execute("PRAGMA wal_autocheckpoint=200") database.execute("PRAGMA journal_size_limit=1536") - let result = database.scalar("PRAGMA user_version") as! Int64 + let result = self.getUserVersion(database) if result != 1 { database.execute("PRAGMA user_version=1") database.execute("CREATE TABLE __meta_tables (name INTEGER)") } - for row in database.prepare("SELECT name FROM __meta_tables").run() { - self.tables.insert(Int32(row[0] as! Int64)) + + for table in self.listTables(database).map({Int32($0)}) { + self.tables.insert(table) } lock.unlock() @@ -142,15 +151,38 @@ public final class SqliteValueBox: ValueBox { } public func begin() { - self.database.transaction() + self.database.execute("BEGIN") } public func commit() { let startTime = CFAbsoluteTimeGetCurrent() - self.database.commit() + self.database.execute("COMMIT") self.commitTime += CFAbsoluteTimeGetCurrent() - startTime } + private func getUserVersion(database: Database) -> Int64 { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(database.handle, "PRAGMA user_version", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + preparedStatement.step() + let value = preparedStatement.int64At(0) + preparedStatement.destroy() + return value + } + + private func listTables(database: Database) -> [Int64] { + var statement: COpaquePointer = nil + sqlite3_prepare_v2(database.handle, "SELECT name FROM __meta_tables", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + var tables: [Int64] = [] + while preparedStatement.step() { + let value = preparedStatement.int64At(0) + tables.append(value) + } + preparedStatement.destroy() + return tables + } + private func getStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement @@ -503,7 +535,7 @@ public final class SqliteValueBox: ValueBox { } } - public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: ValueBoxKey -> Bool, limit: Int) { + public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape keys: ValueBoxKey -> Bool, limit: Int) { if self.tables.contains(table) { let statement: SqlitePreparedStatement diff --git a/Postbox/ValueBox.swift b/Postbox/ValueBox.swift index 83973eeb8f..2ade4659cc 100644 --- a/Postbox/ValueBox.swift +++ b/Postbox/ValueBox.swift @@ -8,7 +8,7 @@ public protocol ValueBox { func endStats() func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) - func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: ValueBoxKey -> Bool, limit: Int) + func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape keys: ValueBoxKey -> Bool, limit: Int) func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? func exists(table: Int32, key: ValueBoxKey) -> Bool func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 276b655c86..772c2efdee 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -16,16 +16,19 @@ final class ViewTracker { private let fetchChatListHole: ChatListHole -> Disposable private let fetchMessageHistoryHole: (MessageHistoryHole, MessageTags?) -> Disposable private let sendUnsentMessage: MessageIndex -> Disposable + private let validateReadState: PeerId -> Disposable private var chatListViews = Bag<(MutableChatListView, Pipe<(ChatListView, ViewUpdateType)>)>() private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] private var unsentMessageView: UnsentMessageHistoryView + private var invalidatedReadStatesView: InvalidatedPeerReadStatesView private var chatListHoleDisposables: [(ChatListHole, Disposable)] = [] private var holeDisposablesByPeerId: [PeerId: [(MessageHistoryHole, Disposable)]] = [:] private var unsentMessageDisposables: [MessageIndex: Disposable] = [:] + private var validateReadStatesDisposables: [PeerId: Disposable] = [:] - init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], renderMessage: IntermediateMessage -> Message, fetchChatListHole: ChatListHole -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, MessageTags?) -> Disposable, sendUnsentMessage: MessageIndex -> Disposable, unsentMessageIndices: [MessageIndex]) { + init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], renderMessage: IntermediateMessage -> Message, fetchChatListHole: ChatListHole -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, MessageTags?) -> Disposable, sendUnsentMessage: MessageIndex -> Disposable, unsentMessageIndices: [MessageIndex], validateReadState: PeerId -> Disposable, invalidatedReadStatePeerIds: [PeerId]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -35,9 +38,12 @@ final class ViewTracker { self.fetchChatListHole = fetchChatListHole self.fetchMessageHistoryHole = fetchMessageHistoryHole self.sendUnsentMessage = sendUnsentMessage + self.validateReadState = validateReadState self.unsentMessageView = UnsentMessageHistoryView(indices: unsentMessageIndices) + self.invalidatedReadStatesView = InvalidatedPeerReadStatesView(peerIds: invalidatedReadStatePeerIds) self.unsentViewUpdated() + self.invalidatedReadStateViewUpdated() } deinit { @@ -91,7 +97,7 @@ final class ViewTracker { self.updateTrackedChatListHoles() } - func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: Set, chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { + func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: Set, chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { @@ -147,6 +153,10 @@ final class ViewTracker { if self.unsentMessageView.replay(unsentMessageOperations) { self.unsentViewUpdated() } + + if self.invalidatedReadStatesView.replay(invalidatedReadStateOperations) { + self.invalidatedReadStateViewUpdated() + } } private func updateTrackedChatListHoles() { @@ -270,7 +280,7 @@ final class ViewTracker { private func unsentViewUpdated() { var removeIndices: [MessageIndex] = [] - for (index, _) in unsentMessageDisposables { + for (index, _) in self.unsentMessageDisposables { var found = false for currentIndex in self.unsentMessageView.indices { if currentIndex == index { @@ -302,4 +312,32 @@ final class ViewTracker { } } } + + private func invalidatedReadStateViewUpdated() { + var removePeerIds: [PeerId] = [] + let currentPeerIds = self.invalidatedReadStatesView.peerIds + for (peerId, _) in self.validateReadStatesDisposables { + if !currentPeerIds.contains(peerId) { + removePeerIds.append(peerId) + } + } + + for peerId in removePeerIds { + self.validateReadStatesDisposables.removeValueForKey(peerId)?.dispose() + } + + for peerId in currentPeerIds { + var found = false + for (currentPeerId, _) in validateReadStatesDisposables { + if peerId == currentPeerId { + found = true + break + } + } + + if !found { + self.validateReadStatesDisposables[peerId] = self.validateReadState(peerId) + } + } + } } diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index 1e0c17ff18..7f569f75a4 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -73,6 +73,8 @@ class ChatListTableTests: XCTestCase { var historyMetadataTable: MessageHistoryMetadataTable? var unsentTable: MessageHistoryUnsentTable? var tagsTable: MessageHistoryTagsTable? + var readStateTable: MessageHistoryReadStateTable? + var invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable? override class func setUp() { super.setUp() @@ -95,7 +97,9 @@ class ChatListTableTests: XCTestCase { self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 11) + self.invalidatedReadStateTable = MessageHistoryInvalidatedReadStateTable(valueBox: self.valueBox!, tableId: 12) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, invalidatedReadStateTable: invalidatedReadStateTable) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, tableId: 5) self.chatListTable = ChatListTable(valueBox: self.valueBox!, tableId: 6, indexTable: self.chatListIndexTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) } diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index bacbc988f7..ddef648dab 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -202,6 +202,8 @@ class MessageHistoryTableTests: XCTestCase { var historyMetadataTable: MessageHistoryMetadataTable? var unsentTable: MessageHistoryUnsentTable? var tagsTable: MessageHistoryTagsTable? + var readStateTable: MessageHistoryReadStateTable? + var invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable? override class func setUp() { super.setUp() @@ -228,7 +230,9 @@ class MessageHistoryTableTests: XCTestCase { self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 10) + self.invalidatedReadStateTable = MessageHistoryInvalidatedReadStateTable(valueBox: self.valueBox!, tableId: 11) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, invalidatedReadStateTable: invalidatedReadStateTable) self.peerTable = PeerTable(valueBox: self.valueBox!, tableId: 6) self.peerTable!.set(peer) } From 81a073645cc2d70a6f4c6d75b30e3caa73856ed1 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 18 Apr 2016 01:03:23 +0300 Subject: [PATCH 030/427] no message --- submodules/SQLite.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/SQLite.swift b/submodules/SQLite.swift index 3816cb553f..1b1e295b29 160000 --- a/submodules/SQLite.swift +++ b/submodules/SQLite.swift @@ -1 +1 @@ -Subproject commit 3816cb553ff2862804fb9081f7822153d84e2c7a +Subproject commit 1b1e295b293aceb3071c0f1a56ec26096aa87caf From 10fe532059edf5ff2fbeab6df21e7b55cf2d3191 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 5 Jun 2016 00:30:30 +0300 Subject: [PATCH 031/427] no message --- Postbox.xcodeproj/project.pbxproj | 14 +- Postbox/InvalidatedPeerReadStatesView.swift | 30 -- Postbox/Message.swift | 35 +- Postbox/MessageHistoryIndexTable.swift | 190 +++++++--- ...sageHistoryInvalidatedReadStateTable.swift | 86 +++-- Postbox/MessageHistoryReadStateTable.swift | 80 +++-- Postbox/MessageHistoryTable.swift | 104 ++++-- Postbox/MessageHistoryView.swift | 339 +++++++++++++----- Postbox/PeerReadState.swift | 15 +- Postbox/Postbox.swift | 302 ++++++++++++---- Postbox/SimpleDictionary.swift | 3 +- Postbox/SqliteValueBox.swift | 2 +- Postbox/SynchronizePeerReadStatesView.swift | 29 ++ Postbox/ViewTracker.swift | 122 ++++--- PostboxTests/ChatListTableTests.swift | 20 +- .../MessageHistoryIndexTableTests.swift | 49 ++- PostboxTests/MessageHistoryTableTests.swift | 65 ++-- PostboxTests/ReadStateTableTests.swift | 244 +++++++++++++ 18 files changed, 1303 insertions(+), 426 deletions(-) delete mode 100644 Postbox/InvalidatedPeerReadStatesView.swift create mode 100644 Postbox/SynchronizePeerReadStatesView.swift create mode 100644 PostboxTests/ReadStateTableTests.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 751e452158..f21bbcfcce 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -15,7 +15,7 @@ D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */; }; D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; - D01F7D9D1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift */; }; + D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; D03BCCF81C73561C0097A291 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; @@ -27,6 +27,7 @@ D044E1641B2AD718001EE087 /* MurMurHash32.h in Headers */ = {isa = PBXBuildFile; fileRef = D044E1611B2AD667001EE087 /* MurMurHash32.h */; settings = {ATTRIBUTES = (Public, ); }; }; D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; + D060B77B1CF4845A0050BE9B /* ReadStateTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */; }; D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-ambiguous-macro -Wno-conditional-uninitialized -Wno-unused-const-variable -Wno-unused-function -Wno-unreachable-code"; }; }; D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516411B2D9CEF00AE42E0 /* sqlite3.h */; }; D07516461B2D9CEF00AE42E0 /* sqlite3ext.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */; }; @@ -91,7 +92,7 @@ D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LmdbValueBox.swift; sourceTree = ""; }; D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; - D01F7D9C1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = InvalidatedPeerReadStatesView.swift; sourceTree = ""; }; + D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryUnsentTable.swift; sourceTree = ""; }; D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageHistoryView.swift; sourceTree = ""; }; D03BCCF71C73561C0097A291 /* Table.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Table.swift; sourceTree = ""; }; @@ -103,6 +104,7 @@ D044E1621B2AD677001EE087 /* MurMurHash32.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MurMurHash32.m; sourceTree = ""; }; D055BD321B7D3D2D00F06C0A /* MediaBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaBox.swift; sourceTree = ""; }; D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResourceStatus.swift; sourceTree = ""; }; + D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ReadStateTableTests.swift; sourceTree = ""; }; D07516401B2D9CEF00AE42E0 /* sqlite3.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = sqlite3.c; sourceTree = ""; }; D07516411B2D9CEF00AE42E0 /* sqlite3.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3.h; sourceTree = ""; }; D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3ext.h; sourceTree = ""; }; @@ -119,7 +121,7 @@ D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = ""; }; - D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; + D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatStateTable.swift; sourceTree = ""; }; @@ -297,7 +299,7 @@ D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */, D0F9E86A1C59719800037222 /* ChatListView.swift */, D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */, - D01F7D9C1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift */, + D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */, ); name = Views; sourceTree = ""; @@ -361,6 +363,7 @@ D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */, D0F9E8681C58FA9300037222 /* ChatListTableTests.swift */, D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */, + D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */, ); path = PostboxTests; sourceTree = ""; @@ -499,7 +502,7 @@ D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */, D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */, - D01F7D9D1CBF8586008765C9 /* InvalidatedPeerReadStatesView.swift in Sources */, + D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, @@ -548,6 +551,7 @@ D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */, D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */, D0F9E8691C58FA9300037222 /* ChatListTableTests.swift in Sources */, + D060B77B1CF4845A0050BE9B /* ReadStateTableTests.swift in Sources */, D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/Postbox/InvalidatedPeerReadStatesView.swift b/Postbox/InvalidatedPeerReadStatesView.swift deleted file mode 100644 index ee44dea37f..0000000000 --- a/Postbox/InvalidatedPeerReadStatesView.swift +++ /dev/null @@ -1,30 +0,0 @@ -import Foundation - -final class InvalidatedPeerReadStatesView { - var peerIds = Set() - - init(peerIds: [PeerId]) { - for peerId in peerIds { - self.peerIds.insert(peerId) - } - } - - func replay(operations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) -> Bool { - var updated = false - for operation in operations { - switch operation { - case let .Insert(peerId): - if !self.peerIds.contains(peerId) { - self.peerIds.insert(peerId) - updated = true - } - case let .Remove(peerId): - if let _ = self.peerIds.remove(peerId) { - updated = true - } - } - } - - return updated - } -} diff --git a/Postbox/Message.swift b/Postbox/Message.swift index c3745d7b00..92c9d2a72e 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -124,15 +124,19 @@ public struct MessageIndex: Equatable, Comparable, Hashable { return self.id.hashValue } - static func absoluteUpperBound() -> MessageIndex { + public static func absoluteUpperBound() -> MessageIndex { return MessageIndex(id: MessageId(peerId: PeerId(namespace: Int32.max, id: Int32.max), namespace: Int32.max, id: Int32.max), timestamp: Int32.max) } - static func absoluteLowerBound() -> MessageIndex { + public static func absoluteLowerBound() -> MessageIndex { return MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0) } - static func upperBound(peerId: PeerId) -> MessageIndex { + public static func lowerBound(peerId: PeerId) -> MessageIndex { + return MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: 0) + } + + public static func upperBound(peerId: PeerId) -> MessageIndex { return MessageIndex(id: MessageId(peerId: peerId, namespace: Int32.max, id: Int32.max), timestamp: Int32.max) } } @@ -215,13 +219,34 @@ public struct StoreMessageForwardInfo { } } -public struct MessageForwardInfo { +public struct MessageForwardInfo: Equatable { public let author: Peer public let source: Peer? public let sourceMessageId: MessageId? public let date: Int32 } +public func ==(lhs: MessageForwardInfo, rhs: MessageForwardInfo) -> Bool { + if !lhs.author.isEqual(rhs.author) { + return false + } + if let lhsSource = lhs.source, rhsSource = rhs.source { + if !lhsSource.isEqual(rhsSource) { + return false + } + } else if (lhs.source == nil) != (rhs.source == nil) { + return false + } + if lhs.sourceMessageId != rhs.sourceMessageId { + return false + } + if lhs.date != rhs.date { + return false + } + + return true +} + public protocol MessageAttribute: Coding { var associatedPeerIds: [PeerId] { get } var associatedMessageIds: [MessageId] { get } @@ -237,7 +262,7 @@ public extension MessageAttribute { } } -public class Message { +public final class Message { public let stableId: UInt32 public let id: MessageId public let timestamp: Int32 diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index c7e4b030a8..9b4f704072 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -14,10 +14,46 @@ enum HistoryIndexEntry { } } -public enum HoleFillType { +public enum HoleFillDirection: Equatable { case UpperToLower case LowerToUpper - case Complete + case AroundIndex(MessageIndex) +} + +public func ==(lhs: HoleFillDirection, rhs: HoleFillDirection) -> Bool { + switch lhs { + case .UpperToLower: + switch rhs { + case .UpperToLower: + return true + default: + return false + } + case .LowerToUpper: + switch rhs { + case .LowerToUpper: + return true + default: + return false + } + case let .AroundIndex(lhsIndex): + switch rhs { + case let .AroundIndex(rhsIndex) where lhsIndex == rhsIndex: + return true + default: + return false + } + } +} + +public struct HoleFill { + public let complete: Bool + public let direction: HoleFillDirection + + public init(complete: Bool, direction: HoleFillDirection) { + self.complete = complete + self.direction = direction + } } public enum AddMessagesLocation { @@ -319,7 +355,7 @@ final class MessageHistoryIndexTable: Table { } } - func fillHole(id: MessageId, fillType: HoleFillType, tagMask: MessageTags?, messages: [InternalStoreMessage], inout operations: [MessageHistoryIndexOperation]) { + func fillHole(id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [InternalStoreMessage], inout operations: [MessageHistoryIndexOperation]) { self.ensureInitialized(id.peerId, operations: &operations) var upperItem: HistoryIndexEntry? @@ -336,6 +372,10 @@ final class MessageHistoryIndexTable: Table { switch upperItem { case let .Hole(upperHole): if let tagMask = tagMask { + if case .AroundIndex = fillType.direction { + assertionFailure(".AroundIndex not supported") + } + var messagesInRange: [InternalStoreMessage] = [] var i = 0 while i < remainingMessages.count { @@ -362,11 +402,10 @@ final class MessageHistoryIndexTable: Table { if i == 0 { if upperHole.min < message.id.id { let holeTags: UInt32 - switch fillType { - case .LowerToUpper, .Complete: - holeTags = clearedTags - case .UpperToLower: - holeTags = upperHole.tags + if fillType.complete || fillType.direction == .LowerToUpper { + holeTags = clearedTags + } else { + holeTags = upperHole.tags } self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: message.id.id - 1), timestamp: message.timestamp), min: upperHole.min, tags: holeTags), operations: &operations) } @@ -380,11 +419,10 @@ final class MessageHistoryIndexTable: Table { if i == messagesInRange.count - 1 { if upperHole.maxIndex.id.id > message.id.id { let holeTags: UInt32 - switch fillType { - case .LowerToUpper: - holeTags = upperHole.tags - case .UpperToLower, .Complete: - holeTags = clearedTags + if fillType.complete || fillType.direction == .UpperToLower { + holeTags = clearedTags + } else { + holeTags = upperHole.tags } self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: upperHole.maxIndex, min: message.id.id + 1, tags: holeTags), operations: &operations) } @@ -401,58 +439,77 @@ final class MessageHistoryIndexTable: Table { while i < remainingMessages.count { let message = remainingMessages[i] if message.id.id >= upperHole.min && message.id.id <= upperHole.maxIndex.id.id { - if (fillType == .UpperToLower || fillType == .Complete) && (minMessageInRange == nil || minMessageInRange!.id > message.id) { + if (minMessageInRange == nil || minMessageInRange!.id > message.id) { minMessageInRange = message - if !removedHole { - removedHole = true - self.justRemove(upperHole.maxIndex, operations: &operations) + if (fillType.complete || fillType.direction == .UpperToLower) { + if !removedHole { + removedHole = true + self.justRemove(upperHole.maxIndex, operations: &operations) + } } } - if (fillType == .LowerToUpper || fillType == .Complete) && (maxMessageInRange == nil || maxMessageInRange!.id < message.id) { + + if (maxMessageInRange == nil || maxMessageInRange!.id < message.id) { maxMessageInRange = message - if !removedHole { - removedHole = true - self.justRemove(upperHole.maxIndex, operations: &operations) + if (fillType.complete || fillType.direction == .LowerToUpper) { + if !removedHole { + removedHole = true + self.justRemove(upperHole.maxIndex, operations: &operations) + } } } + self.justInsertMessage(message, operations: &operations) remainingMessages.removeAtIndex(i) } else { i += 1 } } - switch fillType { - case .Complete: - if !removedHole { - removedHole = true - self.justRemove(upperHole.maxIndex, operations: &operations) + if fillType.complete { + if !removedHole { + removedHole = true + self.justRemove(upperHole.maxIndex, operations: &operations) + } + } else if fillType.direction == .LowerToUpper { + if let maxMessageInRange = maxMessageInRange where maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { + let stableId: UInt32 + let tags: UInt32 = upperHole.tags + if removedHole { + stableId = upperHole.stableId + } else { + stableId = self.metadataTable.getNextStableMessageIndexId() } - case .LowerToUpper: - if let maxMessageInRange = maxMessageInRange where maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { - let stableId: UInt32 - let tags: UInt32 - if removedHole { - stableId = upperHole.stableId - tags = upperHole.tags - } else { - stableId = self.metadataTable.getNextStableMessageIndexId() - tags = MessageTags.All.rawValue - } - self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: upperHole.maxIndex, min: maxMessageInRange.id.id + 1, tags: tags), operations: &operations) - } - case .UpperToLower: - if let minMessageInRange = minMessageInRange where minMessageInRange.id.id - 1 >= upperHole.min { - let stableId: UInt32 - let tags: UInt32 - if removedHole { - stableId = upperHole.stableId - tags = upperHole.tags - } else { - stableId = self.metadataTable.getNextStableMessageIndexId() - tags = MessageTags.All.rawValue - } - self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minMessageInRange.id.id - 1), timestamp: minMessageInRange.timestamp), min: upperHole.min, tags: tags), operations: &operations) + self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: upperHole.maxIndex, min: maxMessageInRange.id.id + 1, tags: tags), operations: &operations) + } + } else if fillType.direction == .UpperToLower { + if let minMessageInRange = minMessageInRange where minMessageInRange.id.id - 1 >= upperHole.min { + let stableId: UInt32 + let tags: UInt32 = upperHole.tags + if removedHole { + stableId = upperHole.stableId + } else { + stableId = self.metadataTable.getNextStableMessageIndexId() } + self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minMessageInRange.id.id - 1), timestamp: minMessageInRange.timestamp), min: upperHole.min, tags: tags), operations: &operations) + } + } else if case .AroundIndex = fillType.direction { + if !removedHole { + self.justRemove(upperHole.maxIndex, operations: &operations) + removedHole = true + } + + if let minMessageInRange = minMessageInRange where minMessageInRange.id.id - 1 >= upperHole.min { + let stableId: UInt32 = upperHole.stableId + let tags: UInt32 = upperHole.tags + + self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minMessageInRange.id.id - 1), timestamp: minMessageInRange.timestamp), min: upperHole.min, tags: tags), operations: &operations) + } + + if let maxMessageInRange = maxMessageInRange where maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { + let stableId: UInt32 = self.metadataTable.getNextStableMessageIndexId() + let tags: UInt32 = upperHole.tags + self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: upperHole.maxIndex, min: maxMessageInRange.id.id + 1, tags: tags), operations: &operations) + } } } case .Message: @@ -510,17 +567,17 @@ final class MessageHistoryIndexTable: Table { } } - private func adjacentItems(id: MessageId) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) { + func adjacentItems(id: MessageId, bindUpper: Bool = true) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) { let key = self.key(id) var lowerItem: HistoryIndexEntry? - self.valueBox.range(self.tableId, start: key, end: self.lowerBound(id.peerId, namespace: id.namespace), values: { key, value in + self.valueBox.range(self.tableId, start: bindUpper ? key : key.successor, end: self.lowerBound(id.peerId, namespace: id.namespace), values: { key, value in lowerItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) return true }, limit: 1) var upperItem: HistoryIndexEntry? - self.valueBox.range(self.tableId, start: key.predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in + self.valueBox.range(self.tableId, start: bindUpper ? key.predecessor : key, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) return true }, limit: 1) @@ -539,10 +596,35 @@ final class MessageHistoryIndexTable: Table { return nil } + func top(peerId: PeerId, namespace: MessageId.Namespace) -> HistoryIndexEntry? { + var operations: [MessageHistoryIndexOperation] = [] + self.ensureInitialized(peerId, operations: &operations) + + var entry: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: self.upperBound(peerId, namespace: namespace), end: self.lowerBound(peerId, namespace: namespace), values: { key, value in + entry = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value) + return false + }, limit: 1) + + return entry + } + func exists(id: MessageId) -> Bool { return self.valueBox.exists(self.tableId, key: self.key(id)) } + func holeContainingId(id: MessageId) -> MessageHistoryHole? { + var result: MessageHistoryHole? + self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: id.peerId, namespace: id.namespace, id: id.id)).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in + if case let .Hole(hole) = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) { + result = hole + } + return true + }, limit: 1) + + return result + } + func incomingMessageCountInRange(peerId: PeerId, namespace: MessageId.Namespace, minId: MessageId.Id, maxId: MessageId.Id) -> (Int, Bool) { var count = 0 var holes = false diff --git a/Postbox/MessageHistoryInvalidatedReadStateTable.swift b/Postbox/MessageHistoryInvalidatedReadStateTable.swift index cd5b1be7a1..57f2628803 100644 --- a/Postbox/MessageHistoryInvalidatedReadStateTable.swift +++ b/Postbox/MessageHistoryInvalidatedReadStateTable.swift @@ -1,11 +1,30 @@ import Foundation -enum IntermediateMessageHistoryInvalidatedReadStateOperation { - case Insert(PeerId) - case Remove(PeerId) +public enum PeerReadStateSynchronizationOperation: Equatable { + case Push(thenSync: Bool) + case Validate } -final class MessageHistoryInvalidatedReadStateTable: Table { +public func ==(lhs: PeerReadStateSynchronizationOperation, rhs: PeerReadStateSynchronizationOperation) -> Bool { + switch lhs { + case let .Push(lhsThenSync): + switch rhs { + case let .Push(rhsThenSync) where lhsThenSync == rhsThenSync: + return true + default: + return false + } + case .Validate: + switch rhs { + case .Validate: + return true + default: + return false + } + } +} + +final class MessageHistorySynchronizeReadStateTable: Table { private let sharedKey = ValueBoxKey(length: 8) private func key(peerId: PeerId) -> ValueBoxKey { @@ -13,7 +32,7 @@ final class MessageHistoryInvalidatedReadStateTable: Table { return self.sharedKey } - private var updatedPeerIds: [PeerId: Bool] = [:] + private var updatedPeerIds: [PeerId: PeerReadStateSynchronizationOperation?] = [:] private func lowerBound() -> ValueBoxKey { let key = ValueBoxKey(length: 1) @@ -27,35 +46,52 @@ final class MessageHistoryInvalidatedReadStateTable: Table { return key } - func add(peerId: PeerId, inout operations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { - if !self.valueBox.exists(self.tableId, key: self.key(peerId)) { - self.valueBox.set(self.tableId, key: self.key(peerId), value: MemoryBuffer()) - operations.append(.Insert(peerId)) - } + func set(peerId: PeerId, operation: PeerReadStateSynchronizationOperation?, inout operations: [PeerId: PeerReadStateSynchronizationOperation?]) { + self.updatedPeerIds[peerId] = operation + operations[peerId] = operation } - func remove(peerId: PeerId, inout operations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { - if self.valueBox.exists(self.tableId, key: self.key(peerId)) { - self.valueBox.remove(self.tableId, key: self.key(peerId)) - operations.append(.Remove(peerId)) - } - } - - func get() -> [PeerId] { - var peerIds: [PeerId] = [] - self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), keys: { key in - peerIds.append(PeerId(key.getInt64(0))) + func get() -> [PeerId: PeerReadStateSynchronizationOperation] { + self.beforeCommit() + + var operations: [PeerId: PeerReadStateSynchronizationOperation] = [:] + self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), values: { key, value in + var operationValue: Int8 = 0 + value.read(&operationValue, offset: 0, length: 1) + + let operation: PeerReadStateSynchronizationOperation + if operationValue == 0 { + var syncValue: Int8 = 0 + value.read(&syncValue, offset: 0, length: 1) + operation = .Push(thenSync: syncValue != 0) + } else { + operation = .Validate + } + + operations[PeerId(key.getInt64(0))] = operation return true }, limit: 0) - return peerIds + return operations } override func beforeCommit() { let key = ValueBoxKey(length: 8) - let buffer = MemoryBuffer() - for (peerId, invalidated) in self.updatedPeerIds { + let buffer = WriteBuffer() + for (peerId, operation) in self.updatedPeerIds { key.setInt64(0, value: peerId.toInt64()) - if invalidated { + if let operation = operation { + buffer.reset() + switch operation { + case let .Push(thenSync): + var operationValue: Int8 = 0 + buffer.write(&operationValue, offset: 0, length: 1) + var syncValue: Int8 = thenSync ? 1 : 0 + buffer.write(&syncValue, offset: 0, length: 1) + case .Validate: + var operationValue: Int8 = 1 + buffer.write(&operationValue, offset: 0, length: 1) + } + self.valueBox.set(self.tableId, key: key, value: buffer) } else { self.valueBox.remove(self.tableId, key: key) diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index 3793491890..f0fb8772fd 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -1,5 +1,12 @@ import Foundation +private let traceReadStates = false + +enum ApplyInteractiveMaxReadIdResult { + case None + case Push(thenSync: Bool) +} + private final class InternalPeerReadStates { var namespaces: [MessageId.Namespace: PeerReadState] @@ -61,6 +68,10 @@ final class MessageHistoryReadStateTable: Table { } func resetStates(peerId: PeerId, namespaces: [MessageId.Namespace: PeerReadState]) -> CombinedPeerReadState? { + if traceReadStates { + print("[ReadStateTable] resetStates peerId: \(peerId), namespaces: \(namespaces)") + } + self.updatedPeerIds.insert(peerId) if let states = self.get(peerId) { @@ -96,6 +107,10 @@ final class MessageHistoryReadStateTable: Table { } if let states = self.get(peerId) { + if traceReadStates { + print("[ReadStateTable] addIncomingMessages peerId: \(peerId), ids: \(ids) (before: \(states.namespaces))") + } + var updated = false var invalidated = false for (namespace, ids) in idsByNamespace { @@ -112,6 +127,10 @@ final class MessageHistoryReadStateTable: Table { if addedUnreadCount != 0 { states.namespaces[namespace] = PeerReadState(maxReadId: currentState.maxReadId, maxKnownId: currentState.maxKnownId, count: currentState.count + addedUnreadCount) updated = true + + if traceReadStates { + print("[ReadStateTable] added \(addedUnreadCount)") + } } } } @@ -122,6 +141,9 @@ final class MessageHistoryReadStateTable: Table { return (updated ? CombinedPeerReadState(states: states.namespaces.map({$0})) : nil, invalidated) } else { + if traceReadStates { + print("[ReadStateTable] addIncomingMessages peerId: \(peerId), just invalidated)") + } return (nil, true) } @@ -139,6 +161,10 @@ final class MessageHistoryReadStateTable: Table { } if let states = self.get(peerId) { + if traceReadStates { + print("[ReadStateTable] deleteMessages peerId: \(peerId), ids: \(ids) (before: \(states.namespaces))") + } + var updated = false var invalidate = false for (namespace, ids) in idsByNamespace { @@ -174,13 +200,28 @@ final class MessageHistoryReadStateTable: Table { return (nil, false) } - func applyMaxReadId(peerId: PeerId, namespace: MessageId.Namespace, maxReadId: MessageId.Id, maxKnownId: MessageId.Id, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool)) -> (CombinedPeerReadState?, Bool) { - if let states = self.get(peerId), state = states.namespaces[namespace] { - if state.maxReadId < maxReadId { - let (deltaCount, holes) = incomingStatsInRange(state.maxReadId + 1, maxReadId) + func applyMaxReadId(messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (CombinedPeerReadState?, Bool) { + if let states = self.get(messageId.peerId), state = states.namespaces[messageId.namespace] { + if traceReadStates { + print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId.id) (before: \(states.namespaces))") + } + + if state.maxReadId < messageId.id || messageId.id == topMessageId { + var (deltaCount, holes) = incomingStatsInRange(state.maxReadId + 1, messageId.id) - states.namespaces[namespace] = PeerReadState(maxReadId: maxReadId, maxKnownId: max(state.maxKnownId, maxReadId), count: state.count - Int32(deltaCount)) - self.updatedPeerIds.insert(peerId) + if traceReadStates { + print("[ReadStateTable] applyMaxReadId after deltaCount: \(deltaCount), holes: \(holes)") + } + + if messageId.id == topMessageId { + if deltaCount != Int(state.count) { + deltaCount = Int(state.count) + holes = true + } + } + + states.namespaces[messageId.namespace] = PeerReadState(maxReadId: messageId.id, maxKnownId: state.maxKnownId, count: state.count - Int32(deltaCount)) + self.updatedPeerIds.insert(messageId.peerId) return (CombinedPeerReadState(states: states.namespaces.map({$0})), holes) } } else { @@ -190,29 +231,14 @@ final class MessageHistoryReadStateTable: Table { return (nil, false) } - func clearUnreadLocally(peerId: PeerId, topId: (PeerId, MessageId.Namespace) -> MessageId.Id?) -> CombinedPeerReadState? { - if let states = self.get(peerId) { - var updatedNamespaces: [MessageId.Namespace: PeerReadState] = [:] - var updated = false - for (namespace, state) in states.namespaces { - if let topMessageId = topId(peerId, namespace) { - let updatedState = PeerReadState(maxReadId: topMessageId, maxKnownId: topMessageId, count: 0) - if updatedState != state { - updated = true - } - updatedNamespaces[namespace] = updatedState - } else { - let updatedState = PeerReadState(maxReadId: state.maxReadId, maxKnownId: state.maxKnownId, count: 0) - updated = true - } - } - if updated { - self.updatedPeerIds.insert(peerId) - return CombinedPeerReadState(states: states.namespaces.map({$0})) - } + func applyInteractiveMaxReadId(messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult) { + let (combinedState, holes) = self.applyMaxReadId(messageId, incomingStatsInRange: incomingStatsInRange, topMessageId: topMessageId) + + if let combinedState = combinedState { + return (combinedState, .Push(thenSync: holes)) } - return nil + return (combinedState, holes ? .Push(thenSync: true) : .None) } override func beforeCommit() { diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 4cc285612c..d83f01ef6c 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -7,6 +7,11 @@ enum MessageHistoryOperation { case UpdateReadState(CombinedPeerReadState) } +struct MessageHistoryAnchorIndex { + let index: MessageIndex + let exact: Bool +} + enum IntermediateMessageHistoryEntry { case Message(IntermediateMessage) case Hole(MessageHistoryHole) @@ -42,16 +47,16 @@ final class MessageHistoryTable: Table { let unsentTable: MessageHistoryUnsentTable let tagsTable: MessageHistoryTagsTable let readStateTable: MessageHistoryReadStateTable - let invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable + let synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable - init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable) { + init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable) { self.messageHistoryIndexTable = messageHistoryIndexTable self.messageMediaTable = messageMediaTable self.historyMetadataTable = historyMetadataTable self.unsentTable = unsentTable self.tagsTable = tagsTable self.readStateTable = readStateTable - self.invalidatedReadStateTable = invalidatedReadStateTable + self.synchronizeReadStateTable = synchronizeReadStateTable super.init(valueBox: valueBox, tableId: tableId) } @@ -106,7 +111,7 @@ final class MessageHistoryTable: Table { return dict } - private func processIndexOperations(peerId: PeerId, operations: [MessageHistoryIndexOperation], inout processedOperationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + private func processIndexOperations(peerId: PeerId, operations: [MessageHistoryIndexOperation], inout processedOperationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0)) let sharedBuffer = WriteBuffer() let sharedEncoder = Encoder() @@ -188,7 +193,7 @@ final class MessageHistoryTable: Table { outputOperations.append(.UpdateReadState(combinedState)) } if invalidate { - self.invalidatedReadStateTable.add(peerId, operations: &invalidatedReadStateOperations) + self.synchronizeReadStateTable.set(peerId, operation: .Validate, operations: &updatedPeerReadStateOperations) } } @@ -213,26 +218,26 @@ final class MessageHistoryTable: Table { return internalStoreMessages } - func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { let messagesByPeerId = self.messagesGroupedByPeerId(messages) for (peerId, peerMessages) in messagesByPeerId { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.addMessages(self.internalStoreMessages(peerMessages), location: location, operations: &operations) - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } } - func addHoles(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + func addHoles(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] for id in messageIds { self.messageHistoryIndexTable.addHole(id, operations: &operations) } - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } } - func removeMessages(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + func removeMessages(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] @@ -243,7 +248,7 @@ final class MessageHistoryTable: Table { for id in messageIds { self.messageHistoryIndexTable.removeMessage(id, operations: &operations) } - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) if let combinedState = combinedState { var outputOperations: [MessageHistoryOperation] = [] @@ -257,24 +262,24 @@ final class MessageHistoryTable: Table { } if invalidate { - self.invalidatedReadStateTable.add(peerId, operations: &invalidatedReadStateOperations) + self.synchronizeReadStateTable.set(peerId, operation: .Validate, operations: &updatedPeerReadStateOperations) } } } - func fillHole(id: MessageId, fillType: HoleFillType, tagMask: MessageTags?, messages: [StoreMessage], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + func fillHole(id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.fillHole(id, fillType: fillType, tagMask: tagMask, messages: self.internalStoreMessages(messages), operations: &operations) - self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - func updateMessage(id: MessageId, message: StoreMessage, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + func updateMessage(id: MessageId, message: StoreMessage, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.updateMessage(id, message: self.internalStoreMessages([message]).first!, operations: &operations) - self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, invalidatedReadStateOperations: &invalidatedReadStateOperations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { for (peerId, namespaces) in states { if let combinedState = self.readStateTable.resetStates(peerId, namespaces: namespaces) { if operationsByPeerId[peerId] == nil { @@ -283,14 +288,19 @@ final class MessageHistoryTable: Table { operationsByPeerId[peerId]!.append(.UpdateReadState(combinedState)) } } - self.invalidatedReadStateTable.remove(peerId, operations: &invalidatedReadStateOperations) + self.synchronizeReadStateTable.set(peerId, operation: nil, operations: &updatedPeerReadStateOperations) } } - func applyIncomingReadMaxId(messageId: MessageId, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { - let (combinedState, invalidated) = self.readStateTable.applyMaxReadId(messageId.peerId, namespace: messageId.namespace, maxReadId: messageId.id, maxKnownId: messageId.id, incomingStatsInRange: { fromId, toId in + func applyIncomingReadMaxId(messageId: MessageId, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + var topMessageId: MessageId.Id? + if let topEntry = self.messageHistoryIndexTable.top(messageId.peerId, namespace: messageId.namespace), case let .Message(index) = topEntry { + topMessageId = index.id.id + } + + let (combinedState, invalidated) = self.readStateTable.applyMaxReadId(messageId, incomingStatsInRange: { fromId, toId in return self.messageHistoryIndexTable.incomingMessageCountInRange(messageId.peerId, namespace: messageId.namespace, minId: fromId, maxId: toId) - }) + }, topMessageId: topMessageId) if let combinedState = combinedState { if operationsByPeerId[messageId.peerId] == nil { @@ -301,7 +311,33 @@ final class MessageHistoryTable: Table { } if invalidated { - self.invalidatedReadStateTable.add(messageId.peerId, operations: &invalidatedReadStateOperations) + self.synchronizeReadStateTable.set(messageId.peerId, operation: .Validate, operations: &updatedPeerReadStateOperations) + } + } + + func applyInteractiveMaxReadId(messageId: MessageId, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + var topMessageId: MessageId.Id? + if let topEntry = self.messageHistoryIndexTable.top(messageId.peerId, namespace: messageId.namespace), case let .Message(index) = topEntry { + topMessageId = index.id.id + } + + let (combinedState, result) = self.readStateTable.applyInteractiveMaxReadId(messageId, incomingStatsInRange: { fromId, toId in + return self.messageHistoryIndexTable.incomingMessageCountInRange(messageId.peerId, namespace: messageId.namespace, minId: fromId, maxId: toId) + }, topMessageId: topMessageId) + + if let combinedState = combinedState { + if operationsByPeerId[messageId.peerId] == nil { + operationsByPeerId[messageId.peerId] = [.UpdateReadState(combinedState)] + } else { + operationsByPeerId[messageId.peerId]!.append(.UpdateReadState(combinedState)) + } + } + + switch result { + case let .Push(thenSync): + self.synchronizeReadStateTable.set(messageId.peerId, operation: .Push(thenSync: thenSync), operations: &updatedPeerReadStateOperations) + case .None: + break } } @@ -1222,6 +1258,30 @@ final class MessageHistoryTable: Table { return entries } + func maxReadIndex(peerId: PeerId) -> MessageHistoryAnchorIndex? { + if let combinedState = self.readStateTable.getCombinedState(peerId), state = combinedState.states.first where state.1.count != 0 { + return self.anchorIndex(MessageId(peerId: peerId, namespace: state.0, id: state.1.maxReadId)) + } + return nil + } + + func anchorIndex(messageId: MessageId) -> MessageHistoryAnchorIndex? { + let (lower, upper) = self.messageHistoryIndexTable.adjacentItems(messageId, bindUpper: false) + if let lower = lower, case let .Hole(hole) = lower where messageId.id >= hole.min && messageId.id <= hole.maxIndex.id.id { + return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: lower.index.timestamp), exact: false) + } + if let upper = upper, case let .Hole(hole) = upper where messageId.id >= hole.min && messageId.id <= hole.maxIndex.id.id { + return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: upper.index.timestamp), exact: false) + } + + if let lower = lower { + return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: lower.index.timestamp), exact: true) + } else if let upper = upper { + return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: upper.index.timestamp), exact: true) + } + return nil + } + func debugList(peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { return self.laterEntries(peerId, index: nil, count: 1000).map({ entry -> RenderedMessageHistoryEntry in switch entry { diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 12e2b49ff9..bf7b27b039 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -1,5 +1,25 @@ import Foundation +public struct MessageHistoryViewId: Equatable { + let peerId: PeerId + let id: Int + let version: Int + + init(peerId: PeerId, id: Int, version: Int = 0) { + self.peerId = peerId + self.id = id + self.version = version + } + + var nextVersion: MessageHistoryViewId { + return MessageHistoryViewId(peerId: self.peerId, id: self.id, version: self.version + 1) + } +} + +public func ==(lhs: MessageHistoryViewId, rhs: MessageHistoryViewId) -> Bool { + return lhs.peerId == rhs.peerId && lhs.id == rhs.id && lhs.version == rhs.version +} + enum MutableMessageHistoryEntry { case IntermediateMessageEntry(IntermediateMessage) case MessageEntry(Message) @@ -68,23 +88,109 @@ final class MutableMessageHistoryViewReplayContext { } final class MutableMessageHistoryView { + private(set) var id: MessageHistoryViewId let tagMask: MessageTags? - private var combinedReadState: CombinedPeerReadState? - private let count: Int + private var anchorIndex: MessageHistoryAnchorIndex + private let combinedReadState: CombinedPeerReadState? private var earlier: MutableMessageHistoryEntry? private var later: MutableMessageHistoryEntry? private var entries: [MutableMessageHistoryEntry] + private let fillCount: Int - init(combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int) { + init(id: MessageHistoryViewId, anchorIndex: MessageHistoryAnchorIndex, combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int) { + self.id = id + self.anchorIndex = anchorIndex self.combinedReadState = combinedReadState self.earlier = earlier self.entries = entries self.later = later self.tagMask = tagMask - self.count = count + self.fillCount = count } - func replay(operations: [MessageHistoryOperation], context: MutableMessageHistoryViewReplayContext) -> Bool { + func incrementVersion() { + self.id = self.id.nextVersion + } + + func updateVisibleRange(earliestVisibleIndex earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex, context: MutableMessageHistoryViewReplayContext) -> Bool { + if (true) { + //return false + } + + var minIndex: Int? + var maxIndex: Int? + + for i in 0 ..< self.entries.count { + if self.entries[i].index >= earliestVisibleIndex { + minIndex = i + break + } + } + + for i in (0 ..< self.entries.count).reverse() { + if self.entries[i].index <= latestVisibleIndex { + maxIndex = i + break + } + } + + if let minIndex = minIndex, maxIndex = maxIndex { + var minClipIndex = minIndex + var maxClipIndex = maxIndex + + while maxClipIndex - minClipIndex <= self.fillCount { + if maxClipIndex != self.entries.count - 1 { + maxClipIndex += 1 + } + + if minClipIndex != 0 { + minClipIndex -= 1 + } else if maxClipIndex == self.entries.count - 1 { + break + } + } + + if minClipIndex != 0 || maxClipIndex != self.entries.count - 1 { + if minClipIndex != 0 { + self.earlier = self.entries[minClipIndex - 1] + } + + if maxClipIndex != self.entries.count - 1 { + self.later = self.entries[maxClipIndex + 1] + } + + for _ in 0 ..< self.entries.count - 1 - maxClipIndex { + /*if case let .MessageEntry(message) = self.entries.last! { + print("remove last \(message.text)") + }*/ + self.entries.removeLast() + } + + for _ in 0 ..< minClipIndex { + /*if case let .MessageEntry(message) = self.entries.first! { + print("remove first \(message.text)") + }*/ + self.entries.removeFirst() + } + + return true + } + } + + return false + } + + func updateAnchorIndex(getIndex: (MessageId) -> MessageHistoryAnchorIndex?) -> Bool { + if !self.anchorIndex.exact { + if let index = getIndex(self.anchorIndex.index.id) { + self.anchorIndex = index + return true + } + } + return false + } + + func replay(operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], context: MutableMessageHistoryViewReplayContext) -> Bool { let tagMask = self.tagMask let unwrappedTagMask: UInt32 = tagMask?.rawValue ?? 0 @@ -93,13 +199,13 @@ final class MutableMessageHistoryView { switch operation { case let .InsertHole(hole): if tagMask == nil || (hole.tags & unwrappedTagMask) != 0 { - if self.add(.HoleEntry(hole)) { + if self.add(.HoleEntry(hole), holeFillDirections: holeFillDirections) { hasChanges = true } } case let .InsertMessage(intermediateMessage): if tagMask == nil || (intermediateMessage.tags.rawValue & unwrappedTagMask) != 0 { - if self.add(.IntermediateMessageEntry(intermediateMessage)) { + if self.add(.IntermediateMessageEntry(intermediateMessage), holeFillDirections: holeFillDirections) { hasChanges = true } } @@ -109,20 +215,20 @@ final class MutableMessageHistoryView { } case let .UpdateReadState(combinedReadState): hasChanges = true - self.combinedReadState = combinedReadState + //self.combinedReadState = combinedReadState } } return hasChanges } - private func add(entry: MutableMessageHistoryEntry) -> Bool { + private func add(entry: MutableMessageHistoryEntry, holeFillDirections: [MessageIndex: HoleFillDirection]) -> Bool { if self.entries.count == 0 { self.entries.append(entry) return true } else { - let first = self.entries[self.entries.count - 1].index - let last = self.entries[0].index + let latestIndex = self.entries[self.entries.count - 1].index + let earliestIndex = self.entries[0].index var next: MessageIndex? if let later = self.later { @@ -131,38 +237,26 @@ final class MutableMessageHistoryView { let index = entry.index - if index < last { + if index < earliestIndex { if self.earlier == nil || self.earlier!.index < index { - if self.entries.count < self.count { - self.entries.insert(entry, atIndex: 0) - } else { - self.earlier = entry - } + self.entries.insert(entry, atIndex: 0) return true } else { return false } - } else if index > first { - if next != nil && index > next! { - if self.later == nil || self.later!.index > index { - if self.entries.count < self.count { - self.entries.append(entry) - } else { - self.later = entry - } + } else if index > latestIndex { + if let later = self.later { + if index < later.index { + self.entries.append(entry) return true } else { return false } } else { self.entries.append(entry) - if self.entries.count > self.count { - self.earlier = self.entries[0] - self.entries.removeAtIndex(0) - } return true } - } else if index != last && index != first { + } else if index != earliestIndex && index != latestIndex { var i = self.entries.count while i >= 1 { if self.entries[i - 1].index < index { @@ -171,10 +265,6 @@ final class MutableMessageHistoryView { i -= 1 } self.entries.insert(entry, atIndex: i) - if self.entries.count > self.count { - self.earlier = self.entries[0] - self.entries.removeAtIndex(0) - } return true } else { return false @@ -214,64 +304,45 @@ final class MutableMessageHistoryView { } func complete(context: MutableMessageHistoryViewReplayContext, fetchEarlier: (MessageIndex?, Int) -> [MutableMessageHistoryEntry], fetchLater: (MessageIndex?, Int) -> [MutableMessageHistoryEntry]) { - if context.removedEntries { - var addedEntries: [MutableMessageHistoryEntry] = [] - - var latestAnchor: MessageIndex? - if let last = self.entries.last { - latestAnchor = last.index - } - - if latestAnchor == nil { - if let later = self.later { - latestAnchor = later.index - } - } - - if let later = self.later { - addedEntries += fetchLater(later.index.predecessor(), self.count) - } - if let earlier = self.earlier { - addedEntries += fetchEarlier(earlier.index.successor(), self.count) - } - - addedEntries += self.entries - addedEntries.sortInPlace({ $0.index < $1.index }) - var i = addedEntries.count - 1 - while i >= 1 { - if addedEntries[i].index.id == addedEntries[i - 1].index.id { - addedEntries.removeAtIndex(i) - } - i -= 1 - } - self.entries = [] - - var anchorIndex = addedEntries.count - 1 - if let latestAnchor = latestAnchor { - var i = addedEntries.count - 1 - while i >= 0 { - if addedEntries[i].index <= latestAnchor { - anchorIndex = i - break + if context.removedEntries && self.entries.count < self.fillCount { + if self.entries.count == 0 { + let anchorIndex = (self.later ?? self.earlier)?.index + + let fetchedEntries = fetchEarlier(anchorIndex, self.fillCount + 2) + if fetchedEntries.count >= self.fillCount + 2 { + self.earlier = fetchedEntries.last + for i in (1 ..< fetchedEntries.count - 1).reverse() { + self.entries.append(fetchedEntries[i]) } - i -= 1 + self.later = fetchedEntries.first + } + } else { + let fetchedEntries = fetchEarlier(self.entries[0].index, self.fillCount - self.entries.count) + for entry in fetchedEntries { + self.entries.insert(entry, atIndex: 0) + } + + if context.invalidEarlier { + var earlyId: MessageIndex? + let i = 0 + if i < self.entries.count { + earlyId = self.entries[i].index + } + + let earlierEntries = fetchEarlier(earlyId, 1) + self.earlier = earlierEntries.first + } + + if context.invalidLater { + var laterId: MessageIndex? + let i = self.entries.count - 1 + if i >= 0 { + laterId = self.entries[i].index + } + + let laterEntries = fetchLater(laterId, 1) + self.later = laterEntries.first } - } - - self.later = nil - if anchorIndex + 1 < addedEntries.count { - self.later = addedEntries[anchorIndex + 1] - } - - i = anchorIndex - while i >= 0 && i > anchorIndex - self.count { - self.entries.insert(addedEntries[i], atIndex: 0) - i -= 1 - } - - self.earlier = nil - if anchorIndex - self.count >= 0 { - self.earlier = addedEntries[anchorIndex - self.count] } } else { if context.invalidEarlier { @@ -313,24 +384,68 @@ final class MutableMessageHistoryView { } } - func firstHole() -> MessageHistoryHole? { - for entry in self.entries.reverse() as ReverseCollection { - if case let .HoleEntry(hole) = entry { - return hole + func firstHole() -> (MessageHistoryHole, HoleFillDirection)? { + if self.entries.isEmpty { + return nil + } + + var referenceIndex = self.entries.count - 1 + for i in 0 ..< self.entries.count { + if self.entries[i].index >= self.anchorIndex.index { + referenceIndex = i + break } } + var i = referenceIndex + var j = referenceIndex + 1 + + while i >= 0 || j < self.entries.count { + if j < self.entries.count { + if case let .HoleEntry(hole) = self.entries[j] { + if self.anchorIndex.index.id.namespace == hole.id.namespace { + if self.anchorIndex.index.id.id >= hole.min && self.anchorIndex.index.id.id <= hole.maxIndex.id.id { + return (hole, .AroundIndex(self.anchorIndex.index)) + } + } + + return (hole, hole.maxIndex <= self.anchorIndex.index ? .UpperToLower : .LowerToUpper) + } + } + + if i >= 0 { + if case let .HoleEntry(hole) = self.entries[i] { + if self.anchorIndex.index.id.namespace == hole.id.namespace { + if self.anchorIndex.index.id.id >= hole.min && self.anchorIndex.index.id.id <= hole.maxIndex.id.id { + return (hole, .AroundIndex(self.anchorIndex.index)) + } + } + + return (hole, hole.maxIndex <= self.anchorIndex.index ? .UpperToLower : .LowerToUpper) + } + } + + i -= 1 + j += 1 + } + return nil } } public final class MessageHistoryView { + public let id: MessageHistoryViewId + public let anchorIndex: MessageIndex public let earlierId: MessageIndex? public let laterId: MessageIndex? public let entries: [MessageHistoryEntry] public let maxReadIndex: MessageIndex? + public let combinedReadState: CombinedPeerReadState? init(_ mutableView: MutableMessageHistoryView) { + self.id = mutableView.id + self.anchorIndex = mutableView.anchorIndex.index + var entries: [MessageHistoryEntry] = [] for entry in mutableView.entries { switch entry { @@ -347,16 +462,42 @@ public final class MessageHistoryView { self.earlierId = mutableView.earlier?.index self.laterId = mutableView.later?.index - if let combinedReadState = mutableView.combinedReadState { + self.combinedReadState = mutableView.combinedReadState + + if let combinedReadState = mutableView.combinedReadState where combinedReadState.count != 0 { var maxIndex: MessageIndex? for (namespace, state) in combinedReadState.states { - for entry in entries { - if entry.index.id.namespace == namespace { - if maxIndex == nil || maxIndex! < entry.index { - maxIndex = entry.index + var maxNamespaceIndex: MessageIndex? + var index = entries.count - 1 + for entry in entries.reverse() { + if entry.index.id.namespace == namespace && entry.index.id.id <= state.maxReadId { + maxNamespaceIndex = entry.index + break + } + index -= 1 + } + if maxNamespaceIndex == nil && index == -1 && entries.count != 0 { + index = 0 + for entry in entries { + if entry.index.id.namespace == namespace { + maxNamespaceIndex = entry.index + break + } + index += 1 + } + } + if let _ = maxNamespaceIndex where index + 1 < entries.count { + for i in index + 1 ..< entries.count { + if case let .MessageEntry(message) = entries[i] where !message.flags.contains(.Incoming) { + maxNamespaceIndex = MessageIndex(message) + } else { + break } } } + if let maxNamespaceIndex = maxNamespaceIndex where maxIndex == nil || maxIndex! < maxNamespaceIndex { + maxIndex = maxNamespaceIndex + } } self.maxReadIndex = maxIndex } else { diff --git a/Postbox/PeerReadState.swift b/Postbox/PeerReadState.swift index a8a66588c4..ab58e09fef 100644 --- a/Postbox/PeerReadState.swift +++ b/Postbox/PeerReadState.swift @@ -1,5 +1,5 @@ -public struct PeerReadState: Equatable { +public struct PeerReadState: Equatable, CustomStringConvertible { public let maxReadId: MessageId.Id public let maxKnownId: MessageId.Id public let count: Int32 @@ -9,12 +9,23 @@ public struct PeerReadState: Equatable { self.maxKnownId = maxKnownId self.count = count } + + public var description: String { + return "(PeerReadState maxReadId: \(maxReadId), maxKnownId: \(maxKnownId), count: \(count))" + } } public func ==(lhs: PeerReadState, rhs: PeerReadState) -> Bool { return lhs.maxReadId == rhs.maxReadId && lhs.maxKnownId == rhs.maxKnownId && lhs.count == rhs.count } -struct CombinedPeerReadState { +public struct CombinedPeerReadState { let states: [(MessageId.Namespace, PeerReadState)] + var count: Int32 { + var result: Int32 = 0 + for (_, state) in self.states { + result += state.count + } + return result + } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 190a3a50db..aa7d968adc 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -2,6 +2,11 @@ import Foundation import SwiftSignalKit import sqlcipher +public enum PreloadedMessageHistoryView { + case Loading + case Preloaded(MessageHistoryView) +} + public protocol PeerChatState: Coding { func equals(other: PeerChatState) -> Bool } @@ -21,7 +26,7 @@ public final class Modifier { self.postbox?.addHole(messageId) } - public func fillHole(hole: MessageHistoryHole, fillType: HoleFillType, tagMask: MessageTags?, messages: [StoreMessage]) { + public func fillHole(hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { self.postbox?.fillHole(hole, fillType: fillType, tagMask: tagMask, messages: messages) } @@ -44,12 +49,16 @@ public final class Modifier { self.postbox?.resetIncomingReadStates(states) } + public func confirmSynchronizedIncomingReadState(peerId: PeerId) { + self.postbox?.confirmSynchronizedIncomingReadState(peerId) + } + public func applyIncomingReadMaxId(messageId: MessageId) { self.postbox?.applyIncomingReadMaxId(messageId) } - public func validateIncomingReadState(peerId: PeerId) { - self.postbox?.validateIncomingReadState(peerId) + public func applyInteractiveReadMaxId(messageId: MessageId) { + self.postbox?.applyInteractiveReadMaxId(messageId) } public func getState() -> Coding? { @@ -72,6 +81,10 @@ public final class Modifier { return self.postbox?.peerTable.get(id) } + public func getPeerReadStates(id: PeerId) -> [(MessageId.Namespace, PeerReadState)]? { + return self.postbox?.readStateTable.getCombinedState(id)?.states + } + public func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer?) { self.postbox?.updatePeers(peers, update: update) } @@ -97,14 +110,16 @@ public final class Postbox { private var valueBox: ValueBox! private var viewTracker: ViewTracker! + private var nextViewId = 0 private var peerPipes: [PeerId: Pipe] = [:] private var currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] private var currentUnsentOperations: [IntermediateMessageHistoryUnsentOperation] = [] - private var currentInvalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation] = [] + private var currentUpdatedSynchronizeReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - private var currentFilledHolesByPeerId = Set() + private var currentRemovedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] = [:] + private var currentFilledHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] = [:] private var currentUpdatedPeers: [PeerId: Peer] = [:] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] @@ -115,8 +130,8 @@ public final class Postbox { self.fetchChatListHoleImpl.set(single(fetch, NoError.self)) } - private let fetchMessageHistoryHoleImpl = Promise<(MessageHistoryHole, MessageTags?) -> Signal>() - public func setFetchMessageHistoryHole(fetch: (MessageHistoryHole, MessageTags?) -> Signal) { + private let fetchMessageHistoryHoleImpl = Promise<(MessageHistoryHole, HoleFillDirection, MessageTags?) -> Signal>() + public func setFetchMessageHistoryHole(fetch: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Signal) { self.fetchMessageHistoryHoleImpl.set(single(fetch, NoError.self)) } @@ -125,9 +140,9 @@ public final class Postbox { self.sendUnsentMessageImpl.set(single(sendUnsentMessage, NoError.self)) } - private let validatePeerReadStateImpl = Promise Signal>() - public func setValidatePeerReadState(validatePeerReadState: PeerId -> Signal) { - self.validatePeerReadStateImpl.set(.single(validatePeerReadState)) + private let synchronizePeerReadStateImpl = Promise<(PeerId, PeerReadStateSynchronizationOperation) -> Signal>() + public func setSynchronizePeerReadState(synchronizePeerReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Signal) { + self.synchronizePeerReadStateImpl.set(.single(synchronizePeerReadState)) } public let mediaBox: MediaBox @@ -149,7 +164,7 @@ public final class Postbox { var messageHistoryTagsTable: MessageHistoryTagsTable! var peerChatStateTable: PeerChatStateTable! var readStateTable: MessageHistoryReadStateTable! - var invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable! + var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable! public init(basePath: String, globalMessageIdsNamespace: MessageId.Namespace, seedConfiguration: SeedConfiguration) { self.basePath = basePath @@ -163,7 +178,7 @@ public final class Postbox { } private func debugSaveState(name: String) { - self.queue.dispatch { + self.queue.justDispatch({ let path = self.basePath + name let _ = try? NSFileManager.defaultManager().removeItemAtPath(path) do { @@ -171,11 +186,11 @@ public final class Postbox { } catch (let e) { print("(Postbox debugSaveState: error \(e))") } - } + }) } private func debugRestoreState(name: String) { - self.queue.dispatch { + self.queue.justDispatch({ let path = self.basePath + name let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath) do { @@ -183,11 +198,11 @@ public final class Postbox { } catch (let e) { print("(Postbox debugRestoreState: error \(e))") } - } + }) } private func openDatabase() { - self.queue.dispatch { + self.queue.justDispatch({ let startTime = CFAbsoluteTimeGetCurrent() do { @@ -196,15 +211,26 @@ public final class Postbox { } //let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath + "/media") - //self.debugSaveState("beforeGetDiff") - //self.debugRestoreState("beforeGetDiff") - //self.debugRestoreState("clean") + + //#if TARGET_IPHONE_SIMULATOR + + //self.debugRestoreState("_empty") + + // debugging large amount of updates + //self.debugSaveState("beforeHoles") + //self.debugRestoreState("beforeHoles") + + // debugging unread counters + //self.debugSaveState("afterLogin") + //self.debugRestoreState("afterLogin") + + //#endif self.valueBox = SqliteValueBox(basePath: self.basePath + "/db") self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 1 + let currentUserVersion: Int32 = 3 if userVersion != currentUserVersion { self.valueBox.drop() @@ -221,8 +247,8 @@ public final class Postbox { self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox, tableId: 5) self.mediaTable = MessageMediaTable(valueBox: self.valueBox, tableId: 6, mediaCleanupTable: self.mediaCleanupTable) self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox, tableId: 14) - self.invalidatedReadStateTable = MessageHistoryInvalidatedReadStateTable(valueBox: self.valueBox, tableId: 15) - self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, tableId: 7, messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, invalidatedReadStateTable: self.invalidatedReadStateTable!) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox, tableId: 15) + self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, tableId: 7, messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable!) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, tableId: 8) self.chatListTable = ChatListTable(valueBox: self.valueBox, tableId: 9, indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, tableId: 13) @@ -239,24 +265,30 @@ public final class Postbox { self.tables.append(self.chatListTable) self.tables.append(self.peerChatStateTable) self.tables.append(self.readStateTable) - self.tables.append(self.invalidatedReadStateTable) + self.tables.append(self.synchronizeReadStateTable) - self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, renderMessage: self.renderIntermediateMessage, fetchChatListHole: self.fetchChatListHoleWrapper, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper, sendUnsentMessage: self.sendUnsentMessageWrapper, unsentMessageIndices: self.messageHistoryUnsentTable!.get(), validateReadState: self.validatePeerReadStateWrapper, invalidatedReadStatePeerIds: self.invalidatedReadStateTable!.get()) + self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, fetchAnchorIndex: self.fetchAnchorIndex, renderMessage: self.renderIntermediateMessage, fetchChatListHole: self.fetchChatListHoleWrapper, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper, sendUnsentMessage: self.sendUnsentMessageWrapper, unsentMessageIndices: self.messageHistoryUnsentTable!.get(), synchronizeReadState: self.synchronizePeerReadStateWrapper, synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - } + }) + } + + private func takeNextViewId() -> Int { + let nextId = self.nextViewId + self.nextViewId += 1 + return nextId } private var cachedState: Coding? private func setState(state: Coding) { - self.queue.dispatch { + self.queue.justDispatch({ self.cachedState = state self.metadataTable.setState(state) self.statePipe.putNext(state) - } + }) } private func getState() -> Coding? { @@ -275,12 +307,12 @@ public final class Postbox { public func state() -> Signal { return Signal { subscriber in let disposable = MetaDisposable() - self.queue.dispatch { + self.queue.justDispatch({ subscriber.putNext(self.getState()) disposable.set(self.statePipe.signal().start(next: { next in subscriber.putNext(next) })) - } + }) return disposable } @@ -299,16 +331,51 @@ public final class Postbox { } private func addMessages(messages: [StoreMessage], location: AddMessagesLocation) { - self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) + self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } private func addHole(id: MessageId) { - self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) + self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func fillHole(hole: MessageHistoryHole, fillType: HoleFillType, tagMask: MessageTags?, messages: [StoreMessage]) { - self.messageHistoryTable.fillHole(hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) - self.currentFilledHolesByPeerId.insert(hole.id.peerId) + private func fillHole(hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.messageHistoryTable.fillHole(hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + for (peerId, operations) in operationsByPeerId { + if self.currentOperationsByPeerId[peerId] == nil { + self.currentOperationsByPeerId[peerId] = operations + } else { + self.currentOperationsByPeerId[peerId]!.appendContentsOf(operations) + } + + var filledMessageIndices: [MessageIndex: HoleFillDirection] = [:] + for operation in operations { + switch operation { + case let .InsertHole(hole): + filledMessageIndices[hole.maxIndex] = fillType.direction + case let .InsertMessage(message): + filledMessageIndices[MessageIndex(message)] = fillType.direction + default: + break + } + } + + if !filledMessageIndices.isEmpty { + if self.currentFilledHolesByPeerId[peerId] == nil { + self.currentFilledHolesByPeerId[peerId] = filledMessageIndices + } else { + for (messageIndex, direction) in filledMessageIndices { + self.currentFilledHolesByPeerId[peerId]![messageIndex] = direction + } + } + } + + if self.currentRemovedHolesByPeerId[peerId] == nil { + self.currentRemovedHolesByPeerId[peerId] = [hole.maxIndex: fillType.direction] + } else { + self.currentRemovedHolesByPeerId[peerId]![hole.maxIndex] = fillType.direction + } + } } private func replaceChatListHole(index: MessageIndex, hole: ChatListHole?) { @@ -316,19 +383,23 @@ public final class Postbox { } private func deleteMessages(messageIds: [MessageId]) { - self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) + self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } private func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]]) { - self.messageHistoryTable.resetIncomingReadStates(states, operationsByPeerId: &self.currentOperationsByPeerId, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) + self.messageHistoryTable.resetIncomingReadStates(states, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + } + + private func confirmSynchronizedIncomingReadState(peerId: PeerId) { + self.synchronizeReadStateTable.set(peerId, operation: nil, operations: &self.currentUpdatedSynchronizeReadStateOperations) } private func applyIncomingReadMaxId(messageId: MessageId) { - self.messageHistoryTable.applyIncomingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) + self.messageHistoryTable.applyIncomingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func validateIncomingReadState(peerId: PeerId) { - self.invalidatedReadStateTable.remove(peerId, operations: &self.currentInvalidatedReadStateOperations) + private func applyInteractiveReadMaxId(messageId: MessageId) { + self.messageHistoryTable.applyInteractiveMaxReadId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } private func fetchEarlierHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { @@ -488,6 +559,10 @@ public final class Postbox { return entries } + private func fetchAnchorIndex(id: MessageId) -> MessageHistoryAnchorIndex? { + return self.messageHistoryTable.anchorIndex(id) + } + private func renderIntermediateMessage(message: IntermediateMessage) -> Message { return self.messageHistoryTable.renderMessage(message, peerTable: self.peerTable) } @@ -498,9 +573,9 @@ public final class Postbox { }).start() } - private func fetchMessageHistoryHoleWrapper(hole: MessageHistoryHole, tagMask: MessageTags?) -> Disposable { + private func fetchMessageHistoryHoleWrapper(hole: MessageHistoryHole, direction: HoleFillDirection, tagMask: MessageTags?) -> Disposable { return (self.fetchMessageHistoryHoleImpl.get() |> mapToSignal { fetch in - return fetch(hole, tagMask) + return fetch(hole, direction, tagMask) }).start() } @@ -515,9 +590,9 @@ public final class Postbox { }).start() } - private func validatePeerReadStateWrapper(peerId: PeerId) -> Disposable { - return (self.validatePeerReadStateImpl.get() |> mapToSignal { validate -> Signal in - return validate(peerId) + private func synchronizePeerReadStateWrapper(peerId: PeerId, operation: PeerReadStateSynchronizationOperation) -> Disposable { + return (self.synchronizePeerReadStateImpl.get() |> mapToSignal { validate -> Signal in + return validate(peerId, operation) }).start() } @@ -528,14 +603,15 @@ public final class Postbox { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } - self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, invalidatedReadStateOperations: self.currentInvalidatedReadStateOperations) + self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations) self.currentOperationsByPeerId.removeAll() self.currentFilledHolesByPeerId.removeAll() + self.currentRemovedHolesByPeerId.removeAll() self.currentUpdatedPeers.removeAll() self.currentReplaceChatListHoles.removeAll() self.currentUnsentOperations.removeAll() - self.currentInvalidatedReadStateOperations.removeAll() + self.currentUpdatedSynchronizeReadStateOperations.removeAll() for table in self.tables { table.beforeCommit() @@ -568,7 +644,7 @@ public final class Postbox { private func updateMessage(index: MessageIndex, update: Message -> StoreMessage) { if let intermediateMessage = self.messageHistoryTable.getMessage(index) { let message = self.renderIntermediateMessage(intermediateMessage) - self.messageHistoryTable.updateMessage(index.id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, invalidatedReadStateOperations: &self.currentInvalidatedReadStateOperations) + self.messageHistoryTable.updateMessage(index.id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } } @@ -586,7 +662,8 @@ public final class Postbox { public func modify(f: Modifier -> T) -> Signal { return Signal { subscriber in - self.queue.dispatch { + self.queue.justDispatch({ + //let startTime = CFAbsoluteTimeGetCurrent() //self.valueBox.beginStats() self.valueBox.begin() let result = f(Modifier(postbox: self)) @@ -594,26 +671,104 @@ public final class Postbox { self.valueBox.commit() subscriber.putNext(result) + //print("modify \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") subscriber.putCompletion() - } + }) return EmptyDisposable } } - - public func tailMessageHistoryViewForPeerId(peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return self.aroundMessageHistoryViewForPeerId(peerId, index: MessageIndex.upperBound(peerId), count: count, tagMask: tagMask) + + public func preloadedAroundUnreadMessageHistoryViewForPeerId(peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + return self.aroundUnreadMessageHistoryViewForPeerId(peerId, count: count, tagMask: tagMask) |> filter { view, _ in + if let maxReadIndex = view.maxReadIndex { + var targetIndex = 0 + for i in 0 ..< view.entries.count { + if view.entries[i].index >= maxReadIndex { + targetIndex = i + break + } + } + + /*print("entries:") + for i in 0 ..< view.entries.count { + switch view.entries[i] { + case let .MessageEntry(message): + print(" \(i == targetIndex ? "*" : " ")\(message.id.id): \(message.text)") + case let .HoleEntry(hole): + print(" \(i == targetIndex ? "*" : " ")\(hole.min) — \(hole.maxIndex.id.id): hole") + } + }*/ + + let maxIndex = min(view.entries.count, targetIndex + count / 2) + if maxIndex >= targetIndex { + for i in targetIndex ..< maxIndex { + if case .HoleEntry = view.entries[i] { + return false + } + } + } + + return true + } else { + return true + } + } |> take(1) |> mapToSignal { _ in + return self.aroundUnreadMessageHistoryViewForPeerId(peerId, count: count, tagMask: tagMask) + } } - public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + public func aroundUnreadMessageHistoryViewForPeerId(peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return Signal { subscriber in let disposable = MetaDisposable() - self.queue.dispatch { + var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId), exact: true) + if let maxReadIndex = self.messageHistoryTable.maxReadIndex(peerId) { + index = maxReadIndex + } + disposable.set(self.aroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, tagMask: tagMask).start(next: { next in + subscriber.putNext(next) + }, error: { error in + subscriber.putError(error) + }, completed: { + subscriber.putCompletion() + })) + } + return disposable + } + } + + public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + return self.aroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, tagMask: tagMask) + } + + private func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + return Signal { subscriber in + let disposable = MetaDisposable() + let startTime = CFAbsoluteTimeGetCurrent() + self.queue.justDispatch({ + //print("+ queue \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) + print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - let mutableView = MutableMessageHistoryView(combinedReadState: self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) + var adjustedAnchorIndex = anchorIndex + /*if entries.count != 0 { + let minIndex = entries[0].index + if anchorIndex < minIndex { + //adjustedAnchorIndex = minIndex + } + }*/ + + let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: adjustedAnchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) mutableView.render(self.renderIntermediateMessage) - subscriber.putNext((MessageHistoryView(mutableView), .Generic)) + //print("+ render \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") + + let initialUpdateType: ViewUpdateType + if let unreadIndex = unreadIndex { + initialUpdateType = .InitialUnread(unreadIndex) + } else { + initialUpdateType = .Generic + } + subscriber.putNext((MessageHistoryView(mutableView), initialUpdateType)) let (index, signal) = self.viewTracker.addMessageHistoryView(peerId, view: mutableView) @@ -631,6 +786,27 @@ public final class Postbox { } return }) + }) + + return disposable + } + } + + public func messageIndexAtId(id: MessageId) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.queue.justDispatch { + if let entry = self.messageHistoryIndexTable.get(id), case let .Message(index) = entry { + subscriber.putNext(index) + subscriber.putCompletion() + } else if let hole = self.messageHistoryIndexTable.holeContainingId(id) { + subscriber.putNext(nil) + subscriber.putCompletion() + } else { + subscriber.putNext(nil) + subscriber.putCompletion() + } } return disposable @@ -645,7 +821,7 @@ public final class Postbox { return Signal { subscriber in let disposable = MetaDisposable() - self.queue.dispatch { + self.queue.justDispatch({ let (entries, earlier, later) = self.fetchAroundChatEntries(index, count: count) let mutableView = MutableChatListView(earlier: earlier, entries: entries, later: later, count: count) @@ -668,7 +844,7 @@ public final class Postbox { } return }) - } + }) return disposable } @@ -677,12 +853,18 @@ public final class Postbox { public func peerWithId(id: PeerId) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() - self.queue.dispatch { + self.queue.justDispatch({ if let peer: Peer = self.peerTable.get(id) { subscriber.putNext(peer) } - } + }) return disposable } } + + public func updateMessageHistoryViewVisibleRange(id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { + self.queue.justDispatch({ + self.viewTracker.updateMessageHistoryViewVisibleRange(id, earliestVisibleIndex: earliestVisibleIndex, latestVisibleIndex: latestVisibleIndex) + }) + } } diff --git a/Postbox/SimpleDictionary.swift b/Postbox/SimpleDictionary.swift index 48d8af5a1f..84d03084c6 100644 --- a/Postbox/SimpleDictionary.swift +++ b/Postbox/SimpleDictionary.swift @@ -41,4 +41,5 @@ public struct SimpleDictionary: SequenceType { return nil } } -} \ No newline at end of file +} + diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 8185d75c6f..4ad5dbf0db 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -380,7 +380,7 @@ public final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: COpaquePointer = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key=?", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT rowid FROM t\(table) WHERE key=?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.existsStatements[table] = preparedStatement resultStatement = preparedStatement diff --git a/Postbox/SynchronizePeerReadStatesView.swift b/Postbox/SynchronizePeerReadStatesView.swift new file mode 100644 index 0000000000..b9a0427a8d --- /dev/null +++ b/Postbox/SynchronizePeerReadStatesView.swift @@ -0,0 +1,29 @@ +import Foundation + +final class SynchronizePeerReadStatesView { + var operations: [PeerId: PeerReadStateSynchronizationOperation] + + init(operations: [PeerId: PeerReadStateSynchronizationOperation]) { + self.operations = operations + } + + func replay(updatedOperations: [PeerId: PeerReadStateSynchronizationOperation?]) -> [PeerId: PeerReadStateSynchronizationOperation?] { + var updates: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + + for (peerId, operation) in updatedOperations { + if let operation = operation { + if self.operations[peerId] == nil || self.operations[peerId]! != operation { + self.operations[peerId] = operation + updates[peerId] = operation + } + } else { + if let _ = self.operations[peerId] { + self.operations.removeValueForKey(peerId) + updates[peerId] = nil + } + } + } + + return updates + } +} diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 772c2efdee..4733db9008 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -2,8 +2,10 @@ import Foundation import SwiftSignalKit public enum ViewUpdateType { + case InitialUnread(MessageIndex) case Generic - case FillHole + case FillHole(insertions: [MessageIndex: HoleFillDirection], deletions: [MessageIndex: HoleFillDirection]) + case UpdateVisible } final class ViewTracker { @@ -12,38 +14,45 @@ final class ViewTracker { private let fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry] private let fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] private let fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] + private let fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex? private let renderMessage: IntermediateMessage -> Message private let fetchChatListHole: ChatListHole -> Disposable - private let fetchMessageHistoryHole: (MessageHistoryHole, MessageTags?) -> Disposable + private let fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable private let sendUnsentMessage: MessageIndex -> Disposable - private let validateReadState: PeerId -> Disposable + private let synchronizeReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Disposable private var chatListViews = Bag<(MutableChatListView, Pipe<(ChatListView, ViewUpdateType)>)>() private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] private var unsentMessageView: UnsentMessageHistoryView - private var invalidatedReadStatesView: InvalidatedPeerReadStatesView + private var synchronizeReadStatesView: SynchronizePeerReadStatesView private var chatListHoleDisposables: [(ChatListHole, Disposable)] = [] private var holeDisposablesByPeerId: [PeerId: [(MessageHistoryHole, Disposable)]] = [:] private var unsentMessageDisposables: [MessageIndex: Disposable] = [:] - private var validateReadStatesDisposables: [PeerId: Disposable] = [:] + private var synchronizeReadStatesDisposables: [PeerId: (PeerReadStateSynchronizationOperation, Disposable)] = [:] - init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], renderMessage: IntermediateMessage -> Message, fetchChatListHole: ChatListHole -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, MessageTags?) -> Disposable, sendUnsentMessage: MessageIndex -> Disposable, unsentMessageIndices: [MessageIndex], validateReadState: PeerId -> Disposable, invalidatedReadStatePeerIds: [PeerId]) { + init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: IntermediateMessage -> Message, fetchChatListHole: ChatListHole -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable, sendUnsentMessage: MessageIndex -> Disposable, unsentMessageIndices: [MessageIndex], synchronizeReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Disposable, synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries self.fetchEarlierChatEntries = fetchEarlierChatEntries self.fetchLaterChatEntries = fetchLaterChatEntries + self.fetchAnchorIndex = fetchAnchorIndex self.renderMessage = renderMessage self.fetchChatListHole = fetchChatListHole self.fetchMessageHistoryHole = fetchMessageHistoryHole self.sendUnsentMessage = sendUnsentMessage - self.validateReadState = validateReadState + self.synchronizeReadState = synchronizeReadState self.unsentMessageView = UnsentMessageHistoryView(indices: unsentMessageIndices) - self.invalidatedReadStatesView = InvalidatedPeerReadStatesView(peerIds: invalidatedReadStatePeerIds) + self.synchronizeReadStatesView = SynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) self.unsentViewUpdated() - self.invalidatedReadStateViewUpdated() + + var initialPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + for (peerId, operation) in synchronizePeerReadStateOperations { + initialPeerReadStateOperations[peerId] = operation + } + self.synchronizeReadStateViewUpdated(initialPeerReadStateOperations) } deinit { @@ -97,7 +106,39 @@ final class ViewTracker { self.updateTrackedChatListHoles() } - func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: Set, chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], invalidatedReadStateOperations: [IntermediateMessageHistoryInvalidatedReadStateOperation]) { + func updateMessageHistoryViewVisibleRange(id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { + if let bag = self.messageHistoryViews[id.peerId] { + for (mutableView, pipe) in bag.copyItems() { + if mutableView.id == id { + let context = MutableMessageHistoryViewReplayContext() + var updated = false + + let updateType: ViewUpdateType = .UpdateVisible + + if mutableView.updateVisibleRange(earliestVisibleIndex: earliestVisibleIndex, latestVisibleIndex: latestVisibleIndex, context: context) { + mutableView.complete(context, fetchEarlier: { index, count in + return self.fetchEarlierHistoryEntries(id.peerId, index, count, mutableView.tagMask) + }, fetchLater: { index, count in + return self.fetchLaterHistoryEntries(id.peerId, index, count, mutableView.tagMask) + }) + mutableView.incrementVersion() + updated = true + } + + if updated { + mutableView.render(self.renderMessage) + pipe.putNext((MessageHistoryView(mutableView), updateType)) + + self.updateTrackedHoles(id.peerId) + } + + break + } + } + } + } + + func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { @@ -108,12 +149,24 @@ final class ViewTracker { let context = MutableMessageHistoryViewReplayContext() var updated = false - if mutableView.replay(operations, context: context) { + let updateType: ViewUpdateType + if let filledIndices = peerIdsWithFilledHoles[peerId] { + updateType = .FillHole(insertions: filledIndices, deletions: removedHolesByPeerId[peerId] ?? [:]) + } else { + updateType = .Generic + } + + if mutableView.replay(operations, holeFillDirections: peerIdsWithFilledHoles[peerId] ?? [:], context: context) { mutableView.complete(context, fetchEarlier: { index, count in return self.fetchEarlierHistoryEntries(peerId, index, count, mutableView.tagMask) }, fetchLater: { index, count in return self.fetchLaterHistoryEntries(peerId, index, count, mutableView.tagMask) }) + mutableView.incrementVersion() + updated = true + } + + if mutableView.updateAnchorIndex(self.fetchAnchorIndex) { updated = true } @@ -123,7 +176,9 @@ final class ViewTracker { if updated { mutableView.render(self.renderMessage) - pipe.putNext((MessageHistoryView(mutableView), peerIdsWithFilledHoles.contains(peerId) ? .FillHole : .Generic)) + + + pipe.putNext((MessageHistoryView(mutableView), updateType)) } } } @@ -154,8 +209,9 @@ final class ViewTracker { self.unsentViewUpdated() } - if self.invalidatedReadStatesView.replay(invalidatedReadStateOperations) { - self.invalidatedReadStateViewUpdated() + let synchronizeReadStateUpdates = self.synchronizeReadStatesView.replay(updatedSynchronizePeerReadStateOperations) + if !synchronizeReadStateUpdates.isEmpty { + self.synchronizeReadStateViewUpdated(synchronizeReadStateUpdates) } } @@ -217,11 +273,11 @@ final class ViewTracker { private func updateTrackedHoles(peerId: PeerId) { if let bag = self.messageHistoryViews[peerId] { var disposeHoles: [Disposable] = [] - var firstHolesAndTags: [(MessageHistoryHole, MessageTags?)] = [] + var firstHolesAndTags: [(MessageHistoryHole, HoleFillDirection, MessageTags?)] = [] for (view, _) in bag.copyItems() { - if let hole = view.firstHole() { - firstHolesAndTags.append((hole, view.tagMask)) + if let (hole, direction) = view.firstHole() { + firstHolesAndTags.append((hole, direction, view.tagMask)) } } @@ -229,7 +285,7 @@ final class ViewTracker { var i = 0 for (hole, disposable) in holes { var exists = false - for (firstHole, _) in firstHolesAndTags { + for (firstHole, _, _) in firstHolesAndTags { if hole == firstHole { exists = true break @@ -265,7 +321,7 @@ final class ViewTracker { self.holeDisposablesByPeerId[peerId] = [] } - self.holeDisposablesByPeerId[peerId]!.append((anyHoleAndTag.0, self.fetchMessageHistoryHole(anyHoleAndTag.0, anyHoleAndTag.1))) + self.holeDisposablesByPeerId[peerId]!.append((anyHoleAndTag.0, self.fetchMessageHistoryHole(anyHoleAndTag.0, anyHoleAndTag.1, anyHoleAndTag.2))) } } } @@ -313,30 +369,14 @@ final class ViewTracker { } } - private func invalidatedReadStateViewUpdated() { - var removePeerIds: [PeerId] = [] - let currentPeerIds = self.invalidatedReadStatesView.peerIds - for (peerId, _) in self.validateReadStatesDisposables { - if !currentPeerIds.contains(peerId) { - removePeerIds.append(peerId) - } - } - - for peerId in removePeerIds { - self.validateReadStatesDisposables.removeValueForKey(peerId)?.dispose() - } - - for peerId in currentPeerIds { - var found = false - for (currentPeerId, _) in validateReadStatesDisposables { - if peerId == currentPeerId { - found = true - break - } + private func synchronizeReadStateViewUpdated(updates: [PeerId: PeerReadStateSynchronizationOperation?]) { + for (peerId, operation) in updates { + if let (_, disposable) = self.synchronizeReadStatesDisposables.removeValueForKey(peerId) { + disposable.dispose() } - if !found { - self.validateReadStatesDisposables[peerId] = self.validateReadState(peerId) + if let operation = operation { + self.synchronizeReadStatesDisposables[peerId] = (operation, self.synchronizeReadState(peerId, operation)) } } } diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index 7f569f75a4..96aa20e6da 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -74,7 +74,7 @@ class ChatListTableTests: XCTestCase { var unsentTable: MessageHistoryUnsentTable? var tagsTable: MessageHistoryTagsTable? var readStateTable: MessageHistoryReadStateTable? - var invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable? + var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? override class func setUp() { super.setUp() @@ -98,8 +98,8 @@ class ChatListTableTests: XCTestCase { self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 11) - self.invalidatedReadStateTable = MessageHistoryInvalidatedReadStateTable(valueBox: self.valueBox!, tableId: 12) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, invalidatedReadStateTable: invalidatedReadStateTable) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, tableId: 12) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, tableId: 5) self.chatListTable = ChatListTable(valueBox: self.valueBox!, tableId: 6, indexTable: self.chatListIndexTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) } @@ -122,7 +122,8 @@ class ChatListTableTests: XCTestCase { private func addMessage(peerId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = []) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] - self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } @@ -130,7 +131,8 @@ class ChatListTableTests: XCTestCase { private func addHole(peerId: Int32, _ id: Int32) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] - self.historyTable!.addHoles([MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.addHoles([MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } @@ -153,15 +155,17 @@ class ChatListTableTests: XCTestCase { private func removeMessages(peerId: Int32, _ ids: [Int32]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] - self.historyTable!.removeMessages(ids.map({ MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.removeMessages(ids.map({ MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } - private func fillHole(peerId: Int32, _ id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32, String, [Media])]) { + private func fillHole(peerId: Int32, _ id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32, String, [Media])]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] - self.historyTable!.fillHole(MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), fillType: fillType, tagMask: nil, messages: messages.map({ StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.fillHole(MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), fillType: fillType, tagMask: nil, messages: messages.map({ StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index 17d46e1a54..6b18fa2a88 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -9,6 +9,12 @@ import Postbox private let peerId = PeerId(namespace: 1, id: 1) private let namespace: Int32 = 1 +private extension MessageIndex { + init(id: Int32, timestamp: Int32) { + self.init(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp) + } +} + private enum Item: Equatable, CustomStringConvertible { case Message(Int32, Int32) case Hole(Int32, Int32, Int32) @@ -103,11 +109,8 @@ class MessageHistoryIndexTableTests: XCTestCase { }, location: .UpperHistoryBlock, operations: &operations) } - func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32)], _ tagMask: MessageTags? = nil) { + func fillHole(id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32)], _ tagMask: MessageTags? = nil) { var operations: [MessageHistoryIndexOperation] = [] - let zeroData = WriteBuffer() - var zero: Int32 = 0 - zeroData.write(&zero, offset: 0, length: 4) self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ return InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: []) @@ -265,42 +268,42 @@ class MessageHistoryIndexTableTests: XCTestCase { } func testFillHoleEmpty() { - fillHole(1, .Complete, []) + fillHole(1, HoleFill(complete: true, direction: .UpperToLower), []) expect([]) } func testFillHoleComplete() { addHole(100) - fillHole(1, .Complete, [(100, 100), (200, 200)]) + fillHole(1, HoleFill(complete: true, direction: .UpperToLower), [(100, 100), (200, 200)]) expect([.Message(100, 100), .Message(200, 200)]) } func testFillHoleUpperToLowerPartial() { addHole(100) - fillHole(1, .UpperToLower, [(100, 100), (200, 200)]) + fillHole(1, HoleFill(complete: false, direction: .UpperToLower), [(100, 100), (200, 200)]) expect([.Hole(1, 99, 100), .Message(100, 100), .Message(200, 200)]) } func testFillHoleUpperToLowerToBounds() { addHole(100) - fillHole(1, .UpperToLower, [(1, 1), (200, 200)]) + fillHole(1, HoleFill(complete: false, direction: .UpperToLower), [(1, 1), (200, 200)]) expect([.Message(1, 1), .Message(200, 200)]) } func testFillHoleLowerToUpperToBounds() { addHole(100) - fillHole(1, .LowerToUpper, [(100, 100), (Int32.max, 200)]) + fillHole(1, HoleFill(complete: false, direction: .LowerToUpper), [(100, 100), (Int32.max, 200)]) expect([.Message(100, 100), .Message(Int32.max, 200)]) } func testFillHoleLowerToUpperPartial() { addHole(100) - fillHole(1, .LowerToUpper, [(100, 100), (200, 200)]) + fillHole(1, HoleFill(complete: false, direction: .LowerToUpper), [(100, 100), (200, 200)]) expect([.Message(100, 100), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) } @@ -310,7 +313,7 @@ class MessageHistoryIndexTableTests: XCTestCase { addMessage(100, 100) addMessage(200, 200) - fillHole(199, .UpperToLower, [(150, 150)]) + fillHole(199, HoleFill(complete: false, direction: .UpperToLower), [(150, 150)]) expect([.Hole(1, 99, 100), .Message(100, 100), .Hole(101, 149, 150), .Message(150, 150), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) } @@ -321,7 +324,7 @@ class MessageHistoryIndexTableTests: XCTestCase { addMessage(100, 100) addMessage(200, 200) - fillHole(199, .LowerToUpper, [(150, 150)]) + fillHole(199, HoleFill(complete: false, direction: .LowerToUpper), [(150, 150)]) expect([.Hole(1, 99, 100), .Message(100, 100), .Message(150, 150), .Hole(151, 199, 200), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) } @@ -332,7 +335,7 @@ class MessageHistoryIndexTableTests: XCTestCase { addMessage(100, 100) addMessage(200, 200) - fillHole(199, .Complete, [(150, 150)]) + fillHole(199, HoleFill(complete: true, direction: .UpperToLower), [(150, 150)]) expect([.Hole(1, 99, 100), .Message(100, 100), .Message(150, 150), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) } @@ -350,7 +353,7 @@ class MessageHistoryIndexTableTests: XCTestCase { addMessage(100, 100) addHole(1) - fillHole(99, .Complete, []) + fillHole(99, HoleFill(complete: true, direction: .UpperToLower), []) expect([.Message(100, 100)]) } @@ -359,7 +362,7 @@ class MessageHistoryIndexTableTests: XCTestCase { addMessage(100, 100) addMessage(101, 101) - fillHole(100, .Complete, [(90, 90)]) + fillHole(100, HoleFill(complete: true, direction: .UpperToLower), [(90, 90)]) expect([.Message(90, 90), .Message(100, 100), .Message(101, 101)]) } @@ -369,7 +372,7 @@ class MessageHistoryIndexTableTests: XCTestCase { addMessage(200, 200) addHole(150) - fillHole(199, .UpperToLower, [(150, 150), (300, 300)]) + fillHole(199, HoleFill(complete: false, direction: .UpperToLower), [(150, 150), (300, 300)]) expect([.Message(100, 100), .Hole(101, 149, 150), .Message(150, 150), .Message(200, 200), .Message(300, 300)]) } @@ -502,4 +505,18 @@ class MessageHistoryIndexTableTests: XCTestCase { addMessagesUpperBlock([(10, 11)]) expect([.Message(10, 10)]) } + + func testFillHoleAtIndex() { + addHole(1) + expect([.Hole(1, Int32.max, Int32.max)]) + fillHole(1, HoleFill(complete: false, direction: .AroundIndex(MessageIndex(id: 10, timestamp: 10))), [(5, 5), (10, 10)]) + expect([.Hole(1, 4, 5), .Message(5, 5), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + } + + func testFillHoleAtIndexComplete() { + addHole(1) + expect([.Hole(1, Int32.max, Int32.max)]) + fillHole(1, HoleFill(complete: true, direction: .AroundIndex(MessageIndex(id: 10, timestamp: 10))), [(5, 5), (10, 10)]) + expect([.Message(5, 5), .Message(10, 10)]) + } } diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index ddef648dab..8a8726aa73 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -203,7 +203,7 @@ class MessageHistoryTableTests: XCTestCase { var unsentTable: MessageHistoryUnsentTable? var tagsTable: MessageHistoryTagsTable? var readStateTable: MessageHistoryReadStateTable? - var invalidatedReadStateTable: MessageHistoryInvalidatedReadStateTable? + var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? override class func setUp() { super.setUp() @@ -231,8 +231,8 @@ class MessageHistoryTableTests: XCTestCase { self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 10) - self.invalidatedReadStateTable = MessageHistoryInvalidatedReadStateTable(valueBox: self.valueBox!, tableId: 11) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, invalidatedReadStateTable: invalidatedReadStateTable) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, tableId: 11) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) self.peerTable = PeerTable(valueBox: self.valueBox!, tableId: 6) self.peerTable!.set(peer) } @@ -255,31 +255,36 @@ class MessageHistoryTableTests: XCTestCase { private func addMessage(id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags = [], _ tags: MessageTags = []) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] - self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } private func updateMessage(previousId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags, _ tags: MessageTags) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] - self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } private func addHole(id: Int32) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] - self.historyTable!.addHoles([MessageId(peerId: peerId, namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.addHoles([MessageId(peerId: peerId, namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } private func removeMessages(ids: [Int32]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] - self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32, String, [Media])], _ tagMask: MessageTags? = nil) { + private func fillHole(id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32, String, [Media])], _ tagMask: MessageTags? = nil) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] - self.historyTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations) + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } private func expectEntries(entries: [Entry], tagMask: MessageTags? = nil) { @@ -578,42 +583,42 @@ class MessageHistoryTableTests: XCTestCase { } func testFillHoleEmpty() { - fillHole(1, .Complete, []) + fillHole(1, HoleFill(complete: true, direction: .UpperToLower), []) expectEntries([]) } func testFillHoleComplete() { addHole(100) - fillHole(1, .Complete, [(100, 100, "m100", []), (200, 200, "m200", [])]) + fillHole(1, HoleFill(complete: true, direction: .UpperToLower), [(100, 100, "m100", []), (200, 200, "m200", [])]) expectEntries([.Message(100, 100, "m100", [], []), .Message(200, 200, "m200", [], [])]) } func testFillHoleUpperToLowerPartial() { addHole(100) - fillHole(1, .UpperToLower, [(100, 100, "m100", []), (200, 200, "m200", [])]) + fillHole(1, HoleFill(complete: false, direction: .UpperToLower), [(100, 100, "m100", []), (200, 200, "m200", [])]) expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Message(200, 200, "m200", [], [])]) } func testFillHoleUpperToLowerToBounds() { addHole(100) - fillHole(1, .UpperToLower, [(1, 1, "m1", []), (200, 200, "m200", [])]) + fillHole(1, HoleFill(complete: false, direction: .UpperToLower), [(1, 1, "m1", []), (200, 200, "m200", [])]) expectEntries([.Message(1, 1, "m1", [], []), .Message(200, 200, "m200", [], [])]) } func testFillHoleLowerToUpperToBounds() { addHole(100) - fillHole(1, .LowerToUpper, [(100, 100, "m100", []), (Int32.max, 200, "m200", [])]) + fillHole(1, HoleFill(complete: false, direction: .LowerToUpper), [(100, 100, "m100", []), (Int32.max, 200, "m200", [])]) expectEntries([.Message(100, 100, "m100", [], []), .Message(Int32.max, 200, "m200", [], [])]) } func testFillHoleLowerToUpperPartial() { addHole(100) - fillHole(1, .LowerToUpper, [(100, 100, "m100", []), (200, 200, "m200", [])]) + fillHole(1, HoleFill(complete: false, direction: .LowerToUpper), [(100, 100, "m100", []), (200, 200, "m200", [])]) expectEntries([.Message(100, 100, "m100", [], []), .Message(200, 200, "m200", [], []), .Hole(201, Int32.max, Int32.max)]) } @@ -623,7 +628,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "m100") addMessage(200, 200, "m200") - fillHole(199, .UpperToLower, [(150, 150, "m150", [])]) + fillHole(199, HoleFill(complete: false, direction: .UpperToLower), [(150, 150, "m150", [])]) expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Hole(101, 149, 150), .Message(150, 150, "m150", [], []), .Message(200, 200, "m200", [], []), .Hole(201, Int32.max, Int32.max)]) } @@ -634,7 +639,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "m100") addMessage(200, 200, "m200") - fillHole(199, .LowerToUpper, [(150, 150, "m150", [])]) + fillHole(199, HoleFill(complete: false, direction: .LowerToUpper), [(150, 150, "m150", [])]) expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Message(150, 150, "m150", [], []), .Hole(151, 199, 200), .Message(200, 200, "m200", [], []), .Hole(201, Int32.max, Int32.max)]) } @@ -645,7 +650,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "m100") addMessage(200, 200, "m200") - fillHole(199, .Complete, [(150, 150, "m150", [])]) + fillHole(199, HoleFill(complete: true, direction: .UpperToLower), [(150, 150, "m150", [])]) expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], []), .Message(150, 150, "m150", [], []), .Message(200, 200, "m200", [], []), .Hole(201, Int32.max, Int32.max)]) } @@ -663,7 +668,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "m100") addHole(1) - fillHole(99, .Complete, []) + fillHole(99, HoleFill(complete: true, direction: .UpperToLower), []) expectEntries([.Message(100, 100, "m100", [], [])]) } @@ -672,7 +677,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "m100") addMessage(101, 101, "m101") - fillHole(100, .Complete, [(90, 90, "m90", [])]) + fillHole(100, HoleFill(complete: true, direction: .UpperToLower), [(90, 90, "m90", [])]) expectEntries([.Message(90, 90, "m90", [], []), .Message(100, 100, "m100", [], []), .Message(101, 101, "m101", [], [])]) } @@ -682,7 +687,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200") addHole(150) - fillHole(199, .UpperToLower, [(150, 150, "m150", []), (300, 300, "m300", [])]) + fillHole(199, HoleFill(complete: false, direction: .UpperToLower), [(150, 150, "m150", []), (300, 300, "m300", [])]) expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 149, 150), .Message(150, 150, "m150", [], []), .Message(200, 200, "m200", [], []), .Message(300, 300, "m300", [], [])]) } @@ -874,7 +879,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200", [], [], [.Second]) addHole(150) - fillHole(199, .UpperToLower, [(180, 180, "m180", [])]) + fillHole(199, HoleFill(complete: false, direction: .UpperToLower), [(180, 180, "m180", [])]) expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 179, 180)], tagMask: [.First]) expectEntries([.Hole(101, 179, 180), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) @@ -886,7 +891,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200", [], [], [.Second]) addHole(150) - fillHole(199, .LowerToUpper, [(180, 180, "m180", [])]) + fillHole(199, HoleFill(complete: false, direction: .LowerToUpper), [(180, 180, "m180", [])]) expectEntries([.Message(100, 100, "m100", [], []), .Hole(181, 199, 200)], tagMask: [.First]) expectEntries([.Hole(181, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) @@ -898,7 +903,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200", [], [], [.Second]) addHole(150) - fillHole(199, .Complete, [(180, 180, "m180", [])]) + fillHole(199, HoleFill(complete: true, direction: .UpperToLower), [(180, 180, "m180", [])]) expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) expectEntries([.Message(200, 200, "m200", [], [])], tagMask: [.Second]) @@ -910,7 +915,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200", [], [], [.Second]) addHole(150) - fillHole(199, .UpperToLower, [(180, 180, "m180", [])], [.First]) + fillHole(199, HoleFill(complete: false, direction: .UpperToLower), [(180, 180, "m180", [])], [.First]) expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 179, 180)], tagMask: [.First]) expectEntries([.Hole(101, 179, 180), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) @@ -922,7 +927,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200", [], [], [.Second]) addHole(150) - fillHole(199, .LowerToUpper, [(180, 180, "m180", [])], [.First]) + fillHole(199, HoleFill(complete: false, direction: .LowerToUpper), [(180, 180, "m180", [])], [.First]) expectEntries([.Message(100, 100, "m100", [], []), .Hole(181, 199, 200)], tagMask: [.First]) expectEntries([.Hole(101, 179, 180), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) @@ -934,7 +939,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200", [], [], [.Second]) addHole(150) - fillHole(199, .Complete, [(180, 180, "m180", [])], [.First]) + fillHole(199, HoleFill(complete: true, direction: .UpperToLower), [(180, 180, "m180", [])], [.First]) expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) expectEntries([.Hole(101, 179, 180), .Hole(181, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) @@ -946,7 +951,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200", [], [], [.Second]) addHole(150) - fillHole(199, .UpperToLower, [], [.First]) + fillHole(199, HoleFill(complete: false, direction: .UpperToLower), [], [.First]) expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) expectEntries([.Hole(101, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) @@ -958,7 +963,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200", [], [], [.Second]) addHole(150) - fillHole(199, .LowerToUpper, [], [.First]) + fillHole(199, HoleFill(complete: false, direction: .LowerToUpper), [], [.First]) expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) expectEntries([.Hole(101, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) @@ -970,7 +975,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(200, 200, "m200", [], [], [.Second]) addHole(150) - fillHole(199, .Complete, [], [.First]) + fillHole(199, HoleFill(complete: true, direction: .UpperToLower), [], [.First]) expectEntries([.Message(100, 100, "m100", [], [])], tagMask: [.First]) expectEntries([.Hole(101, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) diff --git a/PostboxTests/ReadStateTableTests.swift b/PostboxTests/ReadStateTableTests.swift new file mode 100644 index 0000000000..bd7e2e1688 --- /dev/null +++ b/PostboxTests/ReadStateTableTests.swift @@ -0,0 +1,244 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +private let peerId = PeerId(namespace: 1, id: 1) +private let namespace: Int32 = 1 +private let authorPeerId = PeerId(namespace: 1, id: 6) + +private func ==(lhs: [Media], rhs: [Media]) -> Bool { + if lhs.count != rhs.count { + return false + } + + for i in 0 ..< lhs.count { + if !lhs[i].isEqual(rhs[i]) { + return false + } + } + return true +} + +private enum Entry: Equatable, CustomStringConvertible { + case Message(Int32, Int32, String, [Media], MessageFlags) + case Hole(Int32, Int32, Int32) + + var description: String { + switch self { + case let .Message(id, timestamp, text, media, flags): + return "Message(\(id), \(timestamp), \(text), \(media), \(flags))" + case let .Hole(min, max, timestamp): + return "Hole(\(min), \(max), \(timestamp))" + } + } +} + +private func ==(lhs: Entry, rhs: Entry) -> Bool { + switch lhs { + case let .Message(lhsId, lhsTimestamp, lhsText, lhsMedia, lhsFlags): + switch rhs { + case let .Message(rhsId, rhsTimestamp, rhsText, rhsMedia, rhsFlags): + return lhsId == rhsId && lhsTimestamp == rhsTimestamp && lhsText == rhsText && lhsMedia == rhsMedia && lhsFlags == rhsFlags + case .Hole: + return false + } + case let .Hole(lhsMin, lhsMax, lhsMaxTimestamp): + switch rhs { + case .Message: + return false + case let .Hole(rhsMin, rhsMax, rhsMaxTimestamp): + return lhsMin == rhsMin && lhsMax == rhsMax && lhsMaxTimestamp == rhsMaxTimestamp + } + } +} + +private extension MessageTags { + static let First = MessageTags(rawValue: 1 << 0) + static let Second = MessageTags(rawValue: 1 << 1) +} + +class ReadStateTableTests: XCTestCase { + var valueBox: ValueBox? + var path: String? + + var peerTable: PeerTable? + var globalMessageIdsTable: GlobalMessageIdsTable? + var indexTable: MessageHistoryIndexTable? + var mediaTable: MessageMediaTable? + var mediaCleanupTable: MediaCleanupTable? + var historyTable: MessageHistoryTable? + var historyMetadataTable: MessageHistoryMetadataTable? + var unsentTable: MessageHistoryUnsentTable? + var tagsTable: MessageHistoryTagsTable? + var readStateTable: MessageHistoryReadStateTable? + var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? + + override func setUp() { + super.setUp() + + var randomId: Int64 = 0 + arc4random_buf(&randomId, 8) + path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + self.valueBox = SqliteValueBox(basePath: path!) + + let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) + + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 5, namespace: namespace) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 7) + self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, tableId: 8) + self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, tableId: 9) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) + self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 10) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, tableId: 11) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + } + + override func tearDown() { + super.tearDown() + + self.historyTable = nil + self.indexTable = nil + self.mediaTable = nil + self.mediaCleanupTable = nil + self.peerTable = nil + self.historyMetadataTable = nil + + self.valueBox = nil + let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + self.path = nil + } + + private func addMessage(id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags = [], _ tags: MessageTags = []) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + + private func updateMessage(previousId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags, _ tags: MessageTags) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + + private func addHole(id: Int32) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.addHoles([MessageId(peerId: peerId, namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + + private func removeMessages(ids: [Int32]) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + + private func expectApplyRead(messageId: Int32, _ expectInvalidate: Bool) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.applyIncomingReadMaxId(MessageId(peerId: peerId, namespace: namespace, id: messageId), operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + + let invalidated = updatedPeerReadStateOperations.count != 0 + if expectInvalidate != invalidated { + XCTFail("applyRead: invalidated expected \(expectInvalidate), actual: \(invalidated)") + } + } + + private func expectReadState(maxReadId: Int32, _ maxKnownId: Int32, _ count: Int32) { + if let state = self.readStateTable!.getCombinedState(peerId)?.states.first?.1 { + if state.maxReadId != maxReadId || state.maxKnownId != maxKnownId || state.count != count { + XCTFail("Expected\nmaxReadId: \(maxReadId), maxKnownId: \(maxKnownId), count: \(count)\nActual\nmaxReadId: \(state.maxReadId), maxKnownId: \(state.maxKnownId), count: \(state.count)") + } + } else { + XCTFail("Expected\nmaxReadId: (maxReadId), maxKnownId: \(maxKnownId), count: \(count)\nActual\nnil") + } + } + + func testResetState() { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + + expectReadState(100, 120, 130) + } + + func testAddIncomingBeforeKnown() { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + + self.addMessage(99, 99, "", [], [.Incoming]) + + expectReadState(100, 120, 130) + } + + func testAddIncomingAfterKnown() { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + + self.addMessage(130, 130, "", [], [.Incoming]) + + expectReadState(100, 120, 131) + } + + func testApplyReadThenAddIncoming() { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + + self.expectApplyRead(200, false) + + self.addMessage(130, 130, "", [], [.Incoming]) + + expectReadState(200, 200, 0) + } + + func testApplyAddIncomingThenRead() { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + + self.addMessage(130, 130, "", [], [.Incoming]) + + expectReadState(100, 100, 1) + + self.expectApplyRead(200, false) + + expectReadState(200, 200, 0) + } + + func testIgnoreOldRead() { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + + self.expectApplyRead(90, false) + + expectReadState(100, 100, 0) + } + + func testInvalidateReadHole() { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + + self.addMessage(200, 200) + self.addHole(1) + + self.expectApplyRead(200, true) + + expectReadState(200, 200, 0) + } + + +} From f37188879195af0e0359760b4ef080dd8149b27a Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 14 Jun 2016 18:12:22 +0300 Subject: [PATCH 032/427] no message --- Postbox/Coding.swift | 43 ++++++++++ Postbox/MessageHistoryIndexTable.swift | 12 +++ Postbox/MessageHistoryReadStateTable.swift | 49 +++++++---- Postbox/MessageHistoryTable.swift | 96 +++++++++------------- Postbox/MessageHistoryView.swift | 2 +- Postbox/PeerReadState.swift | 12 +-- Postbox/Postbox.swift | 14 +++- 7 files changed, 147 insertions(+), 81 deletions(-) diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 1a2b69c812..97285394f7 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -705,6 +705,49 @@ public final class Decoder { } } + public func decodeObjectArrayWithDecoderForKey(key: StaticString) -> [T] { + if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var array: [T] = [] + array.reserveCapacity(Int(length)) + + var failed = false + var i: Int32 = 0 + while i < length { + var typeHash: Int32 = 0 + memcpy(&typeHash, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var objectLength: Int32 = 0 + memcpy(&objectLength, self.buffer.memory + self.offset, 4) + + let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(objectLength), freeWhenDone: false)) + self.offset += 4 + Int(objectLength) + + if !failed { + if let object = T(decoder: innerDecoder) as? T { + array.append(object) + } else { + failed = true + } + } + + i += 1 + } + + if failed { + return [] + } else { + return array + } + } else { + return [] + } + } + public func decodeObjectArrayForKey(key: StaticString) -> [T] { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 9b4f704072..7c6a2b6a41 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -642,6 +642,18 @@ final class MessageHistoryIndexTable: Table { return true }, limit: 0) + self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: maxId)), end: self.upperBound(peerId, namespace: namespace), values: { key, value in + var flags: Int8 = 0 + value.read(&flags, offset: 0, length: 1) + if (flags & HistoryEntryTypeMask) == HistoryEntryTypeHole { + value.reset() + if case let .Hole(hole) = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value) where hole.min <= maxId && hole.maxIndex.id.id >= maxId { + holes = true + } + } + return false + }, limit: 1) + return (count, holes) } diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index f0fb8772fd..648bd13d75 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -40,14 +40,17 @@ final class MessageHistoryReadStateTable: Table { var stateByNamespace: [MessageId.Namespace: PeerReadState] = [:] for _ in 0 ..< count { var namespaceId: Int32 = 0 - var maxReadId: Int32 = 0 + var maxIncomingReadId: Int32 = 0 + var maxOutgoingReadId: Int32 = 0 var maxKnownId: Int32 = 0 var count: Int32 = 0 value.read(&namespaceId, offset: 0, length: 4) - value.read(&maxReadId, offset: 0, length: 4) + value.read(&maxIncomingReadId, offset: 0, length: 4) + value.read(&maxOutgoingReadId, offset: 0, length: 4) value.read(&maxKnownId, offset: 0, length: 4) value.read(&count, offset: 0, length: 4) - let state = PeerReadState(maxReadId: maxReadId, maxKnownId: maxKnownId, count: count) + + let state = PeerReadState(maxIncomingReadId: maxIncomingReadId, maxOutgoingReadId: maxOutgoingReadId, maxKnownId: maxKnownId, count: count) stateByNamespace[namespaceId] = state } let states = InternalPeerReadStates(namespaces: stateByNamespace) @@ -96,7 +99,7 @@ final class MessageHistoryReadStateTable: Table { } } - func addIncomingMessages(peerId: PeerId, ids: [MessageId]) -> (CombinedPeerReadState?, Bool) { + func addIncomingMessages(peerId: PeerId, ids: Set) -> (CombinedPeerReadState?, Bool) { var idsByNamespace: [MessageId.Namespace: [MessageId.Id]] = [:] for id in ids { if idsByNamespace[id.namespace] != nil { @@ -118,14 +121,14 @@ final class MessageHistoryReadStateTable: Table { var addedUnreadCount: Int32 = 0 var maxIncomingId: Int32 = 0 for id in ids { - if id > currentState.maxKnownId { + if id > currentState.maxKnownId && id > currentState.maxIncomingReadId { addedUnreadCount += 1 maxIncomingId = max(id, maxIncomingId) } } if addedUnreadCount != 0 { - states.namespaces[namespace] = PeerReadState(maxReadId: currentState.maxReadId, maxKnownId: currentState.maxKnownId, count: currentState.count + addedUnreadCount) + states.namespaces[namespace] = PeerReadState(maxIncomingReadId: currentState.maxIncomingReadId, maxOutgoingReadId: currentState.maxOutgoingReadId, maxKnownId: currentState.maxKnownId, count: currentState.count + addedUnreadCount) updated = true if traceReadStates { @@ -171,7 +174,7 @@ final class MessageHistoryReadStateTable: Table { if let currentState = states.namespaces[namespace] { var unreadIds: [MessageId.Id] = [] for id in ids { - if id > currentState.maxReadId { + if id > currentState.maxIncomingReadId { unreadIds.append(id) } } @@ -181,7 +184,7 @@ final class MessageHistoryReadStateTable: Table { invalidate = true } - states.namespaces[namespace] = PeerReadState(maxReadId: currentState.maxReadId, maxKnownId: currentState.maxKnownId, count: currentState.count - knownCount) + states.namespaces[namespace] = PeerReadState(maxIncomingReadId: currentState.maxIncomingReadId, maxOutgoingReadId: currentState.maxOutgoingReadId, maxKnownId: currentState.maxKnownId, count: currentState.count - knownCount) updated = true } else { invalidate = true @@ -200,14 +203,14 @@ final class MessageHistoryReadStateTable: Table { return (nil, false) } - func applyMaxReadId(messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (CombinedPeerReadState?, Bool) { + func applyIncomingMaxReadId(messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (CombinedPeerReadState?, Bool) { if let states = self.get(messageId.peerId), state = states.namespaces[messageId.namespace] { if traceReadStates { print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId.id) (before: \(states.namespaces))") } - if state.maxReadId < messageId.id || messageId.id == topMessageId { - var (deltaCount, holes) = incomingStatsInRange(state.maxReadId + 1, messageId.id) + if state.maxIncomingReadId < messageId.id || messageId.id == topMessageId { + var (deltaCount, holes) = incomingStatsInRange(state.maxIncomingReadId + 1, messageId.id) if traceReadStates { print("[ReadStateTable] applyMaxReadId after deltaCount: \(deltaCount), holes: \(holes)") @@ -220,7 +223,7 @@ final class MessageHistoryReadStateTable: Table { } } - states.namespaces[messageId.namespace] = PeerReadState(maxReadId: messageId.id, maxKnownId: state.maxKnownId, count: state.count - Int32(deltaCount)) + states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: messageId.id, maxOutgoingReadId: state.maxOutgoingReadId, maxKnownId: state.maxKnownId, count: state.count - Int32(deltaCount)) self.updatedPeerIds.insert(messageId.peerId) return (CombinedPeerReadState(states: states.namespaces.map({$0})), holes) } @@ -231,8 +234,22 @@ final class MessageHistoryReadStateTable: Table { return (nil, false) } + func applyOutgoingMaxReadId(messageId: MessageId) -> (CombinedPeerReadState?, Bool) { + if let states = self.get(messageId.peerId), state = states.namespaces[messageId.namespace] { + if state.maxOutgoingReadId < messageId.id { + states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: state.maxIncomingReadId, maxOutgoingReadId: state.maxOutgoingReadId, maxKnownId: state.maxKnownId, count: state.count) + self.updatedPeerIds.insert(messageId.peerId) + return (CombinedPeerReadState(states: states.namespaces.map({$0})), false) + } + } else { + return (nil, true) + } + + return (nil, false) + } + func applyInteractiveMaxReadId(messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult) { - let (combinedState, holes) = self.applyMaxReadId(messageId, incomingStatsInRange: incomingStatsInRange, topMessageId: topMessageId) + let (combinedState, holes) = self.applyIncomingMaxReadId(messageId, incomingStatsInRange: incomingStatsInRange, topMessageId: topMessageId) if let combinedState = combinedState { return (combinedState, .Push(thenSync: holes)) @@ -250,11 +267,13 @@ final class MessageHistoryReadStateTable: Table { sharedBuffer.write(&count, offset: 0, length: 4) for (namespace, state) in states.namespaces { var namespaceId: Int32 = namespace - var maxReadId: Int32 = state.maxReadId + var maxIncomingReadId: Int32 = state.maxIncomingReadId + var maxOutgoingReadId: Int32 = state.maxOutgoingReadId var maxKnownId: Int32 = state.maxKnownId var count: Int32 = state.count sharedBuffer.write(&namespaceId, offset: 0, length: 4) - sharedBuffer.write(&maxReadId, offset: 0, length: 4) + sharedBuffer.write(&maxIncomingReadId, offset: 0, length: 4) + sharedBuffer.write(&maxOutgoingReadId, offset: 0, length: 4) sharedBuffer.write(&maxKnownId, offset: 0, length: 4) sharedBuffer.write(&count, offset: 0, length: 4) } diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index d83f01ef6c..38fdf2891c 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -118,7 +118,7 @@ final class MessageHistoryTable: Table { var outputOperations: [MessageHistoryOperation] = [] var accumulatedRemoveIndices: [MessageIndex] = [] - var addedIncomingMessageIds: [MessageId] = [] + var addedIncomingMessageIds = Set() var removedMessageIds: [MessageId] = [] for operation in operations { switch operation { @@ -167,9 +167,10 @@ final class MessageHistoryTable: Table { } } if message.flags.contains(.Incoming) { - addedIncomingMessageIds.append(message.id) + addedIncomingMessageIds.insert(message.id) } case let .Remove(index): + addedIncomingMessageIds.remove(index.id) self.justRemove(index, unsentMessageOperations: &unsentMessageOperations) accumulatedRemoveIndices.append(index) case let .Update(index, storeMessage): @@ -298,7 +299,7 @@ final class MessageHistoryTable: Table { topMessageId = index.id.id } - let (combinedState, invalidated) = self.readStateTable.applyMaxReadId(messageId, incomingStatsInRange: { fromId, toId in + let (combinedState, invalidated) = self.readStateTable.applyIncomingMaxReadId(messageId, incomingStatsInRange: { fromId, toId in return self.messageHistoryIndexTable.incomingMessageCountInRange(messageId.peerId, namespace: messageId.namespace, minId: fromId, maxId: toId) }, topMessageId: topMessageId) @@ -315,6 +316,22 @@ final class MessageHistoryTable: Table { } } + func applyOutgoingReadMaxId(messageId: MessageId, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + let (combinedState, invalidated) = self.readStateTable.applyOutgoingMaxReadId(messageId) + + if let combinedState = combinedState { + if operationsByPeerId[messageId.peerId] == nil { + operationsByPeerId[messageId.peerId] = [.UpdateReadState(combinedState)] + } else { + operationsByPeerId[messageId.peerId]!.append(.UpdateReadState(combinedState)) + } + } + + if invalidated { + self.synchronizeReadStateTable.set(messageId.peerId, operation: .Validate, operations: &updatedPeerReadStateOperations) + } + } + func applyInteractiveMaxReadId(messageId: MessageId, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { var topMessageId: MessageId.Id? if let topEntry = self.messageHistoryIndexTable.top(messageId.peerId, namespace: messageId.namespace), case let .Message(index) = topEntry { @@ -574,6 +591,18 @@ final class MessageHistoryTable: Table { } } + private func updateMedia(from from: [Media], to: [Media]) { + if from.count != 0 { + assertionFailure() + } + + for media in from { + if let id = media.id { + + } + } + } + private func justUpdate(index: MessageIndex, message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder, inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) -> IntermediateMessage? { if let previousMessage = self.getMessage(index) { self.valueBox.remove(self.tableId, key: self.key(index)) @@ -596,11 +625,10 @@ final class MessageHistoryTable: Table { assertionFailure() } - let previousEmbeddedMediaData = previousMessage.embeddedMediaData - var previousMedia: [Media] = [] - if previousEmbeddedMediaData.length > 4 { + if previousMessage.embeddedMediaData.length > 4 { var embeddedMediaCount: Int32 = 0 + let previousEmbeddedMediaData = previousMessage.embeddedMediaData previousEmbeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) for _ in 0 ..< embeddedMediaCount { var mediaLength: Int32 = 0 @@ -612,65 +640,16 @@ final class MessageHistoryTable: Table { } } + var previousReferencedMedia: [Media] = [] for mediaId in previousMessage.referencedMedia { if let media = self.messageMediaTable.get(mediaId) { previousMedia.append(media) } } - let updatedMedia = message.media - var removedMediaIds: [MediaId] = [] - var updatedEmbeddedMedia: [Media] = [] - var updatedReferencedMediaIds: [MediaId] = [] - - for media in updatedMedia { - assertionFailure() - if let mediaId = media.id { - var found = false - for previous in previousMedia { - if let previousId = previous.id { - if previousId == mediaId { - found = true - break - } - } - } - if !found { - let result = self.messageMediaTable.set(media, index: MessageIndex(message), messageHistoryTable: self) - switch result { - case let .Embed(embedded): - updatedEmbeddedMedia.append(embedded) - case .Reference: - updatedReferencedMediaIds.append(mediaId) - } - } else { - - } - } else { - updatedEmbeddedMedia.append(media) - } - } - for media in previousMedia { - assertionFailure() - if let mediaId = media.id { - var found = false - for updated in updatedMedia { - if let updatedId = updated.id { - if updatedId == mediaId { - found = true - break - } - } - } - if !found { - assertionFailure() - } - } else { - self.messageMediaTable.removeEmbeddedMedia(media) - } - } + self.updateMedia(from: previousMedia, to: message.media) sharedBuffer.reset() @@ -1070,6 +1049,7 @@ final class MessageHistoryTable: Table { var forwardInfo: MessageForwardInfo? if let internalForwardInfo = message.forwardInfo, forwardAuthor = peerTable.get(internalForwardInfo.authorId) { var source: Peer? + if let sourceId = internalForwardInfo.sourceId { source = peerTable.get(sourceId) } @@ -1260,7 +1240,7 @@ final class MessageHistoryTable: Table { func maxReadIndex(peerId: PeerId) -> MessageHistoryAnchorIndex? { if let combinedState = self.readStateTable.getCombinedState(peerId), state = combinedState.states.first where state.1.count != 0 { - return self.anchorIndex(MessageId(peerId: peerId, namespace: state.0, id: state.1.maxReadId)) + return self.anchorIndex(MessageId(peerId: peerId, namespace: state.0, id: state.1.maxIncomingReadId)) } return nil } diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index bf7b27b039..a6403034f7 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -470,7 +470,7 @@ public final class MessageHistoryView { var maxNamespaceIndex: MessageIndex? var index = entries.count - 1 for entry in entries.reverse() { - if entry.index.id.namespace == namespace && entry.index.id.id <= state.maxReadId { + if entry.index.id.namespace == namespace && entry.index.id.id <= state.maxIncomingReadId { maxNamespaceIndex = entry.index break } diff --git a/Postbox/PeerReadState.swift b/Postbox/PeerReadState.swift index ab58e09fef..ee3d20ba52 100644 --- a/Postbox/PeerReadState.swift +++ b/Postbox/PeerReadState.swift @@ -1,22 +1,24 @@ public struct PeerReadState: Equatable, CustomStringConvertible { - public let maxReadId: MessageId.Id + public let maxIncomingReadId: MessageId.Id + public let maxOutgoingReadId: MessageId.Id public let maxKnownId: MessageId.Id public let count: Int32 - public init(maxReadId: MessageId.Id, maxKnownId: MessageId.Id, count: Int32) { - self.maxReadId = maxReadId + public init(maxIncomingReadId: MessageId.Id, maxOutgoingReadId: MessageId.Id, maxKnownId: MessageId.Id, count: Int32) { + self.maxIncomingReadId = maxIncomingReadId + self.maxOutgoingReadId = maxOutgoingReadId self.maxKnownId = maxKnownId self.count = count } public var description: String { - return "(PeerReadState maxReadId: \(maxReadId), maxKnownId: \(maxKnownId), count: \(count))" + return "(PeerReadState maxIncomingReadId: \(maxIncomingReadId), maxOutgoingReadId: \(maxOutgoingReadId) maxKnownId: \(maxKnownId), count: \(count)" } } public func ==(lhs: PeerReadState, rhs: PeerReadState) -> Bool { - return lhs.maxReadId == rhs.maxReadId && lhs.maxKnownId == rhs.maxKnownId && lhs.count == rhs.count + return lhs.maxIncomingReadId == rhs.maxIncomingReadId && lhs.maxOutgoingReadId == rhs.maxOutgoingReadId && lhs.maxKnownId == rhs.maxKnownId && lhs.count == rhs.count } public struct CombinedPeerReadState { diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index aa7d968adc..2e9b6b1536 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -57,6 +57,10 @@ public final class Modifier { self.postbox?.applyIncomingReadMaxId(messageId) } + public func applyOutgoingReadMaxId(messageId: MessageId) { + self.postbox?.applyOutgoingReadMaxId(messageId) + } + public func applyInteractiveReadMaxId(messageId: MessageId) { self.postbox?.applyInteractiveReadMaxId(messageId) } @@ -221,16 +225,18 @@ public final class Postbox { //self.debugRestoreState("beforeHoles") // debugging unread counters - //self.debugSaveState("afterLogin") //self.debugRestoreState("afterLogin") + //self.debugSaveState("previous") + //self.debugRestoreState("previous") + //#endif self.valueBox = SqliteValueBox(basePath: self.basePath + "/db") self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 3 + let currentUserVersion: Int32 = 5 if userVersion != currentUserVersion { self.valueBox.drop() @@ -398,6 +404,10 @@ public final class Postbox { self.messageHistoryTable.applyIncomingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } + private func applyOutgoingReadMaxId(messageId: MessageId) { + self.messageHistoryTable.applyOutgoingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + } + private func applyInteractiveReadMaxId(messageId: MessageId) { self.messageHistoryTable.applyInteractiveMaxReadId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } From c970aef27ede42a0dd594183fc36a6a7b4224801 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 17 Jun 2016 00:59:53 +0300 Subject: [PATCH 033/427] no message --- Postbox/MessageHistoryTable.swift | 139 +++++++++++++++++++++++++++--- Postbox/MessageHistoryView.swift | 64 +++++++++++++- Postbox/MessageMediaTable.swift | 88 ++++++++++++++++++- Postbox/Postbox.swift | 64 ++++++-------- Postbox/ViewTracker.swift | 7 +- 5 files changed, 305 insertions(+), 57 deletions(-) diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 38fdf2891c..9b0809116d 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -5,6 +5,7 @@ enum MessageHistoryOperation { case InsertHole(MessageHistoryHole) case Remove([MessageIndex]) case UpdateReadState(CombinedPeerReadState) + case UpdateEmbeddedMedia(MessageIndex, ReadBuffer) } struct MessageHistoryAnchorIndex { @@ -280,6 +281,32 @@ final class MessageHistoryTable: Table { self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } + func updateMedia(id: MediaId, media: Media?, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedMedia: [MediaId: Media?]) { + if let previousMedia = self.messageMediaTable.get(id, embedded: { index, id in + return self.embeddedMediaForIndex(index, id: id) + }) { + if let media = media { + if !previousMedia.isEqual(media) { + self.messageMediaTable.update(id, media: media, messageHistoryTable: self, operationsByPeerId: &operationsByPeerId) + updatedMedia[id] = media + } + } else { + updatedMedia[id] = nil + if case let .Embedded(index) = self.messageMediaTable.removeReference(id) { + self.updateEmbeddedMedia(index, operationsByPeerId: &operationsByPeerId, update: { previousMedia in + var updated: [Media] = [] + for previous in previousMedia { + if previous.id != id { + updated.append(previous) + } + } + return updated + }) + } + } + } + } + func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { for (peerId, namespaces) in states { if let combinedState = self.readStateTable.resetStates(peerId, namespaces: namespaces) { @@ -591,16 +618,102 @@ final class MessageHistoryTable: Table { } } - private func updateMedia(from from: [Media], to: [Media]) { - if from.count != 0 { - assertionFailure() - } - - for media in from { - if let id = media.id { - + func embeddedMediaForIndex(index: MessageIndex, id: MediaId) -> Media? { + if let message = self.getMessage(index) where message.embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + + for _ in 0 ..< embeddedMediaCount { + let mediaOffset = message.embeddedMediaData.offset + var mediaLength: Int32 = 0 + var copyMedia = true + message.embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let readMedia = Decoder(buffer: MemoryBuffer(memory: message.embeddedMediaData.memory + message.embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + + if let readMediaId = readMedia.id where readMediaId == id { + return readMedia + } + } + message.embeddedMediaData.skip(Int(mediaLength)) } } + + return nil + } + + func updateEmbeddedMedia(index: MessageIndex, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], @noescape update: ([Media]) -> [Media]) { + if let message = self.getMessage(index) { + var embeddedMediaCount: Int32 = 0 + message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + + var previousMedia: [Media] = [] + for _ in 0 ..< embeddedMediaCount { + let mediaOffset = message.embeddedMediaData.offset + var mediaLength: Int32 = 0 + var copyMedia = true + message.embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let readMedia = Decoder(buffer: MemoryBuffer(memory: message.embeddedMediaData.memory + message.embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + previousMedia.append(readMedia) + } + message.embeddedMediaData.skip(Int(mediaLength)) + } + + let updatedMedia = update(previousMedia) + var updated = false + if updatedMedia.count != previousMedia.count { + updated = true + } else { + outer: for i in 0 ..< previousMedia.count { + if !previousMedia[i].isEqual(updatedMedia[i]) { + updated = true + break outer + } + } + } + + if updated { + var updatedEmbeddedMediaCount: Int32 = Int32(updatedMedia.count) + + let updatedEmbeddedMediaBuffer = WriteBuffer() + updatedEmbeddedMediaBuffer.write(&updatedEmbeddedMediaCount, offset: 0, length: 4) + + let encoder = Encoder() + + for media in updatedMedia { + encoder.reset() + encoder.encodeRootObject(media) + let encodedBuffer = encoder.readBufferNoCopy() + var encodedLength: Int32 = Int32(encodedBuffer.length) + updatedEmbeddedMediaBuffer.write(&encodedLength, offset: 0, length: 4) + updatedEmbeddedMediaBuffer.write(encodedBuffer.memory, offset: 0, length: encodedBuffer.length) + } + + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: message.referencedMedia), sharedKey: self.key(index)) + + let operation: MessageHistoryOperation = .UpdateEmbeddedMedia(index, updatedEmbeddedMediaBuffer.makeReadBufferAndReset()) + if operationsByPeerId[index.id.peerId] == nil { + operationsByPeerId[index.id.peerId] = [operation] + } else { + operationsByPeerId[index.id.peerId]!.append(operation) + } + } + } + } + + func updateEmbeddedMedia(index: MessageIndex, mediaId: MediaId, media: Media?, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + self.updateEmbeddedMedia(index, operationsByPeerId: &operationsByPeerId, update: { previousMedia in + var updatedMedia: [Media] = [] + for previous in previousMedia { + if previous.id == mediaId { + if let media = media { + updatedMedia.append(media) + } + } else { + updatedMedia.append(previous) + } + } + return updatedMedia + }) } private func justUpdate(index: MessageIndex, message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder, inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) -> IntermediateMessage? { @@ -642,14 +755,16 @@ final class MessageHistoryTable: Table { var previousReferencedMedia: [Media] = [] for mediaId in previousMessage.referencedMedia { - if let media = self.messageMediaTable.get(mediaId) { + if let media = self.messageMediaTable.get(mediaId, embedded: { _ in + return nil + }) { previousMedia.append(media) } } var removedMediaIds: [MediaId] = [] - self.updateMedia(from: previousMedia, to: message.media) + //self.updateMedia(from: previousMedia, to: message.media) sharedBuffer.reset() @@ -1041,7 +1156,9 @@ final class MessageHistoryTable: Table { } for mediaId in message.referencedMedia { - if let media = self.messageMediaTable.get(mediaId) { + if let media = self.messageMediaTable.get(mediaId, embedded: { _ in + return nil + }) { parsedMedia.append(media) } } diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index a6403034f7..c2b8c41722 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -190,7 +190,7 @@ final class MutableMessageHistoryView { return false } - func replay(operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], context: MutableMessageHistoryViewReplayContext) -> Bool { + func replay(operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], context: MutableMessageHistoryViewReplayContext) -> Bool { let tagMask = self.tagMask let unwrappedTagMask: UInt32 = tagMask?.rawValue ?? 0 @@ -216,6 +216,68 @@ final class MutableMessageHistoryView { case let .UpdateReadState(combinedReadState): hasChanges = true //self.combinedReadState = combinedReadState + case let .UpdateEmbeddedMedia(index, embeddedMediaData): + for i in 0 ..< self.entries.count { + if case let .IntermediateMessageEntry(message) = self.entries[i] where MessageIndex(message) == index { + self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia)) + hasChanges = true + break + } + } + } + } + + if !updatedMedia.isEmpty { + for i in 0 ..< self.entries.count { + switch self.entries[i] { + case let .MessageEntry(message): + var rebuild = false + for media in message.media { + if let mediaId = media.id, updated = updatedMedia[mediaId] { + rebuild = true + break + } + } + + if rebuild { + var messageMedia: [Media] = [] + for media in message.media { + if let mediaId = media.id, updated = updatedMedia[mediaId] { + if let updated = updated { + messageMedia.append(updated) + } + } else { + messageMedia.append(media) + } + } + var updatedMessage = Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages) + self.entries[i] = .MessageEntry(updatedMessage) + hasChanges = true + } + case let .IntermediateMessageEntry(message): + var rebuild = false + for mediaId in message.referencedMedia { + if let media = updatedMedia[mediaId] where media?.id != mediaId { + rebuild = true + break + } + } + if rebuild { + var referencedMedia: [MediaId] = [] + for mediaId in message.referencedMedia { + if let media = updatedMedia[mediaId] where media?.id != mediaId { + if let id = media?.id { + referencedMedia.append(id) + } + } else { + referencedMedia.append(mediaId) + } + } + hasChanges = true + } + default: + break + } } } diff --git a/Postbox/MessageMediaTable.swift b/Postbox/MessageMediaTable.swift index 18b8973907..b9f4069b70 100644 --- a/Postbox/MessageMediaTable.swift +++ b/Postbox/MessageMediaTable.swift @@ -10,6 +10,11 @@ enum InsertMediaResult { case Embed(Media) } +enum RemoveMediaResult { + case Reference + case Embedded(MessageIndex) +} + enum DebugMediaEntry { case Direct(Media, Int) case MessageReference(MessageIndex) @@ -30,7 +35,7 @@ final class MessageMediaTable: Table { return key } - func get(id: MediaId) -> Media? { + func get(id: MediaId, @noescape embedded: (MessageIndex, MediaId) -> Media?) -> Media? { if let value = self.valueBox.get(self.tableId, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) @@ -40,6 +45,19 @@ final class MessageMediaTable: Table { if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + value.offset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media { return media } + } else if type == MediaEntryType.MessageReference.rawValue { + var idPeerId: Int64 = 0 + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idPeerId, offset: 0, length: 8) + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + + let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp) + + return embedded(referencedMessageIndex, id) } } return nil @@ -125,7 +143,7 @@ final class MessageMediaTable: Table { } } - func removeReference(id: MediaId, sharedWriteBuffer: WriteBuffer = WriteBuffer()) { + func removeReference(id: MediaId, sharedWriteBuffer: WriteBuffer = WriteBuffer()) -> RemoveMediaResult { if let value = self.valueBox.get(self.tableId, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) @@ -151,10 +169,28 @@ final class MessageMediaTable: Table { } else { self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) } + + return .Reference } else if type == MediaEntryType.MessageReference.rawValue { + var idPeerId: Int64 = 0 + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idPeerId, offset: 0, length: 8) + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + + let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp) + self.valueBox.remove(self.tableId, key: self.key(id)) + + return .Embedded(referencedMessageIndex) + } else { + assertionFailure() } } + return .Reference } func removeEmbeddedMedia(media: Media) { @@ -164,6 +200,54 @@ final class MessageMediaTable: Table { self.mediaCleanupTable.add(media) } + func update(id: MediaId, media: Media, messageHistoryTable: MessageHistoryTable, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) { + if let updatedId = media.id { + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == MediaEntryType.Direct.rawValue { + var dataLength: Int32 = 0 + value.read(&dataLength, offset: 0, length: 4) + let mediaOffset = value.offset + value.skip(Int(dataLength)) + + var messageReferenceCount: Int32 = 0 + value.read(&messageReferenceCount, offset: 0, length: 4) + + sharedWriteBuffer.reset() + var directType: Int8 = MediaEntryType.Direct.rawValue + sharedWriteBuffer.write(&directType, offset: 0, length: 1) + + sharedEncoder.reset() + sharedEncoder.encodeRootObject(media) + let mediaBuffer = sharedEncoder.memoryBuffer() + var mediaBufferLength = Int32(mediaBuffer.length) + sharedWriteBuffer.write(&mediaBufferLength, offset: 0, length: 4) + sharedWriteBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length) + + sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) + + if id != updatedId { + self.valueBox.remove(self.tableId, key: self.key(id)) + } + self.valueBox.set(self.tableId, key: self.key(updatedId), value: sharedWriteBuffer.readBufferNoCopy()) + } else if type == MediaEntryType.MessageReference.rawValue { + var idPeerId: Int64 = 0 + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idPeerId, offset: 0, length: 8) + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + + let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp) + messageHistoryTable.updateEmbeddedMedia(referencedMessageIndex, mediaId: id, media: media, operationsByPeerId: &operationsByPeerId) + } + } + } + } + func debugList() -> [DebugMediaEntry] { var entries: [DebugMediaEntry] = [] diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 2e9b6b1536..90c71c9f73 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -97,6 +97,23 @@ public final class Modifier { self.postbox?.updateMessage(index, update: update) } + public func updateMedia(id: MediaId, update: Media?) { + self.postbox?.updateMedia(id, update: update) + } + + public func getMessage(id: MessageId) -> Message? { + if let postbox = self.postbox { + if let entry = postbox.messageHistoryIndexTable.get(id) { + if case let .Message(index) = entry { + if let message = postbox.messageHistoryTable.getMessage(index) { + return postbox.renderIntermediateMessage(message) + } + } + } + } + return nil + } + public func filterStoredMessageIds(messageIds: Set) -> Set { if let postbox = self.postbox { return postbox.filterStoredMessageIds(messageIds) @@ -121,6 +138,7 @@ public final class Postbox { private var currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] private var currentUnsentOperations: [IntermediateMessageHistoryUnsentOperation] = [] private var currentUpdatedSynchronizeReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + private var currentUpdatedMedia: [MediaId: Media?] = [:] private var currentRemovedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] = [:] private var currentFilledHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] = [:] @@ -613,7 +631,7 @@ public final class Postbox { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } - self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations) + self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia) self.currentOperationsByPeerId.removeAll() self.currentFilledHolesByPeerId.removeAll() @@ -622,6 +640,7 @@ public final class Postbox { self.currentReplaceChatListHoles.removeAll() self.currentUnsentOperations.removeAll() self.currentUpdatedSynchronizeReadStateOperations.removeAll() + self.currentUpdatedMedia.removeAll() for table in self.tables { table.beforeCommit() @@ -658,6 +677,10 @@ public final class Postbox { } } + private func updateMedia(id: MediaId, update: Media?) { + self.messageHistoryTable.updateMedia(id, media: update, operationsByPeerId: &self.currentOperationsByPeerId, updatedMedia: &self.currentUpdatedMedia) + } + private func filterStoredMessageIds(messageIds: Set) -> Set { var filteredIds = Set() @@ -687,45 +710,6 @@ public final class Postbox { return EmptyDisposable } } - - public func preloadedAroundUnreadMessageHistoryViewForPeerId(peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return self.aroundUnreadMessageHistoryViewForPeerId(peerId, count: count, tagMask: tagMask) |> filter { view, _ in - if let maxReadIndex = view.maxReadIndex { - var targetIndex = 0 - for i in 0 ..< view.entries.count { - if view.entries[i].index >= maxReadIndex { - targetIndex = i - break - } - } - - /*print("entries:") - for i in 0 ..< view.entries.count { - switch view.entries[i] { - case let .MessageEntry(message): - print(" \(i == targetIndex ? "*" : " ")\(message.id.id): \(message.text)") - case let .HoleEntry(hole): - print(" \(i == targetIndex ? "*" : " ")\(hole.min) — \(hole.maxIndex.id.id): hole") - } - }*/ - - let maxIndex = min(view.entries.count, targetIndex + count / 2) - if maxIndex >= targetIndex { - for i in targetIndex ..< maxIndex { - if case .HoleEntry = view.entries[i] { - return false - } - } - } - - return true - } else { - return true - } - } |> take(1) |> mapToSignal { _ in - return self.aroundUnreadMessageHistoryViewForPeerId(peerId, count: count, tagMask: tagMask) - } - } public func aroundUnreadMessageHistoryViewForPeerId(peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return Signal { subscriber in diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 4733db9008..25f19275b9 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -138,12 +138,13 @@ final class ViewTracker { } } - func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?]) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { var updateHoles = false - if let operations = currentOperationsByPeerId[peerId] { + let operations = currentOperationsByPeerId[peerId] + if operations != nil || !updatedMedia.isEmpty { updateHoles = true for (mutableView, pipe) in bag.copyItems() { let context = MutableMessageHistoryViewReplayContext() @@ -156,7 +157,7 @@ final class ViewTracker { updateType = .Generic } - if mutableView.replay(operations, holeFillDirections: peerIdsWithFilledHoles[peerId] ?? [:], context: context) { + if mutableView.replay(operations ?? [], holeFillDirections: peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: updatedMedia, context: context) { mutableView.complete(context, fetchEarlier: { index, count in return self.fetchEarlierHistoryEntries(peerId, index, count, mutableView.tagMask) }, fetchLater: { index, count in From d354e968ecaf96afcd86e82633007ba135f42ddd Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 17 Jun 2016 20:22:09 +0300 Subject: [PATCH 034/427] no message --- Postbox/Coding.swift | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 97285394f7..234df16053 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -36,6 +36,14 @@ public class MemoryBuffer: Equatable, CustomStringConvertible { var capacity: Int var length: Int var freeWhenDone: Bool + + public init(copyOf buffer: MemoryBuffer) { + self.memory = malloc(buffer.length) + memcpy(self.memory, buffer.memory, buffer.length) + self.capacity = buffer.length + self.length = buffer.length + self.freeWhenDone = true + } public init(memory: UnsafeMutablePointer, capacity: Int, length: Int, freeWhenDone: Bool) { self.memory = memory @@ -377,6 +385,15 @@ public final class Encoder { self.buffer.write(&bytesLength, offset: 0, length: 4) self.buffer.write(bytes.memory, offset: 0, length: bytes.offset) } + + public func encodeBytes(bytes: MemoryBuffer, forKey key: StaticString) { + self.encodeKey(key) + var type: Int8 = ValueType.Bytes.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var bytesLength: Int32 = Int32(bytes.length) + self.buffer.write(&bytesLength, offset: 0, length: 4) + self.buffer.write(bytes.memory, offset: 0, length: bytes.length) + } public let sharedWriteBuffer = WriteBuffer() } From 82e22670c095539398e1b6cec6f8aa980506bbdc Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 19 Jun 2016 15:17:53 +0300 Subject: [PATCH 035/427] no message --- Postbox.xcodeproj/project.pbxproj | 7 +- .../xcschemes/PostboxTests.xcscheme | 2 +- .../xcschemes/xcschememanagement.plist | 2 +- Postbox/ChatListIndexTable.swift | 8 +- Postbox/ChatListTable.swift | 32 +-- Postbox/ChatListView.swift | 28 +-- Postbox/Coding.swift | 205 +++++++++--------- Postbox/DeferredString.swift | 124 ----------- Postbox/GlobalMessageIdsTable.swift | 10 +- Postbox/KeychainTable.swift | 10 +- Postbox/LmdbValueBox.swift | 46 ++-- Postbox/Media.swift | 8 +- Postbox/MediaBox.swift | 68 +++--- Postbox/MediaCleanupTable.swift | 4 +- Postbox/Message.swift | 12 +- Postbox/MessageHistoryIndexTable.swift | 48 ++-- ...sageHistoryInvalidatedReadStateTable.swift | 2 +- Postbox/MessageHistoryMetadataTable.swift | 16 +- Postbox/MessageHistoryReadStateTable.swift | 24 +- Postbox/MessageHistoryTable.swift | 121 +++++------ Postbox/MessageHistoryTagsTable.swift | 22 +- Postbox/MessageHistoryUnsentTable.swift | 8 +- Postbox/MessageHistoryView.swift | 37 ++-- Postbox/MessageMediaTable.swift | 15 +- Postbox/MetadataTable.swift | 8 +- Postbox/Peer.swift | 8 +- Postbox/PeerChatStateTable.swift | 6 +- Postbox/PeerTable.swift | 8 +- Postbox/Postbox.swift | 176 +++++++-------- Postbox/SimpleDictionary.swift | 10 +- Postbox/SimpleSet.swift | 6 +- Postbox/SqliteValueBox.swift | 100 ++++----- Postbox/SynchronizePeerReadStatesView.swift | 4 +- Postbox/UnsentMessageHistoryView.swift | 6 +- Postbox/ValueBox.swift | 12 +- Postbox/ValueBoxKey.swift | 28 +-- Postbox/ViewTracker.swift | 50 ++--- PostboxTests/ChatListTableTests.swift | 22 +- PostboxTests/CodingTests.swift | 8 +- .../MessageHistoryIndexTableTests.swift | 16 +- PostboxTests/MessageHistoryTableTests.swift | 34 +-- PostboxTests/ReadStateTableTests.swift | 42 ++-- submodules/SQLite.swift | 2 +- 43 files changed, 624 insertions(+), 781 deletions(-) delete mode 100644 Postbox/DeferredString.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index f21bbcfcce..173fcd650d 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -42,7 +42,6 @@ D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; - D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B76BE61B66639F0095CF45 /* DeferredString.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; @@ -120,7 +119,6 @@ D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; - D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; @@ -275,7 +273,6 @@ isa = PBXGroup; children = ( D0E3A7871B28AE9C00A402D9 /* Coding.swift */, - D0B76BE61B66639F0095CF45 /* DeferredString.swift */, D0F9E8741C5A334100037222 /* SimpleDictionary.swift */, D0C9DA381C65782500855278 /* SimpleSet.swift */, ); @@ -441,7 +438,7 @@ attributes = { LastSwiftMigration = 0700; LastSwiftUpdateCheck = 0700; - LastUpgradeCheck = 0710; + LastUpgradeCheck = 0800; ORGANIZATIONNAME = Telegram; TargetAttributes = { D0E3A7491B28A7E300A402D9 = { @@ -537,7 +534,6 @@ D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */, D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */, - D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */, D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, ); @@ -830,6 +826,7 @@ LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; }; name = Release; }; diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme index fdd99d4b63..4a99b781a6 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme @@ -1,6 +1,6 @@ PostboxTests.xcscheme orderHint - 12 + 3     SuppressBuildableAutocreation diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift index 6e6451d3a1..15a3539f46 100644 --- a/Postbox/ChatListIndexTable.swift +++ b/Postbox/ChatListIndexTable.swift @@ -5,13 +5,13 @@ final class ChatListIndexTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func key(peerId: PeerId) -> ValueBoxKey { + private func key(_ peerId: PeerId) -> ValueBoxKey { let key = ValueBoxKey(length: 8) key.setInt64(0, value: peerId.toInt64()) return key } - func set(index: MessageIndex) { + func set(_ index: MessageIndex) { let writeBuffer = WriteBuffer() var idNamespace: Int32 = index.id.namespace var idId: Int32 = index.id.id @@ -22,11 +22,11 @@ final class ChatListIndexTable: Table { self.valueBox.set(self.tableId, key: self.key(index.id.peerId), value: writeBuffer.readBufferNoCopy()) } - func remove(peerId: PeerId) { + func remove(_ peerId: PeerId) { self.valueBox.remove(self.tableId, key: self.key(peerId)) } - func get(peerId: PeerId) -> MessageIndex? { + func get(_ peerId: PeerId) -> MessageIndex? { if let value = self.valueBox.get(self.tableId, key: self.key(peerId)) { var idNamespace: Int32 = 0 var idId: Int32 = 0 diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index 578a2d6079..54692912a6 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -32,7 +32,7 @@ private enum ChatListEntryType: Int8 { final class ChatListTable: Table { let indexTable: ChatListIndexTable - let emptyMemoryBuffer = MemoryBuffer(memory: nil, capacity: 0, length: 0, freeWhenDone: false) + let emptyMemoryBuffer = MemoryBuffer() let metadataTable: MessageHistoryMetadataTable let seedConfiguration: SeedConfiguration @@ -44,7 +44,7 @@ final class ChatListTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func key(index: MessageIndex, type: ChatListEntryType) -> ValueBoxKey { + private func key(_ index: MessageIndex, type: ChatListEntryType) -> ValueBoxKey { let key = ValueBoxKey(length: 4 + 4 + 4 + 8 + 1) key.setInt32(0, value: index.timestamp) key.setInt32(4, value: index.id.namespace) @@ -75,7 +75,7 @@ final class ChatListTable: Table { } } - func replay(historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]], messageHistoryTable: MessageHistoryTable, inout operations: [ChatListOperation]) { + func replay(_ historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]], messageHistoryTable: MessageHistoryTable, operations: inout [ChatListOperation]) { self.ensureInitialized() for (peerId, _) in historyOperationsByPeerId { @@ -101,7 +101,7 @@ final class ChatListTable: Table { } } - func addHole(hole: ChatListHole, inout operations: [ChatListOperation]) { + func addHole(_ hole: ChatListHole, operations: inout [ChatListOperation]) { self.ensureInitialized() if self.valueBox.get(self.tableId, key: self.key(hole.index, type: .Hole)) == nil { @@ -110,7 +110,7 @@ final class ChatListTable: Table { } } - func replaceHole(index: MessageIndex, hole: ChatListHole?, inout operations: [ChatListOperation]) { + func replaceHole(_ index: MessageIndex, hole: ChatListHole?, operations: inout [ChatListOperation]) { self.ensureInitialized() if self.valueBox.get(self.tableId, key: self.key(index, type: .Hole)) != nil { @@ -128,23 +128,23 @@ final class ChatListTable: Table { } } - private func justInsertMessage(index: MessageIndex) { + private func justInsertMessage(_ index: MessageIndex) { self.valueBox.set(self.tableId, key: self.key(index, type: .Message), value: self.emptyMemoryBuffer) } - private func justRemoveMessage(index: MessageIndex) { + private func justRemoveMessage(_ index: MessageIndex) { self.valueBox.remove(self.tableId, key: self.key(index, type: .Message)) } - private func justInsertHole(hole: ChatListHole) { + private func justInsertHole(_ hole: ChatListHole) { self.valueBox.set(self.tableId, key: self.key(hole.index, type: .Hole), value: self.emptyMemoryBuffer) } - private func justRemoveHole(index: MessageIndex) { + private func justRemoveHole(_ index: MessageIndex) { self.valueBox.remove(self.tableId, key: self.key(index, type: .Hole)) } - func entriesAround(index: MessageIndex, messageHistoryTable: MessageHistoryTable, count: Int) -> (entries: [ChatListIntermediateEntry], lower: ChatListIntermediateEntry?, upper: ChatListIntermediateEntry?) { + func entriesAround(_ index: MessageIndex, messageHistoryTable: MessageHistoryTable, count: Int) -> (entries: [ChatListIntermediateEntry], lower: ChatListIntermediateEntry?, upper: ChatListIntermediateEntry?) { self.ensureInitialized() var lowerEntries: [ChatListIntermediateEntry] = [] @@ -217,16 +217,16 @@ final class ChatListTable: Table { lower = additionalLowerEntries.last additionalLowerEntries.removeLast() } - lowerEntries.appendContentsOf(additionalLowerEntries) + lowerEntries.append(contentsOf: additionalLowerEntries) } var entries: [ChatListIntermediateEntry] = [] - entries.appendContentsOf(lowerEntries.reverse()) - entries.appendContentsOf(upperEntries) + entries.append(contentsOf: lowerEntries.reversed()) + entries.append(contentsOf: upperEntries) return (entries: entries, lower: lower, upper: upper) } - func earlierEntries(index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + func earlierEntries(_ index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { self.ensureInitialized() var entries: [ChatListIntermediateEntry] = [] @@ -254,7 +254,7 @@ final class ChatListTable: Table { return entries } - func laterEntries(index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + func laterEntries(_ index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { self.ensureInitialized() var entries: [ChatListIntermediateEntry] = [] @@ -282,7 +282,7 @@ final class ChatListTable: Table { return entries } - func debugList(messageHistoryTable: MessageHistoryTable) -> [ChatListIntermediateEntry] { + func debugList(_ messageHistoryTable: MessageHistoryTable) -> [ChatListIntermediateEntry] { return self.laterEntries(MessageIndex.absoluteLowerBound(), messageHistoryTable: messageHistoryTable, count: 1000) } } diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index 45acfb3662..7efc992bd3 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -113,7 +113,7 @@ final class MutableChatListView { self.count = count } - func replay(operations: [ChatListOperation], context: MutableChatListViewReplayContext) -> Bool { + func replay(_ operations: [ChatListOperation], context: MutableChatListViewReplayContext) -> Bool { var hasChanges = false for operation in operations { switch operation { @@ -142,7 +142,7 @@ final class MutableChatListView { return hasChanges } - func add(entry: MutableChatListEntry) -> Bool { + func add(_ entry: MutableChatListEntry) -> Bool { if self.entries.count == 0 { self.entries.append(entry) return true @@ -155,7 +155,7 @@ final class MutableChatListView { if entry.index < last.index { if self.earlier == nil || self.earlier!.index < entry.index { if self.entries.count < self.count { - self.entries.insert(entry, atIndex: 0) + self.entries.insert(entry, at: 0) } else { self.earlier = entry } @@ -179,7 +179,7 @@ final class MutableChatListView { self.entries.append(entry) if self.entries.count > self.count { self.earlier = self.entries[0] - self.entries.removeAtIndex(0) + self.entries.remove(at: 0) } return true } @@ -191,10 +191,10 @@ final class MutableChatListView { } i -= 1 } - self.entries.insert(entry, atIndex: i) + self.entries.insert(entry, at: i) if self.entries.count > self.count { self.earlier = self.entries[0] - self.entries.removeAtIndex(0) + self.entries.remove(at: 0) } return true } else { @@ -203,7 +203,7 @@ final class MutableChatListView { } } - func remove(indices: Set, holes: Bool, context: MutableChatListViewReplayContext) -> Bool { + func remove(_ indices: Set, holes: Bool, context: MutableChatListViewReplayContext) -> Bool { var hasChanges = false if let earlier = self.earlier where indices.contains(earlier.index) { var match = false @@ -245,7 +245,7 @@ final class MutableChatListView { match = !holes } if match { - self.entries.removeAtIndex(i) + self.entries.remove(at: i) context.removedEntries = true hasChanges = true } @@ -280,11 +280,11 @@ final class MutableChatListView { } addedEntries += self.entries - addedEntries.sortInPlace({ $0.index < $1.index }) + addedEntries.sort(isOrderedBefore: { $0.index < $1.index }) var i = addedEntries.count - 1 while i >= 1 { if addedEntries[i].index.id == addedEntries[i - 1].index.id { - addedEntries.removeAtIndex(i) + addedEntries.remove(at: i) } i -= 1 } @@ -309,7 +309,7 @@ final class MutableChatListView { i = anchorIndex while i >= 0 && i > anchorIndex - self.count { - self.entries.insert(addedEntries[i], atIndex: 0) + self.entries.insert(addedEntries[i], at: 0) i -= 1 } @@ -352,8 +352,8 @@ final class MutableChatListView { return nil } - func updatePeers(peers: [PeerId: Peer]) -> Bool { - var hasChanges = false + func updatePeers(_ peers: [PeerId: Peer]) -> Bool { + let hasChanges = false /*for i in 0 ..< self.entries.count { switch self.entries[i] { case let .MessageEntry(message): @@ -374,7 +374,7 @@ final class MutableChatListView { return hasChanges } - func render(renderMessage: IntermediateMessage -> Message) { + func render(_ renderMessage: (IntermediateMessage) -> Message) { for i in 0 ..< self.entries.count { if case let .IntermediateMessageEntry(message, combinedReadState) = self.entries[i] { self.entries[i] = .MessageEntry(renderMessage(message), combinedReadState) diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 234df16053..08b32c12a4 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -2,13 +2,13 @@ import Foundation public protocol Coding { init(decoder: Decoder) - func encode(encoder: Encoder) + func encode(_ encoder: Encoder) } private final class EncodableTypeStore { - var dict: [Int32 : Decoder -> Coding] = [:] + var dict: [Int32 : (Decoder) -> Coding] = [:] - func decode(typeHash: Int32, decoder: Decoder) -> Coding? { + func decode(_ typeHash: Int32, decoder: Decoder) -> Coding? { if let typeDecoder = self.dict[typeHash] { return typeDecoder(decoder) } else { @@ -22,7 +22,7 @@ private let typeStore = { () -> EncodableTypeStore in return _typeStore }() -public func declareEncodable(type: Any.Type, f: Decoder -> Coding) { +public func declareEncodable(_ type: Any.Type, f: (Decoder) -> Coding) { let string = "\(type)" let hash = murMurHashString32(string) if typeStore.dict[hash] != nil { @@ -31,6 +31,8 @@ public func declareEncodable(type: Any.Type, f: Decoder -> Coding) { typeStore.dict[murMurHashString32("\(type)")] = f } +private let emptyMemory = malloc(1)! + public class MemoryBuffer: Equatable, CustomStringConvertible { var memory: UnsafeMutablePointer var capacity: Int @@ -52,15 +54,23 @@ public class MemoryBuffer: Equatable, CustomStringConvertible { self.freeWhenDone = freeWhenDone } - public init(data: NSData) { - self.memory = UnsafeMutablePointer(data.bytes) - self.capacity = data.length - self.length = data.length - self.freeWhenDone = false + public init(data: Data) { + if data.count == 0 { + self.memory = emptyMemory + self.capacity = 0 + self.length = 0 + self.freeWhenDone = false + } else { + self.memory = malloc(data.count)! + data.copyBytes(to: UnsafeMutablePointer(self.memory), count: data.count) + self.capacity = data.count + self.length = data.count + self.freeWhenDone = false + } } public init() { - self.memory = nil + self.memory = emptyMemory self.capacity = 0 self.length = 0 self.freeWhenDone = false @@ -106,23 +116,42 @@ public final class WriteBuffer: MemoryBuffer { return ReadBuffer(memory: self.memory, length: self.offset, freeWhenDone: false) } - public func makeData() -> NSData { - return NSData(bytes: self.memory, length: self.offset) + public func makeData() -> Data { + return Data(bytes: UnsafePointer(self.memory), count: self.offset) } public func reset() { self.offset = 0 } - public func write(data: UnsafePointer, offset: Int, length: Int) { + public func write(_ data: UnsafePointer, offset: Int, length: Int) { if self.offset + length > self.capacity { self.capacity = self.offset + length + 256 - self.memory = realloc(self.memory, self.capacity) + if self.length == 0 { + self.memory = malloc(self.capacity)! + } else { + self.memory = realloc(self.memory, self.capacity) + } } memcpy(self.memory + self.offset, data + offset, length) self.offset += length self.length = self.offset } + + public func write(_ data: Data) { + let length = data.count + if self.offset + length > self.capacity { + self.capacity = self.offset + length + 256 + if self.length == 0 { + self.memory = malloc(self.capacity)! + } else { + self.memory = realloc(self.memory, self.capacity) + } + } + data.copyBytes(to: UnsafeMutablePointer(self.memory + offset), count: length) + self.offset += length + self.length = self.offset + } } public final class ReadBuffer: MemoryBuffer { @@ -136,16 +165,16 @@ public final class ReadBuffer: MemoryBuffer { super.init(memory: memoryBufferNoCopy.memory, capacity: memoryBufferNoCopy.length, length: memoryBufferNoCopy.length, freeWhenDone: false) } - func dataNoCopy() -> NSData { - return NSData(bytesNoCopy: self.memory, length: self.length, freeWhenDone: false) + func dataNoCopy() -> Data { + return Data(bytesNoCopy: UnsafeMutablePointer(self.memory), count: self.length, deallocator: .none) } - func read(data: UnsafeMutablePointer, offset: Int, length: Int) { + func read(_ data: UnsafeMutablePointer, offset: Int, length: Int) { memcpy(data + offset, self.memory + self.offset, length) self.offset += length } - func skip(length: Int) { + func skip(_ length: Int) { self.offset += length } @@ -190,7 +219,7 @@ public final class Encoder { return self.buffer.readBufferNoCopy() } - public func makeData() -> NSData { + public func makeData() -> Data { return self.buffer.makeData() } @@ -198,13 +227,13 @@ public final class Encoder { self.buffer.reset() } - public func encodeKey(key: StaticString) { - var length: Int8 = Int8(key.byteSize) + public func encodeKey(_ key: StaticString) { + var length: Int8 = Int8(key.utf8CodeUnitCount) self.buffer.write(&length, offset: 0, length: 1) self.buffer.write(key.utf8Start, offset: 0, length: Int(length)) } - public func encodeInt32(value: Int32, forKey key: StaticString) { + public func encodeInt32(_ value: Int32, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int32.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -212,7 +241,7 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 4) } - public func encodeInt64(value: Int64, forKey key: StaticString) { + public func encodeInt64(_ value: Int64, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int64.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -220,7 +249,7 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 8) } - public func encodeBool(value: Bool, forKey key: StaticString) { + public func encodeBool(_ value: Bool, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Bool.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -228,7 +257,7 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 1) } - public func encodeDouble(value: Double, forKey key: StaticString) { + public func encodeDouble(_ value: Double, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Double.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -236,31 +265,21 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 8) } - public func encodeString(value: String, forKey key: StaticString) { + public func encodeString(_ value: String, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.String.rawValue self.buffer.write(&type, offset: 0, length: 1) - let data = value.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! - var length: Int32 = Int32(data.length) + let data = value.data(using: .utf8, allowLossyConversion: true)! + var length: Int32 = Int32(data.count) self.buffer.write(&length, offset: 0, length: 4) - self.buffer.write(data.bytes, offset: 0, length: Int(length)) + self.buffer.write(data) } - public func encodeString(value: DeferredString, forKey key: StaticString) { - self.encodeKey(key) - var type: Int8 = ValueType.String.rawValue - self.buffer.write(&type, offset: 0, length: 1) - let data = value.data - var length: Int32 = Int32(data.length) - self.buffer.write(&length, offset: 0, length: 4) - self.buffer.write(data.bytes, offset: 0, length: Int(length)) - } - - public func encodeRootObject(value: Coding) { + public func encodeRootObject(_ value: Coding) { self.encodeObject(value, forKey: "_") } - public func encodeObject(value: Coding, forKey key: StaticString) { + public func encodeObject(_ value: Coding, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Object.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -277,31 +296,31 @@ public final class Encoder { self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(length)) } - public func encodeInt32Array(value: [Int32], forKey key: StaticString) { + public func encodeInt32Array(_ value: [Int32], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int32Array.rawValue self.buffer.write(&type, offset: 0, length: 1) var length: Int32 = Int32(value.count) self.buffer.write(&length, offset: 0, length: 4) value.withUnsafeBufferPointer { (data: UnsafeBufferPointer) -> Void in - self.buffer.write(UnsafePointer(data.baseAddress), offset: 0, length: Int(length) * 4) + self.buffer.write(UnsafePointer(data.baseAddress!), offset: 0, length: Int(length) * 4) return } } - public func encodeInt64Array(value: [Int64], forKey key: StaticString) { + public func encodeInt64Array(_ value: [Int64], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int64Array.rawValue self.buffer.write(&type, offset: 0, length: 1) var length: Int32 = Int32(value.count) self.buffer.write(&length, offset: 0, length: 4) value.withUnsafeBufferPointer { (data: UnsafeBufferPointer) -> Void in - self.buffer.write(UnsafePointer(data.baseAddress), offset: 0, length: Int(length) * 8) + self.buffer.write(UnsafePointer(data.baseAddress!), offset: 0, length: Int(length) * 8) return } } - public func encodeObjectArray(value: [T], forKey key: StaticString) { + public func encodeObjectArray(_ value: [T], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.ObjectArray.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -321,7 +340,7 @@ public final class Encoder { } } - public func encodeObjectArray(value: [Coding], forKey key: StaticString) { + public func encodeGenericObjectArray(_ value: [Coding], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.ObjectArray.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -341,7 +360,7 @@ public final class Encoder { } } - public func encodeObjectDictionary(value: [K : V], forKey key: StaticString) { + public func encodeObjectDictionary(_ value: [K : V], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.ObjectDictionary.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -368,7 +387,7 @@ public final class Encoder { } } - public func encodeBytes(bytes: WriteBuffer, forKey key: StaticString) { + public func encodeBytes(_ bytes: WriteBuffer, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Bytes.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -377,7 +396,7 @@ public final class Encoder { self.buffer.write(bytes.memory, offset: 0, length: bytes.offset) } - public func encodeBytes(bytes: ReadBuffer, forKey key: StaticString) { + public func encodeBytes(_ bytes: ReadBuffer, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Bytes.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -386,7 +405,7 @@ public final class Encoder { self.buffer.write(bytes.memory, offset: 0, length: bytes.offset) } - public func encodeBytes(bytes: MemoryBuffer, forKey key: StaticString) { + public func encodeBytes(_ bytes: MemoryBuffer, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Bytes.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -406,7 +425,7 @@ public final class Decoder { self.buffer = buffer } - private class func skipValue(bytes: UnsafePointer, inout offset: Int, length: Int, valueType: ValueType) { + private class func skipValue(_ bytes: UnsafePointer, offset: inout Int, length: Int, valueType: ValueType) { switch valueType { case .Int32: offset += 4 @@ -465,11 +484,11 @@ public final class Decoder { } } - private class func positionOnKey(bytes: UnsafePointer, inout offset: Int, maxOffset: Int, length: Int, key: StaticString, valueType: ValueType) -> Bool + private class func positionOnKey(_ bytes: UnsafePointer, offset: inout Int, maxOffset: Int, length: Int, key: StaticString, valueType: ValueType) -> Bool { let startOffset = offset - let keyLength: Int = key.byteSize + let keyLength: Int = key.utf8CodeUnitCount while (offset < maxOffset) { let readKeyLength = bytes[offset] @@ -495,7 +514,7 @@ public final class Decoder { return false } - private class func positionOnKey(bytes: UnsafePointer, inout offset: Int, maxOffset: Int, length: Int, key: Int16, valueType: ValueType) -> Bool + private class func positionOnKey(_ bytes: UnsafePointer, offset: inout Int, maxOffset: Int, length: Int, key: Int16, valueType: ValueType) -> Bool { var keyValue = key let startOffset = offset @@ -526,7 +545,7 @@ public final class Decoder { return false } - public func decodeInt32ForKey(key: StaticString) -> Int32 { + public func decodeInt32ForKey(_ key: StaticString) -> Int32 { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 memcpy(&value, self.buffer.memory + self.offset, 4) @@ -537,7 +556,7 @@ public final class Decoder { } } - public func decodeInt32ForKey(key: StaticString) -> Int32? { + public func decodeInt32ForKey(_ key: StaticString) -> Int32? { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 memcpy(&value, self.buffer.memory + self.offset, 4) @@ -548,7 +567,7 @@ public final class Decoder { } } - public func decodeInt64ForKey(key: StaticString) -> Int64 { + public func decodeInt64ForKey(_ key: StaticString) -> Int64 { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { var value: Int64 = 0 memcpy(&value, self.buffer.memory + self.offset, 8) @@ -559,7 +578,7 @@ public final class Decoder { } } - public func decodeInt64ForKey(key: StaticString) -> Int64? { + public func decodeInt64ForKey(_ key: StaticString) -> Int64? { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { var value: Int64 = 0 memcpy(&value, self.buffer.memory + self.offset, 8) @@ -570,7 +589,7 @@ public final class Decoder { } } - public func decodeBoolForKey(key: StaticString) -> Bool { + public func decodeBoolForKey(_ key: StaticString) -> Bool { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bool) { var value: Int8 = 0 memcpy(&value, self.buffer.memory + self.offset, 1) @@ -581,7 +600,7 @@ public final class Decoder { } } - public func decodeDoubleForKey(key: StaticString) -> Double { + public func decodeDoubleForKey(_ key: StaticString) -> Double { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Double) { var value: Double = 0 memcpy(&value, self.buffer.memory + self.offset, 8) @@ -592,51 +611,25 @@ public final class Decoder { } } - public func decodeStringForKey(key: StaticString) -> String { + public func decodeStringForKey(_ key: StaticString) -> String { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) - let data = NSData(bytes: self.buffer.memory + (self.offset + 4), length: Int(length)) + let data = Data(bytes: UnsafeMutablePointer(self.buffer.memory).advanced(by: self.offset + 4), count: Int(length)) self.offset += 4 + Int(length) - let value = NSString(data: data, encoding: NSUTF8StringEncoding) - return (value as? String) ?? "" + return String(data: data, encoding: .utf8) ?? "" } else { return "" } } - public func decodeStringForKey(key: StaticString) -> String? { + public func decodeStringForKey(_ key: StaticString) -> String? { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) - let data = NSData(bytes: self.buffer.memory + (self.offset + 4), length: Int(length)) + let data = Data(bytes: UnsafeMutablePointer(self.buffer.memory).advanced(by: self.offset + 4), count: Int(length)) self.offset += 4 + Int(length) - let value = NSString(data: data, encoding: NSUTF8StringEncoding) - return value as? String - } else { - return nil - } - } - - public func decodeStringForKey(key: StaticString) -> DeferredString { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { - var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.offset, 4) - let data = NSData(bytes: self.buffer.memory + (self.offset + 4), length: Int(length)) - self.offset += 4 + Int(length) - return DeferredStringValue(data) - } else { - return DeferredStringValue("") - } - } - - public func decodeStringForKey(key: StaticString) -> DeferredString? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { - var length: Int32 = 0 - memcpy(&length, self.buffer.memory + self.offset, 4) - let data = NSData(bytes: self.buffer.memory + (self.offset + 4), length: Int(length)) - self.offset += 4 + Int(length) - return DeferredStringValue(data) + return String(data: data, encoding: .utf8) } else { return nil } @@ -646,7 +639,7 @@ public final class Decoder { return self.decodeObjectForKey("_") } - public func decodeObjectForKey(key: StaticString) -> Coding? { + public func decodeObjectForKey(_ key: StaticString) -> Coding? { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { var typeHash: Int32 = 0 memcpy(&typeHash, self.buffer.memory + self.offset, 4) @@ -664,7 +657,7 @@ public final class Decoder { } } - public func decodeObjectForKey(key: StaticString, decoder: Decoder -> Coding) -> Coding? { + public func decodeObjectForKey(_ key: StaticString, decoder: (Decoder) -> Coding) -> Coding? { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { var typeHash: Int32 = 0 memcpy(&typeHash, self.buffer.memory + self.offset, 4) @@ -682,7 +675,7 @@ public final class Decoder { } } - public func decodeInt32ArrayForKey(key: StaticString) -> [Int32] { + public func decodeInt32ArrayForKey(_ key: StaticString) -> [Int32] { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32Array) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) @@ -702,7 +695,7 @@ public final class Decoder { } } - public func decodeInt64ArrayForKey(key: StaticString) -> [Int64] { + public func decodeInt64ArrayForKey(_ key: StaticString) -> [Int64] { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64Array) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) @@ -722,7 +715,7 @@ public final class Decoder { } } - public func decodeObjectArrayWithDecoderForKey(key: StaticString) -> [T] { + public func decodeObjectArrayWithDecoderForKey(_ key: StaticString) -> [T] { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) @@ -731,7 +724,7 @@ public final class Decoder { var array: [T] = [] array.reserveCapacity(Int(length)) - var failed = false + let failed = false var i: Int32 = 0 while i < length { var typeHash: Int32 = 0 @@ -745,11 +738,7 @@ public final class Decoder { self.offset += 4 + Int(objectLength) if !failed { - if let object = T(decoder: innerDecoder) as? T { - array.append(object) - } else { - failed = true - } + array.append(T(decoder: innerDecoder)) } i += 1 @@ -765,7 +754,7 @@ public final class Decoder { } } - public func decodeObjectArrayForKey(key: StaticString) -> [T] { + public func decodeObjectArrayForKey(_ key: StaticString) -> [T] { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) @@ -808,7 +797,7 @@ public final class Decoder { } } - public func decodeObjectArrayForKey(key: StaticString) -> [Coding] { + public func decodeObjectArrayForKey(_ key: StaticString) -> [Coding] { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) @@ -851,7 +840,7 @@ public final class Decoder { } } - public func decodeObjectDictionaryForKey(key: StaticString) -> [K : V] { + public func decodeObjectDictionaryForKey(_ key: StaticString) -> [K : V] { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) @@ -905,7 +894,7 @@ public final class Decoder { } } - public func decodeBytesForKeyNoCopy(key: StaticString) -> ReadBuffer! { + public func decodeBytesForKeyNoCopy(_ key: StaticString) -> ReadBuffer! { if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bytes) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) diff --git a/Postbox/DeferredString.swift b/Postbox/DeferredString.swift deleted file mode 100644 index adce77284f..0000000000 --- a/Postbox/DeferredString.swift +++ /dev/null @@ -1,124 +0,0 @@ -import Foundation - -public protocol DeferredString { - var data: NSData { get } - var string: String { get } -} - -private final class DeferredStringImpl { - var string: String! - var data: NSData! - var lock: OSSpinLock = 0 - - init(string: String!, data: NSData!) { - self.string = string - self.data = data - } -} - -public final class DeferredStringValue: DeferredString, CustomStringConvertible, StringLiteralConvertible { - private var impl: DeferredStringImpl - - public typealias ExtendedGraphemeClusterLiteralType = StringLiteralType - public typealias UnicodeScalarLiteralType = StringLiteralType - - public init(unicodeScalarLiteral value: UnicodeScalarLiteralType) { - self.impl = DeferredStringImpl(string: "\(value)", data: nil) - } - - public init(extendedGraphemeClusterLiteral value: ExtendedGraphemeClusterLiteralType) { - self.impl = DeferredStringImpl(string: value, data: nil) - } - - public init(stringLiteral value: StringLiteralType) { - self.impl = DeferredStringImpl(string: value, data: nil) - } - - public init(_ string: String) { - self.impl = DeferredStringImpl(string: string, data: nil) - } - - public init(_ data: NSData) { - self.impl = DeferredStringImpl(string: nil, data: data) - } - - public var description: String { - return self.string - } - - public var data: NSData { - if isUniquelyReferencedNonObjC(&self.impl) { - let value: NSData - - if impl.data != nil { - value = impl.data - } else if impl.string != nil { - value = impl.string.dataUsingEncoding(NSUTF8StringEncoding) ?? NSData() - impl.data = value - } else { - value = NSData() - } - - return value - } else { - let value: NSData - - OSSpinLockLock(&impl.lock) - if impl.data != nil { - value = impl.data - } else if impl.string != nil { - value = impl.string.dataUsingEncoding(NSUTF8StringEncoding) ?? NSData() - impl.data = value - } else { - value = NSData() - } - OSSpinLockUnlock(&impl.lock) - - return value - } - } - - public var string: String { - if isUniquelyReferencedNonObjC(&self.impl) { - let value: String - - if impl.string != nil { - value = impl.string - } else if impl.data != nil { - let fromData = NSString(data: impl.data, encoding: NSUTF8StringEncoding) - if fromData == nil { - value = "" - impl.string = value - } else { - value = fromData as! String - impl.string = value - } - } else { - value = "" - } - - return value - } else { - let value: String - - OSSpinLockLock(&impl.lock) - if impl.string != nil { - value = impl.string - } else if impl.data != nil { - let fromData = NSString(data: impl.data, encoding: NSUTF8StringEncoding) - if fromData == nil { - value = "" - impl.string = value - } else { - value = fromData as! String - impl.string = value - } - } else { - value = "" - } - OSSpinLockUnlock(&impl.lock) - - return value - } - } -} diff --git a/Postbox/GlobalMessageIdsTable.swift b/Postbox/GlobalMessageIdsTable.swift index 6b9fec219a..27063a8471 100644 --- a/Postbox/GlobalMessageIdsTable.swift +++ b/Postbox/GlobalMessageIdsTable.swift @@ -12,12 +12,12 @@ final class GlobalMessageIdsTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func key(id: Int32) -> ValueBoxKey { + private func key(_ id: Int32) -> ValueBoxKey { self.sharedKey.setInt32(0, value: id) return self.sharedKey } - func set(globalId: Int32, id: MessageId) { + func set(_ globalId: Int32, id: MessageId) { self.sharedBuffer.reset() var idPeerId: Int64 = id.peerId.toInt64() var idNamespace: Int32 = id.namespace @@ -26,7 +26,7 @@ final class GlobalMessageIdsTable: Table { self.valueBox.set(self.tableId, key: self.key(globalId), value: self.sharedBuffer) } - func get(globalId: Int32) -> MessageId? { + func get(_ globalId: Int32) -> MessageId? { if let value = self.valueBox.get(self.tableId, key: self.key(globalId)) { var idPeerId: Int64 = 0 var idNamespace: Int32 = 0 @@ -37,7 +37,7 @@ final class GlobalMessageIdsTable: Table { return nil } - func remove(globalId: Int32) { + func remove(_ globalId: Int32) { self.valueBox.remove(self.tableId, key: self.key(globalId)) } -} \ No newline at end of file +} diff --git a/Postbox/KeychainTable.swift b/Postbox/KeychainTable.swift index 00208e6459..82befd5759 100644 --- a/Postbox/KeychainTable.swift +++ b/Postbox/KeychainTable.swift @@ -5,22 +5,22 @@ final class KeychainTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func key(string: String) -> ValueBoxKey { + private func key(_ string: String) -> ValueBoxKey { return ValueBoxKey(string) } - func get(key: String) -> NSData? { + func get(_ key: String) -> Data? { if let value = self.valueBox.get(self.tableId, key: self.key(key)) { - return NSData(bytes: value.memory, length: value.length) + return Data(bytes: UnsafePointer(value.memory), count: value.length) } return nil } - func set(key: String, value: NSData) { + func set(_ key: String, value: Data) { self.valueBox.set(self.tableId, key: self.key(key), value: MemoryBuffer(data: value)) } - func remove(key: String) { + func remove(_ key: String) { self.valueBox.remove(self.tableId, key: self.key(key)) } } diff --git a/Postbox/LmdbValueBox.swift b/Postbox/LmdbValueBox.swift index ec578ff300..d20c12264b 100644 --- a/Postbox/LmdbValueBox.swift +++ b/Postbox/LmdbValueBox.swift @@ -6,9 +6,9 @@ private struct LmdbTable { } private struct LmdbCursor { - var cursor: COpaquePointer + var cursor: OpaquePointer! - func seekTo(key: ValueBoxKey, forward: Bool) -> (ValueBoxKey, ReadBuffer)? { + func seekTo(_ key: ValueBoxKey, forward: Bool) -> (ValueBoxKey, ReadBuffer)? { var mdbKey = MDB_val() var mdbData = MDB_val() @@ -23,7 +23,7 @@ private struct LmdbCursor { let actualKey = ValueBoxKey(length: mdbKey.mv_size) memcpy(actualKey.memory, mdbKey.mv_data, mdbKey.mv_size) - let value = malloc(mdbData.mv_size) + let value = malloc(mdbData.mv_size)! memcpy(value, mdbData.mv_data, mdbData.mv_size) return (actualKey, ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true)) @@ -53,7 +53,7 @@ private struct LmdbCursor { let actualKey = ValueBoxKey(length: mdbKey.mv_size) memcpy(actualKey.memory, mdbKey.mv_data, mdbKey.mv_size) - let value = malloc(mdbData.mv_size) + let value = malloc(mdbData.mv_size)! memcpy(value, mdbData.mv_data, mdbData.mv_size) return (actualKey, ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true)) @@ -79,7 +79,7 @@ private struct LmdbCursor { let actualKey = ValueBoxKey(length: mdbKey.mv_size) memcpy(actualKey.memory, mdbKey.mv_data, mdbKey.mv_size) - let value = malloc(mdbData.mv_size) + let value = malloc(mdbData.mv_size)! memcpy(value, mdbData.mv_data, mdbData.mv_size) return (actualKey, ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true)) @@ -93,10 +93,10 @@ private struct LmdbCursor { } public final class LmdbValueBox: ValueBox { - private var env: COpaquePointer = nil + private var env: OpaquePointer? = nil private var tables: [Int32 : LmdbTable] = [:] - private var sharedTxn: COpaquePointer = nil + private var sharedTxn: OpaquePointer? = nil private var readQueryTime: CFAbsoluteTime = 0.0 private var writeQueryTime: CFAbsoluteTime = 0.0 @@ -113,10 +113,10 @@ public final class LmdbValueBox: ValueBox { var createDirectory = false var isDirectory: ObjCBool = false as ObjCBool - if NSFileManager.defaultManager().fileExistsAtPath(path, isDirectory: &isDirectory) { + if FileManager.default().fileExists(atPath: path, isDirectory: &isDirectory) { if !isDirectory { do { - try NSFileManager.defaultManager().removeItemAtPath(path) + try FileManager.default().removeItem(atPath: path) } catch _ { } createDirectory = true } @@ -127,7 +127,7 @@ public final class LmdbValueBox: ValueBox { if createDirectory { do { - try NSFileManager.defaultManager().createDirectoryAtPath(path, withIntermediateDirectories: true, attributes: nil) + try FileManager.default().createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil) } catch _ { } } @@ -154,7 +154,7 @@ public final class LmdbValueBox: ValueBox { mdb_env_close(self.env) } - private func createTableWithName(name: Int32) -> LmdbTable? { + private func createTableWithName(_ name: Int32) -> LmdbTable? { var dbi = MDB_dbi() let result = mdb_dbi_open(self.sharedTxn, "\(name)", UInt32(MDB_CREATE), &dbi) @@ -204,7 +204,7 @@ public final class LmdbValueBox: ValueBox { } } - public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { + public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, values: @noescape(ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { if start == end || limit == 0 { return } @@ -225,7 +225,7 @@ public final class LmdbValueBox: ValueBox { if let nativeTable = nativeTable { var startTime = CFAbsoluteTimeGetCurrent() - var cursorPtr: COpaquePointer = nil + var cursorPtr: OpaquePointer? = nil let result = mdb_cursor_open(self.sharedTxn, nativeTable.dbi, &cursorPtr) if result != MDB_SUCCESS { print("(LmdbValueBox mdb_cursor_open failed with \(result))") @@ -247,7 +247,7 @@ public final class LmdbValueBox: ValueBox { var count = 0 if value != nil && value!.0 < end { count += 1 - values(value!.0, value!.1) + let _ = values(value!.0, value!.1) } while value != nil && value!.0 < end && count < limit { @@ -261,7 +261,7 @@ public final class LmdbValueBox: ValueBox { if value != nil && value!.0 < end { count += 1 - values(value!.0, value!.1) + let _ = values(value!.0, value!.1) } } } else { @@ -280,7 +280,7 @@ public final class LmdbValueBox: ValueBox { var count = 0 if value != nil && value!.0 > end { count += 1 - values(value!.0, value!.1) + let _ = values(value!.0, value!.1) } while value != nil && value!.0 > end && count < limit { @@ -294,7 +294,7 @@ public final class LmdbValueBox: ValueBox { if value != nil && value!.0 > end { count += 1 - values(value!.0, value!.1) + let _ = values(value!.0, value!.1) } } } @@ -312,13 +312,13 @@ public final class LmdbValueBox: ValueBox { } } - public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape keys: ValueBoxKey -> Bool, limit: Int) { + public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: @noescape(ValueBoxKey) -> Bool, limit: Int) { self.range(table, start: start, end: end, values: { key, _ in return keys(key) }, limit: limit) } - public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? { + public func get(_ table: Int32, key: ValueBoxKey) -> ReadBuffer? { let startTime = CFAbsoluteTimeGetCurrent() var commit = false @@ -347,7 +347,7 @@ public final class LmdbValueBox: ValueBox { let result = mdb_get(self.sharedTxn, nativeTable.dbi, &mdbKey, &mdbData) if result == MDB_SUCCESS { - let value = malloc(mdbData.mv_size) + let value = malloc(mdbData.mv_size)! memcpy(value, mdbData.mv_data, mdbData.mv_size) resultValue = ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true) } else { @@ -366,11 +366,11 @@ public final class LmdbValueBox: ValueBox { return resultValue } - public func exists(table: Int32, key: ValueBoxKey) -> Bool { + public func exists(_ table: Int32, key: ValueBoxKey) -> Bool { return self.get(table, key: key) != nil } - public func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) { + public func set(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) { let startTime = CFAbsoluteTimeGetCurrent() var commit = false @@ -409,7 +409,7 @@ public final class LmdbValueBox: ValueBox { writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } - public func remove(table: Int32, key: ValueBoxKey) { + public func remove(_ table: Int32, key: ValueBoxKey) { let startTime = CFAbsoluteTimeGetCurrent() var commit = false diff --git a/Postbox/Media.swift b/Postbox/Media.swift index 4b0dfe8fd8..85d2097fbf 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -35,14 +35,14 @@ public struct MediaId: Hashable, CustomStringConvertible { buffer.offset += 12 } - public func encodeToBuffer(buffer: WriteBuffer) { + public func encodeToBuffer(_ buffer: WriteBuffer) { var namespace = self.namespace var id = self.id buffer.write(&namespace, offset: 0, length: 4); buffer.write(&id, offset: 0, length: 8); } - public static func encodeArrayToBuffer(array: [MediaId], buffer: WriteBuffer) { + public static func encodeArrayToBuffer(_ array: [MediaId], buffer: WriteBuffer) { var length: Int32 = Int32(array.count) buffer.write(&length, offset: 0, length: 4) for id in array { @@ -50,7 +50,7 @@ public struct MediaId: Hashable, CustomStringConvertible { } } - public static func decodeArrayFromBuffer(buffer: ReadBuffer) -> [MediaId] { + public static func decodeArrayFromBuffer(_ buffer: ReadBuffer) -> [MediaId] { var length: Int32 = 0 memcpy(&length, buffer.memory, 4) buffer.offset += 4 @@ -72,5 +72,5 @@ public protocol Media: Coding { var id: MediaId? { get } var peerIds: [PeerId] { get } - func isEqual(other: Media) -> Bool + func isEqual(_ other: Media) -> Bool } diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 52addd013e..0a1124a70f 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -3,13 +3,13 @@ import SwiftSignalKit private final class ResourceStatusContext { var status: MediaResourceStatus? - let subscribers = Bag Void>() + let subscribers = Bag<(MediaResourceStatus) -> Void>() } private final class ResourceDataContext { var data: MediaResourceData - let progresiveDataSubscribers = Bag Void>() - let completeDataSubscribers = Bag Void>() + let progresiveDataSubscribers = Bag<(MediaResourceData) -> Void>() + let completeDataSubscribers = Bag<(MediaResourceData) -> Void>() var fetchDisposable: Disposable? let fetchSubscribers = Bag() @@ -19,7 +19,7 @@ private final class ResourceDataContext { } } -private func fileSize(path: String) -> Int { +private func fileSize(_ path: String) -> Int { var value = stat() stat(path, &value) return Int(value.st_size) @@ -35,8 +35,8 @@ public final class MediaBox { private var statusContexts: [String: ResourceStatusContext] = [:] private var dataContexts: [String: ResourceDataContext] = [:] - private var wrappedFetchResource = Promise<(MediaResource, Int) -> Signal>() - public var fetchResource: ((MediaResource, Int) -> Signal)? { + private var wrappedFetchResource = Promise<(MediaResource, Int) -> Signal>() + public var fetchResource: ((MediaResource, Int) -> Signal)? { didSet { if let fetchResource = self.fetchResource { wrappedFetchResource.set(.single(fetchResource)) @@ -47,22 +47,22 @@ public final class MediaBox { } lazy var ensureDirectoryCreated: Void = { - try! NSFileManager.defaultManager().createDirectoryAtPath(self.basePath, withIntermediateDirectories: true, attributes: nil) + try! FileManager.default().createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) }() public init(basePath: String) { self.basePath = basePath } - private func pathForId(id: String) -> String { + private func pathForId(_ id: String) -> String { return "\(self.basePath)/\(id)" } - public func resourceStatus(resource: MediaResource) -> Signal { + public func resourceStatus(_ resource: MediaResource) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() - self.statusQueue.dispatch { + self.statusQueue.async { let statusContext: ResourceStatusContext if let current = self.statusContexts[resource.id] { statusContext = current @@ -78,7 +78,7 @@ public final class MediaBox { if let status = statusContext.status { subscriber.putNext(status) } else { - self.dataQueue.dispatch { + self.dataQueue.async { let status: MediaResourceStatus let path = self.pathForId(resource.id) @@ -98,7 +98,7 @@ public final class MediaBox { } } - self.statusQueue.dispatch { + self.statusQueue.async { if let statusContext = self.statusContexts[resource.id] where statusContext.status == nil { statusContext.status = status @@ -111,11 +111,11 @@ public final class MediaBox { } disposable.set(ActionDisposable { - self.statusQueue.dispatch { + self.statusQueue.async { if let current = self.statusContexts[resource.id] { current.subscribers.remove(index) if current.subscribers.isEmpty { - self.statusContexts.removeValueForKey(resource.id) + self.statusContexts.removeValue(forKey: resource.id) } } } @@ -126,11 +126,11 @@ public final class MediaBox { } } - public func resourceData(resource: MediaResource, complete: Bool = true) -> Signal { + public func resourceData(_ resource: MediaResource, complete: Bool = true) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() - self.dataQueue.dispatch { + self.dataQueue.async { let path = self.pathForId(resource.id) let currentSize = fileSize(path) @@ -146,7 +146,7 @@ public final class MediaBox { self.dataContexts[resource.id] = dataContext } - let index: Bag Void>.Index + let index: Bag<(MediaResourceData) -> Void>.Index if complete { index = dataContext.completeDataSubscribers.add { data in subscriber.putNext(data) @@ -170,7 +170,7 @@ public final class MediaBox { } disposable.set(ActionDisposable { - self.dataQueue.dispatch { + self.dataQueue.async { if let dataContext = self.dataContexts[resource.id] { if complete { dataContext.completeDataSubscribers.remove(index) @@ -179,7 +179,7 @@ public final class MediaBox { } if dataContext.progresiveDataSubscribers.isEmpty && dataContext.completeDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { - self.dataContexts.removeValueForKey(resource.id) + self.dataContexts.removeValue(forKey: resource.id) } } } @@ -191,11 +191,11 @@ public final class MediaBox { } } - public func fetchedResource(resource: MediaResource, interactive: Bool) -> Signal { + public func fetchedResource(_ resource: MediaResource, interactive: Bool) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() - self.dataQueue.dispatch { + self.dataQueue.async { let path = self.pathForId(resource.id) let currentSize = fileSize(path) @@ -214,7 +214,7 @@ public final class MediaBox { if dataContext.fetchDisposable == nil { let status: MediaResourceStatus = .Fetching(progress: Float(currentSize) / Float(resource.size)) - self.statusQueue.dispatch { + self.statusQueue.async { if let statusContext = self.statusContexts[resource.id] { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { @@ -225,14 +225,14 @@ public final class MediaBox { var offset = currentSize var fd: Int32? - dataContext.fetchDisposable = (self.wrappedFetchResource.get() |> mapToSignal { fetch -> Signal in + dataContext.fetchDisposable = (self.wrappedFetchResource.get() |> mapToSignal { fetch -> Signal in return fetch(resource, offset) } |> afterDisposed { if let fd = fd { close(fd) } }).start(next: { data in - self.dataQueue.dispatch { + self.dataQueue.async { let _ = self.ensureDirectoryCreated if fd == nil { @@ -243,9 +243,11 @@ public final class MediaBox { } if let fd = fd { - write(fd, data.bytes, data.length) + data.withUnsafeBytes { bytes in + write(fd, bytes, data.count) + } - offset += data.length + offset += data.count let updatedSize = offset for subscriber in dataContext.progresiveDataSubscribers.copyItems() { @@ -265,7 +267,7 @@ public final class MediaBox { status = .Fetching(progress: Float(updatedSize) / Float(resource.size)) } - self.statusQueue.dispatch { + self.statusQueue.async { if let statusContext = self.statusContexts[resource.id] { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { @@ -279,7 +281,7 @@ public final class MediaBox { } disposable.set(ActionDisposable { - self.dataQueue.dispatch { + self.dataQueue.async { if let dataContext = self.dataContexts[resource.id] { dataContext.fetchSubscribers.remove(index) @@ -295,7 +297,7 @@ public final class MediaBox { status = .Remote } - self.statusQueue.dispatch { + self.statusQueue.async { if let statusContext = self.statusContexts[resource.id] where statusContext.status != status { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { @@ -306,7 +308,7 @@ public final class MediaBox { } if dataContext.completeDataSubscribers.isEmpty && dataContext.progresiveDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { - self.dataContexts.removeValueForKey(resource.id) + self.dataContexts.removeValue(forKey: resource.id) } } } @@ -318,8 +320,8 @@ public final class MediaBox { } } - public func cancelInteractiveResourceFetch(resource: MediaResource) { - self.dataQueue.dispatch { + public func cancelInteractiveResourceFetch(_ resource: MediaResource) { + self.dataQueue.async { if let dataContext = self.dataContexts[resource.id] where dataContext.fetchDisposable != nil { dataContext.fetchDisposable?.dispose() dataContext.fetchDisposable = nil @@ -332,7 +334,7 @@ public final class MediaBox { status = .Remote } - self.statusQueue.dispatch { + self.statusQueue.async { if let statusContext = self.statusContexts[resource.id] where statusContext.status != status { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { diff --git a/Postbox/MediaCleanupTable.swift b/Postbox/MediaCleanupTable.swift index 6fde12ccca..042d879c12 100644 --- a/Postbox/MediaCleanupTable.swift +++ b/Postbox/MediaCleanupTable.swift @@ -7,11 +7,11 @@ final class MediaCleanupTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - func add(media: Media, sharedEncoder: Encoder = Encoder()) { + func add(_ media: Media, sharedEncoder: Encoder = Encoder()) { debugMedia.append(media) } func debugList() -> [Media] { return self.debugMedia } -} \ No newline at end of file +} diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 92c9d2a72e..b15a871f66 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -43,7 +43,7 @@ public struct MessageId: Hashable, Comparable, CustomStringConvertible { buffer.offset += 16 } - public func encodeToBuffer(buffer: WriteBuffer) { + public func encodeToBuffer(_ buffer: WriteBuffer) { var peerIdNamespace = self.peerId.namespace var peerIdId = self.peerId.id var namespace = self.namespace @@ -54,7 +54,7 @@ public struct MessageId: Hashable, Comparable, CustomStringConvertible { buffer.write(&id, offset: 0, length: 4); } - public static func encodeArrayToBuffer(array: [MessageId], buffer: WriteBuffer) { + public static func encodeArrayToBuffer(_ array: [MessageId], buffer: WriteBuffer) { var length: Int32 = Int32(array.count) buffer.write(&length, offset: 0, length: 4) for id in array { @@ -62,7 +62,7 @@ public struct MessageId: Hashable, Comparable, CustomStringConvertible { } } - public static func decodeArrayFromBuffer(buffer: ReadBuffer) -> [MessageId] { + public static func decodeArrayFromBuffer(_ buffer: ReadBuffer) -> [MessageId] { var length: Int32 = 0 memcpy(&length, buffer.memory, 4) buffer.offset += 4 @@ -157,7 +157,7 @@ public func <(lhs: MessageIndex, rhs: MessageIndex) -> Bool { return lhs.id.id < rhs.id.id } -public struct MessageTags: OptionSetType { +public struct MessageTags: OptionSet { public var rawValue: UInt32 public init(rawValue: UInt32) { @@ -171,7 +171,7 @@ public struct MessageTags: OptionSetType { public static let All = MessageTags(rawValue: 0xffffffff) } -public struct MessageFlags: OptionSetType { +public struct MessageFlags: OptionSet { public var rawValue: UInt32 public init(rawValue: UInt32) { @@ -292,7 +292,7 @@ public final class Message { } } -public struct StoreMessageFlags: OptionSetType { +public struct StoreMessageFlags: OptionSet { public var rawValue: UInt32 public init(rawValue: UInt32) { diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 7c6a2b6a41..3d6c9906e8 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -73,7 +73,7 @@ private let HistoryEntryTypeMessage: Int8 = 0 private let HistoryEntryTypeHole: Int8 = 1 private let HistoryEntryMessageFlagIncoming: Int8 = 1 << 1 -private func readHistoryIndexEntry(peerId: PeerId, namespace: MessageId.Namespace, key: ValueBoxKey, value: ReadBuffer) -> HistoryIndexEntry { +private func readHistoryIndexEntry(_ peerId: PeerId, namespace: MessageId.Namespace, key: ValueBoxKey, value: ReadBuffer) -> HistoryIndexEntry { var flags: Int8 = 0 value.read(&flags, offset: 0, length: 1) var timestamp: Int32 = 0 @@ -113,7 +113,7 @@ final class MessageHistoryIndexTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func key(id: MessageId) -> ValueBoxKey { + private func key(_ id: MessageId) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 4 + 4) key.setInt64(0, value: id.peerId.toInt64()) key.setInt32(8, value: id.namespace) @@ -121,21 +121,21 @@ final class MessageHistoryIndexTable: Table { return key } - private func lowerBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + private func lowerBound(_ peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 4) key.setInt64(0, value: peerId.toInt64()) key.setInt32(8, value: namespace) return key } - private func upperBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + private func upperBound(_ peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 4) key.setInt64(0, value: peerId.toInt64()) key.setInt32(8, value: namespace) return key.successor } - private func ensureInitialized(peerId: PeerId, inout operations: [MessageHistoryIndexOperation]) { + private func ensureInitialized(_ peerId: PeerId, operations: inout [MessageHistoryIndexOperation]) { if !self.metadataTable.isInitialized(peerId) { for namespace in self.seedConfiguration.initializeMessageNamespacesWithHoles { self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: Int32.max), timestamp: Int32.max), min: 1, tags: MessageTags.All.rawValue), operations: &operations) @@ -145,7 +145,7 @@ final class MessageHistoryIndexTable: Table { } } - func addHole(id: MessageId, inout operations: [MessageHistoryIndexOperation]) { + func addHole(_ id: MessageId, operations: inout [MessageHistoryIndexOperation]) { self.ensureInitialized(id.peerId, operations: &operations) let adjacent = self.adjacentItems(id) @@ -186,7 +186,7 @@ final class MessageHistoryIndexTable: Table { } } - func addMessages(messages: [InternalStoreMessage], location: AddMessagesLocation, inout operations: [MessageHistoryIndexOperation]) { + func addMessages(_ messages: [InternalStoreMessage], location: AddMessagesLocation, operations: inout [MessageHistoryIndexOperation]) { if messages.count == 0 { return } @@ -281,7 +281,7 @@ final class MessageHistoryIndexTable: Table { } } - func removeMessage(id: MessageId, inout operations: [MessageHistoryIndexOperation]) { + func removeMessage(_ id: MessageId, operations: inout [MessageHistoryIndexOperation]) { self.ensureInitialized(id.peerId, operations: &operations) if let existingEntry = self.get(id) { @@ -332,7 +332,7 @@ final class MessageHistoryIndexTable: Table { } } - func updateMessage(id: MessageId, message: InternalStoreMessage, inout operations: [MessageHistoryIndexOperation]) { + func updateMessage(_ id: MessageId, message: InternalStoreMessage, operations: inout [MessageHistoryIndexOperation]) { if let previousEntry = self.get(id), case let .Message(previousIndex) = previousEntry { if previousIndex != MessageIndex(message) { var intermediateOperations: [MessageHistoryIndexOperation] = [] @@ -355,7 +355,7 @@ final class MessageHistoryIndexTable: Table { } } - func fillHole(id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [InternalStoreMessage], inout operations: [MessageHistoryIndexOperation]) { + func fillHole(_ id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [InternalStoreMessage], operations: inout [MessageHistoryIndexOperation]) { self.ensureInitialized(id.peerId, operations: &operations) var upperItem: HistoryIndexEntry? @@ -364,7 +364,7 @@ final class MessageHistoryIndexTable: Table { return true }, limit: 1) - let sortedByIdMessages = messages.sort({$0.id < $1.id}) + let sortedByIdMessages = messages.sorted(isOrderedBefore: {$0.id < $1.id}) var remainingMessages = sortedByIdMessages @@ -382,7 +382,7 @@ final class MessageHistoryIndexTable: Table { let message = remainingMessages[i] if message.id.id >= upperHole.min && message.id.id <= upperHole.maxIndex.id.id { messagesInRange.append(message) - remainingMessages.removeAtIndex(i) + remainingMessages.remove(at: i) } else { i += 1 } @@ -460,7 +460,7 @@ final class MessageHistoryIndexTable: Table { } self.justInsertMessage(message, operations: &operations) - remainingMessages.removeAtIndex(i) + remainingMessages.remove(at: i) } else { i += 1 } @@ -522,7 +522,7 @@ final class MessageHistoryIndexTable: Table { } } - private func justInsertHole(hole: MessageHistoryHole, inout operations: [MessageHistoryIndexOperation]) { + private func justInsertHole(_ hole: MessageHistoryHole, operations: inout [MessageHistoryIndexOperation]) { let value = WriteBuffer() var flags: Int8 = HistoryEntryTypeHole var timestamp: Int32 = hole.maxIndex.timestamp @@ -539,7 +539,7 @@ final class MessageHistoryIndexTable: Table { operations.append(.InsertHole(hole)) } - private func justInsertMessage(message: InternalStoreMessage, inout operations: [MessageHistoryIndexOperation]) { + private func justInsertMessage(_ message: InternalStoreMessage, operations: inout [MessageHistoryIndexOperation]) { let index = MessageIndex(id: message.id, timestamp: message.timestamp) let value = WriteBuffer() @@ -558,7 +558,7 @@ final class MessageHistoryIndexTable: Table { } } - private func justRemove(index: MessageIndex, inout operations: [MessageHistoryIndexOperation]) { + private func justRemove(_ index: MessageIndex, operations: inout [MessageHistoryIndexOperation]) { self.valueBox.remove(self.tableId, key: self.key(index.id)) operations.append(.Remove(index)) @@ -567,7 +567,7 @@ final class MessageHistoryIndexTable: Table { } } - func adjacentItems(id: MessageId, bindUpper: Bool = true) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) { + func adjacentItems(_ id: MessageId, bindUpper: Bool = true) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) { let key = self.key(id) var lowerItem: HistoryIndexEntry? @@ -585,7 +585,7 @@ final class MessageHistoryIndexTable: Table { return (lower: lowerItem, upper: upperItem) } - func get(id: MessageId) -> HistoryIndexEntry? { + func get(_ id: MessageId) -> HistoryIndexEntry? { var operations: [MessageHistoryIndexOperation] = [] self.ensureInitialized(id.peerId, operations: &operations) @@ -596,7 +596,7 @@ final class MessageHistoryIndexTable: Table { return nil } - func top(peerId: PeerId, namespace: MessageId.Namespace) -> HistoryIndexEntry? { + func top(_ peerId: PeerId, namespace: MessageId.Namespace) -> HistoryIndexEntry? { var operations: [MessageHistoryIndexOperation] = [] self.ensureInitialized(peerId, operations: &operations) @@ -609,11 +609,11 @@ final class MessageHistoryIndexTable: Table { return entry } - func exists(id: MessageId) -> Bool { + func exists(_ id: MessageId) -> Bool { return self.valueBox.exists(self.tableId, key: self.key(id)) } - func holeContainingId(id: MessageId) -> MessageHistoryHole? { + func holeContainingId(_ id: MessageId) -> MessageHistoryHole? { var result: MessageHistoryHole? self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: id.peerId, namespace: id.namespace, id: id.id)).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in if case let .Hole(hole) = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) { @@ -625,7 +625,7 @@ final class MessageHistoryIndexTable: Table { return result } - func incomingMessageCountInRange(peerId: PeerId, namespace: MessageId.Namespace, minId: MessageId.Id, maxId: MessageId.Id) -> (Int, Bool) { + func incomingMessageCountInRange(_ peerId: PeerId, namespace: MessageId.Namespace, minId: MessageId.Id, maxId: MessageId.Id) -> (Int, Bool) { var count = 0 var holes = false @@ -657,7 +657,7 @@ final class MessageHistoryIndexTable: Table { return (count, holes) } - func incomingMessageCountInIds(peerId: PeerId, namespace: MessageId.Namespace, ids: [MessageId.Id]) -> (Int, Bool) { + func incomingMessageCountInIds(_ peerId: PeerId, namespace: MessageId.Namespace, ids: [MessageId.Id]) -> (Int, Bool) { var count = 0 var holes = false @@ -686,7 +686,7 @@ final class MessageHistoryIndexTable: Table { return (count, holes) } - func debugList(peerId: PeerId, namespace: MessageId.Namespace) -> [HistoryIndexEntry] { + func debugList(_ peerId: PeerId, namespace: MessageId.Namespace) -> [HistoryIndexEntry] { var list: [HistoryIndexEntry] = [] self.valueBox.range(self.tableId, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in list.append(readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value)) diff --git a/Postbox/MessageHistoryInvalidatedReadStateTable.swift b/Postbox/MessageHistoryInvalidatedReadStateTable.swift index 57f2628803..0aaf0ebd8b 100644 --- a/Postbox/MessageHistoryInvalidatedReadStateTable.swift +++ b/Postbox/MessageHistoryInvalidatedReadStateTable.swift @@ -46,7 +46,7 @@ final class MessageHistorySynchronizeReadStateTable: Table { return key } - func set(peerId: PeerId, operation: PeerReadStateSynchronizationOperation?, inout operations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func set(_ peerId: PeerId, operation: PeerReadStateSynchronizationOperation?, operations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { self.updatedPeerIds[peerId] = operation operations[peerId] = operation } diff --git a/Postbox/MessageHistoryMetadataTable.swift b/Postbox/MessageHistoryMetadataTable.swift index ef0368eda1..382bd13a6b 100644 --- a/Postbox/MessageHistoryMetadataTable.swift +++ b/Postbox/MessageHistoryMetadataTable.swift @@ -25,13 +25,13 @@ final class MessageHistoryMetadataTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func peerHistoryInitializedKey(id: PeerId) -> ValueBoxKey { + private func peerHistoryInitializedKey(_ id: PeerId) -> ValueBoxKey { self.sharedPeerHistoryInitializedKey.setInt64(0, value: id.toInt64()) self.sharedPeerHistoryInitializedKey.setInt8(8, value: MetadataPrefix.PeerHistoryInitialized.rawValue) return self.sharedPeerHistoryInitializedKey } - private func peerNextMessageIdByNamespaceKey(id: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + private func peerNextMessageIdByNamespaceKey(_ id: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { self.sharedPeerNextMessageIdByNamespaceKey.setInt64(0, value: id.toInt64()) self.sharedPeerNextMessageIdByNamespaceKey.setInt8(8, value: MetadataPrefix.PeerNextMessageIdByNamespace.rawValue) self.sharedPeerNextMessageIdByNamespaceKey.setInt32(8 + 1, value: namespace) @@ -39,7 +39,7 @@ final class MessageHistoryMetadataTable: Table { return self.sharedPeerNextMessageIdByNamespaceKey } - private func key(prefix: MetadataPrefix) -> ValueBoxKey { + private func key(_ prefix: MetadataPrefix) -> ValueBoxKey { let key = ValueBoxKey(length: 1) key.setInt8(0, value: prefix.rawValue) return key @@ -47,7 +47,7 @@ final class MessageHistoryMetadataTable: Table { func setInitializedChatList() { self.initializedChatList = true - self.valueBox.set(self.tableId, key: self.key(MetadataPrefix.ChatListInitialized), value: MemoryBuffer(memory: nil, capacity: 0, length: 0, freeWhenDone: false)) + self.valueBox.set(self.tableId, key: self.key(MetadataPrefix.ChatListInitialized), value: MemoryBuffer()) } func isInitializedChatList() -> Bool { @@ -63,13 +63,13 @@ final class MessageHistoryMetadataTable: Table { return false } - func setInitialized(peerId: PeerId) { + func setInitialized(_ peerId: PeerId) { self.initializedHistoryPeerIds.insert(peerId) self.sharedBuffer.reset() self.valueBox.set(self.tableId, key: self.peerHistoryInitializedKey(peerId), value: self.sharedBuffer) } - func isInitialized(peerId: PeerId) -> Bool { + func isInitialized(_ peerId: PeerId) -> Bool { if self.initializedHistoryPeerIds.contains(peerId) { return true } else { @@ -82,7 +82,7 @@ final class MessageHistoryMetadataTable: Table { } } - func getNextMessageIdAndIncrement(peerId: PeerId, namespace: MessageId.Namespace) -> MessageId { + func getNextMessageIdAndIncrement(_ peerId: PeerId, namespace: MessageId.Namespace) -> MessageId { if let messageIdByNamespace = self.peerNextMessageIdByNamespace[peerId] { if let nextId = messageIdByNamespace[namespace] { self.peerNextMessageIdByNamespace[peerId]![namespace] = nextId + 1 @@ -166,4 +166,4 @@ final class MessageHistoryMetadataTable: Table { } } } -} \ No newline at end of file +} diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index 648bd13d75..a9fe295875 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -21,7 +21,7 @@ final class MessageHistoryReadStateTable: Table { private let sharedKey = ValueBoxKey(length: 8) - private func key(id: PeerId) -> ValueBoxKey { + private func key(_ id: PeerId) -> ValueBoxKey { self.sharedKey.setInt64(0, value: id.toInt64()) return self.sharedKey } @@ -30,7 +30,7 @@ final class MessageHistoryReadStateTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func get(id: PeerId) -> InternalPeerReadStates? { + private func get(_ id: PeerId) -> InternalPeerReadStates? { if let states = self.cachedPeerReadStates[id] { return states } else { @@ -63,14 +63,14 @@ final class MessageHistoryReadStateTable: Table { } } - func getCombinedState(peerId: PeerId) -> CombinedPeerReadState? { + func getCombinedState(_ peerId: PeerId) -> CombinedPeerReadState? { if let states = self.get(peerId) { return CombinedPeerReadState(states: states.namespaces.map({$0})) } return nil } - func resetStates(peerId: PeerId, namespaces: [MessageId.Namespace: PeerReadState]) -> CombinedPeerReadState? { + func resetStates(_ peerId: PeerId, namespaces: [MessageId.Namespace: PeerReadState]) -> CombinedPeerReadState? { if traceReadStates { print("[ReadStateTable] resetStates peerId: \(peerId), namespaces: \(namespaces)") } @@ -99,7 +99,7 @@ final class MessageHistoryReadStateTable: Table { } } - func addIncomingMessages(peerId: PeerId, ids: Set) -> (CombinedPeerReadState?, Bool) { + func addIncomingMessages(_ peerId: PeerId, ids: Set) -> (CombinedPeerReadState?, Bool) { var idsByNamespace: [MessageId.Namespace: [MessageId.Id]] = [:] for id in ids { if idsByNamespace[id.namespace] != nil { @@ -115,7 +115,7 @@ final class MessageHistoryReadStateTable: Table { } var updated = false - var invalidated = false + let invalidated = false for (namespace, ids) in idsByNamespace { if let currentState = states.namespaces[namespace] { var addedUnreadCount: Int32 = 0 @@ -149,11 +149,9 @@ final class MessageHistoryReadStateTable: Table { } return (nil, true) } - - return (nil, false) } - func deleteMessages(peerId: PeerId, ids: [MessageId], incomingStatsInIds: (PeerId, MessageId.Namespace, [MessageId.Id]) -> (Int, Bool)) -> (CombinedPeerReadState?, Bool) { + func deleteMessages(_ peerId: PeerId, ids: [MessageId], incomingStatsInIds: (PeerId, MessageId.Namespace, [MessageId.Id]) -> (Int, Bool)) -> (CombinedPeerReadState?, Bool) { var idsByNamespace: [MessageId.Namespace: [MessageId.Id]] = [:] for id in ids { if idsByNamespace[id.namespace] != nil { @@ -199,11 +197,9 @@ final class MessageHistoryReadStateTable: Table { } else { return (nil, true) } - - return (nil, false) } - func applyIncomingMaxReadId(messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (CombinedPeerReadState?, Bool) { + func applyIncomingMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (CombinedPeerReadState?, Bool) { if let states = self.get(messageId.peerId), state = states.namespaces[messageId.namespace] { if traceReadStates { print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId.id) (before: \(states.namespaces))") @@ -234,7 +230,7 @@ final class MessageHistoryReadStateTable: Table { return (nil, false) } - func applyOutgoingMaxReadId(messageId: MessageId) -> (CombinedPeerReadState?, Bool) { + func applyOutgoingMaxReadId(_ messageId: MessageId) -> (CombinedPeerReadState?, Bool) { if let states = self.get(messageId.peerId), state = states.namespaces[messageId.namespace] { if state.maxOutgoingReadId < messageId.id { states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: state.maxIncomingReadId, maxOutgoingReadId: state.maxOutgoingReadId, maxKnownId: state.maxKnownId, count: state.count) @@ -248,7 +244,7 @@ final class MessageHistoryReadStateTable: Table { return (nil, false) } - func applyInteractiveMaxReadId(messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult) { + func applyInteractiveMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult) { let (combinedState, holes) = self.applyIncomingMaxReadId(messageId, incomingStatsInRange: incomingStatsInRange, topMessageId: topMessageId) if let combinedState = combinedState { diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 9b0809116d..f40e4f93de 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -62,7 +62,7 @@ final class MessageHistoryTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func key(index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4)) -> ValueBoxKey { + private func key(_ index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4)) -> ValueBoxKey { key.setInt64(0, value: index.id.peerId.toInt64()) key.setInt32(8, value: index.timestamp) key.setInt32(8 + 4, value: index.id.namespace) @@ -70,19 +70,19 @@ final class MessageHistoryTable: Table { return key } - private func lowerBound(peerId: PeerId) -> ValueBoxKey { + private func lowerBound(_ peerId: PeerId) -> ValueBoxKey { let key = ValueBoxKey(length: 8) key.setInt64(0, value: peerId.toInt64()) return key } - private func upperBound(peerId: PeerId) -> ValueBoxKey { + private func upperBound(_ peerId: PeerId) -> ValueBoxKey { let key = ValueBoxKey(length: 8) key.setInt64(0, value: peerId.toInt64()) return key.successor } - private func messagesGroupedByPeerId(messages: [StoreMessage]) -> [PeerId: [StoreMessage]] { + private func messagesGroupedByPeerId(_ messages: [StoreMessage]) -> [PeerId: [StoreMessage]] { var dict: [PeerId: [StoreMessage]] = [:] for message in messages { @@ -97,7 +97,7 @@ final class MessageHistoryTable: Table { return dict } - private func messageIdsByPeerId(ids: [MessageId]) -> [PeerId: [MessageId]] { + private func messageIdsByPeerId(_ ids: [MessageId]) -> [PeerId: [MessageId]] { var dict: [PeerId: [MessageId]] = [:] for id in ids { @@ -112,7 +112,7 @@ final class MessageHistoryTable: Table { return dict } - private func processIndexOperations(peerId: PeerId, operations: [MessageHistoryIndexOperation], inout processedOperationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + private func processIndexOperations(_ peerId: PeerId, operations: [MessageHistoryIndexOperation], processedOperationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0)) let sharedBuffer = WriteBuffer() let sharedEncoder = Encoder() @@ -120,7 +120,6 @@ final class MessageHistoryTable: Table { var outputOperations: [MessageHistoryOperation] = [] var accumulatedRemoveIndices: [MessageIndex] = [] var addedIncomingMessageIds = Set() - var removedMessageIds: [MessageId] = [] for operation in operations { switch operation { case let .InsertHole(hole): @@ -202,11 +201,11 @@ final class MessageHistoryTable: Table { if processedOperationsByPeerId[peerId] == nil { processedOperationsByPeerId[peerId] = outputOperations } else { - processedOperationsByPeerId[peerId]!.appendContentsOf(outputOperations) + processedOperationsByPeerId[peerId]!.append(contentsOf: outputOperations) } } - private func internalStoreMessages(messages: [StoreMessage]) -> [InternalStoreMessage] { + private func internalStoreMessages(_ messages: [StoreMessage]) -> [InternalStoreMessage] { var internalStoreMessages: [InternalStoreMessage] = [] for message in messages { switch message.id { @@ -220,7 +219,7 @@ final class MessageHistoryTable: Table { return internalStoreMessages } - func addMessages(messages: [StoreMessage], location: AddMessagesLocation, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { let messagesByPeerId = self.messagesGroupedByPeerId(messages) for (peerId, peerMessages) in messagesByPeerId { var operations: [MessageHistoryIndexOperation] = [] @@ -229,7 +228,7 @@ final class MessageHistoryTable: Table { } } - func addHoles(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func addHoles(_ messageIds: [MessageId], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] for id in messageIds { @@ -239,7 +238,7 @@ final class MessageHistoryTable: Table { } } - func removeMessages(messageIds: [MessageId], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func removeMessages(_ messageIds: [MessageId], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] @@ -259,7 +258,7 @@ final class MessageHistoryTable: Table { if operationsByPeerId[peerId] == nil { operationsByPeerId[peerId] = outputOperations } else { - operationsByPeerId[peerId]!.appendContentsOf(outputOperations) + operationsByPeerId[peerId]!.append(contentsOf: outputOperations) } } @@ -269,19 +268,19 @@ final class MessageHistoryTable: Table { } } - func fillHole(id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func fillHole(_ id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.fillHole(id, fillType: fillType, tagMask: tagMask, messages: self.internalStoreMessages(messages), operations: &operations) self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - func updateMessage(id: MessageId, message: StoreMessage, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func updateMessage(_ id: MessageId, message: StoreMessage, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.updateMessage(id, message: self.internalStoreMessages([message]).first!, operations: &operations) self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - func updateMedia(id: MediaId, media: Media?, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedMedia: [MediaId: Media?]) { + func updateMedia(_ id: MediaId, media: Media?, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedMedia: inout [MediaId: Media?]) { if let previousMedia = self.messageMediaTable.get(id, embedded: { index, id in return self.embeddedMediaForIndex(index, id: id) }) { @@ -307,7 +306,7 @@ final class MessageHistoryTable: Table { } } - func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]], inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { for (peerId, namespaces) in states { if let combinedState = self.readStateTable.resetStates(peerId, namespaces: namespaces) { if operationsByPeerId[peerId] == nil { @@ -320,7 +319,7 @@ final class MessageHistoryTable: Table { } } - func applyIncomingReadMaxId(messageId: MessageId, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func applyIncomingReadMaxId(_ messageId: MessageId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { var topMessageId: MessageId.Id? if let topEntry = self.messageHistoryIndexTable.top(messageId.peerId, namespace: messageId.namespace), case let .Message(index) = topEntry { topMessageId = index.id.id @@ -343,7 +342,7 @@ final class MessageHistoryTable: Table { } } - func applyOutgoingReadMaxId(messageId: MessageId, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func applyOutgoingReadMaxId(_ messageId: MessageId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { let (combinedState, invalidated) = self.readStateTable.applyOutgoingMaxReadId(messageId) if let combinedState = combinedState { @@ -359,7 +358,7 @@ final class MessageHistoryTable: Table { } } - func applyInteractiveMaxReadId(messageId: MessageId, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], inout updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?]) { + func applyInteractiveMaxReadId(_ messageId: MessageId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { var topMessageId: MessageId.Id? if let topEntry = self.messageHistoryIndexTable.top(messageId.peerId, namespace: messageId.namespace), case let .Message(index) = topEntry { topMessageId = index.id.id @@ -385,7 +384,7 @@ final class MessageHistoryTable: Table { } } - func topMessage(peerId: PeerId) -> IntermediateMessage? { + func topMessage(_ peerId: PeerId) -> IntermediateMessage? { var currentKey = self.upperBound(peerId) while true { var entry: IntermediateMessageHistoryEntry? @@ -408,7 +407,7 @@ final class MessageHistoryTable: Table { return nil } - func getMessage(index: MessageIndex) -> IntermediateMessage? { + func getMessage(_ index: MessageIndex) -> IntermediateMessage? { let key = self.key(index) if let value = self.valueBox.get(self.tableId, key: key) { let entry = self.readIntermediateEntry(key, value: value) @@ -419,7 +418,7 @@ final class MessageHistoryTable: Table { return nil } - private func justInsertMessage(message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) -> IntermediateMessage { + private func justInsertMessage(_ message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) -> IntermediateMessage { sharedBuffer.reset() var type: Int8 = 0 @@ -479,10 +478,10 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&hasAuthor, offset: 0, length: 1) } - let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! - var length: Int32 = Int32(data.length) + let data = message.text.data(using: .utf8, allowLossyConversion: true)! + var length: Int32 = Int32(data.count) sharedBuffer.write(&length, offset: 0, length: 4) - sharedBuffer.write(data.bytes, offset: 0, length: Int(length)) + sharedBuffer.write(data) let attributesBuffer = WriteBuffer() @@ -543,7 +542,7 @@ final class MessageHistoryTable: Table { return IntermediateMessage(stableId: stableId, id: message.id, timestamp: message.timestamp, flags: flags, tags: message.tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) } - private func justInsertHole(hole: MessageHistoryHole, sharedBuffer: WriteBuffer = WriteBuffer()) { + private func justInsertHole(_ hole: MessageHistoryHole, sharedBuffer: WriteBuffer = WriteBuffer()) { sharedBuffer.reset() var type: Int8 = 1 sharedBuffer.write(&type, offset: 0, length: 1) @@ -556,7 +555,7 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.tableId, key: self.key(hole.maxIndex), value: sharedBuffer.readBufferNoCopy()) } - private func justRemove(index: MessageIndex, inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) { + private func justRemove(_ index: MessageIndex, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) { let key = self.key(index) if let value = self.valueBox.get(self.tableId, key: key) { switch self.readIntermediateEntry(key, value: value) { @@ -595,7 +594,7 @@ final class MessageHistoryTable: Table { } for mediaId in message.referencedMedia { - self.messageMediaTable.removeReference(mediaId) + let _ = self.messageMediaTable.removeReference(mediaId) } case let .Hole(hole): let tags = self.messageHistoryIndexTable.seedConfiguration.existingMessageTags.rawValue & hole.tags @@ -618,15 +617,13 @@ final class MessageHistoryTable: Table { } } - func embeddedMediaForIndex(index: MessageIndex, id: MediaId) -> Media? { + func embeddedMediaForIndex(_ index: MessageIndex, id: MediaId) -> Media? { if let message = self.getMessage(index) where message.embeddedMediaData.length > 4 { var embeddedMediaCount: Int32 = 0 message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) for _ in 0 ..< embeddedMediaCount { - let mediaOffset = message.embeddedMediaData.offset var mediaLength: Int32 = 0 - var copyMedia = true message.embeddedMediaData.read(&mediaLength, offset: 0, length: 4) if let readMedia = Decoder(buffer: MemoryBuffer(memory: message.embeddedMediaData.memory + message.embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { @@ -641,16 +638,14 @@ final class MessageHistoryTable: Table { return nil } - func updateEmbeddedMedia(index: MessageIndex, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], @noescape update: ([Media]) -> [Media]) { + func updateEmbeddedMedia(_ index: MessageIndex, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], update: @noescape([Media]) -> [Media]) { if let message = self.getMessage(index) { var embeddedMediaCount: Int32 = 0 message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) var previousMedia: [Media] = [] for _ in 0 ..< embeddedMediaCount { - let mediaOffset = message.embeddedMediaData.offset var mediaLength: Int32 = 0 - var copyMedia = true message.embeddedMediaData.read(&mediaLength, offset: 0, length: 4) if let readMedia = Decoder(buffer: MemoryBuffer(memory: message.embeddedMediaData.memory + message.embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { previousMedia.append(readMedia) @@ -700,7 +695,7 @@ final class MessageHistoryTable: Table { } } - func updateEmbeddedMedia(index: MessageIndex, mediaId: MediaId, media: Media?, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]]) { + func updateEmbeddedMedia(_ index: MessageIndex, mediaId: MediaId, media: Media?, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]]) { self.updateEmbeddedMedia(index, operationsByPeerId: &operationsByPeerId, update: { previousMedia in var updatedMedia: [Media] = [] for previous in previousMedia { @@ -716,7 +711,7 @@ final class MessageHistoryTable: Table { }) } - private func justUpdate(index: MessageIndex, message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder, inout unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation]) -> IntermediateMessage? { + private func justUpdate(_ index: MessageIndex, message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) -> IntermediateMessage? { if let previousMessage = self.getMessage(index) { self.valueBox.remove(self.tableId, key: self.key(index)) @@ -825,10 +820,10 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&hasAuthor, offset: 0, length: 1) } - let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! - var length: Int32 = Int32(data.length) + let data = message.text.data(using: .utf8, allowLossyConversion: true)! + var length: Int32 = Int32(data.count) sharedBuffer.write(&length, offset: 0, length: 4) - sharedBuffer.write(data.bytes, offset: 0, length: Int(length)) + sharedBuffer.write(data) let attributesBuffer = WriteBuffer() @@ -892,7 +887,7 @@ final class MessageHistoryTable: Table { } } - func unembedMedia(index: MessageIndex, id: MediaId) -> Media? { + func unembedMedia(_ index: MessageIndex, id: MediaId) -> Media? { if let message = self.getMessage(index) where message.embeddedMediaData.length > 4 { var embeddedMediaCount: Int32 = 0 message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) @@ -932,7 +927,7 @@ final class MessageHistoryTable: Table { return nil } - func storeIntermediateMessage(message: IntermediateMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer = WriteBuffer()) { + func storeIntermediateMessage(_ message: IntermediateMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer = WriteBuffer()) { sharedBuffer.reset() var type: Int8 = 0 @@ -989,10 +984,10 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&hasAuthor, offset: 0, length: 1) } - let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! - var length: Int32 = Int32(data.length) + let data = message.text.data(using: .utf8, allowLossyConversion: true)! + var length: Int32 = Int32(data.count) sharedBuffer.write(&length, offset: 0, length: 4) - sharedBuffer.write(data.bytes, offset: 0, length: Int(length)) + sharedBuffer.write(data) sharedBuffer.write(message.attributesData.memory, offset: 0, length: message.attributesData.length) sharedBuffer.write(message.embeddedMediaData.memory, offset: 0, length: message.embeddedMediaData.length) @@ -1009,7 +1004,7 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.tableId, key: self.key(MessageIndex(id: message.id, timestamp: message.timestamp), key: sharedKey), value: sharedBuffer) } - private func readIntermediateEntry(key: ValueBoxKey, value: ReadBuffer) -> IntermediateMessageHistoryEntry { + private func readIntermediateEntry(_ key: ValueBoxKey, value: ReadBuffer) -> IntermediateMessageHistoryEntry { let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4), id: key.getInt32(8 + 4 + 4)), timestamp: key.getInt32(8)) var type: Int8 = 0 @@ -1068,7 +1063,7 @@ final class MessageHistoryTable: Table { var textLength: Int32 = 0 value.read(&textLength, offset: 0, length: 4) - let text = String(data: NSData(bytes: value.memory + value.offset, length: Int(textLength)), encoding: NSUTF8StringEncoding) ?? "" + let text = String(data: Data(bytes: UnsafeMutablePointer(value.memory).advanced(by: value.offset), count: Int(textLength)), encoding: .utf8) ?? "" //let text = NSString(bytes: value.memory + value.offset, length: Int(textLength), encoding: NSUTF8StringEncoding) ?? "" value.skip(Int(textLength)) @@ -1081,7 +1076,7 @@ final class MessageHistoryTable: Table { value.skip(Int(attributeLength)) } let attributesLength = value.offset - attributesOffset - let attributesBytes = malloc(attributesLength) + let attributesBytes = malloc(attributesLength)! memcpy(attributesBytes, value.memory + attributesOffset, attributesLength) let attributesData = ReadBuffer(memory: attributesBytes, length: attributesLength, freeWhenDone: true) @@ -1094,7 +1089,7 @@ final class MessageHistoryTable: Table { value.skip(Int(mediaLength)) } let embeddedMediaLength = value.offset - embeddedMediaOffset - let embeddedMediaBytes = malloc(embeddedMediaLength) + let embeddedMediaBytes = malloc(embeddedMediaLength)! memcpy(embeddedMediaBytes, value.memory + embeddedMediaOffset, embeddedMediaLength) let embeddedMediaData = ReadBuffer(memory: embeddedMediaBytes, length: embeddedMediaLength, freeWhenDone: true) @@ -1123,7 +1118,7 @@ final class MessageHistoryTable: Table { } } - func renderMessage(message: IntermediateMessage, peerTable: PeerTable, addAssociatedMessages: Bool = true) -> Message { + func renderMessage(_ message: IntermediateMessage, peerTable: PeerTable, addAssociatedMessages: Bool = true) -> Message { var parsedAttributes: [MessageAttribute] = [] var parsedMedia: [Media] = [] @@ -1212,7 +1207,7 @@ final class MessageHistoryTable: Table { return Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages) } - func entriesAround(index: MessageIndex, count: Int) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { + func entriesAround(_ index: MessageIndex, count: Int) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { var lowerEntries: [IntermediateMessageHistoryEntry] = [] var upperEntries: [IntermediateMessageHistoryEntry] = [] var lower: IntermediateMessageHistoryEntry? @@ -1247,16 +1242,16 @@ final class MessageHistoryTable: Table { lower = additionalLowerEntries.last additionalLowerEntries.removeLast() } - lowerEntries.appendContentsOf(additionalLowerEntries) + lowerEntries.append(contentsOf: additionalLowerEntries) } var entries: [IntermediateMessageHistoryEntry] = [] - entries.appendContentsOf(lowerEntries.reverse()) - entries.appendContentsOf(upperEntries) + entries.append(contentsOf: lowerEntries.reversed()) + entries.append(contentsOf: upperEntries) return (entries: entries, lower: lower, upper: upper) } - func entriesAround(tagMask: MessageTags, index: MessageIndex, count: Int) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { + func entriesAround(_ tagMask: MessageTags, index: MessageIndex, count: Int) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { let (indices, lower, upper) = self.tagsTable.indicesAround(tagMask, index: index, count: count) var entries: [IntermediateMessageHistoryEntry] = [] @@ -1293,7 +1288,7 @@ final class MessageHistoryTable: Table { return (entries, lowerEntry, upperEntry) } - func earlierEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + func earlierEntries(_ peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { var entries: [IntermediateMessageHistoryEntry] = [] let key: ValueBoxKey if let index = index { @@ -1308,7 +1303,7 @@ final class MessageHistoryTable: Table { return entries } - func earlierEntries(tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + func earlierEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { let indices = self.tagsTable.earlierIndices(tagMask, peerId: peerId, index: index, count: count) var entries: [IntermediateMessageHistoryEntry] = [] @@ -1324,7 +1319,7 @@ final class MessageHistoryTable: Table { return entries } - func laterEntries(peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + func laterEntries(_ peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { var entries: [IntermediateMessageHistoryEntry] = [] let key: ValueBoxKey if let index = index { @@ -1339,7 +1334,7 @@ final class MessageHistoryTable: Table { return entries } - func laterEntries(tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + func laterEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { let indices = self.tagsTable.laterIndices(tagMask, peerId: peerId, index: index, count: count) var entries: [IntermediateMessageHistoryEntry] = [] @@ -1355,14 +1350,14 @@ final class MessageHistoryTable: Table { return entries } - func maxReadIndex(peerId: PeerId) -> MessageHistoryAnchorIndex? { + func maxReadIndex(_ peerId: PeerId) -> MessageHistoryAnchorIndex? { if let combinedState = self.readStateTable.getCombinedState(peerId), state = combinedState.states.first where state.1.count != 0 { return self.anchorIndex(MessageId(peerId: peerId, namespace: state.0, id: state.1.maxIncomingReadId)) } return nil } - func anchorIndex(messageId: MessageId) -> MessageHistoryAnchorIndex? { + func anchorIndex(_ messageId: MessageId) -> MessageHistoryAnchorIndex? { let (lower, upper) = self.messageHistoryIndexTable.adjacentItems(messageId, bindUpper: false) if let lower = lower, case let .Hole(hole) = lower where messageId.id >= hole.min && messageId.id <= hole.maxIndex.id.id { return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: lower.index.timestamp), exact: false) @@ -1379,7 +1374,7 @@ final class MessageHistoryTable: Table { return nil } - func debugList(peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { + func debugList(_ peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { return self.laterEntries(peerId, index: nil, count: 1000).map({ entry -> RenderedMessageHistoryEntry in switch entry { case let .Hole(hole): @@ -1390,7 +1385,7 @@ final class MessageHistoryTable: Table { }) } - func debugList(tagMask: MessageTags, peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { + func debugList(_ tagMask: MessageTags, peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { return self.laterEntries(tagMask, peerId: peerId, index: nil, count: 1000).map({ entry -> RenderedMessageHistoryEntry in switch entry { case let .Hole(hole): diff --git a/Postbox/MessageHistoryTagsTable.swift b/Postbox/MessageHistoryTagsTable.swift index 908b36b014..822ec4d25e 100644 --- a/Postbox/MessageHistoryTagsTable.swift +++ b/Postbox/MessageHistoryTagsTable.swift @@ -7,7 +7,7 @@ class MessageHistoryTagsTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func key(tagMask: MessageTags, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4 + 4)) -> ValueBoxKey { + private func key(_ tagMask: MessageTags, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4 + 4)) -> ValueBoxKey { key.setInt64(0, value: index.id.peerId.toInt64()) key.setUInt32(8, value: tagMask.rawValue) key.setInt32(8 + 4, value: index.timestamp) @@ -16,29 +16,29 @@ class MessageHistoryTagsTable: Table { return key } - private func lowerBound(tagMask: MessageTags, peerId: PeerId) -> ValueBoxKey { + private func lowerBound(_ tagMask: MessageTags, peerId: PeerId) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 4) key.setInt64(0, value: peerId.toInt64()) key.setUInt32(8, value: tagMask.rawValue) return key } - private func upperBound(tagMask: MessageTags, peerId: PeerId) -> ValueBoxKey { + private func upperBound(_ tagMask: MessageTags, peerId: PeerId) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 4) key.setInt64(0, value: peerId.toInt64()) key.setUInt32(8, value: tagMask.rawValue) return key.successor } - func add(tagMask: MessageTags, index: MessageIndex) { + func add(_ tagMask: MessageTags, index: MessageIndex) { self.valueBox.set(self.tableId, key: self.key(tagMask, index: index, key: self.sharedKey), value: MemoryBuffer()) } - func remove(tagMask: MessageTags, index: MessageIndex) { + func remove(_ tagMask: MessageTags, index: MessageIndex) { self.valueBox.remove(self.tableId, key: self.key(tagMask, index: index, key: self.sharedKey)) } - func indicesAround(tagMask: MessageTags, index: MessageIndex, count: Int) -> (indices: [MessageIndex], lower: MessageIndex?, upper: MessageIndex?) { + func indicesAround(_ tagMask: MessageTags, index: MessageIndex, count: Int) -> (indices: [MessageIndex], lower: MessageIndex?, upper: MessageIndex?) { var lowerEntries: [MessageIndex] = [] var upperEntries: [MessageIndex] = [] var lower: MessageIndex? @@ -76,16 +76,16 @@ class MessageHistoryTagsTable: Table { lower = additionalLowerEntries.last additionalLowerEntries.removeLast() } - lowerEntries.appendContentsOf(additionalLowerEntries) + lowerEntries.append(contentsOf: additionalLowerEntries) } var entries: [MessageIndex] = [] - entries.appendContentsOf(lowerEntries.reverse()) - entries.appendContentsOf(upperEntries) + entries.append(contentsOf: lowerEntries.reversed()) + entries.append(contentsOf: upperEntries) return (indices: entries, lower: lower, upper: upper) } - func earlierIndices(tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [MessageIndex] { + func earlierIndices(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [MessageIndex] { var indices: [MessageIndex] = [] let key: ValueBoxKey if let index = index { @@ -101,7 +101,7 @@ class MessageHistoryTagsTable: Table { return indices } - func laterIndices(tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [MessageIndex] { + func laterIndices(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [MessageIndex] { var indices: [MessageIndex] = [] let key: ValueBoxKey if let index = index { diff --git a/Postbox/MessageHistoryUnsentTable.swift b/Postbox/MessageHistoryUnsentTable.swift index 1e7fd9773d..ccb5e8627b 100644 --- a/Postbox/MessageHistoryUnsentTable.swift +++ b/Postbox/MessageHistoryUnsentTable.swift @@ -8,7 +8,7 @@ enum IntermediateMessageHistoryUnsentOperation { final class MessageHistoryUnsentTable: Table { private let sharedKey = ValueBoxKey(length: 4 + 4 + 4 + 8) - private func key(index: MessageIndex) -> ValueBoxKey { + private func key(_ index: MessageIndex) -> ValueBoxKey { self.sharedKey.setInt32(0, value: index.timestamp) self.sharedKey.setInt32(4, value: index.id.namespace) self.sharedKey.setInt32(4 + 4, value: index.id.id) @@ -33,12 +33,12 @@ final class MessageHistoryUnsentTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - func add(index: MessageIndex, inout operations: [IntermediateMessageHistoryUnsentOperation]) { + func add(_ index: MessageIndex, operations: inout [IntermediateMessageHistoryUnsentOperation]) { self.valueBox.set(self.tableId, key: self.key(index), value: MemoryBuffer()) operations.append(.Insert(index)) } - func remove(index: MessageIndex, inout operations: [IntermediateMessageHistoryUnsentOperation]) { + func remove(_ index: MessageIndex, operations: inout [IntermediateMessageHistoryUnsentOperation]) { self.valueBox.remove(self.tableId, key: self.key(index)) operations.append(.Remove(index)) } @@ -55,4 +55,4 @@ final class MessageHistoryUnsentTable: Table { override func beforeCommit() { } -} \ No newline at end of file +} diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index c2b8c41722..e48b4e0a1b 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -112,7 +112,7 @@ final class MutableMessageHistoryView { self.id = self.id.nextVersion } - func updateVisibleRange(earliestVisibleIndex earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex, context: MutableMessageHistoryViewReplayContext) -> Bool { + func updateVisibleRange(earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex, context: MutableMessageHistoryViewReplayContext) -> Bool { if (true) { //return false } @@ -127,7 +127,7 @@ final class MutableMessageHistoryView { } } - for i in (0 ..< self.entries.count).reverse() { + for i in (0 ..< self.entries.count).reversed() { if self.entries[i].index <= latestVisibleIndex { maxIndex = i break @@ -180,7 +180,7 @@ final class MutableMessageHistoryView { return false } - func updateAnchorIndex(getIndex: (MessageId) -> MessageHistoryAnchorIndex?) -> Bool { + func updateAnchorIndex(_ getIndex: (MessageId) -> MessageHistoryAnchorIndex?) -> Bool { if !self.anchorIndex.exact { if let index = getIndex(self.anchorIndex.index.id) { self.anchorIndex = index @@ -190,7 +190,7 @@ final class MutableMessageHistoryView { return false } - func replay(operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], context: MutableMessageHistoryViewReplayContext) -> Bool { + func replay(_ operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], context: MutableMessageHistoryViewReplayContext) -> Bool { let tagMask = self.tagMask let unwrappedTagMask: UInt32 = tagMask?.rawValue ?? 0 @@ -233,7 +233,7 @@ final class MutableMessageHistoryView { case let .MessageEntry(message): var rebuild = false for media in message.media { - if let mediaId = media.id, updated = updatedMedia[mediaId] { + if let mediaId = media.id, _ = updatedMedia[mediaId] { rebuild = true break } @@ -250,7 +250,7 @@ final class MutableMessageHistoryView { messageMedia.append(media) } } - var updatedMessage = Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages) + let updatedMessage = Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages) self.entries[i] = .MessageEntry(updatedMessage) hasChanges = true } @@ -284,7 +284,7 @@ final class MutableMessageHistoryView { return hasChanges } - private func add(entry: MutableMessageHistoryEntry, holeFillDirections: [MessageIndex: HoleFillDirection]) -> Bool { + private func add(_ entry: MutableMessageHistoryEntry, holeFillDirections: [MessageIndex: HoleFillDirection]) -> Bool { if self.entries.count == 0 { self.entries.append(entry) return true @@ -292,16 +292,11 @@ final class MutableMessageHistoryView { let latestIndex = self.entries[self.entries.count - 1].index let earliestIndex = self.entries[0].index - var next: MessageIndex? - if let later = self.later { - next = later.index - } - let index = entry.index if index < earliestIndex { if self.earlier == nil || self.earlier!.index < index { - self.entries.insert(entry, atIndex: 0) + self.entries.insert(entry, at: 0) return true } else { return false @@ -326,7 +321,7 @@ final class MutableMessageHistoryView { } i -= 1 } - self.entries.insert(entry, atIndex: i) + self.entries.insert(entry, at: i) return true } else { return false @@ -334,7 +329,7 @@ final class MutableMessageHistoryView { } } - private func remove(indices: Set, context: MutableMessageHistoryViewReplayContext) -> Bool { + private func remove(_ indices: Set, context: MutableMessageHistoryViewReplayContext) -> Bool { var hasChanges = false if let earlier = self.earlier where indices.contains(earlier.index) { context.invalidEarlier = true @@ -350,7 +345,7 @@ final class MutableMessageHistoryView { var i = self.entries.count - 1 while i >= 0 { if indices.contains(self.entries[i].index) { - self.entries.removeAtIndex(i) + self.entries.remove(at: i) context.removedEntries = true hasChanges = true } @@ -361,7 +356,7 @@ final class MutableMessageHistoryView { return hasChanges } - func updatePeers(peers: [PeerId: Peer]) -> Bool { + func updatePeers(_ peers: [PeerId: Peer]) -> Bool { return false } @@ -373,7 +368,7 @@ final class MutableMessageHistoryView { let fetchedEntries = fetchEarlier(anchorIndex, self.fillCount + 2) if fetchedEntries.count >= self.fillCount + 2 { self.earlier = fetchedEntries.last - for i in (1 ..< fetchedEntries.count - 1).reverse() { + for i in (1 ..< fetchedEntries.count - 1).reversed() { self.entries.append(fetchedEntries[i]) } self.later = fetchedEntries.first @@ -381,7 +376,7 @@ final class MutableMessageHistoryView { } else { let fetchedEntries = fetchEarlier(self.entries[0].index, self.fillCount - self.entries.count) for entry in fetchedEntries { - self.entries.insert(entry, atIndex: 0) + self.entries.insert(entry, at: 0) } if context.invalidEarlier { @@ -431,7 +426,7 @@ final class MutableMessageHistoryView { } } - func render(renderIntermediateMessage: IntermediateMessage -> Message) { + func render(_ renderIntermediateMessage: (IntermediateMessage) -> Message) { if let earlier = self.earlier, case let .IntermediateMessageEntry(intermediateMessage) = earlier { self.earlier = .MessageEntry(renderIntermediateMessage(intermediateMessage)) } @@ -531,7 +526,7 @@ public final class MessageHistoryView { for (namespace, state) in combinedReadState.states { var maxNamespaceIndex: MessageIndex? var index = entries.count - 1 - for entry in entries.reverse() { + for entry in entries.reversed() { if entry.index.id.namespace == namespace && entry.index.id.id <= state.maxIncomingReadId { maxNamespaceIndex = entry.index break diff --git a/Postbox/MessageMediaTable.swift b/Postbox/MessageMediaTable.swift index b9f4069b70..fbef514933 100644 --- a/Postbox/MessageMediaTable.swift +++ b/Postbox/MessageMediaTable.swift @@ -29,13 +29,13 @@ final class MessageMediaTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - func key(id: MediaId, key: ValueBoxKey = ValueBoxKey(length: 4 + 8)) -> ValueBoxKey { + func key(_ id: MediaId, key: ValueBoxKey = ValueBoxKey(length: 4 + 8)) -> ValueBoxKey { key.setInt32(0, value: id.namespace) key.setInt64(4, value: id.id) return key } - func get(id: MediaId, @noescape embedded: (MessageIndex, MediaId) -> Media?) -> Media? { + func get(_ id: MediaId, embedded: @noescape(MessageIndex, MediaId) -> Media?) -> Media? { if let value = self.valueBox.get(self.tableId, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) @@ -63,7 +63,7 @@ final class MessageMediaTable: Table { return nil } - func set(media: Media, index: MessageIndex, messageHistoryTable: MessageHistoryTable, sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) -> InsertMediaResult { + func set(_ media: Media, index: MessageIndex, messageHistoryTable: MessageHistoryTable, sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) -> InsertMediaResult { if let id = media.id { if let value = self.valueBox.get(self.tableId, key: self.key(id)) { var type: Int8 = 0 @@ -143,7 +143,7 @@ final class MessageMediaTable: Table { } } - func removeReference(id: MediaId, sharedWriteBuffer: WriteBuffer = WriteBuffer()) -> RemoveMediaResult { + func removeReference(_ id: MediaId, sharedWriteBuffer: WriteBuffer = WriteBuffer()) -> RemoveMediaResult { if let value = self.valueBox.get(self.tableId, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) @@ -193,14 +193,14 @@ final class MessageMediaTable: Table { return .Reference } - func removeEmbeddedMedia(media: Media) { + func removeEmbeddedMedia(_ media: Media) { if let id = media.id { self.valueBox.remove(self.tableId, key: self.key(id)) } self.mediaCleanupTable.add(media) } - func update(id: MediaId, media: Media, messageHistoryTable: MessageHistoryTable, inout operationsByPeerId: [PeerId: [MessageHistoryOperation]], sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) { + func update(_ id: MediaId, media: Media, messageHistoryTable: MessageHistoryTable, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) { if let updatedId = media.id { if let value = self.valueBox.get(self.tableId, key: self.key(id)) { var type: Int8 = 0 @@ -208,7 +208,6 @@ final class MessageMediaTable: Table { if type == MediaEntryType.Direct.rawValue { var dataLength: Int32 = 0 value.read(&dataLength, offset: 0, length: 4) - let mediaOffset = value.offset value.skip(Int(dataLength)) var messageReferenceCount: Int32 = 0 @@ -288,4 +287,4 @@ final class MessageMediaTable: Table { return entries } -} \ No newline at end of file +} diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index 4815cfb0b8..3ef3729279 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -10,7 +10,7 @@ final class MetadataTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func key(key: MetadataKey) -> ValueBoxKey { + private func key(_ key: MetadataKey) -> ValueBoxKey { let valueBoxKey = ValueBoxKey(length: 4) valueBoxKey.setInt32(0, value: key.rawValue) return valueBoxKey @@ -25,7 +25,7 @@ final class MetadataTable: Table { return nil } - func setUserVersion(version: Int32) { + func setUserVersion(_ version: Int32) { let buffer = WriteBuffer() var varVersion: Int32 = version buffer.write(&varVersion, offset: 0, length: 4) @@ -41,9 +41,9 @@ final class MetadataTable: Table { return nil } - func setState(state: Coding) { + func setState(_ state: Coding) { let encoder = Encoder() encoder.encodeRootObject(state) self.valueBox.set(self.tableId, key: self.key(.State), value: encoder.readBufferNoCopy()) } -} \ No newline at end of file +} diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index d4610b3d8a..c86604375e 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -21,7 +21,7 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { return (Int64(self.namespace) << 32) | Int64(self.id) } - public static func encodeArrayToBuffer(array: [PeerId], buffer: WriteBuffer) { + public static func encodeArrayToBuffer(_ array: [PeerId], buffer: WriteBuffer) { var length: Int32 = Int32(array.count) buffer.write(&length, offset: 0, length: 4) for id in array { @@ -30,7 +30,7 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { } } - public static func decodeArrayFromBuffer(buffer: ReadBuffer) -> [PeerId] { + public static func decodeArrayFromBuffer(_ buffer: ReadBuffer) -> [PeerId] { var length: Int32 = 0 memcpy(&length, buffer.memory, 4) buffer.offset += 4 @@ -68,7 +68,7 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { self.id = id } - public func encodeToBuffer(buffer: WriteBuffer) { + public func encodeToBuffer(_ buffer: WriteBuffer) { var namespace = self.namespace var id = self.id buffer.write(&namespace, offset: 0, length: 4); @@ -95,5 +95,5 @@ public func <(lhs: PeerId, rhs: PeerId) -> Bool { public protocol Peer: Coding { var id: PeerId { get } - func isEqual(other: Peer) -> Bool + func isEqual(_ other: Peer) -> Bool } diff --git a/Postbox/PeerChatStateTable.swift b/Postbox/PeerChatStateTable.swift index 8354705477..d6abd4708b 100644 --- a/Postbox/PeerChatStateTable.swift +++ b/Postbox/PeerChatStateTable.swift @@ -6,12 +6,12 @@ final class PeerChatStateTable: Table { private let sharedKey = ValueBoxKey(length: 8) - private func key(id: PeerId) -> ValueBoxKey { + private func key(_ id: PeerId) -> ValueBoxKey { self.sharedKey.setInt64(0, value: id.toInt64()) return self.sharedKey } - func get(id: PeerId) -> Coding? { + func get(_ id: PeerId) -> Coding? { if let state = self.cachedPeerChatStates[id] { return state } else { @@ -25,7 +25,7 @@ final class PeerChatStateTable: Table { } } - func set(id: PeerId, state: Coding?) { + func set(_ id: PeerId, state: Coding?) { self.cachedPeerChatStates[id] = state self.updatedPeerIds.insert(id) } diff --git a/Postbox/PeerTable.swift b/Postbox/PeerTable.swift index d1c318e547..a85ee8106d 100644 --- a/Postbox/PeerTable.swift +++ b/Postbox/PeerTable.swift @@ -11,17 +11,17 @@ final class PeerTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - private func key(id: PeerId) -> ValueBoxKey { + private func key(_ id: PeerId) -> ValueBoxKey { self.sharedKey.setInt64(0, value: id.toInt64()) return self.sharedKey } - func set(peer: Peer) { + func set(_ peer: Peer) { self.cachedPeers[peer.id] = peer self.updatedPeerIds.insert(peer.id) } - func get(id: PeerId) -> Peer? { + func get(_ id: PeerId) -> Peer? { if let peer = self.cachedPeers[id] { return peer } @@ -46,4 +46,4 @@ final class PeerTable: Table { self.updatedPeerIds.removeAll() } -} \ No newline at end of file +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 90c71c9f73..2db55cc904 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -8,7 +8,7 @@ public enum PreloadedMessageHistoryView { } public protocol PeerChatState: Coding { - func equals(other: PeerChatState) -> Bool + func equals(_ other: PeerChatState) -> Bool } public final class Modifier { @@ -18,50 +18,50 @@ public final class Modifier { self.postbox = postbox } - public func addMessages(messages: [StoreMessage], location: AddMessagesLocation) { + public func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) { self.postbox?.addMessages(messages, location: location) } - public func addHole(messageId: MessageId) { + public func addHole(_ messageId: MessageId) { self.postbox?.addHole(messageId) } - public func fillHole(hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { + public func fillHole(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { self.postbox?.fillHole(hole, fillType: fillType, tagMask: tagMask, messages: messages) } - public func replaceChatListHole(index: MessageIndex, hole: ChatListHole?) { + public func replaceChatListHole(_ index: MessageIndex, hole: ChatListHole?) { self.postbox?.replaceChatListHole(index, hole: hole) } - public func deleteMessages(messageIds: [MessageId]) { + public func deleteMessages(_ messageIds: [MessageId]) { self.postbox?.deleteMessages(messageIds) } - public func deleteMessagesWithGlobalIds(ids: [Int32]) { + public func deleteMessagesWithGlobalIds(_ ids: [Int32]) { if let postbox = self.postbox { let messageIds = postbox.messageIdsForGlobalIds(ids) postbox.deleteMessages(messageIds) } } - public func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]]) { + public func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]]) { self.postbox?.resetIncomingReadStates(states) } - public func confirmSynchronizedIncomingReadState(peerId: PeerId) { + public func confirmSynchronizedIncomingReadState(_ peerId: PeerId) { self.postbox?.confirmSynchronizedIncomingReadState(peerId) } - public func applyIncomingReadMaxId(messageId: MessageId) { + public func applyIncomingReadMaxId(_ messageId: MessageId) { self.postbox?.applyIncomingReadMaxId(messageId) } - public func applyOutgoingReadMaxId(messageId: MessageId) { + public func applyOutgoingReadMaxId(_ messageId: MessageId) { self.postbox?.applyOutgoingReadMaxId(messageId) } - public func applyInteractiveReadMaxId(messageId: MessageId) { + public func applyInteractiveReadMaxId(_ messageId: MessageId) { self.postbox?.applyInteractiveReadMaxId(messageId) } @@ -69,39 +69,39 @@ public final class Modifier { return self.postbox?.getState() } - public func setState(state: Coding) { + public func setState(_ state: Coding) { self.postbox?.setState(state) } - public func getPeerChatState(id: PeerId) -> PeerChatState? { + public func getPeerChatState(_ id: PeerId) -> PeerChatState? { return self.postbox?.peerChatStateTable.get(id) as? PeerChatState } - public func setPeerChatState(id: PeerId, state: PeerChatState) { + public func setPeerChatState(_ id: PeerId, state: PeerChatState) { self.postbox?.peerChatStateTable.set(id, state: state) } - public func getPeer(id: PeerId) -> Peer? { + public func getPeer(_ id: PeerId) -> Peer? { return self.postbox?.peerTable.get(id) } - public func getPeerReadStates(id: PeerId) -> [(MessageId.Namespace, PeerReadState)]? { + public func getPeerReadStates(_ id: PeerId) -> [(MessageId.Namespace, PeerReadState)]? { return self.postbox?.readStateTable.getCombinedState(id)?.states } - public func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer?) { + public func updatePeers(_ peers: [Peer], update: (Peer, Peer) -> Peer?) { self.postbox?.updatePeers(peers, update: update) } - public func updateMessage(index: MessageIndex, update: Message -> StoreMessage) { + public func updateMessage(_ index: MessageIndex, update: (Message) -> StoreMessage) { self.postbox?.updateMessage(index, update: update) } - public func updateMedia(id: MediaId, update: Media?) { + public func updateMedia(_ id: MediaId, update: Media?) { self.postbox?.updateMedia(id, update: update) } - public func getMessage(id: MessageId) -> Message? { + public func getMessage(_ id: MessageId) -> Message? { if let postbox = self.postbox { if let entry = postbox.messageHistoryIndexTable.get(id) { if case let .Message(index) = entry { @@ -114,7 +114,7 @@ public final class Modifier { return nil } - public func filterStoredMessageIds(messageIds: Set) -> Set { + public func filterStoredMessageIds(_ messageIds: Set) -> Set { if let postbox = self.postbox { return postbox.filterStoredMessageIds(messageIds) } @@ -133,7 +133,7 @@ public final class Postbox { private var viewTracker: ViewTracker! private var nextViewId = 0 - private var peerPipes: [PeerId: Pipe] = [:] + private var peerPipes: [PeerId: ValuePipe] = [:] private var currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] private var currentUnsentOperations: [IntermediateMessageHistoryUnsentOperation] = [] @@ -145,21 +145,21 @@ public final class Postbox { private var currentUpdatedPeers: [PeerId: Peer] = [:] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] - private var statePipe: Pipe = Pipe() + private var statePipe: ValuePipe = ValuePipe() - private let fetchChatListHoleImpl = Promise Signal>() - public func setFetchChatListHole(fetch: ChatListHole -> Signal) { - self.fetchChatListHoleImpl.set(single(fetch, NoError.self)) + private let fetchChatListHoleImpl = Promise<(ChatListHole) -> Signal>() + public func setFetchChatListHole(fetch: (ChatListHole) -> Signal) { + self.fetchChatListHoleImpl.set(.single(fetch)) } private let fetchMessageHistoryHoleImpl = Promise<(MessageHistoryHole, HoleFillDirection, MessageTags?) -> Signal>() public func setFetchMessageHistoryHole(fetch: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Signal) { - self.fetchMessageHistoryHoleImpl.set(single(fetch, NoError.self)) + self.fetchMessageHistoryHoleImpl.set(.single(fetch)) } - private let sendUnsentMessageImpl = Promise Signal>() - public func setSendUnsentMessage(sendUnsentMessage: Message -> Signal) { - self.sendUnsentMessageImpl.set(single(sendUnsentMessage, NoError.self)) + private let sendUnsentMessageImpl = Promise<(Message) -> Signal>() + public func setSendUnsentMessage(sendUnsentMessage: (Message) -> Signal) { + self.sendUnsentMessageImpl.set(.single(sendUnsentMessage)) } private let synchronizePeerReadStateImpl = Promise<(PeerId, PeerReadStateSynchronizationOperation) -> Signal>() @@ -193,7 +193,7 @@ public final class Postbox { self.globalMessageIdsNamespace = globalMessageIdsNamespace self.seedConfiguration = seedConfiguration - let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath + "/media") + let _ = try? FileManager.default().removeItem(atPath: self.basePath + "/media") self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.openDatabase() @@ -202,9 +202,9 @@ public final class Postbox { private func debugSaveState(name: String) { self.queue.justDispatch({ let path = self.basePath + name - let _ = try? NSFileManager.defaultManager().removeItemAtPath(path) + let _ = try? FileManager.default().removeItem(atPath: path) do { - try NSFileManager.defaultManager().copyItemAtPath(self.basePath, toPath: path) + try FileManager.default().copyItem(atPath: self.basePath, toPath: path) } catch (let e) { print("(Postbox debugSaveState: error \(e))") } @@ -214,9 +214,9 @@ public final class Postbox { private func debugRestoreState(name: String) { self.queue.justDispatch({ let path = self.basePath + name - let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath) + let _ = try? FileManager.default().removeItem(atPath: self.basePath) do { - try NSFileManager.defaultManager().copyItemAtPath(path, toPath: self.basePath) + try FileManager.default().copyItem(atPath: path, toPath: self.basePath) } catch (let e) { print("(Postbox debugRestoreState: error \(e))") } @@ -228,7 +228,7 @@ public final class Postbox { let startTime = CFAbsoluteTimeGetCurrent() do { - try NSFileManager.defaultManager().createDirectoryAtPath(self.basePath, withIntermediateDirectories: true, attributes: nil) + try FileManager.default().createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) } catch _ { } @@ -305,7 +305,7 @@ public final class Postbox { private var cachedState: Coding? - private func setState(state: Coding) { + private func setState(_ state: Coding) { self.queue.justDispatch({ self.cachedState = state @@ -342,34 +342,34 @@ public final class Postbox { } } - public func keychainEntryForKey(key: String) -> NSData? { + public func keychainEntryForKey(_ key: String) -> Data? { return self.keychainTable.get(key) } - public func setKeychainEntryForKey(key: String, value: NSData) { + public func setKeychainEntryForKey(_ key: String, value: Data) { self.keychainTable.set(key, value: value) } - public func removeKeychainEntryForKey(key: String) { + public func removeKeychainEntryForKey(_ key: String) { self.keychainTable.remove(key) } - private func addMessages(messages: [StoreMessage], location: AddMessagesLocation) { + private func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) { self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func addHole(id: MessageId) { + private func addHole(_ id: MessageId) { self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func fillHole(hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { + private func fillHole(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] self.messageHistoryTable.fillHole(hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) for (peerId, operations) in operationsByPeerId { if self.currentOperationsByPeerId[peerId] == nil { self.currentOperationsByPeerId[peerId] = operations } else { - self.currentOperationsByPeerId[peerId]!.appendContentsOf(operations) + self.currentOperationsByPeerId[peerId]!.append(contentsOf: operations) } var filledMessageIndices: [MessageIndex: HoleFillDirection] = [:] @@ -402,35 +402,35 @@ public final class Postbox { } } - private func replaceChatListHole(index: MessageIndex, hole: ChatListHole?) { + private func replaceChatListHole(_ index: MessageIndex, hole: ChatListHole?) { self.currentReplaceChatListHoles.append((index, hole)) } - private func deleteMessages(messageIds: [MessageId]) { + private func deleteMessages(_ messageIds: [MessageId]) { self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func resetIncomingReadStates(states: [PeerId: [MessageId.Namespace: PeerReadState]]) { + private func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]]) { self.messageHistoryTable.resetIncomingReadStates(states, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func confirmSynchronizedIncomingReadState(peerId: PeerId) { + private func confirmSynchronizedIncomingReadState(_ peerId: PeerId) { self.synchronizeReadStateTable.set(peerId, operation: nil, operations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func applyIncomingReadMaxId(messageId: MessageId) { + private func applyIncomingReadMaxId(_ messageId: MessageId) { self.messageHistoryTable.applyIncomingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func applyOutgoingReadMaxId(messageId: MessageId) { + private func applyOutgoingReadMaxId(_ messageId: MessageId) { self.messageHistoryTable.applyOutgoingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func applyInteractiveReadMaxId(messageId: MessageId) { + private func applyInteractiveReadMaxId(_ messageId: MessageId) { self.messageHistoryTable.applyInteractiveMaxReadId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func fetchEarlierHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { + private func fetchEarlierHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { let intermediateEntries: [IntermediateMessageHistoryEntry] if let tagMask = tagMask { intermediateEntries = self.messageHistoryTable.earlierEntries(tagMask, peerId: peerId, index: index, count: count) @@ -449,7 +449,7 @@ public final class Postbox { return entries } - private func fetchAroundHistoryEntries(index: MessageIndex, count: Int, tagMask: MessageTags? = nil) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?) { + private func fetchAroundHistoryEntries(_ index: MessageIndex, count: Int, tagMask: MessageTags? = nil) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?) { let intermediateEntries: [IntermediateMessageHistoryEntry] let intermediateLower: IntermediateMessageHistoryEntry? @@ -494,7 +494,7 @@ public final class Postbox { return (entries: entries, lower: lower, upper: upper) } - private func fetchLaterHistoryEntries(peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { + private func fetchLaterHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { let intermediateEntries: [IntermediateMessageHistoryEntry] if let tagMask = tagMask { intermediateEntries = self.messageHistoryTable.laterEntries(tagMask, peerId: peerId, index: index, count: count) @@ -513,7 +513,7 @@ public final class Postbox { return entries } - private func fetchAroundChatEntries(index: MessageIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?) { + private func fetchAroundChatEntries(_ index: MessageIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?) { let (intermediateEntries, intermediateLower, intermediateUpper) = self.chatListTable.entriesAround(index, messageHistoryTable: self.messageHistoryTable, count: count) var entries: [MutableChatListEntry] = [] var lower: MutableChatListEntry? @@ -555,7 +555,7 @@ public final class Postbox { return (entries, lower, upper) } - private func fetchEarlierChatEntries(index: MessageIndex?, count: Int) -> [MutableChatListEntry] { + private func fetchEarlierChatEntries(_ index: MessageIndex?, count: Int) -> [MutableChatListEntry] { let intermediateEntries = self.chatListTable.earlierEntries(index, messageHistoryTable: self.messageHistoryTable, count: count) var entries: [MutableChatListEntry] = [] for entry in intermediateEntries { @@ -571,7 +571,7 @@ public final class Postbox { return entries } - private func fetchLaterChatEntries(index: MessageIndex?, count: Int) -> [MutableChatListEntry] { + private func fetchLaterChatEntries(_ index: MessageIndex?, count: Int) -> [MutableChatListEntry] { let intermediateEntries = self.chatListTable.laterEntries(index, messageHistoryTable: self.messageHistoryTable, count: count) var entries: [MutableChatListEntry] = [] for entry in intermediateEntries { @@ -587,27 +587,27 @@ public final class Postbox { return entries } - private func fetchAnchorIndex(id: MessageId) -> MessageHistoryAnchorIndex? { + private func fetchAnchorIndex(_ id: MessageId) -> MessageHistoryAnchorIndex? { return self.messageHistoryTable.anchorIndex(id) } - private func renderIntermediateMessage(message: IntermediateMessage) -> Message { + private func renderIntermediateMessage(_ message: IntermediateMessage) -> Message { return self.messageHistoryTable.renderMessage(message, peerTable: self.peerTable) } - private func fetchChatListHoleWrapper(hole: ChatListHole) -> Disposable { + private func fetchChatListHoleWrapper(_ hole: ChatListHole) -> Disposable { return (self.fetchChatListHoleImpl.get() |> mapToSignal { fetch in return fetch(hole) }).start() } - private func fetchMessageHistoryHoleWrapper(hole: MessageHistoryHole, direction: HoleFillDirection, tagMask: MessageTags?) -> Disposable { + private func fetchMessageHistoryHoleWrapper(_ hole: MessageHistoryHole, direction: HoleFillDirection, tagMask: MessageTags?) -> Disposable { return (self.fetchMessageHistoryHoleImpl.get() |> mapToSignal { fetch in return fetch(hole, direction, tagMask) }).start() } - private func sendUnsentMessageWrapper(index: MessageIndex) -> Disposable { + private func sendUnsentMessageWrapper(_ index: MessageIndex) -> Disposable { return (self.sendUnsentMessageImpl.get() |> deliverOn(self.queue) |> mapToSignal { send -> Signal in if let intermediateMessage = self.messageHistoryTable.getMessage(index) { let message = self.renderIntermediateMessage(intermediateMessage) @@ -618,7 +618,7 @@ public final class Postbox { }).start() } - private func synchronizePeerReadStateWrapper(peerId: PeerId, operation: PeerReadStateSynchronizationOperation) -> Disposable { + private func synchronizePeerReadStateWrapper(_ peerId: PeerId, operation: PeerReadStateSynchronizationOperation) -> Disposable { return (self.synchronizePeerReadStateImpl.get() |> mapToSignal { validate -> Signal in return validate(peerId, operation) }).start() @@ -647,7 +647,7 @@ public final class Postbox { } } - private func messageIdsForGlobalIds(ids: [Int32]) -> [MessageId] { + private func messageIdsForGlobalIds(_ ids: [Int32]) -> [MessageId] { var result: [MessageId] = [] for globalId in ids { if let id = self.globalMessageIdsTable.get(globalId) { @@ -657,7 +657,7 @@ public final class Postbox { return result } - private func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer?) { + private func updatePeers(_ peers: [Peer], update: (Peer, Peer) -> Peer?) { for peer in peers { if let currentPeer = self.peerTable.get(peer.id) { if let updatedPeer = update(currentPeer, peer) { @@ -670,18 +670,18 @@ public final class Postbox { } } - private func updateMessage(index: MessageIndex, update: Message -> StoreMessage) { + private func updateMessage(_ index: MessageIndex, update: (Message) -> StoreMessage) { if let intermediateMessage = self.messageHistoryTable.getMessage(index) { let message = self.renderIntermediateMessage(intermediateMessage) self.messageHistoryTable.updateMessage(index.id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } } - private func updateMedia(id: MediaId, update: Media?) { + private func updateMedia(_ id: MediaId, update: Media?) { self.messageHistoryTable.updateMedia(id, media: update, operationsByPeerId: &self.currentOperationsByPeerId, updatedMedia: &self.currentUpdatedMedia) } - private func filterStoredMessageIds(messageIds: Set) -> Set { + private func filterStoredMessageIds(_ messageIds: Set) -> Set { var filteredIds = Set() for id in messageIds { @@ -693,7 +693,7 @@ public final class Postbox { return filteredIds } - public func modify(f: Modifier -> T) -> Signal { + public func modify(_ f: (Modifier) -> T) -> Signal { return Signal { subscriber in self.queue.justDispatch({ //let startTime = CFAbsoluteTimeGetCurrent() @@ -711,11 +711,11 @@ public final class Postbox { } } - public func aroundUnreadMessageHistoryViewForPeerId(peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return Signal { subscriber in let disposable = MetaDisposable() - self.queue.dispatch { - var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId), exact: true) + self.queue.async { + var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let maxReadIndex = self.messageHistoryTable.maxReadIndex(peerId) { index = maxReadIndex } @@ -731,11 +731,11 @@ public final class Postbox { } } - public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return self.aroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, tagMask: tagMask) } - private func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + private func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return Signal { subscriber in let disposable = MetaDisposable() let startTime = CFAbsoluteTimeGetCurrent() @@ -744,15 +744,7 @@ public final class Postbox { let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - var adjustedAnchorIndex = anchorIndex - /*if entries.count != 0 { - let minIndex = entries[0].index - if anchorIndex < minIndex { - //adjustedAnchorIndex = minIndex - } - }*/ - - let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: adjustedAnchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) + let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) mutableView.render(self.renderIntermediateMessage) //print("+ render \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") @@ -774,7 +766,7 @@ public final class Postbox { pipeDisposable.dispose() if let strongSelf = self { - strongSelf.queue.dispatch { + strongSelf.queue.async { strongSelf.viewTracker.removeMessageHistoryView(peerId, index: index) } } @@ -786,7 +778,7 @@ public final class Postbox { } } - public func messageIndexAtId(id: MessageId) -> Signal { + public func messageIndexAtId(_ id: MessageId) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -794,7 +786,7 @@ public final class Postbox { if let entry = self.messageHistoryIndexTable.get(id), case let .Message(index) = entry { subscriber.putNext(index) subscriber.putCompletion() - } else if let hole = self.messageHistoryIndexTable.holeContainingId(id) { + } else if let _ = self.messageHistoryIndexTable.holeContainingId(id) { subscriber.putNext(nil) subscriber.putCompletion() } else { @@ -807,11 +799,11 @@ public final class Postbox { } } - public func tailChatListView(count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { + public func tailChatListView(_ count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { return self.aroundChatListView(MessageIndex.absoluteUpperBound(), count: count) } - public func aroundChatListView(index: MessageIndex, count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { + public func aroundChatListView(_ index: MessageIndex, count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { return Signal { subscriber in let disposable = MetaDisposable() @@ -832,7 +824,7 @@ public final class Postbox { pipeDisposable.dispose() if let strongSelf = self { - strongSelf.queue.dispatch { + strongSelf.queue.async { strongSelf.viewTracker.removeChatListView(index) } } @@ -844,7 +836,7 @@ public final class Postbox { } } - public func peerWithId(id: PeerId) -> Signal { + public func peerWithId(_ id: PeerId) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.queue.justDispatch({ @@ -856,7 +848,7 @@ public final class Postbox { } } - public func updateMessageHistoryViewVisibleRange(id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { + public func updateMessageHistoryViewVisibleRange(_ id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { self.queue.justDispatch({ self.viewTracker.updateMessageHistoryViewVisibleRange(id, earliestVisibleIndex: earliestVisibleIndex, latestVisibleIndex: latestVisibleIndex) }) diff --git a/Postbox/SimpleDictionary.swift b/Postbox/SimpleDictionary.swift index 84d03084c6..42efa12b69 100644 --- a/Postbox/SimpleDictionary.swift +++ b/Postbox/SimpleDictionary.swift @@ -1,6 +1,6 @@ import Foundation -public struct SimpleDictionary: SequenceType { +public struct SimpleDictionary: Sequence { private var items: [(K, V)] = [] public subscript(key: K) -> V? { @@ -18,7 +18,7 @@ public struct SimpleDictionary: SequenceType { if let value = value { self.items[index] = (k, value) } else { - self.items.removeAtIndex(index) + self.items.remove(at: index) } return } @@ -30,9 +30,11 @@ public struct SimpleDictionary: SequenceType { } } - public func generate() -> AnyGenerator<(K, V)> { + + + public func makeIterator() -> AnyIterator<(K, V)> { var index = 0 - return AnyGenerator { () -> (K, V)? in + return AnyIterator { () -> (K, V)? in if index < self.items.count { let currentIndex = index index += 1 diff --git a/Postbox/SimpleSet.swift b/Postbox/SimpleSet.swift index 74e0e659a5..8e724141cb 100644 --- a/Postbox/SimpleSet.swift +++ b/Postbox/SimpleSet.swift @@ -6,13 +6,13 @@ public struct SimpleSet { public init() { } - public mutating func insert(item: T) { + public mutating func insert(_ item: T) { if !self.contains(item) { self.items.append(item) } } - public func contains(item: T) -> Bool { + public func contains(_ item: T) -> Bool { for currentItem in self.items { if currentItem == item { return true @@ -20,4 +20,4 @@ public struct SimpleSet { } return false } -} \ No newline at end of file +} diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 4ad5dbf0db..8d5d64fbc7 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -3,21 +3,21 @@ import sqlcipher import SwiftSignalKit private struct SqlitePreparedStatement { - let statement: COpaquePointer + let statement: OpaquePointer? - func bind(index: Int, data: UnsafePointer, length: Int) { + func bind(_ index: Int, data: UnsafePointer, length: Int) { sqlite3_bind_blob(statement, Int32(index), data, Int32(length), nil) } - func bindNull(index: Int) { + func bindNull(_ index: Int) { sqlite3_bind_null(statement, Int32(index)) } - func bind(index: Int, number: Int32) { + func bind(_ index: Int, number: Int32) { sqlite3_bind_int(statement, Int32(index), number) } - func bind(index: Int, number: Int64) { + func bind(_ index: Int, number: Int64) { sqlite3_bind_int64(statement, Int32(index), number) } @@ -34,20 +34,20 @@ private struct SqlitePreparedStatement { return result == SQLITE_ROW } - func int64At(index: Int) -> Int64 { + func int64At(_ index: Int) -> Int64 { return sqlite3_column_int64(statement, Int32(index)) } - func valueAt(index: Int) -> ReadBuffer { + func valueAt(_ index: Int) -> ReadBuffer { let valueLength = sqlite3_column_bytes(statement, Int32(index)) let valueData = sqlite3_column_blob(statement, Int32(index)) - let valueMemory = malloc(Int(valueLength)) + let valueMemory = malloc(Int(valueLength))! memcpy(valueMemory, valueData, Int(valueLength)) return ReadBuffer(memory: valueMemory, length: Int(valueLength), freeWhenDone: true) } - func keyAt(index: Int) -> ValueBoxKey { + func keyAt(_ index: Int) -> ValueBoxKey { let valueLength = sqlite3_column_bytes(statement, Int32(index)) let valueData = sqlite3_column_blob(statement, Int32(index)) @@ -62,7 +62,7 @@ private struct SqlitePreparedStatement { } public final class SqliteValueBox: ValueBox { - private let lock = NSRecursiveLock() + private let lock = RecursiveLock() private let basePath: String private var database: Database! @@ -102,7 +102,7 @@ public final class SqliteValueBox: ValueBox { lock.lock() do { - try NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil) + try FileManager.default().createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) } catch _ { } let path = basePath + "/db_sqlite" let database = Database(path) @@ -160,18 +160,18 @@ public final class SqliteValueBox: ValueBox { self.commitTime += CFAbsoluteTimeGetCurrent() - startTime } - private func getUserVersion(database: Database) -> Int64 { - var statement: COpaquePointer = nil + private func getUserVersion(_ database: Database) -> Int64 { + var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, "PRAGMA user_version", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - preparedStatement.step() + let _ = preparedStatement.step() let value = preparedStatement.int64At(0) preparedStatement.destroy() return value } - private func listTables(database: Database) -> [Int64] { - var statement: COpaquePointer = nil + private func listTables(_ database: Database) -> [Int64] { + var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, "SELECT name FROM __meta_tables", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) var tables: [Int64] = [] @@ -183,13 +183,13 @@ public final class SqliteValueBox: ValueBox { return tables } - private func getStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + private func getStatement(_ table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.getStatements[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT value FROM t\(table) WHERE key=?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.getStatements[table] = preparedStatement @@ -203,13 +203,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func rangeKeyAscStatementLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + private func rangeKeyAscStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.rangeKeyAscStatementsLimit[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeKeyAscStatementsLimit[table] = preparedStatement @@ -224,13 +224,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func rangeKeyAscStatementNoLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + private func rangeKeyAscStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.rangeKeyAscStatementsNoLimit[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeKeyAscStatementsNoLimit[table] = preparedStatement @@ -245,13 +245,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func rangeKeyDescStatementLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + private func rangeKeyDescStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.rangeKeyDescStatementsLimit[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeKeyDescStatementsLimit[table] = preparedStatement @@ -267,13 +267,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func rangeKeyDescStatementNoLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + private func rangeKeyDescStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.rangeKeyDescStatementsNoLimit[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeKeyDescStatementsNoLimit[table] = preparedStatement @@ -288,13 +288,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func rangeValueAscStatementLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + private func rangeValueAscStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.rangeValueAscStatementsLimit[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeValueAscStatementsLimit[table] = preparedStatement @@ -309,13 +309,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func rangeValueAscStatementNoLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + private func rangeValueAscStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.rangeValueAscStatementsNoLimit[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeValueAscStatementsNoLimit[table] = preparedStatement @@ -330,13 +330,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func rangeValueDescStatementLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + private func rangeValueDescStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.rangeValueDescStatementsLimit[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeValueDescStatementsLimit[table] = preparedStatement @@ -352,13 +352,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func rangeValueDescStatementNoLimit(table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + private func rangeValueDescStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.rangeKeyDescStatementsNoLimit[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeValueDescStatementsNoLimit[table] = preparedStatement @@ -373,13 +373,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func existsStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + private func existsStatement(_ table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.existsStatements[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "SELECT rowid FROM t\(table) WHERE key=?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.existsStatements[table] = preparedStatement @@ -393,13 +393,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func updateStatement(table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { + private func updateStatement(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.updateStatements[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "UPDATE t\(table) SET value=? WHERE key=?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.updateStatements[table] = preparedStatement @@ -414,13 +414,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func insertStatement(table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { + private func insertStatement(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.insertStatements[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table) (key, value) VALUES(?, ?)", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.insertStatements[table] = preparedStatement @@ -430,7 +430,7 @@ public final class SqliteValueBox: ValueBox { resultStatement.reset() resultStatement.bind(1, data: key.memory, length: key.length) - if value.memory == nil { + if value.length == 0 { resultStatement.bindNull(2) } else { resultStatement.bind(2, data: value.memory, length: value.length) @@ -439,13 +439,13 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func deleteStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + private func deleteStatement(_ table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement if let statement = self.deleteStatements[table] { resultStatement = statement } else { - var statement: COpaquePointer = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(self.database.handle, "DELETE FROM t\(table) WHERE key=?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) self.deleteStatements[table] = preparedStatement @@ -459,7 +459,7 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? { + public func get(_ table: Int32, key: ValueBoxKey) -> ReadBuffer? { let startTime = CFAbsoluteTimeGetCurrent() if self.tables.contains(table) { let statement = self.getStatement(table, key: key) @@ -481,14 +481,14 @@ public final class SqliteValueBox: ValueBox { return nil } - public func exists(table: Int32, key: ValueBoxKey) -> Bool { + public func exists(_ table: Int32, key: ValueBoxKey) -> Bool { if let _ = self.get(table, key: key) { return true } return false } - public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { + public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, values: @noescape(ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { if start == end { return } @@ -535,7 +535,7 @@ public final class SqliteValueBox: ValueBox { } } - public func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape keys: ValueBoxKey -> Bool, limit: Int) { + public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: @noescape(ValueBoxKey) -> Bool, limit: Int) { if self.tables.contains(table) { let statement: SqlitePreparedStatement @@ -577,7 +577,7 @@ public final class SqliteValueBox: ValueBox { } } - public func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) { + public func set(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) { if !self.tables.contains(table) { self.database.execute("CREATE TABLE t\(table) (key BLOB, value BLOB)") self.database.execute("CREATE INDEX t\(table)_key ON t\(table) (key)") @@ -609,7 +609,7 @@ public final class SqliteValueBox: ValueBox { self.writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } - public func remove(table: Int32, key: ValueBoxKey) { + public func remove(_ table: Int32, key: ValueBoxKey) { if self.tables.contains(table) { let startTime = CFAbsoluteTimeGetCurrent() @@ -697,7 +697,7 @@ public final class SqliteValueBox: ValueBox { self.database = nil self.lock.unlock() - let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath) + let _ = try? FileManager.default().removeItem(atPath: self.basePath) self.database = self.openDatabase() tables.removeAll() diff --git a/Postbox/SynchronizePeerReadStatesView.swift b/Postbox/SynchronizePeerReadStatesView.swift index b9a0427a8d..21f48ed93b 100644 --- a/Postbox/SynchronizePeerReadStatesView.swift +++ b/Postbox/SynchronizePeerReadStatesView.swift @@ -7,7 +7,7 @@ final class SynchronizePeerReadStatesView { self.operations = operations } - func replay(updatedOperations: [PeerId: PeerReadStateSynchronizationOperation?]) -> [PeerId: PeerReadStateSynchronizationOperation?] { + func replay(_ updatedOperations: [PeerId: PeerReadStateSynchronizationOperation?]) -> [PeerId: PeerReadStateSynchronizationOperation?] { var updates: [PeerId: PeerReadStateSynchronizationOperation?] = [:] for (peerId, operation) in updatedOperations { @@ -18,7 +18,7 @@ final class SynchronizePeerReadStatesView { } } else { if let _ = self.operations[peerId] { - self.operations.removeValueForKey(peerId) + let _ = self.operations.removeValue(forKey: peerId) updates[peerId] = nil } } diff --git a/Postbox/UnsentMessageHistoryView.swift b/Postbox/UnsentMessageHistoryView.swift index 39e76768a8..0ff6b2efd6 100644 --- a/Postbox/UnsentMessageHistoryView.swift +++ b/Postbox/UnsentMessageHistoryView.swift @@ -7,7 +7,7 @@ final class UnsentMessageHistoryView { self.indices = indices } - func replay(operations: [IntermediateMessageHistoryUnsentOperation]) -> Bool { + func replay(_ operations: [IntermediateMessageHistoryUnsentOperation]) -> Bool { var updated = false for operation in operations { switch operation { @@ -15,7 +15,7 @@ final class UnsentMessageHistoryView { var inserted = false for i in 0 ..< self.indices.count { if self.indices[i] > index { - self.indices.insert(index, atIndex: i) + self.indices.insert(index, at: i) inserted = true break } @@ -27,7 +27,7 @@ final class UnsentMessageHistoryView { case let .Remove(index): for i in 0 ..< self.indices.count { if self.indices[i] == index { - self.indices.removeAtIndex(i) + self.indices.remove(at: i) updated = true break } diff --git a/Postbox/ValueBox.swift b/Postbox/ValueBox.swift index 2ade4659cc..4c6abb138f 100644 --- a/Postbox/ValueBox.swift +++ b/Postbox/ValueBox.swift @@ -7,11 +7,11 @@ public protocol ValueBox { func beginStats() func endStats() - func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) - func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape keys: ValueBoxKey -> Bool, limit: Int) - func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? - func exists(table: Int32, key: ValueBoxKey) -> Bool - func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) - func remove(table: Int32, key: ValueBoxKey) + func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, values: @noescape(ValueBoxKey, ReadBuffer) -> Bool, limit: Int) + func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: @noescape(ValueBoxKey) -> Bool, limit: Int) + func get(_ table: Int32, key: ValueBoxKey) -> ReadBuffer? + func exists(_ table: Int32, key: ValueBoxKey) -> Bool + func set(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) + func remove(_ table: Int32, key: ValueBoxKey) func drop() } diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index 643b5a39fd..5343ee4a5e 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -24,11 +24,11 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { } public init(_ value: String) { - let data = value.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! - self.memory = malloc(data.length) - self.length = data.length + let data = value.data(using: .utf8, allowLossyConversion: true)! + self.memory = malloc(data.count) + self.length = data.count self.impl = ValueBoxKeyImpl(memory: self.memory) - memcpy(self.memory, data.bytes, data.length) + data.copyBytes(to: UnsafeMutablePointer(self.memory), count: data.count) } public init(_ buffer: MemoryBuffer) { @@ -38,51 +38,51 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { memcpy(self.memory, buffer.memory, buffer.length) } - public func setInt32(offset: Int, value: Int32) { + public func setInt32(_ offset: Int, value: Int32) { var bigEndianValue = Int32(bigEndian: value) memcpy(self.memory + offset, &bigEndianValue, 4) } - public func setUInt32(offset: Int, value: UInt32) { + public func setUInt32(_ offset: Int, value: UInt32) { var bigEndianValue = UInt32(bigEndian: value) memcpy(self.memory + offset, &bigEndianValue, 4) } - public func setInt64(offset: Int, value: Int64) { + public func setInt64(_ offset: Int, value: Int64) { var bigEndianValue = Int64(bigEndian: value) memcpy(self.memory + offset, &bigEndianValue, 8) } - public func setInt8(offset: Int, value: Int8) { + public func setInt8(_ offset: Int, value: Int8) { var varValue = value memcpy(self.memory + offset, &varValue, 1) } - public func getInt32(offset: Int) -> Int32 { + public func getInt32(_ offset: Int) -> Int32 { var value: Int32 = 0 memcpy(&value, self.memory + offset, 4) return Int32(bigEndian: value) } - public func getUInt32(offset: Int) -> UInt32 { + public func getUInt32(_ offset: Int) -> UInt32 { var value: UInt32 = 0 memcpy(&value, self.memory + offset, 4) return UInt32(bigEndian: value) } - public func getInt64(offset: Int) -> Int64 { + public func getInt64(_ offset: Int) -> Int64 { var value: Int64 = 0 memcpy(&value, self.memory + offset, 8) return Int64(bigEndian: value) } - public func getInt8(offset: Int) -> Int8 { + public func getInt8(_ offset: Int) -> Int8 { var value: Int8 = 0 memcpy(&value, self.memory + offset, 1) return value } - public func prefix(length: Int) -> ValueBoxKey { + public func prefix(_ length: Int) -> ValueBoxKey { assert(length <= self.length, "length <= self.length") let key = ValueBoxKey(length: length) memcpy(key.memory, self.memory, length) @@ -144,7 +144,7 @@ public func ==(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { return lhs.length == rhs.length && memcmp(lhs.memory, rhs.memory, lhs.length) == 0 } -private func mdb_cmp_memn(a_memory: UnsafeMutablePointer, _ a_length: Int, _ b_memory: UnsafeMutablePointer, _ b_length: Int) -> Int +private func mdb_cmp_memn(_ a_memory: UnsafeMutablePointer, _ a_length: Int, _ b_memory: UnsafeMutablePointer, _ b_length: Int) -> Int { var diff: Int = 0 var len_diff: Int = 0 diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 25f19275b9..9b97614bc3 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -15,14 +15,14 @@ final class ViewTracker { private let fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] private let fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] private let fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex? - private let renderMessage: IntermediateMessage -> Message - private let fetchChatListHole: ChatListHole -> Disposable + private let renderMessage: (IntermediateMessage) -> Message + private let fetchChatListHole: (ChatListHole) -> Disposable private let fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable - private let sendUnsentMessage: MessageIndex -> Disposable + private let sendUnsentMessage: (MessageIndex) -> Disposable private let synchronizeReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Disposable - private var chatListViews = Bag<(MutableChatListView, Pipe<(ChatListView, ViewUpdateType)>)>() - private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] + private var chatListViews = Bag<(MutableChatListView, ValuePipe<(ChatListView, ViewUpdateType)>)>() + private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] private var unsentMessageView: UnsentMessageHistoryView private var synchronizeReadStatesView: SynchronizePeerReadStatesView @@ -31,7 +31,7 @@ final class ViewTracker { private var unsentMessageDisposables: [MessageIndex: Disposable] = [:] private var synchronizeReadStatesDisposables: [PeerId: (PeerReadStateSynchronizationOperation, Disposable)] = [:] - init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: IntermediateMessage -> Message, fetchChatListHole: ChatListHole -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable, sendUnsentMessage: MessageIndex -> Disposable, unsentMessageIndices: [MessageIndex], synchronizeReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Disposable, synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: (IntermediateMessage) -> Message, fetchChatListHole: (ChatListHole) -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable, sendUnsentMessage: (MessageIndex) -> Disposable, unsentMessageIndices: [MessageIndex], synchronizeReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Disposable, synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -67,14 +67,14 @@ final class ViewTracker { } } - func addMessageHistoryView(peerId: PeerId, view: MutableMessageHistoryView) -> (Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>.Index, Signal<(MessageHistoryView, ViewUpdateType), NoError>) { - let record = (view, Pipe<(MessageHistoryView, ViewUpdateType)>()) + func addMessageHistoryView(_ peerId: PeerId, view: MutableMessageHistoryView) -> (Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>.Index, Signal<(MessageHistoryView, ViewUpdateType), NoError>) { + let record = (view, ValuePipe<(MessageHistoryView, ViewUpdateType)>()) - let index: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>.Index + let index: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>.Index if let bag = self.messageHistoryViews[peerId] { index = bag.add(record) } else { - let bag = Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>() + let bag = Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>() index = bag.add(record) self.messageHistoryViews[peerId] = bag } @@ -84,7 +84,7 @@ final class ViewTracker { return (index, record.1.signal()) } - func removeMessageHistoryView(peerId: PeerId, index: Bag<(MutableMessageHistoryView, Pipe<(MessageHistoryView, ViewUpdateType)>)>.Index) { + func removeMessageHistoryView(_ peerId: PeerId, index: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>.Index) { if let bag = self.messageHistoryViews[peerId] { bag.remove(index) @@ -92,8 +92,8 @@ final class ViewTracker { } } - func addChatListView(view: MutableChatListView) -> (Bag<(MutableChatListView, Pipe<(ChatListView, ViewUpdateType)>)>.Index, Signal<(ChatListView, ViewUpdateType), NoError>) { - let record = (view, Pipe<(ChatListView, ViewUpdateType)>()) + func addChatListView(_ view: MutableChatListView) -> (Bag<(MutableChatListView, ValuePipe<(ChatListView, ViewUpdateType)>)>.Index, Signal<(ChatListView, ViewUpdateType), NoError>) { + let record = (view, ValuePipe<(ChatListView, ViewUpdateType)>()) let index = self.chatListViews.add(record) self.updateTrackedChatListHoles() @@ -101,12 +101,12 @@ final class ViewTracker { return (index, record.1.signal()) } - func removeChatListView(index: Bag<(MutableChatListView, Pipe)>.Index) { + func removeChatListView(_ index: Bag<(MutableChatListView, ValuePipe)>.Index) { self.chatListViews.remove(index) self.updateTrackedChatListHoles() } - func updateMessageHistoryViewVisibleRange(id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { + func updateMessageHistoryViewVisibleRange(_ id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { if let bag = self.messageHistoryViews[id.peerId] { for (mutableView, pipe) in bag.copyItems() { if mutableView.id == id { @@ -116,7 +116,7 @@ final class ViewTracker { let updateType: ViewUpdateType = .UpdateVisible if mutableView.updateVisibleRange(earliestVisibleIndex: earliestVisibleIndex, latestVisibleIndex: latestVisibleIndex, context: context) { - mutableView.complete(context, fetchEarlier: { index, count in + mutableView.complete(context: context, fetchEarlier: { index, count in return self.fetchEarlierHistoryEntries(id.peerId, index, count, mutableView.tagMask) }, fetchLater: { index, count in return self.fetchLaterHistoryEntries(id.peerId, index, count, mutableView.tagMask) @@ -138,7 +138,7 @@ final class ViewTracker { } } - func updateViews(currentOperationsByPeerId currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?]) { + func updateViews(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?]) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { @@ -158,7 +158,7 @@ final class ViewTracker { } if mutableView.replay(operations ?? [], holeFillDirections: peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: updatedMedia, context: context) { - mutableView.complete(context, fetchEarlier: { index, count in + mutableView.complete(context: context, fetchEarlier: { index, count in return self.fetchEarlierHistoryEntries(peerId, index, count, mutableView.tagMask) }, fetchLater: { index, count in return self.fetchLaterHistoryEntries(peerId, index, count, mutableView.tagMask) @@ -193,7 +193,7 @@ final class ViewTracker { for (mutableView, pipe) in self.chatListViews.copyItems() { let context = MutableChatListViewReplayContext() if mutableView.replay(chatListOperations, context: context) { - mutableView.complete(context, fetchEarlier: self.fetchEarlierChatEntries, fetchLater: self.fetchLaterChatEntries) + mutableView.complete(context: context, fetchEarlier: self.fetchEarlierChatEntries, fetchLater: self.fetchLaterChatEntries) mutableView.render(self.renderMessage) pipe.putNext((ChatListView(mutableView), .Generic)) } @@ -246,7 +246,7 @@ final class ViewTracker { } if !exists { disposeHoles.append(disposable) - self.chatListHoleDisposables.removeAtIndex(i) + self.chatListHoleDisposables.remove(at: i) } else { i += 1 } @@ -271,7 +271,7 @@ final class ViewTracker { } } - private func updateTrackedHoles(peerId: PeerId) { + private func updateTrackedHoles(_ peerId: PeerId) { if let bag = self.messageHistoryViews[peerId] { var disposeHoles: [Disposable] = [] var firstHolesAndTags: [(MessageHistoryHole, HoleFillDirection, MessageTags?)] = [] @@ -294,7 +294,7 @@ final class ViewTracker { } if !exists { disposeHoles.append(disposable) - self.holeDisposablesByPeerId[peerId]!.removeAtIndex(i) + self.holeDisposablesByPeerId[peerId]!.remove(at: i) } else { i += 1 } @@ -352,7 +352,7 @@ final class ViewTracker { } for index in removeIndices { - self.unsentMessageDisposables.removeValueForKey(index)?.dispose() + self.unsentMessageDisposables.removeValue(forKey: index)?.dispose() } for index in self.unsentMessageView.indices { @@ -370,9 +370,9 @@ final class ViewTracker { } } - private func synchronizeReadStateViewUpdated(updates: [PeerId: PeerReadStateSynchronizationOperation?]) { + private func synchronizeReadStateViewUpdated(_ updates: [PeerId: PeerReadStateSynchronizationOperation?]) { for (peerId, operation) in updates { - if let (_, disposable) = self.synchronizeReadStatesDisposables.removeValueForKey(peerId) { + if let (_, disposable) = self.synchronizeReadStatesDisposables.removeValue(forKey: peerId) { disposable.dispose() } diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index 96aa20e6da..6caf32aefb 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -85,7 +85,7 @@ class ChatListTableTests: XCTestCase { var randomId: Int64 = 0 arc4random_buf(&randomId, 8) - path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + path = NSTemporaryDirectory() + "\(randomId)" self.valueBox = SqliteValueBox(basePath: path!) let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: []) @@ -115,11 +115,11 @@ class ChatListTableTests: XCTestCase { self.chatListTable = nil self.valueBox = nil - let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + let _ = try? FileManager.default().removeItem(atPath: path!) self.path = nil } - private func addMessage(peerId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = []) { + private func addMessage(_ peerId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = []) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] @@ -128,7 +128,7 @@ class ChatListTableTests: XCTestCase { self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } - private func addHole(peerId: Int32, _ id: Int32) { + private func addHole(_ peerId: Int32, _ id: Int32) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] @@ -137,22 +137,22 @@ class ChatListTableTests: XCTestCase { self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } - private func addChatListHole(peerId: Int32, _ id: Int32, _ timestamp: Int32) { + private func addChatListHole(_ peerId: Int32, _ id: Int32, _ timestamp: Int32) { var operations: [ChatListOperation] = [] self.chatListTable!.addHole(ChatListHole(index: MessageIndex(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp)), operations: &operations) } - private func replaceChatListHole(peerId: Int32, _ id: Int32, _ timestamp: Int32, _ otherPeerId: Int32, _ otherId: Int32, _ otherTimestamp: Int32) { + private func replaceChatListHole(_ peerId: Int32, _ id: Int32, _ timestamp: Int32, _ otherPeerId: Int32, _ otherId: Int32, _ otherTimestamp: Int32) { var operations: [ChatListOperation] = [] self.chatListTable!.replaceHole(MessageIndex(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp), hole: ChatListHole(index: MessageIndex(id: MessageId(peerId: PeerId(namespace: namespace, id: otherPeerId), namespace: namespace, id: otherId), timestamp: otherTimestamp)), operations: &operations) } - private func removeChatListHole(peerId: Int32, _ id: Int32, _ timestamp: Int32) { + private func removeChatListHole(_ peerId: Int32, _ id: Int32, _ timestamp: Int32) { var operations: [ChatListOperation] = [] self.chatListTable!.replaceHole(MessageIndex(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp), hole: nil, operations: &operations) } - private func removeMessages(peerId: Int32, _ ids: [Int32]) { + private func removeMessages(_ peerId: Int32, _ ids: [Int32]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] @@ -161,7 +161,7 @@ class ChatListTableTests: XCTestCase { self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } - private func fillHole(peerId: Int32, _ id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32, String, [Media])]) { + private func fillHole(_ peerId: Int32, _ id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32, String, [Media])]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] @@ -170,7 +170,7 @@ class ChatListTableTests: XCTestCase { self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) } - private func expectEntries(entries: [Entry]) { + private func expectEntries(_ entries: [Entry]) { let actualEntries = self.chatListTable!.debugList(self.historyTable!).map({ entry -> Entry in switch entry { case let .Message(message): @@ -276,4 +276,4 @@ class ChatListTableTests: XCTestCase { removeChatListHole(1, 100, 100) expectEntries([]) } -} \ No newline at end of file +} diff --git a/PostboxTests/CodingTests.swift b/PostboxTests/CodingTests.swift index 51963433c8..d38ea7d91e 100644 --- a/PostboxTests/CodingTests.swift +++ b/PostboxTests/CodingTests.swift @@ -14,7 +14,7 @@ class TestParent: Coding, Equatable { self.parentInt32 = parentInt32 } - func encode(encoder: Encoder) { + func encode(_ encoder: Encoder) { encoder.encodeInt32(self.parentInt32, forKey: "parentInt32") } } @@ -47,7 +47,7 @@ class TestObject: TestParent { super.init(parentInt32: parentInt32) } - override func encode(encoder: Encoder) { + override func encode(_ encoder: Encoder) { encoder.encodeInt32(self.int32, forKey: "int32") encoder.encodeInt64(self.int64, forKey: "int64") encoder.encodeDouble(self.double, forKey: "double") @@ -68,7 +68,7 @@ class TestKey: Coding, Hashable { self.value = value } - func encode(encoder: Encoder) { + func encode(_ encoder: Encoder) { encoder.encodeInt32(Int32(self.value), forKey: "value") } @@ -101,7 +101,7 @@ class EmptyState: Coding { required init(decoder: Decoder) { } - func encode(encoder: Encoder) { + func encode(_ encoder: Encoder) { } } diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index 6b18fa2a88..43c44dde75 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -72,7 +72,7 @@ class MessageHistoryIndexTableTests: XCTestCase { var randomId: Int64 = 0 arc4random_buf(&randomId, 8) - path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + path = NSTemporaryDirectory() + "\(randomId)" self.valueBox = SqliteValueBox(basePath: path!) let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: []) @@ -88,28 +88,28 @@ class MessageHistoryIndexTableTests: XCTestCase { self.indexTable = nil self.valueBox = nil - let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + let _ = try? FileManager.default().removeItem(atPath: path!) self.path = nil } - func addHole(id: Int32) { + func addHole(_ id: Int32) { var operations: [MessageHistoryIndexOperation] = [] self.indexTable!.addHole(MessageId(peerId: peerId, namespace: namespace, id: id), operations: &operations) } - func addMessage(id: Int32, _ timestamp: Int32) { + func addMessage(_ id: Int32, _ timestamp: Int32) { var operations: [MessageHistoryIndexOperation] = [] self.indexTable!.addMessages([InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: [])], location: .Random, operations: &operations) } - func addMessagesUpperBlock(messages: [(Int32, Int32)]) { + func addMessagesUpperBlock(_ messages: [(Int32, Int32)]) { var operations: [MessageHistoryIndexOperation] = [] self.indexTable!.addMessages(messages.map { (id, timestamp) in return InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: []) }, location: .UpperHistoryBlock, operations: &operations) } - func fillHole(id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32)], _ tagMask: MessageTags? = nil) { + func fillHole(_ id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32)], _ tagMask: MessageTags? = nil) { var operations: [MessageHistoryIndexOperation] = [] self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ @@ -117,12 +117,12 @@ class MessageHistoryIndexTableTests: XCTestCase { }), operations: &operations) } - func removeMessage(id: Int32) { + func removeMessage(_ id: Int32) { var operations: [MessageHistoryIndexOperation] = [] self.indexTable!.removeMessage(MessageId(peerId: peerId, namespace: namespace, id: id), operations: &operations) } - private func expect(items: [Item]) { + private func expect(_ items: [Item]) { let actualList = self.indexTable!.debugList(peerId, namespace: namespace) let actualItems = actualList.map { return Item($0) } if items != actualItems { diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index 8a8726aa73..713bd60ff2 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -70,11 +70,11 @@ private class TestEmbeddedMedia: Media, CustomStringConvertible { self.data = decoder.decodeStringForKey("s") } - func encode(encoder: Encoder) { + func encode(_ encoder: Encoder) { encoder.encodeString(self.data, forKey: "s") } - func isEqual(other: Media) -> Bool { + func isEqual(_ other: Media) -> Bool { if let other = other as? TestEmbeddedMedia { return self.data == other.data } @@ -101,13 +101,13 @@ private class TestExternalMedia: Media { self.data = decoder.decodeStringForKey("s") } - func encode(encoder: Encoder) { + func encode(_ encoder: Encoder) { encoder.encodeInt32(self.id!.namespace, forKey: "i.n") encoder.encodeInt64(self.id!.id, forKey: "i.i") encoder.encodeString(self.data, forKey: "s") } - func isEqual(other: Media) -> Bool { + func isEqual(_ other: Media) -> Bool { if let other = other as? TestExternalMedia { return self.id == other.id && self.data == other.data } @@ -133,13 +133,13 @@ private class TestPeer: Peer { self.data = decoder.decodeStringForKey("s") } - func encode(encoder: Encoder) { + func encode(_ encoder: Encoder) { encoder.encodeInt32(self.id.namespace, forKey: "i.n") encoder.encodeInt32(self.id.id, forKey: "i.i") encoder.encodeString(self.data, forKey: "s") } - func isEqual(other: Peer) -> Bool { + func isEqual(_ other: Peer) -> Bool { if let other = other as? TestPeer { return self.id == other.id && self.data == other.data } @@ -218,7 +218,7 @@ class MessageHistoryTableTests: XCTestCase { var randomId: Int64 = 0 arc4random_buf(&randomId, 8) - path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + path = NSTemporaryDirectory() + "\(randomId)" self.valueBox = SqliteValueBox(basePath: path!) let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) @@ -248,46 +248,46 @@ class MessageHistoryTableTests: XCTestCase { self.historyMetadataTable = nil self.valueBox = nil - let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + let _ = try? FileManager.default().removeItem(atPath: path!) self.path = nil } - private func addMessage(id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags = [], _ tags: MessageTags = []) { + private func addMessage(_ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags = [], _ tags: MessageTags = []) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func updateMessage(previousId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags, _ tags: MessageTags) { + private func updateMessage(_ previousId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags, _ tags: MessageTags) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func addHole(id: Int32) { + private func addHole(_ id: Int32) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.addHoles([MessageId(peerId: peerId, namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func removeMessages(ids: [Int32]) { + private func removeMessages(_ ids: [Int32]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func fillHole(id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32, String, [Media])], _ tagMask: MessageTags? = nil) { + private func fillHole(_ id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32, String, [Media])], _ tagMask: MessageTags? = nil) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func expectEntries(entries: [Entry], tagMask: MessageTags? = nil) { + private func expectEntries(_ entries: [Entry], tagMask: MessageTags? = nil) { var stableIds = Set() let list: [RenderedMessageHistoryEntry] @@ -330,7 +330,7 @@ class MessageHistoryTableTests: XCTestCase { } } - private func expectUnsent(indices: [(Int32, Int32)]) { + private func expectUnsent(_ indices: [(Int32, Int32)]) { let actualUnsent = self.unsentTable!.get().map({ ($0.id.id, $0.timestamp) }) var match = true if actualUnsent.count == indices.count { @@ -348,14 +348,14 @@ class MessageHistoryTableTests: XCTestCase { } } - private func expectMedia(media: [MediaEntry]) { + private func expectMedia(_ media: [MediaEntry]) { let actualMedia = self.mediaTable!.debugList().map({MediaEntry($0)}) if media != actualMedia { XCTFail("Expected\n\(media)\nActual\n\(actualMedia)") } } - private func expectCleanupMedia(media: [Media]) { + private func expectCleanupMedia(_ media: [Media]) { let actualMedia = self.mediaCleanupTable!.debugList() var equal = true if media.count != actualMedia.count { diff --git a/PostboxTests/ReadStateTableTests.swift b/PostboxTests/ReadStateTableTests.swift index bd7e2e1688..eea587f1aa 100644 --- a/PostboxTests/ReadStateTableTests.swift +++ b/PostboxTests/ReadStateTableTests.swift @@ -82,7 +82,7 @@ class ReadStateTableTests: XCTestCase { var randomId: Int64 = 0 arc4random_buf(&randomId, 8) - path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + path = NSTemporaryDirectory() + "\(randomId)" self.valueBox = SqliteValueBox(basePath: path!) let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) @@ -110,39 +110,39 @@ class ReadStateTableTests: XCTestCase { self.historyMetadataTable = nil self.valueBox = nil - let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + let _ = try? FileManager.default().removeItem(atPath: path!) self.path = nil } - private func addMessage(id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags = [], _ tags: MessageTags = []) { + private func addMessage(_ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags = [], _ tags: MessageTags = []) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func updateMessage(previousId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags, _ tags: MessageTags) { + private func updateMessage(_ previousId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags, _ tags: MessageTags) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func addHole(id: Int32) { + private func addHole(_ id: Int32) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.addHoles([MessageId(peerId: peerId, namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func removeMessages(ids: [Int32]) { + private func removeMessages(_ ids: [Int32]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } - private func expectApplyRead(messageId: Int32, _ expectInvalidate: Bool) { + private func expectApplyRead(_ messageId: Int32, _ expectInvalidate: Bool) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] self.historyTable!.applyIncomingReadMaxId(MessageId(peerId: peerId, namespace: namespace, id: messageId), operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) @@ -153,20 +153,20 @@ class ReadStateTableTests: XCTestCase { } } - private func expectReadState(maxReadId: Int32, _ maxKnownId: Int32, _ count: Int32) { + private func expectReadState(_ maxReadId: Int32, _ maxKnownId: Int32, _ count: Int32) { if let state = self.readStateTable!.getCombinedState(peerId)?.states.first?.1 { - if state.maxReadId != maxReadId || state.maxKnownId != maxKnownId || state.count != count { - XCTFail("Expected\nmaxReadId: \(maxReadId), maxKnownId: \(maxKnownId), count: \(count)\nActual\nmaxReadId: \(state.maxReadId), maxKnownId: \(state.maxKnownId), count: \(state.count)") + if state.maxIncomingReadId != maxReadId || state.maxKnownId != maxKnownId || state.count != count { + XCTFail("Expected\nmaxIncomingReadId: \(maxReadId), maxKnownId: \(maxKnownId), count: \(count)\nActual\nmaxIncomingReadId: \(state.maxIncomingReadId), maxKnownId: \(state.maxKnownId), count: \(state.count)") } } else { - XCTFail("Expected\nmaxReadId: (maxReadId), maxKnownId: \(maxKnownId), count: \(count)\nActual\nnil") + XCTFail("Expected\nmaxReadId: \(maxReadId), maxKnownId: \(maxKnownId), count: \(count)\nActual\nnil") } } func testResetState() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) expectReadState(100, 120, 130) } @@ -174,7 +174,7 @@ class ReadStateTableTests: XCTestCase { func testAddIncomingBeforeKnown() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.addMessage(99, 99, "", [], [.Incoming]) @@ -184,7 +184,7 @@ class ReadStateTableTests: XCTestCase { func testAddIncomingAfterKnown() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.addMessage(130, 130, "", [], [.Incoming]) @@ -194,19 +194,19 @@ class ReadStateTableTests: XCTestCase { func testApplyReadThenAddIncoming() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.expectApplyRead(200, false) self.addMessage(130, 130, "", [], [.Incoming]) - expectReadState(200, 200, 0) + expectReadState(200, 100, 0) } func testApplyAddIncomingThenRead() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.addMessage(130, 130, "", [], [.Incoming]) @@ -214,13 +214,13 @@ class ReadStateTableTests: XCTestCase { self.expectApplyRead(200, false) - expectReadState(200, 200, 0) + expectReadState(200, 100, 0) } func testIgnoreOldRead() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.expectApplyRead(90, false) @@ -230,14 +230,14 @@ class ReadStateTableTests: XCTestCase { func testInvalidateReadHole() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxReadId: 100, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.addMessage(200, 200) self.addHole(1) self.expectApplyRead(200, true) - expectReadState(200, 200, 0) + expectReadState(200, 100, 0) } diff --git a/submodules/SQLite.swift b/submodules/SQLite.swift index 1b1e295b29..95e5898eb3 160000 --- a/submodules/SQLite.swift +++ b/submodules/SQLite.swift @@ -1 +1 @@ -Subproject commit 1b1e295b293aceb3071c0f1a56ec26096aa87caf +Subproject commit 95e5898eb392626957d0cb25662c98c38c6b4677 From 2388cf484efdbe063f278269e7686cc639caa22e Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 6 Jul 2016 15:41:57 +0300 Subject: [PATCH 036/427] no message --- Postbox.xcodeproj/project.pbxproj | 15 + Postbox/Postbox.swift | 26 ++ Postbox/RedBlackTree.swift | 448 +++++++++++++++++++++ PostboxTests/OrderStatisticTreeTests.swift | 125 ++++++ 4 files changed, 614 insertions(+) create mode 100644 Postbox/RedBlackTree.swift create mode 100644 PostboxTests/OrderStatisticTreeTests.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 173fcd650d..2ac23d89f8 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -16,6 +16,7 @@ D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; + D02EB8071D2B07F300D07ED3 /* OrderStatisticTreeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */; }; D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; D03BCCF81C73561C0097A291 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; @@ -38,6 +39,7 @@ D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; + D08CEFB41D2AD8BE0015D3BC /* RedBlackTree.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */; }; D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; @@ -92,6 +94,7 @@ D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; + D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTreeTests.swift; sourceTree = ""; }; D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryUnsentTable.swift; sourceTree = ""; }; D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageHistoryView.swift; sourceTree = ""; }; D03BCCF71C73561C0097A291 /* Table.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Table.swift; sourceTree = ""; }; @@ -115,6 +118,7 @@ D08C71391C501F0700779C0F /* MessageHistoryHole.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHole.swift; sourceTree = ""; }; D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTable.swift; sourceTree = ""; }; D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTable.swift; sourceTree = ""; }; + D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RedBlackTree.swift; sourceTree = ""; }; D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = ""; }; D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; @@ -275,6 +279,7 @@ D0E3A7871B28AE9C00A402D9 /* Coding.swift */, D0F9E8741C5A334100037222 /* SimpleDictionary.swift */, D0C9DA381C65782500855278 /* SimpleSet.swift */, + D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */, ); name = Utils; sourceTree = ""; @@ -361,6 +366,7 @@ D0F9E8681C58FA9300037222 /* ChatListTableTests.swift */, D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */, D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */, + D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */, ); path = PostboxTests; sourceTree = ""; @@ -443,6 +449,7 @@ TargetAttributes = { D0E3A7491B28A7E300A402D9 = { CreatedOnToolsVersion = 6.3.1; + ProvisioningStyle = Manual; }; D0E3A7541B28A7E300A402D9 = { CreatedOnToolsVersion = 6.3.1; @@ -494,6 +501,7 @@ D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */, + D08CEFB41D2AD8BE0015D3BC /* RedBlackTree.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */, @@ -548,6 +556,7 @@ D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */, D0F9E8691C58FA9300037222 /* ChatListTableTests.swift in Sources */, D060B77B1CF4845A0050BE9B /* ReadStateTableTests.swift in Sources */, + D02EB8071D2B07F300D07ED3 /* OrderStatisticTreeTests.swift in Sources */, D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -609,6 +618,7 @@ isa = XCBuildConfiguration; buildSettings = { CLANG_ENABLE_MODULES = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution"; DEFINES_MODULE = YES; DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; @@ -627,6 +637,7 @@ ); PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; }; @@ -744,6 +755,7 @@ baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; buildSettings = { CLANG_ENABLE_MODULES = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; DEFINES_MODULE = YES; DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; @@ -762,6 +774,7 @@ ); PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; }; @@ -771,6 +784,7 @@ isa = XCBuildConfiguration; buildSettings = { CLANG_ENABLE_MODULES = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution"; DEFINES_MODULE = YES; DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; @@ -789,6 +803,7 @@ ); PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; }; diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 2db55cc904..3a3a6c3cf5 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -799,6 +799,32 @@ public final class Postbox { } } + public func messageAtId(_ id: MessageId) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.queue.justDispatch { + if let entry = self.messageHistoryIndexTable.get(id), case let .Message(index) = entry { + if let message = self.messageHistoryTable.getMessage(index) { + subscriber.putNext(self.renderIntermediateMessage(message)) + subscriber.putCompletion() + } else { + subscriber.putNext(nil) + subscriber.putCompletion() + } + } else if let _ = self.messageHistoryIndexTable.holeContainingId(id) { + subscriber.putNext(nil) + subscriber.putCompletion() + } else { + subscriber.putNext(nil) + subscriber.putCompletion() + } + } + + return disposable + } + } + public func tailChatListView(_ count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { return self.aroundChatListView(MessageIndex.absoluteUpperBound(), count: count) } diff --git a/Postbox/RedBlackTree.swift b/Postbox/RedBlackTree.swift new file mode 100644 index 0000000000..c711a6d3ff --- /dev/null +++ b/Postbox/RedBlackTree.swift @@ -0,0 +1,448 @@ +public class RBTNode { + private(set) public var color: Bool + private(set) public var left: RBTNode! + private(set) public var right: RBTNode! + private(set) public var parent: RBTNode! + private(set) public var data: Int + init() { + self.data = -1 + self.color = true + self.left = nil + self.right = nil + self.parent = nil + } + init(rootData: Int) { + self.data = rootData + self.color = true //0 is black 1 is red + self.left = nil + self.right = nil + self.parent = nil + } + deinit { + //print("Node: \(self.data) is bein deinitialized") + } + + public func grandparent() -> RBTNode? { + if self.parent === nil || self.parent.parent === nil { + return nil + } else { + return self.parent.parent + } + } + public func sibling() -> RBTNode? { + if self.parent === nil || self.parent.right === nil || self.parent.left === nil { + return nil + } + if self === self.parent!.left! { + return self.parent!.right! + } else { + return self.parent!.left! + } + } +} +public class RBTree { + private(set) public var root: RBTNode? + init(rootData: Int) { + root = RBTNode(rootData: rootData) + root!.color = false + } + init() { + root = nil + } + public func depth() -> Int { + let n = depth(root!) + return n + } + //return the max depth of the tree + private func depth(_ rooty: RBTNode?) -> Int { + if rooty == nil { + return 0 + } else { + return 1+(max(depth(root!.left), depth(root!.right))) + } + } + + public func inOrder() { + inOrder(root) + } + //Prints the in order traversal of the current tree + private func inOrder(_ root: RBTNode?) { + if self.root == nil { + //print("The tree is empty.") + } + if root == nil { + return + } + inOrder(root!.left) + //print("Data: \(root!.data) Color: \(root!.color)") + inOrder(root!.right) + } + /* + Basic Algorithm: + Let Q be P's right child. + Set P's right child to be Q's left child. + [Set Q's left-child's parent to P] + Set Q's left child to be P. + [Set P's parent to Q] + */ + private func leftRotate(_ x: RBTNode) { + let newRoot = x.right! + x.right = newRoot.left + if newRoot.left !== nil { + newRoot.left.parent = x + } + newRoot.parent = x.parent + if x.parent === nil { + root = newRoot + } else if x === x.parent.left { + x.parent.left = newRoot + } else { + x.parent.right = newRoot + } + newRoot.left = x + x.parent = newRoot + } + /* + Basic Algorithm: + Let P be Q's left child. + Set Q's left child to be P's right child. + [Set P's right-child's parent to Q] + Set P's right child to be Q. + [Set Q's parent to P] + */ + private func rightRotate(_ x: RBTNode) { + let newRoot = x.left! + x.left = newRoot.right + if newRoot.right !== nil { + newRoot.right.parent = x + } + newRoot.parent = x.parent + if x.parent === nil { + root = newRoot + } else if x === x.parent.right { + x.parent.right = newRoot + } else { + x.parent.left = newRoot + } + newRoot.right = x + x.parent = newRoot + } + public func insertFixup(_ value: Int) { + let inserted = find(value) + //print("Inserted Node: \(inserted!.data)") + insertCase1(inserted) + } + //Case where root is the only node + private func insertCase1(_ inserted: RBTNode?) { + let myroot = self.root! + if myroot === inserted! { + self.root!.color = false + } + insertCase2(inserted!) + } + //Case for inserting a node as a child of a black node + private func insertCase2(_ inserted: RBTNode?) { + if inserted!.parent!.color == false { + return + } + insertCase3(inserted!) + } + //Insert case for if the parent is black and parent's siblng is black + private func insertCase3(_ inserted: RBTNode?) { + if inserted!.parent!.sibling() != nil && + inserted!.parent!.sibling()!.color == true { + inserted!.parent!.color = false + inserted!.parent!.sibling()!.color = false + let g = inserted!.grandparent + g()!.color = true + if g()!.parent == nil { + g()!.color = false + } + } + insertCase4(inserted) + } + //Insert case for Node N is left of parent and parent is right of grandparent + private func insertCase4(_ insert: RBTNode?) { + var inserted = insert + if (inserted! === inserted!.parent!.right) && + (inserted!.grandparent()!.left === inserted!.parent!) { + + leftRotate(inserted!.parent) + inserted! = inserted!.left + } else if (inserted! === inserted!.parent!.left) && + (inserted!.grandparent()!.right === inserted!.parent!) { + + rightRotate(inserted!.parent) + inserted! = inserted!.right + } + insertCase5(inserted) + } + //Insert case for Node n where parent is red and parent's sibling is black + private func insertCase5(_ inserted: RBTNode?) { + if inserted!.parent!.color == true && + (inserted!.parent!.sibling()?.color == false || + inserted!.parent!.sibling() == nil) { + + if inserted! === inserted!.parent!.left && inserted!.grandparent()!.left === inserted!.parent! { + inserted!.parent.color = false + inserted!.grandparent()?.color = true + if inserted! === inserted!.parent!.left { + rightRotate(inserted!.grandparent()!) + } + } else if inserted! === inserted!.parent!.right && inserted!.grandparent()!.right === inserted!.parent! { + inserted!.parent.color = false + inserted!.grandparent()?.color = true + leftRotate(inserted!.grandparent()!) + } + } + } + + public func insert(_ value: Int) { + insert(value, parent: root!) + insertFixup(value) + } + //Basic BST insert implementation + private func insert(_ value: Int, parent: RBTNode) { + if self.root == nil { + self.root = RBTNode(rootData: value) + return + } else if value < parent.data { + if let left = parent.left { + insert(value, parent: left) + } else { + parent.left = RBTNode(rootData: value) + parent.left?.parent = parent + } + } else { + if let right = parent.right { + insert(value, parent: right) + } else { + parent.right = RBTNode(rootData: value) + parent.right?.parent = parent + } + } + } + public func find(_ data: Int) -> RBTNode? { + return find(root!, data: data) + } + //Returns the reference to the RBTNode whos data was requested + private func find(_ root: RBTNode, data: Int) -> RBTNode? { + if data == root.data { + return root + } + if root.data != data && root.right == nil && root.left == nil { + return nil + } else if data > root.data { + return find(root.right, data: data) + } else if data < root.data { + return find(root.left, data: data) + } else { + return nil + } + } + + //DELETION HELPER FUNCTIONS + public func remove(_ value: Int) { + let toRemove = find(value) + if toRemove == nil { + return + } + } + //Transplant the positions of two nodes in the RBTree + public func replaceNode(_ n1: RBTNode, n2: RBTNode) { + let temp = n1.data + let temp_color = n1.color + n1.data = n2.data + n1.color = n2.color + n2.data = temp + n2.color = temp_color + } + //returns the node with the minimum value in the subtree + public func minimum(_ node: RBTNode) -> RBTNode { + var minimumNode = node + while minimumNode.left !== nil { + minimumNode = minimumNode.left + } + return minimumNode + } + //Returns the next largest node in the tree + public func successor(_ node: RBTNode) -> RBTNode? { + var nextLargestNode = node + if nextLargestNode.right !== nil { + return minimum(nextLargestNode.right) + } + var successor = nextLargestNode.parent + while successor !== nil && nextLargestNode === successor!.right { + nextLargestNode = successor! + successor = successor!.parent + } + return successor + } + //Returns the next smallest node in the tree + public func predecessor(_ node: RBTNode) -> RBTNode { + var nextSmallestNode = node + if nextSmallestNode.left !== nil { + return minimum(nextSmallestNode.left) + } + var successor = nextSmallestNode.parent + while successor !== nil && nextSmallestNode === successor!.left { + nextSmallestNode = successor! + successor = successor!.parent + } + return successor! + } + //Returns the node with the largest value in the subtree + public func maximum(_ rootnode: RBTNode) -> RBTNode { + var rootNode = rootnode + while rootNode.right !== nil { + rootNode = rootNode.right + } + return rootNode + } + //call to remove a node from the tree + public func delete(_ x: Int) { + let toDel = find(x) + deleteNode(toDel!) + } + //Function call for removing a node + private func deleteNode(_ todel: RBTNode?) { + var toDel = todel + //case for if todel is the only node in the tree + if toDel!.left === nil && toDel!.right === nil && toDel!.parent === nil { + toDel = nil + self.root = nil + return + } + //case for if toDell is a red node w/ no children + if toDel!.left === nil && toDel!.right === nil && toDel!.color == true { + if toDel!.parent.left === toDel! { + toDel!.parent.left = nil + toDel = nil + } else if toDel!.parent === nil { + toDel = nil + } else { + toDel?.parent.right = nil + toDel = nil + } + return + } + //case for toDel having two children + if toDel!.left !== nil && toDel!.right !== nil { + let pred = maximum(toDel!.left!) + toDel!.data = pred.data + toDel! = pred + } + //case for toDel having one child + var child: RBTNode? = nil + if toDel!.right === nil { + child = toDel!.left + } else { + child = toDel!.right + } + if toDel!.color == false && child !== nil { + toDel!.color = child!.color + deleteCase1(toDel!) + } + + if child !== nil { + replaceNode(toDel!, n2: child!) + if toDel!.parent === nil && child !== nil { + child!.color = false + } + } + if toDel!.parent.left === toDel! { + toDel!.parent.left = nil + toDel = nil + return + } else if toDel!.parent === nil { + toDel = nil + return + } else { + toDel?.parent.right = nil + toDel = nil + return + } + } + //delete case for if parent is nil after deletion + private func deleteCase1(_ toDel: RBTNode?) { + if toDel?.parent === nil { + return + } else { + deleteCase2(toDel) + } + } + //case to fix tree after deletion and sibling is red + private func deleteCase2(_ toDel: RBTNode?) { + let sibling = toDel!.sibling() + if sibling?.color == true { + toDel!.parent.color = true + sibling?.color = false + if toDel! === toDel!.parent.left { + leftRotate(toDel!.parent) + } else { + rightRotate(toDel!.parent) + } + deleteCase3(toDel!) + } + } + //delete case for fixing tree when parnet is black and sibling is black and sibling.children are also black + private func deleteCase3(_ toDel: RBTNode?) { + if toDel!.parent?.color == false && + toDel!.sibling()?.color == false && + toDel!.sibling()?.left!.color == false && + toDel!.sibling()?.right!.color == false { + + toDel!.sibling()?.color = true + toDel!.parent?.color = false + } else { + deleteCase4(toDel) + } + } + private func deleteCase4(_ toDel: RBTNode?) { + if toDel!.parent?.color == true && + toDel!.sibling()?.color == false && + toDel!.sibling()?.left!.color == false && + toDel!.sibling()?.right!.color == false { + + toDel!.sibling()?.color = true + toDel!.parent.color = false + } else { + deleteCase5(toDel) + } + } + //delete case for fixing tree if toDel is a left child and sibling(n) is black and children of sibling(n) are black and white respectibely + private func deleteCase5(_ toDel: RBTNode?) { + if toDel! === toDel!.parent?.left && + toDel!.sibling()?.color == false && + toDel!.sibling()?.left.color == true && + toDel!.sibling()?.right.color == false { + + toDel!.sibling()?.color = true + toDel!.sibling()?.left?.color = false + rightRotate(toDel!.sibling()!) + } + //opposite case + else if toDel! === toDel!.parent?.right && + toDel!.sibling()?.color == false && + toDel!.sibling()?.left.color == false && + toDel!.sibling()?.right.color == true { + + toDel!.sibling()?.color = true + toDel!.sibling()?.right?.color = false + leftRotate(toDel!.sibling()!) + } + } + //final rotations to be done after deleting a black node from the tree + private func deleteCase6(_ toDel: RBTNode?) { + let color = toDel!.sibling()?.color + toDel!.parent?.color = color! + toDel!.parent?.color = false + if toDel! === toDel!.parent.left { + toDel!.sibling()?.right?.color = false + leftRotate(toDel!.parent!) + } else { + toDel!.sibling()?.left?.color = false + rightRotate(toDel!.parent!) + } + } +} diff --git a/PostboxTests/OrderStatisticTreeTests.swift b/PostboxTests/OrderStatisticTreeTests.swift new file mode 100644 index 0000000000..10ce2c8eff --- /dev/null +++ b/PostboxTests/OrderStatisticTreeTests.swift @@ -0,0 +1,125 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +private let peerId = PeerId(namespace: 1, id: 1) +private let namespace: Int32 = 1 +private let authorPeerId = PeerId(namespace: 1, id: 6) + +private func ==(lhs: [Media], rhs: [Media]) -> Bool { + if lhs.count != rhs.count { + return false + } + + for i in 0 ..< lhs.count { + if !lhs[i].isEqual(rhs[i]) { + return false + } + } + return true +} + +private enum Entry: Equatable, CustomStringConvertible { + case Message(Int32, Int32, String, [Media], MessageFlags) + case Hole(Int32, Int32, Int32) + + var description: String { + switch self { + case let .Message(id, timestamp, text, media, flags): + return "Message(\(id), \(timestamp), \(text), \(media), \(flags))" + case let .Hole(min, max, timestamp): + return "Hole(\(min), \(max), \(timestamp))" + } + } +} + +private func ==(lhs: Entry, rhs: Entry) -> Bool { + switch lhs { + case let .Message(lhsId, lhsTimestamp, lhsText, lhsMedia, lhsFlags): + switch rhs { + case let .Message(rhsId, rhsTimestamp, rhsText, rhsMedia, rhsFlags): + return lhsId == rhsId && lhsTimestamp == rhsTimestamp && lhsText == rhsText && lhsMedia == rhsMedia && lhsFlags == rhsFlags + case .Hole: + return false + } + case let .Hole(lhsMin, lhsMax, lhsMaxTimestamp): + switch rhs { + case .Message: + return false + case let .Hole(rhsMin, rhsMax, rhsMaxTimestamp): + return lhsMin == rhsMin && lhsMax == rhsMax && lhsMaxTimestamp == rhsMaxTimestamp + } + } +} + +private extension MessageTags { + static let First = MessageTags(rawValue: 1 << 0) + static let Second = MessageTags(rawValue: 1 << 1) +} + +class OrderStatisticTreeTests: XCTestCase { + var valueBox: ValueBox? + var path: String? + + var peerTable: PeerTable? + var globalMessageIdsTable: GlobalMessageIdsTable? + var indexTable: MessageHistoryIndexTable? + var mediaTable: MessageMediaTable? + var mediaCleanupTable: MediaCleanupTable? + var historyTable: MessageHistoryTable? + var historyMetadataTable: MessageHistoryMetadataTable? + var unsentTable: MessageHistoryUnsentTable? + var tagsTable: MessageHistoryTagsTable? + var readStateTable: MessageHistoryReadStateTable? + var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? + + override func setUp() { + super.setUp() + + var randomId: Int64 = 0 + arc4random_buf(&randomId, 8) + path = NSTemporaryDirectory() + "\(randomId)" + self.valueBox = SqliteValueBox(basePath: path!) + + let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) + + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 5, namespace: namespace) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 7) + self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, tableId: 8) + self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, tableId: 9) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) + self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 10) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, tableId: 11) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + } + + override func tearDown() { + super.tearDown() + + self.historyTable = nil + self.indexTable = nil + self.mediaTable = nil + self.mediaCleanupTable = nil + self.peerTable = nil + self.historyMetadataTable = nil + + self.valueBox = nil + let _ = try? FileManager.default().removeItem(atPath: path!) + self.path = nil + } + + func testOST() { + let tree = RBTree(rootData: 0) + for _ in 0 ..< 1000 { + let key = Int(arc4random_uniform(UInt32(Int32.max - 1))) + tree.insert(key) + } + print("OST height: \(tree.depth())") + } +} From 9712c363344b7a2974d2371a08f6b8bda5ad92af Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 9 Aug 2016 00:23:02 +0300 Subject: [PATCH 037/427] no message --- Postbox.xcodeproj/project.pbxproj | 26 + .../xcschemes/xcschememanagement.plist | 2 +- Postbox/ChatListView.swift | 2 +- Postbox/Coding.swift | 14 +- Postbox/ContactTable.swift | 9 + Postbox/LmdbValueBox.swift | 8 +- Postbox/MappedFile.swift | 51 ++ Postbox/MediaBox.swift | 229 ++++++++- Postbox/MessageHistoryIndexTable.swift | 2 +- Postbox/MessageHistoryReadStateTable.swift | 2 +- Postbox/MessageHistoryTagsTable.swift | 19 + Postbox/MessageHistoryView.swift | 110 ++-- Postbox/OrderStatisticTable.swift | 59 +++ Postbox/Postbox.swift | 67 ++- .../RandomAccessMediaResourceContext.swift | 469 ++++++++++++++++++ Postbox/RedBlackTree.swift | 448 ----------------- Postbox/SqliteValueBox.swift | 6 +- PostboxTests/ChatListTableTests.swift | 2 +- .../MessageHistoryIndexTableTests.swift | 2 +- PostboxTests/MessageHistoryTableTests.swift | 2 +- PostboxTests/OrderStatisticTreeTests.swift | 6 +- PostboxTests/RandomAccessResourceTests.swift | 413 +++++++++++++++ PostboxTests/ReadStateTableTests.swift | 2 +- 23 files changed, 1409 insertions(+), 541 deletions(-) create mode 100644 Postbox/ContactTable.swift create mode 100644 Postbox/MappedFile.swift create mode 100644 Postbox/OrderStatisticTable.swift create mode 100644 Postbox/RandomAccessMediaResourceContext.swift create mode 100644 PostboxTests/RandomAccessResourceTests.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 2ac23d89f8..826a081033 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -8,6 +8,7 @@ /* Begin PBXBuildFile section */ D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; + D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB41B85D192002E4EB5 /* lmdb.h */; }; D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-unreachable-code -Wno-conditional-uninitialized -Wno-format-extra-args -Wno-macro-redefined"; }; }; D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB61B85D192002E4EB5 /* midl.c */; settings = {COMPILER_FLAGS = "-Wno-conversion"; }; }; @@ -43,6 +44,8 @@ D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; + D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; + D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; @@ -51,6 +54,8 @@ D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */; }; D0C9DA391C65782500855278 /* SimpleSet.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C9DA381C65782500855278 /* SimpleSet.swift */; }; D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; + D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */; }; + D0D949F51D35353900740E02 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; @@ -86,6 +91,7 @@ /* Begin PBXFileReference section */ D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryView.swift; sourceTree = ""; }; + D0079F591D592E8B00A27A2C /* ContactTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactTable.swift; sourceTree = ""; }; D00E0FB41B85D192002E4EB5 /* lmdb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lmdb.h; path = submodules/lmdb/libraries/liblmdb/lmdb.h; sourceTree = SOURCE_ROOT; }; D00E0FB51B85D192002E4EB5 /* mdb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = mdb.c; path = submodules/lmdb/libraries/liblmdb/mdb.c; sourceTree = SOURCE_ROOT; }; D00E0FB61B85D192002E4EB5 /* midl.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = midl.c; path = submodules/lmdb/libraries/liblmdb/midl.c; sourceTree = SOURCE_ROOT; }; @@ -122,6 +128,8 @@ D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = ""; }; D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; + D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessMediaResourceContext.swift; sourceTree = ""; }; + D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTable.swift; sourceTree = ""; }; D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; @@ -130,6 +138,8 @@ D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryViewTests.swift; sourceTree = ""; }; D0C9DA381C65782500855278 /* SimpleSet.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SimpleSet.swift; sourceTree = ""; }; D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResource.swift; sourceTree = ""; }; + D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessResourceTests.swift; sourceTree = ""; }; + D0D949F41D35353900740E02 /* MappedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MappedFile.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -194,6 +204,7 @@ D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */, D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */, D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */, + D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */, ); name = "Media Box"; sourceTree = ""; @@ -254,8 +265,10 @@ D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */, D0F9E8701C5A0E9B00037222 /* PeerTable.swift */, D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */, + D0079F591D592E8B00A27A2C /* ContactTable.swift */, D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */, D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */, + D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */, ); name = Tables; sourceTree = ""; @@ -280,6 +293,7 @@ D0F9E8741C5A334100037222 /* SimpleDictionary.swift */, D0C9DA381C65782500855278 /* SimpleSet.swift */, D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */, + D0D949F41D35353900740E02 /* MappedFile.swift */, ); name = Utils; sourceTree = ""; @@ -367,6 +381,7 @@ D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */, D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */, D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */, + D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */, ); path = PostboxTests; sourceTree = ""; @@ -501,6 +516,7 @@ D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */, + D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */, D08CEFB41D2AD8BE0015D3BC /* RedBlackTree.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, @@ -512,6 +528,7 @@ D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, + D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */, D0F9E8671C58D08900037222 /* ChatListIndexTable.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, @@ -519,8 +536,10 @@ D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */, D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */, + D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, + D0D949F51D35353900740E02 /* MappedFile.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, @@ -551,6 +570,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */, D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */, D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */, D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */, @@ -640,6 +660,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; }; name = Hockeyapp; }; @@ -655,6 +676,7 @@ LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 3.0; }; name = Hockeyapp; }; @@ -777,6 +799,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 3.0; }; name = Debug; }; @@ -806,6 +829,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; }; name = Release; }; @@ -826,6 +850,7 @@ LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 3.0; }; name = Debug; }; @@ -842,6 +867,7 @@ PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; }; name = Release; }; diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist index dd754c89ca..9eb93eab1b 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist @@ -12,7 +12,7 @@ PostboxTests.xcscheme orderHint - 3 + 4 SuppressBuildableAutocreation diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index 7efc992bd3..f3e0bc64e7 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -280,7 +280,7 @@ final class MutableChatListView { } addedEntries += self.entries - addedEntries.sort(isOrderedBefore: { $0.index < $1.index }) + addedEntries.sort(by: { $0.index < $1.index }) var i = addedEntries.count - 1 while i >= 1 { if addedEntries[i].index.id == addedEntries[i - 1].index.id { diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 08b32c12a4..c03cfacd1c 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -98,7 +98,7 @@ public func ==(lhs: MemoryBuffer, rhs: MemoryBuffer) -> Bool { } public final class WriteBuffer: MemoryBuffer { - var offset = 0 + public var offset = 0 public override init() { super.init(memory: malloc(32), capacity: 32, length: 0, freeWhenDone: true) @@ -155,7 +155,7 @@ public final class WriteBuffer: MemoryBuffer { } public final class ReadBuffer: MemoryBuffer { - var offset = 0 + public var offset = 0 public init(memory: UnsafeMutablePointer, length: Int, freeWhenDone: Bool) { super.init(memory: memory, capacity: length, length: length, freeWhenDone: freeWhenDone) @@ -165,24 +165,24 @@ public final class ReadBuffer: MemoryBuffer { super.init(memory: memoryBufferNoCopy.memory, capacity: memoryBufferNoCopy.length, length: memoryBufferNoCopy.length, freeWhenDone: false) } - func dataNoCopy() -> Data { + public func dataNoCopy() -> Data { return Data(bytesNoCopy: UnsafeMutablePointer(self.memory), count: self.length, deallocator: .none) } - func read(_ data: UnsafeMutablePointer, offset: Int, length: Int) { + public func read(_ data: UnsafeMutablePointer, offset: Int, length: Int) { memcpy(data + offset, self.memory + self.offset, length) self.offset += length } - func skip(_ length: Int) { + public func skip(_ length: Int) { self.offset += length } - func reset() { + public func reset() { self.offset = 0 } - func sharedBufferNoCopy() -> ReadBuffer { + public func sharedBufferNoCopy() -> ReadBuffer { return ReadBuffer(memory: memory, length: length, freeWhenDone: false) } } diff --git a/Postbox/ContactTable.swift b/Postbox/ContactTable.swift new file mode 100644 index 0000000000..2045f4d47d --- /dev/null +++ b/Postbox/ContactTable.swift @@ -0,0 +1,9 @@ +import Foundation + +final class ContactTable: Table { + + + override func beforeCommit() { + + } +} diff --git a/Postbox/LmdbValueBox.swift b/Postbox/LmdbValueBox.swift index d20c12264b..67263c1c05 100644 --- a/Postbox/LmdbValueBox.swift +++ b/Postbox/LmdbValueBox.swift @@ -113,10 +113,10 @@ public final class LmdbValueBox: ValueBox { var createDirectory = false var isDirectory: ObjCBool = false as ObjCBool - if FileManager.default().fileExists(atPath: path, isDirectory: &isDirectory) { - if !isDirectory { + if FileManager.default.fileExists(atPath: path, isDirectory: &isDirectory) { + if !isDirectory.boolValue { do { - try FileManager.default().removeItem(atPath: path) + try FileManager.default.removeItem(atPath: path) } catch _ { } createDirectory = true } @@ -127,7 +127,7 @@ public final class LmdbValueBox: ValueBox { if createDirectory { do { - try FileManager.default().createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil) + try FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil) } catch _ { } } diff --git a/Postbox/MappedFile.swift b/Postbox/MappedFile.swift new file mode 100644 index 0000000000..8c57238fde --- /dev/null +++ b/Postbox/MappedFile.swift @@ -0,0 +1,51 @@ +import Foundation + +public final class MappedFile { + private var handle: Int32 + private var currentSize: Int + private var memory: UnsafeMutablePointer + + public init(path: String) { + self.handle = open(path, O_RDWR | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) + + var value = stat() + stat(path, &value) + self.currentSize = Int(value.st_size) + + self.memory = mmap(nil, self.currentSize, PROT_READ | PROT_WRITE, MAP_SHARED, self.handle, 0) + } + + deinit { + munmap(self.memory, self.currentSize) + close(self.handle) + } + + public var size: Int { + get { + return self.currentSize + } set(value) { + if value != self.currentSize { + munmap(self.memory, self.currentSize) + ftruncate(self.handle, off_t(value)) + self.currentSize = value + self.memory = mmap(nil, self.currentSize, PROT_READ | PROT_WRITE, MAP_SHARED, self.handle, 0) + } + } + } + + public func synchronize() { + msync(self.memory, self.currentSize, MS_ASYNC) + } + + public func write(at range: Range, from data: UnsafePointer) { + memcpy(self.memory.advanced(by: range.lowerBound), data, range.count) + } + + public func read(at range: Range, to data: UnsafeMutablePointer) { + memcpy(data, self.memory.advanced(by: range.lowerBound), range.count) + } + + public func clear() { + memset(self.memory, 0, self.currentSize) + } +} diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 0a1124a70f..3a310e80db 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -25,6 +25,12 @@ private func fileSize(_ path: String) -> Int { return Int(value.st_size) } +public enum ResourceDataRangeMode { + case complete + case incremental + case partial +} + public final class MediaBox { let basePath: String let buffer = WriteBuffer() @@ -34,9 +40,10 @@ public final class MediaBox { private var statusContexts: [String: ResourceStatusContext] = [:] private var dataContexts: [String: ResourceDataContext] = [:] + private var randomAccessContexts: [String: RandomAccessMediaResourceContext] = [:] - private var wrappedFetchResource = Promise<(MediaResource, Int) -> Signal>() - public var fetchResource: ((MediaResource, Int) -> Signal)? { + private var wrappedFetchResource = Promise<(MediaResource, Range) -> Signal>() + public var fetchResource: ((MediaResource, Range) -> Signal)? { didSet { if let fetchResource = self.fetchResource { wrappedFetchResource.set(.single(fetchResource)) @@ -47,15 +54,25 @@ public final class MediaBox { } lazy var ensureDirectoryCreated: Void = { - try! FileManager.default().createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) + try! FileManager.default.createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) }() public init(basePath: String) { self.basePath = basePath + + let _ = self.ensureDirectoryCreated + } + + private func fileNameForId(_ id: String) -> String { + return id } private func pathForId(_ id: String) -> String { - return "\(self.basePath)/\(id)" + return "\(self.basePath)/\(fileNameForId(id))" + } + + private func streamingPathForId(_ id: String) -> String { + return "\(self.basePath)/\(id)_stream" } public func resourceStatus(_ resource: MediaResource) -> Signal { @@ -126,7 +143,7 @@ public final class MediaBox { } } - public func resourceData(_ resource: MediaResource, complete: Bool = true) -> Signal { + public func resourceData(_ resource: MediaResource, pathExtension: String? = nil, complete: Bool = true) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -135,7 +152,15 @@ public final class MediaBox { let currentSize = fileSize(path) if currentSize >= resource.size { - subscriber.putNext(MediaResourceData(path: path, size: currentSize)) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !FileManager.default.fileExists(atPath: pathWithExtension) { + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: currentSize)) + } else { + subscriber.putNext(MediaResourceData(path: path, size: currentSize)) + } subscriber.putCompletion() } else { let dataContext: ResourceDataContext @@ -148,25 +173,79 @@ public final class MediaBox { let index: Bag<(MediaResourceData) -> Void>.Index if complete { + var checkedForSymlink = false index = dataContext.completeDataSubscribers.add { data in - subscriber.putNext(data) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) + } else { + subscriber.putNext(data) + } + if data.size >= resource.size { subscriber.putCompletion() } } + if dataContext.data.size >= resource.size { - subscriber.putNext(dataContext.data) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: dataContext.data.size)) + } else { + subscriber.putNext(dataContext.data) + } } else { - subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0)) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: 0)) + } else { + subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0)) + } } } else { + var checkedForSymlink = false index = dataContext.progresiveDataSubscribers.add { data in - subscriber.putNext(data) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) + } else { + subscriber.putNext(data) + } if data.size >= resource.size { subscriber.putCompletion() } } - subscriber.putNext(dataContext.data) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: dataContext.data.size)) + } else { + subscriber.putNext(dataContext.data) + } } disposable.set(ActionDisposable { @@ -191,7 +270,127 @@ public final class MediaBox { } } - public func fetchedResource(_ resource: MediaResource, interactive: Bool) -> Signal { + private func randomAccessContext(for resource: MediaResource) -> RandomAccessMediaResourceContext { + assert(self.dataQueue.isCurrent()) + + let dataContext: RandomAccessMediaResourceContext + if let current = self.randomAccessContexts[resource.id] { + dataContext = current + } else { + let path = self.pathForId(resource.id) + ".random" + dataContext = RandomAccessMediaResourceContext(path: path, size: resource.size, fetchRange: { [weak self] range in + let disposable = MetaDisposable() + + if let strongSelf = self { + strongSelf.dataQueue.async { + let fetch = strongSelf.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in + return fetch(resource, range) + } + var offset = 0 + disposable.set(fetch.start(next: { [weak strongSelf] data in + if let strongSelf = strongSelf { + strongSelf.dataQueue.async { + if let dataContext = strongSelf.randomAccessContexts[resource.id] { + let storeRange = RandomAccessResourceStoreRange(offset: range.lowerBound + offset, data: data) + offset += data.count + dataContext.storeRanges([storeRange]) + } + } + } + })) + } + } + + return disposable + }) + self.randomAccessContexts[resource.id] = dataContext + } + return dataContext + } + + public func fetchedResourceData(_ resource: MediaResource, in range: Range) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.dataQueue.async { + let dataContext = self.randomAccessContext(for: resource) + + let listener = dataContext.addListenerForFetchedData(in: range) + + disposable.set(ActionDisposable { [weak self] in + if let strongSelf = self { + strongSelf.dataQueue.async { + if let dataContext = strongSelf.randomAccessContexts[resource.id] { + dataContext.removeListenerForFetchedData(listener) + if !dataContext.hasDataListeners() { + let _ = strongSelf.randomAccessContexts.removeValue(forKey: resource.id) + } + } + } + } + }) + } + + return disposable + } + } + + public func resourceData(_ resource: MediaResource, in range: Range, mode: ResourceDataRangeMode = .complete) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.dataQueue.async { + let dataContext = self.randomAccessContext(for: resource) + + let listenerMode: RandomAccessResourceDataRangeMode + switch mode { + case .complete: + listenerMode = .Complete + case .incremental: + listenerMode = .Incremental + case .partial: + listenerMode = .Partial + } + + var offset = 0 + + let listener = dataContext.addListenerForData(in: range, mode: listenerMode, updated: { [weak self] data in + if let strongSelf = self { + strongSelf.dataQueue.async { + subscriber.putNext(data) + + switch mode { + case .complete, .partial: + offset = max(offset, data.count) + case .incremental: + offset += data.count + } + if offset == range.count { + subscriber.putCompletion() + } + } + } + }) + + disposable.set(ActionDisposable { [weak self] in + if let strongSelf = self { + strongSelf.dataQueue.async { + if let dataContext = strongSelf.randomAccessContexts[resource.id] { + dataContext.removeListenerForData(listener) + if !dataContext.hasDataListeners() { + let _ = strongSelf.randomAccessContexts.removeValue(forKey: resource.id) + } + } + } + } + }) + } + + return disposable + } + } + + public func fetchedResource(_ resource: MediaResource) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -225,8 +424,8 @@ public final class MediaBox { var offset = currentSize var fd: Int32? - dataContext.fetchDisposable = (self.wrappedFetchResource.get() |> mapToSignal { fetch -> Signal in - return fetch(resource, offset) + dataContext.fetchDisposable = (self.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in + return fetch(resource, currentSize ..< resource.size) } |> afterDisposed { if let fd = fd { close(fd) @@ -250,6 +449,8 @@ public final class MediaBox { offset += data.count let updatedSize = offset + dataContext.data = MediaResourceData(path: path, size: updatedSize) + for subscriber in dataContext.progresiveDataSubscribers.copyItems() { subscriber(MediaResourceData(path: path, size: updatedSize)) } diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 3d6c9906e8..44e56fbe3f 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -364,7 +364,7 @@ final class MessageHistoryIndexTable: Table { return true }, limit: 1) - let sortedByIdMessages = messages.sorted(isOrderedBefore: {$0.id < $1.id}) + let sortedByIdMessages = messages.sorted(by: {$0.id < $1.id}) var remainingMessages = sortedByIdMessages diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index a9fe295875..d86aafbeee 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -205,7 +205,7 @@ final class MessageHistoryReadStateTable: Table { print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId.id) (before: \(states.namespaces))") } - if state.maxIncomingReadId < messageId.id || messageId.id == topMessageId { + if state.maxIncomingReadId < messageId.id || (messageId.id == topMessageId && state.count != 0) { var (deltaCount, holes) = incomingStatsInRange(state.maxIncomingReadId + 1, messageId.id) if traceReadStates { diff --git a/Postbox/MessageHistoryTagsTable.swift b/Postbox/MessageHistoryTagsTable.swift index 822ec4d25e..55c6412b36 100644 --- a/Postbox/MessageHistoryTagsTable.swift +++ b/Postbox/MessageHistoryTagsTable.swift @@ -38,6 +38,25 @@ class MessageHistoryTagsTable: Table { self.valueBox.remove(self.tableId, key: self.key(tagMask, index: index, key: self.sharedKey)) } + func entryLocation(at index: MessageIndex, tagMask: MessageTags) -> MessageHistoryEntryLocation? { + if let _ = self.valueBox.get(self.tableId, key: self.key(tagMask, index: index)) { + var greaterCount = 0 + self.valueBox.range(self.tableId, start: self.key(tagMask, index: index), end: self.upperBound(tagMask, peerId: index.id.peerId), keys: { _ in + greaterCount += 1 + return true + }, limit: 0) + + var lowerCount = 0 + self.valueBox.range(self.tableId, start: self.key(tagMask, index: index), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { _ in + lowerCount += 1 + return true + }, limit: 0) + + return MessageHistoryEntryLocation(index: lowerCount, count: greaterCount + lowerCount + 1) + } + return nil + } + func indicesAround(_ tagMask: MessageTags, index: MessageIndex, count: Int) -> (indices: [MessageIndex], lower: MessageIndex?, upper: MessageIndex?) { var lowerEntries: [MessageIndex] = [] var upperEntries: [MessageIndex] = [] diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index e48b4e0a1b..05371fe0ae 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -21,31 +21,67 @@ public func ==(lhs: MessageHistoryViewId, rhs: MessageHistoryViewId) -> Bool { } enum MutableMessageHistoryEntry { - case IntermediateMessageEntry(IntermediateMessage) - case MessageEntry(Message) - case HoleEntry(MessageHistoryHole) + case IntermediateMessageEntry(IntermediateMessage, MessageHistoryEntryLocation?) + case MessageEntry(Message, MessageHistoryEntryLocation?) + case HoleEntry(MessageHistoryHole, MessageHistoryEntryLocation?) var index: MessageIndex { switch self { - case let .IntermediateMessageEntry(message): + case let .IntermediateMessageEntry(message, _): return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .MessageEntry(message): + case let .MessageEntry(message, _): return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .HoleEntry(hole): + case let .HoleEntry(hole, _): return hole.maxIndex } } + + func updatedLocation(_ location: MessageHistoryEntryLocation?) -> MutableMessageHistoryEntry { + switch self { + case let .IntermediateMessageEntry(message, _): + return .IntermediateMessageEntry(message, location) + case let .MessageEntry(message, _): + return .MessageEntry(message, location) + case let .HoleEntry(hole, _): + return .HoleEntry(hole, location) + } + } +} + +public struct MessageHistoryEntryLocation: Equatable { + public let index: Int + public let count: Int + + var predecessor: MessageHistoryEntryLocation? { + if index == 0 { + return nil + } else { + return MessageHistoryEntryLocation(index: index - 1, count: count) + } + } + + var successor: MessageHistoryEntryLocation? { + if index == count - 1 { + return nil + } else { + return MessageHistoryEntryLocation(index: index + 1, count: count) + } + } +} + +public func ==(lhs: MessageHistoryEntryLocation, rhs: MessageHistoryEntryLocation) -> Bool { + return lhs.index == rhs.index && lhs.count == rhs.count } public enum MessageHistoryEntry: Comparable { - case MessageEntry(Message) - case HoleEntry(MessageHistoryHole) + case MessageEntry(Message, MessageHistoryEntryLocation?) + case HoleEntry(MessageHistoryHole, MessageHistoryEntryLocation?) public var index: MessageIndex { switch self { - case let .MessageEntry(message): + case let .MessageEntry(message, _): return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .HoleEntry(hole): + case let .HoleEntry(hole, _): return hole.maxIndex } } @@ -53,20 +89,20 @@ public enum MessageHistoryEntry: Comparable { public func ==(lhs: MessageHistoryEntry, rhs: MessageHistoryEntry) -> Bool { switch lhs { - case let .MessageEntry(lhsMessage): + case let .MessageEntry(lhsMessage, lhsLocation): switch rhs { case .HoleEntry: return false - case let .MessageEntry(rhsMessage): - if MessageIndex(lhsMessage) == MessageIndex(rhsMessage) && lhsMessage.flags == rhsMessage.flags { + case let .MessageEntry(rhsMessage, rhsLocation): + if MessageIndex(lhsMessage) == MessageIndex(rhsMessage) && lhsMessage.flags == rhsMessage.flags && lhsLocation == rhsLocation { return true } return false } - case let .HoleEntry(lhsHole): + case let .HoleEntry(lhsHole, lhsLocation): switch rhs { - case let .HoleEntry(rhsHole): - return lhsHole == rhsHole + case let .HoleEntry(rhsHole, rhsLocation): + return lhsHole == rhsHole && lhsLocation == rhsLocation case .MessageEntry: return false } @@ -199,13 +235,13 @@ final class MutableMessageHistoryView { switch operation { case let .InsertHole(hole): if tagMask == nil || (hole.tags & unwrappedTagMask) != 0 { - if self.add(.HoleEntry(hole), holeFillDirections: holeFillDirections) { + if self.add(.HoleEntry(hole, nil), holeFillDirections: holeFillDirections) { hasChanges = true } } case let .InsertMessage(intermediateMessage): if tagMask == nil || (intermediateMessage.tags.rawValue & unwrappedTagMask) != 0 { - if self.add(.IntermediateMessageEntry(intermediateMessage), holeFillDirections: holeFillDirections) { + if self.add(.IntermediateMessageEntry(intermediateMessage, nil), holeFillDirections: holeFillDirections) { hasChanges = true } } @@ -218,8 +254,8 @@ final class MutableMessageHistoryView { //self.combinedReadState = combinedReadState case let .UpdateEmbeddedMedia(index, embeddedMediaData): for i in 0 ..< self.entries.count { - if case let .IntermediateMessageEntry(message) = self.entries[i] where MessageIndex(message) == index { - self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia)) + if case let .IntermediateMessageEntry(message, _) = self.entries[i] where MessageIndex(message) == index { + self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), nil) hasChanges = true break } @@ -230,7 +266,7 @@ final class MutableMessageHistoryView { if !updatedMedia.isEmpty { for i in 0 ..< self.entries.count { switch self.entries[i] { - case let .MessageEntry(message): + case let .MessageEntry(message, _): var rebuild = false for media in message.media { if let mediaId = media.id, _ = updatedMedia[mediaId] { @@ -251,10 +287,10 @@ final class MutableMessageHistoryView { } } let updatedMessage = Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages) - self.entries[i] = .MessageEntry(updatedMessage) + self.entries[i] = .MessageEntry(updatedMessage, nil) hasChanges = true } - case let .IntermediateMessageEntry(message): + case let .IntermediateMessageEntry(message, _): var rebuild = false for mediaId in message.referencedMedia { if let media = updatedMedia[mediaId] where media?.id != mediaId { @@ -427,16 +463,16 @@ final class MutableMessageHistoryView { } func render(_ renderIntermediateMessage: (IntermediateMessage) -> Message) { - if let earlier = self.earlier, case let .IntermediateMessageEntry(intermediateMessage) = earlier { - self.earlier = .MessageEntry(renderIntermediateMessage(intermediateMessage)) + if let earlier = self.earlier, case let .IntermediateMessageEntry(intermediateMessage, location) = earlier { + self.earlier = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) } - if let later = self.later, case let .IntermediateMessageEntry(intermediateMessage) = later { - self.later = .MessageEntry(renderIntermediateMessage(intermediateMessage)) + if let later = self.later, case let .IntermediateMessageEntry(intermediateMessage, location) = later { + self.later = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) } for i in 0 ..< self.entries.count { - if case let .IntermediateMessageEntry(intermediateMessage) = self.entries[i] { - self.entries[i] = .MessageEntry(renderIntermediateMessage(intermediateMessage)) + if case let .IntermediateMessageEntry(intermediateMessage, location) = self.entries[i] { + self.entries[i] = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) } } } @@ -459,7 +495,7 @@ final class MutableMessageHistoryView { while i >= 0 || j < self.entries.count { if j < self.entries.count { - if case let .HoleEntry(hole) = self.entries[j] { + if case let .HoleEntry(hole, _) = self.entries[j] { if self.anchorIndex.index.id.namespace == hole.id.namespace { if self.anchorIndex.index.id.id >= hole.min && self.anchorIndex.index.id.id <= hole.maxIndex.id.id { return (hole, .AroundIndex(self.anchorIndex.index)) @@ -471,7 +507,7 @@ final class MutableMessageHistoryView { } if i >= 0 { - if case let .HoleEntry(hole) = self.entries[i] { + if case let .HoleEntry(hole, _) = self.entries[i] { if self.anchorIndex.index.id.namespace == hole.id.namespace { if self.anchorIndex.index.id.id >= hole.min && self.anchorIndex.index.id.id <= hole.maxIndex.id.id { return (hole, .AroundIndex(self.anchorIndex.index)) @@ -506,12 +542,12 @@ public final class MessageHistoryView { var entries: [MessageHistoryEntry] = [] for entry in mutableView.entries { switch entry { - case let .HoleEntry(hole): - entries.append(.HoleEntry(hole)) - case let .MessageEntry(message): - entries.append(.MessageEntry(message)) + case let .HoleEntry(hole, location): + entries.append(.HoleEntry(hole, location)) + case let .MessageEntry(message, location): + entries.append(.MessageEntry(message, location)) case .IntermediateMessageEntry: - assertionFailure("got IntermediateMessageEntry") + assertionFailure("unexpected IntermediateMessageEntry in MessageHistoryView.init()") } } self.entries = entries @@ -545,7 +581,7 @@ public final class MessageHistoryView { } if let _ = maxNamespaceIndex where index + 1 < entries.count { for i in index + 1 ..< entries.count { - if case let .MessageEntry(message) = entries[i] where !message.flags.contains(.Incoming) { + if case let .MessageEntry(message, _) = entries[i] where !message.flags.contains(.Incoming) { maxNamespaceIndex = MessageIndex(message) } else { break diff --git a/Postbox/OrderStatisticTable.swift b/Postbox/OrderStatisticTable.swift new file mode 100644 index 0000000000..758abf9f86 --- /dev/null +++ b/Postbox/OrderStatisticTable.swift @@ -0,0 +1,59 @@ +import Foundation + +class MessageOrderStatisticTable: Table { + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) + } + + /*private func update(peerId: PeerId, tagMask: MessageTags, id: Int32, count: Int) { + let key = ValueBoxKey(length: 8 + 4 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setUInt32(8, value: tagMask.rawValue) + + var writeValue: Int32 = 0 + let buffer = MemoryBuffer(memory: &writeValue, capacity: 4, length: 4, freeWhenDone: false) + var idx = id + while (idx <= 1000000) { + key.setInt32(8 + 4, value: idx) + var value: Int32 = 0 + if let data = self.valueBox.get(self.tableId, key: key) { + data.read(&value, offset: 0, length: 4) + } + if value == 0 { + self.valueBox.remove(self.tableId, key: key) + } else { + writeValue = value + self.valueBox.set(self.tableId, key: key, value: buffer) + } + idx += idx & -idx + } + } + + private func get(peerId: PeerId, tagMask: MessageTags, id: Int32) -> Int32 { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + + var idx = id + + var sum: Int32 = 0 + while (idx > 0) { + key.setInt32(8, value: idx) + var value: Int32 = 0 + if let data = self.valueBox.get(self.tableId, key: key) { + data.read(&value, offset: 0, length: 4) + } + sum += value + idx -= idx & -idx + } + + return sum + } + + func set(_ id: MessageId, count: Int) { + self.update(peerId: id.peerId, id: id.id, count: count) + } + + func get(_ id: MessageId) -> Int32 { + return self.get(peerId: id.peerId, id: id.id) + }*/ +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 3a3a6c3cf5..ab87c588b7 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -193,7 +193,9 @@ public final class Postbox { self.globalMessageIdsNamespace = globalMessageIdsNamespace self.seedConfiguration = seedConfiguration - let _ = try? FileManager.default().removeItem(atPath: self.basePath + "/media") + print("MediaBox path: \(self.basePath + "/media")") + + //let _ = try? FileManager.default.removeItem(atPath: self.basePath + "/media") self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.openDatabase() @@ -202,9 +204,9 @@ public final class Postbox { private func debugSaveState(name: String) { self.queue.justDispatch({ let path = self.basePath + name - let _ = try? FileManager.default().removeItem(atPath: path) + let _ = try? FileManager.default.removeItem(atPath: path) do { - try FileManager.default().copyItem(atPath: self.basePath, toPath: path) + try FileManager.default.copyItem(atPath: self.basePath, toPath: path) } catch (let e) { print("(Postbox debugSaveState: error \(e))") } @@ -214,9 +216,9 @@ public final class Postbox { private func debugRestoreState(name: String) { self.queue.justDispatch({ let path = self.basePath + name - let _ = try? FileManager.default().removeItem(atPath: self.basePath) + let _ = try? FileManager.default.removeItem(atPath: self.basePath) do { - try FileManager.default().copyItem(atPath: path, toPath: self.basePath) + try FileManager.default.copyItem(atPath: path, toPath: self.basePath) } catch (let e) { print("(Postbox debugRestoreState: error \(e))") } @@ -228,7 +230,7 @@ public final class Postbox { let startTime = CFAbsoluteTimeGetCurrent() do { - try FileManager.default().createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) + try FileManager.default.createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) } catch _ { } @@ -441,9 +443,9 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) + entries.append(.IntermediateMessageEntry(message, nil)) case let .Hole(index): - entries.append(.HoleEntry(index)) + entries.append(.HoleEntry(index, nil)) } } return entries @@ -465,9 +467,9 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) + entries.append(.IntermediateMessageEntry(message, nil)) case let .Hole(index): - entries.append(.HoleEntry(index)) + entries.append(.HoleEntry(index, nil)) } } @@ -475,9 +477,9 @@ public final class Postbox { if let intermediateLower = intermediateLower { switch intermediateLower { case let .Message(message): - lower = .IntermediateMessageEntry(message) + lower = .IntermediateMessageEntry(message, nil) case let .Hole(index): - lower = .HoleEntry(index) + lower = .HoleEntry(index, nil) } } @@ -485,13 +487,17 @@ public final class Postbox { if let intermediateUpper = intermediateUpper { switch intermediateUpper { case let .Message(message): - upper = .IntermediateMessageEntry(message) + upper = .IntermediateMessageEntry(message, nil) case let .Hole(index): - upper = .HoleEntry(index) + upper = .HoleEntry(index, nil) } } - return (entries: entries, lower: lower, upper: upper) + if let tagMask = tagMask { + return addLocationsToMessageHistoryViewEntries(tagMask: tagMask, earlier: lower, later: upper, entries: entries) + } else { + return (entries: entries, lower: lower, upper: upper) + } } private func fetchLaterHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { @@ -505,9 +511,9 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) + entries.append(.IntermediateMessageEntry(message, nil)) case let .Hole(index): - entries.append(.HoleEntry(index)) + entries.append(.HoleEntry(index, nil)) } } return entries @@ -735,6 +741,32 @@ public final class Postbox { return self.aroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, tagMask: tagMask) } + private func addLocationsToMessageHistoryViewEntries(tagMask: MessageTags, earlier: MutableMessageHistoryEntry?, later: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry]) -> ([MutableMessageHistoryEntry], MutableMessageHistoryEntry?, MutableMessageHistoryEntry?) { + if let firstEntry = entries.first { + if let location = self.messageHistoryTagsTable.entryLocation(at: firstEntry.index, tagMask: tagMask) { + var mappedEarlier = earlier?.updatedLocation(location.predecessor) + var mappedEntries: [MutableMessageHistoryEntry] = [] + var previousLocation: MessageHistoryEntryLocation? + for i in 0 ..< entries.count { + if i == 0 { + mappedEntries.append(entries[i].updatedLocation(location)) + previousLocation = location + } else { + previousLocation = previousLocation?.successor + mappedEntries.append(entries[i].updatedLocation(previousLocation)) + } + } + previousLocation = previousLocation?.successor + var mappedLater = later?.updatedLocation(previousLocation) + return (mappedEntries, mappedEarlier, mappedLater) + } else { + return (entries, earlier, later) + } + } else { + return (entries, earlier, later) + } + } + private func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return Signal { subscriber in let disposable = MetaDisposable() @@ -742,6 +774,7 @@ public final class Postbox { self.queue.justDispatch({ //print("+ queue \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) + print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) diff --git a/Postbox/RandomAccessMediaResourceContext.swift b/Postbox/RandomAccessMediaResourceContext.swift new file mode 100644 index 0000000000..08006169c5 --- /dev/null +++ b/Postbox/RandomAccessMediaResourceContext.swift @@ -0,0 +1,469 @@ +import Foundation +import SwiftSignalKit + +public struct RandomAccessResourceStoreRange { + private let offset: Int + private let data: Data + + public init(offset: Int, data: Data) { + self.offset = offset + self.data = data + } +} + +public enum RandomAccessResourceDataRangeMode { + case Complete + case Partial + case Incremental + case None +} + +private final class RandomAccessBlockRangeListener: Hashable { + private let id: Int32 + private let range: Range + private let blockSize: Int + private let blocks: Range + private let mode: RandomAccessResourceDataRangeMode + private let updated: (Data) -> Void + + private var missingBlocks: Set + + init(id: Int32, range: Range, blockSize: Int, blocks: Range, missingBlocks: Set, mode: RandomAccessResourceDataRangeMode, updated: (Data) -> Void) { + self.id = id + self.range = range + self.blockSize = blockSize + self.blocks = blocks + self.mode = mode + self.updated = updated + self.missingBlocks = missingBlocks + } + + var hashValue: Int { + return Int(self.id) + } + + func updateMissingBlocks(addedBlocks: Set, fetchData: @noescape(Range) -> Data) { + if self.missingBlocks.isEmpty { + return + } + + switch self.mode { + case .Complete: + self.missingBlocks.subtract(addedBlocks) + if self.missingBlocks.isEmpty { + self.updated(fetchData(self.range)) + } + case .Incremental, .Partial: + var continuousBlockCount = 0 + for index in CountableRange(self.blocks) { + if self.missingBlocks.contains(index) { + break + } + continuousBlockCount += 1 + } + + self.missingBlocks.subtract(addedBlocks) + + var updatedContinuousBlockCount = 0 + for index in CountableRange(self.blocks) { + if self.missingBlocks.contains(index) { + break + } + updatedContinuousBlockCount += 1 + } + + if updatedContinuousBlockCount > continuousBlockCount { + if self.mode == .Partial { + self.updated(fetchData(self.range)) + } else { + let firstBlock = self.blocks.lowerBound + continuousBlockCount + let lastBlock = self.blocks.lowerBound + updatedContinuousBlockCount + + var startOffset = firstBlock * self.blockSize + if firstBlock == 0 { + startOffset = self.range.lowerBound + } + var endOffset = lastBlock * self.blockSize + if lastBlock == self.blocks.upperBound { + endOffset = self.range.upperBound + } + + self.updated(fetchData(startOffset ..< endOffset)) + } + } + case .None: + break + } + } +} + +private func ==(lhs: RandomAccessBlockRangeListener, rhs: RandomAccessBlockRangeListener) -> Bool { + return lhs.id == rhs.id +} + +private struct FetchRange: Hashable { + let range: Range + + var hashValue: Int { + return self.range.lowerBound ^ self.range.upperBound + } +} + +private func ==(lhs: FetchRange, rhs: FetchRange) -> Bool { + return lhs.range == rhs.range +} + +public final class RandomAccessMediaResourceContext { + private let path: String + private let size: Int + + private var file: MappedFile + private var readyBlocks = Set() + let blockSize: Int + private let fragmentBlockCount: Int + let blockCount: Int + + private var nextBlockRangeListenerId: Int32 = 0 + private var blockRangeListenersByBlockIndex: [Int: [RandomAccessBlockRangeListener]] = [:] + private var blockRangeListenerSet: [Int32: RandomAccessBlockRangeListener] = [:] + private var fetchedBlockRangeListenerSet: [Int32: Range] = [:] + + private var fetchRanges = Set() + private var fetchDisposables: [FetchRange: Disposable] = [:] + + private var fetchRange: (Range) -> Disposable + + public init(path: String, size: Int, fetchRange: (Range) -> Disposable) { + self.path = path + self.size = size + self.fetchRange = fetchRange + + let metadataPath = path + ".meta" + self.file = MappedFile(path: metadataPath) + + self.blockSize = 64 * 1024 + self.fragmentBlockCount = 16 + self.blockCount = size / self.blockSize + (size % self.blockSize == 0 ? 0 : 1) + let expectedSize = 4 + self.blockCount * 1 + + if self.file.size != expectedSize { + self.file.size = expectedSize + var version: Int32 = 1 + self.file.write(at: 0 ..< 4, from: &version) + } + + var version: Int32 = 0 + self.file.read(at: 0 ..< 4, to: &version) + precondition(version == 1) + + for i in 0 ..< blockCount { + var blockStatus: Int8 = 0 + self.file.read(at: (4 + i) ..< (4 + i + 1), to: &blockStatus) + if blockStatus != 0 { + self.readyBlocks.insert(i) + } + } + } + + public func storeRanges(_ ranges: [RandomAccessResourceStoreRange]) { + var blockStatus: Int8 = 1 + + var blocksWithListeners = Set() + + for range in ranges.sorted(by: { $0.offset < $1.offset }) { + var offset = range.offset + let endOffset = offset + range.data.count + assert(offset % self.blockSize == 0) + assert(offset >= 0) + + assert(endOffset == self.size || (endOffset < self.size && endOffset % self.blockSize == 0)) + + var fragmentCache: (Int, MappedFile)? + + var blockIndex = offset / self.blockSize + while offset < endOffset { + let fragmentIndex = blockIndex / self.fragmentBlockCount + + let currentFragmentSize = min(self.size - fragmentIndex * self.fragmentBlockCount * self.blockSize, self.fragmentBlockCount * self.blockSize) + let currentBlockSize = min(self.size - blockIndex * self.blockSize, self.blockSize) + + let fragmentFile: MappedFile + if let fragmentCache = fragmentCache, fragmentCache.0 == fragmentIndex { + fragmentFile = fragmentCache.1 + } else { + fragmentCache?.1.synchronize() + fragmentFile = MappedFile(path: self.path + ".\(fragmentIndex)") + fragmentFile.size = currentFragmentSize + fragmentCache = (fragmentIndex, fragmentFile) + } + + let fragmentBlockIndex = blockIndex % self.fragmentBlockCount + let fragmentBlockOffset = fragmentBlockIndex * self.blockSize + + range.data.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + fragmentFile.write(at: fragmentBlockOffset ..< (fragmentBlockOffset + currentBlockSize), from: bytes.advanced(by: offset - range.offset)) + } + + self.readyBlocks.insert(blockIndex) + if let listeners = self.blockRangeListenersByBlockIndex[blockIndex], !listeners.isEmpty { + blocksWithListeners.insert(blockIndex) + } + + self.file.write(at: (4 + blockIndex) ..< (4 + blockIndex + 1), from: &blockStatus) + + offset += blockSize + blockIndex += 1 + } + + fragmentCache?.1.synchronize() + } + self.file.synchronize() + + var updateListeners = Set() + + for index in blocksWithListeners { + if let listeners = self.blockRangeListenersByBlockIndex.removeValue(forKey: index) { + for listener in listeners { + updateListeners.insert(listener) + } + } + } + + for listener in updateListeners { + listener.updateMissingBlocks(addedBlocks: blocksWithListeners, fetchData: { range in + return self.fetchContiguousReadyData(in: range) + }) + } + } + + public func fetchContiguousReadyData(in dataRange: Range) -> Data { + var data = Data() + + var fragmentCache: (Int, MappedFile)? + + let firstBlock = dataRange.lowerBound / self.blockSize + let lastBlock = dataRange.upperBound / self.blockSize + (dataRange.upperBound % self.blockSize == 0 ? 0 : 1) + + let range = firstBlock ..< lastBlock + + var hadNonReadyBlock = false + for blockIndex in range { + if !self.readyBlocks.contains(blockIndex) { + hadNonReadyBlock = true + break + } + + let fragmentIndex = blockIndex / self.fragmentBlockCount + + let currentFragmentSize = min(self.size - fragmentIndex * self.fragmentBlockCount * self.blockSize, self.fragmentBlockCount * self.blockSize) + let currentBlockSize = min(self.size - blockIndex * self.blockSize, self.blockSize) + + let fragmentFile: MappedFile + if let fragmentCache = fragmentCache, fragmentCache.0 == fragmentIndex { + fragmentFile = fragmentCache.1 + } else { + //fragmentCache?.1.synchronize() + fragmentFile = MappedFile(path: self.path + ".\(fragmentIndex)") + assert(fragmentFile.size == currentFragmentSize) + fragmentCache = (fragmentIndex, fragmentFile) + } + + let fragmentBlockIndex = blockIndex % self.fragmentBlockCount + let fragmentBlockOffset = fragmentBlockIndex * self.blockSize + + var currentBlockStart = 0 + if blockIndex == firstBlock { + currentBlockStart = dataRange.lowerBound % self.blockSize + } + + var currentBlockEnd = currentBlockSize + if blockIndex == lastBlock - 1 && (dataRange.upperBound % self.blockSize) != 0 { + currentBlockEnd = dataRange.upperBound % self.blockSize + } + + data.count += currentBlockEnd - currentBlockStart + data.withUnsafeMutableBytes { (bytes: UnsafeMutablePointer) -> Void in + fragmentFile.read(at: (fragmentBlockOffset + currentBlockStart) ..< (fragmentBlockOffset + currentBlockEnd), to: bytes.advanced(by: data.count - (currentBlockEnd - currentBlockStart))) + } + } + + if !hadNonReadyBlock { + assert(data.count == dataRange.count, "\(data.count) != \(dataRange.count)") + } + + return data + } + + public func missingBlocks(in set: Set) -> Set { + return set.subtracting(self.readyBlocks) + } + + private func missingBlocks(in range: Range) -> Set { + var result = Set() + for index in CountableRange(range) { + if !self.readyBlocks.contains(index) { + result.insert(index) + } + } + return result + } + + public func addListenerForData(in range: Range, mode: RandomAccessResourceDataRangeMode, updated: (Data) -> Void) -> Int32 { + let firstBlock = range.lowerBound / self.blockSize + let lastBlock = range.upperBound / self.blockSize + (range.upperBound % self.blockSize == 0 ? 0 : 1) + + let missingBlocks = self.missingBlocks(in: firstBlock ..< lastBlock) + + if missingBlocks.isEmpty { + updated(self.fetchContiguousReadyData(in: range)) + return -1 + } else { + if missingBlocks.count < (firstBlock ..< lastBlock).count { + switch mode { + case .Complete: + break + case .Incremental, .Partial: + updated(self.fetchContiguousReadyData(in: range)) + case .None: + break + } + } + + let id = self.nextBlockRangeListenerId + self.nextBlockRangeListenerId += 1 + let listener = RandomAccessBlockRangeListener(id: id, range: range, blockSize: self.blockSize, blocks: firstBlock ..< lastBlock, missingBlocks: missingBlocks, mode: mode, updated: { data in + updated(data) + }) + + for index in missingBlocks { + if self.blockRangeListenersByBlockIndex[index] == nil { + self.blockRangeListenersByBlockIndex[index] = [listener] + } else { + self.blockRangeListenersByBlockIndex[index]!.append(listener) + } + } + + self.blockRangeListenerSet[listener.id] = listener + + return listener.id + } + } + + public func removeListenerForData(_ id: Int32) { + if id == -1 { + return + } + + if let listener = self.blockRangeListenerSet.removeValue(forKey: id) { + for index in listener.missingBlocks { + if self.blockRangeListenersByBlockIndex[index] != nil { + if let listenerIndex = self.blockRangeListenersByBlockIndex[index]?.index(where: { $0 === listener }) { + self.blockRangeListenersByBlockIndex[index]?.remove(at: listenerIndex) + if let isEmpty = self.blockRangeListenersByBlockIndex[index]?.isEmpty, isEmpty { + self.blockRangeListenersByBlockIndex.removeValue(forKey: index) + } + } + } + } + } + } + + public func addListenerForFetchedData(in range: Range) -> Int32 { + let firstBlock = range.lowerBound / self.blockSize + let lastBlock = range.upperBound / self.blockSize + (range.upperBound % self.blockSize == 0 ? 0 : 1) + + let missingBlocks = self.missingBlocks(in: firstBlock ..< lastBlock) + + if missingBlocks.isEmpty { + return -1 + } else { + let id = self.nextBlockRangeListenerId + self.nextBlockRangeListenerId += 1 + + self.fetchedBlockRangeListenerSet[id] = firstBlock ..< lastBlock + + self.updateFetchDisposables() + + return id + } + } + + public func removeListenerForFetchedData(_ id: Int32) { + let _ = self.fetchedBlockRangeListenerSet.removeValue(forKey: id) + + self.updateFetchDisposables() + } + + public func hasDataListeners() -> Bool { + return !self.blockRangeListenerSet.isEmpty || !self.fetchedBlockRangeListenerSet.isEmpty + } + + private func updateFetchDisposables() { + var fetchRangeList: [Range] = [] + + for listener in self.fetchedBlockRangeListenerSet.values.sorted(by: { $0.lowerBound < $1.lowerBound }) { + if !fetchRangeList.isEmpty { + if fetchRangeList[fetchRangeList.count - 1].upperBound >= listener.lowerBound { + let upperBound = max(fetchRangeList[fetchRangeList.count - 1].upperBound, listener.upperBound) + fetchRangeList[fetchRangeList.count - 1] = fetchRangeList[fetchRangeList.count - 1].lowerBound ..< upperBound + } else { + fetchRangeList.append(listener) + } + } else { + fetchRangeList.append(listener) + } + } + + for listener in self.fetchedBlockRangeListenerSet.values { + for i in 0 ..< fetchRangeList.count { + if fetchRangeList[i].contains(listener.lowerBound) { + if fetchRangeList[i].lowerBound < listener.lowerBound { + fetchRangeList.insert(listener.lowerBound ..< fetchRangeList[i].upperBound, at: i + 1) + fetchRangeList[i] = fetchRangeList[i].lowerBound ..< listener.lowerBound + } + break + } + } + } + + let blockRanges = Set(fetchRangeList.map({ FetchRange(range: $0) })) + + let removedRanges = self.fetchRanges.subtracting(blockRanges) + let addedRanges = blockRanges.subtracting(self.fetchRanges) + + for blockRange in removedRanges { + self.fetchDisposables.removeValue(forKey: blockRange)?.dispose() + } + + for blockRange in addedRanges { + let disposables = DisposableSet() + + let blocksToFetch = self.missingBlocks(in: blockRange.range) + + var contiguousBlockRanges: [Range] = [] + + for blockIndex in blocksToFetch.sorted() { + if !contiguousBlockRanges.isEmpty { + if contiguousBlockRanges[contiguousBlockRanges.count - 1].upperBound == blockIndex { + contiguousBlockRanges[contiguousBlockRanges.count - 1] = contiguousBlockRanges[contiguousBlockRanges.count - 1].lowerBound ..< (blockIndex + 1) + } else { + contiguousBlockRanges.append(blockIndex ..< (blockIndex + 1)) + } + } else { + contiguousBlockRanges.append(blockIndex ..< (blockIndex + 1)) + } + } + + for blockRange in contiguousBlockRanges { + let lowerBoundOffset = blockRange.lowerBound * self.blockSize + let upperBoundOffset = min(self.size, blockRange.upperBound * self.blockSize) + + disposables.add(self.fetchRange(lowerBoundOffset ..< upperBoundOffset)) + } + + self.fetchDisposables[blockRange] = disposables + } + + self.fetchRanges = blockRanges + } +} diff --git a/Postbox/RedBlackTree.swift b/Postbox/RedBlackTree.swift index c711a6d3ff..e69de29bb2 100644 --- a/Postbox/RedBlackTree.swift +++ b/Postbox/RedBlackTree.swift @@ -1,448 +0,0 @@ -public class RBTNode { - private(set) public var color: Bool - private(set) public var left: RBTNode! - private(set) public var right: RBTNode! - private(set) public var parent: RBTNode! - private(set) public var data: Int - init() { - self.data = -1 - self.color = true - self.left = nil - self.right = nil - self.parent = nil - } - init(rootData: Int) { - self.data = rootData - self.color = true //0 is black 1 is red - self.left = nil - self.right = nil - self.parent = nil - } - deinit { - //print("Node: \(self.data) is bein deinitialized") - } - - public func grandparent() -> RBTNode? { - if self.parent === nil || self.parent.parent === nil { - return nil - } else { - return self.parent.parent - } - } - public func sibling() -> RBTNode? { - if self.parent === nil || self.parent.right === nil || self.parent.left === nil { - return nil - } - if self === self.parent!.left! { - return self.parent!.right! - } else { - return self.parent!.left! - } - } -} -public class RBTree { - private(set) public var root: RBTNode? - init(rootData: Int) { - root = RBTNode(rootData: rootData) - root!.color = false - } - init() { - root = nil - } - public func depth() -> Int { - let n = depth(root!) - return n - } - //return the max depth of the tree - private func depth(_ rooty: RBTNode?) -> Int { - if rooty == nil { - return 0 - } else { - return 1+(max(depth(root!.left), depth(root!.right))) - } - } - - public func inOrder() { - inOrder(root) - } - //Prints the in order traversal of the current tree - private func inOrder(_ root: RBTNode?) { - if self.root == nil { - //print("The tree is empty.") - } - if root == nil { - return - } - inOrder(root!.left) - //print("Data: \(root!.data) Color: \(root!.color)") - inOrder(root!.right) - } - /* - Basic Algorithm: - Let Q be P's right child. - Set P's right child to be Q's left child. - [Set Q's left-child's parent to P] - Set Q's left child to be P. - [Set P's parent to Q] - */ - private func leftRotate(_ x: RBTNode) { - let newRoot = x.right! - x.right = newRoot.left - if newRoot.left !== nil { - newRoot.left.parent = x - } - newRoot.parent = x.parent - if x.parent === nil { - root = newRoot - } else if x === x.parent.left { - x.parent.left = newRoot - } else { - x.parent.right = newRoot - } - newRoot.left = x - x.parent = newRoot - } - /* - Basic Algorithm: - Let P be Q's left child. - Set Q's left child to be P's right child. - [Set P's right-child's parent to Q] - Set P's right child to be Q. - [Set Q's parent to P] - */ - private func rightRotate(_ x: RBTNode) { - let newRoot = x.left! - x.left = newRoot.right - if newRoot.right !== nil { - newRoot.right.parent = x - } - newRoot.parent = x.parent - if x.parent === nil { - root = newRoot - } else if x === x.parent.right { - x.parent.right = newRoot - } else { - x.parent.left = newRoot - } - newRoot.right = x - x.parent = newRoot - } - public func insertFixup(_ value: Int) { - let inserted = find(value) - //print("Inserted Node: \(inserted!.data)") - insertCase1(inserted) - } - //Case where root is the only node - private func insertCase1(_ inserted: RBTNode?) { - let myroot = self.root! - if myroot === inserted! { - self.root!.color = false - } - insertCase2(inserted!) - } - //Case for inserting a node as a child of a black node - private func insertCase2(_ inserted: RBTNode?) { - if inserted!.parent!.color == false { - return - } - insertCase3(inserted!) - } - //Insert case for if the parent is black and parent's siblng is black - private func insertCase3(_ inserted: RBTNode?) { - if inserted!.parent!.sibling() != nil && - inserted!.parent!.sibling()!.color == true { - inserted!.parent!.color = false - inserted!.parent!.sibling()!.color = false - let g = inserted!.grandparent - g()!.color = true - if g()!.parent == nil { - g()!.color = false - } - } - insertCase4(inserted) - } - //Insert case for Node N is left of parent and parent is right of grandparent - private func insertCase4(_ insert: RBTNode?) { - var inserted = insert - if (inserted! === inserted!.parent!.right) && - (inserted!.grandparent()!.left === inserted!.parent!) { - - leftRotate(inserted!.parent) - inserted! = inserted!.left - } else if (inserted! === inserted!.parent!.left) && - (inserted!.grandparent()!.right === inserted!.parent!) { - - rightRotate(inserted!.parent) - inserted! = inserted!.right - } - insertCase5(inserted) - } - //Insert case for Node n where parent is red and parent's sibling is black - private func insertCase5(_ inserted: RBTNode?) { - if inserted!.parent!.color == true && - (inserted!.parent!.sibling()?.color == false || - inserted!.parent!.sibling() == nil) { - - if inserted! === inserted!.parent!.left && inserted!.grandparent()!.left === inserted!.parent! { - inserted!.parent.color = false - inserted!.grandparent()?.color = true - if inserted! === inserted!.parent!.left { - rightRotate(inserted!.grandparent()!) - } - } else if inserted! === inserted!.parent!.right && inserted!.grandparent()!.right === inserted!.parent! { - inserted!.parent.color = false - inserted!.grandparent()?.color = true - leftRotate(inserted!.grandparent()!) - } - } - } - - public func insert(_ value: Int) { - insert(value, parent: root!) - insertFixup(value) - } - //Basic BST insert implementation - private func insert(_ value: Int, parent: RBTNode) { - if self.root == nil { - self.root = RBTNode(rootData: value) - return - } else if value < parent.data { - if let left = parent.left { - insert(value, parent: left) - } else { - parent.left = RBTNode(rootData: value) - parent.left?.parent = parent - } - } else { - if let right = parent.right { - insert(value, parent: right) - } else { - parent.right = RBTNode(rootData: value) - parent.right?.parent = parent - } - } - } - public func find(_ data: Int) -> RBTNode? { - return find(root!, data: data) - } - //Returns the reference to the RBTNode whos data was requested - private func find(_ root: RBTNode, data: Int) -> RBTNode? { - if data == root.data { - return root - } - if root.data != data && root.right == nil && root.left == nil { - return nil - } else if data > root.data { - return find(root.right, data: data) - } else if data < root.data { - return find(root.left, data: data) - } else { - return nil - } - } - - //DELETION HELPER FUNCTIONS - public func remove(_ value: Int) { - let toRemove = find(value) - if toRemove == nil { - return - } - } - //Transplant the positions of two nodes in the RBTree - public func replaceNode(_ n1: RBTNode, n2: RBTNode) { - let temp = n1.data - let temp_color = n1.color - n1.data = n2.data - n1.color = n2.color - n2.data = temp - n2.color = temp_color - } - //returns the node with the minimum value in the subtree - public func minimum(_ node: RBTNode) -> RBTNode { - var minimumNode = node - while minimumNode.left !== nil { - minimumNode = minimumNode.left - } - return minimumNode - } - //Returns the next largest node in the tree - public func successor(_ node: RBTNode) -> RBTNode? { - var nextLargestNode = node - if nextLargestNode.right !== nil { - return minimum(nextLargestNode.right) - } - var successor = nextLargestNode.parent - while successor !== nil && nextLargestNode === successor!.right { - nextLargestNode = successor! - successor = successor!.parent - } - return successor - } - //Returns the next smallest node in the tree - public func predecessor(_ node: RBTNode) -> RBTNode { - var nextSmallestNode = node - if nextSmallestNode.left !== nil { - return minimum(nextSmallestNode.left) - } - var successor = nextSmallestNode.parent - while successor !== nil && nextSmallestNode === successor!.left { - nextSmallestNode = successor! - successor = successor!.parent - } - return successor! - } - //Returns the node with the largest value in the subtree - public func maximum(_ rootnode: RBTNode) -> RBTNode { - var rootNode = rootnode - while rootNode.right !== nil { - rootNode = rootNode.right - } - return rootNode - } - //call to remove a node from the tree - public func delete(_ x: Int) { - let toDel = find(x) - deleteNode(toDel!) - } - //Function call for removing a node - private func deleteNode(_ todel: RBTNode?) { - var toDel = todel - //case for if todel is the only node in the tree - if toDel!.left === nil && toDel!.right === nil && toDel!.parent === nil { - toDel = nil - self.root = nil - return - } - //case for if toDell is a red node w/ no children - if toDel!.left === nil && toDel!.right === nil && toDel!.color == true { - if toDel!.parent.left === toDel! { - toDel!.parent.left = nil - toDel = nil - } else if toDel!.parent === nil { - toDel = nil - } else { - toDel?.parent.right = nil - toDel = nil - } - return - } - //case for toDel having two children - if toDel!.left !== nil && toDel!.right !== nil { - let pred = maximum(toDel!.left!) - toDel!.data = pred.data - toDel! = pred - } - //case for toDel having one child - var child: RBTNode? = nil - if toDel!.right === nil { - child = toDel!.left - } else { - child = toDel!.right - } - if toDel!.color == false && child !== nil { - toDel!.color = child!.color - deleteCase1(toDel!) - } - - if child !== nil { - replaceNode(toDel!, n2: child!) - if toDel!.parent === nil && child !== nil { - child!.color = false - } - } - if toDel!.parent.left === toDel! { - toDel!.parent.left = nil - toDel = nil - return - } else if toDel!.parent === nil { - toDel = nil - return - } else { - toDel?.parent.right = nil - toDel = nil - return - } - } - //delete case for if parent is nil after deletion - private func deleteCase1(_ toDel: RBTNode?) { - if toDel?.parent === nil { - return - } else { - deleteCase2(toDel) - } - } - //case to fix tree after deletion and sibling is red - private func deleteCase2(_ toDel: RBTNode?) { - let sibling = toDel!.sibling() - if sibling?.color == true { - toDel!.parent.color = true - sibling?.color = false - if toDel! === toDel!.parent.left { - leftRotate(toDel!.parent) - } else { - rightRotate(toDel!.parent) - } - deleteCase3(toDel!) - } - } - //delete case for fixing tree when parnet is black and sibling is black and sibling.children are also black - private func deleteCase3(_ toDel: RBTNode?) { - if toDel!.parent?.color == false && - toDel!.sibling()?.color == false && - toDel!.sibling()?.left!.color == false && - toDel!.sibling()?.right!.color == false { - - toDel!.sibling()?.color = true - toDel!.parent?.color = false - } else { - deleteCase4(toDel) - } - } - private func deleteCase4(_ toDel: RBTNode?) { - if toDel!.parent?.color == true && - toDel!.sibling()?.color == false && - toDel!.sibling()?.left!.color == false && - toDel!.sibling()?.right!.color == false { - - toDel!.sibling()?.color = true - toDel!.parent.color = false - } else { - deleteCase5(toDel) - } - } - //delete case for fixing tree if toDel is a left child and sibling(n) is black and children of sibling(n) are black and white respectibely - private func deleteCase5(_ toDel: RBTNode?) { - if toDel! === toDel!.parent?.left && - toDel!.sibling()?.color == false && - toDel!.sibling()?.left.color == true && - toDel!.sibling()?.right.color == false { - - toDel!.sibling()?.color = true - toDel!.sibling()?.left?.color = false - rightRotate(toDel!.sibling()!) - } - //opposite case - else if toDel! === toDel!.parent?.right && - toDel!.sibling()?.color == false && - toDel!.sibling()?.left.color == false && - toDel!.sibling()?.right.color == true { - - toDel!.sibling()?.color = true - toDel!.sibling()?.right?.color = false - leftRotate(toDel!.sibling()!) - } - } - //final rotations to be done after deleting a black node from the tree - private func deleteCase6(_ toDel: RBTNode?) { - let color = toDel!.sibling()?.color - toDel!.parent?.color = color! - toDel!.parent?.color = false - if toDel! === toDel!.parent.left { - toDel!.sibling()?.right?.color = false - leftRotate(toDel!.parent!) - } else { - toDel!.sibling()?.left?.color = false - rightRotate(toDel!.parent!) - } - } -} diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 8d5d64fbc7..cf61a25bd1 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -62,7 +62,7 @@ private struct SqlitePreparedStatement { } public final class SqliteValueBox: ValueBox { - private let lock = RecursiveLock() + private let lock = NSRecursiveLock() private let basePath: String private var database: Database! @@ -102,7 +102,7 @@ public final class SqliteValueBox: ValueBox { lock.lock() do { - try FileManager.default().createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) + try FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) } catch _ { } let path = basePath + "/db_sqlite" let database = Database(path) @@ -697,7 +697,7 @@ public final class SqliteValueBox: ValueBox { self.database = nil self.lock.unlock() - let _ = try? FileManager.default().removeItem(atPath: self.basePath) + let _ = try? FileManager.default.removeItem(atPath: self.basePath) self.database = self.openDatabase() tables.removeAll() diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index 6caf32aefb..3178b94934 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -115,7 +115,7 @@ class ChatListTableTests: XCTestCase { self.chatListTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil } diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index 43c44dde75..1875a1e6f9 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -88,7 +88,7 @@ class MessageHistoryIndexTableTests: XCTestCase { self.indexTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil } diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index 713bd60ff2..0dc11db243 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -248,7 +248,7 @@ class MessageHistoryTableTests: XCTestCase { self.historyMetadataTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil } diff --git a/PostboxTests/OrderStatisticTreeTests.swift b/PostboxTests/OrderStatisticTreeTests.swift index 10ce2c8eff..c4779434db 100644 --- a/PostboxTests/OrderStatisticTreeTests.swift +++ b/PostboxTests/OrderStatisticTreeTests.swift @@ -110,16 +110,16 @@ class OrderStatisticTreeTests: XCTestCase { self.historyMetadataTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil } func testOST() { - let tree = RBTree(rootData: 0) + /*let tree = RBTree(rootData: 0) for _ in 0 ..< 1000 { let key = Int(arc4random_uniform(UInt32(Int32.max - 1))) tree.insert(key) } - print("OST height: \(tree.depth())") + print("OST height: \(tree.depth())")*/ } } diff --git a/PostboxTests/RandomAccessResourceTests.swift b/PostboxTests/RandomAccessResourceTests.swift new file mode 100644 index 0000000000..b0f72a3065 --- /dev/null +++ b/PostboxTests/RandomAccessResourceTests.swift @@ -0,0 +1,413 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +import SwiftSignalKit + +func randomFilePath() -> String { + return NSTemporaryDirectory() + "\(arc4random())\(arc4random())" +} + +class RandomAccessResourceTests: XCTestCase { + override func setUp() { + super.setUp() + + self.continueAfterFailure = false + } + + override func tearDown() { + super.tearDown() + } + + func testCompleteAligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + //let size = 64 * 1024 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + let subdata = sampleData.subdata(with: NSRange(location: range.lowerBound, length: range.count)) + storeRange(RandomAccessResourceStoreRange(offset: range.lowerBound, data: subdata)) + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var blocks: [Int] = [] + for i in 0 ..< context.blockCount { + blocks.append(i) + } + + var selectedRanges: [Range] = [] + + while !blocks.isEmpty { + let arrayOffset = Int(arc4random_uniform(UInt32(blocks.count))) + var rangeEnd = blocks[arrayOffset] + var arrayOffsetEnd = arrayOffset + for i in arrayOffset + 1 ..< blocks.count { + if blocks[i] == rangeEnd + 1 { + rangeEnd = blocks[i] + arrayOffsetEnd = i + } else { + break + } + } + + let arrayOffsetCount = arrayOffsetEnd + 1 - arrayOffset + let selectedArrayOffsetCount = Int(arc4random_uniform(UInt32(arrayOffsetCount + 1))) + let selectedArrayOffsetEnd = arrayOffset + max(0, selectedArrayOffsetCount - 1) + + let range = (blocks[arrayOffset] * context.blockSize) ..< ((blocks[selectedArrayOffsetEnd] + 1) * context.blockSize) + blocks.removeSubrange(arrayOffset ..< (selectedArrayOffsetEnd + 1)) + + selectedRanges.append(Range(range)) + } + + selectedRanges.removeAll() + + selectedRanges.append(10092544..<10354688); selectedRanges.append(1310720..<7274496); selectedRanges.append(7340032..<7798784); selectedRanges.append(1179648..<1310720); selectedRanges.append(524288..<851968); selectedRanges.append(8060928..<9895936); selectedRanges.append(1114112..<1179648); selectedRanges.append(7929856..<8060928); selectedRanges.append(196608..<524288); selectedRanges.append(131072..<196608); selectedRanges.append(7798784..<7864320); selectedRanges.append(917504..<1114112); selectedRanges.append(65536..<131072); selectedRanges.append(7274496..<7340032); selectedRanges.append(0..<65536); selectedRanges.append(7864320..<7929856); selectedRanges.append(851968..<917504); selectedRanges.append(9961472..<10027008); selectedRanges.append(10354688..<10420224); selectedRanges.append(10027008..<10092544); selectedRanges.append(10420224..<10485760); selectedRanges.append(9895936..<9961472) + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var invocationCount = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Complete, updated: { subdata in + XCTAssert(subdata.count == range.count, "\(subdata.count) != \(range.count)") + invocationCount += 1 + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound), bytes, range.count) + } + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(invocationCount == 1, "invocationCount != 1") + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testCompleteUnaligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + 123 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + let subdata = sampleData.subdata(with: NSRange(location: range.lowerBound, length: range.count)) + storeRange(RandomAccessResourceStoreRange(offset: range.lowerBound, data: subdata)) + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var blocks: [Int] = [] + for i in 0 ..< context.blockCount { + blocks.append(i) + } + + var selectedRanges: [Range] = [] + + var dataOffset = 0 + while dataOffset < size { + let partSize = min(size - dataOffset, Int(arc4random_uniform(1024 * 1024))) + selectedRanges.append(dataOffset ..< (dataOffset + partSize)) + dataOffset += partSize + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + let _ = context.addListenerForData(in: Range(range), mode: .Complete, updated: { subdata in + if range.count != subdata.count { + print("\(subdata.count)") + } + XCTAssert(subdata.count == range.count) + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound), bytes, subdata.count) + } + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testIncrementalStoreCompleteSubscriptionAligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + //let size = 64 * 1024 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + var offset = range.lowerBound + while offset < range.upperBound { + let subdata = sampleData.subdata(with: NSRange(location: offset, length: min(range.upperBound - offset, 64 * 1024))) + storeRange(RandomAccessResourceStoreRange(offset: range.lowerBound, data: subdata)) + + offset += 64 * 1024 + } + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var blocks: [Int] = [] + for i in 0 ..< context.blockCount { + blocks.append(i) + } + + var selectedRanges: [Range] = [] + + while !blocks.isEmpty { + let arrayOffset = Int(arc4random_uniform(UInt32(blocks.count))) + var rangeEnd = blocks[arrayOffset] + var arrayOffsetEnd = arrayOffset + for i in arrayOffset + 1 ..< blocks.count { + if blocks[i] == rangeEnd + 1 { + rangeEnd = blocks[i] + arrayOffsetEnd = i + } else { + break + } + } + + let arrayOffsetCount = arrayOffsetEnd + 1 - arrayOffset + let selectedArrayOffsetCount = Int(arc4random_uniform(UInt32(arrayOffsetCount + 1))) + let selectedArrayOffsetEnd = arrayOffset + max(0, selectedArrayOffsetCount - 1) + + let range = (blocks[arrayOffset] * context.blockSize) ..< ((blocks[selectedArrayOffsetEnd] + 1) * context.blockSize) + blocks.removeSubrange(arrayOffset ..< (selectedArrayOffsetEnd + 1)) + + selectedRanges.append(Range(range)) + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var invocations = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Complete, updated: { subdata in + XCTAssert(subdata.count == range.count) + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound), bytes, range.count) + } + invocations += 1 + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(invocations == 1) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testIncrementalStoreCompleteSubscriptionUnaligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + 123 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + var offset = range.lowerBound + while offset < range.upperBound { + let subdata = sampleData.subdata(with: NSRange(location: offset, length: min(range.upperBound - offset, 64 * 1024))) + storeRange(RandomAccessResourceStoreRange(offset: offset, data: subdata)) + + offset += 64 * 1024 + } + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var selectedRanges: [Range] = [] + + var dataOffset = 0 + while dataOffset < size { + let partSize = min(size - dataOffset, Int(arc4random_uniform(1024 * 1024))) + selectedRanges.append(dataOffset ..< (dataOffset + partSize)) + dataOffset += partSize + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var invocations = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Complete, updated: { subdata in + XCTAssert(subdata.count == range.count) + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound), bytes, subdata.count) + } + invocations += 1 + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(invocations == 1) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testIncrementalStoreIncrementalSubscriptionAligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + var offset = range.lowerBound + while offset < range.upperBound { + let subdata = sampleData.subdata(with: NSRange(location: offset, length: min(range.upperBound - offset, 64 * 1024))) + storeRange(RandomAccessResourceStoreRange(offset: offset, data: subdata)) + + offset += 64 * 1024 + } + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var blocks: [Int] = [] + for i in 0 ..< context.blockCount { + blocks.append(i) + } + + var selectedRanges: [Range] = [] + + while !blocks.isEmpty { + let arrayOffset = Int(arc4random_uniform(UInt32(blocks.count))) + var rangeEnd = blocks[arrayOffset] + var arrayOffsetEnd = arrayOffset + for i in arrayOffset + 1 ..< blocks.count { + if blocks[i] == rangeEnd + 1 { + rangeEnd = blocks[i] + arrayOffsetEnd = i + } else { + break + } + } + + let arrayOffsetCount = arrayOffsetEnd + 1 - arrayOffset + let selectedArrayOffsetCount = Int(arc4random_uniform(UInt32(arrayOffsetCount + 1))) + let selectedArrayOffsetEnd = arrayOffset + max(0, selectedArrayOffsetCount - 1) + + let range = (blocks[arrayOffset] * context.blockSize) ..< ((blocks[selectedArrayOffsetEnd] + 1) * context.blockSize) + blocks.removeSubrange(arrayOffset ..< (selectedArrayOffsetEnd + 1)) + + selectedRanges.append(Range(range)) + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var offset = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Incremental, updated: { subdata in + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound + offset), bytes, subdata.count) + } + offset += subdata.count + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(offset == range.count) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testIncrementalStoreIncrementalSubscriptionUnaligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + 123 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + var offset = range.lowerBound + while offset < range.upperBound { + let subdata = sampleData.subdata(with: NSRange(location: offset, length: min(range.upperBound - offset, 64 * 1024))) + storeRange(RandomAccessResourceStoreRange(offset: offset, data: subdata)) + + offset += 64 * 1024 + } + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var selectedRanges: [Range] = [] + + selectedRanges = [0..<615697, 615697..<1040801] + + var dataOffset = 1040801 + while dataOffset < size { + let partSize = min(size - dataOffset, Int(arc4random_uniform(1024 * 1024))) + selectedRanges.append(dataOffset ..< (dataOffset + partSize)) + dataOffset += partSize + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var offset = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Incremental, updated: { subdata in + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound + offset), bytes, subdata.count) + } + offset += subdata.count + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(offset == range.count) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } +} diff --git a/PostboxTests/ReadStateTableTests.swift b/PostboxTests/ReadStateTableTests.swift index eea587f1aa..eb51113df4 100644 --- a/PostboxTests/ReadStateTableTests.swift +++ b/PostboxTests/ReadStateTableTests.swift @@ -110,7 +110,7 @@ class ReadStateTableTests: XCTestCase { self.historyMetadataTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil } From 7a41d7071f017523279158b949c9fbfeedef690f Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 16 Aug 2016 14:05:47 +0300 Subject: [PATCH 038/427] no message --- Postbox.xcodeproj/project.pbxproj | 16 ++ .../xcschemes/xcschememanagement.plist | 2 +- Postbox/ContactPeerIdsView.swift | 26 +++ Postbox/ContactPeersView.swift | 46 +++++ Postbox/ContactTable.swift | 62 +++++++ Postbox/Message.swift | 4 +- Postbox/MessageHistoryIndexTable.swift | 2 +- Postbox/MessageHistoryTable.swift | 2 + Postbox/Peer.swift | 1 + Postbox/PeerNameIndexRepresentation.swift | 37 +++++ Postbox/Postbox.swift | 157 +++++++++++++++++- Postbox/RatingTable.swift | 76 +++++++++ Postbox/SimpleDictionary.swift | 5 +- Postbox/ViewTracker.swift | 44 ++++- 14 files changed, 468 insertions(+), 12 deletions(-) create mode 100644 Postbox/ContactPeerIdsView.swift create mode 100644 Postbox/ContactPeersView.swift create mode 100644 Postbox/PeerNameIndexRepresentation.swift create mode 100644 Postbox/RatingTable.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 826a081033..c8b9a7c4e9 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -9,6 +9,9 @@ /* Begin PBXBuildFile section */ D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; + D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */; }; + D0079F651D5A457A00A27A2C /* ContactPeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F641D5A457A00A27A2C /* ContactPeersView.swift */; }; + D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB41B85D192002E4EB5 /* lmdb.h */; }; D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-unreachable-code -Wno-conditional-uninitialized -Wno-format-extra-args -Wno-macro-redefined"; }; }; D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB61B85D192002E4EB5 /* midl.c */; settings = {COMPILER_FLAGS = "-Wno-conversion"; }; }; @@ -41,6 +44,7 @@ D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; D08CEFB41D2AD8BE0015D3BC /* RedBlackTree.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */; }; + D08D451F1D5D2CA700A7428A /* RatingTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08D451E1D5D2CA700A7428A /* RatingTable.swift */; }; D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; @@ -92,6 +96,9 @@ /* Begin PBXFileReference section */ D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryView.swift; sourceTree = ""; }; D0079F591D592E8B00A27A2C /* ContactTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactTable.swift; sourceTree = ""; }; + D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactPeerIdsView.swift; sourceTree = ""; }; + D0079F641D5A457A00A27A2C /* ContactPeersView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactPeersView.swift; sourceTree = ""; }; + D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameIndexRepresentation.swift; sourceTree = ""; }; D00E0FB41B85D192002E4EB5 /* lmdb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lmdb.h; path = submodules/lmdb/libraries/liblmdb/lmdb.h; sourceTree = SOURCE_ROOT; }; D00E0FB51B85D192002E4EB5 /* mdb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = mdb.c; path = submodules/lmdb/libraries/liblmdb/mdb.c; sourceTree = SOURCE_ROOT; }; D00E0FB61B85D192002E4EB5 /* midl.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = midl.c; path = submodules/lmdb/libraries/liblmdb/midl.c; sourceTree = SOURCE_ROOT; }; @@ -125,6 +132,7 @@ D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTable.swift; sourceTree = ""; }; D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTable.swift; sourceTree = ""; }; D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RedBlackTree.swift; sourceTree = ""; }; + D08D451E1D5D2CA700A7428A /* RatingTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RatingTable.swift; sourceTree = ""; }; D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = ""; }; D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; @@ -269,6 +277,7 @@ D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */, D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */, D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */, + D08D451E1D5D2CA700A7428A /* RatingTable.swift */, ); name = Tables; sourceTree = ""; @@ -282,6 +291,7 @@ D08C71391C501F0700779C0F /* MessageHistoryHole.swift */, D044CA2D1C618373002160FF /* ChatListHole.swift */, D0C674CB1CBB14A700183765 /* PeerReadState.swift */, + D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */, ); name = Objects; sourceTree = ""; @@ -316,6 +326,8 @@ D0F9E86A1C59719800037222 /* ChatListView.swift */, D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */, D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */, + D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */, + D0079F641D5A457A00A27A2C /* ContactPeersView.swift */, ); name = Views; sourceTree = ""; @@ -512,19 +524,23 @@ buildActionMask = 2147483647; files = ( D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, + D08D451F1D5D2CA700A7428A /* RatingTable.swift in Sources */, D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */, + D0079F651D5A457A00A27A2C /* ContactPeersView.swift in Sources */, D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */, D08CEFB41D2AD8BE0015D3BC /* RedBlackTree.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */, + D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */, D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */, D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, + D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */, D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist index 9eb93eab1b..cc1adcd6c7 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist @@ -12,7 +12,7 @@ PostboxTests.xcscheme orderHint - 4 + 5 SuppressBuildableAutocreation diff --git a/Postbox/ContactPeerIdsView.swift b/Postbox/ContactPeerIdsView.swift new file mode 100644 index 0000000000..501e3fc56a --- /dev/null +++ b/Postbox/ContactPeerIdsView.swift @@ -0,0 +1,26 @@ +import Foundation + +final class MutableContactPeerIdsView { + private var peerIds: Set + + init(peerIds: Set) { + self.peerIds = peerIds + } + + func replay(replace replacePeerIds: Set) -> Bool { + if self.peerIds != replacePeerIds { + self.peerIds = replacePeerIds + return true + } else { + return false + } + } +} + +public final class ContactPeerIdsView { + public let peerIds: Set + + init(_ mutableView: MutableContactPeerIdsView) { + self.peerIds = mutableView.peerIds + } +} diff --git a/Postbox/ContactPeersView.swift b/Postbox/ContactPeersView.swift new file mode 100644 index 0000000000..b8a9257b73 --- /dev/null +++ b/Postbox/ContactPeersView.swift @@ -0,0 +1,46 @@ +import Foundation + +final class MutableContactPeersView { + private let index: PeerNameIndex + private var peers: [PeerId: Peer] + private var peerIds: Set + private var accountPeer: Peer? + + init(peers: [PeerId: Peer], index: PeerNameIndex, accountPeer: Peer?) { + self.index = index + self.peers = peers + self.peerIds = Set(peers.map { $0.0 }) + self.accountPeer = accountPeer + } + + func replay(replace replacePeerIds: Set, getPeer: (PeerId) -> Peer?) -> Bool { + let removedPeerIds = self.peerIds.subtracting(replacePeerIds) + let addedPeerIds = replacePeerIds.subtracting(self.peerIds) + + self.peerIds = replacePeerIds + + for peerId in removedPeerIds { + let _ = self.peers.removeValue(forKey: peerId) + } + + for peerId in addedPeerIds { + if let peer = getPeer(peerId) { + self.peers[peerId] = peer + } + } + + return !removedPeerIds.isEmpty || !addedPeerIds.isEmpty + } +} + +public final class ContactPeersView { + public let peers: [Peer] + public let accountPeer: Peer? + + init(_ mutableView: MutableContactPeersView) { + let index = mutableView.index + self.peers = mutableView.peers.map({ $0.1 }).sorted(by: { $0.indexName.indexName(index) < $1.indexName.indexName(index) }) + self.accountPeer = mutableView.accountPeer + } +} + diff --git a/Postbox/ContactTable.swift b/Postbox/ContactTable.swift index 2045f4d47d..0529d6ca71 100644 --- a/Postbox/ContactTable.swift +++ b/Postbox/ContactTable.swift @@ -1,9 +1,71 @@ import Foundation final class ContactTable: Table { + private var originalPeerIds: Set? + private var peerIds: Set? + private var addedPeerIds = Set() + private var removedPeerIds = Set() + + private func key(_ id: PeerId, sharedKey: ValueBoxKey = ValueBoxKey(length: 8)) -> ValueBoxKey { + sharedKey.setInt64(0, value: id.toInt64()) + return sharedKey + } + + private func lowerBound() -> ValueBoxKey { + return self.key(PeerId(namespace: 0, id: 0)) + } + + private func upperBound() -> ValueBoxKey { + return self.key(PeerId(namespace: Int32.max, id: Int32.max)) + } + + func get() -> Set { + if let peerIds = self.peerIds { + return peerIds + } else { + var peerIds = Set() + self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), keys: { key in + peerIds.insert(PeerId(key.getInt64(0))) + return true + }, limit: 0) + self.peerIds = peerIds + self.originalPeerIds = peerIds + return peerIds + } + } + + func replace(_ ids: Set) { + let currentPeerIds = self.get() + + let previousPeerIds: Set + if let originalPeerIds = self.originalPeerIds { + previousPeerIds = originalPeerIds + } else { + previousPeerIds = currentPeerIds + self.originalPeerIds = currentPeerIds + } + + self.removedPeerIds = previousPeerIds.subtracting(ids) + self.addedPeerIds = ids.subtracting(previousPeerIds) + + self.peerIds = ids + } override func beforeCommit() { + let sharedKey = self.key(PeerId(namespace: 0, id: 0)) + for peerId in self.removedPeerIds { + self.valueBox.remove(self.tableId, key: self.key(peerId, sharedKey: sharedKey)) + } + + for peerId in self.addedPeerIds { + self.valueBox.set(self.tableId, key: self.key(peerId, sharedKey: sharedKey), value: MemoryBuffer()) + } + + self.originalPeerIds = self.peerIds + + self.addedPeerIds.removeAll() + self.removedPeerIds.removeAll() } } diff --git a/Postbox/Message.swift b/Postbox/Message.swift index b15a871f66..5312043686 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -182,7 +182,7 @@ public struct MessageFlags: OptionSet { self.rawValue = 0 } - init(_ flags: StoreMessageFlags) { + public init(_ flags: StoreMessageFlags) { var rawValue: UInt32 = 0 if flags.contains(StoreMessageFlags.Unsent) { @@ -276,7 +276,7 @@ public final class Message { public let peers: SimpleDictionary public let associatedMessages: SimpleDictionary - init(stableId: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: MessageForwardInfo?, author: Peer?, text: String, attributes: [MessageAttribute], media: [Media], peers: SimpleDictionary, associatedMessages: SimpleDictionary) { + public init(stableId: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: MessageForwardInfo?, author: Peer?, text: String, attributes: [MessageAttribute], media: [Media], peers: SimpleDictionary, associatedMessages: SimpleDictionary) { self.stableId = stableId self.id = id self.timestamp = timestamp diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 44e56fbe3f..fbf7eaeefc 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -135,7 +135,7 @@ final class MessageHistoryIndexTable: Table { return key.successor } - private func ensureInitialized(_ peerId: PeerId, operations: inout [MessageHistoryIndexOperation]) { + func ensureInitialized(_ peerId: PeerId, operations: inout [MessageHistoryIndexOperation]) { if !self.metadataTable.isInitialized(peerId) { for namespace in self.seedConfiguration.initializeMessageNamespacesWithHoles { self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: Int32.max), timestamp: Int32.max), min: 1, tags: MessageTags.All.rawValue), operations: &operations) diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index f40e4f93de..0ecb1f7cff 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -1289,6 +1289,8 @@ final class MessageHistoryTable: Table { } func earlierEntries(_ peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + + var entries: [IntermediateMessageHistoryEntry] = [] let key: ValueBoxKey if let index = index { diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index c86604375e..d4dc6bcc9d 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -94,6 +94,7 @@ public func <(lhs: PeerId, rhs: PeerId) -> Bool { public protocol Peer: Coding { var id: PeerId { get } + var indexName: PeerIndexNameRepresentation { get } func isEqual(_ other: Peer) -> Bool } diff --git a/Postbox/PeerNameIndexRepresentation.swift b/Postbox/PeerNameIndexRepresentation.swift new file mode 100644 index 0000000000..ba6c28db35 --- /dev/null +++ b/Postbox/PeerNameIndexRepresentation.swift @@ -0,0 +1,37 @@ +import Foundation + +public enum PeerIndexNameRepresentation { + case title(String) + case personName(first: String, last: String) +} + +public enum PeerNameIndex { + case firstNameFirst + case lastNameFirst +} + +extension PeerIndexNameRepresentation { + func indexName(_ index: PeerNameIndex) -> String { + switch self { + case let .title(title): + return title + case let .personName(first, last): + switch index { + case .firstNameFirst: + return first + last + case .lastNameFirst: + return last + first + } + } + } + + func match(query: String) -> Bool { + switch self { + case let .title(title): + return title.lowercased().hasPrefix(query) + case let .personName(first, last): + return first.lowercased().hasPrefix(query) || last.lowercased().hasPrefix(query) + } + return false + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index ab87c588b7..6983346553 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -93,6 +93,14 @@ public final class Modifier { self.postbox?.updatePeers(peers, update: update) } + public func replaceContactPeerIds(_ peerIds: Set) { + self.postbox?.replaceContactPeerIds(peerIds) + } + + public func replaceRecentPeerIds(_ peerIds: [PeerId]) { + self.postbox?.replaceRecentPeerIds(peerIds) + } + public func updateMessage(_ index: MessageIndex, update: (Message) -> StoreMessage) { self.postbox?.updateMessage(index, update: update) } @@ -144,6 +152,7 @@ public final class Postbox { private var currentFilledHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] = [:] private var currentUpdatedPeers: [PeerId: Peer] = [:] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] + private var currentReplacedContactPeerIds: Set? private var statePipe: ValuePipe = ValuePipe() @@ -187,6 +196,11 @@ public final class Postbox { var peerChatStateTable: PeerChatStateTable! var readStateTable: MessageHistoryReadStateTable! var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable! + var contactsTable: ContactTable! + + //temporary + var peerRatingTable: RatingTable! + public init(basePath: String, globalMessageIdsNamespace: MessageId.Namespace, seedConfiguration: SeedConfiguration) { self.basePath = basePath @@ -278,22 +292,30 @@ public final class Postbox { self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, tableId: 8) self.chatListTable = ChatListTable(valueBox: self.valueBox, tableId: 9, indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, tableId: 13) + self.contactsTable = ContactTable(valueBox: self.valueBox, tableId: 16) + self.peerRatingTable = RatingTable(valueBox: self.valueBox, tableId: 17) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) self.tables.append(self.globalMessageIdsTable) self.tables.append(self.messageHistoryMetadataTable) + self.tables.append(self.messageHistoryUnsentTable) + self.tables.append(self.messageHistoryTagsTable) self.tables.append(self.messageHistoryIndexTable) self.tables.append(self.mediaCleanupTable) self.tables.append(self.mediaTable) + self.tables.append(self.readStateTable) + self.tables.append(self.synchronizeReadStateTable) self.tables.append(self.messageHistoryTable) self.tables.append(self.chatListIndexTable) self.tables.append(self.chatListTable) self.tables.append(self.peerChatStateTable) - self.tables.append(self.readStateTable) - self.tables.append(self.synchronizeReadStateTable) + self.tables.append(self.contactsTable) + self.tables.append(self.peerRatingTable) - self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, fetchAnchorIndex: self.fetchAnchorIndex, renderMessage: self.renderIntermediateMessage, fetchChatListHole: self.fetchChatListHoleWrapper, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper, sendUnsentMessage: self.sendUnsentMessageWrapper, unsentMessageIndices: self.messageHistoryUnsentTable!.get(), synchronizeReadState: self.synchronizePeerReadStateWrapper, synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) + self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, fetchAnchorIndex: self.fetchAnchorIndex, renderMessage: self.renderIntermediateMessage, getPeer: { peerId in + return self.peerTable.get(peerId) + }, fetchChatListHole: self.fetchChatListHoleWrapper, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper, sendUnsentMessage: self.sendUnsentMessageWrapper, unsentMessageIndices: self.messageHistoryUnsentTable!.get(), synchronizeReadState: self.synchronizePeerReadStateWrapper, synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") }) @@ -637,7 +659,7 @@ public final class Postbox { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } - self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia) + self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds) self.currentOperationsByPeerId.removeAll() self.currentFilledHolesByPeerId.removeAll() @@ -647,6 +669,7 @@ public final class Postbox { self.currentUnsentOperations.removeAll() self.currentUpdatedSynchronizeReadStateOperations.removeAll() self.currentUpdatedMedia.removeAll() + self.currentReplacedContactPeerIds = nil for table in self.tables { table.beforeCommit() @@ -676,6 +699,16 @@ public final class Postbox { } } + private func replaceContactPeerIds(_ peerIds: Set) { + self.contactsTable.replace(peerIds) + + self.currentReplacedContactPeerIds = peerIds + } + + private func replaceRecentPeerIds(_ peerIds: [PeerId]) { + self.peerRatingTable.replace(items: peerIds) + } + private func updateMessage(_ index: MessageIndex, update: (Message) -> StoreMessage) { if let intermediateMessage = self.messageHistoryTable.getMessage(index) { let message = self.renderIntermediateMessage(intermediateMessage) @@ -737,6 +770,26 @@ public final class Postbox { } } + public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.async { + var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) + if let anchorIndex = self.messageHistoryTable.anchorIndex(messageId) { + index = anchorIndex + } + disposable.set(self.aroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, tagMask: tagMask).start(next: { next in + subscriber.putNext(next) + }, error: { error in + subscriber.putError(error) + }, completed: { + subscriber.putCompletion() + })) + } + return disposable + } + } + public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return self.aroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, tagMask: tagMask) } @@ -895,6 +948,85 @@ public final class Postbox { } } + public func contactPeerIdsView() -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.queue.async { + let view = MutableContactPeerIdsView(peerIds: self.contactsTable.get()) + let (index, signal) = self.viewTracker.addContactPeerIdsView(view) + subscriber.putNext(ContactPeerIdsView(view)) + + let updatedViewDisposable = signal.start(next: { view in + subscriber.putNext(view) + }) + + disposable.set(ActionDisposable { + updatedViewDisposable.dispose() + self.viewTracker.removeContactPeerIdsView(index) + }) + } + + return disposable + } + } + + public func contactPeersView(index: PeerNameIndex, accountPeerId: PeerId) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.queue.async { + var peers: [PeerId: Peer] = [:] + for peerId in self.contactsTable.get() { + if let peer = self.peerTable.get(peerId) { + peers[peerId] = peer + } + } + + let view = MutableContactPeersView(peers: peers, index: index, accountPeer: self.peerTable.get(accountPeerId)) + let (index, signal) = self.viewTracker.addContactPeersView(view) + subscriber.putNext(ContactPeersView(view)) + + let updatedViewDisposable = signal.start(next: { view in + subscriber.putNext(view) + }) + + disposable.set(ActionDisposable { + updatedViewDisposable.dispose() + self.viewTracker.removeContactPeersView(index) + }) + } + + return disposable + } + } + + public func searchContacts(query: String) -> Signal<[Peer], NoError> { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.queue.async { + var peers: [Peer] = [] + for peerId in self.contactsTable.get() { + if let peer = self.peerTable.get(peerId) { + if peer.indexName.match(query: query) { + peers.append(peer) + } + } + } + + peers.sort(by: { $0.indexName.indexName(.lastNameFirst) < $1.indexName.indexName(.lastNameFirst) }) + + subscriber.putNext(peers) + + disposable.set(ActionDisposable { + }) + } + + return disposable + } + } + public func peerWithId(_ id: PeerId) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -912,4 +1044,21 @@ public final class Postbox { self.viewTracker.updateMessageHistoryViewVisibleRange(id, earliestVisibleIndex: earliestVisibleIndex, latestVisibleIndex: latestVisibleIndex) }) } + + public func recentPeers() -> Signal<[Peer], NoError> { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.justDispatch({ + let peerIds = self.peerRatingTable.get() + var peers: [Peer] = [] + for peerId in peerIds { + if let peer: Peer = self.peerTable.get(peerId) { + peers.append(peer) + } + } + subscriber.putNext(peers) + }) + return disposable + } + } } diff --git a/Postbox/RatingTable.swift b/Postbox/RatingTable.swift new file mode 100644 index 0000000000..be80b7bd94 --- /dev/null +++ b/Postbox/RatingTable.swift @@ -0,0 +1,76 @@ +import Foundation + +protocol RatingTableItem { + static func emptyKey() -> ValueBoxKey + static func fromKey(key: ValueBoxKey) -> Self + func ratingKey(rating: Int32, sharedKey: ValueBoxKey) -> ValueBoxKey +} + +extension PeerId: RatingTableItem { + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 4 + 8) + } + + static func fromKey(key: ValueBoxKey) -> PeerId { + return PeerId(key.getInt64(4)) + } + + func ratingKey(rating: Int32, sharedKey: ValueBoxKey) -> ValueBoxKey { + sharedKey.setInt32(0, value: rating) + sharedKey.setInt64(4, value: self.toInt64()) + return sharedKey + } +} + +final class RatingTable: Table { + private var items: [T]? + + func get() -> [T] { + if let items = self.items { + return items + } else { + let lowerBound = T.emptyKey() + let upperBound = T.emptyKey() + memset(lowerBound.memory, 0, lowerBound.length) + memset(upperBound.memory, 0xff, upperBound.length) + + var result: [T] = [] + self.valueBox.range(self.tableId, start: lowerBound, end: upperBound, keys: { key in + result.append(T.fromKey(key: key)) + return true + }, limit: 0) + + self.items = result + return result + } + } + + func replace(items: [T]) { + var keys: [ValueBoxKey] = [] + let lowerBound = T.emptyKey() + let upperBound = T.emptyKey() + memset(lowerBound.memory, 0, lowerBound.length) + memset(upperBound.memory, 0xff, upperBound.length) + self.valueBox.range(self.tableId, start: lowerBound, end: upperBound, keys: { key in + keys.append(key) + return true + }, limit: 0) + + for key in keys { + self.valueBox.remove(self.tableId, key: key) + } + + let sharedKey = T.emptyKey() + var index: Int32 = 0 + for item in items { + self.valueBox.set(self.tableId, key: item.ratingKey(rating: index, sharedKey: sharedKey), value: MemoryBuffer()) + index += 1 + } + + self.items = items + } + + override func beforeCommit() { + + } +} diff --git a/Postbox/SimpleDictionary.swift b/Postbox/SimpleDictionary.swift index 42efa12b69..f9a1d24591 100644 --- a/Postbox/SimpleDictionary.swift +++ b/Postbox/SimpleDictionary.swift @@ -3,6 +3,9 @@ import Foundation public struct SimpleDictionary: Sequence { private var items: [(K, V)] = [] + public init() { + } + public subscript(key: K) -> V? { get { for (k, value) in self.items { @@ -30,8 +33,6 @@ public struct SimpleDictionary: Sequence { } } - - public func makeIterator() -> AnyIterator<(K, V)> { var index = 0 return AnyIterator { () -> (K, V)? in diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 9b97614bc3..7e43a16bcd 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -16,6 +16,7 @@ final class ViewTracker { private let fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] private let fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex? private let renderMessage: (IntermediateMessage) -> Message + private let getPeer: (PeerId) -> Peer? private let fetchChatListHole: (ChatListHole) -> Disposable private let fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable private let sendUnsentMessage: (MessageIndex) -> Disposable @@ -25,13 +26,15 @@ final class ViewTracker { private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] private var unsentMessageView: UnsentMessageHistoryView private var synchronizeReadStatesView: SynchronizePeerReadStatesView + private var contactPeerIdsViews = Bag<(MutableContactPeerIdsView, ValuePipe)>() + private var contactPeersViews = Bag<(MutableContactPeersView, ValuePipe)>() private var chatListHoleDisposables: [(ChatListHole, Disposable)] = [] private var holeDisposablesByPeerId: [PeerId: [(MessageHistoryHole, Disposable)]] = [:] private var unsentMessageDisposables: [MessageIndex: Disposable] = [:] private var synchronizeReadStatesDisposables: [PeerId: (PeerReadStateSynchronizationOperation, Disposable)] = [:] - init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: (IntermediateMessage) -> Message, fetchChatListHole: (ChatListHole) -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable, sendUnsentMessage: (MessageIndex) -> Disposable, unsentMessageIndices: [MessageIndex], synchronizeReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Disposable, synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: (IntermediateMessage) -> Message, getPeer: (PeerId) -> Peer?, fetchChatListHole: (ChatListHole) -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable, sendUnsentMessage: (MessageIndex) -> Disposable, unsentMessageIndices: [MessageIndex], synchronizeReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Disposable, synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -39,6 +42,7 @@ final class ViewTracker { self.fetchLaterChatEntries = fetchLaterChatEntries self.fetchAnchorIndex = fetchAnchorIndex self.renderMessage = renderMessage + self.getPeer = getPeer self.fetchChatListHole = fetchChatListHole self.fetchMessageHistoryHole = fetchMessageHistoryHole self.sendUnsentMessage = sendUnsentMessage @@ -106,6 +110,28 @@ final class ViewTracker { self.updateTrackedChatListHoles() } + func addContactPeerIdsView(_ view: MutableContactPeerIdsView) -> (Bag<(MutableContactPeerIdsView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.contactPeerIdsViews.add(record) + + return (index, record.1.signal()) + } + + func removeContactPeerIdsView(_ index: Bag<(MutableContactPeerIdsView, ValuePipe)>.Index) { + self.contactPeerIdsViews.remove(index) + } + + func addContactPeersView(_ view: MutableContactPeersView) -> (Bag<(MutableContactPeersView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.contactPeersViews.add(record) + + return (index, record.1.signal()) + } + + func removeContactPeersView(_ index: Bag<(MutableContactPeersView, ValuePipe)>.Index) { + self.contactPeersViews.remove(index) + } + func updateMessageHistoryViewVisibleRange(_ id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { if let bag = self.messageHistoryViews[id.peerId] { for (mutableView, pipe) in bag.copyItems() { @@ -138,7 +164,7 @@ final class ViewTracker { } } - func updateViews(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?]) { + func updateViews(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { @@ -214,6 +240,20 @@ final class ViewTracker { if !synchronizeReadStateUpdates.isEmpty { self.synchronizeReadStateViewUpdated(synchronizeReadStateUpdates) } + + if let replaceContactPeerIds = replaceContactPeerIds { + for (mutableView, pipe) in self.contactPeerIdsViews.copyItems() { + if mutableView.replay(replace: replaceContactPeerIds) { + pipe.putNext(ContactPeerIdsView(mutableView)) + } + } + + for (mutableView, pipe) in self.contactPeersViews.copyItems() { + if mutableView.replay(replace: replaceContactPeerIds, getPeer: self.getPeer) { + pipe.putNext(ContactPeersView(mutableView)) + } + } + } } private func updateTrackedChatListHoles() { From 0eb6029dc9d62debc2c2702b734b20a8d6341283 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 19 Aug 2016 16:20:03 +0300 Subject: [PATCH 039/427] no message --- Postbox.xcodeproj/project.pbxproj | 43 ++ Postbox/ChatListHole.swift | 6 +- Postbox/ChatListHolesView.swift | 22 + Postbox/ChatListView.swift | 18 + Postbox/ContactTable.swift | 7 + Postbox/IntermediateMessage.swift | 50 ++ Postbox/IpcNotifier.h | 14 + Postbox/IpcNotifier.mm | 195 ++++++++ Postbox/IpcPipe.swift | 56 +++ Postbox/MediaBox.swift | 4 +- Postbox/Message.swift | 49 -- Postbox/MessageHistoryHolesView.swift | 54 +++ Postbox/MessageHistoryIndexTable.swift | 1 + Postbox/MessageHistoryMetadataTable.swift | 9 + Postbox/MessageHistoryOperation.swift | 9 + Postbox/MessageHistoryReadStateTable.swift | 5 + Postbox/MessageHistoryTable.swift | 8 - Postbox/MessageHistoryView.swift | 22 +- Postbox/MetadataTable.swift | 44 +- Postbox/PeerChatStateTable.swift | 5 + Postbox/PeerTable.swift | 7 +- Postbox/Postbox.h | 1 + Postbox/Postbox.swift | 484 +++++++++++--------- Postbox/PostboxClientId.swift | 8 + Postbox/PostboxTransaction.swift | 61 +++ Postbox/RatingTable.swift | 4 + Postbox/SqliteValueBox.swift | 3 +- Postbox/SynchronizePeerReadStatesView.swift | 31 +- Postbox/Table.swift | 3 + Postbox/UnsentMessageHistoryView.swift | 37 +- Postbox/UnsentMessageIndicesView.swift | 9 + Postbox/ViewTracker.swift | 351 +++++++------- 32 files changed, 1115 insertions(+), 505 deletions(-) create mode 100644 Postbox/ChatListHolesView.swift create mode 100644 Postbox/IntermediateMessage.swift create mode 100644 Postbox/IpcNotifier.h create mode 100644 Postbox/IpcNotifier.mm create mode 100644 Postbox/IpcPipe.swift create mode 100644 Postbox/MessageHistoryHolesView.swift create mode 100644 Postbox/MessageHistoryOperation.swift create mode 100644 Postbox/PostboxClientId.swift create mode 100644 Postbox/PostboxTransaction.swift create mode 100644 Postbox/UnsentMessageIndicesView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index c8b9a7c4e9..4e46330ecb 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -51,6 +51,9 @@ D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; + D0AB0B8C1D65D488002C78E7 /* MessageHistoryHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */; }; + D0AB0B8E1D65D49C002C78E7 /* ChatListHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */; }; + D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; @@ -58,6 +61,13 @@ D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */; }; D0C9DA391C65782500855278 /* SimpleSet.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C9DA381C65782500855278 /* SimpleSet.swift */; }; D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; + D0D510F41D63BA8400A97B8A /* PostboxTransaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F31D63BA8400A97B8A /* PostboxTransaction.swift */; }; + D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */; }; + D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */; }; + D0D510FB1D63C06E00A97B8A /* PostboxClientId.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510FA1D63C06E00A97B8A /* PostboxClientId.swift */; }; + D0D511001D64A58900A97B8A /* IpcPipe.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510FF1D64A58900A97B8A /* IpcPipe.swift */; }; + D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */ = {isa = PBXBuildFile; fileRef = D0D511011D64D73D00A97B8A /* IpcNotifier.mm */; }; + D0D511041D64D91C00A97B8A /* IpcNotifier.h in Headers */ = {isa = PBXBuildFile; fileRef = D0D511031D64D75200A97B8A /* IpcNotifier.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */; }; D0D949F51D35353900740E02 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; @@ -139,6 +149,9 @@ D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessMediaResourceContext.swift; sourceTree = ""; }; D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTable.swift; sourceTree = ""; }; D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; + D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHolesView.swift; sourceTree = ""; }; + D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatListHolesView.swift; sourceTree = ""; }; + D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageIndicesView.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; @@ -146,6 +159,13 @@ D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryViewTests.swift; sourceTree = ""; }; D0C9DA381C65782500855278 /* SimpleSet.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SimpleSet.swift; sourceTree = ""; }; D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResource.swift; sourceTree = ""; }; + D0D510F31D63BA8400A97B8A /* PostboxTransaction.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxTransaction.swift; sourceTree = ""; }; + D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryOperation.swift; sourceTree = ""; }; + D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IntermediateMessage.swift; sourceTree = ""; }; + D0D510FA1D63C06E00A97B8A /* PostboxClientId.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxClientId.swift; sourceTree = ""; }; + D0D510FF1D64A58900A97B8A /* IpcPipe.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IpcPipe.swift; sourceTree = ""; }; + D0D511011D64D73D00A97B8A /* IpcNotifier.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = IpcNotifier.mm; sourceTree = ""; }; + D0D511031D64D75200A97B8A /* IpcNotifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = IpcNotifier.h; sourceTree = ""; }; D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessResourceTests.swift; sourceTree = ""; }; D0D949F41D35353900740E02 /* MappedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MappedFile.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; @@ -263,6 +283,7 @@ D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */, D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */, D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, + D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */, D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */, D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */, D0F9E8641C58CB7F00037222 /* ChatListTable.swift */, @@ -287,6 +308,7 @@ children = ( D0E3A7831B28AE0900A402D9 /* Peer.swift */, D0E3A79D1B28B50400A402D9 /* Message.swift */, + D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */, D0E3A7A11B28B7DC00A402D9 /* Media.swift */, D08C71391C501F0700779C0F /* MessageHistoryHole.swift */, D044CA2D1C618373002160FF /* ChatListHole.swift */, @@ -304,6 +326,7 @@ D0C9DA381C65782500855278 /* SimpleSet.swift */, D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */, D0D949F41D35353900740E02 /* MappedFile.swift */, + D0D510FF1D64A58900A97B8A /* IpcPipe.swift */, ); name = Utils; sourceTree = ""; @@ -328,6 +351,9 @@ D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */, D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */, D0079F641D5A457A00A27A2C /* ContactPeersView.swift */, + D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */, + D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */, + D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */, ); name = Views; sourceTree = ""; @@ -363,8 +389,10 @@ D0E1DE161C5EB06000C7826E /* Tables */, D0E1DE1A1C5EB0DB00C7826E /* Views */, D05F09981C9CAC1100BB6F96 /* Media Box */, + D0D510FA1D63C06E00A97B8A /* PostboxClientId.swift */, D0E3A7811B28ADD000A402D9 /* Postbox.swift */, D044CA291C617D39002160FF /* SeedConfiguration.swift */, + D0D510F31D63BA8400A97B8A /* PostboxTransaction.swift */, D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */, D0E3A74D1B28A7E300A402D9 /* Supporting Files */, ); @@ -378,6 +406,8 @@ D044E1621B2AD677001EE087 /* MurMurHash32.m */, D0E3A74F1B28A7E300A402D9 /* Postbox.h */, D0E3A74E1B28A7E300A402D9 /* Info.plist */, + D0D511031D64D75200A97B8A /* IpcNotifier.h */, + D0D511011D64D73D00A97B8A /* IpcNotifier.mm */, ); name = "Supporting Files"; sourceTree = ""; @@ -417,6 +447,7 @@ D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */, D00E0FBB1B85D192002E4EB5 /* midl.h in Headers */, D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */, + D0D511041D64D91C00A97B8A /* IpcNotifier.h in Headers */, D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */, D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */, D07516461B2D9CEF00AE42E0 /* sqlite3ext.h in Headers */, @@ -545,6 +576,7 @@ D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */, + D0AB0B8C1D65D488002C78E7 /* MessageHistoryHolesView.swift in Sources */, D0F9E8671C58D08900037222 /* ChatListIndexTable.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, @@ -554,9 +586,11 @@ D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */, D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, + D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, D0D949F51D35353900740E02 /* MappedFile.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, + D0AB0B8E1D65D49C002C78E7 /* ChatListHolesView.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */, @@ -564,10 +598,13 @@ D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, + D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D0FC1A3C1CA3FB310056AE9A /* MediaResourceData.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, + D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */, + D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */, D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */, D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */, D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */, @@ -576,9 +613,12 @@ D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */, + D0D510F41D63BA8400A97B8A /* PostboxTransaction.swift in Sources */, + D0D510FB1D63C06E00A97B8A /* PostboxClientId.swift in Sources */, D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */, D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, + D0D511001D64A58900A97B8A /* IpcPipe.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -653,6 +693,7 @@ D086A5721CC0116A00F08284 /* Hockeyapp */ = { isa = XCBuildConfiguration; buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; CLANG_ENABLE_MODULES = YES; "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution"; DEFINES_MODULE = YES; @@ -792,6 +833,7 @@ isa = XCBuildConfiguration; baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; CLANG_ENABLE_MODULES = YES; "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; DEFINES_MODULE = YES; @@ -822,6 +864,7 @@ D0E3A7621B28A7E300A402D9 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; CLANG_ENABLE_MODULES = YES; "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution"; DEFINES_MODULE = YES; diff --git a/Postbox/ChatListHole.swift b/Postbox/ChatListHole.swift index 741dcc62b3..2ac25803ad 100644 --- a/Postbox/ChatListHole.swift +++ b/Postbox/ChatListHole.swift @@ -1,6 +1,6 @@ import Foundation -public struct ChatListHole: Comparable, CustomStringConvertible { +public struct ChatListHole: Hashable, CustomStringConvertible { public let index: MessageIndex public init(index: MessageIndex) { @@ -10,6 +10,10 @@ public struct ChatListHole: Comparable, CustomStringConvertible { public var description: String { return "ChatListHole(\(self.index.id), \(self.index.timestamp))" } + + public var hashValue: Int { + return self.index.hashValue + } } public func ==(lhs: ChatListHole, rhs: ChatListHole) -> Bool { diff --git a/Postbox/ChatListHolesView.swift b/Postbox/ChatListHolesView.swift new file mode 100644 index 0000000000..21b256fc4e --- /dev/null +++ b/Postbox/ChatListHolesView.swift @@ -0,0 +1,22 @@ +import Foundation + +final class MutableChatListHolesView { + private var entries = Set() + + func update(holes: Set) -> Bool { + if self.entries != holes { + self.entries = holes + return true + } else { + return false + } + } +} + +public final class ChatListHolesView { + public let entries: Set + + init(_ mutableView: MutableChatListHolesView) { + self.entries = mutableView.entries + } +} diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index f3e0bc64e7..077deba59e 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -113,6 +113,24 @@ final class MutableChatListView { self.count = count } + func refreshDueToExternalTransaction(fetchAroundChatEntries: (index: MessageIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?)) -> Bool { + var index = MessageIndex.absoluteUpperBound() + if !self.entries.isEmpty { + index = self.entries[self.entries.count / 2].index + } + + var (entries, earlier, later) = fetchAroundChatEntries(index: index, count: self.entries.count) + + if entries != self.entries || earlier != self.earlier || later != self.later { + self.entries = entries + self.earlier = earlier + self.later = later + return true + } else { + return false + } + } + func replay(_ operations: [ChatListOperation], context: MutableChatListViewReplayContext) -> Bool { var hasChanges = false for operation in operations { diff --git a/Postbox/ContactTable.swift b/Postbox/ContactTable.swift index 0529d6ca71..4f55e289d6 100644 --- a/Postbox/ContactTable.swift +++ b/Postbox/ContactTable.swift @@ -52,6 +52,13 @@ final class ContactTable: Table { self.peerIds = ids } + override func clearMemoryCache() { + self.originalPeerIds = nil + self.peerIds = nil + self.addedPeerIds.removeAll() + self.removedPeerIds.removeAll() + } + override func beforeCommit() { let sharedKey = self.key(PeerId(namespace: 0, id: 0)) diff --git a/Postbox/IntermediateMessage.swift b/Postbox/IntermediateMessage.swift new file mode 100644 index 0000000000..de95c68f27 --- /dev/null +++ b/Postbox/IntermediateMessage.swift @@ -0,0 +1,50 @@ +import Foundation + +struct IntermediateMessageForwardInfo { + let authorId: PeerId + let sourceId: PeerId? + let sourceMessageId: MessageId? + let date: Int32 + + init(authorId: PeerId, sourceId: PeerId?, sourceMessageId: MessageId?, date: Int32) { + self.authorId = authorId + self.sourceId = sourceId + self.sourceMessageId = sourceMessageId + self.date = date + } + + init(_ storeInfo: StoreMessageForwardInfo) { + self.authorId = storeInfo.authorId + self.sourceId = storeInfo.sourceId + self.sourceMessageId = storeInfo.sourceMessageId + self.date = storeInfo.date + } +} + +class IntermediateMessage { + let stableId: UInt32 + let id: MessageId + let timestamp: Int32 + let flags: MessageFlags + let tags: MessageTags + let forwardInfo: IntermediateMessageForwardInfo? + let authorId: PeerId? + let text: String + let attributesData: ReadBuffer + let embeddedMediaData: ReadBuffer + let referencedMedia: [MediaId] + + init(stableId: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { + self.stableId = stableId + self.id = id + self.timestamp = timestamp + self.flags = flags + self.tags = tags + self.forwardInfo = forwardInfo + self.authorId = authorId + self.text = text + self.attributesData = attributesData + self.embeddedMediaData = embeddedMediaData + self.referencedMedia = referencedMedia + } +} diff --git a/Postbox/IpcNotifier.h b/Postbox/IpcNotifier.h new file mode 100644 index 0000000000..09f09edb4f --- /dev/null +++ b/Postbox/IpcNotifier.h @@ -0,0 +1,14 @@ +#ifndef IpcNotifier_h +#define IpcNotifier_h + +#import + +@interface RLMNotifier : NSObject + +- (instancetype)initWithBasePath:(NSString *)basePath notify:(void (^)())notify; +- (void)listen; +- (void)notifyOtherRealms; + +@end + +#endif diff --git a/Postbox/IpcNotifier.mm b/Postbox/IpcNotifier.mm new file mode 100644 index 0000000000..60ec850f62 --- /dev/null +++ b/Postbox/IpcNotifier.mm @@ -0,0 +1,195 @@ +#import +#import +#import +#import +#import + +#import "IpcNotifier.h" + +// Write a byte to a pipe to notify anyone waiting for data on the pipe +static void notifyFd(int fd) { + while (true) { + char c = 0; + ssize_t ret = write(fd, &c, 1); + if (ret == 1) { + break; + } + + // If the pipe's buffer is full, we need to read some of the old data in + // it to make space. We don't just read in the code waiting for + // notifications so that we can notify multiple waiters with a single + // write. + assert(ret == -1 && errno == EAGAIN); + char buff[1024]; + read(fd, buff, sizeof buff); + } +} + +namespace { + // A RAII holder for a file descriptor which automatically closes the wrapped fd + // when it's deallocated + class FdHolder { + int fd = -1; + void close() { + if (fd != -1) { + ::close(fd); + } + fd = -1; + } + + FdHolder& operator=(FdHolder const&) = delete; + FdHolder(FdHolder const&) = delete; + + public: + FdHolder() { } + ~FdHolder() { close(); } + operator int() const { return fd; } + + FdHolder& operator=(int newFd) { + close(); + fd = newFd; + return *this; + } + }; +} + +// Inter-thread and inter-process notifications of changes are done using a +// named pipe in the filesystem next to the Realm file. Everyone who wants to be +// notified of commits waits for data to become available on the pipe, and anyone +// who commits a write transaction writes data to the pipe after releasing the +// write lock. Note that no one ever actually *reads* from the pipe: the data +// actually written is meaningless, and trying to read from a pipe from multiple +// processes at once is fraught with race conditions. + +// When a RLMRealm instance is created, we add a CFRunLoopSource to the current +// thread's runloop. On each cycle of the run loop, the run loop checks each of +// its sources for work to do, which in the case of CFRunLoopSource is just +// checking if CFRunLoopSourceSignal has been called since the last time it ran, +// and if so invokes the function pointer supplied when the source is created, +// which in our case just invokes `[realm handleExternalChange]`. + +// Listening for external changes is done using kqueue() on a background thread. +// kqueue() lets us efficiently wait until the amount of data which can be read +// from one or more file descriptors has changed, and tells us which of the file +// descriptors it was that changed. We use this to wait on both the shared named +// pipe, and a local anonymous pipe. When data is written to the named pipe, we +// signal the runloop source and wake up the target runloop, and when data is +// written to the anonymous pipe the background thread removes the runloop +// source from the runloop and and shuts down. + +@implementation RLMNotifier { + // Realm to notify of changes + void (^_notify)(); + // Runloop which notifications are delivered on + CFRunLoopRef _runLoop; + + // Read-write file descriptor for the named pipe which is waited on for + // changes and written to when a commit is made + FdHolder _notifyFd; + // File descriptor for the kqueue + FdHolder _kq; + // The two ends of an anonymous pipe used to notify the kqueue() thread that + // it should be shut down. + FdHolder _shutdownReadFd; + FdHolder _shutdownWriteFd; +} + +- (instancetype)initWithBasePath:(NSString *)basePath notify:(void (^)())notify { + self = [super init]; + if (self) { + _notify = [notify copy]; + + _kq = kqueue(); + if (_kq == -1) { + return nil; + } + + const char *path = [basePath stringByAppendingString:@"postbox.note"].UTF8String; + + // Create and open the named pipe + int ret = mkfifo(path, 0600); + if (ret == -1) { + int err = errno; + if (err == ENOTSUP) { + // Filesystem doesn't support named pipes, so try putting it in tmp instead + // Hash collisions are okay here because they just result in doing + // extra work, as opposed to correctness problems + static NSString *tmpDir = NSTemporaryDirectory(); + path = [tmpDir stringByAppendingFormat:@"poxtbox_%llu.note", (unsigned long long)[basePath hash]].UTF8String; + ret = mkfifo(path, 0600); + err = errno; + } + // the fifo already existing isn't an error + if (ret == -1 && err != EEXIST) { + return nil; + } + } + + _notifyFd = open(path, O_RDWR); + if (_notifyFd == -1) { + return nil; + } + + // Make writing to the pipe return -1 when the pipe's buffer is full + // rather than blocking until there's space available + ret = fcntl(_notifyFd, F_SETFL, O_NONBLOCK); + if (ret == -1) { + return nil; + } + + // Create the anonymous pipe + int pipeFd[2]; + ret = pipe(pipeFd); + if (ret == -1) { + return nil; + } + + _shutdownReadFd = pipeFd[0]; + _shutdownWriteFd = pipeFd[1]; + } + return self; +} + +- (void)listen { + // Set up the kqueue + // EVFILT_READ indicates that we care about data being available to read + // on the given file descriptor. + // EV_CLEAR makes it wait for the amount of data available to be read to + // change rather than just returning when there is any data to read. + struct kevent ke[2]; + EV_SET(&ke[0], _notifyFd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, 0); + EV_SET(&ke[1], _shutdownReadFd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, 0); + int ret = kevent(_kq, ke, 2, nullptr, 0, nullptr); + assert(ret == 0); + + while (true) { + struct kevent event; + // Wait for data to become on either fd + // Return code is number of bytes available or -1 on error + ret = kevent(_kq, nullptr, 0, &event, 1, nullptr); + assert(ret >= 0); + if (ret == 0) { + // Spurious wakeup; just wait again + continue; + } + + // Check which file descriptor had activity: if it's the shutdown + // pipe, then someone called -stop; otherwise it's the named pipe + // and someone committed a write transaction + if (event.ident == (uint32_t)_shutdownReadFd) { + return; + } + assert(event.ident == (uint32_t)_notifyFd); + + _notify(); + } +} + +- (void)stop { + notifyFd(_shutdownWriteFd); +} + +- (void)notifyOtherRealms { + notifyFd(_notifyFd); +} +@end diff --git a/Postbox/IpcPipe.swift b/Postbox/IpcPipe.swift new file mode 100644 index 0000000000..80e762e468 --- /dev/null +++ b/Postbox/IpcPipe.swift @@ -0,0 +1,56 @@ +import Foundation +import SwiftSignalKit + +func ipcNotify(basePath: String, data: Int64) { + DispatchQueue.global(qos: .default).async { + var path = basePath + ".ipc" + let fd = open(path, open(path, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR)) + if fd != -1 { + var value = data + write(fd, &value, 8) + close(fd) + } + } +} + +func ipcNotifications(basePath: String) -> Signal { + return Signal { subscriber in + let queue = Queue() + let disposable = MetaDisposable() + + queue.async { + var path = basePath + ".ipc" + let fd = open(path, open(path, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR)) + if fd != -1 { + let readSource = DispatchSource.makeFileSystemObjectSource(fileDescriptor: fd, eventMask: [.write]) + + var previousValue: Int64 = 0 + readSource.setEventHandler(handler: { + subscriber.putNext(Int64.max) + /*lseek(fd, 0, SEEK_SET) + var value: Int64 = 0 + if read(fd, &value, 8) == 8 { + if previousValue != value { + previousValue = value + subscriber.putNext(value) + } + }*/ + }) + + readSource.resume() + + disposable.set(ActionDisposable { + queue.async { + readSource.cancel() + close(fd) + } + }) + } else { + subscriber.putError(NoError()) + } + } + + return disposable + } +} + diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 3a310e80db..600bd826ad 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -424,9 +424,9 @@ public final class MediaBox { var offset = currentSize var fd: Int32? - dataContext.fetchDisposable = (self.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in + dataContext.fetchDisposable = ((self.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in return fetch(resource, currentSize ..< resource.size) - } |> afterDisposed { + }) |> afterDisposed { if let fd = fd { close(fd) } diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 5312043686..e1f139bd24 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -381,52 +381,3 @@ final class InternalStoreMessage { self.media = media } } - -struct IntermediateMessageForwardInfo { - let authorId: PeerId - let sourceId: PeerId? - let sourceMessageId: MessageId? - let date: Int32 - - init(authorId: PeerId, sourceId: PeerId?, sourceMessageId: MessageId?, date: Int32) { - self.authorId = authorId - self.sourceId = sourceId - self.sourceMessageId = sourceMessageId - self.date = date - } - - init(_ storeInfo: StoreMessageForwardInfo) { - self.authorId = storeInfo.authorId - self.sourceId = storeInfo.sourceId - self.sourceMessageId = storeInfo.sourceMessageId - self.date = storeInfo.date - } -} - -class IntermediateMessage { - let stableId: UInt32 - let id: MessageId - let timestamp: Int32 - let flags: MessageFlags - let tags: MessageTags - let forwardInfo: IntermediateMessageForwardInfo? - let authorId: PeerId? - let text: String - let attributesData: ReadBuffer - let embeddedMediaData: ReadBuffer - let referencedMedia: [MediaId] - - init(stableId: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { - self.stableId = stableId - self.id = id - self.timestamp = timestamp - self.flags = flags - self.tags = tags - self.forwardInfo = forwardInfo - self.authorId = authorId - self.text = text - self.attributesData = attributesData - self.embeddedMediaData = embeddedMediaData - self.referencedMedia = referencedMedia - } -} diff --git a/Postbox/MessageHistoryHolesView.swift b/Postbox/MessageHistoryHolesView.swift new file mode 100644 index 0000000000..755b8047a2 --- /dev/null +++ b/Postbox/MessageHistoryHolesView.swift @@ -0,0 +1,54 @@ +import Foundation + +public struct MessageHistoryHolesViewEntry: Hashable { + public let hole: MessageHistoryHole + public let direction: HoleFillDirection + public let tags: MessageTags? + + public var hashValue: Int { + return self.hole.maxIndex.hashValue + } +} + +public func ==(lhs: MessageHistoryHolesViewEntry, rhs: MessageHistoryHolesViewEntry) -> Bool { + return lhs.hole == rhs.hole && lhs.direction == rhs.direction && lhs.tags == rhs.tags +} + +public func <(lhs: MessageHistoryHolesViewEntry, rhs: MessageHistoryHolesViewEntry) -> Bool { + return lhs.hole.maxIndex < rhs.hole.maxIndex +} + +final class MutableMessageHistoryHolesView { + private var entries: [PeerId: Set] = [:] + + init() { + } + + func update(peerId: PeerId, holes: Set) -> Bool { + if let currentHoles = self.entries[peerId] { + if currentHoles != holes { + if holes.isEmpty { + self.entries.removeValue(forKey: peerId) + } else { + self.entries[peerId] = holes + } + return true + } else { + return false + } + } else if !holes.isEmpty { + self.entries[peerId] = holes + return true + } else { + return false + } + } +} + +public final class MessageHistoryHolesView { + public let entries: [PeerId: Set] + + init(_ mutableView: MutableMessageHistoryHolesView) { + self.entries = mutableView.entries + } +} diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index fbf7eaeefc..fd10190518 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -553,6 +553,7 @@ final class MessageHistoryIndexTable: Table { self.valueBox.set(self.tableId, key: self.key(index.id), value: value) operations.append(.InsertMessage(message)) + if index.id.namespace == self.globalMessageIdsNamespace { self.globalMessageIdsTable.set(index.id.id, id: index.id) } diff --git a/Postbox/MessageHistoryMetadataTable.swift b/Postbox/MessageHistoryMetadataTable.swift index 382bd13a6b..5c6f3c7bf6 100644 --- a/Postbox/MessageHistoryMetadataTable.swift +++ b/Postbox/MessageHistoryMetadataTable.swift @@ -142,6 +142,15 @@ final class MessageHistoryMetadataTable: Table { } } + override func clearMemoryCache() { + self.initializedChatList = nil + self.initializedHistoryPeerIds.removeAll() + self.peerNextMessageIdByNamespace.removeAll() + self.updatedPeerNextMessageIdByNamespace.removeAll() + self.nextMessageStableId = nil + self.nextMessageStableIdUpdated = false + } + override func beforeCommit() { let sharedBuffer = WriteBuffer() for (peerId, namespaces) in self.updatedPeerNextMessageIdByNamespace { diff --git a/Postbox/MessageHistoryOperation.swift b/Postbox/MessageHistoryOperation.swift new file mode 100644 index 0000000000..a71b7be8b2 --- /dev/null +++ b/Postbox/MessageHistoryOperation.swift @@ -0,0 +1,9 @@ +import Foundation + +enum MessageHistoryOperation { + case InsertMessage(IntermediateMessage) + case InsertHole(MessageHistoryHole) + case Remove([MessageIndex]) + case UpdateReadState(CombinedPeerReadState) + case UpdateEmbeddedMedia(MessageIndex, ReadBuffer) +} diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index d86aafbeee..d276067189 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -254,6 +254,11 @@ final class MessageHistoryReadStateTable: Table { return (combinedState, holes ? .Push(thenSync: true) : .None) } + override func clearMemoryCache() { + self.cachedPeerReadStates.removeAll() + self.updatedPeerIds.removeAll() + } + override func beforeCommit() { let sharedBuffer = WriteBuffer() for id in self.updatedPeerIds { diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 0ecb1f7cff..26e70435eb 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -1,13 +1,5 @@ import Foundation -enum MessageHistoryOperation { - case InsertMessage(IntermediateMessage) - case InsertHole(MessageHistoryHole) - case Remove([MessageIndex]) - case UpdateReadState(CombinedPeerReadState) - case UpdateEmbeddedMedia(MessageIndex, ReadBuffer) -} - struct MessageHistoryAnchorIndex { let index: MessageIndex let exact: Bool diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 05371fe0ae..a851cc8b6d 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -149,10 +149,6 @@ final class MutableMessageHistoryView { } func updateVisibleRange(earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex, context: MutableMessageHistoryViewReplayContext) -> Bool { - if (true) { - //return false - } - var minIndex: Int? var maxIndex: Int? @@ -226,6 +222,24 @@ final class MutableMessageHistoryView { return false } + func refreshDueToExternalTransaction(fetchAroundHistoryEntries: (index: MessageIndex, count: Int, tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?)) -> Bool { + var index = MessageIndex.absoluteUpperBound() + if !self.entries.isEmpty { + index = self.entries[self.entries.count / 2].index + } + + let (entries, earlier, later) = fetchAroundHistoryEntries(index: index, count: max(self.fillCount, self.entries.count), tagMask: self.tagMask) + + self.entries = entries + self.earlier = earlier + self.later = later + + /*let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) + + let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count)*/ + return true + } + func replay(_ operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], context: MutableMessageHistoryViewReplayContext) -> Bool { let tagMask = self.tagMask let unwrappedTagMask: UInt32 = tagMask?.rawValue ?? 0 diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index 3ef3729279..b55904b1b6 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -3,9 +3,13 @@ import Foundation private enum MetadataKey: Int32 { case UserVersion = 1 case State = 2 + case TransactionStateVersion = 3 + case MasterClientId = 4 } final class MetadataTable: Table { + private let sharedBuffer = WriteBuffer() + override init(valueBox: ValueBox, tableId: Int32) { super.init(valueBox: valueBox, tableId: tableId) } @@ -26,7 +30,8 @@ final class MetadataTable: Table { } func setUserVersion(_ version: Int32) { - let buffer = WriteBuffer() + sharedBuffer.reset() + let buffer = sharedBuffer var varVersion: Int32 = version buffer.write(&varVersion, offset: 0, length: 4) self.valueBox.set(self.tableId, key: self.key(.UserVersion), value: buffer) @@ -46,4 +51,41 @@ final class MetadataTable: Table { encoder.encodeRootObject(state) self.valueBox.set(self.tableId, key: self.key(.State), value: encoder.readBufferNoCopy()) } + + func transactionStateVersion() -> Int64 { + if let value = self.valueBox.get(self.tableId, key: self.key(.TransactionStateVersion)) { + var version: Int64 = 0 + value.read(&version, offset: 0, length: 8) + return version + } else { + return 0 + } + } + + func incrementTransactionStateVersion() -> Int64 { + var version = self.transactionStateVersion() + 1 + sharedBuffer.reset() + let buffer = sharedBuffer + buffer.write(&version, offset: 0, length: 8) + self.valueBox.set(self.tableId, key: self.key(.TransactionStateVersion), value: buffer) + return version + } + + func masterClientId() -> Int64 { + if let value = self.valueBox.get(self.tableId, key: self.key(.MasterClientId)) { + var clientId: Int64 = 0 + value.read(&clientId, offset: 0, length: 8) + return clientId + } else { + return 0 + } + } + + func setMasterClientId(_ id: Int64) { + sharedBuffer.reset() + let buffer = sharedBuffer + var clientId = id + buffer.write(&clientId, offset: 0, length: 8) + self.valueBox.set(self.tableId, key: self.key(.MasterClientId), value: buffer) + } } diff --git a/Postbox/PeerChatStateTable.swift b/Postbox/PeerChatStateTable.swift index d6abd4708b..8444efc67f 100644 --- a/Postbox/PeerChatStateTable.swift +++ b/Postbox/PeerChatStateTable.swift @@ -30,6 +30,11 @@ final class PeerChatStateTable: Table { self.updatedPeerIds.insert(id) } + override func clearMemoryCache() { + self.cachedPeerChatStates.removeAll() + self.updatedPeerIds.removeAll() + } + override func beforeCommit() { let sharedEncoder = Encoder() for id in self.updatedPeerIds { diff --git a/Postbox/PeerTable.swift b/Postbox/PeerTable.swift index a85ee8106d..4b4b12b91b 100644 --- a/Postbox/PeerTable.swift +++ b/Postbox/PeerTable.swift @@ -3,8 +3,8 @@ import Foundation final class PeerTable: Table { private let sharedEncoder = Encoder() private let sharedKey = ValueBoxKey(length: 8) - private var cachedPeers: [PeerId: Peer] = [:] + private var cachedPeers: [PeerId: Peer] = [:] private var updatedPeerIds = Set() override init(valueBox: ValueBox, tableId: Int32) { @@ -34,6 +34,11 @@ final class PeerTable: Table { return nil } + override func clearMemoryCache() { + self.cachedPeers.removeAll() + self.updatedPeerIds.removeAll() + } + override func beforeCommit() { for peerId in self.updatedPeerIds { if let peer = self.cachedPeers[peerId] { diff --git a/Postbox/Postbox.h b/Postbox/Postbox.h index 5cfad5bac5..4e6ba942bf 100644 --- a/Postbox/Postbox.h +++ b/Postbox/Postbox.h @@ -17,3 +17,4 @@ FOUNDATION_EXPORT const unsigned char PostboxVersionString[]; // In this header, you should import all the public headers of your framework using statements like #import #import +#import diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 6983346553..8d9ccbc042 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -130,14 +130,38 @@ public final class Modifier { } } +private class PipeNotifier: NSObject { + let notifier: RLMNotifier + let thread: Thread + + private init(basePath: String, notify: () -> Void) { + self.notifier = RLMNotifier(basePath: basePath, notify: notify) + self.thread = Thread(target: PipeNotifier.self, selector: #selector(PipeNotifier.threadEntry(_:)), object: self.notifier) + self.thread.start() + } + + @objc static func threadEntry(_ notifier: RLMNotifier!) { + notifier.listen() + } + + func notify() { + notifier.notifyOtherRealms() + } +} + public final class Postbox { private let seedConfiguration: SeedConfiguration private let basePath: String private let globalMessageIdsNamespace: MessageId.Namespace + private let ipcNotificationsDisposable = MetaDisposable() + private var pipeNotifier: PipeNotifier! + private let queue = Queue(name: "org.telegram.postbox.Postbox") private var valueBox: ValueBox! + private var transactionStateVersion: Int64 = 0 + private var viewTracker: ViewTracker! private var nextViewId = 0 @@ -153,28 +177,16 @@ public final class Postbox { private var currentUpdatedPeers: [PeerId: Peer] = [:] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? + private var currentUpdatedMasterClientId: Int64? private var statePipe: ValuePipe = ValuePipe() - - private let fetchChatListHoleImpl = Promise<(ChatListHole) -> Signal>() - public func setFetchChatListHole(fetch: (ChatListHole) -> Signal) { - self.fetchChatListHoleImpl.set(.single(fetch)) - } + private var masterClientId = Promise() - private let fetchMessageHistoryHoleImpl = Promise<(MessageHistoryHole, HoleFillDirection, MessageTags?) -> Signal>() - public func setFetchMessageHistoryHole(fetch: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Signal) { - self.fetchMessageHistoryHoleImpl.set(.single(fetch)) - } - - private let sendUnsentMessageImpl = Promise<(Message) -> Signal>() - public func setSendUnsentMessage(sendUnsentMessage: (Message) -> Signal) { - self.sendUnsentMessageImpl.set(.single(sendUnsentMessage)) - } - - private let synchronizePeerReadStateImpl = Promise<(PeerId, PeerReadStateSynchronizationOperation) -> Signal>() - public func setSynchronizePeerReadState(synchronizePeerReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Signal) { - self.synchronizePeerReadStateImpl.set(.single(synchronizePeerReadState)) - } + private var sessionClientId: Int64 = { + var value: Int64 = 0 + arc4random_buf(&value, 8) + return value + }() public let mediaBox: MediaBox @@ -212,6 +224,28 @@ public final class Postbox { //let _ = try? FileManager.default.removeItem(atPath: self.basePath + "/media") self.mediaBox = MediaBox(basePath: self.basePath + "/media") + + /*self.ipcNotificationsDisposable.set((ipcNotifications(basePath: basePath) + |> deliverOn(self.queue)).start(next: { [weak self] updatedTransactionStateVersion in + if let strongSelf = self, strongSelf.valueBox != nil { + if strongSelf.transactionStateVersion < updatedTransactionStateVersion { + strongSelf.modify({ _ -> Void in + }).start() + } + } + }))*/ + + self.pipeNotifier = PipeNotifier(basePath: basePath, notify: { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + if strongSelf.valueBox != nil { + strongSelf.modify({ _ -> Void in + }).start() + } + } + } + }) + self.openDatabase() } @@ -261,8 +295,8 @@ public final class Postbox { // debugging unread counters //self.debugRestoreState("afterLogin") - //self.debugSaveState("previous") - //self.debugRestoreState("previous") + //self.debugSaveState(name: "previous") + //self.debugRestoreState(name: "previous") //#endif @@ -270,7 +304,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 5 + let currentUserVersion: Int32 = 6 if userVersion != currentUserVersion { self.valueBox.drop() @@ -313,9 +347,11 @@ public final class Postbox { self.tables.append(self.contactsTable) self.tables.append(self.peerRatingTable) + self.transactionStateVersion = self.metadataTable.transactionStateVersion() + self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, fetchAnchorIndex: self.fetchAnchorIndex, renderMessage: self.renderIntermediateMessage, getPeer: { peerId in return self.peerTable.get(peerId) - }, fetchChatListHole: self.fetchChatListHoleWrapper, fetchMessageHistoryHole: self.fetchMessageHistoryHoleWrapper, sendUnsentMessage: self.sendUnsentMessageWrapper, unsentMessageIndices: self.messageHistoryUnsentTable!.get(), synchronizeReadState: self.synchronizePeerReadStateWrapper, synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) + }, unsentMessageIndices: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") }) @@ -330,13 +366,13 @@ public final class Postbox { private var cachedState: Coding? private func setState(_ state: Coding) { - self.queue.justDispatch({ + self.modify({ modifier -> Void in self.cachedState = state self.metadataTable.setState(state) self.statePipe.putNext(state) - }) + }).start() } private func getState() -> Coding? { @@ -353,17 +389,10 @@ public final class Postbox { } public func state() -> Signal { - return Signal { subscriber in - let disposable = MetaDisposable() - self.queue.justDispatch({ - subscriber.putNext(self.getState()) - disposable.set(self.statePipe.signal().start(next: { next in - subscriber.putNext(next) - })) - }) - - return disposable - } + return self.modify { modifier -> Signal in + return Signal.single(self.getState()) + |> then(self.statePipe.signal() |> map { $0 }) + } |> switchToLatest } public func keychainEntryForKey(_ key: String) -> Data? { @@ -622,44 +651,44 @@ public final class Postbox { private func renderIntermediateMessage(_ message: IntermediateMessage) -> Message { return self.messageHistoryTable.renderMessage(message, peerTable: self.peerTable) } - - private func fetchChatListHoleWrapper(_ hole: ChatListHole) -> Disposable { - return (self.fetchChatListHoleImpl.get() |> mapToSignal { fetch in - return fetch(hole) - }).start() - } - private func fetchMessageHistoryHoleWrapper(_ hole: MessageHistoryHole, direction: HoleFillDirection, tagMask: MessageTags?) -> Disposable { - return (self.fetchMessageHistoryHoleImpl.get() |> mapToSignal { fetch in - return fetch(hole, direction, tagMask) - }).start() - } - - private func sendUnsentMessageWrapper(_ index: MessageIndex) -> Disposable { - return (self.sendUnsentMessageImpl.get() |> deliverOn(self.queue) |> mapToSignal { send -> Signal in - if let intermediateMessage = self.messageHistoryTable.getMessage(index) { - let message = self.renderIntermediateMessage(intermediateMessage) - return send(message) - } else { - return never(Void.self, NoError.self) + private func afterBegin() { + let currentTransactionStateVersion = self.metadataTable.transactionStateVersion() + if currentTransactionStateVersion != self.transactionStateVersion { + for table in self.tables { + table.clearMemoryCache() } - }).start() + self.viewTracker.refreshViewsDueToExternalTransaction(fetchAroundChatEntries: self.fetchAroundChatEntries, fetchAroundHistoryEntries: self.fetchAroundHistoryEntries, fetchUnsendMessageIndices: { + return self.messageHistoryUnsentTable!.get() + }, fetchSynchronizePeerReadStateOperations: { + return self.synchronizeReadStateTable!.get() + }) + self.transactionStateVersion = currentTransactionStateVersion + + self.masterClientId.set(.single(self.metadataTable.masterClientId())) + } } - private func synchronizePeerReadStateWrapper(_ peerId: PeerId, operation: PeerReadStateSynchronizationOperation) -> Disposable { - return (self.synchronizePeerReadStateImpl.get() |> mapToSignal { validate -> Signal in - return validate(peerId, operation) - }).start() - } - - private func beforeCommit() { + private func beforeCommit() -> (updatedTransactionStateVersion: Int64?, updatedMasterClientId: Int64?) { var chatListOperations: [ChatListOperation] = [] self.chatListTable.replay(self.currentOperationsByPeerId, messageHistoryTable: self.messageHistoryTable, operations: &chatListOperations) for (index, hole) in self.currentReplaceChatListHoles { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } - self.viewTracker.updateViews(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds) + let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + var updatedTransactionState: Int64? + var updatedMasterClientId: Int64? + if !transaction.isEmpty { + self.viewTracker.updateViews(transaction: transaction) + self.transactionStateVersion = self.metadataTable.incrementTransactionStateVersion() + updatedTransactionState = self.transactionStateVersion + + if let currentUpdatedMasterClientId = self.currentUpdatedMasterClientId { + self.metadataTable.setMasterClientId(currentUpdatedMasterClientId) + updatedMasterClientId = currentUpdatedMasterClientId + } + } self.currentOperationsByPeerId.removeAll() self.currentFilledHolesByPeerId.removeAll() @@ -670,10 +699,13 @@ public final class Postbox { self.currentUpdatedSynchronizeReadStateOperations.removeAll() self.currentUpdatedMedia.removeAll() self.currentReplacedContactPeerIds = nil + self.currentUpdatedMasterClientId = nil for table in self.tables { table.beforeCommit() } + + return (updatedTransactionState, updatedMasterClientId) } private func messageIdsForGlobalIds(_ ids: [Int32]) -> [MessageId] { @@ -734,64 +766,52 @@ public final class Postbox { public func modify(_ f: (Modifier) -> T) -> Signal { return Signal { subscriber in - self.queue.justDispatch({ - //let startTime = CFAbsoluteTimeGetCurrent() - //self.valueBox.beginStats() + self.queue.justDispatch { self.valueBox.begin() + self.afterBegin() let result = f(Modifier(postbox: self)) - self.beforeCommit() + let (updatedTransactionState, updatedMasterClientId) = self.beforeCommit() self.valueBox.commit() subscriber.putNext(result) - //print("modify \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") subscriber.putCompletion() - }) + + if updatedTransactionState != nil || updatedMasterClientId != nil { + self.pipeNotifier.notify() + } + + if let updatedMasterClientId = updatedMasterClientId { + self.masterClientId.set(.single(updatedMasterClientId)) + } + } return EmptyDisposable } } public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return Signal { subscriber in - let disposable = MetaDisposable() - self.queue.async { - var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) - if let maxReadIndex = self.messageHistoryTable.maxReadIndex(peerId) { - index = maxReadIndex - } - disposable.set(self.aroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, tagMask: tagMask).start(next: { next in - subscriber.putNext(next) - }, error: { error in - subscriber.putError(error) - }, completed: { - subscriber.putCompletion() - })) + return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType), NoError> in + var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) + if let maxReadIndex = self.messageHistoryTable.maxReadIndex(peerId) { + index = maxReadIndex } - return disposable - } + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, tagMask: tagMask) + } |> switchToLatest } public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return Signal { subscriber in - let disposable = MetaDisposable() - self.queue.async { - var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) - if let anchorIndex = self.messageHistoryTable.anchorIndex(messageId) { - index = anchorIndex - } - disposable.set(self.aroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, tagMask: tagMask).start(next: { next in - subscriber.putNext(next) - }, error: { error in - subscriber.putError(error) - }, completed: { - subscriber.putCompletion() - })) + return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType), NoError> in + var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) + if let anchorIndex = self.messageHistoryTable.anchorIndex(messageId) { + index = anchorIndex } - return disposable - } + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, tagMask: tagMask) + } |> switchToLatest } public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return self.aroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, tagMask: tagMask) + return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType), NoError> in + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, tagMask: tagMask) + } |> switchToLatest } private func addLocationsToMessageHistoryViewEntries(tagMask: MessageTags, earlier: MutableMessageHistoryEntry?, later: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry]) -> ([MutableMessageHistoryEntry], MutableMessageHistoryEntry?, MutableMessageHistoryEntry?) { @@ -820,95 +840,60 @@ public final class Postbox { } } - private func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return Signal { subscriber in - let disposable = MetaDisposable() - let startTime = CFAbsoluteTimeGetCurrent() - self.queue.justDispatch({ - //print("+ queue \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) - - print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - - let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) - mutableView.render(self.renderIntermediateMessage) - //print("+ render \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - - let initialUpdateType: ViewUpdateType - if let unreadIndex = unreadIndex { - initialUpdateType = .InitialUnread(unreadIndex) - } else { - initialUpdateType = .Generic - } - subscriber.putNext((MessageHistoryView(mutableView), initialUpdateType)) - - let (index, signal) = self.viewTracker.addMessageHistoryView(peerId, view: mutableView) - - let pipeDisposable = signal.start(next: { next in - subscriber.putNext(next) - }) - - disposable.set(ActionDisposable { [weak self] in - pipeDisposable.dispose() - - if let strongSelf = self { - strongSelf.queue.async { - strongSelf.viewTracker.removeMessageHistoryView(peerId, index: index) - } - } - return - }) - }) - - return disposable + private func syncAroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + let startTime = CFAbsoluteTimeGetCurrent() + let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) + print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") + + let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) + mutableView.render(self.renderIntermediateMessage) + + let initialUpdateType: ViewUpdateType + if let unreadIndex = unreadIndex { + initialUpdateType = .InitialUnread(unreadIndex) + } else { + initialUpdateType = .Generic } + + let (index, signal) = self.viewTracker.addMessageHistoryView(peerId, view: mutableView) + + return (.single((MessageHistoryView(mutableView), initialUpdateType)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeMessageHistoryView(peerId, index: index) + } + } + } } public func messageIndexAtId(_ id: MessageId) -> Signal { - return Signal { subscriber in - let disposable = MetaDisposable() - - self.queue.justDispatch { - if let entry = self.messageHistoryIndexTable.get(id), case let .Message(index) = entry { - subscriber.putNext(index) - subscriber.putCompletion() - } else if let _ = self.messageHistoryIndexTable.holeContainingId(id) { - subscriber.putNext(nil) - subscriber.putCompletion() - } else { - subscriber.putNext(nil) - subscriber.putCompletion() - } + return self.modify { modifier -> Signal in + if let entry = self.messageHistoryIndexTable.get(id), case let .Message(index) = entry { + return .single(index) + } else if let _ = self.messageHistoryIndexTable.holeContainingId(id) { + return .single(nil) + } else { + return .single(nil) } - - return disposable - } + } |> switchToLatest } public func messageAtId(_ id: MessageId) -> Signal { - return Signal { subscriber in - let disposable = MetaDisposable() - - self.queue.justDispatch { - if let entry = self.messageHistoryIndexTable.get(id), case let .Message(index) = entry { - if let message = self.messageHistoryTable.getMessage(index) { - subscriber.putNext(self.renderIntermediateMessage(message)) - subscriber.putCompletion() - } else { - subscriber.putNext(nil) - subscriber.putCompletion() - } - } else if let _ = self.messageHistoryIndexTable.holeContainingId(id) { - subscriber.putNext(nil) - subscriber.putCompletion() + return self.modify { modifier -> Signal in + if let entry = self.messageHistoryIndexTable.get(id), case let .Message(index) = entry { + if let message = self.messageHistoryTable.getMessage(index) { + return .single(self.renderIntermediateMessage(message)) } else { - subscriber.putNext(nil) - subscriber.putCompletion() + return .single(nil) } + } else if let _ = self.messageHistoryIndexTable.holeContainingId(id) { + return .single(nil) + } else { + return .single(nil) } - - return disposable - } + } |> switchToLatest } public func tailChatListView(_ count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { @@ -916,36 +901,24 @@ public final class Postbox { } public func aroundChatListView(_ index: MessageIndex, count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { - return Signal { subscriber in - let disposable = MetaDisposable() + return self.modify { modifier -> Signal<(ChatListView, ViewUpdateType), NoError> in + let (entries, earlier, later) = self.fetchAroundChatEntries(index, count: count) - self.queue.justDispatch({ - let (entries, earlier, later) = self.fetchAroundChatEntries(index, count: count) - - let mutableView = MutableChatListView(earlier: earlier, entries: entries, later: later, count: count) - mutableView.render(self.renderIntermediateMessage) - subscriber.putNext((ChatListView(mutableView), .Generic)) - - let (index, signal) = self.viewTracker.addChatListView(mutableView) - - let pipeDisposable = signal.start(next: { next in - subscriber.putNext(next) - }) - - disposable.set(ActionDisposable { [weak self] in - pipeDisposable.dispose() - + let mutableView = MutableChatListView(earlier: earlier, entries: entries, later: later, count: count) + mutableView.render(self.renderIntermediateMessage) + + let (index, signal) = self.viewTracker.addChatListView(mutableView) + + return (.single((ChatListView(mutableView), .Generic)) + |> then(signal)) + |> afterDisposed { [weak self] in if let strongSelf = self { strongSelf.queue.async { strongSelf.viewTracker.removeChatListView(index) } } - return - }) - }) - - return disposable - } + } + } |> switchToLatest } public func contactPeerIdsView() -> Signal { @@ -1028,37 +1001,96 @@ public final class Postbox { } public func peerWithId(_ id: PeerId) -> Signal { - return Signal { subscriber in - let disposable = MetaDisposable() - self.queue.justDispatch({ - if let peer: Peer = self.peerTable.get(id) { - subscriber.putNext(peer) - } - }) - return disposable - } + return self.modify { modifier -> Signal in + if let peer = self.peerTable.get(id) { + return .single(peer) + } else { + return .never() + } + } |> switchToLatest } public func updateMessageHistoryViewVisibleRange(_ id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { - self.queue.justDispatch({ + self.modify({ modifier -> Void in self.viewTracker.updateMessageHistoryViewVisibleRange(id, earliestVisibleIndex: earliestVisibleIndex, latestVisibleIndex: latestVisibleIndex) - }) + }).start() } public func recentPeers() -> Signal<[Peer], NoError> { + return self.modify { modifier -> Signal<[Peer], NoError> in + let peerIds = self.peerRatingTable.get() + var peers: [Peer] = [] + for peerId in peerIds { + if let peer: Peer = self.peerTable.get(peerId) { + peers.append(peer) + } + } + return .single(peers) + } |> switchToLatest + } + + public func messageHistoryHolesView() -> Signal { return Signal { subscriber in let disposable = MetaDisposable() - self.queue.justDispatch({ - let peerIds = self.peerRatingTable.get() - var peers: [Peer] = [] - for peerId in peerIds { - if let peer: Peer = self.peerTable.get(peerId) { - peers.append(peer) - } - } - subscriber.putNext(peers) - }) + self.queue.async { + disposable.set(self.viewTracker.messageHistoryHolesViewSignal().start(next: { view in + subscriber.putNext(view) + })) + } return disposable } } + + public func chatListHolesView() -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.async { + disposable.set(self.viewTracker.chatListHolesViewSignal().start(next: { view in + subscriber.putNext(view) + })) + } + return disposable + } + } + + public func unsentMessageIndicesView() -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.async { + disposable.set(self.viewTracker.unsentMessageIndicesViewSignal().start(next: { view in + subscriber.putNext(view) + })) + } + return disposable + } + } + + public func synchronizePeerReadStatesView() -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.async { + disposable.set(self.viewTracker.synchronizePeerReadStatesViewSignal().start(next: { view in + subscriber.putNext(view) + })) + } + return disposable + } + } + + public func isMasterClient() -> Signal { + return self.modify { modifier -> Signal in + let sessionClientId = self.sessionClientId + return self.masterClientId.get() + |> distinctUntilChanged + |> map({ $0 == sessionClientId }) + } |> switchToLatest + } + + public func becomeMasterClient() { + self.modify({ modifier in + if self.metadataTable.masterClientId() != self.sessionClientId { + self.currentUpdatedMasterClientId = self.sessionClientId + } + }).start() + } } diff --git a/Postbox/PostboxClientId.swift b/Postbox/PostboxClientId.swift new file mode 100644 index 0000000000..5bfe785721 --- /dev/null +++ b/Postbox/PostboxClientId.swift @@ -0,0 +1,8 @@ + +public struct PostboxClientId { + public let stableId: Int32 + + public init(stableId: Int32) { + self.stableId = stableId + } +} diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift new file mode 100644 index 0000000000..764c343f5a --- /dev/null +++ b/Postbox/PostboxTransaction.swift @@ -0,0 +1,61 @@ +import Foundation + +final class PostboxTransaction { + let currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] + let peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]] + let removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] + let chatListOperations: [ChatListOperation] + let currentUpdatedPeers: [PeerId: Peer] + let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] + let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] + let updatedMedia: [MediaId: Media?] + let replaceContactPeerIds: Set? + let currentUpdatedMasterClientId: Int64? + + var isEmpty: Bool { + if !currentOperationsByPeerId.isEmpty { + return false + } + if !peerIdsWithFilledHoles.isEmpty { + return false + } + if !removedHolesByPeerId.isEmpty { + return false + } + if !chatListOperations.isEmpty { + return false + } + if !currentUpdatedPeers.isEmpty { + return false + } + if !unsentMessageOperations.isEmpty { + return false + } + if !updatedSynchronizePeerReadStateOperations.isEmpty { + return false + } + if !updatedMedia.isEmpty { + return false + } + if let replaceContactPeerIds = replaceContactPeerIds, !replaceContactPeerIds.isEmpty { + return false + } + if currentUpdatedMasterClientId != nil { + return false + } + return true + } + + init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + self.currentOperationsByPeerId = currentOperationsByPeerId + self.peerIdsWithFilledHoles = peerIdsWithFilledHoles + self.removedHolesByPeerId = removedHolesByPeerId + self.chatListOperations = chatListOperations + self.currentUpdatedPeers = currentUpdatedPeers + self.unsentMessageOperations = unsentMessageOperations + self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations + self.updatedMedia = updatedMedia + self.replaceContactPeerIds = replaceContactPeerIds + self.currentUpdatedMasterClientId = currentUpdatedMasterClientId + } +} diff --git a/Postbox/RatingTable.swift b/Postbox/RatingTable.swift index be80b7bd94..f91d1b08b7 100644 --- a/Postbox/RatingTable.swift +++ b/Postbox/RatingTable.swift @@ -70,6 +70,10 @@ final class RatingTable: Table { self.items = items } + override func clearMemoryCache() { + self.items = nil + } + override func beforeCommit() { } diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index cf61a25bd1..be1a8063c2 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -115,6 +115,7 @@ public final class SqliteValueBox: ValueBox { database.execute("PRAGMA temp_store=MEMORY") database.execute("PRAGMA wal_autocheckpoint=200") database.execute("PRAGMA journal_size_limit=1536") + sqlite3_busy_timeout(database.handle, 10000000) let result = self.getUserVersion(database) if result != 1 { @@ -151,7 +152,7 @@ public final class SqliteValueBox: ValueBox { } public func begin() { - self.database.execute("BEGIN") + self.database.execute("BEGIN IMMEDIATE") } public func commit() { diff --git a/Postbox/SynchronizePeerReadStatesView.swift b/Postbox/SynchronizePeerReadStatesView.swift index 21f48ed93b..af92a89b5d 100644 --- a/Postbox/SynchronizePeerReadStatesView.swift +++ b/Postbox/SynchronizePeerReadStatesView.swift @@ -1,29 +1,46 @@ import Foundation -final class SynchronizePeerReadStatesView { +final class MutableSynchronizePeerReadStatesView { var operations: [PeerId: PeerReadStateSynchronizationOperation] init(operations: [PeerId: PeerReadStateSynchronizationOperation]) { self.operations = operations } - func replay(_ updatedOperations: [PeerId: PeerReadStateSynchronizationOperation?]) -> [PeerId: PeerReadStateSynchronizationOperation?] { - var updates: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - + func refreshDueToExternalTransaction(fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) -> Bool { + let operations = fetchSynchronizePeerReadStateOperations() + if self.operations != operations { + self.operations = operations + return true + } else { + return false + } + } + + func replay(_ updatedOperations: [PeerId: PeerReadStateSynchronizationOperation?]) -> Bool { + var updated = false for (peerId, operation) in updatedOperations { if let operation = operation { if self.operations[peerId] == nil || self.operations[peerId]! != operation { self.operations[peerId] = operation - updates[peerId] = operation + updated = true } } else { if let _ = self.operations[peerId] { let _ = self.operations.removeValue(forKey: peerId) - updates[peerId] = nil + updated = true } } } - return updates + return updated + } +} + +public final class SynchronizePeerReadStatesView { + public let operations: [PeerId: PeerReadStateSynchronizationOperation] + + init(_ mutableView: MutableSynchronizePeerReadStatesView) { + self.operations = mutableView.operations } } diff --git a/Postbox/Table.swift b/Postbox/Table.swift index 7ec87e866d..08edb399ff 100644 --- a/Postbox/Table.swift +++ b/Postbox/Table.swift @@ -9,6 +9,9 @@ class Table { self.tableId = tableId } + func clearMemoryCache() { + } + func beforeCommit() { } } diff --git a/Postbox/UnsentMessageHistoryView.swift b/Postbox/UnsentMessageHistoryView.swift index 0ff6b2efd6..a6322bf1e7 100644 --- a/Postbox/UnsentMessageHistoryView.swift +++ b/Postbox/UnsentMessageHistoryView.swift @@ -1,10 +1,20 @@ import Foundation final class UnsentMessageHistoryView { - var indices: [MessageIndex] + var indices: Set init(indices: [MessageIndex]) { - self.indices = indices + self.indices = Set(indices) + } + + func refreshDueToExternalTransaction(fetchUnsendMessageIndices: () -> [MessageIndex]) -> Bool { + let indices = Set(fetchUnsendMessageIndices()) + if indices != self.indices { + self.indices = indices + return true + } else { + return false + } } func replay(_ operations: [IntermediateMessageHistoryUnsentOperation]) -> Bool { @@ -12,25 +22,14 @@ final class UnsentMessageHistoryView { for operation in operations { switch operation { case let .Insert(index): - var inserted = false - for i in 0 ..< self.indices.count { - if self.indices[i] > index { - self.indices.insert(index, at: i) - inserted = true - break - } + if !self.indices.contains(index) { + self.indices.insert(index) + updated = true } - if !inserted { - self.indices.append(index) - } - updated = true case let .Remove(index): - for i in 0 ..< self.indices.count { - if self.indices[i] == index { - self.indices.remove(at: i) - updated = true - break - } + if self.indices.contains(index) { + self.indices.remove(index) + updated = true } } } diff --git a/Postbox/UnsentMessageIndicesView.swift b/Postbox/UnsentMessageIndicesView.swift new file mode 100644 index 0000000000..e7047877f5 --- /dev/null +++ b/Postbox/UnsentMessageIndicesView.swift @@ -0,0 +1,9 @@ +import Foundation + +public final class UnsentMessageIndicesView { + public let indices: Set + + init(_ indices: Set) { + self.indices = indices + } +} diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 7e43a16bcd..f0a08804b8 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -17,24 +17,25 @@ final class ViewTracker { private let fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex? private let renderMessage: (IntermediateMessage) -> Message private let getPeer: (PeerId) -> Peer? - private let fetchChatListHole: (ChatListHole) -> Disposable - private let fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable - private let sendUnsentMessage: (MessageIndex) -> Disposable - private let synchronizeReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Disposable private var chatListViews = Bag<(MutableChatListView, ValuePipe<(ChatListView, ViewUpdateType)>)>() private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] - private var unsentMessageView: UnsentMessageHistoryView - private var synchronizeReadStatesView: SynchronizePeerReadStatesView private var contactPeerIdsViews = Bag<(MutableContactPeerIdsView, ValuePipe)>() private var contactPeersViews = Bag<(MutableContactPeersView, ValuePipe)>() - private var chatListHoleDisposables: [(ChatListHole, Disposable)] = [] - private var holeDisposablesByPeerId: [PeerId: [(MessageHistoryHole, Disposable)]] = [:] - private var unsentMessageDisposables: [MessageIndex: Disposable] = [:] - private var synchronizeReadStatesDisposables: [PeerId: (PeerReadStateSynchronizationOperation, Disposable)] = [:] + private let messageHistoryHolesView = MutableMessageHistoryHolesView() + private let messageHistoryHolesViewSubscribers = Bag>() - init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: (IntermediateMessage) -> Message, getPeer: (PeerId) -> Peer?, fetchChatListHole: (ChatListHole) -> Disposable, fetchMessageHistoryHole: (MessageHistoryHole, HoleFillDirection, MessageTags?) -> Disposable, sendUnsentMessage: (MessageIndex) -> Disposable, unsentMessageIndices: [MessageIndex], synchronizeReadState: (PeerId, PeerReadStateSynchronizationOperation) -> Disposable, synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + private let chatListHolesView = MutableChatListHolesView() + private let chatListHolesViewSubscribers = Bag>() + + private var unsentMessageView: UnsentMessageHistoryView + private let unsendMessageIndicesViewSubscribers = Bag>() + + private var synchronizeReadStatesView: MutableSynchronizePeerReadStatesView + private let synchronizePeerReadStatesViewSubscribers = Bag>() + + init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: (IntermediateMessage) -> Message, getPeer: (PeerId) -> Peer?, unsentMessageIndices: [MessageIndex], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -43,32 +44,9 @@ final class ViewTracker { self.fetchAnchorIndex = fetchAnchorIndex self.renderMessage = renderMessage self.getPeer = getPeer - self.fetchChatListHole = fetchChatListHole - self.fetchMessageHistoryHole = fetchMessageHistoryHole - self.sendUnsentMessage = sendUnsentMessage - self.synchronizeReadState = synchronizeReadState self.unsentMessageView = UnsentMessageHistoryView(indices: unsentMessageIndices) - self.synchronizeReadStatesView = SynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) - self.unsentViewUpdated() - - var initialPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - for (peerId, operation) in synchronizePeerReadStateOperations { - initialPeerReadStateOperations[peerId] = operation - } - self.synchronizeReadStateViewUpdated(initialPeerReadStateOperations) - } - - deinit { - for (_, disposable) in chatListHoleDisposables { - disposable.dispose() - } - - for (_, indexToDisposable) in holeDisposablesByPeerId { - for (_, disposable) in indexToDisposable { - disposable.dispose() - } - } + self.synchronizeReadStatesView = MutableSynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) } func addMessageHistoryView(_ peerId: PeerId, view: MutableMessageHistoryView) -> (Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>.Index, Signal<(MessageHistoryView, ViewUpdateType), NoError>) { @@ -164,26 +142,62 @@ final class ViewTracker { } } - func updateViews(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?) { + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (index: MessageIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (index: MessageIndex, count: Int, tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsendMessageIndices: () -> [MessageIndex], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { + var updateTrackedHolesPeerIds: [PeerId] = [] + + for (peerId, bag) in self.messageHistoryViews { + for (mutableView, pipe) in bag.copyItems() { + if mutableView.refreshDueToExternalTransaction(fetchAroundHistoryEntries: fetchAroundHistoryEntries) { + mutableView.incrementVersion() + + mutableView.render(self.renderMessage) + pipe.putNext((MessageHistoryView(mutableView), .Generic)) + + updateTrackedHolesPeerIds.append(peerId) + } + } + } + + for (mutableView, pipe) in self.chatListViews.copyItems() { + if mutableView.refreshDueToExternalTransaction(fetchAroundChatEntries: fetchAroundChatEntries) { + mutableView.render(self.renderMessage) + pipe.putNext((ChatListView(mutableView), .Generic)) + } + } + + for peerId in updateTrackedHolesPeerIds { + self.updateTrackedHoles(peerId) + } + + if self.unsentMessageView.refreshDueToExternalTransaction(fetchUnsendMessageIndices: fetchUnsendMessageIndices) { + self.unsentViewUpdated() + } + + if self.synchronizeReadStatesView.refreshDueToExternalTransaction(fetchSynchronizePeerReadStateOperations: fetchSynchronizePeerReadStateOperations) { + self.synchronizeReadStateViewUpdated() + } + } + + func updateViews(transaction: PostboxTransaction) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { var updateHoles = false - let operations = currentOperationsByPeerId[peerId] - if operations != nil || !updatedMedia.isEmpty { + let operations = transaction.currentOperationsByPeerId[peerId] + if operations != nil || !transaction.updatedMedia.isEmpty { updateHoles = true for (mutableView, pipe) in bag.copyItems() { let context = MutableMessageHistoryViewReplayContext() var updated = false let updateType: ViewUpdateType - if let filledIndices = peerIdsWithFilledHoles[peerId] { - updateType = .FillHole(insertions: filledIndices, deletions: removedHolesByPeerId[peerId] ?? [:]) + if let filledIndices = transaction.peerIdsWithFilledHoles[peerId] { + updateType = .FillHole(insertions: filledIndices, deletions: transaction.removedHolesByPeerId[peerId] ?? [:]) } else { updateType = .Generic } - if mutableView.replay(operations ?? [], holeFillDirections: peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: updatedMedia, context: context) { + if mutableView.replay(operations ?? [], holeFillDirections: transaction.peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: transaction.updatedMedia, context: context) { mutableView.complete(context: context, fetchEarlier: { index, count in return self.fetchEarlierHistoryEntries(peerId, index, count, mutableView.tagMask) }, fetchLater: { index, count in @@ -197,14 +211,13 @@ final class ViewTracker { updated = true } - if mutableView.updatePeers(currentUpdatedPeers) { + if mutableView.updatePeers(transaction.currentUpdatedPeers) { updated = true } if updated { mutableView.render(self.renderMessage) - pipe.putNext((MessageHistoryView(mutableView), updateType)) } } @@ -215,10 +228,10 @@ final class ViewTracker { } } - if chatListOperations.count != 0 { + if transaction.chatListOperations.count != 0 { for (mutableView, pipe) in self.chatListViews.copyItems() { let context = MutableChatListViewReplayContext() - if mutableView.replay(chatListOperations, context: context) { + if mutableView.replay(transaction.chatListOperations, context: context) { mutableView.complete(context: context, fetchEarlier: self.fetchEarlierChatEntries, fetchLater: self.fetchLaterChatEntries) mutableView.render(self.renderMessage) pipe.putNext((ChatListView(mutableView), .Generic)) @@ -232,16 +245,15 @@ final class ViewTracker { self.updateTrackedHoles(peerId) } - if self.unsentMessageView.replay(unsentMessageOperations) { + if self.unsentMessageView.replay(transaction.unsentMessageOperations) { self.unsentViewUpdated() } - let synchronizeReadStateUpdates = self.synchronizeReadStatesView.replay(updatedSynchronizePeerReadStateOperations) - if !synchronizeReadStateUpdates.isEmpty { - self.synchronizeReadStateViewUpdated(synchronizeReadStateUpdates) + if self.synchronizeReadStatesView.replay(transaction.updatedSynchronizePeerReadStateOperations) { + self.synchronizeReadStateViewUpdated() } - if let replaceContactPeerIds = replaceContactPeerIds { + if let replaceContactPeerIds = transaction.replaceContactPeerIds { for (mutableView, pipe) in self.contactPeerIdsViews.copyItems() { if mutableView.replay(replace: replaceContactPeerIds) { pipe.putNext(ContactPeerIdsView(mutableView)) @@ -257,168 +269,135 @@ final class ViewTracker { } private func updateTrackedChatListHoles() { - var disposeHoles: [Disposable] = [] - var firstHoles: [ChatListHole] = [] + var firstHoles = Set() for (view, _) in self.chatListViews.copyItems() { if let hole = view.firstHole() { - var exists = false - for existingHole in firstHoles { - if existingHole == hole { - exists = true - break - } - } - if !exists { - firstHoles.append(hole) - } + firstHoles.insert(hole) } } - var i = 0 - for (hole, disposable) in self.chatListHoleDisposables { - var exists = false - for firstHole in firstHoles { - if hole == firstHole { - exists = true - break - } - } - if !exists { - disposeHoles.append(disposable) - self.chatListHoleDisposables.remove(at: i) - } else { - i += 1 - } - } - - for disposable in disposeHoles { - disposable.dispose() - } - - for hole in firstHoles { - var exists = false - for (existingHole, _) in self.chatListHoleDisposables { - if existingHole == hole { - exists = true - break - } - } - - if !exists { - self.chatListHoleDisposables.append((hole, self.fetchChatListHole(hole))) + if self.chatListHolesView.update(holes: firstHoles) { + for pipe in self.chatListHolesViewSubscribers.copyItems() { + pipe.putNext(ChatListHolesView(self.chatListHolesView)) } } } private func updateTrackedHoles(_ peerId: PeerId) { + var firstHolesAndTags = Set() if let bag = self.messageHistoryViews[peerId] { - var disposeHoles: [Disposable] = [] - var firstHolesAndTags: [(MessageHistoryHole, HoleFillDirection, MessageTags?)] = [] - for (view, _) in bag.copyItems() { if let (hole, direction) = view.firstHole() { - firstHolesAndTags.append((hole, direction, view.tagMask)) + firstHolesAndTags.insert(MessageHistoryHolesViewEntry(hole: hole, direction: direction, tags: view.tagMask)) } } - - if let holes = self.holeDisposablesByPeerId[peerId] { - var i = 0 - for (hole, disposable) in holes { - var exists = false - for (firstHole, _, _) in firstHolesAndTags { - if hole == firstHole { - exists = true - break - } - } - if !exists { - disposeHoles.append(disposable) - self.holeDisposablesByPeerId[peerId]!.remove(at: i) - } else { - i += 1 - } - } - } - - for disposable in disposeHoles { - disposable.dispose() - } - - if let anyHoleAndTag = firstHolesAndTags.first { - if self.holeDisposablesByPeerId[peerId] == nil || self.holeDisposablesByPeerId[peerId]!.count == 0 { - var exists = false - if let existingHoles = self.holeDisposablesByPeerId[peerId] { - for (existingHole, _) in existingHoles { - if existingHole == anyHoleAndTag.0 { - exists = true - break - } - } - } - - if !exists { - if self.holeDisposablesByPeerId[peerId] == nil { - self.holeDisposablesByPeerId[peerId] = [] - } - - self.holeDisposablesByPeerId[peerId]!.append((anyHoleAndTag.0, self.fetchMessageHistoryHole(anyHoleAndTag.0, anyHoleAndTag.1, anyHoleAndTag.2))) - } - } - } - } else if let holes = self.holeDisposablesByPeerId[peerId] { - self.holeDisposablesByPeerId[peerId]?.removeAll() - - for (_, disposable) in holes { - disposable.dispose() + } + + if self.messageHistoryHolesView.update(peerId: peerId, holes: firstHolesAndTags) { + for subscriber in self.messageHistoryHolesViewSubscribers.copyItems() { + subscriber.putNext(MessageHistoryHolesView(self.messageHistoryHolesView)) } } } private func unsentViewUpdated() { - var removeIndices: [MessageIndex] = [] - for (index, _) in self.unsentMessageDisposables { - var found = false - for currentIndex in self.unsentMessageView.indices { - if currentIndex == index { - found = true - break - } - } - - if !found { - removeIndices.append(index) - } - } - - for index in removeIndices { - self.unsentMessageDisposables.removeValue(forKey: index)?.dispose() - } - - for index in self.unsentMessageView.indices { - var found = false - for (currentIndex, _) in unsentMessageDisposables { - if index == currentIndex { - found = true - break - } - } - - if !found { - self.unsentMessageDisposables[index] = self.sendUnsentMessage(index) - } + for subscriber in self.unsendMessageIndicesViewSubscribers.copyItems() { + subscriber.putNext(UnsentMessageIndicesView(self.unsentMessageView.indices)) } } - private func synchronizeReadStateViewUpdated(_ updates: [PeerId: PeerReadStateSynchronizationOperation?]) { - for (peerId, operation) in updates { - if let (_, disposable) = self.synchronizeReadStatesDisposables.removeValue(forKey: peerId) { - disposable.dispose() + private func synchronizeReadStateViewUpdated() { + for subscriber in self.synchronizePeerReadStatesViewSubscribers.copyItems() { + subscriber.putNext(SynchronizePeerReadStatesView(self.synchronizeReadStatesView)) + } + } + + func messageHistoryHolesViewSignal() -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.async { + subscriber.putNext(MessageHistoryHolesView(self.messageHistoryHolesView)) + + let pipe = ValuePipe() + let index = self.messageHistoryHolesViewSubscribers.add(pipe) + + let pipeDisposable = pipe.signal().start(next: { view in + subscriber.putNext(view) + }) + + disposable.set(ActionDisposable { + pipeDisposable.dispose() + self.messageHistoryHolesViewSubscribers.remove(index) + }) } - - if let operation = operation { - self.synchronizeReadStatesDisposables[peerId] = (operation, self.synchronizeReadState(peerId, operation)) + return disposable + } + } + + func chatListHolesViewSignal() -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.async { + subscriber.putNext(ChatListHolesView(self.chatListHolesView)) + + let pipe = ValuePipe() + let index = self.chatListHolesViewSubscribers.add(pipe) + + let pipeDisposable = pipe.signal().start(next: { view in + subscriber.putNext(view) + }) + + disposable.set(ActionDisposable { + pipeDisposable.dispose() + self.chatListHolesViewSubscribers.remove(index) + }) } + return disposable + } + } + + func unsentMessageIndicesViewSignal() -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.async { + subscriber.putNext(UnsentMessageIndicesView(self.unsentMessageView.indices)) + + let pipe = ValuePipe() + let index = self.unsendMessageIndicesViewSubscribers.add(pipe) + + let pipeDisposable = pipe.signal().start(next: { view in + subscriber.putNext(view) + }) + + disposable.set(ActionDisposable { + pipeDisposable.dispose() + self.unsendMessageIndicesViewSubscribers.remove(index) + }) + } + return disposable + } + } + + func synchronizePeerReadStatesViewSignal() -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.queue.async { + subscriber.putNext(SynchronizePeerReadStatesView(self.synchronizeReadStatesView)) + + let pipe = ValuePipe() + let index = self.synchronizePeerReadStatesViewSubscribers.add(pipe) + + let pipeDisposable = pipe.signal().start(next: { view in + subscriber.putNext(view) + }) + + disposable.set(ActionDisposable { + pipeDisposable.dispose() + self.synchronizePeerReadStatesViewSubscribers.remove(index) + }) + } + return disposable } } } From efe9ee147d1746c7b1935b569756e36843de81da Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 23 Aug 2016 16:19:22 +0300 Subject: [PATCH 040/427] no message --- Postbox.xcodeproj/project.pbxproj | 28 - .../xcschemes/xcschememanagement.plist | 2 +- Postbox/ChatListHolesView.swift | 2 +- Postbox/ChatListTable.swift | 2 +- Postbox/ChatListView.swift | 14 +- Postbox/Coding.swift | 78 +- Postbox/ContactPeerIdsView.swift | 2 +- Postbox/ContactPeersView.swift | 8 +- Postbox/IpcNotifier.h | 2 +- Postbox/IpcNotifier.mm | 2 +- Postbox/KeychainTable.swift | 2 +- Postbox/LmdbValueBox.swift | 451 - Postbox/MappedFile.swift | 2 +- Postbox/MediaBox.swift | 8 +- Postbox/Message.swift | 2 +- Postbox/MessageHistoryHolesView.swift | 2 +- Postbox/MessageHistoryIndexTable.swift | 12 +- Postbox/MessageHistoryMetadataTable.swift | 2 +- Postbox/MessageHistoryReadStateTable.swift | 6 +- Postbox/MessageHistoryTable.swift | 20 +- Postbox/MessageHistoryView.swift | 46 +- Postbox/PeerChatStateTable.swift | 4 +- Postbox/Postbox.swift | 63 +- Postbox/PostboxPrivate/module.modulemap | 1 - .../RandomAccessMediaResourceContext.swift | 16 +- Postbox/SqliteValueBox.swift | 18 +- Postbox/ValueBoxKey.swift | 18 +- Postbox/ViewTracker.swift | 4 +- submodules/lmdb/libraries/liblmdb/lmdb.h | 1565 --- submodules/lmdb/libraries/liblmdb/mdb.c | 9519 ----------------- submodules/lmdb/libraries/liblmdb/midl.c | 360 - submodules/lmdb/libraries/liblmdb/midl.h | 186 - 32 files changed, 174 insertions(+), 12273 deletions(-) delete mode 100644 Postbox/LmdbValueBox.swift delete mode 100644 submodules/lmdb/libraries/liblmdb/lmdb.h delete mode 100644 submodules/lmdb/libraries/liblmdb/mdb.c delete mode 100644 submodules/lmdb/libraries/liblmdb/midl.c delete mode 100644 submodules/lmdb/libraries/liblmdb/midl.h diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 4e46330ecb..0cd49c3d09 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -12,11 +12,6 @@ D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */; }; D0079F651D5A457A00A27A2C /* ContactPeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F641D5A457A00A27A2C /* ContactPeersView.swift */; }; D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; - D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB41B85D192002E4EB5 /* lmdb.h */; }; - D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-unreachable-code -Wno-conditional-uninitialized -Wno-format-extra-args -Wno-macro-redefined"; }; }; - D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB61B85D192002E4EB5 /* midl.c */; settings = {COMPILER_FLAGS = "-Wno-conversion"; }; }; - D00E0FBB1B85D192002E4EB5 /* midl.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB71B85D192002E4EB5 /* midl.h */; }; - D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */; }; D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; @@ -109,11 +104,6 @@ D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactPeerIdsView.swift; sourceTree = ""; }; D0079F641D5A457A00A27A2C /* ContactPeersView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactPeersView.swift; sourceTree = ""; }; D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameIndexRepresentation.swift; sourceTree = ""; }; - D00E0FB41B85D192002E4EB5 /* lmdb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lmdb.h; path = submodules/lmdb/libraries/liblmdb/lmdb.h; sourceTree = SOURCE_ROOT; }; - D00E0FB51B85D192002E4EB5 /* mdb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = mdb.c; path = submodules/lmdb/libraries/liblmdb/mdb.c; sourceTree = SOURCE_ROOT; }; - D00E0FB61B85D192002E4EB5 /* midl.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = midl.c; path = submodules/lmdb/libraries/liblmdb/midl.c; sourceTree = SOURCE_ROOT; }; - D00E0FB71B85D192002E4EB5 /* midl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = midl.h; path = submodules/lmdb/libraries/liblmdb/midl.h; sourceTree = SOURCE_ROOT; }; - D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LmdbValueBox.swift; sourceTree = ""; }; D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; @@ -214,17 +204,6 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ - D00E0FBC1B85D194002E4EB5 /* lmdb */ = { - isa = PBXGroup; - children = ( - D00E0FB41B85D192002E4EB5 /* lmdb.h */, - D00E0FB51B85D192002E4EB5 /* mdb.c */, - D00E0FB61B85D192002E4EB5 /* midl.c */, - D00E0FB71B85D192002E4EB5 /* midl.h */, - ); - name = lmdb; - sourceTree = ""; - }; D05F09981C9CAC1100BB6F96 /* Media Box */ = { isa = PBXGroup; children = ( @@ -240,7 +219,6 @@ D07515FC1B2C44A200AE42E0 /* thirdparty */ = { isa = PBXGroup; children = ( - D00E0FBC1B85D194002E4EB5 /* lmdb */, D07516731B2EC8C700AE42E0 /* sqlite.swift */, ); name = thirdparty; @@ -337,7 +315,6 @@ D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */, D0977F9B1B822DB4009994B2 /* ValueBox.swift */, D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */, - D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */, ); name = "Value Box"; sourceTree = ""; @@ -443,9 +420,7 @@ isa = PBXHeadersBuildPhase; buildActionMask = 2147483647; files = ( - D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */, D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */, - D00E0FBB1B85D192002E4EB5 /* midl.h in Headers */, D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */, D0D511041D64D91C00A97B8A /* IpcNotifier.h in Headers */, D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */, @@ -565,7 +540,6 @@ D08CEFB41D2AD8BE0015D3BC /* RedBlackTree.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, - D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */, D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */, D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */, @@ -583,7 +557,6 @@ D0C9DA391C65782500855278 /* SimpleSet.swift in Sources */, D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */, - D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */, D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */, @@ -593,7 +566,6 @@ D0AB0B8E1D65D49C002C78E7 /* ChatListHolesView.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, - D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */, D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist index cc1adcd6c7..9eb93eab1b 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist @@ -12,7 +12,7 @@ PostboxTests.xcscheme orderHint - 5 + 4 SuppressBuildableAutocreation diff --git a/Postbox/ChatListHolesView.swift b/Postbox/ChatListHolesView.swift index 21b256fc4e..70c6d9d360 100644 --- a/Postbox/ChatListHolesView.swift +++ b/Postbox/ChatListHolesView.swift @@ -1,7 +1,7 @@ import Foundation final class MutableChatListHolesView { - private var entries = Set() + fileprivate var entries = Set() func update(holes: Set) -> Bool { if self.entries != holes { diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index 54692912a6..19915e1233 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -83,7 +83,7 @@ final class ChatListTable: Table { if let topMessage = messageHistoryTable.topMessage(peerId) { let index = MessageIndex(id: topMessage.id, timestamp: topMessage.timestamp) - if let currentIndex = currentIndex where currentIndex != index { + if let currentIndex = currentIndex , currentIndex != index { self.justRemoveMessage(currentIndex) } if let currentIndex = currentIndex { diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index 077deba59e..165353f36a 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -101,9 +101,9 @@ final class MutableChatListViewReplayContext { } final class MutableChatListView { - private var earlier: MutableChatListEntry? - private var later: MutableChatListEntry? - private var entries: [MutableChatListEntry] + fileprivate var earlier: MutableChatListEntry? + fileprivate var later: MutableChatListEntry? + fileprivate var entries: [MutableChatListEntry] private var count: Int init(earlier: MutableChatListEntry?, entries: [MutableChatListEntry], later: MutableChatListEntry?, count: Int) { @@ -113,13 +113,13 @@ final class MutableChatListView { self.count = count } - func refreshDueToExternalTransaction(fetchAroundChatEntries: (index: MessageIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?)) -> Bool { + func refreshDueToExternalTransaction(fetchAroundChatEntries: (_ index: MessageIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?)) -> Bool { var index = MessageIndex.absoluteUpperBound() if !self.entries.isEmpty { index = self.entries[self.entries.count / 2].index } - var (entries, earlier, later) = fetchAroundChatEntries(index: index, count: self.entries.count) + var (entries, earlier, later) = fetchAroundChatEntries(index, self.entries.count) if entries != self.entries || earlier != self.earlier || later != self.later { self.entries = entries @@ -223,7 +223,7 @@ final class MutableChatListView { func remove(_ indices: Set, holes: Bool, context: MutableChatListViewReplayContext) -> Bool { var hasChanges = false - if let earlier = self.earlier where indices.contains(earlier.index) { + if let earlier = self.earlier , indices.contains(earlier.index) { var match = false switch earlier { case .HoleEntry: @@ -237,7 +237,7 @@ final class MutableChatListView { } } - if let later = self.later where indices.contains(later.index) { + if let later = self.later , indices.contains(later.index) { var match = false switch later { case .HoleEntry: diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index c03cfacd1c..bda4d11f7d 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -22,7 +22,7 @@ private let typeStore = { () -> EncodableTypeStore in return _typeStore }() -public func declareEncodable(_ type: Any.Type, f: (Decoder) -> Coding) { +public func declareEncodable(_ type: Any.Type, f: @escaping(Decoder) -> Coding) { let string = "\(type)" let hash = murMurHashString32(string) if typeStore.dict[hash] != nil { @@ -34,7 +34,7 @@ public func declareEncodable(_ type: Any.Type, f: (Decoder) -> Coding) { private let emptyMemory = malloc(1)! public class MemoryBuffer: Equatable, CustomStringConvertible { - var memory: UnsafeMutablePointer + var memory: UnsafeMutableRawPointer var capacity: Int var length: Int var freeWhenDone: Bool @@ -47,7 +47,7 @@ public class MemoryBuffer: Equatable, CustomStringConvertible { self.freeWhenDone = true } - public init(memory: UnsafeMutablePointer, capacity: Int, length: Int, freeWhenDone: Bool) { + public init(memory: UnsafeMutableRawPointer, capacity: Int, length: Int, freeWhenDone: Bool) { self.memory = memory self.capacity = capacity self.length = length @@ -62,7 +62,7 @@ public class MemoryBuffer: Equatable, CustomStringConvertible { self.freeWhenDone = false } else { self.memory = malloc(data.count)! - data.copyBytes(to: UnsafeMutablePointer(self.memory), count: data.count) + data.copyBytes(to: self.memory.assumingMemoryBound(to: UInt8.self), count: data.count) self.capacity = data.count self.length = data.count self.freeWhenDone = false @@ -84,7 +84,7 @@ public class MemoryBuffer: Equatable, CustomStringConvertible { public var description: String { let hexString = NSMutableString() - let bytes = UnsafeMutablePointer(self.memory) + let bytes = self.memory.assumingMemoryBound(to: UInt8.self) for i in 0 ..< self.length { hexString.appendFormat("%02x", UInt(bytes[i])) } @@ -117,14 +117,14 @@ public final class WriteBuffer: MemoryBuffer { } public func makeData() -> Data { - return Data(bytes: UnsafePointer(self.memory), count: self.offset) + return Data(bytes: self.memory.assumingMemoryBound(to: UInt8.self), count: self.offset) } public func reset() { self.offset = 0 } - public func write(_ data: UnsafePointer, offset: Int, length: Int) { + public func write(_ data: UnsafeRawPointer, offset: Int, length: Int) { if self.offset + length > self.capacity { self.capacity = self.offset + length + 256 if self.length == 0 { @@ -148,7 +148,7 @@ public final class WriteBuffer: MemoryBuffer { self.memory = realloc(self.memory, self.capacity) } } - data.copyBytes(to: UnsafeMutablePointer(self.memory + offset), count: length) + data.copyBytes(to: self.memory.advanced(by: offset).assumingMemoryBound(to: UInt8.self), count: length) self.offset += length self.length = self.offset } @@ -157,7 +157,7 @@ public final class WriteBuffer: MemoryBuffer { public final class ReadBuffer: MemoryBuffer { public var offset = 0 - public init(memory: UnsafeMutablePointer, length: Int, freeWhenDone: Bool) { + public init(memory: UnsafeMutableRawPointer, length: Int, freeWhenDone: Bool) { super.init(memory: memory, capacity: length, length: length, freeWhenDone: freeWhenDone) } @@ -166,11 +166,11 @@ public final class ReadBuffer: MemoryBuffer { } public func dataNoCopy() -> Data { - return Data(bytesNoCopy: UnsafeMutablePointer(self.memory), count: self.length, deallocator: .none) + return Data(bytesNoCopy: self.memory.assumingMemoryBound(to: UInt8.self), count: self.length, deallocator: .none) } public func read(_ data: UnsafeMutablePointer, offset: Int, length: Int) { - memcpy(data + offset, self.memory + self.offset, length) + memcpy(data + offset, self.memory.advanced(by: self.offset), length) self.offset += length } @@ -284,7 +284,7 @@ public final class Encoder { var type: Int8 = ValueType.Object.rawValue self.buffer.write(&type, offset: 0, length: 1) - let string = "\(value.dynamicType)" + let string = "\(type(of: value))" var typeHash: Int32 = murMurHashString32(string) self.buffer.write(&typeHash, offset: 0, length: 4) @@ -328,7 +328,7 @@ public final class Encoder { self.buffer.write(&length, offset: 0, length: 4) let innerEncoder = Encoder() for object in value { - var typeHash: Int32 = murMurHashString32("\(object.dynamicType)") + var typeHash: Int32 = murMurHashString32("\(type(of: object))") self.buffer.write(&typeHash, offset: 0, length: 4) innerEncoder.reset() @@ -348,7 +348,7 @@ public final class Encoder { self.buffer.write(&length, offset: 0, length: 4) let innerEncoder = Encoder() for object in value { - var typeHash: Int32 = murMurHashString32("\(object.dynamicType)") + var typeHash: Int32 = murMurHashString32("\(type(of: object))") self.buffer.write(&typeHash, offset: 0, length: 4) innerEncoder.reset() @@ -369,7 +369,7 @@ public final class Encoder { let innerEncoder = Encoder() for record in value { - var keyTypeHash: Int32 = murMurHashString32("\(record.0.dynamicType)") + var keyTypeHash: Int32 = murMurHashString32("\(type(of: record.0))") self.buffer.write(&keyTypeHash, offset: 0, length: 4) innerEncoder.reset() record.0.encode(innerEncoder) @@ -377,7 +377,7 @@ public final class Encoder { self.buffer.write(&keyLength, offset: 0, length: 4) self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(keyLength)) - var valueTypeHash: Int32 = murMurHashString32("\(record.1.dynamicType)") + var valueTypeHash: Int32 = murMurHashString32("\(type(of: record.1))") self.buffer.write(&valueTypeHash, offset: 0, length: 4) innerEncoder.reset() record.1.encode(innerEncoder) @@ -484,8 +484,10 @@ public final class Decoder { } } - private class func positionOnKey(_ bytes: UnsafePointer, offset: inout Int, maxOffset: Int, length: Int, key: StaticString, valueType: ValueType) -> Bool + private class func positionOnKey(_ rawBytes: UnsafeRawPointer, offset: inout Int, maxOffset: Int, length: Int, key: StaticString, valueType: ValueType) -> Bool { + let bytes = rawBytes.assumingMemoryBound(to: Int8.self) + let startOffset = offset let keyLength: Int = key.utf8CodeUnitCount @@ -546,7 +548,7 @@ public final class Decoder { } public func decodeInt32ForKey(_ key: StaticString) -> Int32 { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 memcpy(&value, self.buffer.memory + self.offset, 4) self.offset += 4 @@ -557,7 +559,7 @@ public final class Decoder { } public func decodeInt32ForKey(_ key: StaticString) -> Int32? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 memcpy(&value, self.buffer.memory + self.offset, 4) self.offset += 4 @@ -568,7 +570,7 @@ public final class Decoder { } public func decodeInt64ForKey(_ key: StaticString) -> Int64 { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { var value: Int64 = 0 memcpy(&value, self.buffer.memory + self.offset, 8) self.offset += 8 @@ -579,7 +581,7 @@ public final class Decoder { } public func decodeInt64ForKey(_ key: StaticString) -> Int64? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { var value: Int64 = 0 memcpy(&value, self.buffer.memory + self.offset, 8) self.offset += 8 @@ -590,7 +592,7 @@ public final class Decoder { } public func decodeBoolForKey(_ key: StaticString) -> Bool { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bool) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bool) { var value: Int8 = 0 memcpy(&value, self.buffer.memory + self.offset, 1) self.offset += 1 @@ -601,7 +603,7 @@ public final class Decoder { } public func decodeDoubleForKey(_ key: StaticString) -> Double { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Double) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Double) { var value: Double = 0 memcpy(&value, self.buffer.memory + self.offset, 8) self.offset += 8 @@ -612,10 +614,10 @@ public final class Decoder { } public func decodeStringForKey(_ key: StaticString) -> String { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) - let data = Data(bytes: UnsafeMutablePointer(self.buffer.memory).advanced(by: self.offset + 4), count: Int(length)) + let data = Data(bytes: self.buffer.memory.assumingMemoryBound(to: UInt8.self).advanced(by: self.offset + 4), count: Int(length)) self.offset += 4 + Int(length) return String(data: data, encoding: .utf8) ?? "" } else { @@ -624,10 +626,10 @@ public final class Decoder { } public func decodeStringForKey(_ key: StaticString) -> String? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) - let data = Data(bytes: UnsafeMutablePointer(self.buffer.memory).advanced(by: self.offset + 4), count: Int(length)) + let data = Data(bytes: self.buffer.memory.assumingMemoryBound(to: UInt8.self).advanced(by: self.offset + 4), count: Int(length)) self.offset += 4 + Int(length) return String(data: data, encoding: .utf8) } else { @@ -640,7 +642,7 @@ public final class Decoder { } public func decodeObjectForKey(_ key: StaticString) -> Coding? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { var typeHash: Int32 = 0 memcpy(&typeHash, self.buffer.memory + self.offset, 4) self.offset += 4 @@ -658,7 +660,7 @@ public final class Decoder { } public func decodeObjectForKey(_ key: StaticString, decoder: (Decoder) -> Coding) -> Coding? { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { var typeHash: Int32 = 0 memcpy(&typeHash, self.buffer.memory + self.offset, 4) self.offset += 4 @@ -676,7 +678,7 @@ public final class Decoder { } public func decodeInt32ArrayForKey(_ key: StaticString) -> [Int32] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32Array) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32Array) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) var array: [Int32] = [] @@ -696,7 +698,7 @@ public final class Decoder { } public func decodeInt64ArrayForKey(_ key: StaticString) -> [Int64] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64Array) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64Array) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) var array: [Int64] = [] @@ -716,7 +718,7 @@ public final class Decoder { } public func decodeObjectArrayWithDecoderForKey(_ key: StaticString) -> [T] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) self.offset += 4 @@ -755,7 +757,7 @@ public final class Decoder { } public func decodeObjectArrayForKey(_ key: StaticString) -> [T] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) self.offset += 4 @@ -798,7 +800,7 @@ public final class Decoder { } public func decodeObjectArrayForKey(_ key: StaticString) -> [Coding] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) self.offset += 4 @@ -841,7 +843,7 @@ public final class Decoder { } public func decodeObjectDictionaryForKey(_ key: StaticString) -> [K : V] { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) self.offset += 4 @@ -875,7 +877,7 @@ public final class Decoder { let value = failed ? nil : (typeStore.decode(valueHash, decoder: innerDecoder) as? V) - if let key = key, value = value { + if let key = key, let value = value { dictionary[key] = value } else { failed = true @@ -895,11 +897,11 @@ public final class Decoder { } public func decodeBytesForKeyNoCopy(_ key: StaticString) -> ReadBuffer! { - if Decoder.positionOnKey(UnsafePointer(self.buffer.memory), offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bytes) { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bytes) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) self.offset += 4 + Int(length) - return ReadBuffer(memory: UnsafeMutablePointer(self.buffer.memory + (self.offset - Int(length))), length: Int(length), freeWhenDone: false) + return ReadBuffer(memory: self.buffer.memory.advanced(by: self.offset - Int(length)), length: Int(length), freeWhenDone: false) } else { return nil } diff --git a/Postbox/ContactPeerIdsView.swift b/Postbox/ContactPeerIdsView.swift index 501e3fc56a..28661aea0c 100644 --- a/Postbox/ContactPeerIdsView.swift +++ b/Postbox/ContactPeerIdsView.swift @@ -1,7 +1,7 @@ import Foundation final class MutableContactPeerIdsView { - private var peerIds: Set + fileprivate var peerIds: Set init(peerIds: Set) { self.peerIds = peerIds diff --git a/Postbox/ContactPeersView.swift b/Postbox/ContactPeersView.swift index b8a9257b73..42410462c9 100644 --- a/Postbox/ContactPeersView.swift +++ b/Postbox/ContactPeersView.swift @@ -1,10 +1,10 @@ import Foundation final class MutableContactPeersView { - private let index: PeerNameIndex - private var peers: [PeerId: Peer] - private var peerIds: Set - private var accountPeer: Peer? + fileprivate let index: PeerNameIndex + fileprivate var peers: [PeerId: Peer] + fileprivate var peerIds: Set + fileprivate var accountPeer: Peer? init(peers: [PeerId: Peer], index: PeerNameIndex, accountPeer: Peer?) { self.index = index diff --git a/Postbox/IpcNotifier.h b/Postbox/IpcNotifier.h index 09f09edb4f..a7f65b9b16 100644 --- a/Postbox/IpcNotifier.h +++ b/Postbox/IpcNotifier.h @@ -5,7 +5,7 @@ @interface RLMNotifier : NSObject -- (instancetype)initWithBasePath:(NSString *)basePath notify:(void (^)())notify; +- (instancetype _Nonnull)initWithBasePath:(NSString * _Nonnull)basePath notify:(void (^ _Nonnull)())notify; - (void)listen; - (void)notifyOtherRealms; diff --git a/Postbox/IpcNotifier.mm b/Postbox/IpcNotifier.mm index 60ec850f62..62a17a6326 100644 --- a/Postbox/IpcNotifier.mm +++ b/Postbox/IpcNotifier.mm @@ -94,7 +94,7 @@ namespace { FdHolder _shutdownWriteFd; } -- (instancetype)initWithBasePath:(NSString *)basePath notify:(void (^)())notify { +- (instancetype)initWithBasePath:(NSString *)basePath notify:( void (^ _Nonnull)())notify { self = [super init]; if (self) { _notify = [notify copy]; diff --git a/Postbox/KeychainTable.swift b/Postbox/KeychainTable.swift index 82befd5759..451c0e52f3 100644 --- a/Postbox/KeychainTable.swift +++ b/Postbox/KeychainTable.swift @@ -11,7 +11,7 @@ final class KeychainTable: Table { func get(_ key: String) -> Data? { if let value = self.valueBox.get(self.tableId, key: self.key(key)) { - return Data(bytes: UnsafePointer(value.memory), count: value.length) + return Data(bytes: value.memory.assumingMemoryBound(to: UInt8.self), count: value.length) } return nil } diff --git a/Postbox/LmdbValueBox.swift b/Postbox/LmdbValueBox.swift deleted file mode 100644 index 67263c1c05..0000000000 --- a/Postbox/LmdbValueBox.swift +++ /dev/null @@ -1,451 +0,0 @@ -import Foundation -import sqlcipher - -private struct LmdbTable { - var dbi: MDB_dbi -} - -private struct LmdbCursor { - var cursor: OpaquePointer! - - func seekTo(_ key: ValueBoxKey, forward: Bool) -> (ValueBoxKey, ReadBuffer)? { - var mdbKey = MDB_val() - var mdbData = MDB_val() - - mdbKey.mv_data = key.memory - mdbKey.mv_size = key.length - mdbData.mv_data = nil - mdbData.mv_size = 0 - - let result = mdb_cursor_get(self.cursor, &mdbKey, &mdbData, forward ? MDB_SET_RANGE : MDB_SET_KEY) - - if result == MDB_SUCCESS { - let actualKey = ValueBoxKey(length: mdbKey.mv_size) - memcpy(actualKey.memory, mdbKey.mv_data, mdbKey.mv_size) - - let value = malloc(mdbData.mv_size)! - memcpy(value, mdbData.mv_data, mdbData.mv_size) - - return (actualKey, ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true)) - } else if result == MDB_NOTFOUND { - if !forward { - return self.previous() - } else { - return nil - } - } else { - print("(LmdbValueBox mdb_cursor_get failed with \(result))") - return nil - } - } - - func previous() -> (ValueBoxKey, ReadBuffer)? { - var mdbKey = MDB_val() - var mdbData = MDB_val() - - mdbKey.mv_data = nil - mdbKey.mv_size = 0 - mdbData.mv_data = nil - mdbData.mv_size = 0 - - let result = mdb_cursor_get(self.cursor, &mdbKey, &mdbData, MDB_PREV) - if result == MDB_SUCCESS { - let actualKey = ValueBoxKey(length: mdbKey.mv_size) - memcpy(actualKey.memory, mdbKey.mv_data, mdbKey.mv_size) - - let value = malloc(mdbData.mv_size)! - memcpy(value, mdbData.mv_data, mdbData.mv_size) - - return (actualKey, ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true)) - } else if result == MDB_NOTFOUND { - return nil - } else { - print("(LmdbValueBox mdb_cursor_get failed with \(result))") - return nil - } - } - - func next() -> (ValueBoxKey, ReadBuffer)? { - var mdbKey = MDB_val() - var mdbData = MDB_val() - - mdbKey.mv_data = nil - mdbKey.mv_size = 0 - mdbData.mv_data = nil - mdbData.mv_size = 0 - - let result = mdb_cursor_get(self.cursor, &mdbKey, &mdbData, MDB_NEXT) - if result == MDB_SUCCESS { - let actualKey = ValueBoxKey(length: mdbKey.mv_size) - memcpy(actualKey.memory, mdbKey.mv_data, mdbKey.mv_size) - - let value = malloc(mdbData.mv_size)! - memcpy(value, mdbData.mv_data, mdbData.mv_size) - - return (actualKey, ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true)) - } else if result == MDB_NOTFOUND { - return nil - } else { - print("(LmdbValueBox mdb_cursor_get failed with \(result))") - return nil - } - } -} - -public final class LmdbValueBox: ValueBox { - private var env: OpaquePointer? = nil - private var tables: [Int32 : LmdbTable] = [:] - - private var sharedTxn: OpaquePointer? = nil - - private var readQueryTime: CFAbsoluteTime = 0.0 - private var writeQueryTime: CFAbsoluteTime = 0.0 - private var commitTime: CFAbsoluteTime = 0.0 - - public init?(basePath: String) { - var result = mdb_env_create(&self.env) - if result != MDB_SUCCESS { - print("(LmdbValueBox mdb_env_create failed with \(result))") - return nil - } - - let path = basePath + "/lmdb" - - var createDirectory = false - var isDirectory: ObjCBool = false as ObjCBool - if FileManager.default.fileExists(atPath: path, isDirectory: &isDirectory) { - if !isDirectory.boolValue { - do { - try FileManager.default.removeItem(atPath: path) - } catch _ { } - createDirectory = true - } - } - else { - createDirectory = true - } - - if createDirectory { - do { - try FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil) - } catch _ { } - } - - mdb_env_set_mapsize(self.env, 500 * 1024 * 1024); - mdb_env_set_maxdbs(self.env, 64) - - path.withCString { string in - result = mdb_env_open(self.env, string, UInt32(MDB_NOSYNC), 0o664) - } - if result != MDB_SUCCESS { - print("(LmdbValueBox mdb_env_open failed with \(result))") - return nil - } - - var removedReaders: Int32 = 0 - result = mdb_reader_check(self.env, &removedReaders) - - if removedReaders != 0 { - print("(LmdbValueBox removed \(removedReaders) stale readers)") - } - } - - deinit { - mdb_env_close(self.env) - } - - private func createTableWithName(_ name: Int32) -> LmdbTable? { - var dbi = MDB_dbi() - let result = mdb_dbi_open(self.sharedTxn, "\(name)", UInt32(MDB_CREATE), &dbi) - - if result != MDB_SUCCESS { - print("(LmdbValueBox mdb_dbi_open failed with \(result))") - - return nil - } - - return LmdbTable(dbi: dbi) - } - - public func beginStats() { - self.readQueryTime = 0.0 - self.writeQueryTime = 0.0 - self.commitTime = 0.0 - } - - public func endStats() { - print("(LmdbValueBox stats read: \(self.readQueryTime * 1000.0) ms, write: \(self.writeQueryTime * 1000.0) ms, commit: \(self.commitTime * 1000.0) ms") - } - - public func begin() { - if self.sharedTxn != nil { - print("(LmdbValueBox already in transaction)") - } else { - let result = mdb_txn_begin(self.env, nil, 0, &sharedTxn) - if result != MDB_SUCCESS { - print("(LmdbValueBox txn_begin failed with \(result))") - return - } - } - } - - public func commit() { - let startTime = CFAbsoluteTimeGetCurrent() - if self.sharedTxn == nil { - print("(LmdbValueBox already no current transaction)") - } else { - let result = mdb_txn_commit(self.sharedTxn) - self.sharedTxn = nil - self.commitTime += CFAbsoluteTimeGetCurrent() - startTime - if result != MDB_SUCCESS { - print("(LmdbValueBox txn_commit failed with \(result))") - return - } - } - } - - public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, values: @noescape(ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { - if start == end || limit == 0 { - return - } - - var commit = false - if self.sharedTxn == nil { - self.begin() - commit = true - } - - var nativeTable: LmdbTable? - if let existingTable = self.tables[table] { - nativeTable = existingTable - } else if let createdTable = self.createTableWithName(table) { - nativeTable = createdTable - self.tables[table] = createdTable - } - - if let nativeTable = nativeTable { - var startTime = CFAbsoluteTimeGetCurrent() - var cursorPtr: OpaquePointer? = nil - let result = mdb_cursor_open(self.sharedTxn, nativeTable.dbi, &cursorPtr) - if result != MDB_SUCCESS { - print("(LmdbValueBox mdb_cursor_open failed with \(result))") - } else { - let cursor = LmdbCursor(cursor: cursorPtr) - - if start < end { - var value = cursor.seekTo(start, forward: true) - if value != nil { - if value!.0 == start { - value = cursor.next() - } - } - - var currentTime = CFAbsoluteTimeGetCurrent() - readQueryTime += currentTime - startTime - startTime = currentTime - - var count = 0 - if value != nil && value!.0 < end { - count += 1 - let _ = values(value!.0, value!.1) - } - - while value != nil && value!.0 < end && count < limit { - startTime = CFAbsoluteTimeGetCurrent() - - value = cursor.next() - - currentTime = CFAbsoluteTimeGetCurrent() - readQueryTime += currentTime - startTime - startTime = currentTime - - if value != nil && value!.0 < end { - count += 1 - let _ = values(value!.0, value!.1) - } - } - } else { - var startTime = CFAbsoluteTimeGetCurrent() - var value = cursor.seekTo(start, forward: false) - if value != nil { - if value!.0 == start { - value = cursor.previous() - } - } - - var currentTime = CFAbsoluteTimeGetCurrent() - readQueryTime += currentTime - startTime - startTime = currentTime - - var count = 0 - if value != nil && value!.0 > end { - count += 1 - let _ = values(value!.0, value!.1) - } - - while value != nil && value!.0 > end && count < limit { - startTime = CFAbsoluteTimeGetCurrent() - - value = cursor.previous() - - currentTime = CFAbsoluteTimeGetCurrent() - readQueryTime += currentTime - startTime - startTime = currentTime - - if value != nil && value!.0 > end { - count += 1 - let _ = values(value!.0, value!.1) - } - } - } - - mdb_cursor_close(cursorPtr) - } - } - - if commit { - let startTime = CFAbsoluteTimeGetCurrent() - - self.commit() - - readQueryTime += CFAbsoluteTimeGetCurrent() - startTime - } - } - - public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: @noescape(ValueBoxKey) -> Bool, limit: Int) { - self.range(table, start: start, end: end, values: { key, _ in - return keys(key) - }, limit: limit) - } - - public func get(_ table: Int32, key: ValueBoxKey) -> ReadBuffer? { - let startTime = CFAbsoluteTimeGetCurrent() - - var commit = false - if self.sharedTxn == nil { - self.begin() - commit = true - } - - var nativeTable: LmdbTable? - if let existingTable = self.tables[table] { - nativeTable = existingTable - } else if let createdTable = self.createTableWithName(table) { - nativeTable = createdTable - self.tables[table] = createdTable - } - - var resultValue: ReadBuffer? - - if let nativeTable = nativeTable { - var mdbKey = MDB_val() - var mdbData = MDB_val() - - mdbKey.mv_data = key.memory - mdbKey.mv_size = key.length - - let result = mdb_get(self.sharedTxn, nativeTable.dbi, &mdbKey, &mdbData) - - if result == MDB_SUCCESS { - let value = malloc(mdbData.mv_size)! - memcpy(value, mdbData.mv_data, mdbData.mv_size) - resultValue = ReadBuffer(memory: value, length: mdbData.mv_size, freeWhenDone: true) - } else { - if result != MDB_NOTFOUND { - print("(LmdbValueBox mdb_get failed with \(result))") - } - } - } - - if commit { - self.commit() - } - - readQueryTime += CFAbsoluteTimeGetCurrent() - startTime - - return resultValue - } - - public func exists(_ table: Int32, key: ValueBoxKey) -> Bool { - return self.get(table, key: key) != nil - } - - public func set(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) { - let startTime = CFAbsoluteTimeGetCurrent() - - var commit = false - if self.sharedTxn == nil { - self.begin() - commit = true - } - - var nativeTable: LmdbTable? - if let existingTable = self.tables[table] { - nativeTable = existingTable - } else if let createdTable = self.createTableWithName(table) { - nativeTable = createdTable - self.tables[table] = createdTable - } - - if let nativeTable = nativeTable { - var mdbKey = MDB_val() - var mdbData = MDB_val() - - mdbKey.mv_data = key.memory - mdbKey.mv_size = key.length - mdbData.mv_data = value.memory - mdbData.mv_size = value.length - - let result = mdb_put(self.sharedTxn, nativeTable.dbi, &mdbKey, &mdbData, 0) - if result != MDB_SUCCESS { - print("(LmdbValueBox mdb_set failed with \(result))") - } - } - - if commit { - self.commit() - } - - writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime - } - - public func remove(_ table: Int32, key: ValueBoxKey) { - let startTime = CFAbsoluteTimeGetCurrent() - - var commit = false - if self.sharedTxn == nil { - self.begin() - commit = true - } - - var nativeTable: LmdbTable? - if let existingTable = self.tables[table] { - nativeTable = existingTable - } else if let createdTable = self.createTableWithName(table) { - nativeTable = createdTable - self.tables[table] = createdTable - } - - if let nativeTable = nativeTable { - var mdbKey = MDB_val() - - mdbKey.mv_data = key.memory - mdbKey.mv_size = key.length - - let result = mdb_del(self.sharedTxn, nativeTable.dbi, &mdbKey, nil) - if result != MDB_SUCCESS { - print("(LmdbValueBox mdb_set failed with \(result))") - } - } - - if commit { - self.commit() - } - - writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime - } - - public func drop() { - - } -} diff --git a/Postbox/MappedFile.swift b/Postbox/MappedFile.swift index 8c57238fde..d0486c685f 100644 --- a/Postbox/MappedFile.swift +++ b/Postbox/MappedFile.swift @@ -3,7 +3,7 @@ import Foundation public final class MappedFile { private var handle: Int32 private var currentSize: Int - private var memory: UnsafeMutablePointer + private var memory: UnsafeMutableRawPointer public init(path: String) { self.handle = open(path, O_RDWR | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 600bd826ad..d49c6e3e74 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -116,7 +116,7 @@ public final class MediaBox { } self.statusQueue.async { - if let statusContext = self.statusContexts[resource.id] where statusContext.status == nil { + if let statusContext = self.statusContexts[resource.id] , statusContext.status == nil { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { @@ -499,7 +499,7 @@ public final class MediaBox { } self.statusQueue.async { - if let statusContext = self.statusContexts[resource.id] where statusContext.status != status { + if let statusContext = self.statusContexts[resource.id] , statusContext.status != status { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { subscriber(status) @@ -523,7 +523,7 @@ public final class MediaBox { public func cancelInteractiveResourceFetch(_ resource: MediaResource) { self.dataQueue.async { - if let dataContext = self.dataContexts[resource.id] where dataContext.fetchDisposable != nil { + if let dataContext = self.dataContexts[resource.id] , dataContext.fetchDisposable != nil { dataContext.fetchDisposable?.dispose() dataContext.fetchDisposable = nil @@ -536,7 +536,7 @@ public final class MediaBox { } self.statusQueue.async { - if let statusContext = self.statusContexts[resource.id] where statusContext.status != status { + if let statusContext = self.statusContexts[resource.id] , statusContext.status != status { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { subscriber(status) diff --git a/Postbox/Message.swift b/Postbox/Message.swift index e1f139bd24..d48a25c826 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -230,7 +230,7 @@ public func ==(lhs: MessageForwardInfo, rhs: MessageForwardInfo) -> Bool { if !lhs.author.isEqual(rhs.author) { return false } - if let lhsSource = lhs.source, rhsSource = rhs.source { + if let lhsSource = lhs.source, let rhsSource = rhs.source { if !lhsSource.isEqual(rhsSource) { return false } diff --git a/Postbox/MessageHistoryHolesView.swift b/Postbox/MessageHistoryHolesView.swift index 755b8047a2..1db0fbcffe 100644 --- a/Postbox/MessageHistoryHolesView.swift +++ b/Postbox/MessageHistoryHolesView.swift @@ -19,7 +19,7 @@ public func <(lhs: MessageHistoryHolesViewEntry, rhs: MessageHistoryHolesViewEnt } final class MutableMessageHistoryHolesView { - private var entries: [PeerId: Set] = [:] + fileprivate var entries: [PeerId: Set] = [:] init() { } diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index fd10190518..ac8c1ed840 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -289,7 +289,7 @@ final class MessageHistoryIndexTable: Table { let adjacent = self.adjacentItems(id) - if let lowerItem = adjacent.lower, upperItem = adjacent.upper { + if let lowerItem = adjacent.lower, let upperItem = adjacent.upper { switch lowerItem { case let .Message(lowerMessage): switch upperItem { @@ -471,7 +471,7 @@ final class MessageHistoryIndexTable: Table { self.justRemove(upperHole.maxIndex, operations: &operations) } } else if fillType.direction == .LowerToUpper { - if let maxMessageInRange = maxMessageInRange where maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { + if let maxMessageInRange = maxMessageInRange , maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { let stableId: UInt32 let tags: UInt32 = upperHole.tags if removedHole { @@ -482,7 +482,7 @@ final class MessageHistoryIndexTable: Table { self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: upperHole.maxIndex, min: maxMessageInRange.id.id + 1, tags: tags), operations: &operations) } } else if fillType.direction == .UpperToLower { - if let minMessageInRange = minMessageInRange where minMessageInRange.id.id - 1 >= upperHole.min { + if let minMessageInRange = minMessageInRange , minMessageInRange.id.id - 1 >= upperHole.min { let stableId: UInt32 let tags: UInt32 = upperHole.tags if removedHole { @@ -498,14 +498,14 @@ final class MessageHistoryIndexTable: Table { removedHole = true } - if let minMessageInRange = minMessageInRange where minMessageInRange.id.id - 1 >= upperHole.min { + if let minMessageInRange = minMessageInRange , minMessageInRange.id.id - 1 >= upperHole.min { let stableId: UInt32 = upperHole.stableId let tags: UInt32 = upperHole.tags self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minMessageInRange.id.id - 1), timestamp: minMessageInRange.timestamp), min: upperHole.min, tags: tags), operations: &operations) } - if let maxMessageInRange = maxMessageInRange where maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { + if let maxMessageInRange = maxMessageInRange , maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { let stableId: UInt32 = self.metadataTable.getNextStableMessageIndexId() let tags: UInt32 = upperHole.tags self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: upperHole.maxIndex, min: maxMessageInRange.id.id + 1, tags: tags), operations: &operations) @@ -648,7 +648,7 @@ final class MessageHistoryIndexTable: Table { value.read(&flags, offset: 0, length: 1) if (flags & HistoryEntryTypeMask) == HistoryEntryTypeHole { value.reset() - if case let .Hole(hole) = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value) where hole.min <= maxId && hole.maxIndex.id.id >= maxId { + if case let .Hole(hole) = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value) , hole.min <= maxId && hole.maxIndex.id.id >= maxId { holes = true } } diff --git a/Postbox/MessageHistoryMetadataTable.swift b/Postbox/MessageHistoryMetadataTable.swift index 5c6f3c7bf6..c5ad09cc25 100644 --- a/Postbox/MessageHistoryMetadataTable.swift +++ b/Postbox/MessageHistoryMetadataTable.swift @@ -155,7 +155,7 @@ final class MessageHistoryMetadataTable: Table { let sharedBuffer = WriteBuffer() for (peerId, namespaces) in self.updatedPeerNextMessageIdByNamespace { for namespace in namespaces { - if let messageIdByNamespace = self.peerNextMessageIdByNamespace[peerId], maxId = messageIdByNamespace[namespace] { + if let messageIdByNamespace = self.peerNextMessageIdByNamespace[peerId], let maxId = messageIdByNamespace[namespace] { sharedBuffer.reset() var mutableMaxId = maxId sharedBuffer.write(&mutableMaxId, offset: 0, length: 4) diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index d276067189..166ab72382 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -200,7 +200,7 @@ final class MessageHistoryReadStateTable: Table { } func applyIncomingMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (CombinedPeerReadState?, Bool) { - if let states = self.get(messageId.peerId), state = states.namespaces[messageId.namespace] { + if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] { if traceReadStates { print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId.id) (before: \(states.namespaces))") } @@ -231,7 +231,7 @@ final class MessageHistoryReadStateTable: Table { } func applyOutgoingMaxReadId(_ messageId: MessageId) -> (CombinedPeerReadState?, Bool) { - if let states = self.get(messageId.peerId), state = states.namespaces[messageId.namespace] { + if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] { if state.maxOutgoingReadId < messageId.id { states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: state.maxIncomingReadId, maxOutgoingReadId: state.maxOutgoingReadId, maxKnownId: state.maxKnownId, count: state.count) self.updatedPeerIds.insert(messageId.peerId) @@ -262,7 +262,7 @@ final class MessageHistoryReadStateTable: Table { override func beforeCommit() { let sharedBuffer = WriteBuffer() for id in self.updatedPeerIds { - if let wrappedStates = self.cachedPeerReadStates[id], states = wrappedStates { + if let wrappedStates = self.cachedPeerReadStates[id], let states = wrappedStates { sharedBuffer.reset() var count: Int32 = Int32(states.namespaces.count) sharedBuffer.write(&count, offset: 0, length: 4) diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 26e70435eb..a9bf211be4 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -610,7 +610,7 @@ final class MessageHistoryTable: Table { } func embeddedMediaForIndex(_ index: MessageIndex, id: MediaId) -> Media? { - if let message = self.getMessage(index) where message.embeddedMediaData.length > 4 { + if let message = self.getMessage(index), message.embeddedMediaData.length > 4 { var embeddedMediaCount: Int32 = 0 message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) @@ -619,7 +619,7 @@ final class MessageHistoryTable: Table { message.embeddedMediaData.read(&mediaLength, offset: 0, length: 4) if let readMedia = Decoder(buffer: MemoryBuffer(memory: message.embeddedMediaData.memory + message.embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { - if let readMediaId = readMedia.id where readMediaId == id { + if let readMediaId = readMedia.id, readMediaId == id { return readMedia } } @@ -880,7 +880,7 @@ final class MessageHistoryTable: Table { } func unembedMedia(_ index: MessageIndex, id: MediaId) -> Media? { - if let message = self.getMessage(index) where message.embeddedMediaData.length > 4 { + if let message = self.getMessage(index), message.embeddedMediaData.length > 4 { var embeddedMediaCount: Int32 = 0 message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) @@ -897,14 +897,14 @@ final class MessageHistoryTable: Table { message.embeddedMediaData.read(&mediaLength, offset: 0, length: 4) if let media = Decoder(buffer: MemoryBuffer(memory: message.embeddedMediaData.memory + message.embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { - if let mediaId = media.id where mediaId == id { + if let mediaId = media.id, mediaId == id { copyMedia = false extractedMedia = media } } if copyMedia { - updatedEmbeddedMediaBuffer.write(message.embeddedMediaData.memory + mediaOffset, offset: 0, length: message.embeddedMediaData.offset - mediaOffset) + updatedEmbeddedMediaBuffer.write(message.embeddedMediaData.memory.advanced(by: mediaOffset), offset: 0, length: message.embeddedMediaData.offset - mediaOffset) } } @@ -1055,7 +1055,7 @@ final class MessageHistoryTable: Table { var textLength: Int32 = 0 value.read(&textLength, offset: 0, length: 4) - let text = String(data: Data(bytes: UnsafeMutablePointer(value.memory).advanced(by: value.offset), count: Int(textLength)), encoding: .utf8) ?? "" + let text = String(data: Data(bytes: value.memory.assumingMemoryBound(to: UInt8.self).advanced(by: value.offset), count: Int(textLength)), encoding: .utf8) ?? "" //let text = NSString(bytes: value.memory + value.offset, length: Int(textLength), encoding: NSUTF8StringEncoding) ?? "" value.skip(Int(textLength)) @@ -1151,7 +1151,7 @@ final class MessageHistoryTable: Table { } var forwardInfo: MessageForwardInfo? - if let internalForwardInfo = message.forwardInfo, forwardAuthor = peerTable.get(internalForwardInfo.authorId) { + if let internalForwardInfo = message.forwardInfo, let forwardAuthor = peerTable.get(internalForwardInfo.authorId) { var source: Peer? if let sourceId = internalForwardInfo.sourceId { @@ -1345,7 +1345,7 @@ final class MessageHistoryTable: Table { } func maxReadIndex(_ peerId: PeerId) -> MessageHistoryAnchorIndex? { - if let combinedState = self.readStateTable.getCombinedState(peerId), state = combinedState.states.first where state.1.count != 0 { + if let combinedState = self.readStateTable.getCombinedState(peerId), let state = combinedState.states.first , state.1.count != 0 { return self.anchorIndex(MessageId(peerId: peerId, namespace: state.0, id: state.1.maxIncomingReadId)) } return nil @@ -1353,10 +1353,10 @@ final class MessageHistoryTable: Table { func anchorIndex(_ messageId: MessageId) -> MessageHistoryAnchorIndex? { let (lower, upper) = self.messageHistoryIndexTable.adjacentItems(messageId, bindUpper: false) - if let lower = lower, case let .Hole(hole) = lower where messageId.id >= hole.min && messageId.id <= hole.maxIndex.id.id { + if let lower = lower, case let .Hole(hole) = lower, messageId.id >= hole.min && messageId.id <= hole.maxIndex.id.id { return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: lower.index.timestamp), exact: false) } - if let upper = upper, case let .Hole(hole) = upper where messageId.id >= hole.min && messageId.id <= hole.maxIndex.id.id { + if let upper = upper, case let .Hole(hole) = upper, messageId.id >= hole.min && messageId.id <= hole.maxIndex.id.id { return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: upper.index.timestamp), exact: false) } diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index a851cc8b6d..87115c3ad9 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -126,12 +126,12 @@ final class MutableMessageHistoryViewReplayContext { final class MutableMessageHistoryView { private(set) var id: MessageHistoryViewId let tagMask: MessageTags? - private var anchorIndex: MessageHistoryAnchorIndex - private let combinedReadState: CombinedPeerReadState? - private var earlier: MutableMessageHistoryEntry? - private var later: MutableMessageHistoryEntry? - private var entries: [MutableMessageHistoryEntry] - private let fillCount: Int + fileprivate var anchorIndex: MessageHistoryAnchorIndex + fileprivate let combinedReadState: CombinedPeerReadState? + fileprivate var earlier: MutableMessageHistoryEntry? + fileprivate var later: MutableMessageHistoryEntry? + fileprivate var entries: [MutableMessageHistoryEntry] + fileprivate let fillCount: Int init(id: MessageHistoryViewId, anchorIndex: MessageHistoryAnchorIndex, combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int) { self.id = id @@ -166,7 +166,7 @@ final class MutableMessageHistoryView { } } - if let minIndex = minIndex, maxIndex = maxIndex { + if let minIndex = minIndex, let maxIndex = maxIndex { var minClipIndex = minIndex var maxClipIndex = maxIndex @@ -222,13 +222,15 @@ final class MutableMessageHistoryView { return false } - func refreshDueToExternalTransaction(fetchAroundHistoryEntries: (index: MessageIndex, count: Int, tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?)) -> Bool { - var index = MessageIndex.absoluteUpperBound() + func refreshDueToExternalTransaction(fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?)) -> Bool { + var index = MessageIndex.upperBound(peerId: self.anchorIndex.index.id.peerId) if !self.entries.isEmpty { - index = self.entries[self.entries.count / 2].index + if let _ = self.later { + index = self.entries[self.entries.count / 2].index + } } - let (entries, earlier, later) = fetchAroundHistoryEntries(index: index, count: max(self.fillCount, self.entries.count), tagMask: self.tagMask) + let (entries, earlier, later) = fetchAroundHistoryEntries(index, max(self.fillCount, self.entries.count), self.tagMask) self.entries = entries self.earlier = earlier @@ -268,7 +270,7 @@ final class MutableMessageHistoryView { //self.combinedReadState = combinedReadState case let .UpdateEmbeddedMedia(index, embeddedMediaData): for i in 0 ..< self.entries.count { - if case let .IntermediateMessageEntry(message, _) = self.entries[i] where MessageIndex(message) == index { + if case let .IntermediateMessageEntry(message, _) = self.entries[i] , MessageIndex(message) == index { self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), nil) hasChanges = true break @@ -283,7 +285,7 @@ final class MutableMessageHistoryView { case let .MessageEntry(message, _): var rebuild = false for media in message.media { - if let mediaId = media.id, _ = updatedMedia[mediaId] { + if let mediaId = media.id, let _ = updatedMedia[mediaId] { rebuild = true break } @@ -292,7 +294,7 @@ final class MutableMessageHistoryView { if rebuild { var messageMedia: [Media] = [] for media in message.media { - if let mediaId = media.id, updated = updatedMedia[mediaId] { + if let mediaId = media.id, let updated = updatedMedia[mediaId] { if let updated = updated { messageMedia.append(updated) } @@ -307,7 +309,7 @@ final class MutableMessageHistoryView { case let .IntermediateMessageEntry(message, _): var rebuild = false for mediaId in message.referencedMedia { - if let media = updatedMedia[mediaId] where media?.id != mediaId { + if let media = updatedMedia[mediaId] , media?.id != mediaId { rebuild = true break } @@ -315,7 +317,7 @@ final class MutableMessageHistoryView { if rebuild { var referencedMedia: [MediaId] = [] for mediaId in message.referencedMedia { - if let media = updatedMedia[mediaId] where media?.id != mediaId { + if let media = updatedMedia[mediaId] , media?.id != mediaId { if let id = media?.id { referencedMedia.append(id) } @@ -381,12 +383,12 @@ final class MutableMessageHistoryView { private func remove(_ indices: Set, context: MutableMessageHistoryViewReplayContext) -> Bool { var hasChanges = false - if let earlier = self.earlier where indices.contains(earlier.index) { + if let earlier = self.earlier , indices.contains(earlier.index) { context.invalidEarlier = true hasChanges = true } - if let later = self.later where indices.contains(later.index) { + if let later = self.later , indices.contains(later.index) { context.invalidLater = true hasChanges = true } @@ -571,7 +573,7 @@ public final class MessageHistoryView { self.combinedReadState = mutableView.combinedReadState - if let combinedReadState = mutableView.combinedReadState where combinedReadState.count != 0 { + if let combinedReadState = mutableView.combinedReadState , combinedReadState.count != 0 { var maxIndex: MessageIndex? for (namespace, state) in combinedReadState.states { var maxNamespaceIndex: MessageIndex? @@ -593,16 +595,16 @@ public final class MessageHistoryView { index += 1 } } - if let _ = maxNamespaceIndex where index + 1 < entries.count { + if let _ = maxNamespaceIndex , index + 1 < entries.count { for i in index + 1 ..< entries.count { - if case let .MessageEntry(message, _) = entries[i] where !message.flags.contains(.Incoming) { + if case let .MessageEntry(message, _) = entries[i] , !message.flags.contains(.Incoming) { maxNamespaceIndex = MessageIndex(message) } else { break } } } - if let maxNamespaceIndex = maxNamespaceIndex where maxIndex == nil || maxIndex! < maxNamespaceIndex { + if let maxNamespaceIndex = maxNamespaceIndex , maxIndex == nil || maxIndex! < maxNamespaceIndex { maxIndex = maxNamespaceIndex } } diff --git a/Postbox/PeerChatStateTable.swift b/Postbox/PeerChatStateTable.swift index 8444efc67f..e60e622ff0 100644 --- a/Postbox/PeerChatStateTable.swift +++ b/Postbox/PeerChatStateTable.swift @@ -15,7 +15,7 @@ final class PeerChatStateTable: Table { if let state = self.cachedPeerChatStates[id] { return state } else { - if let value = self.valueBox.get(self.tableId, key: self.key(id)), state = Decoder(buffer: value).decodeRootObject() { + if let value = self.valueBox.get(self.tableId, key: self.key(id)), let state = Decoder(buffer: value).decodeRootObject() { self.cachedPeerChatStates[id] = state return state } else { @@ -38,7 +38,7 @@ final class PeerChatStateTable: Table { override func beforeCommit() { let sharedEncoder = Encoder() for id in self.updatedPeerIds { - if let wrappedState = self.cachedPeerChatStates[id], state = wrappedState { + if let wrappedState = self.cachedPeerChatStates[id], let state = wrappedState { sharedEncoder.reset() sharedEncoder.encodeRootObject(state) self.valueBox.set(self.tableId, key: self.key(id), value: sharedEncoder.readBufferNoCopy()) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 8d9ccbc042..ffc7fe4e9f 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -14,7 +14,7 @@ public protocol PeerChatState: Coding { public final class Modifier { private weak var postbox: Postbox? - private init(postbox: Postbox) { + fileprivate init(postbox: Postbox) { self.postbox = postbox } @@ -130,11 +130,11 @@ public final class Modifier { } } -private class PipeNotifier: NSObject { +fileprivate class PipeNotifier: NSObject { let notifier: RLMNotifier let thread: Thread - private init(basePath: String, notify: () -> Void) { + fileprivate init(basePath: String, notify: @escaping () -> Void) { self.notifier = RLMNotifier(basePath: basePath, notify: notify) self.thread = Thread(target: PipeNotifier.self, selector: #selector(PipeNotifier.threadEntry(_:)), object: self.notifier) self.thread.start() @@ -304,7 +304,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 6 + let currentUserVersion: Int32 = 7 if userVersion != currentUserVersion { self.valueBox.drop() @@ -365,17 +365,15 @@ public final class Postbox { private var cachedState: Coding? - private func setState(_ state: Coding) { - self.modify({ modifier -> Void in - self.cachedState = state - - self.metadataTable.setState(state) - - self.statePipe.putNext(state) - }).start() + fileprivate func setState(_ state: Coding) { + self.cachedState = state + + self.metadataTable.setState(state) + + self.statePipe.putNext(state) } - private func getState() -> Coding? { + fileprivate func getState() -> Coding? { if let cachedState = self.cachedState { return cachedState } else { @@ -388,6 +386,7 @@ public final class Postbox { } } + @available(*, deprecated: 1.0) public func state() -> Signal { return self.modify { modifier -> Signal in return Signal.single(self.getState()) @@ -407,15 +406,15 @@ public final class Postbox { self.keychainTable.remove(key) } - private func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) { + fileprivate func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) { self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func addHole(_ id: MessageId) { + fileprivate func addHole(_ id: MessageId) { self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func fillHole(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { + fileprivate func fillHole(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] self.messageHistoryTable.fillHole(hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) for (peerId, operations) in operationsByPeerId { @@ -455,31 +454,31 @@ public final class Postbox { } } - private func replaceChatListHole(_ index: MessageIndex, hole: ChatListHole?) { + fileprivate func replaceChatListHole(_ index: MessageIndex, hole: ChatListHole?) { self.currentReplaceChatListHoles.append((index, hole)) } - private func deleteMessages(_ messageIds: [MessageId]) { + fileprivate func deleteMessages(_ messageIds: [MessageId]) { self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]]) { + fileprivate func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]]) { self.messageHistoryTable.resetIncomingReadStates(states, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func confirmSynchronizedIncomingReadState(_ peerId: PeerId) { + fileprivate func confirmSynchronizedIncomingReadState(_ peerId: PeerId) { self.synchronizeReadStateTable.set(peerId, operation: nil, operations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func applyIncomingReadMaxId(_ messageId: MessageId) { + fileprivate func applyIncomingReadMaxId(_ messageId: MessageId) { self.messageHistoryTable.applyIncomingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func applyOutgoingReadMaxId(_ messageId: MessageId) { + fileprivate func applyOutgoingReadMaxId(_ messageId: MessageId) { self.messageHistoryTable.applyOutgoingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - private func applyInteractiveReadMaxId(_ messageId: MessageId) { + fileprivate func applyInteractiveReadMaxId(_ messageId: MessageId) { self.messageHistoryTable.applyInteractiveMaxReadId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } @@ -648,7 +647,7 @@ public final class Postbox { return self.messageHistoryTable.anchorIndex(id) } - private func renderIntermediateMessage(_ message: IntermediateMessage) -> Message { + fileprivate func renderIntermediateMessage(_ message: IntermediateMessage) -> Message { return self.messageHistoryTable.renderMessage(message, peerTable: self.peerTable) } @@ -708,7 +707,7 @@ public final class Postbox { return (updatedTransactionState, updatedMasterClientId) } - private func messageIdsForGlobalIds(_ ids: [Int32]) -> [MessageId] { + fileprivate func messageIdsForGlobalIds(_ ids: [Int32]) -> [MessageId] { var result: [MessageId] = [] for globalId in ids { if let id = self.globalMessageIdsTable.get(globalId) { @@ -718,7 +717,7 @@ public final class Postbox { return result } - private func updatePeers(_ peers: [Peer], update: (Peer, Peer) -> Peer?) { + fileprivate func updatePeers(_ peers: [Peer], update: (Peer, Peer) -> Peer?) { for peer in peers { if let currentPeer = self.peerTable.get(peer.id) { if let updatedPeer = update(currentPeer, peer) { @@ -731,28 +730,28 @@ public final class Postbox { } } - private func replaceContactPeerIds(_ peerIds: Set) { + fileprivate func replaceContactPeerIds(_ peerIds: Set) { self.contactsTable.replace(peerIds) self.currentReplacedContactPeerIds = peerIds } - private func replaceRecentPeerIds(_ peerIds: [PeerId]) { + fileprivate func replaceRecentPeerIds(_ peerIds: [PeerId]) { self.peerRatingTable.replace(items: peerIds) } - private func updateMessage(_ index: MessageIndex, update: (Message) -> StoreMessage) { + fileprivate func updateMessage(_ index: MessageIndex, update: (Message) -> StoreMessage) { if let intermediateMessage = self.messageHistoryTable.getMessage(index) { let message = self.renderIntermediateMessage(intermediateMessage) self.messageHistoryTable.updateMessage(index.id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } } - private func updateMedia(_ id: MediaId, update: Media?) { + fileprivate func updateMedia(_ id: MediaId, update: Media?) { self.messageHistoryTable.updateMedia(id, media: update, operationsByPeerId: &self.currentOperationsByPeerId, updatedMedia: &self.currentUpdatedMedia) } - private func filterStoredMessageIds(_ messageIds: Set) -> Set { + fileprivate func filterStoredMessageIds(_ messageIds: Set) -> Set { var filteredIds = Set() for id in messageIds { @@ -764,7 +763,7 @@ public final class Postbox { return filteredIds } - public func modify(_ f: (Modifier) -> T) -> Signal { + public func modify(_ f: @escaping(Modifier) -> T) -> Signal { return Signal { subscriber in self.queue.justDispatch { self.valueBox.begin() diff --git a/Postbox/PostboxPrivate/module.modulemap b/Postbox/PostboxPrivate/module.modulemap index 52083fa5cf..b4494ba2ae 100644 --- a/Postbox/PostboxPrivate/module.modulemap +++ b/Postbox/PostboxPrivate/module.modulemap @@ -3,6 +3,5 @@ module sqlcipher { header "sqlcipher/sqlite3ext.h" header "sqlcipher/SQLite-Bridging.h" header "sqlcipher/fts3_tokenizer.h" - header "../../submodules/lmdb/libraries/liblmdb/lmdb.h" export * } diff --git a/Postbox/RandomAccessMediaResourceContext.swift b/Postbox/RandomAccessMediaResourceContext.swift index 08006169c5..8808e6abf7 100644 --- a/Postbox/RandomAccessMediaResourceContext.swift +++ b/Postbox/RandomAccessMediaResourceContext.swift @@ -2,8 +2,8 @@ import Foundation import SwiftSignalKit public struct RandomAccessResourceStoreRange { - private let offset: Int - private let data: Data + fileprivate let offset: Int + fileprivate let data: Data public init(offset: Int, data: Data) { self.offset = offset @@ -19,16 +19,16 @@ public enum RandomAccessResourceDataRangeMode { } private final class RandomAccessBlockRangeListener: Hashable { - private let id: Int32 + fileprivate let id: Int32 private let range: Range private let blockSize: Int private let blocks: Range private let mode: RandomAccessResourceDataRangeMode private let updated: (Data) -> Void - private var missingBlocks: Set + fileprivate var missingBlocks: Set - init(id: Int32, range: Range, blockSize: Int, blocks: Range, missingBlocks: Set, mode: RandomAccessResourceDataRangeMode, updated: (Data) -> Void) { + init(id: Int32, range: Range, blockSize: Int, blocks: Range, missingBlocks: Set, mode: RandomAccessResourceDataRangeMode, updated: @escaping(Data) -> Void) { self.id = id self.range = range self.blockSize = blockSize @@ -42,7 +42,7 @@ private final class RandomAccessBlockRangeListener: Hashable { return Int(self.id) } - func updateMissingBlocks(addedBlocks: Set, fetchData: @noescape(Range) -> Data) { + func updateMissingBlocks(addedBlocks: Set, fetchData: (Range) -> Data) { if self.missingBlocks.isEmpty { return } @@ -133,7 +133,7 @@ public final class RandomAccessMediaResourceContext { private var fetchRange: (Range) -> Disposable - public init(path: String, size: Int, fetchRange: (Range) -> Disposable) { + public init(path: String, size: Int, fetchRange: @escaping(Range) -> Disposable) { self.path = path self.size = size self.fetchRange = fetchRange @@ -308,7 +308,7 @@ public final class RandomAccessMediaResourceContext { return result } - public func addListenerForData(in range: Range, mode: RandomAccessResourceDataRangeMode, updated: (Data) -> Void) -> Int32 { + public func addListenerForData(in range: Range, mode: RandomAccessResourceDataRangeMode, updated: @escaping (Data) -> Void) -> Int32 { let firstBlock = range.lowerBound / self.blockSize let lastBlock = range.upperBound / self.blockSize + (range.upperBound % self.blockSize == 0 ? 0 : 1) diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index be1a8063c2..b492813c03 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -5,7 +5,7 @@ import SwiftSignalKit private struct SqlitePreparedStatement { let statement: OpaquePointer? - func bind(_ index: Int, data: UnsafePointer, length: Int) { + func bind(_ index: Int, data: UnsafeRawPointer, length: Int) { sqlite3_bind_blob(statement, Int32(index), data, Int32(length), nil) } @@ -101,9 +101,7 @@ public final class SqliteValueBox: ValueBox { checkpoints.set(nil) lock.lock() - do { - try FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) - } catch _ { } + let _ = try? FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) let path = basePath + "/db_sqlite" let database = Database(path) @@ -115,6 +113,16 @@ public final class SqliteValueBox: ValueBox { database.execute("PRAGMA temp_store=MEMORY") database.execute("PRAGMA wal_autocheckpoint=200") database.execute("PRAGMA journal_size_limit=1536") + + /*var statement: OpaquePointer? = nil + sqlite3_prepare_v2(database.handle, "PRAGMA integrity_check", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + if preparedStatement.step() { + let value = preparedStatement.stringAt(0) + print("integrity_check: \(value)") + } + preparedStatement.destroy()*/ + sqlite3_busy_timeout(database.handle, 10000000) let result = self.getUserVersion(database) @@ -129,7 +137,7 @@ public final class SqliteValueBox: ValueBox { lock.unlock() checkpoints.set((Signal.single(Void()) |> delay(10.0, queue: Queue.concurrentDefaultQueue()) |> restart).start(next: { [weak self] _ in - if let strongSelf = self where strongSelf.database != nil { + if let strongSelf = self , strongSelf.database != nil { strongSelf.lock.lock() var nLog: Int32 = 0 var nFrames: Int32 = 0 diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index 5343ee4a5e..ffac0122eb 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -1,9 +1,9 @@ import Foundation private final class ValueBoxKeyImpl { - let memory: UnsafeMutablePointer + let memory: UnsafeMutableRawPointer - init(memory: UnsafeMutablePointer) { + init(memory: UnsafeMutableRawPointer) { self.memory = memory } @@ -13,12 +13,12 @@ private final class ValueBoxKeyImpl { } public struct ValueBoxKey: Comparable, CustomStringConvertible { - public let memory: UnsafeMutablePointer + public let memory: UnsafeMutableRawPointer public let length: Int private let impl: ValueBoxKeyImpl public init(length: Int) { - self.memory = malloc(length) + self.memory = malloc(length)! self.length = length self.impl = ValueBoxKeyImpl(memory: self.memory) } @@ -28,7 +28,7 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { self.memory = malloc(data.count) self.length = data.count self.impl = ValueBoxKeyImpl(memory: self.memory) - data.copyBytes(to: UnsafeMutablePointer(self.memory), count: data.count) + data.copyBytes(to: self.memory.assumingMemoryBound(to: UInt8.self), count: data.count) } public init(_ buffer: MemoryBuffer) { @@ -92,7 +92,7 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { public var successor: ValueBoxKey { let key = ValueBoxKey(length: self.length) memcpy(key.memory, self.memory, self.length) - let memory = UnsafeMutablePointer(key.memory) + let memory = key.memory.assumingMemoryBound(to: UInt8.self) var i = self.length - 1 while i >= 0 { var byte = memory[i] @@ -112,7 +112,7 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { public var predecessor: ValueBoxKey { let key = ValueBoxKey(length: self.length) memcpy(key.memory, self.memory, self.length) - let memory = UnsafeMutablePointer(key.memory) + let memory = key.memory.assumingMemoryBound(to: UInt8.self) var i = self.length - 1 while i >= 0 { var byte = memory[i] @@ -131,7 +131,7 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { public var description: String { let string = NSMutableString() - let memory = UnsafeMutablePointer(self.memory) + let memory = self.memory.assumingMemoryBound(to: UInt8.self) for i in 0 ..< self.length { let byte: Int = Int(memory[i]) string.appendFormat("%02x", byte) @@ -144,7 +144,7 @@ public func ==(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { return lhs.length == rhs.length && memcmp(lhs.memory, rhs.memory, lhs.length) == 0 } -private func mdb_cmp_memn(_ a_memory: UnsafeMutablePointer, _ a_length: Int, _ b_memory: UnsafeMutablePointer, _ b_length: Int) -> Int +private func mdb_cmp_memn(_ a_memory: UnsafeMutableRawPointer, _ a_length: Int, _ b_memory: UnsafeMutableRawPointer, _ b_length: Int) -> Int { var diff: Int = 0 var len_diff: Int = 0 diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index f0a08804b8..38c5dbd57c 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -35,7 +35,7 @@ final class ViewTracker { private var synchronizeReadStatesView: MutableSynchronizePeerReadStatesView private let synchronizePeerReadStatesViewSubscribers = Bag>() - init(queue: Queue, fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: (IntermediateMessage) -> Message, getPeer: (PeerId) -> Peer?, unsentMessageIndices: [MessageIndex], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, unsentMessageIndices: [MessageIndex], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -142,7 +142,7 @@ final class ViewTracker { } } - func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (index: MessageIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (index: MessageIndex, count: Int, tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsendMessageIndices: () -> [MessageIndex], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: MessageIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsendMessageIndices: () -> [MessageIndex], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { diff --git a/submodules/lmdb/libraries/liblmdb/lmdb.h b/submodules/lmdb/libraries/liblmdb/lmdb.h deleted file mode 100644 index 82a5410ab1..0000000000 --- a/submodules/lmdb/libraries/liblmdb/lmdb.h +++ /dev/null @@ -1,1565 +0,0 @@ -/** @file lmdb.h - * @brief Lightning memory-mapped database library - * - * @mainpage Lightning Memory-Mapped Database Manager (LMDB) - * - * @section intro_sec Introduction - * LMDB is a Btree-based database management library modeled loosely on the - * BerkeleyDB API, but much simplified. The entire database is exposed - * in a memory map, and all data fetches return data directly - * from the mapped memory, so no malloc's or memcpy's occur during - * data fetches. As such, the library is extremely simple because it - * requires no page caching layer of its own, and it is extremely high - * performance and memory-efficient. It is also fully transactional with - * full ACID semantics, and when the memory map is read-only, the - * database integrity cannot be corrupted by stray pointer writes from - * application code. - * - * The library is fully thread-aware and supports concurrent read/write - * access from multiple processes and threads. Data pages use a copy-on- - * write strategy so no active data pages are ever overwritten, which - * also provides resistance to corruption and eliminates the need of any - * special recovery procedures after a system crash. Writes are fully - * serialized; only one write transaction may be active at a time, which - * guarantees that writers can never deadlock. The database structure is - * multi-versioned so readers run with no locks; writers cannot block - * readers, and readers don't block writers. - * - * Unlike other well-known database mechanisms which use either write-ahead - * transaction logs or append-only data writes, LMDB requires no maintenance - * during operation. Both write-ahead loggers and append-only databases - * require periodic checkpointing and/or compaction of their log or database - * files otherwise they grow without bound. LMDB tracks free pages within - * the database and re-uses them for new write operations, so the database - * size does not grow without bound in normal use. - * - * The memory map can be used as a read-only or read-write map. It is - * read-only by default as this provides total immunity to corruption. - * Using read-write mode offers much higher write performance, but adds - * the possibility for stray application writes thru pointers to silently - * corrupt the database. Of course if your application code is known to - * be bug-free (...) then this is not an issue. - * - * @section caveats_sec Caveats - * Troubleshooting the lock file, plus semaphores on BSD systems: - * - * - A broken lockfile can cause sync issues. - * Stale reader transactions left behind by an aborted program - * cause further writes to grow the database quickly, and - * stale locks can block further operation. - * - * Fix: Check for stale readers periodically, using the - * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. - * Catch stale - * locks with option MDB_ROBUST if supported (non-BSD). Or just - * make all programs using the database close it; the lockfile - * is always reset on first open of the environment. - * - * - On BSD systems or others configured with MDB_USE_POSIX_SEM, - * startup can fail due to semaphores owned by another userid. - * - * Fix: Open and close the database as the user which owns the - * semaphores (likely last user) or as root, while no other - * process is using the database. - * - * Restrictions/caveats (in addition to those listed for some functions): - * - * - Only the database owner should normally use the database on - * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. - * Multiple users can cause startup to fail later, as noted above. - * - * - There is normally no pure read-only mode, since readers need write - * access to locks and lock file. Exceptions: On read-only filesystems - * or with the #MDB_NOLOCK flag described under #mdb_env_open(). - * - * - By default, in versions before 0.9.10, unused portions of the data - * file might receive garbage data from memory freed by other code. - * (This does not happen when using the #MDB_WRITEMAP flag.) As of - * 0.9.10 the default behavior is to initialize such memory before - * writing to the data file. Since there may be a slight performance - * cost due to this initialization, applications may disable it using - * the #MDB_NOMEMINIT flag. Applications handling sensitive data - * which must not be written should not use this flag. This flag is - * irrelevant when using #MDB_WRITEMAP. - * - * - A thread can only use one transaction at a time, plus any child - * transactions. Each transaction belongs to one thread. See below. - * The #MDB_NOTLS flag changes this for read-only transactions. - * - * - Use an MDB_env* in the process which opened it, without fork()ing. - * - * - Do not have open an LMDB database twice in the same process at - * the same time. Not even from a plain open() call - close()ing it - * breaks flock() advisory locking. - * - * - Avoid long-lived transactions. Read transactions prevent - * reuse of pages freed by newer write transactions, thus the - * database can grow quickly. Write transactions prevent - * other write transactions, since writes are serialized. - * - * - Avoid suspending a process with active transactions. These - * would then be "long-lived" as above. Also read transactions - * suspended when writers commit could sometimes see wrong data. - * - * ...when several processes can use a database concurrently: - * - * - Avoid aborting a process with an active transaction. - * The transaction becomes "long-lived" as above until a check - * for stale readers is performed or the lockfile is reset, - * since the process may not remove it from the lockfile. - * Except write-transactions on Unix with MDB_ROBUST or on Windows. - * - * - If you do that anyway, do a periodic check for stale readers. Or - * close the environment once in a while, so the lockfile can get reset. - * - * - Do not use LMDB databases on remote filesystems, even between - * processes on the same host. This breaks flock() on some OSes, - * possibly memory map sync, and certainly sync between programs - * on different hosts. - * - * - Opening a database can fail if another process is opening or - * closing it at exactly the same time. - * - * @author Howard Chu, Symas Corporation. - * - * @copyright Copyright 2011-2014 Howard Chu, Symas Corp. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - * - * @par Derived From: - * This code is derived from btree.c written by Martin Hedenfalk. - * - * Copyright (c) 2009, 2010 Martin Hedenfalk - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#ifndef _LMDB_H_ -#define _LMDB_H_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** Unix permissions for creating files, or dummy definition for Windows */ -#ifdef _MSC_VER -typedef int mdb_mode_t; -#else -typedef mode_t mdb_mode_t; -#endif - -/** An abstraction for a file handle. - * On POSIX systems file handles are small integers. On Windows - * they're opaque pointers. - */ -#ifdef _WIN32 -typedef void *mdb_filehandle_t; -#else -typedef int mdb_filehandle_t; -#endif - -/** @defgroup mdb LMDB API - * @{ - * @brief OpenLDAP Lightning Memory-Mapped Database Manager - */ -/** @defgroup Version Version Macros - * @{ - */ -/** Library major version */ -#define MDB_VERSION_MAJOR 0 -/** Library minor version */ -#define MDB_VERSION_MINOR 9 -/** Library patch version */ -#define MDB_VERSION_PATCH 14 - -/** Combine args a,b,c into a single integer for easy version comparisons */ -#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) - -/** The full library version as a single integer */ -#define MDB_VERSION_FULL \ - MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) - -/** The release date of this library version */ -#define MDB_VERSION_DATE "September 20, 2014" - -/** A stringifier for the version info */ -#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" - -/** A helper for the stringifier macro */ -#define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) - -/** The full library version as a C string */ -#define MDB_VERSION_STRING \ - MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) -/** @} */ - -/** @brief Opaque structure for a database environment. - * - * A DB environment supports multiple databases, all residing in the same - * shared-memory map. - */ -typedef struct MDB_env MDB_env; - -/** @brief Opaque structure for a transaction handle. - * - * All database operations require a transaction handle. Transactions may be - * read-only or read-write. - */ -typedef struct MDB_txn MDB_txn; - -/** @brief A handle for an individual database in the DB environment. */ -typedef unsigned int MDB_dbi; - -/** @brief Opaque structure for navigating through a database */ -typedef struct MDB_cursor MDB_cursor; - -/** @brief Generic structure used for passing keys and data in and out - * of the database. - * - * Values returned from the database are valid only until a subsequent - * update operation, or the end of the transaction. Do not modify or - * free them, they commonly point into the database itself. - * - * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. - * The same applies to data sizes in databases with the #MDB_DUPSORT flag. - * Other data items can in theory be from 0 to 0xffffffff bytes long. - */ -typedef struct MDB_val { - size_t mv_size; /**< size of the data item */ - void *mv_data; /**< address of the data item */ -} MDB_val; - -/** @brief A callback function used to compare two keys in a database */ -typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); - -/** @brief A callback function used to relocate a position-dependent data item - * in a fixed-address database. - * - * The \b newptr gives the item's desired address in - * the memory map, and \b oldptr gives its previous address. The item's actual - * data resides at the address in \b item. This callback is expected to walk - * through the fields of the record in \b item and modify any - * values based at the \b oldptr address to be relative to the \b newptr address. - * @param[in,out] item The item that is to be relocated. - * @param[in] oldptr The previous address. - * @param[in] newptr The new address to relocate to. - * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). - * @todo This feature is currently unimplemented. - */ -typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); - -/** @defgroup mdb_env Environment Flags - * @{ - */ - /** mmap at a fixed address (experimental) */ -#define MDB_FIXEDMAP 0x01 - /** no environment directory */ -#define MDB_NOSUBDIR 0x4000 - /** don't fsync after commit */ -#define MDB_NOSYNC 0x10000 - /** read only */ -#define MDB_RDONLY 0x20000 - /** don't fsync metapage after commit */ -#define MDB_NOMETASYNC 0x40000 - /** use writable mmap */ -#define MDB_WRITEMAP 0x80000 - /** use asynchronous msync when #MDB_WRITEMAP is used */ -#define MDB_MAPASYNC 0x100000 - /** tie reader locktable slots to #MDB_txn objects instead of to threads */ -#define MDB_NOTLS 0x200000 - /** don't do any locking, caller must manage their own locks */ -#define MDB_NOLOCK 0x400000 - /** don't do readahead (no effect on Windows) */ -#define MDB_NORDAHEAD 0x800000 - /** don't initialize malloc'd memory before writing to datafile */ -#define MDB_NOMEMINIT 0x1000000 - /** catch stale locks if supported (not on BSD, needs robust mutexes) */ -#define MDB_ROBUST 0x2000000 -/** @} */ - -/** @defgroup mdb_dbi_open Database Flags - * @{ - */ - /** use reverse string keys */ -#define MDB_REVERSEKEY 0x02 - /** use sorted duplicates */ -#define MDB_DUPSORT 0x04 - /** numeric keys in native byte order. - * The keys must all be of the same size. */ -#define MDB_INTEGERKEY 0x08 - /** with #MDB_DUPSORT, sorted dup items have fixed size */ -#define MDB_DUPFIXED 0x10 - /** with #MDB_DUPSORT, dups are numeric in native byte order */ -#define MDB_INTEGERDUP 0x20 - /** with #MDB_DUPSORT, use reverse string dups */ -#define MDB_REVERSEDUP 0x40 - /** create DB if not already existing */ -#define MDB_CREATE 0x40000 -/** @} */ - -/** @defgroup mdb_put Write Flags - * @{ - */ -/** For put: Don't write if the key already exists. */ -#define MDB_NOOVERWRITE 0x10 -/** Only for #MDB_DUPSORT
    - * For put: don't write if the key and data pair already exist.
    - * For mdb_cursor_del: remove all duplicate data items. - */ -#define MDB_NODUPDATA 0x20 -/** For mdb_cursor_put: overwrite the current key/data pair */ -#define MDB_CURRENT 0x40 -/** For put: Just reserve space for data, don't copy it. Return a - * pointer to the reserved space. - */ -#define MDB_RESERVE 0x10000 -/** Data is being appended, don't split full pages. */ -#define MDB_APPEND 0x20000 -/** Duplicate data is being appended, don't split full pages. */ -#define MDB_APPENDDUP 0x40000 -/** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ -#define MDB_MULTIPLE 0x80000 -/* @} */ - -/** @defgroup mdb_copy Copy Flags - * @{ - */ -/** Compacting copy: Omit free space from copy, and renumber all - * pages sequentially. - */ -#define MDB_CP_COMPACT 0x01 -/* @} */ - -/** @brief Cursor Get operations. - * - * This is the set of all operations for retrieving data - * using a cursor. - */ -typedef enum MDB_cursor_op { - MDB_FIRST, /**< Position at first key/data item */ - MDB_FIRST_DUP, /**< Position at first data item of current key. - Only for #MDB_DUPSORT */ - MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ - MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ - MDB_GET_CURRENT, /**< Return key/data at current cursor position */ - MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items - from current cursor position. Move cursor to prepare - for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ - MDB_LAST, /**< Position at last key/data item */ - MDB_LAST_DUP, /**< Position at last data item of current key. - Only for #MDB_DUPSORT */ - MDB_NEXT, /**< Position at next data item */ - MDB_NEXT_DUP, /**< Position at next data item of current key. - Only for #MDB_DUPSORT */ - MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items - from next cursor position. Move cursor to prepare - for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ - MDB_NEXT_NODUP, /**< Position at first data item of next key */ - MDB_PREV, /**< Position at previous data item */ - MDB_PREV_DUP, /**< Position at previous data item of current key. - Only for #MDB_DUPSORT */ - MDB_PREV_NODUP, /**< Position at last data item of previous key */ - MDB_SET, /**< Position at specified key */ - MDB_SET_KEY, /**< Position at specified key, return key + data */ - MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */ -} MDB_cursor_op; - -/** @defgroup errors Return Codes - * - * BerkeleyDB uses -30800 to -30999, we'll go under them - * @{ - */ - /** Successful result */ -#define MDB_SUCCESS 0 - /** key/data pair already exists */ -#define MDB_KEYEXIST (-30799) - /** key/data pair not found (EOF) */ -#define MDB_NOTFOUND (-30798) - /** Requested page not found - this usually indicates corruption */ -#define MDB_PAGE_NOTFOUND (-30797) - /** Located page was wrong type */ -#define MDB_CORRUPTED (-30796) - /** Update of meta page failed or environment had fatal error */ -#define MDB_PANIC (-30795) - /** Environment version mismatch */ -#define MDB_VERSION_MISMATCH (-30794) - /** File is not a valid LMDB file */ -#define MDB_INVALID (-30793) - /** Environment mapsize reached */ -#define MDB_MAP_FULL (-30792) - /** Environment maxdbs reached */ -#define MDB_DBS_FULL (-30791) - /** Environment maxreaders reached */ -#define MDB_READERS_FULL (-30790) - /** Too many TLS keys in use - Windows only */ -#define MDB_TLS_FULL (-30789) - /** Txn has too many dirty pages */ -#define MDB_TXN_FULL (-30788) - /** Cursor stack too deep - internal error */ -#define MDB_CURSOR_FULL (-30787) - /** Page has not enough space - internal error */ -#define MDB_PAGE_FULL (-30786) - /** Database contents grew beyond environment mapsize */ -#define MDB_MAP_RESIZED (-30785) - /** MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed */ -#define MDB_INCOMPATIBLE (-30784) - /** Invalid reuse of reader locktable slot */ -#define MDB_BAD_RSLOT (-30783) - /** Transaction cannot recover - it must be aborted */ -#define MDB_BAD_TXN (-30782) - /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ -#define MDB_BAD_VALSIZE (-30781) - /** The specified DBI was changed unexpectedly */ -#define MDB_BAD_DBI (-30780) - /** The last defined error code */ -#define MDB_LAST_ERRCODE MDB_BAD_DBI -/** @} */ - -/** @brief Statistics for a database in the environment */ -typedef struct MDB_stat { - unsigned int ms_psize; /**< Size of a database page. - This is currently the same for all databases. */ - unsigned int ms_depth; /**< Depth (height) of the B-tree */ - size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ - size_t ms_leaf_pages; /**< Number of leaf pages */ - size_t ms_overflow_pages; /**< Number of overflow pages */ - size_t ms_entries; /**< Number of data items */ -} MDB_stat; - -/** @brief Information about the environment */ -typedef struct MDB_envinfo { - void *me_mapaddr; /**< Address of map, if fixed */ - size_t me_mapsize; /**< Size of the data memory map */ - size_t me_last_pgno; /**< ID of the last used page */ - size_t me_last_txnid; /**< ID of the last committed transaction */ - unsigned int me_maxreaders; /**< max reader slots in the environment */ - unsigned int me_numreaders; /**< max reader slots used in the environment */ -} MDB_envinfo; - - /** @brief Return the LMDB library version information. - * - * @param[out] major if non-NULL, the library major version number is copied here - * @param[out] minor if non-NULL, the library minor version number is copied here - * @param[out] patch if non-NULL, the library patch version number is copied here - * @retval "version string" The library version as a string - */ -char *mdb_version(int *major, int *minor, int *patch); - - /** @brief Return a string describing a given error code. - * - * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) - * function. If the error code is greater than or equal to 0, then the string - * returned by the system function strerror(3) is returned. If the error code - * is less than 0, an error string corresponding to the LMDB library error is - * returned. See @ref errors for a list of LMDB-specific error codes. - * @param[in] err The error code - * @retval "error message" The description of the error - */ -char *mdb_strerror(int err); - - /** @brief Create an LMDB environment handle. - * - * This function allocates memory for a #MDB_env structure. To release - * the allocated memory and discard the handle, call #mdb_env_close(). - * Before the handle may be used, it must be opened using #mdb_env_open(). - * Various other options may also need to be set before opening the handle, - * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), - * depending on usage requirements. - * @param[out] env The address where the new handle will be stored - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_create(MDB_env **env); - - /** @brief Open an environment handle. - * - * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] path The directory in which the database files reside. This - * directory must already exist and be writable. - * @param[in] flags Special options for this environment. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - * Flags set by mdb_env_set_flags() are also used. - *
      - *
    • #MDB_FIXEDMAP - * use a fixed address for the mmap region. This flag must be specified - * when creating the environment, and is stored persistently in the environment. - * If successful, the memory map will always reside at the same virtual address - * and pointers used to reference data items in the database will be constant - * across multiple invocations. This option may not always work, depending on - * how the operating system has allocated memory to shared libraries and other uses. - * The feature is highly experimental. - *
    • #MDB_NOSUBDIR - * By default, LMDB creates its environment in a directory whose - * pathname is given in \b path, and creates its data and lock files - * under that directory. With this option, \b path is used as-is for - * the database main data file. The database lock file is the \b path - * with "-lock" appended. - *
    • #MDB_RDONLY - * Open the environment in read-only mode. No write operations will be - * allowed. LMDB will still modify the lock file - except on read-only - * filesystems, where LMDB does not use locks. - *
    • #MDB_ROBUST - * Initialize the lockfile to catch stale locks if robust mutexes - * are supported, so aborted processes will not block others. - * Ignored when another process has the environment open. Unsupported - * by liblmdb built with MDB_USE_POSIX_SEM (such as BSD systems). - * Enabled by default on Windows. Some locking slowdown on Unix. - *
    • #MDB_WRITEMAP - * Use a writeable memory map unless MDB_RDONLY is set. This is faster - * and uses fewer mallocs, but loses protection from application bugs - * like wild pointer writes and other bad updates into the database. - * Incompatible with nested transactions. - * Processes with and without MDB_WRITEMAP on the same environment do - * not cooperate well. - *
    • #MDB_NOMETASYNC - * Flush system buffers to disk only once per transaction, omit the - * metadata flush. Defer that until the system flushes files to disk, - * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization - * maintains database integrity, but a system crash may undo the last - * committed transaction. I.e. it preserves the ACI (atomicity, - * consistency, isolation) but not D (durability) database property. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
    • #MDB_NOSYNC - * Don't flush system buffers to disk when committing a transaction. - * This optimization means a system crash can corrupt the database or - * lose the last transactions if buffers are not yet flushed to disk. - * The risk is governed by how often the system flushes dirty buffers - * to disk and how often #mdb_env_sync() is called. However, if the - * filesystem preserves write order and the #MDB_WRITEMAP flag is not - * used, transactions exhibit ACI (atomicity, consistency, isolation) - * properties and only lose D (durability). I.e. database integrity - * is maintained, but a system crash may undo the final transactions. - * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no - * hint for when to write transactions to disk, unless #mdb_env_sync() - * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
    • #MDB_MAPASYNC - * When using #MDB_WRITEMAP, use asynchronous flushes to disk. - * As with #MDB_NOSYNC, a system crash can then corrupt the - * database or lose the last transactions. Calling #mdb_env_sync() - * ensures on-disk database integrity until next commit. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
    • #MDB_NOTLS - * Don't use Thread-Local Storage. Tie reader locktable slots to - * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps - * the slot reseved for the #MDB_txn object. A thread may use parallel - * read-only transactions. A read-only transaction may span threads if - * the user synchronizes its use. Applications that multiplex many - * user threads over individual OS threads need this option. Such an - * application must also serialize the write transactions in an OS - * thread, since LMDB's write locking is unaware of the user threads. - *
    • #MDB_NOLOCK - * Don't do any locking. If concurrent access is anticipated, the - * caller must manage all concurrency itself. For proper operation - * the caller must enforce single-writer semantics, and must ensure - * that no readers are using old transactions while a writer is - * active. The simplest approach is to use an exclusive lock so that - * no readers may be active at all when a writer begins. - *
    • #MDB_NORDAHEAD - * Turn off readahead. Most operating systems perform readahead on - * read requests by default. This option turns it off if the OS - * supports it. Turning it off may help random read performance - * when the DB is larger than RAM and system RAM is full. - * The option is not implemented on Windows. - *
    • #MDB_NOMEMINIT - * Don't initialize malloc'd memory before writing to unused spaces - * in the data file. By default, memory for pages written to the data - * file is obtained using malloc. While these pages may be reused in - * subsequent transactions, freshly malloc'd pages will be initialized - * to zeroes before use. This avoids persisting leftover data from other - * code (that used the heap and subsequently freed the memory) into the - * data file. Note that many other system libraries may allocate - * and free memory from the heap for arbitrary uses. E.g., stdio may - * use the heap for file I/O buffers. This initialization step has a - * modest performance cost so some applications may want to disable - * it using this flag. This option can be a problem for applications - * which handle sensitive data like passwords, and it makes memory - * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, - * which writes directly to the mmap instead of using malloc for pages. The - * initialization is also skipped if #MDB_RESERVE is used; the - * caller is expected to overwrite all of the memory that was - * reserved in that case. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
    - * @param[in] mode The UNIX permissions to set on created files. This parameter - * is ignored on Windows. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • #MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the - * version that created the database environment. - *
    • #MDB_INVALID - the environment file headers are corrupted. - *
    • ENOENT - the directory specified by the path parameter doesn't exist. - *
    • EACCES - the user didn't have permission to access the environment files. - *
    • EAGAIN - the environment was locked by another process. - *
    - */ -int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); - - /** @brief Copy an LMDB environment to the specified path. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] path The directory in which the copy will reside. This - * directory must already exist and be writable but must otherwise be - * empty. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copy(MDB_env *env, const char *path); - - /** @brief Copy an LMDB environment to the specified file descriptor. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] fd The filedescriptor to write the copy to. It must - * have already been opened for Write access. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); - - /** @brief Copy an LMDB environment to the specified path, with options. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] path The directory in which the copy will reside. This - * directory must already exist and be writable but must otherwise be - * empty. - * @param[in] flags Special options for this operation. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *
      - *
    • #MDB_CP_COMPACT - Perform compaction while copying: omit free - * pages and sequentially renumber all pages in output. This option - * consumes more CPU and runs more slowly than the default. - *
    - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); - - /** @brief Copy an LMDB environment to the specified file descriptor, - * with options. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. See - * #mdb_env_copy2() for further details. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] fd The filedescriptor to write the copy to. It must - * have already been opened for Write access. - * @param[in] flags Special options for this operation. - * See #mdb_env_copy2() for options. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); - - /** @brief Return statistics about the LMDB environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] stat The address of an #MDB_stat structure - * where the statistics will be copied - */ -int mdb_env_stat(MDB_env *env, MDB_stat *stat); - - /** @brief Return information about the LMDB environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] stat The address of an #MDB_envinfo structure - * where the information will be copied - */ -int mdb_env_info(MDB_env *env, MDB_envinfo *stat); - - /** @brief Flush the data buffers to disk. - * - * Data is always written to disk when #mdb_txn_commit() is called, - * but the operating system may keep it buffered. LMDB always flushes - * the OS buffers upon commit as well, unless the environment was - * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] force If non-zero, force a synchronous flush. Otherwise - * if the environment has the #MDB_NOSYNC flag set the flushes - * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    • EIO - an error occurred during synchronization. - *
    - */ -int mdb_env_sync(MDB_env *env, int force); - - /** @brief Close the environment and release the memory map. - * - * Only a single thread may call this function. All transactions, databases, - * and cursors must already be closed before calling this function. Attempts to - * use any such handles after calling this function will cause a SIGSEGV. - * The environment handle will be freed and must not be used again after this call. - * @param[in] env An environment handle returned by #mdb_env_create() - */ -void mdb_env_close(MDB_env *env); - - /** @brief Set environment flags. - * - * This may be used to set some flags in addition to those from - * #mdb_env_open(), or to unset these flags. If several threads - * change the flags at the same time, the result is undefined. - * Most flags cannot be changed after #mdb_env_open(). - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] flags The flags to change, bitwise OR'ed together - * @param[in] onoff A non-zero value sets the flags, zero clears them. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); - - /** @brief Get environment flags. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] flags The address of an integer to store the flags - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_env_get_flags(MDB_env *env, unsigned int *flags); - - /** @brief Return the path that was used in #mdb_env_open(). - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] path Address of a string pointer to contain the path. This - * is the actual string in the environment, not a copy. It should not be - * altered in any way. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_env_get_path(MDB_env *env, const char **path); - - /** @brief Return the filedescriptor for the given environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); - - /** @brief Set the size of the memory map to use for this environment. - * - * The size should be a multiple of the OS page size. The default is - * 10485760 bytes. The size of the memory map is also the maximum size - * of the database. The value should be chosen as large as possible, - * to accommodate future growth of the database. - * This function should be called after #mdb_env_create() and before #mdb_env_open(). - * It may be called at later times if no transactions are active in - * this process. Note that the library does not check for this condition, - * the caller must ensure it explicitly. - * - * The new size takes effect immediately for the current process but - * will not be persisted to any others until a write transaction has been - * committed by the current process. Also, only mapsize increases are - * persisted into the environment. - * - * If the mapsize is increased by another process, and data has grown - * beyond the range of the current mapsize, #mdb_txn_begin() will - * return #MDB_MAP_RESIZED. This function may be called with a size - * of zero to adopt the new size. - * - * Any attempt to set a size smaller than the space already consumed - * by the environment will be silently changed to the current size of the used space. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] size The size in bytes - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified, or the environment has - * an active write transaction. - *
    - */ -int mdb_env_set_mapsize(MDB_env *env, size_t size); - - /** @brief Set the maximum number of threads/reader slots for the environment. - * - * This defines the number of slots in the lock table that is used to track readers in the - * the environment. The default is 126. - * Starting a read-only transaction normally ties a lock table slot to the - * current thread until the environment closes or the thread exits. If - * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the - * MDB_txn object until it or the #MDB_env object is destroyed. - * This function may only be called after #mdb_env_create() and before #mdb_env_open(). - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] readers The maximum number of reader lock table slots - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified, or the environment is already open. - *
    - */ -int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); - - /** @brief Get the maximum number of threads/reader slots for the environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] readers Address of an integer to store the number of readers - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); - - /** @brief Set the maximum number of named databases for the environment. - * - * This function is only needed if multiple databases will be used in the - * environment. Simpler applications that use the environment as a single - * unnamed database can ignore this option. - * This function may only be called after #mdb_env_create() and before #mdb_env_open(). - * - * Currently a moderate number of slots are cheap but a huge number gets - * expensive: 7-120 words per transaction, and every #mdb_dbi_open() - * does a linear search of the opened slots. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] dbs The maximum number of databases - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified, or the environment is already open. - *
    - */ -int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); - - /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. - * - * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. - * See @ref MDB_val. - * @param[in] env An environment handle returned by #mdb_env_create() - * @return The maximum size of a key we can write - */ -int mdb_env_get_maxkeysize(MDB_env *env); - - /** @brief Set application information associated with the #MDB_env. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] ctx An arbitrary pointer for whatever the application needs. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_set_userctx(MDB_env *env, void *ctx); - - /** @brief Get the application information associated with the #MDB_env. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @return The pointer set by #mdb_env_set_userctx(). - */ -void *mdb_env_get_userctx(MDB_env *env); - - /** @brief A callback function for most LMDB assert() failures, - * called before printing the message and aborting. - * - * @param[in] env An environment handle returned by #mdb_env_create(). - * @param[in] msg The assertion message, not including newline. - */ -typedef void MDB_assert_func(MDB_env *env, const char *msg); - - /** Set or reset the assert() callback of the environment. - * Disabled if liblmdb is buillt with NDEBUG. - * @note This hack should become obsolete as lmdb's error handling matures. - * @param[in] env An environment handle returned by #mdb_env_create(). - * @param[in] func An #MDB_assert_func function, or 0. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); - - /** @brief Create a transaction for use with the environment. - * - * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). - * @note A transaction and its cursors must only be used by a single - * thread, and a thread may only have a single transaction at a time. - * If #MDB_NOTLS is in use, this does not apply to read-only transactions. - * @note Cursors may not span transactions. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] parent If this parameter is non-NULL, the new transaction - * will be a nested transaction, with the transaction indicated by \b parent - * as its parent. Transactions may be nested to any level. A parent - * transaction and its cursors may not issue any other operations than - * mdb_txn_commit and mdb_txn_abort while it has active child transactions. - * @param[in] flags Special options for this transaction. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *
      - *
    • #MDB_RDONLY - * This transaction will not perform any write operations. - *
    - * @param[out] txn Address where the new #MDB_txn handle will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • #MDB_PANIC - a fatal error occurred earlier and the environment - * must be shut down. - *
    • #MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's - * mapsize and this environment's map must be resized as well. - * See #mdb_env_set_mapsize(). - *
    • #MDB_READERS_FULL - a read-only transaction was requested and - * the reader lock table is full. See #mdb_env_set_maxreaders(). - *
    • ENOMEM - out of memory. - *
    - */ -int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); - - /** @brief Returns the transaction's #MDB_env - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - */ -MDB_env *mdb_txn_env(MDB_txn *txn); - - /** @brief Commit all the operations of a transaction into the database. - * - * The transaction handle is freed. It and its cursors must not be used - * again after this call, except with #mdb_cursor_renew(). - * @note Earlier documentation incorrectly said all cursors would be freed. - * Only write-transactions free cursors. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    • ENOSPC - no more disk space. - *
    • EIO - a low-level I/O error occurred while writing. - *
    • ENOMEM - out of memory. - *
    - */ -int mdb_txn_commit(MDB_txn *txn); - - /** @brief Abandon all the operations of the transaction instead of saving them. - * - * The transaction handle is freed. It and its cursors must not be used - * again after this call, except with #mdb_cursor_renew(). - * @note Earlier documentation incorrectly said all cursors would be freed. - * Only write-transactions free cursors. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - */ -void mdb_txn_abort(MDB_txn *txn); - - /** @brief Reset a read-only transaction. - * - * Abort the transaction like #mdb_txn_abort(), but keep the transaction - * handle. #mdb_txn_renew() may reuse the handle. This saves allocation - * overhead if the process will start a new read-only transaction soon, - * and also locking overhead if #MDB_NOTLS is in use. The reader table - * lock is released, but the table slot stays tied to its thread or - * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free - * its lock table slot if MDB_NOTLS is in use. - * Cursors opened within the transaction must not be used - * again after this call, except with #mdb_cursor_renew(). - * Reader locks generally don't interfere with writers, but they keep old - * versions of database pages allocated. Thus they prevent the old pages - * from being reused when writers commit new data, and so under heavy load - * the database size may grow much more rapidly than otherwise. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - */ -void mdb_txn_reset(MDB_txn *txn); - - /** @brief Renew a read-only transaction. - * - * This acquires a new reader lock for a transaction handle that had been - * released by #mdb_txn_reset(). It must be called before a reset transaction - * may be used again. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • #MDB_PANIC - a fatal error occurred earlier and the environment - * must be shut down. - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_txn_renew(MDB_txn *txn); - -/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ -#define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) -/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ -#define mdb_close(env,dbi) mdb_dbi_close(env,dbi) - - /** @brief Open a database in the environment. - * - * A database handle denotes the name and parameters of a database, - * independently of whether such a database exists. - * The database handle may be discarded by calling #mdb_dbi_close(). - * The old database handle is returned if the database was already open. - * The handle may only be closed once. - * The database handle will be private to the current transaction until - * the transaction is successfully committed. If the transaction is - * aborted the handle will be closed automatically. - * After a successful commit the - * handle will reside in the shared environment, and may be used - * by other transactions. This function must not be called from - * multiple concurrent transactions. A transaction that uses this function - * must finish (either commit or abort) before any other transaction may - * use this function. - * - * To use named databases (with name != NULL), #mdb_env_set_maxdbs() - * must be called before opening the environment. Database names - * are kept as keys in the unnamed database. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] name The name of the database to open. If only a single - * database is needed in the environment, this value may be NULL. - * @param[in] flags Special options for this database. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *
      - *
    • #MDB_REVERSEKEY - * Keys are strings to be compared in reverse order, from the end - * of the strings to the beginning. By default, Keys are treated as strings and - * compared from beginning to end. - *
    • #MDB_DUPSORT - * Duplicate keys may be used in the database. (Or, from another perspective, - * keys may have multiple data items, stored in sorted order.) By default - * keys must be unique and may have only a single data item. - *
    • #MDB_INTEGERKEY - * Keys are binary integers in native byte order. Setting this option - * requires all keys to be the same size, typically sizeof(int) - * or sizeof(size_t). - *
    • #MDB_DUPFIXED - * This flag may only be used in combination with #MDB_DUPSORT. This option - * tells the library that the data items for this database are all the same - * size, which allows further optimizations in storage and retrieval. When - * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE - * cursor operations may be used to retrieve multiple items at once. - *
    • #MDB_INTEGERDUP - * This option specifies that duplicate data items are also integers, and - * should be sorted as such. - *
    • #MDB_REVERSEDUP - * This option specifies that duplicate data items should be compared as - * strings in reverse order. - *
    • #MDB_CREATE - * Create the named database if it doesn't exist. This option is not - * allowed in a read-only transaction or a read-only environment. - *
    - * @param[out] dbi Address where the new #MDB_dbi handle will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • #MDB_NOTFOUND - the specified database doesn't exist in the environment - * and #MDB_CREATE was not specified. - *
    • #MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). - *
    - */ -int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); - - /** @brief Retrieve statistics for a database. - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[out] stat The address of an #MDB_stat structure - * where the statistics will be copied - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); - - /** @brief Retrieve the DB flags for a database handle. - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[out] flags Address where the flags will be returned. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); - - /** @brief Close a database handle. Normally unnecessary. Use with care: - * - * This call is not mutex protected. Handles should only be closed by - * a single thread, and only if no other threads are going to reference - * the database handle or one of its cursors any further. Do not close - * a handle if an existing transaction has modified its database. - * Doing so can cause misbehavior from database corruption to errors - * like MDB_BAD_VALSIZE (since the DB name is gone). - * - * Closing a database handle is not necessary, but lets #mdb_dbi_open() - * reuse the handle value. Usually it's better to set a bigger - * #mdb_env_set_maxdbs(), unless that value would be large. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - */ -void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); - - /** @brief Empty or delete+close a database. - * - * See #mdb_dbi_close() for restrictions about closing the DB handle. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] del 0 to empty the DB, 1 to delete it from the - * environment and close the DB handle. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); - - /** @brief Set a custom key comparison function for a database. - * - * The comparison function is called whenever it is necessary to compare a - * key specified by the application with a key currently stored in the database. - * If no comparison function is specified, and no special key flags were specified - * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating - * before longer keys. - * @warning This function must be called before any data access functions are used, - * otherwise data corruption may occur. The same comparison function must be used by every - * program accessing the database, every time the database is used. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] cmp A #MDB_cmp_func function - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); - - /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. - * - * This comparison function is called whenever it is necessary to compare a data - * item specified by the application with a data item currently stored in the database. - * This function only takes effect if the database was opened with the #MDB_DUPSORT - * flag. - * If no comparison function is specified, and no special key flags were specified - * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating - * before longer items. - * @warning This function must be called before any data access functions are used, - * otherwise data corruption may occur. The same comparison function must be used by every - * program accessing the database, every time the database is used. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] cmp A #MDB_cmp_func function - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); - - /** @brief Set a relocation function for a #MDB_FIXEDMAP database. - * - * @todo The relocation function is called whenever it is necessary to move the data - * of an item to a different position in the database (e.g. through tree - * balancing operations, shifts as a result of adds or deletes, etc.). It is - * intended to allow address/position-dependent data items to be stored in - * a database in an environment opened with the #MDB_FIXEDMAP option. - * Currently the relocation feature is unimplemented and setting - * this function has no effect. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] rel A #MDB_rel_func function - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); - - /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. - * - * See #mdb_set_relfunc and #MDB_rel_func for more details. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] ctx An arbitrary pointer for whatever the application needs. - * It will be passed to the callback function set by #mdb_set_relfunc - * as its \b relctx parameter whenever the callback is invoked. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); - - /** @brief Get items from a database. - * - * This function retrieves key/data pairs from the database. The address - * and length of the data associated with the specified \b key are returned - * in the structure to which \b data refers. - * If the database supports duplicate keys (#MDB_DUPSORT) then the - * first data item for the key will be returned. Retrieval of other - * items requires the use of #mdb_cursor_get(). - * - * @note The memory pointed to by the returned values is owned by the - * database. The caller need not dispose of the memory, and may not - * modify it in any way. For values returned in a read-only transaction - * any modification attempts will cause a SIGSEGV. - * @note Values returned from the database are valid only until a - * subsequent update operation, or the end of the transaction. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] key The key to search for in the database - * @param[out] data The data corresponding to the key - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • #MDB_NOTFOUND - the key was not in the database. - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); - - /** @brief Store items into a database. - * - * This function stores key/data pairs in the database. The default behavior - * is to enter the new key/data pair, replacing any previously existing key - * if duplicates are disallowed, or adding a duplicate data item if - * duplicates are allowed (#MDB_DUPSORT). - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] key The key to store in the database - * @param[in,out] data The data to store - * @param[in] flags Special options for this operation. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *
      - *
    • #MDB_NODUPDATA - enter the new key/data pair only if it does not - * already appear in the database. This flag may only be specified - * if the database was opened with #MDB_DUPSORT. The function will - * return #MDB_KEYEXIST if the key/data pair already appears in the - * database. - *
    • #MDB_NOOVERWRITE - enter the new key/data pair only if the key - * does not already appear in the database. The function will return - * #MDB_KEYEXIST if the key already appears in the database, even if - * the database supports duplicates (#MDB_DUPSORT). The \b data - * parameter will be set to point to the existing item. - *
    • #MDB_RESERVE - reserve space for data of the given size, but - * don't copy the given data. Instead, return a pointer to the - * reserved space, which the caller can fill in later - before - * the next update operation or the transaction ends. This saves - * an extra memcpy if the data is being generated later. - * LMDB does nothing else with this memory, the caller is expected - * to modify all of the space requested. - *
    • #MDB_APPEND - append the given key/data pair to the end of the - * database. No key comparisons are performed. This option allows - * fast bulk loading when keys are already known to be in the - * correct order. Loading unsorted keys with this flag will cause - * data corruption. - *
    • #MDB_APPENDDUP - as above, but for sorted dup data. - *
    - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). - *
    • #MDB_TXN_FULL - the transaction has too many dirty pages. - *
    • EACCES - an attempt was made to write in a read-only transaction. - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, - unsigned int flags); - - /** @brief Delete items from a database. - * - * This function removes key/data pairs from the database. - * If the database does not support sorted duplicate data items - * (#MDB_DUPSORT) the data parameter is ignored. - * If the database supports sorted duplicates and the data parameter - * is NULL, all of the duplicate data items for the key will be - * deleted. Otherwise, if the data parameter is non-NULL - * only the matching data item will be deleted. - * This function will return #MDB_NOTFOUND if the specified key/data - * pair is not in the database. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] key The key to delete from the database - * @param[in] data The data to delete - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EACCES - an attempt was made to write in a read-only transaction. - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); - - /** @brief Create a cursor handle. - * - * A cursor is associated with a specific transaction and database. - * A cursor cannot be used when its database handle is closed. Nor - * when its transaction has ended, except with #mdb_cursor_renew(). - * It can be discarded with #mdb_cursor_close(). - * A cursor in a write-transaction can be closed before its transaction - * ends, and will otherwise be closed when its transaction ends. - * A cursor in a read-only transaction must be closed explicitly, before - * or after its transaction ends. It can be reused with - * #mdb_cursor_renew() before finally closing it. - * @note Earlier documentation said that cursors in every transaction - * were closed when the transaction committed or aborted. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[out] cursor Address where the new #MDB_cursor handle will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); - - /** @brief Close a cursor handle. - * - * The cursor handle will be freed and must not be used again after this call. - * Its transaction must still be live if it is a write-transaction. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - */ -void mdb_cursor_close(MDB_cursor *cursor); - - /** @brief Renew a cursor handle. - * - * A cursor is associated with a specific transaction and database. - * Cursors that are only used in read-only - * transactions may be re-used, to avoid unnecessary malloc/free overhead. - * The cursor may be associated with a new read-only transaction, and - * referencing the same database handle as it was created with. - * This may be done whether the previous transaction is live or dead. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); - - /** @brief Return the cursor's transaction handle. - * - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - */ -MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); - - /** @brief Return the cursor's database handle. - * - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - */ -MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); - - /** @brief Retrieve by cursor. - * - * This function retrieves key/data pairs from the database. The address and length - * of the key are returned in the object to which \b key refers (except for the - * case of the #MDB_SET option, in which the \b key object is unchanged), and - * the address and length of the data are returned in the object to which \b data - * refers. - * See #mdb_get() for restrictions on using the output values. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[in,out] key The key for a retrieved item - * @param[in,out] data The data of a retrieved item - * @param[in] op A cursor operation #MDB_cursor_op - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • #MDB_NOTFOUND - no matching key found. - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, - MDB_cursor_op op); - - /** @brief Store by cursor. - * - * This function stores key/data pairs into the database. - * The cursor is positioned at the new item, or on failure usually near it. - * @note Earlier documentation incorrectly said errors would leave the - * state of the cursor unchanged. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[in] key The key operated on. - * @param[in] data The data operated on. - * @param[in] flags Options for this operation. This parameter - * must be set to 0 or one of the values described here. - *
      - *
    • #MDB_CURRENT - replace the item at the current cursor position. - * The \b key parameter must still be provided, and must match it. - * If using sorted duplicates (#MDB_DUPSORT) the data item must still - * sort into the same place. This is intended to be used when the - * new data is the same size as the old. Otherwise it will simply - * perform a delete of the old record followed by an insert. - *
    • #MDB_NODUPDATA - enter the new key/data pair only if it does not - * already appear in the database. This flag may only be specified - * if the database was opened with #MDB_DUPSORT. The function will - * return #MDB_KEYEXIST if the key/data pair already appears in the - * database. - *
    • #MDB_NOOVERWRITE - enter the new key/data pair only if the key - * does not already appear in the database. The function will return - * #MDB_KEYEXIST if the key already appears in the database, even if - * the database supports duplicates (#MDB_DUPSORT). - *
    • #MDB_RESERVE - reserve space for data of the given size, but - * don't copy the given data. Instead, return a pointer to the - * reserved space, which the caller can fill in later. This saves - * an extra memcpy if the data is being generated later. - *
    • #MDB_APPEND - append the given key/data pair to the end of the - * database. No key comparisons are performed. This option allows - * fast bulk loading when keys are already known to be in the - * correct order. Loading unsorted keys with this flag will cause - * data corruption. - *
    • #MDB_APPENDDUP - as above, but for sorted dup data. - *
    • #MDB_MULTIPLE - store multiple contiguous data elements in a - * single request. This flag may only be specified if the database - * was opened with #MDB_DUPFIXED. The \b data argument must be an - * array of two MDB_vals. The mv_size of the first MDB_val must be - * the size of a single data element. The mv_data of the first MDB_val - * must point to the beginning of the array of contiguous data elements. - * The mv_size of the second MDB_val must be the count of the number - * of data elements to store. On return this field will be set to - * the count of the number of elements actually written. The mv_data - * of the second MDB_val is unused. - *
    - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). - *
    • #MDB_TXN_FULL - the transaction has too many dirty pages. - *
    • EACCES - an attempt was made to modify a read-only database. - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, - unsigned int flags); - - /** @brief Delete current key/data pair - * - * This function deletes the key/data pair to which the cursor refers. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[in] flags Options for this operation. This parameter - * must be set to 0 or one of the values described here. - *
      - *
    • #MDB_NODUPDATA - delete all of the data items for the current key. - * This flag may only be specified if the database was opened with #MDB_DUPSORT. - *
    - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EACCES - an attempt was made to modify a read-only database. - *
    • EINVAL - an invalid parameter was specified. - *
    - */ -int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); - - /** @brief Return count of duplicates for current key. - * - * This call is only valid on databases that support sorted duplicate - * data items #MDB_DUPSORT. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[out] countp Address where the count will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
      - *
    • EINVAL - cursor is not initialized, or an invalid parameter was specified. - *
    - */ -int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); - - /** @brief Compare two data items according to a particular database. - * - * This returns a comparison as if the two data items were keys in the - * specified database. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] a The first item to compare - * @param[in] b The second item to compare - * @return < 0 if a < b, 0 if a == b, > 0 if a > b - */ -int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); - - /** @brief Compare two data items according to a particular database. - * - * This returns a comparison as if the two items were data items of - * the specified database. The database must have the #MDB_DUPSORT flag. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] a The first item to compare - * @param[in] b The second item to compare - * @return < 0 if a < b, 0 if a == b, > 0 if a > b - */ -int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); - - /** @brief A callback function used to print a message from the library. - * - * @param[in] msg The string to be printed. - * @param[in] ctx An arbitrary context pointer for the callback. - * @return < 0 on failure, >= 0 on success. - */ -typedef int (MDB_msg_func)(const char *msg, void *ctx); - - /** @brief Dump the entries in the reader lock table. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] func A #MDB_msg_func function - * @param[in] ctx Anything the message function needs - * @return < 0 on failure, >= 0 on success. - */ -int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); - - /** @brief Check for stale entries in the reader lock table. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] dead Number of stale slots that were cleared - * @return 0 on success, non-zero on failure. - */ -int mdb_reader_check(MDB_env *env, int *dead); -/** @} */ - -#ifdef __cplusplus -} -#endif -/** @page tools LMDB Command Line Tools - The following describes the command line tools that are available for LMDB. - \li \ref mdb_copy_1 - \li \ref mdb_dump_1 - \li \ref mdb_load_1 - \li \ref mdb_stat_1 -*/ - -#endif /* _LMDB_H_ */ diff --git a/submodules/lmdb/libraries/liblmdb/mdb.c b/submodules/lmdb/libraries/liblmdb/mdb.c deleted file mode 100644 index 6179b57868..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mdb.c +++ /dev/null @@ -1,9519 +0,0 @@ -/** @file mdb.c - * @brief Lightning memory-mapped database library - * - * A Btree-based database management library modeled loosely on the - * BerkeleyDB API, but much simplified. - */ -/* - * Copyright 2011-2014 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - * - * This code is derived from btree.c written by Martin Hedenfalk. - * - * Copyright (c) 2009, 2010 Martin Hedenfalk - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE 1 -#endif -#ifdef _WIN32 -#include -#include -/** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it - * as int64 which is wrong. MSVC doesn't define it at all, so just - * don't use it. - */ -#define MDB_PID_T int -#define MDB_THR_T DWORD -#include -#include -#ifdef __GNUC__ -# include -#else -# define LITTLE_ENDIAN 1234 -# define BIG_ENDIAN 4321 -# define BYTE_ORDER LITTLE_ENDIAN -# ifndef SSIZE_MAX -# define SSIZE_MAX INT_MAX -# endif -#endif -#else -#include -#include -#define MDB_PID_T pid_t -#define MDB_THR_T pthread_t -#include -#include -#include -#ifdef HAVE_SYS_FILE_H -#include -#endif -#include -#endif - -#if defined(__mips) && defined(__linux) -/* MIPS has cache coherency issues, requires explicit cache control */ -#include -extern int cacheflush(char *addr, int nbytes, int cache); -#define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache) -#else -#define CACHEFLUSH(addr, bytes, cache) -#endif - - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__sun) || defined(ANDROID) -/* Most platforms have posix_memalign, older may only have memalign */ -#define HAVE_MEMALIGN 1 -#include -#endif - -#if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) -#include -#include /* defines BYTE_ORDER on HPUX and Solaris */ -#endif - -#if defined(__APPLE__) || defined (BSD) -# define MDB_USE_POSIX_SEM 1 -# define MDB_FDATASYNC fsync -#elif defined(ANDROID) -# define MDB_FDATASYNC fsync -#endif - -#ifndef _WIN32 -#include -#ifdef MDB_USE_POSIX_SEM -# define MDB_USE_HASH 1 -#include -#endif -#endif - -#ifdef USE_VALGRIND -#include -#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) -#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) -#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) -#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) -#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) -#else -#define VGMEMP_CREATE(h,r,z) -#define VGMEMP_ALLOC(h,a,s) -#define VGMEMP_FREE(h,a) -#define VGMEMP_DESTROY(h) -#define VGMEMP_DEFINED(a,s) -#endif - -#ifndef BYTE_ORDER -# if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) -/* Solaris just defines one or the other */ -# define LITTLE_ENDIAN 1234 -# define BIG_ENDIAN 4321 -# ifdef _LITTLE_ENDIAN -# define BYTE_ORDER LITTLE_ENDIAN -# else -# define BYTE_ORDER BIG_ENDIAN -# endif -# else -# define BYTE_ORDER __BYTE_ORDER -# endif -#endif - -#ifndef LITTLE_ENDIAN -#define LITTLE_ENDIAN __LITTLE_ENDIAN -#endif -#ifndef BIG_ENDIAN -#define BIG_ENDIAN __BIG_ENDIAN -#endif - -#if defined(__i386) || defined(__x86_64) || defined(_M_IX86) -#define MISALIGNED_OK 1 -#endif - -#include "lmdb.h" -#include "midl.h" - -#if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) -# error "Unknown or unsupported endianness (BYTE_ORDER)" -#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -# error "Two's complement, reasonably sized integer types, please" -#endif - -#ifdef __GNUC__ -/** Put infrequently used env functions in separate section */ -# ifdef __APPLE__ -# define ESECT __attribute__ ((section("__TEXT,text_env"))) -# else -# define ESECT __attribute__ ((section("text_env"))) -# endif -#else -#define ESECT -#endif - -/** @defgroup internal LMDB Internals - * @{ - */ -/** @defgroup compat Compatibility Macros - * A bunch of macros to minimize the amount of platform-specific ifdefs - * needed throughout the rest of the code. When the features this library - * needs are similar enough to POSIX to be hidden in a one-or-two line - * replacement, this macro approach is used. - * @{ - */ - - /** Features under development */ -#ifndef MDB_DEVEL -#define MDB_DEVEL 0 -#endif - -#if MDB_DEVEL && (defined(_WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_POSIX_SEM))) -#define MDB_ROBUST_SUPPORTED 1 -#endif - - /** Wrapper around __func__, which is a C99 feature */ -#if __STDC_VERSION__ >= 199901L -# define mdb_func_ __func__ -#elif __GNUC__ >= 2 || _MSC_VER >= 1300 -# define mdb_func_ __FUNCTION__ -#else -/* If a debug message says (), update the #if statements above */ -# define mdb_func_ "" -#endif - -#ifdef _WIN32 -#define MDB_USE_HASH 1 -#define MDB_PIDLOCK 0 -#define THREAD_RET DWORD -#define pthread_t HANDLE -#define pthread_mutex_t HANDLE -#define pthread_cond_t HANDLE -typedef HANDLE mdb_mutex_t; -#define pthread_key_t DWORD -#define pthread_self() GetCurrentThreadId() -#define pthread_key_create(x,y) \ - ((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0) -#define pthread_key_delete(x) TlsFree(x) -#define pthread_getspecific(x) TlsGetValue(x) -#define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) -#define pthread_mutex_consistent(mutex) 0 -#define pthread_mutex_unlock(x) ReleaseMutex(*x) -#define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) -#define pthread_cond_signal(x) SetEvent(*x) -#define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) -#define THREAD_CREATE(thr,start,arg) thr=CreateThread(NULL,0,start,arg,0,NULL) -#define THREAD_FINISH(thr) WaitForSingleObject(thr, INFINITE) -#define MDB_MUTEX(env, rw) ((env)->me_##rw##mutex) -#define LOCK_MUTEX0(mutex) WaitForSingleObject(mutex, INFINITE) -#define UNLOCK_MUTEX(mutex) ReleaseMutex(mutex) -#define getpid() GetCurrentProcessId() -#define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) -#define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) -#define ErrCode() GetLastError() -#define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;} -#define close(fd) (CloseHandle(fd) ? 0 : -1) -#define munmap(ptr,len) UnmapViewOfFile(ptr) -#ifdef PROCESS_QUERY_LIMITED_INFORMATION -#define MDB_PROCESS_QUERY_LIMITED_INFORMATION PROCESS_QUERY_LIMITED_INFORMATION -#else -#define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 -#endif -#define Z "I" -#else -#define THREAD_RET void * -#define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) -#define THREAD_FINISH(thr) pthread_join(thr,NULL) -#define Z "z" /**< printf format modifier for size_t */ - - /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ -#define MDB_PIDLOCK 1 - -#ifdef MDB_USE_POSIX_SEM - -typedef sem_t *mdb_mutex_t; -#define MDB_MUTEX(env, rw) ((env)->me_##rw##mutex) -#define LOCK_MUTEX0(mutex) mdb_sem_wait(mutex) -#define UNLOCK_MUTEX(mutex) sem_post(mutex) - -static int -mdb_sem_wait(sem_t *sem) -{ - int rc; - while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ; - return rc; -} - -#else - /** Pointer/HANDLE type of shared mutex/semaphore. - */ -typedef pthread_mutex_t *mdb_mutex_t; - /** Mutex for the reader table (rw = r) or write transaction (rw = w). - */ -#define MDB_MUTEX(env, rw) (&(env)->me_txns->mti_##rw##mutex) - /** Lock the reader or writer mutex. - * Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX(). - */ -#define LOCK_MUTEX0(mutex) pthread_mutex_lock(mutex) - /** Unlock the reader or writer mutex. - */ -#define UNLOCK_MUTEX(mutex) pthread_mutex_unlock(mutex) -#endif /* MDB_USE_POSIX_SEM */ - - /** Get the error code for the last failed system function. - */ -#define ErrCode() errno - - /** An abstraction for a file handle. - * On POSIX systems file handles are small integers. On Windows - * they're opaque pointers. - */ -#define HANDLE int - - /** A value for an invalid file handle. - * Mainly used to initialize file variables and signify that they are - * unused. - */ -#define INVALID_HANDLE_VALUE (-1) - - /** Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. - */ -#define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) -#endif - -#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) -#define MNAME_LEN 32 -#else -#define MNAME_LEN (sizeof(pthread_mutex_t)) -#endif - -/** @} */ - -#ifdef MDB_ROBUST_SUPPORTED - /** Lock mutex, handle any error, set rc = result. - * Return 0 on success, nonzero (not rc) on error. - */ -#define LOCK_MUTEX(rc, env, mutex) \ - (((rc) = LOCK_MUTEX0(mutex)) && \ - ((rc) = mdb_mutex_failed(env, mutex, rc))) -static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t mutex, int rc); -#else -#define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex)) -#define mdb_mutex_failed(env, mutex, rc) (rc) -#endif - -#ifndef _WIN32 -/** A flag for opening a file and requesting synchronous data writes. - * This is only used when writing a meta page. It's not strictly needed; - * we could just do a normal write and then immediately perform a flush. - * But if this flag is available it saves us an extra system call. - * - * @note If O_DSYNC is undefined but exists in /usr/include, - * preferably set some compiler flag to get the definition. - * Otherwise compile with the less efficient -DMDB_DSYNC=O_SYNC. - */ -#ifndef MDB_DSYNC -# define MDB_DSYNC O_DSYNC -#endif -#endif - -/** Function for flushing the data of a file. Define this to fsync - * if fdatasync() is not supported. - */ -#ifndef MDB_FDATASYNC -# define MDB_FDATASYNC fdatasync -#endif - -#ifndef MDB_MSYNC -# define MDB_MSYNC(addr,len,flags) msync(addr,len,flags) -#endif - -#ifndef MS_SYNC -#define MS_SYNC 1 -#endif - -#ifndef MS_ASYNC -#define MS_ASYNC 0 -#endif - - /** A page number in the database. - * Note that 64 bit page numbers are overkill, since pages themselves - * already represent 12-13 bits of addressable memory, and the OS will - * always limit applications to a maximum of 63 bits of address space. - * - * @note In the #MDB_node structure, we only store 48 bits of this value, - * which thus limits us to only 60 bits of addressable data. - */ -typedef MDB_ID pgno_t; - - /** A transaction ID. - * See struct MDB_txn.mt_txnid for details. - */ -typedef MDB_ID txnid_t; - -/** @defgroup debug Debug Macros - * @{ - */ -#ifndef MDB_DEBUG - /** Enable debug output. Needs variable argument macros (a C99 feature). - * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs - * read from and written to the database (used for free space management). - */ -#define MDB_DEBUG 0 -#endif - -#if MDB_DEBUG -static int mdb_debug; -static txnid_t mdb_debug_start; - - /** Print a debug message with printf formatting. - * Requires double parenthesis around 2 or more args. - */ -# define DPRINTF(args) ((void) ((mdb_debug) && DPRINTF0 args)) -# define DPRINTF0(fmt, ...) \ - fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__) -#else -# define DPRINTF(args) ((void) 0) -#endif - /** Print a debug string. - * The string is printed literally, with no format processing. - */ -#define DPUTS(arg) DPRINTF(("%s", arg)) - /** Debuging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) -/** @} */ - - /** @brief The maximum size of a database page. - * - * It is 32k or 64k, since value-PAGEBASE must fit in - * #MDB_page.%mp_upper. - * - * LMDB will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. - */ -#define MAX_PAGESIZE (PAGEBASE ? 0x10000 : 0x8000) - - /** The minimum number of keys required in a database page. - * Setting this to a larger value will place a smaller bound on the - * maximum size of a data item. Data items larger than this size will - * be pushed into overflow pages instead of being stored directly in - * the B-tree node. This value used to default to 4. With a page size - * of 4096 bytes that meant that any item larger than 1024 bytes would - * go into an overflow page. That also meant that on average 2-3KB of - * each overflow page was wasted space. The value cannot be lower than - * 2 because then there would no longer be a tree structure. With this - * value, items larger than 2KB will go into overflow pages, and on - * average only 1KB will be wasted. - */ -#define MDB_MINKEYS 2 - - /** A stamp that identifies a file as an LMDB file. - * There's nothing special about this value other than that it is easily - * recognizable, and it will reflect any byte order mismatches. - */ -#define MDB_MAGIC 0xBEEFC0DE - - /** The version number for a database's datafile format. */ -#define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 1) - /** The version number for a database's lockfile format. */ -#define MDB_LOCK_VERSION ((MDB_DEVEL) ? 999 : 1) - - /** @brief The max size of a key we can write, or 0 for dynamic max. - * - * Define this as 0 to compute the max from the page size. 511 - * is default for backwards compat: liblmdb <= 0.9.10 can break - * when modifying a DB with keys/dupsort data bigger than its max. - * #MDB_DEVEL sets the default to 0. - * - * Data items in an #MDB_DUPSORT database are also limited to - * this size, since they're actually keys of a sub-DB. Keys and - * #MDB_DUPSORT data items must fit on a node in a regular page. - */ -#ifndef MDB_MAXKEYSIZE -#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) -#endif - - /** The maximum size of a key we can write to the environment. */ -#if MDB_MAXKEYSIZE -#define ENV_MAXKEY(env) (MDB_MAXKEYSIZE) -#else -#define ENV_MAXKEY(env) ((env)->me_maxkey) -#endif - - /** @brief The maximum size of a data item. - * - * We only store a 32 bit value for node sizes. - */ -#define MAXDATASIZE 0xffffffffUL - -#if MDB_DEBUG - /** Key size which fits in a #DKBUF. - * @ingroup debug - */ -#define DKBUF_MAXKEYSIZE ((MDB_MAXKEYSIZE) > 0 ? (MDB_MAXKEYSIZE) : 511) - /** A key buffer. - * @ingroup debug - * This is used for printing a hex dump of a key's contents. - */ -#define DKBUF char kbuf[DKBUF_MAXKEYSIZE*2+1] - /** Display a key in hex. - * @ingroup debug - * Invoke a function to display a key in hex. - */ -#define DKEY(x) mdb_dkey(x, kbuf) -#else -#define DKBUF -#define DKEY(x) 0 -#endif - - /** An invalid page number. - * Mainly used to denote an empty tree. - */ -#define P_INVALID (~(pgno_t)0) - - /** Test if the flags \b f are set in a flag word \b w. */ -#define F_ISSET(w, f) (((w) & (f)) == (f)) - - /** Round \b n up to an even number. */ -#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ - - /** Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. - */ -typedef uint16_t indx_t; - - /** Default size of memory map. - * This is certainly too small for any actual applications. Apps should always set - * the size explicitly using #mdb_env_set_mapsize(). - */ -#define DEFAULT_MAPSIZE 1048576 - -/** @defgroup readers Reader Lock Table - * Readers don't acquire any locks for their data access. Instead, they - * simply record their transaction ID in the reader table. The reader - * mutex is needed just to find an empty slot in the reader table. The - * slot's address is saved in thread-specific data so that subsequent read - * transactions started by the same thread need no further locking to proceed. - * - * If #MDB_NOTLS is set, the slot address is not saved in thread-specific data. - * - * No reader table is used if the database is on a read-only filesystem, or - * if #MDB_NOLOCK is set. - * - * Since the database uses multi-version concurrency control, readers don't - * actually need any locking. This table is used to keep track of which - * readers are using data from which old transactions, so that we'll know - * when a particular old transaction is no longer in use. Old transactions - * that have discarded any data pages can then have those pages reclaimed - * for use by a later write transaction. - * - * The lock table is constructed such that reader slots are aligned with the - * processor's cache line size. Any slot is only ever used by one thread. - * This alignment guarantees that there will be no contention or cache - * thrashing as threads update their own slot info, and also eliminates - * any need for locking when accessing a slot. - * - * A writer thread will scan every slot in the table to determine the oldest - * outstanding reader transaction. Any freed pages older than this will be - * reclaimed by the writer. The writer doesn't use any locks when scanning - * this table. This means that there's no guarantee that the writer will - * see the most up-to-date reader info, but that's not required for correct - * operation - all we need is to know the upper bound on the oldest reader, - * we don't care at all about the newest reader. So the only consequence of - * reading stale information here is that old pages might hang around a - * while longer before being reclaimed. That's actually good anyway, because - * the longer we delay reclaiming old pages, the more likely it is that a - * string of contiguous pages can be found after coalescing old pages from - * many old transactions together. - * @{ - */ - /** Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. 126 readers plus a - * couple mutexes fit exactly into 8KB on my development machine. - * Applications should set the table size using #mdb_env_set_maxreaders(). - */ -#define DEFAULT_READERS 126 - - /** The size of a CPU cache line in bytes. We want our lock structures - * aligned to this size to avoid false cache line sharing in the - * lock table. - * This value works for most CPUs. For Itanium this should be 128. - */ -#ifndef CACHELINE -#define CACHELINE 64 -#endif - - /** The information we store in a single slot of the reader table. - * In addition to a transaction ID, we also record the process and - * thread ID that owns a slot, so that we can detect stale information, - * e.g. threads or processes that went away without cleaning up. - * @note We currently don't check for stale records. We simply re-init - * the table when we know that we're the only process opening the - * lock file. - */ -typedef struct MDB_rxbody { - /** Current Transaction ID when this transaction began, or (txnid_t)-1. - * Multiple readers that start at the same time will probably have the - * same ID here. Again, it's not important to exclude them from - * anything; all we need to know is which version of the DB they - * started from so we can avoid overwriting any data used in that - * particular version. - */ - txnid_t mrb_txnid; - /** The process ID of the process owning this reader txn. */ - MDB_PID_T mrb_pid; - /** The thread ID of the thread owning this txn. */ - MDB_THR_T mrb_tid; -} MDB_rxbody; - - /** The actual reader record, with cacheline padding. */ -typedef struct MDB_reader { - union { - MDB_rxbody mrx; - /** shorthand for mrb_txnid */ -#define mr_txnid mru.mrx.mrb_txnid -#define mr_pid mru.mrx.mrb_pid -#define mr_tid mru.mrx.mrb_tid - /** cache line alignment */ - char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)]; - } mru; -} MDB_reader; - - /** The header for the reader table. - * The table resides in a memory-mapped file. (This is a different file - * than is used for the main database.) - * - * For POSIX the actual mutexes reside in the shared memory of this - * mapped file. On Windows, mutexes are named objects allocated by the - * kernel; we store the mutex names in this mapped file so that other - * processes can grab them. This same approach is also used on - * MacOSX/Darwin (using named semaphores) since MacOSX doesn't support - * process-shared POSIX mutexes. For these cases where a named object - * is used, the object name is derived from a 64 bit FNV hash of the - * environment pathname. As such, naming collisions are extremely - * unlikely. If a collision occurs, the results are unpredictable. - */ -typedef struct MDB_txbody { - /** Stamp identifying this as an LMDB file. It must be set - * to #MDB_MAGIC. */ - uint32_t mtb_magic; - /** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */ - uint32_t mtb_format; -#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) - char mtb_rmname[MNAME_LEN]; -#else - /** Mutex protecting access to this table. - * This is the #MDB_MUTEX(env,r) reader table lock. - */ - pthread_mutex_t mtb_rmutex; -#endif - /** The ID of the last transaction committed to the database. - * This is recorded here only for convenience; the value can always - * be determined by reading the main database meta pages. - */ - txnid_t mtb_txnid; - /** The number of slots that have been used in the reader table. - * This always records the maximum count, it is not decremented - * when readers release their slots. - */ - unsigned mtb_numreaders; - /** Flags which the lock file was initialized with. */ - unsigned mtb_flags; -} MDB_txbody; - - /** The actual reader table definition. */ -typedef struct MDB_txninfo { - union { - MDB_txbody mtb; -#define mti_magic mt1.mtb.mtb_magic -#define mti_format mt1.mtb.mtb_format -#define mti_rmutex mt1.mtb.mtb_rmutex -#define mti_rmname mt1.mtb.mtb_rmname -#define mti_txnid mt1.mtb.mtb_txnid -#define mti_numreaders mt1.mtb.mtb_numreaders -#define mti_flags mt1.mtb.mtb_flags - char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)]; - } mt1; - union { -#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) - char mt2_wmname[MNAME_LEN]; -#define mti_wmname mt2.mt2_wmname -#else - pthread_mutex_t mt2_wmutex; -#define mti_wmutex mt2.mt2_wmutex -#endif - char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)]; - } mt2; - MDB_reader mti_readers[1]; -} MDB_txninfo; - - /** Lockfile format signature: version, features and field layout */ -#define MDB_LOCK_FORMAT \ - ((uint32_t) \ - ((MDB_LOCK_VERSION) \ - /* Flags which describe functionality */ \ - + (((MDB_PIDLOCK) != 0) << 16))) -/** @} */ - -/** Common header for all page types. - * Overflow records occupy a number of contiguous pages with no - * headers on any page after the first. - */ -typedef struct MDB_page { -#define mp_pgno mp_p.p_pgno -#define mp_next mp_p.p_next - union { - pgno_t p_pgno; /**< page number */ - struct MDB_page *p_next; /**< for in-memory list of freed pages */ - } mp_p; - uint16_t mp_pad; -/** @defgroup mdb_page Page Flags - * @ingroup internal - * Flags for the page headers. - * @{ - */ -#define P_BRANCH 0x01 /**< branch page */ -#define P_LEAF 0x02 /**< leaf page */ -#define P_OVERFLOW 0x04 /**< overflow page */ -#define P_META 0x08 /**< meta page */ -#define P_DIRTY 0x10 /**< dirty page, also set for #P_SUBP pages */ -#define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ -#define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ -#define P_LOOSE 0x4000 /**< page was dirtied then freed, can be reused */ -#define P_KEEP 0x8000 /**< leave this page alone during spill */ -/** @} */ - uint16_t mp_flags; /**< @ref mdb_page */ -#define mp_lower mp_pb.pb.pb_lower -#define mp_upper mp_pb.pb.pb_upper -#define mp_pages mp_pb.pb_pages - union { - struct { - indx_t pb_lower; /**< lower bound of free space */ - indx_t pb_upper; /**< upper bound of free space */ - } pb; - uint32_t pb_pages; /**< number of overflow pages */ - } mp_pb; - indx_t mp_ptrs[1]; /**< dynamic size */ -} MDB_page; - - /** Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ ((unsigned) offsetof(MDB_page, mp_ptrs)) - - /** Address of first usable data byte in a page, after the header */ -#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ)) - - /** ITS#7713, change PAGEBASE to handle 65536 byte pages */ -#define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0) - - /** Number of nodes on a page */ -#define NUMKEYS(p) (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1) - - /** The amount of space remaining in the page */ -#define SIZELEFT(p) (indx_t)((p)->mp_upper - (p)->mp_lower) - - /** The percentage of space used in the page, in tenths of a percent. */ -#define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \ - ((env)->me_psize - PAGEHDRSZ)) - /** The minimum page fill factor, in tenths of a percent. - * Pages emptier than this are candidates for merging. - */ -#define FILL_THRESHOLD 250 - - /** Test if a page is a leaf page */ -#define IS_LEAF(p) F_ISSET((p)->mp_flags, P_LEAF) - /** Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) F_ISSET((p)->mp_flags, P_LEAF2) - /** Test if a page is a branch page */ -#define IS_BRANCH(p) F_ISSET((p)->mp_flags, P_BRANCH) - /** Test if a page is an overflow page */ -#define IS_OVERFLOW(p) F_ISSET((p)->mp_flags, P_OVERFLOW) - /** Test if a page is a sub page */ -#define IS_SUBP(p) F_ISSET((p)->mp_flags, P_SUBP) - - /** The number of overflow pages needed to store the given size. */ -#define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) - - /** Link in #MDB_txn.%mt_loose_pgs list */ -#define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) - - /** Header for a single key/data pair within a page. - * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. - * We guarantee 2-byte alignment for 'MDB_node's. - */ -typedef struct MDB_node { - /** lo and hi are used for data size on leaf nodes and for - * child pgno on branch nodes. On 64 bit platforms, flags - * is also used for pgno. (Branch nodes have no flags). - * They are in host byte order in case that lets some - * accesses be optimized into a 32-bit word access. - */ -#if BYTE_ORDER == LITTLE_ENDIAN - unsigned short mn_lo, mn_hi; /**< part of data size or pgno */ -#else - unsigned short mn_hi, mn_lo; -#endif -/** @defgroup mdb_node Node Flags - * @ingroup internal - * Flags for node headers. - * @{ - */ -#define F_BIGDATA 0x01 /**< data put on overflow page */ -#define F_SUBDATA 0x02 /**< data is a sub-database */ -#define F_DUPDATA 0x04 /**< data has duplicates */ - -/** valid flags for #mdb_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND) - -/** @} */ - unsigned short mn_flags; /**< @ref mdb_node */ - unsigned short mn_ksize; /**< key size */ - char mn_data[1]; /**< key and data are appended here */ -} MDB_node; - - /** Size of the node header, excluding dynamic data at the end */ -#define NODESIZE offsetof(MDB_node, mn_data) - - /** Bit position of top word in page number, for shifting mn_flags */ -#define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0) - - /** Size of a node in a branch page with a given key. - * This is just the node header plus the key, there is no data. - */ -#define INDXSIZE(k) (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size)) - - /** Size of a node in a leaf page with a given key and data. - * This is node header plus key plus data size. - */ -#define LEAFSIZE(k, d) (NODESIZE + (k)->mv_size + (d)->mv_size) - - /** Address of node \b i in page \b p */ -#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + (p)->mp_ptrs[i] + PAGEBASE)) - - /** Address of the key for the node */ -#define NODEKEY(node) (void *)((node)->mn_data) - - /** Address of the data for a node */ -#define NODEDATA(node) (void *)((char *)(node)->mn_data + (node)->mn_ksize) - - /** Get the page number pointed to by a branch node */ -#define NODEPGNO(node) \ - ((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \ - (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0)) - /** Set the page number in a branch node */ -#define SETPGNO(node,pgno) do { \ - (node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \ - if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0) - - /** Get the size of the data in a leaf node */ -#define NODEDSZ(node) ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16)) - /** Set the size of the data for a leaf node */ -#define SETDSZ(node,size) do { \ - (node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0) - /** The size of a key in a node */ -#define NODEKSZ(node) ((node)->mn_ksize) - - /** Copy a page number from src to dst */ -#ifdef MISALIGNED_OK -#define COPY_PGNO(dst,src) dst = src -#else -#if SIZE_MAX > 4294967295UL -#define COPY_PGNO(dst,src) do { \ - unsigned short *s, *d; \ - s = (unsigned short *)&(src); \ - d = (unsigned short *)&(dst); \ - *d++ = *s++; \ - *d++ = *s++; \ - *d++ = *s++; \ - *d = *s; \ -} while (0) -#else -#define COPY_PGNO(dst,src) do { \ - unsigned short *s, *d; \ - s = (unsigned short *)&(src); \ - d = (unsigned short *)&(dst); \ - *d++ = *s++; \ - *d = *s; \ -} while (0) -#endif -#endif - /** The address of a key in a LEAF2 page. - * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs. - * There are no node headers, keys are stored contiguously. - */ -#define LEAF2KEY(p, i, ks) ((char *)(p) + PAGEHDRSZ + ((i)*(ks))) - - /** Set the \b node's key into \b keyptr, if requested. */ -#define MDB_GET_KEY(node, keyptr) { if ((keyptr) != NULL) { \ - (keyptr)->mv_size = NODEKSZ(node); (keyptr)->mv_data = NODEKEY(node); } } - - /** Set the \b node's key into \b key. */ -#define MDB_GET_KEY2(node, key) { key.mv_size = NODEKSZ(node); key.mv_data = NODEKEY(node); } - - /** Information about a single database in the environment. */ -typedef struct MDB_db { - uint32_t md_pad; /**< also ksize for LEAF2 pages */ - uint16_t md_flags; /**< @ref mdb_dbi_open */ - uint16_t md_depth; /**< depth of this tree */ - pgno_t md_branch_pages; /**< number of internal pages */ - pgno_t md_leaf_pages; /**< number of leaf pages */ - pgno_t md_overflow_pages; /**< number of overflow pages */ - size_t md_entries; /**< number of data items */ - pgno_t md_root; /**< the root page of this tree */ -} MDB_db; - - /** mdb_dbi_open flags */ -#define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ -#define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) -#define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ - MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) - - /** Handle for the DB used to track free pages. */ -#define FREE_DBI 0 - /** Handle for the default DB. */ -#define MAIN_DBI 1 - - /** Meta page content. - * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2). - */ -typedef struct MDB_meta { - /** Stamp identifying this as an LMDB file. It must be set - * to #MDB_MAGIC. */ - uint32_t mm_magic; - /** Version number of this lock file. Must be set to #MDB_DATA_VERSION. */ - uint32_t mm_version; - void *mm_address; /**< address for fixed mapping */ - size_t mm_mapsize; /**< size of mmap region */ - MDB_db mm_dbs[2]; /**< first is free space, 2nd is main db */ - /** The size of pages used in this DB */ -#define mm_psize mm_dbs[0].md_pad - /** Any persistent environment flags. @ref mdb_env */ -#define mm_flags mm_dbs[0].md_flags - pgno_t mm_last_pg; /**< last used page in file */ - txnid_t mm_txnid; /**< txnid that committed this page */ -} MDB_meta; - - /** Buffer for a stack-allocated meta page. - * The members define size and alignment, and silence type - * aliasing warnings. They are not used directly; that could - * mean incorrectly using several union members in parallel. - */ -typedef union MDB_metabuf { - MDB_page mb_page; - struct { - char mm_pad[PAGEHDRSZ]; - MDB_meta mm_meta; - } mb_metabuf; -} MDB_metabuf; - - /** Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. - */ -typedef struct MDB_dbx { - MDB_val md_name; /**< name of the database */ - MDB_cmp_func *md_cmp; /**< function for comparing keys */ - MDB_cmp_func *md_dcmp; /**< function for comparing data items */ - MDB_rel_func *md_rel; /**< user relocate function */ - void *md_relctx; /**< user-provided context for md_rel */ -} MDB_dbx; - - /** A database transaction. - * Every operation requires a transaction handle. - */ -struct MDB_txn { - MDB_txn *mt_parent; /**< parent of a nested txn */ - MDB_txn *mt_child; /**< nested txn under this txn */ - pgno_t mt_next_pgno; /**< next unallocated page */ - /** The ID of this transaction. IDs are integers incrementing from 1. - * Only committed write transactions increment the ID. If a transaction - * aborts, the ID may be re-used by the next writer. - */ - txnid_t mt_txnid; - MDB_env *mt_env; /**< the DB environment */ - /** The list of pages that became unused during this transaction. - */ - MDB_IDL mt_free_pgs; - /** The list of loose pages that became unused and may be reused - * in this transaction, linked through #NEXT_LOOSE_PAGE(page). - */ - MDB_page *mt_loose_pgs; - /* #Number of loose pages (#mt_loose_pgs) */ - int mt_loose_count; - /** The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. - */ - MDB_IDL mt_spill_pgs; - union { - /** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */ - MDB_ID2L dirty_list; - /** For read txns: This thread/txn's reader table slot, or NULL. */ - MDB_reader *reader; - } mt_u; - /** Array of records for each DB known in the environment. */ - MDB_dbx *mt_dbxs; - /** Array of MDB_db records for each known DB */ - MDB_db *mt_dbs; - /** Array of sequence numbers for each DB handle */ - unsigned int *mt_dbiseqs; -/** @defgroup mt_dbflag Transaction DB Flags - * @ingroup internal - * @{ - */ -#define DB_DIRTY 0x01 /**< DB was modified or is DUPSORT data */ -#define DB_STALE 0x02 /**< Named-DB record is older than txnID */ -#define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ -#define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ -/** @} */ - /** In write txns, array of cursors for each DB */ - MDB_cursor **mt_cursors; - /** Array of flags for each DB */ - unsigned char *mt_dbflags; - /** Number of DB records in use. This number only ever increments; - * we don't decrement it when individual DB handles are closed. - */ - MDB_dbi mt_numdbs; - -/** @defgroup mdb_txn Transaction Flags - * @ingroup internal - * @{ - */ -#define MDB_TXN_RDONLY 0x01 /**< read-only transaction */ -#define MDB_TXN_ERROR 0x02 /**< txn is unusable after an error */ -#define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ -#define MDB_TXN_SPILLS 0x08 /**< txn or a parent has spilled pages */ -/** @} */ - unsigned int mt_flags; /**< @ref mdb_txn */ - /** #dirty_list room: Array size - \#dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirty_list into mt_parent after freeing hidden mt_parent pages. - */ - unsigned int mt_dirty_room; -}; - -/** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. - * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to - * raise this on a 64 bit machine. - */ -#define CURSOR_STACK 32 - -struct MDB_xcursor; - - /** Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. #MDB_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a #P_SUBP page can be stale. - * (A node with #F_DUPDATA but no #F_SUBDATA contains a subpage). - */ -struct MDB_cursor { - /** Next cursor on this DB in this txn */ - MDB_cursor *mc_next; - /** Backup of the original cursor if this cursor is a shadow */ - MDB_cursor *mc_backup; - /** Context used for databases with #MDB_DUPSORT, otherwise NULL */ - struct MDB_xcursor *mc_xcursor; - /** The transaction that owns this cursor */ - MDB_txn *mc_txn; - /** The database handle this cursor operates on */ - MDB_dbi mc_dbi; - /** The database record for this cursor */ - MDB_db *mc_db; - /** The database auxiliary record for this cursor */ - MDB_dbx *mc_dbx; - /** The @ref mt_dbflag for this database */ - unsigned char *mc_dbflag; - unsigned short mc_snum; /**< number of pushed pages */ - unsigned short mc_top; /**< index of top page, normally mc_snum-1 */ -/** @defgroup mdb_cursor Cursor Flags - * @ingroup internal - * Cursor state flags. - * @{ - */ -#define C_INITIALIZED 0x01 /**< cursor has been initialized and is valid */ -#define C_EOF 0x02 /**< No more data */ -#define C_SUB 0x04 /**< Cursor is a sub-cursor */ -#define C_DEL 0x08 /**< last op was a cursor_del */ -#define C_SPLITTING 0x20 /**< Cursor is in page_split */ -#define C_UNTRACK 0x40 /**< Un-track cursor when closing */ -/** @} */ - unsigned int mc_flags; /**< @ref mdb_cursor */ - MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ -}; - - /** Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. - */ -typedef struct MDB_xcursor { - /** A sub-cursor for traversing the Dup DB */ - MDB_cursor mx_cursor; - /** The database record for this Dup DB */ - MDB_db mx_db; - /** The auxiliary DB record for this Dup DB */ - MDB_dbx mx_dbx; - /** The @ref mt_dbflag for this Dup DB */ - unsigned char mx_dbflag; -} MDB_xcursor; - - /** State of FreeDB old pages, stored in the MDB_env */ -typedef struct MDB_pgstate { - pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ - txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ -} MDB_pgstate; - - /** The database environment. */ -struct MDB_env { - HANDLE me_fd; /**< The main data file */ - HANDLE me_lfd; /**< The lock file */ - HANDLE me_mfd; /**< just for writing the meta pages */ - /** Failed to update the meta page. Probably an I/O error. */ -#define MDB_FATAL_ERROR 0x80000000U - /** Some fields are initialized. */ -#define MDB_ENV_ACTIVE 0x20000000U - /** me_txkey is set */ -#define MDB_ENV_TXKEY 0x10000000U - uint32_t me_flags; /**< @ref mdb_env */ - unsigned int me_psize; /**< DB page size, inited from me_os_psize */ - unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ - unsigned int me_maxreaders; /**< size of the reader table */ - unsigned int me_numreaders; /**< max numreaders set by this env */ - MDB_dbi me_numdbs; /**< number of DBs opened */ - MDB_dbi me_maxdbs; /**< size of the DB table */ - MDB_PID_T me_pid; /**< process ID of this env */ - char *me_path; /**< path to the DB files */ - char *me_map; /**< the memory map of the data file */ - MDB_txninfo *me_txns; /**< the memory map of the lock file or NULL */ - MDB_meta *me_metas[2]; /**< pointers to the two meta pages */ - void *me_pbuf; /**< scratch area for DUPSORT put() */ - MDB_txn *me_txn; /**< current write transaction */ - MDB_txn *me_txn0; /**< prealloc'd write transaction */ - size_t me_mapsize; /**< size of the data memory map */ - off_t me_size; /**< current file size */ - pgno_t me_maxpg; /**< me_mapsize / me_psize */ - MDB_dbx *me_dbxs; /**< array of static DB info */ - uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ - unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ - pthread_key_t me_txkey; /**< thread-key for readers */ - txnid_t me_pgoldest; /**< ID of oldest reader last time we looked */ - MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ -# define me_pglast me_pgstate.mf_pglast -# define me_pghead me_pgstate.mf_pghead - MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ - /** IDL of pages that became unused in a write txn */ - MDB_IDL me_free_pgs; - /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */ - MDB_ID2L me_dirty_list; - /** Max number of freelist items that can fit in a single overflow page */ - int me_maxfree_1pg; - /** Max size of a node on a page */ - unsigned int me_nodemax; -#if !(MDB_MAXKEYSIZE) - unsigned int me_maxkey; /**< max size of a key */ -#endif - int me_live_reader; /**< have liveness lock in reader table */ -#ifdef _WIN32 - int me_pidquery; /**< Used in OpenProcess */ -#endif -#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) - /* Windows mutexes/POSIX semaphores do not reside in shared mem */ - mdb_mutex_t me_rmutex; - mdb_mutex_t me_wmutex; -#endif - void *me_userctx; /**< User-settable context */ - MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ -}; - - /** Nested transaction */ -typedef struct MDB_ntxn { - MDB_txn mnt_txn; /**< the transaction */ - MDB_pgstate mnt_pgstate; /**< parent transaction's saved freestate */ -} MDB_ntxn; - - /** max number of pages to commit in one writev() call */ -#define MDB_COMMIT_PAGES 64 -#if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES -#undef MDB_COMMIT_PAGES -#define MDB_COMMIT_PAGES IOV_MAX -#endif - - /** max bytes to write in one call */ -#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4)) - - /** Check \b txn and \b dbi arguments to a function */ -#define TXN_DBI_EXIST(txn, dbi) \ - ((txn) && (dbi) < (txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & DB_VALID)) - - /** Check for misused \b dbi handles */ -#define TXN_DBI_CHANGED(txn, dbi) \ - ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) - -static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); -static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); -static int mdb_page_touch(MDB_cursor *mc); - -static int mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **mp, int *lvl); -static int mdb_page_search_root(MDB_cursor *mc, - MDB_val *key, int modify); -#define MDB_PS_MODIFY 1 -#define MDB_PS_ROOTONLY 2 -#define MDB_PS_FIRST 4 -#define MDB_PS_LAST 8 -static int mdb_page_search(MDB_cursor *mc, - MDB_val *key, int flags); -static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); - -#define MDB_SPLIT_REPLACE MDB_APPENDDUP /**< newkey is not new */ -static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, - pgno_t newpgno, unsigned int nflags); - -static int mdb_env_read_header(MDB_env *env, MDB_meta *meta); -static int mdb_env_pick_meta(const MDB_env *env); -static int mdb_env_write_meta(MDB_txn *txn); -#if !(defined(_WIN32) || defined(MDB_USE_POSIX_SEM)) /* Drop unused excl arg */ -# define mdb_env_close0(env, excl) mdb_env_close1(env) -#endif -static void mdb_env_close0(MDB_env *env, int excl); - -static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp); -static int mdb_node_add(MDB_cursor *mc, indx_t indx, - MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags); -static void mdb_node_del(MDB_cursor *mc, int ksize); -static void mdb_node_shrink(MDB_page *mp, indx_t indx); -static int mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst); -static int mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data); -static size_t mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data); -static size_t mdb_branch_size(MDB_env *env, MDB_val *key); - -static int mdb_rebalance(MDB_cursor *mc); -static int mdb_update_key(MDB_cursor *mc, MDB_val *key); - -static void mdb_cursor_pop(MDB_cursor *mc); -static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp); - -static int mdb_cursor_del0(MDB_cursor *mc); -static int mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags); -static int mdb_cursor_sibling(MDB_cursor *mc, int move_right); -static int mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); -static int mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); -static int mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op, - int *exactp); -static int mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data); -static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); - -static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); -static void mdb_xcursor_init0(MDB_cursor *mc); -static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); - -static int mdb_drop0(MDB_cursor *mc, int subs); -static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); -static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead); - -/** @cond */ -static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; -/** @endcond */ - -#ifdef _WIN32 -static SECURITY_DESCRIPTOR mdb_null_sd; -static SECURITY_ATTRIBUTES mdb_all_sa; -static int mdb_sec_inited; -#endif - -/** Return the library version info. */ -char * -mdb_version(int *major, int *minor, int *patch) -{ - if (major) *major = MDB_VERSION_MAJOR; - if (minor) *minor = MDB_VERSION_MINOR; - if (patch) *patch = MDB_VERSION_PATCH; - return MDB_VERSION_STRING; -} - -/** Table of descriptions for LMDB @ref errors */ -static char *const mdb_errstr[] = { - "MDB_KEYEXIST: Key/data pair already exists", - "MDB_NOTFOUND: No matching key/data pair found", - "MDB_PAGE_NOTFOUND: Requested page not found", - "MDB_CORRUPTED: Located page was wrong type", - "MDB_PANIC: Update of meta page failed or environment had fatal error", - "MDB_VERSION_MISMATCH: Database environment version mismatch", - "MDB_INVALID: File is not an LMDB file", - "MDB_MAP_FULL: Environment mapsize limit reached", - "MDB_DBS_FULL: Environment maxdbs limit reached", - "MDB_READERS_FULL: Environment maxreaders limit reached", - "MDB_TLS_FULL: Thread-local storage keys full - too many environments open", - "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", - "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", - "MDB_PAGE_FULL: Internal error - page has no more space", - "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", - "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", - "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", - "MDB_BAD_TXN: Transaction cannot recover - it must be aborted", - "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", - "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", -}; - -char * -mdb_strerror(int err) -{ -#ifdef _WIN32 - /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. - * This works as long as no function between the call to mdb_strerror - * and the actual use of the message uses more than 4K of stack. - */ - char pad[4096]; - char buf[1024], *ptr = buf; -#endif - int i; - if (!err) - return ("Successful return: 0"); - - if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { - i = err - MDB_KEYEXIST; - return mdb_errstr[i]; - } - -#ifdef _WIN32 - /* These are the C-runtime error codes we use. The comment indicates - * their numeric value, and the Win32 error they would correspond to - * if the error actually came from a Win32 API. A major mess, we should - * have used LMDB-specific error codes for everything. - */ - switch(err) { - case ENOENT: /* 2, FILE_NOT_FOUND */ - case EIO: /* 5, ACCESS_DENIED */ - case ENOMEM: /* 12, INVALID_ACCESS */ - case EACCES: /* 13, INVALID_DATA */ - case EBUSY: /* 16, CURRENT_DIRECTORY */ - case EINVAL: /* 22, BAD_COMMAND */ - case ENOSPC: /* 28, OUT_OF_PAPER */ - return strerror(err); - default: - ; - } - buf[0] = 0; - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, err, 0, ptr, sizeof(buf), pad); - return ptr; -#else - return strerror(err); -#endif -} - -/** assert(3) variant in cursor context */ -#define mdb_cassert(mc, expr) mdb_assert0((mc)->mc_txn->mt_env, expr, #expr) -/** assert(3) variant in transaction context */ -#define mdb_tassert(mc, expr) mdb_assert0((txn)->mt_env, expr, #expr) -/** assert(3) variant in environment context */ -#define mdb_eassert(env, expr) mdb_assert0(env, expr, #expr) - -#ifndef NDEBUG -# define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \ - mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__)) - -static void -mdb_assert_fail(MDB_env *env, const char *expr_txt, - const char *func, const char *file, int line) -{ - char buf[400]; - sprintf(buf, "%.100s:%d: Assertion '%.200s' failed in %.40s()", - file, line, expr_txt, func); - if (env->me_assert_func) - env->me_assert_func(env, buf); - fprintf(stderr, "%s\n", buf); - abort(); -} -#else -# define mdb_assert0(env, expr, expr_txt) ((void) 0) -#endif /* NDEBUG */ - -#if MDB_DEBUG -/** Return the page number of \b mp which may be sub-page, for debug output */ -static pgno_t -mdb_dbg_pgno(MDB_page *mp) -{ - pgno_t ret; - COPY_PGNO(ret, mp->mp_pgno); - return ret; -} - -/** Display a key in hexadecimal and return the address of the result. - * @param[in] key the key to display - * @param[in] buf the buffer to write into. Should always be #DKBUF. - * @return The key in hexadecimal form. - */ -char * -mdb_dkey(MDB_val *key, char *buf) -{ - char *ptr = buf; - unsigned char *c = key->mv_data; - unsigned int i; - - if (!key) - return ""; - - if (key->mv_size > DKBUF_MAXKEYSIZE) - return "MDB_MAXKEYSIZE"; - /* may want to make this a dynamic check: if the key is mostly - * printable characters, print it as-is instead of converting to hex. - */ -#if 1 - buf[0] = '\0'; - for (i=0; imv_size; i++) - ptr += sprintf(ptr, "%02x", *c++); -#else - sprintf(buf, "%.*s", key->mv_size, key->mv_data); -#endif - return buf; -} - -static const char * -mdb_leafnode_type(MDB_node *n) -{ - static char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; - return F_ISSET(n->mn_flags, F_BIGDATA) ? ": overflow page" : - tp[F_ISSET(n->mn_flags, F_DUPDATA)][F_ISSET(n->mn_flags, F_SUBDATA)]; -} - -/** Display all the keys in the page. */ -void -mdb_page_list(MDB_page *mp) -{ - pgno_t pgno = mdb_dbg_pgno(mp); - const char *type, *state = (mp->mp_flags & P_DIRTY) ? ", dirty" : ""; - MDB_node *node; - unsigned int i, nkeys, nsize, total = 0; - MDB_val key; - DKBUF; - - switch (mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { - case P_BRANCH: type = "Branch page"; break; - case P_LEAF: type = "Leaf page"; break; - case P_LEAF|P_SUBP: type = "Sub-page"; break; - case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; - case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; - case P_OVERFLOW: - fprintf(stderr, "Overflow page %"Z"u pages %u%s\n", - pgno, mp->mp_pages, state); - return; - case P_META: - fprintf(stderr, "Meta-page %"Z"u txnid %"Z"u\n", - pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); - return; - default: - fprintf(stderr, "Bad page %"Z"u flags 0x%u\n", pgno, mp->mp_flags); - return; - } - - nkeys = NUMKEYS(mp); - fprintf(stderr, "%s %"Z"u numkeys %d%s\n", type, pgno, nkeys, state); - - for (i=0; imp_pad; - key.mv_data = LEAF2KEY(mp, i, nsize); - total += nsize; - fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key)); - continue; - } - node = NODEPTR(mp, i); - key.mv_size = node->mn_ksize; - key.mv_data = node->mn_data; - nsize = NODESIZE + key.mv_size; - if (IS_BRANCH(mp)) { - fprintf(stderr, "key %d: page %"Z"u, %s\n", i, NODEPGNO(node), - DKEY(&key)); - total += nsize; - } else { - if (F_ISSET(node->mn_flags, F_BIGDATA)) - nsize += sizeof(pgno_t); - else - nsize += NODEDSZ(node); - total += nsize; - nsize += sizeof(indx_t); - fprintf(stderr, "key %d: nsize %d, %s%s\n", - i, nsize, DKEY(&key), mdb_leafnode_type(node)); - } - total = EVEN(total); - } - fprintf(stderr, "Total: header %d + contents %d + unused %d\n", - IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + mp->mp_lower, total, SIZELEFT(mp)); -} - -void -mdb_cursor_chk(MDB_cursor *mc) -{ - unsigned int i; - MDB_node *node; - MDB_page *mp; - - if (!mc->mc_snum && !(mc->mc_flags & C_INITIALIZED)) return; - for (i=0; imc_top; i++) { - mp = mc->mc_pg[i]; - node = NODEPTR(mp, mc->mc_ki[i]); - if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno) - printf("oops!\n"); - } - if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) - printf("ack!\n"); -} -#endif - -#if (MDB_DEBUG) > 2 -/** Count all the pages in each DB and in the freelist - * and make sure it matches the actual number of pages - * being used. - * All named DBs must be open for a correct count. - */ -static void mdb_audit(MDB_txn *txn) -{ - MDB_cursor mc; - MDB_val key, data; - MDB_ID freecount, count; - MDB_dbi i; - int rc; - - freecount = 0; - mdb_cursor_init(&mc, txn, FREE_DBI, NULL); - while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) - freecount += *(MDB_ID *)data.mv_data; - mdb_tassert(txn, rc == MDB_NOTFOUND); - - count = 0; - for (i = 0; imt_numdbs; i++) { - MDB_xcursor mx; - if (!(txn->mt_dbflags[i] & DB_VALID)) - continue; - mdb_cursor_init(&mc, txn, i, &mx); - if (txn->mt_dbs[i].md_root == P_INVALID) - continue; - count += txn->mt_dbs[i].md_branch_pages + - txn->mt_dbs[i].md_leaf_pages + - txn->mt_dbs[i].md_overflow_pages; - if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) { - rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST); - for (; rc == MDB_SUCCESS; rc = mdb_cursor_sibling(&mc, 1)) { - unsigned j; - MDB_page *mp; - mp = mc.mc_pg[mc.mc_top]; - for (j=0; jmn_flags & F_SUBDATA) { - MDB_db db; - memcpy(&db, NODEDATA(leaf), sizeof(db)); - count += db.md_branch_pages + db.md_leaf_pages + - db.md_overflow_pages; - } - } - } - mdb_tassert(txn, rc == MDB_NOTFOUND); - } - } - if (freecount + count + 2 /* metapages */ != txn->mt_next_pgno) { - fprintf(stderr, "audit: %lu freecount: %lu count: %lu total: %lu next_pgno: %lu\n", - txn->mt_txnid, freecount, count+2, freecount+count+2, txn->mt_next_pgno); - } -} -#endif - -int -mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) -{ - return txn->mt_dbxs[dbi].md_cmp(a, b); -} - -int -mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) -{ - return txn->mt_dbxs[dbi].md_dcmp(a, b); -} - -/** Allocate memory for a page. - * Re-use old malloc'd pages first for singletons, otherwise just malloc. - */ -static MDB_page * -mdb_page_malloc(MDB_txn *txn, unsigned num) -{ - MDB_env *env = txn->mt_env; - MDB_page *ret = env->me_dpages; - size_t psize = env->me_psize, sz = psize, off; - /* For ! #MDB_NOMEMINIT, psize counts how much to init. - * For a single page alloc, we init everything after the page header. - * For multi-page, we init the final page; if the caller needed that - * many pages they will be filling in at least up to the last page. - */ - if (num == 1) { - if (ret) { - VGMEMP_ALLOC(env, ret, sz); - VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); - env->me_dpages = ret->mp_next; - return ret; - } - psize -= off = PAGEHDRSZ; - } else { - sz *= num; - off = sz - psize; - } - if ((ret = malloc(sz)) != NULL) { - VGMEMP_ALLOC(env, ret, sz); - if (!(env->me_flags & MDB_NOMEMINIT)) { - memset((char *)ret + off, 0, psize); - ret->mp_pad = 0; - } - } else { - txn->mt_flags |= MDB_TXN_ERROR; - } - return ret; -} -/** Free a single page. - * Saves single pages to a list, for future reuse. - * (This is not used for multi-page overflow pages.) - */ -static void -mdb_page_free(MDB_env *env, MDB_page *mp) -{ - mp->mp_next = env->me_dpages; - VGMEMP_FREE(env, mp); - env->me_dpages = mp; -} - -/** Free a dirty page */ -static void -mdb_dpage_free(MDB_env *env, MDB_page *dp) -{ - if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { - mdb_page_free(env, dp); - } else { - /* large pages just get freed directly */ - VGMEMP_FREE(env, dp); - free(dp); - } -} - -/** Return all dirty pages to dpage list */ -static void -mdb_dlist_free(MDB_txn *txn) -{ - MDB_env *env = txn->mt_env; - MDB_ID2L dl = txn->mt_u.dirty_list; - unsigned i, n = dl[0].mid; - - for (i = 1; i <= n; i++) { - mdb_dpage_free(env, dl[i].mptr); - } - dl[0].mid = 0; -} - -/** Loosen or free a single page. - * Saves single pages to a list for future reuse - * in this same txn. It has been pulled from the freeDB - * and already resides on the dirty list, but has been - * deleted. Use these pages first before pulling again - * from the freeDB. - * - * If the page wasn't dirtied in this txn, just add it - * to this txn's free list. - */ -static int -mdb_page_loose(MDB_cursor *mc, MDB_page *mp) -{ - int loose = 0; - pgno_t pgno = mp->mp_pgno; - MDB_txn *txn = mc->mc_txn; - - if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) { - if (txn->mt_parent) { - MDB_ID2 *dl = txn->mt_u.dirty_list; - /* If txn has a parent, make sure the page is in our - * dirty list. - */ - if (dl[0].mid) { - unsigned x = mdb_mid2l_search(dl, pgno); - if (x <= dl[0].mid && dl[x].mid == pgno) { - if (mp != dl[x].mptr) { /* bad cursor? */ - mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; - } - /* ok, it's ours */ - loose = 1; - } - } - } else { - /* no parent txn, so it's just ours */ - loose = 1; - } - } - if (loose) { - DPRINTF(("loosen db %d page %"Z"u", DDBI(mc), - mp->mp_pgno)); - NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; - txn->mt_loose_pgs = mp; - txn->mt_loose_count++; - mp->mp_flags |= P_LOOSE; - } else { - int rc = mdb_midl_append(&txn->mt_free_pgs, pgno); - if (rc) - return rc; - } - - return MDB_SUCCESS; -} - -/** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. - * @param[in] mc A cursor handle for the current operation. - * @param[in] pflags Flags of the pages to update: - * P_DIRTY to set P_KEEP, P_DIRTY|P_KEEP to clear it. - * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush(). - * @return 0 on success, non-zero on failure. - */ -static int -mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) -{ - enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP }; - MDB_txn *txn = mc->mc_txn; - MDB_cursor *m3; - MDB_xcursor *mx; - MDB_page *dp, *mp; - MDB_node *leaf; - unsigned i, j; - int rc = MDB_SUCCESS, level; - - /* Mark pages seen by cursors */ - if (mc->mc_flags & C_UNTRACK) - mc = NULL; /* will find mc in mt_cursors */ - for (i = txn->mt_numdbs;; mc = txn->mt_cursors[--i]) { - for (; mc; mc=mc->mc_next) { - if (!(mc->mc_flags & C_INITIALIZED)) - continue; - for (m3 = mc;; m3 = &mx->mx_cursor) { - mp = NULL; - for (j=0; jmc_snum; j++) { - mp = m3->mc_pg[j]; - if ((mp->mp_flags & Mask) == pflags) - mp->mp_flags ^= P_KEEP; - } - mx = m3->mc_xcursor; - /* Proceed to mx if it is at a sub-database */ - if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED))) - break; - if (! (mp && (mp->mp_flags & P_LEAF))) - break; - leaf = NODEPTR(mp, m3->mc_ki[j-1]); - if (!(leaf->mn_flags & F_SUBDATA)) - break; - } - } - if (i == 0) - break; - } - - if (all) { - /* Mark dirty root pages */ - for (i=0; imt_numdbs; i++) { - if (txn->mt_dbflags[i] & DB_DIRTY) { - pgno_t pgno = txn->mt_dbs[i].md_root; - if (pgno == P_INVALID) - continue; - if ((rc = mdb_page_get(txn, pgno, &dp, &level)) != MDB_SUCCESS) - break; - if ((dp->mp_flags & Mask) == pflags && level <= 1) - dp->mp_flags ^= P_KEEP; - } - } - } - - return rc; -} - -static int mdb_page_flush(MDB_txn *txn, int keep); - -/** Spill pages from the dirty list back to disk. - * This is intended to prevent running into #MDB_TXN_FULL situations, - * but note that they may still occur in a few cases: - * 1) our estimate of the txn size could be too small. Currently this - * seems unlikely, except with a large number of #MDB_MULTIPLE items. - * 2) child txns may run out of space if their parents dirtied a - * lot of pages and never spilled them. TODO: we probably should do - * a preemptive spill during #mdb_txn_begin() of a child txn, if - * the parent's dirty_room is below a given threshold. - * - * Otherwise, if not using nested txns, it is expected that apps will - * not run into #MDB_TXN_FULL any more. The pages are flushed to disk - * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared. - * If the txn never references them again, they can be left alone. - * If the txn only reads them, they can be used without any fuss. - * If the txn writes them again, they can be dirtied immediately without - * going thru all of the work of #mdb_page_touch(). Such references are - * handled by #mdb_page_unspill(). - * - * Also note, we never spill DB root pages, nor pages of active cursors, - * because we'll need these back again soon anyway. And in nested txns, - * we can't spill a page in a child txn if it was already spilled in a - * parent txn. That would alter the parent txns' data even though - * the child hasn't committed yet, and we'd have no way to undo it if - * the child aborted. - * - * @param[in] m0 cursor A cursor handle identifying the transaction and - * database for which we are checking space. - * @param[in] key For a put operation, the key being stored. - * @param[in] data For a put operation, the data being stored. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data) -{ - MDB_txn *txn = m0->mc_txn; - MDB_page *dp; - MDB_ID2L dl = txn->mt_u.dirty_list; - unsigned int i, j, need; - int rc; - - if (m0->mc_flags & C_SUB) - return MDB_SUCCESS; - - /* Estimate how much space this op will take */ - i = m0->mc_db->md_depth; - /* Named DBs also dirty the main DB */ - if (m0->mc_dbi > MAIN_DBI) - i += txn->mt_dbs[MAIN_DBI].md_depth; - /* For puts, roughly factor in the key+data size */ - if (key) - i += (LEAFSIZE(key, data) + txn->mt_env->me_psize) / txn->mt_env->me_psize; - i += i; /* double it for good measure */ - need = i; - - if (txn->mt_dirty_room > i) - return MDB_SUCCESS; - - if (!txn->mt_spill_pgs) { - txn->mt_spill_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX); - if (!txn->mt_spill_pgs) - return ENOMEM; - } else { - /* purge deleted slots */ - MDB_IDL sl = txn->mt_spill_pgs; - unsigned int num = sl[0]; - j=0; - for (i=1; i<=num; i++) { - if (!(sl[i] & 1)) - sl[++j] = sl[i]; - } - sl[0] = j; - } - - /* Preserve pages which may soon be dirtied again */ - if ((rc = mdb_pages_xkeep(m0, P_DIRTY, 1)) != MDB_SUCCESS) - goto done; - - /* Less aggressive spill - we originally spilled the entire dirty list, - * with a few exceptions for cursor pages and DB root pages. But this - * turns out to be a lot of wasted effort because in a large txn many - * of those pages will need to be used again. So now we spill only 1/8th - * of the dirty pages. Testing revealed this to be a good tradeoff, - * better than 1/2, 1/4, or 1/10. - */ - if (need < MDB_IDL_UM_MAX / 8) - need = MDB_IDL_UM_MAX / 8; - - /* Save the page IDs of all the pages we're flushing */ - /* flush from the tail forward, this saves a lot of shifting later on. */ - for (i=dl[0].mid; i && need; i--) { - MDB_ID pn = dl[i].mid << 1; - dp = dl[i].mptr; - if (dp->mp_flags & (P_LOOSE|P_KEEP)) - continue; - /* Can't spill twice, make sure it's not already in a parent's - * spill list. - */ - if (txn->mt_parent) { - MDB_txn *tx2; - for (tx2 = txn->mt_parent; tx2; tx2 = tx2->mt_parent) { - if (tx2->mt_spill_pgs) { - j = mdb_midl_search(tx2->mt_spill_pgs, pn); - if (j <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[j] == pn) { - dp->mp_flags |= P_KEEP; - break; - } - } - } - if (tx2) - continue; - } - if ((rc = mdb_midl_append(&txn->mt_spill_pgs, pn))) - goto done; - need--; - } - mdb_midl_sort(txn->mt_spill_pgs); - - /* Flush the spilled part of dirty list */ - if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS) - goto done; - - /* Reset any dirty pages we kept that page_flush didn't see */ - rc = mdb_pages_xkeep(m0, P_DIRTY|P_KEEP, i); - -done: - txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS; - return rc; -} - -/** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */ -static txnid_t -mdb_find_oldest(MDB_txn *txn) -{ - int i; - txnid_t mr, oldest = txn->mt_txnid - 1; - if (txn->mt_env->me_txns) { - MDB_reader *r = txn->mt_env->me_txns->mti_readers; - for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) { - if (r[i].mr_pid) { - mr = r[i].mr_txnid; - if (oldest > mr) - oldest = mr; - } - } - } - return oldest; -} - -/** Add a page to the txn's dirty list */ -static void -mdb_page_dirty(MDB_txn *txn, MDB_page *mp) -{ - MDB_ID2 mid; - int rc, (*insert)(MDB_ID2L, MDB_ID2 *); - - if (txn->mt_env->me_flags & MDB_WRITEMAP) { - insert = mdb_mid2l_append; - } else { - insert = mdb_mid2l_insert; - } - mid.mid = mp->mp_pgno; - mid.mptr = mp; - rc = insert(txn->mt_u.dirty_list, &mid); - mdb_tassert(txn, rc == 0); - txn->mt_dirty_room--; -} - -/** Allocate page numbers and memory for writing. Maintain me_pglast, - * me_pghead and mt_next_pgno. - * - * If there are free pages available from older transactions, they - * are re-used first. Otherwise allocate a new page at mt_next_pgno. - * Do not modify the freedB, just merge freeDB records into me_pghead[] - * and move me_pglast to say which records were consumed. Only this - * function can create me_pghead and move me_pglast/mt_next_pgno. - * @param[in] mc cursor A cursor handle identifying the transaction and - * database for which we are allocating. - * @param[in] num the number of pages to allocate. - * @param[out] mp Address of the allocated page(s). Requests for multiple pages - * will always be satisfied by a single contiguous chunk of memory. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) -{ -#ifdef MDB_PARANOID /* Seems like we can ignore this now */ - /* Get at most more freeDB records once me_pghead - * has enough pages. If not enough, use new pages from the map. - * If and mc is updating the freeDB, only get new - * records if me_pghead is empty. Then the freelist cannot play - * catch-up with itself by growing while trying to save it. - */ - enum { Paranoid = 1, Max_retries = 500 }; -#else - enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; -#endif - int rc, retry = num * 60; - MDB_txn *txn = mc->mc_txn; - MDB_env *env = txn->mt_env; - pgno_t pgno, *mop = env->me_pghead; - unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1; - MDB_page *np; - txnid_t oldest = 0, last; - MDB_cursor_op op; - MDB_cursor m2; - int found_old = 0; - - /* If there are any loose pages, just use them */ - if (num == 1 && txn->mt_loose_pgs) { - np = txn->mt_loose_pgs; - txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); - txn->mt_loose_count--; - DPRINTF(("db %d use loose page %"Z"u", DDBI(mc), - np->mp_pgno)); - *mp = np; - return MDB_SUCCESS; - } - - *mp = NULL; - - /* If our dirty list is already full, we can't do anything */ - if (txn->mt_dirty_room == 0) { - rc = MDB_TXN_FULL; - goto fail; - } - - for (op = MDB_FIRST;; op = MDB_NEXT) { - MDB_val key, data; - MDB_node *leaf; - pgno_t *idl; - - /* Seek a big enough contiguous page range. Prefer - * pages at the tail, just truncating the list. - */ - if (mop_len > n2) { - i = mop_len; - do { - pgno = mop[i]; - if (mop[i-n2] == pgno+n2) - goto search_done; - } while (--i > n2); - if (--retry < 0) - break; - } - - if (op == MDB_FIRST) { /* 1st iteration */ - /* Prepare to fetch more and coalesce */ - last = env->me_pglast; - oldest = env->me_pgoldest; - mdb_cursor_init(&m2, txn, FREE_DBI, NULL); - if (last) { - op = MDB_SET_RANGE; - key.mv_data = &last; /* will look up last+1 */ - key.mv_size = sizeof(last); - } - if (Paranoid && mc->mc_dbi == FREE_DBI) - retry = -1; - } - if (Paranoid && retry < 0 && mop_len) - break; - - last++; - /* Do not fetch more if the record will be too recent */ - if (oldest <= last) { - if (!found_old) { - oldest = mdb_find_oldest(txn); - env->me_pgoldest = oldest; - found_old = 1; - } - if (oldest <= last) - break; - } - rc = mdb_cursor_get(&m2, &key, NULL, op); - if (rc) { - if (rc == MDB_NOTFOUND) - break; - goto fail; - } - last = *(txnid_t*)key.mv_data; - if (oldest <= last) { - if (!found_old) { - oldest = mdb_find_oldest(txn); - env->me_pgoldest = oldest; - found_old = 1; - } - if (oldest <= last) - break; - } - np = m2.mc_pg[m2.mc_top]; - leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); - if ((rc = mdb_node_read(txn, leaf, &data)) != MDB_SUCCESS) - return rc; - - idl = (MDB_ID *) data.mv_data; - i = idl[0]; - if (!mop) { - if (!(env->me_pghead = mop = mdb_midl_alloc(i))) { - rc = ENOMEM; - goto fail; - } - } else { - if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) - goto fail; - mop = env->me_pghead; - } - env->me_pglast = last; -#if (MDB_DEBUG) > 1 - DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u", - last, txn->mt_dbs[FREE_DBI].md_root, i)); - for (j = i; j; j--) - DPRINTF(("IDL %"Z"u", idl[j])); -#endif - /* Merge in descending sorted order */ - mdb_midl_xmerge(mop, idl); - mop_len = mop[0]; - } - - /* Use new pages from the map when nothing suitable in the freeDB */ - i = 0; - pgno = txn->mt_next_pgno; - if (pgno + num >= env->me_maxpg) { - DPUTS("DB size maxed out"); - rc = MDB_MAP_FULL; - goto fail; - } - -search_done: - if (env->me_flags & MDB_WRITEMAP) { - np = (MDB_page *)(env->me_map + env->me_psize * pgno); - } else { - if (!(np = mdb_page_malloc(txn, num))) { - rc = ENOMEM; - goto fail; - } - } - if (i) { - mop[0] = mop_len -= num; - /* Move any stragglers down */ - for (j = i-num; j < mop_len; ) - mop[++j] = mop[++i]; - } else { - txn->mt_next_pgno = pgno + num; - } - np->mp_pgno = pgno; - mdb_page_dirty(txn, np); - *mp = np; - - return MDB_SUCCESS; - -fail: - txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -/** Copy the used portions of a non-overflow page. - * @param[in] dst page to copy into - * @param[in] src page to copy from - * @param[in] psize size of a page - */ -static void -mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize) -{ - enum { Align = sizeof(pgno_t) }; - indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower; - - /* If page isn't full, just copy the used portion. Adjust - * alignment so memcpy may copy words instead of bytes. - */ - if ((unused &= -Align) && !IS_LEAF2(src)) { - upper = (upper + PAGEBASE) & -Align; - memcpy(dst, src, (lower + PAGEBASE + (Align-1)) & -Align); - memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper), - psize - upper); - } else { - memcpy(dst, src, psize - unused); - } -} - -/** Pull a page off the txn's spill list, if present. - * If a page being referenced was spilled to disk in this txn, bring - * it back and make it dirty/writable again. - * @param[in] txn the transaction handle. - * @param[in] mp the page being referenced. It must not be dirty. - * @param[out] ret the writable page, if any. ret is unchanged if - * mp wasn't spilled. - */ -static int -mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) -{ - MDB_env *env = txn->mt_env; - const MDB_txn *tx2; - unsigned x; - pgno_t pgno = mp->mp_pgno, pn = pgno << 1; - - for (tx2 = txn; tx2; tx2=tx2->mt_parent) { - if (!tx2->mt_spill_pgs) - continue; - x = mdb_midl_search(tx2->mt_spill_pgs, pn); - if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { - MDB_page *np; - int num; - if (txn->mt_dirty_room == 0) - return MDB_TXN_FULL; - if (IS_OVERFLOW(mp)) - num = mp->mp_pages; - else - num = 1; - if (env->me_flags & MDB_WRITEMAP) { - np = mp; - } else { - np = mdb_page_malloc(txn, num); - if (!np) - return ENOMEM; - if (num > 1) - memcpy(np, mp, num * env->me_psize); - else - mdb_page_copy(np, mp, env->me_psize); - } - if (tx2 == txn) { - /* If in current txn, this page is no longer spilled. - * If it happens to be the last page, truncate the spill list. - * Otherwise mark it as deleted by setting the LSB. - */ - if (x == txn->mt_spill_pgs[0]) - txn->mt_spill_pgs[0]--; - else - txn->mt_spill_pgs[x] |= 1; - } /* otherwise, if belonging to a parent txn, the - * page remains spilled until child commits - */ - - mdb_page_dirty(txn, np); - np->mp_flags |= P_DIRTY; - *ret = np; - break; - } - } - return MDB_SUCCESS; -} - -/** Touch a page: make it dirty and re-insert into tree with updated pgno. - * @param[in] mc cursor pointing to the page to be touched - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_touch(MDB_cursor *mc) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top], *np; - MDB_txn *txn = mc->mc_txn; - MDB_cursor *m2, *m3; - pgno_t pgno; - int rc; - - if (!F_ISSET(mp->mp_flags, P_DIRTY)) { - if (txn->mt_flags & MDB_TXN_SPILLS) { - np = NULL; - rc = mdb_page_unspill(txn, mp, &np); - if (rc) - goto fail; - if (np) - goto done; - } - if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || - (rc = mdb_page_alloc(mc, 1, &np))) - goto fail; - pgno = np->mp_pgno; - DPRINTF(("touched db %d page %"Z"u -> %"Z"u", DDBI(mc), - mp->mp_pgno, pgno)); - mdb_cassert(mc, mp->mp_pgno != pgno); - mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); - /* Update the parent page, if any, to point to the new page */ - if (mc->mc_top) { - MDB_page *parent = mc->mc_pg[mc->mc_top-1]; - MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]); - SETPGNO(node, pgno); - } else { - mc->mc_db->md_root = pgno; - } - } else if (txn->mt_parent && !IS_SUBP(mp)) { - MDB_ID2 mid, *dl = txn->mt_u.dirty_list; - pgno = mp->mp_pgno; - /* If txn has a parent, make sure the page is in our - * dirty list. - */ - if (dl[0].mid) { - unsigned x = mdb_mid2l_search(dl, pgno); - if (x <= dl[0].mid && dl[x].mid == pgno) { - if (mp != dl[x].mptr) { /* bad cursor? */ - mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; - } - return 0; - } - } - mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX); - /* No - copy it */ - np = mdb_page_malloc(txn, 1); - if (!np) - return ENOMEM; - mid.mid = pgno; - mid.mptr = np; - rc = mdb_mid2l_insert(dl, &mid); - mdb_cassert(mc, rc == 0); - } else { - return 0; - } - - mdb_page_copy(np, mp, txn->mt_env->me_psize); - np->mp_pgno = pgno; - np->mp_flags |= P_DIRTY; - -done: - /* Adjust cursors pointing to mp */ - mc->mc_pg[mc->mc_top] = np; - m2 = txn->mt_cursors[mc->mc_dbi]; - if (mc->mc_flags & C_SUB) { - for (; m2; m2=m2->mc_next) { - m3 = &m2->mc_xcursor->mx_cursor; - if (m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[mc->mc_top] == mp) - m3->mc_pg[mc->mc_top] = np; - } - } else { - for (; m2; m2=m2->mc_next) { - if (m2->mc_snum < mc->mc_snum) continue; - if (m2->mc_pg[mc->mc_top] == mp) { - m2->mc_pg[mc->mc_top] = np; - if ((mc->mc_db->md_flags & MDB_DUPSORT) && - IS_LEAF(np) && - m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) - { - MDB_node *leaf = NODEPTR(np, mc->mc_ki[mc->mc_top]); - if (!(leaf->mn_flags & F_SUBDATA)) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } - } - } - } - return 0; - -fail: - txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_env_sync(MDB_env *env, int force) -{ - int rc = 0; - if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { - if (env->me_flags & MDB_WRITEMAP) { - int flags = ((env->me_flags & MDB_MAPASYNC) && !force) - ? MS_ASYNC : MS_SYNC; - if (MDB_MSYNC(env->me_map, env->me_mapsize, flags)) - rc = ErrCode(); -#ifdef _WIN32 - else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) - rc = ErrCode(); -#endif - } else { - if (MDB_FDATASYNC(env->me_fd)) - rc = ErrCode(); - } - } - return rc; -} - -/** Back up parent txn's cursors, then grab the originals for tracking */ -static int -mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) -{ - MDB_cursor *mc, *bk; - MDB_xcursor *mx; - size_t size; - int i; - - for (i = src->mt_numdbs; --i >= 0; ) { - if ((mc = src->mt_cursors[i]) != NULL) { - size = sizeof(MDB_cursor); - if (mc->mc_xcursor) - size += sizeof(MDB_xcursor); - for (; mc; mc = bk->mc_next) { - bk = malloc(size); - if (!bk) - return ENOMEM; - *bk = *mc; - mc->mc_backup = bk; - mc->mc_db = &dst->mt_dbs[i]; - /* Kill pointers into src - and dst to reduce abuse: The - * user may not use mc until dst ends. Otherwise we'd... - */ - mc->mc_txn = NULL; /* ...set this to dst */ - mc->mc_dbflag = NULL; /* ...and &dst->mt_dbflags[i] */ - if ((mx = mc->mc_xcursor) != NULL) { - *(MDB_xcursor *)(bk+1) = *mx; - mx->mx_cursor.mc_txn = NULL; /* ...and dst. */ - } - mc->mc_next = dst->mt_cursors[i]; - dst->mt_cursors[i] = mc; - } - } - } - return MDB_SUCCESS; -} - -/** Close this write txn's cursors, give parent txn's cursors back to parent. - * @param[in] txn the transaction handle. - * @param[in] merge true to keep changes to parent cursors, false to revert. - * @return 0 on success, non-zero on failure. - */ -static void -mdb_cursors_close(MDB_txn *txn, unsigned merge) -{ - MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; - MDB_xcursor *mx; - int i; - - for (i = txn->mt_numdbs; --i >= 0; ) { - for (mc = cursors[i]; mc; mc = next) { - next = mc->mc_next; - if ((bk = mc->mc_backup) != NULL) { - if (merge) { - /* Commit changes to parent txn */ - mc->mc_next = bk->mc_next; - mc->mc_backup = bk->mc_backup; - mc->mc_txn = bk->mc_txn; - mc->mc_db = bk->mc_db; - mc->mc_dbflag = bk->mc_dbflag; - if ((mx = mc->mc_xcursor) != NULL) - mx->mx_cursor.mc_txn = bk->mc_txn; - } else { - /* Abort nested txn */ - *mc = *bk; - if ((mx = mc->mc_xcursor) != NULL) - *mx = *(MDB_xcursor *)(bk+1); - } - mc = bk; - } - /* Only malloced cursors are permanently tracked. */ - free(mc); - } - cursors[i] = NULL; - } -} - -#if !(MDB_DEBUG) -#define mdb_txn_reset0(txn, act) mdb_txn_reset0(txn) -#endif -static void -mdb_txn_reset0(MDB_txn *txn, const char *act); - -#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ -enum Pidlock_op { - Pidset, Pidcheck -}; -#else -enum Pidlock_op { - Pidset = F_SETLK, Pidcheck = F_GETLK -}; -#endif - -/** Set or check a pid lock. Set returns 0 on success. - * Check returns 0 if the process is certainly dead, nonzero if it may - * be alive (the lock exists or an error happened so we do not know). - * - * On Windows Pidset is a no-op, we merely check for the existence - * of the process with the given pid. On POSIX we use a single byte - * lock on the lockfile, set at an offset equal to the pid. - */ -static int -mdb_reader_pid(MDB_env *env, enum Pidlock_op op, MDB_PID_T pid) -{ -#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ - int ret = 0; - HANDLE h; - if (op == Pidcheck) { - h = OpenProcess(env->me_pidquery, FALSE, pid); - /* No documented "no such process" code, but other program use this: */ - if (!h) - return ErrCode() != ERROR_INVALID_PARAMETER; - /* A process exists until all handles to it close. Has it exited? */ - ret = WaitForSingleObject(h, 0) != 0; - CloseHandle(h); - } - return ret; -#else - for (;;) { - int rc; - struct flock lock_info; - memset(&lock_info, 0, sizeof(lock_info)); - lock_info.l_type = F_WRLCK; - lock_info.l_whence = SEEK_SET; - lock_info.l_start = pid; - lock_info.l_len = 1; - if ((rc = fcntl(env->me_lfd, op, &lock_info)) == 0) { - if (op == F_GETLK && lock_info.l_type != F_UNLCK) - rc = -1; - } else if ((rc = ErrCode()) == EINTR) { - continue; - } - return rc; - } -#endif -} - -/** Common code for #mdb_txn_begin() and #mdb_txn_renew(). - * @param[in] txn the transaction handle to initialize - * @return 0 on success, non-zero on failure. - */ -static int -mdb_txn_renew0(MDB_txn *txn) -{ - MDB_env *env = txn->mt_env; - MDB_txninfo *ti = env->me_txns; - MDB_meta *meta; - unsigned int i, nr; - uint16_t x; - int rc, new_notls = 0; - - /* Setup db info */ - txn->mt_numdbs = env->me_numdbs; - txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ - - if (txn->mt_flags & MDB_TXN_RDONLY) { - if (!ti) { - meta = env->me_metas[ mdb_env_pick_meta(env) ]; - txn->mt_txnid = meta->mm_txnid; - txn->mt_u.reader = NULL; - } else { - MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader : - pthread_getspecific(env->me_txkey); - if (r) { - if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1) - return MDB_BAD_RSLOT; - } else { - MDB_PID_T pid = env->me_pid; - MDB_THR_T tid = pthread_self(); - mdb_mutex_t rmutex = MDB_MUTEX(env, r); - - if (!env->me_live_reader) { - rc = mdb_reader_pid(env, Pidset, pid); - if (rc) - return rc; - env->me_live_reader = 1; - } - - if (LOCK_MUTEX(rc, env, rmutex)) - return rc; - nr = ti->mti_numreaders; - for (i=0; imti_readers[i].mr_pid == 0) - break; - if (i == env->me_maxreaders) { - UNLOCK_MUTEX(rmutex); - return MDB_READERS_FULL; - } - ti->mti_readers[i].mr_pid = pid; - ti->mti_readers[i].mr_tid = tid; - if (i == nr) - ti->mti_numreaders = ++nr; - /* Save numreaders for un-mutexed mdb_env_close() */ - env->me_numreaders = nr; - UNLOCK_MUTEX(rmutex); - - r = &ti->mti_readers[i]; - new_notls = (env->me_flags & MDB_NOTLS); - if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { - r->mr_pid = 0; - return rc; - } - } - txn->mt_txnid = r->mr_txnid = ti->mti_txnid; - txn->mt_u.reader = r; - meta = env->me_metas[txn->mt_txnid & 1]; - } - } else { - if (ti) { - if (LOCK_MUTEX(rc, env, MDB_MUTEX(env, w))) - return rc; - - txn->mt_txnid = ti->mti_txnid; - meta = env->me_metas[txn->mt_txnid & 1]; - } else { - meta = env->me_metas[ mdb_env_pick_meta(env) ]; - txn->mt_txnid = meta->mm_txnid; - } - txn->mt_txnid++; -#if MDB_DEBUG - if (txn->mt_txnid == mdb_debug_start) - mdb_debug = 1; -#endif - txn->mt_dirty_room = MDB_IDL_UM_MAX; - txn->mt_u.dirty_list = env->me_dirty_list; - txn->mt_u.dirty_list[0].mid = 0; - txn->mt_free_pgs = env->me_free_pgs; - txn->mt_free_pgs[0] = 0; - txn->mt_spill_pgs = NULL; - env->me_txn = txn; - memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); - } - - /* Copy the DB info and flags */ - memcpy(txn->mt_dbs, meta->mm_dbs, 2 * sizeof(MDB_db)); - - /* Moved to here to avoid a data race in read TXNs */ - txn->mt_next_pgno = meta->mm_last_pg+1; - - for (i=2; imt_numdbs; i++) { - x = env->me_dbflags[i]; - txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; - txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_STALE : 0; - } - txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; - - if (env->me_maxpg < txn->mt_next_pgno) { - mdb_txn_reset0(txn, "renew0-mapfail"); - if (new_notls) { - txn->mt_u.reader->mr_pid = 0; - txn->mt_u.reader = NULL; - } - return MDB_MAP_RESIZED; - } - - return MDB_SUCCESS; -} - -int -mdb_txn_renew(MDB_txn *txn) -{ - int rc; - - if (!txn || txn->mt_dbxs) /* A reset txn has mt_dbxs==NULL */ - return EINVAL; - - if (txn->mt_env->me_flags & MDB_FATAL_ERROR) { - DPUTS("environment had fatal error, must shutdown!"); - return MDB_PANIC; - } - - rc = mdb_txn_renew0(txn); - if (rc == MDB_SUCCESS) { - DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", - txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', - (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); - } - return rc; -} - -int -mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) -{ - MDB_txn *txn; - MDB_ntxn *ntxn; - int rc, size, tsize = sizeof(MDB_txn); - - if (env->me_flags & MDB_FATAL_ERROR) { - DPUTS("environment had fatal error, must shutdown!"); - return MDB_PANIC; - } - if ((env->me_flags & MDB_RDONLY) && !(flags & MDB_RDONLY)) - return EACCES; - if (parent) { - /* Nested transactions: Max 1 child, write txns only, no writemap */ - if (parent->mt_child || - (flags & MDB_RDONLY) || - (parent->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) || - (env->me_flags & MDB_WRITEMAP)) - { - return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN; - } - tsize = sizeof(MDB_ntxn); - } - size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1); - if (!(flags & MDB_RDONLY)) { - if (!parent) { - txn = env->me_txn0; - txn->mt_flags = 0; - goto ok; - } - size += env->me_maxdbs * sizeof(MDB_cursor *); - /* child txns use parent's dbiseqs */ - if (!parent) - size += env->me_maxdbs * sizeof(unsigned int); - } - - if ((txn = calloc(1, size)) == NULL) { - DPRINTF(("calloc: %s", strerror(errno))); - return ENOMEM; - } - txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); - if (flags & MDB_RDONLY) { - txn->mt_flags |= MDB_TXN_RDONLY; - txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); - txn->mt_dbiseqs = env->me_dbiseqs; - } else { - txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); - if (parent) { - txn->mt_dbiseqs = parent->mt_dbiseqs; - txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); - } else { - txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); - txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); - } - } - txn->mt_env = env; - -ok: - if (parent) { - unsigned int i; - txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); - if (!txn->mt_u.dirty_list || - !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) - { - free(txn->mt_u.dirty_list); - free(txn); - return ENOMEM; - } - txn->mt_txnid = parent->mt_txnid; - txn->mt_dirty_room = parent->mt_dirty_room; - txn->mt_u.dirty_list[0].mid = 0; - txn->mt_spill_pgs = NULL; - txn->mt_next_pgno = parent->mt_next_pgno; - parent->mt_child = txn; - txn->mt_parent = parent; - txn->mt_numdbs = parent->mt_numdbs; - txn->mt_flags = parent->mt_flags; - txn->mt_dbxs = parent->mt_dbxs; - memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); - /* Copy parent's mt_dbflags, but clear DB_NEW */ - for (i=0; imt_numdbs; i++) - txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW; - rc = 0; - ntxn = (MDB_ntxn *)txn; - ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */ - if (env->me_pghead) { - size = MDB_IDL_SIZEOF(env->me_pghead); - env->me_pghead = mdb_midl_alloc(env->me_pghead[0]); - if (env->me_pghead) - memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size); - else - rc = ENOMEM; - } - if (!rc) - rc = mdb_cursor_shadow(parent, txn); - if (rc) - mdb_txn_reset0(txn, "beginchild-fail"); - } else { - rc = mdb_txn_renew0(txn); - } - if (rc) { - if (txn != env->me_txn0) - free(txn); - } else { - *ret = txn; - DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", - txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', - (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); - } - - return rc; -} - -MDB_env * -mdb_txn_env(MDB_txn *txn) -{ - if(!txn) return NULL; - return txn->mt_env; -} - -/** Export or close DBI handles opened in this txn. */ -static void -mdb_dbis_update(MDB_txn *txn, int keep) -{ - int i; - MDB_dbi n = txn->mt_numdbs; - MDB_env *env = txn->mt_env; - unsigned char *tdbflags = txn->mt_dbflags; - - for (i = n; --i >= 2;) { - if (tdbflags[i] & DB_NEW) { - if (keep) { - env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; - } else { - char *ptr = env->me_dbxs[i].md_name.mv_data; - if (ptr) { - env->me_dbxs[i].md_name.mv_data = NULL; - env->me_dbxs[i].md_name.mv_size = 0; - env->me_dbflags[i] = 0; - env->me_dbiseqs[i]++; - free(ptr); - } - } - } - } - if (keep && env->me_numdbs < n) - env->me_numdbs = n; -} - -/** Common code for #mdb_txn_reset() and #mdb_txn_abort(). - * May be called twice for readonly txns: First reset it, then abort. - * @param[in] txn the transaction handle to reset - * @param[in] act why the transaction is being reset - */ -static void -mdb_txn_reset0(MDB_txn *txn, const char *act) -{ - MDB_env *env = txn->mt_env; - - /* Close any DBI handles opened in this txn */ - mdb_dbis_update(txn, 0); - - DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", - act, txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', - (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); - - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { - if (txn->mt_u.reader) { - txn->mt_u.reader->mr_txnid = (txnid_t)-1; - if (!(env->me_flags & MDB_NOTLS)) - txn->mt_u.reader = NULL; /* txn does not own reader */ - } - txn->mt_numdbs = 0; /* close nothing if called again */ - txn->mt_dbxs = NULL; /* mark txn as reset */ - } else { - pgno_t *pghead = env->me_pghead; - env->me_pghead = NULL; - env->me_pglast = 0; - - if (!(env->me_flags & MDB_WRITEMAP)) { - mdb_dlist_free(txn); - } - - if (!txn->mt_parent) { - if (mdb_midl_shrink(&txn->mt_free_pgs)) - env->me_free_pgs = txn->mt_free_pgs; - - env->me_txn = NULL; - /* The writer mutex was locked in mdb_txn_begin. */ - if (env->me_txns) - UNLOCK_MUTEX(MDB_MUTEX(env, w)); - } - - mdb_cursors_close(txn, 0); - - mdb_midl_free(pghead); - - if (txn->mt_parent) { - txn->mt_parent->mt_child = NULL; - env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; - mdb_midl_free(txn->mt_free_pgs); - mdb_midl_free(txn->mt_spill_pgs); - free(txn->mt_u.dirty_list); - } - } -} - -void -mdb_txn_reset(MDB_txn *txn) -{ - if (txn == NULL) - return; - - /* This call is only valid for read-only txns */ - if (!(txn->mt_flags & MDB_TXN_RDONLY)) - return; - - mdb_txn_reset0(txn, "reset"); -} - -void -mdb_txn_abort(MDB_txn *txn) -{ - if (txn == NULL) - return; - - if (txn->mt_child) - mdb_txn_abort(txn->mt_child); - - mdb_txn_reset0(txn, "abort"); - /* Free reader slot tied to this txn (if MDB_NOTLS && writable FS) */ - if ((txn->mt_flags & MDB_TXN_RDONLY) && txn->mt_u.reader) - txn->mt_u.reader->mr_pid = 0; - - if (txn != txn->mt_env->me_txn0) - free(txn); -} - -/** Save the freelist as of this transaction to the freeDB. - * This changes the freelist. Keep trying until it stabilizes. - */ -static int -mdb_freelist_save(MDB_txn *txn) -{ - /* env->me_pghead[] can grow and shrink during this call. - * env->me_pglast and txn->mt_free_pgs[] can only grow. - * Page numbers cannot disappear from txn->mt_free_pgs[]. - */ - MDB_cursor mc; - MDB_env *env = txn->mt_env; - int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; - txnid_t pglast = 0, head_id = 0; - pgno_t freecnt = 0, *free_pgs, *mop; - ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; - - mdb_cursor_init(&mc, txn, FREE_DBI, NULL); - - if (env->me_pghead) { - /* Make sure first page of freeDB is touched and on freelist */ - rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST|MDB_PS_MODIFY); - if (rc && rc != MDB_NOTFOUND) - return rc; - } - - if (!env->me_pghead && txn->mt_loose_pgs) { - /* Put loose page numbers in mt_free_pgs, since - * we may be unable to return them to me_pghead. - */ - MDB_page *mp = txn->mt_loose_pgs; - if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) - return rc; - for (; mp; mp = NEXT_LOOSE_PAGE(mp)) - mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); - txn->mt_loose_pgs = NULL; - txn->mt_loose_count = 0; - } - - /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ - clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) - ? SSIZE_MAX : maxfree_1pg; - - for (;;) { - /* Come back here after each Put() in case freelist changed */ - MDB_val key, data; - pgno_t *pgs; - ssize_t j; - - /* If using records from freeDB which we have not yet - * deleted, delete them and any we reserved for me_pghead. - */ - while (pglast < env->me_pglast) { - rc = mdb_cursor_first(&mc, &key, NULL); - if (rc) - return rc; - pglast = head_id = *(txnid_t *)key.mv_data; - total_room = head_room = 0; - mdb_tassert(txn, pglast <= env->me_pglast); - rc = mdb_cursor_del(&mc, 0); - if (rc) - return rc; - } - - /* Save the IDL of pages freed by this txn, to a single record */ - if (freecnt < txn->mt_free_pgs[0]) { - if (!freecnt) { - /* Make sure last page of freeDB is touched and on freelist */ - rc = mdb_page_search(&mc, NULL, MDB_PS_LAST|MDB_PS_MODIFY); - if (rc && rc != MDB_NOTFOUND) - return rc; - } - free_pgs = txn->mt_free_pgs; - /* Write to last page of freeDB */ - key.mv_size = sizeof(txn->mt_txnid); - key.mv_data = &txn->mt_txnid; - do { - freecnt = free_pgs[0]; - data.mv_size = MDB_IDL_SIZEOF(free_pgs); - rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); - if (rc) - return rc; - /* Retry if mt_free_pgs[] grew during the Put() */ - free_pgs = txn->mt_free_pgs; - } while (freecnt < free_pgs[0]); - mdb_midl_sort(free_pgs); - memcpy(data.mv_data, free_pgs, data.mv_size); -#if (MDB_DEBUG) > 1 - { - unsigned int i = free_pgs[0]; - DPRINTF(("IDL write txn %"Z"u root %"Z"u num %u", - txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); - for (; i; i--) - DPRINTF(("IDL %"Z"u", free_pgs[i])); - } -#endif - continue; - } - - mop = env->me_pghead; - mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; - - /* Reserve records for me_pghead[]. Split it if multi-page, - * to avoid searching freeDB for a page range. Use keys in - * range [1,me_pglast]: Smaller than txnid of oldest reader. - */ - if (total_room >= mop_len) { - if (total_room == mop_len || --more < 0) - break; - } else if (head_room >= maxfree_1pg && head_id > 1) { - /* Keep current record (overflow page), add a new one */ - head_id--; - head_room = 0; - } - /* (Re)write {key = head_id, IDL length = head_room} */ - total_room -= head_room; - head_room = mop_len - total_room; - if (head_room > maxfree_1pg && head_id > 1) { - /* Overflow multi-page for part of me_pghead */ - head_room /= head_id; /* amortize page sizes */ - head_room += maxfree_1pg - head_room % (maxfree_1pg + 1); - } else if (head_room < 0) { - /* Rare case, not bothering to delete this record */ - head_room = 0; - } - key.mv_size = sizeof(head_id); - key.mv_data = &head_id; - data.mv_size = (head_room + 1) * sizeof(pgno_t); - rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); - if (rc) - return rc; - /* IDL is initially empty, zero out at least the length */ - pgs = (pgno_t *)data.mv_data; - j = head_room > clean_limit ? head_room : 0; - do { - pgs[j] = 0; - } while (--j >= 0); - total_room += head_room; - } - - /* Return loose page numbers to me_pghead, though usually none are - * left at this point. The pages themselves remain in dirty_list. - */ - if (txn->mt_loose_pgs) { - MDB_page *mp = txn->mt_loose_pgs; - unsigned count = txn->mt_loose_count; - MDB_IDL loose; - /* Room for loose pages + temp IDL with same */ - if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) - return rc; - mop = env->me_pghead; - loose = mop + MDB_IDL_ALLOCLEN(mop) - count; - for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) - loose[ ++count ] = mp->mp_pgno; - loose[0] = count; - mdb_midl_sort(loose); - mdb_midl_xmerge(mop, loose); - txn->mt_loose_pgs = NULL; - txn->mt_loose_count = 0; - mop_len = mop[0]; - } - - /* Fill in the reserved me_pghead records */ - rc = MDB_SUCCESS; - if (mop_len) { - MDB_val key, data; - - mop += mop_len; - rc = mdb_cursor_first(&mc, &key, &data); - for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { - txnid_t id = *(txnid_t *)key.mv_data; - ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; - MDB_ID save; - - mdb_tassert(txn, len >= 0 && id <= env->me_pglast); - key.mv_data = &id; - if (len > mop_len) { - len = mop_len; - data.mv_size = (len + 1) * sizeof(MDB_ID); - } - data.mv_data = mop -= len; - save = mop[0]; - mop[0] = len; - rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT); - mop[0] = save; - if (rc || !(mop_len -= len)) - break; - } - } - return rc; -} - -/** Flush (some) dirty pages to the map, after clearing their dirty flag. - * @param[in] txn the transaction that's being committed - * @param[in] keep number of initial pages in dirty_list to keep dirty. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_flush(MDB_txn *txn, int keep) -{ - MDB_env *env = txn->mt_env; - MDB_ID2L dl = txn->mt_u.dirty_list; - unsigned psize = env->me_psize, j; - int i, pagecount = dl[0].mid, rc; - size_t size = 0, pos = 0; - pgno_t pgno = 0; - MDB_page *dp = NULL; -#ifdef _WIN32 - OVERLAPPED ov; -#else - struct iovec iov[MDB_COMMIT_PAGES]; - ssize_t wpos = 0, wsize = 0, wres; - size_t next_pos = 1; /* impossible pos, so pos != next_pos */ - int n = 0; -#endif - - j = i = keep; - - if (env->me_flags & MDB_WRITEMAP) { - /* Clear dirty flags */ - while (++i <= pagecount) { - dp = dl[i].mptr; - /* Don't flush this page yet */ - if (dp->mp_flags & (P_LOOSE|P_KEEP)) { - dp->mp_flags &= ~P_KEEP; - dl[++j] = dl[i]; - continue; - } - dp->mp_flags &= ~P_DIRTY; - } - goto done; - } - - /* Write the pages */ - for (;;) { - if (++i <= pagecount) { - dp = dl[i].mptr; - /* Don't flush this page yet */ - if (dp->mp_flags & (P_LOOSE|P_KEEP)) { - dp->mp_flags &= ~P_KEEP; - dl[i].mid = 0; - continue; - } - pgno = dl[i].mid; - /* clear dirty flag */ - dp->mp_flags &= ~P_DIRTY; - pos = pgno * psize; - size = psize; - if (IS_OVERFLOW(dp)) size *= dp->mp_pages; - } -#ifdef _WIN32 - else break; - - /* Windows actually supports scatter/gather I/O, but only on - * unbuffered file handles. Since we're relying on the OS page - * cache for all our data, that's self-defeating. So we just - * write pages one at a time. We use the ov structure to set - * the write offset, to at least save the overhead of a Seek - * system call. - */ - DPRINTF(("committing page %"Z"u", pgno)); - memset(&ov, 0, sizeof(ov)); - ov.Offset = pos & 0xffffffff; - ov.OffsetHigh = pos >> 16 >> 16; - if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) { - rc = ErrCode(); - DPRINTF(("WriteFile: %d", rc)); - return rc; - } -#else - /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ - if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { - if (n) { - /* Write previous page(s) */ -#ifdef MDB_USE_PWRITEV - wres = pwritev(env->me_fd, iov, n, wpos); -#else - if (n == 1) { - wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos); - } else { - if (lseek(env->me_fd, wpos, SEEK_SET) == -1) { - rc = ErrCode(); - DPRINTF(("lseek: %s", strerror(rc))); - return rc; - } - wres = writev(env->me_fd, iov, n); - } -#endif - if (wres != wsize) { - if (wres < 0) { - rc = ErrCode(); - DPRINTF(("Write error: %s", strerror(rc))); - } else { - rc = EIO; /* TODO: Use which error code? */ - DPUTS("short write, filesystem full?"); - } - return rc; - } - n = 0; - } - if (i > pagecount) - break; - wpos = pos; - wsize = 0; - } - DPRINTF(("committing page %"Z"u", pgno)); - next_pos = pos + size; - iov[n].iov_len = size; - iov[n].iov_base = (char *)dp; - wsize += size; - n++; -#endif /* _WIN32 */ - } - - /* MIPS has cache coherency issues, this is a no-op everywhere else - * Note: for any size >= on-chip cache size, entire on-chip cache is - * flushed. - */ - CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE); - - for (i = keep; ++i <= pagecount; ) { - dp = dl[i].mptr; - /* This is a page we skipped above */ - if (!dl[i].mid) { - dl[++j] = dl[i]; - dl[j].mid = dp->mp_pgno; - continue; - } - mdb_dpage_free(env, dp); - } - -done: - i--; - txn->mt_dirty_room += i - j; - dl[0].mid = j; - return MDB_SUCCESS; -} - -int -mdb_txn_commit(MDB_txn *txn) -{ - int rc; - unsigned int i; - MDB_env *env; - - if (txn == NULL || txn->mt_env == NULL) - return EINVAL; - - if (txn->mt_child) { - rc = mdb_txn_commit(txn->mt_child); - txn->mt_child = NULL; - if (rc) - goto fail; - } - - env = txn->mt_env; - - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { - mdb_dbis_update(txn, 1); - txn->mt_numdbs = 2; /* so txn_abort() doesn't close any new handles */ - mdb_txn_abort(txn); - return MDB_SUCCESS; - } - - if (F_ISSET(txn->mt_flags, MDB_TXN_ERROR)) { - DPUTS("error flag is set, can't commit"); - if (txn->mt_parent) - txn->mt_parent->mt_flags |= MDB_TXN_ERROR; - rc = MDB_BAD_TXN; - goto fail; - } - - if (txn->mt_parent) { - MDB_txn *parent = txn->mt_parent; - MDB_page **lp; - MDB_ID2L dst, src; - MDB_IDL pspill; - unsigned x, y, len, ps_len; - - /* Append our free list to parent's */ - rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs); - if (rc) - goto fail; - mdb_midl_free(txn->mt_free_pgs); - /* Failures after this must either undo the changes - * to the parent or set MDB_TXN_ERROR in the parent. - */ - - parent->mt_next_pgno = txn->mt_next_pgno; - parent->mt_flags = txn->mt_flags; - - /* Merge our cursors into parent's and close them */ - mdb_cursors_close(txn, 1); - - /* Update parent's DB table. */ - memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); - parent->mt_numdbs = txn->mt_numdbs; - parent->mt_dbflags[0] = txn->mt_dbflags[0]; - parent->mt_dbflags[1] = txn->mt_dbflags[1]; - for (i=2; imt_numdbs; i++) { - /* preserve parent's DB_NEW status */ - x = parent->mt_dbflags[i] & DB_NEW; - parent->mt_dbflags[i] = txn->mt_dbflags[i] | x; - } - - dst = parent->mt_u.dirty_list; - src = txn->mt_u.dirty_list; - /* Remove anything in our dirty list from parent's spill list */ - if ((pspill = parent->mt_spill_pgs) && (ps_len = pspill[0])) { - x = y = ps_len; - pspill[0] = (pgno_t)-1; - /* Mark our dirty pages as deleted in parent spill list */ - for (i=0, len=src[0].mid; ++i <= len; ) { - MDB_ID pn = src[i].mid << 1; - while (pn > pspill[x]) - x--; - if (pn == pspill[x]) { - pspill[x] = 1; - y = --x; - } - } - /* Squash deleted pagenums if we deleted any */ - for (x=y; ++x <= ps_len; ) - if (!(pspill[x] & 1)) - pspill[++y] = pspill[x]; - pspill[0] = y; - } - - /* Find len = length of merging our dirty list with parent's */ - x = dst[0].mid; - dst[0].mid = 0; /* simplify loops */ - if (parent->mt_parent) { - len = x + src[0].mid; - y = mdb_mid2l_search(src, dst[x].mid + 1) - 1; - for (i = x; y && i; y--) { - pgno_t yp = src[y].mid; - while (yp < dst[i].mid) - i--; - if (yp == dst[i].mid) { - i--; - len--; - } - } - } else { /* Simplify the above for single-ancestor case */ - len = MDB_IDL_UM_MAX - txn->mt_dirty_room; - } - /* Merge our dirty list with parent's */ - y = src[0].mid; - for (i = len; y; dst[i--] = src[y--]) { - pgno_t yp = src[y].mid; - while (yp < dst[x].mid) - dst[i--] = dst[x--]; - if (yp == dst[x].mid) - free(dst[x--].mptr); - } - mdb_tassert(txn, i == x); - dst[0].mid = len; - free(txn->mt_u.dirty_list); - parent->mt_dirty_room = txn->mt_dirty_room; - if (txn->mt_spill_pgs) { - if (parent->mt_spill_pgs) { - /* TODO: Prevent failure here, so parent does not fail */ - rc = mdb_midl_append_list(&parent->mt_spill_pgs, txn->mt_spill_pgs); - if (rc) - parent->mt_flags |= MDB_TXN_ERROR; - mdb_midl_free(txn->mt_spill_pgs); - mdb_midl_sort(parent->mt_spill_pgs); - } else { - parent->mt_spill_pgs = txn->mt_spill_pgs; - } - } - - /* Append our loose page list to parent's */ - for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(lp)) - ; - *lp = txn->mt_loose_pgs; - parent->mt_loose_count += txn->mt_loose_count; - - parent->mt_child = NULL; - mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); - free(txn); - return rc; - } - - if (txn != env->me_txn) { - DPUTS("attempt to commit unknown transaction"); - rc = EINVAL; - goto fail; - } - - mdb_cursors_close(txn, 0); - - if (!txn->mt_u.dirty_list[0].mid && - !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) - goto done; - - DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u", - txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); - - /* Update DB root pointers */ - if (txn->mt_numdbs > 2) { - MDB_cursor mc; - MDB_dbi i; - MDB_val data; - data.mv_size = sizeof(MDB_db); - - mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); - for (i = 2; i < txn->mt_numdbs; i++) { - if (txn->mt_dbflags[i] & DB_DIRTY) { - if (TXN_DBI_CHANGED(txn, i)) { - rc = MDB_BAD_DBI; - goto fail; - } - data.mv_data = &txn->mt_dbs[i]; - rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, 0); - if (rc) - goto fail; - } - } - } - - rc = mdb_freelist_save(txn); - if (rc) - goto fail; - - mdb_midl_free(env->me_pghead); - env->me_pghead = NULL; - if (mdb_midl_shrink(&txn->mt_free_pgs)) - env->me_free_pgs = txn->mt_free_pgs; - -#if (MDB_DEBUG) > 2 - mdb_audit(txn); -#endif - - if ((rc = mdb_page_flush(txn, 0)) || - (rc = mdb_env_sync(env, 0)) || - (rc = mdb_env_write_meta(txn))) - goto fail; - - /* Free P_LOOSE pages left behind in dirty_list */ - if (!(env->me_flags & MDB_WRITEMAP)) - mdb_dlist_free(txn); - -done: - env->me_pglast = 0; - env->me_txn = NULL; - mdb_dbis_update(txn, 1); - - if (env->me_txns) - UNLOCK_MUTEX(MDB_MUTEX(env, w)); - if (txn != env->me_txn0) - free(txn); - - return MDB_SUCCESS; - -fail: - mdb_txn_abort(txn); - return rc; -} - -/** Read the environment parameters of a DB environment before - * mapping it into memory. - * @param[in] env the environment handle - * @param[out] meta address of where to store the meta information - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_env_read_header(MDB_env *env, MDB_meta *meta) -{ - MDB_metabuf pbuf; - MDB_page *p; - MDB_meta *m; - int i, rc, off; - enum { Size = sizeof(pbuf) }; - - /* We don't know the page size yet, so use a minimum value. - * Read both meta pages so we can use the latest one. - */ - - for (i=off=0; i<2; i++, off = meta->mm_psize) { -#ifdef _WIN32 - DWORD len; - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); - ov.Offset = off; - rc = ReadFile(env->me_fd, &pbuf, Size, &len, &ov) ? (int)len : -1; - if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) - rc = 0; -#else - rc = pread(env->me_fd, &pbuf, Size, off); -#endif - if (rc != Size) { - if (rc == 0 && off == 0) - return ENOENT; - rc = rc < 0 ? (int) ErrCode() : MDB_INVALID; - DPRINTF(("read: %s", mdb_strerror(rc))); - return rc; - } - - p = (MDB_page *)&pbuf; - - if (!F_ISSET(p->mp_flags, P_META)) { - DPRINTF(("page %"Z"u not a meta page", p->mp_pgno)); - return MDB_INVALID; - } - - m = METADATA(p); - if (m->mm_magic != MDB_MAGIC) { - DPUTS("meta has invalid magic"); - return MDB_INVALID; - } - - if (m->mm_version != MDB_DATA_VERSION) { - DPRINTF(("database is version %u, expected version %u", - m->mm_version, MDB_DATA_VERSION)); - return MDB_VERSION_MISMATCH; - } - - if (off == 0 || m->mm_txnid > meta->mm_txnid) - *meta = *m; - } - return 0; -} - -static void ESECT -mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) -{ - meta->mm_magic = MDB_MAGIC; - meta->mm_version = MDB_DATA_VERSION; - meta->mm_mapsize = env->me_mapsize; - meta->mm_psize = env->me_psize; - meta->mm_last_pg = 1; - meta->mm_flags = env->me_flags & 0xffff; - meta->mm_flags |= MDB_INTEGERKEY; - meta->mm_dbs[0].md_root = P_INVALID; - meta->mm_dbs[1].md_root = P_INVALID; -} - -/** Write the environment parameters of a freshly created DB environment. - * @param[in] env the environment handle - * @param[out] meta address of where to store the meta information - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_env_init_meta(MDB_env *env, MDB_meta *meta) -{ - MDB_page *p, *q; - int rc; - unsigned int psize; -#ifdef _WIN32 - DWORD len; - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); -#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ - ov.Offset = pos; \ - rc = WriteFile(fd, ptr, size, &len, &ov); } while(0) -#else - int len; -#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ - len = pwrite(fd, ptr, size, pos); \ - rc = (len >= 0); } while(0) -#endif - - DPUTS("writing new meta page"); - - psize = env->me_psize; - - mdb_env_init_meta0(env, meta); - - p = calloc(2, psize); - p->mp_pgno = 0; - p->mp_flags = P_META; - *(MDB_meta *)METADATA(p) = *meta; - - q = (MDB_page *)((char *)p + psize); - q->mp_pgno = 1; - q->mp_flags = P_META; - *(MDB_meta *)METADATA(q) = *meta; - - DO_PWRITE(rc, env->me_fd, p, psize * 2, len, 0); - if (!rc) - rc = ErrCode(); - else if ((unsigned) len == psize * 2) - rc = MDB_SUCCESS; - else - rc = ENOSPC; - free(p); - return rc; -} - -/** Update the environment info to commit a transaction. - * @param[in] txn the transaction that's being committed - * @return 0 on success, non-zero on failure. - */ -static int -mdb_env_write_meta(MDB_txn *txn) -{ - MDB_env *env; - MDB_meta meta, metab, *mp; - size_t mapsize; - off_t off; - int rc, len, toggle; - char *ptr; - HANDLE mfd; -#ifdef _WIN32 - OVERLAPPED ov; -#else - int r2; -#endif - - toggle = txn->mt_txnid & 1; - DPRINTF(("writing meta page %d for root page %"Z"u", - toggle, txn->mt_dbs[MAIN_DBI].md_root)); - - env = txn->mt_env; - mp = env->me_metas[toggle]; - mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; - /* Persist any increases of mapsize config */ - if (mapsize < env->me_mapsize) - mapsize = env->me_mapsize; - - if (env->me_flags & MDB_WRITEMAP) { - mp->mm_mapsize = mapsize; - mp->mm_dbs[0] = txn->mt_dbs[0]; - mp->mm_dbs[1] = txn->mt_dbs[1]; - mp->mm_last_pg = txn->mt_next_pgno - 1; - mp->mm_txnid = txn->mt_txnid; - if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { - unsigned meta_size = env->me_psize; - rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; - ptr = env->me_map; - if (toggle) { -#ifndef _WIN32 /* POSIX msync() requires ptr = start of OS page */ - if (meta_size < env->me_os_psize) - meta_size += meta_size; - else -#endif - ptr += meta_size; - } - if (MDB_MSYNC(ptr, meta_size, rc)) { - rc = ErrCode(); - goto fail; - } - } - goto done; - } - metab.mm_txnid = env->me_metas[toggle]->mm_txnid; - metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg; - - meta.mm_mapsize = mapsize; - meta.mm_dbs[0] = txn->mt_dbs[0]; - meta.mm_dbs[1] = txn->mt_dbs[1]; - meta.mm_last_pg = txn->mt_next_pgno - 1; - meta.mm_txnid = txn->mt_txnid; - - off = offsetof(MDB_meta, mm_mapsize); - ptr = (char *)&meta + off; - len = sizeof(MDB_meta) - off; - if (toggle) - off += env->me_psize; - off += PAGEHDRSZ; - - /* Write to the SYNC fd */ - mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ? - env->me_fd : env->me_mfd; -#ifdef _WIN32 - { - memset(&ov, 0, sizeof(ov)); - ov.Offset = off; - if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov)) - rc = -1; - } -#else - rc = pwrite(mfd, ptr, len, off); -#endif - if (rc != len) { - rc = rc < 0 ? ErrCode() : EIO; - DPUTS("write failed, disk error?"); - /* On a failure, the pagecache still contains the new data. - * Write some old data back, to prevent it from being used. - * Use the non-SYNC fd; we know it will fail anyway. - */ - meta.mm_last_pg = metab.mm_last_pg; - meta.mm_txnid = metab.mm_txnid; -#ifdef _WIN32 - memset(&ov, 0, sizeof(ov)); - ov.Offset = off; - WriteFile(env->me_fd, ptr, len, NULL, &ov); -#else - r2 = pwrite(env->me_fd, ptr, len, off); - (void)r2; /* Silence warnings. We don't care about pwrite's return value */ -#endif -fail: - env->me_flags |= MDB_FATAL_ERROR; - return rc; - } - /* MIPS has cache coherency issues, this is a no-op everywhere else */ - CACHEFLUSH(env->me_map + off, len, DCACHE); -done: - /* Memory ordering issues are irrelevant; since the entire writer - * is wrapped by wmutex, all of these changes will become visible - * after the wmutex is unlocked. Since the DB is multi-version, - * readers will get consistent data regardless of how fresh or - * how stale their view of these values is. - */ - if (env->me_txns) - env->me_txns->mti_txnid = txn->mt_txnid; - - return MDB_SUCCESS; -} - -/** Check both meta pages to see which one is newer. - * @param[in] env the environment handle - * @return meta toggle (0 or 1). - */ -static int -mdb_env_pick_meta(const MDB_env *env) -{ - return (env->me_metas[0]->mm_txnid < env->me_metas[1]->mm_txnid); -} - -int ESECT -mdb_env_create(MDB_env **env) -{ - MDB_env *e; - - e = calloc(1, sizeof(MDB_env)); - if (!e) - return ENOMEM; - - e->me_maxreaders = DEFAULT_READERS; - e->me_maxdbs = e->me_numdbs = 2; - e->me_fd = INVALID_HANDLE_VALUE; - e->me_lfd = INVALID_HANDLE_VALUE; - e->me_mfd = INVALID_HANDLE_VALUE; -#ifdef MDB_USE_POSIX_SEM - e->me_rmutex = SEM_FAILED; - e->me_wmutex = SEM_FAILED; -#endif - e->me_pid = getpid(); - GET_PAGESIZE(e->me_os_psize); - VGMEMP_CREATE(e,0,0); - *env = e; - return MDB_SUCCESS; -} - -static int ESECT -mdb_env_map(MDB_env *env, void *addr) -{ - MDB_page *p; - unsigned int flags = env->me_flags; -#ifdef _WIN32 - int rc; - HANDLE mh; - LONG sizelo, sizehi; - size_t msize; - - if (flags & MDB_RDONLY) { - /* Don't set explicit map size, use whatever exists */ - msize = 0; - sizelo = 0; - sizehi = 0; - } else { - msize = env->me_mapsize; - sizelo = msize & 0xffffffff; - sizehi = msize >> 16 >> 16; /* only needed on Win64 */ - - /* Windows won't create mappings for zero length files. - * and won't map more than the file size. - * Just set the maxsize right now. - */ - if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo - || !SetEndOfFile(env->me_fd) - || SetFilePointer(env->me_fd, 0, NULL, 0) != 0) - return ErrCode(); - } - - mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ? - PAGE_READWRITE : PAGE_READONLY, - sizehi, sizelo, NULL); - if (!mh) - return ErrCode(); - env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ? - FILE_MAP_WRITE : FILE_MAP_READ, - 0, 0, msize, addr); - rc = env->me_map ? 0 : ErrCode(); - CloseHandle(mh); - if (rc) - return rc; -#else - int prot = PROT_READ; - if (flags & MDB_WRITEMAP) { - prot |= PROT_WRITE; - if (ftruncate(env->me_fd, env->me_mapsize) < 0) - return ErrCode(); - } - env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED, - env->me_fd, 0); - if (env->me_map == MAP_FAILED) { - env->me_map = NULL; - return ErrCode(); - } - - if (flags & MDB_NORDAHEAD) { - /* Turn off readahead. It's harmful when the DB is larger than RAM. */ -#ifdef MADV_RANDOM - madvise(env->me_map, env->me_mapsize, MADV_RANDOM); -#else -#ifdef POSIX_MADV_RANDOM - posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM); -#endif /* POSIX_MADV_RANDOM */ -#endif /* MADV_RANDOM */ - } -#endif /* _WIN32 */ - - /* Can happen because the address argument to mmap() is just a - * hint. mmap() can pick another, e.g. if the range is in use. - * The MAP_FIXED flag would prevent that, but then mmap could - * instead unmap existing pages to make room for the new map. - */ - if (addr && env->me_map != addr) - return EBUSY; /* TODO: Make a new MDB_* error code? */ - - p = (MDB_page *)env->me_map; - env->me_metas[0] = METADATA(p); - env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize); - - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_mapsize(MDB_env *env, size_t size) -{ - /* If env is already open, caller is responsible for making - * sure there are no active txns. - */ - if (env->me_map) { - int rc; - void *old; - if (env->me_txn) - return EINVAL; - if (!size) - size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize; - else if (size < env->me_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. - */ - size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; - if (size < minsize) - size = minsize; - } - munmap(env->me_map, env->me_mapsize); - env->me_mapsize = size; - old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; - rc = mdb_env_map(env, old); - if (rc) - return rc; - } - env->me_mapsize = size; - if (env->me_psize) - env->me_maxpg = env->me_mapsize / env->me_psize; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs) -{ - if (env->me_map) - return EINVAL; - env->me_maxdbs = dbs + 2; /* Named databases + main and free DB */ - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) -{ - if (env->me_map || readers < 1) - return EINVAL; - env->me_maxreaders = readers; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) -{ - if (!env || !readers) - return EINVAL; - *readers = env->me_maxreaders; - return MDB_SUCCESS; -} - -/** Further setup required for opening an LMDB environment - */ -static int ESECT -mdb_env_open2(MDB_env *env) -{ - unsigned int flags = env->me_flags; - int i, newenv = 0, rc; - MDB_meta meta; - -#ifdef _WIN32 - /* See if we should use QueryLimited */ - rc = GetVersion(); - if ((rc & 0xff) > 5) - env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION; - else - env->me_pidquery = PROCESS_QUERY_INFORMATION; -#endif /* _WIN32 */ - - memset(&meta, 0, sizeof(meta)); - - if ((i = mdb_env_read_header(env, &meta)) != 0) { - if (i != ENOENT) - return i; - DPUTS("new mdbenv"); - newenv = 1; - env->me_psize = env->me_os_psize; - if (env->me_psize > MAX_PAGESIZE) - env->me_psize = MAX_PAGESIZE; - } else { - env->me_psize = meta.mm_psize; - } - - /* Was a mapsize configured? */ - if (!env->me_mapsize) { - /* If this is a new environment, take the default, - * else use the size recorded in the existing env. - */ - env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize; - } else if (env->me_mapsize < meta.mm_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. - */ - size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; - if (env->me_mapsize < minsize) - env->me_mapsize = minsize; - } - - rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); - if (rc) - return rc; - - if (newenv) { - if (flags & MDB_FIXEDMAP) - meta.mm_address = env->me_map; - i = mdb_env_init_meta(env, &meta); - if (i != MDB_SUCCESS) { - return i; - } - } - - env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; - env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2) - - sizeof(indx_t); -#if !(MDB_MAXKEYSIZE) - env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db)); -#endif - env->me_maxpg = env->me_mapsize / env->me_psize; - -#if MDB_DEBUG - { - int toggle = mdb_env_pick_meta(env); - MDB_db *db = &env->me_metas[toggle]->mm_dbs[MAIN_DBI]; - - DPRINTF(("opened database version %u, pagesize %u", - env->me_metas[0]->mm_version, env->me_psize)); - DPRINTF(("using meta page %d", toggle)); - DPRINTF(("depth: %u", db->md_depth)); - DPRINTF(("entries: %"Z"u", db->md_entries)); - DPRINTF(("branch pages: %"Z"u", db->md_branch_pages)); - DPRINTF(("leaf pages: %"Z"u", db->md_leaf_pages)); - DPRINTF(("overflow pages: %"Z"u", db->md_overflow_pages)); - DPRINTF(("root: %"Z"u", db->md_root)); - } -#endif - - return MDB_SUCCESS; -} - - -/** Release a reader thread's slot in the reader lock table. - * This function is called automatically when a thread exits. - * @param[in] ptr This points to the slot in the reader lock table. - */ -static void -mdb_env_reader_dest(void *ptr) -{ - MDB_reader *reader = ptr; - - reader->mr_pid = 0; -} - -#ifdef _WIN32 -/** Junk for arranging thread-specific callbacks on Windows. This is - * necessarily platform and compiler-specific. Windows supports up - * to 1088 keys. Let's assume nobody opens more than 64 environments - * in a single process, for now. They can override this if needed. - */ -#ifndef MAX_TLS_KEYS -#define MAX_TLS_KEYS 64 -#endif -static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS]; -static int mdb_tls_nkeys; - -static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr) -{ - int i; - switch(reason) { - case DLL_PROCESS_ATTACH: break; - case DLL_THREAD_ATTACH: break; - case DLL_THREAD_DETACH: - for (i=0; ime_txns->mti_txnid = env->me_metas[toggle]->mm_txnid; - -#ifdef _WIN32 - { - OVERLAPPED ov; - /* First acquire a shared lock. The Unlock will - * then release the existing exclusive lock. - */ - memset(&ov, 0, sizeof(ov)); - if (!LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { - rc = ErrCode(); - } else { - UnlockFile(env->me_lfd, 0, 0, 1, 0); - *excl = 0; - } - } -#else - { - struct flock lock_info; - /* The shared lock replaces the existing lock */ - memset((void *)&lock_info, 0, sizeof(lock_info)); - lock_info.l_type = F_RDLCK; - lock_info.l_whence = SEEK_SET; - lock_info.l_start = 0; - lock_info.l_len = 1; - while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && - (rc = ErrCode()) == EINTR) ; - *excl = rc ? -1 : 0; /* error may mean we lost the lock */ - } -#endif - - return rc; -} - -/** Try to get exlusive lock, otherwise shared. - * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. - */ -static int ESECT -mdb_env_excl_lock(MDB_env *env, int *excl) -{ - int rc = 0; -#ifdef _WIN32 - if (LockFile(env->me_lfd, 0, 0, 1, 0)) { - *excl = 1; - } else { - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); - if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { - *excl = 0; - } else { - rc = ErrCode(); - } - } -#else - struct flock lock_info; - memset((void *)&lock_info, 0, sizeof(lock_info)); - lock_info.l_type = F_WRLCK; - lock_info.l_whence = SEEK_SET; - lock_info.l_start = 0; - lock_info.l_len = 1; - while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && - (rc = ErrCode()) == EINTR) ; - if (!rc) { - *excl = 1; - } else -# ifdef MDB_USE_POSIX_SEM - if (*excl < 0) /* always true when !MDB_USE_POSIX_SEM */ -# endif - { - lock_info.l_type = F_RDLCK; - while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) && - (rc = ErrCode()) == EINTR) ; - if (rc == 0) - *excl = 0; - } -#endif - return rc; -} - -#ifdef MDB_USE_HASH -/* - * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code - * - * @(#) $Revision: 5.1 $ - * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ - * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ - * - * http://www.isthe.com/chongo/tech/comp/fnv/index.html - * - *** - * - * Please do not copyright this code. This code is in the public domain. - * - * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, - * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO - * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR - * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF - * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR - * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - * PERFORMANCE OF THIS SOFTWARE. - * - * By: - * chongo /\oo/\ - * http://www.isthe.com/chongo/ - * - * Share and Enjoy! :-) - */ - -typedef unsigned long long mdb_hash_t; -#define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL) - -/** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer - * @param[in] val value to hash - * @param[in] hval initial value for hash - * @return 64 bit hash - * - * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the - * hval arg on the first call. - */ -static mdb_hash_t -mdb_hash_val(MDB_val *val, mdb_hash_t hval) -{ - unsigned char *s = (unsigned char *)val->mv_data; /* unsigned string */ - unsigned char *end = s + val->mv_size; - /* - * FNV-1a hash each octet of the string - */ - while (s < end) { - /* xor the bottom with the current octet */ - hval ^= (mdb_hash_t)*s++; - - /* multiply by the 64 bit FNV magic prime mod 2^64 */ - hval += (hval << 1) + (hval << 4) + (hval << 5) + - (hval << 7) + (hval << 8) + (hval << 40); - } - /* return our new hash value */ - return hval; -} - -/** Hash the string and output the encoded hash. - * This uses modified RFC1924 Ascii85 encoding to accommodate systems with - * very short name limits. We don't care about the encoding being reversible, - * we just want to preserve as many bits of the input as possible in a - * small printable string. - * @param[in] str string to hash - * @param[out] encbuf an array of 11 chars to hold the hash - */ -static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; - -static void -mdb_pack85(unsigned long l, char *out) -{ - int i; - - for (i=0; i<5; i++) { - *out++ = mdb_a85[l % 85]; - l /= 85; - } -} - -static void -mdb_hash_enc(MDB_val *val, char *encbuf) -{ - mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT); - - mdb_pack85(h, encbuf); - mdb_pack85(h>>32, encbuf+5); - encbuf[10] = '\0'; -} -#endif - -/** Open and/or initialize the lock region for the environment. - * @param[in] env The LMDB environment. - * @param[in] lpath The pathname of the file used for the lock region. - * @param[in] mode The Unix permissions for the file, if we create it. - * @param[out] excl Resulting file lock type: -1 none, 0 shared, 1 exclusive - * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) -{ -#ifdef _WIN32 -# define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT -#else -# define MDB_ERRCODE_ROFS EROFS -#ifdef O_CLOEXEC /* Linux: Open file and set FD_CLOEXEC atomically */ -# define MDB_CLOEXEC O_CLOEXEC -#else - int fdflags; -# define MDB_CLOEXEC 0 -#endif -#endif - int rc; - off_t size, rsize; - -#ifdef _WIN32 - env->me_lfd = CreateFile(lpath, GENERIC_READ|GENERIC_WRITE, - FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, - FILE_ATTRIBUTE_NORMAL, NULL); -#else - env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode); -#endif - if (env->me_lfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { - return MDB_SUCCESS; - } - goto fail_errno; - } -#if ! ((MDB_CLOEXEC) || defined(_WIN32)) - /* Lose record locks when exec*() */ - if ((fdflags = fcntl(env->me_lfd, F_GETFD) | FD_CLOEXEC) >= 0) - fcntl(env->me_lfd, F_SETFD, fdflags); -#endif - - if (!(env->me_flags & MDB_NOTLS)) { - rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); - if (rc) - goto fail; - env->me_flags |= MDB_ENV_TXKEY; -#ifdef _WIN32 - /* Windows TLS callbacks need help finding their TLS info. */ - if (mdb_tls_nkeys >= MAX_TLS_KEYS) { - rc = MDB_TLS_FULL; - goto fail; - } - mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey; -#endif - } - - /* Try to get exclusive lock. If we succeed, then - * nobody is using the lock region and we should initialize it. - */ - if ((rc = mdb_env_excl_lock(env, excl))) goto fail; - -#ifdef _WIN32 - size = GetFileSize(env->me_lfd, NULL); -#else - size = lseek(env->me_lfd, 0, SEEK_END); - if (size == -1) goto fail_errno; -#endif - rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo); - if (size < rsize && *excl > 0) { -#ifdef _WIN32 - if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != (DWORD)rsize - || !SetEndOfFile(env->me_lfd)) - goto fail_errno; -#else - if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno; -#endif - } else { - rsize = size; - size = rsize - sizeof(MDB_txninfo); - env->me_maxreaders = size/sizeof(MDB_reader) + 1; - } - { -#ifdef _WIN32 - HANDLE mh; - mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE, - 0, 0, NULL); - if (!mh) goto fail_errno; - env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL); - CloseHandle(mh); - if (!env->me_txns) goto fail_errno; -#else - void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED, - env->me_lfd, 0); - if (m == MAP_FAILED) goto fail_errno; - env->me_txns = m; -#endif - } - if (*excl > 0) { -#ifdef _WIN32 - BY_HANDLE_FILE_INFORMATION stbuf; - struct { - DWORD volume; - DWORD nhigh; - DWORD nlow; - } idbuf; - MDB_val val; - char encbuf[11]; - - if (!mdb_sec_inited) { - InitializeSecurityDescriptor(&mdb_null_sd, - SECURITY_DESCRIPTOR_REVISION); - SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE); - mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES); - mdb_all_sa.bInheritHandle = FALSE; - mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd; - mdb_sec_inited = 1; - } - if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno; - idbuf.volume = stbuf.dwVolumeSerialNumber; - idbuf.nhigh = stbuf.nFileIndexHigh; - idbuf.nlow = stbuf.nFileIndexLow; - val.mv_data = &idbuf; - val.mv_size = sizeof(idbuf); - mdb_hash_enc(&val, encbuf); - sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf); - sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf); - env->me_rmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_rmname); - if (!env->me_rmutex) goto fail_errno; - env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname); - if (!env->me_wmutex) goto fail_errno; - env->me_flags |= MDB_ROBUST; -#elif defined(MDB_USE_POSIX_SEM) - struct stat stbuf; - struct { - dev_t dev; - ino_t ino; - } idbuf; - MDB_val val; - char encbuf[11]; - -#if defined(__NetBSD__) -#define MDB_SHORT_SEMNAMES 1 /* limited to 14 chars */ -#endif - if (fstat(env->me_lfd, &stbuf)) goto fail_errno; - idbuf.dev = stbuf.st_dev; - idbuf.ino = stbuf.st_ino; - val.mv_data = &idbuf; - val.mv_size = sizeof(idbuf); - mdb_hash_enc(&val, encbuf); -#ifdef MDB_SHORT_SEMNAMES - encbuf[9] = '\0'; /* drop name from 15 chars to 14 chars */ -#endif - sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf); - sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf); - /* Clean up after a previous run, if needed: Try to - * remove both semaphores before doing anything else. - */ - sem_unlink(env->me_txns->mti_rmname); - sem_unlink(env->me_txns->mti_wmname); - env->me_rmutex = sem_open(env->me_txns->mti_rmname, - O_CREAT|O_EXCL, mode, 1); - if (env->me_rmutex == SEM_FAILED) goto fail_errno; - env->me_wmutex = sem_open(env->me_txns->mti_wmname, - O_CREAT|O_EXCL, mode, 1); - if (env->me_wmutex == SEM_FAILED) goto fail_errno; -#else /* MDB_USE_POSIX_SEM */ - pthread_mutexattr_t mattr; - - if ((rc = pthread_mutexattr_init(&mattr)) - || (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) -#ifdef MDB_ROBUST_SUPPORTED - || ((env->me_flags & MDB_ROBUST) && - (rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST))) -#endif - || (rc = pthread_mutex_init(&env->me_txns->mti_rmutex, &mattr)) - || (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr))) - goto fail; - pthread_mutexattr_destroy(&mattr); -#endif /* _WIN32 || MDB_USE_POSIX_SEM */ -#ifndef MDB_ROBUST_SUPPORTED - env->me_flags &= ~MDB_ROBUST; -#endif - - env->me_txns->mti_magic = MDB_MAGIC; - env->me_txns->mti_format = MDB_LOCK_FORMAT; - env->me_txns->mti_txnid = 0; - env->me_txns->mti_numreaders = 0; - env->me_txns->mti_flags = env->me_flags; - - } else { - if (env->me_txns->mti_magic != MDB_MAGIC) { - DPUTS("lock region has invalid magic"); - rc = MDB_INVALID; - goto fail; - } - if (env->me_txns->mti_format != MDB_LOCK_FORMAT) { - DPRINTF(("lock region has format+version 0x%x, expected 0x%x", - env->me_txns->mti_format, MDB_LOCK_FORMAT)); - rc = MDB_VERSION_MISMATCH; - goto fail; - } - rc = ErrCode(); - if (rc && rc != EACCES && rc != EAGAIN) { - goto fail; - } - env->me_flags = (env->me_flags & ~MDB_ROBUST) | - (env->me_txns->mti_flags & MDB_ROBUST); -#ifdef _WIN32 - env->me_rmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname); - if (!env->me_rmutex) goto fail_errno; - env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname); - if (!env->me_wmutex) goto fail_errno; -#elif defined(MDB_USE_POSIX_SEM) - env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0); - if (env->me_rmutex == SEM_FAILED) goto fail_errno; - env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0); - if (env->me_wmutex == SEM_FAILED) goto fail_errno; -#endif - } - return MDB_SUCCESS; - -fail_errno: - rc = ErrCode(); -fail: - return rc; -} - - /** The name of the lock file in the DB environment */ -#define LOCKNAME "/lock.mdb" - /** The name of the data file in the DB environment */ -#define DATANAME "/data.mdb" - /** The suffix of the lock file when no subdir is used */ -#define LOCKSUFF "-lock" - /** Only a subset of the @ref mdb_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. - */ -#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) -#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|ROBUST_FLAG| \ - MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) -#ifdef MDB_ROBUST_SUPPORTED -#define ROBUST_FLAG MDB_ROBUST -#else -#define ROBUST_FLAG 0 -#endif - -#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) -# error "Persistent DB flags & env flags overlap, but both go in mm_flags" -#endif - -int ESECT -mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) -{ - int oflags, rc, len, excl = -1; - char *lpath, *dpath; - - if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) - return EINVAL; - - len = strlen(path); - if (flags & MDB_NOSUBDIR) { - rc = len + sizeof(LOCKSUFF) + len + 1; - } else { - rc = len + sizeof(LOCKNAME) + len + sizeof(DATANAME); - } - lpath = malloc(rc); - if (!lpath) - return ENOMEM; - if (flags & MDB_NOSUBDIR) { - dpath = lpath + len + sizeof(LOCKSUFF); - sprintf(lpath, "%s" LOCKSUFF, path); - strcpy(dpath, path); - } else { - dpath = lpath + len + sizeof(LOCKNAME); - sprintf(lpath, "%s" LOCKNAME, path); - sprintf(dpath, "%s" DATANAME, path); - } - - rc = MDB_SUCCESS; - flags |= env->me_flags; - if (flags & MDB_RDONLY) { - /* silently ignore WRITEMAP when we're only getting read access */ - flags &= ~MDB_WRITEMAP; - } else { - if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) && - (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) - rc = ENOMEM; - } - env->me_flags = flags |= MDB_ENV_ACTIVE; - if (rc) - goto leave; - - env->me_path = strdup(path); - env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); - env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); - env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int)); - if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { - rc = ENOMEM; - goto leave; - } - - /* For RDONLY, get lockfile after we know datafile exists */ - if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { - rc = mdb_env_setup_locks(env, lpath, mode, &excl); - if (rc) - goto leave; - } - -#ifdef _WIN32 - if (F_ISSET(flags, MDB_RDONLY)) { - oflags = GENERIC_READ; - len = OPEN_EXISTING; - } else { - oflags = GENERIC_READ|GENERIC_WRITE; - len = OPEN_ALWAYS; - } - mode = FILE_ATTRIBUTE_NORMAL; - env->me_fd = CreateFile(dpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, - NULL, len, mode, NULL); -#else - if (F_ISSET(flags, MDB_RDONLY)) - oflags = O_RDONLY; - else - oflags = O_RDWR | O_CREAT; - - env->me_fd = open(dpath, oflags, mode); -#endif - if (env->me_fd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } - - if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { - rc = mdb_env_setup_locks(env, lpath, mode, &excl); - if (rc) - goto leave; - } - - if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) { - if (flags & (MDB_RDONLY|MDB_WRITEMAP)) { - env->me_mfd = env->me_fd; - } else { - /* Synchronous fd for meta writes. Needed even with - * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. - */ -#ifdef _WIN32 - len = OPEN_EXISTING; - env->me_mfd = CreateFile(dpath, oflags, - FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, - mode | FILE_FLAG_WRITE_THROUGH, NULL); -#else - oflags &= ~O_CREAT; - env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode); -#endif - if (env->me_mfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } - } - DPRINTF(("opened dbenv %p", (void *) env)); - if (excl > 0) { - rc = mdb_env_share_locks(env, &excl); - if (rc) - goto leave; - } - if (!((flags & MDB_RDONLY) || - (env->me_pbuf = calloc(1, env->me_psize)))) - rc = ENOMEM; - if (!(flags & MDB_RDONLY)) { - MDB_txn *txn; - int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs * - (sizeof(MDB_db)+sizeof(MDB_cursor)+sizeof(unsigned int)+1); - txn = calloc(1, size); - if (txn) { - txn->mt_dbs = (MDB_db *)((char *)txn + tsize); - txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); - txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); - txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); - txn->mt_env = env; - env->me_txn0 = txn; - } else { - rc = ENOMEM; - } - } - } - -leave: - if (rc) { - mdb_env_close0(env, excl); - } - free(lpath); - return rc; -} - -/** Destroy resources from mdb_env_open(), clear our readers & DBIs */ -static void ESECT -mdb_env_close0(MDB_env *env, int excl) -{ - int i; - - if (!(env->me_flags & MDB_ENV_ACTIVE)) - return; - - /* Doing this here since me_dbxs may not exist during mdb_env_close */ - for (i = env->me_maxdbs; --i > MAIN_DBI; ) - free(env->me_dbxs[i].md_name.mv_data); - - free(env->me_pbuf); - free(env->me_dbiseqs); - free(env->me_dbflags); - free(env->me_dbxs); - free(env->me_path); - free(env->me_dirty_list); - free(env->me_txn0); - mdb_midl_free(env->me_free_pgs); - - if (env->me_flags & MDB_ENV_TXKEY) { - pthread_key_delete(env->me_txkey); -#ifdef _WIN32 - /* Delete our key from the global list */ - for (i=0; ime_txkey) { - mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1]; - mdb_tls_nkeys--; - break; - } -#endif - } - - if (env->me_map) { - munmap(env->me_map, env->me_mapsize); - } - if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE) - (void) close(env->me_mfd); - if (env->me_fd != INVALID_HANDLE_VALUE) - (void) close(env->me_fd); - if (env->me_txns) { - MDB_PID_T pid = env->me_pid; - /* Clearing readers is done in this function because - * me_txkey with its destructor must be disabled first. - */ - for (i = env->me_numreaders; --i >= 0; ) - if (env->me_txns->mti_readers[i].mr_pid == pid) - env->me_txns->mti_readers[i].mr_pid = 0; -#ifdef _WIN32 - if (env->me_rmutex) { - CloseHandle(env->me_rmutex); - if (env->me_wmutex) CloseHandle(env->me_wmutex); - } - /* Windows automatically destroys the mutexes when - * the last handle closes. - */ -#elif defined(MDB_USE_POSIX_SEM) - if (env->me_rmutex != SEM_FAILED) { - sem_close(env->me_rmutex); - if (env->me_wmutex != SEM_FAILED) - sem_close(env->me_wmutex); - /* If we have the filelock: If we are the - * only remaining user, clean up semaphores. - */ - if (excl == 0) - mdb_env_excl_lock(env, &excl); - if (excl > 0) { - sem_unlink(env->me_txns->mti_rmname); - sem_unlink(env->me_txns->mti_wmname); - } - } -#endif - munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); - } - if (env->me_lfd != INVALID_HANDLE_VALUE) { -#ifdef _WIN32 - if (excl >= 0) { - /* Unlock the lockfile. Windows would have unlocked it - * after closing anyway, but not necessarily at once. - */ - UnlockFile(env->me_lfd, 0, 0, 1, 0); - } -#endif - (void) close(env->me_lfd); - } - - env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); -} - -void ESECT -mdb_env_close(MDB_env *env) -{ - MDB_page *dp; - - if (env == NULL) - return; - - VGMEMP_DESTROY(env); - while ((dp = env->me_dpages) != NULL) { - VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); - env->me_dpages = dp->mp_next; - free(dp); - } - - mdb_env_close0(env, 0); - free(env); -} - -/** Compare two items pointing at aligned size_t's */ -static int -mdb_cmp_long(const MDB_val *a, const MDB_val *b) -{ - return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 : - *(size_t *)a->mv_data > *(size_t *)b->mv_data; -} - -/** Compare two items pointing at aligned unsigned int's */ -static int -mdb_cmp_int(const MDB_val *a, const MDB_val *b) -{ - return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 : - *(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data; -} - -/** Compare two items pointing at unsigned ints of unknown alignment. - * Nodes and keys are guaranteed to be 2-byte aligned. - */ -static int -mdb_cmp_cint(const MDB_val *a, const MDB_val *b) -{ -#if BYTE_ORDER == LITTLE_ENDIAN - unsigned short *u, *c; - int x; - - u = (unsigned short *) ((char *) a->mv_data + a->mv_size); - c = (unsigned short *) ((char *) b->mv_data + a->mv_size); - do { - x = *--u - *--c; - } while(!x && u > (unsigned short *)a->mv_data); - return x; -#else - unsigned short *u, *c, *end; - int x; - - end = (unsigned short *) ((char *) a->mv_data + a->mv_size); - u = (unsigned short *)a->mv_data; - c = (unsigned short *)b->mv_data; - do { - x = *u++ - *c++; - } while(!x && u < end); - return x; -#endif -} - -/** Compare two items pointing at size_t's of unknown alignment. */ -#ifdef MISALIGNED_OK -# define mdb_cmp_clong mdb_cmp_long -#else -# define mdb_cmp_clong mdb_cmp_cint -#endif - -/** Compare two items lexically */ -static int -mdb_cmp_memn(const MDB_val *a, const MDB_val *b) -{ - int diff; - ssize_t len_diff; - unsigned int len; - - len = a->mv_size; - len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; - if (len_diff > 0) { - len = b->mv_size; - len_diff = 1; - } - - diff = memcmp(a->mv_data, b->mv_data, len); - return diff ? diff : len_diff<0 ? -1 : len_diff; -} - -/** Compare two items in reverse byte order */ -static int -mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) -{ - const unsigned char *p1, *p2, *p1_lim; - ssize_t len_diff; - int diff; - - p1_lim = (const unsigned char *)a->mv_data; - p1 = (const unsigned char *)a->mv_data + a->mv_size; - p2 = (const unsigned char *)b->mv_data + b->mv_size; - - len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; - if (len_diff > 0) { - p1_lim += len_diff; - len_diff = 1; - } - - while (p1 > p1_lim) { - diff = *--p1 - *--p2; - if (diff) - return diff; - } - return len_diff<0 ? -1 : len_diff; -} - -/** Search for key within a page, using binary search. - * Returns the smallest entry larger or equal to the key. - * If exactp is non-null, stores whether the found entry was an exact match - * in *exactp (1 or 0). - * Updates the cursor index with the index of the found entry. - * If no entry larger or equal to the key is found, returns NULL. - */ -static MDB_node * -mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) -{ - unsigned int i = 0, nkeys; - int low, high; - int rc = 0; - MDB_page *mp = mc->mc_pg[mc->mc_top]; - MDB_node *node = NULL; - MDB_val nodekey; - MDB_cmp_func *cmp; - DKBUF; - - nkeys = NUMKEYS(mp); - - DPRINTF(("searching %u keys in %s %spage %"Z"u", - nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", - mdb_dbg_pgno(mp))); - - low = IS_LEAF(mp) ? 0 : 1; - high = nkeys - 1; - cmp = mc->mc_dbx->md_cmp; - - /* Branch pages have no data, so if using integer keys, - * alignment is guaranteed. Use faster mdb_cmp_int. - */ - if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { - if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t)) - cmp = mdb_cmp_long; - else - cmp = mdb_cmp_int; - } - - if (IS_LEAF2(mp)) { - nodekey.mv_size = mc->mc_db->md_pad; - node = NODEPTR(mp, 0); /* fake */ - while (low <= high) { - i = (low + high) >> 1; - nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size); - rc = cmp(key, &nodekey); - DPRINTF(("found leaf index %u [%s], rc = %i", - i, DKEY(&nodekey), rc)); - if (rc == 0) - break; - if (rc > 0) - low = i + 1; - else - high = i - 1; - } - } else { - while (low <= high) { - i = (low + high) >> 1; - - node = NODEPTR(mp, i); - nodekey.mv_size = NODEKSZ(node); - nodekey.mv_data = NODEKEY(node); - - rc = cmp(key, &nodekey); -#if MDB_DEBUG - if (IS_LEAF(mp)) - DPRINTF(("found leaf index %u [%s], rc = %i", - i, DKEY(&nodekey), rc)); - else - DPRINTF(("found branch index %u [%s -> %"Z"u], rc = %i", - i, DKEY(&nodekey), NODEPGNO(node), rc)); -#endif - if (rc == 0) - break; - if (rc > 0) - low = i + 1; - else - high = i - 1; - } - } - - if (rc > 0) { /* Found entry is less than the key. */ - i++; /* Skip to get the smallest entry larger than key. */ - if (!IS_LEAF2(mp)) - node = NODEPTR(mp, i); - } - if (exactp) - *exactp = (rc == 0 && nkeys > 0); - /* store the key index */ - mc->mc_ki[mc->mc_top] = i; - if (i >= nkeys) - /* There is no entry larger or equal to the key. */ - return NULL; - - /* nodeptr is fake for LEAF2 */ - return node; -} - -#if 0 -static void -mdb_cursor_adjust(MDB_cursor *mc, func) -{ - MDB_cursor *m2; - - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { - func(mc, m2); - } - } -} -#endif - -/** Pop a page off the top of the cursor's stack. */ -static void -mdb_cursor_pop(MDB_cursor *mc) -{ - if (mc->mc_snum) { -#if MDB_DEBUG - MDB_page *top = mc->mc_pg[mc->mc_top]; -#endif - mc->mc_snum--; - if (mc->mc_snum) - mc->mc_top--; - - DPRINTF(("popped page %"Z"u off db %d cursor %p", top->mp_pgno, - DDBI(mc), (void *) mc)); - } -} - -/** Push a page onto the top of the cursor's stack. */ -static int -mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) -{ - DPRINTF(("pushing page %"Z"u on db %d cursor %p", mp->mp_pgno, - DDBI(mc), (void *) mc)); - - if (mc->mc_snum >= CURSOR_STACK) { - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CURSOR_FULL; - } - - mc->mc_top = mc->mc_snum++; - mc->mc_pg[mc->mc_top] = mp; - mc->mc_ki[mc->mc_top] = 0; - - return MDB_SUCCESS; -} - -/** Find the address of the page corresponding to a given page number. - * @param[in] txn the transaction for this access. - * @param[in] pgno the page number for the page to retrieve. - * @param[out] ret address of a pointer where the page's address will be stored. - * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl) -{ - MDB_env *env = txn->mt_env; - MDB_page *p = NULL; - int level; - - if (!((txn->mt_flags & MDB_TXN_RDONLY) | (env->me_flags & MDB_WRITEMAP))) { - MDB_txn *tx2 = txn; - level = 1; - do { - MDB_ID2L dl = tx2->mt_u.dirty_list; - unsigned x; - /* Spilled pages were dirtied in this txn and flushed - * because the dirty list got full. Bring this page - * back in from the map (but don't unspill it here, - * leave that unless page_touch happens again). - */ - if (tx2->mt_spill_pgs) { - MDB_ID pn = pgno << 1; - x = mdb_midl_search(tx2->mt_spill_pgs, pn); - if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { - p = (MDB_page *)(env->me_map + env->me_psize * pgno); - goto done; - } - } - if (dl[0].mid) { - unsigned x = mdb_mid2l_search(dl, pgno); - if (x <= dl[0].mid && dl[x].mid == pgno) { - p = dl[x].mptr; - goto done; - } - } - level++; - } while ((tx2 = tx2->mt_parent) != NULL); - } - - if (pgno < txn->mt_next_pgno) { - level = 0; - p = (MDB_page *)(env->me_map + env->me_psize * pgno); - } else { - DPRINTF(("page %"Z"u not found", pgno)); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PAGE_NOTFOUND; - } - -done: - *ret = p; - if (lvl) - *lvl = level; - return MDB_SUCCESS; -} - -/** Finish #mdb_page_search() / #mdb_page_search_lowest(). - * The cursor is at the root page, set up the rest of it. - */ -static int -mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top]; - int rc; - DKBUF; - - while (IS_BRANCH(mp)) { - MDB_node *node; - indx_t i; - - DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); - mdb_cassert(mc, NUMKEYS(mp) > 1); - DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0)))); - - if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { - i = 0; - if (flags & MDB_PS_LAST) - i = NUMKEYS(mp) - 1; - } else { - int exact; - node = mdb_node_search(mc, key, &exact); - if (node == NULL) - i = NUMKEYS(mp) - 1; - else { - i = mc->mc_ki[mc->mc_top]; - if (!exact) { - mdb_cassert(mc, i > 0); - i--; - } - } - DPRINTF(("following index %u for key [%s]", i, DKEY(key))); - } - - mdb_cassert(mc, i < NUMKEYS(mp)); - node = NODEPTR(mp, i); - - if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) - return rc; - - mc->mc_ki[mc->mc_top] = i; - if ((rc = mdb_cursor_push(mc, mp))) - return rc; - - if (flags & MDB_PS_MODIFY) { - if ((rc = mdb_page_touch(mc)) != 0) - return rc; - mp = mc->mc_pg[mc->mc_top]; - } - } - - if (!IS_LEAF(mp)) { - DPRINTF(("internal error, index points to a %02X page!?", - mp->mp_flags)); - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; - } - - DPRINTF(("found leaf page %"Z"u for key [%s]", mp->mp_pgno, - key ? DKEY(key) : "null")); - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - return MDB_SUCCESS; -} - -/** Search for the lowest key under the current branch page. - * This just bypasses a NUMKEYS check in the current page - * before calling mdb_page_search_root(), because the callers - * are all in situations where the current page is known to - * be underfilled. - */ -static int -mdb_page_search_lowest(MDB_cursor *mc) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top]; - MDB_node *node = NODEPTR(mp, 0); - int rc; - - if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) - return rc; - - mc->mc_ki[mc->mc_top] = 0; - if ((rc = mdb_cursor_push(mc, mp))) - return rc; - return mdb_page_search_root(mc, NULL, MDB_PS_FIRST); -} - -/** Search for the page a given key should be in. - * Push it and its parent pages on the cursor stack. - * @param[in,out] mc the cursor for this operation. - * @param[in] key the key to search for, or NULL for first/last page. - * @param[in] flags If MDB_PS_MODIFY is set, visited pages in the DB - * are touched (updated with new page numbers). - * If MDB_PS_FIRST or MDB_PS_LAST is set, find first or last leaf. - * This is used by #mdb_cursor_first() and #mdb_cursor_last(). - * If MDB_PS_ROOTONLY set, just fetch root node, no further lookups. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) -{ - int rc; - pgno_t root; - - /* Make sure the txn is still viable, then find the root from - * the txn's db table and set it as the root of the cursor's stack. - */ - if (F_ISSET(mc->mc_txn->mt_flags, MDB_TXN_ERROR)) { - DPUTS("transaction has failed, must abort"); - return MDB_BAD_TXN; - } else { - /* Make sure we're using an up-to-date root */ - if (*mc->mc_dbflag & DB_STALE) { - MDB_cursor mc2; - if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) - return MDB_BAD_DBI; - mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); - rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); - if (rc) - return rc; - { - MDB_val data; - int exact = 0; - uint16_t flags; - MDB_node *leaf = mdb_node_search(&mc2, - &mc->mc_dbx->md_name, &exact); - if (!exact) - return MDB_NOTFOUND; - rc = mdb_node_read(mc->mc_txn, leaf, &data); - if (rc) - return rc; - memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)), - sizeof(uint16_t)); - /* The txn may not know this DBI, or another process may - * have dropped and recreated the DB with other flags. - */ - if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags) - return MDB_INCOMPATIBLE; - memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db)); - } - *mc->mc_dbflag &= ~DB_STALE; - } - root = mc->mc_db->md_root; - - if (root == P_INVALID) { /* Tree is empty. */ - DPUTS("tree is empty"); - return MDB_NOTFOUND; - } - } - - mdb_cassert(mc, root > 1); - if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) - if ((rc = mdb_page_get(mc->mc_txn, root, &mc->mc_pg[0], NULL)) != 0) - return rc; - - mc->mc_snum = 1; - mc->mc_top = 0; - - DPRINTF(("db %d root page %"Z"u has flags 0x%X", - DDBI(mc), root, mc->mc_pg[0]->mp_flags)); - - if (flags & MDB_PS_MODIFY) { - if ((rc = mdb_page_touch(mc))) - return rc; - } - - if (flags & MDB_PS_ROOTONLY) - return MDB_SUCCESS; - - return mdb_page_search_root(mc, key, flags); -} - -static int -mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) -{ - MDB_txn *txn = mc->mc_txn; - pgno_t pg = mp->mp_pgno; - unsigned x = 0, ovpages = mp->mp_pages; - MDB_env *env = txn->mt_env; - MDB_IDL sl = txn->mt_spill_pgs; - MDB_ID pn = pg << 1; - int rc; - - DPRINTF(("free ov page %"Z"u (%d)", pg, ovpages)); - /* If the page is dirty or on the spill list we just acquired it, - * so we should give it back to our current free list, if any. - * Otherwise put it onto the list of pages we freed in this txn. - * - * Won't create me_pghead: me_pglast must be inited along with it. - * Unsupported in nested txns: They would need to hide the page - * range in ancestor txns' dirty and spilled lists. - */ - if (env->me_pghead && - !txn->mt_parent && - ((mp->mp_flags & P_DIRTY) || - (sl && (x = mdb_midl_search(sl, pn)) <= sl[0] && sl[x] == pn))) - { - unsigned i, j; - pgno_t *mop; - MDB_ID2 *dl, ix, iy; - rc = mdb_midl_need(&env->me_pghead, ovpages); - if (rc) - return rc; - if (!(mp->mp_flags & P_DIRTY)) { - /* This page is no longer spilled */ - if (x == sl[0]) - sl[0]--; - else - sl[x] |= 1; - goto release; - } - /* Remove from dirty list */ - dl = txn->mt_u.dirty_list; - x = dl[0].mid--; - for (ix = dl[x]; ix.mptr != mp; ix = iy) { - if (x > 1) { - x--; - iy = dl[x]; - dl[x] = ix; - } else { - mdb_cassert(mc, x > 1); - j = ++(dl[0].mid); - dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; - } - } - if (!(env->me_flags & MDB_WRITEMAP)) - mdb_dpage_free(env, mp); -release: - /* Insert in me_pghead */ - mop = env->me_pghead; - j = mop[0] + ovpages; - for (i = mop[0]; i && mop[i] < pg; i--) - mop[j--] = mop[i]; - while (j>i) - mop[j--] = pg++; - mop[0] += ovpages; - } else { - rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); - if (rc) - return rc; - } - mc->mc_db->md_overflow_pages -= ovpages; - return 0; -} - -/** Return the data associated with a given node. - * @param[in] txn The transaction for this operation. - * @param[in] leaf The node being read. - * @param[out] data Updated to point to the node's data. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data) -{ - MDB_page *omp; /* overflow page */ - pgno_t pgno; - int rc; - - if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { - data->mv_size = NODEDSZ(leaf); - data->mv_data = NODEDATA(leaf); - return MDB_SUCCESS; - } - - /* Read overflow data. - */ - data->mv_size = NODEDSZ(leaf); - memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); - if ((rc = mdb_page_get(txn, pgno, &omp, NULL)) != 0) { - DPRINTF(("read overflow page %"Z"u failed", pgno)); - return rc; - } - data->mv_data = METADATA(omp); - - return MDB_SUCCESS; -} - -int -mdb_get(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data) -{ - MDB_cursor mc; - MDB_xcursor mx; - int exact = 0; - DKBUF; - - DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); - - if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - mdb_cursor_init(&mc, txn, dbi, &mx); - return mdb_cursor_set(&mc, key, data, MDB_SET, &exact); -} - -/** Find a sibling for a page. - * Replaces the page at the top of the cursor's stack with the - * specified sibling, if one exists. - * @param[in] mc The cursor for this operation. - * @param[in] move_right Non-zero if the right sibling is requested, - * otherwise the left sibling. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_cursor_sibling(MDB_cursor *mc, int move_right) -{ - int rc; - MDB_node *indx; - MDB_page *mp; - - if (mc->mc_snum < 2) { - return MDB_NOTFOUND; /* root has no siblings */ - } - - mdb_cursor_pop(mc); - DPRINTF(("parent page is page %"Z"u, index %u", - mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); - - if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) - : (mc->mc_ki[mc->mc_top] == 0)) { - DPRINTF(("no more keys left, moving to %s sibling", - move_right ? "right" : "left")); - if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) { - /* undo cursor_pop before returning */ - mc->mc_top++; - mc->mc_snum++; - return rc; - } - } else { - if (move_right) - mc->mc_ki[mc->mc_top]++; - else - mc->mc_ki[mc->mc_top]--; - DPRINTF(("just moving to %s index key %u", - move_right ? "right" : "left", mc->mc_ki[mc->mc_top])); - } - mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); - - indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(indx), &mp, NULL)) != 0) { - /* mc will be inconsistent if caller does mc_snum++ as above */ - mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - return rc; - } - - mdb_cursor_push(mc, mp); - if (!move_right) - mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1; - - return MDB_SUCCESS; -} - -/** Move the cursor to the next data item. */ -static int -mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) -{ - MDB_page *mp; - MDB_node *leaf; - int rc; - - if (mc->mc_flags & C_EOF) { - return MDB_NOTFOUND; - } - - mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); - - mp = mc->mc_pg[mc->mc_top]; - - if (mc->mc_db->md_flags & MDB_DUPSORT) { - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (op == MDB_NEXT || op == MDB_NEXT_DUP) { - rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT); - if (op != MDB_NEXT || rc != MDB_NOTFOUND) { - if (rc == MDB_SUCCESS) - MDB_GET_KEY(leaf, key); - return rc; - } - } - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if (op == MDB_NEXT_DUP) - return MDB_NOTFOUND; - } - } - - DPRINTF(("cursor_next: top page is %"Z"u in cursor %p", - mdb_dbg_pgno(mp), (void *) mc)); - if (mc->mc_flags & C_DEL) - goto skip; - - if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { - DPUTS("=====> move to next sibling page"); - if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { - mc->mc_flags |= C_EOF; - return rc; - } - mp = mc->mc_pg[mc->mc_top]; - DPRINTF(("next page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); - } else - mc->mc_ki[mc->mc_top]++; - -skip: - DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", - mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); - - if (IS_LEAF2(mp)) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - return MDB_SUCCESS; - } - - mdb_cassert(mc, IS_LEAF(mp)); - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - } - if (data) { - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc != MDB_SUCCESS) - return rc; - } - } - - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -/** Move the cursor to the previous data item. */ -static int -mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) -{ - MDB_page *mp; - MDB_node *leaf; - int rc; - - mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); - - mp = mc->mc_pg[mc->mc_top]; - - if (mc->mc_db->md_flags & MDB_DUPSORT) { - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (op == MDB_PREV || op == MDB_PREV_DUP) { - rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV); - if (op != MDB_PREV || rc != MDB_NOTFOUND) { - if (rc == MDB_SUCCESS) { - MDB_GET_KEY(leaf, key); - mc->mc_flags &= ~C_EOF; - } - return rc; - } - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if (op == MDB_PREV_DUP) - return MDB_NOTFOUND; - } - } - } - - DPRINTF(("cursor_prev: top page is %"Z"u in cursor %p", - mdb_dbg_pgno(mp), (void *) mc)); - - if (mc->mc_ki[mc->mc_top] == 0) { - DPUTS("=====> move to prev sibling page"); - if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { - return rc; - } - mp = mc->mc_pg[mc->mc_top]; - mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; - DPRINTF(("prev page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); - } else - mc->mc_ki[mc->mc_top]--; - - mc->mc_flags &= ~C_EOF; - - DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", - mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); - - if (IS_LEAF2(mp)) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - return MDB_SUCCESS; - } - - mdb_cassert(mc, IS_LEAF(mp)); - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - } - if (data) { - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc != MDB_SUCCESS) - return rc; - } - } - - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -/** Set the cursor on a specific data item. */ -static int -mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, - MDB_cursor_op op, int *exactp) -{ - int rc; - MDB_page *mp; - MDB_node *leaf = NULL; - DKBUF; - - if (key->mv_size == 0) - return MDB_BAD_VALSIZE; - - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - - /* See if we're already on the right page */ - if (mc->mc_flags & C_INITIALIZED) { - MDB_val nodekey; - - mp = mc->mc_pg[mc->mc_top]; - if (!NUMKEYS(mp)) { - mc->mc_ki[mc->mc_top] = 0; - return MDB_NOTFOUND; - } - if (mp->mp_flags & P_LEAF2) { - nodekey.mv_size = mc->mc_db->md_pad; - nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); - } else { - leaf = NODEPTR(mp, 0); - MDB_GET_KEY2(leaf, nodekey); - } - rc = mc->mc_dbx->md_cmp(key, &nodekey); - if (rc == 0) { - /* Probably happens rarely, but first node on the page - * was the one we wanted. - */ - mc->mc_ki[mc->mc_top] = 0; - if (exactp) - *exactp = 1; - goto set1; - } - if (rc > 0) { - unsigned int i; - unsigned int nkeys = NUMKEYS(mp); - if (nkeys > 1) { - if (mp->mp_flags & P_LEAF2) { - nodekey.mv_data = LEAF2KEY(mp, - nkeys-1, nodekey.mv_size); - } else { - leaf = NODEPTR(mp, nkeys-1); - MDB_GET_KEY2(leaf, nodekey); - } - rc = mc->mc_dbx->md_cmp(key, &nodekey); - if (rc == 0) { - /* last node was the one we wanted */ - mc->mc_ki[mc->mc_top] = nkeys-1; - if (exactp) - *exactp = 1; - goto set1; - } - if (rc < 0) { - if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { - /* This is definitely the right page, skip search_page */ - if (mp->mp_flags & P_LEAF2) { - nodekey.mv_data = LEAF2KEY(mp, - mc->mc_ki[mc->mc_top], nodekey.mv_size); - } else { - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - MDB_GET_KEY2(leaf, nodekey); - } - rc = mc->mc_dbx->md_cmp(key, &nodekey); - if (rc == 0) { - /* current node was the one we wanted */ - if (exactp) - *exactp = 1; - goto set1; - } - } - rc = 0; - goto set2; - } - } - /* If any parents have right-sibs, search. - * Otherwise, there's nothing further. - */ - for (i=0; imc_top; i++) - if (mc->mc_ki[i] < - NUMKEYS(mc->mc_pg[i])-1) - break; - if (i == mc->mc_top) { - /* There are no other pages */ - mc->mc_ki[mc->mc_top] = nkeys; - return MDB_NOTFOUND; - } - } - if (!mc->mc_top) { - /* There are no other pages */ - mc->mc_ki[mc->mc_top] = 0; - if (op == MDB_SET_RANGE && !exactp) { - rc = 0; - goto set1; - } else - return MDB_NOTFOUND; - } - } - - rc = mdb_page_search(mc, key, 0); - if (rc != MDB_SUCCESS) - return rc; - - mp = mc->mc_pg[mc->mc_top]; - mdb_cassert(mc, IS_LEAF(mp)); - -set2: - leaf = mdb_node_search(mc, key, exactp); - if (exactp != NULL && !*exactp) { - /* MDB_SET specified and not an exact match. */ - return MDB_NOTFOUND; - } - - if (leaf == NULL) { - DPUTS("===> inexact leaf not found, goto sibling"); - if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) - return rc; /* no entries matched */ - mp = mc->mc_pg[mc->mc_top]; - mdb_cassert(mc, IS_LEAF(mp)); - leaf = NODEPTR(mp, 0); - } - -set1: - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - if (IS_LEAF2(mp)) { - if (op == MDB_SET_RANGE || op == MDB_SET_KEY) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - } - return MDB_SUCCESS; - } - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - } - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) { - rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - } else { - int ex2, *ex2p; - if (op == MDB_GET_BOTH) { - ex2p = &ex2; - ex2 = 0; - } else { - ex2p = NULL; - } - rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p); - if (rc != MDB_SUCCESS) - return rc; - } - } else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { - MDB_val d2; - if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS) - return rc; - rc = mc->mc_dbx->md_dcmp(data, &d2); - if (rc) { - if (op == MDB_GET_BOTH || rc > 0) - return MDB_NOTFOUND; - rc = 0; - *data = d2; - } - - } else { - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - } - } - - /* The key already matches in all other cases */ - if (op == MDB_SET_RANGE || op == MDB_SET_KEY) - MDB_GET_KEY(leaf, key); - DPRINTF(("==> cursor placed on key [%s]", DKEY(key))); - - return rc; -} - -/** Move the cursor to the first item in the database. */ -static int -mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) -{ - int rc; - MDB_node *leaf; - - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); - if (rc != MDB_SUCCESS) - return rc; - } - mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - - leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0); - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - mc->mc_ki[mc->mc_top] = 0; - - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size); - return MDB_SUCCESS; - } - - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc) - return rc; - } else { - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - } - } - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -/** Move the cursor to the last item in the database. */ -static int -mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) -{ - int rc; - MDB_node *leaf; - - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - - if (!(mc->mc_flags & C_EOF)) { - - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = mdb_page_search(mc, NULL, MDB_PS_LAST); - if (rc != MDB_SUCCESS) - return rc; - } - mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - - } - mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; - mc->mc_flags |= C_INITIALIZED|C_EOF; - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size); - return MDB_SUCCESS; - } - - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc) - return rc; - } else { - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - } - } - - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -int -mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, - MDB_cursor_op op) -{ - int rc; - int exact = 0; - int (*mfunc)(MDB_cursor *mc, MDB_val *key, MDB_val *data); - - if (mc == NULL) - return EINVAL; - - if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - switch (op) { - case MDB_GET_CURRENT: - if (!(mc->mc_flags & C_INITIALIZED)) { - rc = EINVAL; - } else { - MDB_page *mp = mc->mc_pg[mc->mc_top]; - int nkeys = NUMKEYS(mp); - if (!nkeys || mc->mc_ki[mc->mc_top] >= nkeys) { - mc->mc_ki[mc->mc_top] = nkeys; - rc = MDB_NOTFOUND; - break; - } - rc = MDB_SUCCESS; - if (IS_LEAF2(mp)) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - } else { - MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - MDB_GET_KEY(leaf, key); - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (mc->mc_flags & C_DEL) - mdb_xcursor_init1(mc, leaf); - rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); - } else { - rc = mdb_node_read(mc->mc_txn, leaf, data); - } - } - } - } - break; - case MDB_GET_BOTH: - case MDB_GET_BOTH_RANGE: - if (data == NULL) { - rc = EINVAL; - break; - } - if (mc->mc_xcursor == NULL) { - rc = MDB_INCOMPATIBLE; - break; - } - /* FALLTHRU */ - case MDB_SET: - case MDB_SET_KEY: - case MDB_SET_RANGE: - if (key == NULL) { - rc = EINVAL; - } else { - rc = mdb_cursor_set(mc, key, data, op, - op == MDB_SET_RANGE ? NULL : &exact); - } - break; - case MDB_GET_MULTIPLE: - if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { - rc = EINVAL; - break; - } - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - rc = MDB_INCOMPATIBLE; - break; - } - rc = MDB_SUCCESS; - if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || - (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) - break; - goto fetchm; - case MDB_NEXT_MULTIPLE: - if (data == NULL) { - rc = EINVAL; - break; - } - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - rc = MDB_INCOMPATIBLE; - break; - } - if (!(mc->mc_flags & C_INITIALIZED)) - rc = mdb_cursor_first(mc, key, data); - else - rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); - if (rc == MDB_SUCCESS) { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - MDB_cursor *mx; -fetchm: - mx = &mc->mc_xcursor->mx_cursor; - data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) * - mx->mc_db->md_pad; - data->mv_data = METADATA(mx->mc_pg[mx->mc_top]); - mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1; - } else { - rc = MDB_NOTFOUND; - } - } - break; - case MDB_NEXT: - case MDB_NEXT_DUP: - case MDB_NEXT_NODUP: - if (!(mc->mc_flags & C_INITIALIZED)) - rc = mdb_cursor_first(mc, key, data); - else - rc = mdb_cursor_next(mc, key, data, op); - break; - case MDB_PREV: - case MDB_PREV_DUP: - case MDB_PREV_NODUP: - if (!(mc->mc_flags & C_INITIALIZED)) { - rc = mdb_cursor_last(mc, key, data); - if (rc) - break; - mc->mc_flags |= C_INITIALIZED; - mc->mc_ki[mc->mc_top]++; - } - rc = mdb_cursor_prev(mc, key, data, op); - break; - case MDB_FIRST: - rc = mdb_cursor_first(mc, key, data); - break; - case MDB_FIRST_DUP: - mfunc = mdb_cursor_first; - mmove: - if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { - rc = EINVAL; - break; - } - if (mc->mc_xcursor == NULL) { - rc = MDB_INCOMPATIBLE; - break; - } - { - MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { - MDB_GET_KEY(leaf, key); - rc = mdb_node_read(mc->mc_txn, leaf, data); - break; - } - } - if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { - rc = EINVAL; - break; - } - rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); - break; - case MDB_LAST: - rc = mdb_cursor_last(mc, key, data); - break; - case MDB_LAST_DUP: - mfunc = mdb_cursor_last; - goto mmove; - default: - DPRINTF(("unhandled/unimplemented cursor operation %u", op)); - rc = EINVAL; - break; - } - - if (mc->mc_flags & C_DEL) - mc->mc_flags ^= C_DEL; - - return rc; -} - -/** Touch all the pages in the cursor stack. Set mc_top. - * Makes sure all the pages are writable, before attempting a write operation. - * @param[in] mc The cursor to operate on. - */ -static int -mdb_cursor_touch(MDB_cursor *mc) -{ - int rc = MDB_SUCCESS; - - if (mc->mc_dbi > MAIN_DBI && !(*mc->mc_dbflag & DB_DIRTY)) { - MDB_cursor mc2; - MDB_xcursor mcx; - if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) - return MDB_BAD_DBI; - mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); - rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); - if (rc) - return rc; - *mc->mc_dbflag |= DB_DIRTY; - } - mc->mc_top = 0; - if (mc->mc_snum) { - do { - rc = mdb_page_touch(mc); - } while (!rc && ++(mc->mc_top) < mc->mc_snum); - mc->mc_top = mc->mc_snum-1; - } - return rc; -} - -/** Do not spill pages to disk if txn is getting full, may fail instead */ -#define MDB_NOSPILL 0x8000 - -int -mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, - unsigned int flags) -{ - enum { MDB_NO_ROOT = MDB_LAST_ERRCODE+10 }; /* internal code */ - MDB_env *env; - MDB_node *leaf = NULL; - MDB_page *fp, *mp; - uint16_t fp_flags; - MDB_val xdata, *rdata, dkey, olddata; - MDB_db dummy; - int do_sub = 0, insert_key, insert_data; - unsigned int mcount = 0, dcount = 0, nospill; - size_t nsize; - int rc, rc2; - unsigned int nflags; - DKBUF; - - if (mc == NULL || key == NULL) - return EINVAL; - - env = mc->mc_txn->mt_env; - - /* Check this first so counter will always be zero on any - * early failures. - */ - if (flags & MDB_MULTIPLE) { - dcount = data[1].mv_size; - data[1].mv_size = 0; - if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) - return MDB_INCOMPATIBLE; - } - - nospill = flags & MDB_NOSPILL; - flags &= ~MDB_NOSPILL; - - if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) - return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - if (key->mv_size-1 >= ENV_MAXKEY(env)) - return MDB_BAD_VALSIZE; - -#if SIZE_MAX > MAXDATASIZE - if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE)) - return MDB_BAD_VALSIZE; -#else - if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env)) - return MDB_BAD_VALSIZE; -#endif - - DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u", - DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size)); - - dkey.mv_size = 0; - - if (flags == MDB_CURRENT) { - if (!(mc->mc_flags & C_INITIALIZED)) - return EINVAL; - rc = MDB_SUCCESS; - } else if (mc->mc_db->md_root == P_INVALID) { - /* new database, cursor has nothing to point to */ - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_flags &= ~C_INITIALIZED; - rc = MDB_NO_ROOT; - } else { - int exact = 0; - MDB_val d2; - if (flags & MDB_APPEND) { - MDB_val k2; - rc = mdb_cursor_last(mc, &k2, &d2); - if (rc == 0) { - rc = mc->mc_dbx->md_cmp(key, &k2); - if (rc > 0) { - rc = MDB_NOTFOUND; - mc->mc_ki[mc->mc_top]++; - } else { - /* new key is <= last key */ - rc = MDB_KEYEXIST; - } - } - } else { - rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); - } - if ((flags & MDB_NOOVERWRITE) && rc == 0) { - DPRINTF(("duplicate key [%s]", DKEY(key))); - *data = d2; - return MDB_KEYEXIST; - } - if (rc && rc != MDB_NOTFOUND) - return rc; - } - - if (mc->mc_flags & C_DEL) - mc->mc_flags ^= C_DEL; - - /* Cursor is positioned, check for room in the dirty list */ - if (!nospill) { - if (flags & MDB_MULTIPLE) { - rdata = &xdata; - xdata.mv_size = data->mv_size * dcount; - } else { - rdata = data; - } - if ((rc2 = mdb_page_spill(mc, key, rdata))) - return rc2; - } - - if (rc == MDB_NO_ROOT) { - MDB_page *np; - /* new database, write a root leaf page */ - DPUTS("allocating new root leaf page"); - if ((rc2 = mdb_page_new(mc, P_LEAF, 1, &np))) { - return rc2; - } - mdb_cursor_push(mc, np); - mc->mc_db->md_root = np->mp_pgno; - mc->mc_db->md_depth++; - *mc->mc_dbflag |= DB_DIRTY; - if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) - == MDB_DUPFIXED) - np->mp_flags |= P_LEAF2; - mc->mc_flags |= C_INITIALIZED; - } else { - /* make sure all cursor pages are writable */ - rc2 = mdb_cursor_touch(mc); - if (rc2) - return rc2; - } - - insert_key = insert_data = rc; - if (insert_key) { - /* The key does not exist */ - DPRINTF(("inserting key at index %i", mc->mc_ki[mc->mc_top])); - if ((mc->mc_db->md_flags & MDB_DUPSORT) && - LEAFSIZE(key, data) > env->me_nodemax) - { - /* Too big for a node, insert in sub-DB. Set up an empty - * "old sub-page" for prep_subDB to expand to a full page. - */ - fp_flags = P_LEAF|P_DIRTY; - fp = env->me_pbuf; - fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */ - fp->mp_lower = fp->mp_upper = (PAGEHDRSZ-PAGEBASE); - olddata.mv_size = PAGEHDRSZ; - goto prep_subDB; - } - } else { - /* there's only a key anyway, so this is a no-op */ - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - char *ptr; - unsigned int ksize = mc->mc_db->md_pad; - if (key->mv_size != ksize) - return MDB_BAD_VALSIZE; - ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); - memcpy(ptr, key->mv_data, ksize); -fix_parent: - /* if overwriting slot 0 of leaf, need to - * update branch key if there is a parent page - */ - if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { - unsigned short top = mc->mc_top; - mc->mc_top--; - /* slot 0 is always an empty key, find real slot */ - while (mc->mc_top && !mc->mc_ki[mc->mc_top]) - mc->mc_top--; - if (mc->mc_ki[mc->mc_top]) - rc2 = mdb_update_key(mc, key); - else - rc2 = MDB_SUCCESS; - mc->mc_top = top; - if (rc2) - return rc2; - } - return MDB_SUCCESS; - } - -more: - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - olddata.mv_size = NODEDSZ(leaf); - olddata.mv_data = NODEDATA(leaf); - - /* DB has dups? */ - if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) { - /* Prepare (sub-)page/sub-DB to accept the new item, - * if needed. fp: old sub-page or a header faking - * it. mp: new (sub-)page. offset: growth in page - * size. xdata: node data with new page or DB. - */ - unsigned i, offset = 0; - mp = fp = xdata.mv_data = env->me_pbuf; - mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; - - /* Was a single item before, must convert now */ - if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { - /* Just overwrite the current item */ - if (flags == MDB_CURRENT) - goto current; - -#if UINT_MAX < SIZE_MAX - if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) - mc->mc_dbx->md_dcmp = mdb_cmp_clong; -#endif - /* does data match? */ - if (!mc->mc_dbx->md_dcmp(data, &olddata)) { - if (flags & MDB_NODUPDATA) - return MDB_KEYEXIST; - /* overwrite it */ - goto current; - } - - /* Back up original data item */ - dkey.mv_size = olddata.mv_size; - dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size); - - /* Make sub-page header for the dup items, with dummy body */ - fp->mp_flags = P_LEAF|P_DIRTY|P_SUBP; - fp->mp_lower = (PAGEHDRSZ-PAGEBASE); - xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size; - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - fp->mp_flags |= P_LEAF2; - fp->mp_pad = data->mv_size; - xdata.mv_size += 2 * data->mv_size; /* leave space for 2 more */ - } else { - xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) + - (dkey.mv_size & 1) + (data->mv_size & 1); - } - fp->mp_upper = xdata.mv_size - PAGEBASE; - olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */ - } else if (leaf->mn_flags & F_SUBDATA) { - /* Data is on sub-DB, just store it */ - flags |= F_DUPDATA|F_SUBDATA; - goto put_sub; - } else { - /* Data is on sub-page */ - fp = olddata.mv_data; - switch (flags) { - default: - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - offset = EVEN(NODESIZE + sizeof(indx_t) + - data->mv_size); - break; - } - offset = fp->mp_pad; - if (SIZELEFT(fp) < offset) { - offset *= 4; /* space for 4 more */ - break; - } - /* FALLTHRU: Big enough MDB_DUPFIXED sub-page */ - case MDB_CURRENT: - fp->mp_flags |= P_DIRTY; - COPY_PGNO(fp->mp_pgno, mp->mp_pgno); - mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; - flags |= F_DUPDATA; - goto put_sub; - } - xdata.mv_size = olddata.mv_size + offset; - } - - fp_flags = fp->mp_flags; - if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) { - /* Too big for a sub-page, convert to sub-DB */ - fp_flags &= ~P_SUBP; -prep_subDB: - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - fp_flags |= P_LEAF2; - dummy.md_pad = fp->mp_pad; - dummy.md_flags = MDB_DUPFIXED; - if (mc->mc_db->md_flags & MDB_INTEGERDUP) - dummy.md_flags |= MDB_INTEGERKEY; - } else { - dummy.md_pad = 0; - dummy.md_flags = 0; - } - dummy.md_depth = 1; - dummy.md_branch_pages = 0; - dummy.md_leaf_pages = 1; - dummy.md_overflow_pages = 0; - dummy.md_entries = NUMKEYS(fp); - xdata.mv_size = sizeof(MDB_db); - xdata.mv_data = &dummy; - if ((rc = mdb_page_alloc(mc, 1, &mp))) - return rc; - offset = env->me_psize - olddata.mv_size; - flags |= F_DUPDATA|F_SUBDATA; - dummy.md_root = mp->mp_pgno; - } - if (mp != fp) { - mp->mp_flags = fp_flags | P_DIRTY; - mp->mp_pad = fp->mp_pad; - mp->mp_lower = fp->mp_lower; - mp->mp_upper = fp->mp_upper + offset; - if (fp_flags & P_LEAF2) { - memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); - } else { - memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE, - olddata.mv_size - fp->mp_upper - PAGEBASE); - for (i=0; imp_ptrs[i] = fp->mp_ptrs[i] + offset; - } - } - - rdata = &xdata; - flags |= F_DUPDATA; - do_sub = 1; - if (!insert_key) - mdb_node_del(mc, 0); - goto new_sub; - } -current: - /* overflow page overwrites need special handling */ - if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { - MDB_page *omp; - pgno_t pg; - int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize); - - memcpy(&pg, olddata.mv_data, sizeof(pg)); - if ((rc2 = mdb_page_get(mc->mc_txn, pg, &omp, &level)) != 0) - return rc2; - ovpages = omp->mp_pages; - - /* Is the ov page large enough? */ - if (ovpages >= dpages) { - if (!(omp->mp_flags & P_DIRTY) && - (level || (env->me_flags & MDB_WRITEMAP))) - { - rc = mdb_page_unspill(mc->mc_txn, omp, &omp); - if (rc) - return rc; - level = 0; /* dirty in this txn or clean */ - } - /* Is it dirty? */ - if (omp->mp_flags & P_DIRTY) { - /* yes, overwrite it. Note in this case we don't - * bother to try shrinking the page if the new data - * is smaller than the overflow threshold. - */ - if (level > 1) { - /* It is writable only in a parent txn */ - size_t sz = (size_t) env->me_psize * ovpages, off; - MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages); - MDB_ID2 id2; - if (!np) - return ENOMEM; - id2.mid = pg; - id2.mptr = np; - rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); - mdb_cassert(mc, rc2 == 0); - if (!(flags & MDB_RESERVE)) { - /* Copy end of page, adjusting alignment so - * compiler may copy words instead of bytes. - */ - off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); - memcpy((size_t *)((char *)np + off), - (size_t *)((char *)omp + off), sz - off); - sz = PAGEHDRSZ; - } - memcpy(np, omp, sz); /* Copy beginning of page */ - omp = np; - } - SETDSZ(leaf, data->mv_size); - if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = METADATA(omp); - else - memcpy(METADATA(omp), data->mv_data, data->mv_size); - return MDB_SUCCESS; - } - } - if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS) - return rc2; - } else if (data->mv_size == olddata.mv_size) { - /* same size, just replace it. Note that we could - * also reuse this node if the new data is smaller, - * but instead we opt to shrink the node in that case. - */ - if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = olddata.mv_data; - else if (!(mc->mc_flags & C_SUB)) - memcpy(olddata.mv_data, data->mv_data, data->mv_size); - else { - memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); - goto fix_parent; - } - return MDB_SUCCESS; - } - mdb_node_del(mc, 0); - } - - rdata = data; - -new_sub: - nflags = flags & NODE_ADD_FLAGS; - nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(env, key, rdata); - if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) { - if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA ) - nflags &= ~MDB_APPEND; /* sub-page may need room to grow */ - if (!insert_key) - nflags |= MDB_SPLIT_REPLACE; - rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags); - } else { - /* There is room already in this leaf page. */ - rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); - if (rc == 0 && insert_key) { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - unsigned i = mc->mc_top; - MDB_page *mp = mc->mc_pg[i]; - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[i] == mp && m3->mc_ki[i] >= mc->mc_ki[i]) { - m3->mc_ki[i]++; - } - } - } - } - - if (rc == MDB_SUCCESS) { - /* Now store the actual data in the child DB. Note that we're - * storing the user data in the keys field, so there are strict - * size limits on dupdata. The actual data fields of the child - * DB are all zero size. - */ - if (do_sub) { - int xflags; - size_t ecount; -put_sub: - xdata.mv_size = 0; - xdata.mv_data = ""; - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (flags & MDB_CURRENT) { - xflags = MDB_CURRENT|MDB_NOSPILL; - } else { - mdb_xcursor_init1(mc, leaf); - xflags = (flags & MDB_NODUPDATA) ? - MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; - } - /* converted, write the original data first */ - if (dkey.mv_size) { - rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); - if (rc) - goto bad_sub; - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2; - unsigned i = mc->mc_top; - MDB_page *mp = mc->mc_pg[i]; - - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (!(m2->mc_flags & C_INITIALIZED)) continue; - if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { - mdb_xcursor_init1(m2, leaf); - } - } - } - /* we've done our job */ - dkey.mv_size = 0; - } - ecount = mc->mc_xcursor->mx_db.md_entries; - if (flags & MDB_APPENDDUP) - xflags |= MDB_APPEND; - rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); - if (flags & F_SUBDATA) { - void *db = NODEDATA(leaf); - memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); - } - insert_data = mc->mc_xcursor->mx_db.md_entries - ecount; - } - /* Increment count unless we just replaced an existing item. */ - if (insert_data) - mc->mc_db->md_entries++; - if (insert_key) { - /* Invalidate txn if we created an empty sub-DB */ - if (rc) - goto bad_sub; - /* If we succeeded and the key didn't exist before, - * make sure the cursor is marked valid. - */ - mc->mc_flags |= C_INITIALIZED; - } - if (flags & MDB_MULTIPLE) { - if (!rc) { - mcount++; - /* let caller know how many succeeded, if any */ - data[1].mv_size = mcount; - if (mcount < dcount) { - data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size; - insert_key = insert_data = 0; - goto more; - } - } - } - return rc; -bad_sub: - if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ - rc = MDB_CORRUPTED; - } - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_cursor_del(MDB_cursor *mc, unsigned int flags) -{ - MDB_node *leaf; - MDB_page *mp; - int rc; - - if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) - return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - if (!(mc->mc_flags & C_INITIALIZED)) - return EINVAL; - - if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) - return MDB_NOTFOUND; - - if (!(flags & MDB_NOSPILL) && (rc = mdb_page_spill(mc, NULL, NULL))) - return rc; - - rc = mdb_cursor_touch(mc); - if (rc) - return rc; - - mp = mc->mc_pg[mc->mc_top]; - if (IS_LEAF2(mp)) - goto del_key; - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (flags & MDB_NODUPDATA) { - /* mdb_cursor_del0() will subtract the final entry */ - mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; - } else { - if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { - mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } - rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL); - if (rc) - return rc; - /* If sub-DB still has entries, we're done */ - if (mc->mc_xcursor->mx_db.md_entries) { - if (leaf->mn_flags & F_SUBDATA) { - /* update subDB info */ - void *db = NODEDATA(leaf); - memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); - } else { - MDB_cursor *m2; - /* shrink fake page */ - mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - /* fix other sub-DB cursors pointed at this fake page */ - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (m2->mc_pg[mc->mc_top] == mp && - m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } - } - mc->mc_db->md_entries--; - mc->mc_flags |= C_DEL; - return rc; - } - /* otherwise fall thru and delete the sub-DB */ - } - - if (leaf->mn_flags & F_SUBDATA) { - /* add all the child DB's pages to the free list */ - rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); - if (rc) - goto fail; - } - } - - /* add overflow pages to free list */ - if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { - MDB_page *omp; - pgno_t pg; - - memcpy(&pg, NODEDATA(leaf), sizeof(pg)); - if ((rc = mdb_page_get(mc->mc_txn, pg, &omp, NULL)) || - (rc = mdb_ovpage_free(mc, omp))) - goto fail; - } - -del_key: - return mdb_cursor_del0(mc); - -fail: - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -/** Allocate and initialize new pages for a database. - * @param[in] mc a cursor on the database being added to. - * @param[in] flags flags defining what type of page is being allocated. - * @param[in] num the number of pages to allocate. This is usually 1, - * unless allocating overflow pages for a large record. - * @param[out] mp Address of a page, or NULL on failure. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) -{ - MDB_page *np; - int rc; - - if ((rc = mdb_page_alloc(mc, num, &np))) - return rc; - DPRINTF(("allocated new mpage %"Z"u, page size %u", - np->mp_pgno, mc->mc_txn->mt_env->me_psize)); - np->mp_flags = flags | P_DIRTY; - np->mp_lower = (PAGEHDRSZ-PAGEBASE); - np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE; - - if (IS_BRANCH(np)) - mc->mc_db->md_branch_pages++; - else if (IS_LEAF(np)) - mc->mc_db->md_leaf_pages++; - else if (IS_OVERFLOW(np)) { - mc->mc_db->md_overflow_pages += num; - np->mp_pages = num; - } - *mp = np; - - return 0; -} - -/** Calculate the size of a leaf node. - * The size depends on the environment's page size; if a data item - * is too large it will be put onto an overflow page and the node - * size will only include the key and not the data. Sizes are always - * rounded up to an even number of bytes, to guarantee 2-byte alignment - * of the #MDB_node headers. - * @param[in] env The environment handle. - * @param[in] key The key for the node. - * @param[in] data The data for the node. - * @return The number of bytes needed to store the node. - */ -static size_t -mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) -{ - size_t sz; - - sz = LEAFSIZE(key, data); - if (sz > env->me_nodemax) { - /* put on overflow page */ - sz -= data->mv_size - sizeof(pgno_t); - } - - return EVEN(sz + sizeof(indx_t)); -} - -/** Calculate the size of a branch node. - * The size should depend on the environment's page size but since - * we currently don't support spilling large keys onto overflow - * pages, it's simply the size of the #MDB_node header plus the - * size of the key. Sizes are always rounded up to an even number - * of bytes, to guarantee 2-byte alignment of the #MDB_node headers. - * @param[in] env The environment handle. - * @param[in] key The key for the node. - * @return The number of bytes needed to store the node. - */ -static size_t -mdb_branch_size(MDB_env *env, MDB_val *key) -{ - size_t sz; - - sz = INDXSIZE(key); - if (sz > env->me_nodemax) { - /* put on overflow page */ - /* not implemented */ - /* sz -= key->size - sizeof(pgno_t); */ - } - - return sz + sizeof(indx_t); -} - -/** Add a node to the page pointed to by the cursor. - * @param[in] mc The cursor for this operation. - * @param[in] indx The index on the page where the new node should be added. - * @param[in] key The key for the new node. - * @param[in] data The data for the new node, if any. - * @param[in] pgno The page number, if adding a branch node. - * @param[in] flags Flags for the node. - * @return 0 on success, non-zero on failure. Possible errors are: - *
      - *
    • ENOMEM - failed to allocate overflow pages for the node. - *
    • MDB_PAGE_FULL - there is insufficient room in the page. This error - * should never happen since all callers already calculate the - * page's free space before calling this function. - *
    - */ -static int -mdb_node_add(MDB_cursor *mc, indx_t indx, - MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags) -{ - unsigned int i; - size_t node_size = NODESIZE; - ssize_t room; - indx_t ofs; - MDB_node *node; - MDB_page *mp = mc->mc_pg[mc->mc_top]; - MDB_page *ofp = NULL; /* overflow page */ - DKBUF; - - mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); - - DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]", - IS_LEAF(mp) ? "leaf" : "branch", - IS_SUBP(mp) ? "sub-" : "", - mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, - key ? key->mv_size : 0, key ? DKEY(key) : "null")); - - if (IS_LEAF2(mp)) { - /* Move higher keys up one slot. */ - int ksize = mc->mc_db->md_pad, dif; - char *ptr = LEAF2KEY(mp, indx, ksize); - dif = NUMKEYS(mp) - indx; - if (dif > 0) - memmove(ptr+ksize, ptr, dif*ksize); - /* insert new key */ - memcpy(ptr, key->mv_data, ksize); - - /* Just using these for counting */ - mp->mp_lower += sizeof(indx_t); - mp->mp_upper -= ksize - sizeof(indx_t); - return MDB_SUCCESS; - } - - room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t); - if (key != NULL) - node_size += key->mv_size; - if (IS_LEAF(mp)) { - mdb_cassert(mc, data); - if (F_ISSET(flags, F_BIGDATA)) { - /* Data already on overflow page. */ - node_size += sizeof(pgno_t); - } else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) { - int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize); - int rc; - /* Put data on overflow page. */ - DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page", - data->mv_size, node_size+data->mv_size)); - node_size = EVEN(node_size + sizeof(pgno_t)); - if ((ssize_t)node_size > room) - goto full; - if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) - return rc; - DPRINTF(("allocated overflow page %"Z"u", ofp->mp_pgno)); - flags |= F_BIGDATA; - goto update; - } else { - node_size += data->mv_size; - } - } - node_size = EVEN(node_size); - if ((ssize_t)node_size > room) - goto full; - -update: - /* Move higher pointers up one slot. */ - for (i = NUMKEYS(mp); i > indx; i--) - mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; - - /* Adjust free space offsets. */ - ofs = mp->mp_upper - node_size; - mdb_cassert(mc, ofs >= mp->mp_lower + sizeof(indx_t)); - mp->mp_ptrs[indx] = ofs; - mp->mp_upper = ofs; - mp->mp_lower += sizeof(indx_t); - - /* Write the node data. */ - node = NODEPTR(mp, indx); - node->mn_ksize = (key == NULL) ? 0 : key->mv_size; - node->mn_flags = flags; - if (IS_LEAF(mp)) - SETDSZ(node,data->mv_size); - else - SETPGNO(node,pgno); - - if (key) - memcpy(NODEKEY(node), key->mv_data, key->mv_size); - - if (IS_LEAF(mp)) { - mdb_cassert(mc, key); - if (ofp == NULL) { - if (F_ISSET(flags, F_BIGDATA)) - memcpy(node->mn_data + key->mv_size, data->mv_data, - sizeof(pgno_t)); - else if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = node->mn_data + key->mv_size; - else - memcpy(node->mn_data + key->mv_size, data->mv_data, - data->mv_size); - } else { - memcpy(node->mn_data + key->mv_size, &ofp->mp_pgno, - sizeof(pgno_t)); - if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = METADATA(ofp); - else - memcpy(METADATA(ofp), data->mv_data, data->mv_size); - } - } - - return MDB_SUCCESS; - -full: - DPRINTF(("not enough room in page %"Z"u, got %u ptrs", - mdb_dbg_pgno(mp), NUMKEYS(mp))); - DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); - DPRINTF(("node size = %"Z"u", node_size)); - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PAGE_FULL; -} - -/** Delete the specified node from a page. - * @param[in] mc Cursor pointing to the node to delete. - * @param[in] ksize The size of a node. Only used if the page is - * part of a #MDB_DUPFIXED database. - */ -static void -mdb_node_del(MDB_cursor *mc, int ksize) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top]; - indx_t indx = mc->mc_ki[mc->mc_top]; - unsigned int sz; - indx_t i, j, numkeys, ptr; - MDB_node *node; - char *base; - - DPRINTF(("delete node %u on %s page %"Z"u", indx, - IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); - numkeys = NUMKEYS(mp); - mdb_cassert(mc, indx < numkeys); - - if (IS_LEAF2(mp)) { - int x = numkeys - 1 - indx; - base = LEAF2KEY(mp, indx, ksize); - if (x) - memmove(base, base + ksize, x * ksize); - mp->mp_lower -= sizeof(indx_t); - mp->mp_upper += ksize - sizeof(indx_t); - return; - } - - node = NODEPTR(mp, indx); - sz = NODESIZE + node->mn_ksize; - if (IS_LEAF(mp)) { - if (F_ISSET(node->mn_flags, F_BIGDATA)) - sz += sizeof(pgno_t); - else - sz += NODEDSZ(node); - } - sz = EVEN(sz); - - ptr = mp->mp_ptrs[indx]; - for (i = j = 0; i < numkeys; i++) { - if (i != indx) { - mp->mp_ptrs[j] = mp->mp_ptrs[i]; - if (mp->mp_ptrs[i] < ptr) - mp->mp_ptrs[j] += sz; - j++; - } - } - - base = (char *)mp + mp->mp_upper + PAGEBASE; - memmove(base + sz, base, ptr - mp->mp_upper); - - mp->mp_lower -= sizeof(indx_t); - mp->mp_upper += sz; -} - -/** Compact the main page after deleting a node on a subpage. - * @param[in] mp The main page to operate on. - * @param[in] indx The index of the subpage on the main page. - */ -static void -mdb_node_shrink(MDB_page *mp, indx_t indx) -{ - MDB_node *node; - MDB_page *sp, *xp; - char *base; - int nsize, delta; - indx_t i, numkeys, ptr; - - node = NODEPTR(mp, indx); - sp = (MDB_page *)NODEDATA(node); - delta = SIZELEFT(sp); - xp = (MDB_page *)((char *)sp + delta); - - /* shift subpage upward */ - if (IS_LEAF2(sp)) { - nsize = NUMKEYS(sp) * sp->mp_pad; - if (nsize & 1) - return; /* do not make the node uneven-sized */ - memmove(METADATA(xp), METADATA(sp), nsize); - } else { - int i; - numkeys = NUMKEYS(sp); - for (i=numkeys-1; i>=0; i--) - xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta; - } - xp->mp_upper = sp->mp_lower; - xp->mp_lower = sp->mp_lower; - xp->mp_flags = sp->mp_flags; - xp->mp_pad = sp->mp_pad; - COPY_PGNO(xp->mp_pgno, mp->mp_pgno); - - nsize = NODEDSZ(node) - delta; - SETDSZ(node, nsize); - - /* shift lower nodes upward */ - ptr = mp->mp_ptrs[indx]; - numkeys = NUMKEYS(mp); - for (i = 0; i < numkeys; i++) { - if (mp->mp_ptrs[i] <= ptr) - mp->mp_ptrs[i] += delta; - } - - base = (char *)mp + mp->mp_upper + PAGEBASE; - memmove(base + delta, base, ptr - mp->mp_upper + NODESIZE + NODEKSZ(node)); - mp->mp_upper += delta; -} - -/** Initial setup of a sorted-dups cursor. - * Sorted duplicates are implemented as a sub-database for the given key. - * The duplicate data items are actually keys of the sub-database. - * Operations on the duplicate data items are performed using a sub-cursor - * initialized when the sub-database is first accessed. This function does - * the preliminary setup of the sub-cursor, filling in the fields that - * depend only on the parent DB. - * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. - */ -static void -mdb_xcursor_init0(MDB_cursor *mc) -{ - MDB_xcursor *mx = mc->mc_xcursor; - - mx->mx_cursor.mc_xcursor = NULL; - mx->mx_cursor.mc_txn = mc->mc_txn; - mx->mx_cursor.mc_db = &mx->mx_db; - mx->mx_cursor.mc_dbx = &mx->mx_dbx; - mx->mx_cursor.mc_dbi = mc->mc_dbi; - mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; - mx->mx_cursor.mc_snum = 0; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_SUB; - mx->mx_dbx.md_name.mv_size = 0; - mx->mx_dbx.md_name.mv_data = NULL; - mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; - mx->mx_dbx.md_dcmp = NULL; - mx->mx_dbx.md_rel = mc->mc_dbx->md_rel; -} - -/** Final setup of a sorted-dups cursor. - * Sets up the fields that depend on the data from the main cursor. - * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. - * @param[in] node The data containing the #MDB_db record for the - * sorted-dup database. - */ -static void -mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) -{ - MDB_xcursor *mx = mc->mc_xcursor; - - if (node->mn_flags & F_SUBDATA) { - memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); - mx->mx_cursor.mc_pg[0] = 0; - mx->mx_cursor.mc_snum = 0; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_SUB; - } else { - MDB_page *fp = NODEDATA(node); - mx->mx_db.md_pad = mc->mc_pg[mc->mc_top]->mp_pad; - mx->mx_db.md_flags = 0; - mx->mx_db.md_depth = 1; - mx->mx_db.md_branch_pages = 0; - mx->mx_db.md_leaf_pages = 1; - mx->mx_db.md_overflow_pages = 0; - mx->mx_db.md_entries = NUMKEYS(fp); - COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); - mx->mx_cursor.mc_snum = 1; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB; - mx->mx_cursor.mc_pg[0] = fp; - mx->mx_cursor.mc_ki[0] = 0; - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - mx->mx_db.md_flags = MDB_DUPFIXED; - mx->mx_db.md_pad = fp->mp_pad; - if (mc->mc_db->md_flags & MDB_INTEGERDUP) - mx->mx_db.md_flags |= MDB_INTEGERKEY; - } - } - DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, - mx->mx_db.md_root)); - mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ -#if UINT_MAX < SIZE_MAX - if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) - mx->mx_dbx.md_cmp = mdb_cmp_clong; -#endif -} - -/** Initialize a cursor for a given transaction and database. */ -static void -mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) -{ - mc->mc_next = NULL; - mc->mc_backup = NULL; - mc->mc_dbi = dbi; - mc->mc_txn = txn; - mc->mc_db = &txn->mt_dbs[dbi]; - mc->mc_dbx = &txn->mt_dbxs[dbi]; - mc->mc_dbflag = &txn->mt_dbflags[dbi]; - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_pg[0] = 0; - mc->mc_flags = 0; - if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { - mdb_tassert(txn, mx != NULL); - mc->mc_xcursor = mx; - mdb_xcursor_init0(mc); - } else { - mc->mc_xcursor = NULL; - } - if (*mc->mc_dbflag & DB_STALE) { - mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); - } -} - -int -mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) -{ - MDB_cursor *mc; - size_t size = sizeof(MDB_cursor); - - if (!ret || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - /* Allow read access to the freelist */ - if (!dbi && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) - return EINVAL; - - if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) - size += sizeof(MDB_xcursor); - - if ((mc = malloc(size)) != NULL) { - mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1)); - if (txn->mt_cursors) { - mc->mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = mc; - mc->mc_flags |= C_UNTRACK; - } - } else { - return ENOMEM; - } - - *ret = mc; - - return MDB_SUCCESS; -} - -int -mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) -{ - if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi)) - return EINVAL; - - if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor); - return MDB_SUCCESS; -} - -/* Return the count of duplicate data items for the current key */ -int -mdb_cursor_count(MDB_cursor *mc, size_t *countp) -{ - MDB_node *leaf; - - if (mc == NULL || countp == NULL) - return EINVAL; - - if (mc->mc_xcursor == NULL) - return MDB_INCOMPATIBLE; - - if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - if (!(mc->mc_flags & C_INITIALIZED)) - return EINVAL; - - if (!mc->mc_snum || (mc->mc_flags & C_EOF)) - return MDB_NOTFOUND; - - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { - *countp = 1; - } else { - if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) - return EINVAL; - - *countp = mc->mc_xcursor->mx_db.md_entries; - } - return MDB_SUCCESS; -} - -void -mdb_cursor_close(MDB_cursor *mc) -{ - if (mc && !mc->mc_backup) { - /* remove from txn, if tracked */ - if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { - MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; - while (*prev && *prev != mc) prev = &(*prev)->mc_next; - if (*prev == mc) - *prev = mc->mc_next; - } - free(mc); - } -} - -MDB_txn * -mdb_cursor_txn(MDB_cursor *mc) -{ - if (!mc) return NULL; - return mc->mc_txn; -} - -MDB_dbi -mdb_cursor_dbi(MDB_cursor *mc) -{ - return mc->mc_dbi; -} - -/** Replace the key for a branch node with a new key. - * @param[in] mc Cursor pointing to the node to operate on. - * @param[in] key The new key to use. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_update_key(MDB_cursor *mc, MDB_val *key) -{ - MDB_page *mp; - MDB_node *node; - char *base; - size_t len; - int delta, ksize, oksize; - indx_t ptr, i, numkeys, indx; - DKBUF; - - indx = mc->mc_ki[mc->mc_top]; - mp = mc->mc_pg[mc->mc_top]; - node = NODEPTR(mp, indx); - ptr = mp->mp_ptrs[indx]; -#if MDB_DEBUG - { - MDB_val k2; - char kbuf2[DKBUF_MAXKEYSIZE*2+1]; - k2.mv_data = NODEKEY(node); - k2.mv_size = node->mn_ksize; - DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Z"u", - indx, ptr, - mdb_dkey(&k2, kbuf2), - DKEY(key), - mp->mp_pgno)); - } -#endif - - /* Sizes must be 2-byte aligned. */ - ksize = EVEN(key->mv_size); - oksize = EVEN(node->mn_ksize); - delta = ksize - oksize; - - /* Shift node contents if EVEN(key length) changed. */ - if (delta) { - if (delta > 0 && SIZELEFT(mp) < delta) { - pgno_t pgno; - /* not enough space left, do a delete and split */ - DPRINTF(("Not enough room, delta = %d, splitting...", delta)); - pgno = NODEPGNO(node); - mdb_node_del(mc, 0); - return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE); - } - - numkeys = NUMKEYS(mp); - for (i = 0; i < numkeys; i++) { - if (mp->mp_ptrs[i] <= ptr) - mp->mp_ptrs[i] -= delta; - } - - base = (char *)mp + mp->mp_upper + PAGEBASE; - len = ptr - mp->mp_upper + NODESIZE; - memmove(base - delta, base, len); - mp->mp_upper -= delta; - - node = NODEPTR(mp, indx); - } - - /* But even if no shift was needed, update ksize */ - if (node->mn_ksize != key->mv_size) - node->mn_ksize = key->mv_size; - - if (key->mv_size) - memcpy(NODEKEY(node), key->mv_data, key->mv_size); - - return MDB_SUCCESS; -} - -static void -mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); - -/** Move a node from csrc to cdst. - */ -static int -mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) -{ - MDB_node *srcnode; - MDB_val key, data; - pgno_t srcpg; - MDB_cursor mn; - int rc; - unsigned short flags; - - DKBUF; - - /* Mark src and dst as dirty. */ - if ((rc = mdb_page_touch(csrc)) || - (rc = mdb_page_touch(cdst))) - return rc; - - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); - data.mv_size = 0; - data.mv_data = NULL; - srcpg = 0; - flags = 0; - } else { - srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]); - mdb_cassert(csrc, !((size_t)srcnode & 1)); - srcpg = NODEPGNO(srcnode); - flags = srcnode->mn_flags; - if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { - unsigned int snum = csrc->mc_snum; - MDB_node *s2; - /* must find the lowest key below src */ - rc = mdb_page_search_lowest(csrc); - if (rc) - return rc; - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); - } else { - s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); - key.mv_size = NODEKSZ(s2); - key.mv_data = NODEKEY(s2); - } - csrc->mc_snum = snum--; - csrc->mc_top = snum; - } else { - key.mv_size = NODEKSZ(srcnode); - key.mv_data = NODEKEY(srcnode); - } - data.mv_size = NODEDSZ(srcnode); - data.mv_data = NODEDATA(srcnode); - } - if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { - unsigned int snum = cdst->mc_snum; - MDB_node *s2; - MDB_val bkey; - /* must find the lowest key below dst */ - mdb_cursor_copy(cdst, &mn); - rc = mdb_page_search_lowest(&mn); - if (rc) - return rc; - if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { - bkey.mv_size = mn.mc_db->md_pad; - bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size); - } else { - s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); - bkey.mv_size = NODEKSZ(s2); - bkey.mv_data = NODEKEY(s2); - } - mn.mc_snum = snum--; - mn.mc_top = snum; - mn.mc_ki[snum] = 0; - rc = mdb_update_key(&mn, &bkey); - if (rc) - return rc; - } - - DPRINTF(("moving %s node %u [%s] on page %"Z"u to node %u on page %"Z"u", - IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", - csrc->mc_ki[csrc->mc_top], - DKEY(&key), - csrc->mc_pg[csrc->mc_top]->mp_pgno, - cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno)); - - /* Add the node to the destination page. - */ - rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags); - if (rc != MDB_SUCCESS) - return rc; - - /* Delete the node from the source page. - */ - mdb_node_del(csrc, key.mv_size); - - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = csrc->mc_dbi; - MDB_page *mp = csrc->mc_pg[csrc->mc_top]; - - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (csrc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == csrc) continue; - if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] == - csrc->mc_ki[csrc->mc_top]) { - m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; - m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; - } - } - } - - /* Update the parent separators. - */ - if (csrc->mc_ki[csrc->mc_top] == 0) { - if (csrc->mc_ki[csrc->mc_top-1] != 0) { - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); - } else { - srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); - key.mv_size = NODEKSZ(srcnode); - key.mv_data = NODEKEY(srcnode); - } - DPRINTF(("update separator for source page %"Z"u to [%s]", - csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); - mdb_cursor_copy(csrc, &mn); - mn.mc_snum--; - mn.mc_top--; - if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) - return rc; - } - if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { - MDB_val nullkey; - indx_t ix = csrc->mc_ki[csrc->mc_top]; - nullkey.mv_size = 0; - csrc->mc_ki[csrc->mc_top] = 0; - rc = mdb_update_key(csrc, &nullkey); - csrc->mc_ki[csrc->mc_top] = ix; - mdb_cassert(csrc, rc == MDB_SUCCESS); - } - } - - if (cdst->mc_ki[cdst->mc_top] == 0) { - if (cdst->mc_ki[cdst->mc_top-1] != 0) { - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); - } else { - srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); - key.mv_size = NODEKSZ(srcnode); - key.mv_data = NODEKEY(srcnode); - } - DPRINTF(("update separator for destination page %"Z"u to [%s]", - cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); - mdb_cursor_copy(cdst, &mn); - mn.mc_snum--; - mn.mc_top--; - if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) - return rc; - } - if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) { - MDB_val nullkey; - indx_t ix = cdst->mc_ki[cdst->mc_top]; - nullkey.mv_size = 0; - cdst->mc_ki[cdst->mc_top] = 0; - rc = mdb_update_key(cdst, &nullkey); - cdst->mc_ki[cdst->mc_top] = ix; - mdb_cassert(csrc, rc == MDB_SUCCESS); - } - } - - return MDB_SUCCESS; -} - -/** Merge one page into another. - * The nodes from the page pointed to by \b csrc will - * be copied to the page pointed to by \b cdst and then - * the \b csrc page will be freed. - * @param[in] csrc Cursor pointing to the source page. - * @param[in] cdst Cursor pointing to the destination page. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) -{ - MDB_page *psrc, *pdst; - MDB_node *srcnode; - MDB_val key, data; - unsigned nkeys; - int rc; - indx_t i, j; - - psrc = csrc->mc_pg[csrc->mc_top]; - pdst = cdst->mc_pg[cdst->mc_top]; - - DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno)); - - mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ - mdb_cassert(csrc, cdst->mc_snum > 1); - - /* Mark dst as dirty. */ - if ((rc = mdb_page_touch(cdst))) - return rc; - - /* Move all nodes from src to dst. - */ - j = nkeys = NUMKEYS(pdst); - if (IS_LEAF2(psrc)) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = METADATA(psrc); - for (i = 0; i < NUMKEYS(psrc); i++, j++) { - rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); - if (rc != MDB_SUCCESS) - return rc; - key.mv_data = (char *)key.mv_data + key.mv_size; - } - } else { - for (i = 0; i < NUMKEYS(psrc); i++, j++) { - srcnode = NODEPTR(psrc, i); - if (i == 0 && IS_BRANCH(psrc)) { - MDB_cursor mn; - MDB_node *s2; - mdb_cursor_copy(csrc, &mn); - /* must find the lowest key below src */ - rc = mdb_page_search_lowest(&mn); - if (rc) - return rc; - if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { - key.mv_size = mn.mc_db->md_pad; - key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size); - } else { - s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); - key.mv_size = NODEKSZ(s2); - key.mv_data = NODEKEY(s2); - } - } else { - key.mv_size = srcnode->mn_ksize; - key.mv_data = NODEKEY(srcnode); - } - - data.mv_size = NODEDSZ(srcnode); - data.mv_data = NODEDATA(srcnode); - rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags); - if (rc != MDB_SUCCESS) - return rc; - } - } - - DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)", - pdst->mp_pgno, NUMKEYS(pdst), - (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); - - /* Unlink the src page from parent and add to free list. - */ - csrc->mc_top--; - mdb_node_del(csrc, 0); - if (csrc->mc_ki[csrc->mc_top] == 0) { - key.mv_size = 0; - rc = mdb_update_key(csrc, &key); - if (rc) { - csrc->mc_top++; - return rc; - } - } - csrc->mc_top++; - - psrc = csrc->mc_pg[csrc->mc_top]; - /* If not operating on FreeDB, allow this page to be reused - * in this txn. Otherwise just add to free list. - */ - rc = mdb_page_loose(csrc, psrc); - if (rc) - return rc; - if (IS_LEAF(psrc)) - csrc->mc_db->md_leaf_pages--; - else - csrc->mc_db->md_branch_pages--; - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = csrc->mc_dbi; - - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (csrc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == csrc) continue; - if (m3->mc_snum < csrc->mc_snum) continue; - if (m3->mc_pg[csrc->mc_top] == psrc) { - m3->mc_pg[csrc->mc_top] = pdst; - m3->mc_ki[csrc->mc_top] += nkeys; - } - } - } - { - unsigned int snum = cdst->mc_snum; - uint16_t depth = cdst->mc_db->md_depth; - mdb_cursor_pop(cdst); - rc = mdb_rebalance(cdst); - /* Did the tree shrink? */ - if (depth > cdst->mc_db->md_depth) - snum--; - cdst->mc_snum = snum; - cdst->mc_top = snum-1; - } - return rc; -} - -/** Copy the contents of a cursor. - * @param[in] csrc The cursor to copy from. - * @param[out] cdst The cursor to copy to. - */ -static void -mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) -{ - unsigned int i; - - cdst->mc_txn = csrc->mc_txn; - cdst->mc_dbi = csrc->mc_dbi; - cdst->mc_db = csrc->mc_db; - cdst->mc_dbx = csrc->mc_dbx; - cdst->mc_snum = csrc->mc_snum; - cdst->mc_top = csrc->mc_top; - cdst->mc_flags = csrc->mc_flags; - - for (i=0; imc_snum; i++) { - cdst->mc_pg[i] = csrc->mc_pg[i]; - cdst->mc_ki[i] = csrc->mc_ki[i]; - } -} - -/** Rebalance the tree after a delete operation. - * @param[in] mc Cursor pointing to the page where rebalancing - * should begin. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_rebalance(MDB_cursor *mc) -{ - MDB_node *node; - int rc; - unsigned int ptop, minkeys; - MDB_cursor mn; - indx_t oldki; - - minkeys = 1 + (IS_BRANCH(mc->mc_pg[mc->mc_top])); - DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)", - IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", - mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), - (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); - - if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= FILL_THRESHOLD && - NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { - DPRINTF(("no need to rebalance page %"Z"u, above fill threshold", - mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); - return MDB_SUCCESS; - } - - if (mc->mc_snum < 2) { - MDB_page *mp = mc->mc_pg[0]; - if (IS_SUBP(mp)) { - DPUTS("Can't rebalance a subpage, ignoring"); - return MDB_SUCCESS; - } - if (NUMKEYS(mp) == 0) { - DPUTS("tree is completely empty"); - mc->mc_db->md_root = P_INVALID; - mc->mc_db->md_depth = 0; - mc->mc_db->md_leaf_pages = 0; - rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); - if (rc) - return rc; - /* Adjust cursors pointing to mp */ - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_flags &= ~C_INITIALIZED; - { - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[0] == mp) { - m3->mc_snum = 0; - m3->mc_top = 0; - m3->mc_flags &= ~C_INITIALIZED; - } - } - } - } else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) { - int i; - DPUTS("collapsing root page!"); - rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); - if (rc) - return rc; - mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0)); - rc = mdb_page_get(mc->mc_txn,mc->mc_db->md_root,&mc->mc_pg[0],NULL); - if (rc) - return rc; - mc->mc_db->md_depth--; - mc->mc_db->md_branch_pages--; - mc->mc_ki[0] = mc->mc_ki[1]; - for (i = 1; imc_db->md_depth; i++) { - mc->mc_pg[i] = mc->mc_pg[i+1]; - mc->mc_ki[i] = mc->mc_ki[i+1]; - } - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[0] == mp) { - m3->mc_snum--; - m3->mc_top--; - for (i=0; imc_snum; i++) { - m3->mc_pg[i] = m3->mc_pg[i+1]; - m3->mc_ki[i] = m3->mc_ki[i+1]; - } - } - } - } - } else - DPUTS("root page doesn't need rebalancing"); - return MDB_SUCCESS; - } - - /* The parent (branch page) must have at least 2 pointers, - * otherwise the tree is invalid. - */ - ptop = mc->mc_top-1; - mdb_cassert(mc, NUMKEYS(mc->mc_pg[ptop]) > 1); - - /* Leaf page fill factor is below the threshold. - * Try to move keys from left or right neighbor, or - * merge with a neighbor page. - */ - - /* Find neighbors. - */ - mdb_cursor_copy(mc, &mn); - mn.mc_xcursor = NULL; - - oldki = mc->mc_ki[mc->mc_top]; - if (mc->mc_ki[ptop] == 0) { - /* We're the leftmost leaf in our parent. - */ - DPUTS("reading right neighbor"); - mn.mc_ki[ptop]++; - node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); - rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); - if (rc) - return rc; - mn.mc_ki[mn.mc_top] = 0; - mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); - } else { - /* There is at least one neighbor to the left. - */ - DPUTS("reading left neighbor"); - mn.mc_ki[ptop]--; - node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); - rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); - if (rc) - return rc; - mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1; - mc->mc_ki[mc->mc_top] = 0; - } - - DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)", - mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), - (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); - - /* If the neighbor page is above threshold and has enough keys, - * move one key from it. Otherwise we should try to merge them. - * (A branch page must never have less than 2 keys.) - */ - minkeys = 1 + (IS_BRANCH(mn.mc_pg[mn.mc_top])); - if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= FILL_THRESHOLD && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { - rc = mdb_node_move(&mn, mc); - if (mc->mc_ki[ptop]) { - oldki++; - } - } else { - if (mc->mc_ki[ptop] == 0) { - rc = mdb_page_merge(&mn, mc); - } else { - oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); - mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; - rc = mdb_page_merge(mc, &mn); - mdb_cursor_copy(&mn, mc); - } - mc->mc_flags &= ~C_EOF; - } - mc->mc_ki[mc->mc_top] = oldki; - return rc; -} - -/** Complete a delete operation started by #mdb_cursor_del(). */ -static int -mdb_cursor_del0(MDB_cursor *mc) -{ - int rc; - MDB_page *mp; - indx_t ki; - unsigned int nkeys; - - ki = mc->mc_ki[mc->mc_top]; - mdb_node_del(mc, mc->mc_db->md_pad); - mc->mc_db->md_entries--; - rc = mdb_rebalance(mc); - - if (rc == MDB_SUCCESS) { - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - mp = mc->mc_pg[mc->mc_top]; - nkeys = NUMKEYS(mp); - - /* if mc points past last node in page, find next sibling */ - if (mc->mc_ki[mc->mc_top] >= nkeys) { - rc = mdb_cursor_sibling(mc, 1); - if (rc == MDB_NOTFOUND) { - mc->mc_flags |= C_EOF; - rc = MDB_SUCCESS; - } - } - - /* Adjust other cursors pointing to mp */ - for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { - m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) - continue; - if (m3 == mc || m3->mc_snum < mc->mc_snum) - continue; - if (m3->mc_pg[mc->mc_top] == mp) { - if (m3->mc_ki[mc->mc_top] >= ki) { - m3->mc_flags |= C_DEL; - if (m3->mc_ki[mc->mc_top] > ki) - m3->mc_ki[mc->mc_top]--; - else if (mc->mc_db->md_flags & MDB_DUPSORT) - m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF; - } - if (m3->mc_ki[mc->mc_top] >= nkeys) { - rc = mdb_cursor_sibling(m3, 1); - if (rc == MDB_NOTFOUND) { - m3->mc_flags |= C_EOF; - rc = MDB_SUCCESS; - } - } - } - } - mc->mc_flags |= C_DEL; - } - - if (rc) - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_del(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data) -{ - if (!key || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) - return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) { - /* must ignore any data */ - data = NULL; - } - - return mdb_del0(txn, dbi, key, data, 0); -} - -static int -mdb_del0(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data, unsigned flags) -{ - MDB_cursor mc; - MDB_xcursor mx; - MDB_cursor_op op; - MDB_val rdata, *xdata; - int rc, exact = 0; - DKBUF; - - DPRINTF(("====> delete db %u key [%s]", dbi, DKEY(key))); - - mdb_cursor_init(&mc, txn, dbi, &mx); - - if (data) { - op = MDB_GET_BOTH; - rdata = *data; - xdata = &rdata; - } else { - op = MDB_SET; - xdata = NULL; - flags |= MDB_NODUPDATA; - } - rc = mdb_cursor_set(&mc, key, xdata, op, &exact); - if (rc == 0) { - /* let mdb_page_split know about this cursor if needed: - * delete will trigger a rebalance; if it needs to move - * a node from one page to another, it will have to - * update the parent's separator key(s). If the new sepkey - * is larger than the current one, the parent page may - * run out of space, triggering a split. We need this - * cursor to be consistent until the end of the rebalance. - */ - mc.mc_flags |= C_UNTRACK; - mc.mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = &mc; - rc = mdb_cursor_del(&mc, flags); - txn->mt_cursors[dbi] = mc.mc_next; - } - return rc; -} - -/** Split a page and insert a new node. - * @param[in,out] mc Cursor pointing to the page and desired insertion index. - * The cursor will be updated to point to the actual page and index where - * the node got inserted after the split. - * @param[in] newkey The key for the newly inserted node. - * @param[in] newdata The data for the newly inserted node. - * @param[in] newpgno The page number, if the new node is a branch node. - * @param[in] nflags The #NODE_ADD_FLAGS for the new node. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, - unsigned int nflags) -{ - unsigned int flags; - int rc = MDB_SUCCESS, new_root = 0, did_split = 0; - indx_t newindx; - pgno_t pgno = 0; - int i, j, split_indx, nkeys, pmax; - MDB_env *env = mc->mc_txn->mt_env; - MDB_node *node; - MDB_val sepkey, rkey, xdata, *rdata = &xdata; - MDB_page *copy = NULL; - MDB_page *mp, *rp, *pp; - int ptop; - MDB_cursor mn; - DKBUF; - - mp = mc->mc_pg[mc->mc_top]; - newindx = mc->mc_ki[mc->mc_top]; - nkeys = NUMKEYS(mp); - - DPRINTF(("-----> splitting %s page %"Z"u and adding [%s] at index %i/%i", - IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, - DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); - - /* Create a right sibling. */ - if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) - return rc; - DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno)); - - if (mc->mc_snum < 2) { - if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp))) - goto done; - /* shift current top to make room for new parent */ - mc->mc_pg[1] = mc->mc_pg[0]; - mc->mc_ki[1] = mc->mc_ki[0]; - mc->mc_pg[0] = pp; - mc->mc_ki[0] = 0; - mc->mc_db->md_root = pp->mp_pgno; - DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno)); - mc->mc_db->md_depth++; - new_root = 1; - - /* Add left (implicit) pointer. */ - if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { - /* undo the pre-push */ - mc->mc_pg[0] = mc->mc_pg[1]; - mc->mc_ki[0] = mc->mc_ki[1]; - mc->mc_db->md_root = mp->mp_pgno; - mc->mc_db->md_depth--; - goto done; - } - mc->mc_snum = 2; - mc->mc_top = 1; - ptop = 0; - } else { - ptop = mc->mc_top-1; - DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno)); - } - - mc->mc_flags |= C_SPLITTING; - mdb_cursor_copy(mc, &mn); - mn.mc_pg[mn.mc_top] = rp; - mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; - - if (nflags & MDB_APPEND) { - mn.mc_ki[mn.mc_top] = 0; - sepkey = *newkey; - split_indx = newindx; - nkeys = 0; - } else { - - split_indx = (nkeys+1) / 2; - - if (IS_LEAF2(rp)) { - char *split, *ins; - int x; - unsigned int lsize, rsize, ksize; - /* Move half of the keys to the right sibling */ - x = mc->mc_ki[mc->mc_top] - split_indx; - ksize = mc->mc_db->md_pad; - split = LEAF2KEY(mp, split_indx, ksize); - rsize = (nkeys - split_indx) * ksize; - lsize = (nkeys - split_indx) * sizeof(indx_t); - mp->mp_lower -= lsize; - rp->mp_lower += lsize; - mp->mp_upper += rsize - lsize; - rp->mp_upper -= rsize - lsize; - sepkey.mv_size = ksize; - if (newindx == split_indx) { - sepkey.mv_data = newkey->mv_data; - } else { - sepkey.mv_data = split; - } - if (x<0) { - ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize); - memcpy(rp->mp_ptrs, split, rsize); - sepkey.mv_data = rp->mp_ptrs; - memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); - memcpy(ins, newkey->mv_data, ksize); - mp->mp_lower += sizeof(indx_t); - mp->mp_upper -= ksize - sizeof(indx_t); - } else { - if (x) - memcpy(rp->mp_ptrs, split, x * ksize); - ins = LEAF2KEY(rp, x, ksize); - memcpy(ins, newkey->mv_data, ksize); - memcpy(ins+ksize, split + x * ksize, rsize - x * ksize); - rp->mp_lower += sizeof(indx_t); - rp->mp_upper -= ksize - sizeof(indx_t); - mc->mc_ki[mc->mc_top] = x; - mc->mc_pg[mc->mc_top] = rp; - } - } else { - int psize, nsize, k; - /* Maximum free space in an empty page */ - pmax = env->me_psize - PAGEHDRSZ; - if (IS_LEAF(mp)) - nsize = mdb_leaf_size(env, newkey, newdata); - else - nsize = mdb_branch_size(env, newkey); - nsize = EVEN(nsize); - - /* grab a page to hold a temporary copy */ - copy = mdb_page_malloc(mc->mc_txn, 1); - if (copy == NULL) { - rc = ENOMEM; - goto done; - } - copy->mp_pgno = mp->mp_pgno; - copy->mp_flags = mp->mp_flags; - copy->mp_lower = (PAGEHDRSZ-PAGEBASE); - copy->mp_upper = env->me_psize - PAGEBASE; - - /* prepare to insert */ - for (i=0, j=0; imp_ptrs[j++] = 0; - } - copy->mp_ptrs[j++] = mp->mp_ptrs[i]; - } - - /* When items are relatively large the split point needs - * to be checked, because being off-by-one will make the - * difference between success or failure in mdb_node_add. - * - * It's also relevant if a page happens to be laid out - * such that one half of its nodes are all "small" and - * the other half of its nodes are "large." If the new - * item is also "large" and falls on the half with - * "large" nodes, it also may not fit. - * - * As a final tweak, if the new item goes on the last - * spot on the page (and thus, onto the new page), bias - * the split so the new page is emptier than the old page. - * This yields better packing during sequential inserts. - */ - if (nkeys < 20 || nsize > pmax/16 || newindx >= nkeys) { - /* Find split point */ - psize = 0; - if (newindx <= split_indx || newindx >= nkeys) { - i = 0; j = 1; - k = newindx >= nkeys ? nkeys : split_indx+2; - } else { - i = nkeys; j = -1; - k = split_indx-1; - } - for (; i!=k; i+=j) { - if (i == newindx) { - psize += nsize; - node = NULL; - } else { - node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); - psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t); - if (IS_LEAF(mp)) { - if (F_ISSET(node->mn_flags, F_BIGDATA)) - psize += sizeof(pgno_t); - else - psize += NODEDSZ(node); - } - psize = EVEN(psize); - } - if (psize > pmax || i == k-j) { - split_indx = i + (j<0); - break; - } - } - } - if (split_indx == newindx) { - sepkey.mv_size = newkey->mv_size; - sepkey.mv_data = newkey->mv_data; - } else { - node = (MDB_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEBASE); - sepkey.mv_size = node->mn_ksize; - sepkey.mv_data = NODEKEY(node); - } - } - } - - DPRINTF(("separator is %d [%s]", split_indx, DKEY(&sepkey))); - - /* Copy separator key to the parent. - */ - if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) { - mn.mc_snum--; - mn.mc_top--; - did_split = 1; - rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0); - if (rc) - goto done; - - /* root split? */ - if (mn.mc_snum == mc->mc_snum) { - mc->mc_pg[mc->mc_snum] = mc->mc_pg[mc->mc_top]; - mc->mc_ki[mc->mc_snum] = mc->mc_ki[mc->mc_top]; - mc->mc_pg[mc->mc_top] = mc->mc_pg[ptop]; - mc->mc_ki[mc->mc_top] = mc->mc_ki[ptop]; - mc->mc_snum++; - mc->mc_top++; - ptop++; - } - /* Right page might now have changed parent. - * Check if left page also changed parent. - */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { - for (i=0; imc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; - } - mc->mc_pg[ptop] = mn.mc_pg[ptop]; - if (mn.mc_ki[ptop]) { - mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; - } else { - /* find right page's left sibling */ - mc->mc_ki[ptop] = mn.mc_ki[ptop]; - mdb_cursor_sibling(mc, 0); - } - } - } else { - mn.mc_top--; - rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0); - mn.mc_top++; - } - mc->mc_flags ^= C_SPLITTING; - if (rc != MDB_SUCCESS) { - goto done; - } - if (nflags & MDB_APPEND) { - mc->mc_pg[mc->mc_top] = rp; - mc->mc_ki[mc->mc_top] = 0; - rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags); - if (rc) - goto done; - for (i=0; imc_top; i++) - mc->mc_ki[i] = mn.mc_ki[i]; - } else if (!IS_LEAF2(mp)) { - /* Move nodes */ - mc->mc_pg[mc->mc_top] = rp; - i = split_indx; - j = 0; - do { - if (i == newindx) { - rkey.mv_data = newkey->mv_data; - rkey.mv_size = newkey->mv_size; - if (IS_LEAF(mp)) { - rdata = newdata; - } else - pgno = newpgno; - flags = nflags; - /* Update index for the new key. */ - mc->mc_ki[mc->mc_top] = j; - } else { - node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); - rkey.mv_data = NODEKEY(node); - rkey.mv_size = node->mn_ksize; - if (IS_LEAF(mp)) { - xdata.mv_data = NODEDATA(node); - xdata.mv_size = NODEDSZ(node); - rdata = &xdata; - } else - pgno = NODEPGNO(node); - flags = node->mn_flags; - } - - if (!IS_LEAF(mp) && j == 0) { - /* First branch index doesn't need key data. */ - rkey.mv_size = 0; - } - - rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags); - if (rc) - goto done; - if (i == nkeys) { - i = 0; - j = 0; - mc->mc_pg[mc->mc_top] = copy; - } else { - i++; - j++; - } - } while (i != split_indx); - - nkeys = NUMKEYS(copy); - for (i=0; imp_ptrs[i] = copy->mp_ptrs[i]; - mp->mp_lower = copy->mp_lower; - mp->mp_upper = copy->mp_upper; - memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1), - env->me_psize - copy->mp_upper - PAGEBASE); - - /* reset back to original page */ - if (newindx < split_indx) { - mc->mc_pg[mc->mc_top] = mp; - if (nflags & MDB_RESERVE) { - node = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - if (!(node->mn_flags & F_BIGDATA)) - newdata->mv_data = NODEDATA(node); - } - } else { - mc->mc_pg[mc->mc_top] = rp; - mc->mc_ki[ptop]++; - /* Make sure mc_ki is still valid. - */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { - for (i=0; i<=ptop; i++) { - mc->mc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; - } - } - } - } - - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - int fixup = NUMKEYS(mp); - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == mc) - continue; - if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) - continue; - if (m3->mc_flags & C_SPLITTING) - continue; - if (new_root) { - int k; - /* root split */ - for (k=m3->mc_top; k>=0; k--) { - m3->mc_ki[k+1] = m3->mc_ki[k]; - m3->mc_pg[k+1] = m3->mc_pg[k]; - } - if (m3->mc_ki[0] >= split_indx) { - m3->mc_ki[0] = 1; - } else { - m3->mc_ki[0] = 0; - } - m3->mc_pg[0] = mc->mc_pg[0]; - m3->mc_snum++; - m3->mc_top++; - } - if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) { - if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE)) - m3->mc_ki[mc->mc_top]++; - if (m3->mc_ki[mc->mc_top] >= fixup) { - m3->mc_pg[mc->mc_top] = rp; - m3->mc_ki[mc->mc_top] -= fixup; - m3->mc_ki[ptop] = mn.mc_ki[ptop]; - } - } else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] && - m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { - m3->mc_ki[ptop]++; - } - } - } - DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); - -done: - if (copy) /* tmp page */ - mdb_page_free(env, copy); - if (rc) - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_put(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data, unsigned int flags) -{ - MDB_cursor mc; - MDB_xcursor mx; - - if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if ((flags & (MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) != flags) - return EINVAL; - - mdb_cursor_init(&mc, txn, dbi, &mx); - return mdb_cursor_put(&mc, key, data, flags); -} - -#ifndef MDB_WBUF -#define MDB_WBUF (1024*1024) -#endif - - /** State needed for a compacting copy. */ -typedef struct mdb_copy { - pthread_mutex_t mc_mutex; - pthread_cond_t mc_cond; - char *mc_wbuf[2]; - char *mc_over[2]; - MDB_env *mc_env; - MDB_txn *mc_txn; - int mc_wlen[2]; - int mc_olen[2]; - pgno_t mc_next_pgno; - HANDLE mc_fd; - int mc_status; - volatile int mc_new; - int mc_toggle; - -} mdb_copy; - - /** Dedicated writer thread for compacting copy. */ -static THREAD_RET ESECT -mdb_env_copythr(void *arg) -{ - mdb_copy *my = arg; - char *ptr; - int toggle = 0, wsize, rc; -#ifdef _WIN32 - DWORD len; -#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) -#else - int len; -#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) -#endif - - pthread_mutex_lock(&my->mc_mutex); - my->mc_new = 0; - pthread_cond_signal(&my->mc_cond); - for(;;) { - while (!my->mc_new) - pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - if (my->mc_new < 0) { - my->mc_new = 0; - break; - } - my->mc_new = 0; - wsize = my->mc_wlen[toggle]; - ptr = my->mc_wbuf[toggle]; -again: - while (wsize > 0) { - DO_WRITE(rc, my->mc_fd, ptr, wsize, len); - if (!rc) { - rc = ErrCode(); - break; - } else if (len > 0) { - rc = MDB_SUCCESS; - ptr += len; - wsize -= len; - continue; - } else { - rc = EIO; - break; - } - } - if (rc) { - my->mc_status = rc; - break; - } - /* If there's an overflow page tail, write it too */ - if (my->mc_olen[toggle]) { - wsize = my->mc_olen[toggle]; - ptr = my->mc_over[toggle]; - my->mc_olen[toggle] = 0; - goto again; - } - my->mc_wlen[toggle] = 0; - toggle ^= 1; - pthread_cond_signal(&my->mc_cond); - } - pthread_cond_signal(&my->mc_cond); - pthread_mutex_unlock(&my->mc_mutex); - return (THREAD_RET)0; -#undef DO_WRITE -} - - /** Tell the writer thread there's a buffer ready to write */ -static int ESECT -mdb_env_cthr_toggle(mdb_copy *my, int st) -{ - int toggle = my->mc_toggle ^ 1; - pthread_mutex_lock(&my->mc_mutex); - if (my->mc_status) { - pthread_mutex_unlock(&my->mc_mutex); - return my->mc_status; - } - while (my->mc_new == 1) - pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - my->mc_new = st; - my->mc_toggle = toggle; - pthread_cond_signal(&my->mc_cond); - pthread_mutex_unlock(&my->mc_mutex); - return 0; -} - - /** Depth-first tree traversal for compacting copy. */ -static int ESECT -mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) -{ - MDB_cursor mc; - MDB_txn *txn = my->mc_txn; - MDB_node *ni; - MDB_page *mo, *mp, *leaf; - char *buf, *ptr; - int rc, toggle; - unsigned int i; - - /* Empty DB, nothing to do */ - if (*pg == P_INVALID) - return MDB_SUCCESS; - - mc.mc_snum = 1; - mc.mc_top = 0; - mc.mc_txn = txn; - - rc = mdb_page_get(my->mc_txn, *pg, &mc.mc_pg[0], NULL); - if (rc) - return rc; - rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST); - if (rc) - return rc; - - /* Make cursor pages writable */ - buf = ptr = malloc(my->mc_env->me_psize * mc.mc_snum); - if (buf == NULL) - return ENOMEM; - - for (i=0; imc_env->me_psize); - mc.mc_pg[i] = (MDB_page *)ptr; - ptr += my->mc_env->me_psize; - } - - /* This is writable space for a leaf page. Usually not needed. */ - leaf = (MDB_page *)ptr; - - toggle = my->mc_toggle; - while (mc.mc_snum > 0) { - unsigned n; - mp = mc.mc_pg[mc.mc_top]; - n = NUMKEYS(mp); - - if (IS_LEAF(mp)) { - if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) { - for (i=0; imn_flags & F_BIGDATA) { - MDB_page *omp; - pgno_t pg; - - /* Need writable leaf */ - if (mp != leaf) { - mc.mc_pg[mc.mc_top] = leaf; - mdb_page_copy(leaf, mp, my->mc_env->me_psize); - mp = leaf; - ni = NODEPTR(mp, i); - } - - memcpy(&pg, NODEDATA(ni), sizeof(pg)); - rc = mdb_page_get(txn, pg, &omp, NULL); - if (rc) - goto done; - if (my->mc_wlen[toggle] >= MDB_WBUF) { - rc = mdb_env_cthr_toggle(my, 1); - if (rc) - goto done; - toggle = my->mc_toggle; - } - mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); - memcpy(mo, omp, my->mc_env->me_psize); - mo->mp_pgno = my->mc_next_pgno; - my->mc_next_pgno += omp->mp_pages; - my->mc_wlen[toggle] += my->mc_env->me_psize; - if (omp->mp_pages > 1) { - my->mc_olen[toggle] = my->mc_env->me_psize * (omp->mp_pages - 1); - my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; - rc = mdb_env_cthr_toggle(my, 1); - if (rc) - goto done; - toggle = my->mc_toggle; - } - memcpy(NODEDATA(ni), &mo->mp_pgno, sizeof(pgno_t)); - } else if (ni->mn_flags & F_SUBDATA) { - MDB_db db; - - /* Need writable leaf */ - if (mp != leaf) { - mc.mc_pg[mc.mc_top] = leaf; - mdb_page_copy(leaf, mp, my->mc_env->me_psize); - mp = leaf; - ni = NODEPTR(mp, i); - } - - memcpy(&db, NODEDATA(ni), sizeof(db)); - my->mc_toggle = toggle; - rc = mdb_env_cwalk(my, &db.md_root, ni->mn_flags & F_DUPDATA); - if (rc) - goto done; - toggle = my->mc_toggle; - memcpy(NODEDATA(ni), &db, sizeof(db)); - } - } - } - } else { - mc.mc_ki[mc.mc_top]++; - if (mc.mc_ki[mc.mc_top] < n) { - pgno_t pg; -again: - ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]); - pg = NODEPGNO(ni); - rc = mdb_page_get(txn, pg, &mp, NULL); - if (rc) - goto done; - mc.mc_top++; - mc.mc_snum++; - mc.mc_ki[mc.mc_top] = 0; - if (IS_BRANCH(mp)) { - /* Whenever we advance to a sibling branch page, - * we must proceed all the way down to its first leaf. - */ - mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); - goto again; - } else - mc.mc_pg[mc.mc_top] = mp; - continue; - } - } - if (my->mc_wlen[toggle] >= MDB_WBUF) { - rc = mdb_env_cthr_toggle(my, 1); - if (rc) - goto done; - toggle = my->mc_toggle; - } - mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); - mdb_page_copy(mo, mp, my->mc_env->me_psize); - mo->mp_pgno = my->mc_next_pgno++; - my->mc_wlen[toggle] += my->mc_env->me_psize; - if (mc.mc_top) { - /* Update parent if there is one */ - ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); - SETPGNO(ni, mo->mp_pgno); - mdb_cursor_pop(&mc); - } else { - /* Otherwise we're done */ - *pg = mo->mp_pgno; - break; - } - } -done: - free(buf); - return rc; -} - - /** Copy environment with compaction. */ -static int ESECT -mdb_env_copyfd1(MDB_env *env, HANDLE fd) -{ - MDB_meta *mm; - MDB_page *mp; - mdb_copy my; - MDB_txn *txn = NULL; - pthread_t thr; - int rc; - -#ifdef _WIN32 - my.mc_mutex = CreateMutex(NULL, FALSE, NULL); - my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL); - my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); - if (my.mc_wbuf[0] == NULL) - return errno; -#else - pthread_mutex_init(&my.mc_mutex, NULL); - pthread_cond_init(&my.mc_cond, NULL); -#ifdef HAVE_MEMALIGN - my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); - if (my.mc_wbuf[0] == NULL) - return errno; -#else - rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); - if (rc) - return rc; -#endif -#endif - memset(my.mc_wbuf[0], 0, MDB_WBUF*2); - my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; - my.mc_wlen[0] = 0; - my.mc_wlen[1] = 0; - my.mc_olen[0] = 0; - my.mc_olen[1] = 0; - my.mc_next_pgno = 2; - my.mc_status = 0; - my.mc_new = 1; - my.mc_toggle = 0; - my.mc_env = env; - my.mc_fd = fd; - THREAD_CREATE(thr, mdb_env_copythr, &my); - - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - if (rc) - return rc; - - mp = (MDB_page *)my.mc_wbuf[0]; - memset(mp, 0, 2*env->me_psize); - mp->mp_pgno = 0; - mp->mp_flags = P_META; - mm = (MDB_meta *)METADATA(mp); - mdb_env_init_meta0(env, mm); - mm->mm_address = env->me_metas[0]->mm_address; - - mp = (MDB_page *)(my.mc_wbuf[0] + env->me_psize); - mp->mp_pgno = 1; - mp->mp_flags = P_META; - *(MDB_meta *)METADATA(mp) = *mm; - mm = (MDB_meta *)METADATA(mp); - - /* Count the number of free pages, subtract from lastpg to find - * number of active pages - */ - { - MDB_ID freecount = 0; - MDB_cursor mc; - MDB_val key, data; - mdb_cursor_init(&mc, txn, FREE_DBI, NULL); - while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) - freecount += *(MDB_ID *)data.mv_data; - freecount += txn->mt_dbs[0].md_branch_pages + - txn->mt_dbs[0].md_leaf_pages + - txn->mt_dbs[0].md_overflow_pages; - - /* Set metapage 1 */ - mm->mm_last_pg = txn->mt_next_pgno - freecount - 1; - mm->mm_dbs[1] = txn->mt_dbs[1]; - mm->mm_dbs[1].md_root = mm->mm_last_pg; - mm->mm_txnid = 1; - } - my.mc_wlen[0] = env->me_psize * 2; - my.mc_txn = txn; - pthread_mutex_lock(&my.mc_mutex); - while(my.mc_new) - pthread_cond_wait(&my.mc_cond, &my.mc_mutex); - pthread_mutex_unlock(&my.mc_mutex); - rc = mdb_env_cwalk(&my, &txn->mt_dbs[1].md_root, 0); - if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle]) - rc = mdb_env_cthr_toggle(&my, 1); - mdb_env_cthr_toggle(&my, -1); - pthread_mutex_lock(&my.mc_mutex); - while(my.mc_new) - pthread_cond_wait(&my.mc_cond, &my.mc_mutex); - pthread_mutex_unlock(&my.mc_mutex); - THREAD_FINISH(thr); - - mdb_txn_abort(txn); -#ifdef _WIN32 - CloseHandle(my.mc_cond); - CloseHandle(my.mc_mutex); - _aligned_free(my.mc_wbuf[0]); -#else - pthread_cond_destroy(&my.mc_cond); - pthread_mutex_destroy(&my.mc_mutex); - free(my.mc_wbuf[0]); -#endif - return rc; -} - - /** Copy environment as-is. */ -static int ESECT -mdb_env_copyfd0(MDB_env *env, HANDLE fd) -{ - MDB_txn *txn = NULL; - mdb_mutex_t wmutex = NULL; - int rc; - size_t wsize; - char *ptr; -#ifdef _WIN32 - DWORD len, w2; -#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) -#else - ssize_t len; - size_t w2; -#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) -#endif - - /* Do the lock/unlock of the reader mutex before starting the - * write txn. Otherwise other read txns could block writers. - */ - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - if (rc) - return rc; - - if (env->me_txns) { - /* We must start the actual read txn after blocking writers */ - mdb_txn_reset0(txn, "reset-stage1"); - - /* Temporarily block writers until we snapshot the meta pages */ - wmutex = MDB_MUTEX(env, w); - if (LOCK_MUTEX(rc, env, wmutex)) - goto leave; - - rc = mdb_txn_renew0(txn); - if (rc) { - UNLOCK_MUTEX(wmutex); - goto leave; - } - } - - wsize = env->me_psize * 2; - ptr = env->me_map; - w2 = wsize; - while (w2 > 0) { - DO_WRITE(rc, fd, ptr, w2, len); - if (!rc) { - rc = ErrCode(); - break; - } else if (len > 0) { - rc = MDB_SUCCESS; - ptr += len; - w2 -= len; - continue; - } else { - /* Non-blocking or async handles are not supported */ - rc = EIO; - break; - } - } - if (wmutex) - UNLOCK_MUTEX(wmutex); - - if (rc) - goto leave; - - w2 = txn->mt_next_pgno * env->me_psize; -#ifdef WIN32 - { - LARGE_INTEGER fsize; - GetFileSizeEx(env->me_fd, &fsize); - if (w2 > fsize.QuadPart) - w2 = fsize.QuadPart; - } -#else - { - struct stat st; - fstat(env->me_fd, &st); - if (w2 > (size_t)st.st_size) - w2 = st.st_size; - } -#endif - wsize = w2 - wsize; - while (wsize > 0) { - if (wsize > MAX_WRITE) - w2 = MAX_WRITE; - else - w2 = wsize; - DO_WRITE(rc, fd, ptr, w2, len); - if (!rc) { - rc = ErrCode(); - break; - } else if (len > 0) { - rc = MDB_SUCCESS; - ptr += len; - wsize -= len; - continue; - } else { - rc = EIO; - break; - } - } - -leave: - mdb_txn_abort(txn); - return rc; -} - -int ESECT -mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags) -{ - if (flags & MDB_CP_COMPACT) - return mdb_env_copyfd1(env, fd); - else - return mdb_env_copyfd0(env, fd); -} - -int ESECT -mdb_env_copyfd(MDB_env *env, HANDLE fd) -{ - return mdb_env_copyfd2(env, fd, 0); -} - -int ESECT -mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) -{ - int rc, len; - char *lpath; - HANDLE newfd = INVALID_HANDLE_VALUE; - - if (env->me_flags & MDB_NOSUBDIR) { - lpath = (char *)path; - } else { - len = strlen(path); - len += sizeof(DATANAME); - lpath = malloc(len); - if (!lpath) - return ENOMEM; - sprintf(lpath, "%s" DATANAME, path); - } - - /* The destination path must exist, but the destination file must not. - * We don't want the OS to cache the writes, since the source data is - * already in the OS cache. - */ -#ifdef _WIN32 - newfd = CreateFile(lpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, - FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); -#else - newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666); -#endif - if (newfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } - - if (env->me_psize >= env->me_os_psize) { -#ifdef O_DIRECT - /* Set O_DIRECT if the file system supports it */ - if ((rc = fcntl(newfd, F_GETFL)) != -1) - (void) fcntl(newfd, F_SETFL, rc | O_DIRECT); -#endif -#ifdef F_NOCACHE /* __APPLE__ */ - rc = fcntl(newfd, F_NOCACHE, 1); - if (rc) { - rc = ErrCode(); - goto leave; - } -#endif - } - - rc = mdb_env_copyfd2(env, newfd, flags); - -leave: - if (!(env->me_flags & MDB_NOSUBDIR)) - free(lpath); - if (newfd != INVALID_HANDLE_VALUE) - if (close(newfd) < 0 && rc == MDB_SUCCESS) - rc = ErrCode(); - - return rc; -} - -int ESECT -mdb_env_copy(MDB_env *env, const char *path) -{ - return mdb_env_copy2(env, path, 0); -} - -int ESECT -mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) -{ - if (flag & (env->me_map ? ~CHANGEABLE : ~(CHANGEABLE|CHANGELESS))) - return EINVAL; - if (onoff) - env->me_flags |= flag; - else - env->me_flags &= ~flag; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_flags(MDB_env *env, unsigned int *arg) -{ - if (!env || !arg) - return EINVAL; - - *arg = env->me_flags; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_userctx(MDB_env *env, void *ctx) -{ - if (!env) - return EINVAL; - env->me_userctx = ctx; - return MDB_SUCCESS; -} - -void * ESECT -mdb_env_get_userctx(MDB_env *env) -{ - return env ? env->me_userctx : NULL; -} - -int ESECT -mdb_env_set_assert(MDB_env *env, MDB_assert_func *func) -{ - if (!env) - return EINVAL; -#ifndef NDEBUG - env->me_assert_func = func; -#endif - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_path(MDB_env *env, const char **arg) -{ - if (!env || !arg) - return EINVAL; - - *arg = env->me_path; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg) -{ - if (!env || !arg) - return EINVAL; - - *arg = env->me_fd; - return MDB_SUCCESS; -} - -/** Common code for #mdb_stat() and #mdb_env_stat(). - * @param[in] env the environment to operate in. - * @param[in] db the #MDB_db record containing the stats to return. - * @param[out] arg the address of an #MDB_stat structure to receive the stats. - * @return 0, this function always succeeds. - */ -static int ESECT -mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg) -{ - arg->ms_psize = env->me_psize; - arg->ms_depth = db->md_depth; - arg->ms_branch_pages = db->md_branch_pages; - arg->ms_leaf_pages = db->md_leaf_pages; - arg->ms_overflow_pages = db->md_overflow_pages; - arg->ms_entries = db->md_entries; - - return MDB_SUCCESS; -} - -int ESECT -mdb_env_stat(MDB_env *env, MDB_stat *arg) -{ - int toggle; - - if (env == NULL || arg == NULL) - return EINVAL; - - toggle = mdb_env_pick_meta(env); - - return mdb_stat0(env, &env->me_metas[toggle]->mm_dbs[MAIN_DBI], arg); -} - -int ESECT -mdb_env_info(MDB_env *env, MDB_envinfo *arg) -{ - int toggle; - - if (env == NULL || arg == NULL) - return EINVAL; - - toggle = mdb_env_pick_meta(env); - arg->me_mapaddr = env->me_metas[toggle]->mm_address; - arg->me_mapsize = env->me_mapsize; - arg->me_maxreaders = env->me_maxreaders; - - /* me_numreaders may be zero if this process never used any readers. Use - * the shared numreader count if it exists. - */ - arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : env->me_numreaders; - - arg->me_last_pgno = env->me_metas[toggle]->mm_last_pg; - arg->me_last_txnid = env->me_metas[toggle]->mm_txnid; - return MDB_SUCCESS; -} - -/** Set the default comparison functions for a database. - * Called immediately after a database is opened to set the defaults. - * The user can then override them with #mdb_set_compare() or - * #mdb_set_dupsort(). - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - */ -static void -mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi) -{ - uint16_t f = txn->mt_dbs[dbi].md_flags; - - txn->mt_dbxs[dbi].md_cmp = - (f & MDB_REVERSEKEY) ? mdb_cmp_memnr : - (f & MDB_INTEGERKEY) ? mdb_cmp_cint : mdb_cmp_memn; - - txn->mt_dbxs[dbi].md_dcmp = - !(f & MDB_DUPSORT) ? 0 : - ((f & MDB_INTEGERDUP) - ? ((f & MDB_DUPFIXED) ? mdb_cmp_int : mdb_cmp_cint) - : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn)); -} - -int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi) -{ - MDB_val key, data; - MDB_dbi i; - MDB_cursor mc; - MDB_db dummy; - int rc, dbflag, exact; - unsigned int unused = 0, seq; - size_t len; - - if (txn->mt_dbxs[FREE_DBI].md_cmp == NULL) { - mdb_default_cmp(txn, FREE_DBI); - } - - if ((flags & VALID_FLAGS) != flags) - return EINVAL; - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - /* main DB? */ - if (!name) { - *dbi = MAIN_DBI; - if (flags & PERSISTENT_FLAGS) { - uint16_t f2 = flags & PERSISTENT_FLAGS; - /* make sure flag changes get committed */ - if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) { - txn->mt_dbs[MAIN_DBI].md_flags |= f2; - txn->mt_flags |= MDB_TXN_DIRTY; - } - } - mdb_default_cmp(txn, MAIN_DBI); - return MDB_SUCCESS; - } - - if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { - mdb_default_cmp(txn, MAIN_DBI); - } - - /* Is the DB already open? */ - len = strlen(name); - for (i=2; imt_numdbs; i++) { - if (!txn->mt_dbxs[i].md_name.mv_size) { - /* Remember this free slot */ - if (!unused) unused = i; - continue; - } - if (len == txn->mt_dbxs[i].md_name.mv_size && - !strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) { - *dbi = i; - return MDB_SUCCESS; - } - } - - /* If no free slot and max hit, fail */ - if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs) - return MDB_DBS_FULL; - - /* Cannot mix named databases with some mainDB flags */ - if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY)) - return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; - - /* Find the DB info */ - dbflag = DB_NEW|DB_VALID; - exact = 0; - key.mv_size = len; - key.mv_data = (void *)name; - mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); - rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact); - if (rc == MDB_SUCCESS) { - /* make sure this is actually a DB */ - MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); - if (!(node->mn_flags & F_SUBDATA)) - return MDB_INCOMPATIBLE; - } else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) { - /* Create if requested */ - data.mv_size = sizeof(MDB_db); - data.mv_data = &dummy; - memset(&dummy, 0, sizeof(dummy)); - dummy.md_root = P_INVALID; - dummy.md_flags = flags & PERSISTENT_FLAGS; - rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA); - dbflag |= DB_DIRTY; - } - - /* OK, got info, add to table */ - if (rc == MDB_SUCCESS) { - unsigned int slot = unused ? unused : txn->mt_numdbs; - txn->mt_dbxs[slot].md_name.mv_data = strdup(name); - txn->mt_dbxs[slot].md_name.mv_size = len; - txn->mt_dbxs[slot].md_rel = NULL; - txn->mt_dbflags[slot] = dbflag; - /* txn-> and env-> are the same in read txns, use - * tmp variable to avoid undefined assignment - */ - seq = ++txn->mt_env->me_dbiseqs[slot]; - txn->mt_dbiseqs[slot] = seq; - - memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); - *dbi = slot; - mdb_default_cmp(txn, slot); - if (!unused) { - txn->mt_numdbs++; - } - } - - return rc; -} - -int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) -{ - if (!arg || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - if (txn->mt_dbflags[dbi] & DB_STALE) { - MDB_cursor mc; - MDB_xcursor mx; - /* Stale, must read the DB's root. cursor_init does it for us. */ - mdb_cursor_init(&mc, txn, dbi, &mx); - } - return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg); -} - -void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) -{ - char *ptr; - if (dbi <= MAIN_DBI || dbi >= env->me_maxdbs) - return; - ptr = env->me_dbxs[dbi].md_name.mv_data; - /* If there was no name, this was already closed */ - if (ptr) { - env->me_dbxs[dbi].md_name.mv_data = NULL; - env->me_dbxs[dbi].md_name.mv_size = 0; - env->me_dbflags[dbi] = 0; - env->me_dbiseqs[dbi]++; - free(ptr); - } -} - -int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) -{ - /* We could return the flags for the FREE_DBI too but what's the point? */ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; - return MDB_SUCCESS; -} - -/** Add all the DB's pages to the free list. - * @param[in] mc Cursor on the DB to free. - * @param[in] subs non-Zero to check for sub-DBs in this DB. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_drop0(MDB_cursor *mc, int subs) -{ - int rc; - - rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); - if (rc == MDB_SUCCESS) { - MDB_txn *txn = mc->mc_txn; - MDB_node *ni; - MDB_cursor mx; - unsigned int i; - - /* LEAF2 pages have no nodes, cannot have sub-DBs */ - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) - mdb_cursor_pop(mc); - - mdb_cursor_copy(mc, &mx); - while (mc->mc_snum > 0) { - MDB_page *mp = mc->mc_pg[mc->mc_top]; - unsigned n = NUMKEYS(mp); - if (IS_LEAF(mp)) { - for (i=0; imn_flags & F_BIGDATA) { - MDB_page *omp; - pgno_t pg; - memcpy(&pg, NODEDATA(ni), sizeof(pg)); - rc = mdb_page_get(txn, pg, &omp, NULL); - if (rc != 0) - goto done; - mdb_cassert(mc, IS_OVERFLOW(omp)); - rc = mdb_midl_append_range(&txn->mt_free_pgs, - pg, omp->mp_pages); - if (rc) - goto done; - } else if (subs && (ni->mn_flags & F_SUBDATA)) { - mdb_xcursor_init1(mc, ni); - rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); - if (rc) - goto done; - } - } - } else { - if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) - goto done; - for (i=0; imt_free_pgs, pg); - } - } - if (!mc->mc_top) - break; - mc->mc_ki[mc->mc_top] = i; - rc = mdb_cursor_sibling(mc, 1); - if (rc) { - if (rc != MDB_NOTFOUND) - goto done; - /* no more siblings, go back to beginning - * of previous level. - */ - mdb_cursor_pop(mc); - mc->mc_ki[0] = 0; - for (i=1; imc_snum; i++) { - mc->mc_ki[i] = 0; - mc->mc_pg[i] = mx.mc_pg[i]; - } - } - } - /* free it */ - rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root); -done: - if (rc) - txn->mt_flags |= MDB_TXN_ERROR; - } else if (rc == MDB_NOTFOUND) { - rc = MDB_SUCCESS; - } - return rc; -} - -int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) -{ - MDB_cursor *mc, *m2; - int rc; - - if ((unsigned)del > 1 || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) - return EACCES; - - if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, dbi)) - return MDB_BAD_DBI; - - rc = mdb_cursor_open(txn, dbi, &mc); - if (rc) - return rc; - - rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT); - /* Invalidate the dropped DB's cursors */ - for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) - m2->mc_flags &= ~(C_INITIALIZED|C_EOF); - if (rc) - goto leave; - - /* Can't delete the main DB */ - if (del && dbi > MAIN_DBI) { - rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, 0); - if (!rc) { - txn->mt_dbflags[dbi] = DB_STALE; - mdb_dbi_close(txn->mt_env, dbi); - } else { - txn->mt_flags |= MDB_TXN_ERROR; - } - } else { - /* reset the DB record, mark it dirty */ - txn->mt_dbflags[dbi] |= DB_DIRTY; - txn->mt_dbs[dbi].md_depth = 0; - txn->mt_dbs[dbi].md_branch_pages = 0; - txn->mt_dbs[dbi].md_leaf_pages = 0; - txn->mt_dbs[dbi].md_overflow_pages = 0; - txn->mt_dbs[dbi].md_entries = 0; - txn->mt_dbs[dbi].md_root = P_INVALID; - - txn->mt_flags |= MDB_TXN_DIRTY; - } -leave: - mdb_cursor_close(mc); - return rc; -} - -int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) -{ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - txn->mt_dbxs[dbi].md_cmp = cmp; - return MDB_SUCCESS; -} - -int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) -{ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - txn->mt_dbxs[dbi].md_dcmp = cmp; - return MDB_SUCCESS; -} - -int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) -{ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - txn->mt_dbxs[dbi].md_rel = rel; - return MDB_SUCCESS; -} - -int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) -{ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - txn->mt_dbxs[dbi].md_relctx = ctx; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_maxkeysize(MDB_env *env) -{ - return ENV_MAXKEY(env); -} - -int ESECT -mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) -{ - unsigned int i, rdrs; - MDB_reader *mr; - char buf[64]; - int rc = 0, first = 1; - - if (!env || !func) - return -1; - if (!env->me_txns) { - return func("(no reader locks)\n", ctx); - } - rdrs = env->me_txns->mti_numreaders; - mr = env->me_txns->mti_readers; - for (i=0; i> 1; - cursor = base + pivot + 1; - val = pid - ids[cursor]; - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - /* found, so it's a duplicate */ - return -1; - } - } - - if( val > 0 ) { - ++cursor; - } - ids[0]++; - for (n = ids[0]; n > cursor; n--) - ids[n] = ids[n-1]; - ids[n] = pid; - return 0; -} - -int ESECT -mdb_reader_check(MDB_env *env, int *dead) -{ - if (!env) - return EINVAL; - if (dead) - *dead = 0; - return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS; -} - -/** As #mdb_reader_check(). rlocked = . */ -static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead) -{ - mdb_mutex_t rmutex = rlocked ? NULL : MDB_MUTEX(env, r); - unsigned int i, j, rdrs; - MDB_reader *mr; - MDB_PID_T *pids, pid; - int rc = MDB_SUCCESS, count = 0; - - rdrs = env->me_txns->mti_numreaders; - pids = malloc((rdrs+1) * sizeof(MDB_PID_T)); - if (!pids) - return ENOMEM; - pids[0] = 0; - mr = env->me_txns->mti_readers; - for (i=0; ime_pid) { - if (mdb_pid_insert(pids, pid) == 0) { - if (!mdb_reader_pid(env, Pidcheck, pid)) { - /* Stale reader found */ - j = i; - if (rmutex) { - if ((rc = LOCK_MUTEX0(rmutex)) != 0) { - if ((rc = mdb_mutex_failed(env, rmutex, rc))) - break; - rdrs = 0; /* the above checked all readers */ - } else { - /* Recheck, a new process may have reused pid */ - if (mdb_reader_pid(env, Pidcheck, pid)) - j = rdrs; - } - } - for (; jme_txns->mti_txnid = env->me_metas[toggle]->mm_txnid; - /* env is hosed if the dead thread was ours */ - if (env->me_txn) { - env->me_flags |= MDB_FATAL_ERROR; - env->me_txn = NULL; - rc = MDB_PANIC; - } - } - DPRINTF(("%cmutex owner died, %s", (rlocked ? 'r' : 'w'), - (rc ? "this process' env is hosed" : "recovering"))); - rc2 = mdb_reader_check0(env, rlocked, NULL); - if (rc2 == 0) - rc2 = pthread_mutex_consistent(mutex); - if (rc || (rc = rc2)) { - DPRINTF(("LOCK_MUTEX recovery failed, %s", mdb_strerror(rc))); - UNLOCK_MUTEX(mutex); - } - } else { -#ifdef _WIN32 - rc = ErrCode(); -#endif - DPRINTF(("LOCK_MUTEX failed, %s", mdb_strerror(rc))); - } - - return rc; -} -#endif /* MDB_ROBUST_SUPPORTED */ -/** @} */ diff --git a/submodules/lmdb/libraries/liblmdb/midl.c b/submodules/lmdb/libraries/liblmdb/midl.c deleted file mode 100644 index 88a3aff10c..0000000000 --- a/submodules/lmdb/libraries/liblmdb/midl.c +++ /dev/null @@ -1,360 +0,0 @@ -/** @file midl.c - * @brief ldap bdb back-end ID List functions */ -/* $OpenLDAP$ */ -/* This work is part of OpenLDAP Software . - * - * Copyright 2000-2014 The OpenLDAP Foundation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#include -#include -#include -#include -#include -#include "midl.h" - -/** @defgroup internal LMDB Internals - * @{ - */ -/** @defgroup idls ID List Management - * @{ - */ -#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) - -unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) -{ - /* - * binary search of id in ids - * if found, returns position of id - * if not found, returns first position greater than id - */ - unsigned base = 0; - unsigned cursor = 1; - int val = 0; - unsigned n = ids[0]; - - while( 0 < n ) { - unsigned pivot = n >> 1; - cursor = base + pivot + 1; - val = CMP( ids[cursor], id ); - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - return cursor; - } - } - - if( val > 0 ) { - ++cursor; - } - return cursor; -} - -#if 0 /* superseded by append/sort */ -int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) -{ - unsigned x, i; - - x = mdb_midl_search( ids, id ); - assert( x > 0 ); - - if( x < 1 ) { - /* internal error */ - return -2; - } - - if ( x <= ids[0] && ids[x] == id ) { - /* duplicate */ - assert(0); - return -1; - } - - if ( ++ids[0] >= MDB_IDL_DB_MAX ) { - /* no room */ - --ids[0]; - return -2; - - } else { - /* insert id */ - for (i=ids[0]; i>x; i--) - ids[i] = ids[i-1]; - ids[x] = id; - } - - return 0; -} -#endif - -MDB_IDL mdb_midl_alloc(int num) -{ - MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); - if (ids) { - *ids++ = num; - *ids = 0; - } - return ids; -} - -void mdb_midl_free(MDB_IDL ids) -{ - if (ids) - free(ids-1); -} - -int mdb_midl_shrink( MDB_IDL *idp ) -{ - MDB_IDL ids = *idp; - if (*(--ids) > MDB_IDL_UM_MAX && - (ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID)))) - { - *ids++ = MDB_IDL_UM_MAX; - *idp = ids; - return 1; - } - return 0; -} - -static int mdb_midl_grow( MDB_IDL *idp, int num ) -{ - MDB_IDL idn = *idp-1; - /* grow it */ - idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); - if (!idn) - return ENOMEM; - *idn++ += num; - *idp = idn; - return 0; -} - -int mdb_midl_need( MDB_IDL *idp, unsigned num ) -{ - MDB_IDL ids = *idp; - num += ids[0]; - if (num > ids[-1]) { - num = (num + num/4 + (256 + 2)) & -256; - if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) - return ENOMEM; - *ids++ = num - 2; - *idp = ids; - } - return 0; -} - -int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) -{ - MDB_IDL ids = *idp; - /* Too big? */ - if (ids[0] >= ids[-1]) { - if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) - return ENOMEM; - ids = *idp; - } - ids[0]++; - ids[ids[0]] = id; - return 0; -} - -int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) -{ - MDB_IDL ids = *idp; - /* Too big? */ - if (ids[0] + app[0] >= ids[-1]) { - if (mdb_midl_grow(idp, app[0])) - return ENOMEM; - ids = *idp; - } - memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); - ids[0] += app[0]; - return 0; -} - -int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) -{ - MDB_ID *ids = *idp, len = ids[0]; - /* Too big? */ - if (len + n > ids[-1]) { - if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) - return ENOMEM; - ids = *idp; - } - ids[0] = len + n; - ids += len; - while (n) - ids[n--] = id++; - return 0; -} - -void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) -{ - MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; - idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ - old_id = idl[j]; - while (i) { - merge_id = merge[i--]; - for (; old_id < merge_id; old_id = idl[--j]) - idl[k--] = old_id; - idl[k--] = merge_id; - } - idl[0] = total; -} - -/* Quicksort + Insertion sort for small arrays */ - -#define SMALL 8 -#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } - -void -mdb_midl_sort( MDB_IDL ids ) -{ - /* Max possible depth of int-indexed tree * 2 items/level */ - int istack[sizeof(int)*CHAR_BIT * 2]; - int i,j,k,l,ir,jstack; - MDB_ID a, itmp; - - ir = (int)ids[0]; - l = 1; - jstack = 0; - for(;;) { - if (ir - l < SMALL) { /* Insertion sort */ - for (j=l+1;j<=ir;j++) { - a = ids[j]; - for (i=j-1;i>=1;i--) { - if (ids[i] >= a) break; - ids[i+1] = ids[i]; - } - ids[i+1] = a; - } - if (jstack == 0) break; - ir = istack[jstack--]; - l = istack[jstack--]; - } else { - k = (l + ir) >> 1; /* Choose median of left, center, right */ - MIDL_SWAP(ids[k], ids[l+1]); - if (ids[l] < ids[ir]) { - MIDL_SWAP(ids[l], ids[ir]); - } - if (ids[l+1] < ids[ir]) { - MIDL_SWAP(ids[l+1], ids[ir]); - } - if (ids[l] < ids[l+1]) { - MIDL_SWAP(ids[l], ids[l+1]); - } - i = l+1; - j = ir; - a = ids[l+1]; - for(;;) { - do i++; while(ids[i] > a); - do j--; while(ids[j] < a); - if (j < i) break; - MIDL_SWAP(ids[i],ids[j]); - } - ids[l+1] = ids[j]; - ids[j] = a; - jstack += 2; - if (ir-i+1 >= j-l) { - istack[jstack] = ir; - istack[jstack-1] = i; - ir = j-1; - } else { - istack[jstack] = j-1; - istack[jstack-1] = l; - l = i; - } - } - } -} - -unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) -{ - /* - * binary search of id in ids - * if found, returns position of id - * if not found, returns first position greater than id - */ - unsigned base = 0; - unsigned cursor = 1; - int val = 0; - unsigned n = (unsigned)ids[0].mid; - - while( 0 < n ) { - unsigned pivot = n >> 1; - cursor = base + pivot + 1; - val = CMP( id, ids[cursor].mid ); - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - return cursor; - } - } - - if( val > 0 ) { - ++cursor; - } - return cursor; -} - -int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) -{ - unsigned x, i; - - x = mdb_mid2l_search( ids, id->mid ); - - if( x < 1 ) { - /* internal error */ - return -2; - } - - if ( x <= ids[0].mid && ids[x].mid == id->mid ) { - /* duplicate */ - return -1; - } - - if ( ids[0].mid >= MDB_IDL_UM_MAX ) { - /* too big */ - return -2; - - } else { - /* insert id */ - ids[0].mid++; - for (i=(unsigned)ids[0].mid; i>x; i--) - ids[i] = ids[i-1]; - ids[x] = *id; - } - - return 0; -} - -int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) -{ - /* Too big? */ - if (ids[0].mid >= MDB_IDL_UM_MAX) { - return -2; - } - ids[0].mid++; - ids[ids[0].mid] = *id; - return 0; -} - -/** @} */ -/** @} */ diff --git a/submodules/lmdb/libraries/liblmdb/midl.h b/submodules/lmdb/libraries/liblmdb/midl.h deleted file mode 100644 index a7f25026ce..0000000000 --- a/submodules/lmdb/libraries/liblmdb/midl.h +++ /dev/null @@ -1,186 +0,0 @@ -/** @file midl.h - * @brief LMDB ID List header file. - * - * This file was originally part of back-bdb but has been - * modified for use in libmdb. Most of the macros defined - * in this file are unused, just left over from the original. - * - * This file is only used internally in libmdb and its definitions - * are not exposed publicly. - */ -/* $OpenLDAP$ */ -/* This work is part of OpenLDAP Software . - * - * Copyright 2000-2014 The OpenLDAP Foundation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#ifndef _MDB_MIDL_H_ -#define _MDB_MIDL_H_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** @defgroup internal LMDB Internals - * @{ - */ - -/** @defgroup idls ID List Management - * @{ - */ - /** A generic unsigned ID number. These were entryIDs in back-bdb. - * Preferably it should have the same size as a pointer. - */ -typedef size_t MDB_ID; - - /** An IDL is an ID List, a sorted array of IDs. The first - * element of the array is a counter for how many actual - * IDs are in the list. In the original back-bdb code, IDLs are - * sorted in ascending order. For libmdb IDLs are sorted in - * descending order. - */ -typedef MDB_ID *MDB_IDL; - -/* IDL sizes - likely should be even bigger - * limiting factors: sizeof(ID), thread stack size - */ -#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ -#define MDB_IDL_DB_SIZE (1< Date: Mon, 5 Sep 2016 23:20:07 +0300 Subject: [PATCH 041/427] no message --- Postbox.xcodeproj/project.pbxproj | 337 +++++++++++++++++- .../xcschemes/PostboxMac.xcscheme | 80 +++++ .../xcschemes/xcschememanagement.plist | 10 + Postbox/Coding.swift | 28 +- Postbox/IpcPipe.swift | 6 +- Postbox/MediaBox.swift | 284 ++++++++------- Postbox/MediaResourceStatus.swift | 6 +- Postbox/MessageHistoryIndexTable.swift | 5 + Postbox/MessageHistoryTable.swift | 60 +++- Postbox/Postbox.swift | 219 ++++++------ .../RandomAccessMediaResourceContext.swift | 6 +- Postbox/SqliteValueBox.swift | 70 +++- Postbox/ViewTracker.swift | 6 +- PostboxMac/Info.plist | 26 ++ PostboxMac/PostboxMac.h | 21 ++ 15 files changed, 888 insertions(+), 276 deletions(-) create mode 100644 Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme create mode 100644 PostboxMac/Info.plist create mode 100644 PostboxMac/PostboxMac.h diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 0cd49c3d09..19595ffea3 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -49,6 +49,75 @@ D0AB0B8C1D65D488002C78E7 /* MessageHistoryHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */; }; D0AB0B8E1D65D49C002C78E7 /* ChatListHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */; }; D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */; }; + D0B418171D7DFAF3004562A4 /* PostboxMac.h in Headers */ = {isa = PBXBuildFile; fileRef = D0B418151D7DFAF3004562A4 /* PostboxMac.h */; settings = {ATTRIBUTES = (Public, ); }; }; + D0B4181C1D7DFDF1004562A4 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; + D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-ambiguous-macro -Wno-conditional-uninitialized -Wno-unused-const-variable -Wno-unused-function -Wno-unreachable-code"; }; }; + D0B4181E1D7DFDF8004562A4 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; + D0B4181F1D7DFDFB004562A4 /* sqlite3.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516411B2D9CEF00AE42E0 /* sqlite3.h */; }; + D0B418201D7DFDFD004562A4 /* sqlite3ext.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */; }; + D0B418211D7DFE05004562A4 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516611B2EC7FE00AE42E0 /* Database.swift */; }; + D0B418221D7DFE0C004562A4 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; + D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8741C5A334100037222 /* SimpleDictionary.swift */; }; + D0B418241D7DFE0C004562A4 /* SimpleSet.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C9DA381C65782500855278 /* SimpleSet.swift */; }; + D0B418251D7DFE0C004562A4 /* RedBlackTree.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */; }; + D0B418261D7DFE0C004562A4 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; + D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510FF1D64A58900A97B8A /* IpcPipe.swift */; }; + D0B418281D7DFE11004562A4 /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; + D0B418291D7DFE11004562A4 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; + D0B4182A1D7DFE11004562A4 /* IntermediateMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */; }; + D0B4182B1D7DFE11004562A4 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; + D0B4182C1D7DFE11004562A4 /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; + D0B4182D1D7DFE11004562A4 /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; + D0B4182E1D7DFE11004562A4 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; + D0B4182F1D7DFE11004562A4 /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; + D0B418301D7DFE16004562A4 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; + D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; + D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; + D0B418331D7DFE1B004562A4 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; + D0B418341D7DFE1B004562A4 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; + D0B418351D7DFE1B004562A4 /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; + D0B418361D7DFE1B004562A4 /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; + D0B418371D7DFE1B004562A4 /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; + D0B418381D7DFE1B004562A4 /* MessageHistoryOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */; }; + D0B418391D7DFE1B004562A4 /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; + D0B4183A1D7DFE1B004562A4 /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; + D0B4183B1D7DFE1B004562A4 /* ChatListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8641C58CB7F00037222 /* ChatListTable.swift */; }; + D0B4183C1D7DFE1B004562A4 /* ChatListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */; }; + D0B4183D1D7DFE1B004562A4 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; + D0B4183E1D7DFE1B004562A4 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; + D0B4183F1D7DFE1B004562A4 /* MetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */; }; + D0B418401D7DFE1B004562A4 /* KeychainTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */; }; + D0B418411D7DFE1B004562A4 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; + D0B418421D7DFE1B004562A4 /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; + D0B418431D7DFE1B004562A4 /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; + D0B418441D7DFE1B004562A4 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; + D0B418451D7DFE1B004562A4 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; + D0B418461D7DFE1B004562A4 /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; + D0B418471D7DFE1B004562A4 /* RatingTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08D451E1D5D2CA700A7428A /* RatingTable.swift */; }; + D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; + D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86A1C59719800037222 /* ChatListView.swift */; }; + D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; + D0B4184B1D7DFE20004562A4 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; + D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */; }; + D0B4184D1D7DFE20004562A4 /* ContactPeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F641D5A457A00A27A2C /* ContactPeersView.swift */; }; + D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */; }; + D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */; }; + D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */; }; + D0B418511D7DFE25004562A4 /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; + D0B418521D7DFE25004562A4 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; + D0B418531D7DFE25004562A4 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; + D0B418541D7DFE25004562A4 /* MediaResourceData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */; }; + D0B418551D7DFE25004562A4 /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; + D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510FA1D63C06E00A97B8A /* PostboxClientId.swift */; }; + D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7811B28ADD000A402D9 /* Postbox.swift */; }; + D0B418581D7DFE29004562A4 /* SeedConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA291C617D39002160FF /* SeedConfiguration.swift */; }; + D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F31D63BA8400A97B8A /* PostboxTransaction.swift */; }; + D0B4185A1D7DFE29004562A4 /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; + D0B4185B1D7DFE2C004562A4 /* MurMurHash32.h in Headers */ = {isa = PBXBuildFile; fileRef = D044E1611B2AD667001EE087 /* MurMurHash32.h */; settings = {ATTRIBUTES = (Public, ); }; }; + D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */ = {isa = PBXBuildFile; fileRef = D044E1621B2AD677001EE087 /* MurMurHash32.m */; }; + D0B4185D1D7DFE35004562A4 /* IpcNotifier.h in Headers */ = {isa = PBXBuildFile; fileRef = D0D511031D64D75200A97B8A /* IpcNotifier.h */; settings = {ATTRIBUTES = (Public, ); }; }; + D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */ = {isa = PBXBuildFile; fileRef = D0D511011D64D73D00A97B8A /* IpcNotifier.mm */; }; + D0B418611D7DFE95004562A4 /* SwiftSignalKitMac.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; @@ -142,6 +211,10 @@ D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHolesView.swift; sourceTree = ""; }; D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatListHolesView.swift; sourceTree = ""; }; D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageIndicesView.swift; sourceTree = ""; }; + D0B418131D7DFAF2004562A4 /* PostboxMac.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = PostboxMac.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + D0B418151D7DFAF3004562A4 /* PostboxMac.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PostboxMac.h; sourceTree = ""; }; + D0B418161D7DFAF3004562A4 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKitMac.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-diblohvjozhgaifjcniwdlixlilx/Build/Products/Debug/SwiftSignalKitMac.framework"; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; @@ -185,6 +258,14 @@ /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ + D0B4180F1D7DFAF2004562A4 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + D0B418611D7DFE95004562A4 /* SwiftSignalKitMac.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; D0E3A7461B28A7E300A402D9 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -253,6 +334,23 @@ name = sqlite.swift; sourceTree = ""; }; + D0B418141D7DFAF3004562A4 /* PostboxMac */ = { + isa = PBXGroup; + children = ( + D0B418151D7DFAF3004562A4 /* PostboxMac.h */, + D0B418161D7DFAF3004562A4 /* Info.plist */, + ); + path = PostboxMac; + sourceTree = ""; + }; + D0B4185F1D7DFE95004562A4 /* Frameworks */ = { + isa = PBXGroup; + children = ( + D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */, + ); + name = Frameworks; + sourceTree = ""; + }; D0E1DE161C5EB06000C7826E /* Tables */ = { isa = PBXGroup; children = ( @@ -342,7 +440,9 @@ D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */, D0E3A74C1B28A7E300A402D9 /* Postbox */, D0E3A7591B28A7E300A402D9 /* PostboxTests */, + D0B418141D7DFAF3004562A4 /* PostboxMac */, D0E3A74B1B28A7E300A402D9 /* Products */, + D0B4185F1D7DFE95004562A4 /* Frameworks */, ); sourceTree = ""; }; @@ -351,6 +451,7 @@ children = ( D0E3A74A1B28A7E300A402D9 /* Postbox.framework */, D0E3A7551B28A7E300A402D9 /* PostboxTests.xctest */, + D0B418131D7DFAF2004562A4 /* PostboxMac.framework */, ); name = Products; sourceTree = ""; @@ -416,6 +517,19 @@ /* End PBXGroup section */ /* Begin PBXHeadersBuildPhase section */ + D0B418101D7DFAF2004562A4 /* Headers */ = { + isa = PBXHeadersBuildPhase; + buildActionMask = 2147483647; + files = ( + D0B418201D7DFDFD004562A4 /* sqlite3ext.h in Headers */, + D0B4185D1D7DFE35004562A4 /* IpcNotifier.h in Headers */, + D0B4185B1D7DFE2C004562A4 /* MurMurHash32.h in Headers */, + D0B4181E1D7DFDF8004562A4 /* SQLite-Bridging.h in Headers */, + D0B4181F1D7DFDFB004562A4 /* sqlite3.h in Headers */, + D0B418171D7DFAF3004562A4 /* PostboxMac.h in Headers */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; D0E3A7471B28A7E300A402D9 /* Headers */ = { isa = PBXHeadersBuildPhase; buildActionMask = 2147483647; @@ -433,6 +547,24 @@ /* End PBXHeadersBuildPhase section */ /* Begin PBXNativeTarget section */ + D0B418121D7DFAF2004562A4 /* PostboxMac */ = { + isa = PBXNativeTarget; + buildConfigurationList = D0B4181B1D7DFAF3004562A4 /* Build configuration list for PBXNativeTarget "PostboxMac" */; + buildPhases = ( + D0B4180E1D7DFAF2004562A4 /* Sources */, + D0B4180F1D7DFAF2004562A4 /* Frameworks */, + D0B418101D7DFAF2004562A4 /* Headers */, + D0B418111D7DFAF2004562A4 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = PostboxMac; + productName = PostboxMac; + productReference = D0B418131D7DFAF2004562A4 /* PostboxMac.framework */; + productType = "com.apple.product-type.framework"; + }; D0E3A7491B28A7E300A402D9 /* Postbox */ = { isa = PBXNativeTarget; buildConfigurationList = D0E3A7601B28A7E300A402D9 /* Build configuration list for PBXNativeTarget "Postbox" */; @@ -480,6 +612,10 @@ LastUpgradeCheck = 0800; ORGANIZATIONNAME = Telegram; TargetAttributes = { + D0B418121D7DFAF2004562A4 = { + CreatedOnToolsVersion = 8.0; + ProvisioningStyle = Manual; + }; D0E3A7491B28A7E300A402D9 = { CreatedOnToolsVersion = 6.3.1; ProvisioningStyle = Manual; @@ -503,11 +639,19 @@ targets = ( D0E3A7491B28A7E300A402D9 /* Postbox */, D0E3A7541B28A7E300A402D9 /* PostboxTests */, + D0B418121D7DFAF2004562A4 /* PostboxMac */, ); }; /* End PBXProject section */ /* Begin PBXResourcesBuildPhase section */ + D0B418111D7DFAF2004562A4 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; D0E3A7481B28A7E300A402D9 /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; @@ -525,6 +669,75 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ + D0B4180E1D7DFAF2004562A4 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + D0B418421D7DFE1B004562A4 /* PeerChatStateTable.swift in Sources */, + D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, + D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, + D0B418211D7DFE05004562A4 /* Database.swift in Sources */, + D0B418251D7DFE0C004562A4 /* RedBlackTree.swift in Sources */, + D0B4183E1D7DFE1B004562A4 /* MediaCleanupTable.swift in Sources */, + D0B418451D7DFE1B004562A4 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, + D0B4183A1D7DFE1B004562A4 /* MessageHistoryUnsentTable.swift in Sources */, + D0B418261D7DFE0C004562A4 /* MappedFile.swift in Sources */, + D0B418511D7DFE25004562A4 /* MediaBox.swift in Sources */, + D0B4181C1D7DFDF1004562A4 /* SQLite-Bridging.m in Sources */, + D0B418351D7DFE1B004562A4 /* MessageHistoryMetadataTable.swift in Sources */, + D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */, + D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */, + D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */, + D0B418291D7DFE11004562A4 /* Message.swift in Sources */, + D0B418361D7DFE1B004562A4 /* MessageHistoryIndexTable.swift in Sources */, + D0B4184D1D7DFE20004562A4 /* ContactPeersView.swift in Sources */, + D0B4182E1D7DFE11004562A4 /* PeerReadState.swift in Sources */, + D0B4183B1D7DFE1B004562A4 /* ChatListTable.swift in Sources */, + D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */, + D0B418531D7DFE25004562A4 /* MediaResource.swift in Sources */, + D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */, + D0B418371D7DFE1B004562A4 /* MessageHistoryTable.swift in Sources */, + D0B4183C1D7DFE1B004562A4 /* ChatListIndexTable.swift in Sources */, + D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, + D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */, + D0B418551D7DFE25004562A4 /* RandomAccessMediaResourceContext.swift in Sources */, + D0B4185A1D7DFE29004562A4 /* ViewTracker.swift in Sources */, + D0B4182D1D7DFE11004562A4 /* ChatListHole.swift in Sources */, + D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */, + D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, + D0B418341D7DFE1B004562A4 /* GlobalMessageIdsTable.swift in Sources */, + D0B4182A1D7DFE11004562A4 /* IntermediateMessage.swift in Sources */, + D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, + D0B418471D7DFE1B004562A4 /* RatingTable.swift in Sources */, + D0B418241D7DFE0C004562A4 /* SimpleSet.swift in Sources */, + D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, + D0B418401D7DFE1B004562A4 /* KeychainTable.swift in Sources */, + D0B418411D7DFE1B004562A4 /* PeerTable.swift in Sources */, + D0B418521D7DFE25004562A4 /* MediaResourceStatus.swift in Sources */, + D0B418461D7DFE1B004562A4 /* OrderStatisticTable.swift in Sources */, + D0B4182C1D7DFE11004562A4 /* MessageHistoryHole.swift in Sources */, + D0B4182B1D7DFE11004562A4 /* Media.swift in Sources */, + D0B418441D7DFE1B004562A4 /* MessageHistoryReadStateTable.swift in Sources */, + D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */, + D0B418431D7DFE1B004562A4 /* ContactTable.swift in Sources */, + D0B418581D7DFE29004562A4 /* SeedConfiguration.swift in Sources */, + D0B4183F1D7DFE1B004562A4 /* MetadataTable.swift in Sources */, + D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */, + D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */, + D0B418391D7DFE1B004562A4 /* MessageHistoryTagsTable.swift in Sources */, + D0B418331D7DFE1B004562A4 /* Table.swift in Sources */, + D0B418381D7DFE1B004562A4 /* MessageHistoryOperation.swift in Sources */, + D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, + D0B4182F1D7DFE11004562A4 /* PeerNameIndexRepresentation.swift in Sources */, + D0B418301D7DFE16004562A4 /* ValueBoxKey.swift in Sources */, + D0B418221D7DFE0C004562A4 /* Coding.swift in Sources */, + D0B418541D7DFE25004562A4 /* MediaResourceData.swift in Sources */, + D0B4183D1D7DFE1B004562A4 /* MessageMediaTable.swift in Sources */, + D0B4184B1D7DFE20004562A4 /* SynchronizePeerReadStatesView.swift in Sources */, + D0B418281D7DFE11004562A4 /* Peer.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; D0E3A7451B28A7E300A402D9 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -652,7 +865,8 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 8.3; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + MACOSX_DEPLOYMENT_TARGET = 10.10; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = iphoneos; TARGETED_DEVICE_FAMILY = "1,2"; @@ -709,6 +923,112 @@ }; name = Hockeyapp; }; + D0B418181D7DFAF3004562A4 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_SUSPICIOUS_MOVES = YES; + CODE_SIGN_IDENTITY = "-"; + COMBINE_HIDPI_IMAGES = YES; + DEBUG_INFORMATION_FORMAT = dwarf; + DEFINES_MODULE = YES; + DEVELOPMENT_TEAM = ""; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + FRAMEWORK_VERSION = A; + INFOPLIST_FILE = PostboxMac/Info.plist; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 10.10; + OTHER_CFLAGS = ( + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_HAS_CODEC=1", + "-DSQLCIPHER_CRYPTO_CC=1", + "-DSQLITE_ENABLE_FTS3", + ); + PRODUCT_BUNDLE_IDENTIFIER = org.telegram.Telegram.PostboxMac; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_VERSION = 3.0; + }; + name = Debug; + }; + D0B418191D7DFAF3004562A4 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_SUSPICIOUS_MOVES = YES; + CODE_SIGN_IDENTITY = "-"; + COMBINE_HIDPI_IMAGES = YES; + DEFINES_MODULE = YES; + DEVELOPMENT_TEAM = ""; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + FRAMEWORK_VERSION = A; + INFOPLIST_FILE = PostboxMac/Info.plist; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 10.10; + OTHER_CFLAGS = ( + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_HAS_CODEC=1", + "-DSQLCIPHER_CRYPTO_CC=1", + "-DSQLITE_ENABLE_FTS3", + ); + PRODUCT_BUNDLE_IDENTIFIER = org.telegram.Telegram.PostboxMac; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; + }; + name = Release; + }; + D0B4181A1D7DFAF3004562A4 /* Hockeyapp */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_SUSPICIOUS_MOVES = YES; + CODE_SIGN_IDENTITY = "-"; + COMBINE_HIDPI_IMAGES = YES; + DEFINES_MODULE = YES; + DEVELOPMENT_TEAM = ""; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + FRAMEWORK_VERSION = A; + INFOPLIST_FILE = PostboxMac/Info.plist; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 10.10; + OTHER_CFLAGS = ( + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_HAS_CODEC=1", + "-DSQLCIPHER_CRYPTO_CC=1", + "-DSQLITE_ENABLE_FTS3", + ); + PRODUCT_BUNDLE_IDENTIFIER = org.telegram.Telegram.PostboxMac; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; + }; + name = Hockeyapp; + }; D0E3A75E1B28A7E300A402D9 /* Debug */ = { isa = XCBuildConfiguration; baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; @@ -748,7 +1068,8 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 8.3; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + MACOSX_DEPLOYMENT_TARGET = 10.10; MTL_ENABLE_DEBUG_INFO = YES; ONLY_ACTIVE_ARCH = YES; SDKROOT = iphoneos; @@ -791,7 +1112,8 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 8.3; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + MACOSX_DEPLOYMENT_TARGET = 10.10; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = iphoneos; TARGETED_DEVICE_FAMILY = "1,2"; @@ -905,6 +1227,15 @@ /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ + D0B4181B1D7DFAF3004562A4 /* Build configuration list for PBXNativeTarget "PostboxMac" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + D0B418181D7DFAF3004562A4 /* Debug */, + D0B418191D7DFAF3004562A4 /* Release */, + D0B4181A1D7DFAF3004562A4 /* Hockeyapp */, + ); + defaultConfigurationIsVisible = 0; + }; D0E3A7441B28A7E300A402D9 /* Build configuration list for PBXProject "Postbox" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme new file mode 100644 index 0000000000..f4a9378c19 --- /dev/null +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist index 9eb93eab1b..82f75996f6 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist @@ -9,6 +9,11 @@ orderHint 1 + PostboxMac.xcscheme + + orderHint + 25 + PostboxTests.xcscheme orderHint @@ -17,6 +22,11 @@ SuppressBuildableAutocreation + D0B418121D7DFAF2004562A4 + + primary + + D0E3A7491B28A7E300A402D9 primary diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index bda4d11f7d..aed292f76b 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -199,6 +199,7 @@ private enum ValueType: Int8 { case ObjectArray = 8 case ObjectDictionary = 9 case Bytes = 10 + case Nil = 11 } public final class Encoder { @@ -233,6 +234,12 @@ public final class Encoder { self.buffer.write(key.utf8Start, offset: 0, length: Int(length)) } + public func encodeNil(forKey key: StaticString) { + self.encodeKey(key) + var type: Int8 = ValueType.Nil.rawValue + self.buffer.write(&type, offset: 0, length: 1) + } + public func encodeInt32(_ value: Int32, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int32.rawValue @@ -481,6 +488,8 @@ public final class Decoder { var length: Int32 = 0 memcpy(&length, bytes + offset, 4) offset += 4 + Int(length) + case .Nil: + break } } @@ -491,24 +500,29 @@ public final class Decoder { let startOffset = offset let keyLength: Int = key.utf8CodeUnitCount - while (offset < maxOffset) - { + while (offset < maxOffset) { let readKeyLength = bytes[offset] + assert(readKeyLength >= 0) offset += 1 offset += Int(readKeyLength) let readValueType = bytes[offset] offset += 1 - if readValueType != valueType.rawValue || keyLength != Int(readKeyLength) || memcmp(bytes + (offset - Int(readKeyLength) - 1), key.utf8Start, keyLength) != 0 { - skipValue(bytes, offset: &offset, length: length, valueType: ValueType(rawValue: readValueType)!) + if keyLength == Int(readKeyLength) && memcmp(bytes + (offset - Int(readKeyLength) - 1), key.utf8Start, keyLength) == 0 { + if readValueType == valueType.rawValue { + return true + } else if readValueType == ValueType.Nil.rawValue { + return false + } else { + skipValue(bytes, offset: &offset, length: length, valueType: ValueType(rawValue: readValueType)!) + } } else { - return true + skipValue(bytes, offset: &offset, length: length, valueType: ValueType(rawValue: readValueType)!) } } - if (startOffset != 0) - { + if (startOffset != 0) { offset = 0 return positionOnKey(bytes, offset: &offset, maxOffset: startOffset, length: length, key: key, valueType: valueType) } diff --git a/Postbox/IpcPipe.swift b/Postbox/IpcPipe.swift index 80e762e468..c3f993b356 100644 --- a/Postbox/IpcPipe.swift +++ b/Postbox/IpcPipe.swift @@ -1,5 +1,9 @@ import Foundation -import SwiftSignalKit +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif func ipcNotify(basePath: String, data: Int64) { DispatchQueue.global(qos: .default).async { diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index d49c6e3e74..92cf175b78 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -1,5 +1,9 @@ import Foundation -import SwiftSignalKit +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif private final class ResourceStatusContext { var status: MediaResourceStatus? @@ -36,6 +40,7 @@ public final class MediaBox { let buffer = WriteBuffer() private let statusQueue = Queue() + private let concurrentQueue = Queue.concurrentDefaultQueue() private let dataQueue = Queue() private var statusContexts: [String: ResourceStatusContext] = [:] @@ -79,64 +84,73 @@ public final class MediaBox { return Signal { subscriber in let disposable = MetaDisposable() - self.statusQueue.async { - let statusContext: ResourceStatusContext - if let current = self.statusContexts[resource.id] { - statusContext = current + self.concurrentQueue.async { + let path = self.pathForId(resource.id) + let currentSize = fileSize(path) + if currentSize >= resource.size { + subscriber.putNext(.Local) + subscriber.putCompletion() } else { - statusContext = ResourceStatusContext() - self.statusContexts[resource.id] = statusContext - } - - let index = statusContext.subscribers.add({ status in - subscriber.putNext(status) - }) - - if let status = statusContext.status { - subscriber.putNext(status) - } else { - self.dataQueue.async { - let status: MediaResourceStatus - - let path = self.pathForId(resource.id) - let currentSize = fileSize(path) - if currentSize >= resource.size { - status = .Local + self.statusQueue.async { + let statusContext: ResourceStatusContext + if let current = self.statusContexts[resource.id] { + statusContext = current } else { - var fetchingData = false - if let dataContext = self.dataContexts[resource.id] { - fetchingData = dataContext.fetchDisposable != nil - } - - if fetchingData { - status = .Fetching(progress: Float(currentSize) / Float(resource.size)) - } else { - status = .Remote - } + statusContext = ResourceStatusContext() + self.statusContexts[resource.id] = statusContext } - self.statusQueue.async { - if let statusContext = self.statusContexts[resource.id] , statusContext.status == nil { - statusContext.status = status + let index = statusContext.subscribers.add({ status in + subscriber.putNext(status) + }) + + if let status = statusContext.status { + subscriber.putNext(status) + } else { + self.dataQueue.async { + let status: MediaResourceStatus - for subscriber in statusContext.subscribers.copyItems() { - subscriber(status) + let path = self.pathForId(resource.id) + let currentSize = fileSize(path) + if currentSize >= resource.size { + status = .Local + } else { + var fetchingData = false + if let dataContext = self.dataContexts[resource.id] { + fetchingData = dataContext.fetchDisposable != nil + } + + if fetchingData { + status = .Fetching(progress: Float(currentSize) / Float(resource.size)) + } else { + status = .Remote + } + } + + self.statusQueue.async { + if let statusContext = self.statusContexts[resource.id] , statusContext.status == nil { + statusContext.status = status + + for subscriber in statusContext.subscribers.copyItems() { + subscriber(status) + } + } } } } + + disposable.set(ActionDisposable { + self.statusQueue.async { + if let current = self.statusContexts[resource.id] { + current.subscribers.remove(index) + if current.subscribers.isEmpty { + self.statusContexts.removeValue(forKey: resource.id) + } + } + } + }) } } - - disposable.set(ActionDisposable { - self.statusQueue.async { - if let current = self.statusContexts[resource.id] { - current.subscribers.remove(index) - if current.subscribers.isEmpty { - self.statusContexts.removeValue(forKey: resource.id) - } - } - } - }) } return disposable @@ -147,7 +161,7 @@ public final class MediaBox { return Signal { subscriber in let disposable = MetaDisposable() - self.dataQueue.async { + self.concurrentQueue.async { let path = self.pathForId(resource.id) let currentSize = fileSize(path) @@ -163,36 +177,79 @@ public final class MediaBox { } subscriber.putCompletion() } else { - let dataContext: ResourceDataContext - if let current = self.dataContexts[resource.id] { - dataContext = current - } else { - dataContext = ResourceDataContext(data: MediaResourceData(path: path, size: currentSize)) - self.dataContexts[resource.id] = dataContext - } - - let index: Bag<(MediaResourceData) -> Void>.Index - if complete { - var checkedForSymlink = false - index = dataContext.completeDataSubscribers.add { data in - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { - checkedForSymlink = true - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) - } - - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) - } else { - subscriber.putNext(data) - } - - if data.size >= resource.size { - subscriber.putCompletion() - } + self.dataQueue.async { + let dataContext: ResourceDataContext + if let current = self.dataContexts[resource.id] { + dataContext = current + } else { + dataContext = ResourceDataContext(data: MediaResourceData(path: path, size: currentSize)) + self.dataContexts[resource.id] = dataContext } - if dataContext.data.size >= resource.size { + let index: Bag<(MediaResourceData) -> Void>.Index + if complete { + var checkedForSymlink = false + index = dataContext.completeDataSubscribers.add { data in + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) + } else { + subscriber.putNext(data) + } + + if data.size >= resource.size { + subscriber.putCompletion() + } + } + + if dataContext.data.size >= resource.size { + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: dataContext.data.size)) + } else { + subscriber.putNext(dataContext.data) + } + } else { + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: 0)) + } else { + subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0)) + } + } + } else { + var checkedForSymlink = false + index = dataContext.progresiveDataSubscribers.add { data in + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) + } else { + subscriber.putNext(data) + } + if data.size >= resource.size { + subscriber.putCompletion() + } + } if let pathExtension = pathExtension { let pathWithExtension = path + ".\(pathExtension)" if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { @@ -204,65 +261,24 @@ public final class MediaBox { } else { subscriber.putNext(dataContext.data) } - } else { - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { - checkedForSymlink = true - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + disposable.set(ActionDisposable { + self.dataQueue.async { + if let dataContext = self.dataContexts[resource.id] { + if complete { + dataContext.completeDataSubscribers.remove(index) + } else { + dataContext.progresiveDataSubscribers.remove(index) + } + + if dataContext.progresiveDataSubscribers.isEmpty && dataContext.completeDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { + self.dataContexts.removeValue(forKey: resource.id) + } } - - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: 0)) - } else { - subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0)) } - } - } else { - var checkedForSymlink = false - index = dataContext.progresiveDataSubscribers.add { data in - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { - checkedForSymlink = true - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) - } - - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) - } else { - subscriber.putNext(data) - } - if data.size >= resource.size { - subscriber.putCompletion() - } - } - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { - checkedForSymlink = true - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) - } - - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: dataContext.data.size)) - } else { - subscriber.putNext(dataContext.data) - } + }) } - - disposable.set(ActionDisposable { - self.dataQueue.async { - if let dataContext = self.dataContexts[resource.id] { - if complete { - dataContext.completeDataSubscribers.remove(index) - } else { - dataContext.progresiveDataSubscribers.remove(index) - } - - if dataContext.progresiveDataSubscribers.isEmpty && dataContext.completeDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { - self.dataContexts.removeValue(forKey: resource.id) - } - } - } - }) } } diff --git a/Postbox/MediaResourceStatus.swift b/Postbox/MediaResourceStatus.swift index c73659ea09..84e844b283 100644 --- a/Postbox/MediaResourceStatus.swift +++ b/Postbox/MediaResourceStatus.swift @@ -1,5 +1,9 @@ import Foundation -import SwiftSignalKit +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif public enum MediaResourceStatus: Equatable { case Remote diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index ac8c1ed840..05ae154959 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -459,6 +459,11 @@ final class MessageHistoryIndexTable: Table { } } + if message.id == upperHole.maxIndex.id { + removedHole = true + self.justRemove(upperHole.maxIndex, operations: &operations) + } + self.justInsertMessage(message, operations: &operations) remainingMessages.remove(at: i) } else { diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index a9bf211be4..180547bbe1 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -1199,7 +1199,13 @@ final class MessageHistoryTable: Table { return Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages) } - func entriesAround(_ index: MessageIndex, count: Int) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { + func entriesAround(_ index: MessageIndex, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { + var indexOperations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.ensureInitialized(index.id.peerId, operations: &indexOperations) + if !indexOperations.isEmpty { + self.processIndexOperations(index.id.peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + var lowerEntries: [IntermediateMessageHistoryEntry] = [] var upperEntries: [IntermediateMessageHistoryEntry] = [] var lower: IntermediateMessageHistoryEntry? @@ -1243,7 +1249,13 @@ final class MessageHistoryTable: Table { return (entries: entries, lower: lower, upper: upper) } - func entriesAround(_ tagMask: MessageTags, index: MessageIndex, count: Int) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { + func entriesAround(_ tagMask: MessageTags, index: MessageIndex, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { + var indexOperations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.ensureInitialized(index.id.peerId, operations: &indexOperations) + if !indexOperations.isEmpty { + self.processIndexOperations(index.id.peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + let (indices, lower, upper) = self.tagsTable.indicesAround(tagMask, index: index, count: count) var entries: [IntermediateMessageHistoryEntry] = [] @@ -1280,8 +1292,12 @@ final class MessageHistoryTable: Table { return (entries, lowerEntry, upperEntry) } - func earlierEntries(_ peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { - + func earlierEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [IntermediateMessageHistoryEntry] { + var indexOperations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.ensureInitialized(peerId, operations: &indexOperations) + if !indexOperations.isEmpty { + self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } var entries: [IntermediateMessageHistoryEntry] = [] let key: ValueBoxKey @@ -1297,7 +1313,13 @@ final class MessageHistoryTable: Table { return entries } - func earlierEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + func earlierEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [IntermediateMessageHistoryEntry] { + var indexOperations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.ensureInitialized(peerId, operations: &indexOperations) + if !indexOperations.isEmpty { + self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + let indices = self.tagsTable.earlierIndices(tagMask, peerId: peerId, index: index, count: count) var entries: [IntermediateMessageHistoryEntry] = [] @@ -1313,7 +1335,13 @@ final class MessageHistoryTable: Table { return entries } - func laterEntries(_ peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + func laterEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [IntermediateMessageHistoryEntry] { + var indexOperations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.ensureInitialized(peerId, operations: &indexOperations) + if !indexOperations.isEmpty { + self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + var entries: [IntermediateMessageHistoryEntry] = [] let key: ValueBoxKey if let index = index { @@ -1328,7 +1356,13 @@ final class MessageHistoryTable: Table { return entries } - func laterEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int) -> [IntermediateMessageHistoryEntry] { + func laterEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [IntermediateMessageHistoryEntry] { + var indexOperations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.ensureInitialized(peerId, operations: &indexOperations) + if !indexOperations.isEmpty { + self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + let indices = self.tagsTable.laterIndices(tagMask, peerId: peerId, index: index, count: count) var entries: [IntermediateMessageHistoryEntry] = [] @@ -1369,7 +1403,11 @@ final class MessageHistoryTable: Table { } func debugList(_ peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { - return self.laterEntries(peerId, index: nil, count: 1000).map({ entry -> RenderedMessageHistoryEntry in + var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + var updatedPeerReadStateOperations: [PeerId : PeerReadStateSynchronizationOperation?] = [:] + + return self.laterEntries(peerId, index: nil, count: 1000, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations).map({ entry -> RenderedMessageHistoryEntry in switch entry { case let .Hole(hole): return .Hole(hole) @@ -1380,7 +1418,11 @@ final class MessageHistoryTable: Table { } func debugList(_ tagMask: MessageTags, peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { - return self.laterEntries(tagMask, peerId: peerId, index: nil, count: 1000).map({ entry -> RenderedMessageHistoryEntry in + var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + var updatedPeerReadStateOperations: [PeerId : PeerReadStateSynchronizationOperation?] = [:] + + return self.laterEntries(tagMask, peerId: peerId, index: nil, count: 1000, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations).map({ entry -> RenderedMessageHistoryEntry in switch entry { case let .Hole(hole): return .Hole(hole) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index ffc7fe4e9f..ec26066924 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -1,7 +1,12 @@ import Foundation -import SwiftSignalKit import sqlcipher +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif + public enum PreloadedMessageHistoryView { case Loading case Preloaded(MessageHistoryView) @@ -221,25 +226,15 @@ public final class Postbox { print("MediaBox path: \(self.basePath + "/media")") - //let _ = try? FileManager.default.removeItem(atPath: self.basePath + "/media") + //let _ = try? FileManager.default.removeItem(atPath: self.basePath) self.mediaBox = MediaBox(basePath: self.basePath + "/media") - /*self.ipcNotificationsDisposable.set((ipcNotifications(basePath: basePath) - |> deliverOn(self.queue)).start(next: { [weak self] updatedTransactionStateVersion in - if let strongSelf = self, strongSelf.valueBox != nil { - if strongSelf.transactionStateVersion < updatedTransactionStateVersion { - strongSelf.modify({ _ -> Void in - }).start() - } - } - }))*/ - self.pipeNotifier = PipeNotifier(basePath: basePath, notify: { [weak self] in if let strongSelf = self { strongSelf.queue.async { if strongSelf.valueBox != nil { - strongSelf.modify({ _ -> Void in + let _ = strongSelf.modify({ _ -> Void in }).start() } } @@ -250,27 +245,23 @@ public final class Postbox { } private func debugSaveState(name: String) { - self.queue.justDispatch({ - let path = self.basePath + name - let _ = try? FileManager.default.removeItem(atPath: path) - do { - try FileManager.default.copyItem(atPath: self.basePath, toPath: path) - } catch (let e) { - print("(Postbox debugSaveState: error \(e))") - } - }) + let path = self.basePath + name + let _ = try? FileManager.default.removeItem(atPath: path) + do { + try FileManager.default.copyItem(atPath: self.basePath, toPath: path) + } catch (let e) { + print("(Postbox debugSaveState: error \(e))") + } } private func debugRestoreState(name: String) { - self.queue.justDispatch({ - let path = self.basePath + name - let _ = try? FileManager.default.removeItem(atPath: self.basePath) - do { - try FileManager.default.copyItem(atPath: path, toPath: self.basePath) - } catch (let e) { - print("(Postbox debugRestoreState: error \(e))") - } - }) + let path = self.basePath + name + let _ = try? FileManager.default.removeItem(atPath: self.basePath) + do { + try FileManager.default.copyItem(atPath: path, toPath: self.basePath) + } catch (let e) { + print("(Postbox debugRestoreState: error \(e))") + } } private func openDatabase() { @@ -300,11 +291,12 @@ public final class Postbox { //#endif - self.valueBox = SqliteValueBox(basePath: self.basePath + "/db") + self.valueBox = SqliteValueBox(basePath: self.basePath + "/db", queue: self.queue) + self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 7 + let currentUserVersion: Int32 = 8 if userVersion != currentUserVersion { self.valueBox.drop() @@ -395,15 +387,56 @@ public final class Postbox { } public func keychainEntryForKey(_ key: String) -> Data? { - return self.keychainTable.get(key) + let metaDisposable = MetaDisposable() + self.keychainOperationsDisposable.add(metaDisposable) + + let semaphore = DispatchSemaphore(value: 0) + + var entry: Data? = nil + let disposable = (self.modify({ modifier -> Data? in + return self.keychainTable.get(key) + }) |> afterDisposed { [weak self, weak metaDisposable] in + if let strongSelf = self, let metaDisposable = metaDisposable { + strongSelf.keychainOperationsDisposable.remove(metaDisposable) + } + }).start(next: { data in + entry = data + semaphore.signal() + }) + metaDisposable.set(disposable) + + semaphore.wait() + return entry } + private var keychainOperationsDisposable = DisposableSet() + public func setKeychainEntryForKey(_ key: String, value: Data) { - self.keychainTable.set(key, value: value) + let metaDisposable = MetaDisposable() + self.keychainOperationsDisposable.add(metaDisposable) + + let disposable = (self.modify({ modifier -> Void in + self.keychainTable.set(key, value: value) + }) |> afterDisposed { [weak self, weak metaDisposable] in + if let strongSelf = self, let metaDisposable = metaDisposable { + strongSelf.keychainOperationsDisposable.remove(metaDisposable) + } + }).start() + metaDisposable.set(disposable) } public func removeKeychainEntryForKey(_ key: String) { - self.keychainTable.remove(key) + let metaDisposable = MetaDisposable() + self.keychainOperationsDisposable.add(metaDisposable) + + let disposable = (self.modify({ modifier -> Void in + self.keychainTable.remove(key) + }) |> afterDisposed { [weak self, weak metaDisposable] in + if let strongSelf = self, let metaDisposable = metaDisposable { + strongSelf.keychainOperationsDisposable.remove(metaDisposable) + } + }).start() + metaDisposable.set(disposable) } fileprivate func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) { @@ -485,9 +518,9 @@ public final class Postbox { private func fetchEarlierHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { let intermediateEntries: [IntermediateMessageHistoryEntry] if let tagMask = tagMask { - intermediateEntries = self.messageHistoryTable.earlierEntries(tagMask, peerId: peerId, index: index, count: count) + intermediateEntries = self.messageHistoryTable.earlierEntries(tagMask, peerId: peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } else { - intermediateEntries = self.messageHistoryTable.earlierEntries(peerId, index: index, count: count) + intermediateEntries = self.messageHistoryTable.earlierEntries(peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } var entries: [MutableMessageHistoryEntry] = [] for entry in intermediateEntries { @@ -508,9 +541,9 @@ public final class Postbox { let intermediateUpper: IntermediateMessageHistoryEntry? if let tagMask = tagMask { - (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(tagMask, index: index, count: count) + (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(tagMask, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } else { - (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(index, count: count) + (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } var entries: [MutableMessageHistoryEntry] = [] @@ -553,9 +586,9 @@ public final class Postbox { private func fetchLaterHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { let intermediateEntries: [IntermediateMessageHistoryEntry] if let tagMask = tagMask { - intermediateEntries = self.messageHistoryTable.laterEntries(tagMask, peerId: peerId, index: index, count: count) + intermediateEntries = self.messageHistoryTable.laterEntries(tagMask, peerId: peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } else { - intermediateEntries = self.messageHistoryTable.laterEntries(peerId, index: index, count: count) + intermediateEntries = self.messageHistoryTable.laterEntries(peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } var entries: [MutableMessageHistoryEntry] = [] for entry in intermediateEntries { @@ -921,82 +954,62 @@ public final class Postbox { } public func contactPeerIdsView() -> Signal { - return Signal { subscriber in - let disposable = MetaDisposable() + return self.modify { modifier -> Signal in + let view = MutableContactPeerIdsView(peerIds: self.contactsTable.get()) + let (index, signal) = self.viewTracker.addContactPeerIdsView(view) - self.queue.async { - let view = MutableContactPeerIdsView(peerIds: self.contactsTable.get()) - let (index, signal) = self.viewTracker.addContactPeerIdsView(view) - subscriber.putNext(ContactPeerIdsView(view)) - - let updatedViewDisposable = signal.start(next: { view in - subscriber.putNext(view) - }) - - disposable.set(ActionDisposable { - updatedViewDisposable.dispose() - self.viewTracker.removeContactPeerIdsView(index) - }) + return (.single(ContactPeerIdsView(view)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeContactPeerIdsView(index) + } + } } - - return disposable - } + } |> switchToLatest } public func contactPeersView(index: PeerNameIndex, accountPeerId: PeerId) -> Signal { - return Signal { subscriber in - let disposable = MetaDisposable() + return self.modify { modifier -> Signal in + var peers: [PeerId: Peer] = [:] - self.queue.async { - var peers: [PeerId: Peer] = [:] - for peerId in self.contactsTable.get() { - if let peer = self.peerTable.get(peerId) { - peers[peerId] = peer - } + for peerId in self.contactsTable.get() { + if let peer = self.peerTable.get(peerId) { + peers[peerId] = peer } - - let view = MutableContactPeersView(peers: peers, index: index, accountPeer: self.peerTable.get(accountPeerId)) - let (index, signal) = self.viewTracker.addContactPeersView(view) - subscriber.putNext(ContactPeersView(view)) - - let updatedViewDisposable = signal.start(next: { view in - subscriber.putNext(view) - }) - - disposable.set(ActionDisposable { - updatedViewDisposable.dispose() - self.viewTracker.removeContactPeersView(index) - }) } - return disposable - } - } - - public func searchContacts(query: String) -> Signal<[Peer], NoError> { - return Signal { subscriber in - let disposable = MetaDisposable() + let view = MutableContactPeersView(peers: peers, index: index, accountPeer: self.peerTable.get(accountPeerId)) + let (index, signal) = self.viewTracker.addContactPeersView(view) - self.queue.async { - var peers: [Peer] = [] - for peerId in self.contactsTable.get() { - if let peer = self.peerTable.get(peerId) { - if peer.indexName.match(query: query) { - peers.append(peer) + return (.single(ContactPeersView(view)) + |> then(signal)) + |> afterDisposed { + [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeContactPeersView(index) } } } - - peers.sort(by: { $0.indexName.indexName(.lastNameFirst) < $1.indexName.indexName(.lastNameFirst) }) - - subscriber.putNext(peers) - - disposable.set(ActionDisposable { - }) + } |> switchToLatest + } + + public func searchContacts(query: String) -> Signal<[Peer], NoError> { + return self.modify { modifier -> Signal<[Peer], NoError> in + var peers: [Peer] = [] + for peerId in self.contactsTable.get() { + if let peer = self.peerTable.get(peerId) { + if peer.indexName.match(query: query) { + peers.append(peer) + } + } } - return disposable - } + peers.sort(by: { $0.indexName.indexName(.lastNameFirst) < $1.indexName.indexName(.lastNameFirst) }) + return .single(peers) + } |> switchToLatest } public func peerWithId(_ id: PeerId) -> Signal { diff --git a/Postbox/RandomAccessMediaResourceContext.swift b/Postbox/RandomAccessMediaResourceContext.swift index 8808e6abf7..f4bfd3be03 100644 --- a/Postbox/RandomAccessMediaResourceContext.swift +++ b/Postbox/RandomAccessMediaResourceContext.swift @@ -1,5 +1,9 @@ import Foundation -import SwiftSignalKit +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif public struct RandomAccessResourceStoreRange { fileprivate let offset: Int diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index b492813c03..79d90bdf20 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -1,6 +1,10 @@ import Foundation import sqlcipher -import SwiftSignalKit +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif private struct SqlitePreparedStatement { let statement: OpaquePointer? @@ -87,8 +91,11 @@ public final class SqliteValueBox: ValueBox { private let checkpoints = MetaDisposable() - public init(basePath: String) { + private let queue: Queue + + public init(basePath: String, queue: Queue) { self.basePath = basePath + self.queue = queue self.database = self.openDatabase() } @@ -98,6 +105,8 @@ public final class SqliteValueBox: ValueBox { } private func openDatabase() -> Database { + assert(self.queue.isCurrent()) + checkpoints.set(nil) lock.lock() @@ -105,23 +114,25 @@ public final class SqliteValueBox: ValueBox { let path = basePath + "/db_sqlite" let database = Database(path) - database.adjustChunkSize() - database.execute("PRAGMA page_size=1024") database.execute("PRAGMA cache_size=-2097152") database.execute("PRAGMA synchronous=NORMAL") database.execute("PRAGMA journal_mode=WAL") database.execute("PRAGMA temp_store=MEMORY") - database.execute("PRAGMA wal_autocheckpoint=200") + //database.execute("PRAGMA wal_autocheckpoint=200") database.execute("PRAGMA journal_size_limit=1536") - /*var statement: OpaquePointer? = nil + var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, "PRAGMA integrity_check", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - if preparedStatement.step() { - let value = preparedStatement.stringAt(0) - print("integrity_check: \(value)") + while preparedStatement.step() { + let value = preparedStatement.valueAt(0) + let text = String(data: Data(bytes: value.memory.assumingMemoryBound(to: UInt8.self), count: value.length), encoding: .utf8) + print("integrity_check: \(text ?? "")") + assert(text == "ok") + //let value = preparedStatement.stringAt(0) + //print("integrity_check: \(value)") } - preparedStatement.destroy()*/ + preparedStatement.destroy() sqlite3_busy_timeout(database.handle, 10000000) @@ -138,11 +149,13 @@ public final class SqliteValueBox: ValueBox { checkpoints.set((Signal.single(Void()) |> delay(10.0, queue: Queue.concurrentDefaultQueue()) |> restart).start(next: { [weak self] _ in if let strongSelf = self , strongSelf.database != nil { - strongSelf.lock.lock() - var nLog: Int32 = 0 - var nFrames: Int32 = 0 - sqlite3_wal_checkpoint_v2(strongSelf.database.handle, nil, SQLITE_CHECKPOINT_PASSIVE, &nLog, &nFrames) - strongSelf.lock.unlock() + strongSelf.queue.async { + strongSelf.lock.lock() + var nLog: Int32 = 0 + var nFrames: Int32 = 0 + sqlite3_wal_checkpoint_v2(strongSelf.database.handle, nil, SQLITE_CHECKPOINT_PASSIVE, &nLog, &nFrames) + strongSelf.lock.unlock() + } //print("(SQLite WAL size \(nLog) removed \(nFrames))") } })) @@ -160,16 +173,19 @@ public final class SqliteValueBox: ValueBox { } public func begin() { + assert(self.queue.isCurrent()) self.database.execute("BEGIN IMMEDIATE") } public func commit() { + assert(self.queue.isCurrent()) let startTime = CFAbsoluteTimeGetCurrent() self.database.execute("COMMIT") self.commitTime += CFAbsoluteTimeGetCurrent() - startTime } private func getUserVersion(_ database: Database) -> Int64 { + assert(self.queue.isCurrent()) var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, "PRAGMA user_version", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) @@ -180,6 +196,7 @@ public final class SqliteValueBox: ValueBox { } private func listTables(_ database: Database) -> [Int64] { + assert(self.queue.isCurrent()) var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, "SELECT name FROM __meta_tables", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) @@ -193,6 +210,7 @@ public final class SqliteValueBox: ValueBox { } private func getStatement(_ table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.getStatements[table] { @@ -213,6 +231,7 @@ public final class SqliteValueBox: ValueBox { } private func rangeKeyAscStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.rangeKeyAscStatementsLimit[table] { @@ -233,7 +252,9 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func rangeKeyAscStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + private func rangeKeyAscStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> + SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.rangeKeyAscStatementsNoLimit[table] { @@ -277,6 +298,7 @@ public final class SqliteValueBox: ValueBox { } private func rangeKeyDescStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.rangeKeyDescStatementsNoLimit[table] { @@ -298,6 +320,7 @@ public final class SqliteValueBox: ValueBox { } private func rangeValueAscStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.rangeValueAscStatementsLimit[table] { @@ -319,6 +342,7 @@ public final class SqliteValueBox: ValueBox { } private func rangeValueAscStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.rangeValueAscStatementsNoLimit[table] { @@ -340,6 +364,7 @@ public final class SqliteValueBox: ValueBox { } private func rangeValueDescStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.rangeValueDescStatementsLimit[table] { @@ -362,6 +387,7 @@ public final class SqliteValueBox: ValueBox { } private func rangeValueDescStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.rangeKeyDescStatementsNoLimit[table] { @@ -383,6 +409,7 @@ public final class SqliteValueBox: ValueBox { } private func existsStatement(_ table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.existsStatements[table] { @@ -403,6 +430,7 @@ public final class SqliteValueBox: ValueBox { } private func updateStatement(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.updateStatements[table] { @@ -424,6 +452,7 @@ public final class SqliteValueBox: ValueBox { } private func insertStatement(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.insertStatements[table] { @@ -449,6 +478,7 @@ public final class SqliteValueBox: ValueBox { } private func deleteStatement(_ table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement if let statement = self.deleteStatements[table] { @@ -469,6 +499,7 @@ public final class SqliteValueBox: ValueBox { } public func get(_ table: Int32, key: ValueBoxKey) -> ReadBuffer? { + assert(self.queue.isCurrent()) let startTime = CFAbsoluteTimeGetCurrent() if self.tables.contains(table) { let statement = self.getStatement(table, key: key) @@ -491,6 +522,7 @@ public final class SqliteValueBox: ValueBox { } public func exists(_ table: Int32, key: ValueBoxKey) -> Bool { + assert(self.queue.isCurrent()) if let _ = self.get(table, key: key) { return true } @@ -498,6 +530,7 @@ public final class SqliteValueBox: ValueBox { } public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, values: @noescape(ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { + assert(self.queue.isCurrent()) if start == end { return } @@ -545,6 +578,7 @@ public final class SqliteValueBox: ValueBox { } public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: @noescape(ValueBoxKey) -> Bool, limit: Int) { + assert(self.queue.isCurrent()) if self.tables.contains(table) { let statement: SqlitePreparedStatement @@ -587,6 +621,7 @@ public final class SqliteValueBox: ValueBox { } public func set(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) { + assert(self.queue.isCurrent()) if !self.tables.contains(table) { self.database.execute("CREATE TABLE t\(table) (key BLOB, value BLOB)") self.database.execute("CREATE INDEX t\(table)_key ON t\(table) (key)") @@ -619,6 +654,7 @@ public final class SqliteValueBox: ValueBox { } public func remove(_ table: Int32, key: ValueBoxKey) { + assert(self.queue.isCurrent()) if self.tables.contains(table) { let startTime = CFAbsoluteTimeGetCurrent() @@ -633,6 +669,7 @@ public final class SqliteValueBox: ValueBox { } private func clearStatements() { + assert(self.queue.isCurrent()) for (_, statement) in self.getStatements { statement.destroy() } @@ -700,6 +737,7 @@ public final class SqliteValueBox: ValueBox { } public func drop() { + assert(self.queue.isCurrent()) self.clearStatements() self.lock.lock() diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 38c5dbd57c..d994ce8436 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -1,5 +1,9 @@ import Foundation -import SwiftSignalKit +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif public enum ViewUpdateType { case InitialUnread(MessageIndex) diff --git a/PostboxMac/Info.plist b/PostboxMac/Info.plist new file mode 100644 index 0000000000..ab41b55afe --- /dev/null +++ b/PostboxMac/Info.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + FMWK + CFBundleShortVersionString + 1.0 + CFBundleVersion + $(CURRENT_PROJECT_VERSION) + NSHumanReadableCopyright + Copyright © 2016 Telegram. All rights reserved. + NSPrincipalClass + + + diff --git a/PostboxMac/PostboxMac.h b/PostboxMac/PostboxMac.h new file mode 100644 index 0000000000..c1c82e15e8 --- /dev/null +++ b/PostboxMac/PostboxMac.h @@ -0,0 +1,21 @@ +// +// PostboxMac.h +// PostboxMac +// +// Created by Peter on 9/5/16. +// Copyright © 2016 Telegram. All rights reserved. +// + +#import + +//! Project version number for PostboxMac. +FOUNDATION_EXPORT double PostboxMacVersionNumber; + +//! Project version string for PostboxMac. +FOUNDATION_EXPORT const unsigned char PostboxMacVersionString[]; + +// In this header, you should import all the public headers of your framework using statements like #import + + +#import +#import From 51377d919fca12b73466a0ddaf0cb198389c5204 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 5 Sep 2016 23:51:46 +0300 Subject: [PATCH 042/427] no message --- Postbox.xcodeproj/project.pbxproj | 21 +- Postbox/Database.swift | 115 ++ submodules/lmdb/libraries/liblmdb/.gitignore | 20 - submodules/lmdb/libraries/liblmdb/CHANGES | 112 -- submodules/lmdb/libraries/liblmdb/COPYRIGHT | 20 - submodules/lmdb/libraries/liblmdb/Doxyfile | 1631 ----------------- submodules/lmdb/libraries/liblmdb/LICENSE | 47 - submodules/lmdb/libraries/liblmdb/Makefile | 97 - submodules/lmdb/libraries/liblmdb/mdb_copy.1 | 54 - submodules/lmdb/libraries/liblmdb/mdb_copy.c | 82 - submodules/lmdb/libraries/liblmdb/mdb_dump.1 | 75 - submodules/lmdb/libraries/liblmdb/mdb_dump.c | 317 ---- submodules/lmdb/libraries/liblmdb/mdb_load.1 | 77 - submodules/lmdb/libraries/liblmdb/mdb_load.c | 452 ----- submodules/lmdb/libraries/liblmdb/mdb_stat.1 | 64 - submodules/lmdb/libraries/liblmdb/mdb_stat.c | 263 --- submodules/lmdb/libraries/liblmdb/mtest.c | 173 -- submodules/lmdb/libraries/liblmdb/mtest2.c | 123 -- submodules/lmdb/libraries/liblmdb/mtest3.c | 133 -- submodules/lmdb/libraries/liblmdb/mtest4.c | 168 -- submodules/lmdb/libraries/liblmdb/mtest5.c | 135 -- submodules/lmdb/libraries/liblmdb/mtest6.c | 137 -- .../lmdb/libraries/liblmdb/sample-bdb.txt | 73 - .../lmdb/libraries/liblmdb/sample-mdb.txt | 62 - submodules/lmdb/libraries/liblmdb/tooltag | 22 - 25 files changed, 122 insertions(+), 4351 deletions(-) create mode 100644 Postbox/Database.swift delete mode 100644 submodules/lmdb/libraries/liblmdb/.gitignore delete mode 100644 submodules/lmdb/libraries/liblmdb/CHANGES delete mode 100644 submodules/lmdb/libraries/liblmdb/COPYRIGHT delete mode 100644 submodules/lmdb/libraries/liblmdb/Doxyfile delete mode 100644 submodules/lmdb/libraries/liblmdb/LICENSE delete mode 100644 submodules/lmdb/libraries/liblmdb/Makefile delete mode 100644 submodules/lmdb/libraries/liblmdb/mdb_copy.1 delete mode 100644 submodules/lmdb/libraries/liblmdb/mdb_copy.c delete mode 100644 submodules/lmdb/libraries/liblmdb/mdb_dump.1 delete mode 100644 submodules/lmdb/libraries/liblmdb/mdb_dump.c delete mode 100644 submodules/lmdb/libraries/liblmdb/mdb_load.1 delete mode 100644 submodules/lmdb/libraries/liblmdb/mdb_load.c delete mode 100644 submodules/lmdb/libraries/liblmdb/mdb_stat.1 delete mode 100644 submodules/lmdb/libraries/liblmdb/mdb_stat.c delete mode 100644 submodules/lmdb/libraries/liblmdb/mtest.c delete mode 100644 submodules/lmdb/libraries/liblmdb/mtest2.c delete mode 100644 submodules/lmdb/libraries/liblmdb/mtest3.c delete mode 100644 submodules/lmdb/libraries/liblmdb/mtest4.c delete mode 100644 submodules/lmdb/libraries/liblmdb/mtest5.c delete mode 100644 submodules/lmdb/libraries/liblmdb/mtest6.c delete mode 100644 submodules/lmdb/libraries/liblmdb/sample-bdb.txt delete mode 100644 submodules/lmdb/libraries/liblmdb/sample-mdb.txt delete mode 100644 submodules/lmdb/libraries/liblmdb/tooltag diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 19595ffea3..2f8f41cb2a 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -31,7 +31,6 @@ D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-ambiguous-macro -Wno-conditional-uninitialized -Wno-unused-const-variable -Wno-unused-function -Wno-unreachable-code"; }; }; D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516411B2D9CEF00AE42E0 /* sqlite3.h */; }; D07516461B2D9CEF00AE42E0 /* sqlite3ext.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */; }; - D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516611B2EC7FE00AE42E0 /* Database.swift */; }; D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; @@ -43,6 +42,8 @@ D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; + D098C6F11D7E11E9007784E4 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = D098C6F01D7E11E9007784E4 /* Database.swift */; }; + D098C6F21D7E1201007784E4 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = D098C6F01D7E11E9007784E4 /* Database.swift */; }; D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; @@ -55,7 +56,6 @@ D0B4181E1D7DFDF8004562A4 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D0B4181F1D7DFDFB004562A4 /* sqlite3.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516411B2D9CEF00AE42E0 /* sqlite3.h */; }; D0B418201D7DFDFD004562A4 /* sqlite3ext.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */; }; - D0B418211D7DFE05004562A4 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07516611B2EC7FE00AE42E0 /* Database.swift */; }; D0B418221D7DFE0C004562A4 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8741C5A334100037222 /* SimpleDictionary.swift */; }; D0B418241D7DFE0C004562A4 /* SimpleSet.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C9DA381C65782500855278 /* SimpleSet.swift */; }; @@ -193,7 +193,6 @@ D07516411B2D9CEF00AE42E0 /* sqlite3.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3.h; sourceTree = ""; }; D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sqlite3ext.h; sourceTree = ""; }; D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Postbox.xcconfig; path = Postbox/Config/Postbox.xcconfig; sourceTree = ""; }; - D07516611B2EC7FE00AE42E0 /* Database.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Database.swift; path = submodules/sqlite.swift/SQLite/Database.swift; sourceTree = SOURCE_ROOT; }; D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; @@ -205,6 +204,7 @@ D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = ""; }; D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; + D098C6F01D7E11E9007784E4 /* Database.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Database.swift; sourceTree = ""; }; D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessMediaResourceContext.swift; sourceTree = ""; }; D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTable.swift; sourceTree = ""; }; D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; @@ -300,7 +300,7 @@ D07515FC1B2C44A200AE42E0 /* thirdparty */ = { isa = PBXGroup; children = ( - D07516731B2EC8C700AE42E0 /* sqlite.swift */, + D098C6F01D7E11E9007784E4 /* Database.swift */, ); name = thirdparty; sourceTree = ""; @@ -326,14 +326,6 @@ path = sqlcipher; sourceTree = ""; }; - D07516731B2EC8C700AE42E0 /* sqlite.swift */ = { - isa = PBXGroup; - children = ( - D07516611B2EC7FE00AE42E0 /* Database.swift */, - ); - name = sqlite.swift; - sourceTree = ""; - }; D0B418141D7DFAF3004562A4 /* PostboxMac */ = { isa = PBXGroup; children = ( @@ -676,7 +668,6 @@ D0B418421D7DFE1B004562A4 /* PeerChatStateTable.swift in Sources */, D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, - D0B418211D7DFE05004562A4 /* Database.swift in Sources */, D0B418251D7DFE0C004562A4 /* RedBlackTree.swift in Sources */, D0B4183E1D7DFE1B004562A4 /* MediaCleanupTable.swift in Sources */, D0B418451D7DFE1B004562A4 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, @@ -700,6 +691,7 @@ D0B4183C1D7DFE1B004562A4 /* ChatListIndexTable.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */, + D098C6F21D7E1201007784E4 /* Database.swift in Sources */, D0B418551D7DFE25004562A4 /* RandomAccessMediaResourceContext.swift in Sources */, D0B4185A1D7DFE29004562A4 /* ViewTracker.swift in Sources */, D0B4182D1D7DFE11004562A4 /* ChatListHole.swift in Sources */, @@ -768,6 +760,7 @@ D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, D0C9DA391C65782500855278 /* SimpleSet.swift in Sources */, + D098C6F11D7E11E9007784E4 /* Database.swift in Sources */, D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */, D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, @@ -801,7 +794,6 @@ D0D510F41D63BA8400A97B8A /* PostboxTransaction.swift in Sources */, D0D510FB1D63C06E00A97B8A /* PostboxClientId.swift in Sources */, D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */, - D075166A1B2EC7FE00AE42E0 /* Database.swift in Sources */, D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, D0D511001D64A58900A97B8A /* IpcPipe.swift in Sources */, ); @@ -1235,6 +1227,7 @@ D0B4181A1D7DFAF3004562A4 /* Hockeyapp */, ); defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; }; D0E3A7441B28A7E300A402D9 /* Build configuration list for PBXProject "Postbox" */ = { isa = XCConfigurationList; diff --git a/Postbox/Database.swift b/Postbox/Database.swift new file mode 100644 index 0000000000..b0450cd1ac --- /dev/null +++ b/Postbox/Database.swift @@ -0,0 +1,115 @@ +// +// SQLite.swift +// https://github.com/stephencelis/SQLite.swift +// Copyright (c) 2014-2015 Stephen Celis. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + +import sqlcipher + +/// A connection (handle) to SQLite. +public final class Database { + + /// The location of a SQLite database. + public enum Location { + + /// An in-memory database (equivalent to `.URI(":memory:")`). + /// + /// See: + case InMemory + + /// A temporary, file-backed database (equivalent to `.URI("")`). + /// + /// See: + case Temporary + + /// A database located at the given URI filename (or path). + /// + /// See: + /// + /// - parameter filename: A URI filename + case URI(String) + + } + + internal var handle: OpaquePointer? = nil + + /// Whether or not the database was opened in a read-only state. + public var readonly: Bool { return sqlite3_db_readonly(handle, nil) == 1 } + + /// Initializes a new connection to a database. + /// + /// - parameter location: The location of the database. Creates a new database if + /// it doesn’t already exist (unless in read-only mode). + /// + /// Default: `.InMemory`. + /// + /// - parameter readonly: Whether or not to open the database in a read-only + /// state. + /// + /// Default: `false`. + /// + /// - returns: A new database connection. + public init(_ location: Location = .InMemory, readonly: Bool = false) { + let flags = readonly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE + sqlite3_open_v2(location.description, &self.handle, flags | SQLITE_OPEN_FULLMUTEX, nil) + } + + /// Initializes a new connection to a database. + /// + /// - parameter filename: The location of the database. Creates a new database if + /// it doesn’t already exist (unless in read-only mode). + /// + /// - parameter readonly: Whether or not to open the database in a read-only + /// state. + /// + /// Default: `false`. + /// + /// - returns: A new database connection. + public convenience init(_ filename: String, readonly: Bool = false) { + self.init(.URI(filename), readonly: readonly) + } + + deinit { sqlite3_close(self.handle) } // sqlite3_close_v2 in Yosemite/iOS 8? + + // MARK: - Execute + + /// Executes a batch of SQL statements. + /// + /// - parameter SQL: A batch of zero or more semicolon-separated SQL statements. + public func execute(_ SQL: String) { + sqlite3_exec(self.handle, SQL, nil, nil, nil) + } +} + +extension Database.Location: CustomStringConvertible { + + public var description: String { + switch self { + case .InMemory: + return ":memory:" + case .Temporary: + return "" + case .URI(let URI): + return URI + } + } + +} diff --git a/submodules/lmdb/libraries/liblmdb/.gitignore b/submodules/lmdb/libraries/liblmdb/.gitignore deleted file mode 100644 index f3277afe42..0000000000 --- a/submodules/lmdb/libraries/liblmdb/.gitignore +++ /dev/null @@ -1,20 +0,0 @@ -mtest -mtest[23456] -testdb -mdb_copy -mdb_stat -*.[ao] -*.so -*.exe -*[~#] -*.bak -*.orig -*.rej -*.gcov -*.gcda -*.gcno -core -core.* -valgrind.* -man/ -html/ diff --git a/submodules/lmdb/libraries/liblmdb/CHANGES b/submodules/lmdb/libraries/liblmdb/CHANGES deleted file mode 100644 index 34049ba7ac..0000000000 --- a/submodules/lmdb/libraries/liblmdb/CHANGES +++ /dev/null @@ -1,112 +0,0 @@ -LMDB 0.9 Change Log - -LMDB 0.9.14 Release (2014/09/20) - Fix to support 64K page size (ITS#7713) - Fix to persist decreased as well as increased mapsizes (ITS#7789) - Fix cursor bug when deleting last node of a DUPSORT key - Fix mdb_env_info to return FIXEDMAP address - Fix ambiguous error code from writing to closed DBI (ITS#7825) - Fix mdb_copy copying past end of file (ITS#7886) - Fix cursor bugs from page_merge/rebalance - Fix to dirty fewer pages in deletes (mdb_page_loose()) - Fix mdb_dbi_open creating subDBs (ITS#7917) - Fix mdb_cursor_get(_DUP) with single value (ITS#7913) - Fix Windows compat issues in mtests (ITS#7879) - Add compacting variant of mdb_copy - Add BigEndian integer key compare code - Add mdb_dump/mdb_load utilities - -LMDB 0.9.13 Release (2014/06/18) - Fix mdb_page_alloc unlimited overflow page search - Documentation - Re-fix MDB_CURRENT doc (ITS#7793) - Fix MDB_GET_MULTIPLE/MDB_NEXT_MULTIPLE doc - -LMDB 0.9.12 Release (2014/06/13) - Fix MDB_GET_BOTH regression (ITS#7875,#7681) - Fix MDB_MULTIPLE writing multiple keys (ITS#7834) - Fix mdb_rebalance (ITS#7829) - Fix mdb_page_split (ITS#7815) - Fix md_entries count (ITS#7861,#7828,#7793) - Fix MDB_CURRENT (ITS#7793) - Fix possible crash on Windows DLL detach - Misc code cleanup - Documentation - mdb_cursor_put: cursor moves on error (ITS#7771) - - -LMDB 0.9.11 Release (2014/01/15) - Add mdb_env_set_assert() (ITS#7775) - Fix: invalidate txn on page allocation errors (ITS#7377) - Fix xcursor tracking in mdb_cursor_del0() (ITS#7771) - Fix corruption from deletes (ITS#7756) - Fix Windows/MSVC build issues - Raise safe limit of max MDB_MAXKEYSIZE - Misc code cleanup - Documentation - Remove spurious note about non-overlapping flags (ITS#7665) - -LMDB 0.9.10 Release (2013/11/12) - Add MDB_NOMEMINIT option - Fix mdb_page_split() again (ITS#7589) - Fix MDB_NORDAHEAD definition (ITS#7734) - Fix mdb_cursor_del() positioning (ITS#7733) - Partial fix for larger page sizes (ITS#7713) - Fix Windows64/MSVC build issues - -LMDB 0.9.9 Release (2013/10/24) - Add mdb_env_get_fd() - Add MDB_NORDAHEAD option - Add MDB_NOLOCK option - Avoid wasting space in mdb_page_split() (ITS#7589) - Fix mdb_page_merge() cursor fixup (ITS#7722) - Fix mdb_cursor_del() on last delete (ITS#7718) - Fix adding WRITEMAP on existing env (ITS#7715) - Fix nested txns (ITS#7515) - Fix mdb_env_copy() O_DIRECT bug (ITS#7682) - Fix mdb_cursor_set(SET_RANGE) return code (ITS#7681) - Fix mdb_rebalance() cursor fixup (ITS#7701) - Misc code cleanup - Documentation - Note that by default, readers need write access - - -LMDB 0.9.8 Release (2013/09/09) - Allow mdb_env_set_mapsize() on an open environment - Fix mdb_dbi_flags() (ITS#7672) - Fix mdb_page_unspill() in nested txns - Fix mdb_cursor_get(CURRENT|NEXT) after a delete - Fix mdb_cursor_get(DUP) to always return key (ITS#7671) - Fix mdb_cursor_del() to always advance to next item (ITS#7670) - Fix mdb_cursor_set(SET_RANGE) for tree with single page (ITS#7681) - Fix mdb_env_copy() retry open if O_DIRECT fails (ITS#7682) - Tweak mdb_page_spill() to be less aggressive - Documentation - Update caveats since mdb_reader_check() added in 0.9.7 - -LMDB 0.9.7 Release (2013/08/17) - Don't leave stale lockfile on failed RDONLY open (ITS#7664) - Fix mdb_page_split() ref beyond cursor depth - Fix read txn data race (ITS#7635) - Fix mdb_rebalance (ITS#7536, #7538) - Fix mdb_drop() (ITS#7561) - Misc DEBUG macro fixes - Add MDB_NOTLS envflag - Add mdb_env_copyfd() - Add mdb_txn_env() (ITS#7660) - Add mdb_dbi_flags() (ITS#7661) - Add mdb_env_get_maxkeysize() - Add mdb_env_reader_list()/mdb_env_reader_check() - Add mdb_page_spill/unspill, remove hard txn size limit - Use shorter names for semaphores (ITS#7615) - Build - Fix install target (ITS#7656) - Documentation - Misc updates for cursors, DB handles, data lifetime - -LMDB 0.9.6 Release (2013/02/25) - Many fixes/enhancements - -LMDB 0.9.5 Release (2012/11/30) - Renamed from libmdb to liblmdb - Many fixes/enhancements diff --git a/submodules/lmdb/libraries/liblmdb/COPYRIGHT b/submodules/lmdb/libraries/liblmdb/COPYRIGHT deleted file mode 100644 index e0eb484c7a..0000000000 --- a/submodules/lmdb/libraries/liblmdb/COPYRIGHT +++ /dev/null @@ -1,20 +0,0 @@ -Copyright 2011-2014 Howard Chu, Symas Corp. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted only as authorized by the OpenLDAP -Public License. - -A copy of this license is available in the file LICENSE in the -top-level directory of the distribution or, alternatively, at -. - -OpenLDAP is a registered trademark of the OpenLDAP Foundation. - -Individual files and/or contributed packages may be copyright by -other parties and/or subject to additional restrictions. - -This work also contains materials derived from public sources. - -Additional information about OpenLDAP can be obtained at -. diff --git a/submodules/lmdb/libraries/liblmdb/Doxyfile b/submodules/lmdb/libraries/liblmdb/Doxyfile deleted file mode 100644 index 92d17b09eb..0000000000 --- a/submodules/lmdb/libraries/liblmdb/Doxyfile +++ /dev/null @@ -1,1631 +0,0 @@ -# Doxyfile 1.7.1 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all -# text before the first occurrence of this tag. Doxygen uses libiconv (or the -# iconv built into libc) for the transcoding. See -# http://www.gnu.org/software/libiconv for the list of possible encodings. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = LMDB - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of -# source files, where putting all generated files in the same directory would -# otherwise cause performance problems for the file system. - -CREATE_SUBDIRS = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, -# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, -# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English -# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, -# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak, -# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is -# used as the annotated text. Otherwise, the brief description is used as-is. -# If left blank, the following values are used ("$name" is automatically -# replaced with the name of the entity): "The $name class" "The $name widget" -# "The $name file" "is" "provides" "specifies" "contains" -# "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like regular Qt-style comments -# (thus requiring an explicit @brief command for a brief description.) - -JAVADOC_AUTOBRIEF = NO - -# If the QT_AUTOBRIEF tag is set to YES then Doxygen will -# interpret the first line (until the first dot) of a Qt-style -# comment as the brief description. If set to NO, the comments -# will behave just like regular Qt-style comments (thus requiring -# an explicit \brief command for a brief description.) - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce -# a new page for each member. If set to NO, the documentation of a member will -# be part of the file/class/namespace that contains it. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C -# sources only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = YES - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java -# sources only. Doxygen will then generate output that is more tailored for -# Java. For instance, namespaces will be presented as packages, qualified -# scopes will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources only. Doxygen will then generate output that is more tailored for -# Fortran. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for -# VHDL. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given extension. -# Doxygen has a built-in mapping, but you can override or extend it using this -# tag. The format is ext=language, where ext is a file extension, and language -# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, -# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make -# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C -# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions -# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. - -EXTENSION_MAPPING = - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should -# set this tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. -# func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. -# Doxygen will parse them like normal C++ but will assume all classes use public -# instead of private inheritance when no explicit protection keyword is present. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate getter -# and setter methods for a property. Setting this option to YES (the default) -# will make doxygen to replace the get and set methods by a property in the -# documentation. This will only work if the methods are indeed getting or -# setting a simple type. If this is not the case, or you want to show the -# methods anyway, you should set this option to NO. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -INLINE_GROUPED_CLASSES = YES -# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum -# is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically -# be useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. - -TYPEDEF_HIDES_STRUCT = YES - -# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to -# determine which symbols to keep in memory and which to flush to disk. -# When the cache is full, less often used symbols will be written to disk. -# For small to medium size projects (<1000 input files) the default value is -# probably good enough. For larger projects a too small cache size can cause -# doxygen to be busy swapping symbols to and from disk most of the time -# causing a significant performance penality. -# If the system has enough physical memory increasing the cache will improve the -# performance by keeping more symbols in memory. Note that the value works on -# a logarithmic scale so increasing the size by one will rougly double the -# memory usage. The cache size is given by this formula: -# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, -# corresponding to a cache size of 2^16 = 65536 symbols - -SYMBOL_CACHE_SIZE = 0 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base -# name of the file that contains the anonymous namespace. By default -# anonymous namespace are hidden. - -EXTRACT_ANON_NSPACES = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen -# will list include files with double quotes in the documentation -# rather than with sharp brackets. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = NO - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen -# will sort the (brief and detailed) documentation of class members so that -# constructors and destructors are listed first. If set to NO (the default) -# the constructors will appear in the respective orders defined by -# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. -# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO -# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the -# hierarchy of group names into alphabetical order. If set to NO (the default) -# the group names will appear in their defined order. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. - -SORT_BY_SCOPE_NAME = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# If the sources in your project are distributed over multiple directories -# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy -# in the documentation. The default is NO. - -SHOW_DIRECTORIES = NO - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. -# This will remove the Files entry from the Quick Index and from the -# Folder Tree View (if specified). The default is YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the -# Namespaces page. -# This will remove the Namespaces entry from the Quick Index -# and from the Folder Tree View (if specified). The default is YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command , where is the value of -# the FILE_VERSION_FILTER tag, and is the name of an input file -# provided by doxygen. Whatever the program writes to standard output -# is used as the file version. See the manual for examples. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. The create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. -# You can optionally specify a file name after the option, if omitted -# DoxygenLayout.xml will be used as the name of the layout file. - -LAYOUT_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# This WARN_NO_PARAMDOC option can be abled to get warnings for -# functions that are documented, but have no documentation for their parameters -# or return value. If set to NO (the default) doxygen will only warn about -# wrong or incomplete parameter documentation, but not about the absence of -# documentation. - -WARN_NO_PARAMDOC = NO - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. Optionally the format may contain -# $version, which will be replaced by the version of the file (if it could -# be obtained via FILE_VERSION_FILTER) - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = lmdb.h midl.h mdb.c midl.c - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is -# also the default input encoding. Doxygen uses libiconv (or the iconv built -# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for -# the list of possible encodings. - -INPUT_ENCODING = UTF-8 - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx -# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 - -FILE_PATTERNS = - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = NO - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or -# directories that are symbolic links (a Unix filesystem feature) are excluded -# from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. Note that the wildcards are matched -# against the file with absolute path, so to exclude all test directories -# for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command , where -# is the value of the INPUT_FILTER tag, and is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. -# If FILTER_PATTERNS is specified, this tag will be -# ignored. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. -# Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. -# The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER -# is applied to all files. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) -# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from -# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will -# link to the source code. -# Otherwise they will link to the documentation. - -REFERENCES_LINK_SOURCE = YES - -# If the USE_HTAGS tag is set to YES then the references to source code -# will point to the HTML generated by the htags(1) tool instead of doxygen -# built-in source browser. The htags tool is part of GNU's global source -# tagging system (see http://www.gnu.org/software/global/global.html). You -# will need version 4.8.6 or higher. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = YES - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. -# Doxygen will adjust the colors in the stylesheet and background images -# according to this color. Hue is specified as an angle on a colorwheel, -# see http://en.wikipedia.org/wiki/Hue for more information. -# For instance the value 0 represents red, 60 is yellow, 120 is green, -# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. -# The allowed range is 0 to 359. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of -# the colors in the HTML output. For a value of 0 the output will use -# grayscales only. A value of 255 will produce the most vivid colors. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to -# the luminance component of the colors in the HTML output. Values below -# 100 gradually make the output lighter, whereas values above 100 make -# the output darker. The value divided by 100 is the actual gamma applied, -# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, -# and 100 does not change the gamma. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting -# this to NO can help when comparing the output of multiple runs. - -HTML_TIMESTAMP = YES - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. For this to work a browser that supports -# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox -# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). - -HTML_DYNAMIC_SECTIONS = NO - -# If the GENERATE_DOCSET tag is set to YES, additional index files -# will be generated that can be used as input for Apple's Xcode 3 -# integrated development environment, introduced with OSX 10.5 (Leopard). -# To create a documentation set, doxygen will generate a Makefile in the -# HTML output directory. Running make will produce the docset in that -# directory and running "make install" will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find -# it at startup. -# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. - -GENERATE_DOCSET = NO - -# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the -# feed. A documentation feed provides an umbrella under which multiple -# documentation sets from a single provider (such as a company or product suite) -# can be grouped. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that -# should uniquely identify the documentation set bundle. This should be a -# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen -# will append .docset to the name. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING -# is used to encode HtmlHelp index (hhk), content (hhc) and project file -# content. - -CHM_INDEX_ENCODING = - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated -# that can be used as input for Qt's qhelpgenerator to generate a -# Qt Compressed Help (.qch) of the generated HTML documentation. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can -# be used to specify the file name of the resulting .qch file. -# The path specified is relative to the HTML output folder. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating -# Qt Help Project output. For more information please see -# http://doc.trolltech.com/qthelpproject.html#namespace - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating -# Qt Help Project output. For more information please see -# http://doc.trolltech.com/qthelpproject.html#virtual-folders - -QHP_VIRTUAL_FOLDER = doc - -# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to -# add. For more information please see -# http://doc.trolltech.com/qthelpproject.html#custom-filters - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see -# -# Qt Help Project / Custom Filters. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's -# filter section matches. -# -# Qt Help Project / Filter Attributes. - -QHP_SECT_FILTER_ATTRS = - -# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can -# be used to specify the location of Qt's qhelpgenerator. -# If non-empty doxygen will try to run qhelpgenerator on the generated -# .qhp file. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files -# will be generated, which together with the HTML files, form an Eclipse help -# plugin. To install this plugin and make it available under the help contents -# menu in Eclipse, the contents of the directory containing the HTML and XML -# files needs to be copied into the plugins directory of eclipse. The name of -# the directory within the plugins directory should be the same as -# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before -# the help appears. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have -# this name. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. -# If the tag value is set to YES, a side panel will be generated -# containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). -# Windows users are probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, -# and Class Hierarchy pages using a tree view instead of an ordered list. - -USE_INLINE_TREES = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open -# links to external symbols imported via tag files in a separate window. - -EXT_LINKS_IN_WINDOW = NO - -# Use this tag to change the font size of Latex formulas included -# as images in the HTML documentation. The default is 10. Note that -# when you change the font size after a successful doxygen run you need -# to manually remove any form_*.png images from the HTML output directory -# to force them to be regenerated. - -FORMULA_FONTSIZE = 10 - -# Use the FORMULA_TRANPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are -# not supported properly for IE 6.0, but are supported on all modern browsers. -# Note that when changing this option you need to delete any form_*.png files -# in the HTML output before the changes have effect. - -FORMULA_TRANSPARENT = YES - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box -# for the HTML output. The underlying search engine uses javascript -# and DHTML and should work on any modern browser. Note that when using -# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets -# (GENERATE_DOCSET) there is already a search function so this one should -# typically be disabled. For large projects the javascript based search engine -# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. - -SEARCHENGINE = YES - -# When the SERVER_BASED_SEARCH tag is enabled the search engine will be -# implemented using a PHP enabled web server instead of at the web client -# using Javascript. Doxygen will generate the search PHP script and index -# file to put on the web server. The advantage of the server -# based approach is that it scales better to large projects and allows -# full text search. The disadvances is that it is more difficult to setup -# and does not have live searching capabilities. - -SERVER_BASED_SEARCH = NO - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = NO - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. -# Note that when enabling USE_PDFLATEX this option is only used for -# generating bitmaps for formulas in the HTML output, but not in the -# Makefile that is written to the output directory. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = YES - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = YES - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -# If LATEX_SOURCE_CODE is set to YES then doxygen will include -# source code with syntax highlighting in the LaTeX output. -# Note that which sources are shown also depends on other settings -# such as SOURCE_BROWSER. - -LATEX_SOURCE_CODE = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = YES - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. -# This is useful -# if you want to understand what is going on. -# On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_DEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. To prevent a macro definition from being -# undefined via #undef or recursively expanded use the := operator -# instead of the = operator. - -PREDEFINED = DEBUG=2 __GNUC__=1 - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse -# the parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = tooltag=./man1 - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base -# or super classes. Setting the tag to NO turns the diagrams off. Note that -# this option is superseded by the HAVE_DOT option below. This is only a -# fallback. It is recommended to install and use dot, since it yields more -# powerful graphs. - -CLASS_DIAGRAMS = YES - -# You can define message sequence charts within doxygen comments using the \msc -# command. Doxygen will then run the mscgen tool (see -# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the -# documentation. The MSCGEN_PATH tag allows you to specify the directory where -# the mscgen tool resides. If left empty the tool is assumed to be found in the -# default search path. - -MSCGEN_PATH = - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is -# allowed to run in parallel. When set to 0 (the default) doxygen will -# base this on the number of processors available in the system. You can set it -# explicitly to a value larger than 0 to get control over the balance -# between CPU load and processing speed. - -DOT_NUM_THREADS = 0 - -# By default doxygen will write a font called FreeSans.ttf to the output -# directory and reference it in all dot files that doxygen generates. This -# font does not include all possible unicode characters however, so when you need -# these (or just want a differently looking font) you can specify the font name -# using DOT_FONTNAME. You need need to make sure dot is able to find the font, -# which can be done by putting it in a standard location or by setting the -# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory -# containing the font. - -DOT_FONTNAME = FreeSans.ttf - -# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. -# The default size is 10pt. - -DOT_FONTSIZE = 10 - -# By default doxygen will tell dot to use the output directory to look for the -# FreeSans.ttf font (which doxygen will put there itself). If you specify a -# different font using DOT_FONTNAME you can set the path where dot -# can find it using this tag. - -DOT_FONTPATH = - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for groups, showing the direct groups dependencies - -GROUP_GRAPHS = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = NO - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT options are set to YES then -# doxygen will generate a call dependency graph for every global function -# or class method. Note that enabling this option will significantly increase -# the time of a run. So in most cases it will be better to enable call graphs -# for selected functions only using the \callgraph command. - -CALL_GRAPH = NO - -# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then -# doxygen will generate a caller dependency graph for every global function -# or class method. Note that enabling this option will significantly increase -# the time of a run. So in most cases it will be better to enable caller -# graphs for selected functions only using the \callergraph command. - -CALLER_GRAPH = NO - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES -# then doxygen will show the dependencies a directory has on other directories -# in a graphical way. The dependency relations are determined by the #include -# relations between the files in the directories. - -DIRECTORY_GRAPH = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found in the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of -# nodes that will be shown in the graph. If the number of nodes in a graph -# becomes larger than this value, doxygen will truncate the graph, which is -# visualized by representing a node as a red box. Note that doxygen if the -# number of direct children of the root node in a graph is already larger than -# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note -# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. - -DOT_GRAPH_MAX_NODES = 50 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes -# that lay further from the root node will be omitted. Note that setting this -# option to 1 or 2 may greatly reduce the computation time needed for large -# code bases. Also note that the size of a graph can be further restricted by -# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. - -MAX_DOT_GRAPH_DEPTH = 0 - -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, because dot on Windows does not -# seem to support this out of the box. Warning: Depending on the platform used, -# enabling this option may lead to badly anti-aliased labels on the edges of -# a graph (i.e. they become hard to read). - -DOT_TRANSPARENT = NO - -# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output -# files in one run (i.e. multiple -o and -T options on the command line). This -# makes dot run faster, but since only newer versions of dot (>1.8.10) -# support this, this feature is disabled by default. - -DOT_MULTI_TARGETS = YES - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES diff --git a/submodules/lmdb/libraries/liblmdb/LICENSE b/submodules/lmdb/libraries/liblmdb/LICENSE deleted file mode 100644 index 05ad7571e4..0000000000 --- a/submodules/lmdb/libraries/liblmdb/LICENSE +++ /dev/null @@ -1,47 +0,0 @@ -The OpenLDAP Public License - Version 2.8, 17 August 2003 - -Redistribution and use of this software and associated documentation -("Software"), with or without modification, are permitted provided -that the following conditions are met: - -1. Redistributions in source form must retain copyright statements - and notices, - -2. Redistributions in binary form must reproduce applicable copyright - statements and notices, this list of conditions, and the following - disclaimer in the documentation and/or other materials provided - with the distribution, and - -3. Redistributions must contain a verbatim copy of this document. - -The OpenLDAP Foundation may revise this license from time to time. -Each revision is distinguished by a version number. You may use -this Software under terms of this license revision or under the -terms of any subsequent revision of the license. - -THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS -CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY -AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT -SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) -OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -The names of the authors and copyright holders must not be used in -advertising or otherwise to promote the sale, use or other dealing -in this Software without specific, written prior permission. Title -to copyright in this Software shall at all times remain with copyright -holders. - -OpenLDAP is a registered trademark of the OpenLDAP Foundation. - -Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, -California, USA. All Rights Reserved. Permission to copy and -distribute verbatim copies of this document is granted. diff --git a/submodules/lmdb/libraries/liblmdb/Makefile b/submodules/lmdb/libraries/liblmdb/Makefile deleted file mode 100644 index 25c1095466..0000000000 --- a/submodules/lmdb/libraries/liblmdb/Makefile +++ /dev/null @@ -1,97 +0,0 @@ -# Makefile for liblmdb (Lightning memory-mapped database library). - -######################################################################## -# Configuration. The compiler options must enable threaded compilation. -# -# Preprocessor macros (for CPPFLAGS) of interest... -# Note that the defaults should already be correct for most -# platforms; you should not need to change any of these. -# Read their descriptions in mdb.c if you do: -# -# - MDB_USE_POSIX_SEM -# - MDB_DSYNC -# - MDB_FDATASYNC -# - MDB_USE_PWRITEV -# -# There may be other macros in mdb.c of interest. You should -# read mdb.c before changing any of them. -# -CC = gcc -W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized -THREADS = -pthread -OPT = -O2 -g -CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS) -LDLIBS = -SOLIBS = -prefix = /usr/local - -######################################################################## - -IHDRS = lmdb.h -ILIBS = liblmdb.a liblmdb.so -IPROGS = mdb_stat mdb_copy mdb_dump mdb_load -IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 -PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 -all: $(ILIBS) $(PROGS) - -install: $(ILIBS) $(IPROGS) $(IHDRS) - for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done - for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done - for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done - for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done - -clean: - rm -rf $(PROGS) *.[ao] *.so *~ testdb - -test: all - mkdir testdb - ./mtest && ./mdb_stat testdb - -liblmdb.a: mdb.o midl.o - ar rs $@ mdb.o midl.o - -liblmdb.so: mdb.o midl.o -# $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS) - $(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.o midl.o $(SOLIBS) - -mdb_stat: mdb_stat.o liblmdb.a -mdb_copy: mdb_copy.o liblmdb.a -mdb_dump: mdb_dump.o liblmdb.a -mdb_load: mdb_load.o liblmdb.a -mtest: mtest.o liblmdb.a -mtest2: mtest2.o liblmdb.a -mtest3: mtest3.o liblmdb.a -mtest4: mtest4.o liblmdb.a -mtest5: mtest5.o liblmdb.a -mtest6: mtest6.o liblmdb.a - -mdb.o: mdb.c lmdb.h midl.h - $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c - -midl.o: midl.c midl.h - $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c - -%: %.o - $(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@ - -%.o: %.c lmdb.h - $(CC) $(CFLAGS) $(CPPFLAGS) -c $< - -COV_FLAGS=-fprofile-arcs -ftest-coverage -COV_OBJS=xmdb.o xmidl.o - -coverage: xmtest - for i in mtest*.c [0-9]*.c; do j=`basename \$$i .c`; $(MAKE) $$j.o; \ - gcc -o x$$j $$j.o $(COV_OBJS) -pthread $(COV_FLAGS); \ - rm -rf testdb; mkdir testdb; ./x$$j; done - gcov xmdb.c - gcov xmidl.c - -xmtest: mtest.o xmdb.o xmidl.o - gcc -o xmtest mtest.o xmdb.o xmidl.o -pthread $(COV_FLAGS) - -xmdb.o: mdb.c lmdb.h midl.h - $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c mdb.c -o $@ - -xmidl.o: midl.c midl.h - $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c midl.c -o $@ diff --git a/submodules/lmdb/libraries/liblmdb/mdb_copy.1 b/submodules/lmdb/libraries/liblmdb/mdb_copy.1 deleted file mode 100644 index 094b260563..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mdb_copy.1 +++ /dev/null @@ -1,54 +0,0 @@ -.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. -.\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.SH NAME -mdb_copy \- LMDB environment copy tool -.SH SYNOPSIS -.B mdb_copy -[\c -.BR \-V ] -[\c -.BR \-c ] -[\c -.BR \-n ] -.B srcpath -[\c -.BR dstpath ] -.SH DESCRIPTION -The -.B mdb_copy -utility copies an LMDB environment. The environment can -be copied regardless of whether it is currently in use. -No lockfile is created, since it gets recreated at need. - -If -.I dstpath -is specified it must be the path of an empty directory -for storing the backup. Otherwise, the backup will be -written to stdout. - -.SH OPTIONS -.TP -.BR \-V -Write the library version number to the standard output, and exit. -.TP -.BR \-c -Compact while copying. Only current data pages will be copied; freed -or unused pages will be omitted from the copy. This option will -slow down the backup process as it is more CPU-intensive. -.TP -.BR \-n -Open LDMB environment(s) which do not use subdirectories. - -.SH DIAGNOSTICS -Exit status is zero if no errors occur. -Errors result in a non-zero exit status and -a diagnostic message being written to standard error. -.SH CAVEATS -This utility can trigger significant file size growth if run -in parallel with write transactions, because pages which they -free during copying cannot be reused until the copy is done. -.SH "SEE ALSO" -.BR mdb_stat (1) -.SH AUTHOR -Howard Chu of Symas Corporation diff --git a/submodules/lmdb/libraries/liblmdb/mdb_copy.c b/submodules/lmdb/libraries/liblmdb/mdb_copy.c deleted file mode 100644 index e7f965c03a..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mdb_copy.c +++ /dev/null @@ -1,82 +0,0 @@ -/* mdb_copy.c - memory-mapped database backup tool */ -/* - * Copyright 2012 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ -#ifdef _WIN32 -#include -#define MDB_STDOUT GetStdHandle(STD_OUTPUT_HANDLE) -#else -#define MDB_STDOUT 1 -#endif -#include -#include -#include -#include "lmdb.h" - -static void -sighandle(int sig) -{ -} - -int main(int argc,char * argv[]) -{ - int rc; - MDB_env *env; - const char *progname = argv[0], *act; - unsigned flags = MDB_RDONLY; - unsigned cpflags = 0; - - for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { - if (argv[1][1] == 'n' && argv[1][2] == '\0') - flags |= MDB_NOSUBDIR; - else if (argv[1][1] == 'c' && argv[1][2] == '\0') - cpflags |= MDB_CP_COMPACT; - else if (argv[1][1] == 'V' && argv[1][2] == '\0') { - printf("%s\n", MDB_VERSION_STRING); - exit(0); - } else - argc = 0; - } - - if (argc<2 || argc>3) { - fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname); - exit(EXIT_FAILURE); - } - -#ifdef SIGPIPE - signal(SIGPIPE, sighandle); -#endif -#ifdef SIGHUP - signal(SIGHUP, sighandle); -#endif - signal(SIGINT, sighandle); - signal(SIGTERM, sighandle); - - act = "opening environment"; - rc = mdb_env_create(&env); - if (rc == MDB_SUCCESS) { - rc = mdb_env_open(env, argv[1], flags, 0664); - } - if (rc == MDB_SUCCESS) { - act = "copying"; - if (argc == 2) - rc = mdb_env_copyfd2(env, MDB_STDOUT, cpflags); - else - rc = mdb_env_copy2(env, argv[2], cpflags); - } - if (rc) - fprintf(stderr, "%s: %s failed, error %d (%s)\n", - progname, act, rc, mdb_strerror(rc)); - mdb_env_close(env); - - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/submodules/lmdb/libraries/liblmdb/mdb_dump.1 b/submodules/lmdb/libraries/liblmdb/mdb_dump.1 deleted file mode 100644 index 6fcc930811..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mdb_dump.1 +++ /dev/null @@ -1,75 +0,0 @@ -.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved. -.\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.SH NAME -mdb_dump \- LMDB environment export tool -.SH SYNOPSIS -.B mdb_dump -.BR \ envpath -[\c -.BR \-V ] -[\c -.BI \-f \ file\fR] -[\c -.BR \-l ] -[\c -.BR \-n ] -[\c -.BR \-p ] -[\c -.BR \-a \ | -.BI \-s \ subdb\fR] -.SH DESCRIPTION -The -.B mdb_dump -utility reads a database and writes its contents to the -standard output using a portable flat-text format -understood by the -.BR mdb_load (1) -utility. -.SH OPTIONS -.TP -.BR \-V -Write the library version number to the standard output, and exit. -.TP -.BR \-f \ file -Write to the specified file instead of to the standard output. -.TP -.BR \-l -List the databases stored in the environment. Just the -names will be listed, no data will be output. -.TP -.BR \-n -Dump an LMDB database which does not use subdirectories. -.TP -.BR \-p -If characters in either the key or data items are printing characters (as -defined by isprint(3)), output them directly. This option permits users to -use standard text editors and tools to modify the contents of databases. - -Note: different systems may have different notions about what characters -are considered printing characters, and databases dumped in this manner may -be less portable to external systems. -.TP -.BR \-a -Dump all of the subdatabases in the environment. -.TP -.BR \-s \ subdb -Dump a specific subdatabase. If no database is specified, only the main database is dumped. -.SH DIAGNOSTICS -Exit status is zero if no errors occur. -Errors result in a non-zero exit status and -a diagnostic message being written to standard error. - -Dumping and reloading databases that use user-defined comparison functions -will result in new databases that use the default comparison functions. -\fBIn this case it is quite likely that the reloaded database will be -damaged beyond repair permitting neither record storage nor retrieval.\fP - -The only available workaround is to modify the source for the -.BR mdb_load (1) -utility to load the database using the correct comparison functions. -.SH "SEE ALSO" -.BR mdb_load (1) -.SH AUTHOR -Howard Chu of Symas Corporation diff --git a/submodules/lmdb/libraries/liblmdb/mdb_dump.c b/submodules/lmdb/libraries/liblmdb/mdb_dump.c deleted file mode 100644 index 0eb85fd20e..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mdb_dump.c +++ /dev/null @@ -1,317 +0,0 @@ -/* mdb_dump.c - memory-mapped database dump tool */ -/* - * Copyright 2011-2014 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ -#include -#include -#include -#include -#include -#include -#include -#include "lmdb.h" - -#ifdef _WIN32 -#define Z "I" -#else -#define Z "z" -#endif - -#define PRINT 1 -static int mode; - -typedef struct flagbit { - int bit; - char *name; -} flagbit; - -flagbit dbflags[] = { - { MDB_REVERSEKEY, "reversekey" }, - { MDB_DUPSORT, "dupsort" }, - { MDB_INTEGERKEY, "integerkey" }, - { MDB_DUPFIXED, "dupfixed" }, - { MDB_INTEGERDUP, "integerdup" }, - { MDB_REVERSEDUP, "reversedup" }, - { 0, NULL } -}; - -static volatile sig_atomic_t gotsig; - -static void dumpsig( int sig ) -{ - gotsig=1; -} - -static const char hexc[] = "0123456789abcdef"; - -static void hex(unsigned char c) -{ - putchar(hexc[c >> 4]); - putchar(hexc[c & 0xf]); -} - -static void text(MDB_val *v) -{ - unsigned char *c, *end; - - putchar(' '); - c = v->mv_data; - end = c + v->mv_size; - while (c < end) { - if (isprint(*c)) { - putchar(*c); - } else { - putchar('\\'); - hex(*c); - } - c++; - } - putchar('\n'); -} - -static void byte(MDB_val *v) -{ - unsigned char *c, *end; - - putchar(' '); - c = v->mv_data; - end = c + v->mv_size; - while (c < end) { - hex(*c++); - } - putchar('\n'); -} - -/* Dump in BDB-compatible format */ -static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) -{ - MDB_cursor *mc; - MDB_stat ms; - MDB_val key, data; - MDB_envinfo info; - unsigned int flags; - int rc, i; - - rc = mdb_dbi_flags(txn, dbi, &flags); - if (rc) return rc; - - rc = mdb_stat(txn, dbi, &ms); - if (rc) return rc; - - rc = mdb_env_info(mdb_txn_env(txn), &info); - if (rc) return rc; - - printf("VERSION=3\n"); - printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); - if (name) - printf("database=%s\n", name); - printf("type=btree\n"); - printf("mapsize=%" Z "u\n", info.me_mapsize); - if (info.me_mapaddr) - printf("mapaddr=%p\n", info.me_mapaddr); - printf("maxreaders=%u\n", info.me_maxreaders); - - if (flags & MDB_DUPSORT) - printf("duplicates=1\n"); - - for (i=0; dbflags[i].bit; i++) - if (flags & dbflags[i].bit) - printf("%s=1\n", dbflags[i].name); - - printf("db_pagesize=%d\n", ms.ms_psize); - printf("HEADER=END\n"); - - rc = mdb_cursor_open(txn, dbi, &mc); - if (rc) return rc; - - while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT) == MDB_SUCCESS)) { - if (gotsig) { - rc = EINTR; - break; - } - if (mode & PRINT) { - text(&key); - text(&data); - } else { - byte(&key); - byte(&data); - } - } - printf("DATA=END\n"); - if (rc == MDB_NOTFOUND) - rc = MDB_SUCCESS; - - return rc; -} - -static void usage(char *prog) -{ - fprintf(stderr, "usage: %s dbpath [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb]\n", prog); - exit(EXIT_FAILURE); -} - -int main(int argc, char *argv[]) -{ - int i, rc; - MDB_env *env; - MDB_txn *txn; - MDB_dbi dbi; - char *prog = argv[0]; - char *envname; - char *subname = NULL; - int alldbs = 0, envflags = 0, list = 0; - - if (argc < 2) { - usage(prog); - } - - /* -a: dump main DB and all subDBs - * -s: dump only the named subDB - * -n: use NOSUBDIR flag on env_open - * -p: use printable characters - * -f: write to file instead of stdout - * -V: print version and exit - * (default) dump only the main DB - */ - while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) { - switch(i) { - case 'V': - printf("%s\n", MDB_VERSION_STRING); - exit(0); - break; - case 'l': - list = 1; - /*FALLTHROUGH*/; - case 'a': - if (subname) - usage(prog); - alldbs++; - break; - case 'f': - if (freopen(optarg, "w", stdout) == NULL) { - fprintf(stderr, "%s: %s: reopen: %s\n", - prog, optarg, strerror(errno)); - exit(EXIT_FAILURE); - } - break; - case 'n': - envflags |= MDB_NOSUBDIR; - break; - case 'p': - mode |= PRINT; - break; - case 's': - if (alldbs) - usage(prog); - subname = optarg; - break; - default: - usage(prog); - } - } - - if (optind != argc - 1) - usage(prog); - -#ifdef SIGPIPE - signal(SIGPIPE, dumpsig); -#endif -#ifdef SIGHUP - signal(SIGHUP, dumpsig); -#endif - signal(SIGINT, dumpsig); - signal(SIGTERM, dumpsig); - - envname = argv[optind]; - rc = mdb_env_create(&env); - if (rc) { - fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); - return EXIT_FAILURE; - } - - if (alldbs || subname) { - mdb_env_set_maxdbs(env, 2); - } - - rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); - if (rc) { - fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto env_close; - } - - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - if (rc) { - fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); - goto env_close; - } - - rc = mdb_open(txn, subname, 0, &dbi); - if (rc) { - fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - - if (alldbs) { - MDB_cursor *cursor; - MDB_val key; - int count = 0; - - rc = mdb_cursor_open(txn, dbi, &cursor); - if (rc) { - fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { - char *str; - MDB_dbi db2; - if (memchr(key.mv_data, '\0', key.mv_size)) - continue; - count++; - str = malloc(key.mv_size+1); - memcpy(str, key.mv_data, key.mv_size); - str[key.mv_size] = '\0'; - rc = mdb_open(txn, str, 0, &db2); - if (rc == MDB_SUCCESS) { - if (list) { - printf("%s\n", str); - list++; - } else { - rc = dumpit(txn, db2, str); - if (rc) - break; - } - mdb_close(env, db2); - } - free(str); - if (rc) continue; - } - mdb_cursor_close(cursor); - if (!count) { - fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname); - rc = MDB_NOTFOUND; - } else if (rc == MDB_NOTFOUND) { - rc = MDB_SUCCESS; - } - } else { - rc = dumpit(txn, dbi, subname); - } - if (rc && rc != MDB_NOTFOUND) - fprintf(stderr, "%s: %s: %s\n", prog, envname, mdb_strerror(rc)); - - mdb_close(env, dbi); -txn_abort: - mdb_txn_abort(txn); -env_close: - mdb_env_close(env); - - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/submodules/lmdb/libraries/liblmdb/mdb_load.1 b/submodules/lmdb/libraries/liblmdb/mdb_load.1 deleted file mode 100644 index 7280240103..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mdb_load.1 +++ /dev/null @@ -1,77 +0,0 @@ -.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved. -.\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.SH NAME -mdb_load \- LMDB environment import tool -.SH SYNOPSIS -.B mdb_load -.BR \ envpath -[\c -.BR \-V ] -[\c -.BI \-f \ file\fR] -[\c -.BR \-n ] -[\c -.BI \-s \ subdb\fR] -[\c -.BR \-N ] -[\c -.BR \-T ] -.SH DESCRIPTION -The -.B mdb_load -utility reads from the standard input and loads it into the -LMDB environment -.BR envpath . - -The input to -.B mdb_load -must be in the output format specified by the -.BR mdb_dump (1) -utility or as specified by the -.B -T -option below. -.SH OPTIONS -.TP -.BR \-V -Write the library version number to the standard output, and exit. -.TP -.BR \-f \ file -Read from the specified file instead of from the standard input. -.TP -.BR \-n -Load an LMDB database which does not use subdirectories. -.TP -.BR \-s \ subdb -Load a specific subdatabase. If no database is specified, data is loaded into the main database. -.TP -.BR \-N -Don't overwrite existing records when loading into an already existing database; just skip them. -.TP -.BR \-T -Load data from simple text files. The input must be paired lines of text, where the first -line of the pair is the key item, and the second line of the pair is its corresponding -data item. - -A simple escape mechanism, where newline and backslash (\\) characters are special, is -applied to the text input. Newline characters are interpreted as record separators. -Backslash characters in the text will be interpreted in one of two ways: If the backslash -character precedes another backslash character, the pair will be interpreted as a literal -backslash. If the backslash character precedes any other character, the two characters -following the backslash will be interpreted as a hexadecimal specification of a single -character; for example, \\0a is a newline character in the ASCII character set. - -For this reason, any backslash or newline characters that naturally occur in the text -input must be escaped to avoid misinterpretation by -.BR mdb_load . - -.SH DIAGNOSTICS -Exit status is zero if no errors occur. -Errors result in a non-zero exit status and -a diagnostic message being written to standard error. - -.SH "SEE ALSO" -.BR mdb_dump (1) -.SH AUTHOR -Howard Chu of Symas Corporation diff --git a/submodules/lmdb/libraries/liblmdb/mdb_load.c b/submodules/lmdb/libraries/liblmdb/mdb_load.c deleted file mode 100644 index e0d95e13c3..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mdb_load.c +++ /dev/null @@ -1,452 +0,0 @@ -/* mdb_load.c - memory-mapped database load tool */ -/* - * Copyright 2011-2014 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ -#include -#include -#include -#include -#include -#include -#include "lmdb.h" - -#define PRINT 1 -#define NOHDR 2 -static int mode; - -static char *subname = NULL; - -static size_t lineno; -static int version; - -static int flags; - -static char *prog; - -static int Eof; - -static MDB_envinfo info; - -static MDB_val kbuf, dbuf; - -#ifdef _WIN32 -#define Z "I" -#else -#define Z "z" -#endif - -#define STRLENOF(s) (sizeof(s)-1) - -typedef struct flagbit { - int bit; - char *name; - int len; -} flagbit; - -#define S(s) s, STRLENOF(s) - -flagbit dbflags[] = { - { MDB_REVERSEKEY, S("reversekey") }, - { MDB_DUPSORT, S("dupsort") }, - { MDB_INTEGERKEY, S("integerkey") }, - { MDB_DUPFIXED, S("dupfixed") }, - { MDB_INTEGERDUP, S("integerdup") }, - { MDB_REVERSEDUP, S("reversedup") }, - { 0, NULL, 0 } -}; - -static void readhdr(void) -{ - char *ptr; - - while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) { - lineno++; - if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { - version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION=")); - if (version > 3) { - fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n", - prog, lineno, version); - exit(EXIT_FAILURE); - } - } else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) { - break; - } else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) { - if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) - mode |= PRINT; - else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { - fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n", - prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT=")); - exit(EXIT_FAILURE); - } - } else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) { - ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); - if (ptr) *ptr = '\0'; - if (subname) free(subname); - subname = strdup((char *)dbuf.mv_data+STRLENOF("database=")); - } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) { - if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { - fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n", - prog, lineno, (char *)dbuf.mv_data+STRLENOF("type=")); - exit(EXIT_FAILURE); - } - } else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) { - int i; - ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); - if (ptr) *ptr = '\0'; - i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr); - if (i != 1) { - fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n", - prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr=")); - exit(EXIT_FAILURE); - } - } else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) { - int i; - ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); - if (ptr) *ptr = '\0'; - i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize); - if (i != 1) { - fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n", - prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize=")); - exit(EXIT_FAILURE); - } - } else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) { - int i; - ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); - if (ptr) *ptr = '\0'; - i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders); - if (i != 1) { - fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n", - prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders=")); - exit(EXIT_FAILURE); - } - } else { - int i; - for (i=0; dbflags[i].bit; i++) { - if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) && - ((char *)dbuf.mv_data)[dbflags[i].len] == '=') { - flags |= dbflags[i].bit; - break; - } - } - if (!dbflags[i].bit) { - ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size); - if (!ptr) { - fprintf(stderr, "%s: line %" Z "d: unexpected format\n", - prog, lineno); - exit(EXIT_FAILURE); - } else { - *ptr = '\0'; - fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n", - prog, lineno, (char *)dbuf.mv_data); - } - } - } - } -} - -static void badend(void) -{ - fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n", - prog, lineno); -} - -static int unhex(unsigned char *c2) -{ - int x, c; - x = *c2++ & 0x4f; - if (x & 0x40) - x -= 55; - c = x << 4; - x = *c2 & 0x4f; - if (x & 0x40) - x -= 55; - c |= x; - return c; -} - -static int readline(MDB_val *out, MDB_val *buf) -{ - unsigned char *c1, *c2, *end; - size_t len; - int c; - - if (!(mode & NOHDR)) { - c = fgetc(stdin); - if (c == EOF) { - Eof = 1; - return EOF; - } - if (c != ' ') { - lineno++; - if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { -badend: - Eof = 1; - badend(); - return EOF; - } - if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END"))) - return EOF; - goto badend; - } - } - if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { - Eof = 1; - return EOF; - } - lineno++; - - c1 = buf->mv_data; - len = strlen((char *)c1); - - /* Is buffer too short? */ - while (c1[len-1] != '\n') { - buf->mv_data = realloc(buf->mv_data, buf->mv_size*2); - if (!buf->mv_data) { - Eof = 1; - fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n", - prog, lineno); - return EOF; - } - c1 = buf->mv_data; - c1 += buf->mv_size; - if (fgets((char *)c1, buf->mv_size, stdin) == NULL) { - Eof = 1; - badend(); - return EOF; - } - buf->mv_size *= 2; - len = strlen((char *)c1); - } - c1 = c2 = buf->mv_data; - len = strlen((char *)c1); - c1[--len] = '\0'; - end = c1 + len; - - if (mode & PRINT) { - while (c2 < end) { - if (*c2 == '\\') { - if (c2[1] == '\\') { - c1++; c2 += 2; - } else { - if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { - Eof = 1; - badend(); - return EOF; - } - *c1++ = unhex(++c2); - c2 += 2; - } - } else { - c1++; c2++; - } - } - } else { - /* odd length not allowed */ - if (len & 1) { - Eof = 1; - badend(); - return EOF; - } - while (c2 < end) { - if (!isxdigit(*c2) || !isxdigit(c2[1])) { - Eof = 1; - badend(); - return EOF; - } - *c1++ = unhex(c2); - c2 += 2; - } - } - c2 = out->mv_data = buf->mv_data; - out->mv_size = c1 - c2; - - return 0; -} - -static void usage(void) -{ - fprintf(stderr, "usage: %s dbpath [-V] [-f input] [-n] [-s name] [-N] [-T]\n", prog); - exit(EXIT_FAILURE); -} - -int main(int argc, char *argv[]) -{ - int i, rc; - MDB_env *env; - MDB_txn *txn; - MDB_cursor *mc; - MDB_dbi dbi; - char *envname; - int envflags = 0, putflags = 0; - int dohdr = 0; - - prog = argv[0]; - - if (argc < 2) { - usage(); - } - - /* -f: load file instead of stdin - * -n: use NOSUBDIR flag on env_open - * -s: load into named subDB - * -N: use NOOVERWRITE on puts - * -T: read plaintext - * -V: print version and exit - */ - while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) { - switch(i) { - case 'V': - printf("%s\n", MDB_VERSION_STRING); - exit(0); - break; - case 'f': - if (freopen(optarg, "r", stdin) == NULL) { - fprintf(stderr, "%s: %s: reopen: %s\n", - prog, optarg, strerror(errno)); - exit(EXIT_FAILURE); - } - break; - case 'n': - envflags |= MDB_NOSUBDIR; - break; - case 's': - subname = strdup(optarg); - break; - case 'N': - putflags = MDB_NOOVERWRITE|MDB_NODUPDATA; - break; - case 'T': - mode |= NOHDR; - break; - default: - usage(); - } - } - - if (optind != argc - 1) - usage(); - - dbuf.mv_size = 4096; - dbuf.mv_data = malloc(dbuf.mv_size); - - if (!(mode & NOHDR)) - readhdr(); - - envname = argv[optind]; - rc = mdb_env_create(&env); - if (rc) { - fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); - return EXIT_FAILURE; - } - - mdb_env_set_maxdbs(env, 2); - - if (info.me_maxreaders) - mdb_env_set_maxreaders(env, info.me_maxreaders); - - if (info.me_mapsize) - mdb_env_set_mapsize(env, info.me_mapsize); - - if (info.me_mapaddr) - envflags |= MDB_FIXEDMAP; - - rc = mdb_env_open(env, envname, envflags, 0664); - if (rc) { - fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto env_close; - } - - kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2; - kbuf.mv_data = malloc(kbuf.mv_size); - - while(!Eof) { - MDB_val key, data; - int batch = 0; - flags = 0; - - if (!dohdr) { - dohdr = 1; - } else if (!(mode & NOHDR)) - readhdr(); - - rc = mdb_txn_begin(env, NULL, 0, &txn); - if (rc) { - fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); - goto env_close; - } - - rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi); - if (rc) { - fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - - rc = mdb_cursor_open(txn, dbi, &mc); - if (rc) { - fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - - while(1) { - rc = readline(&key, &kbuf); - if (rc == EOF) - break; - if (rc) - goto txn_abort; - - rc = readline(&data, &dbuf); - if (rc) - goto txn_abort; - - rc = mdb_cursor_put(mc, &key, &data, putflags); - if (rc == MDB_KEYEXIST && putflags) - continue; - if (rc) - goto txn_abort; - batch++; - if (batch == 100) { - rc = mdb_txn_commit(txn); - if (rc) { - fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", - prog, lineno, mdb_strerror(rc)); - goto env_close; - } - rc = mdb_txn_begin(env, NULL, 0, &txn); - if (rc) { - fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); - goto env_close; - } - rc = mdb_cursor_open(txn, dbi, &mc); - if (rc) { - fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - batch = 0; - } - } - rc = mdb_txn_commit(txn); - txn = NULL; - if (rc) { - fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", - prog, lineno, mdb_strerror(rc)); - goto env_close; - } - mdb_dbi_close(env, dbi); - } - -txn_abort: - mdb_txn_abort(txn); -env_close: - mdb_env_close(env); - - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/submodules/lmdb/libraries/liblmdb/mdb_stat.1 b/submodules/lmdb/libraries/liblmdb/mdb_stat.1 deleted file mode 100644 index 3d8d461d99..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mdb_stat.1 +++ /dev/null @@ -1,64 +0,0 @@ -.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. -.\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.SH NAME -mdb_stat \- LMDB environment status tool -.SH SYNOPSIS -.B mdb_stat -.BR \ envpath -[\c -.BR \-V ] -[\c -.BR \-e ] -[\c -.BR \-f [ f [ f ]]] -[\c -.BR \-n ] -[\c -.BR \-r [ r ]] -[\c -.BR \-a \ | -.BI \-s \ subdb\fR] -.SH DESCRIPTION -The -.B mdb_stat -utility displays the status of an LMDB environment. -.SH OPTIONS -.TP -.BR \-V -Write the library version number to the standard output, and exit. -.TP -.BR \-e -Display information about the database environment. -.TP -.BR \-f -Display information about the environment freelist. -If \fB\-ff\fP is given, summarize each freelist entry. -If \fB\-fff\fP is given, display the full list of page IDs in the freelist. -.TP -.BR \-n -Display the status of an LMDB database which does not use subdirectories. -.TP -.BR \-r -Display information about the environment reader table. -Shows the process ID, thread ID, and transaction ID for each active -reader slot. The process ID and transaction ID are in decimal, the -thread ID is in hexadecimal. The transaction ID is displayed as "-" -if the reader does not currently have a read transaction open. -If \fB\-rr\fP is given, check for stale entries in the reader -table and clear them. The reader table will be printed again -after the check is performed. -.TP -.BR \-a -Display the status of all of the subdatabases in the environment. -.TP -.BR \-s \ subdb -Display the status of a specific subdatabase. -.SH DIAGNOSTICS -Exit status is zero if no errors occur. -Errors result in a non-zero exit status and -a diagnostic message being written to standard error. -.SH "SEE ALSO" -.BR mdb_copy (1) -.SH AUTHOR -Howard Chu of Symas Corporation diff --git a/submodules/lmdb/libraries/liblmdb/mdb_stat.c b/submodules/lmdb/libraries/liblmdb/mdb_stat.c deleted file mode 100644 index 1e9229296f..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mdb_stat.c +++ /dev/null @@ -1,263 +0,0 @@ -/* mdb_stat.c - memory-mapped database status tool */ -/* - * Copyright 2011-2013 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ -#include -#include -#include -#include -#include "lmdb.h" - -#ifdef _WIN32 -#define Z "I" -#else -#define Z "z" -#endif - -static void prstat(MDB_stat *ms) -{ -#if 0 - printf(" Page size: %u\n", ms->ms_psize); -#endif - printf(" Tree depth: %u\n", ms->ms_depth); - printf(" Branch pages: %"Z"u\n", ms->ms_branch_pages); - printf(" Leaf pages: %"Z"u\n", ms->ms_leaf_pages); - printf(" Overflow pages: %"Z"u\n", ms->ms_overflow_pages); - printf(" Entries: %"Z"u\n", ms->ms_entries); -} - -static void usage(char *prog) -{ - fprintf(stderr, "usage: %s dbpath [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog); - exit(EXIT_FAILURE); -} - -int main(int argc, char *argv[]) -{ - int i, rc; - MDB_env *env; - MDB_txn *txn; - MDB_dbi dbi; - MDB_stat mst; - MDB_envinfo mei; - char *prog = argv[0]; - char *envname; - char *subname = NULL; - int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0; - - if (argc < 2) { - usage(prog); - } - - /* -a: print stat of main DB and all subDBs - * -s: print stat of only the named subDB - * -e: print env info - * -f: print freelist info - * -r: print reader info - * -n: use NOSUBDIR flag on env_open - * -V: print version and exit - * (default) print stat of only the main DB - */ - while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) { - switch(i) { - case 'V': - printf("%s\n", MDB_VERSION_STRING); - exit(0); - break; - case 'a': - if (subname) - usage(prog); - alldbs++; - break; - case 'e': - envinfo++; - break; - case 'f': - freinfo++; - break; - case 'n': - envflags |= MDB_NOSUBDIR; - break; - case 'r': - rdrinfo++; - break; - case 's': - if (alldbs) - usage(prog); - subname = optarg; - break; - default: - usage(prog); - } - } - - if (optind != argc - 1) - usage(prog); - - envname = argv[optind]; - rc = mdb_env_create(&env); - if (rc) { - fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); - return EXIT_FAILURE; - } - - if (alldbs || subname) { - mdb_env_set_maxdbs(env, 4); - } - - rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); - if (rc) { - fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto env_close; - } - - if (envinfo) { - (void)mdb_env_stat(env, &mst); - (void)mdb_env_info(env, &mei); - printf("Environment Info\n"); - printf(" Map address: %p\n", mei.me_mapaddr); - printf(" Map size: %"Z"u\n", mei.me_mapsize); - printf(" Page size: %u\n", mst.ms_psize); - printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize); - printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1); - printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid); - printf(" Max readers: %u\n", mei.me_maxreaders); - printf(" Number of readers used: %u\n", mei.me_numreaders); - } - - if (rdrinfo) { - printf("Reader Table Status\n"); - rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); - if (rdrinfo > 1) { - int dead; - mdb_reader_check(env, &dead); - printf(" %d stale readers cleared.\n", dead); - rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); - } - if (!(subname || alldbs || freinfo)) - goto env_close; - } - - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - if (rc) { - fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); - goto env_close; - } - - if (freinfo) { - MDB_cursor *cursor; - MDB_val key, data; - size_t pages = 0, *iptr; - - printf("Freelist Status\n"); - dbi = 0; - rc = mdb_cursor_open(txn, dbi, &cursor); - if (rc) { - fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - rc = mdb_stat(txn, dbi, &mst); - if (rc) { - fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - prstat(&mst); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - iptr = data.mv_data; - pages += *iptr; - if (freinfo > 1) { - char *bad = ""; - size_t pg, prev; - ssize_t i, j, span = 0; - j = *iptr++; - for (i = j, prev = 1; --i >= 0; ) { - pg = iptr[i]; - if (pg <= prev) - bad = " [bad sequence]"; - prev = pg; - pg += span; - for (; i >= span && iptr[i-span] == pg; span++, pg++) ; - } - printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n", - *(size_t *)key.mv_data, j, span, bad); - if (freinfo > 2) { - for (--j; j >= 0; ) { - pg = iptr[j]; - for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ; - printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n", - pg, span); - } - } - } - } - mdb_cursor_close(cursor); - printf(" Free pages: %"Z"u\n", pages); - } - - rc = mdb_open(txn, subname, 0, &dbi); - if (rc) { - fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - - rc = mdb_stat(txn, dbi, &mst); - if (rc) { - fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - printf("Status of %s\n", subname ? subname : "Main DB"); - prstat(&mst); - - if (alldbs) { - MDB_cursor *cursor; - MDB_val key; - - rc = mdb_cursor_open(txn, dbi, &cursor); - if (rc) { - fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { - char *str; - MDB_dbi db2; - if (memchr(key.mv_data, '\0', key.mv_size)) - continue; - str = malloc(key.mv_size+1); - memcpy(str, key.mv_data, key.mv_size); - str[key.mv_size] = '\0'; - rc = mdb_open(txn, str, 0, &db2); - if (rc == MDB_SUCCESS) - printf("Status of %s\n", str); - free(str); - if (rc) continue; - rc = mdb_stat(txn, db2, &mst); - if (rc) { - fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); - goto txn_abort; - } - prstat(&mst); - mdb_close(env, db2); - } - mdb_cursor_close(cursor); - } - - if (rc == MDB_NOTFOUND) - rc = MDB_SUCCESS; - - mdb_close(env, dbi); -txn_abort: - mdb_txn_abort(txn); -env_close: - mdb_env_close(env); - - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/submodules/lmdb/libraries/liblmdb/mtest.c b/submodules/lmdb/libraries/liblmdb/mtest.c deleted file mode 100644 index 79b4175e79..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mtest.c +++ /dev/null @@ -1,173 +0,0 @@ -/* mtest.c - memory-mapped database tester/toy */ -/* - * Copyright 2011-2014 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ -#include -#include -#include -#include "lmdb.h" - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ - "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) - -int main(int argc,char * argv[]) -{ - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDB_stat mst; - MDB_cursor *cursor, *cur2; - MDB_cursor_op op; - int count; - int *values; - char sval[32] = ""; - - srand(time(NULL)); - - count = (rand()%384) + 64; - values = (int *)malloc(count*sizeof(int)); - - for(i = 0;i -1; i-= (rand()%5)) { - j++; - txn=NULL; - E(mdb_txn_begin(env, NULL, 0, &txn)); - sprintf(sval, "%03x ", values[i]); - if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { - j--; - mdb_txn_abort(txn); - } else { - E(mdb_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdb_env_stat(env, &mst)); - E(mdb_txn_begin(env, NULL, 1, &txn)); - E(mdb_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - printf("Cursor last\n"); - E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - printf("Cursor prev\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - printf("Cursor last/prev\n"); - E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - E(mdb_cursor_get(cursor, &key, &data, MDB_PREV)); - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - - mdb_txn_abort(txn); - - printf("Deleting with cursor\n"); - E(mdb_txn_begin(env, NULL, 0, &txn)); - E(mdb_cursor_open(txn, dbi, &cur2)); - for (i=0; i<50; i++) { - if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT))) - break; - printf("key: %p %.*s, data: %p %.*s\n", - key.mv_data, (int) key.mv_size, (char *) key.mv_data, - data.mv_data, (int) data.mv_size, (char *) data.mv_data); - E(mdb_del(txn, dbi, &key, NULL)); - } - - printf("Restarting cursor in txn\n"); - for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { - if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op))) - break; - printf("key: %p %.*s, data: %p %.*s\n", - key.mv_data, (int) key.mv_size, (char *) key.mv_data, - data.mv_data, (int) data.mv_size, (char *) data.mv_data); - } - mdb_cursor_close(cur2); - E(mdb_txn_commit(txn)); - - printf("Restarting cursor outside txn\n"); - E(mdb_txn_begin(env, NULL, 0, &txn)); - E(mdb_cursor_open(txn, dbi, &cursor)); - for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { - if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op))) - break; - printf("key: %p %.*s, data: %p %.*s\n", - key.mv_data, (int) key.mv_size, (char *) key.mv_data, - data.mv_data, (int) data.mv_size, (char *) data.mv_data); - } - mdb_cursor_close(cursor); - mdb_close(env, dbi); - - mdb_txn_abort(txn); - mdb_env_close(env); - - return 0; -} diff --git a/submodules/lmdb/libraries/liblmdb/mtest2.c b/submodules/lmdb/libraries/liblmdb/mtest2.c deleted file mode 100644 index f1a3dbd6c8..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mtest2.c +++ /dev/null @@ -1,123 +0,0 @@ -/* mtest2.c - memory-mapped database tester/toy */ -/* - * Copyright 2011-2014 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Just like mtest.c, but using a subDB instead of the main DB */ - -#include -#include -#include -#include "lmdb.h" - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ - "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) - -int main(int argc,char * argv[]) -{ - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDB_stat mst; - MDB_cursor *cursor; - int count; - int *values; - char sval[32] = ""; - - srand(time(NULL)); - - count = (rand()%384) + 64; - values = (int *)malloc(count*sizeof(int)); - - for(i = 0;i -1; i-= (rand()%5)) { - j++; - txn=NULL; - E(mdb_txn_begin(env, NULL, 0, &txn)); - sprintf(sval, "%03x ", values[i]); - if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { - j--; - mdb_txn_abort(txn); - } else { - E(mdb_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdb_env_stat(env, &mst)); - E(mdb_txn_begin(env, NULL, 1, &txn)); - E(mdb_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - mdb_cursor_close(cursor); - mdb_close(env, dbi); - - mdb_txn_abort(txn); - mdb_env_close(env); - - return 0; -} diff --git a/submodules/lmdb/libraries/liblmdb/mtest3.c b/submodules/lmdb/libraries/liblmdb/mtest3.c deleted file mode 100644 index f705c52dec..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mtest3.c +++ /dev/null @@ -1,133 +0,0 @@ -/* mtest3.c - memory-mapped database tester/toy */ -/* - * Copyright 2011-2014 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Tests for sorted duplicate DBs */ -#include -#include -#include -#include -#include "lmdb.h" - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ - "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) - -int main(int argc,char * argv[]) -{ - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDB_stat mst; - MDB_cursor *cursor; - int count; - int *values; - char sval[32]; - char kval[sizeof(int)]; - - srand(time(NULL)); - - memset(sval, 0, sizeof(sval)); - - count = (rand()%384) + 64; - values = (int *)malloc(count*sizeof(int)); - - for(i = 0;i -1; i-= (rand()%5)) { - j++; - txn=NULL; - E(mdb_txn_begin(env, NULL, 0, &txn)); - sprintf(kval, "%03x", values[i & ~0x0f]); - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { - j--; - mdb_txn_abort(txn); - } else { - E(mdb_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdb_env_stat(env, &mst)); - E(mdb_txn_begin(env, NULL, 1, &txn)); - E(mdb_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - mdb_cursor_close(cursor); - mdb_close(env, dbi); - - mdb_txn_abort(txn); - mdb_env_close(env); - - return 0; -} diff --git a/submodules/lmdb/libraries/liblmdb/mtest4.c b/submodules/lmdb/libraries/liblmdb/mtest4.c deleted file mode 100644 index da5a953044..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mtest4.c +++ /dev/null @@ -1,168 +0,0 @@ -/* mtest4.c - memory-mapped database tester/toy */ -/* - * Copyright 2011-2014 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Tests for sorted duplicate DBs with fixed-size keys */ -#include -#include -#include -#include -#include "lmdb.h" - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ - "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) - -int main(int argc,char * argv[]) -{ - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDB_stat mst; - MDB_cursor *cursor; - int count; - int *values; - char sval[8]; - char kval[sizeof(int)]; - - memset(sval, 0, sizeof(sval)); - - count = 510; - values = (int *)malloc(count*sizeof(int)); - - for(i = 0;i -1; i-= (rand()%3)) { - j++; - txn=NULL; - E(mdb_txn_begin(env, NULL, 0, &txn)); - sprintf(sval, "%07x", values[i]); - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { - j--; - mdb_txn_abort(txn); - } else { - E(mdb_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdb_env_stat(env, &mst)); - E(mdb_txn_begin(env, NULL, 1, &txn)); - E(mdb_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - mdb_cursor_close(cursor); - mdb_close(env, dbi); - - mdb_txn_abort(txn); - mdb_env_close(env); - - return 0; -} diff --git a/submodules/lmdb/libraries/liblmdb/mtest5.c b/submodules/lmdb/libraries/liblmdb/mtest5.c deleted file mode 100644 index 39a8c728ac..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mtest5.c +++ /dev/null @@ -1,135 +0,0 @@ -/* mtest5.c - memory-mapped database tester/toy */ -/* - * Copyright 2011-2014 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Tests for sorted duplicate DBs using cursor_put */ -#include -#include -#include -#include -#include "lmdb.h" - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ - "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) - -int main(int argc,char * argv[]) -{ - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDB_stat mst; - MDB_cursor *cursor; - int count; - int *values; - char sval[32]; - char kval[sizeof(int)]; - - srand(time(NULL)); - - memset(sval, 0, sizeof(sval)); - - count = (rand()%384) + 64; - values = (int *)malloc(count*sizeof(int)); - - for(i = 0;i -1; i-= (rand()%5)) { - j++; - txn=NULL; - E(mdb_txn_begin(env, NULL, 0, &txn)); - sprintf(kval, "%03x", values[i & ~0x0f]); - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { - j--; - mdb_txn_abort(txn); - } else { - E(mdb_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdb_env_stat(env, &mst)); - E(mdb_txn_begin(env, NULL, 1, &txn)); - E(mdb_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - mdb_cursor_close(cursor); - mdb_close(env, dbi); - - mdb_txn_abort(txn); - mdb_env_close(env); - - return 0; -} diff --git a/submodules/lmdb/libraries/liblmdb/mtest6.c b/submodules/lmdb/libraries/liblmdb/mtest6.c deleted file mode 100644 index 07d5758932..0000000000 --- a/submodules/lmdb/libraries/liblmdb/mtest6.c +++ /dev/null @@ -1,137 +0,0 @@ -/* mtest6.c - memory-mapped database tester/toy */ -/* - * Copyright 2011-2014 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Tests for DB splits and merges */ -#include -#include -#include -#include -#include "lmdb.h" - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ - "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) - -char dkbuf[1024]; - -int main(int argc,char * argv[]) -{ - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDB_stat mst; - MDB_cursor *cursor; - int count; - int *values; - long kval; - char *sval; - - srand(time(NULL)); - - E(mdb_env_create(&env)); - E(mdb_env_set_mapsize(env, 10485760)); - E(mdb_env_set_maxdbs(env, 4)); - E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); - E(mdb_txn_begin(env, NULL, 0, &txn)); - E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi)); - E(mdb_cursor_open(txn, dbi, &cursor)); - E(mdb_stat(txn, dbi, &mst)); - - sval = calloc(1, mst.ms_psize / 4); - key.mv_size = sizeof(long); - key.mv_data = &kval; - data.mv_size = mst.ms_psize / 4 - 30; - data.mv_data = sval; - - printf("Adding 12 values, should yield 3 splits\n"); - for (i=0;i<12;i++) { - kval = i*5; - sprintf(sval, "%08x", kval); - (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); - } - printf("Adding 12 more values, should yield 3 splits\n"); - for (i=0;i<12;i++) { - kval = i*5+4; - sprintf(sval, "%08x", kval); - (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); - } - printf("Adding 12 more values, should yield 3 splits\n"); - for (i=0;i<12;i++) { - kval = i*5+1; - sprintf(sval, "%08x", kval); - (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); - } - E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST)); - - do { - printf("key: %p %s, data: %p %.*s\n", - key.mv_data, mdb_dkey(&key, dkbuf), - data.mv_data, (int) data.mv_size, (char *) data.mv_data); - } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0); - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - mdb_cursor_close(cursor); - mdb_txn_commit(txn); - -#if 0 - j=0; - - for (i= count - 1; i > -1; i-= (rand()%5)) { - j++; - txn=NULL; - E(mdb_txn_begin(env, NULL, 0, &txn)); - sprintf(kval, "%03x", values[i & ~0x0f]); - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { - j--; - mdb_txn_abort(txn); - } else { - E(mdb_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdb_env_stat(env, &mst)); - E(mdb_txn_begin(env, NULL, 1, &txn)); - E(mdb_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); - mdb_cursor_close(cursor); - mdb_close(env, dbi); - - mdb_txn_abort(txn); -#endif - mdb_env_close(env); - - return 0; -} diff --git a/submodules/lmdb/libraries/liblmdb/sample-bdb.txt b/submodules/lmdb/libraries/liblmdb/sample-bdb.txt deleted file mode 100644 index 6a959bd685..0000000000 --- a/submodules/lmdb/libraries/liblmdb/sample-bdb.txt +++ /dev/null @@ -1,73 +0,0 @@ -/* sample-bdb.txt - BerkeleyDB toy/sample - * - * Do a line-by-line comparison of this and sample-mdb.txt - */ -/* - * Copyright 2012 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ -#include -#include -#include - -int main(int argc,char * argv[]) -{ - int rc; - DB_ENV *env; - DB *dbi; - DBT key, data; - DB_TXN *txn; - DBC *cursor; - char sval[32], kval[32]; - - /* Note: Most error checking omitted for simplicity */ - -#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD) - rc = db_env_create(&env, 0); - rc = env->open(env, "./testdb", FLAGS, 0664); - rc = db_create(&dbi, env, 0); - rc = env->txn_begin(env, NULL, &txn, 0); - rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664); - - memset(&key, 0, sizeof(DBT)); - memset(&data, 0, sizeof(DBT)); - key.size = sizeof(int); - key.data = sval; - data.size = sizeof(sval); - data.data = sval; - - sprintf(sval, "%03x %d foo bar", 32, 3141592); - rc = dbi->put(dbi, txn, &key, &data, 0); - rc = txn->commit(txn, 0); - if (rc) { - fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc)); - goto leave; - } - rc = env->txn_begin(env, NULL, &txn, 0); - rc = dbi->cursor(dbi, txn, &cursor, 0); - key.flags = DB_DBT_USERMEM; - key.data = kval; - key.ulen = sizeof(kval); - data.flags = DB_DBT_USERMEM; - data.data = sval; - data.ulen = sizeof(sval); - while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", - key.data, (int) key.size, (char *) key.data, - data.data, (int) data.size, (char *) data.data); - } - rc = cursor->c_close(cursor); - rc = txn->abort(txn); -leave: - rc = dbi->close(dbi, 0); - rc = env->close(env, 0); - return rc; -} diff --git a/submodules/lmdb/libraries/liblmdb/sample-mdb.txt b/submodules/lmdb/libraries/liblmdb/sample-mdb.txt deleted file mode 100644 index a233ec5dd3..0000000000 --- a/submodules/lmdb/libraries/liblmdb/sample-mdb.txt +++ /dev/null @@ -1,62 +0,0 @@ -/* sample-mdb.txt - MDB toy/sample - * - * Do a line-by-line comparison of this and sample-bdb.txt - */ -/* - * Copyright 2012 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ -#include -#include "lmdb.h" - -int main(int argc,char * argv[]) -{ - int rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDB_cursor *cursor; - char sval[32]; - - /* Note: Most error checking omitted for simplicity */ - - rc = mdb_env_create(&env); - rc = mdb_env_open(env, "./testdb", 0, 0664); - rc = mdb_txn_begin(env, NULL, 0, &txn); - rc = mdb_open(txn, NULL, 0, &dbi); - - key.mv_size = sizeof(int); - key.mv_data = sval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - - sprintf(sval, "%03x %d foo bar", 32, 3141592); - rc = mdb_put(txn, dbi, &key, &data, 0); - rc = mdb_txn_commit(txn); - if (rc) { - fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc)); - goto leave; - } - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - rc = mdb_cursor_open(txn, dbi, &cursor); - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", - key.mv_data, (int) key.mv_size, (char *) key.mv_data, - data.mv_data, (int) data.mv_size, (char *) data.mv_data); - } - mdb_cursor_close(cursor); - mdb_txn_abort(txn); -leave: - mdb_close(env, dbi); - mdb_env_close(env); - return 0; -} diff --git a/submodules/lmdb/libraries/liblmdb/tooltag b/submodules/lmdb/libraries/liblmdb/tooltag deleted file mode 100644 index 229bf16baa..0000000000 --- a/submodules/lmdb/libraries/liblmdb/tooltag +++ /dev/null @@ -1,22 +0,0 @@ - - - mdb_copy_1 - mdb_copy - environment copy tool - mdb_copy.1 - - - mdb_dump_1 - mdb_dump - environment export tool - mdb_dump.1 - - - mdb_load_1 - mdb_load - environment import tool - mdb_load.1 - - - mdb_stat_1 - mdb_stat - environment status tool - mdb_stat.1 - - From 39d571aa3ce6cff2b437f4b98cc970b150a94d11 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 6 Sep 2016 00:06:27 +0300 Subject: [PATCH 043/427] no message --- .gitmodules | 3 --- submodules/SQLite.swift | 1 - 2 files changed, 4 deletions(-) delete mode 160000 submodules/SQLite.swift diff --git a/.gitmodules b/.gitmodules index fe42ab59be..f6409150e8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "submodules/SQLite.swift"] - path = submodules/SQLite.swift - url = git@github.com:peter-iakovlev/SQLite.swift.git [submodule "submodules/lmdb"] path = submodules/lmdb url = https://github.com/LMDB/lmdb.git diff --git a/submodules/SQLite.swift b/submodules/SQLite.swift deleted file mode 160000 index 95e5898eb3..0000000000 --- a/submodules/SQLite.swift +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 95e5898eb392626957d0cb25662c98c38c6b4677 From c6034ac794c65d562a89117c00adf453a54ddaf3 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 7 Oct 2016 19:14:00 +0300 Subject: [PATCH 044/427] no message --- Postbox.xcodeproj/project.pbxproj | 28 +++++++ Postbox/CachedMediaResource.swift | 4 + Postbox/CachedPeerData.swift | 6 ++ Postbox/CachedPeerDataTable.swift | 55 +++++++++++++ Postbox/ChatListView.swift | 68 +++++++++++----- Postbox/MediaBox.swift | 26 +++++- Postbox/MessageHistoryReadStateTable.swift | 2 +- Postbox/MessageHistoryView.swift | 51 ++++++++---- Postbox/PeerNotificationSettings.swift | 4 + Postbox/PeerNotificationSettingsTable.swift | 54 +++++++++++++ Postbox/PeerReadState.swift | 44 +++++++++- Postbox/PeerStatusTable.swift | 54 +++++++++++++ Postbox/PeerView.swift | 90 +++++++++++++++++++++ Postbox/Postbox.swift | 85 ++++++++++++++++--- Postbox/PostboxTransaction.swift | 13 ++- Postbox/SqliteValueBox.swift | 4 +- Postbox/ViewTracker.swift | 59 ++++++++++++-- 17 files changed, 590 insertions(+), 57 deletions(-) create mode 100644 Postbox/CachedMediaResource.swift create mode 100644 Postbox/CachedPeerData.swift create mode 100644 Postbox/CachedPeerDataTable.swift create mode 100644 Postbox/PeerNotificationSettings.swift create mode 100644 Postbox/PeerNotificationSettingsTable.swift create mode 100644 Postbox/PeerStatusTable.swift create mode 100644 Postbox/PeerView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 2f8f41cb2a..4cf1bc56bc 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -16,6 +16,11 @@ D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; D02EB8071D2B07F300D07ED3 /* OrderStatisticTreeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */; }; + D03120F81DA53FF4006A2A60 /* PeerStatusTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerStatusTable.swift */; }; + D03120FA1DA540F0006A2A60 /* CachedPeerData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F91DA540F0006A2A60 /* CachedPeerData.swift */; }; + D03120FC1DA55427006A2A60 /* PeerNotificationSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */; }; + D03120FE1DA562E9006A2A60 /* CachedPeerDataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */; }; + D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */; }; D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; D03BCCF81C73561C0097A291 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; @@ -134,6 +139,8 @@ D0D511041D64D91C00A97B8A /* IpcNotifier.h in Headers */ = {isa = PBXBuildFile; fileRef = D0D511031D64D75200A97B8A /* IpcNotifier.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */; }; D0D949F51D35353900740E02 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; + D0DE76F91D9293B8002B8809 /* CachedMediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */; }; + D0DF0C8F1D81A350008AEB01 /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DF0C8E1D81A350008AEB01 /* PeerView.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; @@ -177,6 +184,11 @@ D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTreeTests.swift; sourceTree = ""; }; + D03120F71DA53FF4006A2A60 /* PeerStatusTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerStatusTable.swift; sourceTree = ""; }; + D03120F91DA540F0006A2A60 /* CachedPeerData.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CachedPeerData.swift; sourceTree = ""; }; + D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNotificationSettings.swift; sourceTree = ""; }; + D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CachedPeerDataTable.swift; sourceTree = ""; }; + D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNotificationSettingsTable.swift; sourceTree = ""; }; D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryUnsentTable.swift; sourceTree = ""; }; D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageHistoryView.swift; sourceTree = ""; }; D03BCCF71C73561C0097A291 /* Table.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Table.swift; sourceTree = ""; }; @@ -231,6 +243,8 @@ D0D511031D64D75200A97B8A /* IpcNotifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = IpcNotifier.h; sourceTree = ""; }; D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessResourceTests.swift; sourceTree = ""; }; D0D949F41D35353900740E02 /* MappedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MappedFile.swift; sourceTree = ""; }; + D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CachedMediaResource.swift; sourceTree = ""; }; + D0DF0C8E1D81A350008AEB01 /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -291,6 +305,7 @@ D055BD321B7D3D2D00F06C0A /* MediaBox.swift */, D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */, D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */, + D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */, D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */, D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */, ); @@ -361,6 +376,9 @@ D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */, D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */, D0F9E8701C5A0E9B00037222 /* PeerTable.swift */, + D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */, + D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */, + D03120F71DA53FF4006A2A60 /* PeerStatusTable.swift */, D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */, D0079F591D592E8B00A27A2C /* ContactTable.swift */, D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */, @@ -382,6 +400,8 @@ D044CA2D1C618373002160FF /* ChatListHole.swift */, D0C674CB1CBB14A700183765 /* PeerReadState.swift */, D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */, + D03120F91DA540F0006A2A60 /* CachedPeerData.swift */, + D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */, ); name = Objects; sourceTree = ""; @@ -421,6 +441,7 @@ D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */, D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */, D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */, + D0DF0C8E1D81A350008AEB01 /* PeerView.swift */, ); name = Views; sourceTree = ""; @@ -740,9 +761,11 @@ D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */, + D0DF0C8F1D81A350008AEB01 /* PeerView.swift in Sources */, D0079F651D5A457A00A27A2C /* ContactPeersView.swift in Sources */, D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */, D08CEFB41D2AD8BE0015D3BC /* RedBlackTree.swift in Sources */, + D03120FE1DA562E9006A2A60 /* CachedPeerDataTable.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */, @@ -765,11 +788,13 @@ D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */, D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, + D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */, D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, D0D949F51D35353900740E02 /* MappedFile.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, D0AB0B8E1D65D49C002C78E7 /* ChatListHolesView.swift in Sources */, + D03120FA1DA540F0006A2A60 /* CachedPeerData.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, @@ -781,11 +806,14 @@ D0FC1A3C1CA3FB310056AE9A /* MediaResourceData.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, + D0DE76F91D9293B8002B8809 /* CachedMediaResource.swift in Sources */, D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */, D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */, + D03120F81DA53FF4006A2A60 /* PeerStatusTable.swift in Sources */, D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */, D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */, D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */, + D03120FC1DA55427006A2A60 /* PeerNotificationSettings.swift in Sources */, D03BCCF81C73561C0097A291 /* Table.swift in Sources */, D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, diff --git a/Postbox/CachedMediaResource.swift b/Postbox/CachedMediaResource.swift new file mode 100644 index 0000000000..464695d4e4 --- /dev/null +++ b/Postbox/CachedMediaResource.swift @@ -0,0 +1,4 @@ + +public protocol CachedMediaResource { + var id: String { get } +} diff --git a/Postbox/CachedPeerData.swift b/Postbox/CachedPeerData.swift new file mode 100644 index 0000000000..3833f5576c --- /dev/null +++ b/Postbox/CachedPeerData.swift @@ -0,0 +1,6 @@ + +public protocol CachedPeerData: Coding { + var peerIds: Set { get } + + func isEqual(to: CachedPeerData) -> Bool +} diff --git a/Postbox/CachedPeerDataTable.swift b/Postbox/CachedPeerDataTable.swift new file mode 100644 index 0000000000..d3db7014f8 --- /dev/null +++ b/Postbox/CachedPeerDataTable.swift @@ -0,0 +1,55 @@ +import Foundation + +final class CachedPeerDataTable: Table { + private let sharedEncoder = Encoder() + private let sharedKey = ValueBoxKey(length: 8) + + private var cachedDatas: [PeerId: CachedPeerData] = [:] + private var updatedPeerIds = Set() + + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) + } + + private func key(_ id: PeerId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: id.toInt64()) + return self.sharedKey + } + + func set(id: PeerId, data: CachedPeerData) { + self.cachedDatas[id] = data + self.updatedPeerIds.insert(id) + } + + func get(_ id: PeerId) -> CachedPeerData? { + if let status = self.cachedDatas[id] { + return status + } + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let data = Decoder(buffer: value).decodeRootObject() as? CachedPeerData { + self.cachedDatas[id] = data + return data + } + } + return nil + } + + override func clearMemoryCache() { + self.cachedDatas.removeAll() + self.updatedPeerIds.removeAll() + } + + override func beforeCommit() { + for peerId in self.updatedPeerIds { + if let data = self.cachedDatas[peerId] { + self.sharedEncoder.reset() + self.sharedEncoder.encodeRootObject(data) + + self.valueBox.set(self.tableId, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + } + } + + self.updatedPeerIds.removeAll() + self.cachedDatas.removeAll() + } +} diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index 165353f36a..2652332ead 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -1,13 +1,13 @@ import Foundation public enum ChatListEntry: Comparable { - case MessageEntry(Message, Int) + case MessageEntry(Message, CombinedPeerReadState?, PeerNotificationSettings?) case HoleEntry(ChatListHole) case Nothing(MessageIndex) public var index: MessageIndex { switch self { - case let .MessageEntry(message, _): + case let .MessageEntry(message, _, _): return MessageIndex(message) case let .HoleEntry(hole): return hole.index @@ -19,8 +19,15 @@ public enum ChatListEntry: Comparable { public func ==(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { if lhs.index == rhs.index { - if case let .MessageEntry(_, lhsCount) = lhs, case let .MessageEntry(_, rhsCount) = rhs { - if lhsCount != rhsCount { + if case let .MessageEntry(_, lhsReadState, lhsSettings) = lhs, case let .MessageEntry(_, rhsReadState, rhsSettings) = rhs { + if lhsReadState != rhsReadState { + return false + } + if let lhsSettings = lhsSettings, let rhsSettings = rhsSettings { + if !lhsSettings.isEqual(to: rhsSettings) { + return false + } + } else if (lhsSettings != nil) != (rhsSettings != nil) { return false } } @@ -34,16 +41,16 @@ public func <(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { } enum MutableChatListEntry: Equatable { - case IntermediateMessageEntry(IntermediateMessage, CombinedPeerReadState?) - case MessageEntry(Message, CombinedPeerReadState?) + case IntermediateMessageEntry(IntermediateMessage, CombinedPeerReadState?, PeerNotificationSettings?) + case MessageEntry(Message, CombinedPeerReadState?, PeerNotificationSettings?) case HoleEntry(ChatListHole) case Nothing(MessageIndex) var index: MessageIndex { switch self { - case let .IntermediateMessageEntry(message, _): + case let .IntermediateMessageEntry(message, _, _): return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .MessageEntry(message, _): + case let .MessageEntry(message, _, _): return MessageIndex(message) case let .HoleEntry(hole): return hole.index @@ -131,12 +138,12 @@ final class MutableChatListView { } } - func replay(_ operations: [ChatListOperation], context: MutableChatListViewReplayContext) -> Bool { + func replay(_ operations: [ChatListOperation], updatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], context: MutableChatListViewReplayContext) -> Bool { var hasChanges = false for operation in operations { switch operation { case let .InsertMessage(message, combinedReadState): - if self.add(.IntermediateMessageEntry(message, combinedReadState)) { + if self.add(.IntermediateMessageEntry(message, combinedReadState, nil)) { hasChanges = true } case let .InsertNothing(index): @@ -157,6 +164,22 @@ final class MutableChatListView { } } } + for i in 0 ..< self.entries.count { + switch self.entries[i] { + case let .IntermediateMessageEntry(message, readState, _): + if let settings = updatedPeerNotificationSettings[message.id.peerId] { + self.entries[i] = .IntermediateMessageEntry(message, readState, settings) + hasChanges = true + } + case let .MessageEntry(message, readState, _): + if let settings = updatedPeerNotificationSettings[message.id.peerId] { + self.entries[i] = .MessageEntry(message, readState, settings) + hasChanges = true + } + default: + continue + } + } return hasChanges } @@ -392,10 +415,19 @@ final class MutableChatListView { return hasChanges } - func render(_ renderMessage: (IntermediateMessage) -> Message) { + func render(_ renderMessage: (IntermediateMessage) -> Message, getPeerNotificationSettings: (PeerId) -> PeerNotificationSettings?) { for i in 0 ..< self.entries.count { - if case let .IntermediateMessageEntry(message, combinedReadState) = self.entries[i] { - self.entries[i] = .MessageEntry(renderMessage(message), combinedReadState) + switch self.entries[i] { + case let .IntermediateMessageEntry(message, combinedReadState, notificationSettings): + let updatedNotificationSettings: PeerNotificationSettings? + if let notificationSettings = notificationSettings { + updatedNotificationSettings = notificationSettings + } else { + updatedNotificationSettings = getPeerNotificationSettings(message.id.peerId) + } + self.entries[i] = .MessageEntry(renderMessage(message), combinedReadState, updatedNotificationSettings) + default: + break } } } @@ -410,14 +442,8 @@ public final class ChatListView { var entries: [ChatListEntry] = [] for entry in mutableView.entries { switch entry { - case let .MessageEntry(message, combinedReadState): - var unreadCount: Int32 = 0 - if let combinedReadState = combinedReadState { - for (_, state) in combinedReadState.states { - unreadCount += state.count - } - } - entries.append(.MessageEntry(message, Int(unreadCount))) + case let .MessageEntry(message, combinedReadState, notificationSettings): + entries.append(.MessageEntry(message, combinedReadState, notificationSettings)) case let .Nothing(index): entries.append(.Nothing(index)) case let .HoleEntry(hole): diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 92cf175b78..e0d92424a7 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -42,6 +42,7 @@ public final class MediaBox { private let statusQueue = Queue() private let concurrentQueue = Queue.concurrentDefaultQueue() private let dataQueue = Queue() + private let cacheQueue = Queue() private var statusContexts: [String: ResourceStatusContext] = [:] private var dataContexts: [String: ResourceDataContext] = [:] @@ -77,7 +78,30 @@ public final class MediaBox { } private func streamingPathForId(_ id: String) -> String { - return "\(self.basePath)/\(id)_stream" + return "\(self.basePath)/\(fileNameForId(id))_stream" + } + + private func cachePathForId(_ id: String) -> String { + return "\(self.basePath)/cache/\(fileNameForId(id))" + } + + public func cachedResource(_ resource: CachedMediaResource) -> Signal { + return Signal { subscriber in + self.concurrentQueue.async { + let path = self.cachePathForId(resource.id) + let currentSize = fileSize(path) + subscriber.putNext(MediaResourceData(path: path, size: currentSize)) + subscriber.putCompletion() + } + + return EmptyDisposable + } + } + + public func cacheResource(_ resource: CachedMediaResource, data: Data) { + self.cacheQueue.async { + let _ = try? data.write(to: URL(fileURLWithPath: self.cachePathForId(resource.id)), options: [.atomic]) + } } public func resourceStatus(_ resource: MediaResource) -> Signal { diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index 166ab72382..a6555d9c6a 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -233,7 +233,7 @@ final class MessageHistoryReadStateTable: Table { func applyOutgoingMaxReadId(_ messageId: MessageId) -> (CombinedPeerReadState?, Bool) { if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] { if state.maxOutgoingReadId < messageId.id { - states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: state.maxIncomingReadId, maxOutgoingReadId: state.maxOutgoingReadId, maxKnownId: state.maxKnownId, count: state.count) + states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: state.maxIncomingReadId, maxOutgoingReadId: messageId.id, maxKnownId: state.maxKnownId, count: state.count) self.updatedPeerIds.insert(messageId.peerId) return (CombinedPeerReadState(states: states.namespaces.map({$0})), false) } diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 87115c3ad9..77230a6353 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -74,12 +74,12 @@ public func ==(lhs: MessageHistoryEntryLocation, rhs: MessageHistoryEntryLocatio } public enum MessageHistoryEntry: Comparable { - case MessageEntry(Message, MessageHistoryEntryLocation?) + case MessageEntry(Message, Bool, MessageHistoryEntryLocation?) case HoleEntry(MessageHistoryHole, MessageHistoryEntryLocation?) public var index: MessageIndex { switch self { - case let .MessageEntry(message, _): + case let .MessageEntry(message, _, _): return MessageIndex(id: message.id, timestamp: message.timestamp) case let .HoleEntry(hole, _): return hole.maxIndex @@ -89,12 +89,12 @@ public enum MessageHistoryEntry: Comparable { public func ==(lhs: MessageHistoryEntry, rhs: MessageHistoryEntry) -> Bool { switch lhs { - case let .MessageEntry(lhsMessage, lhsLocation): + case let .MessageEntry(lhsMessage, lhsRead, lhsLocation): switch rhs { case .HoleEntry: return false - case let .MessageEntry(rhsMessage, rhsLocation): - if MessageIndex(lhsMessage) == MessageIndex(rhsMessage) && lhsMessage.flags == rhsMessage.flags && lhsLocation == rhsLocation { + case let .MessageEntry(rhsMessage, rhsRead, rhsLocation): + if MessageIndex(lhsMessage) == MessageIndex(rhsMessage) && lhsMessage.flags == rhsMessage.flags && lhsLocation == rhsLocation && lhsRead == rhsRead { return true } return false @@ -128,6 +128,7 @@ final class MutableMessageHistoryView { let tagMask: MessageTags? fileprivate var anchorIndex: MessageHistoryAnchorIndex fileprivate let combinedReadState: CombinedPeerReadState? + fileprivate var transientReadState: CombinedPeerReadState? fileprivate var earlier: MutableMessageHistoryEntry? fileprivate var later: MutableMessageHistoryEntry? fileprivate var entries: [MutableMessageHistoryEntry] @@ -137,6 +138,7 @@ final class MutableMessageHistoryView { self.id = id self.anchorIndex = anchorIndex self.combinedReadState = combinedReadState + self.transientReadState = combinedReadState self.earlier = earlier self.entries = entries self.later = later @@ -267,7 +269,7 @@ final class MutableMessageHistoryView { } case let .UpdateReadState(combinedReadState): hasChanges = true - //self.combinedReadState = combinedReadState + self.transientReadState = combinedReadState case let .UpdateEmbeddedMedia(index, embeddedMediaData): for i in 0 ..< self.entries.count { if case let .IntermediateMessageEntry(message, _) = self.entries[i] , MessageIndex(message) == index { @@ -556,14 +558,33 @@ public final class MessageHistoryView { self.anchorIndex = mutableView.anchorIndex.index var entries: [MessageHistoryEntry] = [] - for entry in mutableView.entries { - switch entry { - case let .HoleEntry(hole, location): - entries.append(.HoleEntry(hole, location)) - case let .MessageEntry(message, location): - entries.append(.MessageEntry(message, location)) - case .IntermediateMessageEntry: - assertionFailure("unexpected IntermediateMessageEntry in MessageHistoryView.init()") + if let transientReadState = mutableView.transientReadState { + for entry in mutableView.entries { + switch entry { + case let .HoleEntry(hole, location): + entries.append(.HoleEntry(hole, location)) + case let .MessageEntry(message, location): + let read: Bool + if message.flags.contains(.Incoming) { + read = false + } else { + read = transientReadState.isOutgoingMessageIdRead(message.id) + } + entries.append(.MessageEntry(message, read, location)) + case .IntermediateMessageEntry: + assertionFailure("unexpected IntermediateMessageEntry in MessageHistoryView.init()") + } + } + } else { + for entry in mutableView.entries { + switch entry { + case let .HoleEntry(hole, location): + entries.append(.HoleEntry(hole, location)) + case let .MessageEntry(message, location): + entries.append(.MessageEntry(message, false, location)) + case .IntermediateMessageEntry: + assertionFailure("unexpected IntermediateMessageEntry in MessageHistoryView.init()") + } } } self.entries = entries @@ -597,7 +618,7 @@ public final class MessageHistoryView { } if let _ = maxNamespaceIndex , index + 1 < entries.count { for i in index + 1 ..< entries.count { - if case let .MessageEntry(message, _) = entries[i] , !message.flags.contains(.Incoming) { + if case let .MessageEntry(message, _, _) = entries[i] , !message.flags.contains(.Incoming) { maxNamespaceIndex = MessageIndex(message) } else { break diff --git a/Postbox/PeerNotificationSettings.swift b/Postbox/PeerNotificationSettings.swift new file mode 100644 index 0000000000..32ad5004c5 --- /dev/null +++ b/Postbox/PeerNotificationSettings.swift @@ -0,0 +1,4 @@ + +public protocol PeerNotificationSettings: Coding { + func isEqual(to: PeerNotificationSettings) -> Bool +} diff --git a/Postbox/PeerNotificationSettingsTable.swift b/Postbox/PeerNotificationSettingsTable.swift new file mode 100644 index 0000000000..21a7c891d7 --- /dev/null +++ b/Postbox/PeerNotificationSettingsTable.swift @@ -0,0 +1,54 @@ +import Foundation + +final class PeerNotificationSettingsTable: Table { + private let sharedEncoder = Encoder() + private let sharedKey = ValueBoxKey(length: 8) + + private var cachedSettings: [PeerId: PeerNotificationSettings] = [:] + private var updatedPeerIds = Set() + + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) + } + + private func key(_ id: PeerId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: id.toInt64()) + return self.sharedKey + } + + func set(id: PeerId, settings: PeerNotificationSettings) { + self.cachedSettings[id] = settings + self.updatedPeerIds.insert(id) + } + + func get(_ id: PeerId) -> PeerNotificationSettings? { + if let settings = self.cachedSettings[id] { + return settings + } + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let settings = Decoder(buffer: value).decodeRootObject() as? PeerNotificationSettings { + self.cachedSettings[id] = settings + return settings + } + } + return nil + } + + override func clearMemoryCache() { + self.cachedSettings.removeAll() + self.updatedPeerIds.removeAll() + } + + override func beforeCommit() { + for peerId in self.updatedPeerIds { + if let settings = self.cachedSettings[peerId] { + self.sharedEncoder.reset() + self.sharedEncoder.encodeRootObject(settings) + + self.valueBox.set(self.tableId, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + } + } + + self.updatedPeerIds.removeAll() + } +} diff --git a/Postbox/PeerReadState.swift b/Postbox/PeerReadState.swift index ee3d20ba52..a0fa71469c 100644 --- a/Postbox/PeerReadState.swift +++ b/Postbox/PeerReadState.swift @@ -21,13 +21,53 @@ public func ==(lhs: PeerReadState, rhs: PeerReadState) -> Bool { return lhs.maxIncomingReadId == rhs.maxIncomingReadId && lhs.maxOutgoingReadId == rhs.maxOutgoingReadId && lhs.maxKnownId == rhs.maxKnownId && lhs.count == rhs.count } -public struct CombinedPeerReadState { +public struct CombinedPeerReadState: Equatable { let states: [(MessageId.Namespace, PeerReadState)] - var count: Int32 { + public var count: Int32 { var result: Int32 = 0 for (_, state) in self.states { result += state.count } return result } + + public static func ==(lhs: CombinedPeerReadState, rhs: CombinedPeerReadState) -> Bool { + if lhs.states.count != rhs.states.count { + return false + } + for (lhsNamespace, lhsState) in lhs.states { + var rhsFound = false + inner: for (rhsNamespace, rhsState) in rhs.states { + if rhsNamespace == lhsNamespace { + if lhsState != rhsState { + return false + } + rhsFound = true + break inner + } + } + if !rhsFound { + return false + } + } + return true + } + + public func isOutgoingMessageIdRead(_ id: MessageId) -> Bool { + for (namespace, readState) in self.states { + if namespace == id.namespace { + return readState.maxOutgoingReadId >= id.id + } + } + return false + } + + public func isIncomingMessageIdRead(_ id: MessageId) -> Bool { + for (namespace, readState) in self.states { + if namespace == id.namespace { + return readState.maxIncomingReadId >= id.id + } + } + return false + } } diff --git a/Postbox/PeerStatusTable.swift b/Postbox/PeerStatusTable.swift new file mode 100644 index 0000000000..ab50d10809 --- /dev/null +++ b/Postbox/PeerStatusTable.swift @@ -0,0 +1,54 @@ +import Foundation + +final class PeerStatusTable: Table { + private let sharedEncoder = Encoder() + private let sharedKey = ValueBoxKey(length: 8) + + private var cachedStatuses: [PeerId: Coding] = [:] + private var updatedPeerIds = Set() + + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) + } + + private func key(_ id: PeerId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: id.toInt64()) + return self.sharedKey + } + + func set(id: PeerId, status: Coding) { + self.cachedStatuses[id] = status + self.updatedPeerIds.insert(id) + } + + func get(_ id: PeerId) -> Coding? { + if let status = self.cachedStatuses[id] { + return status + } + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let status = Decoder(buffer: value).decodeRootObject() { + self.cachedStatuses[id] = status + return status + } + } + return nil + } + + override func clearMemoryCache() { + self.cachedStatuses.removeAll() + self.updatedPeerIds.removeAll() + } + + override func beforeCommit() { + for peerId in self.updatedPeerIds { + if let status = self.cachedStatuses[peerId] { + self.sharedEncoder.reset() + self.sharedEncoder.encodeRootObject(status) + + self.valueBox.set(self.tableId, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + } + } + + self.updatedPeerIds.removeAll() + } +} diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift new file mode 100644 index 0000000000..39d004e2cb --- /dev/null +++ b/Postbox/PeerView.swift @@ -0,0 +1,90 @@ +import Foundation + +final class MutablePeerView { + let peerId: PeerId + var notificationSettings: PeerNotificationSettings? + var cachedData: CachedPeerData? + var peers: [PeerId: Peer] = [:] + + init(peerId: PeerId, notificationSettings: PeerNotificationSettings?, cachedData: CachedPeerData?, getPeer: (PeerId) -> Peer?) { + self.peerId = peerId + self.notificationSettings = notificationSettings + self.cachedData = cachedData + var peerIds = Set() + peerIds.insert(peerId) + if let cachedData = cachedData { + peerIds.formUnion(cachedData.peerIds) + } + for id in peerIds { + if let peer = getPeer(id) { + self.peers[id] = peer + } + } + } + + func replay(updatedPeers: [PeerId: Peer], updatedNotificationSettings: [PeerId: PeerNotificationSettings], updatedCachedPeerData: [PeerId: CachedPeerData], getPeer: (PeerId) -> Peer?) -> Bool { + var updated = false + + if let cachedData = updatedCachedPeerData[self.peerId], self.cachedData == nil || self.cachedData!.peerIds != cachedData.peerIds { + self.cachedData = cachedData + updated = true + + var peerIds = Set() + peerIds.insert(self.peerId) + peerIds.formUnion(cachedData.peerIds) + + for id in peerIds { + if let peer = updatedPeers[id] { + self.peers[id] = peer + } else if let peer = getPeer(id) { + self.peers[id] = peer + } + } + + var removePeerIds: [PeerId] = [] + for peerId in self.peers.keys { + if !peerIds.contains(peerId) { + removePeerIds.append(peerId) + } + } + + for peerId in removePeerIds { + self.peers.removeValue(forKey: peerId) + } + } else { + var peerIds = Set() + peerIds.insert(self.peerId) + if let cachedData = self.cachedData { + peerIds.formUnion(cachedData.peerIds) + } + + for id in peerIds { + if let peer = updatedPeers[id] { + self.peers[id] = peer + updated = true + } + } + } + + if let notificationSettings = updatedNotificationSettings[self.peerId] { + self.notificationSettings = notificationSettings + updated = true + } + + return updated + } +} + +public final class PeerView { + public let peerId: PeerId + public let cachedData: CachedPeerData? + public let notificationSettings: PeerNotificationSettings? + public let peers: [PeerId: Peer] + + init(_ mutableView: MutablePeerView) { + self.peerId = mutableView.peerId + self.cachedData = mutableView.cachedData + self.notificationSettings = mutableView.notificationSettings + self.peers = mutableView.peers + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index ec26066924..2dc54776f8 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -94,10 +94,22 @@ public final class Modifier { return self.postbox?.readStateTable.getCombinedState(id)?.states } + public func getPeerNotificationSettings(_ id: PeerId) -> PeerNotificationSettings? { + return self.postbox?.peerNotificationSettingsTable.get(id) + } + public func updatePeers(_ peers: [Peer], update: (Peer, Peer) -> Peer?) { self.postbox?.updatePeers(peers, update: update) } + public func updatePeerNotificationSettings(_ notificationSettings: [PeerId: PeerNotificationSettings]) { + self.postbox?.updatePeerNotificationSettings(notificationSettings) + } + + public func updatePeerCachedData(peerIds: Set, update: (PeerId, CachedPeerData?) -> CachedPeerData?) { + self.postbox?.updatePeerCachedData(peerIds: peerIds, update: update) + } + public func replaceContactPeerIds(_ peerIds: Set) { self.postbox?.replaceContactPeerIds(peerIds) } @@ -180,6 +192,9 @@ public final class Postbox { private var currentRemovedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] = [:] private var currentFilledHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] = [:] private var currentUpdatedPeers: [PeerId: Peer] = [:] + private var currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings] = [:] + private var currentUpdatedCachedPeerData: [PeerId: CachedPeerData] = [:] + private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? private var currentUpdatedMasterClientId: Int64? @@ -200,6 +215,9 @@ public final class Postbox { var metadataTable: MetadataTable! var keychainTable: KeychainTable! var peerTable: PeerTable! + var peerNotificationSettingsTable: PeerNotificationSettingsTable! + var cachedPeerDataTable: CachedPeerDataTable! + var peerStatusTable: PeerStatusTable! var globalMessageIdsTable: GlobalMessageIdsTable! var messageHistoryIndexTable: MessageHistoryIndexTable! var messageHistoryTable: MessageHistoryTable! @@ -296,7 +314,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 8 + let currentUserVersion: Int32 = 10 if userVersion != currentUserVersion { self.valueBox.drop() @@ -320,6 +338,9 @@ public final class Postbox { self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, tableId: 13) self.contactsTable = ContactTable(valueBox: self.valueBox, tableId: 16) self.peerRatingTable = RatingTable(valueBox: self.valueBox, tableId: 17) + self.cachedPeerDataTable = CachedPeerDataTable(valueBox: self.valueBox, tableId: 18) + self.peerNotificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox, tableId: 19) + self.peerStatusTable = PeerStatusTable(valueBox: self.valueBox, tableId: 20) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -338,11 +359,18 @@ public final class Postbox { self.tables.append(self.peerChatStateTable) self.tables.append(self.contactsTable) self.tables.append(self.peerRatingTable) + self.tables.append(self.peerNotificationSettingsTable) + self.tables.append(self.cachedPeerDataTable) + self.tables.append(self.peerStatusTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, fetchAnchorIndex: self.fetchAnchorIndex, renderMessage: self.renderIntermediateMessage, getPeer: { peerId in return self.peerTable.get(peerId) + }, getPeerNotificationSettings: { peerId in + return self.peerNotificationSettingsTable.get(peerId) + }, getCachedPeerData: { peerId in + return self.cachedPeerDataTable.get(peerId) }, unsentMessageIndices: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") @@ -611,7 +639,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId))) + entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId))) case let .Hole(hole): entries.append(.HoleEntry(hole)) case let .Nothing(index): @@ -622,7 +650,7 @@ public final class Postbox { if let intermediateLower = intermediateLower { switch intermediateLower { case let .Message(message): - lower = .IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId)) + lower = .IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId)) case let .Hole(hole): lower = .HoleEntry(hole) case let .Nothing(index): @@ -633,7 +661,7 @@ public final class Postbox { if let intermediateUpper = intermediateUpper { switch intermediateUpper { case let .Message(message): - upper = .IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId)) + upper = .IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId)) case let .Hole(hole): upper = .HoleEntry(hole) case let .Nothing(index): @@ -650,7 +678,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId))) + entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId))) case let .Hole(hole): entries.append(.HoleEntry(hole)) case let .Nothing(index): @@ -666,7 +694,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId))) + entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId))) case let .Nothing(index): entries.append(.Nothing(index)) case let .Hole(index): @@ -708,7 +736,7 @@ public final class Postbox { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } - let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -732,6 +760,8 @@ public final class Postbox { self.currentUpdatedMedia.removeAll() self.currentReplacedContactPeerIds = nil self.currentUpdatedMasterClientId = nil + self.currentUpdatedPeerNotificationSettings.removeAll() + self.currentUpdatedCachedPeerData.removeAll() for table in self.tables { table.beforeCommit() @@ -763,6 +793,26 @@ public final class Postbox { } } + fileprivate func updatePeerNotificationSettings(_ notificationSettings: [PeerId: PeerNotificationSettings]) { + for (peerId, settings) in notificationSettings { + let currentSettings = self.peerNotificationSettingsTable.get(peerId) + if currentSettings == nil || !(currentSettings!.isEqual(to: settings)) { + self.peerNotificationSettingsTable.set(id: peerId, settings: settings) + self.currentUpdatedPeerNotificationSettings[peerId] = settings + } + } + } + + fileprivate func updatePeerCachedData(peerIds: Set, update: (PeerId, CachedPeerData?) -> CachedPeerData?) { + for peerId in peerIds { + let currentData = self.cachedPeerDataTable.get(peerId) + if let updatedData = update(peerId, currentData) { + self.cachedPeerDataTable.set(id: peerId, data: updatedData) + self.currentUpdatedCachedPeerData[peerId] = updatedData + } + } + } + fileprivate func replaceContactPeerIds(_ peerIds: Set) { self.contactsTable.replace(peerIds) @@ -937,7 +987,7 @@ public final class Postbox { let (entries, earlier, later) = self.fetchAroundChatEntries(index, count: count) let mutableView = MutableChatListView(earlier: earlier, entries: entries, later: later, count: count) - mutableView.render(self.renderIntermediateMessage) + mutableView.render(self.renderIntermediateMessage, getPeerNotificationSettings: { self.peerNotificationSettingsTable.get($0) }) let (index, signal) = self.viewTracker.addChatListView(mutableView) @@ -1012,7 +1062,24 @@ public final class Postbox { } |> switchToLatest } - public func peerWithId(_ id: PeerId) -> Signal { + public func peerView(id: PeerId) -> Signal { + return self.modify { modifier -> Signal in + let view = MutablePeerView(peerId: id, notificationSettings: self.peerNotificationSettingsTable.get(id), cachedData: self.cachedPeerDataTable.get(id), getPeer: { self.peerTable.get($0) }) + let (index, signal) = self.viewTracker.addPeerView(view) + + return (.single(PeerView(view)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removePeerView(index) + } + } + } + } |> switchToLatest + } + + public func loadedPeerWithId(_ id: PeerId) -> Signal { return self.modify { modifier -> Signal in if let peer = self.peerTable.get(id) { return .single(peer) diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index 764c343f5a..4ef39497fc 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -6,6 +6,9 @@ final class PostboxTransaction { let removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] let chatListOperations: [ChatListOperation] let currentUpdatedPeers: [PeerId: Peer] + let currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings] + let currentUpdatedCachedPeerData: [PeerId: CachedPeerData] + let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] let updatedMedia: [MediaId: Media?] @@ -28,6 +31,12 @@ final class PostboxTransaction { if !currentUpdatedPeers.isEmpty { return false } + if !currentUpdatedPeerNotificationSettings.isEmpty { + return false + } + if !currentUpdatedCachedPeerData.isEmpty { + return false + } if !unsentMessageOperations.isEmpty { return false } @@ -46,12 +55,14 @@ final class PostboxTransaction { return true } - init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles self.removedHolesByPeerId = removedHolesByPeerId self.chatListOperations = chatListOperations self.currentUpdatedPeers = currentUpdatedPeers + self.currentUpdatedPeerNotificationSettings = currentUpdatedPeerNotificationSettings; + self.currentUpdatedCachedPeerData = currentUpdatedCachedPeerData self.unsentMessageOperations = unsentMessageOperations self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations self.updatedMedia = updatedMedia diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 79d90bdf20..12ffba1c27 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -121,7 +121,7 @@ public final class SqliteValueBox: ValueBox { //database.execute("PRAGMA wal_autocheckpoint=200") database.execute("PRAGMA journal_size_limit=1536") - var statement: OpaquePointer? = nil + /*var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, "PRAGMA integrity_check", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) while preparedStatement.step() { @@ -132,7 +132,7 @@ public final class SqliteValueBox: ValueBox { //let value = preparedStatement.stringAt(0) //print("integrity_check: \(value)") } - preparedStatement.destroy() + preparedStatement.destroy()*/ sqlite3_busy_timeout(database.handle, 10000000) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index d994ce8436..c339fa0f00 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -21,6 +21,8 @@ final class ViewTracker { private let fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex? private let renderMessage: (IntermediateMessage) -> Message private let getPeer: (PeerId) -> Peer? + private let getPeerNotificationSettings: (PeerId) -> PeerNotificationSettings? + private let getCachedPeerData: (PeerId) -> CachedPeerData? private var chatListViews = Bag<(MutableChatListView, ValuePipe<(ChatListView, ViewUpdateType)>)>() private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] @@ -39,7 +41,9 @@ final class ViewTracker { private var synchronizeReadStatesView: MutableSynchronizePeerReadStatesView private let synchronizePeerReadStatesViewSubscribers = Bag>() - init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, unsentMessageIndices: [MessageIndex], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + private var peerViews = Bag<(MutablePeerView, ValuePipe)>() + + init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, unsentMessageIndices: [MessageIndex], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -48,6 +52,8 @@ final class ViewTracker { self.fetchAnchorIndex = fetchAnchorIndex self.renderMessage = renderMessage self.getPeer = getPeer + self.getPeerNotificationSettings = getPeerNotificationSettings + self.getCachedPeerData = getCachedPeerData self.unsentMessageView = UnsentMessageHistoryView(indices: unsentMessageIndices) self.synchronizeReadStatesView = MutableSynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) @@ -146,6 +152,17 @@ final class ViewTracker { } } + func addPeerView(_ view: MutablePeerView) -> (Bag<(MutablePeerView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.peerViews.add(record) + + return (index, record.1.signal()) + } + + func removePeerView(_ index: Bag<(MutablePeerView, ValuePipe)>.Index) { + self.peerViews.remove(index) + } + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: MessageIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsendMessageIndices: () -> [MessageIndex], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] @@ -164,7 +181,7 @@ final class ViewTracker { for (mutableView, pipe) in self.chatListViews.copyItems() { if mutableView.refreshDueToExternalTransaction(fetchAroundChatEntries: fetchAroundChatEntries) { - mutableView.render(self.renderMessage) + mutableView.render(self.renderMessage, getPeerNotificationSettings: self.getPeerNotificationSettings) pipe.putNext((ChatListView(mutableView), .Generic)) } } @@ -180,6 +197,32 @@ final class ViewTracker { if self.synchronizeReadStatesView.refreshDueToExternalTransaction(fetchSynchronizePeerReadStateOperations: fetchSynchronizePeerReadStateOperations) { self.synchronizeReadStateViewUpdated() } + + for (mutableView, pipe) in self.peerViews.copyItems() { + var updatedPeers: [PeerId: Peer] = [:] + if let peer = self.getPeer(mutableView.peerId) { + updatedPeers[peer.id] = peer + } + + var updatedNotificationSettings: [PeerId: PeerNotificationSettings] = [:] + if let notificationSettings = self.getPeerNotificationSettings(mutableView.peerId) { + updatedNotificationSettings[mutableView.peerId] = notificationSettings + } + + var updatedCachedPeerData: [PeerId: CachedPeerData] = [:] + if let cachedPeerData = self.getCachedPeerData(mutableView.peerId) { + updatedCachedPeerData[mutableView.peerId] = cachedPeerData + for peerIds in cachedPeerData.peerIds { + if let peer = self.getPeer(mutableView.peerId) { + updatedPeers[peer.id] = peer + } + } + } + + if mutableView.replay(updatedPeers: updatedPeers, updatedNotificationSettings: updatedNotificationSettings, updatedCachedPeerData: updatedCachedPeerData, getPeer: self.getPeer) { + pipe.putNext(PeerView(mutableView)) + } + } } func updateViews(transaction: PostboxTransaction) { @@ -232,12 +275,12 @@ final class ViewTracker { } } - if transaction.chatListOperations.count != 0 { + if !transaction.chatListOperations.isEmpty || !transaction.currentUpdatedPeerNotificationSettings.isEmpty { for (mutableView, pipe) in self.chatListViews.copyItems() { let context = MutableChatListViewReplayContext() - if mutableView.replay(transaction.chatListOperations, context: context) { + if mutableView.replay(transaction.chatListOperations, updatedPeerNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, context: context) { mutableView.complete(context: context, fetchEarlier: self.fetchEarlierChatEntries, fetchLater: self.fetchLaterChatEntries) - mutableView.render(self.renderMessage) + mutableView.render(self.renderMessage, getPeerNotificationSettings: self.getPeerNotificationSettings) pipe.putNext((ChatListView(mutableView), .Generic)) } } @@ -270,6 +313,12 @@ final class ViewTracker { } } } + + for (mutableView, pipe) in self.peerViews.copyItems() { + if mutableView.replay(updatedPeers: transaction.currentUpdatedPeers, updatedNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, updatedCachedPeerData: transaction.currentUpdatedCachedPeerData, getPeer: self.getPeer) { + pipe.putNext(PeerView(mutableView)) + } + } } private func updateTrackedChatListHoles() { From 40183e92e6ba4d9faa2e78c267aa8deafff84297 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Oct 2016 21:55:35 +0200 Subject: [PATCH 045/427] no message --- Postbox/ViewTracker.swift | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index c339fa0f00..2d427ecc33 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -281,6 +281,13 @@ final class ViewTracker { if mutableView.replay(transaction.chatListOperations, updatedPeerNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, context: context) { mutableView.complete(context: context, fetchEarlier: self.fetchEarlierChatEntries, fetchLater: self.fetchLaterChatEntries) mutableView.render(self.renderMessage, getPeerNotificationSettings: self.getPeerNotificationSettings) + var updateType: ViewUpdateType = .Generic + for operation in transaction.chatListOperations { + if case .RemoveHoles = operation { + updateType = .UpdateVisible + break + } + } pipe.putNext((ChatListView(mutableView), .Generic)) } } From 12de311c0f8f52630f4f0615105b7c4091f51109 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 10 Oct 2016 11:09:32 +0200 Subject: [PATCH 046/427] no message --- Postbox.xcodeproj/project.pbxproj | 150 ++++++++++++++++-------------- 1 file changed, 82 insertions(+), 68 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 4cf1bc56bc..4d6f1205f7 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -67,38 +67,9 @@ D0B418251D7DFE0C004562A4 /* RedBlackTree.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */; }; D0B418261D7DFE0C004562A4 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510FF1D64A58900A97B8A /* IpcPipe.swift */; }; - D0B418281D7DFE11004562A4 /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; - D0B418291D7DFE11004562A4 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; - D0B4182A1D7DFE11004562A4 /* IntermediateMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */; }; - D0B4182B1D7DFE11004562A4 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; - D0B4182C1D7DFE11004562A4 /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; - D0B4182D1D7DFE11004562A4 /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; - D0B4182E1D7DFE11004562A4 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; - D0B4182F1D7DFE11004562A4 /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; D0B418301D7DFE16004562A4 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; - D0B418331D7DFE1B004562A4 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; - D0B418341D7DFE1B004562A4 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; - D0B418351D7DFE1B004562A4 /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; - D0B418361D7DFE1B004562A4 /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; - D0B418371D7DFE1B004562A4 /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; - D0B418381D7DFE1B004562A4 /* MessageHistoryOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */; }; - D0B418391D7DFE1B004562A4 /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; - D0B4183A1D7DFE1B004562A4 /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; - D0B4183B1D7DFE1B004562A4 /* ChatListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8641C58CB7F00037222 /* ChatListTable.swift */; }; - D0B4183C1D7DFE1B004562A4 /* ChatListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */; }; - D0B4183D1D7DFE1B004562A4 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; - D0B4183E1D7DFE1B004562A4 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; - D0B4183F1D7DFE1B004562A4 /* MetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */; }; - D0B418401D7DFE1B004562A4 /* KeychainTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */; }; - D0B418411D7DFE1B004562A4 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; - D0B418421D7DFE1B004562A4 /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; - D0B418431D7DFE1B004562A4 /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; - D0B418441D7DFE1B004562A4 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; - D0B418451D7DFE1B004562A4 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; - D0B418461D7DFE1B004562A4 /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; - D0B418471D7DFE1B004562A4 /* RatingTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08D451E1D5D2CA700A7428A /* RatingTable.swift */; }; D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86A1C59719800037222 /* ChatListView.swift */; }; D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; @@ -108,11 +79,6 @@ D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */; }; D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */; }; D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */; }; - D0B418511D7DFE25004562A4 /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; - D0B418521D7DFE25004562A4 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; - D0B418531D7DFE25004562A4 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; - D0B418541D7DFE25004562A4 /* MediaResourceData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */; }; - D0B418551D7DFE25004562A4 /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510FA1D63C06E00A97B8A /* PostboxClientId.swift */; }; D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7811B28ADD000A402D9 /* Postbox.swift */; }; D0B418581D7DFE29004562A4 /* SeedConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA291C617D39002160FF /* SeedConfiguration.swift */; }; @@ -123,6 +89,47 @@ D0B4185D1D7DFE35004562A4 /* IpcNotifier.h in Headers */ = {isa = PBXBuildFile; fileRef = D0D511031D64D75200A97B8A /* IpcNotifier.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */ = {isa = PBXBuildFile; fileRef = D0D511011D64D73D00A97B8A /* IpcNotifier.mm */; }; D0B418611D7DFE95004562A4 /* SwiftSignalKitMac.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */; }; + D0B843DE1DAB9198005F29E1 /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; + D0B843DF1DAB9198005F29E1 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; + D0B843E01DAB9198005F29E1 /* IntermediateMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */; }; + D0B843E11DAB9198005F29E1 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; + D0B843E21DAB9198005F29E1 /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; + D0B843E31DAB9198005F29E1 /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; + D0B843E41DAB9198005F29E1 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; + D0B843E51DAB9198005F29E1 /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; + D0B843E61DAB9198005F29E1 /* CachedPeerData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F91DA540F0006A2A60 /* CachedPeerData.swift */; }; + D0B843E71DAB9198005F29E1 /* PeerNotificationSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */; }; + D0B843EB1DAB91A0005F29E1 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; + D0B843EC1DAB91A0005F29E1 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; + D0B843ED1DAB91A0005F29E1 /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; + D0B843EE1DAB91A0005F29E1 /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; + D0B843EF1DAB91A0005F29E1 /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; + D0B843F01DAB91A0005F29E1 /* MessageHistoryOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */; }; + D0B843F11DAB91A0005F29E1 /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; + D0B843F21DAB91A0005F29E1 /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; + D0B843F31DAB91A0005F29E1 /* ChatListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8641C58CB7F00037222 /* ChatListTable.swift */; }; + D0B843F41DAB91A0005F29E1 /* ChatListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */; }; + D0B843F51DAB91A0005F29E1 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; + D0B843F61DAB91A0005F29E1 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; + D0B843F71DAB91A0005F29E1 /* MetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */; }; + D0B843F81DAB91A0005F29E1 /* KeychainTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */; }; + D0B843F91DAB91A0005F29E1 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; + D0B843FA1DAB91A0005F29E1 /* PeerNotificationSettingsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */; }; + D0B843FB1DAB91A0005F29E1 /* CachedPeerDataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */; }; + D0B843FC1DAB91A0005F29E1 /* PeerStatusTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerStatusTable.swift */; }; + D0B843FD1DAB91A0005F29E1 /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; + D0B843FE1DAB91A0005F29E1 /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; + D0B843FF1DAB91A0005F29E1 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; + D0B844001DAB91A0005F29E1 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; + D0B844011DAB91A0005F29E1 /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; + D0B844021DAB91A0005F29E1 /* RatingTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08D451E1D5D2CA700A7428A /* RatingTable.swift */; }; + D0B844031DAB91A7005F29E1 /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DF0C8E1D81A350008AEB01 /* PeerView.swift */; }; + D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; + D0B844061DAB91B5005F29E1 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; + D0B844071DAB91B5005F29E1 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; + D0B844081DAB91B5005F29E1 /* CachedMediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */; }; + D0B844091DAB91B5005F29E1 /* MediaResourceData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */; }; + D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; @@ -686,68 +693,75 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - D0B418421D7DFE1B004562A4 /* PeerChatStateTable.swift in Sources */, + D0B844091DAB91B5005F29E1 /* MediaResourceData.swift in Sources */, D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, D0B418251D7DFE0C004562A4 /* RedBlackTree.swift in Sources */, - D0B4183E1D7DFE1B004562A4 /* MediaCleanupTable.swift in Sources */, - D0B418451D7DFE1B004562A4 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, - D0B4183A1D7DFE1B004562A4 /* MessageHistoryUnsentTable.swift in Sources */, + D0B843EF1DAB91A0005F29E1 /* MessageHistoryTable.swift in Sources */, + D0B844071DAB91B5005F29E1 /* MediaResource.swift in Sources */, D0B418261D7DFE0C004562A4 /* MappedFile.swift in Sources */, - D0B418511D7DFE25004562A4 /* MediaBox.swift in Sources */, + D0B843ED1DAB91A0005F29E1 /* MessageHistoryMetadataTable.swift in Sources */, + D0B843FD1DAB91A0005F29E1 /* PeerChatStateTable.swift in Sources */, + D0B844021DAB91A0005F29E1 /* RatingTable.swift in Sources */, + D0B843EE1DAB91A0005F29E1 /* MessageHistoryIndexTable.swift in Sources */, + D0B843FA1DAB91A0005F29E1 /* PeerNotificationSettingsTable.swift in Sources */, D0B4181C1D7DFDF1004562A4 /* SQLite-Bridging.m in Sources */, - D0B418351D7DFE1B004562A4 /* MessageHistoryMetadataTable.swift in Sources */, + D0B843FB1DAB91A0005F29E1 /* CachedPeerDataTable.swift in Sources */, D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */, + D0B843F41DAB91A0005F29E1 /* ChatListIndexTable.swift in Sources */, D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */, + D0B843DF1DAB9198005F29E1 /* Message.swift in Sources */, + D0B843F11DAB91A0005F29E1 /* MessageHistoryTagsTable.swift in Sources */, + D0B844001DAB91A0005F29E1 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */, - D0B418291D7DFE11004562A4 /* Message.swift in Sources */, - D0B418361D7DFE1B004562A4 /* MessageHistoryIndexTable.swift in Sources */, + D0B843EB1DAB91A0005F29E1 /* Table.swift in Sources */, + D0B843F91DAB91A0005F29E1 /* PeerTable.swift in Sources */, + D0B843DE1DAB9198005F29E1 /* Peer.swift in Sources */, D0B4184D1D7DFE20004562A4 /* ContactPeersView.swift in Sources */, - D0B4182E1D7DFE11004562A4 /* PeerReadState.swift in Sources */, - D0B4183B1D7DFE1B004562A4 /* ChatListTable.swift in Sources */, D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */, - D0B418531D7DFE25004562A4 /* MediaResource.swift in Sources */, + D0B843FE1DAB91A0005F29E1 /* ContactTable.swift in Sources */, D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */, - D0B418371D7DFE1B004562A4 /* MessageHistoryTable.swift in Sources */, - D0B4183C1D7DFE1B004562A4 /* ChatListIndexTable.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, + D0B843F01DAB91A0005F29E1 /* MessageHistoryOperation.swift in Sources */, + D0B843F31DAB91A0005F29E1 /* ChatListTable.swift in Sources */, D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */, + D0B843F51DAB91A0005F29E1 /* MessageMediaTable.swift in Sources */, D098C6F21D7E1201007784E4 /* Database.swift in Sources */, - D0B418551D7DFE25004562A4 /* RandomAccessMediaResourceContext.swift in Sources */, + D0B844061DAB91B5005F29E1 /* MediaResourceStatus.swift in Sources */, D0B4185A1D7DFE29004562A4 /* ViewTracker.swift in Sources */, - D0B4182D1D7DFE11004562A4 /* ChatListHole.swift in Sources */, D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */, D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, - D0B418341D7DFE1B004562A4 /* GlobalMessageIdsTable.swift in Sources */, - D0B4182A1D7DFE11004562A4 /* IntermediateMessage.swift in Sources */, D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, - D0B418471D7DFE1B004562A4 /* RatingTable.swift in Sources */, D0B418241D7DFE0C004562A4 /* SimpleSet.swift in Sources */, + D0B844081DAB91B5005F29E1 /* CachedMediaResource.swift in Sources */, + D0B843E01DAB9198005F29E1 /* IntermediateMessage.swift in Sources */, D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, - D0B418401D7DFE1B004562A4 /* KeychainTable.swift in Sources */, - D0B418411D7DFE1B004562A4 /* PeerTable.swift in Sources */, - D0B418521D7DFE25004562A4 /* MediaResourceStatus.swift in Sources */, - D0B418461D7DFE1B004562A4 /* OrderStatisticTable.swift in Sources */, - D0B4182C1D7DFE11004562A4 /* MessageHistoryHole.swift in Sources */, - D0B4182B1D7DFE11004562A4 /* Media.swift in Sources */, - D0B418441D7DFE1B004562A4 /* MessageHistoryReadStateTable.swift in Sources */, + D0B843E31DAB9198005F29E1 /* ChatListHole.swift in Sources */, + D0B843E11DAB9198005F29E1 /* Media.swift in Sources */, + D0B843E51DAB9198005F29E1 /* PeerNameIndexRepresentation.swift in Sources */, + D0B843EC1DAB91A0005F29E1 /* GlobalMessageIdsTable.swift in Sources */, D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */, - D0B418431D7DFE1B004562A4 /* ContactTable.swift in Sources */, + D0B843F61DAB91A0005F29E1 /* MediaCleanupTable.swift in Sources */, + D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */, D0B418581D7DFE29004562A4 /* SeedConfiguration.swift in Sources */, - D0B4183F1D7DFE1B004562A4 /* MetadataTable.swift in Sources */, D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */, D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */, - D0B418391D7DFE1B004562A4 /* MessageHistoryTagsTable.swift in Sources */, - D0B418331D7DFE1B004562A4 /* Table.swift in Sources */, - D0B418381D7DFE1B004562A4 /* MessageHistoryOperation.swift in Sources */, + D0B843F21DAB91A0005F29E1 /* MessageHistoryUnsentTable.swift in Sources */, + D0B843FC1DAB91A0005F29E1 /* PeerStatusTable.swift in Sources */, + D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */, D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, - D0B4182F1D7DFE11004562A4 /* PeerNameIndexRepresentation.swift in Sources */, + D0B843F71DAB91A0005F29E1 /* MetadataTable.swift in Sources */, + D0B844031DAB91A7005F29E1 /* PeerView.swift in Sources */, D0B418301D7DFE16004562A4 /* ValueBoxKey.swift in Sources */, D0B418221D7DFE0C004562A4 /* Coding.swift in Sources */, - D0B418541D7DFE25004562A4 /* MediaResourceData.swift in Sources */, - D0B4183D1D7DFE1B004562A4 /* MessageMediaTable.swift in Sources */, + D0B843E21DAB9198005F29E1 /* MessageHistoryHole.swift in Sources */, + D0B843E41DAB9198005F29E1 /* PeerReadState.swift in Sources */, + D0B843E71DAB9198005F29E1 /* PeerNotificationSettings.swift in Sources */, + D0B843E61DAB9198005F29E1 /* CachedPeerData.swift in Sources */, + D0B844011DAB91A0005F29E1 /* OrderStatisticTable.swift in Sources */, + D0B843FF1DAB91A0005F29E1 /* MessageHistoryReadStateTable.swift in Sources */, D0B4184B1D7DFE20004562A4 /* SynchronizePeerReadStatesView.swift in Sources */, - D0B418281D7DFE11004562A4 /* Peer.swift in Sources */, + D0B843F81DAB91A0005F29E1 /* KeychainTable.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; From 154a7019a8712a32f4fb677d9aee4023dae0a219 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 11 Oct 2016 19:42:45 +0200 Subject: [PATCH 047/427] no message --- Postbox.xcodeproj/project.pbxproj | 18 +++-- Postbox/ContactPeersView.swift | 66 ++++++++++++++----- ...tusTable.swift => PeerPresenceTable.swift} | 26 ++++---- Postbox/PeerStatus.swift | 4 ++ Postbox/PeerView.swift | 31 ++++++++- Postbox/Postbox.swift | 36 ++++++++-- Postbox/PostboxTransaction.swift | 7 +- Postbox/ViewTracker.swift | 33 ++++++---- 8 files changed, 164 insertions(+), 57 deletions(-) rename Postbox/{PeerStatusTable.swift => PeerPresenceTable.swift} (61%) create mode 100644 Postbox/PeerStatus.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 4d6f1205f7..98086a6525 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -16,7 +16,7 @@ D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; D02EB8071D2B07F300D07ED3 /* OrderStatisticTreeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */; }; - D03120F81DA53FF4006A2A60 /* PeerStatusTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerStatusTable.swift */; }; + D03120F81DA53FF4006A2A60 /* PeerPresenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */; }; D03120FA1DA540F0006A2A60 /* CachedPeerData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F91DA540F0006A2A60 /* CachedPeerData.swift */; }; D03120FC1DA55427006A2A60 /* PeerNotificationSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */; }; D03120FE1DA562E9006A2A60 /* CachedPeerDataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */; }; @@ -116,7 +116,7 @@ D0B843F91DAB91A0005F29E1 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; D0B843FA1DAB91A0005F29E1 /* PeerNotificationSettingsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */; }; D0B843FB1DAB91A0005F29E1 /* CachedPeerDataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */; }; - D0B843FC1DAB91A0005F29E1 /* PeerStatusTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerStatusTable.swift */; }; + D0B843FC1DAB91A0005F29E1 /* PeerPresenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */; }; D0B843FD1DAB91A0005F29E1 /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; D0B843FE1DAB91A0005F29E1 /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; D0B843FF1DAB91A0005F29E1 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; @@ -130,6 +130,7 @@ D0B844081DAB91B5005F29E1 /* CachedMediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */; }; D0B844091DAB91B5005F29E1 /* MediaResourceData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */; }; D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; + D0B844511DAC04FE005F29E1 /* PeerStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerStatus.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; @@ -191,7 +192,7 @@ D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTreeTests.swift; sourceTree = ""; }; - D03120F71DA53FF4006A2A60 /* PeerStatusTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerStatusTable.swift; sourceTree = ""; }; + D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerPresenceTable.swift; sourceTree = ""; }; D03120F91DA540F0006A2A60 /* CachedPeerData.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CachedPeerData.swift; sourceTree = ""; }; D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNotificationSettings.swift; sourceTree = ""; }; D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CachedPeerDataTable.swift; sourceTree = ""; }; @@ -234,6 +235,7 @@ D0B418151D7DFAF3004562A4 /* PostboxMac.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PostboxMac.h; sourceTree = ""; }; D0B418161D7DFAF3004562A4 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKitMac.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-diblohvjozhgaifjcniwdlixlilx/Build/Products/Debug/SwiftSignalKitMac.framework"; sourceTree = ""; }; + D0B844501DAC04FE005F29E1 /* PeerStatus.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerStatus.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; @@ -385,7 +387,7 @@ D0F9E8701C5A0E9B00037222 /* PeerTable.swift */, D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */, D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */, - D03120F71DA53FF4006A2A60 /* PeerStatusTable.swift */, + D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */, D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */, D0079F591D592E8B00A27A2C /* ContactTable.swift */, D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */, @@ -408,6 +410,7 @@ D0C674CB1CBB14A700183765 /* PeerReadState.swift */, D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */, D03120F91DA540F0006A2A60 /* CachedPeerData.swift */, + D0B844501DAC04FE005F29E1 /* PeerStatus.swift */, D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */, ); name = Objects; @@ -747,7 +750,7 @@ D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */, D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */, D0B843F21DAB91A0005F29E1 /* MessageHistoryUnsentTable.swift in Sources */, - D0B843FC1DAB91A0005F29E1 /* PeerStatusTable.swift in Sources */, + D0B843FC1DAB91A0005F29E1 /* PeerPresenceTable.swift in Sources */, D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */, D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, D0B843F71DAB91A0005F29E1 /* MetadataTable.swift in Sources */, @@ -816,6 +819,7 @@ D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, + D0B844511DAC04FE005F29E1 /* PeerStatus.swift in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D0FC1A3C1CA3FB310056AE9A /* MediaResourceData.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, @@ -823,7 +827,7 @@ D0DE76F91D9293B8002B8809 /* CachedMediaResource.swift in Sources */, D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */, D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */, - D03120F81DA53FF4006A2A60 /* PeerStatusTable.swift in Sources */, + D03120F81DA53FF4006A2A60 /* PeerPresenceTable.swift in Sources */, D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */, D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */, D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */, @@ -984,6 +988,7 @@ "-DSQLCIPHER_CRYPTO_CC=1", "-DSQLITE_ENABLE_FTS3", ); + OTHER_SWIFT_FLAGS = "-DDEBUG"; PRODUCT_BUNDLE_IDENTIFIER = org.telegram.Telegram.PostboxMac; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; @@ -1180,6 +1185,7 @@ "-DSQLCIPHER_CRYPTO_CC=1", "-DSQLITE_ENABLE_FTS3", ); + OTHER_SWIFT_FLAGS = "-DDEBUG"; PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; diff --git a/Postbox/ContactPeersView.swift b/Postbox/ContactPeersView.swift index 42410462c9..379cc89719 100644 --- a/Postbox/ContactPeersView.swift +++ b/Postbox/ContactPeersView.swift @@ -3,43 +3,77 @@ import Foundation final class MutableContactPeersView { fileprivate let index: PeerNameIndex fileprivate var peers: [PeerId: Peer] + fileprivate var peerPresences: [PeerId: PeerPresence] fileprivate var peerIds: Set fileprivate var accountPeer: Peer? - init(peers: [PeerId: Peer], index: PeerNameIndex, accountPeer: Peer?) { + init(peers: [PeerId: Peer], peerPresences: [PeerId: PeerPresence], index: PeerNameIndex, accountPeer: Peer?) { self.index = index self.peers = peers self.peerIds = Set(peers.map { $0.0 }) + self.peerPresences = peerPresences self.accountPeer = accountPeer } - func replay(replace replacePeerIds: Set, getPeer: (PeerId) -> Peer?) -> Bool { - let removedPeerIds = self.peerIds.subtracting(replacePeerIds) - let addedPeerIds = replacePeerIds.subtracting(self.peerIds) - - self.peerIds = replacePeerIds - - for peerId in removedPeerIds { - let _ = self.peers.removeValue(forKey: peerId) - } - - for peerId in addedPeerIds { - if let peer = getPeer(peerId) { - self.peers[peerId] = peer + func replay(replacePeerIds: Set?, updatedPeerPresences: [PeerId: PeerPresence], getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) -> Bool { + var updated = false + if let replacePeerIds = replacePeerIds { + let removedPeerIds = self.peerIds.subtracting(replacePeerIds) + let addedPeerIds = replacePeerIds.subtracting(self.peerIds) + + self.peerIds = replacePeerIds + + for peerId in removedPeerIds { + let _ = self.peers.removeValue(forKey: peerId) + let _ = self.peerPresences.removeValue(forKey: peerId) + } + + for peerId in addedPeerIds { + if let peer = getPeer(peerId) { + self.peers[peerId] = peer + } + if let presence = getPeerPresence(peerId) { + self.peerPresences[peerId] = presence + } + } + + if !removedPeerIds.isEmpty || !addedPeerIds.isEmpty { + updated = true } } - return !removedPeerIds.isEmpty || !addedPeerIds.isEmpty + for peerId in self.peerIds { + if let presence = updatedPeerPresences[peerId] { + updated = true + self.peerPresences[peerId] = presence + } + } + + return updated } } public final class ContactPeersView { public let peers: [Peer] + public let peerPresences: [PeerId: PeerPresence] public let accountPeer: Peer? init(_ mutableView: MutableContactPeersView) { let index = mutableView.index - self.peers = mutableView.peers.map({ $0.1 }).sorted(by: { $0.indexName.indexName(index) < $1.indexName.indexName(index) }) + if let accountPeer = mutableView.accountPeer { + var peers: [Peer] = [] + peers.reserveCapacity(mutableView.peers.count) + let accountPeerId = accountPeer.id + for peer in mutableView.peers.values { + if peer.id != accountPeerId { + peers.append(peer) + } + } + self.peers = peers + } else { + self.peers = mutableView.peers.map({ $0.1 }) + } + self.peerPresences = mutableView.peerPresences self.accountPeer = mutableView.accountPeer } } diff --git a/Postbox/PeerStatusTable.swift b/Postbox/PeerPresenceTable.swift similarity index 61% rename from Postbox/PeerStatusTable.swift rename to Postbox/PeerPresenceTable.swift index ab50d10809..18678c8436 100644 --- a/Postbox/PeerStatusTable.swift +++ b/Postbox/PeerPresenceTable.swift @@ -1,10 +1,10 @@ import Foundation -final class PeerStatusTable: Table { +final class PeerPresenceTable: Table { private let sharedEncoder = Encoder() private let sharedKey = ValueBoxKey(length: 8) - private var cachedStatuses: [PeerId: Coding] = [:] + private var cachedPresences: [PeerId: PeerPresence] = [:] private var updatedPeerIds = Set() override init(valueBox: ValueBox, tableId: Int32) { @@ -16,34 +16,34 @@ final class PeerStatusTable: Table { return self.sharedKey } - func set(id: PeerId, status: Coding) { - self.cachedStatuses[id] = status + func set(id: PeerId, presence: PeerPresence) { + self.cachedPresences[id] = presence self.updatedPeerIds.insert(id) } - func get(_ id: PeerId) -> Coding? { - if let status = self.cachedStatuses[id] { - return status + func get(_ id: PeerId) -> PeerPresence? { + if let presence = self.cachedPresences[id] { + return presence } if let value = self.valueBox.get(self.tableId, key: self.key(id)) { - if let status = Decoder(buffer: value).decodeRootObject() { - self.cachedStatuses[id] = status - return status + if let presence = Decoder(buffer: value).decodeRootObject() as? PeerPresence { + self.cachedPresences[id] = presence + return presence } } return nil } override func clearMemoryCache() { - self.cachedStatuses.removeAll() + self.cachedPresences.removeAll() self.updatedPeerIds.removeAll() } override func beforeCommit() { for peerId in self.updatedPeerIds { - if let status = self.cachedStatuses[peerId] { + if let presence = self.cachedPresences[peerId] { self.sharedEncoder.reset() - self.sharedEncoder.encodeRootObject(status) + self.sharedEncoder.encodeRootObject(presence) self.valueBox.set(self.tableId, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) } diff --git a/Postbox/PeerStatus.swift b/Postbox/PeerStatus.swift new file mode 100644 index 0000000000..4b3ceb358f --- /dev/null +++ b/Postbox/PeerStatus.swift @@ -0,0 +1,4 @@ + +public protocol PeerPresence: class, Coding { + func isEqual(to: PeerPresence) -> Bool +} diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index 39d004e2cb..1d59ffe14d 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -5,8 +5,9 @@ final class MutablePeerView { var notificationSettings: PeerNotificationSettings? var cachedData: CachedPeerData? var peers: [PeerId: Peer] = [:] + var peerPresences: [PeerId: PeerPresence] = [:] - init(peerId: PeerId, notificationSettings: PeerNotificationSettings?, cachedData: CachedPeerData?, getPeer: (PeerId) -> Peer?) { + init(peerId: PeerId, notificationSettings: PeerNotificationSettings?, cachedData: CachedPeerData?, getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) { self.peerId = peerId self.notificationSettings = notificationSettings self.cachedData = cachedData @@ -19,10 +20,13 @@ final class MutablePeerView { if let peer = getPeer(id) { self.peers[id] = peer } + if let presence = getPeerPresence(id) { + self.peerPresences[id] = presence + } } } - func replay(updatedPeers: [PeerId: Peer], updatedNotificationSettings: [PeerId: PeerNotificationSettings], updatedCachedPeerData: [PeerId: CachedPeerData], getPeer: (PeerId) -> Peer?) -> Bool { + func replay(updatedPeers: [PeerId: Peer], updatedNotificationSettings: [PeerId: PeerNotificationSettings], updatedCachedPeerData: [PeerId: CachedPeerData], updatedPeerPresences: [PeerId: PeerPresence], getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) -> Bool { var updated = false if let cachedData = updatedCachedPeerData[self.peerId], self.cachedData == nil || self.cachedData!.peerIds != cachedData.peerIds { @@ -39,6 +43,12 @@ final class MutablePeerView { } else if let peer = getPeer(id) { self.peers[id] = peer } + + if let presence = updatedPeerPresences[id] { + self.peerPresences[id] = presence + } else if let presence = getPeerPresence(id) { + self.peerPresences[id] = presence + } } var removePeerIds: [PeerId] = [] @@ -51,6 +61,17 @@ final class MutablePeerView { for peerId in removePeerIds { self.peers.removeValue(forKey: peerId) } + + removePeerIds.removeAll() + for peerId in self.peerPresences.keys { + if !peerIds.contains(peerId) { + removePeerIds.append(peerId) + } + } + + for peerId in removePeerIds { + self.peerPresences.removeValue(forKey: peerId) + } } else { var peerIds = Set() peerIds.insert(self.peerId) @@ -63,6 +84,10 @@ final class MutablePeerView { self.peers[id] = peer updated = true } + if let presence = updatedPeerPresences[id] { + self.peerPresences[id] = presence + updated = true + } } } @@ -80,11 +105,13 @@ public final class PeerView { public let cachedData: CachedPeerData? public let notificationSettings: PeerNotificationSettings? public let peers: [PeerId: Peer] + public let peerPresences: [PeerId: PeerPresence] init(_ mutableView: MutablePeerView) { self.peerId = mutableView.peerId self.cachedData = mutableView.cachedData self.notificationSettings = mutableView.notificationSettings self.peers = mutableView.peers + self.peerPresences = mutableView.peerPresences } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 2dc54776f8..54c8d4701a 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -110,6 +110,10 @@ public final class Modifier { self.postbox?.updatePeerCachedData(peerIds: peerIds, update: update) } + public func updatePeerPresences(_ peerPresences: [PeerId: PeerPresence]) { + self.postbox?.updatePeerPresences(peerPresences) + } + public func replaceContactPeerIds(_ peerIds: Set) { self.postbox?.replaceContactPeerIds(peerIds) } @@ -194,6 +198,7 @@ public final class Postbox { private var currentUpdatedPeers: [PeerId: Peer] = [:] private var currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings] = [:] private var currentUpdatedCachedPeerData: [PeerId: CachedPeerData] = [:] + private var currentUpdatedPeerPresences: [PeerId: PeerPresence] = [:] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -217,7 +222,7 @@ public final class Postbox { var peerTable: PeerTable! var peerNotificationSettingsTable: PeerNotificationSettingsTable! var cachedPeerDataTable: CachedPeerDataTable! - var peerStatusTable: PeerStatusTable! + var peerPresenceTable: PeerPresenceTable! var globalMessageIdsTable: GlobalMessageIdsTable! var messageHistoryIndexTable: MessageHistoryIndexTable! var messageHistoryTable: MessageHistoryTable! @@ -340,7 +345,7 @@ public final class Postbox { self.peerRatingTable = RatingTable(valueBox: self.valueBox, tableId: 17) self.cachedPeerDataTable = CachedPeerDataTable(valueBox: self.valueBox, tableId: 18) self.peerNotificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox, tableId: 19) - self.peerStatusTable = PeerStatusTable(valueBox: self.valueBox, tableId: 20) + self.peerPresenceTable = PeerPresenceTable(valueBox: self.valueBox, tableId: 20) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -361,7 +366,7 @@ public final class Postbox { self.tables.append(self.peerRatingTable) self.tables.append(self.peerNotificationSettingsTable) self.tables.append(self.cachedPeerDataTable) - self.tables.append(self.peerStatusTable) + self.tables.append(self.peerPresenceTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() @@ -371,7 +376,9 @@ public final class Postbox { return self.peerNotificationSettingsTable.get(peerId) }, getCachedPeerData: { peerId in return self.cachedPeerDataTable.get(peerId) - }, unsentMessageIndices: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) + }, getPeerPresence: { peerId in + return self.peerPresenceTable.get(peerId) + },unsentMessageIndices: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") }) @@ -736,7 +743,7 @@ public final class Postbox { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } - let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -762,6 +769,7 @@ public final class Postbox { self.currentUpdatedMasterClientId = nil self.currentUpdatedPeerNotificationSettings.removeAll() self.currentUpdatedCachedPeerData.removeAll() + self.currentUpdatedPeerPresences.removeAll() for table in self.tables { table.beforeCommit() @@ -813,6 +821,16 @@ public final class Postbox { } } + fileprivate func updatePeerPresences(_ peerPresences: [PeerId: PeerPresence]) { + for (peerId, presence) in peerPresences { + let currentPresence = self.peerPresenceTable.get(peerId) + if currentPresence == nil || !(currentPresence!.isEqual(to: presence)) { + self.peerPresenceTable.set(id: peerId, presence: presence) + self.currentUpdatedPeerPresences[peerId] = presence + } + } + } + fileprivate func replaceContactPeerIds(_ peerIds: Set) { self.contactsTable.replace(peerIds) @@ -1023,14 +1041,18 @@ public final class Postbox { public func contactPeersView(index: PeerNameIndex, accountPeerId: PeerId) -> Signal { return self.modify { modifier -> Signal in var peers: [PeerId: Peer] = [:] + var peerPresences: [PeerId: PeerPresence] = [:] for peerId in self.contactsTable.get() { if let peer = self.peerTable.get(peerId) { peers[peerId] = peer } + if let presence = self.peerPresenceTable.get(peerId) { + peerPresences[peerId] = presence + } } - let view = MutableContactPeersView(peers: peers, index: index, accountPeer: self.peerTable.get(accountPeerId)) + let view = MutableContactPeersView(peers: peers, peerPresences: peerPresences, index: index, accountPeer: self.peerTable.get(accountPeerId)) let (index, signal) = self.viewTracker.addContactPeersView(view) return (.single(ContactPeersView(view)) @@ -1064,7 +1086,7 @@ public final class Postbox { public func peerView(id: PeerId) -> Signal { return self.modify { modifier -> Signal in - let view = MutablePeerView(peerId: id, notificationSettings: self.peerNotificationSettingsTable.get(id), cachedData: self.cachedPeerDataTable.get(id), getPeer: { self.peerTable.get($0) }) + let view = MutablePeerView(peerId: id, notificationSettings: self.peerNotificationSettingsTable.get(id), cachedData: self.cachedPeerDataTable.get(id), getPeer: { self.peerTable.get($0) }, getPeerPresence: { self.peerPresenceTable.get($0) }) let (index, signal) = self.viewTracker.addPeerView(view) return (.single(PeerView(view)) diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index 4ef39497fc..7cddf9ee27 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -8,6 +8,7 @@ final class PostboxTransaction { let currentUpdatedPeers: [PeerId: Peer] let currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings] let currentUpdatedCachedPeerData: [PeerId: CachedPeerData] + let currentUpdatedPeerPresences: [PeerId: PeerPresence] let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -37,6 +38,9 @@ final class PostboxTransaction { if !currentUpdatedCachedPeerData.isEmpty { return false } + if !currentUpdatedPeerPresences.isEmpty { + return false + } if !unsentMessageOperations.isEmpty { return false } @@ -55,7 +59,7 @@ final class PostboxTransaction { return true } - init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles self.removedHolesByPeerId = removedHolesByPeerId @@ -63,6 +67,7 @@ final class PostboxTransaction { self.currentUpdatedPeers = currentUpdatedPeers self.currentUpdatedPeerNotificationSettings = currentUpdatedPeerNotificationSettings; self.currentUpdatedCachedPeerData = currentUpdatedCachedPeerData + self.currentUpdatedPeerPresences = currentUpdatedPeerPresences self.unsentMessageOperations = unsentMessageOperations self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations self.updatedMedia = updatedMedia diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 2d427ecc33..f818c5fbfe 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -23,6 +23,7 @@ final class ViewTracker { private let getPeer: (PeerId) -> Peer? private let getPeerNotificationSettings: (PeerId) -> PeerNotificationSettings? private let getCachedPeerData: (PeerId) -> CachedPeerData? + private let getPeerPresence: (PeerId) -> PeerPresence? private var chatListViews = Bag<(MutableChatListView, ValuePipe<(ChatListView, ViewUpdateType)>)>() private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] @@ -43,7 +44,7 @@ final class ViewTracker { private var peerViews = Bag<(MutablePeerView, ValuePipe)>() - init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, unsentMessageIndices: [MessageIndex], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, unsentMessageIndices: [MessageIndex], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -54,6 +55,7 @@ final class ViewTracker { self.getPeer = getPeer self.getPeerNotificationSettings = getPeerNotificationSettings self.getCachedPeerData = getCachedPeerData + self.getPeerPresence = getPeerPresence self.unsentMessageView = UnsentMessageHistoryView(indices: unsentMessageIndices) self.synchronizeReadStatesView = MutableSynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) @@ -201,7 +203,11 @@ final class ViewTracker { for (mutableView, pipe) in self.peerViews.copyItems() { var updatedPeers: [PeerId: Peer] = [:] if let peer = self.getPeer(mutableView.peerId) { - updatedPeers[peer.id] = peer + updatedPeers[mutableView.peerId] = peer + } + var updatedPeerPresences: [PeerId: PeerPresence] = [:] + if let presence = self.getPeerPresence(mutableView.peerId) { + updatedPeerPresences[mutableView.peerId] = presence } var updatedNotificationSettings: [PeerId: PeerNotificationSettings] = [:] @@ -212,14 +218,17 @@ final class ViewTracker { var updatedCachedPeerData: [PeerId: CachedPeerData] = [:] if let cachedPeerData = self.getCachedPeerData(mutableView.peerId) { updatedCachedPeerData[mutableView.peerId] = cachedPeerData - for peerIds in cachedPeerData.peerIds { - if let peer = self.getPeer(mutableView.peerId) { - updatedPeers[peer.id] = peer + for peerId in cachedPeerData.peerIds { + if let peer = self.getPeer(peerId) { + updatedPeers[peerId] = peer + } + if let presence = self.getPeerPresence(peerId) { + updatedPeerPresences[peerId] = presence } } } - if mutableView.replay(updatedPeers: updatedPeers, updatedNotificationSettings: updatedNotificationSettings, updatedCachedPeerData: updatedCachedPeerData, getPeer: self.getPeer) { + if mutableView.replay(updatedPeers: updatedPeers, updatedNotificationSettings: updatedNotificationSettings, updatedCachedPeerData: updatedCachedPeerData, updatedPeerPresences: updatedPeerPresences, getPeer: self.getPeer, getPeerPresence: self.getPeerPresence) { pipe.putNext(PeerView(mutableView)) } } @@ -313,16 +322,16 @@ final class ViewTracker { pipe.putNext(ContactPeerIdsView(mutableView)) } } - - for (mutableView, pipe) in self.contactPeersViews.copyItems() { - if mutableView.replay(replace: replaceContactPeerIds, getPeer: self.getPeer) { - pipe.putNext(ContactPeersView(mutableView)) - } + } + + for (mutableView, pipe) in self.contactPeersViews.copyItems() { + if mutableView.replay(replacePeerIds: transaction.replaceContactPeerIds, updatedPeerPresences: transaction.currentUpdatedPeerPresences, getPeer: self.getPeer, getPeerPresence: self.getPeerPresence) { + pipe.putNext(ContactPeersView(mutableView)) } } for (mutableView, pipe) in self.peerViews.copyItems() { - if mutableView.replay(updatedPeers: transaction.currentUpdatedPeers, updatedNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, updatedCachedPeerData: transaction.currentUpdatedCachedPeerData, getPeer: self.getPeer) { + if mutableView.replay(updatedPeers: transaction.currentUpdatedPeers, updatedNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, updatedCachedPeerData: transaction.currentUpdatedCachedPeerData, updatedPeerPresences: transaction.currentUpdatedPeerPresences, getPeer: self.getPeer, getPeerPresence: self.getPeerPresence) { pipe.putNext(PeerView(mutableView)) } } From 44664c3b88e3af84e9031cd2b47cac2a85094ae1 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 11 Oct 2016 19:45:22 +0200 Subject: [PATCH 048/427] no message --- Postbox.xcodeproj/project.pbxproj | 50 ++++++++++--------- .../{PeerStatus.swift => PeerPresence.swift} | 0 2 files changed, 26 insertions(+), 24 deletions(-) rename Postbox/{PeerStatus.swift => PeerPresence.swift} (100%) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 98086a6525..b595f61d40 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -24,6 +24,17 @@ D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; D03BCCF81C73561C0097A291 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; + D03C535B1DAD5C9C004C17B3 /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; + D03C535C1DAD5C9C004C17B3 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; + D03C535D1DAD5C9C004C17B3 /* IntermediateMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */; }; + D03C535E1DAD5C9C004C17B3 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; + D03C535F1DAD5C9C004C17B3 /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; + D03C53601DAD5C9C004C17B3 /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; + D03C53611DAD5C9C004C17B3 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; + D03C53621DAD5C9C004C17B3 /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; + D03C53631DAD5C9C004C17B3 /* CachedPeerData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F91DA540F0006A2A60 /* CachedPeerData.swift */; }; + D03C53641DAD5C9C004C17B3 /* PeerPresence.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerPresence.swift */; }; + D03C53651DAD5C9C004C17B3 /* PeerNotificationSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */; }; D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA291C617D39002160FF /* SeedConfiguration.swift */; }; D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; @@ -89,16 +100,6 @@ D0B4185D1D7DFE35004562A4 /* IpcNotifier.h in Headers */ = {isa = PBXBuildFile; fileRef = D0D511031D64D75200A97B8A /* IpcNotifier.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */ = {isa = PBXBuildFile; fileRef = D0D511011D64D73D00A97B8A /* IpcNotifier.mm */; }; D0B418611D7DFE95004562A4 /* SwiftSignalKitMac.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */; }; - D0B843DE1DAB9198005F29E1 /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; - D0B843DF1DAB9198005F29E1 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; - D0B843E01DAB9198005F29E1 /* IntermediateMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */; }; - D0B843E11DAB9198005F29E1 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; - D0B843E21DAB9198005F29E1 /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; - D0B843E31DAB9198005F29E1 /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; - D0B843E41DAB9198005F29E1 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; - D0B843E51DAB9198005F29E1 /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; - D0B843E61DAB9198005F29E1 /* CachedPeerData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F91DA540F0006A2A60 /* CachedPeerData.swift */; }; - D0B843E71DAB9198005F29E1 /* PeerNotificationSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */; }; D0B843EB1DAB91A0005F29E1 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; D0B843EC1DAB91A0005F29E1 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; D0B843ED1DAB91A0005F29E1 /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; @@ -130,7 +131,7 @@ D0B844081DAB91B5005F29E1 /* CachedMediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */; }; D0B844091DAB91B5005F29E1 /* MediaResourceData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */; }; D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; - D0B844511DAC04FE005F29E1 /* PeerStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerStatus.swift */; }; + D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerPresence.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; @@ -235,7 +236,7 @@ D0B418151D7DFAF3004562A4 /* PostboxMac.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PostboxMac.h; sourceTree = ""; }; D0B418161D7DFAF3004562A4 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKitMac.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-diblohvjozhgaifjcniwdlixlilx/Build/Products/Debug/SwiftSignalKitMac.framework"; sourceTree = ""; }; - D0B844501DAC04FE005F29E1 /* PeerStatus.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerStatus.swift; sourceTree = ""; }; + D0B844501DAC04FE005F29E1 /* PeerPresence.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerPresence.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; @@ -410,7 +411,7 @@ D0C674CB1CBB14A700183765 /* PeerReadState.swift */, D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */, D03120F91DA540F0006A2A60 /* CachedPeerData.swift */, - D0B844501DAC04FE005F29E1 /* PeerStatus.swift */, + D0B844501DAC04FE005F29E1 /* PeerPresence.swift */, D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */, ); name = Objects; @@ -697,6 +698,7 @@ buildActionMask = 2147483647; files = ( D0B844091DAB91B5005F29E1 /* MediaResourceData.swift in Sources */, + D03C535C1DAD5C9C004C17B3 /* Message.swift in Sources */, D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, D0B418251D7DFE0C004562A4 /* RedBlackTree.swift in Sources */, @@ -705,6 +707,7 @@ D0B418261D7DFE0C004562A4 /* MappedFile.swift in Sources */, D0B843ED1DAB91A0005F29E1 /* MessageHistoryMetadataTable.swift in Sources */, D0B843FD1DAB91A0005F29E1 /* PeerChatStateTable.swift in Sources */, + D03C535B1DAD5C9C004C17B3 /* Peer.swift in Sources */, D0B844021DAB91A0005F29E1 /* RatingTable.swift in Sources */, D0B843EE1DAB91A0005F29E1 /* MessageHistoryIndexTable.swift in Sources */, D0B843FA1DAB91A0005F29E1 /* PeerNotificationSettingsTable.swift in Sources */, @@ -713,20 +716,22 @@ D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */, D0B843F41DAB91A0005F29E1 /* ChatListIndexTable.swift in Sources */, D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */, - D0B843DF1DAB9198005F29E1 /* Message.swift in Sources */, D0B843F11DAB91A0005F29E1 /* MessageHistoryTagsTable.swift in Sources */, D0B844001DAB91A0005F29E1 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, + D03C53651DAD5C9C004C17B3 /* PeerNotificationSettings.swift in Sources */, D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */, D0B843EB1DAB91A0005F29E1 /* Table.swift in Sources */, + D03C535E1DAD5C9C004C17B3 /* Media.swift in Sources */, D0B843F91DAB91A0005F29E1 /* PeerTable.swift in Sources */, - D0B843DE1DAB9198005F29E1 /* Peer.swift in Sources */, D0B4184D1D7DFE20004562A4 /* ContactPeersView.swift in Sources */, D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */, + D03C535D1DAD5C9C004C17B3 /* IntermediateMessage.swift in Sources */, D0B843FE1DAB91A0005F29E1 /* ContactTable.swift in Sources */, D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, D0B843F01DAB91A0005F29E1 /* MessageHistoryOperation.swift in Sources */, D0B843F31DAB91A0005F29E1 /* ChatListTable.swift in Sources */, + D03C53631DAD5C9C004C17B3 /* CachedPeerData.swift in Sources */, D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */, D0B843F51DAB91A0005F29E1 /* MessageMediaTable.swift in Sources */, D098C6F21D7E1201007784E4 /* Database.swift in Sources */, @@ -736,33 +741,30 @@ D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, D0B418241D7DFE0C004562A4 /* SimpleSet.swift in Sources */, + D03C53601DAD5C9C004C17B3 /* ChatListHole.swift in Sources */, D0B844081DAB91B5005F29E1 /* CachedMediaResource.swift in Sources */, - D0B843E01DAB9198005F29E1 /* IntermediateMessage.swift in Sources */, D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, - D0B843E31DAB9198005F29E1 /* ChatListHole.swift in Sources */, - D0B843E11DAB9198005F29E1 /* Media.swift in Sources */, - D0B843E51DAB9198005F29E1 /* PeerNameIndexRepresentation.swift in Sources */, D0B843EC1DAB91A0005F29E1 /* GlobalMessageIdsTable.swift in Sources */, D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */, + D03C53611DAD5C9C004C17B3 /* PeerReadState.swift in Sources */, D0B843F61DAB91A0005F29E1 /* MediaCleanupTable.swift in Sources */, D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */, D0B418581D7DFE29004562A4 /* SeedConfiguration.swift in Sources */, D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */, + D03C535F1DAD5C9C004C17B3 /* MessageHistoryHole.swift in Sources */, D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */, D0B843F21DAB91A0005F29E1 /* MessageHistoryUnsentTable.swift in Sources */, D0B843FC1DAB91A0005F29E1 /* PeerPresenceTable.swift in Sources */, D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */, D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, + D03C53641DAD5C9C004C17B3 /* PeerPresence.swift in Sources */, D0B843F71DAB91A0005F29E1 /* MetadataTable.swift in Sources */, D0B844031DAB91A7005F29E1 /* PeerView.swift in Sources */, D0B418301D7DFE16004562A4 /* ValueBoxKey.swift in Sources */, D0B418221D7DFE0C004562A4 /* Coding.swift in Sources */, - D0B843E21DAB9198005F29E1 /* MessageHistoryHole.swift in Sources */, - D0B843E41DAB9198005F29E1 /* PeerReadState.swift in Sources */, - D0B843E71DAB9198005F29E1 /* PeerNotificationSettings.swift in Sources */, - D0B843E61DAB9198005F29E1 /* CachedPeerData.swift in Sources */, D0B844011DAB91A0005F29E1 /* OrderStatisticTable.swift in Sources */, D0B843FF1DAB91A0005F29E1 /* MessageHistoryReadStateTable.swift in Sources */, + D03C53621DAD5C9C004C17B3 /* PeerNameIndexRepresentation.swift in Sources */, D0B4184B1D7DFE20004562A4 /* SynchronizePeerReadStatesView.swift in Sources */, D0B843F81DAB91A0005F29E1 /* KeychainTable.swift in Sources */, ); @@ -819,7 +821,7 @@ D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, - D0B844511DAC04FE005F29E1 /* PeerStatus.swift in Sources */, + D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D0FC1A3C1CA3FB310056AE9A /* MediaResourceData.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, diff --git a/Postbox/PeerStatus.swift b/Postbox/PeerPresence.swift similarity index 100% rename from Postbox/PeerStatus.swift rename to Postbox/PeerPresence.swift From 6bde6d964cd57cb84325a80213d943c6a6d5ed77 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 24 Oct 2016 12:31:15 +0300 Subject: [PATCH 049/427] no message --- Postbox.xcodeproj/project.pbxproj | 22 +- Postbox/ContactTable.swift | 15 + Postbox/ItemCollection.swift | 81 +++++ Postbox/ItemCollectionInfoTable.swift | 118 +++++++ Postbox/ItemCollectionItemTable.swift | 136 +++++++ Postbox/ItemCollectionsView.swift | 200 +++++++++++ Postbox/Media.swift | 6 +- Postbox/MediaBox.swift | 487 +++++++++++++++----------- Postbox/MediaResource.swift | 38 +- Postbox/MediaResourceData.swift | 6 - Postbox/MessageHistoryTable.swift | 6 + Postbox/Peer.swift | 2 +- Postbox/PeerView.swift | 8 +- Postbox/Postbox.swift | 42 ++- Postbox/ViewTracker.swift | 4 +- 15 files changed, 948 insertions(+), 223 deletions(-) create mode 100644 Postbox/ItemCollection.swift create mode 100644 Postbox/ItemCollectionInfoTable.swift create mode 100644 Postbox/ItemCollectionItemTable.swift create mode 100644 Postbox/ItemCollectionsView.swift delete mode 100644 Postbox/MediaResourceData.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index b595f61d40..b4e89f3f2c 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -15,6 +15,10 @@ D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; + D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */; }; + D021E0D61DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */; }; + D021E0D81DB4FD1300C6B04F /* ItemCollectionItemTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */; }; + D021E0DC1DB5237C00C6B04F /* ItemCollectionsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */; }; D02EB8071D2B07F300D07ED3 /* OrderStatisticTreeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */; }; D03120F81DA53FF4006A2A60 /* PeerPresenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */; }; D03120FA1DA540F0006A2A60 /* CachedPeerData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F91DA540F0006A2A60 /* CachedPeerData.swift */; }; @@ -129,7 +133,6 @@ D0B844061DAB91B5005F29E1 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; D0B844071DAB91B5005F29E1 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; D0B844081DAB91B5005F29E1 /* CachedMediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */; }; - D0B844091DAB91B5005F29E1 /* MediaResourceData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */; }; D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerPresence.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; @@ -170,7 +173,6 @@ D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8741C5A334100037222 /* SimpleDictionary.swift */; }; - D0FC1A3C1CA3FB310056AE9A /* MediaResourceData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -192,6 +194,10 @@ D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; + D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollection.swift; sourceTree = ""; }; + D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollectionInfoTable.swift; sourceTree = ""; }; + D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollectionItemTable.swift; sourceTree = ""; }; + D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollectionsView.swift; sourceTree = ""; }; D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTreeTests.swift; sourceTree = ""; }; D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerPresenceTable.swift; sourceTree = ""; }; D03120F91DA540F0006A2A60 /* CachedPeerData.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CachedPeerData.swift; sourceTree = ""; }; @@ -278,7 +284,6 @@ D0F9E8701C5A0E9B00037222 /* PeerTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerTable.swift; sourceTree = ""; }; D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GlobalMessageIdsTable.swift; sourceTree = ""; }; D0F9E8741C5A334100037222 /* SimpleDictionary.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SimpleDictionary.swift; sourceTree = ""; }; - D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResourceData.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -316,7 +321,6 @@ D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */, D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */, D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */, - D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */, D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */, ); name = "Media Box"; @@ -395,6 +399,8 @@ D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */, D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */, D08D451E1D5D2CA700A7428A /* RatingTable.swift */, + D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */, + D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */, ); name = Tables; sourceTree = ""; @@ -413,6 +419,7 @@ D03120F91DA540F0006A2A60 /* CachedPeerData.swift */, D0B844501DAC04FE005F29E1 /* PeerPresence.swift */, D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */, + D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */, ); name = Objects; sourceTree = ""; @@ -453,6 +460,7 @@ D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */, D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */, D0DF0C8E1D81A350008AEB01 /* PeerView.swift */, + D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */, ); name = Views; sourceTree = ""; @@ -697,7 +705,6 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - D0B844091DAB91B5005F29E1 /* MediaResourceData.swift in Sources */, D03C535C1DAD5C9C004C17B3 /* Message.swift in Sources */, D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, @@ -787,10 +794,12 @@ D03120FE1DA562E9006A2A60 /* CachedPeerDataTable.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, + D021E0D61DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift in Sources */, D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */, D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */, D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */, + D021E0D81DB4FD1300C6B04F /* ItemCollectionItemTable.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */, D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, @@ -808,6 +817,7 @@ D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */, + D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */, D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, D0D949F51D35353900740E02 /* MappedFile.swift in Sources */, @@ -823,7 +833,6 @@ D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, - D0FC1A3C1CA3FB310056AE9A /* MediaResourceData.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D0DE76F91D9293B8002B8809 /* CachedMediaResource.swift in Sources */, @@ -835,6 +844,7 @@ D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */, D03120FC1DA55427006A2A60 /* PeerNotificationSettings.swift in Sources */, D03BCCF81C73561C0097A291 /* Table.swift in Sources */, + D021E0DC1DB5237C00C6B04F /* ItemCollectionsView.swift in Sources */, D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, diff --git a/Postbox/ContactTable.swift b/Postbox/ContactTable.swift index 4f55e289d6..6ac7f1f901 100644 --- a/Postbox/ContactTable.swift +++ b/Postbox/ContactTable.swift @@ -20,6 +20,21 @@ final class ContactTable: Table { return self.key(PeerId(namespace: Int32.max, id: Int32.max)) } + func isContact(peerId: PeerId) -> Bool { + if let peerIds = self.peerIds { + return peerIds.contains(peerId) + } else { + var peerIds = Set() + self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), keys: { key in + peerIds.insert(PeerId(key.getInt64(0))) + return true + }, limit: 0) + self.peerIds = peerIds + self.originalPeerIds = peerIds + return peerIds.contains(peerId) + } + } + func get() -> Set { if let peerIds = self.peerIds { return peerIds diff --git a/Postbox/ItemCollection.swift b/Postbox/ItemCollection.swift new file mode 100644 index 0000000000..396d2df6cc --- /dev/null +++ b/Postbox/ItemCollection.swift @@ -0,0 +1,81 @@ +import Foundation + +public struct ItemCollectionId: Comparable, Hashable { + public typealias Namespace = Int32 + public typealias Id = Int64 + + public let namespace: ItemCollectionId.Namespace + public let id: ItemCollectionId.Id + + public init(namespace: ItemCollectionId.Namespace, id: ItemCollectionId.Id) { + self.namespace = namespace + self.id = id + } + + public static func ==(lhs: ItemCollectionId, rhs: ItemCollectionId) -> Bool { + return lhs.namespace == rhs.namespace && lhs.id == rhs.id + } + + public static func <(lhs: ItemCollectionId, rhs: ItemCollectionId) -> Bool { + if lhs.namespace == rhs.namespace { + return lhs.id < rhs.id + } else { + return lhs.namespace < rhs.namespace + } + } + + public var hashValue: Int { + return self.id.hashValue + } +} + +public protocol ItemCollectionInfo: Coding { + +} + +public struct ItemCollectionItemIndex: Comparable, Hashable { + public typealias Index = Int32 + public typealias Id = Int64 + + public let index: ItemCollectionItemIndex.Index + public let id: ItemCollectionItemIndex.Id + + public init(index: ItemCollectionItemIndex.Index, id: ItemCollectionItemIndex.Id) { + self.index = index + self.id = id + } + + public static func ==(lhs: ItemCollectionItemIndex, rhs: ItemCollectionItemIndex) -> Bool { + return lhs.index == rhs.index && lhs.id == rhs.id + } + + public static func <(lhs: ItemCollectionItemIndex, rhs: ItemCollectionItemIndex) -> Bool { + if lhs.index == rhs.index { + return lhs.id < rhs.id + } else { + return lhs.index < rhs.index + } + } + + public var hashValue: Int { + return self.id.hashValue ^ self.index.hashValue + } + + static var lowerBound: ItemCollectionItemIndex { + return ItemCollectionItemIndex(index: 0, id: 0) + } + + static var upperBound: ItemCollectionItemIndex { + return ItemCollectionItemIndex(index: Int32.max, id: Int64.max) + } +} + +public protocol ItemCollectionItem: Coding { + var index: ItemCollectionItemIndex { get } +} + +public struct ItemCollectionSummary { + let id: ItemCollectionId + let info: ItemCollectionInfo + let itemIds: [ItemCollectionItem] +} diff --git a/Postbox/ItemCollectionInfoTable.swift b/Postbox/ItemCollectionInfoTable.swift new file mode 100644 index 0000000000..726d36e2bb --- /dev/null +++ b/Postbox/ItemCollectionInfoTable.swift @@ -0,0 +1,118 @@ +import Foundation + +final class ItemCollectionInfoTable: Table { + private let sharedKey = ValueBoxKey(length: 4 + 4 + 8) + + private var cachedInfos: [ItemCollectionId.Namespace: [(Int, ItemCollectionId, ItemCollectionInfo)]] = [:] + + private func key(collectionId: ItemCollectionId, index: Int32) -> ValueBoxKey { + self.sharedKey.setInt32(0, value: collectionId.namespace) + self.sharedKey.setInt32(4, value: index) + self.sharedKey.setInt64(4 + 4, value: collectionId.id) + return self.sharedKey + } + + private func lowerBound(namespace: ItemCollectionId.Namespace) -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setInt32(0, value: namespace) + return key + } + + private func upperBound(namespace: ItemCollectionId.Namespace) -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setInt32(0, value: namespace) + return key.successor + } + + func getInfos(namespace: ItemCollectionId.Namespace) -> [(Int, ItemCollectionId, ItemCollectionInfo)] { + if let cachedInfo = self.cachedInfos[namespace] { + return cachedInfo + } else { + var infos: [(Int, ItemCollectionId, ItemCollectionInfo)] = [] + self.valueBox.range(self.tableId, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), values: { key, value in + if let info = Decoder(buffer: value).decodeRootObject() as? ItemCollectionInfo { + infos.append((Int(key.getInt32(4)), ItemCollectionId(namespace: namespace, id: key.getInt64(4 + 4)), info)) + } + return true + }, limit: 0) + self.cachedInfos[namespace] = infos + return infos + } + } + + func lowerCollectionId(namespaceList: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, index: Int32) -> (ItemCollectionId, Int32)? { + var currentNamespace = collectionId.namespace + var currentKey = self.key(collectionId: collectionId, index: index) + while true { + var resultCollectionIdAndIndex: (ItemCollectionId, Int32)? + self.valueBox.range(self.tableId, start: currentKey, end: self.lowerBound(namespace: currentNamespace), keys: { key in + resultCollectionIdAndIndex = (ItemCollectionId(namespace: currentNamespace, id: key.getInt64(4 + 4)), key.getInt32(4)) + return true + }, limit: 1) + if let resultCollectionIdAndIndex = resultCollectionIdAndIndex { + return resultCollectionIdAndIndex + } else { + let index = namespaceList.index(of: currentNamespace)! + if index == 0 { + return nil + } else { + currentNamespace = namespaceList[index - 1] + currentKey = self.upperBound(namespace: currentNamespace) + } + } + } + } + + func higherCollectionId(namespaceList: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, index: Int32) -> (ItemCollectionId, Int32)? { + var currentNamespace = collectionId.namespace + var currentKey = self.key(collectionId: collectionId, index: index) + while true { + var resultCollectionIdAndIndex: (ItemCollectionId, Int32)? + self.valueBox.range(self.tableId, start: currentKey, end: self.upperBound(namespace: currentNamespace), keys: { key in + resultCollectionIdAndIndex = (ItemCollectionId(namespace: currentNamespace, id: key.getInt64(4 + 4)), key.getInt32(4)) + return true + }, limit: 1) + if let resultCollectionIdAndIndex = resultCollectionIdAndIndex { + return resultCollectionIdAndIndex + } else { + let index = namespaceList.index(of: currentNamespace)! + if index == namespaceList.count - 1 { + return nil + } else { + currentNamespace = namespaceList[index + 1] + currentKey = self.lowerBound(namespace: currentNamespace) + } + } + } + } + + func replaceInfos(namespace: ItemCollectionId.Namespace, infos: [(ItemCollectionId, ItemCollectionInfo)]) { + self.cachedInfos.removeAll() + + var currentCollectionKeys: [ValueBoxKey] = [] + self.valueBox.range(self.tableId, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in + currentCollectionKeys.append(key) + return true + }, limit: 0) + + for key in currentCollectionKeys { + self.valueBox.remove(self.tableId, key: key) + } + + var index: Int32 = 0 + let sharedEncoder = Encoder() + for (id, info) in infos { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(info) + self.valueBox.set(self.tableId, key: self.key(collectionId: id, index: index), value: sharedEncoder.readBufferNoCopy()) + index += 1 + } + } + + override func clearMemoryCache() { + self.cachedInfos.removeAll() + } + + override func beforeCommit() { + } +} diff --git a/Postbox/ItemCollectionItemTable.swift b/Postbox/ItemCollectionItemTable.swift new file mode 100644 index 0000000000..ce96f5380c --- /dev/null +++ b/Postbox/ItemCollectionItemTable.swift @@ -0,0 +1,136 @@ +import Foundation + +enum ItemCollectionOperation { + case insertItem(ItemCollectionId, ItemCollectionItem) + case removeItem(ItemCollectionId, ItemCollectionItemIndex.Id) +} + +final class ItemCollectionItemTable: Table { + private let sharedKey = ValueBoxKey(length: 4 + 8 + 4 + 8) + + private func key(collectionId: ItemCollectionId, index: ItemCollectionItemIndex) -> ValueBoxKey { + self.sharedKey.setInt32(0, value: collectionId.namespace) + self.sharedKey.setInt64(4, value: collectionId.id) + self.sharedKey.setInt32(4 + 8, value: index.index) + self.sharedKey.setInt64(4 + 8 + 4, value: index.id) + return self.sharedKey + } + + private func lowerBound(namespace: ItemCollectionId.Namespace) -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setInt32(0, value: namespace) + return key + } + + private func upperBound(namespace: ItemCollectionId.Namespace) -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setInt32(0, value: namespace) + return key.successor + } + + private func lowerBound(collectionId: ItemCollectionId) -> ValueBoxKey { + let key = ValueBoxKey(length: 4 + 8) + key.setInt32(0, value: collectionId.namespace) + key.setInt64(4, value: collectionId.id) + return key + } + + private func upperBound(collectionId: ItemCollectionId) -> ValueBoxKey { + let key = ValueBoxKey(length: 4 + 8) + key.setInt32(0, value: collectionId.namespace) + key.setInt64(4, value: collectionId.id) + return key.successor + } + + func lowerItems(collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] { + var items: [ItemCollectionItem] = [] + self.valueBox.range(self.tableId, start: self.key(collectionId: collectionId, index: itemIndex), end: self.lowerBound(collectionId: collectionId), values: { _, value in + if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { + items.append(item) + } + return true + }, limit: count) + return items + } + + func higherItems(collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] { + var items: [ItemCollectionItem] = [] + self.valueBox.range(self.tableId, start: self.key(collectionId: collectionId, index: itemIndex), end: self.upperBound(collectionId: collectionId), values: { _, value in + if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { + items.append(item) + } + return true + }, limit: count) + return items + } + + func getSummaryIndices(namespace: ItemCollectionId.Namespace) -> [ItemCollectionId: [ItemCollectionItemIndex]] { + var summaryIndices: [ItemCollectionId: [ItemCollectionItemIndex]] = [:] + self.valueBox.range(self.tableId, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in + let collectionId = ItemCollectionId(namespace: namespace, id: key.getInt64(4)) + let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) + if summaryIndices[collectionId] != nil { + summaryIndices[collectionId]!.append(itemIndex) + } else { + summaryIndices[collectionId] = [itemIndex] + } + return true + }, limit: 0) + return summaryIndices + } + + func getItems(namespace: ItemCollectionId.Namespace) -> [ItemCollectionId: [ItemCollectionItem]] { + var items: [ItemCollectionId: [ItemCollectionItem]] = [:] + self.valueBox.range(self.tableId, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), values: { key, value in + let collectionId = ItemCollectionId(namespace: namespace, id: key.getInt64(4)) + //let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) + if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { + if items[collectionId] != nil { + items[collectionId]!.append(item) + } else { + items[collectionId] = [item] + } + } + return true + }, limit: 0) + return items + } + + func replaceItems(collectionId: ItemCollectionId, items: [ItemCollectionItem]) { + var currentIndices = Set() + self.valueBox.range(self.tableId, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId), keys: { key in + let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) + currentIndices.insert(itemIndex) + return true + }, limit: 0) + + var updatedIndices = Set(items.map({ $0.index })) + var itemByIndex: [ItemCollectionItemIndex: ItemCollectionItem] = [:] + for item in items { + itemByIndex[item.index] = item + } + + let addedIndices = updatedIndices.subtracting(currentIndices) + let removedIndices = currentIndices.subtracting(updatedIndices) + + for index in removedIndices { + self.valueBox.remove(self.tableId, key: self.key(collectionId: collectionId, index: index)) + } + + let sharedEncoder = Encoder() + for index in addedIndices { + let item = itemByIndex[index]! + sharedEncoder.reset() + sharedEncoder.encodeRootObject(item) + self.valueBox.set(self.tableId, key: self.key(collectionId: collectionId, index: index), value: sharedEncoder.readBufferNoCopy()) + } + } + + override func clearMemoryCache() { + + } + + override func beforeCommit() { + + } +} diff --git a/Postbox/ItemCollectionsView.swift b/Postbox/ItemCollectionsView.swift new file mode 100644 index 0000000000..c03c4c6c5b --- /dev/null +++ b/Postbox/ItemCollectionsView.swift @@ -0,0 +1,200 @@ +import Foundation + +public struct ItemCollectionViewEntryIndex: Comparable { + public let collectionIndex: Int32 + public let collectionId: ItemCollectionId + public let itemIndex: ItemCollectionItemIndex + + public static func ==(lhs: ItemCollectionViewEntryIndex, rhs: ItemCollectionViewEntryIndex) -> Bool { + return lhs.collectionIndex == rhs.collectionIndex && lhs.collectionId == rhs.collectionId && lhs.itemIndex == rhs.itemIndex + } + + public static func <(lhs: ItemCollectionViewEntryIndex, rhs: ItemCollectionViewEntryIndex) -> Bool { + if lhs.collectionIndex == rhs.collectionIndex { + if lhs.itemIndex == rhs.itemIndex { + return lhs.collectionId < rhs.collectionId + } else { + return lhs.itemIndex < rhs.itemIndex + } + } else { + return lhs.collectionIndex < rhs.collectionIndex + } + } +} + +public struct ItemCollectionViewEntry { + public let index: ItemCollectionViewEntryIndex + public let item: ItemCollectionItem +} + +private func fetchLowerEntries(namespaces: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, collectionIndex: Int32, itemIndex: ItemCollectionItemIndex, count: Int, lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, lowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) -> [ItemCollectionViewEntry] { + + var entries: [ItemCollectionViewEntry] = [] + + var currentCollectionIndex = collectionIndex + var currentCollectionId = collectionId + var currentItemIndex = itemIndex + + while true { + let remainingCount = count - entries.count + assert(remainingCount > 0) + var collectionItems = lowerItems(currentCollectionId, currentItemIndex, remainingCount) + for item in collectionItems { + entries.append(ItemCollectionViewEntry(index: ItemCollectionViewEntryIndex(collectionIndex: currentCollectionIndex, collectionId: currentCollectionId, itemIndex: item.index), item: item)) + } + if entries.count >= count { + break + } else { + assert(collectionItems.count < remainingCount) + + if let (previousCollectionId, previousCollectionIndex) = lowerCollectionId(namespaces, currentCollectionId, currentCollectionIndex) { + currentCollectionIndex = previousCollectionIndex + currentCollectionId = previousCollectionId + currentItemIndex = ItemCollectionItemIndex.upperBound + } else { + break + } + } + } + + return entries +} + +private func fetchHigherEntries(namespaces: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, collectionIndex: Int32, itemIndex: ItemCollectionItemIndex, count: Int, higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, higherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) -> [ItemCollectionViewEntry] { + + var entries: [ItemCollectionViewEntry] = [] + + var currentCollectionIndex = collectionIndex + var currentCollectionId = collectionId + var currentItemIndex = itemIndex + + while true { + let remainingCount = count - entries.count + assert(remainingCount > 0) + var collectionItems = higherItems(currentCollectionId, currentItemIndex, remainingCount) + for item in collectionItems { + entries.append(ItemCollectionViewEntry(index: ItemCollectionViewEntryIndex(collectionIndex: currentCollectionIndex, collectionId: currentCollectionId, itemIndex: item.index), item: item)) + } + if entries.count >= count { + break + } else { + assert(collectionItems.count < remainingCount) + + if let (nextCollectionId, nextCollectionIndex) = higherCollectionId(namespaces, currentCollectionId, currentCollectionIndex) { + currentCollectionIndex = nextCollectionIndex + currentCollectionId = nextCollectionId + currentItemIndex = ItemCollectionItemIndex.lowerBound + } else { + break + } + } + } + + return entries +} + +private func aroundEntries(namespaces: [ItemCollectionId.Namespace], + collectionId: ItemCollectionId, + collectionIndex: Int32, + itemIndex: ItemCollectionItemIndex, + count: Int, + lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, + fetchLowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem], + higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, + fetchHigherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) -> ([ItemCollectionViewEntry], ItemCollectionViewEntry?, ItemCollectionViewEntry?) { + var lowerEntries: [ItemCollectionViewEntry] = [] + var upperEntries: [ItemCollectionViewEntry] = [] + var lower: ItemCollectionViewEntry? + var upper: ItemCollectionViewEntry? + + lowerEntries.append(contentsOf: fetchLowerEntries(namespaces: namespaces, collectionId: collectionId, collectionIndex: collectionIndex, itemIndex: itemIndex, count: count / 2 + 1, lowerCollectionId: lowerCollectionId, lowerItems: fetchLowerItems)) + + let lowerIndices = lowerEntries.map { $0.index } + assert(lowerIndices.sorted() == lowerIndices.reversed()) + + if lowerEntries.count >= count / 2 + 1 { + lower = lowerEntries.last + lowerEntries.removeLast() + } + + upperEntries.append(contentsOf: fetchHigherEntries(namespaces: namespaces, collectionId: collectionId, collectionIndex: collectionIndex, itemIndex: ItemCollectionItemIndex(index: itemIndex.index, id: max(0, itemIndex.id - 1)), count: count - lowerEntries.count + 1, higherCollectionId: higherCollectionId, higherItems: fetchHigherItems)) + + let upperIndices = upperEntries.map { $0.index } + assert(upperIndices.sorted() == upperIndices) + + if upperEntries.count >= count - lowerEntries.count + 1 { + upper = upperEntries.last + upperEntries.removeLast() + } + + if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count { + var additionalLowerEntries: [ItemCollectionViewEntry] = fetchLowerEntries(namespaces: namespaces, collectionId: lowerEntries.last!.index.collectionId, collectionIndex: lowerEntries.last!.index.collectionIndex, itemIndex: lowerEntries.last!.index.itemIndex, count: count - lowerEntries.count - upperEntries.count + 1, lowerCollectionId: lowerCollectionId, lowerItems: fetchLowerItems) + + if additionalLowerEntries.count >= count - lowerEntries.count + upperEntries.count + 1 { + lower = additionalLowerEntries.last + additionalLowerEntries.removeLast() + } + lowerEntries.append(contentsOf: additionalLowerEntries) + } + + var entries: [ItemCollectionViewEntry] = [] + entries.append(contentsOf: lowerEntries.reversed()) + entries.append(contentsOf: upperEntries) + return (entries: entries, lower: lower, upper: upper) +} + +final class MutableItemCollectionsView { + let namespaces: [ItemCollectionId.Namespace] + + var collectionInfos: [(ItemCollectionId, ItemCollectionInfo, ItemCollectionItem?)] + var entries: [ItemCollectionViewEntry] + var lower: ItemCollectionViewEntry? + var higher: ItemCollectionViewEntry? + + init(namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int, getInfos: (_ namespace: ItemCollectionId.Namespace) -> [(Int, ItemCollectionId, ItemCollectionInfo)], lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, lowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem], higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, higherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) { + self.namespaces = namespaces + + self.collectionInfos = [] + for namespace in namespaces { + for (_, id, info) in getInfos(namespace) { + let item = higherItems(id, ItemCollectionItemIndex.lowerBound, 1).first + self.collectionInfos.append((id, info, item)) + } + } + + let selectedAroundIndex: ItemCollectionViewEntryIndex + if let aroundIndex = aroundIndex { + selectedAroundIndex = aroundIndex + } else { + selectedAroundIndex = ItemCollectionViewEntryIndex(collectionIndex: 0, collectionId: ItemCollectionId(namespace: namespaces[0], id: 0), itemIndex: ItemCollectionItemIndex.lowerBound) + } + + let (entries, lower, higher) = aroundEntries(namespaces: namespaces, + collectionId: selectedAroundIndex.collectionId, + collectionIndex: selectedAroundIndex.collectionIndex, + itemIndex: selectedAroundIndex.itemIndex, + count: count, + lowerCollectionId: lowerCollectionId, + fetchLowerItems: lowerItems, + higherCollectionId: higherCollectionId, + fetchHigherItems: higherItems) + + self.entries = entries + self.lower = lower + self.higher = higher + } +} + +public final class ItemCollectionsView { + public let collectionInfos: [(ItemCollectionId, ItemCollectionInfo, ItemCollectionItem?)] + public let entries: [ItemCollectionViewEntry] + public let lower: ItemCollectionViewEntry? + public let higher: ItemCollectionViewEntry? + + init(_ mutableView: MutableItemCollectionsView) { + self.collectionInfos = mutableView.collectionInfos + self.entries = mutableView.entries + self.lower = mutableView.lower + self.higher = mutableView.higher + } +} diff --git a/Postbox/Media.swift b/Postbox/Media.swift index 85d2097fbf..76f629a4bd 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -4,8 +4,8 @@ public struct MediaId: Hashable, CustomStringConvertible { public typealias Namespace = Int32 public typealias Id = Int64 - let namespace: Namespace - let id: Id + public let namespace: Namespace + public let id: Id public var hashValue: Int { get { @@ -68,7 +68,7 @@ public func ==(lhs: MediaId, rhs: MediaId) -> Bool { return lhs.id == rhs.id && lhs.namespace == rhs.namespace } -public protocol Media: Coding { +public protocol Media: class, Coding { var id: MediaId? { get } var peerIds: [PeerId] { get } diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index e0d92424a7..48610ff890 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -23,10 +23,13 @@ private final class ResourceDataContext { } } -private func fileSize(_ path: String) -> Int { +private func fileSize(_ path: String) -> Int? { var value = stat() - stat(path, &value) - return Int(value.st_size) + if stat(path, &value) == 0 { + return Int(value.st_size) + } else { + return nil + } } public enum ResourceDataRangeMode { @@ -35,6 +38,53 @@ public enum ResourceDataRangeMode { case partial } +private struct ResourceStorePaths { + let partial: String + let complete: String +} + +public struct MediaResourceData { + public let path: String + public let size: Int + public let complete: Bool +} + +public struct MediaResourceDataFetchResult { + public let data: Data + public let complete: Bool + + public init(data: Data, complete: Bool) { + self.data = data + self.complete = complete + } +} + +public struct CachedMediaResourceRepresentationResult { + public let temporaryPath: String + + public init(temporaryPath: String) { + self.temporaryPath = temporaryPath + } +} + +private struct CachedMediaResourceRepresentationKey: Hashable { + let resourceId: MediaResourceId + let representation: CachedMediaResourceRepresentation + + static func ==(lhs: CachedMediaResourceRepresentationKey, rhs: CachedMediaResourceRepresentationKey) -> Bool { + return lhs.resourceId.isEqual(to: rhs.resourceId) && lhs.representation.isEqual(to: rhs.representation) + } + + var hashValue: Int { + return self.resourceId.hashValue + } +} + +private final class CachedMediaResourceRepresentationContext { + let dataSubscribers = Bag<(MediaResourceData) -> Void>() + var disposable: Disposable? +} + public final class MediaBox { let basePath: String let buffer = WriteBuffer() @@ -44,12 +94,13 @@ public final class MediaBox { private let dataQueue = Queue() private let cacheQueue = Queue() - private var statusContexts: [String: ResourceStatusContext] = [:] - private var dataContexts: [String: ResourceDataContext] = [:] - private var randomAccessContexts: [String: RandomAccessMediaResourceContext] = [:] + private var statusContexts: [WrappedMediaResourceId: ResourceStatusContext] = [:] + private var dataContexts: [WrappedMediaResourceId: ResourceDataContext] = [:] + private var randomAccessContexts: [WrappedMediaResourceId: RandomAccessMediaResourceContext] = [:] + private var cachedRepresentationContexts: [CachedMediaResourceRepresentationKey: CachedMediaResourceRepresentationContext] = [:] - private var wrappedFetchResource = Promise<(MediaResource, Range) -> Signal>() - public var fetchResource: ((MediaResource, Range) -> Signal)? { + private var wrappedFetchResource = Promise<(MediaResource, Range) -> Signal>() + public var fetchResource: ((MediaResource, Range) -> Signal)? { didSet { if let fetchResource = self.fetchResource { wrappedFetchResource.set(.single(fetchResource)) @@ -59,8 +110,20 @@ public final class MediaBox { } } + public var wrappedFetchCachedResourceRepresentation = Promise<(MediaResource, MediaResourceData, CachedMediaResourceRepresentation) -> Signal>() + public var fetchCachedResourceRepresentation: ((MediaResource, MediaResourceData, CachedMediaResourceRepresentation) -> Signal)? { + didSet { + if let fetchCachedResourceRepresentation = self.fetchCachedResourceRepresentation { + wrappedFetchCachedResourceRepresentation.set(.single(fetchCachedResourceRepresentation)) + } else { + wrappedFetchCachedResourceRepresentation.set(.never()) + } + } + } + lazy var ensureDirectoryCreated: Void = { try! FileManager.default.createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) + try! FileManager.default.createDirectory(atPath: self.basePath + "/cache", withIntermediateDirectories: true, attributes: nil) }() public init(basePath: String) { @@ -69,38 +132,35 @@ public final class MediaBox { let _ = self.ensureDirectoryCreated } - private func fileNameForId(_ id: String) -> String { - return id + private func fileNameForId(_ id: MediaResourceId) -> String { + return "\(id.uniqueId)" } - private func pathForId(_ id: String) -> String { + private func pathForId(_ id: MediaResourceId) -> String { return "\(self.basePath)/\(fileNameForId(id))" } - private func streamingPathForId(_ id: String) -> String { - return "\(self.basePath)/\(fileNameForId(id))_stream" + private func storePathsForId(_ id: MediaResourceId) -> ResourceStorePaths { + return ResourceStorePaths(partial: "\(self.basePath)/\(fileNameForId(id))_partial", complete: "\(self.basePath)/\(fileNameForId(id))") } - private func cachePathForId(_ id: String) -> String { - return "\(self.basePath)/cache/\(fileNameForId(id))" + private func cachedRepresentationPathForId(_ id: MediaResourceId, representation: CachedMediaResourceRepresentation) -> String { + return "\(self.basePath)/cache/\(fileNameForId(id)):\(representation.uniqueId)" } - public func cachedResource(_ resource: CachedMediaResource) -> Signal { - return Signal { subscriber in - self.concurrentQueue.async { - let path = self.cachePathForId(resource.id) - let currentSize = fileSize(path) - subscriber.putNext(MediaResourceData(path: path, size: currentSize)) - subscriber.putCompletion() - } - - return EmptyDisposable + public func storeResourceData(_ id: MediaResourceId, data: Data) { + self.dataQueue.async { + let paths = self.storePathsForId(id) + let _ = try? data.write(to: URL(fileURLWithPath: paths.complete), options: [.atomic]) } } - public func cacheResource(_ resource: CachedMediaResource, data: Data) { - self.cacheQueue.async { - let _ = try? data.write(to: URL(fileURLWithPath: self.cachePathForId(resource.id)), options: [.atomic]) + public func moveResourceData(from: MediaResourceId, to: MediaResourceId) { + self.dataQueue.async { + let pathsFrom = self.storePathsForId(from) + let pathsTo = self.storePathsForId(to) + link(pathsFrom.partial, pathsTo.partial) + link(pathsFrom.complete, pathsTo.complete) } } @@ -109,19 +169,18 @@ public final class MediaBox { let disposable = MetaDisposable() self.concurrentQueue.async { - let path = self.pathForId(resource.id) - let currentSize = fileSize(path) - if currentSize >= resource.size { + let paths = self.storePathsForId(resource.id) + if let _ = fileSize(paths.complete) { subscriber.putNext(.Local) subscriber.putCompletion() } else { self.statusQueue.async { let statusContext: ResourceStatusContext - if let current = self.statusContexts[resource.id] { + if let current = self.statusContexts[WrappedMediaResourceId(resource.id)] { statusContext = current } else { statusContext = ResourceStatusContext() - self.statusContexts[resource.id] = statusContext + self.statusContexts[WrappedMediaResourceId(resource.id)] = statusContext } let index = statusContext.subscribers.add({ status in @@ -134,25 +193,24 @@ public final class MediaBox { self.dataQueue.async { let status: MediaResourceStatus - let path = self.pathForId(resource.id) - let currentSize = fileSize(path) - if currentSize >= resource.size { + if let _ = fileSize(paths.complete) { status = .Local } else { var fetchingData = false - if let dataContext = self.dataContexts[resource.id] { + if let dataContext = self.dataContexts[WrappedMediaResourceId(resource.id)] { fetchingData = dataContext.fetchDisposable != nil } if fetchingData { - status = .Fetching(progress: Float(currentSize) / Float(resource.size)) + //status = .Fetching(progress: Float(currentSize) / Float(resourceSize)) + status = .Fetching(progress: 0.0) } else { status = .Remote } } self.statusQueue.async { - if let statusContext = self.statusContexts[resource.id] , statusContext.status == nil { + if let statusContext = self.statusContexts[WrappedMediaResourceId(resource.id)] , statusContext.status == nil { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { @@ -165,10 +223,10 @@ public final class MediaBox { disposable.set(ActionDisposable { self.statusQueue.async { - if let current = self.statusContexts[resource.id] { + if let current = self.statusContexts[WrappedMediaResourceId(resource.id)] { current.subscribers.remove(index) if current.subscribers.isEmpty { - self.statusContexts.removeValue(forKey: resource.id) + self.statusContexts.removeValue(forKey: WrappedMediaResourceId(resource.id)) } } } @@ -182,126 +240,68 @@ public final class MediaBox { } public func resourceData(_ resource: MediaResource, pathExtension: String? = nil, complete: Bool = true) -> Signal { + assert(pathExtension == nil) return Signal { subscriber in let disposable = MetaDisposable() self.concurrentQueue.async { - let path = self.pathForId(resource.id) - let currentSize = fileSize(path) - - if currentSize >= resource.size { - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !FileManager.default.fileExists(atPath: pathWithExtension) { - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) - } - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: currentSize)) - } else { - subscriber.putNext(MediaResourceData(path: path, size: currentSize)) - } + let paths = self.storePathsForId(resource.id) + if let completeSize = fileSize(paths.complete) { + subscriber.putNext(MediaResourceData(path: paths.complete, size: completeSize, complete: true)) subscriber.putCompletion() } else { self.dataQueue.async { - let dataContext: ResourceDataContext - if let current = self.dataContexts[resource.id] { - dataContext = current + let resourceId = WrappedMediaResourceId(resource.id) + var currentContext: ResourceDataContext? = self.dataContexts[resourceId] + if let currentContext = currentContext, currentContext.data.complete { + subscriber.putNext(currentContext.data) + subscriber.putCompletion() + } else if let completeSize = fileSize(paths.complete) { + subscriber.putNext(MediaResourceData(path: paths.complete, size: completeSize, complete: true)) + subscriber.putCompletion() } else { - dataContext = ResourceDataContext(data: MediaResourceData(path: path, size: currentSize)) - self.dataContexts[resource.id] = dataContext - } - - let index: Bag<(MediaResourceData) -> Void>.Index - if complete { - var checkedForSymlink = false - index = dataContext.completeDataSubscribers.add { data in - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { - checkedForSymlink = true - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) - } - - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) - } else { + let dataContext: ResourceDataContext + if let currentContext = currentContext { + dataContext = currentContext + } else { + let partialSize = fileSize(paths.partial) ?? 0 + dataContext = ResourceDataContext(data: MediaResourceData(path: paths.partial, size: partialSize, complete: false)) + self.dataContexts[resourceId] = dataContext + } + + let index: Bag<(MediaResourceData) -> Void>.Index + if complete { + index = dataContext.completeDataSubscribers.add { data in subscriber.putNext(data) - } - - if data.size >= resource.size { subscriber.putCompletion() } - } - - if dataContext.data.size >= resource.size { - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { - checkedForSymlink = true - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) - } - - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: dataContext.data.size)) - } else { - subscriber.putNext(dataContext.data) - } + subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0, complete: false)) } else { - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { - checkedForSymlink = true - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) - } - - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: 0)) - } else { - subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0)) - } - } - } else { - var checkedForSymlink = false - index = dataContext.progresiveDataSubscribers.add { data in - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { - checkedForSymlink = true - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) - } - - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) - } else { + index = dataContext.progresiveDataSubscribers.add { data in subscriber.putNext(data) + if data.complete { + subscriber.putCompletion() + } } - if data.size >= resource.size { - subscriber.putCompletion() - } - } - if let pathExtension = pathExtension { - let pathWithExtension = path + ".\(pathExtension)" - if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { - checkedForSymlink = true - let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) - } - - subscriber.putNext(MediaResourceData(path: pathWithExtension, size: dataContext.data.size)) - } else { subscriber.putNext(dataContext.data) } - } - - disposable.set(ActionDisposable { - self.dataQueue.async { - if let dataContext = self.dataContexts[resource.id] { - if complete { - dataContext.completeDataSubscribers.remove(index) - } else { - dataContext.progresiveDataSubscribers.remove(index) - } - - if dataContext.progresiveDataSubscribers.isEmpty && dataContext.completeDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { - self.dataContexts.removeValue(forKey: resource.id) + + disposable.set(ActionDisposable { + self.dataQueue.async { + if let dataContext = self.dataContexts[resourceId] { + if complete { + dataContext.completeDataSubscribers.remove(index) + } else { + dataContext.progresiveDataSubscribers.remove(index) + } + + if dataContext.progresiveDataSubscribers.isEmpty && dataContext.completeDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { + self.dataContexts.removeValue(forKey: resourceId) + } } } - } - }) + }) + } } } } @@ -310,60 +310,63 @@ public final class MediaBox { } } - private func randomAccessContext(for resource: MediaResource) -> RandomAccessMediaResourceContext { + private func randomAccessContext(for resource: MediaResource, size: Int) -> RandomAccessMediaResourceContext { assert(self.dataQueue.isCurrent()) + let resourceId = WrappedMediaResourceId(resource.id) + let dataContext: RandomAccessMediaResourceContext - if let current = self.randomAccessContexts[resource.id] { + if let current = self.randomAccessContexts[resourceId] { dataContext = current } else { let path = self.pathForId(resource.id) + ".random" - dataContext = RandomAccessMediaResourceContext(path: path, size: resource.size, fetchRange: { [weak self] range in + dataContext = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { [weak self] range in let disposable = MetaDisposable() if let strongSelf = self { strongSelf.dataQueue.async { - let fetch = strongSelf.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in + let fetch = strongSelf.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in return fetch(resource, range) } var offset = 0 - disposable.set(fetch.start(next: { [weak strongSelf] data in + disposable.set(fetch.start(next: { [weak strongSelf] result in if let strongSelf = strongSelf { strongSelf.dataQueue.async { - if let dataContext = strongSelf.randomAccessContexts[resource.id] { - let storeRange = RandomAccessResourceStoreRange(offset: range.lowerBound + offset, data: data) - offset += data.count + if let dataContext = strongSelf.randomAccessContexts[resourceId] { + let storeRange = RandomAccessResourceStoreRange(offset: range.lowerBound + offset, data: result.data) + offset += result.data.count dataContext.storeRanges([storeRange]) } } } - })) + })) } } return disposable }) - self.randomAccessContexts[resource.id] = dataContext + self.randomAccessContexts[resourceId] = dataContext } return dataContext } - public func fetchedResourceData(_ resource: MediaResource, in range: Range) -> Signal { + public func fetchedResourceData(_ resource: MediaResource, size: Int, in range: Range) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.dataQueue.async { - let dataContext = self.randomAccessContext(for: resource) + let resourceId = WrappedMediaResourceId(resource.id) + let dataContext = self.randomAccessContext(for: resource, size: size) let listener = dataContext.addListenerForFetchedData(in: range) disposable.set(ActionDisposable { [weak self] in if let strongSelf = self { strongSelf.dataQueue.async { - if let dataContext = strongSelf.randomAccessContexts[resource.id] { + if let dataContext = strongSelf.randomAccessContexts[resourceId] { dataContext.removeListenerForFetchedData(listener) if !dataContext.hasDataListeners() { - let _ = strongSelf.randomAccessContexts.removeValue(forKey: resource.id) + let _ = strongSelf.randomAccessContexts.removeValue(forKey: resourceId) } } } @@ -375,12 +378,13 @@ public final class MediaBox { } } - public func resourceData(_ resource: MediaResource, in range: Range, mode: ResourceDataRangeMode = .complete) -> Signal { + public func resourceData(_ resource: MediaResource, size: Int, in range: Range, mode: ResourceDataRangeMode = .complete) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.dataQueue.async { - let dataContext = self.randomAccessContext(for: resource) + let resourceId = WrappedMediaResourceId(resource.id) + let dataContext = self.randomAccessContext(for: resource, size: size) let listenerMode: RandomAccessResourceDataRangeMode switch mode { @@ -415,10 +419,10 @@ public final class MediaBox { disposable.set(ActionDisposable { [weak self] in if let strongSelf = self { strongSelf.dataQueue.async { - if let dataContext = strongSelf.randomAccessContexts[resource.id] { + if let dataContext = strongSelf.randomAccessContexts[resourceId] { dataContext.removeListenerForData(listener) if !dataContext.hasDataListeners() { - let _ = strongSelf.randomAccessContexts.removeValue(forKey: resource.id) + let _ = strongSelf.randomAccessContexts.removeValue(forKey: resourceId) } } } @@ -435,26 +439,28 @@ public final class MediaBox { let disposable = MetaDisposable() self.dataQueue.async { - let path = self.pathForId(resource.id) - let currentSize = fileSize(path) + let resourceId = WrappedMediaResourceId(resource.id) + let paths = self.storePathsForId(resource.id) - if currentSize >= resource.size { + if let _ = fileSize(paths.complete) { subscriber.putCompletion() } else { + let currentSize = fileSize(paths.partial) ?? 0 let dataContext: ResourceDataContext - if let current = self.dataContexts[resource.id] { + if let current = self.dataContexts[resourceId] { dataContext = current } else { - dataContext = ResourceDataContext(data: MediaResourceData(path: path, size: currentSize)) - self.dataContexts[resource.id] = dataContext + dataContext = ResourceDataContext(data: MediaResourceData(path: paths.partial, size: currentSize, complete: false)) + self.dataContexts[resourceId] = dataContext } let index: Bag.Index = dataContext.fetchSubscribers.add(Void()) if dataContext.fetchDisposable == nil { - let status: MediaResourceStatus = .Fetching(progress: Float(currentSize) / Float(resource.size)) + //let status: MediaResourceStatus = .Fetching(progress: Float(currentSize) / Float(resourceSize)) + let status: MediaResourceStatus = .Fetching(progress: 0.0) self.statusQueue.async { - if let statusContext = self.statusContexts[resource.id] { + if let statusContext = self.statusContexts[resourceId] { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { subscriber(status) @@ -464,52 +470,70 @@ public final class MediaBox { var offset = currentSize var fd: Int32? - dataContext.fetchDisposable = ((self.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in - return fetch(resource, currentSize ..< resource.size) + let dataQueue = self.dataQueue + dataContext.fetchDisposable = ((self.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in + return fetch(resource, currentSize ..< Int.max) }) |> afterDisposed { - if let fd = fd { - close(fd) + dataQueue.async { + if let fd = fd { + close(fd) + } } - }).start(next: { data in + }).start(next: { result in self.dataQueue.async { let _ = self.ensureDirectoryCreated if fd == nil { - let handle = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) + let handle = open(paths.partial, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) if handle >= 0 { fd = handle } } - if let fd = fd { - data.withUnsafeBytes { bytes in - write(fd, bytes, data.count) + if let thisFd = fd { + if !result.data.isEmpty { + let writeResult = result.data.withUnsafeBytes { bytes -> Int in + return write(thisFd, bytes, result.data.count) + } + if writeResult != result.data.count { + print("write error \(errno)") + } } - offset += data.count + offset += result.data.count let updatedSize = offset - dataContext.data = MediaResourceData(path: path, size: updatedSize) - - for subscriber in dataContext.progresiveDataSubscribers.copyItems() { - subscriber(MediaResourceData(path: path, size: updatedSize)) + let updatedData: MediaResourceData + if result.complete { + let linkResult = link(paths.partial, paths.complete) + assert(linkResult == 0) + updatedData = MediaResourceData(path: paths.complete, size: updatedSize, complete: true) + } else { + updatedData = MediaResourceData(path: paths.partial, size: updatedSize, complete: false) } - if updatedSize >= resource.size { + dataContext.data = updatedData + + for subscriber in dataContext.progresiveDataSubscribers.copyItems() { + subscriber(updatedData) + } + + if updatedData.complete { for subscriber in dataContext.completeDataSubscribers.copyItems() { - subscriber(MediaResourceData(path: path, size: updatedSize)) + subscriber(updatedData) } } let status: MediaResourceStatus - if updatedSize >= resource.size { + if updatedData.complete { status = .Local } else { - status = .Fetching(progress: Float(updatedSize) / Float(resource.size)) + //status = .Fetching(progress: Float(updatedSize) / Float(resourceSize)) + status = .Fetching(progress: 0.0) } self.statusQueue.async { - if let statusContext = self.statusContexts[resource.id] { + if let statusContext = self.statusContexts[resourceId] { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { subscriber(status) @@ -523,23 +547,22 @@ public final class MediaBox { disposable.set(ActionDisposable { self.dataQueue.async { - if let dataContext = self.dataContexts[resource.id] { + if let dataContext = self.dataContexts[resourceId] { dataContext.fetchSubscribers.remove(index) if dataContext.fetchSubscribers.isEmpty { dataContext.fetchDisposable?.dispose() dataContext.fetchDisposable = nil - let currentSize = fileSize(path) let status: MediaResourceStatus - if currentSize >= resource.size { + if dataContext.data.complete { status = .Local } else { status = .Remote } self.statusQueue.async { - if let statusContext = self.statusContexts[resource.id] , statusContext.status != status { + if let statusContext = self.statusContexts[resourceId], statusContext.status != status { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { subscriber(status) @@ -549,7 +572,7 @@ public final class MediaBox { } if dataContext.completeDataSubscribers.isEmpty && dataContext.progresiveDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { - self.dataContexts.removeValue(forKey: resource.id) + self.dataContexts.removeValue(forKey: resourceId) } } } @@ -563,20 +586,20 @@ public final class MediaBox { public func cancelInteractiveResourceFetch(_ resource: MediaResource) { self.dataQueue.async { - if let dataContext = self.dataContexts[resource.id] , dataContext.fetchDisposable != nil { + let resourceId = WrappedMediaResourceId(resource.id) + if let dataContext = self.dataContexts[resourceId], dataContext.fetchDisposable != nil { dataContext.fetchDisposable?.dispose() dataContext.fetchDisposable = nil - - let currentSize = fileSize(self.pathForId(resource.id)) + let status: MediaResourceStatus - if currentSize >= resource.size { + if dataContext.data.complete { status = .Local } else { status = .Remote } self.statusQueue.async { - if let statusContext = self.statusContexts[resource.id] , statusContext.status != status { + if let statusContext = self.statusContexts[resourceId], statusContext.status != status { statusContext.status = status for subscriber in statusContext.subscribers.copyItems() { subscriber(status) @@ -586,4 +609,72 @@ public final class MediaBox { } } } + + public func cachedResourceRepresentation(_ resource: MediaResource, representation: CachedMediaResourceRepresentation) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + self.concurrentQueue.async { + let path = self.cachedRepresentationPathForId(resource.id, representation: representation) + if let size = fileSize(path) { + subscriber.putNext(MediaResourceData(path: path, size: size, complete: true)) + subscriber.putCompletion() + } else { + self.dataQueue.async { + let key = CachedMediaResourceRepresentationKey(resourceId: resource.id, representation: representation) + let context: CachedMediaResourceRepresentationContext + if let currentContext = self.cachedRepresentationContexts[key] { + context = currentContext + } else { + context = CachedMediaResourceRepresentationContext() + self.cachedRepresentationContexts[key] = context + } + + let index = context.dataSubscribers.add({ [weak self] data in + subscriber.putNext(data) + if data.complete { + subscriber.putCompletion() + } + }) + + disposable.set(ActionDisposable { + self.dataQueue.async { + if let context = self.cachedRepresentationContexts[key] { + context.dataSubscribers.remove(index) + if context.dataSubscribers.isEmpty { + context.disposable?.dispose() + self.cachedRepresentationContexts.removeValue(forKey: key) + } + } + } + }) + + if context.disposable == nil { + let signal = self.resourceData(resource, complete: true) + |> mapToSignal { resourceData in + return self.wrappedFetchCachedResourceRepresentation.get() + |> take(1) + |> mapToSignal { fetch in + return fetch(resource, resourceData, representation) + } + } + |> deliverOn(self.dataQueue) + context.disposable = signal.start(next: { [weak self] next in + rename(next.temporaryPath, path) + + if let strongSelf = self, let context = strongSelf.cachedRepresentationContexts[key] { + strongSelf.cachedRepresentationContexts.removeValue(forKey: key) + if let size = fileSize(path) { + for subscriber in context.dataSubscribers.copyItems() { + subscriber(MediaResourceData(path: path, size: size, complete: true)) + } + } + } + }) + } + } + } + } + return disposable + } + } } diff --git a/Postbox/MediaResource.swift b/Postbox/MediaResource.swift index 3d83936ca7..537e300cd9 100644 --- a/Postbox/MediaResource.swift +++ b/Postbox/MediaResource.swift @@ -1,5 +1,39 @@ +import Foundation + +public protocol MediaResourceId { + var uniqueId: String { get } + var hashValue: Int { get } + func isEqual(to: MediaResourceId) -> Bool +} + +struct WrappedMediaResourceId: Hashable { + let id: MediaResourceId + + init(_ id: MediaResourceId) { + self.id = id + } + + static func ==(lhs: WrappedMediaResourceId, rhs: WrappedMediaResourceId) -> Bool { + return lhs.id.isEqual(to: rhs.id) + } + + var hashValue: Int { + return self.id.hashValue + } +} public protocol MediaResource { - var id: String { get } - var size: Int { get } + var id: MediaResourceId { get } + var size: Int? { get } +} + +public extension MediaResource { + var size: Int? { + return nil + } +} + +public protocol CachedMediaResourceRepresentation { + var uniqueId: String { get } + func isEqual(to: CachedMediaResourceRepresentation) -> Bool } diff --git a/Postbox/MediaResourceData.swift b/Postbox/MediaResourceData.swift deleted file mode 100644 index 7542471cdc..0000000000 --- a/Postbox/MediaResourceData.swift +++ /dev/null @@ -1,6 +0,0 @@ -import Foundation - -public struct MediaResourceData { - public let path: String - public let size: Int -} diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 180547bbe1..b334fe1730 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -706,6 +706,12 @@ final class MessageHistoryTable: Table { private func justUpdate(_ index: MessageIndex, message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) -> IntermediateMessage? { if let previousMessage = self.getMessage(index) { self.valueBox.remove(self.tableId, key: self.key(index)) + if !previousMessage.tags.isEmpty { + self.tagsTable.remove(previousMessage.tags, index: index) + } + if !message.tags.isEmpty { + self.tagsTable.add(message.tags, index: MessageIndex(message)) + } switch (previousMessage.flags.contains(.Unsent) && !previousMessage.flags.contains(.Failed), message.flags.contains(.Unsent) && !message.flags.contains(.Failed)) { case (true, false): diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index d4dc6bcc9d..ad533ee81b 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -92,7 +92,7 @@ public func <(lhs: PeerId, rhs: PeerId) -> Bool { return false } -public protocol Peer: Coding { +public protocol Peer: class, Coding { var id: PeerId { get } var indexName: PeerIndexNameRepresentation { get } diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index 1d59ffe14d..391b292ca0 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -6,11 +6,13 @@ final class MutablePeerView { var cachedData: CachedPeerData? var peers: [PeerId: Peer] = [:] var peerPresences: [PeerId: PeerPresence] = [:] + var peerIsContact: Bool - init(peerId: PeerId, notificationSettings: PeerNotificationSettings?, cachedData: CachedPeerData?, getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) { + init(peerId: PeerId, notificationSettings: PeerNotificationSettings?, cachedData: CachedPeerData?, peerIsContact: Bool, getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) { self.peerId = peerId self.notificationSettings = notificationSettings self.cachedData = cachedData + self.peerIsContact = peerIsContact var peerIds = Set() peerIds.insert(peerId) if let cachedData = cachedData { @@ -26,7 +28,7 @@ final class MutablePeerView { } } - func replay(updatedPeers: [PeerId: Peer], updatedNotificationSettings: [PeerId: PeerNotificationSettings], updatedCachedPeerData: [PeerId: CachedPeerData], updatedPeerPresences: [PeerId: PeerPresence], getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) -> Bool { + func replay(updatedPeers: [PeerId: Peer], updatedNotificationSettings: [PeerId: PeerNotificationSettings], updatedCachedPeerData: [PeerId: CachedPeerData], updatedPeerPresences: [PeerId: PeerPresence], replaceContactPeerIds: Set?, getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) -> Bool { var updated = false if let cachedData = updatedCachedPeerData[self.peerId], self.cachedData == nil || self.cachedData!.peerIds != cachedData.peerIds { @@ -106,6 +108,7 @@ public final class PeerView { public let notificationSettings: PeerNotificationSettings? public let peers: [PeerId: Peer] public let peerPresences: [PeerId: PeerPresence] + public let peerIsContact: Bool init(_ mutableView: MutablePeerView) { self.peerId = mutableView.peerId @@ -113,5 +116,6 @@ public final class PeerView { self.notificationSettings = mutableView.notificationSettings self.peers = mutableView.peers self.peerPresences = mutableView.peerPresences + self.peerIsContact = mutableView.peerIsContact } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 54c8d4701a..f97c8e1a96 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -130,6 +130,10 @@ public final class Modifier { self.postbox?.updateMedia(id, update: update) } + public func replaceItemCollections(namespace: ItemCollectionId.Namespace, itemCollections: [(ItemCollectionId, ItemCollectionInfo, [ItemCollectionItem])]) { + self.postbox?.replaceItemCollections(namespace: namespace, itemCollections: itemCollections) + } + public func getMessage(_ id: MessageId) -> Message? { if let postbox = self.postbox { if let entry = postbox.messageHistoryIndexTable.get(id) { @@ -237,6 +241,8 @@ public final class Postbox { var readStateTable: MessageHistoryReadStateTable! var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable! var contactsTable: ContactTable! + var itemCollectionInfoTable: ItemCollectionInfoTable! + var itemCollectionItemTable: ItemCollectionItemTable! //temporary var peerRatingTable: RatingTable! @@ -296,7 +302,7 @@ public final class Postbox { } catch _ { } - //let _ = try? NSFileManager.defaultManager().removeItemAtPath(self.basePath + "/media") + //let _ = try? FileManager.default.removeItem(atPath: self.basePath + "/media") //#if TARGET_IPHONE_SIMULATOR @@ -319,7 +325,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 10 + let currentUserVersion: Int32 = 11 if userVersion != currentUserVersion { self.valueBox.drop() @@ -346,6 +352,8 @@ public final class Postbox { self.cachedPeerDataTable = CachedPeerDataTable(valueBox: self.valueBox, tableId: 18) self.peerNotificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox, tableId: 19) self.peerPresenceTable = PeerPresenceTable(valueBox: self.valueBox, tableId: 20) + self.itemCollectionInfoTable = ItemCollectionInfoTable(valueBox: self.valueBox, tableId: 21) + self.itemCollectionItemTable = ItemCollectionItemTable(valueBox: self.valueBox, tableId: 22) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -367,6 +375,8 @@ public final class Postbox { self.tables.append(self.peerNotificationSettingsTable) self.tables.append(self.cachedPeerDataTable) self.tables.append(self.peerPresenceTable) + self.tables.append(self.itemCollectionInfoTable) + self.tables.append(self.itemCollectionItemTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() @@ -852,6 +862,15 @@ public final class Postbox { self.messageHistoryTable.updateMedia(id, media: update, operationsByPeerId: &self.currentOperationsByPeerId, updatedMedia: &self.currentUpdatedMedia) } + fileprivate func replaceItemCollections(namespace: ItemCollectionId.Namespace, itemCollections: [(ItemCollectionId, ItemCollectionInfo, [ItemCollectionItem])]) { + var infos: [(ItemCollectionId, ItemCollectionInfo)] = [] + for (id, info, items) in itemCollections { + infos.append(id, info) + self.itemCollectionItemTable.replaceItems(collectionId: id, items: items) + } + self.itemCollectionInfoTable.replaceInfos(namespace: namespace, infos: infos) + } + fileprivate func filterStoredMessageIds(_ messageIds: Set) -> Set { var filteredIds = Set() @@ -1086,7 +1105,7 @@ public final class Postbox { public func peerView(id: PeerId) -> Signal { return self.modify { modifier -> Signal in - let view = MutablePeerView(peerId: id, notificationSettings: self.peerNotificationSettingsTable.get(id), cachedData: self.cachedPeerDataTable.get(id), getPeer: { self.peerTable.get($0) }, getPeerPresence: { self.peerPresenceTable.get($0) }) + let view = MutablePeerView(peerId: id, notificationSettings: self.peerNotificationSettingsTable.get(id), cachedData: self.cachedPeerDataTable.get(id), peerIsContact: self.contactsTable.isContact(peerId: id), getPeer: { self.peerTable.get($0) }, getPeerPresence: { self.peerPresenceTable.get($0) }) let (index, signal) = self.viewTracker.addPeerView(view) return (.single(PeerView(view)) @@ -1178,6 +1197,23 @@ public final class Postbox { } } + public func itemCollectionsView(namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int) -> Signal { + return self.modify { modifier -> ItemCollectionsView in + let mutableView = MutableItemCollectionsView(namespaces: namespaces, aroundIndex: aroundIndex, count: count, getInfos: { namespace in + return self.itemCollectionInfoTable.getInfos(namespace: namespace) + }, lowerCollectionId: { namespaceList, collectionId, collectionIndex in + return self.itemCollectionInfoTable.lowerCollectionId(namespaceList: namespaceList, collectionId: collectionId, index: collectionIndex) + }, lowerItems: { collectionId, itemIndex, count in + return self.itemCollectionItemTable.lowerItems(collectionId: collectionId, itemIndex: itemIndex, count: count) + }, higherCollectionId: { namespaceList, collectionId, collectionIndex in + return self.itemCollectionInfoTable.higherCollectionId(namespaceList: namespaceList, collectionId: collectionId, index: collectionIndex) + }, higherItems: { collectionId, itemIndex, count in + return self.itemCollectionItemTable.higherItems(collectionId: collectionId, itemIndex: itemIndex, count: count) + }) + return ItemCollectionsView(mutableView) + } + } + public func isMasterClient() -> Signal { return self.modify { modifier -> Signal in let sessionClientId = self.sessionClientId diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index f818c5fbfe..96960d4a9d 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -228,7 +228,7 @@ final class ViewTracker { } } - if mutableView.replay(updatedPeers: updatedPeers, updatedNotificationSettings: updatedNotificationSettings, updatedCachedPeerData: updatedCachedPeerData, updatedPeerPresences: updatedPeerPresences, getPeer: self.getPeer, getPeerPresence: self.getPeerPresence) { + if mutableView.replay(updatedPeers: updatedPeers, updatedNotificationSettings: updatedNotificationSettings, updatedCachedPeerData: updatedCachedPeerData, updatedPeerPresences: updatedPeerPresences, replaceContactPeerIds: nil, getPeer: self.getPeer, getPeerPresence: self.getPeerPresence) { pipe.putNext(PeerView(mutableView)) } } @@ -331,7 +331,7 @@ final class ViewTracker { } for (mutableView, pipe) in self.peerViews.copyItems() { - if mutableView.replay(updatedPeers: transaction.currentUpdatedPeers, updatedNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, updatedCachedPeerData: transaction.currentUpdatedCachedPeerData, updatedPeerPresences: transaction.currentUpdatedPeerPresences, getPeer: self.getPeer, getPeerPresence: self.getPeerPresence) { + if mutableView.replay(updatedPeers: transaction.currentUpdatedPeers, updatedNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, updatedCachedPeerData: transaction.currentUpdatedCachedPeerData, updatedPeerPresences: transaction.currentUpdatedPeerPresences, replaceContactPeerIds: transaction.replaceContactPeerIds, getPeer: self.getPeer, getPeerPresence: self.getPeerPresence) { pipe.putNext(PeerView(mutableView)) } } From 4d308bcb6934bf876f584e1c8b75e2c838ed05a2 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 2 Nov 2016 03:14:12 +0300 Subject: [PATCH 050/427] no message --- Postbox.xcodeproj/project.pbxproj | 6 -- Postbox/CachedMediaResource.swift | 4 - Postbox/ItemCollectionsView.swift | 4 + Postbox/MediaBox.swift | 2 - Postbox/Message.swift | 8 +- Postbox/MessageHistoryIndexTable.swift | 20 ++++ Postbox/MessageHistoryOperation.swift | 1 + Postbox/MessageHistoryTable.swift | 134 +++++++++++++++++------- Postbox/MessageHistoryUnsentTable.swift | 37 ++++--- Postbox/MessageHistoryView.swift | 24 +++++ Postbox/Postbox.swift | 41 +++++--- Postbox/UnsentMessageHistoryView.swift | 26 ++--- Postbox/UnsentMessageIndicesView.swift | 8 +- Postbox/ViewTracker.swift | 24 ++--- 14 files changed, 225 insertions(+), 114 deletions(-) delete mode 100644 Postbox/CachedMediaResource.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index b4e89f3f2c..9c9d7efdfd 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -132,7 +132,6 @@ D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; D0B844061DAB91B5005F29E1 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; D0B844071DAB91B5005F29E1 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; - D0B844081DAB91B5005F29E1 /* CachedMediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */; }; D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerPresence.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; @@ -151,7 +150,6 @@ D0D511041D64D91C00A97B8A /* IpcNotifier.h in Headers */ = {isa = PBXBuildFile; fileRef = D0D511031D64D75200A97B8A /* IpcNotifier.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */; }; D0D949F51D35353900740E02 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; - D0DE76F91D9293B8002B8809 /* CachedMediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */; }; D0DF0C8F1D81A350008AEB01 /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DF0C8E1D81A350008AEB01 /* PeerView.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -259,7 +257,6 @@ D0D511031D64D75200A97B8A /* IpcNotifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = IpcNotifier.h; sourceTree = ""; }; D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessResourceTests.swift; sourceTree = ""; }; D0D949F41D35353900740E02 /* MappedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MappedFile.swift; sourceTree = ""; }; - D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CachedMediaResource.swift; sourceTree = ""; }; D0DF0C8E1D81A350008AEB01 /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -320,7 +317,6 @@ D055BD321B7D3D2D00F06C0A /* MediaBox.swift */, D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */, D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */, - D0DE76F81D9293B8002B8809 /* CachedMediaResource.swift */, D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */, ); name = "Media Box"; @@ -749,7 +745,6 @@ D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, D0B418241D7DFE0C004562A4 /* SimpleSet.swift in Sources */, D03C53601DAD5C9C004C17B3 /* ChatListHole.swift in Sources */, - D0B844081DAB91B5005F29E1 /* CachedMediaResource.swift in Sources */, D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, D0B843EC1DAB91A0005F29E1 /* GlobalMessageIdsTable.swift in Sources */, D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */, @@ -835,7 +830,6 @@ D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, - D0DE76F91D9293B8002B8809 /* CachedMediaResource.swift in Sources */, D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */, D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */, D03120F81DA53FF4006A2A60 /* PeerPresenceTable.swift in Sources */, diff --git a/Postbox/CachedMediaResource.swift b/Postbox/CachedMediaResource.swift deleted file mode 100644 index 464695d4e4..0000000000 --- a/Postbox/CachedMediaResource.swift +++ /dev/null @@ -1,4 +0,0 @@ - -public protocol CachedMediaResource { - var id: String { get } -} diff --git a/Postbox/ItemCollectionsView.swift b/Postbox/ItemCollectionsView.swift index c03c4c6c5b..dbc74c1b56 100644 --- a/Postbox/ItemCollectionsView.swift +++ b/Postbox/ItemCollectionsView.swift @@ -20,6 +20,10 @@ public struct ItemCollectionViewEntryIndex: Comparable { return lhs.collectionIndex < rhs.collectionIndex } } + + public static func lowerBound(collectionIndex: Int32, collectionId: ItemCollectionId) -> ItemCollectionViewEntryIndex { + return ItemCollectionViewEntryIndex(collectionIndex: collectionIndex, collectionId: collectionId, itemIndex: ItemCollectionItemIndex(index: 0, id: 0)) + } } public struct ItemCollectionViewEntry { diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 48610ff890..6d6ebb2f4b 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -87,7 +87,6 @@ private final class CachedMediaResourceRepresentationContext { public final class MediaBox { let basePath: String - let buffer = WriteBuffer() private let statusQueue = Queue() private let concurrentQueue = Queue.concurrentDefaultQueue() @@ -243,7 +242,6 @@ public final class MediaBox { assert(pathExtension == nil) return Signal { subscriber in let disposable = MetaDisposable() - self.concurrentQueue.async { let paths = self.storePathsForId(resource.id) if let completeSize = fileSize(paths.complete) { diff --git a/Postbox/Message.swift b/Postbox/Message.swift index d48a25c826..923c712c9b 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -77,12 +77,16 @@ public struct MessageId: Hashable, Comparable, CustomStringConvertible { } public func ==(lhs: MessageId, rhs: MessageId) -> Bool { - return lhs.id == rhs.id && lhs.namespace == rhs.namespace + return lhs.id == rhs.id && lhs.namespace == rhs.namespace && lhs.peerId == rhs.peerId } public func <(lhs: MessageId, rhs: MessageId) -> Bool { if lhs.namespace == rhs.namespace { - return lhs.id < rhs.id + if lhs.id == rhs.id { + return lhs.peerId < rhs.peerId + } else { + return lhs.id < rhs.id + } } else { return lhs.namespace < rhs.namespace } diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 05ae154959..4e32585703 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -66,6 +66,7 @@ enum MessageHistoryIndexOperation { case InsertHole(MessageHistoryHole) case Remove(MessageIndex) case Update(MessageIndex, InternalStoreMessage) + case UpdateTimestamp(MessageIndex, Int32) } private let HistoryEntryTypeMask: Int8 = 1 @@ -96,6 +97,15 @@ private func readHistoryIndexEntry(_ peerId: PeerId, namespace: MessageId.Namesp } } +private func modifyHistoryIndexEntryTimestamp(value: ReadBuffer, timestamp: Int32) -> MemoryBuffer { + let buffer = WriteBuffer() + buffer.write(value.memory.advanced(by: 0), offset: 0, length: 1) + var varTimestamp: Int32 = timestamp + buffer.write(&varTimestamp, offset: 0, length: 4) + buffer.write(value.memory.advanced(by: 5), offset: 0, length: value.length - 5) + return buffer +} + final class MessageHistoryIndexTable: Table { let globalMessageIdsNamespace: Int32 let globalMessageIdsTable: GlobalMessageIdsTable @@ -355,6 +365,16 @@ final class MessageHistoryIndexTable: Table { } } + func updateTimestamp(_ id: MessageId, timestamp: Int32, operations: inout [MessageHistoryIndexOperation]) { + if let previousData = self.valueBox.get(self.tableId, key: self.key(id)), let previousEntry = self.get(id), case let .Message(previousIndex) = previousEntry, previousIndex.timestamp != timestamp { + let updatedEntry = modifyHistoryIndexEntryTimestamp(value: previousData, timestamp: timestamp) + self.valueBox.remove(self.tableId, key: self.key(id)) + self.valueBox.set(self.tableId, key: self.key(id), value: updatedEntry) + + operations.append(.UpdateTimestamp(MessageIndex(id: id, timestamp: previousIndex.timestamp), timestamp)) + } + } + func fillHole(_ id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [InternalStoreMessage], operations: inout [MessageHistoryIndexOperation]) { self.ensureInitialized(id.peerId, operations: &operations) diff --git a/Postbox/MessageHistoryOperation.swift b/Postbox/MessageHistoryOperation.swift index a71b7be8b2..4bfef815ae 100644 --- a/Postbox/MessageHistoryOperation.swift +++ b/Postbox/MessageHistoryOperation.swift @@ -6,4 +6,5 @@ enum MessageHistoryOperation { case Remove([MessageIndex]) case UpdateReadState(CombinedPeerReadState) case UpdateEmbeddedMedia(MessageIndex, ReadBuffer) + case UpdateTimestamp(MessageIndex, Int32) } diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index b334fe1730..012b87dfe1 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -142,7 +142,7 @@ final class MessageHistoryTable: Table { let message = self.justInsertMessage(storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder) outputOperations.append(.InsertMessage(message)) if message.flags.contains(.Unsent) && !message.flags.contains(.Failed) { - self.unsentTable.add(MessageIndex(message), operations: &unsentMessageOperations) + self.unsentTable.add(message.id, operations: &unsentMessageOperations) } let tags = message.tags.rawValue if tags != 0 { @@ -174,6 +174,13 @@ final class MessageHistoryTable: Table { if let message = self.justUpdate(index, message: storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder, unsentMessageOperations: &unsentMessageOperations) { outputOperations.append(.InsertMessage(message)) } + case let .UpdateTimestamp(index, timestamp): + if accumulatedRemoveIndices.count != 0 { + outputOperations.append(.Remove(accumulatedRemoveIndices)) + accumulatedRemoveIndices.removeAll() + } + self.justUpdateTimestamp(index, timestamp: timestamp) + outputOperations.append(.UpdateTimestamp(index, timestamp)) } } if accumulatedRemoveIndices.count != 0 { @@ -272,6 +279,12 @@ final class MessageHistoryTable: Table { self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } + func updateMessageTimestamp(_ id: MessageId, timestamp: Int32, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + var operations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.updateTimestamp(id, timestamp: timestamp, operations: &operations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + func updateMedia(_ id: MediaId, media: Media?, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedMedia: inout [MediaId: Media?]) { if let previousMedia = self.messageMediaTable.get(id, embedded: { index, id in return self.embeddedMediaForIndex(index, id: id) @@ -410,6 +423,14 @@ final class MessageHistoryTable: Table { return nil } + func offsetPendingMessagesTimestamps(lowerBound: MessageId, timestamp: Int32, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + for messageId in self.unsentTable.get() { + if messageId.peerId == lowerBound.peerId && messageId.namespace == lowerBound.namespace && messageId.id > lowerBound.id { + self.updateMessageTimestamp(messageId, timestamp: timestamp, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + } + } + private func justInsertMessage(_ message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) -> IntermediateMessage { sharedBuffer.reset() @@ -567,7 +588,7 @@ final class MessageHistoryTable: Table { } if message.flags.contains(.Unsent) && !message.flags.contains(.Failed) { - self.unsentTable.remove(index, operations: &unsentMessageOperations) + self.unsentTable.remove(index.id, operations: &unsentMessageOperations) } let tags = message.tags.rawValue @@ -705,6 +726,47 @@ final class MessageHistoryTable: Table { private func justUpdate(_ index: MessageIndex, message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) -> IntermediateMessage? { if let previousMessage = self.getMessage(index) { + var previousEmbeddedMediaWithIds: [(MediaId, Media)] = [] + if previousMessage.embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + let previousEmbeddedMediaData = previousMessage.embeddedMediaData + previousEmbeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + previousEmbeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: previousEmbeddedMediaData.memory + previousEmbeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + if let mediaId = media.id { + previousEmbeddedMediaWithIds.append((mediaId, media)) + } + } + previousEmbeddedMediaData.skip(Int(mediaLength)) + } + } + + var previousMediaIds = Set() + for (mediaId, _) in previousEmbeddedMediaWithIds { + previousMediaIds.insert(mediaId) + } + for mediaId in previousMessage.referencedMedia { + previousMediaIds.insert(mediaId) + } + + var updatedMediaIds = Set() + for media in message.media { + if let mediaId = media.id { + updatedMediaIds.insert(mediaId) + } + } + + if previousMediaIds != updatedMediaIds || index.id != message.id { + for (_, media) in previousEmbeddedMediaWithIds { + self.messageMediaTable.removeEmbeddedMedia(media) + } + for mediaId in previousMessage.referencedMedia { + self.messageMediaTable.removeReference(mediaId) + } + } + self.valueBox.remove(self.tableId, key: self.key(index)) if !previousMessage.tags.isEmpty { self.tagsTable.remove(previousMessage.tags, index: index) @@ -715,13 +777,13 @@ final class MessageHistoryTable: Table { switch (previousMessage.flags.contains(.Unsent) && !previousMessage.flags.contains(.Failed), message.flags.contains(.Unsent) && !message.flags.contains(.Failed)) { case (true, false): - self.unsentTable.remove(index, operations: &unsentMessageOperations) + self.unsentTable.remove(index.id, operations: &unsentMessageOperations) case (false, true): - self.unsentTable.add(MessageIndex(message), operations: &unsentMessageOperations) + self.unsentTable.add(message.id, operations: &unsentMessageOperations) case (true, true): if index != MessageIndex(message) { - self.unsentTable.remove(index, operations: &unsentMessageOperations) - self.unsentTable.add(MessageIndex(message), operations: &unsentMessageOperations) + self.unsentTable.remove(index.id, operations: &unsentMessageOperations) + self.unsentTable.add(message.id, operations: &unsentMessageOperations) } case (false, false): break @@ -731,34 +793,6 @@ final class MessageHistoryTable: Table { assertionFailure() } - var previousMedia: [Media] = [] - if previousMessage.embeddedMediaData.length > 4 { - var embeddedMediaCount: Int32 = 0 - let previousEmbeddedMediaData = previousMessage.embeddedMediaData - previousEmbeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) - for _ in 0 ..< embeddedMediaCount { - var mediaLength: Int32 = 0 - previousEmbeddedMediaData.read(&mediaLength, offset: 0, length: 4) - if let media = Decoder(buffer: MemoryBuffer(memory: previousEmbeddedMediaData.memory + previousEmbeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { - previousMedia.append(media) - } - previousEmbeddedMediaData.skip(Int(mediaLength)) - } - } - - var previousReferencedMedia: [Media] = [] - for mediaId in previousMessage.referencedMedia { - if let media = self.messageMediaTable.get(mediaId, embedded: { _ in - return nil - }) { - previousMedia.append(media) - } - } - - var removedMediaIds: [MediaId] = [] - - //self.updateMedia(from: previousMedia, to: message.media) - sharedBuffer.reset() var type: Int8 = 0 @@ -844,10 +878,10 @@ final class MessageHistoryTable: Table { if let mediaId = media.id { let mediaInsertResult = self.messageMediaTable.set(media, index: MessageIndex(message), messageHistoryTable: self) switch mediaInsertResult { - case let .Embed(media): - embeddedMedia.append(media) - case .Reference: - referencedMedia.append(mediaId) + case let .Embed(media): + embeddedMedia.append(media) + case .Reference: + referencedMedia.append(mediaId) } } else { embeddedMedia.append(media) @@ -885,6 +919,30 @@ final class MessageHistoryTable: Table { } } + private func justUpdateTimestamp(_ index: MessageIndex, timestamp: Int32) { + if let previousMessage = self.getMessage(index) { + self.valueBox.remove(self.tableId, key: self.key(index)) + var updatedMessage = IntermediateMessage(stableId: previousMessage.stableId, id: previousMessage.id, timestamp: timestamp, flags: previousMessage.flags, tags: previousMessage.tags, forwardInfo: previousMessage.forwardInfo, authorId: previousMessage.authorId, text: previousMessage.text, attributesData: previousMessage.attributesData, embeddedMediaData: previousMessage.embeddedMediaData, referencedMedia: previousMessage.referencedMedia) + self.storeIntermediateMessage(updatedMessage, sharedKey: self.key(index)) + + let tags = previousMessage.tags.rawValue + if tags != 0 { + for i in 0 ..< 32 { + let currentTags = tags >> UInt32(i) + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + let tag = MessageTags(rawValue: 1 << UInt32(i)) + self.tagsTable.remove(tag, index: index) + self.tagsTable.add(tag, index: MessageIndex(id: index.id, timestamp: timestamp)) + } + } + } + } + } + func unembedMedia(_ index: MessageIndex, id: MediaId) -> Media? { if let message = self.getMessage(index), message.embeddedMediaData.length > 4 { var embeddedMediaCount: Int32 = 0 diff --git a/Postbox/MessageHistoryUnsentTable.swift b/Postbox/MessageHistoryUnsentTable.swift index ccb5e8627b..a0324ab3fe 100644 --- a/Postbox/MessageHistoryUnsentTable.swift +++ b/Postbox/MessageHistoryUnsentTable.swift @@ -1,18 +1,17 @@ import Foundation enum IntermediateMessageHistoryUnsentOperation { - case Insert(MessageIndex) - case Remove(MessageIndex) + case Insert(MessageId) + case Remove(MessageId) } final class MessageHistoryUnsentTable: Table { - private let sharedKey = ValueBoxKey(length: 4 + 4 + 4 + 8) + private let sharedKey = ValueBoxKey(length: 4 + 4 + 8) - private func key(_ index: MessageIndex) -> ValueBoxKey { - self.sharedKey.setInt32(0, value: index.timestamp) - self.sharedKey.setInt32(4, value: index.id.namespace) - self.sharedKey.setInt32(4 + 4, value: index.id.id) - self.sharedKey.setInt64(4 + 4 + 4, value: index.id.peerId.toInt64()) + private func key(_ id: MessageId) -> ValueBoxKey { + self.sharedKey.setInt32(0, value: id.namespace) + self.sharedKey.setInt32(4, value: id.id) + self.sharedKey.setInt64(4 + 4, value: id.peerId.toInt64()) return self.sharedKey } @@ -24,7 +23,7 @@ final class MessageHistoryUnsentTable: Table { } private func upperBound() -> ValueBoxKey { - let key = ValueBoxKey(length: 4 + 4 + 4 + 8) + let key = ValueBoxKey(length: 4 + 4 + 8) memset(key.memory, 0xff, key.length) return key } @@ -33,23 +32,23 @@ final class MessageHistoryUnsentTable: Table { super.init(valueBox: valueBox, tableId: tableId) } - func add(_ index: MessageIndex, operations: inout [IntermediateMessageHistoryUnsentOperation]) { - self.valueBox.set(self.tableId, key: self.key(index), value: MemoryBuffer()) - operations.append(.Insert(index)) + func add(_ id: MessageId, operations: inout [IntermediateMessageHistoryUnsentOperation]) { + self.valueBox.set(self.tableId, key: self.key(id), value: MemoryBuffer()) + operations.append(.Insert(id)) } - func remove(_ index: MessageIndex, operations: inout [IntermediateMessageHistoryUnsentOperation]) { - self.valueBox.remove(self.tableId, key: self.key(index)) - operations.append(.Remove(index)) + func remove(_ id: MessageId, operations: inout [IntermediateMessageHistoryUnsentOperation]) { + self.valueBox.remove(self.tableId, key: self.key(id)) + operations.append(.Remove(id)) } - func get() -> [MessageIndex] { - var indices: [MessageIndex] = [] + func get() -> [MessageId] { + var ids: [MessageId] = [] self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), keys: { key in - indices.append(MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0))) + ids.append(MessageId(peerId: PeerId(key.getInt64(4 + 4)), namespace: key.getInt32(0), id: key.getInt32(4))) return true }, limit: 0) - return indices + return ids } override func beforeCommit() { diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 77230a6353..631ab927e2 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -46,6 +46,20 @@ enum MutableMessageHistoryEntry { return .HoleEntry(hole, location) } } + + func updatedTimestamp(_ timestamp: Int32) -> MutableMessageHistoryEntry { + switch self { + case let .IntermediateMessageEntry(message, location): + var updatedMessage = IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) + return .IntermediateMessageEntry(updatedMessage, location) + case let .MessageEntry(message, location): + var updatedMessage = Message(stableId: message.stableId, id: message.id, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages) + return .MessageEntry(updatedMessage, location) + case let .HoleEntry(hole, location): + var updatedHole = MessageHistoryHole(stableId: hole.stableId, maxIndex: MessageIndex(id: hole.maxIndex.id, timestamp: timestamp), min: hole.min, tags: hole.tags) + return .HoleEntry(updatedHole, location) + } + } } public struct MessageHistoryEntryLocation: Equatable { @@ -278,6 +292,16 @@ final class MutableMessageHistoryView { break } } + case let .UpdateTimestamp(index, timestamp): + for i in 0 ..< self.entries.count { + let entry = self.entries[i] + if entry.index == index { + self.remove(Set([index]), context: context) + self.add(entry.updatedTimestamp(timestamp), holeFillDirections: [:]) + hasChanges = true + break + } + } } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index f97c8e1a96..a344e41ec0 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -122,8 +122,12 @@ public final class Modifier { self.postbox?.replaceRecentPeerIds(peerIds) } - public func updateMessage(_ index: MessageIndex, update: (Message) -> StoreMessage) { - self.postbox?.updateMessage(index, update: update) + public func updateMessage(_ id: MessageId, update: (Message) -> StoreMessage) { + self.postbox?.updateMessage(id, update: update) + } + + public func offsetPendingMessagesTimestamps(lowerBound: MessageId, timestamp: Int32) { + self.postbox?.offsetPendingMessagesTimestamps(lowerBound: lowerBound, timestamp: timestamp) } public func updateMedia(_ id: MediaId, update: Media?) { @@ -325,7 +329,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 11 + let currentUserVersion: Int32 = 14 if userVersion != currentUserVersion { self.valueBox.drop() @@ -388,7 +392,7 @@ public final class Postbox { return self.cachedPeerDataTable.get(peerId) }, getPeerPresence: { peerId in return self.peerPresenceTable.get(peerId) - },unsentMessageIndices: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) + },unsentMessageIds: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") }) @@ -735,7 +739,7 @@ public final class Postbox { for table in self.tables { table.clearMemoryCache() } - self.viewTracker.refreshViewsDueToExternalTransaction(fetchAroundChatEntries: self.fetchAroundChatEntries, fetchAroundHistoryEntries: self.fetchAroundHistoryEntries, fetchUnsendMessageIndices: { + self.viewTracker.refreshViewsDueToExternalTransaction(fetchAroundChatEntries: self.fetchAroundChatEntries, fetchAroundHistoryEntries: self.fetchAroundHistoryEntries, fetchUnsentMessageIds: { return self.messageHistoryUnsentTable!.get() }, fetchSynchronizePeerReadStateOperations: { return self.synchronizeReadStateTable!.get() @@ -851,13 +855,17 @@ public final class Postbox { self.peerRatingTable.replace(items: peerIds) } - fileprivate func updateMessage(_ index: MessageIndex, update: (Message) -> StoreMessage) { - if let intermediateMessage = self.messageHistoryTable.getMessage(index) { + fileprivate func updateMessage(_ id: MessageId, update: (Message) -> StoreMessage) { + if let indexEntry = self.messageHistoryIndexTable.get(id), let intermediateMessage = self.messageHistoryTable.getMessage(indexEntry.index) { let message = self.renderIntermediateMessage(intermediateMessage) - self.messageHistoryTable.updateMessage(index.id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + self.messageHistoryTable.updateMessage(id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } } + fileprivate func offsetPendingMessagesTimestamps(lowerBound: MessageId, timestamp: Int32) { + self.messageHistoryTable.offsetPendingMessagesTimestamps(lowerBound: lowerBound, timestamp: timestamp, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + } + fileprivate func updateMedia(_ id: MediaId, update: Media?) { self.messageHistoryTable.updateMedia(id, media: update, operationsByPeerId: &self.currentOperationsByPeerId, updatedMedia: &self.currentUpdatedMedia) } @@ -883,9 +891,9 @@ public final class Postbox { return filteredIds } - public func modify(_ f: @escaping(Modifier) -> T) -> Signal { + public func modify(userInteractive: Bool = false, _ f: @escaping(Modifier) -> T) -> Signal { return Signal { subscriber in - self.queue.justDispatch { + let f: () -> Void = { self.valueBox.begin() self.afterBegin() let result = f(Modifier(postbox: self)) @@ -903,18 +911,23 @@ public final class Postbox { self.masterClientId.set(.single(updatedMasterClientId)) } } + if userInteractive { + self.queue.justDispatchWithQoS(qos: DispatchQoS.userInteractive, f) + } else { + self.queue.justDispatch(f) + } return EmptyDisposable } } public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType), NoError> in + return self.modify(userInteractive: true, { modifier -> Signal<(MessageHistoryView, ViewUpdateType), NoError> in var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let maxReadIndex = self.messageHistoryTable.maxReadIndex(peerId) { index = maxReadIndex } return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, tagMask: tagMask) - } |> switchToLatest + }) |> switchToLatest } public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { @@ -1173,11 +1186,11 @@ public final class Postbox { } } - public func unsentMessageIndicesView() -> Signal { + public func unsentMessageIdsView() -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.queue.async { - disposable.set(self.viewTracker.unsentMessageIndicesViewSignal().start(next: { view in + disposable.set(self.viewTracker.unsentMessageIdsViewSignal().start(next: { view in subscriber.putNext(view) })) } diff --git a/Postbox/UnsentMessageHistoryView.swift b/Postbox/UnsentMessageHistoryView.swift index a6322bf1e7..6769deb967 100644 --- a/Postbox/UnsentMessageHistoryView.swift +++ b/Postbox/UnsentMessageHistoryView.swift @@ -1,16 +1,16 @@ import Foundation final class UnsentMessageHistoryView { - var indices: Set + var ids: Set - init(indices: [MessageIndex]) { - self.indices = Set(indices) + init(ids: [MessageId]) { + self.ids = Set(ids) } - func refreshDueToExternalTransaction(fetchUnsendMessageIndices: () -> [MessageIndex]) -> Bool { - let indices = Set(fetchUnsendMessageIndices()) - if indices != self.indices { - self.indices = indices + func refreshDueToExternalTransaction(fetchUnsentMessageIds: () -> [MessageId]) -> Bool { + let ids = Set(fetchUnsentMessageIds()) + if ids != self.ids { + self.ids = ids return true } else { return false @@ -21,14 +21,14 @@ final class UnsentMessageHistoryView { var updated = false for operation in operations { switch operation { - case let .Insert(index): - if !self.indices.contains(index) { - self.indices.insert(index) + case let .Insert(id): + if !self.ids.contains(id) { + self.ids.insert(id) updated = true } - case let .Remove(index): - if self.indices.contains(index) { - self.indices.remove(index) + case let .Remove(id): + if self.ids.contains(id) { + self.ids.remove(id) updated = true } } diff --git a/Postbox/UnsentMessageIndicesView.swift b/Postbox/UnsentMessageIndicesView.swift index e7047877f5..59848c6b7c 100644 --- a/Postbox/UnsentMessageIndicesView.swift +++ b/Postbox/UnsentMessageIndicesView.swift @@ -1,9 +1,9 @@ import Foundation -public final class UnsentMessageIndicesView { - public let indices: Set +public final class UnsentMessageIdsView { + public let ids: Set - init(_ indices: Set) { - self.indices = indices + init(_ ids: Set) { + self.ids = ids } } diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 96960d4a9d..14b5b0c173 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -37,14 +37,14 @@ final class ViewTracker { private let chatListHolesViewSubscribers = Bag>() private var unsentMessageView: UnsentMessageHistoryView - private let unsendMessageIndicesViewSubscribers = Bag>() + private let unsendMessageIdsViewSubscribers = Bag>() private var synchronizeReadStatesView: MutableSynchronizePeerReadStatesView private let synchronizePeerReadStatesViewSubscribers = Bag>() private var peerViews = Bag<(MutablePeerView, ValuePipe)>() - init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, unsentMessageIndices: [MessageIndex], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -57,7 +57,7 @@ final class ViewTracker { self.getCachedPeerData = getCachedPeerData self.getPeerPresence = getPeerPresence - self.unsentMessageView = UnsentMessageHistoryView(indices: unsentMessageIndices) + self.unsentMessageView = UnsentMessageHistoryView(ids: unsentMessageIds) self.synchronizeReadStatesView = MutableSynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) } @@ -165,7 +165,7 @@ final class ViewTracker { self.peerViews.remove(index) } - func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: MessageIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsendMessageIndices: () -> [MessageIndex], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: MessageIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { @@ -192,7 +192,7 @@ final class ViewTracker { self.updateTrackedHoles(peerId) } - if self.unsentMessageView.refreshDueToExternalTransaction(fetchUnsendMessageIndices: fetchUnsendMessageIndices) { + if self.unsentMessageView.refreshDueToExternalTransaction(fetchUnsentMessageIds: fetchUnsentMessageIds) { self.unsentViewUpdated() } @@ -371,8 +371,8 @@ final class ViewTracker { } private func unsentViewUpdated() { - for subscriber in self.unsendMessageIndicesViewSubscribers.copyItems() { - subscriber.putNext(UnsentMessageIndicesView(self.unsentMessageView.indices)) + for subscriber in self.unsendMessageIdsViewSubscribers.copyItems() { + subscriber.putNext(UnsentMessageIdsView(self.unsentMessageView.ids)) } } @@ -426,14 +426,14 @@ final class ViewTracker { } } - func unsentMessageIndicesViewSignal() -> Signal { + func unsentMessageIdsViewSignal() -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.queue.async { - subscriber.putNext(UnsentMessageIndicesView(self.unsentMessageView.indices)) + subscriber.putNext(UnsentMessageIdsView(self.unsentMessageView.ids)) - let pipe = ValuePipe() - let index = self.unsendMessageIndicesViewSubscribers.add(pipe) + let pipe = ValuePipe() + let index = self.unsendMessageIdsViewSubscribers.add(pipe) let pipeDisposable = pipe.signal().start(next: { view in subscriber.putNext(view) @@ -441,7 +441,7 @@ final class ViewTracker { disposable.set(ActionDisposable { pipeDisposable.dispose() - self.unsendMessageIndicesViewSubscribers.remove(index) + self.unsendMessageIdsViewSubscribers.remove(index) }) } return disposable From 6c5875013a28eda32d723fe49717168b8c704e80 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 3 Nov 2016 23:01:04 +0300 Subject: [PATCH 051/427] no message --- Postbox.xcodeproj/project.pbxproj | 12 +++++ Postbox/Coding.swift | 2 +- Postbox/InitialMessageHistoryData.swift | 6 +++ Postbox/PeerChatInterfaceState.swift | 12 +++++ Postbox/PeerChatInterfaceStateTable.swift | 64 +++++++++++++++++++++++ Postbox/Postbox.swift | 50 ++++++++++++++---- 6 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 Postbox/InitialMessageHistoryData.swift create mode 100644 Postbox/PeerChatInterfaceState.swift create mode 100644 Postbox/PeerChatInterfaceStateTable.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 9c9d7efdfd..519ce3fb02 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -54,6 +54,9 @@ D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; + D07CFF811DCA765D00761F81 /* PeerChatInterfaceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */; }; + D07CFF831DCA909100761F81 /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; + D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */; }; D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; @@ -221,6 +224,9 @@ D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; + D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatInterfaceState.swift; sourceTree = ""; }; + D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatInterfaceStateTable.swift; sourceTree = ""; }; + D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = InitialMessageHistoryData.swift; sourceTree = ""; }; D08C71391C501F0700779C0F /* MessageHistoryHole.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHole.swift; sourceTree = ""; }; D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTable.swift; sourceTree = ""; }; D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTable.swift; sourceTree = ""; }; @@ -397,6 +403,7 @@ D08D451E1D5D2CA700A7428A /* RatingTable.swift */, D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */, D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */, + D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */, ); name = Tables; sourceTree = ""; @@ -416,6 +423,7 @@ D0B844501DAC04FE005F29E1 /* PeerPresence.swift */, D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */, D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */, + D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */, ); name = Objects; sourceTree = ""; @@ -447,6 +455,7 @@ isa = PBXGroup; children = ( D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */, + D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */, D0F9E86A1C59719800037222 /* ChatListView.swift */, D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */, D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */, @@ -778,6 +787,7 @@ files = ( D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, D08D451F1D5D2CA700A7428A /* RatingTable.swift in Sources */, + D07CFF831DCA909100761F81 /* PeerChatInterfaceStateTable.swift in Sources */, D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, @@ -793,6 +803,7 @@ D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */, D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */, + D07CFF811DCA765D00761F81 /* PeerChatInterfaceState.swift in Sources */, D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */, D021E0D81DB4FD1300C6B04F /* ItemCollectionItemTable.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, @@ -823,6 +834,7 @@ D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, + D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */, D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index aed292f76b..67eec67e5f 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -910,7 +910,7 @@ public final class Decoder { } } - public func decodeBytesForKeyNoCopy(_ key: StaticString) -> ReadBuffer! { + public func decodeBytesForKeyNoCopy(_ key: StaticString) -> ReadBuffer? { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bytes) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) diff --git a/Postbox/InitialMessageHistoryData.swift b/Postbox/InitialMessageHistoryData.swift new file mode 100644 index 0000000000..178b92fda9 --- /dev/null +++ b/Postbox/InitialMessageHistoryData.swift @@ -0,0 +1,6 @@ +import Foundation + +public struct InitialMessageHistoryData { + public let peer: Peer? + public let chatInterfaceState: PeerChatInterfaceState? +} diff --git a/Postbox/PeerChatInterfaceState.swift b/Postbox/PeerChatInterfaceState.swift new file mode 100644 index 0000000000..da475f2186 --- /dev/null +++ b/Postbox/PeerChatInterfaceState.swift @@ -0,0 +1,12 @@ + +public protocol PeerChatListEmbeddedInterfaceState: Coding { + var timestamp: Int32 { get } + + func isEqual(to: PeerChatListEmbeddedInterfaceState) -> Bool +} + +public protocol PeerChatInterfaceState: Coding { + var chatListEmbeddedState: PeerChatListEmbeddedInterfaceState? { get } + + func isEqual(to: PeerChatInterfaceState) -> Bool +} diff --git a/Postbox/PeerChatInterfaceStateTable.swift b/Postbox/PeerChatInterfaceStateTable.swift new file mode 100644 index 0000000000..2c2dbe1937 --- /dev/null +++ b/Postbox/PeerChatInterfaceStateTable.swift @@ -0,0 +1,64 @@ +import Foundation + +final class PeerChatInterfaceStateTable: Table { + private var states: [PeerId: PeerChatInterfaceState?] = [:] + private var peerIdsWithUpdatedStates = Set() + + private let sharedKey = ValueBoxKey(length: 8) + + private func key(_ peerId: PeerId, sharedKey: ValueBoxKey = ValueBoxKey(length: 8)) -> ValueBoxKey { + sharedKey.setInt64(0, value: peerId.toInt64()) + return sharedKey + } + + func get(_ peerId: PeerId) -> PeerChatInterfaceState? { + if let cachedValue = self.states[peerId] { + return cachedValue + } else if let value = self.valueBox.get(self.tableId, key: self.key(peerId, sharedKey: self.sharedKey)), let state = Decoder(buffer: value).decodeRootObject() as? PeerChatInterfaceState { + self.states[peerId] = state + return state + } else { + self.states[peerId] = nil + return nil + } + } + + func set(_ peerId: PeerId, state: PeerChatInterfaceState?) { + let currentState = self.get(peerId) + var updated = false + if let currentState = currentState, let state = state { + if !currentState.isEqual(to: state) { + updated = true + } + } else if (currentState != nil) != (state != nil) { + updated = true + } + if updated { + self.states[peerId] = state + self.peerIdsWithUpdatedStates.insert(peerId) + } + } + + override func clearMemoryCache() { + self.states.removeAll() + self.peerIdsWithUpdatedStates.removeAll() + } + + override func beforeCommit() { + if !self.peerIdsWithUpdatedStates.isEmpty { + let sharedEncoder = Encoder() + for peerId in self.peerIdsWithUpdatedStates { + if let state = self.states[peerId] { + if let state = state { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(state) + self.valueBox.set(self.tableId, key: self.key(peerId, sharedKey: self.sharedKey), value: sharedEncoder.readBufferNoCopy()) + } else { + self.valueBox.remove(self.tableId, key: self.key(peerId, sharedKey: self.sharedKey)) + } + } + } + self.peerIdsWithUpdatedStates.removeAll() + } + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index a344e41ec0..d0598c9e9d 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -86,6 +86,10 @@ public final class Modifier { self.postbox?.peerChatStateTable.set(id, state: state) } + public func updatePeerChatInterfaceState(_ id: PeerId, update: (PeerChatInterfaceState?) -> (PeerChatInterfaceState?)) { + self.postbox?.updatePeerChatInterfaceState(id, update: update) + } + public func getPeer(_ id: PeerId) -> Peer? { return self.postbox?.peerTable.get(id) } @@ -247,6 +251,7 @@ public final class Postbox { var contactsTable: ContactTable! var itemCollectionInfoTable: ItemCollectionInfoTable! var itemCollectionItemTable: ItemCollectionItemTable! + var peerChatInterfaceStateTable: PeerChatInterfaceStateTable! //temporary var peerRatingTable: RatingTable! @@ -358,6 +363,7 @@ public final class Postbox { self.peerPresenceTable = PeerPresenceTable(valueBox: self.valueBox, tableId: 20) self.itemCollectionInfoTable = ItemCollectionInfoTable(valueBox: self.valueBox, tableId: 21) self.itemCollectionItemTable = ItemCollectionItemTable(valueBox: self.valueBox, tableId: 22) + self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox, tableId: 23) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -381,6 +387,7 @@ public final class Postbox { self.tables.append(self.peerPresenceTable) self.tables.append(self.itemCollectionInfoTable) self.tables.append(self.itemCollectionItemTable) + self.tables.append(self.peerChatInterfaceStateTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() @@ -845,6 +852,11 @@ public final class Postbox { } } + fileprivate func updatePeerChatInterfaceState(_ id: PeerId, update: (PeerChatInterfaceState?) -> (PeerChatInterfaceState?)) { + let updatedState = update(self.peerChatInterfaceStateTable.get(id)) + self.peerChatInterfaceStateTable.set(id, state: updatedState) + } + fileprivate func replaceContactPeerIds(_ peerIds: Set) { self.contactsTable.replace(peerIds) @@ -920,8 +932,8 @@ public final class Postbox { } } - public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return self.modify(userInteractive: true, { modifier -> Signal<(MessageHistoryView, ViewUpdateType), NoError> in + public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + return self.modify(userInteractive: true, { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let maxReadIndex = self.messageHistoryTable.maxReadIndex(peerId) { index = maxReadIndex @@ -930,8 +942,8 @@ public final class Postbox { }) |> switchToLatest } - public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType), NoError> in + public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let anchorIndex = self.messageHistoryTable.anchorIndex(messageId) { index = anchorIndex @@ -940,8 +952,8 @@ public final class Postbox { } |> switchToLatest } - public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { - return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType), NoError> in + public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, tagMask: tagMask) } |> switchToLatest } @@ -972,7 +984,7 @@ public final class Postbox { } } - private func syncAroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { + private func syncAroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { let startTime = CFAbsoluteTimeGetCurrent() let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") @@ -989,8 +1001,10 @@ public final class Postbox { let (index, signal) = self.viewTracker.addMessageHistoryView(peerId, view: mutableView) - return (.single((MessageHistoryView(mutableView), initialUpdateType)) - |> then(signal)) + let initialData = self.initialMessageHistoryData(peerId: peerId) + + return (.single((MessageHistoryView(mutableView), initialUpdateType, initialData)) + |> then(signal |> map { ($0.0, $0.1, nil) })) |> afterDisposed { [weak self] in if let strongSelf = self { strongSelf.queue.async { @@ -1000,6 +1014,10 @@ public final class Postbox { } } + private func initialMessageHistoryData(peerId: PeerId) -> InitialMessageHistoryData { + return InitialMessageHistoryData(peer: self.peerTable.get(peerId), chatInterfaceState: self.peerChatInterfaceStateTable.get(peerId)) + } + public func messageIndexAtId(_ id: MessageId) -> Signal { return self.modify { modifier -> Signal in if let entry = self.messageHistoryIndexTable.get(id), case let .Message(index) = entry { @@ -1028,6 +1046,20 @@ public final class Postbox { } |> switchToLatest } + public func messagesAtIds(_ ids: [MessageId]) -> Signal<[Message], NoError> { + return self.modify { modifier -> Signal<[Message], NoError> in + var messages: [Message] = [] + for id in ids { + if let entry = self.messageHistoryIndexTable.get(id), case let .Message(index) = entry { + if let message = self.messageHistoryTable.getMessage(index) { + messages.append(self.renderIntermediateMessage(message)) + } + } + } + return .single(messages) + } |> switchToLatest + } + public func tailChatListView(_ count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { return self.aroundChatListView(MessageIndex.absoluteUpperBound(), count: count) } From 295f44792afe96a4b541718ef02e9b5242752192 Mon Sep 17 00:00:00 2001 From: overtake Date: Fri, 4 Nov 2016 00:06:24 +0300 Subject: [PATCH 052/427] files --- Postbox.xcodeproj/project.pbxproj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 9c9d7efdfd..74dce2b0a2 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -7,6 +7,10 @@ objects = { /* Begin PBXBuildFile section */ + C26A71921DC9E96F00F69385 /* ItemCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */; }; + C26A71931DC9E97700F69385 /* ItemCollectionInfoTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */; }; + C26A71941DC9E97E00F69385 /* ItemCollectionsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */; }; + C26A71951DC9E98900F69385 /* ItemCollectionItemTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */; }; D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */; }; @@ -701,6 +705,10 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + C26A71951DC9E98900F69385 /* ItemCollectionItemTable.swift in Sources */, + C26A71941DC9E97E00F69385 /* ItemCollectionsView.swift in Sources */, + C26A71931DC9E97700F69385 /* ItemCollectionInfoTable.swift in Sources */, + C26A71921DC9E96F00F69385 /* ItemCollection.swift in Sources */, D03C535C1DAD5C9C004C17B3 /* Message.swift in Sources */, D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, From fe5ab3647b718ebb4819afe99b7ba769031ec3c7 Mon Sep 17 00:00:00 2001 From: overtake Date: Fri, 4 Nov 2016 00:07:09 +0300 Subject: [PATCH 053/427] no message --- .../xcschemes/Postbox.xcscheme | 99 +++++++++++++++++++ .../xcschemes/PostboxMac.xcscheme | 80 +++++++++++++++ .../xcschemes/xcschememanagement.plist | 37 +++++++ 3 files changed, 216 insertions(+) create mode 100644 Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/Postbox.xcscheme create mode 100644 Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/PostboxMac.xcscheme create mode 100644 Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist diff --git a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/Postbox.xcscheme b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/Postbox.xcscheme new file mode 100644 index 0000000000..46e76ad439 --- /dev/null +++ b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/Postbox.xcscheme @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/PostboxMac.xcscheme b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/PostboxMac.xcscheme new file mode 100644 index 0000000000..f4a9378c19 --- /dev/null +++ b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/PostboxMac.xcscheme @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 0000000000..f92a60b4aa --- /dev/null +++ b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,37 @@ + + + + + SchemeUserState + + Postbox.xcscheme + + orderHint + 5 + + PostboxMac.xcscheme + + orderHint + 6 + + + SuppressBuildableAutocreation + + D0B418121D7DFAF2004562A4 + + primary + + + D0E3A7491B28A7E300A402D9 + + primary + + + D0E3A7541B28A7E300A402D9 + + primary + + + + + From 9c3067aa07e312f40a1bd4b88ba95af750c72b02 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 4 Nov 2016 01:39:26 +0300 Subject: [PATCH 054/427] no message --- Postbox.xcodeproj/project.pbxproj | 154 ++++++++++++---------- Postbox/ChatListTable.swift | 61 +++++---- Postbox/ChatListView.swift | 109 +++++++++------ Postbox/MessageHistoryTable.swift | 10 ++ Postbox/PeerChatInterfaceStateTable.swift | 12 +- Postbox/Postbox.swift | 37 +++--- Postbox/PostboxTransaction.swift | 7 +- 7 files changed, 238 insertions(+), 152 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 519ce3fb02..8c3ebdccf2 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -28,17 +28,6 @@ D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; D03BCCF81C73561C0097A291 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; - D03C535B1DAD5C9C004C17B3 /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; - D03C535C1DAD5C9C004C17B3 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; - D03C535D1DAD5C9C004C17B3 /* IntermediateMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */; }; - D03C535E1DAD5C9C004C17B3 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; - D03C535F1DAD5C9C004C17B3 /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; - D03C53601DAD5C9C004C17B3 /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; - D03C53611DAD5C9C004C17B3 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; - D03C53621DAD5C9C004C17B3 /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; - D03C53631DAD5C9C004C17B3 /* CachedPeerData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F91DA540F0006A2A60 /* CachedPeerData.swift */; }; - D03C53641DAD5C9C004C17B3 /* PeerPresence.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerPresence.swift */; }; - D03C53651DAD5C9C004C17B3 /* PeerNotificationSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */; }; D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA291C617D39002160FF /* SeedConfiguration.swift */; }; D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; @@ -48,6 +37,48 @@ D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; D060B77B1CF4845A0050BE9B /* ReadStateTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */; }; + D073CE741DCBF3B4007511FD /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; + D073CE751DCBF3B4007511FD /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; + D073CE761DCBF3B4007511FD /* IntermediateMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */; }; + D073CE771DCBF3B4007511FD /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; + D073CE781DCBF3B4007511FD /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; + D073CE791DCBF3B4007511FD /* ChatListHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2D1C618373002160FF /* ChatListHole.swift */; }; + D073CE7A1DCBF3B4007511FD /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; + D073CE7B1DCBF3B4007511FD /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; + D073CE7C1DCBF3B4007511FD /* CachedPeerData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F91DA540F0006A2A60 /* CachedPeerData.swift */; }; + D073CE7D1DCBF3B4007511FD /* PeerPresence.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerPresence.swift */; }; + D073CE7E1DCBF3B4007511FD /* PeerNotificationSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */; }; + D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */; }; + D073CE801DCBF3B4007511FD /* PeerChatInterfaceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */; }; + D073CE841DCBF3BB007511FD /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; + D073CE851DCBF3BB007511FD /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; + D073CE861DCBF3BB007511FD /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; + D073CE871DCBF3BB007511FD /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; + D073CE881DCBF3BB007511FD /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; + D073CE891DCBF3BB007511FD /* MessageHistoryOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */; }; + D073CE8A1DCBF3BB007511FD /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; + D073CE8B1DCBF3BB007511FD /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; + D073CE8C1DCBF3BB007511FD /* ChatListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8641C58CB7F00037222 /* ChatListTable.swift */; }; + D073CE8D1DCBF3BB007511FD /* ChatListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */; }; + D073CE8E1DCBF3BB007511FD /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; + D073CE8F1DCBF3BB007511FD /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; + D073CE901DCBF3BB007511FD /* MetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */; }; + D073CE911DCBF3BB007511FD /* KeychainTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */; }; + D073CE921DCBF3BB007511FD /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; + D073CE931DCBF3BB007511FD /* PeerNotificationSettingsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */; }; + D073CE941DCBF3BB007511FD /* CachedPeerDataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */; }; + D073CE951DCBF3BB007511FD /* PeerPresenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */; }; + D073CE961DCBF3BB007511FD /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; + D073CE971DCBF3BB007511FD /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; + D073CE981DCBF3BB007511FD /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; + D073CE991DCBF3BB007511FD /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; + D073CE9A1DCBF3BB007511FD /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; + D073CE9B1DCBF3BB007511FD /* RatingTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08D451E1D5D2CA700A7428A /* RatingTable.swift */; }; + D073CE9C1DCBF3BB007511FD /* ItemCollectionInfoTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */; }; + D073CE9D1DCBF3BB007511FD /* ItemCollectionItemTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */; }; + D073CE9E1DCBF3BB007511FD /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; + D073CE9F1DCBF3C1007511FD /* InitialMessageHistoryData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */; }; + D073CEA01DCBF3C1007511FD /* ItemCollectionsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */; }; D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-ambiguous-macro -Wno-conditional-uninitialized -Wno-unused-const-variable -Wno-unused-function -Wno-unreachable-code"; }; }; D07516451B2D9CEF00AE42E0 /* sqlite3.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516411B2D9CEF00AE42E0 /* sqlite3.h */; }; D07516461B2D9CEF00AE42E0 /* sqlite3ext.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516421B2D9CEF00AE42E0 /* sqlite3ext.h */; }; @@ -107,30 +138,6 @@ D0B4185D1D7DFE35004562A4 /* IpcNotifier.h in Headers */ = {isa = PBXBuildFile; fileRef = D0D511031D64D75200A97B8A /* IpcNotifier.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */ = {isa = PBXBuildFile; fileRef = D0D511011D64D73D00A97B8A /* IpcNotifier.mm */; }; D0B418611D7DFE95004562A4 /* SwiftSignalKitMac.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */; }; - D0B843EB1DAB91A0005F29E1 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; - D0B843EC1DAB91A0005F29E1 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; - D0B843ED1DAB91A0005F29E1 /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; - D0B843EE1DAB91A0005F29E1 /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; - D0B843EF1DAB91A0005F29E1 /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; - D0B843F01DAB91A0005F29E1 /* MessageHistoryOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */; }; - D0B843F11DAB91A0005F29E1 /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; - D0B843F21DAB91A0005F29E1 /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; - D0B843F31DAB91A0005F29E1 /* ChatListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8641C58CB7F00037222 /* ChatListTable.swift */; }; - D0B843F41DAB91A0005F29E1 /* ChatListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */; }; - D0B843F51DAB91A0005F29E1 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; - D0B843F61DAB91A0005F29E1 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; - D0B843F71DAB91A0005F29E1 /* MetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */; }; - D0B843F81DAB91A0005F29E1 /* KeychainTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */; }; - D0B843F91DAB91A0005F29E1 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; - D0B843FA1DAB91A0005F29E1 /* PeerNotificationSettingsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */; }; - D0B843FB1DAB91A0005F29E1 /* CachedPeerDataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */; }; - D0B843FC1DAB91A0005F29E1 /* PeerPresenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */; }; - D0B843FD1DAB91A0005F29E1 /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; - D0B843FE1DAB91A0005F29E1 /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; - D0B843FF1DAB91A0005F29E1 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; - D0B844001DAB91A0005F29E1 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; - D0B844011DAB91A0005F29E1 /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; - D0B844021DAB91A0005F29E1 /* RatingTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08D451E1D5D2CA700A7428A /* RatingTable.swift */; }; D0B844031DAB91A7005F29E1 /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DF0C8E1D81A350008AEB01 /* PeerView.swift */; }; D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; D0B844061DAB91B5005F29E1 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; @@ -710,74 +717,81 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - D03C535C1DAD5C9C004C17B3 /* Message.swift in Sources */, D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, + D073CE7E1DCBF3B4007511FD /* PeerNotificationSettings.swift in Sources */, D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, D0B418251D7DFE0C004562A4 /* RedBlackTree.swift in Sources */, - D0B843EF1DAB91A0005F29E1 /* MessageHistoryTable.swift in Sources */, + D073CE851DCBF3BB007511FD /* GlobalMessageIdsTable.swift in Sources */, + D073CE9C1DCBF3BB007511FD /* ItemCollectionInfoTable.swift in Sources */, D0B844071DAB91B5005F29E1 /* MediaResource.swift in Sources */, D0B418261D7DFE0C004562A4 /* MappedFile.swift in Sources */, - D0B843ED1DAB91A0005F29E1 /* MessageHistoryMetadataTable.swift in Sources */, - D0B843FD1DAB91A0005F29E1 /* PeerChatStateTable.swift in Sources */, - D03C535B1DAD5C9C004C17B3 /* Peer.swift in Sources */, - D0B844021DAB91A0005F29E1 /* RatingTable.swift in Sources */, - D0B843EE1DAB91A0005F29E1 /* MessageHistoryIndexTable.swift in Sources */, - D0B843FA1DAB91A0005F29E1 /* PeerNotificationSettingsTable.swift in Sources */, + D073CE771DCBF3B4007511FD /* Media.swift in Sources */, + D073CE961DCBF3BB007511FD /* PeerChatStateTable.swift in Sources */, + D073CE971DCBF3BB007511FD /* ContactTable.swift in Sources */, + D073CE791DCBF3B4007511FD /* ChatListHole.swift in Sources */, D0B4181C1D7DFDF1004562A4 /* SQLite-Bridging.m in Sources */, - D0B843FB1DAB91A0005F29E1 /* CachedPeerDataTable.swift in Sources */, + D073CE8E1DCBF3BB007511FD /* MessageMediaTable.swift in Sources */, + D073CE9E1DCBF3BB007511FD /* PeerChatInterfaceStateTable.swift in Sources */, + D073CE871DCBF3BB007511FD /* MessageHistoryIndexTable.swift in Sources */, D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */, - D0B843F41DAB91A0005F29E1 /* ChatListIndexTable.swift in Sources */, D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */, - D0B843F11DAB91A0005F29E1 /* MessageHistoryTagsTable.swift in Sources */, - D0B844001DAB91A0005F29E1 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, - D03C53651DAD5C9C004C17B3 /* PeerNotificationSettings.swift in Sources */, + D073CE911DCBF3BB007511FD /* KeychainTable.swift in Sources */, + D073CE9D1DCBF3BB007511FD /* ItemCollectionItemTable.swift in Sources */, D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */, - D0B843EB1DAB91A0005F29E1 /* Table.swift in Sources */, - D03C535E1DAD5C9C004C17B3 /* Media.swift in Sources */, - D0B843F91DAB91A0005F29E1 /* PeerTable.swift in Sources */, D0B4184D1D7DFE20004562A4 /* ContactPeersView.swift in Sources */, D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */, - D03C535D1DAD5C9C004C17B3 /* IntermediateMessage.swift in Sources */, - D0B843FE1DAB91A0005F29E1 /* ContactTable.swift in Sources */, + D073CE8A1DCBF3BB007511FD /* MessageHistoryTagsTable.swift in Sources */, + D073CE861DCBF3BB007511FD /* MessageHistoryMetadataTable.swift in Sources */, D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */, + D073CE781DCBF3B4007511FD /* MessageHistoryHole.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, - D0B843F01DAB91A0005F29E1 /* MessageHistoryOperation.swift in Sources */, - D0B843F31DAB91A0005F29E1 /* ChatListTable.swift in Sources */, - D03C53631DAD5C9C004C17B3 /* CachedPeerData.swift in Sources */, + D073CE891DCBF3BB007511FD /* MessageHistoryOperation.swift in Sources */, + D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */, + D073CEA01DCBF3C1007511FD /* ItemCollectionsView.swift in Sources */, D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */, - D0B843F51DAB91A0005F29E1 /* MessageMediaTable.swift in Sources */, D098C6F21D7E1201007784E4 /* Database.swift in Sources */, + D073CE951DCBF3BB007511FD /* PeerPresenceTable.swift in Sources */, + D073CE841DCBF3BB007511FD /* Table.swift in Sources */, D0B844061DAB91B5005F29E1 /* MediaResourceStatus.swift in Sources */, + D073CE761DCBF3B4007511FD /* IntermediateMessage.swift in Sources */, D0B4185A1D7DFE29004562A4 /* ViewTracker.swift in Sources */, + D073CE881DCBF3BB007511FD /* MessageHistoryTable.swift in Sources */, + D073CE7C1DCBF3B4007511FD /* CachedPeerData.swift in Sources */, + D073CE9F1DCBF3C1007511FD /* InitialMessageHistoryData.swift in Sources */, D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */, + D073CE8B1DCBF3BB007511FD /* MessageHistoryUnsentTable.swift in Sources */, D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, D0B418241D7DFE0C004562A4 /* SimpleSet.swift in Sources */, - D03C53601DAD5C9C004C17B3 /* ChatListHole.swift in Sources */, + D073CE7B1DCBF3B4007511FD /* PeerNameIndexRepresentation.swift in Sources */, + D073CE7D1DCBF3B4007511FD /* PeerPresence.swift in Sources */, + D073CE9B1DCBF3BB007511FD /* RatingTable.swift in Sources */, D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, - D0B843EC1DAB91A0005F29E1 /* GlobalMessageIdsTable.swift in Sources */, + D073CE9A1DCBF3BB007511FD /* OrderStatisticTable.swift in Sources */, + D073CE751DCBF3B4007511FD /* Message.swift in Sources */, + D073CE901DCBF3BB007511FD /* MetadataTable.swift in Sources */, + D073CE941DCBF3BB007511FD /* CachedPeerDataTable.swift in Sources */, D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */, - D03C53611DAD5C9C004C17B3 /* PeerReadState.swift in Sources */, - D0B843F61DAB91A0005F29E1 /* MediaCleanupTable.swift in Sources */, D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */, D0B418581D7DFE29004562A4 /* SeedConfiguration.swift in Sources */, D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */, - D03C535F1DAD5C9C004C17B3 /* MessageHistoryHole.swift in Sources */, D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */, - D0B843F21DAB91A0005F29E1 /* MessageHistoryUnsentTable.swift in Sources */, - D0B843FC1DAB91A0005F29E1 /* PeerPresenceTable.swift in Sources */, + D073CE931DCBF3BB007511FD /* PeerNotificationSettingsTable.swift in Sources */, D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */, D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, - D03C53641DAD5C9C004C17B3 /* PeerPresence.swift in Sources */, - D0B843F71DAB91A0005F29E1 /* MetadataTable.swift in Sources */, + D073CE8C1DCBF3BB007511FD /* ChatListTable.swift in Sources */, + D073CE921DCBF3BB007511FD /* PeerTable.swift in Sources */, + D073CE741DCBF3B4007511FD /* Peer.swift in Sources */, + D073CE7A1DCBF3B4007511FD /* PeerReadState.swift in Sources */, + D073CE801DCBF3B4007511FD /* PeerChatInterfaceState.swift in Sources */, D0B844031DAB91A7005F29E1 /* PeerView.swift in Sources */, D0B418301D7DFE16004562A4 /* ValueBoxKey.swift in Sources */, + D073CE8F1DCBF3BB007511FD /* MediaCleanupTable.swift in Sources */, D0B418221D7DFE0C004562A4 /* Coding.swift in Sources */, - D0B844011DAB91A0005F29E1 /* OrderStatisticTable.swift in Sources */, - D0B843FF1DAB91A0005F29E1 /* MessageHistoryReadStateTable.swift in Sources */, - D03C53621DAD5C9C004C17B3 /* PeerNameIndexRepresentation.swift in Sources */, D0B4184B1D7DFE20004562A4 /* SynchronizePeerReadStatesView.swift in Sources */, - D0B843F81DAB91A0005F29E1 /* KeychainTable.swift in Sources */, + D073CE991DCBF3BB007511FD /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, + D073CE981DCBF3BB007511FD /* MessageHistoryReadStateTable.swift in Sources */, + D073CE8D1DCBF3BB007511FD /* ChatListIndexTable.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index 19915e1233..01a99a49bf 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -1,7 +1,7 @@ import Foundation enum ChatListOperation { - case InsertMessage(IntermediateMessage, CombinedPeerReadState?) + case InsertMessage(MessageIndex, IntermediateMessage, CombinedPeerReadState?, PeerChatListEmbeddedInterfaceState?) case InsertHole(ChatListHole) case InsertNothing(MessageIndex) case RemoveMessage([MessageIndex]) @@ -9,14 +9,14 @@ enum ChatListOperation { } enum ChatListIntermediateEntry { - case Message(IntermediateMessage) + case Message(MessageIndex, IntermediateMessage, PeerChatListEmbeddedInterfaceState?) case Hole(ChatListHole) case Nothing(MessageIndex) var index: MessageIndex { switch self { - case let .Message(message): - return MessageIndex(id: message.id, timestamp: message.timestamp) + case let .Message(index, _, _): + return index case let .Hole(hole): return hole.index case let .Nothing(index): @@ -75,23 +75,38 @@ final class ChatListTable: Table { } } - func replay(_ historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]], messageHistoryTable: MessageHistoryTable, operations: inout [ChatListOperation]) { + func replay(historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]], updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, operations: inout [ChatListOperation]) { self.ensureInitialized() - - for (peerId, _) in historyOperationsByPeerId { + var changedPeerIds = Set() + for peerId in historyOperationsByPeerId.keys { + changedPeerIds.insert(peerId) + } + for peerId in updatedPeerChatListEmbeddedStates.keys { + changedPeerIds.insert(peerId) + } + for peerId in changedPeerIds { let currentIndex: MessageIndex? = self.indexTable.get(peerId) - if let topMessage = messageHistoryTable.topMessage(peerId) { - let index = MessageIndex(id: topMessage.id, timestamp: topMessage.timestamp) + + let updatedIndex: MessageIndex? + let topMessage = messageHistoryTable.topMessage(peerId) + let embeddedChatState = peerChatInterfaceStateTable.get(peerId)?.chatListEmbeddedState + + if let topMessage = topMessage { + var updatedTimestamp = topMessage.timestamp + if let embeddedChatState = embeddedChatState { + updatedTimestamp = max(updatedTimestamp, embeddedChatState.timestamp) + } + var updatedIndex = MessageIndex(id: topMessage.id, timestamp: updatedTimestamp) - if let currentIndex = currentIndex , currentIndex != index { + if let currentIndex = currentIndex, currentIndex != updatedIndex { self.justRemoveMessage(currentIndex) } if let currentIndex = currentIndex { operations.append(.RemoveMessage([currentIndex])) } - self.indexTable.set(index) - self.justInsertMessage(index) - operations.append(.InsertMessage(topMessage, messageHistoryTable.readStateTable.getCombinedState(peerId))) + self.indexTable.set(updatedIndex) + self.justInsertMessage(updatedIndex) + operations.append(.InsertMessage(updatedIndex, topMessage, messageHistoryTable.readStateTable.getCombinedState(peerId), embeddedChatState)) } else { if let currentIndex = currentIndex { operations.append(.RemoveMessage([currentIndex])) @@ -144,7 +159,7 @@ final class ChatListTable: Table { self.valueBox.remove(self.tableId, key: self.key(index, type: .Hole)) } - func entriesAround(_ index: MessageIndex, messageHistoryTable: MessageHistoryTable, count: Int) -> (entries: [ChatListIntermediateEntry], lower: ChatListIntermediateEntry?, upper: ChatListIntermediateEntry?) { + func entriesAround(_ index: MessageIndex, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> (entries: [ChatListIntermediateEntry], lower: ChatListIntermediateEntry?, upper: ChatListIntermediateEntry?) { self.ensureInitialized() var lowerEntries: [ChatListIntermediateEntry] = [] @@ -157,7 +172,7 @@ final class ChatListTable: Table { let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { if let message = messageHistoryTable.getMessage(index) { - lowerEntries.append(.Message(message)) + lowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) } else { lowerEntries.append(.Nothing(index)) } @@ -176,7 +191,7 @@ final class ChatListTable: Table { let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { if let message = messageHistoryTable.getMessage(index) { - upperEntries.append(.Message(message)) + upperEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) } else { upperEntries.append(.Nothing(index)) } @@ -204,7 +219,7 @@ final class ChatListTable: Table { let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { if let message = messageHistoryTable.getMessage(index) { - additionalLowerEntries.append(.Message(message)) + additionalLowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) } else { additionalLowerEntries.append(.Nothing(index)) } @@ -226,7 +241,7 @@ final class ChatListTable: Table { return (entries: entries, lower: lower, upper: upper) } - func earlierEntries(_ index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + func earlierEntries(_ index: MessageIndex?, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> [ChatListIntermediateEntry] { self.ensureInitialized() var entries: [ChatListIntermediateEntry] = [] @@ -242,7 +257,7 @@ final class ChatListTable: Table { let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { if let message = messageHistoryTable.getMessage(index) { - entries.append(.Message(message)) + entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) } else { entries.append(.Nothing(index)) } @@ -254,7 +269,7 @@ final class ChatListTable: Table { return entries } - func laterEntries(_ index: MessageIndex?, messageHistoryTable: MessageHistoryTable, count: Int) -> [ChatListIntermediateEntry] { + func laterEntries(_ index: MessageIndex?, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> [ChatListIntermediateEntry] { self.ensureInitialized() var entries: [ChatListIntermediateEntry] = [] @@ -270,7 +285,7 @@ final class ChatListTable: Table { let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { if let message = messageHistoryTable.getMessage(index) { - entries.append(.Message(message)) + entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) } else { entries.append(.Nothing(index)) } @@ -282,7 +297,7 @@ final class ChatListTable: Table { return entries } - func debugList(_ messageHistoryTable: MessageHistoryTable) -> [ChatListIntermediateEntry] { - return self.laterEntries(MessageIndex.absoluteLowerBound(), messageHistoryTable: messageHistoryTable, count: 1000) + func debugList(_ messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable) -> [ChatListIntermediateEntry] { + return self.laterEntries(MessageIndex.absoluteLowerBound(), messageHistoryTable: messageHistoryTable, peerChatInterfaceStateTable: peerChatInterfaceStateTable, count: 1000) } } diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index 2652332ead..6cfb85387d 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -1,14 +1,14 @@ import Foundation public enum ChatListEntry: Comparable { - case MessageEntry(Message, CombinedPeerReadState?, PeerNotificationSettings?) + case MessageEntry(MessageIndex, Message, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) case HoleEntry(ChatListHole) case Nothing(MessageIndex) public var index: MessageIndex { switch self { - case let .MessageEntry(message, _, _): - return MessageIndex(message) + case let .MessageEntry(index, _, _, _, _): + return index case let .HoleEntry(hole): return hole.index case let .Nothing(index): @@ -18,22 +18,47 @@ public enum ChatListEntry: Comparable { } public func ==(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { - if lhs.index == rhs.index { - if case let .MessageEntry(_, lhsReadState, lhsSettings) = lhs, case let .MessageEntry(_, rhsReadState, rhsSettings) = rhs { - if lhsReadState != rhsReadState { - return false - } - if let lhsSettings = lhsSettings, let rhsSettings = rhsSettings { - if !lhsSettings.isEqual(to: rhsSettings) { + switch lhs { + case let .MessageEntry(lhsIndex, lhsMessage, lhsReadState, lhsSettings, lhsEmbeddedState): + switch rhs { + case let .MessageEntry(rhsIndex, rhsMessage, rhsReadState, rhsSettings, rhsEmbeddedState): + if lhsIndex != rhsIndex { + return false + } + if lhsReadState != rhsReadState { + return false + } + if let lhsSettings = lhsSettings, let rhsSettings = rhsSettings { + if !lhsSettings.isEqual(to: rhsSettings) { + return false + } + } else if (lhsSettings != nil) != (rhsSettings != nil) { + return false + } + if let lhsEmbeddedState = lhsEmbeddedState, let rhsEmbeddedState = rhsEmbeddedState { + if !lhsEmbeddedState.isEqual(to: rhsEmbeddedState) { + return false + } + } else if (lhsEmbeddedState != nil) != (rhsEmbeddedState != nil) { + return false + } + return true + default: return false - } - } else if (lhsSettings != nil) != (rhsSettings != nil) { + } + case let .HoleEntry(hole): + if case .HoleEntry(hole) = rhs { + return true + } else { + return false + } + case let .Nothing(index): + if case .Nothing(index) = rhs { + return true + } else { return false } - } - return true } - return false } public func <(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { @@ -41,17 +66,17 @@ public func <(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { } enum MutableChatListEntry: Equatable { - case IntermediateMessageEntry(IntermediateMessage, CombinedPeerReadState?, PeerNotificationSettings?) - case MessageEntry(Message, CombinedPeerReadState?, PeerNotificationSettings?) + case IntermediateMessageEntry(MessageIndex, IntermediateMessage, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) + case MessageEntry(MessageIndex, Message, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) case HoleEntry(ChatListHole) case Nothing(MessageIndex) var index: MessageIndex { switch self { - case let .IntermediateMessageEntry(message, _, _): - return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .MessageEntry(message, _, _): - return MessageIndex(message) + case let .IntermediateMessageEntry(index, _, _, _, _): + return index + case let .MessageEntry(index, _, _, _, _): + return index case let .HoleEntry(hole): return hole.index case let .Nothing(index): @@ -142,8 +167,8 @@ final class MutableChatListView { var hasChanges = false for operation in operations { switch operation { - case let .InsertMessage(message, combinedReadState): - if self.add(.IntermediateMessageEntry(message, combinedReadState, nil)) { + case let .InsertMessage(index, message, combinedReadState, embeddedState): + if self.add(.IntermediateMessageEntry(index, message, combinedReadState, nil, embeddedState)) { hasChanges = true } case let .InsertNothing(index): @@ -164,20 +189,22 @@ final class MutableChatListView { } } } - for i in 0 ..< self.entries.count { - switch self.entries[i] { - case let .IntermediateMessageEntry(message, readState, _): - if let settings = updatedPeerNotificationSettings[message.id.peerId] { - self.entries[i] = .IntermediateMessageEntry(message, readState, settings) - hasChanges = true - } - case let .MessageEntry(message, readState, _): - if let settings = updatedPeerNotificationSettings[message.id.peerId] { - self.entries[i] = .MessageEntry(message, readState, settings) - hasChanges = true - } - default: - continue + if !updatedPeerNotificationSettings.isEmpty { + for i in 0 ..< self.entries.count { + switch self.entries[i] { + case let .IntermediateMessageEntry(index, message, readState, _, embeddedState): + if let settings = updatedPeerNotificationSettings[message.id.peerId] { + self.entries[i] = .IntermediateMessageEntry(index, message, readState, settings, embeddedState) + hasChanges = true + } + case let .MessageEntry(index, message, readState, _, embeddedState): + if let settings = updatedPeerNotificationSettings[message.id.peerId] { + self.entries[i] = .MessageEntry(index, message, readState, settings, embeddedState) + hasChanges = true + } + default: + continue + } } } return hasChanges @@ -418,14 +445,14 @@ final class MutableChatListView { func render(_ renderMessage: (IntermediateMessage) -> Message, getPeerNotificationSettings: (PeerId) -> PeerNotificationSettings?) { for i in 0 ..< self.entries.count { switch self.entries[i] { - case let .IntermediateMessageEntry(message, combinedReadState, notificationSettings): + case let .IntermediateMessageEntry(index, message, combinedReadState, notificationSettings, embeddedState): let updatedNotificationSettings: PeerNotificationSettings? if let notificationSettings = notificationSettings { updatedNotificationSettings = notificationSettings } else { updatedNotificationSettings = getPeerNotificationSettings(message.id.peerId) } - self.entries[i] = .MessageEntry(renderMessage(message), combinedReadState, updatedNotificationSettings) + self.entries[i] = .MessageEntry(index, renderMessage(message), combinedReadState, updatedNotificationSettings, embeddedState) default: break } @@ -442,8 +469,8 @@ public final class ChatListView { var entries: [ChatListEntry] = [] for entry in mutableView.entries { switch entry { - case let .MessageEntry(message, combinedReadState, notificationSettings): - entries.append(.MessageEntry(message, combinedReadState, notificationSettings)) + case let .MessageEntry(index, message, combinedReadState, notificationSettings, embeddedState): + entries.append(.MessageEntry(index, message, combinedReadState, notificationSettings, embeddedState)) case let .Nothing(index): entries.append(.Nothing(index)) case let .HoleEntry(hole): diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 012b87dfe1..66632dfe5e 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -419,6 +419,16 @@ final class MessageHistoryTable: Table { if case let .Message(message) = entry { return message } + } else if let tableIndex = self.messageHistoryIndexTable.get(index.id) { + if case let .Message(updatedIndex) = tableIndex { + let key = self.key(updatedIndex) + if let value = self.valueBox.get(self.tableId, key: key) { + let entry = self.readIntermediateEntry(key, value: value) + if case let .Message(message) = entry { + return message + } + } + } } return nil } diff --git a/Postbox/PeerChatInterfaceStateTable.swift b/Postbox/PeerChatInterfaceStateTable.swift index 2c2dbe1937..e92e1288a7 100644 --- a/Postbox/PeerChatInterfaceStateTable.swift +++ b/Postbox/PeerChatInterfaceStateTable.swift @@ -23,20 +23,30 @@ final class PeerChatInterfaceStateTable: Table { } } - func set(_ peerId: PeerId, state: PeerChatInterfaceState?) { + func set(_ peerId: PeerId, state: PeerChatInterfaceState?) -> (updated: Bool, updatedEmbeddedState: Bool) { let currentState = self.get(peerId) var updated = false + var updatedEmbeddedState = false if let currentState = currentState, let state = state { if !currentState.isEqual(to: state) { updated = true + if let currentEmbeddedState = currentState.chatListEmbeddedState, let embeddedState = state.chatListEmbeddedState { + if !currentEmbeddedState.isEqual(to: embeddedState) { + updatedEmbeddedState = true + } + } else if (currentState.chatListEmbeddedState != nil) != (state.chatListEmbeddedState != nil) { + updatedEmbeddedState = true + } } } else if (currentState != nil) != (state != nil) { updated = true + updatedEmbeddedState = true } if updated { self.states[peerId] = state self.peerIdsWithUpdatedStates.insert(peerId) } + return (updated, updatedEmbeddedState) } override func clearMemoryCache() { diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index d0598c9e9d..333a096336 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -211,6 +211,7 @@ public final class Postbox { private var currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings] = [:] private var currentUpdatedCachedPeerData: [PeerId: CachedPeerData] = [:] private var currentUpdatedPeerPresences: [PeerId: PeerPresence] = [:] + private var currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -659,15 +660,15 @@ public final class Postbox { } private func fetchAroundChatEntries(_ index: MessageIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?) { - let (intermediateEntries, intermediateLower, intermediateUpper) = self.chatListTable.entriesAround(index, messageHistoryTable: self.messageHistoryTable, count: count) + let (intermediateEntries, intermediateLower, intermediateUpper) = self.chatListTable.entriesAround(index, messageHistoryTable: self.messageHistoryTable, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable, count: count) var entries: [MutableChatListEntry] = [] var lower: MutableChatListEntry? var upper: MutableChatListEntry? for entry in intermediateEntries { switch entry { - case let .Message(message): - entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId))) + case let .Message(index, message, embeddedState): + entries.append(.IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState)) case let .Hole(hole): entries.append(.HoleEntry(hole)) case let .Nothing(index): @@ -677,8 +678,8 @@ public final class Postbox { if let intermediateLower = intermediateLower { switch intermediateLower { - case let .Message(message): - lower = .IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId)) + case let .Message(index, message, embeddedState): + lower = .IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState) case let .Hole(hole): lower = .HoleEntry(hole) case let .Nothing(index): @@ -688,8 +689,8 @@ public final class Postbox { if let intermediateUpper = intermediateUpper { switch intermediateUpper { - case let .Message(message): - upper = .IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId)) + case let .Message(index, message, embeddedState): + upper = .IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState) case let .Hole(hole): upper = .HoleEntry(hole) case let .Nothing(index): @@ -701,12 +702,12 @@ public final class Postbox { } private func fetchEarlierChatEntries(_ index: MessageIndex?, count: Int) -> [MutableChatListEntry] { - let intermediateEntries = self.chatListTable.earlierEntries(index, messageHistoryTable: self.messageHistoryTable, count: count) + let intermediateEntries = self.chatListTable.earlierEntries(index, messageHistoryTable: self.messageHistoryTable, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable, count: count) var entries: [MutableChatListEntry] = [] for entry in intermediateEntries { switch entry { - case let .Message(message): - entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId))) + case let .Message(index, message, embeddedState): + entries.append(.IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState)) case let .Hole(hole): entries.append(.HoleEntry(hole)) case let .Nothing(index): @@ -717,12 +718,12 @@ public final class Postbox { } private func fetchLaterChatEntries(_ index: MessageIndex?, count: Int) -> [MutableChatListEntry] { - let intermediateEntries = self.chatListTable.laterEntries(index, messageHistoryTable: self.messageHistoryTable, count: count) + let intermediateEntries = self.chatListTable.laterEntries(index, messageHistoryTable: self.messageHistoryTable, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable, count: count) var entries: [MutableChatListEntry] = [] for entry in intermediateEntries { switch entry { - case let .Message(message): - entries.append(.IntermediateMessageEntry(message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId))) + case let .Message(index, message, embeddedState): + entries.append(.IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState)) case let .Nothing(index): entries.append(.Nothing(index)) case let .Hole(index): @@ -759,12 +760,12 @@ public final class Postbox { private func beforeCommit() -> (updatedTransactionStateVersion: Int64?, updatedMasterClientId: Int64?) { var chatListOperations: [ChatListOperation] = [] - self.chatListTable.replay(self.currentOperationsByPeerId, messageHistoryTable: self.messageHistoryTable, operations: &chatListOperations) + self.chatListTable.replay(historyOperationsByPeerId: self.currentOperationsByPeerId, updatedPeerChatListEmbeddedStates: currentUpdatedPeerChatListEmbeddedStates, messageHistoryTable: self.messageHistoryTable, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable, operations: &chatListOperations) for (index, hole) in self.currentReplaceChatListHoles { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } - let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -791,6 +792,7 @@ public final class Postbox { self.currentUpdatedPeerNotificationSettings.removeAll() self.currentUpdatedCachedPeerData.removeAll() self.currentUpdatedPeerPresences.removeAll() + self.currentUpdatedPeerChatListEmbeddedStates.removeAll() for table in self.tables { table.beforeCommit() @@ -854,7 +856,10 @@ public final class Postbox { fileprivate func updatePeerChatInterfaceState(_ id: PeerId, update: (PeerChatInterfaceState?) -> (PeerChatInterfaceState?)) { let updatedState = update(self.peerChatInterfaceStateTable.get(id)) - self.peerChatInterfaceStateTable.set(id, state: updatedState) + let (_, updatedEmbeddedState) = self.peerChatInterfaceStateTable.set(id, state: updatedState) + if updatedEmbeddedState { + self.currentUpdatedPeerChatListEmbeddedStates[id] = updatedState?.chatListEmbeddedState + } } fileprivate func replaceContactPeerIds(_ peerIds: Set) { diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index 7cddf9ee27..58ad00f548 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -9,6 +9,7 @@ final class PostboxTransaction { let currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings] let currentUpdatedCachedPeerData: [PeerId: CachedPeerData] let currentUpdatedPeerPresences: [PeerId: PeerPresence] + let currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -41,6 +42,9 @@ final class PostboxTransaction { if !currentUpdatedPeerPresences.isEmpty { return false } + if !currentUpdatedPeerChatListEmbeddedStates.isEmpty { + return false + } if !unsentMessageOperations.isEmpty { return false } @@ -59,7 +63,7 @@ final class PostboxTransaction { return true } - init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles self.removedHolesByPeerId = removedHolesByPeerId @@ -68,6 +72,7 @@ final class PostboxTransaction { self.currentUpdatedPeerNotificationSettings = currentUpdatedPeerNotificationSettings; self.currentUpdatedCachedPeerData = currentUpdatedCachedPeerData self.currentUpdatedPeerPresences = currentUpdatedPeerPresences + self.currentUpdatedPeerChatListEmbeddedStates = currentUpdatedPeerChatListEmbeddedStates self.unsentMessageOperations = unsentMessageOperations self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations self.updatedMedia = updatedMedia From 2c5dd8804bb7df7a03d68b53ef3241c8f2cbd529 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 4 Nov 2016 12:59:00 +0300 Subject: [PATCH 055/427] no message --- Postbox/Coding.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 67eec67e5f..78dc390e3c 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -91,6 +91,14 @@ public class MemoryBuffer: Equatable, CustomStringConvertible { return hexString as String } + + public func makeData() -> Data { + if self.length == 0 { + return Data() + } else { + return Data(bytes: self.memory, count: self.length) + } + } } public func ==(lhs: MemoryBuffer, rhs: MemoryBuffer) -> Bool { @@ -116,7 +124,7 @@ public final class WriteBuffer: MemoryBuffer { return ReadBuffer(memory: self.memory, length: self.offset, freeWhenDone: false) } - public func makeData() -> Data { + override public func makeData() -> Data { return Data(bytes: self.memory.assumingMemoryBound(to: UInt8.self), count: self.offset) } From dbf3430581a770354543005c6ed92948e28e75c1 Mon Sep 17 00:00:00 2001 From: overtake Date: Fri, 4 Nov 2016 17:20:27 +0300 Subject: [PATCH 056/427] no message --- Postbox.xcodeproj/project.pbxproj | 6 ++++++ Postbox/Message.swift | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index a96dafcd99..f05adcf061 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -11,6 +11,9 @@ C26A71931DC9E97700F69385 /* ItemCollectionInfoTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */; }; C26A71941DC9E97E00F69385 /* ItemCollectionsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */; }; C26A71951DC9E98900F69385 /* ItemCollectionItemTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */; }; + C291942B1DCBDECA00359491 /* PeerChatInterfaceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */; }; + C291942C1DCBDECA00359491 /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; + C291942D1DCBDEEC00359491 /* InitialMessageHistoryData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */; }; D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */; }; @@ -714,6 +717,9 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + C291942D1DCBDEEC00359491 /* InitialMessageHistoryData.swift in Sources */, + C291942B1DCBDECA00359491 /* PeerChatInterfaceState.swift in Sources */, + C291942C1DCBDECA00359491 /* PeerChatInterfaceStateTable.swift in Sources */, C26A71951DC9E98900F69385 /* ItemCollectionItemTable.swift in Sources */, C26A71941DC9E97E00F69385 /* ItemCollectionsView.swift in Sources */, C26A71931DC9E97700F69385 /* ItemCollectionInfoTable.swift in Sources */, diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 923c712c9b..c7ce7ea976 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -69,7 +69,7 @@ public struct MessageId: Hashable, Comparable, CustomStringConvertible { var i = 0 var array: [MessageId] = [] while i < Int(length) { - array[i] = MessageId(buffer) + array.append(MessageId(buffer)) i += 1 } return array From 93827a3dda769bbc2b782f58e87a43f5a46ca9c5 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 15 Nov 2016 19:01:52 +0300 Subject: [PATCH 057/427] no message --- Postbox.xcodeproj/project.pbxproj | 12 +++- .../xcschemes/PostboxMac.xcscheme | 2 +- .../xcschemes/PostboxTests.xcscheme | 2 +- Postbox/Coding.swift | 13 +++++ Postbox/IntermediateMessage.swift | 4 +- Postbox/Message.swift | 14 ++++- Postbox/MessageHistoryTable.swift | 32 ++++++---- Postbox/MessageHistoryView.swift | 58 +++++++++++++++---- Postbox/PeerChatTopTaggedMessageIds.swift | 15 +++++ Postbox/Postbox.swift | 2 +- Postbox/SimpleDictionary.swift | 36 +++++++++++- Postbox/ViewTracker.swift | 2 +- PostboxTests/ChatListTableTests.swift | 22 ++++--- .../MessageHistoryIndexTableTests.swift | 4 +- PostboxTests/MessageHistoryTableTests.swift | 28 +++++---- PostboxTests/OrderStatisticTreeTests.swift | 4 +- PostboxTests/ReadStateTableTests.swift | 4 +- 17 files changed, 200 insertions(+), 54 deletions(-) create mode 100644 Postbox/PeerChatTopTaggedMessageIds.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 8c3ebdccf2..1d86e8a9ee 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -169,6 +169,8 @@ D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; + D0F7AB321DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; + D0F7AB331DCFAB1C009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */; }; D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; @@ -282,6 +284,7 @@ D0E3A7871B28AE9C00A402D9 /* Coding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Coding.swift; sourceTree = ""; }; D0E3A79D1B28B50400A402D9 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = ""; }; D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = ""; }; + D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatTopTaggedMessageIds.swift; sourceTree = ""; }; D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageMediaTable.swift; sourceTree = ""; }; D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTableTests.swift; sourceTree = ""; }; D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaCleanupTable.swift; sourceTree = ""; }; @@ -411,6 +414,7 @@ D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */, D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */, D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */, + D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */, ); name = Tables; sourceTree = ""; @@ -781,6 +785,7 @@ D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, D073CE8C1DCBF3BB007511FD /* ChatListTable.swift in Sources */, D073CE921DCBF3BB007511FD /* PeerTable.swift in Sources */, + D0F7AB331DCFAB1C009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */, D073CE741DCBF3B4007511FD /* Peer.swift in Sources */, D073CE7A1DCBF3B4007511FD /* PeerReadState.swift in Sources */, D073CE801DCBF3B4007511FD /* PeerChatInterfaceState.swift in Sources */, @@ -863,6 +868,7 @@ D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */, D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */, D03120FC1DA55427006A2A60 /* PeerNotificationSettings.swift in Sources */, + D0F7AB321DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */, D03BCCF81C73561C0097A291 /* Table.swift in Sources */, D021E0DC1DB5237C00C6B04F /* ItemCollectionsView.swift in Sources */, D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, @@ -973,7 +979,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; - SWIFT_VERSION = 3.0; + SWIFT_VERSION = 3.0.1; }; name = Hockeyapp; }; @@ -1223,7 +1229,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; - SWIFT_VERSION = 3.0; + SWIFT_VERSION = 3.0.1; }; name = Debug; }; @@ -1254,7 +1260,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; - SWIFT_VERSION = 3.0; + SWIFT_VERSION = 3.0.1; }; name = Release; }; diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme index f4a9378c19..f05213e2b6 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme @@ -1,6 +1,6 @@ ReadBuffer? { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bytes) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.offset, 4) + self.offset += 4 + Int(length) + let copyBytes = malloc(Int(length))! + memcpy(copyBytes, self.buffer.memory.advanced(by: self.offset - Int(length)), Int(length)) + return ReadBuffer(memory: copyBytes, length: Int(length), freeWhenDone: true) + } else { + return nil + } + } } diff --git a/Postbox/IntermediateMessage.swift b/Postbox/IntermediateMessage.swift index de95c68f27..d3b02068ce 100644 --- a/Postbox/IntermediateMessage.swift +++ b/Postbox/IntermediateMessage.swift @@ -23,6 +23,7 @@ struct IntermediateMessageForwardInfo { class IntermediateMessage { let stableId: UInt32 + let stableVersion: UInt32 let id: MessageId let timestamp: Int32 let flags: MessageFlags @@ -34,8 +35,9 @@ class IntermediateMessage { let embeddedMediaData: ReadBuffer let referencedMedia: [MediaId] - init(stableId: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { + init(stableId: UInt32, stableVersion: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { self.stableId = stableId + self.stableVersion = stableVersion self.id = id self.timestamp = timestamp self.flags = flags diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 923c712c9b..3b03f86136 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -69,7 +69,7 @@ public struct MessageId: Hashable, Comparable, CustomStringConvertible { var i = 0 var array: [MessageId] = [] while i < Int(length) { - array[i] = MessageId(buffer) + array.append(MessageId(buffer)) i += 1 } return array @@ -201,12 +201,17 @@ public struct MessageFlags: OptionSet { rawValue |= MessageFlags.Incoming.rawValue } + if flags.contains(StoreMessageFlags.Personal) { + rawValue |= MessageFlags.Personal.rawValue + } + self.rawValue = rawValue } public static let Unsent = MessageFlags(rawValue: 1) public static let Failed = MessageFlags(rawValue: 2) public static let Incoming = MessageFlags(rawValue: 4) + public static let Personal = MessageFlags(rawValue: 8) } public struct StoreMessageForwardInfo { @@ -268,6 +273,7 @@ public extension MessageAttribute { public final class Message { public let stableId: UInt32 + public let stableVersion: UInt32 public let id: MessageId public let timestamp: Int32 public let flags: MessageFlags @@ -279,9 +285,11 @@ public final class Message { public let media: [Media] public let peers: SimpleDictionary public let associatedMessages: SimpleDictionary + public let associatedMessageIds: [MessageId] - public init(stableId: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: MessageForwardInfo?, author: Peer?, text: String, attributes: [MessageAttribute], media: [Media], peers: SimpleDictionary, associatedMessages: SimpleDictionary) { + public init(stableId: UInt32, stableVersion: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: MessageForwardInfo?, author: Peer?, text: String, attributes: [MessageAttribute], media: [Media], peers: SimpleDictionary, associatedMessages: SimpleDictionary, associatedMessageIds: [MessageId]) { self.stableId = stableId + self.stableVersion = stableVersion self.id = id self.timestamp = timestamp self.flags = flags @@ -293,6 +301,7 @@ public final class Message { self.media = media self.peers = peers self.associatedMessages = associatedMessages + self.associatedMessageIds = associatedMessageIds } } @@ -310,6 +319,7 @@ public struct StoreMessageFlags: OptionSet { public static let Unsent = StoreMessageFlags(rawValue: 1) public static let Failed = StoreMessageFlags(rawValue: 2) public static let Incoming = StoreMessageFlags(rawValue: 4) + public static let Personal = StoreMessageFlags(rawValue: 8) } public enum StoreMessageId { diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 66632dfe5e..7cb2fb5100 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -450,6 +450,9 @@ final class MessageHistoryTable: Table { var stableId: UInt32 = self.historyMetadataTable.getNextStableMessageIndexId() sharedBuffer.write(&stableId, offset: 0, length: 4) + var stableVersion: UInt32 = 0 + sharedBuffer.write(&stableVersion, offset: 0, length: 4) + var flags = MessageFlags(message.flags) sharedBuffer.write(&flags.rawValue, offset: 0, length: 4) @@ -562,7 +565,7 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) - return IntermediateMessage(stableId: stableId, id: message.id, timestamp: message.timestamp, flags: flags, tags: message.tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) + return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, timestamp: message.timestamp, flags: flags, tags: message.tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) } private func justInsertHole(_ hole: MessageHistoryHole, sharedBuffer: WriteBuffer = WriteBuffer()) { @@ -706,7 +709,7 @@ final class MessageHistoryTable: Table { updatedEmbeddedMediaBuffer.write(encodedBuffer.memory, offset: 0, length: encodedBuffer.length) } - self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: message.referencedMedia), sharedKey: self.key(index)) + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: message.referencedMedia), sharedKey: self.key(index)) let operation: MessageHistoryOperation = .UpdateEmbeddedMedia(index, updatedEmbeddedMediaBuffer.makeReadBufferAndReset()) if operationsByPeerId[index.id.peerId] == nil { @@ -799,10 +802,6 @@ final class MessageHistoryTable: Table { break } - if previousMessage.tags != message.tags { - assertionFailure() - } - sharedBuffer.reset() var type: Int8 = 0 @@ -811,6 +810,9 @@ final class MessageHistoryTable: Table { var stableId: UInt32 = previousMessage.stableId sharedBuffer.write(&stableId, offset: 0, length: 4) + var stableVersion: UInt32 = previousMessage.stableVersion + 1 + sharedBuffer.write(&stableVersion, offset: 0, length: 4) + var flags = MessageFlags(message.flags) sharedBuffer.write(&flags.rawValue, offset: 0, length: 4) @@ -923,7 +925,7 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) - return IntermediateMessage(stableId: stableId, id: message.id, timestamp: message.timestamp, flags: flags, tags: tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) + return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, timestamp: message.timestamp, flags: flags, tags: tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) } else { return nil } @@ -932,7 +934,7 @@ final class MessageHistoryTable: Table { private func justUpdateTimestamp(_ index: MessageIndex, timestamp: Int32) { if let previousMessage = self.getMessage(index) { self.valueBox.remove(self.tableId, key: self.key(index)) - var updatedMessage = IntermediateMessage(stableId: previousMessage.stableId, id: previousMessage.id, timestamp: timestamp, flags: previousMessage.flags, tags: previousMessage.tags, forwardInfo: previousMessage.forwardInfo, authorId: previousMessage.authorId, text: previousMessage.text, attributesData: previousMessage.attributesData, embeddedMediaData: previousMessage.embeddedMediaData, referencedMedia: previousMessage.referencedMedia) + var updatedMessage = IntermediateMessage(stableId: previousMessage.stableId, stableVersion: previousMessage.stableVersion + 1, id: previousMessage.id, timestamp: timestamp, flags: previousMessage.flags, tags: previousMessage.tags, forwardInfo: previousMessage.forwardInfo, authorId: previousMessage.authorId, text: previousMessage.text, attributesData: previousMessage.attributesData, embeddedMediaData: previousMessage.embeddedMediaData, referencedMedia: previousMessage.referencedMedia) self.storeIntermediateMessage(updatedMessage, sharedKey: self.key(index)) let tags = previousMessage.tags.rawValue @@ -985,7 +987,7 @@ final class MessageHistoryTable: Table { if let extractedMedia = extractedMedia { var updatedReferencedMedia = message.referencedMedia updatedReferencedMedia.append(extractedMedia.id!) - self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) return extractedMedia } @@ -1002,6 +1004,9 @@ final class MessageHistoryTable: Table { var stableId: UInt32 = message.stableId sharedBuffer.write(&stableId, offset: 0, length: 4) + var stableVersion: UInt32 = message.stableVersion + sharedBuffer.write(&stableVersion, offset: 0, length: 4) + var flagsValue: UInt32 = message.flags.rawValue sharedBuffer.write(&flagsValue, offset: 0, length: 4) @@ -1079,6 +1084,9 @@ final class MessageHistoryTable: Table { var stableId: UInt32 = 0 value.read(&stableId, offset: 0, length: 4) + var stableVersion: UInt32 = 0 + value.read(&stableVersion, offset: 0, length: 4) + var flagsValue: UInt32 = 0 value.read(&flagsValue, offset: 0, length: 4) let flags = MessageFlags(rawValue: flagsValue) @@ -1170,7 +1178,7 @@ final class MessageHistoryTable: Table { referencedMediaIds.append(MediaId(namespace: idNamespace, id: idId)) } - return .Message(IntermediateMessage(stableId: stableId, id: index.id, timestamp: index.timestamp, flags: flags, tags: tags, forwardInfo: forwardInfo, authorId: authorId, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)) + return .Message(IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: index.id, timestamp: index.timestamp, flags: flags, tags: tags, forwardInfo: forwardInfo, authorId: authorId, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)) } else { var stableId: UInt32 = 0 value.read(&stableId, offset: 0, length: 4) @@ -1252,6 +1260,7 @@ final class MessageHistoryTable: Table { } } + var associatedMessageIds: [MessageId] = [] var associatedMessages = SimpleDictionary() for attribute in parsedAttributes { for peerId in attribute.associatedPeerIds { @@ -1259,6 +1268,7 @@ final class MessageHistoryTable: Table { peers[peer.id] = peer } } + associatedMessageIds.append(contentsOf: attribute.associatedMessageIds) for messageId in attribute.associatedMessageIds { if let entry = self.messageHistoryIndexTable.get(messageId) { if case let .Message(index) = entry { @@ -1270,7 +1280,7 @@ final class MessageHistoryTable: Table { } } - return Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages) + return Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages, associatedMessageIds: associatedMessageIds) } func entriesAround(_ index: MessageIndex, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 631ab927e2..a1e9d9b744 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -50,10 +50,10 @@ enum MutableMessageHistoryEntry { func updatedTimestamp(_ timestamp: Int32) -> MutableMessageHistoryEntry { switch self { case let .IntermediateMessageEntry(message, location): - var updatedMessage = IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) + var updatedMessage = IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) return .IntermediateMessageEntry(updatedMessage, location) case let .MessageEntry(message, location): - var updatedMessage = Message(stableId: message.stableId, id: message.id, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages) + var updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) return .MessageEntry(updatedMessage, location) case let .HoleEntry(hole, location): var updatedHole = MessageHistoryHole(stableId: hole.stableId, maxIndex: MessageIndex(id: hole.maxIndex.id, timestamp: timestamp), min: hole.min, tags: hole.tags) @@ -208,16 +208,10 @@ final class MutableMessageHistoryView { } for _ in 0 ..< self.entries.count - 1 - maxClipIndex { - /*if case let .MessageEntry(message) = self.entries.last! { - print("remove last \(message.text)") - }*/ self.entries.removeLast() } for _ in 0 ..< minClipIndex { - /*if case let .MessageEntry(message) = self.entries.first! { - print("remove first \(message.text)") - }*/ self.entries.removeFirst() } @@ -258,7 +252,7 @@ final class MutableMessageHistoryView { return true } - func replay(_ operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], context: MutableMessageHistoryViewReplayContext) -> Bool { + func replay(_ operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], context: MutableMessageHistoryViewReplayContext, renderIntermediateMessage: (IntermediateMessage) -> Message) -> Bool { let tagMask = self.tagMask let unwrappedTagMask: UInt32 = tagMask?.rawValue ?? 0 @@ -287,7 +281,7 @@ final class MutableMessageHistoryView { case let .UpdateEmbeddedMedia(index, embeddedMediaData): for i in 0 ..< self.entries.count { if case let .IntermediateMessageEntry(message, _) = self.entries[i] , MessageIndex(message) == index { - self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), nil) + self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), nil) hasChanges = true break } @@ -328,7 +322,7 @@ final class MutableMessageHistoryView { messageMedia.append(media) } } - let updatedMessage = Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages) + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) self.entries[i] = .MessageEntry(updatedMessage, nil) hasChanges = true } @@ -359,6 +353,33 @@ final class MutableMessageHistoryView { } } + for operation in operations { + switch operation { + case let .InsertMessage(intermediateMessage): + for i in 0 ..< self.entries.count { + switch self.entries[i] { + case let .MessageEntry(message, location): + if message.associatedMessageIds.count != message.associatedMessages.count { + if message.associatedMessageIds.contains(intermediateMessage.id) && message.associatedMessages[intermediateMessage.id] == nil { + var updatedAssociatedMessages = message.associatedMessages + let renderedMessage = renderIntermediateMessage(intermediateMessage) + updatedAssociatedMessages[intermediateMessage.id] = renderedMessage + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) + self.entries[i] = .MessageEntry(updatedMessage, location) + hasChanges = true + } + } + break + default: + break + } + } + break + default: + break + } + } + return hasChanges } @@ -419,6 +440,11 @@ final class MutableMessageHistoryView { hasChanges = true } + var ids = Set() + for index in indices { + ids.insert(index.id) + } + if self.entries.count != 0 { var i = self.entries.count - 1 while i >= 0 { @@ -426,6 +452,16 @@ final class MutableMessageHistoryView { self.entries.remove(at: i) context.removedEntries = true hasChanges = true + } else { + switch self.entries[i] { + case let .MessageEntry(message, location): + if let updatedAssociatedMessages = message.associatedMessages.filteredOut(keysIn: ids) { + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) + self.entries[i] = .MessageEntry(updatedMessage, location) + } + default: + break + } } i -= 1 } diff --git a/Postbox/PeerChatTopTaggedMessageIds.swift b/Postbox/PeerChatTopTaggedMessageIds.swift new file mode 100644 index 0000000000..f7ac6d0dd5 --- /dev/null +++ b/Postbox/PeerChatTopTaggedMessageIds.swift @@ -0,0 +1,15 @@ +import Foundation + +final class PeerChatTopTaggedMessageIdsTable: Table { + private var cachedTopIds: [PeerId: [MessageId.Namespace: MessageId?]] = [:] + private var updatedPeerIds = Set() + + private let sharedKey = ValueBoxKey(length: 8 + 4 + 4) + + override func beforeCommit() { + for peerId in self.updatedPeerIds { + + } + self.updatedPeerIds.removeAll() + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 333a096336..7612ea2b8c 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -335,7 +335,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 14 + let currentUserVersion: Int32 = 15 if userVersion != currentUserVersion { self.valueBox.drop() diff --git a/Postbox/SimpleDictionary.swift b/Postbox/SimpleDictionary.swift index f9a1d24591..32ae469d43 100644 --- a/Postbox/SimpleDictionary.swift +++ b/Postbox/SimpleDictionary.swift @@ -1,11 +1,45 @@ import Foundation -public struct SimpleDictionary: Sequence { +public struct SimpleDictionary: Sequence { private var items: [(K, V)] = [] + public var count: Int { + return self.items.count + } + + public var isEmpty: Bool { + return self.items.isEmpty + } + public init() { } + private init(items: [(K, V)] = []) { + self.items = items + } + + public func filteredOut(keysIn: Set) -> SimpleDictionary? { + var hasUpdates = false + for (key, _) in self.items { + if keysIn.contains(key) { + hasUpdates = true + break + } + } + if hasUpdates { + var updatedItems: [(K, V)] = [] + for (key, value) in self.items { + if !keysIn.contains(key) { + updatedItems.append((key, value)) + break + } + } + return SimpleDictionary(items: updatedItems) + } else { + return nil + } + } + public subscript(key: K) -> V? { get { for (k, value) in self.items { diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 14b5b0c173..12684afbf2 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -253,7 +253,7 @@ final class ViewTracker { updateType = .Generic } - if mutableView.replay(operations ?? [], holeFillDirections: transaction.peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: transaction.updatedMedia, context: context) { + if mutableView.replay(operations ?? [], holeFillDirections: transaction.peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: transaction.updatedMedia, context: context, renderIntermediateMessage: self.renderMessage) { mutableView.complete(context: context, fetchEarlier: { index, count in return self.fetchEarlierHistoryEntries(peerId, index, count, mutableView.tagMask) }, fetchLater: { index, count in diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index 3178b94934..f0e6bd6c36 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -6,6 +6,8 @@ import XCTest import Postbox @testable import Postbox +import SwiftSignalKit + private let namespace: Int32 = 1 private let authorPeerId = PeerId(namespace: 2, id: 3) @@ -75,6 +77,7 @@ class ChatListTableTests: XCTestCase { var tagsTable: MessageHistoryTagsTable? var readStateTable: MessageHistoryReadStateTable? var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? + var peerChatInterfaceStateTable: PeerChatInterfaceStateTable? override class func setUp() { super.setUp() @@ -86,7 +89,7 @@ class ChatListTableTests: XCTestCase { var randomId: Int64 = 0 arc4random_buf(&randomId, 8) path = NSTemporaryDirectory() + "\(randomId)" - self.valueBox = SqliteValueBox(basePath: path!) + self.valueBox = SqliteValueBox(basePath: path!, queue: Queue.mainQueue()) let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: []) @@ -102,6 +105,7 @@ class ChatListTableTests: XCTestCase { self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, tableId: 5) self.chatListTable = ChatListTable(valueBox: self.valueBox!, tableId: 6, indexTable: self.chatListIndexTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) + self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox!, tableId: 20) } override func tearDown() { @@ -123,18 +127,20 @@ class ChatListTableTests: XCTestCase { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] - self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) + self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) } private func addHole(_ peerId: Int32, _ id: Int32) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] self.historyTable!.addHoles([MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] - self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) + self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) } private func addChatListHole(_ peerId: Int32, _ id: Int32, _ timestamp: Int32) { @@ -156,24 +162,26 @@ class ChatListTableTests: XCTestCase { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] self.historyTable!.removeMessages(ids.map({ MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] - self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) + self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) } private func fillHole(_ peerId: Int32, _ id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32, String, [Media])]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] self.historyTable!.fillHole(MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), fillType: fillType, tagMask: nil, messages: messages.map({ StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] - self.chatListTable!.replay(operationsByPeerId, messageHistoryTable: self.historyTable!, operations: &operations) + self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) } private func expectEntries(_ entries: [Entry]) { - let actualEntries = self.chatListTable!.debugList(self.historyTable!).map({ entry -> Entry in + let actualEntries = self.chatListTable!.debugList(self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!).map({ entry -> Entry in switch entry { - case let .Message(message): + case let .Message(_, message, _): if message.authorId != authorPeerId { XCTFail("Expected authorId \(authorPeerId), actual \(message.authorId)") } diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index 1875a1e6f9..c39f0cb771 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -6,6 +6,8 @@ import XCTest import Postbox @testable import Postbox +import SwiftSignalKit + private let peerId = PeerId(namespace: 1, id: 1) private let namespace: Int32 = 1 @@ -73,7 +75,7 @@ class MessageHistoryIndexTableTests: XCTestCase { var randomId: Int64 = 0 arc4random_buf(&randomId, 8) path = NSTemporaryDirectory() + "\(randomId)" - self.valueBox = SqliteValueBox(basePath: path!) + self.valueBox = SqliteValueBox(basePath: path!, queue: Queue.mainQueue()) let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: []) diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index 0dc11db243..7c6775adf7 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -6,6 +6,8 @@ import XCTest import Postbox @testable import Postbox +import SwiftSignalKit + private let peerId = PeerId(namespace: 1, id: 1) private let namespace: Int32 = 1 private let authorPeerId = PeerId(namespace: 1, id: 6) @@ -120,6 +122,10 @@ private class TestExternalMedia: Media { } private class TestPeer: Peer { + var indexName: PeerIndexNameRepresentation { + return .title("Test") + } + let id: PeerId let data: String @@ -160,7 +166,7 @@ private enum MediaEntry: Equatable { case let .Direct(media, referenceCount): self = .Direct(media, referenceCount) case let .MessageReference(index): - self = MessageReference(index.id.id) + self = .MessageReference(index.id.id) } } } @@ -219,7 +225,7 @@ class MessageHistoryTableTests: XCTestCase { var randomId: Int64 = 0 arc4random_buf(&randomId, 8) path = NSTemporaryDirectory() + "\(randomId)" - self.valueBox = SqliteValueBox(basePath: path!) + self.valueBox = SqliteValueBox(basePath: path!, queue: Queue.mainQueue()) let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) @@ -330,12 +336,12 @@ class MessageHistoryTableTests: XCTestCase { } } - private func expectUnsent(_ indices: [(Int32, Int32)]) { - let actualUnsent = self.unsentTable!.get().map({ ($0.id.id, $0.timestamp) }) + private func expectUnsent(_ indices: [Int32]) { + let actualUnsent = self.unsentTable!.get().map({ $0.id }) var match = true if actualUnsent.count == indices.count { for i in 0 ..< indices.count { - if indices[i].0 != actualUnsent[i].0 || indices[i].1 != actualUnsent[i].1 { + if indices[i] != actualUnsent[i] { match = false break } @@ -778,13 +784,13 @@ class MessageHistoryTableTests: XCTestCase { func testAddUnsent() { addMessage(100, 100, "m100", [], [.Unsent]) expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) - expectUnsent([(100, 100)]) + expectUnsent([100]) } func testRemoveUnsent() { addMessage(100, 100, "m100", [], [.Unsent]) expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) - expectUnsent([(100, 100)]) + expectUnsent([100]) removeMessages([100]) expectEntries([]) @@ -794,7 +800,7 @@ class MessageHistoryTableTests: XCTestCase { func testUpdateUnsentToSentSameIndex() { addMessage(100, 100, "m100", [], [.Unsent]) expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) - expectUnsent([(100, 100)]) + expectUnsent([100]) updateMessage(100, 100, 100, "m100", [], [], []) expectEntries([.Message(100, 100, "m100", [], [])]) @@ -804,7 +810,7 @@ class MessageHistoryTableTests: XCTestCase { func testUpdateUnsentToFailed() { addMessage(100, 100, "m100", [], [.Unsent]) expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) - expectUnsent([(100, 100)]) + expectUnsent([100]) updateMessage(100, 100, 100, "m100", [], [.Unsent, .Failed], []) expectEntries([.Message(100, 100, "m100", [], [.Unsent, .Failed])]) @@ -814,7 +820,7 @@ class MessageHistoryTableTests: XCTestCase { func testUpdateDifferentIndex() { addMessage(100, 100, "m100", [], [.Unsent]) expectEntries([.Message(100, 100, "m100", [], [.Unsent])]) - expectUnsent([(100, 100)]) + expectUnsent([100]) updateMessage(100, 200, 200, "m100", [], [], []) expectEntries([.Message(200, 200, "m100", [], [])]) @@ -826,7 +832,7 @@ class MessageHistoryTableTests: XCTestCase { addMessage(100, 100, "m100", [], [.Unsent]) expectEntries([.Hole(1, 99, 100), .Message(100, 100, "m100", [], [.Unsent]), .Hole(101, Int32.max, Int32.max)]) - expectUnsent([(100, 100)]) + expectUnsent([100]) updateMessage(100, 200, 200, "m100", [], [], []) expectEntries([.Hole(1, 199, 200), .Message(200, 200, "m100", [], []), .Hole(201, Int32.max, Int32.max)]) diff --git a/PostboxTests/OrderStatisticTreeTests.swift b/PostboxTests/OrderStatisticTreeTests.swift index c4779434db..2e17bf0b1e 100644 --- a/PostboxTests/OrderStatisticTreeTests.swift +++ b/PostboxTests/OrderStatisticTreeTests.swift @@ -6,6 +6,8 @@ import XCTest import Postbox @testable import Postbox +import SwiftSignalKit + private let peerId = PeerId(namespace: 1, id: 1) private let namespace: Int32 = 1 private let authorPeerId = PeerId(namespace: 1, id: 6) @@ -83,7 +85,7 @@ class OrderStatisticTreeTests: XCTestCase { var randomId: Int64 = 0 arc4random_buf(&randomId, 8) path = NSTemporaryDirectory() + "\(randomId)" - self.valueBox = SqliteValueBox(basePath: path!) + self.valueBox = SqliteValueBox(basePath: path!, queue: Queue.mainQueue()) let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) diff --git a/PostboxTests/ReadStateTableTests.swift b/PostboxTests/ReadStateTableTests.swift index eb51113df4..8005f60b71 100644 --- a/PostboxTests/ReadStateTableTests.swift +++ b/PostboxTests/ReadStateTableTests.swift @@ -6,6 +6,8 @@ import XCTest import Postbox @testable import Postbox +import SwiftSignalKit + private let peerId = PeerId(namespace: 1, id: 1) private let namespace: Int32 = 1 private let authorPeerId = PeerId(namespace: 1, id: 6) @@ -83,7 +85,7 @@ class ReadStateTableTests: XCTestCase { var randomId: Int64 = 0 arc4random_buf(&randomId, 8) path = NSTemporaryDirectory() + "\(randomId)" - self.valueBox = SqliteValueBox(basePath: path!) + self.valueBox = SqliteValueBox(basePath: path!, queue: Queue.mainQueue()) let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) From 45f81135c2b666fe6baf5ef17ba4d733bb5863bf Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 17 Nov 2016 22:16:50 +0300 Subject: [PATCH 058/427] no message --- Postbox.xcodeproj/project.pbxproj | 12 + Postbox/CachedPeerDataTable.swift | 12 +- Postbox/ChatListIndexTable.swift | 10 +- Postbox/ChatListTable.swift | 30 +- Postbox/ChatListView.swift | 3 + Postbox/ContactTable.swift | 12 +- Postbox/GlobalMessageIdsTable.swift | 18 +- Postbox/ItemCacheMetaTable.swift | 80 ++++ Postbox/ItemCacheTable.swift | 68 +++ Postbox/ItemCollectionInfoTable.swift | 16 +- Postbox/ItemCollectionItemTable.swift | 18 +- Postbox/KeychainTable.swift | 10 +- Postbox/MediaCleanupTable.swift | 11 +- Postbox/MediaResource.swift | 10 +- Postbox/MessageHistoryIndexTable.swift | 46 +- ...sageHistoryInvalidatedReadStateTable.swift | 14 +- Postbox/MessageHistoryMetadataTable.swift | 28 +- Postbox/MessageHistoryReadStateTable.swift | 14 +- Postbox/MessageHistoryTable.swift | 63 ++- Postbox/MessageHistoryTagsTable.swift | 28 +- Postbox/MessageHistoryUnsentTable.swift | 14 +- Postbox/MessageHistoryView.swift | 6 +- Postbox/MessageMediaTable.swift | 40 +- Postbox/MetadataTable.swift | 28 +- Postbox/OrderStatisticTable.swift | 12 +- Postbox/PeerChatInterfaceStateTable.swift | 10 +- Postbox/PeerChatStateTable.swift | 10 +- Postbox/PeerChatTopTaggedMessageIds.swift | 4 + Postbox/PeerNotificationSettingsTable.swift | 12 +- Postbox/PeerPresenceTable.swift | 12 +- Postbox/PeerTable.swift | 12 +- Postbox/Postbox.swift | 81 ++-- Postbox/RatingTable.swift | 12 +- Postbox/SqliteValueBox.swift | 408 +++++++++++++----- Postbox/Table.swift | 6 +- Postbox/ValueBox.swift | 24 +- PostboxTests/ChatListTableTests.swift | 27 +- .../MessageHistoryIndexTableTests.swift | 6 +- PostboxTests/MessageHistoryTableTests.swift | 46 +- PostboxTests/OrderStatisticTreeTests.swift | 21 +- PostboxTests/ReadStateTableTests.swift | 21 +- 41 files changed, 861 insertions(+), 454 deletions(-) create mode 100644 Postbox/ItemCacheMetaTable.swift create mode 100644 Postbox/ItemCacheTable.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 1d86e8a9ee..3e00341027 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -169,6 +169,10 @@ D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; + D0F3CC721DDE1CDC008148FA /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; + D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; + D0F3CC751DDE2845008148FA /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; + D0F3CC761DDE2845008148FA /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; D0F7AB321DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; D0F7AB331DCFAB1C009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; @@ -284,6 +288,8 @@ D0E3A7871B28AE9C00A402D9 /* Coding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Coding.swift; sourceTree = ""; }; D0E3A79D1B28B50400A402D9 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = ""; }; D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = ""; }; + D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheTable.swift; sourceTree = ""; }; + D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheMetaTable.swift; sourceTree = ""; }; D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatTopTaggedMessageIds.swift; sourceTree = ""; }; D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageMediaTable.swift; sourceTree = ""; }; D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTableTests.swift; sourceTree = ""; }; @@ -415,6 +421,8 @@ D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */, D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */, D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */, + D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */, + D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */, ); name = Tables; sourceTree = ""; @@ -748,6 +756,7 @@ D073CE861DCBF3BB007511FD /* MessageHistoryMetadataTable.swift in Sources */, D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */, D073CE781DCBF3B4007511FD /* MessageHistoryHole.swift in Sources */, + D0F3CC751DDE2845008148FA /* ItemCacheMetaTable.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, D073CE891DCBF3BB007511FD /* MessageHistoryOperation.swift in Sources */, D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */, @@ -796,6 +805,7 @@ D0B4184B1D7DFE20004562A4 /* SynchronizePeerReadStatesView.swift in Sources */, D073CE991DCBF3BB007511FD /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, D073CE981DCBF3BB007511FD /* MessageHistoryReadStateTable.swift in Sources */, + D0F3CC761DDE2845008148FA /* ItemCacheTable.swift in Sources */, D073CE8D1DCBF3BB007511FD /* ChatListIndexTable.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -805,6 +815,7 @@ buildActionMask = 2147483647; files = ( D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, + D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */, D08D451F1D5D2CA700A7428A /* RatingTable.swift in Sources */, D07CFF831DCA909100761F81 /* PeerChatInterfaceStateTable.swift in Sources */, D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */, @@ -858,6 +869,7 @@ D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */, + D0F3CC721DDE1CDC008148FA /* ItemCacheTable.swift in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, diff --git a/Postbox/CachedPeerDataTable.swift b/Postbox/CachedPeerDataTable.swift index d3db7014f8..2b5027f67d 100644 --- a/Postbox/CachedPeerDataTable.swift +++ b/Postbox/CachedPeerDataTable.swift @@ -1,14 +1,18 @@ import Foundation final class CachedPeerDataTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private let sharedEncoder = Encoder() private let sharedKey = ValueBoxKey(length: 8) private var cachedDatas: [PeerId: CachedPeerData] = [:] private var updatedPeerIds = Set() - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) + override init(valueBox: ValueBox, table: ValueBoxTable) { + super.init(valueBox: valueBox, table: table) } private func key(_ id: PeerId) -> ValueBoxKey { @@ -25,7 +29,7 @@ final class CachedPeerDataTable: Table { if let status = self.cachedDatas[id] { return status } - if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let value = self.valueBox.get(self.table, key: self.key(id)) { if let data = Decoder(buffer: value).decodeRootObject() as? CachedPeerData { self.cachedDatas[id] = data return data @@ -45,7 +49,7 @@ final class CachedPeerDataTable: Table { self.sharedEncoder.reset() self.sharedEncoder.encodeRootObject(data) - self.valueBox.set(self.tableId, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) } } diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift index 15a3539f46..c99cbfbeba 100644 --- a/Postbox/ChatListIndexTable.swift +++ b/Postbox/ChatListIndexTable.swift @@ -1,8 +1,8 @@ import Foundation final class ChatListIndexTable: Table { - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) } private func key(_ peerId: PeerId) -> ValueBoxKey { @@ -19,15 +19,15 @@ final class ChatListIndexTable: Table { writeBuffer.write(&idNamespace, offset: 0, length: 4) writeBuffer.write(&idId, offset: 0, length: 4) writeBuffer.write(&idTimestamp, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.key(index.id.peerId), value: writeBuffer.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(index.id.peerId), value: writeBuffer.readBufferNoCopy()) } func remove(_ peerId: PeerId) { - self.valueBox.remove(self.tableId, key: self.key(peerId)) + self.valueBox.remove(self.table, key: self.key(peerId)) } func get(_ peerId: PeerId) -> MessageIndex? { - if let value = self.valueBox.get(self.tableId, key: self.key(peerId)) { + if let value = self.valueBox.get(self.table, key: self.key(peerId)) { var idNamespace: Int32 = 0 var idId: Int32 = 0 var idTimestamp: Int32 = 0 diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index 01a99a49bf..be9d38bd7e 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -31,17 +31,21 @@ private enum ChatListEntryType: Int8 { } final class ChatListTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + let indexTable: ChatListIndexTable let emptyMemoryBuffer = MemoryBuffer() let metadataTable: MessageHistoryMetadataTable let seedConfiguration: SeedConfiguration - init(valueBox: ValueBox, tableId: Int32, indexTable: ChatListIndexTable, metadataTable: MessageHistoryMetadataTable, seedConfiguration: SeedConfiguration) { + init(valueBox: ValueBox, table: ValueBoxTable, indexTable: ChatListIndexTable, metadataTable: MessageHistoryMetadataTable, seedConfiguration: SeedConfiguration) { self.indexTable = indexTable self.metadataTable = metadataTable self.seedConfiguration = seedConfiguration - super.init(valueBox: valueBox, tableId: tableId) + super.init(valueBox: valueBox, table: table) } private func key(_ index: MessageIndex, type: ChatListEntryType) -> ValueBoxKey { @@ -119,7 +123,7 @@ final class ChatListTable: Table { func addHole(_ hole: ChatListHole, operations: inout [ChatListOperation]) { self.ensureInitialized() - if self.valueBox.get(self.tableId, key: self.key(hole.index, type: .Hole)) == nil { + if self.valueBox.get(self.table, key: self.key(hole.index, type: .Hole)) == nil { self.justInsertHole(hole) operations.append(.InsertHole(hole)) } @@ -128,7 +132,7 @@ final class ChatListTable: Table { func replaceHole(_ index: MessageIndex, hole: ChatListHole?, operations: inout [ChatListOperation]) { self.ensureInitialized() - if self.valueBox.get(self.tableId, key: self.key(index, type: .Hole)) != nil { + if self.valueBox.get(self.table, key: self.key(index, type: .Hole)) != nil { if let hole = hole { if hole.index != index { self.justRemoveHole(index) @@ -144,19 +148,19 @@ final class ChatListTable: Table { } private func justInsertMessage(_ index: MessageIndex) { - self.valueBox.set(self.tableId, key: self.key(index, type: .Message), value: self.emptyMemoryBuffer) + self.valueBox.set(self.table, key: self.key(index, type: .Message), value: self.emptyMemoryBuffer) } private func justRemoveMessage(_ index: MessageIndex) { - self.valueBox.remove(self.tableId, key: self.key(index, type: .Message)) + self.valueBox.remove(self.table, key: self.key(index, type: .Message)) } private func justInsertHole(_ hole: ChatListHole) { - self.valueBox.set(self.tableId, key: self.key(hole.index, type: .Hole), value: self.emptyMemoryBuffer) + self.valueBox.set(self.table, key: self.key(hole.index, type: .Hole), value: self.emptyMemoryBuffer) } private func justRemoveHole(_ index: MessageIndex) { - self.valueBox.remove(self.tableId, key: self.key(index, type: .Hole)) + self.valueBox.remove(self.table, key: self.key(index, type: .Hole)) } func entriesAround(_ index: MessageIndex, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> (entries: [ChatListIntermediateEntry], lower: ChatListIntermediateEntry?, upper: ChatListIntermediateEntry?) { @@ -167,7 +171,7 @@ final class ChatListTable: Table { var lower: ChatListIntermediateEntry? var upper: ChatListIntermediateEntry? - self.valueBox.range(self.tableId, start: self.key(index, type: .Message), end: self.lowerBound(), keys: { key in + self.valueBox.range(self.table, start: self.key(index, type: .Message), end: self.lowerBound(), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { @@ -186,7 +190,7 @@ final class ChatListTable: Table { lowerEntries.removeLast() } - self.valueBox.range(self.tableId, start: self.key(index, type: .Message).predecessor, end: self.upperBound(), keys: { key in + self.valueBox.range(self.table, start: self.key(index, type: .Message).predecessor, end: self.upperBound(), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { @@ -214,7 +218,7 @@ final class ChatListTable: Table { case .Hole: startEntryType = .Hole } - self.valueBox.range(self.tableId, start: self.key(lowerEntries.last!.index, type: startEntryType), end: self.lowerBound(), keys: { key in + self.valueBox.range(self.table, start: self.key(lowerEntries.last!.index, type: startEntryType), end: self.lowerBound(), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { @@ -252,7 +256,7 @@ final class ChatListTable: Table { key = self.upperBound() } - self.valueBox.range(self.tableId, start: key, end: self.lowerBound(), keys: { key in + self.valueBox.range(self.table, start: key, end: self.lowerBound(), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { @@ -280,7 +284,7 @@ final class ChatListTable: Table { key = self.lowerBound() } - self.valueBox.range(self.tableId, start: key, end: self.upperBound(), keys: { key in + self.valueBox.range(self.table, start: key, end: self.upperBound(), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) let type: Int8 = key.getInt8(4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index 6cfb85387d..b47cf6b3f4 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -28,6 +28,9 @@ public func ==(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { if lhsReadState != rhsReadState { return false } + if lhsMessage.stableVersion != rhsMessage.stableVersion { + return false + } if let lhsSettings = lhsSettings, let rhsSettings = rhsSettings { if !lhsSettings.isEqual(to: rhsSettings) { return false diff --git a/Postbox/ContactTable.swift b/Postbox/ContactTable.swift index 6ac7f1f901..0019e40e78 100644 --- a/Postbox/ContactTable.swift +++ b/Postbox/ContactTable.swift @@ -1,6 +1,10 @@ import Foundation final class ContactTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private var originalPeerIds: Set? private var peerIds: Set? @@ -25,7 +29,7 @@ final class ContactTable: Table { return peerIds.contains(peerId) } else { var peerIds = Set() - self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), keys: { key in + self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), keys: { key in peerIds.insert(PeerId(key.getInt64(0))) return true }, limit: 0) @@ -40,7 +44,7 @@ final class ContactTable: Table { return peerIds } else { var peerIds = Set() - self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), keys: { key in + self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), keys: { key in peerIds.insert(PeerId(key.getInt64(0))) return true }, limit: 0) @@ -78,11 +82,11 @@ final class ContactTable: Table { let sharedKey = self.key(PeerId(namespace: 0, id: 0)) for peerId in self.removedPeerIds { - self.valueBox.remove(self.tableId, key: self.key(peerId, sharedKey: sharedKey)) + self.valueBox.remove(self.table, key: self.key(peerId, sharedKey: sharedKey)) } for peerId in self.addedPeerIds { - self.valueBox.set(self.tableId, key: self.key(peerId, sharedKey: sharedKey), value: MemoryBuffer()) + self.valueBox.set(self.table, key: self.key(peerId, sharedKey: sharedKey), value: MemoryBuffer()) } self.originalPeerIds = self.peerIds diff --git a/Postbox/GlobalMessageIdsTable.swift b/Postbox/GlobalMessageIdsTable.swift index 27063a8471..f07248be1c 100644 --- a/Postbox/GlobalMessageIdsTable.swift +++ b/Postbox/GlobalMessageIdsTable.swift @@ -1,19 +1,23 @@ import Foundation final class GlobalMessageIdsTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + let namespace: Int32 - let sharedKey = ValueBoxKey(length: 4) + let sharedKey = ValueBoxKey(length: 8) let sharedBuffer = WriteBuffer() - init(valueBox: ValueBox, tableId: Int32, namespace: Int32) { + init(valueBox: ValueBox, table: ValueBoxTable, namespace: Int32) { self.namespace = namespace - super.init(valueBox: valueBox, tableId: tableId) + super.init(valueBox: valueBox, table: table) } private func key(_ id: Int32) -> ValueBoxKey { - self.sharedKey.setInt32(0, value: id) + self.sharedKey.setInt64(0, value: Int64(id)) return self.sharedKey } @@ -23,11 +27,11 @@ final class GlobalMessageIdsTable: Table { var idNamespace: Int32 = id.namespace self.sharedBuffer.write(&idPeerId, offset: 0, length: 8) self.sharedBuffer.write(&idNamespace, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.key(globalId), value: self.sharedBuffer) + self.valueBox.set(self.table, key: self.key(globalId), value: self.sharedBuffer) } func get(_ globalId: Int32) -> MessageId? { - if let value = self.valueBox.get(self.tableId, key: self.key(globalId)) { + if let value = self.valueBox.get(self.table, key: self.key(globalId)) { var idPeerId: Int64 = 0 var idNamespace: Int32 = 0 value.read(&idPeerId, offset: 0, length: 8) @@ -38,6 +42,6 @@ final class GlobalMessageIdsTable: Table { } func remove(_ globalId: Int32) { - self.valueBox.remove(self.tableId, key: self.key(globalId)) + self.valueBox.remove(self.table, key: self.key(globalId)) } } diff --git a/Postbox/ItemCacheMetaTable.swift b/Postbox/ItemCacheMetaTable.swift new file mode 100644 index 0000000000..f4d3c6573f --- /dev/null +++ b/Postbox/ItemCacheMetaTable.swift @@ -0,0 +1,80 @@ +import Foundation + +public typealias ItemCacheCollectionId = Int8 + +public struct ItemCacheCollectionSpec { + public let lowWaterItemCount: Int32 + public let highWaterItemCount: Int32 + + public init(lowWaterItemCount: Int32, highWaterItemCount: Int32) { + self.lowWaterItemCount = lowWaterItemCount + self.highWaterItemCount = highWaterItemCount + } +} + +struct ItemCacheCollectionState: Coding { + let nextAccessIndex: Int32 + + init(decoder: Decoder) { + self.nextAccessIndex = decoder.decodeInt32ForKey("i") + } + + func encode(_ encoder: Encoder) { + encoder.encodeInt32(self.nextAccessIndex, forKey: "i") + } +} + +final class ItemCacheMetaTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + + private let sharedKey = ValueBoxKey(length: 8) + + private var cachedCollectionStates: [ItemCacheCollectionId: ItemCacheCollectionState] = [:] + private var updatedCollectionStateIds = Set() + + private func key(_ id: ItemCacheCollectionId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: Int64(id)) + return self.sharedKey + } + + private func get(_ id: ItemCacheCollectionId) -> ItemCacheCollectionState? { + if let cached = self.cachedCollectionStates[id] { + return cached + } else { + if let value = self.valueBox.get(self.table, key: self.key(id)), let state = Decoder(buffer: value).decodeRootObject() as? ItemCacheCollectionState { + self.cachedCollectionStates[id] = state + return state + } else { + return nil + } + } + } + + private func set(_ id: ItemCacheCollectionId, state: ItemCacheCollectionState) { + self.cachedCollectionStates[id] = state + self.updatedCollectionStateIds.insert(id) + } + + override func clearMemoryCache() { + self.cachedCollectionStates.removeAll() + self.updatedCollectionStateIds.removeAll() + } + + override func beforeCommit() { + if !self.updatedCollectionStateIds.isEmpty { + let sharedEncoder = Encoder() + for id in self.updatedCollectionStateIds { + if let state = self.cachedCollectionStates[id] { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(state) + self.valueBox.set(self.table, key: self.key(id), value: sharedEncoder.readBufferNoCopy()) + } + } + } + + self.updatedCollectionStateIds.removeAll() + } +} + diff --git a/Postbox/ItemCacheTable.swift b/Postbox/ItemCacheTable.swift new file mode 100644 index 0000000000..9b6b1f2aa5 --- /dev/null +++ b/Postbox/ItemCacheTable.swift @@ -0,0 +1,68 @@ +import Foundation + +public final class ItemCacheEntryId { + public let collectionId: ItemCacheCollectionId + public let key: ValueBoxKey + + public init(collectionId: ItemCacheCollectionId, key: ValueBoxKey) { + self.collectionId = collectionId + self.key = key + } +} + +private enum ItemCacheSection: Int8 { + case items = 0 + case accessIndexToItemId = 1 + case itemIdToAccessIndex = 2 +} + +final class ItemCacheTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private func itemKey(id: ItemCacheEntryId) -> ValueBoxKey { + let key = ValueBoxKey(length: 1 + 1 + id.key.length) + key.setInt8(0, value: ItemCacheSection.items.rawValue) + key.setInt8(1, value: id.collectionId) + memcpy(key.memory.advanced(by: 2), id.key.memory, id.key.length) + return key + } + + private func itemIdToAccessIndexKey(id: ItemCacheEntryId) -> ValueBoxKey { + let key = ValueBoxKey(length: 1 + 1 + id.key.length) + key.setInt8(0, value: ItemCacheSection.accessIndexToItemId.rawValue) + key.setInt8(1, value: id.collectionId) + memcpy(key.memory.advanced(by: 2), id.key.memory, id.key.length) + return key + } + + private func accessIndexToItemId(collectionId: ItemCacheCollectionId, index: Int32) -> ValueBoxKey { + let key = ValueBoxKey(length: 1 + 1 + 4) + key.setInt8(0, value: ItemCacheSection.accessIndexToItemId.rawValue) + key.setInt8(1, value: collectionId) + key.setInt32(2, value: index) + return key + } + + func put(id: ItemCacheEntryId, entry: Coding, metaTable: ItemCacheMetaTable) { + let encoder = Encoder() + encoder.encodeRootObject(entry) + self.valueBox.set(self.table, key: self.itemKey(id: id), value: encoder.readBufferNoCopy()) + } + + func retrieve(id: ItemCacheEntryId, metaTable: ItemCacheMetaTable) -> Coding? { + if let value = self.valueBox.get(self.table, key: self.itemKey(id: id)), let entry = Decoder(buffer: value).decodeRootObject() { + return entry + } + return nil + } + + override func clearMemoryCache() { + + } + + override func beforeCommit() { + + } +} diff --git a/Postbox/ItemCollectionInfoTable.swift b/Postbox/ItemCollectionInfoTable.swift index 726d36e2bb..d41894589c 100644 --- a/Postbox/ItemCollectionInfoTable.swift +++ b/Postbox/ItemCollectionInfoTable.swift @@ -1,6 +1,10 @@ import Foundation final class ItemCollectionInfoTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + private let sharedKey = ValueBoxKey(length: 4 + 4 + 8) private var cachedInfos: [ItemCollectionId.Namespace: [(Int, ItemCollectionId, ItemCollectionInfo)]] = [:] @@ -29,7 +33,7 @@ final class ItemCollectionInfoTable: Table { return cachedInfo } else { var infos: [(Int, ItemCollectionId, ItemCollectionInfo)] = [] - self.valueBox.range(self.tableId, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), values: { key, value in + self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), values: { key, value in if let info = Decoder(buffer: value).decodeRootObject() as? ItemCollectionInfo { infos.append((Int(key.getInt32(4)), ItemCollectionId(namespace: namespace, id: key.getInt64(4 + 4)), info)) } @@ -45,7 +49,7 @@ final class ItemCollectionInfoTable: Table { var currentKey = self.key(collectionId: collectionId, index: index) while true { var resultCollectionIdAndIndex: (ItemCollectionId, Int32)? - self.valueBox.range(self.tableId, start: currentKey, end: self.lowerBound(namespace: currentNamespace), keys: { key in + self.valueBox.range(self.table, start: currentKey, end: self.lowerBound(namespace: currentNamespace), keys: { key in resultCollectionIdAndIndex = (ItemCollectionId(namespace: currentNamespace, id: key.getInt64(4 + 4)), key.getInt32(4)) return true }, limit: 1) @@ -68,7 +72,7 @@ final class ItemCollectionInfoTable: Table { var currentKey = self.key(collectionId: collectionId, index: index) while true { var resultCollectionIdAndIndex: (ItemCollectionId, Int32)? - self.valueBox.range(self.tableId, start: currentKey, end: self.upperBound(namespace: currentNamespace), keys: { key in + self.valueBox.range(self.table, start: currentKey, end: self.upperBound(namespace: currentNamespace), keys: { key in resultCollectionIdAndIndex = (ItemCollectionId(namespace: currentNamespace, id: key.getInt64(4 + 4)), key.getInt32(4)) return true }, limit: 1) @@ -90,13 +94,13 @@ final class ItemCollectionInfoTable: Table { self.cachedInfos.removeAll() var currentCollectionKeys: [ValueBoxKey] = [] - self.valueBox.range(self.tableId, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in + self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in currentCollectionKeys.append(key) return true }, limit: 0) for key in currentCollectionKeys { - self.valueBox.remove(self.tableId, key: key) + self.valueBox.remove(self.table, key: key) } var index: Int32 = 0 @@ -104,7 +108,7 @@ final class ItemCollectionInfoTable: Table { for (id, info) in infos { sharedEncoder.reset() sharedEncoder.encodeRootObject(info) - self.valueBox.set(self.tableId, key: self.key(collectionId: id, index: index), value: sharedEncoder.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(collectionId: id, index: index), value: sharedEncoder.readBufferNoCopy()) index += 1 } } diff --git a/Postbox/ItemCollectionItemTable.swift b/Postbox/ItemCollectionItemTable.swift index ce96f5380c..6245739168 100644 --- a/Postbox/ItemCollectionItemTable.swift +++ b/Postbox/ItemCollectionItemTable.swift @@ -6,6 +6,10 @@ enum ItemCollectionOperation { } final class ItemCollectionItemTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + private let sharedKey = ValueBoxKey(length: 4 + 8 + 4 + 8) private func key(collectionId: ItemCollectionId, index: ItemCollectionItemIndex) -> ValueBoxKey { @@ -44,7 +48,7 @@ final class ItemCollectionItemTable: Table { func lowerItems(collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] { var items: [ItemCollectionItem] = [] - self.valueBox.range(self.tableId, start: self.key(collectionId: collectionId, index: itemIndex), end: self.lowerBound(collectionId: collectionId), values: { _, value in + self.valueBox.range(self.table, start: self.key(collectionId: collectionId, index: itemIndex), end: self.lowerBound(collectionId: collectionId), values: { _, value in if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { items.append(item) } @@ -55,7 +59,7 @@ final class ItemCollectionItemTable: Table { func higherItems(collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] { var items: [ItemCollectionItem] = [] - self.valueBox.range(self.tableId, start: self.key(collectionId: collectionId, index: itemIndex), end: self.upperBound(collectionId: collectionId), values: { _, value in + self.valueBox.range(self.table, start: self.key(collectionId: collectionId, index: itemIndex), end: self.upperBound(collectionId: collectionId), values: { _, value in if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { items.append(item) } @@ -66,7 +70,7 @@ final class ItemCollectionItemTable: Table { func getSummaryIndices(namespace: ItemCollectionId.Namespace) -> [ItemCollectionId: [ItemCollectionItemIndex]] { var summaryIndices: [ItemCollectionId: [ItemCollectionItemIndex]] = [:] - self.valueBox.range(self.tableId, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in + self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in let collectionId = ItemCollectionId(namespace: namespace, id: key.getInt64(4)) let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) if summaryIndices[collectionId] != nil { @@ -81,7 +85,7 @@ final class ItemCollectionItemTable: Table { func getItems(namespace: ItemCollectionId.Namespace) -> [ItemCollectionId: [ItemCollectionItem]] { var items: [ItemCollectionId: [ItemCollectionItem]] = [:] - self.valueBox.range(self.tableId, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), values: { key, value in + self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), values: { key, value in let collectionId = ItemCollectionId(namespace: namespace, id: key.getInt64(4)) //let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { @@ -98,7 +102,7 @@ final class ItemCollectionItemTable: Table { func replaceItems(collectionId: ItemCollectionId, items: [ItemCollectionItem]) { var currentIndices = Set() - self.valueBox.range(self.tableId, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId), keys: { key in + self.valueBox.range(self.table, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId), keys: { key in let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) currentIndices.insert(itemIndex) return true @@ -114,7 +118,7 @@ final class ItemCollectionItemTable: Table { let removedIndices = currentIndices.subtracting(updatedIndices) for index in removedIndices { - self.valueBox.remove(self.tableId, key: self.key(collectionId: collectionId, index: index)) + self.valueBox.remove(self.table, key: self.key(collectionId: collectionId, index: index)) } let sharedEncoder = Encoder() @@ -122,7 +126,7 @@ final class ItemCollectionItemTable: Table { let item = itemByIndex[index]! sharedEncoder.reset() sharedEncoder.encodeRootObject(item) - self.valueBox.set(self.tableId, key: self.key(collectionId: collectionId, index: index), value: sharedEncoder.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(collectionId: collectionId, index: index), value: sharedEncoder.readBufferNoCopy()) } } diff --git a/Postbox/KeychainTable.swift b/Postbox/KeychainTable.swift index 451c0e52f3..1c2fd8b68d 100644 --- a/Postbox/KeychainTable.swift +++ b/Postbox/KeychainTable.swift @@ -1,8 +1,8 @@ import Foundation final class KeychainTable: Table { - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) } private func key(_ string: String) -> ValueBoxKey { @@ -10,17 +10,17 @@ final class KeychainTable: Table { } func get(_ key: String) -> Data? { - if let value = self.valueBox.get(self.tableId, key: self.key(key)) { + if let value = self.valueBox.get(self.table, key: self.key(key)) { return Data(bytes: value.memory.assumingMemoryBound(to: UInt8.self), count: value.length) } return nil } func set(_ key: String, value: Data) { - self.valueBox.set(self.tableId, key: self.key(key), value: MemoryBuffer(data: value)) + self.valueBox.set(self.table, key: self.key(key), value: MemoryBuffer(data: value)) } func remove(_ key: String) { - self.valueBox.remove(self.tableId, key: self.key(key)) + self.valueBox.remove(self.table, key: self.key(key)) } } diff --git a/Postbox/MediaCleanupTable.swift b/Postbox/MediaCleanupTable.swift index 042d879c12..4b5d67590d 100644 --- a/Postbox/MediaCleanupTable.swift +++ b/Postbox/MediaCleanupTable.swift @@ -1,17 +1,18 @@ import Foundation final class MediaCleanupTable: Table { - var debugMedia: [Media] = [] + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) + override init(valueBox: ValueBox, table: ValueBoxTable) { + super.init(valueBox: valueBox, table: table) } func add(_ media: Media, sharedEncoder: Encoder = Encoder()) { - debugMedia.append(media) } func debugList() -> [Media] { - return self.debugMedia + return [] } } diff --git a/Postbox/MediaResource.swift b/Postbox/MediaResource.swift index 537e300cd9..2b31114fb4 100644 --- a/Postbox/MediaResource.swift +++ b/Postbox/MediaResource.swift @@ -6,18 +6,18 @@ public protocol MediaResourceId { func isEqual(to: MediaResourceId) -> Bool } -struct WrappedMediaResourceId: Hashable { - let id: MediaResourceId +public struct WrappedMediaResourceId: Hashable { + public let id: MediaResourceId - init(_ id: MediaResourceId) { + public init(_ id: MediaResourceId) { self.id = id } - static func ==(lhs: WrappedMediaResourceId, rhs: WrappedMediaResourceId) -> Bool { + public static func ==(lhs: WrappedMediaResourceId, rhs: WrappedMediaResourceId) -> Bool { return lhs.id.isEqual(to: rhs.id) } - var hashValue: Int { + public var hashValue: Int { return self.id.hashValue } } diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 4e32585703..ebf2df414c 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -107,6 +107,10 @@ private func modifyHistoryIndexEntryTimestamp(value: ReadBuffer, timestamp: Int3 } final class MessageHistoryIndexTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + let globalMessageIdsNamespace: Int32 let globalMessageIdsTable: GlobalMessageIdsTable let metadataTable: MessageHistoryMetadataTable @@ -114,13 +118,13 @@ final class MessageHistoryIndexTable: Table { var cachedMaxEntryByPeerId: [PeerId: [MessageId.Namespace: ValueBoxKey]] = [:] - init(valueBox: ValueBox, tableId: Int32, globalMessageIdsTable: GlobalMessageIdsTable, metadataTable: MessageHistoryMetadataTable, seedConfiguration: SeedConfiguration) { + init(valueBox: ValueBox, table: ValueBoxTable, globalMessageIdsTable: GlobalMessageIdsTable, metadataTable: MessageHistoryMetadataTable, seedConfiguration: SeedConfiguration) { self.globalMessageIdsTable = globalMessageIdsTable self.globalMessageIdsNamespace = globalMessageIdsTable.namespace self.seedConfiguration = seedConfiguration self.metadataTable = metadataTable - super.init(valueBox: valueBox, tableId: tableId) + super.init(valueBox: valueBox, table: table) } private func key(_ id: MessageId) -> ValueBoxKey { @@ -231,7 +235,7 @@ final class MessageHistoryIndexTable: Table { var modifyHole: (MessageIndex, MessageHistoryHole)? let startKey = self.key(MessageId(peerId: peerId, namespace: namespace, id: lowerIndex.id.id)) - self.valueBox.range(self.tableId, start: startKey, end: self.upperBound(peerId, namespace: namespace), values: { key, value in + self.valueBox.range(self.table, start: startKey, end: self.upperBound(peerId, namespace: namespace), values: { key, value in let entry = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value) if case let .Hole(hole) = entry { if lowerIndex.id.id <= hole.min { @@ -261,7 +265,7 @@ final class MessageHistoryIndexTable: Table { let index = MessageIndex(id: message.id, timestamp: message.timestamp) var upperItem: HistoryIndexEntry? - self.valueBox.range(self.tableId, start: self.key(index.id).predecessor, end: self.upperBound(index.id.peerId, namespace: index.id.namespace), values: { key, value in + self.valueBox.range(self.table, start: self.key(index.id).predecessor, end: self.upperBound(index.id.peerId, namespace: index.id.namespace), values: { key, value in upperItem = readHistoryIndexEntry(index.id.peerId, namespace: index.id.namespace, key: key, value: value) return true }, limit: 1) @@ -366,10 +370,10 @@ final class MessageHistoryIndexTable: Table { } func updateTimestamp(_ id: MessageId, timestamp: Int32, operations: inout [MessageHistoryIndexOperation]) { - if let previousData = self.valueBox.get(self.tableId, key: self.key(id)), let previousEntry = self.get(id), case let .Message(previousIndex) = previousEntry, previousIndex.timestamp != timestamp { + if let previousData = self.valueBox.get(self.table, key: self.key(id)), let previousEntry = self.get(id), case let .Message(previousIndex) = previousEntry, previousIndex.timestamp != timestamp { let updatedEntry = modifyHistoryIndexEntryTimestamp(value: previousData, timestamp: timestamp) - self.valueBox.remove(self.tableId, key: self.key(id)) - self.valueBox.set(self.tableId, key: self.key(id), value: updatedEntry) + self.valueBox.remove(self.table, key: self.key(id)) + self.valueBox.set(self.table, key: self.key(id), value: updatedEntry) operations.append(.UpdateTimestamp(MessageIndex(id: id, timestamp: previousIndex.timestamp), timestamp)) } @@ -379,7 +383,7 @@ final class MessageHistoryIndexTable: Table { self.ensureInitialized(id.peerId, operations: &operations) var upperItem: HistoryIndexEntry? - self.valueBox.range(self.tableId, start: self.key(id).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in + self.valueBox.range(self.table, start: self.key(id).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) return true }, limit: 1) @@ -559,7 +563,7 @@ final class MessageHistoryIndexTable: Table { value.write(&stableId, offset: 0, length: 4) value.write(&min, offset: 0, length: 4) value.write(&tags, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.key(hole.id), value: value) + self.valueBox.set(self.table, key: self.key(hole.id), value: value) operations.append(.InsertHole(hole)) } @@ -575,7 +579,7 @@ final class MessageHistoryIndexTable: Table { var timestamp: Int32 = index.timestamp value.write(&flags, offset: 0, length: 1) value.write(×tamp, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.key(index.id), value: value) + self.valueBox.set(self.table, key: self.key(index.id), value: value) operations.append(.InsertMessage(message)) @@ -585,7 +589,7 @@ final class MessageHistoryIndexTable: Table { } private func justRemove(_ index: MessageIndex, operations: inout [MessageHistoryIndexOperation]) { - self.valueBox.remove(self.tableId, key: self.key(index.id)) + self.valueBox.remove(self.table, key: self.key(index.id)) operations.append(.Remove(index)) if index.id.namespace == self.globalMessageIdsNamespace { @@ -597,13 +601,13 @@ final class MessageHistoryIndexTable: Table { let key = self.key(id) var lowerItem: HistoryIndexEntry? - self.valueBox.range(self.tableId, start: bindUpper ? key : key.successor, end: self.lowerBound(id.peerId, namespace: id.namespace), values: { key, value in + self.valueBox.range(self.table, start: bindUpper ? key : key.successor, end: self.lowerBound(id.peerId, namespace: id.namespace), values: { key, value in lowerItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) return true }, limit: 1) var upperItem: HistoryIndexEntry? - self.valueBox.range(self.tableId, start: bindUpper ? key.predecessor : key, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in + self.valueBox.range(self.table, start: bindUpper ? key.predecessor : key, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) return true }, limit: 1) @@ -616,7 +620,7 @@ final class MessageHistoryIndexTable: Table { self.ensureInitialized(id.peerId, operations: &operations) let key = self.key(id) - if let value = self.valueBox.get(self.tableId, key: key) { + if let value = self.valueBox.get(self.table, key: key) { return readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) } return nil @@ -627,7 +631,7 @@ final class MessageHistoryIndexTable: Table { self.ensureInitialized(peerId, operations: &operations) var entry: HistoryIndexEntry? - self.valueBox.range(self.tableId, start: self.upperBound(peerId, namespace: namespace), end: self.lowerBound(peerId, namespace: namespace), values: { key, value in + self.valueBox.range(self.table, start: self.upperBound(peerId, namespace: namespace), end: self.lowerBound(peerId, namespace: namespace), values: { key, value in entry = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value) return false }, limit: 1) @@ -636,12 +640,12 @@ final class MessageHistoryIndexTable: Table { } func exists(_ id: MessageId) -> Bool { - return self.valueBox.exists(self.tableId, key: self.key(id)) + return self.valueBox.exists(self.table, key: self.key(id)) } func holeContainingId(_ id: MessageId) -> MessageHistoryHole? { var result: MessageHistoryHole? - self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: id.peerId, namespace: id.namespace, id: id.id)).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in + self.valueBox.range(self.table, start: self.key(MessageId(peerId: id.peerId, namespace: id.namespace, id: id.id)).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in if case let .Hole(hole) = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) { result = hole } @@ -655,7 +659,7 @@ final class MessageHistoryIndexTable: Table { var count = 0 var holes = false - self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: minId)).predecessor, end: self.key(MessageId(peerId: peerId, namespace: namespace, id: maxId)).successor, values: { _, value in + self.valueBox.range(self.table, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: minId)).predecessor, end: self.key(MessageId(peerId: peerId, namespace: namespace, id: maxId)).successor, values: { _, value in var flags: Int8 = 0 value.read(&flags, offset: 0, length: 1) if (flags & HistoryEntryTypeMask) == HistoryEntryTypeMessage { @@ -668,7 +672,7 @@ final class MessageHistoryIndexTable: Table { return true }, limit: 0) - self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: maxId)), end: self.upperBound(peerId, namespace: namespace), values: { key, value in + self.valueBox.range(self.table, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: maxId)), end: self.upperBound(peerId, namespace: namespace), values: { key, value in var flags: Int8 = 0 value.read(&flags, offset: 0, length: 1) if (flags & HistoryEntryTypeMask) == HistoryEntryTypeHole { @@ -688,7 +692,7 @@ final class MessageHistoryIndexTable: Table { var holes = false for id in ids { - self.valueBox.range(self.tableId, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: id)).predecessor, end: self.upperBound(peerId, namespace: namespace), values: { key, value in + self.valueBox.range(self.table, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: id)).predecessor, end: self.upperBound(peerId, namespace: namespace), values: { key, value in let entryId = key.getInt32(8 + 4) var flags: Int8 = 0 value.read(&flags, offset: 0, length: 1) @@ -714,7 +718,7 @@ final class MessageHistoryIndexTable: Table { func debugList(_ peerId: PeerId, namespace: MessageId.Namespace) -> [HistoryIndexEntry] { var list: [HistoryIndexEntry] = [] - self.valueBox.range(self.tableId, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in + self.valueBox.range(self.table, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in list.append(readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value)) return true diff --git a/Postbox/MessageHistoryInvalidatedReadStateTable.swift b/Postbox/MessageHistoryInvalidatedReadStateTable.swift index 0aaf0ebd8b..e33f9d16f3 100644 --- a/Postbox/MessageHistoryInvalidatedReadStateTable.swift +++ b/Postbox/MessageHistoryInvalidatedReadStateTable.swift @@ -25,6 +25,10 @@ public func ==(lhs: PeerReadStateSynchronizationOperation, rhs: PeerReadStateSyn } final class MessageHistorySynchronizeReadStateTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private let sharedKey = ValueBoxKey(length: 8) private func key(peerId: PeerId) -> ValueBoxKey { @@ -35,8 +39,8 @@ final class MessageHistorySynchronizeReadStateTable: Table { private var updatedPeerIds: [PeerId: PeerReadStateSynchronizationOperation?] = [:] private func lowerBound() -> ValueBoxKey { - let key = ValueBoxKey(length: 1) - key.setInt8(0, value: 0) + let key = ValueBoxKey(length: 8) + key.setInt64(0, value: 0) return key } @@ -55,7 +59,7 @@ final class MessageHistorySynchronizeReadStateTable: Table { self.beforeCommit() var operations: [PeerId: PeerReadStateSynchronizationOperation] = [:] - self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), values: { key, value in + self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), values: { key, value in var operationValue: Int8 = 0 value.read(&operationValue, offset: 0, length: 1) @@ -92,9 +96,9 @@ final class MessageHistorySynchronizeReadStateTable: Table { buffer.write(&operationValue, offset: 0, length: 1) } - self.valueBox.set(self.tableId, key: key, value: buffer) + self.valueBox.set(self.table, key: key, value: buffer) } else { - self.valueBox.remove(self.tableId, key: key) + self.valueBox.remove(self.table, key: key) } } self.updatedPeerIds.removeAll() diff --git a/Postbox/MessageHistoryMetadataTable.swift b/Postbox/MessageHistoryMetadataTable.swift index c5ad09cc25..7f34c663f8 100644 --- a/Postbox/MessageHistoryMetadataTable.swift +++ b/Postbox/MessageHistoryMetadataTable.swift @@ -8,6 +8,10 @@ private enum MetadataPrefix: Int8 { } final class MessageHistoryMetadataTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + let sharedPeerHistoryInitializedKey = ValueBoxKey(length: 8 + 1) let sharedPeerNextMessageIdByNamespaceKey = ValueBoxKey(length: 8 + 1 + 4) let sharedBuffer = WriteBuffer() @@ -21,10 +25,6 @@ final class MessageHistoryMetadataTable: Table { private var nextMessageStableId: UInt32? private var nextMessageStableIdUpdated = false - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) - } - private func peerHistoryInitializedKey(_ id: PeerId) -> ValueBoxKey { self.sharedPeerHistoryInitializedKey.setInt64(0, value: id.toInt64()) self.sharedPeerHistoryInitializedKey.setInt8(8, value: MetadataPrefix.PeerHistoryInitialized.rawValue) @@ -47,7 +47,7 @@ final class MessageHistoryMetadataTable: Table { func setInitializedChatList() { self.initializedChatList = true - self.valueBox.set(self.tableId, key: self.key(MetadataPrefix.ChatListInitialized), value: MemoryBuffer()) + self.valueBox.set(self.table, key: self.key(MetadataPrefix.ChatListInitialized), value: MemoryBuffer()) } func isInitializedChatList() -> Bool { @@ -55,7 +55,7 @@ final class MessageHistoryMetadataTable: Table { return initializedChatList } - if self.valueBox.get(self.tableId, key: self.key(MetadataPrefix.ChatListInitialized)) != nil { + if self.valueBox.get(self.table, key: self.key(MetadataPrefix.ChatListInitialized)) != nil { self.initializedChatList = true return true } @@ -66,14 +66,14 @@ final class MessageHistoryMetadataTable: Table { func setInitialized(_ peerId: PeerId) { self.initializedHistoryPeerIds.insert(peerId) self.sharedBuffer.reset() - self.valueBox.set(self.tableId, key: self.peerHistoryInitializedKey(peerId), value: self.sharedBuffer) + self.valueBox.set(self.table, key: self.peerHistoryInitializedKey(peerId), value: self.sharedBuffer) } func isInitialized(_ peerId: PeerId) -> Bool { if self.initializedHistoryPeerIds.contains(peerId) { return true } else { - if self.valueBox.exists(self.tableId, key: self.peerHistoryInitializedKey(peerId)) { + if self.valueBox.exists(self.table, key: self.peerHistoryInitializedKey(peerId)) { self.initializedHistoryPeerIds.insert(peerId) return true } else { @@ -94,7 +94,7 @@ final class MessageHistoryMetadataTable: Table { return MessageId(peerId: peerId, namespace: namespace, id: nextId) } else { var nextId: Int32 = 1 - if let value = self.valueBox.get(self.tableId, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) { + if let value = self.valueBox.get(self.table, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) { value.read(&nextId, offset: 0, length: 4) } self.peerNextMessageIdByNamespace[peerId]![namespace] = nextId + 1 @@ -107,7 +107,7 @@ final class MessageHistoryMetadataTable: Table { } } else { var nextId: Int32 = 1 - if let value = self.valueBox.get(self.tableId, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) { + if let value = self.valueBox.get(self.table, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) { value.read(&nextId, offset: 0, length: 4) } @@ -127,7 +127,7 @@ final class MessageHistoryMetadataTable: Table { self.nextMessageStableIdUpdated = true return nextId } else { - if let value = self.valueBox.get(self.tableId, key: self.key(.NextStableMessageId)) { + if let value = self.valueBox.get(self.table, key: self.key(.NextStableMessageId)) { var nextId: UInt32 = 0 value.read(&nextId, offset: 0, length: 4) self.nextMessageStableId = nextId + 1 @@ -159,9 +159,9 @@ final class MessageHistoryMetadataTable: Table { sharedBuffer.reset() var mutableMaxId = maxId sharedBuffer.write(&mutableMaxId, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace), value: sharedBuffer) + self.valueBox.set(self.table, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace), value: sharedBuffer) } else { - self.valueBox.remove(self.tableId, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) + self.valueBox.remove(self.table, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) } } } @@ -170,7 +170,7 @@ final class MessageHistoryMetadataTable: Table { if self.nextMessageStableIdUpdated { if let nextMessageStableId = self.nextMessageStableId { var nextId: UInt32 = nextMessageStableId - self.valueBox.set(self.tableId, key: self.key(.NextStableMessageId), value: MemoryBuffer(memory: &nextId, capacity: 4, length: 4, freeWhenDone: false)) + self.valueBox.set(self.table, key: self.key(.NextStableMessageId), value: MemoryBuffer(memory: &nextId, capacity: 4, length: 4, freeWhenDone: false)) self.nextMessageStableIdUpdated = false } } diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index a6555d9c6a..0be6dfaa79 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -16,6 +16,10 @@ private final class InternalPeerReadStates { } final class MessageHistoryReadStateTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private var cachedPeerReadStates: [PeerId: InternalPeerReadStates?] = [:] private var updatedPeerIds = Set() @@ -26,15 +30,15 @@ final class MessageHistoryReadStateTable: Table { return self.sharedKey } - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) + override init(valueBox: ValueBox, table: ValueBoxTable) { + super.init(valueBox: valueBox, table: table) } private func get(_ id: PeerId) -> InternalPeerReadStates? { if let states = self.cachedPeerReadStates[id] { return states } else { - if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let value = self.valueBox.get(self.table, key: self.key(id)) { var count: Int32 = 0 value.read(&count, offset: 0, length: 4) var stateByNamespace: [MessageId.Namespace: PeerReadState] = [:] @@ -278,9 +282,9 @@ final class MessageHistoryReadStateTable: Table { sharedBuffer.write(&maxKnownId, offset: 0, length: 4) sharedBuffer.write(&count, offset: 0, length: 4) } - self.valueBox.set(self.tableId, key: self.key(id), value: sharedBuffer) + self.valueBox.set(self.table, key: self.key(id), value: sharedBuffer) } else { - self.valueBox.remove(self.tableId, key: self.key(id)) + self.valueBox.remove(self.table, key: self.key(id)) } } self.updatedPeerIds.removeAll() diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 7cb2fb5100..ffcfeb46e6 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -34,6 +34,10 @@ enum RenderedMessageHistoryEntry { } final class MessageHistoryTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + let messageHistoryIndexTable: MessageHistoryIndexTable let messageMediaTable: MessageMediaTable let historyMetadataTable: MessageHistoryMetadataTable @@ -42,7 +46,7 @@ final class MessageHistoryTable: Table { let readStateTable: MessageHistoryReadStateTable let synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable - init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable) { + init(valueBox: ValueBox, table: ValueBoxTable, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable) { self.messageHistoryIndexTable = messageHistoryIndexTable self.messageMediaTable = messageMediaTable self.historyMetadataTable = historyMetadataTable @@ -51,7 +55,7 @@ final class MessageHistoryTable: Table { self.readStateTable = readStateTable self.synchronizeReadStateTable = synchronizeReadStateTable - super.init(valueBox: valueBox, tableId: tableId) + super.init(valueBox: valueBox, table: table) } private func key(_ index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4)) -> ValueBoxKey { @@ -393,7 +397,7 @@ final class MessageHistoryTable: Table { var currentKey = self.upperBound(peerId) while true { var entry: IntermediateMessageHistoryEntry? - self.valueBox.range(self.tableId, start: currentKey, end: self.lowerBound(peerId), values: { key, value in + self.valueBox.range(self.table, start: currentKey, end: self.lowerBound(peerId), values: { key, value in entry = self.readIntermediateEntry(key, value: value) return true }, limit: 1) @@ -414,7 +418,7 @@ final class MessageHistoryTable: Table { func getMessage(_ index: MessageIndex) -> IntermediateMessage? { let key = self.key(index) - if let value = self.valueBox.get(self.tableId, key: key) { + if let value = self.valueBox.get(self.table, key: key) { let entry = self.readIntermediateEntry(key, value: value) if case let .Message(message) = entry { return message @@ -422,7 +426,7 @@ final class MessageHistoryTable: Table { } else if let tableIndex = self.messageHistoryIndexTable.get(index.id) { if case let .Message(updatedIndex) = tableIndex { let key = self.key(updatedIndex) - if let value = self.valueBox.get(self.tableId, key: key) { + if let value = self.valueBox.get(self.table, key: key) { let entry = self.readIntermediateEntry(key, value: value) if case let .Message(message) = entry { return message @@ -563,7 +567,7 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&idId, offset: 0, length: 8) } - self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) + self.valueBox.set(self.table, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, timestamp: message.timestamp, flags: flags, tags: message.tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) } @@ -578,12 +582,12 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&minId, offset: 0, length: 4) var tags: UInt32 = hole.tags sharedBuffer.write(&tags, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.key(hole.maxIndex), value: sharedBuffer.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(hole.maxIndex), value: sharedBuffer.readBufferNoCopy()) } private func justRemove(_ index: MessageIndex, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) { let key = self.key(index) - if let value = self.valueBox.get(self.tableId, key: key) { + if let value = self.valueBox.get(self.table, key: key) { switch self.readIntermediateEntry(key, value: value) { case let .Message(message): let embeddedMediaData = message.embeddedMediaData @@ -639,7 +643,7 @@ final class MessageHistoryTable: Table { } } - self.valueBox.remove(self.tableId, key: key) + self.valueBox.remove(self.table, key: key) } } @@ -780,7 +784,7 @@ final class MessageHistoryTable: Table { } } - self.valueBox.remove(self.tableId, key: self.key(index)) + self.valueBox.remove(self.table, key: self.key(index)) if !previousMessage.tags.isEmpty { self.tagsTable.remove(previousMessage.tags, index: index) } @@ -923,7 +927,7 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&idId, offset: 0, length: 8) } - self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) + self.valueBox.set(self.table, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, timestamp: message.timestamp, flags: flags, tags: tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) } else { @@ -933,7 +937,7 @@ final class MessageHistoryTable: Table { private func justUpdateTimestamp(_ index: MessageIndex, timestamp: Int32) { if let previousMessage = self.getMessage(index) { - self.valueBox.remove(self.tableId, key: self.key(index)) + self.valueBox.remove(self.table, key: self.key(index)) var updatedMessage = IntermediateMessage(stableId: previousMessage.stableId, stableVersion: previousMessage.stableVersion + 1, id: previousMessage.id, timestamp: timestamp, flags: previousMessage.flags, tags: previousMessage.tags, forwardInfo: previousMessage.forwardInfo, authorId: previousMessage.authorId, text: previousMessage.text, attributesData: previousMessage.attributesData, embeddedMediaData: previousMessage.embeddedMediaData, referencedMedia: previousMessage.referencedMedia) self.storeIntermediateMessage(updatedMessage, sharedKey: self.key(index)) @@ -1072,7 +1076,7 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&idId, offset: 0, length: 8) } - self.valueBox.set(self.tableId, key: self.key(MessageIndex(id: message.id, timestamp: message.timestamp), key: sharedKey), value: sharedBuffer) + self.valueBox.set(self.table, key: self.key(MessageIndex(id: message.id, timestamp: message.timestamp), key: sharedKey), value: sharedBuffer) } private func readIntermediateEntry(_ key: ValueBoxKey, value: ReadBuffer) -> IntermediateMessageHistoryEntry { @@ -1295,7 +1299,7 @@ final class MessageHistoryTable: Table { var lower: IntermediateMessageHistoryEntry? var upper: IntermediateMessageHistoryEntry? - self.valueBox.range(self.tableId, start: self.key(index), end: self.lowerBound(index.id.peerId), values: { key, value in + self.valueBox.range(self.table, start: self.key(index), end: self.lowerBound(index.id.peerId), values: { key, value in lowerEntries.append(self.readIntermediateEntry(key, value: value)) return true }, limit: count / 2 + 1) @@ -1305,7 +1309,7 @@ final class MessageHistoryTable: Table { lowerEntries.removeLast() } - self.valueBox.range(self.tableId, start: self.key(index).predecessor, end: self.upperBound(index.id.peerId), values: { key, value in + self.valueBox.range(self.table, start: self.key(index).predecessor, end: self.upperBound(index.id.peerId), values: { key, value in upperEntries.append(self.readIntermediateEntry(key, value: value)) return true }, limit: count - lowerEntries.count + 1) @@ -1316,7 +1320,7 @@ final class MessageHistoryTable: Table { if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count { var additionalLowerEntries: [IntermediateMessageHistoryEntry] = [] - self.valueBox.range(self.tableId, start: self.key(lowerEntries.last!.index), end: self.lowerBound(index.id.peerId), values: { key, value in + self.valueBox.range(self.table, start: self.key(lowerEntries.last!.index), end: self.lowerBound(index.id.peerId), values: { key, value in additionalLowerEntries.append(self.readIntermediateEntry(key, value: value)) return true }, limit: count - lowerEntries.count - upperEntries.count + 1) @@ -1345,7 +1349,7 @@ final class MessageHistoryTable: Table { var entries: [IntermediateMessageHistoryEntry] = [] for index in indices { let key = self.key(index) - if let value = self.valueBox.get(self.tableId, key: key) { + if let value = self.valueBox.get(self.table, key: key) { entries.append(readIntermediateEntry(key, value: value)) } else { assertionFailure() @@ -1357,7 +1361,7 @@ final class MessageHistoryTable: Table { if let lowerIndex = lower { let key = self.key(lowerIndex) - if let value = self.valueBox.get(self.tableId, key: key) { + if let value = self.valueBox.get(self.table, key: key) { lowerEntry = readIntermediateEntry(key, value: value) } else { assertionFailure() @@ -1366,7 +1370,7 @@ final class MessageHistoryTable: Table { if let upperIndex = upper { let key = self.key(upperIndex) - if let value = self.valueBox.get(self.tableId, key: key) { + if let value = self.valueBox.get(self.table, key: key) { upperEntry = readIntermediateEntry(key, value: value) } else { assertionFailure() @@ -1390,7 +1394,7 @@ final class MessageHistoryTable: Table { } else { key = self.upperBound(peerId) } - self.valueBox.range(self.tableId, start: key, end: self.lowerBound(peerId), values: { key, value in + self.valueBox.range(self.table, start: key, end: self.lowerBound(peerId), values: { key, value in entries.append(self.readIntermediateEntry(key, value: value)) return true }, limit: count) @@ -1409,7 +1413,7 @@ final class MessageHistoryTable: Table { var entries: [IntermediateMessageHistoryEntry] = [] for index in indices { let key = self.key(index) - if let value = self.valueBox.get(self.tableId, key: key) { + if let value = self.valueBox.get(self.table, key: key) { entries.append(readIntermediateEntry(key, value: value)) } else { assertionFailure() @@ -1433,7 +1437,7 @@ final class MessageHistoryTable: Table { } else { key = self.lowerBound(peerId) } - self.valueBox.range(self.tableId, start: key, end: self.upperBound(peerId), values: { key, value in + self.valueBox.range(self.table, start: key, end: self.upperBound(peerId), values: { key, value in entries.append(self.readIntermediateEntry(key, value: value)) return true }, limit: count) @@ -1452,7 +1456,7 @@ final class MessageHistoryTable: Table { var entries: [IntermediateMessageHistoryEntry] = [] for index in indices { let key = self.key(index) - if let value = self.valueBox.get(self.tableId, key: key) { + if let value = self.valueBox.get(self.table, key: key) { entries.append(readIntermediateEntry(key, value: value)) } else { assertionFailure() @@ -1485,6 +1489,19 @@ final class MessageHistoryTable: Table { } return nil } + + func findMessageId(peerId: PeerId, namespace: MessageId.Namespace, timestamp: Int32) -> MessageId? { + var result: MessageId? + self.valueBox.range(self.table, start: self.key(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: timestamp)), end: self.key(MessageIndex(id: MessageId(peerId: peerId, namespace: Int32.max, id: Int32.max), timestamp: timestamp)), values: { key, value in + let entry = self.readIntermediateEntry(key, value: value) + if case let .Message(message) = entry { + result = entry.index.id + return false + } + return true + }, limit: 0) + return result + } func debugList(_ peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] diff --git a/Postbox/MessageHistoryTagsTable.swift b/Postbox/MessageHistoryTagsTable.swift index 55c6412b36..3514b13ae2 100644 --- a/Postbox/MessageHistoryTagsTable.swift +++ b/Postbox/MessageHistoryTagsTable.swift @@ -1,12 +1,12 @@ import Foundation class MessageHistoryTagsTable: Table { - private let sharedKey = ValueBoxKey(length: 8 + 4 + 4 + 4 + 4) - - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) } + private let sharedKey = ValueBoxKey(length: 8 + 4 + 4 + 4 + 4) + private func key(_ tagMask: MessageTags, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4 + 4)) -> ValueBoxKey { key.setInt64(0, value: index.id.peerId.toInt64()) key.setUInt32(8, value: tagMask.rawValue) @@ -31,23 +31,23 @@ class MessageHistoryTagsTable: Table { } func add(_ tagMask: MessageTags, index: MessageIndex) { - self.valueBox.set(self.tableId, key: self.key(tagMask, index: index, key: self.sharedKey), value: MemoryBuffer()) + self.valueBox.set(self.table, key: self.key(tagMask, index: index, key: self.sharedKey), value: MemoryBuffer()) } func remove(_ tagMask: MessageTags, index: MessageIndex) { - self.valueBox.remove(self.tableId, key: self.key(tagMask, index: index, key: self.sharedKey)) + self.valueBox.remove(self.table, key: self.key(tagMask, index: index, key: self.sharedKey)) } func entryLocation(at index: MessageIndex, tagMask: MessageTags) -> MessageHistoryEntryLocation? { - if let _ = self.valueBox.get(self.tableId, key: self.key(tagMask, index: index)) { + if let _ = self.valueBox.get(self.table, key: self.key(tagMask, index: index)) { var greaterCount = 0 - self.valueBox.range(self.tableId, start: self.key(tagMask, index: index), end: self.upperBound(tagMask, peerId: index.id.peerId), keys: { _ in + self.valueBox.range(self.table, start: self.key(tagMask, index: index), end: self.upperBound(tagMask, peerId: index.id.peerId), keys: { _ in greaterCount += 1 return true }, limit: 0) var lowerCount = 0 - self.valueBox.range(self.tableId, start: self.key(tagMask, index: index), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { _ in + self.valueBox.range(self.table, start: self.key(tagMask, index: index), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { _ in lowerCount += 1 return true }, limit: 0) @@ -63,7 +63,7 @@ class MessageHistoryTagsTable: Table { var lower: MessageIndex? var upper: MessageIndex? - self.valueBox.range(self.tableId, start: self.key(tagMask, index: index), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { key in + self.valueBox.range(self.table, start: self.key(tagMask, index: index), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) lowerEntries.append(index) return true @@ -74,7 +74,7 @@ class MessageHistoryTagsTable: Table { lowerEntries.removeLast() } - self.valueBox.range(self.tableId, start: self.key(tagMask, index: index).predecessor, end: self.upperBound(tagMask, peerId: index.id.peerId), keys: { key in + self.valueBox.range(self.table, start: self.key(tagMask, index: index).predecessor, end: self.upperBound(tagMask, peerId: index.id.peerId), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) upperEntries.append(index) return true @@ -86,7 +86,7 @@ class MessageHistoryTagsTable: Table { if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count { var additionalLowerEntries: [MessageIndex] = [] - self.valueBox.range(self.tableId, start: self.key(tagMask, index: lowerEntries.last!), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { key in + self.valueBox.range(self.table, start: self.key(tagMask, index: lowerEntries.last!), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) additionalLowerEntries.append(index) return true @@ -112,7 +112,7 @@ class MessageHistoryTagsTable: Table { } else { key = self.upperBound(tagMask, peerId: peerId) } - self.valueBox.range(self.tableId, start: key, end: self.lowerBound(tagMask, peerId: peerId), keys: { key in + self.valueBox.range(self.table, start: key, end: self.lowerBound(tagMask, peerId: peerId), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) indices.append(index) return true @@ -128,7 +128,7 @@ class MessageHistoryTagsTable: Table { } else { key = self.lowerBound(tagMask, peerId: peerId) } - self.valueBox.range(self.tableId, start: key, end: self.upperBound(tagMask, peerId: peerId), keys: { key in + self.valueBox.range(self.table, start: key, end: self.upperBound(tagMask, peerId: peerId), keys: { key in let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4)) indices.append(index) return true diff --git a/Postbox/MessageHistoryUnsentTable.swift b/Postbox/MessageHistoryUnsentTable.swift index a0324ab3fe..783f91d25c 100644 --- a/Postbox/MessageHistoryUnsentTable.swift +++ b/Postbox/MessageHistoryUnsentTable.swift @@ -6,6 +6,10 @@ enum IntermediateMessageHistoryUnsentOperation { } final class MessageHistoryUnsentTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + private let sharedKey = ValueBoxKey(length: 4 + 4 + 8) private func key(_ id: MessageId) -> ValueBoxKey { @@ -28,23 +32,19 @@ final class MessageHistoryUnsentTable: Table { return key } - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) - } - func add(_ id: MessageId, operations: inout [IntermediateMessageHistoryUnsentOperation]) { - self.valueBox.set(self.tableId, key: self.key(id), value: MemoryBuffer()) + self.valueBox.set(self.table, key: self.key(id), value: MemoryBuffer()) operations.append(.Insert(id)) } func remove(_ id: MessageId, operations: inout [IntermediateMessageHistoryUnsentOperation]) { - self.valueBox.remove(self.tableId, key: self.key(id)) + self.valueBox.remove(self.table, key: self.key(id)) operations.append(.Remove(id)) } func get() -> [MessageId] { var ids: [MessageId] = [] - self.valueBox.range(self.tableId, start: self.lowerBound(), end: self.upperBound(), keys: { key in + self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), keys: { key in ids.append(MessageId(peerId: PeerId(key.getInt64(4 + 4)), namespace: key.getInt32(0), id: key.getInt32(4))) return true }, limit: 0) diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index a1e9d9b744..d82a30d75a 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -592,7 +592,11 @@ final class MutableMessageHistoryView { } } - return (hole, hole.maxIndex <= self.anchorIndex.index ? .UpperToLower : .LowerToUpper) + if hole.maxIndex.timestamp == Int32.max && self.anchorIndex.index.timestamp == Int32.max { + return (hole, .UpperToLower) + } else { + return (hole, hole.maxIndex <= self.anchorIndex.index ? .UpperToLower : .LowerToUpper) + } } } diff --git a/Postbox/MessageMediaTable.swift b/Postbox/MessageMediaTable.swift index fbef514933..b9bf16adb2 100644 --- a/Postbox/MessageMediaTable.swift +++ b/Postbox/MessageMediaTable.swift @@ -21,12 +21,8 @@ enum DebugMediaEntry { } final class MessageMediaTable: Table { - let mediaCleanupTable: MediaCleanupTable - - init(valueBox: ValueBox, tableId: Int32, mediaCleanupTable: MediaCleanupTable) { - self.mediaCleanupTable = mediaCleanupTable - - super.init(valueBox: valueBox, tableId: tableId) + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) } func key(_ id: MediaId, key: ValueBoxKey = ValueBoxKey(length: 4 + 8)) -> ValueBoxKey { @@ -36,7 +32,7 @@ final class MessageMediaTable: Table { } func get(_ id: MediaId, embedded: @noescape(MessageIndex, MediaId) -> Media?) -> Media? { - if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let value = self.valueBox.get(self.table, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) if type == MediaEntryType.Direct.rawValue { @@ -65,7 +61,7 @@ final class MessageMediaTable: Table { func set(_ media: Media, index: MessageIndex, messageHistoryTable: MessageHistoryTable, sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) -> InsertMediaResult { if let id = media.id { - if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let value = self.valueBox.get(self.table, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) if type == MediaEntryType.Direct.rawValue { @@ -81,7 +77,7 @@ final class MessageMediaTable: Table { messageReferenceCount += 1 sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) return .Reference } else if type == MediaEntryType.MessageReference.rawValue { @@ -114,7 +110,7 @@ final class MessageMediaTable: Table { var messageReferenceCount: Int32 = 2 sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) } return .Reference @@ -134,7 +130,7 @@ final class MessageMediaTable: Table { sharedWriteBuffer.write(&idId, offset: 0, length: 4) sharedWriteBuffer.write(&idTimestamp, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) return .Embed(media) } @@ -144,7 +140,7 @@ final class MessageMediaTable: Table { } func removeReference(_ id: MediaId, sharedWriteBuffer: WriteBuffer = WriteBuffer()) -> RemoveMediaResult { - if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let value = self.valueBox.get(self.table, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) if type == MediaEntryType.Direct.rawValue { @@ -162,12 +158,9 @@ final class MessageMediaTable: Table { sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) if messageReferenceCount <= 0 { - if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + mediaOffset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media { - self.mediaCleanupTable.add(media) - } - self.valueBox.remove(self.tableId, key: self.key(id)) + self.valueBox.remove(self.table, key: self.key(id)) } else { - self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) } return .Reference @@ -183,7 +176,7 @@ final class MessageMediaTable: Table { let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp) - self.valueBox.remove(self.tableId, key: self.key(id)) + self.valueBox.remove(self.table, key: self.key(id)) return .Embedded(referencedMessageIndex) } else { @@ -195,14 +188,13 @@ final class MessageMediaTable: Table { func removeEmbeddedMedia(_ media: Media) { if let id = media.id { - self.valueBox.remove(self.tableId, key: self.key(id)) + self.valueBox.remove(self.table, key: self.key(id)) } - self.mediaCleanupTable.add(media) } func update(_ id: MediaId, media: Media, messageHistoryTable: MessageHistoryTable, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) { if let updatedId = media.id { - if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let value = self.valueBox.get(self.table, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) if type == MediaEntryType.Direct.rawValue { @@ -227,9 +219,9 @@ final class MessageMediaTable: Table { sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) if id != updatedId { - self.valueBox.remove(self.tableId, key: self.key(id)) + self.valueBox.remove(self.table, key: self.key(id)) } - self.valueBox.set(self.tableId, key: self.key(updatedId), value: sharedWriteBuffer.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(updatedId), value: sharedWriteBuffer.readBufferNoCopy()) } else if type == MediaEntryType.MessageReference.rawValue { var idPeerId: Int64 = 0 var idNamespace: Int32 = 0 @@ -252,7 +244,7 @@ final class MessageMediaTable: Table { let upperBoundKey = ValueBoxKey(length: 8 + 4) memset(upperBoundKey.memory, 0xff, 8 + 4) - self.valueBox.range(self.tableId, start: ValueBoxKey(length: 0), end: upperBoundKey, values: { key, value in + self.valueBox.range(self.table, start: ValueBoxKey(length: 0), end: upperBoundKey, values: { key, value in var type: Int8 = 0 value.read(&type, offset: 0, length: 1) if type == MediaEntryType.Direct.rawValue { diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index b55904b1b6..03dc16dd6b 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -8,20 +8,24 @@ private enum MetadataKey: Int32 { } final class MetadataTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private let sharedBuffer = WriteBuffer() - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) + override init(valueBox: ValueBox, table: ValueBoxTable) { + super.init(valueBox: valueBox, table: table) } private func key(_ key: MetadataKey) -> ValueBoxKey { - let valueBoxKey = ValueBoxKey(length: 4) - valueBoxKey.setInt32(0, value: key.rawValue) + let valueBoxKey = ValueBoxKey(length: 8) + valueBoxKey.setInt64(0, value: Int64(key.rawValue)) return valueBoxKey } func userVersion() -> Int32? { - if let value = self.valueBox.get(self.tableId, key: self.key(.UserVersion)) { + if let value = self.valueBox.get(self.table, key: self.key(.UserVersion)) { var version: Int32 = 0 value.read(&version, offset: 0, length: 4) return version @@ -34,11 +38,11 @@ final class MetadataTable: Table { let buffer = sharedBuffer var varVersion: Int32 = version buffer.write(&varVersion, offset: 0, length: 4) - self.valueBox.set(self.tableId, key: self.key(.UserVersion), value: buffer) + self.valueBox.set(self.table, key: self.key(.UserVersion), value: buffer) } func state() -> Coding? { - if let value = self.valueBox.get(self.tableId, key: self.key(.State)) { + if let value = self.valueBox.get(self.table, key: self.key(.State)) { if let state = Decoder(buffer: value).decodeRootObject() { return state } @@ -49,11 +53,11 @@ final class MetadataTable: Table { func setState(_ state: Coding) { let encoder = Encoder() encoder.encodeRootObject(state) - self.valueBox.set(self.tableId, key: self.key(.State), value: encoder.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(.State), value: encoder.readBufferNoCopy()) } func transactionStateVersion() -> Int64 { - if let value = self.valueBox.get(self.tableId, key: self.key(.TransactionStateVersion)) { + if let value = self.valueBox.get(self.table, key: self.key(.TransactionStateVersion)) { var version: Int64 = 0 value.read(&version, offset: 0, length: 8) return version @@ -67,12 +71,12 @@ final class MetadataTable: Table { sharedBuffer.reset() let buffer = sharedBuffer buffer.write(&version, offset: 0, length: 8) - self.valueBox.set(self.tableId, key: self.key(.TransactionStateVersion), value: buffer) + self.valueBox.set(self.table, key: self.key(.TransactionStateVersion), value: buffer) return version } func masterClientId() -> Int64 { - if let value = self.valueBox.get(self.tableId, key: self.key(.MasterClientId)) { + if let value = self.valueBox.get(self.table, key: self.key(.MasterClientId)) { var clientId: Int64 = 0 value.read(&clientId, offset: 0, length: 8) return clientId @@ -86,6 +90,6 @@ final class MetadataTable: Table { let buffer = sharedBuffer var clientId = id buffer.write(&clientId, offset: 0, length: 8) - self.valueBox.set(self.tableId, key: self.key(.MasterClientId), value: buffer) + self.valueBox.set(self.table, key: self.key(.MasterClientId), value: buffer) } } diff --git a/Postbox/OrderStatisticTable.swift b/Postbox/OrderStatisticTable.swift index 758abf9f86..8957cc9469 100644 --- a/Postbox/OrderStatisticTable.swift +++ b/Postbox/OrderStatisticTable.swift @@ -1,8 +1,8 @@ import Foundation class MessageOrderStatisticTable: Table { - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) } /*private func update(peerId: PeerId, tagMask: MessageTags, id: Int32, count: Int) { @@ -16,14 +16,14 @@ class MessageOrderStatisticTable: Table { while (idx <= 1000000) { key.setInt32(8 + 4, value: idx) var value: Int32 = 0 - if let data = self.valueBox.get(self.tableId, key: key) { + if let data = self.valueBox.get(self.table, key: key) { data.read(&value, offset: 0, length: 4) } if value == 0 { - self.valueBox.remove(self.tableId, key: key) + self.valueBox.remove(self.table, key: key) } else { writeValue = value - self.valueBox.set(self.tableId, key: key, value: buffer) + self.valueBox.set(self.table, key: key, value: buffer) } idx += idx & -idx } @@ -39,7 +39,7 @@ class MessageOrderStatisticTable: Table { while (idx > 0) { key.setInt32(8, value: idx) var value: Int32 = 0 - if let data = self.valueBox.get(self.tableId, key: key) { + if let data = self.valueBox.get(self.table, key: key) { data.read(&value, offset: 0, length: 4) } sum += value diff --git a/Postbox/PeerChatInterfaceStateTable.swift b/Postbox/PeerChatInterfaceStateTable.swift index e92e1288a7..a102bcb205 100644 --- a/Postbox/PeerChatInterfaceStateTable.swift +++ b/Postbox/PeerChatInterfaceStateTable.swift @@ -1,6 +1,10 @@ import Foundation final class PeerChatInterfaceStateTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private var states: [PeerId: PeerChatInterfaceState?] = [:] private var peerIdsWithUpdatedStates = Set() @@ -14,7 +18,7 @@ final class PeerChatInterfaceStateTable: Table { func get(_ peerId: PeerId) -> PeerChatInterfaceState? { if let cachedValue = self.states[peerId] { return cachedValue - } else if let value = self.valueBox.get(self.tableId, key: self.key(peerId, sharedKey: self.sharedKey)), let state = Decoder(buffer: value).decodeRootObject() as? PeerChatInterfaceState { + } else if let value = self.valueBox.get(self.table, key: self.key(peerId, sharedKey: self.sharedKey)), let state = Decoder(buffer: value).decodeRootObject() as? PeerChatInterfaceState { self.states[peerId] = state return state } else { @@ -62,9 +66,9 @@ final class PeerChatInterfaceStateTable: Table { if let state = state { sharedEncoder.reset() sharedEncoder.encodeRootObject(state) - self.valueBox.set(self.tableId, key: self.key(peerId, sharedKey: self.sharedKey), value: sharedEncoder.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(peerId, sharedKey: self.sharedKey), value: sharedEncoder.readBufferNoCopy()) } else { - self.valueBox.remove(self.tableId, key: self.key(peerId, sharedKey: self.sharedKey)) + self.valueBox.remove(self.table, key: self.key(peerId, sharedKey: self.sharedKey)) } } } diff --git a/Postbox/PeerChatStateTable.swift b/Postbox/PeerChatStateTable.swift index e60e622ff0..6bf55bbc83 100644 --- a/Postbox/PeerChatStateTable.swift +++ b/Postbox/PeerChatStateTable.swift @@ -1,6 +1,10 @@ import Foundation final class PeerChatStateTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private var cachedPeerChatStates: [PeerId: Coding?] = [:] private var updatedPeerIds = Set() @@ -15,7 +19,7 @@ final class PeerChatStateTable: Table { if let state = self.cachedPeerChatStates[id] { return state } else { - if let value = self.valueBox.get(self.tableId, key: self.key(id)), let state = Decoder(buffer: value).decodeRootObject() { + if let value = self.valueBox.get(self.table, key: self.key(id)), let state = Decoder(buffer: value).decodeRootObject() { self.cachedPeerChatStates[id] = state return state } else { @@ -41,9 +45,9 @@ final class PeerChatStateTable: Table { if let wrappedState = self.cachedPeerChatStates[id], let state = wrappedState { sharedEncoder.reset() sharedEncoder.encodeRootObject(state) - self.valueBox.set(self.tableId, key: self.key(id), value: sharedEncoder.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(id), value: sharedEncoder.readBufferNoCopy()) } else { - self.valueBox.remove(self.tableId, key: self.key(id)) + self.valueBox.remove(self.table, key: self.key(id)) } } self.updatedPeerIds.removeAll() diff --git a/Postbox/PeerChatTopTaggedMessageIds.swift b/Postbox/PeerChatTopTaggedMessageIds.swift index f7ac6d0dd5..2b3d3a9c7b 100644 --- a/Postbox/PeerChatTopTaggedMessageIds.swift +++ b/Postbox/PeerChatTopTaggedMessageIds.swift @@ -1,6 +1,10 @@ import Foundation final class PeerChatTopTaggedMessageIdsTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + private var cachedTopIds: [PeerId: [MessageId.Namespace: MessageId?]] = [:] private var updatedPeerIds = Set() diff --git a/Postbox/PeerNotificationSettingsTable.swift b/Postbox/PeerNotificationSettingsTable.swift index 21a7c891d7..00d8547b1f 100644 --- a/Postbox/PeerNotificationSettingsTable.swift +++ b/Postbox/PeerNotificationSettingsTable.swift @@ -1,16 +1,16 @@ import Foundation final class PeerNotificationSettingsTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private let sharedEncoder = Encoder() private let sharedKey = ValueBoxKey(length: 8) private var cachedSettings: [PeerId: PeerNotificationSettings] = [:] private var updatedPeerIds = Set() - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) - } - private func key(_ id: PeerId) -> ValueBoxKey { self.sharedKey.setInt64(0, value: id.toInt64()) return self.sharedKey @@ -25,7 +25,7 @@ final class PeerNotificationSettingsTable: Table { if let settings = self.cachedSettings[id] { return settings } - if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let value = self.valueBox.get(self.table, key: self.key(id)) { if let settings = Decoder(buffer: value).decodeRootObject() as? PeerNotificationSettings { self.cachedSettings[id] = settings return settings @@ -45,7 +45,7 @@ final class PeerNotificationSettingsTable: Table { self.sharedEncoder.reset() self.sharedEncoder.encodeRootObject(settings) - self.valueBox.set(self.tableId, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) } } diff --git a/Postbox/PeerPresenceTable.swift b/Postbox/PeerPresenceTable.swift index 18678c8436..80dca619eb 100644 --- a/Postbox/PeerPresenceTable.swift +++ b/Postbox/PeerPresenceTable.swift @@ -1,16 +1,16 @@ import Foundation final class PeerPresenceTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private let sharedEncoder = Encoder() private let sharedKey = ValueBoxKey(length: 8) private var cachedPresences: [PeerId: PeerPresence] = [:] private var updatedPeerIds = Set() - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) - } - private func key(_ id: PeerId) -> ValueBoxKey { self.sharedKey.setInt64(0, value: id.toInt64()) return self.sharedKey @@ -25,7 +25,7 @@ final class PeerPresenceTable: Table { if let presence = self.cachedPresences[id] { return presence } - if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let value = self.valueBox.get(self.table, key: self.key(id)) { if let presence = Decoder(buffer: value).decodeRootObject() as? PeerPresence { self.cachedPresences[id] = presence return presence @@ -45,7 +45,7 @@ final class PeerPresenceTable: Table { self.sharedEncoder.reset() self.sharedEncoder.encodeRootObject(presence) - self.valueBox.set(self.tableId, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) } } diff --git a/Postbox/PeerTable.swift b/Postbox/PeerTable.swift index 4b4b12b91b..5439cd664a 100644 --- a/Postbox/PeerTable.swift +++ b/Postbox/PeerTable.swift @@ -1,16 +1,16 @@ import Foundation final class PeerTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + private let sharedEncoder = Encoder() private let sharedKey = ValueBoxKey(length: 8) private var cachedPeers: [PeerId: Peer] = [:] private var updatedPeerIds = Set() - override init(valueBox: ValueBox, tableId: Int32) { - super.init(valueBox: valueBox, tableId: tableId) - } - private func key(_ id: PeerId) -> ValueBoxKey { self.sharedKey.setInt64(0, value: id.toInt64()) return self.sharedKey @@ -25,7 +25,7 @@ final class PeerTable: Table { if let peer = self.cachedPeers[id] { return peer } - if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + if let value = self.valueBox.get(self.table, key: self.key(id)) { if let peer = Decoder(buffer: value).decodeRootObject() as? Peer { self.cachedPeers[id] = peer return peer @@ -45,7 +45,7 @@ final class PeerTable: Table { self.sharedEncoder.reset() self.sharedEncoder.encodeRootObject(peer) - self.valueBox.set(self.tableId, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 7612ea2b8c..351051a8fa 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -161,6 +161,18 @@ public final class Modifier { } return Set() } + + public func storedMessageId(peerId: PeerId, namespace: MessageId.Namespace, timestamp: Int32) -> MessageId? { + return self.postbox?.storedMessageId(peerId: peerId, namespace: namespace, timestamp: timestamp) + } + + public func putItemCacheEntry(id: ItemCacheEntryId, entry: Coding, collectionSpec: ItemCacheCollectionSpec) { + self.postbox?.putItemCacheEntry(id: id, entry: entry, collectionSpec: collectionSpec) + } + + public func retrieveItemCacheEntry(id: ItemCacheEntryId) -> Coding? { + return self.postbox?.retrieveItemCacheEntry(id: id) + } } fileprivate class PipeNotifier: NSObject { @@ -240,7 +252,6 @@ public final class Postbox { var messageHistoryIndexTable: MessageHistoryIndexTable! var messageHistoryTable: MessageHistoryTable! var mediaTable: MessageMediaTable! - var mediaCleanupTable: MediaCleanupTable! var chatListIndexTable: ChatListIndexTable! var chatListTable: ChatListTable! var messageHistoryMetadataTable: MessageHistoryMetadataTable! @@ -253,6 +264,8 @@ public final class Postbox { var itemCollectionInfoTable: ItemCollectionInfoTable! var itemCollectionItemTable: ItemCollectionItemTable! var peerChatInterfaceStateTable: PeerChatInterfaceStateTable! + var itemCacheMetaTable: ItemCacheMetaTable! + var itemCacheTable: ItemCacheTable! //temporary var peerRatingTable: RatingTable! @@ -332,39 +345,40 @@ public final class Postbox { self.valueBox = SqliteValueBox(basePath: self.basePath + "/db", queue: self.queue) - self.metadataTable = MetadataTable(valueBox: self.valueBox, tableId: 0) + self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 15 + let currentUserVersion: Int32 = 16 if userVersion != currentUserVersion { self.valueBox.drop() self.metadataTable.setUserVersion(currentUserVersion) } - self.keychainTable = KeychainTable(valueBox: self.valueBox, tableId: 1) - self.peerTable = PeerTable(valueBox: self.valueBox, tableId: 2) - self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox, tableId: 3, namespace: self.globalMessageIdsNamespace) - self.messageHistoryMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox, tableId: 10) - self.messageHistoryUnsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox, tableId: 11) - self.messageHistoryTagsTable = MessageHistoryTagsTable(valueBox: self.valueBox, tableId: 12) - self.messageHistoryIndexTable = MessageHistoryIndexTable(valueBox: self.valueBox, tableId: 4, globalMessageIdsTable: self.globalMessageIdsTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) - self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox, tableId: 5) - self.mediaTable = MessageMediaTable(valueBox: self.valueBox, tableId: 6, mediaCleanupTable: self.mediaCleanupTable) - self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox, tableId: 14) - self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox, tableId: 15) - self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, tableId: 7, messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable!) - self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, tableId: 8) - self.chatListTable = ChatListTable(valueBox: self.valueBox, tableId: 9, indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) - self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, tableId: 13) - self.contactsTable = ContactTable(valueBox: self.valueBox, tableId: 16) - self.peerRatingTable = RatingTable(valueBox: self.valueBox, tableId: 17) - self.cachedPeerDataTable = CachedPeerDataTable(valueBox: self.valueBox, tableId: 18) - self.peerNotificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox, tableId: 19) - self.peerPresenceTable = PeerPresenceTable(valueBox: self.valueBox, tableId: 20) - self.itemCollectionInfoTable = ItemCollectionInfoTable(valueBox: self.valueBox, tableId: 21) - self.itemCollectionItemTable = ItemCollectionItemTable(valueBox: self.valueBox, tableId: 22) - self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox, tableId: 23) + self.keychainTable = KeychainTable(valueBox: self.valueBox, table: KeychainTable.tableSpec(1)) + self.peerTable = PeerTable(valueBox: self.valueBox, table: PeerTable.tableSpec(2)) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox, table: GlobalMessageIdsTable.tableSpec(3), namespace: self.globalMessageIdsNamespace) + self.messageHistoryMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox, table: MessageHistoryMetadataTable.tableSpec(10)) + self.messageHistoryUnsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox, table: MessageHistoryUnsentTable.tableSpec(11)) + self.messageHistoryTagsTable = MessageHistoryTagsTable(valueBox: self.valueBox, table: MessageHistoryTagsTable.tableSpec(12)) + self.messageHistoryIndexTable = MessageHistoryIndexTable(valueBox: self.valueBox, table: MessageHistoryIndexTable.tableSpec(4), globalMessageIdsTable: self.globalMessageIdsTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox, table: MessageMediaTable.tableSpec(6)) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox, table: MessageHistoryReadStateTable.tableSpec(14)) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox, table: MessageHistorySynchronizeReadStateTable.tableSpec(15)) + self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8)) + self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) + self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, table: PeerChatStateTable.tableSpec(13)) + self.contactsTable = ContactTable(valueBox: self.valueBox, table: ContactTable.tableSpec(16)) + self.peerRatingTable = RatingTable(valueBox: self.valueBox, table: RatingTable.tableSpec(17)) + self.cachedPeerDataTable = CachedPeerDataTable(valueBox: self.valueBox, table: CachedPeerDataTable.tableSpec(18)) + self.peerNotificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox, table: PeerNotificationSettingsTable.tableSpec(19)) + self.peerPresenceTable = PeerPresenceTable(valueBox: self.valueBox, table: PeerPresenceTable.tableSpec(20)) + self.itemCollectionInfoTable = ItemCollectionInfoTable(valueBox: self.valueBox, table: ItemCollectionInfoTable.tableSpec(21)) + self.itemCollectionItemTable = ItemCollectionItemTable(valueBox: self.valueBox, table: ItemCollectionItemTable.tableSpec(22)) + self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox, table: PeerChatInterfaceStateTable.tableSpec(23)) + self.itemCacheMetaTable = ItemCacheMetaTable(valueBox: self.valueBox, table: ItemCacheMetaTable.tableSpec(24)) + self.itemCacheTable = ItemCacheTable(valueBox: self.valueBox, table: ItemCacheTable.tableSpec(25)) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -373,7 +387,6 @@ public final class Postbox { self.tables.append(self.messageHistoryUnsentTable) self.tables.append(self.messageHistoryTagsTable) self.tables.append(self.messageHistoryIndexTable) - self.tables.append(self.mediaCleanupTable) self.tables.append(self.mediaTable) self.tables.append(self.readStateTable) self.tables.append(self.synchronizeReadStateTable) @@ -389,6 +402,8 @@ public final class Postbox { self.tables.append(self.itemCollectionInfoTable) self.tables.append(self.itemCollectionItemTable) self.tables.append(self.peerChatInterfaceStateTable) + self.tables.append(self.itemCacheMetaTable) + self.tables.append(self.itemCacheTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() @@ -908,6 +923,18 @@ public final class Postbox { return filteredIds } + fileprivate func storedMessageId(peerId: PeerId, namespace: MessageId.Namespace, timestamp: Int32) -> MessageId? { + return self.messageHistoryTable.findMessageId(peerId: peerId, namespace: namespace, timestamp: timestamp) + } + + fileprivate func putItemCacheEntry(id: ItemCacheEntryId, entry: Coding, collectionSpec: ItemCacheCollectionSpec) { + self.itemCacheTable.put(id: id, entry: entry, metaTable: self.itemCacheMetaTable) + } + + fileprivate func retrieveItemCacheEntry(id: ItemCacheEntryId) -> Coding? { + return self.itemCacheTable.retrieve(id: id, metaTable: self.itemCacheMetaTable) + } + public func modify(userInteractive: Bool = false, _ f: @escaping(Modifier) -> T) -> Signal { return Signal { subscriber in let f: () -> Void = { diff --git a/Postbox/RatingTable.swift b/Postbox/RatingTable.swift index f91d1b08b7..42a1421197 100644 --- a/Postbox/RatingTable.swift +++ b/Postbox/RatingTable.swift @@ -23,6 +23,10 @@ extension PeerId: RatingTableItem { } final class RatingTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + private var items: [T]? func get() -> [T] { @@ -35,7 +39,7 @@ final class RatingTable: Table { memset(upperBound.memory, 0xff, upperBound.length) var result: [T] = [] - self.valueBox.range(self.tableId, start: lowerBound, end: upperBound, keys: { key in + self.valueBox.range(self.table, start: lowerBound, end: upperBound, keys: { key in result.append(T.fromKey(key: key)) return true }, limit: 0) @@ -51,19 +55,19 @@ final class RatingTable: Table { let upperBound = T.emptyKey() memset(lowerBound.memory, 0, lowerBound.length) memset(upperBound.memory, 0xff, upperBound.length) - self.valueBox.range(self.tableId, start: lowerBound, end: upperBound, keys: { key in + self.valueBox.range(self.table, start: lowerBound, end: upperBound, keys: { key in keys.append(key) return true }, limit: 0) for key in keys { - self.valueBox.remove(self.tableId, key: key) + self.valueBox.remove(self.table, key: key) } let sharedKey = T.emptyKey() var index: Int32 = 0 for item in items { - self.valueBox.set(self.tableId, key: item.ratingKey(rating: index, sharedKey: sharedKey), value: MemoryBuffer()) + self.valueBox.set(self.table, key: item.ratingKey(rating: index, sharedKey: sharedKey), value: MemoryBuffer()) index += 1 } diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 12ffba1c27..f1a99b436d 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -6,6 +6,15 @@ import sqlcipher import SwiftSignalKit #endif +private func checkTableKey(_ table: ValueBoxTable, _ key: ValueBoxKey) { + switch table.keyType { + case .binary: + break + case .int64: + assert(key.length == 8) + } +} + private struct SqlitePreparedStatement { let statement: OpaquePointer? @@ -13,6 +22,10 @@ private struct SqlitePreparedStatement { sqlite3_bind_blob(statement, Int32(index), data, Int32(length), nil) } + func bind(_ index: Int, number: Int64) { + sqlite3_bind_int64(statement, Int32(index), number) + } + func bindNull(_ index: Int) { sqlite3_bind_null(statement, Int32(index)) } @@ -21,10 +34,6 @@ private struct SqlitePreparedStatement { sqlite3_bind_int(statement, Int32(index), number) } - func bind(_ index: Int, number: Int64) { - sqlite3_bind_int64(statement, Int32(index), number) - } - func reset() { sqlite3_reset(statement) sqlite3_clear_bindings(statement) @@ -65,12 +74,12 @@ private struct SqlitePreparedStatement { } } -public final class SqliteValueBox: ValueBox { +final class SqliteValueBox: ValueBox { private let lock = NSRecursiveLock() private let basePath: String private var database: Database! - private var tables = Set() + private var tables: [Int32: ValueBoxTable] = [:] private var getStatements: [Int32 : SqlitePreparedStatement] = [:] private var rangeKeyAscStatementsLimit: [Int32 : SqlitePreparedStatement] = [:] private var rangeKeyAscStatementsNoLimit: [Int32 : SqlitePreparedStatement] = [:] @@ -83,6 +92,7 @@ public final class SqliteValueBox: ValueBox { private var existsStatements: [Int32 : SqlitePreparedStatement] = [:] private var updateStatements: [Int32 : SqlitePreparedStatement] = [:] private var insertStatements: [Int32 : SqlitePreparedStatement] = [:] + private var insertOrReplaceStatements: [Int32 : SqlitePreparedStatement] = [:] private var deleteStatements: [Int32 : SqlitePreparedStatement] = [:] private var readQueryTime: CFAbsoluteTime = 0.0 @@ -137,13 +147,14 @@ public final class SqliteValueBox: ValueBox { sqlite3_busy_timeout(database.handle, 10000000) let result = self.getUserVersion(database) - if result != 1 { - database.execute("PRAGMA user_version=1") - database.execute("CREATE TABLE __meta_tables (name INTEGER)") + if result != 2 { + database.execute("PRAGMA user_version=2") + database.execute("DROP TABLE IF EXISTS __meta_tables") + database.execute("CREATE TABLE __meta_tables (name INTEGER, keyType INTEGER)") } - for table in self.listTables(database).map({Int32($0)}) { - self.tables.insert(table) + for table in self.listTables(database) { + self.tables[table.id] = table } lock.unlock() @@ -195,279 +206,394 @@ public final class SqliteValueBox: ValueBox { return value } - private func listTables(_ database: Database) -> [Int64] { + private func listTables(_ database: Database) -> [ValueBoxTable] { assert(self.queue.isCurrent()) var statement: OpaquePointer? = nil - sqlite3_prepare_v2(database.handle, "SELECT name FROM __meta_tables", -1, &statement, nil) + sqlite3_prepare_v2(database.handle, "SELECT name, keyType FROM __meta_tables", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - var tables: [Int64] = [] + var tables: [ValueBoxTable] = [] while preparedStatement.step() { let value = preparedStatement.int64At(0) - tables.append(value) + let keyType = preparedStatement.int64At(1) + tables.append(ValueBoxTable(id: Int32(value), keyType: ValueBoxKeyType(rawValue: Int32(keyType))!)) } preparedStatement.destroy() return tables } - private func getStatement(_ table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + private func checkTable(_ table: ValueBoxTable) { + if let currentTable = self.tables[table.id] { + precondition(currentTable.keyType == table.keyType) + } else { + switch table.keyType { + case .binary: + self.database.execute("CREATE TABLE t\(table.id) (key BLOB, value BLOB)") + self.database.execute("CREATE INDEX t\(table.id)_key ON t\(table.id) (key)") + case .int64: + self.database.execute("CREATE TABLE t\(table.id) (key INTEGER PRIMARY KEY, value BLOB)") + } + self.tables[table.id] = table + self.database.execute("INSERT INTO __meta_tables(name, keyType) VALUES (\(table.id), \(table.keyType.rawValue))") + } + } + + private func getStatement(_ table: ValueBoxTable, key: ValueBoxKey) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, key) + let resultStatement: SqlitePreparedStatement - if let statement = self.getStatements[table] { + if let statement = self.getStatements[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT value FROM t\(table) WHERE key=?", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT value FROM t\(table.id) WHERE key=?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.getStatements[table] = preparedStatement + self.getStatements[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: key.memory, length: key.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: key.memory, length: key.length) + case .int64: + resultStatement.bind(1, number: key.getInt64(0)) + } return resultStatement } - private func rangeKeyAscStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + private func rangeKeyAscStatementLimit(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, start) + checkTableKey(table, end) + let resultStatement: SqlitePreparedStatement - if let statement = self.rangeKeyAscStatementsLimit[table] { + if let statement = self.rangeKeyAscStatementsLimit[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.rangeKeyAscStatementsLimit[table] = preparedStatement + self.rangeKeyAscStatementsLimit[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: start.memory, length: start.length) - resultStatement.bind(2, data: end.memory, length: end.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + case .int64: + resultStatement.bind(1, number: start.getInt64(0)) + resultStatement.bind(2, number: end.getInt64(0)) + } resultStatement.bind(3, number: Int32(limit)) return resultStatement } - private func rangeKeyAscStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> + private func rangeKeyAscStatementNoLimit(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { - assert(self.queue.isCurrent()) + assert(self.queue.isCurrent()) + checkTableKey(table, start) + checkTableKey(table, end) + let resultStatement: SqlitePreparedStatement - if let statement = self.rangeKeyAscStatementsNoLimit[table] { + if let statement = self.rangeKeyAscStatementsNoLimit[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.rangeKeyAscStatementsNoLimit[table] = preparedStatement + self.rangeKeyAscStatementsNoLimit[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: start.memory, length: start.length) - resultStatement.bind(2, data: end.memory, length: end.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + case .int64: + resultStatement.bind(1, number: start.getInt64(0)) + resultStatement.bind(2, number: end.getInt64(0)) + } return resultStatement } - private func rangeKeyDescStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + private func rangeKeyDescStatementLimit(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { let resultStatement: SqlitePreparedStatement + checkTableKey(table, start) + checkTableKey(table, end) - if let statement = self.rangeKeyDescStatementsLimit[table] { + if let statement = self.rangeKeyDescStatementsLimit[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.rangeKeyDescStatementsLimit[table] = preparedStatement + self.rangeKeyDescStatementsLimit[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: start.memory, length: start.length) - resultStatement.bind(2, data: end.memory, length: end.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + case .int64: + resultStatement.bind(1, number: start.getInt64(0)) + resultStatement.bind(2, number: end.getInt64(0)) + } resultStatement.bind(3, number: Int32(limit)) return resultStatement } - private func rangeKeyDescStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + private func rangeKeyDescStatementNoLimit(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement + checkTableKey(table, start) + checkTableKey(table, end) - if let statement = self.rangeKeyDescStatementsNoLimit[table] { + if let statement = self.rangeKeyDescStatementsNoLimit[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.rangeKeyDescStatementsNoLimit[table] = preparedStatement + self.rangeKeyDescStatementsNoLimit[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: start.memory, length: start.length) - resultStatement.bind(2, data: end.memory, length: end.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + case .int64: + resultStatement.bind(1, number: start.getInt64(0)) + resultStatement.bind(2, number: end.getInt64(0)) + } return resultStatement } - private func rangeValueAscStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + private func rangeValueAscStatementLimit(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, start) + checkTableKey(table, end) + let resultStatement: SqlitePreparedStatement - if let statement = self.rangeValueAscStatementsLimit[table] { + if let statement = self.rangeValueAscStatementsLimit[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.rangeValueAscStatementsLimit[table] = preparedStatement + self.rangeValueAscStatementsLimit[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: start.memory, length: start.length) - resultStatement.bind(2, data: end.memory, length: end.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + case .int64: + resultStatement.bind(1, number: start.getInt64(0)) + resultStatement.bind(2, number: end.getInt64(0)) + } resultStatement.bind(3, number: Int32(limit)) return resultStatement } - private func rangeValueAscStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + private func rangeValueAscStatementNoLimit(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, start) + checkTableKey(table, end) + let resultStatement: SqlitePreparedStatement - if let statement = self.rangeValueAscStatementsNoLimit[table] { + if let statement = self.rangeValueAscStatementsNoLimit[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.rangeValueAscStatementsNoLimit[table] = preparedStatement + self.rangeValueAscStatementsNoLimit[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: start.memory, length: start.length) - resultStatement.bind(2, data: end.memory, length: end.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + case .int64: + resultStatement.bind(1, number: start.getInt64(0)) + resultStatement.bind(2, number: end.getInt64(0)) + } return resultStatement } - private func rangeValueDescStatementLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + private func rangeValueDescStatementLimit(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, start) + checkTableKey(table, end) + let resultStatement: SqlitePreparedStatement - if let statement = self.rangeValueDescStatementsLimit[table] { + if let statement = self.rangeValueDescStatementsLimit[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.rangeValueDescStatementsLimit[table] = preparedStatement + self.rangeValueDescStatementsLimit[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: start.memory, length: start.length) - resultStatement.bind(2, data: end.memory, length: end.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + case .int64: + resultStatement.bind(1, number: start.getInt64(0)) + resultStatement.bind(2, number: end.getInt64(0)) + } resultStatement.bind(3, number: Int32(limit)) return resultStatement } - private func rangeValueDescStatementNoLimit(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { + private func rangeValueDescStatementNoLimit(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, start) + checkTableKey(table, end) + let resultStatement: SqlitePreparedStatement - if let statement = self.rangeKeyDescStatementsNoLimit[table] { + if let statement = self.rangeKeyDescStatementsNoLimit[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.rangeValueDescStatementsNoLimit[table] = preparedStatement + self.rangeValueDescStatementsNoLimit[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: start.memory, length: start.length) - resultStatement.bind(2, data: end.memory, length: end.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: start.memory, length: start.length) + resultStatement.bind(2, data: end.memory, length: end.length) + case .int64: + resultStatement.bind(1, number: start.getInt64(0)) + resultStatement.bind(2, number: end.getInt64(0)) + } return resultStatement } - private func existsStatement(_ table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + private func existsStatement(_ table: ValueBoxTable, key: ValueBoxKey) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, key) + let resultStatement: SqlitePreparedStatement - if let statement = self.existsStatements[table] { + if let statement = self.existsStatements[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT rowid FROM t\(table) WHERE key=?", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "SELECT rowid FROM t\(table.id) WHERE key=?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.existsStatements[table] = preparedStatement + self.existsStatements[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: key.memory, length: key.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: key.memory, length: key.length) + case .int64: + resultStatement.bind(1, number: key.getInt64(0)) + } return resultStatement } - private func updateStatement(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { + private func updateStatement(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, key) + let resultStatement: SqlitePreparedStatement - if let statement = self.updateStatements[table] { + if let statement = self.updateStatements[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "UPDATE t\(table) SET value=? WHERE key=?", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "UPDATE t\(table.id) SET value=? WHERE key=?", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.updateStatements[table] = preparedStatement + self.updateStatements[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() resultStatement.bind(1, data: value.memory, length: value.length) - resultStatement.bind(2, data: key.memory, length: key.length) + switch table.keyType { + case .binary: + resultStatement.bind(2, data: key.memory, length: key.length) + case .int64: + resultStatement.bind(2, number: key.getInt64(0)) + } return resultStatement } - private func insertStatement(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { + private func insertStatement(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, key) + let resultStatement: SqlitePreparedStatement - if let statement = self.insertStatements[table] { + if let statement = self.insertStatements[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table) (key, value) VALUES(?, ?)", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table.id) (key, value) VALUES(?, ?)", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.insertStatements[table] = preparedStatement + self.insertStatements[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: key.memory, length: key.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: key.memory, length: key.length) + case .int64: + resultStatement.bind(1, number: key.getInt64(0)) + } if value.length == 0 { resultStatement.bindNull(2) } else { @@ -477,31 +603,71 @@ public final class SqliteValueBox: ValueBox { return resultStatement } - private func deleteStatement(_ table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { + private func insertOrReplaceStatement(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) + checkTableKey(table, key) + let resultStatement: SqlitePreparedStatement - if let statement = self.deleteStatements[table] { + if let statement = self.insertOrReplaceStatements[table.id] { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "DELETE FROM t\(table) WHERE key=?", -1, &statement, nil) + sqlite3_prepare_v2(self.database.handle, "INSERT OR REPLACE INTO t\(table.id) (key, value) VALUES(?, ?)", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) - self.deleteStatements[table] = preparedStatement + self.insertOrReplaceStatements[table.id] = preparedStatement resultStatement = preparedStatement } resultStatement.reset() - resultStatement.bind(1, data: key.memory, length: key.length) + switch table.keyType { + case .binary: + resultStatement.bind(1, data: key.memory, length: key.length) + case .int64: + resultStatement.bind(1, number: key.getInt64(0)) + } + if value.length == 0 { + resultStatement.bindNull(2) + } else { + resultStatement.bind(2, data: value.memory, length: value.length) + } return resultStatement } - public func get(_ table: Int32, key: ValueBoxKey) -> ReadBuffer? { + private func deleteStatement(_ table: ValueBoxTable, key: ValueBoxKey) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) + checkTableKey(table, key) + + let resultStatement: SqlitePreparedStatement + + if let statement = self.deleteStatements[table.id] { + resultStatement = statement + } else { + var statement: OpaquePointer? = nil + sqlite3_prepare_v2(self.database.handle, "DELETE FROM t\(table.id) WHERE key=?", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.deleteStatements[table.id] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + switch table.keyType { + case .binary: + resultStatement.bind(1, data: key.memory, length: key.length) + case .int64: + resultStatement.bind(1, number: key.getInt64(0)) + } + + return resultStatement + } + + public func get(_ table: ValueBoxTable, key: ValueBoxKey) -> ReadBuffer? { assert(self.queue.isCurrent()) let startTime = CFAbsoluteTimeGetCurrent() - if self.tables.contains(table) { + if let _ = self.tables[table.id] { let statement = self.getStatement(table, key: key) var buffer: ReadBuffer? @@ -521,7 +687,7 @@ public final class SqliteValueBox: ValueBox { return nil } - public func exists(_ table: Int32, key: ValueBoxKey) -> Bool { + public func exists(_ table: ValueBoxTable, key: ValueBoxKey) -> Bool { assert(self.queue.isCurrent()) if let _ = self.get(table, key: key) { return true @@ -529,13 +695,13 @@ public final class SqliteValueBox: ValueBox { return false } - public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, values: @noescape(ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { + public func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) { assert(self.queue.isCurrent()) if start == end { return } - if self.tables.contains(table) { + if let _ = self.tables[table.id] { let statement: SqlitePreparedStatement var startTime = CFAbsoluteTimeGetCurrent() @@ -577,9 +743,9 @@ public final class SqliteValueBox: ValueBox { } } - public func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: @noescape(ValueBoxKey) -> Bool, limit: Int) { + public func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, keys: (ValueBoxKey) -> Bool, limit: Int) { assert(self.queue.isCurrent()) - if self.tables.contains(table) { + if let _ = self.tables[table.id] { let statement: SqlitePreparedStatement var startTime = CFAbsoluteTimeGetCurrent() @@ -620,43 +786,44 @@ public final class SqliteValueBox: ValueBox { } } - public func set(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) { + public func set(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) { assert(self.queue.isCurrent()) - if !self.tables.contains(table) { - self.database.execute("CREATE TABLE t\(table) (key BLOB, value BLOB)") - self.database.execute("CREATE INDEX t\(table)_key ON t\(table) (key)") - self.tables.insert(table) - self.database.execute("INSERT INTO __meta_tables(name) VALUES (\(table))") - } + self.checkTable(table) let startTime = CFAbsoluteTimeGetCurrent() - var exists = false - let existsStatement = self.existsStatement(table, key: key) - if existsStatement.step() { - exists = true - } - existsStatement.reset() - - if exists { - let statement = self.updateStatement(table, key: key, value: value) + if case .int64 = table.keyType { + let statement = self.insertOrReplaceStatement(table, key: key, value: value) while statement.step() { } statement.reset() } else { - let statement = self.insertStatement(table, key: key, value: value) - while statement.step() { + var exists = false + let existsStatement = self.existsStatement(table, key: key) + if existsStatement.step() { + exists = true + } + existsStatement.reset() + + if exists { + let statement = self.updateStatement(table, key: key, value: value) + while statement.step() { + } + statement.reset() + } else { + let statement = self.insertStatement(table, key: key, value: value) + while statement.step() { + } + statement.reset() } - statement.reset() } self.writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } - public func remove(_ table: Int32, key: ValueBoxKey) { + public func remove(_ table: ValueBoxTable, key: ValueBoxKey) { assert(self.queue.isCurrent()) - if self.tables.contains(table) { - + if let _ = self.tables[table.id] { let startTime = CFAbsoluteTimeGetCurrent() let statement = self.deleteStatement(table, key: key) @@ -730,6 +897,11 @@ public final class SqliteValueBox: ValueBox { } self.insertStatements.removeAll() + for (_, statement) in self.insertOrReplaceStatements { + statement.destroy() + } + self.insertOrReplaceStatements.removeAll() + for (_, statement) in self.deleteStatements { statement.destroy() } diff --git a/Postbox/Table.swift b/Postbox/Table.swift index 08edb399ff..9f51743dab 100644 --- a/Postbox/Table.swift +++ b/Postbox/Table.swift @@ -2,11 +2,11 @@ import Foundation class Table { final let valueBox: ValueBox - final let tableId: Int32 + final let table: ValueBoxTable - init(valueBox: ValueBox, tableId: Int32) { + init(valueBox: ValueBox, table: ValueBoxTable) { self.valueBox = valueBox - self.tableId = tableId + self.table = table } func clearMemoryCache() { diff --git a/Postbox/ValueBox.swift b/Postbox/ValueBox.swift index 4c6abb138f..1de04fe06f 100644 --- a/Postbox/ValueBox.swift +++ b/Postbox/ValueBox.swift @@ -1,17 +1,27 @@ import Foundation -public protocol ValueBox { +enum ValueBoxKeyType: Int32 { + case binary + case int64 +} + +struct ValueBoxTable { + let id: Int32 + let keyType: ValueBoxKeyType +} + +protocol ValueBox { func begin() func commit() func beginStats() func endStats() - func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, values: @noescape(ValueBoxKey, ReadBuffer) -> Bool, limit: Int) - func range(_ table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: @noescape(ValueBoxKey) -> Bool, limit: Int) - func get(_ table: Int32, key: ValueBoxKey) -> ReadBuffer? - func exists(_ table: Int32, key: ValueBoxKey) -> Bool - func set(_ table: Int32, key: ValueBoxKey, value: MemoryBuffer) - func remove(_ table: Int32, key: ValueBoxKey) + func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, values: @noescape(ValueBoxKey, ReadBuffer) -> Bool, limit: Int) + func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, keys: @noescape(ValueBoxKey) -> Bool, limit: Int) + func get(_ table: ValueBoxTable, key: ValueBoxKey) -> ReadBuffer? + func exists(_ table: ValueBoxTable, key: ValueBoxKey) -> Bool + func set(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) + func remove(_ table: ValueBoxTable, key: ValueBoxKey) func drop() } diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index f0e6bd6c36..3dfd380568 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -68,7 +68,6 @@ class ChatListTableTests: XCTestCase { var globalMessageIdsTable: GlobalMessageIdsTable? var indexTable: MessageHistoryIndexTable? var mediaTable: MessageMediaTable? - var mediaCleanupTable: MediaCleanupTable? var historyTable: MessageHistoryTable? var chatListIndexTable: ChatListIndexTable? var chatListTable: ChatListTable? @@ -93,19 +92,18 @@ class ChatListTableTests: XCTestCase { let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: []) - self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 7, namespace: namespace) - self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 8) - self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, tableId: 9) - self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, tableId: 10) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) - self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) - self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) - self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 11) - self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, tableId: 12) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) - self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, tableId: 5) - self.chatListTable = ChatListTable(valueBox: self.valueBox!, tableId: 6, indexTable: self.chatListIndexTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) - self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox!, tableId: 20) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, table: GlobalMessageIdsTable.tableSpec(7), namespace: namespace) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, table: MessageHistoryMetadataTable.tableSpec(8)) + self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, table: MessageHistoryUnsentTable.tableSpec(9)) + self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, table: MessageHistoryTagsTable.tableSpec(10)) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, table: MessageHistoryIndexTable.tableSpec(1), globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, table: MessageMediaTable.tableSpec(2)) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, table: MessageHistoryReadStateTable.tableSpec(11)) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, table: MessageHistorySynchronizeReadStateTable.tableSpec(12)) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, table: ChatListIndexTable.tableSpec(5)) + self.chatListTable = ChatListTable(valueBox: self.valueBox!, table: ChatListTable.tableSpec(6), indexTable: self.chatListIndexTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) + self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox!, table: PeerChatInterfaceStateTable.tableSpec(20)) } override func tearDown() { @@ -114,7 +112,6 @@ class ChatListTableTests: XCTestCase { self.historyTable = nil self.indexTable = nil self.mediaTable = nil - self.mediaCleanupTable = nil self.chatListIndexTable = nil self.chatListTable = nil diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index c39f0cb771..f55ee0490a 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -79,9 +79,9 @@ class MessageHistoryIndexTableTests: XCTestCase { let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: []) - self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 2, namespace: namespace) - self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 8) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, table: GlobalMessageIdsTable.tableSpec(2), namespace: namespace) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, table: MessageHistoryMetadataTable.tableSpec(8)) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, table: MessageHistoryIndexTable.tableSpec(1), globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) } override func tearDown() { diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index 7c6775adf7..153912cd12 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -203,7 +203,6 @@ class MessageHistoryTableTests: XCTestCase { var globalMessageIdsTable: GlobalMessageIdsTable? var indexTable: MessageHistoryIndexTable? var mediaTable: MessageMediaTable? - var mediaCleanupTable: MediaCleanupTable? var historyTable: MessageHistoryTable? var historyMetadataTable: MessageHistoryMetadataTable? var unsentTable: MessageHistoryUnsentTable? @@ -229,17 +228,16 @@ class MessageHistoryTableTests: XCTestCase { let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) - self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 5, namespace: namespace) - self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 7) - self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, tableId: 8) - self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, tableId: 9) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) - self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) - self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) - self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 10) - self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, tableId: 11) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) - self.peerTable = PeerTable(valueBox: self.valueBox!, tableId: 6) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, table: GlobalMessageIdsTable.tableSpec(5), namespace: namespace) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, table: MessageHistoryMetadataTable.tableSpec(7)) + self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, table: MessageHistoryUnsentTable.tableSpec(8)) + self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, table: MessageHistoryTagsTable.tableSpec(9)) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, table: MessageHistoryIndexTable.tableSpec(1), globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, table: MessageMediaTable.tableSpec(2)) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, table: MessageHistoryReadStateTable.tableSpec(10)) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, table: MessageHistorySynchronizeReadStateTable.tableSpec(11)) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.peerTable = PeerTable(valueBox: self.valueBox!, table: PeerTable.tableSpec(6)) self.peerTable!.set(peer) } @@ -249,7 +247,6 @@ class MessageHistoryTableTests: XCTestCase { self.historyTable = nil self.indexTable = nil self.mediaTable = nil - self.mediaCleanupTable = nil self.peerTable = nil self.historyMetadataTable = nil @@ -361,25 +358,6 @@ class MessageHistoryTableTests: XCTestCase { } } - private func expectCleanupMedia(_ media: [Media]) { - let actualMedia = self.mediaCleanupTable!.debugList() - var equal = true - if media.count != actualMedia.count { - equal = false - } else { - for i in 0 ..< media.count { - if !media[i].isEqual(actualMedia[i]) { - equal = false - break - } - } - } - - if !equal { - XCTFail("Expected\n\(media)\nActual\n\(actualMedia)") - } - } - func testInsertMessageIntoEmpty() { addMessage(100, 100, "t100") addMessage(200, 200, "t200") @@ -444,7 +422,6 @@ class MessageHistoryTableTests: XCTestCase { expectEntries([]) expectMedia([]) - expectCleanupMedia([media]) } func testRemoveOnlyReferenceToExternalMedia() { @@ -454,7 +431,6 @@ class MessageHistoryTableTests: XCTestCase { expectEntries([]) expectMedia([]) - expectCleanupMedia([media]) } func testRemoveReferenceToExternalMedia() { @@ -465,13 +441,11 @@ class MessageHistoryTableTests: XCTestCase { expectEntries([.Message(200, 200, "t200", [media], [])]) expectMedia([.Direct(media, 1)]) - expectCleanupMedia([]) removeMessages([200]) expectEntries([]) expectMedia([]) - expectCleanupMedia([media]) } func testAddHoleToEmpty() { diff --git a/PostboxTests/OrderStatisticTreeTests.swift b/PostboxTests/OrderStatisticTreeTests.swift index 2e17bf0b1e..3e40008b8b 100644 --- a/PostboxTests/OrderStatisticTreeTests.swift +++ b/PostboxTests/OrderStatisticTreeTests.swift @@ -71,7 +71,6 @@ class OrderStatisticTreeTests: XCTestCase { var globalMessageIdsTable: GlobalMessageIdsTable? var indexTable: MessageHistoryIndexTable? var mediaTable: MessageMediaTable? - var mediaCleanupTable: MediaCleanupTable? var historyTable: MessageHistoryTable? var historyMetadataTable: MessageHistoryMetadataTable? var unsentTable: MessageHistoryUnsentTable? @@ -89,16 +88,15 @@ class OrderStatisticTreeTests: XCTestCase { let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) - self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 5, namespace: namespace) - self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 7) - self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, tableId: 8) - self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, tableId: 9) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) - self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) - self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) - self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 10) - self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, tableId: 11) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, table: GlobalMessageIdsTable.tableSpec(5), namespace: namespace) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, table: MessageHistoryMetadataTable.tableSpec(7)) + self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, table: MessageHistoryUnsentTable.tableSpec(8)) + self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, table: MessageHistoryTagsTable.tableSpec(9)) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, table: MessageHistoryIndexTable.tableSpec(1), globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, table: MessageMediaTable.tableSpec(2)) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, table: MessageHistoryReadStateTable.tableSpec(10)) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, table: MessageHistorySynchronizeReadStateTable.tableSpec(11)) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) } override func tearDown() { @@ -107,7 +105,6 @@ class OrderStatisticTreeTests: XCTestCase { self.historyTable = nil self.indexTable = nil self.mediaTable = nil - self.mediaCleanupTable = nil self.peerTable = nil self.historyMetadataTable = nil diff --git a/PostboxTests/ReadStateTableTests.swift b/PostboxTests/ReadStateTableTests.swift index 8005f60b71..ddb5b1642a 100644 --- a/PostboxTests/ReadStateTableTests.swift +++ b/PostboxTests/ReadStateTableTests.swift @@ -71,7 +71,6 @@ class ReadStateTableTests: XCTestCase { var globalMessageIdsTable: GlobalMessageIdsTable? var indexTable: MessageHistoryIndexTable? var mediaTable: MessageMediaTable? - var mediaCleanupTable: MediaCleanupTable? var historyTable: MessageHistoryTable? var historyMetadataTable: MessageHistoryMetadataTable? var unsentTable: MessageHistoryUnsentTable? @@ -89,16 +88,15 @@ class ReadStateTableTests: XCTestCase { let seedConfiguration = SeedConfiguration(initializeChatListWithHoles: [], initializeMessageNamespacesWithHoles: [], existingMessageTags: [.First, .Second]) - self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, tableId: 5, namespace: namespace) - self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, tableId: 7) - self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, tableId: 8) - self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, tableId: 9) - self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1, globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) - self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) - self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) - self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, tableId: 10) - self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, tableId: 11) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox!, table: GlobalMessageIdsTable.tableSpec(5), namespace: namespace) + self.historyMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox!, table: MessageHistoryMetadataTable.tableSpec(7)) + self.unsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox!, table: MessageHistoryUnsentTable.tableSpec(8)) + self.tagsTable = MessageHistoryTagsTable(valueBox: self.valueBox!, table: MessageHistoryTagsTable.tableSpec(9)) + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, table: MessageHistoryIndexTable.tableSpec(1), globalMessageIdsTable: self.globalMessageIdsTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, table: MessageMediaTable.tableSpec(2)) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, table: MessageHistoryReadStateTable.tableSpec(10)) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, table: MessageHistorySynchronizeReadStateTable.tableSpec(11)) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) } override func tearDown() { @@ -107,7 +105,6 @@ class ReadStateTableTests: XCTestCase { self.historyTable = nil self.indexTable = nil self.mediaTable = nil - self.mediaCleanupTable = nil self.peerTable = nil self.historyMetadataTable = nil From a82e5f148abd6c86afd9a74b7e1e3f534bf1c9bb Mon Sep 17 00:00:00 2001 From: overtake Date: Fri, 18 Nov 2016 12:16:53 +0300 Subject: [PATCH 059/427] =?UTF-8?q?=F0=9F=98=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Postbox.xcodeproj/project.pbxproj | 30 ++----------------- .../xcschemes/Postbox.xcscheme | 2 +- .../xcschemes/PostboxMac.xcscheme | 2 +- .../xcschemes/xcschememanagement.plist | 4 +-- .../xcschemes/PostboxMac.xcscheme | 2 +- .../xcschemes/PostboxTests.xcscheme | 2 +- 6 files changed, 9 insertions(+), 33 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 8c48bef415..8c3ebdccf2 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -7,13 +7,6 @@ objects = { /* Begin PBXBuildFile section */ - C26A71921DC9E96F00F69385 /* ItemCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */; }; - C26A71931DC9E97700F69385 /* ItemCollectionInfoTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */; }; - C26A71941DC9E97E00F69385 /* ItemCollectionsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */; }; - C26A71951DC9E98900F69385 /* ItemCollectionItemTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */; }; - C291942B1DCBDECA00359491 /* PeerChatInterfaceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */; }; - C291942C1DCBDECA00359491 /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; - C291942D1DCBDEEC00359491 /* InitialMessageHistoryData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */; }; D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */; }; @@ -176,8 +169,6 @@ D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; - D0F7AB321DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; - D0F7AB331DCFAB1C009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */; }; D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; @@ -291,7 +282,6 @@ D0E3A7871B28AE9C00A402D9 /* Coding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Coding.swift; sourceTree = ""; }; D0E3A79D1B28B50400A402D9 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = ""; }; D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = ""; }; - D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatTopTaggedMessageIds.swift; sourceTree = ""; }; D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageMediaTable.swift; sourceTree = ""; }; D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTableTests.swift; sourceTree = ""; }; D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaCleanupTable.swift; sourceTree = ""; }; @@ -421,7 +411,6 @@ D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */, D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */, D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */, - D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */, ); name = Tables; sourceTree = ""; @@ -728,17 +717,6 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( -<<<<<<< HEAD - C291942D1DCBDEEC00359491 /* InitialMessageHistoryData.swift in Sources */, - C291942B1DCBDECA00359491 /* PeerChatInterfaceState.swift in Sources */, - C291942C1DCBDECA00359491 /* PeerChatInterfaceStateTable.swift in Sources */, - C26A71951DC9E98900F69385 /* ItemCollectionItemTable.swift in Sources */, - C26A71941DC9E97E00F69385 /* ItemCollectionsView.swift in Sources */, - C26A71931DC9E97700F69385 /* ItemCollectionInfoTable.swift in Sources */, - C26A71921DC9E96F00F69385 /* ItemCollection.swift in Sources */, - D03C535C1DAD5C9C004C17B3 /* Message.swift in Sources */, -======= ->>>>>>> 2c5dd8804bb7df7a03d68b53ef3241c8f2cbd529 D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, D073CE7E1DCBF3B4007511FD /* PeerNotificationSettings.swift in Sources */, D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, @@ -803,7 +781,6 @@ D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, D073CE8C1DCBF3BB007511FD /* ChatListTable.swift in Sources */, D073CE921DCBF3BB007511FD /* PeerTable.swift in Sources */, - D0F7AB331DCFAB1C009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */, D073CE741DCBF3B4007511FD /* Peer.swift in Sources */, D073CE7A1DCBF3B4007511FD /* PeerReadState.swift in Sources */, D073CE801DCBF3B4007511FD /* PeerChatInterfaceState.swift in Sources */, @@ -886,7 +863,6 @@ D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */, D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */, D03120FC1DA55427006A2A60 /* PeerNotificationSettings.swift in Sources */, - D0F7AB321DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */, D03BCCF81C73561C0097A291 /* Table.swift in Sources */, D021E0DC1DB5237C00C6B04F /* ItemCollectionsView.swift in Sources */, D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, @@ -997,7 +973,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; - SWIFT_VERSION = 3.0.1; + SWIFT_VERSION = 3.0; }; name = Hockeyapp; }; @@ -1247,7 +1223,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; - SWIFT_VERSION = 3.0.1; + SWIFT_VERSION = 3.0; }; name = Debug; }; @@ -1278,7 +1254,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; - SWIFT_VERSION = 3.0.1; + SWIFT_VERSION = 3.0; }; name = Release; }; diff --git a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/Postbox.xcscheme b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/Postbox.xcscheme index 46e76ad439..ba5245b1f0 100644 --- a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/Postbox.xcscheme +++ b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/Postbox.xcscheme @@ -1,6 +1,6 @@ Postbox.xcscheme orderHint - 5 + 10 PostboxMac.xcscheme orderHint - 6 + 11 SuppressBuildableAutocreation diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme index f05213e2b6..f4a9378c19 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme @@ -1,6 +1,6 @@ Date: Tue, 22 Nov 2016 21:30:16 +0300 Subject: [PATCH 060/427] no message --- Postbox/Coding.swift | 4 + Postbox/MediaBox.swift | 91 ++++++++++++++++++----- Postbox/PeerNameIndexRepresentation.swift | 4 +- 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 9c1608895f..6399702203 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -31,6 +31,10 @@ public func declareEncodable(_ type: Any.Type, f: @escaping(Decoder) -> Coding) typeStore.dict[murMurHashString32("\(type)")] = f } +public func persistentHash32(_ string: String) -> Int32 { + return murMurHashString32(string) +} + private let emptyMemory = malloc(1)! public class MemoryBuffer: Equatable, CustomStringConvertible { diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 6d6ebb2f4b..ba2ddab1b8 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -201,8 +201,14 @@ public final class MediaBox { } if fetchingData { - //status = .Fetching(progress: Float(currentSize) / Float(resourceSize)) - status = .Fetching(progress: 0.0) + let currentSize = fileSize(paths.partial) ?? 0 + + if let resourceSize = resource.size { + status = .Fetching(progress: Float(currentSize) / Float(resourceSize)) + } else { + status = .Fetching(progress: 0.0) + } + } else { status = .Remote } @@ -239,24 +245,50 @@ public final class MediaBox { } public func resourceData(_ resource: MediaResource, pathExtension: String? = nil, complete: Bool = true) -> Signal { - assert(pathExtension == nil) return Signal { subscriber in let disposable = MetaDisposable() self.concurrentQueue.async { let paths = self.storePathsForId(resource.id) if let completeSize = fileSize(paths.complete) { - subscriber.putNext(MediaResourceData(path: paths.complete, size: completeSize, complete: true)) - subscriber.putCompletion() + if let pathExtension = pathExtension { + let symlinkPath = paths.complete + ".\(pathExtension)" + if fileSize(symlinkPath) == nil { + let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) + } + subscriber.putNext(MediaResourceData(path: symlinkPath, size: completeSize, complete: true)) + subscriber.putCompletion() + } else { + subscriber.putNext(MediaResourceData(path: paths.complete, size: completeSize, complete: true)) + subscriber.putCompletion() + } } else { self.dataQueue.async { let resourceId = WrappedMediaResourceId(resource.id) var currentContext: ResourceDataContext? = self.dataContexts[resourceId] if let currentContext = currentContext, currentContext.data.complete { - subscriber.putNext(currentContext.data) - subscriber.putCompletion() + if let pathExtension = pathExtension { + let symlinkPath = paths.complete + ".\(pathExtension)" + if fileSize(symlinkPath) == nil { + let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) + } + subscriber.putNext(MediaResourceData(path: symlinkPath, size: currentContext.data.size, complete: currentContext.data.complete)) + subscriber.putCompletion() + } else { + subscriber.putNext(currentContext.data) + subscriber.putCompletion() + } } else if let completeSize = fileSize(paths.complete) { - subscriber.putNext(MediaResourceData(path: paths.complete, size: completeSize, complete: true)) - subscriber.putCompletion() + if let pathExtension = pathExtension { + let symlinkPath = paths.complete + ".\(pathExtension)" + if fileSize(symlinkPath) == nil { + let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) + } + subscriber.putNext(MediaResourceData(path: symlinkPath, size: completeSize, complete: true)) + subscriber.putCompletion() + } else { + subscriber.putNext(MediaResourceData(path: paths.complete, size: completeSize, complete: true)) + subscriber.putCompletion() + } } else { let dataContext: ResourceDataContext if let currentContext = currentContext { @@ -270,15 +302,33 @@ public final class MediaBox { let index: Bag<(MediaResourceData) -> Void>.Index if complete { index = dataContext.completeDataSubscribers.add { data in - subscriber.putNext(data) - subscriber.putCompletion() + if let pathExtension = pathExtension, data.complete { + let symlinkPath = paths.complete + ".\(pathExtension)" + if fileSize(symlinkPath) == nil { + let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) + } + subscriber.putNext(MediaResourceData(path: symlinkPath, size: data.size, complete: data.complete)) + subscriber.putCompletion() + } else { + subscriber.putNext(data) + subscriber.putCompletion() + } } subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0, complete: false)) } else { index = dataContext.progresiveDataSubscribers.add { data in - subscriber.putNext(data) - if data.complete { + if let pathExtension = pathExtension, data.complete { + let symlinkPath = paths.complete + ".\(pathExtension)" + if fileSize(symlinkPath) == nil { + let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) + } + subscriber.putNext(MediaResourceData(path: symlinkPath, size: data.size, complete: data.complete)) subscriber.putCompletion() + } else { + subscriber.putNext(data) + if data.complete { + subscriber.putCompletion() + } } } subscriber.putNext(dataContext.data) @@ -455,8 +505,12 @@ public final class MediaBox { let index: Bag.Index = dataContext.fetchSubscribers.add(Void()) if dataContext.fetchDisposable == nil { - //let status: MediaResourceStatus = .Fetching(progress: Float(currentSize) / Float(resourceSize)) - let status: MediaResourceStatus = .Fetching(progress: 0.0) + let status: MediaResourceStatus + if let resourceSize = resource.size { + status = .Fetching(progress: Float(currentSize) / Float(resourceSize)) + } else { + status = .Fetching(progress: 0.0) + } self.statusQueue.async { if let statusContext = self.statusContexts[resourceId] { statusContext.status = status @@ -526,8 +580,11 @@ public final class MediaBox { if updatedData.complete { status = .Local } else { - //status = .Fetching(progress: Float(updatedSize) / Float(resourceSize)) - status = .Fetching(progress: 0.0) + if let resourceSize = resource.size { + status = .Fetching(progress: Float(updatedSize) / Float(resourceSize)) + } else { + status = .Fetching(progress: 0.0) + } } self.statusQueue.async { diff --git a/Postbox/PeerNameIndexRepresentation.swift b/Postbox/PeerNameIndexRepresentation.swift index ba6c28db35..4b9ee60c50 100644 --- a/Postbox/PeerNameIndexRepresentation.swift +++ b/Postbox/PeerNameIndexRepresentation.swift @@ -11,7 +11,7 @@ public enum PeerNameIndex { } extension PeerIndexNameRepresentation { - func indexName(_ index: PeerNameIndex) -> String { + public func indexName(_ index: PeerNameIndex) -> String { switch self { case let .title(title): return title @@ -25,7 +25,7 @@ extension PeerIndexNameRepresentation { } } - func match(query: String) -> Bool { + public func match(query: String) -> Bool { switch self { case let .title(title): return title.lowercased().hasPrefix(query) From 40dd5579e738d6a918cdad1c3d88efd4c7f26a83 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 28 Nov 2016 13:28:31 +0300 Subject: [PATCH 061/427] no message --- Postbox/MediaBox.swift | 4 ++-- Postbox/RandomAccessMediaResourceContext.swift | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index ba2ddab1b8..fc3ec43501 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -414,7 +414,7 @@ public final class MediaBox { if let dataContext = strongSelf.randomAccessContexts[resourceId] { dataContext.removeListenerForFetchedData(listener) if !dataContext.hasDataListeners() { - let _ = strongSelf.randomAccessContexts.removeValue(forKey: resourceId) + //let _ = strongSelf.randomAccessContexts.removeValue(forKey: resourceId) } } } @@ -470,7 +470,7 @@ public final class MediaBox { if let dataContext = strongSelf.randomAccessContexts[resourceId] { dataContext.removeListenerForData(listener) if !dataContext.hasDataListeners() { - let _ = strongSelf.randomAccessContexts.removeValue(forKey: resourceId) + //let _ = strongSelf.randomAccessContexts.removeValue(forKey: resourceId) } } } diff --git a/Postbox/RandomAccessMediaResourceContext.swift b/Postbox/RandomAccessMediaResourceContext.swift index f4bfd3be03..54e5288aed 100644 --- a/Postbox/RandomAccessMediaResourceContext.swift +++ b/Postbox/RandomAccessMediaResourceContext.swift @@ -177,7 +177,7 @@ public final class RandomAccessMediaResourceContext { for range in ranges.sorted(by: { $0.offset < $1.offset }) { var offset = range.offset let endOffset = offset + range.data.count - assert(offset % self.blockSize == 0) + assert(offset % self.blockSize == 0 || (endOffset == self.size && range.data.count == 0)) assert(offset >= 0) assert(endOffset == self.size || (endOffset < self.size && endOffset % self.blockSize == 0)) From 9c3583c56662629bc33ad1003877b5049e0e3bb4 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 16 Dec 2016 20:30:06 +0300 Subject: [PATCH 062/427] no message --- Postbox.xcodeproj/project.pbxproj | 21 +- Postbox/ChatListIndexTable.swift | 101 +++++++-- Postbox/ChatListTable.swift | 13 ++ Postbox/Coding.swift | 61 ++++++ Postbox/Media.swift | 12 +- Postbox/MessageHistoryTable.swift | 3 +- Postbox/MurMurHash32.h | 2 + Postbox/MurMurHash32.m | 18 ++ Postbox/Peer.swift | 4 +- Postbox/PeerNameIndexRepresentation.swift | 67 ++++-- Postbox/PeerNameIndexTable.swift | 252 ++++++++++++++++++++++ Postbox/PeerNameTokenIndexTable.swift | 211 ++++++++++++++++++ Postbox/Postbox.swift | 56 ++++- Postbox/StringIndexTokens.swift | 74 +++++++ Postbox/ValueBoxKey.swift | 21 +- 15 files changed, 866 insertions(+), 50 deletions(-) create mode 100644 Postbox/PeerNameIndexTable.swift create mode 100644 Postbox/PeerNameTokenIndexTable.swift create mode 100644 Postbox/StringIndexTokens.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 3e00341027..1e091e1db1 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -85,6 +85,9 @@ D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; + D07827C11E0079CB00071108 /* StringIndexTokens.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C01E0079CB00071108 /* StringIndexTokens.swift */; }; + D07827C31E008F7300071108 /* PeerNameTokenIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */; }; + D07827C51E00B23F00071108 /* PeerNameIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */; }; D07CFF811DCA765D00761F81 /* PeerChatInterfaceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */; }; D07CFF831DCA909100761F81 /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */; }; @@ -237,6 +240,9 @@ D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; + D07827C01E0079CB00071108 /* StringIndexTokens.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = StringIndexTokens.swift; sourceTree = ""; }; + D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameTokenIndexTable.swift; sourceTree = ""; }; + D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameIndexTable.swift; sourceTree = ""; }; D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatInterfaceState.swift; sourceTree = ""; }; D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatInterfaceStateTable.swift; sourceTree = ""; }; D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = InitialMessageHistoryData.swift; sourceTree = ""; }; @@ -423,6 +429,8 @@ D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */, D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */, D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */, + D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */, + D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */, ); name = Tables; sourceTree = ""; @@ -456,6 +464,7 @@ D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */, D0D949F41D35353900740E02 /* MappedFile.swift */, D0D510FF1D64A58900A97B8A /* IpcPipe.swift */, + D07827C01E0079CB00071108 /* StringIndexTokens.swift */, ); name = Utils; sourceTree = ""; @@ -850,6 +859,7 @@ D098C6F11D7E11E9007784E4 /* Database.swift in Sources */, D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */, + D07827C31E008F7300071108 /* PeerNameTokenIndexTable.swift in Sources */, D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */, @@ -859,6 +869,7 @@ D0D949F51D35353900740E02 /* MappedFile.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, D0AB0B8E1D65D49C002C78E7 /* ChatListHolesView.swift in Sources */, + D07827C51E00B23F00071108 /* PeerNameIndexTable.swift in Sources */, D03120FA1DA540F0006A2A60 /* CachedPeerData.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, @@ -869,6 +880,7 @@ D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */, + D07827C11E0079CB00071108 /* StringIndexTokens.swift in Sources */, D0F3CC721DDE1CDC008148FA /* ItemCacheTable.swift in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, @@ -991,7 +1003,8 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; - SWIFT_VERSION = 3.0.1; + SWIFT_REFLECTION_METADATA_LEVEL = none; + SWIFT_VERSION = 3.0; }; name = Hockeyapp; }; @@ -1241,7 +1254,8 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; - SWIFT_VERSION = 3.0.1; + SWIFT_REFLECTION_METADATA_LEVEL = none; + SWIFT_VERSION = 3.0; }; name = Debug; }; @@ -1272,7 +1286,8 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; - SWIFT_VERSION = 3.0.1; + SWIFT_REFLECTION_METADATA_LEVEL = none; + SWIFT_VERSION = 3.0; }; name = Release; }; diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift index c99cbfbeba..aa571dbfb3 100644 --- a/Postbox/ChatListIndexTable.swift +++ b/Postbox/ChatListIndexTable.swift @@ -5,37 +5,96 @@ final class ChatListIndexTable: Table { return ValueBoxTable(id: id, keyType: .int64) } + private let peerNameIndexTable: PeerNameIndexTable + + private let sharedKey = ValueBoxKey(length: 8) + + private var cachedIndices: [PeerId: MessageIndex?] = [:] + private var updatedPreviousCachedIndices: [PeerId: MessageIndex?] = [:] + + init(valueBox: ValueBox, table: ValueBoxTable, peerNameIndexTable: PeerNameIndexTable) { + self.peerNameIndexTable = peerNameIndexTable + + super.init(valueBox: valueBox, table: table) + } + private func key(_ peerId: PeerId) -> ValueBoxKey { - let key = ValueBoxKey(length: 8) - key.setInt64(0, value: peerId.toInt64()) - return key + self.sharedKey.setInt64(0, value: peerId.toInt64()) + return self.sharedKey } func set(_ index: MessageIndex) { - let writeBuffer = WriteBuffer() - var idNamespace: Int32 = index.id.namespace - var idId: Int32 = index.id.id - var idTimestamp: Int32 = index.timestamp - writeBuffer.write(&idNamespace, offset: 0, length: 4) - writeBuffer.write(&idId, offset: 0, length: 4) - writeBuffer.write(&idTimestamp, offset: 0, length: 4) - self.valueBox.set(self.table, key: self.key(index.id.peerId), value: writeBuffer.readBufferNoCopy()) + self.updatedPreviousCachedIndices[index.id.peerId] = self.get(index.id.peerId) + self.cachedIndices[index.id.peerId] = index } func remove(_ peerId: PeerId) { - self.valueBox.remove(self.table, key: self.key(peerId)) + self.updatedPreviousCachedIndices[peerId] = self.get(peerId) + self.cachedIndices[peerId] = nil } func get(_ peerId: PeerId) -> MessageIndex? { - if let value = self.valueBox.get(self.table, key: self.key(peerId)) { - var idNamespace: Int32 = 0 - var idId: Int32 = 0 - var idTimestamp: Int32 = 0 - value.read(&idNamespace, offset: 0, length: 4) - value.read(&idId, offset: 0, length: 4) - value.read(&idTimestamp, offset: 0, length: 4) - return MessageIndex(id: MessageId(peerId: peerId, namespace: idNamespace, id: idId), timestamp: idTimestamp) + if let cached = self.cachedIndices[peerId] { + return cached + } else { + if let value = self.valueBox.get(self.table, key: self.key(peerId)) { + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + let index = MessageIndex(id: MessageId(peerId: peerId, namespace: idNamespace, id: idId), timestamp: idTimestamp) + self.cachedIndices[peerId] = index + return index + } else { + return nil + } + } + } + + override func clearMemoryCache() { + self.cachedIndices.removeAll() + assert(self.updatedPreviousCachedIndices.isEmpty) + } + + override func beforeCommit() { + if !self.updatedPreviousCachedIndices.isEmpty { + var addedPeerIds = Set() + var removedPeerIds = Set() + + for (peerId, previousIndex) in self.updatedPreviousCachedIndices { + let index = self.cachedIndices[peerId]! + if let index = index { + if previousIndex == nil { + addedPeerIds.insert(peerId) + } + + let writeBuffer = WriteBuffer() + var idNamespace: Int32 = index.id.namespace + var idId: Int32 = index.id.id + var idTimestamp: Int32 = index.timestamp + writeBuffer.write(&idNamespace, offset: 0, length: 4) + writeBuffer.write(&idId, offset: 0, length: 4) + writeBuffer.write(&idTimestamp, offset: 0, length: 4) + self.valueBox.set(self.table, key: self.key(index.id.peerId), value: writeBuffer.readBufferNoCopy()) + } else { + if previousIndex != nil { + removedPeerIds.insert(peerId) + } + + self.valueBox.remove(self.table, key: self.key(peerId)) + } + } + self.updatedPreviousCachedIndices.removeAll() + + for peerId in addedPeerIds { + self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.chats], includes: true) + } + + for peerId in removedPeerIds { + self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.chats], includes: false) + } } - return nil } } diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index be9d38bd7e..e2cb8686f7 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -245,6 +245,19 @@ final class ChatListTable: Table { return (entries: entries, lower: lower, upper: upper) } + func topPeerIds(count: Int) -> [PeerId] { + var peerIds: [PeerId] = [] + self.valueBox.range(self.table, start: self.upperBound(), end: self.lowerBound(), keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) + let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + if type == ChatListEntryType.Message.rawValue { + peerIds.append(index.id.peerId) + } + return true + }, limit: count) + return peerIds + } + func earlierEntries(_ index: MessageIndex?, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> [ChatListIntermediateEntry] { self.ensureInitialized() diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 6399702203..af779414c7 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -103,6 +103,10 @@ public class MemoryBuffer: Equatable, CustomStringConvertible { return Data(bytes: self.memory, count: self.length) } } + + public func withDataNoCopy(_ f: (Data) -> Void) { + f(Data(bytesNoCopy: self.memory, count: self.length, deallocator: .none)) + } } public func ==(lhs: MemoryBuffer, rhs: MemoryBuffer) -> Bool { @@ -922,6 +926,63 @@ public final class Decoder { } } + public func decodeObjectDictionaryForKey(_ key: StaticString, keyDecoder: (Decoder) -> K) -> [K : V] { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var dictionary: [K : V] = [:] + + var failed = false + var i: Int32 = 0 + while i < length { + var keyHash: Int32 = 0 + memcpy(&keyHash, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var keyLength: Int32 = 0 + memcpy(&keyLength, self.buffer.memory + self.offset, 4) + + var innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(keyLength), freeWhenDone: false)) + self.offset += 4 + Int(keyLength) + + var key: K? + if !failed { + key = keyDecoder(innerDecoder) + } + + var valueHash: Int32 = 0 + memcpy(&valueHash, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var valueLength: Int32 = 0 + memcpy(&valueLength, self.buffer.memory + self.offset, 4) + + innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(valueLength), freeWhenDone: false)) + self.offset += 4 + Int(valueLength) + + let value = failed ? nil : (typeStore.decode(valueHash, decoder: innerDecoder) as? V) + + if let key = key, let value = value { + dictionary[key] = value + } else { + failed = true + } + + i += 1 + } + + if failed { + return [:] + } else { + return dictionary + } + } else { + return [:] + } + } + public func decodeBytesForKeyNoCopy(_ key: StaticString) -> ReadBuffer? { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bytes) { var length: Int32 = 0 diff --git a/Postbox/Media.swift b/Postbox/Media.swift index 76f629a4bd..5fa0f0c7ab 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -1,6 +1,6 @@ import Foundation -public struct MediaId: Hashable, CustomStringConvertible { +public struct MediaId: Hashable, Coding, CustomStringConvertible { public typealias Namespace = Int32 public typealias Id = Int64 @@ -35,6 +35,16 @@ public struct MediaId: Hashable, CustomStringConvertible { buffer.offset += 12 } + public init(decoder: Decoder) { + self.namespace = decoder.decodeInt32ForKey("n") + self.id = decoder.decodeInt64ForKey("i") + } + + public func encode(_ encoder: Encoder) { + encoder.encodeInt32(self.namespace, forKey: "n") + encoder.encodeInt64(self.id, forKey: "i") + } + public func encodeToBuffer(_ buffer: WriteBuffer) { var namespace = self.namespace var id = self.id diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index ffcfeb46e6..55a3753d8b 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -1487,7 +1487,8 @@ final class MessageHistoryTable: Table { } else if let upper = upper { return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: upper.index.timestamp), exact: true) } - return nil + + return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: 1), exact: false) } func findMessageId(peerId: PeerId, namespace: MessageId.Namespace, timestamp: Int32) -> MessageId? { diff --git a/Postbox/MurMurHash32.h b/Postbox/MurMurHash32.h index 42512ef238..46f06ba46d 100644 --- a/Postbox/MurMurHash32.h +++ b/Postbox/MurMurHash32.h @@ -2,8 +2,10 @@ #define Postbox_MurMurHash32_h #import +#import int32_t murMurHash32(void *bytes, int length); int32_t murMurHashString32(const char *s); +NSString *postboxTransformedString(CFStringRef string, bool replaceWithTransliteratedVersion, bool appendTransliteratedVersion); #endif diff --git a/Postbox/MurMurHash32.m b/Postbox/MurMurHash32.m index 25afc09f39..6cf5a19aea 100644 --- a/Postbox/MurMurHash32.m +++ b/Postbox/MurMurHash32.m @@ -96,3 +96,21 @@ int32_t murMurHashString32(const char *s) return result; } + +NSString *postboxTransformedString(CFStringRef string, bool replaceWithTransliteratedVersion, bool appendTransliteratedVersion) { + NSMutableString *mutableString = [[NSMutableString alloc] initWithString:(__bridge NSString * _Nonnull)(string)]; + CFStringTransform((CFMutableStringRef)mutableString, NULL, kCFStringTransformStripCombiningMarks, false); + + if (replaceWithTransliteratedVersion || appendTransliteratedVersion) { + NSMutableString *transliteratedString = [[NSMutableString alloc] initWithString:mutableString]; + CFStringTransform((CFMutableStringRef)transliteratedString, NULL, kCFStringTransformToLatin, false); + if (replaceWithTransliteratedVersion) { + return transliteratedString; + } else { + [mutableString appendString:@" "]; + [mutableString appendString:transliteratedString]; + } + } + + return mutableString; +} diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index ad533ee81b..f5c9a454e2 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -13,8 +13,8 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { } public init(_ n: Int64) { - self.namespace = Int32((n >> 32) & 0xffffffff) - self.id = Int32(n & 0xffffffff) + self.namespace = Int32((n >> 32) & 0x7fffffff) + self.id = Int32(n & 0x7fffffff) } public func toInt64() -> Int64 { diff --git a/Postbox/PeerNameIndexRepresentation.swift b/Postbox/PeerNameIndexRepresentation.swift index 4b9ee60c50..fd6182f0d7 100644 --- a/Postbox/PeerNameIndexRepresentation.swift +++ b/Postbox/PeerNameIndexRepresentation.swift @@ -1,8 +1,25 @@ import Foundation -public enum PeerIndexNameRepresentation { - case title(String) - case personName(first: String, last: String) +public enum PeerIndexNameRepresentation: Equatable { + case title(title: String, addressName: String?) + case personName(first: String, last: String, addressName: String?) + + public static func ==(lhs: PeerIndexNameRepresentation, rhs: PeerIndexNameRepresentation) -> Bool { + switch lhs { + case let .title(lhsTitle, lhsAddressName): + if case let .title(rhsTitle, rhsAddressName) = rhs, lhsTitle == rhsTitle, lhsAddressName == rhsAddressName { + return true + } else { + return false + } + case let .personName(lhsFirst, lhsLast, lhsAddressName): + if case let .personName(rhsFirst, rhsLast, rhsAddressName) = rhs, lhsFirst == rhsFirst, lhsLast == rhsLast, lhsAddressName == rhsAddressName { + return true + } else { + return false + } + } + } } public enum PeerNameIndex { @@ -13,9 +30,9 @@ public enum PeerNameIndex { extension PeerIndexNameRepresentation { public func indexName(_ index: PeerNameIndex) -> String { switch self { - case let .title(title): + case let .title(title, _): return title - case let .personName(first, last): + case let .personName(first, last, _): switch index { case .firstNameFirst: return first + last @@ -25,13 +42,39 @@ extension PeerIndexNameRepresentation { } } - public func match(query: String) -> Bool { - switch self { - case let .title(title): - return title.lowercased().hasPrefix(query) - case let .personName(first, last): - return first.lowercased().hasPrefix(query) || last.lowercased().hasPrefix(query) + public func matchesByTokens(_ other: String) -> Bool { + var foundAtLeastOne = false + for searchToken in stringIndexTokens(other, transliteration: .none) { + var found = false + for token in self.indexTokens { + if searchToken.isPrefix(to: token) { + found = true + break + } + } + if !found { + return false + } + foundAtLeastOne = true + } + return foundAtLeastOne + } + + var indexTokens: [ValueBoxKey] { + switch self { + case let .title(title, addressName): + var tokens: [ValueBoxKey] = stringIndexTokens(title, transliteration: .combined) + if let addressName = addressName { + tokens.append(contentsOf: stringIndexTokens(addressName, transliteration: .none)) + } + return tokens + case let .personName(first, last, addressName): + var tokens: [ValueBoxKey] = stringIndexTokens(first, transliteration: .combined) + tokens.append(contentsOf: stringIndexTokens(last, transliteration: .combined)) + if let addressName = addressName { + tokens.append(contentsOf: stringIndexTokens(addressName, transliteration: .none)) + } + return tokens } - return false } } diff --git a/Postbox/PeerNameIndexTable.swift b/Postbox/PeerNameIndexTable.swift new file mode 100644 index 0000000000..43164fc5da --- /dev/null +++ b/Postbox/PeerNameIndexTable.swift @@ -0,0 +1,252 @@ +import Foundation + +struct PeerNameIndexCategories: OptionSet { + var rawValue: Int32 + + init() { + self.rawValue = 0 + } + + init(rawValue: Int32) { + self.rawValue = rawValue + } + + static let chats = PeerNameIndexCategories(rawValue: 1 << 0) + static let contacts = PeerNameIndexCategories(rawValue: 1 << 1) +} + +private final class PeerNameIndexCategoriesEntry { + let categories: PeerNameIndexCategories + let tokens: [ValueBoxKey] + + init(categories: PeerNameIndexCategories, tokens: [ValueBoxKey]) { + self.categories = categories + self.tokens = tokens + } + + init(buffer: MemoryBuffer) { + assert(buffer.length >= 8) + + self.categories = PeerNameIndexCategories(rawValue: buffer.memory.load(fromByteOffset: 0, as: Int32.self)) + let tokenCount = buffer.memory.load(fromByteOffset: 4, as: Int32.self) + var offset = 8 + var tokens: [ValueBoxKey] = [] + for i in 0 ..< tokenCount { + let length = buffer.memory.load(fromByteOffset: offset, as: Int32.self) + offset += 4 + tokens.append(ValueBoxKey(MemoryBuffer(memory: buffer.memory.advanced(by: offset), capacity: Int(length), length: Int(length), freeWhenDone: false))) + offset += Int(length) + let paddingLength = length % 4 + offset += Int(paddingLength) + } + self.tokens = tokens + } + + func write(to buffer: WriteBuffer) { + var rawValue: Int32 = self.categories.rawValue + buffer.write(&rawValue, offset: 0, length: 4) + var count: Int32 = Int32(self.tokens.count) + buffer.write(&count, offset: 0, length: 4) + for token in self.tokens { + var length = Int32(token.length) + buffer.write(&length, offset: 0, length: 4) + buffer.write(token.memory, offset: 0, length: token.length) + var paddingLength = token.length % 4 + var zero: Int8 = 0 + while paddingLength > 0 { + buffer.write(&zero, offset: 0, length: 1) + paddingLength -= 1 + } + } + } +} + +private final class PeerNameIndexCategoriesEntryUpdate { + let initialCategories: PeerNameIndexCategories + let initialTokens: [ValueBoxKey] + + private(set) var updatedCategories: PeerNameIndexCategories? + private(set) var updatedName: PeerIndexNameRepresentation? + + init(initialCategories: PeerNameIndexCategories, initialTokens: [ValueBoxKey]) { + self.initialCategories = initialCategories + self.initialTokens = initialTokens + } + + func updateCategory(_ category: PeerNameIndexCategories, includes: Bool) { + var currentCategories: PeerNameIndexCategories + if let updatedCategories = self.updatedCategories { + currentCategories = updatedCategories + } else { + currentCategories = self.initialCategories + } + if includes { + currentCategories.insert(category) + self.updatedCategories = currentCategories + } else { + currentCategories.remove(category) + self.updatedCategories = currentCategories + } + } + + func updateName(_ name: PeerIndexNameRepresentation) { + self.updatedName = name + } +} + +final class PeerNameIndexTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + + private let peerTable: PeerTable + private let peerNameTokenIndexTable: PeerNameTokenIndexTable + + private let sharedKey = ValueBoxKey(length: 8) + + private var entryUpdates: [PeerId: PeerNameIndexCategoriesEntryUpdate] = [:] + + init(valueBox: ValueBox, table: ValueBoxTable, peerTable: PeerTable, peerNameTokenIndexTable: PeerNameTokenIndexTable) { + self.peerTable = peerTable + self.peerNameTokenIndexTable = peerNameTokenIndexTable + + super.init(valueBox: valueBox, table: table) + } + + private func key(_ peerId: PeerId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: peerId.toInt64()) + return self.sharedKey + } + + private func updateEntry(_ peerId: PeerId, _ f: (PeerNameIndexCategoriesEntryUpdate) -> Void) { + let entryUpdate: PeerNameIndexCategoriesEntryUpdate + if let current = self.entryUpdates[peerId] { + entryUpdate = current + } else { + let entry: PeerNameIndexCategoriesEntry + if let value = self.valueBox.get(self.table, key: self.key(peerId)) { + entry = PeerNameIndexCategoriesEntry(buffer: value) + } else { + entry = PeerNameIndexCategoriesEntry(categories: [], tokens: []) + } + entryUpdate = PeerNameIndexCategoriesEntryUpdate(initialCategories: entry.categories, initialTokens: entry.tokens) + self.entryUpdates[peerId] = entryUpdate + } + f(entryUpdate) + } + + func setPeerCategoryState(peerId: PeerId, category: PeerNameIndexCategories, includes: Bool) { + self.updateEntry(peerId, { entryUpdate in + entryUpdate.updateCategory(category, includes: includes) + }) + } + + func markPeerNameUpdated(peerId: PeerId, name: PeerIndexNameRepresentation) { + self.updateEntry(peerId, { entryUpdate in + entryUpdate.updateName(name) + }) + } + + override func clearMemoryCache() { + assert(self.entryUpdates.isEmpty) + } + + override func beforeCommit() { + if !self.entryUpdates.isEmpty { + let sharedBuffer = WriteBuffer() + for (peerId, entryUpdate) in self.entryUpdates { + if let updatedCategories = entryUpdate.updatedCategories { + let wasEmpty = entryUpdate.initialCategories.isEmpty + if updatedCategories.isEmpty != wasEmpty { + if updatedCategories.isEmpty { + if !entryUpdate.initialTokens.isEmpty { + self.peerNameTokenIndexTable.remove(peerId: peerId, tokens: entryUpdate.initialTokens) + } + if !entryUpdate.initialCategories.isEmpty { + self.valueBox.remove(self.table, key: self.key(peerId)) + } + } else { + let updatedTokens: [ValueBoxKey] + if let updatedName = entryUpdate.updatedName { + updatedTokens = updatedName.indexTokens + } else { + if let peer = self.peerTable.get(peerId) { + updatedTokens = peer.indexName.indexTokens + } else { + assertionFailure() + updatedTokens = [] + } + } + self.peerNameTokenIndexTable.add(peerId: peerId, tokens: updatedTokens) + sharedBuffer.reset() + PeerNameIndexCategoriesEntry(categories: updatedCategories, tokens: updatedTokens).write(to: sharedBuffer) + self.valueBox.set(self.table, key: self.key(peerId), value: sharedBuffer) + } + } else { + if let updatedName = entryUpdate.updatedName { + if !entryUpdate.initialTokens.isEmpty { + self.peerNameTokenIndexTable.remove(peerId: peerId, tokens: entryUpdate.initialTokens) + } + let updatedTokens = updatedName.indexTokens + self.peerNameTokenIndexTable.add(peerId: peerId, tokens: updatedTokens) + sharedBuffer.reset() + PeerNameIndexCategoriesEntry(categories: updatedCategories, tokens: updatedTokens).write(to: sharedBuffer) + self.valueBox.set(self.table, key: self.key(peerId), value: sharedBuffer) + } else { + sharedBuffer.reset() + PeerNameIndexCategoriesEntry(categories: updatedCategories, tokens: entryUpdate.initialTokens).write(to: sharedBuffer) + self.valueBox.set(self.table, key: self.key(peerId), value: sharedBuffer) + } + } + } else if let updatedName = entryUpdate.updatedName { + if !entryUpdate.initialCategories.isEmpty { + if !entryUpdate.initialTokens.isEmpty { + self.peerNameTokenIndexTable.remove(peerId: peerId, tokens: entryUpdate.initialTokens) + } + let updatedTokens = updatedName.indexTokens + self.peerNameTokenIndexTable.add(peerId: peerId, tokens: updatedTokens) + sharedBuffer.reset() + PeerNameIndexCategoriesEntry(categories: entryUpdate.initialCategories, tokens: updatedTokens).write(to: sharedBuffer) + self.valueBox.set(self.table, key: self.key(peerId), value: sharedBuffer) + } + } + } + self.entryUpdates.removeAll() + } + } + + func matchingPeerIds(tokens: (regular: [ValueBoxKey], transliterated: [ValueBoxKey]?), categories: PeerNameIndexCategories, chatListIndexTable: ChatListIndexTable, contactTable: ContactTable) -> (chats: [PeerId], contacts: [PeerId]) { + if categories.isEmpty { + return ([], []) + } else { + var contacts: [PeerId] = [] + var chatIndices: [PeerId: MessageIndex] = [:] + var peerIds = self.peerNameTokenIndexTable.matchingPeerIds(tokens: tokens.regular) + if let transliterated = tokens.transliterated, tokens.regular != transliterated { + let transliteratedPeerIds = self.peerNameTokenIndexTable.matchingPeerIds(tokens: transliterated) + peerIds.formUnion(transliteratedPeerIds) + } + for peerId in peerIds { + var foundInChats = false + if categories.contains(.chats) { + if let index = chatListIndexTable.get(peerId) { + foundInChats = true + chatIndices[peerId] = index + } + } + if !foundInChats { + if categories.contains(.contacts) { + if contactTable.isContact(peerId: peerId) { + contacts.append(peerId) + } + } + } + } + + let chats = chatIndices.keys.sorted(by: { lhs, rhs -> Bool in + return chatIndices[lhs]! > chatIndices[rhs]! + }) + return (chats, contacts) + } + } +} diff --git a/Postbox/PeerNameTokenIndexTable.swift b/Postbox/PeerNameTokenIndexTable.swift new file mode 100644 index 0000000000..cad285753f --- /dev/null +++ b/Postbox/PeerNameTokenIndexTable.swift @@ -0,0 +1,211 @@ +import Foundation + +private final class PeerNameTokenIndexTableEntry { + var sortedPeerIds: [Int64] = [] + + init() { + } + + init(buffer: MemoryBuffer) { + assert(buffer.length % 8 == 0) + + self.sortedPeerIds.reserveCapacity(buffer.length % 8) + withExtendedLifetime(buffer, { + let memory = buffer.memory.assumingMemoryBound(to: Int64.self) + for i in 0 ..< buffer.length / 8 { + self.sortedPeerIds.append(memory[i]) + } + }) + } + + private func index(of peerId: Int64) -> Int? { + var lowerIndex = 0 + var upperIndex = self.sortedPeerIds.count - 1 + + if lowerIndex > upperIndex { + return nil + } + + while (true) { + let currentIndex = (lowerIndex + upperIndex) / 2 + if self.sortedPeerIds[currentIndex] == peerId { + return currentIndex + } else if lowerIndex > upperIndex { + return nil + } else { + if self.sortedPeerIds[currentIndex] > peerId { + upperIndex = currentIndex - 1 + } else { + lowerIndex = currentIndex + 1 + } + } + } + } + + func remove(_ peerId: PeerId) -> Bool { + if let index = self.index(of: peerId.toInt64()) { + self.sortedPeerIds.remove(at: index) + return true + } else { + return false + } + } + + func insert(_ peerId: PeerId) -> Bool { + let insertItem = peerId.toInt64() + var lo = 0 + var hi = self.sortedPeerIds.count - 1 + while lo <= hi { + let mid = (lo + hi)/2 + if self.sortedPeerIds[mid] < insertItem { + lo = mid + 1 + } else if insertItem < self.sortedPeerIds[mid] { + hi = mid - 1 + } else { + return false + } + } + self.sortedPeerIds.insert(insertItem, at: lo) + return true + } + + func withMemoryBuffer(_ f: (MemoryBuffer) -> Void) { + let buffer = MemoryBuffer(memory: malloc(self.sortedPeerIds.count * 8), capacity: self.sortedPeerIds.count * 8, length: self.sortedPeerIds.count * 8, freeWhenDone: true) + let memory = buffer.memory.assumingMemoryBound(to: Int64.self) + var index = 0 + for peerId in self.sortedPeerIds { + memory[index] = peerId + index += 1 + } + f(buffer) + } +} + +final class PeerNameTokenIndexTable: Table { + private var cachedEntries: [ValueBoxKey: PeerNameTokenIndexTableEntry] = [:] + private var updatedCachedEntries = Set() + + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private func getEntry(token: ValueBoxKey) -> PeerNameTokenIndexTableEntry? { + if let cached = self.cachedEntries[token] { + return cached + } else { + if let value = self.valueBox.get(self.table, key: token) { + let entry = PeerNameTokenIndexTableEntry(buffer: value) + self.cachedEntries[token] = entry + return entry + } else { + return nil + } + } + } + + private func add(token: ValueBoxKey, peerId: PeerId) { + let entry: PeerNameTokenIndexTableEntry + var addToCache = false + if let current = self.getEntry(token: token) { + entry = current + } else { + entry = PeerNameTokenIndexTableEntry() + addToCache = true + } + if entry.insert(peerId) { + self.updatedCachedEntries.insert(token) + if addToCache { + self.cachedEntries[token] = entry + } + } + } + + private func remove(token: ValueBoxKey, peerId: PeerId) { + let entry: PeerNameTokenIndexTableEntry + var addToCache = false + if let current = self.getEntry(token: token) { + entry = current + } else { + entry = PeerNameTokenIndexTableEntry() + addToCache = true + } + if entry.remove(peerId) { + self.updatedCachedEntries.insert(token) + if addToCache { + self.cachedEntries[token] = entry + } + } + } + + func add(peerId: PeerId, tokens: [ValueBoxKey]) { + for token in tokens { + self.add(token: token, peerId: peerId) + } + } + + func remove(peerId: PeerId, tokens: [ValueBoxKey]) { + for token in tokens { + self.remove(token: token, peerId: peerId) + } + } + + func matchingPeerIds(tokens: [ValueBoxKey]) -> Set { + var peerIds: Set? + for token in tokens { + if let peerIds = peerIds, peerIds.isEmpty { + return Set() + } + var currentPeerIds = Set() + self.valueBox.range(self.table, start: token.predecessor, end: token.successor, values: { key, value in + if let cached = self.cachedEntries[key] { + for peerId in cached.sortedPeerIds { + currentPeerIds.insert(PeerId(peerId)) + } + } else { + for peerId in PeerNameTokenIndexTableEntry(buffer: value).sortedPeerIds { + currentPeerIds.insert(PeerId(peerId)) + } + } + return true + }, limit: 0) + if let previousPeerIds = peerIds { + peerIds = previousPeerIds.intersection(currentPeerIds) + } else { + peerIds = currentPeerIds + } + } + if let peerIds = peerIds { + return peerIds + } else { + return Set() + } + } + + override func clearMemoryCache() { + assert(self.cachedEntries.isEmpty) + assert(self.updatedCachedEntries.isEmpty) + } + + override func beforeCommit() { + if !self.updatedCachedEntries.isEmpty { + for token in self.updatedCachedEntries { + if let entry = self.cachedEntries[token] { + entry.withMemoryBuffer { buffer in + if buffer.length == 0 { + self.valueBox.remove(self.table, key: token) + } else { + self.valueBox.set(self.table, key: token, value: buffer) + } + } + } else { + assertionFailure() + } + } + self.updatedCachedEntries.removeAll() + } + + if !self.cachedEntries.isEmpty { + self.cachedEntries.removeAll() + } + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 351051a8fa..f3981b51db 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -266,6 +266,8 @@ public final class Postbox { var peerChatInterfaceStateTable: PeerChatInterfaceStateTable! var itemCacheMetaTable: ItemCacheMetaTable! var itemCacheTable: ItemCacheTable! + var peerNameTokenIndexTable: PeerNameTokenIndexTable! + var peerNameIndexTable: PeerNameIndexTable! //temporary var peerRatingTable: RatingTable! @@ -348,7 +350,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 16 + let currentUserVersion: Int32 = 1 if userVersion != currentUserVersion { self.valueBox.drop() @@ -366,8 +368,6 @@ public final class Postbox { self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox, table: MessageHistoryReadStateTable.tableSpec(14)) self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox, table: MessageHistorySynchronizeReadStateTable.tableSpec(15)) self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable!) - self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8)) - self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, table: PeerChatStateTable.tableSpec(13)) self.contactsTable = ContactTable(valueBox: self.valueBox, table: ContactTable.tableSpec(16)) self.peerRatingTable = RatingTable(valueBox: self.valueBox, table: RatingTable.tableSpec(17)) @@ -379,6 +379,10 @@ public final class Postbox { self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox, table: PeerChatInterfaceStateTable.tableSpec(23)) self.itemCacheMetaTable = ItemCacheMetaTable(valueBox: self.valueBox, table: ItemCacheMetaTable.tableSpec(24)) self.itemCacheTable = ItemCacheTable(valueBox: self.valueBox, table: ItemCacheTable.tableSpec(25)) + self.peerNameTokenIndexTable = PeerNameTokenIndexTable(valueBox: self.valueBox, table: PeerNameTokenIndexTable.tableSpec(26)) + self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) + self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8), peerNameIndexTable: self.peerNameIndexTable) + self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -404,6 +408,8 @@ public final class Postbox { self.tables.append(self.peerChatInterfaceStateTable) self.tables.append(self.itemCacheMetaTable) self.tables.append(self.itemCacheTable) + self.tables.append(self.peerNameIndexTable) + self.tables.append(self.peerNameTokenIndexTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() @@ -832,9 +838,13 @@ public final class Postbox { if let updatedPeer = update(currentPeer, peer) { self.peerTable.set(updatedPeer) self.currentUpdatedPeers[updatedPeer.id] = updatedPeer + if currentPeer.indexName != updatedPeer.indexName { + self.peerNameIndexTable.markPeerNameUpdated(peerId: peer.id, name: updatedPeer.indexName) + } } } else { self.peerTable.set(peer) + self.peerNameIndexTable.markPeerNameUpdated(peerId: peer.id, name: peer.indexName) } } } @@ -1166,17 +1176,45 @@ public final class Postbox { public func searchContacts(query: String) -> Signal<[Peer], NoError> { return self.modify { modifier -> Signal<[Peer], NoError> in - var peers: [Peer] = [] - for peerId in self.contactsTable.get() { + let (_, contactPeerIds) = self.peerNameIndexTable.matchingPeerIds(tokens: (regular: stringIndexTokens(query, transliteration: .none), transliterated: stringIndexTokens(query, transliteration: .transliterated)), categories: [.contacts], chatListIndexTable: self.chatListIndexTable, contactTable: self.contactsTable) + + var contactPeers: [Peer] = [] + for peerId in contactPeerIds { if let peer = self.peerTable.get(peerId) { - if peer.indexName.match(query: query) { - peers.append(peer) + contactPeers.append(peer) + } + } + + contactPeers.sort(by: { $0.indexName.indexName(.lastNameFirst) < $1.indexName.indexName(.lastNameFirst) }) + return .single(contactPeers) + } |> switchToLatest + } + + public func searchPeers(query: String) -> Signal<[Peer], NoError> { + return self.modify { modifier -> Signal<[Peer], NoError> in + var peerIds = Set() + var chatPeers: [Peer] = [] + + let (chatPeerIds, contactPeerIds) = self.peerNameIndexTable.matchingPeerIds(tokens: (regular: stringIndexTokens(query, transliteration: .none), transliterated: stringIndexTokens(query, transliteration: .transliterated)), categories: [.chats, .contacts], chatListIndexTable: self.chatListIndexTable, contactTable: self.contactsTable) + + for peerId in chatPeerIds { + if let peer = self.peerTable.get(peerId) { + chatPeers.append(peer) + peerIds.insert(peerId) + } + } + + var contactPeers: [Peer] = [] + for peerId in contactPeerIds { + if !peerIds.contains(peerId) { + if let peer = self.peerTable.get(peerId) { + contactPeers.append(peer) } } } - peers.sort(by: { $0.indexName.indexName(.lastNameFirst) < $1.indexName.indexName(.lastNameFirst) }) - return .single(peers) + contactPeers.sort(by: { $0.indexName.indexName(.lastNameFirst) < $1.indexName.indexName(.lastNameFirst) }) + return .single(chatPeers + contactPeers) } |> switchToLatest } diff --git a/Postbox/StringIndexTokens.swift b/Postbox/StringIndexTokens.swift new file mode 100644 index 0000000000..ad6f5c37f4 --- /dev/null +++ b/Postbox/StringIndexTokens.swift @@ -0,0 +1,74 @@ +import Foundation + +enum StringIndexTokenTransliteration { + case none + case combined + case transliterated +} + +func stringIndexTokens(_ string: String, transliteration: StringIndexTokenTransliteration) -> [ValueBoxKey] { + var nsString = string.lowercased() as NSString + + var isLatin = true + for i in 0 ..< nsString.length { + let c = nsString.character(at: i) + if c >= 128 { + isLatin = false + break + } + } + + if !isLatin { + nsString = postboxTransformedString(nsString, transliteration == .transliterated, transliteration == .combined) as NSString + } + + let flag = UInt(kCFStringTokenizerUnitWord) + let tokenizer = CFStringTokenizerCreate(kCFAllocatorDefault, nsString, CFRangeMake(0, nsString.length), flag, CFLocaleCopyCurrent()) + var tokenType = CFStringTokenizerAdvanceToNextToken(tokenizer) + var tokens: [ValueBoxKey] = [] + + var addedTokens = Set() + while tokenType != [] { + let currentTokenRange = CFStringTokenizerGetCurrentTokenRange(tokenizer) + + if currentTokenRange.location >= 0 && currentTokenRange.length != 0 { + let token = ValueBoxKey(length: currentTokenRange.length * 2) + nsString.getCharacters(token.memory.assumingMemoryBound(to: unichar.self), range: NSMakeRange(currentTokenRange.location, currentTokenRange.length)) + if !addedTokens.contains(token) { + tokens.append(token) + addedTokens.insert(token) + } + } + tokenType = CFStringTokenizerAdvanceToNextToken(tokenizer) + } + + return tokens +} + +func matchStringIndexTokens(_ tokens: [ValueBoxKey], with other: [ValueBoxKey]) -> Bool { + if other.isEmpty { + return false + } else if other.count == 1 { + let otherToken = other[0] + for token in tokens { + if otherToken.isPrefix(to: token) { + return true + } + } + } else { + for otherToken in other { + var found = false + for token in tokens { + if otherToken.isPrefix(to: token) { + found = true + break + } + } + if !found { + return false + } + } + return true + } + return false +} diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index ffac0122eb..414d7bd532 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -12,7 +12,7 @@ private final class ValueBoxKeyImpl { } } -public struct ValueBoxKey: Comparable, CustomStringConvertible { +public struct ValueBoxKey: Hashable, CustomStringConvertible { public let memory: UnsafeMutableRawPointer public let length: Int private let impl: ValueBoxKeyImpl @@ -89,6 +89,16 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { return key } + public func isPrefix(to other: ValueBoxKey) -> Bool { + if self.length == 0 { + return true + } else if self.length <= other.length { + return memcmp(self.memory, other.memory, self.length) == 0 + } else { + return false + } + } + public var successor: ValueBoxKey { let key = ValueBoxKey(length: self.length) memcpy(key.memory, self.memory, self.length) @@ -138,6 +148,15 @@ public struct ValueBoxKey: Comparable, CustomStringConvertible { } return string as String } + + public var hashValue: Int { + var hash = 37 + let bytes = self.memory.assumingMemoryBound(to: Int8.self) + for i in 0 ..< self.length { + hash = (hash &* 54059) ^ (Int(bytes[i]) &* 76963) + } + return hash + } } public func ==(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { From 6d636ad0a9055854353c3886a1e6a0e36a251dab Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 16 Dec 2016 20:32:22 +0300 Subject: [PATCH 063/427] no message --- Postbox.xcodeproj/project.pbxproj | 126 ++++++++++++++++-------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 1e091e1db1..dcf5db850e 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -50,33 +50,6 @@ D073CE7E1DCBF3B4007511FD /* PeerNotificationSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */; }; D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */; }; D073CE801DCBF3B4007511FD /* PeerChatInterfaceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */; }; - D073CE841DCBF3BB007511FD /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; - D073CE851DCBF3BB007511FD /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; - D073CE861DCBF3BB007511FD /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; - D073CE871DCBF3BB007511FD /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; - D073CE881DCBF3BB007511FD /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; - D073CE891DCBF3BB007511FD /* MessageHistoryOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */; }; - D073CE8A1DCBF3BB007511FD /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; - D073CE8B1DCBF3BB007511FD /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; - D073CE8C1DCBF3BB007511FD /* ChatListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8641C58CB7F00037222 /* ChatListTable.swift */; }; - D073CE8D1DCBF3BB007511FD /* ChatListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */; }; - D073CE8E1DCBF3BB007511FD /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; - D073CE8F1DCBF3BB007511FD /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; - D073CE901DCBF3BB007511FD /* MetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */; }; - D073CE911DCBF3BB007511FD /* KeychainTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */; }; - D073CE921DCBF3BB007511FD /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; - D073CE931DCBF3BB007511FD /* PeerNotificationSettingsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */; }; - D073CE941DCBF3BB007511FD /* CachedPeerDataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */; }; - D073CE951DCBF3BB007511FD /* PeerPresenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */; }; - D073CE961DCBF3BB007511FD /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; - D073CE971DCBF3BB007511FD /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; - D073CE981DCBF3BB007511FD /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; - D073CE991DCBF3BB007511FD /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; - D073CE9A1DCBF3BB007511FD /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; - D073CE9B1DCBF3BB007511FD /* RatingTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08D451E1D5D2CA700A7428A /* RatingTable.swift */; }; - D073CE9C1DCBF3BB007511FD /* ItemCollectionInfoTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */; }; - D073CE9D1DCBF3BB007511FD /* ItemCollectionItemTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */; }; - D073CE9E1DCBF3BB007511FD /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; D073CE9F1DCBF3C1007511FD /* InitialMessageHistoryData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */; }; D073CEA01DCBF3C1007511FD /* ItemCollectionsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */; }; D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-ambiguous-macro -Wno-conditional-uninitialized -Wno-unused-const-variable -Wno-unused-function -Wno-unreachable-code"; }; }; @@ -174,10 +147,40 @@ D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; D0F3CC721DDE1CDC008148FA /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; - D0F3CC751DDE2845008148FA /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; - D0F3CC761DDE2845008148FA /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; D0F7AB321DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; - D0F7AB331DCFAB1C009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; + D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C01E0079CB00071108 /* StringIndexTokens.swift */; }; + D0F7B1C31E045C6A007EB8A5 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; + D0F7B1C41E045C6A007EB8A5 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; + D0F7B1C51E045C6A007EB8A5 /* MessageHistoryMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */; }; + D0F7B1C61E045C6A007EB8A5 /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; + D0F7B1C71E045C6A007EB8A5 /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; + D0F7B1C81E045C6A007EB8A5 /* MessageHistoryOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */; }; + D0F7B1C91E045C6A007EB8A5 /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; + D0F7B1CA1E045C6A007EB8A5 /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; + D0F7B1CB1E045C6A007EB8A5 /* ChatListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8641C58CB7F00037222 /* ChatListTable.swift */; }; + D0F7B1CC1E045C6A007EB8A5 /* ChatListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */; }; + D0F7B1CD1E045C6A007EB8A5 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; + D0F7B1CE1E045C6A007EB8A5 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; + D0F7B1CF1E045C6A007EB8A5 /* MetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86C1C5A0E5D00037222 /* MetadataTable.swift */; }; + D0F7B1D01E045C6A007EB8A5 /* KeychainTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */; }; + D0F7B1D11E045C6A007EB8A5 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; + D0F7B1D21E045C6A007EB8A5 /* PeerNotificationSettingsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */; }; + D0F7B1D31E045C6A007EB8A5 /* CachedPeerDataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */; }; + D0F7B1D41E045C6A007EB8A5 /* PeerPresenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120F71DA53FF4006A2A60 /* PeerPresenceTable.swift */; }; + D0F7B1D51E045C6A007EB8A5 /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; + D0F7B1D61E045C6A007EB8A5 /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; + D0F7B1D71E045C6A007EB8A5 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; + D0F7B1D81E045C6A007EB8A5 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; + D0F7B1D91E045C6A007EB8A5 /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; + D0F7B1DA1E045C6A007EB8A5 /* RatingTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08D451E1D5D2CA700A7428A /* RatingTable.swift */; }; + D0F7B1DB1E045C6A007EB8A5 /* ItemCollectionInfoTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */; }; + D0F7B1DC1E045C6A007EB8A5 /* ItemCollectionItemTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */; }; + D0F7B1DD1E045C6A007EB8A5 /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; + D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; + D0F7B1DF1E045C6A007EB8A5 /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; + D0F7B1E01E045C6A007EB8A5 /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; + D0F7B1E11E045C6A007EB8A5 /* PeerNameTokenIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */; }; + D0F7B1E21E045C6A007EB8A5 /* PeerNameIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */; }; D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */; }; D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; @@ -739,83 +742,86 @@ buildActionMask = 2147483647; files = ( D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, + D0F7B1DF1E045C6A007EB8A5 /* ItemCacheMetaTable.swift in Sources */, + D0F7B1C41E045C6A007EB8A5 /* GlobalMessageIdsTable.swift in Sources */, D073CE7E1DCBF3B4007511FD /* PeerNotificationSettings.swift in Sources */, + D0F7B1DC1E045C6A007EB8A5 /* ItemCollectionItemTable.swift in Sources */, + D0F7B1C71E045C6A007EB8A5 /* MessageHistoryTable.swift in Sources */, D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, D0B418251D7DFE0C004562A4 /* RedBlackTree.swift in Sources */, - D073CE851DCBF3BB007511FD /* GlobalMessageIdsTable.swift in Sources */, - D073CE9C1DCBF3BB007511FD /* ItemCollectionInfoTable.swift in Sources */, + D0F7B1C61E045C6A007EB8A5 /* MessageHistoryIndexTable.swift in Sources */, + D0F7B1CD1E045C6A007EB8A5 /* MessageMediaTable.swift in Sources */, D0B844071DAB91B5005F29E1 /* MediaResource.swift in Sources */, + D0F7B1D71E045C6A007EB8A5 /* MessageHistoryReadStateTable.swift in Sources */, D0B418261D7DFE0C004562A4 /* MappedFile.swift in Sources */, D073CE771DCBF3B4007511FD /* Media.swift in Sources */, - D073CE961DCBF3BB007511FD /* PeerChatStateTable.swift in Sources */, - D073CE971DCBF3BB007511FD /* ContactTable.swift in Sources */, D073CE791DCBF3B4007511FD /* ChatListHole.swift in Sources */, D0B4181C1D7DFDF1004562A4 /* SQLite-Bridging.m in Sources */, - D073CE8E1DCBF3BB007511FD /* MessageMediaTable.swift in Sources */, - D073CE9E1DCBF3BB007511FD /* PeerChatInterfaceStateTable.swift in Sources */, - D073CE871DCBF3BB007511FD /* MessageHistoryIndexTable.swift in Sources */, D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */, + D0F7B1D21E045C6A007EB8A5 /* PeerNotificationSettingsTable.swift in Sources */, D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */, - D073CE911DCBF3BB007511FD /* KeychainTable.swift in Sources */, - D073CE9D1DCBF3BB007511FD /* ItemCollectionItemTable.swift in Sources */, + D0F7B1C91E045C6A007EB8A5 /* MessageHistoryTagsTable.swift in Sources */, + D0F7B1CE1E045C6A007EB8A5 /* MediaCleanupTable.swift in Sources */, D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */, D0B4184D1D7DFE20004562A4 /* ContactPeersView.swift in Sources */, + D0F7B1CC1E045C6A007EB8A5 /* ChatListIndexTable.swift in Sources */, D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */, - D073CE8A1DCBF3BB007511FD /* MessageHistoryTagsTable.swift in Sources */, - D073CE861DCBF3BB007511FD /* MessageHistoryMetadataTable.swift in Sources */, + D0F7B1D31E045C6A007EB8A5 /* CachedPeerDataTable.swift in Sources */, D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */, + D0F7B1D51E045C6A007EB8A5 /* PeerChatStateTable.swift in Sources */, + D0F7B1C51E045C6A007EB8A5 /* MessageHistoryMetadataTable.swift in Sources */, D073CE781DCBF3B4007511FD /* MessageHistoryHole.swift in Sources */, - D0F3CC751DDE2845008148FA /* ItemCacheMetaTable.swift in Sources */, + D0F7B1C31E045C6A007EB8A5 /* Table.swift in Sources */, + D0F7B1CF1E045C6A007EB8A5 /* MetadataTable.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, - D073CE891DCBF3BB007511FD /* MessageHistoryOperation.swift in Sources */, D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */, + D0F7B1D91E045C6A007EB8A5 /* OrderStatisticTable.swift in Sources */, D073CEA01DCBF3C1007511FD /* ItemCollectionsView.swift in Sources */, D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */, + D0F7B1D61E045C6A007EB8A5 /* ContactTable.swift in Sources */, + D0F7B1D11E045C6A007EB8A5 /* PeerTable.swift in Sources */, + D0F7B1DD1E045C6A007EB8A5 /* PeerChatInterfaceStateTable.swift in Sources */, D098C6F21D7E1201007784E4 /* Database.swift in Sources */, - D073CE951DCBF3BB007511FD /* PeerPresenceTable.swift in Sources */, - D073CE841DCBF3BB007511FD /* Table.swift in Sources */, D0B844061DAB91B5005F29E1 /* MediaResourceStatus.swift in Sources */, + D0F7B1DB1E045C6A007EB8A5 /* ItemCollectionInfoTable.swift in Sources */, + D0F7B1D41E045C6A007EB8A5 /* PeerPresenceTable.swift in Sources */, D073CE761DCBF3B4007511FD /* IntermediateMessage.swift in Sources */, D0B4185A1D7DFE29004562A4 /* ViewTracker.swift in Sources */, - D073CE881DCBF3BB007511FD /* MessageHistoryTable.swift in Sources */, + D0F7B1E21E045C6A007EB8A5 /* PeerNameIndexTable.swift in Sources */, D073CE7C1DCBF3B4007511FD /* CachedPeerData.swift in Sources */, + D0F7B1DA1E045C6A007EB8A5 /* RatingTable.swift in Sources */, D073CE9F1DCBF3C1007511FD /* InitialMessageHistoryData.swift in Sources */, + D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopTaggedMessageIds.swift in Sources */, D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */, - D073CE8B1DCBF3BB007511FD /* MessageHistoryUnsentTable.swift in Sources */, D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, + D0F7B1E01E045C6A007EB8A5 /* ItemCacheTable.swift in Sources */, + D0F7B1E11E045C6A007EB8A5 /* PeerNameTokenIndexTable.swift in Sources */, D0B418241D7DFE0C004562A4 /* SimpleSet.swift in Sources */, D073CE7B1DCBF3B4007511FD /* PeerNameIndexRepresentation.swift in Sources */, D073CE7D1DCBF3B4007511FD /* PeerPresence.swift in Sources */, - D073CE9B1DCBF3BB007511FD /* RatingTable.swift in Sources */, D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, - D073CE9A1DCBF3BB007511FD /* OrderStatisticTable.swift in Sources */, D073CE751DCBF3B4007511FD /* Message.swift in Sources */, - D073CE901DCBF3BB007511FD /* MetadataTable.swift in Sources */, - D073CE941DCBF3BB007511FD /* CachedPeerDataTable.swift in Sources */, + D0F7B1CB1E045C6A007EB8A5 /* ChatListTable.swift in Sources */, + D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */, + D0F7B1CA1E045C6A007EB8A5 /* MessageHistoryUnsentTable.swift in Sources */, + D0F7B1D01E045C6A007EB8A5 /* KeychainTable.swift in Sources */, D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */, + D0F7B1D81E045C6A007EB8A5 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */, D0B418581D7DFE29004562A4 /* SeedConfiguration.swift in Sources */, D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */, D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */, - D073CE931DCBF3BB007511FD /* PeerNotificationSettingsTable.swift in Sources */, D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */, D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, - D073CE8C1DCBF3BB007511FD /* ChatListTable.swift in Sources */, - D073CE921DCBF3BB007511FD /* PeerTable.swift in Sources */, - D0F7AB331DCFAB1C009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */, D073CE741DCBF3B4007511FD /* Peer.swift in Sources */, D073CE7A1DCBF3B4007511FD /* PeerReadState.swift in Sources */, D073CE801DCBF3B4007511FD /* PeerChatInterfaceState.swift in Sources */, + D0F7B1C81E045C6A007EB8A5 /* MessageHistoryOperation.swift in Sources */, D0B844031DAB91A7005F29E1 /* PeerView.swift in Sources */, D0B418301D7DFE16004562A4 /* ValueBoxKey.swift in Sources */, - D073CE8F1DCBF3BB007511FD /* MediaCleanupTable.swift in Sources */, D0B418221D7DFE0C004562A4 /* Coding.swift in Sources */, D0B4184B1D7DFE20004562A4 /* SynchronizePeerReadStatesView.swift in Sources */, - D073CE991DCBF3BB007511FD /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, - D073CE981DCBF3BB007511FD /* MessageHistoryReadStateTable.swift in Sources */, - D0F3CC761DDE2845008148FA /* ItemCacheTable.swift in Sources */, - D073CE8D1DCBF3BB007511FD /* ChatListIndexTable.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; From 6b78eb5c2d655ac3164be9ff5471cb3a189658eb Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 22 Dec 2016 03:18:02 +0300 Subject: [PATCH 064/427] no message --- Postbox.xcodeproj/project.pbxproj | 12 +- Postbox/Message.swift | 6 + Postbox/MessageHistoryView.swift | 70 ++++++++++- Postbox/PeerChatTopIndexableMessageIds.swift | 116 +++++++++++++++++++ Postbox/PeerChatTopTaggedMessageIds.swift | 19 --- Postbox/Postbox.swift | 39 +++++-- 6 files changed, 224 insertions(+), 38 deletions(-) create mode 100644 Postbox/PeerChatTopIndexableMessageIds.swift delete mode 100644 Postbox/PeerChatTopTaggedMessageIds.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index dcf5db850e..40b6e0bbdd 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -147,7 +147,7 @@ D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; D0F3CC721DDE1CDC008148FA /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; - D0F7AB321DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; + D0F7AB321DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */; }; D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C01E0079CB00071108 /* StringIndexTokens.swift */; }; D0F7B1C31E045C6A007EB8A5 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; D0F7B1C41E045C6A007EB8A5 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; @@ -176,7 +176,7 @@ D0F7B1DB1E045C6A007EB8A5 /* ItemCollectionInfoTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */; }; D0F7B1DC1E045C6A007EB8A5 /* ItemCollectionItemTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */; }; D0F7B1DD1E045C6A007EB8A5 /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; - D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopTaggedMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */; }; + D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopIndexableMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */; }; D0F7B1DF1E045C6A007EB8A5 /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; D0F7B1E01E045C6A007EB8A5 /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; D0F7B1E11E045C6A007EB8A5 /* PeerNameTokenIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */; }; @@ -299,7 +299,7 @@ D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = ""; }; D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheTable.swift; sourceTree = ""; }; D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheMetaTable.swift; sourceTree = ""; }; - D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatTopTaggedMessageIds.swift; sourceTree = ""; }; + D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatTopIndexableMessageIds.swift; sourceTree = ""; }; D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageMediaTable.swift; sourceTree = ""; }; D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTableTests.swift; sourceTree = ""; }; D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaCleanupTable.swift; sourceTree = ""; }; @@ -429,7 +429,7 @@ D021E0D51DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift */, D021E0D71DB4FD1300C6B04F /* ItemCollectionItemTable.swift */, D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */, - D0F7AB311DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift */, + D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */, D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */, D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */, D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */, @@ -791,7 +791,7 @@ D073CE7C1DCBF3B4007511FD /* CachedPeerData.swift in Sources */, D0F7B1DA1E045C6A007EB8A5 /* RatingTable.swift in Sources */, D073CE9F1DCBF3C1007511FD /* InitialMessageHistoryData.swift in Sources */, - D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopTaggedMessageIds.swift in Sources */, + D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopIndexableMessageIds.swift in Sources */, D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */, D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, @@ -898,7 +898,7 @@ D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */, D044CA2C1C617E2D002160FF /* MessageHistoryMetadataTable.swift in Sources */, D03120FC1DA55427006A2A60 /* PeerNotificationSettings.swift in Sources */, - D0F7AB321DCFAB18009AD9A1 /* PeerChatTopTaggedMessageIds.swift in Sources */, + D0F7AB321DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift in Sources */, D03BCCF81C73561C0097A291 /* Table.swift in Sources */, D021E0DC1DB5237C00C6B04F /* ItemCollectionsView.swift in Sources */, D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 3b03f86136..886401447c 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -205,6 +205,10 @@ public struct MessageFlags: OptionSet { rawValue |= MessageFlags.Personal.rawValue } + if flags.contains(StoreMessageFlags.TopIndexable) { + rawValue |= MessageFlags.TopIndexable.rawValue + } + self.rawValue = rawValue } @@ -212,6 +216,7 @@ public struct MessageFlags: OptionSet { public static let Failed = MessageFlags(rawValue: 2) public static let Incoming = MessageFlags(rawValue: 4) public static let Personal = MessageFlags(rawValue: 8) + public static let TopIndexable = MessageFlags(rawValue: 16) } public struct StoreMessageForwardInfo { @@ -320,6 +325,7 @@ public struct StoreMessageFlags: OptionSet { public static let Failed = StoreMessageFlags(rawValue: 2) public static let Incoming = StoreMessageFlags(rawValue: 4) public static let Personal = StoreMessageFlags(rawValue: 8) + public static let TopIndexable = StoreMessageFlags(rawValue: 16) } public enum StoreMessageId { diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index d82a30d75a..a7f8988516 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -137,6 +137,20 @@ final class MutableMessageHistoryViewReplayContext { } } +enum MessageHistoryTopTaggedMessage { + case message(Message) + case intermediate(IntermediateMessage) + + var id: MessageId { + switch self { + case let .message(message): + return message.id + case let .intermediate(message): + return message.id + } + } +} + final class MutableMessageHistoryView { private(set) var id: MessageHistoryViewId let tagMask: MessageTags? @@ -148,7 +162,9 @@ final class MutableMessageHistoryView { fileprivate var entries: [MutableMessageHistoryEntry] fileprivate let fillCount: Int - init(id: MessageHistoryViewId, anchorIndex: MessageHistoryAnchorIndex, combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int) { + fileprivate var topTaggedMessages: [MessageId.Namespace: MessageHistoryTopTaggedMessage?] + + init(id: MessageHistoryViewId, anchorIndex: MessageHistoryAnchorIndex, combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int, topTaggedMessages: [MessageId.Namespace: MessageHistoryTopTaggedMessage?]) { self.id = id self.anchorIndex = anchorIndex self.combinedReadState = combinedReadState @@ -158,6 +174,7 @@ final class MutableMessageHistoryView { self.later = later self.tagMask = tagMask self.fillCount = count + self.topTaggedMessages = topTaggedMessages } func incrementVersion() { @@ -246,9 +263,6 @@ final class MutableMessageHistoryView { self.earlier = earlier self.later = later - /*let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) - - let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count)*/ return true } @@ -380,6 +394,31 @@ final class MutableMessageHistoryView { } } + for operation in operations { + switch operation { + case let .InsertMessage(message): + if message.flags.contains(.TopIndexable) { + if let currentTopMessage = self.topTaggedMessages[message.id.namespace] { + if currentTopMessage == nil || currentTopMessage!.id < message.id { + self.topTaggedMessages[message.id.namespace] = MessageHistoryTopTaggedMessage.intermediate(message) + hasChanges = true + } + } + } + case let .Remove(indices): + if !self.topTaggedMessages.isEmpty { + for index in indices { + if let maybeCurrentTopMessage = self.topTaggedMessages[index.id.namespace], let currentTopMessage = maybeCurrentTopMessage, index.id == currentTopMessage.id { + let item: MessageHistoryTopTaggedMessage? = nil + self.topTaggedMessages[index.id.namespace] = item + } + } + } + default: + break + } + } + return hasChanges } @@ -548,11 +587,18 @@ final class MutableMessageHistoryView { self.later = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) } - for i in 0 ..< self.entries.count { + for i in 0 ..< self.entries.count { if case let .IntermediateMessageEntry(intermediateMessage, location) = self.entries[i] { self.entries[i] = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) } } + + for namespace in self.topTaggedMessages.keys { + if let entry = self.topTaggedMessages[namespace]!, case let .intermediate(message) = entry { + let item: MessageHistoryTopTaggedMessage? = .message(renderIntermediateMessage(message)) + self.topTaggedMessages[namespace] = item + } + } } func firstHole() -> (MessageHistoryHole, HoleFillDirection)? { @@ -616,6 +662,7 @@ public final class MessageHistoryView { public let entries: [MessageHistoryEntry] public let maxReadIndex: MessageIndex? public let combinedReadState: CombinedPeerReadState? + public let topTaggedMessages: [Message] init(_ mutableView: MutableMessageHistoryView) { self.id = mutableView.id @@ -653,6 +700,19 @@ public final class MessageHistoryView { } self.entries = entries + var topTaggedMessages: [Message] = [] + for (namespace, message) in mutableView.topTaggedMessages { + if let message = message { + switch message { + case let .message(message): + topTaggedMessages.append(message) + default: + assertionFailure("unexpected intermediate tagged message entry in MessageHistoryView.init()") + } + } + } + self.topTaggedMessages = topTaggedMessages + self.earlierId = mutableView.earlier?.index self.laterId = mutableView.later?.index diff --git a/Postbox/PeerChatTopIndexableMessageIds.swift b/Postbox/PeerChatTopIndexableMessageIds.swift new file mode 100644 index 0000000000..9f4bf66eab --- /dev/null +++ b/Postbox/PeerChatTopIndexableMessageIds.swift @@ -0,0 +1,116 @@ +import Foundation + +private struct PeerChatTopTaggedUpdateRecord: Hashable { + let peerId: PeerId + let namespace: MessageId.Namespace + + static func ==(lhs: PeerChatTopTaggedUpdateRecord, rhs: PeerChatTopTaggedUpdateRecord) -> Bool { + return lhs.peerId == rhs.peerId && lhs.namespace == rhs.namespace + } + + var hashValue: Int { + return self.peerId.hashValue + } +} + +final class PeerChatTopTaggedMessageIdsTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private var cachedTopIds: [PeerId: [MessageId.Namespace: MessageId?]] = [:] + private var updatedPeerIds = Set() + + private let sharedKey = ValueBoxKey(length: 8 + 4 + 4) + + private func key(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: peerId.toInt64()) + self.sharedKey.setInt32(8, value: namespace) + return self.sharedKey + } + + func get(peerId: PeerId, namespace: MessageId.Namespace) -> MessageId? { + if let cachedDict = self.cachedTopIds[peerId] { + if let maybeCachedId = cachedDict[namespace] { + return maybeCachedId + } else { + if let value = self.valueBox.get(self.table, key: self.key(peerId: peerId, namespace: namespace)) { + var messageIdId: Int32 = 0 + value.read(&messageIdId, offset: 0, length: 4) + self.cachedTopIds[peerId]![namespace] = MessageId(peerId: peerId, namespace: namespace, id: messageIdId) + return MessageId(peerId: peerId, namespace: namespace, id: messageIdId) + } else { + let item: MessageId? = nil + self.cachedTopIds[peerId]![namespace] = item + return nil + } + } + } else { + if let value = self.valueBox.get(self.table, key: self.key(peerId: peerId, namespace: namespace)) { + var messageIdId: Int32 = 0 + value.read(&messageIdId, offset: 0, length: 4) + self.cachedTopIds[peerId] = [namespace: MessageId(peerId: peerId, namespace: namespace, id: messageIdId)] + return MessageId(peerId: peerId, namespace: namespace, id: messageIdId) + } else { + let item: MessageId? = nil + self.cachedTopIds[peerId] = [namespace: item] + return nil + } + } + } + + private func set(peerId: PeerId, namespace: MessageId.Namespace, id: MessageId?) { + if let _ = self.cachedTopIds[peerId] { + self.cachedTopIds[peerId]![namespace] = id + } else { + self.cachedTopIds[peerId] = [namespace: id] + } + self.updatedPeerIds.insert(PeerChatTopTaggedUpdateRecord(peerId: peerId, namespace: namespace)) + } + + func replay(historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]]) { + var updatedRecords = Set() + for (peerId, operations) in historyOperationsByPeerId { + for operation in operations { + switch operation { + case let .InsertMessage(message): + if message.flags.contains(.TopIndexable) { + let currentTopMessageId = self.get(peerId: message.id.peerId, namespace: message.id.namespace) + if currentTopMessageId == nil || currentTopMessageId! < message.id { + self.set(peerId: message.id.peerId, namespace: message.id.namespace, id: message.id) + } + } + case let .Remove(indices): + for index in indices { + if let _ = self.get(peerId: index.id.peerId, namespace: index.id.namespace) { + self.set(peerId: index.id.peerId, namespace: index.id.namespace, id: nil) + } + } + default: + break + } + } + } + } + + override func clearMemoryCache() { + assert(self.updatedPeerIds.isEmpty) + self.cachedTopIds.removeAll() + } + + override func beforeCommit() { + if !self.updatedPeerIds.isEmpty { + for record in self.updatedPeerIds { + if let cachedDict = self.cachedTopIds[record.peerId], let maybeMessageId = cachedDict[record.namespace] { + if let maybeMessageId = maybeMessageId { + var messageIdId: Int32 = maybeMessageId.id + self.valueBox.set(self.table, key: self.key(peerId: record.peerId, namespace: record.namespace), value: MemoryBuffer(memory: &messageIdId, capacity: 4, length: 4, freeWhenDone: false)) + } else { + self.valueBox.remove(self.table, key: self.key(peerId: record.peerId, namespace: record.namespace)) + } + } + } + self.updatedPeerIds.removeAll() + } + } +} diff --git a/Postbox/PeerChatTopTaggedMessageIds.swift b/Postbox/PeerChatTopTaggedMessageIds.swift deleted file mode 100644 index 2b3d3a9c7b..0000000000 --- a/Postbox/PeerChatTopTaggedMessageIds.swift +++ /dev/null @@ -1,19 +0,0 @@ -import Foundation - -final class PeerChatTopTaggedMessageIdsTable: Table { - static func tableSpec(_ id: Int32) -> ValueBoxTable { - return ValueBoxTable(id: id, keyType: .binary) - } - - private var cachedTopIds: [PeerId: [MessageId.Namespace: MessageId?]] = [:] - private var updatedPeerIds = Set() - - private let sharedKey = ValueBoxKey(length: 8 + 4 + 4) - - override func beforeCommit() { - for peerId in self.updatedPeerIds { - - } - self.updatedPeerIds.removeAll() - } -} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index f3981b51db..948f7db184 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -268,6 +268,7 @@ public final class Postbox { var itemCacheTable: ItemCacheTable! var peerNameTokenIndexTable: PeerNameTokenIndexTable! var peerNameIndexTable: PeerNameIndexTable! + var peerChatTopTaggedMessageIdsTable: PeerChatTopTaggedMessageIdsTable! //temporary var peerRatingTable: RatingTable! @@ -383,6 +384,7 @@ public final class Postbox { self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8), peerNameIndexTable: self.peerNameIndexTable) self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) + self.peerChatTopTaggedMessageIdsTable = PeerChatTopTaggedMessageIdsTable(valueBox: self.valueBox, table: PeerChatTopTaggedMessageIdsTable.tableSpec(28)) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -410,6 +412,7 @@ public final class Postbox { self.tables.append(self.itemCacheTable) self.tables.append(self.peerNameIndexTable) self.tables.append(self.peerNameTokenIndexTable) + self.tables.append(self.peerChatTopTaggedMessageIdsTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() @@ -786,6 +789,8 @@ public final class Postbox { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } + self.peerChatTopTaggedMessageIdsTable.replay(historyOperationsByPeerId: self.currentOperationsByPeerId) + let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? @@ -974,29 +979,29 @@ public final class Postbox { } } - public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { return self.modify(userInteractive: true, { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let maxReadIndex = self.messageHistoryTable.maxReadIndex(peerId) { index = maxReadIndex } - return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, tagMask: tagMask) + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask) }) |> switchToLatest } - public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let anchorIndex = self.messageHistoryTable.anchorIndex(messageId) { index = anchorIndex } - return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, tagMask: tagMask) + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask) } |> switchToLatest } - public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in - return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, tagMask: tagMask) + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask) } |> switchToLatest } @@ -1026,12 +1031,30 @@ public final class Postbox { } } - private func syncAroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + private func syncAroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { let startTime = CFAbsoluteTimeGetCurrent() let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) + var topTaggedMessages: [MessageId.Namespace: MessageHistoryTopTaggedMessage?] = [:] + for namespace in topTaggedMessageIdNamespaces { + if let messageId = self.peerChatTopTaggedMessageIdsTable.get(peerId: peerId, namespace: namespace) { + if let indexEntry = self.messageHistoryIndexTable.get(messageId), case let .Message(index) = indexEntry { + if let message = self.messageHistoryTable.getMessage(index) { + topTaggedMessages[namespace] = MessageHistoryTopTaggedMessage.intermediate(message) + } else { + assertionFailure() + } + } else { + assertionFailure() + } + } else { + let item: MessageHistoryTopTaggedMessage? = nil + topTaggedMessages[namespace] = item + } + } + + let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count, topTaggedMessages: topTaggedMessages) mutableView.render(self.renderIntermediateMessage) let initialUpdateType: ViewUpdateType From 5889a687334e02523d0cf921525cde06b1ce2bc8 Mon Sep 17 00:00:00 2001 From: overtake Date: Thu, 22 Dec 2016 13:18:51 +0300 Subject: [PATCH 065/427] typings to core --- .../xcschemes/xcschememanagement.plist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist index f57f8370f6..1d47b86ce2 100644 --- a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist @@ -7,12 +7,12 @@ Postbox.xcscheme orderHint - 10 + 7 PostboxMac.xcscheme orderHint - 11 + 8 SuppressBuildableAutocreation From 8fdd767461acc13e792018b1e90c61c6e1f43409 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 27 Dec 2016 15:13:12 +0300 Subject: [PATCH 066/427] no message --- Postbox/MessageHistoryView.swift | 26 ++++++++++++++++++++++++-- Postbox/Postbox.swift | 24 ++++++++++++++++-------- Postbox/ViewTracker.swift | 4 ++-- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index a7f8988516..a5b685dfa8 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -1,5 +1,13 @@ import Foundation +public enum AdditionalMessageHistoryViewData { + case cachedPeerData(PeerId) +} + +public enum AdditionalMessageHistoryViewDataEntry { + case cachedPeerData(PeerId, CachedPeerData?) +} + public struct MessageHistoryViewId: Equatable { let peerId: PeerId let id: Int @@ -163,8 +171,9 @@ final class MutableMessageHistoryView { fileprivate let fillCount: Int fileprivate var topTaggedMessages: [MessageId.Namespace: MessageHistoryTopTaggedMessage?] + fileprivate var additionalDatas: [AdditionalMessageHistoryViewDataEntry] - init(id: MessageHistoryViewId, anchorIndex: MessageHistoryAnchorIndex, combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int, topTaggedMessages: [MessageId.Namespace: MessageHistoryTopTaggedMessage?]) { + init(id: MessageHistoryViewId, anchorIndex: MessageHistoryAnchorIndex, combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int, topTaggedMessages: [MessageId.Namespace: MessageHistoryTopTaggedMessage?], additionalDatas: [AdditionalMessageHistoryViewDataEntry]) { self.id = id self.anchorIndex = anchorIndex self.combinedReadState = combinedReadState @@ -175,6 +184,7 @@ final class MutableMessageHistoryView { self.tagMask = tagMask self.fillCount = count self.topTaggedMessages = topTaggedMessages + self.additionalDatas = additionalDatas } func incrementVersion() { @@ -266,7 +276,7 @@ final class MutableMessageHistoryView { return true } - func replay(_ operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], context: MutableMessageHistoryViewReplayContext, renderIntermediateMessage: (IntermediateMessage) -> Message) -> Bool { + func replay(_ operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], updatedCachedPeerData: [PeerId: CachedPeerData], context: MutableMessageHistoryViewReplayContext, renderIntermediateMessage: (IntermediateMessage) -> Message) -> Bool { let tagMask = self.tagMask let unwrappedTagMask: UInt32 = tagMask?.rawValue ?? 0 @@ -419,6 +429,16 @@ final class MutableMessageHistoryView { } } + for i in 0 ..< self.additionalDatas.count { + switch self.additionalDatas[i] { + case let .cachedPeerData(peerId, data): + if let updatedData = updatedCachedPeerData[peerId] { + self.additionalDatas[i] = .cachedPeerData(peerId, updatedData) + hasChanges = true + } + } + } + return hasChanges } @@ -663,6 +683,7 @@ public final class MessageHistoryView { public let maxReadIndex: MessageIndex? public let combinedReadState: CombinedPeerReadState? public let topTaggedMessages: [Message] + public let additionalData: [AdditionalMessageHistoryViewDataEntry] init(_ mutableView: MutableMessageHistoryView) { self.id = mutableView.id @@ -712,6 +733,7 @@ public final class MessageHistoryView { } } self.topTaggedMessages = topTaggedMessages + self.additionalData = mutableView.additionalDatas self.earlierId = mutableView.earlier?.index self.laterId = mutableView.later?.index diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 948f7db184..b94c6902e6 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -979,29 +979,29 @@ public final class Postbox { } } - public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil, additionalData: [AdditionalMessageHistoryViewData]) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { return self.modify(userInteractive: true, { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let maxReadIndex = self.messageHistoryTable.maxReadIndex(peerId) { index = maxReadIndex } - return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask) + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask, additionalData: additionalData) }) |> switchToLatest } - public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil, additionalData: [AdditionalMessageHistoryViewData] = []) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let anchorIndex = self.messageHistoryTable.anchorIndex(messageId) { index = anchorIndex } - return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask) + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask, additionalData: additionalData) } |> switchToLatest } - public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil, additionalData: [AdditionalMessageHistoryViewData] = []) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in - return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask) + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask, additionalData: additionalData) } |> switchToLatest } @@ -1031,7 +1031,7 @@ public final class Postbox { } } - private func syncAroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + private func syncAroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags?, additionalData: [AdditionalMessageHistoryViewData]) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { let startTime = CFAbsoluteTimeGetCurrent() let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") @@ -1054,7 +1054,15 @@ public final class Postbox { } } - let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count, topTaggedMessages: topTaggedMessages) + var additionalDataEntries: [AdditionalMessageHistoryViewDataEntry] = [] + for data in additionalData { + switch data { + case let .cachedPeerData(peerId): + additionalDataEntries.append(.cachedPeerData(peerId, self.cachedPeerDataTable.get(peerId))) + } + } + + let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count, topTaggedMessages: topTaggedMessages, additionalDatas: additionalDataEntries) mutableView.render(self.renderIntermediateMessage) let initialUpdateType: ViewUpdateType diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 12684afbf2..a0cc0a8a4f 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -240,7 +240,7 @@ final class ViewTracker { for (peerId, bag) in self.messageHistoryViews { var updateHoles = false let operations = transaction.currentOperationsByPeerId[peerId] - if operations != nil || !transaction.updatedMedia.isEmpty { + if operations != nil || !transaction.updatedMedia.isEmpty || !transaction.currentUpdatedCachedPeerData.isEmpty { updateHoles = true for (mutableView, pipe) in bag.copyItems() { let context = MutableMessageHistoryViewReplayContext() @@ -253,7 +253,7 @@ final class ViewTracker { updateType = .Generic } - if mutableView.replay(operations ?? [], holeFillDirections: transaction.peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: transaction.updatedMedia, context: context, renderIntermediateMessage: self.renderMessage) { + if mutableView.replay(operations ?? [], holeFillDirections: transaction.peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: transaction.updatedMedia, updatedCachedPeerData: transaction.currentUpdatedCachedPeerData, context: context, renderIntermediateMessage: self.renderMessage) { mutableView.complete(context: context, fetchEarlier: { index, count in return self.fetchEarlierHistoryEntries(peerId, index, count, mutableView.tagMask) }, fetchLater: { index, count in From af145a119421e64981b59ad91a740ab2be59208a Mon Sep 17 00:00:00 2001 From: overtake Date: Tue, 27 Dec 2016 19:39:02 +0300 Subject: [PATCH 067/427] channels --- .../xcschemes/xcschememanagement.plist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist index 1d47b86ce2..f92a60b4aa 100644 --- a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist @@ -7,12 +7,12 @@ Postbox.xcscheme orderHint - 7 + 5 PostboxMac.xcscheme orderHint - 8 + 6 SuppressBuildableAutocreation From 84c0a86d8f3a27f4f32f8b83bea0a9e10a29e4b4 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 5 Jan 2017 01:14:32 +0400 Subject: [PATCH 068/427] no message --- Postbox.xcodeproj/project.pbxproj | 16 +++ Postbox/ChatListIndexTable.swift | 99 +++++++++++++++-- Postbox/IntermediateMessage.swift | 4 +- Postbox/MediaBox.swift | 17 +++ Postbox/Message.swift | 16 ++- Postbox/MessageGloballyUniqueIdTable.swift | 43 ++++++++ Postbox/MessageHistoryMetadataTable.swift | 71 ++++++++++++ Postbox/MessageHistoryReadStateTable.swift | 102 +++++++++++------- Postbox/MessageHistoryTable.swift | 61 ++++++++--- Postbox/MessageHistoryView.swift | 12 +-- Postbox/PeerNameIndexTable.swift | 2 +- Postbox/PeerNotificationSettings.swift | 2 + Postbox/PeerNotificationSettingsTable.swift | 53 ++++++--- Postbox/PeerOperationLogMetadataTable.swift | 41 +++++++ Postbox/PeerOperationLogTable.swift | 53 +++++++++ Postbox/Postbox.swift | 52 +++++++-- Postbox/PostboxTransaction.swift | 12 ++- Postbox/UnreadMessageCountsView.swift | 69 ++++++++++++ Postbox/ValueBoxKey.swift | 18 ++-- Postbox/ViewTracker.swift | 25 ++++- PostboxTests/ChatListTableTests.swift | 14 ++- .../MessageHistoryIndexTableTests.swift | 6 +- PostboxTests/MessageHistoryTableTests.swift | 8 +- PostboxTests/ReadStateTableTests.swift | 4 +- 24 files changed, 686 insertions(+), 114 deletions(-) create mode 100644 Postbox/MessageGloballyUniqueIdTable.swift create mode 100644 Postbox/PeerOperationLogMetadataTable.swift create mode 100644 Postbox/PeerOperationLogTable.swift create mode 100644 Postbox/UnreadMessageCountsView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 40b6e0bbdd..b4aab6edf5 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -76,10 +76,14 @@ D098C6F21D7E1201007784E4 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = D098C6F01D7E11E9007784E4 /* Database.swift */; }; D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; + D0A18D671E16874D004C6734 /* UnreadMessageCountsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */; }; D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; D0AB0B8C1D65D488002C78E7 /* MessageHistoryHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */; }; D0AB0B8E1D65D49C002C78E7 /* ChatListHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */; }; D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */; }; + D0AD23271E194D1C00A7089A /* PeerOperationLogTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */; }; + D0AD23291E196B6400A7089A /* PeerOperationLogMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */; }; + D0AD232B1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AD232A1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift */; }; D0B418171D7DFAF3004562A4 /* PostboxMac.h in Headers */ = {isa = PBXBuildFile; fileRef = D0B418151D7DFAF3004562A4 /* PostboxMac.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0B4181C1D7DFDF1004562A4 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-ambiguous-macro -Wno-conditional-uninitialized -Wno-unused-const-variable -Wno-unused-function -Wno-unreachable-code"; }; }; @@ -260,10 +264,14 @@ D098C6F01D7E11E9007784E4 /* Database.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Database.swift; sourceTree = ""; }; D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessMediaResourceContext.swift; sourceTree = ""; }; D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTable.swift; sourceTree = ""; }; + D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnreadMessageCountsView.swift; sourceTree = ""; }; D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHolesView.swift; sourceTree = ""; }; D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatListHolesView.swift; sourceTree = ""; }; D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageIndicesView.swift; sourceTree = ""; }; + D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerOperationLogTable.swift; sourceTree = ""; }; + D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerOperationLogMetadataTable.swift; sourceTree = ""; }; + D0AD232A1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageGloballyUniqueIdTable.swift; sourceTree = ""; }; D0B418131D7DFAF2004562A4 /* PostboxMac.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = PostboxMac.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0B418151D7DFAF3004562A4 /* PostboxMac.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PostboxMac.h; sourceTree = ""; }; D0B418161D7DFAF3004562A4 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -404,6 +412,7 @@ children = ( D03BCCF71C73561C0097A291 /* Table.swift */, D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */, + D0AD232A1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift */, D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */, D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */, D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, @@ -434,6 +443,8 @@ D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */, D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */, D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */, + D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */, + D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */, ); name = Tables; sourceTree = ""; @@ -497,6 +508,7 @@ D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */, D0DF0C8E1D81A350008AEB01 /* PeerView.swift */, D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */, + D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */, ); name = Views; sourceTree = ""; @@ -853,6 +865,7 @@ D021E0D81DB4FD1300C6B04F /* ItemCollectionItemTable.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */, + D0AD23291E196B6400A7089A /* PeerOperationLogMetadataTable.swift in Sources */, D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, @@ -868,7 +881,9 @@ D07827C31E008F7300071108 /* PeerNameTokenIndexTable.swift in Sources */, D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, + D0AD232B1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift in Sources */, D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */, + D0AD23271E194D1C00A7089A /* PeerOperationLogTable.swift in Sources */, D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */, D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, @@ -884,6 +899,7 @@ D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */, D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, + D0A18D671E16874D004C6734 /* UnreadMessageCountsView.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */, D07827C11E0079CB00071108 /* StringIndexTokens.swift in Sources */, diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift index aa571dbfb3..138325fe3a 100644 --- a/Postbox/ChatListIndexTable.swift +++ b/Postbox/ChatListIndexTable.swift @@ -6,14 +6,20 @@ final class ChatListIndexTable: Table { } private let peerNameIndexTable: PeerNameIndexTable + private let metadataTable: MessageHistoryMetadataTable + private let readStateTable: MessageHistoryReadStateTable + private let notificationSettingsTable: PeerNotificationSettingsTable private let sharedKey = ValueBoxKey(length: 8) private var cachedIndices: [PeerId: MessageIndex?] = [:] private var updatedPreviousCachedIndices: [PeerId: MessageIndex?] = [:] - init(valueBox: ValueBox, table: ValueBoxTable, peerNameIndexTable: PeerNameIndexTable) { + init(valueBox: ValueBox, table: ValueBoxTable, peerNameIndexTable: PeerNameIndexTable, metadataTable: MessageHistoryMetadataTable, readStateTable: MessageHistoryReadStateTable, notificationSettingsTable: PeerNotificationSettingsTable) { self.peerNameIndexTable = peerNameIndexTable + self.metadataTable = metadataTable + self.readStateTable = readStateTable + self.notificationSettingsTable = notificationSettingsTable super.init(valueBox: valueBox, table: table) } @@ -58,16 +64,16 @@ final class ChatListIndexTable: Table { assert(self.updatedPreviousCachedIndices.isEmpty) } - override func beforeCommit() { - if !self.updatedPreviousCachedIndices.isEmpty { - var addedPeerIds = Set() - var removedPeerIds = Set() + func commitWithTransactionUnreadCountDeltas(_ deltas: [PeerId: Int32], transactionParticipationInTotalUnreadCountUpdates: (added: Set, removed: Set), updatedTotalUnreadCount: inout Int32?) { + if !self.updatedPreviousCachedIndices.isEmpty || !deltas.isEmpty || !transactionParticipationInTotalUnreadCountUpdates.added.isEmpty || !transactionParticipationInTotalUnreadCountUpdates.removed.isEmpty { + var addedChatListPeerIds = Set() + var removedChatListPeerIds = Set() for (peerId, previousIndex) in self.updatedPreviousCachedIndices { let index = self.cachedIndices[peerId]! if let index = index { if previousIndex == nil { - addedPeerIds.insert(peerId) + addedChatListPeerIds.insert(peerId) } let writeBuffer = WriteBuffer() @@ -80,7 +86,7 @@ final class ChatListIndexTable: Table { self.valueBox.set(self.table, key: self.key(index.id.peerId), value: writeBuffer.readBufferNoCopy()) } else { if previousIndex != nil { - removedPeerIds.insert(peerId) + removedChatListPeerIds.insert(peerId) } self.valueBox.remove(self.table, key: self.key(peerId)) @@ -88,13 +94,88 @@ final class ChatListIndexTable: Table { } self.updatedPreviousCachedIndices.removeAll() - for peerId in addedPeerIds { + let addedUnreadCountPeerIds = addedChatListPeerIds.union(transactionParticipationInTotalUnreadCountUpdates.added) + let removedUnreadCountPeerIds = removedChatListPeerIds.union(transactionParticipationInTotalUnreadCountUpdates.removed) + + var totalUnreadCount = self.metadataTable.getChatListTotalUnreadCount() + for (peerId, delta) in deltas { + if !addedUnreadCountPeerIds.contains(peerId) && !removedUnreadCountPeerIds.contains(peerId) { + if let _ = self.get(peerId), let notificationSettings = self.notificationSettingsTable.get(peerId), !notificationSettings.isRemovedFromTotalUnreadCount { + totalUnreadCount += delta + } + } + } + + for peerId in addedChatListPeerIds { self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.chats], includes: true) } - for peerId in removedPeerIds { + for peerId in addedUnreadCountPeerIds { + let addedToList = addedChatListPeerIds.contains(peerId) + let startedParticipationInUnreadCount = transactionParticipationInTotalUnreadCountUpdates.added.contains(peerId) + + if addedToList && startedParticipationInUnreadCount { + if let combinedState = self.readStateTable.getCombinedState(peerId) { + totalUnreadCount += combinedState.count + } + } else if addedToList { + if let notificationSettings = self.notificationSettingsTable.get(peerId), !notificationSettings.isRemovedFromTotalUnreadCount { + if let combinedState = self.readStateTable.getCombinedState(peerId) { + totalUnreadCount += combinedState.count + } + } + } else if startedParticipationInUnreadCount { + if let _ = self.get(peerId) { + if let combinedState = self.readStateTable.getCombinedState(peerId) { + totalUnreadCount += combinedState.count + } + } + } else { + assertionFailure() + } + } + + for peerId in removedChatListPeerIds { self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.chats], includes: false) } + + for peerId in removedUnreadCountPeerIds { + var currentPeerUnreadCount: Int32 = 0 + if let combinedState = self.readStateTable.getCombinedState(peerId) { + currentPeerUnreadCount = combinedState.count + } + + if let delta = deltas[peerId] { + currentPeerUnreadCount -= delta + } + + var removedFromList = removedChatListPeerIds.contains(peerId) + var removedFromParticipationInUnreadCount = transactionParticipationInTotalUnreadCountUpdates.removed.contains(peerId) + if removedFromList && removedFromParticipationInUnreadCount { + totalUnreadCount -= currentPeerUnreadCount + } else if removedFromList { + if let notificationSettings = self.notificationSettingsTable.get(peerId), !notificationSettings.isRemovedFromTotalUnreadCount { + totalUnreadCount -= currentPeerUnreadCount + } + } else if removedFromParticipationInUnreadCount { + if let _ = self.get(peerId) { + totalUnreadCount -= currentPeerUnreadCount + } + } else { + assertionFailure() + } + } + + //assert(totalUnreadCount >= 0) + + if self.metadataTable.getChatListTotalUnreadCount() != totalUnreadCount { + self.metadataTable.setChatListTotalUnreadCount(totalUnreadCount) + updatedTotalUnreadCount = totalUnreadCount + } } } + + override func beforeCommit() { + assert(self.updatedPreviousCachedIndices.isEmpty) + } } diff --git a/Postbox/IntermediateMessage.swift b/Postbox/IntermediateMessage.swift index d3b02068ce..21acc5d6a6 100644 --- a/Postbox/IntermediateMessage.swift +++ b/Postbox/IntermediateMessage.swift @@ -25,6 +25,7 @@ class IntermediateMessage { let stableId: UInt32 let stableVersion: UInt32 let id: MessageId + let globallyUniqueId: Int64? let timestamp: Int32 let flags: MessageFlags let tags: MessageTags @@ -35,10 +36,11 @@ class IntermediateMessage { let embeddedMediaData: ReadBuffer let referencedMedia: [MediaId] - init(stableId: UInt32, stableVersion: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { + init(stableId: UInt32, stableVersion: UInt32, id: MessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { self.stableId = stableId self.stableVersion = stableVersion self.id = id + self.globallyUniqueId = globallyUniqueId self.timestamp = timestamp self.flags = flags self.tags = tags diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index fc3ec43501..abcf5a552f 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -244,6 +244,23 @@ public final class MediaBox { } } + public func completedResourcePath(_ resource: MediaResource, pathExtension: String? = nil) -> String? { + let paths = self.storePathsForId(resource.id) + if let completeSize = fileSize(paths.complete) { + if let pathExtension = pathExtension { + let symlinkPath = paths.complete + ".\(pathExtension)" + if fileSize(symlinkPath) == nil { + let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) + } + return symlinkPath + } else { + return paths.complete + } + } else { + return nil + } + } + public func resourceData(_ resource: MediaResource, pathExtension: String? = nil, complete: Bool = true) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 886401447c..e2ad1c66d9 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -279,7 +279,9 @@ public extension MessageAttribute { public final class Message { public let stableId: UInt32 public let stableVersion: UInt32 + public let id: MessageId + public let globallyUniqueId: Int64? public let timestamp: Int32 public let flags: MessageFlags public let tags: MessageTags @@ -292,10 +294,11 @@ public final class Message { public let associatedMessages: SimpleDictionary public let associatedMessageIds: [MessageId] - public init(stableId: UInt32, stableVersion: UInt32, id: MessageId, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: MessageForwardInfo?, author: Peer?, text: String, attributes: [MessageAttribute], media: [Media], peers: SimpleDictionary, associatedMessages: SimpleDictionary, associatedMessageIds: [MessageId]) { + public init(stableId: UInt32, stableVersion: UInt32, id: MessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: MessageForwardInfo?, author: Peer?, text: String, attributes: [MessageAttribute], media: [Media], peers: SimpleDictionary, associatedMessages: SimpleDictionary, associatedMessageIds: [MessageId]) { self.stableId = stableId self.stableVersion = stableVersion self.id = id + self.globallyUniqueId = globallyUniqueId self.timestamp = timestamp self.flags = flags self.tags = tags @@ -345,6 +348,7 @@ public enum StoreMessageId { public final class StoreMessage { public let id: StoreMessageId public let timestamp: Int32 + public let globallyUniqueId: Int64? public let flags: StoreMessageFlags public let tags: MessageTags public let forwardInfo: StoreMessageForwardInfo? @@ -353,8 +357,9 @@ public final class StoreMessage { public let attributes: [MessageAttribute] public let media: [Media] - public init(id: MessageId, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + public init(id: MessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { self.id = .Id(id) + self.globallyUniqueId = globallyUniqueId self.timestamp = timestamp self.flags = flags self.tags = tags @@ -365,9 +370,10 @@ public final class StoreMessage { self.media = media } - public init(peerId: PeerId, namespace: MessageId.Namespace, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + public init(peerId: PeerId, namespace: MessageId.Namespace, globallyUniqueId: Int64?, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { self.id = .Partial(peerId, namespace) self.timestamp = timestamp + self.globallyUniqueId = globallyUniqueId self.flags = flags self.tags = tags self.forwardInfo = forwardInfo @@ -381,6 +387,7 @@ public final class StoreMessage { final class InternalStoreMessage { let id: MessageId let timestamp: Int32 + let globallyUniqueId: Int64? let flags: StoreMessageFlags let tags: MessageTags let forwardInfo: StoreMessageForwardInfo? @@ -389,9 +396,10 @@ final class InternalStoreMessage { let attributes: [MessageAttribute] let media: [Media] - init(id: MessageId, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + init(id: MessageId, timestamp: Int32, globallyUniqueId: Int64?, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { self.id = id self.timestamp = timestamp + self.globallyUniqueId = globallyUniqueId self.flags = flags self.tags = tags self.forwardInfo = forwardInfo diff --git a/Postbox/MessageGloballyUniqueIdTable.swift b/Postbox/MessageGloballyUniqueIdTable.swift new file mode 100644 index 0000000000..85bedd1b66 --- /dev/null +++ b/Postbox/MessageGloballyUniqueIdTable.swift @@ -0,0 +1,43 @@ +import Foundation + +final class MessageGloballyUniqueIdTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + + private let sharedKey = ValueBoxKey(length: 8) + private let sharedBuffer = WriteBuffer() + + private func key(_ id: Int64) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: id) + return self.sharedKey + } + + func set(_ globallyUniqueId: Int64, id: MessageId) { + self.sharedBuffer.reset() + var idPeerId: Int64 = id.peerId.toInt64() + var idNamespace: Int32 = id.namespace + var idId: Int32 = id.id + self.sharedBuffer.write(&idPeerId, offset: 0, length: 8) + self.sharedBuffer.write(&idNamespace, offset: 0, length: 4) + self.sharedBuffer.write(&idId, offset: 0, length: 4) + self.valueBox.set(self.table, key: self.key(globallyUniqueId), value: self.sharedBuffer) + } + + func get(_ globallyUniqueId: Int64) -> MessageId? { + if let value = self.valueBox.get(self.table, key: self.key(globallyUniqueId)) { + var idPeerId: Int64 = 0 + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + value.read(&idPeerId, offset: 0, length: 8) + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + return MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId) + } + return nil + } + + func remove(_ globallyUniqueId: Int64) { + self.valueBox.remove(self.table, key: self.key(globallyUniqueId)) + } +} diff --git a/Postbox/MessageHistoryMetadataTable.swift b/Postbox/MessageHistoryMetadataTable.swift index 7f34c663f8..4318505eca 100644 --- a/Postbox/MessageHistoryMetadataTable.swift +++ b/Postbox/MessageHistoryMetadataTable.swift @@ -5,6 +5,8 @@ private enum MetadataPrefix: Int8 { case PeerHistoryInitialized = 1 case PeerNextMessageIdByNamespace = 2 case NextStableMessageId = 3 + case ChatListTotalUnreadCount = 4 + case NextPeerOperationLogIndex = 5 } final class MessageHistoryMetadataTable: Table { @@ -25,6 +27,12 @@ final class MessageHistoryMetadataTable: Table { private var nextMessageStableId: UInt32? private var nextMessageStableIdUpdated = false + private var chatListTotalUnreadCount: Int32? + private var chatListTotalUnreadCountUpdated = false + + private var nextPeerOperationLogIndex: UInt32? + private var nextPeerOperationLogIndexUpdated = false + private func peerHistoryInitializedKey(_ id: PeerId) -> ValueBoxKey { self.sharedPeerHistoryInitializedKey.setInt64(0, value: id.toInt64()) self.sharedPeerHistoryInitializedKey.setInt8(8, value: MetadataPrefix.PeerHistoryInitialized.rawValue) @@ -142,6 +150,51 @@ final class MessageHistoryMetadataTable: Table { } } + func getNextPeerOperationLogIndex() -> UInt32 { + if let nextId = self.nextPeerOperationLogIndex { + self.nextPeerOperationLogIndex = nextId + 1 + self.nextPeerOperationLogIndexUpdated = true + return nextId + } else { + if let value = self.valueBox.get(self.table, key: self.key(.NextPeerOperationLogIndex)) { + var nextId: UInt32 = 0 + value.read(&nextId, offset: 0, length: 4) + self.nextPeerOperationLogIndex = nextId + 1 + self.nextPeerOperationLogIndexUpdated = true + return nextId + } else { + let nextId: UInt32 = 1 + self.nextPeerOperationLogIndex = nextId + 1 + self.nextPeerOperationLogIndexUpdated = true + return nextId + } + } + } + + func getChatListTotalUnreadCount() -> Int32 { + if let cached = self.chatListTotalUnreadCount { + return cached + } else { + if let value = self.valueBox.get(self.table, key: self.key(.ChatListTotalUnreadCount)) { + var count: Int32 = 0 + value.read(&count, offset: 0, length: 4) + self.chatListTotalUnreadCount = count + return count + } else { + self.chatListTotalUnreadCount = 0 + return 0 + } + } + } + + func setChatListTotalUnreadCount(_ value: Int32) { + let current = self.getChatListTotalUnreadCount() + if current != value { + self.chatListTotalUnreadCount = value + self.chatListTotalUnreadCountUpdated = true + } + } + override func clearMemoryCache() { self.initializedChatList = nil self.initializedHistoryPeerIds.removeAll() @@ -149,6 +202,8 @@ final class MessageHistoryMetadataTable: Table { self.updatedPeerNextMessageIdByNamespace.removeAll() self.nextMessageStableId = nil self.nextMessageStableIdUpdated = false + self.chatListTotalUnreadCount = nil + self.chatListTotalUnreadCountUpdated = false } override func beforeCommit() { @@ -174,5 +229,21 @@ final class MessageHistoryMetadataTable: Table { self.nextMessageStableIdUpdated = false } } + + if self.nextPeerOperationLogIndexUpdated { + if let nextPeerOperationLogIndex = self.nextPeerOperationLogIndex { + var nextId: UInt32 = nextPeerOperationLogIndex + self.valueBox.set(self.table, key: self.key(.NextPeerOperationLogIndex), value: MemoryBuffer(memory: &nextId, capacity: 4, length: 4, freeWhenDone: false)) + self.nextPeerOperationLogIndexUpdated = false + } + } + + if self.chatListTotalUnreadCountUpdated { + if let value = self.chatListTotalUnreadCount { + var count: Int32 = value + self.valueBox.set(self.table, key: self.key(.ChatListTotalUnreadCount), value: MemoryBuffer(memory: &count, capacity: 4, length: 4, freeWhenDone: false)) + } + self.chatListTotalUnreadCountUpdated = false + } } } diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index 0be6dfaa79..4a308040f0 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -21,7 +21,7 @@ final class MessageHistoryReadStateTable: Table { } private var cachedPeerReadStates: [PeerId: InternalPeerReadStates?] = [:] - private var updatedPeerIds = Set() + private var updatedInitialPeerReadStates: [PeerId: [MessageId.Namespace: PeerReadState]] = [:] private let sharedKey = ValueBoxKey(length: 8) @@ -74,29 +74,33 @@ final class MessageHistoryReadStateTable: Table { return nil } + private func markReadStatesAsUpdated(_ peerId: PeerId, namespaces: [MessageId.Namespace: PeerReadState]) { + if self.updatedInitialPeerReadStates[peerId] == nil { + self.updatedInitialPeerReadStates[peerId] = namespaces + } + } + func resetStates(_ peerId: PeerId, namespaces: [MessageId.Namespace: PeerReadState]) -> CombinedPeerReadState? { if traceReadStates { print("[ReadStateTable] resetStates peerId: \(peerId), namespaces: \(namespaces)") } - self.updatedPeerIds.insert(peerId) - if let states = self.get(peerId) { var updated = false for (namespace, state) in namespaces { if states.namespaces[namespace] == nil || states.namespaces[namespace]! != state { + self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces) updated = true } states.namespaces[namespace] = state } if updated { - self.updatedPeerIds.insert(peerId) return CombinedPeerReadState(states: states.namespaces.map({$0})) } else { return nil } } else { - self.updatedPeerIds.insert(peerId) + self.markReadStatesAsUpdated(peerId, namespaces: [:]) let states = InternalPeerReadStates(namespaces: namespaces) self.cachedPeerReadStates[peerId] = states return CombinedPeerReadState(states: states.namespaces.map({$0})) @@ -132,6 +136,8 @@ final class MessageHistoryReadStateTable: Table { } if addedUnreadCount != 0 { + self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces) + states.namespaces[namespace] = PeerReadState(maxIncomingReadId: currentState.maxIncomingReadId, maxOutgoingReadId: currentState.maxOutgoingReadId, maxKnownId: currentState.maxKnownId, count: currentState.count + addedUnreadCount) updated = true @@ -142,10 +148,6 @@ final class MessageHistoryReadStateTable: Table { } } - if updated { - self.updatedPeerIds.insert(peerId) - } - return (updated ? CombinedPeerReadState(states: states.namespaces.map({$0})) : nil, invalidated) } else { if traceReadStates { @@ -186,6 +188,8 @@ final class MessageHistoryReadStateTable: Table { invalidate = true } + self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces) + states.namespaces[namespace] = PeerReadState(maxIncomingReadId: currentState.maxIncomingReadId, maxOutgoingReadId: currentState.maxOutgoingReadId, maxKnownId: currentState.maxKnownId, count: currentState.count - knownCount) updated = true } else { @@ -193,10 +197,6 @@ final class MessageHistoryReadStateTable: Table { } } - if updated { - self.updatedPeerIds.insert(peerId) - } - return (updated ? CombinedPeerReadState(states: states.namespaces.map({$0})) : nil, invalidate) } else { return (nil, true) @@ -223,8 +223,9 @@ final class MessageHistoryReadStateTable: Table { } } + self.markReadStatesAsUpdated(messageId.peerId, namespaces: states.namespaces) + states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: messageId.id, maxOutgoingReadId: state.maxOutgoingReadId, maxKnownId: state.maxKnownId, count: state.count - Int32(deltaCount)) - self.updatedPeerIds.insert(messageId.peerId) return (CombinedPeerReadState(states: states.namespaces.map({$0})), holes) } } else { @@ -237,8 +238,9 @@ final class MessageHistoryReadStateTable: Table { func applyOutgoingMaxReadId(_ messageId: MessageId) -> (CombinedPeerReadState?, Bool) { if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] { if state.maxOutgoingReadId < messageId.id { + self.markReadStatesAsUpdated(messageId.peerId, namespaces: states.namespaces) + states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: state.maxIncomingReadId, maxOutgoingReadId: messageId.id, maxKnownId: state.maxKnownId, count: state.count) - self.updatedPeerIds.insert(messageId.peerId) return (CombinedPeerReadState(states: states.namespaces.map({$0})), false) } } else { @@ -258,35 +260,63 @@ final class MessageHistoryReadStateTable: Table { return (combinedState, holes ? .Push(thenSync: true) : .None) } + func transactionUnreadCountDeltas() -> [PeerId: Int32] { + var deltas: [PeerId: Int32] = [:] + for (id, initialNamespaces) in self.updatedInitialPeerReadStates { + var initialCount: Int32 = 0 + for (_, state) in initialNamespaces { + initialCount += state.count + } + + var updatedCount: Int32 = 0 + if let maybeStates = self.cachedPeerReadStates[id] { + if let states = maybeStates { + for (_, state) in states.namespaces { + updatedCount += state.count + } + } + } else { + assertionFailure() + } + + if initialCount != updatedCount { + deltas[id] = updatedCount - initialCount + } + } + return deltas + } + override func clearMemoryCache() { self.cachedPeerReadStates.removeAll() - self.updatedPeerIds.removeAll() + assert(self.updatedInitialPeerReadStates.isEmpty) } override func beforeCommit() { - let sharedBuffer = WriteBuffer() - for id in self.updatedPeerIds { - if let wrappedStates = self.cachedPeerReadStates[id], let states = wrappedStates { - sharedBuffer.reset() - var count: Int32 = Int32(states.namespaces.count) - sharedBuffer.write(&count, offset: 0, length: 4) - for (namespace, state) in states.namespaces { - var namespaceId: Int32 = namespace - var maxIncomingReadId: Int32 = state.maxIncomingReadId - var maxOutgoingReadId: Int32 = state.maxOutgoingReadId - var maxKnownId: Int32 = state.maxKnownId - var count: Int32 = state.count - sharedBuffer.write(&namespaceId, offset: 0, length: 4) - sharedBuffer.write(&maxIncomingReadId, offset: 0, length: 4) - sharedBuffer.write(&maxOutgoingReadId, offset: 0, length: 4) - sharedBuffer.write(&maxKnownId, offset: 0, length: 4) + if !self.updatedInitialPeerReadStates.isEmpty { + let sharedBuffer = WriteBuffer() + for (id, initialNamespaces) in self.updatedInitialPeerReadStates { + if let wrappedStates = self.cachedPeerReadStates[id], let states = wrappedStates { + sharedBuffer.reset() + var count: Int32 = Int32(states.namespaces.count) sharedBuffer.write(&count, offset: 0, length: 4) + for (namespace, state) in states.namespaces { + var namespaceId: Int32 = namespace + var maxIncomingReadId: Int32 = state.maxIncomingReadId + var maxOutgoingReadId: Int32 = state.maxOutgoingReadId + var maxKnownId: Int32 = state.maxKnownId + var count: Int32 = state.count + sharedBuffer.write(&namespaceId, offset: 0, length: 4) + sharedBuffer.write(&maxIncomingReadId, offset: 0, length: 4) + sharedBuffer.write(&maxOutgoingReadId, offset: 0, length: 4) + sharedBuffer.write(&maxKnownId, offset: 0, length: 4) + sharedBuffer.write(&count, offset: 0, length: 4) + } + self.valueBox.set(self.table, key: self.key(id), value: sharedBuffer) + } else { + self.valueBox.remove(self.table, key: self.key(id)) } - self.valueBox.set(self.table, key: self.key(id), value: sharedBuffer) - } else { - self.valueBox.remove(self.table, key: self.key(id)) } + self.updatedInitialPeerReadStates.removeAll() } - self.updatedPeerIds.removeAll() } } diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 55a3753d8b..0d2f874a7f 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -212,11 +212,11 @@ final class MessageHistoryTable: Table { var internalStoreMessages: [InternalStoreMessage] = [] for message in messages { switch message.id { - case let .Id(id): - internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) - case let .Partial(peerId, namespace): - let id = self.historyMetadataTable.getNextMessageIdAndIncrement(peerId, namespace: namespace) - internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) + case let .Id(id): + internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, globallyUniqueId: message.globallyUniqueId, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) + case let .Partial(peerId, namespace): + let id = self.historyMetadataTable.getNextMessageIdAndIncrement(peerId, namespace: namespace) + internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, globallyUniqueId: message.globallyUniqueId, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) } } return internalStoreMessages @@ -457,6 +457,13 @@ final class MessageHistoryTable: Table { var stableVersion: UInt32 = 0 sharedBuffer.write(&stableVersion, offset: 0, length: 4) + var hasGloballyUniqueId: Int8 = message.globallyUniqueId != nil ? 1 : 0 + sharedBuffer.write(&hasGloballyUniqueId, offset: 0, length: 1) + if let globallyUniqueId = message.globallyUniqueId { + var globallyUniqueIdValue = globallyUniqueId + sharedBuffer.write(&globallyUniqueIdValue, offset: 0, length: 8) + } + var flags = MessageFlags(message.flags) sharedBuffer.write(&flags.rawValue, offset: 0, length: 4) @@ -569,7 +576,7 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.table, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) - return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, timestamp: message.timestamp, flags: flags, tags: message.tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) + return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: flags, tags: message.tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) } private func justInsertHole(_ hole: MessageHistoryHole, sharedBuffer: WriteBuffer = WriteBuffer()) { @@ -668,7 +675,7 @@ final class MessageHistoryTable: Table { return nil } - func updateEmbeddedMedia(_ index: MessageIndex, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], update: @noescape([Media]) -> [Media]) { + func updateEmbeddedMedia(_ index: MessageIndex, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], update: ([Media]) -> [Media]) { if let message = self.getMessage(index) { var embeddedMediaCount: Int32 = 0 message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) @@ -713,7 +720,7 @@ final class MessageHistoryTable: Table { updatedEmbeddedMediaBuffer.write(encodedBuffer.memory, offset: 0, length: encodedBuffer.length) } - self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: message.referencedMedia), sharedKey: self.key(index)) + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: message.referencedMedia), sharedKey: self.key(index)) let operation: MessageHistoryOperation = .UpdateEmbeddedMedia(index, updatedEmbeddedMediaBuffer.makeReadBufferAndReset()) if operationsByPeerId[index.id.peerId] == nil { @@ -780,7 +787,7 @@ final class MessageHistoryTable: Table { self.messageMediaTable.removeEmbeddedMedia(media) } for mediaId in previousMessage.referencedMedia { - self.messageMediaTable.removeReference(mediaId) + let _ = self.messageMediaTable.removeReference(mediaId) } } @@ -817,6 +824,14 @@ final class MessageHistoryTable: Table { var stableVersion: UInt32 = previousMessage.stableVersion + 1 sharedBuffer.write(&stableVersion, offset: 0, length: 4) + let globallyUniqueId: Int64? = previousMessage.globallyUniqueId + var hasGloballyUniqueId: Int8 = globallyUniqueId != nil ? 1 : 0 + sharedBuffer.write(&hasGloballyUniqueId, offset: 0, length: 1) + if let globallyUniqueId = globallyUniqueId { + var globallyUniqueIdValue = globallyUniqueId + sharedBuffer.write(&globallyUniqueIdValue, offset: 0, length: 8) + } + var flags = MessageFlags(message.flags) sharedBuffer.write(&flags.rawValue, offset: 0, length: 4) @@ -929,7 +944,7 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.table, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) - return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, timestamp: message.timestamp, flags: flags, tags: tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) + return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: flags, tags: tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) } else { return nil } @@ -938,7 +953,7 @@ final class MessageHistoryTable: Table { private func justUpdateTimestamp(_ index: MessageIndex, timestamp: Int32) { if let previousMessage = self.getMessage(index) { self.valueBox.remove(self.table, key: self.key(index)) - var updatedMessage = IntermediateMessage(stableId: previousMessage.stableId, stableVersion: previousMessage.stableVersion + 1, id: previousMessage.id, timestamp: timestamp, flags: previousMessage.flags, tags: previousMessage.tags, forwardInfo: previousMessage.forwardInfo, authorId: previousMessage.authorId, text: previousMessage.text, attributesData: previousMessage.attributesData, embeddedMediaData: previousMessage.embeddedMediaData, referencedMedia: previousMessage.referencedMedia) + let updatedMessage = IntermediateMessage(stableId: previousMessage.stableId, stableVersion: previousMessage.stableVersion + 1, id: previousMessage.id, globallyUniqueId: previousMessage.globallyUniqueId, timestamp: timestamp, flags: previousMessage.flags, tags: previousMessage.tags, forwardInfo: previousMessage.forwardInfo, authorId: previousMessage.authorId, text: previousMessage.text, attributesData: previousMessage.attributesData, embeddedMediaData: previousMessage.embeddedMediaData, referencedMedia: previousMessage.referencedMedia) self.storeIntermediateMessage(updatedMessage, sharedKey: self.key(index)) let tags = previousMessage.tags.rawValue @@ -991,7 +1006,7 @@ final class MessageHistoryTable: Table { if let extractedMedia = extractedMedia { var updatedReferencedMedia = message.referencedMedia updatedReferencedMedia.append(extractedMedia.id!) - self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) return extractedMedia } @@ -1011,6 +1026,13 @@ final class MessageHistoryTable: Table { var stableVersion: UInt32 = message.stableVersion sharedBuffer.write(&stableVersion, offset: 0, length: 4) + var hasGloballyUniqueId: Int8 = message.globallyUniqueId != nil ? 1 : 0 + sharedBuffer.write(&hasGloballyUniqueId, offset: 0, length: 1) + if let globallyUniqueId = message.globallyUniqueId { + var globallyUniqueIdValue = globallyUniqueId + sharedBuffer.write(&globallyUniqueIdValue, offset: 0, length: 8) + } + var flagsValue: UInt32 = message.flags.rawValue sharedBuffer.write(&flagsValue, offset: 0, length: 4) @@ -1091,6 +1113,15 @@ final class MessageHistoryTable: Table { var stableVersion: UInt32 = 0 value.read(&stableVersion, offset: 0, length: 4) + var hasGloballyUniqueId: Int8 = 0 + var globallyUniqueId: Int64? + value.read(&hasGloballyUniqueId, offset: 0, length: 1) + if hasGloballyUniqueId != 0 { + var globallyUniqueIdValue: Int64 = 0 + value.read(&globallyUniqueIdValue, offset: 0, length: 8) + globallyUniqueId = globallyUniqueIdValue + } + var flagsValue: UInt32 = 0 value.read(&flagsValue, offset: 0, length: 4) let flags = MessageFlags(rawValue: flagsValue) @@ -1182,7 +1213,7 @@ final class MessageHistoryTable: Table { referencedMediaIds.append(MediaId(namespace: idNamespace, id: idId)) } - return .Message(IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: index.id, timestamp: index.timestamp, flags: flags, tags: tags, forwardInfo: forwardInfo, authorId: authorId, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)) + return .Message(IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: index.id, globallyUniqueId: globallyUniqueId, timestamp: index.timestamp, flags: flags, tags: tags, forwardInfo: forwardInfo, authorId: authorId, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)) } else { var stableId: UInt32 = 0 value.read(&stableId, offset: 0, length: 4) @@ -1284,7 +1315,7 @@ final class MessageHistoryTable: Table { } } - return Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages, associatedMessageIds: associatedMessageIds) + return Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages, associatedMessageIds: associatedMessageIds) } func entriesAround(_ index: MessageIndex, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { @@ -1495,7 +1526,7 @@ final class MessageHistoryTable: Table { var result: MessageId? self.valueBox.range(self.table, start: self.key(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: timestamp)), end: self.key(MessageIndex(id: MessageId(peerId: peerId, namespace: Int32.max, id: Int32.max), timestamp: timestamp)), values: { key, value in let entry = self.readIntermediateEntry(key, value: value) - if case let .Message(message) = entry { + if case .Message = entry { result = entry.index.id return false } diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index a5b685dfa8..17c070ed0f 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -58,10 +58,10 @@ enum MutableMessageHistoryEntry { func updatedTimestamp(_ timestamp: Int32) -> MutableMessageHistoryEntry { switch self { case let .IntermediateMessageEntry(message, location): - var updatedMessage = IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) + var updatedMessage = IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) return .IntermediateMessageEntry(updatedMessage, location) case let .MessageEntry(message, location): - var updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) + var updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) return .MessageEntry(updatedMessage, location) case let .HoleEntry(hole, location): var updatedHole = MessageHistoryHole(stableId: hole.stableId, maxIndex: MessageIndex(id: hole.maxIndex.id, timestamp: timestamp), min: hole.min, tags: hole.tags) @@ -305,7 +305,7 @@ final class MutableMessageHistoryView { case let .UpdateEmbeddedMedia(index, embeddedMediaData): for i in 0 ..< self.entries.count { if case let .IntermediateMessageEntry(message, _) = self.entries[i] , MessageIndex(message) == index { - self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), nil) + self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), nil) hasChanges = true break } @@ -346,7 +346,7 @@ final class MutableMessageHistoryView { messageMedia.append(media) } } - let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) self.entries[i] = .MessageEntry(updatedMessage, nil) hasChanges = true } @@ -388,7 +388,7 @@ final class MutableMessageHistoryView { var updatedAssociatedMessages = message.associatedMessages let renderedMessage = renderIntermediateMessage(intermediateMessage) updatedAssociatedMessages[intermediateMessage.id] = renderedMessage - let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId:message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) self.entries[i] = .MessageEntry(updatedMessage, location) hasChanges = true } @@ -515,7 +515,7 @@ final class MutableMessageHistoryView { switch self.entries[i] { case let .MessageEntry(message, location): if let updatedAssociatedMessages = message.associatedMessages.filteredOut(keysIn: ids) { - let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) self.entries[i] = .MessageEntry(updatedMessage, location) } default: diff --git a/Postbox/PeerNameIndexTable.swift b/Postbox/PeerNameIndexTable.swift index 43164fc5da..ebd23ea4e1 100644 --- a/Postbox/PeerNameIndexTable.swift +++ b/Postbox/PeerNameIndexTable.swift @@ -31,7 +31,7 @@ private final class PeerNameIndexCategoriesEntry { let tokenCount = buffer.memory.load(fromByteOffset: 4, as: Int32.self) var offset = 8 var tokens: [ValueBoxKey] = [] - for i in 0 ..< tokenCount { + for _ in 0 ..< tokenCount { let length = buffer.memory.load(fromByteOffset: offset, as: Int32.self) offset += 4 tokens.append(ValueBoxKey(MemoryBuffer(memory: buffer.memory.advanced(by: offset), capacity: Int(length), length: Int(length), freeWhenDone: false))) diff --git a/Postbox/PeerNotificationSettings.swift b/Postbox/PeerNotificationSettings.swift index 32ad5004c5..d0ab6d404f 100644 --- a/Postbox/PeerNotificationSettings.swift +++ b/Postbox/PeerNotificationSettings.swift @@ -1,4 +1,6 @@ public protocol PeerNotificationSettings: Coding { + var isRemovedFromTotalUnreadCount: Bool { get } + func isEqual(to: PeerNotificationSettings) -> Bool } diff --git a/Postbox/PeerNotificationSettingsTable.swift b/Postbox/PeerNotificationSettingsTable.swift index 00d8547b1f..6b79c40c4b 100644 --- a/Postbox/PeerNotificationSettingsTable.swift +++ b/Postbox/PeerNotificationSettingsTable.swift @@ -9,7 +9,7 @@ final class PeerNotificationSettingsTable: Table { private let sharedKey = ValueBoxKey(length: 8) private var cachedSettings: [PeerId: PeerNotificationSettings] = [:] - private var updatedPeerIds = Set() + private var updatedInitialSettings: [PeerId: PeerNotificationSettings?] = [:] private func key(_ id: PeerId) -> ValueBoxKey { self.sharedKey.setInt64(0, value: id.toInt64()) @@ -17,8 +17,13 @@ final class PeerNotificationSettingsTable: Table { } func set(id: PeerId, settings: PeerNotificationSettings) { - self.cachedSettings[id] = settings - self.updatedPeerIds.insert(id) + let current = self.get(id) + if current == nil || !current!.isEqual(to: settings) { + if self.updatedInitialSettings[id] == nil { + self.updatedInitialSettings[id] = current + } + self.cachedSettings[id] = settings + } } func get(_ id: PeerId) -> PeerNotificationSettings? { @@ -36,19 +41,43 @@ final class PeerNotificationSettingsTable: Table { override func clearMemoryCache() { self.cachedSettings.removeAll() - self.updatedPeerIds.removeAll() + self.updatedInitialSettings.removeAll() } - override func beforeCommit() { - for peerId in self.updatedPeerIds { - if let settings = self.cachedSettings[peerId] { - self.sharedEncoder.reset() - self.sharedEncoder.encodeRootObject(settings) - - self.valueBox.set(self.table, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + func transactionParticipationInTotalUnreadCountUpdates() -> (added: Set, removed: Set) { + var added = Set() + var removed = Set() + + for (peerId, initialSettings) in self.updatedInitialSettings { + var wasParticipating = false + if let initialSettings = initialSettings { + wasParticipating = !initialSettings.isRemovedFromTotalUnreadCount + } + let isParticipating = !self.cachedSettings[peerId]!.isRemovedFromTotalUnreadCount + if wasParticipating != isParticipating { + if isParticipating { + added.insert(peerId) + } else { + removed.insert(peerId) + } } } - self.updatedPeerIds.removeAll() + return (added, removed) + } + + override func beforeCommit() { + if !self.updatedInitialSettings.isEmpty { + for (peerId, _) in self.updatedInitialSettings { + if let settings = self.cachedSettings[peerId] { + self.sharedEncoder.reset() + self.sharedEncoder.encodeRootObject(settings) + + self.valueBox.set(self.table, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + } + } + + self.updatedInitialSettings.removeAll() + } } } diff --git a/Postbox/PeerOperationLogMetadataTable.swift b/Postbox/PeerOperationLogMetadataTable.swift new file mode 100644 index 0000000000..ae413c7ec7 --- /dev/null +++ b/Postbox/PeerOperationLogMetadataTable.swift @@ -0,0 +1,41 @@ +import Foundation + +private enum PeerOperationLogIndexNamespace: Int8 { + case nextTagLocalIndex = 0 +} + +final class PeerOperationLogMetadataTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private func keyForNextLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> ValueBoxKey { + let key = ValueBoxKey(length: 1 + 8 + 1) + key.setInt8(0, value: PeerOperationLogIndexNamespace.nextTagLocalIndex.rawValue) + key.setInt64(1, value: peerId.toInt64()) + key.setInt8(9, value: tag.rawValue) + return key + } + + func getNextLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> Int32 { + if let value = self.valueBox.get(self.table, key: self.keyForNextLocalIndex(peerId: peerId, tag: tag)) { + assert(value.length == 4) + var index: Int32 = 0 + value.read(&index, offset: 0, length: 4) + return index + } else { + return 0 + } + } + + func setNextLocalIndex(peerId: PeerId, tag: PeerOperationLogTag, index: Int32) { + var indexValue: Int32 = index + self.valueBox.set(self.table, key: self.keyForNextLocalIndex(peerId: peerId, tag: tag), value: MemoryBuffer(memory: &indexValue, capacity: 4, length: 4, freeWhenDone: false)) + } + + func takeNextLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> Int32 { + let index = self.getNextLocalIndex(peerId: peerId, tag: tag) + self.setNextLocalIndex(peerId: peerId, tag: tag, index: index) + return index + } +} diff --git a/Postbox/PeerOperationLogTable.swift b/Postbox/PeerOperationLogTable.swift new file mode 100644 index 0000000000..a6d83f96f9 --- /dev/null +++ b/Postbox/PeerOperationLogTable.swift @@ -0,0 +1,53 @@ +import Foundation + +public enum PeerOperationLogTag: Int8 { + case inbox = 0 + case outbox = 1 +} + +public struct PeerOperationLogEntry { + let tagLocalIndex: Int32 + let contents: Coding +} + +final class PeerOperationLogQueue: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private let metadataTable: PeerOperationLogMetadataTable + + init(valueBox: ValueBox, table: ValueBoxTable, metadataTable: PeerOperationLogMetadataTable) { + self.metadataTable = metadataTable + + super.init(valueBox: valueBox, table: table) + } + + private func key(peerId: PeerId, tag: PeerOperationLogTag, index: Int32) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 1 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setInt8(1, value: tag.rawValue) + key.setInt32(9, value: index) + return key + } + + func addEntryAndTakeNextTagLocalIndex(peerId: PeerId, tag: PeerOperationLogTag, contents: Coding) -> Int32 { + let index = self.metadataTable.takeNextLocalIndex(peerId: peerId, tag: tag) + let encoder = Encoder() + encoder.encodeRootObject(contents) + self.valueBox.set(self.table, key: self.key(peerId: peerId, tag: tag, index: index), value: encoder.readBufferNoCopy()) + return index + } + + func removeEntries(peerId: PeerId, tag: PeerOperationLogTag, withIndicesLowerThan index: Int32) { + var indices: [Int32] = [] + self.valueBox.range(self.table, start: self.key(peerId: peerId, tag: tag, index: 0).predecessor, end: self.key(peerId: peerId, tag: tag, index: index), keys: { key in + indices.append(key.getInt32(9)) + return true + }, limit: 0) + + for index in indices { + self.valueBox.remove(self.table, key: self.key(peerId: peerId, tag: tag, index: index)) + } + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index b94c6902e6..155a1b195a 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -224,6 +224,7 @@ public final class Postbox { private var currentUpdatedCachedPeerData: [PeerId: CachedPeerData] = [:] private var currentUpdatedPeerPresences: [PeerId: PeerPresence] = [:] private var currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] + private var currentUpdatedTotalUnreadCount: Int32? private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -287,12 +288,12 @@ public final class Postbox { self.pipeNotifier = PipeNotifier(basePath: basePath, notify: { [weak self] in if let strongSelf = self { - strongSelf.queue.async { + /*strongSelf.queue.async { if strongSelf.valueBox != nil { let _ = strongSelf.modify({ _ -> Void in }).start() } - } + }*/ } }) @@ -351,7 +352,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 1 + let currentUserVersion: Int32 = 5 if userVersion != currentUserVersion { self.valueBox.drop() @@ -382,7 +383,7 @@ public final class Postbox { self.itemCacheTable = ItemCacheTable(valueBox: self.valueBox, table: ItemCacheTable.tableSpec(25)) self.peerNameTokenIndexTable = PeerNameTokenIndexTable(valueBox: self.valueBox, table: PeerNameTokenIndexTable.tableSpec(26)) self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) - self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8), peerNameIndexTable: self.peerNameIndexTable) + self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8), peerNameIndexTable: self.peerNameIndexTable, metadataTable: self.messageHistoryMetadataTable, readStateTable: self.readStateTable, notificationSettingsTable: self.peerNotificationSettingsTable) self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.peerChatTopTaggedMessageIdsTable = PeerChatTopTaggedMessageIdsTable(valueBox: self.valueBox, table: PeerChatTopTaggedMessageIdsTable.tableSpec(28)) @@ -424,7 +425,11 @@ public final class Postbox { return self.cachedPeerDataTable.get(peerId) }, getPeerPresence: { peerId in return self.peerPresenceTable.get(peerId) - },unsentMessageIds: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) + }, getTotalUnreadCount: { + return self.messageHistoryMetadataTable.getChatListTotalUnreadCount() + }, getPeerReadState: { peerId in + return self.readStateTable.getCombinedState(peerId) + }, unsentMessageIds: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") }) @@ -791,7 +796,11 @@ public final class Postbox { self.peerChatTopTaggedMessageIdsTable.replay(historyOperationsByPeerId: self.currentOperationsByPeerId) - let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transactionUnreadCountDeltas = self.readStateTable.transactionUnreadCountDeltas() + let transactionParticipationInTotalUnreadCountUpdates = self.peerNotificationSettingsTable.transactionParticipationInTotalUnreadCountUpdates() + self.chatListIndexTable.commitWithTransactionUnreadCountDeltas(transactionUnreadCountDeltas, transactionParticipationInTotalUnreadCountUpdates: transactionParticipationInTotalUnreadCountUpdates, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) + + let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -963,7 +972,7 @@ public final class Postbox { subscriber.putCompletion() if updatedTransactionState != nil || updatedMasterClientId != nil { - self.pipeNotifier.notify() + //self.pipeNotifier.notify() } if let updatedMasterClientId = updatedMasterClientId { @@ -1276,6 +1285,35 @@ public final class Postbox { } |> switchToLatest } + public func unreadMessageCountsView(items: [UnreadMessageCountsItem]) -> Signal { + return self.modify { modifier -> Signal in + let entries: [UnreadMessageCountsItemEntry] = items.map { item in + switch item { + case .total: + return .total(self.messageHistoryMetadataTable.getChatListTotalUnreadCount()) + case let .peer(peerId): + var count: Int32 = 0 + if let combinedState = self.readStateTable.getCombinedState(peerId) { + count = combinedState.count + } + return .peer(peerId, count) + } + } + let view = MutableUnreadMessageCountsView(entries: entries) + let (index, signal) = self.viewTracker.addUnreadMessageCountsView(view) + + return (.single(UnreadMessageCountsView(view)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeUnreadMessageCountsView(index) + } + } + } + } |> switchToLatest + } + public func updateMessageHistoryViewVisibleRange(_ id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { self.modify({ modifier -> Void in self.viewTracker.updateMessageHistoryViewVisibleRange(id, earliestVisibleIndex: earliestVisibleIndex, latestVisibleIndex: latestVisibleIndex) diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index 58ad00f548..673b7f5636 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -10,6 +10,8 @@ final class PostboxTransaction { let currentUpdatedCachedPeerData: [PeerId: CachedPeerData] let currentUpdatedPeerPresences: [PeerId: PeerPresence] let currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] + let currentUpdatedTotalUnreadCount: Int32? + let peerIdsWithUpdatedUnreadCounts: Set let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -60,10 +62,16 @@ final class PostboxTransaction { if currentUpdatedMasterClientId != nil { return false } + if currentUpdatedTotalUnreadCount != nil { + return false + } + if !peerIdsWithUpdatedUnreadCounts.isEmpty { + return false + } return true } - init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles self.removedHolesByPeerId = removedHolesByPeerId @@ -73,6 +81,8 @@ final class PostboxTransaction { self.currentUpdatedCachedPeerData = currentUpdatedCachedPeerData self.currentUpdatedPeerPresences = currentUpdatedPeerPresences self.currentUpdatedPeerChatListEmbeddedStates = currentUpdatedPeerChatListEmbeddedStates + self.currentUpdatedTotalUnreadCount = currentUpdatedTotalUnreadCount + self.peerIdsWithUpdatedUnreadCounts = peerIdsWithUpdatedUnreadCounts self.unsentMessageOperations = unsentMessageOperations self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations self.updatedMedia = updatedMedia diff --git a/Postbox/UnreadMessageCountsView.swift b/Postbox/UnreadMessageCountsView.swift new file mode 100644 index 0000000000..21551728f4 --- /dev/null +++ b/Postbox/UnreadMessageCountsView.swift @@ -0,0 +1,69 @@ +import Foundation + +public enum UnreadMessageCountsItem { + case total + case peer(PeerId) +} + +enum UnreadMessageCountsItemEntry { + case total(Int32) + case peer(PeerId, Int32) +} + +final class MutableUnreadMessageCountsView { + fileprivate var entries: [UnreadMessageCountsItemEntry] + + init(entries: [UnreadMessageCountsItemEntry]) { + self.entries = entries + } + + func replay(peerIdsWithUpdatedUnreadCounts: Set, getTotalUnreadCount: () -> Int32, getPeerReadState: (PeerId) -> CombinedPeerReadState?) -> Bool { + var updated = false + + for i in 0 ..< self.entries.count { + switch self.entries[i] { + case let .total(count): + let updatedCount = getTotalUnreadCount() + if updatedCount != count { + self.entries[i] = .total(updatedCount) + updated = true + } + case let .peer(peerId, count): + if peerIdsWithUpdatedUnreadCounts.contains(peerId) { + var updatedCount: Int32 = 0 + if let combinedState = getPeerReadState(peerId) { + updatedCount = combinedState.count + } + self.entries[i] = .peer(peerId, updatedCount) + updated = true + } + } + } + + return updated + } +} + +public final class UnreadMessageCountsView { + private let entries: [UnreadMessageCountsItemEntry] + + init(_ view: MutableUnreadMessageCountsView) { + self.entries = view.entries + } + + public func count(for item: UnreadMessageCountsItem) -> Int32? { + for entry in self.entries { + switch entry { + case let .total(count): + if case .total = item { + return count + } + case let .peer(peerId, count): + if case .peer(peerId) = item { + return count + } + } + } + return nil + } +} diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index 414d7bd532..e88a1b97a2 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -12,7 +12,7 @@ private final class ValueBoxKeyImpl { } } -public struct ValueBoxKey: Hashable, CustomStringConvertible { +public struct ValueBoxKey: Hashable, CustomStringConvertible, Comparable { public let memory: UnsafeMutableRawPointer public let length: Int private let impl: ValueBoxKeyImpl @@ -157,10 +157,14 @@ public struct ValueBoxKey: Hashable, CustomStringConvertible { } return hash } -} - -public func ==(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { - return lhs.length == rhs.length && memcmp(lhs.memory, rhs.memory, lhs.length) == 0 + + public static func ==(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { + return lhs.length == rhs.length && memcmp(lhs.memory, rhs.memory, lhs.length) == 0 + } + + public static func <(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { + return mdb_cmp_memn(lhs.memory, lhs.length, rhs.memory, rhs.length) < 0 + } } private func mdb_cmp_memn(_ a_memory: UnsafeMutableRawPointer, _ a_length: Int, _ b_memory: UnsafeMutableRawPointer, _ b_length: Int) -> Int @@ -179,7 +183,3 @@ private func mdb_cmp_memn(_ a_memory: UnsafeMutableRawPointer, _ a_length: Int, diff = Int(memcmp(a_memory, b_memory, len)) return diff != 0 ? diff : len_diff < 0 ? -1 : len_diff } - -public func <(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { - return mdb_cmp_memn(lhs.memory, lhs.length, rhs.memory, rhs.length) < 0 -} diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index a0cc0a8a4f..f424a9572b 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -24,6 +24,8 @@ final class ViewTracker { private let getPeerNotificationSettings: (PeerId) -> PeerNotificationSettings? private let getCachedPeerData: (PeerId) -> CachedPeerData? private let getPeerPresence: (PeerId) -> PeerPresence? + private let getTotalUnreadCount: () -> Int32 + private let getPeerReadState: (PeerId) -> CombinedPeerReadState? private var chatListViews = Bag<(MutableChatListView, ValuePipe<(ChatListView, ViewUpdateType)>)>() private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] @@ -44,7 +46,9 @@ final class ViewTracker { private var peerViews = Bag<(MutablePeerView, ValuePipe)>() - init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + private var unreadMessageCountsViews = Bag<(MutableUnreadMessageCountsView, ValuePipe)>() + + init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -56,6 +60,8 @@ final class ViewTracker { self.getPeerNotificationSettings = getPeerNotificationSettings self.getCachedPeerData = getCachedPeerData self.getPeerPresence = getPeerPresence + self.getTotalUnreadCount = getTotalUnreadCount + self.getPeerReadState = getPeerReadState self.unsentMessageView = UnsentMessageHistoryView(ids: unsentMessageIds) self.synchronizeReadStatesView = MutableSynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) @@ -165,6 +171,17 @@ final class ViewTracker { self.peerViews.remove(index) } + func addUnreadMessageCountsView(_ view: MutableUnreadMessageCountsView) -> (Bag<(MutableUnreadMessageCountsView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.unreadMessageCountsViews.add(record) + + return (index, record.1.signal()) + } + + func removeUnreadMessageCountsView(_ index: Bag<(MutableUnreadMessageCountsView, ValuePipe)>.Index) { + self.peerViews.remove(index) + } + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: MessageIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] @@ -316,6 +333,12 @@ final class ViewTracker { self.synchronizeReadStateViewUpdated() } + for (view, pipe) in self.unreadMessageCountsViews.copyItems() { + if view.replay(peerIdsWithUpdatedUnreadCounts: transaction.peerIdsWithUpdatedUnreadCounts, getTotalUnreadCount: self.getTotalUnreadCount, getPeerReadState: self.getPeerReadState) { + pipe.putNext(UnreadMessageCountsView(view)) + } + } + if let replaceContactPeerIds = transaction.replaceContactPeerIds { for (mutableView, pipe) in self.contactPeerIdsViews.copyItems() { if mutableView.replay(replace: replaceContactPeerIds) { diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index 3dfd380568..b7ea687151 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -77,6 +77,10 @@ class ChatListTableTests: XCTestCase { var readStateTable: MessageHistoryReadStateTable? var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? var peerChatInterfaceStateTable: PeerChatInterfaceStateTable? + var peerTable: PeerTable? + var peerNameTokenIndexTable: PeerNameTokenIndexTable? + var peerNameIndexTable: PeerNameIndexTable? + var notificationSettingsTable: PeerNotificationSettingsTable? override class func setUp() { super.setUp() @@ -101,7 +105,11 @@ class ChatListTableTests: XCTestCase { self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, table: MessageHistoryReadStateTable.tableSpec(11)) self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, table: MessageHistorySynchronizeReadStateTable.tableSpec(12)) self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) - self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, table: ChatListIndexTable.tableSpec(5)) + self.peerTable = PeerTable(valueBox: self.valueBox!, table: PeerTable.tableSpec(20)) + self.peerNameTokenIndexTable = PeerNameTokenIndexTable(valueBox: self.valueBox!, table: PeerNameTokenIndexTable.tableSpec(21)) + self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox!, table: PeerNameIndexTable.tableSpec(22), peerTable: self.peerTable!, peerNameTokenIndexTable: self.peerNameTokenIndexTable!) + self.notificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox!, table: PeerNotificationSettingsTable.tableSpec(23)) + self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, table: ChatListIndexTable.tableSpec(5), peerNameIndexTable: self.peerNameIndexTable!, metadataTable: self.historyMetadataTable!, readStateTable: self.readStateTable!, notificationSettingsTable: self.notificationSettingsTable!) self.chatListTable = ChatListTable(valueBox: self.valueBox!, table: ChatListTable.tableSpec(6), indexTable: self.chatListIndexTable!, metadataTable: self.historyMetadataTable!, seedConfiguration: seedConfiguration) self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox!, table: PeerChatInterfaceStateTable.tableSpec(20)) } @@ -125,7 +133,7 @@ class ChatListTableTests: XCTestCase { var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] - self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), globallyUniqueId: nil, timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) } @@ -170,7 +178,7 @@ class ChatListTableTests: XCTestCase { var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] - self.historyTable!.fillHole(MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), fillType: fillType, tagMask: nil, messages: messages.map({ StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.fillHole(MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), fillType: fillType, tagMask: nil, messages: messages.map({ StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0.0), globallyUniqueId: nil, timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) } diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index f55ee0490a..e304ac53d2 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -101,13 +101,13 @@ class MessageHistoryIndexTableTests: XCTestCase { func addMessage(_ id: Int32, _ timestamp: Int32) { var operations: [MessageHistoryIndexOperation] = [] - self.indexTable!.addMessages([InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: [])], location: .Random, operations: &operations) + self.indexTable!.addMessages([InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, globallyUniqueId: nil, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: [])], location: .Random, operations: &operations) } func addMessagesUpperBlock(_ messages: [(Int32, Int32)]) { var operations: [MessageHistoryIndexOperation] = [] self.indexTable!.addMessages(messages.map { (id, timestamp) in - return InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: []) + return InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, globallyUniqueId: nil, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: []) }, location: .UpperHistoryBlock, operations: &operations) } @@ -115,7 +115,7 @@ class MessageHistoryIndexTableTests: XCTestCase { var operations: [MessageHistoryIndexOperation] = [] self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ - return InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: []) + return InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, globallyUniqueId: nil, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: []) }), operations: &operations) } diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index 153912cd12..74b607c9dc 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -123,7 +123,7 @@ private class TestExternalMedia: Media { private class TestPeer: Peer { var indexName: PeerIndexNameRepresentation { - return .title("Test") + return .title(title: "Test", addressName: nil) } let id: PeerId @@ -259,14 +259,14 @@ class MessageHistoryTableTests: XCTestCase { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), globallyUniqueId: nil, timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } private func updateMessage(_ previousId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags, _ tags: MessageTags) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), globallyUniqueId: nil, timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } private func addHole(_ id: Int32) { @@ -287,7 +287,7 @@ class MessageHistoryTableTests: XCTestCase { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), globallyUniqueId: nil, timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } private func expectEntries(_ entries: [Entry], tagMask: MessageTags? = nil) { diff --git a/PostboxTests/ReadStateTableTests.swift b/PostboxTests/ReadStateTableTests.swift index ddb5b1642a..59e84ad7ac 100644 --- a/PostboxTests/ReadStateTableTests.swift +++ b/PostboxTests/ReadStateTableTests.swift @@ -117,14 +117,14 @@ class ReadStateTableTests: XCTestCase { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), globallyUniqueId: nil, timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } private func updateMessage(_ previousId: Int32, _ id: Int32, _ timestamp: Int32, _ text: String = "", _ media: [Media] = [], _ flags: StoreMessageFlags, _ tags: MessageTags) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.updateMessage(MessageId(peerId: peerId, namespace: namespace, id: previousId), message: StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), globallyUniqueId: nil, timestamp: timestamp, flags: flags, tags: tags, forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } private func addHole(_ id: Int32) { From 4c392a3ace3669eb6bc8475529a8ab3ecb99208f Mon Sep 17 00:00:00 2001 From: overtake Date: Mon, 16 Jan 2017 16:02:06 +0300 Subject: [PATCH 069/427] no message --- Postbox.xcodeproj/project.pbxproj | 2 ++ .../xcschemes/xcschememanagement.plist | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index b4aab6edf5..b106f894a0 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -7,6 +7,7 @@ objects = { /* Begin PBXBuildFile section */ + C2AC9C131E1E5D200085C7DE /* UnreadMessageCountsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */; }; D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */; }; @@ -753,6 +754,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + C2AC9C131E1E5D200085C7DE /* UnreadMessageCountsView.swift in Sources */, D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, D0F7B1DF1E045C6A007EB8A5 /* ItemCacheMetaTable.swift in Sources */, D0F7B1C41E045C6A007EB8A5 /* GlobalMessageIdsTable.swift in Sources */, diff --git a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist index f92a60b4aa..a3703be5c4 100644 --- a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist @@ -6,11 +6,15 @@ Postbox.xcscheme + isShown + orderHint 5 PostboxMac.xcscheme + isShown + orderHint 6 From 9835ac1b881f47f52a6d5b6d5adfeb71b1adb95f Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 17 Jan 2017 18:33:36 +0300 Subject: [PATCH 070/427] no message --- Postbox.xcodeproj/project.pbxproj | 8 + Postbox/Coding.swift | 4 +- Postbox/Message.swift | 36 ++ Postbox/MessageHistoryIndexTable.swift | 8 +- Postbox/MessageHistoryTable.swift | 12 +- Postbox/Peer.swift | 7 +- .../PeerMergedOperationLogIndexTable.swift | 62 ++++ Postbox/PeerMergedOperationLogView.swift | 96 +++++ Postbox/PeerNameIndexTable.swift | 2 +- Postbox/PeerOperationLogMetadataTable.swift | 33 +- Postbox/PeerOperationLogTable.swift | 330 +++++++++++++++++- Postbox/PeerView.swift | 25 ++ Postbox/Postbox.swift | 107 +++++- Postbox/PostboxTransaction.swift | 7 +- Postbox/SeedConfiguration.swift | 4 +- Postbox/ValueBoxKey.swift | 11 + Postbox/ViewTracker.swift | 24 +- 17 files changed, 743 insertions(+), 33 deletions(-) create mode 100644 Postbox/PeerMergedOperationLogIndexTable.swift create mode 100644 Postbox/PeerMergedOperationLogView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index b4aab6edf5..4bba6c3596 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -13,6 +13,7 @@ D0079F651D5A457A00A27A2C /* ContactPeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F641D5A457A00A27A2C /* ContactPeersView.swift */; }; D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; + D010B61A1E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */; }; D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */; }; @@ -149,6 +150,7 @@ D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; + D0F019FD1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */; }; D0F3CC721DDE1CDC008148FA /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; D0F7AB321DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */; }; @@ -216,6 +218,7 @@ D0079F641D5A457A00A27A2C /* ContactPeersView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactPeersView.swift; sourceTree = ""; }; D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameIndexRepresentation.swift; sourceTree = ""; }; D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; + D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerMergedOperationLogIndexTable.swift; sourceTree = ""; }; D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollection.swift; sourceTree = ""; }; @@ -305,6 +308,7 @@ D0E3A7871B28AE9C00A402D9 /* Coding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Coding.swift; sourceTree = ""; }; D0E3A79D1B28B50400A402D9 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = ""; }; D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = ""; }; + D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerMergedOperationLogView.swift; sourceTree = ""; }; D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheTable.swift; sourceTree = ""; }; D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheMetaTable.swift; sourceTree = ""; }; D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatTopIndexableMessageIds.swift; sourceTree = ""; }; @@ -444,6 +448,7 @@ D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */, D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */, D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */, + D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */, D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */, ); name = Tables; @@ -509,6 +514,7 @@ D0DF0C8E1D81A350008AEB01 /* PeerView.swift */, D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */, D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */, + D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */, ); name = Views; sourceTree = ""; @@ -871,6 +877,7 @@ D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */, D0AB0B8C1D65D488002C78E7 /* MessageHistoryHolesView.swift in Sources */, + D010B61A1E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift in Sources */, D0F9E8671C58D08900037222 /* ChatListIndexTable.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, @@ -894,6 +901,7 @@ D03120FA1DA540F0006A2A60 /* CachedPeerData.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, + D0F019FD1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift in Sources */, D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */, diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index af779414c7..45ea6857f4 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -38,9 +38,9 @@ public func persistentHash32(_ string: String) -> Int32 { private let emptyMemory = malloc(1)! public class MemoryBuffer: Equatable, CustomStringConvertible { - var memory: UnsafeMutableRawPointer + public internal(set) var memory: UnsafeMutableRawPointer var capacity: Int - var length: Int + public internal(set) var length: Int var freeWhenDone: Bool public init(copyOf buffer: MemoryBuffer) { diff --git a/Postbox/Message.swift b/Postbox/Message.swift index e2ad1c66d9..f358da19ce 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -209,6 +209,10 @@ public struct MessageFlags: OptionSet { rawValue |= MessageFlags.TopIndexable.rawValue } + if flags.contains(StoreMessageFlags.Sending) { + rawValue |= MessageFlags.Sending.rawValue + } + self.rawValue = rawValue } @@ -217,6 +221,7 @@ public struct MessageFlags: OptionSet { public static let Incoming = MessageFlags(rawValue: 4) public static let Personal = MessageFlags(rawValue: 8) public static let TopIndexable = MessageFlags(rawValue: 16) + public static let Sending = MessageFlags(rawValue: 32) } public struct StoreMessageForwardInfo { @@ -324,11 +329,42 @@ public struct StoreMessageFlags: OptionSet { self.rawValue = 0 } + public init(_ flags: MessageFlags) { + var rawValue: UInt32 = 0 + + if flags.contains(.Unsent) { + rawValue |= StoreMessageFlags.Unsent.rawValue + } + + if flags.contains(.Failed) { + rawValue |= StoreMessageFlags.Failed.rawValue + } + + if flags.contains(.Incoming) { + rawValue |= StoreMessageFlags.Incoming.rawValue + } + + if flags.contains(.Personal) { + rawValue |= StoreMessageFlags.Personal.rawValue + } + + if flags.contains(.TopIndexable) { + rawValue |= StoreMessageFlags.TopIndexable.rawValue + } + + if flags.contains(.Sending) { + rawValue |= StoreMessageFlags.Sending.rawValue + } + + self.rawValue = rawValue + } + public static let Unsent = StoreMessageFlags(rawValue: 1) public static let Failed = StoreMessageFlags(rawValue: 2) public static let Incoming = StoreMessageFlags(rawValue: 4) public static let Personal = StoreMessageFlags(rawValue: 8) public static let TopIndexable = StoreMessageFlags(rawValue: 16) + public static let Sending = StoreMessageFlags(rawValue: 32) } public enum StoreMessageId { diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index ebf2df414c..02acd75da6 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -151,8 +151,12 @@ final class MessageHistoryIndexTable: Table { func ensureInitialized(_ peerId: PeerId, operations: inout [MessageHistoryIndexOperation]) { if !self.metadataTable.isInitialized(peerId) { - for namespace in self.seedConfiguration.initializeMessageNamespacesWithHoles { - self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: Int32.max), timestamp: Int32.max), min: 1, tags: MessageTags.All.rawValue), operations: &operations) + var processedMessageNamespaces = Set() + for (peerNamespace, messageNamespace) in self.seedConfiguration.initializeMessageNamespacesWithHoles { + if peerId.namespace == peerNamespace, !processedMessageNamespaces.contains(messageNamespace) { + processedMessageNamespaces.insert(messageNamespace) + self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: messageNamespace, id: Int32.max), timestamp: Int32.max), min: 1, tags: MessageTags.All.rawValue), operations: &operations) + } } self.metadataTable.setInitialized(peerId) diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 0d2f874a7f..d5c7ec0ace 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -782,7 +782,7 @@ final class MessageHistoryTable: Table { } } - if previousMediaIds != updatedMediaIds || index.id != message.id { + if previousMediaIds != updatedMediaIds || index != MessageIndex(message) { for (_, media) in previousEmbeddedMediaWithIds { self.messageMediaTable.removeEmbeddedMedia(media) } @@ -1278,13 +1278,21 @@ final class MessageHistoryTable: Table { } var author: Peer? + var peers = SimpleDictionary() if let authorId = message.authorId { author = peerTable.get(authorId) } - var peers = SimpleDictionary() if let chatPeer = peerTable.get(message.id.peerId) { peers[chatPeer.id] = chatPeer + + if let associatedPeerIds = chatPeer.associatedPeerIds { + for peerId in associatedPeerIds { + if let peer = peerTable.get(peerId) { + peers[peer.id] = peer + } + } + } } for media in parsedMedia { diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index f5c9a454e2..12b070bc13 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -14,11 +14,13 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { public init(_ n: Int64) { self.namespace = Int32((n >> 32) & 0x7fffffff) - self.id = Int32(n & 0x7fffffff) + self.id = unsafeBitCast(UInt32(n & 0xffffffff), to: Int32.self) } public func toInt64() -> Int64 { - return (Int64(self.namespace) << 32) | Int64(self.id) + + + return (Int64(self.namespace) << 32) | unsafeBitCast(UInt64(unsafeBitCast(self.id, to: UInt32.self)), to: Int64.self) } public static func encodeArrayToBuffer(_ array: [PeerId], buffer: WriteBuffer) { @@ -95,6 +97,7 @@ public func <(lhs: PeerId, rhs: PeerId) -> Bool { public protocol Peer: class, Coding { var id: PeerId { get } var indexName: PeerIndexNameRepresentation { get } + var associatedPeerIds: [PeerId]? { get } func isEqual(_ other: Peer) -> Bool } diff --git a/Postbox/PeerMergedOperationLogIndexTable.swift b/Postbox/PeerMergedOperationLogIndexTable.swift new file mode 100644 index 0000000000..a7cc240f14 --- /dev/null +++ b/Postbox/PeerMergedOperationLogIndexTable.swift @@ -0,0 +1,62 @@ +import Foundation + +final class PeerMergedOperationLogIndexTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private let metadataTable: PeerOperationLogMetadataTable + + init(valueBox: ValueBox, table: ValueBoxTable, metadataTable: PeerOperationLogMetadataTable) { + self.metadataTable = metadataTable + + super.init(valueBox: valueBox, table: table) + } + + private func key(tag: PeerOperationLogTag, index: Int32) -> ValueBoxKey { + let key = ValueBoxKey(length: 5) + key.setUInt8(0, value: tag.rawValue) + key.setInt32(1, value: index) + return key + } + + func add(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32) -> Int32 { + let index = self.metadataTable.takeNextMergedIndex() + let buffer = WriteBuffer() + var peerIdValue: Int64 = peerId.toInt64() + var tagLocalIndexValue: Int32 = tagLocalIndex + buffer.write(&peerIdValue, offset: 0, length: 8) + buffer.write(&tagLocalIndexValue, offset: 0, length: 4) + self.valueBox.set(self.table, key: self.key(tag: tag, index: index), value: buffer) + return index + } + + func remove(tag: PeerOperationLogTag, mergedIndices: [Int32]) { + for index in mergedIndices { + self.valueBox.remove(self.table, key: self.key(tag: tag, index: index)) + } + } + + func getTagLocalIndices(tag: PeerOperationLogTag, fromMergedIndex: Int32, limit: Int) -> [(PeerId, Int32)] { + var result: [(PeerId, Int32)] = [] + self.valueBox.range(self.table, start: self.key(tag: tag, index: fromMergedIndex == 0 ? 0 : fromMergedIndex - 1), end: self.key(tag: tag, index: Int32.max), values: { key, value in + var peerIdValue: Int64 = 0 + var tagLocalIndexValue: Int32 = 0 + value.read(&peerIdValue, offset: 0, length: 8) + value.read(&tagLocalIndexValue, offset: 0, length: 4) + result.append((PeerId(peerIdValue), tagLocalIndexValue)) + return true + }, limit: limit) + return result + } + + func tailIndex(tag: PeerOperationLogTag) -> Int32? { + var result: Int32? + self.valueBox.range(self.table, start: self.key(tag: tag, index: Int32.max), end: self.key(tag: tag, index: 0), keys: { + key in + result = key.getInt32(1) + return false + }, limit: 1) + return result + } +} diff --git a/Postbox/PeerMergedOperationLogView.swift b/Postbox/PeerMergedOperationLogView.swift new file mode 100644 index 0000000000..73171c16a7 --- /dev/null +++ b/Postbox/PeerMergedOperationLogView.swift @@ -0,0 +1,96 @@ +import Foundation + +final class MutablePeerMergedOperationLogView { + let tag: PeerOperationLogTag + var entries: [PeerMergedOperationLogEntry] + var tailIndex: Int32? + let limit: Int + + init(tag: PeerOperationLogTag, limit: Int, getOperations: (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], getTailIndex: (PeerOperationLogTag) -> Int32?) { + self.tag = tag + self.entries = getOperations(tag, 0, limit) + self.tailIndex = getTailIndex(tag) + self.limit = limit + } + + func replay(operations: [PeerMergedOperationLogOperation], getOperations: (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], getTailIndex: (PeerOperationLogTag) -> Int32?) -> Bool { + var updated = false + var invalidatedTail = false + + for operation in operations { + switch operation { + case let .append(entry): + if entry.tag == self.tag { + if let tailIndex = self.tailIndex { + assert(entry.mergedIndex > tailIndex) + self.tailIndex = entry.mergedIndex + if self.entries.count < self.limit { + self.entries.append(entry) + updated = true + } + } else { + updated = true + assert(self.entries.isEmpty) + self.entries.append(entry) + self.tailIndex = entry.mergedIndex + } + } + case let .remove(tag, mergedIndices): + if tag == self.tag { + updated = true + for i in (0 ..< self.entries.count).reversed() { + if mergedIndices.contains(self.entries[i].mergedIndex) { + self.entries.remove(at: i) + } + } + if let tailIndex = self.tailIndex, mergedIndices.contains(tailIndex) { + self.tailIndex = nil + invalidatedTail = true + } + } + } + } + + if updated { + if invalidatedTail { + self.tailIndex = getTailIndex(self.tag) + } + if self.entries.count < self.limit { + if let tailIndex = self.tailIndex { + if self.entries.isEmpty || self.entries.last!.mergedIndex < tailIndex { + var fromIndex: Int32 = 0 + if !self.entries.isEmpty { + fromIndex = self.entries.last!.mergedIndex + 1 + } + for entry in getOperations(self.tag, fromIndex, self.limit - self.entries.count) { + self.entries.append(entry) + } + for i in 0 ..< self.entries.count { + if i != 0 { + assert(self.entries[i].mergedIndex == self.entries[i - 1].mergedIndex + 1) + } + } + if !self.entries.isEmpty { + assert(self.entries.last!.mergedIndex <= tailIndex) + } + } + } + } else { + assert(self.tailIndex != nil) + if let tailIndex = self.tailIndex { + assert(self.entries.last!.mergedIndex == tailIndex) + } + } + } + + return updated + } +} + +public final class PeerMergedOperationLogView { + public let entries: [PeerMergedOperationLogEntry] + + init(_ view: MutablePeerMergedOperationLogView) { + self.entries = view.entries + } +} diff --git a/Postbox/PeerNameIndexTable.swift b/Postbox/PeerNameIndexTable.swift index ebd23ea4e1..f5991b76c0 100644 --- a/Postbox/PeerNameIndexTable.swift +++ b/Postbox/PeerNameIndexTable.swift @@ -173,7 +173,7 @@ final class PeerNameIndexTable: Table { if let peer = self.peerTable.get(peerId) { updatedTokens = peer.indexName.indexTokens } else { - assertionFailure() + //assertionFailure() updatedTokens = [] } } diff --git a/Postbox/PeerOperationLogMetadataTable.swift b/Postbox/PeerOperationLogMetadataTable.swift index ae413c7ec7..aacfda417b 100644 --- a/Postbox/PeerOperationLogMetadataTable.swift +++ b/Postbox/PeerOperationLogMetadataTable.swift @@ -2,6 +2,7 @@ import Foundation private enum PeerOperationLogIndexNamespace: Int8 { case nextTagLocalIndex = 0 + case nextMergedIndex = 1 } final class PeerOperationLogMetadataTable: Table { @@ -9,11 +10,17 @@ final class PeerOperationLogMetadataTable: Table { return ValueBoxTable(id: id, keyType: .binary) } + private func keyForNextMergedIndex() -> ValueBoxKey { + let key = ValueBoxKey(length: 1) + key.setInt8(0, value: PeerOperationLogIndexNamespace.nextMergedIndex.rawValue) + return key + } + private func keyForNextLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> ValueBoxKey { let key = ValueBoxKey(length: 1 + 8 + 1) key.setInt8(0, value: PeerOperationLogIndexNamespace.nextTagLocalIndex.rawValue) key.setInt64(1, value: peerId.toInt64()) - key.setInt8(9, value: tag.rawValue) + key.setUInt8(9, value: tag.rawValue) return key } @@ -35,7 +42,29 @@ final class PeerOperationLogMetadataTable: Table { func takeNextLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> Int32 { let index = self.getNextLocalIndex(peerId: peerId, tag: tag) - self.setNextLocalIndex(peerId: peerId, tag: tag, index: index) + self.setNextLocalIndex(peerId: peerId, tag: tag, index: index + 1) + return index + } + + func getNextMergedIndex() -> Int32 { + if let value = self.valueBox.get(self.table, key: self.keyForNextMergedIndex()) { + assert(value.length == 4) + var index: Int32 = 0 + value.read(&index, offset: 0, length: 4) + return index + } else { + return 1 + } + } + + private func setNextMergedIndex(_ index: Int32) { + var indexValue: Int32 = index + self.valueBox.set(self.table, key: self.keyForNextMergedIndex(), value: MemoryBuffer(memory: &indexValue, capacity: 4, length: 4, freeWhenDone: false)) + } + + func takeNextMergedIndex() -> Int32 { + let index = self.getNextMergedIndex() + self.setNextMergedIndex(index + 1) return index } } diff --git a/Postbox/PeerOperationLogTable.swift b/Postbox/PeerOperationLogTable.swift index a6d83f96f9..c2fb7a048d 100644 --- a/Postbox/PeerOperationLogTable.swift +++ b/Postbox/PeerOperationLogTable.swift @@ -1,24 +1,119 @@ import Foundation -public enum PeerOperationLogTag: Int8 { - case inbox = 0 - case outbox = 1 +enum PeerMergedOperationLogOperation { + case append(PeerMergedOperationLogEntry) + case remove(tag: PeerOperationLogTag, mergedIndices: Set) +} + +public struct PeerMergedOperationLogEntry { + public let peerId: PeerId + public let tag: PeerOperationLogTag + public let tagLocalIndex: Int32 + public let mergedIndex: Int32 + public let contents: Coding +} + +public enum StorePeerOperationLogEntryTagLocalIndex { + case automatic + case manual(Int32) +} + +public enum StorePeerOperationLogEntryTagMergedIndex { + case none + case automatic } public struct PeerOperationLogEntry { - let tagLocalIndex: Int32 - let contents: Coding + public let peerId: PeerId + public let tag: PeerOperationLogTag + public let tagLocalIndex: Int32 + public let mergedIndex: Int32? + public let contents: Coding + + public func withUpdatedContents(_ contents: Coding) -> PeerOperationLogEntry { + return PeerOperationLogEntry(peerId: self.peerId, tag: self.tag, tagLocalIndex: self.tagLocalIndex, mergedIndex: self.mergedIndex, contents: contents) + } + + public var mergedEntry: PeerMergedOperationLogEntry? { + if let mergedIndex = self.mergedIndex { + return PeerMergedOperationLogEntry(peerId: self.peerId, tag: self.tag, tagLocalIndex: self.tagLocalIndex, mergedIndex: mergedIndex, contents: self.contents) + } else { + return nil + } + } } -final class PeerOperationLogQueue: Table { +public struct PeerOperationLogTag: Equatable { + let rawValue: UInt8 + + public init(value: Int) { + self.rawValue = UInt8(value) + } + + public static func ==(lhs: PeerOperationLogTag, rhs: PeerOperationLogTag) -> Bool { + return lhs.rawValue == rhs.rawValue + } +} + +public enum PeerOperationLogEntryUpdateContents { + case none + case update(Coding) +} + +public enum PeerOperationLogEntryUpdateTagMergedIndex { + case none + case remove + case newAutomatic +} + +public struct PeerOperationLogEntryUpdate { + let mergedIndex: PeerOperationLogEntryUpdateTagMergedIndex + let contents: PeerOperationLogEntryUpdateContents + + public init(mergedIndex: PeerOperationLogEntryUpdateTagMergedIndex, contents: PeerOperationLogEntryUpdateContents) { + self.mergedIndex = mergedIndex + self.contents = contents + } +} + +private func parseEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32, _ value: ReadBuffer) -> PeerOperationLogEntry? { + var hasMergedIndex: Int8 = 0 + value.read(&hasMergedIndex, offset: 0, length: 1) + var mergedIndex: Int32? + if hasMergedIndex != 0 { + var mergedIndexValue: Int32 = 0 + value.read(&mergedIndexValue, offset: 0, length: 4) + mergedIndex = mergedIndexValue + } + var contentLength: Int32 = 0 + value.read(&contentLength, offset: 0, length: 4) + assert(value.length - value.offset == Int(contentLength)) + if let contents = Decoder(buffer: MemoryBuffer(memory: value.memory.advanced(by: value.offset), capacity: Int(contentLength), length: Int(contentLength), freeWhenDone: false)).decodeRootObject() { + return PeerOperationLogEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, mergedIndex: mergedIndex, contents: contents) + } else { + return nil + } +} + +private func parseMergedEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32, _ value: ReadBuffer) -> PeerMergedOperationLogEntry? { + if let entry = parseEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, value), let mergedIndex = entry.mergedIndex { + return PeerMergedOperationLogEntry(peerId: entry.peerId, tag: entry.tag, tagLocalIndex: entry.tagLocalIndex, mergedIndex: mergedIndex, contents: entry.contents) + } else { + return nil + } +} + +final class PeerOperationLogTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .binary) } private let metadataTable: PeerOperationLogMetadataTable + private let mergedIndexTable: PeerMergedOperationLogIndexTable - init(valueBox: ValueBox, table: ValueBoxTable, metadataTable: PeerOperationLogMetadataTable) { + init(valueBox: ValueBox, table: ValueBoxTable, metadataTable: PeerOperationLogMetadataTable, mergedIndexTable: PeerMergedOperationLogIndexTable) { self.metadataTable = metadataTable + self.mergedIndexTable = mergedIndexTable super.init(valueBox: valueBox, table: table) } @@ -26,28 +121,229 @@ final class PeerOperationLogQueue: Table { private func key(peerId: PeerId, tag: PeerOperationLogTag, index: Int32) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 1 + 4) key.setInt64(0, value: peerId.toInt64()) - key.setInt8(1, value: tag.rawValue) + key.setUInt8(1, value: tag.rawValue) key.setInt32(9, value: index) return key } - func addEntryAndTakeNextTagLocalIndex(peerId: PeerId, tag: PeerOperationLogTag, contents: Coding) -> Int32 { - let index = self.metadataTable.takeNextLocalIndex(peerId: peerId, tag: tag) - let encoder = Encoder() - encoder.encodeRootObject(contents) - self.valueBox.set(self.table, key: self.key(peerId: peerId, tag: tag, index: index), value: encoder.readBufferNoCopy()) - return index + func getNextEntryLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> Int32 { + return self.metadataTable.getNextLocalIndex(peerId: peerId, tag: tag) } - func removeEntries(peerId: PeerId, tag: PeerOperationLogTag, withIndicesLowerThan index: Int32) { + func addEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: StorePeerOperationLogEntryTagLocalIndex, tagMergedIndex: StorePeerOperationLogEntryTagMergedIndex, contents: Coding, operations: inout [PeerMergedOperationLogOperation]) { + let index: Int32 + switch tagLocalIndex { + case .automatic: + index = self.metadataTable.takeNextLocalIndex(peerId: peerId, tag: tag) + case let .manual(manualIndex): + index = manualIndex + } + + var mergedIndex: Int32? + switch tagMergedIndex { + case .automatic: + mergedIndex = self.mergedIndexTable.add(peerId: peerId, tag: tag, tagLocalIndex: index) + case .none: + break + } + + let buffer = WriteBuffer() + var hasMergedIndex: Int8 = mergedIndex != nil ? 1 : 0 + buffer.write(&hasMergedIndex, offset: 0, length: 1) + if let mergedIndex = mergedIndex { + var mergedIndexValue: Int32 = mergedIndex + buffer.write(&mergedIndexValue, offset: 0, length: 4) + } + + let encoder = Encoder() + encoder.encodeRootObject(contents) + let contentBuffer = encoder.readBufferNoCopy() + var contentBufferLength: Int32 = Int32(contentBuffer.length) + buffer.write(&contentBufferLength, offset: 0, length: 4) + buffer.write(contentBuffer.memory, offset: 0, length: contentBuffer.length) + + self.valueBox.set(self.table, key: self.key(peerId: peerId, tag: tag, index: index), value: buffer) + if let mergedIndex = mergedIndex { + operations.append(.append(PeerMergedOperationLogEntry(peerId: peerId, tag: tag, tagLocalIndex: index, mergedIndex: mergedIndex, contents: contents))) + } + } + + func removeEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex index: Int32, operations: inout [PeerMergedOperationLogOperation]) -> Bool { var indices: [Int32] = [] - self.valueBox.range(self.table, start: self.key(peerId: peerId, tag: tag, index: 0).predecessor, end: self.key(peerId: peerId, tag: tag, index: index), keys: { key in - indices.append(key.getInt32(9)) + var mergedIndices: [Int32] = [] + var removed = false + if let value = self.valueBox.get(self.table, key: self.key(peerId: peerId, tag: tag, index: index)) { + indices.append(index) + var hasMergedIndex: Int8 = 0 + value.read(&hasMergedIndex, offset: 0, length: 1) + if hasMergedIndex != 0 { + var mergedIndex: Int32 = 0 + value.read(&mergedIndex, offset: 0, length: 4) + mergedIndices.append(mergedIndex) + } + removed = true + } + + for index in indices { + self.valueBox.remove(self.table, key: self.key(peerId: peerId, tag: tag, index: index)) + } + + if !mergedIndices.isEmpty { + self.mergedIndexTable.remove(tag: tag, mergedIndices: mergedIndices) + operations.append(.remove(tag: tag, mergedIndices: Set(mergedIndices))) + } + return removed + } + + func removeAllEntries(peerId: PeerId, tag: PeerOperationLogTag, operations: inout [PeerMergedOperationLogOperation]) { + var indices: [Int32] = [] + var mergedIndices: [Int32] = [] + self.valueBox.range(self.table, start: self.key(peerId: peerId, tag: tag, index: 0).predecessor, end: self.key(peerId: peerId, tag: tag, index: Int32.max).successor, values: { key, value in + let index = key.getInt32(9) + indices.append(index) + var hasMergedIndex: Int8 = 0 + value.read(&hasMergedIndex, offset: 0, length: 1) + if hasMergedIndex != 0 { + var mergedIndex: Int32 = 0 + value.read(&mergedIndex, offset: 0, length: 4) + mergedIndices.append(mergedIndex) + } return true }, limit: 0) for index in indices { self.valueBox.remove(self.table, key: self.key(peerId: peerId, tag: tag, index: index)) } + + if !mergedIndices.isEmpty { + self.mergedIndexTable.remove(tag: tag, mergedIndices: mergedIndices) + operations.append(.remove(tag: tag, mergedIndices: Set(mergedIndices))) + } + } + + func removeEntries(peerId: PeerId, tag: PeerOperationLogTag, withTagLocalIndicesEqualToOrLowerThan maxTagLocalIndex: Int32, operations: inout [PeerMergedOperationLogOperation]) { + var indices: [Int32] = [] + var mergedIndices: [Int32] = [] + self.valueBox.range(self.table, start: self.key(peerId: peerId, tag: tag, index: 0).predecessor, end: self.key(peerId: peerId, tag: tag, index: maxTagLocalIndex).successor, values: { key, value in + let index = key.getInt32(9) + indices.append(index) + var hasMergedIndex: Int8 = 0 + value.read(&hasMergedIndex, offset: 0, length: 1) + if hasMergedIndex != 0 { + var mergedIndex: Int32 = 0 + value.read(&mergedIndex, offset: 0, length: 4) + mergedIndices.append(mergedIndex) + } + return true + }, limit: 0) + + for index in indices { + self.valueBox.remove(self.table, key: self.key(peerId: peerId, tag: tag, index: index)) + } + + if !mergedIndices.isEmpty { + self.mergedIndexTable.remove(tag: tag, mergedIndices: mergedIndices) + operations.append(.remove(tag: tag, mergedIndices: Set(mergedIndices))) + } + } + + func getMergedEntries(tag: PeerOperationLogTag, fromIndex: Int32, limit: Int) -> [PeerMergedOperationLogEntry] { + var entries: [PeerMergedOperationLogEntry] = [] + for (peerId, tagLocalIndex) in self.mergedIndexTable.getTagLocalIndices(tag: tag, fromMergedIndex: fromIndex, limit: limit) { + if let value = self.valueBox.get(self.table, key: self.key(peerId: peerId, tag: tag, index: tagLocalIndex)) { + if let entry = parseMergedEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, value) { + entries.append(entry) + } else { + assertionFailure() + } + } else { + assertionFailure() + } + } + return entries + } + + func enumerateEntries(peerId: PeerId, tag: PeerOperationLogTag, _ f: (PeerOperationLogEntry) -> Bool) { + self.valueBox.range(self.table, start: self.key(peerId: peerId, tag: tag, index: 0).predecessor, end: self.key(peerId: peerId, tag: tag, index: Int32.max).successor, values: { key, value in + if let entry = parseEntry(peerId: peerId, tag: tag, tagLocalIndex: key.getInt32(9), value) { + if !f(entry) { + return false + } + } else { + assertionFailure() + } + return true + }, limit: 0) + } + + func updateEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32, f: (PeerOperationLogEntry?) -> PeerOperationLogEntryUpdate, operations: inout [PeerMergedOperationLogOperation]) { + let key = self.key(peerId: peerId, tag: tag, index: tagLocalIndex) + if let value = self.valueBox.get(self.table, key: key) { + var hasMergedIndex: Int8 = 0 + value.read(&hasMergedIndex, offset: 0, length: 1) + var mergedIndex: Int32? + if hasMergedIndex != 0 { + var mergedIndexValue: Int32 = 0 + value.read(&mergedIndexValue, offset: 0, length: 4) + mergedIndex = mergedIndexValue + } + let previousMergedIndex = mergedIndex + var contentLength: Int32 = 0 + value.read(&contentLength, offset: 0, length: 4) + assert(value.length - value.offset == Int(contentLength)) + if let contents = Decoder(buffer: MemoryBuffer(memory: value.memory.advanced(by: value.offset), capacity: Int(contentLength), length: Int(contentLength), freeWhenDone: false)).decodeRootObject() { + let entryUpdate = f(PeerOperationLogEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, mergedIndex: mergedIndex, contents: contents)) + switch entryUpdate.mergedIndex { + case .none: + break + case .remove: + if let mergedIndexValue = mergedIndex { + mergedIndex = nil + self.mergedIndexTable.remove(tag: tag, mergedIndices: [mergedIndexValue]) + operations.append(.remove(tag: tag, mergedIndices: Set([mergedIndexValue]))) + } + case .newAutomatic: + if let mergedIndexValue = mergedIndex { + self.mergedIndexTable.remove(tag: tag, mergedIndices: [mergedIndexValue]) + operations.append(.remove(tag: tag, mergedIndices: Set([mergedIndexValue]))) + } + let updatedMergedIndexValue = self.mergedIndexTable.add(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex) + mergedIndex = updatedMergedIndexValue + operations.append(.append(PeerMergedOperationLogEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, mergedIndex: updatedMergedIndexValue, contents: contents))) + } + var updatedContents: Coding? + switch entryUpdate.contents { + case .none: + break + case let .update(contents): + updatedContents = contents + } + if previousMergedIndex != mergedIndex || updatedContents != nil { + let buffer = WriteBuffer() + var hasMergedIndex: Int8 = mergedIndex != nil ? 1 : 0 + buffer.write(&hasMergedIndex, offset: 0, length: 1) + if let mergedIndex = mergedIndex { + var mergedIndexValue: Int32 = mergedIndex + buffer.write(&mergedIndexValue, offset: 0, length: 4) + } + + let encoder = Encoder() + if let updatedContents = updatedContents { + encoder.encodeRootObject(updatedContents) + } else { + encoder.encodeRootObject(contents) + } + let contentBuffer = encoder.readBufferNoCopy() + var contentBufferLength: Int32 = Int32(contentBuffer.length) + buffer.write(&contentBufferLength, offset: 0, length: 4) + buffer.write(contentBuffer.memory, offset: 0, length: contentBuffer.length) + self.valueBox.set(self.table, key: key, value: buffer) + } + } else { + assertionFailure() + } + } else { + f(nil) + } } } diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index 391b292ca0..b5ade8d940 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -15,6 +15,11 @@ final class MutablePeerView { self.peerIsContact = peerIsContact var peerIds = Set() peerIds.insert(peerId) + if let peer = getPeer(peerId), let associatedPeerIds = peer.associatedPeerIds { + for peerId in associatedPeerIds { + peerIds.insert(peerId) + } + } if let cachedData = cachedData { peerIds.formUnion(cachedData.peerIds) } @@ -26,6 +31,16 @@ final class MutablePeerView { self.peerPresences[id] = presence } } + if let peer = self.peers[peerId], let associatedPeerIds = peer.associatedPeerIds { + for id in associatedPeerIds { + if let peer = getPeer(id) { + self.peers[id] = peer + } + if let presence = getPeerPresence(id) { + self.peerPresences[id] = presence + } + } + } } func replay(updatedPeers: [PeerId: Peer], updatedNotificationSettings: [PeerId: PeerNotificationSettings], updatedCachedPeerData: [PeerId: CachedPeerData], updatedPeerPresences: [PeerId: PeerPresence], replaceContactPeerIds: Set?, getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) -> Bool { @@ -37,6 +52,11 @@ final class MutablePeerView { var peerIds = Set() peerIds.insert(self.peerId) + if let peer = getPeer(self.peerId), let associatedPeerIds = peer.associatedPeerIds { + for peerId in associatedPeerIds { + peerIds.insert(peerId) + } + } peerIds.formUnion(cachedData.peerIds) for id in peerIds { @@ -77,6 +97,11 @@ final class MutablePeerView { } else { var peerIds = Set() peerIds.insert(self.peerId) + if let peer = getPeer(self.peerId), let associatedPeerIds = peer.associatedPeerIds { + for peerId in associatedPeerIds { + peerIds.insert(peerId) + } + } if let cachedData = self.cachedData { peerIds.formUnion(cachedData.peerIds) } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 155a1b195a..75ad3ecbc5 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -173,6 +173,46 @@ public final class Modifier { public func retrieveItemCacheEntry(id: ItemCacheEntryId) -> Coding? { return self.postbox?.retrieveItemCacheEntry(id: id) } + + public func operationLogGetNextEntryLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> Int32 { + if let postbox = self.postbox { + return postbox.operationLogGetNextEntryLocalIndex(peerId: peerId, tag: tag) + } else { + return 0 + } + } + + public func operationLogAddEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: StorePeerOperationLogEntryTagLocalIndex, tagMergedIndex: StorePeerOperationLogEntryTagMergedIndex, contents: Coding) { + self.postbox?.operationLogAddEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, tagMergedIndex: tagMergedIndex, contents: contents) + } + + public func operationLogRemoveEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32) -> Bool { + if let postbox = self.postbox { + return postbox.operationLogRemoveEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex) + } else { + return false + } + } + + public func operationLogRemoveAllEntries(peerId: PeerId, tag: PeerOperationLogTag) { + self.postbox?.operationLogRemoveAllEntries(peerId: peerId, tag: tag) + } + + public func operationLogRemoveEntries(peerId: PeerId, tag: PeerOperationLogTag, withTagLocalIndicesEqualToOrLowerThan maxTagLocalIndex: Int32) { + self.postbox?.operationLogRemoveEntries(peerId: peerId, tag: tag, withTagLocalIndicesEqualToOrLowerThan: maxTagLocalIndex) + } + + public func operationLogUpdateEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32, _ f: (PeerOperationLogEntry?) -> PeerOperationLogEntryUpdate) { + self.postbox?.operationLogUpdateEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, f) + } + + public func operationLogEnumerateEntries(peerId: PeerId, tag: PeerOperationLogTag, _ f: (PeerOperationLogEntry) -> Bool) { + self.postbox?.operationLogEnumerateEntries(peerId: peerId, tag: tag, f) + } + + /*public func operationLogTransformOperationContent(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32, _ f: (PeerMergedOperationLogEntry?) -> PeerMergedOperationLogEntryTransform) { + self.postbox?.operationLogTransformOperationContent(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, f) + }*/ } fileprivate class PipeNotifier: NSObject { @@ -225,6 +265,7 @@ public final class Postbox { private var currentUpdatedPeerPresences: [PeerId: PeerPresence] = [:] private var currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] private var currentUpdatedTotalUnreadCount: Int32? + private var currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation] = [] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -270,6 +311,9 @@ public final class Postbox { var peerNameTokenIndexTable: PeerNameTokenIndexTable! var peerNameIndexTable: PeerNameIndexTable! var peerChatTopTaggedMessageIdsTable: PeerChatTopTaggedMessageIdsTable! + var peerOperationLogMetadataTable: PeerOperationLogMetadataTable! + var peerMergedOperationLogIndexTable: PeerMergedOperationLogIndexTable! + var peerOperationLogTable: PeerOperationLogTable! //temporary var peerRatingTable: RatingTable! @@ -386,6 +430,9 @@ public final class Postbox { self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8), peerNameIndexTable: self.peerNameIndexTable, metadataTable: self.messageHistoryMetadataTable, readStateTable: self.readStateTable, notificationSettingsTable: self.peerNotificationSettingsTable) self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.peerChatTopTaggedMessageIdsTable = PeerChatTopTaggedMessageIdsTable(valueBox: self.valueBox, table: PeerChatTopTaggedMessageIdsTable.tableSpec(28)) + self.peerOperationLogMetadataTable = PeerOperationLogMetadataTable(valueBox: self.valueBox, table: PeerOperationLogMetadataTable.tableSpec(29)) + self.peerMergedOperationLogIndexTable = PeerMergedOperationLogIndexTable(valueBox: self.valueBox, table: PeerMergedOperationLogIndexTable.tableSpec(30), metadataTable: self.peerOperationLogMetadataTable!) + self.peerOperationLogTable = PeerOperationLogTable(valueBox: self.valueBox, table: PeerOperationLogTable.tableSpec(31), metadataTable: self.peerOperationLogMetadataTable, mergedIndexTable: self.peerMergedOperationLogIndexTable) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -414,6 +461,9 @@ public final class Postbox { self.tables.append(self.peerNameIndexTable) self.tables.append(self.peerNameTokenIndexTable) self.tables.append(self.peerChatTopTaggedMessageIdsTable) + self.tables.append(self.peerOperationLogMetadataTable) + self.tables.append(self.peerMergedOperationLogIndexTable) + self.tables.append(self.peerOperationLogTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() @@ -429,6 +479,10 @@ public final class Postbox { return self.messageHistoryMetadataTable.getChatListTotalUnreadCount() }, getPeerReadState: { peerId in return self.readStateTable.getCombinedState(peerId) + }, operationLogGetOperations: { tag, fromIndex, limit in + return self.peerOperationLogTable.getMergedEntries(tag: tag, fromIndex: fromIndex, limit: limit) + }, operationLogGetTailIndex: { tag in + return self.peerMergedOperationLogIndexTable.tailIndex(tag: tag) }, unsentMessageIds: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") @@ -800,7 +854,7 @@ public final class Postbox { let transactionParticipationInTotalUnreadCountUpdates = self.peerNotificationSettingsTable.transactionParticipationInTotalUnreadCountUpdates() self.chatListIndexTable.commitWithTransactionUnreadCountDeltas(transactionUnreadCountDeltas, transactionParticipationInTotalUnreadCountUpdates: transactionParticipationInTotalUnreadCountUpdates, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) - let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -828,6 +882,8 @@ public final class Postbox { self.currentUpdatedCachedPeerData.removeAll() self.currentUpdatedPeerPresences.removeAll() self.currentUpdatedPeerChatListEmbeddedStates.removeAll() + self.currentUpdatedTotalUnreadCount = nil + self.currentPeerMergedOperationLogOperations.removeAll() for table in self.tables { table.beforeCommit() @@ -1398,6 +1454,55 @@ public final class Postbox { } } + public func mergedOperationLogView(tag: PeerOperationLogTag, limit: Int) -> Signal { + return self.modify { modifier -> Signal in + let view = MutablePeerMergedOperationLogView(tag: tag, limit: limit, getOperations: { tag, fromIndex, limit in + return self.peerOperationLogTable.getMergedEntries(tag: tag, fromIndex: fromIndex, limit: limit) + }, getTailIndex: { tag in + return self.peerMergedOperationLogIndexTable.tailIndex(tag: tag) + }) + let (index, signal) = self.viewTracker.addPeerMergedOperationLogView(view) + + return (.single(PeerMergedOperationLogView(view)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removePeerMergedOperationLogView(index) + } + } + } + } |> switchToLatest + } + + fileprivate func operationLogGetNextEntryLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> Int32 { + return self.peerOperationLogTable.getNextEntryLocalIndex(peerId: peerId, tag: tag) + } + + fileprivate func operationLogAddEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: StorePeerOperationLogEntryTagLocalIndex, tagMergedIndex: StorePeerOperationLogEntryTagMergedIndex, contents: Coding) { + self.peerOperationLogTable.addEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, tagMergedIndex: tagMergedIndex, contents: contents, operations: &self.currentPeerMergedOperationLogOperations) + } + + fileprivate func operationLogRemoveEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32) -> Bool { + return self.peerOperationLogTable.removeEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, operations: &self.currentPeerMergedOperationLogOperations) + } + + fileprivate func operationLogRemoveAllEntries(peerId: PeerId, tag: PeerOperationLogTag) { + self.peerOperationLogTable.removeAllEntries(peerId: peerId, tag: tag, operations: &self.currentPeerMergedOperationLogOperations) + } + + fileprivate func operationLogRemoveEntries(peerId: PeerId, tag: PeerOperationLogTag, withTagLocalIndicesEqualToOrLowerThan maxTagLocalIndex: Int32) { + self.peerOperationLogTable.removeEntries(peerId: peerId, tag: tag, withTagLocalIndicesEqualToOrLowerThan: maxTagLocalIndex, operations: &self.currentPeerMergedOperationLogOperations) + } + + fileprivate func operationLogUpdateEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32, _ f: (PeerOperationLogEntry?) -> PeerOperationLogEntryUpdate) { + self.peerOperationLogTable.updateEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, f: f, operations: &self.currentPeerMergedOperationLogOperations) + } + + fileprivate func operationLogEnumerateEntries(peerId: PeerId, tag: PeerOperationLogTag, _ f: (PeerOperationLogEntry) -> Bool) { + self.peerOperationLogTable.enumerateEntries(peerId: peerId, tag: tag, f) + } + public func isMasterClient() -> Signal { return self.modify { modifier -> Signal in let sessionClientId = self.sessionClientId diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index 673b7f5636..94144804db 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -12,6 +12,7 @@ final class PostboxTransaction { let currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] let currentUpdatedTotalUnreadCount: Int32? let peerIdsWithUpdatedUnreadCounts: Set + let currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation] let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -68,10 +69,13 @@ final class PostboxTransaction { if !peerIdsWithUpdatedUnreadCounts.isEmpty { return false } + if !currentPeerMergedOperationLogOperations.isEmpty { + return false + } return true } - init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles self.removedHolesByPeerId = removedHolesByPeerId @@ -83,6 +87,7 @@ final class PostboxTransaction { self.currentUpdatedPeerChatListEmbeddedStates = currentUpdatedPeerChatListEmbeddedStates self.currentUpdatedTotalUnreadCount = currentUpdatedTotalUnreadCount self.peerIdsWithUpdatedUnreadCounts = peerIdsWithUpdatedUnreadCounts + self.currentPeerMergedOperationLogOperations = currentPeerMergedOperationLogOperations self.unsentMessageOperations = unsentMessageOperations self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations self.updatedMedia = updatedMedia diff --git a/Postbox/SeedConfiguration.swift b/Postbox/SeedConfiguration.swift index cd0f8320be..60845ed053 100644 --- a/Postbox/SeedConfiguration.swift +++ b/Postbox/SeedConfiguration.swift @@ -2,10 +2,10 @@ import Foundation public final class SeedConfiguration { let initializeChatListWithHoles: [ChatListHole] - let initializeMessageNamespacesWithHoles: [MessageId.Namespace] + let initializeMessageNamespacesWithHoles: [(PeerId.Namespace, MessageId.Namespace)] let existingMessageTags: MessageTags - public init(initializeChatListWithHoles: [ChatListHole], initializeMessageNamespacesWithHoles: [MessageId.Namespace], existingMessageTags: MessageTags) { + public init(initializeChatListWithHoles: [ChatListHole], initializeMessageNamespacesWithHoles: [(PeerId.Namespace, MessageId.Namespace)], existingMessageTags: MessageTags) { self.initializeChatListWithHoles = initializeChatListWithHoles self.initializeMessageNamespacesWithHoles = initializeMessageNamespacesWithHoles self.existingMessageTags = existingMessageTags diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index e88a1b97a2..2099139e9b 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -58,6 +58,11 @@ public struct ValueBoxKey: Hashable, CustomStringConvertible, Comparable { memcpy(self.memory + offset, &varValue, 1) } + public func setUInt8(_ offset: Int, value: UInt8) { + var varValue = value + memcpy(self.memory + offset, &varValue, 1) + } + public func getInt32(_ offset: Int) -> Int32 { var value: Int32 = 0 memcpy(&value, self.memory + offset, 4) @@ -82,6 +87,12 @@ public struct ValueBoxKey: Hashable, CustomStringConvertible, Comparable { return value } + public func getUInt8(_ offset: Int) -> UInt8 { + var value: UInt8 = 0 + memcpy(&value, self.memory + offset, 1) + return value + } + public func prefix(_ length: Int) -> ValueBoxKey { assert(length <= self.length, "length <= self.length") let key = ValueBoxKey(length: length) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index f424a9572b..9489400b3c 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -26,6 +26,8 @@ final class ViewTracker { private let getPeerPresence: (PeerId) -> PeerPresence? private let getTotalUnreadCount: () -> Int32 private let getPeerReadState: (PeerId) -> CombinedPeerReadState? + private let operationLogGetOperations: (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry] + private let operationLogGetTailIndex: (PeerOperationLogTag) -> Int32? private var chatListViews = Bag<(MutableChatListView, ValuePipe<(ChatListView, ViewUpdateType)>)>() private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] @@ -47,8 +49,9 @@ final class ViewTracker { private var peerViews = Bag<(MutablePeerView, ValuePipe)>() private var unreadMessageCountsViews = Bag<(MutableUnreadMessageCountsView, ValuePipe)>() + private var peerMergedOperationLogViews = Bag<(MutablePeerMergedOperationLogView, ValuePipe)>() - init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, operationLogGetOperations: @escaping (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], operationLogGetTailIndex: @escaping (PeerOperationLogTag) -> Int32?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -62,6 +65,8 @@ final class ViewTracker { self.getPeerPresence = getPeerPresence self.getTotalUnreadCount = getTotalUnreadCount self.getPeerReadState = getPeerReadState + self.operationLogGetOperations = operationLogGetOperations + self.operationLogGetTailIndex = operationLogGetTailIndex self.unsentMessageView = UnsentMessageHistoryView(ids: unsentMessageIds) self.synchronizeReadStatesView = MutableSynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) @@ -182,6 +187,17 @@ final class ViewTracker { self.peerViews.remove(index) } + func addPeerMergedOperationLogView(_ view: MutablePeerMergedOperationLogView) -> (Bag<(MutablePeerMergedOperationLogView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.peerMergedOperationLogViews.add(record) + + return (index, record.1.signal()) + } + + func removePeerMergedOperationLogView(_ index: Bag<(MutablePeerMergedOperationLogView, ValuePipe)>.Index) { + self.peerMergedOperationLogViews.remove(index) + } + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: MessageIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] @@ -358,6 +374,12 @@ final class ViewTracker { pipe.putNext(PeerView(mutableView)) } } + + for (mutableView, pipe) in self.peerMergedOperationLogViews.copyItems() { + if mutableView.replay(operations: transaction.currentPeerMergedOperationLogOperations, getOperations: self.operationLogGetOperations, getTailIndex: self.operationLogGetTailIndex) { + pipe.putNext(PeerMergedOperationLogView(mutableView)) + } + } } private func updateTrackedChatListHoles() { From d0b0d0dd4fdfcc7f4a40fee4e78fe96e6c63e40d Mon Sep 17 00:00:00 2001 From: overtake Date: Wed, 18 Jan 2017 11:28:55 +0300 Subject: [PATCH 071/427] change --- Postbox.xcodeproj/project.pbxproj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index a8cadc21be..c016fbea44 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -7,6 +7,10 @@ objects = { /* Begin PBXBuildFile section */ + C2A315BC1E2E730400D89000 /* PeerOperationLogTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */; }; + C2A315BD1E2E732000D89000 /* PeerOperationLogMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */; }; + C2A315BE1E2E733900D89000 /* PeerMergedOperationLogIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */; }; + C2A315BF1E2E733900D89000 /* PeerMergedOperationLogView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */; }; C2AC9C131E1E5D200085C7DE /* UnreadMessageCountsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */; }; D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; @@ -760,6 +764,10 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + C2A315BE1E2E733900D89000 /* PeerMergedOperationLogIndexTable.swift in Sources */, + C2A315BF1E2E733900D89000 /* PeerMergedOperationLogView.swift in Sources */, + C2A315BD1E2E732000D89000 /* PeerOperationLogMetadataTable.swift in Sources */, + C2A315BC1E2E730400D89000 /* PeerOperationLogTable.swift in Sources */, C2AC9C131E1E5D200085C7DE /* UnreadMessageCountsView.swift in Sources */, D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, D0F7B1DF1E045C6A007EB8A5 /* ItemCacheMetaTable.swift in Sources */, From 48c1c227a5cc25cba0abd5191091466e38717890 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 22 Jan 2017 23:45:06 +0300 Subject: [PATCH 072/427] no message --- Postbox.xcodeproj/project.pbxproj | 22 +- Postbox/ChatListIndexTable.swift | 58 +++- Postbox/ChatListTable.swift | 146 +++++---- Postbox/ChatListView.swift | 42 +-- Postbox/Message.swift | 51 ++- Postbox/MessageGloballyUniqueIdTable.swift | 21 +- Postbox/MessageHistoryHolesView.swift | 2 +- Postbox/MessageHistoryIndexTable.swift | 172 ++++++++++- ...sageHistoryInvalidatedReadStateTable.swift | 15 +- Postbox/MessageHistoryReadStateTable.swift | 292 +++++++++++++----- Postbox/MessageHistoryTable.swift | 247 +++++++++++---- Postbox/MessageHistoryView.swift | 75 ++++- Postbox/Peer.swift | 1 + Postbox/PeerNameIndexTable.swift | 2 +- Postbox/PeerReadState.swift | 87 ++++-- Postbox/Postbox.swift | 158 ++++++++-- Postbox/PostboxTransaction.swift | 7 +- ...tampBasedMessageAttributesIndexTable.swift | 35 +++ ...TimestampBasedMessageAttributesTable.swift | 89 ++++++ .../TimestampBasedMessageAttributesView.swift | 52 ++++ Postbox/ValueBoxKey.swift | 11 + Postbox/ViewTracker.swift | 29 +- PostboxTests/ChatListTableTests.swift | 6 +- .../MessageHistoryIndexTableTests.swift | 110 ++++++- PostboxTests/MessageHistoryTableTests.swift | 68 +++- PostboxTests/OrderStatisticTreeTests.swift | 4 +- PostboxTests/RandomAccessResourceTests.swift | 6 +- PostboxTests/ReadStateTableTests.swift | 27 +- 28 files changed, 1499 insertions(+), 336 deletions(-) create mode 100644 Postbox/TimestampBasedMessageAttributesIndexTable.swift create mode 100644 Postbox/TimestampBasedMessageAttributesTable.swift create mode 100644 Postbox/TimestampBasedMessageAttributesView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 4bba6c3596..d4b22bc570 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -14,6 +14,7 @@ D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; D010B61A1E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */; }; + D019B1CF1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */; }; D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */; }; @@ -79,12 +80,15 @@ D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; D0A18D671E16874D004C6734 /* UnreadMessageCountsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */; }; D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; + D0AAD1B21E3266B100D5B9DE /* TimestampBasedMessageAttributesIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AAD1B01E3266B100D5B9DE /* TimestampBasedMessageAttributesIndexTable.swift */; }; + D0AAD1B31E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AAD1B11E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift */; }; + D0AAD1B51E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AAD1B41E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift */; }; + D0AAD1B61E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AAD1B41E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift */; }; D0AB0B8C1D65D488002C78E7 /* MessageHistoryHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */; }; D0AB0B8E1D65D49C002C78E7 /* ChatListHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */; }; D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */; }; D0AD23271E194D1C00A7089A /* PeerOperationLogTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */; }; D0AD23291E196B6400A7089A /* PeerOperationLogMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */; }; - D0AD232B1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AD232A1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift */; }; D0B418171D7DFAF3004562A4 /* PostboxMac.h in Headers */ = {isa = PBXBuildFile; fileRef = D0B418151D7DFAF3004562A4 /* PostboxMac.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0B4181C1D7DFDF1004562A4 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */ = {isa = PBXBuildFile; fileRef = D07516401B2D9CEF00AE42E0 /* sqlite3.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-ambiguous-macro -Wno-conditional-uninitialized -Wno-unused-const-variable -Wno-unused-function -Wno-unreachable-code"; }; }; @@ -219,6 +223,7 @@ D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameIndexRepresentation.swift; sourceTree = ""; }; D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerMergedOperationLogIndexTable.swift; sourceTree = ""; }; + D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageGloballyUniqueIdTable.swift; sourceTree = ""; }; D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollection.swift; sourceTree = ""; }; @@ -269,12 +274,14 @@ D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTable.swift; sourceTree = ""; }; D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnreadMessageCountsView.swift; sourceTree = ""; }; D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; + D0AAD1B01E3266B100D5B9DE /* TimestampBasedMessageAttributesIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TimestampBasedMessageAttributesIndexTable.swift; sourceTree = ""; }; + D0AAD1B11E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TimestampBasedMessageAttributesTable.swift; sourceTree = ""; }; + D0AAD1B41E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TimestampBasedMessageAttributesView.swift; sourceTree = ""; }; D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHolesView.swift; sourceTree = ""; }; D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatListHolesView.swift; sourceTree = ""; }; D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageIndicesView.swift; sourceTree = ""; }; D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerOperationLogTable.swift; sourceTree = ""; }; D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerOperationLogMetadataTable.swift; sourceTree = ""; }; - D0AD232A1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageGloballyUniqueIdTable.swift; sourceTree = ""; }; D0B418131D7DFAF2004562A4 /* PostboxMac.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = PostboxMac.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0B418151D7DFAF3004562A4 /* PostboxMac.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PostboxMac.h; sourceTree = ""; }; D0B418161D7DFAF3004562A4 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -415,8 +422,8 @@ isa = PBXGroup; children = ( D03BCCF71C73561C0097A291 /* Table.swift */, + D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */, D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */, - D0AD232A1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift */, D044CA2B1C617E2D002160FF /* MessageHistoryMetadataTable.swift */, D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */, D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, @@ -450,6 +457,8 @@ D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */, D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */, D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */, + D0AAD1B11E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift */, + D0AAD1B01E3266B100D5B9DE /* TimestampBasedMessageAttributesIndexTable.swift */, ); name = Tables; sourceTree = ""; @@ -515,6 +524,7 @@ D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */, D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */, D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */, + D0AAD1B41E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift */, ); name = Views; sourceTree = ""; @@ -789,6 +799,7 @@ D0F7B1D51E045C6A007EB8A5 /* PeerChatStateTable.swift in Sources */, D0F7B1C51E045C6A007EB8A5 /* MessageHistoryMetadataTable.swift in Sources */, D073CE781DCBF3B4007511FD /* MessageHistoryHole.swift in Sources */, + D0AAD1B61E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift in Sources */, D0F7B1C31E045C6A007EB8A5 /* Table.swift in Sources */, D0F7B1CF1E045C6A007EB8A5 /* MetadataTable.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, @@ -877,6 +888,7 @@ D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */, D0AB0B8C1D65D488002C78E7 /* MessageHistoryHolesView.swift in Sources */, + D0AAD1B51E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift in Sources */, D010B61A1E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift in Sources */, D0F9E8671C58D08900037222 /* ChatListIndexTable.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, @@ -888,7 +900,6 @@ D07827C31E008F7300071108 /* PeerNameTokenIndexTable.swift in Sources */, D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, - D0AD232B1E1974C000A7089A /* MessageGloballyUniqueIdTable.swift in Sources */, D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */, D0AD23271E194D1C00A7089A /* PeerOperationLogTable.swift in Sources */, D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */, @@ -897,6 +908,7 @@ D0D949F51D35353900740E02 /* MappedFile.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, D0AB0B8E1D65D49C002C78E7 /* ChatListHolesView.swift in Sources */, + D0AAD1B21E3266B100D5B9DE /* TimestampBasedMessageAttributesIndexTable.swift in Sources */, D07827C51E00B23F00071108 /* PeerNameIndexTable.swift in Sources */, D03120FA1DA540F0006A2A60 /* CachedPeerData.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, @@ -913,6 +925,7 @@ D07827C11E0079CB00071108 /* StringIndexTokens.swift in Sources */, D0F3CC721DDE1CDC008148FA /* ItemCacheTable.swift in Sources */, D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, + D019B1CF1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */, @@ -926,6 +939,7 @@ D03BCCF81C73561C0097A291 /* Table.swift in Sources */, D021E0DC1DB5237C00C6B04F /* ItemCollectionsView.swift in Sources */, D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, + D0AAD1B31E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift in Sources */, D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */, diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift index 138325fe3a..413aabf967 100644 --- a/Postbox/ChatListIndexTable.swift +++ b/Postbox/ChatListIndexTable.swift @@ -12,8 +12,8 @@ final class ChatListIndexTable: Table { private let sharedKey = ValueBoxKey(length: 8) - private var cachedIndices: [PeerId: MessageIndex?] = [:] - private var updatedPreviousCachedIndices: [PeerId: MessageIndex?] = [:] + private var cachedIndices: [PeerId: ChatListIndex?] = [:] + private var updatedPreviousCachedIndices: [PeerId: ChatListIndex?] = [:] init(valueBox: ValueBox, table: ValueBoxTable, peerNameIndexTable: PeerNameIndexTable, metadataTable: MessageHistoryMetadataTable, readStateTable: MessageHistoryReadStateTable, notificationSettingsTable: PeerNotificationSettingsTable) { self.peerNameIndexTable = peerNameIndexTable @@ -29,9 +29,9 @@ final class ChatListIndexTable: Table { return self.sharedKey } - func set(_ index: MessageIndex) { - self.updatedPreviousCachedIndices[index.id.peerId] = self.get(index.id.peerId) - self.cachedIndices[index.id.peerId] = index + func set(_ index: ChatListIndex) { + self.updatedPreviousCachedIndices[index.messageIndex.id.peerId] = self.get(index.messageIndex.id.peerId) + self.cachedIndices[index.messageIndex.id.peerId] = index } func remove(_ peerId: PeerId) { @@ -39,18 +39,20 @@ final class ChatListIndexTable: Table { self.cachedIndices[peerId] = nil } - func get(_ peerId: PeerId) -> MessageIndex? { + func get(_ peerId: PeerId) -> ChatListIndex? { if let cached = self.cachedIndices[peerId] { return cached } else { if let value = self.valueBox.get(self.table, key: self.key(peerId)) { + var pinningIndexValue: UInt16 = 0 var idNamespace: Int32 = 0 var idId: Int32 = 0 var idTimestamp: Int32 = 0 + value.read(&pinningIndexValue, offset: 0, length: 2) value.read(&idNamespace, offset: 0, length: 4) value.read(&idId, offset: 0, length: 4) value.read(&idTimestamp, offset: 0, length: 4) - let index = MessageIndex(id: MessageId(peerId: peerId, namespace: idNamespace, id: idId), timestamp: idTimestamp) + let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: idNamespace, id: idId), timestamp: idTimestamp)) self.cachedIndices[peerId] = index return index } else { @@ -64,7 +66,7 @@ final class ChatListIndexTable: Table { assert(self.updatedPreviousCachedIndices.isEmpty) } - func commitWithTransactionUnreadCountDeltas(_ deltas: [PeerId: Int32], transactionParticipationInTotalUnreadCountUpdates: (added: Set, removed: Set), updatedTotalUnreadCount: inout Int32?) { + func commitWithTransactionUnreadCountDeltas(_ deltas: [PeerId: Int32], transactionParticipationInTotalUnreadCountUpdates: (added: Set, removed: Set), getPeer: (PeerId) -> Peer?, updatedTotalUnreadCount: inout Int32?) { if !self.updatedPreviousCachedIndices.isEmpty || !deltas.isEmpty || !transactionParticipationInTotalUnreadCountUpdates.added.isEmpty || !transactionParticipationInTotalUnreadCountUpdates.removed.isEmpty { var addedChatListPeerIds = Set() var removedChatListPeerIds = Set() @@ -77,13 +79,15 @@ final class ChatListIndexTable: Table { } let writeBuffer = WriteBuffer() - var idNamespace: Int32 = index.id.namespace - var idId: Int32 = index.id.id - var idTimestamp: Int32 = index.timestamp + var pinningIndexValue: UInt16 = keyValueForChatListPinningIndex(index.pinningIndex) + var idNamespace: Int32 = index.messageIndex.id.namespace + var idId: Int32 = index.messageIndex.id.id + var idTimestamp: Int32 = index.messageIndex.timestamp + writeBuffer.write(&pinningIndexValue, offset: 0, length: 2) writeBuffer.write(&idNamespace, offset: 0, length: 4) writeBuffer.write(&idId, offset: 0, length: 4) writeBuffer.write(&idTimestamp, offset: 0, length: 4) - self.valueBox.set(self.table, key: self.key(index.id.peerId), value: writeBuffer.readBufferNoCopy()) + self.valueBox.set(self.table, key: self.key(index.messageIndex.id.peerId), value: writeBuffer.readBufferNoCopy()) } else { if previousIndex != nil { removedChatListPeerIds.insert(peerId) @@ -100,7 +104,15 @@ final class ChatListIndexTable: Table { var totalUnreadCount = self.metadataTable.getChatListTotalUnreadCount() for (peerId, delta) in deltas { if !addedUnreadCountPeerIds.contains(peerId) && !removedUnreadCountPeerIds.contains(peerId) { - if let _ = self.get(peerId), let notificationSettings = self.notificationSettingsTable.get(peerId), !notificationSettings.isRemovedFromTotalUnreadCount { + var notificationSettings: PeerNotificationSettings? + if let peer = getPeer(peerId) { + if let notificationSettingsPeerId = peer.notificationSettingsPeerId { + notificationSettings = self.notificationSettingsTable.get(notificationSettingsPeerId) + } else { + notificationSettings = self.notificationSettingsTable.get(peerId) + } + } + if let _ = self.get(peerId), let notificationSettings = notificationSettings, !notificationSettings.isRemovedFromTotalUnreadCount { totalUnreadCount += delta } } @@ -119,7 +131,15 @@ final class ChatListIndexTable: Table { totalUnreadCount += combinedState.count } } else if addedToList { - if let notificationSettings = self.notificationSettingsTable.get(peerId), !notificationSettings.isRemovedFromTotalUnreadCount { + var notificationSettings: PeerNotificationSettings? + if let peer = getPeer(peerId) { + if let notificationSettingsPeerId = peer.notificationSettingsPeerId { + notificationSettings = self.notificationSettingsTable.get(notificationSettingsPeerId) + } else { + notificationSettings = self.notificationSettingsTable.get(peerId) + } + } + if let notificationSettings = notificationSettings, !notificationSettings.isRemovedFromTotalUnreadCount { if let combinedState = self.readStateTable.getCombinedState(peerId) { totalUnreadCount += combinedState.count } @@ -154,7 +174,15 @@ final class ChatListIndexTable: Table { if removedFromList && removedFromParticipationInUnreadCount { totalUnreadCount -= currentPeerUnreadCount } else if removedFromList { - if let notificationSettings = self.notificationSettingsTable.get(peerId), !notificationSettings.isRemovedFromTotalUnreadCount { + var notificationSettings: PeerNotificationSettings? + if let peer = getPeer(peerId) { + if let notificationSettingsPeerId = peer.notificationSettingsPeerId { + notificationSettings = self.notificationSettingsTable.get(notificationSettingsPeerId) + } else { + notificationSettings = self.notificationSettingsTable.get(peerId) + } + } + if let notificationSettings = notificationSettings, !notificationSettings.isRemovedFromTotalUnreadCount { totalUnreadCount -= currentPeerUnreadCount } } else if removedFromParticipationInUnreadCount { diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index e2cb8686f7..28b78a13a6 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -1,24 +1,24 @@ import Foundation enum ChatListOperation { - case InsertMessage(MessageIndex, IntermediateMessage, CombinedPeerReadState?, PeerChatListEmbeddedInterfaceState?) + case InsertMessage(ChatListIndex, IntermediateMessage, CombinedPeerReadState?, PeerChatListEmbeddedInterfaceState?) case InsertHole(ChatListHole) - case InsertNothing(MessageIndex) - case RemoveMessage([MessageIndex]) - case RemoveHoles([MessageIndex]) + case InsertNothing(ChatListIndex) + case RemoveMessage([ChatListIndex]) + case RemoveHoles([ChatListIndex]) } enum ChatListIntermediateEntry { - case Message(MessageIndex, IntermediateMessage, PeerChatListEmbeddedInterfaceState?) + case Message(ChatListIndex, IntermediateMessage, PeerChatListEmbeddedInterfaceState?) case Hole(ChatListHole) - case Nothing(MessageIndex) + case Nothing(ChatListIndex) - var index: MessageIndex { + var index: ChatListIndex { switch self { case let .Message(index, _, _): return index case let .Hole(hole): - return hole.index + return ChatListIndex(pinningIndex: nil, messageIndex: hole.index) case let .Nothing(index): return index } @@ -30,6 +30,22 @@ private enum ChatListEntryType: Int8 { case Hole = 2 } +func chatListPinningIndexFromKeyValue(_ value: UInt16) -> UInt16? { + if value == 0 { + return nil + } else { + return UInt16.max - 1 - value + } +} + +func keyValueForChatListPinningIndex(_ index: UInt16?) -> UInt16 { + if let index = index { + return UInt16.max - 1 - index + } else { + return 0 + } +} + final class ChatListTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .binary) @@ -48,25 +64,28 @@ final class ChatListTable: Table { super.init(valueBox: valueBox, table: table) } - private func key(_ index: MessageIndex, type: ChatListEntryType) -> ValueBoxKey { - let key = ValueBoxKey(length: 4 + 4 + 4 + 8 + 1) - key.setInt32(0, value: index.timestamp) - key.setInt32(4, value: index.id.namespace) - key.setInt32(4 + 4, value: index.id.id) - key.setInt64(4 + 4 + 4, value: index.id.peerId.toInt64()) - key.setInt8(4 + 4 + 4 + 8, value: type.rawValue) + private func key(_ index: ChatListIndex, type: ChatListEntryType) -> ValueBoxKey { + let key = ValueBoxKey(length: 2 + 4 + 4 + 4 + 8 + 1) + key.setUInt16(0, value: keyValueForChatListPinningIndex(index.pinningIndex)) + key.setInt32(2, value: index.messageIndex.timestamp) + key.setInt32(2 + 4, value: index.messageIndex.id.namespace) + key.setInt32(2 + 4 + 4, value: index.messageIndex.id.id) + key.setInt64(2 + 4 + 4 + 4, value: index.messageIndex.id.peerId.toInt64()) + key.setInt8(2 + 4 + 4 + 4 + 8, value: type.rawValue) return key } private func lowerBound() -> ValueBoxKey { - let key = ValueBoxKey(length: 4) - key.setInt32(0, value: 0) + let key = ValueBoxKey(length: 2 + 4) + key.setUInt16(0, value: 0) + key.setInt32(2, value: 0) return key } private func upperBound() -> ValueBoxKey { - let key = ValueBoxKey(length: 4) - key.setInt32(0, value: Int32.max) + let key = ValueBoxKey(length: 2 + 4) + key.setUInt16(0, value: UInt16.max) + key.setInt32(2, value: Int32.max) return key } @@ -89,9 +108,9 @@ final class ChatListTable: Table { changedPeerIds.insert(peerId) } for peerId in changedPeerIds { - let currentIndex: MessageIndex? = self.indexTable.get(peerId) + let currentIndex: ChatListIndex? = self.indexTable.get(peerId) - let updatedIndex: MessageIndex? + let updatedIndex: ChatListIndex? let topMessage = messageHistoryTable.topMessage(peerId) let embeddedChatState = peerChatInterfaceStateTable.get(peerId)?.chatListEmbeddedState @@ -100,16 +119,16 @@ final class ChatListTable: Table { if let embeddedChatState = embeddedChatState { updatedTimestamp = max(updatedTimestamp, embeddedChatState.timestamp) } - var updatedIndex = MessageIndex(id: topMessage.id, timestamp: updatedTimestamp) + let updatedIndex: ChatListIndex = ChatListIndex(pinningIndex: currentIndex?.pinningIndex, messageIndex: MessageIndex(id: topMessage.id, timestamp: updatedTimestamp)) if let currentIndex = currentIndex, currentIndex != updatedIndex { - self.justRemoveMessage(currentIndex) + self.justRemoveIndex(currentIndex) } if let currentIndex = currentIndex { operations.append(.RemoveMessage([currentIndex])) } self.indexTable.set(updatedIndex) - self.justInsertMessage(updatedIndex) + self.justInsertIndex(updatedIndex) operations.append(.InsertMessage(updatedIndex, topMessage, messageHistoryTable.readStateTable.getCombinedState(peerId), embeddedChatState)) } else { if let currentIndex = currentIndex { @@ -123,7 +142,7 @@ final class ChatListTable: Table { func addHole(_ hole: ChatListHole, operations: inout [ChatListOperation]) { self.ensureInitialized() - if self.valueBox.get(self.table, key: self.key(hole.index, type: .Hole)) == nil { + if self.valueBox.get(self.table, key: self.key(ChatListIndex(pinningIndex: nil, messageIndex: hole.index), type: .Hole)) == nil { self.justInsertHole(hole) operations.append(.InsertHole(hole)) } @@ -132,38 +151,38 @@ final class ChatListTable: Table { func replaceHole(_ index: MessageIndex, hole: ChatListHole?, operations: inout [ChatListOperation]) { self.ensureInitialized() - if self.valueBox.get(self.table, key: self.key(index, type: .Hole)) != nil { + if self.valueBox.get(self.table, key: self.key(ChatListIndex(pinningIndex: nil, messageIndex: index), type: .Hole)) != nil { if let hole = hole { if hole.index != index { self.justRemoveHole(index) self.justInsertHole(hole) - operations.append(.RemoveHoles([index])) + operations.append(.RemoveHoles([ChatListIndex(pinningIndex: nil, messageIndex: index)])) operations.append(.InsertHole(hole)) } } else{ self.justRemoveHole(index) - operations.append(.RemoveHoles([index])) + operations.append(.RemoveHoles([ChatListIndex(pinningIndex: nil, messageIndex: index)])) } } } - private func justInsertMessage(_ index: MessageIndex) { + private func justInsertIndex(_ index: ChatListIndex) { self.valueBox.set(self.table, key: self.key(index, type: .Message), value: self.emptyMemoryBuffer) } - private func justRemoveMessage(_ index: MessageIndex) { + private func justRemoveIndex(_ index: ChatListIndex) { self.valueBox.remove(self.table, key: self.key(index, type: .Message)) } private func justInsertHole(_ hole: ChatListHole) { - self.valueBox.set(self.table, key: self.key(hole.index, type: .Hole), value: self.emptyMemoryBuffer) + self.valueBox.set(self.table, key: self.key(ChatListIndex(pinningIndex: nil, messageIndex: hole.index), type: .Hole), value: self.emptyMemoryBuffer) } private func justRemoveHole(_ index: MessageIndex) { - self.valueBox.remove(self.table, key: self.key(index, type: .Hole)) + self.valueBox.remove(self.table, key: self.key(ChatListIndex(pinningIndex: nil, messageIndex: index), type: .Hole)) } - func entriesAround(_ index: MessageIndex, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> (entries: [ChatListIntermediateEntry], lower: ChatListIntermediateEntry?, upper: ChatListIntermediateEntry?) { + func entriesAround(_ index: ChatListIndex, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> (entries: [ChatListIntermediateEntry], lower: ChatListIntermediateEntry?, upper: ChatListIntermediateEntry?) { self.ensureInitialized() var lowerEntries: [ChatListIntermediateEntry] = [] @@ -172,16 +191,17 @@ final class ChatListTable: Table { var upper: ChatListIntermediateEntry? self.valueBox.range(self.table, start: self.key(index, type: .Message), end: self.lowerBound(), keys: { key in - let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) - let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + let pinningIndexValue: UInt16 = key.getUInt16(0) + let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) + let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index) { - lowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) + if let message = messageHistoryTable.getMessage(index.messageIndex) { + lowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { lowerEntries.append(.Nothing(index)) } } else { - lowerEntries.append(.Hole(ChatListHole(index: index))) + lowerEntries.append(.Hole(ChatListHole(index: index.messageIndex))) } return true }, limit: count / 2 + 1) @@ -191,16 +211,17 @@ final class ChatListTable: Table { } self.valueBox.range(self.table, start: self.key(index, type: .Message).predecessor, end: self.upperBound(), keys: { key in - let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) - let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + let pinningIndexValue: UInt16 = key.getUInt16(0) + let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) + let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index) { - upperEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) + if let message = messageHistoryTable.getMessage(index.messageIndex) { + upperEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { upperEntries.append(.Nothing(index)) } } else { - upperEntries.append(.Hole(ChatListHole(index: index))) + upperEntries.append(.Hole(ChatListHole(index: index.messageIndex))) } return true }, limit: count - lowerEntries.count + 1) @@ -219,16 +240,17 @@ final class ChatListTable: Table { startEntryType = .Hole } self.valueBox.range(self.table, start: self.key(lowerEntries.last!.index, type: startEntryType), end: self.lowerBound(), keys: { key in - let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) - let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + let pinningIndexValue: UInt16 = key.getUInt16(0) + let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) + let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index) { - additionalLowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) + if let message = messageHistoryTable.getMessage(index.messageIndex) { + additionalLowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { additionalLowerEntries.append(.Nothing(index)) } } else { - additionalLowerEntries.append(.Hole(ChatListHole(index: index))) + additionalLowerEntries.append(.Hole(ChatListHole(index: index.messageIndex))) } return true }, limit: count - lowerEntries.count - upperEntries.count + 1) @@ -258,7 +280,7 @@ final class ChatListTable: Table { return peerIds } - func earlierEntries(_ index: MessageIndex?, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> [ChatListIntermediateEntry] { + func earlierEntries(_ index: ChatListIndex?, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> [ChatListIntermediateEntry] { self.ensureInitialized() var entries: [ChatListIntermediateEntry] = [] @@ -270,23 +292,24 @@ final class ChatListTable: Table { } self.valueBox.range(self.table, start: key, end: self.lowerBound(), keys: { key in - let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) - let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + let pinningIndexValue: UInt16 = key.getUInt16(0) + let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) + let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index) { - entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) + if let message = messageHistoryTable.getMessage(index.messageIndex) { + entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { entries.append(.Nothing(index)) } } else { - entries.append(.Hole(ChatListHole(index: index))) + entries.append(.Hole(ChatListHole(index: index.messageIndex))) } return true }, limit: count) return entries } - func laterEntries(_ index: MessageIndex?, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> [ChatListIntermediateEntry] { + func laterEntries(_ index: ChatListIndex?, messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, count: Int) -> [ChatListIntermediateEntry] { self.ensureInitialized() var entries: [ChatListIntermediateEntry] = [] @@ -298,16 +321,17 @@ final class ChatListTable: Table { } self.valueBox.range(self.table, start: key, end: self.upperBound(), keys: { key in - let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)), timestamp: key.getInt32(0)) - let type: Int8 = key.getInt8(4 + 4 + 4 + 8) + let pinningIndexValue: UInt16 = key.getUInt16(0) + let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) + let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index) { - entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.id.peerId)?.chatListEmbeddedState)) + if let message = messageHistoryTable.getMessage(index.messageIndex) { + entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { entries.append(.Nothing(index)) } } else { - entries.append(.Hole(ChatListHole(index: index))) + entries.append(.Hole(ChatListHole(index: index.messageIndex))) } return true }, limit: count) @@ -315,6 +339,6 @@ final class ChatListTable: Table { } func debugList(_ messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable) -> [ChatListIntermediateEntry] { - return self.laterEntries(MessageIndex.absoluteLowerBound(), messageHistoryTable: messageHistoryTable, peerChatInterfaceStateTable: peerChatInterfaceStateTable, count: 1000) + return self.laterEntries(ChatListIndex.absoluteLowerBound, messageHistoryTable: messageHistoryTable, peerChatInterfaceStateTable: peerChatInterfaceStateTable, count: 1000) } } diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index b47cf6b3f4..b592d81ab8 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -1,16 +1,16 @@ import Foundation public enum ChatListEntry: Comparable { - case MessageEntry(MessageIndex, Message, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) + case MessageEntry(ChatListIndex, Message, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) case HoleEntry(ChatListHole) - case Nothing(MessageIndex) + case Nothing(ChatListIndex) - public var index: MessageIndex { + public var index: ChatListIndex { switch self { case let .MessageEntry(index, _, _, _, _): return index case let .HoleEntry(hole): - return hole.index + return ChatListIndex(pinningIndex: nil, messageIndex: hole.index) case let .Nothing(index): return index } @@ -69,19 +69,19 @@ public func <(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { } enum MutableChatListEntry: Equatable { - case IntermediateMessageEntry(MessageIndex, IntermediateMessage, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) - case MessageEntry(MessageIndex, Message, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) + case IntermediateMessageEntry(ChatListIndex, IntermediateMessage, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) + case MessageEntry(ChatListIndex, Message, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) case HoleEntry(ChatListHole) - case Nothing(MessageIndex) + case Nothing(ChatListIndex) - var index: MessageIndex { + var index: ChatListIndex { switch self { case let .IntermediateMessageEntry(index, _, _, _, _): return index case let .MessageEntry(index, _, _, _, _): return index case let .HoleEntry(hole): - return hole.index + return ChatListIndex(pinningIndex: nil, messageIndex: hole.index) case let .Nothing(index): return index } @@ -148,8 +148,8 @@ final class MutableChatListView { self.count = count } - func refreshDueToExternalTransaction(fetchAroundChatEntries: (_ index: MessageIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?)) -> Bool { - var index = MessageIndex.absoluteUpperBound() + func refreshDueToExternalTransaction(fetchAroundChatEntries: (_ index: ChatListIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?)) -> Bool { + var index = ChatListIndex.absoluteUpperBound if !self.entries.isEmpty { index = self.entries[self.entries.count / 2].index } @@ -274,7 +274,7 @@ final class MutableChatListView { } } - func remove(_ indices: Set, holes: Bool, context: MutableChatListViewReplayContext) -> Bool { + func remove(_ indices: Set, holes: Bool, context: MutableChatListViewReplayContext) -> Bool { var hasChanges = false if let earlier = self.earlier , indices.contains(earlier.index) { var match = false @@ -328,11 +328,11 @@ final class MutableChatListView { return hasChanges } - func complete(context: MutableChatListViewReplayContext, fetchEarlier: (MessageIndex?, Int) -> [MutableChatListEntry], fetchLater: (MessageIndex?, Int) -> [MutableChatListEntry]) { + func complete(context: MutableChatListViewReplayContext, fetchEarlier: (ChatListIndex?, Int) -> [MutableChatListEntry], fetchLater: (ChatListIndex?, Int) -> [MutableChatListEntry]) { if context.removedEntries { var addedEntries: [MutableChatListEntry] = [] - var latestAnchor: MessageIndex? + var latestAnchor: ChatListIndex? if let last = self.entries.last { latestAnchor = last.index } @@ -344,17 +344,17 @@ final class MutableChatListView { } if let later = self.later { - addedEntries += fetchLater(later.index.predecessor(), self.count) + addedEntries += fetchLater(later.index.predecessor, self.count) } if let earlier = self.earlier { - addedEntries += fetchEarlier(earlier.index.successor(), self.count) + addedEntries += fetchEarlier(earlier.index.successor, self.count) } addedEntries += self.entries addedEntries.sort(by: { $0.index < $1.index }) var i = addedEntries.count - 1 while i >= 1 { - if addedEntries[i].index.id == addedEntries[i - 1].index.id { + if addedEntries[i].index.messageIndex.id == addedEntries[i - 1].index.messageIndex.id { addedEntries.remove(at: i) } i -= 1 @@ -390,7 +390,7 @@ final class MutableChatListView { } } else { if context.invalidEarlier { - var earlyId: MessageIndex? + var earlyId: ChatListIndex? let i = 0 if i < self.entries.count { earlyId = self.entries[i].index @@ -401,7 +401,7 @@ final class MutableChatListView { } if context.invalidLater { - var laterId: MessageIndex? + var laterId: ChatListIndex? let i = self.entries.count - 1 if i >= 0 { laterId = self.entries[i].index @@ -465,8 +465,8 @@ final class MutableChatListView { public final class ChatListView { public let entries: [ChatListEntry] - public let earlierIndex: MessageIndex? - public let laterIndex: MessageIndex? + public let earlierIndex: ChatListIndex? + public let laterIndex: ChatListIndex? init(_ mutableView: MutableChatListView) { var entries: [ChatListEntry] = [] diff --git a/Postbox/Message.swift b/Postbox/Message.swift index f358da19ce..40d41a466a 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -121,7 +121,7 @@ public struct MessageIndex: Equatable, Comparable, Hashable { } public func successor() -> MessageIndex { - return MessageIndex(id: MessageId(peerId: self.id.peerId, namespace: self.id.namespace, id: self.id.id + 1), timestamp: self.timestamp) + return MessageIndex(id: MessageId(peerId: self.id.peerId, namespace: self.id.namespace, id: self.id.id == Int32.max ? self.id.id : (self.id.id + 1)), timestamp: self.timestamp) } public var hashValue: Int { @@ -143,6 +143,10 @@ public struct MessageIndex: Equatable, Comparable, Hashable { public static func upperBound(peerId: PeerId) -> MessageIndex { return MessageIndex(id: MessageId(peerId: peerId, namespace: Int32.max, id: Int32.max), timestamp: Int32.max) } + + public static func upperBound(peerId: PeerId, timestamp: Int32, namespace: MessageId.Namespace) -> MessageIndex { + return MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: Int32.max), timestamp: timestamp) + } } public func ==(lhs: MessageIndex, rhs: MessageIndex) -> Bool { @@ -161,6 +165,51 @@ public func <(lhs: MessageIndex, rhs: MessageIndex) -> Bool { return lhs.id.id < rhs.id.id } +public struct ChatListIndex: Comparable, Hashable { + public let pinningIndex: UInt16? + public let messageIndex: MessageIndex + + public init(pinningIndex: UInt16?, messageIndex: MessageIndex) { + self.pinningIndex = pinningIndex + self.messageIndex = messageIndex + } + + public static func ==(lhs: ChatListIndex, rhs: ChatListIndex) -> Bool { + return lhs.pinningIndex == rhs.pinningIndex && lhs.messageIndex == rhs.messageIndex + } + + public static func <(lhs: ChatListIndex, rhs: ChatListIndex) -> Bool { + if let lhsPinningIndex = lhs.pinningIndex, let rhsPinningIndex = rhs.pinningIndex { + if lhsPinningIndex < rhsPinningIndex { + return true + } else if lhsPinningIndex > rhsPinningIndex { + return false + } + } + return lhs.messageIndex < rhs.messageIndex + } + + public var hashValue: Int { + return self.messageIndex.hashValue + } + + public static var absoluteUpperBound: ChatListIndex { + return ChatListIndex(pinningIndex: 0, messageIndex: MessageIndex.absoluteUpperBound()) + } + + public static var absoluteLowerBound: ChatListIndex { + return ChatListIndex(pinningIndex: nil, messageIndex: MessageIndex.absoluteLowerBound()) + } + + public var predecessor: ChatListIndex { + return ChatListIndex(pinningIndex: self.pinningIndex, messageIndex: self.messageIndex.predecessor()) + } + + public var successor: ChatListIndex { + return ChatListIndex(pinningIndex: self.pinningIndex, messageIndex: self.messageIndex.successor()) + } +} + public struct MessageTags: OptionSet { public var rawValue: UInt32 diff --git a/Postbox/MessageGloballyUniqueIdTable.swift b/Postbox/MessageGloballyUniqueIdTable.swift index 85bedd1b66..16635af47f 100644 --- a/Postbox/MessageGloballyUniqueIdTable.swift +++ b/Postbox/MessageGloballyUniqueIdTable.swift @@ -2,18 +2,19 @@ import Foundation final class MessageGloballyUniqueIdTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { - return ValueBoxTable(id: id, keyType: .int64) + return ValueBoxTable(id: id, keyType: .binary) } - private let sharedKey = ValueBoxKey(length: 8) + private let sharedKey = ValueBoxKey(length: 8 + 8) private let sharedBuffer = WriteBuffer() - private func key(_ id: Int64) -> ValueBoxKey { - self.sharedKey.setInt64(0, value: id) + private func key(peerId: PeerId, id: Int64) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: peerId.toInt64()) + self.sharedKey.setInt64(8, value: id) return self.sharedKey } - func set(_ globallyUniqueId: Int64, id: MessageId) { + func set(peerId: PeerId, globallyUniqueId: Int64, id: MessageId) { self.sharedBuffer.reset() var idPeerId: Int64 = id.peerId.toInt64() var idNamespace: Int32 = id.namespace @@ -21,11 +22,11 @@ final class MessageGloballyUniqueIdTable: Table { self.sharedBuffer.write(&idPeerId, offset: 0, length: 8) self.sharedBuffer.write(&idNamespace, offset: 0, length: 4) self.sharedBuffer.write(&idId, offset: 0, length: 4) - self.valueBox.set(self.table, key: self.key(globallyUniqueId), value: self.sharedBuffer) + self.valueBox.set(self.table, key: self.key(peerId: peerId, id: globallyUniqueId), value: self.sharedBuffer) } - func get(_ globallyUniqueId: Int64) -> MessageId? { - if let value = self.valueBox.get(self.table, key: self.key(globallyUniqueId)) { + func get(peerId: PeerId, globallyUniqueId: Int64) -> MessageId? { + if let value = self.valueBox.get(self.table, key: self.key(peerId: peerId, id: globallyUniqueId)) { var idPeerId: Int64 = 0 var idNamespace: Int32 = 0 var idId: Int32 = 0 @@ -37,7 +38,7 @@ final class MessageGloballyUniqueIdTable: Table { return nil } - func remove(_ globallyUniqueId: Int64) { - self.valueBox.remove(self.table, key: self.key(globallyUniqueId)) + func remove(peerId: PeerId, globallyUniqueId: Int64) { + self.valueBox.remove(self.table, key: self.key(peerId: peerId, id: globallyUniqueId)) } } diff --git a/Postbox/MessageHistoryHolesView.swift b/Postbox/MessageHistoryHolesView.swift index 1db0fbcffe..28afe47750 100644 --- a/Postbox/MessageHistoryHolesView.swift +++ b/Postbox/MessageHistoryHolesView.swift @@ -2,7 +2,7 @@ import Foundation public struct MessageHistoryHolesViewEntry: Hashable { public let hole: MessageHistoryHole - public let direction: HoleFillDirection + public let direction: MessageHistoryViewRelativeHoleDirection public let tags: MessageTags? public var hashValue: Int { diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 02acd75da6..71e9343341 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -17,7 +17,7 @@ enum HistoryIndexEntry { public enum HoleFillDirection: Equatable { case UpperToLower case LowerToUpper - case AroundIndex(MessageIndex) + case AroundIndex(MessageIndex, lowerComplete: Bool, upperComplete: Bool) } public func ==(lhs: HoleFillDirection, rhs: HoleFillDirection) -> Bool { @@ -36,12 +36,11 @@ public func ==(lhs: HoleFillDirection, rhs: HoleFillDirection) -> Bool { default: return false } - case let .AroundIndex(lhsIndex): - switch rhs { - case let .AroundIndex(rhsIndex) where lhsIndex == rhsIndex: + case let .AroundIndex(index, lowerComplete, upperComplete): + if case .AroundIndex(index, lowerComplete, upperComplete) = rhs { return true - default: - return false + } else { + return false } } } @@ -383,6 +382,160 @@ final class MessageHistoryIndexTable: Table { } } + func fillMultipleHoles(mainHoleId: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [InternalStoreMessage], operations: inout [MessageHistoryIndexOperation]) { + let peerId = mainHoleId.peerId + self.ensureInitialized(peerId, operations: &operations) + + let sortedByIdMessages = messages.sorted(by: {$0.id < $1.id}) + + var collectedHoles: [MessageId] = [] + var messagesByHole: [MessageId: [InternalStoreMessage]] = [:] + var holesByHole: [MessageId: MessageHistoryHole] = [:] + + var filledUpperBound: MessageId.Id? + var filledLowerBound: MessageId.Id? + + var adjustedMainHoleId: MessageId? + do { + var upperItem: HistoryIndexEntry? + self.valueBox.range(self.table, start: self.key(mainHoleId).predecessor, end: self.upperBound(peerId, namespace: mainHoleId.namespace), values: { key, value in + upperItem = readHistoryIndexEntry(peerId, namespace: mainHoleId.namespace, key: key, value: value) + return true + }, limit: 1) + if let upperItem = upperItem, case let .Hole(upperHole) = upperItem { + collectedHoles.append(upperHole.maxIndex.id) + messagesByHole[upperHole.maxIndex.id] = [] + adjustedMainHoleId = upperHole.maxIndex.id + holesByHole[upperHole.maxIndex.id] = upperHole + + if !sortedByIdMessages.isEmpty { + var currentLowerBound = sortedByIdMessages[0].id.id + var currentUpperBound = sortedByIdMessages[sortedByIdMessages.count - 1].id.id + + switch fillType.direction { + case .LowerToUpper: + currentLowerBound = min(currentLowerBound, upperHole.min) + case .UpperToLower: + currentUpperBound = max(currentUpperBound, upperHole.maxIndex.id.id) + case .AroundIndex: + break + } + + filledLowerBound = currentLowerBound + filledUpperBound = currentUpperBound + } + } + } + + if filledLowerBound == nil { + if !sortedByIdMessages.isEmpty { + let currentLowerBound = sortedByIdMessages[0].id.id + let currentUpperBound = sortedByIdMessages[sortedByIdMessages.count - 1].id.id + filledLowerBound = currentLowerBound + filledUpperBound = currentUpperBound + } + } + + var remainingMessages: [InternalStoreMessage] = [] + + if !sortedByIdMessages.isEmpty { + let lowestMessageId = filledLowerBound! + let highestMessageId = filledUpperBound! + + self.valueBox.range(self.table, start: self.key(MessageId(peerId: mainHoleId.peerId, namespace: mainHoleId.namespace, id: lowestMessageId)), end: self.key(MessageId(peerId: mainHoleId.peerId, namespace: mainHoleId.namespace, id: highestMessageId)), values: { key, value in + let item = readHistoryIndexEntry(peerId, namespace: mainHoleId.namespace, key: key, value: value) + if case let .Hole(itemHole) = item { + if itemHole.min <= highestMessageId && itemHole.maxIndex.id.id >= lowestMessageId { + if messagesByHole[itemHole.maxIndex.id] == nil { + collectedHoles.append(itemHole.maxIndex.id) + holesByHole[itemHole.maxIndex.id] = itemHole + messagesByHole[itemHole.maxIndex.id] = [] + } + } + } + return true + }, limit: 0) + } + + for message in sortedByIdMessages { + var upperItem: HistoryIndexEntry? + self.valueBox.range(self.table, start: self.key(message.id).predecessor, end: self.upperBound(peerId, namespace: message.id.namespace), values: { key, value in + upperItem = readHistoryIndexEntry(peerId, namespace: message.id.namespace, key: key, value: value) + return true + }, limit: 1) + if let upperItem = upperItem, case let .Hole(upperHole) = upperItem, message.id.id >= upperHole.min && message.id.id <= upperHole.maxIndex.id.id { + if messagesByHole[upperHole.maxIndex.id] == nil { + messagesByHole[upperHole.maxIndex.id] = [message] + collectedHoles.append(upperHole.maxIndex.id) + holesByHole[upperHole.maxIndex.id] = upperHole + } else { + messagesByHole[upperHole.maxIndex.id]!.append(message) + } + } else { + remainingMessages.append(message) + } + } + + for holeId in collectedHoles { + let holeMessages = messagesByHole[holeId]! + let currentFillType: HoleFill + + var adjustedLowerComplete = false + var adjustedUpperComplete = false + + if !sortedByIdMessages.isEmpty { + let currentHole = holesByHole[holeId]! + + if filledLowerBound! <= currentHole.min { + adjustedLowerComplete = true + } + if filledUpperBound! >= currentHole.maxIndex.id.id { + adjustedUpperComplete = true + } + } else { + adjustedLowerComplete = true + adjustedUpperComplete = true + } + + if let adjustedMainHoleId = adjustedMainHoleId { + if holeId == adjustedMainHoleId { + switch fillType.direction { + case let .AroundIndex(index, lowerComplete, upperComplete): + if lowerComplete { + adjustedLowerComplete = true + } + if upperComplete { + adjustedUpperComplete = true + } + + currentFillType = HoleFill(complete: fillType.complete, direction: .AroundIndex(index, lowerComplete: adjustedLowerComplete, upperComplete: adjustedUpperComplete)) + case .LowerToUpper: + currentFillType = HoleFill(complete: fillType.complete || adjustedUpperComplete, direction: .LowerToUpper) + case .UpperToLower: + currentFillType = HoleFill(complete: fillType.complete || adjustedLowerComplete, direction: .UpperToLower) + } + } else { + if holeId < adjustedMainHoleId { + currentFillType = HoleFill(complete: adjustedLowerComplete, direction: .UpperToLower) + } else { + currentFillType = HoleFill(complete: adjustedUpperComplete, direction: .LowerToUpper) + } + } + } else { + if holeId < mainHoleId { + currentFillType = HoleFill(complete: adjustedLowerComplete, direction: .UpperToLower) + } else { + currentFillType = HoleFill(complete: adjustedUpperComplete, direction: .LowerToUpper) + } + } + self.fillHole(holeId, fillType: currentFillType, tagMask: tagMask, messages: holeMessages, operations: &operations) + } + + for message in remainingMessages { + self.addMessages([message], location: .Random, operations: &operations) + } + } + func fillHole(_ id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [InternalStoreMessage], operations: inout [MessageHistoryIndexOperation]) { self.ensureInitialized(id.peerId, operations: &operations) @@ -402,6 +555,7 @@ final class MessageHistoryIndexTable: Table { if let tagMask = tagMask { if case .AroundIndex = fillType.direction { assertionFailure(".AroundIndex not supported") + return } var messagesInRange: [InternalStoreMessage] = [] @@ -525,20 +679,20 @@ final class MessageHistoryIndexTable: Table { } self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minMessageInRange.id.id - 1), timestamp: minMessageInRange.timestamp), min: upperHole.min, tags: tags), operations: &operations) } - } else if case .AroundIndex = fillType.direction { + } else if case let .AroundIndex(_, lowerComplete, upperComplete) = fillType.direction { if !removedHole { self.justRemove(upperHole.maxIndex, operations: &operations) removedHole = true } - if let minMessageInRange = minMessageInRange , minMessageInRange.id.id - 1 >= upperHole.min { + if let minMessageInRange = minMessageInRange, minMessageInRange.id.id - 1 >= upperHole.min && !lowerComplete { let stableId: UInt32 = upperHole.stableId let tags: UInt32 = upperHole.tags self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minMessageInRange.id.id - 1), timestamp: minMessageInRange.timestamp), min: upperHole.min, tags: tags), operations: &operations) } - if let maxMessageInRange = maxMessageInRange , maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { + if let maxMessageInRange = maxMessageInRange, maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id && !upperComplete { let stableId: UInt32 = self.metadataTable.getNextStableMessageIndexId() let tags: UInt32 = upperHole.tags self.justInsertHole(MessageHistoryHole(stableId: stableId, maxIndex: upperHole.maxIndex, min: maxMessageInRange.id.id + 1, tags: tags), operations: &operations) diff --git a/Postbox/MessageHistoryInvalidatedReadStateTable.swift b/Postbox/MessageHistoryInvalidatedReadStateTable.swift index e33f9d16f3..779fc4ef6e 100644 --- a/Postbox/MessageHistoryInvalidatedReadStateTable.swift +++ b/Postbox/MessageHistoryInvalidatedReadStateTable.swift @@ -1,15 +1,15 @@ import Foundation public enum PeerReadStateSynchronizationOperation: Equatable { - case Push(thenSync: Bool) + case Push(state: CombinedPeerReadState?, thenSync: Bool) case Validate } public func ==(lhs: PeerReadStateSynchronizationOperation, rhs: PeerReadStateSynchronizationOperation) -> Bool { switch lhs { - case let .Push(lhsThenSync): + case let .Push(lhsState, lhsThenSync): switch rhs { - case let .Push(rhsThenSync) where lhsThenSync == rhsThenSync: + case let .Push(rhsState, rhsThenSync) where lhsState == rhsState && lhsThenSync == rhsThenSync: return true default: return false @@ -55,11 +55,12 @@ final class MessageHistorySynchronizeReadStateTable: Table { operations[peerId] = operation } - func get() -> [PeerId: PeerReadStateSynchronizationOperation] { + func get(getCombinedPeerReadState: (PeerId) -> CombinedPeerReadState?) -> [PeerId: PeerReadStateSynchronizationOperation] { self.beforeCommit() var operations: [PeerId: PeerReadStateSynchronizationOperation] = [:] self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), values: { key, value in + let peerId = PeerId(key.getInt64(0)) var operationValue: Int8 = 0 value.read(&operationValue, offset: 0, length: 1) @@ -67,12 +68,12 @@ final class MessageHistorySynchronizeReadStateTable: Table { if operationValue == 0 { var syncValue: Int8 = 0 value.read(&syncValue, offset: 0, length: 1) - operation = .Push(thenSync: syncValue != 0) + operation = .Push(state: getCombinedPeerReadState(peerId), thenSync: syncValue != 0) } else { operation = .Validate } - operations[PeerId(key.getInt64(0))] = operation + operations[peerId] = operation return true }, limit: 0) return operations @@ -86,7 +87,7 @@ final class MessageHistorySynchronizeReadStateTable: Table { if let operation = operation { buffer.reset() switch operation { - case let .Push(thenSync): + case let .Push(_, thenSync): var operationValue: Int8 = 0 buffer.write(&operationValue, offset: 0, length: 1) var syncValue: Int8 = thenSync ? 1 : 0 diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index 4a308040f0..c339161f22 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -44,17 +44,50 @@ final class MessageHistoryReadStateTable: Table { var stateByNamespace: [MessageId.Namespace: PeerReadState] = [:] for _ in 0 ..< count { var namespaceId: Int32 = 0 - var maxIncomingReadId: Int32 = 0 - var maxOutgoingReadId: Int32 = 0 - var maxKnownId: Int32 = 0 - var count: Int32 = 0 value.read(&namespaceId, offset: 0, length: 4) - value.read(&maxIncomingReadId, offset: 0, length: 4) - value.read(&maxOutgoingReadId, offset: 0, length: 4) - value.read(&maxKnownId, offset: 0, length: 4) - value.read(&count, offset: 0, length: 4) - let state = PeerReadState(maxIncomingReadId: maxIncomingReadId, maxOutgoingReadId: maxOutgoingReadId, maxKnownId: maxKnownId, count: count) + let state: PeerReadState + var kind: Int8 = 0 + value.read(&kind, offset: 0, length: 1) + if kind == 0 { + var maxIncomingReadId: Int32 = 0 + var maxOutgoingReadId: Int32 = 0 + var maxKnownId: Int32 = 0 + var count: Int32 = 0 + + value.read(&maxIncomingReadId, offset: 0, length: 4) + value.read(&maxOutgoingReadId, offset: 0, length: 4) + value.read(&maxKnownId, offset: 0, length: 4) + value.read(&count, offset: 0, length: 4) + + state = .idBased(maxIncomingReadId: maxIncomingReadId, maxOutgoingReadId: maxOutgoingReadId, maxKnownId: maxKnownId, count: count) + } else { + var maxIncomingReadTimestamp: Int32 = 0 + var maxIncomingReadIdPeerId: Int64 = 0 + var maxIncomingReadIdNamespace: Int32 = 0 + var maxIncomingReadIdId: Int32 = 0 + + var maxOutgoingReadTimestamp: Int32 = 0 + var maxOutgoingReadIdPeerId: Int64 = 0 + var maxOutgoingReadIdNamespace: Int32 = 0 + var maxOutgoingReadIdId: Int32 = 0 + + var count: Int32 = 0 + + value.read(&maxIncomingReadTimestamp, offset: 0, length: 4) + value.read(&maxIncomingReadIdPeerId, offset: 0, length: 8) + value.read(&maxIncomingReadIdNamespace, offset: 0, length: 4) + value.read(&maxIncomingReadIdId, offset: 0, length: 4) + + value.read(&maxOutgoingReadTimestamp, offset: 0, length: 4) + value.read(&maxOutgoingReadIdPeerId, offset: 0, length: 8) + value.read(&maxOutgoingReadIdNamespace, offset: 0, length: 4) + value.read(&maxOutgoingReadIdId, offset: 0, length: 4) + + value.read(&count, offset: 0, length: 4) + + state = .indexBased(maxIncomingReadIndex: MessageIndex(id: MessageId(peerId: PeerId(maxIncomingReadIdPeerId), namespace: maxIncomingReadIdNamespace, id: maxIncomingReadIdId), timestamp: maxIncomingReadTimestamp), maxOutgoingReadIndex: MessageIndex(id: MessageId(peerId: PeerId(maxOutgoingReadIdPeerId), namespace: maxOutgoingReadIdNamespace, id: maxOutgoingReadIdId), timestamp: maxOutgoingReadTimestamp), count: count) + } stateByNamespace[namespaceId] = state } let states = InternalPeerReadStates(namespaces: stateByNamespace) @@ -107,38 +140,43 @@ final class MessageHistoryReadStateTable: Table { } } - func addIncomingMessages(_ peerId: PeerId, ids: Set) -> (CombinedPeerReadState?, Bool) { - var idsByNamespace: [MessageId.Namespace: [MessageId.Id]] = [:] - for id in ids { - if idsByNamespace[id.namespace] != nil { - idsByNamespace[id.namespace]!.append(id.id) + func addIncomingMessages(_ peerId: PeerId, indices: Set) -> (CombinedPeerReadState?, Bool) { + var indicesByNamespace: [MessageId.Namespace: [MessageIndex]] = [:] + for index in indices { + if indicesByNamespace[index.id.namespace] != nil { + indicesByNamespace[index.id.namespace]!.append(index) } else { - idsByNamespace[id.namespace] = [id.id] + indicesByNamespace[index.id.namespace] = [index] } } if let states = self.get(peerId) { if traceReadStates { - print("[ReadStateTable] addIncomingMessages peerId: \(peerId), ids: \(ids) (before: \(states.namespaces))") + print("[ReadStateTable] addIncomingMessages peerId: \(peerId), indices: \(indices) (before: \(states.namespaces))") } var updated = false let invalidated = false - for (namespace, ids) in idsByNamespace { + for (namespace, namespaceIndices) in indicesByNamespace { if let currentState = states.namespaces[namespace] { var addedUnreadCount: Int32 = 0 - var maxIncomingId: Int32 = 0 - for id in ids { - if id > currentState.maxKnownId && id > currentState.maxIncomingReadId { - addedUnreadCount += 1 - maxIncomingId = max(id, maxIncomingId) + for index in namespaceIndices { + switch currentState { + case let .idBased(maxIncomingReadId, _, maxKnownId, _): + if index.id.id > maxKnownId && index.id.id > maxIncomingReadId { + addedUnreadCount += 1 + } + case let .indexBased(maxIncomingReadIndex, _, _): + if index > maxIncomingReadIndex { + addedUnreadCount += 1 + } } } if addedUnreadCount != 0 { self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces) - states.namespaces[namespace] = PeerReadState(maxIncomingReadId: currentState.maxIncomingReadId, maxOutgoingReadId: currentState.maxOutgoingReadId, maxKnownId: currentState.maxKnownId, count: currentState.count + addedUnreadCount) + states.namespaces[namespace] = currentState.withAddedCount(addedUnreadCount) updated = true if traceReadStates { @@ -157,40 +195,40 @@ final class MessageHistoryReadStateTable: Table { } } - func deleteMessages(_ peerId: PeerId, ids: [MessageId], incomingStatsInIds: (PeerId, MessageId.Namespace, [MessageId.Id]) -> (Int, Bool)) -> (CombinedPeerReadState?, Bool) { - var idsByNamespace: [MessageId.Namespace: [MessageId.Id]] = [:] - for id in ids { - if idsByNamespace[id.namespace] != nil { - idsByNamespace[id.namespace]!.append(id.id) + func deleteMessages(_ peerId: PeerId, indices: [MessageIndex], incomingStatsInIndices: (PeerId, MessageId.Namespace, [MessageIndex]) -> (Int, Bool)) -> (CombinedPeerReadState?, Bool) { + var indicesByNamespace: [MessageId.Namespace: [MessageIndex]] = [:] + for index in indices { + if indicesByNamespace[index.id.namespace] != nil { + indicesByNamespace[index.id.namespace]!.append(index) } else { - idsByNamespace[id.namespace] = [id.id] + indicesByNamespace[index.id.namespace] = [index] } } if let states = self.get(peerId) { if traceReadStates { - print("[ReadStateTable] deleteMessages peerId: \(peerId), ids: \(ids) (before: \(states.namespaces))") + print("[ReadStateTable] deleteMessages peerId: \(peerId), ids: \(indices) (before: \(states.namespaces))") } var updated = false var invalidate = false - for (namespace, ids) in idsByNamespace { + for (namespace, namespaceIndices) in indicesByNamespace { if let currentState = states.namespaces[namespace] { - var unreadIds: [MessageId.Id] = [] - for id in ids { - if id > currentState.maxIncomingReadId { - unreadIds.append(id) + var unreadIndices: [MessageIndex] = [] + for index in namespaceIndices { + if !currentState.isIncomingMessageIndexRead(index) { + unreadIndices.append(index) } } - let (knownCount, holes) = incomingStatsInIds(peerId, namespace, unreadIds) + let (knownCount, holes) = incomingStatsInIndices(peerId, namespace, unreadIndices) if holes { invalidate = true } self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces) - states.namespaces[namespace] = PeerReadState(maxIncomingReadId: currentState.maxIncomingReadId, maxOutgoingReadId: currentState.maxOutgoingReadId, maxKnownId: currentState.maxKnownId, count: currentState.count - knownCount) + states.namespaces[namespace] = currentState.withAddedCount(Int32(-knownCount)) updated = true } else { invalidate = true @@ -206,27 +244,33 @@ final class MessageHistoryReadStateTable: Table { func applyIncomingMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (CombinedPeerReadState?, Bool) { if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] { if traceReadStates { - print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId.id) (before: \(states.namespaces))") + print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId) (before: \(states.namespaces))") } - if state.maxIncomingReadId < messageId.id || (messageId.id == topMessageId && state.count != 0) { - var (deltaCount, holes) = incomingStatsInRange(state.maxIncomingReadId + 1, messageId.id) - - if traceReadStates { - print("[ReadStateTable] applyMaxReadId after deltaCount: \(deltaCount), holes: \(holes)") + switch state { + case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count): + if maxIncomingReadId < messageId.id || (messageId.id == topMessageId && state.count != 0) { + var (deltaCount, holes) = incomingStatsInRange(maxIncomingReadId + 1, messageId.id) + + if traceReadStates { + print("[ReadStateTable] applyMaxReadId after deltaCount: \(deltaCount), holes: \(holes)") + } + + if messageId.id == topMessageId { + if deltaCount != Int(state.count) { + deltaCount = Int(state.count) + holes = true + } + } + + self.markReadStatesAsUpdated(messageId.peerId, namespaces: states.namespaces) + + states.namespaces[messageId.namespace] = .idBased(maxIncomingReadId: messageId.id, maxOutgoingReadId: maxOutgoingReadId, maxKnownId: maxKnownId, count: count - Int32(deltaCount)) + return (CombinedPeerReadState(states: states.namespaces.map({$0})), holes) } - - if messageId.id == topMessageId { - if deltaCount != Int(state.count) { - deltaCount = Int(state.count) - holes = true - } - } - - self.markReadStatesAsUpdated(messageId.peerId, namespaces: states.namespaces) - - states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: messageId.id, maxOutgoingReadId: state.maxOutgoingReadId, maxKnownId: state.maxKnownId, count: state.count - Int32(deltaCount)) - return (CombinedPeerReadState(states: states.namespaces.map({$0})), holes) + case .indexBased: + assertionFailure() + break } } else { return (nil, true) @@ -235,13 +279,48 @@ final class MessageHistoryReadStateTable: Table { return (nil, false) } + func applyIncomingMaxReadIndex(_ messageIndex: MessageIndex, incomingStatsInRange: (MessageIndex, MessageIndex) -> (count: Int, holes: Bool, readMesageIds: [MessageId])) -> (CombinedPeerReadState?, Bool, [MessageId]) { + if let states = self.get(messageIndex.id.peerId), let state = states.namespaces[messageIndex.id.namespace] { + if traceReadStates { + print("[ReadStateTable] applyIncomingMaxReadIndex peerId: \(messageIndex.id.peerId), maxReadIndex: \(messageIndex) (before: \(states.namespaces))") + } + + switch state { + case .idBased: + assertionFailure() + case let .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count): + if maxIncomingReadIndex < messageIndex { + var (deltaCount, holes, messageIds) = incomingStatsInRange(maxIncomingReadIndex.successor(), messageIndex) + + if traceReadStates { + print("[ReadStateTable] applyIncomingMaxReadIndex after deltaCount: \(deltaCount), holes: \(holes)") + } + + self.markReadStatesAsUpdated(messageIndex.id.peerId, namespaces: states.namespaces) + + states.namespaces[messageIndex.id.namespace] = .indexBased(maxIncomingReadIndex: messageIndex, maxOutgoingReadIndex: maxOutgoingReadIndex, count: count - Int32(deltaCount)) + return (CombinedPeerReadState(states: states.namespaces.map({$0})), holes, messageIds) + } + } + } else { + return (nil, true, []) + } + + return (nil, false, []) + } + func applyOutgoingMaxReadId(_ messageId: MessageId) -> (CombinedPeerReadState?, Bool) { if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] { - if state.maxOutgoingReadId < messageId.id { - self.markReadStatesAsUpdated(messageId.peerId, namespaces: states.namespaces) - - states.namespaces[messageId.namespace] = PeerReadState(maxIncomingReadId: state.maxIncomingReadId, maxOutgoingReadId: messageId.id, maxKnownId: state.maxKnownId, count: state.count) - return (CombinedPeerReadState(states: states.namespaces.map({$0})), false) + switch state { + case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count): + if maxOutgoingReadId < messageId.id { + self.markReadStatesAsUpdated(messageId.peerId, namespaces: states.namespaces) + states.namespaces[messageId.namespace] = .idBased(maxIncomingReadId: maxIncomingReadId, maxOutgoingReadId: messageId.id, maxKnownId: maxKnownId, count: count) + return (CombinedPeerReadState(states: states.namespaces.map({$0})), false) + } + case .indexBased: + assertionFailure() + break } } else { return (nil, true) @@ -250,14 +329,51 @@ final class MessageHistoryReadStateTable: Table { return (nil, false) } - func applyInteractiveMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult) { - let (combinedState, holes) = self.applyIncomingMaxReadId(messageId, incomingStatsInRange: incomingStatsInRange, topMessageId: topMessageId) - - if let combinedState = combinedState { - return (combinedState, .Push(thenSync: holes)) + func applyOutgoingMaxReadIndex(_ messageIndex: MessageIndex, outgoingIndexStatsInRange: (MessageIndex, MessageIndex) -> [MessageId]) -> (CombinedPeerReadState?, Bool, [MessageId]) { + if let states = self.get(messageIndex.id.peerId), let state = states.namespaces[messageIndex.id.namespace] { + switch state { + case .idBased: + assertionFailure() + break + case let .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count): + if maxOutgoingReadIndex < messageIndex { + let messageIds: [MessageId] = outgoingIndexStatsInRange(maxOutgoingReadIndex.successor(), messageIndex) + + self.markReadStatesAsUpdated(messageIndex.id.peerId, namespaces: states.namespaces) + states.namespaces[messageIndex.id.namespace] = .indexBased(maxIncomingReadIndex: maxIncomingReadIndex, maxOutgoingReadIndex: messageIndex, count: count) + return (CombinedPeerReadState(states: states.namespaces.map({$0})), false, messageIds) + } + } + } else { + return (nil, true, []) } - return (combinedState, holes ? .Push(thenSync: true) : .None) + return (nil, false, []) + } + + func applyInteractiveMaxReadIndex(_ messageIndex: MessageIndex, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), incomingIndexStatsInRange: (MessageIndex, MessageIndex) -> (count: Int, holes: Bool, readMesageIds: [MessageId]), topMessageId: MessageId.Id?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult, readMesageIds: [MessageId]) { + if let states = self.get(messageIndex.id.peerId), let state = states.namespaces[messageIndex.id.namespace] { + switch state { + case .idBased: + let (combinedState, holes) = self.applyIncomingMaxReadId(messageIndex.id, incomingStatsInRange: incomingStatsInRange, topMessageId: topMessageId) + + if let combinedState = combinedState { + return (combinedState, .Push(thenSync: holes), []) + } + + return (combinedState, holes ? .Push(thenSync: true) : .None, []) + case .indexBased: + let (combinedState, holes, messageIds) = self.applyIncomingMaxReadIndex(messageIndex, incomingStatsInRange: incomingIndexStatsInRange) + + if let combinedState = combinedState { + return (combinedState, .Push(thenSync: holes), messageIds) + } + + return (combinedState, holes ? .Push(thenSync: true) : .None, messageIds) + } + } else { + return (nil, .Push(thenSync: true), []) + } } func transactionUnreadCountDeltas() -> [PeerId: Int32] { @@ -301,15 +417,43 @@ final class MessageHistoryReadStateTable: Table { sharedBuffer.write(&count, offset: 0, length: 4) for (namespace, state) in states.namespaces { var namespaceId: Int32 = namespace - var maxIncomingReadId: Int32 = state.maxIncomingReadId - var maxOutgoingReadId: Int32 = state.maxOutgoingReadId - var maxKnownId: Int32 = state.maxKnownId - var count: Int32 = state.count sharedBuffer.write(&namespaceId, offset: 0, length: 4) - sharedBuffer.write(&maxIncomingReadId, offset: 0, length: 4) - sharedBuffer.write(&maxOutgoingReadId, offset: 0, length: 4) - sharedBuffer.write(&maxKnownId, offset: 0, length: 4) - sharedBuffer.write(&count, offset: 0, length: 4) + + switch state { + case var .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count): + var kind: Int8 = 0 + sharedBuffer.write(&kind, offset: 0, length: 1) + + sharedBuffer.write(&maxIncomingReadId, offset: 0, length: 4) + sharedBuffer.write(&maxOutgoingReadId, offset: 0, length: 4) + sharedBuffer.write(&maxKnownId, offset: 0, length: 4) + sharedBuffer.write(&count, offset: 0, length: 4) + case var .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count): + var kind: Int8 = 1 + sharedBuffer.write(&kind, offset: 0, length: 1) + + var maxIncomingReadTimestamp: Int32 = maxIncomingReadIndex.timestamp + var maxIncomingReadIdPeerId: Int64 = maxIncomingReadIndex.id.peerId.toInt64() + var maxIncomingReadIdNamespace: Int32 = maxIncomingReadIndex.id.namespace + var maxIncomingReadIdId: Int32 = maxIncomingReadIndex.id.id + + var maxOutgoingReadTimestamp: Int32 = maxOutgoingReadIndex.timestamp + var maxOutgoingReadIdPeerId: Int64 = maxOutgoingReadIndex.id.peerId.toInt64() + var maxOutgoingReadIdNamespace: Int32 = maxOutgoingReadIndex.id.namespace + var maxOutgoingReadIdId: Int32 = maxOutgoingReadIndex.id.id + + sharedBuffer.write(&maxIncomingReadTimestamp, offset: 0, length: 4) + sharedBuffer.write(&maxIncomingReadIdPeerId, offset: 0, length: 8) + sharedBuffer.write(&maxIncomingReadIdNamespace, offset: 0, length: 4) + sharedBuffer.write(&maxIncomingReadIdId, offset: 0, length: 4) + + sharedBuffer.write(&maxOutgoingReadTimestamp, offset: 0, length: 4) + sharedBuffer.write(&maxOutgoingReadIdPeerId, offset: 0, length: 8) + sharedBuffer.write(&maxOutgoingReadIdNamespace, offset: 0, length: 4) + sharedBuffer.write(&maxOutgoingReadIdId, offset: 0, length: 4) + + sharedBuffer.write(&count, offset: 0, length: 4) + } } self.valueBox.set(self.table, key: self.key(id), value: sharedBuffer) } else { diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index d5c7ec0ace..93ef4e30cf 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -41,15 +41,17 @@ final class MessageHistoryTable: Table { let messageHistoryIndexTable: MessageHistoryIndexTable let messageMediaTable: MessageMediaTable let historyMetadataTable: MessageHistoryMetadataTable + let globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable let unsentTable: MessageHistoryUnsentTable let tagsTable: MessageHistoryTagsTable let readStateTable: MessageHistoryReadStateTable let synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable - init(valueBox: ValueBox, table: ValueBoxTable, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable) { + init(valueBox: ValueBox, table: ValueBoxTable, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable) { self.messageHistoryIndexTable = messageHistoryIndexTable self.messageMediaTable = messageMediaTable self.historyMetadataTable = historyMetadataTable + self.globallyUniqueMessageIdsTable = globallyUniqueMessageIdsTable self.unsentTable = unsentTable self.tagsTable = tagsTable self.readStateTable = readStateTable @@ -108,6 +110,30 @@ final class MessageHistoryTable: Table { return dict } + private func processIndexOperationsCommitAccumulatedRemoveIndices(remove: Bool, peerId: PeerId, accumulatedRemoveIndices: inout [MessageIndex], updatedCombinedState: inout CombinedPeerReadState?, invalidateReadState: inout Bool, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], outputOperations: inout [MessageHistoryOperation]) { + if !accumulatedRemoveIndices.isEmpty { + outputOperations.append(.Remove(accumulatedRemoveIndices)) + + let (combinedState, invalidate) = self.readStateTable.deleteMessages(peerId, indices: accumulatedRemoveIndices, incomingStatsInIndices: { peerId, namespace, indices in + return self.incomingMessageStatsInIndices(peerId, namespace: namespace, indices: indices) + }) + if let combinedState = combinedState { + updatedCombinedState = combinedState + } + if invalidate { + invalidateReadState = true + } + + if remove { + for index in accumulatedRemoveIndices { + self.justRemove(index, unsentMessageOperations: &unsentMessageOperations) + } + } + + accumulatedRemoveIndices.removeAll() + } + } + private func processIndexOperations(_ peerId: PeerId, operations: [MessageHistoryIndexOperation], processedOperationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0)) let sharedBuffer = WriteBuffer() @@ -115,14 +141,31 @@ final class MessageHistoryTable: Table { var outputOperations: [MessageHistoryOperation] = [] var accumulatedRemoveIndices: [MessageIndex] = [] - var addedIncomingMessageIds = Set() + var accumulatedAddedIncomingMessageIndices = Set() + + var updatedCombinedState: CombinedPeerReadState? + var invalidateReadState = false + + let commitAccumulatedAddedIndices: () -> Void = { + if !accumulatedAddedIncomingMessageIndices.isEmpty { + let (combinedState, invalidate) = self.readStateTable.addIncomingMessages(peerId, indices: accumulatedAddedIncomingMessageIndices) + if let combinedState = combinedState { + updatedCombinedState = combinedState + } + if invalidate { + invalidateReadState = true + } + + accumulatedAddedIncomingMessageIndices.removeAll() + } + } + for operation in operations { switch operation { case let .InsertHole(hole): - if accumulatedRemoveIndices.count != 0 { - outputOperations.append(.Remove(accumulatedRemoveIndices)) - accumulatedRemoveIndices.removeAll() - } + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + commitAccumulatedAddedIndices() + self.justInsertHole(hole) outputOperations.append(.InsertHole(hole)) @@ -139,10 +182,8 @@ final class MessageHistoryTable: Table { } } case let .InsertMessage(storeMessage): - if accumulatedRemoveIndices.count != 0 { - outputOperations.append(.Remove(accumulatedRemoveIndices)) - accumulatedRemoveIndices.removeAll() - } + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + let message = self.justInsertMessage(storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder) outputOperations.append(.InsertMessage(message)) if message.flags.contains(.Unsent) && !message.flags.contains(.Failed) { @@ -163,42 +204,42 @@ final class MessageHistoryTable: Table { } } if message.flags.contains(.Incoming) { - addedIncomingMessageIds.insert(message.id) + accumulatedAddedIncomingMessageIndices.insert(MessageIndex(message)) } case let .Remove(index): - addedIncomingMessageIds.remove(index.id) - self.justRemove(index, unsentMessageOperations: &unsentMessageOperations) + commitAccumulatedAddedIndices() + accumulatedRemoveIndices.append(index) case let .Update(index, storeMessage): - accumulatedRemoveIndices.append(index) - if accumulatedRemoveIndices.count != 0 { - outputOperations.append(.Remove(accumulatedRemoveIndices)) - accumulatedRemoveIndices.removeAll() - } + commitAccumulatedAddedIndices() + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + if let message = self.justUpdate(index, message: storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder, unsentMessageOperations: &unsentMessageOperations) { + outputOperations.append(.Remove(accumulatedRemoveIndices)) outputOperations.append(.InsertMessage(message)) + + if message.flags.contains(.Incoming) { + accumulatedAddedIncomingMessageIndices.insert(MessageIndex(message)) + } } case let .UpdateTimestamp(index, timestamp): - if accumulatedRemoveIndices.count != 0 { - outputOperations.append(.Remove(accumulatedRemoveIndices)) - accumulatedRemoveIndices.removeAll() - } + commitAccumulatedAddedIndices() + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + self.justUpdateTimestamp(index, timestamp: timestamp) outputOperations.append(.UpdateTimestamp(index, timestamp)) } } - if accumulatedRemoveIndices.count != 0 { - outputOperations.append(.Remove(accumulatedRemoveIndices)) + + commitAccumulatedAddedIndices() + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + + if let updatedCombinedState = updatedCombinedState { + outputOperations.append(.UpdateReadState(updatedCombinedState)) } - if !addedIncomingMessageIds.isEmpty { - let (combinedState, invalidate) = self.readStateTable.addIncomingMessages(peerId, ids: addedIncomingMessageIds) - if let combinedState = combinedState { - outputOperations.append(.UpdateReadState(combinedState)) - } - if invalidate { - self.synchronizeReadStateTable.set(peerId, operation: .Validate, operations: &updatedPeerReadStateOperations) - } + if invalidateReadState { + self.synchronizeReadStateTable.set(peerId, operation: .Validate, operations: &updatedPeerReadStateOperations) } if processedOperationsByPeerId[peerId] == nil { @@ -245,29 +286,10 @@ final class MessageHistoryTable: Table { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] - let (combinedState, invalidate) = self.readStateTable.deleteMessages(peerId, ids: messageIds, incomingStatsInIds: { peerId, namespace, ids in - return self.messageHistoryIndexTable.incomingMessageCountInIds(peerId, namespace: namespace, ids: ids) - }) - for id in messageIds { self.messageHistoryIndexTable.removeMessage(id, operations: &operations) } self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) - - if let combinedState = combinedState { - var outputOperations: [MessageHistoryOperation] = [] - outputOperations.append(.UpdateReadState(combinedState)) - - if operationsByPeerId[peerId] == nil { - operationsByPeerId[peerId] = outputOperations - } else { - operationsByPeerId[peerId]!.append(contentsOf: outputOperations) - } - } - - if invalidate { - self.synchronizeReadStateTable.set(peerId, operation: .Validate, operations: &updatedPeerReadStateOperations) - } } } @@ -277,6 +299,12 @@ final class MessageHistoryTable: Table { self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } + func fillMultipleHoles(mainHoleId: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + var operations: [MessageHistoryIndexOperation] = [] + self.messageHistoryIndexTable.fillMultipleHoles(mainHoleId: mainHoleId, fillType: fillType, tagMask: tagMask, messages: self.internalStoreMessages(messages), operations: &operations) + self.processIndexOperations(mainHoleId.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + func updateMessage(_ id: MessageId, message: StoreMessage, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.updateMessage(id, message: self.internalStoreMessages([message]).first!, operations: &operations) @@ -367,30 +395,54 @@ final class MessageHistoryTable: Table { } } - func applyInteractiveMaxReadId(_ messageId: MessageId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func applyOutgoingReadMaxIndex(_ messageIndex: MessageIndex, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [MessageId] { + let (combinedState, invalidated, messageIds) = self.readStateTable.applyOutgoingMaxReadIndex(messageIndex, outgoingIndexStatsInRange: { fromIndex, toIndex in + return self.outgoingMessageCountInRange(messageIndex.id.peerId, namespace: messageIndex.id.namespace, fromIndex: fromIndex, toIndex: toIndex) + }) + + if let combinedState = combinedState { + if operationsByPeerId[messageIndex.id.peerId] == nil { + operationsByPeerId[messageIndex.id.peerId] = [.UpdateReadState(combinedState)] + } else { + operationsByPeerId[messageIndex.id.peerId]!.append(.UpdateReadState(combinedState)) + } + } + + if invalidated { + self.synchronizeReadStateTable.set(messageIndex.id.peerId, operation: .Validate, operations: &updatedPeerReadStateOperations) + } + + return messageIds + } + + func applyInteractiveMaxReadIndex(_ messageIndex: MessageIndex, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [MessageId] { var topMessageId: MessageId.Id? - if let topEntry = self.messageHistoryIndexTable.top(messageId.peerId, namespace: messageId.namespace), case let .Message(index) = topEntry { + if let topEntry = self.messageHistoryIndexTable.top(messageIndex.id.peerId, namespace: messageIndex.id.namespace), case let .Message(index) = topEntry { topMessageId = index.id.id } - let (combinedState, result) = self.readStateTable.applyInteractiveMaxReadId(messageId, incomingStatsInRange: { fromId, toId in - return self.messageHistoryIndexTable.incomingMessageCountInRange(messageId.peerId, namespace: messageId.namespace, minId: fromId, maxId: toId) + let (combinedState, result, messageIds) = self.readStateTable.applyInteractiveMaxReadIndex(messageIndex, incomingStatsInRange: { fromId, toId in + return self.messageHistoryIndexTable.incomingMessageCountInRange(messageIndex.id.peerId, namespace: messageIndex.id.namespace, minId: fromId, maxId: toId) + }, incomingIndexStatsInRange: { fromIndex, toIndex in + return self.incomingMessageCountInRange(messageIndex.id.peerId, namespace: messageIndex.id.namespace, fromIndex: fromIndex, toIndex: toIndex) }, topMessageId: topMessageId) if let combinedState = combinedState { - if operationsByPeerId[messageId.peerId] == nil { - operationsByPeerId[messageId.peerId] = [.UpdateReadState(combinedState)] + if operationsByPeerId[messageIndex.id.peerId] == nil { + operationsByPeerId[messageIndex.id.peerId] = [.UpdateReadState(combinedState)] } else { - operationsByPeerId[messageId.peerId]!.append(.UpdateReadState(combinedState)) + operationsByPeerId[messageIndex.id.peerId]!.append(.UpdateReadState(combinedState)) } } switch result { case let .Push(thenSync): - self.synchronizeReadStateTable.set(messageId.peerId, operation: .Push(thenSync: thenSync), operations: &updatedPeerReadStateOperations) + self.synchronizeReadStateTable.set(messageIndex.id.peerId, operation: .Push(state: self.readStateTable.getCombinedState(messageIndex.id.peerId), thenSync: thenSync), operations: &updatedPeerReadStateOperations) case .None: break } + + return messageIds } func topMessage(_ peerId: PeerId) -> IntermediateMessage? { @@ -462,6 +514,7 @@ final class MessageHistoryTable: Table { if let globallyUniqueId = message.globallyUniqueId { var globallyUniqueIdValue = globallyUniqueId sharedBuffer.write(&globallyUniqueIdValue, offset: 0, length: 8) + self.globallyUniqueMessageIdsTable.set(peerId: message.id.peerId, globallyUniqueId: globallyUniqueId, id: message.id) } var flags = MessageFlags(message.flags) @@ -615,6 +668,10 @@ final class MessageHistoryTable: Table { self.unsentTable.remove(index.id, operations: &unsentMessageOperations) } + if let globallyUniqueId = message.globallyUniqueId { + self.globallyUniqueMessageIdsTable.remove(peerId: message.id.peerId, globallyUniqueId: globallyUniqueId) + } + let tags = message.tags.rawValue if tags != 0 { for i in 0 ..< 32 { @@ -799,6 +856,17 @@ final class MessageHistoryTable: Table { self.tagsTable.add(message.tags, index: MessageIndex(message)) } + if message.globallyUniqueId != previousMessage.globallyUniqueId { + if let globallyUniqueId = previousMessage.globallyUniqueId { + self.globallyUniqueMessageIdsTable.remove(peerId: message.id.peerId, globallyUniqueId: globallyUniqueId) + } + if let globallyUniqueId = message.globallyUniqueId { + self.globallyUniqueMessageIdsTable.set(peerId: message.id.peerId, globallyUniqueId: globallyUniqueId, id: message.id) + } + } else if let globallyUniqueId = message.globallyUniqueId, previousMessage.id != message.id { + self.globallyUniqueMessageIdsTable.set(peerId: message.id.peerId, globallyUniqueId: globallyUniqueId, id: message.id) + } + switch (previousMessage.flags.contains(.Unsent) && !previousMessage.flags.contains(.Failed), message.flags.contains(.Unsent) && !message.flags.contains(.Failed)) { case (true, false): self.unsentTable.remove(index.id, operations: &unsentMessageOperations) @@ -1507,7 +1575,12 @@ final class MessageHistoryTable: Table { func maxReadIndex(_ peerId: PeerId) -> MessageHistoryAnchorIndex? { if let combinedState = self.readStateTable.getCombinedState(peerId), let state = combinedState.states.first , state.1.count != 0 { - return self.anchorIndex(MessageId(peerId: peerId, namespace: state.0, id: state.1.maxIncomingReadId)) + switch state.1 { + case let .idBased(maxIncomingReadId, _, _, _): + return self.anchorIndex(MessageId(peerId: peerId, namespace: state.0, id: maxIncomingReadId)) + case let .indexBased(maxIncomingReadIndex, _, _): + return MessageHistoryAnchorIndex(index: maxIncomingReadIndex, exact: true) + } } return nil } @@ -1542,6 +1615,60 @@ final class MessageHistoryTable: Table { }, limit: 0) return result } + + func incomingMessageStatsInIndices(_ peerId: PeerId, namespace: MessageId.Namespace, indices: [MessageIndex]) -> (Int, Bool) { + var count: Int = 0 + var holes = false + for index in indices { + let key = self.key(index) + if let value = self.valueBox.get(self.table, key: key) { + let entry = self.readIntermediateEntry(key, value: value) + if case let .Message(message) = entry { + if message.id.namespace == namespace && message.flags.contains(.Incoming) { + count += 1 + } + } else { + holes = true + } + } + } + return (count, holes) + } + + func incomingMessageCountInRange(_ peerId: PeerId, namespace: MessageId.Namespace, fromIndex: MessageIndex, toIndex: MessageIndex) -> (Int, Bool, [MessageId]) { + var count: Int = 0 + var messageIds: [MessageId] = [] + var holes = false + self.valueBox.range(self.table, start: self.key(fromIndex).predecessor, end: self.key(toIndex).successor, values: { key, value in + let entry = self.readIntermediateEntry(key, value: value) + if case let .Message(message) = entry { + if message.id.namespace == namespace && message.flags.contains(.Incoming) { + count += 1 + messageIds.append(message.id) + } + } else { + holes = true + } + return true + }, limit: 0) + + return (count, holes, messageIds) + } + + func outgoingMessageCountInRange(_ peerId: PeerId, namespace: MessageId.Namespace, fromIndex: MessageIndex, toIndex: MessageIndex) -> [MessageId] { + var messageIds: [MessageId] = [] + self.valueBox.range(self.table, start: self.key(fromIndex).predecessor, end: self.key(toIndex).successor, values: { key, value in + let entry = self.readIntermediateEntry(key, value: value) + if case let .Message(message) = entry { + if message.id.namespace == namespace && !message.flags.contains(.Incoming) { + messageIds.append(message.id) + } + } + return true + }, limit: 0) + + return messageIds + } func debugList(_ peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 17c070ed0f..b60432eb93 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -58,13 +58,13 @@ enum MutableMessageHistoryEntry { func updatedTimestamp(_ timestamp: Int32) -> MutableMessageHistoryEntry { switch self { case let .IntermediateMessageEntry(message, location): - var updatedMessage = IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) + let updatedMessage = IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) return .IntermediateMessageEntry(updatedMessage, location) case let .MessageEntry(message, location): - var updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) return .MessageEntry(updatedMessage, location) case let .HoleEntry(hole, location): - var updatedHole = MessageHistoryHole(stableId: hole.stableId, maxIndex: MessageIndex(id: hole.maxIndex.id, timestamp: timestamp), min: hole.min, tags: hole.tags) + let updatedHole = MessageHistoryHole(stableId: hole.stableId, maxIndex: MessageIndex(id: hole.maxIndex.id, timestamp: timestamp), min: hole.min, tags: hole.tags) return .HoleEntry(updatedHole, location) } } @@ -159,6 +159,35 @@ enum MessageHistoryTopTaggedMessage { } } +public enum MessageHistoryViewRelativeHoleDirection: Equatable { + case UpperToLower + case LowerToUpper + case AroundIndex(MessageIndex) + + public static func ==(lhs: MessageHistoryViewRelativeHoleDirection, rhs: MessageHistoryViewRelativeHoleDirection) -> Bool { + switch lhs { + case .UpperToLower: + if case .UpperToLower = rhs { + return true + } else { + return false + } + case .LowerToUpper: + if case .LowerToUpper = rhs { + return true + } else { + return false + } + case let .AroundIndex(index): + if case .AroundIndex(index) = rhs { + return true + } else { + return false + } + } + } +} + final class MutableMessageHistoryView { private(set) var id: MessageHistoryViewId let tagMask: MessageTags? @@ -621,7 +650,7 @@ final class MutableMessageHistoryView { } } - func firstHole() -> (MessageHistoryHole, HoleFillDirection)? { + func firstHole() -> (MessageHistoryHole, MessageHistoryViewRelativeHoleDirection)? { if self.entries.isEmpty { return nil } @@ -689,6 +718,9 @@ public final class MessageHistoryView { self.id = mutableView.id self.anchorIndex = mutableView.anchorIndex.index + var earlierId = mutableView.earlier?.index + var laterId = mutableView.later?.index + var entries: [MessageHistoryEntry] = [] if let transientReadState = mutableView.transientReadState { for entry in mutableView.entries { @@ -700,7 +732,7 @@ public final class MessageHistoryView { if message.flags.contains(.Incoming) { read = false } else { - read = transientReadState.isOutgoingMessageIdRead(message.id) + read = transientReadState.isOutgoingMessageIndexRead(MessageIndex(message)) } entries.append(.MessageEntry(message, read, location)) case .IntermediateMessageEntry: @@ -719,6 +751,33 @@ public final class MessageHistoryView { } } } + if !entries.isEmpty { + var referenceIndex = entries.count - 1 + for i in 0 ..< entries.count { + if entries[i].index >= self.anchorIndex { + referenceIndex = i + break + } + } + for i in referenceIndex ..< entries.count { + if case .HoleEntry = entries[i] { + if i != entries.count - 1 { + entries.removeSubrange(i + 1 ..< entries.count) + laterId = nil + } + break + } + } + for i in (0 ... referenceIndex).reversed() { + if case .HoleEntry = entries[i] { + if i != 0 { + entries.removeSubrange(0 ..< i) + earlierId = nil + } + break + } + } + } self.entries = entries var topTaggedMessages: [Message] = [] @@ -735,8 +794,8 @@ public final class MessageHistoryView { self.topTaggedMessages = topTaggedMessages self.additionalData = mutableView.additionalDatas - self.earlierId = mutableView.earlier?.index - self.laterId = mutableView.later?.index + self.earlierId = earlierId + self.laterId = laterId self.combinedReadState = mutableView.combinedReadState @@ -746,7 +805,7 @@ public final class MessageHistoryView { var maxNamespaceIndex: MessageIndex? var index = entries.count - 1 for entry in entries.reversed() { - if entry.index.id.namespace == namespace && entry.index.id.id <= state.maxIncomingReadId { + if entry.index.id.namespace == namespace && state.isIncomingMessageIndexRead(entry.index) { maxNamespaceIndex = entry.index break } diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index 12b070bc13..44dc53fef6 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -98,6 +98,7 @@ public protocol Peer: class, Coding { var id: PeerId { get } var indexName: PeerIndexNameRepresentation { get } var associatedPeerIds: [PeerId]? { get } + var notificationSettingsPeerId: PeerId? { get } func isEqual(_ other: Peer) -> Bool } diff --git a/Postbox/PeerNameIndexTable.swift b/Postbox/PeerNameIndexTable.swift index f5991b76c0..5365b53ef3 100644 --- a/Postbox/PeerNameIndexTable.swift +++ b/Postbox/PeerNameIndexTable.swift @@ -220,7 +220,7 @@ final class PeerNameIndexTable: Table { return ([], []) } else { var contacts: [PeerId] = [] - var chatIndices: [PeerId: MessageIndex] = [:] + var chatIndices: [PeerId: ChatListIndex] = [:] var peerIds = self.peerNameTokenIndexTable.matchingPeerIds(tokens: tokens.regular) if let transliterated = tokens.transliterated, tokens.regular != transliterated { let transliteratedPeerIds = self.peerNameTokenIndexTable.matchingPeerIds(tokens: transliterated) diff --git a/Postbox/PeerReadState.swift b/Postbox/PeerReadState.swift index a0fa71469c..0f7f5bf258 100644 --- a/Postbox/PeerReadState.swift +++ b/Postbox/PeerReadState.swift @@ -1,24 +1,69 @@ -public struct PeerReadState: Equatable, CustomStringConvertible { - public let maxIncomingReadId: MessageId.Id - public let maxOutgoingReadId: MessageId.Id - public let maxKnownId: MessageId.Id - public let count: Int32 +public enum PeerReadState: Equatable, CustomStringConvertible { + case idBased(maxIncomingReadId: MessageId.Id, maxOutgoingReadId: MessageId.Id, maxKnownId: MessageId.Id, count: Int32) + case indexBased(maxIncomingReadIndex: MessageIndex, maxOutgoingReadIndex: MessageIndex, count: Int32) - public init(maxIncomingReadId: MessageId.Id, maxOutgoingReadId: MessageId.Id, maxKnownId: MessageId.Id, count: Int32) { - self.maxIncomingReadId = maxIncomingReadId - self.maxOutgoingReadId = maxOutgoingReadId - self.maxKnownId = maxKnownId - self.count = count + public var count: Int32 { + switch self { + case let .idBased(_, _, _, count): + return count + case let .indexBased(_, _, count): + return count + } + } + + func withAddedCount(_ value: Int32) -> PeerReadState { + switch self { + case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count): + return .idBased(maxIncomingReadId: maxIncomingReadId, maxOutgoingReadId: maxOutgoingReadId, maxKnownId: maxKnownId, count: count + value) + case let .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count): + return .indexBased(maxIncomingReadIndex: maxIncomingReadIndex, maxOutgoingReadIndex: maxOutgoingReadIndex, count: count + value) + } } public var description: String { - return "(PeerReadState maxIncomingReadId: \(maxIncomingReadId), maxOutgoingReadId: \(maxOutgoingReadId) maxKnownId: \(maxKnownId), count: \(count)" + switch self { + case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count): + return "(PeerReadState maxIncomingReadId: \(maxIncomingReadId), maxOutgoingReadId: \(maxOutgoingReadId) maxKnownId: \(maxKnownId), count: \(count)" + case let .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count): + return "(PeerReadState maxIncomingReadIndex: \(maxIncomingReadIndex), maxOutgoingReadIndex: \(maxOutgoingReadIndex), count: \(count)" + } + } + + public static func ==(lhs: PeerReadState, rhs: PeerReadState) -> Bool { + switch lhs { + case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count): + if case .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count) = rhs { + return true + } else { + return false + } + case let .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count): + if case .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count) = rhs { + return true + } else { + return false + } + } + } + + func isIncomingMessageIndexRead(_ index: MessageIndex) -> Bool { + switch self { + case let .idBased(maxIncomingReadId, _, _, _): + return maxIncomingReadId >= index.id.id + case let .indexBased(maxIncomingReadIndex, _, _): + return maxIncomingReadIndex >= index + } + } + + func isOutgoingMessageIndexRead(_ index: MessageIndex) -> Bool { + switch self { + case let .idBased(_, maxOutgoingReadId, _, _): + return maxOutgoingReadId >= index.id.id + case let .indexBased(_, maxOutgoingReadIndex, _): + return maxOutgoingReadIndex >= index + } } -} - -public func ==(lhs: PeerReadState, rhs: PeerReadState) -> Bool { - return lhs.maxIncomingReadId == rhs.maxIncomingReadId && lhs.maxOutgoingReadId == rhs.maxOutgoingReadId && lhs.maxKnownId == rhs.maxKnownId && lhs.count == rhs.count } public struct CombinedPeerReadState: Equatable { @@ -53,19 +98,19 @@ public struct CombinedPeerReadState: Equatable { return true } - public func isOutgoingMessageIdRead(_ id: MessageId) -> Bool { + public func isOutgoingMessageIndexRead(_ index: MessageIndex) -> Bool { for (namespace, readState) in self.states { - if namespace == id.namespace { - return readState.maxOutgoingReadId >= id.id + if namespace == index.id.namespace { + return readState.isOutgoingMessageIndexRead(index) } } return false } - public func isIncomingMessageIdRead(_ id: MessageId) -> Bool { + public func isIncomingMessageIndexRead(_ index: MessageIndex) -> Bool { for (namespace, readState) in self.states { - if namespace == id.namespace { - return readState.maxIncomingReadId >= id.id + if namespace == index.id.namespace { + return readState.isIncomingMessageIndexRead(index) } } return false diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 75ad3ecbc5..5c134dfdc2 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -35,6 +35,10 @@ public final class Modifier { self.postbox?.fillHole(hole, fillType: fillType, tagMask: tagMask, messages: messages) } + public func fillMultipleHoles(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { + self.postbox?.fillMultipleHoles(hole, fillType: fillType, tagMask: tagMask, messages: messages) + } + public func replaceChatListHole(_ index: MessageIndex, hole: ChatListHole?) { self.postbox?.replaceChatListHole(index, hole: hole) } @@ -50,6 +54,10 @@ public final class Modifier { } } + public func messageIdForGloballyUniqueMessageId(peerId: PeerId, id: Int64) -> MessageId? { + return self.postbox?.messageIdForGloballyUniqueMessageId(peerId: peerId, id: id) + } + public func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]]) { self.postbox?.resetIncomingReadStates(states) } @@ -66,8 +74,20 @@ public final class Modifier { self.postbox?.applyOutgoingReadMaxId(messageId) } - public func applyInteractiveReadMaxId(_ messageId: MessageId) { - self.postbox?.applyInteractiveReadMaxId(messageId) + public func applyInteractiveReadMaxIndex(_ messageIndex: MessageIndex) -> [MessageId] { + if let postbox = self.postbox { + return postbox.applyInteractiveReadMaxIndex(messageIndex) + } else { + return [] + } + } + + public func applyOutgoingReadMaxIndex(_ messageIndex: MessageIndex) -> [MessageId] { + if let postbox = self.postbox { + return postbox.applyOutgoingReadMaxIndex(messageIndex) + } else { + return [] + } } public func getState() -> Coding? { @@ -98,6 +118,10 @@ public final class Modifier { return self.postbox?.readStateTable.getCombinedState(id)?.states } + public func getCombinedPeerReadState(_ id: PeerId) -> CombinedPeerReadState? { + return self.postbox?.readStateTable.getCombinedState(id) + } + public func getPeerNotificationSettings(_ id: PeerId) -> PeerNotificationSettings? { return self.postbox?.peerNotificationSettingsTable.get(id) } @@ -210,9 +234,13 @@ public final class Modifier { self.postbox?.operationLogEnumerateEntries(peerId: peerId, tag: tag, f) } - /*public func operationLogTransformOperationContent(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32, _ f: (PeerMergedOperationLogEntry?) -> PeerMergedOperationLogEntryTransform) { - self.postbox?.operationLogTransformOperationContent(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, f) - }*/ + public func addTimestampBasedMessageAttribute(tag: UInt16, timestamp: Int32, messageId: MessageId) { + self.postbox?.addTimestampBasedMessageAttribute(tag: tag, timestamp: timestamp, messageId: messageId) + } + + public func removeTimestampBasedMessageAttribute(tag: UInt16, messageId: MessageId) { + self.postbox?.removeTimestampBasedMessageAttribute(tag: tag, messageId: messageId) + } } fileprivate class PipeNotifier: NSObject { @@ -266,6 +294,7 @@ public final class Postbox { private var currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] private var currentUpdatedTotalUnreadCount: Int32? private var currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation] = [] + private var currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation] = [] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -291,6 +320,7 @@ public final class Postbox { var cachedPeerDataTable: CachedPeerDataTable! var peerPresenceTable: PeerPresenceTable! var globalMessageIdsTable: GlobalMessageIdsTable! + var globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable! var messageHistoryIndexTable: MessageHistoryIndexTable! var messageHistoryTable: MessageHistoryTable! var mediaTable: MessageMediaTable! @@ -314,6 +344,8 @@ public final class Postbox { var peerOperationLogMetadataTable: PeerOperationLogMetadataTable! var peerMergedOperationLogIndexTable: PeerMergedOperationLogIndexTable! var peerOperationLogTable: PeerOperationLogTable! + var timestampBasedMessageAttributesTable: TimestampBasedMessageAttributesTable! + var timestampBasedMessageAttributesIndexTable: TimestampBasedMessageAttributesIndexTable! //temporary var peerRatingTable: RatingTable! @@ -387,7 +419,7 @@ public final class Postbox { //self.debugRestoreState("afterLogin") //self.debugSaveState(name: "previous") - //self.debugRestoreState(name: "previous") + self.debugRestoreState(name: "previous") //#endif @@ -396,7 +428,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 5 + let currentUserVersion: Int32 = 9 if userVersion != currentUserVersion { self.valueBox.drop() @@ -406,6 +438,7 @@ public final class Postbox { self.keychainTable = KeychainTable(valueBox: self.valueBox, table: KeychainTable.tableSpec(1)) self.peerTable = PeerTable(valueBox: self.valueBox, table: PeerTable.tableSpec(2)) self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox, table: GlobalMessageIdsTable.tableSpec(3), namespace: self.globalMessageIdsNamespace) + self.globallyUniqueMessageIdsTable = MessageGloballyUniqueIdTable(valueBox: self.valueBox, table: MessageGloballyUniqueIdTable.tableSpec(32)) self.messageHistoryMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox, table: MessageHistoryMetadataTable.tableSpec(10)) self.messageHistoryUnsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox, table: MessageHistoryUnsentTable.tableSpec(11)) self.messageHistoryTagsTable = MessageHistoryTagsTable(valueBox: self.valueBox, table: MessageHistoryTagsTable.tableSpec(12)) @@ -413,7 +446,9 @@ public final class Postbox { self.mediaTable = MessageMediaTable(valueBox: self.valueBox, table: MessageMediaTable.tableSpec(6)) self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox, table: MessageHistoryReadStateTable.tableSpec(14)) self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox, table: MessageHistorySynchronizeReadStateTable.tableSpec(15)) - self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.timestampBasedMessageAttributesIndexTable = TimestampBasedMessageAttributesIndexTable(valueBox: self.valueBox!, table: TimestampBasedMessageAttributesTable.tableSpec(33)) + self.timestampBasedMessageAttributesTable = TimestampBasedMessageAttributesTable(valueBox: self.valueBox!, table: TimestampBasedMessageAttributesTable.tableSpec(34), indexTable: self.timestampBasedMessageAttributesIndexTable) + self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable!, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable!) self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, table: PeerChatStateTable.tableSpec(13)) self.contactsTable = ContactTable(valueBox: self.valueBox, table: ContactTable.tableSpec(16)) self.peerRatingTable = RatingTable(valueBox: self.valueBox, table: RatingTable.tableSpec(17)) @@ -437,6 +472,7 @@ public final class Postbox { self.tables.append(self.keychainTable) self.tables.append(self.peerTable) self.tables.append(self.globalMessageIdsTable) + self.tables.append(self.globallyUniqueMessageIdsTable) self.tables.append(self.messageHistoryMetadataTable) self.tables.append(self.messageHistoryUnsentTable) self.tables.append(self.messageHistoryTagsTable) @@ -483,7 +519,11 @@ public final class Postbox { return self.peerOperationLogTable.getMergedEntries(tag: tag, fromIndex: fromIndex, limit: limit) }, operationLogGetTailIndex: { tag in return self.peerMergedOperationLogIndexTable.tailIndex(tag: tag) - }, unsentMessageIds: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get()) + }, getTimestampBasedMessageAttributesHead: { tag in + return self.timestampBasedMessageAttributesTable.head(tag: tag) + }, unsentMessageIds: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get(getCombinedPeerReadState: { peerId in + return self.readStateTable.getCombinedState(peerId) + })) print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") }) @@ -627,6 +667,46 @@ public final class Postbox { } } + fileprivate func fillMultipleHoles(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + self.messageHistoryTable.fillMultipleHoles(mainHoleId: hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + for (peerId, operations) in operationsByPeerId { + if self.currentOperationsByPeerId[peerId] == nil { + self.currentOperationsByPeerId[peerId] = operations + } else { + self.currentOperationsByPeerId[peerId]!.append(contentsOf: operations) + } + + var filledMessageIndices: [MessageIndex: HoleFillDirection] = [:] + for operation in operations { + switch operation { + case let .InsertHole(hole): + filledMessageIndices[hole.maxIndex] = fillType.direction + case let .InsertMessage(message): + filledMessageIndices[MessageIndex(message)] = fillType.direction + default: + break + } + } + + if !filledMessageIndices.isEmpty { + if self.currentFilledHolesByPeerId[peerId] == nil { + self.currentFilledHolesByPeerId[peerId] = filledMessageIndices + } else { + for (messageIndex, direction) in filledMessageIndices { + self.currentFilledHolesByPeerId[peerId]![messageIndex] = direction + } + } + } + + if self.currentRemovedHolesByPeerId[peerId] == nil { + self.currentRemovedHolesByPeerId[peerId] = [hole.maxIndex: fillType.direction] + } else { + self.currentRemovedHolesByPeerId[peerId]![hole.maxIndex] = fillType.direction + } + } + } + fileprivate func replaceChatListHole(_ index: MessageIndex, hole: ChatListHole?) { self.currentReplaceChatListHoles.append((index, hole)) } @@ -651,8 +731,12 @@ public final class Postbox { self.messageHistoryTable.applyOutgoingReadMaxId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } - fileprivate func applyInteractiveReadMaxId(_ messageId: MessageId) { - self.messageHistoryTable.applyInteractiveMaxReadId(messageId, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + fileprivate func applyInteractiveReadMaxIndex(_ messageIndex: MessageIndex) -> [MessageId] { + return self.messageHistoryTable.applyInteractiveMaxReadIndex(messageIndex, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + } + + fileprivate func applyOutgoingReadMaxIndex(_ messageIndex: MessageIndex) -> [MessageId] { + return self.messageHistoryTable.applyOutgoingReadMaxIndex(messageIndex, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } private func fetchEarlierHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { @@ -742,7 +826,7 @@ public final class Postbox { return entries } - private func fetchAroundChatEntries(_ index: MessageIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?) { + private func fetchAroundChatEntries(_ index: ChatListIndex, count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?) { let (intermediateEntries, intermediateLower, intermediateUpper) = self.chatListTable.entriesAround(index, messageHistoryTable: self.messageHistoryTable, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable, count: count) var entries: [MutableChatListEntry] = [] var lower: MutableChatListEntry? @@ -784,7 +868,7 @@ public final class Postbox { return (entries, lower, upper) } - private func fetchEarlierChatEntries(_ index: MessageIndex?, count: Int) -> [MutableChatListEntry] { + private func fetchEarlierChatEntries(_ index: ChatListIndex?, count: Int) -> [MutableChatListEntry] { let intermediateEntries = self.chatListTable.earlierEntries(index, messageHistoryTable: self.messageHistoryTable, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable, count: count) var entries: [MutableChatListEntry] = [] for entry in intermediateEntries { @@ -800,7 +884,7 @@ public final class Postbox { return entries } - private func fetchLaterChatEntries(_ index: MessageIndex?, count: Int) -> [MutableChatListEntry] { + private func fetchLaterChatEntries(_ index: ChatListIndex?, count: Int) -> [MutableChatListEntry] { let intermediateEntries = self.chatListTable.laterEntries(index, messageHistoryTable: self.messageHistoryTable, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable, count: count) var entries: [MutableChatListEntry] = [] for entry in intermediateEntries { @@ -833,7 +917,9 @@ public final class Postbox { self.viewTracker.refreshViewsDueToExternalTransaction(fetchAroundChatEntries: self.fetchAroundChatEntries, fetchAroundHistoryEntries: self.fetchAroundHistoryEntries, fetchUnsentMessageIds: { return self.messageHistoryUnsentTable!.get() }, fetchSynchronizePeerReadStateOperations: { - return self.synchronizeReadStateTable!.get() + return self.synchronizeReadStateTable!.get(getCombinedPeerReadState: { peerId in + return self.readStateTable.getCombinedState(peerId) + }) }) self.transactionStateVersion = currentTransactionStateVersion @@ -852,9 +938,11 @@ public final class Postbox { let transactionUnreadCountDeltas = self.readStateTable.transactionUnreadCountDeltas() let transactionParticipationInTotalUnreadCountUpdates = self.peerNotificationSettingsTable.transactionParticipationInTotalUnreadCountUpdates() - self.chatListIndexTable.commitWithTransactionUnreadCountDeltas(transactionUnreadCountDeltas, transactionParticipationInTotalUnreadCountUpdates: transactionParticipationInTotalUnreadCountUpdates, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) + self.chatListIndexTable.commitWithTransactionUnreadCountDeltas(transactionUnreadCountDeltas, transactionParticipationInTotalUnreadCountUpdates: transactionParticipationInTotalUnreadCountUpdates, getPeer: { peerId in + return self.peerTable.get(peerId) + }, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) - let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -884,6 +972,7 @@ public final class Postbox { self.currentUpdatedPeerChatListEmbeddedStates.removeAll() self.currentUpdatedTotalUnreadCount = nil self.currentPeerMergedOperationLogOperations.removeAll() + self.currentTimestampBasedMessageAttributesOperations.removeAll() for table in self.tables { table.beforeCommit() @@ -902,6 +991,10 @@ public final class Postbox { return result } + fileprivate func messageIdForGloballyUniqueMessageId(peerId: PeerId, id: Int64) -> MessageId? { + return self.globallyUniqueMessageIdsTable.get(peerId: peerId, globallyUniqueId: id) + } + fileprivate func updatePeers(_ peers: [Peer], update: (Peer, Peer) -> Peer?) { for peer in peers { if let currentPeer = self.peerTable.get(peer.id) { @@ -1199,10 +1292,10 @@ public final class Postbox { } public func tailChatListView(_ count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { - return self.aroundChatListView(MessageIndex.absoluteUpperBound(), count: count) + return self.aroundChatListView(ChatListIndex.absoluteUpperBound, count: count) } - public func aroundChatListView(_ index: MessageIndex, count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { + public func aroundChatListView(_ index: ChatListIndex, count: Int) -> Signal<(ChatListView, ViewUpdateType), NoError> { return self.modify { modifier -> Signal<(ChatListView, ViewUpdateType), NoError> in let (entries, earlier, later) = self.fetchAroundChatEntries(index, count: count) @@ -1475,6 +1568,25 @@ public final class Postbox { } |> switchToLatest } + public func timestampBasedMessageAttributesView(tag: UInt16) -> Signal { + return self.modify { modifier -> Signal in + let view = MutableTimestampBasedMessageAttributesView(tag: tag, getHead: { tag in + return self.timestampBasedMessageAttributesTable.head(tag: tag) + }) + let (index, signal) = self.viewTracker.addTimestampBasedMessageAttributesView(view) + + return (.single(TimestampBasedMessageAttributesView(view)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeTimestampBasedMessageAttributesView(index) + } + } + } + } |> switchToLatest + } + fileprivate func operationLogGetNextEntryLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> Int32 { return self.peerOperationLogTable.getNextEntryLocalIndex(peerId: peerId, tag: tag) } @@ -1503,6 +1615,14 @@ public final class Postbox { self.peerOperationLogTable.enumerateEntries(peerId: peerId, tag: tag, f) } + fileprivate func addTimestampBasedMessageAttribute(tag: UInt16, timestamp: Int32, messageId: MessageId) { + self.timestampBasedMessageAttributesTable.set(tag: tag, id: messageId, timestamp: timestamp, operations: &self.currentTimestampBasedMessageAttributesOperations) + } + + fileprivate func removeTimestampBasedMessageAttribute(tag: UInt16, messageId: MessageId) { + self.timestampBasedMessageAttributesTable.remove(tag: tag, id: messageId, operations: &self.currentTimestampBasedMessageAttributesOperations) + } + public func isMasterClient() -> Signal { return self.modify { modifier -> Signal in let sessionClientId = self.sessionClientId diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index 94144804db..a6c85c40c1 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -13,6 +13,7 @@ final class PostboxTransaction { let currentUpdatedTotalUnreadCount: Int32? let peerIdsWithUpdatedUnreadCounts: Set let currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation] + let currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation] let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -72,10 +73,13 @@ final class PostboxTransaction { if !currentPeerMergedOperationLogOperations.isEmpty { return false } + if !currentTimestampBasedMessageAttributesOperations.isEmpty { + return false + } return true } - init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles self.removedHolesByPeerId = removedHolesByPeerId @@ -88,6 +92,7 @@ final class PostboxTransaction { self.currentUpdatedTotalUnreadCount = currentUpdatedTotalUnreadCount self.peerIdsWithUpdatedUnreadCounts = peerIdsWithUpdatedUnreadCounts self.currentPeerMergedOperationLogOperations = currentPeerMergedOperationLogOperations + self.currentTimestampBasedMessageAttributesOperations = currentTimestampBasedMessageAttributesOperations self.unsentMessageOperations = unsentMessageOperations self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations self.updatedMedia = updatedMedia diff --git a/Postbox/TimestampBasedMessageAttributesIndexTable.swift b/Postbox/TimestampBasedMessageAttributesIndexTable.swift new file mode 100644 index 0000000000..dd6f66fa92 --- /dev/null +++ b/Postbox/TimestampBasedMessageAttributesIndexTable.swift @@ -0,0 +1,35 @@ +import Foundation + +final class TimestampBasedMessageAttributesIndexTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private func key(tag: UInt16, id: MessageId) -> ValueBoxKey { + let key = ValueBoxKey(length: 2 + 4 + 8 + 4) + key.setUInt16(0, value: tag) + key.setInt64(2, value: id.peerId.toInt64()) + key.setInt32(2 + 8, value: id.namespace) + key.setInt32(2 + 8 + 4, value: id.id) + return key + } + + func set(tag: UInt16, id: MessageId, timestamp: Int32) { + var timestampValue = timestamp + self.valueBox.set(self.table, key: self.key(tag: tag, id: id), value: MemoryBuffer(memory: ×tampValue, capacity: 4, length: 4, freeWhenDone: false)) + } + + func get(tag: UInt16, id: MessageId) -> Int32? { + if let value = self.valueBox.get(self.table, key: self.key(tag: tag, id: id)) { + var timestampValue: Int32 = 0 + value.read(×tampValue, offset: 0, length: 4) + return timestampValue + } else { + return nil + } + } + + func remove(tag: UInt16, id: MessageId) { + self.valueBox.remove(self.table, key: self.key(tag: tag, id: id)) + } +} diff --git a/Postbox/TimestampBasedMessageAttributesTable.swift b/Postbox/TimestampBasedMessageAttributesTable.swift new file mode 100644 index 0000000000..0b13add854 --- /dev/null +++ b/Postbox/TimestampBasedMessageAttributesTable.swift @@ -0,0 +1,89 @@ +import Foundation + +public struct TimestampBasedMessageAttributesEntry { + public let tag: UInt16 + public let timestamp: Int32 + public let messageId: MessageId + + public var index: MessageIndex { + return MessageIndex(id: self.messageId, timestamp: timestamp) + } +} + +enum TimestampBasedMessageAttributesOperation { + case add(TimestampBasedMessageAttributesEntry) + case remove(TimestampBasedMessageAttributesEntry) +} + +final class TimestampBasedMessageAttributesTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private let indexTable: TimestampBasedMessageAttributesIndexTable + + init(valueBox: ValueBox, table: ValueBoxTable, indexTable: TimestampBasedMessageAttributesIndexTable) { + self.indexTable = indexTable + + super.init(valueBox: valueBox, table: table) + } + + private func key(tag: UInt16, timestamp: Int32, id: MessageId) -> ValueBoxKey { + let key = ValueBoxKey(length: 2 + 4 + 8 + 4 + 4) + key.setUInt16(0, value: tag) + key.setInt32(2, value: timestamp) + key.setInt64(2 + 4, value: id.peerId.toInt64()) + key.setInt32(2 + 4 + 8, value: id.namespace) + key.setInt32(2 + 4 + 8 + 4, value: id.id) + return key + } + + private func lowerBound(tag: UInt16) -> ValueBoxKey { + let key = ValueBoxKey(length: 3) + key.setUInt16(0, value: tag) + key.setInt8(2, value: 0) + return key + } + + private func upperBound(tag: UInt16) -> ValueBoxKey { + let key = ValueBoxKey(length: 3) + key.setUInt16(0, value: tag + 1) + key.setInt8(2, value: 0) + return key + } + + func set(tag: UInt16, id: MessageId, timestamp: Int32, operations: inout [TimestampBasedMessageAttributesOperation]) { + if let previousTimestamp = self.indexTable.get(tag: tag, id: id) { + if previousTimestamp == timestamp { + return + } else { + self.valueBox.remove(self.table, key: self.key(tag: tag, timestamp: previousTimestamp, id: id)) + operations.append(.remove(TimestampBasedMessageAttributesEntry(tag: tag, timestamp: previousTimestamp, messageId: id))) + } + } + self.valueBox.set(self.table, key: self.key(tag: tag, timestamp: timestamp, id: id), value: MemoryBuffer()) + self.indexTable.set(tag: tag, id: id, timestamp: timestamp) + operations.append(.add(TimestampBasedMessageAttributesEntry(tag: tag, timestamp: timestamp, messageId: id))) + } + + func remove(tag: UInt16, id: MessageId, operations: inout [TimestampBasedMessageAttributesOperation]) { + if let previousTimestamp = self.indexTable.get(tag: tag, id: id) { + self.valueBox.remove(self.table, key: self.key(tag: tag, timestamp: previousTimestamp, id: id)) + self.indexTable.remove(tag: tag, id: id) + operations.append(.remove(TimestampBasedMessageAttributesEntry(tag: tag, timestamp: previousTimestamp, messageId: id))) + } + } + + func head(tag: UInt16) -> TimestampBasedMessageAttributesEntry? { + var result: TimestampBasedMessageAttributesEntry? + self.valueBox.range(self.table, start: self.lowerBound(tag: tag), end: self.upperBound(tag: tag), keys: { key in + let timestamp = key.getInt32(2) + let idPeerId = key.getInt64(2 + 4) + let idNamespace = key.getInt32(2 + 4 + 8) + let idId = key.getInt32(2 + 4 + 8 + 4) + result = TimestampBasedMessageAttributesEntry(tag: tag, timestamp: timestamp, messageId: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId)) + return false + }, limit: 1) + return result + } +} diff --git a/Postbox/TimestampBasedMessageAttributesView.swift b/Postbox/TimestampBasedMessageAttributesView.swift new file mode 100644 index 0000000000..c8fe4a4036 --- /dev/null +++ b/Postbox/TimestampBasedMessageAttributesView.swift @@ -0,0 +1,52 @@ +import Foundation + +final class MutableTimestampBasedMessageAttributesView { + let tag: UInt16 + var head: TimestampBasedMessageAttributesEntry? + + init(tag: UInt16, getHead: (UInt16) -> TimestampBasedMessageAttributesEntry?) { + self.tag = tag + self.head = getHead(tag) + } + + func replay(operations: [TimestampBasedMessageAttributesOperation], getHead: (UInt16) -> TimestampBasedMessageAttributesEntry?) -> Bool { + var updated = false + var invalidatedHead = false + for operation in operations { + switch operation { + case let .add(entry): + if entry.tag == self.tag { + if let head = self.head { + if entry.index < head.index { + self.head = entry + updated = true + } + } else { + self.head = entry + updated = true + } + } + case let .remove(entry): + if entry.tag == self.tag { + if let head = self.head, head.index == entry.index { + self.head = nil + updated = true + invalidatedHead = true + } + } + } + } + if invalidatedHead { + self.head = getHead(self.tag) + } + return updated + } +} + +public final class TimestampBasedMessageAttributesView { + public let head: TimestampBasedMessageAttributesEntry? + + init(_ view: MutableTimestampBasedMessageAttributesView) { + self.head = view.head + } +} diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index 2099139e9b..ebb9eb280a 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -63,6 +63,11 @@ public struct ValueBoxKey: Hashable, CustomStringConvertible, Comparable { memcpy(self.memory + offset, &varValue, 1) } + public func setUInt16(_ offset: Int, value: UInt16) { + var varValue = value + memcpy(self.memory + offset, &varValue, 2) + } + public func getInt32(_ offset: Int) -> Int32 { var value: Int32 = 0 memcpy(&value, self.memory + offset, 4) @@ -93,6 +98,12 @@ public struct ValueBoxKey: Hashable, CustomStringConvertible, Comparable { return value } + public func getUInt16(_ offset: Int) -> UInt16 { + var value: UInt16 = 0 + memcpy(&value, self.memory + offset, 2) + return value + } + public func prefix(_ length: Int) -> ValueBoxKey { assert(length <= self.length, "length <= self.length") let key = ValueBoxKey(length: length) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 9489400b3c..6737eb718f 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -16,8 +16,8 @@ final class ViewTracker { private let queue: Queue private let fetchEarlierHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry] private let fetchLaterHistoryEntries: (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry] - private let fetchEarlierChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] - private let fetchLaterChatEntries: (MessageIndex?, Int) -> [MutableChatListEntry] + private let fetchEarlierChatEntries: (ChatListIndex?, Int) -> [MutableChatListEntry] + private let fetchLaterChatEntries: (ChatListIndex?, Int) -> [MutableChatListEntry] private let fetchAnchorIndex: (MessageId) -> MessageHistoryAnchorIndex? private let renderMessage: (IntermediateMessage) -> Message private let getPeer: (PeerId) -> Peer? @@ -51,7 +51,10 @@ final class ViewTracker { private var unreadMessageCountsViews = Bag<(MutableUnreadMessageCountsView, ValuePipe)>() private var peerMergedOperationLogViews = Bag<(MutablePeerMergedOperationLogView, ValuePipe)>() - init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (MessageIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, operationLogGetOperations: @escaping (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], operationLogGetTailIndex: @escaping (PeerOperationLogTag) -> Int32?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + private let getTimestampBasedMessageAttributesHead: (UInt16) -> TimestampBasedMessageAttributesEntry? + private var timestampBasedMessageAttributesViews = Bag<(MutableTimestampBasedMessageAttributesView, ValuePipe)>() + + init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, operationLogGetOperations: @escaping (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], operationLogGetTailIndex: @escaping (PeerOperationLogTag) -> Int32?, getTimestampBasedMessageAttributesHead: @escaping (UInt16) -> TimestampBasedMessageAttributesEntry?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -67,6 +70,7 @@ final class ViewTracker { self.getPeerReadState = getPeerReadState self.operationLogGetOperations = operationLogGetOperations self.operationLogGetTailIndex = operationLogGetTailIndex + self.getTimestampBasedMessageAttributesHead = getTimestampBasedMessageAttributesHead self.unsentMessageView = UnsentMessageHistoryView(ids: unsentMessageIds) self.synchronizeReadStatesView = MutableSynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) @@ -198,7 +202,18 @@ final class ViewTracker { self.peerMergedOperationLogViews.remove(index) } - func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: MessageIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { + func addTimestampBasedMessageAttributesView(_ view: MutableTimestampBasedMessageAttributesView) -> (Bag<(MutableTimestampBasedMessageAttributesView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.timestampBasedMessageAttributesViews.add(record) + + return (index, record.1.signal()) + } + + func removeTimestampBasedMessageAttributesView(_ index: Bag<(MutableTimestampBasedMessageAttributesView, ValuePipe)>.Index) { + self.timestampBasedMessageAttributesViews.remove(index) + } + + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: ChatListIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { @@ -380,6 +395,12 @@ final class ViewTracker { pipe.putNext(PeerMergedOperationLogView(mutableView)) } } + + for (mutableView, pipe) in self.timestampBasedMessageAttributesViews.copyItems() { + if mutableView.replay(operations: transaction.currentTimestampBasedMessageAttributesOperations, getHead: self.getTimestampBasedMessageAttributesHead) { + pipe.putNext(TimestampBasedMessageAttributesView(mutableView)) + } + } } private func updateTrackedChatListHoles() { diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index b7ea687151..02507ecddb 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -81,6 +81,7 @@ class ChatListTableTests: XCTestCase { var peerNameTokenIndexTable: PeerNameTokenIndexTable? var peerNameIndexTable: PeerNameIndexTable? var notificationSettingsTable: PeerNotificationSettingsTable? + var globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable? override class func setUp() { super.setUp() @@ -104,7 +105,8 @@ class ChatListTableTests: XCTestCase { self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, table: MessageMediaTable.tableSpec(2)) self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, table: MessageHistoryReadStateTable.tableSpec(11)) self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, table: MessageHistorySynchronizeReadStateTable.tableSpec(12)) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.globallyUniqueMessageIdsTable = MessageGloballyUniqueIdTable(valueBox: self.valueBox!, table: MessageGloballyUniqueIdTable.tableSpec(24)) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) self.peerTable = PeerTable(valueBox: self.valueBox!, table: PeerTable.tableSpec(20)) self.peerNameTokenIndexTable = PeerNameTokenIndexTable(valueBox: self.valueBox!, table: PeerNameTokenIndexTable.tableSpec(21)) self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox!, table: PeerNameIndexTable.tableSpec(22), peerTable: self.peerTable!, peerNameTokenIndexTable: self.peerNameTokenIndexTable!) @@ -194,7 +196,7 @@ class ChatListTableTests: XCTestCase { case let .Hole(hole): return .Hole(hole.index.id.peerId.id, hole.index.id.id, hole.index.timestamp) case let .Nothing(index): - return .Nothing(index.id.peerId.id, index.id.id, index.timestamp) + return .Nothing(index.messageIndex.id.peerId.id, index.messageIndex.id.id, index.messageIndex.timestamp) } }) if entries != actualEntries { diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index e304ac53d2..324e4fc201 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -119,6 +119,13 @@ class MessageHistoryIndexTableTests: XCTestCase { }), operations: &operations) } + func fillMultipleHoles(_ mainId: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32)], _ tagMask: MessageTags? = nil) { + var operations: [MessageHistoryIndexOperation] = [] + self.indexTable!.fillMultipleHoles(mainHoleId: MessageId(peerId: peerId, namespace: namespace, id: mainId), fillType: fillType, tagMask: tagMask, messages: messages.map({ + return InternalStoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1, globallyUniqueId: nil, flags: [], tags: [], forwardInfo: nil, authorId: peerId, text: "", attributes: [], media: []) + }), operations: &operations) + } + func removeMessage(_ id: Int32) { var operations: [MessageHistoryIndexOperation] = [] self.indexTable!.removeMessage(MessageId(peerId: peerId, namespace: namespace, id: id), operations: &operations) @@ -511,14 +518,113 @@ class MessageHistoryIndexTableTests: XCTestCase { func testFillHoleAtIndex() { addHole(1) expect([.Hole(1, Int32.max, Int32.max)]) - fillHole(1, HoleFill(complete: false, direction: .AroundIndex(MessageIndex(id: 10, timestamp: 10))), [(5, 5), (10, 10)]) + fillHole(1, HoleFill(complete: false, direction: .AroundIndex(MessageIndex(id: 10, timestamp: 10), lowerComplete: false, upperComplete: false)), [(5, 5), (10, 10)]) expect([.Hole(1, 4, 5), .Message(5, 5), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) } func testFillHoleAtIndexComplete() { addHole(1) expect([.Hole(1, Int32.max, Int32.max)]) - fillHole(1, HoleFill(complete: true, direction: .AroundIndex(MessageIndex(id: 10, timestamp: 10))), [(5, 5), (10, 10)]) + fillHole(1, HoleFill(complete: true, direction: .AroundIndex(MessageIndex(id: 10, timestamp: 10), lowerComplete: false, upperComplete: false)), [(5, 5), (10, 10)]) expect([.Message(5, 5), .Message(10, 10)]) } + + func testFillMultipleHolesSingleHole() { + addHole(1) + expect([.Hole(1, Int32.max, Int32.max)]) + + fillMultipleHoles(1, HoleFill(complete: false, direction: .UpperToLower), [(5, 5), (10, 10)]) + expect([.Hole(1, 4, 5), .Message(5, 5), .Message(10, 10)]) + } + + func testFillMultipleHolesTwoHolesUpperToLowerNotComplete() { + addHole(1) + addMessage(10, 10) + expect([.Hole(1, 9, 10), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + + fillMultipleHoles(12, HoleFill(complete: false, direction: .UpperToLower), [(8, 8), (15, 15), (20, 20)]) + + expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20)]) + } + + func testFillMultipleHolesTwoHolesUpperToLowerNotCompleteSkipOverOne() { + addHole(1) + addMessage(10, 10) + addMessage(13, 13) + expect([.Hole(1, 9, 10), .Message(10, 10), .Hole(11, 12, 13), .Message(13, 13), .Hole(14, Int32.max, Int32.max)]) + + fillMultipleHoles(20, HoleFill(complete: false, direction: .UpperToLower), [(8, 8), (15, 15), (20, 20)]) + + expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(13, 13), .Message(15, 15), .Message(20, 20)]) + } + + func testFillMultipleHolesTwoHolesUpperToLowerComplete() { + addHole(1) + addMessage(10, 10) + expect([.Hole(1, 9, 10), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + + fillMultipleHoles(12, HoleFill(complete: true, direction: .UpperToLower), [(8, 8), (15, 15), (20, 20)]) + + expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20)]) + } + + func testFillMultipleHolesTwoHolesLowerToUpperNotComplete() { + addHole(1) + addMessage(10, 10) + expect([.Hole(1, 9, 10), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + + fillMultipleHoles(12, HoleFill(complete: false, direction: .LowerToUpper), [(8, 8), (15, 15), (20, 20)]) + + expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20), .Hole(21, Int32.max, Int32.max)]) + } + + func testFillMultipleHolesTwoHolesLowerToUpperComplete() { + addHole(1) + addMessage(10, 10) + expect([.Hole(1, 9, 10), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + + fillMultipleHoles(12, HoleFill(complete: true, direction: .LowerToUpper), [(8, 8), (15, 15), (20, 20)]) + + expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20)]) + } + + func testFillMultipleHolesTwoHolesAroundIndexNotComplete() { + addHole(1) + addMessage(10, 10) + expect([.Hole(1, 9, 10), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + + fillMultipleHoles(12, HoleFill(complete: false, direction: .AroundIndex(MessageIndex(id: 15, timestamp: 15), lowerComplete: false, upperComplete: false)), [(8, 8), (15, 15), (20, 20)]) + + expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20), .Hole(21, Int32.max, Int32.max)]) + } + + func testFillMultipleHolesTwoHolesAroundIndexUpperComplete() { + addHole(1) + addMessage(10, 10) + expect([.Hole(1, 9, 10), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + + fillMultipleHoles(12, HoleFill(complete: false, direction: .AroundIndex(MessageIndex(id: 15, timestamp: 15), lowerComplete: false, upperComplete: true)), [(8, 8), (15, 15), (20, 20)]) + + expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20)]) + } + + func testFillMultipleHolesTwoHolesAroundIndexLowerComplete() { + addHole(1) + addMessage(10, 10) + expect([.Hole(1, 9, 10), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + + fillMultipleHoles(12, HoleFill(complete: false, direction: .AroundIndex(MessageIndex(id: 15, timestamp: 15), lowerComplete: true, upperComplete: false)), [(8, 8), (15, 15), (20, 20)]) + + expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20), .Hole(21, Int32.max, Int32.max)]) + } + + func testFillMultipleHolesTwoHolesAroundIndexAutoComplete() { + addHole(1) + addMessage(10, 10) + expect([.Hole(1, 9, 10), .Message(10, 10), .Hole(11, Int32.max, Int32.max)]) + + fillMultipleHoles(12, HoleFill(complete: false, direction: .AroundIndex(MessageIndex(id: 15, timestamp: 15), lowerComplete: false, upperComplete: false)), [(8, 8), (15, 15), (20, 20)]) + + expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20), .Hole(21, Int32.max, Int32.max)]) + } } diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index 74b607c9dc..6432fd8c6d 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -122,6 +122,10 @@ private class TestExternalMedia: Media { } private class TestPeer: Peer { + public let notificationSettingsPeerId: PeerId? = nil + + let associatedPeerIds: [PeerId]? = nil + var indexName: PeerIndexNameRepresentation { return .title(title: "Test", addressName: nil) } @@ -209,6 +213,7 @@ class MessageHistoryTableTests: XCTestCase { var tagsTable: MessageHistoryTagsTable? var readStateTable: MessageHistoryReadStateTable? var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? + var globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable? override class func setUp() { super.setUp() @@ -236,7 +241,8 @@ class MessageHistoryTableTests: XCTestCase { self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, table: MessageMediaTable.tableSpec(2)) self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, table: MessageHistoryReadStateTable.tableSpec(10)) self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, table: MessageHistorySynchronizeReadStateTable.tableSpec(11)) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.globallyUniqueMessageIdsTable = MessageGloballyUniqueIdTable(valueBox: self.valueBox!, table: MessageGloballyUniqueIdTable.tableSpec(12)) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) self.peerTable = PeerTable(valueBox: self.valueBox!, table: PeerTable.tableSpec(6)) self.peerTable!.set(peer) } @@ -290,6 +296,13 @@ class MessageHistoryTableTests: XCTestCase { self.historyTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), globallyUniqueId: nil, timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } + private func fillMultipleHoles(_ id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32, String, [Media])], _ tagMask: MessageTags? = nil, _ tags: MessageTags = []) { + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + self.historyTable!.fillMultipleHoles(mainHoleId: MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, tagMask: tagMask, messages: messages.map({ StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), globallyUniqueId: nil, timestamp: $0.1, flags: [], tags: tags, forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + private func expectEntries(_ entries: [Entry], tagMask: MessageTags? = nil) { var stableIds = Set() @@ -961,4 +974,57 @@ class MessageHistoryTableTests: XCTestCase { expectEntries([.Hole(101, 199, 200), .Message(200, 200, "m200", [], [])], tagMask: [.Second]) expectEntries([.Message(100, 100, "m100", [], []), .Hole(101, 199, 200), .Message(200, 200, "m200", [], [])]) } + + func testTagsFillMultipleHolesSingleHole() { + addHole(1) + addMessage(100, 100, "m100", [], [], [.First]) + addMessage(200, 200, "m200", [], [], [.First]) + addMessage(300, 300, "m300", [], [], [.First]) + addMessage(400, 400, "m400", [], [], [.First]) + + expectEntries([ + .Hole(1, 99, 100), + .Message(100, 100, "m100", [], []), + .Hole(101, 199, 200), + .Message(200, 200, "m200", [], []), + .Hole(201, 299, 300), + .Message(300, 300, "m300", [], []), + .Hole(301, 399, 400), + .Message(400, 400, "m400", [], []), + .Hole(401, Int32.max, Int32.max) + ], tagMask: [.First]) + + expectEntries([ + .Hole(1, 99, 100), + .Hole(101, 199, 200), + .Hole(201, 299, 300), + .Hole(301, 399, 400), + .Hole(401, Int32.max, Int32.max) + ], tagMask: [.Second]) + + fillMultipleHoles(500, HoleFill(complete: false, direction: .UpperToLower), [(500, 500, "m500", []), (350, 350, "m350", [])], [.Second], [.Second]) + + expectEntries([ + .Hole(1, 99, 100), + .Hole(101, 199, 200), + .Hole(201, 299, 300), + .Hole(301, 349, 350), + .Message(350, 350, "m350", [], []), + .Message(500, 500, "m500", [], []) + ], tagMask: [.Second]) + + expectEntries([ + .Hole(1, 99, 100), + .Message(100, 100, "m100", [], []), + .Hole(101, 199, 200), + .Message(200, 200, "m200", [], []), + .Hole(201, 299, 300), + .Message(300, 300, "m300", [], []), + .Hole(301, 349, 350), + .Hole(351, 399, 400), + .Message(400, 400, "m400", [], []), + .Hole(401, 499, 500), + .Hole(501, Int32.max, Int32.max) + ], tagMask: [.First]) + } } diff --git a/PostboxTests/OrderStatisticTreeTests.swift b/PostboxTests/OrderStatisticTreeTests.swift index 3e40008b8b..d5d0c040ca 100644 --- a/PostboxTests/OrderStatisticTreeTests.swift +++ b/PostboxTests/OrderStatisticTreeTests.swift @@ -77,6 +77,7 @@ class OrderStatisticTreeTests: XCTestCase { var tagsTable: MessageHistoryTagsTable? var readStateTable: MessageHistoryReadStateTable? var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? + var globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable? override func setUp() { super.setUp() @@ -96,7 +97,8 @@ class OrderStatisticTreeTests: XCTestCase { self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, table: MessageMediaTable.tableSpec(2)) self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, table: MessageHistoryReadStateTable.tableSpec(10)) self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, table: MessageHistorySynchronizeReadStateTable.tableSpec(11)) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.globallyUniqueMessageIdsTable = MessageGloballyUniqueIdTable(valueBox: self.valueBox!, table: MessageGloballyUniqueIdTable.tableSpec(12)) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) } override func tearDown() { diff --git a/PostboxTests/RandomAccessResourceTests.swift b/PostboxTests/RandomAccessResourceTests.swift index b0f72a3065..83dd893c70 100644 --- a/PostboxTests/RandomAccessResourceTests.swift +++ b/PostboxTests/RandomAccessResourceTests.swift @@ -152,7 +152,7 @@ class RandomAccessResourceTests: XCTestCase { XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) } - func testIncrementalStoreCompleteSubscriptionAligned() { + /*func testIncrementalStoreCompleteSubscriptionAligned() { let path = randomFilePath() let size = 10 * 1024 * 1024 @@ -222,11 +222,11 @@ class RandomAccessResourceTests: XCTestCase { invocations += 1 }) let _ = context.addListenerForFetchedData(in: Range(range)) - XCTAssert(invocations == 1) + //XCTAssert(invocations == 1) } XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) - } + }*/ func testIncrementalStoreCompleteSubscriptionUnaligned() { let path = randomFilePath() diff --git a/PostboxTests/ReadStateTableTests.swift b/PostboxTests/ReadStateTableTests.swift index 59e84ad7ac..7e9ac3482b 100644 --- a/PostboxTests/ReadStateTableTests.swift +++ b/PostboxTests/ReadStateTableTests.swift @@ -77,6 +77,7 @@ class ReadStateTableTests: XCTestCase { var tagsTable: MessageHistoryTagsTable? var readStateTable: MessageHistoryReadStateTable? var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? + var globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable? override func setUp() { super.setUp() @@ -96,7 +97,8 @@ class ReadStateTableTests: XCTestCase { self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, table: MessageMediaTable.tableSpec(2)) self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox!, table: MessageHistoryReadStateTable.tableSpec(10)) self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox!, table: MessageHistorySynchronizeReadStateTable.tableSpec(11)) - self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) + self.globallyUniqueMessageIdsTable = MessageGloballyUniqueIdTable(valueBox: self.valueBox!, table: MessageGloballyUniqueIdTable.tableSpec(12)) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) } override func tearDown() { @@ -154,8 +156,13 @@ class ReadStateTableTests: XCTestCase { private func expectReadState(_ maxReadId: Int32, _ maxKnownId: Int32, _ count: Int32) { if let state = self.readStateTable!.getCombinedState(peerId)?.states.first?.1 { - if state.maxIncomingReadId != maxReadId || state.maxKnownId != maxKnownId || state.count != count { - XCTFail("Expected\nmaxIncomingReadId: \(maxReadId), maxKnownId: \(maxKnownId), count: \(count)\nActual\nmaxIncomingReadId: \(state.maxIncomingReadId), maxKnownId: \(state.maxKnownId), count: \(state.count)") + switch state { + case let .idBased(maxIncomingReadId, maxOutgoingReadId, stateMaxKnownId, stateCount): + if maxIncomingReadId != maxReadId || stateMaxKnownId != maxKnownId || stateCount != count { + XCTFail("Expected\nmaxIncomingReadId: \(maxReadId), maxKnownId: \(maxKnownId), count: \(count)\nActual\nmaxIncomingReadId: \(maxIncomingReadId), maxKnownId: \(stateMaxKnownId), count: \(stateCount)") + } + case .indexBased: + XCTFail() } } else { XCTFail("Expected\nmaxReadId: \(maxReadId), maxKnownId: \(maxKnownId), count: \(count)\nActual\nnil") @@ -165,7 +172,7 @@ class ReadStateTableTests: XCTestCase { func testResetState() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: .idBased(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) expectReadState(100, 120, 130) } @@ -173,7 +180,7 @@ class ReadStateTableTests: XCTestCase { func testAddIncomingBeforeKnown() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: .idBased(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.addMessage(99, 99, "", [], [.Incoming]) @@ -183,7 +190,7 @@ class ReadStateTableTests: XCTestCase { func testAddIncomingAfterKnown() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: .idBased(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 120, count: 130)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.addMessage(130, 130, "", [], [.Incoming]) @@ -193,7 +200,7 @@ class ReadStateTableTests: XCTestCase { func testApplyReadThenAddIncoming() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: .idBased(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.expectApplyRead(200, false) @@ -205,7 +212,7 @@ class ReadStateTableTests: XCTestCase { func testApplyAddIncomingThenRead() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: .idBased(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.addMessage(130, 130, "", [], [.Incoming]) @@ -219,7 +226,7 @@ class ReadStateTableTests: XCTestCase { func testIgnoreOldRead() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: .idBased(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.expectApplyRead(90, false) @@ -229,7 +236,7 @@ class ReadStateTableTests: XCTestCase { func testInvalidateReadHole() { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] - self.historyTable!.resetIncomingReadStates([peerId: [namespace: PeerReadState(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.historyTable!.resetIncomingReadStates([peerId: [namespace: .idBased(maxIncomingReadId: 100, maxOutgoingReadId: 0, maxKnownId: 100, count: 0)]], operationsByPeerId: &operationsByPeerId, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) self.addMessage(200, 200) self.addHole(1) From 4f3d14645db0c16a7ba0daee3bc65e4e8dcf61a4 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 Jan 2017 10:44:38 +0300 Subject: [PATCH 073/427] no message --- Postbox/Message.swift | 6 +++++- Postbox/MessageHistoryTable.swift | 2 +- Postbox/Postbox.swift | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 40d41a466a..16e3951c7a 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -330,7 +330,7 @@ public extension MessageAttribute { } } -public final class Message { +public final class Message: CustomStringConvertible { public let stableId: UInt32 public let stableVersion: UInt32 @@ -365,6 +365,10 @@ public final class Message { self.associatedMessages = associatedMessages self.associatedMessageIds = associatedMessageIds } + + public var description: String { + return "Message(stableId: \(self.stableId), id: \(self.id), text: \(self.text))" + } } public struct StoreMessageFlags: OptionSet { diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 93ef4e30cf..b26e9a6e20 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -215,7 +215,7 @@ final class MessageHistoryTable: Table { processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) if let message = self.justUpdate(index, message: storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder, unsentMessageOperations: &unsentMessageOperations) { - outputOperations.append(.Remove(accumulatedRemoveIndices)) + outputOperations.append(.Remove([index])) outputOperations.append(.InsertMessage(message)) if message.flags.contains(.Incoming) { diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 5c134dfdc2..3be0572d2f 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -419,7 +419,7 @@ public final class Postbox { //self.debugRestoreState("afterLogin") //self.debugSaveState(name: "previous") - self.debugRestoreState(name: "previous") + //self.debugRestoreState(name: "previous") //#endif From 101585d8143c65399d2640a40af324a2d332e532 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 Jan 2017 13:48:57 +0300 Subject: [PATCH 074/427] no message --- Postbox/MessageHistoryIndexTable.swift | 26 ++++++++++-- Postbox/MessageHistoryView.swift | 42 ++++++++++++++++++- Postbox/Postbox.swift | 2 +- .../MessageHistoryIndexTableTests.swift | 2 +- 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 71e9343341..001bceaa31 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -415,14 +415,35 @@ final class MessageHistoryIndexTable: Table { switch fillType.direction { case .LowerToUpper: currentLowerBound = min(currentLowerBound, upperHole.min) + if fillType.complete { + currentUpperBound = Int32.max + } case .UpperToLower: currentUpperBound = max(currentUpperBound, upperHole.maxIndex.id.id) + if fillType.complete { + currentLowerBound = 1 + } case .AroundIndex: break } filledLowerBound = currentLowerBound filledUpperBound = currentUpperBound + } else { + switch fillType.direction { + case .LowerToUpper: + filledLowerBound = upperHole.min + if fillType.complete { + filledUpperBound = Int32.max + } + case .UpperToLower: + filledUpperBound = upperHole.maxIndex.id.id + if fillType.complete { + filledLowerBound = 1 + } + case .AroundIndex: + break + } } } } @@ -438,10 +459,7 @@ final class MessageHistoryIndexTable: Table { var remainingMessages: [InternalStoreMessage] = [] - if !sortedByIdMessages.isEmpty { - let lowestMessageId = filledLowerBound! - let highestMessageId = filledUpperBound! - + if let lowestMessageId = filledLowerBound, let highestMessageId = filledUpperBound { self.valueBox.range(self.table, start: self.key(MessageId(peerId: mainHoleId.peerId, namespace: mainHoleId.namespace, id: lowestMessageId)), end: self.key(MessageId(peerId: mainHoleId.peerId, namespace: mainHoleId.namespace, id: highestMessageId)), values: { key, value in let item = readHistoryIndexEntry(peerId, namespace: mainHoleId.namespace, key: key, value: value) if case let .Hole(itemHole) = item { diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index b60432eb93..ac9dc7aebc 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -565,7 +565,35 @@ final class MutableMessageHistoryView { func complete(context: MutableMessageHistoryViewReplayContext, fetchEarlier: (MessageIndex?, Int) -> [MutableMessageHistoryEntry], fetchLater: (MessageIndex?, Int) -> [MutableMessageHistoryEntry]) { if context.removedEntries && self.entries.count < self.fillCount { if self.entries.count == 0 { - let anchorIndex = (self.later ?? self.earlier)?.index + var anchorIndex: MessageIndex? + + if context.invalidLater { + var laterId: MessageIndex? + let i = self.entries.count - 1 + if i >= 0 { + laterId = self.entries[i].index + } + + let laterEntries = fetchLater(laterId, 1) + anchorIndex = laterEntries.first?.index + } else { + anchorIndex = self.later?.index + } + + if anchorIndex == nil { + if context.invalidEarlier { + var earlyId: MessageIndex? + let i = 0 + if i < self.entries.count { + earlyId = self.entries[i].index + } + + let earlierEntries = fetchEarlier(earlyId, 1) + anchorIndex = earlierEntries.first?.index + } else { + anchorIndex = self.earlier?.index + } + } let fetchedEntries = fetchEarlier(anchorIndex, self.fillCount + 2) if fetchedEntries.count >= self.fillCount + 2 { @@ -574,6 +602,18 @@ final class MutableMessageHistoryView { self.entries.append(fetchedEntries[i]) } self.later = fetchedEntries.first + } else if fetchedEntries.count >= self.fillCount + 1 { + self.earlier = fetchedEntries.last + for i in (1 ..< fetchedEntries.count).reversed() { + self.entries.append(fetchedEntries[i]) + } + self.later = nil + } else { + for i in (0 ..< fetchedEntries.count).reversed() { + self.entries.append(fetchedEntries[i]) + } + self.earlier = nil + self.later = nil } } else { let fetchedEntries = fetchEarlier(self.entries[0].index, self.fillCount - self.entries.count) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 3be0572d2f..5c134dfdc2 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -419,7 +419,7 @@ public final class Postbox { //self.debugRestoreState("afterLogin") //self.debugSaveState(name: "previous") - //self.debugRestoreState(name: "previous") + self.debugRestoreState(name: "previous") //#endif diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index 324e4fc201..6935684750 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -565,7 +565,7 @@ class MessageHistoryIndexTableTests: XCTestCase { fillMultipleHoles(12, HoleFill(complete: true, direction: .UpperToLower), [(8, 8), (15, 15), (20, 20)]) - expect([.Hole(1, 7, 8), .Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20)]) + expect([.Message(8, 8), .Message(10, 10), .Message(15, 15), .Message(20, 20)]) } func testFillMultipleHolesTwoHolesLowerToUpperNotComplete() { From 99883e6998edbc24117aa2514f03bbd2660050cb Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 24 Jan 2017 19:42:03 +0300 Subject: [PATCH 075/427] no message --- Postbox.xcodeproj/project.pbxproj | 12 ++++++++ Postbox/ContactTable.swift | 4 +-- Postbox/MediaResource.swift | 5 ++++ Postbox/MessageView.swift | 48 +++++++++++++++++++++++++++++++ Postbox/PeerChatListPolicy.swift | 7 +++++ Postbox/Postbox.swift | 21 ++++++++++++-- Postbox/ViewTracker.swift | 22 ++++++++++++++ 7 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 Postbox/MessageView.swift create mode 100644 Postbox/PeerChatListPolicy.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index a1c3c11fd5..585e8e157e 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -17,6 +17,10 @@ D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */; }; D0079F651D5A457A00A27A2C /* ContactPeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F641D5A457A00A27A2C /* ContactPeersView.swift */; }; D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; + D00C7CD41E365C4E0080C3D5 /* PeerChatListPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CD31E365C4E0080C3D5 /* PeerChatListPolicy.swift */; }; + D00C7CD51E365C4E0080C3D5 /* PeerChatListPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CD31E365C4E0080C3D5 /* PeerChatListPolicy.swift */; }; + D00C7CE31E37861C0080C3D5 /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CE21E37861C0080C3D5 /* MessageView.swift */; }; + D00C7CE41E37861C0080C3D5 /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CE21E37861C0080C3D5 /* MessageView.swift */; }; D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; D010B61A1E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */; }; D019B1CF1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */; }; @@ -226,6 +230,8 @@ D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactPeerIdsView.swift; sourceTree = ""; }; D0079F641D5A457A00A27A2C /* ContactPeersView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactPeersView.swift; sourceTree = ""; }; D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameIndexRepresentation.swift; sourceTree = ""; }; + D00C7CD31E365C4E0080C3D5 /* PeerChatListPolicy.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatListPolicy.swift; sourceTree = ""; }; + D00C7CE21E37861C0080C3D5 /* MessageView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageView.swift; sourceTree = ""; }; D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerMergedOperationLogIndexTable.swift; sourceTree = ""; }; D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageGloballyUniqueIdTable.swift; sourceTree = ""; }; @@ -484,6 +490,7 @@ D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */, D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */, D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */, + D00C7CD31E365C4E0080C3D5 /* PeerChatListPolicy.swift */, ); name = Objects; sourceTree = ""; @@ -530,6 +537,7 @@ D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */, D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */, D0AAD1B41E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift */, + D00C7CE21E37861C0080C3D5 /* MessageView.swift */, ); name = Views; sourceTree = ""; @@ -802,6 +810,7 @@ D0F7B1CE1E045C6A007EB8A5 /* MediaCleanupTable.swift in Sources */, D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */, D0B4184D1D7DFE20004562A4 /* ContactPeersView.swift in Sources */, + D00C7CE41E37861C0080C3D5 /* MessageView.swift in Sources */, D0F7B1CC1E045C6A007EB8A5 /* ChatListIndexTable.swift in Sources */, D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */, D0F7B1D31E045C6A007EB8A5 /* CachedPeerDataTable.swift in Sources */, @@ -841,6 +850,7 @@ D073CE7D1DCBF3B4007511FD /* PeerPresence.swift in Sources */, D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, D073CE751DCBF3B4007511FD /* Message.swift in Sources */, + D00C7CD51E365C4E0080C3D5 /* PeerChatListPolicy.swift in Sources */, D0F7B1CB1E045C6A007EB8A5 /* ChatListTable.swift in Sources */, D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */, D0F7B1CA1E045C6A007EB8A5 /* MessageHistoryUnsentTable.swift in Sources */, @@ -888,6 +898,7 @@ D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, D044CA2A1C617D39002160FF /* SeedConfiguration.swift in Sources */, D07CFF811DCA765D00761F81 /* PeerChatInterfaceState.swift in Sources */, + D00C7CE31E37861C0080C3D5 /* MessageView.swift in Sources */, D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */, D021E0D81DB4FD1300C6B04F /* ItemCollectionItemTable.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, @@ -929,6 +940,7 @@ D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */, D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, + D00C7CD41E365C4E0080C3D5 /* PeerChatListPolicy.swift in Sources */, D0A18D671E16874D004C6734 /* UnreadMessageCountsView.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */, diff --git a/Postbox/ContactTable.swift b/Postbox/ContactTable.swift index 0019e40e78..f9b8d05fab 100644 --- a/Postbox/ContactTable.swift +++ b/Postbox/ContactTable.swift @@ -2,7 +2,7 @@ import Foundation final class ContactTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { - return ValueBoxTable(id: id, keyType: .int64) + return ValueBoxTable(id: id, keyType: .binary) } private var originalPeerIds: Set? @@ -32,7 +32,7 @@ final class ContactTable: Table { self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), keys: { key in peerIds.insert(PeerId(key.getInt64(0))) return true - }, limit: 0) + }, limit: 0) self.peerIds = peerIds self.originalPeerIds = peerIds return peerIds.contains(peerId) diff --git a/Postbox/MediaResource.swift b/Postbox/MediaResource.swift index 2b31114fb4..f204bc2441 100644 --- a/Postbox/MediaResource.swift +++ b/Postbox/MediaResource.swift @@ -25,12 +25,17 @@ public struct WrappedMediaResourceId: Hashable { public protocol MediaResource { var id: MediaResourceId { get } var size: Int? { get } + var streamable: Bool { get } } public extension MediaResource { var size: Int? { return nil } + + var streamable: Bool { + return false + } } public protocol CachedMediaResourceRepresentation { diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift new file mode 100644 index 0000000000..3436e183bb --- /dev/null +++ b/Postbox/MessageView.swift @@ -0,0 +1,48 @@ +import Foundation + +final class MutableMessageView { + let messageId: MessageId + var message: Message? + + init(messageId: MessageId, message: Message?) { + self.messageId = messageId + self.message = message + } + + func replay(_ operations: [MessageHistoryOperation], updatedMedia: [MediaId: Media?], renderIntermediateMessage: (IntermediateMessage) -> Message) -> Bool { + var updated = false + for operation in operations { + switch operation { + case let .Remove(indices): + if let message = self.message { + if indices.contains(MessageIndex(message)) { + self.message = nil + updated = true + } + } + case let .InsertMessage(message): + if message.id == self.messageId { + self.message = renderIntermediateMessage(message) + updated = true + } + case let .UpdateEmbeddedMedia(index, embeddedMedia): + break + case let .UpdateTimestamp(index, timestamp): + break + default: + break + } + } + return updated + } +} + +public final class MessageView { + public let messageId: MessageId + public let message: Message? + + init(_ view: MutableMessageView) { + self.messageId = view.messageId + self.message = view.message + } +} diff --git a/Postbox/PeerChatListPolicy.swift b/Postbox/PeerChatListPolicy.swift new file mode 100644 index 0000000000..be5b3645dc --- /dev/null +++ b/Postbox/PeerChatListPolicy.swift @@ -0,0 +1,7 @@ +import Foundation + +public enum PeerChatListInclusion { + case never + case ifNotEmpty + case always(minTimestamp: Int32) +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 5c134dfdc2..ee95adb0af 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -419,7 +419,7 @@ public final class Postbox { //self.debugRestoreState("afterLogin") //self.debugSaveState(name: "previous") - self.debugRestoreState(name: "previous") + //self.debugRestoreState(name: "previous") //#endif @@ -428,7 +428,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 9 + let currentUserVersion: Int32 = 10 if userVersion != currentUserVersion { self.valueBox.drop() @@ -1623,6 +1623,23 @@ public final class Postbox { self.timestampBasedMessageAttributesTable.remove(tag: tag, id: messageId, operations: &self.currentTimestampBasedMessageAttributesOperations) } + public func messageView(_ messageId: MessageId) -> Signal { + return self.modify { modifier -> Signal in + let view = MutableMessageView(messageId: messageId, message: modifier.getMessage(messageId)) + let (index, signal) = self.viewTracker.addMessageView(view) + + return (.single(MessageView(view)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeMessageView(index) + } + } + } + } |> switchToLatest + } + public func isMasterClient() -> Signal { return self.modify { modifier -> Signal in let sessionClientId = self.sessionClientId diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 6737eb718f..1009775888 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -54,6 +54,8 @@ final class ViewTracker { private let getTimestampBasedMessageAttributesHead: (UInt16) -> TimestampBasedMessageAttributesEntry? private var timestampBasedMessageAttributesViews = Bag<(MutableTimestampBasedMessageAttributesView, ValuePipe)>() + private var messageViews = Bag<(MutableMessageView, ValuePipe)>() + init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, operationLogGetOperations: @escaping (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], operationLogGetTailIndex: @escaping (PeerOperationLogTag) -> Int32?, getTimestampBasedMessageAttributesHead: @escaping (UInt16) -> TimestampBasedMessageAttributesEntry?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries @@ -213,6 +215,17 @@ final class ViewTracker { self.timestampBasedMessageAttributesViews.remove(index) } + func addMessageView(_ view: MutableMessageView) -> (Bag<(MutableMessageView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.messageViews.add(record) + + return (index, record.1.signal()) + } + + func removeMessageView(_ index: Bag<(MutableMessageView, ValuePipe)>.Index) { + self.messageViews.remove(index) + } + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: ChatListIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] @@ -332,6 +345,15 @@ final class ViewTracker { } } + for (mutableView, pipe) in self.messageViews.copyItems() { + let operations = transaction.currentOperationsByPeerId[mutableView.messageId.peerId] + if operations != nil || !transaction.updatedMedia.isEmpty || !transaction.currentUpdatedCachedPeerData.isEmpty { + if mutableView.replay(operations ?? [], updatedMedia: transaction.updatedMedia, renderIntermediateMessage: self.renderMessage) { + pipe.putNext(MessageView(mutableView)) + } + } + } + if !transaction.chatListOperations.isEmpty || !transaction.currentUpdatedPeerNotificationSettings.isEmpty { for (mutableView, pipe) in self.chatListViews.copyItems() { let context = MutableChatListViewReplayContext() From 3da3ebe3e2079ea55c62abba698cd8be3e75331a Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 31 Jan 2017 00:28:16 +0300 Subject: [PATCH 076/427] no message --- Postbox.xcodeproj/project.pbxproj | 34 ++- Postbox/ContactPeersView.swift | 5 +- Postbox/MultiplePeersView.swift | 52 ++++ ...licy.swift => PeerChatListInclusion.swift} | 2 +- Postbox/PeerNotificationSettingsTable.swift | 22 ++ Postbox/Postbox.swift | 91 ++++++- Postbox/PostboxTransaction.swift | 7 +- Postbox/PreferencesEntry.swift | 5 + Postbox/PreferencesTable.swift | 58 +++++ Postbox/PreferencesView.swift | 55 +++++ Postbox/SqliteValueBox.swift | 230 ++++++++++++------ Postbox/ValueBox.swift | 4 +- Postbox/ValueBoxKey.swift | 12 + Postbox/ViewTracker.swift | 81 ++++-- 14 files changed, 540 insertions(+), 118 deletions(-) create mode 100644 Postbox/MultiplePeersView.swift rename Postbox/{PeerChatListPolicy.swift => PeerChatListInclusion.swift} (82%) create mode 100644 Postbox/PreferencesEntry.swift create mode 100644 Postbox/PreferencesTable.swift create mode 100644 Postbox/PreferencesView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 585e8e157e..c8b5922ab2 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -17,8 +17,8 @@ D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */; }; D0079F651D5A457A00A27A2C /* ContactPeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F641D5A457A00A27A2C /* ContactPeersView.swift */; }; D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */; }; - D00C7CD41E365C4E0080C3D5 /* PeerChatListPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CD31E365C4E0080C3D5 /* PeerChatListPolicy.swift */; }; - D00C7CD51E365C4E0080C3D5 /* PeerChatListPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CD31E365C4E0080C3D5 /* PeerChatListPolicy.swift */; }; + D00C7CD41E365C4E0080C3D5 /* PeerChatListInclusion.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CD31E365C4E0080C3D5 /* PeerChatListInclusion.swift */; }; + D00C7CD51E365C4E0080C3D5 /* PeerChatListInclusion.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CD31E365C4E0080C3D5 /* PeerChatListInclusion.swift */; }; D00C7CE31E37861C0080C3D5 /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CE21E37861C0080C3D5 /* MessageView.swift */; }; D00C7CE41E37861C0080C3D5 /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00C7CE21E37861C0080C3D5 /* MessageView.swift */; }; D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; @@ -75,6 +75,12 @@ D07CFF811DCA765D00761F81 /* PeerChatInterfaceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */; }; D07CFF831DCA909100761F81 /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */; }; + D08775001E3E3D9F00A97350 /* PreferencesTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08774FF1E3E3D9F00A97350 /* PreferencesTable.swift */; }; + D08775011E3E3D9F00A97350 /* PreferencesTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08774FF1E3E3D9F00A97350 /* PreferencesTable.swift */; }; + D08775031E3E3E7400A97350 /* PreferencesEntry.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08775021E3E3E7400A97350 /* PreferencesEntry.swift */; }; + D08775041E3E3E7400A97350 /* PreferencesEntry.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08775021E3E3E7400A97350 /* PreferencesEntry.swift */; }; + D08775061E3E3F2100A97350 /* PreferencesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08775051E3E3F2100A97350 /* PreferencesView.swift */; }; + D08775071E3E3F2100A97350 /* PreferencesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08775051E3E3F2100A97350 /* PreferencesView.swift */; }; D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; @@ -138,6 +144,7 @@ D0B844071DAB91B5005F29E1 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerPresence.swift */; }; + D0BC386C1E3FCEE50044D6FE /* MultiplePeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; @@ -230,7 +237,7 @@ D0079F621D5A242500A27A2C /* ContactPeerIdsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactPeerIdsView.swift; sourceTree = ""; }; D0079F641D5A457A00A27A2C /* ContactPeersView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactPeersView.swift; sourceTree = ""; }; D0079F6A1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameIndexRepresentation.swift; sourceTree = ""; }; - D00C7CD31E365C4E0080C3D5 /* PeerChatListPolicy.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatListPolicy.swift; sourceTree = ""; }; + D00C7CD31E365C4E0080C3D5 /* PeerChatListInclusion.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatListInclusion.swift; sourceTree = ""; }; D00C7CE21E37861C0080C3D5 /* MessageView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageView.swift; sourceTree = ""; }; D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerMergedOperationLogIndexTable.swift; sourceTree = ""; }; @@ -272,6 +279,9 @@ D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatInterfaceState.swift; sourceTree = ""; }; D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatInterfaceStateTable.swift; sourceTree = ""; }; D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = InitialMessageHistoryData.swift; sourceTree = ""; }; + D08774FF1E3E3D9F00A97350 /* PreferencesTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PreferencesTable.swift; sourceTree = ""; }; + D08775021E3E3E7400A97350 /* PreferencesEntry.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PreferencesEntry.swift; sourceTree = ""; }; + D08775051E3E3F2100A97350 /* PreferencesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PreferencesView.swift; sourceTree = ""; }; D08C71391C501F0700779C0F /* MessageHistoryHole.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHole.swift; sourceTree = ""; }; D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTable.swift; sourceTree = ""; }; D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTable.swift; sourceTree = ""; }; @@ -298,6 +308,7 @@ D0B418161D7DFAF3004562A4 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKitMac.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-diblohvjozhgaifjcniwdlixlilx/Build/Products/Debug/SwiftSignalKitMac.framework"; sourceTree = ""; }; D0B844501DAC04FE005F29E1 /* PeerPresence.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerPresence.swift; sourceTree = ""; }; + D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MultiplePeersView.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; @@ -470,6 +481,7 @@ D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */, D0AAD1B11E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift */, D0AAD1B01E3266B100D5B9DE /* TimestampBasedMessageAttributesIndexTable.swift */, + D08774FF1E3E3D9F00A97350 /* PreferencesTable.swift */, ); name = Tables; sourceTree = ""; @@ -490,7 +502,8 @@ D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */, D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */, D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */, - D00C7CD31E365C4E0080C3D5 /* PeerChatListPolicy.swift */, + D00C7CD31E365C4E0080C3D5 /* PeerChatListInclusion.swift */, + D08775021E3E3E7400A97350 /* PreferencesEntry.swift */, ); name = Objects; sourceTree = ""; @@ -538,6 +551,8 @@ D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */, D0AAD1B41E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift */, D00C7CE21E37861C0080C3D5 /* MessageView.swift */, + D08775051E3E3F2100A97350 /* PreferencesView.swift */, + D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */, ); name = Views; sourceTree = ""; @@ -811,6 +826,7 @@ D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */, D0B4184D1D7DFE20004562A4 /* ContactPeersView.swift in Sources */, D00C7CE41E37861C0080C3D5 /* MessageView.swift in Sources */, + D08775011E3E3D9F00A97350 /* PreferencesTable.swift in Sources */, D0F7B1CC1E045C6A007EB8A5 /* ChatListIndexTable.swift in Sources */, D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */, D0F7B1D31E045C6A007EB8A5 /* CachedPeerDataTable.swift in Sources */, @@ -818,6 +834,7 @@ D0F7B1D51E045C6A007EB8A5 /* PeerChatStateTable.swift in Sources */, D0F7B1C51E045C6A007EB8A5 /* MessageHistoryMetadataTable.swift in Sources */, D073CE781DCBF3B4007511FD /* MessageHistoryHole.swift in Sources */, + D08775071E3E3F2100A97350 /* PreferencesView.swift in Sources */, D0AAD1B61E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift in Sources */, D0F7B1C31E045C6A007EB8A5 /* Table.swift in Sources */, D0F7B1CF1E045C6A007EB8A5 /* MetadataTable.swift in Sources */, @@ -850,7 +867,7 @@ D073CE7D1DCBF3B4007511FD /* PeerPresence.swift in Sources */, D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, D073CE751DCBF3B4007511FD /* Message.swift in Sources */, - D00C7CD51E365C4E0080C3D5 /* PeerChatListPolicy.swift in Sources */, + D00C7CD51E365C4E0080C3D5 /* PeerChatListInclusion.swift in Sources */, D0F7B1CB1E045C6A007EB8A5 /* ChatListTable.swift in Sources */, D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */, D0F7B1CA1E045C6A007EB8A5 /* MessageHistoryUnsentTable.swift in Sources */, @@ -860,6 +877,7 @@ D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */, D0B418581D7DFE29004562A4 /* SeedConfiguration.swift in Sources */, D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */, + D08775041E3E3E7400A97350 /* PreferencesEntry.swift in Sources */, D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */, D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */, D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, @@ -878,6 +896,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + D08775061E3E3F2100A97350 /* PreferencesView.swift in Sources */, D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */, D08D451F1D5D2CA700A7428A /* RatingTable.swift in Sources */, @@ -937,10 +956,11 @@ D0F019FD1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift in Sources */, D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, + D0BC386C1E3FCEE50044D6FE /* MultiplePeersView.swift in Sources */, D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */, D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, - D00C7CD41E365C4E0080C3D5 /* PeerChatListPolicy.swift in Sources */, + D00C7CD41E365C4E0080C3D5 /* PeerChatListInclusion.swift in Sources */, D0A18D671E16874D004C6734 /* UnreadMessageCountsView.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */, @@ -951,7 +971,9 @@ D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */, + D08775031E3E3E7400A97350 /* PreferencesEntry.swift in Sources */, D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */, + D08775001E3E3D9F00A97350 /* PreferencesTable.swift in Sources */, D03120F81DA53FF4006A2A60 /* PeerPresenceTable.swift in Sources */, D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */, D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */, diff --git a/Postbox/ContactPeersView.swift b/Postbox/ContactPeersView.swift index 379cc89719..c7342a67e9 100644 --- a/Postbox/ContactPeersView.swift +++ b/Postbox/ContactPeersView.swift @@ -1,14 +1,12 @@ import Foundation final class MutableContactPeersView { - fileprivate let index: PeerNameIndex fileprivate var peers: [PeerId: Peer] fileprivate var peerPresences: [PeerId: PeerPresence] fileprivate var peerIds: Set fileprivate var accountPeer: Peer? - init(peers: [PeerId: Peer], peerPresences: [PeerId: PeerPresence], index: PeerNameIndex, accountPeer: Peer?) { - self.index = index + init(peers: [PeerId: Peer], peerPresences: [PeerId: PeerPresence], accountPeer: Peer?) { self.peers = peers self.peerIds = Set(peers.map { $0.0 }) self.peerPresences = peerPresences @@ -59,7 +57,6 @@ public final class ContactPeersView { public let accountPeer: Peer? init(_ mutableView: MutableContactPeersView) { - let index = mutableView.index if let accountPeer = mutableView.accountPeer { var peers: [Peer] = [] peers.reserveCapacity(mutableView.peers.count) diff --git a/Postbox/MultiplePeersView.swift b/Postbox/MultiplePeersView.swift new file mode 100644 index 0000000000..ca1317b06d --- /dev/null +++ b/Postbox/MultiplePeersView.swift @@ -0,0 +1,52 @@ +import Foundation + +final class MutableMultiplePeersView { + let peerIds: Set + + var peers: [PeerId: Peer] = [:] + var presences: [PeerId: PeerPresence] = [:] + + init(peerIds: [PeerId], getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) { + self.peerIds = Set(peerIds) + + for peerId in self.peerIds { + if let peer = getPeer(peerId) { + self.peers[peerId] = peer + } + if let presence = getPeerPresence(peerId) { + self.presences[peerId] = presence + } + } + } + + func replay(updatedPeers: [PeerId: Peer], updatedPeerPresences: [PeerId: PeerPresence]) -> Bool { + if updatedPeers.isEmpty && updatedPeerPresences.isEmpty { + return false + } + + var updated = false + + for peerId in self.peerIds { + if let peer = updatedPeers[peerId] { + self.peers[peerId] = peer + updated = true + } + if let presence = updatedPeerPresences[peerId] { + self.presences[peerId] = presence + updated = true + } + } + + return updated + } +} + +public final class MultiplePeersView { + public let peers: [PeerId: Peer] + public let presences: [PeerId: PeerPresence] + + init(_ view: MutableMultiplePeersView) { + self.peers = view.peers + self.presences = view.presences + } +} diff --git a/Postbox/PeerChatListPolicy.swift b/Postbox/PeerChatListInclusion.swift similarity index 82% rename from Postbox/PeerChatListPolicy.swift rename to Postbox/PeerChatListInclusion.swift index be5b3645dc..d6c2a7273d 100644 --- a/Postbox/PeerChatListPolicy.swift +++ b/Postbox/PeerChatListInclusion.swift @@ -2,6 +2,6 @@ import Foundation public enum PeerChatListInclusion { case never - case ifNotEmpty + case ifHasMessages case always(minTimestamp: Int32) } diff --git a/Postbox/PeerNotificationSettingsTable.swift b/Postbox/PeerNotificationSettingsTable.swift index 6b79c40c4b..d13559e380 100644 --- a/Postbox/PeerNotificationSettingsTable.swift +++ b/Postbox/PeerNotificationSettingsTable.swift @@ -66,6 +66,28 @@ final class PeerNotificationSettingsTable: Table { return (added, removed) } + func resetAll(to settings: PeerNotificationSettings) -> [PeerId] { + let lowerBound = ValueBoxKey(length: 8) + lowerBound.setInt64(0, value: 0) + let upperBound = ValueBoxKey(length: 8) + upperBound.setInt64(0, value: Int64.max) + var peerIds: [PeerId] = [] + self.valueBox.range(self.table, start: lowerBound, end: upperBound, keys: { key in + peerIds.append(PeerId(key.getInt64(0))) + return true + }, limit: 0) + + var updatedPeerIds: [PeerId] = [] + for peerId in peerIds { + if let current = self.get(peerId), !current.isEqual(to: settings) { + updatedPeerIds.append(peerId) + self.set(id: peerId, settings: settings) + } + } + + return updatedPeerIds + } + override func beforeCommit() { if !self.updatedInitialSettings.isEmpty { for (peerId, _) in self.updatedInitialSettings { diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index ee95adb0af..2cfbc94e81 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -134,6 +134,10 @@ public final class Modifier { self.postbox?.updatePeerNotificationSettings(notificationSettings) } + public func resetAllPeerNotificationSettings(_ notificationSettings: PeerNotificationSettings) { + self.postbox?.resetAllPeerNotificationSettings(notificationSettings) + } + public func updatePeerCachedData(peerIds: Set, update: (PeerId, CachedPeerData?) -> CachedPeerData?) { self.postbox?.updatePeerCachedData(peerIds: peerIds, update: update) } @@ -241,6 +245,18 @@ public final class Modifier { public func removeTimestampBasedMessageAttribute(tag: UInt16, messageId: MessageId) { self.postbox?.removeTimestampBasedMessageAttribute(tag: tag, messageId: messageId) } + + public func getPreferencesEntry(key: ValueBoxKey) -> PreferencesEntry? { + return self.postbox?.getPreferencesEntry(key: key) + } + + public func setPreferencesEntry(key: ValueBoxKey, value: PreferencesEntry?) { + self.postbox?.setPreferencesEntry(key: key, value: value) + } + + public func updatePreferencesEntry(key: ValueBoxKey, _ f: (PreferencesEntry?) -> PreferencesEntry?) { + self.postbox?.setPreferencesEntry(key: key, value: f(self.postbox?.getPreferencesEntry(key: key))) + } } fileprivate class PipeNotifier: NSObject { @@ -295,6 +311,7 @@ public final class Postbox { private var currentUpdatedTotalUnreadCount: Int32? private var currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation] = [] private var currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation] = [] + private var currentPreferencesOperations: [PreferencesOperation] = [] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -346,6 +363,7 @@ public final class Postbox { var peerOperationLogTable: PeerOperationLogTable! var timestampBasedMessageAttributesTable: TimestampBasedMessageAttributesTable! var timestampBasedMessageAttributesIndexTable: TimestampBasedMessageAttributesIndexTable! + var preferencesTable: PreferencesTable! //temporary var peerRatingTable: RatingTable! @@ -363,14 +381,14 @@ public final class Postbox { self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.pipeNotifier = PipeNotifier(basePath: basePath, notify: { [weak self] in - if let strongSelf = self { + //if let strongSelf = self { /*strongSelf.queue.async { if strongSelf.valueBox != nil { let _ = strongSelf.modify({ _ -> Void in }).start() } }*/ - } + //} }) self.openDatabase() @@ -468,6 +486,7 @@ public final class Postbox { self.peerOperationLogMetadataTable = PeerOperationLogMetadataTable(valueBox: self.valueBox, table: PeerOperationLogMetadataTable.tableSpec(29)) self.peerMergedOperationLogIndexTable = PeerMergedOperationLogIndexTable(valueBox: self.valueBox, table: PeerMergedOperationLogIndexTable.tableSpec(30), metadataTable: self.peerOperationLogMetadataTable!) self.peerOperationLogTable = PeerOperationLogTable(valueBox: self.valueBox, table: PeerOperationLogTable.tableSpec(31), metadataTable: self.peerOperationLogMetadataTable, mergedIndexTable: self.peerMergedOperationLogIndexTable) + self.preferencesTable = PreferencesTable(valueBox: self.valueBox, table: PreferencesTable.tableSpec(35)) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -500,6 +519,7 @@ public final class Postbox { self.tables.append(self.peerOperationLogMetadataTable) self.tables.append(self.peerMergedOperationLogIndexTable) self.tables.append(self.peerOperationLogTable) + self.tables.append(self.preferencesTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() @@ -521,6 +541,8 @@ public final class Postbox { return self.peerMergedOperationLogIndexTable.tailIndex(tag: tag) }, getTimestampBasedMessageAttributesHead: { tag in return self.timestampBasedMessageAttributesTable.head(tag: tag) + }, getPreferencesEntry: { key in + return self.preferencesTable.get(key: key) }, unsentMessageIds: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get(getCombinedPeerReadState: { peerId in return self.readStateTable.getCombinedState(peerId) })) @@ -942,7 +964,7 @@ public final class Postbox { return self.peerTable.get(peerId) }, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) - let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -973,6 +995,7 @@ public final class Postbox { self.currentUpdatedTotalUnreadCount = nil self.currentPeerMergedOperationLogOperations.removeAll() self.currentTimestampBasedMessageAttributesOperations.removeAll() + self.currentPreferencesOperations.removeAll() for table in self.tables { table.beforeCommit() @@ -1022,6 +1045,12 @@ public final class Postbox { } } + fileprivate func resetAllPeerNotificationSettings(_ notificationSettings: PeerNotificationSettings) { + for peerId in self.peerNotificationSettingsTable.resetAll(to: notificationSettings) { + self.currentUpdatedPeerNotificationSettings[peerId] = notificationSettings + } + } + fileprivate func updatePeerCachedData(peerIds: Set, update: (PeerId, CachedPeerData?) -> CachedPeerData?) { for peerId in peerIds { let currentData = self.cachedPeerDataTable.get(peerId) @@ -1166,7 +1195,7 @@ public final class Postbox { private func addLocationsToMessageHistoryViewEntries(tagMask: MessageTags, earlier: MutableMessageHistoryEntry?, later: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry]) -> ([MutableMessageHistoryEntry], MutableMessageHistoryEntry?, MutableMessageHistoryEntry?) { if let firstEntry = entries.first { if let location = self.messageHistoryTagsTable.entryLocation(at: firstEntry.index, tagMask: tagMask) { - var mappedEarlier = earlier?.updatedLocation(location.predecessor) + let mappedEarlier = earlier?.updatedLocation(location.predecessor) var mappedEntries: [MutableMessageHistoryEntry] = [] var previousLocation: MessageHistoryEntryLocation? for i in 0 ..< entries.count { @@ -1179,7 +1208,7 @@ public final class Postbox { } } previousLocation = previousLocation?.successor - var mappedLater = later?.updatedLocation(previousLocation) + let mappedLater = later?.updatedLocation(previousLocation) return (mappedEntries, mappedEarlier, mappedLater) } else { return (entries, earlier, later) @@ -1333,7 +1362,7 @@ public final class Postbox { } |> switchToLatest } - public func contactPeersView(index: PeerNameIndex, accountPeerId: PeerId) -> Signal { + public func contactPeersView(accountPeerId: PeerId) -> Signal { return self.modify { modifier -> Signal in var peers: [PeerId: Peer] = [:] var peerPresences: [PeerId: PeerPresence] = [:] @@ -1347,7 +1376,7 @@ public final class Postbox { } } - let view = MutableContactPeersView(peers: peers, peerPresences: peerPresences, index: index, accountPeer: self.peerTable.get(accountPeerId)) + let view = MutableContactPeersView(peers: peers, peerPresences: peerPresences, accountPeer: self.peerTable.get(accountPeerId)) let (index, signal) = self.viewTracker.addContactPeersView(view) return (.single(ContactPeersView(view)) @@ -1424,6 +1453,23 @@ public final class Postbox { } |> switchToLatest } + public func multiplePeersView(_ ids: [PeerId]) -> Signal { + return self.modify { modifier -> Signal in + let view = MutableMultiplePeersView(peerIds: ids, getPeer: { self.peerTable.get($0) }, getPeerPresence: { self.peerPresenceTable.get($0) }) + let (index, signal) = self.viewTracker.addMultiplePeersView(view) + + return (.single(MultiplePeersView(view)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeMultiplePeersView(index) + } + } + } + } |> switchToLatest + } + public func loadedPeerWithId(_ id: PeerId) -> Signal { return self.modify { modifier -> Signal in if let peer = self.peerTable.get(id) { @@ -1464,7 +1510,7 @@ public final class Postbox { } public func updateMessageHistoryViewVisibleRange(_ id: MessageHistoryViewId, earliestVisibleIndex: MessageIndex, latestVisibleIndex: MessageIndex) { - self.modify({ modifier -> Void in + let _ = self.modify({ modifier -> Void in self.viewTracker.updateMessageHistoryViewVisibleRange(id, earliestVisibleIndex: earliestVisibleIndex, latestVisibleIndex: latestVisibleIndex) }).start() } @@ -1640,6 +1686,33 @@ public final class Postbox { } |> switchToLatest } + public func preferencesView(keys: [ValueBoxKey]) -> Signal { + return self.modify { modifier -> Signal in + let view = MutablePreferencesView(keys: Set(keys), get: { key in + return self.preferencesTable.get(key: key) + }) + let (index, signal) = self.viewTracker.addPreferencesView(view) + + return (.single(PreferencesView(view)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removePreferencesView(index) + } + } + } + } |> switchToLatest + } + + fileprivate func getPreferencesEntry(key: ValueBoxKey) -> PreferencesEntry? { + return self.preferencesTable.get(key: key) + } + + fileprivate func setPreferencesEntry(key: ValueBoxKey, value: PreferencesEntry?) { + self.preferencesTable.set(key: key, value: value, operations: &self.currentPreferencesOperations) + } + public func isMasterClient() -> Signal { return self.modify { modifier -> Signal in let sessionClientId = self.sessionClientId @@ -1650,7 +1723,7 @@ public final class Postbox { } public func becomeMasterClient() { - self.modify({ modifier in + let _ = self.modify({ modifier in if self.metadataTable.masterClientId() != self.sessionClientId { self.currentUpdatedMasterClientId = self.sessionClientId } diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index a6c85c40c1..b7fbdfd0af 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -14,6 +14,7 @@ final class PostboxTransaction { let peerIdsWithUpdatedUnreadCounts: Set let currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation] let currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation] + let currentPreferencesOperations: [PreferencesOperation] let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -76,10 +77,13 @@ final class PostboxTransaction { if !currentTimestampBasedMessageAttributesOperations.isEmpty { return false } + if !currentPreferencesOperations.isEmpty { + return false + } return true } - init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles self.removedHolesByPeerId = removedHolesByPeerId @@ -95,6 +99,7 @@ final class PostboxTransaction { self.currentTimestampBasedMessageAttributesOperations = currentTimestampBasedMessageAttributesOperations self.unsentMessageOperations = unsentMessageOperations self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations + self.currentPreferencesOperations = currentPreferencesOperations self.updatedMedia = updatedMedia self.replaceContactPeerIds = replaceContactPeerIds self.currentUpdatedMasterClientId = currentUpdatedMasterClientId diff --git a/Postbox/PreferencesEntry.swift b/Postbox/PreferencesEntry.swift new file mode 100644 index 0000000000..c795d92e5d --- /dev/null +++ b/Postbox/PreferencesEntry.swift @@ -0,0 +1,5 @@ +import Foundation + +public protocol PreferencesEntry: Coding { + func isEqual(to: PreferencesEntry) -> Bool +} diff --git a/Postbox/PreferencesTable.swift b/Postbox/PreferencesTable.swift new file mode 100644 index 0000000000..cbc5f6c103 --- /dev/null +++ b/Postbox/PreferencesTable.swift @@ -0,0 +1,58 @@ +import Foundation + +enum PreferencesOperation { + case update(ValueBoxKey, PreferencesEntry?) +} + +private struct CachedEntry { + let entry: PreferencesEntry? +} + +final class PreferencesTable: Table { + private var cachedEntries: [ValueBoxKey: CachedEntry] = [:] + private var updatedEntryKeys = Set() + + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + func get(key: ValueBoxKey) -> PreferencesEntry? { + if let cached = self.cachedEntries[key] { + return cached.entry + } else { + if let value = self.valueBox.get(self.table, key: key), let object = Decoder(buffer: value).decodeRootObject() as? PreferencesEntry { + self.cachedEntries[key] = CachedEntry(entry: object) + return object + } else { + self.cachedEntries[key] = CachedEntry(entry: nil) + return nil + } + } + } + + func set(key: ValueBoxKey, value: PreferencesEntry?, operations: inout [PreferencesOperation]) { + self.cachedEntries[key] = CachedEntry(entry: value) + updatedEntryKeys.insert(key) + operations.append(.update(key, value)) + } + + override func clearMemoryCache() { + assert(self.updatedEntryKeys.isEmpty) + } + + override func beforeCommit() { + if !self.updatedEntryKeys.isEmpty { + for key in self.updatedEntryKeys { + if let value = self.cachedEntries[key]?.entry { + let encoder = Encoder() + encoder.encodeRootObject(value) + self.valueBox.set(self.table, key: key, value: encoder.readBufferNoCopy()) + } else { + self.valueBox.remove(self.table, key: key) + } + } + + self.updatedEntryKeys.removeAll() + } + } +} diff --git a/Postbox/PreferencesView.swift b/Postbox/PreferencesView.swift new file mode 100644 index 0000000000..bdc6195672 --- /dev/null +++ b/Postbox/PreferencesView.swift @@ -0,0 +1,55 @@ +import Foundation + +final class MutablePreferencesView { + fileprivate let keys: Set + fileprivate var values: [ValueBoxKey: PreferencesEntry] + + init(keys: Set, get: (ValueBoxKey) -> PreferencesEntry?) { + self.keys = keys + var values: [ValueBoxKey: PreferencesEntry] = [:] + for key in keys { + if let value = get(key) { + values[key] = value + } + } + self.values = values + } + + func replay(operations: [PreferencesOperation], get: (ValueBoxKey) -> PreferencesEntry?) -> Bool { + var updated = false + for operation in operations { + switch operation { + case let .update(key, value): + if self.keys.contains(key) { + let currentValue = self.values[key] + var updatedValue = false + if let value = value, let currentValue = currentValue { + if !value.isEqual(to: currentValue) { + updatedValue = true + } + } else if (value != nil) != (currentValue != nil) { + updatedValue = true + } + if updatedValue { + if let value = value { + self.values[key] = value + } else { + self.values.removeValue(forKey: key) + } + updated = true + } + } + } + } + + return updated + } +} + +public final class PreferencesView { + public let values: [ValueBoxKey: PreferencesEntry] + + init(_ view: MutablePreferencesView) { + self.values = view.values + } +} diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index f1a99b436d..3d219bdb6a 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -69,6 +69,14 @@ private struct SqlitePreparedStatement { return key } + func int64KeyAt(_ index: Int) -> ValueBoxKey { + let value = sqlite3_column_int64(statement, Int32(index)) + + let key = ValueBoxKey(length: 9) + key.setInt64(0, value: value) + return key + } + func destroy() { sqlite3_finalize(statement) } @@ -158,15 +166,14 @@ final class SqliteValueBox: ValueBox { } lock.unlock() - checkpoints.set((Signal.single(Void()) |> delay(10.0, queue: Queue.concurrentDefaultQueue()) |> restart).start(next: { [weak self] _ in - if let strongSelf = self , strongSelf.database != nil { - strongSelf.queue.async { - strongSelf.lock.lock() - var nLog: Int32 = 0 - var nFrames: Int32 = 0 - sqlite3_wal_checkpoint_v2(strongSelf.database.handle, nil, SQLITE_CHECKPOINT_PASSIVE, &nLog, &nFrames) - strongSelf.lock.unlock() - } + checkpoints.set((Signal.single(Void()) |> delay(10.0, queue: self.queue) |> restart).start(next: { [weak self] _ in + if let strongSelf = self, strongSelf.database != nil { + assert(strongSelf.queue.isCurrent()) + strongSelf.lock.lock() + var nLog: Int32 = 0 + var nFrames: Int32 = 0 + sqlite3_wal_checkpoint_v2(strongSelf.database.handle, nil, SQLITE_CHECKPOINT_PASSIVE, &nLog, &nFrames) + strongSelf.lock.unlock() //print("(SQLite WAL size \(nLog) removed \(nFrames))") } })) @@ -706,40 +713,78 @@ final class SqliteValueBox: ValueBox { var startTime = CFAbsoluteTimeGetCurrent() - if start < end { - if limit <= 0 { - statement = self.rangeValueAscStatementNoLimit(table, start: start, end: end) - } else { - statement = self.rangeValueAscStatementLimit(table, start: start, end: end, limit: limit) - } - } else { - if limit <= 0 { - statement = self.rangeValueDescStatementNoLimit(table, start: end, end: start) - } else { - statement = self.rangeValueDescStatementLimit(table, start: end, end: start, limit: limit) - } + switch table.keyType { + case .binary: + if start < end { + if limit <= 0 { + statement = self.rangeValueAscStatementNoLimit(table, start: start, end: end) + } else { + statement = self.rangeValueAscStatementLimit(table, start: start, end: end, limit: limit) + } + } else { + if limit <= 0 { + statement = self.rangeValueDescStatementNoLimit(table, start: end, end: start) + } else { + statement = self.rangeValueDescStatementLimit(table, start: end, end: start, limit: limit) + } + } + + var currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + startTime = currentTime + + while statement.step() { + startTime = CFAbsoluteTimeGetCurrent() + + let key = statement.keyAt(0) + let value = statement.valueAt(1) + + currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + if !values(key, value) { + break + } + } + + statement.reset() + case .int64: + if start.reversed < end.reversed { + if limit <= 0 { + statement = self.rangeValueAscStatementNoLimit(table, start: start, end: end) + } else { + statement = self.rangeValueAscStatementLimit(table, start: start, end: end, limit: limit) + } + } else { + if limit <= 0 { + statement = self.rangeValueDescStatementNoLimit(table, start: end, end: start) + } else { + statement = self.rangeValueDescStatementLimit(table, start: end, end: start, limit: limit) + } + } + + var currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + startTime = currentTime + + while statement.step() { + startTime = CFAbsoluteTimeGetCurrent() + + let key = statement.int64KeyAt(0) + let value = statement.valueAt(1) + + currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + if !values(key, value) { + break + } + } + + statement.reset() } - - var currentTime = CFAbsoluteTimeGetCurrent() - self.readQueryTime += currentTime - startTime - - startTime = currentTime - - while statement.step() { - startTime = CFAbsoluteTimeGetCurrent() - - let key = statement.keyAt(0) - let value = statement.valueAt(1) - - currentTime = CFAbsoluteTimeGetCurrent() - self.readQueryTime += currentTime - startTime - - if !values(key, value) { - break - } - } - - statement.reset() } } @@ -750,39 +795,76 @@ final class SqliteValueBox: ValueBox { var startTime = CFAbsoluteTimeGetCurrent() - if start < end { - if limit <= 0 { - statement = self.rangeKeyAscStatementNoLimit(table, start: start, end: end) - } else { - statement = self.rangeKeyAscStatementLimit(table, start: start, end: end, limit: limit) - } - } else { - if limit <= 0 { - statement = self.rangeKeyDescStatementNoLimit(table, start: end, end: start) - } else { - statement = self.rangeKeyDescStatementLimit(table, start: end, end: start, limit: limit) - } + switch table.keyType { + case .binary: + if start < end { + if limit <= 0 { + statement = self.rangeKeyAscStatementNoLimit(table, start: start, end: end) + } else { + statement = self.rangeKeyAscStatementLimit(table, start: start, end: end, limit: limit) + } + } else { + if limit <= 0 { + statement = self.rangeKeyDescStatementNoLimit(table, start: end, end: start) + } else { + statement = self.rangeKeyDescStatementLimit(table, start: end, end: start, limit: limit) + } + } + + var currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + startTime = currentTime + + while statement.step() { + startTime = CFAbsoluteTimeGetCurrent() + + let key = statement.keyAt(0) + + currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + if !keys(key) { + break + } + } + + statement.reset() + case .int64: + if start.reversed < end.reversed { + if limit <= 0 { + statement = self.rangeKeyAscStatementNoLimit(table, start: start, end: end) + } else { + statement = self.rangeKeyAscStatementLimit(table, start: start, end: end, limit: limit) + } + } else { + if limit <= 0 { + statement = self.rangeKeyDescStatementNoLimit(table, start: end, end: start) + } else { + statement = self.rangeKeyDescStatementLimit(table, start: end, end: start, limit: limit) + } + } + + var currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + startTime = currentTime + + while statement.step() { + startTime = CFAbsoluteTimeGetCurrent() + + let key = statement.int64KeyAt(0) + + currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + if !keys(key) { + break + } + } + + statement.reset() } - - var currentTime = CFAbsoluteTimeGetCurrent() - self.readQueryTime += currentTime - startTime - - startTime = currentTime - - while statement.step() { - startTime = CFAbsoluteTimeGetCurrent() - - let key = statement.keyAt(0) - - currentTime = CFAbsoluteTimeGetCurrent() - self.readQueryTime += currentTime - startTime - - if !keys(key) { - break - } - } - - statement.reset() } } diff --git a/Postbox/ValueBox.swift b/Postbox/ValueBox.swift index 1de04fe06f..a9f8561c92 100644 --- a/Postbox/ValueBox.swift +++ b/Postbox/ValueBox.swift @@ -17,8 +17,8 @@ protocol ValueBox { func beginStats() func endStats() - func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, values: @noescape(ValueBoxKey, ReadBuffer) -> Bool, limit: Int) - func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, keys: @noescape(ValueBoxKey) -> Bool, limit: Int) + func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) + func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, keys: (ValueBoxKey) -> Bool, limit: Int) func get(_ table: ValueBoxTable, key: ValueBoxKey) -> ReadBuffer? func exists(_ table: ValueBoxTable, key: ValueBoxKey) -> Bool func set(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index ebb9eb280a..aabc158b50 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -121,6 +121,18 @@ public struct ValueBoxKey: Hashable, CustomStringConvertible, Comparable { } } + public var reversed: ValueBoxKey { + let key = ValueBoxKey(length: self.length) + let keyMemory = key.memory.assumingMemoryBound(to: UInt8.self) + let selfMemory = self.memory.assumingMemoryBound(to: UInt8.self) + var i = self.length - 1 + while i >= 0 { + keyMemory[i] = selfMemory[self.length - 1 - i] + i -= 1 + } + return key + } + public var successor: ValueBoxKey { let key = ValueBoxKey(length: self.length) memcpy(key.memory, self.memory, self.length) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 1009775888..583e1ebcaf 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -28,6 +28,7 @@ final class ViewTracker { private let getPeerReadState: (PeerId) -> CombinedPeerReadState? private let operationLogGetOperations: (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry] private let operationLogGetTailIndex: (PeerOperationLogTag) -> Int32? + private let getPreferencesEntry: (ValueBoxKey) -> PreferencesEntry? private var chatListViews = Bag<(MutableChatListView, ValuePipe<(ChatListView, ViewUpdateType)>)>() private var messageHistoryViews: [PeerId: Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>] = [:] @@ -55,8 +56,10 @@ final class ViewTracker { private var timestampBasedMessageAttributesViews = Bag<(MutableTimestampBasedMessageAttributesView, ValuePipe)>() private var messageViews = Bag<(MutableMessageView, ValuePipe)>() + private var preferencesViews = Bag<(MutablePreferencesView, ValuePipe)>() + private var multiplePeersViews = Bag<(MutableMultiplePeersView, ValuePipe)>() - init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, operationLogGetOperations: @escaping (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], operationLogGetTailIndex: @escaping (PeerOperationLogTag) -> Int32?, getTimestampBasedMessageAttributesHead: @escaping (UInt16) -> TimestampBasedMessageAttributesEntry?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { + init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, operationLogGetOperations: @escaping (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], operationLogGetTailIndex: @escaping (PeerOperationLogTag) -> Int32?, getTimestampBasedMessageAttributesHead: @escaping (UInt16) -> TimestampBasedMessageAttributesEntry?, getPreferencesEntry: @escaping (ValueBoxKey) -> PreferencesEntry?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue self.fetchEarlierHistoryEntries = fetchEarlierHistoryEntries self.fetchLaterHistoryEntries = fetchLaterHistoryEntries @@ -73,6 +76,7 @@ final class ViewTracker { self.operationLogGetOperations = operationLogGetOperations self.operationLogGetTailIndex = operationLogGetTailIndex self.getTimestampBasedMessageAttributesHead = getTimestampBasedMessageAttributesHead + self.getPreferencesEntry = getPreferencesEntry self.unsentMessageView = UnsentMessageHistoryView(ids: unsentMessageIds) self.synchronizeReadStatesView = MutableSynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) @@ -226,6 +230,28 @@ final class ViewTracker { self.messageViews.remove(index) } + func addPreferencesView(_ view: MutablePreferencesView) -> (Bag<(MutablePreferencesView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.preferencesViews.add(record) + + return (index, record.1.signal()) + } + + func removePreferencesView(_ index: Bag<(MutablePreferencesView, ValuePipe)>.Index) { + self.preferencesViews.remove(index) + } + + func addMultiplePeersView(_ view: MutableMultiplePeersView) -> (Bag<(MutableMultiplePeersView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.multiplePeersViews.add(record) + + return (index, record.1.signal()) + } + + func removeMultiplePeersView(_ index: Bag<(MutableMultiplePeersView, ValuePipe)>.Index) { + self.multiplePeersViews.remove(index) + } + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: ChatListIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] @@ -263,28 +289,29 @@ final class ViewTracker { for (mutableView, pipe) in self.peerViews.copyItems() { var updatedPeers: [PeerId: Peer] = [:] - if let peer = self.getPeer(mutableView.peerId) { - updatedPeers[mutableView.peerId] = peer - } var updatedPeerPresences: [PeerId: PeerPresence] = [:] - if let presence = self.getPeerPresence(mutableView.peerId) { - updatedPeerPresences[mutableView.peerId] = presence - } - var updatedNotificationSettings: [PeerId: PeerNotificationSettings] = [:] - if let notificationSettings = self.getPeerNotificationSettings(mutableView.peerId) { - updatedNotificationSettings[mutableView.peerId] = notificationSettings - } - var updatedCachedPeerData: [PeerId: CachedPeerData] = [:] - if let cachedPeerData = self.getCachedPeerData(mutableView.peerId) { - updatedCachedPeerData[mutableView.peerId] = cachedPeerData - for peerId in cachedPeerData.peerIds { - if let peer = self.getPeer(peerId) { - updatedPeers[peerId] = peer + + let peerId = mutableView.peerId + + if let peer = self.getPeer(peerId) { + updatedPeers[peerId] = peer + } + if let presence = self.getPeerPresence(peerId) { + updatedPeerPresences[peerId] = presence + } + if let notificationSettings = self.getPeerNotificationSettings(peerId) { + updatedNotificationSettings[peerId] = notificationSettings + } + if let cachedPeerData = self.getCachedPeerData(peerId) { + updatedCachedPeerData[peerId] = cachedPeerData + for cachedPeerId in cachedPeerData.peerIds { + if let peer = self.getPeer(cachedPeerId) { + updatedPeers[cachedPeerId] = peer } - if let presence = self.getPeerPresence(peerId) { - updatedPeerPresences[peerId] = presence + if let presence = self.getPeerPresence(cachedPeerId) { + updatedPeerPresences[cachedPeerId] = presence } } } @@ -360,10 +387,10 @@ final class ViewTracker { if mutableView.replay(transaction.chatListOperations, updatedPeerNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, context: context) { mutableView.complete(context: context, fetchEarlier: self.fetchEarlierChatEntries, fetchLater: self.fetchLaterChatEntries) mutableView.render(self.renderMessage, getPeerNotificationSettings: self.getPeerNotificationSettings) - var updateType: ViewUpdateType = .Generic + //var updateType: ViewUpdateType = .Generic for operation in transaction.chatListOperations { if case .RemoveHoles = operation { - updateType = .UpdateVisible + //updateType = .UpdateVisible break } } @@ -423,6 +450,18 @@ final class ViewTracker { pipe.putNext(TimestampBasedMessageAttributesView(mutableView)) } } + + for (mutableView, pipe) in self.preferencesViews.copyItems() { + if mutableView.replay(operations: transaction.currentPreferencesOperations, get: self.getPreferencesEntry) { + pipe.putNext(PreferencesView(mutableView)) + } + } + + for (mutableView, pipe) in self.multiplePeersViews.copyItems() { + if mutableView.replay(updatedPeers: transaction.currentUpdatedPeers, updatedPeerPresences: transaction.currentUpdatedPeerPresences) { + pipe.putNext(MultiplePeersView(mutableView)) + } + } } private func updateTrackedChatListHoles() { From b5f298977cc8a5e891a05526e5f673d88380f3a8 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 31 Jan 2017 14:41:50 +0300 Subject: [PATCH 077/427] no message --- Postbox.xcodeproj/project.pbxproj | 6 + Postbox/ChatListIndexTable.swift | 193 ++++++++++++++++++----- Postbox/ChatListTable.swift | 161 +++++++++++++------ Postbox/ChatListView.swift | 93 ++++++----- Postbox/PeerChatListInclusion.swift | 53 ++++++- Postbox/PeerNameIndexTable.swift | 2 +- Postbox/Postbox.swift | 79 ++++++---- Postbox/RenderedPeer.swift | 26 ++++ Postbox/SimpleDictionary.swift | 16 ++ Postbox/ViewTracker.swift | 8 +- PostboxTests/ChatListTableTests.swift | 212 ++++++++++++++++++++------ 11 files changed, 633 insertions(+), 216 deletions(-) create mode 100644 Postbox/RenderedPeer.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index c8b5922ab2..d4344efb1a 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -145,6 +145,8 @@ D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B844501DAC04FE005F29E1 /* PeerPresence.swift */; }; D0BC386C1E3FCEE50044D6FE /* MultiplePeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */; }; + D0BC38721E409E670044D6FE /* RenderedPeer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC38711E409E670044D6FE /* RenderedPeer.swift */; }; + D0BC38731E409E670044D6FE /* RenderedPeer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC38711E409E670044D6FE /* RenderedPeer.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; @@ -309,6 +311,7 @@ D0B418601D7DFE95004562A4 /* SwiftSignalKitMac.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKitMac.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-diblohvjozhgaifjcniwdlixlilx/Build/Products/Debug/SwiftSignalKitMac.framework"; sourceTree = ""; }; D0B844501DAC04FE005F29E1 /* PeerPresence.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerPresence.swift; sourceTree = ""; }; D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MultiplePeersView.swift; sourceTree = ""; }; + D0BC38711E409E670044D6FE /* RenderedPeer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RenderedPeer.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; @@ -504,6 +507,7 @@ D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */, D00C7CD31E365C4E0080C3D5 /* PeerChatListInclusion.swift */, D08775021E3E3E7400A97350 /* PreferencesEntry.swift */, + D0BC38711E409E670044D6FE /* RenderedPeer.swift */, ); name = Objects; sourceTree = ""; @@ -856,6 +860,7 @@ D073CE7C1DCBF3B4007511FD /* CachedPeerData.swift in Sources */, D0F7B1DA1E045C6A007EB8A5 /* RatingTable.swift in Sources */, D073CE9F1DCBF3C1007511FD /* InitialMessageHistoryData.swift in Sources */, + D0BC38731E409E670044D6FE /* RenderedPeer.swift in Sources */, D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopIndexableMessageIds.swift in Sources */, D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */, D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, @@ -971,6 +976,7 @@ D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */, + D0BC38721E409E670044D6FE /* RenderedPeer.swift in Sources */, D08775031E3E3E7400A97350 /* PreferencesEntry.swift in Sources */, D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */, D08775001E3E3D9F00A97350 /* PreferencesTable.swift in Sources */, diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift index 413aabf967..dbb3dadef3 100644 --- a/Postbox/ChatListIndexTable.swift +++ b/Postbox/ChatListIndexTable.swift @@ -1,5 +1,37 @@ import Foundation +struct ChatListInclusionIndex { + let topMessageIndex: MessageIndex? + let inclusion: PeerChatListInclusion + + func includedIndex(peerId: PeerId) -> ChatListIndex? { + switch inclusion { + case .notSpecified, .never: + return nil + case .ifHasMessages: + if let topMessageIndex = self.topMessageIndex { + return ChatListIndex(pinningIndex: nil, messageIndex: topMessageIndex) + } else { + return nil + } + case let .ifHasMessagesOrOneOf(pinningIndex, minTimestamp): + if let minTimestamp = minTimestamp { + if let topMessageIndex = self.topMessageIndex, topMessageIndex.timestamp >= minTimestamp { + return ChatListIndex(pinningIndex: pinningIndex, messageIndex: topMessageIndex) + } else { + return ChatListIndex(pinningIndex: pinningIndex, messageIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: minTimestamp)) + } + } else if let topMessageIndex = self.topMessageIndex { + return ChatListIndex(pinningIndex: pinningIndex, messageIndex: topMessageIndex) + } else if let pinningIndex = pinningIndex { + return ChatListIndex(pinningIndex: pinningIndex, messageIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: 0)) + } else { + return nil + } + } + } +} + final class ChatListIndexTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .int64) @@ -12,8 +44,8 @@ final class ChatListIndexTable: Table { private let sharedKey = ValueBoxKey(length: 8) - private var cachedIndices: [PeerId: ChatListIndex?] = [:] - private var updatedPreviousCachedIndices: [PeerId: ChatListIndex?] = [:] + private var cachedIndices: [PeerId: ChatListInclusionIndex] = [:] + private var updatedPreviousCachedIndices: [PeerId: ChatListInclusionIndex] = [:] init(valueBox: ValueBox, table: ValueBoxTable, peerNameIndexTable: PeerNameIndexTable, metadataTable: MessageHistoryMetadataTable, readStateTable: MessageHistoryReadStateTable, notificationSettingsTable: PeerNotificationSettingsTable) { self.peerNameIndexTable = peerNameIndexTable @@ -29,34 +61,81 @@ final class ChatListIndexTable: Table { return self.sharedKey } - func set(_ index: ChatListIndex) { - self.updatedPreviousCachedIndices[index.messageIndex.id.peerId] = self.get(index.messageIndex.id.peerId) - self.cachedIndices[index.messageIndex.id.peerId] = index + func setTopMessageIndex(peerId: PeerId, index: MessageIndex?) -> ChatListInclusionIndex { + let current = self.get(peerId) + if self.updatedPreviousCachedIndices[peerId] == nil { + self.updatedPreviousCachedIndices[peerId] = current + } + let updated = ChatListInclusionIndex(topMessageIndex: index, inclusion: current.inclusion) + self.cachedIndices[peerId] = updated + return updated } - func remove(_ peerId: PeerId) { - self.updatedPreviousCachedIndices[peerId] = self.get(peerId) - self.cachedIndices[peerId] = nil + func setInclusion(peerId: PeerId, inclusion: PeerChatListInclusion) -> ChatListInclusionIndex { + let current = self.get(peerId) + if self.updatedPreviousCachedIndices[peerId] == nil { + self.updatedPreviousCachedIndices[peerId] = current + } + let updated = ChatListInclusionIndex(topMessageIndex: current.topMessageIndex, inclusion: inclusion) + self.cachedIndices[peerId] = updated + return updated } - func get(_ peerId: PeerId) -> ChatListIndex? { + func get(_ peerId: PeerId) -> ChatListInclusionIndex { if let cached = self.cachedIndices[peerId] { return cached } else { if let value = self.valueBox.get(self.table, key: self.key(peerId)) { - var pinningIndexValue: UInt16 = 0 - var idNamespace: Int32 = 0 - var idId: Int32 = 0 - var idTimestamp: Int32 = 0 - value.read(&pinningIndexValue, offset: 0, length: 2) - value.read(&idNamespace, offset: 0, length: 4) - value.read(&idId, offset: 0, length: 4) - value.read(&idTimestamp, offset: 0, length: 4) - let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: idNamespace, id: idId), timestamp: idTimestamp)) - self.cachedIndices[peerId] = index - return index + let topMessageIndex: MessageIndex? + + var hasIndex: Int8 = 0 + value.read(&hasIndex, offset: 0, length: 1) + if hasIndex != 0 { + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + topMessageIndex = MessageIndex(id: MessageId(peerId: peerId, namespace: idNamespace, id: idId), timestamp: idTimestamp) + } else { + topMessageIndex = nil + } + + let inclusion: PeerChatListInclusion + + var inclusionId: Int8 = 0 + value.read(&inclusionId, offset: 0, length: 1) + if inclusionId == 0 { + inclusion = .notSpecified + } else if inclusionId == 1 { + inclusion = .never + } else if inclusionId == 2 { + inclusion = .ifHasMessages + } else if inclusionId == 3 { + var pinningIndexValue: UInt16 = 0 + value.read(&pinningIndexValue, offset: 0, length: 2) + + var hasMinTimestamp: Int8 = 0 + value.read(&hasMinTimestamp, offset: 0, length: 1) + let minTimestamp: Int32? + if hasMinTimestamp != 0 { + var minTimestampValue: Int32 = 0 + value.read(&minTimestampValue, offset: 0, length: 4) + minTimestamp = minTimestampValue + } else { + minTimestamp = nil + } + inclusion = .ifHasMessagesOrOneOf(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), minTimestamp: minTimestamp) + } else { + preconditionFailure() + } + + let inclusionIndex = ChatListInclusionIndex(topMessageIndex: topMessageIndex, inclusion: inclusion) + self.cachedIndices[peerId] = inclusionIndex + return inclusionIndex } else { - return nil + return ChatListInclusionIndex(topMessageIndex: nil, inclusion: .notSpecified) } } } @@ -73,28 +152,60 @@ final class ChatListIndexTable: Table { for (peerId, previousIndex) in self.updatedPreviousCachedIndices { let index = self.cachedIndices[peerId]! - if let index = index { - if previousIndex == nil { + if index.includedIndex(peerId: peerId) != nil { + if previousIndex.includedIndex(peerId: peerId) == nil { addedChatListPeerIds.insert(peerId) } - - let writeBuffer = WriteBuffer() - var pinningIndexValue: UInt16 = keyValueForChatListPinningIndex(index.pinningIndex) - var idNamespace: Int32 = index.messageIndex.id.namespace - var idId: Int32 = index.messageIndex.id.id - var idTimestamp: Int32 = index.messageIndex.timestamp - writeBuffer.write(&pinningIndexValue, offset: 0, length: 2) + } else if previousIndex.includedIndex(peerId: peerId) != nil { + removedChatListPeerIds.insert(peerId) + } + + let writeBuffer = WriteBuffer() + + if let topMessageIndex = index.topMessageIndex { + var hasIndex: Int8 = 1 + writeBuffer.write(&hasIndex, offset: 0, length: 1) + var idNamespace: Int32 = topMessageIndex.id.namespace + var idId: Int32 = topMessageIndex.id.id + var idTimestamp: Int32 = topMessageIndex.timestamp writeBuffer.write(&idNamespace, offset: 0, length: 4) writeBuffer.write(&idId, offset: 0, length: 4) writeBuffer.write(&idTimestamp, offset: 0, length: 4) - self.valueBox.set(self.table, key: self.key(index.messageIndex.id.peerId), value: writeBuffer.readBufferNoCopy()) } else { - if previousIndex != nil { - removedChatListPeerIds.insert(peerId) - } - - self.valueBox.remove(self.table, key: self.key(peerId)) + var hasIndex: Int8 = 0 + writeBuffer.write(&hasIndex, offset: 0, length: 1) } + + switch index.inclusion { + case .notSpecified: + var key: Int8 = 0 + writeBuffer.write(&key, offset: 0, length: 1) + case .never: + var key: Int8 = 1 + writeBuffer.write(&key, offset: 0, length: 1) + case .ifHasMessages: + var key: Int8 = 2 + writeBuffer.write(&key, offset: 0, length: 1) + case let .ifHasMessagesOrOneOf(pinningIndex, minTimestamp): + var key: Int8 = 3 + writeBuffer.write(&key, offset: 0, length: 1) + + var pinningIndexValue: UInt16 = keyValueForChatListPinningIndex(pinningIndex) + writeBuffer.write(&pinningIndexValue, offset: 0, length: 2) + + if let minTimestamp = minTimestamp { + var hasMinTimestamp: Int8 = 1 + writeBuffer.write(&hasMinTimestamp, offset: 0, length: 1) + + var minTimestampValue = minTimestamp + writeBuffer.write(&minTimestampValue, offset: 0, length: 4) + } else { + var hasMinTimestamp: Int8 = 0 + writeBuffer.write(&hasMinTimestamp, offset: 0, length: 1) + } + } + + self.valueBox.set(self.table, key: self.key(peerId), value: writeBuffer.readBufferNoCopy()) } self.updatedPreviousCachedIndices.removeAll() @@ -112,7 +223,7 @@ final class ChatListIndexTable: Table { notificationSettings = self.notificationSettingsTable.get(peerId) } } - if let _ = self.get(peerId), let notificationSettings = notificationSettings, !notificationSettings.isRemovedFromTotalUnreadCount { + if let _ = self.get(peerId).includedIndex(peerId: peerId), let notificationSettings = notificationSettings, !notificationSettings.isRemovedFromTotalUnreadCount { totalUnreadCount += delta } } @@ -145,7 +256,7 @@ final class ChatListIndexTable: Table { } } } else if startedParticipationInUnreadCount { - if let _ = self.get(peerId) { + if let _ = self.get(peerId).includedIndex(peerId: peerId) { if let combinedState = self.readStateTable.getCombinedState(peerId) { totalUnreadCount += combinedState.count } @@ -169,8 +280,8 @@ final class ChatListIndexTable: Table { currentPeerUnreadCount -= delta } - var removedFromList = removedChatListPeerIds.contains(peerId) - var removedFromParticipationInUnreadCount = transactionParticipationInTotalUnreadCountUpdates.removed.contains(peerId) + let removedFromList = removedChatListPeerIds.contains(peerId) + let removedFromParticipationInUnreadCount = transactionParticipationInTotalUnreadCountUpdates.removed.contains(peerId) if removedFromList && removedFromParticipationInUnreadCount { totalUnreadCount -= currentPeerUnreadCount } else if removedFromList { @@ -186,7 +297,7 @@ final class ChatListIndexTable: Table { totalUnreadCount -= currentPeerUnreadCount } } else if removedFromParticipationInUnreadCount { - if let _ = self.get(peerId) { + if let _ = self.get(peerId).includedIndex(peerId: peerId) { totalUnreadCount -= currentPeerUnreadCount } } else { diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index 28b78a13a6..daea865d32 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -1,17 +1,15 @@ import Foundation enum ChatListOperation { - case InsertMessage(ChatListIndex, IntermediateMessage, CombinedPeerReadState?, PeerChatListEmbeddedInterfaceState?) + case InsertEntry(ChatListIndex, IntermediateMessage?, CombinedPeerReadState?, PeerChatListEmbeddedInterfaceState?) case InsertHole(ChatListHole) - case InsertNothing(ChatListIndex) - case RemoveMessage([ChatListIndex]) + case RemoveEntry([ChatListIndex]) case RemoveHoles([ChatListIndex]) } enum ChatListIntermediateEntry { - case Message(ChatListIndex, IntermediateMessage, PeerChatListEmbeddedInterfaceState?) + case Message(ChatListIndex, IntermediateMessage?, PeerChatListEmbeddedInterfaceState?) case Hole(ChatListHole) - case Nothing(ChatListIndex) var index: ChatListIndex { switch self { @@ -19,8 +17,6 @@ enum ChatListIntermediateEntry { return index case let .Hole(hole): return ChatListIndex(pinningIndex: nil, messageIndex: hole.index) - case let .Nothing(index): - return index } } } @@ -98,7 +94,20 @@ final class ChatListTable: Table { } } - func replay(historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]], updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, operations: inout [ChatListOperation]) { + func updateInclusion(peerId: PeerId, updatedChatListInclusions: inout [PeerId: PeerChatListInclusion], _ f: (PeerChatListInclusion) -> PeerChatListInclusion) { + let currentInclusion: PeerChatListInclusion + if let updated = updatedChatListInclusions[peerId] { + currentInclusion = updated + } else { + currentInclusion = self.indexTable.get(peerId).inclusion + } + let updatedInclusion = f(currentInclusion) + if currentInclusion != updatedInclusion { + updatedChatListInclusions[peerId] = updatedInclusion + } + } + + func replay(historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]], updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], updatedChatListInclusions: [PeerId: PeerChatListInclusion], messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, operations: inout [ChatListOperation]) { self.ensureInitialized() var changedPeerIds = Set() for peerId in historyOperationsByPeerId.keys { @@ -107,33 +116,47 @@ final class ChatListTable: Table { for peerId in updatedPeerChatListEmbeddedStates.keys { changedPeerIds.insert(peerId) } + for peerId in updatedChatListInclusions.keys { + changedPeerIds.insert(peerId) + } for peerId in changedPeerIds { - let currentIndex: ChatListIndex? = self.indexTable.get(peerId) + let currentIndex: ChatListIndex? = self.indexTable.get(peerId).includedIndex(peerId: peerId) - let updatedIndex: ChatListIndex? let topMessage = messageHistoryTable.topMessage(peerId) let embeddedChatState = peerChatInterfaceStateTable.get(peerId)?.chatListEmbeddedState + let rawTopMessageIndex: MessageIndex? + let topMessageIndex: MessageIndex? if let topMessage = topMessage { var updatedTimestamp = topMessage.timestamp + rawTopMessageIndex = MessageIndex(id: topMessage.id, timestamp: topMessage.timestamp) if let embeddedChatState = embeddedChatState { updatedTimestamp = max(updatedTimestamp, embeddedChatState.timestamp) } - let updatedIndex: ChatListIndex = ChatListIndex(pinningIndex: currentIndex?.pinningIndex, messageIndex: MessageIndex(id: topMessage.id, timestamp: updatedTimestamp)) - - if let currentIndex = currentIndex, currentIndex != updatedIndex { + topMessageIndex = MessageIndex(id: topMessage.id, timestamp: updatedTimestamp) + } else { + topMessageIndex = nil + rawTopMessageIndex = nil + } + + var updatedIndex = self.indexTable.setTopMessageIndex(peerId: peerId, index: topMessageIndex) + if let updatedInclusion = updatedChatListInclusions[peerId] { + updatedIndex = self.indexTable.setInclusion(peerId: peerId, inclusion: updatedInclusion) + } + + if let updatedOrderingIndex = updatedIndex.includedIndex(peerId: peerId) { + if let currentIndex = currentIndex, currentIndex != updatedOrderingIndex { self.justRemoveIndex(currentIndex) } if let currentIndex = currentIndex { - operations.append(.RemoveMessage([currentIndex])) + operations.append(.RemoveEntry([currentIndex])) } - self.indexTable.set(updatedIndex) - self.justInsertIndex(updatedIndex) - operations.append(.InsertMessage(updatedIndex, topMessage, messageHistoryTable.readStateTable.getCombinedState(peerId), embeddedChatState)) + self.justInsertIndex(updatedOrderingIndex, topMessageIndex: rawTopMessageIndex) + operations.append(.InsertEntry(updatedOrderingIndex, topMessage, messageHistoryTable.readStateTable.getCombinedState(peerId), embeddedChatState)) } else { if let currentIndex = currentIndex { - operations.append(.RemoveMessage([currentIndex])) - operations.append(.InsertNothing(currentIndex)) + self.justRemoveIndex(currentIndex) + operations.append(.RemoveEntry([currentIndex])) } } } @@ -166,8 +189,17 @@ final class ChatListTable: Table { } } - private func justInsertIndex(_ index: ChatListIndex) { - self.valueBox.set(self.table, key: self.key(index, type: .Message), value: self.emptyMemoryBuffer) + private func justInsertIndex(_ index: ChatListIndex, topMessageIndex: MessageIndex?) { + let buffer = WriteBuffer() + if let topMessageIndex = topMessageIndex { + var idNamespace = topMessageIndex.id.namespace + buffer.write(&idNamespace, offset: 0, length: 4) + var idId = topMessageIndex.id.id + buffer.write(&idId, offset: 0, length: 4) + var indexTimestamp = topMessageIndex.timestamp + buffer.write(&indexTimestamp, offset: 0, length: 4) + } + self.valueBox.set(self.table, key: self.key(index, type: .Message), value: buffer) } private func justRemoveIndex(_ index: ChatListIndex) { @@ -190,16 +222,23 @@ final class ChatListTable: Table { var lower: ChatListIntermediateEntry? var upper: ChatListIntermediateEntry? - self.valueBox.range(self.table, start: self.key(index, type: .Message), end: self.lowerBound(), keys: { key in + self.valueBox.range(self.table, start: self.key(index, type: .Message), end: self.lowerBound(), values: { key, value in let pinningIndexValue: UInt16 = key.getUInt16(0) let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index.messageIndex) { - lowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) - } else { - lowerEntries.append(.Nothing(index)) + var message: IntermediateMessage? + if value.length != 0 { + var idNamespace: Int32 = 0 + value.read(&idNamespace, offset: 0, length: 4) + var idId: Int32 = 0 + value.read(&idId, offset: 0, length: 4) + var indexTimestamp: Int32 = 0 + value.read(&indexTimestamp, offset: 0, length: 4) + + message = messageHistoryTable.getMessage(MessageIndex(id: MessageId(peerId: index.messageIndex.id.peerId, namespace: idNamespace, id: idId), timestamp: indexTimestamp)) } + lowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { lowerEntries.append(.Hole(ChatListHole(index: index.messageIndex))) } @@ -210,16 +249,23 @@ final class ChatListTable: Table { lowerEntries.removeLast() } - self.valueBox.range(self.table, start: self.key(index, type: .Message).predecessor, end: self.upperBound(), keys: { key in + self.valueBox.range(self.table, start: self.key(index, type: .Message).predecessor, end: self.upperBound(), values: { key, value in let pinningIndexValue: UInt16 = key.getUInt16(0) let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index.messageIndex) { - upperEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) - } else { - upperEntries.append(.Nothing(index)) + var message: IntermediateMessage? + if value.length != 0 { + var idNamespace: Int32 = 0 + value.read(&idNamespace, offset: 0, length: 4) + var idId: Int32 = 0 + value.read(&idId, offset: 0, length: 4) + var indexTimestamp: Int32 = 0 + value.read(&indexTimestamp, offset: 0, length: 4) + + message = messageHistoryTable.getMessage(MessageIndex(id: MessageId(peerId: index.messageIndex.id.peerId, namespace: idNamespace, id: idId), timestamp: indexTimestamp)) } + upperEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { upperEntries.append(.Hole(ChatListHole(index: index.messageIndex))) } @@ -234,21 +280,28 @@ final class ChatListTable: Table { var additionalLowerEntries: [ChatListIntermediateEntry] = [] let startEntryType: ChatListEntryType switch lowerEntries.last! { - case .Message, .Nothing: + case .Message: startEntryType = .Message case .Hole: startEntryType = .Hole } - self.valueBox.range(self.table, start: self.key(lowerEntries.last!.index, type: startEntryType), end: self.lowerBound(), keys: { key in + self.valueBox.range(self.table, start: self.key(lowerEntries.last!.index, type: startEntryType), end: self.lowerBound(), values: { key, value in let pinningIndexValue: UInt16 = key.getUInt16(0) let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index.messageIndex) { - additionalLowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) - } else { - additionalLowerEntries.append(.Nothing(index)) + var message: IntermediateMessage? + if value.length != 0 { + var idNamespace: Int32 = 0 + value.read(&idNamespace, offset: 0, length: 4) + var idId: Int32 = 0 + value.read(&idId, offset: 0, length: 4) + var indexTimestamp: Int32 = 0 + value.read(&indexTimestamp, offset: 0, length: 4) + + message = messageHistoryTable.getMessage(MessageIndex(id: MessageId(peerId: index.messageIndex.id.peerId, namespace: idNamespace, id: idId), timestamp: indexTimestamp)) } + additionalLowerEntries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { additionalLowerEntries.append(.Hole(ChatListHole(index: index.messageIndex))) } @@ -291,16 +344,23 @@ final class ChatListTable: Table { key = self.upperBound() } - self.valueBox.range(self.table, start: key, end: self.lowerBound(), keys: { key in + self.valueBox.range(self.table, start: key, end: self.lowerBound(), values: { key, value in let pinningIndexValue: UInt16 = key.getUInt16(0) let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index.messageIndex) { - entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) - } else { - entries.append(.Nothing(index)) + var message: IntermediateMessage? + if value.length != 0 { + var idNamespace: Int32 = 0 + value.read(&idNamespace, offset: 0, length: 4) + var idId: Int32 = 0 + value.read(&idId, offset: 0, length: 4) + var indexTimestamp: Int32 = 0 + value.read(&indexTimestamp, offset: 0, length: 4) + + message = messageHistoryTable.getMessage(MessageIndex(id: MessageId(peerId: index.messageIndex.id.peerId, namespace: idNamespace, id: idId), timestamp: indexTimestamp)) } + entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { entries.append(.Hole(ChatListHole(index: index.messageIndex))) } @@ -320,16 +380,23 @@ final class ChatListTable: Table { key = self.lowerBound() } - self.valueBox.range(self.table, start: key, end: self.upperBound(), keys: { key in + self.valueBox.range(self.table, start: key, end: self.upperBound(), values: { key, value in let pinningIndexValue: UInt16 = key.getUInt16(0) let index = ChatListIndex(pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), messageIndex: MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(2 + 4 + 4 + 4)), namespace: key.getInt32(2 + 4), id: key.getInt32(2 + 4 + 4)), timestamp: key.getInt32(2))) let type: Int8 = key.getInt8(2 + 4 + 4 + 4 + 8) if type == ChatListEntryType.Message.rawValue { - if let message = messageHistoryTable.getMessage(index.messageIndex) { - entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) - } else { - entries.append(.Nothing(index)) + var message: IntermediateMessage? + if value.length != 0 { + var idNamespace: Int32 = 0 + value.read(&idNamespace, offset: 0, length: 4) + var idId: Int32 = 0 + value.read(&idId, offset: 0, length: 4) + var indexTimestamp: Int32 = 0 + value.read(&indexTimestamp, offset: 0, length: 4) + + message = messageHistoryTable.getMessage(MessageIndex(id: MessageId(peerId: index.messageIndex.id.peerId, namespace: idNamespace, id: idId), timestamp: indexTimestamp)) } + entries.append(.Message(index, message, peerChatInterfaceStateTable.get(index.messageIndex.id.peerId)?.chatListEmbeddedState)) } else { entries.append(.Hole(ChatListHole(index: index.messageIndex))) } diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index b592d81ab8..7d34a5c498 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -1,34 +1,31 @@ import Foundation public enum ChatListEntry: Comparable { - case MessageEntry(ChatListIndex, Message, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) + case MessageEntry(ChatListIndex, Message?, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?, RenderedPeer) case HoleEntry(ChatListHole) - case Nothing(ChatListIndex) public var index: ChatListIndex { switch self { - case let .MessageEntry(index, _, _, _, _): + case let .MessageEntry(index, _, _, _, _, _): return index case let .HoleEntry(hole): return ChatListIndex(pinningIndex: nil, messageIndex: hole.index) - case let .Nothing(index): - return index } } } public func ==(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { switch lhs { - case let .MessageEntry(lhsIndex, lhsMessage, lhsReadState, lhsSettings, lhsEmbeddedState): + case let .MessageEntry(lhsIndex, lhsMessage, lhsReadState, lhsSettings, lhsEmbeddedState, lhsPeer): switch rhs { - case let .MessageEntry(rhsIndex, rhsMessage, rhsReadState, rhsSettings, rhsEmbeddedState): + case let .MessageEntry(rhsIndex, rhsMessage, rhsReadState, rhsSettings, rhsEmbeddedState, rhsPeer): if lhsIndex != rhsIndex { return false } if lhsReadState != rhsReadState { return false } - if lhsMessage.stableVersion != rhsMessage.stableVersion { + if lhsMessage?.stableVersion != rhsMessage?.stableVersion { return false } if let lhsSettings = lhsSettings, let rhsSettings = rhsSettings { @@ -45,6 +42,9 @@ public func ==(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { } else if (lhsEmbeddedState != nil) != (rhsEmbeddedState != nil) { return false } + if lhsPeer != rhsPeer { + return false + } return true default: return false @@ -55,12 +55,6 @@ public func ==(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { } else { return false } - case let .Nothing(index): - if case .Nothing(index) = rhs { - return true - } else { - return false - } } } @@ -69,21 +63,18 @@ public func <(lhs: ChatListEntry, rhs: ChatListEntry) -> Bool { } enum MutableChatListEntry: Equatable { - case IntermediateMessageEntry(ChatListIndex, IntermediateMessage, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) - case MessageEntry(ChatListIndex, Message, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) + case IntermediateMessageEntry(ChatListIndex, IntermediateMessage?, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?) + case MessageEntry(ChatListIndex, Message?, CombinedPeerReadState?, PeerNotificationSettings?, PeerChatListEmbeddedInterfaceState?, RenderedPeer) case HoleEntry(ChatListHole) - case Nothing(ChatListIndex) var index: ChatListIndex { switch self { case let .IntermediateMessageEntry(index, _, _, _, _): return index - case let .MessageEntry(index, _, _, _, _): + case let .MessageEntry(index, _, _, _, _, _): return index case let .HoleEntry(hole): return ChatListIndex(pinningIndex: nil, messageIndex: hole.index) - case let .Nothing(index): - return index } } } @@ -115,13 +106,6 @@ func ==(lhs: MutableChatListEntry, rhs: MutableChatListEntry) -> Bool { default: return false } - case .Nothing: - switch rhs { - case .Nothing: - return true - default: - return false - } } } @@ -150,11 +134,11 @@ final class MutableChatListView { func refreshDueToExternalTransaction(fetchAroundChatEntries: (_ index: ChatListIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?)) -> Bool { var index = ChatListIndex.absoluteUpperBound - if !self.entries.isEmpty { + if !self.entries.isEmpty && self.later != nil { index = self.entries[self.entries.count / 2].index } - var (entries, earlier, later) = fetchAroundChatEntries(index, self.entries.count) + let (entries, earlier, later) = fetchAroundChatEntries(index, self.entries.count) if entries != self.entries || earlier != self.earlier || later != self.later { self.entries = entries @@ -170,19 +154,15 @@ final class MutableChatListView { var hasChanges = false for operation in operations { switch operation { - case let .InsertMessage(index, message, combinedReadState, embeddedState): + case let .InsertEntry(index, message, combinedReadState, embeddedState): if self.add(.IntermediateMessageEntry(index, message, combinedReadState, nil, embeddedState)) { hasChanges = true } - case let .InsertNothing(index): - if self.add(.Nothing(index)) { - hasChanges = true - } case let .InsertHole(index): if self.add(.HoleEntry(index)) { hasChanges = true } - case let .RemoveMessage(indices): + case let .RemoveEntry(indices): if self.remove(Set(indices), holes: false, context: context) { hasChanges = true } @@ -196,13 +176,13 @@ final class MutableChatListView { for i in 0 ..< self.entries.count { switch self.entries[i] { case let .IntermediateMessageEntry(index, message, readState, _, embeddedState): - if let settings = updatedPeerNotificationSettings[message.id.peerId] { + if let settings = updatedPeerNotificationSettings[index.messageIndex.id.peerId] { self.entries[i] = .IntermediateMessageEntry(index, message, readState, settings, embeddedState) hasChanges = true } - case let .MessageEntry(index, message, readState, _, embeddedState): - if let settings = updatedPeerNotificationSettings[message.id.peerId] { - self.entries[i] = .MessageEntry(index, message, readState, settings, embeddedState) + case let .MessageEntry(index, message, readState, _, embeddedState, peer): + if let settings = updatedPeerNotificationSettings[index.messageIndex.id.peerId] { + self.entries[i] = .MessageEntry(index, message, readState, settings, embeddedState, peer) hasChanges = true } default: @@ -281,7 +261,7 @@ final class MutableChatListView { switch earlier { case .HoleEntry: match = holes - case .IntermediateMessageEntry, .MessageEntry, .Nothing: + case .IntermediateMessageEntry, .MessageEntry: match = !holes } if match { @@ -295,7 +275,7 @@ final class MutableChatListView { switch later { case .HoleEntry: match = holes - case .IntermediateMessageEntry, .MessageEntry, .Nothing: + case .IntermediateMessageEntry, .MessageEntry: match = !holes } if match { @@ -312,7 +292,7 @@ final class MutableChatListView { switch self.entries[i] { case .HoleEntry: match = holes - case .IntermediateMessageEntry, .MessageEntry, .Nothing: + case .IntermediateMessageEntry, .MessageEntry: match = !holes } if match { @@ -445,7 +425,7 @@ final class MutableChatListView { return hasChanges } - func render(_ renderMessage: (IntermediateMessage) -> Message, getPeerNotificationSettings: (PeerId) -> PeerNotificationSettings?) { + func render(_ renderMessage: (IntermediateMessage) -> Message, getPeer: (PeerId) -> Peer?, getPeerNotificationSettings: (PeerId) -> PeerNotificationSettings?) { for i in 0 ..< self.entries.count { switch self.entries[i] { case let .IntermediateMessageEntry(index, message, combinedReadState, notificationSettings, embeddedState): @@ -453,9 +433,26 @@ final class MutableChatListView { if let notificationSettings = notificationSettings { updatedNotificationSettings = notificationSettings } else { - updatedNotificationSettings = getPeerNotificationSettings(message.id.peerId) + updatedNotificationSettings = getPeerNotificationSettings(index.messageIndex.id.peerId) } - self.entries[i] = .MessageEntry(index, renderMessage(message), combinedReadState, updatedNotificationSettings, embeddedState) + let renderedMessage: Message? + if let message = message { + renderedMessage = renderMessage(message) + } else { + renderedMessage = nil + } + var peers = SimpleDictionary() + if let peer = getPeer(index.messageIndex.id.peerId) { + peers[peer.id] = peer + if let associatedPeerIds = peer.associatedPeerIds { + for associatedPeerId in associatedPeerIds { + if let associatedPeer = getPeer(associatedPeerId) { + peers[associatedPeer.id] = associatedPeer + } + } + } + } + self.entries[i] = .MessageEntry(index, renderedMessage, combinedReadState, updatedNotificationSettings, embeddedState, RenderedPeer(peerId: index.messageIndex.id.peerId, peers: peers)) default: break } @@ -472,10 +469,8 @@ public final class ChatListView { var entries: [ChatListEntry] = [] for entry in mutableView.entries { switch entry { - case let .MessageEntry(index, message, combinedReadState, notificationSettings, embeddedState): - entries.append(.MessageEntry(index, message, combinedReadState, notificationSettings, embeddedState)) - case let .Nothing(index): - entries.append(.Nothing(index)) + case let .MessageEntry(index, message, combinedReadState, notificationSettings, embeddedState, peer): + entries.append(.MessageEntry(index, message, combinedReadState, notificationSettings, embeddedState, peer)) case let .HoleEntry(hole): entries.append(.HoleEntry(hole)) case .IntermediateMessageEntry: diff --git a/Postbox/PeerChatListInclusion.swift b/Postbox/PeerChatListInclusion.swift index d6c2a7273d..47f3170e7a 100644 --- a/Postbox/PeerChatListInclusion.swift +++ b/Postbox/PeerChatListInclusion.swift @@ -1,7 +1,56 @@ import Foundation -public enum PeerChatListInclusion { +public enum PeerChatListInclusion: Equatable { + case notSpecified case never case ifHasMessages - case always(minTimestamp: Int32) + case ifHasMessagesOrOneOf(pinningIndex: UInt16?, minTimestamp: Int32?) + + public func withSetIfHasMessagesOrMaxMinTimestamp(_ minTimestamp: Int32) -> PeerChatListInclusion { + switch self { + case let .ifHasMessagesOrOneOf(pinningIndex, currentMinTimestamp): + var maxTimestamp: Int32 = minTimestamp + if let currentMinTimestamp = currentMinTimestamp, currentMinTimestamp > maxTimestamp { + maxTimestamp = currentMinTimestamp + } + return .ifHasMessagesOrOneOf(pinningIndex: pinningIndex, minTimestamp: maxTimestamp) + default: + return .ifHasMessagesOrOneOf(pinningIndex: nil, minTimestamp: minTimestamp) + } + } + + public static func ==(lhs: PeerChatListInclusion, rhs: PeerChatListInclusion) -> Bool { + switch lhs { + case .notSpecified: + if case .notSpecified = rhs { + return true + } else { + return false + } + case .never: + if case .never = rhs { + return true + } else { + return false + } + case .ifHasMessages: + if case .ifHasMessages = rhs { + return true + } else { + return false + } + case let .ifHasMessagesOrOneOf(lhsPinningIndex, lhsMinTimestamp): + if case let .ifHasMessagesOrOneOf(rhsPinningIndex, rhsMinTimestamp) = rhs { + if lhsPinningIndex != rhsPinningIndex { + return false + } + if lhsMinTimestamp != rhsMinTimestamp { + return false + } + return true + } else { + return false + } + } + } } diff --git a/Postbox/PeerNameIndexTable.swift b/Postbox/PeerNameIndexTable.swift index 5365b53ef3..26a91736b5 100644 --- a/Postbox/PeerNameIndexTable.swift +++ b/Postbox/PeerNameIndexTable.swift @@ -229,7 +229,7 @@ final class PeerNameIndexTable: Table { for peerId in peerIds { var foundInChats = false if categories.contains(.chats) { - if let index = chatListIndexTable.get(peerId) { + if let index = chatListIndexTable.get(peerId).includedIndex(peerId: peerId) { foundInChats = true chatIndices[peerId] = index } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 2cfbc94e81..77f22c4739 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -126,10 +126,25 @@ public final class Modifier { return self.postbox?.peerNotificationSettingsTable.get(id) } - public func updatePeers(_ peers: [Peer], update: (Peer, Peer) -> Peer?) { + public func updatePeersInternal(_ peers: [Peer], update: (Peer?, Peer) -> Peer?) { self.postbox?.updatePeers(peers, update: update) } + public func getPeerChatListInclusion(_ id: PeerId) -> PeerChatListInclusion { + if let postbox = self.postbox { + return postbox.getPeerChatListInclusion(id) + } + return .never + } + + public func getTopPeerMessageId(peerId: PeerId, namespace: MessageId.Namespace) -> MessageId? { + return self.postbox?.messageHistoryIndexTable.top(peerId, namespace: namespace)?.index.id + } + + public func updatePeerChatListInclusion(_ id: PeerId, inclusion: PeerChatListInclusion) { + self.postbox?.updatePeerChatListInclusion(id, inclusion: inclusion) + } + public func updatePeerNotificationSettings(_ notificationSettings: [PeerId: PeerNotificationSettings]) { self.postbox?.updatePeerNotificationSettings(notificationSettings) } @@ -297,6 +312,7 @@ public final class Postbox { private var peerPipes: [PeerId: ValuePipe] = [:] private var currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + private var currentUpdatedChatListInclusions: [PeerId: PeerChatListInclusion] = [:] private var currentUnsentOperations: [IntermediateMessageHistoryUnsentOperation] = [] private var currentUpdatedSynchronizeReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] private var currentUpdatedMedia: [MediaId: Media?] = [:] @@ -446,7 +462,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 10 + let currentUserVersion: Int32 = 11 if userVersion != currentUserVersion { self.valueBox.drop() @@ -857,33 +873,27 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(index, message, embeddedState): - entries.append(.IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState)) + entries.append(.IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(index.messageIndex.id.peerId), self.peerNotificationSettingsTable.get(index.messageIndex.id.peerId), embeddedState)) case let .Hole(hole): entries.append(.HoleEntry(hole)) - case let .Nothing(index): - entries.append(.Nothing(index)) } } if let intermediateLower = intermediateLower { switch intermediateLower { case let .Message(index, message, embeddedState): - lower = .IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState) + lower = .IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(index.messageIndex.id.peerId), self.peerNotificationSettingsTable.get(index.messageIndex.id.peerId), embeddedState) case let .Hole(hole): lower = .HoleEntry(hole) - case let .Nothing(index): - lower = .Nothing(index) } } if let intermediateUpper = intermediateUpper { switch intermediateUpper { case let .Message(index, message, embeddedState): - upper = .IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState) + upper = .IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(index.messageIndex.id.peerId), self.peerNotificationSettingsTable.get(index.messageIndex.id.peerId), embeddedState) case let .Hole(hole): upper = .HoleEntry(hole) - case let .Nothing(index): - upper = .Nothing(index) } } @@ -896,11 +906,9 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(index, message, embeddedState): - entries.append(.IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState)) + entries.append(.IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(index.messageIndex.id.peerId), self.peerNotificationSettingsTable.get(index.messageIndex.id.peerId), embeddedState)) case let .Hole(hole): entries.append(.HoleEntry(hole)) - case let .Nothing(index): - entries.append(.Nothing(index)) } } return entries @@ -912,9 +920,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(index, message, embeddedState): - entries.append(.IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(message.id.peerId), self.peerNotificationSettingsTable.get(message.id.peerId), embeddedState)) - case let .Nothing(index): - entries.append(.Nothing(index)) + entries.append(.IntermediateMessageEntry(index, message, self.readStateTable.getCombinedState(index.messageIndex.id.peerId), self.peerNotificationSettingsTable.get(index.messageIndex.id.peerId), embeddedState)) case let .Hole(index): entries.append(.HoleEntry(index)) } @@ -951,7 +957,7 @@ public final class Postbox { private func beforeCommit() -> (updatedTransactionStateVersion: Int64?, updatedMasterClientId: Int64?) { var chatListOperations: [ChatListOperation] = [] - self.chatListTable.replay(historyOperationsByPeerId: self.currentOperationsByPeerId, updatedPeerChatListEmbeddedStates: currentUpdatedPeerChatListEmbeddedStates, messageHistoryTable: self.messageHistoryTable, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable, operations: &chatListOperations) + self.chatListTable.replay(historyOperationsByPeerId: self.currentOperationsByPeerId, updatedPeerChatListEmbeddedStates: currentUpdatedPeerChatListEmbeddedStates, updatedChatListInclusions: self.currentUpdatedChatListInclusions, messageHistoryTable: self.messageHistoryTable, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable, operations: &chatListOperations) for (index, hole) in self.currentReplaceChatListHoles { self.chatListTable.replaceHole(index, hole: hole, operations: &chatListOperations) } @@ -979,6 +985,7 @@ public final class Postbox { } self.currentOperationsByPeerId.removeAll() + self.currentUpdatedChatListInclusions.removeAll() self.currentFilledHolesByPeerId.removeAll() self.currentRemovedHolesByPeerId.removeAll() self.currentUpdatedPeers.removeAll() @@ -1018,23 +1025,33 @@ public final class Postbox { return self.globallyUniqueMessageIdsTable.get(peerId: peerId, globallyUniqueId: id) } - fileprivate func updatePeers(_ peers: [Peer], update: (Peer, Peer) -> Peer?) { + fileprivate func updatePeers(_ peers: [Peer], update: (Peer?, Peer) -> Peer?) { for peer in peers { - if let currentPeer = self.peerTable.get(peer.id) { - if let updatedPeer = update(currentPeer, peer) { - self.peerTable.set(updatedPeer) - self.currentUpdatedPeers[updatedPeer.id] = updatedPeer - if currentPeer.indexName != updatedPeer.indexName { - self.peerNameIndexTable.markPeerNameUpdated(peerId: peer.id, name: updatedPeer.indexName) - } + let currentPeer = self.peerTable.get(peer.id) + if let updatedPeer = update(currentPeer, peer) { + self.peerTable.set(updatedPeer) + self.currentUpdatedPeers[updatedPeer.id] = updatedPeer + if currentPeer?.indexName != updatedPeer.indexName { + self.peerNameIndexTable.markPeerNameUpdated(peerId: peer.id, name: updatedPeer.indexName) } - } else { - self.peerTable.set(peer) - self.peerNameIndexTable.markPeerNameUpdated(peerId: peer.id, name: peer.indexName) } } } + fileprivate func getPeerChatListInclusion(_ id: PeerId) -> PeerChatListInclusion { + if let inclusion = self.currentUpdatedChatListInclusions[id] { + return inclusion + } else { + return self.chatListIndexTable.get(id).inclusion + } + } + + fileprivate func updatePeerChatListInclusion(_ id: PeerId, inclusion: PeerChatListInclusion) { + self.chatListTable.updateInclusion(peerId: id, updatedChatListInclusions: &self.currentUpdatedChatListInclusions, { _ in + return inclusion + }) + } + fileprivate func updatePeerNotificationSettings(_ notificationSettings: [PeerId: PeerNotificationSettings]) { for (peerId, settings) in notificationSettings { let currentSettings = self.peerNotificationSettingsTable.get(peerId) @@ -1329,7 +1346,9 @@ public final class Postbox { let (entries, earlier, later) = self.fetchAroundChatEntries(index, count: count) let mutableView = MutableChatListView(earlier: earlier, entries: entries, later: later, count: count) - mutableView.render(self.renderIntermediateMessage, getPeerNotificationSettings: { self.peerNotificationSettingsTable.get($0) }) + mutableView.render(self.renderIntermediateMessage, getPeer: { id in + return self.peerTable.get(id) + }, getPeerNotificationSettings: { self.peerNotificationSettingsTable.get($0) }) let (index, signal) = self.viewTracker.addChatListView(mutableView) diff --git a/Postbox/RenderedPeer.swift b/Postbox/RenderedPeer.swift new file mode 100644 index 0000000000..707fdbfa47 --- /dev/null +++ b/Postbox/RenderedPeer.swift @@ -0,0 +1,26 @@ +import Foundation + +public final class RenderedPeer: Equatable { + public let peerId: PeerId + public let peers: SimpleDictionary + + public init(peerId: PeerId, peers: SimpleDictionary) { + self.peerId = peerId + self.peers = peers + } + + public static func ==(lhs: RenderedPeer, rhs: RenderedPeer) -> Bool { + if lhs.peerId != rhs.peerId { + return false + } + if lhs.peers.count != rhs.peers.count { + return false + } + if !lhs.peers.isEqual(other: rhs.peers, with: { p1, p2 in + return p1.isEqual(p2) + }) { + return false + } + return true + } +} diff --git a/Postbox/SimpleDictionary.swift b/Postbox/SimpleDictionary.swift index 32ae469d43..704f5b7ee9 100644 --- a/Postbox/SimpleDictionary.swift +++ b/Postbox/SimpleDictionary.swift @@ -78,5 +78,21 @@ public struct SimpleDictionary: Sequence { return nil } } + + public func isEqual(other: SimpleDictionary, with f: (V, V) -> Bool) -> Bool { + if self.items.count != other.items.count { + return false + } + for i in 0 ..< self.items.count { + if self.items[i].0 != other.items[i].0 { + return false + } + if !f(self.items[i].1, other.items[i].1) { + return false + } + } + + return true + } } diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 583e1ebcaf..367a886212 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -270,7 +270,9 @@ final class ViewTracker { for (mutableView, pipe) in self.chatListViews.copyItems() { if mutableView.refreshDueToExternalTransaction(fetchAroundChatEntries: fetchAroundChatEntries) { - mutableView.render(self.renderMessage, getPeerNotificationSettings: self.getPeerNotificationSettings) + mutableView.render(self.renderMessage, getPeer: { id in + return self.getPeer(id) + }, getPeerNotificationSettings: self.getPeerNotificationSettings) pipe.putNext((ChatListView(mutableView), .Generic)) } } @@ -386,7 +388,9 @@ final class ViewTracker { let context = MutableChatListViewReplayContext() if mutableView.replay(transaction.chatListOperations, updatedPeerNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, context: context) { mutableView.complete(context: context, fetchEarlier: self.fetchEarlierChatEntries, fetchLater: self.fetchLaterChatEntries) - mutableView.render(self.renderMessage, getPeerNotificationSettings: self.getPeerNotificationSettings) + mutableView.render(self.renderMessage, getPeer: { id in + return self.getPeer(id) + }, getPeerNotificationSettings: self.getPeerNotificationSettings) //var updateType: ViewUpdateType = .Generic for operation in transaction.chatListOperations { if case .RemoveHoles = operation { diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index 02507ecddb..21ec3f1544 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -13,16 +13,13 @@ private let namespace: Int32 = 1 private let authorPeerId = PeerId(namespace: 2, id: 3) private enum Entry: Equatable, CustomStringConvertible { - case Message(Int32, Int32, Int32) - case Nothing(Int32, Int32, Int32) + case Message(Int32, Int32, Int32, Bool) case Hole(Int32, Int32, Int32) var description: String { switch self { - case let .Message(peerId, id, timestamp): - return "Message(\(peerId), \(id), \(timestamp))" - case let .Nothing(peerId, id, timestamp): - return "Nothing(\(peerId), \(id), \(timestamp))" + case let .Message(peerId, id, timestamp, exists): + return "Message(\(peerId), \(id), \(timestamp), \(exists))" case let .Hole(peerId, id, timestamp): return "Hole(\(peerId), \(id), \(timestamp))" } @@ -31,21 +28,10 @@ private enum Entry: Equatable, CustomStringConvertible { private func ==(lhs: Entry, rhs: Entry) -> Bool { switch lhs { - case let .Message(lhsPeerId, lhsId, lhsTimestamp): + case let .Message(lhsPeerId, lhsId, lhsTimestamp, lhsExists): switch rhs { - case let .Message(rhsPeerId, rhsId, rhsTimestamp): - return lhsPeerId == rhsPeerId && lhsId == rhsId && lhsTimestamp == rhsTimestamp - case .Nothing: - return false - case .Hole: - return false - } - case let .Nothing(lhsPeerId, lhsId, lhsTimestamp): - switch rhs { - case .Message: - return false - case let .Nothing(rhsPeerId, rhsId, rhsTimestamp): - return lhsPeerId == rhsPeerId && lhsId == rhsId && lhsTimestamp == rhsTimestamp + case let .Message(rhsPeerId, rhsId, rhsTimestamp, rhsExists): + return lhsPeerId == rhsPeerId && lhsId == rhsId && lhsTimestamp == rhsTimestamp && lhsExists == rhsExists case .Hole: return false } @@ -53,8 +39,6 @@ private func ==(lhs: Entry, rhs: Entry) -> Bool { switch rhs { case .Message: return false - case .Nothing: - return false case let .Hole(rhsPeerId, rhsId, rhsTimestamp): return lhsPeerId == rhsPeerId && lhsId == rhsId && lhsTimestamp == rhsTimestamp } @@ -137,7 +121,31 @@ class ChatListTableTests: XCTestCase { let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), globallyUniqueId: nil, timestamp: timestamp, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: text, attributes: [], media: media)], location: .Random, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] - self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) + self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, updatedChatListInclusions: [:], messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) + var updatedTotalUnreadCount: Int32? + self.chatListIndexTable?.commitWithTransactionUnreadCountDeltas([:], transactionParticipationInTotalUnreadCountUpdates: ([], []), getPeer: { _ in + return nil + }, updatedTotalUnreadCount: &updatedTotalUnreadCount) + self.chatListIndexTable?.clearMemoryCache() + } + + private func updateInclusion(_ peerId: Int32, f: (PeerChatListInclusion) -> PeerChatListInclusion) { + var updatedChatListInclusions: [PeerId: PeerChatListInclusion] = [:] + self.chatListTable!.updateInclusion(peerId: PeerId(namespace: namespace, id: peerId), updatedChatListInclusions: &updatedChatListInclusions, f) + + var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] + var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] + var updatedPeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] + let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] + + var operations: [ChatListOperation] = [] + + self.chatListTable!.replay(historyOperationsByPeerId: [:], updatedPeerChatListEmbeddedStates: [:], updatedChatListInclusions: updatedChatListInclusions, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) + var updatedTotalUnreadCount: Int32? + self.chatListIndexTable?.commitWithTransactionUnreadCountDeltas([:], transactionParticipationInTotalUnreadCountUpdates: ([], []), getPeer: { _ in + return nil + }, updatedTotalUnreadCount: &updatedTotalUnreadCount) + self.chatListIndexTable?.clearMemoryCache() } private func addHole(_ peerId: Int32, _ id: Int32) { @@ -147,7 +155,7 @@ class ChatListTableTests: XCTestCase { let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] self.historyTable!.addHoles([MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id)], operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] - self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) + self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, updatedChatListInclusions: [:], messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) } private func addChatListHole(_ peerId: Int32, _ id: Int32, _ timestamp: Int32) { @@ -172,7 +180,7 @@ class ChatListTableTests: XCTestCase { let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] self.historyTable!.removeMessages(ids.map({ MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] - self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) + self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, updatedChatListInclusions: [:], messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) } private func fillHole(_ peerId: Int32, _ id: Int32, _ fillType: HoleFill, _ messages: [(Int32, Int32, String, [Media])]) { @@ -182,21 +190,19 @@ class ChatListTableTests: XCTestCase { let updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?] = [:] self.historyTable!.fillHole(MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: id), fillType: fillType, tagMask: nil, messages: messages.map({ StoreMessage(id: MessageId(peerId: PeerId(namespace: namespace, id: peerId), namespace: namespace, id: $0.0), globallyUniqueId: nil, timestamp: $0.1, flags: [], tags: [], forwardInfo: nil, authorId: authorPeerId, text: $0.2, attributes: [], media: $0.3) }), operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) var operations: [ChatListOperation] = [] - self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) + self.chatListTable!.replay(historyOperationsByPeerId: operationsByPeerId, updatedPeerChatListEmbeddedStates: updatedPeerChatListEmbeddedStates, updatedChatListInclusions: [:], messageHistoryTable: self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!, operations: &operations) } private func expectEntries(_ entries: [Entry]) { let actualEntries = self.chatListTable!.debugList(self.historyTable!, peerChatInterfaceStateTable: self.peerChatInterfaceStateTable!).map({ entry -> Entry in switch entry { - case let .Message(_, message, _): - if message.authorId != authorPeerId { - XCTFail("Expected authorId \(authorPeerId), actual \(message.authorId)") + case let .Message(index, message, _): + if let message = message, message.authorId != authorPeerId { + XCTFail("Expected authorId \(authorPeerId), actual \(String(describing: message.authorId))") } - return .Message(message.id.peerId.id, message.id.id, message.timestamp) + return .Message(index.messageIndex.id.peerId.id, index.messageIndex.id.id, index.messageIndex.timestamp, message != nil) case let .Hole(hole): return .Hole(hole.index.id.peerId.id, hole.index.id.id, hole.index.timestamp) - case let .Nothing(index): - return .Nothing(index.messageIndex.id.peerId.id, index.messageIndex.id.id, index.messageIndex.timestamp) } }) if entries != actualEntries { @@ -209,78 +215,184 @@ class ChatListTableTests: XCTestCase { } func testAddSingleMessage() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + addMessage(1, 100, 100) - expectEntries([.Message(1, 100, 100)]) + expectEntries([.Message(1, 100, 100, true)]) } func testInsertLaterMessage() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + addMessage(1, 100, 100) addMessage(1, 200, 200) - expectEntries([.Message(1, 200, 200)]) + expectEntries([.Message(1, 200, 200, true)]) } func testInsertEarlierMessage() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + addMessage(1, 100, 100) addMessage(1, 10, 20) - expectEntries([.Message(1, 100, 100)]) + expectEntries([.Message(1, 100, 100, true)]) } func testInsertTwoChatMessages() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + updateInclusion(2, f: { _ in + return .ifHasMessages + }) + addMessage(1, 100, 100) addMessage(2, 10, 20) - expectEntries([.Message(2, 10, 20), .Message(1, 100, 100)]) + expectEntries([.Message(2, 10, 20, true), .Message(1, 100, 100, true)]) } func testMoveChatUpper() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + updateInclusion(2, f: { _ in + return .ifHasMessages + }) + addMessage(1, 100, 100) addMessage(2, 10, 20) addMessage(2, 120, 120) - expectEntries([.Message(1, 100, 100), .Message(2, 120, 120)]) + expectEntries([.Message(1, 100, 100, true), .Message(2, 120, 120, true)]) } - func testRemoveSingleMessage() { + func testRemoveSingleMessageInclusionIfHasMessages() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) addMessage(1, 100, 100) removeMessages(1, [100]) - expectEntries([.Nothing(1, 100, 100)]) + expectEntries([]) + //expectEntries([.Message(1, 100, 100, false)]) + } + + func testAddSingleMessageInclusionNever() { + addMessage(1, 100, 100) + expectEntries([]) + } + + func testEmptyWithInclusionMinIndex() { + updateInclusion(1, f: { _ in + return .ifHasMessagesOrOneOf(pinningIndex: nil, minTimestamp: 50) + }) + expectEntries([.Message(1, 0, 50, false)]) + addMessage(1, 20, 20) + expectEntries([.Message(1, 0, 50, true)]) + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + expectEntries([.Message(1, 20, 20, true)]) + } + + func testEmptyWithInclusionPinningIndex() { + updateInclusion(1, f: { _ in + return .ifHasMessagesOrOneOf(pinningIndex: 0, minTimestamp: nil) + }) + expectEntries([.Message(1, 0, 0, false)]) + } + + func testRemoveSingleMessageInclusionWithMinIndexUpdate() { + updateInclusion(1, f: { _ in + return .ifHasMessagesOrOneOf(pinningIndex: nil, minTimestamp: 50) + }) + addMessage(1, 100, 100) + removeMessages(1, [100]) + expectEntries([.Message(1, 0, 50, false)]) + updateInclusion(1, f: { _ in + return .ifHasMessagesOrOneOf(pinningIndex: nil, minTimestamp: 200) + }) + expectEntries([.Message(1, 0, 200, false)]) + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + expectEntries([]) + addMessage(1, 200, 200) + expectEntries([.Message(1, 200, 200, true)]) } func testOverrideNothing() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + addMessage(1, 100, 100) removeMessages(1, [100]) addMessage(1, 100, 100) - expectEntries([.Message(1, 100, 100)]) + expectEntries([.Message(1, 100, 100, true)]) } func testInsertHoleIntoEmpty() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + addChatListHole(1, 10, 10) expectEntries([.Hole(1, 10, 10)]) } func testInsertHoleLower() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + addMessage(1, 100, 100) addChatListHole(1, 10, 10) - expectEntries([.Hole(1, 10, 10), .Message(1, 100, 100)]) + expectEntries([.Hole(1, 10, 10), .Message(1, 100, 100, true)]) } func testInsertHoleUpper() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + addMessage(1, 100, 100) addChatListHole(1, 200, 200) - expectEntries([.Message(1, 100, 100), .Hole(1, 200, 200)]) + expectEntries([.Message(1, 100, 100, true), .Hole(1, 200, 200)]) } func testIgnoreRemoveHole() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + addChatListHole(1, 100, 100) removeMessages(1, [100]) expectEntries([.Hole(1, 100, 100)]) addMessage(1, 100, 100) - expectEntries([.Message(1, 100, 100), .Hole(1, 100, 100)]) + expectEntries([.Message(1, 100, 100, true), .Hole(1, 100, 100)]) removeMessages(1, [100]) - expectEntries([.Nothing(1, 100, 100), .Hole(1, 100, 100)]) + expectEntries([.Hole(1, 100, 100)]) + + updateInclusion(1, f: { _ in + return .ifHasMessagesOrOneOf(pinningIndex: nil, minTimestamp: 100) + }) + expectEntries([.Message(1, 0, 100, false), .Hole(1, 100, 100)]) } func testReplaceHoleWithHole() { + updateInclusion(1, f: { _ in + return .ifHasMessages + }) + updateInclusion(2, f: { _ in + return .ifHasMessages + }) + addChatListHole(1, 100, 100) replaceChatListHole(1, 100, 100, 2, 200, 200) expectEntries([.Hole(2, 200, 200)]) @@ -291,4 +403,16 @@ class ChatListTableTests: XCTestCase { removeChatListHole(1, 100, 100) expectEntries([]) } + + func testInclusionUpdate() { + //expectEntries([.Message(1, 0, 1, false)]) + addMessage(1, 100, 100) + //expectEntries([.Message(1, 100, 100, true)]) + addMessage(1, 200, 200) + updateInclusion(1, f: { _ in + return .ifHasMessagesOrOneOf(pinningIndex: nil, minTimestamp: 1) + }) + addMessage(1, 300, 300) + expectEntries([.Message(1, 300, 300, true)]) + } } From 9f89633d03f980290166210bca4ef4ee3e90dd3c Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 31 Jan 2017 15:14:32 +0300 Subject: [PATCH 078/427] no message --- Postbox/MessageHistoryTable.swift | 15 +++++++++++++++ Postbox/Postbox.swift | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index b26e9a6e20..9f67a1e93c 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -293,6 +293,11 @@ final class MessageHistoryTable: Table { } } + func clearHistory(peerId: PeerId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + let indices = self.allIndices(peerId) + self.removeMessages(indices.map { $0.id }, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + func fillHole(_ id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.fillHole(id, fillType: fillType, tagMask: tagMask, messages: self.internalStoreMessages(messages), operations: &operations) @@ -1669,6 +1674,16 @@ final class MessageHistoryTable: Table { return messageIds } + + func allIndices(_ peerId: PeerId) -> [MessageIndex] { + var indices: [MessageIndex] = [] + self.valueBox.range(self.table, start: self.key(MessageIndex.lowerBound(peerId: peerId)).predecessor, end: self.key(MessageIndex.upperBound(peerId: peerId)).successor, keys: { key in + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4), id: key.getInt32(8 + 4 + 4)), timestamp: key.getInt32(8)) + indices.append(index) + return true + }, limit: 0) + return indices + } func debugList(_ peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 77f22c4739..5d36794e2e 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -47,6 +47,10 @@ public final class Modifier { self.postbox?.deleteMessages(messageIds) } + public func clearHistory(_ peerId: PeerId) { + self.postbox?.clearHistory(peerId) + } + public func deleteMessagesWithGlobalIds(_ ids: [Int32]) { if let postbox = self.postbox { let messageIds = postbox.messageIdsForGlobalIds(ids) @@ -753,6 +757,10 @@ public final class Postbox { self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } + fileprivate func clearHistory(_ peerId: PeerId) { + self.messageHistoryTable.clearHistory(peerId: peerId, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + } + fileprivate func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]]) { self.messageHistoryTable.resetIncomingReadStates(states, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } From 51c4d15c9f6c16c0829dfebfa5cc180e1cbdaf48 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 31 Jan 2017 18:08:16 +0300 Subject: [PATCH 079/427] no message --- Postbox/ChatListTable.swift | 25 ++++++++++++++++++++++ Postbox/Message.swift | 8 +++++-- Postbox/MessageHistoryMetadataTable.swift | 3 +++ Postbox/MessageHistoryReadStateTable.swift | 8 +++---- Postbox/MessageHistoryTable.swift | 16 ++++++++++---- Postbox/MessageHistoryView.swift | 2 +- Postbox/PeerChatListInclusion.swift | 18 ++++++++++++++++ Postbox/Postbox.swift | 16 ++++++++++++++ 8 files changed, 85 insertions(+), 11 deletions(-) diff --git a/Postbox/ChatListTable.swift b/Postbox/ChatListTable.swift index daea865d32..7ad2267924 100644 --- a/Postbox/ChatListTable.swift +++ b/Postbox/ChatListTable.swift @@ -107,6 +107,31 @@ final class ChatListTable: Table { } } + func getPinnedPeerIds() -> [PeerId] { + var peerIds: [PeerId] = [] + self.valueBox.range(self.table, start: self.upperBound(), end: self.key(ChatListIndex(pinningIndex: UInt16.max - 1, messageIndex: MessageIndex.absoluteUpperBound()), type: ChatListEntryType.Message).successor, keys: { key in + peerIds.append(PeerId(key.getInt64(2 + 4 + 4 + 4))) + return true + }, limit: 0) + return peerIds + } + + func setPinnedPeerIds(peerIds: [PeerId], updatedChatListInclusions: inout [PeerId: PeerChatListInclusion]) { + let updatedIds = Set(peerIds) + for peerId in self.getPinnedPeerIds() { + if !updatedIds.contains(peerId) { + self.updateInclusion(peerId: peerId, updatedChatListInclusions: &updatedChatListInclusions, { inclusion in + return inclusion.withoutPinningIndex() + }) + } + } + for i in 0 ..< peerIds.count { + self.updateInclusion(peerId: peerIds[i], updatedChatListInclusions: &updatedChatListInclusions, { inclusion in + return inclusion.withPinningIndex(UInt16(i)) + }) + } + } + func replay(historyOperationsByPeerId: [PeerId : [MessageHistoryOperation]], updatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], updatedChatListInclusions: [PeerId: PeerChatListInclusion], messageHistoryTable: MessageHistoryTable, peerChatInterfaceStateTable: PeerChatInterfaceStateTable, operations: inout [ChatListOperation]) { self.ensureInitialized() var changedPeerIds = Set() diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 16e3951c7a..308e123371 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -180,11 +180,15 @@ public struct ChatListIndex: Comparable, Hashable { public static func <(lhs: ChatListIndex, rhs: ChatListIndex) -> Bool { if let lhsPinningIndex = lhs.pinningIndex, let rhsPinningIndex = rhs.pinningIndex { - if lhsPinningIndex < rhsPinningIndex { + if lhsPinningIndex > rhsPinningIndex { return true - } else if lhsPinningIndex > rhsPinningIndex { + } else if lhsPinningIndex < rhsPinningIndex { return false } + } else if lhs.pinningIndex != nil { + return false + } else if rhs.pinningIndex != nil { + return true } return lhs.messageIndex < rhs.messageIndex } diff --git a/Postbox/MessageHistoryMetadataTable.swift b/Postbox/MessageHistoryMetadataTable.swift index 4318505eca..7ebe43becb 100644 --- a/Postbox/MessageHistoryMetadataTable.swift +++ b/Postbox/MessageHistoryMetadataTable.swift @@ -33,6 +33,9 @@ final class MessageHistoryMetadataTable: Table { private var nextPeerOperationLogIndex: UInt32? private var nextPeerOperationLogIndexUpdated = false + private var currentPinnedChatPeerIds: Set? + private var currentPinnedChatPeerIdsUpdated = false + private func peerHistoryInitializedKey(_ id: PeerId) -> ValueBoxKey { self.sharedPeerHistoryInitializedKey.setInt64(0, value: id.toInt64()) self.sharedPeerHistoryInitializedKey.setInt8(8, value: MetadataPrefix.PeerHistoryInitialized.rawValue) diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index c339161f22..d9f8157c0b 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -241,7 +241,7 @@ final class MessageHistoryReadStateTable: Table { } } - func applyIncomingMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: MessageId.Id?) -> (CombinedPeerReadState?, Bool) { + func applyIncomingMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: (MessageId.Id, Bool)?) -> (CombinedPeerReadState?, Bool) { if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] { if traceReadStates { print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId) (before: \(states.namespaces))") @@ -249,14 +249,14 @@ final class MessageHistoryReadStateTable: Table { switch state { case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count): - if maxIncomingReadId < messageId.id || (messageId.id == topMessageId && state.count != 0) { + if maxIncomingReadId < messageId.id || (topMessageId != nil && (messageId.id == topMessageId!.0 || topMessageId!.1) && state.count != 0) { var (deltaCount, holes) = incomingStatsInRange(maxIncomingReadId + 1, messageId.id) if traceReadStates { print("[ReadStateTable] applyMaxReadId after deltaCount: \(deltaCount), holes: \(holes)") } - if messageId.id == topMessageId { + if let topMessageId = topMessageId, (messageId.id == topMessageId.0 || topMessageId.1) { if deltaCount != Int(state.count) { deltaCount = Int(state.count) holes = true @@ -351,7 +351,7 @@ final class MessageHistoryReadStateTable: Table { return (nil, false, []) } - func applyInteractiveMaxReadIndex(_ messageIndex: MessageIndex, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), incomingIndexStatsInRange: (MessageIndex, MessageIndex) -> (count: Int, holes: Bool, readMesageIds: [MessageId]), topMessageId: MessageId.Id?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult, readMesageIds: [MessageId]) { + func applyInteractiveMaxReadIndex(_ messageIndex: MessageIndex, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), incomingIndexStatsInRange: (MessageIndex, MessageIndex) -> (count: Int, holes: Bool, readMesageIds: [MessageId]), topMessageId: (MessageId.Id, Bool)?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult, readMesageIds: [MessageId]) { if let states = self.get(messageIndex.id.peerId), let state = states.namespaces[messageIndex.id.namespace] { switch state { case .idBased: diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 9f67a1e93c..bec654d7b8 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -362,9 +362,13 @@ final class MessageHistoryTable: Table { } func applyIncomingReadMaxId(_ messageId: MessageId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { - var topMessageId: MessageId.Id? + var topMessageId: (MessageId.Id, Bool)? if let topEntry = self.messageHistoryIndexTable.top(messageId.peerId, namespace: messageId.namespace), case let .Message(index) = topEntry { - topMessageId = index.id.id + if let message = self.getMessage(index) { + topMessageId = (index.id.id, !message.flags.contains(.Incoming)) + } else { + topMessageId = (index.id.id, false) + } } let (combinedState, invalidated) = self.readStateTable.applyIncomingMaxReadId(messageId, incomingStatsInRange: { fromId, toId in @@ -421,9 +425,13 @@ final class MessageHistoryTable: Table { } func applyInteractiveMaxReadIndex(_ messageIndex: MessageIndex, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [MessageId] { - var topMessageId: MessageId.Id? + var topMessageId: (MessageId.Id, Bool)? if let topEntry = self.messageHistoryIndexTable.top(messageIndex.id.peerId, namespace: messageIndex.id.namespace), case let .Message(index) = topEntry { - topMessageId = index.id.id + if let message = self.getMessage(index) { + topMessageId = (index.id.id, !message.flags.contains(.Incoming)) + } else { + topMessageId = (index.id.id, false) + } } let (combinedState, result, messageIds) = self.readStateTable.applyInteractiveMaxReadIndex(messageIndex, incomingStatsInRange: { fromId, toId in diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index ac9dc7aebc..8aa7c48a48 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -821,7 +821,7 @@ public final class MessageHistoryView { self.entries = entries var topTaggedMessages: [Message] = [] - for (namespace, message) in mutableView.topTaggedMessages { + for (_, message) in mutableView.topTaggedMessages { if let message = message { switch message { case let .message(message): diff --git a/Postbox/PeerChatListInclusion.swift b/Postbox/PeerChatListInclusion.swift index 47f3170e7a..f018146aa4 100644 --- a/Postbox/PeerChatListInclusion.swift +++ b/Postbox/PeerChatListInclusion.swift @@ -19,6 +19,24 @@ public enum PeerChatListInclusion: Equatable { } } + public func withPinningIndex(_ pinningIndex: UInt16?) -> PeerChatListInclusion { + switch self { + case let .ifHasMessagesOrOneOf(_, minTimestamp): + return .ifHasMessagesOrOneOf(pinningIndex: pinningIndex, minTimestamp: minTimestamp) + default: + return .ifHasMessagesOrOneOf(pinningIndex: pinningIndex, minTimestamp: nil) + } + } + + public func withoutPinningIndex() -> PeerChatListInclusion { + switch self { + case let .ifHasMessagesOrOneOf(_, minTimestamp): + return .ifHasMessagesOrOneOf(pinningIndex: nil, minTimestamp: minTimestamp) + default: + return self + } + } + public static func ==(lhs: PeerChatListInclusion, rhs: PeerChatListInclusion) -> Bool { switch lhs { case .notSpecified: diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 5d36794e2e..0b0d62032b 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -276,6 +276,18 @@ public final class Modifier { public func updatePreferencesEntry(key: ValueBoxKey, _ f: (PreferencesEntry?) -> PreferencesEntry?) { self.postbox?.setPreferencesEntry(key: key, value: f(self.postbox?.getPreferencesEntry(key: key))) } + + public func getPinnedPeerIds() -> [PeerId] { + if let postbox = self.postbox { + return postbox.chatListTable.getPinnedPeerIds() + } else { + return [] + } + } + + public func setPinnedPeerIds(_ peerIds: [PeerId]) { + self.postbox?.setPinnedPeerIds(peerIds) + } } fileprivate class PipeNotifier: NSObject { @@ -1060,6 +1072,10 @@ public final class Postbox { }) } + fileprivate func setPinnedPeerIds(_ peerIds: [PeerId]) { + self.chatListTable.setPinnedPeerIds(peerIds: peerIds, updatedChatListInclusions: &self.currentUpdatedChatListInclusions) + } + fileprivate func updatePeerNotificationSettings(_ notificationSettings: [PeerId: PeerNotificationSettings]) { for (peerId, settings) in notificationSettings { let currentSettings = self.peerNotificationSettingsTable.get(peerId) From e1195986b906f4c6787bae9b5cfab910c96668a8 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 2 Feb 2017 11:47:26 +0300 Subject: [PATCH 080/427] no message --- Postbox.xcodeproj/project.pbxproj | 12 +- Postbox/Coding.swift | 127 +++++++++-- Postbox/ItemCollection.swift | 7 +- Postbox/ItemCollectionItemTable.swift | 134 +++++++++-- Postbox/PeerNameIndexTable.swift | 65 +++++- Postbox/PeerNameTokenIndexTable.swift | 211 ----------------- Postbox/Postbox.swift | 17 +- Postbox/ReverseIndexReferenceTable.swift | 274 +++++++++++++++++++++++ Postbox/ValueBoxKey.swift | 6 + 9 files changed, 576 insertions(+), 277 deletions(-) delete mode 100644 Postbox/PeerNameTokenIndexTable.swift create mode 100644 Postbox/ReverseIndexReferenceTable.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index d4344efb1a..12c9d23fdf 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -70,7 +70,7 @@ D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; D07827C11E0079CB00071108 /* StringIndexTokens.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C01E0079CB00071108 /* StringIndexTokens.swift */; }; - D07827C31E008F7300071108 /* PeerNameTokenIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */; }; + D07827C31E008F7300071108 /* ReverseIndexReferenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C21E008F7300071108 /* ReverseIndexReferenceTable.swift */; }; D07827C51E00B23F00071108 /* PeerNameIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */; }; D07CFF811DCA765D00761F81 /* PeerChatInterfaceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */; }; D07CFF831DCA909100761F81 /* PeerChatInterfaceStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */; }; @@ -207,7 +207,7 @@ D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopIndexableMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */; }; D0F7B1DF1E045C6A007EB8A5 /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; D0F7B1E01E045C6A007EB8A5 /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; - D0F7B1E11E045C6A007EB8A5 /* PeerNameTokenIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */; }; + D0F7B1E11E045C6A007EB8A5 /* ReverseIndexReferenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C21E008F7300071108 /* ReverseIndexReferenceTable.swift */; }; D0F7B1E21E045C6A007EB8A5 /* PeerNameIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */; }; D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */; }; @@ -276,7 +276,7 @@ D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; D07827C01E0079CB00071108 /* StringIndexTokens.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = StringIndexTokens.swift; sourceTree = ""; }; - D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameTokenIndexTable.swift; sourceTree = ""; }; + D07827C21E008F7300071108 /* ReverseIndexReferenceTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ReverseIndexReferenceTable.swift; sourceTree = ""; }; D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNameIndexTable.swift; sourceTree = ""; }; D07CFF801DCA765D00761F81 /* PeerChatInterfaceState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatInterfaceState.swift; sourceTree = ""; }; D07CFF821DCA909100761F81 /* PeerChatInterfaceStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatInterfaceStateTable.swift; sourceTree = ""; }; @@ -477,7 +477,7 @@ D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */, D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */, D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */, - D07827C21E008F7300071108 /* PeerNameTokenIndexTable.swift */, + D07827C21E008F7300071108 /* ReverseIndexReferenceTable.swift */, D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */, D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */, D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */, @@ -866,7 +866,7 @@ D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, D0F7B1E01E045C6A007EB8A5 /* ItemCacheTable.swift in Sources */, - D0F7B1E11E045C6A007EB8A5 /* PeerNameTokenIndexTable.swift in Sources */, + D0F7B1E11E045C6A007EB8A5 /* ReverseIndexReferenceTable.swift in Sources */, D0B418241D7DFE0C004562A4 /* SimpleSet.swift in Sources */, D073CE7B1DCBF3B4007511FD /* PeerNameIndexRepresentation.swift in Sources */, D073CE7D1DCBF3B4007511FD /* PeerPresence.swift in Sources */, @@ -942,7 +942,7 @@ D098C6F11D7E11E9007784E4 /* Database.swift in Sources */, D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */, - D07827C31E008F7300071108 /* PeerNameTokenIndexTable.swift in Sources */, + D07827C31E008F7300071108 /* ReverseIndexReferenceTable.swift in Sources */, D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */, diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 45ea6857f4..fd92f1dd08 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -185,7 +185,7 @@ public final class ReadBuffer: MemoryBuffer { return Data(bytesNoCopy: self.memory.assumingMemoryBound(to: UInt8.self), count: self.length, deallocator: .none) } - public func read(_ data: UnsafeMutablePointer, offset: Int, length: Int) { + public func read(_ data: UnsafeMutableRawPointer, offset: Int, length: Int) { memcpy(data + offset, self.memory.advanced(by: self.offset), length) self.offset += length } @@ -216,6 +216,8 @@ private enum ValueType: Int8 { case ObjectDictionary = 9 case Bytes = 10 case Nil = 11 + case StringArray = 12 + case BytesArray = 13 } public final class Encoder { @@ -326,7 +328,7 @@ public final class Encoder { var length: Int32 = Int32(value.count) self.buffer.write(&length, offset: 0, length: 4) value.withUnsafeBufferPointer { (data: UnsafeBufferPointer) -> Void in - self.buffer.write(UnsafePointer(data.baseAddress!), offset: 0, length: Int(length) * 4) + self.buffer.write(UnsafeRawPointer(data.baseAddress!), offset: 0, length: Int(length) * 4) return } } @@ -338,7 +340,7 @@ public final class Encoder { var length: Int32 = Int32(value.count) self.buffer.write(&length, offset: 0, length: 4) value.withUnsafeBufferPointer { (data: UnsafeBufferPointer) -> Void in - self.buffer.write(UnsafePointer(data.baseAddress!), offset: 0, length: Int(length) * 8) + self.buffer.write(UnsafeRawPointer(data.baseAddress!), offset: 0, length: Int(length) * 8) return } } @@ -363,6 +365,35 @@ public final class Encoder { } } + public func encodeStringArray(_ value: [String], forKey key: StaticString) { + self.encodeKey(key) + var type: Int8 = ValueType.StringArray.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var length: Int32 = Int32(value.count) + self.buffer.write(&length, offset: 0, length: 4) + + for object in value { + let data = object.data(using: .utf8, allowLossyConversion: true)! + var length: Int32 = Int32(data.count) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(data) + } + } + + public func encodeBytesArray(_ value: [MemoryBuffer], forKey key: StaticString) { + self.encodeKey(key) + var type: Int8 = ValueType.BytesArray.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var length: Int32 = Int32(value.count) + self.buffer.write(&length, offset: 0, length: 4) + + for object in value { + var length: Int32 = Int32(object.length) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(object.memory, offset: 0, length: object.length) + } + } + public func encodeGenericObjectArray(_ value: [Coding], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.ObjectArray.rawValue @@ -383,7 +414,7 @@ public final class Encoder { } } - public func encodeObjectDictionary(_ value: [K : V], forKey key: StaticString) { + public func encodeObjectDictionary(_ value: [K : V], forKey key: StaticString) where K: Coding, K: Hashable { self.encodeKey(key) var type: Int8 = ValueType.ObjectDictionary.rawValue self.buffer.write(&type, offset: 0, length: 1) @@ -506,6 +537,17 @@ public final class Decoder { offset += 4 + Int(length) case .Nil: break + case .StringArray, .BytesArray: + var length: Int32 = 0 + memcpy(&length, bytes + offset, 4) + offset += 4 + var i: Int32 = 0 + while i < length { + var stringLength: Int32 = 0 + memcpy(&stringLength, bytes + offset, 4) + offset += 4 + Int(stringLength) + i += 1 + } } } @@ -747,7 +789,7 @@ public final class Decoder { } } - public func decodeObjectArrayWithDecoderForKey(_ key: StaticString) -> [T] { + public func decodeObjectArrayWithDecoderForKey(_ key: StaticString) -> [T] where T: Coding { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) @@ -756,7 +798,6 @@ public final class Decoder { var array: [T] = [] array.reserveCapacity(Int(length)) - let failed = false var i: Int32 = 0 while i < length { var typeHash: Int32 = 0 @@ -769,24 +810,76 @@ public final class Decoder { let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(objectLength), freeWhenDone: false)) self.offset += 4 + Int(objectLength) - if !failed { - array.append(T(decoder: innerDecoder)) - } + array.append(T(decoder: innerDecoder)) i += 1 } - if failed { - return [] - } else { - return array - } + return array } else { return [] } } - public func decodeObjectArrayForKey(_ key: StaticString) -> [T] { + public func decodeStringArrayForKey(_ key: StaticString) -> [String] { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .StringArray) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var array: [String] = [] + array.reserveCapacity(Int(length)) + + var i: Int32 = 0 + while i < length { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.offset, 4) + let data = Data(bytes: self.buffer.memory.assumingMemoryBound(to: UInt8.self).advanced(by: self.offset + 4), count: Int(length)) + self.offset += 4 + Int(length) + if let string = String(data: data, encoding: .utf8) { + array.append(string) + } else { + assertionFailure() + array.append("") + } + + i += 1 + } + + return array + } else { + return [] + } + } + + public func decodeBytesArrayForKey(_ key: StaticString) -> [MemoryBuffer] { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .BytesArray) { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var array: [MemoryBuffer] = [] + array.reserveCapacity(Int(length)) + + var i: Int32 = 0 + while i < length { + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.offset, 4) + let bytes = malloc(Int(length))! + memcpy(bytes, self.buffer.memory.advanced(by: self.offset + 4), Int(length)) + array.append(MemoryBuffer(memory: bytes, capacity: Int(length), length: Int(length), freeWhenDone: true)) + self.offset += 4 + Int(length) + + i += 1 + } + + return array + } else { + return [] + } + } + + public func decodeObjectArrayForKey(_ key: StaticString) -> [T] where T: Coding { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectArray) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) @@ -872,7 +965,7 @@ public final class Decoder { } } - public func decodeObjectDictionaryForKey(_ key: StaticString) -> [K : V] { + public func decodeObjectDictionaryForKey(_ key: StaticString) -> [K : V] where K: Coding, K: Hashable { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) @@ -926,7 +1019,7 @@ public final class Decoder { } } - public func decodeObjectDictionaryForKey(_ key: StaticString, keyDecoder: (Decoder) -> K) -> [K : V] { + public func decodeObjectDictionaryForKey(_ key: StaticString, keyDecoder: (Decoder) -> K) -> [K : V] where K: Hashable { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) diff --git a/Postbox/ItemCollection.swift b/Postbox/ItemCollection.swift index 396d2df6cc..b2f5673e58 100644 --- a/Postbox/ItemCollection.swift +++ b/Postbox/ItemCollection.swift @@ -72,10 +72,5 @@ public struct ItemCollectionItemIndex: Comparable, Hashable { public protocol ItemCollectionItem: Coding { var index: ItemCollectionItemIndex { get } -} - -public struct ItemCollectionSummary { - let id: ItemCollectionId - let info: ItemCollectionInfo - let itemIds: [ItemCollectionItem] + var indexKeys: [MemoryBuffer] { get } } diff --git a/Postbox/ItemCollectionItemTable.swift b/Postbox/ItemCollectionItemTable.swift index 6245739168..a8cc9c602c 100644 --- a/Postbox/ItemCollectionItemTable.swift +++ b/Postbox/ItemCollectionItemTable.swift @@ -5,13 +5,78 @@ enum ItemCollectionOperation { case removeItem(ItemCollectionId, ItemCollectionItemIndex.Id) } +struct ItemCollectionItemReverseIndexReference: ReverseIndexReference { + let collectionId: ItemCollectionId + let itemIndex: ItemCollectionItemIndex + + var hashValue: Int { + return self.itemIndex.hashValue ^ self.collectionId.hashValue + } + + static func ==(lhs: ItemCollectionItemReverseIndexReference, rhs: ItemCollectionItemReverseIndexReference) -> Bool { + return lhs.collectionId == rhs.collectionId && lhs.itemIndex == rhs.itemIndex + } + + static func <(lhs: ItemCollectionItemReverseIndexReference, rhs: ItemCollectionItemReverseIndexReference) -> Bool { + if lhs.collectionId != rhs.collectionId { + return lhs.collectionId < rhs.collectionId + } else { + return lhs.itemIndex < rhs.itemIndex + } + } + + static func decodeArray(_ buffer: MemoryBuffer) -> [ItemCollectionItemReverseIndexReference] { + assert(buffer.length % (4 + 8 + 4 + 8) == 0) + var references: [ItemCollectionItemReverseIndexReference] = [] + references.reserveCapacity(buffer.length % (4 + 8 + 4 + 8)) + withExtendedLifetime(buffer, { + let readBuffer = ReadBuffer(memoryBufferNoCopy: buffer) + for _ in 0 ..< buffer.length / (4 + 8 + 4 + 8) { + var collectionIdNamespace: Int32 = 0 + var collectionIdId: Int64 = 0 + var indexIdIndex: Int32 = 0 + var indexIdId: Int64 = 0 + readBuffer.read(&collectionIdNamespace, offset: 0, length: 4) + readBuffer.read(&collectionIdId, offset: 0, length: 8) + readBuffer.read(&indexIdIndex, offset: 0, length: 4) + readBuffer.read(&indexIdId, offset: 0, length: 8) + references.append(ItemCollectionItemReverseIndexReference(collectionId: ItemCollectionId(namespace: collectionIdNamespace, id: collectionIdId), itemIndex: ItemCollectionItemIndex(index: indexIdIndex, id: indexIdId))) + } + }) + return references + } + + static func encodeArray(_ array: [ItemCollectionItemReverseIndexReference]) -> MemoryBuffer { + let buffer = WriteBuffer() + for reference in array { + var collectionIdNamespace: Int32 = reference.collectionId.namespace + var collectionIdId: Int64 = reference.collectionId.id + var indexIdIndex: Int32 = reference.itemIndex.index + var indexIdId: Int64 = reference.itemIndex.id + + buffer.write(&collectionIdNamespace, offset: 0, length: 4) + buffer.write(&collectionIdId, offset: 0, length: 8) + buffer.write(&indexIdIndex, offset: 0, length: 4) + buffer.write(&indexIdId, offset: 0, length: 8) + } + return buffer + } +} + final class ItemCollectionItemTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .binary) } + private let reverseIndexTable: ReverseIndexReferenceTable + private let sharedKey = ValueBoxKey(length: 4 + 8 + 4 + 8) + init(valueBox: ValueBox, table: ValueBoxTable, reverseIndexTable: ReverseIndexReferenceTable) { + self.reverseIndexTable = reverseIndexTable + super.init(valueBox: valueBox, table: table) + } + private func key(collectionId: ItemCollectionId, index: ItemCollectionItemIndex) -> ValueBoxKey { self.sharedKey.setInt32(0, value: collectionId.namespace) self.sharedKey.setInt64(4, value: collectionId.id) @@ -51,6 +116,8 @@ final class ItemCollectionItemTable: Table { self.valueBox.range(self.table, start: self.key(collectionId: collectionId, index: itemIndex), end: self.lowerBound(collectionId: collectionId), values: { _, value in if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { items.append(item) + } else { + assertionFailure() } return true }, limit: count) @@ -62,27 +129,14 @@ final class ItemCollectionItemTable: Table { self.valueBox.range(self.table, start: self.key(collectionId: collectionId, index: itemIndex), end: self.upperBound(collectionId: collectionId), values: { _, value in if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { items.append(item) + } else { + assertionFailure() } return true }, limit: count) return items } - func getSummaryIndices(namespace: ItemCollectionId.Namespace) -> [ItemCollectionId: [ItemCollectionItemIndex]] { - var summaryIndices: [ItemCollectionId: [ItemCollectionItemIndex]] = [:] - self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in - let collectionId = ItemCollectionId(namespace: namespace, id: key.getInt64(4)) - let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) - if summaryIndices[collectionId] != nil { - summaryIndices[collectionId]!.append(itemIndex) - } else { - summaryIndices[collectionId] = [itemIndex] - } - return true - }, limit: 0) - return summaryIndices - } - func getItems(namespace: ItemCollectionId.Namespace) -> [ItemCollectionId: [ItemCollectionItem]] { var items: [ItemCollectionId: [ItemCollectionItem]] = [:] self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), values: { key, value in @@ -94,6 +148,8 @@ final class ItemCollectionItemTable: Table { } else { items[collectionId] = [item] } + } else { + assertionFailure() } return true }, limit: 0) @@ -101,24 +157,40 @@ final class ItemCollectionItemTable: Table { } func replaceItems(collectionId: ItemCollectionId, items: [ItemCollectionItem]) { - var currentIndices = Set() - self.valueBox.range(self.table, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId), keys: { key in - let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) - currentIndices.insert(itemIndex) - return true - }, limit: 0) - - var updatedIndices = Set(items.map({ $0.index })) + let updatedIndices = Set(items.map({ $0.index })) var itemByIndex: [ItemCollectionItemIndex: ItemCollectionItem] = [:] for item in items { itemByIndex[item.index] = item } + var currentIndices = Set() + + var removedIndexKeys: [ItemCollectionItemIndex: [MemoryBuffer]] = [:] + + self.valueBox.range(self.table, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId), values: { key, value in + let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) + currentIndices.insert(itemIndex) + + if !updatedIndices.contains(itemIndex) { + if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { + if !item.indexKeys.isEmpty { + removedIndexKeys[itemIndex] = item.indexKeys + } + } else { + assertionFailure() + } + } + return true + }, limit: 0) + let addedIndices = updatedIndices.subtracting(currentIndices) let removedIndices = currentIndices.subtracting(updatedIndices) for index in removedIndices { self.valueBox.remove(self.table, key: self.key(collectionId: collectionId, index: index)) + if let indexKeys = removedIndexKeys[index] { + self.reverseIndexTable.remove(namespace: ReverseIndexNamespace(collectionId.namespace), reference: ItemCollectionItemReverseIndexReference(collectionId: collectionId, itemIndex: index), tokens: indexKeys.map({ ValueBoxKey($0) })) + } } let sharedEncoder = Encoder() @@ -127,9 +199,25 @@ final class ItemCollectionItemTable: Table { sharedEncoder.reset() sharedEncoder.encodeRootObject(item) self.valueBox.set(self.table, key: self.key(collectionId: collectionId, index: index), value: sharedEncoder.readBufferNoCopy()) + if !item.indexKeys.isEmpty { + self.reverseIndexTable.add(namespace: ReverseIndexNamespace(collectionId.namespace), reference: ItemCollectionItemReverseIndexReference(collectionId: collectionId, itemIndex: index), tokens: item.indexKeys.map({ ValueBoxKey($0) })) + } } } + func exactIndexedItems(namespace: ItemCollectionId.Namespace, key: ValueBoxKey) -> [ItemCollectionItem] { + let references = self.reverseIndexTable.exactReferences(namespace: ReverseIndexNamespace(namespace), token: key) + var result: [ItemCollectionItem] = [] + for reference in references { + if let value = self.valueBox.get(self.table, key: self.key(collectionId: reference.collectionId, index: reference.itemIndex)), let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { + result.append(item) + } else { + assertionFailure() + } + } + return result + } + override func clearMemoryCache() { } diff --git a/Postbox/PeerNameIndexTable.swift b/Postbox/PeerNameIndexTable.swift index 26a91736b5..865f5ceac7 100644 --- a/Postbox/PeerNameIndexTable.swift +++ b/Postbox/PeerNameIndexTable.swift @@ -94,19 +94,61 @@ private final class PeerNameIndexCategoriesEntryUpdate { } } +struct PeerIdReverseIndexReference: ReverseIndexReference { + let value: Int64 + + var hashValue: Int { + return self.value.hashValue + } + + static func ==(lhs: PeerIdReverseIndexReference, rhs: PeerIdReverseIndexReference) -> Bool { + return lhs.value == rhs.value + } + + static func <(lhs: PeerIdReverseIndexReference, rhs: PeerIdReverseIndexReference) -> Bool { + return lhs.value < rhs.value + } + + static func decodeArray(_ buffer: MemoryBuffer) -> [PeerIdReverseIndexReference] { + assert(buffer.length % 8 == 0) + var sortedPeerIds: [PeerIdReverseIndexReference] = [] + sortedPeerIds.reserveCapacity(buffer.length % 8) + withExtendedLifetime(buffer, { + let memory = buffer.memory.assumingMemoryBound(to: Int64.self) + for i in 0 ..< buffer.length / 8 { + sortedPeerIds.append(PeerIdReverseIndexReference(value: memory[i])) + } + }) + return sortedPeerIds + } + + static func encodeArray(_ array: [PeerIdReverseIndexReference]) -> MemoryBuffer { + let buffer = MemoryBuffer(memory: malloc(array.count * 8), capacity: array.count * 8, length: array.count * 8, freeWhenDone: true) + let memory = buffer.memory.assumingMemoryBound(to: Int64.self) + var index = 0 + for peerId in array { + memory[index] = peerId.value + index += 1 + } + return buffer + } +} + +private let reverseIndexNamespace = ReverseIndexNamespace(nil) + final class PeerNameIndexTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .int64) } private let peerTable: PeerTable - private let peerNameTokenIndexTable: PeerNameTokenIndexTable + private let peerNameTokenIndexTable: ReverseIndexReferenceTable private let sharedKey = ValueBoxKey(length: 8) private var entryUpdates: [PeerId: PeerNameIndexCategoriesEntryUpdate] = [:] - init(valueBox: ValueBox, table: ValueBoxTable, peerTable: PeerTable, peerNameTokenIndexTable: PeerNameTokenIndexTable) { + init(valueBox: ValueBox, table: ValueBoxTable, peerTable: PeerTable, peerNameTokenIndexTable: ReverseIndexReferenceTable) { self.peerTable = peerTable self.peerNameTokenIndexTable = peerNameTokenIndexTable @@ -160,7 +202,7 @@ final class PeerNameIndexTable: Table { if updatedCategories.isEmpty != wasEmpty { if updatedCategories.isEmpty { if !entryUpdate.initialTokens.isEmpty { - self.peerNameTokenIndexTable.remove(peerId: peerId, tokens: entryUpdate.initialTokens) + self.peerNameTokenIndexTable.remove(namespace: reverseIndexNamespace, reference: PeerIdReverseIndexReference(value: peerId.toInt64()), tokens: entryUpdate.initialTokens) } if !entryUpdate.initialCategories.isEmpty { self.valueBox.remove(self.table, key: self.key(peerId)) @@ -177,7 +219,7 @@ final class PeerNameIndexTable: Table { updatedTokens = [] } } - self.peerNameTokenIndexTable.add(peerId: peerId, tokens: updatedTokens) + self.peerNameTokenIndexTable.add(namespace: reverseIndexNamespace, reference: PeerIdReverseIndexReference(value: peerId.toInt64()), tokens: updatedTokens) sharedBuffer.reset() PeerNameIndexCategoriesEntry(categories: updatedCategories, tokens: updatedTokens).write(to: sharedBuffer) self.valueBox.set(self.table, key: self.key(peerId), value: sharedBuffer) @@ -185,10 +227,10 @@ final class PeerNameIndexTable: Table { } else { if let updatedName = entryUpdate.updatedName { if !entryUpdate.initialTokens.isEmpty { - self.peerNameTokenIndexTable.remove(peerId: peerId, tokens: entryUpdate.initialTokens) + self.peerNameTokenIndexTable.remove(namespace: reverseIndexNamespace, reference: PeerIdReverseIndexReference(value: peerId.toInt64()), tokens: entryUpdate.initialTokens) } let updatedTokens = updatedName.indexTokens - self.peerNameTokenIndexTable.add(peerId: peerId, tokens: updatedTokens) + self.peerNameTokenIndexTable.add(namespace: reverseIndexNamespace, reference: PeerIdReverseIndexReference(value: peerId.toInt64()), tokens: updatedTokens) sharedBuffer.reset() PeerNameIndexCategoriesEntry(categories: updatedCategories, tokens: updatedTokens).write(to: sharedBuffer) self.valueBox.set(self.table, key: self.key(peerId), value: sharedBuffer) @@ -201,10 +243,10 @@ final class PeerNameIndexTable: Table { } else if let updatedName = entryUpdate.updatedName { if !entryUpdate.initialCategories.isEmpty { if !entryUpdate.initialTokens.isEmpty { - self.peerNameTokenIndexTable.remove(peerId: peerId, tokens: entryUpdate.initialTokens) + self.peerNameTokenIndexTable.remove(namespace: reverseIndexNamespace, reference: PeerIdReverseIndexReference(value: peerId.toInt64()), tokens: entryUpdate.initialTokens) } let updatedTokens = updatedName.indexTokens - self.peerNameTokenIndexTable.add(peerId: peerId, tokens: updatedTokens) + self.peerNameTokenIndexTable.add(namespace: reverseIndexNamespace, reference: PeerIdReverseIndexReference(value: peerId.toInt64()), tokens: updatedTokens) sharedBuffer.reset() PeerNameIndexCategoriesEntry(categories: entryUpdate.initialCategories, tokens: updatedTokens).write(to: sharedBuffer) self.valueBox.set(self.table, key: self.key(peerId), value: sharedBuffer) @@ -221,12 +263,13 @@ final class PeerNameIndexTable: Table { } else { var contacts: [PeerId] = [] var chatIndices: [PeerId: ChatListIndex] = [:] - var peerIds = self.peerNameTokenIndexTable.matchingPeerIds(tokens: tokens.regular) + var peerIds = self.peerNameTokenIndexTable.matchingReferences(namespace: reverseIndexNamespace, tokens: tokens.regular) if let transliterated = tokens.transliterated, tokens.regular != transliterated { - let transliteratedPeerIds = self.peerNameTokenIndexTable.matchingPeerIds(tokens: transliterated) + let transliteratedPeerIds = self.peerNameTokenIndexTable.matchingReferences(namespace: reverseIndexNamespace, tokens: transliterated) peerIds.formUnion(transliteratedPeerIds) } - for peerId in peerIds { + for peerIdReference in peerIds { + let peerId = PeerId(peerIdReference.value) var foundInChats = false if categories.contains(.chats) { if let index = chatListIndexTable.get(peerId).includedIndex(peerId: peerId) { diff --git a/Postbox/PeerNameTokenIndexTable.swift b/Postbox/PeerNameTokenIndexTable.swift deleted file mode 100644 index cad285753f..0000000000 --- a/Postbox/PeerNameTokenIndexTable.swift +++ /dev/null @@ -1,211 +0,0 @@ -import Foundation - -private final class PeerNameTokenIndexTableEntry { - var sortedPeerIds: [Int64] = [] - - init() { - } - - init(buffer: MemoryBuffer) { - assert(buffer.length % 8 == 0) - - self.sortedPeerIds.reserveCapacity(buffer.length % 8) - withExtendedLifetime(buffer, { - let memory = buffer.memory.assumingMemoryBound(to: Int64.self) - for i in 0 ..< buffer.length / 8 { - self.sortedPeerIds.append(memory[i]) - } - }) - } - - private func index(of peerId: Int64) -> Int? { - var lowerIndex = 0 - var upperIndex = self.sortedPeerIds.count - 1 - - if lowerIndex > upperIndex { - return nil - } - - while (true) { - let currentIndex = (lowerIndex + upperIndex) / 2 - if self.sortedPeerIds[currentIndex] == peerId { - return currentIndex - } else if lowerIndex > upperIndex { - return nil - } else { - if self.sortedPeerIds[currentIndex] > peerId { - upperIndex = currentIndex - 1 - } else { - lowerIndex = currentIndex + 1 - } - } - } - } - - func remove(_ peerId: PeerId) -> Bool { - if let index = self.index(of: peerId.toInt64()) { - self.sortedPeerIds.remove(at: index) - return true - } else { - return false - } - } - - func insert(_ peerId: PeerId) -> Bool { - let insertItem = peerId.toInt64() - var lo = 0 - var hi = self.sortedPeerIds.count - 1 - while lo <= hi { - let mid = (lo + hi)/2 - if self.sortedPeerIds[mid] < insertItem { - lo = mid + 1 - } else if insertItem < self.sortedPeerIds[mid] { - hi = mid - 1 - } else { - return false - } - } - self.sortedPeerIds.insert(insertItem, at: lo) - return true - } - - func withMemoryBuffer(_ f: (MemoryBuffer) -> Void) { - let buffer = MemoryBuffer(memory: malloc(self.sortedPeerIds.count * 8), capacity: self.sortedPeerIds.count * 8, length: self.sortedPeerIds.count * 8, freeWhenDone: true) - let memory = buffer.memory.assumingMemoryBound(to: Int64.self) - var index = 0 - for peerId in self.sortedPeerIds { - memory[index] = peerId - index += 1 - } - f(buffer) - } -} - -final class PeerNameTokenIndexTable: Table { - private var cachedEntries: [ValueBoxKey: PeerNameTokenIndexTableEntry] = [:] - private var updatedCachedEntries = Set() - - static func tableSpec(_ id: Int32) -> ValueBoxTable { - return ValueBoxTable(id: id, keyType: .binary) - } - - private func getEntry(token: ValueBoxKey) -> PeerNameTokenIndexTableEntry? { - if let cached = self.cachedEntries[token] { - return cached - } else { - if let value = self.valueBox.get(self.table, key: token) { - let entry = PeerNameTokenIndexTableEntry(buffer: value) - self.cachedEntries[token] = entry - return entry - } else { - return nil - } - } - } - - private func add(token: ValueBoxKey, peerId: PeerId) { - let entry: PeerNameTokenIndexTableEntry - var addToCache = false - if let current = self.getEntry(token: token) { - entry = current - } else { - entry = PeerNameTokenIndexTableEntry() - addToCache = true - } - if entry.insert(peerId) { - self.updatedCachedEntries.insert(token) - if addToCache { - self.cachedEntries[token] = entry - } - } - } - - private func remove(token: ValueBoxKey, peerId: PeerId) { - let entry: PeerNameTokenIndexTableEntry - var addToCache = false - if let current = self.getEntry(token: token) { - entry = current - } else { - entry = PeerNameTokenIndexTableEntry() - addToCache = true - } - if entry.remove(peerId) { - self.updatedCachedEntries.insert(token) - if addToCache { - self.cachedEntries[token] = entry - } - } - } - - func add(peerId: PeerId, tokens: [ValueBoxKey]) { - for token in tokens { - self.add(token: token, peerId: peerId) - } - } - - func remove(peerId: PeerId, tokens: [ValueBoxKey]) { - for token in tokens { - self.remove(token: token, peerId: peerId) - } - } - - func matchingPeerIds(tokens: [ValueBoxKey]) -> Set { - var peerIds: Set? - for token in tokens { - if let peerIds = peerIds, peerIds.isEmpty { - return Set() - } - var currentPeerIds = Set() - self.valueBox.range(self.table, start: token.predecessor, end: token.successor, values: { key, value in - if let cached = self.cachedEntries[key] { - for peerId in cached.sortedPeerIds { - currentPeerIds.insert(PeerId(peerId)) - } - } else { - for peerId in PeerNameTokenIndexTableEntry(buffer: value).sortedPeerIds { - currentPeerIds.insert(PeerId(peerId)) - } - } - return true - }, limit: 0) - if let previousPeerIds = peerIds { - peerIds = previousPeerIds.intersection(currentPeerIds) - } else { - peerIds = currentPeerIds - } - } - if let peerIds = peerIds { - return peerIds - } else { - return Set() - } - } - - override func clearMemoryCache() { - assert(self.cachedEntries.isEmpty) - assert(self.updatedCachedEntries.isEmpty) - } - - override func beforeCommit() { - if !self.updatedCachedEntries.isEmpty { - for token in self.updatedCachedEntries { - if let entry = self.cachedEntries[token] { - entry.withMemoryBuffer { buffer in - if buffer.length == 0 { - self.valueBox.remove(self.table, key: token) - } else { - self.valueBox.set(self.table, key: token, value: buffer) - } - } - } else { - assertionFailure() - } - } - self.updatedCachedEntries.removeAll() - } - - if !self.cachedEntries.isEmpty { - self.cachedEntries.removeAll() - } - } -} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 0b0d62032b..57fca3495f 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -189,6 +189,14 @@ public final class Modifier { self.postbox?.replaceItemCollections(namespace: namespace, itemCollections: itemCollections) } + public func searchItemCollection(namespace: ItemCollectionId.Namespace, key: MemoryBuffer) -> [ItemCollectionItem] { + if let postbox = self.postbox { + return postbox.itemCollectionItemTable.exactIndexedItems(namespace: namespace, key: ValueBoxKey(key)) + } else { + return [] + } + } + public func getMessage(_ id: MessageId) -> Message? { if let postbox = self.postbox { if let entry = postbox.messageHistoryIndexTable.get(id) { @@ -384,10 +392,11 @@ public final class Postbox { var contactsTable: ContactTable! var itemCollectionInfoTable: ItemCollectionInfoTable! var itemCollectionItemTable: ItemCollectionItemTable! + var itemCollectionReverseIndexTable: ReverseIndexReferenceTable! var peerChatInterfaceStateTable: PeerChatInterfaceStateTable! var itemCacheMetaTable: ItemCacheMetaTable! var itemCacheTable: ItemCacheTable! - var peerNameTokenIndexTable: PeerNameTokenIndexTable! + var peerNameTokenIndexTable: ReverseIndexReferenceTable! var peerNameIndexTable: PeerNameIndexTable! var peerChatTopTaggedMessageIdsTable: PeerChatTopTaggedMessageIdsTable! var peerOperationLogMetadataTable: PeerOperationLogMetadataTable! @@ -506,11 +515,12 @@ public final class Postbox { self.peerNotificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox, table: PeerNotificationSettingsTable.tableSpec(19)) self.peerPresenceTable = PeerPresenceTable(valueBox: self.valueBox, table: PeerPresenceTable.tableSpec(20)) self.itemCollectionInfoTable = ItemCollectionInfoTable(valueBox: self.valueBox, table: ItemCollectionInfoTable.tableSpec(21)) - self.itemCollectionItemTable = ItemCollectionItemTable(valueBox: self.valueBox, table: ItemCollectionItemTable.tableSpec(22)) + self.itemCollectionReverseIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(36)) + self.itemCollectionItemTable = ItemCollectionItemTable(valueBox: self.valueBox, table: ItemCollectionItemTable.tableSpec(22), reverseIndexTable: self.itemCollectionReverseIndexTable) self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox, table: PeerChatInterfaceStateTable.tableSpec(23)) self.itemCacheMetaTable = ItemCacheMetaTable(valueBox: self.valueBox, table: ItemCacheMetaTable.tableSpec(24)) self.itemCacheTable = ItemCacheTable(valueBox: self.valueBox, table: ItemCacheTable.tableSpec(25)) - self.peerNameTokenIndexTable = PeerNameTokenIndexTable(valueBox: self.valueBox, table: PeerNameTokenIndexTable.tableSpec(26)) + self.peerNameTokenIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(26)) self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8), peerNameIndexTable: self.peerNameIndexTable, metadataTable: self.messageHistoryMetadataTable, readStateTable: self.readStateTable, notificationSettingsTable: self.peerNotificationSettingsTable) self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) @@ -542,6 +552,7 @@ public final class Postbox { self.tables.append(self.peerPresenceTable) self.tables.append(self.itemCollectionInfoTable) self.tables.append(self.itemCollectionItemTable) + self.tables.append(self.itemCollectionReverseIndexTable) self.tables.append(self.peerChatInterfaceStateTable) self.tables.append(self.itemCacheMetaTable) self.tables.append(self.itemCacheTable) diff --git a/Postbox/ReverseIndexReferenceTable.swift b/Postbox/ReverseIndexReferenceTable.swift new file mode 100644 index 0000000000..81a89fc08a --- /dev/null +++ b/Postbox/ReverseIndexReferenceTable.swift @@ -0,0 +1,274 @@ +import Foundation + +protocol ReverseIndexReference: Comparable, Hashable { + static func decodeArray(_ buffer: MemoryBuffer) -> [Self] + static func encodeArray(_ array: [Self]) -> MemoryBuffer +} + +private final class ReverseIndexReferencesEntry { + var orderedReferences: [T] = [] + + init() { + } + + init(buffer: MemoryBuffer) { + self.orderedReferences = T.decodeArray(buffer) + } + + private func index(of reference: T) -> Int? { + var lowerIndex = 0 + var upperIndex = self.orderedReferences.count - 1 + + if lowerIndex > upperIndex { + return nil + } + + while (true) { + let currentIndex = (lowerIndex + upperIndex) / 2 + if self.orderedReferences[currentIndex] == reference { + return currentIndex + } else if lowerIndex > upperIndex { + return nil + } else { + if self.orderedReferences[currentIndex] > reference { + upperIndex = currentIndex - 1 + } else { + lowerIndex = currentIndex + 1 + } + } + } + } + + func remove(_ reference: T) -> Bool { + if let index = self.index(of: reference) { + self.orderedReferences.remove(at: index) + return true + } else { + return false + } + } + + func insert(_ reference: T) -> Bool { + let insertItem = reference + var lo = 0 + var hi = self.orderedReferences.count - 1 + while lo <= hi { + let mid = (lo + hi)/2 + if self.orderedReferences[mid] < insertItem { + lo = mid + 1 + } else if insertItem < self.orderedReferences[mid] { + hi = mid - 1 + } else { + return false + } + } + self.orderedReferences.insert(insertItem, at: lo) + return true + } + + func withMemoryBuffer(_ f: (MemoryBuffer) -> Void) { + f(T.encodeArray(self.orderedReferences)) + } +} + +struct ReverseIndexNamespace: Hashable { + let value: Int32? + + init(_ value: Int32?) { + self.value = value + } + + var hashValue: Int { + if let value = self.value { + return value.hashValue + } else { + return 0 + } + } + + static func ==(lhs: ReverseIndexNamespace, rhs: ReverseIndexNamespace) -> Bool { + return lhs.value == rhs.value + } +} + +final class ReverseIndexReferenceTable: Table { + private var cachedEntriesByNamespace: [ReverseIndexNamespace: [ValueBoxKey: ReverseIndexReferencesEntry]] = [:] + private var updatedCachedEntriesByNamespace: [ReverseIndexNamespace: Set] = [:] + + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private func key(namespace: ReverseIndexNamespace, token: ValueBoxKey) -> ValueBoxKey { + if let value = namespace.value { + let key = ValueBoxKey(length: 4 + token.length) + key.setInt32(0, value: value) + memcpy(key.memory.advanced(by: 4), token.memory, token.length) + return key + } else { + return token + } + } + + private func getEntry(namespace: ReverseIndexNamespace, token: ValueBoxKey) -> ReverseIndexReferencesEntry? { + if let cachedNamespace = self.cachedEntriesByNamespace[namespace], let cached = cachedNamespace[token] { + return cached + } else { + if let value = self.valueBox.get(self.table, key: self.key(namespace: namespace, token: token)) { + let entry = ReverseIndexReferencesEntry(buffer: value) + if self.cachedEntriesByNamespace[namespace] == nil { + self.cachedEntriesByNamespace[namespace] = [token: entry] + } else { + self.cachedEntriesByNamespace[namespace]![token] = entry + } + return entry + } else { + return nil + } + } + } + + private func add(namespace: ReverseIndexNamespace, token: ValueBoxKey, reference: T) { + let entry: ReverseIndexReferencesEntry + var addToCache = false + if let current = self.getEntry(namespace: namespace, token: token) { + entry = current + } else { + entry = ReverseIndexReferencesEntry() + addToCache = true + } + if entry.insert(reference) { + if self.updatedCachedEntriesByNamespace[namespace] == nil { + self.updatedCachedEntriesByNamespace[namespace] = Set() + } + self.updatedCachedEntriesByNamespace[namespace]!.insert(token) + if addToCache { + if self.cachedEntriesByNamespace[namespace] == nil { + self.cachedEntriesByNamespace[namespace] = [token: entry] + } else { + self.cachedEntriesByNamespace[namespace]![token] = entry + } + } + } + } + + private func remove(namespace: ReverseIndexNamespace, token: ValueBoxKey, reference: T) { + let entry: ReverseIndexReferencesEntry + var addToCache = false + if let current = self.getEntry(namespace: namespace, token: token) { + entry = current + } else { + entry = ReverseIndexReferencesEntry() + addToCache = true + } + if entry.remove(reference) { + if self.updatedCachedEntriesByNamespace[namespace] == nil { + self.updatedCachedEntriesByNamespace[namespace] = Set() + } + self.updatedCachedEntriesByNamespace[namespace]!.insert(token) + if addToCache { + if self.cachedEntriesByNamespace[namespace] == nil { + self.cachedEntriesByNamespace[namespace] = [token: entry] + } else { + self.cachedEntriesByNamespace[namespace]![token] = entry + } + } + } + } + + func add(namespace: ReverseIndexNamespace, reference: T, tokens: [ValueBoxKey]) { + for token in tokens { + self.add(namespace: namespace, token: token, reference: reference) + } + } + + func remove(namespace: ReverseIndexNamespace, reference: T, tokens: [ValueBoxKey]) { + for token in tokens { + self.remove(namespace: namespace, token: token, reference: reference) + } + } + + func matchingReferences(namespace: ReverseIndexNamespace, tokens: [ValueBoxKey]) -> Set { + var references: Set? + for token in tokens { + if let references = references, references.isEmpty { + return Set() + } + var currentReferences = Set() + self.valueBox.range(self.table, start: self.key(namespace: namespace, token: token).predecessor, end: self.key(namespace: namespace, token: token).successor, values: { key, value in + if let cachedNamespace = self.cachedEntriesByNamespace[namespace], let cached = cachedNamespace[token] { + for reference in cached.orderedReferences { + currentReferences.insert(reference) + } + } else { + for reference in ReverseIndexReferencesEntry(buffer: value).orderedReferences { + currentReferences.insert(reference) + } + } + return true + }, limit: 0) + if let previousReferences = references { + references = previousReferences.intersection(currentReferences) + } else { + references = currentReferences + } + } + if let references = references { + return references + } else { + return Set() + } + } + + func exactReferences(namespace: ReverseIndexNamespace, token: ValueBoxKey) -> [T] { + if let value = self.valueBox.get(self.table, key: self.key(namespace: namespace, token: token)) { + var currentReferences: [T] = [] + if let cachedNamespace = self.cachedEntriesByNamespace[namespace], let cached = cachedNamespace[token] { + for reference in cached.orderedReferences { + currentReferences.append(reference) + } + } else { + for reference in ReverseIndexReferencesEntry(buffer: value).orderedReferences { + currentReferences.append(reference) + } + } + return currentReferences + } else { + return [] + } + } + + override func clearMemoryCache() { + for (_, cachedEntries) in self.cachedEntriesByNamespace { + assert(cachedEntries.isEmpty) + } + for (_, updatedCachedEntries) in self.updatedCachedEntriesByNamespace { + assert(updatedCachedEntries.isEmpty) + } + } + + override func beforeCommit() { + if !self.updatedCachedEntriesByNamespace.isEmpty { + for (namespace, updatedCachedEntries) in self.updatedCachedEntriesByNamespace { + for token in updatedCachedEntries { + if let cachedNamespace = self.cachedEntriesByNamespace[namespace], let cached = cachedNamespace[token] { + cached.withMemoryBuffer { buffer in + if buffer.length == 0 { + self.valueBox.remove(self.table, key: self.key(namespace: namespace, token: token)) + } else { + self.valueBox.set(self.table, key: self.key(namespace: namespace, token: token), value: buffer) + } + } + } else { + assertionFailure() + } + } + } + self.updatedCachedEntriesByNamespace.removeAll() + } + + if !self.cachedEntriesByNamespace.isEmpty { + self.cachedEntriesByNamespace.removeAll() + } + } +} diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index aabc158b50..d54e8503cc 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -199,6 +199,12 @@ public struct ValueBoxKey: Hashable, CustomStringConvertible, Comparable { public static func <(lhs: ValueBoxKey, rhs: ValueBoxKey) -> Bool { return mdb_cmp_memn(lhs.memory, lhs.length, rhs.memory, rhs.length) < 0 } + + public func toMemoryBuffer() -> MemoryBuffer { + let data = malloc(self.length)! + memcpy(data, self.memory, self.length) + return MemoryBuffer(memory: data, capacity: self.length, length: self.length, freeWhenDone: true) + } } private func mdb_cmp_memn(_ a_memory: UnsafeMutableRawPointer, _ a_length: Int, _ b_memory: UnsafeMutableRawPointer, _ b_length: Int) -> Int From 9d53f1ec93e0a59bdddd0a2f60d75de69da58df0 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 7 Feb 2017 22:33:14 +0300 Subject: [PATCH 081/427] no message --- Postbox.xcodeproj/project.pbxproj | 34 ++++ .../xcschemes/xcschememanagement.plist | 2 +- Postbox/ItemCollectionsView.swift | 26 ++- Postbox/MessageHistoryReadStateTable.swift | 4 +- Postbox/MetadataTable.swift | 21 +- Postbox/OrderedItemListEntry.swift | 15 ++ Postbox/OrderedItemListIndexTable.swift | 43 +++++ Postbox/OrderedItemListTable.swift | 181 ++++++++++++++++++ Postbox/OrderedItemListView.swift | 48 +++++ Postbox/Postbox.swift | 129 ++++++++++--- Postbox/PostboxStateView.swift | 22 +++ Postbox/PostboxTransaction.swift | 12 +- Postbox/ValueBoxKey.swift | 11 +- Postbox/ViewTracker.swift | 56 ++++++ PostboxTests/ChatListTableTests.swift | 4 +- PostboxTests/OrderedItemListTableTests.swift | 124 ++++++++++++ 16 files changed, 689 insertions(+), 43 deletions(-) create mode 100644 Postbox/OrderedItemListEntry.swift create mode 100644 Postbox/OrderedItemListIndexTable.swift create mode 100644 Postbox/OrderedItemListTable.swift create mode 100644 Postbox/OrderedItemListView.swift create mode 100644 Postbox/PostboxStateView.swift create mode 100644 PostboxTests/OrderedItemListTableTests.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 12c9d23fdf..e18e16a104 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -45,6 +45,8 @@ D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044E15D1B2ACB9C001EE087 /* CodingTests.swift */; }; D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */ = {isa = PBXBuildFile; fileRef = D044E1621B2AD677001EE087 /* MurMurHash32.m */; }; D044E1641B2AD718001EE087 /* MurMurHash32.h in Headers */ = {isa = PBXBuildFile; fileRef = D044E1611B2AD667001EE087 /* MurMurHash32.h */; settings = {ATTRIBUTES = (Public, ); }; }; + D049EAF01E44D9B900A2CD3A /* PostboxStateView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D049EAEF1E44D9B900A2CD3A /* PostboxStateView.swift */; }; + D049EAF11E44D9B900A2CD3A /* PostboxStateView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D049EAEF1E44D9B900A2CD3A /* PostboxStateView.swift */; }; D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; D060B77B1CF4845A0050BE9B /* ReadStateTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */; }; @@ -81,6 +83,8 @@ D08775041E3E3E7400A97350 /* PreferencesEntry.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08775021E3E3E7400A97350 /* PreferencesEntry.swift */; }; D08775061E3E3F2100A97350 /* PreferencesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08775051E3E3F2100A97350 /* PreferencesView.swift */; }; D08775071E3E3F2100A97350 /* PreferencesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08775051E3E3F2100A97350 /* PreferencesView.swift */; }; + D087C20C1E43C11C00D686F8 /* OrderedItemListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D087C20B1E43C11C00D686F8 /* OrderedItemListView.swift */; }; + D087C20D1E43C11C00D686F8 /* OrderedItemListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D087C20B1E43C11C00D686F8 /* OrderedItemListView.swift */; }; D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; @@ -209,6 +213,13 @@ D0F7B1E01E045C6A007EB8A5 /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; D0F7B1E11E045C6A007EB8A5 /* ReverseIndexReferenceTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C21E008F7300071108 /* ReverseIndexReferenceTable.swift */; }; D0F7B1E21E045C6A007EB8A5 /* PeerNameIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */; }; + D0F82CFD1E4345D7007E499C /* OrderedItemListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F82CFC1E4345D7007E499C /* OrderedItemListTable.swift */; }; + D0F82CFE1E4345D7007E499C /* OrderedItemListTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F82CFC1E4345D7007E499C /* OrderedItemListTable.swift */; }; + D0F82D0C1E439FCC007E499C /* OrderedItemListEntry.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F82D0B1E439FCC007E499C /* OrderedItemListEntry.swift */; }; + D0F82D0D1E439FCC007E499C /* OrderedItemListEntry.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F82D0B1E439FCC007E499C /* OrderedItemListEntry.swift */; }; + D0F82D0F1E43A024007E499C /* OrderedItemListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F82D0E1E43A024007E499C /* OrderedItemListIndexTable.swift */; }; + D0F82D101E43A024007E499C /* OrderedItemListIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F82D0E1E43A024007E499C /* OrderedItemListIndexTable.swift */; }; + D0F82D121E43B5D4007E499C /* OrderedItemListTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F82D111E43B5D4007E499C /* OrderedItemListTableTests.swift */; }; D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */; }; D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; @@ -265,6 +276,7 @@ D044E15D1B2ACB9C001EE087 /* CodingTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CodingTests.swift; sourceTree = ""; }; D044E1611B2AD667001EE087 /* MurMurHash32.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MurMurHash32.h; sourceTree = ""; }; D044E1621B2AD677001EE087 /* MurMurHash32.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MurMurHash32.m; sourceTree = ""; }; + D049EAEF1E44D9B900A2CD3A /* PostboxStateView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxStateView.swift; sourceTree = ""; }; D055BD321B7D3D2D00F06C0A /* MediaBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaBox.swift; sourceTree = ""; }; D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResourceStatus.swift; sourceTree = ""; }; D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ReadStateTableTests.swift; sourceTree = ""; }; @@ -284,6 +296,7 @@ D08774FF1E3E3D9F00A97350 /* PreferencesTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PreferencesTable.swift; sourceTree = ""; }; D08775021E3E3E7400A97350 /* PreferencesEntry.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PreferencesEntry.swift; sourceTree = ""; }; D08775051E3E3F2100A97350 /* PreferencesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PreferencesView.swift; sourceTree = ""; }; + D087C20B1E43C11C00D686F8 /* OrderedItemListView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderedItemListView.swift; sourceTree = ""; }; D08C71391C501F0700779C0F /* MessageHistoryHole.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHole.swift; sourceTree = ""; }; D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTable.swift; sourceTree = ""; }; D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTable.swift; sourceTree = ""; }; @@ -344,6 +357,10 @@ D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheTable.swift; sourceTree = ""; }; D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheMetaTable.swift; sourceTree = ""; }; D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatTopIndexableMessageIds.swift; sourceTree = ""; }; + D0F82CFC1E4345D7007E499C /* OrderedItemListTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderedItemListTable.swift; sourceTree = ""; }; + D0F82D0B1E439FCC007E499C /* OrderedItemListEntry.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderedItemListEntry.swift; sourceTree = ""; }; + D0F82D0E1E43A024007E499C /* OrderedItemListIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderedItemListIndexTable.swift; sourceTree = ""; }; + D0F82D111E43B5D4007E499C /* OrderedItemListTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderedItemListTableTests.swift; sourceTree = ""; }; D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageMediaTable.swift; sourceTree = ""; }; D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTableTests.swift; sourceTree = ""; }; D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaCleanupTable.swift; sourceTree = ""; }; @@ -485,6 +502,8 @@ D0AAD1B11E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift */, D0AAD1B01E3266B100D5B9DE /* TimestampBasedMessageAttributesIndexTable.swift */, D08774FF1E3E3D9F00A97350 /* PreferencesTable.swift */, + D0F82CFC1E4345D7007E499C /* OrderedItemListTable.swift */, + D0F82D0E1E43A024007E499C /* OrderedItemListIndexTable.swift */, ); name = Tables; sourceTree = ""; @@ -508,6 +527,7 @@ D00C7CD31E365C4E0080C3D5 /* PeerChatListInclusion.swift */, D08775021E3E3E7400A97350 /* PreferencesEntry.swift */, D0BC38711E409E670044D6FE /* RenderedPeer.swift */, + D0F82D0B1E439FCC007E499C /* OrderedItemListEntry.swift */, ); name = Objects; sourceTree = ""; @@ -539,6 +559,7 @@ D0E1DE1A1C5EB0DB00C7826E /* Views */ = { isa = PBXGroup; children = ( + D049EAEF1E44D9B900A2CD3A /* PostboxStateView.swift */, D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */, D07CFF841DCA99C400761F81 /* InitialMessageHistoryData.swift */, D0F9E86A1C59719800037222 /* ChatListView.swift */, @@ -557,6 +578,7 @@ D00C7CE21E37861C0080C3D5 /* MessageView.swift */, D08775051E3E3F2100A97350 /* PreferencesView.swift */, D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */, + D087C20B1E43C11C00D686F8 /* OrderedItemListView.swift */, ); name = Views; sourceTree = ""; @@ -630,6 +652,7 @@ D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */, D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */, D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */, + D0F82D111E43B5D4007E499C /* OrderedItemListTableTests.swift */, ); path = PostboxTests; sourceTree = ""; @@ -822,6 +845,7 @@ D073CE771DCBF3B4007511FD /* Media.swift in Sources */, D073CE791DCBF3B4007511FD /* ChatListHole.swift in Sources */, D0B4181C1D7DFDF1004562A4 /* SQLite-Bridging.m in Sources */, + D0F82D0D1E439FCC007E499C /* OrderedItemListEntry.swift in Sources */, D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */, D0F7B1D21E045C6A007EB8A5 /* PeerNotificationSettingsTable.swift in Sources */, D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */, @@ -836,11 +860,14 @@ D0F7B1D31E045C6A007EB8A5 /* CachedPeerDataTable.swift in Sources */, D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */, D0F7B1D51E045C6A007EB8A5 /* PeerChatStateTable.swift in Sources */, + D0F82CFE1E4345D7007E499C /* OrderedItemListTable.swift in Sources */, D0F7B1C51E045C6A007EB8A5 /* MessageHistoryMetadataTable.swift in Sources */, D073CE781DCBF3B4007511FD /* MessageHistoryHole.swift in Sources */, D08775071E3E3F2100A97350 /* PreferencesView.swift in Sources */, + D049EAF11E44D9B900A2CD3A /* PostboxStateView.swift in Sources */, D0AAD1B61E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift in Sources */, D0F7B1C31E045C6A007EB8A5 /* Table.swift in Sources */, + D0F82D101E43A024007E499C /* OrderedItemListIndexTable.swift in Sources */, D0F7B1CF1E045C6A007EB8A5 /* MetadataTable.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */, @@ -872,6 +899,7 @@ D073CE7D1DCBF3B4007511FD /* PeerPresence.swift in Sources */, D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, D073CE751DCBF3B4007511FD /* Message.swift in Sources */, + D087C20D1E43C11C00D686F8 /* OrderedItemListView.swift in Sources */, D00C7CD51E365C4E0080C3D5 /* PeerChatListInclusion.swift in Sources */, D0F7B1CB1E045C6A007EB8A5 /* ChatListTable.swift in Sources */, D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */, @@ -914,6 +942,7 @@ D0079F651D5A457A00A27A2C /* ContactPeersView.swift in Sources */, D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */, D08CEFB41D2AD8BE0015D3BC /* RedBlackTree.swift in Sources */, + D0F82D0C1E439FCC007E499C /* OrderedItemListEntry.swift in Sources */, D03120FE1DA562E9006A2A60 /* CachedPeerDataTable.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, @@ -926,12 +955,14 @@ D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */, D021E0D81DB4FD1300C6B04F /* ItemCollectionItemTable.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, + D049EAF01E44D9B900A2CD3A /* PostboxStateView.swift in Sources */, D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */, D0AD23291E196B6400A7089A /* PeerOperationLogMetadataTable.swift in Sources */, D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */, + D0F82CFD1E4345D7007E499C /* OrderedItemListTable.swift in Sources */, D0AB0B8C1D65D488002C78E7 /* MessageHistoryHolesView.swift in Sources */, D0AAD1B51E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift in Sources */, D010B61A1E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift in Sources */, @@ -945,6 +976,7 @@ D07827C31E008F7300071108 /* ReverseIndexReferenceTable.swift in Sources */, D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, + D0F82D0F1E43A024007E499C /* OrderedItemListIndexTable.swift in Sources */, D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */, D0AD23271E194D1C00A7089A /* PeerOperationLogTable.swift in Sources */, D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */, @@ -963,6 +995,7 @@ D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, D0BC386C1E3FCEE50044D6FE /* MultiplePeersView.swift in Sources */, D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */, + D087C20C1E43C11C00D686F8 /* OrderedItemListView.swift in Sources */, D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, D00C7CD41E365C4E0080C3D5 /* PeerChatListInclusion.swift in Sources */, @@ -1005,6 +1038,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + D0F82D121E43B5D4007E499C /* OrderedItemListTableTests.swift in Sources */, D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */, D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */, D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */, diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist index 82f75996f6..d2fb9e5ddf 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist @@ -12,7 +12,7 @@ PostboxMac.xcscheme orderHint - 25 + 21 PostboxTests.xcscheme diff --git a/Postbox/ItemCollectionsView.swift b/Postbox/ItemCollectionsView.swift index dbc74c1b56..7b8157be81 100644 --- a/Postbox/ItemCollectionsView.swift +++ b/Postbox/ItemCollectionsView.swift @@ -5,6 +5,12 @@ public struct ItemCollectionViewEntryIndex: Comparable { public let collectionId: ItemCollectionId public let itemIndex: ItemCollectionItemIndex + public init(collectionIndex: Int32, collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex) { + self.collectionIndex = collectionIndex + self.collectionId = collectionId + self.itemIndex = itemIndex + } + public static func ==(lhs: ItemCollectionViewEntryIndex, rhs: ItemCollectionViewEntryIndex) -> Bool { return lhs.collectionIndex == rhs.collectionIndex && lhs.collectionId == rhs.collectionId && lhs.itemIndex == rhs.itemIndex } @@ -42,7 +48,7 @@ private func fetchLowerEntries(namespaces: [ItemCollectionId.Namespace], collect while true { let remainingCount = count - entries.count assert(remainingCount > 0) - var collectionItems = lowerItems(currentCollectionId, currentItemIndex, remainingCount) + let collectionItems = lowerItems(currentCollectionId, currentItemIndex, remainingCount) for item in collectionItems { entries.append(ItemCollectionViewEntry(index: ItemCollectionViewEntryIndex(collectionIndex: currentCollectionIndex, collectionId: currentCollectionId, itemIndex: item.index), item: item)) } @@ -75,7 +81,7 @@ private func fetchHigherEntries(namespaces: [ItemCollectionId.Namespace], collec while true { let remainingCount = count - entries.count assert(remainingCount > 0) - var collectionItems = higherItems(currentCollectionId, currentItemIndex, remainingCount) + let collectionItems = higherItems(currentCollectionId, currentItemIndex, remainingCount) for item in collectionItems { entries.append(ItemCollectionViewEntry(index: ItemCollectionViewEntryIndex(collectionIndex: currentCollectionIndex, collectionId: currentCollectionId, itemIndex: item.index), item: item)) } @@ -148,6 +154,7 @@ private func aroundEntries(namespaces: [ItemCollectionId.Namespace], } final class MutableItemCollectionsView { + let orderedItemListsViews: [MutableOrderedItemListView] let namespaces: [ItemCollectionId.Namespace] var collectionInfos: [(ItemCollectionId, ItemCollectionInfo, ItemCollectionItem?)] @@ -155,7 +162,8 @@ final class MutableItemCollectionsView { var lower: ItemCollectionViewEntry? var higher: ItemCollectionViewEntry? - init(namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int, getInfos: (_ namespace: ItemCollectionId.Namespace) -> [(Int, ItemCollectionId, ItemCollectionInfo)], lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, lowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem], higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, higherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) { + init(orderedItemListsViews: [MutableOrderedItemListView], namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int, getInfos: (_ namespace: ItemCollectionId.Namespace) -> [(Int, ItemCollectionId, ItemCollectionInfo)], lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, lowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem], higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, higherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) { + self.orderedItemListsViews = orderedItemListsViews self.namespaces = namespaces self.collectionInfos = [] @@ -187,15 +195,27 @@ final class MutableItemCollectionsView { self.lower = lower self.higher = higher } + + func replay(orderedItemListOperations: [Int32: [OrderedItemListOperation]]) -> Bool { + var updated = false + for view in self.orderedItemListsViews { + if view.replay(operations: orderedItemListOperations) { + updated = true + } + } + return updated + } } public final class ItemCollectionsView { + public let orderedItemListsViews: [OrderedItemListView] public let collectionInfos: [(ItemCollectionId, ItemCollectionInfo, ItemCollectionItem?)] public let entries: [ItemCollectionViewEntry] public let lower: ItemCollectionViewEntry? public let higher: ItemCollectionViewEntry? init(_ mutableView: MutableItemCollectionsView) { + self.orderedItemListsViews = mutableView.orderedItemListsViews.map { OrderedItemListView($0) } self.collectionInfos = mutableView.collectionInfos self.entries = mutableView.entries self.lower = mutableView.lower diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index d9f8157c0b..ca23924717 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -290,7 +290,7 @@ final class MessageHistoryReadStateTable: Table { assertionFailure() case let .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count): if maxIncomingReadIndex < messageIndex { - var (deltaCount, holes, messageIds) = incomingStatsInRange(maxIncomingReadIndex.successor(), messageIndex) + let (deltaCount, holes, messageIds) = incomingStatsInRange(maxIncomingReadIndex.successor(), messageIndex) if traceReadStates { print("[ReadStateTable] applyIncomingMaxReadIndex after deltaCount: \(deltaCount), holes: \(holes)") @@ -428,7 +428,7 @@ final class MessageHistoryReadStateTable: Table { sharedBuffer.write(&maxOutgoingReadId, offset: 0, length: 4) sharedBuffer.write(&maxKnownId, offset: 0, length: 4) sharedBuffer.write(&count, offset: 0, length: 4) - case var .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count): + case .indexBased(let maxIncomingReadIndex, let maxOutgoingReadIndex, var count): var kind: Int8 = 1 sharedBuffer.write(&kind, offset: 0, length: 1) diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index 03dc16dd6b..f42089b735 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -12,6 +12,8 @@ final class MetadataTable: Table { return ValueBoxTable(id: id, keyType: .int64) } + private var cachedState: Coding? + private let sharedBuffer = WriteBuffer() override init(valueBox: ValueBox, table: ValueBoxTable) { @@ -42,15 +44,22 @@ final class MetadataTable: Table { } func state() -> Coding? { - if let value = self.valueBox.get(self.table, key: self.key(.State)) { - if let state = Decoder(buffer: value).decodeRootObject() { - return state + if let cachedState = self.cachedState { + return cachedState + } else { + if let value = self.valueBox.get(self.table, key: self.key(.State)) { + if let state = Decoder(buffer: value).decodeRootObject() { + self.cachedState = state + return state + } } + return nil } - return nil } func setState(_ state: Coding) { + self.cachedState = state + let encoder = Encoder() encoder.encodeRootObject(state) self.valueBox.set(self.table, key: self.key(.State), value: encoder.readBufferNoCopy()) @@ -92,4 +101,8 @@ final class MetadataTable: Table { buffer.write(&clientId, offset: 0, length: 8) self.valueBox.set(self.table, key: self.key(.MasterClientId), value: buffer) } + + override func clearMemoryCache() { + self.cachedState = nil + } } diff --git a/Postbox/OrderedItemListEntry.swift b/Postbox/OrderedItemListEntry.swift new file mode 100644 index 0000000000..c77487ffad --- /dev/null +++ b/Postbox/OrderedItemListEntry.swift @@ -0,0 +1,15 @@ +import Foundation + +public protocol OrderedItemListEntryContents: Coding { + +} + +public struct OrderedItemListEntry { + public let id: MemoryBuffer + public let contents: OrderedItemListEntryContents + + public init(id: MemoryBuffer, contents: OrderedItemListEntryContents) { + self.id = id + self.contents = contents + } +} diff --git a/Postbox/OrderedItemListIndexTable.swift b/Postbox/OrderedItemListIndexTable.swift new file mode 100644 index 0000000000..0875efb1d8 --- /dev/null +++ b/Postbox/OrderedItemListIndexTable.swift @@ -0,0 +1,43 @@ +import Foundation + +final class OrderedItemListIndexTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private func key(collectionId: Int32, id: MemoryBuffer) -> ValueBoxKey { + let key = ValueBoxKey(length: 4 + id.length) + key.setInt32(0, value: collectionId) + memcpy(key.memory.advanced(by: 4), id.memory, id.length) + return key + } + + func get(collectionId: Int32, id: MemoryBuffer) -> Coding? { + if let value = self.valueBox.get(self.table, key: self.key(collectionId: collectionId, id: id)), let content = Decoder(buffer: value).decodeRootObject() { + return content + } else { + return nil + } + } + + func remove(collectionId: Int32, id: MemoryBuffer) { + self.valueBox.remove(self.table, key: self.key(collectionId: collectionId, id: id)) + } + + func set(collectionId: Int32, id: MemoryBuffer, content: Coding) { + let encoder = Encoder() + encoder.encodeRootObject(content) + self.valueBox.set(self.table, key: self.key(collectionId: collectionId, id: id), value: encoder.readBufferNoCopy()) + } + + func getAllItemIds(collectionId: Int32) -> [MemoryBuffer] { + var result: [MemoryBuffer] = [] + self.valueBox.range(self.table, start: self.key(collectionId: collectionId, id: MemoryBuffer()), end: self.key(collectionId: collectionId + 1, id: MemoryBuffer()), keys: { key in + let id = MemoryBuffer(memory: malloc(key.length - 4)!, capacity: key.length - 4, length: key.length - 4, freeWhenDone: true) + memcpy(id.memory, key.memory.advanced(by: 4), key.length - 4) + result.append(id) + return true + }, limit: 0) + return result + } +} diff --git a/Postbox/OrderedItemListTable.swift b/Postbox/OrderedItemListTable.swift new file mode 100644 index 0000000000..f09eaac970 --- /dev/null +++ b/Postbox/OrderedItemListTable.swift @@ -0,0 +1,181 @@ +import Foundation + +enum OrderedItemListOperation { + case replace([OrderedItemListEntry]) + case addOrMoveToFirstPosition(OrderedItemListEntry, Int?) +} + +private enum OrderedItemListKeyNamespace: UInt8 { + case indexToId = 0 + case idToIndex = 1 +} + +final class OrderedItemListTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private let indexTable: OrderedItemListIndexTable + + init(valueBox: ValueBox, table: ValueBoxTable, indexTable: OrderedItemListIndexTable) { + self.indexTable = indexTable + + super.init(valueBox: valueBox, table: table) + } + + private func keyIndexToId(collectionId: Int32, itemIndex: UInt32) -> ValueBoxKey { + let key = ValueBoxKey(length: 1 + 4 + 4) + key.setUInt8(0, value: OrderedItemListKeyNamespace.indexToId.rawValue) + key.setInt32(1, value: collectionId) + key.setUInt32(5, value: itemIndex) + return key + } + + private func keyIdToIndex(collectionId: Int32, id: MemoryBuffer) -> ValueBoxKey { + let key = ValueBoxKey(length: 1 + 4 + id.length) + key.setUInt8(0, value: OrderedItemListKeyNamespace.idToIndex.rawValue) + key.setInt32(1, value: collectionId) + memcpy(key.memory.advanced(by: 5), id.memory, id.length) + return key + } + + func getItemIds(collectionId: Int32) -> [MemoryBuffer] { + var itemIds: [MemoryBuffer] = [] + self.valueBox.range(self.table, start: self.keyIndexToId(collectionId: collectionId, itemIndex: 0).predecessor, end: self.keyIndexToId(collectionId: collectionId, itemIndex: UInt32.max), values: { _, value in + itemIds.append(value) + return true + }, limit: 0) + return itemIds + } + + private func getIndex(collectionId: Int32, id: MemoryBuffer) -> UInt32? { + if let value = self.valueBox.get(self.table, key: self.keyIdToIndex(collectionId: collectionId, id: id)) { + var index: UInt32 = 0 + value.read(&index, offset: 0, length: 4) + return index + } else { + return nil + } + } + + func getItems(collectionId: Int32) -> [OrderedItemListEntry] { + var currentIds: [MemoryBuffer] = [] + self.valueBox.range(self.table, start: self.keyIndexToId(collectionId: collectionId, itemIndex: 0).predecessor, end: self.keyIndexToId(collectionId: collectionId, itemIndex: UInt32.max), values: { _, value in + currentIds.append(value) + return true + }, limit: 0) + var items: [OrderedItemListEntry] = [] + for id in currentIds { + if let contents = self.indexTable.get(collectionId: collectionId, id: id) as? OrderedItemListEntryContents { + items.append(OrderedItemListEntry(id: id, contents: contents)) + } else { + assertionFailure() + } + } + return items + } + + func replaceItems(collectionId: Int32, items: [OrderedItemListEntry], operations: inout [Int32: [OrderedItemListOperation]]) { + if operations[collectionId] == nil { + operations[collectionId] = [.replace(items)] + } else { + operations[collectionId]!.append(.replace(items)) + } + + var currentIds: [MemoryBuffer] = [] + var currentIndices: [UInt32] = [] + self.valueBox.range(self.table, start: self.keyIndexToId(collectionId: collectionId, itemIndex: 0).predecessor, end: self.keyIndexToId(collectionId: collectionId, itemIndex: UInt32.max), values: { key, value in + currentIndices.append(key.getUInt32(1 + 4)) + currentIds.append(value) + return true + }, limit: 0) + + for index in currentIndices { + self.valueBox.remove(self.table, key: self.keyIndexToId(collectionId: collectionId, itemIndex: index)) + } + for id in currentIds { + self.valueBox.remove(self.table, key: self.keyIdToIndex(collectionId: collectionId, id: id)) + self.indexTable.remove(collectionId: collectionId, id: id) + } + + for i in 0 ..< items.count { + self.valueBox.set(self.table, key: self.keyIndexToId(collectionId: collectionId, itemIndex: UInt32(i)), value: items[i].id) + var indexValue: UInt32 = UInt32(i) + self.valueBox.set(self.table, key: self.keyIdToIndex(collectionId: collectionId, id: items[i].id), value: MemoryBuffer(memory: &indexValue, capacity: 4, length: 4, freeWhenDone: false)) + self.indexTable.set(collectionId: collectionId, id: items[i].id, content: items[i].contents) + } + } + + func addItemOrMoveToFirstPosition(collectionId: Int32, item: OrderedItemListEntry, removeTailIfCountExceeds: Int?, operations: inout [Int32: [OrderedItemListOperation]]) { + if let index = self.getIndex(collectionId: collectionId, id: item.id), index == 0 { + return + } + + if operations[collectionId] == nil { + operations[collectionId] = [.addOrMoveToFirstPosition(item, removeTailIfCountExceeds)] + } else { + operations[collectionId]!.append(.addOrMoveToFirstPosition(item, removeTailIfCountExceeds)) + } + + var orderedIds = self.getItemIds(collectionId: collectionId) + + let offsetUntilIndex: Int + if let index = orderedIds.index(of: item.id) { + offsetUntilIndex = index + self.valueBox.remove(self.table, key: self.keyIndexToId(collectionId: collectionId, itemIndex: UInt32(index))) + } else { + if let removeTailIfCountExceeds = removeTailIfCountExceeds, orderedIds.count + 1 > removeTailIfCountExceeds { + self.indexTable.remove(collectionId: collectionId, id: orderedIds[orderedIds.count - 1]) + self.valueBox.remove(self.table, key: self.keyIdToIndex(collectionId: collectionId, id: orderedIds[orderedIds.count - 1])) + self.valueBox.remove(self.table, key: self.keyIndexToId(collectionId: collectionId, itemIndex: UInt32(orderedIds.count - 1))) + orderedIds.removeLast() + } + + offsetUntilIndex = orderedIds.count + self.indexTable.set(collectionId: collectionId, id: item.id, content: item.contents) + } + + for i in 0 ..< offsetUntilIndex { + var updatedIndex: UInt32 = UInt32(i + 1) + self.valueBox.set(self.table, key: self.keyIdToIndex(collectionId: collectionId, id: orderedIds[i]), value: MemoryBuffer(memory: &updatedIndex, capacity: 4, length: 4, freeWhenDone: false)) + self.valueBox.set(self.table, key: self.keyIndexToId(collectionId: collectionId, itemIndex: updatedIndex), value: orderedIds[i]) + } + + self.valueBox.set(self.table, key: self.keyIndexToId(collectionId: collectionId, itemIndex: 0), value: item.id) + var itemIndex: UInt32 = 0 + self.valueBox.set(self.table, key: self.keyIdToIndex(collectionId: collectionId, id: item.id), value: MemoryBuffer(memory: &itemIndex, capacity: 4, length: 4, freeWhenDone: false)) + } + + func testIntegrity(collectionId: Int32) -> Bool { + let orderedIds = self.getItemIds(collectionId: collectionId).map { ValueBoxKey($0) } + + let existingIds = Set(orderedIds) + let indexIds = Set(self.indexTable.getAllItemIds(collectionId: collectionId).map { ValueBoxKey($0) }) + if indexIds != existingIds { + return false + } + + var allIndexedKeys: [ValueBoxKey] = [] + var allIndices: [UInt32] = [] + self.valueBox.range(self.table, start: self.keyIdToIndex(collectionId: collectionId, id: MemoryBuffer()), end: self.keyIdToIndex(collectionId: collectionId + 1, id: MemoryBuffer()), values: { key, value in + let id = MemoryBuffer(memory: malloc(key.length - 5)!, capacity: key.length - 5, length: key.length - 5, freeWhenDone: true) + memcpy(id.memory, key.memory.advanced(by: 5), key.length - 5) + allIndexedKeys.append(ValueBoxKey(id)) + var index: UInt32 = 0 + value.read(&index, offset: 0, length: 4) + allIndices.append(index) + return true + }, limit: 0) + + if Set(allIndexedKeys) != existingIds { + print("\(allIndexedKeys) != \(existingIds)") + return false + } + + if allIndices.sorted() != Array((0 ..< orderedIds.count).map { UInt32($0) }) { + return false + } + + return true + } +} diff --git a/Postbox/OrderedItemListView.swift b/Postbox/OrderedItemListView.swift new file mode 100644 index 0000000000..bd71a6c584 --- /dev/null +++ b/Postbox/OrderedItemListView.swift @@ -0,0 +1,48 @@ +import Foundation + +final class MutableOrderedItemListView { + let collectionId: Int32 + var items: [OrderedItemListEntry] + + init(collectionId: Int32, getItems: (Int32) -> [OrderedItemListEntry]) { + self.collectionId = collectionId + self.items = getItems(collectionId) + } + + func replay(operations: [Int32: [OrderedItemListOperation]]) -> Bool { + var updated = false + + if let operations = operations[self.collectionId] { + for operation in operations { + switch operation { + case let .replace(items): + self.items = items + updated = true + case let .addOrMoveToFirstPosition(item, maxCount): + if let index = self.items.index(where: { $0.id == item.id }) { + self.items.remove(at: index) + self.items.insert(item, at: 0) + } else { + self.items.insert(item, at: 0) + if let maxCount = maxCount, self.items.count > maxCount { + self.items.removeLast() + } + } + updated = true + } + } + } + + return updated + } +} + +public final class OrderedItemListView { + public let collectionId: Int32 + public let items: [OrderedItemListEntry] + + init(_ view: MutableOrderedItemListView) { + self.collectionId = view.collectionId + self.items = view.items + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 57fca3495f..3014d9adf6 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -197,6 +197,22 @@ public final class Modifier { } } + public func replaceOrderedItemListItems(collectionId: Int32, items: [OrderedItemListEntry]) { + self.postbox?.replaceOrderedItemListItems(collectionId: collectionId, items: items) + } + + public func addOrMoveToFirstPositionOrderedItemListItem(collectionId: Int32, item: OrderedItemListEntry, removeTailIfCountExceeds: Int?) { + self.postbox?.addOrMoveToFirstPositionOrderedItemListItem(collectionId: collectionId, item: item, removeTailIfCountExceeds: removeTailIfCountExceeds) + } + + public func getOrderedListItemIds(collectionId: Int32) -> [MemoryBuffer] { + if let postbox = postbox { + return postbox.getOrderedListItemIds(collectionId: collectionId) + } else { + return [] + } + } + public func getMessage(_ id: MessageId) -> Message? { if let postbox = self.postbox { if let entry = postbox.messageHistoryIndexTable.get(id) { @@ -335,6 +351,7 @@ public final class Postbox { private var peerPipes: [PeerId: ValuePipe] = [:] + private var currentUpdatedState: Coding? private var currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] private var currentUpdatedChatListInclusions: [PeerId: PeerChatListInclusion] = [:] private var currentUnsentOperations: [IntermediateMessageHistoryUnsentOperation] = [] @@ -352,6 +369,7 @@ public final class Postbox { private var currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation] = [] private var currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation] = [] private var currentPreferencesOperations: [PreferencesOperation] = [] + private var currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]] = [:] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -405,6 +423,8 @@ public final class Postbox { var timestampBasedMessageAttributesTable: TimestampBasedMessageAttributesTable! var timestampBasedMessageAttributesIndexTable: TimestampBasedMessageAttributesIndexTable! var preferencesTable: PreferencesTable! + var orderedItemListTable: OrderedItemListTable! + var orderedItemListIndexTable: OrderedItemListIndexTable! //temporary var peerRatingTable: RatingTable! @@ -529,6 +549,8 @@ public final class Postbox { self.peerMergedOperationLogIndexTable = PeerMergedOperationLogIndexTable(valueBox: self.valueBox, table: PeerMergedOperationLogIndexTable.tableSpec(30), metadataTable: self.peerOperationLogMetadataTable!) self.peerOperationLogTable = PeerOperationLogTable(valueBox: self.valueBox, table: PeerOperationLogTable.tableSpec(31), metadataTable: self.peerOperationLogMetadataTable, mergedIndexTable: self.peerMergedOperationLogIndexTable) self.preferencesTable = PreferencesTable(valueBox: self.valueBox, table: PreferencesTable.tableSpec(35)) + self.orderedItemListIndexTable = OrderedItemListIndexTable(valueBox: self.valueBox, table: OrderedItemListIndexTable.tableSpec(37)) + self.orderedItemListTable = OrderedItemListTable(valueBox: self.valueBox, table: OrderedItemListTable.tableSpec(38), indexTable: self.orderedItemListIndexTable) self.tables.append(self.keychainTable) self.tables.append(self.peerTable) @@ -563,6 +585,8 @@ public final class Postbox { self.tables.append(self.peerMergedOperationLogIndexTable) self.tables.append(self.peerOperationLogTable) self.tables.append(self.preferencesTable) + self.tables.append(self.orderedItemListTable) + self.tables.append(self.orderedItemListIndexTable) self.transactionStateVersion = self.metadataTable.transactionStateVersion() @@ -600,35 +624,15 @@ public final class Postbox { return nextId } - private var cachedState: Coding? - fileprivate func setState(_ state: Coding) { - self.cachedState = state - + self.currentUpdatedState = state self.metadataTable.setState(state) self.statePipe.putNext(state) } fileprivate func getState() -> Coding? { - if let cachedState = self.cachedState { - return cachedState - } else { - if let state = self.metadataTable.state() { - self.cachedState = state - return state - } - - return nil - } - } - - @available(*, deprecated: 1.0) - public func state() -> Signal { - return self.modify { modifier -> Signal in - return Signal.single(self.getState()) - |> then(self.statePipe.signal() |> map { $0 }) - } |> switchToLatest + return self.metadataTable.state() } public func keychainEntryForKey(_ key: String) -> Data? { @@ -1001,7 +1005,7 @@ public final class Postbox { return self.peerTable.get(peerId) }, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) - let transaction = PostboxTransaction(currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentUpdatedState: self.currentUpdatedState, currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, currentOrderedItemListOperations: self.currentOrderedItemListOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -1015,6 +1019,7 @@ public final class Postbox { } } + self.currentUpdatedState = nil self.currentOperationsByPeerId.removeAll() self.currentUpdatedChatListInclusions.removeAll() self.currentFilledHolesByPeerId.removeAll() @@ -1034,6 +1039,7 @@ public final class Postbox { self.currentPeerMergedOperationLogOperations.removeAll() self.currentTimestampBasedMessageAttributesOperations.removeAll() self.currentPreferencesOperations.removeAll() + self.currentOrderedItemListOperations.removeAll() for table in self.tables { table.beforeCommit() @@ -1582,6 +1588,24 @@ public final class Postbox { } |> switchToLatest } + public func stateView() -> Signal { + return self.modify { modifier -> Signal in + let mutableView = MutablePostboxStateView(state: self.getState()) + + let (index, signal) = self.viewTracker.addPostboxStateView(mutableView) + + return (.single(PostboxStateView(mutableView)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removePostboxStateView(index) + } + } + } + } |> switchToLatest + } + public func messageHistoryHolesView() -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -1630,9 +1654,15 @@ public final class Postbox { } } - public func itemCollectionsView(namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int) -> Signal { - return self.modify { modifier -> ItemCollectionsView in - let mutableView = MutableItemCollectionsView(namespaces: namespaces, aroundIndex: aroundIndex, count: count, getInfos: { namespace in + public func itemCollectionsView(orderedItemListCollectionIds: [Int32], namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int) -> Signal { + return self.modify { modifier -> Signal in + let itemListViews = orderedItemListCollectionIds.map { collectionId -> MutableOrderedItemListView in + return MutableOrderedItemListView(collectionId: collectionId, getItems: { collectionId in + return self.orderedItemListTable.getItems(collectionId: collectionId) + }) + } + + let mutableView = MutableItemCollectionsView(orderedItemListsViews: itemListViews, namespaces: namespaces, aroundIndex: aroundIndex, count: count, getInfos: { namespace in return self.itemCollectionInfoTable.getInfos(namespace: namespace) }, lowerCollectionId: { namespaceList, collectionId, collectionIndex in return self.itemCollectionInfoTable.lowerCollectionId(namespaceList: namespaceList, collectionId: collectionId, index: collectionIndex) @@ -1643,8 +1673,39 @@ public final class Postbox { }, higherItems: { collectionId, itemIndex, count in return self.itemCollectionItemTable.higherItems(collectionId: collectionId, itemIndex: itemIndex, count: count) }) - return ItemCollectionsView(mutableView) - } + + let (index, signal) = self.viewTracker.addItemCollectionView(mutableView) + + return (.single(ItemCollectionsView(mutableView)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeItemCollectionView(index) + } + } + } + } |> switchToLatest + } + + public func orderedItemListView(collectionId: Int32) -> Signal { + return self.modify { modifier -> Signal in + let mutableView = MutableOrderedItemListView(collectionId: collectionId, getItems: { collectionId in + return self.orderedItemListTable.getItems(collectionId: collectionId) + }) + + let (index, signal) = self.viewTracker.addOrderedItemListView(mutableView) + + return (.single(OrderedItemListView(mutableView)) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeOrderedItemListView(index) + } + } + } + } |> switchToLatest } public func mergedOperationLogView(tag: PeerOperationLogTag, limit: Int) -> Signal { @@ -1767,6 +1828,18 @@ public final class Postbox { self.preferencesTable.set(key: key, value: value, operations: &self.currentPreferencesOperations) } + fileprivate func replaceOrderedItemListItems(collectionId: Int32, items: [OrderedItemListEntry]) { + self.orderedItemListTable.replaceItems(collectionId: collectionId, items: items, operations: &self.currentOrderedItemListOperations) + } + + fileprivate func addOrMoveToFirstPositionOrderedItemListItem(collectionId: Int32, item: OrderedItemListEntry, removeTailIfCountExceeds: Int?) { + self.orderedItemListTable.addItemOrMoveToFirstPosition(collectionId: collectionId, item: item, removeTailIfCountExceeds: removeTailIfCountExceeds, operations: &self.currentOrderedItemListOperations) + } + + fileprivate func getOrderedListItemIds(collectionId: Int32) -> [MemoryBuffer] { + return self.orderedItemListTable.getItemIds(collectionId: collectionId) + } + public func isMasterClient() -> Signal { return self.modify { modifier -> Signal in let sessionClientId = self.sessionClientId diff --git a/Postbox/PostboxStateView.swift b/Postbox/PostboxStateView.swift new file mode 100644 index 0000000000..0644ecf82b --- /dev/null +++ b/Postbox/PostboxStateView.swift @@ -0,0 +1,22 @@ +import Foundation + +final class MutablePostboxStateView { + var state: Coding? + + init(state: Coding?) { + self.state = state + } + + func replay(updatedState: Coding) -> Bool { + self.state = updatedState + return true + } +} + +public final class PostboxStateView { + public let state: Coding? + + init(_ view: MutablePostboxStateView) { + self.state = view.state + } +} diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index b7fbdfd0af..3f319555e3 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -1,6 +1,7 @@ import Foundation final class PostboxTransaction { + let currentUpdatedState: Coding? let currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] let peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]] let removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] @@ -15,6 +16,7 @@ final class PostboxTransaction { let currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation] let currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation] let currentPreferencesOperations: [PreferencesOperation] + let currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]] let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -23,6 +25,9 @@ final class PostboxTransaction { let currentUpdatedMasterClientId: Int64? var isEmpty: Bool { + if currentUpdatedState != nil { + return false + } if !currentOperationsByPeerId.isEmpty { return false } @@ -80,10 +85,14 @@ final class PostboxTransaction { if !currentPreferencesOperations.isEmpty { return false } + if !currentOrderedItemListOperations.isEmpty { + return false + } return true } - init(currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentUpdatedState: Coding?, currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + self.currentUpdatedState = currentUpdatedState self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles self.removedHolesByPeerId = removedHolesByPeerId @@ -100,6 +109,7 @@ final class PostboxTransaction { self.unsentMessageOperations = unsentMessageOperations self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations self.currentPreferencesOperations = currentPreferencesOperations + self.currentOrderedItemListOperations = currentOrderedItemListOperations self.updatedMedia = updatedMedia self.replaceContactPeerIds = replaceContactPeerIds self.currentUpdatedMasterClientId = currentUpdatedMasterClientId diff --git a/Postbox/ValueBoxKey.swift b/Postbox/ValueBoxKey.swift index d54e8503cc..2e0553ae90 100644 --- a/Postbox/ValueBoxKey.swift +++ b/Postbox/ValueBoxKey.swift @@ -165,8 +165,15 @@ public struct ValueBoxKey: Hashable, CustomStringConvertible, Comparable { memory[i] = byte break } else { - byte = 0xff - memory[i] = byte + if i == 0 { + assert(self.length > 1) + let previousKey = ValueBoxKey(length: self.length - 1) + memcpy(previousKey.memory, self.memory, self.length - 1) + return previousKey + } else { + byte = 0xff + memory[i] = byte + } } i -= 1 } diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 367a886212..43804688c3 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -55,9 +55,12 @@ final class ViewTracker { private let getTimestampBasedMessageAttributesHead: (UInt16) -> TimestampBasedMessageAttributesEntry? private var timestampBasedMessageAttributesViews = Bag<(MutableTimestampBasedMessageAttributesView, ValuePipe)>() + private var postboxStateViews = Bag<(MutablePostboxStateView, ValuePipe)>() private var messageViews = Bag<(MutableMessageView, ValuePipe)>() private var preferencesViews = Bag<(MutablePreferencesView, ValuePipe)>() private var multiplePeersViews = Bag<(MutableMultiplePeersView, ValuePipe)>() + private var itemCollectionsViews = Bag<(MutableItemCollectionsView, ValuePipe)>() + private var orderedItemListViews = Bag<(MutableOrderedItemListView, ValuePipe)>() init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, operationLogGetOperations: @escaping (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], operationLogGetTailIndex: @escaping (PeerOperationLogTag) -> Int32?, getTimestampBasedMessageAttributesHead: @escaping (UInt16) -> TimestampBasedMessageAttributesEntry?, getPreferencesEntry: @escaping (ValueBoxKey) -> PreferencesEntry?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue @@ -82,6 +85,17 @@ final class ViewTracker { self.synchronizeReadStatesView = MutableSynchronizePeerReadStatesView(operations: synchronizePeerReadStateOperations) } + func addPostboxStateView(_ view: MutablePostboxStateView) -> (Bag<(MutablePostboxStateView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.postboxStateViews.add(record) + + return (index, record.1.signal()) + } + + func removePostboxStateView(_ index: Bag<(MutablePostboxStateView, ValuePipe)>.Index) { + self.postboxStateViews.remove(index) + } + func addMessageHistoryView(_ peerId: PeerId, view: MutableMessageHistoryView) -> (Bag<(MutableMessageHistoryView, ValuePipe<(MessageHistoryView, ViewUpdateType)>)>.Index, Signal<(MessageHistoryView, ViewUpdateType), NoError>) { let record = (view, ValuePipe<(MessageHistoryView, ViewUpdateType)>()) @@ -252,6 +266,28 @@ final class ViewTracker { self.multiplePeersViews.remove(index) } + func addOrderedItemListView(_ view: MutableOrderedItemListView) -> (Bag<(MutableOrderedItemListView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.orderedItemListViews.add(record) + + return (index, record.1.signal()) + } + + func removeOrderedItemListView(_ index: Bag<(MutableOrderedItemListView, ValuePipe)>.Index) { + self.orderedItemListViews.remove(index) + } + + func addItemCollectionView(_ view: MutableItemCollectionsView) -> (Bag<(MutableItemCollectionsView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.itemCollectionsViews.add(record) + + return (index, record.1.signal()) + } + + func removeItemCollectionView(_ index: Bag<(MutableItemCollectionsView, ValuePipe)>.Index) { + self.itemCollectionsViews.remove(index) + } + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: ChatListIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] @@ -327,6 +363,14 @@ final class ViewTracker { func updateViews(transaction: PostboxTransaction) { var updateTrackedHolesPeerIds: [PeerId] = [] + if let currentUpdatedState = transaction.currentUpdatedState { + for (mutableView, pipe) in self.postboxStateViews.copyItems() { + if mutableView.replay(updatedState: currentUpdatedState) { + pipe.putNext(PostboxStateView(mutableView)) + } + } + } + for (peerId, bag) in self.messageHistoryViews { var updateHoles = false let operations = transaction.currentOperationsByPeerId[peerId] @@ -466,6 +510,18 @@ final class ViewTracker { pipe.putNext(MultiplePeersView(mutableView)) } } + + for (mutableView, pipe) in self.orderedItemListViews.copyItems() { + if mutableView.replay(operations: transaction.currentOrderedItemListOperations) { + pipe.putNext(OrderedItemListView(mutableView)) + } + } + + for (mutableView, pipe) in self.itemCollectionsViews.copyItems() { + if mutableView.replay(orderedItemListOperations: transaction.currentOrderedItemListOperations) { + pipe.putNext(ItemCollectionsView(mutableView)) + } + } } private func updateTrackedChatListHoles() { diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index 21ec3f1544..520680cbcf 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -62,7 +62,7 @@ class ChatListTableTests: XCTestCase { var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable? var peerChatInterfaceStateTable: PeerChatInterfaceStateTable? var peerTable: PeerTable? - var peerNameTokenIndexTable: PeerNameTokenIndexTable? + var peerNameTokenIndexTable: ReverseIndexReferenceTable? var peerNameIndexTable: PeerNameIndexTable? var notificationSettingsTable: PeerNotificationSettingsTable? var globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable? @@ -92,7 +92,7 @@ class ChatListTableTests: XCTestCase { self.globallyUniqueMessageIdsTable = MessageGloballyUniqueIdTable(valueBox: self.valueBox!, table: MessageGloballyUniqueIdTable.tableSpec(24)) self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, table: MessageHistoryTable.tableSpec(4), messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!, historyMetadataTable: self.historyMetadataTable!, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable!, unsentTable: self.unsentTable!, tagsTable: self.tagsTable!, readStateTable: self.readStateTable!, synchronizeReadStateTable: self.synchronizeReadStateTable!) self.peerTable = PeerTable(valueBox: self.valueBox!, table: PeerTable.tableSpec(20)) - self.peerNameTokenIndexTable = PeerNameTokenIndexTable(valueBox: self.valueBox!, table: PeerNameTokenIndexTable.tableSpec(21)) + self.peerNameTokenIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox!, table: ReverseIndexReferenceTable.tableSpec(21)) self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox!, table: PeerNameIndexTable.tableSpec(22), peerTable: self.peerTable!, peerNameTokenIndexTable: self.peerNameTokenIndexTable!) self.notificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox!, table: PeerNotificationSettingsTable.tableSpec(23)) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox!, table: ChatListIndexTable.tableSpec(5), peerNameIndexTable: self.peerNameIndexTable!, metadataTable: self.historyMetadataTable!, readStateTable: self.readStateTable!, notificationSettingsTable: self.notificationSettingsTable!) diff --git a/PostboxTests/OrderedItemListTableTests.swift b/PostboxTests/OrderedItemListTableTests.swift new file mode 100644 index 0000000000..a1197ad009 --- /dev/null +++ b/PostboxTests/OrderedItemListTableTests.swift @@ -0,0 +1,124 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +import SwiftSignalKit + +private struct TestListItem: OrderedItemListEntryContents { + init() { + + } + + init(decoder: Decoder) { + } + + func encode(_ encoder: Encoder) { + + } +} + +private let declaredEncodables: Void = { + declareEncodable(TestListItem.self, f: { TestListItem(decoder: $0) }) + return () +}() + +class OrderedItemListTableTests: XCTestCase { + var valueBox: ValueBox? + var path: String? + + var itemListTable: OrderedItemListTable? + var itemListIndexTable: OrderedItemListIndexTable? + + override func setUp() { + super.setUp() + + let _ = declaredEncodables + + var randomId: Int64 = 0 + arc4random_buf(&randomId, 8) + path = NSTemporaryDirectory() + "\(randomId)" + self.valueBox = SqliteValueBox(basePath: path!, queue: Queue.mainQueue()) + + self.itemListIndexTable = OrderedItemListIndexTable(valueBox: self.valueBox!, table: OrderedItemListIndexTable.tableSpec(0)) + self.itemListTable = OrderedItemListTable(valueBox: self.valueBox!, table: OrderedItemListTable.tableSpec(1), indexTable: self.itemListIndexTable!) + } + + override func tearDown() { + super.tearDown() + + self.valueBox = nil + let _ = try? FileManager.default.removeItem(atPath: path!) + self.path = nil + } + + private func expectIds(_ ids: [Int32]) { + let actualIds = self.itemListTable!.getItems(collectionId: 0).map { entry -> Int32 in + var value: Int32 = 0 + memcpy(&value, entry.id.memory, 4) + return value + } + XCTAssert(ids == actualIds, "Expected\n\(ids)\nActual\n\(actualIds)") + } + + private func setIds(_ ids: [Int32]) { + self.itemListTable!.replaceItems(collectionId: 0, items: ids.map { id -> OrderedItemListEntry in + var idValue: Int32 = id + let buffer = MemoryBuffer(memory: malloc(4)!, capacity: 4, length: 4, freeWhenDone: true) + memcpy(buffer.memory, &idValue, 4) + return OrderedItemListEntry(id: buffer, contents: TestListItem()) + }) + XCTAssert(self.itemListTable!.testIntegrity(collectionId: 0), "Index integrity violated") + } + + private func addOrMoveId(_ id: Int32, _ maxCount: Int? = nil) { + var idValue: Int32 = id + let buffer = MemoryBuffer(memory: malloc(4)!, capacity: 4, length: 4, freeWhenDone: true) + memcpy(buffer.memory, &idValue, 4) + self.itemListTable!.addItemOrMoveToFirstPosition(collectionId: 0, item: OrderedItemListEntry(id: buffer, contents: TestListItem()), removeTailIfCountExceeds: maxCount) + XCTAssert(self.itemListTable!.testIntegrity(collectionId: 0), "Index integrity violated") + } + + func testEmpty() { + expectIds([]) + } + + func testSetIds() { + expectIds([]) + setIds([10, 20, 30]) + expectIds([10, 20, 30]) + setIds([40, 50, 60]) + expectIds([40, 50, 60]) + setIds([10, 20, 30, 40]) + expectIds([10, 20, 30, 40]) + } + + func testAddItem() { + expectIds([]) + addOrMoveId(10) + expectIds([10]) + addOrMoveId(20) + expectIds([20, 10]) + addOrMoveId(30) + expectIds([30, 20, 10]) + addOrMoveId(40, 4) + expectIds([40, 30, 20, 10]) + addOrMoveId(50, 4) + expectIds([50, 40, 30, 20]) + } + + func testMoveItem() { + expectIds([]) + setIds([10, 20, 30]) + expectIds([10, 20, 30]) + addOrMoveId(10) + expectIds([10, 20, 30]) + addOrMoveId(20) + expectIds([20, 10, 30]) + addOrMoveId(30) + expectIds([30, 20, 10]) + } +} From dc0504f23dcb2281b0a64b6af61268d7af341caf Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 7 Feb 2017 22:46:24 +0300 Subject: [PATCH 082/427] no message --- Postbox.xcodeproj/project.pbxproj | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index e18e16a104..52d67c2083 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -47,6 +47,11 @@ D044E1641B2AD718001EE087 /* MurMurHash32.h in Headers */ = {isa = PBXBuildFile; fileRef = D044E1611B2AD667001EE087 /* MurMurHash32.h */; settings = {ATTRIBUTES = (Public, ); }; }; D049EAF01E44D9B900A2CD3A /* PostboxStateView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D049EAEF1E44D9B900A2CD3A /* PostboxStateView.swift */; }; D049EAF11E44D9B900A2CD3A /* PostboxStateView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D049EAEF1E44D9B900A2CD3A /* PostboxStateView.swift */; }; + D050F2651E4A5B4800988324 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; + D050F2661E4A5B5A00988324 /* MessageGloballyUniqueIdTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */; }; + D050F2671E4A5B5A00988324 /* TimestampBasedMessageAttributesTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AAD1B11E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift */; }; + D050F2681E4A5B5A00988324 /* TimestampBasedMessageAttributesIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AAD1B01E3266B100D5B9DE /* TimestampBasedMessageAttributesIndexTable.swift */; }; + D050F2691E4A5B5A00988324 /* MultiplePeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */; }; D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; D060B77B1CF4845A0050BE9B /* ReadStateTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */; }; @@ -674,6 +679,7 @@ files = ( D0B418201D7DFDFD004562A4 /* sqlite3ext.h in Headers */, D0B4185D1D7DFE35004562A4 /* IpcNotifier.h in Headers */, + D050F2651E4A5B4800988324 /* fts3_tokenizer.h in Headers */, D0B4185B1D7DFE2C004562A4 /* MurMurHash32.h in Headers */, D0B4181E1D7DFDF8004562A4 /* SQLite-Bridging.h in Headers */, D0B4181F1D7DFDFB004562A4 /* sqlite3.h in Headers */, @@ -824,6 +830,10 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + D050F2661E4A5B5A00988324 /* MessageGloballyUniqueIdTable.swift in Sources */, + D050F2671E4A5B5A00988324 /* TimestampBasedMessageAttributesTable.swift in Sources */, + D050F2681E4A5B5A00988324 /* TimestampBasedMessageAttributesIndexTable.swift in Sources */, + D050F2691E4A5B5A00988324 /* MultiplePeersView.swift in Sources */, C2A315BE1E2E733900D89000 /* PeerMergedOperationLogIndexTable.swift in Sources */, C2A315BF1E2E733900D89000 /* PeerMergedOperationLogView.swift in Sources */, C2A315BD1E2E732000D89000 /* PeerOperationLogMetadataTable.swift in Sources */, From 0ea6f9831a970ecbfae53ec0a96756010694ef42 Mon Sep 17 00:00:00 2001 From: Peter Date: Sat, 11 Feb 2017 17:03:28 +0300 Subject: [PATCH 083/427] no message --- Postbox.xcodeproj/project.pbxproj | 40 ++++++++++++++++++++---- Postbox/IncrementalUpgrade.swift | 10 ++++++ Postbox/IncrementalUpgrade_v11_v12.swift | 15 +++++++++ Postbox/Postbox.swift | 12 +------ Postbox/PostboxAccess.swift | 23 ++++++++++++++ Postbox/PostboxClientId.swift | 8 ----- 6 files changed, 83 insertions(+), 25 deletions(-) create mode 100644 Postbox/IncrementalUpgrade.swift create mode 100644 Postbox/IncrementalUpgrade_v11_v12.swift create mode 100644 Postbox/PostboxAccess.swift delete mode 100644 Postbox/PostboxClientId.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 52d67c2083..32bd946899 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -137,7 +137,6 @@ D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8B1D65D488002C78E7 /* MessageHistoryHolesView.swift */; }; D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8D1D65D49C002C78E7 /* ChatListHolesView.swift */; }; D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */; }; - D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510FA1D63C06E00A97B8A /* PostboxClientId.swift */; }; D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7811B28ADD000A402D9 /* Postbox.swift */; }; D0B418581D7DFE29004562A4 /* SeedConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = D044CA291C617D39002160FF /* SeedConfiguration.swift */; }; D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F31D63BA8400A97B8A /* PostboxTransaction.swift */; }; @@ -166,12 +165,17 @@ D0D510F41D63BA8400A97B8A /* PostboxTransaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F31D63BA8400A97B8A /* PostboxTransaction.swift */; }; D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */; }; D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */; }; - D0D510FB1D63C06E00A97B8A /* PostboxClientId.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510FA1D63C06E00A97B8A /* PostboxClientId.swift */; }; D0D511001D64A58900A97B8A /* IpcPipe.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D510FF1D64A58900A97B8A /* IpcPipe.swift */; }; D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */ = {isa = PBXBuildFile; fileRef = D0D511011D64D73D00A97B8A /* IpcNotifier.mm */; }; D0D511041D64D91C00A97B8A /* IpcNotifier.h in Headers */ = {isa = PBXBuildFile; fileRef = D0D511031D64D75200A97B8A /* IpcNotifier.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */; }; D0D949F51D35353900740E02 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; + D0DA44411E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */; }; + D0DA44421E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */; }; + D0DA44451E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44441E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift */; }; + D0DA44461E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44441E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift */; }; + D0DA44481E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */; }; + D0DA44491E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */; }; D0DF0C8F1D81A350008AEB01 /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DF0C8E1D81A350008AEB01 /* PeerView.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -340,12 +344,14 @@ D0D510F31D63BA8400A97B8A /* PostboxTransaction.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxTransaction.swift; sourceTree = ""; }; D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryOperation.swift; sourceTree = ""; }; D0D510F81D63BCC200A97B8A /* IntermediateMessage.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IntermediateMessage.swift; sourceTree = ""; }; - D0D510FA1D63C06E00A97B8A /* PostboxClientId.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxClientId.swift; sourceTree = ""; }; D0D510FF1D64A58900A97B8A /* IpcPipe.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IpcPipe.swift; sourceTree = ""; }; D0D511011D64D73D00A97B8A /* IpcNotifier.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = IpcNotifier.mm; sourceTree = ""; }; D0D511031D64D75200A97B8A /* IpcNotifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = IpcNotifier.h; sourceTree = ""; }; D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessResourceTests.swift; sourceTree = ""; }; D0D949F41D35353900740E02 /* MappedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MappedFile.swift; sourceTree = ""; }; + D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IncrementalUpgrade.swift; sourceTree = ""; }; + D0DA44441E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IncrementalUpgrade_v11_v12.swift; sourceTree = ""; }; + D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxAccess.swift; sourceTree = ""; }; D0DF0C8E1D81A350008AEB01 /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -465,6 +471,23 @@ name = Frameworks; sourceTree = ""; }; + D0DA443F1E4C7834005FDCA7 /* Upgrade */ = { + isa = PBXGroup; + children = ( + D0DA44431E4C796C005FDCA7 /* v11-v12 */, + D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */, + ); + name = Upgrade; + sourceTree = ""; + }; + D0DA44431E4C796C005FDCA7 /* v11-v12 */ = { + isa = PBXGroup; + children = ( + D0DA44441E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift */, + ); + name = "v11-v12"; + sourceTree = ""; + }; D0E1DE161C5EB06000C7826E /* Tables */ = { isa = PBXGroup; children = ( @@ -614,6 +637,7 @@ D0E3A74C1B28A7E300A402D9 /* Postbox */ = { isa = PBXGroup; children = ( + D0DA443F1E4C7834005FDCA7 /* Upgrade */, D075163D1B2D9CEF00AE42E0 /* PostboxPrivate */, D07515FC1B2C44A200AE42E0 /* thirdparty */, D0E1DE181C5EB09300C7826E /* Utils */, @@ -622,8 +646,8 @@ D0E1DE161C5EB06000C7826E /* Tables */, D0E1DE1A1C5EB0DB00C7826E /* Views */, D05F09981C9CAC1100BB6F96 /* Media Box */, - D0D510FA1D63C06E00A97B8A /* PostboxClientId.swift */, D0E3A7811B28ADD000A402D9 /* Postbox.swift */, + D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */, D044CA291C617D39002160FF /* SeedConfiguration.swift */, D0D510F31D63BA8400A97B8A /* PostboxTransaction.swift */, D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */, @@ -833,6 +857,7 @@ D050F2661E4A5B5A00988324 /* MessageGloballyUniqueIdTable.swift in Sources */, D050F2671E4A5B5A00988324 /* TimestampBasedMessageAttributesTable.swift in Sources */, D050F2681E4A5B5A00988324 /* TimestampBasedMessageAttributesIndexTable.swift in Sources */, + D0DA44421E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */, D050F2691E4A5B5A00988324 /* MultiplePeersView.swift in Sources */, C2A315BE1E2E733900D89000 /* PeerMergedOperationLogIndexTable.swift in Sources */, C2A315BF1E2E733900D89000 /* PeerMergedOperationLogView.swift in Sources */, @@ -868,7 +893,6 @@ D0F7B1CC1E045C6A007EB8A5 /* ChatListIndexTable.swift in Sources */, D0B4185E1D7DFE54004562A4 /* IpcNotifier.mm in Sources */, D0F7B1D31E045C6A007EB8A5 /* CachedPeerDataTable.swift in Sources */, - D0B418561D7DFE29004562A4 /* PostboxClientId.swift in Sources */, D0F7B1D51E045C6A007EB8A5 /* PeerChatStateTable.swift in Sources */, D0F82CFE1E4345D7007E499C /* OrderedItemListTable.swift in Sources */, D0F7B1C51E045C6A007EB8A5 /* MessageHistoryMetadataTable.swift in Sources */, @@ -901,6 +925,7 @@ D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopIndexableMessageIds.swift in Sources */, D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */, D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, + D0DA44461E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift in Sources */, D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, D0F7B1E01E045C6A007EB8A5 /* ItemCacheTable.swift in Sources */, D0F7B1E11E045C6A007EB8A5 /* ReverseIndexReferenceTable.swift in Sources */, @@ -908,6 +933,7 @@ D073CE7B1DCBF3B4007511FD /* PeerNameIndexRepresentation.swift in Sources */, D073CE7D1DCBF3B4007511FD /* PeerPresence.swift in Sources */, D0B4184C1D7DFE20004562A4 /* ContactPeerIdsView.swift in Sources */, + D0DA44491E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */, D073CE751DCBF3B4007511FD /* Message.swift in Sources */, D087C20D1E43C11C00D686F8 /* OrderedItemListView.swift in Sources */, D00C7CD51E365C4E0080C3D5 /* PeerChatListInclusion.swift in Sources */, @@ -942,6 +968,7 @@ D08775061E3E3F2100A97350 /* PreferencesView.swift in Sources */, D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */, + D0DA44411E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */, D08D451F1D5D2CA700A7428A /* RatingTable.swift in Sources */, D07CFF831DCA909100761F81 /* PeerChatInterfaceStateTable.swift in Sources */, D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */, @@ -1010,6 +1037,7 @@ D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, D00C7CD41E365C4E0080C3D5 /* PeerChatListInclusion.swift in Sources */, D0A18D671E16874D004C6734 /* UnreadMessageCountsView.swift in Sources */, + D0DA44451E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */, D07827C11E0079CB00071108 /* StringIndexTokens.swift in Sources */, @@ -1017,6 +1045,7 @@ D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D019B1CF1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift in Sources */, D0F9E86D1C5A0E5D00037222 /* MetadataTable.swift in Sources */, + D0DA44481E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D0D510F61D63BBE100A97B8A /* MessageHistoryOperation.swift in Sources */, D0BC38721E409E670044D6FE /* RenderedPeer.swift in Sources */, @@ -1037,7 +1066,6 @@ D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */, D0D510F41D63BA8400A97B8A /* PostboxTransaction.swift in Sources */, - D0D510FB1D63C06E00A97B8A /* PostboxClientId.swift in Sources */, D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */, D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, D0D511001D64A58900A97B8A /* IpcPipe.swift in Sources */, diff --git a/Postbox/IncrementalUpgrade.swift b/Postbox/IncrementalUpgrade.swift new file mode 100644 index 0000000000..e8431d30b2 --- /dev/null +++ b/Postbox/IncrementalUpgrade.swift @@ -0,0 +1,10 @@ +import Foundation +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif + +protocol IncrementalUpgrade { + func upgrade(from previous: ValueBox, tmpBasePath: String) -> Signal +} diff --git a/Postbox/IncrementalUpgrade_v11_v12.swift b/Postbox/IncrementalUpgrade_v11_v12.swift new file mode 100644 index 0000000000..f822c0461b --- /dev/null +++ b/Postbox/IncrementalUpgrade_v11_v12.swift @@ -0,0 +1,15 @@ +import Foundation + +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif + +final class IncrementalUpgrade_v11_v12: IncrementalUpgrade { + func upgrade(from previous: ValueBox, tmpBasePath: String) -> Signal { + assertionFailure() + + return .never() + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 3014d9adf6..fa5a3fda9d 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -1,5 +1,4 @@ import Foundation -import sqlcipher #if os(macOS) import SwiftSignalKitMac @@ -488,15 +487,6 @@ public final class Postbox { //#if TARGET_IPHONE_SIMULATOR - //self.debugRestoreState("_empty") - - // debugging large amount of updates - //self.debugSaveState("beforeHoles") - //self.debugRestoreState("beforeHoles") - - // debugging unread counters - //self.debugRestoreState("afterLogin") - //self.debugSaveState(name: "previous") //self.debugRestoreState(name: "previous") @@ -507,7 +497,7 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 11 + let currentUserVersion: Int32 = 12 if userVersion != currentUserVersion { self.valueBox.drop() diff --git a/Postbox/PostboxAccess.swift b/Postbox/PostboxAccess.swift new file mode 100644 index 0000000000..36690f3485 --- /dev/null +++ b/Postbox/PostboxAccess.swift @@ -0,0 +1,23 @@ +import Foundation +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif + +public enum PostboxAuthorizationChallenge { + case numericPassword(length: Int32) + case arbitraryPassword() +} + +public enum PostboxAccess { + case unlocked(Postbox) + case locked(PostboxAuthorizationChallenge) +} + +public func accessPostbox(basePath: String, password: String?) -> Signal { + return Signal { subscriber in + return ActionDisposable { + } + } +} diff --git a/Postbox/PostboxClientId.swift b/Postbox/PostboxClientId.swift deleted file mode 100644 index 5bfe785721..0000000000 --- a/Postbox/PostboxClientId.swift +++ /dev/null @@ -1,8 +0,0 @@ - -public struct PostboxClientId { - public let stableId: Int32 - - public init(stableId: Int32) { - self.stableId = stableId - } -} From f0e2cbee88de0df75eebe0a683a7745bff1a2c9f Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 19 Feb 2017 23:22:06 +0300 Subject: [PATCH 084/427] no message --- Postbox.xcodeproj/project.pbxproj | 38 ++++++ Postbox/AccountManager.swift | 156 ++++++++++++++++++++++ Postbox/AccountManagerMetadataTable.swift | 37 +++++ Postbox/AccountManagerRecordTable.swift | 46 +++++++ Postbox/AccountRecord.swift | 71 ++++++++++ Postbox/AccountRecordsView.swift | 78 +++++++++++ Postbox/Coding.swift | 40 +++--- Postbox/MessageHistoryTable.swift | 12 +- Postbox/PeerView.swift | 2 +- Postbox/Postbox.swift | 32 +++-- Postbox/SqliteValueBox.swift | 57 ++++++++ Postbox/ValueBox.swift | 1 + 12 files changed, 539 insertions(+), 31 deletions(-) create mode 100644 Postbox/AccountManager.swift create mode 100644 Postbox/AccountManagerMetadataTable.swift create mode 100644 Postbox/AccountManagerRecordTable.swift create mode 100644 Postbox/AccountRecord.swift create mode 100644 Postbox/AccountRecordsView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 32bd946899..ddfa9d9b5e 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -155,6 +155,16 @@ D0BC386C1E3FCEE50044D6FE /* MultiplePeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */; }; D0BC38721E409E670044D6FE /* RenderedPeer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC38711E409E670044D6FE /* RenderedPeer.swift */; }; D0BC38731E409E670044D6FE /* RenderedPeer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC38711E409E670044D6FE /* RenderedPeer.swift */; }; + D0BEAF631E54B2FA00BD963D /* AccountManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF621E54B2FA00BD963D /* AccountManager.swift */; }; + D0BEAF641E54B2FA00BD963D /* AccountManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF621E54B2FA00BD963D /* AccountManager.swift */; }; + D0BEAF671E54B33900BD963D /* AccountRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF661E54B33900BD963D /* AccountRecord.swift */; }; + D0BEAF681E54B33900BD963D /* AccountRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF661E54B33900BD963D /* AccountRecord.swift */; }; + D0BEAF6A1E54B5FB00BD963D /* AccountManagerMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF691E54B5FB00BD963D /* AccountManagerMetadataTable.swift */; }; + D0BEAF6B1E54B5FB00BD963D /* AccountManagerMetadataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF691E54B5FB00BD963D /* AccountManagerMetadataTable.swift */; }; + D0BEAF6D1E54B77900BD963D /* AccountManagerRecordTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF6C1E54B77900BD963D /* AccountManagerRecordTable.swift */; }; + D0BEAF6E1E54B77900BD963D /* AccountManagerRecordTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF6C1E54B77900BD963D /* AccountManagerRecordTable.swift */; }; + D0BEAF701E54BC1E00BD963D /* AccountRecordsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF6F1E54BC1E00BD963D /* AccountRecordsView.swift */; }; + D0BEAF711E54BC1E00BD963D /* AccountRecordsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF6F1E54BC1E00BD963D /* AccountRecordsView.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; @@ -334,6 +344,11 @@ D0B844501DAC04FE005F29E1 /* PeerPresence.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerPresence.swift; sourceTree = ""; }; D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MultiplePeersView.swift; sourceTree = ""; }; D0BC38711E409E670044D6FE /* RenderedPeer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RenderedPeer.swift; sourceTree = ""; }; + D0BEAF621E54B2FA00BD963D /* AccountManager.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountManager.swift; sourceTree = ""; }; + D0BEAF661E54B33900BD963D /* AccountRecord.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountRecord.swift; sourceTree = ""; }; + D0BEAF691E54B5FB00BD963D /* AccountManagerMetadataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountManagerMetadataTable.swift; sourceTree = ""; }; + D0BEAF6C1E54B77900BD963D /* AccountManagerRecordTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountManagerRecordTable.swift; sourceTree = ""; }; + D0BEAF6F1E54BC1E00BD963D /* AccountRecordsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountRecordsView.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; @@ -471,6 +486,18 @@ name = Frameworks; sourceTree = ""; }; + D0BEAF651E54B32B00BD963D /* Account Manager */ = { + isa = PBXGroup; + children = ( + D0BEAF621E54B2FA00BD963D /* AccountManager.swift */, + D0BEAF661E54B33900BD963D /* AccountRecord.swift */, + D0BEAF691E54B5FB00BD963D /* AccountManagerMetadataTable.swift */, + D0BEAF6C1E54B77900BD963D /* AccountManagerRecordTable.swift */, + D0BEAF6F1E54BC1E00BD963D /* AccountRecordsView.swift */, + ); + name = "Account Manager"; + sourceTree = ""; + }; D0DA443F1E4C7834005FDCA7 /* Upgrade */ = { isa = PBXGroup; children = ( @@ -651,6 +678,7 @@ D044CA291C617D39002160FF /* SeedConfiguration.swift */, D0D510F31D63BA8400A97B8A /* PostboxTransaction.swift */, D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */, + D0BEAF651E54B32B00BD963D /* Account Manager */, D0E3A74D1B28A7E300A402D9 /* Supporting Files */, ); path = Postbox; @@ -872,6 +900,7 @@ D0F7B1C71E045C6A007EB8A5 /* MessageHistoryTable.swift in Sources */, D0B418571D7DFE29004562A4 /* Postbox.swift in Sources */, D0B418251D7DFE0C004562A4 /* RedBlackTree.swift in Sources */, + D0BEAF681E54B33900BD963D /* AccountRecord.swift in Sources */, D0F7B1C61E045C6A007EB8A5 /* MessageHistoryIndexTable.swift in Sources */, D0F7B1CD1E045C6A007EB8A5 /* MessageMediaTable.swift in Sources */, D0B844071DAB91B5005F29E1 /* MediaResource.swift in Sources */, @@ -904,6 +933,7 @@ D0F82D101E43A024007E499C /* OrderedItemListIndexTable.swift in Sources */, D0F7B1CF1E045C6A007EB8A5 /* MetadataTable.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, + D0BEAF641E54B2FA00BD963D /* AccountManager.swift in Sources */, D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */, D0F7B1D91E045C6A007EB8A5 /* OrderStatisticTable.swift in Sources */, D073CEA01DCBF3C1007511FD /* ItemCollectionsView.swift in Sources */, @@ -920,6 +950,7 @@ D0F7B1E21E045C6A007EB8A5 /* PeerNameIndexTable.swift in Sources */, D073CE7C1DCBF3B4007511FD /* CachedPeerData.swift in Sources */, D0F7B1DA1E045C6A007EB8A5 /* RatingTable.swift in Sources */, + D0BEAF711E54BC1E00BD963D /* AccountRecordsView.swift in Sources */, D073CE9F1DCBF3C1007511FD /* InitialMessageHistoryData.swift in Sources */, D0BC38731E409E670044D6FE /* RenderedPeer.swift in Sources */, D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopIndexableMessageIds.swift in Sources */, @@ -940,7 +971,9 @@ D0F7B1CB1E045C6A007EB8A5 /* ChatListTable.swift in Sources */, D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */, D0F7B1CA1E045C6A007EB8A5 /* MessageHistoryUnsentTable.swift in Sources */, + D0BEAF6B1E54B5FB00BD963D /* AccountManagerMetadataTable.swift in Sources */, D0F7B1D01E045C6A007EB8A5 /* KeychainTable.swift in Sources */, + D0BEAF6E1E54B77900BD963D /* AccountManagerRecordTable.swift in Sources */, D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */, D0F7B1D81E045C6A007EB8A5 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, D0B8440A1DAB91B5005F29E1 /* RandomAccessMediaResourceContext.swift in Sources */, @@ -983,6 +1016,7 @@ D03120FE1DA562E9006A2A60 /* CachedPeerDataTable.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, + D0BEAF671E54B33900BD963D /* AccountRecord.swift in Sources */, D021E0D61DB4FCFC00C6B04F /* ItemCollectionInfoTable.swift in Sources */, D0079F6B1D5B3AAB00A27A2C /* PeerNameIndexRepresentation.swift in Sources */, D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, @@ -1015,6 +1049,7 @@ D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D0F82D0F1E43A024007E499C /* OrderedItemListIndexTable.swift in Sources */, D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */, + D0BEAF631E54B2FA00BD963D /* AccountManager.swift in Sources */, D0AD23271E194D1C00A7089A /* PeerOperationLogTable.swift in Sources */, D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */, D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */, @@ -1031,6 +1066,7 @@ D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, D0BC386C1E3FCEE50044D6FE /* MultiplePeersView.swift in Sources */, + D0BEAF701E54BC1E00BD963D /* AccountRecordsView.swift in Sources */, D07CFF851DCA99C400761F81 /* InitialMessageHistoryData.swift in Sources */, D087C20C1E43C11C00D686F8 /* OrderedItemListView.swift in Sources */, D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */, @@ -1051,7 +1087,9 @@ D0BC38721E409E670044D6FE /* RenderedPeer.swift in Sources */, D08775031E3E3E7400A97350 /* PreferencesEntry.swift in Sources */, D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */, + D0BEAF6A1E54B5FB00BD963D /* AccountManagerMetadataTable.swift in Sources */, D08775001E3E3D9F00A97350 /* PreferencesTable.swift in Sources */, + D0BEAF6D1E54B77900BD963D /* AccountManagerRecordTable.swift in Sources */, D03120F81DA53FF4006A2A60 /* PeerPresenceTable.swift in Sources */, D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */, D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */, diff --git a/Postbox/AccountManager.swift b/Postbox/AccountManager.swift new file mode 100644 index 0000000000..e24379cb5d --- /dev/null +++ b/Postbox/AccountManager.swift @@ -0,0 +1,156 @@ +import Foundation +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif + +public struct AccountManagerModifier { + public let getRecords: () -> [AccountRecord] + public let updateRecord: (AccountRecordId, (AccountRecord?) -> (AccountRecord?)) -> Void + public let getCurrentId: () -> AccountRecordId? + public let setCurrentId: (AccountRecordId) -> Void + public let createRecord: ([AccountRecordAttribute]) -> AccountRecordId +} + +public final class AccountManager { + private let queue: Queue + private let basePath: String + private let valueBox: ValueBox + + private var tables: [Table] = [] + + private let metadataTable: AccountManagerMetadataTable + private let recordTable: AccountManagerRecordTable + + private var currentRecordOperations: [AccountManagerRecordOperation] = [] + private var currentMetadataOperations: [AccountManagerMetadataOperation] = [] + + private var recordsViews = Bag<(MutableAccountRecordsView, ValuePipe)>() + + fileprivate init(queue: Queue, basePath: String) { + self.queue = queue + self.basePath = basePath + let _ = try? FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: false, attributes: nil) + self.valueBox = SqliteValueBox(basePath: basePath + "/db", queue: queue) + + self.metadataTable = AccountManagerMetadataTable(valueBox: self.valueBox, table: AccountManagerMetadataTable.tableSpec(0)) + self.recordTable = AccountManagerRecordTable(valueBox: self.valueBox, table: AccountManagerRecordTable.tableSpec(1)) + + self.tables.append(self.metadataTable) + self.tables.append(self.recordTable) + } + + public func modify(_ f: @escaping (AccountManagerModifier) -> T) -> Signal { + return Signal { subscriber in + self.queue.async { + self.valueBox.begin() + + let modifier = AccountManagerModifier(getRecords: { + return self.recordTable.getRecords() + }, updateRecord: { id, update in + let current = self.recordTable.getRecord(id: id) + let updated = update(current) + if updated != current { + self.recordTable.setRecord(id: id, record: updated, operations: &self.currentRecordOperations) + } + }, getCurrentId: { + return self.metadataTable.getCurrentAccountId() + }, setCurrentId: { id in + self.metadataTable.setCurrentAccountId(id, operations: &self.currentMetadataOperations) + }, createRecord: { attributes in + let id = generateAccountRecordId() + let record = AccountRecord(id: id, attributes: attributes) + self.recordTable.setRecord(id: id, record: record, operations: &self.currentRecordOperations) + return id + }) + + let result = f(modifier) + + self.beforeCommit() + + self.valueBox.commit() + + subscriber.putNext(result) + subscriber.putCompletion() + } + return EmptyDisposable + } + } + + private func beforeCommit() { + if !self.currentRecordOperations.isEmpty || !self.currentMetadataOperations.isEmpty { + for (view, pipe) in self.recordsViews.copyItems() { + if view.replay(operations: self.currentRecordOperations, metadataOperations: self.currentMetadataOperations) { + pipe.putNext(AccountRecordsView(view)) + } + } + } + + self.currentRecordOperations.removeAll() + self.currentMetadataOperations.removeAll() + + for table in self.tables { + table.beforeCommit() + } + } + + public func accountRecords() -> Signal { + return self.modify { modifier -> Signal in + return self.accountRecordsInternal(modifier: modifier) + } |> switchToLatest + } + + private func accountRecordsInternal(modifier: AccountManagerModifier) -> Signal { + let mutableView = MutableAccountRecordsView(getRecords: { + return self.recordTable.getRecords() + }, currentId: self.metadataTable.getCurrentAccountId()) + let pipe = ValuePipe() + let index = self.recordsViews.add((mutableView, pipe)) + + return (.single(AccountRecordsView(mutableView)) + |> then(pipe.signal())) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.recordsViews.remove(index) + } + } + } + } + + public func allocatedCurrentAccountId() -> Signal { + return self.modify { modifier -> Signal in + let current = modifier.getCurrentId() + let id: AccountRecordId + if let current = current { + id = current + } else { + id = generateAccountRecordId() + modifier.setCurrentId(id) + modifier.updateRecord(id, { _ in + return AccountRecord(id: id, attributes: []) + }) + } + + let signal = self.accountRecordsInternal(modifier: modifier) |> map { view -> AccountRecordId? in + return view.currentRecord?.id + } + + return signal + } |> switchToLatest |> distinctUntilChanged(isEqual: { lhs, rhs in + return lhs == rhs + }) + } +} + +public func accountManager(basePath: String) -> Signal { + return Signal { subscriber in + let queue = Queue() + queue.async { + subscriber.putNext(AccountManager(queue: queue, basePath: basePath)) + subscriber.putCompletion() + } + return EmptyDisposable + } +} diff --git a/Postbox/AccountManagerMetadataTable.swift b/Postbox/AccountManagerMetadataTable.swift new file mode 100644 index 0000000000..3e31283f42 --- /dev/null +++ b/Postbox/AccountManagerMetadataTable.swift @@ -0,0 +1,37 @@ +import Foundation + +enum AccountManagerMetadataOperation { + case updateCurrentAccountId(AccountRecordId) +} + +private enum MetadataKey: Int64 { + case currentAccountId = 0 +} + +final class AccountManagerMetadataTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + + private func key(_ key: MetadataKey) -> ValueBoxKey { + let result = ValueBoxKey(length: 8) + result.setInt64(0, value: key.rawValue) + return result + } + + func getCurrentAccountId() -> AccountRecordId? { + if let value = self.valueBox.get(self.table, key: self.key(.currentAccountId)) { + var id: Int64 = 0 + value.read(&id, offset: 0, length: 8) + return AccountRecordId(rawValue: id) + } else { + return nil + } + } + + func setCurrentAccountId(_ id: AccountRecordId, operations: inout [AccountManagerMetadataOperation]) { + var rawValue = id.rawValue + self.valueBox.set(self.table, key: self.key(.currentAccountId), value: MemoryBuffer(memory: &rawValue, capacity: 8, length: 8, freeWhenDone: false)) + operations.append(.updateCurrentAccountId(id)) + } +} diff --git a/Postbox/AccountManagerRecordTable.swift b/Postbox/AccountManagerRecordTable.swift new file mode 100644 index 0000000000..1d48924997 --- /dev/null +++ b/Postbox/AccountManagerRecordTable.swift @@ -0,0 +1,46 @@ +import Foundation + +enum AccountManagerRecordOperation { + case set(id: AccountRecordId, record: AccountRecord?) +} + +final class AccountManagerRecordTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + + private func key(_ key: AccountRecordId) -> ValueBoxKey { + let result = ValueBoxKey(length: 8) + result.setInt64(0, value: key.rawValue) + return result + } + + func getRecords() -> [AccountRecord] { + var records: [AccountRecord] = [] + self.valueBox.scan(self.table, values: { _, value in + let record = AccountRecord(decoder: Decoder(buffer: value)) + records.append(record) + return true + }) + return records + } + + func getRecord(id: AccountRecordId) -> AccountRecord? { + if let value = self.valueBox.get(self.table, key: self.key(id)) { + return AccountRecord(decoder: Decoder(buffer: value)) + } else { + return nil + } + } + + func setRecord(id: AccountRecordId, record: AccountRecord?, operations: inout [AccountManagerRecordOperation]) { + if let record = record { + let encoder = Encoder() + record.encode(encoder) + self.valueBox.set(self.table, key: self.key(id), value: encoder.readBufferNoCopy()) + } else { + self.valueBox.remove(self.table, key: self.key(id)) + } + operations.append(.set(id: id, record: record)) + } +} diff --git a/Postbox/AccountRecord.swift b/Postbox/AccountRecord.swift new file mode 100644 index 0000000000..56c03c4228 --- /dev/null +++ b/Postbox/AccountRecord.swift @@ -0,0 +1,71 @@ +import Foundation + +public protocol AccountRecordAttribute: Coding { + func isEqual(to: AccountRecordAttribute) -> Bool +} + +public struct AccountRecordId: Comparable, Hashable { + let rawValue: Int64 + + init(rawValue: Int64) { + self.rawValue = rawValue + } + + public var int64: Int64 { + return self.rawValue + } + + public var hashValue: Int { + return self.rawValue.hashValue + } + + public static func ==(lhs: AccountRecordId, rhs: AccountRecordId) -> Bool { + return lhs.rawValue == rhs.rawValue + } + + public static func <(lhs: AccountRecordId, rhs: AccountRecordId) -> Bool { + return lhs.rawValue < rhs.rawValue + } +} + +public func generateAccountRecordId() -> AccountRecordId { + var id: Int64 = 0 + arc4random_buf(&id, 8) + return AccountRecordId(rawValue: id) +} + +public struct AccountRecord: Coding, Equatable { + public let id: AccountRecordId + public let attributes: [AccountRecordAttribute] + + public init(id: AccountRecordId, attributes: [AccountRecordAttribute]) { + self.id = id + self.attributes = attributes + } + + public init(decoder: Decoder) { + self.id = AccountRecordId(rawValue: decoder.decodeInt64ForKey("id")) + self.attributes = (decoder.decodeObjectArrayForKey("attributes") as [Coding]).map { $0 as! AccountRecordAttribute } + } + + public func encode(_ encoder: Encoder) { + encoder.encodeInt64(self.id.int64, forKey: "id") + let attributes: [Coding] = self.attributes.map { $0 } + encoder.encodeGenericObjectArray(attributes, forKey: "attributes") + } + + public static func ==(lhs: AccountRecord, rhs: AccountRecord) -> Bool { + if lhs.id != rhs.id { + return false + } + if lhs.attributes.count != rhs.attributes.count { + return false + } + for i in 0 ..< lhs.attributes.count { + if !lhs.attributes[i].isEqual(to: rhs.attributes[i]) { + return false + } + } + return true + } +} diff --git a/Postbox/AccountRecordsView.swift b/Postbox/AccountRecordsView.swift new file mode 100644 index 0000000000..44dcb79b05 --- /dev/null +++ b/Postbox/AccountRecordsView.swift @@ -0,0 +1,78 @@ +import Foundation + +final class MutableAccountRecordsView { + fileprivate var records: [AccountRecord] + fileprivate var currentId: AccountRecordId? + + init(getRecords: () -> [AccountRecord], currentId: AccountRecordId?) { + self.records = getRecords() + self.currentId = currentId + } + + func replay(operations: [AccountManagerRecordOperation], metadataOperations: [AccountManagerMetadataOperation]) -> Bool { + var updated = false + + for operation in operations { + switch operation { + case let .set(id, record): + if let record = record { + updated = true + var found = false + for i in 0 ..< self.records.count { + if self.records[i].id == id { + self.records[i] = record + found = true + break + } + } + + if !found { + self.records.append(record) + self.records.sort(by: { lhs, rhs in + return lhs.id < rhs.id + }) + } + } else { + for i in 0 ..< self.records.count { + if self.records[i].id == id { + self.records.remove(at: i) + updated = true + break + } + } + } + } + } + + for operation in metadataOperations { + switch operation { + case let .updateCurrentAccountId(id): + updated = true + self.currentId = id + } + } + + return updated + } +} + +public final class AccountRecordsView { + public let records: [AccountRecord] + public let currentRecord: AccountRecord? + + init(_ view: MutableAccountRecordsView) { + self.records = view.records + if let currentId = view.currentId { + var currentRecord: AccountRecord? + for record in view.records { + if record.id == currentId { + currentRecord = record + break + } + } + self.currentRecord = currentRecord + } else { + self.currentRecord = nil + } + } +} diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index fd92f1dd08..bd3eac7815 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -365,6 +365,26 @@ public final class Encoder { } } + public func encodeGenericObjectArray(_ value: [Coding], forKey key: StaticString) { + self.encodeKey(key) + var type: Int8 = ValueType.ObjectArray.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var length: Int32 = Int32(value.count) + self.buffer.write(&length, offset: 0, length: 4) + let innerEncoder = Encoder() + for object in value { + var typeHash: Int32 = murMurHashString32("\(type(of: object))") + self.buffer.write(&typeHash, offset: 0, length: 4) + + innerEncoder.reset() + object.encode(innerEncoder) + + var length: Int32 = Int32(innerEncoder.buffer.offset) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(length)) + } + } + public func encodeStringArray(_ value: [String], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.StringArray.rawValue @@ -394,26 +414,6 @@ public final class Encoder { } } - public func encodeGenericObjectArray(_ value: [Coding], forKey key: StaticString) { - self.encodeKey(key) - var type: Int8 = ValueType.ObjectArray.rawValue - self.buffer.write(&type, offset: 0, length: 1) - var length: Int32 = Int32(value.count) - self.buffer.write(&length, offset: 0, length: 4) - let innerEncoder = Encoder() - for object in value { - var typeHash: Int32 = murMurHashString32("\(type(of: object))") - self.buffer.write(&typeHash, offset: 0, length: 4) - - innerEncoder.reset() - object.encode(innerEncoder) - - var length: Int32 = Int32(innerEncoder.buffer.offset) - self.buffer.write(&length, offset: 0, length: 4) - self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(length)) - } - } - public func encodeObjectDictionary(_ value: [K : V], forKey key: StaticString) where K: Coding, K: Hashable { self.encodeKey(key) var type: Int8 = ValueType.ObjectDictionary.rawValue diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index bec654d7b8..a1de367194 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -263,13 +263,21 @@ final class MessageHistoryTable: Table { return internalStoreMessages } - func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [Int64: MessageId] { let messagesByPeerId = self.messagesGroupedByPeerId(messages) + var globallyUniqueIdToMessageId: [Int64: MessageId] = [:] for (peerId, peerMessages) in messagesByPeerId { var operations: [MessageHistoryIndexOperation] = [] - self.messageHistoryIndexTable.addMessages(self.internalStoreMessages(peerMessages), location: location, operations: &operations) + let internalPeerMessages = self.internalStoreMessages(peerMessages) + for message in internalPeerMessages { + if let globallyUniqueId = message.globallyUniqueId { + globallyUniqueIdToMessageId[globallyUniqueId] = message.id + } + } + self.messageHistoryIndexTable.addMessages(internalPeerMessages, location: location, operations: &operations) self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } + return globallyUniqueIdToMessageId } func addHoles(_ messageIds: [MessageId], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index b5ade8d940..d23d7ed984 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -46,7 +46,7 @@ final class MutablePeerView { func replay(updatedPeers: [PeerId: Peer], updatedNotificationSettings: [PeerId: PeerNotificationSettings], updatedCachedPeerData: [PeerId: CachedPeerData], updatedPeerPresences: [PeerId: PeerPresence], replaceContactPeerIds: Set?, getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) -> Bool { var updated = false - if let cachedData = updatedCachedPeerData[self.peerId], self.cachedData == nil || self.cachedData!.peerIds != cachedData.peerIds { + if let cachedData = updatedCachedPeerData[self.peerId], self.cachedData == nil || !self.cachedData!.isEqual(to: cachedData) { self.cachedData = cachedData updated = true diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index fa5a3fda9d..b6cde5d3de 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -22,8 +22,12 @@ public final class Modifier { self.postbox = postbox } - public func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) { - self.postbox?.addMessages(messages, location: location) + public func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) -> [Int64: MessageId] { + if let postbox = self.postbox { + return postbox.addMessages(messages, location: location) + } else { + return [:] + } } public func addHole(_ messageId: MessageId) { @@ -311,6 +315,14 @@ public final class Modifier { public func setPinnedPeerIds(_ peerIds: [PeerId]) { self.postbox?.setPinnedPeerIds(peerIds) } + + public func getTotalUnreadCount() -> Int32 { + if let postbox = self.postbox { + return postbox.messageHistoryMetadataTable.getChatListTotalUnreadCount() + } else { + return 0 + } + } } fileprivate class PipeNotifier: NSObject { @@ -338,7 +350,7 @@ public final class Postbox { private let globalMessageIdsNamespace: MessageId.Namespace private let ipcNotificationsDisposable = MetaDisposable() - private var pipeNotifier: PipeNotifier! + //private var pipeNotifier: PipeNotifier! private let queue = Queue(name: "org.telegram.postbox.Postbox") private var valueBox: ValueBox! @@ -440,7 +452,7 @@ public final class Postbox { self.mediaBox = MediaBox(basePath: self.basePath + "/media") - self.pipeNotifier = PipeNotifier(basePath: basePath, notify: { [weak self] in + /*self.pipeNotifier = PipeNotifier(basePath: basePath, notify: { [weak self] in //if let strongSelf = self { /*strongSelf.queue.async { if strongSelf.valueBox != nil { @@ -449,11 +461,15 @@ public final class Postbox { } }*/ //} - }) + })*/ self.openDatabase() } + deinit { + assert(true) + } + private func debugSaveState(name: String) { let path = self.basePath + name let _ = try? FileManager.default.removeItem(atPath: path) @@ -678,8 +694,8 @@ public final class Postbox { metaDisposable.set(disposable) } - fileprivate func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) { - self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + fileprivate func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) -> [Int64: MessageId] { + return self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } fileprivate func addHole(_ id: MessageId) { @@ -1593,7 +1609,7 @@ public final class Postbox { } } } - } |> switchToLatest + } |> switchToLatest } public func messageHistoryHolesView() -> Signal { diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 3d219bdb6a..87f69bc768 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -97,6 +97,7 @@ final class SqliteValueBox: ValueBox { private var rangeValueAscStatementsNoLimit: [Int32 : SqlitePreparedStatement] = [:] private var rangeValueDescStatementsLimit: [Int32 : SqlitePreparedStatement] = [:] private var rangeValueDescStatementsNoLimit: [Int32 : SqlitePreparedStatement] = [:] + private var scanStatements: [Int32 : SqlitePreparedStatement] = [:] private var existsStatements: [Int32 : SqlitePreparedStatement] = [:] private var updateStatements: [Int32 : SqlitePreparedStatement] = [:] private var insertStatements: [Int32 : SqlitePreparedStatement] = [:] @@ -520,6 +521,26 @@ final class SqliteValueBox: ValueBox { return resultStatement } + private func scanStatement(_ table: ValueBoxTable) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) + + let resultStatement: SqlitePreparedStatement + + if let statement = self.scanStatements[table.id] { + resultStatement = statement + } else { + var statement: OpaquePointer? = nil + sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) ORDER BY key ASC", -1, &statement, nil) + let preparedStatement = SqlitePreparedStatement(statement: statement) + self.scanStatements[table.id] = preparedStatement + resultStatement = preparedStatement + } + + resultStatement.reset() + + return resultStatement + } + private func existsStatement(_ table: ValueBoxTable, key: ValueBoxKey) -> SqlitePreparedStatement { assert(self.queue.isCurrent()) checkTableKey(table, key) @@ -868,6 +889,37 @@ final class SqliteValueBox: ValueBox { } } + public func scan(_ table: ValueBoxTable, values: (ValueBoxKey, ReadBuffer) -> Bool) { + assert(self.queue.isCurrent()) + + if let _ = self.tables[table.id] { + let statement: SqlitePreparedStatement = self.scanStatement(table) + + var startTime = CFAbsoluteTimeGetCurrent() + + var currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + startTime = currentTime + + while statement.step() { + startTime = CFAbsoluteTimeGetCurrent() + + let key = statement.keyAt(0) + let value = statement.valueAt(1) + + currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + if !values(key, value) { + break + } + } + + statement.reset() + } + } + public func set(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) { assert(self.queue.isCurrent()) self.checkTable(table) @@ -964,6 +1016,11 @@ final class SqliteValueBox: ValueBox { } self.rangeValueDescStatementsNoLimit.removeAll() + for (_, statement) in self.scanStatements { + statement.destroy() + } + self.scanStatements.removeAll() + for (_, statement) in self.existsStatements { statement.destroy() } diff --git a/Postbox/ValueBox.swift b/Postbox/ValueBox.swift index a9f8561c92..9fe68e47b7 100644 --- a/Postbox/ValueBox.swift +++ b/Postbox/ValueBox.swift @@ -19,6 +19,7 @@ protocol ValueBox { func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) func range(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, keys: (ValueBoxKey) -> Bool, limit: Int) + func scan(_ table: ValueBoxTable, values: (ValueBoxKey, ReadBuffer) -> Bool) func get(_ table: ValueBoxTable, key: ValueBoxKey) -> ReadBuffer? func exists(_ table: ValueBoxTable, key: ValueBoxKey) -> Bool func set(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) From 7540c7494c4aac07a39398d61deffd28f15b16b2 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 20 Feb 2017 19:13:17 +0300 Subject: [PATCH 085/427] no message --- Postbox/Postbox.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index b6cde5d3de..19fc031289 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -1862,4 +1862,12 @@ public final class Postbox { } }).start() } + + public func clearCaches() { + let _ = self.modify({ _ in + for table in self.tables { + table.clearMemoryCache() + } + }).start() + } } From fb5b83ca7c2f33615026582ed485deb36f8762b2 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 24 Feb 2017 17:41:39 +0300 Subject: [PATCH 086/427] no message --- Postbox/Postbox.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 19fc031289..74a7f472b6 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -164,6 +164,10 @@ public final class Modifier { self.postbox?.updatePeerCachedData(peerIds: peerIds, update: update) } + public func getPeerCachedData(peerId: PeerId) -> CachedPeerData? { + return self.postbox?.cachedPeerDataTable.get(peerId) + } + public func updatePeerPresences(_ peerPresences: [PeerId: PeerPresence]) { self.postbox?.updatePeerPresences(peerPresences) } From bbfe6045e9d5d4adb3938ed8bc22352c4c1ec16a Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 27 Feb 2017 21:26:03 +0400 Subject: [PATCH 087/427] no message --- Postbox/ViewTracker.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 43804688c3..5f68ef49bb 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -208,7 +208,7 @@ final class ViewTracker { } func removeUnreadMessageCountsView(_ index: Bag<(MutableUnreadMessageCountsView, ValuePipe)>.Index) { - self.peerViews.remove(index) + self.unreadMessageCountsViews.remove(index) } func addPeerMergedOperationLogView(_ view: MutablePeerMergedOperationLogView) -> (Bag<(MutablePeerMergedOperationLogView, ValuePipe)>.Index, Signal) { From f5b7e6ddb15ae3b4503168bb215a44f9b3ed428a Mon Sep 17 00:00:00 2001 From: overtake Date: Mon, 27 Feb 2017 20:13:34 +0200 Subject: [PATCH 088/427] -peerView dispose bugfix --- Postbox/ViewTracker.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 43804688c3..5f68ef49bb 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -208,7 +208,7 @@ final class ViewTracker { } func removeUnreadMessageCountsView(_ index: Bag<(MutableUnreadMessageCountsView, ValuePipe)>.Index) { - self.peerViews.remove(index) + self.unreadMessageCountsViews.remove(index) } func addPeerMergedOperationLogView(_ view: MutablePeerMergedOperationLogView) -> (Bag<(MutablePeerMergedOperationLogView, ValuePipe)>.Index, Signal) { From 0fd73c6c8de6d500d347204bdbb93edb286c5fdf Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 28 Feb 2017 22:51:45 +0400 Subject: [PATCH 089/427] no message --- Postbox/ItemCollectionItemTable.swift | 14 ++++++++++++++ Postbox/Postbox.swift | 8 ++++++++ Postbox/ViewTracker.swift | 24 ++++++++++++++++-------- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Postbox/ItemCollectionItemTable.swift b/Postbox/ItemCollectionItemTable.swift index a8cc9c602c..13ddc31925 100644 --- a/Postbox/ItemCollectionItemTable.swift +++ b/Postbox/ItemCollectionItemTable.swift @@ -156,6 +156,20 @@ final class ItemCollectionItemTable: Table { return items } + func collectionItems(collectionId: ItemCollectionId) -> [ItemCollectionItem] { + var items: [ItemCollectionItem] = [] + self.valueBox.range(self.table, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId), values: { key, value in + let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) + if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { + items.append(item) + } else { + assertionFailure() + } + return true + }, limit: 0) + return items + } + func replaceItems(collectionId: ItemCollectionId, items: [ItemCollectionItem]) { let updatedIndices = Set(items.map({ $0.index })) var itemByIndex: [ItemCollectionItemIndex: ItemCollectionItem] = [:] diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 74a7f472b6..ae62686f1e 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -196,6 +196,14 @@ public final class Modifier { self.postbox?.replaceItemCollections(namespace: namespace, itemCollections: itemCollections) } + public func getItemCollectionItems(collectionId: ItemCollectionId) -> [ItemCollectionItem] { + if let postbox = self.postbox { + return postbox.itemCollectionItemTable.collectionItems(collectionId: collectionId) + } else { + return [] + } + } + public func searchItemCollection(namespace: ItemCollectionId.Namespace, key: MemoryBuffer) -> [ItemCollectionItem] { if let postbox = self.postbox { return postbox.itemCollectionItemTable.exactIndexedItems(namespace: namespace, key: ValueBoxKey(key)) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 5f68ef49bb..a4a2b2b0d6 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -583,8 +583,10 @@ final class ViewTracker { }) disposable.set(ActionDisposable { - pipeDisposable.dispose() - self.messageHistoryHolesViewSubscribers.remove(index) + self.queue.async { + pipeDisposable.dispose() + self.messageHistoryHolesViewSubscribers.remove(index) + } }) } return disposable @@ -605,8 +607,10 @@ final class ViewTracker { }) disposable.set(ActionDisposable { - pipeDisposable.dispose() - self.chatListHolesViewSubscribers.remove(index) + self.queue.async { + pipeDisposable.dispose() + self.chatListHolesViewSubscribers.remove(index) + } }) } return disposable @@ -627,8 +631,10 @@ final class ViewTracker { }) disposable.set(ActionDisposable { - pipeDisposable.dispose() - self.unsendMessageIdsViewSubscribers.remove(index) + self.queue.async { + pipeDisposable.dispose() + self.unsendMessageIdsViewSubscribers.remove(index) + } }) } return disposable @@ -649,8 +655,10 @@ final class ViewTracker { }) disposable.set(ActionDisposable { - pipeDisposable.dispose() - self.synchronizePeerReadStatesViewSubscribers.remove(index) + self.queue.async { + pipeDisposable.dispose() + self.synchronizePeerReadStatesViewSubscribers.remove(index) + } }) } return disposable From a71641c2f74157508a8aa536c539c39a9b09d467 Mon Sep 17 00:00:00 2001 From: overtake Date: Tue, 28 Feb 2017 21:52:15 +0300 Subject: [PATCH 090/427] no message --- Postbox/Postbox.swift | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 74a7f472b6..3576e20077 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -196,6 +196,34 @@ public final class Modifier { self.postbox?.replaceItemCollections(namespace: namespace, itemCollections: itemCollections) } + public func getItemCollections(namespace: ItemCollectionId.Namespace) -> [(ItemCollectionId, ItemCollectionInfo, [ItemCollectionItem])] { + if let postbox = postbox { + let items = postbox.itemCollectionItemTable.getItems(namespace: namespace) + let infos = postbox.itemCollectionInfoTable.getInfos(namespace: namespace) + + var result:[(ItemCollectionId, ItemCollectionInfo, [ItemCollectionItem])] = [] + for info in infos { + if let items = items[info.1] { + result.append((info.1, info.2, items)) + } + } + return result + + } else { + return [] + } + } + + public func getItemCollection(namespace: ItemCollectionId.Namespace, id: ItemCollectionId) -> (ItemCollectionInfo, [ItemCollectionItem])? { + let collections = getItemCollections(namespace: namespace) + for collection in collections { + if collection.0 == id { + return (collection.1, collection.2) + } + } + return nil + } + public func searchItemCollection(namespace: ItemCollectionId.Namespace, key: MemoryBuffer) -> [ItemCollectionItem] { if let postbox = self.postbox { return postbox.itemCollectionItemTable.exactIndexedItems(namespace: namespace, key: ValueBoxKey(key)) From 76f893d9a03f577d752164d7e5891ed57864d269 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 3 Mar 2017 13:47:22 +0400 Subject: [PATCH 091/427] no message --- Postbox/MediaBox.swift | 103 +++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index abcf5a552f..fb5145ac2e 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -12,8 +12,9 @@ private final class ResourceStatusContext { private final class ResourceDataContext { var data: MediaResourceData - let progresiveDataSubscribers = Bag<(MediaResourceData) -> Void>() - let completeDataSubscribers = Bag<(MediaResourceData) -> Void>() + var processedFetch: Bool = false + let progresiveDataSubscribers = Bag<(waitUntilFetchStatus: Bool, sink: (MediaResourceData) -> Void)>() + let completeDataSubscribers = Bag<(waitUntilFetchStatus: Bool, sink: (MediaResourceData) -> Void)>() var fetchDisposable: Disposable? let fetchSubscribers = Bag() @@ -85,6 +86,11 @@ private final class CachedMediaResourceRepresentationContext { var disposable: Disposable? } +public enum ResourceDataRequestOption { + case complete(waitUntilFetchStatus: Bool) + case incremental(waitUntilFetchStatus: Bool) +} + public final class MediaBox { let basePath: String @@ -246,7 +252,7 @@ public final class MediaBox { public func completedResourcePath(_ resource: MediaResource, pathExtension: String? = nil) -> String? { let paths = self.storePathsForId(resource.id) - if let completeSize = fileSize(paths.complete) { + if let _ = fileSize(paths.complete) { if let pathExtension = pathExtension { let symlinkPath = paths.complete + ".\(pathExtension)" if fileSize(symlinkPath) == nil { @@ -261,7 +267,7 @@ public final class MediaBox { } } - public func resourceData(_ resource: MediaResource, pathExtension: String? = nil, complete: Bool = true) -> Signal { + public func resourceData(_ resource: MediaResource, pathExtension: String? = nil, option: ResourceDataRequestOption = .complete(waitUntilFetchStatus: false)) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.concurrentQueue.async { @@ -281,7 +287,7 @@ public final class MediaBox { } else { self.dataQueue.async { let resourceId = WrappedMediaResourceId(resource.id) - var currentContext: ResourceDataContext? = self.dataContexts[resourceId] + let currentContext: ResourceDataContext? = self.dataContexts[resourceId] if let currentContext = currentContext, currentContext.data.complete { if let pathExtension = pathExtension { let symlinkPath = paths.complete + ".\(pathExtension)" @@ -317,47 +323,53 @@ public final class MediaBox { } let index: Bag<(MediaResourceData) -> Void>.Index - if complete { - index = dataContext.completeDataSubscribers.add { data in - if let pathExtension = pathExtension, data.complete { - let symlinkPath = paths.complete + ".\(pathExtension)" - if fileSize(symlinkPath) == nil { - let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) - } - subscriber.putNext(MediaResourceData(path: symlinkPath, size: data.size, complete: data.complete)) - subscriber.putCompletion() - } else { - subscriber.putNext(data) - subscriber.putCompletion() - } - } - subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0, complete: false)) - } else { - index = dataContext.progresiveDataSubscribers.add { data in - if let pathExtension = pathExtension, data.complete { - let symlinkPath = paths.complete + ".\(pathExtension)" - if fileSize(symlinkPath) == nil { - let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) - } - subscriber.putNext(MediaResourceData(path: symlinkPath, size: data.size, complete: data.complete)) - subscriber.putCompletion() - } else { - subscriber.putNext(data) - if data.complete { + switch option { + case let .complete(waitUntilFetchStatus): + index = dataContext.completeDataSubscribers.add((waitUntilFetchStatus, { data in + if let pathExtension = pathExtension, data.complete { + let symlinkPath = paths.complete + ".\(pathExtension)" + if fileSize(symlinkPath) == nil { + let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) + } + subscriber.putNext(MediaResourceData(path: symlinkPath, size: data.size, complete: data.complete)) + subscriber.putCompletion() + } else { + subscriber.putNext(data) subscriber.putCompletion() } + })) + if !waitUntilFetchStatus || dataContext.processedFetch { + subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0, complete: false)) + } + case let .incremental(waitUntilFetchStatus): + index = dataContext.progresiveDataSubscribers.add((waitUntilFetchStatus, { data in + if let pathExtension = pathExtension, data.complete { + let symlinkPath = paths.complete + ".\(pathExtension)" + if fileSize(symlinkPath) == nil { + let _ = try? FileManager.default.createSymbolicLink(atPath: symlinkPath, withDestinationPath: URL(fileURLWithPath: paths.complete).lastPathComponent) + } + subscriber.putNext(MediaResourceData(path: symlinkPath, size: data.size, complete: data.complete)) + subscriber.putCompletion() + } else { + subscriber.putNext(data) + if data.complete { + subscriber.putCompletion() + } + } + })) + if !waitUntilFetchStatus || dataContext.processedFetch { + subscriber.putNext(dataContext.data) } - } - subscriber.putNext(dataContext.data) } disposable.set(ActionDisposable { self.dataQueue.async { if let dataContext = self.dataContexts[resourceId] { - if complete { - dataContext.completeDataSubscribers.remove(index) - } else { - dataContext.progresiveDataSubscribers.remove(index) + switch option { + case .complete: + dataContext.completeDataSubscribers.remove(index) + case .incremental: + dataContext.progresiveDataSubscribers.remove(index) } if dataContext.progresiveDataSubscribers.isEmpty && dataContext.completeDataSubscribers.isEmpty && dataContext.fetchSubscribers.isEmpty { @@ -583,14 +595,23 @@ public final class MediaBox { dataContext.data = updatedData - for subscriber in dataContext.progresiveDataSubscribers.copyItems() { + let hadProcessedFetch = dataContext.processedFetch + dataContext.processedFetch = true + + for (_, subscriber) in dataContext.progresiveDataSubscribers.copyItems() { subscriber(updatedData) } if updatedData.complete { - for subscriber in dataContext.completeDataSubscribers.copyItems() { + for (_, subscriber) in dataContext.completeDataSubscribers.copyItems() { subscriber(updatedData) } + } else if !hadProcessedFetch { + for (waitUntilFetchStatus, subscriber) in dataContext.completeDataSubscribers.copyItems() { + if waitUntilFetchStatus { + subscriber(updatedData) + } + } } let status: MediaResourceStatus @@ -721,7 +742,7 @@ public final class MediaBox { }) if context.disposable == nil { - let signal = self.resourceData(resource, complete: true) + let signal = self.resourceData(resource, option: .complete(waitUntilFetchStatus: false)) |> mapToSignal { resourceData in return self.wrappedFetchCachedResourceRepresentation.get() |> take(1) From 047e41cc81499e3a013331aeca170a2d0a2a16c4 Mon Sep 17 00:00:00 2001 From: Peter Date: Sat, 4 Mar 2017 23:13:39 +0400 Subject: [PATCH 092/427] no message --- Postbox.xcodeproj/project.pbxproj | 20 ++-- Postbox/IncrementalUpgrade_v11_v12.swift | 15 --- Postbox/MetadataTable.swift | 48 ++++++++++ Postbox/PostboxAccess.swift | 44 ++++++++- Postbox/SqliteInterface.swift | 116 +++++++++++++++++++++++ 5 files changed, 212 insertions(+), 31 deletions(-) delete mode 100644 Postbox/IncrementalUpgrade_v11_v12.swift create mode 100644 Postbox/SqliteInterface.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index ddfa9d9b5e..fe24793299 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -36,6 +36,8 @@ D03120FC1DA55427006A2A60 /* PeerNotificationSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */; }; D03120FE1DA562E9006A2A60 /* CachedPeerDataTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */; }; D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */; }; + D03229EE1E6B33FD0000AF9C /* SqliteInterface.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03229ED1E6B33FD0000AF9C /* SqliteInterface.swift */; }; + D03229EF1E6B33FD0000AF9C /* SqliteInterface.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03229ED1E6B33FD0000AF9C /* SqliteInterface.swift */; }; D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; D03BCCF81C73561C0097A291 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; @@ -182,8 +184,6 @@ D0D949F51D35353900740E02 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; D0DA44411E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */; }; D0DA44421E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */; }; - D0DA44451E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44441E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift */; }; - D0DA44461E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44441E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift */; }; D0DA44481E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */; }; D0DA44491E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */; }; D0DF0C8F1D81A350008AEB01 /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DF0C8E1D81A350008AEB01 /* PeerView.swift */; }; @@ -286,6 +286,7 @@ D03120FB1DA55427006A2A60 /* PeerNotificationSettings.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNotificationSettings.swift; sourceTree = ""; }; D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CachedPeerDataTable.swift; sourceTree = ""; }; D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNotificationSettingsTable.swift; sourceTree = ""; }; + D03229ED1E6B33FD0000AF9C /* SqliteInterface.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteInterface.swift; sourceTree = ""; }; D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryUnsentTable.swift; sourceTree = ""; }; D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageHistoryView.swift; sourceTree = ""; }; D03BCCF71C73561C0097A291 /* Table.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Table.swift; sourceTree = ""; }; @@ -365,7 +366,6 @@ D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessResourceTests.swift; sourceTree = ""; }; D0D949F41D35353900740E02 /* MappedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MappedFile.swift; sourceTree = ""; }; D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IncrementalUpgrade.swift; sourceTree = ""; }; - D0DA44441E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IncrementalUpgrade_v11_v12.swift; sourceTree = ""; }; D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxAccess.swift; sourceTree = ""; }; D0DF0C8E1D81A350008AEB01 /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; @@ -501,20 +501,11 @@ D0DA443F1E4C7834005FDCA7 /* Upgrade */ = { isa = PBXGroup; children = ( - D0DA44431E4C796C005FDCA7 /* v11-v12 */, D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */, ); name = Upgrade; sourceTree = ""; }; - D0DA44431E4C796C005FDCA7 /* v11-v12 */ = { - isa = PBXGroup; - children = ( - D0DA44441E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift */, - ); - name = "v11-v12"; - sourceTree = ""; - }; D0E1DE161C5EB06000C7826E /* Tables */ = { isa = PBXGroup; children = ( @@ -597,6 +588,7 @@ D0D949F41D35353900740E02 /* MappedFile.swift */, D0D510FF1D64A58900A97B8A /* IpcPipe.swift */, D07827C01E0079CB00071108 /* StringIndexTokens.swift */, + D03229ED1E6B33FD0000AF9C /* SqliteInterface.swift */, ); name = Utils; sourceTree = ""; @@ -883,6 +875,7 @@ buildActionMask = 2147483647; files = ( D050F2661E4A5B5A00988324 /* MessageGloballyUniqueIdTable.swift in Sources */, + D03229EF1E6B33FD0000AF9C /* SqliteInterface.swift in Sources */, D050F2671E4A5B5A00988324 /* TimestampBasedMessageAttributesTable.swift in Sources */, D050F2681E4A5B5A00988324 /* TimestampBasedMessageAttributesIndexTable.swift in Sources */, D0DA44421E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */, @@ -956,7 +949,6 @@ D0F7B1DE1E045C6A007EB8A5 /* PeerChatTopIndexableMessageIds.swift in Sources */, D0B4184A1D7DFE20004562A4 /* UnsentMessageHistoryView.swift in Sources */, D0B418231D7DFE0C004562A4 /* SimpleDictionary.swift in Sources */, - D0DA44461E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift in Sources */, D0B4185C1D7DFE2F004562A4 /* MurMurHash32.m in Sources */, D0F7B1E01E045C6A007EB8A5 /* ItemCacheTable.swift in Sources */, D0F7B1E11E045C6A007EB8A5 /* ReverseIndexReferenceTable.swift in Sources */, @@ -999,6 +991,7 @@ buildActionMask = 2147483647; files = ( D08775061E3E3F2100A97350 /* PreferencesView.swift in Sources */, + D03229EE1E6B33FD0000AF9C /* SqliteInterface.swift in Sources */, D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */, D0DA44411E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */, @@ -1073,7 +1066,6 @@ D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */, D00C7CD41E365C4E0080C3D5 /* PeerChatListInclusion.swift in Sources */, D0A18D671E16874D004C6734 /* UnreadMessageCountsView.swift in Sources */, - D0DA44451E4C798E005FDCA7 /* IncrementalUpgrade_v11_v12.swift in Sources */, D0D511021D64D73D00A97B8A /* IpcNotifier.mm in Sources */, D0B844511DAC04FE005F29E1 /* PeerPresence.swift in Sources */, D07827C11E0079CB00071108 /* StringIndexTokens.swift in Sources */, diff --git a/Postbox/IncrementalUpgrade_v11_v12.swift b/Postbox/IncrementalUpgrade_v11_v12.swift deleted file mode 100644 index f822c0461b..0000000000 --- a/Postbox/IncrementalUpgrade_v11_v12.swift +++ /dev/null @@ -1,15 +0,0 @@ -import Foundation - -#if os(macOS) - import SwiftSignalKitMac -#else - import SwiftSignalKit -#endif - -final class IncrementalUpgrade_v11_v12: IncrementalUpgrade { - func upgrade(from previous: ValueBox, tmpBasePath: String) -> Signal { - assertionFailure() - - return .never() - } -} diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index f42089b735..4d2070e844 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -5,6 +5,40 @@ private enum MetadataKey: Int32 { case State = 2 case TransactionStateVersion = 3 case MasterClientId = 4 + case AccessChallenge = 5 +} + +enum PostboxAccessChallengeData: Coding { + case none + case numericalPassword(String) + case plaintextPassword(String) + + init(decoder: Decoder) { + switch decoder.decodeInt32ForKey("r") as Int32 { + case 0: + self = .none + case 1: + self = .numericalPassword(decoder.decodeStringForKey("t")) + case 2: + self = .plaintextPassword(decoder.decodeStringForKey("t")) + default: + assertionFailure() + self = .none + } + } + + func encode(_ encoder: Encoder) { + switch self { + case .none: + encoder.encodeInt32(0, forKey: "r") + case let .numericalPassword(text): + encoder.encodeInt32(1, forKey: "r") + encoder.encodeString(text, forKey: "t") + case let .plaintextPassword(text): + encoder.encodeInt32(2, forKey: "r") + encoder.encodeString(text, forKey: "t") + } + } } final class MetadataTable: Table { @@ -102,6 +136,20 @@ final class MetadataTable: Table { self.valueBox.set(self.table, key: self.key(.MasterClientId), value: buffer) } + func accessChallengeData() -> PostboxAccessChallengeData { + if let value = self.valueBox.get(self.table, key: self.key(.AccessChallenge)) { + return PostboxAccessChallengeData(decoder: Decoder(buffer: value)) + } else { + return .none + } + } + + func setAccessChallengeData(_ data: PostboxAccessChallengeData) { + let encoder = Encoder() + data.encode(encoder) + self.valueBox.set(self.table, key: self.key(.AccessChallenge), value: encoder.readBufferNoCopy()) + } + override func clearMemoryCache() { self.cachedState = nil } diff --git a/Postbox/PostboxAccess.swift b/Postbox/PostboxAccess.swift index 36690f3485..fa266d7102 100644 --- a/Postbox/PostboxAccess.swift +++ b/Postbox/PostboxAccess.swift @@ -7,16 +7,56 @@ import Foundation public enum PostboxAuthorizationChallenge { case numericPassword(length: Int32) - case arbitraryPassword() + case arbitraryPassword } public enum PostboxAccess { - case unlocked(Postbox) + case unlocked case locked(PostboxAuthorizationChallenge) } +private final class PostboxAccessHelper { + let queue: Queue + let valueBox: ValueBox + let metadataTable: MetadataTable + + init(queue: Queue, basePath: String) { + self.queue = queue + self.valueBox = SqliteValueBox(basePath: basePath + "/db", queue: self.queue) + self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) + } +} + public func accessPostbox(basePath: String, password: String?) -> Signal { return Signal { subscriber in + let queue = Queue() + + queue.async { + let postbox = PostboxAccessHelper(queue: queue, basePath: basePath) + let challengeData = postbox.metadataTable.accessChallengeData() + switch challengeData { + case .none: + subscriber.putNext(.unlocked) + subscriber.putCompletion() + case let .numericalPassword(text): + if text == password { + subscriber.putNext(.unlocked) + subscriber.putCompletion() + } else { + subscriber.putNext(.locked(.numericPassword(length: Int32(text.characters.count)))) + subscriber.putCompletion() + } + case let .plaintextPassword(text): + if text == password { + subscriber.putNext(.unlocked) + subscriber.putCompletion() + } else { + subscriber.putNext(.locked(.arbitraryPassword)) + subscriber.putCompletion() + } + } + } + return ActionDisposable { } } diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift new file mode 100644 index 0000000000..661c8017f5 --- /dev/null +++ b/Postbox/SqliteInterface.swift @@ -0,0 +1,116 @@ +import Foundation +import sqlcipher + +private final class SqliteInterfaceStatement { + let statement: OpaquePointer? + + init(statement: OpaquePointer?) { + self.statement = statement + } + + func bind(_ index: Int, data: UnsafeRawPointer, length: Int) { + sqlite3_bind_blob(statement, Int32(index), data, Int32(length), nil) + } + + func bind(_ index: Int, number: Int64) { + sqlite3_bind_int64(statement, Int32(index), number) + } + + func bindNull(_ index: Int) { + sqlite3_bind_null(statement, Int32(index)) + } + + func bind(_ index: Int, number: Int32) { + sqlite3_bind_int(statement, Int32(index), number) + } + + func reset() { + sqlite3_reset(statement) + sqlite3_clear_bindings(statement) + } + + func step() -> Bool { + let result = sqlite3_step(statement) + if result != SQLITE_ROW && result != SQLITE_DONE { + assertionFailure("Sqlite error \(result)") + } + return result == SQLITE_ROW + } + + func int32At(_ index: Int) -> Int32 { + return sqlite3_column_int(statement, Int32(index)) + } + + func int64At(_ index: Int) -> Int64 { + return sqlite3_column_int64(statement, Int32(index)) + } + + func valueAt(_ index: Int) -> ReadBuffer { + let valueLength = sqlite3_column_bytes(statement, Int32(index)) + let valueData = sqlite3_column_blob(statement, Int32(index)) + + let valueMemory = malloc(Int(valueLength))! + memcpy(valueMemory, valueData, Int(valueLength)) + return ReadBuffer(memory: valueMemory, length: Int(valueLength), freeWhenDone: true) + } + + func keyAt(_ index: Int) -> ValueBoxKey { + let valueLength = sqlite3_column_bytes(statement, Int32(index)) + let valueData = sqlite3_column_blob(statement, Int32(index)) + + let key = ValueBoxKey(length: Int(valueLength)) + memcpy(key.memory, valueData, Int(valueLength)) + return key + } + + func destroy() { + sqlite3_finalize(statement) + } +} + +public final class SqliteStatementCursor { + private let statement: SqliteInterfaceStatement + + fileprivate init(statement: SqliteInterfaceStatement) { + self.statement = statement + } + + public func getInt32(at index: Int) -> Int32 { + return self.statement.int32At(index) + } + + public func getInt64(at index: Int) -> Int64 { + return self.statement.int64At(index) + } + + public func getString(at index: Int) -> String { + let value = self.statement.valueAt(index) + if let string = String(data: value.makeData(), encoding: .utf8) { + return string + } else { + return "" + } + } +} + +public final class SqliteInterface { + private let database: Database + + init(databasePath: String) { + self.database = Database(databasePath) + } + + func select(_ query: String, _ f: (SqliteStatementCursor) -> Bool) { + var statement: OpaquePointer? = nil + sqlite3_prepare_v2(database.handle, query, -1, &statement, nil) + let preparedStatement = SqliteInterfaceStatement(statement: statement) + let cursor = SqliteStatementCursor(statement: preparedStatement) + while preparedStatement.step() { + if !f(cursor) { + break + } + } + preparedStatement.reset() + preparedStatement.destroy() + } +} From c3e29b7dcdf4ebe3191dd3cd86c5c1e7909818ed Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 6 Mar 2017 01:36:17 +0300 Subject: [PATCH 093/427] no message --- Postbox/ContactTable.swift | 76 +++++++++++++++++--------------------- Postbox/Postbox.swift | 6 +-- 2 files changed, 36 insertions(+), 46 deletions(-) diff --git a/Postbox/ContactTable.swift b/Postbox/ContactTable.swift index f9b8d05fab..38c65d5747 100644 --- a/Postbox/ContactTable.swift +++ b/Postbox/ContactTable.swift @@ -5,11 +5,16 @@ final class ContactTable: Table { return ValueBoxTable(id: id, keyType: .binary) } - private var originalPeerIds: Set? + private let peerNameIndexTable: PeerNameIndexTable + + private var peerIdsBeforeModification: Set? private var peerIds: Set? - private var addedPeerIds = Set() - private var removedPeerIds = Set() + init(valueBox: ValueBox, table: ValueBoxTable, peerNameIndexTable: PeerNameIndexTable) { + self.peerNameIndexTable = peerNameIndexTable + + super.init(valueBox: valueBox, table: table) + } private func key(_ id: PeerId, sharedKey: ValueBoxKey = ValueBoxKey(length: 8)) -> ValueBoxKey { sharedKey.setInt64(0, value: id.toInt64()) @@ -25,18 +30,7 @@ final class ContactTable: Table { } func isContact(peerId: PeerId) -> Bool { - if let peerIds = self.peerIds { - return peerIds.contains(peerId) - } else { - var peerIds = Set() - self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), keys: { key in - peerIds.insert(PeerId(key.getInt64(0))) - return true - }, limit: 0) - self.peerIds = peerIds - self.originalPeerIds = peerIds - return peerIds.contains(peerId) - } + return self.get().contains(peerId) } func get() -> Set { @@ -49,49 +43,45 @@ final class ContactTable: Table { return true }, limit: 0) self.peerIds = peerIds - self.originalPeerIds = peerIds return peerIds } } func replace(_ ids: Set) { - let currentPeerIds = self.get() - - let previousPeerIds: Set - if let originalPeerIds = self.originalPeerIds { - previousPeerIds = originalPeerIds - } else { - previousPeerIds = currentPeerIds - self.originalPeerIds = currentPeerIds + if self.peerIdsBeforeModification == nil { + self.peerIdsBeforeModification = self.get() } - self.removedPeerIds = previousPeerIds.subtracting(ids) - self.addedPeerIds = ids.subtracting(previousPeerIds) - self.peerIds = ids } override func clearMemoryCache() { - self.originalPeerIds = nil + assert(self.peerIdsBeforeModification == nil) self.peerIds = nil - self.addedPeerIds.removeAll() - self.removedPeerIds.removeAll() } override func beforeCommit() { - let sharedKey = self.key(PeerId(namespace: 0, id: 0)) - - for peerId in self.removedPeerIds { - self.valueBox.remove(self.table, key: self.key(peerId, sharedKey: sharedKey)) + if let peerIdsBeforeModification = self.peerIdsBeforeModification { + if let peerIds = self.peerIds { + let removedPeerIds = peerIdsBeforeModification.subtracting(peerIds) + let addedPeerIds = peerIds.subtracting(peerIdsBeforeModification) + + let sharedKey = self.key(PeerId(namespace: 0, id: 0)) + + for peerId in removedPeerIds { + self.valueBox.remove(self.table, key: self.key(peerId, sharedKey: sharedKey)) + self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.contacts], includes: false) + } + + for peerId in addedPeerIds { + self.valueBox.set(self.table, key: self.key(peerId, sharedKey: sharedKey), value: MemoryBuffer()) + self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.contacts], includes: true) + } + } else { + assertionFailure() + } + + self.peerIdsBeforeModification = nil } - - for peerId in self.addedPeerIds { - self.valueBox.set(self.table, key: self.key(peerId, sharedKey: sharedKey), value: MemoryBuffer()) - } - - self.originalPeerIds = self.peerIds - - self.addedPeerIds.removeAll() - self.removedPeerIds.removeAll() } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 18b4eeed62..ba3841ab83 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -574,7 +574,9 @@ public final class Postbox { self.timestampBasedMessageAttributesTable = TimestampBasedMessageAttributesTable(valueBox: self.valueBox!, table: TimestampBasedMessageAttributesTable.tableSpec(34), indexTable: self.timestampBasedMessageAttributesIndexTable) self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable!, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable!) self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, table: PeerChatStateTable.tableSpec(13)) - self.contactsTable = ContactTable(valueBox: self.valueBox, table: ContactTable.tableSpec(16)) + self.peerNameTokenIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(26)) + self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) + self.contactsTable = ContactTable(valueBox: self.valueBox, table: ContactTable.tableSpec(16), peerNameIndexTable: self.peerNameIndexTable) self.peerRatingTable = RatingTable(valueBox: self.valueBox, table: RatingTable.tableSpec(17)) self.cachedPeerDataTable = CachedPeerDataTable(valueBox: self.valueBox, table: CachedPeerDataTable.tableSpec(18)) self.peerNotificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox, table: PeerNotificationSettingsTable.tableSpec(19)) @@ -585,8 +587,6 @@ public final class Postbox { self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox, table: PeerChatInterfaceStateTable.tableSpec(23)) self.itemCacheMetaTable = ItemCacheMetaTable(valueBox: self.valueBox, table: ItemCacheMetaTable.tableSpec(24)) self.itemCacheTable = ItemCacheTable(valueBox: self.valueBox, table: ItemCacheTable.tableSpec(25)) - self.peerNameTokenIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(26)) - self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8), peerNameIndexTable: self.peerNameIndexTable, metadataTable: self.messageHistoryMetadataTable, readStateTable: self.readStateTable, notificationSettingsTable: self.peerNotificationSettingsTable) self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.peerChatTopTaggedMessageIdsTable = PeerChatTopTaggedMessageIdsTable(valueBox: self.valueBox, table: PeerChatTopTaggedMessageIdsTable.tableSpec(28)) From 4d49e3be128755b7b16851aa6bdbaab6811d3d9e Mon Sep 17 00:00:00 2001 From: overtake Date: Mon, 6 Mar 2017 21:55:26 +0300 Subject: [PATCH 094/427] deployment target --- Postbox.xcodeproj/project.pbxproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index fe24793299..0a43dc1b49 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -1240,7 +1240,7 @@ INFOPLIST_FILE = PostboxMac/Info.plist; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 10.10; + MACOSX_DEPLOYMENT_TARGET = 10.11; OTHER_CFLAGS = ( "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", @@ -1276,7 +1276,7 @@ INFOPLIST_FILE = PostboxMac/Info.plist; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 10.10; + MACOSX_DEPLOYMENT_TARGET = 10.11; OTHER_CFLAGS = ( "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", @@ -1311,7 +1311,7 @@ INFOPLIST_FILE = PostboxMac/Info.plist; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 10.10; + MACOSX_DEPLOYMENT_TARGET = 10.11; OTHER_CFLAGS = ( "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", From 36adf94aef16de73b09274011c6f33450ca07cf3 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 7 Mar 2017 01:42:21 +0300 Subject: [PATCH 095/427] no message --- Postbox/MetadataTable.swift | 6 +++--- Postbox/Postbox.swift | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index 4d2070e844..dec5657a5b 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -8,12 +8,12 @@ private enum MetadataKey: Int32 { case AccessChallenge = 5 } -enum PostboxAccessChallengeData: Coding { +public enum PostboxAccessChallengeData: Coding { case none case numericalPassword(String) case plaintextPassword(String) - init(decoder: Decoder) { + public init(decoder: Decoder) { switch decoder.decodeInt32ForKey("r") as Int32 { case 0: self = .none @@ -27,7 +27,7 @@ enum PostboxAccessChallengeData: Coding { } } - func encode(_ encoder: Encoder) { + public func encode(_ encoder: Encoder) { switch self { case .none: encoder.encodeInt32(0, forKey: "r") diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index ba3841ab83..629f60fa57 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -362,6 +362,18 @@ public final class Modifier { return 0 } } + + public func getAccessChallengeData() -> PostboxAccessChallengeData { + if let postbox = self.postbox { + return postbox.metadataTable.accessChallengeData() + } else { + return .none + } + } + + public func setAccessChallengeData(_ data: PostboxAccessChallengeData) { + self.postbox?.metadataTable.setAccessChallengeData(data) + } } fileprivate class PipeNotifier: NSObject { From c60255eaa3df643e30a0526618c2dba50757fcd8 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 7 Mar 2017 16:10:56 +0300 Subject: [PATCH 096/427] no message --- Postbox/Postbox.swift | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 629f60fa57..0a5dad0964 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -53,6 +53,13 @@ public final class Modifier { public func clearHistory(_ peerId: PeerId) { self.postbox?.clearHistory(peerId) } + public func messageIdsForGlobalIds(_ ids: [Int32]) -> [MessageId] { + if let postbox = self.postbox { + return postbox.messageIdsForGlobalIds(ids) + } else { + return [] + } + } public func deleteMessagesWithGlobalIds(_ ids: [Int32]) { if let postbox = self.postbox { From 7caa5add948d96f628ab7c7fd44937b5711bcf6a Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 9 Mar 2017 21:14:01 +0300 Subject: [PATCH 097/427] no message --- Postbox/OrderStatisticTable.swift | 472 ++++++++++++++++++++++++++---- Postbox/SqliteInterface.swift | 10 + 2 files changed, 431 insertions(+), 51 deletions(-) diff --git a/Postbox/OrderStatisticTable.swift b/Postbox/OrderStatisticTable.swift index 8957cc9469..7ba2fe6f2d 100644 --- a/Postbox/OrderStatisticTable.swift +++ b/Postbox/OrderStatisticTable.swift @@ -1,59 +1,429 @@ import Foundation +private struct MessageOrderKey: Comparable { + let timestamp: Int32 + let namespace: MessageId.Namespace + let id: MessageId.Id + + static func ==(lhs: MessageOrderKey, rhs: MessageOrderKey) -> Bool { + return lhs.timestamp == rhs.timestamp && lhs.namespace == rhs.namespace && lhs.id == rhs.id + } + + static func <(lhs: MessageOrderKey, rhs: MessageOrderKey) -> Bool { + if lhs.timestamp != rhs.timestamp { + return lhs.timestamp < rhs.timestamp + } + + if lhs.namespace != rhs.namespace { + return lhs.namespace < rhs.namespace + } + + return lhs.id < rhs.id + } +} + +private enum BTreeNodePosition { + case left + case right +} + +private final class BTreeNode { + let reference: Int32 + let keys: [MessageOrderKey] + let values: [Int32] + let childrenReferences: [Int32]? + + var isLeaf: Bool { + return self.childrenReferences == nil + } + + var numberOfKeys: Int { + return self.keys.count + } + + init(reference: Int32) { + self.reference = reference + self.keys = [] + self.values = [] + self.childrenReferences = nil + } + + init(reference: Int32, keys: [MessageOrderKey], values: [Int32], childrenReferences: [Int32]? = nil) { + self.reference = reference + self.keys = keys + self.values = values + self.childrenReferences = childrenReferences + } + + func replacingValueAt(_ index: Int, with value: Int32) -> BTreeNode { + var values = self.values + values[index] = value + return BTreeNode(reference: self.reference, keys: self.keys, values: values, childrenReferences: self.childrenReferences) + } + + func insertingKeyValueAt(_ index: Int, key: MessageOrderKey, value: Int32) -> BTreeNode { + var keys = self.keys + var values = self.values + keys.insert(key, at: index) + values.insert(value, at: index) + return BTreeNode(reference: self.reference, keys: keys, values: values, childrenReferences: self.childrenReferences) + } + + func removingKeyValueAt(_ index: Int) -> BTreeNode { + var keys = self.keys + var values = self.values + keys.remove(at: index) + values.remove(at: index) + return BTreeNode(reference: self.reference, keys: keys, values: values, childrenReferences: self.childrenReferences) + } + + func removingKeyValueInRange(_ range: CountableRange) -> BTreeNode { + var keys = self.keys + var values = self.values + keys.removeSubrange(range) + values.removeSubrange(range) + return BTreeNode(reference: self.reference, keys: keys, values: values, childrenReferences: self.childrenReferences) + } + + func insertingChildAt(_ index: Int, reference: Int32) -> BTreeNode { + var childrenReferences = self.childrenReferences! + childrenReferences.insert(reference, at: index) + return BTreeNode(reference: self.reference, keys: self.keys, values: self.values, childrenReferences: childrenReferences) + } + + func replacingChildrenReferences(_ childrenReferences: [Int32]?) -> BTreeNode { + return BTreeNode(reference: self.reference, keys: self.keys, values: self.values, childrenReferences: childrenReferences) + } + + func removingChildrenReferencesInRange(_ range: CountableRange) -> BTreeNode { + var childrenReferences = self.childrenReferences + childrenReferences?.removeSubrange(range) + return BTreeNode(reference: self.reference, keys: self.keys, values: self.values, childrenReferences: childrenReferences) + } +} + + +private final class BTreeAccess { + let order: Int = 2 + + func getNode(_ reference: Int32) -> BTreeNode? { + return nil + } + + func replaceNode(_ reference: Int32, with: BTreeNode) { + } + + func allocateNodeReference() -> Int32 { + return 0 + } + + func value(node: BTreeNode, key: MessageOrderKey) -> Int32? { + var index = node.keys.startIndex + + while (index + 1) < node.keys.endIndex && node.keys[index] < key { + index = index + 1 + } + + if key == node.keys[index] { + return node.values[index] + } else if key < node.keys[index] { + if let childrenReferences = node.childrenReferences { + if let child = self.getNode(childrenReferences[index]) { + return self.value(node: child, key: key) + } else { + assertionFailure() + return nil + } + } else { + return nil + } + //return children?[index].value(for: key) + } else { + if let child = self.getNode(index + 1) { + return self.value(node: child, key: key) + } else { + assertionFailure() + return nil + } + //return children?[(index + 1)].value(for: key) + } + } + + func insert(node: BTreeNode, value: Int32, key: MessageOrderKey) { + var index = node.keys.startIndex + + while index < node.keys.endIndex && node.keys[index] < key { + index = index + 1 + } + + if index < node.keys.endIndex && node.keys[index] == key { + self.replaceNode(node.reference, with: node.replacingValueAt(index, with: value)) + //values[index] = value + return + } + + if node.isLeaf { + self.replaceNode(node.reference, with: node.insertingKeyValueAt(index, key: key, value: value)) + //keys.insert(key, at: index) + //values.insert(value, at: index) + } else { + if let child = self.getNode(node.childrenReferences![index]) { + self.insert(node: child, value: value, key: key) + if child.numberOfKeys > self.order * 2 { + self.split(node: node, child: child, atIndex: index) + } + } else { + assertionFailure() + } + //children![index].insert(value, for: key) + //if children![index].numberOfKeys > self.order * 2 { + // split(child: children![index], atIndex: index) + //} + } + } + + private func split(node: BTreeNode, child: BTreeNode, atIndex index: Int) { + let middleIndex = child.numberOfKeys / 2 + + var updatedNode = node.insertingKeyValueAt(index, key: child.keys[middleIndex], value: child.values[middleIndex]) + //keys.insert(child.keys[middleIndex], at: index) + //values.insert(child.values[middleIndex], at: index) + + var updatedChild = child.removingKeyValueAt(middleIndex) + //child.keys.remove(at: middleIndex) + //child.values.remove(at: middleIndex) + + var rightSibling = BTreeNode(reference: self.allocateNodeReference(), keys: Array(child.keys[updatedChild.keys.indices.suffix(from: middleIndex)]), values: Array(child.values[updatedChild.values.indices.suffix(from: middleIndex)])) + + updatedChild = updatedChild.removingKeyValueInRange(updatedChild.keys.indices.suffix(from: middleIndex)) + //child.keys.removeSubrange(child.keys.indices.suffix(from: middleIndex)) + //child.values.removeSubrange(child.values.indices.suffix(from: middleIndex)) + + updatedNode = updatedNode.insertingChildAt(index + 1, reference: rightSibling.reference) + //children!.insert(rightSibling, at: (index + 1)) + + self.replaceNode(node.reference, with: updatedNode) + + if updatedChild.childrenReferences != nil { + rightSibling = rightSibling.replacingChildrenReferences(Array(updatedChild.childrenReferences![updatedChild.childrenReferences!.indices.suffix(from: (middleIndex + 1))])) + updatedChild = updatedChild.removingChildrenReferencesInRange(updatedChild.childrenReferences!.indices.suffix(from: (middleIndex + 1))) + } + + self.replaceNode(child.reference, with: updatedChild) + self.replaceNode(rightSibling.reference, with: rightSibling) + } + + /*private var inorderPredecessor: BTreeNode { + if isLeaf { + return self + } else { + return children!.last!.inorderPredecessor + } + } + + func remove(_ key: MessageOrderKey) { + var index = keys.startIndex + + while (index + 1) < keys.endIndex && keys[index] < key { + index = (index + 1) + } + + if keys[index] == key { + if isLeaf { + keys.remove(at: index) + values.remove(at: index) + } else { + let predecessor = children![index].inorderPredecessor + keys[index] = predecessor.keys.last! + values[index] = predecessor.values.last! + children![index].remove(keys[index]) + if children![index].numberOfKeys < self.order { + fix(childWithTooFewKeys: children![index], atIndex: index) + } + } + } else if key < keys[index] { + if let leftChild = children?[index] { + leftChild.remove(key) + if leftChild.numberOfKeys < self.order { + fix(childWithTooFewKeys: leftChild, atIndex: index) + } + } else { + // key is not present + } + } else { + if let rightChild = children?[(index + 1)] { + rightChild.remove(key) + if rightChild.numberOfKeys < self.order { + fix(childWithTooFewKeys: rightChild, atIndex: (index + 1)) + } + } else { + // key is not present + } + } + } + + private func fix(childWithTooFewKeys child: BTreeNode, atIndex index: Int) { + if (index - 1) >= 0 && children![(index - 1)].numberOfKeys > self.order { + move(keyAtIndex: (index - 1), to: child, from: children![(index - 1)], at: .left) + } else if (index + 1) < children!.count && children![(index + 1)].numberOfKeys > self.order { + move(keyAtIndex: index, to: child, from: children![(index + 1)], at: .right) + } else if (index - 1) >= 0 { + merge(child: child, atIndex: index, to: .left) + } else { + merge(child: child, atIndex: index, to: .right) + } + } + + private func move(keyAtIndex index: Int, to targetNode: BTreeNode, + from node: BTreeNode, at position: BTreeNodePosition) { + switch position { + case .left: + targetNode.keys.insert(keys[index], at: targetNode.keys.startIndex) + targetNode.values.insert(values[index], at: targetNode.values.startIndex) + keys[index] = node.keys.last! + values[index] = node.values.last! + node.keys.removeLast() + node.values.removeLast() + if !targetNode.isLeaf { + targetNode.children!.insert(node.children!.last!, + at: targetNode.children!.startIndex) + node.children!.removeLast() + } + + case .right: + targetNode.keys.insert(keys[index], at: targetNode.keys.endIndex) + targetNode.values.insert(values[index], at: targetNode.values.endIndex) + keys[index] = node.keys.first! + values[index] = node.values.first! + node.keys.removeFirst() + node.values.removeFirst() + if !targetNode.isLeaf { + targetNode.children!.insert(node.children!.first!, + at: targetNode.children!.endIndex) + node.children!.removeFirst() + } + } + } + + private func merge(child: BTreeNode, atIndex index: Int, to position: BTreeNodePosition) { + switch position { + case .left: + // We can merge to the left sibling + + children![(index - 1)].keys = children![(index - 1)].keys + + [keys[(index - 1)]] + child.keys + + children![(index - 1)].values = children![(index - 1)].values + + [values[(index - 1)]] + child.values + + keys.remove(at: (index - 1)) + values.remove(at: (index - 1)) + + if !child.isLeaf { + children![(index - 1)].children = + children![(index - 1)].children! + child.children! + } + + case .right: + // We should merge to the right sibling + + children![(index + 1)].keys = child.keys + [keys[index]] + + children![(index + 1)].keys + + children![(index + 1)].values = child.values + [values[index]] + + children![(index + 1)].values + + keys.remove(at: index) + values.remove(at: index) + + if !child.isLeaf { + children![(index + 1)].children = + child.children! + children![(index + 1)].children! + } + } + children!.remove(at: index) + }*/ +} + +/*private class BTree { + public let order: Int + + var rootNode: BTreeNode! + + public init?(order: Int) { + guard order > 0 else { + assertionFailure() + return nil + } + self.order = order + rootNode = BTreeNode(order: self.order) + } + + public func value(for key: MessageOrderKey) -> Int32? { + guard rootNode.numberOfKeys > 0 else { + return nil + } + + return rootNode.value(for: key) + } + + public func insert(_ value: Int32, for key: MessageOrderKey) { + rootNode.insert(value, for: key) + + if rootNode.numberOfKeys > order * 2 { + splitRoot() + } + } + + private func splitRoot() { + let middleIndexOfOldRoot = rootNode.numberOfKeys / 2 + + let newRoot = BTreeNode( + order: self.order, + keys: [rootNode.keys[middleIndexOfOldRoot]], + values: [rootNode.values[middleIndexOfOldRoot]], + children: [rootNode] + ) + rootNode.keys.remove(at: middleIndexOfOldRoot) + rootNode.values.remove(at: middleIndexOfOldRoot) + + let newRightChild = BTreeNode( + order: self.order, + keys: Array(rootNode.keys[rootNode.keys.indices.suffix(from: middleIndexOfOldRoot)]), + values: Array(rootNode.values[rootNode.values.indices.suffix(from: middleIndexOfOldRoot)]) + ) + rootNode.keys.removeSubrange(rootNode.keys.indices.suffix(from: middleIndexOfOldRoot)) + rootNode.values.removeSubrange(rootNode.values.indices.suffix(from: middleIndexOfOldRoot)) + + if rootNode.children != nil { + newRightChild.children = Array( + rootNode.children![rootNode.children!.indices.suffix(from: (middleIndexOfOldRoot + 1))] + ) + rootNode.children!.removeSubrange( + rootNode.children!.indices.suffix(from: (middleIndexOfOldRoot + 1)) + ) + } + + newRoot.children!.append(newRightChild) + rootNode = newRoot + } + + public func remove(_ key: MessageOrderKey) { + guard rootNode.numberOfKeys > 0 else { + return + } + + rootNode.remove(key) + + if rootNode.numberOfKeys == 0 && !rootNode.isLeaf { + rootNode = rootNode.children!.first! + } + } +}*/ + class MessageOrderStatisticTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .binary) } - /*private func update(peerId: PeerId, tagMask: MessageTags, id: Int32, count: Int) { - let key = ValueBoxKey(length: 8 + 4 + 4) - key.setInt64(0, value: peerId.toInt64()) - key.setUInt32(8, value: tagMask.rawValue) - - var writeValue: Int32 = 0 - let buffer = MemoryBuffer(memory: &writeValue, capacity: 4, length: 4, freeWhenDone: false) - var idx = id - while (idx <= 1000000) { - key.setInt32(8 + 4, value: idx) - var value: Int32 = 0 - if let data = self.valueBox.get(self.table, key: key) { - data.read(&value, offset: 0, length: 4) - } - if value == 0 { - self.valueBox.remove(self.table, key: key) - } else { - writeValue = value - self.valueBox.set(self.table, key: key, value: buffer) - } - idx += idx & -idx - } - } - - private func get(peerId: PeerId, tagMask: MessageTags, id: Int32) -> Int32 { - let key = ValueBoxKey(length: 8 + 4) - key.setInt64(0, value: peerId.toInt64()) - - var idx = id - - var sum: Int32 = 0 - while (idx > 0) { - key.setInt32(8, value: idx) - var value: Int32 = 0 - if let data = self.valueBox.get(self.table, key: key) { - data.read(&value, offset: 0, length: 4) - } - sum += value - idx -= idx & -idx - } - - return sum - } - - func set(_ id: MessageId, count: Int) { - self.update(peerId: id.peerId, id: id.id, count: count) - } - - func get(_ id: MessageId) -> Int32 { - return self.get(peerId: id.peerId, id: id.id) - }*/ } diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index 661c8017f5..b4fd8ceee0 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -100,6 +100,16 @@ public final class SqliteInterface { self.database = Database(databasePath) } + func unlock(password: String) -> Bool { + self.database.execute("PRAGMA key=\"\(password)\"") + var result = false + self.select("count(*) FROM sqlite_master", { _ in + result = true + return false + }) + return result + } + func select(_ query: String, _ f: (SqliteStatementCursor) -> Bool) { var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, query, -1, &statement, nil) From 54d3af26b14df29937875e9fb557719dd9e76546 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 9 Mar 2017 21:14:48 +0300 Subject: [PATCH 098/427] no message --- Postbox/SqliteInterface.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index b4fd8ceee0..5ee633095a 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -100,7 +100,7 @@ public final class SqliteInterface { self.database = Database(databasePath) } - func unlock(password: String) -> Bool { + public func unlock(password: String) -> Bool { self.database.execute("PRAGMA key=\"\(password)\"") var result = false self.select("count(*) FROM sqlite_master", { _ in @@ -110,7 +110,7 @@ public final class SqliteInterface { return result } - func select(_ query: String, _ f: (SqliteStatementCursor) -> Bool) { + public func select(_ query: String, _ f: (SqliteStatementCursor) -> Bool) { var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, query, -1, &statement, nil) let preparedStatement = SqliteInterfaceStatement(statement: statement) From 8da40d0b014f9208102fdaee0261bbc99e8f16c1 Mon Sep 17 00:00:00 2001 From: overtake Date: Thu, 9 Mar 2017 21:15:30 +0300 Subject: [PATCH 099/427] some passcode changes --- Postbox/MetadataTable.swift | 22 ++++++++++++++++------ Postbox/PostboxAccess.swift | 4 ++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index dec5657a5b..317ff58a17 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -10,17 +10,17 @@ private enum MetadataKey: Int32 { public enum PostboxAccessChallengeData: Coding { case none - case numericalPassword(String) - case plaintextPassword(String) + case numericalPassword(String, Int32?) + case plaintextPassword(String, Int32?) public init(decoder: Decoder) { switch decoder.decodeInt32ForKey("r") as Int32 { case 0: self = .none case 1: - self = .numericalPassword(decoder.decodeStringForKey("t")) + self = .numericalPassword(decoder.decodeStringForKey("t"), decoder.decodeInt32ForKey("a")) case 2: - self = .plaintextPassword(decoder.decodeStringForKey("t")) + self = .plaintextPassword(decoder.decodeStringForKey("t"), decoder.decodeInt32ForKey("a")) default: assertionFailure() self = .none @@ -31,12 +31,22 @@ public enum PostboxAccessChallengeData: Coding { switch self { case .none: encoder.encodeInt32(0, forKey: "r") - case let .numericalPassword(text): + case let .numericalPassword(text, timeout): encoder.encodeInt32(1, forKey: "r") encoder.encodeString(text, forKey: "t") - case let .plaintextPassword(text): + if let timeout = timeout { + encoder.encodeInt32(timeout, forKey: "a") + } else { + encoder.encodeNil(forKey: "a") + } + case let .plaintextPassword(text, timeout): encoder.encodeInt32(2, forKey: "r") encoder.encodeString(text, forKey: "t") + if let timeout = timeout { + encoder.encodeInt32(timeout, forKey: "a") + } else { + encoder.encodeNil(forKey: "a") + } } } } diff --git a/Postbox/PostboxAccess.swift b/Postbox/PostboxAccess.swift index fa266d7102..eb2802f3fb 100644 --- a/Postbox/PostboxAccess.swift +++ b/Postbox/PostboxAccess.swift @@ -38,7 +38,7 @@ public func accessPostbox(basePath: String, password: String?) -> Signal Signal Date: Thu, 9 Mar 2017 22:29:33 +0300 Subject: [PATCH 100/427] no message --- Postbox.xcodeproj/project.pbxproj | 3 --- Postbox/PostboxPrivate/sqlcipher/sqlite3.h | 8 ++++++++ Postbox/SqliteInterface.swift | 12 ++++-------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 0a43dc1b49..15ad5eea4e 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -1245,7 +1245,6 @@ "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_ENABLE_FTS3", ); OTHER_SWIFT_FLAGS = "-DDEBUG"; PRODUCT_BUNDLE_IDENTIFIER = org.telegram.Telegram.PostboxMac; @@ -1281,7 +1280,6 @@ "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_ENABLE_FTS3", ); PRODUCT_BUNDLE_IDENTIFIER = org.telegram.Telegram.PostboxMac; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1316,7 +1314,6 @@ "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_ENABLE_FTS3", ); PRODUCT_BUNDLE_IDENTIFIER = org.telegram.Telegram.PostboxMac; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/Postbox/PostboxPrivate/sqlcipher/sqlite3.h b/Postbox/PostboxPrivate/sqlcipher/sqlite3.h index 9879f80d40..08aa2b41cf 100644 --- a/Postbox/PostboxPrivate/sqlcipher/sqlite3.h +++ b/Postbox/PostboxPrivate/sqlcipher/sqlite3.h @@ -34,6 +34,14 @@ #define _SQLITE3_H_ #include /* Needed for the definition of va_list */ +#ifndef SQLITE_HAS_CODEC +# define SQLITE_HAS_CODEC 1 +#endif + +#ifndef DSQLCIPHER_CRYPTO_CC +# define DSQLCIPHER_CRYPTO_CC 1 +#endif + /* ** Make sure we can call this stuff from C++. */ diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index 5ee633095a..d993c6e5dc 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -100,14 +100,10 @@ public final class SqliteInterface { self.database = Database(databasePath) } - public func unlock(password: String) -> Bool { - self.database.execute("PRAGMA key=\"\(password)\"") - var result = false - self.select("count(*) FROM sqlite_master", { _ in - result = true - return false - }) - return result + public func unlock(password: Data) -> Bool { + return password.withUnsafeBytes { (bytes: UnsafePointer) -> Bool in + return sqlite3_key(self.database.handle, bytes, Int32(password.count)) == SQLITE_OK + } } public func select(_ query: String, _ f: (SqliteStatementCursor) -> Bool) { From 2cba4c7b0c2773f0dcde1e108d834ad98dd66fff Mon Sep 17 00:00:00 2001 From: overtake Date: Thu, 9 Mar 2017 22:30:19 +0300 Subject: [PATCH 101/427] access --- Postbox/SqliteInterface.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index 5ee633095a..a8ec30412c 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -96,7 +96,7 @@ public final class SqliteStatementCursor { public final class SqliteInterface { private let database: Database - init(databasePath: String) { + public init(databasePath: String) { self.database = Database(databasePath) } From 7f5a4ac3e1ac3c21f0cb31a7c9fa4c8b4f8f37e0 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 10 Mar 2017 21:34:39 +0300 Subject: [PATCH 102/427] no message --- Postbox/SqliteInterface.swift | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index d993c6e5dc..0a41960a58 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -106,6 +106,23 @@ public final class SqliteInterface { } } + public func selectWithKey(_ query: String, index: Int, key: Data, _ f: (SqliteStatementCursor) -> Bool) { + var statement: OpaquePointer? = nil + sqlite3_prepare_v2(database.handle, query, -1, &statement, nil) + let preparedStatement = SqliteInterfaceStatement(statement: statement) + key.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + preparedStatement.bind(index, data: bytes, length: key.count) + } + let cursor = SqliteStatementCursor(statement: preparedStatement) + while preparedStatement.step() { + if !f(cursor) { + break + } + } + preparedStatement.reset() + preparedStatement.destroy() + } + public func select(_ query: String, _ f: (SqliteStatementCursor) -> Bool) { var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, query, -1, &statement, nil) From b84989e58c7630c82e9e42e7e58077b81b8a6a47 Mon Sep 17 00:00:00 2001 From: overtake Date: Fri, 10 Mar 2017 21:44:05 +0300 Subject: [PATCH 103/427] no message --- Postbox/SqliteInterface.swift | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index 3379920898..bf02131ae9 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -91,6 +91,10 @@ public final class SqliteStatementCursor { return "" } } + + public func getData(at index: Int) -> Data { + return self.statement.valueAt(index).makeData() + } } public final class SqliteInterface { @@ -119,4 +123,21 @@ public final class SqliteInterface { preparedStatement.reset() preparedStatement.destroy() } + + public func selectWithKey(_ query: String, index: Int, key: Data, _ f: (SqliteStatementCursor) -> Bool) { + var statement: OpaquePointer? = nil + sqlite3_prepare_v2(database.handle, query, -1, &statement, nil) + let preparedStatement = SqliteInterfaceStatement(statement: statement) + key.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + preparedStatement.bind(index, data: bytes, length: key.count) + } + let cursor = SqliteStatementCursor(statement: preparedStatement) + while preparedStatement.step() { + if !f(cursor) { + break + } + } + preparedStatement.reset() + preparedStatement.destroy() + } } From a3c5fed02429e730dde5508b4652b3e9d7b32787 Mon Sep 17 00:00:00 2001 From: Peter Date: Sat, 11 Mar 2017 12:44:37 +0300 Subject: [PATCH 104/427] no message --- Postbox/Coding.swift | 12 ++++++++---- Postbox/MessageHistoryTable.swift | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index bd3eac7815..d5acbb6d98 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -294,10 +294,14 @@ public final class Encoder { self.encodeKey(key) var type: Int8 = ValueType.String.rawValue self.buffer.write(&type, offset: 0, length: 1) - let data = value.data(using: .utf8, allowLossyConversion: true)! - var length: Int32 = Int32(data.count) - self.buffer.write(&length, offset: 0, length: 4) - self.buffer.write(data) + if let data = value.data(using: .utf8, allowLossyConversion: true) { + var length: Int32 = Int32(data.count) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(data) + } else { + var length: Int32 = 0 + self.buffer.write(&length, offset: 0, length: 4) + } } public func encodeRootObject(_ value: Coding) { diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index a1de367194..b6495e7f4b 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -589,10 +589,14 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&hasAuthor, offset: 0, length: 1) } - let data = message.text.data(using: .utf8, allowLossyConversion: true)! - var length: Int32 = Int32(data.count) - sharedBuffer.write(&length, offset: 0, length: 4) - sharedBuffer.write(data) + if let data = message.text.data(using: .utf8, allowLossyConversion: true) { + var length: Int32 = Int32(data.count) + sharedBuffer.write(&length, offset: 0, length: 4) + sharedBuffer.write(data) + } else { + var length: Int32 = 0 + sharedBuffer.write(&length, offset: 0, length: 4) + } let attributesBuffer = WriteBuffer() From b3da75142a9ace2965f3664ff23300535b6ff5cc Mon Sep 17 00:00:00 2001 From: overtake Date: Sat, 11 Mar 2017 12:46:26 +0300 Subject: [PATCH 105/427] remove selectWithKey duplicate --- Postbox/SqliteInterface.swift | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index 13364ea5f1..b8128a308c 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -140,21 +140,5 @@ public final class SqliteInterface { preparedStatement.reset() preparedStatement.destroy() } - - public func selectWithKey(_ query: String, index: Int, key: Data, _ f: (SqliteStatementCursor) -> Bool) { - var statement: OpaquePointer? = nil - sqlite3_prepare_v2(database.handle, query, -1, &statement, nil) - let preparedStatement = SqliteInterfaceStatement(statement: statement) - key.withUnsafeBytes { (bytes: UnsafePointer) -> Void in - preparedStatement.bind(index, data: bytes, length: key.count) - } - let cursor = SqliteStatementCursor(statement: preparedStatement) - while preparedStatement.step() { - if !f(cursor) { - break - } - } - preparedStatement.reset() - preparedStatement.destroy() - } + } From b424fae986bac1f822bb156b86d5477a9ae59ab2 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 12 Mar 2017 19:02:36 +0300 Subject: [PATCH 106/427] no message --- Postbox/OrderStatisticTable.swift | 253 +++++++++++-------- Postbox/Postbox.swift | 8 + PostboxTests/OrderStatisticTreeTests.swift | 20 +- PostboxTests/OrderedItemListTableTests.swift | 6 +- 4 files changed, 172 insertions(+), 115 deletions(-) diff --git a/Postbox/OrderStatisticTable.swift b/Postbox/OrderStatisticTable.swift index 7ba2fe6f2d..d805b2aef5 100644 --- a/Postbox/OrderStatisticTable.swift +++ b/Postbox/OrderStatisticTable.swift @@ -1,6 +1,6 @@ import Foundation -private struct MessageOrderKey: Comparable { +struct MessageOrderKey: Hashable, Comparable { let timestamp: Int32 let namespace: MessageId.Namespace let id: MessageId.Id @@ -20,14 +20,18 @@ private struct MessageOrderKey: Comparable { return lhs.id < rhs.id } + + var hashValue: Int { + return self.timestamp.hashValue + } } -private enum BTreeNodePosition { +enum BTreeNodePosition { case left case right } -private final class BTreeNode { +final class BTreeNode { let reference: Int32 let keys: [MessageOrderKey] let values: [Int32] @@ -103,19 +107,106 @@ private final class BTreeNode { } -private final class BTreeAccess { - let order: Int = 2 +final class BTreeAccess { + let order: Int + + var memory: [Int32: BTreeNode] = [:] + + var rootNodeReference: Int32! + var nextReference: Int32 = 0 func getNode(_ reference: Int32) -> BTreeNode? { - return nil + return self.memory[reference] } - func replaceNode(_ reference: Int32, with: BTreeNode) { + func replaceNode(_ reference: Int32, with node: BTreeNode) { + self.memory[reference] = node } func allocateNodeReference() -> Int32 { - return 0 + let value = self.nextReference + self.nextReference += 1 + return value } + + init(order: Int) { + self.order = order + let reference = self.allocateNodeReference() + self.rootNodeReference = reference + self.replaceNode(reference, with: BTreeNode(reference: reference)) + } + + func value(for key: MessageOrderKey) -> Int32? { + if let rootNode = self.getNode(self.rootNodeReference) { + if rootNode.numberOfKeys > 0 { + return value(node: rootNode, key: key) + } else { + return nil + } + } else { + assertionFailure() + return nil + } + } + + public func insert(_ value: Int32, for key: MessageOrderKey) { + if let rootNode = self.getNode(self.rootNodeReference) { + let updatedNode = self.insert(node: rootNode, value: value, key: key) + if updatedNode.numberOfKeys > self.order * 1 { + self.splitRoot() + } + } else { + assertionFailure() + } + } + + private func splitRoot() { + var updatedRootNode = self.getNode(self.rootNodeReference)! + let middleIndexOfOldRoot = updatedRootNode.numberOfKeys / 2 + + var newRoot = BTreeNode(reference: self.allocateNodeReference(), keys: [updatedRootNode.keys[middleIndexOfOldRoot]], values: [updatedRootNode.values[middleIndexOfOldRoot]], childrenReferences: [updatedRootNode.reference]) + + updatedRootNode = updatedRootNode.removingKeyValueAt(middleIndexOfOldRoot) + //rootNode.keys.remove(at: middleIndexOfOldRoot) + //rootNode.values.remove(at: middleIndexOfOldRoot) + + var newRightChild = BTreeNode(reference: self.allocateNodeReference(), keys: Array(updatedRootNode.keys[updatedRootNode.keys.indices.suffix(from: middleIndexOfOldRoot)]), values: Array(updatedRootNode.values[updatedRootNode.values.indices.suffix(from: middleIndexOfOldRoot)])) + updatedRootNode = updatedRootNode.removingKeyValueInRange(updatedRootNode.keys.indices.suffix(from: middleIndexOfOldRoot)) + //rootNode.keys.removeSubrange(rootNode.keys.indices.suffix(from: middleIndexOfOldRoot)) + //rootNode.values.removeSubrange(rootNode.values.indices.suffix(from: middleIndexOfOldRoot)) + + if updatedRootNode.childrenReferences != nil { + newRightChild = newRightChild.replacingChildrenReferences(Array( + updatedRootNode.childrenReferences![updatedRootNode.childrenReferences!.indices.suffix(from: (middleIndexOfOldRoot + 1))] + )) + updatedRootNode = updatedRootNode.removingChildrenReferencesInRange(updatedRootNode.childrenReferences!.indices.suffix(from: middleIndexOfOldRoot + 1)) + //rootNode.children!.removeSubrange( + // rootNode.children!.indices.suffix(from: (middleIndexOfOldRoot + 1)) + //) + } + + newRoot = newRoot.insertingChildAt(newRoot.childrenReferences!.count, reference: newRightChild.reference) + //newRoot.children!.append(newRightChild) + + self.replaceNode(newRoot.reference, with: newRoot) + self.replaceNode(newRightChild.reference, with: newRightChild) + self.replaceNode(updatedRootNode.reference, with: updatedRootNode) + + self.rootNodeReference = newRoot.reference + //rootNode = newRoot + } + + /*public func remove(_ key: MessageOrderKey) { + guard rootNode.numberOfKeys > 0 else { + return + } + + rootNode.remove(key) + + if rootNode.numberOfKeys == 0 && !rootNode.isLeaf { + rootNode = rootNode.children!.first! + } + }*/ func value(node: BTreeNode, key: MessageOrderKey) -> Int32? { var index = node.keys.startIndex @@ -139,17 +230,22 @@ private final class BTreeAccess { } //return children?[index].value(for: key) } else { - if let child = self.getNode(index + 1) { - return self.value(node: child, key: key) + if let childrenReferences = node.childrenReferences { + if let child = self.getNode(childrenReferences[index + 1]) { + return self.value(node: child, key: key) + } else { + assertionFailure() + return nil + } } else { - assertionFailure() return nil } + //return children?[(index + 1)].value(for: key) } } - func insert(node: BTreeNode, value: Int32, key: MessageOrderKey) { + func insert(node: BTreeNode, value: Int32, key: MessageOrderKey) -> BTreeNode { var index = node.keys.startIndex while index < node.keys.endIndex && node.keys[index] < key { @@ -157,32 +253,39 @@ private final class BTreeAccess { } if index < node.keys.endIndex && node.keys[index] == key { - self.replaceNode(node.reference, with: node.replacingValueAt(index, with: value)) + let updatedNode = node.replacingValueAt(index, with: value) + self.replaceNode(node.reference, with: updatedNode) //values[index] = value - return - } - - if node.isLeaf { - self.replaceNode(node.reference, with: node.insertingKeyValueAt(index, key: key, value: value)) - //keys.insert(key, at: index) - //values.insert(value, at: index) + return updatedNode } else { - if let child = self.getNode(node.childrenReferences![index]) { - self.insert(node: child, value: value, key: key) - if child.numberOfKeys > self.order * 2 { - self.split(node: node, child: child, atIndex: index) - } + if node.isLeaf { + let updatedNode = node.insertingKeyValueAt(index, key: key, value: value) + self.replaceNode(node.reference, with: updatedNode) + //keys.insert(key, at: index) + //values.insert(value, at: index) + return updatedNode } else { - assertionFailure() + if let child = self.getNode(node.childrenReferences![index]) { + let updatedChild = self.insert(node: child, value: value, key: key) + + if updatedChild.numberOfKeys > self.order * 2 { + return self.split(node: node, child: updatedChild, atIndex: index) + } else { + return node + } + } else { + assertionFailure() + return node + } + //children![index].insert(value, for: key) + //if children![index].numberOfKeys > self.order * 2 { + // split(child: children![index], atIndex: index) + //} } - //children![index].insert(value, for: key) - //if children![index].numberOfKeys > self.order * 2 { - // split(child: children![index], atIndex: index) - //} } } - private func split(node: BTreeNode, child: BTreeNode, atIndex index: Int) { + private func split(node: BTreeNode, child: BTreeNode, atIndex index: Int) -> BTreeNode { let middleIndex = child.numberOfKeys / 2 var updatedNode = node.insertingKeyValueAt(index, key: child.keys[middleIndex], value: child.values[middleIndex]) @@ -193,7 +296,7 @@ private final class BTreeAccess { //child.keys.remove(at: middleIndex) //child.values.remove(at: middleIndex) - var rightSibling = BTreeNode(reference: self.allocateNodeReference(), keys: Array(child.keys[updatedChild.keys.indices.suffix(from: middleIndex)]), values: Array(child.values[updatedChild.values.indices.suffix(from: middleIndex)])) + var rightSibling = BTreeNode(reference: self.allocateNodeReference(), keys: Array(updatedChild.keys[updatedChild.keys.indices.suffix(from: middleIndex)]), values: Array(updatedChild.values[updatedChild.values.indices.suffix(from: middleIndex)])) updatedChild = updatedChild.removingKeyValueInRange(updatedChild.keys.indices.suffix(from: middleIndex)) //child.keys.removeSubrange(child.keys.indices.suffix(from: middleIndex)) @@ -211,6 +314,16 @@ private final class BTreeAccess { self.replaceNode(child.reference, with: updatedChild) self.replaceNode(rightSibling.reference, with: rightSibling) + + return updatedNode + } + + private func inorderPredecessor(node: BTreeNode) -> BTreeNode { + if node.isLeaf { + return node + } else { + return self.inorderPredecessor(node: self.getNode(node.childrenReferences!.last!)!) + } } /*private var inorderPredecessor: BTreeNode { @@ -345,82 +458,6 @@ private final class BTreeAccess { }*/ } -/*private class BTree { - public let order: Int - - var rootNode: BTreeNode! - - public init?(order: Int) { - guard order > 0 else { - assertionFailure() - return nil - } - self.order = order - rootNode = BTreeNode(order: self.order) - } - - public func value(for key: MessageOrderKey) -> Int32? { - guard rootNode.numberOfKeys > 0 else { - return nil - } - - return rootNode.value(for: key) - } - - public func insert(_ value: Int32, for key: MessageOrderKey) { - rootNode.insert(value, for: key) - - if rootNode.numberOfKeys > order * 2 { - splitRoot() - } - } - - private func splitRoot() { - let middleIndexOfOldRoot = rootNode.numberOfKeys / 2 - - let newRoot = BTreeNode( - order: self.order, - keys: [rootNode.keys[middleIndexOfOldRoot]], - values: [rootNode.values[middleIndexOfOldRoot]], - children: [rootNode] - ) - rootNode.keys.remove(at: middleIndexOfOldRoot) - rootNode.values.remove(at: middleIndexOfOldRoot) - - let newRightChild = BTreeNode( - order: self.order, - keys: Array(rootNode.keys[rootNode.keys.indices.suffix(from: middleIndexOfOldRoot)]), - values: Array(rootNode.values[rootNode.values.indices.suffix(from: middleIndexOfOldRoot)]) - ) - rootNode.keys.removeSubrange(rootNode.keys.indices.suffix(from: middleIndexOfOldRoot)) - rootNode.values.removeSubrange(rootNode.values.indices.suffix(from: middleIndexOfOldRoot)) - - if rootNode.children != nil { - newRightChild.children = Array( - rootNode.children![rootNode.children!.indices.suffix(from: (middleIndexOfOldRoot + 1))] - ) - rootNode.children!.removeSubrange( - rootNode.children!.indices.suffix(from: (middleIndexOfOldRoot + 1)) - ) - } - - newRoot.children!.append(newRightChild) - rootNode = newRoot - } - - public func remove(_ key: MessageOrderKey) { - guard rootNode.numberOfKeys > 0 else { - return - } - - rootNode.remove(key) - - if rootNode.numberOfKeys == 0 && !rootNode.isLeaf { - rootNode = rootNode.children!.first! - } - } -}*/ - class MessageOrderStatisticTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .binary) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 0a5dad0964..1a94eb0e89 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -179,6 +179,14 @@ public final class Modifier { self.postbox?.updatePeerPresences(peerPresences) } + public func getContactPeerIds() -> Set { + if let postbox = self.postbox { + return postbox.contactsTable.get() + } else { + return Set() + } + } + public func replaceContactPeerIds(_ peerIds: Set) { self.postbox?.replaceContactPeerIds(peerIds) } diff --git a/PostboxTests/OrderStatisticTreeTests.swift b/PostboxTests/OrderStatisticTreeTests.swift index d5d0c040ca..3a767b5e5b 100644 --- a/PostboxTests/OrderStatisticTreeTests.swift +++ b/PostboxTests/OrderStatisticTreeTests.swift @@ -116,11 +116,21 @@ class OrderStatisticTreeTests: XCTestCase { } func testOST() { - /*let tree = RBTree(rootData: 0) - for _ in 0 ..< 1000 { - let key = Int(arc4random_uniform(UInt32(Int32.max - 1))) - tree.insert(key) + let access = BTreeAccess(order: 100) + var entries: [MessageOrderKey: Int32] = [:] + for i in 0 ..< 1000 { + let k = Int32(bitPattern: arc4random()) + let key = MessageOrderKey(timestamp: k, namespace: 0, id: 0) + let value = Int32(bitPattern: arc4random()) + access.insert(value, for: key) + entries[key] = value + } + for (key, value) in entries { + if let result = access.value(for: key) { + XCTAssert(result == value) + } else { + XCTAssert(false) + } } - print("OST height: \(tree.depth())")*/ } } diff --git a/PostboxTests/OrderedItemListTableTests.swift b/PostboxTests/OrderedItemListTableTests.swift index a1197ad009..ff860b7321 100644 --- a/PostboxTests/OrderedItemListTableTests.swift +++ b/PostboxTests/OrderedItemListTableTests.swift @@ -65,20 +65,22 @@ class OrderedItemListTableTests: XCTestCase { } private func setIds(_ ids: [Int32]) { + var operations: [Int32 : [OrderedItemListOperation]] = [:] self.itemListTable!.replaceItems(collectionId: 0, items: ids.map { id -> OrderedItemListEntry in var idValue: Int32 = id let buffer = MemoryBuffer(memory: malloc(4)!, capacity: 4, length: 4, freeWhenDone: true) memcpy(buffer.memory, &idValue, 4) return OrderedItemListEntry(id: buffer, contents: TestListItem()) - }) + }, operations: &operations) XCTAssert(self.itemListTable!.testIntegrity(collectionId: 0), "Index integrity violated") } private func addOrMoveId(_ id: Int32, _ maxCount: Int? = nil) { + var operations: [Int32 : [OrderedItemListOperation]] = [:] var idValue: Int32 = id let buffer = MemoryBuffer(memory: malloc(4)!, capacity: 4, length: 4, freeWhenDone: true) memcpy(buffer.memory, &idValue, 4) - self.itemListTable!.addItemOrMoveToFirstPosition(collectionId: 0, item: OrderedItemListEntry(id: buffer, contents: TestListItem()), removeTailIfCountExceeds: maxCount) + self.itemListTable!.addItemOrMoveToFirstPosition(collectionId: 0, item: OrderedItemListEntry(id: buffer, contents: TestListItem()), removeTailIfCountExceeds: maxCount, operations: &operations) XCTAssert(self.itemListTable!.testIntegrity(collectionId: 0), "Index integrity violated") } From 53f23831cc33b7b09e2adea8e27441bfa00fb132 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 12 Mar 2017 21:53:41 +0300 Subject: [PATCH 107/427] no message --- Postbox/MessageHistoryReadStateTable.swift | 66 +++++++++++++++------- Postbox/MessageHistoryTable.swift | 16 ++++-- 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index ca23924717..e9385e1919 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -241,7 +241,7 @@ final class MessageHistoryReadStateTable: Table { } } - func applyIncomingMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: (MessageId.Id, Bool)?) -> (CombinedPeerReadState?, Bool) { + func applyIncomingMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Namespace, MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: (MessageId.Id, Bool)?) -> (CombinedPeerReadState?, Bool) { if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] { if traceReadStates { print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId) (before: \(states.namespaces))") @@ -250,7 +250,7 @@ final class MessageHistoryReadStateTable: Table { switch state { case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count): if maxIncomingReadId < messageId.id || (topMessageId != nil && (messageId.id == topMessageId!.0 || topMessageId!.1) && state.count != 0) { - var (deltaCount, holes) = incomingStatsInRange(maxIncomingReadId + 1, messageId.id) + var (deltaCount, holes) = incomingStatsInRange(messageId.namespace, maxIncomingReadId + 1, messageId.id) if traceReadStates { print("[ReadStateTable] applyMaxReadId after deltaCount: \(deltaCount), holes: \(holes)") @@ -351,25 +351,51 @@ final class MessageHistoryReadStateTable: Table { return (nil, false, []) } - func applyInteractiveMaxReadIndex(_ messageIndex: MessageIndex, incomingStatsInRange: (MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), incomingIndexStatsInRange: (MessageIndex, MessageIndex) -> (count: Int, holes: Bool, readMesageIds: [MessageId]), topMessageId: (MessageId.Id, Bool)?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult, readMesageIds: [MessageId]) { - if let states = self.get(messageIndex.id.peerId), let state = states.namespaces[messageIndex.id.namespace] { - switch state { - case .idBased: - let (combinedState, holes) = self.applyIncomingMaxReadId(messageIndex.id, incomingStatsInRange: incomingStatsInRange, topMessageId: topMessageId) - - if let combinedState = combinedState { - return (combinedState, .Push(thenSync: holes), []) + func applyInteractiveMaxReadIndex(_ messageIndex: MessageIndex, incomingStatsInRange: (MessageId.Namespace, MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), incomingIndexStatsInRange: (MessageIndex, MessageIndex) -> (count: Int, holes: Bool, readMesageIds: [MessageId]), topMessageId: (MessageId.Id, Bool)?, topMessageIndexByNamespace: (MessageId.Namespace) -> MessageIndex?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult, readMesageIds: [MessageId]) { + if let states = self.get(messageIndex.id.peerId) { + if let state = states.namespaces[messageIndex.id.namespace] { + switch state { + case .idBased: + let (combinedState, holes) = self.applyIncomingMaxReadId(messageIndex.id, incomingStatsInRange: incomingStatsInRange, topMessageId: topMessageId) + + if let combinedState = combinedState { + return (combinedState, .Push(thenSync: holes), []) + } + + return (combinedState, holes ? .Push(thenSync: true) : .None, []) + case .indexBased: + let (combinedState, holes, messageIds) = self.applyIncomingMaxReadIndex(messageIndex, incomingStatsInRange: incomingIndexStatsInRange) + + if let combinedState = combinedState { + return (combinedState, .Push(thenSync: holes), messageIds) + } + + return (combinedState, holes ? .Push(thenSync: true) : .None, messageIds) + } + } else { + for (namespace, state) in states.namespaces { + if let topIndex = topMessageIndexByNamespace(namespace), topIndex <= messageIndex { + switch state { + case .idBased: + let (combinedState, holes) = self.applyIncomingMaxReadId(topIndex.id, incomingStatsInRange: incomingStatsInRange, topMessageId: nil) + + if let combinedState = combinedState { + return (combinedState, .Push(thenSync: holes), []) + } + + return (combinedState, holes ? .Push(thenSync: true) : .None, []) + case .indexBased: + let (combinedState, holes, messageIds) = self.applyIncomingMaxReadIndex(topIndex, incomingStatsInRange: incomingIndexStatsInRange) + + if let combinedState = combinedState { + return (combinedState, .Push(thenSync: holes), messageIds) + } + + return (combinedState, holes ? .Push(thenSync: true) : .None, messageIds) + } } - - return (combinedState, holes ? .Push(thenSync: true) : .None, []) - case .indexBased: - let (combinedState, holes, messageIds) = self.applyIncomingMaxReadIndex(messageIndex, incomingStatsInRange: incomingIndexStatsInRange) - - if let combinedState = combinedState { - return (combinedState, .Push(thenSync: holes), messageIds) - } - - return (combinedState, holes ? .Push(thenSync: true) : .None, messageIds) + } + return (nil, .Push(thenSync: true), []) } } else { return (nil, .Push(thenSync: true), []) diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index b6495e7f4b..dcf36a9908 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -379,8 +379,8 @@ final class MessageHistoryTable: Table { } } - let (combinedState, invalidated) = self.readStateTable.applyIncomingMaxReadId(messageId, incomingStatsInRange: { fromId, toId in - return self.messageHistoryIndexTable.incomingMessageCountInRange(messageId.peerId, namespace: messageId.namespace, minId: fromId, maxId: toId) + let (combinedState, invalidated) = self.readStateTable.applyIncomingMaxReadId(messageId, incomingStatsInRange: { namespace, fromId, toId in + return self.messageHistoryIndexTable.incomingMessageCountInRange(messageId.peerId, namespace: namespace, minId: fromId, maxId: toId) }, topMessageId: topMessageId) if let combinedState = combinedState { @@ -442,11 +442,17 @@ final class MessageHistoryTable: Table { } } - let (combinedState, result, messageIds) = self.readStateTable.applyInteractiveMaxReadIndex(messageIndex, incomingStatsInRange: { fromId, toId in - return self.messageHistoryIndexTable.incomingMessageCountInRange(messageIndex.id.peerId, namespace: messageIndex.id.namespace, minId: fromId, maxId: toId) + let (combinedState, result, messageIds) = self.readStateTable.applyInteractiveMaxReadIndex(messageIndex, incomingStatsInRange: { namespace, fromId, toId in + return self.messageHistoryIndexTable.incomingMessageCountInRange(messageIndex.id.peerId, namespace: namespace, minId: fromId, maxId: toId) }, incomingIndexStatsInRange: { fromIndex, toIndex in return self.incomingMessageCountInRange(messageIndex.id.peerId, namespace: messageIndex.id.namespace, fromIndex: fromIndex, toIndex: toIndex) - }, topMessageId: topMessageId) + }, topMessageId: topMessageId, topMessageIndexByNamespace: { namespace in + if let topEntry = self.messageHistoryIndexTable.top(messageIndex.id.peerId, namespace: namespace), case let .Message(index) = topEntry { + return index + } else { + return nil + } + }) if let combinedState = combinedState { if operationsByPeerId[messageIndex.id.peerId] == nil { From 246bf1bc332a87d2601601b05a459dcf8b8e6a62 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 12 Mar 2017 22:38:46 +0300 Subject: [PATCH 108/427] no message --- Postbox/PeerView.swift | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index d23d7ed984..494f39270c 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -123,6 +123,20 @@ final class MutablePeerView { updated = true } + if let replaceContactPeerIds = replaceContactPeerIds { + if self.peerIsContact { + if !replaceContactPeerIds.contains(self.peerId) { + self.peerIsContact = false + updated = true + } + } else { + if replaceContactPeerIds.contains(self.peerId) { + self.peerIsContact = true + updated = true + } + } + } + return updated } } From 3730bfa8207bb8880b59b095f1398afd7f41de39 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 14 Mar 2017 22:12:11 +0300 Subject: [PATCH 109/427] no message --- Postbox.xcodeproj/project.pbxproj | 18 +++++ Postbox/ItemCollectionInfoTable.swift | 4 ++ Postbox/ItemCollectionInfosView.swift | 97 +++++++++++++++++++++++++++ Postbox/ItemCollectionItemTable.swift | 7 +- Postbox/PeerOperationLogTable.swift | 2 +- Postbox/Postbox.swift | 34 +++++++++- Postbox/PostboxTransaction.swift | 12 +++- Postbox/PostboxView.swift | 43 ++++++++++++ Postbox/SqliteValueBox.swift | 11 +-- Postbox/ViewTracker.swift | 21 +++++- Postbox/Views.swift | 30 +++++++++ 11 files changed, 266 insertions(+), 13 deletions(-) create mode 100644 Postbox/ItemCollectionInfosView.swift create mode 100644 Postbox/PostboxView.swift create mode 100644 Postbox/Views.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 15ad5eea4e..299149a99b 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -251,6 +251,12 @@ D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8701C5A0E9B00037222 /* PeerTable.swift */; }; D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */; }; D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8741C5A334100037222 /* SimpleDictionary.swift */; }; + D0FA0AC71E77F0A2005BB9B7 /* ItemCollectionInfosView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FA0AC61E77F0A2005BB9B7 /* ItemCollectionInfosView.swift */; }; + D0FA0AC81E77F0A2005BB9B7 /* ItemCollectionInfosView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FA0AC61E77F0A2005BB9B7 /* ItemCollectionInfosView.swift */; }; + D0FA0ACA1E780A26005BB9B7 /* PostboxView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FA0AC91E780A26005BB9B7 /* PostboxView.swift */; }; + D0FA0ACB1E780A26005BB9B7 /* PostboxView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FA0AC91E780A26005BB9B7 /* PostboxView.swift */; }; + D0FA0ACD1E781067005BB9B7 /* Views.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FA0ACC1E781067005BB9B7 /* Views.swift */; }; + D0FA0ACE1E781067005BB9B7 /* Views.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0FA0ACC1E781067005BB9B7 /* Views.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -399,6 +405,9 @@ D0F9E8701C5A0E9B00037222 /* PeerTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerTable.swift; sourceTree = ""; }; D0F9E8721C5A1EE500037222 /* GlobalMessageIdsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GlobalMessageIdsTable.swift; sourceTree = ""; }; D0F9E8741C5A334100037222 /* SimpleDictionary.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SimpleDictionary.swift; sourceTree = ""; }; + D0FA0AC61E77F0A2005BB9B7 /* ItemCollectionInfosView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollectionInfosView.swift; sourceTree = ""; }; + D0FA0AC91E780A26005BB9B7 /* PostboxView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxView.swift; sourceTree = ""; }; + D0FA0ACC1E781067005BB9B7 /* Views.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Views.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -619,6 +628,7 @@ D0AB0B8F1D65D4AB002C78E7 /* UnsentMessageIndicesView.swift */, D0DF0C8E1D81A350008AEB01 /* PeerView.swift */, D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */, + D0FA0AC61E77F0A2005BB9B7 /* ItemCollectionInfosView.swift */, D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */, D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */, D0AAD1B41E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift */, @@ -626,6 +636,8 @@ D08775051E3E3F2100A97350 /* PreferencesView.swift */, D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */, D087C20B1E43C11C00D686F8 /* OrderedItemListView.swift */, + D0FA0AC91E780A26005BB9B7 /* PostboxView.swift */, + D0FA0ACC1E781067005BB9B7 /* Views.swift */, ); name = Views; sourceTree = ""; @@ -909,6 +921,7 @@ D0F7B1C91E045C6A007EB8A5 /* MessageHistoryTagsTable.swift in Sources */, D0F7B1CE1E045C6A007EB8A5 /* MediaCleanupTable.swift in Sources */, D0B418271D7DFE0C004562A4 /* IpcPipe.swift in Sources */, + D0FA0ACB1E780A26005BB9B7 /* PostboxView.swift in Sources */, D0B4184D1D7DFE20004562A4 /* ContactPeersView.swift in Sources */, D00C7CE41E37861C0080C3D5 /* MessageView.swift in Sources */, D08775011E3E3D9F00A97350 /* PreferencesTable.swift in Sources */, @@ -925,6 +938,7 @@ D0F7B1C31E045C6A007EB8A5 /* Table.swift in Sources */, D0F82D101E43A024007E499C /* OrderedItemListIndexTable.swift in Sources */, D0F7B1CF1E045C6A007EB8A5 /* MetadataTable.swift in Sources */, + D0FA0ACE1E781067005BB9B7 /* Views.swift in Sources */, D0B418501D7DFE20004562A4 /* UnsentMessageIndicesView.swift in Sources */, D0BEAF641E54B2FA00BD963D /* AccountManager.swift in Sources */, D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */, @@ -939,6 +953,7 @@ D0F7B1DB1E045C6A007EB8A5 /* ItemCollectionInfoTable.swift in Sources */, D0F7B1D41E045C6A007EB8A5 /* PeerPresenceTable.swift in Sources */, D073CE761DCBF3B4007511FD /* IntermediateMessage.swift in Sources */, + D0FA0AC81E77F0A2005BB9B7 /* ItemCollectionInfosView.swift in Sources */, D0B4185A1D7DFE29004562A4 /* ViewTracker.swift in Sources */, D0F7B1E21E045C6A007EB8A5 /* PeerNameIndexTable.swift in Sources */, D073CE7C1DCBF3B4007511FD /* CachedPeerData.swift in Sources */, @@ -1025,6 +1040,7 @@ D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, + D0FA0ACA1E780A26005BB9B7 /* PostboxView.swift in Sources */, D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */, D0F82CFD1E4345D7007E499C /* OrderedItemListTable.swift in Sources */, D0AB0B8C1D65D488002C78E7 /* MessageHistoryHolesView.swift in Sources */, @@ -1041,6 +1057,7 @@ D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D0F82D0F1E43A024007E499C /* OrderedItemListIndexTable.swift in Sources */, + D0FA0ACD1E781067005BB9B7 /* Views.swift in Sources */, D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */, D0BEAF631E54B2FA00BD963D /* AccountManager.swift in Sources */, D0AD23271E194D1C00A7089A /* PeerOperationLogTable.swift in Sources */, @@ -1055,6 +1072,7 @@ D03120FA1DA540F0006A2A60 /* CachedPeerData.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, + D0FA0AC71E77F0A2005BB9B7 /* ItemCollectionInfosView.swift in Sources */, D0F019FD1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift in Sources */, D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, diff --git a/Postbox/ItemCollectionInfoTable.swift b/Postbox/ItemCollectionInfoTable.swift index 950a23ad93..74272657d4 100644 --- a/Postbox/ItemCollectionInfoTable.swift +++ b/Postbox/ItemCollectionInfoTable.swift @@ -1,5 +1,9 @@ import Foundation +enum ItemCollectionInfosOperation { + case replaceInfos(ItemCollectionId.Namespace) +} + final class ItemCollectionInfoTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .binary) diff --git a/Postbox/ItemCollectionInfosView.swift b/Postbox/ItemCollectionInfosView.swift new file mode 100644 index 0000000000..b9ff98ae4b --- /dev/null +++ b/Postbox/ItemCollectionInfosView.swift @@ -0,0 +1,97 @@ +import Foundation + +public final class ItemCollectionInfoEntry { + public let id: ItemCollectionId + public let info: ItemCollectionInfo + public let firstItem: ItemCollectionItem? + + init(id: ItemCollectionId, info: ItemCollectionInfo, firstItem: ItemCollectionItem?) { + self.id = id + self.info = info + self.firstItem = firstItem + } +} + +final class MutableItemCollectionInfosView: MutablePostboxView { + let namespaces: [ItemCollectionId.Namespace] + var entriesByNamespace: [ItemCollectionId.Namespace: [ItemCollectionInfoEntry]] + + init(postbox: Postbox, namespaces: [ItemCollectionId.Namespace]) { + self.namespaces = namespaces + + var entriesByNamespace: [ItemCollectionId.Namespace: [ItemCollectionInfoEntry]] = [:] + for namespace in namespaces { + let infos = postbox.itemCollectionInfoTable.getInfos(namespace: namespace) + var entries: [ItemCollectionInfoEntry] = [] + for (_, id, info) in infos { + let firstItem = postbox.itemCollectionItemTable.higherItems(collectionId: id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first + entries.append(ItemCollectionInfoEntry(id: id, info: info, firstItem: firstItem)) + } + entriesByNamespace[namespace] = entries + } + self.entriesByNamespace = entriesByNamespace + } + + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { + if transaction.currentItemCollectionInfosOperations.isEmpty && transaction.currentItemCollectionItemsOperations.isEmpty { + return false + } + + var updated = false + + var reloadInfosNamespaces = Set() + var reloadTopItemCollectionIds = Set() + for operation in transaction.currentItemCollectionInfosOperations { + switch operation { + case let .replaceInfos(namespace): + reloadInfosNamespaces.insert(namespace) + } + } + for (id, operations) in transaction.currentItemCollectionItemsOperations { + for operation in operations { + switch operation { + case .replaceItems: + reloadTopItemCollectionIds.insert(id) + } + } + } + if !reloadInfosNamespaces.isEmpty { + updated = true + + var entriesByNamespace: [ItemCollectionId.Namespace: [ItemCollectionInfoEntry]] = [:] + for namespace in self.namespaces { + let infos = postbox.itemCollectionInfoTable.getInfos(namespace: namespace) + var entries: [ItemCollectionInfoEntry] = [] + for (_, id, info) in infos { + let firstItem = postbox.itemCollectionItemTable.higherItems(collectionId: id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first + entries.append(ItemCollectionInfoEntry(id: id, info: info, firstItem: firstItem)) + } + entriesByNamespace[namespace] = entries + } + self.entriesByNamespace = entriesByNamespace + } else if !reloadTopItemCollectionIds.isEmpty { + for (namespace, entries) in self.entriesByNamespace { + for i in 0 ..< entries.count { + if reloadTopItemCollectionIds.contains(entries[i].id) { + updated = true + let firstItem = postbox.itemCollectionItemTable.higherItems(collectionId: entries[i].id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first + self.entriesByNamespace[namespace]![i] = ItemCollectionInfoEntry(id: entries[i].id, info: entries[i].info, firstItem: firstItem) + } + } + } + } + return updated + } + + func immutableView() -> PostboxView { + return ItemCollectionInfosView(self) + } +} + +public final class ItemCollectionInfosView: PostboxView { + public let entriesByNamespace: [ItemCollectionId.Namespace: [ItemCollectionInfoEntry]] + + init(_ view: MutableItemCollectionInfosView) { + self.entriesByNamespace = view.entriesByNamespace + } +} diff --git a/Postbox/ItemCollectionItemTable.swift b/Postbox/ItemCollectionItemTable.swift index 13ddc31925..6c52c6ee69 100644 --- a/Postbox/ItemCollectionItemTable.swift +++ b/Postbox/ItemCollectionItemTable.swift @@ -1,8 +1,7 @@ import Foundation -enum ItemCollectionOperation { - case insertItem(ItemCollectionId, ItemCollectionItem) - case removeItem(ItemCollectionId, ItemCollectionItemIndex.Id) +enum ItemCollectionItemsOperation { + case replaceItems } struct ItemCollectionItemReverseIndexReference: ReverseIndexReference { @@ -159,7 +158,7 @@ final class ItemCollectionItemTable: Table { func collectionItems(collectionId: ItemCollectionId) -> [ItemCollectionItem] { var items: [ItemCollectionItem] = [] self.valueBox.range(self.table, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId), values: { key, value in - let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) + //let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) if let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { items.append(item) } else { diff --git a/Postbox/PeerOperationLogTable.swift b/Postbox/PeerOperationLogTable.swift index c2fb7a048d..29b9403034 100644 --- a/Postbox/PeerOperationLogTable.swift +++ b/Postbox/PeerOperationLogTable.swift @@ -343,7 +343,7 @@ final class PeerOperationLogTable: Table { assertionFailure() } } else { - f(nil) + let _ = f(nil) } } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 1a94eb0e89..72d5fdfa3d 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -447,6 +447,8 @@ public final class Postbox { private var currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation] = [] private var currentPreferencesOperations: [PreferencesOperation] = [] private var currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]] = [:] + private var currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]] = [:] + private var currentItemCollectionInfosOperations: [ItemCollectionInfosOperation] = [] private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -1077,11 +1079,11 @@ public final class Postbox { return self.peerTable.get(peerId) }, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) - let transaction = PostboxTransaction(currentUpdatedState: self.currentUpdatedState, currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, currentOrderedItemListOperations: self.currentOrderedItemListOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentUpdatedState: self.currentUpdatedState, currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, currentOrderedItemListOperations: self.currentOrderedItemListOperations, currentItemCollectionItemsOperations: self.currentItemCollectionItemsOperations, currentItemCollectionInfosOperations: self.currentItemCollectionInfosOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { - self.viewTracker.updateViews(transaction: transaction) + self.viewTracker.updateViews(postbox: self, transaction: transaction) self.transactionStateVersion = self.metadataTable.incrementTransactionStateVersion() updatedTransactionState = self.transactionStateVersion @@ -1112,6 +1114,8 @@ public final class Postbox { self.currentTimestampBasedMessageAttributesOperations.removeAll() self.currentPreferencesOperations.removeAll() self.currentOrderedItemListOperations.removeAll() + self.currentItemCollectionItemsOperations.removeAll() + self.currentItemCollectionInfosOperations.removeAll() for table in self.tables { table.beforeCommit() @@ -1239,8 +1243,13 @@ public final class Postbox { for (id, info, items) in itemCollections { infos.append(id, info) self.itemCollectionItemTable.replaceItems(collectionId: id, items: items) + if self.currentItemCollectionItemsOperations[id] == nil { + self.currentItemCollectionItemsOperations[id] = [] + } + self.currentItemCollectionItemsOperations[id]!.append(.replaceItems) } self.itemCollectionInfoTable.replaceInfos(namespace: namespace, infos: infos) + self.currentItemCollectionInfosOperations.append(.replaceInfos(namespace)) } fileprivate func filterStoredMessageIds(_ messageIds: Set) -> Set { @@ -1892,6 +1901,27 @@ public final class Postbox { } |> switchToLatest } + public func combinedView(keys: [PostboxViewKey]) -> Signal { + return self.modify { modifier -> Signal in + var views: [PostboxViewKey: MutablePostboxView] = [:] + for key in keys { + views[key] = postboxViewForKey(postbox: self, key: key) + } + let view = CombinedMutableView(views: views) + let (index, signal) = self.viewTracker.addCombinedView(view) + + return (.single(view.immutableView()) + |> then(signal)) + |> afterDisposed { [weak self] in + if let strongSelf = self { + strongSelf.queue.async { + strongSelf.viewTracker.removeCombinedView(index) + } + } + } + } |> switchToLatest + } + fileprivate func getPreferencesEntry(key: ValueBoxKey) -> PreferencesEntry? { return self.preferencesTable.get(key: key) } diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index 3f319555e3..9891efc156 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -17,6 +17,8 @@ final class PostboxTransaction { let currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation] let currentPreferencesOperations: [PreferencesOperation] let currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]] + let currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]] + let currentItemCollectionInfosOperations: [ItemCollectionInfosOperation] let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -88,10 +90,16 @@ final class PostboxTransaction { if !currentOrderedItemListOperations.isEmpty { return false } + if !currentItemCollectionItemsOperations.isEmpty { + return false + } + if !currentItemCollectionInfosOperations.isEmpty { + return false + } return true } - init(currentUpdatedState: Coding?, currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentUpdatedState: Coding?, currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]], currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]], currentItemCollectionInfosOperations: [ItemCollectionInfosOperation], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentUpdatedState = currentUpdatedState self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles @@ -110,6 +118,8 @@ final class PostboxTransaction { self.updatedSynchronizePeerReadStateOperations = updatedSynchronizePeerReadStateOperations self.currentPreferencesOperations = currentPreferencesOperations self.currentOrderedItemListOperations = currentOrderedItemListOperations + self.currentItemCollectionItemsOperations = currentItemCollectionItemsOperations + self.currentItemCollectionInfosOperations = currentItemCollectionInfosOperations self.updatedMedia = updatedMedia self.replaceContactPeerIds = replaceContactPeerIds self.currentUpdatedMasterClientId = currentUpdatedMasterClientId diff --git a/Postbox/PostboxView.swift b/Postbox/PostboxView.swift new file mode 100644 index 0000000000..4ee0772f8b --- /dev/null +++ b/Postbox/PostboxView.swift @@ -0,0 +1,43 @@ +import Foundation + +public protocol PostboxView { +} + +protocol MutablePostboxView { + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool + func immutableView() -> PostboxView +} + +final class CombinedMutableView { + private let views: [PostboxViewKey: MutablePostboxView] + + init(views: [PostboxViewKey: MutablePostboxView]) { + self.views = views + } + + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { + var updated = false + for (_, view) in self.views { + if view.replay(postbox: postbox, transaction: transaction) { + updated = true + } + } + return updated + } + + func immutableView() -> CombinedView { + var result: [PostboxViewKey: PostboxView] = [:] + for (key, view) in self.views { + result[key] = view.immutableView() + } + return CombinedView(views: result) + } +} + +public final class CombinedView { + public let views: [PostboxViewKey: PostboxView] + + init(views: [PostboxViewKey: PostboxView]) { + self.views = views + } +} diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 87f69bc768..d5c7f02bc3 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -119,6 +119,7 @@ final class SqliteValueBox: ValueBox { } deinit { + assert(self.queue.isCurrent()) self.clearStatements() checkpoints.dispose() } @@ -137,7 +138,7 @@ final class SqliteValueBox: ValueBox { database.execute("PRAGMA synchronous=NORMAL") database.execute("PRAGMA journal_mode=WAL") database.execute("PRAGMA temp_store=MEMORY") - //database.execute("PRAGMA wal_autocheckpoint=200") + database.execute("PRAGMA wal_autocheckpoint=400") database.execute("PRAGMA journal_size_limit=1536") /*var statement: OpaquePointer? = nil @@ -167,17 +168,18 @@ final class SqliteValueBox: ValueBox { } lock.unlock() - checkpoints.set((Signal.single(Void()) |> delay(10.0, queue: self.queue) |> restart).start(next: { [weak self] _ in + /*checkpoints.set((Signal.single(Void()) |> delay(10.0, queue: self.queue) |> restart).start(next: { [weak self] _ in if let strongSelf = self, strongSelf.database != nil { assert(strongSelf.queue.isCurrent()) strongSelf.lock.lock() var nLog: Int32 = 0 var nFrames: Int32 = 0 - sqlite3_wal_checkpoint_v2(strongSelf.database.handle, nil, SQLITE_CHECKPOINT_PASSIVE, &nLog, &nFrames) + let result = sqlite3_wal_checkpoint_v2(strongSelf.database.handle, nil, SQLITE_CHECKPOINT_PASSIVE, &nLog, &nFrames) + assert(result == SQLITE_OK) strongSelf.lock.unlock() //print("(SQLite WAL size \(nLog) removed \(nFrames))") } - })) + }))*/ return database } @@ -337,6 +339,7 @@ final class SqliteValueBox: ValueBox { } private func rangeKeyDescStatementLimit(_ table: ValueBoxTable, start: ValueBoxKey, end: ValueBoxKey, limit: Int) -> SqlitePreparedStatement { + assert(self.queue.isCurrent()) let resultStatement: SqlitePreparedStatement checkTableKey(table, start) checkTableKey(table, end) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index a4a2b2b0d6..7c7205db74 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -55,6 +55,8 @@ final class ViewTracker { private let getTimestampBasedMessageAttributesHead: (UInt16) -> TimestampBasedMessageAttributesEntry? private var timestampBasedMessageAttributesViews = Bag<(MutableTimestampBasedMessageAttributesView, ValuePipe)>() + private var combinedViews = Bag<(CombinedMutableView, ValuePipe)>() + private var postboxStateViews = Bag<(MutablePostboxStateView, ValuePipe)>() private var messageViews = Bag<(MutableMessageView, ValuePipe)>() private var preferencesViews = Bag<(MutablePreferencesView, ValuePipe)>() @@ -288,6 +290,17 @@ final class ViewTracker { self.itemCollectionsViews.remove(index) } + func addCombinedView(_ view: CombinedMutableView) -> (Bag<(CombinedMutableView, ValuePipe)>.Index, Signal) { + let record = (view, ValuePipe()) + let index = self.combinedViews.add(record) + + return (index, record.1.signal()) + } + + func removeCombinedView(_ index: Bag<(CombinedMutableView, ValuePipe)>.Index) { + self.combinedViews.remove(index) + } + func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: ChatListIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] @@ -360,7 +373,7 @@ final class ViewTracker { } } - func updateViews(transaction: PostboxTransaction) { + func updateViews(postbox: Postbox, transaction: PostboxTransaction) { var updateTrackedHolesPeerIds: [PeerId] = [] if let currentUpdatedState = transaction.currentUpdatedState { @@ -522,6 +535,12 @@ final class ViewTracker { pipe.putNext(ItemCollectionsView(mutableView)) } } + + for (mutableView, pipe) in self.combinedViews.copyItems() { + if mutableView.replay(postbox: postbox, transaction: transaction) { + pipe.putNext(mutableView.immutableView()) + } + } } private func updateTrackedChatListHoles() { diff --git a/Postbox/Views.swift b/Postbox/Views.swift new file mode 100644 index 0000000000..911bbfcc9c --- /dev/null +++ b/Postbox/Views.swift @@ -0,0 +1,30 @@ +import Foundation + +public enum PostboxViewKey: Hashable { + case itemCollectionInfos(namespaces: [ItemCollectionId.Namespace]) + + public var hashValue: Int { + switch self { + case .itemCollectionInfos: + return 0 + } + } + + public static func ==(lhs: PostboxViewKey, rhs: PostboxViewKey) -> Bool { + switch lhs { + case let .itemCollectionInfos(lhsNamespaces): + if case let .itemCollectionInfos(rhsNamespaces) = rhs, lhsNamespaces == rhsNamespaces { + return true + } else { + return false + } + } + } +} + +func postboxViewForKey(postbox: Postbox, key: PostboxViewKey) -> MutablePostboxView { + switch key { + case let .itemCollectionInfos(namespaces): + return MutableItemCollectionInfosView(postbox: postbox, namespaces: namespaces) + } +} From c31ea00f17baf54b03be702046b4758cd67f75ab Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 15 Mar 2017 13:32:32 +0300 Subject: [PATCH 110/427] no message --- Postbox.xcodeproj/project.pbxproj | 6 ++++++ Postbox/PeerChatStateView.swift | 34 +++++++++++++++++++++++++++++++ Postbox/Postbox.swift | 11 ++++++++-- Postbox/PostboxTransaction.swift | 7 ++++++- Postbox/SqliteValueBox.swift | 2 +- Postbox/Views.swift | 11 ++++++++++ 6 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 Postbox/PeerChatStateView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 299149a99b..9dc6454040 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -198,6 +198,8 @@ D0F019FD1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */; }; D0F3CC721DDE1CDC008148FA /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; + D0F53BF31E794C6700117362 /* PeerChatStateView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F53BF21E794C6700117362 /* PeerChatStateView.swift */; }; + D0F53BF41E794C6700117362 /* PeerChatStateView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F53BF21E794C6700117362 /* PeerChatStateView.swift */; }; D0F7AB321DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */; }; D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */ = {isa = PBXBuildFile; fileRef = D07827C01E0079CB00071108 /* StringIndexTokens.swift */; }; D0F7B1C31E045C6A007EB8A5 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03BCCF71C73561C0097A291 /* Table.swift */; }; @@ -388,6 +390,7 @@ D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerMergedOperationLogView.swift; sourceTree = ""; }; D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheTable.swift; sourceTree = ""; }; D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheMetaTable.swift; sourceTree = ""; }; + D0F53BF21E794C6700117362 /* PeerChatStateView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatStateView.swift; sourceTree = ""; }; D0F7AB311DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatTopIndexableMessageIds.swift; sourceTree = ""; }; D0F82CFC1E4345D7007E499C /* OrderedItemListTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderedItemListTable.swift; sourceTree = ""; }; D0F82D0B1E439FCC007E499C /* OrderedItemListEntry.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderedItemListEntry.swift; sourceTree = ""; }; @@ -638,6 +641,7 @@ D087C20B1E43C11C00D686F8 /* OrderedItemListView.swift */, D0FA0AC91E780A26005BB9B7 /* PostboxView.swift */, D0FA0ACC1E781067005BB9B7 /* Views.swift */, + D0F53BF21E794C6700117362 /* PeerChatStateView.swift */, ); name = Views; sourceTree = ""; @@ -913,6 +917,7 @@ D0B418261D7DFE0C004562A4 /* MappedFile.swift in Sources */, D073CE771DCBF3B4007511FD /* Media.swift in Sources */, D073CE791DCBF3B4007511FD /* ChatListHole.swift in Sources */, + D0F53BF41E794C6700117362 /* PeerChatStateView.swift in Sources */, D0B4181C1D7DFDF1004562A4 /* SQLite-Bridging.m in Sources */, D0F82D0D1E439FCC007E499C /* OrderedItemListEntry.swift in Sources */, D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */, @@ -1032,6 +1037,7 @@ D07CFF811DCA765D00761F81 /* PeerChatInterfaceState.swift in Sources */, D00C7CE31E37861C0080C3D5 /* MessageView.swift in Sources */, D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */, + D0F53BF31E794C6700117362 /* PeerChatStateView.swift in Sources */, D021E0D81DB4FD1300C6B04F /* ItemCollectionItemTable.swift in Sources */, D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, D049EAF01E44D9B900A2CD3A /* PostboxStateView.swift in Sources */, diff --git a/Postbox/PeerChatStateView.swift b/Postbox/PeerChatStateView.swift new file mode 100644 index 0000000000..f3ea668727 --- /dev/null +++ b/Postbox/PeerChatStateView.swift @@ -0,0 +1,34 @@ +import Foundation + +final class MutablePeerChatStateView: MutablePostboxView { + let peerId: PeerId + var chatState: Coding? + + init(postbox: Postbox, peerId: PeerId) { + self.peerId = peerId + self.chatState = postbox.peerChatStateTable.get(peerId) + } + + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { + if transaction.currentUpdatedPeerChatStates.contains(self.peerId) { + self.chatState = postbox.peerChatStateTable.get(self.peerId) + return true + } else { + return false + } + } + + func immutableView() -> PostboxView { + return PeerChatStateView(self) + } +} + +public final class PeerChatStateView: PostboxView { + public let peerId: PeerId + public let chatState: Coding? + + init(_ view: MutablePeerChatStateView) { + self.peerId = view.peerId + self.chatState = view.chatState + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 72d5fdfa3d..c4635c38fa 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -117,7 +117,7 @@ public final class Modifier { } public func setPeerChatState(_ id: PeerId, state: PeerChatState) { - self.postbox?.peerChatStateTable.set(id, state: state) + self.postbox?.setPeerChatState(id, state: state) } public func updatePeerChatInterfaceState(_ id: PeerId, update: (PeerChatInterfaceState?) -> (PeerChatInterfaceState?)) { @@ -449,6 +449,7 @@ public final class Postbox { private var currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]] = [:] private var currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]] = [:] private var currentItemCollectionInfosOperations: [ItemCollectionInfosOperation] = [] + private var currentUpdatedPeerChatStates = Set() private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -1079,7 +1080,7 @@ public final class Postbox { return self.peerTable.get(peerId) }, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) - let transaction = PostboxTransaction(currentUpdatedState: self.currentUpdatedState, currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, currentOrderedItemListOperations: self.currentOrderedItemListOperations, currentItemCollectionItemsOperations: self.currentItemCollectionItemsOperations, currentItemCollectionInfosOperations: self.currentItemCollectionInfosOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentUpdatedState: self.currentUpdatedState, currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, currentOrderedItemListOperations: self.currentOrderedItemListOperations, currentItemCollectionItemsOperations: self.currentItemCollectionItemsOperations, currentItemCollectionInfosOperations: self.currentItemCollectionInfosOperations, currentUpdatedPeerChatStates: self.currentUpdatedPeerChatStates, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -1116,6 +1117,7 @@ public final class Postbox { self.currentOrderedItemListOperations.removeAll() self.currentItemCollectionItemsOperations.removeAll() self.currentItemCollectionInfosOperations.removeAll() + self.currentUpdatedPeerChatStates.removeAll() for table in self.tables { table.beforeCommit() @@ -1205,6 +1207,11 @@ public final class Postbox { } } + fileprivate func setPeerChatState(_ id: PeerId, state: PeerChatState) { + self.peerChatStateTable.set(id, state: state) + self.currentUpdatedPeerChatStates.insert(id) + } + fileprivate func updatePeerChatInterfaceState(_ id: PeerId, update: (PeerChatInterfaceState?) -> (PeerChatInterfaceState?)) { let updatedState = update(self.peerChatInterfaceStateTable.get(id)) let (_, updatedEmbeddedState) = self.peerChatInterfaceStateTable.set(id, state: updatedState) diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index 9891efc156..8c1e1e74df 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -19,6 +19,7 @@ final class PostboxTransaction { let currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]] let currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]] let currentItemCollectionInfosOperations: [ItemCollectionInfosOperation] + let currentUpdatedPeerChatStates: Set let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -96,10 +97,13 @@ final class PostboxTransaction { if !currentItemCollectionInfosOperations.isEmpty { return false } + if !currentUpdatedPeerChatStates.isEmpty { + return false + } return true } - init(currentUpdatedState: Coding?, currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]], currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]], currentItemCollectionInfosOperations: [ItemCollectionInfosOperation], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentUpdatedState: Coding?, currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]], currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]], currentItemCollectionInfosOperations: [ItemCollectionInfosOperation], currentUpdatedPeerChatStates: Set, updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentUpdatedState = currentUpdatedState self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles @@ -120,6 +124,7 @@ final class PostboxTransaction { self.currentOrderedItemListOperations = currentOrderedItemListOperations self.currentItemCollectionItemsOperations = currentItemCollectionItemsOperations self.currentItemCollectionInfosOperations = currentItemCollectionInfosOperations + self.currentUpdatedPeerChatStates = currentUpdatedPeerChatStates self.updatedMedia = updatedMedia self.replaceContactPeerIds = replaceContactPeerIds self.currentUpdatedMasterClientId = currentUpdatedMasterClientId diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index d5c7f02bc3..cb8fead940 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -138,7 +138,7 @@ final class SqliteValueBox: ValueBox { database.execute("PRAGMA synchronous=NORMAL") database.execute("PRAGMA journal_mode=WAL") database.execute("PRAGMA temp_store=MEMORY") - database.execute("PRAGMA wal_autocheckpoint=400") + database.execute("PRAGMA wal_autocheckpoint=500") database.execute("PRAGMA journal_size_limit=1536") /*var statement: OpaquePointer? = nil diff --git a/Postbox/Views.swift b/Postbox/Views.swift index 911bbfcc9c..a51089ecbd 100644 --- a/Postbox/Views.swift +++ b/Postbox/Views.swift @@ -2,11 +2,14 @@ import Foundation public enum PostboxViewKey: Hashable { case itemCollectionInfos(namespaces: [ItemCollectionId.Namespace]) + case peerChatState(peerId: PeerId) public var hashValue: Int { switch self { case .itemCollectionInfos: return 0 + case let .peerChatState(peerId): + return peerId.hashValue } } @@ -18,6 +21,12 @@ public enum PostboxViewKey: Hashable { } else { return false } + case let .peerChatState(peerId): + if case .peerChatState(peerId) = rhs { + return true + } else { + return false + } } } } @@ -26,5 +35,7 @@ func postboxViewForKey(postbox: Postbox, key: PostboxViewKey) -> MutablePostboxV switch key { case let .itemCollectionInfos(namespaces): return MutableItemCollectionInfosView(postbox: postbox, namespaces: namespaces) + case let .peerChatState(peerId): + return MutablePeerChatStateView(postbox: postbox, peerId: peerId) } } From 1ce4c9f832101d5e0a991b0be245469f36a057ee Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 15 Mar 2017 17:29:26 +0300 Subject: [PATCH 111/427] no message --- Postbox/MessageHistoryTable.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index dcf36a9908..26bc3e9b2f 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -219,7 +219,10 @@ final class MessageHistoryTable: Table { outputOperations.append(.InsertMessage(message)) if message.flags.contains(.Incoming) { - accumulatedAddedIncomingMessageIndices.insert(MessageIndex(message)) + if index != MessageIndex(message) { + accumulatedRemoveIndices.append(index) + accumulatedAddedIncomingMessageIndices.insert(MessageIndex(message)) + } } } case let .UpdateTimestamp(index, timestamp): From f20a270fcac8407da097264ef10f1da9ecf4dae2 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 16 Mar 2017 00:22:34 +0300 Subject: [PATCH 112/427] no message --- Postbox/ItemCollection.swift | 29 +++++++++++++++++++++++++++ Postbox/ItemCollectionInfosView.swift | 10 +++++---- Postbox/ItemCollectionItemTable.swift | 10 +++++++++ Postbox/Peer.swift | 2 -- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/Postbox/ItemCollection.swift b/Postbox/ItemCollection.swift index b2f5673e58..fc05a634cb 100644 --- a/Postbox/ItemCollection.swift +++ b/Postbox/ItemCollection.swift @@ -27,6 +27,35 @@ public struct ItemCollectionId: Comparable, Hashable { public var hashValue: Int { return self.id.hashValue } + + public static func encodeArrayToBuffer(_ array: [ItemCollectionId], buffer: WriteBuffer) { + var length: Int32 = Int32(array.count) + buffer.write(&length, offset: 0, length: 4) + for id in array { + var idNamespace = id.namespace + buffer.write(&idNamespace, offset: 0, length: 4) + var idId = id.id + buffer.write(&idId, offset: 0, length: 8) + } + } + + public static func decodeArrayFromBuffer(_ buffer: ReadBuffer) -> [ItemCollectionId] { + var length: Int32 = 0 + memcpy(&length, buffer.memory, 4) + buffer.offset += 4 + var i = 0 + var array: [ItemCollectionId] = [] + array.reserveCapacity(Int(length)) + while i < Int(length) { + var idNamespace: Int32 = 0 + buffer.read(&idNamespace, offset: 0, length: 4) + var idId: Int64 = 0 + buffer.read(&idId, offset: 0, length: 8) + array.append(ItemCollectionId(namespace: idNamespace, id: idId)) + i += 1 + } + return array + } } public protocol ItemCollectionInfo: Coding { diff --git a/Postbox/ItemCollectionInfosView.swift b/Postbox/ItemCollectionInfosView.swift index b9ff98ae4b..922440425f 100644 --- a/Postbox/ItemCollectionInfosView.swift +++ b/Postbox/ItemCollectionInfosView.swift @@ -3,11 +3,13 @@ import Foundation public final class ItemCollectionInfoEntry { public let id: ItemCollectionId public let info: ItemCollectionInfo + public let count: Int32 public let firstItem: ItemCollectionItem? - init(id: ItemCollectionId, info: ItemCollectionInfo, firstItem: ItemCollectionItem?) { + init(id: ItemCollectionId, info: ItemCollectionInfo, count: Int32, firstItem: ItemCollectionItem?) { self.id = id self.info = info + self.count = count self.firstItem = firstItem } } @@ -25,7 +27,7 @@ final class MutableItemCollectionInfosView: MutablePostboxView { var entries: [ItemCollectionInfoEntry] = [] for (_, id, info) in infos { let firstItem = postbox.itemCollectionItemTable.higherItems(collectionId: id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first - entries.append(ItemCollectionInfoEntry(id: id, info: info, firstItem: firstItem)) + entries.append(ItemCollectionInfoEntry(id: id, info: info, count: postbox.itemCollectionItemTable.itemCount(collectionId: id), firstItem: firstItem)) } entriesByNamespace[namespace] = entries } @@ -64,7 +66,7 @@ final class MutableItemCollectionInfosView: MutablePostboxView { var entries: [ItemCollectionInfoEntry] = [] for (_, id, info) in infos { let firstItem = postbox.itemCollectionItemTable.higherItems(collectionId: id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first - entries.append(ItemCollectionInfoEntry(id: id, info: info, firstItem: firstItem)) + entries.append(ItemCollectionInfoEntry(id: id, info: info, count: postbox.itemCollectionItemTable.itemCount(collectionId: id), firstItem: firstItem)) } entriesByNamespace[namespace] = entries } @@ -75,7 +77,7 @@ final class MutableItemCollectionInfosView: MutablePostboxView { if reloadTopItemCollectionIds.contains(entries[i].id) { updated = true let firstItem = postbox.itemCollectionItemTable.higherItems(collectionId: entries[i].id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first - self.entriesByNamespace[namespace]![i] = ItemCollectionInfoEntry(id: entries[i].id, info: entries[i].info, firstItem: firstItem) + self.entriesByNamespace[namespace]![i] = ItemCollectionInfoEntry(id: entries[i].id, info: entries[i].info, count: postbox.itemCollectionItemTable.itemCount(collectionId: entries[i].id), firstItem: firstItem) } } } diff --git a/Postbox/ItemCollectionItemTable.swift b/Postbox/ItemCollectionItemTable.swift index 6c52c6ee69..416e086943 100644 --- a/Postbox/ItemCollectionItemTable.swift +++ b/Postbox/ItemCollectionItemTable.swift @@ -123,6 +123,16 @@ final class ItemCollectionItemTable: Table { return items } + func itemCount(collectionId: ItemCollectionId) -> Int32 { + var count: Int32 = 0 + self.valueBox.range(self.table, start: self.upperBound(collectionId: collectionId), end: self.lowerBound(collectionId: collectionId), keys: { key in + let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4)) + count = itemIndex.index + 1 + return true + }, limit: 1) + return count + } + func higherItems(collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] { var items: [ItemCollectionItem] = [] self.valueBox.range(self.table, start: self.key(collectionId: collectionId, index: itemIndex), end: self.upperBound(collectionId: collectionId), values: { _, value in diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index 44dc53fef6..90bc134dc8 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -18,8 +18,6 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { } public func toInt64() -> Int64 { - - return (Int64(self.namespace) << 32) | unsafeBitCast(UInt64(unsafeBitCast(self.id, to: UInt32.self)), to: Int64.self) } From 370f76636a85b5186ce36a8589867c7e3c257896 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 16 Mar 2017 10:58:00 +0300 Subject: [PATCH 113/427] no message --- Postbox/AccountManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Postbox/AccountManager.swift b/Postbox/AccountManager.swift index e24379cb5d..7ab16482e0 100644 --- a/Postbox/AccountManager.swift +++ b/Postbox/AccountManager.swift @@ -43,7 +43,7 @@ public final class AccountManager { public func modify(_ f: @escaping (AccountManagerModifier) -> T) -> Signal { return Signal { subscriber in - self.queue.async { + self.queue.justDispatch { self.valueBox.begin() let modifier = AccountManagerModifier(getRecords: { From c2192b598ca99fee6fec567b2886fc9afa0ddced Mon Sep 17 00:00:00 2001 From: Peter Date: Sat, 18 Mar 2017 17:57:52 +0300 Subject: [PATCH 114/427] no message --- Postbox.xcodeproj/project.pbxproj | 6 +++ Postbox/ItemCollectionInfoTable.swift | 17 +++++++ Postbox/ItemCollectionInfoView.swift | 65 +++++++++++++++++++++++++++ Postbox/Postbox.swift | 45 +++++++++++++++++++ Postbox/Views.swift | 11 +++++ 5 files changed, 144 insertions(+) create mode 100644 Postbox/ItemCollectionInfoView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 9dc6454040..78c0f51ebb 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -157,6 +157,8 @@ D0BC386C1E3FCEE50044D6FE /* MultiplePeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */; }; D0BC38721E409E670044D6FE /* RenderedPeer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC38711E409E670044D6FE /* RenderedPeer.swift */; }; D0BC38731E409E670044D6FE /* RenderedPeer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC38711E409E670044D6FE /* RenderedPeer.swift */; }; + D0BE38391E7C1FD4000079AF /* ItemCollectionInfoView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BE38381E7C1FD4000079AF /* ItemCollectionInfoView.swift */; }; + D0BE383A1E7C1FD4000079AF /* ItemCollectionInfoView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BE38381E7C1FD4000079AF /* ItemCollectionInfoView.swift */; }; D0BEAF631E54B2FA00BD963D /* AccountManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF621E54B2FA00BD963D /* AccountManager.swift */; }; D0BEAF641E54B2FA00BD963D /* AccountManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF621E54B2FA00BD963D /* AccountManager.swift */; }; D0BEAF671E54B33900BD963D /* AccountRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF661E54B33900BD963D /* AccountRecord.swift */; }; @@ -353,6 +355,7 @@ D0B844501DAC04FE005F29E1 /* PeerPresence.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerPresence.swift; sourceTree = ""; }; D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MultiplePeersView.swift; sourceTree = ""; }; D0BC38711E409E670044D6FE /* RenderedPeer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RenderedPeer.swift; sourceTree = ""; }; + D0BE38381E7C1FD4000079AF /* ItemCollectionInfoView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollectionInfoView.swift; sourceTree = ""; }; D0BEAF621E54B2FA00BD963D /* AccountManager.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountManager.swift; sourceTree = ""; }; D0BEAF661E54B33900BD963D /* AccountRecord.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountRecord.swift; sourceTree = ""; }; D0BEAF691E54B5FB00BD963D /* AccountManagerMetadataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountManagerMetadataTable.swift; sourceTree = ""; }; @@ -632,6 +635,7 @@ D0DF0C8E1D81A350008AEB01 /* PeerView.swift */, D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */, D0FA0AC61E77F0A2005BB9B7 /* ItemCollectionInfosView.swift */, + D0BE38381E7C1FD4000079AF /* ItemCollectionInfoView.swift */, D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */, D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */, D0AAD1B41E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift */, @@ -949,6 +953,7 @@ D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */, D0F7B1D91E045C6A007EB8A5 /* OrderStatisticTable.swift in Sources */, D073CEA01DCBF3C1007511FD /* ItemCollectionsView.swift in Sources */, + D0BE383A1E7C1FD4000079AF /* ItemCollectionInfoView.swift in Sources */, D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */, D0F7B1D61E045C6A007EB8A5 /* ContactTable.swift in Sources */, D0F7B1D11E045C6A007EB8A5 /* PeerTable.swift in Sources */, @@ -1069,6 +1074,7 @@ D0AD23271E194D1C00A7089A /* PeerOperationLogTable.swift in Sources */, D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */, D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */, + D0BE38391E7C1FD4000079AF /* ItemCollectionInfoView.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, D0D949F51D35353900740E02 /* MappedFile.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, diff --git a/Postbox/ItemCollectionInfoTable.swift b/Postbox/ItemCollectionInfoTable.swift index 74272657d4..07ad8f124b 100644 --- a/Postbox/ItemCollectionInfoTable.swift +++ b/Postbox/ItemCollectionInfoTable.swift @@ -49,6 +49,23 @@ final class ItemCollectionInfoTable: Table { } } + func getInfo(id: ItemCollectionId) -> ItemCollectionInfo? { + var infoKey: ValueBoxKey? + self.valueBox.range(self.table, start: self.lowerBound(namespace: id.namespace), end: self.upperBound(namespace: id.namespace), keys: { key in + let keyId = ItemCollectionId(namespace: id.namespace, id: key.getInt64(4 + 4)) + if keyId == id { + infoKey = key + return false + } + return true + }, limit: 0) + if let infoKey = infoKey, let value = self.valueBox.get(self.table, key: infoKey), let info = Decoder(buffer: value).decodeRootObject() as? ItemCollectionInfo { + return info + } else { + return nil + } + } + func lowerCollectionId(namespaceList: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, index: Int32) -> (ItemCollectionId, Int32)? { var currentNamespace = collectionId.namespace var currentKey = self.key(collectionId: collectionId, index: index) diff --git a/Postbox/ItemCollectionInfoView.swift b/Postbox/ItemCollectionInfoView.swift new file mode 100644 index 0000000000..63f70b0ae4 --- /dev/null +++ b/Postbox/ItemCollectionInfoView.swift @@ -0,0 +1,65 @@ +import Foundation + +final class MutableItemCollectionInfoView: MutablePostboxView { + let id: ItemCollectionId + var info: ItemCollectionInfo? + + init(postbox: Postbox, id: ItemCollectionId) { + self.id = id + let infos = postbox.itemCollectionInfoTable.getInfos(namespace: id.namespace) + for (_, infoId, info) in infos { + if id == infoId { + self.info = info + break + } + } + } + + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { + if transaction.currentItemCollectionInfosOperations.isEmpty { + return false + } + + var updated = false + + var reloadInfosNamespaces = Set() + for operation in transaction.currentItemCollectionInfosOperations { + switch operation { + case let .replaceInfos(namespace): + reloadInfosNamespaces.insert(namespace) + } + } + + if !reloadInfosNamespaces.isEmpty && reloadInfosNamespaces.contains(self.id.namespace) { + updated = true + + let infos = postbox.itemCollectionInfoTable.getInfos(namespace: id.namespace) + var found = false + for (_, infoId, info) in infos { + if id == infoId { + self.info = info + found = true + break + } + } + if !found { + self.info = nil + } + } + return updated + } + + func immutableView() -> PostboxView { + return ItemCollectionInfoView(self) + } +} + +public final class ItemCollectionInfoView: PostboxView { + public let id: ItemCollectionId + public let info: ItemCollectionInfo? + + init(_ view: MutableItemCollectionInfoView) { + self.id = view.id + self.info = view.info + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index c4635c38fa..446dc0e784 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -211,6 +211,30 @@ public final class Modifier { self.postbox?.replaceItemCollections(namespace: namespace, itemCollections: itemCollections) } + public func replaceItemCollectionInfos(namespace: ItemCollectionId.Namespace, itemCollectionInfos: [(ItemCollectionId, ItemCollectionInfo)]) { + self.postbox?.replaceItemCollectionInfos(namespace: namespace, itemCollectionInfos: itemCollectionInfos) + } + + public func replaceItemCollectionItems(collectionId: ItemCollectionId, items: [ItemCollectionItem]) { + self.postbox?.replaceItemCollectionItems(collectionId: collectionId, items: items) + } + + public func removeItemCollection(collectionId: ItemCollectionId) { + self.postbox?.removeItemCollection(collectionId: collectionId) + } + + public func getItemCollectionsInfos(namespace: ItemCollectionId.Namespace) -> [(ItemCollectionId, ItemCollectionInfo)] { + if let postbox = self.postbox { + return postbox.itemCollectionInfoTable.getInfos(namespace: namespace).map { ($0.1, $0.2) } + } else { + return [] + } + } + + public func getItemCollectionInfo(collectionId: ItemCollectionId) -> ItemCollectionInfo? { + return self.postbox?.itemCollectionInfoTable.getInfo(id: collectionId) + } + public func getItemCollectionItems(collectionId: ItemCollectionId) -> [ItemCollectionItem] { if let postbox = self.postbox { return postbox.itemCollectionItemTable.collectionItems(collectionId: collectionId) @@ -1259,6 +1283,27 @@ public final class Postbox { self.currentItemCollectionInfosOperations.append(.replaceInfos(namespace)) } + fileprivate func replaceItemCollectionInfos(namespace: ItemCollectionId.Namespace, itemCollectionInfos: [(ItemCollectionId, ItemCollectionInfo)]) { + self.itemCollectionInfoTable.replaceInfos(namespace: namespace, infos: itemCollectionInfos) + self.currentItemCollectionInfosOperations.append(.replaceInfos(namespace)) + } + + fileprivate func replaceItemCollectionItems(collectionId: ItemCollectionId, items: [ItemCollectionItem]) { + self.itemCollectionItemTable.replaceItems(collectionId: collectionId, items: items) + if self.currentItemCollectionItemsOperations[collectionId] == nil { + self.currentItemCollectionItemsOperations[collectionId] = [] + } + self.currentItemCollectionItemsOperations[collectionId]!.append(.replaceItems) + } + + fileprivate func removeItemCollection(collectionId: ItemCollectionId) { + var infos = self.itemCollectionInfoTable.getInfos(namespace: collectionId.namespace) + if let index = infos.index(where: { $0.1 == collectionId }) { + infos.remove(at: index) + self.replaceItemCollectionInfos(namespace: collectionId.namespace, itemCollectionInfos: infos.map { ($0.1, $0.2) }) + } + } + fileprivate func filterStoredMessageIds(_ messageIds: Set) -> Set { var filteredIds = Set() diff --git a/Postbox/Views.swift b/Postbox/Views.swift index a51089ecbd..1f811d2374 100644 --- a/Postbox/Views.swift +++ b/Postbox/Views.swift @@ -2,6 +2,7 @@ import Foundation public enum PostboxViewKey: Hashable { case itemCollectionInfos(namespaces: [ItemCollectionId.Namespace]) + case itemCollectionInfo(id: ItemCollectionId) case peerChatState(peerId: PeerId) public var hashValue: Int { @@ -10,6 +11,8 @@ public enum PostboxViewKey: Hashable { return 0 case let .peerChatState(peerId): return peerId.hashValue + case let .itemCollectionInfo(id): + return id.hashValue } } @@ -21,6 +24,12 @@ public enum PostboxViewKey: Hashable { } else { return false } + case let .itemCollectionInfo(id): + if case .itemCollectionInfo(id) = rhs { + return true + } else { + return false + } case let .peerChatState(peerId): if case .peerChatState(peerId) = rhs { return true @@ -35,6 +44,8 @@ func postboxViewForKey(postbox: Postbox, key: PostboxViewKey) -> MutablePostboxV switch key { case let .itemCollectionInfos(namespaces): return MutableItemCollectionInfosView(postbox: postbox, namespaces: namespaces) + case let .itemCollectionInfo(id): + return MutableItemCollectionInfoView(postbox: postbox, id: id) case let .peerChatState(peerId): return MutablePeerChatStateView(postbox: postbox, peerId: peerId) } From 3bc78cb58d494580bef91f27d315f09d47bfb937 Mon Sep 17 00:00:00 2001 From: overtake Date: Sat, 18 Mar 2017 17:59:16 +0300 Subject: [PATCH 115/427] no message --- Postbox/Postbox.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index c4635c38fa..a86a674446 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -155,6 +155,10 @@ public final class Modifier { return self.postbox?.messageHistoryIndexTable.top(peerId, namespace: namespace)?.index.id } + public func getTopPeerMessageIndex(peerId: PeerId, namespace: MessageId.Namespace) -> MessageIndex? { + return self.postbox?.messageHistoryIndexTable.top(peerId, namespace: namespace)?.index + } + public func updatePeerChatListInclusion(_ id: PeerId, inclusion: PeerChatListInclusion) { self.postbox?.updatePeerChatListInclusion(id, inclusion: inclusion) } From 7b231edd2f03b67bce3a71c70be085b22aadc593 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 20 Mar 2017 18:37:15 +0300 Subject: [PATCH 116/427] no message --- Postbox/ItemCollectionsView.swift | 121 ++++++++++++++++++++++++++++-- Postbox/Postbox.swift | 25 +++--- Postbox/ViewTracker.swift | 2 +- 3 files changed, 127 insertions(+), 21 deletions(-) diff --git a/Postbox/ItemCollectionsView.swift b/Postbox/ItemCollectionsView.swift index 7b8157be81..437e810507 100644 --- a/Postbox/ItemCollectionsView.swift +++ b/Postbox/ItemCollectionsView.swift @@ -156,13 +156,85 @@ private func aroundEntries(namespaces: [ItemCollectionId.Namespace], final class MutableItemCollectionsView { let orderedItemListsViews: [MutableOrderedItemListView] let namespaces: [ItemCollectionId.Namespace] + let requestedAroundIndex: ItemCollectionViewEntryIndex? + let requestedCount: Int var collectionInfos: [(ItemCollectionId, ItemCollectionInfo, ItemCollectionItem?)] var entries: [ItemCollectionViewEntry] var lower: ItemCollectionViewEntry? var higher: ItemCollectionViewEntry? - init(orderedItemListsViews: [MutableOrderedItemListView], namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int, getInfos: (_ namespace: ItemCollectionId.Namespace) -> [(Int, ItemCollectionId, ItemCollectionInfo)], lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, lowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem], higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, higherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) { + init(postbox: Postbox, orderedItemListsViews: [MutableOrderedItemListView], namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int) { + self.orderedItemListsViews = orderedItemListsViews + self.namespaces = namespaces + self.requestedAroundIndex = aroundIndex + self.requestedCount = count + + self.collectionInfos = [] + self.entries = [] + self.lower = nil + self.higher = nil + + self.reload(postbox: postbox, aroundIndex: aroundIndex, count: count) + } + + private func lowerItems(postbox: Postbox, collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] { + return postbox.itemCollectionItemTable.lowerItems(collectionId: collectionId, itemIndex: itemIndex, count: count) + } + + private func higherItems(postbox: Postbox, collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] { + return postbox.itemCollectionItemTable.higherItems(collectionId: collectionId, itemIndex: itemIndex, count: count) + } + + private func lowerCollectionId(postbox: Postbox, namespaceList: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, collectionIndex: Int32) -> (ItemCollectionId, Int32)? { + return postbox.itemCollectionInfoTable.lowerCollectionId(namespaceList: namespaceList, collectionId: collectionId, index: collectionIndex) + } + + private func higherCollectionId(postbox: Postbox, namespaceList: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, collectionIndex: Int32) -> (ItemCollectionId, Int32)? { + return postbox.itemCollectionInfoTable.higherCollectionId(namespaceList: namespaceList, collectionId: collectionId, index: collectionIndex) + } + + private func reload(postbox: Postbox, aroundIndex: ItemCollectionViewEntryIndex?, count: Int) { + self.collectionInfos = [] + for namespace in namespaces { + for (_, id, info) in postbox.itemCollectionInfoTable.getInfos(namespace: namespace) { + let item = self.higherItems(postbox: postbox, collectionId: id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first + self.collectionInfos.append((id, info, item)) + } + } + + let selectedAroundIndex: ItemCollectionViewEntryIndex + if let aroundIndex = aroundIndex { + selectedAroundIndex = aroundIndex + } else { + selectedAroundIndex = ItemCollectionViewEntryIndex(collectionIndex: 0, collectionId: ItemCollectionId(namespace: namespaces[0], id: 0), itemIndex: ItemCollectionItemIndex.lowerBound) + } + + let (entries, lower, higher) = aroundEntries(namespaces: namespaces, + collectionId: selectedAroundIndex.collectionId, + collectionIndex: selectedAroundIndex.collectionIndex, + itemIndex: selectedAroundIndex.itemIndex, + count: count, + lowerCollectionId: { namespaceList, collectionId, collectionIndex in + return self.lowerCollectionId(postbox: postbox, namespaceList: namespaceList, collectionId: collectionId, collectionIndex: collectionIndex) + }, + fetchLowerItems: { collectionId, itemIndex, count in + return self.lowerItems(postbox: postbox, collectionId: collectionId, itemIndex: itemIndex, count: count) + }, + higherCollectionId: { namespaceList, collectionId, collectionIndex in + return self.higherCollectionId(postbox: postbox, namespaceList: namespaceList, collectionId: collectionId, collectionIndex: collectionIndex) + }, + fetchHigherItems: { + collectionId, itemIndex, count in + return self.higherItems(postbox: postbox, collectionId: collectionId, itemIndex: itemIndex, count: count) + }) + + self.entries = entries + self.lower = lower + self.higher = higher + } + + /*init(orderedItemListsViews: [MutableOrderedItemListView], namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int, getInfos: (_ namespace: ItemCollectionId.Namespace) -> [(Int, ItemCollectionId, ItemCollectionInfo)], lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, lowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem], higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, higherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) { self.orderedItemListsViews = orderedItemListsViews self.namespaces = namespaces @@ -194,15 +266,52 @@ final class MutableItemCollectionsView { self.entries = entries self.lower = lower self.higher = higher - } + }*/ - func replay(orderedItemListOperations: [Int32: [OrderedItemListOperation]]) -> Bool { + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { var updated = false - for view in self.orderedItemListsViews { - if view.replay(operations: orderedItemListOperations) { - updated = true + + if !transaction.currentOrderedItemListOperations.isEmpty { + for view in self.orderedItemListsViews { + if view.replay(operations: transaction.currentOrderedItemListOperations) { + updated = true + } } } + + + var reloadNamespaces = Set() + for operation in transaction.currentItemCollectionInfosOperations { + switch operation { + case let .replaceInfos(namespace): + reloadNamespaces.insert(namespace) + } + } + + for (id, operations) in transaction.currentItemCollectionItemsOperations { + for operation in operations { + switch operation { + case .replaceItems: + reloadNamespaces.insert(id.namespace) + } + } + } + + var shouldReloadEntries = false + if !reloadNamespaces.isEmpty { + for namespace in self.namespaces { + if reloadNamespaces.contains(namespace) { + shouldReloadEntries = true + break + } + } + } + + if shouldReloadEntries { + self.reload(postbox: postbox, aroundIndex: self.requestedAroundIndex, count: self.requestedCount) + updated = true + } + return updated } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 446dc0e784..0df11db2f6 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -15,6 +15,11 @@ public protocol PeerChatState: Coding { func equals(_ other: PeerChatState) -> Bool } +public enum PostboxUpdateMessage { + case update(StoreMessage) + case skip +} + public final class Modifier { private weak var postbox: Postbox? @@ -195,7 +200,7 @@ public final class Modifier { self.postbox?.replaceRecentPeerIds(peerIds) } - public func updateMessage(_ id: MessageId, update: (Message) -> StoreMessage) { + public func updateMessage(_ id: MessageId, update: (Message) -> PostboxUpdateMessage) { self.postbox?.updateMessage(id, update: update) } @@ -1254,10 +1259,12 @@ public final class Postbox { self.peerRatingTable.replace(items: peerIds) } - fileprivate func updateMessage(_ id: MessageId, update: (Message) -> StoreMessage) { + fileprivate func updateMessage(_ id: MessageId, update: (Message) -> PostboxUpdateMessage) { if let indexEntry = self.messageHistoryIndexTable.get(id), let intermediateMessage = self.messageHistoryTable.getMessage(indexEntry.index) { let message = self.renderIntermediateMessage(intermediateMessage) - self.messageHistoryTable.updateMessage(id, message: update(message), operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + if case let .update(updatedMessage) = update(message) { + self.messageHistoryTable.updateMessage(id, message: updatedMessage, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + } } } @@ -1795,17 +1802,7 @@ public final class Postbox { }) } - let mutableView = MutableItemCollectionsView(orderedItemListsViews: itemListViews, namespaces: namespaces, aroundIndex: aroundIndex, count: count, getInfos: { namespace in - return self.itemCollectionInfoTable.getInfos(namespace: namespace) - }, lowerCollectionId: { namespaceList, collectionId, collectionIndex in - return self.itemCollectionInfoTable.lowerCollectionId(namespaceList: namespaceList, collectionId: collectionId, index: collectionIndex) - }, lowerItems: { collectionId, itemIndex, count in - return self.itemCollectionItemTable.lowerItems(collectionId: collectionId, itemIndex: itemIndex, count: count) - }, higherCollectionId: { namespaceList, collectionId, collectionIndex in - return self.itemCollectionInfoTable.higherCollectionId(namespaceList: namespaceList, collectionId: collectionId, index: collectionIndex) - }, higherItems: { collectionId, itemIndex, count in - return self.itemCollectionItemTable.higherItems(collectionId: collectionId, itemIndex: itemIndex, count: count) - }) + let mutableView = MutableItemCollectionsView(postbox: self, orderedItemListsViews: itemListViews, namespaces: namespaces, aroundIndex: aroundIndex, count: count) let (index, signal) = self.viewTracker.addItemCollectionView(mutableView) diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 7c7205db74..5a71adc859 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -531,7 +531,7 @@ final class ViewTracker { } for (mutableView, pipe) in self.itemCollectionsViews.copyItems() { - if mutableView.replay(orderedItemListOperations: transaction.currentOrderedItemListOperations) { + if mutableView.replay(postbox: postbox, transaction: transaction) { pipe.putNext(ItemCollectionsView(mutableView)) } } From 0b7bf07482ae0b1e5f19ffbbadad4cf30dbf1ac0 Mon Sep 17 00:00:00 2001 From: overtake Date: Tue, 21 Mar 2017 14:35:39 +0300 Subject: [PATCH 117/427] no message --- Postbox/Postbox.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 8ee7381b24..da40ea8c17 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -1313,6 +1313,7 @@ public final class Postbox { infos.remove(at: index) self.replaceItemCollectionInfos(namespace: collectionId.namespace, itemCollectionInfos: infos.map { ($0.1, $0.2) }) } + self.replaceItemCollectionItems(collectionId: collectionId, items: []) } fileprivate func filterStoredMessageIds(_ messageIds: Set) -> Set { From 4b1fb83ae760302039e3c3b6190967ddce1b26c5 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 21 Mar 2017 14:37:01 +0300 Subject: [PATCH 118/427] no message --- Postbox.xcodeproj/project.pbxproj | 6 ++ Postbox/ItemCollectionIdsView.swift | 55 +++++++++++++ Postbox/ItemCollectionInfoTable.swift | 26 ++++++ Postbox/ItemCollectionsView.swift | 67 ++++----------- Postbox/OrderedItemListView.swift | 16 ++-- Postbox/Postbox.swift | 114 ++++++++++++++++++++------ Postbox/ViewTracker.swift | 18 ---- Postbox/Views.swift | 22 +++++ 8 files changed, 227 insertions(+), 97 deletions(-) create mode 100644 Postbox/ItemCollectionIdsView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 78c0f51ebb..ab2d6f7cac 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -190,6 +190,8 @@ D0DA44491E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */; }; D0DF0C8F1D81A350008AEB01 /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DF0C8E1D81A350008AEB01 /* PeerView.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; + D0E23DE21E808A9400B9B6D2 /* ItemCollectionIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E23DE11E808A9400B9B6D2 /* ItemCollectionIdsView.swift */; }; + D0E23DE31E808A9400B9B6D2 /* ItemCollectionIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E23DE11E808A9400B9B6D2 /* ItemCollectionIdsView.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7811B28ADD000A402D9 /* Postbox.swift */; }; @@ -380,6 +382,7 @@ D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxAccess.swift; sourceTree = ""; }; D0DF0C8E1D81A350008AEB01 /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; + D0E23DE11E808A9400B9B6D2 /* ItemCollectionIdsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollectionIdsView.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; D0E3A74F1B28A7E300A402D9 /* Postbox.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Postbox.h; sourceTree = ""; }; @@ -635,6 +638,7 @@ D0DF0C8E1D81A350008AEB01 /* PeerView.swift */, D021E0DB1DB5237C00C6B04F /* ItemCollectionsView.swift */, D0FA0AC61E77F0A2005BB9B7 /* ItemCollectionInfosView.swift */, + D0E23DE11E808A9400B9B6D2 /* ItemCollectionIdsView.swift */, D0BE38381E7C1FD4000079AF /* ItemCollectionInfoView.swift */, D0A18D661E16874D004C6734 /* UnreadMessageCountsView.swift */, D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */, @@ -926,6 +930,7 @@ D0F82D0D1E439FCC007E499C /* OrderedItemListEntry.swift in Sources */, D0B418591D7DFE29004562A4 /* PostboxTransaction.swift in Sources */, D0F7B1D21E045C6A007EB8A5 /* PeerNotificationSettingsTable.swift in Sources */, + D0E23DE31E808A9400B9B6D2 /* ItemCollectionIdsView.swift in Sources */, D0B4181D1D7DFDF4004562A4 /* sqlite3.c in Sources */, D0F7B1C91E045C6A007EB8A5 /* MessageHistoryTagsTable.swift in Sources */, D0F7B1CE1E045C6A007EB8A5 /* MediaCleanupTable.swift in Sources */, @@ -1047,6 +1052,7 @@ D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, D049EAF01E44D9B900A2CD3A /* PostboxStateView.swift in Sources */, D0079F631D5A242500A27A2C /* ContactPeerIdsView.swift in Sources */, + D0E23DE21E808A9400B9B6D2 /* ItemCollectionIdsView.swift in Sources */, D0AD23291E196B6400A7089A /* PeerOperationLogMetadataTable.swift in Sources */, D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, diff --git a/Postbox/ItemCollectionIdsView.swift b/Postbox/ItemCollectionIdsView.swift new file mode 100644 index 0000000000..cde4e4b6d8 --- /dev/null +++ b/Postbox/ItemCollectionIdsView.swift @@ -0,0 +1,55 @@ +import Foundation + +final class MutableItemCollectionIdsView: MutablePostboxView { + let namespaces: [ItemCollectionId.Namespace] + var idsByNamespace: [ItemCollectionId.Namespace: Set] + + init(postbox: Postbox, namespaces: [ItemCollectionId.Namespace]) { + self.namespaces = namespaces + + var idsByNamespace: [ItemCollectionId.Namespace: Set] = [:] + for namespace in namespaces { + let ids = postbox.itemCollectionInfoTable.getIds(namespace: namespace) + idsByNamespace[namespace] = Set(ids) + } + self.idsByNamespace = idsByNamespace + } + + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { + if transaction.currentItemCollectionInfosOperations.isEmpty { + return false + } + + var updated = false + + var reloadInfosNamespaces = Set() + for operation in transaction.currentItemCollectionInfosOperations { + switch operation { + case let .replaceInfos(namespace): + reloadInfosNamespaces.insert(namespace) + } + } + if !reloadInfosNamespaces.isEmpty { + for namespace in self.namespaces { + if reloadInfosNamespaces.contains(namespace) { + updated = true + let ids = postbox.itemCollectionInfoTable.getIds(namespace: namespace) + self.idsByNamespace[namespace] = Set(ids) + } + } + } + return updated + } + + func immutableView() -> PostboxView { + return ItemCollectionIdsView(self) + } +} + +public final class ItemCollectionIdsView: PostboxView { + public let idsByNamespace: [ItemCollectionId.Namespace: Set] + + init(_ view: MutableItemCollectionIdsView) { + self.idsByNamespace = view.idsByNamespace + } +} diff --git a/Postbox/ItemCollectionInfoTable.swift b/Postbox/ItemCollectionInfoTable.swift index 07ad8f124b..72be13bf66 100644 --- a/Postbox/ItemCollectionInfoTable.swift +++ b/Postbox/ItemCollectionInfoTable.swift @@ -49,6 +49,32 @@ final class ItemCollectionInfoTable: Table { } } + func getIds(namespace: ItemCollectionId.Namespace) -> [ItemCollectionId] { + if let cachedInfo = self.cachedInfos[namespace] { + return cachedInfo.map { $0.1 } + } else { + var ids: [ItemCollectionId] = [] + self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in + ids.append(ItemCollectionId(namespace: namespace, id: key.getInt64(4 + 4))) + return true + }, limit: 0) + return ids + } + } + + func getIndex(id: ItemCollectionId) -> Int32? { + var index: Int32? + self.valueBox.range(self.table, start: self.lowerBound(namespace: id.namespace), end: self.upperBound(namespace: id.namespace), keys: { key in + let keyId = ItemCollectionId(namespace: id.namespace, id: key.getInt64(4 + 4)) + if keyId == id { + index = key.getInt32(4) + return false + } + return true + }, limit: 0) + return index + } + func getInfo(id: ItemCollectionId) -> ItemCollectionInfo? { var infoKey: ValueBoxKey? self.valueBox.range(self.table, start: self.lowerBound(namespace: id.namespace), end: self.upperBound(namespace: id.namespace), keys: { key in diff --git a/Postbox/ItemCollectionsView.swift b/Postbox/ItemCollectionsView.swift index 437e810507..4102087550 100644 --- a/Postbox/ItemCollectionsView.swift +++ b/Postbox/ItemCollectionsView.swift @@ -104,10 +104,9 @@ private func fetchHigherEntries(namespaces: [ItemCollectionId.Namespace], collec } private func aroundEntries(namespaces: [ItemCollectionId.Namespace], - collectionId: ItemCollectionId, - collectionIndex: Int32, - itemIndex: ItemCollectionItemIndex, + aroundIndex: ItemCollectionViewEntryIndex?, count: Int, + collectionIndexById: (ItemCollectionId) -> Int32?, lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, fetchLowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem], higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, @@ -117,6 +116,17 @@ private func aroundEntries(namespaces: [ItemCollectionId.Namespace], var lower: ItemCollectionViewEntry? var upper: ItemCollectionViewEntry? + let selectedAroundIndex: ItemCollectionViewEntryIndex + if let aroundIndex = aroundIndex, let aroundCollectionIndex = collectionIndexById(aroundIndex.collectionId) { + selectedAroundIndex = ItemCollectionViewEntryIndex(collectionIndex: aroundCollectionIndex, collectionId: aroundIndex.collectionId, itemIndex: aroundIndex.itemIndex) + } else { + selectedAroundIndex = ItemCollectionViewEntryIndex(collectionIndex: 0, collectionId: ItemCollectionId(namespace: namespaces[0], id: 0), itemIndex: ItemCollectionItemIndex.lowerBound) + } + + let collectionId: ItemCollectionId = selectedAroundIndex.collectionId + let collectionIndex: Int32 = selectedAroundIndex.collectionIndex + let itemIndex: ItemCollectionItemIndex = selectedAroundIndex.itemIndex + lowerEntries.append(contentsOf: fetchLowerEntries(namespaces: namespaces, collectionId: collectionId, collectionIndex: collectionIndex, itemIndex: itemIndex, count: count / 2 + 1, lowerCollectionId: lowerCollectionId, lowerItems: fetchLowerItems)) let lowerIndices = lowerEntries.map { $0.index } @@ -203,18 +213,11 @@ final class MutableItemCollectionsView { } } - let selectedAroundIndex: ItemCollectionViewEntryIndex - if let aroundIndex = aroundIndex { - selectedAroundIndex = aroundIndex - } else { - selectedAroundIndex = ItemCollectionViewEntryIndex(collectionIndex: 0, collectionId: ItemCollectionId(namespace: namespaces[0], id: 0), itemIndex: ItemCollectionItemIndex.lowerBound) - } - let (entries, lower, higher) = aroundEntries(namespaces: namespaces, - collectionId: selectedAroundIndex.collectionId, - collectionIndex: selectedAroundIndex.collectionIndex, - itemIndex: selectedAroundIndex.itemIndex, - count: count, + aroundIndex: aroundIndex, + count: count, collectionIndexById: { id in + return postbox.itemCollectionInfoTable.getIndex(id: id) + }, lowerCollectionId: { namespaceList, collectionId, collectionIndex in return self.lowerCollectionId(postbox: postbox, namespaceList: namespaceList, collectionId: collectionId, collectionIndex: collectionIndex) }, @@ -234,46 +237,12 @@ final class MutableItemCollectionsView { self.higher = higher } - /*init(orderedItemListsViews: [MutableOrderedItemListView], namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int, getInfos: (_ namespace: ItemCollectionId.Namespace) -> [(Int, ItemCollectionId, ItemCollectionInfo)], lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, lowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem], higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, higherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) { - self.orderedItemListsViews = orderedItemListsViews - self.namespaces = namespaces - - self.collectionInfos = [] - for namespace in namespaces { - for (_, id, info) in getInfos(namespace) { - let item = higherItems(id, ItemCollectionItemIndex.lowerBound, 1).first - self.collectionInfos.append((id, info, item)) - } - } - - let selectedAroundIndex: ItemCollectionViewEntryIndex - if let aroundIndex = aroundIndex { - selectedAroundIndex = aroundIndex - } else { - selectedAroundIndex = ItemCollectionViewEntryIndex(collectionIndex: 0, collectionId: ItemCollectionId(namespace: namespaces[0], id: 0), itemIndex: ItemCollectionItemIndex.lowerBound) - } - - let (entries, lower, higher) = aroundEntries(namespaces: namespaces, - collectionId: selectedAroundIndex.collectionId, - collectionIndex: selectedAroundIndex.collectionIndex, - itemIndex: selectedAroundIndex.itemIndex, - count: count, - lowerCollectionId: lowerCollectionId, - fetchLowerItems: lowerItems, - higherCollectionId: higherCollectionId, - fetchHigherItems: higherItems) - - self.entries = entries - self.lower = lower - self.higher = higher - }*/ - func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { var updated = false if !transaction.currentOrderedItemListOperations.isEmpty { for view in self.orderedItemListsViews { - if view.replay(operations: transaction.currentOrderedItemListOperations) { + if view.replay(postbox: postbox, transaction: transaction) { updated = true } } diff --git a/Postbox/OrderedItemListView.swift b/Postbox/OrderedItemListView.swift index bd71a6c584..23c07d6969 100644 --- a/Postbox/OrderedItemListView.swift +++ b/Postbox/OrderedItemListView.swift @@ -1,18 +1,18 @@ import Foundation -final class MutableOrderedItemListView { +final class MutableOrderedItemListView: MutablePostboxView { let collectionId: Int32 var items: [OrderedItemListEntry] - init(collectionId: Int32, getItems: (Int32) -> [OrderedItemListEntry]) { + init(postbox: Postbox, collectionId: Int32) { self.collectionId = collectionId - self.items = getItems(collectionId) + self.items = postbox.orderedItemListTable.getItems(collectionId: collectionId) } - func replay(operations: [Int32: [OrderedItemListOperation]]) -> Bool { + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { var updated = false - if let operations = operations[self.collectionId] { + if let operations = transaction.currentOrderedItemListOperations[self.collectionId] { for operation in operations { switch operation { case let .replace(items): @@ -35,9 +35,13 @@ final class MutableOrderedItemListView { return updated } + + func immutableView() -> PostboxView { + return OrderedItemListView(self) + } } -public final class OrderedItemListView { +public final class OrderedItemListView: PostboxView { public let collectionId: Int32 public let items: [OrderedItemListEntry] diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 0df11db2f6..f1867d92c2 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -22,12 +22,14 @@ public enum PostboxUpdateMessage { public final class Modifier { private weak var postbox: Postbox? + var disposed = false fileprivate init(postbox: Postbox) { self.postbox = postbox } public func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) -> [Int64: MessageId] { + assert(!self.disposed) if let postbox = self.postbox { return postbox.addMessages(messages, location: location) } else { @@ -36,29 +38,36 @@ public final class Modifier { } public func addHole(_ messageId: MessageId) { + assert(!self.disposed) self.postbox?.addHole(messageId) } public func fillHole(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { + assert(!self.disposed) self.postbox?.fillHole(hole, fillType: fillType, tagMask: tagMask, messages: messages) } public func fillMultipleHoles(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { + assert(!self.disposed) self.postbox?.fillMultipleHoles(hole, fillType: fillType, tagMask: tagMask, messages: messages) } public func replaceChatListHole(_ index: MessageIndex, hole: ChatListHole?) { + assert(!self.disposed) self.postbox?.replaceChatListHole(index, hole: hole) } public func deleteMessages(_ messageIds: [MessageId]) { + assert(!self.disposed) self.postbox?.deleteMessages(messageIds) } public func clearHistory(_ peerId: PeerId) { + assert(!self.disposed) self.postbox?.clearHistory(peerId) } public func messageIdsForGlobalIds(_ ids: [Int32]) -> [MessageId] { + assert(!self.disposed) if let postbox = self.postbox { return postbox.messageIdsForGlobalIds(ids) } else { @@ -67,6 +76,7 @@ public final class Modifier { } public func deleteMessagesWithGlobalIds(_ ids: [Int32]) { + assert(!self.disposed) if let postbox = self.postbox { let messageIds = postbox.messageIdsForGlobalIds(ids) postbox.deleteMessages(messageIds) @@ -74,26 +84,32 @@ public final class Modifier { } public func messageIdForGloballyUniqueMessageId(peerId: PeerId, id: Int64) -> MessageId? { + assert(!self.disposed) return self.postbox?.messageIdForGloballyUniqueMessageId(peerId: peerId, id: id) } public func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]]) { + assert(!self.disposed) self.postbox?.resetIncomingReadStates(states) } public func confirmSynchronizedIncomingReadState(_ peerId: PeerId) { + assert(!self.disposed) self.postbox?.confirmSynchronizedIncomingReadState(peerId) } public func applyIncomingReadMaxId(_ messageId: MessageId) { + assert(!self.disposed) self.postbox?.applyIncomingReadMaxId(messageId) } public func applyOutgoingReadMaxId(_ messageId: MessageId) { + assert(!self.disposed) self.postbox?.applyOutgoingReadMaxId(messageId) } public func applyInteractiveReadMaxIndex(_ messageIndex: MessageIndex) -> [MessageId] { + assert(!self.disposed) if let postbox = self.postbox { return postbox.applyInteractiveReadMaxIndex(messageIndex) } else { @@ -102,6 +118,7 @@ public final class Modifier { } public func applyOutgoingReadMaxIndex(_ messageIndex: MessageIndex) -> [MessageId] { + assert(!self.disposed) if let postbox = self.postbox { return postbox.applyOutgoingReadMaxIndex(messageIndex) } else { @@ -110,46 +127,57 @@ public final class Modifier { } public func getState() -> Coding? { + assert(!self.disposed) return self.postbox?.getState() } public func setState(_ state: Coding) { + assert(!self.disposed) self.postbox?.setState(state) } public func getPeerChatState(_ id: PeerId) -> PeerChatState? { + assert(!self.disposed) return self.postbox?.peerChatStateTable.get(id) as? PeerChatState } public func setPeerChatState(_ id: PeerId, state: PeerChatState) { + assert(!self.disposed) self.postbox?.setPeerChatState(id, state: state) } public func updatePeerChatInterfaceState(_ id: PeerId, update: (PeerChatInterfaceState?) -> (PeerChatInterfaceState?)) { + assert(!self.disposed) self.postbox?.updatePeerChatInterfaceState(id, update: update) } public func getPeer(_ id: PeerId) -> Peer? { + assert(!self.disposed) return self.postbox?.peerTable.get(id) } public func getPeerReadStates(_ id: PeerId) -> [(MessageId.Namespace, PeerReadState)]? { + assert(!self.disposed) return self.postbox?.readStateTable.getCombinedState(id)?.states } public func getCombinedPeerReadState(_ id: PeerId) -> CombinedPeerReadState? { + assert(!self.disposed) return self.postbox?.readStateTable.getCombinedState(id) } public func getPeerNotificationSettings(_ id: PeerId) -> PeerNotificationSettings? { + assert(!self.disposed) return self.postbox?.peerNotificationSettingsTable.get(id) } public func updatePeersInternal(_ peers: [Peer], update: (Peer?, Peer) -> Peer?) { + assert(!self.disposed) self.postbox?.updatePeers(peers, update: update) } public func getPeerChatListInclusion(_ id: PeerId) -> PeerChatListInclusion { + assert(!self.disposed) if let postbox = self.postbox { return postbox.getPeerChatListInclusion(id) } @@ -157,34 +185,42 @@ public final class Modifier { } public func getTopPeerMessageId(peerId: PeerId, namespace: MessageId.Namespace) -> MessageId? { + assert(!self.disposed) return self.postbox?.messageHistoryIndexTable.top(peerId, namespace: namespace)?.index.id } public func updatePeerChatListInclusion(_ id: PeerId, inclusion: PeerChatListInclusion) { + assert(!self.disposed) self.postbox?.updatePeerChatListInclusion(id, inclusion: inclusion) } public func updatePeerNotificationSettings(_ notificationSettings: [PeerId: PeerNotificationSettings]) { + assert(!self.disposed) self.postbox?.updatePeerNotificationSettings(notificationSettings) } public func resetAllPeerNotificationSettings(_ notificationSettings: PeerNotificationSettings) { + assert(!self.disposed) self.postbox?.resetAllPeerNotificationSettings(notificationSettings) } public func updatePeerCachedData(peerIds: Set, update: (PeerId, CachedPeerData?) -> CachedPeerData?) { + assert(!self.disposed) self.postbox?.updatePeerCachedData(peerIds: peerIds, update: update) } public func getPeerCachedData(peerId: PeerId) -> CachedPeerData? { + assert(!self.disposed) return self.postbox?.cachedPeerDataTable.get(peerId) } public func updatePeerPresences(_ peerPresences: [PeerId: PeerPresence]) { + assert(!self.disposed) self.postbox?.updatePeerPresences(peerPresences) } public func getContactPeerIds() -> Set { + assert(!self.disposed) if let postbox = self.postbox { return postbox.contactsTable.get() } else { @@ -193,42 +229,52 @@ public final class Modifier { } public func replaceContactPeerIds(_ peerIds: Set) { + assert(!self.disposed) self.postbox?.replaceContactPeerIds(peerIds) } public func replaceRecentPeerIds(_ peerIds: [PeerId]) { + assert(!self.disposed) self.postbox?.replaceRecentPeerIds(peerIds) } public func updateMessage(_ id: MessageId, update: (Message) -> PostboxUpdateMessage) { + assert(!self.disposed) self.postbox?.updateMessage(id, update: update) } public func offsetPendingMessagesTimestamps(lowerBound: MessageId, timestamp: Int32) { + assert(!self.disposed) self.postbox?.offsetPendingMessagesTimestamps(lowerBound: lowerBound, timestamp: timestamp) } public func updateMedia(_ id: MediaId, update: Media?) { + assert(!self.disposed) self.postbox?.updateMedia(id, update: update) } public func replaceItemCollections(namespace: ItemCollectionId.Namespace, itemCollections: [(ItemCollectionId, ItemCollectionInfo, [ItemCollectionItem])]) { + assert(!self.disposed) self.postbox?.replaceItemCollections(namespace: namespace, itemCollections: itemCollections) } public func replaceItemCollectionInfos(namespace: ItemCollectionId.Namespace, itemCollectionInfos: [(ItemCollectionId, ItemCollectionInfo)]) { + assert(!self.disposed) self.postbox?.replaceItemCollectionInfos(namespace: namespace, itemCollectionInfos: itemCollectionInfos) } public func replaceItemCollectionItems(collectionId: ItemCollectionId, items: [ItemCollectionItem]) { + assert(!self.disposed) self.postbox?.replaceItemCollectionItems(collectionId: collectionId, items: items) } public func removeItemCollection(collectionId: ItemCollectionId) { + assert(!self.disposed) self.postbox?.removeItemCollection(collectionId: collectionId) } public func getItemCollectionsInfos(namespace: ItemCollectionId.Namespace) -> [(ItemCollectionId, ItemCollectionInfo)] { + assert(!self.disposed) if let postbox = self.postbox { return postbox.itemCollectionInfoTable.getInfos(namespace: namespace).map { ($0.1, $0.2) } } else { @@ -237,10 +283,12 @@ public final class Modifier { } public func getItemCollectionInfo(collectionId: ItemCollectionId) -> ItemCollectionInfo? { + assert(!self.disposed) return self.postbox?.itemCollectionInfoTable.getInfo(id: collectionId) } public func getItemCollectionItems(collectionId: ItemCollectionId) -> [ItemCollectionItem] { + assert(!self.disposed) if let postbox = self.postbox { return postbox.itemCollectionItemTable.collectionItems(collectionId: collectionId) } else { @@ -249,6 +297,7 @@ public final class Modifier { } public func getCollectionsItems(namespace: ItemCollectionId.Namespace) -> [(ItemCollectionId, ItemCollectionInfo, [ItemCollectionItem])] { + assert(!self.disposed) if let postbox = postbox { let infos = postbox.itemCollectionInfoTable.getInfos(namespace: namespace) var result: [(ItemCollectionId, ItemCollectionInfo, [ItemCollectionItem])] = [] @@ -263,6 +312,7 @@ public final class Modifier { } public func getItemCollectionInfoItems(namespace: ItemCollectionId.Namespace, id:ItemCollectionId) -> (ItemCollectionInfo, [ItemCollectionItem])? { + assert(!self.disposed) if let postbox = postbox { let infos = postbox.itemCollectionInfoTable.getInfos(namespace: namespace) for info in infos { @@ -276,6 +326,7 @@ public final class Modifier { } public func searchItemCollection(namespace: ItemCollectionId.Namespace, key: MemoryBuffer) -> [ItemCollectionItem] { + assert(!self.disposed) if let postbox = self.postbox { return postbox.itemCollectionItemTable.exactIndexedItems(namespace: namespace, key: ValueBoxKey(key)) } else { @@ -284,14 +335,17 @@ public final class Modifier { } public func replaceOrderedItemListItems(collectionId: Int32, items: [OrderedItemListEntry]) { + assert(!self.disposed) self.postbox?.replaceOrderedItemListItems(collectionId: collectionId, items: items) } public func addOrMoveToFirstPositionOrderedItemListItem(collectionId: Int32, item: OrderedItemListEntry, removeTailIfCountExceeds: Int?) { + assert(!self.disposed) self.postbox?.addOrMoveToFirstPositionOrderedItemListItem(collectionId: collectionId, item: item, removeTailIfCountExceeds: removeTailIfCountExceeds) } public func getOrderedListItemIds(collectionId: Int32) -> [MemoryBuffer] { + assert(!self.disposed) if let postbox = postbox { return postbox.getOrderedListItemIds(collectionId: collectionId) } else { @@ -299,7 +353,17 @@ public final class Modifier { } } + public func getOrderedListItems(collectionId: Int32) -> [OrderedItemListEntry] { + assert(!self.disposed) + if let postbox = postbox { + return postbox.orderedItemListTable.getItems(collectionId: collectionId) + } else { + return [] + } + } + public func getMessage(_ id: MessageId) -> Message? { + assert(!self.disposed) if let postbox = self.postbox { if let entry = postbox.messageHistoryIndexTable.get(id) { if case let .Message(index) = entry { @@ -313,6 +377,7 @@ public final class Modifier { } public func filterStoredMessageIds(_ messageIds: Set) -> Set { + assert(!self.disposed) if let postbox = self.postbox { return postbox.filterStoredMessageIds(messageIds) } @@ -320,18 +385,22 @@ public final class Modifier { } public func storedMessageId(peerId: PeerId, namespace: MessageId.Namespace, timestamp: Int32) -> MessageId? { + assert(!self.disposed) return self.postbox?.storedMessageId(peerId: peerId, namespace: namespace, timestamp: timestamp) } public func putItemCacheEntry(id: ItemCacheEntryId, entry: Coding, collectionSpec: ItemCacheCollectionSpec) { + assert(!self.disposed) self.postbox?.putItemCacheEntry(id: id, entry: entry, collectionSpec: collectionSpec) } public func retrieveItemCacheEntry(id: ItemCacheEntryId) -> Coding? { + assert(!self.disposed) return self.postbox?.retrieveItemCacheEntry(id: id) } public func operationLogGetNextEntryLocalIndex(peerId: PeerId, tag: PeerOperationLogTag) -> Int32 { + assert(!self.disposed) if let postbox = self.postbox { return postbox.operationLogGetNextEntryLocalIndex(peerId: peerId, tag: tag) } else { @@ -340,10 +409,12 @@ public final class Modifier { } public func operationLogAddEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: StorePeerOperationLogEntryTagLocalIndex, tagMergedIndex: StorePeerOperationLogEntryTagMergedIndex, contents: Coding) { + assert(!self.disposed) self.postbox?.operationLogAddEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, tagMergedIndex: tagMergedIndex, contents: contents) } public func operationLogRemoveEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32) -> Bool { + assert(!self.disposed) if let postbox = self.postbox { return postbox.operationLogRemoveEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex) } else { @@ -352,42 +423,52 @@ public final class Modifier { } public func operationLogRemoveAllEntries(peerId: PeerId, tag: PeerOperationLogTag) { + assert(!self.disposed) self.postbox?.operationLogRemoveAllEntries(peerId: peerId, tag: tag) } public func operationLogRemoveEntries(peerId: PeerId, tag: PeerOperationLogTag, withTagLocalIndicesEqualToOrLowerThan maxTagLocalIndex: Int32) { + assert(!self.disposed) self.postbox?.operationLogRemoveEntries(peerId: peerId, tag: tag, withTagLocalIndicesEqualToOrLowerThan: maxTagLocalIndex) } public func operationLogUpdateEntry(peerId: PeerId, tag: PeerOperationLogTag, tagLocalIndex: Int32, _ f: (PeerOperationLogEntry?) -> PeerOperationLogEntryUpdate) { + assert(!self.disposed) self.postbox?.operationLogUpdateEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, f) } public func operationLogEnumerateEntries(peerId: PeerId, tag: PeerOperationLogTag, _ f: (PeerOperationLogEntry) -> Bool) { + assert(!self.disposed) self.postbox?.operationLogEnumerateEntries(peerId: peerId, tag: tag, f) } public func addTimestampBasedMessageAttribute(tag: UInt16, timestamp: Int32, messageId: MessageId) { + assert(!self.disposed) self.postbox?.addTimestampBasedMessageAttribute(tag: tag, timestamp: timestamp, messageId: messageId) } public func removeTimestampBasedMessageAttribute(tag: UInt16, messageId: MessageId) { + assert(!self.disposed) self.postbox?.removeTimestampBasedMessageAttribute(tag: tag, messageId: messageId) } public func getPreferencesEntry(key: ValueBoxKey) -> PreferencesEntry? { + assert(!self.disposed) return self.postbox?.getPreferencesEntry(key: key) } public func setPreferencesEntry(key: ValueBoxKey, value: PreferencesEntry?) { + assert(!self.disposed) self.postbox?.setPreferencesEntry(key: key, value: value) } public func updatePreferencesEntry(key: ValueBoxKey, _ f: (PreferencesEntry?) -> PreferencesEntry?) { + assert(!self.disposed) self.postbox?.setPreferencesEntry(key: key, value: f(self.postbox?.getPreferencesEntry(key: key))) } public func getPinnedPeerIds() -> [PeerId] { + assert(!self.disposed) if let postbox = self.postbox { return postbox.chatListTable.getPinnedPeerIds() } else { @@ -396,10 +477,12 @@ public final class Modifier { } public func setPinnedPeerIds(_ peerIds: [PeerId]) { + assert(!self.disposed) self.postbox?.setPinnedPeerIds(peerIds) } public func getTotalUnreadCount() -> Int32 { + assert(!self.disposed) if let postbox = self.postbox { return postbox.messageHistoryMetadataTable.getChatListTotalUnreadCount() } else { @@ -408,6 +491,7 @@ public final class Modifier { } public func getAccessChallengeData() -> PostboxAccessChallengeData { + assert(!self.disposed) if let postbox = self.postbox { return postbox.metadataTable.accessChallengeData() } else { @@ -416,6 +500,7 @@ public final class Modifier { } public func setAccessChallengeData(_ data: PostboxAccessChallengeData) { + assert(!self.disposed) self.postbox?.metadataTable.setAccessChallengeData(data) } } @@ -1309,6 +1394,7 @@ public final class Postbox { infos.remove(at: index) self.replaceItemCollectionInfos(namespace: collectionId.namespace, itemCollectionInfos: infos.map { ($0.1, $0.2) }) } + self.replaceItemCollectionItems(collectionId: collectionId, items: []) } fileprivate func filterStoredMessageIds(_ messageIds: Set) -> Set { @@ -1340,7 +1426,9 @@ public final class Postbox { let f: () -> Void = { self.valueBox.begin() self.afterBegin() - let result = f(Modifier(postbox: self)) + let modifier = Modifier(postbox: self) + let result = f(modifier) + modifier.disposed = true let (updatedTransactionState, updatedMasterClientId) = self.beforeCommit() self.valueBox.commit() @@ -1797,9 +1885,7 @@ public final class Postbox { public func itemCollectionsView(orderedItemListCollectionIds: [Int32], namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int) -> Signal { return self.modify { modifier -> Signal in let itemListViews = orderedItemListCollectionIds.map { collectionId -> MutableOrderedItemListView in - return MutableOrderedItemListView(collectionId: collectionId, getItems: { collectionId in - return self.orderedItemListTable.getItems(collectionId: collectionId) - }) + return MutableOrderedItemListView(postbox: self, collectionId: collectionId) } let mutableView = MutableItemCollectionsView(postbox: self, orderedItemListsViews: itemListViews, namespaces: namespaces, aroundIndex: aroundIndex, count: count) @@ -1818,26 +1904,6 @@ public final class Postbox { } |> switchToLatest } - public func orderedItemListView(collectionId: Int32) -> Signal { - return self.modify { modifier -> Signal in - let mutableView = MutableOrderedItemListView(collectionId: collectionId, getItems: { collectionId in - return self.orderedItemListTable.getItems(collectionId: collectionId) - }) - - let (index, signal) = self.viewTracker.addOrderedItemListView(mutableView) - - return (.single(OrderedItemListView(mutableView)) - |> then(signal)) - |> afterDisposed { [weak self] in - if let strongSelf = self { - strongSelf.queue.async { - strongSelf.viewTracker.removeOrderedItemListView(index) - } - } - } - } |> switchToLatest - } - public func mergedOperationLogView(tag: PeerOperationLogTag, limit: Int) -> Signal { return self.modify { modifier -> Signal in let view = MutablePeerMergedOperationLogView(tag: tag, limit: limit, getOperations: { tag, fromIndex, limit in diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 5a71adc859..7495e4cc4f 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -62,7 +62,6 @@ final class ViewTracker { private var preferencesViews = Bag<(MutablePreferencesView, ValuePipe)>() private var multiplePeersViews = Bag<(MutableMultiplePeersView, ValuePipe)>() private var itemCollectionsViews = Bag<(MutableItemCollectionsView, ValuePipe)>() - private var orderedItemListViews = Bag<(MutableOrderedItemListView, ValuePipe)>() init(queue: Queue, fetchEarlierHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchLaterHistoryEntries: @escaping (PeerId, MessageIndex?, Int, MessageTags?) -> [MutableMessageHistoryEntry], fetchEarlierChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchLaterChatEntries: @escaping (ChatListIndex?, Int) -> [MutableChatListEntry], fetchAnchorIndex: @escaping (MessageId) -> MessageHistoryAnchorIndex?, renderMessage: @escaping (IntermediateMessage) -> Message, getPeer: @escaping (PeerId) -> Peer?, getPeerNotificationSettings: @escaping (PeerId) -> PeerNotificationSettings?, getCachedPeerData: @escaping (PeerId) -> CachedPeerData?, getPeerPresence: @escaping (PeerId) -> PeerPresence?, getTotalUnreadCount: @escaping () -> Int32, getPeerReadState: @escaping (PeerId) -> CombinedPeerReadState?, operationLogGetOperations: @escaping (PeerOperationLogTag, Int32, Int) -> [PeerMergedOperationLogEntry], operationLogGetTailIndex: @escaping (PeerOperationLogTag) -> Int32?, getTimestampBasedMessageAttributesHead: @escaping (UInt16) -> TimestampBasedMessageAttributesEntry?, getPreferencesEntry: @escaping (ValueBoxKey) -> PreferencesEntry?, unsentMessageIds: [MessageId], synchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation]) { self.queue = queue @@ -268,17 +267,6 @@ final class ViewTracker { self.multiplePeersViews.remove(index) } - func addOrderedItemListView(_ view: MutableOrderedItemListView) -> (Bag<(MutableOrderedItemListView, ValuePipe)>.Index, Signal) { - let record = (view, ValuePipe()) - let index = self.orderedItemListViews.add(record) - - return (index, record.1.signal()) - } - - func removeOrderedItemListView(_ index: Bag<(MutableOrderedItemListView, ValuePipe)>.Index) { - self.orderedItemListViews.remove(index) - } - func addItemCollectionView(_ view: MutableItemCollectionsView) -> (Bag<(MutableItemCollectionsView, ValuePipe)>.Index, Signal) { let record = (view, ValuePipe()) let index = self.itemCollectionsViews.add(record) @@ -524,12 +512,6 @@ final class ViewTracker { } } - for (mutableView, pipe) in self.orderedItemListViews.copyItems() { - if mutableView.replay(operations: transaction.currentOrderedItemListOperations) { - pipe.putNext(OrderedItemListView(mutableView)) - } - } - for (mutableView, pipe) in self.itemCollectionsViews.copyItems() { if mutableView.replay(postbox: postbox, transaction: transaction) { pipe.putNext(ItemCollectionsView(mutableView)) diff --git a/Postbox/Views.swift b/Postbox/Views.swift index 1f811d2374..c0ccb6b3b1 100644 --- a/Postbox/Views.swift +++ b/Postbox/Views.swift @@ -2,17 +2,23 @@ import Foundation public enum PostboxViewKey: Hashable { case itemCollectionInfos(namespaces: [ItemCollectionId.Namespace]) + case itemCollectionIds(namespaces: [ItemCollectionId.Namespace]) case itemCollectionInfo(id: ItemCollectionId) case peerChatState(peerId: PeerId) + case orderedItemList(id: Int32) public var hashValue: Int { switch self { case .itemCollectionInfos: return 0 + case .itemCollectionIds: + return 1 case let .peerChatState(peerId): return peerId.hashValue case let .itemCollectionInfo(id): return id.hashValue + case let .orderedItemList(id): + return id.hashValue } } @@ -24,6 +30,12 @@ public enum PostboxViewKey: Hashable { } else { return false } + case let .itemCollectionIds(lhsNamespaces): + if case let .itemCollectionIds(rhsNamespaces) = rhs, lhsNamespaces == rhsNamespaces { + return true + } else { + return false + } case let .itemCollectionInfo(id): if case .itemCollectionInfo(id) = rhs { return true @@ -36,6 +48,12 @@ public enum PostboxViewKey: Hashable { } else { return false } + case let .orderedItemList(id): + if case .orderedItemList(id) = rhs { + return true + } else { + return false + } } } } @@ -44,9 +62,13 @@ func postboxViewForKey(postbox: Postbox, key: PostboxViewKey) -> MutablePostboxV switch key { case let .itemCollectionInfos(namespaces): return MutableItemCollectionInfosView(postbox: postbox, namespaces: namespaces) + case let .itemCollectionIds(namespaces): + return MutableItemCollectionIdsView(postbox: postbox, namespaces: namespaces) case let .itemCollectionInfo(id): return MutableItemCollectionInfoView(postbox: postbox, id: id) case let .peerChatState(peerId): return MutablePeerChatStateView(postbox: postbox, peerId: peerId) + case let .orderedItemList(id): + return MutableOrderedItemListView(postbox: postbox, collectionId: id) } } From 04d78772be1cf5a492eb246e71f4e9ae85a73c53 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 23 Mar 2017 21:26:23 +0300 Subject: [PATCH 119/427] no message --- Postbox/MediaBox.swift | 290 ++++++++++++++++++----- Postbox/MediaResource.swift | 5 + Postbox/PeerMergedOperationLogView.swift | 4 +- Postbox/Postbox.swift | 5 + 4 files changed, 241 insertions(+), 63 deletions(-) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index fb5145ac2e..512cdd351c 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -50,7 +50,14 @@ public struct MediaResourceData { public let complete: Bool } -public struct MediaResourceDataFetchResult { +public enum MediaResourceDataFetchResult { + case dataPart(data: Data, range: Range, complete: Bool) + case replaceHeader(data: Data, range: Range) + case moveLocalFile(path: String) + case reset +} + +/*public struct MediaResourceDataFetchResult { public let data: Data public let complete: Bool @@ -58,7 +65,7 @@ public struct MediaResourceDataFetchResult { self.data = data self.complete = complete } -} +}*/ public struct CachedMediaResourceRepresentationResult { public let temporaryPath: String @@ -410,9 +417,14 @@ public final class MediaBox { if let strongSelf = strongSelf { strongSelf.dataQueue.async { if let dataContext = strongSelf.randomAccessContexts[resourceId] { - let storeRange = RandomAccessResourceStoreRange(offset: range.lowerBound + offset, data: result.data) - offset += result.data.count - dataContext.storeRanges([storeRange]) + switch result { + case let .dataPart(data, dataRange, _): + let storeRange = RandomAccessResourceStoreRange(offset: range.lowerBound + offset, data: data.subdata(in: dataRange)) + offset += data.count + dataContext.storeRanges([storeRange]) + default: + assertionFailure() + } } } } @@ -560,79 +572,233 @@ public final class MediaBox { close(fd) } } - }).start(next: { result in + }).start(next: { resultOption in self.dataQueue.async { let _ = self.ensureDirectoryCreated - if fd == nil { - let handle = open(paths.partial, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) - if handle >= 0 { - fd = handle - } - } - - if let thisFd = fd { - if !result.data.isEmpty { - let writeResult = result.data.withUnsafeBytes { bytes -> Int in - return write(thisFd, bytes, result.data.count) + switch resultOption { + case let .dataPart(data, dataRange, complete): + if fd == nil { + let handle = open(paths.partial, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) + if handle >= 0 { + fd = handle + } } - if writeResult != result.data.count { - print("write error \(errno)") + + if let thisFd = fd { + if !dataRange.isEmpty { + let writeResult = data.withUnsafeBytes { bytes -> Int in + return write(thisFd, bytes.advanced(by: dataRange.lowerBound), dataRange.count) + } + if writeResult != dataRange.count { + print("write error \(errno)") + } + } + + offset += dataRange.count + let updatedSize = offset + + let updatedData: MediaResourceData + if complete { + let linkResult = link(paths.partial, paths.complete) + assert(linkResult == 0) + updatedData = MediaResourceData(path: paths.complete, size: updatedSize, complete: true) + } else { + updatedData = MediaResourceData(path: paths.partial, size: updatedSize, complete: false) + } + + dataContext.data = updatedData + + let hadProcessedFetch = dataContext.processedFetch + dataContext.processedFetch = true + + for (_, subscriber) in dataContext.progresiveDataSubscribers.copyItems() { + subscriber(updatedData) + } + + if updatedData.complete { + for (_, subscriber) in dataContext.completeDataSubscribers.copyItems() { + subscriber(updatedData) + } + } else if !hadProcessedFetch { + for (waitUntilFetchStatus, subscriber) in dataContext.completeDataSubscribers.copyItems() { + if waitUntilFetchStatus { + subscriber(updatedData) + } + } + } + + let status: MediaResourceStatus + if updatedData.complete { + status = .Local + } else { + if let resourceSize = resource.size { + status = .Fetching(progress: Float(updatedSize) / Float(resourceSize)) + } else { + status = .Fetching(progress: 0.0) + } + } + + self.statusQueue.async { + if let statusContext = self.statusContexts[resourceId] { + statusContext.status = status + for subscriber in statusContext.subscribers.copyItems() { + subscriber(status) + } + } + } + } + case let .replaceHeader(data, dataRange): + if let thisFd = fd { + close(thisFd) + fd = nil + } + + if fd == nil { + let handle = open(paths.partial, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR) + if handle >= 0 { + fd = handle + } + } + + if let thisFd = fd { + if !dataRange.isEmpty { + lseek(thisFd, 0, SEEK_SET) + let writeResult = data.withUnsafeBytes { bytes -> Int in + return write(thisFd, bytes.advanced(by: dataRange.lowerBound), dataRange.count) + } + lseek(thisFd, Int64(offset), SEEK_SET) + if writeResult != dataRange.count { + print("write error \(errno)") + } + + close(thisFd) + fd = nil + } + } + case .reset: + if fd == nil { + let handle = open(paths.partial, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) + if handle >= 0 { + fd = handle + } + } + + if let fd = fd { + ftruncate(fd, 0) + lseek(fd, 0, SEEK_SET) + } else { + assertionFailure() } - } - offset += result.data.count - let updatedSize = offset + offset = 0 + let updatedSize = offset + + let updatedData: MediaResourceData + updatedData = MediaResourceData(path: paths.partial, size: updatedSize, complete: false) + + dataContext.data = updatedData + + let hadProcessedFetch = dataContext.processedFetch + dataContext.processedFetch = true + + for (_, subscriber) in dataContext.progresiveDataSubscribers.copyItems() { + subscriber(updatedData) + } + + if updatedData.complete { + for (_, subscriber) in dataContext.completeDataSubscribers.copyItems() { + subscriber(updatedData) + } + } else if !hadProcessedFetch { + for (waitUntilFetchStatus, subscriber) in dataContext.completeDataSubscribers.copyItems() { + if waitUntilFetchStatus { + subscriber(updatedData) + } + } + } + + let status: MediaResourceStatus + if updatedData.complete { + status = .Local + } else { + if let resourceSize = resource.size { + status = .Fetching(progress: Float(updatedSize) / Float(resourceSize)) + } else { + status = .Fetching(progress: 0.0) + } + } + + self.statusQueue.async { + if let statusContext = self.statusContexts[resourceId] { + statusContext.status = status + for subscriber in statusContext.subscribers.copyItems() { + subscriber(status) + } + } + } + case let .moveLocalFile(tempPath): + if let fd = fd { + close(fd) + } + unlink(paths.partial) + do { + try FileManager.default.moveItem(atPath: tempPath, toPath: paths.partial) + } catch { + assertionFailure() + } - let updatedData: MediaResourceData - if result.complete { + guard let offset = fileSize(paths.partial) else { + assertionFailure() + return + } + let updatedSize = offset + + let updatedData: MediaResourceData let linkResult = link(paths.partial, paths.complete) assert(linkResult == 0) updatedData = MediaResourceData(path: paths.complete, size: updatedSize, complete: true) - } else { - updatedData = MediaResourceData(path: paths.partial, size: updatedSize, complete: false) - } - - dataContext.data = updatedData - - let hadProcessedFetch = dataContext.processedFetch - dataContext.processedFetch = true - - for (_, subscriber) in dataContext.progresiveDataSubscribers.copyItems() { - subscriber(updatedData) - } - - if updatedData.complete { - for (_, subscriber) in dataContext.completeDataSubscribers.copyItems() { + + dataContext.data = updatedData + + let hadProcessedFetch = dataContext.processedFetch + dataContext.processedFetch = true + + for (_, subscriber) in dataContext.progresiveDataSubscribers.copyItems() { subscriber(updatedData) } - } else if !hadProcessedFetch { - for (waitUntilFetchStatus, subscriber) in dataContext.completeDataSubscribers.copyItems() { - if waitUntilFetchStatus { + + if updatedData.complete { + for (_, subscriber) in dataContext.completeDataSubscribers.copyItems() { subscriber(updatedData) } - } - } - - let status: MediaResourceStatus - if updatedData.complete { - status = .Local - } else { - if let resourceSize = resource.size { - status = .Fetching(progress: Float(updatedSize) / Float(resourceSize)) - } else { - status = .Fetching(progress: 0.0) - } - } - - self.statusQueue.async { - if let statusContext = self.statusContexts[resourceId] { - statusContext.status = status - for subscriber in statusContext.subscribers.copyItems() { - subscriber(status) + } else if !hadProcessedFetch { + for (waitUntilFetchStatus, subscriber) in dataContext.completeDataSubscribers.copyItems() { + if waitUntilFetchStatus { + subscriber(updatedData) + } + } + } + + let status: MediaResourceStatus + if updatedData.complete { + status = .Local + } else { + if let resourceSize = resource.size { + status = .Fetching(progress: Float(updatedSize) / Float(resourceSize)) + } else { + status = .Fetching(progress: 0.0) + } + } + + self.statusQueue.async { + if let statusContext = self.statusContexts[resourceId] { + statusContext.status = status + for subscriber in statusContext.subscribers.copyItems() { + subscriber(status) + } } } - } } } }) diff --git a/Postbox/MediaResource.swift b/Postbox/MediaResource.swift index f204bc2441..96c028cba1 100644 --- a/Postbox/MediaResource.swift +++ b/Postbox/MediaResource.swift @@ -26,6 +26,7 @@ public protocol MediaResource { var id: MediaResourceId { get } var size: Int? { get } var streamable: Bool { get } + var headerSize: Int32 { get } } public extension MediaResource { @@ -36,6 +37,10 @@ public extension MediaResource { var streamable: Bool { return false } + + var headerSize: Int32 { + return 0 + } } public protocol CachedMediaResourceRepresentation { diff --git a/Postbox/PeerMergedOperationLogView.swift b/Postbox/PeerMergedOperationLogView.swift index 73171c16a7..e07117b152 100644 --- a/Postbox/PeerMergedOperationLogView.swift +++ b/Postbox/PeerMergedOperationLogView.swift @@ -30,7 +30,9 @@ final class MutablePeerMergedOperationLogView { } } else { updated = true - assert(self.entries.isEmpty) + if !self.entries.isEmpty { + assertionFailure("self.entries.isEmpty == false for tag \(self.tag)") + } self.entries.append(entry) self.tailIndex = entry.mergedIndex } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 2d4b5c30a9..33b858a76f 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -146,6 +146,11 @@ public final class Modifier { self.postbox?.setPeerChatState(id, state: state) } + public func getPeerChatInterfaceState(_ id: PeerId) -> PeerChatInterfaceState? { + assert(!self.disposed) + return self.postbox?.peerChatInterfaceStateTable.get(id) + } + public func updatePeerChatInterfaceState(_ id: PeerId, update: (PeerChatInterfaceState?) -> (PeerChatInterfaceState?)) { assert(!self.disposed) self.postbox?.updatePeerChatInterfaceState(id, update: update) From e187267899b64ab99cb4fa5723d46ce5ac0e6d38 Mon Sep 17 00:00:00 2001 From: overtake Date: Sun, 26 Mar 2017 23:07:07 +0300 Subject: [PATCH 120/427] changes --- Postbox.xcodeproj/project.pbxproj | 1 + .../xcschemes/xcschememanagement.plist | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index ab2d6f7cac..b096659bb8 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -1302,6 +1302,7 @@ CLANG_WARN_SUSPICIOUS_MOVES = YES; CODE_SIGN_IDENTITY = "-"; COMBINE_HIDPI_IMAGES = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; DEFINES_MODULE = YES; DEVELOPMENT_TEAM = ""; DYLIB_COMPATIBILITY_VERSION = 1; diff --git a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist index a3703be5c4..f82c04594c 100644 --- a/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/mikhailfilimonov.xcuserdatad/xcschemes/xcschememanagement.plist @@ -7,14 +7,14 @@ Postbox.xcscheme isShown - + orderHint 5 PostboxMac.xcscheme isShown - + orderHint 6 @@ -24,17 +24,17 @@ D0B418121D7DFAF2004562A4 primary - + D0E3A7491B28A7E300A402D9 primary - + D0E3A7541B28A7E300A402D9 primary - + From 4847a6ca82606f467c252255544276558aee7492 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 28 Mar 2017 21:44:41 +0300 Subject: [PATCH 121/427] no message --- Postbox/ItemCollectionItemTable.swift | 16 +++++++--- Postbox/MediaBox.swift | 43 ++++++++++++++++++--------- Postbox/Postbox.swift | 21 ++++++++++++- 3 files changed, 61 insertions(+), 19 deletions(-) diff --git a/Postbox/ItemCollectionItemTable.swift b/Postbox/ItemCollectionItemTable.swift index 416e086943..e92858a07e 100644 --- a/Postbox/ItemCollectionItemTable.swift +++ b/Postbox/ItemCollectionItemTable.swift @@ -228,17 +228,25 @@ final class ItemCollectionItemTable: Table { } } - func exactIndexedItems(namespace: ItemCollectionId.Namespace, key: ValueBoxKey) -> [ItemCollectionItem] { + func exactIndexedItems(namespace: ItemCollectionId.Namespace, key: ValueBoxKey) -> [ItemCollectionId: [ItemCollectionItem]] { let references = self.reverseIndexTable.exactReferences(namespace: ReverseIndexNamespace(namespace), token: key) - var result: [ItemCollectionItem] = [] + var resultsByCollectionId: [ItemCollectionId: [(ItemCollectionItemIndex, ItemCollectionItem)]] = [:] for reference in references { if let value = self.valueBox.get(self.table, key: self.key(collectionId: reference.collectionId, index: reference.itemIndex)), let item = Decoder(buffer: value).decodeRootObject() as? ItemCollectionItem { - result.append(item) + if resultsByCollectionId[reference.collectionId] == nil { + resultsByCollectionId[reference.collectionId] = [(reference.itemIndex, item)] + } else { + resultsByCollectionId[reference.collectionId]!.append((reference.itemIndex, item)) + } } else { assertionFailure() } } - return result + var results: [ItemCollectionId: [ItemCollectionItem]] = [:] + for collectionId in resultsByCollectionId.keys { + results[collectionId] = resultsByCollectionId[collectionId]!.sorted(by: { $0.0 < $1.0 }).map({ $0.1 }) + } + return results } override func clearMemoryCache() { diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 512cdd351c..e903f4a2cf 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -869,7 +869,7 @@ public final class MediaBox { } } - public func cachedResourceRepresentation(_ resource: MediaResource, representation: CachedMediaResourceRepresentation) -> Signal { + public func cachedResourceRepresentation(_ resource: MediaResource, representation: CachedMediaResourceRepresentation, complete: Bool) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.concurrentQueue.async { @@ -888,8 +888,10 @@ public final class MediaBox { self.cachedRepresentationContexts[key] = context } - let index = context.dataSubscribers.add({ [weak self] data in - subscriber.putNext(data) + let index = context.dataSubscribers.add({ data in + if !complete || data.complete { + subscriber.putNext(data) + } if data.complete { subscriber.putCompletion() } @@ -909,22 +911,35 @@ public final class MediaBox { if context.disposable == nil { let signal = self.resourceData(resource, option: .complete(waitUntilFetchStatus: false)) - |> mapToSignal { resourceData in - return self.wrappedFetchCachedResourceRepresentation.get() - |> take(1) - |> mapToSignal { fetch in - return fetch(resource, resourceData, representation) + |> mapToSignal { resourceData -> Signal in + if resourceData.complete { + return self.wrappedFetchCachedResourceRepresentation.get() + |> take(1) + |> mapToSignal { fetch in + return fetch(resource, resourceData, representation) + |> map { Optional($0) } + } + } else { + return .single(nil) } } |> deliverOn(self.dataQueue) context.disposable = signal.start(next: { [weak self] next in - rename(next.temporaryPath, path) - - if let strongSelf = self, let context = strongSelf.cachedRepresentationContexts[key] { - strongSelf.cachedRepresentationContexts.removeValue(forKey: key) - if let size = fileSize(path) { + if let next = next { + rename(next.temporaryPath, path) + + if let strongSelf = self, let context = strongSelf.cachedRepresentationContexts[key] { + strongSelf.cachedRepresentationContexts.removeValue(forKey: key) + if let size = fileSize(path) { + for subscriber in context.dataSubscribers.copyItems() { + subscriber(MediaResourceData(path: path, size: size, complete: true)) + } + } + } + } else { + if let strongSelf = self, let context = strongSelf.cachedRepresentationContexts[key] { for subscriber in context.dataSubscribers.copyItems() { - subscriber(MediaResourceData(path: path, size: size, complete: true)) + subscriber(MediaResourceData(path: path, size: 0, complete: false)) } } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 33b858a76f..d9cd74f5be 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -337,7 +337,26 @@ public final class Modifier { public func searchItemCollection(namespace: ItemCollectionId.Namespace, key: MemoryBuffer) -> [ItemCollectionItem] { assert(!self.disposed) if let postbox = self.postbox { - return postbox.itemCollectionItemTable.exactIndexedItems(namespace: namespace, key: ValueBoxKey(key)) + let itemsByCollectionId = postbox.itemCollectionItemTable.exactIndexedItems(namespace: namespace, key: ValueBoxKey(key)) + let infoIds = postbox.itemCollectionInfoTable.getIds(namespace: namespace) + var infoIndices: [ItemCollectionId: Int] = [:] + for i in 0 ..< infoIds.count { + infoIndices[infoIds[i]] = i + } + let sortedKeys = itemsByCollectionId.keys.sorted(by: { lhs, rhs in + if let lhsIndex = infoIndices[lhs], let rhsIndex = infoIndices[rhs] { + return lhsIndex < rhsIndex + } else if let _ = infoIndices[lhs] { + return true + } else { + return false + } + }) + var result: [ItemCollectionItem] = [] + for key in sortedKeys { + result.append(contentsOf: itemsByCollectionId[key]!) + } + return result } else { return [] } From 13bc5bbebc97c2f36adb087113a9c3b1e6552443 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 30 Mar 2017 18:00:34 +0300 Subject: [PATCH 122/427] no message --- Postbox/MessageHistoryIndexTable.swift | 46 +-- Postbox/MessageHistoryOperation.swift | 2 +- Postbox/MessageHistoryTable.swift | 73 +++- Postbox/MessageHistoryTagsTable.swift | 9 + Postbox/MessageHistoryView.swift | 376 +++++++++++++++---- Postbox/MessageView.swift | 10 +- Postbox/PeerChatTopIndexableMessageIds.swift | 2 +- Postbox/Postbox.swift | 64 +--- 8 files changed, 433 insertions(+), 149 deletions(-) diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 001bceaa31..629d1497b1 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -63,7 +63,7 @@ public enum AddMessagesLocation { enum MessageHistoryIndexOperation { case InsertMessage(InternalStoreMessage) case InsertHole(MessageHistoryHole) - case Remove(MessageIndex) + case Remove(index: MessageIndex, isMessage: Bool) case Update(MessageIndex, InternalStoreMessage) case UpdateTimestamp(MessageIndex, Int32) } @@ -171,7 +171,7 @@ final class MessageHistoryIndexTable: Table { switch lowerItem { case let .Hole(lowerHole): if lowerHole.tags != MessageTags.All.rawValue { - self.justRemove(lowerHole.maxIndex, operations: &operations) + self.justRemove(lowerHole.maxIndex, isMessage: false, operations: &operations) self.justInsertHole(MessageHistoryHole(stableId: lowerHole.stableId, maxIndex: lowerHole.maxIndex, min: lowerHole.min, tags: MessageTags.All.rawValue), operations: &operations) } case let .Message(lowerMessage): @@ -251,11 +251,11 @@ final class MessageHistoryIndexTable: Table { }, limit: 0) for index in removeHoles { - self.justRemove(index, operations: &operations) + self.justRemove(index, isMessage: false, operations: &operations) } if let modifyHole = modifyHole { - self.justRemove(modifyHole.0, operations: &operations) + self.justRemove(modifyHole.0, isMessage: false, operations: &operations) self.justInsertHole(modifyHole.1, operations: &operations) } } @@ -278,7 +278,7 @@ final class MessageHistoryIndexTable: Table { if let upperItem = upperItem { switch upperItem { case let .Hole(upperHole): - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) if upperHole.maxIndex.id.id > index.id.id + 1 { self.justInsertHole(MessageHistoryHole(stableId: self.metadataTable.getNextStableMessageIndexId(), maxIndex: upperHole.maxIndex, min: index.id.id + 1, tags: upperHole.tags), operations: &operations) } @@ -301,8 +301,8 @@ final class MessageHistoryIndexTable: Table { func removeMessage(_ id: MessageId, operations: inout [MessageHistoryIndexOperation]) { self.ensureInitialized(id.peerId, operations: &operations) - if let existingEntry = self.get(id) { - self.justRemove(existingEntry.index, operations: &operations) + if let existingEntry = self.get(id), case .Message = existingEntry { + self.justRemove(existingEntry.index, isMessage: true, operations: &operations) let adjacent = self.adjacentItems(id) @@ -311,7 +311,7 @@ final class MessageHistoryIndexTable: Table { case let .Message(lowerMessage): switch upperItem { case let .Hole(upperHole): - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) self.justInsertHole(MessageHistoryHole(stableId: upperHole.stableId, maxIndex: upperHole.maxIndex, min: lowerMessage.id.id + 1, tags: upperHole.tags), operations: &operations) case .Message: break @@ -319,18 +319,18 @@ final class MessageHistoryIndexTable: Table { case let .Hole(lowerHole): switch upperItem { case let .Hole(upperHole): - self.justRemove(lowerHole.maxIndex, operations: &operations) - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(lowerHole.maxIndex, isMessage: false, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) self.justInsertHole(MessageHistoryHole(stableId: upperHole.stableId, maxIndex: upperHole.maxIndex, min: lowerHole.min, tags: upperHole.tags | lowerHole.tags), operations: &operations) case let .Message(upperMessage): - self.justRemove(lowerHole.maxIndex, operations: &operations) + self.justRemove(lowerHole.maxIndex, isMessage: false, operations: &operations) self.justInsertHole(MessageHistoryHole(stableId: lowerHole.stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerHole.min, tags: lowerHole.tags), operations: &operations) } } } else if let lowerItem = adjacent.lower { switch lowerItem { case let .Hole(lowerHole): - self.justRemove(lowerHole.maxIndex, operations: &operations) + self.justRemove(lowerHole.maxIndex, isMessage: false, operations: &operations) self.justInsertHole(MessageHistoryHole(stableId: lowerHole.stableId, maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerHole.min, tags: lowerHole.tags), operations: &operations) break case .Message: @@ -339,7 +339,7 @@ final class MessageHistoryIndexTable: Table { } else if let upperItem = adjacent.upper { switch upperItem { case let .Hole(upperHole): - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) self.justInsertHole(MessageHistoryHole(stableId: upperHole.stableId, maxIndex: upperHole.maxIndex, min: 1, tags: upperHole.tags), operations: &operations) break case .Message: @@ -358,7 +358,7 @@ final class MessageHistoryIndexTable: Table { for operation in intermediateOperations { switch operation { - case let .Remove(index) where index == previousIndex: + case let .Remove(index, _) where index == previousIndex: operations.append(.Update(previousIndex, message)) case let .InsertMessage(insertMessage) where MessageIndex(insertMessage) == MessageIndex(message): break @@ -589,10 +589,10 @@ final class MessageHistoryIndexTable: Table { } if messagesInRange.isEmpty { - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) self.justInsertHole(MessageHistoryHole(stableId: upperHole.stableId, maxIndex: upperHole.maxIndex, min: upperHole.min, tags: upperHole.tags & ~tagMask.rawValue), operations: &operations) } else { - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) let clearedTags = upperHole.tags & ~tagMask.rawValue @@ -644,7 +644,7 @@ final class MessageHistoryIndexTable: Table { if (fillType.complete || fillType.direction == .UpperToLower) { if !removedHole { removedHole = true - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) } } } @@ -654,14 +654,14 @@ final class MessageHistoryIndexTable: Table { if (fillType.complete || fillType.direction == .LowerToUpper) { if !removedHole { removedHole = true - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) } } } if message.id == upperHole.maxIndex.id { removedHole = true - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) } self.justInsertMessage(message, operations: &operations) @@ -673,7 +673,7 @@ final class MessageHistoryIndexTable: Table { if fillType.complete { if !removedHole { removedHole = true - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) } } else if fillType.direction == .LowerToUpper { if let maxMessageInRange = maxMessageInRange , maxMessageInRange.id.id != Int32.max && maxMessageInRange.id.id + 1 <= upperHole.maxIndex.id.id { @@ -699,7 +699,7 @@ final class MessageHistoryIndexTable: Table { } } else if case let .AroundIndex(_, lowerComplete, upperComplete) = fillType.direction { if !removedHole { - self.justRemove(upperHole.maxIndex, operations: &operations) + self.justRemove(upperHole.maxIndex, isMessage: false, operations: &operations) removedHole = true } @@ -764,10 +764,10 @@ final class MessageHistoryIndexTable: Table { } } - private func justRemove(_ index: MessageIndex, operations: inout [MessageHistoryIndexOperation]) { + private func justRemove(_ index: MessageIndex, isMessage: Bool, operations: inout [MessageHistoryIndexOperation]) { self.valueBox.remove(self.table, key: self.key(index.id)) - operations.append(.Remove(index)) + operations.append(.Remove(index: index, isMessage: isMessage)) if index.id.namespace == self.globalMessageIdsNamespace { self.globalMessageIdsTable.remove(index.id.id) } diff --git a/Postbox/MessageHistoryOperation.swift b/Postbox/MessageHistoryOperation.swift index 4bfef815ae..673ab88b90 100644 --- a/Postbox/MessageHistoryOperation.swift +++ b/Postbox/MessageHistoryOperation.swift @@ -3,7 +3,7 @@ import Foundation enum MessageHistoryOperation { case InsertMessage(IntermediateMessage) case InsertHole(MessageHistoryHole) - case Remove([MessageIndex]) + case Remove([(MessageIndex, Bool, MessageTags)]) case UpdateReadState(CombinedPeerReadState) case UpdateEmbeddedMedia(MessageIndex, ReadBuffer) case UpdateTimestamp(MessageIndex, Int32) diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 26bc3e9b2f..5d73514d3d 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -110,11 +110,10 @@ final class MessageHistoryTable: Table { return dict } - private func processIndexOperationsCommitAccumulatedRemoveIndices(remove: Bool, peerId: PeerId, accumulatedRemoveIndices: inout [MessageIndex], updatedCombinedState: inout CombinedPeerReadState?, invalidateReadState: inout Bool, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], outputOperations: inout [MessageHistoryOperation]) { + private func processIndexOperationsCommitAccumulatedRemoveIndices(remove: Bool, peerId: PeerId, accumulatedRemoveIndices: inout [(MessageIndex, Bool)], updatedCombinedState: inout CombinedPeerReadState?, invalidateReadState: inout Bool, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], outputOperations: inout [MessageHistoryOperation]) { if !accumulatedRemoveIndices.isEmpty { - outputOperations.append(.Remove(accumulatedRemoveIndices)) - let (combinedState, invalidate) = self.readStateTable.deleteMessages(peerId, indices: accumulatedRemoveIndices, incomingStatsInIndices: { peerId, namespace, indices in + let (combinedState, invalidate) = self.readStateTable.deleteMessages(peerId, indices: accumulatedRemoveIndices.map { $0.0 }, incomingStatsInIndices: { peerId, namespace, indices in return self.incomingMessageStatsInIndices(peerId, namespace: namespace, indices: indices) }) if let combinedState = combinedState { @@ -125,9 +124,30 @@ final class MessageHistoryTable: Table { } if remove { - for index in accumulatedRemoveIndices { - self.justRemove(index, unsentMessageOperations: &unsentMessageOperations) + var indicesWithMetadata: [(MessageIndex, Bool, MessageTags)] = [] + if accumulatedRemoveIndices.count == 1 && accumulatedRemoveIndices[0].0.timestamp == Int32.max { + assert(true) } + for index in accumulatedRemoveIndices { + let tags = self.justRemove(index.0, unsentMessageOperations: &unsentMessageOperations) + if let tags = tags { + indicesWithMetadata.append((index.0, index.1, tags)) + } else { + indicesWithMetadata.append((index.0, index.1, MessageTags())) + } + } + assert(accumulatedRemoveIndices.count == indicesWithMetadata.count) + outputOperations.append(.Remove(indicesWithMetadata)) + } else { + var indicesWithMetadata: [(MessageIndex, Bool, MessageTags)] = [] + for index in accumulatedRemoveIndices { + let tags = self.tagsForIndex(index.0) + if let tags = tags { + indicesWithMetadata.append((index.0, index.1, tags)) + } + } + assert(accumulatedRemoveIndices.count == indicesWithMetadata.count) + outputOperations.append(.Remove(indicesWithMetadata)) } accumulatedRemoveIndices.removeAll() @@ -140,7 +160,7 @@ final class MessageHistoryTable: Table { let sharedEncoder = Encoder() var outputOperations: [MessageHistoryOperation] = [] - var accumulatedRemoveIndices: [MessageIndex] = [] + var accumulatedRemoveIndices: [(MessageIndex, Bool)] = [] var accumulatedAddedIncomingMessageIndices = Set() var updatedCombinedState: CombinedPeerReadState? @@ -206,21 +226,21 @@ final class MessageHistoryTable: Table { if message.flags.contains(.Incoming) { accumulatedAddedIncomingMessageIndices.insert(MessageIndex(message)) } - case let .Remove(index): + case let .Remove(index, isMessage): commitAccumulatedAddedIndices() - accumulatedRemoveIndices.append(index) + accumulatedRemoveIndices.append((index, isMessage)) case let .Update(index, storeMessage): commitAccumulatedAddedIndices() processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) - if let message = self.justUpdate(index, message: storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder, unsentMessageOperations: &unsentMessageOperations) { - outputOperations.append(.Remove([index])) + if let (message, previousTags) = self.justUpdate(index, message: storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder, unsentMessageOperations: &unsentMessageOperations) { + outputOperations.append(.Remove([(index, true, previousTags)])) outputOperations.append(.InsertMessage(message)) if message.flags.contains(.Incoming) { if index != MessageIndex(message) { - accumulatedRemoveIndices.append(index) + accumulatedRemoveIndices.append((index, true)) accumulatedAddedIncomingMessageIndices.insert(MessageIndex(message)) } } @@ -679,9 +699,24 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.table, key: self.key(hole.maxIndex), value: sharedBuffer.readBufferNoCopy()) } - private func justRemove(_ index: MessageIndex, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) { + private func tagsForIndex(_ index: MessageIndex) -> MessageTags? { let key = self.key(index) if let value = self.valueBox.get(self.table, key: key) { + switch self.readIntermediateEntry(key, value: value) { + case let .Message(message): + return message.tags + case let .Hole(hole): + return MessageTags(rawValue: hole.tags) + } + } else { + return nil + } + } + + private func justRemove(_ index: MessageIndex, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) -> MessageTags? { + let key = self.key(index) + if let value = self.valueBox.get(self.table, key: key) { + let resultTags: MessageTags switch self.readIntermediateEntry(key, value: value) { case let .Message(message): let embeddedMediaData = message.embeddedMediaData @@ -724,6 +759,8 @@ final class MessageHistoryTable: Table { for mediaId in message.referencedMedia { let _ = self.messageMediaTable.removeReference(mediaId) } + + resultTags = message.tags case let .Hole(hole): let tags = self.messageHistoryIndexTable.seedConfiguration.existingMessageTags.rawValue & hole.tags if tags != 0 { @@ -739,9 +776,13 @@ final class MessageHistoryTable: Table { } } } + resultTags = MessageTags(rawValue: hole.tags) } self.valueBox.remove(self.table, key: key) + return resultTags + } else { + return nil } } @@ -839,7 +880,7 @@ final class MessageHistoryTable: Table { }) } - private func justUpdate(_ index: MessageIndex, message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) -> IntermediateMessage? { + private func justUpdate(_ index: MessageIndex, message: InternalStoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) -> (IntermediateMessage, MessageTags)? { if let previousMessage = self.getMessage(index) { var previousEmbeddedMediaWithIds: [(MediaId, Media)] = [] if previousMessage.embeddedMediaData.length > 4 { @@ -1046,7 +1087,7 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.table, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) - return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: flags, tags: tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) + return (IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: flags, tags: tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia), previousMessage.tags) } else { return nil } @@ -1713,6 +1754,10 @@ final class MessageHistoryTable: Table { }, limit: 0) return indices } + + func getMessageCountInRange(peerId: PeerId, tagMask: MessageTags, lowerBound: MessageIndex, upperBound: MessageIndex) -> Int32 { + return self.tagsTable.getMessageCountInRange(tagMask: tagMask, peerId: peerId, lowerBound: lowerBound, upperBound: upperBound) + } func debugList(_ peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] diff --git a/Postbox/MessageHistoryTagsTable.swift b/Postbox/MessageHistoryTagsTable.swift index 3514b13ae2..ff1c142115 100644 --- a/Postbox/MessageHistoryTagsTable.swift +++ b/Postbox/MessageHistoryTagsTable.swift @@ -135,4 +135,13 @@ class MessageHistoryTagsTable: Table { }, limit: count) return indices } + + func getMessageCountInRange(tagMask: MessageTags, peerId: PeerId, lowerBound: MessageIndex, upperBound: MessageIndex) -> Int32 { + var count: Int32 = 0 + self.valueBox.range(self.table, start: self.key(tagMask, index: lowerBound).predecessor, end: self.key(tagMask, index: upperBound.successor()), keys: { _ in + count += 1 + return true + }, limit: 0) + return count + } } diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 8aa7c48a48..27dfeb965d 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -29,40 +29,141 @@ public func ==(lhs: MessageHistoryViewId, rhs: MessageHistoryViewId) -> Bool { } enum MutableMessageHistoryEntry { - case IntermediateMessageEntry(IntermediateMessage, MessageHistoryEntryLocation?) - case MessageEntry(Message, MessageHistoryEntryLocation?) + case IntermediateMessageEntry(IntermediateMessage, MessageHistoryEntryLocation?, MessageHistoryEntryMonthLocation?) + case MessageEntry(Message, MessageHistoryEntryLocation?, MessageHistoryEntryMonthLocation?) case HoleEntry(MessageHistoryHole, MessageHistoryEntryLocation?) var index: MessageIndex { switch self { - case let .IntermediateMessageEntry(message, _): + case let .IntermediateMessageEntry(message, _, _): return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .MessageEntry(message, _): + case let .MessageEntry(message, _, _): return MessageIndex(id: message.id, timestamp: message.timestamp) case let .HoleEntry(hole, _): return hole.maxIndex } } + var tags: MessageTags { + switch self { + case let .IntermediateMessageEntry(message, _, _): + return message.tags + case let .MessageEntry(message, _, _): + return message.tags + case let .HoleEntry(hole, _): + return MessageTags(rawValue: hole.tags) + } + } + func updatedLocation(_ location: MessageHistoryEntryLocation?) -> MutableMessageHistoryEntry { switch self { - case let .IntermediateMessageEntry(message, _): - return .IntermediateMessageEntry(message, location) - case let .MessageEntry(message, _): - return .MessageEntry(message, location) + case let .IntermediateMessageEntry(message, _, monthLocation): + return .IntermediateMessageEntry(message, location, monthLocation) + case let .MessageEntry(message, _, monthLocation): + return .MessageEntry(message, location, monthLocation) case let .HoleEntry(hole, _): return .HoleEntry(hole, location) } } + func updatedMonthLocation(_ monthLocation: MessageHistoryEntryMonthLocation?) -> MutableMessageHistoryEntry { + switch self { + case let .IntermediateMessageEntry(message, location, _): + return .IntermediateMessageEntry(message, location, monthLocation) + case let .MessageEntry(message, location, _): + return .MessageEntry(message, location, monthLocation) + case .HoleEntry: + return self + } + } + + func offsetLocationForInsertedIndex(_ index: MessageIndex, isMessage: Bool) -> MutableMessageHistoryEntry { + switch self { + case let .HoleEntry(hole, location): + if let location = location { + if hole.maxIndex > index { + return .HoleEntry(hole, MessageHistoryEntryLocation(index: location.index + 1, count: location.count + 1)) + } else { + return .HoleEntry(hole, MessageHistoryEntryLocation(index: location.index, count: location.count + 1)) + } + } else { + return self + } + case let .IntermediateMessageEntry(message, location, monthLocation): + if let location = location { + if MessageIndex(id: message.id, timestamp: message.timestamp) > index { + return .IntermediateMessageEntry(message, MessageHistoryEntryLocation(index: location.index + 1, count: location.count + 1), monthLocation) + } else { + return .IntermediateMessageEntry(message, MessageHistoryEntryLocation(index: location.index, count: location.count - 1), monthLocation) + } + } else { + return self + } + case let .MessageEntry(message, location, monthLocation): + if let location = location { + if MessageIndex(id: message.id, timestamp: message.timestamp) > index { + return .MessageEntry(message, MessageHistoryEntryLocation(index: location.index + 1, count: location.count + 1), monthLocation) + } else { + return .MessageEntry(message, MessageHistoryEntryLocation(index: location.index, count: location.count + 1), monthLocation) + } + } else { + return self + } + } + } + + func offsetLocationForRemovedIndex(_ index: MessageIndex, wasMessage: Bool) -> MutableMessageHistoryEntry { + switch self { + case let .HoleEntry(hole, location): + if let location = location { + if hole.maxIndex > index { + assert(location.index > 0) + assert(location.count != 0) + return .HoleEntry(hole, MessageHistoryEntryLocation(index: location.index - 1, count: location.count - 1)) + } else { + assert(location.count != 0) + return .HoleEntry(hole, MessageHistoryEntryLocation(index: location.index, count: location.count - 1)) + } + } else { + return self + } + case let .IntermediateMessageEntry(message, location, monthLocation): + if let location = location { + if MessageIndex(id: message.id, timestamp: message.timestamp) > index { + assert(location.index > 0) + assert(location.count != 0) + return .IntermediateMessageEntry(message, MessageHistoryEntryLocation(index: location.index - 1, count: location.count - 1), monthLocation) + } else { + assert(location.count != 0) + return .IntermediateMessageEntry(message, MessageHistoryEntryLocation(index: location.index, count: location.count - 1), monthLocation) + } + } else { + return self + } + case let .MessageEntry(message, location, monthLocation): + if let location = location { + if MessageIndex(id: message.id, timestamp: message.timestamp) > index { + assert(location.index > 0) + assert(location.count != 0) + return .MessageEntry(message, MessageHistoryEntryLocation(index: location.index - 1, count: location.count - 1), monthLocation) + } else { + assert(location.count != 0) + return .MessageEntry(message, MessageHistoryEntryLocation(index: location.index, count: location.count - 1), monthLocation) + } + } else { + return self + } + } + } + func updatedTimestamp(_ timestamp: Int32) -> MutableMessageHistoryEntry { switch self { - case let .IntermediateMessageEntry(message, location): + case let .IntermediateMessageEntry(message, location, monthLocation): let updatedMessage = IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) - return .IntermediateMessageEntry(updatedMessage, location) - case let .MessageEntry(message, location): + return .IntermediateMessageEntry(updatedMessage, location, monthLocation) + case let .MessageEntry(message, location, monthLocation): let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) - return .MessageEntry(updatedMessage, location) + return .MessageEntry(updatedMessage, location, monthLocation) case let .HoleEntry(hole, location): let updatedHole = MessageHistoryHole(stableId: hole.stableId, maxIndex: MessageIndex(id: hole.maxIndex.id, timestamp: timestamp), min: hole.min, tags: hole.tags) return .HoleEntry(updatedHole, location) @@ -82,26 +183,30 @@ public struct MessageHistoryEntryLocation: Equatable { } } - var successor: MessageHistoryEntryLocation? { - if index == count - 1 { - return nil - } else { - return MessageHistoryEntryLocation(index: index + 1, count: count) - } + var successor: MessageHistoryEntryLocation { + return MessageHistoryEntryLocation(index: index + 1, count: count) + } + + public static func ==(lhs: MessageHistoryEntryLocation, rhs: MessageHistoryEntryLocation) -> Bool { + return lhs.index == rhs.index && lhs.count == rhs.count } } -public func ==(lhs: MessageHistoryEntryLocation, rhs: MessageHistoryEntryLocation) -> Bool { - return lhs.index == rhs.index && lhs.count == rhs.count +public struct MessageHistoryEntryMonthLocation: Equatable { + public let indexInMonth: Int32 + + public static func ==(lhs: MessageHistoryEntryMonthLocation, rhs: MessageHistoryEntryMonthLocation) -> Bool { + return lhs.indexInMonth == rhs.indexInMonth + } } public enum MessageHistoryEntry: Comparable { - case MessageEntry(Message, Bool, MessageHistoryEntryLocation?) + case MessageEntry(Message, Bool, MessageHistoryEntryLocation?, MessageHistoryEntryMonthLocation?) case HoleEntry(MessageHistoryHole, MessageHistoryEntryLocation?) public var index: MessageIndex { switch self { - case let .MessageEntry(message, _, _): + case let .MessageEntry(message, _, _, _): return MessageIndex(id: message.id, timestamp: message.timestamp) case let .HoleEntry(hole, _): return hole.maxIndex @@ -111,12 +216,12 @@ public enum MessageHistoryEntry: Comparable { public func ==(lhs: MessageHistoryEntry, rhs: MessageHistoryEntry) -> Bool { switch lhs { - case let .MessageEntry(lhsMessage, lhsRead, lhsLocation): + case let .MessageEntry(lhsMessage, lhsRead, lhsLocation, lhsMonthLocation): switch rhs { case .HoleEntry: return false - case let .MessageEntry(rhsMessage, rhsRead, rhsLocation): - if MessageIndex(lhsMessage) == MessageIndex(rhsMessage) && lhsMessage.flags == rhsMessage.flags && lhsLocation == rhsLocation && lhsRead == rhsRead { + case let .MessageEntry(rhsMessage, rhsRead, rhsLocation, rhsMonthLocation): + if MessageIndex(lhsMessage) == MessageIndex(rhsMessage) && lhsMessage.flags == rhsMessage.flags && lhsLocation == rhsLocation && lhsRead == rhsRead && lhsMonthLocation != rhsMonthLocation { return true } return false @@ -188,9 +293,75 @@ public enum MessageHistoryViewRelativeHoleDirection: Equatable { } } +public struct MessageHistoryViewOrderStatistics: OptionSet { + public var rawValue: Int32 + + public init() { + self.rawValue = 0 + } + + public init(rawValue: Int32) { + self.rawValue = rawValue + } + + public static let combinedLocation = MessageHistoryViewOrderStatistics(rawValue: 1 << 0) + public static let locationWithinMonth = MessageHistoryViewOrderStatistics(rawValue: 1 << 1) +} + +private struct MessageMonthIndex: Equatable { + let year: Int32 + let month: Int32 + + var timestamp: Int32 { + var timeinfo = tm() + timeinfo.tm_year = self.year + timeinfo.tm_mon = self.month + return Int32(timegm(&timeinfo)) + } + + init(year: Int32, month: Int32) { + self.year = year + self.month = month + } + + init(timestamp: Int32) { + var t = Int(timestamp) + var timeinfo = tm() + gmtime_r(&t, &timeinfo) + self.year = timeinfo.tm_year + self.month = timeinfo.tm_mon + } + + static func ==(lhs: MessageMonthIndex, rhs: MessageMonthIndex) -> Bool { + return lhs.month == rhs.month && lhs.year == rhs.year + } + + var successor: MessageMonthIndex { + if self.month == 11 { + return MessageMonthIndex(year: self.year + 1, month: 0) + } else { + return MessageMonthIndex(year: self.year, month: self.month + 1) + } + } + + var predecessor: MessageMonthIndex { + if self.month == 0 { + return MessageMonthIndex(year: self.year - 1, month: 11) + } else { + return MessageMonthIndex(year: self.year, month: self.month - 1) + } + } +} + +private func monthUpperBoundIndex(peerId: PeerId, index: MessageMonthIndex) -> MessageIndex { + return MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: index.successor.timestamp) +} + final class MutableMessageHistoryView { private(set) var id: MessageHistoryViewId let tagMask: MessageTags? + private let orderStatistics: MessageHistoryViewOrderStatistics + fileprivate var anchorIndex: MessageHistoryAnchorIndex fileprivate let combinedReadState: CombinedPeerReadState? fileprivate var transientReadState: CombinedPeerReadState? @@ -202,8 +373,9 @@ final class MutableMessageHistoryView { fileprivate var topTaggedMessages: [MessageId.Namespace: MessageHistoryTopTaggedMessage?] fileprivate var additionalDatas: [AdditionalMessageHistoryViewDataEntry] - init(id: MessageHistoryViewId, anchorIndex: MessageHistoryAnchorIndex, combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int, topTaggedMessages: [MessageId.Namespace: MessageHistoryTopTaggedMessage?], additionalDatas: [AdditionalMessageHistoryViewDataEntry]) { + init(id: MessageHistoryViewId, postbox: Postbox, orderStatistics: MessageHistoryViewOrderStatistics, peerId: PeerId, anchorIndex: MessageHistoryAnchorIndex, combinedReadState: CombinedPeerReadState?, earlier: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry], later: MutableMessageHistoryEntry?, tagMask: MessageTags?, count: Int, topTaggedMessages: [MessageId.Namespace: MessageHistoryTopTaggedMessage?], additionalDatas: [AdditionalMessageHistoryViewDataEntry], getMessageCountInRange: (MessageIndex, MessageIndex) -> Int32) { self.id = id + self.orderStatistics = orderStatistics self.anchorIndex = anchorIndex self.combinedReadState = combinedReadState self.transientReadState = combinedReadState @@ -214,6 +386,45 @@ final class MutableMessageHistoryView { self.fillCount = count self.topTaggedMessages = topTaggedMessages self.additionalDatas = additionalDatas + + if let tagMask = self.tagMask, !orderStatistics.isEmpty && !self.entries.isEmpty { + if orderStatistics.contains(.combinedLocation) { + if let location = postbox.messageHistoryTagsTable.entryLocation(at: self.entries[0].index, tagMask: tagMask) { + self.entries[0] = self.entries[0].updatedLocation(location) + + var previousLocation = location + for i in 1 ..< self.entries.count { + previousLocation = previousLocation.successor + self.entries[i] = self.entries[i].updatedLocation(previousLocation) + } + } else { + assertionFailure() + } + } + + if orderStatistics.contains(.locationWithinMonth) { + var topMessageEntryIndex: Int? + var index = self.entries.count - 1 + loop: for entry in self.entries.reversed() { + switch entry { + case .IntermediateMessageEntry, .MessageEntry: + topMessageEntryIndex = index + break loop + default: + break + } + index -= 1 + } + if let topMessageEntryIndex = topMessageEntryIndex { + let topMessageIndex = self.entries[topMessageEntryIndex].index + + let monthIndex = MessageMonthIndex(timestamp: topMessageIndex.timestamp) + + let laterCount = postbox.messageHistoryTagsTable.getMessageCountInRange(tagMask: tagMask, peerId: peerId, lowerBound: topMessageIndex.successor(), upperBound: monthUpperBoundIndex(peerId: peerId, index: monthIndex)) + self.entries[topMessageEntryIndex] = self.entries[topMessageEntryIndex].updatedMonthLocation(MessageHistoryEntryMonthLocation(indexInMonth: laterCount)) + } + } + } } func incrementVersion() { @@ -320,12 +531,12 @@ final class MutableMessageHistoryView { } case let .InsertMessage(intermediateMessage): if tagMask == nil || (intermediateMessage.tags.rawValue & unwrappedTagMask) != 0 { - if self.add(.IntermediateMessageEntry(intermediateMessage, nil), holeFillDirections: holeFillDirections) { + if self.add(.IntermediateMessageEntry(intermediateMessage, nil, nil), holeFillDirections: holeFillDirections) { hasChanges = true } } case let .Remove(indices): - if self.remove(Set(indices), context: context) { + if self.remove(indices, context: context) { hasChanges = true } case let .UpdateReadState(combinedReadState): @@ -333,8 +544,8 @@ final class MutableMessageHistoryView { self.transientReadState = combinedReadState case let .UpdateEmbeddedMedia(index, embeddedMediaData): for i in 0 ..< self.entries.count { - if case let .IntermediateMessageEntry(message, _) = self.entries[i] , MessageIndex(message) == index { - self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), nil) + if case let .IntermediateMessageEntry(message, location, monthLocation) = self.entries[i] , MessageIndex(message) == index { + self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), location, monthLocation) hasChanges = true break } @@ -343,8 +554,8 @@ final class MutableMessageHistoryView { for i in 0 ..< self.entries.count { let entry = self.entries[i] if entry.index == index { - self.remove(Set([index]), context: context) - self.add(entry.updatedTimestamp(timestamp), holeFillDirections: [:]) + let _ = self.remove([(index, true, entry.tags)], context: context) + let _ = self.add(entry.updatedTimestamp(timestamp), holeFillDirections: [:]) hasChanges = true break } @@ -355,7 +566,7 @@ final class MutableMessageHistoryView { if !updatedMedia.isEmpty { for i in 0 ..< self.entries.count { switch self.entries[i] { - case let .MessageEntry(message, _): + case let .MessageEntry(message, location, monthLocation): var rebuild = false for media in message.media { if let mediaId = media.id, let _ = updatedMedia[mediaId] { @@ -376,10 +587,10 @@ final class MutableMessageHistoryView { } } let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) - self.entries[i] = .MessageEntry(updatedMessage, nil) + self.entries[i] = .MessageEntry(updatedMessage, location, monthLocation) hasChanges = true } - case let .IntermediateMessageEntry(message, _): + case let .IntermediateMessageEntry(message, location, monthLocation): var rebuild = false for mediaId in message.referencedMedia { if let media = updatedMedia[mediaId] , media?.id != mediaId { @@ -411,14 +622,14 @@ final class MutableMessageHistoryView { case let .InsertMessage(intermediateMessage): for i in 0 ..< self.entries.count { switch self.entries[i] { - case let .MessageEntry(message, location): + case let .MessageEntry(message, location, monthLocation): if message.associatedMessageIds.count != message.associatedMessages.count { if message.associatedMessageIds.contains(intermediateMessage.id) && message.associatedMessages[intermediateMessage.id] == nil { var updatedAssociatedMessages = message.associatedMessages let renderedMessage = renderIntermediateMessage(intermediateMessage) updatedAssociatedMessages[intermediateMessage.id] = renderedMessage let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId:message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) - self.entries[i] = .MessageEntry(updatedMessage, location) + self.entries[i] = .MessageEntry(updatedMessage, location, monthLocation) hasChanges = true } } @@ -446,7 +657,7 @@ final class MutableMessageHistoryView { } case let .Remove(indices): if !self.topTaggedMessages.isEmpty { - for index in indices { + for (index, _, _) in indices { if let maybeCurrentTopMessage = self.topTaggedMessages[index.id.namespace], let currentTopMessage = maybeCurrentTopMessage, index.id == currentTopMessage.id { let item: MessageHistoryTopTaggedMessage? = nil self.topTaggedMessages[index.id.namespace] = item @@ -460,7 +671,7 @@ final class MutableMessageHistoryView { for i in 0 ..< self.additionalDatas.count { switch self.additionalDatas[i] { - case let .cachedPeerData(peerId, data): + case let .cachedPeerData(peerId, _): if let updatedData = updatedCachedPeerData[peerId] { self.additionalDatas[i] = .cachedPeerData(peerId, updatedData) hasChanges = true @@ -472,9 +683,11 @@ final class MutableMessageHistoryView { } private func add(_ entry: MutableMessageHistoryEntry, holeFillDirections: [MessageIndex: HoleFillDirection]) -> Bool { + let updated: Bool + if self.entries.count == 0 { self.entries.append(entry) - return true + updated = true } else { let latestIndex = self.entries[self.entries.count - 1].index let earliestIndex = self.entries[0].index @@ -484,21 +697,21 @@ final class MutableMessageHistoryView { if index < earliestIndex { if self.earlier == nil || self.earlier!.index < index { self.entries.insert(entry, at: 0) - return true + updated = true } else { - return false + updated = false } } else if index > latestIndex { if let later = self.later { if index < later.index { self.entries.append(entry) - return true + updated = true } else { - return false + updated = false } } else { self.entries.append(entry) - return true + updated = true } } else if index != earliestIndex && index != latestIndex { var i = self.entries.count @@ -509,14 +722,39 @@ final class MutableMessageHistoryView { i -= 1 } self.entries.insert(entry, at: i) - return true + updated = true } else { - return false + updated = false } } + + if !self.orderStatistics.isEmpty { + let entryIndex = entry.index + let isMessage: Bool + switch entry { + case .HoleEntry: + isMessage = false + case .IntermediateMessageEntry, .MessageEntry: + isMessage = true + } + + if self.orderStatistics.contains(.combinedLocation) { + for i in 0 ..< self.entries.count { + if self.entries[i].index != entryIndex { + self.entries[i] = self.entries[i].offsetLocationForInsertedIndex(entryIndex, isMessage: isMessage) + } + } + } + if self.orderStatistics.contains(.locationWithinMonth) { + + } + } + + return updated } - private func remove(_ indices: Set, context: MutableMessageHistoryViewReplayContext) -> Bool { + private func remove(_ indicesAndFlags: [(MessageIndex, Bool, MessageTags)], context: MutableMessageHistoryViewReplayContext) -> Bool { + let indices = Set(indicesAndFlags.map { $0.0 }) var hasChanges = false if let earlier = self.earlier , indices.contains(earlier.index) { context.invalidEarlier = true @@ -536,16 +774,18 @@ final class MutableMessageHistoryView { if self.entries.count != 0 { var i = self.entries.count - 1 while i >= 0 { - if indices.contains(self.entries[i].index) { + let entry = self.entries[i] + if indices.contains(entry.index) { self.entries.remove(at: i) + context.removedEntries = true hasChanges = true } else { - switch self.entries[i] { - case let .MessageEntry(message, location): + switch entry { + case let .MessageEntry(message, location, monthLocation): if let updatedAssociatedMessages = message.associatedMessages.filteredOut(keysIn: ids) { let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) - self.entries[i] = .MessageEntry(updatedMessage, location) + self.entries[i] = .MessageEntry(updatedMessage, location, monthLocation) } default: break @@ -555,6 +795,16 @@ final class MutableMessageHistoryView { } } + if let tagMask = self.tagMask { + for (index, wasMessage, tags) in indicesAndFlags { + if (tagMask.rawValue & tags.rawValue) != 0 { + for i in 0 ..< self.entries.count { + self.entries[i] = self.entries[i].offsetLocationForRemovedIndex(index, wasMessage: wasMessage) + } + } + } + } + return hasChanges } @@ -669,16 +919,16 @@ final class MutableMessageHistoryView { } func render(_ renderIntermediateMessage: (IntermediateMessage) -> Message) { - if let earlier = self.earlier, case let .IntermediateMessageEntry(intermediateMessage, location) = earlier { - self.earlier = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) + if let earlier = self.earlier, case let .IntermediateMessageEntry(intermediateMessage, location, monthLocation) = earlier { + self.earlier = .MessageEntry(renderIntermediateMessage(intermediateMessage), location, monthLocation) } - if let later = self.later, case let .IntermediateMessageEntry(intermediateMessage, location) = later { - self.later = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) + if let later = self.later, case let .IntermediateMessageEntry(intermediateMessage, location, monthLocation) = later { + self.later = .MessageEntry(renderIntermediateMessage(intermediateMessage), location, monthLocation) } for i in 0 ..< self.entries.count { - if case let .IntermediateMessageEntry(intermediateMessage, location) = self.entries[i] { - self.entries[i] = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) + if case let .IntermediateMessageEntry(intermediateMessage, location, monthLocation) = self.entries[i] { + self.entries[i] = .MessageEntry(renderIntermediateMessage(intermediateMessage), location, monthLocation) } } @@ -767,14 +1017,14 @@ public final class MessageHistoryView { switch entry { case let .HoleEntry(hole, location): entries.append(.HoleEntry(hole, location)) - case let .MessageEntry(message, location): + case let .MessageEntry(message, location, monthLocation): let read: Bool if message.flags.contains(.Incoming) { read = false } else { read = transientReadState.isOutgoingMessageIndexRead(MessageIndex(message)) } - entries.append(.MessageEntry(message, read, location)) + entries.append(.MessageEntry(message, read, location, monthLocation)) case .IntermediateMessageEntry: assertionFailure("unexpected IntermediateMessageEntry in MessageHistoryView.init()") } @@ -784,8 +1034,8 @@ public final class MessageHistoryView { switch entry { case let .HoleEntry(hole, location): entries.append(.HoleEntry(hole, location)) - case let .MessageEntry(message, location): - entries.append(.MessageEntry(message, false, location)) + case let .MessageEntry(message, location, monthLocation): + entries.append(.MessageEntry(message, false, location, monthLocation)) case .IntermediateMessageEntry: assertionFailure("unexpected IntermediateMessageEntry in MessageHistoryView.init()") } @@ -863,7 +1113,7 @@ public final class MessageHistoryView { } if let _ = maxNamespaceIndex , index + 1 < entries.count { for i in index + 1 ..< entries.count { - if case let .MessageEntry(message, _, _) = entries[i] , !message.flags.contains(.Incoming) { + if case let .MessageEntry(message, _, _, _) = entries[i] , !message.flags.contains(.Incoming) { maxNamespaceIndex = MessageIndex(message) } else { break diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift index 3436e183bb..23bda53c5c 100644 --- a/Postbox/MessageView.swift +++ b/Postbox/MessageView.swift @@ -15,9 +15,13 @@ final class MutableMessageView { switch operation { case let .Remove(indices): if let message = self.message { - if indices.contains(MessageIndex(message)) { - self.message = nil - updated = true + let messageIndex = MessageIndex(message) + for (index, _, _) in indices { + if index == messageIndex { + self.message = nil + updated = true + break + } } } case let .InsertMessage(message): diff --git a/Postbox/PeerChatTopIndexableMessageIds.swift b/Postbox/PeerChatTopIndexableMessageIds.swift index 9f4bf66eab..4a6da515aa 100644 --- a/Postbox/PeerChatTopIndexableMessageIds.swift +++ b/Postbox/PeerChatTopIndexableMessageIds.swift @@ -81,7 +81,7 @@ final class PeerChatTopTaggedMessageIdsTable: Table { } } case let .Remove(indices): - for index in indices { + for (index, _, _) in indices { if let _ = self.get(peerId: index.id.peerId, namespace: index.id.namespace) { self.set(peerId: index.id.peerId, namespace: index.id.namespace, id: nil) } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index d9cd74f5be..499be08744 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -1040,7 +1040,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message, nil)) + entries.append(.IntermediateMessageEntry(message, nil, nil)) case let .Hole(index): entries.append(.HoleEntry(index, nil)) } @@ -1064,7 +1064,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message, nil)) + entries.append(.IntermediateMessageEntry(message, nil, nil)) case let .Hole(index): entries.append(.HoleEntry(index, nil)) } @@ -1074,7 +1074,7 @@ public final class Postbox { if let intermediateLower = intermediateLower { switch intermediateLower { case let .Message(message): - lower = .IntermediateMessageEntry(message, nil) + lower = .IntermediateMessageEntry(message, nil, nil) case let .Hole(index): lower = .HoleEntry(index, nil) } @@ -1084,17 +1084,13 @@ public final class Postbox { if let intermediateUpper = intermediateUpper { switch intermediateUpper { case let .Message(message): - upper = .IntermediateMessageEntry(message, nil) + upper = .IntermediateMessageEntry(message, nil, nil) case let .Hole(index): upper = .HoleEntry(index, nil) } } - if let tagMask = tagMask { - return addLocationsToMessageHistoryViewEntries(tagMask: tagMask, earlier: lower, later: upper, entries: entries) - } else { - return (entries: entries, lower: lower, upper: upper) - } + return (entries: entries, lower: lower, upper: upper) } private func fetchLaterHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { @@ -1108,7 +1104,7 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message, nil)) + entries.append(.IntermediateMessageEntry(message, nil, nil)) case let .Hole(index): entries.append(.HoleEntry(index, nil)) } @@ -1480,59 +1476,33 @@ public final class Postbox { } } - public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil, additionalData: [AdditionalMessageHistoryViewData]) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + public func aroundUnreadMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags?, orderStatistics: MessageHistoryViewOrderStatistics, additionalData: [AdditionalMessageHistoryViewData]) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { return self.modify(userInteractive: true, { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let maxReadIndex = self.messageHistoryTable.maxReadIndex(peerId) { index = maxReadIndex } - return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask, additionalData: additionalData) + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask, orderStatistics: orderStatistics, additionalData: additionalData) }) |> switchToLatest } - public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil, additionalData: [AdditionalMessageHistoryViewData] = []) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + public func aroundIdMessageHistoryViewForPeerId(_ peerId: PeerId, count: Int, messageId: MessageId, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags?, orderStatistics: MessageHistoryViewOrderStatistics, additionalData: [AdditionalMessageHistoryViewData] = []) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in var index = MessageHistoryAnchorIndex(index: MessageIndex.upperBound(peerId: peerId), exact: true) if let anchorIndex = self.messageHistoryTable.anchorIndex(messageId) { index = anchorIndex } - return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask, additionalData: additionalData) + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index.index, count: count, anchorIndex: index, unreadIndex: index.index, fixedCombinedReadState: nil, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask, orderStatistics: orderStatistics, additionalData: additionalData) } |> switchToLatest } - public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags? = nil, additionalData: [AdditionalMessageHistoryViewData] = []) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + public func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageIndex, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags?, orderStatistics: MessageHistoryViewOrderStatistics, additionalData: [AdditionalMessageHistoryViewData] = []) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { return self.modify { modifier -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> in - return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask, additionalData: additionalData) + return self.syncAroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, topTaggedMessageIdNamespaces: topTaggedMessageIdNamespaces, tagMask: tagMask, orderStatistics: orderStatistics, additionalData: additionalData) } |> switchToLatest } - private func addLocationsToMessageHistoryViewEntries(tagMask: MessageTags, earlier: MutableMessageHistoryEntry?, later: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry]) -> ([MutableMessageHistoryEntry], MutableMessageHistoryEntry?, MutableMessageHistoryEntry?) { - if let firstEntry = entries.first { - if let location = self.messageHistoryTagsTable.entryLocation(at: firstEntry.index, tagMask: tagMask) { - let mappedEarlier = earlier?.updatedLocation(location.predecessor) - var mappedEntries: [MutableMessageHistoryEntry] = [] - var previousLocation: MessageHistoryEntryLocation? - for i in 0 ..< entries.count { - if i == 0 { - mappedEntries.append(entries[i].updatedLocation(location)) - previousLocation = location - } else { - previousLocation = previousLocation?.successor - mappedEntries.append(entries[i].updatedLocation(previousLocation)) - } - } - previousLocation = previousLocation?.successor - let mappedLater = later?.updatedLocation(previousLocation) - return (mappedEntries, mappedEarlier, mappedLater) - } else { - return (entries, earlier, later) - } - } else { - return (entries, earlier, later) - } - } - - private func syncAroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags?, additionalData: [AdditionalMessageHistoryViewData]) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { + private func syncAroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, topTaggedMessageIdNamespaces: Set, tagMask: MessageTags?, orderStatistics: MessageHistoryViewOrderStatistics, additionalData: [AdditionalMessageHistoryViewData]) -> Signal<(MessageHistoryView, ViewUpdateType, InitialMessageHistoryData?), NoError> { let startTime = CFAbsoluteTimeGetCurrent() let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") @@ -1563,7 +1533,13 @@ public final class Postbox { } } - let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count, topTaggedMessages: topTaggedMessages, additionalDatas: additionalDataEntries) + let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), postbox: self, orderStatistics: orderStatistics, peerId: peerId, anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count, topTaggedMessages: topTaggedMessages, additionalDatas: additionalDataEntries, getMessageCountInRange: { lowerBound, upperBound in + if let tagMask = tagMask { + return self.messageHistoryTable.getMessageCountInRange(peerId: peerId, tagMask: tagMask, lowerBound: lowerBound, upperBound: upperBound) + } else { + return 0 + } + }) mutableView.render(self.renderIntermediateMessage) let initialUpdateType: ViewUpdateType From 6aba279ef7c0862854c465a75d62554a501b6069 Mon Sep 17 00:00:00 2001 From: overtake Date: Thu, 30 Mar 2017 18:16:49 +0300 Subject: [PATCH 123/427] no message --- Postbox/MediaBox.swift | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index e903f4a2cf..6616cdf65b 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -39,6 +39,11 @@ public enum ResourceDataRangeMode { case partial } +public enum FetchResourceSourceType { + case local + case remote +} + private struct ResourceStorePaths { let partial: String let complete: String @@ -523,7 +528,7 @@ public final class MediaBox { } } - public func fetchedResource(_ resource: MediaResource) -> Signal { + public func fetchedResource(_ resource: MediaResource, implNext:Bool = false) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -532,6 +537,9 @@ public final class MediaBox { let paths = self.storePathsForId(resource.id) if let _ = fileSize(paths.complete) { + if implNext { + subscriber.putNext(.local) + } subscriber.putCompletion() } else { let currentSize = fileSize(paths.partial) ?? 0 @@ -631,6 +639,10 @@ public final class MediaBox { let status: MediaResourceStatus if updatedData.complete { status = .Local + if implNext { + subscriber.putNext(.remote) + } + subscriber.putCompletion() } else { if let resourceSize = resource.size { status = .Fetching(progress: Float(updatedSize) / Float(resourceSize)) From 3fb0dc5421c78578e1a30d01b377aff396e9b29b Mon Sep 17 00:00:00 2001 From: Peter Date: Sat, 1 Apr 2017 16:19:54 +0300 Subject: [PATCH 124/427] no message --- Postbox/Database.swift | 9 ++++++--- Postbox/MessageHistoryView.swift | 7 ++++++- Postbox/Peer.swift | 8 ++++++++ Postbox/SqliteInterface.swift | 8 ++++++-- Postbox/SqliteValueBox.swift | 9 ++++++++- 5 files changed, 34 insertions(+), 7 deletions(-) diff --git a/Postbox/Database.swift b/Postbox/Database.swift index b0450cd1ac..dc9660fc2a 100644 --- a/Postbox/Database.swift +++ b/Postbox/Database.swift @@ -67,9 +67,12 @@ public final class Database { /// Default: `false`. /// /// - returns: A new database connection. - public init(_ location: Location = .InMemory, readonly: Bool = false) { + public init?(_ location: Location = .InMemory, readonly: Bool = false) { let flags = readonly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE - sqlite3_open_v2(location.description, &self.handle, flags | SQLITE_OPEN_FULLMUTEX, nil) + let res = sqlite3_open_v2(location.description, &self.handle, flags | SQLITE_OPEN_FULLMUTEX, nil) + if res != SQLITE_OK { + return nil + } } /// Initializes a new connection to a database. @@ -83,7 +86,7 @@ public final class Database { /// Default: `false`. /// /// - returns: A new database connection. - public convenience init(_ filename: String, readonly: Bool = false) { + public convenience init?(_ filename: String, readonly: Bool = false) { self.init(.URI(filename), readonly: readonly) } diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 27dfeb965d..523f1053a5 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -420,7 +420,12 @@ final class MutableMessageHistoryView { let monthIndex = MessageMonthIndex(timestamp: topMessageIndex.timestamp) - let laterCount = postbox.messageHistoryTagsTable.getMessageCountInRange(tagMask: tagMask, peerId: peerId, lowerBound: topMessageIndex.successor(), upperBound: monthUpperBoundIndex(peerId: peerId, index: monthIndex)) + let laterCount: Int32 + if self.later == nil { + laterCount = 0 + } else { + laterCount = postbox.messageHistoryTagsTable.getMessageCountInRange(tagMask: tagMask, peerId: peerId, lowerBound: topMessageIndex.successor(), upperBound: monthUpperBoundIndex(peerId: peerId, index: monthIndex)) + } self.entries[topMessageEntryIndex] = self.entries[topMessageEntryIndex].updatedMonthLocation(MessageHistoryEntryMonthLocation(indexInMonth: laterCount)) } } diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index 90bc134dc8..b2100dfd5d 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -100,3 +100,11 @@ public protocol Peer: class, Coding { func isEqual(_ other: Peer) -> Bool } + +public func arePeersEqual(_ lhs: Peer?, _ rhs: Peer?) -> Bool { + if let lhs = lhs, let rhs = rhs { + return lhs.isEqual(rhs) + } else { + return (lhs != nil) == (rhs != nil) + } +} diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index b8128a308c..8aef02210b 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -100,8 +100,12 @@ public final class SqliteStatementCursor { public final class SqliteInterface { private let database: Database - public init(databasePath: String) { - self.database = Database(databasePath) + public init?(databasePath: String) { + if let database = Database(databasePath) { + self.database = database + } else { + return nil + } } public func unlock(password: Data) -> Bool { diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index cb8fead940..5cc4f2e4c1 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -132,7 +132,14 @@ final class SqliteValueBox: ValueBox { let _ = try? FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) let path = basePath + "/db_sqlite" - let database = Database(path) + let database: Database + if let result = Database(path) { + database = result + } else { + assertionFailure() + let _ = try? FileManager.default.removeItem(atPath: path) + database = Database(path)! + } database.execute("PRAGMA cache_size=-2097152") database.execute("PRAGMA synchronous=NORMAL") From 4c6f23eb3272443f4e2ee5058063f95b56c12e7d Mon Sep 17 00:00:00 2001 From: overtake Date: Sat, 1 Apr 2017 16:23:32 +0300 Subject: [PATCH 125/427] no message --- Postbox.xcodeproj/project.pbxproj | 133 +++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 1 deletion(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index b096659bb8..d6d2227607 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -1165,6 +1165,133 @@ /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ + C22069C81E8EB4BF00E82730 /* AppStore */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 1; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + MACOSX_DEPLOYMENT_TARGET = 10.10; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + VALIDATE_PRODUCT = YES; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = AppStore; + }; + C22069C91E8EB4BF00E82730 /* AppStore */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CLANG_ENABLE_MODULES = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution"; + DEFINES_MODULE = YES; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + ENABLE_BITCODE = YES; + FRAMEWORK_SEARCH_PATHS = "$(inherited)"; + INFOPLIST_FILE = Postbox/Info.plist; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + OTHER_CFLAGS = ( + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_HAS_CODEC=1", + "-DSQLCIPHER_CRYPTO_CC=1", + "-DSQLITE_ENABLE_FTS3", + ); + PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; + PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; + SKIP_INSTALL = YES; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_REFLECTION_METADATA_LEVEL = none; + SWIFT_VERSION = 3.0; + }; + name = AppStore; + }; + C22069CA1E8EB4BF00E82730 /* AppStore */ = { + isa = XCBuildConfiguration; + buildSettings = { + FRAMEWORK_SEARCH_PATHS = ( + "$(SDKROOT)/Developer/Library/Frameworks", + "$(inherited)", + "/Users/peter/Documents/PostBoxTest/submodules/SSignalKit/build/Debug-iphoneos", + ); + INFOPLIST_FILE = PostboxTests/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; + }; + name = AppStore; + }; + C22069CB1E8EB4BF00E82730 /* AppStore */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_SUSPICIOUS_MOVES = YES; + CODE_SIGN_IDENTITY = "-"; + COMBINE_HIDPI_IMAGES = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + DEFINES_MODULE = YES; + DEVELOPMENT_TEAM = ""; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + FRAMEWORK_VERSION = A; + INFOPLIST_FILE = PostboxMac/Info.plist; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 10.11; + OTHER_CFLAGS = ( + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_HAS_CODEC=1", + "-DSQLCIPHER_CRYPTO_CC=1", + ); + PRODUCT_BUNDLE_IDENTIFIER = org.telegram.Telegram.PostboxMac; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; + }; + name = AppStore; + }; D086A5711CC0116A00F08284 /* Hockeyapp */ = { isa = XCBuildConfiguration; baseConfigurationReference = D07516491B2D9E2500AE42E0 /* Postbox.xcconfig */; @@ -1264,7 +1391,7 @@ CLANG_WARN_DOCUMENTATION_COMMENTS = YES; CLANG_WARN_INFINITE_RECURSION = YES; CLANG_WARN_SUSPICIOUS_MOVES = YES; - CODE_SIGN_IDENTITY = "-"; + CODE_SIGN_IDENTITY = "Mac Developer"; COMBINE_HIDPI_IMAGES = YES; DEBUG_INFORMATION_FORMAT = dwarf; DEFINES_MODULE = YES; @@ -1567,6 +1694,7 @@ buildConfigurations = ( D0B418181D7DFAF3004562A4 /* Debug */, D0B418191D7DFAF3004562A4 /* Release */, + C22069CB1E8EB4BF00E82730 /* AppStore */, D0B4181A1D7DFAF3004562A4 /* Hockeyapp */, ); defaultConfigurationIsVisible = 0; @@ -1577,6 +1705,7 @@ buildConfigurations = ( D0E3A75E1B28A7E300A402D9 /* Debug */, D0E3A75F1B28A7E300A402D9 /* Release */, + C22069C81E8EB4BF00E82730 /* AppStore */, D086A5711CC0116A00F08284 /* Hockeyapp */, ); defaultConfigurationIsVisible = 0; @@ -1587,6 +1716,7 @@ buildConfigurations = ( D0E3A7611B28A7E300A402D9 /* Debug */, D0E3A7621B28A7E300A402D9 /* Release */, + C22069C91E8EB4BF00E82730 /* AppStore */, D086A5721CC0116A00F08284 /* Hockeyapp */, ); defaultConfigurationIsVisible = 0; @@ -1597,6 +1727,7 @@ buildConfigurations = ( D0E3A7641B28A7E300A402D9 /* Debug */, D0E3A7651B28A7E300A402D9 /* Release */, + C22069CA1E8EB4BF00E82730 /* AppStore */, D086A5731CC0116A00F08284 /* Hockeyapp */, ); defaultConfigurationIsVisible = 0; From bd4e29ecbe3eb7734602b16d85ca9d5ab390e81a Mon Sep 17 00:00:00 2001 From: Peter Date: Sat, 1 Apr 2017 16:48:13 +0300 Subject: [PATCH 126/427] no message --- Postbox/Database.swift | 4 ++-- Postbox/SqliteInterface.swift | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Postbox/Database.swift b/Postbox/Database.swift index dc9660fc2a..8939a7c452 100644 --- a/Postbox/Database.swift +++ b/Postbox/Database.swift @@ -97,8 +97,8 @@ public final class Database { /// Executes a batch of SQL statements. /// /// - parameter SQL: A batch of zero or more semicolon-separated SQL statements. - public func execute(_ SQL: String) { - sqlite3_exec(self.handle, SQL, nil, nil, nil) + public func execute(_ SQL: String) -> Bool { + return sqlite3_exec(self.handle, SQL, nil, nil, nil) == SQLITE_OK } } diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index 8aef02210b..44a0f0e33d 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -102,6 +102,9 @@ public final class SqliteInterface { public init?(databasePath: String) { if let database = Database(databasePath) { + if !database.execute("pragma schema_version") { + return nil + } self.database = database } else { return nil From b192142d347955818a4a6dcd6003917e8cd0cc50 Mon Sep 17 00:00:00 2001 From: overtake Date: Mon, 3 Apr 2017 16:15:42 +0300 Subject: [PATCH 127/427] changes --- Postbox/SqliteInterface.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index 44a0f0e33d..5d392f28e3 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -102,9 +102,9 @@ public final class SqliteInterface { public init?(databasePath: String) { if let database = Database(databasePath) { - if !database.execute("pragma schema_version") { - return nil - } +// if !database.execute("pragma schema_version") { +// return nil +// } self.database = database } else { return nil From 403a723c6f433b940768c1d80a892df96f2b85dc Mon Sep 17 00:00:00 2001 From: Peter Date: Sat, 8 Apr 2017 17:22:56 +0300 Subject: [PATCH 128/427] no message --- Postbox.xcodeproj/project.pbxproj | 18 +- Postbox/IncrementalUpgrade.swift | 10 - .../PeerMergedOperationLogIndexTable.swift | 8 +- Postbox/PeerOperationLogTable.swift | 15 +- Postbox/Postbox.swift | 393 +++++++++--------- Postbox/PostboxUpgrade_12to13.swift | 11 + Postbox/SqliteInterface.swift | 3 - Postbox/SqliteValueBox.swift | 4 + Postbox/Upgrades.swift | 11 + Postbox/ValueBox.swift | 1 + 10 files changed, 261 insertions(+), 213 deletions(-) delete mode 100644 Postbox/IncrementalUpgrade.swift create mode 100644 Postbox/PostboxUpgrade_12to13.swift create mode 100644 Postbox/Upgrades.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index b096659bb8..7335250cd8 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -184,8 +184,6 @@ D0D511041D64D91C00A97B8A /* IpcNotifier.h in Headers */ = {isa = PBXBuildFile; fileRef = D0D511031D64D75200A97B8A /* IpcNotifier.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */; }; D0D949F51D35353900740E02 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; - D0DA44411E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */; }; - D0DA44421E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */; }; D0DA44481E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */; }; D0DA44491E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */; }; D0DF0C8F1D81A350008AEB01 /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DF0C8E1D81A350008AEB01 /* PeerView.swift */; }; @@ -200,6 +198,10 @@ D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; D0F019FD1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */; }; + D0F02CDF1E99223D0065DEE2 /* Upgrades.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F02CDE1E99223D0065DEE2 /* Upgrades.swift */; }; + D0F02CE01E99223E0065DEE2 /* Upgrades.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F02CDE1E99223D0065DEE2 /* Upgrades.swift */; }; + D0F02CE21E9922F50065DEE2 /* PostboxUpgrade_12to13.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F02CE11E9922F50065DEE2 /* PostboxUpgrade_12to13.swift */; }; + D0F02CE31E9922F50065DEE2 /* PostboxUpgrade_12to13.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F02CE11E9922F50065DEE2 /* PostboxUpgrade_12to13.swift */; }; D0F3CC721DDE1CDC008148FA /* ItemCacheTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */; }; D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */; }; D0F53BF31E794C6700117362 /* PeerChatStateView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F53BF21E794C6700117362 /* PeerChatStateView.swift */; }; @@ -378,7 +380,6 @@ D0D511031D64D75200A97B8A /* IpcNotifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = IpcNotifier.h; sourceTree = ""; }; D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessResourceTests.swift; sourceTree = ""; }; D0D949F41D35353900740E02 /* MappedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MappedFile.swift; sourceTree = ""; }; - D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IncrementalUpgrade.swift; sourceTree = ""; }; D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxAccess.swift; sourceTree = ""; }; D0DF0C8E1D81A350008AEB01 /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; @@ -394,6 +395,8 @@ D0E3A79D1B28B50400A402D9 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = ""; }; D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = ""; }; D0F019FC1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerMergedOperationLogView.swift; sourceTree = ""; }; + D0F02CDE1E99223D0065DEE2 /* Upgrades.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Upgrades.swift; sourceTree = ""; }; + D0F02CE11E9922F50065DEE2 /* PostboxUpgrade_12to13.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxUpgrade_12to13.swift; sourceTree = ""; }; D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheTable.swift; sourceTree = ""; }; D0F3CC731DDE1EB9008148FA /* ItemCacheMetaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCacheMetaTable.swift; sourceTree = ""; }; D0F53BF21E794C6700117362 /* PeerChatStateView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatStateView.swift; sourceTree = ""; }; @@ -519,7 +522,8 @@ D0DA443F1E4C7834005FDCA7 /* Upgrade */ = { isa = PBXGroup; children = ( - D0DA44401E4C7868005FDCA7 /* IncrementalUpgrade.swift */, + D0F02CDE1E99223D0065DEE2 /* Upgrades.swift */, + D0F02CE11E9922F50065DEE2 /* PostboxUpgrade_12to13.swift */, ); name = Upgrade; sourceTree = ""; @@ -902,7 +906,6 @@ D03229EF1E6B33FD0000AF9C /* SqliteInterface.swift in Sources */, D050F2671E4A5B5A00988324 /* TimestampBasedMessageAttributesTable.swift in Sources */, D050F2681E4A5B5A00988324 /* TimestampBasedMessageAttributesIndexTable.swift in Sources */, - D0DA44421E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */, D050F2691E4A5B5A00988324 /* MultiplePeersView.swift in Sources */, C2A315BE1E2E733900D89000 /* PeerMergedOperationLogIndexTable.swift in Sources */, C2A315BF1E2E733900D89000 /* PeerMergedOperationLogView.swift in Sources */, @@ -994,6 +997,7 @@ D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */, D0F7B1CA1E045C6A007EB8A5 /* MessageHistoryUnsentTable.swift in Sources */, D0BEAF6B1E54B5FB00BD963D /* AccountManagerMetadataTable.swift in Sources */, + D0F02CE01E99223E0065DEE2 /* Upgrades.swift in Sources */, D0F7B1D01E045C6A007EB8A5 /* KeychainTable.swift in Sources */, D0BEAF6E1E54B77900BD963D /* AccountManagerRecordTable.swift in Sources */, D0B4184F1D7DFE20004562A4 /* ChatListHolesView.swift in Sources */, @@ -1007,6 +1011,7 @@ D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, D073CE741DCBF3B4007511FD /* Peer.swift in Sources */, D073CE7A1DCBF3B4007511FD /* PeerReadState.swift in Sources */, + D0F02CE31E9922F50065DEE2 /* PostboxUpgrade_12to13.swift in Sources */, D073CE801DCBF3B4007511FD /* PeerChatInterfaceState.swift in Sources */, D0F7B1C81E045C6A007EB8A5 /* MessageHistoryOperation.swift in Sources */, D0B844031DAB91A7005F29E1 /* PeerView.swift in Sources */, @@ -1024,7 +1029,6 @@ D03229EE1E6B33FD0000AF9C /* SqliteInterface.swift in Sources */, D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, D0F3CC741DDE1EB9008148FA /* ItemCacheMetaTable.swift in Sources */, - D0DA44411E4C7868005FDCA7 /* IncrementalUpgrade.swift in Sources */, D08D451F1D5D2CA700A7428A /* RatingTable.swift in Sources */, D07CFF831DCA909100761F81 /* PeerChatInterfaceStateTable.swift in Sources */, D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */, @@ -1116,6 +1120,7 @@ D08775031E3E3E7400A97350 /* PreferencesEntry.swift in Sources */, D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */, D0BEAF6A1E54B5FB00BD963D /* AccountManagerMetadataTable.swift in Sources */, + D0F02CDF1E99223D0065DEE2 /* Upgrades.swift in Sources */, D08775001E3E3D9F00A97350 /* PreferencesTable.swift in Sources */, D0BEAF6D1E54B77900BD963D /* AccountManagerRecordTable.swift in Sources */, D03120F81DA53FF4006A2A60 /* PeerPresenceTable.swift in Sources */, @@ -1129,6 +1134,7 @@ D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, D0AAD1B31E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift in Sources */, D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */, + D0F02CE21E9922F50065DEE2 /* PostboxUpgrade_12to13.swift in Sources */, D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */, D0D510F41D63BA8400A97B8A /* PostboxTransaction.swift in Sources */, diff --git a/Postbox/IncrementalUpgrade.swift b/Postbox/IncrementalUpgrade.swift deleted file mode 100644 index e8431d30b2..0000000000 --- a/Postbox/IncrementalUpgrade.swift +++ /dev/null @@ -1,10 +0,0 @@ -import Foundation -#if os(macOS) - import SwiftSignalKitMac -#else - import SwiftSignalKit -#endif - -protocol IncrementalUpgrade { - func upgrade(from previous: ValueBox, tmpBasePath: String) -> Signal -} diff --git a/Postbox/PeerMergedOperationLogIndexTable.swift b/Postbox/PeerMergedOperationLogIndexTable.swift index a7cc240f14..e3a4762c4f 100644 --- a/Postbox/PeerMergedOperationLogIndexTable.swift +++ b/Postbox/PeerMergedOperationLogIndexTable.swift @@ -33,18 +33,20 @@ final class PeerMergedOperationLogIndexTable: Table { func remove(tag: PeerOperationLogTag, mergedIndices: [Int32]) { for index in mergedIndices { + assert(self.valueBox.exists(self.table, key: self.key(tag: tag, index: index))) self.valueBox.remove(self.table, key: self.key(tag: tag, index: index)) } } - func getTagLocalIndices(tag: PeerOperationLogTag, fromMergedIndex: Int32, limit: Int) -> [(PeerId, Int32)] { - var result: [(PeerId, Int32)] = [] + func getTagLocalIndices(tag: PeerOperationLogTag, fromMergedIndex: Int32, limit: Int) -> [(PeerId, Int32, Int32)] { + var result: [(PeerId, Int32, Int32)] = [] self.valueBox.range(self.table, start: self.key(tag: tag, index: fromMergedIndex == 0 ? 0 : fromMergedIndex - 1), end: self.key(tag: tag, index: Int32.max), values: { key, value in + assert(key.getUInt8(0) == tag.rawValue) var peerIdValue: Int64 = 0 var tagLocalIndexValue: Int32 = 0 value.read(&peerIdValue, offset: 0, length: 8) value.read(&tagLocalIndexValue, offset: 0, length: 4) - result.append((PeerId(peerIdValue), tagLocalIndexValue)) + result.append((PeerId(peerIdValue), tagLocalIndexValue, key.getInt32(1))) return true }, limit: limit) return result diff --git a/Postbox/PeerOperationLogTable.swift b/Postbox/PeerOperationLogTable.swift index 29b9403034..0722f6fc6f 100644 --- a/Postbox/PeerOperationLogTable.swift +++ b/Postbox/PeerOperationLogTable.swift @@ -121,7 +121,7 @@ final class PeerOperationLogTable: Table { private func key(peerId: PeerId, tag: PeerOperationLogTag, index: Int32) -> ValueBoxKey { let key = ValueBoxKey(length: 8 + 1 + 4) key.setInt64(0, value: peerId.toInt64()) - key.setUInt8(1, value: tag.rawValue) + key.setUInt8(8, value: tag.rawValue) key.setInt32(9, value: index) return key } @@ -249,7 +249,7 @@ final class PeerOperationLogTable: Table { func getMergedEntries(tag: PeerOperationLogTag, fromIndex: Int32, limit: Int) -> [PeerMergedOperationLogEntry] { var entries: [PeerMergedOperationLogEntry] = [] - for (peerId, tagLocalIndex) in self.mergedIndexTable.getTagLocalIndices(tag: tag, fromMergedIndex: fromIndex, limit: limit) { + for (peerId, tagLocalIndex, mergedIndex) in self.mergedIndexTable.getTagLocalIndices(tag: tag, fromMergedIndex: fromIndex, limit: limit) { if let value = self.valueBox.get(self.table, key: self.key(peerId: peerId, tag: tag, index: tagLocalIndex)) { if let entry = parseMergedEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, value) { entries.append(entry) @@ -257,6 +257,7 @@ final class PeerOperationLogTable: Table { assertionFailure() } } else { + self.mergedIndexTable.remove(tag: tag, mergedIndices: [mergedIndex]) assertionFailure() } } @@ -334,10 +335,12 @@ final class PeerOperationLogTable: Table { encoder.encodeRootObject(contents) } let contentBuffer = encoder.readBufferNoCopy() - var contentBufferLength: Int32 = Int32(contentBuffer.length) - buffer.write(&contentBufferLength, offset: 0, length: 4) - buffer.write(contentBuffer.memory, offset: 0, length: contentBuffer.length) - self.valueBox.set(self.table, key: key, value: buffer) + withExtendedLifetime(encoder, { + var contentBufferLength: Int32 = Int32(contentBuffer.length) + buffer.write(&contentBufferLength, offset: 0, length: 4) + buffer.write(contentBuffer.memory, offset: 0, length: contentBuffer.length) + self.valueBox.set(self.table, key: key, value: buffer) + }) } } else { assertionFailure() diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 499be08744..cc9ee81aeb 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -552,24 +552,69 @@ fileprivate class PipeNotifier: NSObject { } } +public enum PostboxResult { + case upgrading + case postbox(Postbox) +} + +public func openPostbox(basePath: String, globalMessageIdsNamespace: MessageId.Namespace, seedConfiguration: SeedConfiguration) -> Signal { + let queue = Queue(name: "org.telegram.postbox.Postbox") + return Signal { subscriber in + queue.async { + let _ = try? FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) + + loop: while true { + let valueBox = SqliteValueBox(basePath: basePath + "/db", queue: queue) + + let metadataTable = MetadataTable(valueBox: valueBox, table: MetadataTable.tableSpec(0)) + + let userVersion: Int32? = metadataTable.userVersion() + let currentUserVersion: Int32 = 13 + + if let userVersion = userVersion { + if userVersion != currentUserVersion { + if let operation = registeredUpgrades()[userVersion] { + switch operation { + case let .inplace(f): + valueBox.begin() + f(metadataTable, valueBox) + valueBox.commit() + } + continue loop + } else { + assertionFailure() + valueBox.drop() + } + } + } else { + metadataTable.setUserVersion(currentUserVersion) + } + + subscriber.putNext(.postbox(Postbox(queue: queue, basePath: basePath, globalMessageIdsNamespace: globalMessageIdsNamespace, seedConfiguration: seedConfiguration, valueBox: valueBox))) + subscriber.putCompletion() + break + } + } + + return EmptyDisposable + } +} + public final class Postbox { + private let queue: Queue private let seedConfiguration: SeedConfiguration private let basePath: String private let globalMessageIdsNamespace: MessageId.Namespace + private let valueBox: ValueBox private let ipcNotificationsDisposable = MetaDisposable() //private var pipeNotifier: PipeNotifier! - private let queue = Queue(name: "org.telegram.postbox.Postbox") - private var valueBox: ValueBox! - private var transactionStateVersion: Int64 = 0 private var viewTracker: ViewTracker! private var nextViewId = 0 - private var peerPipes: [PeerId: ValuePipe] = [:] - private var currentUpdatedState: Coding? private var currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] private var currentUpdatedChatListInclusions: [PeerId: PeerChatListInclusion] = [:] @@ -597,7 +642,7 @@ public final class Postbox { private var currentReplacedContactPeerIds: Set? private var currentUpdatedMasterClientId: Int64? - private var statePipe: ValuePipe = ValuePipe() + private let statePipe: ValuePipe = ValuePipe() private var masterClientId = Promise() private var sessionClientId: Int64 = { @@ -608,60 +653,62 @@ public final class Postbox { public let mediaBox: MediaBox - var tables: [Table] = [] + let tables: [Table] - var metadataTable: MetadataTable! - var keychainTable: KeychainTable! - var peerTable: PeerTable! - var peerNotificationSettingsTable: PeerNotificationSettingsTable! - var cachedPeerDataTable: CachedPeerDataTable! - var peerPresenceTable: PeerPresenceTable! - var globalMessageIdsTable: GlobalMessageIdsTable! - var globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable! - var messageHistoryIndexTable: MessageHistoryIndexTable! - var messageHistoryTable: MessageHistoryTable! - var mediaTable: MessageMediaTable! - var chatListIndexTable: ChatListIndexTable! - var chatListTable: ChatListTable! - var messageHistoryMetadataTable: MessageHistoryMetadataTable! - var messageHistoryUnsentTable: MessageHistoryUnsentTable! - var messageHistoryTagsTable: MessageHistoryTagsTable! - var peerChatStateTable: PeerChatStateTable! - var readStateTable: MessageHistoryReadStateTable! - var synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable! - var contactsTable: ContactTable! - var itemCollectionInfoTable: ItemCollectionInfoTable! - var itemCollectionItemTable: ItemCollectionItemTable! - var itemCollectionReverseIndexTable: ReverseIndexReferenceTable! - var peerChatInterfaceStateTable: PeerChatInterfaceStateTable! - var itemCacheMetaTable: ItemCacheMetaTable! - var itemCacheTable: ItemCacheTable! - var peerNameTokenIndexTable: ReverseIndexReferenceTable! - var peerNameIndexTable: PeerNameIndexTable! - var peerChatTopTaggedMessageIdsTable: PeerChatTopTaggedMessageIdsTable! - var peerOperationLogMetadataTable: PeerOperationLogMetadataTable! - var peerMergedOperationLogIndexTable: PeerMergedOperationLogIndexTable! - var peerOperationLogTable: PeerOperationLogTable! - var timestampBasedMessageAttributesTable: TimestampBasedMessageAttributesTable! - var timestampBasedMessageAttributesIndexTable: TimestampBasedMessageAttributesIndexTable! - var preferencesTable: PreferencesTable! - var orderedItemListTable: OrderedItemListTable! - var orderedItemListIndexTable: OrderedItemListIndexTable! + let metadataTable: MetadataTable + let keychainTable: KeychainTable + let peerTable: PeerTable + let peerNotificationSettingsTable: PeerNotificationSettingsTable + let cachedPeerDataTable: CachedPeerDataTable + let peerPresenceTable: PeerPresenceTable + let globalMessageIdsTable: GlobalMessageIdsTable + let globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable + let messageHistoryIndexTable: MessageHistoryIndexTable + let messageHistoryTable: MessageHistoryTable + let mediaTable: MessageMediaTable + let chatListIndexTable: ChatListIndexTable + let chatListTable: ChatListTable + let messageHistoryMetadataTable: MessageHistoryMetadataTable + let messageHistoryUnsentTable: MessageHistoryUnsentTable + let messageHistoryTagsTable: MessageHistoryTagsTable + let peerChatStateTable: PeerChatStateTable + let readStateTable: MessageHistoryReadStateTable + let synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable + let contactsTable: ContactTable + let itemCollectionInfoTable: ItemCollectionInfoTable + let itemCollectionItemTable: ItemCollectionItemTable + let itemCollectionReverseIndexTable: ReverseIndexReferenceTable + let peerChatInterfaceStateTable: PeerChatInterfaceStateTable + let itemCacheMetaTable: ItemCacheMetaTable + let itemCacheTable: ItemCacheTable + let peerNameTokenIndexTable: ReverseIndexReferenceTable + let peerNameIndexTable: PeerNameIndexTable + let peerChatTopTaggedMessageIdsTable: PeerChatTopTaggedMessageIdsTable + let peerOperationLogMetadataTable: PeerOperationLogMetadataTable + let peerMergedOperationLogIndexTable: PeerMergedOperationLogIndexTable + let peerOperationLogTable: PeerOperationLogTable + let timestampBasedMessageAttributesTable: TimestampBasedMessageAttributesTable + let timestampBasedMessageAttributesIndexTable: TimestampBasedMessageAttributesIndexTable + let preferencesTable: PreferencesTable + let orderedItemListTable: OrderedItemListTable + let orderedItemListIndexTable: OrderedItemListIndexTable //temporary - var peerRatingTable: RatingTable! + let peerRatingTable: RatingTable - public init(basePath: String, globalMessageIdsNamespace: MessageId.Namespace, seedConfiguration: SeedConfiguration) { + fileprivate init(queue: Queue, basePath: String, globalMessageIdsNamespace: MessageId.Namespace, seedConfiguration: SeedConfiguration, valueBox: ValueBox) { + assert(queue.isCurrent()) + + self.queue = queue self.basePath = basePath self.globalMessageIdsNamespace = globalMessageIdsNamespace self.seedConfiguration = seedConfiguration print("MediaBox path: \(self.basePath + "/media")") - //let _ = try? FileManager.default.removeItem(atPath: self.basePath) - self.mediaBox = MediaBox(basePath: self.basePath + "/media") + self.valueBox = valueBox /*self.pipeNotifier = PipeNotifier(basePath: basePath, notify: { [weak self] in //if let strongSelf = self { @@ -674,7 +721,117 @@ public final class Postbox { //} })*/ - self.openDatabase() + let startTime = CFAbsoluteTimeGetCurrent() + + self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) + + self.keychainTable = KeychainTable(valueBox: self.valueBox, table: KeychainTable.tableSpec(1)) + self.peerTable = PeerTable(valueBox: self.valueBox, table: PeerTable.tableSpec(2)) + self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox, table: GlobalMessageIdsTable.tableSpec(3), namespace: self.globalMessageIdsNamespace) + self.globallyUniqueMessageIdsTable = MessageGloballyUniqueIdTable(valueBox: self.valueBox, table: MessageGloballyUniqueIdTable.tableSpec(32)) + self.messageHistoryMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox, table: MessageHistoryMetadataTable.tableSpec(10)) + self.messageHistoryUnsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox, table: MessageHistoryUnsentTable.tableSpec(11)) + self.messageHistoryTagsTable = MessageHistoryTagsTable(valueBox: self.valueBox, table: MessageHistoryTagsTable.tableSpec(12)) + self.messageHistoryIndexTable = MessageHistoryIndexTable(valueBox: self.valueBox, table: MessageHistoryIndexTable.tableSpec(4), globalMessageIdsTable: self.globalMessageIdsTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox, table: MessageMediaTable.tableSpec(6)) + self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox, table: MessageHistoryReadStateTable.tableSpec(14)) + self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox, table: MessageHistorySynchronizeReadStateTable.tableSpec(15)) + self.timestampBasedMessageAttributesIndexTable = TimestampBasedMessageAttributesIndexTable(valueBox: self.valueBox, table: TimestampBasedMessageAttributesTable.tableSpec(33)) + self.timestampBasedMessageAttributesTable = TimestampBasedMessageAttributesTable(valueBox: self.valueBox, table: TimestampBasedMessageAttributesTable.tableSpec(34), indexTable: self.timestampBasedMessageAttributesIndexTable) + self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable, unsentTable: self.messageHistoryUnsentTable, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable) + self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, table: PeerChatStateTable.tableSpec(13)) + self.peerNameTokenIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(26)) + self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) + self.contactsTable = ContactTable(valueBox: self.valueBox, table: ContactTable.tableSpec(16), peerNameIndexTable: self.peerNameIndexTable) + self.peerRatingTable = RatingTable(valueBox: self.valueBox, table: RatingTable.tableSpec(17)) + self.cachedPeerDataTable = CachedPeerDataTable(valueBox: self.valueBox, table: CachedPeerDataTable.tableSpec(18)) + self.peerNotificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox, table: PeerNotificationSettingsTable.tableSpec(19)) + self.peerPresenceTable = PeerPresenceTable(valueBox: self.valueBox, table: PeerPresenceTable.tableSpec(20)) + self.itemCollectionInfoTable = ItemCollectionInfoTable(valueBox: self.valueBox, table: ItemCollectionInfoTable.tableSpec(21)) + self.itemCollectionReverseIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(36)) + self.itemCollectionItemTable = ItemCollectionItemTable(valueBox: self.valueBox, table: ItemCollectionItemTable.tableSpec(22), reverseIndexTable: self.itemCollectionReverseIndexTable) + self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox, table: PeerChatInterfaceStateTable.tableSpec(23)) + self.itemCacheMetaTable = ItemCacheMetaTable(valueBox: self.valueBox, table: ItemCacheMetaTable.tableSpec(24)) + self.itemCacheTable = ItemCacheTable(valueBox: self.valueBox, table: ItemCacheTable.tableSpec(25)) + self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8), peerNameIndexTable: self.peerNameIndexTable, metadataTable: self.messageHistoryMetadataTable, readStateTable: self.readStateTable, notificationSettingsTable: self.peerNotificationSettingsTable) + self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) + self.peerChatTopTaggedMessageIdsTable = PeerChatTopTaggedMessageIdsTable(valueBox: self.valueBox, table: PeerChatTopTaggedMessageIdsTable.tableSpec(28)) + self.peerOperationLogMetadataTable = PeerOperationLogMetadataTable(valueBox: self.valueBox, table: PeerOperationLogMetadataTable.tableSpec(29)) + self.peerMergedOperationLogIndexTable = PeerMergedOperationLogIndexTable(valueBox: self.valueBox, table: PeerMergedOperationLogIndexTable.tableSpec(30), metadataTable: self.peerOperationLogMetadataTable) + self.peerOperationLogTable = PeerOperationLogTable(valueBox: self.valueBox, table: PeerOperationLogTable.tableSpec(31), metadataTable: self.peerOperationLogMetadataTable, mergedIndexTable: self.peerMergedOperationLogIndexTable) + self.preferencesTable = PreferencesTable(valueBox: self.valueBox, table: PreferencesTable.tableSpec(35)) + self.orderedItemListIndexTable = OrderedItemListIndexTable(valueBox: self.valueBox, table: OrderedItemListIndexTable.tableSpec(37)) + self.orderedItemListTable = OrderedItemListTable(valueBox: self.valueBox, table: OrderedItemListTable.tableSpec(38), indexTable: self.orderedItemListIndexTable) + + var tables: [Table] = [] + tables.append(self.metadataTable) + tables.append(self.keychainTable) + tables.append(self.peerTable) + tables.append(self.globalMessageIdsTable) + tables.append(self.globallyUniqueMessageIdsTable) + tables.append(self.messageHistoryMetadataTable) + tables.append(self.messageHistoryUnsentTable) + tables.append(self.messageHistoryTagsTable) + tables.append(self.messageHistoryIndexTable) + tables.append(self.mediaTable) + tables.append(self.readStateTable) + tables.append(self.synchronizeReadStateTable) + tables.append(self.messageHistoryTable) + tables.append(self.chatListIndexTable) + tables.append(self.chatListTable) + tables.append(self.peerChatStateTable) + tables.append(self.contactsTable) + tables.append(self.peerRatingTable) + tables.append(self.peerNotificationSettingsTable) + tables.append(self.cachedPeerDataTable) + tables.append(self.peerPresenceTable) + tables.append(self.itemCollectionInfoTable) + tables.append(self.itemCollectionItemTable) + tables.append(self.itemCollectionReverseIndexTable) + tables.append(self.peerChatInterfaceStateTable) + tables.append(self.itemCacheMetaTable) + tables.append(self.itemCacheTable) + tables.append(self.peerNameIndexTable) + tables.append(self.peerNameTokenIndexTable) + tables.append(self.peerChatTopTaggedMessageIdsTable) + tables.append(self.peerOperationLogMetadataTable) + tables.append(self.peerMergedOperationLogIndexTable) + tables.append(self.peerOperationLogTable) + tables.append(self.timestampBasedMessageAttributesTable) + tables.append(self.timestampBasedMessageAttributesIndexTable) + tables.append(self.preferencesTable) + tables.append(self.orderedItemListTable) + tables.append(self.orderedItemListIndexTable) + + self.tables = tables + + self.transactionStateVersion = self.metadataTable.transactionStateVersion() + + self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, fetchAnchorIndex: self.fetchAnchorIndex, renderMessage: self.renderIntermediateMessage, getPeer: { peerId in + return self.peerTable.get(peerId) + }, getPeerNotificationSettings: { peerId in + return self.peerNotificationSettingsTable.get(peerId) + }, getCachedPeerData: { peerId in + return self.cachedPeerDataTable.get(peerId) + }, getPeerPresence: { peerId in + return self.peerPresenceTable.get(peerId) + }, getTotalUnreadCount: { + return self.messageHistoryMetadataTable.getChatListTotalUnreadCount() + }, getPeerReadState: { peerId in + return self.readStateTable.getCombinedState(peerId) + }, operationLogGetOperations: { tag, fromIndex, limit in + return self.peerOperationLogTable.getMergedEntries(tag: tag, fromIndex: fromIndex, limit: limit) + }, operationLogGetTailIndex: { tag in + return self.peerMergedOperationLogIndexTable.tailIndex(tag: tag) + }, getTimestampBasedMessageAttributesHead: { tag in + return self.timestampBasedMessageAttributesTable.head(tag: tag) + }, getPreferencesEntry: { key in + return self.preferencesTable.get(key: key) + }, unsentMessageIds: self.messageHistoryUnsentTable.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable.get(getCombinedPeerReadState: { peerId in + return self.readStateTable.getCombinedState(peerId) + })) + + print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") } deinit { @@ -701,140 +858,6 @@ public final class Postbox { } } - private func openDatabase() { - self.queue.justDispatch({ - let startTime = CFAbsoluteTimeGetCurrent() - - do { - try FileManager.default.createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) - } catch _ { - } - - //let _ = try? FileManager.default.removeItem(atPath: self.basePath + "/media") - - //#if TARGET_IPHONE_SIMULATOR - - //self.debugSaveState(name: "previous") - //self.debugRestoreState(name: "previous") - - //#endif - - self.valueBox = SqliteValueBox(basePath: self.basePath + "/db", queue: self.queue) - - self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) - - let userVersion: Int32? = self.metadataTable.userVersion() - let currentUserVersion: Int32 = 12 - - if userVersion != currentUserVersion { - self.valueBox.drop() - self.metadataTable.setUserVersion(currentUserVersion) - } - - self.keychainTable = KeychainTable(valueBox: self.valueBox, table: KeychainTable.tableSpec(1)) - self.peerTable = PeerTable(valueBox: self.valueBox, table: PeerTable.tableSpec(2)) - self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox, table: GlobalMessageIdsTable.tableSpec(3), namespace: self.globalMessageIdsNamespace) - self.globallyUniqueMessageIdsTable = MessageGloballyUniqueIdTable(valueBox: self.valueBox, table: MessageGloballyUniqueIdTable.tableSpec(32)) - self.messageHistoryMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox, table: MessageHistoryMetadataTable.tableSpec(10)) - self.messageHistoryUnsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox, table: MessageHistoryUnsentTable.tableSpec(11)) - self.messageHistoryTagsTable = MessageHistoryTagsTable(valueBox: self.valueBox, table: MessageHistoryTagsTable.tableSpec(12)) - self.messageHistoryIndexTable = MessageHistoryIndexTable(valueBox: self.valueBox, table: MessageHistoryIndexTable.tableSpec(4), globalMessageIdsTable: self.globalMessageIdsTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) - self.mediaTable = MessageMediaTable(valueBox: self.valueBox, table: MessageMediaTable.tableSpec(6)) - self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox, table: MessageHistoryReadStateTable.tableSpec(14)) - self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox, table: MessageHistorySynchronizeReadStateTable.tableSpec(15)) - self.timestampBasedMessageAttributesIndexTable = TimestampBasedMessageAttributesIndexTable(valueBox: self.valueBox!, table: TimestampBasedMessageAttributesTable.tableSpec(33)) - self.timestampBasedMessageAttributesTable = TimestampBasedMessageAttributesTable(valueBox: self.valueBox!, table: TimestampBasedMessageAttributesTable.tableSpec(34), indexTable: self.timestampBasedMessageAttributesIndexTable) - self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable!, unsentTable: self.messageHistoryUnsentTable!, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable!) - self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, table: PeerChatStateTable.tableSpec(13)) - self.peerNameTokenIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(26)) - self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) - self.contactsTable = ContactTable(valueBox: self.valueBox, table: ContactTable.tableSpec(16), peerNameIndexTable: self.peerNameIndexTable) - self.peerRatingTable = RatingTable(valueBox: self.valueBox, table: RatingTable.tableSpec(17)) - self.cachedPeerDataTable = CachedPeerDataTable(valueBox: self.valueBox, table: CachedPeerDataTable.tableSpec(18)) - self.peerNotificationSettingsTable = PeerNotificationSettingsTable(valueBox: self.valueBox, table: PeerNotificationSettingsTable.tableSpec(19)) - self.peerPresenceTable = PeerPresenceTable(valueBox: self.valueBox, table: PeerPresenceTable.tableSpec(20)) - self.itemCollectionInfoTable = ItemCollectionInfoTable(valueBox: self.valueBox, table: ItemCollectionInfoTable.tableSpec(21)) - self.itemCollectionReverseIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(36)) - self.itemCollectionItemTable = ItemCollectionItemTable(valueBox: self.valueBox, table: ItemCollectionItemTable.tableSpec(22), reverseIndexTable: self.itemCollectionReverseIndexTable) - self.peerChatInterfaceStateTable = PeerChatInterfaceStateTable(valueBox: self.valueBox, table: PeerChatInterfaceStateTable.tableSpec(23)) - self.itemCacheMetaTable = ItemCacheMetaTable(valueBox: self.valueBox, table: ItemCacheMetaTable.tableSpec(24)) - self.itemCacheTable = ItemCacheTable(valueBox: self.valueBox, table: ItemCacheTable.tableSpec(25)) - self.chatListIndexTable = ChatListIndexTable(valueBox: self.valueBox, table: ChatListIndexTable.tableSpec(8), peerNameIndexTable: self.peerNameIndexTable, metadataTable: self.messageHistoryMetadataTable, readStateTable: self.readStateTable, notificationSettingsTable: self.peerNotificationSettingsTable) - self.chatListTable = ChatListTable(valueBox: self.valueBox, table: ChatListTable.tableSpec(9), indexTable: self.chatListIndexTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) - self.peerChatTopTaggedMessageIdsTable = PeerChatTopTaggedMessageIdsTable(valueBox: self.valueBox, table: PeerChatTopTaggedMessageIdsTable.tableSpec(28)) - self.peerOperationLogMetadataTable = PeerOperationLogMetadataTable(valueBox: self.valueBox, table: PeerOperationLogMetadataTable.tableSpec(29)) - self.peerMergedOperationLogIndexTable = PeerMergedOperationLogIndexTable(valueBox: self.valueBox, table: PeerMergedOperationLogIndexTable.tableSpec(30), metadataTable: self.peerOperationLogMetadataTable!) - self.peerOperationLogTable = PeerOperationLogTable(valueBox: self.valueBox, table: PeerOperationLogTable.tableSpec(31), metadataTable: self.peerOperationLogMetadataTable, mergedIndexTable: self.peerMergedOperationLogIndexTable) - self.preferencesTable = PreferencesTable(valueBox: self.valueBox, table: PreferencesTable.tableSpec(35)) - self.orderedItemListIndexTable = OrderedItemListIndexTable(valueBox: self.valueBox, table: OrderedItemListIndexTable.tableSpec(37)) - self.orderedItemListTable = OrderedItemListTable(valueBox: self.valueBox, table: OrderedItemListTable.tableSpec(38), indexTable: self.orderedItemListIndexTable) - - self.tables.append(self.keychainTable) - self.tables.append(self.peerTable) - self.tables.append(self.globalMessageIdsTable) - self.tables.append(self.globallyUniqueMessageIdsTable) - self.tables.append(self.messageHistoryMetadataTable) - self.tables.append(self.messageHistoryUnsentTable) - self.tables.append(self.messageHistoryTagsTable) - self.tables.append(self.messageHistoryIndexTable) - self.tables.append(self.mediaTable) - self.tables.append(self.readStateTable) - self.tables.append(self.synchronizeReadStateTable) - self.tables.append(self.messageHistoryTable) - self.tables.append(self.chatListIndexTable) - self.tables.append(self.chatListTable) - self.tables.append(self.peerChatStateTable) - self.tables.append(self.contactsTable) - self.tables.append(self.peerRatingTable) - self.tables.append(self.peerNotificationSettingsTable) - self.tables.append(self.cachedPeerDataTable) - self.tables.append(self.peerPresenceTable) - self.tables.append(self.itemCollectionInfoTable) - self.tables.append(self.itemCollectionItemTable) - self.tables.append(self.itemCollectionReverseIndexTable) - self.tables.append(self.peerChatInterfaceStateTable) - self.tables.append(self.itemCacheMetaTable) - self.tables.append(self.itemCacheTable) - self.tables.append(self.peerNameIndexTable) - self.tables.append(self.peerNameTokenIndexTable) - self.tables.append(self.peerChatTopTaggedMessageIdsTable) - self.tables.append(self.peerOperationLogMetadataTable) - self.tables.append(self.peerMergedOperationLogIndexTable) - self.tables.append(self.peerOperationLogTable) - self.tables.append(self.preferencesTable) - self.tables.append(self.orderedItemListTable) - self.tables.append(self.orderedItemListIndexTable) - - self.transactionStateVersion = self.metadataTable.transactionStateVersion() - - self.viewTracker = ViewTracker(queue: self.queue, fetchEarlierHistoryEntries: self.fetchEarlierHistoryEntries, fetchLaterHistoryEntries: self.fetchLaterHistoryEntries, fetchEarlierChatEntries: self.fetchEarlierChatEntries, fetchLaterChatEntries: self.fetchLaterChatEntries, fetchAnchorIndex: self.fetchAnchorIndex, renderMessage: self.renderIntermediateMessage, getPeer: { peerId in - return self.peerTable.get(peerId) - }, getPeerNotificationSettings: { peerId in - return self.peerNotificationSettingsTable.get(peerId) - }, getCachedPeerData: { peerId in - return self.cachedPeerDataTable.get(peerId) - }, getPeerPresence: { peerId in - return self.peerPresenceTable.get(peerId) - }, getTotalUnreadCount: { - return self.messageHistoryMetadataTable.getChatListTotalUnreadCount() - }, getPeerReadState: { peerId in - return self.readStateTable.getCombinedState(peerId) - }, operationLogGetOperations: { tag, fromIndex, limit in - return self.peerOperationLogTable.getMergedEntries(tag: tag, fromIndex: fromIndex, limit: limit) - }, operationLogGetTailIndex: { tag in - return self.peerMergedOperationLogIndexTable.tailIndex(tag: tag) - }, getTimestampBasedMessageAttributesHead: { tag in - return self.timestampBasedMessageAttributesTable.head(tag: tag) - }, getPreferencesEntry: { key in - return self.preferencesTable.get(key: key) - }, unsentMessageIds: self.messageHistoryUnsentTable!.get(), synchronizePeerReadStateOperations: self.synchronizeReadStateTable!.get(getCombinedPeerReadState: { peerId in - return self.readStateTable.getCombinedState(peerId) - })) - - print("(Postbox initialization took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - }) - } - private func takeNextViewId() -> Int { let nextId = self.nextViewId self.nextViewId += 1 @@ -1191,9 +1214,9 @@ public final class Postbox { table.clearMemoryCache() } self.viewTracker.refreshViewsDueToExternalTransaction(fetchAroundChatEntries: self.fetchAroundChatEntries, fetchAroundHistoryEntries: self.fetchAroundHistoryEntries, fetchUnsentMessageIds: { - return self.messageHistoryUnsentTable!.get() + return self.messageHistoryUnsentTable.get() }, fetchSynchronizePeerReadStateOperations: { - return self.synchronizeReadStateTable!.get(getCombinedPeerReadState: { peerId in + return self.synchronizeReadStateTable.get(getCombinedPeerReadState: { peerId in return self.readStateTable.getCombinedState(peerId) }) }) diff --git a/Postbox/PostboxUpgrade_12to13.swift b/Postbox/PostboxUpgrade_12to13.swift new file mode 100644 index 0000000000..873abb859f --- /dev/null +++ b/Postbox/PostboxUpgrade_12to13.swift @@ -0,0 +1,11 @@ +import Foundation + +func postboxUpgrade_12to13(metadataTable: MetadataTable, valueBox: ValueBox) { + // drop PeerMergedOperationLogIndexTable + valueBox.dropTable(ValueBoxTable(id: 30, keyType: .binary)) + + // drop PeerOperationLogTable + valueBox.dropTable(ValueBoxTable(id: 31, keyType: .binary)) + + metadataTable.setUserVersion(13) +} diff --git a/Postbox/SqliteInterface.swift b/Postbox/SqliteInterface.swift index 44a0f0e33d..8aef02210b 100644 --- a/Postbox/SqliteInterface.swift +++ b/Postbox/SqliteInterface.swift @@ -102,9 +102,6 @@ public final class SqliteInterface { public init?(databasePath: String) { if let database = Database(databasePath) { - if !database.execute("pragma schema_version") { - return nil - } self.database = database } else { return nil diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 5cc4f2e4c1..d5183e31fc 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -1057,6 +1057,10 @@ final class SqliteValueBox: ValueBox { self.deleteStatements.removeAll() } + public func dropTable(_ table: ValueBoxTable) { + self.database.execute("REMOVE FROM t\(table.id)") + } + public func drop() { assert(self.queue.isCurrent()) self.clearStatements() diff --git a/Postbox/Upgrades.swift b/Postbox/Upgrades.swift new file mode 100644 index 0000000000..6c70a9d015 --- /dev/null +++ b/Postbox/Upgrades.swift @@ -0,0 +1,11 @@ +import Foundation + +enum PostboxUpgradeOperation { + case inplace((MetadataTable, ValueBox) -> Void) +} + +func registeredUpgrades() -> [Int32: PostboxUpgradeOperation] { + var dict: [Int32: PostboxUpgradeOperation] = [:] + dict[12] = .inplace(postboxUpgrade_12to13) + return dict +} diff --git a/Postbox/ValueBox.swift b/Postbox/ValueBox.swift index 9fe68e47b7..3dd987a8af 100644 --- a/Postbox/ValueBox.swift +++ b/Postbox/ValueBox.swift @@ -24,5 +24,6 @@ protocol ValueBox { func exists(_ table: ValueBoxTable, key: ValueBoxKey) -> Bool func set(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) func remove(_ table: ValueBoxTable, key: ValueBoxKey) + func dropTable(_ table: ValueBoxTable) func drop() } From b0e511bea65fb3237d242c68277b9d35aa02901c Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Apr 2017 16:32:13 +0300 Subject: [PATCH 129/427] no message --- Postbox/SqliteValueBox.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index d5183e31fc..e32a96123d 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -1058,7 +1058,7 @@ final class SqliteValueBox: ValueBox { } public func dropTable(_ table: ValueBoxTable) { - self.database.execute("REMOVE FROM t\(table.id)") + let _ = self.database.execute("DELETE FROM t\(table.id)") } public func drop() { From 1dcf9ee2ebff0e4aa58a3106e0b8747a9978e46b Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 12 Apr 2017 13:02:33 +0300 Subject: [PATCH 130/427] no message --- Postbox/AccountManagerRecordTable.swift | 4 +- Postbox/ChatListIndexTable.swift | 4 +- Postbox/Database.swift | 1 + Postbox/ItemCacheMetaTable.swift | 4 +- Postbox/ItemCacheTable.swift | 4 +- Postbox/ItemCollectionInfoTable.swift | 4 +- Postbox/ItemCollectionItemTable.swift | 4 +- Postbox/MessageHistoryTable.swift | 83 ++++++++++++++++++++++--- Postbox/MessageMediaTable.swift | 20 ++++-- Postbox/MetadataTable.swift | 41 +++++++++--- Postbox/OrderedItemListIndexTable.swift | 4 +- Postbox/PeerOperationLogTable.swift | 10 +-- Postbox/Postbox.swift | 10 +++ Postbox/PreferencesTable.swift | 4 +- Postbox/SqliteValueBox.swift | 6 +- 15 files changed, 169 insertions(+), 34 deletions(-) diff --git a/Postbox/AccountManagerRecordTable.swift b/Postbox/AccountManagerRecordTable.swift index 1d48924997..19742e034a 100644 --- a/Postbox/AccountManagerRecordTable.swift +++ b/Postbox/AccountManagerRecordTable.swift @@ -37,7 +37,9 @@ final class AccountManagerRecordTable: Table { if let record = record { let encoder = Encoder() record.encode(encoder) - self.valueBox.set(self.table, key: self.key(id), value: encoder.readBufferNoCopy()) + withExtendedLifetime(encoder, { + self.valueBox.set(self.table, key: self.key(id), value: encoder.readBufferNoCopy()) + }) } else { self.valueBox.remove(self.table, key: self.key(id)) } diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift index dbb3dadef3..83495c2d5e 100644 --- a/Postbox/ChatListIndexTable.swift +++ b/Postbox/ChatListIndexTable.swift @@ -205,7 +205,9 @@ final class ChatListIndexTable: Table { } } - self.valueBox.set(self.table, key: self.key(peerId), value: writeBuffer.readBufferNoCopy()) + withExtendedLifetime(writeBuffer, { + self.valueBox.set(self.table, key: self.key(peerId), value: writeBuffer.readBufferNoCopy()) + }) } self.updatedPreviousCachedIndices.removeAll() diff --git a/Postbox/Database.swift b/Postbox/Database.swift index 8939a7c452..ba023b6db3 100644 --- a/Postbox/Database.swift +++ b/Postbox/Database.swift @@ -71,6 +71,7 @@ public final class Database { let flags = readonly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE let res = sqlite3_open_v2(location.description, &self.handle, flags | SQLITE_OPEN_FULLMUTEX, nil) if res != SQLITE_OK { + preconditionFailure("sqlite3_open_v2: \(res)") return nil } } diff --git a/Postbox/ItemCacheMetaTable.swift b/Postbox/ItemCacheMetaTable.swift index f4d3c6573f..2a7cd00ba4 100644 --- a/Postbox/ItemCacheMetaTable.swift +++ b/Postbox/ItemCacheMetaTable.swift @@ -69,7 +69,9 @@ final class ItemCacheMetaTable: Table { if let state = self.cachedCollectionStates[id] { sharedEncoder.reset() sharedEncoder.encodeRootObject(state) - self.valueBox.set(self.table, key: self.key(id), value: sharedEncoder.readBufferNoCopy()) + withExtendedLifetime(sharedEncoder, { + self.valueBox.set(self.table, key: self.key(id), value: sharedEncoder.readBufferNoCopy()) + }) } } } diff --git a/Postbox/ItemCacheTable.swift b/Postbox/ItemCacheTable.swift index 9b6b1f2aa5..ba6751785f 100644 --- a/Postbox/ItemCacheTable.swift +++ b/Postbox/ItemCacheTable.swift @@ -48,7 +48,9 @@ final class ItemCacheTable: Table { func put(id: ItemCacheEntryId, entry: Coding, metaTable: ItemCacheMetaTable) { let encoder = Encoder() encoder.encodeRootObject(entry) - self.valueBox.set(self.table, key: self.itemKey(id: id), value: encoder.readBufferNoCopy()) + withExtendedLifetime(encoder, { + self.valueBox.set(self.table, key: self.itemKey(id: id), value: encoder.readBufferNoCopy()) + }) } func retrieve(id: ItemCacheEntryId, metaTable: ItemCacheMetaTable) -> Coding? { diff --git a/Postbox/ItemCollectionInfoTable.swift b/Postbox/ItemCollectionInfoTable.swift index 72be13bf66..2f320b47db 100644 --- a/Postbox/ItemCollectionInfoTable.swift +++ b/Postbox/ItemCollectionInfoTable.swift @@ -156,7 +156,9 @@ final class ItemCollectionInfoTable: Table { for (id, info) in infos { sharedEncoder.reset() sharedEncoder.encodeRootObject(info) - self.valueBox.set(self.table, key: self.key(collectionId: id, index: index), value: sharedEncoder.readBufferNoCopy()) + withExtendedLifetime(sharedEncoder, { + self.valueBox.set(self.table, key: self.key(collectionId: id, index: index), value: sharedEncoder.readBufferNoCopy()) + }) index += 1 } } diff --git a/Postbox/ItemCollectionItemTable.swift b/Postbox/ItemCollectionItemTable.swift index e92858a07e..9263de669a 100644 --- a/Postbox/ItemCollectionItemTable.swift +++ b/Postbox/ItemCollectionItemTable.swift @@ -221,7 +221,9 @@ final class ItemCollectionItemTable: Table { let item = itemByIndex[index]! sharedEncoder.reset() sharedEncoder.encodeRootObject(item) - self.valueBox.set(self.table, key: self.key(collectionId: collectionId, index: index), value: sharedEncoder.readBufferNoCopy()) + withExtendedLifetime(sharedEncoder, { + self.valueBox.set(self.table, key: self.key(collectionId: collectionId, index: index), value: sharedEncoder.readBufferNoCopy()) + }) if !item.indexKeys.isEmpty { self.reverseIndexTable.add(namespace: ReverseIndexNamespace(collectionId.namespace), reference: ItemCollectionItemReverseIndexReference(collectionId: collectionId, itemIndex: index), tokens: item.indexKeys.map({ ValueBoxKey($0) })) } diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 5d73514d3d..d4cacf9436 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -329,6 +329,11 @@ final class MessageHistoryTable: Table { self.removeMessages(indices.map { $0.id }, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) } + func removeAllMessagesWithAuthor(peerId: PeerId, authorId: PeerId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + let indices = self.allIndicesWithAuthor(peerId, authorId: authorId) + self.removeMessages(indices.map { $0.id }, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + } + func fillHole(_ id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.fillHole(id, fillType: fillType, tagMask: tagMask, messages: self.internalStoreMessages(messages), operations: &operations) @@ -696,7 +701,9 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&minId, offset: 0, length: 4) var tags: UInt32 = hole.tags sharedBuffer.write(&tags, offset: 0, length: 4) - self.valueBox.set(self.table, key: self.key(hole.maxIndex), value: sharedBuffer.readBufferNoCopy()) + withExtendedLifetime(sharedBuffer, { + self.valueBox.set(self.table, key: self.key(hole.maxIndex), value: sharedBuffer.readBufferNoCopy()) + }) } private func tagsForIndex(_ index: MessageIndex) -> MessageTags? { @@ -846,13 +853,17 @@ final class MessageHistoryTable: Table { for media in updatedMedia { encoder.reset() encoder.encodeRootObject(media) - let encodedBuffer = encoder.readBufferNoCopy() - var encodedLength: Int32 = Int32(encodedBuffer.length) - updatedEmbeddedMediaBuffer.write(&encodedLength, offset: 0, length: 4) - updatedEmbeddedMediaBuffer.write(encodedBuffer.memory, offset: 0, length: encodedBuffer.length) + withExtendedLifetime(encoder, { + let encodedBuffer = encoder.readBufferNoCopy() + var encodedLength: Int32 = Int32(encodedBuffer.length) + updatedEmbeddedMediaBuffer.write(&encodedLength, offset: 0, length: 4) + updatedEmbeddedMediaBuffer.write(encodedBuffer.memory, offset: 0, length: encodedBuffer.length) + }) } - self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: message.referencedMedia), sharedKey: self.key(index)) + withExtendedLifetime(updatedEmbeddedMediaBuffer, { + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: message.referencedMedia), sharedKey: self.key(index)) + }) let operation: MessageHistoryOperation = .UpdateEmbeddedMedia(index, updatedEmbeddedMediaBuffer.makeReadBufferAndReset()) if operationsByPeerId[index.id.peerId] == nil { @@ -1149,7 +1160,9 @@ final class MessageHistoryTable: Table { if let extractedMedia = extractedMedia { var updatedReferencedMedia = message.referencedMedia updatedReferencedMedia.append(extractedMedia.id!) - self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) + withExtendedLifetime(updatedEmbeddedMediaBuffer, { + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) + }) return extractedMedia } @@ -1244,6 +1257,50 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.table, key: self.key(MessageIndex(id: message.id, timestamp: message.timestamp), key: sharedKey), value: sharedBuffer) } + private func extractIntermediateEntryAuthor(value: ReadBuffer) -> PeerId? { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == 0 { + value.skip(4) // stableId + value.skip(4) // stableVersion + + var hasGloballyUniqueId: Int8 = 0 + value.read(&hasGloballyUniqueId, offset: 0, length: 1) + if hasGloballyUniqueId != 0 { + value.skip(8) // globallyUniqueId + } + + value.skip(4) // flags + value.skip(4) // tags + + var forwardInfoFlags: Int8 = 0 + value.read(&forwardInfoFlags, offset: 0, length: 1) + if forwardInfoFlags != 0 { + value.skip(8) // forwardAuthorId + value.skip(4) // forwardDate + + if (forwardInfoFlags & 2) != 0 { + value.skip(8) // forwardSourceIdValue + } + + if (forwardInfoFlags & 4) != 0 { + value.skip(8) // forwardSourceMessagePeerId + value.skip(4) // forwardSourceMessageNamespace + value.skip(4) // forwardSourceMessageIdId + } + } + + var hasAuthor: Int8 = 0 + value.read(&hasAuthor, offset: 0, length: 1) + if hasAuthor == 1 { + var varAuthorId: Int64 = 0 + value.read(&varAuthorId, offset: 0, length: 8) + return PeerId(varAuthorId) + } + } + return nil + } + private func readIntermediateEntry(_ key: ValueBoxKey, value: ReadBuffer) -> IntermediateMessageHistoryEntry { let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4), id: key.getInt32(8 + 4 + 4)), timestamp: key.getInt32(8)) @@ -1755,6 +1812,18 @@ final class MessageHistoryTable: Table { return indices } + func allIndicesWithAuthor(_ peerId: PeerId, authorId: PeerId) -> [MessageIndex] { + var indices: [MessageIndex] = [] + self.valueBox.range(self.table, start: self.key(MessageIndex.lowerBound(peerId: peerId)).predecessor, end: self.key(MessageIndex.upperBound(peerId: peerId)).successor, values: { key, value in + if extractIntermediateEntryAuthor(value: value) == authorId { + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4), id: key.getInt32(8 + 4 + 4)), timestamp: key.getInt32(8)) + indices.append(index) + } + return true + }, limit: 0) + return indices + } + func getMessageCountInRange(peerId: PeerId, tagMask: MessageTags, lowerBound: MessageIndex, upperBound: MessageIndex) -> Int32 { return self.tagsTable.getMessageCountInRange(tagMask: tagMask, peerId: peerId, lowerBound: lowerBound, upperBound: upperBound) } diff --git a/Postbox/MessageMediaTable.swift b/Postbox/MessageMediaTable.swift index b9bf16adb2..8b9e15aee2 100644 --- a/Postbox/MessageMediaTable.swift +++ b/Postbox/MessageMediaTable.swift @@ -77,7 +77,9 @@ final class MessageMediaTable: Table { messageReferenceCount += 1 sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) - self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + withExtendedLifetime(sharedWriteBuffer, { + self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + }) return .Reference } else if type == MediaEntryType.MessageReference.rawValue { @@ -110,7 +112,9 @@ final class MessageMediaTable: Table { var messageReferenceCount: Int32 = 2 sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) - self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + withExtendedLifetime(sharedWriteBuffer, { + self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + }) } return .Reference @@ -130,7 +134,9 @@ final class MessageMediaTable: Table { sharedWriteBuffer.write(&idId, offset: 0, length: 4) sharedWriteBuffer.write(&idTimestamp, offset: 0, length: 4) - self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + withExtendedLifetime(sharedWriteBuffer, { + self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + }) return .Embed(media) } @@ -160,7 +166,9 @@ final class MessageMediaTable: Table { if messageReferenceCount <= 0 { self.valueBox.remove(self.table, key: self.key(id)) } else { - self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + withExtendedLifetime(sharedWriteBuffer, { + self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + }) } return .Reference @@ -221,7 +229,9 @@ final class MessageMediaTable: Table { if id != updatedId { self.valueBox.remove(self.table, key: self.key(id)) } - self.valueBox.set(self.table, key: self.key(updatedId), value: sharedWriteBuffer.readBufferNoCopy()) + withExtendedLifetime(sharedWriteBuffer, { + self.valueBox.set(self.table, key: self.key(updatedId), value: sharedWriteBuffer.readBufferNoCopy()) + }) } else if type == MediaEntryType.MessageReference.rawValue { var idPeerId: Int64 = 0 var idNamespace: Int32 = 0 diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index 317ff58a17..287ec88db6 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -8,19 +8,19 @@ private enum MetadataKey: Int32 { case AccessChallenge = 5 } -public enum PostboxAccessChallengeData: Coding { +public enum PostboxAccessChallengeData: Coding, Equatable { case none - case numericalPassword(String, Int32?) - case plaintextPassword(String, Int32?) + case numericalPassword(value: String, timeout: Int32?) + case plaintextPassword(value: String, timeout: Int32?) public init(decoder: Decoder) { switch decoder.decodeInt32ForKey("r") as Int32 { case 0: self = .none case 1: - self = .numericalPassword(decoder.decodeStringForKey("t"), decoder.decodeInt32ForKey("a")) + self = .numericalPassword(value: decoder.decodeStringForKey("t"), timeout: decoder.decodeInt32ForKey("a")) case 2: - self = .plaintextPassword(decoder.decodeStringForKey("t"), decoder.decodeInt32ForKey("a")) + self = .plaintextPassword(value: decoder.decodeStringForKey("t"), timeout: decoder.decodeInt32ForKey("a")) default: assertionFailure() self = .none @@ -49,6 +49,29 @@ public enum PostboxAccessChallengeData: Coding { } } } + + public static func ==(lhs: PostboxAccessChallengeData, rhs: PostboxAccessChallengeData) -> Bool { + switch lhs { + case .none: + if case .none = rhs { + return true + } else { + return false + } + case let .numericalPassword(lhsText, lhsTimeout): + if case let .numericalPassword(rhsText, rhsTimeout) = rhs, lhsText == rhsText, lhsTimeout == rhsTimeout { + return true + } else { + return false + } + case let .plaintextPassword(lhsText, lhsTimeout): + if case let .plaintextPassword(rhsText, rhsTimeout) = rhs, lhsText == rhsText, lhsTimeout == rhsTimeout { + return true + } else { + return false + } + } + } } final class MetadataTable: Table { @@ -106,7 +129,9 @@ final class MetadataTable: Table { let encoder = Encoder() encoder.encodeRootObject(state) - self.valueBox.set(self.table, key: self.key(.State), value: encoder.readBufferNoCopy()) + withExtendedLifetime(encoder, { + self.valueBox.set(self.table, key: self.key(.State), value: encoder.readBufferNoCopy()) + }) } func transactionStateVersion() -> Int64 { @@ -157,7 +182,9 @@ final class MetadataTable: Table { func setAccessChallengeData(_ data: PostboxAccessChallengeData) { let encoder = Encoder() data.encode(encoder) - self.valueBox.set(self.table, key: self.key(.AccessChallenge), value: encoder.readBufferNoCopy()) + withExtendedLifetime(encoder, { + self.valueBox.set(self.table, key: self.key(.AccessChallenge), value: encoder.readBufferNoCopy()) + }) } override func clearMemoryCache() { diff --git a/Postbox/OrderedItemListIndexTable.swift b/Postbox/OrderedItemListIndexTable.swift index 0875efb1d8..d31e7448ec 100644 --- a/Postbox/OrderedItemListIndexTable.swift +++ b/Postbox/OrderedItemListIndexTable.swift @@ -27,7 +27,9 @@ final class OrderedItemListIndexTable: Table { func set(collectionId: Int32, id: MemoryBuffer, content: Coding) { let encoder = Encoder() encoder.encodeRootObject(content) - self.valueBox.set(self.table, key: self.key(collectionId: collectionId, id: id), value: encoder.readBufferNoCopy()) + withExtendedLifetime(encoder, { + self.valueBox.set(self.table, key: self.key(collectionId: collectionId, id: id), value: encoder.readBufferNoCopy()) + }) } func getAllItemIds(collectionId: Int32) -> [MemoryBuffer] { diff --git a/Postbox/PeerOperationLogTable.swift b/Postbox/PeerOperationLogTable.swift index 0722f6fc6f..71f8413f7a 100644 --- a/Postbox/PeerOperationLogTable.swift +++ b/Postbox/PeerOperationLogTable.swift @@ -157,10 +157,12 @@ final class PeerOperationLogTable: Table { let encoder = Encoder() encoder.encodeRootObject(contents) - let contentBuffer = encoder.readBufferNoCopy() - var contentBufferLength: Int32 = Int32(contentBuffer.length) - buffer.write(&contentBufferLength, offset: 0, length: 4) - buffer.write(contentBuffer.memory, offset: 0, length: contentBuffer.length) + withExtendedLifetime(encoder, { + let contentBuffer = encoder.readBufferNoCopy() + var contentBufferLength: Int32 = Int32(contentBuffer.length) + buffer.write(&contentBufferLength, offset: 0, length: 4) + buffer.write(contentBuffer.memory, offset: 0, length: contentBuffer.length) + }) self.valueBox.set(self.table, key: self.key(peerId: peerId, tag: tag, index: index), value: buffer) if let mergedIndex = mergedIndex { diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index cc9ee81aeb..27c2407288 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -66,6 +66,12 @@ public final class Modifier { assert(!self.disposed) self.postbox?.clearHistory(peerId) } + + public func removeAllMessagesWithAuthor(_ peerId: PeerId, authorId: PeerId) { + assert(!self.disposed) + self.postbox?.removeAllMessagesWithAuthor(peerId, authorId: authorId) + } + public func messageIdsForGlobalIds(_ ids: [Int32]) -> [MessageId] { assert(!self.disposed) if let postbox = self.postbox { @@ -1028,6 +1034,10 @@ public final class Postbox { self.messageHistoryTable.clearHistory(peerId: peerId, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } + fileprivate func removeAllMessagesWithAuthor(_ peerId: PeerId, authorId: PeerId) { + self.messageHistoryTable.removeAllMessagesWithAuthor(peerId: peerId, authorId: authorId, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + } + fileprivate func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]]) { self.messageHistoryTable.resetIncomingReadStates(states, operationsByPeerId: &self.currentOperationsByPeerId, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) } diff --git a/Postbox/PreferencesTable.swift b/Postbox/PreferencesTable.swift index cbc5f6c103..da465671af 100644 --- a/Postbox/PreferencesTable.swift +++ b/Postbox/PreferencesTable.swift @@ -46,7 +46,9 @@ final class PreferencesTable: Table { if let value = self.cachedEntries[key]?.entry { let encoder = Encoder() encoder.encodeRootObject(value) - self.valueBox.set(self.table, key: key, value: encoder.readBufferNoCopy()) + withExtendedLifetime(encoder, { + self.valueBox.set(self.table, key: key, value: encoder.readBufferNoCopy()) + }) } else { self.valueBox.remove(self.table, key: key) } diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index e32a96123d..3680431478 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -136,9 +136,9 @@ final class SqliteValueBox: ValueBox { if let result = Database(path) { database = result } else { - assertionFailure() - let _ = try? FileManager.default.removeItem(atPath: path) - database = Database(path)! + preconditionFailure("Couldn't open database") + //let _ = try? FileManager.default.removeItem(atPath: path) + //database = Database(path)! } database.execute("PRAGMA cache_size=-2097152") From cd0476e1cc5758e41a282afe99cd8dbfec963952 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 13 Apr 2017 20:43:34 +0300 Subject: [PATCH 131/427] no message --- Postbox.xcodeproj/project.pbxproj | 6 ++ Postbox/AccessChallengeDataView.swift | 34 ++++++++++ Postbox/AccountManager.swift | 2 +- Postbox/MediaBox.swift | 2 +- Postbox/MetadataTable.swift | 95 ++++++++++++++++++++++++--- Postbox/Postbox.swift | 15 +++-- Postbox/PostboxAccess.swift | 4 +- Postbox/PostboxTransaction.swift | 7 +- Postbox/PreferencesView.swift | 16 +++-- Postbox/ViewTracker.swift | 2 +- Postbox/Views.swift | 22 +++++++ 11 files changed, 178 insertions(+), 27 deletions(-) create mode 100644 Postbox/AccessChallengeDataView.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index bdf7c30fc9..edbcdb9b82 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -55,6 +55,8 @@ D050F2681E4A5B5A00988324 /* TimestampBasedMessageAttributesIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0AAD1B01E3266B100D5B9DE /* TimestampBasedMessageAttributesIndexTable.swift */; }; D050F2691E4A5B5A00988324 /* MultiplePeersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BC386B1E3FCEE50044D6FE /* MultiplePeersView.swift */; }; D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D055BD321B7D3D2D00F06C0A /* MediaBox.swift */; }; + D0575AE31E9ECBB2006F2541 /* AccessChallengeDataView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0575AE21E9ECBB2006F2541 /* AccessChallengeDataView.swift */; }; + D0575AE41E9ECBB2006F2541 /* AccessChallengeDataView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0575AE21E9ECBB2006F2541 /* AccessChallengeDataView.swift */; }; D05F09A61C9E9F9300BB6F96 /* MediaResourceStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */; }; D060B77B1CF4845A0050BE9B /* ReadStateTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */; }; D073CE741DCBF3B4007511FD /* Peer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7831B28AE0900A402D9 /* Peer.swift */; }; @@ -312,6 +314,7 @@ D044E1621B2AD677001EE087 /* MurMurHash32.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MurMurHash32.m; sourceTree = ""; }; D049EAEF1E44D9B900A2CD3A /* PostboxStateView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxStateView.swift; sourceTree = ""; }; D055BD321B7D3D2D00F06C0A /* MediaBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaBox.swift; sourceTree = ""; }; + D0575AE21E9ECBB2006F2541 /* AccessChallengeDataView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccessChallengeDataView.swift; sourceTree = ""; }; D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResourceStatus.swift; sourceTree = ""; }; D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ReadStateTableTests.swift; sourceTree = ""; }; D07516401B2D9CEF00AE42E0 /* sqlite3.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = sqlite3.c; sourceTree = ""; }; @@ -654,6 +657,7 @@ D0FA0AC91E780A26005BB9B7 /* PostboxView.swift */, D0FA0ACC1E781067005BB9B7 /* Views.swift */, D0F53BF21E794C6700117362 /* PeerChatStateView.swift */, + D0575AE21E9ECBB2006F2541 /* AccessChallengeDataView.swift */, ); name = Views; sourceTree = ""; @@ -910,6 +914,7 @@ C2A315BE1E2E733900D89000 /* PeerMergedOperationLogIndexTable.swift in Sources */, C2A315BF1E2E733900D89000 /* PeerMergedOperationLogView.swift in Sources */, C2A315BD1E2E732000D89000 /* PeerOperationLogMetadataTable.swift in Sources */, + D0575AE41E9ECBB2006F2541 /* AccessChallengeDataView.swift in Sources */, C2A315BC1E2E730400D89000 /* PeerOperationLogTable.swift in Sources */, C2AC9C131E1E5D200085C7DE /* UnreadMessageCountsView.swift in Sources */, D0B418481D7DFE20004562A4 /* MessageHistoryView.swift in Sources */, @@ -1033,6 +1038,7 @@ D07CFF831DCA909100761F81 /* PeerChatInterfaceStateTable.swift in Sources */, D0F9E86F1C5A0E7600037222 /* KeychainTable.swift in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, + D0575AE31E9ECBB2006F2541 /* AccessChallengeDataView.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */, D0DF0C8F1D81A350008AEB01 /* PeerView.swift in Sources */, diff --git a/Postbox/AccessChallengeDataView.swift b/Postbox/AccessChallengeDataView.swift new file mode 100644 index 0000000000..74913071ee --- /dev/null +++ b/Postbox/AccessChallengeDataView.swift @@ -0,0 +1,34 @@ +import Foundation + +final class MutableAccessChallengeDataView: MutablePostboxView { + var data: PostboxAccessChallengeData + + init(postbox: Postbox) { + self.data = postbox.metadataTable.accessChallengeData() + } + + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { + var updated = false + + if let data = transaction.updatedAccessChallengeData { + if self.data != data { + self.data = data + updated = true + } + } + + return updated + } + + func immutableView() -> PostboxView { + return AccessChallengeDataView(self) + } +} + +public final class AccessChallengeDataView: PostboxView { + public let data: PostboxAccessChallengeData + + init(_ view: MutableAccessChallengeDataView) { + self.data = view.data + } +} diff --git a/Postbox/AccountManager.swift b/Postbox/AccountManager.swift index 7ab16482e0..43f62d3439 100644 --- a/Postbox/AccountManager.swift +++ b/Postbox/AccountManager.swift @@ -31,7 +31,7 @@ public final class AccountManager { fileprivate init(queue: Queue, basePath: String) { self.queue = queue self.basePath = basePath - let _ = try? FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: false, attributes: nil) + let _ = try? FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) self.valueBox = SqliteValueBox(basePath: basePath + "/db", queue: queue) self.metadataTable = AccountManagerMetadataTable(valueBox: self.valueBox, table: AccountManagerMetadataTable.tableSpec(0)) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 6616cdf65b..ef9ac03ffc 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -528,7 +528,7 @@ public final class MediaBox { } } - public func fetchedResource(_ resource: MediaResource, implNext:Bool = false) -> Signal { + public func fetchedResource(_ resource: MediaResource, implNext: Bool = false) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index 287ec88db6..50ec4da92c 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -8,19 +8,43 @@ private enum MetadataKey: Int32 { case AccessChallenge = 5 } +public struct AccessChallengeAttempts: Coding, Equatable { + public let count: Int32 + public let timestamp: Int32 + + public init(count: Int32, timestamp: Int32) { + self.count = count + self.timestamp = timestamp + } + + public init(decoder: Decoder) { + self.count = decoder.decodeInt32ForKey("c") + self.timestamp = decoder.decodeInt32ForKey("t") + } + + public func encode(_ encoder: Encoder) { + encoder.encodeInt32(self.count, forKey: "c") + encoder.encodeInt32(self.timestamp, forKey: "t") + } + + public static func ==(lhs: AccessChallengeAttempts, rhs: AccessChallengeAttempts) -> Bool { + return lhs.count == rhs.count && lhs.timestamp == rhs.timestamp + } +} + public enum PostboxAccessChallengeData: Coding, Equatable { case none - case numericalPassword(value: String, timeout: Int32?) - case plaintextPassword(value: String, timeout: Int32?) + case numericalPassword(value: String, timeout: Int32?, attempts: AccessChallengeAttempts?) + case plaintextPassword(value: String, timeout: Int32?, attempts: AccessChallengeAttempts?) public init(decoder: Decoder) { switch decoder.decodeInt32ForKey("r") as Int32 { case 0: self = .none case 1: - self = .numericalPassword(value: decoder.decodeStringForKey("t"), timeout: decoder.decodeInt32ForKey("a")) + self = .numericalPassword(value: decoder.decodeStringForKey("t"), timeout: decoder.decodeInt32ForKey("a"), attempts: decoder.decodeObjectForKey("att", decoder: { AccessChallengeAttempts(decoder: $0) }) as? AccessChallengeAttempts) case 2: - self = .plaintextPassword(value: decoder.decodeStringForKey("t"), timeout: decoder.decodeInt32ForKey("a")) + self = .plaintextPassword(value: decoder.decodeStringForKey("t"), timeout: decoder.decodeInt32ForKey("a"), attempts: decoder.decodeObjectForKey("att", decoder: { AccessChallengeAttempts(decoder: $0) }) as? AccessChallengeAttempts) default: assertionFailure() self = .none @@ -31,7 +55,7 @@ public enum PostboxAccessChallengeData: Coding, Equatable { switch self { case .none: encoder.encodeInt32(0, forKey: "r") - case let .numericalPassword(text, timeout): + case let .numericalPassword(text, timeout, attempts): encoder.encodeInt32(1, forKey: "r") encoder.encodeString(text, forKey: "t") if let timeout = timeout { @@ -39,7 +63,12 @@ public enum PostboxAccessChallengeData: Coding, Equatable { } else { encoder.encodeNil(forKey: "a") } - case let .plaintextPassword(text, timeout): + if let attempts = attempts { + encoder.encodeObject(attempts, forKey: "att") + } else { + encoder.encodeNil(forKey: "att") + } + case let .plaintextPassword(text, timeout, attempts): encoder.encodeInt32(2, forKey: "r") encoder.encodeString(text, forKey: "t") if let timeout = timeout { @@ -47,6 +76,11 @@ public enum PostboxAccessChallengeData: Coding, Equatable { } else { encoder.encodeNil(forKey: "a") } + if let attempts = attempts { + encoder.encodeObject(attempts, forKey: "att") + } else { + encoder.encodeNil(forKey: "att") + } } } @@ -58,20 +92,61 @@ public enum PostboxAccessChallengeData: Coding, Equatable { } else { return false } - case let .numericalPassword(lhsText, lhsTimeout): - if case let .numericalPassword(rhsText, rhsTimeout) = rhs, lhsText == rhsText, lhsTimeout == rhsTimeout { + case let .numericalPassword(lhsText, lhsTimeout, lhsAttempts): + if case let .numericalPassword(rhsText, rhsTimeout, rhsAttempts) = rhs, lhsText == rhsText, lhsTimeout == rhsTimeout, lhsAttempts == rhsAttempts { return true } else { return false } - case let .plaintextPassword(lhsText, lhsTimeout): - if case let .plaintextPassword(rhsText, rhsTimeout) = rhs, lhsText == rhsText, lhsTimeout == rhsTimeout { + case let .plaintextPassword(lhsText, lhsTimeout, lhsAttempts): + if case let .plaintextPassword(rhsText, rhsTimeout, rhsAttempts) = rhs, lhsText == rhsText, lhsTimeout == rhsTimeout, lhsAttempts == rhsAttempts { return true } else { return false } } } + + public var isLockable: Bool { + if case .none = self { + return false + } else { + return true + } + } + + public var autolockDeadline: Int32? { + switch self { + case .none: + return nil + case let .numericalPassword(_, timeout, _): + return timeout + case let .plaintextPassword(_, timeout, _): + return timeout + } + } + + public var attempts: AccessChallengeAttempts? { + switch self { + case .none: + return nil + case let .numericalPassword(_, _, attempts): + return attempts + case let .plaintextPassword(_, _, attempts): + return attempts + } + } + + public func withUpdatedAutolockDeadline(_ autolockDeadline: Int32?) -> PostboxAccessChallengeData { + switch self { + case .none: + return self + case let .numericalPassword(value, _, attempts): + return .numericalPassword(value: value, timeout: autolockDeadline, attempts: attempts) + case let .plaintextPassword(value, _, attempts): + return .plaintextPassword(value: value, timeout: autolockDeadline, attempts: attempts) + } + } } final class MetadataTable: Table { diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 27c2407288..8e7b6be3b6 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -535,7 +535,7 @@ public final class Modifier { public func setAccessChallengeData(_ data: PostboxAccessChallengeData) { assert(!self.disposed) - self.postbox?.metadataTable.setAccessChallengeData(data) + self.postbox?.setAccessChallengeData(data) } } @@ -643,6 +643,7 @@ public final class Postbox { private var currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]] = [:] private var currentItemCollectionInfosOperations: [ItemCollectionInfosOperation] = [] private var currentUpdatedPeerChatStates = Set() + private var currentUpdatedAccessChallengeData: PostboxAccessChallengeData? private var currentReplaceChatListHoles: [(MessageIndex, ChatListHole?)] = [] private var currentReplacedContactPeerIds: Set? @@ -1251,7 +1252,7 @@ public final class Postbox { return self.peerTable.get(peerId) }, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) - let transaction = PostboxTransaction(currentUpdatedState: self.currentUpdatedState, currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, currentOrderedItemListOperations: self.currentOrderedItemListOperations, currentItemCollectionItemsOperations: self.currentItemCollectionItemsOperations, currentItemCollectionInfosOperations: self.currentItemCollectionInfosOperations, currentUpdatedPeerChatStates: self.currentUpdatedPeerChatStates, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentUpdatedState: self.currentUpdatedState, currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, currentOrderedItemListOperations: self.currentOrderedItemListOperations, currentItemCollectionItemsOperations: self.currentItemCollectionItemsOperations, currentItemCollectionInfosOperations: self.currentItemCollectionInfosOperations, currentUpdatedPeerChatStates: self.currentUpdatedPeerChatStates, updatedAccessChallengeData: self.currentUpdatedAccessChallengeData, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -1289,6 +1290,7 @@ public final class Postbox { self.currentItemCollectionItemsOperations.removeAll() self.currentItemCollectionInfosOperations.removeAll() self.currentUpdatedPeerChatStates.removeAll() + self.currentUpdatedAccessChallengeData = nil for table in self.tables { table.beforeCommit() @@ -2036,9 +2038,7 @@ public final class Postbox { public func preferencesView(keys: [ValueBoxKey]) -> Signal { return self.modify { modifier -> Signal in - let view = MutablePreferencesView(keys: Set(keys), get: { key in - return self.preferencesTable.get(key: key) - }) + let view = MutablePreferencesView(postbox: self, keys: Set(keys)) let (index, signal) = self.viewTracker.addPreferencesView(view) return (.single(PreferencesView(view)) @@ -2094,6 +2094,11 @@ public final class Postbox { return self.orderedItemListTable.getItemIds(collectionId: collectionId) } + fileprivate func setAccessChallengeData(_ data: PostboxAccessChallengeData) { + self.currentUpdatedAccessChallengeData = data + self.metadataTable.setAccessChallengeData(data) + } + public func isMasterClient() -> Signal { return self.modify { modifier -> Signal in let sessionClientId = self.sessionClientId diff --git a/Postbox/PostboxAccess.swift b/Postbox/PostboxAccess.swift index eb2802f3fb..4e2d344e5e 100644 --- a/Postbox/PostboxAccess.swift +++ b/Postbox/PostboxAccess.swift @@ -38,7 +38,7 @@ public func accessPostbox(basePath: String, password: String?) -> Signal Signal + let updatedAccessChallengeData: PostboxAccessChallengeData? let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -100,10 +101,13 @@ final class PostboxTransaction { if !currentUpdatedPeerChatStates.isEmpty { return false } + if self.updatedAccessChallengeData != nil { + return false + } return true } - init(currentUpdatedState: Coding?, currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]], currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]], currentItemCollectionInfosOperations: [ItemCollectionInfosOperation], currentUpdatedPeerChatStates: Set, updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentUpdatedState: Coding?, currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]], currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]], currentItemCollectionInfosOperations: [ItemCollectionInfosOperation], currentUpdatedPeerChatStates: Set, updatedAccessChallengeData: PostboxAccessChallengeData?, updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentUpdatedState = currentUpdatedState self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles @@ -125,6 +129,7 @@ final class PostboxTransaction { self.currentItemCollectionItemsOperations = currentItemCollectionItemsOperations self.currentItemCollectionInfosOperations = currentItemCollectionInfosOperations self.currentUpdatedPeerChatStates = currentUpdatedPeerChatStates + self.updatedAccessChallengeData = updatedAccessChallengeData self.updatedMedia = updatedMedia self.replaceContactPeerIds = replaceContactPeerIds self.currentUpdatedMasterClientId = currentUpdatedMasterClientId diff --git a/Postbox/PreferencesView.swift b/Postbox/PreferencesView.swift index bdc6195672..4a7b809e03 100644 --- a/Postbox/PreferencesView.swift +++ b/Postbox/PreferencesView.swift @@ -1,23 +1,23 @@ import Foundation -final class MutablePreferencesView { +final class MutablePreferencesView: MutablePostboxView { fileprivate let keys: Set fileprivate var values: [ValueBoxKey: PreferencesEntry] - init(keys: Set, get: (ValueBoxKey) -> PreferencesEntry?) { + init(postbox: Postbox, keys: Set) { self.keys = keys var values: [ValueBoxKey: PreferencesEntry] = [:] for key in keys { - if let value = get(key) { + if let value = postbox.preferencesTable.get(key: key) { values[key] = value } } self.values = values } - func replay(operations: [PreferencesOperation], get: (ValueBoxKey) -> PreferencesEntry?) -> Bool { + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { var updated = false - for operation in operations { + for operation in transaction.currentPreferencesOperations { switch operation { case let .update(key, value): if self.keys.contains(key) { @@ -44,9 +44,13 @@ final class MutablePreferencesView { return updated } + + func immutableView() -> PostboxView { + return PreferencesView(self) + } } -public final class PreferencesView { +public final class PreferencesView: PostboxView { public let values: [ValueBoxKey: PreferencesEntry] init(_ view: MutablePreferencesView) { diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 7495e4cc4f..b5f4ca9d38 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -501,7 +501,7 @@ final class ViewTracker { } for (mutableView, pipe) in self.preferencesViews.copyItems() { - if mutableView.replay(operations: transaction.currentPreferencesOperations, get: self.getPreferencesEntry) { + if mutableView.replay(postbox: postbox, transaction: transaction) { pipe.putNext(PreferencesView(mutableView)) } } diff --git a/Postbox/Views.swift b/Postbox/Views.swift index c0ccb6b3b1..43fa9950df 100644 --- a/Postbox/Views.swift +++ b/Postbox/Views.swift @@ -6,6 +6,8 @@ public enum PostboxViewKey: Hashable { case itemCollectionInfo(id: ItemCollectionId) case peerChatState(peerId: PeerId) case orderedItemList(id: Int32) + case accessChallengeData + case preferences(keys: Set) public var hashValue: Int { switch self { @@ -19,6 +21,10 @@ public enum PostboxViewKey: Hashable { return id.hashValue case let .orderedItemList(id): return id.hashValue + case .accessChallengeData: + return 2 + case .preferences: + return 3 } } @@ -54,6 +60,18 @@ public enum PostboxViewKey: Hashable { } else { return false } + case .accessChallengeData: + if case .accessChallengeData = rhs { + return true + } else { + return false + } + case let .preferences(lhsKeys): + if case let .preferences(rhsKeys) = rhs, lhsKeys == rhsKeys { + return true + } else { + return false + } } } } @@ -70,5 +88,9 @@ func postboxViewForKey(postbox: Postbox, key: PostboxViewKey) -> MutablePostboxV return MutablePeerChatStateView(postbox: postbox, peerId: peerId) case let .orderedItemList(id): return MutableOrderedItemListView(postbox: postbox, collectionId: id) + case .accessChallengeData: + return MutableAccessChallengeDataView(postbox: postbox) + case let .preferences(keys): + return MutablePreferencesView(postbox: postbox, keys: keys) } } From 0e0646ddc9ef0ec92865f265261aea62e9eb9ecc Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 14 Apr 2017 16:04:22 +0300 Subject: [PATCH 132/427] no message --- Postbox/OrderedItemListTable.swift | 29 +++++++++++++++++++++++++++++ Postbox/OrderedItemListView.swift | 8 ++++++++ Postbox/Postbox.swift | 18 ++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/Postbox/OrderedItemListTable.swift b/Postbox/OrderedItemListTable.swift index f09eaac970..c11fa02f3b 100644 --- a/Postbox/OrderedItemListTable.swift +++ b/Postbox/OrderedItemListTable.swift @@ -3,6 +3,7 @@ import Foundation enum OrderedItemListOperation { case replace([OrderedItemListEntry]) case addOrMoveToFirstPosition(OrderedItemListEntry, Int?) + case remove(MemoryBuffer) } private enum OrderedItemListKeyNamespace: UInt8 { @@ -75,6 +76,14 @@ final class OrderedItemListTable: Table { return items } + func getItem(collectionId: Int32, itemId: MemoryBuffer) -> OrderedItemListEntry? { + if let contents = self.indexTable.get(collectionId: collectionId, id: itemId) as? OrderedItemListEntryContents { + return OrderedItemListEntry(id: itemId, contents: contents) + } else { + return nil + } + } + func replaceItems(collectionId: Int32, items: [OrderedItemListEntry], operations: inout [Int32: [OrderedItemListOperation]]) { if operations[collectionId] == nil { operations[collectionId] = [.replace(items)] @@ -146,6 +155,26 @@ final class OrderedItemListTable: Table { self.valueBox.set(self.table, key: self.keyIdToIndex(collectionId: collectionId, id: item.id), value: MemoryBuffer(memory: &itemIndex, capacity: 4, length: 4, freeWhenDone: false)) } + func remove(collectionId: Int32, itemId: MemoryBuffer, operations: inout [Int32: [OrderedItemListOperation]]) { + if let index = self.getIndex(collectionId: collectionId, id: itemId) { + var orderedIds = self.getItemIds(collectionId: collectionId) + + self.valueBox.remove(self.table, key: self.keyIdToIndex(collectionId: collectionId, id: itemId)) + self.valueBox.remove(self.table, key: self.keyIndexToId(collectionId: collectionId, itemIndex: index)) + + for i in (Int(index) + 1) ..< orderedIds.count { + var updatedIndex: UInt32 = UInt32(i - 1) + self.valueBox.set(self.table, key: self.keyIdToIndex(collectionId: collectionId, id: orderedIds[i]), value: MemoryBuffer(memory: &updatedIndex, capacity: 4, length: 4, freeWhenDone: false)) + self.valueBox.set(self.table, key: self.keyIndexToId(collectionId: collectionId, itemIndex: updatedIndex), value: orderedIds[i]) + } + + if operations[collectionId] == nil { + operations[collectionId] = [] + } + operations[collectionId]!.append(.remove(itemId)) + } + } + func testIntegrity(collectionId: Int32) -> Bool { let orderedIds = self.getItemIds(collectionId: collectionId).map { ValueBoxKey($0) } diff --git a/Postbox/OrderedItemListView.swift b/Postbox/OrderedItemListView.swift index 23c07d6969..7f51202e2d 100644 --- a/Postbox/OrderedItemListView.swift +++ b/Postbox/OrderedItemListView.swift @@ -29,6 +29,14 @@ final class MutableOrderedItemListView: MutablePostboxView { } } updated = true + case let .remove(itemId): + inner: for i in 0 ..< self.items.count { + if self.items[i].id == itemId { + self.items.remove(at: i) + updated = true + break inner + } + } } } } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 8e7b6be3b6..4eac61547a 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -396,6 +396,16 @@ public final class Modifier { } } + public func getOrderedItemListItem(collectionId: Int32, itemId: MemoryBuffer) -> OrderedItemListEntry? { + assert(!self.disposed) + return self.postbox?.getOrderedItemListItem(collectionId: collectionId, itemId: itemId) + } + + public func removeOrderedItemListItem(collectionId: Int32, itemId: MemoryBuffer) { + assert(!self.disposed) + self.postbox?.removeOrderedItemListItem(collectionId: collectionId, itemId: itemId) + } + public func getMessage(_ id: MessageId) -> Message? { assert(!self.disposed) if let postbox = self.postbox { @@ -2090,10 +2100,18 @@ public final class Postbox { self.orderedItemListTable.addItemOrMoveToFirstPosition(collectionId: collectionId, item: item, removeTailIfCountExceeds: removeTailIfCountExceeds, operations: &self.currentOrderedItemListOperations) } + fileprivate func removeOrderedItemListItem(collectionId: Int32, itemId: MemoryBuffer) { + self.orderedItemListTable.remove(collectionId: collectionId, itemId: itemId, operations: &self.currentOrderedItemListOperations) + } + fileprivate func getOrderedListItemIds(collectionId: Int32) -> [MemoryBuffer] { return self.orderedItemListTable.getItemIds(collectionId: collectionId) } + fileprivate func getOrderedItemListItem(collectionId: Int32, itemId: MemoryBuffer) -> OrderedItemListEntry? { + return self.orderedItemListTable.getItem(collectionId: collectionId, itemId: itemId) + } + fileprivate func setAccessChallengeData(_ data: PostboxAccessChallengeData) { self.currentUpdatedAccessChallengeData = data self.metadataTable.setAccessChallengeData(data) From b19a770951cbb335348ab76601b7a2e671dd3f44 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 18 Apr 2017 20:05:09 +0300 Subject: [PATCH 133/427] no message --- Postbox/MediaBox.swift | 53 +++++++++++++++++++++++++------ Postbox/MediaResource.swift | 3 ++ Postbox/MessageHistoryTable.swift | 51 +++++++++++++++++++++++++++++ Postbox/MessageMediaTable.swift | 2 +- Postbox/Postbox.swift | 9 ++++++ 5 files changed, 107 insertions(+), 11 deletions(-) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index ef9ac03ffc..b66a83e838 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -116,8 +116,8 @@ public final class MediaBox { private var randomAccessContexts: [WrappedMediaResourceId: RandomAccessMediaResourceContext] = [:] private var cachedRepresentationContexts: [CachedMediaResourceRepresentationKey: CachedMediaResourceRepresentationContext] = [:] - private var wrappedFetchResource = Promise<(MediaResource, Range) -> Signal>() - public var fetchResource: ((MediaResource, Range) -> Signal)? { + private var wrappedFetchResource = Promise<(MediaResource, Range, MediaResourceFetchTag?) -> Signal>() + public var fetchResource: ((MediaResource, Range, MediaResourceFetchTag?) -> Signal)? { didSet { if let fetchResource = self.fetchResource { wrappedFetchResource.set(.single(fetchResource)) @@ -399,7 +399,7 @@ public final class MediaBox { } } - private func randomAccessContext(for resource: MediaResource, size: Int) -> RandomAccessMediaResourceContext { + private func randomAccessContext(for resource: MediaResource, size: Int, tag: MediaResourceFetchTag?) -> RandomAccessMediaResourceContext { assert(self.dataQueue.isCurrent()) let resourceId = WrappedMediaResourceId(resource.id) @@ -415,7 +415,7 @@ public final class MediaBox { if let strongSelf = self { strongSelf.dataQueue.async { let fetch = strongSelf.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in - return fetch(resource, range) + return fetch(resource, range, tag) } var offset = 0 disposable.set(fetch.start(next: { [weak strongSelf] result in @@ -444,13 +444,13 @@ public final class MediaBox { return dataContext } - public func fetchedResourceData(_ resource: MediaResource, size: Int, in range: Range) -> Signal { + public func fetchedResourceData(_ resource: MediaResource, size: Int, in range: Range, tag: MediaResourceFetchTag?) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.dataQueue.async { let resourceId = WrappedMediaResourceId(resource.id) - let dataContext = self.randomAccessContext(for: resource, size: size) + let dataContext = self.randomAccessContext(for: resource, size: size, tag: tag) let listener = dataContext.addListenerForFetchedData(in: range) @@ -472,13 +472,13 @@ public final class MediaBox { } } - public func resourceData(_ resource: MediaResource, size: Int, in range: Range, mode: ResourceDataRangeMode = .complete) -> Signal { + public func resourceData(_ resource: MediaResource, size: Int, in range: Range, tag: MediaResourceFetchTag?, mode: ResourceDataRangeMode = .complete) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.dataQueue.async { let resourceId = WrappedMediaResourceId(resource.id) - let dataContext = self.randomAccessContext(for: resource, size: size) + let dataContext = self.randomAccessContext(for: resource, size: size, tag: tag) let listenerMode: RandomAccessResourceDataRangeMode switch mode { @@ -528,7 +528,7 @@ public final class MediaBox { } } - public func fetchedResource(_ resource: MediaResource, implNext: Bool = false) -> Signal { + public func fetchedResource(_ resource: MediaResource, tag: MediaResourceFetchTag?, implNext: Bool = false) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -573,7 +573,7 @@ public final class MediaBox { var fd: Int32? let dataQueue = self.dataQueue dataContext.fetchDisposable = ((self.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in - return fetch(resource, currentSize ..< Int.max) + return fetch(resource, currentSize ..< Int.max, tag) }) |> afterDisposed { dataQueue.async { if let fd = fd { @@ -963,4 +963,37 @@ public final class MediaBox { return disposable } } + + public func collectResourceCacheUsage(_ ids: [MediaResourceId]) -> Signal<[WrappedMediaResourceId: Int64], NoError> { + return Signal { subscriber in + self.dataQueue.async { + var result: [WrappedMediaResourceId: Int64] = [:] + for id in ids { + let wrappedId = WrappedMediaResourceId(id) + let paths = self.storePathsForId(id) + if let size = fileSize(paths.complete) { + result[wrappedId] = Int64(size) + } else if let size = fileSize(paths.partial) { + result[wrappedId] = Int64(size) + } + } + subscriber.putNext(result) + subscriber.putCompletion() + } + return EmptyDisposable + } + } + + public func removeCachedResources(_ ids: Set) -> Signal { + return Signal { subscriber in + self.dataQueue.async { + for id in ids { + let paths = self.storePathsForId(id.id) + unlink(paths.complete) + unlink(paths.partial) + } + } + return EmptyDisposable + } + } } diff --git a/Postbox/MediaResource.swift b/Postbox/MediaResource.swift index 96c028cba1..c24634ea23 100644 --- a/Postbox/MediaResource.swift +++ b/Postbox/MediaResource.swift @@ -47,3 +47,6 @@ public protocol CachedMediaResourceRepresentation { var uniqueId: String { get } func isEqual(to: CachedMediaResourceRepresentation) -> Bool } + +public protocol MediaResourceFetchTag { +} diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index d4cacf9436..2fbef602be 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -1827,6 +1827,57 @@ final class MessageHistoryTable: Table { func getMessageCountInRange(peerId: PeerId, tagMask: MessageTags, lowerBound: MessageIndex, upperBound: MessageIndex) -> Int32 { return self.tagsTable.getMessageCountInRange(tagMask: tagMask, peerId: peerId, lowerBound: lowerBound, upperBound: upperBound) } + + func enumerateMedia(lowerBound: MessageIndex?, limit: Int) -> ([PeerId: Set], [MediaId: Media], MessageIndex?) { + var mediaRefs: [MediaId: Media] = [:] + var result: [PeerId: Set] = [:] + var lastIndex: MessageIndex? + var count = 0 + self.valueBox.range(self.table, start: self.key(lowerBound == nil ? MessageIndex.absoluteLowerBound() : lowerBound!), end: self.key(MessageIndex.absoluteUpperBound()), values: { key, value in + count += 1 + + let entry = self.readIntermediateEntry(key, value: value) + lastIndex = entry.index + + if case let .Message(message) = entry { + var parsedMedia: [Media] = [] + + let embeddedMediaData = message.embeddedMediaData.sharedBufferNoCopy() + if embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + parsedMedia.append(media) + } + embeddedMediaData.skip(Int(mediaLength)) + } + } + + for mediaId in message.referencedMedia { + if let media = self.messageMediaTable.get(mediaId, embedded: { _ in + return nil + }) { + parsedMedia.append(media) + } + } + + for media in parsedMedia { + if let id = media.id { + mediaRefs[id] = media + if result[message.id.peerId] == nil { + result[message.id.peerId] = Set() + } + result[message.id.peerId]!.insert(id) + } + } + } + return true + }, limit: limit) + return (result, mediaRefs, count == 0 ? nil : lastIndex) + } func debugList(_ peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] diff --git a/Postbox/MessageMediaTable.swift b/Postbox/MessageMediaTable.swift index 8b9e15aee2..e0fdfd35c4 100644 --- a/Postbox/MessageMediaTable.swift +++ b/Postbox/MessageMediaTable.swift @@ -31,7 +31,7 @@ final class MessageMediaTable: Table { return key } - func get(_ id: MediaId, embedded: @noescape(MessageIndex, MediaId) -> Media?) -> Media? { + func get(_ id: MediaId, embedded: (MessageIndex, MediaId) -> Media?) -> Media? { if let value = self.valueBox.get(self.table, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 4eac61547a..f35f6f59a5 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -547,6 +547,15 @@ public final class Modifier { assert(!self.disposed) self.postbox?.setAccessChallengeData(data) } + + public func enumerateMedia(lowerBound: MessageIndex?, limit: Int) -> ([PeerId: Set], [MediaId: Media], MessageIndex?) { + assert(!self.disposed) + if let postbox = self.postbox { + return postbox.messageHistoryTable.enumerateMedia(lowerBound: lowerBound, limit: limit) + } else { + return ([:], [:], nil) + } + } } fileprivate class PipeNotifier: NSObject { From a3ba5f30434073291d072b2ecb36432fded48732 Mon Sep 17 00:00:00 2001 From: overtake Date: Thu, 20 Apr 2017 18:45:56 +0300 Subject: [PATCH 134/427] no message --- Postbox/MediaBox.swift | 53 +++++++++++++++++++++++++------ Postbox/MediaResource.swift | 3 ++ Postbox/MessageHistoryTable.swift | 51 +++++++++++++++++++++++++++++ Postbox/MessageMediaTable.swift | 2 +- Postbox/Postbox.swift | 9 ++++++ 5 files changed, 107 insertions(+), 11 deletions(-) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index ef9ac03ffc..b66a83e838 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -116,8 +116,8 @@ public final class MediaBox { private var randomAccessContexts: [WrappedMediaResourceId: RandomAccessMediaResourceContext] = [:] private var cachedRepresentationContexts: [CachedMediaResourceRepresentationKey: CachedMediaResourceRepresentationContext] = [:] - private var wrappedFetchResource = Promise<(MediaResource, Range) -> Signal>() - public var fetchResource: ((MediaResource, Range) -> Signal)? { + private var wrappedFetchResource = Promise<(MediaResource, Range, MediaResourceFetchTag?) -> Signal>() + public var fetchResource: ((MediaResource, Range, MediaResourceFetchTag?) -> Signal)? { didSet { if let fetchResource = self.fetchResource { wrappedFetchResource.set(.single(fetchResource)) @@ -399,7 +399,7 @@ public final class MediaBox { } } - private func randomAccessContext(for resource: MediaResource, size: Int) -> RandomAccessMediaResourceContext { + private func randomAccessContext(for resource: MediaResource, size: Int, tag: MediaResourceFetchTag?) -> RandomAccessMediaResourceContext { assert(self.dataQueue.isCurrent()) let resourceId = WrappedMediaResourceId(resource.id) @@ -415,7 +415,7 @@ public final class MediaBox { if let strongSelf = self { strongSelf.dataQueue.async { let fetch = strongSelf.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in - return fetch(resource, range) + return fetch(resource, range, tag) } var offset = 0 disposable.set(fetch.start(next: { [weak strongSelf] result in @@ -444,13 +444,13 @@ public final class MediaBox { return dataContext } - public func fetchedResourceData(_ resource: MediaResource, size: Int, in range: Range) -> Signal { + public func fetchedResourceData(_ resource: MediaResource, size: Int, in range: Range, tag: MediaResourceFetchTag?) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.dataQueue.async { let resourceId = WrappedMediaResourceId(resource.id) - let dataContext = self.randomAccessContext(for: resource, size: size) + let dataContext = self.randomAccessContext(for: resource, size: size, tag: tag) let listener = dataContext.addListenerForFetchedData(in: range) @@ -472,13 +472,13 @@ public final class MediaBox { } } - public func resourceData(_ resource: MediaResource, size: Int, in range: Range, mode: ResourceDataRangeMode = .complete) -> Signal { + public func resourceData(_ resource: MediaResource, size: Int, in range: Range, tag: MediaResourceFetchTag?, mode: ResourceDataRangeMode = .complete) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.dataQueue.async { let resourceId = WrappedMediaResourceId(resource.id) - let dataContext = self.randomAccessContext(for: resource, size: size) + let dataContext = self.randomAccessContext(for: resource, size: size, tag: tag) let listenerMode: RandomAccessResourceDataRangeMode switch mode { @@ -528,7 +528,7 @@ public final class MediaBox { } } - public func fetchedResource(_ resource: MediaResource, implNext: Bool = false) -> Signal { + public func fetchedResource(_ resource: MediaResource, tag: MediaResourceFetchTag?, implNext: Bool = false) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -573,7 +573,7 @@ public final class MediaBox { var fd: Int32? let dataQueue = self.dataQueue dataContext.fetchDisposable = ((self.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in - return fetch(resource, currentSize ..< Int.max) + return fetch(resource, currentSize ..< Int.max, tag) }) |> afterDisposed { dataQueue.async { if let fd = fd { @@ -963,4 +963,37 @@ public final class MediaBox { return disposable } } + + public func collectResourceCacheUsage(_ ids: [MediaResourceId]) -> Signal<[WrappedMediaResourceId: Int64], NoError> { + return Signal { subscriber in + self.dataQueue.async { + var result: [WrappedMediaResourceId: Int64] = [:] + for id in ids { + let wrappedId = WrappedMediaResourceId(id) + let paths = self.storePathsForId(id) + if let size = fileSize(paths.complete) { + result[wrappedId] = Int64(size) + } else if let size = fileSize(paths.partial) { + result[wrappedId] = Int64(size) + } + } + subscriber.putNext(result) + subscriber.putCompletion() + } + return EmptyDisposable + } + } + + public func removeCachedResources(_ ids: Set) -> Signal { + return Signal { subscriber in + self.dataQueue.async { + for id in ids { + let paths = self.storePathsForId(id.id) + unlink(paths.complete) + unlink(paths.partial) + } + } + return EmptyDisposable + } + } } diff --git a/Postbox/MediaResource.swift b/Postbox/MediaResource.swift index 96c028cba1..c24634ea23 100644 --- a/Postbox/MediaResource.swift +++ b/Postbox/MediaResource.swift @@ -47,3 +47,6 @@ public protocol CachedMediaResourceRepresentation { var uniqueId: String { get } func isEqual(to: CachedMediaResourceRepresentation) -> Bool } + +public protocol MediaResourceFetchTag { +} diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index d4cacf9436..2fbef602be 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -1827,6 +1827,57 @@ final class MessageHistoryTable: Table { func getMessageCountInRange(peerId: PeerId, tagMask: MessageTags, lowerBound: MessageIndex, upperBound: MessageIndex) -> Int32 { return self.tagsTable.getMessageCountInRange(tagMask: tagMask, peerId: peerId, lowerBound: lowerBound, upperBound: upperBound) } + + func enumerateMedia(lowerBound: MessageIndex?, limit: Int) -> ([PeerId: Set], [MediaId: Media], MessageIndex?) { + var mediaRefs: [MediaId: Media] = [:] + var result: [PeerId: Set] = [:] + var lastIndex: MessageIndex? + var count = 0 + self.valueBox.range(self.table, start: self.key(lowerBound == nil ? MessageIndex.absoluteLowerBound() : lowerBound!), end: self.key(MessageIndex.absoluteUpperBound()), values: { key, value in + count += 1 + + let entry = self.readIntermediateEntry(key, value: value) + lastIndex = entry.index + + if case let .Message(message) = entry { + var parsedMedia: [Media] = [] + + let embeddedMediaData = message.embeddedMediaData.sharedBufferNoCopy() + if embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + parsedMedia.append(media) + } + embeddedMediaData.skip(Int(mediaLength)) + } + } + + for mediaId in message.referencedMedia { + if let media = self.messageMediaTable.get(mediaId, embedded: { _ in + return nil + }) { + parsedMedia.append(media) + } + } + + for media in parsedMedia { + if let id = media.id { + mediaRefs[id] = media + if result[message.id.peerId] == nil { + result[message.id.peerId] = Set() + } + result[message.id.peerId]!.insert(id) + } + } + } + return true + }, limit: limit) + return (result, mediaRefs, count == 0 ? nil : lastIndex) + } func debugList(_ peerId: PeerId, peerTable: PeerTable) -> [RenderedMessageHistoryEntry] { var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] diff --git a/Postbox/MessageMediaTable.swift b/Postbox/MessageMediaTable.swift index 8b9e15aee2..e0fdfd35c4 100644 --- a/Postbox/MessageMediaTable.swift +++ b/Postbox/MessageMediaTable.swift @@ -31,7 +31,7 @@ final class MessageMediaTable: Table { return key } - func get(_ id: MediaId, embedded: @noescape(MessageIndex, MediaId) -> Media?) -> Media? { + func get(_ id: MediaId, embedded: (MessageIndex, MediaId) -> Media?) -> Media? { if let value = self.valueBox.get(self.table, key: self.key(id)) { var type: Int8 = 0 value.read(&type, offset: 0, length: 1) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 4eac61547a..f35f6f59a5 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -547,6 +547,15 @@ public final class Modifier { assert(!self.disposed) self.postbox?.setAccessChallengeData(data) } + + public func enumerateMedia(lowerBound: MessageIndex?, limit: Int) -> ([PeerId: Set], [MediaId: Media], MessageIndex?) { + assert(!self.disposed) + if let postbox = self.postbox { + return postbox.messageHistoryTable.enumerateMedia(lowerBound: lowerBound, limit: limit) + } else { + return ([:], [:], nil) + } + } } fileprivate class PipeNotifier: NSObject { From a73bcc3180f38e8cd24b49b6fc05a5e5f232b338 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 27 Apr 2017 22:47:14 +0300 Subject: [PATCH 135/427] no message --- Postbox/Coding.swift | 106 ++++++++++++++++++++++- Postbox/Database.swift | 11 ++- Postbox/MappedFile.swift | 4 +- Postbox/MediaBox.swift | 10 ++- Postbox/PeerMergedOperationLogView.swift | 4 +- Postbox/SqliteValueBox.swift | 100 +++++++++++++-------- 6 files changed, 190 insertions(+), 45 deletions(-) diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index d5acbb6d98..a2a4f9779a 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -252,6 +252,15 @@ public final class Encoder { self.buffer.write(key.utf8Start, offset: 0, length: Int(length)) } + public func encodeKey(_ key: String) { + let data = key.data(using: .utf8)! + data.withUnsafeBytes { (keyBytes: UnsafePointer) -> Void in + var length: Int8 = Int8(data.count) + self.buffer.write(&length, offset: 0, length: 1) + self.buffer.write(keyBytes, offset: 0, length: Int(length)) + } + } + public func encodeNil(forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Nil.rawValue @@ -266,6 +275,14 @@ public final class Encoder { self.buffer.write(&v, offset: 0, length: 4) } + public func encodeInt32(_ value: Int32, forKey key: String) { + self.encodeKey(key) + var type: Int8 = ValueType.Int32.rawValue + self.buffer.write(&type, offset: 0, length: 1) + var v = value + self.buffer.write(&v, offset: 0, length: 4) + } + public func encodeInt64(_ value: Int64, forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int64.rawValue @@ -325,6 +342,23 @@ public final class Encoder { self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(length)) } + public func encodeObjectWithEncoder(_ value: T, encoder: (Encoder) -> Void, forKey key: String) { + self.encodeKey(key) + var type: Int8 = ValueType.Object.rawValue + self.buffer.write(&type, offset: 0, length: 1) + + let string = "\(type(of: value))" + var typeHash: Int32 = murMurHashString32(string) + self.buffer.write(&typeHash, offset: 0, length: 4) + + let innerEncoder = Encoder() + encoder(innerEncoder) + + var length: Int32 = Int32(innerEncoder.buffer.offset) + self.buffer.write(&length, offset: 0, length: 4) + self.buffer.write(innerEncoder.buffer.memory, offset: 0, length: Int(length)) + } + public func encodeInt32Array(_ value: [Int32], forKey key: StaticString) { self.encodeKey(key) var type: Int8 = ValueType.Int32Array.rawValue @@ -591,6 +625,47 @@ public final class Decoder { return false } + + private class func positionOnStringKey(_ rawBytes: UnsafeRawPointer, offset: inout Int, maxOffset: Int, length: Int, key: String, valueType: ValueType) -> Bool + { + let bytes = rawBytes.assumingMemoryBound(to: Int8.self) + + let startOffset = offset + + let keyData = key.data(using: .utf8)! + + return keyData.withUnsafeBytes { (keyBytes: UnsafePointer) -> Bool in + let keyLength: Int = keyData.count + while (offset < maxOffset) { + let readKeyLength = bytes[offset] + assert(readKeyLength >= 0) + offset += 1 + offset += Int(readKeyLength) + + let readValueType = bytes[offset] + offset += 1 + + if keyLength == Int(readKeyLength) && memcmp(bytes + (offset - Int(readKeyLength) - 1), keyBytes, keyLength) == 0 { + if readValueType == valueType.rawValue { + return true + } else if readValueType == ValueType.Nil.rawValue { + return false + } else { + skipValue(bytes, offset: &offset, length: length, valueType: ValueType(rawValue: readValueType)!) + } + } else { + skipValue(bytes, offset: &offset, length: length, valueType: ValueType(rawValue: readValueType)!) + } + } + + if (startOffset != 0) { + offset = 0 + return positionOnStringKey(bytes, offset: &offset, maxOffset: startOffset, length: length, key: key, valueType: valueType) + } + + return false + } + } private class func positionOnKey(_ bytes: UnsafePointer, offset: inout Int, maxOffset: Int, length: Int, key: Int16, valueType: ValueType) -> Bool { @@ -634,6 +709,17 @@ public final class Decoder { } } + public func decodeInt32ForKey(_ key: String) -> Int32 { + if Decoder.positionOnStringKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { + var value: Int32 = 0 + memcpy(&value, self.buffer.memory + self.offset, 4) + self.offset += 4 + return value + } else { + return 0 + } + } + public func decodeInt32ForKey(_ key: StaticString) -> Int32? { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 @@ -753,6 +839,24 @@ public final class Decoder { } } + public func decodeObjectForKeyThrowing(_ key: StaticString, decoder: (Decoder) throws -> Any) throws -> Any? { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Object) { + var typeHash: Int32 = 0 + memcpy(&typeHash, self.buffer.memory + self.offset, 4) + self.offset += 4 + + var length: Int32 = 0 + memcpy(&length, self.buffer.memory + self.offset, 4) + + let innerDecoder = Decoder(buffer: ReadBuffer(memory: self.buffer.memory + (self.offset + 4), length: Int(length), freeWhenDone: false)) + self.offset += 4 + Int(length) + + return try decoder(innerDecoder) + } else { + return nil + } + } + public func decodeInt32ArrayForKey(_ key: StaticString) -> [Int32] { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32Array) { var length: Int32 = 0 @@ -1023,7 +1127,7 @@ public final class Decoder { } } - public func decodeObjectDictionaryForKey(_ key: StaticString, keyDecoder: (Decoder) -> K) -> [K : V] where K: Hashable { + public func decodeObjectDictionaryForKey(_ key: StaticString, keyDecoder: (Decoder) -> K) -> [K : V] where K: Hashable, K: Hashable { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .ObjectDictionary) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) diff --git a/Postbox/Database.swift b/Postbox/Database.swift index ba023b6db3..1712061b6d 100644 --- a/Postbox/Database.swift +++ b/Postbox/Database.swift @@ -68,8 +68,15 @@ public final class Database { /// /// - returns: A new database connection. public init?(_ location: Location = .InMemory, readonly: Bool = false) { - let flags = readonly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE - let res = sqlite3_open_v2(location.description, &self.handle, flags | SQLITE_OPEN_FULLMUTEX, nil) + switch location { + case let .URI(uri): + let _ = open(uri + "-guard", O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) + break + default: + break + } + //let flags = readonly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE + let res = sqlite3_open(location.description, &self.handle) if res != SQLITE_OK { preconditionFailure("sqlite3_open_v2: \(res)") return nil diff --git a/Postbox/MappedFile.swift b/Postbox/MappedFile.swift index d0486c685f..23fb19dfcf 100644 --- a/Postbox/MappedFile.swift +++ b/Postbox/MappedFile.swift @@ -37,11 +37,11 @@ public final class MappedFile { msync(self.memory, self.currentSize, MS_ASYNC) } - public func write(at range: Range, from data: UnsafePointer) { + public func write(at range: Range, from data: UnsafeRawPointer) { memcpy(self.memory.advanced(by: range.lowerBound), data, range.count) } - public func read(at range: Range, to data: UnsafeMutablePointer) { + public func read(at range: Range, to data: UnsafeMutableRawPointer) { memcpy(data, self.memory.advanced(by: range.lowerBound), range.count) } diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index b66a83e838..13abd5cd0b 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -576,8 +576,9 @@ public final class MediaBox { return fetch(resource, currentSize ..< Int.max, tag) }) |> afterDisposed { dataQueue.async { - if let fd = fd { - close(fd) + if let thisFd = fd { + close(thisFd) + fd = nil } } }).start(next: { resultOption in @@ -750,8 +751,9 @@ public final class MediaBox { } } case let .moveLocalFile(tempPath): - if let fd = fd { - close(fd) + if let thisFd = thisFd { + close(thisFd) + fd = nil } unlink(paths.partial) do { diff --git a/Postbox/PeerMergedOperationLogView.swift b/Postbox/PeerMergedOperationLogView.swift index e07117b152..c403ced292 100644 --- a/Postbox/PeerMergedOperationLogView.swift +++ b/Postbox/PeerMergedOperationLogView.swift @@ -69,7 +69,7 @@ final class MutablePeerMergedOperationLogView { } for i in 0 ..< self.entries.count { if i != 0 { - assert(self.entries[i].mergedIndex == self.entries[i - 1].mergedIndex + 1) + assert(self.entries[i].mergedIndex >= self.entries[i - 1].mergedIndex + 1) } } if !self.entries.isEmpty { @@ -80,7 +80,7 @@ final class MutablePeerMergedOperationLogView { } else { assert(self.tailIndex != nil) if let tailIndex = self.tailIndex { - assert(self.entries.last!.mergedIndex == tailIndex) + assert(self.entries.last!.mergedIndex <= tailIndex) } } } diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 3680431478..a9b2ab6097 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -141,12 +141,19 @@ final class SqliteValueBox: ValueBox { //database = Database(path)! } - database.execute("PRAGMA cache_size=-2097152") - database.execute("PRAGMA synchronous=NORMAL") - database.execute("PRAGMA journal_mode=WAL") - database.execute("PRAGMA temp_store=MEMORY") - database.execute("PRAGMA wal_autocheckpoint=500") - database.execute("PRAGMA journal_size_limit=1536") + sqlite3_busy_timeout(database.handle, 10000000) + + var resultCode: Bool + + //database.execute("PRAGMA cache_size=-2097152") + resultCode = database.execute("PRAGMA journal_mode=WAL") + assert(resultCode) + resultCode = database.execute("PRAGMA synchronous=NORMAL") + assert(resultCode) + //database.execute("PRAGMA temp_store=MEMORY") + resultCode = database.execute("PRAGMA wal_autocheckpoint=500") + assert(resultCode) + //database.execute("PRAGMA journal_size_limit=1536") /*var statement: OpaquePointer? = nil sqlite3_prepare_v2(database.handle, "PRAGMA integrity_check", -1, &statement, nil) @@ -161,13 +168,14 @@ final class SqliteValueBox: ValueBox { } preparedStatement.destroy()*/ - sqlite3_busy_timeout(database.handle, 10000000) - let result = self.getUserVersion(database) if result != 2 { - database.execute("PRAGMA user_version=2") - database.execute("DROP TABLE IF EXISTS __meta_tables") - database.execute("CREATE TABLE __meta_tables (name INTEGER, keyType INTEGER)") + resultCode = database.execute("PRAGMA user_version=2") + assert(resultCode) + resultCode = database.execute("DROP TABLE IF EXISTS __meta_tables") + assert(resultCode) + resultCode = database.execute("CREATE TABLE __meta_tables (name INTEGER, keyType INTEGER)") + assert(resultCode) } for table in self.listTables(database) { @@ -202,20 +210,23 @@ final class SqliteValueBox: ValueBox { public func begin() { assert(self.queue.isCurrent()) - self.database.execute("BEGIN IMMEDIATE") + let resultCode = self.database.execute("BEGIN IMMEDIATE") + assert(resultCode) } public func commit() { assert(self.queue.isCurrent()) let startTime = CFAbsoluteTimeGetCurrent() - self.database.execute("COMMIT") + let resultCode = self.database.execute("COMMIT") + assert(resultCode) self.commitTime += CFAbsoluteTimeGetCurrent() - startTime } private func getUserVersion(_ database: Database) -> Int64 { assert(self.queue.isCurrent()) var statement: OpaquePointer? = nil - sqlite3_prepare_v2(database.handle, "PRAGMA user_version", -1, &statement, nil) + let status = sqlite3_prepare_v2(database.handle, "PRAGMA user_version", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) let _ = preparedStatement.step() let value = preparedStatement.int64At(0) @@ -226,7 +237,8 @@ final class SqliteValueBox: ValueBox { private func listTables(_ database: Database) -> [ValueBoxTable] { assert(self.queue.isCurrent()) var statement: OpaquePointer? = nil - sqlite3_prepare_v2(database.handle, "SELECT name, keyType FROM __meta_tables", -1, &statement, nil) + let status = sqlite3_prepare_v2(database.handle, "SELECT name, keyType FROM __meta_tables", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) var tables: [ValueBoxTable] = [] while preparedStatement.step() { @@ -244,13 +256,18 @@ final class SqliteValueBox: ValueBox { } else { switch table.keyType { case .binary: - self.database.execute("CREATE TABLE t\(table.id) (key BLOB, value BLOB)") - self.database.execute("CREATE INDEX t\(table.id)_key ON t\(table.id) (key)") + var resultCode: Bool + resultCode = self.database.execute("CREATE TABLE t\(table.id) (key BLOB, value BLOB)") + assert(resultCode) + resultCode = self.database.execute("CREATE INDEX t\(table.id)_key ON t\(table.id) (key)") + assert(resultCode) case .int64: - self.database.execute("CREATE TABLE t\(table.id) (key INTEGER PRIMARY KEY, value BLOB)") + let resultCode = self.database.execute("CREATE TABLE t\(table.id) (key INTEGER PRIMARY KEY, value BLOB)") + assert(resultCode) } self.tables[table.id] = table - self.database.execute("INSERT INTO __meta_tables(name, keyType) VALUES (\(table.id), \(table.keyType.rawValue))") + let resultCode = self.database.execute("INSERT INTO __meta_tables(name, keyType) VALUES (\(table.id), \(table.keyType.rawValue))") + assert(resultCode) } } @@ -264,7 +281,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT value FROM t\(table.id) WHERE key=?", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT value FROM t\(table.id) WHERE key=?", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.getStatements[table.id] = preparedStatement resultStatement = preparedStatement @@ -293,7 +311,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeKeyAscStatementsLimit[table.id] = preparedStatement resultStatement = preparedStatement @@ -325,7 +344,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeKeyAscStatementsNoLimit[table.id] = preparedStatement resultStatement = preparedStatement @@ -355,7 +375,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeKeyDescStatementsLimit[table.id] = preparedStatement resultStatement = preparedStatement @@ -386,7 +407,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT key FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeKeyDescStatementsNoLimit[table.id] = preparedStatement resultStatement = preparedStatement @@ -417,7 +439,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC LIMIT ?", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeValueAscStatementsLimit[table.id] = preparedStatement resultStatement = preparedStatement @@ -448,7 +471,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key ASC", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeValueAscStatementsNoLimit[table.id] = preparedStatement resultStatement = preparedStatement @@ -479,7 +503,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC LIMIT ?", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeValueDescStatementsLimit[table.id] = preparedStatement resultStatement = preparedStatement @@ -511,7 +536,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) WHERE key > ? AND key < ? ORDER BY key DESC", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.rangeValueDescStatementsNoLimit[table.id] = preparedStatement resultStatement = preparedStatement @@ -540,7 +566,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) ORDER BY key ASC", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT key, value FROM t\(table.id) ORDER BY key ASC", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.scanStatements[table.id] = preparedStatement resultStatement = preparedStatement @@ -561,7 +588,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "SELECT rowid FROM t\(table.id) WHERE key=?", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "SELECT rowid FROM t\(table.id) WHERE key=?", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.existsStatements[table.id] = preparedStatement resultStatement = preparedStatement @@ -589,7 +617,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "UPDATE t\(table.id) SET value=? WHERE key=?", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "UPDATE t\(table.id) SET value=? WHERE key=?", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.updateStatements[table.id] = preparedStatement resultStatement = preparedStatement @@ -618,7 +647,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table.id) (key, value) VALUES(?, ?)", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table.id) (key, value) VALUES(?, ?)", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.insertStatements[table.id] = preparedStatement resultStatement = preparedStatement @@ -651,7 +681,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "INSERT OR REPLACE INTO t\(table.id) (key, value) VALUES(?, ?)", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "INSERT OR REPLACE INTO t\(table.id) (key, value) VALUES(?, ?)", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.insertOrReplaceStatements[table.id] = preparedStatement resultStatement = preparedStatement @@ -684,7 +715,8 @@ final class SqliteValueBox: ValueBox { resultStatement = statement } else { var statement: OpaquePointer? = nil - sqlite3_prepare_v2(self.database.handle, "DELETE FROM t\(table.id) WHERE key=?", -1, &statement, nil) + let status = sqlite3_prepare_v2(self.database.handle, "DELETE FROM t\(table.id) WHERE key=?", -1, &statement, nil) + assert(status == SQLITE_OK) let preparedStatement = SqlitePreparedStatement(statement: statement) self.deleteStatements[table.id] = preparedStatement resultStatement = preparedStatement From 17935702ca05e94b21058c461d31fbe5bb1226a0 Mon Sep 17 00:00:00 2001 From: overtake Date: Thu, 27 Apr 2017 23:26:14 +0300 Subject: [PATCH 136/427] no message --- Postbox/SqliteValueBox.swift | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 3680431478..ff2b9161b2 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -39,10 +39,17 @@ private struct SqlitePreparedStatement { sqlite3_clear_bindings(statement) } - func step() -> Bool { + func step(_ initial:Bool = false, path:String? = nil) -> Bool { let result = sqlite3_step(statement) if result != SQLITE_ROW && result != SQLITE_DONE { - assertionFailure("Sqlite error \(result)") + if initial { + if let path = path { + try? FileManager.default.removeItem(atPath: path) + preconditionFailure() + } + } else { + assertionFailure("Sqlite error \(result)") + } } return result == SQLITE_ROW } @@ -229,7 +236,8 @@ final class SqliteValueBox: ValueBox { sqlite3_prepare_v2(database.handle, "SELECT name, keyType FROM __meta_tables", -1, &statement, nil) let preparedStatement = SqlitePreparedStatement(statement: statement) var tables: [ValueBoxTable] = [] - while preparedStatement.step() { + + while preparedStatement.step(true, path: basePath + "/db_sqlite") { let value = preparedStatement.int64At(0) let keyType = preparedStatement.int64At(1) tables.append(ValueBoxTable(id: Int32(value), keyType: ValueBoxKeyType(rawValue: Int32(keyType))!)) From 23bccb2d6df236da25e3fa66a4c72a3bea0cbddd Mon Sep 17 00:00:00 2001 From: overtake Date: Fri, 28 Apr 2017 11:31:00 +0300 Subject: [PATCH 137/427] - a lot of changes and features --- Postbox/MediaBox.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 13abd5cd0b..9ebe9d4782 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -751,7 +751,7 @@ public final class MediaBox { } } case let .moveLocalFile(tempPath): - if let thisFd = thisFd { + if let thisFd = fd { close(thisFd) fd = nil } From fd7f064e9f7fad2f0fb613d31ea20c9f7f24603f Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 2 May 2017 20:17:30 +0300 Subject: [PATCH 138/427] no message --- Postbox/MediaBox.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 13abd5cd0b..9ebe9d4782 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -751,7 +751,7 @@ public final class MediaBox { } } case let .moveLocalFile(tempPath): - if let thisFd = thisFd { + if let thisFd = fd { close(thisFd) fd = nil } From 33c847b71b4d8d26f270bf1afdd7b37bb88a2e05 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 9 May 2017 20:50:07 +0300 Subject: [PATCH 139/427] no message --- Postbox/IntermediateMessage.swift | 4 +- Postbox/Message.swift | 27 ++++++++++-- Postbox/MessageHistoryTable.swift | 71 +++++++++++++++++++++++++------ Postbox/MessageHistoryView.swift | 12 +++--- 4 files changed, 89 insertions(+), 25 deletions(-) diff --git a/Postbox/IntermediateMessage.swift b/Postbox/IntermediateMessage.swift index 21acc5d6a6..1d7416bb89 100644 --- a/Postbox/IntermediateMessage.swift +++ b/Postbox/IntermediateMessage.swift @@ -29,6 +29,7 @@ class IntermediateMessage { let timestamp: Int32 let flags: MessageFlags let tags: MessageTags + let globalTags: GlobalMessageTags let forwardInfo: IntermediateMessageForwardInfo? let authorId: PeerId? let text: String @@ -36,7 +37,7 @@ class IntermediateMessage { let embeddedMediaData: ReadBuffer let referencedMedia: [MediaId] - init(stableId: UInt32, stableVersion: UInt32, id: MessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { + init(stableId: UInt32, stableVersion: UInt32, id: MessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: MessageFlags, tags: MessageTags, globalTags: GlobalMessageTags, forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { self.stableId = stableId self.stableVersion = stableVersion self.id = id @@ -44,6 +45,7 @@ class IntermediateMessage { self.timestamp = timestamp self.flags = flags self.tags = tags + self.globalTags = globalTags self.forwardInfo = forwardInfo self.authorId = authorId self.text = text diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 308e123371..5d316bca8b 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -228,6 +228,18 @@ public struct MessageTags: OptionSet { public static let All = MessageTags(rawValue: 0xffffffff) } +public struct GlobalMessageTags: OptionSet { + public var rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public init() { + self.rawValue = 0 + } +} + public struct MessageFlags: OptionSet { public var rawValue: UInt32 @@ -343,6 +355,7 @@ public final class Message: CustomStringConvertible { public let timestamp: Int32 public let flags: MessageFlags public let tags: MessageTags + public let globalTags: GlobalMessageTags public let forwardInfo: MessageForwardInfo? public let author: Peer? public let text: String @@ -352,7 +365,7 @@ public final class Message: CustomStringConvertible { public let associatedMessages: SimpleDictionary public let associatedMessageIds: [MessageId] - public init(stableId: UInt32, stableVersion: UInt32, id: MessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: MessageFlags, tags: MessageTags, forwardInfo: MessageForwardInfo?, author: Peer?, text: String, attributes: [MessageAttribute], media: [Media], peers: SimpleDictionary, associatedMessages: SimpleDictionary, associatedMessageIds: [MessageId]) { + public init(stableId: UInt32, stableVersion: UInt32, id: MessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: MessageFlags, tags: MessageTags, globalTags: GlobalMessageTags, forwardInfo: MessageForwardInfo?, author: Peer?, text: String, attributes: [MessageAttribute], media: [Media], peers: SimpleDictionary, associatedMessages: SimpleDictionary, associatedMessageIds: [MessageId]) { self.stableId = stableId self.stableVersion = stableVersion self.id = id @@ -360,6 +373,7 @@ public final class Message: CustomStringConvertible { self.timestamp = timestamp self.flags = flags self.tags = tags + self.globalTags = globalTags self.forwardInfo = forwardInfo self.author = author self.text = text @@ -444,18 +458,20 @@ public final class StoreMessage { public let globallyUniqueId: Int64? public let flags: StoreMessageFlags public let tags: MessageTags + public let globalTags: GlobalMessageTags public let forwardInfo: StoreMessageForwardInfo? public let authorId: PeerId? public let text: String public let attributes: [MessageAttribute] public let media: [Media] - public init(id: MessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + public init(id: MessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, globalTags: GlobalMessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { self.id = .Id(id) self.globallyUniqueId = globallyUniqueId self.timestamp = timestamp self.flags = flags self.tags = tags + self.globalTags = globalTags self.forwardInfo = forwardInfo self.authorId = authorId self.text = text @@ -463,12 +479,13 @@ public final class StoreMessage { self.media = media } - public init(peerId: PeerId, namespace: MessageId.Namespace, globallyUniqueId: Int64?, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + public init(peerId: PeerId, namespace: MessageId.Namespace, globallyUniqueId: Int64?, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, globalTags: GlobalMessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { self.id = .Partial(peerId, namespace) self.timestamp = timestamp self.globallyUniqueId = globallyUniqueId self.flags = flags self.tags = tags + self.globalTags = globalTags self.forwardInfo = forwardInfo self.authorId = authorId self.text = text @@ -483,18 +500,20 @@ final class InternalStoreMessage { let globallyUniqueId: Int64? let flags: StoreMessageFlags let tags: MessageTags + let globalTags: GlobalMessageTags let forwardInfo: StoreMessageForwardInfo? let authorId: PeerId? let text: String let attributes: [MessageAttribute] let media: [Media] - init(id: MessageId, timestamp: Int32, globallyUniqueId: Int64?, flags: StoreMessageFlags, tags: MessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + init(id: MessageId, timestamp: Int32, globallyUniqueId: Int64?, flags: StoreMessageFlags, tags: MessageTags, globalTags: GlobalMessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { self.id = id self.timestamp = timestamp self.globallyUniqueId = globallyUniqueId self.flags = flags self.tags = tags + self.globalTags = globalTags self.forwardInfo = forwardInfo self.authorId = authorId self.text = text diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 2fbef602be..a2a126a451 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -33,6 +33,21 @@ enum RenderedMessageHistoryEntry { } } +private struct MessageDataFlags: OptionSet { + var rawValue: Int8 + + init() { + self.rawValue = 0 + } + + init(rawValue: Int8) { + self.rawValue = rawValue + } + + static let hasGloballyUniqueId = MessageDataFlags(rawValue: 1 << 0) + static let hasGlobalTags = MessageDataFlags(rawValue: 1 << 1) +} + final class MessageHistoryTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .binary) @@ -277,10 +292,10 @@ final class MessageHistoryTable: Table { for message in messages { switch message.id { case let .Id(id): - internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, globallyUniqueId: message.globallyUniqueId, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) + internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, globallyUniqueId: message.globallyUniqueId, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) case let .Partial(peerId, namespace): let id = self.historyMetadataTable.getNextMessageIdAndIncrement(peerId, namespace: namespace) - internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, globallyUniqueId: message.globallyUniqueId, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) + internalStoreMessages.append(InternalStoreMessage(id: id, timestamp: message.timestamp, globallyUniqueId: message.globallyUniqueId, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributes: message.attributes, media: message.media)) } } return internalStoreMessages @@ -564,13 +579,23 @@ final class MessageHistoryTable: Table { var stableVersion: UInt32 = 0 sharedBuffer.write(&stableVersion, offset: 0, length: 4) - var hasGloballyUniqueId: Int8 = message.globallyUniqueId != nil ? 1 : 0 - sharedBuffer.write(&hasGloballyUniqueId, offset: 0, length: 1) + var dataFlags: MessageDataFlags = [] + if message.globallyUniqueId != nil { + dataFlags.insert(.hasGloballyUniqueId) + } + if !message.globalTags.isEmpty { + dataFlags.insert(.hasGlobalTags) + } + sharedBuffer.write(&dataFlags, offset: 0, length: 1) if let globallyUniqueId = message.globallyUniqueId { var globallyUniqueIdValue = globallyUniqueId sharedBuffer.write(&globallyUniqueIdValue, offset: 0, length: 8) self.globallyUniqueMessageIdsTable.set(peerId: message.id.peerId, globallyUniqueId: globallyUniqueId, id: message.id) } + if !message.globalTags.isEmpty { + var globalTagsValue: UInt32 = message.globalTags.rawValue + sharedBuffer.write(&globalTagsValue, offset: 0, length: 4) + } var flags = MessageFlags(message.flags) sharedBuffer.write(&flags.rawValue, offset: 0, length: 4) @@ -688,7 +713,7 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.table, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) - return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: flags, tags: message.tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) + return IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia) } private func justInsertHole(_ hole: MessageHistoryHole, sharedBuffer: WriteBuffer = WriteBuffer()) { @@ -862,7 +887,7 @@ final class MessageHistoryTable: Table { } withExtendedLifetime(updatedEmbeddedMediaBuffer, { - self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: message.referencedMedia), sharedKey: self.key(index)) + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: message.referencedMedia), sharedKey: self.key(index)) }) let operation: MessageHistoryOperation = .UpdateEmbeddedMedia(index, updatedEmbeddedMediaBuffer.makeReadBufferAndReset()) @@ -953,6 +978,15 @@ final class MessageHistoryTable: Table { self.globallyUniqueMessageIdsTable.set(peerId: message.id.peerId, globallyUniqueId: globallyUniqueId, id: message.id) } + if !message.globalTags.isEmpty || !previousMessage.globalTags.isEmpty { + assertionFailure("implement global tags") + if index != MessageIndex(message) { + + } else if message.globalTags != previousMessage.globalTags { + + } + } + switch (previousMessage.flags.contains(.Unsent) && !previousMessage.flags.contains(.Failed), message.flags.contains(.Unsent) && !message.flags.contains(.Failed)) { case (true, false): self.unsentTable.remove(index.id, operations: &unsentMessageOperations) @@ -1098,7 +1132,7 @@ final class MessageHistoryTable: Table { self.valueBox.set(self.table, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) - return (IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: flags, tags: tags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia), previousMessage.tags) + return (IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: flags, tags: tags, globalTags: message.globalTags, forwardInfo: intermediateForwardInfo, authorId: message.authorId, text: message.text, attributesData: attributesBuffer.makeReadBufferAndReset(), embeddedMediaData: embeddedMediaBuffer.makeReadBufferAndReset(), referencedMedia: referencedMedia), previousMessage.tags) } else { return nil } @@ -1107,7 +1141,7 @@ final class MessageHistoryTable: Table { private func justUpdateTimestamp(_ index: MessageIndex, timestamp: Int32) { if let previousMessage = self.getMessage(index) { self.valueBox.remove(self.table, key: self.key(index)) - let updatedMessage = IntermediateMessage(stableId: previousMessage.stableId, stableVersion: previousMessage.stableVersion + 1, id: previousMessage.id, globallyUniqueId: previousMessage.globallyUniqueId, timestamp: timestamp, flags: previousMessage.flags, tags: previousMessage.tags, forwardInfo: previousMessage.forwardInfo, authorId: previousMessage.authorId, text: previousMessage.text, attributesData: previousMessage.attributesData, embeddedMediaData: previousMessage.embeddedMediaData, referencedMedia: previousMessage.referencedMedia) + let updatedMessage = IntermediateMessage(stableId: previousMessage.stableId, stableVersion: previousMessage.stableVersion + 1, id: previousMessage.id, globallyUniqueId: previousMessage.globallyUniqueId, timestamp: timestamp, flags: previousMessage.flags, tags: previousMessage.tags, globalTags: previousMessage.globalTags, forwardInfo: previousMessage.forwardInfo, authorId: previousMessage.authorId, text: previousMessage.text, attributesData: previousMessage.attributesData, embeddedMediaData: previousMessage.embeddedMediaData, referencedMedia: previousMessage.referencedMedia) self.storeIntermediateMessage(updatedMessage, sharedKey: self.key(index)) let tags = previousMessage.tags.rawValue @@ -1161,7 +1195,7 @@ final class MessageHistoryTable: Table { var updatedReferencedMedia = message.referencedMedia updatedReferencedMedia.append(extractedMedia.id!) withExtendedLifetime(updatedEmbeddedMediaBuffer, { - self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) + self.storeIntermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) }) return extractedMedia @@ -1313,15 +1347,24 @@ final class MessageHistoryTable: Table { var stableVersion: UInt32 = 0 value.read(&stableVersion, offset: 0, length: 4) - var hasGloballyUniqueId: Int8 = 0 + var dataFlagsValue: Int8 = 0 + value.read(&dataFlagsValue, offset: 0, length: 1) + let dataFlags = MessageDataFlags(rawValue: dataFlagsValue) + var globallyUniqueId: Int64? - value.read(&hasGloballyUniqueId, offset: 0, length: 1) - if hasGloballyUniqueId != 0 { + if dataFlags.contains(.hasGloballyUniqueId) { var globallyUniqueIdValue: Int64 = 0 value.read(&globallyUniqueIdValue, offset: 0, length: 8) globallyUniqueId = globallyUniqueIdValue } + var globalTags: GlobalMessageTags = [] + if dataFlags.contains(.hasGlobalTags) { + var globalTagsValue: UInt32 = 0 + value.read(&globalTagsValue, offset: 0, length: 4) + globalTags = GlobalMessageTags(rawValue: globalTagsValue) + } + var flagsValue: UInt32 = 0 value.read(&flagsValue, offset: 0, length: 4) let flags = MessageFlags(rawValue: flagsValue) @@ -1413,7 +1456,7 @@ final class MessageHistoryTable: Table { referencedMediaIds.append(MediaId(namespace: idNamespace, id: idId)) } - return .Message(IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: index.id, globallyUniqueId: globallyUniqueId, timestamp: index.timestamp, flags: flags, tags: tags, forwardInfo: forwardInfo, authorId: authorId, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)) + return .Message(IntermediateMessage(stableId: stableId, stableVersion: stableVersion, id: index.id, globallyUniqueId: globallyUniqueId, timestamp: index.timestamp, flags: flags, tags: tags, globalTags: globalTags, forwardInfo: forwardInfo, authorId: authorId, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)) } else { var stableId: UInt32 = 0 value.read(&stableId, offset: 0, length: 4) @@ -1523,7 +1566,7 @@ final class MessageHistoryTable: Table { } } - return Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages, associatedMessageIds: associatedMessageIds) + return Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages, associatedMessageIds: associatedMessageIds) } func entriesAround(_ index: MessageIndex, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index 523f1053a5..a3d7563366 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -159,10 +159,10 @@ enum MutableMessageHistoryEntry { func updatedTimestamp(_ timestamp: Int32) -> MutableMessageHistoryEntry { switch self { case let .IntermediateMessageEntry(message, location, monthLocation): - let updatedMessage = IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) + let updatedMessage = IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia) return .IntermediateMessageEntry(updatedMessage, location, monthLocation) case let .MessageEntry(message, location, monthLocation): - let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) return .MessageEntry(updatedMessage, location, monthLocation) case let .HoleEntry(hole, location): let updatedHole = MessageHistoryHole(stableId: hole.stableId, maxIndex: MessageIndex(id: hole.maxIndex.id, timestamp: timestamp), min: hole.min, tags: hole.tags) @@ -550,7 +550,7 @@ final class MutableMessageHistoryView { case let .UpdateEmbeddedMedia(index, embeddedMediaData): for i in 0 ..< self.entries.count { if case let .IntermediateMessageEntry(message, location, monthLocation) = self.entries[i] , MessageIndex(message) == index { - self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), location, monthLocation) + self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), location, monthLocation) hasChanges = true break } @@ -591,7 +591,7 @@ final class MutableMessageHistoryView { messageMedia.append(media) } } - let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds) self.entries[i] = .MessageEntry(updatedMessage, location, monthLocation) hasChanges = true } @@ -633,7 +633,7 @@ final class MutableMessageHistoryView { var updatedAssociatedMessages = message.associatedMessages let renderedMessage = renderIntermediateMessage(intermediateMessage) updatedAssociatedMessages[intermediateMessage.id] = renderedMessage - let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId:message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId:message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) self.entries[i] = .MessageEntry(updatedMessage, location, monthLocation) hasChanges = true } @@ -789,7 +789,7 @@ final class MutableMessageHistoryView { switch entry { case let .MessageEntry(message, location, monthLocation): if let updatedAssociatedMessages = message.associatedMessages.filteredOut(keysIn: ids) { - let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) + let updatedMessage = Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: updatedAssociatedMessages, associatedMessageIds: message.associatedMessageIds) self.entries[i] = .MessageEntry(updatedMessage, location, monthLocation) } default: From b61e408cd853af4802927dd89245d843f423c9a3 Mon Sep 17 00:00:00 2001 From: overtake Date: Thu, 11 May 2017 20:27:07 +0300 Subject: [PATCH 140/427] secret chat bug fixes --- Postbox/Coding.swift | 12 +----------- Postbox/PeerMergedOperationLogView.swift | 10 ++++++++++ Postbox/PeerOperationLogTable.swift | 21 ++++++++++++--------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index a2a4f9779a..611f6ed211 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -708,17 +708,7 @@ public final class Decoder { return 0 } } - - public func decodeInt32ForKey(_ key: String) -> Int32 { - if Decoder.positionOnStringKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { - var value: Int32 = 0 - memcpy(&value, self.buffer.memory + self.offset, 4) - self.offset += 4 - return value - } else { - return 0 - } - } + public func decodeInt32ForKey(_ key: StaticString) -> Int32? { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { diff --git a/Postbox/PeerMergedOperationLogView.swift b/Postbox/PeerMergedOperationLogView.swift index c403ced292..7993c10b8d 100644 --- a/Postbox/PeerMergedOperationLogView.swift +++ b/Postbox/PeerMergedOperationLogView.swift @@ -37,6 +37,16 @@ final class MutablePeerMergedOperationLogView { self.tailIndex = entry.mergedIndex } } + case let .updateContents(entry): + if entry.tag == self.tag { + loop: for i in 0 ..< self.entries.count { + if self.entries[i].tagLocalIndex == entry.tagLocalIndex { + self.entries[i] = entry + updated = true + break loop + } + } + } case let .remove(tag, mergedIndices): if tag == self.tag { updated = true diff --git a/Postbox/PeerOperationLogTable.swift b/Postbox/PeerOperationLogTable.swift index 71f8413f7a..e88a27dedd 100644 --- a/Postbox/PeerOperationLogTable.swift +++ b/Postbox/PeerOperationLogTable.swift @@ -3,6 +3,7 @@ import Foundation enum PeerMergedOperationLogOperation { case append(PeerMergedOperationLogEntry) case remove(tag: PeerOperationLogTag, mergedIndices: Set) + case updateContents(PeerMergedOperationLogEntry) } public struct PeerMergedOperationLogEntry { @@ -296,9 +297,18 @@ final class PeerOperationLogTable: Table { assert(value.length - value.offset == Int(contentLength)) if let contents = Decoder(buffer: MemoryBuffer(memory: value.memory.advanced(by: value.offset), capacity: Int(contentLength), length: Int(contentLength), freeWhenDone: false)).decodeRootObject() { let entryUpdate = f(PeerOperationLogEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, mergedIndex: mergedIndex, contents: contents)) - switch entryUpdate.mergedIndex { + var updatedContents: Coding? + switch entryUpdate.contents { case .none: break + case let .update(contents): + updatedContents = contents + } + switch entryUpdate.mergedIndex { + case .none: + if let previousMergedIndex = previousMergedIndex, let updatedContents = updatedContents { + operations.append(.updateContents(PeerMergedOperationLogEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, mergedIndex: previousMergedIndex, contents: updatedContents))) + } case .remove: if let mergedIndexValue = mergedIndex { mergedIndex = nil @@ -312,14 +322,7 @@ final class PeerOperationLogTable: Table { } let updatedMergedIndexValue = self.mergedIndexTable.add(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex) mergedIndex = updatedMergedIndexValue - operations.append(.append(PeerMergedOperationLogEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, mergedIndex: updatedMergedIndexValue, contents: contents))) - } - var updatedContents: Coding? - switch entryUpdate.contents { - case .none: - break - case let .update(contents): - updatedContents = contents + operations.append(.append(PeerMergedOperationLogEntry(peerId: peerId, tag: tag, tagLocalIndex: tagLocalIndex, mergedIndex: updatedMergedIndexValue, contents: updatedContents ?? contents))) } if previousMergedIndex != mergedIndex || updatedContents != nil { let buffer = WriteBuffer() From 0677381484532e8a6c3c84e6f737724781294ec7 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 26 May 2017 13:09:35 +0300 Subject: [PATCH 141/427] no message --- Postbox.xcodeproj/project.pbxproj | 18 + Postbox/AccountRecord.swift | 2 +- Postbox/Coding.swift | 63 ++- Postbox/GlobalMessageHistoryTagsTable.swift | 209 +++++++++ Postbox/GlobalMessageTagsView.swift | 469 ++++++++++++++++++++ Postbox/ItemCacheMetaTable.swift | 2 +- Postbox/ManagedFile.swift | 73 +++ Postbox/Media.swift | 4 +- Postbox/MediaBox.swift | 77 ++-- Postbox/Message.swift | 59 ++- Postbox/MessageHistoryTable.swift | 312 ++++++++++--- Postbox/MetadataTable.swift | 10 +- Postbox/Postbox.swift | 72 ++- Postbox/PostboxTransaction.swift | 7 +- Postbox/SeedConfiguration.swift | 4 +- Postbox/SqliteValueBox.swift | 5 +- Postbox/Views.swift | 11 + 17 files changed, 1248 insertions(+), 149 deletions(-) create mode 100644 Postbox/GlobalMessageHistoryTagsTable.swift create mode 100644 Postbox/GlobalMessageTagsView.swift create mode 100644 Postbox/ManagedFile.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index edbcdb9b82..5b633cafd8 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -24,6 +24,8 @@ D00EED1E1C81F28D00341DFF /* MessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */; }; D010B61A1E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */; }; D019B1CF1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */; }; + D01BAA551ED1D70C00295217 /* ManagedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01BAA541ED1D70C00295217 /* ManagedFile.swift */; }; + D01BAA561ED1D70C00295217 /* ManagedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01BAA541ED1D70C00295217 /* ManagedFile.swift */; }; D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */; }; @@ -189,6 +191,10 @@ D0DA44481E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */; }; D0DA44491E4C7D1E005FDCA7 /* PostboxAccess.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */; }; D0DF0C8F1D81A350008AEB01 /* PeerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0DF0C8E1D81A350008AEB01 /* PeerView.swift */; }; + D0E1D30C1ECA1F5500FCEEF1 /* GlobalMessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1D30B1ECA1F5500FCEEF1 /* GlobalMessageHistoryTagsTable.swift */; }; + D0E1D30D1ECA1F5500FCEEF1 /* GlobalMessageHistoryTagsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1D30B1ECA1F5500FCEEF1 /* GlobalMessageHistoryTagsTable.swift */; }; + D0E1D30F1ECA53F900FCEEF1 /* GlobalMessageTagsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1D30E1ECA53F900FCEEF1 /* GlobalMessageTagsView.swift */; }; + D0E1D3101ECA53F900FCEEF1 /* GlobalMessageTagsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1D30E1ECA53F900FCEEF1 /* GlobalMessageTagsView.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; D0E23DE21E808A9400B9B6D2 /* ItemCollectionIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E23DE11E808A9400B9B6D2 /* ItemCollectionIdsView.swift */; }; D0E23DE31E808A9400B9B6D2 /* ItemCollectionIdsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E23DE11E808A9400B9B6D2 /* ItemCollectionIdsView.swift */; }; @@ -290,6 +296,7 @@ D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTagsTable.swift; sourceTree = ""; }; D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerMergedOperationLogIndexTable.swift; sourceTree = ""; }; D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageGloballyUniqueIdTable.swift; sourceTree = ""; }; + D01BAA541ED1D70C00295217 /* ManagedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ManagedFile.swift; sourceTree = ""; }; D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollection.swift; sourceTree = ""; }; @@ -385,6 +392,8 @@ D0D949F41D35353900740E02 /* MappedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MappedFile.swift; sourceTree = ""; }; D0DA44471E4C7D1E005FDCA7 /* PostboxAccess.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxAccess.swift; sourceTree = ""; }; D0DF0C8E1D81A350008AEB01 /* PeerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerView.swift; sourceTree = ""; }; + D0E1D30B1ECA1F5500FCEEF1 /* GlobalMessageHistoryTagsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GlobalMessageHistoryTagsTable.swift; sourceTree = ""; }; + D0E1D30E1ECA53F900FCEEF1 /* GlobalMessageTagsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GlobalMessageTagsView.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; D0E23DE11E808A9400B9B6D2 /* ItemCollectionIdsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollectionIdsView.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -542,6 +551,7 @@ D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */, D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */, + D0E1D30B1ECA1F5500FCEEF1 /* GlobalMessageHistoryTagsTable.swift */, D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */, D0F9E8641C58CB7F00037222 /* ChatListTable.swift */, D0F9E8661C58D08900037222 /* ChatListIndexTable.swift */, @@ -614,6 +624,7 @@ D0D510FF1D64A58900A97B8A /* IpcPipe.swift */, D07827C01E0079CB00071108 /* StringIndexTokens.swift */, D03229ED1E6B33FD0000AF9C /* SqliteInterface.swift */, + D01BAA541ED1D70C00295217 /* ManagedFile.swift */, ); name = Utils; sourceTree = ""; @@ -658,6 +669,7 @@ D0FA0ACC1E781067005BB9B7 /* Views.swift */, D0F53BF21E794C6700117362 /* PeerChatStateView.swift */, D0575AE21E9ECBB2006F2541 /* AccessChallengeDataView.swift */, + D0E1D30E1ECA53F900FCEEF1 /* GlobalMessageTagsView.swift */, ); name = Views; sourceTree = ""; @@ -954,6 +966,7 @@ D0F82CFE1E4345D7007E499C /* OrderedItemListTable.swift in Sources */, D0F7B1C51E045C6A007EB8A5 /* MessageHistoryMetadataTable.swift in Sources */, D073CE781DCBF3B4007511FD /* MessageHistoryHole.swift in Sources */, + D0E1D3101ECA53F900FCEEF1 /* GlobalMessageTagsView.swift in Sources */, D08775071E3E3F2100A97350 /* PreferencesView.swift in Sources */, D049EAF11E44D9B900A2CD3A /* PostboxStateView.swift in Sources */, D0AAD1B61E32673C00D5B9DE /* TimestampBasedMessageAttributesView.swift in Sources */, @@ -979,6 +992,7 @@ D0FA0AC81E77F0A2005BB9B7 /* ItemCollectionInfosView.swift in Sources */, D0B4185A1D7DFE29004562A4 /* ViewTracker.swift in Sources */, D0F7B1E21E045C6A007EB8A5 /* PeerNameIndexTable.swift in Sources */, + D0E1D30D1ECA1F5500FCEEF1 /* GlobalMessageHistoryTagsTable.swift in Sources */, D073CE7C1DCBF3B4007511FD /* CachedPeerData.swift in Sources */, D0F7B1DA1E045C6A007EB8A5 /* RatingTable.swift in Sources */, D0BEAF711E54BC1E00BD963D /* AccountRecordsView.swift in Sources */, @@ -1001,6 +1015,7 @@ D0F7B1CB1E045C6A007EB8A5 /* ChatListTable.swift in Sources */, D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */, D0F7B1CA1E045C6A007EB8A5 /* MessageHistoryUnsentTable.swift in Sources */, + D01BAA561ED1D70C00295217 /* ManagedFile.swift in Sources */, D0BEAF6B1E54B5FB00BD963D /* AccountManagerMetadataTable.swift in Sources */, D0F02CE01E99223E0065DEE2 /* Upgrades.swift in Sources */, D0F7B1D01E045C6A007EB8A5 /* KeychainTable.swift in Sources */, @@ -1078,6 +1093,7 @@ D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, D0C9DA391C65782500855278 /* SimpleSet.swift in Sources */, D098C6F11D7E11E9007784E4 /* Database.swift in Sources */, + D0E1D30F1ECA53F900FCEEF1 /* GlobalMessageTagsView.swift in Sources */, D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */, D07827C31E008F7300071108 /* ReverseIndexReferenceTable.swift in Sources */, @@ -1103,6 +1119,7 @@ D0FA0AC71E77F0A2005BB9B7 /* ItemCollectionInfosView.swift in Sources */, D0F019FD1E1DA0CC00F05AB3 /* PeerMergedOperationLogView.swift in Sources */, D0F9E8651C58CB7F00037222 /* ChatListTable.swift in Sources */, + D0E1D30C1ECA1F5500FCEEF1 /* GlobalMessageHistoryTagsTable.swift in Sources */, D0F9E8711C5A0E9B00037222 /* PeerTable.swift in Sources */, D0BC386C1E3FCEE50044D6FE /* MultiplePeersView.swift in Sources */, D0BEAF701E54BC1E00BD963D /* AccountRecordsView.swift in Sources */, @@ -1125,6 +1142,7 @@ D0BC38721E409E670044D6FE /* RenderedPeer.swift in Sources */, D08775031E3E3E7400A97350 /* PreferencesEntry.swift in Sources */, D0AB0B901D65D4AB002C78E7 /* UnsentMessageIndicesView.swift in Sources */, + D01BAA551ED1D70C00295217 /* ManagedFile.swift in Sources */, D0BEAF6A1E54B5FB00BD963D /* AccountManagerMetadataTable.swift in Sources */, D0F02CDF1E99223D0065DEE2 /* Upgrades.swift in Sources */, D08775001E3E3D9F00A97350 /* PreferencesTable.swift in Sources */, diff --git a/Postbox/AccountRecord.swift b/Postbox/AccountRecord.swift index 56c03c4228..ea096d1747 100644 --- a/Postbox/AccountRecord.swift +++ b/Postbox/AccountRecord.swift @@ -44,7 +44,7 @@ public struct AccountRecord: Coding, Equatable { } public init(decoder: Decoder) { - self.id = AccountRecordId(rawValue: decoder.decodeInt64ForKey("id")) + self.id = AccountRecordId(rawValue: decoder.decodeInt64ForKey("id", orElse: 0)) self.attributes = (decoder.decodeObjectArrayForKey("attributes") as [Coding]).map { $0 as! AccountRecordAttribute } } diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index a2a4f9779a..99aea86f36 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -698,29 +698,29 @@ public final class Decoder { return false } - public func decodeInt32ForKey(_ key: StaticString) -> Int32 { + public func decodeInt32ForKey(_ key: StaticString, orElse: Int32) -> Int32 { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 memcpy(&value, self.buffer.memory + self.offset, 4) self.offset += 4 return value } else { - return 0 + return orElse } } - public func decodeInt32ForKey(_ key: String) -> Int32 { + public func decodeInt32ForKey(_ key: String, orElse: Int32) -> Int32 { if Decoder.positionOnStringKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 memcpy(&value, self.buffer.memory + self.offset, 4) self.offset += 4 return value } else { - return 0 + return orElse } } - public func decodeInt32ForKey(_ key: StaticString) -> Int32? { + public func decodeOptionalInt32ForKey(_ key: StaticString) -> Int32? { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { var value: Int32 = 0 memcpy(&value, self.buffer.memory + self.offset, 4) @@ -731,18 +731,29 @@ public final class Decoder { } } - public func decodeInt64ForKey(_ key: StaticString) -> Int64 { + public func decodeOptionalInt32ForKey(_ key: String) -> Int32? { + if Decoder.positionOnStringKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int32) { + var value: Int32 = 0 + memcpy(&value, self.buffer.memory + self.offset, 4) + self.offset += 4 + return value + } else { + return nil + } + } + + public func decodeInt64ForKey(_ key: StaticString, orElse: Int64) -> Int64 { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { var value: Int64 = 0 memcpy(&value, self.buffer.memory + self.offset, 8) self.offset += 8 return value } else { - return 0 + return orElse } } - public func decodeInt64ForKey(_ key: StaticString) -> Int64? { + public func decodeOptionalInt64ForKey(_ key: StaticString) -> Int64? { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Int64) { var value: Int64 = 0 memcpy(&value, self.buffer.memory + self.offset, 8) @@ -753,18 +764,40 @@ public final class Decoder { } } - public func decodeBoolForKey(_ key: StaticString) -> Bool { + public func decodeBoolForKey(_ key: StaticString, orElse: Bool) -> Bool { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bool) { var value: Int8 = 0 memcpy(&value, self.buffer.memory + self.offset, 1) self.offset += 1 return value != 0 } else { - return false + return orElse } } - public func decodeDoubleForKey(_ key: StaticString) -> Double { + public func decodeOptionalBoolForKey(_ key: StaticString) -> Bool? { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Bool) { + var value: Int8 = 0 + memcpy(&value, self.buffer.memory + self.offset, 1) + self.offset += 1 + return value != 0 + } else { + return nil + } + } + + public func decodeDoubleForKey(_ key: StaticString, orElse: Double) -> Double { + if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Double) { + var value: Double = 0 + memcpy(&value, self.buffer.memory + self.offset, 8) + self.offset += 8 + return value + } else { + return orElse + } + } + + public func decodeOptionalDoubleForKey(_ key: StaticString) -> Double? { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .Double) { var value: Double = 0 memcpy(&value, self.buffer.memory + self.offset, 8) @@ -775,19 +808,19 @@ public final class Decoder { } } - public func decodeStringForKey(_ key: StaticString) -> String { + public func decodeStringForKey(_ key: StaticString, orElse: String) -> String { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) let data = Data(bytes: self.buffer.memory.assumingMemoryBound(to: UInt8.self).advanced(by: self.offset + 4), count: Int(length)) self.offset += 4 + Int(length) - return String(data: data, encoding: .utf8) ?? "" + return String(data: data, encoding: .utf8) ?? orElse } else { - return "" + return orElse } } - public func decodeStringForKey(_ key: StaticString) -> String? { + public func decodeOptionalStringForKey(_ key: StaticString) -> String? { if Decoder.positionOnKey(self.buffer.memory, offset: &self.offset, maxOffset: self.buffer.length, length: self.buffer.length, key: key, valueType: .String) { var length: Int32 = 0 memcpy(&length, self.buffer.memory + self.offset, 4) diff --git a/Postbox/GlobalMessageHistoryTagsTable.swift b/Postbox/GlobalMessageHistoryTagsTable.swift new file mode 100644 index 0000000000..410d926d10 --- /dev/null +++ b/Postbox/GlobalMessageHistoryTagsTable.swift @@ -0,0 +1,209 @@ +import Foundation + +enum IntermediateGlobalMessageTagsEntry { + case message(IntermediateMessage) + case hole(MessageIndex) + + var index: MessageIndex { + switch self { + case let .message(message): + return MessageIndex(message) + case let .hole(index): + return index + } + } +} + +enum GlobalMessageHistoryTagsTableEntry { + case message(MessageIndex) + case hole(MessageIndex) + + var index: MessageIndex { + switch self { + case let .message(index): + return index + case let .hole(index): + return index + } + } +} + +enum GlobalMessageHistoryTagsOperation { + case insertMessage(GlobalMessageTags, IntermediateMessage) + case insertHole(GlobalMessageTags, MessageIndex) + case remove([(GlobalMessageTags, MessageIndex)]) + case updateTimestamp(GlobalMessageTags, MessageIndex, Int32) +} + +private func parseEntry(key: ValueBoxKey, value: ReadBuffer) -> GlobalMessageHistoryTagsTableEntry { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4 + 4)), namespace: key.getInt32(4 + 4), id: key.getInt32(4 + 4 + 4)), timestamp: key.getInt32(4)) + if type == 0 { + return .message(index) + } else if type == 1 { + return .hole(index) + } else { + preconditionFailure() + } +} + +class GlobalMessageHistoryTagsTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .binary) + } + + private let sharedKey = ValueBoxKey(length: 4 + 4 + 4 + 4 + 8) + + private var cachedInitializedTags = Set() + + private func key(_ tagMask: GlobalMessageTags, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 4 + 4 + 4 + 4 + 8)) -> ValueBoxKey { + key.setUInt32(0, value: tagMask.rawValue) + key.setInt32(4, value: index.timestamp) + key.setInt32(4 + 4, value: index.id.namespace) + key.setInt32(4 + 4 + 4, value: index.id.id) + key.setInt64(4 + 4 + 4 + 4, value: index.id.peerId.toInt64()) + return key + } + + private func lowerBound(_ tagMask: GlobalMessageTags) -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setUInt32(0, value: tagMask.rawValue) + return key + } + + private func upperBound(_ tagMask: GlobalMessageTags) -> ValueBoxKey { + let key = ValueBoxKey(length: 4) + key.setUInt32(0, value: tagMask.rawValue) + return key.successor + } + + func ensureInitialized(_ tagMask: GlobalMessageTags) { + for tag in tagMask { + if !self.cachedInitializedTags.contains(tag) { + var isEmpty = true + self.valueBox.range(self.table, start: self.lowerBound(tag), end: self.upperBound(tag), keys: { _ in + isEmpty = false + return false + }, limit: 1) + if isEmpty { + self.addHole(tag, index: MessageIndex.absoluteUpperBound()) + } + self.cachedInitializedTags.insert(tag) + } + } + } + + func addMessage(_ tagMask: GlobalMessageTags, index: MessageIndex) -> Bool { + assert(tagMask.isSingleTag) + var upperIsHole = false + self.valueBox.range(self.table, start: self.key(tagMask, index: index, key: self.sharedKey), end: self.upperBound(tagMask), values: { key, value in + let entry = parseEntry(key: key, value: value) + if case .hole = entry { + upperIsHole = true + } + return false + }, limit: 1) + if !upperIsHole { + var type: Int8 = 0 + self.valueBox.set(self.table, key: self.key(tagMask, index: index, key: self.sharedKey), value: MemoryBuffer(memory: &type, capacity: 1, length: 1, freeWhenDone: false)) + return true + } else { + return false + } + } + + func addHole(_ tagMask: GlobalMessageTags, index: MessageIndex) { + assert(tagMask.isSingleTag) + var type: Int8 = 1 + self.valueBox.set(self.table, key: self.key(tagMask, index: index, key: self.sharedKey), value: MemoryBuffer(memory: &type, capacity: 1, length: 1, freeWhenDone: false)) + } + + func remove(_ tagMask: GlobalMessageTags, index: MessageIndex) { + assert(tagMask.isSingleTag) + self.valueBox.remove(self.table, key: self.key(tagMask, index: index, key: self.sharedKey)) + } + + func get(_ tagMask: GlobalMessageTags, index: MessageIndex) -> GlobalMessageHistoryTagsTableEntry? { + let key = self.key(tagMask, index: index) + if let value = self.valueBox.get(self.table, key: key) { + return parseEntry(key: key, value: value) + } else { + return nil + } + } + + func entriesAround(_ tagMask: GlobalMessageTags, index: MessageIndex, count: Int) -> (entries: [GlobalMessageHistoryTagsTableEntry], lower: GlobalMessageHistoryTagsTableEntry?, upper: GlobalMessageHistoryTagsTableEntry?) { + var lowerEntries: [GlobalMessageHistoryTagsTableEntry] = [] + var upperEntries: [GlobalMessageHistoryTagsTableEntry] = [] + var lower: GlobalMessageHistoryTagsTableEntry? + var upper: GlobalMessageHistoryTagsTableEntry? + + self.valueBox.range(self.table, start: self.key(tagMask, index: index), end: self.lowerBound(tagMask), values: { key, value in + lowerEntries.append(parseEntry(key: key, value: value)) + return true + }, limit: count / 2 + 1) + + if lowerEntries.count >= count / 2 + 1 { + lower = lowerEntries.last + lowerEntries.removeLast() + } + + self.valueBox.range(self.table, start: self.key(tagMask, index: index).predecessor, end: self.upperBound(tagMask), values: { key, value in + upperEntries.append(parseEntry(key: key, value: value)) + return true + }, limit: count - lowerEntries.count + 1) + if upperEntries.count >= count - lowerEntries.count + 1 { + upper = upperEntries.last + upperEntries.removeLast() + } + + if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count { + var additionalLowerEntries: [GlobalMessageHistoryTagsTableEntry] = [] + self.valueBox.range(self.table, start: self.key(tagMask, index: lowerEntries.last!.index), end: self.lowerBound(tagMask), values: { key, value in + additionalLowerEntries.append(parseEntry(key: key, value: value)) + return true + }, limit: count - lowerEntries.count - upperEntries.count + 1) + if additionalLowerEntries.count >= count - lowerEntries.count + upperEntries.count + 1 { + lower = additionalLowerEntries.last + additionalLowerEntries.removeLast() + } + lowerEntries.append(contentsOf: additionalLowerEntries) + } + + var entries: [GlobalMessageHistoryTagsTableEntry] = [] + entries.append(contentsOf: lowerEntries.reversed()) + entries.append(contentsOf: upperEntries) + return (entries: entries, lower: lower, upper: upper) + } + + func earlierEntries(_ tagMask: GlobalMessageTags, index: MessageIndex?, count: Int) -> [GlobalMessageHistoryTagsTableEntry] { + var indices: [GlobalMessageHistoryTagsTableEntry] = [] + let key: ValueBoxKey + if let index = index { + key = self.key(tagMask, index: index) + } else { + key = self.upperBound(tagMask) + } + self.valueBox.range(self.table, start: key, end: self.lowerBound(tagMask), values: { key, value in + indices.append(parseEntry(key: key, value: value)) + return true + }, limit: count) + return indices + } + + func laterEntries(_ tagMask: GlobalMessageTags, index: MessageIndex?, count: Int) -> [GlobalMessageHistoryTagsTableEntry] { + var indices: [GlobalMessageHistoryTagsTableEntry] = [] + let key: ValueBoxKey + if let index = index { + key = self.key(tagMask, index: index) + } else { + key = self.lowerBound(tagMask) + } + self.valueBox.range(self.table, start: key, end: self.upperBound(tagMask), values: { key, value in + indices.append(parseEntry(key: key, value: value)) + return true + }, limit: count) + return indices + } +} diff --git a/Postbox/GlobalMessageTagsView.swift b/Postbox/GlobalMessageTagsView.swift new file mode 100644 index 0000000000..9c3aaabaeb --- /dev/null +++ b/Postbox/GlobalMessageTagsView.swift @@ -0,0 +1,469 @@ +import Foundation + +private enum InternalGlobalMessageTagsEntry: Comparable { + case intermediateMessage(IntermediateMessage) + case message(Message) + case hole(MessageIndex) + + var index: MessageIndex { + switch self { + case let .intermediateMessage(message): + return MessageIndex(message) + case let .message(message): + return MessageIndex(message) + case let .hole(index): + return index + } + } + + static func ==(lhs: InternalGlobalMessageTagsEntry, rhs: InternalGlobalMessageTagsEntry) -> Bool { + switch lhs { + case let .intermediateMessage(lhsMessage): + if case let .intermediateMessage(rhsMessage) = rhs { + if lhsMessage.stableVersion != rhsMessage.stableVersion { + return false + } + if MessageIndex(lhsMessage) != MessageIndex(rhsMessage) { + return false + } + return true + } else { + return false + } + case let .message(lhsMessage): + if case let .message(rhsMessage) = rhs { + if lhsMessage.stableVersion != rhsMessage.stableVersion { + return false + } + if MessageIndex(lhsMessage) != MessageIndex(rhsMessage) { + return false + } + return true + } else { + return false + } + case let .hole(index): + if case .hole(index) = rhs { + return true + } else { + return false + } + } + } + + static func <(lhs: InternalGlobalMessageTagsEntry, rhs: InternalGlobalMessageTagsEntry) -> Bool { + return lhs.index < rhs.index + } +} + +public enum GlobalMessageTagsEntry { + case message(Message) + case hole(MessageIndex) + + public var index: MessageIndex { + switch self { + case let .message(message): + return MessageIndex(message) + case let .hole(index): + return index + } + } +} + +final class MutableGlobalMessageTagsViewReplayContext { + var invalidEarlier: Bool = false + var invalidLater: Bool = false + var removedEntries: Bool = false + + func empty() -> Bool { + return !self.removedEntries && !invalidEarlier && !invalidLater + } +} + +final class MutableGlobalMessageTagsView: MutablePostboxView { + private let globalTag: GlobalMessageTags + private let count: Int + private let groupingPredicate: ((Message, Message) -> Bool)? + + fileprivate var entries: [InternalGlobalMessageTagsEntry] + fileprivate var earlier: MessageIndex? + fileprivate var later: MessageIndex? + + init(postbox: Postbox, globalTag: GlobalMessageTags, position: MessageIndex, count: Int, groupingPredicate: ((Message, Message) -> Bool)?) { + self.globalTag = globalTag + self.count = count + self.groupingPredicate = groupingPredicate + + let (entries, lower, upper) = postbox.messageHistoryTable.entriesAround(globalTagMask: globalTag, index: position, count: count) + + self.entries = entries.map { entry -> InternalGlobalMessageTagsEntry in + switch entry { + case let .message(message): + return .intermediateMessage(message) + case let .hole(index): + return .hole(index) + } + } + self.earlier = lower + self.later = upper + + self.render(postbox: postbox) + } + + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { + var hasChanges = false + + let context = MutableGlobalMessageTagsViewReplayContext() + + for operation in transaction.currentGlobalTagsOperations { + switch operation { + case let .insertMessage(tags, message): + if (self.globalTag.rawValue & tags.rawValue) != 0 { + if self.add(.intermediateMessage(message)) { + hasChanges = true + } + } + case let .insertHole(tags, index): + if (self.globalTag.rawValue & tags.rawValue) != 0 { + if self.add(.hole(index)) { + hasChanges = true + } + } + case let .remove(tagsAndIndices): + var indices = Set() + for (tags, index) in tagsAndIndices { + if (self.globalTag.rawValue & tags.rawValue) != 0 { + indices.insert(index) + } + } + if !indices.isEmpty { + if self.remove(indices, context: context) { + hasChanges = true + } + } + case let .updateTimestamp(tags, previousIndex, updatedTimestamp): + if (self.globalTag.rawValue & tags.rawValue) != 0 { + inner: for i in 0 ..< self.entries.count { + let entry = self.entries[i] + if entry.index == previousIndex { + let updatedIndex = MessageIndex(id: entry.index.id, timestamp: updatedTimestamp) + if self.remove(Set([entry.index]), context: context) { + hasChanges = true + } + switch entry { + case .hole: + if self.add(.hole(updatedIndex)) { + hasChanges = true + } + case let .intermediateMessage(message): + if self.add(.intermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: updatedTimestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia))) { + hasChanges = true + } + case let .message(message): + if self.add(.message(Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: updatedTimestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds))) { + hasChanges = true + } + } + break inner + } + } + } + } + } + + if hasChanges || !context.empty() { + self.complete(postbox: postbox, context: context) + self.render(postbox: postbox) + + self.render(postbox: postbox) + } + + return hasChanges + } + + private func add(_ entry: InternalGlobalMessageTagsEntry) -> Bool { + if self.entries.count == 0 { + self.entries.append(entry) + return true + } else { + let first = self.entries[self.entries.count - 1] + let last = self.entries[0] + + let next = self.later + + if entry.index < last.index { + if self.earlier == nil || self.earlier! < entry.index { + if self.entries.count < self.count { + self.entries.insert(entry, at: 0) + } else { + self.earlier = entry.index + } + return true + } else { + return false + } + } else if entry.index > first.index { + if next != nil && entry.index > next! { + if self.later == nil || self.later! > entry.index { + if self.entries.count < self.count { + self.entries.append(entry) + } else { + self.later = entry.index + } + return true + } else { + return false + } + } else { + self.entries.append(entry) + if self.entries.count > self.count { + self.earlier = self.entries[0].index + self.entries.remove(at: 0) + } + return true + } + } else if entry != last && entry != first { + var i = self.entries.count + while i >= 1 { + if self.entries[i - 1].index < entry.index { + break + } + i -= 1 + } + self.entries.insert(entry, at: i) + if self.entries.count > self.count { + self.earlier = self.entries[0].index + self.entries.remove(at: 0) + } + return true + } else { + return false + } + } + } + + private func remove(_ indices: Set, context: MutableGlobalMessageTagsViewReplayContext) -> Bool { + var hasChanges = false + if let earlier = self.earlier, indices.contains(earlier) { + context.invalidEarlier = true + hasChanges = true + } + + if let later = self.later , indices.contains(later) { + context.invalidLater = true + hasChanges = true + } + + if self.entries.count != 0 { + var i = self.entries.count - 1 + while i >= 0 { + if indices.contains(self.entries[i].index) { + self.entries.remove(at: i) + context.removedEntries = true + hasChanges = true + } + i -= 1 + } + } + + return hasChanges + } + + private func complete(postbox: Postbox, context: MutableGlobalMessageTagsViewReplayContext) { + if context.removedEntries { + self.completeWithReset(postbox: postbox) + } else { + if context.invalidEarlier { + var earlyId: MessageIndex? + let i = 0 + if i < self.entries.count { + earlyId = self.entries[i].index + } + + let earlierEntries = postbox.messageHistoryTable.earlierEntries(globalTagMask: self.globalTag, index: earlyId, count: 1).map { entry -> InternalGlobalMessageTagsEntry in + switch entry { + case let .message(message): + return .intermediateMessage(message) + case let .hole(index): + return .hole(index) + } + } + self.earlier = earlierEntries.first?.index + } + + if context.invalidLater { + var laterId: MessageIndex? + let i = self.entries.count - 1 + if i >= 0 { + laterId = self.entries[i].index + } + + let laterEntries = postbox.messageHistoryTable.laterEntries(globalTagMask: self.globalTag, index: laterId, count: 1).map { entry -> InternalGlobalMessageTagsEntry in + switch entry { + case let .message(message): + return .intermediateMessage(message) + case let .hole(index): + return .hole(index) + } + } + self.later = laterEntries.first?.index + } + } + } + + private func completeWithReset(postbox: Postbox) { + var addedEntries: [InternalGlobalMessageTagsEntry] = [] + + var latestAnchor: MessageIndex? + if let last = self.entries.last { + latestAnchor = last.index + } + + if latestAnchor == nil { + if let later = self.later { + latestAnchor = later + } + } + + if let later = self.later { + addedEntries += postbox.messageHistoryTable.laterEntries(globalTagMask: self.globalTag, index: later.predecessor(), count: self.count).map { entry -> InternalGlobalMessageTagsEntry in + switch entry { + case let .message(message): + return .intermediateMessage(message) + case let .hole(index): + return .hole(index) + } + } + } + if let earlier = self.earlier { + addedEntries += postbox.messageHistoryTable.earlierEntries(globalTagMask: self.globalTag, index: earlier.successor(), count: self.count).map { entry -> InternalGlobalMessageTagsEntry in + switch entry { + case let .message(message): + return .intermediateMessage(message) + case let .hole(index): + return .hole(index) + } + } + } + + addedEntries += self.entries + addedEntries.sort(by: { $0.index < $1.index }) + var i = addedEntries.count - 1 + while i >= 1 { + if addedEntries[i].index.id == addedEntries[i - 1].index.id { + addedEntries.remove(at: i) + } + i -= 1 + } + self.entries = [] + + var anchorIndex = addedEntries.count - 1 + if let latestAnchor = latestAnchor { + var i = addedEntries.count - 1 + while i >= 0 { + if addedEntries[i].index <= latestAnchor { + anchorIndex = i + break + } + i -= 1 + } + } + + /*let indices = self.groupedIndices(addedEntries) + + if indices.count > self.count { + + } else { + self.later = nil + self.earlier = nil + self.entries = addedEntries + }*/ + + self.later = nil + if anchorIndex + 1 < addedEntries.count { + self.later = addedEntries[anchorIndex + 1].index + } + + i = anchorIndex + while i >= 0 && i > anchorIndex - self.count { + self.entries.insert(addedEntries[i], at: 0) + i -= 1 + } + + self.earlier = nil + if anchorIndex - self.count >= 0 { + self.earlier = addedEntries[anchorIndex - self.count].index + } + } + + private func groupedIndices(_ entries: [InternalGlobalMessageTagsEntry]) -> [[Int]] { + if entries.isEmpty { + return [] + } + if let groupingPredicate = self.groupingPredicate { + var result: [[Int]] = [[0]] + for i in 1 ..< entries.count { + switch entries[i] { + case .hole: + result.append([i]) + case let .message(message): + switch entries[i - 1] { + case .hole: + result.append([i]) + case let .message(previousMessage): + if !groupingPredicate(message, previousMessage) { + result.append([i]) + } else { + result[result.count - 1].append(i) + } + case .intermediateMessage: + assertionFailure() + result.append([i]) + } + case .intermediateMessage: + assertionFailure() + result.append([i]) + } + } + return result + } else { + return (0 ..< entries.count).map { [$0] } + } + } + + private func render(postbox: Postbox) { + for i in 0 ..< self.entries.count { + if case let .intermediateMessage(message) = self.entries[i] { + self.entries[i] = .message(postbox.renderIntermediateMessage(message)) + } + } + } + + func immutableView() -> PostboxView { + return GlobalMessageTagsView(self) + } +} + +public final class GlobalMessageTagsView: PostboxView { + public let entries: [GlobalMessageTagsEntry] + public let earlier: MessageIndex? + public let later: MessageIndex? + + init(_ view: MutableGlobalMessageTagsView) { + var entries: [GlobalMessageTagsEntry] = [] + for entry in view.entries { + switch entry { + case let .message(message): + entries.append(.message(message)) + case let .hole(index): + entries.append(.hole(index)) + case .intermediateMessage: + assertionFailure() + break + } + } + self.entries = entries + self.earlier = view.earlier + self.later = view.later + } +} diff --git a/Postbox/ItemCacheMetaTable.swift b/Postbox/ItemCacheMetaTable.swift index 2a7cd00ba4..56db5ddd86 100644 --- a/Postbox/ItemCacheMetaTable.swift +++ b/Postbox/ItemCacheMetaTable.swift @@ -16,7 +16,7 @@ struct ItemCacheCollectionState: Coding { let nextAccessIndex: Int32 init(decoder: Decoder) { - self.nextAccessIndex = decoder.decodeInt32ForKey("i") + self.nextAccessIndex = decoder.decodeInt32ForKey("i", orElse: 0) } func encode(_ encoder: Encoder) { diff --git a/Postbox/ManagedFile.swift b/Postbox/ManagedFile.swift new file mode 100644 index 0000000000..61243b496a --- /dev/null +++ b/Postbox/ManagedFile.swift @@ -0,0 +1,73 @@ +import Foundation +#if os(macOS) + import SwiftSignalKitMac +#else + import SwiftSignalKit +#endif + +public enum ManagedFileMode { + case read + case readwrite + case append +} + +private func wrappedWrite(_ fd: Int32, _ data: UnsafeRawPointer, _ count: Int) -> Int { + return write(fd, data, count) +} + +private func wrappedRead(_ fd: Int32, _ data: UnsafeMutableRawPointer, _ count: Int) -> Int { + return read(fd, data, count) +} + +public final class ManagedFile { + private let queue: Queue + private let fd: Int32 + private let mode: ManagedFileMode + + public init?(queue: Queue, path: String, mode: ManagedFileMode) { + self.queue = queue + self.mode = mode + let fileMode: Int32 + let accessMode: UInt16 + switch mode { + case .read: + fileMode = O_RDONLY + accessMode = S_IRUSR + case .readwrite: + fileMode = O_RDWR | O_CREAT + accessMode = S_IRUSR | S_IWUSR + case .append: + fileMode = O_WRONLY | O_CREAT | O_APPEND + accessMode = S_IRUSR | S_IWUSR + } + let fd = open(path, fileMode, accessMode) + if fd >= 0 { + self.fd = fd + } else { + return nil + } + } + + deinit { + assert(self.queue.isCurrent()) + close(self.fd) + } + + public func write(_ data: UnsafeRawPointer, count: Int) -> Int { + assert(self.queue.isCurrent()) + return wrappedWrite(self.fd, data, count) + } + + public func read(_ data: UnsafeMutableRawPointer, _ count: Int) -> Int { + assert(self.queue.isCurrent()) + return wrappedRead(self.fd, data, count) + } + + public func seek(position: Int64) { + lseek(self.fd, position, SEEK_SET) + } + + public func truncate(count: Int64) { + ftruncate(self.fd, count) + } +} diff --git a/Postbox/Media.swift b/Postbox/Media.swift index 5fa0f0c7ab..de3bd8834f 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -36,8 +36,8 @@ public struct MediaId: Hashable, Coding, CustomStringConvertible { } public init(decoder: Decoder) { - self.namespace = decoder.decodeInt32ForKey("n") - self.id = decoder.decodeInt64ForKey("i") + self.namespace = decoder.decodeInt32ForKey("n", orElse: 0) + self.id = decoder.decodeInt64ForKey("i", orElse: 0) } public func encode(_ encoder: Encoder) { diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 9ebe9d4782..368f7c946a 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -570,15 +570,14 @@ public final class MediaBox { } var offset = currentSize - var fd: Int32? + let file = Atomic(value: nil) let dataQueue = self.dataQueue dataContext.fetchDisposable = ((self.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in return fetch(resource, currentSize ..< Int.max, tag) }) |> afterDisposed { dataQueue.async { - if let thisFd = fd { - close(thisFd) - fd = nil + let _ = file.modify { current in + return nil } } }).start(next: { resultOption in @@ -587,20 +586,25 @@ public final class MediaBox { switch resultOption { case let .dataPart(data, dataRange, complete): - if fd == nil { - let handle = open(paths.partial, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) - if handle >= 0 { - fd = handle + var currentFile: ManagedFile? + let _ = file.modify { current in + if let current = current { + currentFile = current + return current + } else { + let newFile = ManagedFile(queue: self.dataQueue, path: paths.partial, mode: .append) + currentFile = newFile + return newFile } } - if let thisFd = fd { + if let currentFile = currentFile { if !dataRange.isEmpty { let writeResult = data.withUnsafeBytes { bytes -> Int in - return write(thisFd, bytes.advanced(by: dataRange.lowerBound), dataRange.count) + return currentFile.write(bytes.advanced(by: dataRange.lowerBound), count: dataRange.count) } if writeResult != dataRange.count { - print("write error \(errno)") + assertionFailure("write error \(errno)") } } @@ -662,44 +666,39 @@ public final class MediaBox { } } case let .replaceHeader(data, dataRange): - if let thisFd = fd { - close(thisFd) - fd = nil + let currentFile = ManagedFile(queue: self.dataQueue, path: paths.partial, mode: .readwrite) + let _ = file.modify { _ in + return nil } - if fd == nil { - let handle = open(paths.partial, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR) - if handle >= 0 { - fd = handle - } - } - - if let thisFd = fd { + if let currentFile = currentFile { if !dataRange.isEmpty { - lseek(thisFd, 0, SEEK_SET) + currentFile.seek(position: 0) let writeResult = data.withUnsafeBytes { bytes -> Int in - return write(thisFd, bytes.advanced(by: dataRange.lowerBound), dataRange.count) + return currentFile.write(bytes.advanced(by: dataRange.lowerBound), count: dataRange.count) } - lseek(thisFd, Int64(offset), SEEK_SET) + currentFile.seek(position: Int64(offset)) if writeResult != dataRange.count { - print("write error \(errno)") + assertionFailure("write error \(errno)") } - - close(thisFd) - fd = nil } } case .reset: - if fd == nil { - let handle = open(paths.partial, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) - if handle >= 0 { - fd = handle + var currentFile: ManagedFile? + let _ = file.modify { current in + if let current = current { + currentFile = current + return current + } else { + let newFile = ManagedFile(queue: self.dataQueue, path: paths.partial, mode: .append) + currentFile = newFile + return newFile } } - if let fd = fd { - ftruncate(fd, 0) - lseek(fd, 0, SEEK_SET) + if let currentFile = currentFile { + currentFile.truncate(count: 0) + currentFile.seek(position: 0) } else { assertionFailure() } @@ -751,10 +750,10 @@ public final class MediaBox { } } case let .moveLocalFile(tempPath): - if let thisFd = fd { - close(thisFd) - fd = nil + let _ = file.modify { _ in + return nil } + unlink(paths.partial) do { try FileManager.default.moveItem(atPath: tempPath, toPath: paths.partial) diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 5d316bca8b..621c3c2935 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -214,7 +214,7 @@ public struct ChatListIndex: Comparable, Hashable { } } -public struct MessageTags: OptionSet { +public struct MessageTags: OptionSet, Sequence { public var rawValue: UInt32 public init(rawValue: UInt32) { @@ -226,9 +226,28 @@ public struct MessageTags: OptionSet { } public static let All = MessageTags(rawValue: 0xffffffff) + + public func makeIterator() -> AnyIterator { + var index = 0 + return AnyIterator { () -> MessageTags? in + while index < 31 { + let currentTags = self.rawValue >> UInt32(index) + let tag = MessageTags(rawValue: 1 << UInt32(index)) + index += 1 + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + return tag + } + } + return nil + } + } } -public struct GlobalMessageTags: OptionSet { +public struct GlobalMessageTags: OptionSet, Sequence, Hashable { public var rawValue: UInt32 public init(rawValue: UInt32) { @@ -238,6 +257,34 @@ public struct GlobalMessageTags: OptionSet { public init() { self.rawValue = 0 } + + var isSingleTag: Bool { + let t = Int32(bitPattern: self.rawValue) + return t != 0 && t == (t & (-t)) + } + + public func makeIterator() -> AnyIterator { + var index = 0 + return AnyIterator { () -> GlobalMessageTags? in + while index < 31 { + let currentTags = self.rawValue >> UInt32(index) + let tag = GlobalMessageTags(rawValue: 1 << UInt32(index)) + index += 1 + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + return tag + } + } + return nil + } + } + + public var hashValue: Int { + return self.rawValue.hashValue + } } public struct MessageFlags: OptionSet { @@ -492,6 +539,14 @@ public final class StoreMessage { self.attributes = attributes self.media = media } + + public var index: MessageIndex? { + if case let .Id(id) = self.id { + return MessageIndex(id: id, timestamp: self.timestamp) + } else { + return nil + } + } } final class InternalStoreMessage { diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index a2a126a451..d502a97efe 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -25,10 +25,10 @@ enum RenderedMessageHistoryEntry { var index: MessageIndex { switch self { - case let .RenderedMessage(message): - return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .Hole(hole): - return hole.maxIndex + case let .RenderedMessage(message): + return MessageIndex(id: message.id, timestamp: message.timestamp) + case let .Hole(hole): + return hole.maxIndex } } } @@ -59,16 +59,18 @@ final class MessageHistoryTable: Table { let globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable let unsentTable: MessageHistoryUnsentTable let tagsTable: MessageHistoryTagsTable + let globalTagsTable: GlobalMessageHistoryTagsTable let readStateTable: MessageHistoryReadStateTable let synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable - init(valueBox: ValueBox, table: ValueBoxTable, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable) { + init(valueBox: ValueBox, table: ValueBoxTable, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, globalTagsTable: GlobalMessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable) { self.messageHistoryIndexTable = messageHistoryIndexTable self.messageMediaTable = messageMediaTable self.historyMetadataTable = historyMetadataTable self.globallyUniqueMessageIdsTable = globallyUniqueMessageIdsTable self.unsentTable = unsentTable self.tagsTable = tagsTable + self.globalTagsTable = globalTagsTable self.readStateTable = readStateTable self.synchronizeReadStateTable = synchronizeReadStateTable @@ -125,9 +127,8 @@ final class MessageHistoryTable: Table { return dict } - private func processIndexOperationsCommitAccumulatedRemoveIndices(remove: Bool, peerId: PeerId, accumulatedRemoveIndices: inout [(MessageIndex, Bool)], updatedCombinedState: inout CombinedPeerReadState?, invalidateReadState: inout Bool, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], outputOperations: inout [MessageHistoryOperation]) { + private func processIndexOperationsCommitAccumulatedRemoveIndices(remove: Bool, peerId: PeerId, accumulatedRemoveIndices: inout [(MessageIndex, Bool)], updatedCombinedState: inout CombinedPeerReadState?, invalidateReadState: inout Bool, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], outputOperations: inout [MessageHistoryOperation], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { if !accumulatedRemoveIndices.isEmpty { - let (combinedState, invalidate) = self.readStateTable.deleteMessages(peerId, indices: accumulatedRemoveIndices.map { $0.0 }, incomingStatsInIndices: { peerId, namespace, indices in return self.incomingMessageStatsInIndices(peerId, namespace: namespace, indices: indices) }) @@ -140,36 +141,51 @@ final class MessageHistoryTable: Table { if remove { var indicesWithMetadata: [(MessageIndex, Bool, MessageTags)] = [] + var globalIndicesWithMetadata: [(GlobalMessageTags, MessageIndex)] = [] if accumulatedRemoveIndices.count == 1 && accumulatedRemoveIndices[0].0.timestamp == Int32.max { assert(true) } for index in accumulatedRemoveIndices { - let tags = self.justRemove(index.0, unsentMessageOperations: &unsentMessageOperations) - if let tags = tags { + let tagsAndGlobalTags = self.justRemove(index.0, unsentMessageOperations: &unsentMessageOperations) + if let (tags, globalTags) = tagsAndGlobalTags { indicesWithMetadata.append((index.0, index.1, tags)) + + if !globalTags.isEmpty { + globalIndicesWithMetadata.append(globalTags, index.0) + } } else { indicesWithMetadata.append((index.0, index.1, MessageTags())) } } assert(accumulatedRemoveIndices.count == indicesWithMetadata.count) outputOperations.append(.Remove(indicesWithMetadata)) + if !globalIndicesWithMetadata.isEmpty { + globalTagsOperations.append(.remove(globalIndicesWithMetadata)) + } } else { var indicesWithMetadata: [(MessageIndex, Bool, MessageTags)] = [] + var globalIndicesWithMetadata: [(GlobalMessageTags, MessageIndex)] = [] for index in accumulatedRemoveIndices { - let tags = self.tagsForIndex(index.0) - if let tags = tags { + if let (tags, globalTags) = self.tagsForIndex(index.0) { indicesWithMetadata.append((index.0, index.1, tags)) + + if !globalTags.isEmpty { + globalIndicesWithMetadata.append(globalTags, index.0) + } } } assert(accumulatedRemoveIndices.count == indicesWithMetadata.count) outputOperations.append(.Remove(indicesWithMetadata)) + if !globalIndicesWithMetadata.isEmpty { + globalTagsOperations.append(.remove(globalIndicesWithMetadata)) + } } accumulatedRemoveIndices.removeAll() } } - private func processIndexOperations(_ peerId: PeerId, operations: [MessageHistoryIndexOperation], processedOperationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + private func processIndexOperations(_ peerId: PeerId, operations: [MessageHistoryIndexOperation], processedOperationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0)) let sharedBuffer = WriteBuffer() let sharedEncoder = Encoder() @@ -198,7 +214,7 @@ final class MessageHistoryTable: Table { for operation in operations { switch operation { case let .InsertHole(hole): - processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations, globalTagsOperations: &globalTagsOperations) commitAccumulatedAddedIndices() self.justInsertHole(hole) @@ -217,7 +233,7 @@ final class MessageHistoryTable: Table { } } case let .InsertMessage(storeMessage): - processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations, globalTagsOperations: &globalTagsOperations) let message = self.justInsertMessage(storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder) outputOperations.append(.InsertMessage(message)) @@ -238,6 +254,22 @@ final class MessageHistoryTable: Table { } } } + let globalTags = message.globalTags.rawValue + if globalTags != 0 { + for i in 0 ..< 32 { + let currentTags = globalTags >> UInt32(i) + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + let tag = GlobalMessageTags(rawValue: 1 << UInt32(i)) + if self.globalTagsTable.addMessage(tag, index: MessageIndex(message)) { + globalTagsOperations.append(.insertMessage(tag, message)) + } + } + } + } if message.flags.contains(.Incoming) { accumulatedAddedIncomingMessageIndices.insert(MessageIndex(message)) } @@ -247,7 +279,7 @@ final class MessageHistoryTable: Table { accumulatedRemoveIndices.append((index, isMessage)) case let .Update(index, storeMessage): commitAccumulatedAddedIndices() - processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations, globalTagsOperations: &globalTagsOperations) if let (message, previousTags) = self.justUpdate(index, message: storeMessage, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder, unsentMessageOperations: &unsentMessageOperations) { outputOperations.append(.Remove([(index, true, previousTags)])) @@ -262,15 +294,20 @@ final class MessageHistoryTable: Table { } case let .UpdateTimestamp(index, timestamp): commitAccumulatedAddedIndices() - processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations, globalTagsOperations: &globalTagsOperations) - self.justUpdateTimestamp(index, timestamp: timestamp) + let tagsAndGlobalTags = self.justUpdateTimestamp(index, timestamp: timestamp) outputOperations.append(.UpdateTimestamp(index, timestamp)) + if let (_, globalTags) = tagsAndGlobalTags { + if !globalTags.isEmpty { + globalTagsOperations.append(.updateTimestamp(globalTags, index, timestamp)) + } + } } } commitAccumulatedAddedIndices() - processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations) + processIndexOperationsCommitAccumulatedRemoveIndices(remove: true, peerId: peerId, accumulatedRemoveIndices: &accumulatedRemoveIndices, updatedCombinedState: &updatedCombinedState, invalidateReadState: &invalidateReadState, unsentMessageOperations: &unsentMessageOperations, outputOperations: &outputOperations, globalTagsOperations: &globalTagsOperations) if let updatedCombinedState = updatedCombinedState { outputOperations.append(.UpdateReadState(updatedCombinedState)) @@ -301,9 +338,10 @@ final class MessageHistoryTable: Table { return internalStoreMessages } - func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [Int64: MessageId] { + func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) -> [Int64: MessageId] { let messagesByPeerId = self.messagesGroupedByPeerId(messages) var globallyUniqueIdToMessageId: [Int64: MessageId] = [:] + var globalTagsInitialized = Set() for (peerId, peerMessages) in messagesByPeerId { var operations: [MessageHistoryIndexOperation] = [] let internalPeerMessages = self.internalStoreMessages(peerMessages) @@ -311,66 +349,74 @@ final class MessageHistoryTable: Table { if let globallyUniqueId = message.globallyUniqueId { globallyUniqueIdToMessageId[globallyUniqueId] = message.id } + if !message.globalTags.isEmpty { + for tag in message.globalTags { + if !globalTagsInitialized.contains(tag) { + self.globalTagsTable.ensureInitialized(tag) + globalTagsInitialized.insert(tag) + } + } + } } self.messageHistoryIndexTable.addMessages(internalPeerMessages, location: location, operations: &operations) - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } return globallyUniqueIdToMessageId } - func addHoles(_ messageIds: [MessageId], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func addHoles(_ messageIds: [MessageId], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] for id in messageIds { self.messageHistoryIndexTable.addHole(id, operations: &operations) } - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } } - func removeMessages(_ messageIds: [MessageId], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func removeMessages(_ messageIds: [MessageId], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { for (peerId, messageIds) in self.messageIdsByPeerId(messageIds) { var operations: [MessageHistoryIndexOperation] = [] for id in messageIds { self.messageHistoryIndexTable.removeMessage(id, operations: &operations) } - self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } } - func clearHistory(peerId: PeerId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func clearHistory(peerId: PeerId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { let indices = self.allIndices(peerId) - self.removeMessages(indices.map { $0.id }, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.removeMessages(indices.map { $0.id }, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } - func removeAllMessagesWithAuthor(peerId: PeerId, authorId: PeerId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func removeAllMessagesWithAuthor(peerId: PeerId, authorId: PeerId, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { let indices = self.allIndicesWithAuthor(peerId, authorId: authorId) - self.removeMessages(indices.map { $0.id }, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.removeMessages(indices.map { $0.id }, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } - func fillHole(_ id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func fillHole(_ id: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.fillHole(id, fillType: fillType, tagMask: tagMask, messages: self.internalStoreMessages(messages), operations: &operations) - self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } - func fillMultipleHoles(mainHoleId: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func fillMultipleHoles(mainHoleId: MessageId, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage], operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.fillMultipleHoles(mainHoleId: mainHoleId, fillType: fillType, tagMask: tagMask, messages: self.internalStoreMessages(messages), operations: &operations) - self.processIndexOperations(mainHoleId.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(mainHoleId.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } - func updateMessage(_ id: MessageId, message: StoreMessage, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func updateMessage(_ id: MessageId, message: StoreMessage, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.updateMessage(id, message: self.internalStoreMessages([message]).first!, operations: &operations) - self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } - func updateMessageTimestamp(_ id: MessageId, timestamp: Int32, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func updateMessageTimestamp(_ id: MessageId, timestamp: Int32, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { var operations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.updateTimestamp(id, timestamp: timestamp, operations: &operations) - self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(id.peerId, operations: operations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } func updateMedia(_ id: MediaId, media: Media?, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], updatedMedia: inout [MediaId: Media?]) { @@ -559,10 +605,10 @@ final class MessageHistoryTable: Table { return nil } - func offsetPendingMessagesTimestamps(lowerBound: MessageId, timestamp: Int32, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) { + func offsetPendingMessagesTimestamps(lowerBound: MessageId, timestamp: Int32, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) { for messageId in self.unsentTable.get() { if messageId.peerId == lowerBound.peerId && messageId.namespace == lowerBound.namespace && messageId.id > lowerBound.id { - self.updateMessageTimestamp(messageId, timestamp: timestamp, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.updateMessageTimestamp(messageId, timestamp: timestamp, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } } } @@ -731,24 +777,25 @@ final class MessageHistoryTable: Table { }) } - private func tagsForIndex(_ index: MessageIndex) -> MessageTags? { + private func tagsForIndex(_ index: MessageIndex) -> (MessageTags, GlobalMessageTags)? { let key = self.key(index) if let value = self.valueBox.get(self.table, key: key) { switch self.readIntermediateEntry(key, value: value) { case let .Message(message): - return message.tags + return (message.tags, message.globalTags) case let .Hole(hole): - return MessageTags(rawValue: hole.tags) + return (MessageTags(rawValue: hole.tags), GlobalMessageTags()) } } else { return nil } } - private func justRemove(_ index: MessageIndex, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) -> MessageTags? { + private func justRemove(_ index: MessageIndex, unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation]) -> (MessageTags, GlobalMessageTags)? { let key = self.key(index) if let value = self.valueBox.get(self.table, key: key) { let resultTags: MessageTags + let resultGlobalTags: GlobalMessageTags switch self.readIntermediateEntry(key, value: value) { case let .Message(message): let embeddedMediaData = message.embeddedMediaData @@ -787,12 +834,27 @@ final class MessageHistoryTable: Table { } } } + let globalTags = message.globalTags.rawValue + if globalTags != 0 { + for i in 0 ..< 32 { + let currentTags = globalTags >> UInt32(i) + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + let tag = GlobalMessageTags(rawValue: 1 << UInt32(i)) + self.globalTagsTable.remove(tag, index: index) + } + } + } for mediaId in message.referencedMedia { let _ = self.messageMediaTable.removeReference(mediaId) } resultTags = message.tags + resultGlobalTags = message.globalTags case let .Hole(hole): let tags = self.messageHistoryIndexTable.seedConfiguration.existingMessageTags.rawValue & hole.tags if tags != 0 { @@ -809,10 +871,11 @@ final class MessageHistoryTable: Table { } } resultTags = MessageTags(rawValue: hole.tags) + resultGlobalTags = GlobalMessageTags() } self.valueBox.remove(self.table, key: key) - return resultTags + return (resultTags, resultGlobalTags) } else { return nil } @@ -967,6 +1030,19 @@ final class MessageHistoryTable: Table { self.tagsTable.add(message.tags, index: MessageIndex(message)) } + if previousMessage.globalTags != message.globalTags || (MessageIndex(previousMessage) != index && (!previousMessage.globalTags.isEmpty || !message.globalTags.isEmpty)) { + if !previousMessage.globalTags.isEmpty { + for tag in previousMessage.globalTags { + self.globalTagsTable.remove(tag, index: index) + } + } + if !message.globalTags.isEmpty { + for tag in message.globalTags { + let _ = self.globalTagsTable.addMessage(tag, index: MessageIndex(message)) + } + } + } + if message.globallyUniqueId != previousMessage.globallyUniqueId { if let globallyUniqueId = previousMessage.globallyUniqueId { self.globallyUniqueMessageIdsTable.remove(peerId: message.id.peerId, globallyUniqueId: globallyUniqueId) @@ -1138,7 +1214,7 @@ final class MessageHistoryTable: Table { } } - private func justUpdateTimestamp(_ index: MessageIndex, timestamp: Int32) { + private func justUpdateTimestamp(_ index: MessageIndex, timestamp: Int32) -> (MessageTags, GlobalMessageTags)? { if let previousMessage = self.getMessage(index) { self.valueBox.remove(self.table, key: self.key(index)) let updatedMessage = IntermediateMessage(stableId: previousMessage.stableId, stableVersion: previousMessage.stableVersion + 1, id: previousMessage.id, globallyUniqueId: previousMessage.globallyUniqueId, timestamp: timestamp, flags: previousMessage.flags, tags: previousMessage.tags, globalTags: previousMessage.globalTags, forwardInfo: previousMessage.forwardInfo, authorId: previousMessage.authorId, text: previousMessage.text, attributesData: previousMessage.attributesData, embeddedMediaData: previousMessage.embeddedMediaData, referencedMedia: previousMessage.referencedMedia) @@ -1159,6 +1235,26 @@ final class MessageHistoryTable: Table { } } } + + let globalTags = previousMessage.globalTags.rawValue + if globalTags != 0 { + for i in 0 ..< 32 { + let currentTags = globalTags >> UInt32(i) + if currentTags == 0 { + break + } + + if (currentTags & 1) != 0 { + let tag = GlobalMessageTags(rawValue: 1 << UInt32(i)) + self.globalTagsTable.remove(tag, index: index) + let _ = self.globalTagsTable.addMessage(tag, index: MessageIndex(id: index.id, timestamp: timestamp)) + } + } + } + + return (previousMessage.tags, previousMessage.globalTags) + } else { + return nil } } @@ -1569,11 +1665,11 @@ final class MessageHistoryTable: Table { return Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, timestamp: message.timestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, forwardInfo: forwardInfo, author: author, text: message.text, attributes: parsedAttributes, media: parsedMedia, peers: peers, associatedMessages: associatedMessages, associatedMessageIds: associatedMessageIds) } - func entriesAround(_ index: MessageIndex, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { + func entriesAround(_ index: MessageIndex, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { var indexOperations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.ensureInitialized(index.id.peerId, operations: &indexOperations) if !indexOperations.isEmpty { - self.processIndexOperations(index.id.peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(index.id.peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } var lowerEntries: [IntermediateMessageHistoryEntry] = [] @@ -1619,11 +1715,11 @@ final class MessageHistoryTable: Table { return (entries: entries, lower: lower, upper: upper) } - func entriesAround(_ tagMask: MessageTags, index: MessageIndex, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { + func entriesAround(_ tagMask: MessageTags, index: MessageIndex, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) -> (entries: [IntermediateMessageHistoryEntry], lower: IntermediateMessageHistoryEntry?, upper: IntermediateMessageHistoryEntry?) { var indexOperations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.ensureInitialized(index.id.peerId, operations: &indexOperations) if !indexOperations.isEmpty { - self.processIndexOperations(index.id.peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(index.id.peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } let (indices, lower, upper) = self.tagsTable.indicesAround(tagMask, index: index, count: count) @@ -1662,11 +1758,11 @@ final class MessageHistoryTable: Table { return (entries, lowerEntry, upperEntry) } - func earlierEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [IntermediateMessageHistoryEntry] { + func earlierEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) -> [IntermediateMessageHistoryEntry] { var indexOperations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.ensureInitialized(peerId, operations: &indexOperations) if !indexOperations.isEmpty { - self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } var entries: [IntermediateMessageHistoryEntry] = [] @@ -1683,11 +1779,11 @@ final class MessageHistoryTable: Table { return entries } - func earlierEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [IntermediateMessageHistoryEntry] { + func earlierEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) -> [IntermediateMessageHistoryEntry] { var indexOperations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.ensureInitialized(peerId, operations: &indexOperations) if !indexOperations.isEmpty { - self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } let indices = self.tagsTable.earlierIndices(tagMask, peerId: peerId, index: index, count: count) @@ -1705,11 +1801,11 @@ final class MessageHistoryTable: Table { return entries } - func laterEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [IntermediateMessageHistoryEntry] { + func laterEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) -> [IntermediateMessageHistoryEntry] { var indexOperations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.ensureInitialized(peerId, operations: &indexOperations) if !indexOperations.isEmpty { - self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } var entries: [IntermediateMessageHistoryEntry] = [] @@ -1726,11 +1822,11 @@ final class MessageHistoryTable: Table { return entries } - func laterEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?]) -> [IntermediateMessageHistoryEntry] { + func laterEntries(_ tagMask: MessageTags, peerId: PeerId, index: MessageIndex?, count: Int, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], unsentMessageOperations: inout [IntermediateMessageHistoryUnsentOperation], updatedPeerReadStateOperations: inout [PeerId: PeerReadStateSynchronizationOperation?], globalTagsOperations: inout [GlobalMessageHistoryTagsOperation]) -> [IntermediateMessageHistoryEntry] { var indexOperations: [MessageHistoryIndexOperation] = [] self.messageHistoryIndexTable.ensureInitialized(peerId, operations: &indexOperations) if !indexOperations.isEmpty { - self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations) + self.processIndexOperations(peerId, operations: indexOperations, processedOperationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations) } let indices = self.tagsTable.laterIndices(tagMask, peerId: peerId, index: index, count: count) @@ -1748,6 +1844,96 @@ final class MessageHistoryTable: Table { return entries } + private func globalTagsIntermediateEntry(_ entry: IntermediateMessageHistoryEntry) -> IntermediateGlobalMessageTagsEntry? { + switch entry { + case let .Message(message): + return .message(message) + case .Hole: + return nil + } + } + + func entriesAround(globalTagMask: GlobalMessageTags, index: MessageIndex, count: Int) -> (entries: [IntermediateGlobalMessageTagsEntry], lower: MessageIndex?, upper: MessageIndex?) { + self.globalTagsTable.ensureInitialized(globalTagMask) + + let (globalEntries, lower, upper) = self.globalTagsTable.entriesAround(globalTagMask, index: index, count: count) + + var entries: [IntermediateGlobalMessageTagsEntry] = [] + for entry in globalEntries { + switch entry { + case let .hole(index): + entries.append(.hole(index)) + case let .message(index): + let key = self.key(index) + if let value = self.valueBox.get(self.table, key: key) { + if let entry = self.globalTagsIntermediateEntry(readIntermediateEntry(key, value: value)) { + entries.append(entry) + } else { + assertionFailure() + } + } else { + assertionFailure() + } + } + } + + return (entries, lower?.index, upper?.index) + } + + func earlierEntries(globalTagMask: GlobalMessageTags, index: MessageIndex?, count: Int) -> [IntermediateGlobalMessageTagsEntry] { + self.globalTagsTable.ensureInitialized(globalTagMask) + + let globalEntries = self.globalTagsTable.earlierEntries(globalTagMask, index: index, count: count) + + var entries: [IntermediateGlobalMessageTagsEntry] = [] + for entry in globalEntries { + switch entry { + case let .hole(index): + entries.append(.hole(index)) + case let .message(index): + let key = self.key(index) + if let value = self.valueBox.get(self.table, key: key) { + if let entry = self.globalTagsIntermediateEntry(readIntermediateEntry(key, value: value)) { + entries.append(entry) + } else { + assertionFailure() + } + } else { + assertionFailure() + } + } + } + + return entries + } + + func laterEntries(globalTagMask: GlobalMessageTags, index: MessageIndex?, count: Int) -> [IntermediateGlobalMessageTagsEntry] { + self.globalTagsTable.ensureInitialized(globalTagMask) + + let globalEntries = self.globalTagsTable.laterEntries(globalTagMask, index: index, count: count) + + var entries: [IntermediateGlobalMessageTagsEntry] = [] + for entry in globalEntries { + switch entry { + case let .hole(index): + entries.append(.hole(index)) + case let .message(index): + let key = self.key(index) + if let value = self.valueBox.get(self.table, key: key) { + if let entry = self.globalTagsIntermediateEntry(readIntermediateEntry(key, value: value)) { + entries.append(entry) + } else { + assertionFailure() + } + } else { + assertionFailure() + } + } + } + + return entries + } + func maxReadIndex(_ peerId: PeerId) -> MessageHistoryAnchorIndex? { if let combinedState = self.readStateTable.getCombinedState(peerId), let state = combinedState.states.first , state.1.count != 0 { switch state.1 { @@ -1926,8 +2112,9 @@ final class MessageHistoryTable: Table { var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId : PeerReadStateSynchronizationOperation?] = [:] + var globalTagsOperations: [GlobalMessageHistoryTagsOperation] = [] - return self.laterEntries(peerId, index: nil, count: 1000, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations).map({ entry -> RenderedMessageHistoryEntry in + return self.laterEntries(peerId, index: nil, count: 1000, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations).map({ entry -> RenderedMessageHistoryEntry in switch entry { case let .Hole(hole): return .Hole(hole) @@ -1941,13 +2128,14 @@ final class MessageHistoryTable: Table { var operationsByPeerId: [PeerId : [MessageHistoryOperation]] = [:] var unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] = [] var updatedPeerReadStateOperations: [PeerId : PeerReadStateSynchronizationOperation?] = [:] + var globalTagsOperations: [GlobalMessageHistoryTagsOperation] = [] - return self.laterEntries(tagMask, peerId: peerId, index: nil, count: 1000, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations).map({ entry -> RenderedMessageHistoryEntry in + return self.laterEntries(tagMask, peerId: peerId, index: nil, count: 1000, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: &unsentMessageOperations, updatedPeerReadStateOperations: &updatedPeerReadStateOperations, globalTagsOperations: &globalTagsOperations).map({ entry -> RenderedMessageHistoryEntry in switch entry { - case let .Hole(hole): - return .Hole(hole) - case let .Message(message): - return .RenderedMessage(self.renderMessage(message, peerTable: peerTable)) + case let .Hole(hole): + return .Hole(hole) + case let .Message(message): + return .RenderedMessage(self.renderMessage(message, peerTable: peerTable)) } }) } diff --git a/Postbox/MetadataTable.swift b/Postbox/MetadataTable.swift index 50ec4da92c..1983ac1129 100644 --- a/Postbox/MetadataTable.swift +++ b/Postbox/MetadataTable.swift @@ -18,8 +18,8 @@ public struct AccessChallengeAttempts: Coding, Equatable { } public init(decoder: Decoder) { - self.count = decoder.decodeInt32ForKey("c") - self.timestamp = decoder.decodeInt32ForKey("t") + self.count = decoder.decodeInt32ForKey("c", orElse: 0) + self.timestamp = decoder.decodeInt32ForKey("t", orElse: 0) } public func encode(_ encoder: Encoder) { @@ -38,13 +38,13 @@ public enum PostboxAccessChallengeData: Coding, Equatable { case plaintextPassword(value: String, timeout: Int32?, attempts: AccessChallengeAttempts?) public init(decoder: Decoder) { - switch decoder.decodeInt32ForKey("r") as Int32 { + switch decoder.decodeInt32ForKey("r", orElse: 0) { case 0: self = .none case 1: - self = .numericalPassword(value: decoder.decodeStringForKey("t"), timeout: decoder.decodeInt32ForKey("a"), attempts: decoder.decodeObjectForKey("att", decoder: { AccessChallengeAttempts(decoder: $0) }) as? AccessChallengeAttempts) + self = .numericalPassword(value: decoder.decodeStringForKey("t", orElse: ""), timeout: decoder.decodeOptionalInt32ForKey("a"), attempts: decoder.decodeObjectForKey("att", decoder: { AccessChallengeAttempts(decoder: $0) }) as? AccessChallengeAttempts) case 2: - self = .plaintextPassword(value: decoder.decodeStringForKey("t"), timeout: decoder.decodeInt32ForKey("a"), attempts: decoder.decodeObjectForKey("att", decoder: { AccessChallengeAttempts(decoder: $0) }) as? AccessChallengeAttempts) + self = .plaintextPassword(value: decoder.decodeStringForKey("t", orElse: ""), timeout: decoder.decodeOptionalInt32ForKey("a"), attempts: decoder.decodeObjectForKey("att", decoder: { AccessChallengeAttempts(decoder: $0) }) as? AccessChallengeAttempts) default: assertionFailure() self = .none diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index f35f6f59a5..b4ee5161d0 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -556,6 +556,11 @@ public final class Modifier { return ([:], [:], nil) } } + + public func replaceGlobalMessageTagsHole(globalTags: GlobalMessageTags, index: MessageIndex, with updatedIndex: MessageIndex?, messages: [StoreMessage]) { + assert(!self.disposed) + self.postbox?.replaceGlobalMessageTagsHole(globalTags: globalTags, index: index, with: updatedIndex, messages: messages) + } } fileprivate class PipeNotifier: NSObject { @@ -646,6 +651,7 @@ public final class Postbox { private var currentUnsentOperations: [IntermediateMessageHistoryUnsentOperation] = [] private var currentUpdatedSynchronizeReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] = [:] private var currentUpdatedMedia: [MediaId: Media?] = [:] + private var currentGlobalTagsOperations: [GlobalMessageHistoryTagsOperation] = [] private var currentRemovedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] = [:] private var currentFilledHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]] = [:] @@ -697,6 +703,7 @@ public final class Postbox { let messageHistoryMetadataTable: MessageHistoryMetadataTable let messageHistoryUnsentTable: MessageHistoryUnsentTable let messageHistoryTagsTable: MessageHistoryTagsTable + let globalMessageHistoryTagsTable: GlobalMessageHistoryTagsTable let peerChatStateTable: PeerChatStateTable let readStateTable: MessageHistoryReadStateTable let synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable @@ -758,13 +765,14 @@ public final class Postbox { self.messageHistoryMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox, table: MessageHistoryMetadataTable.tableSpec(10)) self.messageHistoryUnsentTable = MessageHistoryUnsentTable(valueBox: self.valueBox, table: MessageHistoryUnsentTable.tableSpec(11)) self.messageHistoryTagsTable = MessageHistoryTagsTable(valueBox: self.valueBox, table: MessageHistoryTagsTable.tableSpec(12)) + self.globalMessageHistoryTagsTable = GlobalMessageHistoryTagsTable(valueBox: self.valueBox, table: GlobalMessageHistoryTagsTable.tableSpec(39)) self.messageHistoryIndexTable = MessageHistoryIndexTable(valueBox: self.valueBox, table: MessageHistoryIndexTable.tableSpec(4), globalMessageIdsTable: self.globalMessageIdsTable, metadataTable: self.messageHistoryMetadataTable, seedConfiguration: self.seedConfiguration) self.mediaTable = MessageMediaTable(valueBox: self.valueBox, table: MessageMediaTable.tableSpec(6)) self.readStateTable = MessageHistoryReadStateTable(valueBox: self.valueBox, table: MessageHistoryReadStateTable.tableSpec(14)) self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox, table: MessageHistorySynchronizeReadStateTable.tableSpec(15)) self.timestampBasedMessageAttributesIndexTable = TimestampBasedMessageAttributesIndexTable(valueBox: self.valueBox, table: TimestampBasedMessageAttributesTable.tableSpec(33)) self.timestampBasedMessageAttributesTable = TimestampBasedMessageAttributesTable(valueBox: self.valueBox, table: TimestampBasedMessageAttributesTable.tableSpec(34), indexTable: self.timestampBasedMessageAttributesIndexTable) - self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable, unsentTable: self.messageHistoryUnsentTable, tagsTable: self.messageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable) + self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable, unsentTable: self.messageHistoryUnsentTable, tagsTable: self.messageHistoryTagsTable, globalTagsTable: self.globalMessageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable) self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, table: PeerChatStateTable.tableSpec(13)) self.peerNameTokenIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(26)) self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) @@ -798,6 +806,7 @@ public final class Postbox { tables.append(self.messageHistoryMetadataTable) tables.append(self.messageHistoryUnsentTable) tables.append(self.messageHistoryTagsTable) + tables.append(self.globalMessageHistoryTagsTable) tables.append(self.messageHistoryIndexTable) tables.append(self.mediaTable) tables.append(self.readStateTable) @@ -955,16 +964,16 @@ public final class Postbox { } fileprivate func addMessages(_ messages: [StoreMessage], location: AddMessagesLocation) -> [Int64: MessageId] { - return self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + return self.messageHistoryTable.addMessages(messages, location: location, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } fileprivate func addHole(_ id: MessageId) { - self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + self.messageHistoryTable.addHoles([id], operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } fileprivate func fillHole(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.messageHistoryTable.fillHole(hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + self.messageHistoryTable.fillHole(hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) for (peerId, operations) in operationsByPeerId { if self.currentOperationsByPeerId[peerId] == nil { self.currentOperationsByPeerId[peerId] = operations @@ -1004,7 +1013,7 @@ public final class Postbox { fileprivate func fillMultipleHoles(_ hole: MessageHistoryHole, fillType: HoleFill, tagMask: MessageTags?, messages: [StoreMessage]) { var operationsByPeerId: [PeerId: [MessageHistoryOperation]] = [:] - self.messageHistoryTable.fillMultipleHoles(mainHoleId: hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + self.messageHistoryTable.fillMultipleHoles(mainHoleId: hole.id, fillType: fillType, tagMask: tagMask, messages: messages, operationsByPeerId: &operationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) for (peerId, operations) in operationsByPeerId { if self.currentOperationsByPeerId[peerId] == nil { self.currentOperationsByPeerId[peerId] = operations @@ -1047,15 +1056,15 @@ public final class Postbox { } fileprivate func deleteMessages(_ messageIds: [MessageId]) { - self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + self.messageHistoryTable.removeMessages(messageIds, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } fileprivate func clearHistory(_ peerId: PeerId) { - self.messageHistoryTable.clearHistory(peerId: peerId, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + self.messageHistoryTable.clearHistory(peerId: peerId, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } fileprivate func removeAllMessagesWithAuthor(_ peerId: PeerId, authorId: PeerId) { - self.messageHistoryTable.removeAllMessagesWithAuthor(peerId: peerId, authorId: authorId, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + self.messageHistoryTable.removeAllMessagesWithAuthor(peerId: peerId, authorId: authorId, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: ¤tUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } fileprivate func resetIncomingReadStates(_ states: [PeerId: [MessageId.Namespace: PeerReadState]]) { @@ -1085,9 +1094,9 @@ public final class Postbox { private func fetchEarlierHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { let intermediateEntries: [IntermediateMessageHistoryEntry] if let tagMask = tagMask { - intermediateEntries = self.messageHistoryTable.earlierEntries(tagMask, peerId: peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + intermediateEntries = self.messageHistoryTable.earlierEntries(tagMask, peerId: peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } else { - intermediateEntries = self.messageHistoryTable.earlierEntries(peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + intermediateEntries = self.messageHistoryTable.earlierEntries(peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } var entries: [MutableMessageHistoryEntry] = [] for entry in intermediateEntries { @@ -1108,9 +1117,9 @@ public final class Postbox { let intermediateUpper: IntermediateMessageHistoryEntry? if let tagMask = tagMask { - (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(tagMask, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(tagMask, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } else { - (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + (intermediateEntries, intermediateLower, intermediateUpper) = self.messageHistoryTable.entriesAround(index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } var entries: [MutableMessageHistoryEntry] = [] @@ -1149,9 +1158,9 @@ public final class Postbox { private func fetchLaterHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { let intermediateEntries: [IntermediateMessageHistoryEntry] if let tagMask = tagMask { - intermediateEntries = self.messageHistoryTable.laterEntries(tagMask, peerId: peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + intermediateEntries = self.messageHistoryTable.laterEntries(tagMask, peerId: peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } else { - intermediateEntries = self.messageHistoryTable.laterEntries(peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + intermediateEntries = self.messageHistoryTable.laterEntries(peerId, index: index, count: count, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } var entries: [MutableMessageHistoryEntry] = [] for entry in intermediateEntries { @@ -1233,7 +1242,7 @@ public final class Postbox { return self.messageHistoryTable.anchorIndex(id) } - fileprivate func renderIntermediateMessage(_ message: IntermediateMessage) -> Message { + func renderIntermediateMessage(_ message: IntermediateMessage) -> Message { return self.messageHistoryTable.renderMessage(message, peerTable: self.peerTable) } @@ -1271,7 +1280,7 @@ public final class Postbox { return self.peerTable.get(peerId) }, updatedTotalUnreadCount: &self.currentUpdatedTotalUnreadCount) - let transaction = PostboxTransaction(currentUpdatedState: self.currentUpdatedState, currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, currentOrderedItemListOperations: self.currentOrderedItemListOperations, currentItemCollectionItemsOperations: self.currentItemCollectionItemsOperations, currentItemCollectionInfosOperations: self.currentItemCollectionInfosOperations, currentUpdatedPeerChatStates: self.currentUpdatedPeerChatStates, updatedAccessChallengeData: self.currentUpdatedAccessChallengeData, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) + let transaction = PostboxTransaction(currentUpdatedState: self.currentUpdatedState, currentOperationsByPeerId: self.currentOperationsByPeerId, peerIdsWithFilledHoles: self.currentFilledHolesByPeerId, removedHolesByPeerId: self.currentRemovedHolesByPeerId, chatListOperations: chatListOperations, currentUpdatedPeers: self.currentUpdatedPeers, currentUpdatedPeerNotificationSettings: self.currentUpdatedPeerNotificationSettings, currentUpdatedCachedPeerData: self.currentUpdatedCachedPeerData, currentUpdatedPeerPresences: currentUpdatedPeerPresences, currentUpdatedPeerChatListEmbeddedStates: self.currentUpdatedPeerChatListEmbeddedStates, currentUpdatedTotalUnreadCount: self.currentUpdatedTotalUnreadCount, peerIdsWithUpdatedUnreadCounts: Set(transactionUnreadCountDeltas.keys), currentPeerMergedOperationLogOperations: self.currentPeerMergedOperationLogOperations, currentTimestampBasedMessageAttributesOperations: self.currentTimestampBasedMessageAttributesOperations, unsentMessageOperations: self.currentUnsentOperations, updatedSynchronizePeerReadStateOperations: self.currentUpdatedSynchronizeReadStateOperations, currentPreferencesOperations: self.currentPreferencesOperations, currentOrderedItemListOperations: self.currentOrderedItemListOperations, currentItemCollectionItemsOperations: self.currentItemCollectionItemsOperations, currentItemCollectionInfosOperations: self.currentItemCollectionInfosOperations, currentUpdatedPeerChatStates: self.currentUpdatedPeerChatStates, updatedAccessChallengeData: self.currentUpdatedAccessChallengeData, currentGlobalTagsOperations: self.currentGlobalTagsOperations, updatedMedia: self.currentUpdatedMedia, replaceContactPeerIds: self.currentReplacedContactPeerIds, currentUpdatedMasterClientId: currentUpdatedMasterClientId) var updatedTransactionState: Int64? var updatedMasterClientId: Int64? if !transaction.isEmpty { @@ -1294,6 +1303,7 @@ public final class Postbox { self.currentReplaceChatListHoles.removeAll() self.currentUnsentOperations.removeAll() self.currentUpdatedSynchronizeReadStateOperations.removeAll() + self.currentGlobalTagsOperations.removeAll() self.currentUpdatedMedia.removeAll() self.currentReplacedContactPeerIds = nil self.currentUpdatedMasterClientId = nil @@ -1426,13 +1436,13 @@ public final class Postbox { if let indexEntry = self.messageHistoryIndexTable.get(id), let intermediateMessage = self.messageHistoryTable.getMessage(indexEntry.index) { let message = self.renderIntermediateMessage(intermediateMessage) if case let .update(updatedMessage) = update(message) { - self.messageHistoryTable.updateMessage(id, message: updatedMessage, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + self.messageHistoryTable.updateMessage(id, message: updatedMessage, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } } } fileprivate func offsetPendingMessagesTimestamps(lowerBound: MessageId, timestamp: Int32) { - self.messageHistoryTable.offsetPendingMessagesTimestamps(lowerBound: lowerBound, timestamp: timestamp, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations) + self.messageHistoryTable.offsetPendingMessagesTimestamps(lowerBound: lowerBound, timestamp: timestamp, operationsByPeerId: &self.currentOperationsByPeerId, unsentMessageOperations: &self.currentUnsentOperations, updatedPeerReadStateOperations: &self.currentUpdatedSynchronizeReadStateOperations, globalTagsOperations: &self.currentGlobalTagsOperations) } fileprivate func updateMedia(_ id: MediaId, update: Media?) { @@ -1499,6 +1509,32 @@ public final class Postbox { return self.itemCacheTable.retrieve(id: id, metaTable: self.itemCacheMetaTable) } + fileprivate func replaceGlobalMessageTagsHole(globalTags: GlobalMessageTags, index: MessageIndex, with updatedIndex: MessageIndex?, messages: [StoreMessage]) { + var allTagsMatch = true + for tag in globalTags { + self.globalMessageHistoryTagsTable.ensureInitialized(tag) + + if let entry = self.globalMessageHistoryTagsTable.get(tag, index: index), case .hole = entry { + + } else { + allTagsMatch = false + } + } + if allTagsMatch { + for tag in globalTags { + self.globalMessageHistoryTagsTable.remove(tag, index: index) + self.currentGlobalTagsOperations.append(.remove([(tag, index)])) + + if let updatedIndex = updatedIndex { + self.globalMessageHistoryTagsTable.addHole(tag, index: updatedIndex) + self.currentGlobalTagsOperations.append(.insertHole(tag, updatedIndex)) + } + } + + let _ = self.addMessages(messages, location: .Random) + } + } + public func modify(userInteractive: Bool = false, _ f: @escaping(Modifier) -> T) -> Signal { return Signal { subscriber in let f: () -> Void = { diff --git a/Postbox/PostboxTransaction.swift b/Postbox/PostboxTransaction.swift index fe70fbff78..0f9d178af3 100644 --- a/Postbox/PostboxTransaction.swift +++ b/Postbox/PostboxTransaction.swift @@ -21,6 +21,7 @@ final class PostboxTransaction { let currentItemCollectionInfosOperations: [ItemCollectionInfosOperation] let currentUpdatedPeerChatStates: Set let updatedAccessChallengeData: PostboxAccessChallengeData? + let currentGlobalTagsOperations: [GlobalMessageHistoryTagsOperation] let unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation] let updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?] @@ -104,10 +105,13 @@ final class PostboxTransaction { if self.updatedAccessChallengeData != nil { return false } + if !self.currentGlobalTagsOperations.isEmpty { + return false + } return true } - init(currentUpdatedState: Coding?, currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]], currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]], currentItemCollectionInfosOperations: [ItemCollectionInfosOperation], currentUpdatedPeerChatStates: Set, updatedAccessChallengeData: PostboxAccessChallengeData?, updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { + init(currentUpdatedState: Coding?, currentOperationsByPeerId: [PeerId: [MessageHistoryOperation]], peerIdsWithFilledHoles: [PeerId: [MessageIndex: HoleFillDirection]], removedHolesByPeerId: [PeerId: [MessageIndex: HoleFillDirection]], chatListOperations: [ChatListOperation], currentUpdatedPeers: [PeerId: Peer], currentUpdatedPeerNotificationSettings: [PeerId: PeerNotificationSettings], currentUpdatedCachedPeerData: [PeerId: CachedPeerData], currentUpdatedPeerPresences: [PeerId: PeerPresence], currentUpdatedPeerChatListEmbeddedStates: [PeerId: PeerChatListEmbeddedInterfaceState?], currentUpdatedTotalUnreadCount: Int32?, peerIdsWithUpdatedUnreadCounts: Set, currentPeerMergedOperationLogOperations: [PeerMergedOperationLogOperation], currentTimestampBasedMessageAttributesOperations: [TimestampBasedMessageAttributesOperation], unsentMessageOperations: [IntermediateMessageHistoryUnsentOperation], updatedSynchronizePeerReadStateOperations: [PeerId: PeerReadStateSynchronizationOperation?], currentPreferencesOperations: [PreferencesOperation], currentOrderedItemListOperations: [Int32: [OrderedItemListOperation]], currentItemCollectionItemsOperations: [ItemCollectionId: [ItemCollectionItemsOperation]], currentItemCollectionInfosOperations: [ItemCollectionInfosOperation], currentUpdatedPeerChatStates: Set, updatedAccessChallengeData: PostboxAccessChallengeData?, currentGlobalTagsOperations: [GlobalMessageHistoryTagsOperation], updatedMedia: [MediaId: Media?], replaceContactPeerIds: Set?, currentUpdatedMasterClientId: Int64?) { self.currentUpdatedState = currentUpdatedState self.currentOperationsByPeerId = currentOperationsByPeerId self.peerIdsWithFilledHoles = peerIdsWithFilledHoles @@ -130,6 +134,7 @@ final class PostboxTransaction { self.currentItemCollectionInfosOperations = currentItemCollectionInfosOperations self.currentUpdatedPeerChatStates = currentUpdatedPeerChatStates self.updatedAccessChallengeData = updatedAccessChallengeData + self.currentGlobalTagsOperations = currentGlobalTagsOperations self.updatedMedia = updatedMedia self.replaceContactPeerIds = replaceContactPeerIds self.currentUpdatedMasterClientId = currentUpdatedMasterClientId diff --git a/Postbox/SeedConfiguration.swift b/Postbox/SeedConfiguration.swift index 60845ed053..6ea76003e9 100644 --- a/Postbox/SeedConfiguration.swift +++ b/Postbox/SeedConfiguration.swift @@ -4,10 +4,12 @@ public final class SeedConfiguration { let initializeChatListWithHoles: [ChatListHole] let initializeMessageNamespacesWithHoles: [(PeerId.Namespace, MessageId.Namespace)] let existingMessageTags: MessageTags + let existingGlobalMessageTags: GlobalMessageTags - public init(initializeChatListWithHoles: [ChatListHole], initializeMessageNamespacesWithHoles: [(PeerId.Namespace, MessageId.Namespace)], existingMessageTags: MessageTags) { + public init(initializeChatListWithHoles: [ChatListHole], initializeMessageNamespacesWithHoles: [(PeerId.Namespace, MessageId.Namespace)], existingMessageTags: MessageTags, existingGlobalMessageTags: GlobalMessageTags) { self.initializeChatListWithHoles = initializeChatListWithHoles self.initializeMessageNamespacesWithHoles = initializeMessageNamespacesWithHoles self.existingMessageTags = existingMessageTags + self.existingGlobalMessageTags = existingGlobalMessageTags } } diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index fa6ef647f6..fac054801e 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -39,7 +39,7 @@ private struct SqlitePreparedStatement { sqlite3_clear_bindings(statement) } - func step(_ initial:Bool = false, path:String? = nil) -> Bool { + func step(_ initial: Bool = false, path: String? = nil) -> Bool { let result = sqlite3_step(statement) if result != SQLITE_ROW && result != SQLITE_DONE { if initial { @@ -49,6 +49,7 @@ private struct SqlitePreparedStatement { } } else { assertionFailure("Sqlite error \(result)") + return false } } return result == SQLITE_ROW @@ -79,7 +80,7 @@ private struct SqlitePreparedStatement { func int64KeyAt(_ index: Int) -> ValueBoxKey { let value = sqlite3_column_int64(statement, Int32(index)) - let key = ValueBoxKey(length: 9) + let key = ValueBoxKey(length: 8) key.setInt64(0, value: value) return key } diff --git a/Postbox/Views.swift b/Postbox/Views.swift index 43fa9950df..a8993553e9 100644 --- a/Postbox/Views.swift +++ b/Postbox/Views.swift @@ -8,6 +8,7 @@ public enum PostboxViewKey: Hashable { case orderedItemList(id: Int32) case accessChallengeData case preferences(keys: Set) + case globalMessageTags(globalTag: GlobalMessageTags, position: MessageIndex, count: Int, groupingPredicate: ((Message, Message) -> Bool)?) public var hashValue: Int { switch self { @@ -25,6 +26,8 @@ public enum PostboxViewKey: Hashable { return 2 case .preferences: return 3 + case .globalMessageTags: + return 4 } } @@ -72,6 +75,12 @@ public enum PostboxViewKey: Hashable { } else { return false } + case let .globalMessageTags(globalTag, position, count, _): + if case .globalMessageTags(globalTag, position, count, _) = rhs { + return true + } else { + return false + } } } } @@ -92,5 +101,7 @@ func postboxViewForKey(postbox: Postbox, key: PostboxViewKey) -> MutablePostboxV return MutableAccessChallengeDataView(postbox: postbox) case let .preferences(keys): return MutablePreferencesView(postbox: postbox, keys: keys) + case let .globalMessageTags(globalTag, position, count, groupingPredicate): + return MutableGlobalMessageTagsView(postbox: postbox, globalTag: globalTag, position: position, count: count, groupingPredicate: groupingPredicate) } } From 357ddd05d372a92719898bf96b273aa829cb86f8 Mon Sep 17 00:00:00 2001 From: overtake Date: Mon, 5 Jun 2017 14:03:41 +0300 Subject: [PATCH 142/427] - localisation and release version --- Postbox/Postbox.swift | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index b4ee5161d0..da9059817e 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -587,12 +587,35 @@ public enum PostboxResult { case postbox(Postbox) } +func debugSaveState(basePath:String, name: String) { + let path = basePath + name + let _ = try? FileManager.default.removeItem(atPath: path) + do { + try FileManager.default.copyItem(atPath: basePath, toPath: path) + } catch (let e) { + print("(Postbox debugSaveState: error \(e))") + } +} + +func debugRestoreState(basePath:String, name: String) { + let path = basePath + name + let _ = try? FileManager.default.removeItem(atPath: basePath) + do { + try FileManager.default.copyItem(atPath: path, toPath: basePath) + } catch (let e) { + print("(Postbox debugRestoreState: error \(e))") + } +} + public func openPostbox(basePath: String, globalMessageIdsNamespace: MessageId.Namespace, seedConfiguration: SeedConfiguration) -> Signal { let queue = Queue(name: "org.telegram.postbox.Postbox") return Signal { subscriber in queue.async { let _ = try? FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) + //debugSaveState(basePath: basePath, name: "beforeHoles") + //debugRestoreState(basePath: basePath, name: "beforeHoles") + loop: while true { let valueBox = SqliteValueBox(basePath: basePath + "/db", queue: queue) @@ -873,25 +896,7 @@ public final class Postbox { assert(true) } - private func debugSaveState(name: String) { - let path = self.basePath + name - let _ = try? FileManager.default.removeItem(atPath: path) - do { - try FileManager.default.copyItem(atPath: self.basePath, toPath: path) - } catch (let e) { - print("(Postbox debugSaveState: error \(e))") - } - } - - private func debugRestoreState(name: String) { - let path = self.basePath + name - let _ = try? FileManager.default.removeItem(atPath: self.basePath) - do { - try FileManager.default.copyItem(atPath: path, toPath: self.basePath) - } catch (let e) { - print("(Postbox debugRestoreState: error \(e))") - } - } + private func takeNextViewId() -> Int { let nextId = self.nextViewId From 10e3a2188b22e9033febb3609d0f3a54b89768a9 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 6 Jun 2017 12:56:45 +0300 Subject: [PATCH 143/427] no message --- Postbox.xcodeproj/project.pbxproj | 12 +++ .../xcschemes/PostboxMac.xcscheme | 2 +- .../xcschemes/PostboxTests.xcscheme | 2 +- Postbox/ChatListView.swift | 8 +- Postbox/MessageHistoryTable.swift | 8 +- Postbox/Peer.swift | 2 +- Postbox/PeerNameIndexTable.swift | 2 +- Postbox/PeerTable.swift | 33 ++++++- Postbox/PeerView.swift | 74 +++++++++----- Postbox/Postbox.swift | 13 ++- Postbox/PostboxUpgrade_13to14.swift | 41 ++++++++ Postbox/ReverseAssociatedPeerTable.swift | 99 +++++++++++++++++++ Postbox/Upgrades.swift | 1 + Postbox/ViewTracker.swift | 34 +------ Postbox/Views.swift | 11 +++ 15 files changed, 263 insertions(+), 79 deletions(-) create mode 100644 Postbox/PostboxUpgrade_13to14.swift create mode 100644 Postbox/ReverseAssociatedPeerTable.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 5b633cafd8..d918e00b64 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -174,6 +174,10 @@ D0BEAF701E54BC1E00BD963D /* AccountRecordsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF6F1E54BC1E00BD963D /* AccountRecordsView.swift */; }; D0BEAF711E54BC1E00BD963D /* AccountRecordsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0BEAF6F1E54BC1E00BD963D /* AccountRecordsView.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; + D0C0B5AB1EE1AB08000F4D2C /* ReverseAssociatedPeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C0B5AA1EE1AB08000F4D2C /* ReverseAssociatedPeerTable.swift */; }; + D0C0B5AC1EE1AB08000F4D2C /* ReverseAssociatedPeerTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C0B5AA1EE1AB08000F4D2C /* ReverseAssociatedPeerTable.swift */; }; + D0C0B5AE1EE1B3F4000F4D2C /* PostboxUpgrade_13to14.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C0B5AD1EE1B3F4000F4D2C /* PostboxUpgrade_13to14.swift */; }; + D0C0B5AF1EE1B3F4000F4D2C /* PostboxUpgrade_13to14.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C0B5AD1EE1B3F4000F4D2C /* PostboxUpgrade_13to14.swift */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; D0C674CC1CBB14A700183765 /* PeerReadState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674CB1CBB14A700183765 /* PeerReadState.swift */; }; D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */; }; @@ -376,6 +380,8 @@ D0BEAF6C1E54B77900BD963D /* AccountManagerRecordTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountManagerRecordTable.swift; sourceTree = ""; }; D0BEAF6F1E54BC1E00BD963D /* AccountRecordsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountRecordsView.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + D0C0B5AA1EE1AB08000F4D2C /* ReverseAssociatedPeerTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ReverseAssociatedPeerTable.swift; sourceTree = ""; }; + D0C0B5AD1EE1B3F4000F4D2C /* PostboxUpgrade_13to14.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxUpgrade_13to14.swift; sourceTree = ""; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; D0C674CB1CBB14A700183765 /* PeerReadState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerReadState.swift; sourceTree = ""; }; D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerChatStateTable.swift; sourceTree = ""; }; @@ -536,6 +542,7 @@ children = ( D0F02CDE1E99223D0065DEE2 /* Upgrades.swift */, D0F02CE11E9922F50065DEE2 /* PostboxUpgrade_12to13.swift */, + D0C0B5AD1EE1B3F4000F4D2C /* PostboxUpgrade_13to14.swift */, ); name = Upgrade; sourceTree = ""; @@ -577,6 +584,7 @@ D0F3CC711DDE1CDC008148FA /* ItemCacheTable.swift */, D07827C21E008F7300071108 /* ReverseIndexReferenceTable.swift */, D07827C41E00B23F00071108 /* PeerNameIndexTable.swift */, + D0C0B5AA1EE1AB08000F4D2C /* ReverseAssociatedPeerTable.swift */, D0AD23261E194D1C00A7089A /* PeerOperationLogTable.swift */, D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */, D0AD23281E196B6400A7089A /* PeerOperationLogMetadataTable.swift */, @@ -978,6 +986,7 @@ D0BEAF641E54B2FA00BD963D /* AccountManager.swift in Sources */, D073CE7F1DCBF3B4007511FD /* ItemCollection.swift in Sources */, D0F7B1D91E045C6A007EB8A5 /* OrderStatisticTable.swift in Sources */, + D0C0B5AF1EE1B3F4000F4D2C /* PostboxUpgrade_13to14.swift in Sources */, D073CEA01DCBF3C1007511FD /* ItemCollectionsView.swift in Sources */, D0BE383A1E7C1FD4000079AF /* ItemCollectionInfoView.swift in Sources */, D0B418491D7DFE20004562A4 /* ChatListView.swift in Sources */, @@ -1027,6 +1036,7 @@ D0B418311D7DFE16004562A4 /* ValueBox.swift in Sources */, D08775041E3E3E7400A97350 /* PreferencesEntry.swift in Sources */, D0B4184E1D7DFE20004562A4 /* MessageHistoryHolesView.swift in Sources */, + D0C0B5AC1EE1AB08000F4D2C /* ReverseAssociatedPeerTable.swift in Sources */, D0B844051DAB91B5005F29E1 /* MediaBox.swift in Sources */, D0B418321D7DFE16004562A4 /* SqliteValueBox.swift in Sources */, D073CE741DCBF3B4007511FD /* Peer.swift in Sources */, @@ -1105,6 +1115,7 @@ D0BEAF631E54B2FA00BD963D /* AccountManager.swift in Sources */, D0AD23271E194D1C00A7089A /* PeerOperationLogTable.swift in Sources */, D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */, + D0C0B5AE1EE1B3F4000F4D2C /* PostboxUpgrade_13to14.swift in Sources */, D0D510F91D63BCC200A97B8A /* IntermediateMessage.swift in Sources */, D0BE38391E7C1FD4000079AF /* ItemCollectionInfoView.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, @@ -1154,6 +1165,7 @@ D03120FC1DA55427006A2A60 /* PeerNotificationSettings.swift in Sources */, D0F7AB321DCFAB18009AD9A1 /* PeerChatTopIndexableMessageIds.swift in Sources */, D03BCCF81C73561C0097A291 /* Table.swift in Sources */, + D0C0B5AB1EE1AB08000F4D2C /* ReverseAssociatedPeerTable.swift in Sources */, D021E0DC1DB5237C00C6B04F /* ItemCollectionsView.swift in Sources */, D0C735281C864DF300BB3149 /* PeerChatStateTable.swift in Sources */, D0AAD1B31E3266B100D5B9DE /* TimestampBasedMessageAttributesTable.swift in Sources */, diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme index f4a9378c19..c2494365aa 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxMac.xcscheme @@ -1,6 +1,6 @@ () if let peer = getPeer(index.messageIndex.id.peerId) { peers[peer.id] = peer - if let associatedPeerIds = peer.associatedPeerIds { - for associatedPeerId in associatedPeerIds { - if let associatedPeer = getPeer(associatedPeerId) { - peers[associatedPeer.id] = associatedPeer - } + if let associatedPeerId = peer.associatedPeerId { + if let associatedPeer = getPeer(associatedPeerId) { + peers[associatedPeer.id] = associatedPeer } } } diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index d502a97efe..549d49fca3 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -1625,11 +1625,9 @@ final class MessageHistoryTable: Table { if let chatPeer = peerTable.get(message.id.peerId) { peers[chatPeer.id] = chatPeer - if let associatedPeerIds = chatPeer.associatedPeerIds { - for peerId in associatedPeerIds { - if let peer = peerTable.get(peerId) { - peers[peer.id] = peer - } + if let associatedPeerId = chatPeer.associatedPeerId { + if let peer = peerTable.get(associatedPeerId) { + peers[peer.id] = peer } } } diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index b2100dfd5d..3e8f22f758 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -95,7 +95,7 @@ public func <(lhs: PeerId, rhs: PeerId) -> Bool { public protocol Peer: class, Coding { var id: PeerId { get } var indexName: PeerIndexNameRepresentation { get } - var associatedPeerIds: [PeerId]? { get } + var associatedPeerId: PeerId? { get } var notificationSettingsPeerId: PeerId? { get } func isEqual(_ other: Peer) -> Bool diff --git a/Postbox/PeerNameIndexTable.swift b/Postbox/PeerNameIndexTable.swift index 865f5ceac7..8baaceebfa 100644 --- a/Postbox/PeerNameIndexTable.swift +++ b/Postbox/PeerNameIndexTable.swift @@ -257,7 +257,7 @@ final class PeerNameIndexTable: Table { } } - func matchingPeerIds(tokens: (regular: [ValueBoxKey], transliterated: [ValueBoxKey]?), categories: PeerNameIndexCategories, chatListIndexTable: ChatListIndexTable, contactTable: ContactTable) -> (chats: [PeerId], contacts: [PeerId]) { + func matchingPeerIds(tokens: (regular: [ValueBoxKey], transliterated: [ValueBoxKey]?), categories: PeerNameIndexCategories, chatListIndexTable: ChatListIndexTable, contactTable: ContactTable, reverseAssociatedPeerTable: ReverseAssociatedPeerTable) -> (chats: [PeerId], contacts: [PeerId]) { if categories.isEmpty { return ([], []) } else { diff --git a/Postbox/PeerTable.swift b/Postbox/PeerTable.swift index 5439cd664a..fb628d307f 100644 --- a/Postbox/PeerTable.swift +++ b/Postbox/PeerTable.swift @@ -5,11 +5,19 @@ final class PeerTable: Table { return ValueBoxTable(id: id, keyType: .int64) } + private let reverseAssociatedTable: ReverseAssociatedPeerTable + private let sharedEncoder = Encoder() private let sharedKey = ValueBoxKey(length: 8) private var cachedPeers: [PeerId: Peer] = [:] - private var updatedPeerIds = Set() + private var updatedInitialPeers: [PeerId: Peer?] = [:] + + init(valueBox: ValueBox, table: ValueBoxTable, reverseAssociatedTable: ReverseAssociatedPeerTable) { + self.reverseAssociatedTable = reverseAssociatedTable + + super.init(valueBox: valueBox, table: table) + } private func key(_ id: PeerId) -> ValueBoxKey { self.sharedKey.setInt64(0, value: id.toInt64()) @@ -17,8 +25,11 @@ final class PeerTable: Table { } func set(_ peer: Peer) { + let previous = self.get(peer.id) self.cachedPeers[peer.id] = peer - self.updatedPeerIds.insert(peer.id) + if self.updatedInitialPeers[peer.id] == nil { + self.updatedInitialPeers[peer.id] = previous + } } func get(_ id: PeerId) -> Peer? { @@ -36,19 +47,31 @@ final class PeerTable: Table { override func clearMemoryCache() { self.cachedPeers.removeAll() - self.updatedPeerIds.removeAll() + assert(self.updatedInitialPeers.isEmpty) } override func beforeCommit() { - for peerId in self.updatedPeerIds { + for (peerId, previousPeer) in self.updatedInitialPeers { if let peer = self.cachedPeers[peerId] { self.sharedEncoder.reset() self.sharedEncoder.encodeRootObject(peer) self.valueBox.set(self.table, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy()) + + let previousAssociation = previousPeer?.associatedPeerId + if previousAssociation != peer.associatedPeerId { + if let previousAssociation = previousAssociation { + self.reverseAssociatedTable.removeReverseAssociation(target: previousAssociation, from: peerId) + } + if let associatedPeerId = peer.associatedPeerId { + self.reverseAssociatedTable.addReverseAssociation(target: associatedPeerId, from: peerId) + } + } + } else { + assertionFailure() } } - self.updatedPeerIds.removeAll() + self.updatedInitialPeers.removeAll() } } diff --git a/Postbox/PeerView.swift b/Postbox/PeerView.swift index 494f39270c..25d5a185c1 100644 --- a/Postbox/PeerView.swift +++ b/Postbox/PeerView.swift @@ -1,6 +1,6 @@ import Foundation -final class MutablePeerView { +final class MutablePeerView: MutablePostboxView { let peerId: PeerId var notificationSettings: PeerNotificationSettings? var cachedData: CachedPeerData? @@ -8,17 +8,27 @@ final class MutablePeerView { var peerPresences: [PeerId: PeerPresence] = [:] var peerIsContact: Bool - init(peerId: PeerId, notificationSettings: PeerNotificationSettings?, cachedData: CachedPeerData?, peerIsContact: Bool, getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) { + init(postbox: Postbox, peerId: PeerId) { + let notificationSettings = postbox.peerNotificationSettingsTable.get(peerId) + let cachedData = postbox.cachedPeerDataTable.get(peerId) + let peerIsContact = postbox.contactsTable.isContact(peerId: peerId) + + let getPeer: (PeerId) -> Peer? = { peerId in + return postbox.peerTable.get(peerId) + } + + let getPeerPresence: (PeerId) -> PeerPresence? = { peerId in + return postbox.peerPresenceTable.get(peerId) + } + self.peerId = peerId self.notificationSettings = notificationSettings self.cachedData = cachedData self.peerIsContact = peerIsContact var peerIds = Set() peerIds.insert(peerId) - if let peer = getPeer(peerId), let associatedPeerIds = peer.associatedPeerIds { - for peerId in associatedPeerIds { - peerIds.insert(peerId) - } + if let peer = getPeer(peerId), let associatedPeerId = peer.associatedPeerId { + peerIds.insert(associatedPeerId) } if let cachedData = cachedData { peerIds.formUnion(cachedData.peerIds) @@ -31,19 +41,35 @@ final class MutablePeerView { self.peerPresences[id] = presence } } - if let peer = self.peers[peerId], let associatedPeerIds = peer.associatedPeerIds { - for id in associatedPeerIds { - if let peer = getPeer(id) { - self.peers[id] = peer - } - if let presence = getPeerPresence(id) { - self.peerPresences[id] = presence - } + if let peer = self.peers[peerId], let associatedPeerId = peer.associatedPeerId { + if let peer = getPeer(associatedPeerId) { + self.peers[associatedPeerId] = peer + } + if let presence = getPeerPresence(associatedPeerId) { + self.peerPresences[associatedPeerId] = presence } } } - func replay(updatedPeers: [PeerId: Peer], updatedNotificationSettings: [PeerId: PeerNotificationSettings], updatedCachedPeerData: [PeerId: CachedPeerData], updatedPeerPresences: [PeerId: PeerPresence], replaceContactPeerIds: Set?, getPeer: (PeerId) -> Peer?, getPeerPresence: (PeerId) -> PeerPresence?) -> Bool { + func reset(postbox: Postbox) -> Bool { + return false + } + + func replay(postbox: Postbox, transaction: PostboxTransaction) -> Bool { + let updatedPeers = transaction.currentUpdatedPeers + let updatedNotificationSettings = transaction.currentUpdatedPeerNotificationSettings + let updatedCachedPeerData = transaction.currentUpdatedCachedPeerData + let updatedPeerPresences = transaction.currentUpdatedPeerPresences + let replaceContactPeerIds = transaction.replaceContactPeerIds + + let getPeer: (PeerId) -> Peer? = { peerId in + return postbox.peerTable.get(peerId) + } + + let getPeerPresence: (PeerId) -> PeerPresence? = { peerId in + return postbox.peerPresenceTable.get(peerId) + } + var updated = false if let cachedData = updatedCachedPeerData[self.peerId], self.cachedData == nil || !self.cachedData!.isEqual(to: cachedData) { @@ -52,10 +78,8 @@ final class MutablePeerView { var peerIds = Set() peerIds.insert(self.peerId) - if let peer = getPeer(self.peerId), let associatedPeerIds = peer.associatedPeerIds { - for peerId in associatedPeerIds { - peerIds.insert(peerId) - } + if let peer = getPeer(self.peerId), let associatedPeerId = peer.associatedPeerId { + peerIds.insert(associatedPeerId) } peerIds.formUnion(cachedData.peerIds) @@ -97,10 +121,8 @@ final class MutablePeerView { } else { var peerIds = Set() peerIds.insert(self.peerId) - if let peer = getPeer(self.peerId), let associatedPeerIds = peer.associatedPeerIds { - for peerId in associatedPeerIds { - peerIds.insert(peerId) - } + if let peer = getPeer(self.peerId), let associatedPeerId = peer.associatedPeerId { + peerIds.insert(associatedPeerId) } if let cachedData = self.cachedData { peerIds.formUnion(cachedData.peerIds) @@ -139,9 +161,13 @@ final class MutablePeerView { return updated } + + func immutableView() -> PostboxView { + return PeerView(self) + } } -public final class PeerView { +public final class PeerView: PostboxView { public let peerId: PeerId public let cachedData: CachedPeerData? public let notificationSettings: PeerNotificationSettings? diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index b4ee5161d0..052021f3e7 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -716,6 +716,7 @@ public final class Postbox { let itemCacheTable: ItemCacheTable let peerNameTokenIndexTable: ReverseIndexReferenceTable let peerNameIndexTable: PeerNameIndexTable + let reverseAssociatedPeerTable: ReverseAssociatedPeerTable let peerChatTopTaggedMessageIdsTable: PeerChatTopTaggedMessageIdsTable let peerOperationLogMetadataTable: PeerOperationLogMetadataTable let peerMergedOperationLogIndexTable: PeerMergedOperationLogIndexTable @@ -759,7 +760,8 @@ public final class Postbox { self.metadataTable = MetadataTable(valueBox: self.valueBox, table: MetadataTable.tableSpec(0)) self.keychainTable = KeychainTable(valueBox: self.valueBox, table: KeychainTable.tableSpec(1)) - self.peerTable = PeerTable(valueBox: self.valueBox, table: PeerTable.tableSpec(2)) + self.reverseAssociatedPeerTable = ReverseAssociatedPeerTable(valueBox: self.valueBox, table:ReverseAssociatedPeerTable.tableSpec(40)) + self.peerTable = PeerTable(valueBox: self.valueBox, table: PeerTable.tableSpec(2), reverseAssociatedTable: self.reverseAssociatedPeerTable) self.globalMessageIdsTable = GlobalMessageIdsTable(valueBox: self.valueBox, table: GlobalMessageIdsTable.tableSpec(3), namespace: self.globalMessageIdsNamespace) self.globallyUniqueMessageIdsTable = MessageGloballyUniqueIdTable(valueBox: self.valueBox, table: MessageGloballyUniqueIdTable.tableSpec(32)) self.messageHistoryMetadataTable = MessageHistoryMetadataTable(valueBox: self.valueBox, table: MessageHistoryMetadataTable.tableSpec(10)) @@ -827,6 +829,7 @@ public final class Postbox { tables.append(self.itemCacheMetaTable) tables.append(self.itemCacheTable) tables.append(self.peerNameIndexTable) + tables.append(self.reverseAssociatedPeerTable) tables.append(self.peerNameTokenIndexTable) tables.append(self.peerChatTopTaggedMessageIdsTable) tables.append(self.peerOperationLogMetadataTable) @@ -1252,7 +1255,7 @@ public final class Postbox { for table in self.tables { table.clearMemoryCache() } - self.viewTracker.refreshViewsDueToExternalTransaction(fetchAroundChatEntries: self.fetchAroundChatEntries, fetchAroundHistoryEntries: self.fetchAroundHistoryEntries, fetchUnsentMessageIds: { + self.viewTracker.refreshViewsDueToExternalTransaction(postbox: self, fetchAroundChatEntries: self.fetchAroundChatEntries, fetchAroundHistoryEntries: self.fetchAroundHistoryEntries, fetchUnsentMessageIds: { return self.messageHistoryUnsentTable.get() }, fetchSynchronizePeerReadStateOperations: { return self.synchronizeReadStateTable.get(getCombinedPeerReadState: { peerId in @@ -1776,7 +1779,7 @@ public final class Postbox { public func searchContacts(query: String) -> Signal<[Peer], NoError> { return self.modify { modifier -> Signal<[Peer], NoError> in - let (_, contactPeerIds) = self.peerNameIndexTable.matchingPeerIds(tokens: (regular: stringIndexTokens(query, transliteration: .none), transliterated: stringIndexTokens(query, transliteration: .transliterated)), categories: [.contacts], chatListIndexTable: self.chatListIndexTable, contactTable: self.contactsTable) + let (_, contactPeerIds) = self.peerNameIndexTable.matchingPeerIds(tokens: (regular: stringIndexTokens(query, transliteration: .none), transliterated: stringIndexTokens(query, transliteration: .transliterated)), categories: [.contacts], chatListIndexTable: self.chatListIndexTable, contactTable: self.contactsTable, reverseAssociatedPeerTable: self.reverseAssociatedPeerTable) var contactPeers: [Peer] = [] for peerId in contactPeerIds { @@ -1795,7 +1798,7 @@ public final class Postbox { var peerIds = Set() var chatPeers: [Peer] = [] - let (chatPeerIds, contactPeerIds) = self.peerNameIndexTable.matchingPeerIds(tokens: (regular: stringIndexTokens(query, transliteration: .none), transliterated: stringIndexTokens(query, transliteration: .transliterated)), categories: [.chats, .contacts], chatListIndexTable: self.chatListIndexTable, contactTable: self.contactsTable) + let (chatPeerIds, contactPeerIds) = self.peerNameIndexTable.matchingPeerIds(tokens: (regular: stringIndexTokens(query, transliteration: .none), transliterated: stringIndexTokens(query, transliteration: .transliterated)), categories: [.chats, .contacts], chatListIndexTable: self.chatListIndexTable, contactTable: self.contactsTable, reverseAssociatedPeerTable: self.reverseAssociatedPeerTable) for peerId in chatPeerIds { if let peer = self.peerTable.get(peerId) { @@ -1820,7 +1823,7 @@ public final class Postbox { public func peerView(id: PeerId) -> Signal { return self.modify { modifier -> Signal in - let view = MutablePeerView(peerId: id, notificationSettings: self.peerNotificationSettingsTable.get(id), cachedData: self.cachedPeerDataTable.get(id), peerIsContact: self.contactsTable.isContact(peerId: id), getPeer: { self.peerTable.get($0) }, getPeerPresence: { self.peerPresenceTable.get($0) }) + let view = MutablePeerView(postbox: self, peerId: id) let (index, signal) = self.viewTracker.addPeerView(view) return (.single(PeerView(view)) diff --git a/Postbox/PostboxUpgrade_13to14.swift b/Postbox/PostboxUpgrade_13to14.swift new file mode 100644 index 0000000000..2517910a57 --- /dev/null +++ b/Postbox/PostboxUpgrade_13to14.swift @@ -0,0 +1,41 @@ +import Foundation + +private func writePeerIds(_ buffer: WriteBuffer, _ peerIds: Set) { + for id in peerIds { + var value: Int64 = id.toInt64() + buffer.write(&value, offset: 0, length: 8) + } +} + +func postboxUpgrade_13to14(metadataTable: MetadataTable, valueBox: ValueBox) { + var reverseAssociations: [PeerId: Set] = [:] + + let peerTable = ValueBoxTable(id: 2, keyType: .int64) + valueBox.scan(peerTable, values: { _, value in + if let peer = Decoder(buffer: value).decodeRootObject() as? Peer { + if let association = peer.associatedPeerId { + if reverseAssociations[association] == nil { + reverseAssociations[association] = Set() + } + reverseAssociations[association]!.insert(peer.id) + } + } else { + assertionFailure() + } + return true + }) + + let reverseAssociatedPeerTable = ValueBoxTable(id: 40, keyType: .int64) + + let sharedKey = ValueBoxKey(length: 8) + let sharedBuffer = WriteBuffer() + for (peerId, associations) in reverseAssociations { + sharedBuffer.reset() + writePeerIds(sharedBuffer, associations) + + sharedKey.setInt64(0, value: peerId.toInt64()) + valueBox.set(reverseAssociatedPeerTable, key: sharedKey, value: sharedBuffer) + } + + metadataTable.setUserVersion(14) +} diff --git a/Postbox/ReverseAssociatedPeerTable.swift b/Postbox/ReverseAssociatedPeerTable.swift new file mode 100644 index 0000000000..b2ca5870cd --- /dev/null +++ b/Postbox/ReverseAssociatedPeerTable.swift @@ -0,0 +1,99 @@ +import Foundation + +private func readPeerIds(_ buffer: ReadBuffer) -> Set { + assert(buffer.length % 8 == 0) + let count = buffer.length / 8 + var result = Set() + for _ in 0 ..< count { + var value: Int64 = 0 + buffer.read(&value, offset: 0, length: 8) + result.insert(PeerId(value)) + } + return result +} + +private func writePeerIds(_ buffer: WriteBuffer, _ peerIds: Set) { + for id in peerIds { + var value: Int64 = id.toInt64() + buffer.write(&value, offset: 0, length: 8) + } +} + +final class ReverseAssociatedPeerTable: Table { + static func tableSpec(_ id: Int32) -> ValueBoxTable { + return ValueBoxTable(id: id, keyType: .int64) + } + + private let sharedKey = ValueBoxKey(length: 8) + + private var cachedAssociations: [PeerId: Set] = [:] + private var updatedAssociations = Set() + + private func key(_ peerId: PeerId) -> ValueBoxKey { + self.sharedKey.setInt64(0, value: peerId.toInt64()) + return self.sharedKey + } + + func get(peerId: PeerId) -> Set { + if let cached = self.cachedAssociations[peerId] { + return cached + } else { + if let value = self.valueBox.get(self.table, key: self.key(peerId)) { + let peerIds = readPeerIds(value) + self.cachedAssociations[peerId] = peerIds + return peerIds + } else { + self.cachedAssociations[peerId] = Set() + return Set() + } + } + } + + func addReverseAssociation(target targetPeerId: PeerId, from sourcePeerId: PeerId) { + var value = self.get(peerId: targetPeerId) + if value.contains(sourcePeerId) { + assertionFailure() + } else { + value.insert(sourcePeerId) + self.cachedAssociations[targetPeerId] = value + self.updatedAssociations.insert(targetPeerId) + } + } + + func removeReverseAssociation(target targetPeerId: PeerId, from sourcePeerId: PeerId) { + var value = self.get(peerId: targetPeerId) + if !value.contains(sourcePeerId) { + assertionFailure() + } else { + value.remove(sourcePeerId) + self.cachedAssociations[targetPeerId] = value + self.updatedAssociations.insert(targetPeerId) + } + } + + override func clearMemoryCache() { + self.cachedAssociations.removeAll() + assert(self.updatedAssociations.isEmpty) + } + + override func beforeCommit() { + if !self.updatedAssociations.isEmpty { + let buffer = WriteBuffer() + for peerId in self.updatedAssociations { + if let peerIds = self.cachedAssociations[peerId] { + if peerIds.isEmpty { + self.valueBox.remove(self.table, key: self.key(peerId)) + } else { + buffer.reset() + writePeerIds(buffer, peerIds) + self.valueBox.set(self.table, key: self.key(peerId), value: buffer) + } + } else { + assertionFailure() + } + } + + self.updatedAssociations.removeAll() + } + } +} diff --git a/Postbox/Upgrades.swift b/Postbox/Upgrades.swift index 6c70a9d015..21e053a2d6 100644 --- a/Postbox/Upgrades.swift +++ b/Postbox/Upgrades.swift @@ -7,5 +7,6 @@ enum PostboxUpgradeOperation { func registeredUpgrades() -> [Int32: PostboxUpgradeOperation] { var dict: [Int32: PostboxUpgradeOperation] = [:] dict[12] = .inplace(postboxUpgrade_12to13) + dict[13] = .inplace(postboxUpgrade_13to14) return dict } diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index b5f4ca9d38..0ff4d25f89 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -289,7 +289,7 @@ final class ViewTracker { self.combinedViews.remove(index) } - func refreshViewsDueToExternalTransaction(fetchAroundChatEntries: (_ index: ChatListIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { + func refreshViewsDueToExternalTransaction(postbox: Postbox, fetchAroundChatEntries: (_ index: ChatListIndex, _ count: Int) -> (entries: [MutableChatListEntry], earlier: MutableChatListEntry?, later: MutableChatListEntry?), fetchAroundHistoryEntries: (_ index: MessageIndex, _ count: Int, _ tagMask: MessageTags?) -> (entries: [MutableMessageHistoryEntry], lower: MutableMessageHistoryEntry?, upper: MutableMessageHistoryEntry?), fetchUnsentMessageIds: () -> [MessageId], fetchSynchronizePeerReadStateOperations: () -> [PeerId: PeerReadStateSynchronizationOperation]) { var updateTrackedHolesPeerIds: [PeerId] = [] for (peerId, bag) in self.messageHistoryViews { @@ -327,35 +327,7 @@ final class ViewTracker { } for (mutableView, pipe) in self.peerViews.copyItems() { - var updatedPeers: [PeerId: Peer] = [:] - var updatedPeerPresences: [PeerId: PeerPresence] = [:] - var updatedNotificationSettings: [PeerId: PeerNotificationSettings] = [:] - var updatedCachedPeerData: [PeerId: CachedPeerData] = [:] - - let peerId = mutableView.peerId - - if let peer = self.getPeer(peerId) { - updatedPeers[peerId] = peer - } - if let presence = self.getPeerPresence(peerId) { - updatedPeerPresences[peerId] = presence - } - if let notificationSettings = self.getPeerNotificationSettings(peerId) { - updatedNotificationSettings[peerId] = notificationSettings - } - if let cachedPeerData = self.getCachedPeerData(peerId) { - updatedCachedPeerData[peerId] = cachedPeerData - for cachedPeerId in cachedPeerData.peerIds { - if let peer = self.getPeer(cachedPeerId) { - updatedPeers[cachedPeerId] = peer - } - if let presence = self.getPeerPresence(cachedPeerId) { - updatedPeerPresences[cachedPeerId] = presence - } - } - } - - if mutableView.replay(updatedPeers: updatedPeers, updatedNotificationSettings: updatedNotificationSettings, updatedCachedPeerData: updatedCachedPeerData, updatedPeerPresences: updatedPeerPresences, replaceContactPeerIds: nil, getPeer: self.getPeer, getPeerPresence: self.getPeerPresence) { + if mutableView.reset(postbox: postbox) { pipe.putNext(PeerView(mutableView)) } } @@ -483,7 +455,7 @@ final class ViewTracker { } for (mutableView, pipe) in self.peerViews.copyItems() { - if mutableView.replay(updatedPeers: transaction.currentUpdatedPeers, updatedNotificationSettings: transaction.currentUpdatedPeerNotificationSettings, updatedCachedPeerData: transaction.currentUpdatedCachedPeerData, updatedPeerPresences: transaction.currentUpdatedPeerPresences, replaceContactPeerIds: transaction.replaceContactPeerIds, getPeer: self.getPeer, getPeerPresence: self.getPeerPresence) { + if mutableView.replay(postbox: postbox, transaction: transaction) { pipe.putNext(PeerView(mutableView)) } } diff --git a/Postbox/Views.swift b/Postbox/Views.swift index a8993553e9..109c129844 100644 --- a/Postbox/Views.swift +++ b/Postbox/Views.swift @@ -9,6 +9,7 @@ public enum PostboxViewKey: Hashable { case accessChallengeData case preferences(keys: Set) case globalMessageTags(globalTag: GlobalMessageTags, position: MessageIndex, count: Int, groupingPredicate: ((Message, Message) -> Bool)?) + case peer(peerId: PeerId) public var hashValue: Int { switch self { @@ -28,6 +29,8 @@ public enum PostboxViewKey: Hashable { return 3 case .globalMessageTags: return 4 + case let .peer(peerId): + return peerId.hashValue } } @@ -81,6 +84,12 @@ public enum PostboxViewKey: Hashable { } else { return false } + case let .peer(peerId): + if case .peer(peerId) = rhs { + return true + } else { + return false + } } } } @@ -103,5 +112,7 @@ func postboxViewForKey(postbox: Postbox, key: PostboxViewKey) -> MutablePostboxV return MutablePreferencesView(postbox: postbox, keys: keys) case let .globalMessageTags(globalTag, position, count, groupingPredicate): return MutableGlobalMessageTagsView(postbox: postbox, globalTag: globalTag, position: position, count: count, groupingPredicate: groupingPredicate) + case let .peer(peerId): + return MutablePeerView(postbox: postbox, peerId: peerId) } } From cdef6a6b9962e49ca7558464502c0da49ae271eb Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 13 Jun 2017 22:22:21 +0300 Subject: [PATCH 144/427] no message --- Postbox/Message.swift | 18 ++++++++++++++++++ Postbox/MessageHistoryView.swift | 11 +++++++++-- Postbox/Postbox.swift | 2 ++ Postbox/ViewTracker.swift | 2 +- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 621c3c2935..76357a4a11 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -540,6 +540,20 @@ public final class StoreMessage { self.media = media } + public init(id: StoreMessageId, globallyUniqueId: Int64?, timestamp: Int32, flags: StoreMessageFlags, tags: MessageTags, globalTags: GlobalMessageTags, forwardInfo: StoreMessageForwardInfo?, authorId: PeerId?, text: String, attributes: [MessageAttribute], media: [Media]) { + self.id = id + self.timestamp = timestamp + self.globallyUniqueId = globallyUniqueId + self.flags = flags + self.tags = tags + self.globalTags = globalTags + self.forwardInfo = forwardInfo + self.authorId = authorId + self.text = text + self.attributes = attributes + self.media = media + } + public var index: MessageIndex? { if case let .Id(id) = self.id { return MessageIndex(id: id, timestamp: self.timestamp) @@ -547,6 +561,10 @@ public final class StoreMessage { return nil } } + + public func withUpdatedAttributes(_ attributes: [MessageAttribute]) -> StoreMessage { + return StoreMessage(id: self.id, globallyUniqueId: self.globallyUniqueId, timestamp: self.timestamp, flags: self.flags, tags: self.tags, globalTags: self.globalTags, forwardInfo: self.forwardInfo, authorId: self.authorId, text: self.text, attributes: attributes, media: self.media) + } } final class InternalStoreMessage { diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index a3d7563366..ae3efccc84 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -2,10 +2,12 @@ import Foundation public enum AdditionalMessageHistoryViewData { case cachedPeerData(PeerId) + case peerChatState(PeerId) } public enum AdditionalMessageHistoryViewDataEntry { case cachedPeerData(PeerId, CachedPeerData?) + case peerChatState(PeerId, PeerChatState?) } public struct MessageHistoryViewId: Equatable { @@ -521,7 +523,7 @@ final class MutableMessageHistoryView { return true } - func replay(_ operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], updatedCachedPeerData: [PeerId: CachedPeerData], context: MutableMessageHistoryViewReplayContext, renderIntermediateMessage: (IntermediateMessage) -> Message) -> Bool { + func replay(postbox: Postbox, transaction: PostboxTransaction, operations: [MessageHistoryOperation], holeFillDirections: [MessageIndex: HoleFillDirection], updatedMedia: [MediaId: Media?], updatedCachedPeerData: [PeerId: CachedPeerData], context: MutableMessageHistoryViewReplayContext, renderIntermediateMessage: (IntermediateMessage) -> Message) -> Bool { let tagMask = self.tagMask let unwrappedTagMask: UInt32 = tagMask?.rawValue ?? 0 @@ -681,6 +683,11 @@ final class MutableMessageHistoryView { self.additionalDatas[i] = .cachedPeerData(peerId, updatedData) hasChanges = true } + case let .peerChatState(peerId, _): + if transaction.currentUpdatedPeerChatStates.contains(peerId) { + self.additionalDatas[i] = .peerChatState(peerId, postbox.peerChatStateTable.get(peerId) as? PeerChatState) + hasChanges = true + } } } @@ -1049,7 +1056,7 @@ public final class MessageHistoryView { if !entries.isEmpty { var referenceIndex = entries.count - 1 for i in 0 ..< entries.count { - if entries[i].index >= self.anchorIndex { + if entries[i].index > self.anchorIndex { referenceIndex = i break } diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 052021f3e7..769b40d3de 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -1623,6 +1623,8 @@ public final class Postbox { switch data { case let .cachedPeerData(peerId): additionalDataEntries.append(.cachedPeerData(peerId, self.cachedPeerDataTable.get(peerId))) + case let .peerChatState(peerId): + additionalDataEntries.append(.peerChatState(peerId, self.peerChatStateTable.get(peerId) as? PeerChatState)) } } diff --git a/Postbox/ViewTracker.swift b/Postbox/ViewTracker.swift index 0ff4d25f89..7edb0680f9 100644 --- a/Postbox/ViewTracker.swift +++ b/Postbox/ViewTracker.swift @@ -360,7 +360,7 @@ final class ViewTracker { updateType = .Generic } - if mutableView.replay(operations ?? [], holeFillDirections: transaction.peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: transaction.updatedMedia, updatedCachedPeerData: transaction.currentUpdatedCachedPeerData, context: context, renderIntermediateMessage: self.renderMessage) { + if mutableView.replay(postbox: postbox, transaction: transaction, operations: operations ?? [], holeFillDirections: transaction.peerIdsWithFilledHoles[peerId] ?? [:], updatedMedia: transaction.updatedMedia, updatedCachedPeerData: transaction.currentUpdatedCachedPeerData, context: context, renderIntermediateMessage: self.renderMessage) { mutableView.complete(context: context, fetchEarlier: { index, count in return self.fetchEarlierHistoryEntries(peerId, index, count, mutableView.tagMask) }, fetchLater: { index, count in From b96742a0d40b6a11dd591e3d858750ee26108a78 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 16 Jun 2017 12:15:42 +0300 Subject: [PATCH 145/427] no message --- Postbox.xcodeproj/project.pbxproj | 20 ++++--- Postbox/ChatListIndexTable.swift | 69 +++++++++++++++++++---- Postbox/Database.swift | 8 ++- Postbox/PeerNameIndexRepresentation.swift | 40 +++++++++++-- Postbox/PeerNameIndexTable.swift | 10 +++- Postbox/Postbox.swift | 44 ++++++++++++--- Postbox/SqliteValueBox.swift | 6 +- 7 files changed, 158 insertions(+), 39 deletions(-) diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index d918e00b64..6a95e318d4 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -1267,10 +1267,11 @@ IPHONEOS_DEPLOYMENT_TARGET = 8.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; OTHER_CFLAGS = ( - "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_ENABLE_FTS3", + "-DSQLITE_OMIT_DEPRECATED", + "-DSQLITE_ENABLE_FTS5", + "-DSQLITE_DEFAULT_MEMSTATUS=0", ); PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1394,10 +1395,11 @@ IPHONEOS_DEPLOYMENT_TARGET = 8.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; OTHER_CFLAGS = ( - "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_ENABLE_FTS3", + "-DSQLITE_OMIT_DEPRECATED", + "-DSQLITE_ENABLE_FTS5", + "-DSQLITE_DEFAULT_MEMSTATUS=0", ); PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1642,10 +1644,11 @@ IPHONEOS_DEPLOYMENT_TARGET = 8.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; OTHER_CFLAGS = ( - "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_ENABLE_FTS3", + "-DSQLITE_OMIT_DEPRECATED", + "-DSQLITE_ENABLE_FTS5", + "-DSQLITE_DEFAULT_MEMSTATUS=0", ); OTHER_SWIFT_FLAGS = "-DDEBUG"; PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; @@ -1675,10 +1678,11 @@ IPHONEOS_DEPLOYMENT_TARGET = 8.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; OTHER_CFLAGS = ( - "-DSQLITE_TEMP_STORE=2", "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_ENABLE_FTS3", + "-DSQLITE_OMIT_DEPRECATED", + "-DSQLITE_ENABLE_FTS5", + "-DSQLITE_DEFAULT_MEMSTATUS=0", ); PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/Postbox/ChatListIndexTable.swift b/Postbox/ChatListIndexTable.swift index 83495c2d5e..57ce1ea40e 100644 --- a/Postbox/ChatListIndexTable.swift +++ b/Postbox/ChatListIndexTable.swift @@ -3,6 +3,7 @@ import Foundation struct ChatListInclusionIndex { let topMessageIndex: MessageIndex? let inclusion: PeerChatListInclusion + var tags: [UInt16]? func includedIndex(peerId: PeerId) -> ChatListIndex? { switch inclusion { @@ -32,6 +33,17 @@ struct ChatListInclusionIndex { } } +private struct ChatListIndexFlags: OptionSet { + var rawValue: Int8 + + init(rawValue: Int8) { + self.rawValue = rawValue + } + + static let hasIndex = ChatListIndexFlags(rawValue: 1 << 0) + static let hasTags = ChatListIndexFlags(rawValue: 1 << 1) +} + final class ChatListIndexTable: Table { static func tableSpec(_ id: Int32) -> ValueBoxTable { return ValueBoxTable(id: id, keyType: .int64) @@ -66,7 +78,7 @@ final class ChatListIndexTable: Table { if self.updatedPreviousCachedIndices[peerId] == nil { self.updatedPreviousCachedIndices[peerId] = current } - let updated = ChatListInclusionIndex(topMessageIndex: index, inclusion: current.inclusion) + let updated = ChatListInclusionIndex(topMessageIndex: index, inclusion: current.inclusion, tags: current.tags) self.cachedIndices[peerId] = updated return updated } @@ -76,7 +88,7 @@ final class ChatListIndexTable: Table { if self.updatedPreviousCachedIndices[peerId] == nil { self.updatedPreviousCachedIndices[peerId] = current } - let updated = ChatListInclusionIndex(topMessageIndex: current.topMessageIndex, inclusion: inclusion) + let updated = ChatListInclusionIndex(topMessageIndex: current.topMessageIndex, inclusion: inclusion, tags: current.tags) self.cachedIndices[peerId] = updated return updated } @@ -88,9 +100,11 @@ final class ChatListIndexTable: Table { if let value = self.valueBox.get(self.table, key: self.key(peerId)) { let topMessageIndex: MessageIndex? - var hasIndex: Int8 = 0 - value.read(&hasIndex, offset: 0, length: 1) - if hasIndex != 0 { + var flagsValue: Int8 = 0 + value.read(&flagsValue, offset: 0, length: 1) + let flags = ChatListIndexFlags(rawValue: flagsValue) + + if flags.contains(.hasIndex) { var idNamespace: Int32 = 0 var idId: Int32 = 0 var idTimestamp: Int32 = 0 @@ -131,11 +145,24 @@ final class ChatListIndexTable: Table { preconditionFailure() } - let inclusionIndex = ChatListInclusionIndex(topMessageIndex: topMessageIndex, inclusion: inclusion) + var tags: [UInt16]? + if flags.contains(.hasTags) { + var count: UInt16 = 0 + value.read(&count, offset: 0, length: 2) + var resultTags: [UInt16] = [] + for _ in 0 ..< Int(count) { + var tag: UInt16 = 0 + value.read(&tag, offset: 0, length: 2) + resultTags.append(tag) + } + tags = resultTags + } + + let inclusionIndex = ChatListInclusionIndex(topMessageIndex: topMessageIndex, inclusion: inclusion, tags: tags) self.cachedIndices[peerId] = inclusionIndex return inclusionIndex } else { - return ChatListInclusionIndex(topMessageIndex: nil, inclusion: .notSpecified) + return ChatListInclusionIndex(topMessageIndex: nil, inclusion: .notSpecified, tags: nil) } } } @@ -162,18 +189,26 @@ final class ChatListIndexTable: Table { let writeBuffer = WriteBuffer() + var flags: ChatListIndexFlags = [] + + if index.topMessageIndex != nil { + flags.insert(.hasIndex) + } + + if index.tags != nil { + flags.insert(.hasTags) + } + + var flagsValue = flags.rawValue + writeBuffer.write(&flagsValue, offset: 0, length: 1) + if let topMessageIndex = index.topMessageIndex { - var hasIndex: Int8 = 1 - writeBuffer.write(&hasIndex, offset: 0, length: 1) var idNamespace: Int32 = topMessageIndex.id.namespace var idId: Int32 = topMessageIndex.id.id var idTimestamp: Int32 = topMessageIndex.timestamp writeBuffer.write(&idNamespace, offset: 0, length: 4) writeBuffer.write(&idId, offset: 0, length: 4) writeBuffer.write(&idTimestamp, offset: 0, length: 4) - } else { - var hasIndex: Int8 = 0 - writeBuffer.write(&hasIndex, offset: 0, length: 1) } switch index.inclusion { @@ -205,6 +240,16 @@ final class ChatListIndexTable: Table { } } + if let tags = index.tags { + var count = UInt16(tags.count) + writeBuffer.write(&count, offset: 0, length: 2) + + for tag in tags { + var tagValue: UInt16 = tag + writeBuffer.write(&tagValue, offset: 0, length: 2) + } + } + withExtendedLifetime(writeBuffer, { self.valueBox.set(self.table, key: self.key(peerId), value: writeBuffer.readBufferNoCopy()) }) diff --git a/Postbox/Database.swift b/Postbox/Database.swift index 1712061b6d..fb557e8c4e 100644 --- a/Postbox/Database.swift +++ b/Postbox/Database.swift @@ -106,7 +106,13 @@ public final class Database { /// /// - parameter SQL: A batch of zero or more semicolon-separated SQL statements. public func execute(_ SQL: String) -> Bool { - return sqlite3_exec(self.handle, SQL, nil, nil, nil) == SQLITE_OK + let res = sqlite3_exec(self.handle, SQL, nil, nil, nil) + if res == SQLITE_OK { + return true + } else { + print("SQL error \(res) on SQL") + return false + } } } diff --git a/Postbox/PeerNameIndexRepresentation.swift b/Postbox/PeerNameIndexRepresentation.swift index fd6182f0d7..1974c29db3 100644 --- a/Postbox/PeerNameIndexRepresentation.swift +++ b/Postbox/PeerNameIndexRepresentation.swift @@ -2,7 +2,7 @@ import Foundation public enum PeerIndexNameRepresentation: Equatable { case title(title: String, addressName: String?) - case personName(first: String, last: String, addressName: String?) + case personName(first: String, last: String, addressName: String?, phoneNumber: String?) public static func ==(lhs: PeerIndexNameRepresentation, rhs: PeerIndexNameRepresentation) -> Bool { switch lhs { @@ -12,14 +12,41 @@ public enum PeerIndexNameRepresentation: Equatable { } else { return false } - case let .personName(lhsFirst, lhsLast, lhsAddressName): - if case let .personName(rhsFirst, rhsLast, rhsAddressName) = rhs, lhsFirst == rhsFirst, lhsLast == rhsLast, lhsAddressName == rhsAddressName { + case let .personName(lhsFirst, lhsLast, lhsAddressName, lhsPhoneNumber): + if case let .personName(rhsFirst, rhsLast, rhsAddressName, rhsPhoneNumber) = rhs, lhsFirst == rhsFirst, lhsLast == rhsLast, lhsAddressName == rhsAddressName, lhsPhoneNumber == rhsPhoneNumber { return true } else { return false } } } + + public var isEmpty: Bool { + switch self { + case let .title(title, addressName): + if !title.isEmpty { + return false + } + if let addressName = addressName, !addressName.isEmpty { + return false + } + return true + case let .personName(first, last, addressName, phoneNumber): + if !first.isEmpty { + return false + } + if !last.isEmpty { + return false + } + if let addressName = addressName, !addressName.isEmpty { + return false + } + if let phoneNumber = phoneNumber, !phoneNumber.isEmpty { + return false + } + return true + } + } } public enum PeerNameIndex { @@ -32,7 +59,7 @@ extension PeerIndexNameRepresentation { switch self { case let .title(title, _): return title - case let .personName(first, last, _): + case let .personName(first, last, _, _): switch index { case .firstNameFirst: return first + last @@ -68,12 +95,15 @@ extension PeerIndexNameRepresentation { tokens.append(contentsOf: stringIndexTokens(addressName, transliteration: .none)) } return tokens - case let .personName(first, last, addressName): + case let .personName(first, last, addressName, phoneNumber): var tokens: [ValueBoxKey] = stringIndexTokens(first, transliteration: .combined) tokens.append(contentsOf: stringIndexTokens(last, transliteration: .combined)) if let addressName = addressName { tokens.append(contentsOf: stringIndexTokens(addressName, transliteration: .none)) } + if let phoneNumber = phoneNumber { + tokens.append(contentsOf: stringIndexTokens(phoneNumber, transliteration: .none)) + } return tokens } } diff --git a/Postbox/PeerNameIndexTable.swift b/Postbox/PeerNameIndexTable.swift index 8baaceebfa..69a1feef4c 100644 --- a/Postbox/PeerNameIndexTable.swift +++ b/Postbox/PeerNameIndexTable.swift @@ -213,7 +213,15 @@ final class PeerNameIndexTable: Table { updatedTokens = updatedName.indexTokens } else { if let peer = self.peerTable.get(peerId) { - updatedTokens = peer.indexName.indexTokens + if let associatedPeerId = peer.associatedPeerId { + if let associatedPeer = self.peerTable.get(associatedPeerId) { + updatedTokens = associatedPeer.indexName.indexTokens + } else { + updatedTokens = [] + } + } else { + updatedTokens = peer.indexName.indexTokens + } } else { //assertionFailure() updatedTokens = [] diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 769b40d3de..09e00aaf9d 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -1351,8 +1351,23 @@ public final class Postbox { if let updatedPeer = update(currentPeer, peer) { self.peerTable.set(updatedPeer) self.currentUpdatedPeers[updatedPeer.id] = updatedPeer - if currentPeer?.indexName != updatedPeer.indexName { - self.peerNameIndexTable.markPeerNameUpdated(peerId: peer.id, name: updatedPeer.indexName) + var previousIndexNameWasEmpty = true + + if let currentPeer = currentPeer { + if !currentPeer.indexName.isEmpty { + previousIndexNameWasEmpty = false + } + } + + let indexNameIsEmpty = updatedPeer.indexName.isEmpty + + if !previousIndexNameWasEmpty || !indexNameIsEmpty { + if currentPeer?.indexName != updatedPeer.indexName { + self.peerNameIndexTable.markPeerNameUpdated(peerId: peer.id, name: updatedPeer.indexName) + for reverseAssociatedPeerId in self.reverseAssociatedPeerTable.get(peerId: peer.id) { + self.peerNameIndexTable.markPeerNameUpdated(peerId: reverseAssociatedPeerId, name: updatedPeer.indexName) + } + } } } } @@ -1795,30 +1810,41 @@ public final class Postbox { } |> switchToLatest } - public func searchPeers(query: String) -> Signal<[Peer], NoError> { - return self.modify { modifier -> Signal<[Peer], NoError> in + public func searchPeers(query: String) -> Signal<[RenderedPeer], NoError> { + return self.modify { modifier -> Signal<[RenderedPeer], NoError> in var peerIds = Set() - var chatPeers: [Peer] = [] + var chatPeers: [RenderedPeer] = [] let (chatPeerIds, contactPeerIds) = self.peerNameIndexTable.matchingPeerIds(tokens: (regular: stringIndexTokens(query, transliteration: .none), transliterated: stringIndexTokens(query, transliteration: .transliterated)), categories: [.chats, .contacts], chatListIndexTable: self.chatListIndexTable, contactTable: self.contactsTable, reverseAssociatedPeerTable: self.reverseAssociatedPeerTable) for peerId in chatPeerIds { if let peer = self.peerTable.get(peerId) { - chatPeers.append(peer) + var peers = SimpleDictionary() + peers[peer.id] = peer + if let associatedPeerId = peer.associatedPeerId { + if let associatedPeer = self.peerTable.get(associatedPeerId) { + peers[associatedPeer.id] = associatedPeer + } + } + chatPeers.append(RenderedPeer(peerId: peer.id, peers: peers)) peerIds.insert(peerId) } } - var contactPeers: [Peer] = [] + var contactPeers: [RenderedPeer] = [] for peerId in contactPeerIds { if !peerIds.contains(peerId) { if let peer = self.peerTable.get(peerId) { - contactPeers.append(peer) + var peers = SimpleDictionary() + peers[peer.id] = peer + contactPeers.append(RenderedPeer(peerId: peer.id, peers: peers)) } } } - contactPeers.sort(by: { $0.indexName.indexName(.lastNameFirst) < $1.indexName.indexName(.lastNameFirst) }) + contactPeers.sort(by: { lhs, rhs in + lhs.peers[lhs.peerId]!.indexName.indexName(.lastNameFirst) < rhs.peers[rhs.peerId]!.indexName.indexName(.lastNameFirst) + }) return .single(chatPeers + contactPeers) } |> switchToLatest } diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index fac054801e..4551e289d2 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -149,7 +149,7 @@ final class SqliteValueBox: ValueBox { //database = Database(path)! } - sqlite3_busy_timeout(database.handle, 10000000) + sqlite3_busy_timeout(database.handle, 1000 * 10000) var resultCode: Bool @@ -158,9 +158,9 @@ final class SqliteValueBox: ValueBox { assert(resultCode) resultCode = database.execute("PRAGMA synchronous=NORMAL") assert(resultCode) - //database.execute("PRAGMA temp_store=MEMORY") - resultCode = database.execute("PRAGMA wal_autocheckpoint=500") + resultCode = database.execute("PRAGMA temp_store=MEMORY") assert(resultCode) + //resultCode = database.execute("PRAGMA wal_autocheckpoint=500") //database.execute("PRAGMA journal_size_limit=1536") /*var statement: OpaquePointer? = nil From d1daff6b1e3ef287819ecdd114436d9f94155c43 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 18 Jun 2017 16:26:27 +0300 Subject: [PATCH 146/427] no message --- Postbox/MessageHistoryTable.swift | 2 +- Postbox/Postbox.swift | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index 549d49fca3..db2547e53f 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -1962,7 +1962,7 @@ final class MessageHistoryTable: Table { return MessageHistoryAnchorIndex(index: MessageIndex(id: messageId, timestamp: 1), exact: false) } - func findMessageId(peerId: PeerId, namespace: MessageId.Namespace, timestamp: Int32) -> MessageId? { + func findMessageId(peerId: PeerId, timestamp: Int32) -> MessageId? { var result: MessageId? self.valueBox.range(self.table, start: self.key(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: timestamp)), end: self.key(MessageIndex(id: MessageId(peerId: peerId, namespace: Int32.max, id: Int32.max), timestamp: timestamp)), values: { key, value in let entry = self.readIntermediateEntry(key, value: value) diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 09e00aaf9d..b2432ab105 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -420,6 +420,11 @@ public final class Modifier { return nil } + public func findClosestMessageIdByTimestamp(peerId: PeerId, timestamp: Int32) -> MessageId? { + assert(!self.disposed) + return self.postbox?.messageHistoryTable.findMessageId(peerId: peerId, timestamp: timestamp) + } + public func filterStoredMessageIds(_ messageIds: Set) -> Set { assert(!self.disposed) if let postbox = self.postbox { @@ -1516,7 +1521,11 @@ public final class Postbox { } fileprivate func storedMessageId(peerId: PeerId, namespace: MessageId.Namespace, timestamp: Int32) -> MessageId? { - return self.messageHistoryTable.findMessageId(peerId: peerId, namespace: namespace, timestamp: timestamp) + if let id = self.messageHistoryTable.findMessageId(peerId: peerId, timestamp: timestamp), id.namespace == namespace { + return id + } else { + return nil + } } fileprivate func putItemCacheEntry(id: ItemCacheEntryId, entry: Coding, collectionSpec: ItemCacheCollectionSpec) { From 4a004016c00ac3860032014e64b90d84dcbb1662 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 19 Jun 2017 20:36:07 +0300 Subject: [PATCH 147/427] no message --- Postbox.xcodeproj/project.pbxproj | 10 +- Postbox/Database.swift | 6 +- Postbox/MessageHistoryTable.swift | 42 +- Postbox/MessageHistoryTextIndexTable.swift | 99 + Postbox/Postbox.swift | 32 +- Postbox/PostboxPrivate/sqlcipher/sqlite3.c | 88936 ++++++++++++---- Postbox/PostboxPrivate/sqlcipher/sqlite3.h | 3896 +- Postbox/PostboxPrivate/sqlcipher/sqlite3ext.h | 93 +- Postbox/SeedConfiguration.swift | 4 +- Postbox/SqliteValueBox.swift | 362 +- Postbox/ValueBox.swift | 7 + PostboxTests/ChatListTableTests.swift | 24 +- PostboxTests/CodingTests.swift | 20 +- .../MessageHistoryIndexTableTests.swift | 10 +- PostboxTests/MessageHistoryTableTests.swift | 178 +- PostboxTests/OrderStatisticTreeTests.swift | 8 +- PostboxTests/ReadStateTableTests.swift | 20 +- 17 files changed, 74094 insertions(+), 19653 deletions(-) create mode 100644 Postbox/MessageHistoryTextIndexTable.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 6a95e318d4..d815eda34c 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -26,6 +26,8 @@ D019B1CF1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */; }; D01BAA551ED1D70C00295217 /* ManagedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01BAA541ED1D70C00295217 /* ManagedFile.swift */; }; D01BAA561ED1D70C00295217 /* ManagedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01BAA541ED1D70C00295217 /* ManagedFile.swift */; }; + D01C7EDE1EF73F71008305F1 /* MessageHistoryTextIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01C7EDD1EF73F71008305F1 /* MessageHistoryTextIndexTable.swift */; }; + D01C7EDF1EF73F71008305F1 /* MessageHistoryTextIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01C7EDD1EF73F71008305F1 /* MessageHistoryTextIndexTable.swift */; }; D01F7D9B1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */; }; D01F7D9D1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */; }; D021E0D41DB4FAE100C6B04F /* ItemCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */; }; @@ -301,6 +303,7 @@ D010B6191E1E463900C3E282 /* PeerMergedOperationLogIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerMergedOperationLogIndexTable.swift; sourceTree = ""; }; D019B1CE1E2E770700F80DB3 /* MessageGloballyUniqueIdTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageGloballyUniqueIdTable.swift; sourceTree = ""; }; D01BAA541ED1D70C00295217 /* ManagedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ManagedFile.swift; sourceTree = ""; }; + D01C7EDD1EF73F71008305F1 /* MessageHistoryTextIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTextIndexTable.swift; sourceTree = ""; }; D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryInvalidatedReadStateTable.swift; sourceTree = ""; }; D01F7D9C1CBF8586008765C9 /* SynchronizePeerReadStatesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SynchronizePeerReadStatesView.swift; sourceTree = ""; }; D021E0D31DB4FAE100C6B04F /* ItemCollection.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ItemCollection.swift; sourceTree = ""; }; @@ -558,6 +561,7 @@ D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, D0D510F51D63BBE100A97B8A /* MessageHistoryOperation.swift */, D00EED1D1C81F28D00341DFF /* MessageHistoryTagsTable.swift */, + D01C7EDD1EF73F71008305F1 /* MessageHistoryTextIndexTable.swift */, D0E1D30B1ECA1F5500FCEEF1 /* GlobalMessageHistoryTagsTable.swift */, D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */, D0F9E8641C58CB7F00037222 /* ChatListTable.swift */, @@ -1048,6 +1052,7 @@ D0B418301D7DFE16004562A4 /* ValueBoxKey.swift in Sources */, D0B418221D7DFE0C004562A4 /* Coding.swift in Sources */, D0B4184B1D7DFE20004562A4 /* SynchronizePeerReadStatesView.swift in Sources */, + D01C7EDF1EF73F71008305F1 /* MessageHistoryTextIndexTable.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1177,6 +1182,7 @@ D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */, D07516441B2D9CEF00AE42E0 /* sqlite3.c in Sources */, D0D511001D64A58900A97B8A /* IpcPipe.swift in Sources */, + D01C7EDE1EF73F71008305F1 /* MessageHistoryTextIndexTable.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1269,7 +1275,6 @@ OTHER_CFLAGS = ( "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_OMIT_DEPRECATED", "-DSQLITE_ENABLE_FTS5", "-DSQLITE_DEFAULT_MEMSTATUS=0", ); @@ -1397,7 +1402,6 @@ OTHER_CFLAGS = ( "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_OMIT_DEPRECATED", "-DSQLITE_ENABLE_FTS5", "-DSQLITE_DEFAULT_MEMSTATUS=0", ); @@ -1646,7 +1650,6 @@ OTHER_CFLAGS = ( "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_OMIT_DEPRECATED", "-DSQLITE_ENABLE_FTS5", "-DSQLITE_DEFAULT_MEMSTATUS=0", ); @@ -1680,7 +1683,6 @@ OTHER_CFLAGS = ( "-DSQLITE_HAS_CODEC=1", "-DSQLCIPHER_CRYPTO_CC=1", - "-DSQLITE_OMIT_DEPRECATED", "-DSQLITE_ENABLE_FTS5", "-DSQLITE_DEFAULT_MEMSTATUS=0", ); diff --git a/Postbox/Database.swift b/Postbox/Database.swift index fb557e8c4e..c2bf43ad81 100644 --- a/Postbox/Database.swift +++ b/Postbox/Database.swift @@ -110,7 +110,11 @@ public final class Database { if res == SQLITE_OK { return true } else { - print("SQL error \(res) on SQL") + if let error = sqlite3_errmsg(self.handle), let str = NSString(utf8String: error) { + print("SQL error \(res): \(str) on SQL") + } else { + print("SQL error \(res) on SQL") + } return false } } diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift index db2547e53f..943de8d34f 100644 --- a/Postbox/MessageHistoryTable.swift +++ b/Postbox/MessageHistoryTable.swift @@ -62,8 +62,9 @@ final class MessageHistoryTable: Table { let globalTagsTable: GlobalMessageHistoryTagsTable let readStateTable: MessageHistoryReadStateTable let synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable + let textIndexTable: MessageHistoryTextIndexTable - init(valueBox: ValueBox, table: ValueBoxTable, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, globalTagsTable: GlobalMessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable) { + init(valueBox: ValueBox, table: ValueBoxTable, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable, historyMetadataTable: MessageHistoryMetadataTable, globallyUniqueMessageIdsTable: MessageGloballyUniqueIdTable, unsentTable: MessageHistoryUnsentTable, tagsTable: MessageHistoryTagsTable, globalTagsTable: GlobalMessageHistoryTagsTable, readStateTable: MessageHistoryReadStateTable, synchronizeReadStateTable: MessageHistorySynchronizeReadStateTable, textIndexTable: MessageHistoryTextIndexTable) { self.messageHistoryIndexTable = messageHistoryIndexTable self.messageMediaTable = messageMediaTable self.historyMetadataTable = historyMetadataTable @@ -73,6 +74,7 @@ final class MessageHistoryTable: Table { self.globalTagsTable = globalTagsTable self.readStateTable = readStateTable self.synchronizeReadStateTable = synchronizeReadStateTable + self.textIndexTable = textIndexTable super.init(valueBox: valueBox, table: table) } @@ -643,6 +645,10 @@ final class MessageHistoryTable: Table { sharedBuffer.write(&globalTagsValue, offset: 0, length: 4) } + if self.messageHistoryIndexTable.seedConfiguration.peerNamespacesRequiringMessageTextIndex.contains(message.id.peerId.namespace) { + self.textIndexTable.add(messageId: message.id, text: message.text, tags: message.tags) + } + var flags = MessageFlags(message.flags) sharedBuffer.write(&flags.rawValue, offset: 0, length: 4) @@ -853,6 +859,10 @@ final class MessageHistoryTable: Table { let _ = self.messageMediaTable.removeReference(mediaId) } + if self.messageHistoryIndexTable.seedConfiguration.peerNamespacesRequiringMessageTextIndex.contains(message.id.peerId.namespace) { + self.textIndexTable.remove(messageId: message.id) + } + resultTags = message.tags resultGlobalTags = message.globalTags case let .Hole(hole): @@ -1077,6 +1087,13 @@ final class MessageHistoryTable: Table { break } + if self.messageHistoryIndexTable.seedConfiguration.peerNamespacesRequiringMessageTextIndex.contains(message.id.peerId.namespace) { + if previousMessage.id != message.id || previousMessage.text != message.text || previousMessage.tags != message.tags { + self.textIndexTable.remove(messageId: previousMessage.id) + self.textIndexTable.add(messageId: message.id, text: message.text, tags: message.tags) + } + } + sharedBuffer.reset() var type: Int8 = 0 @@ -1975,6 +1992,29 @@ final class MessageHistoryTable: Table { return result } + func findClosestMessageId(peerId: PeerId, timestamp: Int32) -> MessageId? { + var result: MessageId? + self.valueBox.range(self.table, start: self.key(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: timestamp)), end: self.lowerBound(peerId), values: { key, value in + let entry = self.readIntermediateEntry(key, value: value) + if case .Message = entry { + result = entry.index.id + return false + } + return true + }, limit: 0) + if result == nil { + self.valueBox.range(self.table, start: self.key(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: timestamp)), end: self.upperBound(peerId), values: { key, value in + let entry = self.readIntermediateEntry(key, value: value) + if case .Message = entry { + result = entry.index.id + return false + } + return true + }, limit: 0) + } + return result + } + func incomingMessageStatsInIndices(_ peerId: PeerId, namespace: MessageId.Namespace, indices: [MessageIndex]) -> (Int, Bool) { var count: Int = 0 var holes = false diff --git a/Postbox/MessageHistoryTextIndexTable.swift b/Postbox/MessageHistoryTextIndexTable.swift new file mode 100644 index 0000000000..29049ea9cc --- /dev/null +++ b/Postbox/MessageHistoryTextIndexTable.swift @@ -0,0 +1,99 @@ +import Foundation + +private func collectionId(_ peerId: PeerId) -> String { + return "p\(UInt64(bitPattern: peerId.toInt64()))" +} + +private func itemId(_ messageId: MessageId) -> String { + return "p\(UInt64(bitPattern: messageId.peerId.toInt64()))a\(UInt32(bitPattern: messageId.namespace))b\(UInt32(bitPattern: messageId.id))" +} + +private func messageTags(_ tags: MessageTags) -> String { + var result = "" + for tag in tags { + if !result.isEmpty { + result += " " + } + result += "t\(tag.rawValue)" + } + return result +} + +private func parseMessageId(_ value: String) -> MessageId? { + if !value.hasPrefix("p") { + return nil + } + guard let aRange = value.range(of: "a") else { + return nil + } + guard let bRange = value.range(of: "b") else { + return nil + } + let pString = value.substring(with: value.index(value.startIndex, offsetBy: 1) ..< aRange.lowerBound) + let nString = value.substring(with: aRange.upperBound ..< bRange.lowerBound) + let iString = value.substring(with: bRange.upperBound ..< value.endIndex) + + guard let pValue = UInt64(pString) else { + return nil + } + guard let nValue = UInt32(nString) else { + return nil + } + guard let iValue = UInt32(iString) else { + return nil + } + + return MessageId(peerId: PeerId(Int64(bitPattern: pValue)), namespace: Int32(bitPattern: nValue), id: Int32(bitPattern: iValue)) +} + +private let alphanumerics = CharacterSet.alphanumerics + +final class MessageHistoryTextIndexTable { + static func tableSpec(_ id: Int32) -> ValueBoxFullTextTable { + return ValueBoxFullTextTable(id: id) + } + + private let valueBox: ValueBox + private let table: ValueBoxFullTextTable + + init(valueBox: ValueBox, table: ValueBoxFullTextTable) { + self.valueBox = valueBox + self.table = table + } + + func add(messageId: MessageId, text: String, tags: MessageTags) { + self.valueBox.fullTextSet(self.table, collectionId: collectionId(messageId.peerId), itemId: itemId(messageId), contents: text, tags: messageTags(tags)) + } + + func remove(messageId: MessageId) { + self.valueBox.fullTextRemove(self.table, itemId: itemId(messageId)) + } + + func search(peerId: PeerId?, text: String, tags: MessageTags?) -> [MessageId] { + var escapedText = String(text.characters.map({ c in + var codeUnits: [UnicodeScalar] = [] + for codeUnit in String(c).unicodeScalars { + codeUnits.append(codeUnit) + } + for codeUnit in codeUnits { + if !alphanumerics.contains(codeUnit) { + return " " + } + } + return c + })) + if !escapedText.isEmpty { + escapedText += "*" + } + var result: [MessageId] = [] + self.valueBox.fullTextMatch(self.table, collectionId: peerId.flatMap { collectionId($0) }, query: escapedText, tags: tags.flatMap(messageTags), values: { _, itemId in + if let messageId = parseMessageId(itemId) { + result.append(messageId) + } else { + assertionFailure() + } + return true + }) + return result + } +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index b2432ab105..c27a1e4938 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -420,11 +420,16 @@ public final class Modifier { return nil } - public func findClosestMessageIdByTimestamp(peerId: PeerId, timestamp: Int32) -> MessageId? { + public func findMessageIdByTimestamp(peerId: PeerId, timestamp: Int32) -> MessageId? { assert(!self.disposed) return self.postbox?.messageHistoryTable.findMessageId(peerId: peerId, timestamp: timestamp) } + public func findClosestMessageIdByTimestamp(peerId: PeerId, timestamp: Int32) -> MessageId? { + assert(!self.disposed) + return self.postbox?.messageHistoryTable.findClosestMessageId(peerId: peerId, timestamp: timestamp) + } + public func filterStoredMessageIds(_ messageIds: Set) -> Set { assert(!self.disposed) if let postbox = self.postbox { @@ -566,6 +571,15 @@ public final class Modifier { assert(!self.disposed) self.postbox?.replaceGlobalMessageTagsHole(globalTags: globalTags, index: index, with: updatedIndex, messages: messages) } + + public func searchMessages(peerId: PeerId?, query: String, tags: MessageTags?) -> [Message] { + assert(!self.disposed) + if let postbox = self.postbox { + return postbox.searchMessages(peerId: peerId, query: query, tags: tags) + } else { + return [] + } + } } fileprivate class PipeNotifier: NSObject { @@ -731,6 +745,7 @@ public final class Postbox { let preferencesTable: PreferencesTable let orderedItemListTable: OrderedItemListTable let orderedItemListIndexTable: OrderedItemListIndexTable + let textIndexTable: MessageHistoryTextIndexTable //temporary let peerRatingTable: RatingTable @@ -779,7 +794,8 @@ public final class Postbox { self.synchronizeReadStateTable = MessageHistorySynchronizeReadStateTable(valueBox: self.valueBox, table: MessageHistorySynchronizeReadStateTable.tableSpec(15)) self.timestampBasedMessageAttributesIndexTable = TimestampBasedMessageAttributesIndexTable(valueBox: self.valueBox, table: TimestampBasedMessageAttributesTable.tableSpec(33)) self.timestampBasedMessageAttributesTable = TimestampBasedMessageAttributesTable(valueBox: self.valueBox, table: TimestampBasedMessageAttributesTable.tableSpec(34), indexTable: self.timestampBasedMessageAttributesIndexTable) - self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable, unsentTable: self.messageHistoryUnsentTable, tagsTable: self.messageHistoryTagsTable, globalTagsTable: self.globalMessageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable) + self.textIndexTable = MessageHistoryTextIndexTable(valueBox: self.valueBox, table: MessageHistoryTextIndexTable.tableSpec(41)) + self.messageHistoryTable = MessageHistoryTable(valueBox: self.valueBox, table: MessageHistoryTable.tableSpec(7), messageHistoryIndexTable: self.messageHistoryIndexTable, messageMediaTable: self.mediaTable, historyMetadataTable: self.messageHistoryMetadataTable, globallyUniqueMessageIdsTable: self.globallyUniqueMessageIdsTable, unsentTable: self.messageHistoryUnsentTable, tagsTable: self.messageHistoryTagsTable, globalTagsTable: self.globalMessageHistoryTagsTable, readStateTable: self.readStateTable, synchronizeReadStateTable: self.synchronizeReadStateTable, textIndexTable: self.textIndexTable) self.peerChatStateTable = PeerChatStateTable(valueBox: self.valueBox, table: PeerChatStateTable.tableSpec(13)) self.peerNameTokenIndexTable = ReverseIndexReferenceTable(valueBox: self.valueBox, table: ReverseIndexReferenceTable.tableSpec(26)) self.peerNameIndexTable = PeerNameIndexTable(valueBox: self.valueBox, table: PeerNameIndexTable.tableSpec(27), peerTable: self.peerTable, peerNameTokenIndexTable: self.peerNameTokenIndexTable) @@ -1562,6 +1578,18 @@ public final class Postbox { } } + fileprivate func searchMessages(peerId: PeerId?, query: String, tags: MessageTags?) -> [Message] { + var result: [Message] = [] + for messageId in self.textIndexTable.search(peerId: peerId, text: query, tags: tags) { + if let indexEntry = self.messageHistoryIndexTable.get(messageId), case let .Message(index) = indexEntry, let message = self.messageHistoryTable.getMessage(index) { + result.append(self.messageHistoryTable.renderMessage(message, peerTable: self.peerTable)) + } else { + assertionFailure() + } + } + return result + } + public func modify(userInteractive: Bool = false, _ f: @escaping(Modifier) -> T) -> Signal { return Signal { subscriber in let f: () -> Void = { diff --git a/Postbox/PostboxPrivate/sqlcipher/sqlite3.c b/Postbox/PostboxPrivate/sqlcipher/sqlite3.c index 0052bfcdb8..59d03f428e 100644 --- a/Postbox/PostboxPrivate/sqlcipher/sqlite3.c +++ b/Postbox/PostboxPrivate/sqlcipher/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.8.6. By combining all the individual C code files into this +** version 3.15.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -9,7 +9,7 @@ ** ** This file is all you need to compile SQLite. To use SQLite in other ** programs, you need this file and the "sqlite3.h" header file that defines -** the programming interface to the SQLite library. (If you do not have +** the programming interface to the SQLite library. (If you do not have ** the "sqlite3.h" header file at hand, you will find a copy embedded within ** the text of this file. Search for "Begin file sqlite3.h" to find the start ** of the embedded sqlite3.h header file.) Additional code files may be needed @@ -17,14 +17,14 @@ ** language. The code for the "sqlite3" command-line shell is also in a ** separate file. This file contains only code for the core SQLite library. */ +#define NDEBUG +#define assert(e) ((void)0) + #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 #ifndef SQLITE_PRIVATE # define SQLITE_PRIVATE static #endif -#ifndef SQLITE_API -# define SQLITE_API -#endif /************** Begin file sqliteInt.h ***************************************/ /* ** 2001 September 15 @@ -40,8 +40,140 @@ ** Internal interface definitions for SQLite. ** */ -#ifndef _SQLITEINT_H_ -#define _SQLITEINT_H_ +#ifndef SQLITEINT_H +#define SQLITEINT_H + +/* Special Comments: +** +** Some comments have special meaning to the tools that measure test +** coverage: +** +** NO_TEST - The branches on this line are not +** measured by branch coverage. This is +** used on lines of code that actually +** implement parts of coverage testing. +** +** OPTIMIZATION-IF-TRUE - This branch is allowed to alway be false +** and the correct answer is still obtained, +** though perhaps more slowly. +** +** OPTIMIZATION-IF-FALSE - This branch is allowed to alway be true +** and the correct answer is still obtained, +** though perhaps more slowly. +** +** PREVENTS-HARMLESS-OVERREAD - This branch prevents a buffer overread +** that would be harmless and undetectable +** if it did occur. +** +** In all cases, the special comment must be enclosed in the usual +** slash-asterisk...asterisk-slash comment marks, with no spaces between the +** asterisks and the comment text. +*/ + +/* +** Make sure the Tcl calling convention macro is defined. This macro is +** only used by test code and Tcl integration code. +*/ +#ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +#endif + +/* +** Make sure that rand_s() is available on Windows systems with MSVC 2005 +** or higher. +*/ +#if defined(_MSC_VER) && _MSC_VER>=1400 +# define _CRT_RAND_S +#endif + +/* +** Include the header file used to customize the compiler options for MSVC. +** This should be done first so that it can successfully prevent spurious +** compiler warnings due to subsequent content in this file and other files +** that are included by this file. +*/ +/************** Include msvc.h in the middle of sqliteInt.h ******************/ +/************** Begin file msvc.h ********************************************/ +/* +** 2015 January 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to MSVC. +*/ +#ifndef SQLITE_MSVC_H +#define SQLITE_MSVC_H + +#if defined(_MSC_VER) +#pragma warning(disable : 4054) +#pragma warning(disable : 4055) +#pragma warning(disable : 4100) +#pragma warning(disable : 4127) +#pragma warning(disable : 4130) +#pragma warning(disable : 4152) +#pragma warning(disable : 4189) +#pragma warning(disable : 4206) +#pragma warning(disable : 4210) +#pragma warning(disable : 4232) +#pragma warning(disable : 4244) +#pragma warning(disable : 4305) +#pragma warning(disable : 4306) +#pragma warning(disable : 4702) +#pragma warning(disable : 4706) +#endif /* defined(_MSC_VER) */ + +#endif /* SQLITE_MSVC_H */ + +/************** End of msvc.h ************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* +** Special setup for VxWorks +*/ +/************** Include vxworks.h in the middle of sqliteInt.h ***************/ +/************** Begin file vxworks.h *****************************************/ +/* +** 2015-03-02 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Wind River's VxWorks +*/ +#if defined(__RTP__) || defined(_WRS_KERNEL) +/* This is VxWorks. Set up things specially for that OS +*/ +#include +#include /* amalgamator: dontcache */ +#define OS_VXWORKS 1 +#define SQLITE_OS_OTHER 0 +#define SQLITE_HOMEGROWN_RECURSIVE_MUTEX 1 +#define SQLITE_OMIT_LOAD_EXTENSION 1 +#define SQLITE_ENABLE_LOCKING_STYLE 0 +#define HAVE_UTIME 1 +#else +/* This is not VxWorks. */ +#define OS_VXWORKS 0 +#define HAVE_FCHOWN 1 +#define HAVE_READLINK 1 +#define HAVE_LSTAT 1 +#endif /* defined(_WRS_KERNEL) */ + +/************** End of vxworks.h *********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ /* ** These #defines should enable >2GB file support on POSIX if the @@ -75,6 +207,22 @@ # define _LARGEFILE_SOURCE 1 #endif +/* What version of GCC is being used. 0 means GCC is not being used */ +#ifdef __GNUC__ +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif + +/* Needed for various definitions... */ +#if defined(__GNUC__) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE +#endif + +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE +#endif + /* ** For MinGW, check to see if we can include the header file containing its ** version information, among other things. Normally, this internal MinGW @@ -138,15 +286,15 @@ ** ** The official C-language API documentation for SQLite is derived ** from comments in this file. This file is the authoritative source -** on how SQLite interfaces are suppose to operate. +** on how SQLite interfaces are supposed to operate. ** ** The name of this file under configuration management is "sqlite.h.in". ** The makefile makes some minor changes to this file (such as inserting ** the version number) and changes its name to "sqlite3.h" as ** part of the build process. */ -#ifndef _SQLITE3_H_ -#define _SQLITE3_H_ +#ifndef SQLITE3_H +#define SQLITE3_H #include /* Needed for the definition of va_list */ /* @@ -158,21 +306,34 @@ extern "C" { /* -** Add the ability to override 'extern' +** Provide the ability to override linkage features of the interface. */ #ifndef SQLITE_EXTERN # define SQLITE_EXTERN extern #endif - #ifndef SQLITE_API # define SQLITE_API #endif - - +#ifndef SQLITE_CDECL +# define SQLITE_CDECL +#endif +#ifndef SQLITE_APICALL +# define SQLITE_APICALL +#endif +#ifndef SQLITE_STDCALL +# define SQLITE_STDCALL SQLITE_APICALL +#endif +#ifndef SQLITE_CALLBACK +# define SQLITE_CALLBACK +#endif +#ifndef SQLITE_SYSAPI +# define SQLITE_SYSAPI +#endif + /* ** These no-op macros are used in front of interfaces to mark those ** interfaces as either deprecated or experimental. New applications -** should not use deprecated interfaces - they are support for backwards +** should not use deprecated interfaces - they are supported for backwards ** compatibility only. Application writers should be aware that ** experimental interfaces are subject to change in point releases. ** @@ -210,7 +371,8 @@ extern "C" { ** be held constant and Z will be incremented or else Y will be incremented ** and Z will be reset to zero. ** -** Since version 3.6.18, SQLite source code has been stored in the +** Since [version 3.6.18] ([dateof:3.6.18]), +** SQLite source code has been stored in the ** Fossil configuration management ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite @@ -222,9 +384,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.8.6" -#define SQLITE_VERSION_NUMBER 3008006 -#define SQLITE_SOURCE_ID "2014-08-15 11:46:33 9491ba7d738528f168657adb43a198238abde19e" +#define SQLITE_VERSION "3.15.2" +#define SQLITE_VERSION_NUMBER 3015002 +#define SQLITE_SOURCE_ID "2016-11-28 19:13:37 bbd85d235f7037c6a033a9690534391ffeacecc8" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -235,7 +397,7 @@ extern "C" { ** but are associated with the library instead of the header file. ^(Cautious ** programmers might include assert() statements in their application to ** verify that values returned by these interfaces match the macros in -** the header, and thus insure that the application is +** the header, and thus ensure that the application is ** compiled with matching library and header files. ** ** 
    @@ -316,7 +478,7 @@ SQLITE_API const char *sqlite3_compileoption_get(int N);
 ** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but
 ** can be fully or partially disabled using a call to [sqlite3_config()]
 ** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD],
-** or [SQLITE_CONFIG_MUTEX].  ^(The return value of the
+** or [SQLITE_CONFIG_SERIALIZED].  ^(The return value of the
 ** sqlite3_threadsafe() function shows only the compile-time setting of
 ** thread safety, not any run-time changes to that setting made by
 ** sqlite3_config(). In other words, the return value from sqlite3_threadsafe()
@@ -381,6 +543,7 @@ typedef sqlite_uint64 sqlite3_uint64;
 
 /*
 ** CAPI3REF: Closing A Database Connection
+** DESTRUCTOR: sqlite3
 **
 ** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors
 ** for the [sqlite3] object.
@@ -432,6 +595,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 
 /*
 ** CAPI3REF: One-Step Query Execution Interface
+** METHOD: sqlite3
 **
 ** The sqlite3_exec() interface is a convenience wrapper around
 ** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()],
@@ -456,7 +620,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** from [sqlite3_malloc()] and passed back through the 5th parameter.
 ** To avoid memory leaks, the application should invoke [sqlite3_free()]
 ** on error message strings returned through the 5th parameter of
-** of sqlite3_exec() after the error message string is no longer needed.
+** sqlite3_exec() after the error message string is no longer needed.
 ** ^If the 5th parameter to sqlite3_exec() is not NULL and no errors
 ** occur, then sqlite3_exec() sets the pointer in its 5th parameter to
 ** NULL before returning.
@@ -483,7 +647,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** Restrictions:
 **
 ** 
      
-** 
    •  The application must insure that the 1st parameter to sqlite3_exec()
+** 
    •  The application must ensure that the 1st parameter to sqlite3_exec()
 **      is a valid and open [database connection].
 ** 
    •  The application must not close the [database connection] specified by
 **      the 1st parameter to sqlite3_exec() while sqlite3_exec() is running.
@@ -552,7 +716,8 @@ SQLITE_API int sqlite3_exec(
 ** [result codes].  However, experience has shown that many of
 ** these result codes are too coarse-grained.  They do not provide as
 ** much information about problems as programmers might like.  In an effort to
-** address this, newer versions of SQLite (version 3.3.8 and later) include
+** address this, newer versions of SQLite (version 3.3.8 [dateof:3.3.8]
+** and later) include
 ** support for additional result codes that provide more detailed information
 ** about errors. These [extended result codes] are enabled or disabled
 ** on a per database connection basis using the
@@ -586,6 +751,8 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_IOERR_MMAP              (SQLITE_IOERR | (24<<8))
 #define SQLITE_IOERR_GETTEMPPATH       (SQLITE_IOERR | (25<<8))
 #define SQLITE_IOERR_CONVPATH          (SQLITE_IOERR | (26<<8))
+#define SQLITE_IOERR_VNODE             (SQLITE_IOERR | (27<<8))
+#define SQLITE_IOERR_AUTH              (SQLITE_IOERR | (28<<8))
 #define SQLITE_LOCKED_SHAREDCACHE      (SQLITE_LOCKED |  (1<<8))
 #define SQLITE_BUSY_RECOVERY           (SQLITE_BUSY   |  (1<<8))
 #define SQLITE_BUSY_SNAPSHOT           (SQLITE_BUSY   |  (2<<8))
@@ -612,6 +779,8 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_NOTICE_RECOVER_WAL      (SQLITE_NOTICE | (1<<8))
 #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8))
 #define SQLITE_WARNING_AUTOINDEX       (SQLITE_WARNING | (1<<8))
+#define SQLITE_AUTH_USER               (SQLITE_AUTH | (1<<8))
+#define SQLITE_OK_LOAD_PERMANENTLY     (SQLITE_OK | (1<<8))
 
 /*
 ** CAPI3REF: Flags For File Open Operations
@@ -870,14 +1039,16 @@ struct sqlite3_io_methods {
 ** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()]
 ** interface.
 **
+** 
        
+** 
      • [[SQLITE_FCNTL_LOCKSTATE]]
 ** The [SQLITE_FCNTL_LOCKSTATE] opcode is used for debugging.  This
 ** opcode causes the xFileControl method to write the current state of
 ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED],
 ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE])
 ** into an integer that the pArg argument points to. This capability
-** is used during testing and only needs to be supported when SQLITE_TEST
-** is defined.
-** 
          
+** is used during testing and is only available when the SQLITE_TEST
+** compile-time option is used.
+**
 ** 
        • [[SQLITE_FCNTL_SIZE_HINT]]
 ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS
 ** layer a hint of how large the database file will grow to be during the
@@ -898,8 +1069,13 @@ struct sqlite3_io_methods {
 ** 
        • [[SQLITE_FCNTL_FILE_POINTER]]
 ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer
 ** to the [sqlite3_file] object associated with a particular database
-** connection.  See the [sqlite3_file_control()] documentation for
-** additional information.
+** connection.  See also [SQLITE_FCNTL_JOURNAL_POINTER].
+**
+** 
        • [[SQLITE_FCNTL_JOURNAL_POINTER]]
+** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer
+** to the [sqlite3_file] object associated with the journal file (either
+** the [rollback journal] or the [write-ahead log]) for a particular database
+** connection.  See also [SQLITE_FCNTL_FILE_POINTER].
 **
 ** 
        • [[SQLITE_FCNTL_SYNC_OMITTED]]
 ** No longer in use.
@@ -986,6 +1162,15 @@ struct sqlite3_io_methods {
 ** pointer in case this file-control is not implemented.  This file-control
 ** is intended for diagnostic use only.
 **
+** 
        • [[SQLITE_FCNTL_VFS_POINTER]]
+** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level
+** [VFSes] currently in use.  ^(The argument X in
+** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be
+** of type "[sqlite3_vfs] **".  This opcodes will set *X
+** to a pointer to the top-level VFS.)^
+** ^When there are multiple VFS shims in the stack, this opcode finds the
+** upper-most shim only.
+**
 ** 
        • [[SQLITE_FCNTL_PRAGMA]]
 ** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] 
 ** file control is sent to the open [sqlite3_file] object corresponding
@@ -1002,7 +1187,9 @@ struct sqlite3_io_methods {
 ** [PRAGMA] processing continues.  ^If the [SQLITE_FCNTL_PRAGMA]
 ** file control returns [SQLITE_OK], then the parser assumes that the
 ** VFS has handled the PRAGMA itself and the parser generates a no-op
-** prepared statement.  ^If the [SQLITE_FCNTL_PRAGMA] file control returns
+** prepared statement if result string is NULL, or that returns a copy
+** of the result string if the string is non-NULL.
+** ^If the [SQLITE_FCNTL_PRAGMA] file control returns
 ** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means
 ** that the VFS encountered an error while handling the [PRAGMA] and the
 ** compilation of the PRAGMA fails with an error.  ^The [SQLITE_FCNTL_PRAGMA]
@@ -1054,18 +1241,39 @@ struct sqlite3_io_methods {
 ** on whether or not the file has been renamed, moved, or deleted since it
 ** was first opened.
 **
+** 
        • [[SQLITE_FCNTL_WIN32_GET_HANDLE]]
+** The [SQLITE_FCNTL_WIN32_GET_HANDLE] opcode can be used to obtain the
+** underlying native file handle associated with a file handle.  This file
+** control interprets its argument as a pointer to a native file handle and
+** writes the resulting value there.
+**
 ** 
        • [[SQLITE_FCNTL_WIN32_SET_HANDLE]]
 ** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging.  This
 ** opcode causes the xFileControl method to swap the file handle with the one
 ** pointed to by the pArg argument.  This capability is used during testing
 ** and only needs to be supported when SQLITE_TEST is defined.
 **
+** 
        • [[SQLITE_FCNTL_WAL_BLOCK]]
+** The [SQLITE_FCNTL_WAL_BLOCK] is a signal to the VFS layer that it might
+** be advantageous to block on the next WAL lock if the lock is not immediately
+** available.  The WAL subsystem issues this signal during rare
+** circumstances in order to fix a problem with priority inversion.
+** Applications should not use this file-control.
+**
+** 
        • [[SQLITE_FCNTL_ZIPVFS]]
+** The [SQLITE_FCNTL_ZIPVFS] opcode is implemented by zipvfs only. All other
+** VFS should return SQLITE_NOTFOUND for this opcode.
+**
+** 
        • [[SQLITE_FCNTL_RBU]]
+** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by
+** the RBU extension only.  All other VFS should return SQLITE_NOTFOUND for
+** this opcode.  
 ** 
        
 */
 #define SQLITE_FCNTL_LOCKSTATE               1
-#define SQLITE_GET_LOCKPROXYFILE             2
-#define SQLITE_SET_LOCKPROXYFILE             3
-#define SQLITE_LAST_ERRNO                    4
+#define SQLITE_FCNTL_GET_LOCKPROXYFILE       2
+#define SQLITE_FCNTL_SET_LOCKPROXYFILE       3
+#define SQLITE_FCNTL_LAST_ERRNO              4
 #define SQLITE_FCNTL_SIZE_HINT               5
 #define SQLITE_FCNTL_CHUNK_SIZE              6
 #define SQLITE_FCNTL_FILE_POINTER            7
@@ -1084,6 +1292,18 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_SYNC                   21
 #define SQLITE_FCNTL_COMMIT_PHASETWO        22
 #define SQLITE_FCNTL_WIN32_SET_HANDLE       23
+#define SQLITE_FCNTL_WAL_BLOCK              24
+#define SQLITE_FCNTL_ZIPVFS                 25
+#define SQLITE_FCNTL_RBU                    26
+#define SQLITE_FCNTL_VFS_POINTER            27
+#define SQLITE_FCNTL_JOURNAL_POINTER        28
+#define SQLITE_FCNTL_WIN32_GET_HANDLE       29
+
+/* deprecated names */
+#define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
+#define SQLITE_SET_LOCKPROXYFILE      SQLITE_FCNTL_SET_LOCKPROXYFILE
+#define SQLITE_LAST_ERRNO             SQLITE_FCNTL_LAST_ERRNO
+
 
 /*
 ** CAPI3REF: Mutex Handle
@@ -1097,6 +1317,16 @@ struct sqlite3_io_methods {
 */
 typedef struct sqlite3_mutex sqlite3_mutex;
 
+/*
+** CAPI3REF: Loadable Extension Thunk
+**
+** A pointer to the opaque sqlite3_api_routines structure is passed as
+** the third parameter to entry points of [loadable extensions].  This
+** structure must be typedefed in order to work around compiler warnings
+** on some platforms.
+*/
+typedef struct sqlite3_api_routines sqlite3_api_routines;
+
 /*
 ** CAPI3REF: OS Interface Object
 **
@@ -1290,7 +1520,7 @@ struct sqlite3_vfs {
   const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName);
   /*
   ** The methods above are in versions 1 through 3 of the sqlite_vfs object.
-  ** New fields may be appended in figure versions.  The iVersion
+  ** New fields may be appended in future versions.  The iVersion
   ** value will increment whenever this happens. 
   */
 };
@@ -1335,7 +1565,7 @@ struct sqlite3_vfs {
 ** 
      
 **
 ** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as
-** was given no the corresponding lock.  
+** was given on the corresponding lock.  
 **
 ** The xShmLock method can transition between unlocked and SHARED or
 ** between unlocked and EXCLUSIVE.  It cannot transition between SHARED
@@ -1446,9 +1676,11 @@ SQLITE_API int sqlite3_os_end(void);
 ** applications and so this routine is usually not necessary.  It is
 ** provided to support rare applications with unusual needs.
 **
-** The sqlite3_config() interface is not threadsafe.  The application
-** must insure that no other SQLite interfaces are invoked by other
-** threads while sqlite3_config() is running.  Furthermore, sqlite3_config()
+** The sqlite3_config() interface is not threadsafe. The application
+** must ensure that no other SQLite interfaces are invoked by other
+** threads while sqlite3_config() is running.
+**
+** The sqlite3_config() interface
 ** may only be invoked prior to library initialization using
 ** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()].
 ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before
@@ -1470,6 +1702,7 @@ SQLITE_API int sqlite3_config(int, ...);
 
 /*
 ** CAPI3REF: Configure database connections
+** METHOD: sqlite3
 **
 ** The sqlite3_db_config() interface is used to make configuration
 ** changes to a [database connection].  The interface is similar to
@@ -1618,31 +1851,33 @@ struct sqlite3_mem_methods {
 ** SQLITE_CONFIG_SERIALIZED configuration option.
 **
 ** [[SQLITE_CONFIG_MALLOC]] 
      SQLITE_CONFIG_MALLOC
      
-** 
       ^(This option takes a single argument which is a pointer to an
-** instance of the [sqlite3_mem_methods] structure.  The argument specifies
+** 
       ^(The SQLITE_CONFIG_MALLOC option takes a single argument which is 
+** a pointer to an instance of the [sqlite3_mem_methods] structure.
+** The argument specifies
 ** alternative low-level memory allocation routines to be used in place of
 ** the memory allocation routines built into SQLite.)^ ^SQLite makes
 ** its own private copy of the content of the [sqlite3_mem_methods] structure
 ** before the [sqlite3_config()] call returns.
      
 **
 ** [[SQLITE_CONFIG_GETMALLOC]] 
      SQLITE_CONFIG_GETMALLOC
      
-** 
       ^(This option takes a single argument which is a pointer to an
-** instance of the [sqlite3_mem_methods] structure.  The [sqlite3_mem_methods]
+** 
       ^(The SQLITE_CONFIG_GETMALLOC option takes a single argument which
+** is a pointer to an instance of the [sqlite3_mem_methods] structure.
+** The [sqlite3_mem_methods]
 ** structure is filled with the currently defined memory allocation routines.)^
 ** This option can be used to overload the default memory allocation
 ** routines with a wrapper that simulations memory allocation failure or
 ** tracks memory usage, for example. 
      
 **
 ** [[SQLITE_CONFIG_MEMSTATUS]] 
      SQLITE_CONFIG_MEMSTATUS
      
-** 
       ^This option takes single argument of type int, interpreted as a 
-** boolean, which enables or disables the collection of memory allocation 
-** statistics. ^(When memory allocation statistics are disabled, the 
-** following SQLite interfaces become non-operational:
+** 
       ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int,
+** interpreted as a boolean, which enables or disables the collection of
+** memory allocation statistics. ^(When memory allocation statistics are
+** disabled, the following SQLite interfaces become non-operational:
 **   
        
 **   
      •  [sqlite3_memory_used()]
 **   
      •  [sqlite3_memory_highwater()]
 **   
      •  [sqlite3_soft_heap_limit64()]
-**   
      •  [sqlite3_status()]
+**   
      •  [sqlite3_status64()]
 **   
      )^
 ** ^Memory allocation statistics are enabled by default unless SQLite is
 ** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory
@@ -1650,53 +1885,72 @@ struct sqlite3_mem_methods {
 ** 
      
 **
 ** [[SQLITE_CONFIG_SCRATCH]] 
      SQLITE_CONFIG_SCRATCH
      
-** 
       ^This option specifies a static memory buffer that SQLite can use for
-** scratch memory.  There are three arguments:  A pointer an 8-byte
+** 
       ^The SQLITE_CONFIG_SCRATCH option specifies a static memory buffer
+** that SQLite can use for scratch memory.  ^(There are three arguments
+** to SQLITE_CONFIG_SCRATCH:  A pointer an 8-byte
 ** aligned memory buffer from which the scratch allocations will be
 ** drawn, the size of each scratch allocation (sz),
-** and the maximum number of scratch allocations (N).  The sz
-** argument must be a multiple of 16.
+** and the maximum number of scratch allocations (N).)^
 ** The first argument must be a pointer to an 8-byte aligned buffer
 ** of at least sz*N bytes of memory.
-** ^SQLite will use no more than two scratch buffers per thread.  So
-** N should be set to twice the expected maximum number of threads.
-** ^SQLite will never require a scratch buffer that is more than 6
-** times the database page size. ^If SQLite needs needs additional
+** ^SQLite will not use more than one scratch buffers per thread.
+** ^SQLite will never request a scratch buffer that is more than 6
+** times the database page size.
+** ^If SQLite needs needs additional
 ** scratch memory beyond what is provided by this configuration option, then 
-** [sqlite3_malloc()] will be used to obtain the memory needed.
      
+** [sqlite3_malloc()] will be used to obtain the memory needed.
      
+** ^When the application provides any amount of scratch memory using
+** SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary large
+** [sqlite3_malloc|heap allocations].
+** This can help [Robson proof|prevent memory allocation failures] due to heap
+** fragmentation in low-memory embedded systems.
+** 
 **
 ** [[SQLITE_CONFIG_PAGECACHE]] 
      SQLITE_CONFIG_PAGECACHE
      
-** 
       ^This option specifies a static memory buffer that SQLite can use for
-** the database page cache with the default page cache implementation.  
-** This configuration should not be used if an application-define page
-** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option.
-** There are three arguments to this option: A pointer to 8-byte aligned
-** memory, the size of each page buffer (sz), and the number of pages (N).
+** 
       ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool
+** that SQLite can use for the database page cache with the default page
+** cache implementation.  
+** This configuration option is a no-op if an application-define page
+** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2].
+** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to
+** 8-byte aligned memory (pMem), the size of each page cache line (sz),
+** and the number of cache lines (N).
 ** The sz argument should be the size of the largest database page
-** (a power of two between 512 and 32768) plus a little extra for each
-** page header.  ^The page header size is 20 to 40 bytes depending on
-** the host architecture.  ^It is harmless, apart from the wasted memory,
-** to make sz a little too large.  The first
-** argument should point to an allocation of at least sz*N bytes of memory.
-** ^SQLite will use the memory provided by the first argument to satisfy its
-** memory needs for the first N pages that it adds to cache.  ^If additional
-** page cache memory is needed beyond what is provided by this option, then
-** SQLite goes to [sqlite3_malloc()] for the additional storage space.
-** The pointer in the first argument must
-** be aligned to an 8-byte boundary or subsequent behavior of SQLite
-** will be undefined.
      
+** (a power of two between 512 and 65536) plus some extra bytes for each
+** page header.  ^The number of extra bytes needed by the page header
+** can be determined using [SQLITE_CONFIG_PCACHE_HDRSZ].
+** ^It is harmless, apart from the wasted memory,
+** for the sz parameter to be larger than necessary.  The pMem
+** argument must be either a NULL pointer or a pointer to an 8-byte
+** aligned block of memory of at least sz*N bytes, otherwise
+** subsequent behavior is undefined.
+** ^When pMem is not NULL, SQLite will strive to use the memory provided
+** to satisfy page cache needs, falling back to [sqlite3_malloc()] if
+** a page cache line is larger than sz bytes or if all of the pMem buffer
+** is exhausted.
+** ^If pMem is NULL and N is non-zero, then each database connection
+** does an initial bulk allocation for page cache memory
+** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or
+** of -1024*N bytes if N is negative, . ^If additional
+** page cache memory is needed beyond what is provided by the initial
+** allocation, then SQLite goes to [sqlite3_malloc()] separately for each
+** additional cache line. 
 **
 ** [[SQLITE_CONFIG_HEAP]] 
      SQLITE_CONFIG_HEAP
      
-** 
       ^This option specifies a static memory buffer that SQLite will use
-** for all of its dynamic memory allocation needs beyond those provided
-** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE].
-** There are three arguments: An 8-byte aligned pointer to the memory,
+** 
       ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer 
+** that SQLite will use for all of its dynamic memory allocation needs
+** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and
+** [SQLITE_CONFIG_PAGECACHE].
+** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled
+** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns
+** [SQLITE_ERROR] if invoked otherwise.
+** ^There are three arguments to SQLITE_CONFIG_HEAP:
+** An 8-byte aligned pointer to the memory,
 ** the number of bytes in the memory buffer, and the minimum allocation size.
 ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts
 ** to using its default memory allocator (the system malloc() implementation),
 ** undoing any prior invocation of [SQLITE_CONFIG_MALLOC].  ^If the
-** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or
-** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory
+** memory pointer is not NULL then the alternative memory
 ** allocator is engaged to handle all of SQLites memory allocation needs.
 ** The first pointer (the memory pointer) must be aligned to an 8-byte
 ** boundary or subsequent behavior of SQLite will be undefined.
@@ -1704,11 +1958,11 @@ struct sqlite3_mem_methods {
 ** for the minimum allocation size are 2**5 through 2**8.
      
 **
 ** [[SQLITE_CONFIG_MUTEX]] 
      SQLITE_CONFIG_MUTEX
      
-** 
       ^(This option takes a single argument which is a pointer to an
-** instance of the [sqlite3_mutex_methods] structure.  The argument specifies
-** alternative low-level mutex routines to be used in place
-** the mutex routines built into SQLite.)^  ^SQLite makes a copy of the
-** content of the [sqlite3_mutex_methods] structure before the call to
+** 
       ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a
+** pointer to an instance of the [sqlite3_mutex_methods] structure.
+** The argument specifies alternative low-level mutex routines to be used
+** in place the mutex routines built into SQLite.)^  ^SQLite makes a copy of
+** the content of the [sqlite3_mutex_methods] structure before the call to
 ** [sqlite3_config()] returns. ^If SQLite is compiled with
 ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then
 ** the entire mutexing subsystem is omitted from the build and hence calls to
@@ -1716,8 +1970,8 @@ struct sqlite3_mem_methods {
 ** return [SQLITE_ERROR].
      
 **
 ** [[SQLITE_CONFIG_GETMUTEX]] 
      SQLITE_CONFIG_GETMUTEX
      
-** 
       ^(This option takes a single argument which is a pointer to an
-** instance of the [sqlite3_mutex_methods] structure.  The
+** 
       ^(The SQLITE_CONFIG_GETMUTEX option takes a single argument which
+** is a pointer to an instance of the [sqlite3_mutex_methods] structure.  The
 ** [sqlite3_mutex_methods]
 ** structure is filled with the currently defined mutex routines.)^
 ** This option can be used to overload the default mutex allocation
@@ -1729,25 +1983,25 @@ struct sqlite3_mem_methods {
 ** return [SQLITE_ERROR].
      
 **
 ** [[SQLITE_CONFIG_LOOKASIDE]] 
      SQLITE_CONFIG_LOOKASIDE
      
-** 
       ^(This option takes two arguments that determine the default
-** memory allocation for the lookaside memory allocator on each
-** [database connection].  The first argument is the
+** 
       ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine
+** the default size of lookaside memory on each [database connection].
+** The first argument is the
 ** size of each lookaside buffer slot and the second is the number of
-** slots allocated to each database connection.)^  ^(This option sets the
-** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE]
-** verb to [sqlite3_db_config()] can be used to change the lookaside
+** slots allocated to each database connection.)^  ^(SQLITE_CONFIG_LOOKASIDE
+** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE]
+** option to [sqlite3_db_config()] can be used to change the lookaside
 ** configuration on individual connections.)^ 
      
 **
 ** [[SQLITE_CONFIG_PCACHE2]] 
      SQLITE_CONFIG_PCACHE2
      
-** 
       ^(This option takes a single argument which is a pointer to
-** an [sqlite3_pcache_methods2] object.  This object specifies the interface
-** to a custom page cache implementation.)^  ^SQLite makes a copy of the
-** object and uses it for page cache memory allocations.
      
+** 
       ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is 
+** a pointer to an [sqlite3_pcache_methods2] object.  This object specifies
+** the interface to a custom page cache implementation.)^
+** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
      
 **
 ** [[SQLITE_CONFIG_GETPCACHE2]] 
      SQLITE_CONFIG_GETPCACHE2
      
-** 
       ^(This option takes a single argument which is a pointer to an
-** [sqlite3_pcache_methods2] object.  SQLite copies of the current
-** page cache implementation into that object.)^ 
      
+** 
       ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which
+** is a pointer to an [sqlite3_pcache_methods2] object.  SQLite copies of
+** the current page cache implementation into that object.)^ 
      
 **
 ** [[SQLITE_CONFIG_LOG]] 
      SQLITE_CONFIG_LOG
      
 ** 
       The SQLITE_CONFIG_LOG option is used to configure the SQLite
@@ -1770,10 +2024,11 @@ struct sqlite3_mem_methods {
 ** function must be threadsafe. 
      
 **
 ** [[SQLITE_CONFIG_URI]] 
      SQLITE_CONFIG_URI
-** 
      ^(This option takes a single argument of type int. If non-zero, then
-** URI handling is globally enabled. If the parameter is zero, then URI handling
-** is globally disabled.)^ ^If URI handling is globally enabled, all filenames
-** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or
+** 
      ^(The SQLITE_CONFIG_URI option takes a single argument of type int.
+** If non-zero, then URI handling is globally enabled. If the parameter is zero,
+** then URI handling is globally disabled.)^ ^If URI handling is globally
+** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()],
+** [sqlite3_open16()] or
 ** specified as part of [ATTACH] commands are interpreted as URIs, regardless
 ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database
 ** connection is opened. ^If it is globally disabled, filenames are
@@ -1783,9 +2038,10 @@ struct sqlite3_mem_methods {
 ** [SQLITE_USE_URI] symbol defined.)^
 **
 ** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]] 
      SQLITE_CONFIG_COVERING_INDEX_SCAN
-** 
      ^This option takes a single integer argument which is interpreted as
-** a boolean in order to enable or disable the use of covering indices for
-** full table scans in the query optimizer.  ^The default setting is determined
+** 
      ^The SQLITE_CONFIG_COVERING_INDEX_SCAN option takes a single integer
+** argument which is interpreted as a boolean in order to enable or disable
+** the use of covering indices for full table scans in the query optimizer.
+** ^The default setting is determined
 ** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on"
 ** if that compile-time option is omitted.
 ** The ability to disable the use of covering indices for full table scans
@@ -1825,18 +2081,51 @@ struct sqlite3_mem_methods {
 ** ^The default setting can be overridden by each database connection using
 ** either the [PRAGMA mmap_size] command, or by using the
 ** [SQLITE_FCNTL_MMAP_SIZE] file control.  ^(The maximum allowed mmap size
-** cannot be changed at run-time.  Nor may the maximum allowed mmap size
-** exceed the compile-time maximum mmap size set by the
+** will be silently truncated if necessary so that it does not exceed the
+** compile-time maximum mmap size set by the
 ** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^
 ** ^If either argument to this option is negative, then that argument is
 ** changed to its compile-time default.
 **
 ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]]
 ** 
      SQLITE_CONFIG_WIN32_HEAPSIZE
-** 
      ^This option is only available if SQLite is compiled for Windows
-** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined.
-** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value
+** 
      ^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is
+** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro
+** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value
 ** that specifies the maximum size of the created heap.
+**
+** [[SQLITE_CONFIG_PCACHE_HDRSZ]]
+** 
      SQLITE_CONFIG_PCACHE_HDRSZ
+** 
      ^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which
+** is a pointer to an integer and writes into that integer the number of extra
+** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE].
+** The amount of extra space required can change depending on the compiler,
+** target platform, and SQLite version.
+**
+** [[SQLITE_CONFIG_PMASZ]]
+** 
      SQLITE_CONFIG_PMASZ
+** 
      ^The SQLITE_CONFIG_PMASZ option takes a single parameter which
+** is an unsigned integer and sets the "Minimum PMA Size" for the multithreaded
+** sorter to that integer.  The default minimum PMA Size is set by the
+** [SQLITE_SORTER_PMASZ] compile-time option.  New threads are launched
+** to help with sort operations when multithreaded sorting
+** is enabled (using the [PRAGMA threads] command) and the amount of content
+** to be sorted exceeds the page size times the minimum of the
+** [PRAGMA cache_size] setting and this value.
+**
+** [[SQLITE_CONFIG_STMTJRNL_SPILL]]
+** 
      SQLITE_CONFIG_STMTJRNL_SPILL
+** 
      ^The SQLITE_CONFIG_STMTJRNL_SPILL option takes a single parameter which
+** becomes the [statement journal] spill-to-disk threshold.  
+** [Statement journals] are held in memory until their size (in bytes)
+** exceeds this threshold, at which point they are written to disk.
+** Or if the threshold is -1, statement journals are always held
+** exclusively in memory.
+** Since many statement journals never become large, setting the spill
+** threshold to a value such as 64KiB can greatly reduce the amount of
+** I/O required to support statement rollback.
+** The default value for this setting is controlled by the
+** [SQLITE_STMTJRNL_SPILL] compile-time option.
 ** 
 */
 #define SQLITE_CONFIG_SINGLETHREAD  1  /* nil */
@@ -1862,6 +2151,9 @@ struct sqlite3_mem_methods {
 #define SQLITE_CONFIG_SQLLOG       21  /* xSqllog, void* */
 #define SQLITE_CONFIG_MMAP_SIZE    22  /* sqlite3_int64, sqlite3_int64 */
 #define SQLITE_CONFIG_WIN32_HEAPSIZE      23  /* int nByte */
+#define SQLITE_CONFIG_PCACHE_HDRSZ        24  /* int *psz */
+#define SQLITE_CONFIG_PMASZ               25  /* unsigned int szPma */
+#define SQLITE_CONFIG_STMTJRNL_SPILL      26  /* int nByte */
 
 /*
 ** CAPI3REF: Database Connection Configuration Options
@@ -1919,15 +2211,58 @@ struct sqlite3_mem_methods {
 ** following this call.  The second parameter may be a NULL pointer, in
 ** which case the trigger setting is not reported back. 
      
 **
+** 
      SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
      
+** 
       ^This option is used to enable or disable the two-argument
+** version of the [fts3_tokenizer()] function which is part of the
+** [FTS3] full-text search engine extension.
+** There should be two additional arguments.
+** The first argument is an integer which is 0 to disable fts3_tokenizer() or
+** positive to enable fts3_tokenizer() or negative to leave the setting
+** unchanged.
+** The second parameter is a pointer to an integer into which
+** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled
+** following this call.  The second parameter may be a NULL pointer, in
+** which case the new setting is not reported back. 
      
+**
+** 
      SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
      
+** 
       ^This option is used to enable or disable the [sqlite3_load_extension()]
+** interface independently of the [load_extension()] SQL function.
+** The [sqlite3_enable_load_extension()] API enables or disables both the
+** C-API [sqlite3_load_extension()] and the SQL function [load_extension()].
+** There should be two additional arguments.
+** When the first argument to this interface is 1, then only the C-API is
+** enabled and the SQL function remains disabled.  If the first argument to
+** this interface is 0, then both the C-API and the SQL function are disabled.
+** If the first argument is -1, then no changes are made to state of either the
+** C-API or the SQL function.
+** The second parameter is a pointer to an integer into which
+** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface
+** is disabled or enabled following this call.  The second parameter may
+** be a NULL pointer, in which case the new setting is not reported back.
+** 
      
+**
+** 
      SQLITE_DBCONFIG_MAINDBNAME
      
+** 
       ^This option is used to change the name of the "main" database
+** schema.  ^The sole argument is a pointer to a constant UTF8 string
+** which will become the new schema name in place of "main".  ^SQLite
+** does not make a copy of the new main schema name string, so the application
+** must ensure that the argument passed into this DBCONFIG option is unchanged
+** until after the database connection closes.
+** 
      
+**
 ** 
 */
-#define SQLITE_DBCONFIG_LOOKASIDE       1001  /* void* int int */
-#define SQLITE_DBCONFIG_ENABLE_FKEY     1002  /* int int* */
-#define SQLITE_DBCONFIG_ENABLE_TRIGGER  1003  /* int int* */
+#define SQLITE_DBCONFIG_MAINDBNAME            1000 /* const char* */
+#define SQLITE_DBCONFIG_LOOKASIDE             1001 /* void* int int */
+#define SQLITE_DBCONFIG_ENABLE_FKEY           1002 /* int int* */
+#define SQLITE_DBCONFIG_ENABLE_TRIGGER        1003 /* int int* */
+#define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */
+#define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
 
 
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
+** METHOD: sqlite3
 **
 ** ^The sqlite3_extended_result_codes() routine enables or disables the
 ** [extended result codes] feature of SQLite. ^The extended result
@@ -1937,6 +2272,7 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff);
 
 /*
 ** CAPI3REF: Last Insert Rowid
+** METHOD: sqlite3
 **
 ** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables)
 ** has a unique 64-bit signed
@@ -1988,48 +2324,47 @@ SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*);
 
 /*
 ** CAPI3REF: Count The Number Of Rows Modified
+** METHOD: sqlite3
 **
-** ^This function returns the number of database rows that were changed
-** or inserted or deleted by the most recently completed SQL statement
-** on the [database connection] specified by the first parameter.
-** ^(Only changes that are directly specified by the [INSERT], [UPDATE],
-** or [DELETE] statement are counted.  Auxiliary changes caused by
-** triggers or [foreign key actions] are not counted.)^ Use the
-** [sqlite3_total_changes()] function to find the total number of changes
-** including changes caused by triggers and foreign key actions.
+** ^This function returns the number of rows modified, inserted or
+** deleted by the most recently completed INSERT, UPDATE or DELETE
+** statement on the database connection specified by the only parameter.
+** ^Executing any other type of SQL statement does not modify the value
+** returned by this function.
 **
-** ^Changes to a view that are simulated by an [INSTEAD OF trigger]
-** are not counted.  Only real table changes are counted.
+** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are
+** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], 
+** [foreign key actions] or [REPLACE] constraint resolution are not counted.
+** 
+** Changes to a view that are intercepted by 
+** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value 
+** returned by sqlite3_changes() immediately after an INSERT, UPDATE or 
+** DELETE statement run on a view is always zero. Only changes made to real 
+** tables are counted.
 **
-** ^(A "row change" is a change to a single row of a single table
-** caused by an INSERT, DELETE, or UPDATE statement.  Rows that
-** are changed as side effects of [REPLACE] constraint resolution,
-** rollback, ABORT processing, [DROP TABLE], or by any other
-** mechanisms do not count as direct row changes.)^
-**
-** A "trigger context" is a scope of execution that begins and
-** ends with the script of a [CREATE TRIGGER | trigger]. 
-** Most SQL statements are
-** evaluated outside of any trigger.  This is the "top level"
-** trigger context.  If a trigger fires from the top level, a
-** new trigger context is entered for the duration of that one
-** trigger.  Subtriggers create subcontexts for their duration.
-**
-** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does
-** not create a new trigger context.
-**
-** ^This function returns the number of direct row changes in the
-** most recent INSERT, UPDATE, or DELETE statement within the same
-** trigger context.
-**
-** ^Thus, when called from the top level, this function returns the
-** number of changes in the most recent INSERT, UPDATE, or DELETE
-** that also occurred at the top level.  ^(Within the body of a trigger,
-** the sqlite3_changes() interface can be called to find the number of
-** changes in the most recently completed INSERT, UPDATE, or DELETE
-** statement within the body of the same trigger.
-** However, the number returned does not include changes
-** caused by subtriggers since those have their own context.)^
+** Things are more complicated if the sqlite3_changes() function is
+** executed while a trigger program is running. This may happen if the
+** program uses the [changes() SQL function], or if some other callback
+** function invokes sqlite3_changes() directly. Essentially:
+** 
+** 
        
+**   
      •  ^(Before entering a trigger program the value returned by
+**        sqlite3_changes() function is saved. After the trigger program 
+**        has finished, the original value is restored.)^
+** 
+**   
      •  ^(Within a trigger program each INSERT, UPDATE and DELETE 
+**        statement sets the value returned by sqlite3_changes() 
+**        upon completion as normal. Of course, this value will not include 
+**        any changes performed by sub-triggers, as the sqlite3_changes() 
+**        value will be saved and restored after each sub-trigger has run.)^
+** 
      
+** 
+** ^This means that if the changes() SQL function (or similar) is used
+** by the first INSERT, UPDATE or DELETE statement within a trigger, it 
+** returns the value as set when the calling statement began executing.
+** ^If it is used by the second or subsequent such statement within a trigger 
+** program, the value returned reflects the number of rows modified by the 
+** previous INSERT, UPDATE or DELETE statement within the same trigger.
 **
 ** See also the [sqlite3_total_changes()] interface, the
 ** [count_changes pragma], and the [changes() SQL function].
@@ -2042,21 +2377,19 @@ SQLITE_API int sqlite3_changes(sqlite3*);
 
 /*
 ** CAPI3REF: Total Number Of Rows Modified
+** METHOD: sqlite3
 **
-** ^This function returns the number of row changes caused by [INSERT],
-** [UPDATE] or [DELETE] statements since the [database connection] was opened.
-** ^(The count returned by sqlite3_total_changes() includes all changes
-** from all [CREATE TRIGGER | trigger] contexts and changes made by
-** [foreign key actions]. However,
-** the count does not include changes used to implement [REPLACE] constraints,
-** do rollbacks or ABORT processing, or [DROP TABLE] processing.  The
-** count does not include rows of views that fire an [INSTEAD OF trigger],
-** though if the INSTEAD OF trigger makes changes of its own, those changes 
-** are counted.)^
-** ^The sqlite3_total_changes() function counts the changes as soon as
-** the statement that makes them is completed (when the statement handle
-** is passed to [sqlite3_reset()] or [sqlite3_finalize()]).
-**
+** ^This function returns the total number of rows inserted, modified or
+** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed
+** since the database connection was opened, including those executed as
+** part of trigger programs. ^Executing any other type of SQL statement
+** does not affect the value returned by sqlite3_total_changes().
+** 
+** ^Changes made as part of [foreign key actions] are included in the
+** count, but those made as part of REPLACE constraint resolution are
+** not. ^Changes to a view that are intercepted by INSTEAD OF triggers 
+** are not counted.
+** 
 ** See also the [sqlite3_changes()] interface, the
 ** [count_changes pragma], and the [total_changes() SQL function].
 **
@@ -2068,6 +2401,7 @@ SQLITE_API int sqlite3_total_changes(sqlite3*);
 
 /*
 ** CAPI3REF: Interrupt A Long-Running Query
+** METHOD: sqlite3
 **
 ** ^This function causes any pending database operation to abort and
 ** return at its earliest opportunity. This routine is typically
@@ -2143,6 +2477,8 @@ SQLITE_API int sqlite3_complete16(const void *sql);
 
 /*
 ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors
+** KEYWORDS: {busy-handler callback} {busy handler}
+** METHOD: sqlite3
 **
 ** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X
 ** that might be invoked with argument P whenever
@@ -2159,7 +2495,7 @@ SQLITE_API int sqlite3_complete16(const void *sql);
 ** ^The first argument to the busy handler is a copy of the void* pointer which
 ** is the third argument to sqlite3_busy_handler().  ^The second argument to
 ** the busy handler callback is the number of times that the busy handler has
-** been invoked for the same locking event.  ^If the
+** been invoked previously for the same locking event.  ^If the
 ** busy callback returns 0, then no additional attempts are made to
 ** access the database and [SQLITE_BUSY] is returned
 ** to the application.
@@ -2198,10 +2534,11 @@ SQLITE_API int sqlite3_complete16(const void *sql);
 ** A busy handler must not close the database connection
 ** or [prepared statement] that invoked the busy handler.
 */
-SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*);
+SQLITE_API int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*);
 
 /*
 ** CAPI3REF: Set A Busy Timeout
+** METHOD: sqlite3
 **
 ** ^This routine sets a [sqlite3_busy_handler | busy handler] that sleeps
 ** for a specified amount of time when a table is locked.  ^The handler
@@ -2214,7 +2551,7 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*);
 ** turns off all busy handlers.
 **
 ** ^(There can only be a single busy handler for a particular
-** [database connection] any any given moment.  If another busy handler
+** [database connection] at any given moment.  If another busy handler
 ** was defined  (using [sqlite3_busy_handler()]) prior to calling
 ** this routine, that other busy handler is cleared.)^
 **
@@ -2224,6 +2561,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms);
 
 /*
 ** CAPI3REF: Convenience Routines For Running Queries
+** METHOD: sqlite3
 **
 ** This is a legacy interface that is preserved for backwards compatibility.
 ** Use of this interface is not recommended.
@@ -2309,6 +2647,10 @@ SQLITE_API void sqlite3_free_table(char **result);
 **
 ** These routines are work-alikes of the "printf()" family of functions
 ** from the standard C library.
+** These routines understand most of the common K&R formatting options,
+** plus some additional non-standard formats, detailed below.
+** Note that some of the more obscure formatting options from recent
+** C-library standards are omitted from this implementation.
 **
 ** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their
 ** results into memory obtained from [sqlite3_malloc()].
@@ -2341,7 +2683,7 @@ SQLITE_API void sqlite3_free_table(char **result);
 ** These routines all implement some additional formatting
 ** options that are useful for constructing SQL statements.
 ** All of the usual printf() formatting options apply.  In addition, there
-** is are "%q", "%Q", and "%z" options.
+** is are "%q", "%Q", "%w" and "%z" options.
 **
 ** ^(The %q option works like %s in that it substitutes a nul-terminated
 ** string from the argument list.  But %q also doubles every '\'' character.
@@ -2394,6 +2736,12 @@ SQLITE_API void sqlite3_free_table(char **result);
 ** The code above will render a correct SQL statement in the zSQL
 ** variable even if the zText variable is a NULL pointer.
 **
+** ^(The "%w" formatting option is like "%q" except that it expects to
+** be contained within double-quotes instead of single quotes, and it
+** escapes the double-quote character instead of the single-quote
+** character.)^  The "%w" formatting option is intended for safely inserting
+** table and column names into a constructed SQL statement.
+**
 ** ^(The "%z" formatting option works like "%s" but with the
 ** addition that after the string has been read and copied into
 ** the result, [sqlite3_free()] is called on the input string.)^
@@ -2418,6 +2766,10 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 ** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns
 ** a NULL pointer.
 **
+** ^The sqlite3_malloc64(N) routine works just like
+** sqlite3_malloc(N) except that N is an unsigned 64-bit integer instead
+** of a signed 32-bit integer.
+**
 ** ^Calling sqlite3_free() with a pointer previously returned
 ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so
 ** that it might be reused.  ^The sqlite3_free() routine is
@@ -2429,24 +2781,38 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 ** might result if sqlite3_free() is called with a non-NULL pointer that
 ** was not obtained from sqlite3_malloc() or sqlite3_realloc().
 **
-** ^(The sqlite3_realloc() interface attempts to resize a
-** prior memory allocation to be at least N bytes, where N is the
-** second parameter.  The memory allocation to be resized is the first
-** parameter.)^ ^ If the first parameter to sqlite3_realloc()
+** ^The sqlite3_realloc(X,N) interface attempts to resize a
+** prior memory allocation X to be at least N bytes.
+** ^If the X parameter to sqlite3_realloc(X,N)
 ** is a NULL pointer then its behavior is identical to calling
-** sqlite3_malloc(N) where N is the second parameter to sqlite3_realloc().
-** ^If the second parameter to sqlite3_realloc() is zero or
+** sqlite3_malloc(N).
+** ^If the N parameter to sqlite3_realloc(X,N) is zero or
 ** negative then the behavior is exactly the same as calling
-** sqlite3_free(P) where P is the first parameter to sqlite3_realloc().
-** ^sqlite3_realloc() returns a pointer to a memory allocation
-** of at least N bytes in size or NULL if sufficient memory is unavailable.
+** sqlite3_free(X).
+** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation
+** of at least N bytes in size or NULL if insufficient memory is available.
 ** ^If M is the size of the prior allocation, then min(N,M) bytes
 ** of the prior allocation are copied into the beginning of buffer returned
-** by sqlite3_realloc() and the prior allocation is freed.
-** ^If sqlite3_realloc() returns NULL, then the prior allocation
-** is not freed.
+** by sqlite3_realloc(X,N) and the prior allocation is freed.
+** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the
+** prior allocation is not freed.
 **
-** ^The memory returned by sqlite3_malloc() and sqlite3_realloc()
+** ^The sqlite3_realloc64(X,N) interfaces works the same as
+** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead
+** of a 32-bit signed integer.
+**
+** ^If X is a memory allocation previously obtained from sqlite3_malloc(),
+** sqlite3_malloc64(), sqlite3_realloc(), or sqlite3_realloc64(), then
+** sqlite3_msize(X) returns the size of that memory allocation in bytes.
+** ^The value returned by sqlite3_msize(X) might be larger than the number
+** of bytes requested when X was allocated.  ^If X is a NULL pointer then
+** sqlite3_msize(X) returns zero.  If X points to something that is not
+** the beginning of memory allocation, or if it points to a formerly
+** valid memory allocation that has now been freed, then the behavior
+** of sqlite3_msize(X) is undefined and possibly harmful.
+**
+** ^The memory returned by sqlite3_malloc(), sqlite3_realloc(),
+** sqlite3_malloc64(), and sqlite3_realloc64()
 ** is always aligned to at least an 8 byte boundary, or to a
 ** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time
 ** option is used.
@@ -2474,8 +2840,11 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 ** [sqlite3_free()] or [sqlite3_realloc()].
 */
 SQLITE_API void *sqlite3_malloc(int);
+SQLITE_API void *sqlite3_malloc64(sqlite3_uint64);
 SQLITE_API void *sqlite3_realloc(void*, int);
+SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64);
 SQLITE_API void sqlite3_free(void*);
+SQLITE_API sqlite3_uint64 sqlite3_msize(void*);
 
 /*
 ** CAPI3REF: Memory Allocator Statistics
@@ -2513,13 +2882,14 @@ SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag);
 ** applications to access the same PRNG for other purposes.
 **
 ** ^A call to this routine stores N bytes of randomness into buffer P.
-** ^If N is less than one, then P can be a NULL pointer.
+** ^The P parameter can be a NULL pointer.
 **
 ** ^If this routine has not been previously called or if the previous
-** call had N less than one, then the PRNG is seeded using randomness
-** obtained from the xRandomness method of the default [sqlite3_vfs] object.
-** ^If the previous call to this routine had an N of 1 or more then
-** the pseudo-randomness is generated
+** call had N less than one or a NULL pointer for P, then the PRNG is
+** seeded using randomness obtained from the xRandomness method of
+** the default [sqlite3_vfs] object.
+** ^If the previous call to this routine had an N of 1 or more and a
+** non-NULL P then the pseudo-randomness is generated
 ** internally and without recourse to the [sqlite3_vfs] xRandomness
 ** method.
 */
@@ -2527,6 +2897,7 @@ SQLITE_API void sqlite3_randomness(int N, void *P);
 
 /*
 ** CAPI3REF: Compile-Time Authorization Callbacks
+** METHOD: sqlite3
 **
 ** ^This routine registers an authorizer callback with a particular
 ** [database connection], supplied in the first argument.
@@ -2683,6 +3054,10 @@ SQLITE_API int sqlite3_set_authorizer(
 
 /*
 ** CAPI3REF: Tracing And Profiling Functions
+** METHOD: sqlite3
+**
+** These routines are deprecated. Use the [sqlite3_trace_v2()] interface
+** instead of the routines described here.
 **
 ** These routines register callback functions that can be used for
 ** tracing and profiling the execution of SQL statements.
@@ -2709,12 +3084,107 @@ SQLITE_API int sqlite3_set_authorizer(
 ** sqlite3_profile() function is considered experimental and is
 ** subject to change in future versions of SQLite.
 */
-SQLITE_API void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*);
-SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*,
+SQLITE_API SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*,
+   void(*xTrace)(void*,const char*), void*);
+SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
    void(*xProfile)(void*,const char*,sqlite3_uint64), void*);
 
+/*
+** CAPI3REF: SQL Trace Event Codes
+** KEYWORDS: SQLITE_TRACE
+**
+** These constants identify classes of events that can be monitored
+** using the [sqlite3_trace_v2()] tracing logic.  The third argument
+** to [sqlite3_trace_v2()] is an OR-ed combination of one or more of
+** the following constants.  ^The first argument to the trace callback
+** is one of the following constants.
+**
+** New tracing constants may be added in future releases.
+**
+** ^A trace callback has four arguments: xCallback(T,C,P,X).
+** ^The T argument is one of the integer type codes above.
+** ^The C argument is a copy of the context pointer passed in as the
+** fourth argument to [sqlite3_trace_v2()].
+** The P and X arguments are pointers whose meanings depend on T.
+**
+** 
      
+** [[SQLITE_TRACE_STMT]] 
      SQLITE_TRACE_STMT
      
+** 
      ^An SQLITE_TRACE_STMT callback is invoked when a prepared statement
+** first begins running and possibly at other times during the
+** execution of the prepared statement, such as at the start of each
+** trigger subprogram. ^The P argument is a pointer to the
+** [prepared statement]. ^The X argument is a pointer to a string which
+** is the unexpanded SQL text of the prepared statement or an SQL comment 
+** that indicates the invocation of a trigger.  ^The callback can compute
+** the same text that would have been returned by the legacy [sqlite3_trace()]
+** interface by using the X argument when X begins with "--" and invoking
+** [sqlite3_expanded_sql(P)] otherwise.
+**
+** [[SQLITE_TRACE_PROFILE]] 
      SQLITE_TRACE_PROFILE
      
+** 
      ^An SQLITE_TRACE_PROFILE callback provides approximately the same
+** information as is provided by the [sqlite3_profile()] callback.
+** ^The P argument is a pointer to the [prepared statement] and the
+** X argument points to a 64-bit integer which is the estimated of
+** the number of nanosecond that the prepared statement took to run.
+** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes.
+**
+** [[SQLITE_TRACE_ROW]] 
      SQLITE_TRACE_ROW
      
+** 
      ^An SQLITE_TRACE_ROW callback is invoked whenever a prepared
+** statement generates a single row of result.  
+** ^The P argument is a pointer to the [prepared statement] and the
+** X argument is unused.
+**
+** [[SQLITE_TRACE_CLOSE]] 
      SQLITE_TRACE_CLOSE
      
+** 
      ^An SQLITE_TRACE_CLOSE callback is invoked when a database
+** connection closes.
+** ^The P argument is a pointer to the [database connection] object
+** and the X argument is unused.
+** 
      
+*/
+#define SQLITE_TRACE_STMT       0x01
+#define SQLITE_TRACE_PROFILE    0x02
+#define SQLITE_TRACE_ROW        0x04
+#define SQLITE_TRACE_CLOSE      0x08
+
+/*
+** CAPI3REF: SQL Trace Hook
+** METHOD: sqlite3
+**
+** ^The sqlite3_trace_v2(D,M,X,P) interface registers a trace callback
+** function X against [database connection] D, using property mask M
+** and context pointer P.  ^If the X callback is
+** NULL or if the M mask is zero, then tracing is disabled.  The
+** M argument should be the bitwise OR-ed combination of
+** zero or more [SQLITE_TRACE] constants.
+**
+** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides 
+** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2().
+**
+** ^The X callback is invoked whenever any of the events identified by 
+** mask M occur.  ^The integer return value from the callback is currently
+** ignored, though this may change in future releases.  Callback
+** implementations should return zero to ensure future compatibility.
+**
+** ^A trace callback is invoked with four arguments: callback(T,C,P,X).
+** ^The T argument is one of the [SQLITE_TRACE]
+** constants to indicate why the callback was invoked.
+** ^The C argument is a copy of the context pointer.
+** The P and X arguments are pointers whose meanings depend on T.
+**
+** The sqlite3_trace_v2() interface is intended to replace the legacy
+** interfaces [sqlite3_trace()] and [sqlite3_profile()], both of which
+** are deprecated.
+*/
+SQLITE_API int sqlite3_trace_v2(
+  sqlite3*,
+  unsigned uMask,
+  int(*xCallback)(unsigned,void*,void*,void*),
+  void *pCtx
+);
+
 /*
 ** CAPI3REF: Query Progress Callbacks
+** METHOD: sqlite3
 **
 ** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback
 ** function X to be invoked periodically during long running calls to
@@ -2748,6 +3218,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 
 /*
 ** CAPI3REF: Opening A New Database Connection
+** CONSTRUCTOR: sqlite3
 **
 ** ^These routines open an SQLite database file as specified by the 
 ** filename argument. ^The filename argument is interpreted as UTF-8 for
@@ -2762,9 +3233,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** an English language description of the error following a failure of any
 ** of the sqlite3_open() routines.
 **
-** ^The default encoding for the database will be UTF-8 if
-** sqlite3_open() or sqlite3_open_v2() is called and
-** UTF-16 in the native byte order if sqlite3_open16() is used.
+** ^The default encoding will be UTF-8 for databases created using
+** sqlite3_open() or sqlite3_open_v2().  ^The default encoding for databases
+** created using sqlite3_open16() will be UTF-16 in the native byte order.
 **
 ** Whether or not an error occurs when it is opened, resources
 ** associated with the [database connection] handle should be released by
@@ -2852,13 +3323,14 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** then it is interpreted as an absolute path. ^If the path does not begin 
 ** with a '/' (meaning that the authority section is omitted from the URI)
 ** then the path is interpreted as a relative path. 
-** ^On windows, the first component of an absolute path 
-** is a drive specification (e.g. "C:").
+** ^(On windows, the first component of an absolute path 
+** is a drive specification (e.g. "C:").)^
 **
 ** [[core URI query parameters]]
 ** The query component of a URI may contain parameters that are interpreted
 ** either by SQLite itself, or by a [VFS | custom VFS implementation].
-** SQLite interprets the following three query parameters:
+** SQLite and its built-in [VFSes] interpret the
+** following query parameters:
 **
 ** 
        
 **   
      •  vfs: ^The "vfs" parameter may be used to specify the name of
@@ -2893,11 +3365,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 **     a URI filename, its value overrides any behavior requested by setting
 **     SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag.
 **
-**  
      •  psow: ^The psow parameter may be "true" (or "on" or "yes" or
-**     "1") or "false" (or "off" or "no" or "0") to indicate that the
+**  
      •  psow: ^The psow parameter indicates whether or not the
 **     [powersafe overwrite] property does or does not apply to the
-**     storage media on which the database file resides.  ^The psow query
-**     parameter only works for the built-in unix and Windows VFSes.
+**     storage media on which the database file resides.
 **
 **  
      •  nolock: ^The nolock parameter is a boolean query parameter
 **     which if set disables file locking in rollback journal modes.  This
@@ -3034,12 +3504,15 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int
 
 /*
 ** CAPI3REF: Error Codes And Messages
+** METHOD: sqlite3
 **
-** ^The sqlite3_errcode() interface returns the numeric [result code] or
-** [extended result code] for the most recent failed sqlite3_* API call
-** associated with a [database connection]. If a prior API call failed
-** but the most recent API call succeeded, the return value from
-** sqlite3_errcode() is undefined.  ^The sqlite3_extended_errcode()
+** ^If the most recent sqlite3_* API call associated with 
+** [database connection] D failed, then the sqlite3_errcode(D) interface
+** returns the numeric [result code] or [extended result code] for that
+** API call.
+** If the most recent API call was successful,
+** then the return value from sqlite3_errcode() is undefined.
+** ^The sqlite3_extended_errcode()
 ** interface is the same except that it always returns the 
 ** [extended result code] even when extended result codes are
 ** disabled.
@@ -3077,33 +3550,34 @@ SQLITE_API const void *sqlite3_errmsg16(sqlite3*);
 SQLITE_API const char *sqlite3_errstr(int);
 
 /*
-** CAPI3REF: SQL Statement Object
+** CAPI3REF: Prepared Statement Object
 ** KEYWORDS: {prepared statement} {prepared statements}
 **
-** An instance of this object represents a single SQL statement.
-** This object is variously known as a "prepared statement" or a
-** "compiled SQL statement" or simply as a "statement".
+** An instance of this object represents a single SQL statement that
+** has been compiled into binary form and is ready to be evaluated.
 **
-** The life of a statement object goes something like this:
+** Think of each SQL statement as a separate computer program.  The
+** original SQL text is source code.  A prepared statement object 
+** is the compiled object code.  All SQL must be converted into a
+** prepared statement before it can be run.
+**
+** The life-cycle of a prepared statement object usually goes like this:
 **
 ** 
          
-** 
        1.  Create the object using [sqlite3_prepare_v2()] or a related
-**      function.
-** 
        2.  Bind values to [host parameters] using the sqlite3_bind_*()
+** 
        3.  Create the prepared statement object using [sqlite3_prepare_v2()].
+** 
        4.  Bind values to [parameters] using the sqlite3_bind_*()
 **      interfaces.
 ** 
        5.  Run the SQL by calling [sqlite3_step()] one or more times.
-** 
        6.  Reset the statement using [sqlite3_reset()] then go back
+** 
        7.  Reset the prepared statement using [sqlite3_reset()] then go back
 **      to step 2.  Do this zero or more times.
 ** 
        8.  Destroy the object using [sqlite3_finalize()].
 ** 
        
-**
-** Refer to documentation on individual methods above for additional
-** information.
 */
 typedef struct sqlite3_stmt sqlite3_stmt;
 
 /*
 ** CAPI3REF: Run-time Limits
+** METHOD: sqlite3
 **
 ** ^(This interface allows the size of various constructs to be limited
 ** on a connection by connection basis.  The first parameter is the
@@ -3193,6 +3667,10 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal);
 **
 ** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
        SQLITE_LIMIT_TRIGGER_DEPTH
        
 ** 
        The maximum depth of recursion for triggers.
        )^
+**
+** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
        SQLITE_LIMIT_WORKER_THREADS
        
+** 
        The maximum number of auxiliary worker threads that a single
+** [prepared statement] may start.
        )^
 ** 
 */
 #define SQLITE_LIMIT_LENGTH                    0
@@ -3206,10 +3684,13 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal);
 #define SQLITE_LIMIT_LIKE_PATTERN_LENGTH       8
 #define SQLITE_LIMIT_VARIABLE_NUMBER           9
 #define SQLITE_LIMIT_TRIGGER_DEPTH            10
+#define SQLITE_LIMIT_WORKER_THREADS           11
 
 /*
 ** CAPI3REF: Compiling An SQL Statement
 ** KEYWORDS: {SQL statement compiler}
+** METHOD: sqlite3
+** CONSTRUCTOR: sqlite3_stmt
 **
 ** To execute an SQL query, it must first be compiled into a byte-code
 ** program using one of these routines.
@@ -3223,16 +3704,14 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal);
 ** interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2()
 ** use UTF-16.
 **
-** ^If the nByte argument is less than zero, then zSql is read up to the
-** first zero terminator. ^If nByte is non-negative, then it is the maximum
-** number of  bytes read from zSql.  ^When nByte is non-negative, the
-** zSql string ends at either the first '\000' or '\u0000' character or
-** the nByte-th byte, whichever comes first. If the caller knows
-** that the supplied string is nul-terminated, then there is a small
-** performance advantage to be gained by passing an nByte parameter that
-** is equal to the number of bytes in the input string including
-** the nul-terminator bytes as this saves SQLite from having to
-** make a copy of the input string.
+** ^If the nByte argument is negative, then zSql is read up to the
+** first zero terminator. ^If nByte is positive, then it is the
+** number of bytes read from zSql.  ^If nByte is zero, then no prepared
+** statement is generated.
+** If the caller knows that the supplied string is nul-terminated, then
+** there is a small performance advantage to passing an nByte parameter that
+** is the number of bytes in the input string including
+** the nul-terminator.
 **
 ** ^If pzTail is not NULL then *pzTail is made to point to the first byte
 ** past the end of the first SQL statement in zSql.  These routines only
@@ -3319,15 +3798,41 @@ SQLITE_API int sqlite3_prepare16_v2(
 
 /*
 ** CAPI3REF: Retrieving Statement SQL
+** METHOD: sqlite3_stmt
 **
-** ^This interface can be used to retrieve a saved copy of the original
-** SQL text used to create a [prepared statement] if that statement was
-** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()].
+** ^The sqlite3_sql(P) interface returns a pointer to a copy of the UTF-8
+** SQL text used to create [prepared statement] P if P was
+** created by either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()].
+** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8
+** string containing the SQL text of prepared statement P with
+** [bound parameters] expanded.
+**
+** ^(For example, if a prepared statement is created using the SQL
+** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345
+** and parameter :xyz is unbound, then sqlite3_sql() will return
+** the original string, "SELECT $abc,:xyz" but sqlite3_expanded_sql()
+** will return "SELECT 2345,NULL".)^
+**
+** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory
+** is available to hold the result, or if the result would exceed the
+** the maximum string length determined by the [SQLITE_LIMIT_LENGTH].
+**
+** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of
+** bound parameter expansions.  ^The [SQLITE_OMIT_TRACE] compile-time
+** option causes sqlite3_expanded_sql() to always return NULL.
+**
+** ^The string returned by sqlite3_sql(P) is managed by SQLite and is
+** automatically freed when the prepared statement is finalized.
+** ^The string returned by sqlite3_expanded_sql(P), on the other hand,
+** is obtained from [sqlite3_malloc()] and must be free by the application
+** by passing it to [sqlite3_free()].
 */
 SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt);
+SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt);
 
 /*
 ** CAPI3REF: Determine If An SQL Statement Writes The Database
+** METHOD: sqlite3_stmt
 **
 ** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if
 ** and only if the [prepared statement] X makes no direct changes to
@@ -3359,10 +3864,12 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
 
 /*
 ** CAPI3REF: Determine If A Prepared Statement Has Been Reset
+** METHOD: sqlite3_stmt
 **
 ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the
 ** [prepared statement] S has been stepped at least once using 
-** [sqlite3_step(S)] but has not run to completion and/or has not 
+** [sqlite3_step(S)] but has neither run to completion (returned
+** [SQLITE_DONE] from [sqlite3_step(S)]) nor
 ** been reset using [sqlite3_reset(S)].  ^The sqlite3_stmt_busy(S)
 ** interface returns false if S is a NULL pointer.  If S is not a 
 ** NULL pointer and is not a pointer to a valid [prepared statement]
@@ -3389,7 +3896,9 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*);
 ** Some interfaces require a protected sqlite3_value.  Other interfaces
 ** will accept either a protected or an unprotected sqlite3_value.
 ** Every interface that accepts sqlite3_value arguments specifies
-** whether or not it requires a protected sqlite3_value.
+** whether or not it requires a protected sqlite3_value.  The
+** [sqlite3_value_dup()] interface can be used to construct a new 
+** protected sqlite3_value from an unprotected sqlite3_value.
 **
 ** The terms "protected" and "unprotected" refer to whether or not
 ** a mutex is held.  An internal mutex is held for a protected
@@ -3433,6 +3942,7 @@ typedef struct sqlite3_context sqlite3_context;
 ** CAPI3REF: Binding Values To Prepared Statements
 ** KEYWORDS: {host parameter} {host parameters} {host parameter name}
 ** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding}
+** METHOD: sqlite3_stmt
 **
 ** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants,
 ** literals may be replaced by a [parameter] that matches one of following
@@ -3479,18 +3989,18 @@ typedef struct sqlite3_context sqlite3_context;
 ** If the fourth parameter to sqlite3_bind_blob() is negative, then
 ** the behavior is undefined.
 ** If a non-negative fourth parameter is provided to sqlite3_bind_text()
-** or sqlite3_bind_text16() then that parameter must be the byte offset
+** or sqlite3_bind_text16() or sqlite3_bind_text64() then
+** that parameter must be the byte offset
 ** where the NUL terminator would occur assuming the string were NUL
 ** terminated.  If any NUL characters occur at byte offsets less than 
 ** the value of the fourth parameter then the resulting string value will
 ** contain embedded NULs.  The result of expressions involving strings
 ** with embedded NULs is undefined.
 **
-** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and
-** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or
+** ^The fifth argument to the BLOB and string binding interfaces
+** is a destructor used to dispose of the BLOB or
 ** string after SQLite has finished with it.  ^The destructor is called
-** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(),
-** sqlite3_bind_text(), or sqlite3_bind_text16() fails.  
+** to dispose of the BLOB or string even if the call to bind API fails.
 ** ^If the fifth argument is
 ** the special value [SQLITE_STATIC], then SQLite assumes that the
 ** information is in static, unmanaged space and does not need to be freed.
@@ -3498,6 +4008,14 @@ typedef struct sqlite3_context sqlite3_context;
 ** SQLite makes its own private copy of the data immediately, before
 ** the sqlite3_bind_*() routine returns.
 **
+** ^The sixth argument to sqlite3_bind_text64() must be one of
+** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]
+** to specify the encoding of the text in the third parameter.  If
+** the sixth argument to sqlite3_bind_text64() is not one of the
+** allowed values shown above, or if the text encoding is different
+** from the encoding specified by the sixth parameter, then the behavior
+** is undefined.
+**
 ** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that
 ** is filled with zeroes.  ^A zeroblob uses a fixed amount of memory
 ** (just an integer to hold its size) while it is being processed.
@@ -3518,6 +4036,9 @@ typedef struct sqlite3_context sqlite3_context;
 **
 ** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an
 ** [error code] if anything goes wrong.
+** ^[SQLITE_TOOBIG] might be returned if the size of a string or BLOB
+** exceeds limits imposed by [sqlite3_limit]([SQLITE_LIMIT_LENGTH]) or
+** [SQLITE_MAX_LENGTH].
 ** ^[SQLITE_RANGE] is returned if the parameter
 ** index is out of range.  ^[SQLITE_NOMEM] is returned if malloc() fails.
 **
@@ -3525,17 +4046,23 @@ typedef struct sqlite3_context sqlite3_context;
 ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()].
 */
 SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*));
+SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64,
+                        void(*)(void*));
 SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double);
 SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int);
 SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64);
 SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int);
-SQLITE_API int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*));
+SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*));
 SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*));
+SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64,
+                         void(*)(void*), unsigned char encoding);
 SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
 SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
+SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64);
 
 /*
 ** CAPI3REF: Number Of SQL Parameters
+** METHOD: sqlite3_stmt
 **
 ** ^This routine can be used to find the number of [SQL parameters]
 ** in a [prepared statement].  SQL parameters are tokens of the
@@ -3556,6 +4083,7 @@ SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*);
 
 /*
 ** CAPI3REF: Name Of A Host Parameter
+** METHOD: sqlite3_stmt
 **
 ** ^The sqlite3_bind_parameter_name(P,N) interface returns
 ** the name of the N-th [SQL parameter] in the [prepared statement] P.
@@ -3583,6 +4111,7 @@ SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int);
 
 /*
 ** CAPI3REF: Index Of A Parameter With A Given Name
+** METHOD: sqlite3_stmt
 **
 ** ^Return the index of an SQL parameter given its name.  ^The
 ** index value returned is suitable for use as the second
@@ -3593,12 +4122,13 @@ SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int);
 **
 ** See also: [sqlite3_bind_blob|sqlite3_bind()],
 ** [sqlite3_bind_parameter_count()], and
-** [sqlite3_bind_parameter_index()].
+** [sqlite3_bind_parameter_name()].
 */
 SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName);
 
 /*
 ** CAPI3REF: Reset All Bindings On A Prepared Statement
+** METHOD: sqlite3_stmt
 **
 ** ^Contrary to the intuition of many, [sqlite3_reset()] does not reset
 ** the [sqlite3_bind_blob | bindings] on a [prepared statement].
@@ -3608,6 +4138,7 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*);
 
 /*
 ** CAPI3REF: Number Of Columns In A Result Set
+** METHOD: sqlite3_stmt
 **
 ** ^Return the number of columns in the result set returned by the
 ** [prepared statement]. ^This routine returns 0 if pStmt is an SQL
@@ -3619,6 +4150,7 @@ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt);
 
 /*
 ** CAPI3REF: Column Names In A Result Set
+** METHOD: sqlite3_stmt
 **
 ** ^These routines return the name assigned to a particular column
 ** in the result set of a [SELECT] statement.  ^The sqlite3_column_name()
@@ -3648,6 +4180,7 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N);
 
 /*
 ** CAPI3REF: Source Of Data In A Query Result
+** METHOD: sqlite3_stmt
 **
 ** ^These routines provide a means to determine the database, table, and
 ** table column that is the origin of a particular result column in
@@ -3700,6 +4233,7 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int);
 
 /*
 ** CAPI3REF: Declared Datatype Of A Query Result
+** METHOD: sqlite3_stmt
 **
 ** ^(The first parameter is a [prepared statement].
 ** If this statement is a [SELECT] statement and the Nth column of the
@@ -3732,6 +4266,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 
 /*
 ** CAPI3REF: Evaluate An SQL Statement
+** METHOD: sqlite3_stmt
 **
 ** After a [prepared statement] has been prepared using either
 ** [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or one of the legacy
@@ -3788,7 +4323,8 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 ** other than [SQLITE_ROW] before any subsequent invocation of
 ** sqlite3_step().  Failure to reset the prepared statement using 
 ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from
-** sqlite3_step().  But after version 3.6.23.1, sqlite3_step() began
+** sqlite3_step().  But after [version 3.6.23.1] ([dateof:3.6.23.1],
+** sqlite3_step() began
 ** calling [sqlite3_reset()] automatically in this circumstance rather
 ** than returning [SQLITE_MISUSE].  This is not considered a compatibility
 ** break because any application that ever receives an SQLITE_MISUSE error
@@ -3811,6 +4347,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*);
 
 /*
 ** CAPI3REF: Number of columns in a result set
+** METHOD: sqlite3_stmt
 **
 ** ^The sqlite3_data_count(P) interface returns the number of columns in the
 ** current row of the result set of [prepared statement] P.
@@ -3864,8 +4401,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 /*
 ** CAPI3REF: Result Values From A Query
 ** KEYWORDS: {column access functions}
-**
-** These routines form the "result set" interface.
+** METHOD: sqlite3_stmt
 **
 ** ^These routines return information about a single column of the current
 ** result row of a query.  ^In every case the first argument is a pointer
@@ -3926,13 +4462,14 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** even empty strings, are always zero-terminated.  ^The return
 ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer.
 **
-** ^The object returned by [sqlite3_column_value()] is an
-** [unprotected sqlite3_value] object.  An unprotected sqlite3_value object
-** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()].
+** Warning: ^The object returned by [sqlite3_column_value()] is an
+** [unprotected sqlite3_value] object.  In a multithreaded environment,
+** an unprotected sqlite3_value object may only be used safely with
+** [sqlite3_bind_value()] and [sqlite3_result_value()].
 ** If the [unprotected sqlite3_value] object returned by
 ** [sqlite3_column_value()] is used in any other way, including calls
 ** to routines like [sqlite3_value_int()], [sqlite3_value_text()],
-** or [sqlite3_value_bytes()], then the behavior is undefined.
+** or [sqlite3_value_bytes()], the behavior is not threadsafe.
 **
 ** These routines attempt to convert the value where appropriate.  ^For
 ** example, if the internal representation is FLOAT and a text result
@@ -3963,12 +4500,6 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** 
 **
    )^ ** -** The table above makes reference to standard C library functions atoi() -** and atof(). SQLite does not really use these functions. It has its -** own equivalent internal routines. The atoi() and atof() names are -** used in the table for brevity and because they are familiar to most -** C programmers. -** ** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. @@ -3993,7 +4524,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** The safest and easiest to remember policy is to invoke these routines +** The safest policy is to invoke these routines ** in one of the following ways: ** **
      @@ -4013,7 +4544,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ^The pointers returned are valid until a type conversion occurs as ** described above, or until [sqlite3_step()] or [sqlite3_reset()] or ** [sqlite3_finalize()] is called. ^The memory space used to hold strings -** and BLOBs is freed automatically. Do not pass the pointers returned +** and BLOBs is freed automatically. Do not pass the pointers returned ** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** @@ -4036,6 +4567,7 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); /* ** CAPI3REF: Destroy A Prepared Statement Object +** DESTRUCTOR: sqlite3_stmt ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. ** ^If the most recent evaluation of the statement encountered no errors @@ -4063,6 +4595,7 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); /* ** CAPI3REF: Reset A Prepared Statement Object +** METHOD: sqlite3_stmt ** ** The sqlite3_reset() function is called to reset a [prepared statement] ** object back to its initial state, ready to be re-executed. @@ -4092,6 +4625,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** KEYWORDS: {function creation routines} ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} +** METHOD: sqlite3 ** ** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior @@ -4222,9 +4756,9 @@ SQLITE_API int sqlite3_create_function_v2( ** These constant define integer codes that represent the various ** text encodings supported by SQLite. */ -#define SQLITE_UTF8 1 -#define SQLITE_UTF16LE 2 -#define SQLITE_UTF16BE 3 +#define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ +#define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ +#define SQLITE_UTF16BE 3 /* IMP: R-51971-34154 */ #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ @@ -4246,8 +4780,8 @@ SQLITE_API int sqlite3_create_function_v2( ** These functions are [deprecated]. In order to maintain ** backwards compatibility with older code, these functions continue ** to be supported. However, new applications should avoid -** the use of these functions. To help encourage people to avoid -** using these functions, we are not going to tell you what they do. +** the use of these functions. To encourage programmers to avoid +** these functions, we will not explain what they do. */ #ifndef SQLITE_OMIT_DEPRECATED SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); @@ -4260,11 +4794,12 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 #endif /* -** CAPI3REF: Obtaining SQL Function Parameter Values +** CAPI3REF: Obtaining SQL Values +** METHOD: sqlite3_value ** ** The C-language implementation of SQL functions and aggregates uses ** this set of interface routines to access the parameter values on -** the function or aggregate. +** the function or aggregate. ** ** The xFunc (for scalar functions) or xStep (for aggregates) parameters ** to [sqlite3_create_function()] and [sqlite3_create_function16()] @@ -4279,7 +4814,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** object results in undefined behavior. ** ** ^These routines work just like the corresponding [column access functions] -** except that these routines take a single [protected sqlite3_value] object +** except that these routines take a single [protected sqlite3_value] object ** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. ** ** ^The sqlite3_value_text16() interface extracts a UTF-16 string @@ -4317,8 +4852,42 @@ SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); SQLITE_API int sqlite3_value_type(sqlite3_value*); SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); +/* +** CAPI3REF: Finding The Subtype Of SQL Values +** METHOD: sqlite3_value +** +** The sqlite3_value_subtype(V) function returns the subtype for +** an [application-defined SQL function] argument V. The subtype +** information can be used to pass a limited amount of context from +** one SQL function to another. Use the [sqlite3_result_subtype()] +** routine to set the subtype for the return value of an SQL function. +** +** SQLite makes no use of subtype itself. It merely passes the subtype +** from the result of one [application-defined SQL function] into the +** input of another. +*/ +SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); + +/* +** CAPI3REF: Copy And Free SQL Values +** METHOD: sqlite3_value +** +** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] +** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** is a [protected sqlite3_value] object even if the input is not. +** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a +** memory allocation fails. +** +** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object +** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer +** then sqlite3_value_free(V) is a harmless no-op. +*/ +SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value*); +SQLITE_API void sqlite3_value_free(sqlite3_value*); + /* ** CAPI3REF: Obtain Aggregate Function Context +** METHOD: sqlite3_context ** ** Implementations of aggregate SQL functions use this ** routine to allocate memory for storing their state. @@ -4363,6 +4932,7 @@ SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); /* ** CAPI3REF: User Data For Functions +** METHOD: sqlite3_context ** ** ^The sqlite3_user_data() interface returns a copy of ** the pointer that was the pUserData parameter (the 5th parameter) @@ -4377,6 +4947,7 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context*); /* ** CAPI3REF: Database Connection For Functions +** METHOD: sqlite3_context ** ** ^The sqlite3_context_db_handle() interface returns a copy of ** the pointer to the [database connection] (the 1st parameter) @@ -4388,6 +4959,7 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); /* ** CAPI3REF: Function Auxiliary Data +** METHOD: sqlite3_context ** ** These functions may be used by (non-aggregate) SQL functions to ** associate metadata with argument values. If the same value is passed to @@ -4415,12 +4987,13 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** SQLite will invoke the destructor function X with parameter P exactly ** once, when the metadata is discarded. ** SQLite is free to discard the metadata at any time, including:
        -**
      • when the corresponding function parameter changes, or -**
      • when [sqlite3_reset()] or [sqlite3_finalize()] is called for the -** SQL statement, or -**
      • when sqlite3_set_auxdata() is invoked again on the same parameter, or -**
      • during the original sqlite3_set_auxdata() call when a memory -** allocation error occurs.
      )^ +**
    • ^(when the corresponding function parameter changes)^, or +**
    • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement)^, or +**
    • ^(when sqlite3_set_auxdata() is invoked again on the same +** parameter)^, or +**
    • ^(during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.)^
    ** ** Note the last bullet in particular. The destructor X in ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the @@ -4460,6 +5033,7 @@ typedef void (*sqlite3_destructor_type)(void*); /* ** CAPI3REF: Setting The Result Of An SQL Function +** METHOD: sqlite3_context ** ** These routines are used by the xFunc or xFinal callbacks that ** implement SQL functions and aggregates. See @@ -4475,9 +5049,9 @@ typedef void (*sqlite3_destructor_type)(void*); ** to by the second parameter and which is N bytes long where N is the ** third parameter. ** -** ^The sqlite3_result_zeroblob() interfaces set the result of -** the application-defined function to be a BLOB containing all zero -** bytes and N bytes in size, where N is the value of the 2nd parameter. +** ^The sqlite3_result_zeroblob(C,N) and sqlite3_result_zeroblob64(C,N) +** interfaces set the result of the application-defined function to be +** a BLOB containing all zero bytes and N bytes in size. ** ** ^The sqlite3_result_double() interface sets the result from ** an application-defined function to be a floating point value specified @@ -4526,6 +5100,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^The sqlite3_result_text64() interface sets the return value of an +** application-defined function to be a text string in an encoding +** specified by the fifth (and last) parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces @@ -4555,7 +5133,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** from [sqlite3_malloc()] before it returns. ** ** ^The sqlite3_result_value() interface sets the result of -** the application-defined function to be a copy the +** the application-defined function to be a copy of the ** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The ** sqlite3_result_value() interface makes a copy of the [sqlite3_value] ** so that the [sqlite3_value] specified in the parameter may change or @@ -4569,6 +5147,8 @@ typedef void (*sqlite3_destructor_type)(void*); ** the [sqlite3_context] pointer, the results are undefined. */ SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); SQLITE_API void sqlite3_result_double(sqlite3_context*, double); SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); @@ -4579,14 +5159,33 @@ SQLITE_API void sqlite3_result_int(sqlite3_context*, int); SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); SQLITE_API void sqlite3_result_null(sqlite3_context*); SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, + void(*)(void*), unsigned char encoding); SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); + + +/* +** CAPI3REF: Setting The Subtype Of An SQL Function +** METHOD: sqlite3_context +** +** The sqlite3_result_subtype(C,T) function causes the subtype of +** the result from the [application-defined SQL function] with +** [sqlite3_context] C to be the value T. Only the lower 8 bits +** of the subtype T are preserved in current versions of SQLite; +** higher order bits are discarded. +** The number of subtype bytes preserved by SQLite might increase +** in future releases of SQLite. +*/ +SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); /* ** CAPI3REF: Define New Collating Sequences +** METHOD: sqlite3 ** ** ^These functions add, remove, or modify a [collation] associated ** with the [database connection] specified as the first argument. @@ -4689,6 +5288,7 @@ SQLITE_API int sqlite3_create_collation16( /* ** CAPI3REF: Collation Needed Callbacks +** METHOD: sqlite3 ** ** ^To avoid having to register all collation sequences before a database ** can be used, a single callback function may be registered with the @@ -4896,6 +5496,7 @@ SQLITE_API char *sqlite3_data_directory; /* ** CAPI3REF: Test For Auto-Commit Mode ** KEYWORDS: {autocommit mode} +** METHOD: sqlite3 ** ** ^The sqlite3_get_autocommit() interface returns non-zero or ** zero if the given database connection is or is not in autocommit mode, @@ -4918,6 +5519,7 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*); /* ** CAPI3REF: Find The Database Handle Of A Prepared Statement +** METHOD: sqlite3_stmt ** ** ^The sqlite3_db_handle interface returns the [database connection] handle ** to which a [prepared statement] belongs. ^The [database connection] @@ -4930,6 +5532,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); /* ** CAPI3REF: Return The Filename For A Database Connection +** METHOD: sqlite3 ** ** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename ** associated with database N of connection D. ^The main database file @@ -4946,6 +5549,7 @@ SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only +** METHOD: sqlite3 ** ** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N ** of connection D is read-only, 0 if it is read/write, or -1 if N is not @@ -4955,6 +5559,7 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Find the next prepared statement +** METHOD: sqlite3 ** ** ^This interface returns a pointer to the next [prepared statement] after ** pStmt associated with the [database connection] pDb. ^If pStmt is NULL @@ -4970,6 +5575,7 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); /* ** CAPI3REF: Commit And Rollback Notification Callbacks +** METHOD: sqlite3 ** ** ^The sqlite3_commit_hook() interface registers a callback ** function to be invoked whenever a transaction is [COMMIT | committed]. @@ -5019,11 +5625,12 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); /* ** CAPI3REF: Data Change Notification Callbacks +** METHOD: sqlite3 ** ** ^The sqlite3_update_hook() interface registers a callback function ** with the [database connection] identified by the first argument ** to be invoked whenever a row is updated, inserted or deleted in -** a rowid table. +** a [rowid table]. ** ^Any callback set by a previous call to this function ** for the same database connection is overridden. ** @@ -5062,8 +5669,8 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** on the same [database connection] D, or NULL for ** the first call on D. ** -** See also the [sqlite3_commit_hook()] and [sqlite3_rollback_hook()] -** interfaces. +** See also the [sqlite3_commit_hook()], [sqlite3_rollback_hook()], +** and [sqlite3_preupdate_hook()] interfaces. */ SQLITE_API void *sqlite3_update_hook( sqlite3*, @@ -5080,7 +5687,8 @@ SQLITE_API void *sqlite3_update_hook( ** and disabled if the argument is false.)^ ** ** ^Cache sharing is enabled and disabled for an entire process. -** This is a change as of SQLite version 3.5.0. In prior versions of SQLite, +** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). +** In prior versions of SQLite, ** sharing was enabled or disabled for each thread separately. ** ** ^(The cache sharing mode set by this interface effects all subsequent @@ -5095,6 +5703,11 @@ SQLITE_API void *sqlite3_update_hook( ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** +** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0 +** and will always return SQLITE_MISUSE. On those systems, +** shared cache mode should be enabled per-database connection via +** [sqlite3_open_v2()] with [SQLITE_OPEN_SHAREDCACHE]. +** ** This interface is threadsafe on processors where writing a ** 32-bit integer is atomic. ** @@ -5120,6 +5733,7 @@ SQLITE_API int sqlite3_release_memory(int); /* ** CAPI3REF: Free Memory Used By A Database Connection +** METHOD: sqlite3 ** ** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap ** memory as possible from database connection D. Unlike the @@ -5168,7 +5782,8 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** from the heap. ** )^ ** -** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** Beginning with SQLite [version 3.7.3] ([dateof:3.7.3]), +** the soft heap limit is enforced ** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] ** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], ** the soft heap limit is enforced on every memory allocation. Without @@ -5197,21 +5812,29 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); /* ** CAPI3REF: Extract Metadata About A Column Of A Table +** METHOD: sqlite3 ** -** ^This routine returns metadata about a specific column of a specific -** database table accessible using the [database connection] handle -** passed as the first function argument. +** ^(The sqlite3_table_column_metadata(X,D,T,C,....) routine returns +** information about column C of table T in database D +** on [database connection] X.)^ ^The sqlite3_table_column_metadata() +** interface returns SQLITE_OK and fills in the non-NULL pointers in +** the final five arguments with appropriate values if the specified +** column exists. ^The sqlite3_table_column_metadata() interface returns +** SQLITE_ERROR and if the specified column does not exist. +** ^If the column-name parameter to sqlite3_table_column_metadata() is a +** NULL pointer, then this routine simply checks for the existence of the +** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it +** does not. ** ** ^The column is identified by the second, third and fourth parameters to -** this function. ^The second parameter is either the name of the database +** this function. ^(The second parameter is either the name of the database ** (i.e. "main", "temp", or an attached database) containing the specified -** table or NULL. ^If it is NULL, then all attached databases are searched +** table or NULL.)^ ^If it is NULL, then all attached databases are searched ** for the table using the same algorithm used by the database engine to ** resolve unqualified table references. ** ** ^The third and fourth parameters to this function are the table and column -** name of the desired column, respectively. Neither of these parameters -** may be NULL. +** name of the desired column, respectively. ** ** ^Metadata is returned by writing to the memory locations passed as the 5th ** and subsequent parameters to this function. ^Any of these arguments may be @@ -5230,16 +5853,17 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** )^ ** ** ^The memory pointed to by the character pointers returned for the -** declaration type and collation sequence is valid only until the next +** declaration type and collation sequence is valid until the next ** call to any SQLite API function. ** ** ^If the specified table is actually a view, an [error code] is returned. ** -** ^If the specified column is "rowid", "oid" or "_rowid_" and an +** ^If the specified column is "rowid", "oid" or "_rowid_" and the table +** is not a [WITHOUT ROWID] table and an ** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output ** parameters are set for the explicitly declared column. ^(If there is no -** explicitly declared [INTEGER PRIMARY KEY] column, then the output -** parameters are set as follows: +** [INTEGER PRIMARY KEY] column, then the outputs +** for the [rowid] are set as follows: ** ** 
     **     data type: "INTEGER"
@@ -5249,13 +5873,9 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N);
 **     auto increment: 0
 **
    )^ ** -** ^(This function may load one or more schemas from database files. If an -** error occurs during this process, or if the requested table or column -** cannot be found, an [error code] is returned and an error message left -** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^ -** -** ^This API is only available if the library was compiled with the -** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined. +** ^This function causes all database schemas to be read from disk and +** parsed, if that has not already been done, and returns an error if +** any errors are encountered while loading the schema. */ SQLITE_API int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ @@ -5271,6 +5891,7 @@ SQLITE_API int sqlite3_table_column_metadata( /* ** CAPI3REF: Load An Extension +** METHOD: sqlite3 ** ** ^This interface loads an SQLite extension library from the named file. ** @@ -5298,9 +5919,18 @@ SQLITE_API int sqlite3_table_column_metadata( ** should free this memory by calling [sqlite3_free()]. ** ** ^Extension loading must be enabled using -** [sqlite3_enable_load_extension()] prior to calling this API, +** [sqlite3_enable_load_extension()] or +** [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],1,NULL) +** prior to calling this API, ** otherwise an error will be returned. ** +** Security warning: It is recommended that the +** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this +** interface. The use of the [sqlite3_enable_load_extension()] interface +** should be avoided. This will keep the SQL function [load_extension()] +** disabled and prevent SQL injections from giving attackers +** access to extension loading capabilities. +** ** See also the [load_extension() SQL function]. */ SQLITE_API int sqlite3_load_extension( @@ -5312,6 +5942,7 @@ SQLITE_API int sqlite3_load_extension( /* ** CAPI3REF: Enable Or Disable Extension Loading +** METHOD: sqlite3 ** ** ^So as not to open security holes in older applications that are ** unprepared to deal with [extension loading], and as a means of disabling @@ -5322,6 +5953,17 @@ SQLITE_API int sqlite3_load_extension( ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. +** +** ^This interface enables or disables both the C-API +** [sqlite3_load_extension()] and the SQL function [load_extension()]. +** ^(Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) +** to enable or disable only the C-API.)^ +** +** Security warning: It is recommended that extension loading +** be disabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** rather than this interface, so the [load_extension()] SQL function +** remains disabled. This will prevent SQL injections from giving attackers +** access to extension loading capabilities. */ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); @@ -5335,7 +5977,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^(Even though the function prototype shows that xEntryPoint() takes ** no arguments and returns void, SQLite invokes xEntryPoint() with three -** arguments and expects and integer result as if the signature of the +** arguments and expects an integer result as if the signature of the ** entry point where as follows: ** ** 
    @@ -5361,7 +6003,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff);
 ** See also: [sqlite3_reset_auto_extension()]
 ** and [sqlite3_cancel_auto_extension()]
 */
-SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void));
+SQLITE_API int sqlite3_auto_extension(void(*xEntryPoint)(void));
 
 /*
 ** CAPI3REF: Cancel Automatic Extension Loading
@@ -5373,7 +6015,7 @@ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void));
 ** unregistered and it returns 0 if X was not on the list of initialization
 ** routines.
 */
-SQLITE_API int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void));
+SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void));
 
 /*
 ** CAPI3REF: Reset Automatic Extension Loading
@@ -5483,6 +6125,17 @@ struct sqlite3_module {
 ** ^Information about the ORDER BY clause is stored in aOrderBy[].
 ** ^Each term of aOrderBy records a column of the ORDER BY clause.
 **
+** The colUsed field indicates which columns of the virtual table may be
+** required by the current scan. Virtual table columns are numbered from
+** zero in the order in which they appear within the CREATE TABLE statement
+** passed to sqlite3_declare_vtab(). For the first 63 columns (columns 0-62),
+** the corresponding bit is set within the colUsed mask if the column may be
+** required by SQLite. If the table has at least 64 columns and any column
+** to the right of the first 63 is required, then bit 63 of colUsed is also
+** set. In other words, column iCol may be required if the expression
+** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to 
+** non-zero.
+**
 ** The [xBestIndex] method must fill aConstraintUsage[] with information
 ** about what parameters to pass to xFilter.  ^If argvIndex>0 then
 ** the right-hand side of the corresponding aConstraint[] is evaluated
@@ -5508,19 +6161,39 @@ struct sqlite3_module {
 ** ^The estimatedRows value is an estimate of the number of rows that
 ** will be returned by the strategy.
 **
+** The xBestIndex method may optionally populate the idxFlags field with a 
+** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag -
+** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite
+** assumes that the strategy may visit at most one row. 
+**
+** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then
+** SQLite also assumes that if a call to the xUpdate() method is made as
+** part of the same statement to delete or update a virtual table row and the
+** implementation returns SQLITE_CONSTRAINT, then there is no need to rollback
+** any database changes. In other words, if the xUpdate() returns
+** SQLITE_CONSTRAINT, the database contents must be exactly as they were
+** before xUpdate was called. By contrast, if SQLITE_INDEX_SCAN_UNIQUE is not
+** set and xUpdate returns SQLITE_CONSTRAINT, any database changes made by
+** the xUpdate method are automatically rolled back by SQLite.
+**
 ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info
-** structure for SQLite version 3.8.2. If a virtual table extension is
+** structure for SQLite [version 3.8.2] ([dateof:3.8.2]). 
+** If a virtual table extension is
 ** used with an SQLite version earlier than 3.8.2, the results of attempting 
 ** to read or write the estimatedRows field are undefined (but are likely 
 ** to included crashing the application). The estimatedRows field should
 ** therefore only be used if [sqlite3_libversion_number()] returns a
-** value greater than or equal to 3008002.
+** value greater than or equal to 3008002. Similarly, the idxFlags field
+** was added for [version 3.9.0] ([dateof:3.9.0]). 
+** It may therefore only be used if
+** sqlite3_libversion_number() returns a value greater than or equal to
+** 3009000.
 */
 struct sqlite3_index_info {
   /* Inputs */
   int nConstraint;           /* Number of entries in aConstraint */
   struct sqlite3_index_constraint {
-     int iColumn;              /* Column on left-hand side of constraint */
+     int iColumn;              /* Column constrained.  -1 for ROWID */
      unsigned char op;         /* Constraint operator */
      unsigned char usable;     /* True if this constraint is usable */
      int iTermOffset;          /* Used internally - xBestIndex should ignore */
@@ -5542,8 +6215,17 @@ struct sqlite3_index_info {
   double estimatedCost;           /* Estimated cost of using this index */
   /* Fields below are only available in SQLite 3.8.2 and later */
   sqlite3_int64 estimatedRows;    /* Estimated number of rows returned */
+  /* Fields below are only available in SQLite 3.9.0 and later */
+  int idxFlags;              /* Mask of SQLITE_INDEX_SCAN_* flags */
+  /* Fields below are only available in SQLite 3.10.0 and later */
+  sqlite3_uint64 colUsed;    /* Input: Mask of columns used by statement */
 };
 
+/*
+** CAPI3REF: Virtual Table Scan Flags
+*/
+#define SQLITE_INDEX_SCAN_UNIQUE      1     /* Scan visits at most 1 row */
+
 /*
 ** CAPI3REF: Virtual Table Constraint Operator Codes
 **
@@ -5552,15 +6234,19 @@ struct sqlite3_index_info {
 ** an operator that is part of a constraint term in the wHERE clause of
 ** a query that uses a [virtual table].
 */
-#define SQLITE_INDEX_CONSTRAINT_EQ    2
-#define SQLITE_INDEX_CONSTRAINT_GT    4
-#define SQLITE_INDEX_CONSTRAINT_LE    8
-#define SQLITE_INDEX_CONSTRAINT_LT    16
-#define SQLITE_INDEX_CONSTRAINT_GE    32
-#define SQLITE_INDEX_CONSTRAINT_MATCH 64
+#define SQLITE_INDEX_CONSTRAINT_EQ      2
+#define SQLITE_INDEX_CONSTRAINT_GT      4
+#define SQLITE_INDEX_CONSTRAINT_LE      8
+#define SQLITE_INDEX_CONSTRAINT_LT     16
+#define SQLITE_INDEX_CONSTRAINT_GE     32
+#define SQLITE_INDEX_CONSTRAINT_MATCH  64
+#define SQLITE_INDEX_CONSTRAINT_LIKE   65
+#define SQLITE_INDEX_CONSTRAINT_GLOB   66
+#define SQLITE_INDEX_CONSTRAINT_REGEXP 67
 
 /*
 ** CAPI3REF: Register A Virtual Table Implementation
+** METHOD: sqlite3
 **
 ** ^These routines are used to register a new [virtual table module] name.
 ** ^Module names must be registered before
@@ -5618,7 +6304,7 @@ SQLITE_API int sqlite3_create_module_v2(
 */
 struct sqlite3_vtab {
   const sqlite3_module *pModule;  /* The module for this virtual table */
-  int nRef;                       /* NO LONGER USED */
+  int nRef;                       /* Number of open cursors */
   char *zErrMsg;                  /* Error message from sqlite3_mprintf() */
   /* Virtual table implementations will typically add additional fields */
 };
@@ -5657,6 +6343,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL);
 
 /*
 ** CAPI3REF: Overload A Function For A Virtual Table
+** METHOD: sqlite3
 **
 ** ^(Virtual tables can provide alternative implementations of functions
 ** using the [xFindFunction] method of the [virtual table module].  
@@ -5699,6 +6386,8 @@ typedef struct sqlite3_blob sqlite3_blob;
 
 /*
 ** CAPI3REF: Open A BLOB For Incremental I/O
+** METHOD: sqlite3
+** CONSTRUCTOR: sqlite3_blob
 **
 ** ^(This interfaces opens a [BLOB handle | handle] to the BLOB located
 ** in row iRow, column zColumn, table zTable in database zDb;
@@ -5708,26 +6397,42 @@ typedef struct sqlite3_blob sqlite3_blob;
 **     SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow;
 **
    )^ ** +** ^(Parameter zDb is not the filename that contains the database, but +** rather the symbolic name of the database. For attached databases, this is +** the name that appears after the AS keyword in the [ATTACH] statement. +** For the main database file, the database name is "main". For TEMP +** tables, the database name is "temp".)^ +** ** ^If the flags parameter is non-zero, then the BLOB is opened for read -** and write access. ^If it is zero, the BLOB is opened for read access. -** ^It is not possible to open a column that is part of an index or primary -** key for writing. ^If [foreign key constraints] are enabled, it is -** not possible to open a column that is part of a [child key] for writing. +** and write access. ^If the flags parameter is zero, the BLOB is opened for +** read-only access. ** -** ^Note that the database name is not the filename that contains -** the database but rather the symbolic name of the database that -** appears after the AS keyword when the database is connected using [ATTACH]. -** ^For the main database file, the database name is "main". -** ^For TEMP tables, the database name is "temp". +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored +** in *ppBlob. Otherwise an [error code] is returned and, unless the error +** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided +** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** on *ppBlob after this function it returns. +** +** This function fails with SQLITE_ERROR if any of the following are true: +**
      +**
    • ^(Database zDb does not exist)^, +**
    • ^(Table zTable does not exist within database zDb)^, +**
    • ^(Table zTable is a WITHOUT ROWID table)^, +**
    • ^(Column zColumn does not exist)^, +**
    • ^(Row iRow is not present in the table)^, +**
    • ^(The specified column of row iRow contains a value that is not +** a TEXT or BLOB value)^, +**
    • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +** constraint and the blob is being opened for read/write access)^, +**
    • ^([foreign key constraints | Foreign key constraints] are enabled, +** column zColumn is part of a [child key] definition and the blob is +** being opened for read/write access)^. +**
    +** +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** -** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is written -** to *ppBlob. Otherwise an [error code] is returned and *ppBlob is set -** to be a null pointer.)^ -** ^This function sets the [database connection] error code and message -** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()] and related -** functions. ^Note that the *ppBlob variable is always initialized in a -** way that makes it safe to invoke [sqlite3_blob_close()] on *ppBlob -** regardless of the success or failure of this routine. ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects @@ -5745,13 +6450,9 @@ typedef struct sqlite3_blob sqlite3_blob; ** interface. Use the [UPDATE] SQL command to change the size of a ** blob. ** -** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] -** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. -** ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces -** and the built-in [zeroblob] SQL function can be used, if desired, -** to create an empty, zero-filled blob in which to read or write using -** this interface. +** and the built-in [zeroblob] SQL function may be used to create a +** zero-filled blob to read or write using the incremental-blob interface. ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. @@ -5768,6 +6469,7 @@ SQLITE_API int sqlite3_blob_open( /* ** CAPI3REF: Move a BLOB Handle to a New Row +** METHOD: sqlite3_blob ** ** ^This function is used to move an existing blob handle so that it points ** to a different row of the same database table. ^The new row is identified @@ -5788,34 +6490,34 @@ SQLITE_API int sqlite3_blob_open( ** ** ^This function sets the database handle error code and message. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); /* ** CAPI3REF: Close A BLOB Handle +** DESTRUCTOR: sqlite3_blob ** -** ^Closes an open [BLOB handle]. +** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed +** unconditionally. Even if this routine returns an error code, the +** handle is still closed.)^ ** -** ^Closing a BLOB shall cause the current transaction to commit -** if there are no other BLOBs, no pending prepared statements, and the -** database connection is in [autocommit mode]. -** ^If any writes were made to the BLOB, they might be held in cache -** until the close operation if they will fit. +** ^If the blob handle being closed was opened for read-write access, and if +** the database is in auto-commit mode and there are no other open read-write +** blob handles or active write statements, the current transaction is +** committed. ^If an error occurs while committing the transaction, an error +** code is returned and the transaction rolled back. ** -** ^(Closing the BLOB often forces the changes -** out to disk and so if any I/O errors occur, they will likely occur -** at the time when the BLOB is closed. Any errors that occur during -** closing are reported as a non-zero return value.)^ -** -** ^(The BLOB is closed unconditionally. Even if this routine returns -** an error code, the BLOB is still closed.)^ -** -** ^Calling this routine with a null pointer (such as would be returned -** by a failed call to [sqlite3_blob_open()]) is a harmless no-op. +** Calling this function with an argument that is not a NULL pointer or an +** open blob handle results in undefined behaviour. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to +** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function +** is passed a valid open blob handle, the values returned by the +** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. */ SQLITE_API int sqlite3_blob_close(sqlite3_blob *); /* ** CAPI3REF: Return The Size Of An Open BLOB +** METHOD: sqlite3_blob ** ** ^Returns the size in bytes of the BLOB accessible via the ** successfully opened [BLOB handle] in its only argument. ^The @@ -5831,6 +6533,7 @@ SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); /* ** CAPI3REF: Read Data From A BLOB Incrementally +** METHOD: sqlite3_blob ** ** ^(This function is used to read data from an open [BLOB handle] into a ** caller-supplied buffer. N bytes of data are copied into buffer Z @@ -5859,22 +6562,29 @@ SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); /* ** CAPI3REF: Write Data Into A BLOB Incrementally +** METHOD: sqlite3_blob ** -** ^This function is used to write data into an open [BLOB handle] from a -** caller-supplied buffer. ^N bytes of data are copied from the buffer Z -** into the open BLOB, starting at offset iOffset. +** ^(This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset.)^ +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** ** ^If the [BLOB handle] passed as the first argument was not opened for ** writing (the flags parameter to [sqlite3_blob_open()] was zero), ** this function returns [SQLITE_READONLY]. ** -** ^This function may only modify the contents of the BLOB; it is +** This function may only modify the contents of the BLOB; it is ** not possible to increase the size of a BLOB using this API. ** ^If offset iOffset is less than N bytes from the end of the BLOB, -** [SQLITE_ERROR] is returned and no data is written. ^If N is -** less than zero [SQLITE_ERROR] is returned and no data is written. -** The size of the BLOB (and hence the maximum value of N+iOffset) -** can be determined using the [sqlite3_blob_bytes()] interface. +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** than zero [SQLITE_ERROR] is returned and no data is written. ** ** ^An attempt to write to an expired [BLOB handle] fails with an ** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred @@ -5883,9 +6593,6 @@ SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); ** have been overwritten by the statement that expired the BLOB handle ** or by other independent statements. ** -** ^(On success, sqlite3_blob_write() returns SQLITE_OK. -** Otherwise, an [error code] or an [extended error code] is returned.)^ -** ** This routine only works on a [BLOB handle] which has been created ** by a prior successful call to [sqlite3_blob_open()] and which has not ** been closed by [sqlite3_blob_close()]. Passing any other pointer in @@ -5938,34 +6645,34 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** The SQLite source code contains multiple implementations ** of these mutex routines. An appropriate implementation -** is selected automatically at compile-time. ^(The following +** is selected automatically at compile-time. The following ** implementations are available in the SQLite core: ** **
      **
    • SQLITE_MUTEX_PTHREADS **
    • SQLITE_MUTEX_W32 **
    • SQLITE_MUTEX_NOOP -**
    )^ +** ** -** ^The SQLITE_MUTEX_NOOP implementation is a set of routines +** The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in -** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** a single-threaded application. The SQLITE_MUTEX_PTHREADS and ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor ** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex ** implementation is included with the library. In this case the ** application must supply a custom mutex implementation using the ** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function ** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize().)^ +** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new -** mutex and returns a pointer to it. ^If it returns NULL -** that means that a mutex could not be allocated. ^SQLite -** will unwind its stack and return an error. ^(The argument -** to sqlite3_mutex_alloc() is one of these integer constants: +** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() +** routine returns NULL if it is unable to allocate the requested +** mutex. The argument to sqlite3_mutex_alloc() must one of these +** integer constants: ** **
      **
    • SQLITE_MUTEX_FAST @@ -5978,7 +6685,11 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); **
    • SQLITE_MUTEX_STATIC_PMEM **
    • SQLITE_MUTEX_STATIC_APP1 **
    • SQLITE_MUTEX_STATIC_APP2 -**
    )^ +**
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 +** ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) ** cause sqlite3_mutex_alloc() to create @@ -5986,14 +6697,14 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** is used but not necessarily so when SQLITE_MUTEX_FAST is used. ** The mutex implementation does not need to make a distinction ** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does -** not want to. ^SQLite will only request a recursive mutex in -** cases where it really needs one. ^If a faster non-recursive mutex +** not want to. SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex ** implementation is available on the host platform, the mutex subsystem ** might return such a mutex in response to SQLITE_MUTEX_FAST. ** ** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other ** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return -** a pointer to a static preexisting mutex. ^Six static mutexes are +** a pointer to a static preexisting mutex. ^Nine static mutexes are ** used by the current version of SQLite. Future versions of SQLite ** may add additional static mutexes. Static mutexes are for internal ** use by SQLite only. Applications that use SQLite mutexes should @@ -6002,16 +6713,13 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. ^But for the static +** returns a different mutex on every call. ^For the static ** mutex types, the same mutex is returned on every call that has ** the same type number. ** ** ^The sqlite3_mutex_free() routine deallocates a previously -** allocated dynamic mutex. ^SQLite is careful to deallocate every -** dynamic mutex that it allocates. The dynamic mutexes must not be in -** use when they are deallocated. Attempting to deallocate a static -** mutex results in undefined behavior. ^SQLite never deallocates -** a static mutex. +** allocated dynamic mutex. Attempting to deallocate a static +** mutex results in undefined behavior. ** ** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt ** to enter a mutex. ^If another thread is already within the mutex, @@ -6019,23 +6727,21 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] ** upon successful entry. ^(Mutexes created using ** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. -** In such cases the, +** In such cases, the ** mutex must be exited an equal number of times before another thread -** can enter.)^ ^(If the same thread tries to enter any other -** kind of mutex more than once, the behavior is undefined. -** SQLite will never exhibit -** such behavior in its own use of mutexes.)^ +** can enter.)^ If the same thread tries to enter any mutex other +** than an SQLITE_MUTEX_RECURSIVE more than once, the behavior is undefined. ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable behavior.)^ +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable +** behavior.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was -** previously entered by the same thread. ^(The behavior +** previously entered by the same thread. The behavior ** is undefined if the mutex is not currently entered by the -** calling thread or is not currently allocated. SQLite will -** never do either.)^ +** calling thread or is not currently allocated. ** ** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or ** sqlite3_mutex_leave() is a NULL pointer, then all three routines @@ -6056,9 +6762,9 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** used to allocate and use mutexes. ** ** Usually, the default mutex implementations provided by SQLite are -** sufficient, however the user has the option of substituting a custom +** sufficient, however the application has the option of substituting a custom ** implementation for specialized deployments or systems for which SQLite -** does not provide a suitable implementation. In this case, the user +** does not provide a suitable implementation. In this case, the application ** creates and populates an instance of this structure to pass ** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. ** Additionally, an instance of this structure can be used as an @@ -6099,13 +6805,13 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). ** -** The xMutexInit() method must be threadsafe. ^It must be harmless to +** The xMutexInit() method must be threadsafe. It must be harmless to ** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** -** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] -** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory +** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). Similarly, xMutexAlloc() must not use SQLite memory ** allocation for a static mutex. ^However xMutexAlloc() may use SQLite ** memory allocation for a fast or recursive mutex. ** @@ -6131,29 +6837,29 @@ struct sqlite3_mutex_methods { ** CAPI3REF: Mutex Verification Routines ** ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines -** are intended for use inside assert() statements. ^The SQLite core +** are intended for use inside assert() statements. The SQLite core ** never uses these routines except inside an assert() and applications -** are advised to follow the lead of the core. ^The SQLite core only +** are advised to follow the lead of the core. The SQLite core only ** provides implementations for these routines when it is compiled -** with the SQLITE_DEBUG flag. ^External mutex implementations +** with the SQLITE_DEBUG flag. External mutex implementations ** are only required to provide these routines if SQLITE_DEBUG is ** defined and if NDEBUG is not defined. ** -** ^These routines should return true if the mutex in their argument +** These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** ^The implementation is not required to provide versions of these +** The implementation is not required to provide versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. ** -** ^If the argument to sqlite3_mutex_held() is a NULL pointer then +** If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since ** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is -** the appropriate thing to do. ^The sqlite3_mutex_notheld() +** the appropriate thing to do. The sqlite3_mutex_notheld() ** interface should also return 1 when given a NULL pointer. */ #ifndef NDEBUG @@ -6177,16 +6883,20 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_MEM 3 /* sqlite3_malloc() */ #define SQLITE_MUTEX_STATIC_MEM2 4 /* NOT USED */ #define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */ -#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_random() */ +#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_randomness() */ #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ #define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ +#define SQLITE_MUTEX_STATIC_VFS1 11 /* For use by built-in VFS */ +#define SQLITE_MUTEX_STATIC_VFS2 12 /* For use by extension VFS */ +#define SQLITE_MUTEX_STATIC_VFS3 13 /* For use by application VFS */ /* ** CAPI3REF: Retrieve the mutex for a database connection +** METHOD: sqlite3 ** ** ^This interface returns a pointer the [sqlite3_mutex] object that ** serializes access to the [database connection] given in the argument @@ -6198,6 +6908,7 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); /* ** CAPI3REF: Low-Level Control Of Database Files +** METHOD: sqlite3 ** ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated @@ -6275,17 +6986,20 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 -#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ +#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 #define SQLITE_TESTCTRL_BYTEORDER 22 #define SQLITE_TESTCTRL_ISINIT 23 -#define SQLITE_TESTCTRL_LAST 23 +#define SQLITE_TESTCTRL_SORTER_MMAP 24 +#define SQLITE_TESTCTRL_IMPOSTER 25 +#define SQLITE_TESTCTRL_LAST 25 /* ** CAPI3REF: SQLite Runtime Status ** -** ^This interface is used to retrieve runtime status information +** ^These interfaces are used to retrieve runtime status information ** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes @@ -6299,19 +7013,22 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** ^(Other parameters record only the highwater mark and not the current ** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** ^The sqlite3_status() routine returns SQLITE_OK on success and a -** non-zero [error code] on failure. +** ^The sqlite3_status() and sqlite3_status64() routines return +** SQLITE_OK on success and a non-zero [error code] on failure. ** -** This routine is threadsafe but is not atomic. This routine can be -** called while other threads are running the same or different SQLite -** interfaces. However the values returned in *pCurrent and -** *pHighwater reflect the status of SQLite at different points in time -** and it is possible that another thread might change the parameter -** in between the times when *pCurrent and *pHighwater are written. +** If either the current value or the highwater mark is too large to +** be represented by a 32-bit integer, then the values returned by +** sqlite3_status() are undefined. ** ** See also: [sqlite3_db_status()] */ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int sqlite3_status64( + int op, + sqlite3_int64 *pCurrent, + sqlite3_int64 *pHighwater, + int resetFlag +); /* @@ -6390,7 +7107,8 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** The value written into the *pCurrent parameter is undefined.)^ ** ** [[SQLITE_STATUS_PARSER_STACK]] ^(
    SQLITE_STATUS_PARSER_STACK
    -**
    This parameter records the deepest parser stack. It is only +**
    The *pHighwater parameter records the deepest parser stack. +** The *pCurrent value is undefined. The *pHighwater value is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
    )^ ** ** @@ -6409,6 +7127,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF /* ** CAPI3REF: Database Connection Status +** METHOD: sqlite3 ** ** ^This interface is used to retrieve runtime status information ** about a single [database connection]. ^The first argument is the @@ -6471,12 +7190,24 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** the current value is always zero.)^ ** ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
    SQLITE_DBSTATUS_CACHE_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** +** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] +** ^(
    SQLITE_DBSTATUS_CACHE_USED_SHARED
    +**
    This parameter is similar to DBSTATUS_CACHE_USED, except that if a +** pager cache is shared between two or more connections the bytes of heap +** memory used by that pager cache is divided evenly between the attached +** connections.)^ In other words, if none of the pager caches associated +** with the database connection are shared, this request returns the same +** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are +** shared, the value returned by this call will be smaller than that returned +** by DBSTATUS_CACHE_USED. ^The highwater mark associated with +** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. +** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** memory used to store the schema for all databases associated ** with the connection - main, temp, and any [ATTACH]-ed databases.)^ ** ^The full amount of memory used by the schemas is reported, even if the @@ -6485,7 +7216,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. ** ** [[SQLITE_DBSTATUS_STMT_USED]] ^(
    SQLITE_DBSTATUS_STMT_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** and lookaside memory used by all prepared statements associated with ** the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. @@ -6532,11 +7263,13 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_CACHE_MISS 8 #define SQLITE_DBSTATUS_CACHE_WRITE 9 #define SQLITE_DBSTATUS_DEFERRED_FKS 10 -#define SQLITE_DBSTATUS_MAX 10 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 +#define SQLITE_DBSTATUS_MAX 11 /* Largest defined DBSTATUS */ /* ** CAPI3REF: Prepared Statement Status +** METHOD: sqlite3_stmt ** ** ^(Each prepared statement maintains various ** [SQLITE_STMTSTATUS counters] that measure the number @@ -6885,6 +7618,10 @@ typedef struct sqlite3_backup sqlite3_backup; ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with ** an error. ** +** ^A call to sqlite3_backup_init() will fail, returning NULL, if +** there is already a read or read-write transaction open on the +** destination database. +** ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is ** returned and an error code and error message are stored in the ** destination [database connection] D. @@ -6977,20 +7714,20 @@ typedef struct sqlite3_backup sqlite3_backup; ** is not a permanent error and does not affect the return value of ** sqlite3_backup_finish(). ** -** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]] +** [[sqlite3_backup_remaining()]] [[sqlite3_backup_pagecount()]] ** sqlite3_backup_remaining() and sqlite3_backup_pagecount() ** -** ^Each call to sqlite3_backup_step() sets two values inside -** the [sqlite3_backup] object: the number of pages still to be backed -** up and the total number of pages in the source database file. -** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces -** retrieve these two values, respectively. -** -** ^The values returned by these functions are only updated by -** sqlite3_backup_step(). ^If the source database is modified during a backup -** operation, then the values are not updated to account for any extra -** pages that need to be updated or the size of the source database file -** changing. +** ^The sqlite3_backup_remaining() routine returns the number of pages still +** to be backed up at the conclusion of the most recent sqlite3_backup_step(). +** ^The sqlite3_backup_pagecount() routine returns the total number of pages +** in the source database at the conclusion of the most recent +** sqlite3_backup_step(). +** ^(The values returned by these functions are only updated by +** sqlite3_backup_step(). If the source database is modified in a way that +** changes the size of the source database or the number of pages remaining, +** those changes are not reflected in the output of sqlite3_backup_pagecount() +** and sqlite3_backup_remaining() until after the next +** sqlite3_backup_step().)^ ** ** Concurrent Usage of Database Handles ** @@ -7036,6 +7773,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); /* ** CAPI3REF: Unlock Notification +** METHOD: sqlite3 ** ** ^When running in shared-cache mode, a database operation may fail with ** an [SQLITE_LOCKED] error if the required locks on the shared-cache or @@ -7169,18 +7907,43 @@ SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); /* ** CAPI3REF: String Globbing * -** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches -** the glob pattern P, and it returns non-zero if string X does not match -** the glob pattern P. ^The definition of glob pattern matching used in +** ^The [sqlite3_strglob(P,X)] interface returns zero if and only if +** string X matches the [GLOB] pattern P. +** ^The definition of [GLOB] pattern matching used in ** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the -** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case -** sensitive. +** SQL dialect understood by SQLite. ^The [sqlite3_strglob(P,X)] function +** is case sensitive. ** ** Note that this routine returns zero on a match and non-zero if the strings ** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strlike()]. */ SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); +/* +** CAPI3REF: String LIKE Matching +* +** ^The [sqlite3_strlike(P,X,E)] interface returns zero if and only if +** string X matches the [LIKE] pattern P with escape character E. +** ^The definition of [LIKE] pattern matching used in +** [sqlite3_strlike(P,X,E)] is the same as for the "X LIKE P ESCAPE E" +** operator in the SQL dialect understood by SQLite. ^For "X LIKE P" without +** the ESCAPE clause, set the E parameter of [sqlite3_strlike(P,X,E)] to 0. +** ^As with the LIKE operator, the [sqlite3_strlike(P,X,E)] function is case +** insensitive - equivalent upper and lower case ASCII characters match +** one another. +** +** ^The [sqlite3_strlike(P,X,E)] function matches Unicode characters, though +** only ASCII characters are case folded. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strglob()]. +*/ +SQLITE_API int sqlite3_strlike(const char *zGlob, const char *zStr, unsigned int cEsc); + /* ** CAPI3REF: Error Logging Interface ** @@ -7206,14 +7969,13 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); /* ** CAPI3REF: Write-Ahead Log Commit Hook +** METHOD: sqlite3 ** ** ^The [sqlite3_wal_hook()] function is used to register a callback that -** will be invoked each time a database connection commits data to a -** [write-ahead log] (i.e. whenever a transaction is committed in -** [journal_mode | journal_mode=WAL mode]). +** is invoked each time data is committed to a database in wal mode. ** -** ^The callback is invoked by SQLite after the commit has taken place and -** the associated write-lock on the database released, so the implementation +** ^(The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released)^, so the implementation ** may read, write or [checkpoint] the database as required. ** ** ^The first parameter passed to the callback function when it is invoked @@ -7237,7 +7999,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** previously registered write-ahead log callback. ^Note that the ** [sqlite3_wal_autocheckpoint()] interface and the ** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will -** those overwrite any prior [sqlite3_wal_hook()] settings. +** overwrite any prior [sqlite3_wal_hook()] settings. */ SQLITE_API void *sqlite3_wal_hook( sqlite3*, @@ -7247,6 +8009,7 @@ SQLITE_API void *sqlite3_wal_hook( /* ** CAPI3REF: Configure an auto-checkpoint +** METHOD: sqlite3 ** ** ^The [sqlite3_wal_autocheckpoint(D,N)] is a wrapper around ** [sqlite3_wal_hook()] that causes any database on [database connection] D @@ -7277,98 +8040,117 @@ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); /* ** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 ** -** ^The [sqlite3_wal_checkpoint(D,X)] interface causes database named X -** on [database connection] D to be [checkpointed]. ^If X is NULL or an -** empty string, then a checkpoint is run on all databases of -** connection D. ^If the database connection D is not in -** [WAL | write-ahead log mode] then this interface is a harmless no-op. -** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a -** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. -** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL -** or RESET checkpoint. +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ ** -** ^The [wal_checkpoint pragma] can be used to invoke this interface -** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] can be used to cause this interface to be -** run whenever the WAL reaches a certain size threshold. +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. ** -** See also: [sqlite3_wal_checkpoint_v2()] +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. */ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 ** -** Run a checkpoint operation on WAL database zDb attached to database -** handle db. The specific operation is determined by the value of the -** eMode parameter: +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ ** **
    **
    SQLITE_CHECKPOINT_PASSIVE
    -** Checkpoint as many frames as possible without waiting for any database -** readers or writers to finish. Sync the db file if all frames in the log -** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] -** is never invoked. +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (it invokes the +** ^This mode blocks (it invokes the ** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database -** snapshot. It then checkpoints all frames in the log file and syncs the -** database file. This call blocks database writers while it is running, -** but not database readers. +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. ** **
    SQLITE_CHECKPOINT_RESTART
    -** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the -** [sqlite3_busy_handler|busy-handler callback]) -** until all readers are reading from the database file only. This ensures -** that the next client to write to the database file restarts the log file -** from the beginning. This call blocks database writers while it is running, -** but not database readers. +** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the +** [busy-handler callback]) +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. +** +**
    SQLITE_CHECKPOINT_TRUNCATE
    +** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. **
    ** -** If pnLog is not NULL, then *pnLog is set to the total number of frames in -** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to -** the total number of checkpointed frames (including any that were already -** checkpointed when this function is called). *pnLog and *pnCkpt may be -** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. -** If no values are available because of an error, they are both set to -1 -** before returning to communicate this to the caller. +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. ** -** All calls obtain an exclusive "checkpoint" lock on the database file. If +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If ** any other process is running a checkpoint operation at the same time, the -** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive -** "writer" lock on the database file. If the writer lock cannot be obtained -** immediately, and a busy-handler is configured, it is invoked and the writer -** lock retried until either the busy-handler returns 0 or the lock is -** successfully obtained. The busy-handler is also invoked while waiting for -** database readers as described above. If the busy-handler returns 0 before +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as ** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible -** without blocking any further. SQLITE_BUSY is returned in this case. +** without blocking any further. ^SQLITE_BUSY is returned in this case. ** -** If parameter zDb is NULL or points to a zero length string, then the -** specified operation is attempted on all WAL databases. In this case the -** values written to output parameters *pnLog and *pnCkpt are undefined. If +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If ** an SQLITE_BUSY error is encountered when processing one or more of the ** attached WAL databases, the operation is still attempted on any remaining -** attached databases and SQLITE_BUSY is returned to the caller. If any other +** attached databases and SQLITE_BUSY is returned at the end. ^If any other ** error occurs while processing an attached database, processing is abandoned -** and the error code returned to the caller immediately. If no error +** and the error code is returned to the caller immediately. ^If no error ** (SQLITE_BUSY or otherwise) is encountered while processing the attached ** databases, SQLITE_OK is returned. ** -** If database zDb is the name of an attached database that is not in WAL -** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If ** zDb is not NULL (or a zero length string) and is not the name of any ** attached database, SQLITE_ERROR is returned to the caller. +** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. */ SQLITE_API int sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ @@ -7379,16 +8161,18 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ); /* -** CAPI3REF: Checkpoint operation parameters +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} ** -** These constants can be used as the 3rd parameter to -** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] -** documentation for additional information about the meaning and use of -** each of these values. +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 -#define SQLITE_CHECKPOINT_FULL 1 -#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* ** CAPI3REF: Virtual Table Interface Configuration @@ -7477,7 +8261,380 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 +/* +** CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus options} +** +** The following constants can be used for the T parameter to the +** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a +** different metric for sqlite3_stmt_scanstatus() to return. +** +** When the value returned to V is a string, space to hold that string is +** managed by the prepared statement S and will be automatically freed when +** S is finalized. +** +**
    +** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be +** set to the total number of times that the X-th loop has run.
    +** +** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
    +** +** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    +**
    ^The "double" variable pointed to by the T parameter will be set to the +** query planner's estimate for the average number of rows output from each +** iteration of the X-th loop. If the query planner's estimates was accurate, +** then this value will approximate the quotient NVISIT/NLOOP and the +** product of this value for all prior loops with the same SELECTID will +** be the NLOOP value for the current loop. +** +** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. +** +** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. +** +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +**
    ^The "int" variable pointed to by the T parameter will be set to the +** "select-id" for the X-th loop. The select-id identifies which query or +** subquery the loop is part of. The main query has a select-id of zero. +** The select-id is the same value as is output in the first column +** of an [EXPLAIN QUERY PLAN] query. +**
    +*/ +#define SQLITE_SCANSTAT_NLOOP 0 +#define SQLITE_SCANSTAT_NVISIT 1 +#define SQLITE_SCANSTAT_EST 2 +#define SQLITE_SCANSTAT_NAME 3 +#define SQLITE_SCANSTAT_EXPLAIN 4 +#define SQLITE_SCANSTAT_SELECTID 5 +/* +** CAPI3REF: Prepared Statement Scan Status +** METHOD: sqlite3_stmt +** +** This interface returns information about the predicted and measured +** performance for pStmt. Advanced applications can use this +** interface to compare the predicted and the measured performance and +** issue warnings and/or rerun [ANALYZE] if discrepancies are found. +** +** Since this interface is expected to be rarely used, it is only +** available if SQLite is compiled using the [SQLITE_ENABLE_STMT_SCANSTATUS] +** compile-time option. +** +** The "iScanStatusOp" parameter determines which status information to return. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. +** ^The requested measurement is written into a variable pointed to by +** the "pOut" parameter. +** Parameter "idx" identifies the specific loop to retrieve statistics for. +** Loops are numbered starting from zero. ^If idx is out of range - less than +** zero or greater than or equal to the total number of loops used to implement +** the statement - a non-zero value is returned and the variable that pOut +** points to is unchanged. +** +** ^Statistics might not be available for all loops in all statements. ^In cases +** where there exist loops with no available statistics, this function behaves +** as if the loop did not exist - it returns non-zero and leave the variable +** that pOut points to unchanged. +** +** See also: [sqlite3_stmt_scanstatus_reset()] +*/ +SQLITE_API int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Zero Scan-Status Counters +** METHOD: sqlite3_stmt +** +** ^Zero all [sqlite3_stmt_scanstatus()] related event counters. +** +** This API is only available if the library is built with pre-processor +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +*/ +SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); + +/* +** CAPI3REF: Flush caches to disk mid-transaction +** +** ^If a write-transaction is open on [database connection] D when the +** [sqlite3_db_cacheflush(D)] interface invoked, any dirty +** pages in the pager-cache that are not currently in use are written out +** to disk. A dirty page may be in use if a database cursor created by an +** active SQL statement is reading from it, or if it is page 1 of a database +** file (page 1 is always "in use"). ^The [sqlite3_db_cacheflush(D)] +** interface flushes caches for all schemas - "main", "temp", and +** any [attached] databases. +** +** ^If this function needs to obtain extra database locks before dirty pages +** can be flushed to disk, it does so. ^If those locks cannot be obtained +** immediately and there is a busy-handler callback configured, it is invoked +** in the usual manner. ^If the required lock still cannot be obtained, then +** the database is skipped and an attempt made to flush any dirty pages +** belonging to the next (if any) database. ^If any databases are skipped +** because locks cannot be obtained, but no other error occurs, this +** function returns SQLITE_BUSY. +** +** ^If any other error occurs while flushing dirty pages to disk (for +** example an IO error or out-of-memory condition), then processing is +** abandoned and an SQLite [error code] is returned to the caller immediately. +** +** ^Otherwise, if no error occurs, [sqlite3_db_cacheflush()] returns SQLITE_OK. +** +** ^This function does not set the database handle error code or message +** returned by the [sqlite3_errcode()] and [sqlite3_errmsg()] functions. +*/ +SQLITE_API int sqlite3_db_cacheflush(sqlite3*); + +/* +** CAPI3REF: The pre-update hook. +** +** ^These interfaces are only available if SQLite is compiled using the +** [SQLITE_ENABLE_PREUPDATE_HOOK] compile-time option. +** +** ^The [sqlite3_preupdate_hook()] interface registers a callback function +** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation +** on a [rowid table]. +** ^At most one preupdate hook may be registered at a time on a single +** [database connection]; each call to [sqlite3_preupdate_hook()] overrides +** the previous setting. +** ^The preupdate hook is disabled by invoking [sqlite3_preupdate_hook()] +** with a NULL pointer as the second parameter. +** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as +** the first parameter to callbacks. +** +** ^The preupdate hook only fires for changes to [rowid tables]; the preupdate +** hook is not invoked for changes to [virtual tables] or [WITHOUT ROWID] +** tables. +** +** ^The second parameter to the preupdate callback is a pointer to +** the [database connection] that registered the preupdate hook. +** ^The third parameter to the preupdate callback is one of the constants +** [SQLITE_INSERT], [SQLITE_DELETE], or [SQLITE_UPDATE] to identify the +** kind of update operation that is about to occur. +** ^(The fourth parameter to the preupdate callback is the name of the +** database within the database connection that is being modified. This +** will be "main" for the main database or "temp" for TEMP tables or +** the name given after the AS keyword in the [ATTACH] statement for attached +** databases.)^ +** ^The fifth parameter to the preupdate callback is the name of the +** table that is being modified. +** ^The sixth parameter to the preupdate callback is the initial [rowid] of the +** row being changes for SQLITE_UPDATE and SQLITE_DELETE changes and is +** undefined for SQLITE_INSERT changes. +** ^The seventh parameter to the preupdate callback is the final [rowid] of +** the row being changed for SQLITE_UPDATE and SQLITE_INSERT changes and is +** undefined for SQLITE_DELETE changes. +** +** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], +** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces +** provide additional information about a preupdate event. These routines +** may only be called from within a preupdate callback. Invoking any of +** these routines from outside of a preupdate callback or with a +** [database connection] pointer that is different from the one supplied +** to the preupdate callback results in undefined and probably undesirable +** behavior. +** +** ^The [sqlite3_preupdate_count(D)] interface returns the number of columns +** in the row that is being inserted, updated, or deleted. +** +** ^The [sqlite3_preupdate_old(D,N,P)] interface writes into P a pointer to +** a [protected sqlite3_value] that contains the value of the Nth column of +** the table row before it is updated. The N parameter must be between 0 +** and one less than the number of columns or the behavior will be +** undefined. This must only be used within SQLITE_UPDATE and SQLITE_DELETE +** preupdate callbacks; if it is used by an SQLITE_INSERT callback then the +** behavior is undefined. The [sqlite3_value] that P points to +** will be destroyed when the preupdate callback returns. +** +** ^The [sqlite3_preupdate_new(D,N,P)] interface writes into P a pointer to +** a [protected sqlite3_value] that contains the value of the Nth column of +** the table row after it is updated. The N parameter must be between 0 +** and one less than the number of columns or the behavior will be +** undefined. This must only be used within SQLITE_INSERT and SQLITE_UPDATE +** preupdate callbacks; if it is used by an SQLITE_DELETE callback then the +** behavior is undefined. The [sqlite3_value] that P points to +** will be destroyed when the preupdate callback returns. +** +** ^The [sqlite3_preupdate_depth(D)] interface returns 0 if the preupdate +** callback was invoked as a result of a direct insert, update, or delete +** operation; or 1 for inserts, updates, or deletes invoked by top-level +** triggers; or 2 for changes resulting from triggers called by top-level +** triggers; and so forth. +** +** See also: [sqlite3_update_hook()] +*/ +SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_preupdate_hook( + sqlite3 *db, + void(*xPreUpdate)( + void *pCtx, /* Copy of third arg to preupdate_hook() */ + sqlite3 *db, /* Database handle */ + int op, /* SQLITE_UPDATE, DELETE or INSERT */ + char const *zDb, /* Database name */ + char const *zName, /* Table name */ + sqlite3_int64 iKey1, /* Rowid of row about to be deleted/updated */ + sqlite3_int64 iKey2 /* New rowid value (for a rowid UPDATE) */ + ), + void* +); +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_preupdate_old(sqlite3 *, int, sqlite3_value **); +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_preupdate_count(sqlite3 *); +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_preupdate_depth(sqlite3 *); +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **); + +/* +** CAPI3REF: Low-level system error code +** +** ^Attempt to return the underlying operating system error code or error +** number that caused the most recent I/O error or failure to open a file. +** The return value is OS-dependent. For example, on unix systems, after +** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be +** called to get back the underlying "errno" that caused the problem, such +** as ENOSPC, EAUTH, EISDIR, and so forth. +*/ +SQLITE_API int sqlite3_system_errno(sqlite3*); + +/* +** CAPI3REF: Database Snapshot +** KEYWORDS: {snapshot} +** EXPERIMENTAL +** +** An instance of the snapshot object records the state of a [WAL mode] +** database for some specific point in history. +** +** In [WAL mode], multiple [database connections] that are open on the +** same database file can each be reading a different historical version +** of the database file. When a [database connection] begins a read +** transaction, that connection sees an unchanging copy of the database +** as it existed for the point in time when the transaction first started. +** Subsequent changes to the database from other connections are not seen +** by the reader until a new read transaction is started. +** +** The sqlite3_snapshot object records state information about an historical +** version of the database file so that it is possible to later open a new read +** transaction that sees that historical version of the database rather than +** the most recent version. +** +** The constructor for this object is [sqlite3_snapshot_get()]. The +** [sqlite3_snapshot_open()] method causes a fresh read transaction to refer +** to an historical snapshot (if possible). The destructor for +** sqlite3_snapshot objects is [sqlite3_snapshot_free()]. +*/ +typedef struct sqlite3_snapshot sqlite3_snapshot; + +/* +** CAPI3REF: Record A Database Snapshot +** EXPERIMENTAL +** +** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a +** new [sqlite3_snapshot] object that records the current state of +** schema S in database connection D. ^On success, the +** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly +** created [sqlite3_snapshot] object into *P and returns SQLITE_OK. +** ^If schema S of [database connection] D is not a [WAL mode] database +** that is in a read transaction, then [sqlite3_snapshot_get(D,S,P)] +** leaves the *P value unchanged and returns an appropriate [error code]. +** +** The [sqlite3_snapshot] object returned from a successful call to +** [sqlite3_snapshot_get()] must be freed using [sqlite3_snapshot_free()] +** to avoid a memory leak. +** +** The [sqlite3_snapshot_get()] interface is only available when the +** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot **ppSnapshot +); + +/* +** CAPI3REF: Start a read transaction on an historical snapshot +** EXPERIMENTAL +** +** ^The [sqlite3_snapshot_open(D,S,P)] interface starts a +** read transaction for schema S of +** [database connection] D such that the read transaction +** refers to historical [snapshot] P, rather than the most +** recent change to the database. +** ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK on success +** or an appropriate [error code] if it fails. +** +** ^In order to succeed, a call to [sqlite3_snapshot_open(D,S,P)] must be +** the first operation following the [BEGIN] that takes the schema S +** out of [autocommit mode]. +** ^In other words, schema S must not currently be in +** a transaction for [sqlite3_snapshot_open(D,S,P)] to work, but the +** database connection D must be out of [autocommit mode]. +** ^A [snapshot] will fail to open if it has been overwritten by a +** [checkpoint]. +** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the +** database connection D does not know that the database file for +** schema S is in [WAL mode]. A database connection might not know +** that the database file is in [WAL mode] if there has been no prior +** I/O on that database connection, or if the database entered [WAL mode] +** after the most recent I/O on the database connection.)^ +** (Hint: Run "[PRAGMA application_id]" against a newly opened +** database connection in order to make it ready to use snapshots.) +** +** The [sqlite3_snapshot_open()] interface is only available when the +** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot *pSnapshot +); + +/* +** CAPI3REF: Destroy a snapshot +** EXPERIMENTAL +** +** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. +** The application must eventually free every [sqlite3_snapshot] object +** using this routine to avoid a memory leak. +** +** The [sqlite3_snapshot_free()] interface is only available when the +** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); + +/* +** CAPI3REF: Compare the ages of two snapshot handles. +** EXPERIMENTAL +** +** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages +** of two valid snapshot handles. +** +** If the two snapshot handles are not associated with the same database +** file, the result of the comparison is undefined. +** +** Additionally, the result of the comparison is only valid if both of the +** snapshot handles were obtained by calling sqlite3_snapshot_get() since the +** last time the wal file was deleted. The wal file is deleted when the +** database is changed back to rollback mode or when the number of database +** clients drops to zero. If either snapshot handle was obtained before the +** wal file was last deleted, the value returned by this function +** is undefined. +** +** Otherwise, this API returns a negative value if P1 refers to an older +** snapshot than P2, zero if the two handles refer to the same database +** snapshot, and a positive value if P1 is a newer snapshot than P2. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( + sqlite3_snapshot *p1, + sqlite3_snapshot *p2 +); /* ** Undo the hack that converts floating point types to integer for @@ -7490,8 +8647,9 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); #if 0 } /* End of the 'extern "C"' block */ #endif -#endif /* _SQLITE3_H_ */ +#endif /* SQLITE3_H */ +/******** Begin file sqlite3rtree.h *********/ /* ** 2010 August 30 ** @@ -7591,6 +8749,8 @@ struct sqlite3_rtree_query_info { int eParentWithin; /* Visibility of parent node */ int eWithin; /* OUT: Visiblity */ sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ + /* The following fields are only available in 3.8.11 and later */ + sqlite3_value **apSqlParam; /* Original SQL values of parameters */ }; /* @@ -7607,6 +8767,1871 @@ struct sqlite3_rtree_query_info { #endif /* ifndef _SQLITE3RTREE_H_ */ +/******** End of sqlite3rtree.h *********/ +/******** Begin file sqlite3session.h *********/ + +#if !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) +#define __SQLITESESSION_H_ 1 + +/* +** Make sure we can call this stuff from C++. +*/ +#if 0 +extern "C" { +#endif + + +/* +** CAPI3REF: Session Object Handle +*/ +typedef struct sqlite3_session sqlite3_session; + +/* +** CAPI3REF: Changeset Iterator Handle +*/ +typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; + +/* +** CAPI3REF: Create A New Session Object +** +** Create a new session object attached to database handle db. If successful, +** a pointer to the new object is written to *ppSession and SQLITE_OK is +** returned. If an error occurs, *ppSession is set to NULL and an SQLite +** error code (e.g. SQLITE_NOMEM) is returned. +** +** It is possible to create multiple session objects attached to a single +** database handle. +** +** Session objects created using this function should be deleted using the +** [sqlite3session_delete()] function before the database handle that they +** are attached to is itself closed. If the database handle is closed before +** the session object is deleted, then the results of calling any session +** module function, including [sqlite3session_delete()] on the session object +** are undefined. +** +** Because the session module uses the [sqlite3_preupdate_hook()] API, it +** is not possible for an application to register a pre-update hook on a +** database handle that has one or more session objects attached. Nor is +** it possible to create a session object attached to a database handle for +** which a pre-update hook is already defined. The results of attempting +** either of these things are undefined. +** +** The session object will be used to create changesets for tables in +** database zDb, where zDb is either "main", or "temp", or the name of an +** attached database. It is not an error if database zDb is not attached +** to the database when the session object is created. +*/ +int sqlite3session_create( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (e.g. "main") */ + sqlite3_session **ppSession /* OUT: New session object */ +); + +/* +** CAPI3REF: Delete A Session Object +** +** Delete a session object previously allocated using +** [sqlite3session_create()]. Once a session object has been deleted, the +** results of attempting to use pSession with any other session module +** function are undefined. +** +** Session objects must be deleted before the database handle to which they +** are attached is closed. Refer to the documentation for +** [sqlite3session_create()] for details. +*/ +void sqlite3session_delete(sqlite3_session *pSession); + + +/* +** CAPI3REF: Enable Or Disable A Session Object +** +** Enable or disable the recording of changes by a session object. When +** enabled, a session object records changes made to the database. When +** disabled - it does not. A newly created session object is enabled. +** Refer to the documentation for [sqlite3session_changeset()] for further +** details regarding how enabling and disabling a session object affects +** the eventual changesets. +** +** Passing zero to this function disables the session. Passing a value +** greater than zero enables it. Passing a value less than zero is a +** no-op, and may be used to query the current state of the session. +** +** The return value indicates the final state of the session object: 0 if +** the session is disabled, or 1 if it is enabled. +*/ +int sqlite3session_enable(sqlite3_session *pSession, int bEnable); + +/* +** CAPI3REF: Set Or Clear the Indirect Change Flag +** +** Each change recorded by a session object is marked as either direct or +** indirect. A change is marked as indirect if either: +** +**
      +**
    • The session object "indirect" flag is set when the change is +** made, or +**
    • The change is made by an SQL trigger or foreign key action +** instead of directly as a result of a users SQL statement. +**
    +** +** If a single row is affected by more than one operation within a session, +** then the change is considered indirect if all operations meet the criteria +** for an indirect change above, or direct otherwise. +** +** This function is used to set, clear or query the session object indirect +** flag. If the second argument passed to this function is zero, then the +** indirect flag is cleared. If it is greater than zero, the indirect flag +** is set. Passing a value less than zero does not modify the current value +** of the indirect flag, and may be used to query the current state of the +** indirect flag for the specified session object. +** +** The return value indicates the final state of the indirect flag: 0 if +** it is clear, or 1 if it is set. +*/ +int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); + +/* +** CAPI3REF: Attach A Table To A Session Object +** +** If argument zTab is not NULL, then it is the name of a table to attach +** to the session object passed as the first argument. All subsequent changes +** made to the table while the session object is enabled will be recorded. See +** documentation for [sqlite3session_changeset()] for further details. +** +** Or, if argument zTab is NULL, then changes are recorded for all tables +** in the database. If additional tables are added to the database (by +** executing "CREATE TABLE" statements) after this call is made, changes for +** the new tables are also recorded. +** +** Changes can only be recorded for tables that have a PRIMARY KEY explicitly +** defined as part of their CREATE TABLE statement. It does not matter if the +** PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) or not. The PRIMARY +** KEY may consist of a single column, or may be a composite key. +** +** It is not an error if the named table does not exist in the database. Nor +** is it an error if the named table does not have a PRIMARY KEY. However, +** no changes will be recorded in either of these scenarios. +** +** Changes are not recorded for individual rows that have NULL values stored +** in one or more of their PRIMARY KEY columns. +** +** SQLITE_OK is returned if the call completes without error. Or, if an error +** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned. +*/ +int sqlite3session_attach( + sqlite3_session *pSession, /* Session object */ + const char *zTab /* Table name */ +); + +/* +** CAPI3REF: Set a table filter on a Session Object. +** +** The second argument (xFilter) is the "filter callback". For changes to rows +** in tables that are not attached to the Session object, the filter is called +** to determine whether changes to the table's rows should be tracked or not. +** If xFilter returns 0, changes is not tracked. Note that once a table is +** attached, xFilter will not be called again. +*/ +void sqlite3session_table_filter( + sqlite3_session *pSession, /* Session object */ + int(*xFilter)( + void *pCtx, /* Copy of third arg to _filter_table() */ + const char *zTab /* Table name */ + ), + void *pCtx /* First argument passed to xFilter */ +); + +/* +** CAPI3REF: Generate A Changeset From A Session Object +** +** Obtain a changeset containing changes to the tables attached to the +** session object passed as the first argument. If successful, +** set *ppChangeset to point to a buffer containing the changeset +** and *pnChangeset to the size of the changeset in bytes before returning +** SQLITE_OK. If an error occurs, set both *ppChangeset and *pnChangeset to +** zero and return an SQLite error code. +** +** A changeset consists of zero or more INSERT, UPDATE and/or DELETE changes, +** each representing a change to a single row of an attached table. An INSERT +** change contains the values of each field of a new database row. A DELETE +** contains the original values of each field of a deleted database row. An +** UPDATE change contains the original values of each field of an updated +** database row along with the updated values for each updated non-primary-key +** column. It is not possible for an UPDATE change to represent a change that +** modifies the values of primary key columns. If such a change is made, it +** is represented in a changeset as a DELETE followed by an INSERT. +** +** Changes are not recorded for rows that have NULL values stored in one or +** more of their PRIMARY KEY columns. If such a row is inserted or deleted, +** no corresponding change is present in the changesets returned by this +** function. If an existing row with one or more NULL values stored in +** PRIMARY KEY columns is updated so that all PRIMARY KEY columns are non-NULL, +** only an INSERT is appears in the changeset. Similarly, if an existing row +** with non-NULL PRIMARY KEY values is updated so that one or more of its +** PRIMARY KEY columns are set to NULL, the resulting changeset contains a +** DELETE change only. +** +** The contents of a changeset may be traversed using an iterator created +** using the [sqlite3changeset_start()] API. A changeset may be applied to +** a database with a compatible schema using the [sqlite3changeset_apply()] +** API. +** +** Within a changeset generated by this function, all changes related to a +** single table are grouped together. In other words, when iterating through +** a changeset or when applying a changeset to a database, all changes related +** to a single table are processed before moving on to the next table. Tables +** are sorted in the same order in which they were attached (or auto-attached) +** to the sqlite3_session object. The order in which the changes related to +** a single table are stored is undefined. +** +** Following a successful call to this function, it is the responsibility of +** the caller to eventually free the buffer that *ppChangeset points to using +** [sqlite3_free()]. +** +**

    Changeset Generation

    +** +** Once a table has been attached to a session object, the session object +** records the primary key values of all new rows inserted into the table. +** It also records the original primary key and other column values of any +** deleted or updated rows. For each unique primary key value, data is only +** recorded once - the first time a row with said primary key is inserted, +** updated or deleted in the lifetime of the session. +** +** There is one exception to the previous paragraph: when a row is inserted, +** updated or deleted, if one or more of its primary key columns contain a +** NULL value, no record of the change is made. +** +** The session object therefore accumulates two types of records - those +** that consist of primary key values only (created when the user inserts +** a new record) and those that consist of the primary key values and the +** original values of other table columns (created when the users deletes +** or updates a record). +** +** When this function is called, the requested changeset is created using +** both the accumulated records and the current contents of the database +** file. Specifically: +** +**
      +**
    • For each record generated by an insert, the database is queried +** for a row with a matching primary key. If one is found, an INSERT +** change is added to the changeset. If no such row is found, no change +** is added to the changeset. +** +**
    • For each record generated by an update or delete, the database is +** queried for a row with a matching primary key. If such a row is +** found and one or more of the non-primary key fields have been +** modified from their original values, an UPDATE change is added to +** the changeset. Or, if no such row is found in the table, a DELETE +** change is added to the changeset. If there is a row with a matching +** primary key in the database, but all fields contain their original +** values, no change is added to the changeset. +**
    +** +** This means, amongst other things, that if a row is inserted and then later +** deleted while a session object is active, neither the insert nor the delete +** will be present in the changeset. Or if a row is deleted and then later a +** row with the same primary key values inserted while a session object is +** active, the resulting changeset will contain an UPDATE change instead of +** a DELETE and an INSERT. +** +** When a session object is disabled (see the [sqlite3session_enable()] API), +** it does not accumulate records when rows are inserted, updated or deleted. +** This may appear to have some counter-intuitive effects if a single row +** is written to more than once during a session. For example, if a row +** is inserted while a session object is enabled, then later deleted while +** the same session object is disabled, no INSERT record will appear in the +** changeset, even though the delete took place while the session was disabled. +** Or, if one field of a row is updated while a session is disabled, and +** another field of the same row is updated while the session is enabled, the +** resulting changeset will contain an UPDATE change that updates both fields. +*/ +int sqlite3session_changeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Load The Difference Between Tables Into A Session +** +** If it is not already attached to the session object passed as the first +** argument, this function attaches table zTbl in the same manner as the +** [sqlite3session_attach()] function. If zTbl does not exist, or if it +** does not have a primary key, this function is a no-op (but does not return +** an error). +** +** Argument zFromDb must be the name of a database ("main", "temp" etc.) +** attached to the same database handle as the session object that contains +** a table compatible with the table attached to the session by this function. +** A table is considered compatible if it: +** +**
      +**
    • Has the same name, +**
    • Has the same set of columns declared in the same order, and +**
    • Has the same PRIMARY KEY definition. +**
    +** +** If the tables are not compatible, SQLITE_SCHEMA is returned. If the tables +** are compatible but do not have any PRIMARY KEY columns, it is not an error +** but no changes are added to the session object. As with other session +** APIs, tables without PRIMARY KEYs are simply ignored. +** +** This function adds a set of changes to the session object that could be +** used to update the table in database zFrom (call this the "from-table") +** so that its content is the same as the table attached to the session +** object (call this the "to-table"). Specifically: +** +**
      +**
    • For each row (primary key) that exists in the to-table but not in +** the from-table, an INSERT record is added to the session object. +** +**
    • For each row (primary key) that exists in the to-table but not in +** the from-table, a DELETE record is added to the session object. +** +**
    • For each row (primary key) that exists in both tables, but features +** different in each, an UPDATE record is added to the session. +**
    +** +** To clarify, if this function is called and then a changeset constructed +** using [sqlite3session_changeset()], then after applying that changeset to +** database zFrom the contents of the two compatible tables would be +** identical. +** +** It an error if database zFrom does not exist or does not contain the +** required compatible table. +** +** If the operation successful, SQLITE_OK is returned. Otherwise, an SQLite +** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to free this buffer using +** sqlite3_free(). +*/ +int sqlite3session_diff( + sqlite3_session *pSession, + const char *zFromDb, + const char *zTbl, + char **pzErrMsg +); + + +/* +** CAPI3REF: Generate A Patchset From A Session Object +** +** The differences between a patchset and a changeset are that: +** +**
      +**
    • DELETE records consist of the primary key fields only. The +** original values of other fields are omitted. +**
    • The original values of any modified fields are omitted from +** UPDATE records. +**
    +** +** A patchset blob may be used with up to date versions of all +** sqlite3changeset_xxx API functions except for sqlite3changeset_invert(), +** which returns SQLITE_CORRUPT if it is passed a patchset. Similarly, +** attempting to use a patchset blob with old versions of the +** sqlite3changeset_xxx APIs also provokes an SQLITE_CORRUPT error. +** +** Because the non-primary key "old.*" fields are omitted, no +** SQLITE_CHANGESET_DATA conflicts can be detected or reported if a patchset +** is passed to the sqlite3changeset_apply() API. Other conflict types work +** in the same way as for changesets. +** +** Changes within a patchset are ordered in the same way as for changesets +** generated by the sqlite3session_changeset() function (i.e. all changes for +** a single table are grouped together, tables appear in the order in which +** they were attached to the session object). +*/ +int sqlite3session_patchset( + sqlite3_session *pSession, /* Session object */ + int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ + void **ppPatchset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Test if a changeset has recorded any changes. +** +** Return non-zero if no changes to attached tables have been recorded by +** the session object passed as the first argument. Otherwise, if one or +** more changes have been recorded, return zero. +** +** Even if this function returns zero, it is possible that calling +** [sqlite3session_changeset()] on the session handle may still return a +** changeset that contains no changes. This can happen when a row in +** an attached table is modified and then later on the original values +** are restored. However, if this function returns non-zero, then it is +** guaranteed that a call to sqlite3session_changeset() will return a +** changeset containing zero changes. +*/ +int sqlite3session_isempty(sqlite3_session *pSession); + +/* +** CAPI3REF: Create An Iterator To Traverse A Changeset +** +** Create an iterator used to iterate through the contents of a changeset. +** If successful, *pp is set to point to the iterator handle and SQLITE_OK +** is returned. Otherwise, if an error occurs, *pp is set to zero and an +** SQLite error code is returned. +** +** The following functions can be used to advance and query a changeset +** iterator created by this function: +** +**
      +**
    • [sqlite3changeset_next()] +**
    • [sqlite3changeset_op()] +**
    • [sqlite3changeset_new()] +**
    • [sqlite3changeset_old()] +**
    +** +** It is the responsibility of the caller to eventually destroy the iterator +** by passing it to [sqlite3changeset_finalize()]. The buffer containing the +** changeset (pChangeset) must remain valid until after the iterator is +** destroyed. +** +** Assuming the changeset blob was created by one of the +** [sqlite3session_changeset()], [sqlite3changeset_concat()] or +** [sqlite3changeset_invert()] functions, all changes within the changeset +** that apply to a single table are grouped together. This means that when +** an application iterates through a changeset using an iterator created by +** this function, all changes that relate to a single table are visited +** consecutively. There is no chance that the iterator will visit a change +** the applies to table X, then one for table Y, and then later on visit +** another change for table X. +*/ +int sqlite3changeset_start( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset /* Pointer to blob containing changeset */ +); + + +/* +** CAPI3REF: Advance A Changeset Iterator +** +** This function may only be used with iterators created by function +** [sqlite3changeset_start()]. If it is called on an iterator passed to +** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE +** is returned and the call has no effect. +** +** Immediately after an iterator is created by sqlite3changeset_start(), it +** does not point to any change in the changeset. Assuming the changeset +** is not empty, the first call to this function advances the iterator to +** point to the first change in the changeset. Each subsequent call advances +** the iterator to point to the next change in the changeset (if any). If +** no error occurs and the iterator points to a valid change after a call +** to sqlite3changeset_next() has advanced it, SQLITE_ROW is returned. +** Otherwise, if all changes in the changeset have already been visited, +** SQLITE_DONE is returned. +** +** If an error occurs, an SQLite error code is returned. Possible error +** codes include SQLITE_CORRUPT (if the changeset buffer is corrupt) or +** SQLITE_NOMEM. +*/ +int sqlite3changeset_next(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Obtain The Current Operation From A Changeset Iterator +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned [SQLITE_ROW]. If this +** is not the case, this function returns [SQLITE_MISUSE]. +** +** If argument pzTab is not NULL, then *pzTab is set to point to a +** nul-terminated utf-8 encoded string containing the name of the table +** affected by the current change. The buffer remains valid until either +** sqlite3changeset_next() is called on the iterator or until the +** conflict-handler function returns. If pnCol is not NULL, then *pnCol is +** set to the number of columns in the table affected by the change. If +** pbIncorrect is not NULL, then *pbIndirect is set to true (1) if the change +** is an indirect change, or false (0) otherwise. See the documentation for +** [sqlite3session_indirect()] for a description of direct and indirect +** changes. Finally, if pOp is not NULL, then *pOp is set to one of +** [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], depending on the +** type of change that the iterator currently points to. +** +** If no error occurs, SQLITE_OK is returned. If an error does occur, an +** SQLite error code is returned. The values of the output variables may not +** be trusted in this case. +*/ +int sqlite3changeset_op( + sqlite3_changeset_iter *pIter, /* Iterator object */ + const char **pzTab, /* OUT: Pointer to table name */ + int *pnCol, /* OUT: Number of columns in table */ + int *pOp, /* OUT: SQLITE_INSERT, DELETE or UPDATE */ + int *pbIndirect /* OUT: True for an 'indirect' change */ +); + +/* +** CAPI3REF: Obtain The Primary Key Definition Of A Table +** +** For each modified table, a changeset includes the following: +** +**
      +**
    • The number of columns in the table, and +**
    • Which of those columns make up the tables PRIMARY KEY. +**
    +** +** This function is used to find which columns comprise the PRIMARY KEY of +** the table modified by the change that iterator pIter currently points to. +** If successful, *pabPK is set to point to an array of nCol entries, where +** nCol is the number of columns in the table. Elements of *pabPK are set to +** 0x01 if the corresponding column is part of the tables primary key, or +** 0x00 if it is not. +** +** If argument pnCol is not NULL, then *pnCol is set to the number of columns +** in the table. +** +** If this function is called when the iterator does not point to a valid +** entry, SQLITE_MISUSE is returned and the output variables zeroed. Otherwise, +** SQLITE_OK is returned and the output variables populated as described +** above. +*/ +int sqlite3changeset_pk( + sqlite3_changeset_iter *pIter, /* Iterator object */ + unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ + int *pnCol /* OUT: Number of entries in output array */ +); + +/* +** CAPI3REF: Obtain old.* Values From A Changeset Iterator +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_DELETE] or [SQLITE_UPDATE]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** original row values stored as part of the UPDATE or DELETE change and +** returns SQLITE_OK. The name of the function comes from the fact that this +** is similar to the "old.*" columns available to update or delete triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +int sqlite3changeset_old( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain new.* Values From A Changeset Iterator +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_UPDATE] or [SQLITE_INSERT]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** new row values stored as part of the UPDATE or INSERT change and +** returns SQLITE_OK. If the change is an UPDATE and does not include +** a new value for the requested column, *ppValue is set to NULL and +** SQLITE_OK returned. The name of the function comes from the fact that +** this is similar to the "new.*" columns available to update or delete +** triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +int sqlite3changeset_new( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator +** +** This function should only be used with iterator objects passed to a +** conflict-handler callback by [sqlite3changeset_apply()] with either +** [SQLITE_CHANGESET_DATA] or [SQLITE_CHANGESET_CONFLICT]. If this function +** is called on any other iterator, [SQLITE_MISUSE] is returned and *ppValue +** is set to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the +** "conflicting row" associated with the current conflict-handler callback +** and returns SQLITE_OK. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +int sqlite3changeset_conflict( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Value from conflicting row */ +); + +/* +** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations +** +** This function may only be called with an iterator passed to an +** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case +** it sets the output variable to the total number of known foreign key +** violations in the destination database and returns SQLITE_OK. +** +** In all other cases this function returns SQLITE_MISUSE. +*/ +int sqlite3changeset_fk_conflicts( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int *pnOut /* OUT: Number of FK violations */ +); + + +/* +** CAPI3REF: Finalize A Changeset Iterator +** +** This function is used to finalize an iterator allocated with +** [sqlite3changeset_start()]. +** +** This function should only be called on iterators created using the +** [sqlite3changeset_start()] function. If an application calls this +** function with an iterator passed to a conflict-handler by +** [sqlite3changeset_apply()], [SQLITE_MISUSE] is immediately returned and the +** call has no effect. +** +** If an error was encountered within a call to an sqlite3changeset_xxx() +** function (for example an [SQLITE_CORRUPT] in [sqlite3changeset_next()] or an +** [SQLITE_NOMEM] in [sqlite3changeset_new()]) then an error code corresponding +** to that error is returned by this function. Otherwise, SQLITE_OK is +** returned. This is to allow the following pattern (pseudo-code): +** +** sqlite3changeset_start(); +** while( SQLITE_ROW==sqlite3changeset_next() ){ +** // Do something with change. +** } +** rc = sqlite3changeset_finalize(); +** if( rc!=SQLITE_OK ){ +** // An error has occurred +** } +*/ +int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Invert A Changeset +** +** This function is used to "invert" a changeset object. Applying an inverted +** changeset to a database reverses the effects of applying the uninverted +** changeset. Specifically: +** +**
      +**
    • Each DELETE change is changed to an INSERT, and +**
    • Each INSERT change is changed to a DELETE, and +**
    • For each UPDATE change, the old.* and new.* values are exchanged. +**
    +** +** This function does not change the order in which changes appear within +** the changeset. It merely reverses the sense of each individual change. +** +** If successful, a pointer to a buffer containing the inverted changeset +** is stored in *ppOut, the size of the same buffer is stored in *pnOut, and +** SQLITE_OK is returned. If an error occurs, both *pnOut and *ppOut are +** zeroed and an SQLite error code returned. +** +** It is the responsibility of the caller to eventually call sqlite3_free() +** on the *ppOut pointer to free the buffer allocation following a successful +** call to this function. +** +** WARNING/TODO: This function currently assumes that the input is a valid +** changeset. If it is not, the results are undefined. +*/ +int sqlite3changeset_invert( + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + +/* +** CAPI3REF: Concatenate Two Changeset Objects +** +** This function is used to concatenate two changesets, A and B, into a +** single changeset. The result is a changeset equivalent to applying +** changeset A followed by changeset B. +** +** This function combines the two input changesets using an +** sqlite3_changegroup object. Calling it produces similar results as the +** following code fragment: +** +** sqlite3_changegroup *pGrp; +** rc = sqlite3_changegroup_new(&pGrp); +** if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA); +** if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nB, pB); +** if( rc==SQLITE_OK ){ +** rc = sqlite3changegroup_output(pGrp, pnOut, ppOut); +** }else{ +** *ppOut = 0; +** *pnOut = 0; +** } +** +** Refer to the sqlite3_changegroup documentation below for details. +*/ +int sqlite3changeset_concat( + int nA, /* Number of bytes in buffer pA */ + void *pA, /* Pointer to buffer containing changeset A */ + int nB, /* Number of bytes in buffer pB */ + void *pB, /* Pointer to buffer containing changeset B */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: Buffer containing output changeset */ +); + + +/* +** CAPI3REF: Changegroup Handle +*/ +typedef struct sqlite3_changegroup sqlite3_changegroup; + +/* +** CAPI3REF: Create A New Changegroup Object +** +** An sqlite3_changegroup object is used to combine two or more changesets +** (or patchsets) into a single changeset (or patchset). A single changegroup +** object may combine changesets or patchsets, but not both. The output is +** always in the same format as the input. +** +** If successful, this function returns SQLITE_OK and populates (*pp) with +** a pointer to a new sqlite3_changegroup object before returning. The caller +** should eventually free the returned object using a call to +** sqlite3changegroup_delete(). If an error occurs, an SQLite error code +** (i.e. SQLITE_NOMEM) is returned and *pp is set to NULL. +** +** The usual usage pattern for an sqlite3_changegroup object is as follows: +** +**
      +**
    • It is created using a call to sqlite3changegroup_new(). +** +**
    • Zero or more changesets (or patchsets) are added to the object +** by calling sqlite3changegroup_add(). +** +**
    • The result of combining all input changesets together is obtained +** by the application via a call to sqlite3changegroup_output(). +** +**
    • The object is deleted using a call to sqlite3changegroup_delete(). +**
    +** +** Any number of calls to add() and output() may be made between the calls to +** new() and delete(), and in any order. +** +** As well as the regular sqlite3changegroup_add() and +** sqlite3changegroup_output() functions, also available are the streaming +** versions sqlite3changegroup_add_strm() and sqlite3changegroup_output_strm(). +*/ +int sqlite3changegroup_new(sqlite3_changegroup **pp); + +/* +** CAPI3REF: Add A Changeset To A Changegroup +** +** Add all changes within the changeset (or patchset) in buffer pData (size +** nData bytes) to the changegroup. +** +** If the buffer contains a patchset, then all prior calls to this function +** on the same changegroup object must also have specified patchsets. Or, if +** the buffer contains a changeset, so must have the earlier calls to this +** function. Otherwise, SQLITE_ERROR is returned and no changes are added +** to the changegroup. +** +** Rows within the changeset and changegroup are identified by the values in +** their PRIMARY KEY columns. A change in the changeset is considered to +** apply to the same row as a change already present in the changegroup if +** the two rows have the same primary key. +** +** Changes to rows that do not already appear in the changegroup are +** simply copied into it. Or, if both the new changeset and the changegroup +** contain changes that apply to a single row, the final contents of the +** changegroup depends on the type of each change, as follows: +** +** +** +** +**
    Existing Change New Change Output Change +**
    INSERT INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    INSERT UPDATE +** The INSERT change remains in the changegroup. The values in the +** INSERT change are modified as if the row was inserted by the +** existing change and then updated according to the new change. +**
    INSERT DELETE +** The existing INSERT is removed from the changegroup. The DELETE is +** not added. +**
    UPDATE INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    UPDATE UPDATE +** The existing UPDATE remains within the changegroup. It is amended +** so that the accompanying values are as if the row was updated once +** by the existing change and then again by the new change. +**
    UPDATE DELETE +** The existing UPDATE is replaced by the new DELETE within the +** changegroup. +**
    DELETE INSERT +** If one or more of the column values in the row inserted by the +** new change differ from those in the row deleted by the existing +** change, the existing DELETE is replaced by an UPDATE within the +** changegroup. Otherwise, if the inserted row is exactly the same +** as the deleted row, the existing DELETE is simply discarded. +**
    DELETE UPDATE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    DELETE DELETE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    +** +** If the new changeset contains changes to a table that is already present +** in the changegroup, then the number of columns and the position of the +** primary key columns for the table must be consistent. If this is not the +** case, this function fails with SQLITE_SCHEMA. If the input changeset +** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is +** returned. Or, if an out-of-memory condition occurs during processing, this +** function returns SQLITE_NOMEM. In all cases, if an error occurs the +** final contents of the changegroup is undefined. +** +** If no error occurs, SQLITE_OK is returned. +*/ +int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); + +/* +** CAPI3REF: Obtain A Composite Changeset From A Changegroup +** +** Obtain a buffer containing a changeset (or patchset) representing the +** current contents of the changegroup. If the inputs to the changegroup +** were themselves changesets, the output is a changeset. Or, if the +** inputs were patchsets, the output is also a patchset. +** +** As with the output of the sqlite3session_changeset() and +** sqlite3session_patchset() functions, all changes related to a single +** table are grouped together in the output of this function. Tables appear +** in the same order as for the very first changeset added to the changegroup. +** If the second or subsequent changesets added to the changegroup contain +** changes for tables that do not appear in the first changeset, they are +** appended onto the end of the output changeset, again in the order in +** which they are first encountered. +** +** If an error occurs, an SQLite error code is returned and the output +** variables (*pnData) and (*ppData) are set to 0. Otherwise, SQLITE_OK +** is returned and the output variables are set to the size of and a +** pointer to the output buffer, respectively. In this case it is the +** responsibility of the caller to eventually free the buffer using a +** call to sqlite3_free(). +*/ +int sqlite3changegroup_output( + sqlite3_changegroup*, + int *pnData, /* OUT: Size of output buffer in bytes */ + void **ppData /* OUT: Pointer to output buffer */ +); + +/* +** CAPI3REF: Delete A Changegroup Object +*/ +void sqlite3changegroup_delete(sqlite3_changegroup*); + +/* +** CAPI3REF: Apply A Changeset To A Database +** +** Apply a changeset to a database. This function attempts to update the +** "main" database attached to handle db with the changes found in the +** changeset passed via the second and third arguments. +** +** The fourth argument (xFilter) passed to this function is the "filter +** callback". If it is not NULL, then for each table affected by at least one +** change in the changeset, the filter callback is invoked with +** the table name as the second argument, and a copy of the context pointer +** passed as the sixth argument to this function as the first. If the "filter +** callback" returns zero, then no attempt is made to apply any changes to +** the table. Otherwise, if the return value is non-zero or the xFilter +** argument to this function is NULL, all changes related to the table are +** attempted. +** +** For each table that is not excluded by the filter callback, this function +** tests that the target database contains a compatible table. A table is +** considered compatible if all of the following are true: +** +**
      +**
    • The table has the same name as the name recorded in the +** changeset, and +**
    • The table has the same number of columns as recorded in the +** changeset, and +**
    • The table has primary key columns in the same position as +** recorded in the changeset. +**
    +** +** If there is no compatible table, it is not an error, but none of the +** changes associated with the table are applied. A warning message is issued +** via the sqlite3_log() mechanism with the error code SQLITE_SCHEMA. At most +** one such warning is issued for each table in the changeset. +** +** For each change for which there is a compatible table, an attempt is made +** to modify the table contents according to the UPDATE, INSERT or DELETE +** change. If a change cannot be applied cleanly, the conflict handler +** function passed as the fifth argument to sqlite3changeset_apply() may be +** invoked. A description of exactly when the conflict handler is invoked for +** each type of change is below. +** +** Unlike the xFilter argument, xConflict may not be passed NULL. The results +** of passing anything other than a valid function pointer as the xConflict +** argument are undefined. +** +** Each time the conflict handler function is invoked, it must return one +** of [SQLITE_CHANGESET_OMIT], [SQLITE_CHANGESET_ABORT] or +** [SQLITE_CHANGESET_REPLACE]. SQLITE_CHANGESET_REPLACE may only be returned +** if the second argument passed to the conflict handler is either +** SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If the conflict-handler +** returns an illegal value, any changes already made are rolled back and +** the call to sqlite3changeset_apply() returns SQLITE_MISUSE. Different +** actions are taken by sqlite3changeset_apply() depending on the value +** returned by each invocation of the conflict-handler function. Refer to +** the documentation for the three +** [SQLITE_CHANGESET_OMIT|available return values] for details. +** +**
    +**
    DELETE Changes
    +** For each DELETE change, this function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all non-primary key columns also match the values stored in +** the changeset the row is deleted from the target database. +** +** If a row with matching primary key values is found, but one or more of +** the non-primary key fields contains a value different from the original +** row value stored in the changeset, the conflict-handler function is +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the DELETE operation is attempted, but SQLite returns SQLITE_CONSTRAINT +** (which can only happen if a foreign key constraint is violated), the +** conflict-handler function is invoked with [SQLITE_CHANGESET_CONSTRAINT] +** passed as the second argument. This includes the case where the DELETE +** operation is attempted because an earlier call to the conflict handler +** function returned [SQLITE_CHANGESET_REPLACE]. +** +**
    INSERT Changes
    +** For each INSERT change, an attempt is made to insert the new row into +** the database. +** +** If the attempt to insert the row fails because the database already +** contains a row with the same primary key values, the conflict handler +** function is invoked with the second argument set to +** [SQLITE_CHANGESET_CONFLICT]. +** +** If the attempt to insert the row fails because of some other constraint +** violation (e.g. NOT NULL or UNIQUE), the conflict handler function is +** invoked with the second argument set to [SQLITE_CHANGESET_CONSTRAINT]. +** This includes the case where the INSERT operation is re-attempted because +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +** +**
    UPDATE Changes
    +** For each UPDATE change, this function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all non-primary key columns also match the values stored in +** the changeset the row is updated within the target database. +** +** If a row with matching primary key values is found, but one or more of +** the non-primary key fields contains a value different from an original +** row value stored in the changeset, the conflict-handler function is +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since +** UPDATE changes only contain values for non-primary key fields that are +** to be modified, only those fields need to match the original values to +** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the UPDATE operation is attempted, but SQLite returns +** SQLITE_CONSTRAINT, the conflict-handler function is invoked with +** [SQLITE_CHANGESET_CONSTRAINT] passed as the second argument. +** This includes the case where the UPDATE operation is attempted after +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +**
    +** +** It is safe to execute SQL statements, including those that write to the +** table that the callback related to, from within the xConflict callback. +** This can be used to further customize the applications conflict +** resolution strategy. +** +** All changes made by this function are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. +*/ +int sqlite3changeset_apply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); + +/* +** CAPI3REF: Constants Passed To The Conflict Handler +** +** Values that may be passed as the second argument to a conflict-handler. +** +**
    +**
    SQLITE_CHANGESET_DATA
    +** The conflict handler is invoked with CHANGESET_DATA as the second argument +** when processing a DELETE or UPDATE change if a row with the required +** PRIMARY KEY fields is present in the database, but one or more other +** (non primary-key) fields modified by the update do not contain the +** expected "before" values. +** +** The conflicting row, in this case, is the database row with the matching +** primary key. +** +**
    SQLITE_CHANGESET_NOTFOUND
    +** The conflict handler is invoked with CHANGESET_NOTFOUND as the second +** argument when processing a DELETE or UPDATE change if a row with the +** required PRIMARY KEY fields is not present in the database. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
    SQLITE_CHANGESET_CONFLICT
    +** CHANGESET_CONFLICT is passed as the second argument to the conflict +** handler while processing an INSERT change if the operation would result +** in duplicate primary key values. +** +** The conflicting row in this case is the database row with the matching +** primary key. +** +**
    SQLITE_CHANGESET_FOREIGN_KEY
    +** If foreign key handling is enabled, and applying a changeset leaves the +** database in a state containing foreign key violations, the conflict +** handler is invoked with CHANGESET_FOREIGN_KEY as the second argument +** exactly once before the changeset is committed. If the conflict handler +** returns CHANGESET_OMIT, the changes, including those that caused the +** foreign key constraint violation, are committed. Or, if it returns +** CHANGESET_ABORT, the changeset is rolled back. +** +** No current or conflicting row information is provided. The only function +** it is possible to call on the supplied sqlite3_changeset_iter handle +** is sqlite3changeset_fk_conflicts(). +** +**
    SQLITE_CHANGESET_CONSTRAINT
    +** If any other constraint violation occurs while applying a change (i.e. +** a UNIQUE, CHECK or NOT NULL constraint), the conflict handler is +** invoked with CHANGESET_CONSTRAINT as the second argument. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
    +*/ +#define SQLITE_CHANGESET_DATA 1 +#define SQLITE_CHANGESET_NOTFOUND 2 +#define SQLITE_CHANGESET_CONFLICT 3 +#define SQLITE_CHANGESET_CONSTRAINT 4 +#define SQLITE_CHANGESET_FOREIGN_KEY 5 + +/* +** CAPI3REF: Constants Returned By The Conflict Handler +** +** A conflict handler callback must return one of the following three values. +** +**
    +**
    SQLITE_CHANGESET_OMIT
    +** If a conflict handler returns this value no special action is taken. The +** change that caused the conflict is not applied. The session module +** continues to the next change in the changeset. +** +**
    SQLITE_CHANGESET_REPLACE
    +** This value may only be returned if the second argument to the conflict +** handler was SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If this +** is not the case, any changes applied so far are rolled back and the +** call to sqlite3changeset_apply() returns SQLITE_MISUSE. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_DATA conflict +** handler, then the conflicting row is either updated or deleted, depending +** on the type of change. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_CONFLICT conflict +** handler, then the conflicting row is removed from the database and a +** second attempt to apply the change is made. If this second attempt fails, +** the original row is restored to the database before continuing. +** +**
    SQLITE_CHANGESET_ABORT
    +** If this value is returned, any changes applied so far are rolled back +** and the call to sqlite3changeset_apply() returns SQLITE_ABORT. +**
    +*/ +#define SQLITE_CHANGESET_OMIT 0 +#define SQLITE_CHANGESET_REPLACE 1 +#define SQLITE_CHANGESET_ABORT 2 + +/* +** CAPI3REF: Streaming Versions of API functions. +** +** The six streaming API xxx_strm() functions serve similar purposes to the +** corresponding non-streaming API functions: +** +** +** +**
    Streaming functionNon-streaming equivalent
    sqlite3changeset_apply_str[sqlite3changeset_apply] +**
    sqlite3changeset_concat_str[sqlite3changeset_concat] +**
    sqlite3changeset_invert_str[sqlite3changeset_invert] +**
    sqlite3changeset_start_str[sqlite3changeset_start] +**
    sqlite3session_changeset_str[sqlite3session_changeset] +**
    sqlite3session_patchset_str[sqlite3session_patchset] +**
    +** +** Non-streaming functions that accept changesets (or patchsets) as input +** require that the entire changeset be stored in a single buffer in memory. +** Similarly, those that return a changeset or patchset do so by returning +** a pointer to a single large buffer allocated using sqlite3_malloc(). +** Normally this is convenient. However, if an application running in a +** low-memory environment is required to handle very large changesets, the +** large contiguous memory allocations required can become onerous. +** +** In order to avoid this problem, instead of a single large buffer, input +** is passed to a streaming API functions by way of a callback function that +** the sessions module invokes to incrementally request input data as it is +** required. In all cases, a pair of API function parameters such as +** +** 
    +**        int nChangeset,
+**        void *pChangeset,
+**
    +** +** Is replaced by: +** +** 
    +**        int (*xInput)(void *pIn, void *pData, int *pnData),
+**        void *pIn,
+**
    +** +** Each time the xInput callback is invoked by the sessions module, the first +** argument passed is a copy of the supplied pIn context pointer. The second +** argument, pData, points to a buffer (*pnData) bytes in size. Assuming no +** error occurs the xInput method should copy up to (*pnData) bytes of data +** into the buffer and set (*pnData) to the actual number of bytes copied +** before returning SQLITE_OK. If the input is completely exhausted, (*pnData) +** should be set to zero to indicate this. Or, if an error occurs, an SQLite +** error code should be returned. In all cases, if an xInput callback returns +** an error, all processing is abandoned and the streaming API function +** returns a copy of the error code to the caller. +** +** In the case of sqlite3changeset_start_strm(), the xInput callback may be +** invoked by the sessions module at any point during the lifetime of the +** iterator. If such an xInput callback returns an error, the iterator enters +** an error state, whereby all subsequent calls to iterator functions +** immediately fail with the same error code as returned by xInput. +** +** Similarly, streaming API functions that return changesets (or patchsets) +** return them in chunks by way of a callback function instead of via a +** pointer to a single large buffer. In this case, a pair of parameters such +** as: +** +** 
    +**        int *pnChangeset,
+**        void **ppChangeset,
+**
    +** +** Is replaced by: +** +** 
    +**        int (*xOutput)(void *pOut, const void *pData, int nData),
+**        void *pOut
+**
    +** +** The xOutput callback is invoked zero or more times to return data to +** the application. The first parameter passed to each call is a copy of the +** pOut pointer supplied by the application. The second parameter, pData, +** points to a buffer nData bytes in size containing the chunk of output +** data being returned. If the xOutput callback successfully processes the +** supplied data, it should return SQLITE_OK to indicate success. Otherwise, +** it should return some other SQLite error code. In this case processing +** is immediately abandoned and the streaming API function returns a copy +** of the xOutput error code to the application. +** +** The sessions module never invokes an xOutput callback with the third +** parameter set to a value less than or equal to zero. Other than this, +** no guarantees are made as to the size of the chunks of data returned. +*/ +int sqlite3changeset_apply_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); +int sqlite3changeset_concat_strm( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3changeset_invert_strm( + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3changeset_start_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +int sqlite3session_changeset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3session_patchset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3changegroup_add_strm(sqlite3_changegroup*, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +int sqlite3changegroup_output_strm(sqlite3_changegroup*, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); + + +/* +** Make sure we can call this stuff from C++. +*/ +#if 0 +} +#endif + +#endif /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */ + +/******** End of sqlite3session.h *********/ +/******** Begin file fts5.h *********/ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + + +#if 0 +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. The exception is if the table was created +** with the offsets=0 option specified. In this case *piOff is always +** set to -1. +** +** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) +** if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. Any column filter that applies to +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback +** function may be used to access the properties of each matched row. +** Invoking Api.xUserData() returns a copy of the pointer passed as +** the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension functions +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** of the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, an +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iCol>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). +** +** xPhraseNext() +** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 3 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and initialize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**
    • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
    • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
    • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
    • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
    +** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
    1. By mapping all synonyms to a single token. In this case, the +** In the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
    2. By adding multiple synonyms for a single term to the FTS index. +** In this case, when tokenizing query text, the tokenizer may +** provide multiple synonyms for a single term within the document. +** FTS5 then queries the index for each synonym individually. For +** example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
    3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entires in the +** FTS index corresponding to both forms of the first token. +**
    +** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is subsituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + +/******** End of fts5.h *********/ /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -7721,14 +10746,16 @@ struct sqlite3_rtree_query_info { #endif /* -** The maximum number of in-memory pages to use for the main database -** table and for temporary tables. The SQLITE_DEFAULT_CACHE_SIZE +** The suggested maximum number of in-memory pages to use for +** the main database table and for temporary tables. +** +** IMPLEMENTATION-OF: R-30185-15359 The default suggested cache size is -2000, +** which means the cache size is limited to 2048000 bytes of memory. +** IMPLEMENTATION-OF: R-48205-43578 The default suggested cache size can be +** altered using the SQLITE_DEFAULT_CACHE_SIZE compile-time options. */ #ifndef SQLITE_DEFAULT_CACHE_SIZE -# define SQLITE_DEFAULT_CACHE_SIZE 2000 -#endif -#ifndef SQLITE_DEFAULT_TEMP_CACHE_SIZE -# define SQLITE_DEFAULT_TEMP_CACHE_SIZE 500 +# define SQLITE_DEFAULT_CACHE_SIZE -2000 #endif /* @@ -7741,8 +10768,9 @@ struct sqlite3_rtree_query_info { /* ** The maximum number of attached databases. This must be between 0 -** and 62. The upper bound on 62 is because a 64-bit integer bitmap -** is used internally to track attached databases. +** and 125. The upper bound of 125 is because the attached databases are +** counted using a signed 8-bit integer which has a maximum value of 127 +** and we have to allow 2 extra counts for the "main" and "temp" databases. */ #ifndef SQLITE_MAX_ATTACHED # define SQLITE_MAX_ATTACHED 10 @@ -7777,7 +10805,7 @@ struct sqlite3_rtree_query_info { ** The default size of a database page. */ #ifndef SQLITE_DEFAULT_PAGE_SIZE -# define SQLITE_DEFAULT_PAGE_SIZE 1024 +# define SQLITE_DEFAULT_PAGE_SIZE 4096 #endif #if SQLITE_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE # undef SQLITE_DEFAULT_PAGE_SIZE @@ -7842,15 +10870,6 @@ struct sqlite3_rtree_query_info { #pragma warn -spa /* Suspicious pointer arithmetic */ #endif -/* Needed for various definitions... */ -#ifndef _GNU_SOURCE -# define _GNU_SOURCE -#endif - -#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) -# define _BSD_SOURCE -#endif - /* ** Include standard header files as necessary */ @@ -7867,7 +10886,7 @@ struct sqlite3_rtree_query_info { ** to the next, so we have developed the following set of #if statements ** to generate appropriate macros for a wide range of compilers. ** -** The correct "ANSI" way to do this is to use the intptr_t type. +** The correct "ANSI" way to do this is to use the intptr_t type. ** Unfortunately, that typedef is not available on all compilers, or ** if it is available, it requires an #include of specific headers ** that vary from one machine to the next. @@ -7891,6 +10910,36 @@ struct sqlite3_rtree_query_info { # define SQLITE_PTR_TO_INT(X) ((int)(X)) #endif +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define SQLITE_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define SQLITE_NOINLINE __declspec(noinline) +#else +# define SQLITE_NOINLINE +#endif + +/* +** Make sure that the compiler intrinsics we desire are enabled when +** compiling with an appropriate version of MSVC unless prevented by +** the SQLITE_DISABLE_INTRINSIC define. +*/ +#if !defined(SQLITE_DISABLE_INTRINSIC) +# if defined(_MSC_VER) && _MSC_VER>=1400 +# if !defined(_WIN32_WCE) +# include +# pragma intrinsic(_byteswap_ushort) +# pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_ReadWriteBarrier) +# else +# include +# endif +# endif +#endif + /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never @@ -7919,10 +10968,9 @@ struct sqlite3_rtree_query_info { #endif /* -** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. -** It determines whether or not the features related to -** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can -** be overridden at runtime using the sqlite3_config() API. +** EVIDENCE-OF: R-25715-37072 Memory allocation statistics are enabled by +** default unless SQLite is compiled with SQLITE_DEFAULT_MEMSTATUS=0 in +** which case memory allocation statistics are disabled by default. */ #if !defined(SQLITE_DEFAULT_MEMSTATUS) # define SQLITE_DEFAULT_MEMSTATUS 1 @@ -7990,7 +11038,7 @@ struct sqlite3_rtree_query_info { ** is set. Thus NDEBUG becomes an opt-in rather than an opt-out ** feature. */ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif #if defined(NDEBUG) && defined(SQLITE_DEBUG) @@ -8005,7 +11053,7 @@ struct sqlite3_rtree_query_info { #endif /* -** The testcase() macro is used to aid in coverage testing. When +** The testcase() macro is used to aid in coverage testing. When ** doing coverage testing, the condition inside the argument to ** testcase() must be evaluated both true and false in order to ** get full branch coverage. The testcase() macro is inserted @@ -8051,7 +11099,7 @@ SQLITE_PRIVATE void sqlite3Coverage(int); #endif /* -** The ALWAYS and NEVER macros surround boolean expressions which +** The ALWAYS and NEVER macros surround boolean expressions which ** are intended to always be true or false, respectively. Such ** expressions could be omitted from the code completely. But they ** are included in a few cases in order to enhance the resilience @@ -8065,7 +11113,7 @@ SQLITE_PRIVATE void sqlite3Coverage(int); ** be true and false so that the unreachable code they specify will ** not be counted as untested code. */ -#if defined(SQLITE_COVERAGE_TEST) +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) # define ALWAYS(X) (1) # define NEVER(X) (0) #elif !defined(NDEBUG) @@ -8077,7 +11125,55 @@ SQLITE_PRIVATE void sqlite3Coverage(int); #endif /* -** Return true (non-zero) if the input is a integer that is too large +** Some malloc failures are only possible if SQLITE_TEST_REALLOC_STRESS is +** defined. We need to defend against those failures when testing with +** SQLITE_TEST_REALLOC_STRESS, but we don't want the unreachable branches +** during a normal build. The following macro can be used to disable tests +** that are always false except when SQLITE_TEST_REALLOC_STRESS is set. +*/ +#if defined(SQLITE_TEST_REALLOC_STRESS) +# define ONLY_IF_REALLOC_STRESS(X) (X) +#elif !defined(NDEBUG) +# define ONLY_IF_REALLOC_STRESS(X) ((X)?(assert(0),1):0) +#else +# define ONLY_IF_REALLOC_STRESS(X) (0) +#endif + +/* +** Declarations used for tracing the operating system interfaces. +*/ +#if defined(SQLITE_FORCE_OS_TRACE) || defined(SQLITE_TEST) || \ + (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) + extern int sqlite3OSTrace; +# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X +# define SQLITE_HAVE_OS_TRACE +#else +# define OSTRACE(X) +# undef SQLITE_HAVE_OS_TRACE +#endif + +/* +** Is the sqlite3ErrName() function needed in the build? Currently, +** it is needed by "mutex_w32.c" (when debugging), "os_win.c" (when +** OSTRACE is enabled), and by several "test*.c" files (which are +** compiled using SQLITE_TEST). +*/ +#if defined(SQLITE_HAVE_OS_TRACE) || defined(SQLITE_TEST) || \ + (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) +# define SQLITE_NEED_ERR_NAME +#else +# undef SQLITE_NEED_ERR_NAME +#endif + +/* +** SQLITE_ENABLE_EXPLAIN_COMMENTS is incompatible with SQLITE_OMIT_EXPLAIN +*/ +#ifdef SQLITE_OMIT_EXPLAIN +# undef SQLITE_ENABLE_EXPLAIN_COMMENTS +#endif + +/* +** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() ** macros to verify that we have tested SQLite for large-file support. */ @@ -8109,8 +11205,8 @@ SQLITE_PRIVATE void sqlite3Coverage(int); ** This is the header file for the generic hash-table implementation ** used in SQLite. */ -#ifndef _SQLITE_HASH_H_ -#define _SQLITE_HASH_H_ +#ifndef SQLITE_HASH_H +#define SQLITE_HASH_H /* Forward declarations of structures. */ typedef struct Hash Hash; @@ -8156,15 +11252,15 @@ struct Hash { struct HashElem { HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ - const char *pKey; int nKey; /* Key associated with this element */ + const char *pKey; /* Key associated with this element */ }; /* ** Access routines. To delete, insert a NULL pointer. */ SQLITE_PRIVATE void sqlite3HashInit(Hash*); -SQLITE_PRIVATE void *sqlite3HashInsert(Hash*, const char *pKey, int nKey, void *pData); -SQLITE_PRIVATE void *sqlite3HashFind(const Hash*, const char *pKey, int nKey); +SQLITE_PRIVATE void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); +SQLITE_PRIVATE void *sqlite3HashFind(const Hash*, const char *pKey); SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* @@ -8190,7 +11286,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); */ /* #define sqliteHashCount(H) ((H)->count) // NOT USED */ -#endif /* _SQLITE_HASH_H_ */ +#endif /* SQLITE_HASH_H */ /************** End of hash.h ************************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -8222,76 +11318,76 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_AS 24 #define TK_WITHOUT 25 #define TK_COMMA 26 -#define TK_ID 27 -#define TK_INDEXED 28 -#define TK_ABORT 29 -#define TK_ACTION 30 -#define TK_AFTER 31 -#define TK_ANALYZE 32 -#define TK_ASC 33 -#define TK_ATTACH 34 -#define TK_BEFORE 35 -#define TK_BY 36 -#define TK_CASCADE 37 -#define TK_CAST 38 -#define TK_COLUMNKW 39 -#define TK_CONFLICT 40 -#define TK_DATABASE 41 -#define TK_DESC 42 -#define TK_DETACH 43 -#define TK_EACH 44 -#define TK_FAIL 45 -#define TK_FOR 46 -#define TK_IGNORE 47 -#define TK_INITIALLY 48 -#define TK_INSTEAD 49 -#define TK_LIKE_KW 50 -#define TK_MATCH 51 -#define TK_NO 52 -#define TK_KEY 53 -#define TK_OF 54 -#define TK_OFFSET 55 -#define TK_PRAGMA 56 -#define TK_RAISE 57 -#define TK_RECURSIVE 58 -#define TK_REPLACE 59 -#define TK_RESTRICT 60 -#define TK_ROW 61 -#define TK_TRIGGER 62 -#define TK_VACUUM 63 -#define TK_VIEW 64 -#define TK_VIRTUAL 65 -#define TK_WITH 66 -#define TK_REINDEX 67 -#define TK_RENAME 68 -#define TK_CTIME_KW 69 -#define TK_ANY 70 -#define TK_OR 71 -#define TK_AND 72 -#define TK_IS 73 -#define TK_BETWEEN 74 -#define TK_IN 75 -#define TK_ISNULL 76 -#define TK_NOTNULL 77 -#define TK_NE 78 -#define TK_EQ 79 -#define TK_GT 80 -#define TK_LE 81 -#define TK_LT 82 -#define TK_GE 83 -#define TK_ESCAPE 84 -#define TK_BITAND 85 -#define TK_BITOR 86 -#define TK_LSHIFT 87 -#define TK_RSHIFT 88 -#define TK_PLUS 89 -#define TK_MINUS 90 -#define TK_STAR 91 -#define TK_SLASH 92 -#define TK_REM 93 -#define TK_CONCAT 94 -#define TK_COLLATE 95 -#define TK_BITNOT 96 +#define TK_OR 27 +#define TK_AND 28 +#define TK_IS 29 +#define TK_MATCH 30 +#define TK_LIKE_KW 31 +#define TK_BETWEEN 32 +#define TK_IN 33 +#define TK_ISNULL 34 +#define TK_NOTNULL 35 +#define TK_NE 36 +#define TK_EQ 37 +#define TK_GT 38 +#define TK_LE 39 +#define TK_LT 40 +#define TK_GE 41 +#define TK_ESCAPE 42 +#define TK_BITAND 43 +#define TK_BITOR 44 +#define TK_LSHIFT 45 +#define TK_RSHIFT 46 +#define TK_PLUS 47 +#define TK_MINUS 48 +#define TK_STAR 49 +#define TK_SLASH 50 +#define TK_REM 51 +#define TK_CONCAT 52 +#define TK_COLLATE 53 +#define TK_BITNOT 54 +#define TK_ID 55 +#define TK_INDEXED 56 +#define TK_ABORT 57 +#define TK_ACTION 58 +#define TK_AFTER 59 +#define TK_ANALYZE 60 +#define TK_ASC 61 +#define TK_ATTACH 62 +#define TK_BEFORE 63 +#define TK_BY 64 +#define TK_CASCADE 65 +#define TK_CAST 66 +#define TK_COLUMNKW 67 +#define TK_CONFLICT 68 +#define TK_DATABASE 69 +#define TK_DESC 70 +#define TK_DETACH 71 +#define TK_EACH 72 +#define TK_FAIL 73 +#define TK_FOR 74 +#define TK_IGNORE 75 +#define TK_INITIALLY 76 +#define TK_INSTEAD 77 +#define TK_NO 78 +#define TK_KEY 79 +#define TK_OF 80 +#define TK_OFFSET 81 +#define TK_PRAGMA 82 +#define TK_RAISE 83 +#define TK_RECURSIVE 84 +#define TK_REPLACE 85 +#define TK_RESTRICT 86 +#define TK_ROW 87 +#define TK_TRIGGER 88 +#define TK_VACUUM 89 +#define TK_VIEW 90 +#define TK_VIRTUAL 91 +#define TK_WITH 92 +#define TK_REINDEX 93 +#define TK_RENAME 94 +#define TK_CTIME_KW 95 +#define TK_ANY 96 #define TK_STRING 97 #define TK_JOIN_KW 98 #define TK_CONSTRAINT 99 @@ -8327,9 +11423,9 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_LIMIT 129 #define TK_WHERE 130 #define TK_INTO 131 -#define TK_INTEGER 132 -#define TK_FLOAT 133 -#define TK_BLOB 134 +#define TK_FLOAT 132 +#define TK_BLOB 133 +#define TK_INTEGER 134 #define TK_VARIABLE 135 #define TK_CASE 136 #define TK_WHEN 137 @@ -8345,23 +11441,33 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_TO_REAL 147 #define TK_ISNOT 148 #define TK_END_OF_FILE 149 -#define TK_ILLEGAL 150 -#define TK_SPACE 151 -#define TK_UNCLOSED_STRING 152 -#define TK_FUNCTION 153 -#define TK_COLUMN 154 -#define TK_AGG_FUNCTION 155 -#define TK_AGG_COLUMN 156 -#define TK_UMINUS 157 -#define TK_UPLUS 158 -#define TK_REGISTER 159 +#define TK_UNCLOSED_STRING 150 +#define TK_FUNCTION 151 +#define TK_COLUMN 152 +#define TK_AGG_FUNCTION 153 +#define TK_AGG_COLUMN 154 +#define TK_UMINUS 155 +#define TK_UPLUS 156 +#define TK_REGISTER 157 +#define TK_VECTOR 158 +#define TK_SELECT_COLUMN 159 +#define TK_ASTERISK 160 +#define TK_SPAN 161 +#define TK_SPACE 162 +#define TK_ILLEGAL 163 + +/* The token codes above must all fit in 8 bits */ +#define TKFLG_MASK 0xff + +/* Flags that can be added to a token code when it is not +** being stored in a u8: */ +#define TKFLG_DONTFOLD 0x100 /* Omit constant folding optimizations */ /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ #include #include #include -#include #include /* @@ -8386,7 +11492,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* ** OMIT_TEMPDB is set to 1 if SQLITE_OMIT_TEMPDB is defined, or 0 -** afterward. Having this macro allows us to cause the C compiler +** afterward. Having this macro allows us to cause the C compiler ** to omit code used by TEMP tables without messy #ifndef statements. */ #ifdef SQLITE_OMIT_TEMPDB @@ -8423,6 +11529,36 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); # define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif +/* +** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if +** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it +** to zero. +*/ +#if SQLITE_TEMP_STORE==3 || SQLITE_THREADSAFE==0 +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 0 +#endif +#ifndef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 8 +#endif +#ifndef SQLITE_DEFAULT_WORKER_THREADS +# define SQLITE_DEFAULT_WORKER_THREADS 0 +#endif +#if SQLITE_DEFAULT_WORKER_THREADS>SQLITE_MAX_WORKER_THREADS +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS +#endif + +/* +** The default initial allocation for the pagecache when using separate +** pagecaches for each database connection. A positive number is the +** number of pages. A negative number N translations means that a buffer +** of -1024*N bytes is allocated and used for as many pages as it will hold. +*/ +#ifndef SQLITE_DEFAULT_PCACHE_INITSZ +# define SQLITE_DEFAULT_PCACHE_INITSZ 100 +#endif + /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. @@ -8434,8 +11570,17 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* ** Macros to compute minimum and maximum of two numbers. */ -#define MIN(A,B) ((A)<(B)?(A):(B)) -#define MAX(A,B) ((A)>(B)?(A):(B)) +#ifndef MIN +# define MIN(A,B) ((A)<(B)?(A):(B)) +#endif +#ifndef MAX +# define MAX(A,B) ((A)>(B)?(A):(B)) +#endif + +/* +** Swap two objects of type TYPE. +*/ +#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} /* ** Check to see if this machine uses EBCDIC. (Yes, believe it or @@ -8526,7 +11671,7 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantaties are suppose to be estimates, +** of 40. However, since LogEst quantities are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** ** "LogEst" is short for "Logarithmic Estimate". @@ -8538,13 +11683,48 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** 4 -> 20 1000 -> 99 1048576 -> 200 ** 10 -> 33 1024 -> 100 4294967296 -> 320 ** -** The LogEst can be negative to indicate fractional values. +** The LogEst can be negative to indicate fractional values. ** Examples: ** ** 0.5 -> -10 0.1 -> -33 0.0625 -> -40 */ typedef INT16_TYPE LogEst; +/* +** Set the SQLITE_PTRSIZE macro to the number of bytes in a pointer +*/ +#ifndef SQLITE_PTRSIZE +# if defined(__SIZEOF_POINTER__) +# define SQLITE_PTRSIZE __SIZEOF_POINTER__ +# elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(_M_ARM) || defined(__arm__) || defined(__x86) +# define SQLITE_PTRSIZE 4 +# else +# define SQLITE_PTRSIZE 8 +# endif +#endif + +/* The uptr type is an unsigned integer large enough to hold a pointer +*/ +#if defined(HAVE_STDINT_H) + typedef uintptr_t uptr; +#elif SQLITE_PTRSIZE==4 + typedef u32 uptr; +#else + typedef u64 uptr; +#endif + +/* +** The SQLITE_WITHIN(P,S,E) macro checks to see if pointer P points to +** something between S (inclusive) and E (exclusive). +** +** In other words, S is a buffer and E is a pointer to the first byte after +** the end of buffer S. This macro returns true if P points to something +** contained within the buffer S. +*/ +#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) + + /* ** Macros to determine whether the machine is big or little endian, ** and whether or not that determination is run-time or compile-time. @@ -8554,11 +11734,6 @@ typedef INT16_TYPE LogEst; ** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined ** at run-time. */ -#ifdef SQLITE_AMALGAMATION -SQLITE_PRIVATE const int sqlite3one = 1; -#else -SQLITE_PRIVATE const int sqlite3one; -#endif #if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ @@ -8576,6 +11751,11 @@ SQLITE_PRIVATE const int sqlite3one; # define SQLITE_UTF16NATIVE SQLITE_UTF16BE #endif #if !defined(SQLITE_BYTEORDER) +# ifdef SQLITE_AMALGAMATION + const int sqlite3one = 1; +# else + extern const int sqlite3one; +# endif # define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) @@ -8590,7 +11770,7 @@ SQLITE_PRIVATE const int sqlite3one; #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -/* +/* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. */ @@ -8607,7 +11787,7 @@ SQLITE_PRIVATE const int sqlite3one; ** all alignment restrictions correct. ** ** Except, if SQLITE_4_BYTE_ALIGNED_MALLOC is defined, then the -** underlying malloc() implemention might return us 4-byte aligned +** underlying malloc() implementation might return us 4-byte aligned ** pointers. In that case, only verify 4-byte alignment. */ #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC @@ -8629,16 +11809,14 @@ SQLITE_PRIVATE const int sqlite3one; */ #ifdef __APPLE__ # include -# if TARGET_OS_IPHONE -# undef SQLITE_MAX_MMAP_SIZE -# define SQLITE_MAX_MMAP_SIZE 0 -# endif #endif #ifndef SQLITE_MAX_MMAP_SIZE # if defined(__linux__) \ || defined(_WIN32) \ || (defined(__APPLE__) && defined(__MACH__)) \ - || defined(__sun) + || defined(__sun) \ + || defined(__FreeBSD__) \ + || defined(__DragonFly__) # define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ # else # define SQLITE_MAX_MMAP_SIZE 0 @@ -8674,9 +11852,19 @@ SQLITE_PRIVATE const int sqlite3one; # undef SQLITE_ENABLE_STAT3_OR_STAT4 #endif +/* +** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not +** the Select query generator tracing logic is turned on. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_SELECTTRACE) +# define SELECTTRACE_ENABLED 1 +#else +# define SELECTTRACE_ENABLED 0 +#endif + /* ** An instance of the following structure is used to store the busy-handler -** callback for a given sqlite handle. +** callback for a given sqlite handle. ** ** The sqlite.busyHandler member of the sqlite struct contains the busy ** callback for the database handle. Each pager opened via the sqlite @@ -8721,9 +11909,9 @@ struct BusyHandler { /* ** The following value as a destructor means to use sqlite3DbFree(). -** The sqlite3DbFree() routine requires two parameters instead of the -** one parameter that destructors normally want. So we have to introduce -** this magic value that the code knows to handle differently. Any +** The sqlite3DbFree() routine requires two parameters instead of the +** one parameter that destructors normally want. So we have to introduce +** this magic value that the code knows to handle differently. Any ** pointer will work here as long as it is distinct from SQLITE_STATIC ** and SQLITE_TRANSIENT. */ @@ -8747,19 +11935,19 @@ struct BusyHandler { #define SQLITE_WSD const #define GLOBAL(t,v) (*(t*)sqlite3_wsd_find((void*)&(v), sizeof(v))) #define sqlite3GlobalConfig GLOBAL(struct Sqlite3Config, sqlite3Config) -SQLITE_API int sqlite3_wsd_init(int N, int J); -SQLITE_API void *sqlite3_wsd_find(void *K, int L); +SQLITE_API int sqlite3_wsd_init(int N, int J); +SQLITE_API void *sqlite3_wsd_find(void *K, int L); #else - #define SQLITE_WSD + #define SQLITE_WSD #define GLOBAL(t,v) v #define sqlite3GlobalConfig sqlite3Config #endif /* ** The following macros are used to suppress compiler warnings and to -** make it clear to human readers when a function parameter is deliberately +** make it clear to human readers when a function parameter is deliberately ** left unused within the body of a function. This usually happens when -** a function is called via a function pointer. For example the +** a function is called via a function pointer. For example the ** implementation of an SQL aggregate step callback may not use the ** parameter indicating the number of arguments passed to the aggregate, ** if it knows that this is enforced elsewhere. @@ -8802,16 +11990,19 @@ typedef struct LookasideSlot LookasideSlot; typedef struct Module Module; typedef struct NameContext NameContext; typedef struct Parse Parse; +typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; +typedef struct SQLiteThread SQLiteThread; typedef struct SelectDest SelectDest; typedef struct SrcList SrcList; typedef struct StrAccum StrAccum; typedef struct Table Table; typedef struct TableLock TableLock; typedef struct Token Token; +typedef struct TreeView TreeView; typedef struct Trigger Trigger; typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; @@ -8823,7 +12014,7 @@ typedef struct WhereInfo WhereInfo; typedef struct With With; /* -** Defer sourcing vdbe.h and btree.h until after the "u8" and +** Defer sourcing vdbe.h and btree.h until after the "u8" and ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. */ @@ -8844,13 +12035,13 @@ typedef struct With With; ** subsystem. See comments in the source code for a detailed description ** of what each interface routine does. */ -#ifndef _BTREE_H_ -#define _BTREE_H_ +#ifndef SQLITE_BTREE_H +#define SQLITE_BTREE_H /* TODO: This definition is just included so other modules compile. It ** needs to be revisited. */ -#define SQLITE_N_BTREE_META 10 +#define SQLITE_N_BTREE_META 16 /* ** If defined as non-zero, auto-vacuum is enabled by default. Otherwise @@ -8870,6 +12061,7 @@ typedef struct With With; typedef struct Btree Btree; typedef struct BtCursor BtCursor; typedef struct BtShared BtShared; +typedef struct BtreePayload BtreePayload; SQLITE_PRIVATE int sqlite3BtreeOpen( @@ -8894,27 +12086,25 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree*,int); #if SQLITE_MAX_MMAP_SIZE>0 SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); #endif SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags(Btree*,unsigned); -SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree*); SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree*); -#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree*); SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); -#endif SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int); SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int); SQLITE_PRIVATE int sqlite3BtreeCommit(Btree*); -SQLITE_PRIVATE int sqlite3BtreeRollback(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeRollback(Btree*,int,int); SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree*,int); SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree*, int*, int flags); SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree*); @@ -8922,7 +12112,9 @@ SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree*); SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree*); SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *pBtree); +#ifndef SQLITE_OMIT_SHARED_CACHE SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock); +#endif SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *, int, int); SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *); @@ -8947,7 +12139,7 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, int*); SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor*); -SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree*, int); +SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree*, int, int); SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); @@ -8965,6 +12157,11 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); ** For example, the free-page-count field is located at byte offset 36 of ** the database file header. The incr-vacuum-flag field is located at ** byte offset 64 (== 36+4*7). +** +** The BTREE_DATA_VERSION value is not really a value stored in the header. +** It is a read-only number computed by the pager. But we merge it with +** the header value access routines since its access pattern is the same. +** Call it a "virtual meta value". */ #define BTREE_FREE_PAGE_COUNT 0 #define BTREE_SCHEMA_VERSION 1 @@ -8975,12 +12172,78 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 #define BTREE_APPLICATION_ID 8 +#define BTREE_DATA_VERSION 15 /* A virtual meta-value */ /* -** Values that may be OR'd together to form the second argument of an -** sqlite3BtreeCursorHints() call. +** Kinds of hints that can be passed into the sqlite3BtreeCursorHint() +** interface. +** +** BTREE_HINT_RANGE (arguments: Expr*, Mem*) +** +** The first argument is an Expr* (which is guaranteed to be constant for +** the lifetime of the cursor) that defines constraints on which rows +** might be fetched with this cursor. The Expr* tree may contain +** TK_REGISTER nodes that refer to values stored in the array of registers +** passed as the second parameter. In other words, if Expr.op==TK_REGISTER +** then the value of the node is the value in Mem[pExpr.iTable]. Any +** TK_COLUMN node in the expression tree refers to the Expr.iColumn-th +** column of the b-tree of the cursor. The Expr tree will not contain +** any function calls nor subqueries nor references to b-trees other than +** the cursor being hinted. +** +** The design of the _RANGE hint is aid b-tree implementations that try +** to prefetch content from remote machines - to provide those +** implementations with limits on what needs to be prefetched and thereby +** reduce network bandwidth. +** +** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by +** standard SQLite. The other hints are provided for extentions that use +** the SQLite parser and code generator but substitute their own storage +** engine. */ -#define BTREE_BULKLOAD 0x00000001 +#define BTREE_HINT_RANGE 0 /* Range constraints on queries */ + +/* +** Values that may be OR'd together to form the argument to the +** BTREE_HINT_FLAGS hint for sqlite3BtreeCursorHint(): +** +** The BTREE_BULKLOAD flag is set on index cursors when the index is going +** to be filled with content that is already in sorted order. +** +** The BTREE_SEEK_EQ flag is set on cursors that will get OP_SeekGE or +** OP_SeekLE opcodes for a range search, but where the range of entries +** selected will all have the same key. In other words, the cursor will +** be used only for equality key searches. +** +*/ +#define BTREE_BULKLOAD 0x00000001 /* Used to full index in sorted order */ +#define BTREE_SEEK_EQ 0x00000002 /* EQ seeks only - no range seeks */ + +/* +** Flags passed as the third argument to sqlite3BtreeCursor(). +** +** For read-only cursors the wrFlag argument is always zero. For read-write +** cursors it may be set to either (BTREE_WRCSR|BTREE_FORDELETE) or just +** (BTREE_WRCSR). If the BTREE_FORDELETE bit is set, then the cursor will +** only be used by SQLite for the following: +** +** * to seek to and then delete specific entries, and/or +** +** * to read values that will be used to create keys that other +** BTREE_FORDELETE cursors will seek to and delete. +** +** The BTREE_FORDELETE flag is an optimization hint. It is not used by +** by this, the native b-tree engine of SQLite, but it is available to +** alternative storage engines that might be substituted in place of this +** b-tree system. For alternative storage engines in which a delete of +** the main table row automatically deletes corresponding index rows, +** the FORDELETE flag hint allows those alternative storage engines to +** skip a lot of work. Namely: FORDELETE cursors may treat all SEEK +** and DELETE operations as no-ops, and any READ operation against a +** FORDELETE cursor may return a null row: 0x01 0x00. +*/ +#define BTREE_WRCSR 0x00000004 /* read-write cursor */ +#define BTREE_FORDELETE 0x00000008 /* Cursor is for seek/delete only */ SQLITE_PRIVATE int sqlite3BtreeCursor( Btree*, /* BTree containing table to open */ @@ -8991,6 +12254,10 @@ SQLITE_PRIVATE int sqlite3BtreeCursor( ); SQLITE_PRIVATE int sqlite3BtreeCursorSize(void); SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor*, int, ...); +#endif SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( @@ -9000,32 +12267,65 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( int bias, int *pRes ); -SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*, int*); -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*); -SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, - const void *pData, int nData, - int nZero, int bias, int seekResult); +SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor*, int*); +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*, u8 flags); + +/* Allowed flags for the 2nd argument to sqlite3BtreeDelete() */ +#define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ +#define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ + +/* An instance of the BtreePayload object describes the content of a single +** entry in either an index or table btree. +** +** Index btrees (used for indexes and also WITHOUT ROWID tables) contain +** an arbitrary key and no data. These btrees have pKey,nKey set to their +** key and pData,nData,nZero set to zero. +** +** Table btrees (used for rowid tables) contain an integer rowid used as +** the key and passed in the nKey field. The pKey field is zero. +** pData,nData hold the content of the new entry. nZero extra zero bytes +** are appended to the end of the content when constructing the entry. +** +** This object is used to pass information into sqlite3BtreeInsert(). The +** same information used to be passed as five separate parameters. But placing +** the information into this object helps to keep the interface more +** organized and understandable, and it also helps the resulting code to +** run a little faster by using fewer registers for parameter passing. +*/ +struct BtreePayload { + const void *pKey; /* Key content for indexes. NULL for tables */ + sqlite3_int64 nKey; /* Size of pKey for indexes. PRIMARY KEY for tabs */ + const void *pData; /* Data for tables. NULL for indexes */ + int nData; /* Size of pData. 0 if none. */ + int nZero; /* Extra zero data appended after pData,nData */ +}; + +SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, + int bias, int seekResult); SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int *pRes); -SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor*, i64 *pSize); +SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*); -SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt); -SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt); -SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor*, u32 *pSize); +SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); +SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*); SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); +#ifndef SQLITE_OMIT_INCRBLOB SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); +#endif SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); -SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); +SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); +SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void); #ifndef NDEBUG SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*); @@ -9052,15 +12352,19 @@ SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); #ifndef SQLITE_OMIT_SHARED_CACHE SQLITE_PRIVATE void sqlite3BtreeEnter(Btree*); SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3*); +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree*); #else # define sqlite3BtreeEnter(X) # define sqlite3BtreeEnterAll(X) +# define sqlite3BtreeSharable(X) 0 +# define sqlite3BtreeEnterCursor(X) +# define sqlite3BtreeConnectionCount(X) 1 #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE -SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); SQLITE_PRIVATE void sqlite3BtreeLeave(Btree*); -SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3*); #ifndef NDEBUG @@ -9071,9 +12375,7 @@ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); #endif #else -# define sqlite3BtreeSharable(X) 0 # define sqlite3BtreeLeave(X) -# define sqlite3BtreeEnterCursor(X) # define sqlite3BtreeLeaveCursor(X) # define sqlite3BtreeLeaveAll(X) @@ -9083,7 +12385,7 @@ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); #endif -#endif /* _BTREE_H_ */ +#endif /* SQLITE_BTREE_H */ /************** End of btree.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -9106,8 +12408,8 @@ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); ** or VDBE. The VDBE implements an abstract machine that runs a ** simple program to access and modify the underlying database. */ -#ifndef _SQLITE_VDBE_H_ -#define _SQLITE_VDBE_H_ +#ifndef SQLITE_VDBE_H +#define SQLITE_VDBE_H /* #include */ /* @@ -9132,24 +12434,29 @@ typedef struct SubProgram SubProgram; struct VdbeOp { u8 opcode; /* What operation to perform */ signed char p4type; /* One of the P4_xxx constants for p4 */ - u8 opflags; /* Mask of the OPFLG_* flags in opcodes.h */ + u8 notUsed1; u8 p5; /* Fifth parameter is an unsigned character */ int p1; /* First operand */ int p2; /* Second parameter (often the jump destination) */ int p3; /* The third parameter */ - union { /* fourth parameter */ + union p4union { /* fourth parameter */ int i; /* Integer value if p4type==P4_INT32 */ void *p; /* Generic pointer */ char *z; /* Pointer to data for string (char array) types */ i64 *pI64; /* Used when p4type is P4_INT64 */ double *pReal; /* Used when p4type is P4_REAL */ FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ + sqlite3_context *pCtx; /* Used when p4type is P4_FUNCCTX */ CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ Mem *pMem; /* Used when p4type is P4_MEM */ VTable *pVtab; /* Used when p4type is P4_VTAB */ KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + Table *pTab; /* Used when p4type is P4_TABLE */ +#ifdef SQLITE_ENABLE_CURSOR_HINTS + Expr *pExpr; /* Used when p4type is P4_EXPR */ +#endif int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -9174,7 +12481,6 @@ struct SubProgram { int nOp; /* Elements in aOp[] */ int nMem; /* Number of memory cells required */ int nCsr; /* Number of cursors required */ - int nOnce; /* Number of OP_Once instructions */ void *token; /* id that may be used to recursive triggers */ SubProgram *pNext; /* Next sub-program already visited */ }; @@ -9200,6 +12506,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_COLLSEQ (-4) /* P4 is a pointer to a CollSeq structure */ #define P4_FUNCDEF (-5) /* P4 is a pointer to a FuncDef structure */ #define P4_KEYINFO (-6) /* P4 is a pointer to a KeyInfo structure */ +#define P4_EXPR (-7) /* P4 is a pointer to an Expr tree */ #define P4_MEM (-8) /* P4 is a pointer to a Mem* structure */ #define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ #define P4_VTAB (-10) /* P4 is a pointer to an sqlite3_vtab structure */ @@ -9210,6 +12517,8 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ +#define P4_TABLE (-20) /* P4 is a pointer to a Table structure */ +#define P4_FUNCCTX (-21) /* P4 is a pointer to an sqlite3_context object */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -9251,199 +12560,211 @@ typedef struct VdbeOpList VdbeOpList; /************** Include opcodes.h in the middle of vdbe.h ********************/ /************** Begin file opcodes.h *****************************************/ /* Automatically generated. Do not edit */ -/* See the mkopcodeh.awk script for details */ -#define OP_Function 1 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_Savepoint 2 -#define OP_AutoCommit 3 -#define OP_Transaction 4 -#define OP_SorterNext 5 -#define OP_PrevIfOpen 6 -#define OP_NextIfOpen 7 -#define OP_Prev 8 -#define OP_Next 9 -#define OP_AggStep 10 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_Checkpoint 11 -#define OP_JournalMode 12 -#define OP_Vacuum 13 -#define OP_VFilter 14 /* synopsis: iplan=r[P3] zplan='P4' */ -#define OP_VUpdate 15 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 16 -#define OP_Gosub 17 -#define OP_Return 18 +/* See the tool/mkopcodeh.tcl script for details */ +#define OP_Savepoint 0 +#define OP_AutoCommit 1 +#define OP_Transaction 2 +#define OP_SorterNext 3 +#define OP_PrevIfOpen 4 +#define OP_NextIfOpen 5 +#define OP_Prev 6 +#define OP_Next 7 +#define OP_Checkpoint 8 +#define OP_JournalMode 9 +#define OP_Vacuum 10 +#define OP_VFilter 11 /* synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 12 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 13 +#define OP_Gosub 14 +#define OP_InitCoroutine 15 +#define OP_Yield 16 +#define OP_MustBeInt 17 +#define OP_Jump 18 #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_InitCoroutine 20 -#define OP_EndCoroutine 21 -#define OP_Yield 22 -#define OP_HaltIfNull 23 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 24 -#define OP_Integer 25 /* synopsis: r[P2]=P1 */ -#define OP_Int64 26 /* synopsis: r[P2]=P4 */ -#define OP_String 27 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 28 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 29 /* synopsis: r[P1]=NULL */ -#define OP_Blob 30 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 31 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 32 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 33 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 34 /* synopsis: r[P2]=r[P1] */ -#define OP_ResultRow 35 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 36 -#define OP_AddImm 37 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_MustBeInt 38 -#define OP_RealAffinity 39 -#define OP_Permutation 40 -#define OP_Compare 41 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_Jump 42 -#define OP_Once 43 -#define OP_If 44 -#define OP_IfNot 45 -#define OP_Column 46 /* synopsis: r[P3]=PX */ -#define OP_Affinity 47 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 48 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 49 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 50 -#define OP_SetCookie 51 -#define OP_ReopenIdx 52 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenRead 53 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 54 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenAutoindex 55 /* synopsis: nColumn=P2 */ -#define OP_OpenEphemeral 56 /* synopsis: nColumn=P2 */ -#define OP_SorterOpen 57 -#define OP_OpenPseudo 58 /* synopsis: P3 columns in r[P2] */ -#define OP_Close 59 -#define OP_SeekLT 60 /* synopsis: key=r[P3@P4] */ -#define OP_SeekLE 61 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGE 62 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGT 63 /* synopsis: key=r[P3@P4] */ -#define OP_Seek 64 /* synopsis: intkey=r[P2] */ -#define OP_NoConflict 65 /* synopsis: key=r[P3@P4] */ -#define OP_NotFound 66 /* synopsis: key=r[P3@P4] */ -#define OP_Found 67 /* synopsis: key=r[P3@P4] */ -#define OP_NotExists 68 /* synopsis: intkey=r[P3] */ -#define OP_Sequence 69 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 70 /* synopsis: r[P2]=rowid */ -#define OP_Or 71 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ -#define OP_And 72 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_Insert 73 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_InsertInt 74 /* synopsis: intkey=P3 data=r[P2] */ -#define OP_Delete 75 -#define OP_IsNull 76 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ -#define OP_NotNull 77 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ -#define OP_Ne 78 /* same as TK_NE, synopsis: if r[P1]!=r[P3] goto P2 */ -#define OP_Eq 79 /* same as TK_EQ, synopsis: if r[P1]==r[P3] goto P2 */ -#define OP_Gt 80 /* same as TK_GT, synopsis: if r[P1]>r[P3] goto P2 */ -#define OP_Le 81 /* same as TK_LE, synopsis: if r[P1]<=r[P3] goto P2 */ -#define OP_Lt 82 /* same as TK_LT, synopsis: if r[P1]=r[P3] goto P2 */ -#define OP_ResetCount 84 -#define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ -#define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ -#define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ -#define OP_Add 89 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ -#define OP_Subtract 90 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ -#define OP_Multiply 91 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ -#define OP_Divide 92 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ -#define OP_Remainder 93 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ -#define OP_Concat 94 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_SorterCompare 95 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_BitNot 96 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ +#define OP_Once 20 +#define OP_If 21 +#define OP_IfNot 22 +#define OP_SeekLT 23 /* synopsis: key=r[P3@P4] */ +#define OP_SeekLE 24 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGE 25 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGT 26 /* synopsis: key=r[P3@P4] */ +#define OP_Or 27 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ +#define OP_And 28 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ +#define OP_NoConflict 29 /* synopsis: key=r[P3@P4] */ +#define OP_NotFound 30 /* synopsis: key=r[P3@P4] */ +#define OP_Found 31 /* synopsis: key=r[P3@P4] */ +#define OP_SeekRowid 32 /* synopsis: intkey=r[P3] */ +#define OP_NotExists 33 /* synopsis: intkey=r[P3] */ +#define OP_IsNull 34 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ +#define OP_NotNull 35 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ +#define OP_Ne 36 /* same as TK_NE, synopsis: IF r[P3]!=r[P1] */ +#define OP_Eq 37 /* same as TK_EQ, synopsis: IF r[P3]==r[P1] */ +#define OP_Gt 38 /* same as TK_GT, synopsis: IF r[P3]>r[P1] */ +#define OP_Le 39 /* same as TK_LE, synopsis: IF r[P3]<=r[P1] */ +#define OP_Lt 40 /* same as TK_LT, synopsis: IF r[P3]=r[P1] */ +#define OP_ElseNotEq 42 /* same as TK_ESCAPE */ +#define OP_BitAnd 43 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 44 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 45 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 47 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 48 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 49 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 50 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 51 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 52 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_Last 53 +#define OP_BitNot 54 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ +#define OP_SorterSort 55 +#define OP_Sort 56 +#define OP_Rewind 57 +#define OP_IdxLE 58 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGT 59 /* synopsis: key=r[P3@P4] */ +#define OP_IdxLT 60 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGE 61 /* synopsis: key=r[P3@P4] */ +#define OP_RowSetRead 62 /* synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 63 /* synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 64 +#define OP_FkIfZero 65 /* synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IfPos 66 /* synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IfNotZero 67 /* synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 */ +#define OP_DecrJumpZero 68 /* synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 69 +#define OP_VNext 70 +#define OP_Init 71 /* synopsis: Start at P2 */ +#define OP_Return 72 +#define OP_EndCoroutine 73 +#define OP_HaltIfNull 74 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 75 +#define OP_Integer 76 /* synopsis: r[P2]=P1 */ +#define OP_Int64 77 /* synopsis: r[P2]=P4 */ +#define OP_String 78 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 79 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 80 /* synopsis: r[P1]=NULL */ +#define OP_Blob 81 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 82 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 83 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 84 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 85 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 86 /* synopsis: r[P2]=r[P1] */ +#define OP_ResultRow 87 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 88 +#define OP_Function0 89 /* synopsis: r[P3]=func(r[P2@P5]) */ +#define OP_Function 90 /* synopsis: r[P3]=func(r[P2@P5]) */ +#define OP_AddImm 91 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 92 +#define OP_Cast 93 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 94 +#define OP_Compare 95 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_Column 96 /* synopsis: r[P3]=PX */ #define OP_String8 97 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_SorterData 98 /* synopsis: r[P2]=data */ -#define OP_RowKey 99 /* synopsis: r[P2]=key */ -#define OP_RowData 100 /* synopsis: r[P2]=data */ -#define OP_Rowid 101 /* synopsis: r[P2]=rowid */ -#define OP_NullRow 102 -#define OP_Last 103 -#define OP_SorterSort 104 -#define OP_Sort 105 -#define OP_Rewind 106 -#define OP_SorterInsert 107 -#define OP_IdxInsert 108 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 109 /* synopsis: key=r[P2@P3] */ -#define OP_IdxRowid 110 /* synopsis: r[P2]=rowid */ -#define OP_IdxLE 111 /* synopsis: key=r[P3@P4] */ -#define OP_IdxGT 112 /* synopsis: key=r[P3@P4] */ -#define OP_IdxLT 113 /* synopsis: key=r[P3@P4] */ -#define OP_IdxGE 114 /* synopsis: key=r[P3@P4] */ -#define OP_Destroy 115 -#define OP_Clear 116 -#define OP_ResetSorter 117 -#define OP_CreateIndex 118 /* synopsis: r[P2]=root iDb=P1 */ -#define OP_CreateTable 119 /* synopsis: r[P2]=root iDb=P1 */ -#define OP_ParseSchema 120 -#define OP_LoadAnalysis 121 -#define OP_DropTable 122 -#define OP_DropIndex 123 -#define OP_DropTrigger 124 -#define OP_IntegrityCk 125 -#define OP_RowSetAdd 126 /* synopsis: rowset(P1)=r[P2] */ -#define OP_RowSetRead 127 /* synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 128 /* synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 129 -#define OP_Param 130 -#define OP_FkCounter 131 /* synopsis: fkctr[P1]+=P2 */ -#define OP_FkIfZero 132 /* synopsis: if fkctr[P1]==0 goto P2 */ -#define OP_Real 133 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_MemMax 134 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_IfPos 135 /* synopsis: if r[P1]>0 goto P2 */ -#define OP_IfNeg 136 /* synopsis: r[P1]+=P3, if r[P1]<0 goto P2 */ -#define OP_IfZero 137 /* synopsis: r[P1]+=P3, if r[P1]==0 goto P2 */ -#define OP_AggFinal 138 /* synopsis: accum=r[P1] N=P2 */ -#define OP_IncrVacuum 139 -#define OP_Expire 140 -#define OP_TableLock 141 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 142 -#define OP_ToText 143 /* same as TK_TO_TEXT */ -#define OP_ToBlob 144 /* same as TK_TO_BLOB */ -#define OP_ToNumeric 145 /* same as TK_TO_NUMERIC */ -#define OP_ToInt 146 /* same as TK_TO_INT */ -#define OP_ToReal 147 /* same as TK_TO_REAL */ -#define OP_VCreate 148 -#define OP_VDestroy 149 -#define OP_VOpen 150 -#define OP_VColumn 151 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VNext 152 -#define OP_VRename 153 -#define OP_Pagecount 154 -#define OP_MaxPgcnt 155 -#define OP_Init 156 /* synopsis: Start at P2 */ -#define OP_Noop 157 -#define OP_Explain 158 - +#define OP_Affinity 98 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 99 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 100 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 101 +#define OP_SetCookie 102 +#define OP_ReopenIdx 103 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 104 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 105 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenAutoindex 106 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 107 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 108 +#define OP_SequenceTest 109 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 110 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 111 +#define OP_ColumnsUsed 112 +#define OP_Sequence 113 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 114 /* synopsis: r[P2]=rowid */ +#define OP_Insert 115 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_InsertInt 116 /* synopsis: intkey=P3 data=r[P2] */ +#define OP_Delete 117 +#define OP_ResetCount 118 +#define OP_SorterCompare 119 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 120 /* synopsis: r[P2]=data */ +#define OP_RowKey 121 /* synopsis: r[P2]=key */ +#define OP_RowData 122 /* synopsis: r[P2]=data */ +#define OP_Rowid 123 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 124 +#define OP_SorterInsert 125 +#define OP_IdxInsert 126 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 127 /* synopsis: key=r[P2@P3] */ +#define OP_Seek 128 /* synopsis: Move P3 to P1.rowid */ +#define OP_IdxRowid 129 /* synopsis: r[P2]=rowid */ +#define OP_Destroy 130 +#define OP_Clear 131 +#define OP_Real 132 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_ResetSorter 133 +#define OP_CreateIndex 134 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_CreateTable 135 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_ParseSchema 136 +#define OP_LoadAnalysis 137 +#define OP_DropTable 138 +#define OP_DropIndex 139 +#define OP_DropTrigger 140 +#define OP_IntegrityCk 141 +#define OP_RowSetAdd 142 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 143 +#define OP_FkCounter 144 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 145 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 146 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggStep0 147 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep 148 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggFinal 149 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 150 +#define OP_TableLock 151 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 152 +#define OP_VCreate 153 +#define OP_VDestroy 154 +#define OP_VOpen 155 +#define OP_VColumn 156 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 157 +#define OP_Pagecount 158 +#define OP_MaxPgcnt 159 +#define OP_CursorHint 160 +#define OP_Noop 161 +#define OP_Explain 162 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c ** are encoded into bitvectors as follows: */ -#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */ -#define OPFLG_OUT2_PRERELEASE 0x0002 /* out2-prerelease: */ -#define OPFLG_IN1 0x0004 /* in1: P1 is an input */ -#define OPFLG_IN2 0x0008 /* in2: P2 is an input */ -#define OPFLG_IN3 0x0010 /* in3: P3 is an input */ -#define OPFLG_OUT2 0x0020 /* out2: P2 is an output */ -#define OPFLG_OUT3 0x0040 /* out3: P3 is an output */ +#define OPFLG_JUMP 0x01 /* jump: P2 holds jmp target */ +#define OPFLG_IN1 0x02 /* in1: P1 is an input */ +#define OPFLG_IN2 0x04 /* in2: P2 is an input */ +#define OPFLG_IN3 0x08 /* in3: P3 is an input */ +#define OPFLG_OUT2 0x10 /* out2: P2 is an output */ +#define OPFLG_OUT3 0x20 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,\ -/* 8 */ 0x01, 0x01, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00,\ -/* 16 */ 0x01, 0x01, 0x04, 0x24, 0x01, 0x04, 0x05, 0x10,\ -/* 24 */ 0x00, 0x02, 0x02, 0x02, 0x02, 0x00, 0x02, 0x02,\ -/* 32 */ 0x00, 0x00, 0x20, 0x00, 0x00, 0x04, 0x05, 0x04,\ -/* 40 */ 0x00, 0x00, 0x01, 0x01, 0x05, 0x05, 0x00, 0x00,\ -/* 48 */ 0x00, 0x02, 0x02, 0x10, 0x00, 0x00, 0x00, 0x00,\ -/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11,\ -/* 64 */ 0x08, 0x11, 0x11, 0x11, 0x11, 0x02, 0x02, 0x4c,\ -/* 72 */ 0x4c, 0x00, 0x00, 0x00, 0x05, 0x05, 0x15, 0x15,\ -/* 80 */ 0x15, 0x15, 0x15, 0x15, 0x00, 0x4c, 0x4c, 0x4c,\ -/* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x00,\ -/* 96 */ 0x24, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01,\ -/* 104 */ 0x01, 0x01, 0x01, 0x08, 0x08, 0x00, 0x02, 0x01,\ -/* 112 */ 0x01, 0x01, 0x01, 0x02, 0x00, 0x00, 0x02, 0x02,\ -/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x45,\ -/* 128 */ 0x15, 0x01, 0x02, 0x00, 0x01, 0x02, 0x08, 0x05,\ -/* 136 */ 0x05, 0x05, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04,\ -/* 144 */ 0x04, 0x04, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00,\ -/* 152 */ 0x01, 0x00, 0x02, 0x02, 0x01, 0x00, 0x00,} +/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01,\ +/* 8 */ 0x00, 0x10, 0x00, 0x01, 0x00, 0x01, 0x01, 0x01,\ +/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x03, 0x03, 0x09,\ +/* 24 */ 0x09, 0x09, 0x09, 0x26, 0x26, 0x09, 0x09, 0x09,\ +/* 32 */ 0x09, 0x09, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ +/* 40 */ 0x0b, 0x0b, 0x01, 0x26, 0x26, 0x26, 0x26, 0x26,\ +/* 48 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x01, 0x12, 0x01,\ +/* 56 */ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x23, 0x0b,\ +/* 64 */ 0x01, 0x01, 0x03, 0x03, 0x03, 0x01, 0x01, 0x01,\ +/* 72 */ 0x02, 0x02, 0x08, 0x00, 0x10, 0x10, 0x10, 0x10,\ +/* 80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x10, 0x10, 0x00,\ +/* 88 */ 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x00, 0x00,\ +/* 96 */ 0x00, 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ +/* 104 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 112 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 120 */ 0x00, 0x00, 0x00, 0x10, 0x00, 0x04, 0x04, 0x00,\ +/* 128 */ 0x00, 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10,\ +/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x10,\ +/* 144 */ 0x00, 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 152 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x10,\ +/* 160 */ 0x00, 0x00, 0x00,} + +/* The sqlite3P2Values() routine is able to run faster if it knows +** the value of the largest JUMP opcode. The smaller the maximum +** JUMP opcode the better, so the mkopcodeh.tcl script that +** generated this include file strives to group all JUMP opcodes +** together near the beginning of the list. +*/ +#define SQLITE_MX_JUMP_OPCODE 71 /* Maximum JUMP opcode */ /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -9456,17 +12777,28 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe*,int,const char*); +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); -SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); +SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe*,int); +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); +#else +# define sqlite3VdbeVerifyNoMallocRequired(A,B) +#endif +SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5); SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); -SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe*, int addr); +SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); @@ -9474,6 +12806,7 @@ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); @@ -9501,10 +12834,11 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); +SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); -typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); #ifndef SQLITE_OMIT_TRIGGER @@ -9571,8 +12905,14 @@ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); # define VDBE_OFFSET_LINENO(x) 0 #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); +#else +# define sqlite3VdbeScanStatus(a,b,c,d,e) #endif +#endif /* SQLITE_VDBE_H */ + /************** End of vdbe.h ************************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ /************** Include pager.h in the middle of sqliteInt.h *****************/ @@ -9593,8 +12933,8 @@ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); ** at a time and provides a journal for rollback. */ -#ifndef _PAGER_H_ -#define _PAGER_H_ +#ifndef SQLITE_PAGER_H +#define SQLITE_PAGER_H /* ** Default maximum size for persistent journal files. A negative @@ -9647,7 +12987,11 @@ typedef struct PgHdr DbPage; #define PAGER_LOCKINGMODE_EXCLUSIVE 1 /* -** Numeric constants that encode the journalmode. +** Numeric constants that encode the journalmode. +** +** The numeric values encoded here (other than PAGER_JOURNALMODE_QUERY) +** are exposed in the API via the "PRAGMA journal_mode" command and +** therefore cannot be changed without a compatibility break. */ #define PAGER_JOURNALMODE_QUERY (-1) /* Query the value of journalmode */ #define PAGER_JOURNALMODE_DELETE 0 /* Commit by deleting journal file */ @@ -9658,22 +13002,28 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ /* -** Flags that make up the mask passed to sqlite3PagerAcquire(). +** Flags that make up the mask passed to sqlite3PagerGet(). */ #define PAGER_GET_NOCONTENT 0x01 /* Do not load data from disk */ #define PAGER_GET_READONLY 0x02 /* Read-only page is acceptable */ /* ** Flags for sqlite3PagerSetFlags() +** +** Value constraints (enforced via assert()): +** PAGER_FULLFSYNC == SQLITE_FullFSync +** PAGER_CKPT_FULLFSYNC == SQLITE_CkptFullFSync +** PAGER_CACHE_SPILL == SQLITE_CacheSpill */ #define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ #define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ #define PAGER_SYNCHRONOUS_FULL 0x03 /* PRAGMA synchronous=FULL */ -#define PAGER_SYNCHRONOUS_MASK 0x03 /* Mask for three values above */ -#define PAGER_FULLFSYNC 0x04 /* PRAGMA fullfsync=ON */ -#define PAGER_CKPT_FULLFSYNC 0x08 /* PRAGMA checkpoint_fullfsync=ON */ -#define PAGER_CACHESPILL 0x10 /* PRAGMA cache_spill=ON */ -#define PAGER_FLAGS_MASK 0x1c /* All above except SYNCHRONOUS */ +#define PAGER_SYNCHRONOUS_EXTRA 0x04 /* PRAGMA synchronous=EXTRA */ +#define PAGER_SYNCHRONOUS_MASK 0x07 /* Mask for four values above */ +#define PAGER_FULLFSYNC 0x08 /* PRAGMA fullfsync=ON */ +#define PAGER_CKPT_FULLFSYNC 0x10 /* PRAGMA checkpoint_fullfsync=ON */ +#define PAGER_CACHESPILL 0x20 /* PRAGMA cache_spill=ON */ +#define PAGER_FLAGS_MASK 0x38 /* All above except SYNCHRONOUS */ /* ** The remainder of this file contains the declarations of the functions @@ -9697,8 +13047,12 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); +#ifdef SQLITE_HAS_CODEC +SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*,Pager*); +#endif SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); +SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); SQLITE_PRIVATE void sqlite3PagerShrink(Pager*); SQLITE_PRIVATE void sqlite3PagerSetFlags(Pager*,unsigned); @@ -9708,10 +13062,10 @@ SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*); SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager*); SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *, i64); SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager*); +SQLITE_PRIVATE int sqlite3PagerFlush(Pager*); /* Functions used to obtain and release page references. */ -SQLITE_PRIVATE int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); -#define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0) +SQLITE_PRIVATE int sqlite3PagerGet(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); SQLITE_PRIVATE void sqlite3PagerRef(DbPage*); SQLITE_PRIVATE void sqlite3PagerUnref(DbPage*); @@ -9743,6 +13097,13 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerUseWal(Pager *pPager); +# ifdef SQLITE_ENABLE_SNAPSHOT +SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); +# endif +#else +# define sqlite3PagerUseWal(x) 0 #endif #ifdef SQLITE_ENABLE_ZIPVFS @@ -9751,22 +13112,27 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager); /* Functions used to query pager state and configuration. */ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager*); -SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); +SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); +#endif SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); -SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager*); +SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager*); SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); +SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager*); SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); -SQLITE_PRIVATE int sqlite3PagerNosync(Pager*); SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); -SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *); +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager*); SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); +SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16); + #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) SQLITE_PRIVATE void *sqlite3PagerCodec(DbPage *); #endif @@ -9786,7 +13152,7 @@ SQLITE_PRIVATE void sqlite3PagerRefdump(Pager*); # define enable_simulated_io_errors() #endif -#endif /* _PAGER_H_ */ +#endif /* SQLITE_PAGER_H */ /************** End of pager.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -9820,7 +13186,7 @@ struct PgHdr { sqlite3_pcache_page *pPage; /* Pcache object page handle */ void *pData; /* Page data */ void *pExtra; /* Extra content */ - PgHdr *pDirty; /* Transient list of dirty pages */ + PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ Pager *pPager; /* The pager this page is part of */ Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES @@ -9840,14 +13206,15 @@ struct PgHdr { }; /* Bit values for PgHdr.flags */ -#define PGHDR_DIRTY 0x002 /* Page has changed */ -#define PGHDR_NEED_SYNC 0x004 /* Fsync the rollback journal before - ** writing this page to the database */ -#define PGHDR_NEED_READ 0x008 /* Content is unread */ -#define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ -#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_CLEAN 0x001 /* Page not on the PCache.pDirty list */ +#define PGHDR_DIRTY 0x002 /* Page is on the PCache.pDirty list */ +#define PGHDR_WRITEABLE 0x004 /* Journaled and ready to modify */ +#define PGHDR_NEED_SYNC 0x008 /* Fsync the rollback journal before + ** writing this page to the database */ +#define PGHDR_DONT_WRITE 0x010 /* Do not write content to disk */ +#define PGHDR_MMAP 0x020 /* This is an mmap page object */ -#define PGHDR_MMAP 0x040 /* This is an mmap page object */ +#define PGHDR_WAL_APPEND 0x040 /* Appended to wal file */ /* Initialize and shutdown the page cache subsystem */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void); @@ -9862,7 +13229,7 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *, int sz, int n); ** Under memory stress, invoke xStress to try to make pages clean. ** Only clean and unpinned pages can be reclaimed. */ -SQLITE_PRIVATE void sqlite3PcacheOpen( +SQLITE_PRIVATE int sqlite3PcacheOpen( int szPage, /* Size of every page */ int szExtra, /* Extra space associated with each page */ int bPurgeable, /* True if pages are on backing store */ @@ -9872,7 +13239,7 @@ SQLITE_PRIVATE void sqlite3PcacheOpen( ); /* Modify the page-size after the cache has been created. */ -SQLITE_PRIVATE void sqlite3PcacheSetPageSize(PCache *, int); +SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *, int); /* Return the size in bytes of a PCache object. Used to preallocate ** storage space. @@ -9882,13 +13249,16 @@ SQLITE_PRIVATE int sqlite3PcacheSize(void); /* One release per successful fetch. Page is pinned until released. ** Reference counted. */ -SQLITE_PRIVATE int sqlite3PcacheFetch(PCache*, Pgno, int createFlag, PgHdr**); +SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch(PCache*, Pgno, int createFlag); +SQLITE_PRIVATE int sqlite3PcacheFetchStress(PCache*, Pgno, sqlite3_pcache_page**); +SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish(PCache*, Pgno, sqlite3_pcache_page *pPage); SQLITE_PRIVATE void sqlite3PcacheRelease(PgHdr*); SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ +SQLITE_PRIVATE void sqlite3PcacheClearWritable(PCache*); /* Change a page number. Used by incr-vacuum. */ SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr*, Pgno); @@ -9927,6 +13297,11 @@ SQLITE_PRIVATE int sqlite3PcachePagecount(PCache*); SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); #endif +#if defined(SQLITE_DEBUG) +/* Check invariants on a PgHdr object */ +SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr*); +#endif + /* Set and get the suggested cache-size for the specified pager-cache. ** ** If no global maximum is configured, then the system attempts to limit @@ -9938,6 +13313,13 @@ SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *, int); SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *); #endif +/* Set or get the suggested spill-size for the specified pager-cache. +** +** The spill-size is the minimum number of pages in cache before the cache +** will attempt to spill dirty pages by calling xStress. +*/ +SQLITE_PRIVATE int sqlite3PcacheSetSpillsize(PCache *, int); + /* Free up as much memory as possible from the page cache */ SQLITE_PRIVATE void sqlite3PcacheShrink(PCache*); @@ -9952,11 +13334,17 @@ SQLITE_PRIVATE void sqlite3PcacheStats(int*,int*,int*,int*); SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); +/* Return the header size */ +SQLITE_PRIVATE int sqlite3HeaderSizePcache(void); +SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void); + +/* Number of dirty pages as a percentage of the configured cache size */ +SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache*); + #endif /* _PCACHE_H_ */ /************** End of pcache.h **********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ - /************** Include os.h in the middle of sqliteInt.h ********************/ /************** Begin file os.h **********************************************/ /* @@ -10002,8 +13390,8 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); ** This file contains pre-processor directives related to operating system ** detection and/or setup. */ -#ifndef _OS_SETUP_H_ -#define _OS_SETUP_H_ +#ifndef SQLITE_OS_SETUP_H +#define SQLITE_OS_SETUP_H /* ** Figure out if we are dealing with Unix, Windows, or some other operating @@ -10043,7 +13431,7 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); # endif #endif -#endif /* _OS_SETUP_H_ */ +#endif /* SQLITE_OS_SETUP_H */ /************** End of os_setup.h ********************************************/ /************** Continuing where we left off in os.h *************************/ @@ -10142,7 +13530,7 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); ** shared locks begins at SHARED_FIRST. ** ** The same locking strategy and -** byte ranges are used for Unix. This leaves open the possiblity of having +** byte ranges are used for Unix. This leaves open the possibility of having ** clients on win95, winNT, and unix all talking to the same shared file ** and all locking correctly. To do so would require that samba (or whatever ** tool is being used for file sharing) implements locks correctly between @@ -10182,7 +13570,7 @@ SQLITE_PRIVATE int sqlite3OsInit(void); /* ** Functions for accessing sqlite3_file methods */ -SQLITE_PRIVATE int sqlite3OsClose(sqlite3_file*); +SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); @@ -10219,6 +13607,7 @@ SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); #endif /* SQLITE_OMIT_LOAD_EXTENSION */ SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); /* @@ -10226,7 +13615,7 @@ SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); ** sqlite3_malloc() to obtain space for the file-handle structure. */ SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); -SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); +SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); #endif /* _SQLITE_OS_H_ */ @@ -10261,7 +13650,7 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); ** Figure out what version of the code to use. The choices are ** ** SQLITE_MUTEX_OMIT No mutex logic. Not even stubs. The -** mutexes implemention cannot be overridden +** mutexes implementation cannot be overridden ** at start-time. ** ** SQLITE_MUTEX_NOOP For single-threaded applications. No @@ -10308,6 +13697,36 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); /************** End of mutex.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ +/* The SQLITE_EXTRA_DURABLE compile-time option used to set the default +** synchronous setting to EXTRA. It is no longer supported. +*/ +#ifdef SQLITE_EXTRA_DURABLE +# warning Use SQLITE_DEFAULT_SYNCHRONOUS=3 instead of SQLITE_EXTRA_DURABLE +# define SQLITE_DEFAULT_SYNCHRONOUS 3 +#endif + +/* +** Default synchronous levels. +** +** Note that (for historcal reasons) the PAGER_SYNCHRONOUS_* macros differ +** from the SQLITE_DEFAULT_SYNCHRONOUS value by 1. +** +** PAGER_SYNCHRONOUS DEFAULT_SYNCHRONOUS +** OFF 1 0 +** NORMAL 2 1 +** FULL 3 2 +** EXTRA 4 3 +** +** The "PRAGMA synchronous" statement also uses the zero-based numbers. +** In other words, the zero-based numbers are used for all external interfaces +** and the one-based values are used internally. +*/ +#ifndef SQLITE_DEFAULT_SYNCHRONOUS +# define SQLITE_DEFAULT_SYNCHRONOUS (PAGER_SYNCHRONOUS_FULL-1) +#endif +#ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS +# define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS +#endif /* ** Each database file to be accessed by the system is an instance @@ -10317,9 +13736,10 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); ** databases may be attached. */ struct Db { - char *zName; /* Name of this database */ + char *zDbSName; /* Name of this database. (schema name, not filename) */ Btree *pBt; /* The B*Tree structure for this database file */ u8 safety_level; /* How aggressive at syncing data to disk */ + u8 bSyncSet; /* True if "PRAGMA synchronous=N" has been run */ Schema *pSchema; /* Pointer to database schema (possibly shared) */ }; @@ -10330,7 +13750,7 @@ struct Db { ** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing. ** In shared cache mode, a single Schema object can be shared by multiple ** Btrees that refer to the same underlying BtShared object. -** +** ** Schema objects are automatically deallocated when the last Btree that ** references them is destroyed. The TEMP Schema is manually freed by ** sqlite3_close(). @@ -10355,7 +13775,7 @@ struct Schema { }; /* -** These macros can be used to test, set, or clear bits in the +** These macros can be used to test, set, or clear bits in the ** Db.pSchema->flags field. */ #define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))==(P)) @@ -10381,7 +13801,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_TRIGGER_DEPTH+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -10404,8 +13824,8 @@ struct Schema { ** lookaside allocations are not used to construct the schema objects. */ struct Lookaside { + u32 bDisable; /* Only operate the lookaside when zero */ u16 sz; /* Size of each buffer in bytes */ - u8 bEnabled; /* False to disable new lookaside allocations */ u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ int nOut; /* Number of buffers currently checked out */ int mxOut; /* Highwater mark for nOut */ @@ -10419,15 +13839,65 @@ struct LookasideSlot { }; /* -** A hash table for function definitions. +** A hash table for built-in function definitions. (Application-defined +** functions use a regular table table from hash.h.) ** ** Hash each FuncDef structure into one of the FuncDefHash.a[] slots. -** Collisions are on the FuncDef.pHash chain. +** Collisions are on the FuncDef.u.pHash chain. */ +#define SQLITE_FUNC_HASH_SZ 23 struct FuncDefHash { - FuncDef *a[23]; /* Hash table for functions */ + FuncDef *a[SQLITE_FUNC_HASH_SZ]; /* Hash table for functions */ }; +#ifdef SQLITE_USER_AUTHENTICATION +/* +** Information held in the "sqlite3" database connection object and used +** to manage user authentication. +*/ +typedef struct sqlite3_userauth sqlite3_userauth; +struct sqlite3_userauth { + u8 authLevel; /* Current authentication level */ + int nAuthPW; /* Size of the zAuthPW in bytes */ + char *zAuthPW; /* Password used to authenticate */ + char *zAuthUser; /* User name used to authenticate */ +}; + +/* Allowed values for sqlite3_userauth.authLevel */ +#define UAUTH_Unknown 0 /* Authentication not yet checked */ +#define UAUTH_Fail 1 /* User authentication failed */ +#define UAUTH_User 2 /* Authenticated as a normal user */ +#define UAUTH_Admin 3 /* Authenticated as an administrator */ + +/* Functions used only by user authorization logic */ +SQLITE_PRIVATE int sqlite3UserAuthTable(const char*); +SQLITE_PRIVATE int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*); +SQLITE_PRIVATE void sqlite3UserAuthInit(sqlite3*); +SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); + +#endif /* SQLITE_USER_AUTHENTICATION */ + +/* +** typedef for the authorization callback function. +*/ +#ifdef SQLITE_USER_AUTHENTICATION + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*, const char*); +#else + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*); +#endif + +#ifndef SQLITE_OMIT_DEPRECATED +/* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing +** in the style of sqlite3_trace() +*/ +#define SQLITE_TRACE_LEGACY 0x80 +#else +#define SQLITE_TRACE_LEGACY 0 +#endif /* SQLITE_OMIT_DEPRECATED */ + + /* ** Each database connection is an instance of the following structure. */ @@ -10444,42 +13914,56 @@ struct sqlite3 { unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ + int iSysErrno; /* Errno value from last system error */ u16 dbOptFlags; /* Flags to enable/disable optimizations */ + u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ + u8 bBenignMalloc; /* Do not require OOMs if true */ u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ + u8 mTrace; /* zero or more SQLITE_TRACE flags */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ + int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ int newTnum; /* Rootpage of table being initialized */ u8 iDb; /* Which db file is being initialized */ u8 busy; /* TRUE if currently initializing */ u8 orphanTrigger; /* Last statement is orphaned TEMP trigger */ + u8 imposterTable; /* Building an imposter table */ } init; int nVdbeActive; /* Number of VDBEs currently running */ int nVdbeRead; /* Number of active VDBEs that read or write */ int nVdbeWrite; /* Number of active VDBEs that read and write */ int nVdbeExec; /* Number of nested calls to VdbeExec() */ + int nVDestroy; /* Number of active OP_VDestroy operations */ int nExtension; /* Number of loaded extensions */ void **aExtension; /* Array of shared library handles */ - void (*xTrace)(void*,const char*); /* Trace function */ + int (*xTrace)(u32,void*,void*,void*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ void (*xProfile)(void*,const char*,u64); /* Profiling function */ void *pProfileArg; /* Argument to profile function */ - void *pCommitArg; /* Argument to xCommitCallback() */ + void *pCommitArg; /* Argument to xCommitCallback() */ int (*xCommitCallback)(void*); /* Invoked at every commit. */ - void *pRollbackArg; /* Argument to xRollbackCallback() */ + void *pRollbackArg; /* Argument to xRollbackCallback() */ void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + void *pPreUpdateArg; /* First argument to xPreUpdateCallback */ + void (*xPreUpdateCallback)( /* Registered using sqlite3_preupdate_hook() */ + void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64 + ); + PreUpdate *pPreUpdate; /* Context for active pre-update callback */ +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ #ifndef SQLITE_OMIT_WAL int (*xWalCallback)(void *, sqlite3 *, const char *, int); void *pWalArg; @@ -10494,8 +13978,7 @@ struct sqlite3 { } u1; Lookaside lookaside; /* Lookaside malloc configuration */ #ifndef SQLITE_OMIT_AUTHORIZATION - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); - /* Access authorization function */ + sqlite3_xauth xAuth; /* Access authorization function */ void *pAuthArg; /* 1st argument to the access auth function */ #endif #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -10510,7 +13993,7 @@ struct sqlite3 { VTable **aVTrans; /* Virtual tables with open transactions */ VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ #endif - FuncDefHash aFunc; /* Hash table of connection functions */ + Hash aFunc; /* Hash table of connection functions */ Hash aCollSeq; /* All collating sequences */ BusyHandler busyHandler; /* Busy callback */ Db aDbStatic[2]; /* Static space for the 2 default backends */ @@ -10521,10 +14004,9 @@ struct sqlite3 { i64 nDeferredCons; /* Net deferred constraints this transaction. */ i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ - #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - /* The following variables are all protected by the STATIC_MASTER - ** mutex, not by sqlite3.mutex. They are used by code in notify.c. + /* The following variables are all protected by the STATIC_MASTER + ** mutex, not by sqlite3.mutex. They are used by code in notify.c. ** ** When X.pUnlockConnection==Y, that means that X is waiting for Y to ** unlock so that it can proceed. @@ -10539,22 +14021,31 @@ struct sqlite3 { void (*xUnlockNotify)(void **, int); /* Unlock notify callback */ sqlite3 *pNextBlocked; /* Next in list of all blocked connections */ #endif +#ifdef SQLITE_USER_AUTHENTICATION + sqlite3_userauth auth; /* User authentication information */ +#endif }; /* ** A macro to discover the encoding of a database. */ -#define ENC(db) ((db)->aDb[0].pSchema->enc) +#define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) +#define ENC(db) ((db)->enc) /* ** Possible values for the sqlite3.flags. +** +** Value constraints (enforced via assert()): +** SQLITE_FullFSync == PAGER_FULLFSYNC +** SQLITE_CkptFullFSync == PAGER_CKPT_FULLFSYNC +** SQLITE_CacheSpill == PAGER_CACHE_SPILL */ #define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ #define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ -#define SQLITE_FullFSync 0x00000004 /* Use full fsync on the backend */ -#define SQLITE_CkptFullFSync 0x00000008 /* Use full fsync for checkpoint */ -#define SQLITE_CacheSpill 0x00000010 /* OK to spill pager cache */ -#define SQLITE_FullColNames 0x00000020 /* Show full column names on SELECT */ +#define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ +#define SQLITE_FullFSync 0x00000008 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00000010 /* Use full fsync for checkpoint */ +#define SQLITE_CacheSpill 0x00000020 /* OK to spill pager cache */ #define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ #define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ @@ -10575,10 +14066,14 @@ struct sqlite3 { #define SQLITE_AutoIndex 0x00100000 /* Enable automatic indexes */ #define SQLITE_PreferBuiltin 0x00200000 /* Preference to built-in funcs */ #define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ -#define SQLITE_EnableTrigger 0x00800000 /* True to enable triggers */ -#define SQLITE_DeferFKs 0x01000000 /* Defer all FK constraints */ -#define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ -#define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_LoadExtFunc 0x00800000 /* Enable load_extension() SQL func */ +#define SQLITE_EnableTrigger 0x01000000 /* True to enable triggers */ +#define SQLITE_DeferFKs 0x02000000 /* Defer all FK constraints */ +#define SQLITE_QueryOnly 0x04000000 /* Disable database changes */ +#define SQLITE_VdbeEQP 0x08000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_Vacuum 0x10000000 /* Currently in a VACUUM */ +#define SQLITE_CellSizeCk 0x20000000 /* Check btree cell sizes on load */ +#define SQLITE_Fts3Tokenizer 0x40000000 /* Enable fts3_tokenizer(2) */ /* @@ -10597,8 +14092,8 @@ struct sqlite3 { #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ -#define SQLITE_AdjustOutEst 0x1000 /* Adjust output estimates using WHERE */ +#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ +#define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -10632,28 +14127,33 @@ struct sqlite3 { /* ** Each SQL function is defined by an instance of the following -** structure. A pointer to this structure is stored in the sqlite.aFunc -** hash table. When multiple functions have the same name, the hash table -** points to a linked list of these structures. +** structure. For global built-in functions (ex: substr(), max(), count()) +** a pointer to this structure is held in the sqlite3BuiltinFunctions object. +** For per-connection application-defined functions, a pointer to this +** structure is held in the db->aHash hash table. +** +** The u.pHash field is used by the global built-ins. The u.pDestructor +** field is used by per-connection app-def functions. */ struct FuncDef { - i16 nArg; /* Number of arguments. -1 means unlimited */ + i8 nArg; /* Number of arguments. -1 means unlimited */ u16 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */ - void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */ - void (*xFinalize)(sqlite3_context*); /* Aggregate finalizer */ - char *zName; /* SQL name of the function. */ - FuncDef *pHash; /* Next with a different name but the same hash */ - FuncDestructor *pDestructor; /* Reference counted destructor function */ + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */ + void (*xFinalize)(sqlite3_context*); /* Agg finalizer */ + const char *zName; /* SQL name of the function. */ + union { + FuncDef *pHash; /* Next with a different name but the same hash */ + FuncDestructor *pDestructor; /* Reference counted destructor function */ + } u; }; /* ** This structure encapsulates a user-function destructor callback (as ** configured using create_function_v2()) and a reference counter. When ** create_function_v2() is called to create a function with a destructor, -** a single object of this type is allocated. FuncDestructor.nRef is set to +** a single object of this type is allocated. FuncDestructor.nRef is set to ** the number of FuncDef objects created (either 1 or 3, depending on whether ** or not the specified encoding is SQLITE_ANY). The FuncDef.pDestructor ** member of each of the new FuncDef objects is set to point to the allocated @@ -10671,35 +14171,52 @@ struct FuncDestructor { /* ** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF -** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. And +** SQLITE_FUNC_CONSTANT must be the same as SQLITE_DETERMINISTIC. There ** are assert() statements in the code to verify this. +** +** Value constraints (enforced via assert()): +** SQLITE_FUNC_MINMAX == NC_MinMaxAgg == SF_MinMaxAgg +** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG +** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG +** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API +** SQLITE_FUNC_ENCMASK depends on SQLITE_UTF* macros in the API */ -#define SQLITE_FUNC_ENCMASK 0x003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ -#define SQLITE_FUNC_LIKE 0x004 /* Candidate for the LIKE optimization */ -#define SQLITE_FUNC_CASE 0x008 /* Case-sensitive LIKE-type function */ -#define SQLITE_FUNC_EPHEM 0x010 /* Ephemeral. Delete with VDBE */ -#define SQLITE_FUNC_NEEDCOLL 0x020 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_LENGTH 0x040 /* Built-in length() function */ -#define SQLITE_FUNC_TYPEOF 0x080 /* Built-in typeof() function */ -#define SQLITE_FUNC_COUNT 0x100 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ -#define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ -#define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +#define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ +#define SQLITE_FUNC_CASE 0x0008 /* Case-sensitive LIKE-type function */ +#define SQLITE_FUNC_EPHEM 0x0010 /* Ephemeral. Delete with VDBE */ +#define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ +#define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ +#define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */ +#define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ +#define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a + ** single query - might change over time */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are ** used to create the initializers for the FuncDef structures. ** ** FUNCTION(zName, nArg, iArg, bNC, xFunc) -** Used to create a scalar function definition of a function zName +** Used to create a scalar function definition of a function zName ** implemented by C function xFunc that accepts nArg arguments. The ** value passed as iArg is cast to a (void*) and made available -** as the user-data (sqlite3_user_data()) for the function. If +** as the user-data (sqlite3_user_data()) for the function. If ** argument bNC is true, then the SQLITE_FUNC_NEEDCOLL flag is set. ** ** VFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. ** +** DFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and +** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions +** and functions like sqlite_version() that can change, but not during +** a single query. +** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by ** the C functions xStep and xFinal. The first four parameters @@ -10707,8 +14224,8 @@ struct FuncDestructor { ** FUNCTION(). ** ** LIKEFUNC(zName, nArg, pArg, flags) -** Used to create a scalar function definition of a function zName -** that accepts nArg arguments and is implemented by a call to C +** Used to create a scalar function definition of a function zName +** that accepts nArg arguments and is implemented by a call to C ** function likeFunc. Argument pArg is cast to a (void *) and made ** available as the function user-data (sqlite3_user_data()). The ** FuncDef.flags variable is set to the value passed as the flags @@ -10716,22 +14233,28 @@ struct FuncDestructor { */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } +#define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, 0, #zName, 0, 0} + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + pArg, 0, xFunc, 0, #zName, } #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ - (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} + (void *)arg, 0, likeFunc, 0, #zName, {0} } #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} +#define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} /* ** All current savepoints are stored in a linked list starting at @@ -10765,6 +14288,7 @@ struct Module { const char *zName; /* Name passed to create_module() */ void *pAux; /* pAux passed to create_module() */ void (*xDestroy)(void *); /* Module destructor function */ + Table *pEpoTab; /* Eponymous table for this module */ }; /* @@ -10772,14 +14296,12 @@ struct Module { ** of this structure. */ struct Column { - char *zName; /* Name of this column */ + char *zName; /* Name of this column, \000, then the type */ Expr *pDflt; /* Default value of this column */ - char *zDflt; /* Original text of the default value */ - char *zType; /* Data type for this column */ char *zColl; /* Collating sequence. If NULL, use the default */ u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ - u8 szEst; /* Estimated size of this column. INT==1 */ + u8 szEst; /* Estimated size of value in this column. sizeof(INT)==1 */ u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; @@ -10787,6 +14309,7 @@ struct Column { */ #define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ #define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ +#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ /* ** A "Collating Sequence" is defined by an instance of the following @@ -10810,34 +14333,35 @@ struct CollSeq { */ #define SQLITE_SO_ASC 0 /* Sort in ascending order */ #define SQLITE_SO_DESC 1 /* Sort in ascending order */ +#define SQLITE_SO_UNDEFINED -1 /* No sort order specified */ /* ** Column affinity types. ** ** These used to have mnemonic name like 'i' for SQLITE_AFF_INTEGER and ** 't' for SQLITE_AFF_TEXT. But we can save a little space and improve -** the speed a little by numbering the values consecutively. +** the speed a little by numbering the values consecutively. ** -** But rather than start with 0 or 1, we begin with 'a'. That way, +** But rather than start with 0 or 1, we begin with 'A'. That way, ** when multiple affinity types are concatenated into a string and ** used as the P4 operand, they will be more readable. ** ** Note also that the numeric types are grouped together so that testing -** for a numeric type is a single comparison. +** for a numeric type is a single comparison. And the BLOB type is first. */ -#define SQLITE_AFF_TEXT 'a' -#define SQLITE_AFF_NONE 'b' -#define SQLITE_AFF_NUMERIC 'c' -#define SQLITE_AFF_INTEGER 'd' -#define SQLITE_AFF_REAL 'e' +#define SQLITE_AFF_BLOB 'A' +#define SQLITE_AFF_TEXT 'B' +#define SQLITE_AFF_NUMERIC 'C' +#define SQLITE_AFF_INTEGER 'D' +#define SQLITE_AFF_REAL 'E' #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) /* ** The SQLITE_AFF_MASK values masks off the significant bits of an -** affinity value. +** affinity value. */ -#define SQLITE_AFF_MASK 0x67 +#define SQLITE_AFF_MASK 0x47 /* ** Additional bit values that can be ORed with an affinity without @@ -10848,27 +14372,28 @@ struct CollSeq { ** operator is NULL. It is added to certain comparison operators to ** prove that the operands are always NOT NULL. */ -#define SQLITE_JUMPIFNULL 0x08 /* jumps if either operand is NULL */ -#define SQLITE_STOREP2 0x10 /* Store result in reg[P2] rather than jump */ +#define SQLITE_KEEPNULL 0x08 /* Used by vector == or <> */ +#define SQLITE_JUMPIFNULL 0x10 /* jumps if either operand is NULL */ +#define SQLITE_STOREP2 0x20 /* Store result in reg[P2] rather than jump */ #define SQLITE_NULLEQ 0x80 /* NULL=NULL */ -#define SQLITE_NOTNULL 0x88 /* Assert that operands are never NULL */ +#define SQLITE_NOTNULL 0x90 /* Assert that operands are never NULL */ /* ** An object of this type is created for each virtual table present in -** the database schema. +** the database schema. ** ** If the database schema is shared, then there is one instance of this ** structure for each database connection (sqlite3*) that uses the shared ** schema. This is because each database connection requires its own unique -** instance of the sqlite3_vtab* handle used to access the virtual table -** implementation. sqlite3_vtab* handles can not be shared between -** database connections, even when the rest of the in-memory database +** instance of the sqlite3_vtab* handle used to access the virtual table +** implementation. sqlite3_vtab* handles can not be shared between +** database connections, even when the rest of the in-memory database ** schema is shared, as the implementation often stores the database ** connection handle passed to it via the xConnect() or xCreate() method ** during initialization internally. This database connection handle may -** then be used by the virtual table implementation to access real tables -** within the database. So that they appear as part of the callers -** transaction, these accesses need to be made via the same database +** then be used by the virtual table implementation to access real tables +** within the database. So that they appear as part of the callers +** transaction, these accesses need to be made via the same database ** connection as that used to execute SQL operations on the virtual table. ** ** All VTable objects that correspond to a single table in a shared @@ -10880,19 +14405,19 @@ struct CollSeq { ** sqlite3_vtab* handle in the compiled query. ** ** When an in-memory Table object is deleted (for example when the -** schema is being reloaded for some reason), the VTable objects are not -** deleted and the sqlite3_vtab* handles are not xDisconnect()ed +** schema is being reloaded for some reason), the VTable objects are not +** deleted and the sqlite3_vtab* handles are not xDisconnect()ed ** immediately. Instead, they are moved from the Table.pVTable list to ** another linked list headed by the sqlite3.pDisconnect member of the -** corresponding sqlite3 structure. They are then deleted/xDisconnected +** corresponding sqlite3 structure. They are then deleted/xDisconnected ** next time a statement is prepared using said sqlite3*. This is done ** to avoid deadlock issues involving multiple sqlite3.mutex mutexes. ** Refer to comments above function sqlite3VtabUnlockList() for an ** explanation as to why it is safe to add an entry to an sqlite3.pDisconnect ** list without holding the corresponding sqlite3.mutex mutex. ** -** The memory for objects of this type is always allocated by -** sqlite3DbMalloc(), using the connection handle stored in VTable.db as +** The memory for objects of this type is always allocated by +** sqlite3DbMalloc(), using the connection handle stored in VTable.db as ** the first argument. */ struct VTable { @@ -10906,34 +14431,8 @@ struct VTable { }; /* -** Each SQL table is represented in memory by an instance of the -** following structure. -** -** Table.zName is the name of the table. The case of the original -** CREATE TABLE statement is stored, but case is not significant for -** comparisons. -** -** Table.nCol is the number of columns in this table. Table.aCol is a -** pointer to an array of Column structures, one for each column. -** -** If the table has an INTEGER PRIMARY KEY, then Table.iPKey is the index of -** the column that is that key. Otherwise Table.iPKey is negative. Note -** that the datatype of the PRIMARY KEY must be INTEGER for this field to -** be set. An INTEGER PRIMARY KEY is used as the rowid for each row of -** the table. If a table has no INTEGER PRIMARY KEY, then a random rowid -** is generated for each row of the table. TF_HasPrimaryKey is set if -** the table has any PRIMARY KEY, INTEGER or otherwise. -** -** Table.tnum is the page number for the root BTree page of the table in the -** database file. If Table.iDb is the index of the database table backend -** in sqlite.aDb[]. 0 is for the main database and 1 is for the file that -** holds temporary tables and indices. If TF_Ephemeral is set -** then the table is stored in a file that is automatically deleted -** when the VDBE cursor to the table is closed. In this case Table.tnum -** refers VDBE cursor number that holds the table open, not to the root -** page number. Transient tables are used to hold the results of a -** sub-query that appears instead of a real table name in the FROM clause -** of a SELECT statement. +** The schema for each SQL table and view is represented in memory +** by an instance of the following structure. */ struct Table { char *zName; /* Name of the table or view */ @@ -10942,14 +14441,13 @@ struct Table { Select *pSelect; /* NULL for tables. Points to definition if a view. */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ -#ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ -#endif - LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ - int tnum; /* Root BTree node for this table (see note above) */ - i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ + /* ... also used as column name list in a VIEW */ + int tnum; /* Root BTree page for this table */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ u16 nRef; /* Number of pointers to this Table */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ LogEst szTabRow; /* Estimated size of each table row in bytes */ #ifdef SQLITE_ENABLE_COSTMULT LogEst costMult; /* Cost multiplier for using this table */ @@ -10961,7 +14459,7 @@ struct Table { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE int nModuleArg; /* Number of arguments to the module */ - char **azModuleArg; /* Text of all module args. [0] is module name */ + char **azModuleArg; /* 0: module 1: schema 2: vtab name 3...: args */ VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ @@ -10971,13 +14469,21 @@ struct Table { /* ** Allowed values for Table.tabFlags. +** +** TF_OOOHidden applies to tables or view that have hidden columns that are +** followed by non-hidden columns. Example: "CREATE VIRTUAL TABLE x USING +** vtab1(a HIDDEN, b);". Since "b" is a non-hidden column but "a" is hidden, +** the TF_OOOHidden attribute would apply in this case. Such tables require +** special handling during INSERT processing. */ #define TF_Readonly 0x01 /* Read-only system table */ #define TF_Ephemeral 0x02 /* An ephemeral table */ #define TF_HasPrimaryKey 0x04 /* Table has a primary key */ #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ #define TF_Virtual 0x10 /* Is a virtual table */ -#define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */ +#define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ +#define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ /* @@ -10987,14 +14493,31 @@ struct Table { */ #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) -# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) #else # define IsVirtual(X) 0 -# define IsHiddenColumn(X) 0 #endif +/* +** Macros to determine if a column is hidden. IsOrdinaryHiddenColumn() +** only works for non-virtual tables (ordinary tables and views) and is +** always false unless SQLITE_ENABLE_HIDDEN_COLUMNS is defined. The +** IsHiddenColumn() macro is general purpose. +*/ +#if defined(SQLITE_ENABLE_HIDDEN_COLUMNS) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +# define IsOrdinaryHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +#elif !defined(SQLITE_OMIT_VIRTUALTABLE) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +# define IsOrdinaryHiddenColumn(X) 0 +#else +# define IsHiddenColumn(X) 0 +# define IsOrdinaryHiddenColumn(X) 0 +#endif + + /* Does the table have a rowid */ #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) +#define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) /* ** Each foreign key constraint is an instance of the following structure. @@ -11062,7 +14585,7 @@ struct FKey { ** key is set to NULL. CASCADE means that a DELETE or UPDATE of the ** referenced table row is propagated into the row that holds the ** foreign key. -** +** ** The following symbolic values are used to record which type ** of action to take. */ @@ -11083,7 +14606,7 @@ struct FKey { /* ** An instance of the following structure is passed as the first -** argument to sqlite3VdbeKeyCompare and is used to control the +** argument to sqlite3VdbeKeyCompare and is used to control the ** comparison of the two index keys. ** ** Note that aSortOrder[] and aColl[] have nField+1 slots. There @@ -11101,9 +14624,8 @@ struct KeyInfo { }; /* -** An instance of the following structure holds information about a -** single index record that has already been parsed out into individual -** values. +** This object holds a record which has been parsed out into individual +** fields, for the purposes of doing a comparison. ** ** A record is an object that contains one or more fields of data. ** Records are used to store the content of a table row and to store @@ -11111,20 +14633,40 @@ struct KeyInfo { ** the OP_MakeRecord opcode of the VDBE and is disassembled by the ** OP_Column opcode. ** -** This structure holds a record that has already been disassembled -** into its constituent fields. +** An instance of this object serves as a "key" for doing a search on +** an index b+tree. The goal of the search is to find the entry that +** is closed to the key described by this object. This object might hold +** just a prefix of the key. The number of fields is given by +** pKeyInfo->nField. ** -** The r1 and r2 member variables are only used by the optimized comparison -** functions vdbeRecordCompareInt() and vdbeRecordCompareString(). +** The r1 and r2 fields are the values to return if this key is less than +** or greater than a key in the btree, respectively. These are normally +** -1 and +1 respectively, but might be inverted to +1 and -1 if the b-tree +** is in DESC order. +** +** The key comparison functions actually return default_rc when they find +** an equals comparison. default_rc can be -1, 0, or +1. If there are +** multiple entries in the b-tree with the same key (when only looking +** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to +** cause the search to find the last match, or +1 to cause the search to +** find the first match. +** +** The key comparison functions will set eqSeen to true if they ever +** get and equal results when comparing this structure to a b-tree record. +** When default_rc!=0, the search might end up on the record immediately +** before the first match or immediately after the last match. The +** eqSeen field will indicate whether or not an exact match exists in the +** b-tree. */ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ + Mem *aMem; /* Values */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ - u8 isCorrupt; /* Corruption detected by xRecordCompare() */ - Mem *aMem; /* Values */ - int r1; /* Value to return if (lhs > rhs) */ - int r2; /* Value to return if (rhs < lhs) */ + u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ + i8 r1; /* Value to return if (lhs > rhs) */ + i8 r2; /* Value to return if (rhs < lhs) */ + u8 eqSeen; /* True if an equality comparison has been seen */ }; @@ -11142,7 +14684,7 @@ struct UnpackedRecord { ** In the Table structure describing Ex1, nCol==3 because there are ** three columns in the table. In the Index structure describing ** Ex2, nColumn==2 since 2 of the 3 columns of Ex1 are indexed. -** The value of aiColumn is {2, 0}. aiColumn[0]==2 because the +** The value of aiColumn is {2, 0}. aiColumn[0]==2 because the ** first column to be indexed (c3) has an index of 2 in Ex1.aCol[]. ** The second column to be indexed (c1) has an index of 0 in ** Ex1.aCol[], hence Ex2.aiColumn[1]==0. @@ -11150,9 +14692,17 @@ struct UnpackedRecord { ** The Index.onError field determines whether or not the indexed columns ** must be unique and what to do if they are not. When Index.onError=OE_None, ** it means this is not a unique index. Otherwise it is a unique index -** and the value of Index.onError indicate the which conflict resolution +** and the value of Index.onError indicate the which conflict resolution ** algorithm to employ whenever an attempt is made to insert a non-unique ** element. +** +** While parsing a CREATE TABLE or CREATE INDEX statement in order to +** generate VDBE code (as opposed to parsing one read from an sqlite_master +** table as part of parsing an existing database schema), transient instances +** of this structure may be created. In this case the Index.tnum variable is +** used to store the address of a VDBE instruction, not a database page +** number (it cannot - the database page is not allocated until the VDBE +** program is executed). See convertToWithoutRowidTable() for details. */ struct Index { char *zName; /* Name of this index */ @@ -11163,9 +14713,9 @@ struct Index { Index *pNext; /* The next index associated with the same table */ Schema *pSchema; /* Schema containing this index */ u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ - char **azColl; /* Array of collation sequence names for index */ + const char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ - KeyInfo *pKeyInfo; /* A KeyInfo object suitable for this index */ + ExprList *aColExpr; /* Column expressions */ int tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ @@ -11176,11 +14726,14 @@ struct Index { unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ + unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ + tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif }; @@ -11197,8 +14750,14 @@ struct Index { /* Return true if index X is a UNIQUE index */ #define IsUniqueIndex(X) ((X)->onError!=OE_None) +/* The Index.aiColumn[] values are normally positive integer. But +** there are some negative values that have special meaning: +*/ +#define XN_ROWID (-1) /* Indexed column is the rowid */ +#define XN_EXPR (-2) /* Indexed column is an expression */ + /* -** Each sample stored in the sqlite_stat3 table is represented in memory +** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the ** analyze.c source file for additional information. */ @@ -11293,9 +14852,9 @@ typedef int ynVar; ** to represent the greater-than-or-equal-to operator in the expression ** tree. ** -** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, +** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, ** or TK_STRING), then Expr.token contains the text of the SQL literal. If -** the expression is a variable (TK_VARIABLE), then Expr.token contains the +** the expression is a variable (TK_VARIABLE), then Expr.token contains the ** variable name. Finally, if the expression is an SQL function (TK_FUNCTION), ** then Expr.token contains the name of the function. ** @@ -11306,7 +14865,7 @@ typedef int ynVar; ** a CASE expression or an IN expression of the form " IN (, ...)". ** Expr.x.pSelect is used if the expression is a sub-select or an expression of ** the form " IN (SELECT ...)". If the EP_xIsSelect bit is set in the -** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is +** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is ** valid. ** ** An expression of the form ID or ID.ID refers to a column in a table. @@ -11317,8 +14876,8 @@ typedef int ynVar; ** value is also stored in the Expr.iAgg column in the aggregate so that ** it can be accessed after all aggregates are computed. ** -** If the expression is an unbound variable marker (a question mark -** character '?' in the original SQL) then the Expr.iTable holds the index +** If the expression is an unbound variable marker (a question mark +** character '?' in the original SQL) then the Expr.iTable holds the index ** number for that variable. ** ** If the expression is a subquery then Expr.iColumn holds an integer @@ -11357,7 +14916,7 @@ struct Expr { /* If the EP_TokenOnly flag is set in the Expr.flags mask, then no ** space is allocated for the fields below this point. An attempt to - ** access them will result in a segfault or malfunction. + ** access them will result in a segfault or malfunction. *********************************************************************/ Expr *pLeft; /* Left subnode */ @@ -11378,9 +14937,11 @@ struct Expr { int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old - ** EP_Unlikely: 1000 times likelihood */ + ** EP_Unlikely: 134217728 times likelihood + ** TK_SELECT: 1st register of result vector */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. - ** TK_VARIABLE: variable number (always >= 1). */ + ** TK_VARIABLE: variable number (always >= 1). + ** TK_SELECT_COLUMN: column of the result vector */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 op2; /* TK_REGISTER: original value of Expr.op @@ -11393,7 +14954,7 @@ struct Expr { /* ** The following are the meanings of bits in the Expr.flags field. */ -#define EP_FromJoin 0x000001 /* Originated in ON or USING clause of a join */ +#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ #define EP_Agg 0x000002 /* Contains one or more aggregate functions */ #define EP_Resolved 0x000004 /* IDs have been resolved to COLUMNs */ #define EP_Error 0x000008 /* Expression contains one or more errors */ @@ -11412,10 +14973,19 @@ struct Expr { #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_Constant 0x080000 /* Node is a constant */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ +#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ +#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ +#define EP_Alias 0x400000 /* Is an alias for a result set column */ +#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ /* -** These macros can be used to test, set, or clear bits in the +** Combinations of two or more EP_* flags +*/ +#define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */ + +/* +** These macros can be used to test, set, or clear bits in the ** Expr.flags field. */ #define ExprHasProperty(E,P) (((E)->flags&(P))!=0) @@ -11434,8 +15004,8 @@ struct Expr { #endif /* -** Macros to determine the number of bytes required by a normal Expr -** struct, an Expr struct with the EP_Reduced flag set in Expr.flags +** Macros to determine the number of bytes required by a normal Expr +** struct, an Expr struct with the EP_Reduced flag set in Expr.flags ** and an Expr struct with the EP_TokenOnly flag set. */ #define EXPR_FULLSIZE sizeof(Expr) /* Full size */ @@ -11443,7 +15013,7 @@ struct Expr { #define EXPR_TOKENONLYSIZE offsetof(Expr,pLeft) /* Fewer features */ /* -** Flags passed to the sqlite3ExprDup() function. See the header comment +** Flags passed to the sqlite3ExprDup() function. See the header comment ** above sqlite3ExprDup() for details. */ #define EXPRDUP_REDUCE 0x0001 /* Used reduced-size Expr nodes */ @@ -11525,7 +15095,11 @@ struct IdList { ** tables in a join to 32 instead of 64. But it also reduces the size ** of the library by 738 bytes on ix86. */ -typedef u64 Bitmask; +#ifdef SQLITE_BITMASK_TYPE + typedef SQLITE_BITMASK_TYPE Bitmask; +#else + typedef u64 Bitmask; +#endif /* ** The number of bits in a Bitmask. "BMS" means "BitMask Size". @@ -11537,6 +15111,7 @@ typedef u64 Bitmask; */ #define MASKBIT(n) (((Bitmask)1)<<(n)) #define MASKBIT32(n) (((unsigned int)1)<<(n)) +#define ALLBITS ((Bitmask)-1) /* ** The following structure describes the FROM clause of a SELECT statement. @@ -11570,11 +15145,15 @@ struct SrcList { int addrFillSub; /* Address of subroutine to manifest a subquery */ int regReturn; /* Register holding return address of addrFillSub */ int regResult; /* Registers holding results of a co-routine */ - u8 jointype; /* Type of join between this able and the previous */ - unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ - unsigned isCorrelated :1; /* True if sub-query is correlated */ - unsigned viaCoroutine :1; /* Implemented as a co-routine */ - unsigned isRecursive :1; /* True for recursive reference in WITH */ + struct { + u8 jointype; /* Type of join between this table and the previous */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isTabFunc :1; /* True if table-valued-function syntax */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ + } fg; #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif @@ -11582,8 +15161,11 @@ struct SrcList { Expr *pOn; /* The ON clause of a join */ IdList *pUsing; /* The USING clause of a join */ Bitmask colUsed; /* Bit N (1<" clause */ - Index *pIndex; /* Index structure corresponding to zIndex, if any */ + union { + char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ + ExprList *pFuncArg; /* Arguments to table-valued-function */ + } u1; + Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ } a[1]; /* One entry for each identifier on the list */ }; @@ -11602,21 +15184,28 @@ struct SrcList { /* ** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin() ** and the WhereInfo.wctrlFlags member. +** +** Value constraints (enforced via assert()): +** WHERE_USE_LIMIT == SF_FixedLimit */ #define WHERE_ORDERBY_NORMAL 0x0000 /* No-op */ #define WHERE_ORDERBY_MIN 0x0001 /* ORDER BY processing for min() func */ #define WHERE_ORDERBY_MAX 0x0002 /* ORDER BY processing for max() func */ #define WHERE_ONEPASS_DESIRED 0x0004 /* Want to do one-pass UPDATE/DELETE */ -#define WHERE_DUPLICATES_OK 0x0008 /* Ok to return a row more than once */ -#define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ -#define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ -#define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ -#define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ -#define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ -#define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ -#define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ -#define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ -#define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ +#define WHERE_ONEPASS_MULTIROW 0x0008 /* ONEPASS is ok with multiple rows */ +#define WHERE_DUPLICATES_OK 0x0010 /* Ok to return a row more than once */ +#define WHERE_OR_SUBCLAUSE 0x0020 /* Processing a sub-WHERE as part of + ** the OR optimization */ +#define WHERE_GROUPBY 0x0040 /* pOrderBy is really a GROUP BY */ +#define WHERE_DISTINCTBY 0x0080 /* pOrderby is really a DISTINCT clause */ +#define WHERE_WANT_DISTINCT 0x0100 /* All output needs to be distinct */ +#define WHERE_SORTBYGROUP 0x0200 /* Support sqlite3WhereIsSorted() */ +#define WHERE_SEEK_TABLE 0x0400 /* Do not defer seeks on main table */ +#define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ + /* 0x1000 not currently used */ + /* 0x2000 not currently used */ +#define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ + /* 0x8000 not currently used */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -11634,12 +15223,12 @@ struct SrcList { ** pEList corresponds to the result set of a SELECT and is NULL for ** other statements. ** -** NameContexts can be nested. When resolving names, the inner-most +** NameContexts can be nested. When resolving names, the inner-most ** context is searched first. If no match is found, the next outer ** context is checked. If there is still no match, the next context ** is checked. This process continues until either a match is found ** or all contexts are check. When a match is found, the nRef member of -** the context containing the match is incremented. +** the context containing the match is incremented. ** ** Each subquery gets a new NameContext. The pNext field points to the ** NameContext in the parent query. Thus the process of scanning the @@ -11654,17 +15243,25 @@ struct NameContext { NameContext *pNext; /* Next outer name context. NULL for outermost */ int nRef; /* Number of names resolved by this context */ int nErr; /* Number of errors encountered while resolving names */ - u8 ncFlags; /* Zero or more NC_* flags defined below */ + u16 ncFlags; /* Zero or more NC_* flags defined below */ }; /* ** Allowed values for the NameContext, ncFlags field. +** +** Value constraints (all checked via assert()): +** NC_HasAgg == SF_HasAgg +** NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX +** */ -#define NC_AllowAgg 0x01 /* Aggregate functions are allowed here */ -#define NC_HasAgg 0x02 /* One or more aggregate functions seen */ -#define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ -#define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ -#define NC_PartIdx 0x10 /* True if resolving a partial index WHERE */ +#define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ +#define NC_PartIdx 0x0002 /* True if resolving a partial index WHERE */ +#define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ +#define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ +#define NC_HasAgg 0x0010 /* One or more aggregate functions seen */ +#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ +#define NC_VarSelect 0x0040 /* A correlated subquery has been seen */ +#define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ /* ** An instance of the following structure contains all information @@ -11689,10 +15286,13 @@ struct NameContext { struct Select { ExprList *pEList; /* The fields of the result */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ - u16 selFlags; /* Various SF_* values */ + LogEst nSelectRow; /* Estimated number of result rows */ + u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ +#if SELECTTRACE_ENABLED + char zSelName[12]; /* Symbolic name of this SELECT use for debugging */ +#endif int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ - u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ ExprList *pGroupBy; /* The GROUP BY clause */ @@ -11708,20 +15308,30 @@ struct Select { /* ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". +** +** Value constraints (all checked via assert()) +** SF_HasAgg == NC_HasAgg +** SF_MinMaxAgg == NC_MinMaxAgg == SQLITE_FUNC_MINMAX +** SF_FixedLimit == WHERE_USE_LIMIT */ -#define SF_Distinct 0x0001 /* Output should be DISTINCT */ -#define SF_Resolved 0x0002 /* Identifiers have been resolved */ -#define SF_Aggregate 0x0004 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ - /* 0x0040 NOT USED */ -#define SF_Values 0x0080 /* Synthesized from VALUES clause */ - /* 0x0100 NOT USED */ -#define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ -#define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ -#define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ -#define SF_Compound 0x1000 /* Part of a compound query */ +#define SF_Distinct 0x00001 /* Output should be DISTINCT */ +#define SF_All 0x00002 /* Includes the ALL keyword */ +#define SF_Resolved 0x00004 /* Identifiers have been resolved */ +#define SF_Aggregate 0x00008 /* Contains agg functions or a GROUP BY */ +#define SF_HasAgg 0x00010 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x00020 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x00040 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x00080 /* FROM subqueries have Table metadata */ +#define SF_Compound 0x00100 /* Part of a compound query */ +#define SF_Values 0x00200 /* Synthesized from VALUES clause */ +#define SF_MultiValue 0x00400 /* Single VALUES term with multiple rows */ +#define SF_NestedFrom 0x00800 /* Part of a parenthesized FROM clause */ +#define SF_MinMaxAgg 0x01000 /* Aggregate containing min() or max() */ +#define SF_Recursive 0x02000 /* The recursive part of a recursive CTE */ +#define SF_FixedLimit 0x04000 /* nSelectRow set by a constant LIMIT */ +#define SF_MaybeConvert 0x08000 /* Need convertCompoundSelectToSubquery() */ +#define SF_Converted 0x10000 /* By convertCompoundSelectToSubquery() */ +#define SF_IncludeHidden 0x20000 /* Include hidden columns in output */ /* @@ -11729,7 +15339,7 @@ struct Select { ** by one of the following macros. The "SRT" prefix means "SELECT Result ** Type". ** -** SRT_Union Store results as a key in a temporary index +** SRT_Union Store results as a key in a temporary index ** identified by pDest->iSDParm. ** ** SRT_Except Remove results from the temporary index pDest->iSDParm. @@ -11753,7 +15363,7 @@ struct Select { ** of the query. This destination implies "LIMIT 1". ** ** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iSDParm. +** row of result as the key in table pDest->iSDParm. ** Apply the affinity pDest->affSdst before storing ** results. Used to implement "IN (SELECT ...)". ** @@ -11813,7 +15423,7 @@ struct Select { */ struct SelectDest { u8 eDest; /* How to dispose of the results. On of SRT_* above. */ - char affSdst; /* Affinity used when eDest==SRT_Set */ + char *zAffSdst; /* Affinity used when eDest==SRT_Set */ int iSDParm; /* A parameter used by the eDest disposal method */ int iSdst; /* Base register where results are written */ int nSdst; /* Number of registers allocated */ @@ -11821,11 +15431,11 @@ struct SelectDest { }; /* -** During code generation of statements that do inserts into AUTOINCREMENT +** During code generation of statements that do inserts into AUTOINCREMENT ** tables, the following information is attached to the Table.u.autoInc.p ** pointer of each autoincrement table to record some side information that ** the code generator needs. We have to keep per-table autoincrement -** information in case inserts are down within triggers. Triggers do not +** information in case inserts are done within triggers. Triggers do not ** normally coordinate their activities, but we do need to coordinate the ** loading and saving of autoincrement information. */ @@ -11844,7 +15454,7 @@ struct AutoincInfo { #endif /* -** At least one instance of the following structure is created for each +** At least one instance of the following structure is created for each ** trigger that may be fired while parsing an INSERT, UPDATE or DELETE ** statement. All such objects are stored in the linked list headed at ** Parse.pTriggerPrg and deleted once statement compilation has been @@ -11857,7 +15467,7 @@ struct AutoincInfo { ** values for both pTrigger and orconf. ** ** The TriggerPrg.aColmask[0] variable is set to a mask of old.* columns -** accessed (or set to 0 for triggers fired as a result of INSERT +** accessed (or set to 0 for triggers fired as a result of INSERT ** statements). Similarly, the TriggerPrg.aColmask[1] variable is set to ** a mask of new.* columns used by the program. */ @@ -11898,7 +15508,7 @@ struct TriggerPrg { ** is constant but the second part is reset at the beginning and end of ** each recursion. ** -** The nTableLock and aTableLock variables are only used if the shared-cache +** The nTableLock and aTableLock variables are only used if the shared-cache ** feature is enabled (if sqlite3Tsd()->useSharedData is true). They are ** used to store the set of table-locks required by the statement being ** compiled. Function sqlite3TableLock() is used to add entries to the @@ -11917,22 +15527,54 @@ struct Parse { u8 mayAbort; /* True if statement may throw an ABORT exception */ u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ - int aTempReg[8]; /* Holding area for temporary registers */ + u8 disableLookaside; /* Number of times lookaside has been disabled */ + u8 nColCache; /* Number of entries in aColCache[] */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ int nTab; /* Number of previously allocated VDBE cursors */ int nMem; /* Number of memory cells used so far */ - int nSet; /* Number of sets used so far */ - int nOnce; /* Number of OP_Once instructions so far */ int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ - int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ + int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ int ckBase; /* Base register of data during check constraints */ - int iPartIdxTab; /* Table corresponding to a partial index */ + int iSelfTab; /* Table of an index whose exprs are being coded */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int nLabel; /* Number of labels used */ int *aLabel; /* Space to hold the labels */ + ExprList *pConstExpr;/* Constant expressions */ + Token constraintName;/* Name of the constraint currently being parsed */ + yDbMask writeMask; /* Start a write transaction on these databases */ + yDbMask cookieMask; /* Bitmask of schema verified databases */ + int regRowid; /* Register holding rowid of CREATE TABLE entry */ + int regRoot; /* Register holding root page number for new objects */ + int nMaxArg; /* Max args passed to user function by sub-program */ +#if SELECTTRACE_ENABLED + int nSelect; /* Number of SELECT statements seen */ + int nSelectIndent; /* How far to indent SELECTTRACE() output */ +#endif +#ifndef SQLITE_OMIT_SHARED_CACHE + int nTableLock; /* Number of locks in aTableLock */ + TableLock *aTableLock; /* Required table locks for shared-cache mode */ +#endif + AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ + Parse *pToplevel; /* Parse structure for main program (or NULL) */ + Table *pTriggerTab; /* Table triggers are being coded for */ + int addrCrTab; /* Address of OP_CreateTable opcode on CREATE TABLE */ + u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ + u32 oldmask; /* Mask of old.* columns referenced */ + u32 newmask; /* Mask of new.* columns referenced */ + u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ + u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ + u8 disableTriggers; /* True to disable triggers */ + + /************************************************************************** + ** Fields above must be initialized to zero. The fields that follow, + ** down to the beginning of the recursive section, do not need to be + ** initialized as they will be set before being used. The boundary is + ** determined by offsetof(Parse,aColCache). + **************************************************************************/ + struct yColCache { int iTable; /* Table cursor number */ i16 iColumn; /* Table column number */ @@ -11941,49 +15583,25 @@ struct Parse { int iReg; /* Reg with value of this column. 0 means none. */ int lru; /* Least recently used entry has the smallest value */ } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ - ExprList *pConstExpr;/* Constant expressions */ - Token constraintName;/* Name of the constraint currently being parsed */ - yDbMask writeMask; /* Start a write transaction on these databases */ - yDbMask cookieMask; /* Bitmask of schema verified databases */ - int cookieValue[SQLITE_MAX_ATTACHED+2]; /* Values of cookies to verify */ - int regRowid; /* Register holding rowid of CREATE TABLE entry */ - int regRoot; /* Register holding root page number for new objects */ - int nMaxArg; /* Max args passed to user function by sub-program */ -#ifndef SQLITE_OMIT_SHARED_CACHE - int nTableLock; /* Number of locks in aTableLock */ - TableLock *aTableLock; /* Required table locks for shared-cache mode */ -#endif - AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ - - /* Information used while coding trigger programs. */ - Parse *pToplevel; /* Parse structure for main program (or NULL) */ - Table *pTriggerTab; /* Table triggers are being coded for */ - int addrCrTab; /* Address of OP_CreateTable opcode on CREATE TABLE */ - int addrSkipPK; /* Address of instruction to skip PRIMARY KEY index */ - u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ - u32 oldmask; /* Mask of old.* columns referenced */ - u32 newmask; /* Mask of new.* columns referenced */ - u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ - u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ - u8 disableTriggers; /* True to disable triggers */ + int aTempReg[8]; /* Holding area for temporary registers */ + Token sNameToken; /* Token with unqualified schema object name */ /************************************************************************ ** Above is constant between recursions. Below is reset before and after ** each recursion. The boundary between these two regions is determined - ** using offsetof(Parse,nVar) so the nVar field must be the first field - ** in the recursive region. + ** using offsetof(Parse,sLastToken) so the sLastToken field must be the + ** first field in the recursive region. ************************************************************************/ - int nVar; /* Number of '?' variables seen in the SQL so far */ + Token sLastToken; /* The last token parsed */ + ynVar nVar; /* Number of '?' variables seen in the SQL so far */ int nzVar; /* Number of available slots in azVar[] */ u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ - u8 bFreeWith; /* True if pWith should be freed with parser */ u8 explain; /* True if the EXPLAIN flag is found on the query */ #ifndef SQLITE_OMIT_VIRTUALTABLE u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ int nVtabLock; /* Number of virtual tables to lock */ #endif - int nAlias; /* Number of aliased result set columns */ int nHeight; /* Expression tree height of current sub-select */ #ifndef SQLITE_OMIT_EXPLAIN int iSelectId; /* ID of current select for EXPLAIN output */ @@ -11995,8 +15613,6 @@ struct Parse { Table *pNewTable; /* A table being constructed by CREATE TABLE */ Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ - Token sNameToken; /* Token with unqualified schema object name */ - Token sLastToken; /* The last token parsed */ #ifndef SQLITE_OMIT_VIRTUALTABLE Token sArg; /* Complete text of a module argument */ Table **apVtabLock; /* Pointer to virtual tables needing locking */ @@ -12004,8 +15620,17 @@ struct Parse { Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ With *pWith; /* Current WITH clause, or NULL */ + With *pWithToFree; /* Free this WITH object at the end of the parse */ }; +/* +** Sizes and pointers of various parts of the Parse object. +*/ +#define PARSE_HDR_SZ offsetof(Parse,aColCache) /* Recursive part w/o aColCache*/ +#define PARSE_RECURSE_SZ offsetof(Parse,sLastToken) /* Recursive part */ +#define PARSE_TAIL_SZ (sizeof(Parse)-PARSE_RECURSE_SZ) /* Non-recursive part */ +#define PARSE_TAIL(X) (((char*)(X))+PARSE_RECURSE_SZ) /* Pointer to tail */ + /* ** Return true if currently inside an sqlite3_declare_vtab() call. */ @@ -12026,25 +15651,42 @@ struct AuthContext { /* ** Bitfield flags for P5 value in various opcodes. +** +** Value constraints (enforced via assert()): +** OPFLAG_LENGTHARG == SQLITE_FUNC_LENGTH +** OPFLAG_TYPEOFARG == SQLITE_FUNC_TYPEOF +** OPFLAG_BULKCSR == BTREE_BULKLOAD +** OPFLAG_SEEKEQ == BTREE_SEEK_EQ +** OPFLAG_FORDELETE == BTREE_FORDELETE +** OPFLAG_SAVEPOSITION == BTREE_SAVEPOSITION +** OPFLAG_AUXDELETE == BTREE_AUXDELETE */ -#define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ +#define OPFLAG_NCHANGE 0x01 /* OP_Insert: Set to update db->nChange */ + /* Also used in P2 (not P5) of OP_Delete */ +#define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ -#define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +#define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */ +#endif #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ -#define OPFLAG_P2ISREG 0x02 /* P2 to OP_Open** is a register number */ +#define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */ +#define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */ +#define OPFLAG_P2ISREG 0x10 /* P2 to OP_Open** is a register number */ #define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ +#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete: keep cursor position */ +#define OPFLAG_AUXDELETE 0x04 /* OP_Delete: index in a DELETE op */ /* * Each trigger present in the database schema is stored as an instance of - * struct Trigger. + * struct Trigger. * * Pointers to instances of struct Trigger are stored in two ways. - * 1. In the "trigHash" hash table (part of the sqlite3* that represents the + * 1. In the "trigHash" hash table (part of the sqlite3* that represents the * database). This allows Trigger structures to be retrieved by name. * 2. All triggers associated with a single table form a linked list, using the * pNext member of struct Trigger. A pointer to the first element of the @@ -12070,7 +15712,7 @@ struct Trigger { /* ** A trigger is either a BEFORE or an AFTER trigger. The following constants -** determine which. +** determine which. ** ** If there are multiple triggers, you might of some BEFORE and some AFTER. ** In that cases, the constants below can be ORed together. @@ -12080,48 +15722,48 @@ struct Trigger { /* * An instance of struct TriggerStep is used to store a single SQL statement - * that is a part of a trigger-program. + * that is a part of a trigger-program. * * Instances of struct TriggerStep are stored in a singly linked list (linked - * using the "pNext" member) referenced by the "step_list" member of the + * using the "pNext" member) referenced by the "step_list" member of the * associated struct Trigger instance. The first element of the linked list is * the first step of the trigger-program. - * + * * The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or - * "SELECT" statement. The meanings of the other members is determined by the + * "SELECT" statement. The meanings of the other members is determined by the * value of "op" as follows: * * (op == TK_INSERT) * orconf -> stores the ON CONFLICT algorithm * pSelect -> If this is an INSERT INTO ... SELECT ... statement, then * this stores a pointer to the SELECT statement. Otherwise NULL. - * target -> A token holding the quoted name of the table to insert into. + * zTarget -> Dequoted name of the table to insert into. * pExprList -> If this is an INSERT INTO ... VALUES ... statement, then * this stores values to be inserted. Otherwise NULL. - * pIdList -> If this is an INSERT INTO ... () VALUES ... + * pIdList -> If this is an INSERT INTO ... () VALUES ... * statement, then this stores the column-names to be * inserted into. * * (op == TK_DELETE) - * target -> A token holding the quoted name of the table to delete from. + * zTarget -> Dequoted name of the table to delete from. * pWhere -> The WHERE clause of the DELETE statement if one is specified. * Otherwise NULL. - * + * * (op == TK_UPDATE) - * target -> A token holding the quoted name of the table to update rows of. + * zTarget -> Dequoted name of the table to update. * pWhere -> The WHERE clause of the UPDATE statement if one is specified. * Otherwise NULL. * pExprList -> A list of the columns to update and the expressions to update * them to. See sqlite3Update() documentation of "pChanges" * argument. - * + * */ struct TriggerStep { u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT */ u8 orconf; /* OE_Rollback etc. */ Trigger *pTrig; /* The trigger that this step is a part of */ - Select *pSelect; /* SELECT statment or RHS of INSERT INTO .. SELECT ... */ - Token target; /* Target table for DELETE, UPDATE, INSERT */ + Select *pSelect; /* SELECT statement or RHS of INSERT INTO SELECT ... */ + char *zTarget; /* Target table for DELETE, UPDATE, INSERT */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ ExprList *pExprList; /* SET clause for UPDATE. */ IdList *pIdList; /* Column names for INSERT */ @@ -12132,7 +15774,7 @@ struct TriggerStep { /* ** The following structure contains information used by the sqliteFix... ** routines as they walk the parse tree to make database references -** explicit. +** explicit. */ typedef struct DbFixer DbFixer; struct DbFixer { @@ -12152,14 +15794,20 @@ struct StrAccum { sqlite3 *db; /* Optional database for lookaside. Can be NULL */ char *zBase; /* A base allocation. Not from malloc. */ char *zText; /* The string collected so far */ - int nChar; /* Length of the string so far */ - int nAlloc; /* Amount of space allocated in zText */ - int mxAlloc; /* Maximum allowed string length */ - u8 useMalloc; /* 0: none, 1: sqlite3DbMalloc, 2: sqlite3_malloc */ + u32 nChar; /* Length of the string so far */ + u32 nAlloc; /* Amount of space allocated in zText */ + u32 mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ u8 accError; /* STRACCUM_NOMEM or STRACCUM_TOOBIG */ + u8 printfFlags; /* SQLITE_PRINTF flags below */ }; #define STRACCUM_NOMEM 1 #define STRACCUM_TOOBIG 2 +#define SQLITE_PRINTF_INTERNAL 0x01 /* Internal-use-only converters allowed */ +#define SQLITE_PRINTF_SQLFUNC 0x02 /* SQL function arguments to VXPrintf */ +#define SQLITE_PRINTF_MALLOCED 0x04 /* True if xText is allocated space */ + +#define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) + /* ** A pointer to this structure is used to communicate information @@ -12187,6 +15835,7 @@ struct Sqlite3Config { int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ + int nStmtSpill; /* Stmt-journal spill-to-disk threshold */ sqlite3_mem_methods m; /* Low-level memory allocation interface */ sqlite3_mutex_methods mutex; /* Low-level mutex interface */ sqlite3_pcache_methods2 pcache2; /* Low-level page-cache interface */ @@ -12203,6 +15852,7 @@ struct Sqlite3Config { int nPage; /* Number of pages in pPage[] */ int mxParserStack; /* maximum depth of the parser stack */ int sharedCacheEnabled; /* true if shared-cache mode enabled */ + u32 szPma; /* Maximum Sorter PMA size */ /* The above might be initialized to non-zero. The following need to always ** initially be zero, however. */ int isInit; /* True after initialization has finished */ @@ -12229,6 +15879,7 @@ struct Sqlite3Config { int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ #endif int bLocaltimeFault; /* True to fail localtime() calls */ + int iOnceResetThreshold; /* When to reset OP_Once counters */ }; /* @@ -12253,16 +15904,21 @@ struct Sqlite3Config { ** Context pointer passed down through the tree-walk. */ struct Walker { + Parse *pParse; /* Parser context. */ int (*xExprCallback)(Walker*, Expr*); /* Callback for expressions */ int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ - Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 eCode; /* A small processing code */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ - int i; /* Integer value */ + int n; /* A counter */ + int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ + struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ + int *aiCol; /* array of column indexes */ + struct IdxCover *pIdxCover; /* Check for index coverage */ } u; }; @@ -12272,6 +15928,7 @@ SQLITE_PRIVATE int sqlite3WalkExprList(Walker*, ExprList*); SQLITE_PRIVATE int sqlite3WalkSelect(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker*, Select*); +SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker*, Expr*); /* ** Return code from the parse-tree walking primitives and their @@ -12292,10 +15949,21 @@ struct With { char *zName; /* Name of this CTE */ ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ - const char *zErr; /* Error message for circular references */ + const char *zCteErr; /* Error message for circular references */ } a[1]; }; +#ifdef SQLITE_DEBUG +/* +** An instance of the TreeView object is used for printing the content of +** data structures on sqlite3DebugPrintf() using a tree-like view. +*/ +struct TreeView { + int iLevel; /* Which level of the tree we are on */ + u8 bLine[100]; /* Draw vertical in column i if bLine[i] is true */ +}; +#endif /* SQLITE_DEBUG */ + /* ** Assuming zIn points to the first byte of a UTF-8 character, ** advance zIn to point to the first byte of the next UTF-8 character. @@ -12319,15 +15987,31 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); #define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__) #define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__) #define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__) +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NomemError(int); +SQLITE_PRIVATE int sqlite3IoerrnomemError(int); +# define SQLITE_NOMEM_BKPT sqlite3NomemError(__LINE__) +# define SQLITE_IOERR_NOMEM_BKPT sqlite3IoerrnomemError(__LINE__) +#else +# define SQLITE_NOMEM_BKPT SQLITE_NOMEM +# define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM +#endif +/* +** FTS3 and FTS4 both require virtual table support +*/ +#if defined(SQLITE_OMIT_VIRTUALTABLE) +# undef SQLITE_ENABLE_FTS3 +# undef SQLITE_ENABLE_FTS4 +#endif /* ** FTS4 is really an extension for FTS3. It is enabled using the -** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all -** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also call +** the SQLITE_ENABLE_FTS4 macro to serve as an alias for SQLITE_ENABLE_FTS3. */ #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) -# define SQLITE_ENABLE_FTS3 +# define SQLITE_ENABLE_FTS3 1 #endif /* @@ -12352,6 +16036,7 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); # define sqlite3Isdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x04) # define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) # define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) +# define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) #else # define sqlite3Toupper(x) toupper((unsigned char)(x)) # define sqlite3Isspace(x) isspace((unsigned char)(x)) @@ -12360,26 +16045,32 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); # define sqlite3Isdigit(x) isdigit((unsigned char)(x)) # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) +# define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') +#endif +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +SQLITE_PRIVATE int sqlite3IsIdChar(u8); #endif /* ** Internal function prototypes */ -#define sqlite3StrICmp sqlite3_stricmp +SQLITE_PRIVATE int sqlite3StrICmp(const char*,const char*); SQLITE_PRIVATE int sqlite3Strlen30(const char*); +SQLITE_PRIVATE char *sqlite3ColumnType(Column*,char*); #define sqlite3StrNICmp sqlite3_strnicmp SQLITE_PRIVATE int sqlite3MallocInit(void); SQLITE_PRIVATE void sqlite3MallocEnd(void); -SQLITE_PRIVATE void *sqlite3Malloc(int); -SQLITE_PRIVATE void *sqlite3MallocZero(int); -SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3*, int); -SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3*, int); +SQLITE_PRIVATE void *sqlite3Malloc(u64); +SQLITE_PRIVATE void *sqlite3MallocZero(u64); +SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3*, u64); +SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3*, u64); +SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3*, u64); SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3*,const char*); -SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3*,const char*, int); -SQLITE_PRIVATE void *sqlite3Realloc(void*, int); -SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *, void *, int); -SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *, void *, int); +SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3*,const char*, u64); +SQLITE_PRIVATE void *sqlite3Realloc(void*, u64); +SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64); +SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *, void *, u64); SQLITE_PRIVATE void sqlite3DbFree(sqlite3*, void*); SQLITE_PRIVATE int sqlite3MallocSize(void*); SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3*, void*); @@ -12388,7 +16079,9 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void*); SQLITE_PRIVATE void *sqlite3PageMalloc(int); SQLITE_PRIVATE void sqlite3PageFree(void*); SQLITE_PRIVATE void sqlite3MemSetDefault(void); +#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); +#endif SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); /* @@ -12402,18 +16095,22 @@ SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); #ifdef SQLITE_USE_ALLOCA # define sqlite3StackAllocRaw(D,N) alloca(N) # define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) -# define sqlite3StackFree(D,P) +# define sqlite3StackFree(D,P) #else # define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) # define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) # define sqlite3StackFree(D,P) sqlite3DbFree(D,P) #endif -#ifdef SQLITE_ENABLE_MEMSYS3 -SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); -#endif +/* Do not allow both MEMSYS5 and MEMSYS3 to be defined together. If they +** are, disable MEMSYS3 +*/ #ifdef SQLITE_ENABLE_MEMSYS5 SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void); +#undef SQLITE_ENABLE_MEMSYS3 +#endif +#ifdef SQLITE_ENABLE_MEMSYS3 +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); #endif @@ -12424,10 +16121,20 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int); SQLITE_PRIVATE int sqlite3MutexInit(void); SQLITE_PRIVATE int sqlite3MutexEnd(void); #endif +#if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP) +SQLITE_PRIVATE void sqlite3MemoryBarrier(void); +#else +# define sqlite3MemoryBarrier() +#endif -SQLITE_PRIVATE int sqlite3StatusValue(int); -SQLITE_PRIVATE void sqlite3StatusAdd(int, int); -SQLITE_PRIVATE void sqlite3StatusSet(int, int); +SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int); +SQLITE_PRIVATE void sqlite3StatusUp(int, int); +SQLITE_PRIVATE void sqlite3StatusDown(int, int); +SQLITE_PRIVATE void sqlite3StatusHighwater(int, int); + +/* Access to mutexes used by sqlite3_status() */ +SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void); +SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void); #ifndef SQLITE_OMIT_FLOATING_POINT SQLITE_PRIVATE int sqlite3IsNaN(double); @@ -12445,45 +16152,30 @@ struct PrintfArguments { sqlite3_value **apArg; /* The argument values */ }; -#define SQLITE_PRINTF_INTERNAL 0x01 -#define SQLITE_PRINTF_SQLFUNC 0x02 -SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, u32, const char*, va_list); -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, u32, const char*, ...); +SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, const char*, va_list); +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, const char*, ...); SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); -SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3*,char*,const char*,...); -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) SQLITE_PRIVATE void sqlite3DebugPrintf(const char*, ...); #endif #if defined(SQLITE_TEST) SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); #endif -/* Output formatting for SQLITE_TESTCTRL_EXPLAIN */ -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) -SQLITE_PRIVATE void sqlite3ExplainBegin(Vdbe*); -SQLITE_PRIVATE void sqlite3ExplainPrintf(Vdbe*, const char*, ...); -SQLITE_PRIVATE void sqlite3ExplainNL(Vdbe*); -SQLITE_PRIVATE void sqlite3ExplainPush(Vdbe*); -SQLITE_PRIVATE void sqlite3ExplainPop(Vdbe*); -SQLITE_PRIVATE void sqlite3ExplainFinish(Vdbe*); -SQLITE_PRIVATE void sqlite3ExplainSelect(Vdbe*, Select*); -SQLITE_PRIVATE void sqlite3ExplainExpr(Vdbe*, Expr*); -SQLITE_PRIVATE void sqlite3ExplainExprList(Vdbe*, ExprList*); -SQLITE_PRIVATE const char *sqlite3VdbeExplanation(Vdbe*); -#else -# define sqlite3ExplainBegin(X) -# define sqlite3ExplainSelect(A,B) -# define sqlite3ExplainExpr(A,B) -# define sqlite3ExplainExprList(A,B) -# define sqlite3ExplainFinish(X) -# define sqlite3VdbeExplanation(X) 0 +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); +SQLITE_PRIVATE void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, const char*); +SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); +SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); +SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); #endif -SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*, ...); +SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); -SQLITE_PRIVATE int sqlite3Dequote(char*); +SQLITE_PRIVATE void sqlite3Dequote(char*); +SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*, char **); SQLITE_PRIVATE void sqlite3FinishCoding(Parse*); @@ -12492,43 +16184,55 @@ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int); SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int); SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int); SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int); +#endif SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*); +SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); -SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); +SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int); SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); +SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList*); SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); SQLITE_PRIVATE int sqlite3InitCallback(void*, int, char**, char**); SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3*); SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int); SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); -SQLITE_PRIVATE void sqlite3BeginParse(Parse*,int); SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); +SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); +SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*); SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index*, i16); SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); -SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*); +#if SQLITE_ENABLE_HIDDEN_COLUMNS +SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table*, Column*); +#else +# define sqlite3ColumnPropertiesFromName(T,C) /* no-op */ +#endif +SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*,Token*); SQLITE_PRIVATE void sqlite3AddNotNull(Parse*, int); SQLITE_PRIVATE void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int); SQLITE_PRIVATE void sqlite3AddCheckConstraint(Parse*, Expr*); -SQLITE_PRIVATE void sqlite3AddColumnType(Parse*,Token*); SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,ExprSpan*); SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*); SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, sqlite3_vfs**,char**,char **); SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); -SQLITE_PRIVATE int sqlite3CodeOnce(Parse *); #ifdef SQLITE_OMIT_BUILTIN_TEST # define sqlite3FaultSim(X) SQLITE_OK @@ -12538,11 +16242,14 @@ SQLITE_PRIVATE int sqlite3FaultSim(int); SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32); SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec*, u32); +SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec*, u32); SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec*, u32); SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec*, u32, void*); SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec*); SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec*); +#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); +#endif SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); SQLITE_PRIVATE void sqlite3RowSetClear(RowSet*); @@ -12550,7 +16257,7 @@ SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); -SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); +SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int); #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); @@ -12580,18 +16287,19 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*) SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, Expr*, IdList*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse*, SrcList*, ExprList*); SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, struct SrcList_item *); SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList*); SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3*, IdList*); SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3*, SrcList*); SQLITE_PRIVATE Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**); -SQLITE_PRIVATE Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, - Expr*, int, int); +SQLITE_PRIVATE void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, + Expr*, int, int, u8); SQLITE_PRIVATE void sqlite3DropIndex(Parse*, SrcList*, int); SQLITE_PRIVATE int sqlite3Select(Parse*, Select*, SelectDest*); SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, - Expr*,ExprList*,u16,Expr*,Expr*); + Expr*,ExprList*,u32,Expr*,Expr*); SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); @@ -12603,14 +16311,20 @@ SQLITE_PRIVATE void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); SQLITE_PRIVATE void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); -SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo*); +SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereOrderedInnerLoop(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); +#define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ +#define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ +#define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); +SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg(Parse*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse*, int, int, int); @@ -12620,34 +16334,42 @@ SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeAtInit(Parse*, Expr*, int, u8); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); -SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, u8); +SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ +#define SQLITE_ECEL_REF 0x04 /* Use ExprList.u.x.iOrderByCol */ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse*, Expr*, int, int); SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse*, Expr*, int, int); +SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int); SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3*,const char*, const char*); -SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*); -SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *); +#define LOCATE_VIEW 0x01 +#define LOCATE_NOERR 0x02 +SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*); +SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *); SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3*,const char*, const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); -SQLITE_PRIVATE void sqlite3Vacuum(Parse*); -SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*); +SQLITE_PRIVATE void sqlite3Vacuum(Parse*,Token*); +SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*, int); SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, Token*); SQLITE_PRIVATE int sqlite3ExprCompare(Expr*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList*, ExprList*, int); SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Expr*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); +SQLITE_PRIVATE int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse*); +#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE void sqlite3PrngSaveState(void); SQLITE_PRIVATE void sqlite3PrngRestoreState(void); +#endif SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*,int); SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int); SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); @@ -12659,19 +16381,24 @@ SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *); SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); +SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); +#endif SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); -SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); -SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); +SQLITE_PRIVATE void sqlite3GenerateRowDelete( + Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, - u8,u8,int,int*); + u8,u8,int,int*,int*); SQLITE_PRIVATE void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); -SQLITE_PRIVATE int sqlite3OpenTableAndIndices(Parse*, Table*, int, int, u8*, int*, int*); +SQLITE_PRIVATE int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*); SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse*, int, int); SQLITE_PRIVATE void sqlite3MultiWrite(Parse*); SQLITE_PRIVATE void sqlite3MayAbort(Parse*); @@ -12683,11 +16410,16 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3*,IdList*); SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,Select*,int); -SQLITE_PRIVATE void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*); -SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8); -SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3*); +#if SELECTTRACE_ENABLED +SQLITE_PRIVATE void sqlite3SelectSetName(Select*,const char*); +#else +# define sqlite3SelectSetName(A,B) +#endif +SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); +SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void); SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void); -SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void); +SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3*); SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); @@ -12718,6 +16450,7 @@ SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) +# define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else # define sqlite3TriggersExist(B,C,D,E,F) 0 # define sqlite3DeleteTrigger(A,B) @@ -12727,6 +16460,7 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Tab # define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F) # define sqlite3TriggerList(X, Y) 0 # define sqlite3ParseToplevel(p) p +# define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 #endif @@ -12764,60 +16498,52 @@ SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst,LogEst); #ifndef SQLITE_OMIT_VIRTUALTABLE SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ + defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); +#endif /* ** Routines to read and write variable-length integers. These used to ** be defined locally, but now we use the varint routines in the util.c -** file. Code should use the MACRO forms below, as the Varint32 versions -** are coded to assume the single byte case is already handled (which -** the MACRO form does). +** file. */ SQLITE_PRIVATE int sqlite3PutVarint(unsigned char*, u64); -SQLITE_PRIVATE int sqlite3PutVarint32(unsigned char*, u32); SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *, u64 *); SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *, u32 *); SQLITE_PRIVATE int sqlite3VarintLen(u64 v); /* -** The header of a record consists of a sequence variable-length integers. -** These integers are almost always small and are encoded as a single byte. -** The following macros take advantage this fact to provide a fast encode -** and decode of the integers in a record header. It is faster for the common -** case where the integer is a single byte. It is a little slower when the -** integer is two or more bytes. But overall it is faster. -** -** The following expressions are equivalent: -** -** x = sqlite3GetVarint32( A, &B ); -** x = sqlite3PutVarint32( A, B ); -** -** x = getVarint32( A, B ); -** x = putVarint32( A, B ); -** +** The common case is for a varint to be a single byte. They following +** macros handle the common case without a procedure call, but then call +** the procedure for larger varints. */ #define getVarint32(A,B) \ (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) #define putVarint32(A,B) \ (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ - sqlite3PutVarint32((A),(B))) + sqlite3PutVarint((A),(B))) #define getVarint sqlite3GetVarint #define putVarint sqlite3PutVarint -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *, Index *); +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); +SQLITE_PRIVATE char sqlite3TableColumnAffinity(Table*,int); SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); -SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...); +SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); +SQLITE_PRIVATE void sqlite3Error(sqlite3*,int); +SQLITE_PRIVATE void sqlite3SystemError(sqlite3*,int); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); -#if defined(SQLITE_TEST) +#if defined(SQLITE_NEED_ERR_NAME) SQLITE_PRIVATE const char *sqlite3ErrName(int); #endif @@ -12826,7 +16552,7 @@ SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*, int); SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); @@ -12845,7 +16571,7 @@ SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z,u8); SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); -SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, +SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); @@ -12855,11 +16581,12 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); #ifndef SQLITE_AMALGAMATION SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; +SQLITE_PRIVATE const char sqlite3StrBINARY[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; SQLITE_PRIVATE const Token sqlite3IntTokens[]; SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; -SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; +SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; #ifndef SQLITE_OMIT_WSD SQLITE_PRIVATE int sqlite3PendingByte; #endif @@ -12871,10 +16598,12 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*); -SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *, Expr *, int, int); +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr *, int, int); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); +SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); +SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); @@ -12892,7 +16621,6 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3*,Index*); SQLITE_PRIVATE void sqlite3DefaultRowEst(Index*); SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3*, int); SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*); -SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse*, int, int); SQLITE_PRIVATE void sqlite3SchemaClear(void *); SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *, Btree *); SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *); @@ -12903,18 +16631,20 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); #endif -SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, +SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), FuncDestructor *pDestructor ); +SQLITE_PRIVATE void sqlite3OomFault(sqlite3*); +SQLITE_PRIVATE void sqlite3OomClear(sqlite3*); SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); -SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, char*, int, int); +SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum*,const char*,int); SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum*,const char*); -SQLITE_PRIVATE void sqlite3AppendSpace(StrAccum*,int); +SQLITE_PRIVATE void sqlite3AppendChar(StrAccum*,int,char); SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum*); SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest*,int,int); @@ -12923,18 +16653,26 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *); SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse*, Expr*); +#else +# define sqlite3ExprCheckIN(x,y) SQLITE_OK +#endif + #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void); -SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); +SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( + Parse*,Index*,UnpackedRecord**,Expr*,int,int,int*); SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord*); SQLITE_PRIVATE int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**); +SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3*, Index*, int); #endif /* ** The interface to the LEMON-generated parser */ -SQLITE_PRIVATE void *sqlite3ParserAlloc(void*(*)(size_t)); +SQLITE_PRIVATE void *sqlite3ParserAlloc(void*(*)(u64)); SQLITE_PRIVATE void sqlite3ParserFree(void*, void(*)(void*)); SQLITE_PRIVATE void sqlite3Parser(void*, int, Token, Parse*); #ifdef YYTRACKMAXSTACKDEPTH @@ -12964,7 +16702,7 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); # define sqlite3VtabRollback(X) # define sqlite3VtabCommit(X) # define sqlite3VtabInSync(db) 0 -# define sqlite3VtabLock(X) +# define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) # define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK @@ -12983,6 +16721,8 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int); SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*); @@ -13020,7 +16760,7 @@ SQLITE_PRIVATE void sqlite3WithPush(Parse*, With*, u8); ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign ** key functionality is available. If OMIT_TRIGGER is defined but ** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In -** this case foreign keys are parsed, but no other functionality is +** this case foreign keys are parsed, but no other functionality is ** provided (enforcement of FK constraints requires the triggers sub-system). */ #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) @@ -13079,28 +16819,22 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void); #define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ #define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ -SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*); +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*); +SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); #ifdef SQLITE_ENABLE_ATOMIC_WRITE -SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); -SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *); -SQLITE_PRIVATE int sqlite3JournalExists(sqlite3_file *p); -#else - #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile) - #define sqlite3JournalExists(p) 1 #endif +SQLITE_PRIVATE int sqlite3JournalIsInMemory(sqlite3_file *p); SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *); -SQLITE_PRIVATE int sqlite3MemJournalSize(void); -SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *); +SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p); #if SQLITE_MAX_EXPR_DEPTH>0 -SQLITE_PRIVATE void sqlite3ExprSetHeight(Parse *pParse, Expr *p); SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *); SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse*, int); #else - #define sqlite3ExprSetHeight(x,y) #define sqlite3SelectExprHeight(x) 0 #define sqlite3ExprCheckHeight(x,y) #endif @@ -13125,12 +16859,12 @@ SQLITE_PRIVATE void sqlite3ParserTrace(FILE*, char *); /* ** If the SQLITE_ENABLE IOTRACE exists then the global variable ** sqlite3IoTrace is a pointer to a printf-like routine used to -** print I/O tracing messages. +** print I/O tracing messages. */ #ifdef SQLITE_ENABLE_IOTRACE # define IOTRACE(A) if( sqlite3IoTrace ){ sqlite3IoTrace A; } SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe*); -SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*,...); +SQLITE_API SQLITE_EXTERN void (SQLITE_CDECL *sqlite3IoTrace)(const char*,...); #else # define IOTRACE(A) # define sqlite3VdbeIOTraceSql(X) @@ -13159,7 +16893,7 @@ SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*,...); ** that allocations that might have been satisfied by lookaside are not ** passed back to non-lookaside free() routines. Asserts such as the ** example above are placed on the non-lookaside free() routines to verify -** this constraint. +** this constraint. ** ** All of this is no-op for a production build. It only comes into ** play when the SQLITE_MEMDEBUG compile-time option is used. @@ -13174,12 +16908,28 @@ SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); # define sqlite3MemdebugNoType(X,Y) 1 #endif #define MEMTYPE_HEAP 0x01 /* General heap allocations */ -#define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ +#define MEMTYPE_LOOKASIDE 0x02 /* Heap that might have been lookaside */ #define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ #define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ -#define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ -#endif /* _SQLITEINT_H_ */ +/* +** Threading interface +*/ +#if SQLITE_MAX_WORKER_THREADS>0 +SQLITE_PRIVATE int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); +SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread*, void**); +#endif + +#if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST) +SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3*); +#endif + +SQLITE_PRIVATE int sqlite3ExprVectorSize(Expr *pExpr); +SQLITE_PRIVATE int sqlite3ExprIsVector(Expr *pExpr); +SQLITE_PRIVATE Expr *sqlite3VectorFieldSubexpr(Expr*, int); +SQLITE_PRIVATE Expr *sqlite3ExprForVectorField(Parse*,Expr*,int); + +#endif /* SQLITEINT_H */ /************** End of sqliteInt.h *******************************************/ /************** Begin file crypto.c ******************************************/ @@ -13217,6 +16967,7 @@ SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); #ifdef SQLITE_HAS_CODEC /* #include */ +/* #include "sqliteInt.h" */ /************** Include btreeInt.h in the middle of crypto.c *****************/ /************** Begin file btreeInt.h ****************************************/ /* @@ -13230,7 +16981,7 @@ SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file implements a external (disk-based) database using BTrees. +** This file implements an external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to ** ** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3: @@ -13356,7 +17107,7 @@ SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); ** ** The flags define the format of this btree page. The leaf flag means that ** this page has no children. The zerodata flag means that this page carries -** only keys and no data. The intkey flag means that the key is a integer +** only keys and no data. The intkey flag means that the key is an integer ** which is stored in the key size entry of the cell header rather than in ** the payload area. ** @@ -13434,6 +17185,7 @@ SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); ** 4 Number of leaf pointers on this page ** * zero or more pages numbers of leaves */ +/* #include "sqliteInt.h" */ /* The following value is the maximum cell size assuming a maximum page @@ -13451,6 +17203,7 @@ SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); /* Forward declarations */ typedef struct MemPage MemPage; typedef struct BtLock BtLock; +typedef struct CellInfo CellInfo; /* ** This is a magic string that appears at the beginning of every @@ -13493,12 +17246,13 @@ typedef struct BtLock BtLock; struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ - u8 intKey; /* True if intkey flag is set */ - u8 leaf; /* True if leaf flag is set */ - u8 hasData; /* True if this page stores data */ + u8 intKey; /* True if table b-trees. False for index b-trees */ + u8 intKeyLeaf; /* True if the leaf of an intKey table */ + u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ u8 max1bytePayload; /* min(maxLocal,127) */ + u8 bBusy; /* Prevent endless loops on corrupt database files */ u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ u16 cellOffset; /* Index in aData of first cell pointer */ @@ -13512,7 +17266,10 @@ struct MemPage { u8 *aData; /* Pointer to disk image of the page data */ u8 *aDataEnd; /* One byte past the end of usable data */ u8 *aCellIdx; /* The cell index area */ + u8 *aDataOfst; /* Same as aData for leaves. aData+4 for interior */ DbPage *pDbPage; /* Pager page handle */ + u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ + void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ Pgno pgno; /* Page number for this page */ }; @@ -13568,8 +17325,10 @@ struct Btree { u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ u8 sharable; /* True if we can share pBt with another db */ u8 locked; /* True if db currently has pBt locked */ + u8 hasIncrblobCur; /* True if there are one or more Incrblob cursors */ int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ int nBackup; /* Number of backup operations reading this btree */ + u32 iDataVersion; /* Combines with pBt->pPager->iDataVersion */ Btree *pNext; /* List of other sharable Btrees from the same db */ Btree *pPrev; /* Back pointer of the same list */ #ifndef SQLITE_OMIT_SHARED_CACHE @@ -13636,6 +17395,9 @@ struct BtShared { #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ +#ifdef SQLITE_HAS_CODEC + u8 optimalReserve; /* Desired amount of reserved space per page */ +#endif u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ @@ -13655,7 +17417,7 @@ struct BtShared { BtLock *pLock; /* List of locks held on this shared-btree struct */ Btree *pWriter; /* Btree with currently open write transaction */ #endif - u8 *pTmpSpace; /* BtShared.pageSize bytes of space for tmp use */ + u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ }; /* @@ -13674,15 +17436,11 @@ struct BtShared { ** about a cell. The parseCellPtr() function fills in this structure ** based on information extract from the raw disk page. */ -typedef struct CellInfo CellInfo; struct CellInfo { - i64 nKey; /* The key for INTKEY tables, or number of bytes in key */ - u8 *pCell; /* Pointer to the start of cell content */ - u32 nData; /* Number of bytes of data */ - u32 nPayload; /* Total amount of payload */ - u16 nHeader; /* Size of the cell content header in bytes */ - u16 nLocal; /* Amount of payload held locally */ - u16 iOverflow; /* Offset to overflow page number. Zero if no overflow */ + i64 nKey; /* The key for INTKEY tables, or nPayload otherwise */ + u8 *pPayload; /* Pointer to the start of payload */ + u32 nPayload; /* Bytes of payload */ + u16 nLocal; /* Amount of payload held locally, not on overflow */ u16 nSize; /* Size of the cell content on the main b-tree page */ }; @@ -13710,23 +17468,35 @@ struct CellInfo { ** ** Fields in this structure are accessed under the BtShared.mutex ** found at self->pBt->mutex. +** +** skipNext meaning: +** eState==SKIPNEXT && skipNext>0: Next sqlite3BtreeNext() is no-op. +** eState==SKIPNEXT && skipNext<0: Next sqlite3BtreePrevious() is no-op. +** eState==FAULT: Cursor fault with skipNext as error code. */ struct BtCursor { Btree *pBtree; /* The Btree to which this cursor belongs */ BtShared *pBt; /* The BtShared this cursor points to */ - BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ - struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ + BtCursor *pNext; /* Forms a linked list of all cursors */ Pgno *aOverflow; /* Cache of overflow page locations */ CellInfo info; /* A parse of the cell we are pointing at */ i64 nKey; /* Size of pKey, or last integer key */ void *pKey; /* Saved key that was cursor last known position */ Pgno pgnoRoot; /* The root page of this tree */ int nOvflAlloc; /* Allocated size of aOverflow[] array */ - int skipNext; /* Prev() is noop if negative. Next() is noop if positive */ + int skipNext; /* Prev() is noop if negative. Next() is noop if positive. + ** Error code if eState==CURSOR_FAULT */ u8 curFlags; /* zero or more BTCF_* flags defined below */ + u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ - u8 hints; /* As configured by CursorSetHints() */ - i16 iPage; /* Index of current page in apPage */ + u8 hints; /* As configured by CursorSetHints() */ + /* All fields above are zeroed when the cursor is allocated. See + ** sqlite3BtreeCursorZero(). Fields that follow must be manually + ** initialized. */ + i8 iPage; /* Index of current page in apPage */ + u8 curIntKey; /* Value of apPage[0]->intKey */ + struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ + void *padding1; /* Make object size a multiple of 16 */ u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ }; @@ -13739,6 +17509,7 @@ struct BtCursor { #define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ #define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ +#define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ /* ** Potential values for BtCursor.eState. @@ -13764,11 +17535,11 @@ struct BtCursor { ** seek the cursor to the saved position. ** ** CURSOR_FAULT: -** A unrecoverable error (an I/O error or a malloc failure) has occurred +** An unrecoverable error (an I/O error or a malloc failure) has occurred ** on a different connection that shares the BtShared cache with this ** cursor. The error has left the cache in an inconsistent state. ** Do nothing else with this cursor. Any attempt to use the cursor -** should return the error code stored in BtCursor.skip +** should return the error code stored in BtCursor.skipNext */ #define CURSOR_INVALID 0 #define CURSOR_VALID 1 @@ -13878,7 +17649,10 @@ struct IntegrityCk { int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ int mallocFailed; /* A memory allocation error has occurred */ + const char *zPfx; /* Error message prefix */ + int v1, v2; /* Values for up to two %d fields in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ + u32 *heap; /* Min-heap used for analyzing cell coverage */ }; /* @@ -13889,6 +17663,23 @@ struct IntegrityCk { #define get4byte sqlite3Get4byte #define put4byte sqlite3Put4byte +/* +** get2byteAligned(), unlike get2byte(), requires that its argument point to a +** two-byte aligned address. get2bytea() is only used for accessing the +** cell addresses in a btree header. +*/ +#if SQLITE_BYTEORDER==4321 +# define get2byteAligned(x) (*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && GCC_VERSION>=4008000 +# define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 +# define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) +#else +# define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) +#endif + /************** End of btreeInt.h ********************************************/ /************** Continuing where we left off in crypto.c *********************/ /************** Include crypto.h in the middle of crypto.c *******************/ @@ -13939,7 +17730,11 @@ struct IntegrityCk { #define FILE_HEADER_SZ 16 #ifndef CIPHER_VERSION -#define CIPHER_VERSION "3.2.0" +#ifdef SQLCIPHER_FIPS +#define CIPHER_VERSION "3.4.1 FIPS" +#else +#define CIPHER_VERSION "3.4.1" +#endif #endif #ifndef CIPHER @@ -14015,17 +17810,17 @@ struct IntegrityCk { #endif /* extensions defined in pager.c */ -SQLITE_PRIVATE void sqlite3pager_get_codec(Pager *pPager, void **ctx); -SQLITE_PRIVATE int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno); +SQLITE_API void sqlite3pager_get_codec(Pager *pPager, void **ctx); +SQLITE_API int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno); SQLITE_PRIVATE sqlite3_file *sqlite3Pager_get_fd(Pager *pPager); -SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetCodec( +SQLITE_API void sqlite3pager_sqlite3PagerSetCodec( Pager *pPager, void *(*xCodec)(void*,void*,Pgno,int), void (*xCodecSizeChng)(void*,int,int), void (*xCodecFree)(void*), void *pCodec ); -SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetError(Pager *pPager, int error); +SQLITE_API void sqlite3pager_sqlite3PagerSetError(Pager *pPager, int error); /* end extensions defined in pager.c */ /* @@ -14051,6 +17846,19 @@ static void cipher_bin2hex(const unsigned char* in, int sz, char *out) { } } +static int cipher_isHex(const unsigned char *hex, int sz){ + int i; + for(i = 0; i < sz; i++) { + unsigned char c = hex[i]; + if ((c < '0' || c > '9') && + (c < 'A' || c > 'F') && + (c < 'a' || c > 'f')) { + return 0; + } + } + return 1; +} + /* extensions defined in crypto_impl.c */ typedef struct codec_ctx codec_ctx; @@ -14075,6 +17883,9 @@ int sqlcipher_codec_ctx_set_pagesize(codec_ctx *, int); int sqlcipher_codec_ctx_get_pagesize(codec_ctx *); int sqlcipher_codec_ctx_get_reservesize(codec_ctx *); +void sqlcipher_set_default_pagesize(int page_size); +int sqlcipher_get_default_pagesize(); + void sqlcipher_set_default_kdf_iter(int iter); int sqlcipher_get_default_kdf_iter(); @@ -14114,7 +17925,8 @@ static void sqlcipher_profile_callback(void *file, const char *sql, sqlite3_uint static int sqlcipher_codec_get_store_pass(codec_ctx *ctx); static void sqlcipher_codec_get_pass(codec_ctx *ctx, void **zKey, int *nKey); static void sqlcipher_codec_set_store_pass(codec_ctx *ctx, int value); - +int sqlcipher_codec_fips_status(codec_ctx *ctx); +const char* sqlcipher_codec_get_provider_version(codec_ctx *ctx); #endif #endif /* END SQLCIPHER */ @@ -14165,6 +17977,7 @@ static int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, } int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLeft, const char *zRight) { + char *pragma_cipher_deprecated_msg = "PRAGMA cipher command is deprecated, please remove from usage."; struct Db *pDb = &db->aDb[iDb]; codec_ctx *ctx = NULL; int rc; @@ -14175,13 +17988,24 @@ int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLef CODEC_TRACE(("sqlcipher_codec_pragma: entered db=%p iDb=%d pParse=%p zLeft=%s zRight=%s ctx=%p\n", db, iDb, pParse, zLeft, zRight, ctx)); + if( sqlite3StrICmp(zLeft, "cipher_fips_status")== 0 && !zRight ){ + if(ctx) { + char *fips_mode_status = sqlite3_mprintf("%d", sqlcipher_codec_fips_status(ctx)); + codec_vdbe_return_static_string(pParse, "cipher_fips_status", fips_mode_status); + sqlite3_free(fips_mode_status); + } + } else if( sqlite3StrICmp(zLeft, "cipher_store_pass")==0 && zRight ) { - sqlcipher_codec_set_store_pass(ctx, sqlite3GetBoolean(zRight, 1)); + if(ctx) { + sqlcipher_codec_set_store_pass(ctx, sqlite3GetBoolean(zRight, 1)); + } } else if( sqlite3StrICmp(zLeft, "cipher_store_pass")==0 && !zRight ) { - char *store_pass_value = sqlite3_mprintf("%d", sqlcipher_codec_get_store_pass(ctx)); - codec_vdbe_return_static_string(pParse, "cipher_store_pass", store_pass_value); - sqlite3_free(store_pass_value); + if(ctx){ + char *store_pass_value = sqlite3_mprintf("%d", sqlcipher_codec_get_store_pass(ctx)); + codec_vdbe_return_static_string(pParse, "cipher_store_pass", store_pass_value); + sqlite3_free(store_pass_value); + } } if( sqlite3StrICmp(zLeft, "cipher_profile")== 0 && zRight ){ char *profile_status = sqlite3_mprintf("%d", sqlcipher_cipher_profile(db, zRight)); @@ -14207,13 +18031,21 @@ int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLef sqlcipher_codec_get_cipher_provider(ctx)); } } else + if( sqlite3StrICmp(zLeft, "cipher_provider_version")==0 && !zRight){ + if(ctx) { codec_vdbe_return_static_string(pParse, "cipher_provider_version", + sqlcipher_codec_get_provider_version(ctx)); + } + } else if( sqlite3StrICmp(zLeft, "cipher_version")==0 && !zRight ){ codec_vdbe_return_static_string(pParse, "cipher_version", codec_get_cipher_version()); }else if( sqlite3StrICmp(zLeft, "cipher")==0 ){ if(ctx) { if( zRight ) { - sqlcipher_codec_ctx_set_cipher(ctx, zRight, 2); // change cipher for both + rc = sqlcipher_codec_ctx_set_cipher(ctx, zRight, 2); // change cipher for both + codec_vdbe_return_static_string(pParse, "cipher", pragma_cipher_deprecated_msg); + sqlite3_log(SQLITE_WARNING, pragma_cipher_deprecated_msg); + return rc; }else { codec_vdbe_return_static_string(pParse, "cipher", sqlcipher_codec_ctx_get_cipher(ctx, 2)); @@ -14272,6 +18104,15 @@ int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLef } } }else + if( sqlite3StrICmp(zLeft,"cipher_default_page_size")==0 ){ + if( zRight ) { + sqlcipher_set_default_pagesize(atoi(zRight)); + } else { + char *default_page_size = sqlite3_mprintf("%d", sqlcipher_get_default_pagesize()); + codec_vdbe_return_static_string(pParse, "cipher_default_page_size", default_page_size); + sqlite3_free(default_page_size); + } + }else if( sqlite3StrICmp(zLeft,"cipher_default_use_hmac")==0 ){ if( zRight ) { sqlcipher_set_default_use_hmac(sqlite3GetBoolean(zRight,1)); @@ -14422,7 +18263,11 @@ SQLITE_PRIVATE int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, in /* point the internal codec argument against the contet to be prepared */ rc = sqlcipher_codec_ctx_init(&ctx, pDb, pDb->pBt->pBt->pPager, fd, zKey, nKey); - if(rc != SQLITE_OK) return rc; /* initialization failed, do not attach potentially corrupted context */ + if(rc != SQLITE_OK) { + /* initialization failed, do not attach potentially corrupted context */ + sqlite3_mutex_leave(db->mutex); + return rc; + } sqlite3pager_sqlite3PagerSetCodec(sqlite3BtreePager(pDb->pBt), sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); @@ -14455,7 +18300,7 @@ static int sqlcipher_find_db_index(sqlite3 *db, const char *zDb) { } for(db_index = 0; db_index < db->nDb; db_index++) { struct Db *pDb = &db->aDb[db_index]; - if(strcmp(pDb->zName, zDb) == 0) { + if(strcmp(pDb->zDbSName, zDb) == 0) { return db_index; } } @@ -14527,7 +18372,7 @@ SQLITE_API int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, sqlite3PagerPagecount(pPager, &page_count); for(pgno = 1; rc == SQLITE_OK && pgno <= (unsigned int)page_count; pgno++) { /* pgno's start at 1 see pager.c:pagerAcquire */ if(!sqlite3pager_is_mj_pgno(pPager, pgno)) { /* skip this page (see pager.c:pagerAcquire for reasoning) */ - rc = sqlite3PagerGet(pPager, pgno, &page); + rc = sqlite3PagerGet(pPager, pgno, &page, 0); if(rc == SQLITE_OK) { /* write page see pager_incr_changecounter for example */ rc = sqlite3PagerWrite(page); if(rc == SQLITE_OK) { @@ -14548,7 +18393,7 @@ SQLITE_API int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, sqlcipher_codec_key_copy(ctx, CIPHER_WRITE_CTX); } else { CODEC_TRACE(("sqlite3_rekey_v2: rollback\n")); - sqlite3BtreeRollback(pDb->pBt, SQLITE_ABORT_ROLLBACK); + sqlite3BtreeRollback(pDb->pBt, SQLITE_ABORT_ROLLBACK, 0); } sqlite3_mutex_leave(db->mutex); @@ -14663,7 +18508,7 @@ void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **ar int saved_flags; /* Saved value of the db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ - void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ + int (*saved_xTrace)(u32,void*,void*,void*); /* Saved db->xTrace */ int rc = SQLITE_OK; /* Return code from service routines */ char *zSql = NULL; /* SQL statements */ char *pzErrMsg = NULL; @@ -14813,6 +18658,8 @@ end_of_export: /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC +/* #include "sqliteInt.h" */ +/* #include "btreeInt.h" */ /************** Include sqlcipher.h in the middle of crypto_impl.c ***********/ /************** Begin file sqlcipher.h ***************************************/ /* @@ -14872,6 +18719,8 @@ typedef struct { int (*ctx_cmp)(void *c1, void *c2); int (*ctx_init)(void **ctx); int (*ctx_free)(void **ctx); + int (*fips_status)(void *ctx); + const char* (*get_provider_version)(void *ctx); } sqlcipher_provider; /* utility functions */ @@ -14893,8 +18742,9 @@ sqlcipher_provider* sqlcipher_get_provider(); /************** End of sqlcipher.h *******************************************/ /************** Continuing where we left off in crypto_impl.c ****************/ +/* #include "crypto.h" */ #ifndef OMIT_MEMLOCK -#if defined(__unix__) || defined(__APPLE__) +#if defined(__unix__) || defined(__APPLE__) || defined(_AIX) #include #elif defined(_WIN32) # include @@ -14928,6 +18778,7 @@ typedef struct { static unsigned int default_flags = DEFAULT_CIPHER_FLAGS; static unsigned char hmac_salt_mask = HMAC_SALT_MASK; static int default_kdf_iter = PBKDF2_ITER; +static int default_page_size = 1024; static unsigned int sqlcipher_activate_count = 0; static sqlite3_mutex* sqlcipher_provider_mutex = NULL; static sqlcipher_provider *default_provider = NULL; @@ -15292,15 +19143,15 @@ static int sqlcipher_cipher_ctx_set_keyspec(cipher_ctx *ctx, const unsigned char return SQLITE_OK; } -static int sqlcipher_codec_get_store_pass(codec_ctx *ctx) { +int sqlcipher_codec_get_store_pass(codec_ctx *ctx) { return ctx->read_ctx->store_pass; } -static void sqlcipher_codec_set_store_pass(codec_ctx *ctx, int value) { +void sqlcipher_codec_set_store_pass(codec_ctx *ctx, int value) { ctx->read_ctx->store_pass = value; } -static void sqlcipher_codec_get_pass(codec_ctx *ctx, void **zKey, int *nKey) { +void sqlcipher_codec_get_pass(codec_ctx *ctx, void **zKey, int *nKey) { *zKey = ctx->read_ctx->pass; *nKey = ctx->read_ctx->pass_sz; } @@ -15345,8 +19196,11 @@ int sqlcipher_codec_ctx_set_cipher(codec_ctx *ctx, const char *cipher_name, int cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; int rc; - c_ctx->provider->set_cipher(c_ctx->provider_ctx, cipher_name); - + rc = c_ctx->provider->set_cipher(c_ctx->provider_ctx, cipher_name); + if(rc != SQLITE_OK){ + sqlcipher_codec_ctx_set_error(ctx, rc); + return rc; + } c_ctx->key_sz = c_ctx->provider->get_key_sz(c_ctx->provider_ctx); c_ctx->iv_sz = c_ctx->provider->get_iv_sz(c_ctx->provider_ctx); c_ctx->block_sz = c_ctx->provider->get_block_sz(c_ctx->provider_ctx); @@ -15519,6 +19373,14 @@ int sqlcipher_codec_ctx_get_pagesize(codec_ctx *ctx) { return ctx->page_sz; } +void sqlcipher_set_default_pagesize(int page_size) { + default_page_size = page_size; +} + +int sqlcipher_get_default_pagesize() { + return default_page_size; +} + int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, sqlite3_file *fd, const void *zKey, int nKey) { int rc; codec_ctx *ctx; @@ -15549,7 +19411,7 @@ int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, sqlite3_f in encrypted and thus sqlite can't effectively determine the pagesize. this causes an issue in cases where bytes 16 & 17 of the page header are a power of 2 as reported by John Lehman */ - if((rc = sqlcipher_codec_ctx_set_pagesize(ctx, SQLITE_DEFAULT_PAGE_SIZE)) != SQLITE_OK) return rc; + if((rc = sqlcipher_codec_ctx_set_pagesize(ctx, default_page_size)) != SQLITE_OK) return rc; if((rc = sqlcipher_cipher_ctx_init(&ctx->read_ctx)) != SQLITE_OK) return rc; if((rc = sqlcipher_cipher_ctx_init(&ctx->write_ctx)) != SQLITE_OK) return rc; @@ -15734,12 +19596,12 @@ static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { if(ctx->read_ctx->provider->random(ctx->read_ctx->provider_ctx, ctx->kdf_salt, FILE_HEADER_SZ) != SQLITE_OK) return SQLITE_ERROR; ctx->need_kdf_salt = 0; } - if (c_ctx->pass_sz == ((c_ctx->key_sz * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0) { + if (c_ctx->pass_sz == ((c_ctx->key_sz * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0 && cipher_isHex(c_ctx->pass + 2, c_ctx->key_sz * 2)) { int n = c_ctx->pass_sz - 3; /* adjust for leading x' and tailing ' */ const unsigned char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ CODEC_TRACE(("cipher_ctx_key_derive: using raw key from hex\n")); cipher_hex2bin(z, n, c_ctx->key); - } else if (c_ctx->pass_sz == (((c_ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0) { + } else if (c_ctx->pass_sz == (((c_ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0 && cipher_isHex(c_ctx->pass + 2, (c_ctx->key_sz + ctx->kdf_salt_sz) * 2)) { const unsigned char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ CODEC_TRACE(("cipher_ctx_key_derive: using raw key from hex\n")); cipher_hex2bin(z, (c_ctx->key_sz * 2), c_ctx->key); @@ -15868,7 +19730,7 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { int saved_flags; int saved_nChange; int saved_nTotalChange; - void (*saved_xTrace)(void*,const char*); + int (*saved_xTrace)(u32,void*,void*,void*); /* Saved db->xTrace */ Db *pDb = 0; sqlite3 *db = ctx->pBt->db; const char *db_filename = sqlite3_db_filename(db, "main"); @@ -16007,11 +19869,9 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { db->nTotalChange = saved_nTotalChange; db->xTrace = saved_xTrace; db->autoCommit = 1; - if( pDb ){ - sqlite3BtreeClose(pDb->pBt); - pDb->pBt = 0; - pDb->pSchema = 0; - } + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; sqlite3ResetAllSchemasOfConnection(db); remove(migrated_db_filename); sqlite3_free(migrated_db_filename); @@ -16054,17 +19914,22 @@ int sqlcipher_codec_add_random(codec_ctx *ctx, const char *zRight, int random_sz int sqlcipher_cipher_profile(sqlite3 *db, const char *destination){ FILE *f; - if( strcmp(destination,"stdout")==0 ){ + if(sqlite3StrICmp(destination, "stdout") == 0){ f = stdout; - }else if( strcmp(destination, "stderr")==0 ){ + }else if(sqlite3StrICmp(destination, "stderr") == 0){ f = stderr; - }else if( strcmp(destination, "off")==0 ){ + }else if(sqlite3StrICmp(destination, "off") == 0){ f = 0; }else{ - f = fopen(destination, "wb"); - if( f==0 ){ - return SQLITE_ERROR; - } +#if defined(_WIN32) && (__STDC_VERSION__ > 199901L) || defined(SQLITE_OS_WINRT) + if(fopen_s(&f, destination, "a") != 0){ +#else + f = fopen(destination, "a"); + if(f == 0){ +#endif + return SQLITE_ERROR; + } + } sqlite3_profile(db, sqlcipher_profile_callback, f); return SQLITE_OK; @@ -16073,9 +19938,16 @@ int sqlcipher_cipher_profile(sqlite3 *db, const char *destination){ static void sqlcipher_profile_callback(void *file, const char *sql, sqlite3_uint64 run_time){ FILE *f = (FILE*)file; double elapsed = run_time/1000000.0; - if( f ) fprintf(f, "Elapsed time:%.3f ms - %s\n", elapsed, sql); + if(f) fprintf(f, "Elapsed time:%.3f ms - %s\n", elapsed, sql); } +int sqlcipher_codec_fips_status(codec_ctx *ctx) { + return ctx->read_ctx->provider->fips_status(ctx->read_ctx); +} + +const char* sqlcipher_codec_get_provider_version(codec_ctx *ctx) { + return ctx->read_ctx->provider->get_provider_version(ctx->read_ctx); +} #endif /* END SQLCIPHER */ @@ -16115,6 +19987,8 @@ static void sqlcipher_profile_callback(void *file, const char *sql, sqlite3_uint /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #ifdef SQLCIPHER_CRYPTO_LIBTOMCRYPT +/* #include "sqliteInt.h" */ +/* #include "sqlcipher.h" */ #include #define FORTUNA_MAX_SZ 32 @@ -16206,6 +20080,10 @@ static const char* sqlcipher_ltc_get_provider_name(void *ctx) { return "libtomcrypt"; } +static const char* sqlcipher_ltc_get_provider_version(void *ctx) { + return SCRYPT; +} + static int sqlcipher_ltc_random(void *ctx, void *buffer, int length) { #ifndef SQLCIPHER_LTC_NO_MUTEX_RAND sqlite3_mutex_enter(ltc_rand_mutex); @@ -16309,6 +20187,10 @@ static int sqlcipher_ltc_ctx_free(void **ctx) { return SQLITE_OK; } +static int sqlcipher_ltc_fips_status(void *ctx) { + return 0; +} + int sqlcipher_ltc_setup(sqlcipher_provider *p) { p->activate = sqlcipher_ltc_activate; p->deactivate = sqlcipher_ltc_deactivate; @@ -16328,6 +20210,8 @@ int sqlcipher_ltc_setup(sqlcipher_provider *p) { p->ctx_init = sqlcipher_ltc_ctx_init; p->ctx_free = sqlcipher_ltc_ctx_free; p->add_random = sqlcipher_ltc_add_random; + p->fips_status = sqlcipher_ltc_fips_status; + p->get_provider_version = sqlcipher_ltc_get_provider_version; return SQLITE_OK; } @@ -16370,6 +20254,9 @@ int sqlcipher_ltc_setup(sqlcipher_provider *p) { /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #ifdef SQLCIPHER_CRYPTO_OPENSSL +/* #include "sqliteInt.h" */ +/* #include "crypto.h" */ +/* #include "sqlcipher.h" */ #include #include #include @@ -16378,11 +20265,33 @@ typedef struct { EVP_CIPHER *evp_cipher; } openssl_ctx; - static unsigned int openssl_external_init = 0; static unsigned int openssl_init_count = 0; static sqlite3_mutex* openssl_rand_mutex = NULL; +#if OPENSSL_VERSION_NUMBER < 0x10100000L +static HMAC_CTX *HMAC_CTX_new(void) +{ + HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); + if (ctx != NULL) { + HMAC_CTX_init(ctx); + } + return ctx; +} + +// Per 1.1.0 (https://wiki.openssl.org/index.php/1.1_API_Changes) +// HMAC_CTX_free should call HMAC_CTX_cleanup, then EVP_MD_CTX_Cleanup. +// HMAC_CTX_cleanup internally calls EVP_MD_CTX_cleanup so these +// calls are not needed. +static void HMAC_CTX_free(HMAC_CTX *ctx) +{ + if (ctx != NULL) { + HMAC_CTX_cleanup(ctx); + OPENSSL_free(ctx); + } +} +#endif + static int sqlcipher_openssl_add_random(void *ctx, void *buffer, int length) { #ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND sqlite3_mutex_enter(openssl_rand_mutex); @@ -16413,6 +20322,15 @@ static int sqlcipher_openssl_activate(void *ctx) { openssl_external_init = 1; } +#ifdef SQLCIPHER_FIPS + if(!FIPS_mode()){ + if(!FIPS_mode_set(1)){ + ERR_load_crypto_strings(); + ERR_print_errors_fp(stderr); + } + } +#endif + if(openssl_init_count == 0 && openssl_external_init == 0) { /* if the library was not externally initialized, then should be now */ OpenSSL_add_all_algorithms(); @@ -16445,6 +20363,8 @@ static int sqlcipher_openssl_deactivate(void *ctx) { is called by SQLCipher internally. This should prevent SQLCipher from "cleaning up" openssl when it was initialized externally by the program */ EVP_cleanup(); + } else { + openssl_external_init = 0; } #ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND sqlite3_mutex_free(openssl_rand_mutex); @@ -16459,6 +20379,10 @@ static const char* sqlcipher_openssl_get_provider_name(void *ctx) { return "openssl"; } +static const char* sqlcipher_openssl_get_provider_version(void *ctx) { + return OPENSSL_VERSION_TEXT; +} + /* generate a defined number of random bytes */ static int sqlcipher_openssl_random (void *ctx, void *buffer, int length) { int rc = 0; @@ -16479,14 +20403,14 @@ static int sqlcipher_openssl_random (void *ctx, void *buffer, int length) { } static int sqlcipher_openssl_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { - HMAC_CTX hctx; unsigned int outlen; - HMAC_CTX_init(&hctx); - HMAC_Init_ex(&hctx, hmac_key, key_sz, EVP_sha1(), NULL); - HMAC_Update(&hctx, in, in_sz); - HMAC_Update(&hctx, in2, in2_sz); - HMAC_Final(&hctx, out, &outlen); - HMAC_CTX_cleanup(&hctx); + HMAC_CTX* hctx = HMAC_CTX_new(); + if(hctx == NULL) return SQLITE_ERROR; + HMAC_Init_ex(hctx, hmac_key, key_sz, EVP_sha1(), NULL); + HMAC_Update(hctx, in, in_sz); + HMAC_Update(hctx, in2, in2_sz); + HMAC_Final(hctx, out, &outlen); + HMAC_CTX_free(hctx); return SQLITE_OK; } @@ -16496,26 +20420,29 @@ static int sqlcipher_openssl_kdf(void *ctx, const unsigned char *pass, int pass_ } static int sqlcipher_openssl_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { - EVP_CIPHER_CTX ectx; int tmp_csz, csz; - - EVP_CipherInit(&ectx, ((openssl_ctx *)ctx)->evp_cipher, NULL, NULL, mode); - EVP_CIPHER_CTX_set_padding(&ectx, 0); // no padding - EVP_CipherInit(&ectx, NULL, key, iv, mode); - EVP_CipherUpdate(&ectx, out, &tmp_csz, in, in_sz); + EVP_CIPHER_CTX* ectx = EVP_CIPHER_CTX_new(); + if(ectx == NULL) return SQLITE_ERROR; + EVP_CipherInit_ex(ectx, ((openssl_ctx *)ctx)->evp_cipher, NULL, NULL, NULL, mode); + EVP_CIPHER_CTX_set_padding(ectx, 0); // no padding + EVP_CipherInit_ex(ectx, NULL, NULL, key, iv, mode); + EVP_CipherUpdate(ectx, out, &tmp_csz, in, in_sz); csz = tmp_csz; out += tmp_csz; - EVP_CipherFinal(&ectx, out, &tmp_csz); + EVP_CipherFinal_ex(ectx, out, &tmp_csz); csz += tmp_csz; - EVP_CIPHER_CTX_cleanup(&ectx); + EVP_CIPHER_CTX_free(ectx); assert(in_sz == csz); return SQLITE_OK; } static int sqlcipher_openssl_set_cipher(void *ctx, const char *cipher_name) { openssl_ctx *o_ctx = (openssl_ctx *)ctx; - o_ctx->evp_cipher = (EVP_CIPHER *) EVP_get_cipherbyname(cipher_name); - return SQLITE_OK; + EVP_CIPHER* cipher = (EVP_CIPHER *) EVP_get_cipherbyname(cipher_name); + if(cipher != NULL) { + o_ctx->evp_cipher = cipher; + } + return cipher != NULL ? SQLITE_OK : SQLITE_ERROR; } static const char* sqlcipher_openssl_get_cipher(void *ctx) { @@ -16560,6 +20487,14 @@ static int sqlcipher_openssl_ctx_free(void **ctx) { return SQLITE_OK; } +static int sqlcipher_openssl_fips_status(void *ctx) { +#ifdef SQLCIPHER_FIPS + return FIPS_mode(); +#else + return 0; +#endif +} + int sqlcipher_openssl_setup(sqlcipher_provider *p) { p->activate = sqlcipher_openssl_activate; p->deactivate = sqlcipher_openssl_deactivate; @@ -16579,6 +20514,8 @@ int sqlcipher_openssl_setup(sqlcipher_provider *p) { p->ctx_init = sqlcipher_openssl_ctx_init; p->ctx_free = sqlcipher_openssl_ctx_free; p->add_random = sqlcipher_openssl_add_random; + p->fips_status = sqlcipher_openssl_fips_status; + p->get_provider_version = sqlcipher_openssl_get_provider_version; return SQLITE_OK; } @@ -16621,8 +20558,11 @@ int sqlcipher_openssl_setup(sqlcipher_provider *p) { /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #ifdef SQLCIPHER_CRYPTO_CC +/* #include "crypto.h" */ +/* #include "sqlcipher.h" */ #include #include +#include static int sqlcipher_cc_add_random(void *ctx, void *buffer, int length) { return SQLITE_OK; @@ -16637,6 +20577,20 @@ static const char* sqlcipher_cc_get_provider_name(void *ctx) { return "commoncrypto"; } +static const char* sqlcipher_cc_get_provider_version(void *ctx) { +#if TARGET_OS_MAC + CFTypeRef version; + CFBundleRef bundle = CFBundleGetBundleWithIdentifier(CFSTR("com.apple.security")); + if(bundle == NULL) { + return "unknown"; + } + version = CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("CFBundleShortVersionString")); + return CFStringGetCStringPtr(version, kCFStringEncodingUTF8); +#else + return "unknown"; +#endif +} + static int sqlcipher_cc_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { CCHmacContext hmac_context; CCHmacInit(&hmac_context, kCCHmacAlgSHA1, hmac_key, key_sz); @@ -16663,7 +20617,7 @@ static int sqlcipher_cc_cipher(void *ctx, int mode, unsigned char *key, int key_ CCCryptorFinal(cryptor, out, in_sz - csz, &tmp_csz); csz += tmp_csz; CCCryptorRelease(cryptor); - assert(size == csz); + assert(in_sz == csz); return SQLITE_OK; } @@ -16708,6 +20662,10 @@ static int sqlcipher_cc_ctx_free(void **ctx) { return SQLITE_OK; } +static int sqlcipher_cc_fips_status(void *ctx) { + return 0; +} + int sqlcipher_cc_setup(sqlcipher_provider *p) { p->random = sqlcipher_cc_random; p->get_provider_name = sqlcipher_cc_get_provider_name; @@ -16725,6 +20683,8 @@ int sqlcipher_cc_setup(sqlcipher_provider *p) { p->ctx_init = sqlcipher_cc_ctx_init; p->ctx_free = sqlcipher_cc_ctx_free; p->add_random = sqlcipher_cc_add_random; + p->fips_status = sqlcipher_cc_fips_status; + p->get_provider_version = sqlcipher_cc_get_provider_version; return SQLITE_OK; } @@ -16746,8 +20706,9 @@ int sqlcipher_cc_setup(sqlcipher_provider *p) { ** ************************************************************************* ** -** This file contains definitions of global variables and contants. +** This file contains definitions of global variables and constants. */ +/* #include "sqliteInt.h" */ /* An array to map all upper-case characters into their corresponding ** lower-case character. @@ -16781,16 +20742,16 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, /* 3x */ 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, /* 4x */ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, /* 5x */ - 96, 97, 66, 67, 68, 69, 70, 71, 72, 73,106,107,108,109,110,111, /* 6x */ - 112, 81, 82, 83, 84, 85, 86, 87, 88, 89,122,123,124,125,126,127, /* 7x */ + 96, 97, 98, 99,100,101,102,103,104,105,106,107,108,109,110,111, /* 6x */ + 112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127, /* 7x */ 128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, /* 8x */ - 144,145,146,147,148,149,150,151,152,153,154,155,156,157,156,159, /* 9x */ + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159, /* 9x */ 160,161,162,163,164,165,166,167,168,169,170,171,140,141,142,175, /* Ax */ 176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191, /* Bx */ 192,129,130,131,132,133,134,135,136,137,202,203,204,205,206,207, /* Cx */ 208,145,146,147,148,149,150,151,152,153,218,219,220,221,222,223, /* Dx */ - 224,225,162,163,164,165,166,167,168,169,232,203,204,205,206,207, /* Ex */ - 239,240,241,242,243,244,245,246,247,248,249,219,220,221,222,255, /* Fx */ + 224,225,162,163,164,165,166,167,168,169,234,235,236,237,238,239, /* Ex */ + 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, /* Fx */ #endif }; @@ -16805,6 +20766,7 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { ** isxdigit() 0x08 ** toupper() 0x20 ** SQLite identifier character 0x40 +** Quote character 0x80 ** ** Bit 0x20 is set if the mapped character requires translation to upper ** case. i.e. if the character is a lower-case ASCII character. @@ -16813,16 +20775,13 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { ** ** (x & ~(map[x]&0x20)) ** -** Standard function tolower() is implemented using the sqlite3UpperToLower[] +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] ** array. tolower() is used more often than toupper() by SQLite. ** -** Bit 0x40 is set if the character non-alphanumeric and can be used in an +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an ** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any ** non-ASCII UTF character. Hence the test for whether or not a character is ** part of an identifier is 0x46. -** -** SQLite's versions are identical to the standard versions assuming a -** locale of "C". They are implemented as macros in sqliteInt.h. */ #ifdef SQLITE_ASCII SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { @@ -16830,7 +20789,7 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ - 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */ + 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ @@ -16838,8 +20797,8 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ - 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ - 0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ @@ -16864,14 +20823,48 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { }; #endif +/* EVIDENCE-OF: R-02982-34736 In order to maintain full backwards +** compatibility for legacy applications, the URI filename capability is +** disabled by default. +** +** EVIDENCE-OF: R-38799-08373 URI filenames can be enabled or disabled +** using the SQLITE_USE_URI=1 or SQLITE_USE_URI=0 compile-time options. +** +** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** SQLITE_USE_URI symbol defined. +*/ #ifndef SQLITE_USE_URI # define SQLITE_USE_URI 0 #endif +/* EVIDENCE-OF: R-38720-18127 The default setting is determined by the +** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if +** that compile-time option is omitted. +*/ #ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 #endif +/* The minimum PMA size is set to this value multiplied by the database +** page size in bytes. +*/ +#ifndef SQLITE_SORTER_PMASZ +# define SQLITE_SORTER_PMASZ 250 +#endif + +/* Statement journals spill to disk when their size exceeds the following +** threshold (in bytes). 0 means that statement journals are created and +** written to disk immediately (the default behavior for SQLite versions +** before 3.12.0). -1 means always keep the entire statement journal in +** memory. (The statement journal is also always held entirely in memory +** if journal_mode=MEMORY or if temp_store=MEMORY, regardless of this +** setting.) +*/ +#ifndef SQLITE_STMTJRNL_SPILL +# define SQLITE_STMTJRNL_SPILL (64*1024) +#endif + /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -16886,6 +20879,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* neverCorrupt */ 128, /* szLookaside */ 500, /* nLookaside */ + SQLITE_STMTJRNL_SPILL, /* nStmtSpill */ {0,0,0,0,0,0,0,0}, /* m */ {0,0,0,0,0,0,0,0,0}, /* mutex */ {0,0,0,0,0,0,0,0,0,0,0,0,0},/* pcache2 */ @@ -16899,9 +20893,10 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nScratch */ (void*)0, /* pPage */ 0, /* szPage */ - 0, /* nPage */ + SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ + SQLITE_SORTER_PMASZ, /* szPma */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ @@ -16923,7 +20918,8 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { #ifndef SQLITE_OMIT_BUILTIN_TEST 0, /* xTestCallback */ #endif - 0 /* bLocaltimeFault */ + 0, /* bLocaltimeFault */ + 0x7ffffffe /* iOnceResetThreshold */ }; /* @@ -16931,7 +20927,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { ** database connections. After initialization, this table is ** read-only. */ -SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; +SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; /* ** Constant tokens for values 0 and 1. @@ -16946,7 +20942,7 @@ SQLITE_PRIVATE const Token sqlite3IntTokens[] = { ** The value of the "pending" byte must be 0x40000000 (1 byte past the ** 1-gibabyte boundary) in a compatible database. SQLite never uses ** the database page that contains the pending byte. It never attempts -** to read or write that page. The pending byte page is set assign +** to read or write that page. The pending byte page is set aside ** for use by the VFS layers as space for managing file locks. ** ** During testing, it is often desirable to move the pending byte to @@ -16957,13 +20953,14 @@ SQLITE_PRIVATE const Token sqlite3IntTokens[] = { ** ** IMPORTANT: Changing the pending byte to any value other than ** 0x40000000 results in an incompatible database file format! -** Changing the pending byte during operating results in undefined -** and dileterious behavior. +** Changing the pending byte during operation will result in undefined +** and incorrect behavior. */ #ifndef SQLITE_OMIT_WSD SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; #endif +/* #include "opcodes.h" */ /* ** Properties of opcodes. The OPFLG_INITIALIZER macro is ** created by mkopcodeh.awk during compilation. Data is obtained @@ -16972,6 +20969,11 @@ SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; */ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; +/* +** Name of the default collating sequence +*/ +SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; + /************** End of global.c **********************************************/ /************** Begin file ctime.c *******************************************/ /* @@ -16992,6 +20994,7 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +/* #include "sqliteInt.h" */ /* ** An array of names of all compile-time options. This array should @@ -17008,88 +21011,112 @@ static const char * const azCompileOpt[] = { #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) -#ifdef SQLITE_32BIT_ROWID +#if SQLITE_32BIT_ROWID "32BIT_ROWID", #endif -#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC +#if SQLITE_4_BYTE_ALIGNED_MALLOC "4_BYTE_ALIGNED_MALLOC", #endif -#ifdef SQLITE_CASE_SENSITIVE_LIKE +#if SQLITE_CASE_SENSITIVE_LIKE "CASE_SENSITIVE_LIKE", #endif -#ifdef SQLITE_CHECK_PAGES +#if SQLITE_CHECK_PAGES "CHECK_PAGES", #endif -#ifdef SQLITE_COVERAGE_TEST +#if defined(__clang__) && defined(__clang_major__) + "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__), +#elif defined(_MSC_VER) + "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), +#elif defined(__GNUC__) && defined(__VERSION__) + "COMPILER=gcc-" __VERSION__, +#endif +#if SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#ifdef SQLITE_DEBUG +#if SQLITE_DEBUG "DEBUG", #endif -#ifdef SQLITE_DEFAULT_LOCKING_MODE +#if SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif #if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), #endif -#ifdef SQLITE_DISABLE_DIRSYNC +#if SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif -#ifdef SQLITE_DISABLE_LFS +#if SQLITE_DISABLE_LFS "DISABLE_LFS", #endif -#ifdef SQLITE_ENABLE_ATOMIC_WRITE +#if SQLITE_ENABLE_8_3_NAMES + "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), +#endif +#if SQLITE_ENABLE_API_ARMOR + "ENABLE_API_ARMOR", +#endif +#if SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif -#ifdef SQLITE_ENABLE_CEROD +#if SQLITE_ENABLE_CEROD "ENABLE_CEROD", #endif -#ifdef SQLITE_ENABLE_COLUMN_METADATA +#if SQLITE_ENABLE_COLUMN_METADATA "ENABLE_COLUMN_METADATA", #endif -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT +#if SQLITE_ENABLE_DBSTAT_VTAB + "ENABLE_DBSTAT_VTAB", +#endif +#if SQLITE_ENABLE_EXPENSIVE_ASSERT "ENABLE_EXPENSIVE_ASSERT", #endif -#ifdef SQLITE_ENABLE_FTS1 +#if SQLITE_ENABLE_FTS1 "ENABLE_FTS1", #endif -#ifdef SQLITE_ENABLE_FTS2 +#if SQLITE_ENABLE_FTS2 "ENABLE_FTS2", #endif -#ifdef SQLITE_ENABLE_FTS3 +#if SQLITE_ENABLE_FTS3 "ENABLE_FTS3", #endif -#ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +#if SQLITE_ENABLE_FTS3_PARENTHESIS "ENABLE_FTS3_PARENTHESIS", #endif -#ifdef SQLITE_ENABLE_FTS4 +#if SQLITE_ENABLE_FTS4 "ENABLE_FTS4", #endif -#ifdef SQLITE_ENABLE_ICU +#if SQLITE_ENABLE_FTS5 + "ENABLE_FTS5", +#endif +#if SQLITE_ENABLE_ICU "ENABLE_ICU", #endif -#ifdef SQLITE_ENABLE_IOTRACE +#if SQLITE_ENABLE_IOTRACE "ENABLE_IOTRACE", #endif -#ifdef SQLITE_ENABLE_LOAD_EXTENSION +#if SQLITE_ENABLE_JSON1 + "ENABLE_JSON1", +#endif +#if SQLITE_ENABLE_LOAD_EXTENSION "ENABLE_LOAD_EXTENSION", #endif -#ifdef SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_ENABLE_LOCKING_STYLE "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), #endif -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +#if SQLITE_ENABLE_MEMORY_MANAGEMENT "ENABLE_MEMORY_MANAGEMENT", #endif -#ifdef SQLITE_ENABLE_MEMSYS3 +#if SQLITE_ENABLE_MEMSYS3 "ENABLE_MEMSYS3", #endif -#ifdef SQLITE_ENABLE_MEMSYS5 +#if SQLITE_ENABLE_MEMSYS5 "ENABLE_MEMSYS5", #endif -#ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK +#if SQLITE_ENABLE_OVERSIZE_CELL_CHECK "ENABLE_OVERSIZE_CELL_CHECK", #endif -#ifdef SQLITE_ENABLE_RTREE +#if SQLITE_ENABLE_RTREE "ENABLE_RTREE", #endif #if defined(SQLITE_ENABLE_STAT4) @@ -17097,31 +21124,34 @@ static const char * const azCompileOpt[] = { #elif defined(SQLITE_ENABLE_STAT3) "ENABLE_STAT3", #endif -#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY +#if SQLITE_ENABLE_UNLOCK_NOTIFY "ENABLE_UNLOCK_NOTIFY", #endif -#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT +#if SQLITE_ENABLE_UPDATE_DELETE_LIMIT "ENABLE_UPDATE_DELETE_LIMIT", #endif -#ifdef SQLITE_HAS_CODEC +#if SQLITE_HAS_CODEC "HAS_CODEC", #endif -#ifdef SQLITE_HAVE_ISNAN +#if HAVE_ISNAN || SQLITE_HAVE_ISNAN "HAVE_ISNAN", #endif -#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX +#if SQLITE_HOMEGROWN_RECURSIVE_MUTEX "HOMEGROWN_RECURSIVE_MUTEX", #endif -#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS +#if SQLITE_IGNORE_AFP_LOCK_ERRORS "IGNORE_AFP_LOCK_ERRORS", #endif -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS +#if SQLITE_IGNORE_FLOCK_LOCK_ERRORS "IGNORE_FLOCK_LOCK_ERRORS", #endif #ifdef SQLITE_INT64_TYPE "INT64_TYPE", #endif -#ifdef SQLITE_LOCK_TRACE +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + "LIKE_DOESNT_MATCH_BLOBS", +#endif +#if SQLITE_LOCK_TRACE "LOCK_TRACE", #endif #if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) @@ -17130,223 +21160,226 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_MAX_SCHEMA_RETRY "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), #endif -#ifdef SQLITE_MEMDEBUG +#if SQLITE_MEMDEBUG "MEMDEBUG", #endif -#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT +#if SQLITE_MIXED_ENDIAN_64BIT_FLOAT "MIXED_ENDIAN_64BIT_FLOAT", #endif -#ifdef SQLITE_NO_SYNC +#if SQLITE_NO_SYNC "NO_SYNC", #endif -#ifdef SQLITE_OMIT_ALTERTABLE +#if SQLITE_OMIT_ALTERTABLE "OMIT_ALTERTABLE", #endif -#ifdef SQLITE_OMIT_ANALYZE +#if SQLITE_OMIT_ANALYZE "OMIT_ANALYZE", #endif -#ifdef SQLITE_OMIT_ATTACH +#if SQLITE_OMIT_ATTACH "OMIT_ATTACH", #endif -#ifdef SQLITE_OMIT_AUTHORIZATION +#if SQLITE_OMIT_AUTHORIZATION "OMIT_AUTHORIZATION", #endif -#ifdef SQLITE_OMIT_AUTOINCREMENT +#if SQLITE_OMIT_AUTOINCREMENT "OMIT_AUTOINCREMENT", #endif -#ifdef SQLITE_OMIT_AUTOINIT +#if SQLITE_OMIT_AUTOINIT "OMIT_AUTOINIT", #endif -#ifdef SQLITE_OMIT_AUTOMATIC_INDEX +#if SQLITE_OMIT_AUTOMATIC_INDEX "OMIT_AUTOMATIC_INDEX", #endif -#ifdef SQLITE_OMIT_AUTORESET +#if SQLITE_OMIT_AUTORESET "OMIT_AUTORESET", #endif -#ifdef SQLITE_OMIT_AUTOVACUUM +#if SQLITE_OMIT_AUTOVACUUM "OMIT_AUTOVACUUM", #endif -#ifdef SQLITE_OMIT_BETWEEN_OPTIMIZATION +#if SQLITE_OMIT_BETWEEN_OPTIMIZATION "OMIT_BETWEEN_OPTIMIZATION", #endif -#ifdef SQLITE_OMIT_BLOB_LITERAL +#if SQLITE_OMIT_BLOB_LITERAL "OMIT_BLOB_LITERAL", #endif -#ifdef SQLITE_OMIT_BTREECOUNT +#if SQLITE_OMIT_BTREECOUNT "OMIT_BTREECOUNT", #endif -#ifdef SQLITE_OMIT_BUILTIN_TEST +#if SQLITE_OMIT_BUILTIN_TEST "OMIT_BUILTIN_TEST", #endif -#ifdef SQLITE_OMIT_CAST +#if SQLITE_OMIT_CAST "OMIT_CAST", #endif -#ifdef SQLITE_OMIT_CHECK +#if SQLITE_OMIT_CHECK "OMIT_CHECK", #endif -#ifdef SQLITE_OMIT_COMPLETE +#if SQLITE_OMIT_COMPLETE "OMIT_COMPLETE", #endif -#ifdef SQLITE_OMIT_COMPOUND_SELECT +#if SQLITE_OMIT_COMPOUND_SELECT "OMIT_COMPOUND_SELECT", #endif -#ifdef SQLITE_OMIT_CTE +#if SQLITE_OMIT_CTE "OMIT_CTE", #endif -#ifdef SQLITE_OMIT_DATETIME_FUNCS +#if SQLITE_OMIT_DATETIME_FUNCS "OMIT_DATETIME_FUNCS", #endif -#ifdef SQLITE_OMIT_DECLTYPE +#if SQLITE_OMIT_DECLTYPE "OMIT_DECLTYPE", #endif -#ifdef SQLITE_OMIT_DEPRECATED +#if SQLITE_OMIT_DEPRECATED "OMIT_DEPRECATED", #endif -#ifdef SQLITE_OMIT_DISKIO +#if SQLITE_OMIT_DISKIO "OMIT_DISKIO", #endif -#ifdef SQLITE_OMIT_EXPLAIN +#if SQLITE_OMIT_EXPLAIN "OMIT_EXPLAIN", #endif -#ifdef SQLITE_OMIT_FLAG_PRAGMAS +#if SQLITE_OMIT_FLAG_PRAGMAS "OMIT_FLAG_PRAGMAS", #endif -#ifdef SQLITE_OMIT_FLOATING_POINT +#if SQLITE_OMIT_FLOATING_POINT "OMIT_FLOATING_POINT", #endif -#ifdef SQLITE_OMIT_FOREIGN_KEY +#if SQLITE_OMIT_FOREIGN_KEY "OMIT_FOREIGN_KEY", #endif -#ifdef SQLITE_OMIT_GET_TABLE +#if SQLITE_OMIT_GET_TABLE "OMIT_GET_TABLE", #endif -#ifdef SQLITE_OMIT_INCRBLOB +#if SQLITE_OMIT_INCRBLOB "OMIT_INCRBLOB", #endif -#ifdef SQLITE_OMIT_INTEGRITY_CHECK +#if SQLITE_OMIT_INTEGRITY_CHECK "OMIT_INTEGRITY_CHECK", #endif -#ifdef SQLITE_OMIT_LIKE_OPTIMIZATION +#if SQLITE_OMIT_LIKE_OPTIMIZATION "OMIT_LIKE_OPTIMIZATION", #endif -#ifdef SQLITE_OMIT_LOAD_EXTENSION +#if SQLITE_OMIT_LOAD_EXTENSION "OMIT_LOAD_EXTENSION", #endif -#ifdef SQLITE_OMIT_LOCALTIME +#if SQLITE_OMIT_LOCALTIME "OMIT_LOCALTIME", #endif -#ifdef SQLITE_OMIT_LOOKASIDE +#if SQLITE_OMIT_LOOKASIDE "OMIT_LOOKASIDE", #endif -#ifdef SQLITE_OMIT_MEMORYDB +#if SQLITE_OMIT_MEMORYDB "OMIT_MEMORYDB", #endif -#ifdef SQLITE_OMIT_OR_OPTIMIZATION +#if SQLITE_OMIT_OR_OPTIMIZATION "OMIT_OR_OPTIMIZATION", #endif -#ifdef SQLITE_OMIT_PAGER_PRAGMAS +#if SQLITE_OMIT_PAGER_PRAGMAS "OMIT_PAGER_PRAGMAS", #endif -#ifdef SQLITE_OMIT_PRAGMA +#if SQLITE_OMIT_PRAGMA "OMIT_PRAGMA", #endif -#ifdef SQLITE_OMIT_PROGRESS_CALLBACK +#if SQLITE_OMIT_PROGRESS_CALLBACK "OMIT_PROGRESS_CALLBACK", #endif -#ifdef SQLITE_OMIT_QUICKBALANCE +#if SQLITE_OMIT_QUICKBALANCE "OMIT_QUICKBALANCE", #endif -#ifdef SQLITE_OMIT_REINDEX +#if SQLITE_OMIT_REINDEX "OMIT_REINDEX", #endif -#ifdef SQLITE_OMIT_SCHEMA_PRAGMAS +#if SQLITE_OMIT_SCHEMA_PRAGMAS "OMIT_SCHEMA_PRAGMAS", #endif -#ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS +#if SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS "OMIT_SCHEMA_VERSION_PRAGMAS", #endif -#ifdef SQLITE_OMIT_SHARED_CACHE +#if SQLITE_OMIT_SHARED_CACHE "OMIT_SHARED_CACHE", #endif -#ifdef SQLITE_OMIT_SUBQUERY +#if SQLITE_OMIT_SUBQUERY "OMIT_SUBQUERY", #endif -#ifdef SQLITE_OMIT_TCL_VARIABLE +#if SQLITE_OMIT_TCL_VARIABLE "OMIT_TCL_VARIABLE", #endif -#ifdef SQLITE_OMIT_TEMPDB +#if SQLITE_OMIT_TEMPDB "OMIT_TEMPDB", #endif -#ifdef SQLITE_OMIT_TRACE +#if SQLITE_OMIT_TRACE "OMIT_TRACE", #endif -#ifdef SQLITE_OMIT_TRIGGER +#if SQLITE_OMIT_TRIGGER "OMIT_TRIGGER", #endif -#ifdef SQLITE_OMIT_TRUNCATE_OPTIMIZATION +#if SQLITE_OMIT_TRUNCATE_OPTIMIZATION "OMIT_TRUNCATE_OPTIMIZATION", #endif -#ifdef SQLITE_OMIT_UTF16 +#if SQLITE_OMIT_UTF16 "OMIT_UTF16", #endif -#ifdef SQLITE_OMIT_VACUUM +#if SQLITE_OMIT_VACUUM "OMIT_VACUUM", #endif -#ifdef SQLITE_OMIT_VIEW +#if SQLITE_OMIT_VIEW "OMIT_VIEW", #endif -#ifdef SQLITE_OMIT_VIRTUALTABLE +#if SQLITE_OMIT_VIRTUALTABLE "OMIT_VIRTUALTABLE", #endif -#ifdef SQLITE_OMIT_WAL +#if SQLITE_OMIT_WAL "OMIT_WAL", #endif -#ifdef SQLITE_OMIT_WSD +#if SQLITE_OMIT_WSD "OMIT_WSD", #endif -#ifdef SQLITE_OMIT_XFER_OPT +#if SQLITE_OMIT_XFER_OPT "OMIT_XFER_OPT", #endif -#ifdef SQLITE_PERFORMANCE_TRACE +#if SQLITE_PERFORMANCE_TRACE "PERFORMANCE_TRACE", #endif -#ifdef SQLITE_PROXY_DEBUG +#if SQLITE_PROXY_DEBUG "PROXY_DEBUG", #endif -#ifdef SQLITE_RTREE_INT_ONLY +#if SQLITE_RTREE_INT_ONLY "RTREE_INT_ONLY", #endif -#ifdef SQLITE_SECURE_DELETE +#if SQLITE_SECURE_DELETE "SECURE_DELETE", #endif -#ifdef SQLITE_SMALL_STACK +#if SQLITE_SMALL_STACK "SMALL_STACK", #endif -#ifdef SQLITE_SOUNDEX +#if SQLITE_SOUNDEX "SOUNDEX", #endif -#ifdef SQLITE_SYSTEM_MALLOC +#if SQLITE_SYSTEM_MALLOC "SYSTEM_MALLOC", #endif -#ifdef SQLITE_TCL +#if SQLITE_TCL "TCL", #endif #if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), #endif -#ifdef SQLITE_TEST +#if SQLITE_TEST "TEST", #endif #if defined(SQLITE_THREADSAFE) "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), #endif -#ifdef SQLITE_USE_ALLOCA +#if SQLITE_USE_ALLOCA "USE_ALLOCA", #endif -#ifdef SQLITE_WIN32_MALLOC +#if SQLITE_USER_AUTHENTICATION + "USER_AUTHENTICATION", +#endif +#if SQLITE_WIN32_MALLOC "WIN32_MALLOC", #endif -#ifdef SQLITE_ZERO_MALLOC +#if SQLITE_ZERO_MALLOC "ZERO_MALLOC" #endif }; @@ -17360,6 +21393,13 @@ static const char * const azCompileOpt[] = { */ SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ int i, n; + +#if SQLITE_ENABLE_API_ARMOR + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; n = sqlite3Strlen30(zOptName); @@ -17367,7 +21407,7 @@ SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ ** linear search is adequate. No need for a binary search. */ for(i=0; iaDb[] (or -1) */ + u8 nullRow; /* True if pointing to a row with no data */ + u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + u8 isTable; /* True for rowid tables. False for indexes */ +#ifdef SQLITE_DEBUG + u8 seekOp; /* Most recent seek operation on this cursor */ + u8 wrFlag; /* The wrFlag argument to sqlite3BtreeCursor() */ +#endif + Bool isEphemeral:1; /* True for an ephemeral table */ + Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ + Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ + Pgno pgnoRoot; /* Root page of the open btree cursor */ + i16 nField; /* Number of fields in the header */ + u16 nHdrParsed; /* Number of header fields parsed so far */ + union { + BtCursor *pCursor; /* CURTYPE_BTREE. Btree cursor */ + sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB. Vtab cursor */ + int pseudoTableReg; /* CURTYPE_PSEUDO. Reg holding content. */ + VdbeSorter *pSorter; /* CURTYPE_SORTER. Sorter object */ + } uc; Btree *pBt; /* Separate file holding temporary table */ KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ int seekResult; /* Result of previous sqlite3BtreeMoveto() */ - int pseudoTableReg; /* Register holding pseudotable content. */ - i16 nField; /* Number of fields in the header */ - u16 nHdrParsed; /* Number of header fields parsed so far */ -#ifdef SQLITE_DEBUG - u8 seekOp; /* Most recent seek operation on this cursor */ -#endif - i8 iDb; /* Index of cursor database in db->aDb[] (or -1) */ - u8 nullRow; /* True if pointing to a row with no data */ - u8 rowidIsValid; /* True if lastRowid is valid */ - u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ - Bool isEphemeral:1; /* True for an ephemeral table */ - Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ - Bool isTable:1; /* True if a table requiring integer keys */ - Bool isOrdered:1; /* True if the underlying table is BTREE_UNORDERED */ - Pgno pgnoRoot; /* Root page of the open btree cursor */ - sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ - i64 lastRowid; /* Rowid being deleted by OP_Delete */ - VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ + VdbeCursor *pAltCursor; /* Associated index cursor from which to read */ + int *aAltMap; /* Mapping from table to index column numbers */ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + u64 maskUsed; /* Mask of columns used by this cursor */ +#endif /* Cached information about the header for the data record that the ** cursor is currently pointing to. Only valid if cacheStatus matches @@ -17509,12 +21569,18 @@ struct VdbeCursor { u32 szRow; /* Byte available in aRow */ u32 iHdrOffset; /* Offset to next unparsed byte of the header */ const u8 *aRow; /* Data for the current row, if all on one page */ + u32 *aOffset; /* Pointer to aType[nField] */ u32 aType[1]; /* Type values for all entries in the record */ /* 2*nField extra array elements allocated for aType[], beyond the one ** static element declared in the structure. nField total array slots for ** aType[] and nField+1 array slots for aOffset[] */ }; -typedef struct VdbeCursor VdbeCursor; + + +/* +** A value for VdbeCursor.cacheStatus that means the cache is always invalid. +*/ +#define CACHE_STALE 0 /* ** When a sub-program is executed (OP_Program), a structure of this type @@ -17542,55 +21608,61 @@ struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ + i64 *anExec; /* Event counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ - u8 *aOnceFlag; /* Array of OP_Once flags for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ void *token; /* Copy of SubProgram.token */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ + AuxData *pAuxData; /* Linked list of auxdata allocations */ int nCursor; /* Number of entries in apCsr */ int pc; /* Program Counter in parent (calling) frame */ int nOp; /* Size of aOp array */ int nMem; /* Number of entries in aMem */ - int nOnceFlag; /* Number of entries in aOnceFlag */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ - int nChange; /* Statement changes (Vdbe.nChanges) */ + int nChange; /* Statement changes (Vdbe.nChange) */ + int nDbChange; /* Value of db->nChange */ }; #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) -/* -** A value for VdbeCursor.cacheValid that means the cache is always invalid. -*/ -#define CACHE_STALE 0 - /* ** Internally, the vdbe manipulates nearly all SQL values as Mem ** structures. Each Mem struct may cache multiple representations (string, ** integer etc.) of the same value. */ struct Mem { - sqlite3 *db; /* The associated database connection */ - char *z; /* String or BLOB value */ - double r; /* Real value */ - union { + union MemValue { + double r; /* Real value used when MEM_Real is set in flags */ i64 i; /* Integer value used when MEM_Int is set in flags */ int nZero; /* Used when bit MEM_Zero is set in flags */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ } u; - int n; /* Number of characters in string value, excluding '\0' */ u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ + u8 eSubtype; /* Subtype for this value */ + int n; /* Number of characters in string value, excluding '\0' */ + char *z; /* String or BLOB value */ + /* ShallowCopy only needs to copy the information above */ + char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ + int szMalloc; /* Size of the zMalloc allocation */ + u32 uTemp; /* Transient storage for serial_type in OP_MakeRecord */ + sqlite3 *db; /* The associated database connection */ + void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ #endif - void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ - char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ }; +/* +** Size of struct Mem not including the Mem.zMalloc member or anything that +** follows. +*/ +#define MEMCELLSIZE offsetof(Mem,zMalloc) + /* One or more of the following flags are set to indicate the validOK ** representations of the value stored in the Mem struct. ** @@ -17613,7 +21685,7 @@ struct Mem { #define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ -#define MEM_TypeMask 0x01ff /* Mask of type bits */ +#define MEM_TypeMask 0x81ff /* Mask of type bits */ /* Whenever Mem contains a valid string or blob representation, one of @@ -17627,11 +21699,18 @@ struct Mem { #define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ #define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ #define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ +#define MEM_Subtype 0x8000 /* Mem.eSubtype is valid */ #ifdef SQLITE_OMIT_INCRBLOB #undef MEM_Zero #define MEM_Zero 0x0000 #endif +/* Return TRUE if Mem X contains dynamically allocated content - anything +** that needs to be deallocated to avoid a leak. +*/ +#define VdbeMemDynamic(X) \ + (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) + /* ** Clear any existing type flags from a Mem and replace them with f */ @@ -17647,7 +21726,7 @@ struct Mem { #endif /* -** Each auxilliary data pointer stored by a user defined function +** Each auxiliary data pointer stored by a user defined function ** implementation calling sqlite3_set_auxdata() is stored in an instance ** of this structure. All such structures associated with a single VM ** are stored in a linked list headed at Vdbe.pAuxData. All are destroyed @@ -17662,7 +21741,7 @@ struct AuxData { }; /* -** The "context" argument for a installable function. A pointer to an +** The "context" argument for an installable function. A pointer to an ** instance of this structure is the first argument to the routines used ** implement the SQL functions. ** @@ -17675,27 +21754,16 @@ struct AuxData { ** (Mem) which are only defined there. */ struct sqlite3_context { - FuncDef *pFunc; /* Pointer to function information. MUST BE FIRST */ - Mem s; /* The return value is stored here */ - Mem *pMem; /* Memory cell used to store aggregate context */ - CollSeq *pColl; /* Collating sequence */ - Vdbe *pVdbe; /* The VM that owns this context */ - int iOp; /* Instruction number of OP_Function */ - int isError; /* Error code returned by the function. */ - u8 skipFlag; /* Skip skip accumulator loading if true */ - u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ -}; - -/* -** An Explain object accumulates indented output which is helpful -** in describing recursive data structures. -*/ -struct Explain { - Vdbe *pVdbe; /* Attach the explanation to this Vdbe */ - StrAccum str; /* The string being accumulated */ - int nIndent; /* Number of elements in aIndent */ - u16 aIndent[100]; /* Levels of indentation */ - char zBase[100]; /* Initial space */ + Mem *pOut; /* The return value is stored here */ + FuncDef *pFunc; /* Pointer to function information */ + Mem *pMem; /* Memory cell used to store aggregate context */ + Vdbe *pVdbe; /* The VM that owns this context */ + int iOp; /* Instruction number of OP_Function */ + int isError; /* Error code returned by the function. */ + u8 skipFlag; /* Skip accumulator loading if true */ + u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ + u8 argc; /* Number of arguments */ + sqlite3_value *argv[1]; /* Argument set */ }; /* A bitfield type for use inside of structures. Always follow with :N where @@ -17703,110 +21771,140 @@ struct Explain { */ typedef unsigned bft; /* Bit Field Type */ +typedef struct ScanStatus ScanStatus; +struct ScanStatus { + int addrExplain; /* OP_Explain for loop */ + int addrLoop; /* Address of "loops" counter */ + int addrVisit; /* Address of "rows visited" counter */ + int iSelectID; /* The "Select-ID" for this loop */ + LogEst nEst; /* Estimated output rows per loop */ + char *zName; /* Name of table or index */ +}; + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. ** ** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare() ** is really a pointer to an instance of this structure. -** -** The Vdbe.inVtabMethod variable is set to non-zero for the duration of -** any virtual table method invocations made by the vdbe program. It is -** set to 2 for xDestroy method calls and 1 for all other methods. This -** variable is used for two purposes: to allow xDestroy methods to execute -** "DROP TABLE" statements and to prevent some nasty side effects of -** malloc failure when SQLite is invoked recursively by a virtual table -** method function. */ struct Vdbe { sqlite3 *db; /* The database connection that owns this statement */ + Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ + Parse *pParse; /* Parsing context used to create this Vdbe */ + ynVar nVar; /* Number of entries in aVar[] */ + ynVar nzVar; /* Number of entries in azVar[] */ + u32 magic; /* Magic number for sanity checking */ + int nMem; /* Number of memory locations currently allocated */ + int nCursor; /* Number of slots in apCsr[] */ + u32 cacheCtr; /* VdbeCursor row cache generation counter */ + int pc; /* The program counter */ + int rc; /* Value to return */ + int nChange; /* Number of db changes made since last reset */ + int iStatement; /* Statement number (or 0 if has not opened stmt) */ + i64 iCurrentTime; /* Value of julianday('now') for this statement */ + i64 nFkConstraint; /* Number of imm. FK constraints this VM */ + i64 nStmtDefCons; /* Number of def. constraints when stmt started */ + i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ + + /* When allocating a new Vdbe object, all of the fields below should be + ** initialized to zero or NULL */ + Op *aOp; /* Space to hold the virtual machine's program */ Mem *aMem; /* The memory locations */ Mem **apArg; /* Arguments to currently executing user function */ Mem *aColName; /* Column names to return */ Mem *pResultSet; /* Pointer to an array of results */ - Parse *pParse; /* Parsing context used to create this Vdbe */ - int nMem; /* Number of memory locations currently allocated */ - int nOp; /* Number of instructions in the program */ - int nCursor; /* Number of slots in apCsr[] */ - u32 magic; /* Magic number for sanity checking */ char *zErrMsg; /* Error message written here */ - Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ VdbeCursor **apCsr; /* One element of this array for each open cursor */ Mem *aVar; /* Values for the OP_Variable opcode. */ char **azVar; /* Name of variables */ - ynVar nVar; /* Number of entries in aVar[] */ - ynVar nzVar; /* Number of entries in azVar[] */ - u32 cacheCtr; /* VdbeCursor row cache generation counter */ - int pc; /* The program counter */ - int rc; /* Value to return */ +#ifndef SQLITE_OMIT_TRACE + i64 startTime; /* Time when query started - used for profiling */ +#endif + int nOp; /* Number of instructions in the program */ +#ifdef SQLITE_DEBUG + int rcApp; /* errcode set by sqlite3_result_error_code() */ +#endif u16 nResColumn; /* Number of columns in one row of the result set */ u8 errorAction; /* Recovery action to do in case of an error */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ - bft explain:2; /* True if EXPLAIN present on SQL command */ - bft inVtabMethod:2; /* See comments above */ - bft changeCntOn:1; /* True to update the change-counter */ bft expired:1; /* True if the VM needs to be recompiled */ + bft doingRerun:1; /* True if rerunning after an auto-reprepare */ + bft explain:2; /* True if EXPLAIN present on SQL command */ + bft changeCntOn:1; /* True to update the change-counter */ bft runOnlyOnce:1; /* Automatically expire on reset */ bft usesStmtJournal:1; /* True if uses a statement journal */ bft readOnly:1; /* True for statements that do not write */ bft bIsReader:1; /* True for statements that read */ bft isPrepareV2:1; /* True if prepared with prepare_v2() */ - bft doingRerun:1; /* True if rerunning after an auto-reprepare */ - int nChange; /* Number of db changes made since last reset */ yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ yDbMask lockMask; /* Subset of btreeMask that requires a lock */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ u32 aCounter[5]; /* Counters used by sqlite3_stmt_status() */ -#ifndef SQLITE_OMIT_TRACE - i64 startTime; /* Time when query started - used for profiling */ -#endif - i64 iCurrentTime; /* Value of julianday('now') for this statement */ - i64 nFkConstraint; /* Number of imm. FK constraints this VM */ - i64 nStmtDefCons; /* Number of def. constraints when stmt started */ - i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ char *zSql; /* Text of the SQL statement that generated this */ void *pFree; /* Free this when deleting the vdbe */ -#ifdef SQLITE_ENABLE_TREE_EXPLAIN - Explain *pExplain; /* The explainer */ - char *zExplain; /* Explanation of data structures */ -#endif VdbeFrame *pFrame; /* Parent frame */ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ int nFrame; /* Number of frames in pFrame list */ u32 expmask; /* Binding to these vars invalidates VM */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ - int nOnceFlag; /* Size of array aOnceFlag[] */ - u8 *aOnceFlag; /* Flags for OP_Once */ AuxData *pAuxData; /* Linked list of auxdata allocations */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + i64 *anExec; /* Number of times each op has been executed */ + int nScan; /* Entries in aScan[] */ + ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ +#endif }; /* ** The following are allowed values for Vdbe.magic */ -#define VDBE_MAGIC_INIT 0x26bceaa5 /* Building a VDBE program */ -#define VDBE_MAGIC_RUN 0xbdf20da3 /* VDBE is ready to execute */ -#define VDBE_MAGIC_HALT 0x519c2973 /* VDBE has completed execution */ -#define VDBE_MAGIC_DEAD 0xb606c3c8 /* The VDBE has been deallocated */ +#define VDBE_MAGIC_INIT 0x16bceaa5 /* Building a VDBE program */ +#define VDBE_MAGIC_RUN 0x2df20da3 /* VDBE is ready to execute */ +#define VDBE_MAGIC_HALT 0x319c2973 /* VDBE has completed execution */ +#define VDBE_MAGIC_RESET 0x48fa9f76 /* Reset and ready to run again */ +#define VDBE_MAGIC_DEAD 0x5606c3c8 /* The VDBE has been deallocated */ + +/* +** Structure used to store the context required by the +** sqlite3_preupdate_*() API functions. +*/ +struct PreUpdate { + Vdbe *v; + VdbeCursor *pCsr; /* Cursor to read old values from */ + int op; /* One of SQLITE_INSERT, UPDATE, DELETE */ + u8 *aRecord; /* old.* database record */ + KeyInfo keyinfo; + UnpackedRecord *pUnpacked; /* Unpacked version of aRecord[] */ + UnpackedRecord *pNewUnpacked; /* Unpacked version of new.* record */ + int iNewReg; /* Register for new.* values */ + i64 iKey1; /* First key value passed to hook */ + i64 iKey2; /* Second key value passed to hook */ + Mem *aNew; /* Array of new.* values */ + Table *pTab; /* Schema object being upated */ +}; /* ** Function prototypes */ +SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, int*); +SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); #endif SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int); +SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int, u32*); SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); -SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); -SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); -SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); +SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); +SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor*, i64*); SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); @@ -17823,46 +21921,51 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem*, i64); #else SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem*, double); #endif +SQLITE_PRIVATE void sqlite3VdbeMemInit(Mem*,sqlite3*,u16); SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, int); +SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, u8, u8); SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem*,u8,u8); SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,int,Mem*); SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); -SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p); -#define VdbeMemDynamic(X) \ - (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) -#define VdbeMemRelease(X) \ - if( VdbeMemDynamic(X) ) sqlite3VdbeMemReleaseExternal(X); SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); SQLITE_PRIVATE const char *sqlite3OpcodeName(int); SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); +SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int n); SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +SQLITE_PRIVATE void sqlite3VdbePreUpdateHook(Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int); +#endif SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p); -SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); +SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); -SQLITE_PRIVATE int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); -SQLITE_PRIVATE int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); +SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); +SQLITE_PRIVATE int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); SQLITE_PRIVATE int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); -#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +#if !defined(SQLITE_OMIT_SHARED_CACHE) SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe*); #else # define sqlite3VdbeEnter(X) +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe*); +#else # define sqlite3VdbeLeave(X) #endif @@ -17892,7 +21995,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); #define ExpandBlob(P) SQLITE_OK #endif -#endif /* !defined(_VDBEINT_H_) */ +#endif /* !defined(SQLITE_VDBEINT_H) */ /************** End of vdbeInt.h *********************************************/ /************** Continuing where we left off in status.c *********************/ @@ -17900,12 +22003,34 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); /* ** Variables in which to record status information. */ +#if SQLITE_PTRSIZE>4 +typedef sqlite3_int64 sqlite3StatValueType; +#else +typedef u32 sqlite3StatValueType; +#endif typedef struct sqlite3StatType sqlite3StatType; static SQLITE_WSD struct sqlite3StatType { - int nowValue[10]; /* Current value */ - int mxValue[10]; /* Maximum value */ + sqlite3StatValueType nowValue[10]; /* Current value */ + sqlite3StatValueType mxValue[10]; /* Maximum value */ } sqlite3Stat = { {0,}, {0,} }; +/* +** Elements of sqlite3Stat[] are protected by either the memory allocator +** mutex, or by the pcache1 mutex. The following array determines which. +*/ +static const char statMutex[] = { + 0, /* SQLITE_STATUS_MEMORY_USED */ + 1, /* SQLITE_STATUS_PAGECACHE_USED */ + 1, /* SQLITE_STATUS_PAGECACHE_OVERFLOW */ + 0, /* SQLITE_STATUS_SCRATCH_USED */ + 0, /* SQLITE_STATUS_SCRATCH_OVERFLOW */ + 0, /* SQLITE_STATUS_MALLOC_SIZE */ + 0, /* SQLITE_STATUS_PARSER_STACK */ + 1, /* SQLITE_STATUS_PAGECACHE_SIZE */ + 0, /* SQLITE_STATUS_SCRATCH_SIZE */ + 0, /* SQLITE_STATUS_MALLOC_COUNT */ +}; + /* The "wsdStat" macro will resolve to the status information ** state vector. If writable static data is unsupported on the target, @@ -17922,58 +22047,114 @@ static SQLITE_WSD struct sqlite3StatType { #endif /* -** Return the current value of a status parameter. +** Return the current value of a status parameter. The caller must +** be holding the appropriate mutex. */ -SQLITE_PRIVATE int sqlite3StatusValue(int op){ +#define assert(e) ((void)0) +SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int op){ wsdStatInit; assert( op>=0 && op=0 && op=0 && op=0 && opwsdStat.mxValue[op] ){ wsdStat.mxValue[op] = wsdStat.nowValue[op]; } } +SQLITE_PRIVATE void sqlite3StatusDown(int op, int N){ + wsdStatInit; + assert( N>=0 ); + assert( op>=0 && op=0 && op=0 ); + newValue = (sqlite3StatValueType)X; assert( op>=0 && opwsdStat.mxValue[op] ){ - wsdStat.mxValue[op] = wsdStat.nowValue[op]; + assert( op>=0 && opwsdStat.mxValue[op] ){ + wsdStat.mxValue[op] = newValue; } } /* ** Query status information. -** -** This implementation assumes that reading or writing an aligned -** 32-bit integer is an atomic operation. If that assumption is not true, -** then this routine is not threadsafe. */ -SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ +SQLITE_API int sqlite3_status64( + int op, + sqlite3_int64 *pCurrent, + sqlite3_int64 *pHighwater, + int resetFlag +){ + sqlite3_mutex *pMutex; wsdStatInit; if( op<0 || op>=ArraySize(wsdStat.nowValue) ){ return SQLITE_MISUSE_BKPT; } +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif + pMutex = statMutex[op] ? sqlite3Pcache1Mutex() : sqlite3MallocMutex(); + sqlite3_mutex_enter(pMutex); *pCurrent = wsdStat.nowValue[op]; *pHighwater = wsdStat.mxValue[op]; if( resetFlag ){ wsdStat.mxValue[op] = wsdStat.nowValue[op]; } + sqlite3_mutex_leave(pMutex); + (void)pMutex; /* Prevent warning when SQLITE_THREADSAFE=0 */ return SQLITE_OK; } +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ + sqlite3_int64 iCur = 0, iHwtr = 0; + int rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif + rc = sqlite3_status64(op, &iCur, &iHwtr, resetFlag); + if( rc==0 ){ + *pCurrent = (int)iCur; + *pHighwater = (int)iHwtr; + } + return rc; +} /* ** Query status information for a single database connection @@ -17986,6 +22167,11 @@ SQLITE_API int sqlite3_db_status( int resetFlag /* Reset high-water mark if true */ ){ int rc = SQLITE_OK; /* Return code */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { @@ -18018,6 +22204,7 @@ SQLITE_API int sqlite3_db_status( ** by all pagers associated with the given database connection. The ** highwater mark is meaningless and is returned as zero. */ + case SQLITE_DBSTATUS_CACHE_USED_SHARED: case SQLITE_DBSTATUS_CACHE_USED: { int totalUsed = 0; int i; @@ -18026,7 +22213,11 @@ SQLITE_API int sqlite3_db_status( Btree *pBt = db->aDb[i].pBt; if( pBt ){ Pager *pPager = sqlite3BtreePager(pBt); - totalUsed += sqlite3PagerMemUsed(pPager); + int nByte = sqlite3PagerMemUsed(pPager); + if( op==SQLITE_DBSTATUS_CACHE_USED_SHARED ){ + nByte = nByte / sqlite3BtreeConnectionCount(pBt); + } + totalUsed += nByte; } } sqlite3BtreeLeaveAll(db); @@ -18057,10 +22248,10 @@ SQLITE_API int sqlite3_db_status( + pSchema->idxHash.count + pSchema->fkeyHash.count ); - nByte += sqlite3MallocSize(pSchema->tblHash.ht); - nByte += sqlite3MallocSize(pSchema->trigHash.ht); - nByte += sqlite3MallocSize(pSchema->idxHash.ht); - nByte += sqlite3MallocSize(pSchema->fkeyHash.ht); + nByte += sqlite3_msize(pSchema->tblHash.ht); + nByte += sqlite3_msize(pSchema->trigHash.ht); + nByte += sqlite3_msize(pSchema->idxHash.ht); + nByte += sqlite3_msize(pSchema->fkeyHash.ht); for(p=sqliteHashFirst(&pSchema->trigHash); p; p=sqliteHashNext(p)){ sqlite3DeleteTrigger(db, (Trigger*)sqliteHashData(p)); @@ -18094,7 +22285,7 @@ SQLITE_API int sqlite3_db_status( } db->pnBytesFreed = 0; - *pHighwater = 0; + *pHighwater = 0; /* IMP: R-64479-57858 */ *pCurrent = nByte; break; @@ -18119,7 +22310,9 @@ SQLITE_API int sqlite3_db_status( sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet); } } - *pHighwater = 0; + *pHighwater = 0; /* IMP: R-42420-56072 */ + /* IMP: R-54100-20147 */ + /* IMP: R-29431-39229 */ *pCurrent = nRet; break; } @@ -18129,7 +22322,7 @@ SQLITE_API int sqlite3_db_status( ** have been satisfied. The *pHighwater is always set to zero. */ case SQLITE_DBSTATUS_DEFERRED_FKS: { - *pHighwater = 0; + *pHighwater = 0; /* IMP: R-11967-56545 */ *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0; break; } @@ -18162,7 +22355,7 @@ SQLITE_API int sqlite3_db_status( ** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. ** All other code has file scope. ** -** SQLite processes all times and dates as Julian Day numbers. The +** SQLite processes all times and dates as julian day numbers. The ** dates and times are stored as the number of days since noon ** in Greenwich on November 24, 4714 B.C. according to the Gregorian ** calendar system. @@ -18170,14 +22363,14 @@ SQLITE_API int sqlite3_db_status( ** 1970-01-01 00:00:00 is JD 2440587.5 ** 2000-01-01 00:00:00 is JD 2451544.5 ** -** This implemention requires years to be expressed as a 4-digit number +** This implementation requires years to be expressed as a 4-digit number ** which means that only dates between 0000-01-01 and 9999-12-31 can ** be represented, even though julian day numbers allow a much wider ** range of dates. ** ** The Gregorian calendar system is used for all dates and times, ** even those that predate the Gregorian calendar. Historians usually -** use the Julian calendar for dates prior to 1582-10-15 and for some +** use the julian calendar for dates prior to 1582-10-15 and for some ** dates afterwards, depending on locale. Beware of this difference. ** ** The conversion algorithms are implemented based on descriptions @@ -18189,12 +22382,22 @@ SQLITE_API int sqlite3_db_status( ** Willmann-Bell, Inc ** Richmond, Virginia (USA) */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ #include #ifndef SQLITE_OMIT_DATETIME_FUNCS +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So declare a substitute. The substitute function itself is +** defined in "os_win.c". +*/ +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +struct tm *__cdecl localtime(const time_t *); +#endif /* ** A structure for holding a single date and time. @@ -18210,38 +22413,54 @@ struct DateTime { char validHMS; /* True (1) if h,m,s are valid */ char validJD; /* True (1) if iJD is valid */ char validTZ; /* True (1) if tz is valid */ + char tzSet; /* Timezone was set explicitly */ }; /* -** Convert zDate into one or more integers. Additional arguments -** come in groups of 5 as follows: +** Convert zDate into one or more integers according to the conversion +** specifier zFormat. ** -** N number of digits in the integer -** min minimum allowed value of the integer -** max maximum allowed value of the integer -** nextC first character after the integer -** pVal where to write the integers value. +** zFormat[] contains 4 characters for each integer converted, except for +** the last integer which is specified by three characters. The meaning +** of a four-character format specifiers ABCD is: +** +** A: number of digits to convert. Always "2" or "4". +** B: minimum value. Always "0" or "1". +** C: maximum value, decoded as: +** a: 12 +** b: 14 +** c: 24 +** d: 31 +** e: 59 +** f: 9999 +** D: the separator character, or \000 to indicate this is the +** last number to convert. +** +** Example: To translate an ISO-8601 date YYYY-MM-DD, the format would +** be "40f-21a-20c". The "40f-" indicates the 4-digit year followed by "-". +** The "21a-" indicates the 2-digit month followed by "-". The "20c" indicates +** the 2-digit day which is the last integer in the set. ** -** Conversions continue until one with nextC==0 is encountered. ** The function returns the number of successful conversions. */ -static int getDigits(const char *zDate, ...){ +static int getDigits(const char *zDate, const char *zFormat, ...){ + /* The aMx[] array translates the 3rd character of each format + ** spec into a max size: a b c d e f */ + static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; va_list ap; - int val; - int N; - int min; - int max; - int nextC; - int *pVal; int cnt = 0; - va_start(ap, zDate); + char nextC; + va_start(ap, zFormat); do{ - N = va_arg(ap, int); - min = va_arg(ap, int); - max = va_arg(ap, int); - nextC = va_arg(ap, int); - pVal = va_arg(ap, int*); + char N = zFormat[0] - '0'; + char min = zFormat[1] - '0'; + int val = 0; + u16 max; + + assert( zFormat[2]>='a' && zFormat[2]<='f' ); + max = aMx[zFormat[2] - 'a']; + nextC = zFormat[3]; val = 0; while( N-- ){ if( !sqlite3Isdigit(*zDate) ){ @@ -18250,12 +22469,13 @@ static int getDigits(const char *zDate, ...){ val = val*10 + *zDate - '0'; zDate++; } - if( valmax || (nextC!=0 && nextC!=*zDate) ){ + if( val<(int)min || val>(int)max || (nextC!=0 && nextC!=*zDate) ){ goto end_getDigits; } - *pVal = val; + *va_arg(ap,int*) = val; zDate++; cnt++; + zFormat += 4; }while( nextC ); end_getDigits: va_end(ap); @@ -18296,13 +22516,14 @@ static int parseTimezone(const char *zDate, DateTime *p){ return c!=0; } zDate++; - if( getDigits(zDate, 2, 0, 14, ':', &nHr, 2, 0, 59, 0, &nMn)!=2 ){ + if( getDigits(zDate, "20b:20e", &nHr, &nMn)!=2 ){ return 1; } zDate += 5; p->tz = sgn*(nMn + nHr*60); zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } + p->tzSet = 1; return *zDate!=0; } @@ -18316,13 +22537,13 @@ zulu_time: static int parseHhMmSs(const char *zDate, DateTime *p){ int h, m, s; double ms = 0.0; - if( getDigits(zDate, 2, 0, 24, ':', &h, 2, 0, 59, 0, &m)!=2 ){ + if( getDigits(zDate, "20c:20e", &h, &m)!=2 ){ return 1; } zDate += 5; if( *zDate==':' ){ zDate++; - if( getDigits(zDate, 2, 0, 59, 0, &s)!=1 ){ + if( getDigits(zDate, "20e", &s)!=1 ){ return 1; } zDate += 2; @@ -18410,7 +22631,7 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ }else{ neg = 0; } - if( getDigits(zDate,4,0,9999,'-',&Y,2,1,12,'-',&M,2,1,31,0,&D)!=3 ){ + if( getDigits(zDate, "40f-21a-21d", &Y, &M, &D)!=3 ){ return 1; } zDate += 10; @@ -18449,7 +22670,7 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ } /* -** Attempt to parse the given string into a Julian Day Number. Return +** Attempt to parse the given string into a julian day number. Return ** the number of errors. ** ** The following are acceptable forms for the input string: @@ -18500,7 +22721,7 @@ static void computeYMD(DateTime *p){ A = Z + 1 + A - (A/4); B = A + 1524; C = (int)((B - 122.1)/365.25); - D = (36525*C)/100; + D = (36525*(C&32767))/100; E = (int)((B-D)/30.6001); X1 = (int)(30.6001*E); p->D = B - D - X1; @@ -18545,6 +22766,7 @@ static void clearYMD_HMS_TZ(DateTime *p){ p->validTZ = 0; } +#ifndef SQLITE_OMIT_LOCALTIME /* ** On recent Windows platforms, the localtime_s() function is available ** as part of the "Secure CRT". It is essentially equivalent to @@ -18557,12 +22779,12 @@ static void clearYMD_HMS_TZ(DateTime *p){ ** already, check for an MSVC build environment that provides ** localtime_s(). */ -#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ - defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S \ + && defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#undef HAVE_LOCALTIME_S #define HAVE_LOCALTIME_S 1 #endif -#ifndef SQLITE_OMIT_LOCALTIME /* ** The following routine implements the rough equivalent of localtime_r() ** using whatever operating-system specific localtime facility that @@ -18578,8 +22800,7 @@ static void clearYMD_HMS_TZ(DateTime *p){ */ static int osLocaltime(time_t *t, struct tm *pTm){ int rc; -#if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \ - && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S) +#if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S struct tm *pX; #if SQLITE_THREADSAFE>0 sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); @@ -18596,7 +22817,7 @@ static int osLocaltime(time_t *t, struct tm *pTm){ #ifndef SQLITE_OMIT_BUILTIN_TEST if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; #endif -#if defined(HAVE_LOCALTIME_R) && HAVE_LOCALTIME_R +#if HAVE_LOCALTIME_R rc = localtime_r(t, pTm)==0; #else rc = localtime_s(pTm, t); @@ -18735,13 +22956,18 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ } #ifndef SQLITE_OMIT_LOCALTIME else if( strcmp(z, "utc")==0 ){ - sqlite3_int64 c1; - computeJD(p); - c1 = localtimeOffset(p, pCtx, &rc); - if( rc==SQLITE_OK ){ - p->iJD -= c1; - clearYMD_HMS_TZ(p); - p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + if( p->tzSet==0 ){ + sqlite3_int64 c1; + computeJD(p); + c1 = localtimeOffset(p, pCtx, &rc); + if( rc==SQLITE_OK ){ + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + } + p->tzSet = 1; + }else{ + rc = SQLITE_OK; } } #endif @@ -19020,7 +23246,7 @@ static void dateFunc( ** %f ** fractional seconds SS.SSS ** %H hour 00-24 ** %j day of year 000-366 -** %J ** Julian day number +** %J ** julian day number ** %m month 01-12 ** %M minute 00-59 ** %s seconds since 1970-01-01 @@ -19040,8 +23266,10 @@ static void strftimeFunc( size_t i,j; char *z; sqlite3 *db; - const char *zFmt = (const char*)sqlite3_value_text(argv[0]); + const char *zFmt; char zBuf[100]; + if( argc==0 ) return; + zFmt = (const char*)sqlite3_value_text(argv[0]); if( zFmt==0 || isDate(context, argc-1, argv+1, &x) ) return; db = sqlite3_context_db_handle(context); for(i=0, n=1; zFmt[i]; i++, n++){ @@ -19087,7 +23315,7 @@ static void strftimeFunc( sqlite3_result_error_toobig(context); return; }else{ - z = sqlite3DbMallocRaw(db, (int)n); + z = sqlite3DbMallocRawNN(db, (int)n); if( z==0 ){ sqlite3_result_error_nomem(context); return; @@ -19223,7 +23451,6 @@ static void currentTimeFunc( ){ time_t t; char *zFormat = (char *)sqlite3_user_data(context); - sqlite3 *db; sqlite3_int64 iT; struct tm *pTm; struct tm sNow; @@ -19235,7 +23462,7 @@ static void currentTimeFunc( iT = sqlite3StmtCurrentTime(context); if( iT<=0 ) return; t = iT/1000 - 10000*(sqlite3_int64)21086676; -#ifdef HAVE_GMTIME_R +#if HAVE_GMTIME_R pTm = gmtime_r(&t, &sNow); #else sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); @@ -19256,29 +23483,23 @@ static void currentTimeFunc( ** external linkage. */ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ - static SQLITE_WSD FuncDef aDateTimeFuncs[] = { + static FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - FUNCTION(julianday, -1, 0, 0, juliandayFunc ), - FUNCTION(date, -1, 0, 0, dateFunc ), - FUNCTION(time, -1, 0, 0, timeFunc ), - FUNCTION(datetime, -1, 0, 0, datetimeFunc ), - FUNCTION(strftime, -1, 0, 0, strftimeFunc ), - FUNCTION(current_time, 0, 0, 0, ctimeFunc ), - FUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), - FUNCTION(current_date, 0, 0, 0, cdateFunc ), + DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), + DFUNCTION(date, -1, 0, 0, dateFunc ), + DFUNCTION(time, -1, 0, 0, timeFunc ), + DFUNCTION(datetime, -1, 0, 0, datetimeFunc ), + DFUNCTION(strftime, -1, 0, 0, strftimeFunc ), + DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), + DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), + DFUNCTION(current_date, 0, 0, 0, cdateFunc ), #else STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), STR_FUNCTION(current_date, 0, "%Y-%m-%d", 0, currentTimeFunc), STR_FUNCTION(current_timestamp, 0, "%Y-%m-%d %H:%M:%S", 0, currentTimeFunc), #endif }; - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aDateTimeFuncs); - - for(i=0; ipMethods ){ - rc = pId->pMethods->xClose(pId); + pId->pMethods->xClose(pId); pId->pMethods = 0; } - return rc; } SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file *id, void *pBuf, int amt, i64 offset){ DO_OS_MALLOC_TEST(id); @@ -19394,8 +23634,8 @@ SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ #ifdef SQLITE_TEST if( op!=SQLITE_FCNTL_COMMIT_PHASETWO ){ /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite - ** is using a regular VFS, it is called after the corresponding - ** transaction has been committed. Injecting a fault at this point + ** is using a regular VFS, it is called after the corresponding + ** transaction has been committed. Injecting a fault at this point ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM ** but the transaction is committed anyway. ** @@ -19464,10 +23704,10 @@ SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ ** VFS methods. */ SQLITE_PRIVATE int sqlite3OsOpen( - sqlite3_vfs *pVfs, - const char *zPath, - sqlite3_file *pFile, - int flags, + sqlite3_vfs *pVfs, + const char *zPath, + sqlite3_file *pFile, + int flags, int *pFlagsOut ){ int rc; @@ -19486,18 +23726,18 @@ SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dir return pVfs->xDelete(pVfs, zPath, dirSync); } SQLITE_PRIVATE int sqlite3OsAccess( - sqlite3_vfs *pVfs, - const char *zPath, - int flags, + sqlite3_vfs *pVfs, + const char *zPath, + int flags, int *pResOut ){ DO_OS_MALLOC_TEST(0); return pVfs->xAccess(pVfs, zPath, flags, pResOut); } SQLITE_PRIVATE int sqlite3OsFullPathname( - sqlite3_vfs *pVfs, - const char *zPath, - int nPathOut, + sqlite3_vfs *pVfs, + const char *zPath, + int nPathOut, char *zPathOut ){ DO_OS_MALLOC_TEST(0); @@ -19524,6 +23764,9 @@ SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufO SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); } +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs *pVfs){ + return pVfs->xGetLastError ? pVfs->xGetLastError(pVfs, 0, 0) : 0; +} SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ int rc; /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() @@ -19543,13 +23786,13 @@ SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p } SQLITE_PRIVATE int sqlite3OsOpenMalloc( - sqlite3_vfs *pVfs, - const char *zFile, - sqlite3_file **ppFile, + sqlite3_vfs *pVfs, + const char *zFile, + sqlite3_file **ppFile, int flags, int *pOutFlags ){ - int rc = SQLITE_NOMEM; + int rc; sqlite3_file *pFile; pFile = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile); if( pFile ){ @@ -19559,15 +23802,15 @@ SQLITE_PRIVATE int sqlite3OsOpenMalloc( }else{ *ppFile = pFile; } + }else{ + rc = SQLITE_NOMEM_BKPT; } return rc; } -SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *pFile){ - int rc = SQLITE_OK; +SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *pFile){ assert( pFile ); - rc = sqlite3OsClose(pFile); + sqlite3OsClose(pFile); sqlite3_free(pFile); - return rc; } /* @@ -19578,7 +23821,7 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *pFile){ */ SQLITE_PRIVATE int sqlite3OsInit(void){ void *p = sqlite3_malloc(10); - if( p==0 ) return SQLITE_NOMEM; + if( p==0 ) return SQLITE_NOMEM_BKPT; sqlite3_free(p); return sqlite3_os_init(); } @@ -19645,6 +23888,10 @@ SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ int rc = sqlite3_initialize(); if( rc ) return rc; #endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pVfs==0 ) return SQLITE_MISUSE_BKPT; +#endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); @@ -19701,6 +23948,7 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ ** during a hash table resize is a benign fault. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_BUILTIN_TEST @@ -19782,6 +24030,7 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void){ ** are merely placeholders. Real drivers must be substituted using ** sqlite3_config() before SQLite will operate. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is the default. It is @@ -19868,6 +24117,7 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** be necessary when compiling for Delphi, ** for example. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is the default. It is @@ -19905,9 +24155,9 @@ static malloc_zone_t* _sqliteZone_; ** The malloc.h header file is needed for malloc_usable_size() function ** on some systems (e.g. Linux). */ -#if defined(HAVE_MALLOC_H) && defined(HAVE_MALLOC_USABLE_SIZE) -# define SQLITE_USE_MALLOC_H -# define SQLITE_USE_MALLOC_USABLE_SIZE +#if HAVE_MALLOC_H && HAVE_MALLOC_USABLE_SIZE +# define SQLITE_USE_MALLOC_H 1 +# define SQLITE_USE_MALLOC_USABLE_SIZE 1 /* ** The MSVCRT has malloc_usable_size(), but it is called _msize(). The ** use of _msize() is automatic, but can be disabled by compiling with @@ -19998,10 +24248,11 @@ static void sqlite3MemFree(void *pPrior){ */ static int sqlite3MemSize(void *pPrior){ #ifdef SQLITE_MALLOCSIZE - return pPrior ? (int)SQLITE_MALLOCSIZE(pPrior) : 0; + assert( pPrior!=0 ); + return (int)SQLITE_MALLOCSIZE(pPrior); #else sqlite3_int64 *p; - if( pPrior==0 ) return 0; + assert( pPrior!=0 ); p = (sqlite3_int64*)pPrior; p--; return (int)p[0]; @@ -20014,7 +24265,7 @@ static int sqlite3MemSize(void *pPrior){ ** ** For this low-level interface, we know that pPrior!=0. Cases where ** pPrior==0 while have been intercepted by higher-level routine and -** redirected to xMalloc. Similarly, we know that nByte>0 becauses +** redirected to xMalloc. Similarly, we know that nByte>0 because ** cases where nByte<=0 will have been intercepted by higher-level ** routines and redirected to xFree. */ @@ -20143,6 +24394,7 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** This file contains implementations of the low-level memory allocation ** routines specified in the sqlite3_mem_methods object. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is used only if the @@ -20517,7 +24769,7 @@ SQLITE_PRIVATE void sqlite3MemdebugSetType(void *p, u8 eType){ ** This routine is designed for use within an assert() statement, to ** verify the type of an allocation. For example: ** -** assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); +** assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); */ SQLITE_PRIVATE int sqlite3MemdebugHasType(void *p, u8 eType){ int rc = 1; @@ -20539,7 +24791,7 @@ SQLITE_PRIVATE int sqlite3MemdebugHasType(void *p, u8 eType){ ** This routine is designed for use within an assert() statement, to ** verify the type of an allocation. For example: ** -** assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); +** assert( sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); */ SQLITE_PRIVATE int sqlite3MemdebugNoType(void *p, u8 eType){ int rc = 1; @@ -20677,6 +24929,7 @@ SQLITE_PRIVATE int sqlite3MemdebugMallocCount(){ ** This version of the memory allocation subsystem is included ** in the build only if SQLITE_ENABLE_MEMSYS3 is defined. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is only built into the library @@ -21129,7 +25382,7 @@ static void memsys3FreeUnsafe(void *pOld){ */ static int memsys3Size(void *p){ Mem3Block *pBlock; - if( p==0 ) return 0; + assert( p!=0 ); pBlock = (Mem3Block*)p; assert( (pBlock[-1].u.hdr.size4x&1)!=0 ); return (pBlock[-1].u.hdr.size4x&~3)*2 - 4; @@ -21368,10 +25621,10 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ ** ** This memory allocator uses the following algorithm: ** -** 1. All memory allocations sizes are rounded up to a power of 2. +** 1. All memory allocation sizes are rounded up to a power of 2. ** ** 2. If two adjacent free blocks are the halves of a larger block, -** then the two blocks are coalesed into the single larger block. +** then the two blocks are coalesced into the single larger block. ** ** 3. New memory is allocated from the first available free block. ** @@ -21391,6 +25644,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ ** The sqlite3_status() logic tracks the maximum values of n and M so ** that an application can, at any time, verify this constraint. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is used only when @@ -21444,6 +25698,7 @@ static SQLITE_WSD struct Mem5Global { */ sqlite3_mutex *mutex; +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Performance statistics */ @@ -21455,11 +25710,12 @@ static SQLITE_WSD struct Mem5Global { u32 maxOut; /* Maximum instantaneous currentOut */ u32 maxCount; /* Maximum instantaneous currentCount */ u32 maxRequest; /* Largest allocation (exclusive of internal frag) */ +#endif /* ** Lists of free blocks. aiFreelist[0] is a list of free blocks of ** size mem5.szAtom. aiFreelist[1] holds blocks of size szAtom*2. - ** and so forth. + ** aiFreelist[2] holds free blocks of size szAtom*4. And so forth. */ int aiFreelist[LOGMAX+1]; @@ -21525,9 +25781,7 @@ static void memsys5Link(int i, int iLogsize){ } /* -** If the STATIC_MEM mutex is not already held, obtain it now. The mutex -** will already be held (obtained by code in malloc.c) if -** sqlite3GlobalConfig.bMemStat is true. +** Obtain or release the mutex needed to access global data structures. */ static void memsys5Enter(void){ sqlite3_mutex_enter(mem5.mutex); @@ -21537,17 +25791,15 @@ static void memsys5Leave(void){ } /* -** Return the size of an outstanding allocation, in bytes. The -** size returned omits the 8-byte header overhead. This only -** works for chunks that are currently checked out. +** Return the size of an outstanding allocation, in bytes. +** This only works for chunks that are currently checked out. */ static int memsys5Size(void *p){ - int iSize = 0; - if( p ){ - int i = (int)(((u8 *)p-mem5.zPool)/mem5.szAtom); - assert( i>=0 && i=0 && i0 ); + /* No more than 1GiB per allocation */ + if( nByte > 0x40000000 ) return 0; + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* Keep track of the maximum allocation request. Even unfulfilled ** requests are counted */ if( (u32)nByte>mem5.maxRequest ){ mem5.maxRequest = nByte; } +#endif - /* Abort if the requested allocation size is larger than the largest - ** power of two that we can represent using 32-bit signed integers. - */ - if( nByte > 0x40000000 ){ - return 0; - } /* Round nByte up to the next valid power of two */ - for(iFullSz=mem5.szAtom, iLogsize=0; iFullSz0 ); assert( mem5.currentOut>=(size*mem5.szAtom) ); mem5.currentCount--; mem5.currentOut -= size*mem5.szAtom; assert( mem5.currentOut>0 || mem5.currentCount==0 ); assert( mem5.currentCount>0 || mem5.currentOut==0 ); +#endif mem5.aCtrl[iBlock] = CTRL_FREE | iLogsize; while( ALWAYS(iLogsize>iLogsize) & 1 ){ iBuddy = iBlock - size; + assert( iBuddy>=0 ); }else{ iBuddy = iBlock + size; + if( iBuddy>=mem5.nBlock ) break; } - assert( iBuddy>=0 ); - if( (iBuddy+(1<mem5.nBlock ) break; if( mem5.aCtrl[iBuddy]!=(CTRL_FREE | iLogsize) ) break; memsys5Unlink(iBuddy, iLogsize); iLogsize++; @@ -21741,13 +25997,11 @@ static void *memsys5Realloc(void *pPrior, int nBytes){ if( nBytes<=nOld ){ return pPrior; } - memsys5Enter(); - p = memsys5MallocUnsafe(nBytes); + p = memsys5Malloc(nBytes); if( p ){ memcpy(p, pPrior, nOld); - memsys5FreeUnsafe(pPrior); + memsys5Free(pPrior); } - memsys5Leave(); return p; } @@ -21934,6 +26188,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ ** ** This file contains code that is common across all mutex implementations. */ +/* #include "sqliteInt.h" */ #if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT) /* @@ -21942,7 +26197,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ ** allocate a mutex while the system is uninitialized. */ static SQLITE_WSD int mutexIsInit = 0; -#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_DEBUG && !defined(SQLITE_MUTEX_OMIT) */ #ifndef SQLITE_MUTEX_OMIT @@ -21965,11 +26220,18 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ }else{ pFrom = sqlite3NoopMutex(); } - memcpy(pTo, pFrom, offsetof(sqlite3_mutex_methods, xMutexAlloc)); - memcpy(&pTo->xMutexFree, &pFrom->xMutexFree, - sizeof(*pTo) - offsetof(sqlite3_mutex_methods, xMutexFree)); + pTo->xMutexInit = pFrom->xMutexInit; + pTo->xMutexEnd = pFrom->xMutexEnd; + pTo->xMutexFree = pFrom->xMutexFree; + pTo->xMutexEnter = pFrom->xMutexEnter; + pTo->xMutexTry = pFrom->xMutexTry; + pTo->xMutexLeave = pFrom->xMutexLeave; + pTo->xMutexHeld = pFrom->xMutexHeld; + pTo->xMutexNotheld = pFrom->xMutexNotheld; + sqlite3MemoryBarrier(); pTo->xMutexAlloc = pFrom->xMutexAlloc; } + assert( sqlite3GlobalConfig.mutex.xMutexInit ); rc = sqlite3GlobalConfig.mutex.xMutexInit(); #ifdef SQLITE_DEBUG @@ -22002,7 +26264,9 @@ SQLITE_PRIVATE int sqlite3MutexEnd(void){ SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; + if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -22011,6 +26275,7 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ return 0; } assert( GLOBAL(int, mutexIsInit) ); + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -22019,6 +26284,7 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ */ SQLITE_API void sqlite3_mutex_free(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexFree ); sqlite3GlobalConfig.mutex.xMutexFree(p); } } @@ -22029,6 +26295,7 @@ SQLITE_API void sqlite3_mutex_free(sqlite3_mutex *p){ */ SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexEnter ); sqlite3GlobalConfig.mutex.xMutexEnter(p); } } @@ -22040,6 +26307,7 @@ SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex *p){ SQLITE_API int sqlite3_mutex_try(sqlite3_mutex *p){ int rc = SQLITE_OK; if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexTry ); return sqlite3GlobalConfig.mutex.xMutexTry(p); } return rc; @@ -22053,6 +26321,7 @@ SQLITE_API int sqlite3_mutex_try(sqlite3_mutex *p){ */ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexLeave ); sqlite3GlobalConfig.mutex.xMutexLeave(p); } } @@ -22063,9 +26332,11 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ ** intended for use inside assert() statements. */ SQLITE_API int sqlite3_mutex_held(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } #endif @@ -22101,6 +26372,7 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ ** that does error checking on mutexes to make sure they are being ** called correctly. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_MUTEX_OMIT @@ -22182,7 +26454,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1]; + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_VFS3 - 1]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: @@ -22195,8 +26467,12 @@ static sqlite3_mutex *debugMutexAlloc(int id){ break; } default: { - assert( id-2 >= 0 ); - assert( id-2 < (int)(sizeof(aStatic)/sizeof(aStatic[0])) ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( id-2<0 || id-2>=ArraySize(aStatic) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif pNew = &aStatic[id-2]; pNew->id = id; break; @@ -22211,8 +26487,13 @@ static sqlite3_mutex *debugMutexAlloc(int id){ static void debugMutexFree(sqlite3_mutex *pX){ sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; assert( p->cnt==0 ); - assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); - sqlite3_free(p); + if( p->id==SQLITE_MUTEX_RECURSIVE || p->id==SQLITE_MUTEX_FAST ){ + sqlite3_free(p); + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + (void)SQLITE_MISUSE_BKPT; +#endif + } } /* @@ -22295,6 +26576,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ************************************************************************* ** This file contains the C functions that implement mutexes for pthreads */ +/* #include "sqliteInt.h" */ /* ** The code in this file is only used if we are compiling threadsafe @@ -22323,15 +26605,19 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ */ struct sqlite3_mutex { pthread_mutex_t mutex; /* Mutex controlling the lock */ -#if SQLITE_MUTEX_NREF +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) int id; /* Mutex type */ +#endif +#if SQLITE_MUTEX_NREF volatile int nRef; /* Number of entrances */ volatile pthread_t owner; /* Thread that is within this mutex */ int trace; /* True to trace changes */ #endif }; #if SQLITE_MUTEX_NREF -#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0, 0, (pthread_t)0, 0 } +#define SQLITE3_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER,0,0,(pthread_t)0,0} +#elif defined(SQLITE_ENABLE_API_ARMOR) +#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0 } #else #define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER } #endif @@ -22361,6 +26647,19 @@ static int pthreadMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization +** and also for the implementation of xShmBarrier in the VFS in cases +** where SQLite is compiled without mutexes. +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) && GCC_VERSION>=4001000 + __sync_synchronize(); +#endif +} + /* ** Initialize and deinitialize the mutex subsystem. */ @@ -22386,6 +26685,9 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 **
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -22422,6 +26724,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; sqlite3_mutex *p; @@ -22440,9 +26745,6 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&p->mutex, &recursiveAttr); pthread_mutexattr_destroy(&recursiveAttr); -#endif -#if SQLITE_MUTEX_NREF - p->id = iType; #endif } break; @@ -22450,23 +26752,24 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ case SQLITE_MUTEX_FAST: { p = sqlite3MallocZero( sizeof(*p) ); if( p ){ -#if SQLITE_MUTEX_NREF - p->id = iType; -#endif pthread_mutex_init(&p->mutex, 0); } break; } default: { - assert( iType-2 >= 0 ); - assert( iType-2 < ArraySize(staticMutexes) ); - p = &staticMutexes[iType-2]; -#if SQLITE_MUTEX_NREF - p->id = iType; +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } #endif + p = &staticMutexes[iType-2]; break; } } +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + if( p ) p->id = iType; +#endif return p; } @@ -22478,9 +26781,18 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ */ static void pthreadMutexFree(sqlite3_mutex *p){ assert( p->nRef==0 ); - assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); - pthread_mutex_destroy(&p->mutex); - sqlite3_free(p); +#if SQLITE_ENABLE_API_ARMOR + if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ) +#endif + { + pthread_mutex_destroy(&p->mutex); + sqlite3_free(p); + } +#ifdef SQLITE_ENABLE_API_ARMOR + else{ + (void)SQLITE_MISUSE_BKPT; + } +#endif } /* @@ -22494,6 +26806,7 @@ static void pthreadMutexFree(sqlite3_mutex *p){ ** can enter. If the same thread tries to enter any other kind of mutex ** more than once, the behavior is undefined. */ +#define assert(e) ((void)0) static void pthreadMutexEnter(sqlite3_mutex *p){ assert( p->id==SQLITE_MUTEX_RECURSIVE || pthreadMutexNotheld(p) ); @@ -22654,6 +26967,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ************************************************************************* ** This file contains the C functions that implement mutexes for Win32. */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN /* @@ -22692,24 +27006,14 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; -# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X -#else -# define OSTRACE(X) -#endif - /* ** Macros for performance tracing. Normally turned off. Only works ** on i486 hardware. */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing +/* +** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. */ /************** Include hwtime.h in the middle of os_common.h ****************/ @@ -22729,8 +27033,8 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H /* ** The following routine only works on pentium-class (or newer) processors. @@ -22798,7 +27102,7 @@ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } #endif -#endif /* !defined(_HWTIME_H_) */ +#endif /* !defined(SQLITE_HWTIME_H) */ /************** End of hwtime.h **********************************************/ /************** Continuing where we left off in os_common.h ******************/ @@ -22819,14 +27123,14 @@ static sqlite_uint64 g_elapsed; ** of code will give us the ability to simulate a disk I/O error. This ** is used for testing the I/O recovery logic. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ -SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ -SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ -SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ -SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ -SQLITE_API int sqlite3_diskfull_pending = 0; -SQLITE_API int sqlite3_diskfull = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_io_error_hit; +SQLITE_API extern int sqlite3_io_error_hardhit; +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_persist; +SQLITE_API extern int sqlite3_io_error_benign; +SQLITE_API extern int sqlite3_diskfull_pending; +SQLITE_API extern int sqlite3_diskfull; #define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ @@ -22852,17 +27156,17 @@ static void local_ioerr(){ #define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) -#endif +#endif /* defined(SQLITE_TEST) */ /* ** When testing, keep a count of the number of open files. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_open_file_count = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_open_file_count; #define OpenCounter(X) sqlite3_open_file_count+=(X) #else #define OpenCounter(X) -#endif +#endif /* defined(SQLITE_TEST) */ #endif /* !defined(_OS_COMMON_H_) */ @@ -22888,8 +27192,8 @@ SQLITE_API int sqlite3_open_file_count = 0; ** ** This file contains code that is specific to Windows. */ -#ifndef _OS_WIN_H_ -#define _OS_WIN_H_ +#ifndef SQLITE_OS_WIN_H +#define SQLITE_OS_WIN_H /* ** Include the primary Windows SDK header file. @@ -22940,7 +27244,28 @@ SQLITE_API int sqlite3_open_file_count = 0; # define SQLITE_OS_WINRT 0 #endif -#endif /* _OS_WIN_H_ */ +/* +** For WinCE, some API function parameters do not appear to be declared as +** volatile. +*/ +#if SQLITE_OS_WINCE +# define SQLITE_WIN32_VOLATILE +#else +# define SQLITE_WIN32_VOLATILE volatile +#endif + +/* +** For some Windows sub-platforms, the _beginthreadex() / _endthreadex() +** functions are not available (e.g. those not using MSVC, Cygwin, etc). +*/ +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ + SQLITE_THREADSAFE>0 && !defined(__CYGWIN__) +# define SQLITE_OS_WIN_THREADS 1 +#else +# define SQLITE_OS_WIN_THREADS 0 +#endif + +#endif /* SQLITE_OS_WIN_H */ /************** End of os_win.h **********************************************/ /************** Continuing where we left off in mutex_w32.c ******************/ @@ -22998,6 +27323,24 @@ static int winMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization +** and also for the xShmBarrier method of the VFS in cases when SQLite is +** compiled without mutexes (SQLITE_THREADSAFE=0). +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) + __sync_synchronize(); +#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ + defined(_MSC_VER) && _MSC_VER>=1300 + _ReadWriteBarrier(); +#elif defined(MemoryBarrier) + MemoryBarrier(); +#endif +} + /* ** Initialize and deinitialize the mutex subsystem. */ @@ -23010,6 +27353,9 @@ static sqlite3_mutex winMutex_staticMutexes[] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; @@ -23020,7 +27366,7 @@ static int winMutex_isNt = -1; /* <0 means "need to query" */ ** of the sqlite3_initialize() and sqlite3_shutdown() processing, the ** "interlocked" magic used here is probably not strictly necessary. */ -static LONG volatile winMutex_lock = 0; +static LONG SQLITE_WIN32_VOLATILE winMutex_lock = 0; SQLITE_API int sqlite3_win32_is_nt(void); /* os_win.c */ SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ @@ -23081,6 +27427,9 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 **
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -23115,8 +27464,8 @@ static sqlite3_mutex *winMutexAlloc(int iType){ case SQLITE_MUTEX_RECURSIVE: { p = sqlite3MallocZero( sizeof(*p) ); if( p ){ -#ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_DEBUG #ifdef SQLITE_WIN32_MUTEX_TRACE_DYNAMIC p->trace = 1; #endif @@ -23130,12 +27479,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){ break; } default: { - assert( iType-2 >= 0 ); - assert( iType-2 < ArraySize(winMutex_staticMutexes) ); - assert( winMutex_isInit==1 ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(winMutex_staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif p = &winMutex_staticMutexes[iType-2]; -#ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_DEBUG #ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC p->trace = 1; #endif @@ -23154,13 +27506,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){ */ static void winMutexFree(sqlite3_mutex *p){ assert( p ); -#ifdef SQLITE_DEBUG assert( p->nRef==0 && p->owner==0 ); - assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ){ + DeleteCriticalSection(&p->mutex); + sqlite3_free(p); + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + (void)SQLITE_MISUSE_BKPT; #endif - assert( winMutex_isInit==1 ); - DeleteCriticalSection(&p->mutex); - sqlite3_free(p); + } } /* @@ -23307,6 +27661,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** ** Memory allocation functions used throughout sqlite. */ +/* #include "sqliteInt.h" */ /* #include */ /* @@ -23339,16 +27694,7 @@ typedef struct ScratchFreeslot { */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ - - /* - ** The alarm callback and its arguments. The mem0.mutex lock will - ** be held while the callback is running. Recursive calls into - ** the memory subsystem are allowed, but no new callbacks will be - ** issued. - */ - sqlite3_int64 alarmThreshold; - void (*alarmCallback)(void*, sqlite3_int64,int); - void *alarmArg; + sqlite3_int64 alarmThreshold; /* The soft heap limit */ /* ** Pointers to the end of sqlite3GlobalConfig.pScratch memory @@ -23365,54 +27711,32 @@ static SQLITE_WSD struct Mem0Global { ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; -} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; +} mem0 = { 0, 0, 0, 0, 0, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) /* -** This routine runs when the memory allocator sees that the -** total memory allocation is about to exceed the soft heap -** limit. +** Return the memory allocator mutex. sqlite3_status() needs it. */ -static void softHeapLimitEnforcer( - void *NotUsed, - sqlite3_int64 NotUsed2, - int allocSize -){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - sqlite3_release_memory(allocSize); -} - -/* -** Change the alarm callback -*/ -static int sqlite3MemoryAlarm( - void(*xCallback)(void *pArg, sqlite3_int64 used,int N), - void *pArg, - sqlite3_int64 iThreshold -){ - int nUsed; - sqlite3_mutex_enter(mem0.mutex); - mem0.alarmCallback = xCallback; - mem0.alarmArg = pArg; - mem0.alarmThreshold = iThreshold; - nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed); - sqlite3_mutex_leave(mem0.mutex); - return SQLITE_OK; +SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void){ + return mem0.mutex; } #ifndef SQLITE_OMIT_DEPRECATED /* -** Deprecated external interface. Internal/core SQLite code -** should call sqlite3MemoryAlarm. +** Deprecated external interface. It used to set an alarm callback +** that was invoked when memory usage grew too large. Now it is a +** no-op. */ SQLITE_API int sqlite3_memory_alarm( void(*xCallback)(void *pArg, sqlite3_int64 used,int N), void *pArg, sqlite3_int64 iThreshold ){ - return sqlite3MemoryAlarm(xCallback, pArg, iThreshold); + (void)xCallback; + (void)pArg; + (void)iThreshold; + return SQLITE_OK; } #endif @@ -23423,19 +27747,21 @@ SQLITE_API int sqlite3_memory_alarm( SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; sqlite3_int64 excess; + sqlite3_int64 nUsed; #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); if( rc ) return -1; #endif sqlite3_mutex_enter(mem0.mutex); priorLimit = mem0.alarmThreshold; - sqlite3_mutex_leave(mem0.mutex); - if( n<0 ) return priorLimit; - if( n>0 ){ - sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, n); - }else{ - sqlite3MemoryAlarm(0, 0, 0); + if( n<0 ){ + sqlite3_mutex_leave(mem0.mutex); + return priorLimit; } + mem0.alarmThreshold = n; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (n>0 && n<=nUsed); + sqlite3_mutex_leave(mem0.mutex); excess = sqlite3_memory_used() - n; if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); return priorLimit; @@ -23449,13 +27775,12 @@ SQLITE_API void sqlite3_soft_heap_limit(int n){ ** Initialize the memory allocation subsystem. */ SQLITE_PRIVATE int sqlite3MallocInit(void){ + int rc; if( sqlite3GlobalConfig.m.xMalloc==0 ){ sqlite3MemSetDefault(); } memset(&mem0, 0, sizeof(mem0)); - if( sqlite3GlobalConfig.bCoreMutex ){ - mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); - } + mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 && sqlite3GlobalConfig.nScratch>0 ){ int i, n, sz; @@ -23479,12 +27804,13 @@ SQLITE_PRIVATE int sqlite3MallocInit(void){ sqlite3GlobalConfig.nScratch = 0; } if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512 - || sqlite3GlobalConfig.nPage<1 ){ + || sqlite3GlobalConfig.nPage<=0 ){ sqlite3GlobalConfig.pPage = 0; sqlite3GlobalConfig.szPage = 0; - sqlite3GlobalConfig.nPage = 0; } - return sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData); + rc = sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData); + if( rc!=SQLITE_OK ) memset(&mem0, 0, sizeof(mem0)); + return rc; } /* @@ -23510,10 +27836,8 @@ SQLITE_PRIVATE void sqlite3MallocEnd(void){ ** Return the amount of memory currently checked out. */ SQLITE_API sqlite3_int64 sqlite3_memory_used(void){ - int n, mx; - sqlite3_int64 res; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, 0); - res = (sqlite3_int64)n; /* Work around bug in Borland C. Ticket #3216 */ + sqlite3_int64 res, mx; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, 0); return res; } @@ -23523,30 +27847,19 @@ SQLITE_API sqlite3_int64 sqlite3_memory_used(void){ ** or since the most recent reset. */ SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ - int n, mx; - sqlite3_int64 res; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, resetFlag); - res = (sqlite3_int64)mx; /* Work around bug in Borland C. Ticket #3216 */ - return res; + sqlite3_int64 res, mx; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, resetFlag); + return mx; } /* ** Trigger the alarm */ static void sqlite3MallocAlarm(int nByte){ - void (*xCallback)(void*,sqlite3_int64,int); - sqlite3_int64 nowUsed; - void *pArg; - if( mem0.alarmCallback==0 ) return; - xCallback = mem0.alarmCallback; - nowUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - pArg = mem0.alarmArg; - mem0.alarmCallback = 0; + if( mem0.alarmThreshold<=0 ) return; sqlite3_mutex_leave(mem0.mutex); - xCallback(pArg, nowUsed, nByte); + sqlite3_release_memory(nByte); sqlite3_mutex_enter(mem0.mutex); - mem0.alarmCallback = xCallback; - mem0.alarmArg = pArg; } /* @@ -23558,9 +27871,9 @@ static int mallocWithAlarm(int n, void **pp){ void *p; assert( sqlite3_mutex_held(mem0.mutex) ); nFull = sqlite3GlobalConfig.m.xRoundup(n); - sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); - if( mem0.alarmCallback!=0 ){ - int nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); + if( mem0.alarmThreshold>0 ){ + sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); @@ -23570,15 +27883,15 @@ static int mallocWithAlarm(int n, void **pp){ } p = sqlite3GlobalConfig.m.xMalloc(nFull); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( p==0 && mem0.alarmCallback ){ + if( p==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } #endif if( p ){ nFull = sqlite3MallocSize(p); - sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull); - sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, 1); + sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nFull); + sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); } *pp = p; return nFull; @@ -23588,11 +27901,9 @@ static int mallocWithAlarm(int n, void **pp){ ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. */ -SQLITE_PRIVATE void *sqlite3Malloc(int n){ +SQLITE_PRIVATE void *sqlite3Malloc(u64 n){ void *p; - if( n<=0 /* IMP: R-65312-04917 */ - || n>=0x7fffff00 - ){ + if( n==0 || n>=0x7fffff00 ){ /* A memory allocation of a number of bytes which is near the maximum ** signed integer value might cause an integer overflow inside of the ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving @@ -23601,12 +27912,12 @@ SQLITE_PRIVATE void *sqlite3Malloc(int n){ p = 0; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - mallocWithAlarm(n, &p); + mallocWithAlarm((int)n, &p); sqlite3_mutex_leave(mem0.mutex); }else{ - p = sqlite3GlobalConfig.m.xMalloc(n); + p = sqlite3GlobalConfig.m.xMalloc((int)n); } - assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-04675-44850 */ + assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-11148-40995 */ return p; } @@ -23616,6 +27927,12 @@ SQLITE_PRIVATE void *sqlite3Malloc(int n){ ** allocation. */ SQLITE_API void *sqlite3_malloc(int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return n<=0 ? 0 : sqlite3Malloc(n); +} +SQLITE_API void *sqlite3_malloc64(sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -23646,22 +27963,20 @@ SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ assert( n>0 ); sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusHighwater(SQLITE_STATUS_SCRATCH_SIZE, n); if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ p = mem0.pScratchFree; mem0.pScratchFree = mem0.pScratchFree->pNext; mem0.nScratchFree--; - sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1); - sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + sqlite3StatusUp(SQLITE_STATUS_SCRATCH_USED, 1); sqlite3_mutex_leave(mem0.mutex); }else{ - if( sqlite3GlobalConfig.bMemstat ){ - sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); - n = mallocWithAlarm(n, &p); - if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n); + sqlite3_mutex_leave(mem0.mutex); + p = sqlite3Malloc(n); + if( sqlite3GlobalConfig.bMemstat && p ){ + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusUp(SQLITE_STATUS_SCRATCH_OVERFLOW, sqlite3MallocSize(p)); sqlite3_mutex_leave(mem0.mutex); - }else{ - sqlite3_mutex_leave(mem0.mutex); - p = sqlite3GlobalConfig.m.xMalloc(n); } sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); } @@ -23669,11 +27984,12 @@ SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocations per thread - ** are outstanding at one time. (This is only checked in the - ** single-threaded case since checking in the multi-threaded case - ** would be much more complicated.) */ - assert( scratchAllocOut<=1 ); + /* EVIDENCE-OF: R-12970-05880 SQLite will not use more than one scratch + ** buffers per thread. + ** + ** This can only be checked in single-threaded mode. + */ + assert( scratchAllocOut==0 ); if( p ) scratchAllocOut++; #endif @@ -23691,7 +28007,7 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ scratchAllocOut--; #endif - if( p>=sqlite3GlobalConfig.pScratch && p=db->lookaside.pStart && plookaside.pEnd; + return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); } #else #define isLookaside(A,B) 0 @@ -23739,33 +28055,43 @@ static int isLookaside(sqlite3 *db, void *p){ */ SQLITE_PRIVATE int sqlite3MallocSize(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); return sqlite3GlobalConfig.m.xSize(p); } SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ - assert( db!=0 ); - assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ - return db->lookaside.sz; - }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); - assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + assert( p!=0 ); + if( db==0 || !isLookaside(db,p) ){ +#if SQLITE_DEBUG + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + }else{ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + } +#endif return sqlite3GlobalConfig.m.xSize(p); + }else{ + assert( sqlite3_mutex_held(db->mutex) ); + return db->lookaside.sz; } } +SQLITE_API sqlite3_uint64 sqlite3_msize(void *p){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + return p ? sqlite3GlobalConfig.m.xSize(p) : 0; +} /* ** Free memory previously obtained from sqlite3Malloc(). */ SQLITE_API void sqlite3_free(void *p){ if( p==0 ) return; /* IMP: R-49053-54554 */ - assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p)); - sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); + sqlite3StatusDown(SQLITE_STATUS_MEMORY_USED, sqlite3MallocSize(p)); + sqlite3StatusDown(SQLITE_STATUS_MALLOC_COUNT, 1); sqlite3GlobalConfig.m.xFree(p); sqlite3_mutex_leave(mem0.mutex); }else{ @@ -23773,6 +28099,14 @@ SQLITE_API void sqlite3_free(void *p){ } } +/* +** Add the size of memory allocation "p" to the count in +** *db->pnBytesFreed. +*/ +static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){ + *db->pnBytesFreed += sqlite3DbMallocSize(db,p); +} + /* ** Free memory that might be associated with a particular database ** connection. @@ -23782,7 +28116,7 @@ SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ if( p==0 ) return; if( db ){ if( db->pnBytesFreed ){ - *db->pnBytesFreed += sqlite3DbMallocSize(db, p); + measureAllocationSize(db, p); return; } if( isLookaside(db, p) ){ @@ -23797,8 +28131,8 @@ SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ return; } } - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); sqlite3_free(p); @@ -23807,14 +28141,16 @@ SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ /* ** Change the size of an existing memory allocation */ -SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ +SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ int nOld, nNew, nDiff; void *pNew; + assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, (u8)~MEMTYPE_HEAP) ); if( pOld==0 ){ - return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ + return sqlite3Malloc(nBytes); /* IMP: R-04300-56712 */ } - if( nBytes<=0 ){ - sqlite3_free(pOld); /* IMP: R-31593-10574 */ + if( nBytes==0 ){ + sqlite3_free(pOld); /* IMP: R-26507-47431 */ return 0; } if( nBytes>=0x7fffff00 ){ @@ -23825,33 +28161,31 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second ** argument to xRealloc is always a value returned by a prior call to ** xRoundup. */ - nNew = sqlite3GlobalConfig.m.xRoundup(nBytes); + nNew = sqlite3GlobalConfig.m.xRoundup((int)nBytes); if( nOld==nNew ){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, nBytes); + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= mem0.alarmThreshold-nDiff ){ sqlite3MallocAlarm(nDiff); } - assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); - assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); - if( pNew==0 && mem0.alarmCallback ){ - sqlite3MallocAlarm(nBytes); + if( pNew==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } if( pNew ){ nNew = sqlite3MallocSize(pNew); - sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nNew-nOld); + sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld); } sqlite3_mutex_leave(mem0.mutex); }else{ pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } - assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-04675-44850 */ + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-11148-40995 */ return pNew; } @@ -23860,6 +28194,13 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ ** subsystem is initialized prior to invoking sqliteRealloc. */ SQLITE_API void *sqlite3_realloc(void *pOld, int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + if( n<0 ) n = 0; /* IMP: R-26507-47431 */ + return sqlite3Realloc(pOld, n); +} +SQLITE_API void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -23870,10 +28211,10 @@ SQLITE_API void *sqlite3_realloc(void *pOld, int n){ /* ** Allocate and zero memory. */ -SQLITE_PRIVATE void *sqlite3MallocZero(int n){ +SQLITE_PRIVATE void *sqlite3MallocZero(u64 n){ void *p = sqlite3Malloc(n); if( p ){ - memset(p, 0, n); + memset(p, 0, (size_t)n); } return p; } @@ -23882,17 +28223,32 @@ SQLITE_PRIVATE void *sqlite3MallocZero(int n){ ** Allocate and zero memory. If the allocation fails, make ** the mallocFailed flag in the connection pointer. */ -SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3 *db, int n){ - void *p = sqlite3DbMallocRaw(db, n); - if( p ){ - memset(p, 0, n); - } +SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ + void *p; + testcase( db==0 ); + p = sqlite3DbMallocRaw(db, n); + if( p ) memset(p, 0, (size_t)n); + return p; +} + + +/* Finish the work of sqlite3DbMallocRawNN for the unusual and +** slower case when the allocation cannot be fulfilled using lookaside. +*/ +static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ + void *p; + assert( db!=0 ); + p = sqlite3Malloc(n); + if( !p ) sqlite3OomFault(db); + sqlite3MemdebugSetType(p, + (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); return p; } /* -** Allocate and zero memory. If the allocation fails, make -** the mallocFailed flag in the connection pointer. +** Allocate memory, either lookaside (if possible) or heap. +** If the allocation fails, set the mallocFailed flag in +** the connection pointer. ** ** If db!=0 and db->mallocFailed is true (indicating a prior malloc ** failure on the same database connection) then always return 0. @@ -23907,79 +28263,87 @@ SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3 *db, int n){ ** ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed ** that all prior mallocs (ex: "a") worked too. +** +** The sqlite3MallocRawNN() variant guarantees that the "db" parameter is +** not a NULL pointer. */ -SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, int n){ +SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ void *p; - assert( db==0 || sqlite3_mutex_held(db->mutex) ); - assert( db==0 || db->pnBytesFreed==0 ); + if( db ) return sqlite3DbMallocRawNN(db, n); + p = sqlite3Malloc(n); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + return p; +} +SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ #ifndef SQLITE_OMIT_LOOKASIDE - if( db ){ - LookasideSlot *pBuf; - if( db->mallocFailed ){ - return 0; - } - if( db->lookaside.bEnabled ){ - if( n>db->lookaside.sz ){ - db->lookaside.anStat[1]++; - }else if( (pBuf = db->lookaside.pFree)==0 ){ - db->lookaside.anStat[2]++; - }else{ - db->lookaside.pFree = pBuf->pNext; - db->lookaside.nOut++; - db->lookaside.anStat[0]++; - if( db->lookaside.nOut>db->lookaside.mxOut ){ - db->lookaside.mxOut = db->lookaside.nOut; - } - return (void*)pBuf; + LookasideSlot *pBuf; + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); + if( db->lookaside.bDisable==0 ){ + assert( db->mallocFailed==0 ); + if( n>db->lookaside.sz ){ + db->lookaside.anStat[1]++; + }else if( (pBuf = db->lookaside.pFree)==0 ){ + db->lookaside.anStat[2]++; + }else{ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.nOut++; + db->lookaside.anStat[0]++; + if( db->lookaside.nOut>db->lookaside.mxOut ){ + db->lookaside.mxOut = db->lookaside.nOut; } + return (void*)pBuf; } + }else if( db->mallocFailed ){ + return 0; } #else - if( db && db->mallocFailed ){ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); + if( db->mallocFailed ){ return 0; } #endif - p = sqlite3Malloc(n); - if( !p && db ){ - db->mallocFailed = 1; - } - sqlite3MemdebugSetType(p, MEMTYPE_DB | - ((db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); - return p; + return dbMallocRawFinish(db, n); } +/* Forward declaration */ +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n); + /* ** Resize the block of memory pointed to by p to n bytes. If the ** resize fails, set the mallocFailed flag in the connection object. */ -SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ +SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ + assert( db!=0 ); + if( p==0 ) return sqlite3DbMallocRawNN(db, n); + assert( sqlite3_mutex_held(db->mutex) ); + if( isLookaside(db,p) && n<=db->lookaside.sz ) return p; + return dbReallocFinish(db, p, n); +} +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ void *pNew = 0; assert( db!=0 ); - assert( sqlite3_mutex_held(db->mutex) ); + assert( p!=0 ); if( db->mallocFailed==0 ){ - if( p==0 ){ - return sqlite3DbMallocRaw(db, n); - } if( isLookaside(db, p) ){ - if( n<=db->lookaside.sz ){ - return p; - } - pNew = sqlite3DbMallocRaw(db, n); + pNew = sqlite3DbMallocRawNN(db, n); if( pNew ){ memcpy(pNew, p, db->lookaside.sz); sqlite3DbFree(db, p); } }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - pNew = sqlite3_realloc(p, n); + pNew = sqlite3_realloc64(p, n); if( !pNew ){ - sqlite3MemdebugSetType(p, MEMTYPE_DB|MEMTYPE_HEAP); - db->mallocFailed = 1; + sqlite3OomFault(db); } - sqlite3MemdebugSetType(pNew, MEMTYPE_DB | - (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + sqlite3MemdebugSetType(pNew, + (db->lookaside.bDisable==0 ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); } } return pNew; @@ -23989,7 +28353,7 @@ SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ ** Attempt to reallocate p. If the reallocation fails, then free p ** and set the mallocFailed flag in the database connection. */ -SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, int n){ +SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, u64 n){ void *pNew; pNew = sqlite3DbRealloc(db, p, n); if( !pNew ){ @@ -24019,36 +28383,69 @@ SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3 *db, const char *z){ } return zNew; } -SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, int n){ +SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ char *zNew; + assert( db!=0 ); if( z==0 ){ return 0; } assert( (n&0x7fffffff)==n ); - zNew = sqlite3DbMallocRaw(db, n+1); + zNew = sqlite3DbMallocRawNN(db, n+1); if( zNew ){ - memcpy(zNew, z, n); + memcpy(zNew, z, (size_t)n); zNew[n] = 0; } return zNew; } /* -** Create a string from the zFromat argument and the va_list that follows. -** Store the string in memory obtained from sqliteMalloc() and make *pz -** point to that string. +** Free any prior content in *pz and replace it with a copy of zNew. */ -SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zFormat, ...){ - va_list ap; - char *z; - - va_start(ap, zFormat); - z = sqlite3VMPrintf(db, zFormat, ap); - va_end(ap); +SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ sqlite3DbFree(db, *pz); - *pz = z; + *pz = sqlite3DbStrDup(db, zNew); } +/* +** Call this routine to record the fact that an OOM (out-of-memory) error +** has happened. This routine will set db->mallocFailed, and also +** temporarily disable the lookaside memory allocator and interrupt +** any running VDBEs. +*/ +SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ + if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ + db->mallocFailed = 1; + if( db->nVdbeExec>0 ){ + db->u1.isInterrupted = 1; + } + db->lookaside.bDisable++; + } +} + +/* +** This routine reactivates the memory allocator and clears the +** db->mallocFailed flag as necessary. +** +** The memory allocator is not restarted if there are running +** VDBEs. +*/ +SQLITE_PRIVATE void sqlite3OomClear(sqlite3 *db){ + if( db->mallocFailed && db->nVdbeExec==0 ){ + db->mallocFailed = 0; + db->u1.isInterrupted = 0; + assert( db->lookaside.bDisable>0 ); + db->lookaside.bDisable--; + } +} + +/* +** Take actions at the end of an API call to indicate an OOM error +*/ +static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ + sqlite3OomClear(db); + sqlite3Error(db, SQLITE_NOMEM); + return SQLITE_NOMEM_BKPT; +} /* ** This function must be called before exiting any API function (i.e. @@ -24059,65 +28456,61 @@ SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zFormat ** function. However, if a malloc() failure has occurred since the previous ** invocation SQLITE_NOMEM is returned instead. ** -** If the first argument, db, is not NULL and a malloc() error has occurred, -** then the connection error-code (the value returned by sqlite3_errcode()) -** is set to SQLITE_NOMEM. +** If an OOM as occurred, then the connection error-code (the value +** returned by sqlite3_errcode()) is set to SQLITE_NOMEM. */ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ - /* If the db handle is not NULL, then we must hold the connection handle - ** mutex here. Otherwise the read (and possible write) of db->mallocFailed + /* If the db handle must hold the connection handle mutex here. + ** Otherwise the read (and possible write) of db->mallocFailed ** is unsafe, as is the call to sqlite3Error(). */ - assert( !db || sqlite3_mutex_held(db->mutex) ); - if( db && (db->mallocFailed || rc==SQLITE_IOERR_NOMEM) ){ - sqlite3Error(db, SQLITE_NOMEM, 0); - db->mallocFailed = 0; - rc = SQLITE_NOMEM; + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ + return apiOomError(db); } - return rc & (db ? db->errMask : 0xff); + return rc & db->errMask; } /************** End of malloc.c **********************************************/ /************** Begin file printf.c ******************************************/ /* ** The "printf" code that follows dates from the 1980's. It is in -** the public domain. The original comments are included here for -** completeness. They are very out-of-date but might be useful as -** an historical reference. Most of the "enhancements" have been backed -** out so that the functionality is now the same as standard printf(). +** the public domain. ** ************************************************************************** ** ** This file contains code for a set of "printf"-like routines. These ** routines format strings much like the printf() from the standard C ** library, though the implementation here has enhancements to support -** SQLlite. +** SQLite. */ +/* #include "sqliteInt.h" */ /* ** Conversion types fall into various categories as defined by the ** following enumeration. */ -#define etRADIX 1 /* Integer types. %d, %x, %o, and so forth */ -#define etFLOAT 2 /* Floating point. %f */ -#define etEXP 3 /* Exponentional notation. %e and %E */ -#define etGENERIC 4 /* Floating or exponential, depending on exponent. %g */ -#define etSIZE 5 /* Return number of characters processed so far. %n */ -#define etSTRING 6 /* Strings. %s */ -#define etDYNSTRING 7 /* Dynamically allocated strings. %z */ -#define etPERCENT 8 /* Percent symbol. %% */ -#define etCHARX 9 /* Characters. %c */ +#define etRADIX 0 /* Integer types. %d, %x, %o, and so forth */ +#define etFLOAT 1 /* Floating point. %f */ +#define etEXP 2 /* Exponentional notation. %e and %E */ +#define etGENERIC 3 /* Floating or exponential, depending on exponent. %g */ +#define etSIZE 4 /* Return number of characters processed so far. %n */ +#define etSTRING 5 /* Strings. %s */ +#define etDYNSTRING 6 /* Dynamically allocated strings. %z */ +#define etPERCENT 7 /* Percent symbol. %% */ +#define etCHARX 8 /* Characters. %c */ /* The rest are extensions, not normally found in printf() */ -#define etSQLESCAPE 10 /* Strings with '\'' doubled. %q */ -#define etSQLESCAPE2 11 /* Strings with '\'' doubled and enclosed in '', +#define etSQLESCAPE 9 /* Strings with '\'' doubled. %q */ +#define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', NULL pointers replaced by SQL NULL. %Q */ -#define etTOKEN 12 /* a pointer to a Token structure */ -#define etSRCLIST 13 /* a pointer to a SrcList */ -#define etPOINTER 14 /* The %p conversion */ -#define etSQLESCAPE3 15 /* %w -> Strings with '\"' doubled */ -#define etORDINAL 16 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etTOKEN 11 /* a pointer to a Token structure */ +#define etSRCLIST 12 /* a pointer to a SrcList */ +#define etPOINTER 13 /* The %p conversion */ +#define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ -#define etINVALID 0 /* Any unrecognized conversion type */ +#define etINVALID 16 /* Any unrecognized conversion type */ /* @@ -24218,6 +28611,7 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ ** Set the StrAccum object to an error mode. */ static void setStrAccumError(StrAccum *p, u8 eError){ + assert( eError==STRACCUM_NOMEM || eError==STRACCUM_TOOBIG ); p->accError = eError; p->nAlloc = 0; } @@ -24253,7 +28647,6 @@ static char *getTextArg(PrintfArguments *p){ */ SQLITE_PRIVATE void sqlite3VXPrintf( StrAccum *pAccum, /* Accumulate results here */ - u32 bFlags, /* SQLITE_PRINTF_* flags */ const char *fmt, /* Format string */ va_list ap /* arguments */ ){ @@ -24272,7 +28665,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( etByte flag_long; /* True if "l" flag is present */ etByte flag_longlong; /* True if the "ll" flag is present */ etByte done; /* Loop termination flag */ - etByte xtype = 0; /* Conversion paradigm */ + etByte xtype = etINVALID; /* Conversion paradigm */ u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ u8 useIntern; /* Ok to use internal conversions (ex: %T) */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ @@ -24281,7 +28674,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ - char *zExtra; /* Malloced memory used by some conversion */ + char *zExtra = 0; /* Malloced memory used by some conversion */ #ifndef SQLITE_OMIT_FLOATING_POINT int exp, e2; /* exponent of real numbers */ int nsd; /* Number of significant digits returned */ @@ -24293,20 +28686,24 @@ SQLITE_PRIVATE void sqlite3VXPrintf( char buf[etBUFSIZE]; /* Conversion buffer */ bufpt = 0; - if( bFlags ){ - if( (bArgList = (bFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ + if( pAccum->printfFlags ){ + if( (bArgList = (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ pArgList = va_arg(ap, PrintfArguments*); } - useIntern = bFlags & SQLITE_PRINTF_INTERNAL; + useIntern = pAccum->printfFlags & SQLITE_PRINTF_INTERNAL; }else{ bArgList = useIntern = 0; } for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ bufpt = (char *)fmt; - while( (c=(*++fmt))!='%' && c!=0 ){}; +#if HAVE_STRCHRNUL + fmt = strchrnul(fmt, '%'); +#else + do{ fmt++; }while( *fmt && *fmt != '%' ); +#endif sqlite3StrAccumAppend(pAccum, bufpt, (int)(fmt - bufpt)); - if( c==0 ) break; + if( *fmt==0 ) break; } if( (c=(*++fmt))==0 ){ sqlite3StrAccumAppend(pAccum, "%", 1); @@ -24328,7 +28725,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf( } }while( !done && (c=(*++fmt))!=0 ); /* Get the field width */ - width = 0; if( c=='*' ){ if( bArgList ){ width = (int)getIntArg(pArgList); @@ -24337,18 +28733,27 @@ SQLITE_PRIVATE void sqlite3VXPrintf( } if( width<0 ){ flag_leftjustify = 1; - width = -width; + width = width >= -2147483647 ? -width : 0; } c = *++fmt; }else{ + unsigned wx = 0; while( c>='0' && c<='9' ){ - width = width*10 + c - '0'; + wx = wx*10 + c - '0'; c = *++fmt; } + testcase( wx>0x7fffffff ); + width = wx & 0x7fffffff; } + assert( width>=0 ); +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + /* Get the precision */ if( c=='.' ){ - precision = 0; c = *++fmt; if( c=='*' ){ if( bArgList ){ @@ -24356,17 +28761,30 @@ SQLITE_PRIVATE void sqlite3VXPrintf( }else{ precision = va_arg(ap,int); } - if( precision<0 ) precision = -precision; c = *++fmt; + if( precision<0 ){ + precision = precision >= -2147483647 ? -precision : -1; + } }else{ + unsigned px = 0; while( c>='0' && c<='9' ){ - precision = precision*10 + c - '0'; + px = px*10 + c - '0'; c = *++fmt; } + testcase( px>0x7fffffff ); + precision = px & 0x7fffffff; } }else{ precision = -1; } + assert( precision>=(-1) ); +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ + precision = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + + /* Get the conversion type modifier */ if( c=='l' ){ flag_long = 1; @@ -24394,7 +28812,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf( break; } } - zExtra = 0; /* ** At this point, variables are initialized as follows: @@ -24527,7 +28944,8 @@ SQLITE_PRIVATE void sqlite3VXPrintf( else prefix = 0; } if( xtype==etGENERIC && precision>0 ) precision--; - for(idx=precision, rounder=0.5; idx>0; idx--, rounder*=0.1){} + testcase( precision>0xfff ); + for(idx=precision&0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} if( xtype==etFLOAT ) realvalue += rounder; /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ exp = 0; @@ -24539,21 +28957,16 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( realvalue>0.0 ){ LONGDOUBLE_TYPE scale = 1.0; while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e64*scale && exp<=350 ){ scale *= 1e64; exp+=64; } - while( realvalue>=1e8*scale && exp<=350 ){ scale *= 1e8; exp+=8; } + while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } realvalue /= scale; while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } if( exp>350 ){ - if( prefix=='-' ){ - bufpt = "-Inf"; - }else if( prefix=='+' ){ - bufpt = "+Inf"; - }else{ - bufpt = "Inf"; - } - length = sqlite3Strlen30(bufpt); + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); break; } } @@ -24582,8 +28995,9 @@ SQLITE_PRIVATE void sqlite3VXPrintf( }else{ e2 = exp; } - if( MAX(e2,0)+precision+width > etBUFSIZE - 15 ){ - bufpt = zExtra = sqlite3Malloc( MAX(e2,0)+precision+width+15 ); + if( MAX(e2,0)+(i64)precision+(i64)width > etBUFSIZE - 15 ){ + bufpt = zExtra + = sqlite3Malloc( MAX(e2,0)+(i64)precision+(i64)width+15 ); if( bufpt==0 ){ setStrAccumError(pAccum, STRACCUM_NOMEM); return; @@ -24685,25 +29099,29 @@ SQLITE_PRIVATE void sqlite3VXPrintf( }else{ c = va_arg(ap,int); } - buf[0] = (char)c; - if( precision>=0 ){ - for(idx=1; idx1 ){ + width -= precision-1; + if( width>1 && !flag_leftjustify ){ + sqlite3AppendChar(pAccum, width-1, ' '); + width = 0; + } + sqlite3AppendChar(pAccum, precision-1, c); } + length = 1; + buf[0] = c; bufpt = buf; break; case etSTRING: case etDYNSTRING: if( bArgList ){ bufpt = getTextArg(pArgList); + xtype = etSTRING; }else{ bufpt = va_arg(ap,char*); } if( bufpt==0 ){ bufpt = ""; - }else if( xtype==etDYNSTRING && !bArgList ){ + }else if( xtype==etDYNSTRING ){ zExtra = bufpt; } if( precision>=0 ){ @@ -24712,9 +29130,9 @@ SQLITE_PRIVATE void sqlite3VXPrintf( length = sqlite3Strlen30(bufpt); } break; - case etSQLESCAPE: - case etSQLESCAPE2: - case etSQLESCAPE3: { + case etSQLESCAPE: /* Escape ' characters */ + case etSQLESCAPE2: /* Escape ' and enclose in '...' */ + case etSQLESCAPE3: { /* Escape " characters */ int i, j, k, n, isnull; int needQuote; char ch; @@ -24733,7 +29151,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( ch==q ) n++; } needQuote = !isnull && xtype==etSQLESCAPE2; - n += i + 1 + needQuote*2; + n += i + 3; if( n>etBUFSIZE ){ bufpt = zExtra = sqlite3Malloc( n ); if( bufpt==0 ){ @@ -24792,11 +29210,14 @@ SQLITE_PRIVATE void sqlite3VXPrintf( ** the output. */ width -= length; - if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); sqlite3StrAccumAppend(pAccum, bufpt, length); - if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); - if( zExtra ) sqlite3_free(zExtra); + if( zExtra ){ + sqlite3DbFree(pAccum->db, zExtra); + zExtra = 0; + } }/* End for loop over the format string */ } /* End of function */ @@ -24809,20 +29230,26 @@ SQLITE_PRIVATE void sqlite3VXPrintf( */ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ char *zNew; - assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ + assert( p->nChar+(i64)N >= p->nAlloc ); /* Only called if really needed */ if( p->accError ){ testcase(p->accError==STRACCUM_TOOBIG); testcase(p->accError==STRACCUM_NOMEM); return 0; } - if( !p->useMalloc ){ + if( p->mxAlloc==0 ){ N = p->nAlloc - p->nChar - 1; setStrAccumError(p, STRACCUM_TOOBIG); return N; }else{ - char *zOld = (p->zText==p->zBase ? 0 : p->zText); + char *zOld = isMalloced(p) ? p->zText : 0; i64 szNew = p->nChar; + assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); szNew += N + 1; + if( szNew+p->nChar<=p->mxAlloc ){ + /* Force exponential buffer size growth as long as it does not overflow, + ** to avoid having to call this routine too often */ + szNew += p->nChar; + } if( szNew > p->mxAlloc ){ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_TOOBIG); @@ -24830,15 +29257,17 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ }else{ p->nAlloc = (int)szNew; } - if( p->useMalloc==1 ){ + if( p->db ){ zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); }else{ - zNew = sqlite3_realloc(zOld, p->nAlloc); + zNew = sqlite3_realloc64(zOld, p->nAlloc); } if( zNew ){ assert( p->zText!=0 || p->nChar==0 ); - if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); + if( !isMalloced(p) && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; + p->nAlloc = sqlite3DbMallocSize(p->db, zNew); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; }else{ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_NOMEM); @@ -24849,11 +29278,15 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ } /* -** Append N space characters to the given string buffer. +** Append N copies of character c to the given string buffer. */ -SQLITE_PRIVATE void sqlite3AppendSpace(StrAccum *p, int N){ - if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; - while( (N--)>0 ) p->zText[p->nChar++] = ' '; +SQLITE_PRIVATE void sqlite3AppendChar(StrAccum *p, int N, char c){ + testcase( p->nChar + (i64)N > 0x7fffffff ); + if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ + return; + } + assert( (p->zText==p->zBase)==!isMalloced(p) ); + while( (N--)>0 ) p->zText[p->nChar++] = c; } /* @@ -24864,12 +29297,13 @@ SQLITE_PRIVATE void sqlite3AppendSpace(StrAccum *p, int N){ ** work (enlarging the buffer) using tail recursion, so that the ** sqlite3StrAccumAppend() routine can use fast calling semantics. */ -static void enlargeAndAppend(StrAccum *p, const char *z, int N){ +static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ N = sqlite3StrAccumEnlarge(p, N); if( N>0 ){ memcpy(&p->zText[p->nChar], z, N); p->nChar += N; } + assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); } /* @@ -24877,17 +29311,17 @@ static void enlargeAndAppend(StrAccum *p, const char *z, int N){ ** size of the memory allocation for StrAccum if necessary. */ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ - assert( z!=0 ); + assert( z!=0 || N==0 ); assert( p->zText!=0 || p->nChar==0 || p->accError ); assert( N>=0 ); assert( p->accError==0 || p->nAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ enlargeAndAppend(p,z,N); - return; + }else{ + assert( p->zText ); + p->nChar += N; + memcpy(&p->zText[p->nChar-N], z, N); } - assert( p->zText ); - memcpy(&p->zText[p->nChar], z, N); - p->nChar += N; } /* @@ -24905,15 +29339,13 @@ SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum *p, const char *z){ */ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ + assert( (p->zText==p->zBase)==!isMalloced(p) ); p->zText[p->nChar] = 0; - if( p->useMalloc && p->zText==p->zBase ){ - if( p->useMalloc==1 ){ - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); - }else{ - p->zText = sqlite3_malloc(p->nChar+1); - } + if( p->mxAlloc>0 && !isMalloced(p) ){ + p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); if( p->zText ){ memcpy(p->zText, p->zBase, p->nChar+1); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; }else{ setStrAccumError(p, STRACCUM_NOMEM); } @@ -24926,27 +29358,36 @@ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ ** Reset an StrAccum string. Reclaim all malloced memory. */ SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum *p){ - if( p->zText!=p->zBase ){ - if( p->useMalloc==1 ){ - sqlite3DbFree(p->db, p->zText); - }else{ - sqlite3_free(p->zText); - } + assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); + if( isMalloced(p) ){ + sqlite3DbFree(p->db, p->zText); + p->printfFlags &= ~SQLITE_PRINTF_MALLOCED; } p->zText = 0; } /* -** Initialize a string accumulator +** Initialize a string accumulator. +** +** p: The accumulator to be initialized. +** db: Pointer to a database connection. May be NULL. Lookaside +** memory is used if not NULL. db->mallocFailed is set appropriately +** when not NULL. +** zBase: An initial buffer. May be NULL in which case the initial buffer +** is malloced. +** n: Size of zBase in bytes. If total space requirements never exceed +** n then no memory allocations ever occur. +** mx: Maximum number of bytes to accumulate. If mx==0 then no memory +** allocations will ever occur. */ -SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum *p, char *zBase, int n, int mx){ +SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, int n, int mx){ p->zText = p->zBase = zBase; - p->db = 0; + p->db = db; p->nChar = 0; p->nAlloc = n; p->mxAlloc = mx; - p->useMalloc = 1; p->accError = 0; + p->printfFlags = 0; } /* @@ -24958,13 +29399,13 @@ SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list a char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; assert( db!=0 ); - sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), + sqlite3StrAccumInit(&acc, db, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); - acc.db = db; - sqlite3VXPrintf(&acc, SQLITE_PRINTF_INTERNAL, zFormat, ap); + acc.printfFlags = SQLITE_PRINTF_INTERNAL; + sqlite3VXPrintf(&acc, zFormat, ap); z = sqlite3StrAccumFinish(&acc); if( acc.accError==STRACCUM_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } return z; } @@ -24982,24 +29423,6 @@ SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ return z; } -/* -** Like sqlite3MPrintf(), but call sqlite3DbFree() on zStr after formatting -** the string and before returnning. This routine is intended to be used -** to modify an existing string. For example: -** -** x = sqlite3MPrintf(db, x, "prefix %s suffix", x); -** -*/ -SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3 *db, char *zStr, const char *zFormat, ...){ - va_list ap; - char *z; - va_start(ap, zFormat); - z = sqlite3VMPrintf(db, zFormat, ap); - va_end(ap); - sqlite3DbFree(db, zStr); - return z; -} - /* ** Print into memory obtained from sqlite3_malloc(). Omit the internal ** %-conversion extensions. @@ -25008,12 +29431,18 @@ SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zFormat==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif - sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); - acc.useMalloc = 2; - sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3StrAccumInit(&acc, 0, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); + sqlite3VXPrintf(&acc, zFormat, ap); z = sqlite3StrAccumFinish(&acc); return z; } @@ -25050,9 +29479,15 @@ SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){ SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ StrAccum acc; if( n<=0 ) return zBuf; - sqlite3StrAccumInit(&acc, zBuf, n, 0); - acc.useMalloc = 0; - sqlite3VXPrintf(&acc, 0, zFormat, ap); +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf ) zBuf[0] = 0; + return zBuf; + } +#endif + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); + sqlite3VXPrintf(&acc, zFormat, ap); return sqlite3StrAccumFinish(&acc); } SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ @@ -25072,14 +29507,18 @@ SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ ** sqlite3_log() must render into a static buffer. It cannot dynamically ** allocate memory because it might be called while the memory allocator ** mutex is held. +** +** sqlite3VXPrintf() might ask for *temporary* memory allocations for +** certain format characters (%q) or for very large precisions or widths. +** Care must be taken that any sqlite3_log() calls that occur while the +** memory mutex is held do not use these mechanisms. */ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ StrAccum acc; /* String accumulator */ char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ - sqlite3StrAccumInit(&acc, zMsg, sizeof(zMsg), 0); - acc.useMalloc = 0; - sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); + sqlite3VXPrintf(&acc, zFormat, ap); sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, sqlite3StrAccumFinish(&acc)); } @@ -25096,7 +29535,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...){ } } -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) /* ** A version of printf() that understands %lld. Used for debugging. ** The printf() built into some versions of windows does not understand %lld @@ -25106,10 +29545,9 @@ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ va_list ap; StrAccum acc; char zBuf[500]; - sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); - acc.useMalloc = 0; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); va_start(ap,zFormat); - sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3VXPrintf(&acc, zFormat, ap); va_end(ap); sqlite3StrAccumFinish(&acc); fprintf(stdout,"%s", zBuf); @@ -25117,17 +29555,538 @@ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ } #endif + /* -** variable-argument wrapper around sqlite3VXPrintf(). +** variable-argument wrapper around sqlite3VXPrintf(). The bFlags argument +** can contain the bit SQLITE_PRINTF_INTERNAL enable internal formats. */ -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ...){ +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, const char *zFormat, ...){ va_list ap; va_start(ap,zFormat); - sqlite3VXPrintf(p, bFlags, zFormat, ap); + sqlite3VXPrintf(p, zFormat, ap); va_end(ap); } /************** End of printf.c **********************************************/ +/************** Begin file treeview.c ****************************************/ +/* +** 2015-06-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains C code to implement the TreeView debugging routines. +** These routines print a parse tree to standard output for debugging and +** analysis. +** +** The interfaces in this file is only available when compiling +** with SQLITE_DEBUG. +*/ +/* #include "sqliteInt.h" */ +#ifdef SQLITE_DEBUG + +/* +** Add a new subitem to the tree. The moreToFollow flag indicates that this +** is not the last item in the tree. +*/ +static TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ + if( p==0 ){ + p = sqlite3_malloc64( sizeof(*p) ); + if( p==0 ) return 0; + memset(p, 0, sizeof(*p)); + }else{ + p->iLevel++; + } + assert( moreToFollow==0 || moreToFollow==1 ); + if( p->iLevelbLine) ) p->bLine[p->iLevel] = moreToFollow; + return p; +} + +/* +** Finished with one layer of the tree +*/ +static void sqlite3TreeViewPop(TreeView *p){ + if( p==0 ) return; + p->iLevel--; + if( p->iLevel<0 ) sqlite3_free(p); +} + +/* +** Generate a single line of output for the tree, with a prefix that contains +** all the appropriate tree lines +*/ +static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ + va_list ap; + int i; + StrAccum acc; + char zBuf[500]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + if( p ){ + for(i=0; iiLevel && ibLine)-1; i++){ + sqlite3StrAccumAppend(&acc, p->bLine[i] ? "| " : " ", 4); + } + sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4); + } + va_start(ap, zFormat); + sqlite3VXPrintf(&acc, zFormat, ap); + va_end(ap); + if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1); + sqlite3StrAccumFinish(&acc); + fprintf(stdout,"%s", zBuf); + fflush(stdout); +} + +/* +** Shorthand for starting a new tree item that consists of a single label +*/ +static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){ + p = sqlite3TreeViewPush(p, moreFollows); + sqlite3TreeViewLine(p, "%s", zLabel); +} + +/* +** Generate a human-readable description of a WITH clause. +*/ +SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){ + int i; + if( pWith==0 ) return; + if( pWith->nCte==0 ) return; + if( pWith->pOuter ){ + sqlite3TreeViewLine(pView, "WITH (0x%p, pOuter=0x%p)",pWith,pWith->pOuter); + }else{ + sqlite3TreeViewLine(pView, "WITH (0x%p)", pWith); + } + if( pWith->nCte>0 ){ + pView = sqlite3TreeViewPush(pView, 1); + for(i=0; inCte; i++){ + StrAccum x; + char zLine[1000]; + const struct Cte *pCte = &pWith->a[i]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, "%s", pCte->zName); + if( pCte->pCols && pCte->pCols->nExpr>0 ){ + char cSep = '('; + int j; + for(j=0; jpCols->nExpr; j++){ + sqlite3XPrintf(&x, "%c%s", cSep, pCte->pCols->a[j].zName); + cSep = ','; + } + sqlite3XPrintf(&x, ")"); + } + sqlite3XPrintf(&x, " AS"); + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, inCte-1); + sqlite3TreeViewSelect(pView, pCte->pSelect, 0); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); + } +} + + +/* +** Generate a human-readable description of a Select object. +*/ +SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + int n = 0; + int cnt = 0; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( p->pWith ){ + sqlite3TreeViewWith(pView, p->pWith, 1); + cnt = 1; + sqlite3TreeViewPush(pView, 1); + } + do{ + sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x nSelectRow=%d", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags, + (int)p->nSelectRow + ); + if( cnt++ ) sqlite3TreeViewPop(pView); + if( p->pPrior ){ + n = 1000; + }else{ + n = 0; + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; + if( p->pOffset ) n++; + } + sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); + if( p->pSrc && p->pSrc->nSrc ){ + int i; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3XPrintf(&x, " %s.%s", pItem->zDatabase, pItem->zName); + }else if( pItem->zName ){ + sqlite3XPrintf(&x, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3XPrintf(&x, " tabname=%Q", pItem->pTab->zName); + } + if( pItem->zAlias ){ + sqlite3XPrintf(&x, " (AS %s)", pItem->zAlias); + } + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3XPrintf(&x, " LEFT-JOIN"); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); + } + if( p->pWhere ){ + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); + } + if( p->pGroupBy ){ + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); + } + if( p->pHaving ){ + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOrderBy ){ + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); + } + if( p->pLimit ){ + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOffset ){ + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewPop(pView); + } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, 1); + } + p = p->pPrior; + }while( p!=0 ); + sqlite3TreeViewPop(pView); +} + +/* +** Generate a human-readable explanation of an expression tree. +*/ +SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ + const char *zBinOp = 0; /* Binary operator */ + const char *zUniOp = 0; /* Unary operator */ + char zFlgs[30]; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( pExpr==0 ){ + sqlite3TreeViewLine(pView, "nil"); + sqlite3TreeViewPop(pView); + return; + } + if( pExpr->flags ){ + sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); + }else{ + zFlgs[0] = 0; + } + switch( pExpr->op ){ + case TK_AGG_COLUMN: { + sqlite3TreeViewLine(pView, "AGG{%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + break; + } + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ + sqlite3TreeViewLine(pView, "COLUMN(%d)%s", pExpr->iColumn, zFlgs); + }else{ + sqlite3TreeViewLine(pView, "{%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + } + break; + } + case TK_INTEGER: { + if( pExpr->flags & EP_IntValue ){ + sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); + }else{ + sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); + } + break; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_STRING: { + sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); + break; + } + case TK_NULL: { + sqlite3TreeViewLine(pView,"NULL"); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_VARIABLE: { + sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); + break; + } + case TK_REGISTER: { + sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); + break; + } + case TK_ID: { + sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); + break; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: zBinOp = "LT"; break; + case TK_LE: zBinOp = "LE"; break; + case TK_GT: zBinOp = "GT"; break; + case TK_GE: zBinOp = "GE"; break; + case TK_NE: zBinOp = "NE"; break; + case TK_EQ: zBinOp = "EQ"; break; + case TK_IS: zBinOp = "IS"; break; + case TK_ISNOT: zBinOp = "ISNOT"; break; + case TK_AND: zBinOp = "AND"; break; + case TK_OR: zBinOp = "OR"; break; + case TK_PLUS: zBinOp = "ADD"; break; + case TK_STAR: zBinOp = "MUL"; break; + case TK_MINUS: zBinOp = "SUB"; break; + case TK_REM: zBinOp = "REM"; break; + case TK_BITAND: zBinOp = "BITAND"; break; + case TK_BITOR: zBinOp = "BITOR"; break; + case TK_SLASH: zBinOp = "DIV"; break; + case TK_LSHIFT: zBinOp = "LSHIFT"; break; + case TK_RSHIFT: zBinOp = "RSHIFT"; break; + case TK_CONCAT: zBinOp = "CONCAT"; break; + case TK_DOT: zBinOp = "DOT"; break; + + case TK_UMINUS: zUniOp = "UMINUS"; break; + case TK_UPLUS: zUniOp = "UPLUS"; break; + case TK_BITNOT: zUniOp = "BITNOT"; break; + case TK_NOT: zUniOp = "NOT"; break; + case TK_ISNULL: zUniOp = "ISNULL"; break; + case TK_NOTNULL: zUniOp = "NOTNULL"; break; + + case TK_SPAN: { + sqlite3TreeViewLine(pView, "SPAN %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + + case TK_COLLATE: { + sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + if( ExprHasProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + }else{ + pFarg = pExpr->x.pList; + } + if( pExpr->op==TK_AGG_FUNCTION ){ + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", + pExpr->op2, pExpr->u.zToken); + }else{ + sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); + } + if( pFarg ){ + sqlite3TreeViewExprList(pView, pFarg, 0, 0); + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: { + sqlite3TreeViewLine(pView, "EXISTS-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_SELECT: { + sqlite3TreeViewLine(pView, "SELECT-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_IN: { + sqlite3TreeViewLine(pView, "IN"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + }else{ + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + } + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pX = pExpr->pLeft; + Expr *pY = pExpr->x.pList->a[0].pExpr; + Expr *pZ = pExpr->x.pList->a[1].pExpr; + sqlite3TreeViewLine(pView, "BETWEEN"); + sqlite3TreeViewExpr(pView, pX, 1); + sqlite3TreeViewExpr(pView, pY, 1); + sqlite3TreeViewExpr(pView, pZ, 0); + break; + } + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + */ + sqlite3TreeViewLine(pView, "%s(%d)", + pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); + break; + } + case TK_CASE: { + sqlite3TreeViewLine(pView, "CASE"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + const char *zType = "unk"; + switch( pExpr->affinity ){ + case OE_Rollback: zType = "rollback"; break; + case OE_Abort: zType = "abort"; break; + case OE_Fail: zType = "fail"; break; + case OE_Ignore: zType = "ignore"; break; + } + sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); + break; + } +#endif + case TK_MATCH: { + sqlite3TreeViewLine(pView, "MATCH {%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + break; + } + case TK_VECTOR: { + sqlite3TreeViewBareExprList(pView, pExpr->x.pList, "VECTOR"); + break; + } + case TK_SELECT_COLUMN: { + sqlite3TreeViewLine(pView, "SELECT-COLUMN %d", pExpr->iColumn); + sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0); + break; + } + default: { + sqlite3TreeViewLine(pView, "op=%d", pExpr->op); + break; + } + } + if( zBinOp ){ + sqlite3TreeViewLine(pView, "%s%s", zBinOp, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + }else if( zUniOp ){ + sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + } + sqlite3TreeViewPop(pView); +} + + +/* +** Generate a human-readable explanation of an expression list. +*/ +SQLITE_PRIVATE void sqlite3TreeViewBareExprList( + TreeView *pView, + const ExprList *pList, + const char *zLabel +){ + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); + }else{ + int i; + sqlite3TreeViewLine(pView, "%s", zLabel); + for(i=0; inExpr; i++){ + int j = pList->a[i].u.x.iOrderByCol; + if( j ){ + sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewLine(pView, "iOrderByCol=%d", j); + } + sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); + if( j ) sqlite3TreeViewPop(pView); + } + } +} +SQLITE_PRIVATE void sqlite3TreeViewExprList( + TreeView *pView, + const ExprList *pList, + u8 moreToFollow, + const char *zLabel +){ + pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewBareExprList(pView, pList, zLabel); + sqlite3TreeViewPop(pView); +} + +#endif /* SQLITE_DEBUG */ + +/************** End of treeview.c ********************************************/ /************** Begin file random.c ******************************************/ /* ** 2001 September 15 @@ -25146,6 +30105,7 @@ SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ** Random numbers are used by some of the database backends in order ** to generate random integer keys for tables or random filenames. */ +/* #include "sqliteInt.h" */ /* All threads share a single random number generator. @@ -25178,11 +30138,19 @@ SQLITE_API void sqlite3_randomness(int N, void *pBuf){ #endif #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); - sqlite3_mutex_enter(mutex); + sqlite3_mutex *mutex; #endif - if( N<=0 ){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); +#endif + + sqlite3_mutex_enter(mutex); + if( N<=0 || pBuf==0 ){ wsdPrng.isInit = 0; sqlite3_mutex_leave(mutex); return; @@ -25256,6 +30224,283 @@ SQLITE_PRIVATE void sqlite3PrngRestoreState(void){ #endif /* SQLITE_OMIT_BUILTIN_TEST */ /************** End of random.c **********************************************/ +/************** Begin file threads.c *****************************************/ +/* +** 2012 July 21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file presents a simple cross-platform threading interface for +** use internally by SQLite. +** +** A "thread" can be created using sqlite3ThreadCreate(). This thread +** runs independently of its creator until it is joined using +** sqlite3ThreadJoin(), at which point it terminates. +** +** Threads do not have to be real. It could be that the work of the +** "thread" is done by the main thread at either the sqlite3ThreadCreate() +** or sqlite3ThreadJoin() call. This is, in fact, what happens in +** single threaded systems. Nothing in SQLite requires multiple threads. +** This interface exists so that applications that want to take advantage +** of multiple cores can do so, while also allowing applications to stay +** single-threaded if desired. +*/ +/* #include "sqliteInt.h" */ +#if SQLITE_OS_WIN +/* # include "os_win.h" */ +#endif + +#if SQLITE_MAX_WORKER_THREADS>0 + +/********************************* Unix Pthreads ****************************/ +#if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) && SQLITE_THREADSAFE>0 + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +/* #include */ + +/* A running thread */ +struct SQLiteThread { + pthread_t tid; /* Thread ID */ + int done; /* Set to true when thread finishes */ + void *pOut; /* Result returned by the thread */ + void *(*xTask)(void*); /* The thread routine */ + void *pIn; /* Argument to the thread */ +}; + +/* Create a new thread */ +SQLITE_PRIVATE int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + int rc; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + /* This routine is never used in single-threaded mode */ + assert( sqlite3GlobalConfig.bCoreMutex!=0 ); + + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM_BKPT; + memset(p, 0, sizeof(*p)); + p->xTask = xTask; + p->pIn = pIn; + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** for testing purposes. */ + if( sqlite3FaultSim(200) ){ + rc = 1; + }else{ + rc = pthread_create(&p->tid, 0, xTask, pIn); + } + if( rc ){ + p->done = 1; + p->pOut = xTask(pIn); + } + *ppThread = p; + return SQLITE_OK; +} + +/* Get the results of the thread */ +SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + int rc; + + assert( ppOut!=0 ); + if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; + if( p->done ){ + *ppOut = p->pOut; + rc = SQLITE_OK; + }else{ + rc = pthread_join(p->tid, ppOut) ? SQLITE_ERROR : SQLITE_OK; + } + sqlite3_free(p); + return rc; +} + +#endif /* SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) */ +/******************************** End Unix Pthreads *************************/ + + +/********************************* Win32 Threads ****************************/ +#if SQLITE_OS_WIN_THREADS + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +#include + +/* A running thread */ +struct SQLiteThread { + void *tid; /* The thread handle */ + unsigned id; /* The thread identifier */ + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Thread procedure Win32 compatibility shim */ +static unsigned __stdcall sqlite3ThreadProc( + void *pArg /* IN: Pointer to the SQLiteThread structure */ +){ + SQLiteThread *p = (SQLiteThread *)pArg; + + assert( p!=0 ); +#if 0 + /* + ** This assert appears to trigger spuriously on certain + ** versions of Windows, possibly due to _beginthreadex() + ** and/or CreateThread() not fully setting their thread + ** ID parameter before starting the thread. + */ + assert( p->id==GetCurrentThreadId() ); +#endif + assert( p->xTask!=0 ); + p->pResult = p->xTask(p->pIn); + + _endthreadex(0); + return 0; /* NOT REACHED */ +} + +/* Create a new thread */ +SQLITE_PRIVATE int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM_BKPT; + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** (via the sqlite3FaultSim() term of the conditional) for testing + ** purposes. */ + if( sqlite3GlobalConfig.bCoreMutex==0 || sqlite3FaultSim(200) ){ + memset(p, 0, sizeof(*p)); + }else{ + p->xTask = xTask; + p->pIn = pIn; + p->tid = (void*)_beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); + if( p->tid==0 ){ + memset(p, 0, sizeof(*p)); + } + } + if( p->xTask==0 ){ + p->id = GetCurrentThreadId(); + p->pResult = xTask(pIn); + } + *ppThread = p; + return SQLITE_OK; +} + +SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject); /* os_win.c */ + +/* Get the results of the thread */ +SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + DWORD rc; + BOOL bRc; + + assert( ppOut!=0 ); + if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; + if( p->xTask==0 ){ + /* assert( p->id==GetCurrentThreadId() ); */ + rc = WAIT_OBJECT_0; + assert( p->tid==0 ); + }else{ + assert( p->id!=0 && p->id!=GetCurrentThreadId() ); + rc = sqlite3Win32Wait((HANDLE)p->tid); + assert( rc!=WAIT_IO_COMPLETION ); + bRc = CloseHandle((HANDLE)p->tid); + assert( bRc ); + } + if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; + sqlite3_free(p); + return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; +} + +#endif /* SQLITE_OS_WIN_THREADS */ +/******************************** End Win32 Threads *************************/ + + +/********************************* Single-Threaded **************************/ +#ifndef SQLITE_THREADS_IMPLEMENTED +/* +** This implementation does not actually create a new thread. It does the +** work of the thread in the main thread, when either the thread is created +** or when it is joined +*/ + +/* A running thread */ +struct SQLiteThread { + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Create a new thread */ +SQLITE_PRIVATE int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM_BKPT; + if( (SQLITE_PTR_TO_INT(p)/17)&1 ){ + p->xTask = xTask; + p->pIn = pIn; + }else{ + p->xTask = 0; + p->pResult = xTask(pIn); + } + *ppThread = p; + return SQLITE_OK; +} + +/* Get the results of the thread */ +SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + + assert( ppOut!=0 ); + if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; + if( p->xTask ){ + *ppOut = p->xTask(p->pIn); + }else{ + *ppOut = p->pResult; + } + sqlite3_free(p); + +#if defined(SQLITE_TEST) + { + void *pTstAlloc = sqlite3Malloc(10); + if (!pTstAlloc) return SQLITE_NOMEM_BKPT; + sqlite3_free(pTstAlloc); + } +#endif + + return SQLITE_OK; +} + +#endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ +/****************************** End Single-Threaded *************************/ +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ + +/************** End of threads.c *********************************************/ /************** Begin file utf.c *********************************************/ /* ** 2004 April 13 @@ -25292,15 +30537,17 @@ SQLITE_PRIVATE void sqlite3PrngRestoreState(void){ ** 0xfe 0xff big-endian utf-16 follows ** */ +/* #include "sqliteInt.h" */ /* #include */ +/* #include "vdbeInt.h" */ -#ifndef SQLITE_AMALGAMATION +#if !defined(SQLITE_AMALGAMATION) && SQLITE_BYTEORDER==0 /* ** The following constant value is used by the SQLITE_BIGENDIAN and ** SQLITE_LITTLEENDIAN macros. */ SQLITE_PRIVATE const int sqlite3one = 1; -#endif /* SQLITE_AMALGAMATION */ +#endif /* SQLITE_AMALGAMATION && SQLITE_BYTEORDER==0 */ /* ** This lookup table is used to help decode the first byte of @@ -25456,7 +30703,7 @@ SQLITE_PRIVATE u32 sqlite3Utf8Read( ** desiredEnc. It is an error if the string is already of the desired ** encoding, or if *pMem does not contain a string value. */ -SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ int len; /* Maximum length of output string in bytes */ unsigned char *zOut; /* Output buffer */ unsigned char *zIn; /* Input iterator */ @@ -25488,7 +30735,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ rc = sqlite3VdbeMemMakeWriteable(pMem); if( rc!=SQLITE_OK ){ assert( rc==SQLITE_NOMEM ); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } zIn = (u8*)pMem->z; zTerm = &zIn[pMem->n&~1]; @@ -25530,7 +30777,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ zTerm = &zIn[pMem->n]; zOut = sqlite3DbMallocRaw(pMem->db, len); if( !zOut ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } z = zOut; @@ -25571,12 +30818,13 @@ SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ *z = 0; assert( (pMem->n+(desiredEnc==SQLITE_UTF8?1:2))<=len ); + c = pMem->flags; sqlite3VdbeMemRelease(pMem); - pMem->flags &= ~(MEM_Static|MEM_Dyn|MEM_Ephem); + pMem->flags = MEM_Str|MEM_Term|(c&(MEM_AffMask|MEM_Subtype)); pMem->enc = desiredEnc; - pMem->flags |= (MEM_Term); pMem->z = (char*)zOut; pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->z); translate_out: #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) @@ -25804,8 +31052,9 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ ** strings, and stuff like that. ** */ +/* #include "sqliteInt.h" */ /* #include */ -#ifdef SQLITE_HAVE_ISNAN +#if HAVE_ISNAN || SQLITE_HAVE_ISNAN # include #endif @@ -25846,7 +31095,7 @@ SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ */ SQLITE_PRIVATE int sqlite3IsNaN(double x){ int rc; /* The value return */ -#if !defined(SQLITE_HAVE_ISNAN) +#if !SQLITE_HAVE_ISNAN && !HAVE_ISNAN /* ** Systems that support the isnan() library function should probably ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have @@ -25876,9 +31125,9 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ volatile double y = x; volatile double z = y; rc = (y!=z); -#else /* if defined(SQLITE_HAVE_ISNAN) */ +#else /* if HAVE_ISNAN */ rc = isnan(x); -#endif /* SQLITE_HAVE_ISNAN */ +#endif /* HAVE_ISNAN */ testcase( rc ); return rc; } @@ -25893,10 +31142,53 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ ** than 1GiB) the value returned might be less than the true string length. */ SQLITE_PRIVATE int sqlite3Strlen30(const char *z){ - const char *z2 = z; if( z==0 ) return 0; - while( *z2 ){ z2++; } - return 0x3fffffff & (int)(z2 - z); + return 0x3fffffff & (int)strlen(z); +} + +/* +** Return the declared type of a column. Or return zDflt if the column +** has no declared type. +** +** The column type is an extra string stored after the zero-terminator on +** the column name if and only if the COLFLAG_HASTYPE flag is set. +*/ +SQLITE_PRIVATE char *sqlite3ColumnType(Column *pCol, char *zDflt){ + if( (pCol->colFlags & COLFLAG_HASTYPE)==0 ) return zDflt; + return pCol->zName + strlen(pCol->zName) + 1; +} + +/* +** Helper function for sqlite3Error() - called rarely. Broken out into +** a separate routine to avoid unnecessary register saves on entry to +** sqlite3Error(). +*/ +static SQLITE_NOINLINE void sqlite3ErrorFinish(sqlite3 *db, int err_code){ + if( db->pErr ) sqlite3ValueSetNull(db->pErr); + sqlite3SystemError(db, err_code); +} + +/* +** Set the current error code to err_code and clear any prior error message. +** Also set iSysErrno (by calling sqlite3System) if the err_code indicates +** that would be appropriate. +*/ +SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code){ + assert( db!=0 ); + db->errCode = err_code; + if( err_code || db->pErr ) sqlite3ErrorFinish(db, err_code); +} + +/* +** Load the sqlite3.iSysErrno field if that is an appropriate thing +** to do based on the SQLite error code in rc. +*/ +SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ + if( rc==SQLITE_IOERR_NOMEM ) return; + rc &= 0xff; + if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){ + db->iSysErrno = sqlite3OsGetLastError(db->pVfs); + } } /* @@ -25920,18 +31212,19 @@ SQLITE_PRIVATE int sqlite3Strlen30(const char *z){ ** should be called with err_code set to SQLITE_OK and zFormat set ** to NULL. */ -SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ...){ +SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){ assert( db!=0 ); db->errCode = err_code; - if( zFormat && (db->pErr || (db->pErr = sqlite3ValueNew(db))!=0) ){ + sqlite3SystemError(db, err_code); + if( zFormat==0 ){ + sqlite3Error(db, err_code); + }else if( db->pErr || (db->pErr = sqlite3ValueNew(db))!=0 ){ char *z; va_list ap; va_start(ap, zFormat); z = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); sqlite3ValueSetStr(db->pErr, -1, z, SQLITE_UTF8, SQLITE_DYNAMIC); - }else if( db->pErr ){ - sqlite3ValueSetNull(db->pErr); } } @@ -25945,12 +31238,12 @@ SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ** %T Insert a token ** %S Insert the first element of a SrcList ** -** This function should be used to report any error that occurs whilst +** This function should be used to report any error that occurs while ** compiling an SQL statement (i.e. within sqlite3_prepare()). The ** last thing the sqlite3_prepare() function does is copy the error ** stored by this function into the database handle using sqlite3Error(). -** Function sqlite3Error() should be used during statement execution -** (sqlite3_step() etc.). +** Functions sqlite3Error() or sqlite3ErrorWithMsg() should be used +** during statement execution (sqlite3_step() etc.). */ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ char *zMsg; @@ -25983,21 +31276,16 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ ** occur. ** ** 2002-Feb-14: This routine is extended to remove MS-Access style -** brackets from around identifers. For example: "[a-b-c]" becomes +** brackets from around identifiers. For example: "[a-b-c]" becomes ** "a-b-c". */ -SQLITE_PRIVATE int sqlite3Dequote(char *z){ +SQLITE_PRIVATE void sqlite3Dequote(char *z){ char quote; int i, j; - if( z==0 ) return -1; + if( z==0 ) return; quote = z[0]; - switch( quote ){ - case '\'': break; - case '"': break; - case '`': break; /* For MySQL compatibility */ - case '[': quote = ']'; break; /* For MS SqlServer compatibility */ - default: return -1; - } + if( !sqlite3Isquote(quote) ) return; + if( quote=='[' ) quote = ']'; for(i=1, j=0;; i++){ assert( z[i] ); if( z[i]==quote ){ @@ -26012,7 +31300,14 @@ SQLITE_PRIVATE int sqlite3Dequote(char *z){ } } z[j] = 0; - return j; +} + +/* +** Generate a Token object from a string +*/ +SQLITE_PRIVATE void sqlite3TokenInit(Token *p, char *z){ + p->z = z; + p->n = sqlite3Strlen30(z); } /* Convenient short-hand */ @@ -26029,14 +31324,33 @@ SQLITE_PRIVATE int sqlite3Dequote(char *z){ ** independence" that SQLite uses internally when comparing identifiers. */ SQLITE_API int sqlite3_stricmp(const char *zLeft, const char *zRight){ - register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } + return sqlite3StrICmp(zLeft, zRight); +} +SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ + unsigned char *a, *b; + int c; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; - while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } - return UpperToLower[*a] - UpperToLower[*b]; + for(;;){ + c = (int)UpperToLower[*a] - (int)UpperToLower[*b]; + if( c || *a==0 ) break; + a++; + b++; + } + return c; } SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } @@ -26078,7 +31392,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; - int nonNum = 0; + int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ @@ -26091,7 +31405,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); for(i=3-enc; i=zEnd ) goto do_atof_calc; @@ -26139,7 +31451,12 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en if( *z=='e' || *z=='E' ){ z+=incr; eValid = 0; - if( z>=zEnd ) goto do_atof_calc; + + /* This branch is needed to avoid a (harmless) buffer overread. The + ** special comment alerts the mutation tester that the correct answer + ** is obtained even if the branch is omitted */ + if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ + /* get sign of exponent */ if( *z=='-' ){ esign = -1; @@ -26156,9 +31473,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en } /* skip trailing spaces */ - if( nDigits && eValid ){ - while( z0 ){ - while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10; - }else{ - while( !(s%10) && e>0 ) e--,s/=10; + /* Attempt to reduce exponent. + ** + ** Branches that are not required for the correct answer but which only + ** help to obtain the correct answer faster are marked with special + ** comments, as a hint to the mutation tester. + */ + while( e>0 ){ /*OPTIMIZATION-IF-TRUE*/ + if( esign>0 ){ + if( s>=(LARGEST_INT64/10) ) break; /*OPTIMIZATION-IF-FALSE*/ + s *= 10; + }else{ + if( s%10!=0 ) break; /*OPTIMIZATION-IF-FALSE*/ + s /= 10; + } + e--; } /* adjust the sign of significand */ s = sign<0 ? -s : s; - /* if exponent, scale significand as appropriate - ** and store in result. */ - if( e ){ + if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ + result = (double)s; + }else{ LONGDOUBLE_TYPE scale = 1.0; /* attempt to handle extremely small/large numbers better */ - if( e>307 && e<342 ){ - while( e%308 ) { scale *= 1.0e+1; e -= 1; } - if( esign<0 ){ - result = s / scale; - result /= 1.0e+308; - }else{ - result = s * scale; - result *= 1.0e+308; - } - }else if( e>=342 ){ - if( esign<0 ){ - result = 0.0*s; - }else{ - result = 1e308*1e308*s; /* Infinity */ + if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ + if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ + while( e%308 ) { scale *= 1.0e+1; e -= 1; } + if( esign<0 ){ + result = s / scale; + result /= 1.0e+308; + }else{ + result = s * scale; + result *= 1.0e+308; + } + }else{ assert( e>=342 ); + if( esign<0 ){ + result = 0.0*s; + }else{ + result = 1e308*1e308*s; /* Infinity */ + } } }else{ /* 1.0e+22 is the largest power of 10 than can be @@ -26217,8 +31542,6 @@ do_atof_calc: result = s * scale; } } - } else { - result = (double)s; } } @@ -26226,7 +31549,7 @@ do_atof_calc: *pResult = result; /* return true if number and no extra non-whitespace chracters after */ - return z>=zEnd && nDigits>0 && eValid && nonNum==0; + return z==zEnd && nDigits>0 && eValid && nonNum==0; #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ @@ -26288,7 +31611,7 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc int neg = 0; /* assume positive */ int i; int c = 0; - int nonNum = 0; + int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ const char *zStart; const char *zEnd = zNum + length; assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); @@ -26299,7 +31622,7 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); for(i=3-enc; i19*incr || nonNum ){ + if( &zNum[i]19*incr /* Too many digits */ + || nonNum /* UTF16 with high-order bytes non-zero */ + ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranteeing that it is too large) */ return 1; @@ -26368,7 +31695,6 @@ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') - && sqlite3Isxdigit(z[2]) ){ u64 u = 0; int i, k; @@ -26424,6 +31750,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ } } #endif + while( zNum[0]=='0' ) zNum++; for(i=0; i<11 && (c = zNum[i] - '0')>=0 && c<=9; i++){ v = v*10 + c; } @@ -26487,7 +31814,7 @@ SQLITE_PRIVATE int sqlite3Atoi(const char *z){ ** bit clear. Except, if we get to the 9th byte, it stores the full ** 8 bits and is the last byte. */ -SQLITE_PRIVATE int sqlite3PutVarint(unsigned char *p, u64 v){ +static int SQLITE_NOINLINE putVarint64(unsigned char *p, u64 v){ int i, j, n; u8 buf[10]; if( v & (((u64)0xff000000)<<32) ){ @@ -26511,28 +31838,17 @@ SQLITE_PRIVATE int sqlite3PutVarint(unsigned char *p, u64 v){ } return n; } - -/* -** This routine is a faster version of sqlite3PutVarint() that only -** works for 32-bit positive integers and which is optimized for -** the common case of small integers. A MACRO version, putVarint32, -** is provided which inlines the single-byte case. All code should use -** the MACRO version as this function assumes the single-byte case has -** already been handled. -*/ -SQLITE_PRIVATE int sqlite3PutVarint32(unsigned char *p, u32 v){ -#ifndef putVarint32 - if( (v & ~0x7f)==0 ){ - p[0] = v; +SQLITE_PRIVATE int sqlite3PutVarint(unsigned char *p, u64 v){ + if( v<=0x7f ){ + p[0] = v&0x7f; return 1; } -#endif - if( (v & ~0x3fff)==0 ){ - p[0] = (u8)((v>>7) | 0x80); - p[1] = (u8)(v & 0x7f); + if( v<=0x3fff ){ + p[0] = ((v>>7)&0x7f)|0x80; + p[1] = v&0x7f; return 2; } - return sqlite3PutVarint(p, v); + return putVarint64(p,v); } /* @@ -26625,7 +31941,8 @@ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ /* a: p0<<28 | p2<<14 | p4 (unmasked) */ if (!(a&0x80)) { - /* we can skip these cause they were (effectively) done above in calc'ing s */ + /* we can skip these cause they were (effectively) done above + ** while calculating s */ /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ /* b &= (0x7f<<14)|(0x7f); */ b = b<<7; @@ -26846,11 +32163,8 @@ SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ ** 64-bit integer. */ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ - int i = 0; - do{ - i++; - v >>= 7; - }while( v!=0 && ALWAYS(i<9) ); + int i; + for(i=1; (v >>= 7)!=0; i++){ assert( i<10 ); } return i; } @@ -26859,14 +32173,42 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ ** Read or write a four-byte big-endian integer value. */ SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ +#if SQLITE_BYTEORDER==4321 + u32 x; + memcpy(&x,p,4); + return x; +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(__GNUC__) && GCC_VERSION>=4003000 + u32 x; + memcpy(&x,p,4); + return __builtin_bswap32(x); +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 + u32 x; + memcpy(&x,p,4); + return _byteswap_ulong(x); +#else testcase( p[0]&0x80 ); return ((unsigned)p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3]; +#endif } SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ +#if SQLITE_BYTEORDER==4321 + memcpy(p,&v,4); +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(__GNUC__) && GCC_VERSION>=4003000 + u32 x = __builtin_bswap32(v); + memcpy(p,&x,4); +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 + u32 x = _byteswap_ulong(v); + memcpy(p,&x,4); +#else p[0] = (u8)(v>>24); p[1] = (u8)(v>>16); p[2] = (u8)(v>>8); p[3] = (u8)v; +#endif } @@ -26898,7 +32240,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ char *zBlob; int i; - zBlob = (char *)sqlite3DbMallocRaw(db, n/2 + 1); + zBlob = (char *)sqlite3DbMallocRawNN(db, n/2 + 1); n--; if( zBlob ){ for(i=0; i0 ){ + if( iA>LARGEST_INT64/iB ) return 1; + if( iA0 ){ + if( iBLARGEST_INT64/-iB ) return 1; } - r = iA0*iB1; - }else if( iB1==0 ){ - r = iA1*iB0; - }else{ - /* If both iA1 and iB1 are non-zero, overflow will result */ - return 1; } - testcase( r==(-TWOPOWER31)-1 ); - testcase( r==(-TWOPOWER31) ); - testcase( r==TWOPOWER31 ); - testcase( r==TWOPOWER31-1 ); - if( r<(-TWOPOWER31) || r>=TWOPOWER31 ) return 1; - r *= TWOPOWER32; - if( sqlite3AddInt64(&r, iA0*iB0) ) return 1; - *pA = r; + *pA = iA*iB; return 0; } @@ -27114,7 +32441,7 @@ SQLITE_PRIVATE LogEst sqlite3LogEst(u64 x){ if( x<2 ) return 0; while( x<8 ){ y -= 10; x <<= 1; } }else{ - while( x>255 ){ y += 40; x >>= 4; } + while( x>255 ){ y += 40; x >>= 4; } /*OPTIMIZATION-IF-TRUE*/ while( x>15 ){ y += 10; x >>= 1; } } return a[x&7] + y - 10; @@ -27137,21 +32464,32 @@ SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double x){ } #endif /* SQLITE_OMIT_VIRTUALTABLE */ +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ + defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) /* ** Convert a LogEst into an integer. +** +** Note that this routine is only used when one or more of various +** non-standard compile-time options is enabled. */ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ u64 n; - if( x<10 ) return 1; n = x%10; x /= 10; if( n>=5 ) n -= 2; else if( n>=1 ) n -= 1; - if( x>=3 ){ - return x>60 ? (u64)LARGEST_INT64 : (n+8)<<(x-3); - } - return (n+8)>>(3-x); +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) + if( x>60 ) return (u64)LARGEST_INT64; +#else + /* If only SQLITE_ENABLE_STAT3_OR_STAT4 is on, then the largest input + ** possible to this routine is 310, resulting in a maximum x of 31 */ + assert( x<=60 ); +#endif + return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x); } +#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */ /************** End of util.c ************************************************/ /************** Begin file hash.c ********************************************/ @@ -27169,6 +32507,7 @@ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ ** This is the implementation of generic hash-tables ** used in SQLite. */ +/* #include "sqliteInt.h" */ /* #include */ /* Turn bulk memory into a hash table object by initializing the @@ -27208,12 +32547,15 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash *pH){ /* ** The hashing function. */ -static unsigned int strHash(const char *z, int nKey){ +static unsigned int strHash(const char *z){ unsigned int h = 0; - assert( nKey>=0 ); - while( nKey > 0 ){ - h = (h<<3) ^ h ^ sqlite3UpperToLower[(unsigned char)*z++]; - nKey--; + unsigned char c; + while( (c = (unsigned char)*z++)!=0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). + ** 0x9e3779b1 is 2654435761 which is the closest prime number to + ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ + h += sqlite3UpperToLower[c]; + h *= 0x9e3779b1; } return h; } @@ -27285,7 +32627,7 @@ static int rehash(Hash *pH, unsigned int new_size){ pH->htsize = new_size = sqlite3MallocSize(new_ht)/sizeof(struct _ht); memset(new_ht, 0, new_size*sizeof(struct _ht)); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - unsigned int h = strHash(elem->pKey, elem->nKey) % new_size; + unsigned int h = strHash(elem->pKey) % new_size; next_elem = elem->next; insertElement(pH, &new_ht[h], elem); } @@ -27293,28 +32635,33 @@ static int rehash(Hash *pH, unsigned int new_size){ } /* This function (for internal use only) locates an element in an -** hash table that matches the given key. The hash for this key has -** already been computed and is passed as the 4th parameter. +** hash table that matches the given key. The hash for this key is +** also computed and returned in the *pH parameter. */ -static HashElem *findElementGivenHash( +static HashElem *findElementWithHash( const Hash *pH, /* The pH to be searched */ const char *pKey, /* The key we are searching for */ - int nKey, /* Bytes in key (not counting zero terminator) */ - unsigned int h /* The hash for this key. */ + unsigned int *pHash /* Write the hash value here */ ){ HashElem *elem; /* Used to loop thru the element list */ int count; /* Number of elements left to test */ + unsigned int h; /* The computed hash */ - if( pH->ht ){ - struct _ht *pEntry = &pH->ht[h]; + if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ + struct _ht *pEntry; + h = strHash(pKey) % pH->htsize; + pEntry = &pH->ht[h]; elem = pEntry->chain; count = pEntry->count; }else{ + h = 0; elem = pH->first; count = pH->count; } - while( count-- && ALWAYS(elem) ){ - if( elem->nKey==nKey && sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ + *pHash = h; + while( count-- ){ + assert( elem!=0 ); + if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ return elem; } elem = elem->next; @@ -27357,26 +32704,20 @@ static void removeElementGivenHash( } /* Attempt to locate an element of the hash table pH with a key -** that matches pKey,nKey. Return the data for this element if it is +** that matches pKey. Return the data for this element if it is ** found, or NULL if there is no match. */ -SQLITE_PRIVATE void *sqlite3HashFind(const Hash *pH, const char *pKey, int nKey){ +SQLITE_PRIVATE void *sqlite3HashFind(const Hash *pH, const char *pKey){ HashElem *elem; /* The element that matches key */ unsigned int h; /* A hash on key */ assert( pH!=0 ); assert( pKey!=0 ); - assert( nKey>=0 ); - if( pH->ht ){ - h = strHash(pKey, nKey) % pH->htsize; - }else{ - h = 0; - } - elem = findElementGivenHash(pH, pKey, nKey, h); + elem = findElementWithHash(pH, pKey, &h); return elem ? elem->data : 0; } -/* Insert an element into the hash table pH. The key is pKey,nKey +/* Insert an element into the hash table pH. The key is pKey ** and the data is "data". ** ** If no element exists with a matching key, then a new @@ -27390,20 +32731,14 @@ SQLITE_PRIVATE void *sqlite3HashFind(const Hash *pH, const char *pKey, int nKey) ** If the "data" parameter to this function is NULL, then the ** element corresponding to "key" is removed from the hash table. */ -SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, void *data){ +SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ unsigned int h; /* the hash of the key modulo hash table size */ HashElem *elem; /* Used to loop thru the element list */ HashElem *new_elem; /* New element added to the pH */ assert( pH!=0 ); assert( pKey!=0 ); - assert( nKey>=0 ); - if( pH->htsize ){ - h = strHash(pKey, nKey) % pH->htsize; - }else{ - h = 0; - } - elem = findElementGivenHash(pH,pKey,nKey,h); + elem = findElementWithHash(pH,pKey,&h); if( elem ){ void *old_data = elem->data; if( data==0 ){ @@ -27411,7 +32746,6 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, voi }else{ elem->data = data; elem->pKey = pKey; - assert(nKey==elem->nKey); } return old_data; } @@ -27419,193 +32753,195 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, voi new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); if( new_elem==0 ) return data; new_elem->pKey = pKey; - new_elem->nKey = nKey; new_elem->data = data; pH->count++; if( pH->count>=10 && pH->count > 2*pH->htsize ){ if( rehash(pH, pH->count*2) ){ assert( pH->htsize>0 ); - h = strHash(pKey, nKey) % pH->htsize; + h = strHash(pKey) % pH->htsize; } } - if( pH->ht ){ - insertElement(pH, &pH->ht[h], new_elem); - }else{ - insertElement(pH, 0, new_elem); - } + insertElement(pH, pH->ht ? &pH->ht[h] : 0, new_elem); return 0; } /************** End of hash.c ************************************************/ /************** Begin file opcodes.c *****************************************/ /* Automatically generated. Do not edit */ -/* See the mkopcodec.awk script for details. */ -#if !defined(SQLITE_OMIT_EXPLAIN) || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) +/* See the tool/mkopcodec.tcl script for details. */ +#if !defined(SQLITE_OMIT_EXPLAIN) \ + || defined(VDBE_PROFILE) \ + || defined(SQLITE_DEBUG) #if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) || defined(SQLITE_DEBUG) # define OpHelp(X) "\0" X #else # define OpHelp(X) #endif SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ - static const char *const azName[] = { "?", - /* 1 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), - /* 2 */ "Savepoint" OpHelp(""), - /* 3 */ "AutoCommit" OpHelp(""), - /* 4 */ "Transaction" OpHelp(""), - /* 5 */ "SorterNext" OpHelp(""), - /* 6 */ "PrevIfOpen" OpHelp(""), - /* 7 */ "NextIfOpen" OpHelp(""), - /* 8 */ "Prev" OpHelp(""), - /* 9 */ "Next" OpHelp(""), - /* 10 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 11 */ "Checkpoint" OpHelp(""), - /* 12 */ "JournalMode" OpHelp(""), - /* 13 */ "Vacuum" OpHelp(""), - /* 14 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), - /* 15 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 16 */ "Goto" OpHelp(""), - /* 17 */ "Gosub" OpHelp(""), - /* 18 */ "Return" OpHelp(""), - /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "InitCoroutine" OpHelp(""), - /* 21 */ "EndCoroutine" OpHelp(""), - /* 22 */ "Yield" OpHelp(""), - /* 23 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 24 */ "Halt" OpHelp(""), - /* 25 */ "Integer" OpHelp("r[P2]=P1"), - /* 26 */ "Int64" OpHelp("r[P2]=P4"), - /* 27 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 28 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 29 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 30 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 31 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 32 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 33 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 34 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 35 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 36 */ "CollSeq" OpHelp(""), - /* 37 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 38 */ "MustBeInt" OpHelp(""), - /* 39 */ "RealAffinity" OpHelp(""), - /* 40 */ "Permutation" OpHelp(""), - /* 41 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 42 */ "Jump" OpHelp(""), - /* 43 */ "Once" OpHelp(""), - /* 44 */ "If" OpHelp(""), - /* 45 */ "IfNot" OpHelp(""), - /* 46 */ "Column" OpHelp("r[P3]=PX"), - /* 47 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 48 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 49 */ "Count" OpHelp("r[P2]=count()"), - /* 50 */ "ReadCookie" OpHelp(""), - /* 51 */ "SetCookie" OpHelp(""), - /* 52 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 53 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 54 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 55 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 56 */ "OpenEphemeral" OpHelp("nColumn=P2"), - /* 57 */ "SorterOpen" OpHelp(""), - /* 58 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), - /* 59 */ "Close" OpHelp(""), - /* 60 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 61 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 62 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 63 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 64 */ "Seek" OpHelp("intkey=r[P2]"), - /* 65 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 66 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 67 */ "Found" OpHelp("key=r[P3@P4]"), - /* 68 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 69 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 70 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 71 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), - /* 72 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 73 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 74 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), - /* 75 */ "Delete" OpHelp(""), - /* 76 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), - /* 77 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), - /* 78 */ "Ne" OpHelp("if r[P1]!=r[P3] goto P2"), - /* 79 */ "Eq" OpHelp("if r[P1]==r[P3] goto P2"), - /* 80 */ "Gt" OpHelp("if r[P1]>r[P3] goto P2"), - /* 81 */ "Le" OpHelp("if r[P1]<=r[P3] goto P2"), - /* 82 */ "Lt" OpHelp("if r[P1]=r[P3] goto P2"), - /* 84 */ "ResetCount" OpHelp(""), - /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), - /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), - /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), - /* 89 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), - /* 90 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), - /* 91 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), - /* 92 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), - /* 93 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), - /* 94 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 95 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 96 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), - /* 97 */ "String8" OpHelp("r[P2]='P4'"), - /* 98 */ "SorterData" OpHelp("r[P2]=data"), - /* 99 */ "RowKey" OpHelp("r[P2]=key"), - /* 100 */ "RowData" OpHelp("r[P2]=data"), - /* 101 */ "Rowid" OpHelp("r[P2]=rowid"), - /* 102 */ "NullRow" OpHelp(""), - /* 103 */ "Last" OpHelp(""), - /* 104 */ "SorterSort" OpHelp(""), - /* 105 */ "Sort" OpHelp(""), - /* 106 */ "Rewind" OpHelp(""), - /* 107 */ "SorterInsert" OpHelp(""), - /* 108 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 109 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 110 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 111 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 112 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 113 */ "IdxLT" OpHelp("key=r[P3@P4]"), - /* 114 */ "IdxGE" OpHelp("key=r[P3@P4]"), - /* 115 */ "Destroy" OpHelp(""), - /* 116 */ "Clear" OpHelp(""), - /* 117 */ "ResetSorter" OpHelp(""), - /* 118 */ "CreateIndex" OpHelp("r[P2]=root iDb=P1"), - /* 119 */ "CreateTable" OpHelp("r[P2]=root iDb=P1"), - /* 120 */ "ParseSchema" OpHelp(""), - /* 121 */ "LoadAnalysis" OpHelp(""), - /* 122 */ "DropTable" OpHelp(""), - /* 123 */ "DropIndex" OpHelp(""), - /* 124 */ "DropTrigger" OpHelp(""), - /* 125 */ "IntegrityCk" OpHelp(""), - /* 126 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 127 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 128 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 129 */ "Program" OpHelp(""), - /* 130 */ "Param" OpHelp(""), - /* 131 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 132 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 133 */ "Real" OpHelp("r[P2]=P4"), - /* 134 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 135 */ "IfPos" OpHelp("if r[P1]>0 goto P2"), - /* 136 */ "IfNeg" OpHelp("r[P1]+=P3, if r[P1]<0 goto P2"), - /* 137 */ "IfZero" OpHelp("r[P1]+=P3, if r[P1]==0 goto P2"), - /* 138 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 139 */ "IncrVacuum" OpHelp(""), - /* 140 */ "Expire" OpHelp(""), - /* 141 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 142 */ "VBegin" OpHelp(""), - /* 143 */ "ToText" OpHelp(""), - /* 144 */ "ToBlob" OpHelp(""), - /* 145 */ "ToNumeric" OpHelp(""), - /* 146 */ "ToInt" OpHelp(""), - /* 147 */ "ToReal" OpHelp(""), - /* 148 */ "VCreate" OpHelp(""), - /* 149 */ "VDestroy" OpHelp(""), - /* 150 */ "VOpen" OpHelp(""), - /* 151 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 152 */ "VNext" OpHelp(""), - /* 153 */ "VRename" OpHelp(""), - /* 154 */ "Pagecount" OpHelp(""), - /* 155 */ "MaxPgcnt" OpHelp(""), - /* 156 */ "Init" OpHelp("Start at P2"), - /* 157 */ "Noop" OpHelp(""), - /* 158 */ "Explain" OpHelp(""), + static const char *const azName[] = { + /* 0 */ "Savepoint" OpHelp(""), + /* 1 */ "AutoCommit" OpHelp(""), + /* 2 */ "Transaction" OpHelp(""), + /* 3 */ "SorterNext" OpHelp(""), + /* 4 */ "PrevIfOpen" OpHelp(""), + /* 5 */ "NextIfOpen" OpHelp(""), + /* 6 */ "Prev" OpHelp(""), + /* 7 */ "Next" OpHelp(""), + /* 8 */ "Checkpoint" OpHelp(""), + /* 9 */ "JournalMode" OpHelp(""), + /* 10 */ "Vacuum" OpHelp(""), + /* 11 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 12 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 13 */ "Goto" OpHelp(""), + /* 14 */ "Gosub" OpHelp(""), + /* 15 */ "InitCoroutine" OpHelp(""), + /* 16 */ "Yield" OpHelp(""), + /* 17 */ "MustBeInt" OpHelp(""), + /* 18 */ "Jump" OpHelp(""), + /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), + /* 20 */ "Once" OpHelp(""), + /* 21 */ "If" OpHelp(""), + /* 22 */ "IfNot" OpHelp(""), + /* 23 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 25 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 26 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 27 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), + /* 28 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), + /* 29 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 30 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 31 */ "Found" OpHelp("key=r[P3@P4]"), + /* 32 */ "SeekRowid" OpHelp("intkey=r[P3]"), + /* 33 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 34 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), + /* 35 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), + /* 36 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), + /* 37 */ "Eq" OpHelp("IF r[P3]==r[P1]"), + /* 38 */ "Gt" OpHelp("IF r[P3]>r[P1]"), + /* 39 */ "Le" OpHelp("IF r[P3]<=r[P1]"), + /* 40 */ "Lt" OpHelp("IF r[P3]=r[P1]"), + /* 42 */ "ElseNotEq" OpHelp(""), + /* 43 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 44 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 45 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 47 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 48 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 49 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 50 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 51 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 52 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 53 */ "Last" OpHelp(""), + /* 54 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), + /* 55 */ "SorterSort" OpHelp(""), + /* 56 */ "Sort" OpHelp(""), + /* 57 */ "Rewind" OpHelp(""), + /* 58 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 59 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 60 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 61 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 62 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 63 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 64 */ "Program" OpHelp(""), + /* 65 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 66 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 67 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]-=P3, goto P2"), + /* 68 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 69 */ "IncrVacuum" OpHelp(""), + /* 70 */ "VNext" OpHelp(""), + /* 71 */ "Init" OpHelp("Start at P2"), + /* 72 */ "Return" OpHelp(""), + /* 73 */ "EndCoroutine" OpHelp(""), + /* 74 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 75 */ "Halt" OpHelp(""), + /* 76 */ "Integer" OpHelp("r[P2]=P1"), + /* 77 */ "Int64" OpHelp("r[P2]=P4"), + /* 78 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 79 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 80 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 81 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 82 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 83 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 84 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 85 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 86 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 87 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 88 */ "CollSeq" OpHelp(""), + /* 89 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), + /* 90 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), + /* 91 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 92 */ "RealAffinity" OpHelp(""), + /* 93 */ "Cast" OpHelp("affinity(r[P1])"), + /* 94 */ "Permutation" OpHelp(""), + /* 95 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 96 */ "Column" OpHelp("r[P3]=PX"), + /* 97 */ "String8" OpHelp("r[P2]='P4'"), + /* 98 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 99 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 100 */ "Count" OpHelp("r[P2]=count()"), + /* 101 */ "ReadCookie" OpHelp(""), + /* 102 */ "SetCookie" OpHelp(""), + /* 103 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 104 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 105 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 106 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 107 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 108 */ "SorterOpen" OpHelp(""), + /* 109 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 110 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 111 */ "Close" OpHelp(""), + /* 112 */ "ColumnsUsed" OpHelp(""), + /* 113 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 114 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 115 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 116 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), + /* 117 */ "Delete" OpHelp(""), + /* 118 */ "ResetCount" OpHelp(""), + /* 119 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 120 */ "SorterData" OpHelp("r[P2]=data"), + /* 121 */ "RowKey" OpHelp("r[P2]=key"), + /* 122 */ "RowData" OpHelp("r[P2]=data"), + /* 123 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 124 */ "NullRow" OpHelp(""), + /* 125 */ "SorterInsert" OpHelp(""), + /* 126 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 127 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 128 */ "Seek" OpHelp("Move P3 to P1.rowid"), + /* 129 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 130 */ "Destroy" OpHelp(""), + /* 131 */ "Clear" OpHelp(""), + /* 132 */ "Real" OpHelp("r[P2]=P4"), + /* 133 */ "ResetSorter" OpHelp(""), + /* 134 */ "CreateIndex" OpHelp("r[P2]=root iDb=P1"), + /* 135 */ "CreateTable" OpHelp("r[P2]=root iDb=P1"), + /* 136 */ "ParseSchema" OpHelp(""), + /* 137 */ "LoadAnalysis" OpHelp(""), + /* 138 */ "DropTable" OpHelp(""), + /* 139 */ "DropIndex" OpHelp(""), + /* 140 */ "DropTrigger" OpHelp(""), + /* 141 */ "IntegrityCk" OpHelp(""), + /* 142 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 143 */ "Param" OpHelp(""), + /* 144 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 145 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 146 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 147 */ "AggStep0" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 148 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 149 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 150 */ "Expire" OpHelp(""), + /* 151 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 152 */ "VBegin" OpHelp(""), + /* 153 */ "VCreate" OpHelp(""), + /* 154 */ "VDestroy" OpHelp(""), + /* 155 */ "VOpen" OpHelp(""), + /* 156 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 157 */ "VRename" OpHelp(""), + /* 158 */ "Pagecount" OpHelp(""), + /* 159 */ "MaxPgcnt" OpHelp(""), + /* 160 */ "CursorHint" OpHelp(""), + /* 161 */ "Noop" OpHelp(""), + /* 162 */ "Explain" OpHelp(""), }; return azName[i]; } @@ -27658,6 +32994,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ ** * Definitions of sqlite3_vfs objects for all locking methods ** plus implementations of sqlite3_os_init() and sqlite3_os_end(). */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_UNIX /* This file is used on unix only */ /* @@ -27685,16 +33022,17 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ # endif #endif -/* -** Define the OS_VXWORKS pre-processor macro to 1 if building on -** vxworks, or 0 otherwise. -*/ -#ifndef OS_VXWORKS -# if defined(__RTP__) || defined(_WRS_KERNEL) -# define OS_VXWORKS 1 -# else -# define OS_VXWORKS 0 -# endif +/* Use pread() and pwrite() if they are available */ +#if defined(__APPLE__) +# define HAVE_PREAD 1 +# define HAVE_PWRITE 1 +#endif +#if defined(HAVE_PREAD64) && defined(HAVE_PWRITE64) +# undef USE_PREAD +# define USE_PREAD64 1 +#elif defined(HAVE_PREAD) && defined(HAVE_PWRITE) +# undef USE_PREAD64 +# define USE_PREAD 1 #endif /* @@ -27708,21 +33046,33 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ #include #include #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 -# include +/* # include */ #endif -#if SQLITE_ENABLE_LOCKING_STYLE || OS_VXWORKS +#if SQLITE_ENABLE_LOCKING_STYLE # include -# if OS_VXWORKS -# include -# include -# else -# include -# include -# endif +# include +# include #endif /* SQLITE_ENABLE_LOCKING_STYLE */ -#if defined(__APPLE__) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) +#if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ + (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) +# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ + && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0)) +# define HAVE_GETHOSTUUID 1 +# else +# warning "gethostuuid() is disabled." +# endif +#endif + + +#if OS_VXWORKS +/* # include */ +# include +# include +#endif /* OS_VXWORKS */ + +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE # include #endif @@ -27763,6 +33113,15 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ */ #define MAX_PATHNAME 512 +/* +** Maximum supported symbolic links +*/ +#define SQLITE_MAX_SYMLINKS 100 + +/* Always cast the getpid() return type for compatibility with +** kernel modules in VxWorks. */ +#define osGetpid(X) (pid_t)getpid() + /* ** Only set the lastErrno if the error code is a real error and not ** a normal expected return code of SQLITE_BUSY or SQLITE_OK @@ -27851,7 +33210,7 @@ struct unixFile { ** method was called. If xOpen() is called from a different process id, ** indicating that a fork() has occurred, the PRNG will be reset. */ -static int randomnessPid = 0; +static pid_t randomnessPid = 0; /* ** Allowed values for the unixFile.ctrlFlags bitmask: @@ -27868,7 +33227,6 @@ static int randomnessPid = 0; #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ -#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings have been issued */ /* ** Include code that is common to all os_*.c files @@ -27906,24 +33264,14 @@ static int randomnessPid = 0; # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; -# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X -#else -# define OSTRACE(X) -#endif - /* ** Macros for performance tracing. Normally turned off. Only works ** on i486 hardware. */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing +/* +** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. */ /************** Include hwtime.h in the middle of os_common.h ****************/ @@ -27943,8 +33291,8 @@ static int randomnessPid = 0; ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H /* ** The following routine only works on pentium-class (or newer) processors. @@ -28012,7 +33360,7 @@ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } #endif -#endif /* !defined(_HWTIME_H_) */ +#endif /* !defined(SQLITE_HWTIME_H) */ /************** End of hwtime.h **********************************************/ /************** Continuing where we left off in os_common.h ******************/ @@ -28033,14 +33381,14 @@ static sqlite_uint64 g_elapsed; ** of code will give us the ability to simulate a disk I/O error. This ** is used for testing the I/O recovery logic. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ -SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ -SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ -SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ -SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ -SQLITE_API int sqlite3_diskfull_pending = 0; -SQLITE_API int sqlite3_diskfull = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_io_error_hit; +SQLITE_API extern int sqlite3_io_error_hardhit; +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_persist; +SQLITE_API extern int sqlite3_io_error_benign; +SQLITE_API extern int sqlite3_diskfull_pending; +SQLITE_API extern int sqlite3_diskfull; #define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ @@ -28066,17 +33414,17 @@ static void local_ioerr(){ #define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) -#endif +#endif /* defined(SQLITE_TEST) */ /* ** When testing, keep a count of the number of open files. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_open_file_count = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_open_file_count; #define OpenCounter(X) sqlite3_open_file_count+=(X) #else #define OpenCounter(X) -#endif +#endif /* defined(SQLITE_TEST) */ #endif /* !defined(_OS_COMMON_H_) */ @@ -28121,6 +33469,14 @@ SQLITE_API int sqlite3_open_file_count = 0; # endif #endif +/* +** Explicitly call the 64-bit version of lseek() on Android. Otherwise, lseek() +** is the 32-bit version, even if _FILE_OFFSET_BITS=64 is defined. +*/ +#ifdef __ANDROID__ +# define lseek lseek64 +#endif + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -28133,19 +33489,6 @@ static int posixOpen(const char *zFile, int flags, int mode){ return open(zFile, flags, mode); } -/* -** On some systems, calls to fchown() will trigger a message in a security -** log if they come from non-root processes. So avoid calling fchown() if -** we are not running as root. -*/ -static int posixFchown(int fd, uid_t uid, gid_t gid){ -#if OS_VXWORKS - return 0; -#else - return geteuid() ? 0 : fchown(fd,uid,gid); -#endif -} - /* Forward reference */ static int openDirectory(const char*, int*); static int unixGetpagesize(void); @@ -28199,7 +33542,7 @@ static struct unix_syscall { { "read", (sqlite3_syscall_ptr)read, 0 }, #define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) -#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE { "pread", (sqlite3_syscall_ptr)pread, 0 }, #else { "pread", (sqlite3_syscall_ptr)0, 0 }, @@ -28211,12 +33554,12 @@ static struct unix_syscall { #else { "pread64", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osPread64 ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[10].pCurrent) +#define osPread64 ((ssize_t(*)(int,void*,size_t,off64_t))aSyscall[10].pCurrent) { "write", (sqlite3_syscall_ptr)write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) -#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, #else { "pwrite", (sqlite3_syscall_ptr)0, 0 }, @@ -28229,10 +33572,10 @@ static struct unix_syscall { #else { "pwrite64", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ +#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off64_t))\ aSyscall[13].pCurrent) - { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, + { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, #define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -28254,29 +33597,74 @@ static struct unix_syscall { { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) - { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, +#if defined(HAVE_FCHOWN) + { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, +#else + { "fchown", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, +#define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) + #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 - { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, -#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#else + { "mmap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[22].pCurrent) +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, -#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) +#else + { "munmap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMunmap ((void*(*)(void*,size_t))aSyscall[23].pCurrent) -#if HAVE_MREMAP +#if HAVE_MREMAP && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) - { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, -#define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent) +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[24].pCurrent) +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, +#else + { "getpagesize", (sqlite3_syscall_ptr)0, 0 }, #endif +#define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) + +#if defined(HAVE_READLINK) + { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, +#else + { "readlink", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) + +#if defined(HAVE_LSTAT) + { "lstat", (sqlite3_syscall_ptr)lstat, 0 }, +#else + { "lstat", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) }; /* End of the overrideable system calls */ + +/* +** On some systems, calls to fchown() will trigger a message in a security +** log if they come from non-root processes. So avoid calling fchown() if +** we are not running as root. +*/ +static int robustFchown(int fd, uid_t uid, gid_t gid){ +#if defined(HAVE_FCHOWN) + return osGeteuid() ? 0 : osFchown(fd,uid,gid); +#else + return 0; +#endif +} + /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the ** "unix" VFSes. Return SQLITE_OK opon successfully updating the @@ -28438,22 +33826,22 @@ static int robust_open(const char *z, int f, mode_t m){ ** unixEnterLeave() */ static void unixEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } static void unixLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #ifdef SQLITE_DEBUG static int unixMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +#ifdef SQLITE_HAVE_OS_TRACE /* ** Helper function for printing out trace information from debugging -** binaries. This returns the string represetation of the supplied +** binaries. This returns the string representation of the supplied ** integer lock-type. */ static const char *azFileLock(int eFileLock){ @@ -28530,9 +33918,22 @@ static int lockTrace(int fd, int op, struct flock *p){ /* ** Retry ftruncate() calls that fail due to EINTR +** +** All calls to ftruncate() within this file should be made through +** this wrapper. On the Android platform, bypassing the logic below +** could lead to a corrupt database. */ static int robust_ftruncate(int h, sqlite3_int64 sz){ int rc; +#ifdef __ANDROID__ + /* On Android, ftruncate() always uses 32-bit offsets, even if + ** _FILE_OFFSET_BITS=64 is defined. This means it is unsafe to attempt to + ** truncate a file to any size larger than 2GiB. Silently ignore any + ** such attempts. */ + if( sz>(sqlite3_int64)0x7FFFFFFF ){ + rc = SQLITE_OK; + }else +#endif do{ rc = osFtruncate(h,sz); }while( rc<0 && errno==EINTR ); return rc; } @@ -28548,23 +33949,12 @@ static int robust_ftruncate(int h, sqlite3_int64 sz){ ** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. */ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { + assert( (sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + (sqliteIOErr == SQLITE_IOERR_RDLOCK) || + (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ); switch (posixError) { -#if 0 - /* At one point this code was not commented out. In theory, this branch - ** should never be hit, as this function should only be called after - ** a locking-related function (i.e. fcntl()) has returned non-zero with - ** the value of errno as the first argument. Since a system call has failed, - ** errno should be non-zero. - ** - ** Despite this, if errno really is zero, we still don't want to return - ** SQLITE_OK. The system call failed, and *some* SQLite error should be - ** propagated back to the caller. Commenting this branch out means errno==0 - ** will be handled by the "default:" case below. - */ - case 0: - return SQLITE_OK; -#endif - + case EACCES: case EAGAIN: case ETIMEDOUT: case EBUSY: @@ -28574,41 +33964,9 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { * introspection, in which it actually means what it says */ return SQLITE_BUSY; - case EACCES: - /* EACCES is like EAGAIN during locking operations, but not any other time*/ - if( (sqliteIOErr == SQLITE_IOERR_LOCK) || - (sqliteIOErr == SQLITE_IOERR_UNLOCK) || - (sqliteIOErr == SQLITE_IOERR_RDLOCK) || - (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ){ - return SQLITE_BUSY; - } - /* else fall through */ case EPERM: return SQLITE_PERM; -#if EOPNOTSUPP!=ENOTSUP - case EOPNOTSUPP: - /* something went terribly awry, unless during file system support - * introspection, in which it actually means what it says */ -#endif -#ifdef ENOTSUP - case ENOTSUP: - /* invalid fd, unless during file system support introspection, in which - * it actually means what it says */ -#endif - case EIO: - case EBADF: - case EINVAL: - case ENOTCONN: - case ENODEV: - case ENXIO: - case ENOENT: -#ifdef ESTALE /* ESTALE is not defined on Interix systems */ - case ESTALE: -#endif - case ENOSYS: - /* these should force the client to close the file and reconnect */ - default: return sqliteIOErr; } @@ -28700,7 +34058,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ assert( zAbsoluteName[0]=='/' ); n = (int)strlen(zAbsoluteName); - pNew = sqlite3_malloc( sizeof(*pNew) + (n+1) ); + pNew = sqlite3_malloc64( sizeof(*pNew) + (n+1) ); if( pNew==0 ) return 0; pNew->zCanonicalName = (char*)&pNew[1]; memcpy(pNew->zCanonicalName, zAbsoluteName, n+1); @@ -28892,7 +34250,7 @@ static unixInodeInfo *inodeList = 0; /* ** -** This function - unixLogError_x(), is only ever called via the macro +** This function - unixLogErrorAtLine(), is only ever called via the macro ** unixLogError(). ** ** It is invoked after an error occurs in an OS function and errno has been @@ -28979,6 +34337,14 @@ static void robust_close(unixFile *pFile, int h, int lineno){ } } +/* +** Set the pFile->lastErrno. Do this in a subroutine as that provides +** a convenient place to set a breakpoint. +*/ +static void storeLastErrno(unixFile *pFile, int error){ + pFile->lastErrno = error; +} + /* ** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. */ @@ -29052,8 +34418,8 @@ static int findInodeInfo( fd = pFile->h; rc = osFstat(fd, &statbuf); if( rc!=0 ){ - pFile->lastErrno = errno; -#ifdef EOVERFLOW + storeLastErrno(pFile, errno); +#if defined(EOVERFLOW) && defined(SQLITE_DISABLE_LFS) if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; #endif return SQLITE_IOERR; @@ -29073,12 +34439,12 @@ static int findInodeInfo( if( statbuf.st_size==0 && (pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS)!=0 ){ do{ rc = osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); if( rc!=1 ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return SQLITE_IOERR; } rc = osFstat(fd, &statbuf); if( rc!=0 ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return SQLITE_IOERR; } } @@ -29096,9 +34462,9 @@ static int findInodeInfo( pInode = pInode->pNext; } if( pInode==0 ){ - pInode = sqlite3_malloc( sizeof(*pInode) ); + pInode = sqlite3_malloc64( sizeof(*pInode) ); if( pInode==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(pInode, 0, sizeof(*pInode)); memcpy(&pInode->fileId, &fileId, sizeof(fileId)); @@ -29140,30 +34506,25 @@ static int fileHasMoved(unixFile *pFile){ static void verifyDbFile(unixFile *pFile){ struct stat buf; int rc; - if( pFile->ctrlFlags & UNIXFILE_WARNED ){ - /* One or more of the following warnings have already been issued. Do not - ** repeat them so as not to clutter the error log */ - return; - } + + /* These verifications occurs for the main database only */ + if( pFile->ctrlFlags & UNIXFILE_NOLOCK ) return; + rc = osFstat(pFile->h, &buf); if( rc!=0 ){ sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); - pFile->ctrlFlags |= UNIXFILE_WARNED; return; } - if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + if( buf.st_nlink==0 ){ sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); - pFile->ctrlFlags |= UNIXFILE_WARNED; return; } if( buf.st_nlink>1 ){ sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); - pFile->ctrlFlags |= UNIXFILE_WARNED; return; } if( fileHasMoved(pFile) ){ sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); - pFile->ctrlFlags |= UNIXFILE_WARNED; return; } } @@ -29183,6 +34544,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); + assert( pFile->eFileLock<=SHARED_LOCK ); unixEnterMutex(); /* Because pFile->pInode is shared across threads */ /* Check if a thread in this process holds such a lock */ @@ -29201,7 +34563,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ lock.l_type = F_WRLCK; if( osFcntl(pFile->h, F_GETLK, &lock) ){ rc = SQLITE_IOERR_CHECKRESERVEDLOCK; - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); } else if( lock.l_type!=F_UNLCK ){ reserved = 1; } @@ -29239,9 +34601,7 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ unixInodeInfo *pInode = pFile->pInode; assert( unixMutexHeld() ); assert( pInode!=0 ); - if( ((pFile->ctrlFlags & UNIXFILE_EXCL)!=0 || pInode->bProcessLock) - && ((pFile->ctrlFlags & UNIXFILE_RDONLY)==0) - ){ + if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ if( pInode->bProcessLock==0 ){ struct flock lock; assert( pInode->nLock==0 ); @@ -29291,7 +34651,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** lock transitions in terms of the POSIX advisory shared and exclusive ** lock primitives (called read-locks and write-locks below, to avoid ** confusion with SQLite lock names). The algorithms are complicated - ** slightly in order to be compatible with windows systems simultaneously + ** slightly in order to be compatible with Windows95 systems simultaneously ** accessing the same database file, in case that is ever required. ** ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved @@ -29299,8 +34659,14 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** range', a range of 510 bytes at a well known offset. ** ** To obtain a SHARED lock, a read-lock is obtained on the 'pending - ** byte'. If this is successful, a random byte from the 'shared byte - ** range' is read-locked and the lock on the 'pending byte' released. + ** byte'. If this is successful, 'shared byte range' is read-locked + ** and the lock on the 'pending byte' released. (Legacy note: When + ** SQLite was first developed, Windows95 systems were still very common, + ** and Widnows95 lacks a shared-lock capability. So on Windows95, a + ** single randomly selected by from the 'shared byte range' is locked. + ** Windows95 is now pretty much extinct, but this work-around for the + ** lack of shared-locks on Windows95 lives on, for backwards + ** compatibility.) ** ** A process may only obtain a RESERVED lock after it has a SHARED lock. ** A RESERVED lock is implemented by grabbing a write-lock on the @@ -29319,11 +34685,6 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** range'. Since all other locks require a read-lock on one of the bytes ** within this range, this ensures that no other locks are held on the ** database. - ** - ** The reason a single byte cannot be used instead of the 'shared byte - ** range' is that some versions of windows do not support read-locks. By - ** locking a random byte from a range, concurrent SHARED locks may exist - ** even if the locking primitive used is always a write-lock. */ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; @@ -29334,7 +34695,8 @@ static int unixLock(sqlite3_file *id, int eFileLock){ assert( pFile ); OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, azFileLock(eFileLock), azFileLock(pFile->eFileLock), - azFileLock(pFile->pInode->eFileLock), pFile->pInode->nShared , getpid())); + azFileLock(pFile->pInode->eFileLock), pFile->pInode->nShared, + osGetpid(0))); /* If there is already a lock of this type or more restrictive on the ** unixFile, do nothing. Don't use the end_lock: exit path, as @@ -29401,7 +34763,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( rc!=SQLITE_BUSY ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } goto end_lock; } @@ -29436,7 +34798,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ if( rc ){ if( rc!=SQLITE_BUSY ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } goto end_lock; }else{ @@ -29469,7 +34831,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( rc!=SQLITE_BUSY ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } } } @@ -29542,7 +34904,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ assert( pFile ); OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, eFileLock, pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, - getpid())); + osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); if( pFile->eFileLock<=eFileLock ){ @@ -29576,7 +34938,6 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ ** 4: [RRRR.] */ if( eFileLock==SHARED_LOCK ){ - #if !defined(__APPLE__) || !SQLITE_ENABLE_LOCKING_STYLE (void)handleNFSUnlock; assert( handleNFSUnlock==0 ); @@ -29593,9 +34954,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( unixFileLock(pFile, &lock)==(-1) ){ tErrno = errno; rc = SQLITE_IOERR_UNLOCK; - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } + storeLastErrno(pFile, tErrno); goto end_unlock; } lock.l_type = F_RDLCK; @@ -29606,7 +34965,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } goto end_unlock; } @@ -29617,9 +34976,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( unixFileLock(pFile, &lock)==(-1) ){ tErrno = errno; rc = SQLITE_IOERR_UNLOCK; - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } + storeLastErrno(pFile, tErrno); goto end_unlock; } }else @@ -29637,7 +34994,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ ** SQLITE_BUSY would confuse the upper layer (in practice it causes ** an assert to fail). */ rc = SQLITE_IOERR_RDLOCK; - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); goto end_unlock; } } @@ -29650,7 +35007,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ pInode->eFileLock = SHARED_LOCK; }else{ rc = SQLITE_IOERR_UNLOCK; - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); goto end_unlock; } } @@ -29668,7 +35025,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ pInode->eFileLock = NO_LOCK; }else{ rc = SQLITE_IOERR_UNLOCK; - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); pInode->eFileLock = NO_LOCK; pFile->eFileLock = NO_LOCK; } @@ -29870,17 +35227,7 @@ static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); - - /* Check if a thread in this process holds such a lock */ - if( pFile->eFileLock>SHARED_LOCK ){ - /* Either this connection or some other connection in the same process - ** holds a lock on the file. No need to check further. */ - reserved = 1; - }else{ - /* The lock is held if and only if the lockfile exists */ - const char *zLockFile = (const char*)pFile->lockingContext; - reserved = osAccess(zLockFile, 0)==0; - } + reserved = osAccess((const char*)pFile->lockingContext, 0)==0; OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); *pResOut = reserved; return rc; @@ -29942,8 +35289,8 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { rc = SQLITE_BUSY; } else { rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + if( rc!=SQLITE_BUSY ){ + storeLastErrno(pFile, tErrno); } } return rc; @@ -29970,7 +35317,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { assert( pFile ); OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); + pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); /* no-op if possible */ @@ -29989,15 +35336,13 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { /* To fully unlock the database, delete the lock file */ assert( eFileLock==NO_LOCK ); rc = osRmdir(zLockFile); - if( rc<0 && errno==ENOTDIR ) rc = osUnlink(zLockFile); if( rc<0 ){ int tErrno = errno; - rc = 0; - if( ENOENT != tErrno ){ + if( tErrno==ENOENT ){ + rc = SQLITE_OK; + }else{ rc = SQLITE_IOERR_UNLOCK; - } - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } return rc; } @@ -30009,14 +35354,11 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. Make sure the lock has been released before closing. */ static int dotlockClose(sqlite3_file *id) { - int rc = SQLITE_OK; - if( id ){ - unixFile *pFile = (unixFile*)id; - dotlockUnlock(id, NO_LOCK); - sqlite3_free(pFile->lockingContext); - rc = closeUnixFile(id); - } - return rc; + unixFile *pFile = (unixFile*)id; + assert( id!=0 ); + dotlockUnlock(id, NO_LOCK); + sqlite3_free(pFile->lockingContext); + return closeUnixFile(id); } /****************** End of the dot-file lock implementation ******************* ******************************************************************************/ @@ -30033,10 +35375,9 @@ static int dotlockClose(sqlite3_file *id) { ** still works when you do this, but concurrency is reduced since ** only a single process can be reading the database at a time. ** -** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off or if -** compiling for VXWORKS. +** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off */ -#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS +#if SQLITE_ENABLE_LOCKING_STYLE /* ** Retry flock() calls that fail with EINTR @@ -30083,10 +35424,8 @@ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ int tErrno = errno; /* unlock failed with an error */ lrc = SQLITE_IOERR_UNLOCK; - if( IS_LOCK_ERROR(lrc) ){ - pFile->lastErrno = tErrno; - rc = lrc; - } + storeLastErrno(pFile, tErrno); + rc = lrc; } } else { int tErrno = errno; @@ -30094,7 +35433,7 @@ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ /* someone else might have it reserved */ lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( IS_LOCK_ERROR(lrc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); rc = lrc; } } @@ -30160,7 +35499,7 @@ static int flockLock(sqlite3_file *id, int eFileLock) { /* didn't get, must be busy */ rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } } else { /* got it, set the type and return ok */ @@ -30189,7 +35528,7 @@ static int flockUnlock(sqlite3_file *id, int eFileLock) { assert( pFile ); OSTRACE(("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); + pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); /* no-op if possible */ @@ -30219,12 +35558,9 @@ static int flockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. */ static int flockClose(sqlite3_file *id) { - int rc = SQLITE_OK; - if( id ){ - flockUnlock(id, NO_LOCK); - rc = closeUnixFile(id); - } - return rc; + assert( id!=0 ); + flockUnlock(id, NO_LOCK); + return closeUnixFile(id); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ @@ -30250,7 +35586,7 @@ static int flockClose(sqlite3_file *id) { ** to a non-zero value otherwise *pResOut is set to zero. The return value ** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ -static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { +static int semXCheckReservedLock(sqlite3_file *id, int *pResOut) { int rc = SQLITE_OK; int reserved = 0; unixFile *pFile = (unixFile*)id; @@ -30272,7 +35608,7 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { int tErrno = errno; if( EAGAIN != tErrno ){ rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } else { /* someone else has the lock when we are in NO_LOCK */ reserved = (pFile->eFileLock < SHARED_LOCK); @@ -30317,7 +35653,7 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { ** This routine will only increase a lock. Use the sqlite3OsUnlock() ** routine to lower a locking level. */ -static int semLock(sqlite3_file *id, int eFileLock) { +static int semXLock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; sem_t *pSem = pFile->pInode->pSem; int rc = SQLITE_OK; @@ -30350,14 +35686,14 @@ static int semLock(sqlite3_file *id, int eFileLock) { ** If the locking level of the file descriptor is already at or below ** the requested locking level, this routine is a no-op. */ -static int semUnlock(sqlite3_file *id, int eFileLock) { +static int semXUnlock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; sem_t *pSem = pFile->pInode->pSem; assert( pFile ); assert( pSem ); OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); + pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); /* no-op if possible */ @@ -30376,7 +35712,7 @@ static int semUnlock(sqlite3_file *id, int eFileLock) { int rc, tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } return rc; } @@ -30387,10 +35723,10 @@ static int semUnlock(sqlite3_file *id, int eFileLock) { /* ** Close a file. */ -static int semClose(sqlite3_file *id) { +static int semXClose(sqlite3_file *id) { if( id ){ unixFile *pFile = (unixFile*)id; - semUnlock(id, NO_LOCK); + semXUnlock(id, NO_LOCK); assert( pFile ); unixEnterMutex(); releaseInodeInfo(pFile); @@ -30478,7 +35814,7 @@ static int afpSetLock( setLockFlag ? SQLITE_IOERR_LOCK : SQLITE_IOERR_UNLOCK); #endif /* SQLITE_IGNORE_AFP_LOCK_ERRORS */ if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } return rc; } else { @@ -30571,7 +35907,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ assert( pFile ); OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, azFileLock(eFileLock), azFileLock(pFile->eFileLock), - azFileLock(pInode->eFileLock), pInode->nShared , getpid())); + azFileLock(pInode->eFileLock), pInode->nShared , osGetpid(0))); /* If there is already a lock of this type or more restrictive on the ** unixFile, do nothing. Don't use the afp_end_lock: exit path, as @@ -30661,7 +35997,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); if( IS_LOCK_ERROR(lrc1) ) { - pFile->lastErrno = lrc1Errno; + storeLastErrno(pFile, lrc1Errno); rc = lrc1; goto afp_end_lock; } else if( IS_LOCK_ERROR(lrc2) ){ @@ -30757,7 +36093,7 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { assert( pFile ); OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, - getpid())); + osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); if( pFile->eFileLock<=eFileLock ){ @@ -30849,23 +36185,22 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { */ static int afpClose(sqlite3_file *id) { int rc = SQLITE_OK; - if( id ){ - unixFile *pFile = (unixFile*)id; - afpUnlock(id, NO_LOCK); - unixEnterMutex(); - if( pFile->pInode && pFile->pInode->nLock ){ - /* If there are outstanding locks, do not actually close the file just - ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->aPending. It will be automatically closed when - ** the last lock is cleared. - */ - setPendingFd(pFile); - } - releaseInodeInfo(pFile); - sqlite3_free(pFile->lockingContext); - rc = closeUnixFile(id); - unixLeaveMutex(); + unixFile *pFile = (unixFile*)id; + assert( id!=0 ); + afpUnlock(id, NO_LOCK); + unixEnterMutex(); + if( pFile->pInode && pFile->pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); } + releaseInodeInfo(pFile); + sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); + unixLeaveMutex(); return rc; } @@ -30920,7 +36255,7 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ ** NB: If you define USE_PREAD or USE_PREAD64, then it might also ** be necessary to define _XOPEN_SOURCE to be 500. This varies from ** one system to another. Since SQLite does not define USE_PREAD -** any any form by default, we will not attempt to define _XOPEN_SOURCE. +** in any form by default, we will not attempt to define _XOPEN_SOURCE. ** See tickets #2741 and #2681. ** ** To avoid stomping the errno value on a failed read the lastErrno value @@ -30935,7 +36270,6 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ TIMER_START; assert( cnt==(cnt&0x1ffff) ); assert( id->h>2 ); - cnt &= 0x1ffff; do{ #if defined(USE_PREAD) got = osPread(id->h, pBuf, cnt, offset); @@ -30945,13 +36279,9 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ SimulateIOError( got = -1 ); #else newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; - }else{ - ((unixFile*)id)->lastErrno = 0; - } + SimulateIOError( newOffset = -1 ); + if( newOffset<0 ){ + storeLastErrno((unixFile*)id, errno); return -1; } got = osRead(id->h, pBuf, cnt); @@ -30960,7 +36290,7 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ if( got<0 ){ if( errno==EINTR ){ got = 1; continue; } prior = 0; - ((unixFile*)id)->lastErrno = errno; + storeLastErrno((unixFile*)id, errno); break; }else if( got>0 ){ cnt -= got; @@ -31025,7 +36355,7 @@ static int unixRead( /* lastErrno set by seekAndRead */ return SQLITE_IOERR_READ; }else{ - pFile->lastErrno = 0; /* not a system error */ + storeLastErrno(pFile, 0); /* not a system error */ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[got], 0, amt-got); return SQLITE_IOERR_SHORT_READ; @@ -31050,21 +36380,21 @@ static int seekAndWriteFd( assert( nBuf==(nBuf&0x1ffff) ); assert( fd>2 ); + assert( piErrno!=0 ); nBuf &= 0x1ffff; TIMER_START; #if defined(USE_PREAD) - do{ rc = osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); + do{ rc = (int)osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); #elif defined(USE_PREAD64) - do{ rc = osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); + do{ rc = (int)osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); #else do{ i64 iSeek = lseek(fd, iOff, SEEK_SET); - SimulateIOError( iSeek-- ); - - if( iSeek!=iOff ){ - if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); - return -1; + SimulateIOError( iSeek = -1 ); + if( iSeek<0 ){ + rc = -1; + break; } rc = osWrite(fd, pBuf, nBuf); }while( rc<0 && errno==EINTR ); @@ -31073,7 +36403,7 @@ static int seekAndWriteFd( TIMER_END; OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); - if( rc<0 && piErrno ) *piErrno = errno; + if( rc<0 ) *piErrno = errno; return rc; } @@ -31136,7 +36466,7 @@ static int unixWrite( } #endif -#if SQLITE_MAX_MMAP_SIZE>0 +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -31152,8 +36482,8 @@ static int unixWrite( } } #endif - - while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ + + while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))0 ){ amt -= wrote; offset += wrote; pBuf = &((char*)pBuf)[wrote]; @@ -31161,12 +36491,12 @@ static int unixWrite( SimulateIOError(( wrote=(-1), amt=1 )); SimulateDiskfullError(( wrote=0, amt=1 )); - if( amt>0 ){ + if( amt>wrote ){ if( wrote<0 && pFile->lastErrno!=ENOSPC ){ /* lastErrno set by seekAndWrite */ return SQLITE_IOERR_WRITE; }else{ - pFile->lastErrno = 0; /* not a system error */ + storeLastErrno(pFile, 0); /* not a system error */ return SQLITE_FULL; } } @@ -31187,9 +36517,9 @@ SQLITE_API int sqlite3_fullsync_count = 0; ** We do not trust systems to provide a working fdatasync(). Some do. ** Others do no. To be safe, we will stick with the (slightly slower) ** fsync(). If you know that your system does support fdatasync() correctly, -** then simply compile with -Dfdatasync=fdatasync +** then simply compile with -Dfdatasync=fdatasync or -DHAVE_FDATASYNC */ -#if !defined(fdatasync) +#if !defined(fdatasync) && !HAVE_FDATASYNC # define fdatasync fsync #endif @@ -31257,10 +36587,15 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ #endif /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a - ** no-op + ** no-op. But go ahead and call fstat() to validate the file + ** descriptor as we need a method to provoke a failure during + ** coverate testing. */ #ifdef SQLITE_NO_SYNC - rc = SQLITE_OK; + { + struct stat buf; + rc = osFstat(fd, &buf); + } #elif HAVE_FULLFSYNC if( fullSync ){ rc = osFcntl(fd, F_FULLFSYNC, 0); @@ -31326,16 +36661,20 @@ static int openDirectory(const char *zFilename, int *pFd){ char zDirname[MAX_PATHNAME+1]; sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); - for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); + for(ii=(int)strlen(zDirname); ii>0 && zDirname[ii]!='/'; ii--); if( ii>0 ){ zDirname[ii] = '\0'; - fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); - if( fd>=0 ){ - OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); - } + }else{ + if( zDirname[0]!='/' ) zDirname[0] = '.'; + zDirname[1] = 0; + } + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); + if( fd>=0 ){ + OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); } *pFd = fd; - return (fd>=0?SQLITE_OK:unixLogError(SQLITE_CANTOPEN_BKPT, "open", zDirname)); + if( fd>=0 ) return SQLITE_OK; + return unixLogError(SQLITE_CANTOPEN_BKPT, "openDirectory", zDirname); } /* @@ -31375,7 +36714,7 @@ static int unixSync(sqlite3_file *id, int flags){ rc = full_fsync(pFile->h, isFullsync, isDataOnly); SimulateIOError( rc=1 ); if( rc ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return unixLogError(SQLITE_IOERR_FSYNC, "full_fsync", pFile->zPath); } @@ -31388,10 +36727,11 @@ static int unixSync(sqlite3_file *id, int flags){ OSTRACE(("DIRSYNC %s (have_fullfsync=%d fullsync=%d)\n", pFile->zPath, HAVE_FULLFSYNC, isFullsync)); rc = osOpenDirectory(pFile->zPath, &dirfd); - if( rc==SQLITE_OK && dirfd>=0 ){ + if( rc==SQLITE_OK ){ full_fsync(dirfd, 0, 0); robust_close(pFile, dirfd, __LINE__); - }else if( rc==SQLITE_CANTOPEN ){ + }else{ + assert( rc==SQLITE_CANTOPEN ); rc = SQLITE_OK; } pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; @@ -31417,9 +36757,9 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } - rc = robust_ftruncate(pFile->h, (off_t)nByte); + rc = robust_ftruncate(pFile->h, nByte); if( rc ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); }else{ #ifdef SQLITE_DEBUG @@ -31459,7 +36799,7 @@ static int unixFileSize(sqlite3_file *id, i64 *pSize){ rc = osFstat(((unixFile*)id)->h, &buf); SimulateIOError( rc=1 ); if( rc!=0 ){ - ((unixFile*)id)->lastErrno = errno; + storeLastErrno((unixFile*)id, errno); return SQLITE_IOERR_FSTAT; } *pSize = buf.st_size; @@ -31495,7 +36835,9 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ i64 nSize; /* Required file size */ struct stat buf; /* Used to hold return values of fstat() */ - if( osFstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; + if( osFstat(pFile->h, &buf) ){ + return SQLITE_IOERR_FSTAT; + } nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; if( nSize>(i64)buf.st_size ){ @@ -31510,24 +36852,24 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ }while( err==EINTR ); if( err ) return SQLITE_IOERR_WRITE; #else - /* If the OS does not have posix_fallocate(), fake it. First use - ** ftruncate() to set the file size, then write a single byte to - ** the last byte in each block within the extended region. This - ** is the same technique used by glibc to implement posix_fallocate() - ** on systems that do not have a real fallocate() system call. + /* If the OS does not have posix_fallocate(), fake it. Write a + ** single byte to the last byte in each block that falls entirely + ** within the extended region. Then, if required, a single byte + ** at offset (nSize-1), to set the size of the file correctly. + ** This is a similar technique to that used by glibc on systems + ** that do not have a real fallocate() call. */ int nBlk = buf.st_blksize; /* File-system block size */ + int nWrite = 0; /* Number of bytes written by seekAndWrite */ i64 iWrite; /* Next offset to write to */ - if( robust_ftruncate(pFile->h, nSize) ){ - pFile->lastErrno = errno; - return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); - } - iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; - while( iWrite=buf.st_size ); + assert( ((iWrite+1)%nBlk)==0 ); + for(/*no-op*/; iWrite=nSize ) iWrite = nSize - 1; + nWrite = seekAndWrite(pFile, iWrite, "", 1); if( nWrite!=1 ) return SQLITE_IOERR_WRITE; - iWrite += nBlk; } #endif } @@ -31538,7 +36880,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ int rc; if( pFile->szChunk<=0 ){ if( robust_ftruncate(pFile->h, nByte) ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); } } @@ -31552,7 +36894,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } /* -** If *pArg is inititially negative then this is a query. Set *pArg to +** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. ** ** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. @@ -31580,7 +36922,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(int*)pArg = pFile->eFileLock; return SQLITE_OK; } - case SQLITE_LAST_ERRNO: { + case SQLITE_FCNTL_LAST_ERRNO: { *(int*)pArg = pFile->lastErrno; return SQLITE_OK; } @@ -31608,7 +36950,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_TEMPFILENAME: { - char *zTFile = sqlite3_malloc( pFile->pVfs->mxPathname ); + char *zTFile = sqlite3_malloc64( pFile->pVfs->mxPathname ); if( zTFile ){ unixGetTempname(pFile->pVfs->mxPathname, zTFile); *(char**)pArg = zTFile; @@ -31649,8 +36991,8 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } #endif #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) - case SQLITE_SET_LOCKPROXYFILE: - case SQLITE_GET_LOCKPROXYFILE: { + case SQLITE_FCNTL_SET_LOCKPROXYFILE: + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { return proxyFileControl(id,op,pArg); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ @@ -31759,7 +37101,7 @@ static int unixSectorSize(sqlite3_file *id){ ** Return the device characteristics for the file. ** ** This VFS is set up to return SQLITE_IOCAP_POWERSAFE_OVERWRITE by default. -** However, that choice is contraversial since technically the underlying +** However, that choice is controversial since technically the underlying ** file system does not always provide powersafe overwrites. (In other ** words, after a power-loss event, parts of the file that were never ** written might end up being altered.) However, non-PSOW behavior is very, @@ -31790,7 +37132,9 @@ static int unixDeviceCharacteristics(sqlite3_file *id){ ** Instead, it should be called via macro osGetpagesize(). */ static int unixGetpagesize(void){ -#if defined(_BSD_SOURCE) +#if OS_VXWORKS + return 1024; +#elif defined(_BSD_SOURCE) return getpagesize(); #else return (int)sysconf(_SC_PAGESIZE); @@ -31883,22 +37227,24 @@ struct unixShm { ** otherwise. */ static int unixShmSystemLock( - unixShmNode *pShmNode, /* Apply locks to this open shared-memory segment */ + unixFile *pFile, /* Open connection to the WAL file */ int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ int ofst, /* First byte of the locking range */ int n /* Number of bytes to lock */ ){ - struct flock f; /* The posix advisory locking structure */ - int rc = SQLITE_OK; /* Result code form fcntl() */ + unixShmNode *pShmNode; /* Apply locks to this open shared-memory segment */ + struct flock f; /* The posix advisory locking structure */ + int rc = SQLITE_OK; /* Result code form fcntl() */ /* Access to the unixShmNode object is serialized by the caller */ + pShmNode = pFile->pInode->pShmNode; assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); /* Shared locks never span more than one byte */ assert( n==1 || lockType!=F_RDLCK ); /* Locks are within range */ - assert( n>=1 && n=1 && n<=SQLITE_SHM_NLOCK ); if( pShmNode->h>=0 ){ /* Initialize the locking parameters */ @@ -31976,7 +37322,7 @@ static int unixShmRegionPerMap(void){ static void unixShmPurge(unixFile *pFd){ unixShmNode *p = pFd->pInode->pShmNode; assert( unixMutexHeld() ); - if( p && p->nRef==0 ){ + if( p && ALWAYS(p->nRef==0) ){ int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); @@ -32042,8 +37388,8 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ int nShmFilename; /* Size of the SHM filename in bytes */ /* Allocate space for the new unixShm object. */ - p = sqlite3_malloc( sizeof(*p) ); - if( p==0 ) return SQLITE_NOMEM; + p = sqlite3_malloc64( sizeof(*p) ); + if( p==0 ) return SQLITE_NOMEM_BKPT; memset(p, 0, sizeof(*p)); assert( pDbFd->pShm==0 ); @@ -32055,12 +37401,15 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode = pInode->pShmNode; if( pShmNode==0 ){ struct stat sStat; /* fstat() info for database file */ +#ifndef SQLITE_SHM_DIRECTORY + const char *zBasePath = pDbFd->zPath; +#endif /* Call fstat() to figure out the permissions on the database file. If ** a new *-shm file is created, an attempt will be made to create it ** with the same permissions. */ - if( osFstat(pDbFd->h, &sStat) && pInode->bProcessLock==0 ){ + if( osFstat(pDbFd->h, &sStat) ){ rc = SQLITE_IOERR_FSTAT; goto shm_open_err; } @@ -32068,11 +37417,11 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ #ifdef SQLITE_SHM_DIRECTORY nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 31; #else - nShmFilename = 6 + (int)strlen(pDbFd->zPath); + nShmFilename = 6 + (int)strlen(zBasePath); #endif - pShmNode = sqlite3_malloc( sizeof(*pShmNode) + nShmFilename ); + pShmNode = sqlite3_malloc64( sizeof(*pShmNode) + nShmFilename ); if( pShmNode==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); @@ -32082,16 +37431,18 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", (u32)sStat.st_ino, (u32)sStat.st_dev); #else - sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", pDbFd->zPath); + sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", zBasePath); sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); #endif pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } } if( pInode->bProcessLock==0 ){ @@ -32110,19 +37461,19 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** is owned by the same user that owns the original database. Otherwise, ** the original owner will not be able to connect. */ - osFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); + robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. */ rc = SQLITE_OK; - if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ + if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ if( robust_ftruncate(pShmNode->h, 0) ){ rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); } } if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); } if( rc ) goto shm_open_err; } @@ -32247,7 +37598,8 @@ static int unixShmMap( /* Write to the last byte of each newly allocated or extended page */ assert( (nByte % pgsz)==0 ); for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ - if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ + int x = 0; + if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, &x)!=1 ){ const char *zFile = pShmNode->zFilename; rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); goto shmpage_out; @@ -32262,7 +37614,7 @@ static int unixShmMap( pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ - rc = SQLITE_IOERR_NOMEM; + rc = SQLITE_IOERR_NOMEM_BKPT; goto shmpage_out; } pShmNode->apRegion = apNew; @@ -32280,9 +37632,9 @@ static int unixShmMap( goto shmpage_out; } }else{ - pMem = sqlite3_malloc(szRegion); + pMem = sqlite3_malloc64(szRegion); if( pMem==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto shmpage_out; } memset(pMem, 0, szRegion); @@ -32354,7 +37706,7 @@ static int unixShmLock( /* Unlock the system-level locks */ if( (mask & allMask)==0 ){ - rc = unixShmSystemLock(pShmNode, F_UNLCK, ofst+UNIX_SHM_BASE, n); + rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -32382,7 +37734,7 @@ static int unixShmLock( /* Get shared locks at the system level, if necessary */ if( rc==SQLITE_OK ){ if( (allShared & mask)==0 ){ - rc = unixShmSystemLock(pShmNode, F_RDLCK, ofst+UNIX_SHM_BASE, n); + rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -32407,7 +37759,7 @@ static int unixShmLock( ** also mark the local connection as being locked. */ if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pShmNode, F_WRLCK, ofst+UNIX_SHM_BASE, n); + rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n); if( rc==SQLITE_OK ){ assert( (p->sharedMask & mask)==0 ); p->exclMask |= mask; @@ -32416,7 +37768,7 @@ static int unixShmLock( } sqlite3_mutex_leave(pShmNode->mutex); OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", - p->id, getpid(), p->sharedMask, p->exclMask)); + p->id, osGetpid(0), p->sharedMask, p->exclMask)); return rc; } @@ -32430,7 +37782,8 @@ static void unixShmBarrier( sqlite3_file *fd /* Database file holding the shared memory */ ){ UNUSED_PARAMETER(fd); - unixEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); } @@ -32475,7 +37828,9 @@ static int unixShmUnmap( assert( pShmNode->nRef>0 ); pShmNode->nRef--; if( pShmNode->nRef==0 ){ - if( deleteFlag && pShmNode->h>=0 ) osUnlink(pShmNode->zFilename); + if( deleteFlag && pShmNode->h>=0 ){ + osUnlink(pShmNode->zFilename); + } unixShmPurge(pDbFd); } unixLeaveMutex(); @@ -32538,7 +37893,9 @@ static void unixRemapfile( assert( pFd->mmapSizeActual>=pFd->mmapSize ); assert( MAP_FAILED!=0 ); +#ifdef SQLITE_MMAP_READWRITE if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; +#endif if( pOrig ){ #if HAVE_MREMAP @@ -32610,17 +37967,14 @@ static void unixRemapfile( ** recreated as a result of outstanding references) or an SQLite error ** code otherwise. */ -static int unixMapfile(unixFile *pFd, i64 nByte){ - i64 nMap = nByte; - int rc; - +static int unixMapfile(unixFile *pFd, i64 nMap){ assert( nMap>=0 || pFd->nFetchOut==0 ); + assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); if( pFd->nFetchOut>0 ) return SQLITE_OK; if( nMap<0 ){ struct stat statbuf; /* Low-level file information */ - rc = osFstat(pFd->h, &statbuf); - if( rc!=SQLITE_OK ){ + if( osFstat(pFd->h, &statbuf) ){ return SQLITE_IOERR_FSTAT; } nMap = statbuf.st_size; @@ -32629,12 +37983,9 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ nMap = pFd->mmapSizeMax; } + assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); if( nMap!=pFd->mmapSize ){ - if( nMap>0 ){ - unixRemapfile(pFd, nMap); - }else{ - unixUnmapfile(pFd); - } + unixRemapfile(pFd, nMap); } return SQLITE_OK; @@ -32731,7 +38082,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ ** looks at the filesystem type and tries to guess the best locking ** strategy from that. ** -** For finder-funtion F, two objects are created: +** For finder-function F, two objects are created: ** ** (1) The real finder-function named "FImpt()". ** @@ -32752,7 +38103,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ ** * An I/O method finder function called FINDER that returns a pointer ** to the METHOD object in the previous bullet. */ -#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK) \ +#define IOMETHODS(FINDER,METHOD,VERSION,CLOSE,LOCK,UNLOCK,CKLOCK,SHMMAP) \ static const sqlite3_io_methods METHOD = { \ VERSION, /* iVersion */ \ CLOSE, /* xClose */ \ @@ -32767,7 +38118,7 @@ static const sqlite3_io_methods METHOD = { \ unixFileControl, /* xFileControl */ \ unixSectorSize, /* xSectorSize */ \ unixDeviceCharacteristics, /* xDeviceCapabilities */ \ - unixShmMap, /* xShmMap */ \ + SHMMAP, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmUnmap, /* xShmUnmap */ \ @@ -32793,16 +38144,18 @@ IOMETHODS( unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + unixShmMap /* xShmMap method */ ) IOMETHODS( nolockIoFinder, /* Finder function name */ nolockIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ + 3, /* shared memory is disabled */ nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ - nolockCheckReservedLock /* xCheckReservedLock method */ + nolockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) IOMETHODS( dotlockIoFinder, /* Finder function name */ @@ -32811,10 +38164,11 @@ IOMETHODS( dotlockClose, /* xClose method */ dotlockLock, /* xLock method */ dotlockUnlock, /* xUnlock method */ - dotlockCheckReservedLock /* xCheckReservedLock method */ + dotlockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) -#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS +#if SQLITE_ENABLE_LOCKING_STYLE IOMETHODS( flockIoFinder, /* Finder function name */ flockIoMethods, /* sqlite3_io_methods object name */ @@ -32822,7 +38176,8 @@ IOMETHODS( flockClose, /* xClose method */ flockLock, /* xLock method */ flockUnlock, /* xUnlock method */ - flockCheckReservedLock /* xCheckReservedLock method */ + flockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -32831,10 +38186,11 @@ IOMETHODS( semIoFinder, /* Finder function name */ semIoMethods, /* sqlite3_io_methods object name */ 1, /* shared memory is disabled */ - semClose, /* xClose method */ - semLock, /* xLock method */ - semUnlock, /* xUnlock method */ - semCheckReservedLock /* xCheckReservedLock method */ + semXClose, /* xClose method */ + semXLock, /* xLock method */ + semXUnlock, /* xUnlock method */ + semXCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -32846,7 +38202,8 @@ IOMETHODS( afpClose, /* xClose method */ afpLock, /* xLock method */ afpUnlock, /* xUnlock method */ - afpCheckReservedLock /* xCheckReservedLock method */ + afpCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -32871,7 +38228,8 @@ IOMETHODS( proxyClose, /* xClose method */ proxyLock, /* xLock method */ proxyUnlock, /* xUnlock method */ - proxyCheckReservedLock /* xCheckReservedLock method */ + proxyCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -32884,7 +38242,8 @@ IOMETHODS( unixClose, /* xClose method */ unixLock, /* xLock method */ nfsUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -32954,15 +38313,13 @@ static const sqlite3_io_methods #endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ -#if OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE -/* -** This "finder" function attempts to determine the best locking strategy -** for the database file "filePath". It then returns the sqlite3_io_methods -** object that implements that strategy. -** -** This is for VXWorks only. +#if OS_VXWORKS +/* +** This "finder" function for VxWorks checks to see if posix advisory +** locking works. If it does, then that is what is used. If it does not +** work, then fallback to named semaphore locking. */ -static const sqlite3_io_methods *autolockIoFinderImpl( +static const sqlite3_io_methods *vxworksIoFinderImpl( const char *filePath, /* name of the database file */ unixFile *pNew /* the open file object */ ){ @@ -32988,12 +38345,12 @@ static const sqlite3_io_methods *autolockIoFinderImpl( } } static const sqlite3_io_methods - *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; + *(*const vxworksIoFinder)(const char*,unixFile*) = vxworksIoFinderImpl; -#endif /* OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE */ +#endif /* OS_VXWORKS */ /* -** An abstract type for a pointer to a IO method finder function: +** An abstract type for a pointer to an IO method finder function: */ typedef const sqlite3_io_methods *(*finder_type)(const char*,unixFile*); @@ -33055,7 +38412,7 @@ static int fillInUnixFile( pNew->pId = vxworksFindFileId(zFilename); if( pNew->pId==0 ){ ctrlFlags |= UNIXFILE_NOLOCK; - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } #endif @@ -33109,9 +38466,9 @@ static int fillInUnixFile( ** the afpLockingContext. */ afpLockingContext *pCtx; - pNew->lockingContext = pCtx = sqlite3_malloc( sizeof(*pCtx) ); + pNew->lockingContext = pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ /* NB: zFilename exists and remains valid until the file is closed ** according to requirement F11141. So we do not need to make a @@ -33139,9 +38496,9 @@ static int fillInUnixFile( int nFilename; assert( zFilename!=0 ); nFilename = (int)strlen(zFilename) + 6; - zLockFile = (char *)sqlite3_malloc(nFilename); + zLockFile = (char *)sqlite3_malloc64(nFilename); if( zLockFile==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); } @@ -33164,7 +38521,7 @@ static int fillInUnixFile( if( zSemName[n]=='/' ) zSemName[n] = '_'; pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); if( pNew->pInode->pSem == SEM_FAILED ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; pNew->pInode->aSemName[0] = '\0'; } } @@ -33172,7 +38529,7 @@ static int fillInUnixFile( } #endif - pNew->lastErrno = 0; + storeLastErrno(pNew, 0); #if OS_VXWORKS if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); @@ -33197,29 +38554,31 @@ static int fillInUnixFile( */ static const char *unixTempFileDir(void){ static const char *azDirs[] = { - 0, 0, 0, "/var/tmp", "/usr/tmp", "/tmp", - 0 /* List terminator */ + "." }; - unsigned int i; + unsigned int i = 0; struct stat buf; - const char *zDir = 0; + const char *zDir = sqlite3_temp_directory; - azDirs[0] = sqlite3_temp_directory; - if( !azDirs[1] ) azDirs[1] = getenv("SQLITE_TMPDIR"); - if( !azDirs[2] ) azDirs[2] = getenv("TMPDIR"); - for(i=0; i=sizeof(azDirs)/sizeof(azDirs[0]) ) break; + zDir = azDirs[i++]; } - return zDir; + return 0; } /* @@ -33228,38 +38587,26 @@ static const char *unixTempFileDir(void){ ** pVfs->mxPathname bytes. */ static int unixGetTempname(int nBuf, char *zBuf){ - static const unsigned char zChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789"; - unsigned int i, j; const char *zDir; + int iLimit = 0; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. */ + zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); zDir = unixTempFileDir(); - if( zDir==0 ) zDir = "."; - - /* Check that the output buffer is large enough for the temporary file - ** name. If it is not, return SQLITE_ERROR. - */ - if( (strlen(zDir) + strlen(SQLITE_TEMP_FILE_PREFIX) + 18) >= (size_t)nBuf ){ - return SQLITE_ERROR; - } - + if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; do{ - sqlite3_snprintf(nBuf-18, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX, zDir); - j = (int)strlen(zBuf); - sqlite3_randomness(15, &zBuf[j]); - for(i=0; i<15; i++, j++){ - zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; - } - zBuf[j] = 0; - zBuf[j+1] = 0; + u64 r; + sqlite3_randomness(sizeof(r), &r); + assert( nBuf>2 ); + zBuf[nBuf-2] = 0; + sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", + zDir, r, 0); + if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; }while( osAccess(zBuf,0)==0 ); return SQLITE_OK; } @@ -33307,7 +38654,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** descriptor on the same path, fail, and return an error to SQLite. ** ** Even if a subsequent open() call does succeed, the consequences of - ** not searching for a resusable file descriptor are not dire. */ + ** not searching for a reusable file descriptor are not dire. */ if( 0==osStat(zPath, &sStat) ){ unixInodeInfo *pInode; @@ -33331,6 +38678,27 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ return pUnused; } +/* +** Find the mode, uid and gid of file zFile. +*/ +static int getFileMode( + const char *zFile, /* File name */ + mode_t *pMode, /* OUT: Permissions of zFile */ + uid_t *pUid, /* OUT: uid of zFile. */ + gid_t *pGid /* OUT: gid of zFile. */ +){ + struct stat sStat; /* Output of stat() on database file */ + int rc = SQLITE_OK; + if( 0==osStat(zFile, &sStat) ){ + *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; + }else{ + rc = SQLITE_IOERR_FSTAT; + } + return rc; +} + /* ** This function is called by unixOpen() to determine the unix permissions ** to create new files with. If no error occurs, then SQLITE_OK is returned @@ -33338,7 +38706,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** written to *pMode. If an IO error occurs, an SQLite error code is ** returned and the value of *pMode is not modified. ** -** In most cases cases, this routine sets *pMode to 0, which will become +** In most cases, this routine sets *pMode to 0, which will become ** an indication to robust_open() to create the file using ** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. ** But if the file being opened is a WAL or regular journal file, then @@ -33366,7 +38734,6 @@ static int findCreateFileMode( if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ - struct stat sStat; /* Output of stat() on database file */ /* zPath is a path to a WAL or journal file. The following block derives ** the path to the associated database file from zPath. This block handles @@ -33381,28 +38748,34 @@ static int findCreateFileMode( ** used by the test_multiplex.c module. */ nDb = sqlite3Strlen30(zPath) - 1; -#ifdef SQLITE_ENABLE_8_3_NAMES - while( nDb>0 && sqlite3Isalnum(zPath[nDb]) ) nDb--; - if( nDb==0 || zPath[nDb]!='-' ) return SQLITE_OK; -#else while( zPath[nDb]!='-' ){ +#ifndef SQLITE_ENABLE_8_3_NAMES + /* In the normal case (8+3 filenames disabled) the journal filename + ** is guaranteed to contain a '-' character. */ assert( nDb>0 ); - assert( zPath[nDb]!='\n' ); + assert( sqlite3Isalnum(zPath[nDb]) ); +#else + /* If 8+3 names are possible, then the journal file might not contain + ** a '-' character. So check for that case and return early. */ + if( nDb==0 || zPath[nDb]=='.' ) return SQLITE_OK; +#endif nDb--; } -#endif memcpy(zDb, zPath, nDb); zDb[nDb] = '\0'; - if( 0==osStat(zDb, &sStat) ){ - *pMode = sStat.st_mode & 0777; - *pUid = sStat.st_uid; - *pGid = sStat.st_gid; - }else{ - rc = SQLITE_IOERR_FSTAT; - } + rc = getFileMode(zDb, pMode, pUid, pGid); }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ *pMode = 0600; + }else if( flags & SQLITE_OPEN_URI ){ + /* If this is a main database file and the file was opened using a URI + ** filename, check for the "modeof" parameter. If present, interpret + ** its value as a filename and try to copy the mode, uid and gid from + ** that file. */ + const char *z = sqlite3_uri_parameter(zPath, "modeof"); + if( z ){ + rc = getFileMode(z, pMode, pUid, pGid); + } } return rc; } @@ -33503,8 +38876,8 @@ static int unixOpen( ** the same instant might all reset the PRNG. But multiple resets ** are harmless. */ - if( randomnessPid!=getpid() ){ - randomnessPid = getpid(); + if( randomnessPid!=osGetpid(0) ){ + randomnessPid = osGetpid(0); sqlite3_randomness(0,0); } @@ -33516,9 +38889,9 @@ static int unixOpen( if( pUnused ){ fd = pUnused->fd; }else{ - pUnused = sqlite3_malloc(sizeof(*pUnused)); + pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } p->pUnused = pUnused; @@ -33531,7 +38904,7 @@ static int unixOpen( }else if( !zName ){ /* If zName is NULL, the upper layer is requesting a temp file. */ assert(isDelete && !syncDir); - rc = unixGetTempname(MAX_PATHNAME+2, zTmpname); + rc = unixGetTempname(pVfs->mxPathname, zTmpname); if( rc!=SQLITE_OK ){ return rc; } @@ -33564,7 +38937,8 @@ static int unixOpen( } fd = robust_open(zName, openFlags, openMode); OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); - if( fd<0 && errno!=EISDIR && isReadWrite && !isExclusive ){ + assert( !isExclusive || (openFlags & O_CREAT)!=0 ); + if( fd<0 && errno!=EISDIR && isReadWrite ){ /* Failed to open the file for read/write access. Try read-only. */ flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); @@ -33583,7 +38957,7 @@ static int unixOpen( ** the same as the original database. */ if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ - osFchown(fd, uid, gid); + robustFchown(fd, uid, gid); } } assert( fd>=0 ); @@ -33603,7 +38977,7 @@ static int unixOpen( zPath = sqlite3_mprintf("%s", zName); if( zPath==0 ){ robust_close(p, fd, __LINE__); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } #else osUnlink(zName); @@ -33614,24 +38988,25 @@ static int unixOpen( p->openFlags = openFlags; } #endif - - noLock = eType!=SQLITE_OPEN_MAIN_DB; - #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE if( fstatfs(fd, &fsInfo) == -1 ){ - ((unixFile*)pFile)->lastErrno = errno; + storeLastErrno(p, errno); robust_close(p, fd, __LINE__); return SQLITE_IOERR_ACCESS; } if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; } + if (0 == strncmp("exfat", fsInfo.f_fstypename, 5)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; + } #endif /* Set up appropriate ctrlFlags */ if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; + noLock = eType!=SQLITE_OPEN_MAIN_DB; if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; @@ -33649,19 +39024,6 @@ static int unixOpen( if( envforce!=NULL ){ useProxy = atoi(envforce)>0; }else{ - if( statfs(zPath, &fsInfo) == -1 ){ - /* In theory, the close(fd) call is sub-optimal. If the file opened - ** with fd is a database file, and there are other connections open - ** on that file that are currently holding advisory locks on it, - ** then the call to close() will cancel those locks. In practice, - ** we're assuming that statfs() doesn't fail very often. At least - ** not while other file descriptors opened by the same process on - ** the same file are working. */ - p->lastErrno = errno; - robust_close(p, fd, __LINE__); - rc = SQLITE_IOERR_ACCESS; - goto open_finished; - } useProxy = !(fsInfo.f_flags&MNT_LOCAL); } if( useProxy ){ @@ -33707,7 +39069,7 @@ static int unixDelete( if( osUnlink(zPath)==(-1) ){ if( errno==ENOENT #if OS_VXWORKS - || errno==0x380003 + || osAccess(zPath,0)!=0 #endif ){ rc = SQLITE_IOERR_DELETE_NOENT; @@ -33721,16 +39083,12 @@ static int unixDelete( int fd; rc = osOpenDirectory(zPath, &fd); if( rc==SQLITE_OK ){ -#if OS_VXWORKS - if( fsync(fd)==-1 ) -#else - if( fsync(fd) ) -#endif - { + if( full_fsync(fd,0,0) ){ rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); } robust_close(0, fd, __LINE__); - }else if( rc==SQLITE_CANTOPEN ){ + }else{ + assert( rc==SQLITE_CANTOPEN ); rc = SQLITE_OK; } } @@ -33754,33 +39112,49 @@ static int unixAccess( int flags, /* What do we want to learn about the zPath file? */ int *pResOut /* Write result boolean here */ ){ - int amode = 0; UNUSED_PARAMETER(NotUsed); SimulateIOError( return SQLITE_IOERR_ACCESS; ); - switch( flags ){ - case SQLITE_ACCESS_EXISTS: - amode = F_OK; - break; - case SQLITE_ACCESS_READWRITE: - amode = W_OK|R_OK; - break; - case SQLITE_ACCESS_READ: - amode = R_OK; - break; + assert( pResOut!=0 ); - default: - assert(!"Invalid flags argument"); - } - *pResOut = (osAccess(zPath, amode)==0); - if( flags==SQLITE_ACCESS_EXISTS && *pResOut ){ + /* The spec says there are three possible values for flags. But only + ** two of them are actually used */ + assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE ); + + if( flags==SQLITE_ACCESS_EXISTS ){ struct stat buf; - if( 0==osStat(zPath, &buf) && buf.st_size==0 ){ - *pResOut = 0; - } + *pResOut = (0==osStat(zPath, &buf) && buf.st_size>0); + }else{ + *pResOut = osAccess(zPath, W_OK|R_OK)==0; } return SQLITE_OK; } +/* +** +*/ +static int mkFullPathname( + const char *zPath, /* Input path */ + char *zOut, /* Output buffer */ + int nOut /* Allocated size of buffer zOut */ +){ + int nPath = sqlite3Strlen30(zPath); + int iOff = 0; + if( zPath[0]!='/' ){ + if( osGetcwd(zOut, nOut-2)==0 ){ + return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + } + iOff = sqlite3Strlen30(zOut); + zOut[iOff++] = '/'; + } + if( (iOff+nPath+1)>nOut ){ + /* SQLite assumes that xFullPathname() nul-terminates the output buffer + ** even if it returns an error. */ + zOut[iOff] = '\0'; + return SQLITE_CANTOPEN_BKPT; + } + sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); + return SQLITE_OK; +} /* ** Turn a relative pathname into a full pathname. The relative path @@ -33797,6 +39171,17 @@ static int unixFullPathname( int nOut, /* Size of output buffer in bytes */ char *zOut /* Output buffer */ ){ +#if !defined(HAVE_READLINK) || !defined(HAVE_LSTAT) + return mkFullPathname(zPath, zOut, nOut); +#else + int rc = SQLITE_OK; + int nByte; + int nLink = 1; /* Number of symbolic links followed so far */ + const char *zIn = zPath; /* Input path for each iteration of loop */ + char *zDel = 0; + + assert( pVfs->mxPathname==MAX_PATHNAME ); + UNUSED_PARAMETER(pVfs); /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -33805,21 +39190,62 @@ static int unixFullPathname( */ SimulateIOError( return SQLITE_ERROR ); - assert( pVfs->mxPathname==MAX_PATHNAME ); - UNUSED_PARAMETER(pVfs); + do { - zOut[nOut-1] = '\0'; - if( zPath[0]=='/' ){ - sqlite3_snprintf(nOut, zOut, "%s", zPath); - }else{ - int nCwd; - if( osGetcwd(zOut, nOut-1)==0 ){ - return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + /* Call stat() on path zIn. Set bLink to true if the path is a symbolic + ** link, or false otherwise. */ + int bLink = 0; + struct stat buf; + if( osLstat(zIn, &buf)!=0 ){ + if( errno!=ENOENT ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); + } + }else{ + bLink = S_ISLNK(buf.st_mode); } - nCwd = (int)strlen(zOut); - sqlite3_snprintf(nOut-nCwd, &zOut[nCwd], "/%s", zPath); - } - return SQLITE_OK; + + if( bLink ){ + if( zDel==0 ){ + zDel = sqlite3_malloc(nOut); + if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; + }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ + rc = SQLITE_CANTOPEN_BKPT; + } + + if( rc==SQLITE_OK ){ + nByte = osReadlink(zIn, zDel, nOut-1); + if( nByte<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); + }else{ + if( zDel[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nOut ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zDel[n], zDel, nByte+1); + memcpy(zDel, zIn, n); + nByte += n; + } + } + zDel[nByte] = '\0'; + } + } + + zIn = zDel; + } + + assert( rc!=SQLITE_OK || zIn!=zOut || zIn[0]=='/' ); + if( rc==SQLITE_OK && zIn!=zOut ){ + rc = mkFullPathname(zIn, zOut, nOut); + } + if( bLink==0 ) break; + zIn = zOut; + }while( rc==SQLITE_OK ); + + sqlite3_free(zDel); + return rc; +#endif /* HAVE_READLINK && HAVE_LSTAT */ } @@ -33905,8 +39331,8 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** tests repeatable. */ memset(zBuf, 0, nBuf); - randomnessPid = getpid(); -#if !defined(SQLITE_TEST) + randomnessPid = osGetpid(0); +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) { int fd, got; fd = robust_open("/dev/urandom", O_RDONLY, 0); @@ -33988,11 +39414,8 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; #else struct timeval sNow; - if( gettimeofday(&sNow, 0)==0 ){ - *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; - }else{ - rc = SQLITE_ERROR; - } + (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; #endif #ifdef SQLITE_TEST @@ -34004,6 +39427,7 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ return rc; } +#ifndef SQLITE_OMIT_DEPRECATED /* ** Find the current time (in Universal Coordinated Time). Write the ** current time and date as a Julian Day number into *prNow and @@ -34017,19 +39441,21 @@ static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ *prNow = i/86400000.0; return rc; } +#else +# define unixCurrentTime 0 +#endif /* -** We added the xGetLastError() method with the intention of providing -** better low-level error messages when operating-system problems come up -** during SQLite operation. But so far, none of that has been implemented -** in the core. So this routine is never called. For now, it is merely -** a place-holder. +** The xGetLastError() method is designed to return a better +** low-level error message when operating-system problems come up +** during SQLite operation. Only the integer return code is currently +** used. */ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ UNUSED_PARAMETER(NotUsed); UNUSED_PARAMETER(NotUsed2); UNUSED_PARAMETER(NotUsed3); - return 0; + return errno; } @@ -34087,9 +39513,10 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** ** C APIs ** -** sqlite3_file_control(db, dbname, SQLITE_SET_LOCKPROXYFILE, +** sqlite3_file_control(db, dbname, SQLITE_FCNTL_SET_LOCKPROXYFILE, ** | ":auto:"); -** sqlite3_file_control(db, dbname, SQLITE_GET_LOCKPROXYFILE, &); +** sqlite3_file_control(db, dbname, SQLITE_FCNTL_GET_LOCKPROXYFILE, +** &); ** ** ** SQL pragmas @@ -34130,7 +39557,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** proxy path against the values stored in the conch. The conch file is ** stored in the same directory as the database file and the file name ** is patterned after the database file name as ".-conch". -** If the conch file does not exist, or it's contents do not match the +** If the conch file does not exist, or its contents do not match the ** host ID and/or proxy path, then the lock is escalated to an exclusive ** lock and the conch file contents is updated with the host ID and proxy ** path and the lock is downgraded to a shared lock again. If the conch @@ -34182,7 +39609,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will ** force proxy locking to be used for every database file opened, and 0 ** will force automatic proxy locking to be disabled for all database -** files (explicity calling the SQLITE_SET_LOCKPROXYFILE pragma or +** files (explicitly calling the SQLITE_FCNTL_SET_LOCKPROXYFILE pragma or ** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). */ @@ -34203,6 +39630,7 @@ struct proxyLockingContext { char *lockProxyPath; /* Name of the proxy lock file */ char *dbPath; /* Name of the open file */ int conchHeld; /* 1 if the conch is held, -1 if lockless */ + int nFails; /* Number of conch taking failures */ void *oldLockingContext; /* Original lockingcontext to restore on close */ sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ }; @@ -34224,7 +39652,7 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ { if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", - lPath, errno, getpid())); + lPath, errno, osGetpid(0))); return SQLITE_IOERR_LOCK; } len = strlcat(lPath, "sqliteplocks", maxLen); @@ -34246,7 +39674,7 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ } lPath[i+len]='\0'; strlcat(lPath, ":auto:", maxLen); - OSTRACE(("GETLOCKPATH proxy lock path=%s pid=%d\n", lPath, getpid())); + OSTRACE(("GETLOCKPATH proxy lock path=%s pid=%d\n", lPath, osGetpid(0))); return SQLITE_OK; } @@ -34273,7 +39701,7 @@ static int proxyCreateLockPath(const char *lockPath){ if( err!=EEXIST ) { OSTRACE(("CREATELOCKPATH FAILED creating %s, " "'%s' proxy lock path=%s pid=%d\n", - buf, strerror(err), lockPath, getpid())); + buf, strerror(err), lockPath, osGetpid(0))); return err; } } @@ -34282,7 +39710,7 @@ static int proxyCreateLockPath(const char *lockPath){ } buf[i] = lockPath[i]; } - OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n", lockPath, getpid())); + OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n",lockPath,osGetpid(0))); return 0; } @@ -34316,9 +39744,9 @@ static int proxyCreateUnixFile( if( pUnused ){ fd = pUnused->fd; }else{ - pUnused = sqlite3_malloc(sizeof(*pUnused)); + pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } if( fd<0 ){ @@ -34349,9 +39777,9 @@ static int proxyCreateUnixFile( } } - pNew = (unixFile *)sqlite3_malloc(sizeof(*pNew)); + pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); if( pNew==NULL ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto end_create_proxy; } memset(pNew, 0, sizeof(unixFile)); @@ -34382,8 +39810,10 @@ SQLITE_API int sqlite3_hostid_num = 0; #define PROXY_HOSTIDLEN 16 /* conch file host id length */ +#ifdef HAVE_GETHOSTUUID /* Not always defined in the headers as it ought to be */ extern int gethostuuid(uuid_t id, const struct timespec *wait); +#endif /* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN ** bytes of writable memory. @@ -34391,10 +39821,9 @@ extern int gethostuuid(uuid_t id, const struct timespec *wait); static int proxyGetHostID(unsigned char *pHostID, int *pError){ assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); memset(pHostID, 0, PROXY_HOSTIDLEN); -#if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\ - && __MAC_OS_X_VERSION_MIN_REQUIRED<1050 +#ifdef HAVE_GETHOSTUUID { - static const struct timespec timeout = {1, 0}; /* 1 sec timeout */ + struct timespec timeout = {1, 0}; /* 1 sec timeout */ if( gethostuuid(pHostID, &timeout) ){ int err = errno; if( pError ){ @@ -34509,7 +39938,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ */ struct stat buf; if( osFstat(conchFile->h, &buf) ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return SQLITE_IOERR_LOCK; } @@ -34529,7 +39958,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ char tBuf[PROXY_MAXCONCHLEN]; int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); if( len<0 ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return SQLITE_IOERR_LOCK; } if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ @@ -34549,7 +39978,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ if( 0==proxyBreakConchLock(pFile, myHostID) ){ rc = SQLITE_OK; if( lockType==EXCLUSIVE_LOCK ){ - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); } if( !rc ){ rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); @@ -34587,11 +40016,12 @@ static int proxyTakeConch(unixFile *pFile){ int forceNewLockPath = 0; OSTRACE(("TAKECONCH %d for %s pid=%d\n", conchFile->h, - (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), getpid())); + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + osGetpid(0))); rc = proxyGetHostID(myHostID, &pError); if( (rc&0xff)==SQLITE_IOERR ){ - pFile->lastErrno = pError; + storeLastErrno(pFile, pError); goto end_takeconch; } rc = proxyConchLock(pFile, myHostID, SHARED_LOCK); @@ -34602,7 +40032,7 @@ static int proxyTakeConch(unixFile *pFile){ readLen = seekAndRead((unixFile*)conchFile, 0, readBuf, PROXY_MAXCONCHLEN); if( readLen<0 ){ /* I/O error: lastErrno set by seekAndRead */ - pFile->lastErrno = conchFile->lastErrno; + storeLastErrno(pFile, conchFile->lastErrno); rc = SQLITE_IOERR_READ; goto end_takeconch; }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || @@ -34675,7 +40105,7 @@ static int proxyTakeConch(unixFile *pFile){ rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); } }else{ - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, EXCLUSIVE_LOCK); + rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); } if( rc==SQLITE_OK ){ char writeBuffer[PROXY_MAXCONCHLEN]; @@ -34684,14 +40114,15 @@ static int proxyTakeConch(unixFile *pFile){ writeBuffer[0] = (char)PROXY_CONCHVERSION; memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); if( pCtx->lockProxyPath!=NULL ){ - strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, MAXPATHLEN); + strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, + MAXPATHLEN); }else{ strlcpy(&writeBuffer[PROXY_PATHINDEX], tempLockPath, MAXPATHLEN); } writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); robust_ftruncate(conchFile->h, writeSize); rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); - fsync(conchFile->h); + full_fsync(conchFile->h,0,0); /* If we created a new conch file (not just updated the contents of a ** valid conch file), try to match the permissions of the database */ @@ -34761,7 +40192,7 @@ static int proxyTakeConch(unixFile *pFile){ if( tempLockPath ){ pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); if( !pCtx->lockProxyPath ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } } @@ -34796,7 +40227,7 @@ static int proxyReleaseConch(unixFile *pFile){ conchFile = pCtx->conchFile; OSTRACE(("RELEASECONCH %d for %s pid=%d\n", conchFile->h, (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), - getpid())); + osGetpid(0))); if( pCtx->conchHeld>0 ){ rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); } @@ -34808,7 +40239,7 @@ static int proxyReleaseConch(unixFile *pFile){ /* ** Given the name of a database file, compute the name of its conch file. -** Store the conch filename in memory obtained from sqlite3_malloc(). +** Store the conch filename in memory obtained from sqlite3_malloc64(). ** Make *pConchPath point to the new name. Return SQLITE_OK on success ** or SQLITE_NOMEM if unable to obtain memory. ** @@ -34824,9 +40255,9 @@ static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ /* Allocate space for the conch filename and initialize the name to ** the name of the original database file. */ - *pConchPath = conchPath = (char *)sqlite3_malloc(len + 8); + *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); if( conchPath==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(conchPath, dbPath, len+1); @@ -34896,7 +40327,8 @@ static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ /* afp style keeps a reference to the db path in the filePath field ** of the struct */ assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); - strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, MAXPATHLEN); + strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, + MAXPATHLEN); } else #endif if( pFile->pMethod == &dotlockIoMethods ){ @@ -34937,11 +40369,11 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { } OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, - (lockPath ? lockPath : ":auto:"), getpid())); + (lockPath ? lockPath : ":auto:"), osGetpid(0))); - pCtx = sqlite3_malloc( sizeof(*pCtx) ); + pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(pCtx, 0, sizeof(*pCtx)); @@ -34977,7 +40409,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { if( rc==SQLITE_OK ){ pCtx->dbPath = sqlite3DbStrDup(0, dbPath); if( pCtx->dbPath==NULL ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } if( rc==SQLITE_OK ){ @@ -35009,7 +40441,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { */ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ switch( op ){ - case SQLITE_GET_LOCKPROXYFILE: { + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { unixFile *pFile = (unixFile*)id; if( pFile->pMethod == &proxyIoMethods ){ proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; @@ -35024,13 +40456,16 @@ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_SET_LOCKPROXYFILE: { + case SQLITE_FCNTL_SET_LOCKPROXYFILE: { unixFile *pFile = (unixFile*)id; int rc = SQLITE_OK; int isProxyStyle = (pFile->pMethod == &proxyIoMethods); if( pArg==NULL || (const char *)pArg==0 ){ if( isProxyStyle ){ - /* turn off proxy locking - not supported */ + /* turn off proxy locking - not supported. If support is added for + ** switching proxy locking mode off then it will need to fail if + ** the journal mode is WAL mode. + */ rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; }else{ /* turn off proxy locking - already off - NOOP */ @@ -35160,7 +40595,7 @@ static int proxyUnlock(sqlite3_file *id, int eFileLock) { ** Close a file that uses proxy locks. */ static int proxyClose(sqlite3_file *id) { - if( id ){ + if( ALWAYS(id) ){ unixFile *pFile = (unixFile*)id; proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; unixFile *lockProxy = pCtx->lockProxy; @@ -35275,8 +40710,10 @@ SQLITE_API int sqlite3_os_init(void){ ** array cannot be const. */ static sqlite3_vfs aVfs[] = { -#if SQLITE_ENABLE_LOCKING_STYLE && (OS_VXWORKS || defined(__APPLE__)) +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) UNIXVFS("unix", autolockIoFinder ), +#elif OS_VXWORKS + UNIXVFS("unix", vxworksIoFinder ), #else UNIXVFS("unix", posixIoFinder ), #endif @@ -35286,11 +40723,11 @@ SQLITE_API int sqlite3_os_init(void){ #if OS_VXWORKS UNIXVFS("unix-namedsem", semIoFinder ), #endif -#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_ENABLE_LOCKING_STYLE || OS_VXWORKS UNIXVFS("unix-posix", posixIoFinder ), -#if !OS_VXWORKS - UNIXVFS("unix-flock", flockIoFinder ), #endif +#if SQLITE_ENABLE_LOCKING_STYLE + UNIXVFS("unix-flock", flockIoFinder ), #endif #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) UNIXVFS("unix-afp", afpIoFinder ), @@ -35302,7 +40739,7 @@ SQLITE_API int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==25 ); + assert( ArraySize(aSyscall)==28 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ @@ -35340,6 +40777,7 @@ SQLITE_API int sqlite3_os_end(void){ ** ** This file contains code that is specific to Windows. */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN /* This file is used for Windows only */ /* @@ -35378,24 +40816,14 @@ SQLITE_API int sqlite3_os_end(void){ # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; -# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X -#else -# define OSTRACE(X) -#endif - /* ** Macros for performance tracing. Normally turned off. Only works ** on i486 hardware. */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing +/* +** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. */ /************** Include hwtime.h in the middle of os_common.h ****************/ @@ -35415,8 +40843,8 @@ SQLITE_API int sqlite3_os_end(void){ ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H /* ** The following routine only works on pentium-class (or newer) processors. @@ -35484,7 +40912,7 @@ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } #endif -#endif /* !defined(_HWTIME_H_) */ +#endif /* !defined(SQLITE_HWTIME_H) */ /************** End of hwtime.h **********************************************/ /************** Continuing where we left off in os_common.h ******************/ @@ -35505,14 +40933,14 @@ static sqlite_uint64 g_elapsed; ** of code will give us the ability to simulate a disk I/O error. This ** is used for testing the I/O recovery logic. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ -SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ -SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ -SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ -SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ -SQLITE_API int sqlite3_diskfull_pending = 0; -SQLITE_API int sqlite3_diskfull = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_io_error_hit; +SQLITE_API extern int sqlite3_io_error_hardhit; +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_persist; +SQLITE_API extern int sqlite3_io_error_benign; +SQLITE_API extern int sqlite3_diskfull_pending; +SQLITE_API extern int sqlite3_diskfull; #define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ @@ -35538,17 +40966,17 @@ static void local_ioerr(){ #define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) -#endif +#endif /* defined(SQLITE_TEST) */ /* ** When testing, keep a count of the number of open files. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_open_file_count = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_open_file_count; #define OpenCounter(X) sqlite3_open_file_count+=(X) #else #define OpenCounter(X) -#endif +#endif /* defined(SQLITE_TEST) */ #endif /* !defined(_OS_COMMON_H_) */ @@ -35558,6 +40986,7 @@ SQLITE_API int sqlite3_open_file_count = 0; /* ** Include the header file for the Windows VFS. */ +/* #include "os_win.h" */ /* ** Compiling and using WAL mode requires several APIs that are only @@ -35568,6 +40997,11 @@ SQLITE_API int sqlite3_open_file_count = 0; with SQLITE_OMIT_WAL." #endif +#if !SQLITE_OS_WINNT && SQLITE_MAX_MMAP_SIZE>0 +# error "Memory mapped files require support from the Windows NT kernel,\ + compile with SQLITE_MAX_MMAP_SIZE=0." +#endif + /* ** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions ** based on the sub-platform)? @@ -35605,6 +41039,10 @@ SQLITE_API int sqlite3_open_file_count = 0; # define NTDDI_WINBLUE 0x06030000 #endif +#ifndef NTDDI_WINTHRESHOLD +# define NTDDI_WINTHRESHOLD 0x06040000 +#endif + /* ** Check to see if the GetVersionEx[AW] functions are deprecated on the ** target system. GetVersionEx was first deprecated in Win8.1. @@ -35617,6 +41055,19 @@ SQLITE_API int sqlite3_open_file_count = 0; # endif #endif +/* +** Check to see if the CreateFileMappingA function is supported on the +** target system. It is unavailable when using "mincore.lib" on Win10. +** When compiling for Windows 10, always assume "mincore.lib" is in use. +*/ +#ifndef SQLITE_WIN32_CREATEFILEMAPPINGA +# if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINTHRESHOLD +# define SQLITE_WIN32_CREATEFILEMAPPINGA 0 +# else +# define SQLITE_WIN32_CREATEFILEMAPPINGA 1 +# endif +#endif + /* ** This constant should already be defined (in the "WinDef.h" SDK file). */ @@ -35697,10 +41148,11 @@ SQLITE_API int sqlite3_open_file_count = 0; /* ** Do we need to manually define the Win32 file mapping APIs for use with WAL -** mode (e.g. these APIs are available in the Windows CE SDK; however, they -** are not present in the header file)? +** mode or memory mapped files (e.g. these APIs are available in the Windows +** CE SDK; however, they are not present in the header file)? */ -#if SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) +#if SQLITE_WIN32_FILEMAPPING_API && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) /* ** Two of the file mapping APIs are different under WinRT. Figure out which ** set we need. @@ -35725,10 +41177,12 @@ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); #endif /* SQLITE_OS_WINRT */ /* -** This file mapping API is common to both Win32 and WinRT. +** These file mapping APIs are common to both Win32 and WinRT. */ + +WINBASEAPI BOOL WINAPI FlushViewOfFile(LPCVOID, SIZE_T); WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); -#endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ +#endif /* SQLITE_WIN32_FILEMAPPING_API */ /* ** Some Microsoft compilers lack this definition. @@ -35799,6 +41253,17 @@ struct winFile { #endif }; +/* +** The winVfsAppData structure is used for the pAppData member for all of the +** Win32 VFS variants. +*/ +typedef struct winVfsAppData winVfsAppData; +struct winVfsAppData { + const sqlite3_io_methods *pMethod; /* The file I/O methods to use. */ + void *pAppData; /* The extra pAppData, if any. */ + BOOL bNoLock; /* Non-zero if locking is disabled. */ +}; + /* ** Allowed values for winFile.ctrlFlags */ @@ -35853,11 +41318,23 @@ struct winFile { # define SQLITE_WIN32_HEAP_CREATE (TRUE) #endif +/* + * This is cache size used in the calculation of the initial size of the + * Win32-specific heap. It cannot be negative. + */ +#ifndef SQLITE_WIN32_CACHE_SIZE +# if SQLITE_DEFAULT_CACHE_SIZE>=0 +# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) +# else +# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) +# endif +#endif + /* * The initial size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_INIT_SIZE -# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_DEFAULT_CACHE_SIZE) * \ +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) #endif @@ -35944,9 +41421,9 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetWin32(void); ** can manually set this value to 1 to emulate Win98 behavior. */ #ifdef SQLITE_TEST -SQLITE_API LONG volatile sqlite3_os_type = 0; +SQLITE_API LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; #else -static LONG volatile sqlite3_os_type = 0; +static LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; #endif #ifndef SYSCALL @@ -36020,8 +41497,9 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) -#if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ - !defined(SQLITE_OMIT_WAL)) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) && \ + SQLITE_WIN32_CREATEFILEMAPPINGA { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, #else { "CreateFileMappingA", (SYSCALL)0, 0 }, @@ -36031,7 +41509,7 @@ static struct win_syscall { DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) #if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - !defined(SQLITE_OMIT_WAL)) + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -36251,8 +41729,7 @@ static struct win_syscall { #define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) -#if defined(SQLITE_WIN32_HAS_ANSI) && defined(SQLITE_WIN32_GETVERSIONEX) && \ - SQLITE_WIN32_GETVERSIONEX +#if defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_GETVERSIONEX { "GetVersionExA", (SYSCALL)GetVersionExA, 0 }, #else { "GetVersionExA", (SYSCALL)0, 0 }, @@ -36262,7 +41739,7 @@ static struct win_syscall { LPOSVERSIONINFOA))aSyscall[34].pCurrent) #if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX + SQLITE_WIN32_GETVERSIONEX { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, #else { "GetVersionExW", (SYSCALL)0, 0 }, @@ -36371,7 +41848,8 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[48].pCurrent) #endif -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL)) +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -36441,7 +41919,7 @@ static struct win_syscall { #define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ LPOVERLAPPED))aSyscall[58].pCurrent) -#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, #else { "UnmapViewOfFile", (SYSCALL)0, 0 }, @@ -36477,7 +41955,7 @@ static struct win_syscall { #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[63].pCurrent) -#if SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, #else { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, @@ -36504,7 +41982,7 @@ static struct win_syscall { #define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, #else { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, @@ -36568,7 +42046,7 @@ static struct win_syscall { #define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, #else { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, @@ -36589,10 +42067,36 @@ static struct win_syscall { #else { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, -#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG volatile*, \ - LONG,LONG))aSyscall[76].pCurrent) +#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG \ + SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) #endif /* defined(InterlockedCompareExchange) */ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { "UuidCreate", (SYSCALL)UuidCreate, 0 }, +#else + { "UuidCreate", (SYSCALL)0, 0 }, +#endif + +#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[77].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { "UuidCreateSequential", (SYSCALL)UuidCreateSequential, 0 }, +#else + { "UuidCreateSequential", (SYSCALL)0, 0 }, +#endif + +#define osUuidCreateSequential \ + ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[78].pCurrent) + +#if !defined(SQLITE_NO_SYNC) && SQLITE_MAX_MMAP_SIZE>0 + { "FlushViewOfFile", (SYSCALL)FlushViewOfFile, 0 }, +#else + { "FlushViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osFlushViewOfFile \ + ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -36704,7 +42208,7 @@ SQLITE_API int sqlite3_win32_compact_heap(LPUINT pnLargest){ if( lastErrno==NO_ERROR ){ sqlite3_log(SQLITE_NOMEM, "failed to HeapCompact (no space), heap=%p", (void*)hHeap); - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ sqlite3_log(SQLITE_ERROR, "failed to HeapCompact (%lu), heap=%p", osGetLastError(), (void*)hHeap); @@ -36776,6 +42280,12 @@ SQLITE_API void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */ if( nMin<-1 ) nMin = -1; /* all negative values become -1. */ assert( nMin==-1 || nMin==0 || nMin0 ){ memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); @@ -36824,6 +42334,16 @@ SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds){ #endif } +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ + SQLITE_THREADSAFE>0 +SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; +} +#endif + /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, ** or WinCE. Return false (zero) for Win95, Win98, or WinME. @@ -36836,7 +42356,7 @@ SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds){ ** the LockFileEx() API. */ -#if !defined(SQLITE_WIN32_GETVERSIONEX) || !SQLITE_WIN32_GETVERSIONEX +#if !SQLITE_WIN32_GETVERSIONEX # define osIsNT() (1) #elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) # define osIsNT() (1) @@ -36851,27 +42371,36 @@ SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds){ ** based on the NT kernel. */ SQLITE_API int sqlite3_win32_is_nt(void){ -#if defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX +#if SQLITE_OS_WINRT + /* + ** NOTE: The WinRT sub-platform is always assumed to be based on the NT + ** kernel. + */ + return 1; +#elif SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WIN8 - OSVERSIONINFOW sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - osGetVersionExW(&sInfo); - osInterlockedCompareExchange(&sqlite3_os_type, - (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); -#elif defined(SQLITE_WIN32_HAS_ANSI) +#if defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); osGetVersionExA(&sInfo); osInterlockedCompareExchange(&sqlite3_os_type, (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#elif defined(SQLITE_WIN32_HAS_WIDE) + OSVERSIONINFOW sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExW(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); #endif } return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; #elif SQLITE_TEST return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; #else + /* + ** NOTE: All sub-platforms where the GetVersionEx[AW] functions are + ** deprecated are always assumed to be based on the NT kernel. + */ return 1; #endif } @@ -37005,7 +42534,7 @@ static int winMemInit(void *pAppData){ "failed to HeapCreate (%lu), flags=%u, initSize=%lu, maxSize=%lu", osGetLastError(), SQLITE_WIN32_HEAP_FLAGS, dwInitialSize, dwMaximumSize); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pWinMemData->bOwned = TRUE; assert( pWinMemData->bOwned ); @@ -37015,7 +42544,7 @@ static int winMemInit(void *pAppData){ if( !pWinMemData->hHeap ){ sqlite3_log(SQLITE_NOMEM, "failed to GetProcessHeap (%lu)", osGetLastError()); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pWinMemData->bOwned = FALSE; assert( !pWinMemData->bOwned ); @@ -37082,147 +42611,244 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ #endif /* SQLITE_WIN32_MALLOC */ /* -** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). +** Convert a UTF-8 string to Microsoft Unicode. ** -** Space to hold the returned string is obtained from malloc. +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static LPWSTR winUtf8ToUnicode(const char *zFilename){ +static LPWSTR winUtf8ToUnicode(const char *zText){ int nChar; - LPWSTR zWideFilename; + LPWSTR zWideText; - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); if( nChar==0 ){ return 0; } - zWideFilename = sqlite3MallocZero( nChar*sizeof(zWideFilename[0]) ); - if( zWideFilename==0 ){ + zWideText = sqlite3MallocZero( nChar*sizeof(WCHAR) ); + if( zWideText==0 ){ return 0; } - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, nChar); if( nChar==0 ){ - sqlite3_free(zWideFilename); - zWideFilename = 0; + sqlite3_free(zWideText); + zWideText = 0; } - return zWideFilename; + return zWideText; } /* -** Convert Microsoft Unicode to UTF-8. Space to hold the returned string is -** obtained from sqlite3_malloc(). +** Convert a Microsoft Unicode string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static char *winUnicodeToUtf8(LPCWSTR zWideFilename){ +static char *winUnicodeToUtf8(LPCWSTR zWideText){ int nByte; - char *zFilename; + char *zText; - nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); if( nByte == 0 ){ return 0; } - zFilename = sqlite3MallocZero( nByte ); - if( zFilename==0 ){ + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ return 0; } - nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, 0, 0); if( nByte == 0 ){ - sqlite3_free(zFilename); - zFilename = 0; + sqlite3_free(zText); + zText = 0; } - return zFilename; + return zText; } /* -** Convert an ANSI string to Microsoft Unicode, based on the -** current codepage settings for file apis. +** Convert an ANSI string to Microsoft Unicode, using the ANSI or OEM +** code page. ** -** Space to hold the returned string is obtained -** from sqlite3_malloc. +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static LPWSTR winMbcsToUnicode(const char *zFilename){ +static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ int nByte; - LPWSTR zMbcsFilename; - int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; + LPWSTR zMbcsText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; - nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, NULL, + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, 0)*sizeof(WCHAR); if( nByte==0 ){ return 0; } - zMbcsFilename = sqlite3MallocZero( nByte*sizeof(zMbcsFilename[0]) ); - if( zMbcsFilename==0 ){ + zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); + if( zMbcsText==0 ){ return 0; } - nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, nByte); if( nByte==0 ){ - sqlite3_free(zMbcsFilename); - zMbcsFilename = 0; + sqlite3_free(zMbcsText); + zMbcsText = 0; } - return zMbcsFilename; + return zMbcsText; } /* -** Convert Microsoft Unicode to multi-byte character string, based on the -** user's ANSI codepage. +** Convert a Microsoft Unicode string to a multi-byte character string, +** using the ANSI or OEM code page. ** -** Space to hold the returned string is obtained from -** sqlite3_malloc(). +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static char *winUnicodeToMbcs(LPCWSTR zWideFilename){ +static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ int nByte; - char *zFilename; - int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; + char *zText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; - nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, 0, 0, 0, 0); if( nByte == 0 ){ return 0; } - zFilename = sqlite3MallocZero( nByte ); - if( zFilename==0 ){ + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ return 0; } - nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, zText, nByte, 0, 0); if( nByte == 0 ){ - sqlite3_free(zFilename); - zFilename = 0; + sqlite3_free(zText); + zText = 0; } - return zFilename; + return zText; } /* -** Convert multibyte character string to UTF-8. Space to hold the -** returned string is obtained from sqlite3_malloc(). +** Convert a multi-byte character string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ - char *zFilenameUtf8; +static char *winMbcsToUtf8(const char *zText, int useAnsi){ + char *zTextUtf8; LPWSTR zTmpWide; - zTmpWide = winMbcsToUnicode(zFilename); + zTmpWide = winMbcsToUnicode(zText, useAnsi); if( zTmpWide==0 ){ return 0; } - zFilenameUtf8 = winUnicodeToUtf8(zTmpWide); + zTextUtf8 = winUnicodeToUtf8(zTmpWide); sqlite3_free(zTmpWide); - return zFilenameUtf8; + return zTextUtf8; } /* -** Convert UTF-8 to multibyte character string. Space to hold the -** returned string is obtained from sqlite3_malloc(). +** Convert a UTF-8 string to a multi-byte character string. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ - char *zFilenameMbcs; +static char *winUtf8ToMbcs(const char *zText, int useAnsi){ + char *zTextMbcs; LPWSTR zTmpWide; - zTmpWide = winUtf8ToUnicode(zFilename); + zTmpWide = winUtf8ToUnicode(zText); if( zTmpWide==0 ){ return 0; } - zFilenameMbcs = winUnicodeToMbcs(zTmpWide); + zTextMbcs = winUnicodeToMbcs(zTmpWide, useAnsi); sqlite3_free(zTmpWide); - return zFilenameMbcs; + return zTextMbcs; +} + +/* +** This is a public wrapper for the winUtf8ToUnicode() function. +*/ +SQLITE_API LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToUnicode(zText); +} + +/* +** This is a public wrapper for the winUnicodeToUtf8() function. +*/ +SQLITE_API char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zWideText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUnicodeToUtf8(zWideText); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +SQLITE_API char *sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, useAnsi); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +SQLITE_API char *sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, useAnsi); } /* @@ -37252,7 +42878,7 @@ SQLITE_API int sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ if( zValue && zValue[0] ){ zValueUtf8 = winUnicodeToUtf8(zValue); if ( zValueUtf8==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } sqlite3_free(*ppDirectory); @@ -37324,7 +42950,7 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ sqlite3BeginBenignMalloc(); - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); sqlite3EndBenignMalloc(); /* free the system buffer allocated by FormatMessage */ osLocalFree(zTemp); @@ -37457,25 +43083,26 @@ static int winRetryIoerr(int *pnRetry, DWORD *pError){ /* ** Log a I/O error retry episode. */ -static void winLogIoerr(int nRetry){ +static void winLogIoerr(int nRetry, int lineno){ if( nRetry ){ - sqlite3_log(SQLITE_IOERR, - "delayed %dms for lock/sharing conflict", - winIoerrRetryDelay*nRetry*(nRetry+1)/2 + sqlite3_log(SQLITE_NOTICE, + "delayed %dms for lock/sharing conflict at line %d", + winIoerrRetryDelay*nRetry*(nRetry+1)/2, lineno ); } } -#if SQLITE_OS_WINCE -/************************************************************************* -** This section contains code for WinCE only. -*/ -#if !defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API /* -** The MSVC CRT on Windows CE may not have a localtime() function. So -** create a substitute. +** This #if does not rely on the SQLITE_OS_WINCE define because the +** corresponding section in "date.c" cannot use it. */ -/* #include */ +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So define a substitute. +*/ +/* # include */ struct tm *__cdecl localtime(const time_t *t) { static struct tm y; @@ -37499,6 +43126,10 @@ struct tm *__cdecl localtime(const time_t *t) } #endif +#if SQLITE_OS_WINCE +/************************************************************************* +** This section contains code for WinCE only. +*/ #define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] /* @@ -37529,7 +43160,7 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ zName = winUtf8ToUnicode(zFilename); if( zName==0 ){ /* out of memory */ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } /* Initialize the local lockdata */ @@ -37941,7 +43572,8 @@ static int winClose(sqlite3_file *id){ assert( pFile->pShm==0 ); #endif assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); - OSTRACE(("CLOSE file=%p\n", pFile->h)); + OSTRACE(("CLOSE pid=%lu, pFile=%p, file=%p\n", + osGetCurrentProcessId(), pFile, pFile->h)); #if SQLITE_MAX_MMAP_SIZE>0 winUnmapfile(pFile); @@ -37953,7 +43585,12 @@ static int winClose(sqlite3_file *id){ }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); #if SQLITE_OS_WINCE #define WINCE_DELETION_ATTEMPTS 3 - winceDestroyLock(pFile); + { + winVfsAppData *pAppData = (winVfsAppData*)pFile->pVfs->pAppData; + if( pAppData==NULL || !pAppData->bNoLock ){ + winceDestroyLock(pFile); + } + } if( pFile->zDeleteOnClose ){ int cnt = 0; while( @@ -37970,7 +43607,8 @@ static int winClose(sqlite3_file *id){ pFile->h = NULL; } OpenCounter(-1); - OSTRACE(("CLOSE file=%p, rc=%s\n", pFile->h, rc ? "ok" : "failed")); + OSTRACE(("CLOSE pid=%lu, pFile=%p, file=%p, rc=%s\n", + osGetCurrentProcessId(), pFile, pFile->h, rc ? "ok" : "failed")); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), "winClose", pFile->zPath); @@ -37987,7 +43625,7 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) OVERLAPPED overlapped; /* The offset for ReadFile. */ #endif winFile *pFile = (winFile*)id; /* file handle */ @@ -37998,7 +43636,8 @@ static int winRead( assert( amt>0 ); assert( offset>=0 ); SimulateIOError(return SQLITE_IOERR_READ); - OSTRACE(("READ file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, buffer=%p, amount=%d, " + "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 @@ -38007,7 +43646,8 @@ static int winRead( if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); - OSTRACE(("READ-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("READ-MMAP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; }else{ int nCopy = (int)(pFile->mmapSize - offset); @@ -38019,9 +43659,10 @@ static int winRead( } #endif -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) if( winSeekFile(pFile, offset) ){ - OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_FULL\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_FULL; } while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ @@ -38035,19 +43676,22 @@ static int winRead( DWORD lastErrno; if( winRetryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; - OSTRACE(("READ file=%p, rc=SQLITE_IOERR_READ\n", pFile->h)); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_READ\n", + osGetCurrentProcessId(), pFile, pFile->h)); return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, "winRead", pFile->zPath); } - winLogIoerr(nRetry); + winLogIoerr(nRetry, __LINE__); if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); - OSTRACE(("READ file=%p, rc=SQLITE_IOERR_SHORT_READ\n", pFile->h)); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_SHORT_READ\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_IOERR_SHORT_READ; } - OSTRACE(("READ file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; } @@ -38070,16 +43714,18 @@ static int winWrite( SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); - OSTRACE(("WRITE file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, buffer=%p, amount=%d, " + "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, pFile->h, pBuf, amt, offset, pFile->locktype)); -#if SQLITE_MAX_MMAP_SIZE>0 +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); - OSTRACE(("WRITE-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("WRITE-MMAP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; }else{ int nCopy = (int)(pFile->mmapSize - offset); @@ -38091,13 +43737,13 @@ static int winWrite( } #endif -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) rc = winSeekFile(pFile, offset); if( rc==0 ){ #else { #endif -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) OVERLAPPED overlapped; /* The offset for WriteFile. */ #endif u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ @@ -38105,14 +43751,14 @@ static int winWrite( DWORD nWrite; /* Bytes written by each WriteFile() call */ DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) memset(&overlapped, 0, sizeof(OVERLAPPED)); overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); #endif while( nRem>0 ){ -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ #else if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ @@ -38125,7 +43771,7 @@ static int winWrite( lastErrno = osGetLastError(); break; } -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) offset += nWrite; overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); @@ -38142,17 +43788,20 @@ static int winWrite( if( rc ){ if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) || ( pFile->lastErrno==ERROR_DISK_FULL )){ - OSTRACE(("WRITE file=%p, rc=SQLITE_FULL\n", pFile->h)); + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_FULL\n", + osGetCurrentProcessId(), pFile, pFile->h)); return winLogError(SQLITE_FULL, pFile->lastErrno, "winWrite1", pFile->zPath); } - OSTRACE(("WRITE file=%p, rc=SQLITE_IOERR_WRITE\n", pFile->h)); + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_WRITE\n", + osGetCurrentProcessId(), pFile, pFile->h)); return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, "winWrite2", pFile->zPath); }else{ - winLogIoerr(nRetry); + winLogIoerr(nRetry, __LINE__); } - OSTRACE(("WRITE file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; } @@ -38166,8 +43815,8 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ assert( pFile ); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - OSTRACE(("TRUNCATE file=%p, size=%lld, lock=%d\n", - pFile->h, nByte, pFile->locktype)); + OSTRACE(("TRUNCATE pid=%lu, pFile=%p, file=%p, size=%lld, lock=%d\n", + osGetCurrentProcessId(), pFile, pFile->h, nByte, pFile->locktype)); /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the @@ -38199,7 +43848,8 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ } #endif - OSTRACE(("TRUNCATE file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + OSTRACE(("TRUNCATE pid=%lu, pFile=%p, file=%p, rc=%s\n", + osGetCurrentProcessId(), pFile, pFile->h, sqlite3ErrName(rc))); return rc; } @@ -38223,7 +43873,7 @@ static int winSync(sqlite3_file *id, int flags){ BOOL rc; #endif #if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || \ - (defined(SQLITE_TEST) && defined(SQLITE_DEBUG)) + defined(SQLITE_HAVE_OS_TRACE) /* ** Used when SQLITE_NO_SYNC is not defined and by the assert() and/or ** OSTRACE() macros. @@ -38244,8 +43894,9 @@ static int winSync(sqlite3_file *id, int flags){ */ SimulateDiskfullError( return SQLITE_FULL ); - OSTRACE(("SYNC file=%p, flags=%x, lock=%d\n", - pFile->h, flags, pFile->locktype)); + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, flags=%x, lock=%d\n", + osGetCurrentProcessId(), pFile, pFile->h, flags, + pFile->locktype)); #ifndef SQLITE_TEST UNUSED_PARAMETER(flags); @@ -38260,19 +43911,38 @@ static int winSync(sqlite3_file *id, int flags){ ** no-op */ #ifdef SQLITE_NO_SYNC - OSTRACE(("SYNC-NOP file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("SYNC-NOP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; #else +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->pMapRegion ){ + if( osFlushViewOfFile(pFile->pMapRegion, 0) ){ + OSTRACE(("SYNC-MMAP pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_OK\n", osGetCurrentProcessId(), + pFile, pFile->pMapRegion)); + }else{ + pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC-MMAP pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), + pFile, pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winSync1", pFile->zPath); + } + } +#endif rc = osFlushFileBuffers(pFile->h); SimulateIOError( rc=FALSE ); if( rc ){ - OSTRACE(("SYNC file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; }else{ pFile->lastErrno = osGetLastError(); - OSTRACE(("SYNC file=%p, rc=SQLITE_IOERR_FSYNC\n", pFile->h)); + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_FSYNC\n", + osGetCurrentProcessId(), pFile, pFile->h)); return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, - "winSync", pFile->zPath); + "winSync2", pFile->zPath); } #endif } @@ -38461,6 +44131,12 @@ static int winLock(sqlite3_file *id, int locktype){ return SQLITE_OK; } + /* Do not allow any kind of write-lock on a read-only database + */ + if( (pFile->ctrlFlags & WINFILE_RDONLY)!=0 && locktype>=RESERVED_LOCK ){ + return SQLITE_IOERR_LOCK; + } + /* Make sure the locking sequence is correct */ assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); @@ -38472,9 +44148,8 @@ static int winLock(sqlite3_file *id, int locktype){ ** the PENDING_LOCK byte is temporary. */ newLocktype = pFile->locktype; - if( (pFile->locktype==NO_LOCK) - || ( (locktype==EXCLUSIVE_LOCK) - && (pFile->locktype==RESERVED_LOCK)) + if( pFile->locktype==NO_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) ){ int cnt = 3; while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, @@ -38590,7 +44265,7 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ res = 1; OSTRACE(("TEST-WR-LOCK file=%p, result=%d (local)\n", pFile->h, res)); }else{ - res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE,0,1,0); if( res ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } @@ -38647,8 +44322,46 @@ static int winUnlock(sqlite3_file *id, int locktype){ return rc; } +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int winNolockLock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +static int winNolockCheckReservedLock(sqlite3_file *id, int *pResOut){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(pResOut); + return SQLITE_OK; +} + +static int winNolockUnlock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ + /* -** If *pArg is inititially negative then this is a query. Set *pArg to +** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. ** ** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. @@ -38680,7 +44393,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } - case SQLITE_LAST_ERRNO: { + case SQLITE_FCNTL_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; @@ -38738,6 +44451,12 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } + case SQLITE_FCNTL_WIN32_GET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + *phFile = pFile->h; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } #ifdef SQLITE_TEST case SQLITE_FCNTL_WIN32_SET_HANDLE: { LPHANDLE phFile = (LPHANDLE)pArg; @@ -38830,14 +44549,14 @@ static SYSTEM_INFO winSysInfo; ** winShmLeaveMutex() */ static void winShmEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } static void winShmLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #ifndef NDEBUG static int winShmMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #endif @@ -38880,7 +44599,7 @@ struct winShmNode { int nRef; /* Number of winShm objects pointing to this */ winShm *pFirst; /* All winShm objects pointing to this */ winShmNode *pNext; /* Next in list of all winShmNode objects */ -#ifdef SQLITE_DEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 nextShmId; /* Next available winShm.id value */ #endif }; @@ -38911,7 +44630,7 @@ struct winShm { u8 hasMutex; /* True if holding the winShmNode mutex */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ -#ifdef SQLITE_DEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 id; /* Id of this connection with its winShmNode */ #endif }; @@ -38925,12 +44644,12 @@ struct winShm { /* ** Apply advisory locks for all n bytes beginning at ofst. */ -#define _SHM_UNLCK 1 -#define _SHM_RDLCK 2 -#define _SHM_WRLCK 3 +#define WINSHM_UNLCK 1 +#define WINSHM_RDLCK 2 +#define WINSHM_WRLCK 3 static int winShmSystemLock( winShmNode *pFile, /* Apply locks to this open shared-memory segment */ - int lockType, /* _SHM_UNLCK, _SHM_RDLCK, or _SHM_WRLCK */ + int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ int ofst, /* Offset to first byte to be locked/unlocked */ int nByte /* Number of bytes to lock or unlock */ ){ @@ -38943,12 +44662,12 @@ static int winShmSystemLock( pFile->hFile.h, lockType, ofst, nByte)); /* Release/Acquire the system-level lock */ - if( lockType==_SHM_UNLCK ){ + if( lockType==WINSHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); }else{ /* Initialize the locking parameters */ DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; - if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); } @@ -38960,7 +44679,7 @@ static int winShmSystemLock( } OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", - pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; @@ -39038,12 +44757,12 @@ static int winOpenSharedMemory(winFile *pDbFd){ ** allocate space for a new winShmNode and filename. */ p = sqlite3MallocZero( sizeof(*p) ); - if( p==0 ) return SQLITE_IOERR_NOMEM; + if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; nName = sqlite3Strlen30(pDbFd->zPath); pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); if( pNew==0 ){ sqlite3_free(p); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); @@ -39068,10 +44787,12 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->pNext = winShmNodeList; winShmNodeList = pShmNode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shm_open_err; + } } rc = winOpen(pDbFd->pVfs, @@ -39086,7 +44807,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. */ - if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ + if( winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); if( rc!=SQLITE_OK ){ rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), @@ -39094,15 +44815,15 @@ static int winOpenSharedMemory(winFile *pDbFd){ } } if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, WIN_SHM_DMS, 1); + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); } if( rc ) goto shm_open_err; } /* Make the new connection a child of the winShmNode */ p->pShmNode = pShmNode; -#ifdef SQLITE_DEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) p->id = pShmNode->nextShmId++; #endif pShmNode->nRef++; @@ -39124,7 +44845,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Jump here on any error */ shm_open_err: - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ sqlite3_free(p); sqlite3_free(pNew); @@ -39213,7 +44934,7 @@ static int winShmLock( /* Unlock the system-level locks */ if( (mask & allMask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_UNLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -39241,7 +44962,7 @@ static int winShmLock( /* Get shared locks at the system level, if necessary */ if( rc==SQLITE_OK ){ if( (allShared & mask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -39266,7 +44987,7 @@ static int winShmLock( ** also mark the local connection as being locked. */ if( rc==SQLITE_OK ){ - rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); if( rc==SQLITE_OK ){ assert( (p->sharedMask & mask)==0 ); p->exclMask |= mask; @@ -39290,8 +45011,8 @@ static void winShmBarrier( sqlite3_file *fd /* Database holding the shared memory */ ){ UNUSED_PARAMETER(fd); - /* MemoryBarrier(); // does not work -- do not know why not */ - winShmEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + winShmEnterMutex(); /* Also mutex, for redundancy */ winShmLeaveMutex(); } @@ -39322,16 +45043,16 @@ static int winShmMap( void volatile **pp /* OUT: Mapped memory */ ){ winFile *pDbFd = (winFile*)fd; - winShm *p = pDbFd->pShm; + winShm *pShm = pDbFd->pShm; winShmNode *pShmNode; int rc = SQLITE_OK; - if( !p ){ + if( !pShm ){ rc = winOpenSharedMemory(pDbFd); if( rc!=SQLITE_OK ) return rc; - p = pDbFd->pShm; + pShm = pDbFd->pShm; } - pShmNode = p->pShmNode; + pShmNode = pShm->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); @@ -39371,11 +45092,11 @@ static int winShmMap( } /* Map the requested memory region into this processes address space. */ - apNew = (struct ShmRegion *)sqlite3_realloc( + apNew = (struct ShmRegion *)sqlite3_realloc64( pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) ); if( !apNew ){ - rc = SQLITE_IOERR_NOMEM; + rc = SQLITE_IOERR_NOMEM_BKPT; goto shmpage_out; } pShmNode->aRegion = apNew; @@ -39392,7 +45113,7 @@ static int winShmMap( hMap = osCreateFileMappingW(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); -#elif defined(SQLITE_WIN32_HAS_ANSI) +#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); @@ -39536,17 +45257,19 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ DWORD flags = FILE_MAP_READ; winUnmapfile(pFd); +#ifdef SQLITE_MMAP_READWRITE if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ protect = PAGE_READWRITE; flags |= FILE_MAP_WRITE; } +#endif #if SQLITE_OS_WINRT pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); -#elif defined(SQLITE_WIN32_HAS_ANSI) +#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); @@ -39662,7 +45385,7 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ }else{ /* FIXME: If Windows truly always prevents truncating or deleting a ** file while a mapping is held, then the following winUnmapfile() call - ** is unnecessary can can be omitted - potentially improving + ** is unnecessary can be omitted - potentially improving ** performance. */ winUnmapfile(pFd); } @@ -39707,6 +45430,44 @@ static const sqlite3_io_methods winIoMethod = { winUnfetch /* xUnfetch */ }; +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32 without performing any locking. +*/ +static const sqlite3_io_methods winIoNolockMethod = { + 3, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winNolockLock, /* xLock */ + winNolockUnlock, /* xUnlock */ + winNolockCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ +}; + +static winVfsAppData winAppData = { + &winIoMethod, /* pMethod */ + 0, /* pAppData */ + 0 /* bNoLock */ +}; + +static winVfsAppData winNolockAppData = { + &winIoNolockMethod, /* pMethod */ + 0, /* pAppData */ + 1 /* bNoLock */ +}; + /**************************************************************************** **************************** sqlite3_vfs methods **************************** ** @@ -39727,7 +45488,7 @@ static char *winConvertToUtf8Filename(const void *zFilename){ } #ifdef SQLITE_WIN32_HAS_ANSI else{ - zConverted = sqlite3_win32_mbcs_to_utf8(zFilename); + zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI()); } #endif /* caller will handle out of memory */ @@ -39748,7 +45509,7 @@ static void *winConvertFromUtf8Filename(const char *zFilename){ } #ifdef SQLITE_WIN32_HAS_ANSI else{ - zConverted = sqlite3_win32_utf8_to_mbcs(zFilename); + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); } #endif /* caller will handle out of memory */ @@ -39803,7 +45564,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ zBuf = sqlite3MallocZero( nBuf ); if( !zBuf ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } /* Figure out the effective temporary directory. First, check if one @@ -39861,7 +45622,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zConverted ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( winIsDir(zConverted) ){ sqlite3_snprintf(nMax, zBuf, "%s", zDir); @@ -39874,7 +45635,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zConverted ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( cygwin_conv_path( osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A, zDir, @@ -39895,7 +45656,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ sqlite3_free(zConverted); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } sqlite3_snprintf(nMax, zBuf, "%s", zUtf8); sqlite3_free(zUtf8); @@ -39913,7 +45674,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zWidePath ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osGetTempPathW(nMax, zWidePath)==0 ){ sqlite3_free(zWidePath); @@ -39931,7 +45692,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ sqlite3_free(zWidePath); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } } #ifdef SQLITE_WIN32_HAS_ANSI @@ -39941,7 +45702,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zMbcsPath ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osGetTempPathA(nMax, zMbcsPath)==0 ){ sqlite3_free(zBuf); @@ -39949,14 +45710,14 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ return winLogError(SQLITE_IOERR_GETTEMPPATH, osGetLastError(), "winGetTempname3", 0); } - zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); + zUtf8 = winMbcsToUtf8(zMbcsPath, osAreFileApisANSI()); if( zUtf8 ){ sqlite3_snprintf(nMax, zBuf, "%s", zUtf8); sqlite3_free(zUtf8); }else{ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } } #endif /* SQLITE_WIN32_HAS_ANSI */ @@ -40039,7 +45800,7 @@ static int winIsDir(const void *zConverted){ ** Open a file. */ static int winOpen( - sqlite3_vfs *pVfs, /* Used to get maximum path name length */ + sqlite3_vfs *pVfs, /* Used to get maximum path length and AppData */ const char *zName, /* Name of the file (UTF-8) */ sqlite3_file *id, /* Write the SQLite file handle here */ int flags, /* Open mode flags */ @@ -40054,6 +45815,7 @@ static int winOpen( #if SQLITE_OS_WINCE int isTemp = 0; #endif + winVfsAppData *pAppData; winFile *pFile = (winFile*)id; void *zConverted; /* Filename in OS encoding */ const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ @@ -40148,7 +45910,7 @@ static int winOpen( if( zConverted==0 ){ sqlite3_free(zTmpname); OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( winIsDir(zConverted) ){ @@ -40243,7 +46005,7 @@ static int winOpen( } } #endif - winLogIoerr(cnt); + winLogIoerr(cnt, __LINE__); OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); @@ -40275,15 +46037,20 @@ static int winOpen( "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + pAppData = (winVfsAppData*)pVfs->pAppData; + #if SQLITE_OS_WINCE - if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB - && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK - ){ - osCloseHandle(h); - sqlite3_free(zConverted); - sqlite3_free(zTmpname); - OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); - return rc; + { + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB + && ((pAppData==NULL) || !pAppData->bNoLock) + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK + ){ + osCloseHandle(h); + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; + } } if( isTemp ){ pFile->zDeleteOnClose = zConverted; @@ -40294,7 +46061,7 @@ static int winOpen( } sqlite3_free(zTmpname); - pFile->pMethod = &winIoMethod; + pFile->pMethod = pAppData ? pAppData->pMethod : &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; if( isReadonly ){ @@ -40348,7 +46115,7 @@ static int winDelete( zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("DELETE name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osIsNT() ){ do { @@ -40427,7 +46194,7 @@ static int winDelete( if( rc && rc!=SQLITE_IOERR_DELETE_NOENT ){ rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); }else{ - winLogIoerr(cnt); + winLogIoerr(cnt, __LINE__); } sqlite3_free(zConverted); OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); @@ -40456,7 +46223,7 @@ static int winAccess( zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osIsNT() ){ int cnt = 0; @@ -40477,7 +46244,7 @@ static int winAccess( attr = sAttrData.dwFileAttributes; } }else{ - winLogIoerr(cnt); + winLogIoerr(cnt, __LINE__); if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ sqlite3_free(zConverted); return winLogError(SQLITE_IOERR_ACCESS, lastErrno, "winAccess", @@ -40569,6 +46336,18 @@ static int winFullPathname( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + DWORD nByte; + void *zConverted; + char *zOut; +#endif + + /* If this path name begins with "/X:", where "X" is any alphabetic + ** character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ + zRelative++; + } #if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); @@ -40583,7 +46362,7 @@ static int winFullPathname( */ char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); if( !zOut ){ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( cygwin_conv_path( (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | @@ -40595,7 +46374,7 @@ static int winFullPathname( char *zUtf8 = winConvertToUtf8Filename(zOut); if( !zUtf8 ){ sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", sqlite3_data_directory, winGetDirSep(), zUtf8); @@ -40605,7 +46384,7 @@ static int winFullPathname( }else{ char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); if( !zOut ){ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( cygwin_conv_path( (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), @@ -40617,7 +46396,7 @@ static int winFullPathname( char *zUtf8 = winConvertToUtf8Filename(zOut); if( !zUtf8 ){ sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); sqlite3_free(zUtf8); @@ -40647,17 +46426,6 @@ static int winFullPathname( #endif #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - DWORD nByte; - void *zConverted; - char *zOut; - - /* If this path name begins with "/X:", where "X" is any alphabetic - ** character, discard the initial "/" from the pathname. - */ - if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ - zRelative++; - } - /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the @@ -40677,7 +46445,7 @@ static int winFullPathname( } zConverted = winConvertFromUtf8Filename(zRelative); if( zConverted==0 ){ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osIsNT() ){ LPWSTR zTemp; @@ -40691,7 +46459,7 @@ static int winFullPathname( zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } nByte = osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); if( nByte==0 ){ @@ -40717,7 +46485,7 @@ static int winFullPathname( zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); if( nByte==0 ){ @@ -40727,7 +46495,7 @@ static int winFullPathname( "winFullPathname4", zRelative); } sqlite3_free(zConverted); - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); sqlite3_free(zTemp); } #endif @@ -40736,7 +46504,7 @@ static int winFullPathname( sqlite3_free(zOut); return SQLITE_OK; }else{ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } #endif } @@ -40811,49 +46579,85 @@ static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ #define winDlClose 0 #endif +/* State information for the randomness gatherer. */ +typedef struct EntropyGatherer EntropyGatherer; +struct EntropyGatherer { + unsigned char *a; /* Gather entropy into this buffer */ + int na; /* Size of a[] in bytes */ + int i; /* XOR next input into a[i] */ + int nXor; /* Number of XOR operations done */ +}; + +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) +/* Mix sz bytes of entropy into p. */ +static void xorMemory(EntropyGatherer *p, unsigned char *x, int sz){ + int j, k; + for(j=0, k=p->i; ja[k++] ^= x[j]; + if( k>=p->na ) k = 0; + } + p->i = k; + p->nXor += sz; +} +#endif /* !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) */ /* ** Write up to nBuf bytes of randomness into zBuf. */ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ - int n = 0; +#if defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) UNUSED_PARAMETER(pVfs); -#if defined(SQLITE_TEST) - n = nBuf; memset(zBuf, 0, nBuf); + return nBuf; #else - if( sizeof(SYSTEMTIME)<=nBuf-n ){ + EntropyGatherer e; + UNUSED_PARAMETER(pVfs); + memset(zBuf, 0, nBuf); +#if defined(_MSC_VER) && _MSC_VER>=1400 && !SQLITE_OS_WINCE + rand_s((unsigned int*)zBuf); /* rand_s() is not available with MinGW */ +#endif /* defined(_MSC_VER) && _MSC_VER>=1400 */ + e.a = (unsigned char*)zBuf; + e.na = nBuf; + e.nXor = 0; + e.i = 0; + { SYSTEMTIME x; osGetSystemTime(&x); - memcpy(&zBuf[n], &x, sizeof(x)); - n += sizeof(x); + xorMemory(&e, (unsigned char*)&x, sizeof(SYSTEMTIME)); } - if( sizeof(DWORD)<=nBuf-n ){ + { DWORD pid = osGetCurrentProcessId(); - memcpy(&zBuf[n], &pid, sizeof(pid)); - n += sizeof(pid); + xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); } #if SQLITE_OS_WINRT - if( sizeof(ULONGLONG)<=nBuf-n ){ + { ULONGLONG cnt = osGetTickCount64(); - memcpy(&zBuf[n], &cnt, sizeof(cnt)); - n += sizeof(cnt); + xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); } #else - if( sizeof(DWORD)<=nBuf-n ){ + { DWORD cnt = osGetTickCount(); - memcpy(&zBuf[n], &cnt, sizeof(cnt)); - n += sizeof(cnt); + xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); } -#endif - if( sizeof(LARGE_INTEGER)<=nBuf-n ){ +#endif /* SQLITE_OS_WINRT */ + { LARGE_INTEGER i; osQueryPerformanceCounter(&i); - memcpy(&zBuf[n], &i, sizeof(i)); - n += sizeof(i); + xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); } -#endif - return n; +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { + UUID id; + memset(&id, 0, sizeof(UUID)); + osUuidCreate(&id); + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); + memset(&id, 0, sizeof(UUID)); + osUuidCreateSequential(&id); + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); + } +#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ + return e.nXor>nBuf ? nBuf : e.nXor; +#endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ } @@ -40969,8 +46773,10 @@ static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ ** sqlite3_errmsg(), possibly making IO errors easier to debug. */ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + DWORD e = osGetLastError(); UNUSED_PARAMETER(pVfs); - return winGetLastErrorMsg(osGetLastError(), nBuf, zBuf); + if( nBuf>0 ) winGetLastErrorMsg(e, nBuf, zBuf); + return e; } /* @@ -40978,59 +46784,109 @@ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ */ SQLITE_API int sqlite3_os_init(void){ static sqlite3_vfs winVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #if defined(SQLITE_WIN32_HAS_WIDE) static sqlite3_vfs winLongPathVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32-longpath", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32-longpath", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif + static sqlite3_vfs winNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #endif /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==77 ); + assert( ArraySize(aSyscall)==80 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); @@ -41048,6 +46904,12 @@ SQLITE_API int sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathVfs, 0); #endif + sqlite3_vfs_register(&winNolockVfs, 0); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathNolockVfs, 0); +#endif + return SQLITE_OK; } @@ -41101,13 +46963,15 @@ SQLITE_API int sqlite3_os_end(void){ ** start of a transaction, and is thus usually less than a few thousand, ** but can be as large as 2 billion for a really big database. */ +/* #include "sqliteInt.h" */ /* Size of the Bitvec structure in bytes. */ #define BITVEC_SZ 512 /* Round the union size down to the nearest pointer boundary, since that's how ** it will be aligned within the Bitvec struct. */ -#define BITVEC_USIZE (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) +#define BITVEC_USIZE \ + (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) /* Type of the array "element" for the bitmap representation. ** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. @@ -41192,10 +47056,10 @@ SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32 iSize){ ** If p is NULL (if the bitmap has not been created) or if ** i is out of range, then return false. */ -SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ - if( p==0 ) return 0; - if( i>p->iSize || i==0 ) return 0; +SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec *p, u32 i){ + assert( p!=0 ); i--; + if( i>=p->iSize ) return 0; while( p->iDivisor ){ u32 bin = i/p->iDivisor; i = i%p->iDivisor; @@ -41215,6 +47079,9 @@ SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ return 0; } } +SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ + return p!=0 && sqlite3BitvecTestNotNull(p,i); +} /* ** Set the i-th bit. Return 0 on success and an error code if @@ -41239,7 +47106,7 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ i = i%p->iDivisor; if( p->u.apSub[bin]==0 ){ p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); - if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM; + if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM_BKPT; } p = p->u.apSub[bin]; } @@ -41274,7 +47141,7 @@ bitvec_set_rehash: int rc; u32 *aiValues = sqlite3StackAllocRaw(0, sizeof(p->u.aHash)); if( aiValues==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; }else{ memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); memset(p->u.apSub, 0, sizeof(p->u.apSub)); @@ -41407,7 +47274,7 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ ** bits to act as the reference */ pBitvec = sqlite3BitvecCreate( sz ); pV = sqlite3MallocZero( (sz+7)/8 + 1 ); - pTmpSpace = sqlite3_malloc(BITVEC_SZ); + pTmpSpace = sqlite3_malloc64(BITVEC_SZ); if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; /* NULL pBitvec tests */ @@ -41487,15 +47354,39 @@ bitvec_end: ************************************************************************* ** This file implements that page cache. */ +/* #include "sqliteInt.h" */ /* -** A complete page cache is an instance of this structure. +** A complete page cache is an instance of this structure. Every +** entry in the cache holds a single page of the database file. The +** btree layer only operates on the cached copy of the database pages. +** +** A page cache entry is "clean" if it exactly matches what is currently +** on disk. A page is "dirty" if it has been modified and needs to be +** persisted to disk. +** +** pDirty, pDirtyTail, pSynced: +** All dirty pages are linked into the doubly linked list using +** PgHdr.pDirtyNext and pDirtyPrev. The list is maintained in LRU order +** such that p was added to the list more recently than p->pDirtyNext. +** PCache.pDirty points to the first (newest) element in the list and +** pDirtyTail to the last (oldest). +** +** The PCache.pSynced variable is used to optimize searching for a dirty +** page to eject from the cache mid-transaction. It is better to eject +** a page that does not require a journal sync than one that does. +** Therefore, pSynced is maintained to that it *almost* always points +** to either the oldest page in the pDirty/pDirtyTail list that has a +** clear PGHDR_NEED_SYNC flag or to a page that is older than this one +** (so that the right page to eject can be found by following pDirtyPrev +** pointers). */ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRef; /* Number of referenced pages */ + int nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ + int szSpill; /* Size before spilling occurs */ int szPage; /* Size of every page in this cache */ int szExtra; /* Size of extra space for each page */ u8 bPurgeable; /* True if pages are on backing store */ @@ -41503,105 +47394,177 @@ struct PCache { int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ void *pStress; /* Argument to xStress */ sqlite3_pcache *pCache; /* Pluggable cache module */ - PgHdr *pPage1; /* Reference to page 1 */ }; +/********************************** Test and Debug Logic **********************/ /* -** Some of the assert() macros in this code are too expensive to run -** even during normal debugging. Use them only rarely on long-running -** tests. Enable the expensive asserts using the -** -DSQLITE_ENABLE_EXPENSIVE_ASSERT=1 compile-time option. +** Debug tracing macros. Enable by by changing the "0" to "1" and +** recompiling. +** +** When sqlite3PcacheTrace is 1, single line trace messages are issued. +** When sqlite3PcacheTrace is 2, a dump of the pcache showing all cache entries +** is displayed for many operations, resulting in a lot of output. */ -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT -# define expensive_assert(X) assert(X) -#else -# define expensive_assert(X) +#if defined(SQLITE_DEBUG) && 0 + int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */ + int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */ +# define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;} + void pcacheDump(PCache *pCache){ + int N; + int i, j; + sqlite3_pcache_page *pLower; + PgHdr *pPg; + unsigned char *a; + + if( sqlite3PcacheTrace<2 ) return; + if( pCache->pCache==0 ) return; + N = sqlite3PcachePagecount(pCache); + if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; + for(i=1; i<=N; i++){ + pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); + if( pLower==0 ) continue; + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf("\n"); + if( pPg->pPage==0 ){ + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); + } + } + } + #else +# define pcacheTrace(X) +# define pcacheDump(X) #endif +/* +** Check invariants on a PgHdr entry. Return true if everything is OK. +** Return false if any invariant is violated. +** +** This routine is for use inside of assert() statements only. For +** example: +** +** assert( sqlite3PcachePageSanity(pPg) ); +*/ +#if SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){ + PCache *pCache; + assert( pPg!=0 ); + assert( pPg->pgno>0 ); /* Page number is 1 or more */ + pCache = pPg->pCache; + assert( pCache!=0 ); /* Every page has an associated PCache */ + if( pPg->flags & PGHDR_CLEAN ){ + assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ + assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ + assert( pCache->pDirtyTail!=pPg ); + } + /* WRITEABLE pages must also be DIRTY */ + if( pPg->flags & PGHDR_WRITEABLE ){ + assert( pPg->flags & PGHDR_DIRTY ); /* WRITEABLE implies DIRTY */ + } + /* NEED_SYNC can be set independently of WRITEABLE. This can happen, + ** for example, when using the sqlite3PagerDontWrite() optimization: + ** (1) Page X is journalled, and gets WRITEABLE and NEED_SEEK. + ** (2) Page X moved to freelist, WRITEABLE is cleared + ** (3) Page X reused, WRITEABLE is set again + ** If NEED_SYNC had been cleared in step 2, then it would not be reset + ** in step 3, and page might be written into the database without first + ** syncing the rollback journal, which might cause corruption on a power + ** loss. + ** + ** Another example is when the database page size is smaller than the + ** disk sector size. When any page of a sector is journalled, all pages + ** in that sector are marked NEED_SYNC even if they are still CLEAN, just + ** in case they are later modified, since all pages in the same sector + ** must be journalled and synced before any of those pages can be safely + ** written. + */ + return 1; +} +#endif /* SQLITE_DEBUG */ + + /********************************** Linked List Management ********************/ -#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) -/* -** Check that the pCache->pSynced variable is set correctly. If it -** is not, either fail an assert or return zero. Otherwise, return -** non-zero. This is only used in debugging builds, as follows: -** -** expensive_assert( pcacheCheckSynced(pCache) ); -*/ -static int pcacheCheckSynced(PCache *pCache){ - PgHdr *p; - for(p=pCache->pDirtyTail; p!=pCache->pSynced; p=p->pDirtyPrev){ - assert( p->nRef || (p->flags&PGHDR_NEED_SYNC) ); - } - return (p==0 || p->nRef || (p->flags&PGHDR_NEED_SYNC)==0); -} -#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */ +/* Allowed values for second argument to pcacheManageDirtyList() */ +#define PCACHE_DIRTYLIST_REMOVE 1 /* Remove pPage from dirty list */ +#define PCACHE_DIRTYLIST_ADD 2 /* Add pPage to the dirty list */ +#define PCACHE_DIRTYLIST_FRONT 3 /* Move pPage to the front of the list */ /* -** Remove page pPage from the list of dirty pages. +** Manage pPage's participation on the dirty list. Bits of the addRemove +** argument determines what operation to do. The 0x01 bit means first +** remove pPage from the dirty list. The 0x02 means add pPage back to +** the dirty list. Doing both moves pPage to the front of the dirty list. */ -static void pcacheRemoveFromDirtyList(PgHdr *pPage){ +static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ PCache *p = pPage->pCache; - assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); - assert( pPage->pDirtyPrev || pPage==p->pDirty ); - - /* Update the PCache1.pSynced variable if necessary. */ - if( p->pSynced==pPage ){ - PgHdr *pSynced = pPage->pDirtyPrev; - while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ - pSynced = pSynced->pDirtyPrev; + pcacheTrace(("%p.DIRTYLIST.%s %d\n", p, + addRemove==1 ? "REMOVE" : addRemove==2 ? "ADD" : "FRONT", + pPage->pgno)); + if( addRemove & PCACHE_DIRTYLIST_REMOVE ){ + assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); + assert( pPage->pDirtyPrev || pPage==p->pDirty ); + + /* Update the PCache1.pSynced variable if necessary. */ + if( p->pSynced==pPage ){ + p->pSynced = pPage->pDirtyPrev; } - p->pSynced = pSynced; + + if( pPage->pDirtyNext ){ + pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; + }else{ + assert( pPage==p->pDirtyTail ); + p->pDirtyTail = pPage->pDirtyPrev; + } + if( pPage->pDirtyPrev ){ + pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; + }else{ + /* If there are now no dirty pages in the cache, set eCreate to 2. + ** This is an optimization that allows sqlite3PcacheFetch() to skip + ** searching for a dirty page to eject from the cache when it might + ** otherwise have to. */ + assert( pPage==p->pDirty ); + p->pDirty = pPage->pDirtyNext; + assert( p->bPurgeable || p->eCreate==2 ); + if( p->pDirty==0 ){ /*OPTIMIZATION-IF-TRUE*/ + assert( p->bPurgeable==0 || p->eCreate==1 ); + p->eCreate = 2; + } + } + pPage->pDirtyNext = 0; + pPage->pDirtyPrev = 0; } + if( addRemove & PCACHE_DIRTYLIST_ADD ){ + assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); + + pPage->pDirtyNext = p->pDirty; + if( pPage->pDirtyNext ){ + assert( pPage->pDirtyNext->pDirtyPrev==0 ); + pPage->pDirtyNext->pDirtyPrev = pPage; + }else{ + p->pDirtyTail = pPage; + if( p->bPurgeable ){ + assert( p->eCreate==2 ); + p->eCreate = 1; + } + } + p->pDirty = pPage; - if( pPage->pDirtyNext ){ - pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; - }else{ - assert( pPage==p->pDirtyTail ); - p->pDirtyTail = pPage->pDirtyPrev; - } - if( pPage->pDirtyPrev ){ - pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; - }else{ - assert( pPage==p->pDirty ); - p->pDirty = pPage->pDirtyNext; - if( p->pDirty==0 && p->bPurgeable ){ - assert( p->eCreate==1 ); - p->eCreate = 2; + /* If pSynced is NULL and this page has a clear NEED_SYNC flag, set + ** pSynced to point to it. Checking the NEED_SYNC flag is an + ** optimization, as if pSynced points to a page with the NEED_SYNC + ** flag set sqlite3PcacheFetchStress() searches through all newer + ** entries of the dirty-list for a page with NEED_SYNC clear anyway. */ + if( !p->pSynced + && 0==(pPage->flags&PGHDR_NEED_SYNC) /*OPTIMIZATION-IF-FALSE*/ + ){ + p->pSynced = pPage; } } - pPage->pDirtyNext = 0; - pPage->pDirtyPrev = 0; - - expensive_assert( pcacheCheckSynced(p) ); -} - -/* -** Add page pPage to the head of the dirty list (PCache1.pDirty is set to -** pPage). -*/ -static void pcacheAddToDirtyList(PgHdr *pPage){ - PCache *p = pPage->pCache; - - assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); - - pPage->pDirtyNext = p->pDirty; - if( pPage->pDirtyNext ){ - assert( pPage->pDirtyNext->pDirtyPrev==0 ); - pPage->pDirtyNext->pDirtyPrev = pPage; - }else if( p->bPurgeable ){ - assert( p->eCreate==2 ); - p->eCreate = 1; - } - p->pDirty = pPage; - if( !p->pDirtyTail ){ - p->pDirtyTail = pPage; - } - if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ - p->pSynced = pPage; - } - expensive_assert( pcacheCheckSynced(p) ); + pcacheDump(p); } /* @@ -41609,12 +47572,27 @@ static void pcacheAddToDirtyList(PgHdr *pPage){ ** being used for an in-memory database, this function is a no-op. */ static void pcacheUnpin(PgHdr *p){ - PCache *pCache = p->pCache; - if( pCache->bPurgeable ){ - if( p->pgno==1 ){ - pCache->pPage1 = 0; - } - sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 0); + if( p->pCache->bPurgeable ){ + pcacheTrace(("%p.UNPIN %d\n", p->pCache, p->pgno)); + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); + pcacheDump(p->pCache); + } +} + +/* +** Compute the number of pages of cache requested. p->szCache is the +** cache size requested by the "PRAGMA cache_size" statement. +*/ +static int numberOfCachePages(PCache *p){ + if( p->szCache>=0 ){ + /* IMPLEMENTATION-OF: R-42059-47211 If the argument N is positive then the + ** suggested cache size is set to N. */ + return p->szCache; + }else{ + /* IMPLEMENTATION-OF: R-61436-13639 If the argument N is negative, then + ** the number of cache pages is adjusted to use approximately abs(N*1024) + ** bytes of memory. */ + return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); } } @@ -41650,7 +47628,7 @@ SQLITE_PRIVATE int sqlite3PcacheSize(void){ return sizeof(PCache); } ** The caller discovers how much space needs to be allocated by ** calling sqlite3PcacheSize(). */ -SQLITE_PRIVATE void sqlite3PcacheOpen( +SQLITE_PRIVATE int sqlite3PcacheOpen( int szPage, /* Size of every page */ int szExtra, /* Extra space associated with each page */ int bPurgeable, /* True if pages are on backing store */ @@ -41659,75 +47637,79 @@ SQLITE_PRIVATE void sqlite3PcacheOpen( PCache *p /* Preallocated space for the PCache */ ){ memset(p, 0, sizeof(PCache)); - p->szPage = szPage; + p->szPage = 1; p->szExtra = szExtra; p->bPurgeable = bPurgeable; p->eCreate = 2; p->xStress = xStress; p->pStress = pStress; p->szCache = 100; + p->szSpill = 1; + pcacheTrace(("%p.OPEN szPage %d bPurgeable %d\n",p,szPage,bPurgeable)); + return sqlite3PcacheSetPageSize(p, szPage); } /* ** Change the page size for PCache object. The caller must ensure that there ** are no outstanding page references when this function is called. */ -SQLITE_PRIVATE void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ - assert( pCache->nRef==0 && pCache->pDirty==0 ); - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); - pCache->pCache = 0; - pCache->pPage1 = 0; - } - pCache->szPage = szPage; -} - -/* -** Compute the number of pages of cache requested. -*/ -static int numberOfCachePages(PCache *p){ - if( p->szCache>=0 ){ - return p->szCache; - }else{ - return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); +SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ + assert( pCache->nRefSum==0 && pCache->pDirty==0 ); + if( pCache->szPage ){ + sqlite3_pcache *pNew; + pNew = sqlite3GlobalConfig.pcache2.xCreate( + szPage, pCache->szExtra + ROUND8(sizeof(PgHdr)), + pCache->bPurgeable + ); + if( pNew==0 ) return SQLITE_NOMEM_BKPT; + sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); + } + pCache->pCache = pNew; + pCache->szPage = szPage; + pcacheTrace(("%p.PAGESIZE %d\n",pCache,szPage)); } + return SQLITE_OK; } /* ** Try to obtain a page from the cache. +** +** This routine returns a pointer to an sqlite3_pcache_page object if +** such an object is already in cache, or if a new one is created. +** This routine returns a NULL pointer if the object was not in cache +** and could not be created. +** +** The createFlags should be 0 to check for existing pages and should +** be 3 (not 1, but 3) to try to create a new page. +** +** If the createFlag is 0, then NULL is always returned if the page +** is not already in the cache. If createFlag is 1, then a new page +** is created only if that can be done without spilling dirty pages +** and without exceeding the cache size limit. +** +** The caller needs to invoke sqlite3PcacheFetchFinish() to properly +** initialize the sqlite3_pcache_page object and convert it into a +** PgHdr object. The sqlite3PcacheFetch() and sqlite3PcacheFetchFinish() +** routines are split this way for performance reasons. When separated +** they can both (usually) operate without having to push values to +** the stack on entry and pop them back off on exit, which saves a +** lot of pushing and popping. */ -SQLITE_PRIVATE int sqlite3PcacheFetch( +SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch( PCache *pCache, /* Obtain the page from this cache */ Pgno pgno, /* Page number to obtain */ - int createFlag, /* If true, create page if it does not exist already */ - PgHdr **ppPage /* Write the page here */ + int createFlag /* If true, create page if it does not exist already */ ){ - sqlite3_pcache_page *pPage; - PgHdr *pPgHdr = 0; int eCreate; + sqlite3_pcache_page *pRes; assert( pCache!=0 ); - assert( createFlag==1 || createFlag==0 ); + assert( pCache->pCache!=0 ); + assert( createFlag==3 || createFlag==0 ); assert( pgno>0 ); - - /* If the pluggable cache (sqlite3_pcache*) has not been allocated, - ** allocate it now. - */ - if( !pCache->pCache ){ - sqlite3_pcache *p; - if( !createFlag ){ - *ppPage = 0; - return SQLITE_OK; - } - p = sqlite3GlobalConfig.pcache2.xCreate( - pCache->szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable - ); - if( !p ){ - return SQLITE_NOMEM; - } - sqlite3GlobalConfig.pcache2.xCachesize(p, numberOfCachePages(pCache)); - pCache->pCache = p; - } + assert( pCache->eCreate==((pCache->bPurgeable && pCache->pDirty) ? 1 : 2) ); /* eCreate defines what to do if the page does not exist. ** 0 Do not allocate a new page. (createFlag==0) @@ -41736,18 +47718,45 @@ SQLITE_PRIVATE int sqlite3PcacheFetch( ** 2 Allocate a new page even it doing so is difficult. ** (createFlag==1 AND !(bPurgeable AND pDirty) */ - eCreate = createFlag==0 ? 0 : pCache->eCreate; - assert( (createFlag*(1+(!pCache->bPurgeable||!pCache->pDirty)))==eCreate ); - pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); - if( !pPage && eCreate==1 ){ - PgHdr *pPg; + eCreate = createFlag & pCache->eCreate; + assert( eCreate==0 || eCreate==1 || eCreate==2 ); + assert( createFlag==0 || pCache->eCreate==eCreate ); + assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); + pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); + pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, + createFlag?" create":"",pRes)); + return pRes; +} +/* +** If the sqlite3PcacheFetch() routine is unable to allocate a new +** page because no clean pages are available for reuse and the cache +** size limit has been reached, then this routine can be invoked to +** try harder to allocate a page. This routine might invoke the stress +** callback to spill dirty pages to the journal. It will then try to +** allocate the new page and will only fail to allocate a new page on +** an OOM error. +** +** This routine should be invoked only after sqlite3PcacheFetch() fails. +*/ +SQLITE_PRIVATE int sqlite3PcacheFetchStress( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + sqlite3_pcache_page **ppPage /* Write result here */ +){ + PgHdr *pPg; + if( pCache->eCreate==2 ) return 0; + + if( sqlite3PcachePagecount(pCache)>pCache->szSpill ){ /* Find a dirty page to write-out and recycle. First try to find a ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC ** cleared), but if that is not possible settle for any other ** unreferenced dirty page. - */ - expensive_assert( pcacheCheckSynced(pCache) ); + ** + ** If the LRU page in the dirty list that has a clear PGHDR_NEED_SYNC + ** flag is currently referenced, then the following may leave pSynced + ** set incorrectly (pointing to other than the LRU page with NEED_SYNC + ** cleared). This is Ok, as pSynced is just an optimization. */ for(pPg=pCache->pSynced; pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); pPg=pPg->pDirtyPrev @@ -41763,62 +47772,90 @@ SQLITE_PRIVATE int sqlite3PcacheFetch( "spill page %d making room for %d - cache used: %d/%d", pPg->pgno, pgno, sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), - numberOfCachePages(pCache)); + numberOfCachePages(pCache)); #endif + pcacheTrace(("%p.SPILL %d\n",pCache,pPg->pgno)); rc = pCache->xStress(pCache->pStress, pPg); + pcacheDump(pCache); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ return rc; } } - - pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); } + *ppPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); + return *ppPage==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; +} - if( pPage ){ - pPgHdr = (PgHdr *)pPage->pExtra; +/* +** This is a helper routine for sqlite3PcacheFetchFinish() +** +** In the uncommon case where the page being fetched has not been +** initialized, this routine is invoked to do the initialization. +** This routine is broken out into a separate function since it +** requires extra stack manipulation that can be avoided in the common +** case. +*/ +static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; + assert( pPage!=0 ); + pPgHdr = (PgHdr*)pPage->pExtra; + assert( pPgHdr->pPage==0 ); + memset(&pPgHdr->pDirty, 0, sizeof(PgHdr) - offsetof(PgHdr,pDirty)); + pPgHdr->pPage = pPage; + pPgHdr->pData = pPage->pBuf; + pPgHdr->pExtra = (void *)&pPgHdr[1]; + memset(pPgHdr->pExtra, 0, pCache->szExtra); + pPgHdr->pCache = pCache; + pPgHdr->pgno = pgno; + pPgHdr->flags = PGHDR_CLEAN; + return sqlite3PcacheFetchFinish(pCache,pgno,pPage); +} - if( !pPgHdr->pPage ){ - memset(pPgHdr, 0, sizeof(PgHdr)); - pPgHdr->pPage = pPage; - pPgHdr->pData = pPage->pBuf; - pPgHdr->pExtra = (void *)&pPgHdr[1]; - memset(pPgHdr->pExtra, 0, pCache->szExtra); - pPgHdr->pCache = pCache; - pPgHdr->pgno = pgno; - } - assert( pPgHdr->pCache==pCache ); - assert( pPgHdr->pgno==pgno ); - assert( pPgHdr->pData==pPage->pBuf ); - assert( pPgHdr->pExtra==(void *)&pPgHdr[1] ); +/* +** This routine converts the sqlite3_pcache_page object returned by +** sqlite3PcacheFetch() into an initialized PgHdr object. This routine +** must be called after sqlite3PcacheFetch() in order to get a usable +** result. +*/ +SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; - if( 0==pPgHdr->nRef ){ - pCache->nRef++; - } - pPgHdr->nRef++; - if( pgno==1 ){ - pCache->pPage1 = pPgHdr; - } + assert( pPage!=0 ); + pPgHdr = (PgHdr *)pPage->pExtra; + + if( !pPgHdr->pPage ){ + return pcacheFetchFinishWithInit(pCache, pgno, pPage); } - *ppPage = pPgHdr; - return (pPgHdr==0 && eCreate) ? SQLITE_NOMEM : SQLITE_OK; + pCache->nRefSum++; + pPgHdr->nRef++; + assert( sqlite3PcachePageSanity(pPgHdr) ); + return pPgHdr; } /* ** Decrement the reference count on a page. If the page is clean and the -** reference count drops to 0, then it is made elible for recycling. +** reference count drops to 0, then it is made eligible for recycling. */ -SQLITE_PRIVATE void sqlite3PcacheRelease(PgHdr *p){ +SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); - p->nRef--; - if( p->nRef==0 ){ - PCache *pCache = p->pCache; - pCache->nRef--; - if( (p->flags&PGHDR_DIRTY)==0 ){ + p->pCache->nRefSum--; + if( (--p->nRef)==0 ){ + if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); - }else{ - /* Move the page to the head of the dirty list. */ - pcacheRemoveFromDirtyList(p); - pcacheAddToDirtyList(p); + }else if( p->pDirtyPrev!=0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Move the page to the head of the dirty list. If p->pDirtyPrev==0, + ** then page p is already at the head of the dirty list and the + ** following call would be a no-op. Hence the OPTIMIZATION-IF-FALSE + ** tag above. */ + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } } @@ -41828,7 +47865,9 @@ SQLITE_PRIVATE void sqlite3PcacheRelease(PgHdr *p){ */ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ assert(p->nRef>0); + assert( sqlite3PcachePageSanity(p) ); p->nRef++; + p->pCache->nRefSum++; } /* @@ -41837,17 +47876,13 @@ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ ** page pointed to by p is invalid. */ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ - PCache *pCache; assert( p->nRef==1 ); + assert( sqlite3PcachePageSanity(p) ); if( p->flags&PGHDR_DIRTY ){ - pcacheRemoveFromDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } - pCache = p->pCache; - pCache->nRef--; - if( p->pgno==1 ){ - pCache->pPage1 = 0; - } - sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 1); + p->pCache->nRefSum--; + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); } /* @@ -41855,11 +47890,17 @@ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ ** make it so. */ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ - p->flags &= ~PGHDR_DONT_WRITE; assert( p->nRef>0 ); - if( 0==(p->flags & PGHDR_DIRTY) ){ - p->flags |= PGHDR_DIRTY; - pcacheAddToDirtyList( p); + assert( sqlite3PcachePageSanity(p) ); + if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ /*OPTIMIZATION-IF-FALSE*/ + p->flags &= ~PGHDR_DONT_WRITE; + if( p->flags & PGHDR_CLEAN ){ + p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN); + pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); + assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + } + assert( sqlite3PcachePageSanity(p) ); } } @@ -41868,9 +47909,14 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ ** make it so. */ SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ - if( (p->flags & PGHDR_DIRTY) ){ - pcacheRemoveFromDirtyList(p); - p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); + assert( sqlite3PcachePageSanity(p) ); + if( ALWAYS((p->flags & PGHDR_DIRTY)!=0) ){ + assert( (p->flags & PGHDR_CLEAN)==0 ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + p->flags |= PGHDR_CLEAN; + pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); + assert( sqlite3PcachePageSanity(p) ); if( p->nRef==0 ){ pcacheUnpin(p); } @@ -41882,11 +47928,24 @@ SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ */ SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache *pCache){ PgHdr *p; + pcacheTrace(("%p.CLEAN-ALL\n",pCache)); while( (p = pCache->pDirty)!=0 ){ sqlite3PcacheMakeClean(p); } } +/* +** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. +*/ +SQLITE_PRIVATE void sqlite3PcacheClearWritable(PCache *pCache){ + PgHdr *p; + pcacheTrace(("%p.CLEAR-WRITEABLE\n",pCache)); + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + } + pCache->pSynced = pCache->pDirtyTail; +} + /* ** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ @@ -41905,11 +47964,12 @@ SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; assert( p->nRef>0 ); assert( newPgno>0 ); + assert( sqlite3PcachePageSanity(p) ); + pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ - pcacheRemoveFromDirtyList(p); - pcacheAddToDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } @@ -41926,6 +47986,7 @@ SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ if( pCache->pCache ){ PgHdr *p; PgHdr *pNext; + pcacheTrace(("%p.TRUNCATE %d\n",pCache,pgno)); for(p=pCache->pDirty; p; p=pNext){ pNext = p->pDirtyNext; /* This routine never gets call with a positive pgno except right @@ -41933,14 +47994,19 @@ SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** it must be that pgno==0. */ assert( p->pgno>0 ); - if( ALWAYS(p->pgno>pgno) ){ + if( p->pgno>pgno ){ assert( p->flags&PGHDR_DIRTY ); sqlite3PcacheMakeClean(p); } } - if( pgno==0 && pCache->pPage1 ){ - memset(pCache->pPage1->pData, 0, pCache->szPage); - pgno = 1; + if( pgno==0 && pCache->nRefSum ){ + sqlite3_pcache_page *pPage1; + pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); + if( ALWAYS(pPage1) ){ /* Page 1 is always available in cache, because + ** pCache->nRefSum>0 */ + memset(pPage1->pBuf, 0, pCache->szPage); + pgno = 1; + } } sqlite3GlobalConfig.pcache2.xTruncate(pCache->pCache, pgno+1); } @@ -41950,9 +48016,9 @@ SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** Close a cache. */ SQLITE_PRIVATE void sqlite3PcacheClose(PCache *pCache){ - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); - } + assert( pCache->pCache!=0 ); + pcacheTrace(("%p.CLOSE\n",pCache)); + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } /* @@ -41964,29 +48030,31 @@ SQLITE_PRIVATE void sqlite3PcacheClear(PCache *pCache){ /* ** Merge two lists of pages connected by pDirty and in pgno order. -** Do not both fixing the pDirtyPrev pointers. +** Do not bother fixing the pDirtyPrev pointers. */ static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ PgHdr result, *pTail; pTail = &result; - while( pA && pB ){ + assert( pA!=0 && pB!=0 ); + for(;;){ if( pA->pgnopgno ){ pTail->pDirty = pA; pTail = pA; pA = pA->pDirty; + if( pA==0 ){ + pTail->pDirty = pB; + break; + } }else{ pTail->pDirty = pB; pTail = pB; pB = pB->pDirty; + if( pB==0 ){ + pTail->pDirty = pA; + break; + } } } - if( pA ){ - pTail->pDirty = pA; - }else if( pB ){ - pTail->pDirty = pB; - }else{ - pTail->pDirty = 0; - } return result.pDirty; } @@ -42027,7 +48095,8 @@ static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ } p = a[0]; for(i=1; inRef; + return pCache->nRefSum; } /* @@ -42061,11 +48133,8 @@ SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr *p){ ** Return the total number of pages in the cache. */ SQLITE_PRIVATE int sqlite3PcachePagecount(PCache *pCache){ - int nPage = 0; - if( pCache->pCache ){ - nPage = sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); - } - return nPage; + assert( pCache->pCache!=0 ); + return sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); } #ifdef SQLITE_TEST @@ -42081,20 +48150,55 @@ SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *pCache){ ** Set the suggested cache-size value. */ SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ + assert( pCache->pCache!=0 ); pCache->szCache = mxPage; - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, - numberOfCachePages(pCache)); + sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, + numberOfCachePages(pCache)); +} + +/* +** Set the suggested cache-spill value. Make no changes if if the +** argument is zero. Return the effective cache-spill size, which will +** be the larger of the szSpill and szCache. +*/ +SQLITE_PRIVATE int sqlite3PcacheSetSpillsize(PCache *p, int mxPage){ + int res; + assert( p->pCache!=0 ); + if( mxPage ){ + if( mxPage<0 ){ + mxPage = (int)((-1024*(i64)mxPage)/(p->szPage+p->szExtra)); + } + p->szSpill = mxPage; } + res = numberOfCachePages(p); + if( resszSpill ) res = p->szSpill; + return res; } /* ** Free up as much memory as possible from the page cache. */ SQLITE_PRIVATE void sqlite3PcacheShrink(PCache *pCache){ - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); - } + assert( pCache->pCache!=0 ); + sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); +} + +/* +** Return the size of the header added by this middleware layer +** in the page-cache hierarchy. +*/ +SQLITE_PRIVATE int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } + +/* +** Return the number of dirty pages currently in the cache, as a percentage +** of the configured cache size. +*/ +SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache *pCache){ + PgHdr *pDirty; + int nDirty = 0; + int nCache = numberOfCachePages(pCache); + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; + return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; } #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) @@ -42128,18 +48232,100 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** This file implements the default page cache implementation (the ** sqlite3_pcache interface). It also contains part of the implementation ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. -** If the default page cache implementation is overriden, then neither of +** If the default page cache implementation is overridden, then neither of ** these two features are available. +** +** A Page cache line looks like this: +** +** ------------------------------------------------------------- +** | database page content | PgHdr1 | MemPage | PgHdr | +** ------------------------------------------------------------- +** +** The database page content is up front (so that buffer overreads tend to +** flow harmlessly into the PgHdr1, MemPage, and PgHdr extensions). MemPage +** is the extension added by the btree.c module containing information such +** as the database page number and how that database page is used. PgHdr +** is added by the pcache.c layer and contains information used to keep track +** of which pages are "dirty". PgHdr1 is an extension added by this +** module (pcache1.c). The PgHdr1 header is a subclass of sqlite3_pcache_page. +** PgHdr1 contains information needed to look up a page by its page number. +** The superclass sqlite3_pcache_page.pBuf points to the start of the +** database page content and sqlite3_pcache_page.pExtra points to PgHdr. +** +** The size of the extension (MemPage+PgHdr+PgHdr1) can be determined at +** runtime using sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &size). The +** sizes of the extensions sum to 272 bytes on x64 for 3.8.10, but this +** size can vary according to architecture, compile-time options, and +** SQLite library version number. +** +** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained +** using a separate memory allocation from the database page content. This +** seeks to overcome the "clownshoe" problem (also called "internal +** fragmentation" in academic literature) of allocating a few bytes more +** than a power of two with the memory allocator rounding up to the next +** power of two, and leaving the rounded-up space unused. +** +** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates +** with this module. Information is passed back and forth as PgHdr1 pointers. +** +** The pcache.c and pager.c modules deal pointers to PgHdr objects. +** The btree.c module deals with pointers to MemPage objects. +** +** SOURCE OF PAGE CACHE MEMORY: +** +** Memory for a page might come from any of three sources: +** +** (1) The general-purpose memory allocator - sqlite3Malloc() +** (2) Global page-cache memory provided using sqlite3_config() with +** SQLITE_CONFIG_PAGECACHE. +** (3) PCache-local bulk allocation. +** +** The third case is a chunk of heap memory (defaulting to 100 pages worth) +** that is allocated when the page cache is created. The size of the local +** bulk allocation can be adjusted using +** +** sqlite3_config(SQLITE_CONFIG_PAGECACHE, (void*)0, 0, N). +** +** If N is positive, then N pages worth of memory are allocated using a single +** sqlite3Malloc() call and that memory is used for the first N pages allocated. +** Or if N is negative, then -1024*N bytes of memory are allocated and used +** for as many pages as can be accomodated. +** +** Only one of (2) or (3) can be used. Once the memory available to (2) or +** (3) is exhausted, subsequent allocations fail over to the general-purpose +** memory allocator (1). +** +** Earlier versions of SQLite used only methods (1) and (2). But experiments +** show that method (3) with N==100 provides about a 5% performance boost for +** common workloads. */ - +/* #include "sqliteInt.h" */ typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; +/* +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u8 isPinned; /* Page in use, not on the LRU list */ + u8 isBulkLocal; /* This page from bulk local storage */ + u8 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; + /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set -** of one or more PCaches that are able to recycle each others unpinned +** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of ** the following object. ** @@ -42166,7 +48352,7 @@ struct PGroup { unsigned int nMinPage; /* Sum of nMin for purgeable caches */ unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ unsigned int nCurrentPage; /* Number of purgeable pages allocated */ - PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ + PgHdr1 lru; /* The beginning and end of the LRU list */ }; /* Each page cache is an instance of the following object. Every @@ -42184,8 +48370,9 @@ struct PCache1 { ** The PGroup mutex must be held when accessing nMax. */ PGroup *pGroup; /* PGroup this cache belongs to */ - int szPage; /* Size of allocated pages in bytes */ - int szExtra; /* Size of extra space in bytes */ + int szPage; /* Size of database content section */ + int szExtra; /* sizeof(MemPage)+sizeof(PgHdr) */ + int szAlloc; /* Total size of one pcache line */ int bPurgeable; /* True if cache is purgeable */ unsigned int nMin; /* Minimum number of pages reserved */ unsigned int nMax; /* Configured "cache_size" value */ @@ -42199,27 +48386,13 @@ struct PCache1 { unsigned int nPage; /* Total number of pages in apHash */ unsigned int nHash; /* Number of slots in apHash[] */ PgHdr1 **apHash; /* Hash table for fast lookup by key */ + PgHdr1 *pFree; /* List of unused pcache-local pages */ + void *pBulk; /* Bulk memory used by pcache-local */ }; /* -** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. -*/ -struct PgHdr1 { - sqlite3_pcache_page page; - unsigned int iKey; /* Key value (page number) */ - u8 isPinned; /* Page in use, not on the LRU list */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ -}; - -/* -** Free slots in the allocator used to divide up the buffer provided using -** the SQLITE_CONFIG_PAGECACHE mechanism. +** Free slots in the allocator used to divide up the global page cache +** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. */ struct PgFreeslot { PgFreeslot *pNext; /* Next free slot */ @@ -42237,10 +48410,12 @@ static SQLITE_WSD struct PCacheGlobal { ** The nFreeSlot and pFree values do require mutex protection. */ int isInit; /* True if initialized */ + int separateCache; /* Use a new PGroup for each PCache */ + int nInitPage; /* Initial bulk allocation size */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ - void *pStart, *pEnd; /* Bounds of pagecache malloc range */ + void *pStart, *pEnd; /* Bounds of global page cache memory */ /* Above requires no mutex. Use mutex below for variable that follow. */ sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ @@ -42262,12 +48437,20 @@ static SQLITE_WSD struct PCacheGlobal { /* ** Macros to enter and leave the PCache LRU mutex. */ -#define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) -#define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) +#if !defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 +# define pcache1EnterMutex(X) assert((X)->mutex==0) +# define pcache1LeaveMutex(X) assert((X)->mutex==0) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 0 +#else +# define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) +# define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 1 +#endif /******************************************************************************/ /******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + /* ** This function is called during initialization if a static buffer is ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE @@ -42280,6 +48463,7 @@ static SQLITE_WSD struct PCacheGlobal { SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ if( pcache1.isInit ){ PgFreeslot *p; + if( pBuf==0 ) sz = n = 0; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; pcache1.nSlot = pcache1.nFreeSlot = n; @@ -42297,6 +48481,44 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ } } +/* +** Try to initialize the pCache->pFree and pCache->pBulk fields. Return +** true if pCache->pFree ends up containing one or more free pages. +*/ +static int pcache1InitBulk(PCache1 *pCache){ + i64 szBulk; + char *zBulk; + if( pcache1.nInitPage==0 ) return 0; + /* Do not bother with a bulk allocation if the cache size very small */ + if( pCache->nMax<3 ) return 0; + sqlite3BeginBenignMalloc(); + if( pcache1.nInitPage>0 ){ + szBulk = pCache->szAlloc * (i64)pcache1.nInitPage; + }else{ + szBulk = -1024 * (i64)pcache1.nInitPage; + } + if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ + szBulk = pCache->szAlloc*(i64)pCache->nMax; + } + zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); + sqlite3EndBenignMalloc(); + if( zBulk ){ + int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; + int i; + for(i=0; iszPage]; + pX->page.pBuf = zBulk; + pX->page.pExtra = &pX[1]; + pX->isBulkLocal = 1; + pX->isAnchor = 0; + pX->pNext = pCache->pFree; + pCache->pFree = pX; + zBulk += pCache->szAlloc; + } + } + return pCache->pFree!=0; +} + /* ** Malloc function used within this file to allocate space from the buffer ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no @@ -42309,7 +48531,6 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ static void *pcache1Alloc(int nByte){ void *p = 0; assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); if( nByte<=pcache1.szSlot ){ sqlite3_mutex_enter(pcache1.mutex); p = (PgHdr1 *)pcache1.pFree; @@ -42318,7 +48539,8 @@ static void *pcache1Alloc(int nByte){ pcache1.nFreeSlot--; pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); } sqlite3_mutex_leave(pcache1.mutex); } @@ -42331,7 +48553,8 @@ static void *pcache1Alloc(int nByte){ if( p ){ int sz = sqlite3MallocSize(p); sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); sqlite3_mutex_leave(pcache1.mutex); } #endif @@ -42343,13 +48566,12 @@ static void *pcache1Alloc(int nByte){ /* ** Free an allocated buffer obtained from pcache1Alloc(). */ -static int pcache1Free(void *p){ - int nFreed = 0; - if( p==0 ) return 0; - if( p>=pcache1.pStart && ppNext = pcache1.pFree; pcache1.pFree = pSlot; @@ -42360,15 +48582,17 @@ static int pcache1Free(void *p){ }else{ assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - nFreed = sqlite3MallocSize(p); #ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS - sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -nFreed); - sqlite3_mutex_leave(pcache1.mutex); + { + int nFreed = 0; + nFreed = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); + sqlite3_mutex_leave(pcache1.mutex); + } #endif sqlite3_free(p); } - return nFreed; } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT @@ -42392,58 +48616,72 @@ static int pcache1MemSize(void *p){ /* ** Allocate a new page object initially associated with cache pCache. */ -static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ +static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ PgHdr1 *p = 0; void *pPg; - /* The group mutex must be released before pcache1Alloc() is called. This - ** is because it may call sqlite3_release_memory(), which assumes that - ** this mutex is not held. */ assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - pcache1LeaveMutex(pCache->pGroup); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } -#else - pPg = pcache1Alloc(sizeof(PgHdr1) + pCache->szPage + pCache->szExtra); - p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; + if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ + p = pCache->pFree; + pCache->pFree = p->pNext; + p->pNext = 0; + }else{ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + /* The group mutex must be released before pcache1Alloc() is called. This + ** is because it might call sqlite3_release_memory(), which assumes that + ** this mutex is not held. */ + assert( pcache1.separateCache==0 ); + assert( pCache->pGroup==&pcache1.grp ); + pcache1LeaveMutex(pCache->pGroup); #endif - pcache1EnterMutex(pCache->pGroup); - - if( pPg ){ + if( benignMalloc ){ sqlite3BeginBenignMalloc(); } +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + pPg = pcache1Alloc(pCache->szPage); + p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); + if( !pPg || !p ){ + pcache1Free(pPg); + sqlite3_free(p); + pPg = 0; + } +#else + pPg = pcache1Alloc(pCache->szAlloc); + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; +#endif + if( benignMalloc ){ sqlite3EndBenignMalloc(); } +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + pcache1EnterMutex(pCache->pGroup); +#endif + if( pPg==0 ) return 0; p->page.pBuf = pPg; p->page.pExtra = &p[1]; - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage++; - } - return p; + p->isBulkLocal = 0; + p->isAnchor = 0; } - return 0; + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage++; + } + return p; } /* ** Free a page object allocated by pcache1AllocPage(). -** -** The pointer is allowed to be NULL, which is prudent. But it turns out -** that the current implementation happens to never call this routine -** with a NULL pointer, so we mark the NULL test with ALWAYS(). */ static void pcache1FreePage(PgHdr1 *p){ - if( ALWAYS(p) ){ - PCache1 *pCache = p->pCache; - assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + PCache1 *pCache; + assert( p!=0 ); + pCache = p->pCache; + assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + if( p->isBulkLocal ){ + p->pNext = pCache->pFree; + pCache->pFree = p; + }else{ pcache1Free(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER sqlite3_free(p); #endif - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage--; - } + } + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage--; } } @@ -42497,7 +48735,7 @@ static int pcache1UnderMemoryPressure(PCache1 *pCache){ ** ** The PCache mutex must be held when this function is called. */ -static int pcache1ResizeHash(PCache1 *p){ +static void pcache1ResizeHash(PCache1 *p){ PgHdr1 **apNew; unsigned int nNew; unsigned int i; @@ -42529,8 +48767,6 @@ static int pcache1ResizeHash(PCache1 *p){ p->apHash = apNew; p->nHash = nNew; } - - return (p->apHash ? SQLITE_OK : SQLITE_NOMEM); } /* @@ -42540,41 +48776,35 @@ static int pcache1ResizeHash(PCache1 *p){ ** ** The PGroup mutex must be held when this function is called. */ -static void pcache1PinPage(PgHdr1 *pPage){ +static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ PCache1 *pCache; - PGroup *pGroup; assert( pPage!=0 ); assert( pPage->isPinned==0 ); pCache = pPage->pCache; - pGroup = pCache->pGroup; - assert( pPage->pLruNext || pPage==pGroup->pLruTail ); - assert( pPage->pLruPrev || pPage==pGroup->pLruHead ); - assert( sqlite3_mutex_held(pGroup->mutex) ); - if( pPage->pLruPrev ){ - pPage->pLruPrev->pLruNext = pPage->pLruNext; - }else{ - pGroup->pLruHead = pPage->pLruNext; - } - if( pPage->pLruNext ){ - pPage->pLruNext->pLruPrev = pPage->pLruPrev; - }else{ - pGroup->pLruTail = pPage->pLruPrev; - } + assert( pPage->pLruNext ); + assert( pPage->pLruPrev ); + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; pPage->pLruPrev = 0; pPage->isPinned = 1; + assert( pPage->isAnchor==0 ); + assert( pCache->pGroup->lru.isAnchor==1 ); pCache->nRecyclable--; + return pPage; } /* ** Remove the page supplied as an argument from the hash table ** (PCache1.apHash structure) that it is currently stored in. +** Also free the page if freePage is true. ** ** The PGroup mutex must be held when this function is called. */ -static void pcache1RemoveFromHash(PgHdr1 *pPage){ +static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ unsigned int h; PCache1 *pCache = pPage->pCache; PgHdr1 **pp; @@ -42585,21 +48815,28 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage){ *pp = (*pp)->pNext; pCache->nPage--; + if( freeFlag ) pcache1FreePage(pPage); } /* ** If there are currently more than nMaxPage pages allocated, try ** to recycle pages to reduce the number allocated to nMaxPage. */ -static void pcache1EnforceMaxPage(PGroup *pGroup){ +static void pcache1EnforceMaxPage(PCache1 *pCache){ + PGroup *pGroup = pCache->pGroup; + PgHdr1 *p; assert( sqlite3_mutex_held(pGroup->mutex) ); - while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ - PgHdr1 *p = pGroup->pLruTail; + while( pGroup->nCurrentPage>pGroup->nMaxPage + && (p=pGroup->lru.pLruPrev)->isAnchor==0 + ){ assert( p->pCache->pGroup==pGroup ); assert( p->isPinned==0 ); pcache1PinPage(p); - pcache1RemoveFromHash(p); - pcache1FreePage(p); + pcache1RemoveFromHash(p, 1); + } + if( pCache->nPage==0 && pCache->pBulk ){ + sqlite3_free(pCache->pBulk); + pCache->pBulk = pCache->pFree = 0; } } @@ -42614,12 +48851,30 @@ static void pcache1TruncateUnsafe( PCache1 *pCache, /* The cache to truncate */ unsigned int iLimit /* Drop pages with this pgno or larger */ ){ - TESTONLY( unsigned int nPage = 0; ) /* To assert pCache->nPage is correct */ - unsigned int h; + TESTONLY( int nPage = 0; ) /* To assert pCache->nPage is correct */ + unsigned int h, iStop; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - for(h=0; hnHash; h++){ - PgHdr1 **pp = &pCache->apHash[h]; + assert( pCache->iMaxKey >= iLimit ); + assert( pCache->nHash > 0 ); + if( pCache->iMaxKey - iLimit < pCache->nHash ){ + /* If we are just shaving the last few pages off the end of the + ** cache, then there is no point in scanning the entire hash table. + ** Only scan those hash slots that might contain pages that need to + ** be removed. */ + h = iLimit % pCache->nHash; + iStop = pCache->iMaxKey % pCache->nHash; + TESTONLY( nPage = -10; ) /* Disable the pCache->nPage validity check */ + }else{ + /* This is the general case where many pages are being removed. + ** It is necessary to scan the entire hash table */ + h = pCache->nHash/2; + iStop = h - 1; + } + for(;;){ + PgHdr1 **pp; PgHdr1 *pPage; + assert( hnHash ); + pp = &pCache->apHash[h]; while( (pPage = *pp)!=0 ){ if( pPage->iKey>=iLimit ){ pCache->nPage--; @@ -42628,11 +48883,13 @@ static void pcache1TruncateUnsafe( pcache1FreePage(pPage); }else{ pp = &pPage->pNext; - TESTONLY( nPage++; ) + TESTONLY( if( nPage>=0 ) nPage++; ) } } + if( h==iStop ) break; + h = (h+1) % pCache->nHash; } - assert( pCache->nPage==nPage ); + assert( nPage<0 || pCache->nPage==(unsigned)nPage ); } /******************************************************************************/ @@ -42645,9 +48902,44 @@ static int pcache1Init(void *NotUsed){ UNUSED_PARAMETER(NotUsed); assert( pcache1.isInit==0 ); memset(&pcache1, 0, sizeof(pcache1)); + + + /* + ** The pcache1.separateCache variable is true if each PCache has its own + ** private PGroup (mode-1). pcache1.separateCache is false if the single + ** PGroup in pcache1.grp is used for all page caches (mode-2). + ** + ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT + ** + ** * Use a unified cache in single-threaded applications that have + ** configured a start-time buffer for use as page-cache memory using + ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL + ** pBuf argument. + ** + ** * Otherwise use separate caches (mode-1) + */ +#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) + pcache1.separateCache = 0; +#elif SQLITE_THREADSAFE + pcache1.separateCache = sqlite3GlobalConfig.pPage==0 + || sqlite3GlobalConfig.bCoreMutex>0; +#else + pcache1.separateCache = sqlite3GlobalConfig.pPage==0; +#endif + +#if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ - pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); - pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); + pcache1.grp.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PMEM); + } +#endif + if( pcache1.separateCache + && sqlite3GlobalConfig.nPage!=0 + && sqlite3GlobalConfig.pPage==0 + ){ + pcache1.nInitPage = sqlite3GlobalConfig.nPage; + }else{ + pcache1.nInitPage = 0; } pcache1.grp.mxPinned = 10; pcache1.isInit = 1; @@ -42665,6 +48957,9 @@ static void pcache1Shutdown(void *NotUsed){ memset(&pcache1, 0, sizeof(pcache1)); } +/* forward declaration */ +static void pcache1Destroy(sqlite3_pcache *p); + /* ** Implementation of the sqlite3_pcache.xCreate method. ** @@ -42675,46 +48970,38 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ PGroup *pGroup; /* The group the new page cache will belong to */ int sz; /* Bytes of memory required to allocate the new cache */ - /* - ** The separateCache variable is true if each PCache has its own private - ** PGroup. In other words, separateCache is true for mode (1) where no - ** mutexing is required. - ** - ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT - ** - ** * Always use a unified cache in single-threaded applications - ** - ** * Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off) - ** use separate caches (mode-1) - */ -#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 - const int separateCache = 0; -#else - int separateCache = sqlite3GlobalConfig.bCoreMutex>0; -#endif - assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); assert( szExtra < 300 ); - sz = sizeof(PCache1) + sizeof(PGroup)*separateCache; + sz = sizeof(PCache1) + sizeof(PGroup)*pcache1.separateCache; pCache = (PCache1 *)sqlite3MallocZero(sz); if( pCache ){ - if( separateCache ){ + if( pcache1.separateCache ){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; }else{ pGroup = &pcache1.grp; } + if( pGroup->lru.isAnchor==0 ){ + pGroup->lru.isAnchor = 1; + pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; + } pCache->pGroup = pGroup; pCache->szPage = szPage; pCache->szExtra = szExtra; + pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); pCache->bPurgeable = (bPurgeable ? 1 : 0); + pcache1EnterMutex(pGroup); + pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; - pcache1EnterMutex(pGroup); pGroup->nMinPage += pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; - pcache1LeaveMutex(pGroup); + } + pcache1LeaveMutex(pGroup); + if( pCache->nHash==0 ){ + pcache1Destroy((sqlite3_pcache*)pCache); + pCache = 0; } } return (sqlite3_pcache *)pCache; @@ -42734,7 +49021,7 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pCache->nMax = nMax; pCache->n90pct = pCache->nMax*9/10; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); } } @@ -42752,7 +49039,7 @@ static void pcache1Shrink(sqlite3_pcache *p){ pcache1EnterMutex(pGroup); savedMaxPage = pGroup->nMaxPage; pGroup->nMaxPage = 0; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pGroup->nMaxPage = savedMaxPage; pcache1LeaveMutex(pGroup); } @@ -42770,6 +49057,84 @@ static int pcache1Pagecount(sqlite3_pcache *p){ return n; } + +/* +** Implement steps 3, 4, and 5 of the pcache1Fetch() algorithm described +** in the header of the pcache1Fetch() procedure. +** +** This steps are broken out into a separate procedure because they are +** usually not needed, and by avoiding the stack initialization required +** for these steps, the main pcache1Fetch() procedure can run faster. +*/ +static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( + PCache1 *pCache, + unsigned int iKey, + int createFlag +){ + unsigned int nPinned; + PGroup *pGroup = pCache->pGroup; + PgHdr1 *pPage = 0; + + /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ + assert( pCache->nPage >= pCache->nRecyclable ); + nPinned = pCache->nPage - pCache->nRecyclable; + assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); + assert( pCache->n90pct == pCache->nMax*9/10 ); + if( createFlag==1 && ( + nPinned>=pGroup->mxPinned + || nPinned>=pCache->n90pct + || (pcache1UnderMemoryPressure(pCache) && pCache->nRecyclablenPage>=pCache->nHash ) pcache1ResizeHash(pCache); + assert( pCache->nHash>0 && pCache->apHash ); + + /* Step 4. Try to recycle a page. */ + if( pCache->bPurgeable + && !pGroup->lru.pLruPrev->isAnchor + && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) + ){ + PCache1 *pOther; + pPage = pGroup->lru.pLruPrev; + assert( pPage->isPinned==0 ); + pcache1RemoveFromHash(pPage, 0); + pcache1PinPage(pPage); + pOther = pPage->pCache; + if( pOther->szAlloc != pCache->szAlloc ){ + pcache1FreePage(pPage); + pPage = 0; + }else{ + pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); + } + } + + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. + */ + if( !pPage ){ + pPage = pcache1AllocPage(pCache, createFlag==1); + } + + if( pPage ){ + unsigned int h = iKey % pCache->nHash; + pCache->nPage++; + pPage->iKey = iKey; + pPage->pNext = pCache->apHash[h]; + pPage->pCache = pCache; + pPage->pLruPrev = 0; + pPage->pLruNext = 0; + pPage->isPinned = 1; + *(void **)pPage->page.pExtra = 0; + pCache->apHash[h] = pPage; + if( iKey>pCache->iMaxKey ){ + pCache->iMaxKey = iKey; + } + } + return pPage; +} + /* ** Implementation of the sqlite3_pcache.xFetch method. ** @@ -42823,124 +49188,80 @@ static int pcache1Pagecount(sqlite3_pcache *p){ ** proceed to step 5. ** ** 5. Otherwise, allocate and return a new page buffer. +** +** There are two versions of this routine. pcache1FetchWithMutex() is +** the general case. pcache1FetchNoMutex() is a faster implementation for +** the common case where pGroup->mutex is NULL. The pcache1Fetch() wrapper +** invokes the appropriate routine. */ +static PgHdr1 *pcache1FetchNoMutex( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = 0; + + /* Step 1: Search the hash table for an existing entry. */ + pPage = pCache->apHash[iKey % pCache->nHash]; + while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } + + /* Step 2: If the page was found in the hash table, then return it. + ** If the page was not in the hash table and createFlag is 0, abort. + ** Otherwise (page not in hash and createFlag!=0) continue with + ** subsequent steps to try to create the page. */ + if( pPage ){ + if( !pPage->isPinned ){ + return pcache1PinPage(pPage); + }else{ + return pPage; + } + }else if( createFlag ){ + /* Steps 3, 4, and 5 implemented by this subroutine */ + return pcache1FetchStage2(pCache, iKey, createFlag); + }else{ + return 0; + } +} +#if PCACHE1_MIGHT_USE_GROUP_MUTEX +static PgHdr1 *pcache1FetchWithMutex( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage; + + pcache1EnterMutex(pCache->pGroup); + pPage = pcache1FetchNoMutex(p, iKey, createFlag); + assert( pPage==0 || pCache->iMaxKey>=iKey ); + pcache1LeaveMutex(pCache->pGroup); + return pPage; +} +#endif static sqlite3_pcache_page *pcache1Fetch( sqlite3_pcache *p, unsigned int iKey, int createFlag ){ - unsigned int nPinned; +#if PCACHE1_MIGHT_USE_GROUP_MUTEX || defined(SQLITE_DEBUG) PCache1 *pCache = (PCache1 *)p; - PGroup *pGroup; - PgHdr1 *pPage = 0; +#endif assert( offsetof(PgHdr1,page)==0 ); assert( pCache->bPurgeable || createFlag!=1 ); assert( pCache->bPurgeable || pCache->nMin==0 ); assert( pCache->bPurgeable==0 || pCache->nMin==10 ); assert( pCache->nMin==0 || pCache->bPurgeable ); - pcache1EnterMutex(pGroup = pCache->pGroup); - - /* Step 1: Search the hash table for an existing entry. */ - if( pCache->nHash>0 ){ - unsigned int h = iKey % pCache->nHash; - for(pPage=pCache->apHash[h]; pPage&&pPage->iKey!=iKey; pPage=pPage->pNext); - } - - /* Step 2: Abort if no existing page is found and createFlag is 0 */ - if( pPage ){ - if( !pPage->isPinned ) pcache1PinPage(pPage); - goto fetch_out; - } - if( createFlag==0 ){ - goto fetch_out; - } - - /* The pGroup local variable will normally be initialized by the - ** pcache1EnterMutex() macro above. But if SQLITE_MUTEX_OMIT is defined, - ** then pcache1EnterMutex() is a no-op, so we have to initialize the - ** local variable here. Delaying the initialization of pGroup is an - ** optimization: The common case is to exit the module before reaching - ** this point. - */ -#ifdef SQLITE_MUTEX_OMIT - pGroup = pCache->pGroup; + assert( pCache->nHash>0 ); +#if PCACHE1_MIGHT_USE_GROUP_MUTEX + if( pCache->pGroup->mutex ){ + return (sqlite3_pcache_page*)pcache1FetchWithMutex(p, iKey, createFlag); + }else #endif - - /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ - assert( pCache->nPage >= pCache->nRecyclable ); - nPinned = pCache->nPage - pCache->nRecyclable; - assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); - assert( pCache->n90pct == pCache->nMax*9/10 ); - if( createFlag==1 && ( - nPinned>=pGroup->mxPinned - || nPinned>=pCache->n90pct - || pcache1UnderMemoryPressure(pCache) - )){ - goto fetch_out; + { + return (sqlite3_pcache_page*)pcache1FetchNoMutex(p, iKey, createFlag); } - - if( pCache->nPage>=pCache->nHash && pcache1ResizeHash(pCache) ){ - goto fetch_out; - } - assert( pCache->nHash>0 && pCache->apHash ); - - /* Step 4. Try to recycle a page. */ - if( pCache->bPurgeable && pGroup->pLruTail && ( - (pCache->nPage+1>=pCache->nMax) - || pGroup->nCurrentPage>=pGroup->nMaxPage - || pcache1UnderMemoryPressure(pCache) - )){ - PCache1 *pOther; - pPage = pGroup->pLruTail; - assert( pPage->isPinned==0 ); - pcache1RemoveFromHash(pPage); - pcache1PinPage(pPage); - pOther = pPage->pCache; - - /* We want to verify that szPage and szExtra are the same for pOther - ** and pCache. Assert that we can verify this by comparing sums. */ - assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); - assert( pCache->szExtra<512 ); - assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); - assert( pOther->szExtra<512 ); - - if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ - pcache1FreePage(pPage); - pPage = 0; - }else{ - pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); - } - } - - /* Step 5. If a usable page buffer has still not been found, - ** attempt to allocate a new one. - */ - if( !pPage ){ - if( createFlag==1 ) sqlite3BeginBenignMalloc(); - pPage = pcache1AllocPage(pCache); - if( createFlag==1 ) sqlite3EndBenignMalloc(); - } - - if( pPage ){ - unsigned int h = iKey % pCache->nHash; - pCache->nPage++; - pPage->iKey = iKey; - pPage->pNext = pCache->apHash[h]; - pPage->pCache = pCache; - pPage->pLruPrev = 0; - pPage->pLruNext = 0; - pPage->isPinned = 1; - *(void **)pPage->page.pExtra = 0; - pCache->apHash[h] = pPage; - } - -fetch_out: - if( pPage && iKey>pCache->iMaxKey ){ - pCache->iMaxKey = iKey; - } - pcache1LeaveMutex(pGroup); - return (sqlite3_pcache_page*)pPage; } @@ -42965,22 +49286,16 @@ static void pcache1Unpin( ** part of the PGroup LRU list. */ assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); - assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage ); assert( pPage->isPinned==1 ); if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ - pcache1RemoveFromHash(pPage); - pcache1FreePage(pPage); + pcache1RemoveFromHash(pPage, 1); }else{ /* Add the page to the PGroup LRU list. */ - if( pGroup->pLruHead ){ - pGroup->pLruHead->pLruPrev = pPage; - pPage->pLruNext = pGroup->pLruHead; - pGroup->pLruHead = pPage; - }else{ - pGroup->pLruTail = pPage; - pGroup->pLruHead = pPage; - } + PgHdr1 **ppFirst = &pGroup->lru.pLruNext; + pPage->pLruPrev = &pGroup->lru; + (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; + *ppFirst = pPage; pCache->nRecyclable++; pPage->isPinned = 0; } @@ -43051,14 +49366,15 @@ static void pcache1Destroy(sqlite3_pcache *p){ PGroup *pGroup = pCache->pGroup; assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); pcache1EnterMutex(pGroup); - pcache1TruncateUnsafe(pCache, 0); + if( pCache->nPage ) pcache1TruncateUnsafe(pCache, 0); assert( pGroup->nMaxPage >= pCache->nMax ); pGroup->nMaxPage -= pCache->nMax; assert( pGroup->nMinPage >= pCache->nMin ); pGroup->nMinPage -= pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); + sqlite3_free(pCache->pBulk); sqlite3_free(pCache->apHash); sqlite3_free(pCache); } @@ -43087,6 +49403,19 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void){ sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultMethods); } +/* +** Return the size of the header on each page of this PCACHE implementation. +*/ +SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void){ return ROUND8(sizeof(PgHdr1)); } + +/* +** Return the global mutex used by this PCACHE implementation. The +** sqlite3_status() routine needs access to this mutex. +*/ +SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void){ + return pcache1.mutex; +} + #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* ** This function is called to free superfluous dynamically allocated memory @@ -43101,18 +49430,20 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); assert( sqlite3_mutex_notheld(pcache1.mutex) ); - if( pcache1.pStart==0 ){ + if( sqlite3GlobalConfig.nPage==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); - while( (nReq<0 || nFreeisAnchor==0 + ){ nFree += pcache1MemSize(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER nFree += sqlite3MemSize(p); #endif assert( p->isPinned==0 ); pcache1PinPage(p); - pcache1RemoveFromHash(p); - pcache1FreePage(p); + pcache1RemoveFromHash(p, 1); } pcache1LeaveMutex(&pcache1.grp); } @@ -43133,7 +49464,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ){ PgHdr1 *p; int nRecyclable = 0; - for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){ + for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ assert( p->isPinned==0 ); nRecyclable++; } @@ -43198,16 +49529,18 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ** No INSERTs may occurs after a SMALLEST. An assertion will fail if ** that is attempted. ** -** The cost of an INSERT is roughly constant. (Sometime new memory +** The cost of an INSERT is roughly constant. (Sometimes new memory ** has to be allocated on an INSERT.) The cost of a TEST with a new ** batch number is O(NlogN) where N is the number of elements in the RowSet. ** The cost of a TEST using the same batch number is O(logN). The cost ** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST ** primitives are constant time. The cost of DESTROY is O(N). ** -** There is an added cost of O(N) when switching between TEST and -** SMALLEST primitives. +** TEST and SMALLEST may not be used by the same RowSet. This used to +** be possible, but the feature was not used, so it was removed in order +** to simplify the code. */ +/* #include "sqliteInt.h" */ /* @@ -43326,9 +49659,11 @@ SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ */ static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); - if( p->nFresh==0 ){ + if( p->nFresh==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* We could allocate a fresh RowSetEntry each time one is needed, but it + ** is more efficient to pull a preallocated entry from the pool */ struct RowSetChunk *pNew; - pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); + pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); if( pNew==0 ){ return 0; } @@ -43360,7 +49695,9 @@ SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet *p, i64 rowid){ pEntry->pRight = 0; pLast = p->pLast; if( pLast ){ - if( (p->rsFlags & ROWSET_SORTED)!=0 && rowid<=pLast->v ){ + if( rowid<=pLast->v ){ /*OPTIMIZATION-IF-FALSE*/ + /* Avoid unnecessary sorts by preserving the ROWSET_SORTED flags + ** where possible */ p->rsFlags &= ~ROWSET_SORTED; } pLast->pRight = pEntry; @@ -43384,28 +49721,26 @@ static struct RowSetEntry *rowSetEntryMerge( struct RowSetEntry *pTail; pTail = &head; - while( pA && pB ){ + assert( pA!=0 && pB!=0 ); + for(;;){ assert( pA->pRight==0 || pA->v<=pA->pRight->v ); assert( pB->pRight==0 || pB->v<=pB->pRight->v ); - if( pA->vv ){ - pTail->pRight = pA; + if( pA->v<=pB->v ){ + if( pA->vv ) pTail = pTail->pRight = pA; pA = pA->pRight; - pTail = pTail->pRight; - }else if( pB->vv ){ - pTail->pRight = pB; - pB = pB->pRight; - pTail = pTail->pRight; + if( pA==0 ){ + pTail->pRight = pB; + break; + } }else{ - pA = pA->pRight; + pTail = pTail->pRight = pB; + pB = pB->pRight; + if( pB==0 ){ + pTail->pRight = pA; + break; + } } } - if( pA ){ - assert( pA->pRight==0 || pA->v<=pA->pRight->v ); - pTail->pRight = pA; - }else{ - assert( pB==0 || pB->pRight==0 || pB->v<=pB->pRight->v ); - pTail->pRight = pB; - } return head.pRight; } @@ -43428,9 +49763,10 @@ static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ aBucket[i] = pIn; pIn = pNext; } - pIn = 0; - for(i=0; i1 ){ /*OPTIMIZATION-IF-TRUE*/ + /* This branch causes a *balanced* tree to be generated. A valid tree + ** is still generated without this branch, but the tree is wildly + ** unbalanced and inefficient. */ + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* It is safe to always return here, but the resulting tree + ** would be unbalanced */ + return pLeft; + } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); + }else{ p = *ppList; *ppList = p->pRight; p->pLeft = p->pRight = 0; - return p; } - pLeft = rowSetNDeepTree(ppList, iDepth-1); - p = *ppList; - if( p==0 ){ - return pLeft; - } - p->pLeft = pLeft; - *ppList = p->pRight; - p->pRight = rowSetNDeepTree(ppList, iDepth-1); return p; } @@ -43525,59 +49867,37 @@ static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ return p; } -/* -** Take all the entries on p->pEntry and on the trees in p->pForest and -** sort them all together into one big ordered list on p->pEntry. -** -** This routine should only be called once in the life of a RowSet. -*/ -static void rowSetToList(RowSet *p){ - - /* This routine is called only once */ - assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); - - if( (p->rsFlags & ROWSET_SORTED)==0 ){ - p->pEntry = rowSetEntrySort(p->pEntry); - } - - /* While this module could theoretically support it, sqlite3RowSetNext() - ** is never called after sqlite3RowSetText() for the same RowSet. So - ** there is never a forest to deal with. Should this change, simply - ** remove the assert() and the #if 0. */ - assert( p->pForest==0 ); -#if 0 - while( p->pForest ){ - struct RowSetEntry *pTree = p->pForest->pLeft; - if( pTree ){ - struct RowSetEntry *pHead, *pTail; - rowSetTreeToList(pTree, &pHead, &pTail); - p->pEntry = rowSetEntryMerge(p->pEntry, pHead); - } - p->pForest = p->pForest->pRight; - } -#endif - p->rsFlags |= ROWSET_NEXT; /* Verify this routine is never called again */ -} - /* ** Extract the smallest element from the RowSet. ** Write the element into *pRowid. Return 1 on success. Return ** 0 if the RowSet is already empty. ** ** After this routine has been called, the sqlite3RowSetInsert() -** routine may not be called again. +** routine may not be called again. +** +** This routine may not be called after sqlite3RowSetTest() has +** been used. Older versions of RowSet allowed that, but as the +** capability was not used by the code generator, it was removed +** for code economy. */ SQLITE_PRIVATE int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ assert( p!=0 ); + assert( p->pForest==0 ); /* Cannot be used with sqlite3RowSetText() */ /* Merge the forest into a single sorted list on first call */ - if( (p->rsFlags & ROWSET_NEXT)==0 ) rowSetToList(p); + if( (p->rsFlags & ROWSET_NEXT)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (p->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + p->pEntry = rowSetEntrySort(p->pEntry); + } + p->rsFlags |= ROWSET_SORTED|ROWSET_NEXT; + } /* Return the next entry on the list */ if( p->pEntry ){ *pRowid = p->pEntry->v; p->pEntry = p->pEntry->pRight; - if( p->pEntry==0 ){ + if( p->pEntry==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Free memory immediately, rather than waiting on sqlite3_finalize() */ sqlite3RowSetClear(p); } return 1; @@ -43590,8 +49910,8 @@ SQLITE_PRIVATE int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ ** Check to see if element iRowid was inserted into the rowset as ** part of any insert batch prior to iBatch. Return 1 or 0. ** -** If this is the first test of a new batch and if there exist entires -** on pRowSet->pEntry, then sort those entires into the forest at +** If this is the first test of a new batch and if there exist entries +** on pRowSet->pEntry, then sort those entries into the forest at ** pRowSet->pForest so that they can be tested. */ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ @@ -43600,13 +49920,15 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 /* This routine is never called after sqlite3RowSetNext() */ assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); - /* Sort entries into the forest on the first test of a new batch + /* Sort entries into the forest on the first test of a new batch. + ** To save unnecessary work, only do this when the batch number changes. */ - if( iBatch!=pRowSet->iBatch ){ + if( iBatch!=pRowSet->iBatch ){ /*OPTIMIZATION-IF-FALSE*/ p = pRowSet->pEntry; if( p ){ struct RowSetEntry **ppPrevTree = &pRowSet->pForest; - if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Only sort the current set of entiries if they need it */ p = rowSetEntrySort(p); } for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ @@ -43677,6 +49999,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 ** another is writing. */ #ifndef SQLITE_OMIT_DISKIO +/* #include "sqliteInt.h" */ /************** Include wal.h in the middle of pager.c ***********************/ /************** Begin file wal.h *********************************************/ /* @@ -43695,9 +50018,10 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 ** the implementation of each function in log.c for further details. */ -#ifndef _WAL_H_ -#define _WAL_H_ +#ifndef SQLITE_WAL_H +#define SQLITE_WAL_H +/* #include "sqliteInt.h" */ /* Additional values that can be added to the sync_flags argument of ** sqlite3WalFrames(): @@ -43724,6 +50048,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 +# define sqlite3WalFile(x) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -43806,6 +50131,11 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op); */ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); +#ifdef SQLITE_ENABLE_SNAPSHOT +SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot); +SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). @@ -43813,8 +50143,11 @@ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal); #endif +/* Return the sqlite3_file object for the WAL file */ +SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal); + #endif /* ifndef SQLITE_OMIT_WAL */ -#endif /* _WAL_H_ */ +#endif /* SQLITE_WAL_H */ /************** End of wal.h *************************************************/ /************** Continuing where we left off in pager.c **********************/ @@ -43873,12 +50206,12 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal); ** Definition: Two databases (or the same database at two points it time) ** are said to be "logically equivalent" if they give the same answer to ** all queries. Note in particular the content of freelist leaf -** pages can be changed arbitarily without effecting the logical equivalence +** pages can be changed arbitrarily without affecting the logical equivalence ** of the database. ** ** (7) At any time, if any subset, including the empty set and the total set, ** of the unsynced changes to a rollback journal are removed and the -** journal is rolled back, the resulting database file will be logical +** journal is rolled back, the resulting database file will be logically ** equivalent to the database file at the beginning of the transaction. ** ** (8) When a transaction is rolled back, the xTruncate method of the VFS @@ -44175,7 +50508,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** The exception is when the database file is unlocked as the pager moves ** from ERROR to OPEN state. At this point there may be a hot-journal file -** in the file-system that needs to be rolled back (as part of a OPEN->SHARED +** in the file-system that needs to be rolled back (as part of an OPEN->SHARED ** transition, by the same pager or any other). If the call to xUnlock() ** fails at this point and the pager is left holding an EXCLUSIVE lock, this ** can confuse the call to xCheckReservedLock() call made later as part @@ -44225,6 +50558,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ */ #define MAX_SECTOR_SIZE 0x10000 + /* ** An instance of the following structure is allocated for each active ** savepoint and statement transaction in the system. All such structures @@ -44253,12 +50587,12 @@ struct PagerSavepoint { /* ** Bits of the Pager.doNotSpill flag. See further description below. */ -#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ -#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ -#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ +#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ +#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ +#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ /* -** A open page cache is an instance of struct Pager. A description of +** An open page cache is an instance of struct Pager. A description of ** some of the more important member variables follows: ** ** eState @@ -44337,11 +50671,11 @@ struct PagerSavepoint { ** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF ** case is a user preference. ** -** If the SPILLFLAG_NOSYNC bit is set, writing to the database from pagerStress() -** is permitted, but syncing the journal file is not. This flag is set -** by sqlite3PagerWrite() when the file-system sector-size is larger than -** the database page-size in order to prevent a journal sync from happening -** in between the journalling of two pages on the same sector. +** If the SPILLFLAG_NOSYNC bit is set, writing to the database from +** pagerStress() is permitted, but syncing the journal file is not. +** This flag is set by sqlite3PagerWrite() when the file-system sector-size +** is larger than the database page-size in order to prevent a journal sync +** from happening in between the journalling of two pages on the same sector. ** ** subjInMemory ** @@ -44420,6 +50754,7 @@ struct Pager { u8 useJournal; /* Use a rollback journal on this file */ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ + u8 extraSync; /* sync directory after journal delete */ u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ @@ -44430,7 +50765,7 @@ struct Pager { /************************************************************************** ** The following block contains those class members that change during - ** routine opertion. Class members not in this block are either fixed + ** routine operation. Class members not in this block are either fixed ** when the pager is first created or else only change when there is a ** significant mode change (such as changing the page_size, locking_mode, ** or the journal_mode). From another view, these class members describe @@ -44443,6 +50778,8 @@ struct Pager { u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ + u8 bUseFetch; /* True to use xFetch() */ + u8 hasHeldSharedLock; /* True if a shared lock has ever been held */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -44460,9 +50797,9 @@ struct Pager { sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ + u32 iDataVersion; /* Changes whenever database content changes */ char dbFileVers[16]; /* Changes whenever database file changes */ - u8 bUseFetch; /* True to use xFetch() */ int nMmapOut; /* Number of mmap pages currently outstanding */ sqlite3_int64 szMmap; /* Desired maximum mmap size */ PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ @@ -44603,16 +50940,17 @@ static const unsigned char aJournalMagic[] = { ** ** if( pPager->jfd->pMethods ){ ... */ -#define isOpen(pFd) ((pFd)->pMethods) +#define isOpen(pFd) ((pFd)->pMethods!=0) /* ** Return true if this pager uses a write-ahead log instead of the usual ** rollback journal. Otherwise false. */ #ifndef SQLITE_OMIT_WAL -static int pagerUseWal(Pager *pPager){ +SQLITE_PRIVATE int sqlite3PagerUseWal(Pager *pPager){ return (pPager->pWal!=0); } +# define pagerUseWal(x) sqlite3PagerUseWal(x) #else # define pagerUseWal(x) 0 # define pagerRollbackWal(x) 0 @@ -44665,6 +51003,7 @@ static int assert_pager_state(Pager *p){ ** state. */ if( MEMDB ){ + assert( !isOpen(p->fd) ); assert( p->noSync ); assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_MEMORY @@ -44751,7 +51090,7 @@ static int assert_pager_state(Pager *p){ ** back to OPEN state. */ assert( pPager->errCode!=SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile ); break; } @@ -44826,19 +51165,21 @@ static int subjRequiresPage(PgHdr *pPg){ int i; for(i=0; inSavepoint; i++){ p = &pPager->aSavepoint[i]; - if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){ + if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){ return 1; } } return 0; } +#ifdef SQLITE_DEBUG /* ** Return true if the page is already in the journal file. */ static int pageInJournal(Pager *pPager, PgHdr *pPg){ return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno); } +#endif /* ** Read a 32-bit integer from the given file descriptor. Store the integer @@ -44961,6 +51302,8 @@ static int jrnlBufferSize(Pager *pPager){ return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); } +#else +# define jrnlBufferSize(x) 0 #endif /* @@ -45121,6 +51464,7 @@ static i64 journalHdrOffset(Pager *pPager){ static int zeroJournalHdr(Pager *pPager, int doTruncate){ int rc = SQLITE_OK; /* Return code */ assert( isOpen(pPager->jfd) ); + assert( !sqlite3JournalIsInMemory(pPager->jfd) ); if( pPager->journalOff ){ const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ @@ -45450,7 +51794,8 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ || (0 != (rc = sqlite3OsWrite(pPager->jfd, zMaster, nMaster, iHdrOff+4))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster, nMaster))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster+4, cksum))) - || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, iHdrOff+4+nMaster+8))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, + iHdrOff+4+nMaster+8))) ){ return rc; } @@ -45474,29 +51819,23 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ return rc; } -/* -** Find a page in the hash table given its page number. Return -** a pointer to the page or NULL if the requested page is not -** already in memory. -*/ -static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ - PgHdr *p = 0; /* Return value */ - - /* It is not possible for a call to PcacheFetch() with createFlag==0 to - ** fail, since no attempt to allocate dynamic memory will be made. - */ - (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &p); - return p; -} - /* ** Discard the entire contents of the in-memory page-cache. */ static void pager_reset(Pager *pPager){ + pPager->iDataVersion++; sqlite3BackupRestart(pPager->pBackup); sqlite3PcacheClear(pPager->pPCache); } +/* +** Return the pPager->iDataVersion value +*/ +SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager *pPager){ + assert( pPager->eState>PAGER_OPEN ); + return pPager->iDataVersion; +} + /* ** Free all structures in the Pager.aSavepoint[] array and set both ** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal @@ -45507,7 +51846,7 @@ static void releaseAllSavepoints(Pager *pPager){ for(ii=0; iinSavepoint; ii++){ sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } - if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){ + if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){ sqlite3OsClose(pPager->sjfd); } sqlite3_free(pPager->aSavepoint); @@ -45613,13 +51952,17 @@ static void pager_unlock(Pager *pPager){ ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. */ + assert( pPager->errCode==SQLITE_OK || !MEMDB ); if( pPager->errCode ){ - assert( !MEMDB ); - pager_reset(pPager); - pPager->changeCountDone = pPager->tempFile; - pPager->eState = PAGER_OPEN; - pPager->errCode = SQLITE_OK; + if( pPager->tempFile==0 ){ + pager_reset(pPager); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + }else{ + pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER); + } if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + pPager->errCode = SQLITE_OK; } pPager->journalOff = 0; @@ -45663,6 +52006,29 @@ static int pager_error(Pager *pPager, int rc){ static int pager_truncate(Pager *pPager, Pgno nPage); +/* +** The write transaction open on pPager is being committed (bCommit==1) +** or rolled back (bCommit==0). +** +** Return TRUE if and only if all dirty pages should be flushed to disk. +** +** Rules: +** +** * For non-TEMP databases, always sync to disk. This is necessary +** for transactions to be durable. +** +** * Sync TEMP database only on a COMMIT (not a ROLLBACK) when the backing +** file has been created already (via a spill on pagerStress()) and +** when the number of dirty pages in memory exceeds 25% of the total +** cache size. +*/ +static int pagerFlushOnCommit(Pager *pPager, int bCommit){ + if( pPager->tempFile==0 ) return 1; + if( !bCommit ) return 0; + if( !isOpen(pPager->fd) ) return 0; + return (sqlite3PCachePercentDirty(pPager->pPCache)>=25); +} + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -45745,35 +52111,44 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ - if( sqlite3IsMemJournal(pPager->jfd) ){ - assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); + if( sqlite3JournalIsInMemory(pPager->jfd) ){ + /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */ sqlite3OsClose(pPager->jfd); }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ if( pPager->journalOff==0 ){ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ - rc = zeroJournalHdr(pPager, hasMaster); + rc = zeroJournalHdr(pPager, hasMaster||pPager->tempFile); pPager->journalOff = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal ** file should be closed and deleted. If this connection writes to - ** the database file, it will do so using an in-memory journal. + ** the database file, it will do so using an in-memory journal. */ - int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd)); + int bDelete = !pPager->tempFile; + assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_WAL ); sqlite3OsClose(pPager->jfd); if( bDelete ){ - rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync); } } } @@ -45781,7 +52156,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ #ifdef SQLITE_CHECK_PAGES sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ - PgHdr *p = pager_lookup(pPager, 1); + PgHdr *p = sqlite3PagerLookup(pPager, 1); if( p ){ p->pageHash = 0; sqlite3PagerUnrefNotNull(p); @@ -45792,8 +52167,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - sqlite3PcacheCleanAll(pPager->pPCache); - sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + if( rc==SQLITE_OK ){ + if( pagerFlushOnCommit(pPager, bCommit) ){ + sqlite3PcacheCleanAll(pPager->pPCache); + }else{ + sqlite3PcacheClearWritable(pPager->pPCache); + } + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + } if( pagerUseWal(pPager) ){ /* Drop the WAL write-lock, if any. Also, if the connection was in @@ -45906,6 +52287,20 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif +#ifdef SQLITE_HAS_CODEC +/* +** Make sure the number of reserved bits is the same in the destination +** pager as it is in the source. This comes up when a VACUUM changes the +** number of reserved bits to the "optimal" amount. +*/ +SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ + if( pDest->nReserve!=pSrc->nReserve ){ + pDest->nReserve = pSrc->nReserve; + pagerReportSize(pDest); + } +} +#endif + /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -46008,7 +52403,7 @@ static int pager_playback_one_page( } } - /* If this page has already been played by before during the current + /* If this page has already been played back before during the current ** rollback, then don't bother to play it back again. */ if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ @@ -46060,10 +52455,10 @@ static int pager_playback_one_page( if( pagerUseWal(pPager) ){ pPg = 0; }else{ - pPg = pager_lookup(pPager, pgno); + pPg = sqlite3PagerLookup(pPager, pgno); } assert( pPg || !MEMDB ); - assert( pPager->eState!=PAGER_OPEN || pPg==0 ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -46085,9 +52480,9 @@ static int pager_playback_one_page( pPager->dbFileSize = pgno; } if( pPager->pBackup ){ - CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM); + CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); - CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData); + CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT, aData); } }else if( !isMainJrnl && pPg==0 ){ /* If this is a rollback of a savepoint and data was not written to @@ -46109,11 +52504,10 @@ static int pager_playback_one_page( assert( isSavepnt ); assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)==0 ); pPager->doNotSpill |= SPILLFLAG_ROLLBACK; - rc = sqlite3PagerAcquire(pPager, pgno, &pPg, 1); + rc = sqlite3PagerGet(pPager, pgno, &pPg, 1); assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 ); pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK; if( rc!=SQLITE_OK ) return rc; - pPg->flags &= ~PGHDR_NEED_READ; sqlite3PcacheMakeDirty(pPg); } if( pPg ){ @@ -46127,29 +52521,10 @@ static int pager_playback_one_page( pData = pPg->pData; memcpy(pData, (u8*)aData, pPager->pageSize); pPager->xReiniter(pPg); - if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){ - /* If the contents of this page were just restored from the main - ** journal file, then its content must be as they were when the - ** transaction was first opened. In this case we can mark the page - ** as clean, since there will be no need to write it out to the - ** database. - ** - ** There is one exception to this rule. If the page is being rolled - ** back as part of a savepoint (or statement) rollback from an - ** unsynced portion of the main journal file, then it is not safe - ** to mark the page as clean. This is because marking the page as - ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is - ** already in the journal file (recorded in Pager.pInJournal) and - ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to - ** again within this transaction, it will be marked as dirty but - ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially - ** be written out into the database file before its journal file - ** segment is synced. If a crash occurs during or following this, - ** database corruption may ensue. - */ - assert( !pagerUseWal(pPager) ); - sqlite3PcacheMakeClean(pPg); - } + /* It used to be that sqlite3PcacheMakeClean(pPg) was called here. But + ** that call was dangerous and had no detectable benefit since the cache + ** is normally cleaned by sqlite3PcacheCleanAll() after rollback and so + ** has been removed. */ pager_set_pagehash(pPg); /* If this was page 1, then restore the value of Pager.dbFileVers. @@ -46159,7 +52534,7 @@ static int pager_playback_one_page( } /* Decode the page just read from disk */ - CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM); + CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); sqlite3PcacheRelease(pPg); } return rc; @@ -46225,7 +52600,7 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); if( !pMaster ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); @@ -46240,9 +52615,9 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ rc = sqlite3OsFileSize(pMaster, &nMasterJournal); if( rc!=SQLITE_OK ) goto delmaster_out; nMasterPtr = pVfs->mxPathname+1; - zMasterJournal = sqlite3Malloc((int)nMasterJournal + nMasterPtr + 1); + zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1); if( !zMasterJournal ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto delmaster_out; } zMasterPtr = &zMasterJournal[nMasterJournal+1]; @@ -46309,7 +52684,7 @@ delmaster_out: ** If the file on disk is currently larger than nPage pages, then use the VFS ** xTruncate() method to truncate it. ** -** Or, it might might be the case that the file on disk is smaller than +** Or, it might be the case that the file on disk is smaller than ** nPage pages. Some operating system implementations can get confused if ** you try to truncate a file to some size that is larger than it ** currently is, so detect this case and write a single zero byte to @@ -46368,7 +52743,7 @@ SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *pFile){ /* ** Set the value of the Pager.sectorSize variable for the given ** pager based on the value returned by the xSectorSize method -** of the open database file. The sector size will be used used +** of the open database file. The sector size will be used ** to determine the size and alignment of journal header and ** master journal pointers within created journal files. ** @@ -46490,7 +52865,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** TODO: Technically the following is an error because it assumes that ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, - ** mxPathname is 512, which is the same as the minimum allowable value + ** mxPathname is 512, which is the same as the minimum allowable value ** for pageSize. */ zMaster = pPager->pTmpSpace; @@ -46703,7 +53078,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ ** ** For an encrypted database, the situation is more complex: bytes ** 24..39 of the database are white noise. But the probability of - ** white noising equaling 16 bytes of 0xff is vanishingly small so + ** white noise equaling 16 bytes of 0xff is vanishingly small so ** we should still be ok. */ memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); @@ -46712,7 +53087,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); } } - CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); + CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM_BKPT); PAGER_INCR(sqlite3_pager_readdb_count); PAGER_INCR(pPager->nRead); @@ -46837,9 +53212,7 @@ static int pagerWalFrames( ){ int rc; /* Return code */ int nList; /* Number of pages in pList */ -#if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES) PgHdr *p; /* For looping over pages */ -#endif assert( pPager->pWal ); assert( pList ); @@ -46856,7 +53229,6 @@ static int pagerWalFrames( ** any pages with page numbers greater than nTruncate into the WAL file. ** They will never be read by any client. So remove them from the pDirty ** list here. */ - PgHdr *p; PgHdr **ppNext = &pList; nList = 0; for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ @@ -46876,7 +53248,6 @@ static int pagerWalFrames( pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags ); if( rc==SQLITE_OK && pPager->pBackup ){ - PgHdr *p; for(p=pList; p; p=p->pDirty){ sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); } @@ -46944,22 +53315,20 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->eLock>=SHARED_LOCK ); + assert( isOpen(pPager->fd) ); + assert( pPager->tempFile==0 ); nPage = sqlite3WalDbsize(pPager->pWal); - /* If the database size was not available from the WAL sub-system, - ** determine it based on the size of the database file. If the size - ** of the database file is not an integer multiple of the page-size, - ** round down to the nearest page. Except, any file larger than 0 - ** bytes in size is considered to contain at least one page. + /* If the number of pages in the database is not available from the + ** WAL sub-system, determine the page counte based on the size of + ** the database file. If the size of the database file is not an + ** integer multiple of the page-size, round up the result. */ - if( nPage==0 ){ + if( nPage==0 && ALWAYS(isOpen(pPager->fd)) ){ i64 n = 0; /* Size of db file in bytes */ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - int rc = sqlite3OsFileSize(pPager->fd, &n); - if( rc!=SQLITE_OK ){ - return rc; - } + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; } nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); } @@ -47077,7 +53446,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ if( pSavepoint ){ pDone = sqlite3BitvecCreate(pSavepoint->nOrig); if( !pDone ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } @@ -47173,12 +53542,21 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ } /* -** Change the maximum number of in-memory pages that are allowed. +** Change the maximum number of in-memory pages that are allowed +** before attempting to recycle clean and unused pages. */ SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } +/* +** Change the maximum number of in-memory pages that are allowed +** before attempting to spill pages to journal. +*/ +SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager *pPager, int mxPage){ + return sqlite3PcacheSetSpillsize(pPager->pPCache, mxPage); +} + /* ** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. */ @@ -47215,7 +53593,7 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** The "level" in pgFlags & PAGER_SYNCHRONOUS_MASK sets the robustness ** of the database to damage due to OS crashes or power failures by ** changing the number of syncs()s when writing the journals. -** There are three levels: +** There are four levels: ** ** OFF sqlite3OsSync() is never called. This is the default ** for temporary and transient files. @@ -47235,6 +53613,10 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** assurance that the journal will not be corrupted to the ** point of causing damage to the database during rollback. ** +** EXTRA This is like FULL except that is also syncs the directory +** that contains the rollback journal after the rollback +** journal is unlinked. +** ** The above is for a rollback-journal mode. For WAL mode, OFF continues ** to mean that no syncs ever occur. NORMAL means that the WAL is synced ** prior to the start of checkpoint and that the database file is synced @@ -47242,7 +53624,8 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** was written back into the database. But no sync operations occur for ** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL ** file is synced following each commit operation, in addition to the -** syncs associated with NORMAL. +** syncs associated with NORMAL. There is no difference between FULL +** and EXTRA for WAL mode. ** ** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The ** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync @@ -47261,9 +53644,15 @@ SQLITE_PRIVATE void sqlite3PagerSetFlags( unsigned pgFlags /* Various flags */ ){ unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; - assert( level>=1 && level<=3 ); - pPager->noSync = (level==1 || pPager->tempFile) ?1:0; - pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; + if( pPager->tempFile ){ + pPager->noSync = 1; + pPager->fullSync = 0; + pPager->extraSync = 0; + }else{ + pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; + pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; + pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; + } if( pPager->noSync ){ pPager->syncFlags = 0; pPager->ckptSyncFlags = 0; @@ -47425,16 +53814,20 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR } if( rc==SQLITE_OK ){ pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ) rc = SQLITE_NOMEM; + if( !pNew ) rc = SQLITE_NOMEM_BKPT; } if( rc==SQLITE_OK ){ pager_reset(pPager); - pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); - pPager->pageSize = pageSize; + rc = sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + } + if( rc==SQLITE_OK ){ sqlite3PageFree(pPager->pTmpSpace); pPager->pTmpSpace = pNew; - sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); + pPager->pageSize = pageSize; + }else{ + sqlite3PageFree(pNew); } } @@ -47568,7 +53961,7 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* Return code */ /* Check that this is either a no-op (because the requested lock is - ** already held, or one of the transistions that the busy-handler + ** already held), or one of the transitions that the busy-handler ** may be invoked during, according to the comment above ** sqlite3PagerSetBusyhandler(). */ @@ -47687,7 +54080,7 @@ static int pagerAcquireMapPage( PgHdr **ppPage /* OUT: Acquired page object */ ){ PgHdr *p; /* Memory mapped page to return */ - + if( pPager->pMmapFreelist ){ *ppPage = p = pPager->pMmapFreelist; pPager->pMmapFreelist = p->pDirty; @@ -47697,7 +54090,7 @@ static int pagerAcquireMapPage( *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } p->pExtra = (void *)&p[1]; p->flags = PGHDR_MMAP; @@ -48011,8 +54404,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); + assert( isOpen(pPager->fd) || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -48055,7 +54449,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pList->pgno==1 ) pager_write_changecounter(pList); /* Encode the database */ - CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); + CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM_BKPT, pData); /* Write out the page data. */ rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); @@ -48100,19 +54494,20 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; if( !isOpen(pPager->sjfd) ){ + const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE + | SQLITE_OPEN_DELETEONCLOSE; + int nStmtSpill = sqlite3Config.nStmtSpill; if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ - sqlite3MemJournalOpen(pPager->sjfd); - }else{ - rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL); + nStmtSpill = -1; } + rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nStmtSpill); } return rc; } /* ** Append a record of the current state of page pPg to the sub-journal. -** It is the callers responsibility to use subjRequiresPage() to check -** that it is really required before calling this function. ** ** If successful, set the bit corresponding to pPg->pgno in the bitvecs ** for all open savepoints before returning. @@ -48144,7 +54539,7 @@ static int subjournalPage(PgHdr *pPg){ i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); char *pData2; - CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); rc = write32bits(pPager->sjfd, offset, pPg->pgno); if( rc==SQLITE_OK ){ @@ -48159,6 +54554,13 @@ static int subjournalPage(PgHdr *pPg){ } return rc; } +static int subjournalPageIfRequired(PgHdr *pPg){ + if( subjRequiresPage(pPg) ){ + return subjournalPage(pPg); + }else{ + return SQLITE_OK; + } +} /* ** This function is called by the pcache layer when it has reached some @@ -48196,8 +54598,8 @@ static int pagerStress(void *p, PgHdr *pPg){ ** a rollback or by user request, respectively. ** ** Spilling is also prohibited when in an error state since that could - ** lead to database corruption. In the current implementaton it - ** is impossible for sqlite3PcacheFetch() to be called with createFlag==1 + ** lead to database corruption. In the current implementation it + ** is impossible for sqlite3PcacheFetch() to be called with createFlag==3 ** while in the error state, hence it is impossible for this routine to ** be called in the error state. Nevertheless, we include a NEVER() ** test for the error state as a safeguard against future changes. @@ -48216,9 +54618,7 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseWal(pPager) ){ /* Write a single frame for this page to the log. */ - if( subjRequiresPage(pPg) ){ - rc = subjournalPage(pPg); - } + rc = subjournalPageIfRequired(pPg); if( rc==SQLITE_OK ){ rc = pagerWalFrames(pPager, pPg, 0, 0); } @@ -48231,39 +54631,6 @@ static int pagerStress(void *p, PgHdr *pPg){ rc = syncJournal(pPager, 1); } - /* If the page number of this page is larger than the current size of - ** the database image, it may need to be written to the sub-journal. - ** This is because the call to pager_write_pagelist() below will not - ** actually write data to the file in this case. - ** - ** Consider the following sequence of events: - ** - ** BEGIN; - ** - ** - ** SAVEPOINT sp; - ** - ** pagerStress(page X) - ** ROLLBACK TO sp; - ** - ** If (X>Y), then when pagerStress is called page X will not be written - ** out to the database file, but will be dropped from the cache. Then, - ** following the "ROLLBACK TO sp" statement, reading page X will read - ** data from the database file. This will be the copy of page X as it - ** was when the transaction started, not as it was when "SAVEPOINT sp" - ** was executed. - ** - ** The solution is to write the current data for page X into the - ** sub-journal file now (if it is not already there), so that it will - ** be restored to its current value when the "ROLLBACK TO sp" is - ** executed. - */ - if( NEVER( - rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) - ) ){ - rc = subjournalPage(pPg); - } - /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); @@ -48280,6 +54647,25 @@ static int pagerStress(void *p, PgHdr *pPg){ return pager_error(pPager, rc); } +/* +** Flush all unreferenced dirty pages to disk. +*/ +SQLITE_PRIVATE int sqlite3PagerFlush(Pager *pPager){ + int rc = pPager->errCode; + if( !MEMDB ){ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + assert( assert_pager_state(pPager) ); + while( rc==SQLITE_OK && pList ){ + PgHdr *pNext = pList->pDirty; + if( pList->nRef==0 ){ + rc = pagerStress((void*)pPager, pList); + } + pList = pNext; + } + } + + return rc; +} /* ** Allocate and initialize a new Pager object and put a pointer to it @@ -48336,18 +54722,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int nUri = 0; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle - ** (there are two of them, the main journal and the sub-journal). This - ** is the maximum space required for an in-memory journal file handle - ** and a regular journal file-handle. Note that a "regular journal-handle" - ** may be a wrapper capable of caching the first portion of the journal - ** file in memory to implement the atomic-write optimization (see - ** source file journal.c). - */ - if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){ - journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); - }else{ - journalFileSize = ROUND8(sqlite3MemJournalSize()); - } + ** (there are two of them, the main journal and the sub-journal). */ + journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); /* Set the output variable to NULL in case an error occurs. */ *ppPager = 0; @@ -48357,7 +54733,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( memDb = 1; if( zFilename && zFilename[0] ){ zPathname = sqlite3DbStrDup(0, zFilename); - if( zPathname==0 ) return SQLITE_NOMEM; + if( zPathname==0 ) return SQLITE_NOMEM_BKPT; nPathname = sqlite3Strlen30(zPathname); zFilename = 0; } @@ -48373,7 +54749,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( nPathname = pVfs->mxPathname+1; zPathname = sqlite3DbMallocRaw(0, nPathname*2); if( zPathname==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); @@ -48426,7 +54802,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); if( !pPtr ){ sqlite3DbFree(0, zPathname); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pPager = (Pager*)(pPtr); pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); @@ -48519,7 +54895,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( act_like_temp_file: tempFile = 1; pPager->eState = PAGER_READER; /* Pretend we already have a lock */ - pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE locking mode */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE mode */ pPager->noLock = 1; /* Do no locking */ readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } @@ -48533,22 +54909,23 @@ act_like_temp_file: testcase( rc!=SQLITE_OK ); } - /* If an error occurred in either of the blocks above, free the - ** Pager structure and close the file. + /* Initialize the PCache object. */ + if( rc==SQLITE_OK ){ + assert( nExtra<1000 ); + nExtra = ROUND8(nExtra); + rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + } + + /* If an error occurred above, free the Pager structure and close the file. */ if( rc!=SQLITE_OK ){ - assert( !pPager->pTmpSpace ); sqlite3OsClose(pPager->fd); + sqlite3PageFree(pPager->pTmpSpace); sqlite3_free(pPager); return rc; } - /* Initialize the PCache object. */ - assert( nExtra<1000 ); - nExtra = ROUND8(nExtra); - sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, - !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); - PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename)); IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) @@ -48574,11 +54951,13 @@ act_like_temp_file: pPager->noSync = pPager->tempFile; if( pPager->noSync ){ assert( pPager->fullSync==0 ); + assert( pPager->extraSync==0 ); assert( pPager->syncFlags==0 ); assert( pPager->walSyncFlags==0 ); assert( pPager->ckptSyncFlags==0 ); }else{ pPager->fullSync = 1; + pPager->extraSync = 0; pPager->syncFlags = SQLITE_SYNC_NORMAL; pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; @@ -48695,6 +55074,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ + assert( pPager->tempFile==0 ); rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ /* If the database is zero pages in size, that means that either (1) the @@ -48735,7 +55115,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ *pExists = (first!=0); }else if( rc==SQLITE_CANTOPEN ){ /* If we cannot open the rollback journal file in order to see if - ** its has a zero header, that might be due to an I/O error, or + ** it has a zero header, that might be due to an I/O error, or ** it might be due to the race condition described above and in ** ticket #3883. Either way, assume that the journal is hot. ** This might be a false positive. But if it is, then the @@ -48756,7 +55136,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ /* ** This function is called to obtain a shared lock on the database file. -** It is illegal to call sqlite3PagerAcquire() until after this function +** It is illegal to call sqlite3PagerGet() until after this function ** has been successfully called. If a shared-lock is already held when ** this function is called, it is a no-op. ** @@ -48787,17 +55167,17 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either ** be OPEN or READER. READER is only possible if the pager is or was in - ** exclusive access mode. - */ + ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); - if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } + assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB ); + assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ @@ -48883,7 +55263,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ - rc = pager_playback(pPager, 1); + rc = pager_playback(pPager, !pPager->tempFile); pPager->eState = PAGER_OPEN; } }else if( !pPager->exclusiveMode ){ @@ -48917,18 +55297,14 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile && ( - pPager->pBackup - || sqlite3PcachePagecount(pPager->pPCache)>0 - || USEFETCH(pPager) - )){ - /* The shared-lock has just been acquired on the database file - ** and there are already pages in the cache (from a previous - ** read or write transaction). Check to see if the database - ** has been modified. If the database has changed, flush the - ** cache. + if( !pPager->tempFile && pPager->hasHeldSharedLock ){ + /* The shared-lock has just been acquired then check to + ** see if the database has been modified. If the database has changed, + ** flush the cache. The hasHeldSharedLock flag prevents this from + ** occurring on the very first access to a file, in order to save a + ** single unnecessary sqlite3OsRead() call at the start-up. ** - ** Database changes is detected by looking at 15 bytes beginning + ** Database changes are detected by looking at 15 bytes beginning ** at offset 24 into the file. The first 4 of these 16 bytes are ** a 32-bit counter that is incremented with each change. The ** other bytes change randomly with each file change when @@ -48983,7 +55359,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerBeginReadTransaction(pPager); } - if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ rc = pagerPagecount(pPager, &pPager->dbSize); } @@ -48994,6 +55370,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ assert( pPager->eState==PAGER_OPEN ); }else{ pPager->eState = PAGER_READER; + pPager->hasHeldSharedLock = 1; } return rc; } @@ -49062,7 +55439,7 @@ static void pagerUnlockIfUnused(Pager *pPager){ ** Since Lookup() never goes to disk, it never has to deal with locks ** or journal files. */ -SQLITE_PRIVATE int sqlite3PagerAcquire( +SQLITE_PRIVATE int sqlite3PagerGet( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ @@ -49077,27 +55454,31 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + const int bMmapOk = (pgno>1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) #ifdef SQLITE_HAS_CODEC && pPager->xCodec==0 #endif ); + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); assert( noContent==0 || bMmapOk==0 ); - if( pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } + assert( pPager->hasHeldSharedLock==1 ); /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ - if( bMmapOk && pagerUseWal(pPager) ){ rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); if( rc!=SQLITE_OK ) goto pager_acquire_err; @@ -49111,8 +55492,8 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( ); if( rc==SQLITE_OK && pData ){ - if( pPager->eState>PAGER_READER ){ - (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + if( pPager->eState>PAGER_READER || pPager->tempFile ){ + pPg = sqlite3PagerLookup(pPager, pgno); } if( pPg==0 ){ rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); @@ -49130,7 +55511,21 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( } } - rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); + { + sqlite3_pcache_page *pBase; + pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); + if( pBase==0 ){ + rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pBase==0 ){ + pPg = *ppPage = 0; + rc = SQLITE_NOMEM_BKPT; + goto pager_acquire_err; + } + } + pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); + assert( pPg!=0 ); + } } if( rc!=SQLITE_OK ){ @@ -49140,10 +55535,11 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( pPg = 0; goto pager_acquire_err; } - assert( (*ppPage)->pgno==pgno ); - assert( (*ppPage)->pPager==pPager || (*ppPage)->pPager==0 ); + assert( pPg==(*ppPage) ); + assert( pPg->pgno==pgno ); + assert( pPg->pPager==pPager || pPg->pPager==0 ); - if( (*ppPage)->pPager && !noContent ){ + if( pPg->pPager && !noContent ){ /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); @@ -49154,7 +55550,6 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( /* The pager cache has created a new page. Its content needs to ** be initialized. */ - pPg = *ppPage; pPg->pPager = pPager; /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page @@ -49164,7 +55559,8 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( goto pager_acquire_err; } - if( MEMDB || pPager->dbSizefd) ){ + assert( !isOpen(pPager->fd) || !MEMDB ); + if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -49227,13 +55623,14 @@ pager_acquire_err: ** has ever happened. */ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ - PgHdr *pPg = 0; + sqlite3_pcache_page *pPage; assert( pPager!=0 ); assert( pgno!=0 ); assert( pPager->pPCache!=0 ); - assert( pPager->eState>=PAGER_READER && pPager->eState!=PAGER_ERROR ); - sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); - return pPg; + pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); + assert( pPage==0 || pPager->hasHeldSharedLock ); + if( pPage==0 ) return 0; + return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } /* @@ -49297,7 +55694,7 @@ static int pager_open_journal(Pager *pPager){ if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); if( pPager->pInJournal==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* Open the journal file if it is not already open. */ @@ -49305,24 +55702,24 @@ static int pager_open_journal(Pager *pPager){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3MemJournalOpen(pPager->jfd); }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); + int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + int nSpill; + if( pPager->tempFile ){ + flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + nSpill = sqlite3Config.nStmtSpill; + }else{ + flags |= SQLITE_OPEN_MAIN_JOURNAL; + nSpill = jrnlBufferSize(pPager); + } + /* Verify that the database still has the same name as it did when ** it was originally opened. */ rc = databaseIsUnmoved(pPager); if( rc==SQLITE_OK ){ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + rc = sqlite3JournalOpen ( + pVfs, pPager->zJournal, pPager->jfd, flags, nSpill ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif } } assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); @@ -49389,7 +55786,7 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory if( rc!=SQLITE_OK ){ return rc; } - sqlite3WalExclusiveMode(pPager->pWal, 1); + (void)sqlite3WalExclusiveMode(pPager->pWal, 1); } /* Grab the write lock on the log file. If successful, upgrade to @@ -49436,6 +55833,59 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory return rc; } +/* +** Write page pPg onto the end of the rollback journal. +*/ +static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + int rc; + u32 cksum; + char *pData2; + i64 iOff = pPager->journalOff; + + /* We should never write to the journal file the page that + ** contains the database locks. The following assert verifies + ** that we do not. */ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + + assert( pPager->journalHdr<=pPager->journalOff ); + CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); + cksum = pager_cksum(pPager, (u8*)pData2); + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + + pPager->journalOff += 8 + pPager->pageSize; + pPager->nRec++; + assert( pPager->pInJournal!=0 ); + rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + rc |= addToSavepointBitvecs(pPager, pPg->pgno); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + return rc; +} + /* ** Mark a single data page as writeable. The page is written into the ** main journal or sub-journal as required. If the page is written into @@ -49446,7 +55896,6 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory static int pager_write(PgHdr *pPg){ Pager *pPager = pPg->pPager; int rc = SQLITE_OK; - int inJournal; /* This routine is not called unless a write-transaction has already ** been started. The journal file may or may not be open at this point. @@ -49459,7 +55908,6 @@ static int pager_write(PgHdr *pPg){ assert( assert_pager_state(pPager) ); assert( pPager->errCode==0 ); assert( pPager->readOnly==0 ); - CHECK_PAGE(pPg); /* The journal file needs to be opened. Higher level routines have already @@ -49478,97 +55926,145 @@ static int pager_write(PgHdr *pPg){ assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); assert( assert_pager_state(pPager) ); - /* Mark the page as dirty. If the page has already been written - ** to the journal then we can return right away. - */ + /* Mark the page that is about to be modified as dirty. */ sqlite3PcacheMakeDirty(pPg); - inJournal = pageInJournal(pPager, pPg); - if( inJournal && (pPager->nSavepoint==0 || !subjRequiresPage(pPg)) ){ - assert( !pagerUseWal(pPager) ); - }else{ - - /* The transaction journal now exists and we have a RESERVED or an - ** EXCLUSIVE lock on the main database file. Write the current page to - ** the transaction journal if it is not there already. - */ - if( !inJournal && !pagerUseWal(pPager) ){ - assert( pagerUseWal(pPager)==0 ); - if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ - u32 cksum; - char *pData2; - i64 iOff = pPager->journalOff; - /* We should never write to the journal file the page that - ** contains the database locks. The following assert verifies - ** that we do not. */ - assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - - assert( pPager->journalHdr<=pPager->journalOff ); - CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); - cksum = pager_cksum(pPager, (u8*)pData2); - - /* Even if an IO or diskfull error occurs while journalling the - ** page in the block above, set the need-sync flag for the page. - ** Otherwise, when the transaction is rolled back, the logic in - ** playback_one_page() will think that the page needs to be restored - ** in the database file. And if an IO error occurs while doing so, - ** then corruption may follow. - */ - pPg->flags |= PGHDR_NEED_SYNC; - - rc = write32bits(pPager->jfd, iOff, pPg->pgno); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); - if( rc!=SQLITE_OK ) return rc; - rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); - if( rc!=SQLITE_OK ) return rc; - - IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, - pPager->journalOff, pPager->pageSize)); - PAGER_INCR(sqlite3_pager_writej_count); - PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); - - pPager->journalOff += 8 + pPager->pageSize; - pPager->nRec++; - assert( pPager->pInJournal!=0 ); - rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); - testcase( rc==SQLITE_NOMEM ); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - rc |= addToSavepointBitvecs(pPager, pPg->pgno); - if( rc!=SQLITE_OK ){ - assert( rc==SQLITE_NOMEM ); - return rc; - } - }else{ - if( pPager->eState!=PAGER_WRITER_DBMOD ){ - pPg->flags |= PGHDR_NEED_SYNC; - } - PAGERTRACE(("APPEND %d page %d needSync=%d\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); + /* If a rollback journal is in use, them make sure the page that is about + ** to change is in the rollback journal, or if the page is a new page off + ** then end of the file, make sure it is marked as PGHDR_NEED_SYNC. + */ + assert( (pPager->pInJournal!=0) == isOpen(pPager->jfd) ); + if( pPager->pInJournal!=0 + && sqlite3BitvecTestNotNull(pPager->pInJournal, pPg->pgno)==0 + ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize ){ + rc = pagerAddPageToRollbackJournal(pPg); + if( rc!=SQLITE_OK ){ + return rc; } - } - - /* If the statement journal is open and the page is not in it, - ** then write the current page to the statement journal. Note that - ** the statement journal format differs from the standard journal format - ** in that it omits the checksums and the header. - */ - if( pPager->nSavepoint>0 && subjRequiresPage(pPg) ){ - rc = subjournalPage(pPg); + }else{ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ + pPg->flags |= PGHDR_NEED_SYNC; + } + PAGERTRACE(("APPEND %d page %d needSync=%d\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); } } - /* Update the database size and return. + /* The PGHDR_DIRTY bit is set above when the page was added to the dirty-list + ** and before writing the page into the rollback journal. Wait until now, + ** after the page has been successfully journalled, before setting the + ** PGHDR_WRITEABLE bit that indicates that the page can be safely modified. */ + pPg->flags |= PGHDR_WRITEABLE; + + /* If the statement journal is open and the page is not in it, + ** then write the page into the statement journal. + */ + if( pPager->nSavepoint>0 ){ + rc = subjournalPageIfRequired(pPg); + } + + /* Update the database size and return. */ if( pPager->dbSizepgno ){ pPager->dbSize = pPg->pgno; } return rc; } +/* +** This is a variant of sqlite3PagerWrite() that runs when the sector size +** is larger than the page size. SQLite makes the (reasonable) assumption that +** all bytes of a sector are written together by hardware. Hence, all bytes of +** a sector need to be journalled in case of a power loss in the middle of +** a write. +** +** Usually, the sector size is less than or equal to the page size, in which +** case pages can be individually written. This routine only runs in the +** exceptional case where the page size is smaller than the sector size. +*/ +static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ + int rc = SQLITE_OK; /* Return code */ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + Pager *pPager = pPg->pPager; /* The pager that owns pPg */ + Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + + /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow + ** a journal header to be written between the pages journaled by + ** this function. + */ + assert( !MEMDB ); + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)==0 ); + pPager->doNotSpill |= SPILLFLAG_NOSYNC; + + /* This trick assumes that both the page-size and sector-size are + ** an integer power of 2. It sets variable pg1 to the identifier + ** of the first page of the sector pPg is located on. + */ + pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; + + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; + } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); + + for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ + if( pg!=PAGER_MJ_PGNO(pPager) ){ + rc = sqlite3PagerGet(pPager, pg, &pPage, 0); + if( rc==SQLITE_OK ){ + rc = pager_write(pPage); + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + }else if( (pPage = sqlite3PagerLookup(pPager, pg))!=0 ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + ** starting at pg1, then it needs to be set for all of them. Because + ** writing to any of these nPage pages may damage the others, the + ** journal file must contain sync()ed copies of all of them + ** before any of them can be written out to the database file. + */ + if( rc==SQLITE_OK && needSync ){ + assert( !MEMDB ); + for(ii=0; iiflags |= PGHDR_NEED_SYNC; + sqlite3PagerUnrefNotNull(pPage); + } + } + } + + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)!=0 ); + pPager->doNotSpill &= ~SPILLFLAG_NOSYNC; + return rc; +} + /* ** Mark a data page as writeable. This routine must be called before ** making changes to a page. The caller must check the return value @@ -49583,96 +56079,22 @@ static int pager_write(PgHdr *pPg){ ** If an error occurs, SQLITE_NOMEM or an IO error code is returned ** as appropriate. Otherwise, SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3PagerWrite(DbPage *pDbPage){ - int rc = SQLITE_OK; - - PgHdr *pPg = pDbPage; +SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ Pager *pPager = pPg->pPager; - assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); - - if( pPager->sectorSize > (u32)pPager->pageSize ){ - Pgno nPageCount; /* Total number of pages in database file */ - Pgno pg1; /* First page of the sector pPg is located on. */ - int nPage = 0; /* Number of pages starting at pg1 to journal */ - int ii; /* Loop counter */ - int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ - Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); - - /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow - ** a journal header to be written between the pages journaled by - ** this function. - */ - assert( !MEMDB ); - assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)==0 ); - pPager->doNotSpill |= SPILLFLAG_NOSYNC; - - /* This trick assumes that both the page-size and sector-size are - ** an integer power of 2. It sets variable pg1 to the identifier - ** of the first page of the sector pPg is located on. - */ - pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; - - nPageCount = pPager->dbSize; - if( pPg->pgno>nPageCount ){ - nPage = (pPg->pgno - pg1)+1; - }else if( (pg1+nPagePerSector-1)>nPageCount ){ - nPage = nPageCount+1-pg1; - }else{ - nPage = nPagePerSector; - } - assert(nPage>0); - assert(pg1<=pPg->pgno); - assert((pg1+nPage)>pPg->pgno); - - for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ - if( pg!=PAGER_MJ_PGNO(pPager) ){ - rc = sqlite3PagerGet(pPager, pg, &pPage); - if( rc==SQLITE_OK ){ - rc = pager_write(pPage); - if( pPage->flags&PGHDR_NEED_SYNC ){ - needSync = 1; - } - sqlite3PagerUnrefNotNull(pPage); - } - } - }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ - if( pPage->flags&PGHDR_NEED_SYNC ){ - needSync = 1; - } - sqlite3PagerUnrefNotNull(pPage); - } - } - - /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages - ** starting at pg1, then it needs to be set for all of them. Because - ** writing to any of these nPage pages may damage the others, the - ** journal file must contain sync()ed copies of all of them - ** before any of them can be written out to the database file. - */ - if( rc==SQLITE_OK && needSync ){ - assert( !MEMDB ); - for(ii=0; iiflags |= PGHDR_NEED_SYNC; - sqlite3PagerUnrefNotNull(pPage); - } - } - } - - assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)!=0 ); - pPager->doNotSpill &= ~SPILLFLAG_NOSYNC; + if( pPager->errCode ){ + return pPager->errCode; + }else if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); + return SQLITE_OK; + }else if( pPager->sectorSize > (u32)pPager->pageSize ){ + assert( pPager->tempFile==0 ); + return pagerWriteLargeSector(pPg); }else{ - rc = pager_write(pDbPage); + return pager_write(pPg); } - return rc; } /* @@ -49682,7 +56104,7 @@ SQLITE_PRIVATE int sqlite3PagerWrite(DbPage *pDbPage){ */ #ifndef NDEBUG SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ - return pPg->flags&PGHDR_DIRTY; + return pPg->flags & PGHDR_WRITEABLE; } #endif @@ -49699,13 +56121,21 @@ SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ ** ** Tests show that this optimization can quadruple the speed of large ** DELETE operations. +** +** This optimization cannot be used with a temp-file, as the page may +** have been dirty at the start of the transaction. In that case, if +** memory pressure forces page pPg out of the cache, the data does need +** to be written out to disk so that it may be read back in if the +** current transaction is rolled back. */ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ Pager *pPager = pPg->pPager; - if( (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ + if( !pPager->tempFile && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; + pPg->flags &= ~PGHDR_WRITEABLE; + testcase( pPg->flags & PGHDR_NEED_SYNC ); pager_set_pagehash(pPg); } } @@ -49764,7 +56194,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ assert( !pPager->tempFile && isOpen(pPager->fd) ); /* Open page 1 of the file for writing. */ - rc = sqlite3PagerGet(pPager, 1, &pPgHdr); + rc = sqlite3PagerGet(pPager, 1, &pPgHdr, 0); assert( pPgHdr==0 || rc==SQLITE_OK ); /* If page one was fetched successfully, and this function is not @@ -49784,7 +56214,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ if( DIRECT_MODE ){ const void *zBuf; assert( pPager->dbFileSize>0 ); - CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); + CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM_BKPT, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); pPager->aStat[PAGER_STAT_WRITE]++; @@ -49842,14 +56272,17 @@ SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** returned. */ SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ - int rc = SQLITE_OK; - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_LOCKED - ); + int rc = pPager->errCode; assert( assert_pager_state(pPager) ); - if( 0==pagerUseWal(pPager) ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + if( rc==SQLITE_OK ){ + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } } return rc; } @@ -49897,17 +56330,21 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( /* If a prior error occurred, report that error again. */ if( NEVER(pPager->errCode) ) return pPager->errCode; + /* Provide the ability to easily simulate an I/O error during testing */ + if( sqlite3FaultSim(400) ) return SQLITE_IOERR; + PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); /* If no database changes have been made, return early. */ if( pPager->eStatetempFile ); + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( 0==pagerFlushOnCommit(pPager, 1) ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any - ** backup in progress needs to be restarted. - */ + ** backup in progress needs to be restarted. */ sqlite3BackupRestart(pPager->pBackup); }else{ if( pagerUseWal(pPager) ){ @@ -49916,7 +56353,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( if( pList==0 ){ /* Must have at least one page for the WAL commit flag. ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ - rc = sqlite3PagerGet(pPager, 1, &pPageOne); + rc = sqlite3PagerGet(pPager, 1, &pPageOne, 0); pList = pPageOne; pList->pDirty = 0; } @@ -50088,6 +56525,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + pPager->iDataVersion++; rc = pager_end_transaction(pPager, pPager->setMaster, 1); return pager_error(pPager, rc); } @@ -50171,12 +56609,14 @@ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager *pPager){ return pPager->readOnly; } +#ifdef SQLITE_DEBUG /* -** Return the number of references to the pager. +** Return the sum of the reference counts for all pages held by pPager. */ SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ return sqlite3PcacheRefCount(pPager->pPCache); } +#endif /* ** Return the approximate number of bytes of memory currently @@ -50243,10 +56683,10 @@ SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, i } /* -** Return true if this is an in-memory pager. +** Return true if this is an in-memory or temp-file backed pager. */ SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ - return MEMDB; + return pPager->tempFile; } /* @@ -50259,54 +56699,62 @@ SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ ** occurs while opening the sub-journal file, then an IO error code is ** returned. Otherwise, SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ +static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ int rc = SQLITE_OK; /* Return code */ int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + int ii; /* Iterator variable */ + PagerSavepoint *aNew; /* New Pager.aSavepoint array */ assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); + assert( nSavepoint>nCurrent && pPager->useJournal ); - if( nSavepoint>nCurrent && pPager->useJournal ){ - int ii; /* Iterator variable */ - PagerSavepoint *aNew; /* New Pager.aSavepoint array */ - - /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM - ** if the allocation fails. Otherwise, zero the new portion in case a - ** malloc failure occurs while populating it in the for(...) loop below. - */ - aNew = (PagerSavepoint *)sqlite3Realloc( - pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint - ); - if( !aNew ){ - return SQLITE_NOMEM; - } - memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); - pPager->aSavepoint = aNew; - - /* Populate the PagerSavepoint structures just allocated. */ - for(ii=nCurrent; iidbSize; - if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ - aNew[ii].iOffset = pPager->journalOff; - }else{ - aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); - } - aNew[ii].iSubRec = pPager->nSubRec; - aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); - if( !aNew[ii].pInSavepoint ){ - return SQLITE_NOMEM; - } - if( pagerUseWal(pPager) ){ - sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); - } - pPager->nSavepoint = ii+1; - } - assert( pPager->nSavepoint==nSavepoint ); - assertTruncateConstraint(pPager); + /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM + ** if the allocation fails. Otherwise, zero the new portion in case a + ** malloc failure occurs while populating it in the for(...) loop below. + */ + aNew = (PagerSavepoint *)sqlite3Realloc( + pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint + ); + if( !aNew ){ + return SQLITE_NOMEM_BKPT; } + memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); + pPager->aSavepoint = aNew; + /* Populate the PagerSavepoint structures just allocated. */ + for(ii=nCurrent; iidbSize; + if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ + aNew[ii].iOffset = pPager->journalOff; + }else{ + aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); + } + aNew[ii].iSubRec = pPager->nSubRec; + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + if( !aNew[ii].pInSavepoint ){ + return SQLITE_NOMEM_BKPT; + } + if( pagerUseWal(pPager) ){ + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); + } + pPager->nSavepoint = ii+1; + } + assert( pPager->nSavepoint==nSavepoint ); + assertTruncateConstraint(pPager); return rc; } +SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + + if( nSavepoint>pPager->nSavepoint && pPager->useJournal ){ + return pagerOpenSavepoint(pPager, nSavepoint); + }else{ + return SQLITE_OK; + } +} + /* ** This function is called to rollback or release (commit) a savepoint. @@ -50339,7 +56787,11 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ ** savepoint. If no errors occur, SQLITE_OK is returned. */ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = pPager->errCode; /* Return code */ + int rc = pPager->errCode; + +#ifdef SQLITE_ENABLE_ZIPVFS + if( op==SAVEPOINT_RELEASE ) rc = SQLITE_OK; +#endif assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); @@ -50363,7 +56815,7 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ if( op==SAVEPOINT_RELEASE ){ if( nNew==0 && isOpen(pPager->sjfd) ){ /* Only truncate if it is an in-memory sub-journal. */ - if( sqlite3IsMemJournal(pPager->sjfd) ){ + if( sqlite3JournalIsInMemory(pPager->sjfd) ){ rc = sqlite3OsTruncate(pPager->sjfd, 0); assert( rc==SQLITE_OK ); } @@ -50380,6 +56832,20 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } + +#ifdef SQLITE_ENABLE_ZIPVFS + /* If the cache has been modified but the savepoint cannot be rolled + ** back journal_mode=off, put the pager in the error state. This way, + ** if the VFS used by this pager includes ZipVFS, the entire transaction + ** can be rolled back at the ZipVFS level. */ + else if( + pPager->journalMode==PAGER_JOURNALMODE_OFF + && pPager->eState>=PAGER_WRITER_CACHEMOD + ){ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + } +#endif } return rc; @@ -50402,7 +56868,7 @@ SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager, int nullIfMemDb){ /* ** Return the VFS structure for the pager. */ -SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ +SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ return pPager->pVfs; } @@ -50415,6 +56881,18 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ return pPager->fd; } +/* +** Return the file handle for the journal file (if it exists). +** This will be either the rollback journal or the WAL file. +*/ +SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ +#if SQLITE_OMIT_WAL + return pPager->jfd; +#else + return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; +#endif +} + /* ** Return the full pathname of the journal file. */ @@ -50422,14 +56900,6 @@ SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ return pPager->zJournal; } -/* -** Return true if fsync() calls are disabled for this pager. Return FALSE -** if fsync()s are executed normally. -*/ -SQLITE_PRIVATE int sqlite3PagerNosync(Pager *pPager){ - return pPager->noSync; -} - #ifdef SQLITE_HAS_CODEC /* ** Set or retrieve the codec for this pager @@ -50514,7 +56984,8 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. */ - if( MEMDB ){ + assert( pPager->tempFile || !MEMDB ); + if( pPager->tempFile ){ rc = sqlite3PagerWrite(pPg); if( rc ) return rc; } @@ -50537,9 +57008,8 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** one or more savepoint bitvecs. This is the reason this function ** may return SQLITE_NOMEM. */ - if( pPg->flags&PGHDR_DIRTY - && subjRequiresPage(pPg) - && SQLITE_OK!=(rc = subjournalPage(pPg)) + if( (pPg->flags & PGHDR_DIRTY)!=0 + && SQLITE_OK!=(rc = subjournalPageIfRequired(pPg)) ){ return rc; } @@ -50568,11 +57038,11 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** for the page moved there. */ pPg->flags &= ~PGHDR_NEED_SYNC; - pPgOld = pager_lookup(pPager, pgno); + pPgOld = sqlite3PagerLookup(pPager, pgno); assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); - if( MEMDB ){ + if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ sqlite3PcacheMove(pPgOld, pPager->dbSize+1); @@ -50589,8 +57059,7 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** to exist, in case the transaction needs to roll back. Use pPgOld ** as the original page since it has already been allocated. */ - if( MEMDB ){ - assert( pPgOld ); + if( pPager->tempFile && pPgOld ){ sqlite3PcacheMove(pPgOld, origPgno); sqlite3PagerUnrefNotNull(pPgOld); } @@ -50611,7 +57080,7 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** the journal file twice, but that is not a problem. */ PgHdr *pPgHdr; - rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); + rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr, 0); if( rc!=SQLITE_OK ){ if( needSyncPgno<=pPager->dbOrigSize ){ assert( pPager->pTmpSpace!=0 ); @@ -50628,6 +57097,18 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i } #endif +/* +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to +** the value passed as the third parameter. +*/ +SQLITE_PRIVATE void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ + assert( pPg->pgno!=iNew ); + pPg->flags = flags; + sqlite3PcacheMove(pPg, iNew); +} + /* ** Return a pointer to the data for the specified page. */ @@ -50773,6 +57254,8 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } assert( state==pPager->eState ); } + }else if( eMode==PAGER_JOURNALMODE_OFF ){ + sqlite3OsClose(pPager->jfd); } } @@ -50828,10 +57311,12 @@ SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ ** Unless this is an in-memory or temporary database, clear the pager cache. */ SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *pPager){ - if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); + assert( MEMDB==0 || pPager->tempFile ); + if( pPager->tempFile==0 ) pager_reset(pPager); } #endif + #ifndef SQLITE_OMIT_WAL /* ** This function is called when the user invokes "PRAGMA wal_checkpoint", @@ -50844,7 +57329,8 @@ SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int rc = SQLITE_OK; if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, - pPager->xBusyHandler, pPager->pBusyHandlerArg, + (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + pPager->pBusyHandlerArg, pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); @@ -50862,6 +57348,7 @@ SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager){ */ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ const sqlite3_io_methods *pMethods = pPager->fd->pMethods; + if( pPager->noLock ) return 0; return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); } @@ -51005,11 +57492,40 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); + if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } } return rc; } +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** If this is a WAL database, obtain a snapshot handle for the snapshot +** currently open. Otherwise, return an error. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot){ + int rc = SQLITE_ERROR; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotGet(pPager->pWal, ppSnapshot); + } + return rc; +} + +/* +** If this is a WAL database, store a pointer to pSnapshot. Next time a +** read transaction is opened, attempt to read from the snapshot it +** identifies. If this is not a WAL database, return an error. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot){ + int rc = SQLITE_OK; + if( pPager->pWal ){ + sqlite3WalSnapshotOpen(pPager->pWal, pSnapshot); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS @@ -51021,7 +57537,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ ** is empty, return 0. */ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ - assert( pPager->eState==PAGER_READER ); + assert( pPager->eState>=PAGER_READER ); return sqlite3WalFramesize(pPager->pWal); } #endif @@ -51030,11 +57546,11 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC -SQLITE_PRIVATE void sqlite3pager_get_codec(Pager *pPager, void **ctx) { +SQLITE_API void sqlite3pager_get_codec(Pager *pPager, void **ctx) { *ctx = pPager->pCodec; } -SQLITE_PRIVATE int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno) { +SQLITE_API int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno) { return (PAGER_MJ_PGNO(pPager) == pgno) ? 1 : 0; } @@ -51042,7 +57558,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3Pager_get_fd(Pager *pPager) { return (isOpen(pPager->fd)) ? pPager->fd : NULL; } -SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetCodec( +SQLITE_API void sqlite3pager_sqlite3PagerSetCodec( Pager *pPager, void *(*xCodec)(void*,void*,Pgno,int), void (*xCodecSizeChng)(void*,int,int), @@ -51052,7 +57568,7 @@ SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetCodec( sqlite3PagerSetCodec(pPager, xCodec, xCodecSizeChng, xCodecFree, pCodec); } -SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) { +SQLITE_API void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) { pPager->errCode = error; } @@ -51306,6 +57822,7 @@ SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) */ #ifndef SQLITE_OMIT_WAL +/* #include "wal.h" */ /* ** Trace output macros @@ -51335,7 +57852,8 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0; /* ** Indices of various locking bytes. WAL_NREADER is the number -** of available reader locks and should be at least 3. +** of available reader locks and should be at least 3. The default +** is SQLITE_SHM_NLOCK==8 and WAL_NREADER==5. */ #define WAL_WRITE_LOCK 0 #define WAL_ALL_BUT_WRITE 1 @@ -51355,7 +57873,10 @@ typedef struct WalCkptInfo WalCkptInfo; ** The following object holds a copy of the wal-index header content. ** ** The actual header in the wal-index consists of two copies of this -** object. +** object followed by one instance of the WalCkptInfo object. +** For all versions of SQLite through 3.10.0 and probably beyond, +** the locking bytes (WalCkptInfo.aLock) start at offset 120 and +** the total header size is 136 bytes. ** ** The szPage value can be any power of 2 between 512 and 32768, inclusive. ** Or it can be 1 to represent a 65536-byte page. The latter case was @@ -51388,6 +57909,16 @@ struct WalIndexHdr { ** However, a WAL_WRITE_LOCK thread can move the value of nBackfill from ** mxFrame back to zero when the WAL is reset. ** +** nBackfillAttempted is the largest value of nBackfill that a checkpoint +** has attempted to achieve. Normally nBackfill==nBackfillAtempted, however +** the nBackfillAttempted is set before any backfilling is done and the +** nBackfill is only set after all backfilling completes. So if a checkpoint +** crashes, nBackfillAttempted might be larger than nBackfill. The +** WalIndexHdr.mxFrame must never be less than nBackfillAttempted. +** +** The aLock[] field is a set of bytes used for locking. These bytes should +** never be read or written. +** ** There is one entry in aReadMark[] for each reader lock. If a reader ** holds read-lock K, then the value in aReadMark[K] is no greater than ** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) @@ -51427,6 +57958,9 @@ struct WalIndexHdr { struct WalCkptInfo { u32 nBackfill; /* Number of WAL frames backfilled into DB */ u32 aReadMark[WAL_NREADER]; /* Reader marks */ + u8 aLock[SQLITE_SHM_NLOCK]; /* Reserved space for locks */ + u32 nBackfillAttempted; /* WAL frames perhaps written, or maybe not */ + u32 notUsed0; /* Available for future enhancements */ }; #define READMARK_NOT_USED 0xffffffff @@ -51436,9 +57970,8 @@ struct WalCkptInfo { ** only support mandatory file-locks, we do not read or write data ** from the region of the file on which locks are applied. */ -#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2 + sizeof(WalCkptInfo)) -#define WALINDEX_LOCK_RESERVED 16 -#define WALINDEX_HDR_SIZE (WALINDEX_LOCK_OFFSET+WALINDEX_LOCK_RESERVED) +#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2+offsetof(WalCkptInfo,aLock)) +#define WALINDEX_HDR_SIZE (sizeof(WalIndexHdr)*2+sizeof(WalCkptInfo)) /* Size of header before each frame in wal */ #define WAL_FRAME_HDRSIZE 24 @@ -51491,11 +58024,16 @@ struct Wal { u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ + u32 minFrame; /* Ignore wal frames before this one */ + u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif +#ifdef SQLITE_ENABLE_SNAPSHOT + WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ +#endif }; /* @@ -51585,10 +58123,10 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ if( pWal->nWiData<=iPage ){ int nByte = sizeof(u32*)*(iPage+1); volatile u32 **apNew; - apNew = (volatile u32 **)sqlite3_realloc((void *)pWal->apWiData, nByte); + apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); if( !apNew ){ *ppPage = 0; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset((void*)&apNew[pWal->nWiData], 0, sizeof(u32*)*(iPage+1-pWal->nWiData)); @@ -51600,7 +58138,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ if( pWal->apWiData[iPage]==0 ){ if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); - if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM; + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; }else{ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] @@ -51637,7 +58175,7 @@ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ ** The argument to this macro must be of type u32. On a little-endian ** architecture, it returns the u32 value that results from interpreting ** the 4 bytes as a big-endian value. On a big-endian architecture, it -** returns the value that would be produced by intepreting the 4 bytes +** returns the value that would be produced by interpreting the 4 bytes ** of the input value as a little-endian integer. */ #define BYTESWAP32(x) ( \ @@ -51711,9 +58249,9 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.isInit = 1; pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); - memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); + memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); walShmBarrier(pWal); - memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); + memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); } /* @@ -51741,14 +58279,18 @@ static void walEncodeFrame( assert( WAL_FRAME_HDRSIZE==24 ); sqlite3Put4byte(&aFrame[0], iPage); sqlite3Put4byte(&aFrame[4], nTruncate); - memcpy(&aFrame[8], pWal->hdr.aSalt, 8); + if( pWal->iReCksum==0 ){ + memcpy(&aFrame[8], pWal->hdr.aSalt, 8); - nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); - walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); - walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); + nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); + walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); - sqlite3Put4byte(&aFrame[16], aCksum[0]); - sqlite3Put4byte(&aFrame[20], aCksum[1]); + sqlite3Put4byte(&aFrame[16], aCksum[0]); + sqlite3Put4byte(&aFrame[20], aCksum[1]); + }else{ + memset(&aFrame[8], 0, 16); + } } /* @@ -52014,13 +58556,13 @@ static void walCleanupHash(Wal *pWal){ ** via the hash table even after the cleanup. */ if( iLimit ){ - int i; /* Loop counter */ + int j; /* Loop counter */ int iKey; /* Hash key */ - for(i=1; i<=iLimit; i++){ - for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==i ) break; + for(j=1; j<=iLimit; j++){ + for(iKey=walHash(aPgno[j]); aHash[iKey]; iKey=walNextHash(iKey)){ + if( aHash[iKey]==j ) break; } - assert( aHash[iKey]==i ); + assert( aHash[iKey]==j ); } } #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ @@ -52051,7 +58593,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ assert( idx <= HASHTABLE_NSLOT/2 + 1 ); /* If this is the first entry to be added to this hash-table, zero the - ** entire hash table and aPgno[] array before proceding. + ** entire hash table and aPgno[] array before proceeding. */ if( idx==1 ){ int nByte = (int)((u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]); @@ -52209,9 +58751,9 @@ static int walIndexRecover(Wal *pWal){ /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; - aFrame = (u8 *)sqlite3_malloc(szFrame); + aFrame = (u8 *)sqlite3_malloc64(szFrame); if( !aFrame ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto recovery_error; } aData = &aFrame[WAL_FRAME_HDRSIZE]; @@ -52260,6 +58802,7 @@ finished: */ pInfo = walCkptInfo(pWal); pInfo->nBackfill = 0; + pInfo->nBackfillAttempted = pWal->hdr.mxFrame; pInfo->aReadMark[0] = 0; for(i=1; iaReadMark[i] = READMARK_NOT_USED; if( pWal->hdr.mxFrame ) pInfo->aReadMark[1] = pWal->hdr.mxFrame; @@ -52331,7 +58874,11 @@ SQLITE_PRIVATE int sqlite3WalOpen( /* In the amalgamation, the os_unix.c and os_win.c source files come before ** this source file. Verify that the #defines of the locking byte offsets ** in os_unix.c and os_win.c agree with the WALINDEX_LOCK_OFFSET value. + ** For that matter, if the lock offset ever changes from its initial design + ** value of 120, we need to know that so there is an assert() to check it. */ + assert( 120==WALINDEX_LOCK_OFFSET ); + assert( 136==WALINDEX_HDR_SIZE ); #ifdef WIN_SHM_BASE assert( WIN_SHM_BASE==WALINDEX_LOCK_OFFSET ); #endif @@ -52344,7 +58891,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( *ppWal = 0; pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile); if( !pRet ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pRet->pVfs = pVfs; @@ -52522,7 +59069,7 @@ static void walMergesort( int nMerge = 0; /* Number of elements in list aMerge */ ht_slot *aMerge = 0; /* List to be merged */ int iList; /* Index into input list */ - int iSub = 0; /* Index into aSub array */ + u32 iSub = 0; /* Index into aSub array */ struct Sublist aSub[13]; /* Array of sub-lists */ memset(aSub, 0, sizeof(aSub)); @@ -52533,7 +59080,9 @@ static void walMergesort( nMerge = 1; aMerge = &aList[iList]; for(iSub=0; iList & (1<aList && p->nList<=(1<aList==&aList[iList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); @@ -52544,7 +59093,9 @@ static void walMergesort( for(iSub++; iSubnList<=(1<aList==&aList[nList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); @@ -52567,7 +59118,7 @@ static void walMergesort( ** Free an iterator allocated by walIteratorInit(). */ static void walIteratorFree(WalIterator *p){ - sqlite3ScratchFree(p); + sqlite3_free(p); } /* @@ -52602,9 +59153,9 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3ScratchMalloc(nByte); + p = (WalIterator *)sqlite3_malloc64(nByte); if( !p ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(p, 0, nByte); p->nSegment = nSegment; @@ -52612,11 +59163,11 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ /* Allocate temporary space used by the merge-sort routine. This block ** of memory will be freed before this function returns. */ - aTmp = (ht_slot *)sqlite3ScratchMalloc( + aTmp = (ht_slot *)sqlite3_malloc64( sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) ); if( !aTmp ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } for(i=0; rc==SQLITE_OK && iaSegment[i].aPgno = (u32 *)aPgno; } } - sqlite3ScratchFree(aTmp); + sqlite3_free(aTmp); if( rc!=SQLITE_OK ){ walIteratorFree(p); @@ -52686,6 +59237,39 @@ static int walPagesize(Wal *pWal){ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } +/* +** The following is guaranteed when this function is called: +** +** a) the WRITER lock is held, +** b) the entire log file has been checkpointed, and +** c) any existing readers are reading exclusively from the database +** file - there are no readers that may attempt to read a frame from +** the log file. +** +** This function updates the shared-memory structures so that the next +** client to write to the database (which may be this one) does so by +** writing frames into the start of the log file. +** +** The value of parameter salt1 is used as the aSalt[1] value in the +** new wal-index header. It should be passed a pseudo-random value (i.e. +** one obtained from sqlite3_randomness()). +*/ +static void walRestartHdr(Wal *pWal, u32 salt1){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + memcpy(&pWal->hdr.aSalt[1], &salt1, 4); + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + pInfo->nBackfillAttempted = 0; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -52709,7 +59293,7 @@ static int walPagesize(Wal *pWal){ ** database file. ** ** This routine uses and updates the nBackfill field of the wal-index header. -** This is the only routine tha will increase the value of nBackfill. +** This is the only routine that will increase the value of nBackfill. ** (A WAL reset or recovery will revert nBackfill to zero, but not increase ** its value.) ** @@ -52720,12 +59304,12 @@ static int walPagesize(Wal *pWal){ static int walCheckpoint( Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ - int (*xBusyCall)(void*), /* Function to call when busy */ + int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf /* Temporary buffer to use */ ){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int szPage; /* Database page-size */ WalIterator *pIter = 0; /* Wal iterator context */ u32 iDbpage = 0; /* Next database page to write */ @@ -52734,123 +59318,156 @@ static int walCheckpoint( u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ - int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ szPage = walPagesize(pWal); testcase( szPage<=32768 ); testcase( szPage>=65536 ); pInfo = walCkptInfo(pWal); - if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK; + if( pInfo->nBackfillhdr.mxFrame ){ - /* Allocate the iterator */ - rc = walIteratorInit(pWal, &pIter); - if( rc!=SQLITE_OK ){ - return rc; - } - assert( pIter ); - - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; - - /* Compute in mxSafeFrame the index of the last frame of the WAL that is - ** safe to write into the database. Frames beyond mxSafeFrame might - ** overwrite database pages that are in use by active readers and thus - ** cannot be backfilled from the WAL. - */ - mxSafeFrame = pWal->hdr.mxFrame; - mxPage = pWal->hdr.nPage; - for(i=1; iaReadMark[i]; - if( mxSafeFrame>y ){ - assert( y<=pWal->hdr.mxFrame ); - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - }else if( rc==SQLITE_BUSY ){ - mxSafeFrame = y; - xBusy = 0; - }else{ - goto walcheckpoint_out; - } + /* Allocate the iterator */ + rc = walIteratorInit(pWal, &pIter); + if( rc!=SQLITE_OK ){ + return rc; } - } + assert( pIter ); - if( pInfo->nBackfillnBackfill; + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); - /* Sync the WAL to disk */ - if( sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); - } - - /* If the database may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. + /* Compute in mxSafeFrame the index of the last frame of the WAL that is + ** safe to write into the database. Frames beyond mxSafeFrame might + ** overwrite database pages that are in use by active readers and thus + ** cannot be backfilled from the WAL. */ - if( rc==SQLITE_OK ){ - i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); - } - } - - - /* Iterate through the contents of the WAL, copying data to the db file. */ - while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ - i64 iOffset; - assert( walFramePgno(pWal, iFrame)==iDbpage ); - if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; - iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ - rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - iOffset = (iDbpage-1)*(i64)szPage; - testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - } - - /* If work was actually accomplished... */ - if( rc==SQLITE_OK ){ - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ - i64 szDb = pWal->hdr.nPage*(i64)szPage; - testcase( IS_BIG_INT(szDb) ); - rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK && sync_flags ){ - rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; + for(i=1; iaReadMark[i]; + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; + xBusy = 0; + }else{ + goto walcheckpoint_out; } } - if( rc==SQLITE_OK ){ - pInfo->nBackfill = mxSafeFrame; - } } - /* Release the reader lock held while backfilling */ - walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); + if( pInfo->nBackfillnBackfill; + + pInfo->nBackfillAttempted = mxSafeFrame; + + /* Sync the WAL to disk */ + if( sync_flags ){ + rc = sqlite3OsSync(pWal->pWalFd, sync_flags); + } + + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } + } + + + /* Iterate through the contents of the WAL, copying data to the db file */ + while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + i64 iOffset; + assert( walFramePgno(pWal, iFrame)==iDbpage ); + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ + continue; + } + iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ + rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + iOffset = (iDbpage-1)*(i64)szPage; + testcase( IS_BIG_INT(iOffset) ); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + } + + /* If work was actually accomplished... */ + if( rc==SQLITE_OK ){ + if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ + i64 szDb = pWal->hdr.nPage*(i64)szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + if( rc==SQLITE_OK && sync_flags ){ + rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + } + } + if( rc==SQLITE_OK ){ + pInfo->nBackfill = mxSafeFrame; + } + } + + /* Release the reader lock held while backfilling */ + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); + } + + if( rc==SQLITE_BUSY ){ + /* Reset the return code so as not to report a checkpoint failure + ** just because there are active readers. */ + rc = SQLITE_OK; + } } - if( rc==SQLITE_BUSY ){ - /* Reset the return code so as not to report a checkpoint failure - ** just because there are active readers. */ - rc = SQLITE_OK; - } - - /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal - ** file has been copied into the database file, then block until all - ** readers have finished using the wal file. This ensures that the next - ** process to write to the database restarts the wal file. + /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the + ** entire wal file has been copied into the database file, then block + ** until all readers have finished using the wal file. This ensures that + ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; - }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ - assert( mxSafeFrame==pWal->hdr.mxFrame ); + }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ + u32 salt1; + sqlite3_randomness(4, &salt1); + assert( pInfo->nBackfill==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ + if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ + /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as + ** SQLITE_CHECKPOINT_RESTART with the addition that it also + ** truncates the log file to zero bytes just prior to a + ** successful return. + ** + ** In theory, it might be safe to do this without updating the + ** wal-index header in shared memory, as all subsequent reader or + ** writer clients should see that the entire log file has been + ** checkpointed and behave accordingly. This seems unsafe though, + ** as it would leave the system in a state where the contents of + ** the wal-index header do not match the contents of the + ** file-system. To avoid this, update the wal-index header to + ** indicate that the log file contains zero valid frames. */ + walRestartHdr(pWal, salt1); + rc = sqlite3OsTruncate(pWal->pWalFd, 0); + } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); } } @@ -53013,7 +59630,7 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** wal-index from the WAL before returning. ** ** Set *pChanged to 1 if the wal-index header value in pWal->hdr is -** changed by this opertion. If pWal->hdr is unchanged, set *pChanged +** changed by this operation. If pWal->hdr is unchanged, set *pChanged ** to 0. ** ** If the wal-index header is successfully read, return SQLITE_OK. @@ -53142,6 +59759,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ int mxI; /* Index of largest aReadMark[] value */ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ + u32 mxFrame; /* Wal frame to lock to */ assert( pWal->readLock<0 ); /* Not currently locked */ @@ -53205,7 +59823,12 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ){ + if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame +#ifdef SQLITE_ENABLE_SNAPSHOT + && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 + || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) +#endif + ){ /* The WAL has been completely backfilled (or it is empty). ** and can be safely ignored. */ @@ -53217,7 +59840,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** may have been appended to the log before READ_LOCK(0) was obtained. ** When holding READ_LOCK(0), the reader ignores the entire log file, ** which implies that the database file contains a trustworthy - ** snapshoT. Since holding READ_LOCK(0) prevents a checkpoint from + ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from ** happening, this is usually correct. ** ** However, if frames have been appended to the log (or if the log @@ -53243,70 +59866,88 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ */ mxReadMark = 0; mxI = 0; + mxFrame = pWal->hdr.mxFrame; +#ifdef SQLITE_ENABLE_SNAPSHOT + if( pWal->pSnapshot && pWal->pSnapshot->mxFramepSnapshot->mxFrame; + } +#endif for(i=1; iaReadMark[i]; - if( mxReadMark<=thisMark && thisMark<=pWal->hdr.mxFrame ){ + if( mxReadMark<=thisMark && thisMark<=mxFrame ){ assert( thisMark!=READMARK_NOT_USED ); mxReadMark = thisMark; mxI = i; } } - /* There was once an "if" here. The extra "{" is to preserve indentation. */ - { - if( (pWal->readOnly & WAL_SHM_RDONLY)==0 - && (mxReadMarkhdr.mxFrame || mxI==0) - ){ - for(i=1; iaReadMark[i] = pWal->hdr.mxFrame; - mxI = i; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - break; - }else if( rc!=SQLITE_BUSY ){ - return rc; - } + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkaReadMark[i] = mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; + }else if( rc!=SQLITE_BUSY ){ + return rc; } } - if( mxI==0 ){ - assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; - } + } + if( mxI==0 ){ + assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + } - rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); - if( rc ){ - return rc==SQLITE_BUSY ? WAL_RETRY : rc; - } - /* Now that the read-lock has been obtained, check that neither the - ** value in the aReadMark[] array or the contents of the wal-index - ** header have changed. - ** - ** It is necessary to check that the wal-index header did not change - ** between the time it was read and when the shared-lock was obtained - ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility - ** that the log file may have been wrapped by a writer, or that frames - ** that occur later in the log than pWal->hdr.mxFrame may have been - ** copied into the database by a checkpointer. If either of these things - ** happened, then reading the database with the current value of - ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry - ** instead. - ** - ** This does not guarantee that the copy of the wal-index header is up to - ** date before proceeding. That would not be possible without somehow - ** blocking writers. It only guarantees that a dangerous checkpoint or - ** log-wrap (either of which would require an exclusive lock on - ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. - */ - walShmBarrier(pWal); - if( pInfo->aReadMark[mxI]!=mxReadMark - || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) - ){ - walUnlockShared(pWal, WAL_READ_LOCK(mxI)); - return WAL_RETRY; - }else{ - assert( mxReadMark<=pWal->hdr.mxFrame ); - pWal->readLock = (i16)mxI; - } + rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + if( rc ){ + return rc==SQLITE_BUSY ? WAL_RETRY : rc; + } + /* Now that the read-lock has been obtained, check that neither the + ** value in the aReadMark[] array or the contents of the wal-index + ** header have changed. + ** + ** It is necessary to check that the wal-index header did not change + ** between the time it was read and when the shared-lock was obtained + ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility + ** that the log file may have been wrapped by a writer, or that frames + ** that occur later in the log than pWal->hdr.mxFrame may have been + ** copied into the database by a checkpointer. If either of these things + ** happened, then reading the database with the current value of + ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry + ** instead. + ** + ** Before checking that the live wal-index header has not changed + ** since it was read, set Wal.minFrame to the first frame in the wal + ** file that has not yet been checkpointed. This client will not need + ** to read any frames earlier than minFrame from the wal file - they + ** can be safely read directly from the database file. + ** + ** Because a ShmBarrier() call is made between taking the copy of + ** nBackfill and checking that the wal-header in shared-memory still + ** matches the one cached in pWal->hdr, it is guaranteed that the + ** checkpointer that set nBackfill was not working with a wal-index + ** header newer than that cached in pWal->hdr. If it were, that could + ** cause a problem. The checkpointer could omit to checkpoint + ** a version of page X that lies before pWal->minFrame (call that version + ** A) on the basis that there is a newer version (version B) of the same + ** page later in the wal file. But if version B happens to like past + ** frame pWal->hdr.mxFrame - then the client would incorrectly assume + ** that it can read version A from the database file. However, since + ** we can guarantee that the checkpointer that set nBackfill could not + ** see any pages past pWal->hdr.mxFrame, this problem does not come up. + */ + pWal->minFrame = pInfo->nBackfill+1; + walShmBarrier(pWal); + if( pInfo->aReadMark[mxI]!=mxReadMark + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ + walUnlockShared(pWal, WAL_READ_LOCK(mxI)); + return WAL_RETRY; + }else{ + assert( mxReadMark<=pWal->hdr.mxFrame ); + pWal->readLock = (i16)mxI; } return rc; } @@ -53329,6 +59970,14 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ int rc; /* Return code */ int cnt = 0; /* Number of TryBeginRead attempts */ +#ifdef SQLITE_ENABLE_SNAPSHOT + int bChanged = 0; + WalIndexHdr *pSnapshot = pWal->pSnapshot; + if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ + bChanged = 1; + } +#endif + do{ rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); }while( rc==WAL_RETRY ); @@ -53336,6 +59985,66 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ testcase( (rc&0xff)==SQLITE_IOERR ); testcase( rc==SQLITE_PROTOCOL ); testcase( rc==SQLITE_OK ); + +#ifdef SQLITE_ENABLE_SNAPSHOT + if( rc==SQLITE_OK ){ + if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ + /* At this point the client has a lock on an aReadMark[] slot holding + ** a value equal to or smaller than pSnapshot->mxFrame, but pWal->hdr + ** is populated with the wal-index header corresponding to the head + ** of the wal file. Verify that pSnapshot is still valid before + ** continuing. Reasons why pSnapshot might no longer be valid: + ** + ** (1) The WAL file has been reset since the snapshot was taken. + ** In this case, the salt will have changed. + ** + ** (2) A checkpoint as been attempted that wrote frames past + ** pSnapshot->mxFrame into the database file. Note that the + ** checkpoint need not have completed for this to cause problems. + */ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + + assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 ); + assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame ); + + /* It is possible that there is a checkpointer thread running + ** concurrent with this code. If this is the case, it may be that the + ** checkpointer has already determined that it will checkpoint + ** snapshot X, where X is later in the wal file than pSnapshot, but + ** has not yet set the pInfo->nBackfillAttempted variable to indicate + ** its intent. To avoid the race condition this leads to, ensure that + ** there is no checkpointer process by taking a shared CKPT lock + ** before checking pInfo->nBackfillAttempted. */ + rc = walLockShared(pWal, WAL_CKPT_LOCK); + + if( rc==SQLITE_OK ){ + /* Check that the wal file has not been wrapped. Assuming that it has + ** not, also check that no checkpointer has attempted to checkpoint any + ** frames beyond pSnapshot->mxFrame. If either of these conditions are + ** true, return SQLITE_BUSY_SNAPSHOT. Otherwise, overwrite pWal->hdr + ** with *pSnapshot and set *pChanged as appropriate for opening the + ** snapshot. */ + if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + && pSnapshot->mxFrame>=pInfo->nBackfillAttempted + ){ + assert( pWal->readLock>0 ); + memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); + *pChanged = bChanged; + }else{ + rc = SQLITE_BUSY_SNAPSHOT; + } + + /* Release the shared CKPT lock obtained above. */ + walUnlockShared(pWal, WAL_CKPT_LOCK); + } + + + if( rc!=SQLITE_OK ){ + sqlite3WalEndReadTransaction(pWal); + } + } + } +#endif return rc; } @@ -53367,6 +60076,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ + int iMinHash; /* This routine is only be called from within a read transaction. */ assert( pWal->readLock>=0 || pWal->lockError ); @@ -53407,7 +60117,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( ** This condition filters out entries that were added to the hash ** table after the current read-transaction had started. */ - for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ + iMinHash = walFramePage(pWal->minFrame); + for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ volatile ht_slot *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ @@ -53422,8 +60133,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( nCollide = HASHTABLE_NSLOT; for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ - /* assert( iFrame>iRead ); -- not true if there is corruption */ + if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ + assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } if( (nCollide--)==0 ){ @@ -53439,7 +60150,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - for(iTest=iLast; iTest>0; iTest--){ + assert( pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; @@ -53505,6 +60217,7 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ /* Cannot start a write transaction without first holding a read ** transaction. */ assert( pWal->readLock>=0 ); + assert( pWal->writeLock==0 && pWal->iReCksum==0 ); if( pWal->readOnly ){ return SQLITE_READONLY; @@ -53540,6 +60253,7 @@ SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){ if( pWal->writeLock ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; + pWal->iReCksum = 0; pWal->truncateOnCommit = 0; } return SQLITE_OK; @@ -53588,7 +60302,6 @@ SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *p } if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } - assert( rc==SQLITE_OK ); return rc; } @@ -53637,7 +60350,6 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ return rc; } - /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending @@ -53670,20 +60382,8 @@ static int walRestartLog(Wal *pWal){ ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() - ** to handle if this transaction is rolled back. - */ - int i; /* Loop counter */ - u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ - - pWal->nCkpt++; - pWal->hdr.mxFrame = 0; - sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); - aSalt[1] = salt1; - walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; - pInfo->aReadMark[1] = 0; - for(i=2; iaReadMark[i] = READMARK_NOT_USED; - assert( pInfo->aReadMark[0]==0 ); + ** to handle if this transaction is rolled back. */ + walRestartHdr(pWal, salt1); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ return rc; @@ -53760,7 +60460,7 @@ static int walWriteOneFrame( void *pData; /* Data actually written */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ #if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM; + if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; #else pData = pPage->pData; #endif @@ -53772,6 +60472,59 @@ static int walWriteOneFrame( return rc; } +/* +** This function is called as part of committing a transaction within which +** one or more frames have been overwritten. It updates the checksums for +** all frames written to the wal file by the current transaction starting +** with the earliest to have been overwritten. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int walRewriteChecksums(Wal *pWal, u32 iLast){ + const int szPage = pWal->szPage;/* Database page size */ + int rc = SQLITE_OK; /* Return code */ + u8 *aBuf; /* Buffer to load data from wal file into */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-headers in */ + u32 iRead; /* Next frame to read from wal file */ + i64 iCksumOff; + + aBuf = sqlite3_malloc(szPage + WAL_FRAME_HDRSIZE); + if( aBuf==0 ) return SQLITE_NOMEM_BKPT; + + /* Find the checksum values to use as input for the recalculating the + ** first checksum. If the first frame is frame 1 (implying that the current + ** transaction restarted the wal file), these values must be read from the + ** wal-file header. Otherwise, read them from the frame header of the + ** previous frame. */ + assert( pWal->iReCksum>0 ); + if( pWal->iReCksum==1 ){ + iCksumOff = 24; + }else{ + iCksumOff = walFrameOffset(pWal->iReCksum-1, szPage) + 16; + } + rc = sqlite3OsRead(pWal->pWalFd, aBuf, sizeof(u32)*2, iCksumOff); + pWal->hdr.aFrameCksum[0] = sqlite3Get4byte(aBuf); + pWal->hdr.aFrameCksum[1] = sqlite3Get4byte(&aBuf[sizeof(u32)]); + + iRead = pWal->iReCksum; + pWal->iReCksum = 0; + for(; rc==SQLITE_OK && iRead<=iLast; iRead++){ + i64 iOff = walFrameOffset(iRead, szPage); + rc = sqlite3OsRead(pWal->pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); + if( rc==SQLITE_OK ){ + u32 iPgno, nDbSize; + iPgno = sqlite3Get4byte(aBuf); + nDbSize = sqlite3Get4byte(&aBuf[4]); + + walEncodeFrame(pWal, iPgno, nDbSize, &aBuf[WAL_FRAME_HDRSIZE], aFrame); + rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOff); + } + } + + sqlite3_free(aBuf); + return rc; +} + /* ** Write a set of frames to the log. The caller must hold the write-lock ** on the log file (obtained using sqlite3WalBeginWriteTransaction()). @@ -53792,6 +60545,8 @@ SQLITE_PRIVATE int sqlite3WalFrames( int szFrame; /* The size of a single frame */ i64 iOffset; /* Next byte to write in WAL file */ WalWriter w; /* The writer */ + u32 iFirst = 0; /* First frame that may be overwritten */ + WalIndexHdr *pLive; /* Pointer to shared header */ assert( pList ); assert( pWal->writeLock ); @@ -53807,6 +60562,11 @@ SQLITE_PRIVATE int sqlite3WalFrames( } #endif + pLive = (WalIndexHdr*)walIndexHdr(pWal); + if( memcmp(&pWal->hdr, (void *)pLive, sizeof(WalIndexHdr))!=0 ){ + iFirst = pLive->mxFrame+1; + } + /* See if it is possible to write these frames into the start of the ** log file, instead of appending to it at pWal->hdr.mxFrame. */ @@ -53871,6 +60631,33 @@ SQLITE_PRIVATE int sqlite3WalFrames( /* Write all frames into the log file exactly once */ for(p=pList; p; p=p->pDirty){ int nDbSize; /* 0 normally. Positive == commit flag */ + + /* Check if this page has already been written into the wal file by + ** the current transaction. If so, overwrite the existing frame and + ** set Wal.writeLock to WAL_WRITELOCK_RECKSUM - indicating that + ** checksums must be recomputed when the transaction is committed. */ + if( iFirst && (p->pDirty || isCommit==0) ){ + u32 iWrite = 0; + VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); + assert( rc==SQLITE_OK || iWrite==0 ); + if( iWrite>=iFirst ){ + i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; + void *pData; + if( pWal->iReCksum==0 || iWriteiReCksum ){ + pWal->iReCksum = iWrite; + } +#if defined(SQLITE_HAS_CODEC) + if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; +#else + pData = p->pData; +#endif + rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); + if( rc ) return rc; + p->flags &= ~PGHDR_WAL_APPEND; + continue; + } + } + iFrame++; assert( iOffset==walFrameOffset(iFrame, szPage) ); nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; @@ -53878,6 +60665,13 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( rc ) return rc; pLast = p; iOffset += szFrame; + p->flags |= PGHDR_WAL_APPEND; + } + + /* Recalculate checksums within the wal file if required. */ + if( isCommit && pWal->iReCksum ){ + rc = walRewriteChecksums(pWal, iFrame); + if( rc ) return rc; } /* If this is the end of a transaction, then we might need to pad @@ -53885,7 +60679,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( ** ** Padding and syncing only occur if this set of frames complete a ** transaction and if PRAGMA synchronous=FULL. If synchronous==NORMAL - ** or synchonous==OFF, then no padding or syncing are needed. + ** or synchronous==OFF, then no padding or syncing are needed. ** ** If SQLITE_IOCAP_POWERSAFE_OVERWRITE is defined, then padding is not ** needed and only the sync is done. If padding is needed, then the @@ -53895,16 +60689,21 @@ SQLITE_PRIVATE int sqlite3WalFrames( ** past the sector boundary is written after the sync. */ if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){ + int bSync = 1; if( pWal->padToSectorBoundary ){ int sectorSize = sqlite3SectorSize(pWal->pWalFd); w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; + bSync = (w.iSyncPoint==iOffset); + testcase( bSync ); while( iOffsethdr.mxFrame; for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ + if( (p->flags & PGHDR_WAL_APPEND)==0 ) continue; iFrame++; rc = walIndexAppend(pWal, iFrame, p->pgno); } @@ -53971,7 +60771,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( */ SQLITE_PRIVATE int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ - int eMode, /* PASSIVE, FULL or RESTART */ + int eMode, /* PASSIVE, FULL, RESTART, or TRUNCATE */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ @@ -53983,29 +60783,42 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ + int (*xBusy2)(void*) = xBusy; /* Busy handler for eMode2 */ assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive + ** "checkpoint" lock on the database file. */ rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ - /* Usually this is SQLITE_BUSY meaning that another thread or process - ** is already running a checkpoint, or maybe a recovery. But it might - ** also be SQLITE_IOERR. */ + /* EVIDENCE-OF: R-10421-19736 If any other process is running a + ** checkpoint operation at the same time, the lock cannot be obtained and + ** SQLITE_BUSY is returned. + ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + ** it will not be invoked in this case. + */ + testcase( rc==SQLITE_BUSY ); + testcase( xBusy!=0 ); return rc; } pWal->ckptLock = 1; - /* If this is a blocking-checkpoint, then obtain the write-lock as well - ** to prevent any writers from running while the checkpoint is underway. - ** This has to be done before the call to walIndexReadHdr() below. + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + ** TRUNCATE modes also obtain the exclusive "writer" lock on the database + ** file. ** - ** If the writer lock cannot be obtained, then a passive checkpoint is - ** run instead. Since the checkpointer is not holding the writer lock, - ** there is no point in blocking waiting for any readers. Assuming no - ** other error occurs, this function will return SQLITE_BUSY to the caller. + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. */ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); @@ -54013,6 +60826,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( pWal->writeLock = 1; }else if( rc==SQLITE_BUSY ){ eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; rc = SQLITE_OK; } } @@ -54027,10 +60841,11 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf); + rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); } /* If no error occurred, set the output variables. */ @@ -54142,6 +60957,52 @@ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +#ifdef SQLITE_ENABLE_SNAPSHOT +/* Create a snapshot object. The content of a snapshot is opaque to +** every other subsystem, so the WAL module can put whatever it needs +** in the object. +*/ +SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){ + int rc = SQLITE_OK; + WalIndexHdr *pRet; + + assert( pWal->readLock>=0 && pWal->writeLock==0 ); + + pRet = (WalIndexHdr*)sqlite3_malloc(sizeof(WalIndexHdr)); + if( pRet==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + memcpy(pRet, &pWal->hdr, sizeof(WalIndexHdr)); + *ppSnapshot = (sqlite3_snapshot*)pRet; + } + + return rc; +} + +/* Try to open on pSnapshot when the next read-transaction starts +*/ +SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot){ + pWal->pSnapshot = (WalIndexHdr*)pSnapshot; +} + +/* +** Return a +ve value if snapshot p1 is newer than p2. A -ve value if +** p1 is older than p2 and zero if p1 and p2 are the same snapshot. +*/ +SQLITE_API int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ + WalIndexHdr *pHdr1 = (WalIndexHdr*)p1; + WalIndexHdr *pHdr2 = (WalIndexHdr*)p2; + + /* aSalt[0] is a copy of the value stored in the wal file header. It + ** is incremented each time the wal file is restarted. */ + if( pHdr1->aSalt[0]aSalt[0] ) return -1; + if( pHdr1->aSalt[0]>pHdr2->aSalt[0] ) return +1; + if( pHdr1->mxFramemxFrame ) return -1; + if( pHdr1->mxFrame>pHdr2->mxFrame ) return +1; + return 0; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + #ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a @@ -54154,6 +61015,12 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ } #endif +/* Return the sqlite3_file object for the WAL file +*/ +SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ + return pWal->pWalFd; +} + #endif /* #ifndef SQLITE_OMIT_WAL */ /************** End of wal.c *************************************************/ @@ -54175,6 +61042,7 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ ** big and we want to break it down some. This packaged seemed like ** a good breakout. */ +/* #include "btreeInt.h" */ #ifndef SQLITE_OMIT_SHARED_CACHE #if SQLITE_THREADSAFE @@ -54197,7 +61065,7 @@ static void lockBtreeMutex(Btree *p){ ** Release the BtShared mutex associated with B-Tree handle p and ** clear the p->locked boolean. */ -static void unlockBtreeMutex(Btree *p){ +static void SQLITE_NOINLINE unlockBtreeMutex(Btree *p){ BtShared *pBt = p->pBt; assert( p->locked==1 ); assert( sqlite3_mutex_held(pBt->mutex) ); @@ -54208,6 +61076,9 @@ static void unlockBtreeMutex(Btree *p){ p->locked = 0; } +/* Forward reference */ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p); + /* ** Enter a mutex on the given BTree object. ** @@ -54225,8 +61096,6 @@ static void unlockBtreeMutex(Btree *p){ ** subsequent Btrees that desire a lock. */ SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ - Btree *pLater; - /* Some basic sanity checking on the Btree. The list of Btrees ** connected by pNext and pPrev should be in sorted order by ** Btree.pBt value. All elements of the list should belong to @@ -54251,9 +61120,20 @@ SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ if( !p->sharable ) return; p->wantToLock++; if( p->locked ) return; + btreeLockCarefully(p); +} + +/* This is a helper function for sqlite3BtreeLock(). By moving +** complex, but seldom used logic, out of sqlite3BtreeLock() and +** into this routine, we avoid unnecessary stack pointer changes +** and thus help the sqlite3BtreeLock() routine to run much faster +** in the common case. +*/ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p){ + Btree *pLater; /* In most cases, we should be able to acquire the lock we - ** want without having to go throught the ascending lock + ** want without having to go through the ascending lock ** procedure that follows. Just be sure not to block. */ if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){ @@ -54283,10 +61163,12 @@ SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ } } + /* ** Exit the recursive mutex on a Btree. */ SQLITE_PRIVATE void sqlite3BtreeLeave(Btree *p){ + assert( sqlite3_mutex_held(p->db->mutex) ); if( p->sharable ){ assert( p->wantToLock>0 ); p->wantToLock--; @@ -54314,21 +61196,6 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ #endif -#ifndef SQLITE_OMIT_INCRBLOB -/* -** Enter and leave a mutex on a Btree given a cursor owned by that -** Btree. These entry points are used by incremental I/O and can be -** omitted if that module is not used. -*/ -SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ - sqlite3BtreeEnter(pCur->pBtree); -} -SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ - sqlite3BtreeLeave(pCur->pBtree); -} -#endif /* SQLITE_OMIT_INCRBLOB */ - - /* ** Enter the mutex on every Btree associated with a database ** connection. This is needed (for example) prior to parsing @@ -54362,14 +61229,6 @@ SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ } } -/* -** Return true if a particular Btree requires a lock. Return FALSE if -** no lock is ever required since it is not sharable. -*/ -SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ - return p->sharable; -} - #ifndef NDEBUG /* ** Return true if the current thread holds the database connection @@ -54443,6 +61302,25 @@ SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ } } #endif /* if SQLITE_THREADSAFE */ + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Enter a mutex on a Btree given a cursor owned by that Btree. +** +** These entry points are used by incremental I/O only. Enter() is required +** any time OMIT_SHARED_CACHE is not defined, regardless of whether or not +** the build is threadsafe. Leave() is only required by threadsafe builds. +*/ +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ + sqlite3BtreeEnter(pCur->pBtree); +} +# if SQLITE_THREADSAFE +SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ + sqlite3BtreeLeave(pCur->pBtree); +} +# endif +#endif /* ifndef SQLITE_OMIT_INCRBLOB */ + #endif /* ifndef SQLITE_OMIT_SHARED_CACHE */ /************** End of btmutex.c *********************************************/ @@ -54458,10 +61336,11 @@ SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file implements a external (disk-based) database using BTrees. +** This file implements an external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. ** Including a description of file format and an overview of operation. */ +/* #include "btreeInt.h" */ /* ** The header string that appears at the beginning of every @@ -54623,6 +61502,12 @@ static int hasSharedCacheTableLock( for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){ Index *pIdx = (Index *)sqliteHashData(p); if( pIdx->tnum==(int)iRoot ){ + if( iTab ){ + /* Two or more indexes share the same root page. There must + ** be imposter tables. So just return true. The assert is not + ** useful in that case. */ + return 1; + } iTab = pIdx->pTable->tnum; } } @@ -54792,7 +61677,7 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ if( !pLock ){ pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); if( !pLock ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pLock->iTable = iTable; pLock->pBtree = p; @@ -54892,6 +61777,19 @@ static void releasePage(MemPage *pPage); /* Forward reference */ static int cursorHoldsMutex(BtCursor *p){ return sqlite3_mutex_held(p->pBt->mutex); } + +/* Verify that the cursor and the BtShared agree about what is the current +** database connetion. This is important in shared-cache mode. If the database +** connection pointers get out-of-sync, it is possible for routines like +** btreeInitPage() to reference an stale connection pointer that references a +** a connection that has already closed. This routine is used inside assert() +** statements only and for the purpose of double-checking that the btree code +** does keep the database connection pointers up-to-date. +*/ +static int cursorOwnsBtShared(BtCursor *p){ + assert( cursorHoldsMutex(p) ); + return (p->pBtree->db==p->pBt->db); +} #endif /* @@ -54932,11 +61830,15 @@ static void invalidateIncrblobCursors( int isClearTable /* True if all rows are being deleted */ ){ BtCursor *p; - BtShared *pBt = pBtree->pBt; + if( pBtree->hasIncrblobCur==0 ) return; assert( sqlite3BtreeHoldsMutex(pBtree) ); - for(p=pBt->pCursor; p; p=p->pNext){ - if( (p->curFlags & BTCF_Incrblob)!=0 && (isClearTable || p->info.nKey==iRow) ){ - p->eState = CURSOR_INVALID; + pBtree->hasIncrblobCur = 0; + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( (p->curFlags & BTCF_Incrblob)!=0 ){ + pBtree->hasIncrblobCur = 1; + if( isClearTable || p->info.nKey==iRow ){ + p->eState = CURSOR_INVALID; + } } } } @@ -54987,7 +61889,7 @@ static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ assert( pgno<=pBt->nPage ); pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage); if( !pBt->pHasContent ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){ @@ -55029,6 +61931,47 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ pCur->iPage = -1; } +/* +** The cursor passed as the only argument must point to a valid entry +** when this function is called (i.e. have eState==CURSOR_VALID). This +** function saves the current cursor key in variables pCur->nKey and +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** code otherwise. +** +** If the cursor is open on an intkey table, then the integer key +** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing +** the key. +*/ +static int saveCursorKey(BtCursor *pCur){ + int rc = SQLITE_OK; + assert( CURSOR_VALID==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + if( pCur->curIntKey ){ + /* Only the rowid is required for a table btree */ + pCur->nKey = sqlite3BtreeIntegerKey(pCur); + }else{ + /* For an index btree, save the complete key content */ + void *pKey; + pCur->nKey = sqlite3BtreePayloadSize(pCur); + pKey = sqlite3Malloc( pCur->nKey ); + if( pKey ){ + rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); + if( rc==SQLITE_OK ){ + pCur->pKey = pKey; + }else{ + sqlite3_free(pKey); + } + }else{ + rc = SQLITE_NOMEM_BKPT; + } + } + assert( !pCur->curIntKey || !pCur->pKey ); + return rc; +} /* ** Save the current cursor position in the variables BtCursor.nKey @@ -55040,55 +61983,75 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ static int saveCursorPosition(BtCursor *pCur){ int rc; - assert( CURSOR_VALID==pCur->eState ); + assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState ); assert( 0==pCur->pKey ); assert( cursorHoldsMutex(pCur) ); - rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); - assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ - - /* If this is an intKey table, then the above call to BtreeKeySize() - ** stores the integer key in pCur->nKey. In this case this value is - ** all that is required. Otherwise, if pCur is not open on an intKey - ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. - */ - if( 0==pCur->apPage[0]->intKey ){ - void *pKey = sqlite3Malloc( (int)pCur->nKey ); - if( pKey ){ - rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); - if( rc==SQLITE_OK ){ - pCur->pKey = pKey; - }else{ - sqlite3_free(pKey); - } - }else{ - rc = SQLITE_NOMEM; - } + if( pCur->eState==CURSOR_SKIPNEXT ){ + pCur->eState = CURSOR_VALID; + }else{ + pCur->skipNext = 0; } - assert( !pCur->apPage[0]->intKey || !pCur->pKey ); + rc = saveCursorKey(pCur); if( rc==SQLITE_OK ){ btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } - invalidateOverflowCache(pCur); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl|BTCF_AtLast); return rc; } +/* Forward reference */ +static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); + /* ** Save the positions of all cursors (except pExcept) that are open on -** the table with root-page iRoot. Usually, this is called just before cursor -** pExcept is used to modify the table (BtreeDelete() or BtreeInsert()). +** the table with root-page iRoot. "Saving the cursor position" means that +** the location in the btree is remembered in such a way that it can be +** moved back to the same spot after the btree has been modified. This +** routine is called just before cursor pExcept is used to modify the +** table, for example in BtreeDelete() or BtreeInsert(). +** +** If there are two or more cursors on the same btree, then all such +** cursors should have their BTCF_Multiple flag set. The btreeCursor() +** routine enforces that rule. This routine only needs to be called in +** the uncommon case when pExpect has the BTCF_Multiple flag set. +** +** If pExpect!=NULL and if no other cursors are found on the same root-page, +** then the BTCF_Multiple flag on pExpect is cleared, to avoid another +** pointless call to this routine. +** +** Implementation note: This routine merely checks to see if any cursors +** need to be saved. It calls out to saveCursorsOnList() in the (unusual) +** event that cursors are in need to being saved. */ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ BtCursor *p; assert( sqlite3_mutex_held(pBt->mutex) ); assert( pExcept==0 || pExcept->pBt==pBt ); for(p=pBt->pCursor; p; p=p->pNext){ + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break; + } + if( p ) return saveCursorsOnList(p, iRoot, pExcept); + if( pExcept ) pExcept->curFlags &= ~BTCF_Multiple; + return SQLITE_OK; +} + +/* This helper routine to saveAllCursors does the actual work of saving +** the cursors if and when a cursor is found that actually requires saving. +** The common case is that no cursors need to be saved, so this routine is +** broken out from its caller to avoid unnecessary stack pointer movement. +*/ +static int SQLITE_NOINLINE saveCursorsOnList( + BtCursor *p, /* The first cursor that needs saving */ + Pgno iRoot, /* Only save cursor with this iRoot. Save all if zero */ + BtCursor *pExcept /* Do not save this cursor */ +){ + do{ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ - if( p->eState==CURSOR_VALID ){ + if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ int rc = saveCursorPosition(p); if( SQLITE_OK!=rc ){ return rc; @@ -55098,7 +62061,8 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ btreeReleaseAllCursorPages(p); } } - } + p = p->pNext; + }while( p ); return SQLITE_OK; } @@ -55126,7 +62090,7 @@ static int btreeMoveto( ){ int rc; /* Status code */ UnpackedRecord *pIdxKey; /* Unpacked index key */ - char aSpace[200]; /* Temp space for pIdxKey - to avoid a malloc */ + char aSpace[384]; /* Temp space for pIdxKey - to avoid a malloc */ char *pFree = 0; if( pKey ){ @@ -55134,7 +62098,7 @@ static int btreeMoveto( pIdxKey = sqlite3VdbeAllocUnpackedRecord( pCur->pKeyInfo, aSpace, sizeof(aSpace), &pFree ); - if( pIdxKey==0 ) return SQLITE_NOMEM; + if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 ){ sqlite3DbFree(pCur->pKeyInfo->db, pFree); @@ -55159,17 +62123,19 @@ static int btreeMoveto( */ static int btreeRestoreCursorPosition(BtCursor *pCur){ int rc; - assert( cursorHoldsMutex(pCur) ); + int skipNext; + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState>=CURSOR_REQUIRESEEK ); if( pCur->eState==CURSOR_FAULT ){ return pCur->skipNext; } pCur->eState = CURSOR_INVALID; - rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &pCur->skipNext); + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); if( rc==SQLITE_OK ){ sqlite3_free(pCur->pKey); pCur->pKey = 0; assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); + pCur->skipNext |= skipNext; if( pCur->skipNext && pCur->eState==CURSOR_VALID ){ pCur->eState = CURSOR_SKIPNEXT; } @@ -55183,41 +62149,73 @@ static int btreeRestoreCursorPosition(BtCursor *pCur){ SQLITE_OK) /* -** Determine whether or not a cursor has moved from the position it -** was last placed at. Cursors can move when the row they are pointing -** at is deleted out from under them. +** Determine whether or not a cursor has moved from the position where +** it was last placed, or has been invalidated for any other reason. +** Cursors can move when the row they are pointing at is deleted out +** from under them, for example. Cursor might also move if a btree +** is rebalanced. ** -** This routine returns an error code if something goes wrong. The -** integer *pHasMoved is set as follows: +** Calling this routine with a NULL cursor pointer returns false. ** -** 0: The cursor is unchanged -** 1: The cursor is still pointing at the same row, but the pointers -** returned by sqlite3BtreeKeyFetch() or sqlite3BtreeDataFetch() -** might now be invalid because of a balance() or other change to the -** b-tree. -** 2: The cursor is no longer pointing to the row. The row might have -** been deleted out from under the cursor. +** Use the separate sqlite3BtreeCursorRestore() routine to restore a cursor +** back to where it ought to be if this routine returns true. */ -SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ +SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor *pCur){ + return pCur->eState!=CURSOR_VALID; +} + +/* +** This routine restores a cursor back to its original position after it +** has been moved by some outside activity (such as a btree rebalance or +** a row having been deleted out from under the cursor). +** +** On success, the *pDifferentRow parameter is false if the cursor is left +** pointing at exactly the same row. *pDifferntRow is the row the cursor +** was pointing to has been deleted, forcing the cursor to point to some +** nearby row. +** +** This routine should only be called for a cursor that just returned +** TRUE from sqlite3BtreeCursorHasMoved(). +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow){ int rc; - if( pCur->eState==CURSOR_VALID ){ - *pHasMoved = 0; - return SQLITE_OK; - } + assert( pCur!=0 ); + assert( pCur->eState!=CURSOR_VALID ); rc = restoreCursorPosition(pCur); if( rc ){ - *pHasMoved = 2; + *pDifferentRow = 1; return rc; } - if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){ - *pHasMoved = 2; + if( pCur->eState!=CURSOR_VALID ){ + *pDifferentRow = 1; }else{ - *pHasMoved = 1; + assert( pCur->skipNext==0 ); + *pDifferentRow = 0; } return SQLITE_OK; } +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Provide hints to the cursor. The particular hint given (and the type +** and number of the varargs parameters) is determined by the eHintType +** parameter. See the definitions of the BTREE_HINT_* macros for details. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ + /* Used only by system that substitute their own storage engine */ +} +#endif + +/* +** Provide flag hints to the cursor. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor *pCur, unsigned x){ + assert( x==BTREE_SEEK_EQ || x==BTREE_BULKLOAD || x==0 ); + pCur->hints = x; +} + + #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Given a page number of a regular database page, return the page @@ -55271,7 +62269,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ return; } iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); if( rc!=SQLITE_OK ){ *pRC = rc; return; @@ -55314,7 +62312,7 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ assert( sqlite3_mutex_held(pBt->mutex) ); iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); if( rc!=0 ){ return rc; } @@ -55346,128 +62344,214 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ ** the page, 1 means the second cell, and so forth) return a pointer ** to the cell content. ** +** findCellPastPtr() does the same except it skips past the initial +** 4-byte child pointer found on interior pages, if there is one. +** ** This routine works only for pages that do not contain overflow cells. */ #define findCell(P,I) \ - ((P)->aData + ((P)->maskPage & get2byte(&(P)->aCellIdx[2*(I)]))) -#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) + ((P)->aData + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) +#define findCellPastPtr(P,I) \ + ((P)->aDataOfst + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) /* -** This a more complex version of findCell() that works for -** pages that do contain overflow cells. +** This is common tail processing for btreeParseCellPtr() and +** btreeParseCellPtrIndex() for the case when the cell does not fit entirely +** on a single B-tree page. Make necessary adjustments to the CellInfo +** structure. */ -static u8 *findOverflowCell(MemPage *pPage, int iCell){ - int i; - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - for(i=pPage->nOverflow-1; i>=0; i--){ - int k; - k = pPage->aiOvfl[i]; - if( k<=iCell ){ - if( k==iCell ){ - return pPage->apOvfl[i]; - } - iCell--; - } +static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + /* If the payload will not fit completely on the local page, we have + ** to decide how much to store locally and how much to spill onto + ** overflow pages. The strategy is to minimize the amount of unused + ** space on overflow pages while keeping the amount of local storage + ** in between minLocal and maxLocal. + ** + ** Warning: changing the way overflow payload is distributed in any + ** way will result in an incompatible file format. + */ + int minLocal; /* Minimum amount of payload held locally */ + int maxLocal; /* Maximum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + + minLocal = pPage->minLocal; + maxLocal = pPage->maxLocal; + surplus = minLocal + (pInfo->nPayload - minLocal)%(pPage->pBt->usableSize-4); + testcase( surplus==maxLocal ); + testcase( surplus==maxLocal+1 ); + if( surplus <= maxLocal ){ + pInfo->nLocal = (u16)surplus; + }else{ + pInfo->nLocal = (u16)minLocal; } - return findCell(pPage, iCell); + pInfo->nSize = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell) + 4; } /* -** Parse a cell content block and fill in the CellInfo structure. There -** are two versions of this function. btreeParseCell() takes a -** cell index as the second argument and btreeParseCellPtr() -** takes a pointer to the body of the cell as its second argument. +** The following routines are implementations of the MemPage.xParseCell() +** method. ** -** Within this file, the parseCell() macro can be called instead of -** btreeParseCellPtr(). Using some compilers, this will be faster. +** Parse a cell content block and fill in the CellInfo structure. +** +** btreeParseCellPtr() => table btree leaf nodes +** btreeParseCellNoPayload() => table btree internal nodes +** btreeParseCellPtrIndex() => index btree nodes +** +** There is also a wrapper function btreeParseCell() that works for +** all MemPage types and that references the cell by index rather than +** by pointer. */ +static void btreeParseCellPtrNoPayload( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 ); + assert( pPage->childPtrSize==4 ); +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER(pPage); +#endif + pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nPayload = 0; + pInfo->nLocal = 0; + pInfo->pPayload = 0; + return; +} static void btreeParseCellPtr( MemPage *pPage, /* Page containing the cell */ u8 *pCell, /* Pointer to the cell text. */ CellInfo *pInfo /* Fill in this structure */ ){ - u16 n; /* Number bytes in cell content header */ + u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ + u64 iKey; /* Extracted Key value */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - - pInfo->pCell = pCell; assert( pPage->leaf==0 || pPage->leaf==1 ); - n = pPage->childPtrSize; - assert( n==4-4*pPage->leaf ); - if( pPage->intKey ){ - if( pPage->hasData ){ - assert( n==0 ); - n = getVarint32(pCell, nPayload); - }else{ - nPayload = 0; - } - n += getVarint(&pCell[n], (u64*)&pInfo->nKey); - pInfo->nData = nPayload; - }else{ - pInfo->nData = 0; - n += getVarint32(&pCell[n], nPayload); - pInfo->nKey = nPayload; + assert( pPage->intKeyLeaf ); + assert( pPage->childPtrSize==0 ); + pIter = pCell; + + /* The next block of code is equivalent to: + ** + ** pIter += getVarint32(pIter, nPayload); + ** + ** The code is inlined to avoid a function call. + */ + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( (*pIter)>=0x80 && pIternKey); + ** + ** The code is inlined to avoid a function call. + */ + iKey = *pIter; + if( iKey>=0x80 ){ + u8 *pEnd = &pIter[7]; + iKey &= 0x7f; + while(1){ + iKey = (iKey<<7) | (*++pIter & 0x7f); + if( (*pIter)<0x80 ) break; + if( pIter>=pEnd ){ + iKey = (iKey<<8) | *++pIter; + break; + } + } + } + pIter++; + + pInfo->nKey = *(i64*)&iKey; pInfo->nPayload = nPayload; - pInfo->nHeader = n; + pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==pPage->maxLocal+1 ); - if( likely(nPayload<=pPage->maxLocal) ){ + if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - if( (pInfo->nSize = (u16)(n+nPayload))<4 ) pInfo->nSize = 4; + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; - pInfo->iOverflow = 0; }else{ - /* If the payload will not fit completely on the local page, we have - ** to decide how much to store locally and how much to spill onto - ** overflow pages. The strategy is to minimize the amount of unused - ** space on overflow pages while keeping the amount of local storage - ** in between minLocal and maxLocal. - ** - ** Warning: changing the way overflow payload is distributed in any - ** way will result in an incompatible file format. - */ - int minLocal; /* Minimum amount of payload held locally */ - int maxLocal; /* Maximum amount of payload held locally */ - int surplus; /* Overflow payload available for local storage */ - - minLocal = pPage->minLocal; - maxLocal = pPage->maxLocal; - surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize - 4); - testcase( surplus==maxLocal ); - testcase( surplus==maxLocal+1 ); - if( surplus <= maxLocal ){ - pInfo->nLocal = (u16)surplus; - }else{ - pInfo->nLocal = (u16)minLocal; - } - pInfo->iOverflow = (u16)(pInfo->nLocal + n); - pInfo->nSize = pInfo->iOverflow + 4; + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); + } +} +static void btreeParseCellPtrIndex( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + u8 *pIter; /* For scanning through pCell */ + u32 nPayload; /* Number of bytes of cell payload */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 || pPage->leaf==1 ); + assert( pPage->intKeyLeaf==0 ); + pIter = pCell + pPage->childPtrSize; + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIternKey = nPayload; + pInfo->nPayload = nPayload; + pInfo->pPayload = pIter; + testcase( nPayload==pPage->maxLocal ); + testcase( nPayload==pPage->maxLocal+1 ); + if( nPayload<=pPage->maxLocal ){ + /* This is the (easy) common case where the entire payload fits + ** on the local page. No overflow is required. + */ + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; + pInfo->nLocal = (u16)nPayload; + }else{ + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); } } -#define parseCell(pPage, iCell, pInfo) \ - btreeParseCellPtr((pPage), findCell((pPage), (iCell)), (pInfo)) static void btreeParseCell( MemPage *pPage, /* Page containing the cell */ int iCell, /* The cell index. First cell is 0 */ CellInfo *pInfo /* Fill in this structure */ ){ - parseCell(pPage, iCell, pInfo); + pPage->xParseCell(pPage, findCell(pPage, iCell), pInfo); } /* +** The following routines are implementations of the MemPage.xCellSize +** method. +** ** Compute the total number of bytes that a Cell needs in the cell ** data area of the btree-page. The return number includes the cell ** data header and the local payload, but not any overflow page or ** the space used by the cell pointer. +** +** cellSizePtrNoPayload() => table internal nodes +** cellSizePtr() => all index nodes & table leaf nodes */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = &pCell[pPage->childPtrSize]; - u32 nSize; + u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ #ifdef SQLITE_DEBUG /* The value returned by this function should always be the same as @@ -55475,29 +62559,31 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of ** this function verifies that this invariant is not violated. */ CellInfo debuginfo; - btreeParseCellPtr(pPage, pCell, &debuginfo); + pPage->xParseCell(pPage, pCell, &debuginfo); #endif + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIterintKey ){ - u8 *pEnd; - if( pPage->hasData ){ - pIter += getVarint32(pIter, nSize); - }else{ - nSize = 0; - } - /* pIter now points at the 64-bit integer key value, a variable length ** integer. The following block moves pIter to point at the first byte ** past the end of the key value. */ pEnd = &pIter[9]; while( (*pIter++)&0x80 && pItermaxLocal ); testcase( nSize==pPage->maxLocal+1 ); - if( nSize>pPage->maxLocal ){ + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + if( nSize<4 ) nSize = 4; + }else{ int minLocal = pPage->minLocal; nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); testcase( nSize==pPage->maxLocal ); @@ -55505,24 +62591,39 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ if( nSize>pPage->maxLocal ){ nSize = minLocal; } - nSize += 4; + nSize += 4 + (u16)(pIter - pCell); } - nSize += (u32)(pIter - pCell); - - /* The minimum size of any cell is 4 bytes. */ - if( nSize<4 ){ - nSize = 4; - } - - assert( nSize==debuginfo.nSize ); + assert( nSize==debuginfo.nSize || CORRUPT_DB ); return (u16)nSize; } +static u16 cellSizePtrNoPayload(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#else + UNUSED_PARAMETER(pPage); +#endif + + assert( pPage->childPtrSize==4 ); + pEnd = pIter + 9; + while( (*pIter++)&0x80 && pIterxCellSize(pPage, findCell(pPage, iCell)); } #endif @@ -55536,10 +62637,9 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ CellInfo info; if( *pRC ) return; assert( pCell!=0 ); - btreeParseCellPtr(pPage, pCell, &info); - assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); - if( info.iOverflow ){ - Pgno ovfl = get4byte(&pCell[info.iOverflow]); + pPage->xParseCell(pPage, pCell, &info); + if( info.nLocalpBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); } } @@ -55551,10 +62651,15 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ ** end of the page and all free space is collected into one ** big FreeBlk that occurs in between the header and cell ** pointer array and the cell content area. +** +** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a +** b-tree page so that there are no freeblocks or fragment bytes, all +** unused bytes are contained in the unallocated space region, and all +** cells are packed tightly at the end of the page. */ static int defragmentPage(MemPage *pPage){ int i; /* Loop counter */ - int pc; /* Address of a i-th cell */ + int pc; /* Address of the i-th cell */ int hdr; /* Offset to the page header */ int size; /* Size of a cell */ int usableSize; /* Number of usable bytes on a page */ @@ -55563,6 +62668,7 @@ static int defragmentPage(MemPage *pPage){ int nCell; /* Number of cells on the page */ unsigned char *data; /* The page data */ unsigned char *temp; /* Temp area for cell content */ + unsigned char *src; /* Source of content */ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ @@ -55572,15 +62678,13 @@ static int defragmentPage(MemPage *pPage){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - data = pPage->aData; + temp = 0; + src = data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); usableSize = pPage->pBt->usableSize; - cbrk = get2byte(&data[hdr+5]); - memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); cbrk = usableSize; iCellFirst = cellOffset + 2*nCell; iCellLast = usableSize - 4; @@ -55590,31 +62694,31 @@ static int defragmentPage(MemPage *pPage){ pc = get2byte(pAddr); testcase( pc==iCellFirst ); testcase( pc==iCellLast ); -#if !defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) /* These conditions have already been verified in btreeInitPage() - ** if SQLITE_ENABLE_OVERSIZE_CELL_CHECK is defined + ** if PRAGMA cell_size_check=ON. */ if( pciCellLast ){ return SQLITE_CORRUPT_BKPT; } -#endif assert( pc>=iCellFirst && pc<=iCellLast ); - size = cellSizePtr(pPage, &temp[pc]); + size = pPage->xCellSize(pPage, &src[pc]); cbrk -= size; -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - if( cbrkusableSize ){ return SQLITE_CORRUPT_BKPT; } -#endif assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); testcase( cbrk+size==usableSize ); testcase( pc+size==usableSize ); - memcpy(&data[cbrk], &temp[pc], size); put2byte(pAddr, cbrk); + if( temp==0 ){ + int x; + if( cbrk==pc ) continue; + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + x = get2byte(&data[hdr+5]); + memcpy(&temp[x], &data[x], (cbrk+size) - x); + src = temp; + } + memcpy(&data[cbrk], &src[pc], size); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); @@ -55629,6 +62733,70 @@ static int defragmentPage(MemPage *pPage){ return SQLITE_OK; } +/* +** Search the free-list on page pPg for space to store a cell nByte bytes in +** size. If one can be found, return a pointer to the space and remove it +** from the free-list. +** +** If no suitable space can be found on the free-list, return NULL. +** +** This function may detect corruption within pPg. If corruption is +** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. +** +** Slots on the free list that are between 1 and 3 bytes larger than nByte +** will be ignored if adding the extra space to the fragmentation count +** causes the fragmentation count to exceed 60. +*/ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr = hdr + 1; + int pc = get2byte(&aData[iAddr]); + int x; + int usableSize = pPg->pBt->usableSize; + + assert( pc>0 ); + do{ + int size; /* Size of the free slot */ + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ + if( pc>usableSize-4 || pc=0 ){ + testcase( x==4 ); + testcase( x==3 ); + if( pc < pPg->cellOffset+2*pPg->nCell || size+pc > usableSize ){ + *pRc = SQLITE_CORRUPT_BKPT; + return 0; + }else if( x<4 ){ + /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total + ** number of bytes in fragments may not exceed 60. */ + if( aData[hdr+7]>57 ) return 0; + + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + iAddr = pc; + pc = get2byte(&aData[pc]); + }while( pc ); + + return 0; +} + /* ** Allocate nByte bytes of space from within the B-Tree page passed ** as the first argument. Write into *pIdx the index into pPage->aData[] @@ -55645,11 +62813,9 @@ static int defragmentPage(MemPage *pPage){ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ - int nFrag; /* Number of fragmented bytes on pPage */ int top; /* First byte of cell content area */ + int rc = SQLITE_OK; /* Integer return code */ int gap; /* First byte of gap between cell pointers and cell content */ - int rc; /* Integer return code */ - int usableSize; /* Usable size of the page */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); @@ -55657,62 +62823,50 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - usableSize = pPage->pBt->usableSize; - assert( nByte < usableSize-8 ); + assert( nByte < (int)(pPage->pBt->usableSize-8) ); - nFrag = data[hdr+7]; assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; - top = get2byteNotZero(&data[hdr+5]); - if( gap>top ) return SQLITE_CORRUPT_BKPT; - testcase( gap+2==top ); - testcase( gap+1==top ); - testcase( gap==top ); - - if( nFrag>=60 ){ - /* Always defragment highly fragmented pages */ - rc = defragmentPage(pPage); - if( rc ) return rc; - top = get2byteNotZero(&data[hdr+5]); - }else if( gap+2<=top ){ - /* Search the freelist looking for a free slot big enough to satisfy - ** the request. The allocation is made from the first free slot in - ** the list that is large enough to accommodate it. - */ - int pc, addr; - for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&data[addr], &data[pc], 2); - data[hdr+7] = (u8)(nFrag + x); - }else if( size+pc > usableSize ){ - return SQLITE_CORRUPT_BKPT; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&data[pc+2], x); - } - *pIdx = pc + x; - return SQLITE_OK; - } + assert( gap<=65536 ); + /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size + ** and the reserved space is zero (the usual value for reserved space) + ** then the cell content offset of an empty page wants to be 65536. + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ + top = get2byte(&data[hdr+5]); + assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */ + if( gap>top ){ + if( top==0 && pPage->pBt->usableSize==65536 ){ + top = 65536; + }else{ + return SQLITE_CORRUPT_BKPT; } } - /* Check to make sure there is enough space in the gap to satisfy - ** the allocation. If not, defragment. + /* If there is enough space between gap and top for one more cell pointer + ** array entry offset, and if the freelist is not empty, then search the + ** freelist looking for a free slot big enough to satisfy the request. + */ + testcase( gap+2==top ); + testcase( gap+1==top ); + testcase( gap==top ); + if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( pSpace ){ + assert( pSpace>=data && (pSpace - data)<65536 ); + *pIdx = (int)(pSpace - data); + return SQLITE_OK; + }else if( rc ){ + return rc; + } + } + + /* The request could not be fulfilled using a freelist slot. Check + ** to see if defragmentation is necessary. */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ + assert( pPage->nCell>0 || CORRUPT_DB ); rc = defragmentPage(pPage); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); @@ -55735,90 +62889,104 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ /* ** Return a section of the pPage->aData to the freelist. -** The first byte of the new free block is pPage->aDisk[start] -** and the size of the block is "size" bytes. +** The first byte of the new free block is pPage->aData[iStart] +** and the size of the block is iSize bytes. ** -** Most of the effort here is involved in coalesing adjacent -** free blocks into a single big free block. +** Adjacent freeblocks are coalesced. +** +** Note that even though the freeblock list was checked by btreeInitPage(), +** that routine will not detect overlap between cells or freeblocks. Nor +** does it detect cells or freeblocks that encrouch into the reserved bytes +** at the end of the page. So do additional corruption checks inside this +** routine and return SQLITE_CORRUPT if any problems are found. */ -static int freeSpace(MemPage *pPage, int start, int size){ - int addr, pbegin, hdr; - int iLast; /* Largest possible freeblock offset */ - unsigned char *data = pPage->aData; +static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ + u16 iPtr; /* Address of ptr to next freeblock */ + u16 iFreeBlk; /* Address of the next freeblock */ + u8 hdr; /* Page header size. 0 or 100 */ + u8 nFrag = 0; /* Reduction in fragmentation */ + u16 iOrigSize = iSize; /* Original value of iSize */ + u32 iLast = pPage->pBt->usableSize-4; /* Largest possible freeblock offset */ + u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ + unsigned char *data = pPage->aData; /* Page content */ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( start>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( (start + size) <= (int)pPage->pBt->usableSize ); + assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( size>=0 ); /* Minimum cell size is 4 */ + assert( iSize>=4 ); /* Minimum cell size is 4 */ + assert( iStart<=iLast ); + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){ - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ - memset(&data[start], 0, size); + memset(&data[iStart], 0, iSize); } - /* Add the space back into the linked list of freeblocks. Note that - ** even though the freeblock list was checked by btreeInitPage(), - ** btreeInitPage() did not detect overlapping cells or - ** freeblocks that overlapped cells. Nor does it detect when the - ** cell content area exceeds the value in the page header. If these - ** situations arise, then subsequent insert operations might corrupt - ** the freelist. So we do need to check for corruption while scanning - ** the freelist. + /* The list of freeblocks must be in ascending order. Find the + ** spot on the list where iStart should be inserted. */ hdr = pPage->hdrOffset; - addr = hdr + 1; - iLast = pPage->pBt->usableSize - 4; - assert( start<=iLast ); - while( (pbegin = get2byte(&data[addr]))0 ){ - if( pbeginiLast ){ - return SQLITE_CORRUPT_BKPT; - } - assert( pbegin>addr || pbegin==0 ); - put2byte(&data[addr], start); - put2byte(&data[start], pbegin); - put2byte(&data[start+2], size); - pPage->nFree = pPage->nFree + (u16)size; - - /* Coalesce adjacent free blocks */ - addr = hdr + 1; - while( (pbegin = get2byte(&data[addr]))>0 ){ - int pnext, psize, x; - assert( pbegin>addr ); - assert( pbegin <= (int)pPage->pBt->usableSize-4 ); - pnext = get2byte(&data[pbegin]); - psize = get2byte(&data[pbegin+2]); - if( pbegin + psize + 3 >= pnext && pnext>0 ){ - int frag = pnext - (pbegin+psize); - if( (frag<0) || (frag>(int)data[hdr+7]) ){ + iPtr = hdr + 1; + if( data[iPtr+1]==0 && data[iPtr]==0 ){ + iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ + }else{ + while( (iFreeBlk = get2byte(&data[iPtr]))iLast ) return SQLITE_CORRUPT_BKPT; + assert( iFreeBlk>iPtr || iFreeBlk==0 ); + + /* At this point: + ** iFreeBlk: First freeblock after iStart, or zero if none + ** iPtr: The address of a pointer to iFreeBlk + ** + ** Check to see if iFreeBlk should be coalesced onto the end of iStart. + */ + if( iFreeBlk && iEnd+3>=iFreeBlk ){ + nFrag = iFreeBlk - iEnd; + if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT; + iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); + if( iEnd > pPage->pBt->usableSize ) return SQLITE_CORRUPT_BKPT; + iSize = iEnd - iStart; + iFreeBlk = get2byte(&data[iFreeBlk]); + } + + /* If iPtr is another freeblock (that is, if iPtr is not the freelist + ** pointer in the page header) then check to see if iStart should be + ** coalesced onto the end of iPtr. + */ + if( iPtr>hdr+1 ){ + int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); + if( iPtrEnd+3>=iStart ){ + if( iPtrEnd>iStart ) return SQLITE_CORRUPT_BKPT; + nFrag += iStart - iPtrEnd; + iSize = iEnd - iPtr; + iStart = iPtr; + } + } + if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_BKPT; + data[hdr+7] -= nFrag; } - - /* If the cell content area begins with a freeblock, remove it. */ - if( data[hdr+1]==data[hdr+5] && data[hdr+2]==data[hdr+6] ){ - int top; - pbegin = get2byte(&data[hdr+1]); - memcpy(&data[hdr+1], &data[pbegin], 2); - top = get2byte(&data[hdr+5]) + get2byte(&data[pbegin+2]); - put2byte(&data[hdr+5], top); + if( iStart==get2byte(&data[hdr+5]) ){ + /* The new freeblock is at the beginning of the cell content area, + ** so just extend the cell content area rather than create another + ** freelist entry */ + if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_BKPT; + put2byte(&data[hdr+1], iFreeBlk); + put2byte(&data[hdr+5], iEnd); + }else{ + /* Insert the new freeblock into the freelist */ + put2byte(&data[iPtr], iStart); + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); } - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + pPage->nFree += iOrigSize; return SQLITE_OK; } @@ -55842,18 +63010,41 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); flagByte &= ~PTF_LEAF; pPage->childPtrSize = 4-4*pPage->leaf; + pPage->xCellSize = cellSizePtr; pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an + ** interior table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); + /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a + ** leaf table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; - pPage->hasData = pPage->leaf; + if( pPage->leaf ){ + pPage->intKeyLeaf = 1; + pPage->xParseCell = btreeParseCellPtr; + }else{ + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrNoPayload; + pPage->xParseCell = btreeParseCellPtrNoPayload; + } pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ + /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an + ** interior index b-tree page. */ + assert( (PTF_ZERODATA)==2 ); + /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a + ** leaf index b-tree page. */ + assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; - pPage->hasData = 0; + pPage->intKeyLeaf = 0; + pPage->xParseCell = btreeParseCellPtrIndex; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ + /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is + ** an error. */ return SQLITE_CORRUPT_BKPT; } pPage->max1bytePayload = pBt->max1bytePayload; @@ -55872,6 +63063,7 @@ static int decodeFlags(MemPage *pPage, int flagByte){ static int btreeInitPage(MemPage *pPage){ assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); @@ -55893,21 +63085,34 @@ static int btreeInitPage(MemPage *pPage){ hdr = pPage->hdrOffset; data = pPage->aData; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; + pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; pPage->aDataEnd = &data[usableSize]; pPage->aCellIdx = &data[cellOffset]; + pPage->aDataOfst = &data[pPage->childPtrSize]; + /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates + ** the start of the cell content area. A zero value for this integer is + ** interpreted as 65536. */ top = get2byteNotZero(&data[hdr+5]); + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ return SQLITE_CORRUPT_BKPT; } testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB ); /* A malformed database page might cause us to read past the end ** of page when parsing a cell. @@ -55918,20 +63123,19 @@ static int btreeInitPage(MemPage *pPage){ */ iCellFirst = cellOffset + 2*pPage->nCell; iCellLast = usableSize - 4; -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - { + if( pBt->db->flags & SQLITE_CellSizeCk ){ int i; /* Index into the cell pointer array */ int sz; /* Size of a cell */ if( !pPage->leaf ) iCellLast--; for(i=0; inCell; i++){ - pc = get2byte(&data[cellOffset+i*2]); + pc = get2byteAligned(&data[cellOffset+i*2]); testcase( pc==iCellFirst ); testcase( pc==iCellLast ); if( pciCellLast ){ return SQLITE_CORRUPT_BKPT; } - sz = cellSizePtr(pPage, &data[pc]); + sz = pPage->xCellSize(pPage, &data[pc]); testcase( pc+sz==usableSize ); if( pc+sz>usableSize ){ return SQLITE_CORRUPT_BKPT; @@ -55939,15 +63143,21 @@ static int btreeInitPage(MemPage *pPage){ } if( !pPage->leaf ) iCellLast++; } -#endif - /* Compute the total free space on the page */ + /* Compute the total free space on the page + ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the + ** start of the first freeblock on the page, or is zero if there are no + ** freeblocks. */ pc = get2byte(&data[hdr+1]); - nFree = data[hdr+7] + top; + nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ while( pc>0 ){ u16 next, size; if( pciCellLast ){ - /* Start of free block is off the page */ + /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will + ** always be at least one cell before the first freeblock. + ** + ** Or, the freeblock is off the end of the page + */ return SQLITE_CORRUPT_BKPT; } next = get2byte(&data[pc]); @@ -56005,6 +63215,7 @@ static void zeroPage(MemPage *pPage, int flags){ pPage->cellOffset = first; pPage->aDataEnd = &data[pBt->usableSize]; pPage->aCellIdx = &data[first]; + pPage->aDataOfst = &data[pPage->childPtrSize]; pPage->nOverflow = 0; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); @@ -56019,20 +63230,23 @@ static void zeroPage(MemPage *pPage, int flags){ */ static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ MemPage *pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); - pPage->aData = sqlite3PagerGetData(pDbPage); - pPage->pDbPage = pDbPage; - pPage->pBt = pBt; - pPage->pgno = pgno; - pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; + if( pgno!=pPage->pgno ){ + pPage->aData = sqlite3PagerGetData(pDbPage); + pPage->pDbPage = pDbPage; + pPage->pBt = pBt; + pPage->pgno = pgno; + pPage->hdrOffset = pgno==1 ? 100 : 0; + } + assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); return pPage; } /* ** Get a page from the pager. Initialize the MemPage.pBt and -** MemPage.aData elements if needed. +** MemPage.aData elements if needed. See also: btreeGetUnusedPage(). ** -** If the noContent flag is set, it means that we do not care about -** the content of the page at this time. So do not go to the disk +** If the PAGER_GET_NOCONTENT flag is set, it means that we do not care +** about the content of the page at this time. So do not go to the disk ** to fetch the content. Just fill in the content with zeros for now. ** If in the future we call sqlite3PagerWrite() on this page, that ** means we have started to be concerned about content and the disk @@ -56049,7 +63263,7 @@ static int btreeGetPage( assert( flags==0 || flags==PAGER_GET_NOCONTENT || flags==PAGER_GET_READONLY ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); + rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -56084,35 +63298,63 @@ SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ } /* -** Get a page from the pager and initialize it. This routine is just a -** convenience wrapper around separate calls to btreeGetPage() and -** btreeInitPage(). +** Get a page from the pager and initialize it. ** -** If an error occurs, then the value *ppPage is set to is undefined. It +** If pCur!=0 then the page is being fetched as part of a moveToChild() +** call. Do additional sanity checking on the page in this case. +** And if the fetch fails, this routine must decrement pCur->iPage. +** +** The page is fetched as read-write unless pCur is not NULL and is +** a read-only cursor. +** +** If an error occurs, then *ppPage is undefined. It ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( BtShared *pBt, /* The database file */ Pgno pgno, /* Number of the page to get */ MemPage **ppPage, /* Write the page pointer here */ - int bReadonly /* PAGER_GET_READONLY or 0 */ + BtCursor *pCur, /* Cursor to receive the page, or NULL */ + int bReadOnly /* True for a read-only page */ ){ int rc; + DbPage *pDbPage; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( bReadonly==PAGER_GET_READONLY || bReadonly==0 ); + assert( pCur==0 || ppPage==&pCur->apPage[pCur->iPage] ); + assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); + assert( pCur==0 || pCur->iPage>0 ); if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = btreeGetPage(pBt, pgno, ppPage, bReadonly); - if( rc==SQLITE_OK && (*ppPage)->isInit==0 ){ - rc = btreeInitPage(*ppPage); - if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - } + goto getAndInitPage_error; + } + rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); + if( rc ){ + goto getAndInitPage_error; + } + *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + if( (*ppPage)->isInit==0 ){ + btreePageFromDbPage(pDbPage, pgno, pBt); + rc = btreeInitPage(*ppPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + goto getAndInitPage_error; } } + assert( (*ppPage)->pgno==pgno ); + assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); + /* If obtaining a child page for a cursor, we must verify that the page is + ** compatible with the root page. */ + if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ + rc = SQLITE_CORRUPT_BKPT; + releasePage(*ppPage); + goto getAndInitPage_error; + } + return SQLITE_OK; + +getAndInitPage_error: + if( pCur ) pCur->iPage--; testcase( pgno==0 ); assert( pgno!=0 || rc==SQLITE_CORRUPT ); return rc; @@ -56122,17 +63364,48 @@ static int getAndInitPage( ** Release a MemPage. This should be called once for each prior ** call to btreeGetPage. */ -static void releasePage(MemPage *pPage){ - if( pPage ){ - assert( pPage->aData ); - assert( pPage->pBt ); - assert( pPage->pDbPage!=0 ); - assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); - assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - sqlite3PagerUnrefNotNull(pPage->pDbPage); - } +static void releasePageNotNull(MemPage *pPage){ + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefNotNull(pPage->pDbPage); } +static void releasePage(MemPage *pPage){ + if( pPage ) releasePageNotNull(pPage); +} + +/* +** Get an unused page. +** +** This works just like btreeGetPage() with the addition: +** +** * If the page is already in use for some other purpose, immediately +** release it and return an SQLITE_CURRUPT error. +** * Make sure the isInit flag is clear +*/ +static int btreeGetUnusedPage( + BtShared *pBt, /* The btree */ + Pgno pgno, /* Number of the page to fetch */ + MemPage **ppPage, /* Return the page in this parameter */ + int flags /* PAGER_GET_NOCONTENT or PAGER_GET_READONLY */ +){ + int rc = btreeGetPage(pBt, pgno, ppPage, flags); + if( rc==SQLITE_OK ){ + if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ + releasePage(*ppPage); + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; + } + (*ppPage)->isInit = 0; + }else{ + *ppPage = 0; + } + return rc; +} + /* ** During a rollback, when the pager reloads information into the cache @@ -56240,7 +63513,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( } p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } p->inTrans = TRANS_NONE; p->db = db; @@ -56256,16 +63529,18 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( */ if( isTempDb==0 && (isMemdb==0 || (vfsFlags&SQLITE_OPEN_URI)!=0) ){ if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ + int nFilename = sqlite3Strlen30(zFilename)+1; int nFullPathname = pVfs->mxPathname+1; - char *zFullPathname = sqlite3Malloc(nFullPathname); + char *zFullPathname = sqlite3Malloc(MAX(nFullPathname,nFilename)); MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) + p->sharable = 1; if( !zFullPathname ){ sqlite3_free(p); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( isMemdb ){ - memcpy(zFullPathname, zFilename, sqlite3Strlen30(zFilename)+1); + memcpy(zFullPathname, zFilename, nFilename); }else{ rc = sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); @@ -56322,15 +63597,15 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( ** the right size. This is to guard against size changes that result ** when compiling on a different architecture. */ - assert( sizeof(i64)==8 || sizeof(i64)==4 ); - assert( sizeof(u64)==8 || sizeof(u64)==4 ); + assert( sizeof(i64)==8 ); + assert( sizeof(u64)==8 ); assert( sizeof(u32)==4 ); assert( sizeof(u16)==2 ); assert( sizeof(Pgno)==4 ); pBt = sqlite3MallocZero( sizeof(*pBt) ); if( pBt==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto btree_open_out; } rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, @@ -56353,6 +63628,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #ifdef SQLITE_SECURE_DELETE pBt->btsFlags |= BTS_SECURE_DELETE; #endif + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ @@ -56371,6 +63649,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #endif nReserve = 0; }else{ + /* EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ nReserve = zDbHeader[20]; pBt->btsFlags |= BTS_PAGESIZE_FIXED; #ifndef SQLITE_OMIT_AUTOVACUUM @@ -56386,15 +63667,14 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) /* Add the new BtShared object to the linked list sharable BtShareds. */ + pBt->nRef = 1; if( p->sharable ){ MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) - pBt->nRef = 1; MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);) if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); if( pBt->mutex==0 ){ - rc = SQLITE_NOMEM; - db->mallocFailed = 0; + rc = SQLITE_NOMEM_BKPT; goto btree_open_out; } } @@ -56417,12 +63697,12 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( for(i=0; inDb; i++){ if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ while( pSib->pPrev ){ pSib = pSib->pPrev; } - if( p->pBtpBt ){ + if( (uptr)p->pBt<(uptr)pSib->pBt ){ p->pNext = pSib; p->pPrev = 0; pSib->pPrev = p; }else{ - while( pSib->pNext && pSib->pNext->pBtpBt ){ + while( pSib->pNext && (uptr)pSib->pNext->pBt<(uptr)p->pBt ){ pSib = pSib->pNext; } p->pNext = pSib->pNext; @@ -56460,6 +63740,7 @@ btree_open_out: assert( sqlite3_mutex_held(mutexOpen) ); sqlite3_mutex_leave(mutexOpen); } + assert( rc!=SQLITE_OK || sqlite3BtreeConnectionCount(*ppBtree)>0 ); return rc; } @@ -56505,7 +63786,8 @@ static int removeFromSharingList(BtShared *pBt){ /* ** Make sure pBt->pTmpSpace points to an allocation of -** MX_CELL_SIZE(pBt) bytes. +** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child +** pointer. */ static void allocateTempSpace(BtShared *pBt){ if( !pBt->pTmpSpace ){ @@ -56520,8 +63802,16 @@ static void allocateTempSpace(BtShared *pBt){ ** it into a database page. This is not actually a problem, but it ** does cause a valgrind error when the 1 or 2 bytes of unitialized ** data is passed to system call write(). So to avoid this error, - ** zero the first 4 bytes of temp space here. */ - if( pBt->pTmpSpace ) memset(pBt->pTmpSpace, 0, 4); + ** zero the first 4 bytes of temp space here. + ** + ** Also: Provide four bytes of initialized space before the + ** beginning of pTmpSpace as an area available to prepend the + ** left-child pointer to the beginning of a cell. + */ + if( pBt->pTmpSpace ){ + memset(pBt->pTmpSpace, 0, 8); + pBt->pTmpSpace += 4; + } } } @@ -56529,8 +63819,11 @@ static void allocateTempSpace(BtShared *pBt){ ** Free the pBt->pTmpSpace allocation */ static void freeTempSpace(BtShared *pBt){ - sqlite3PageFree( pBt->pTmpSpace); - pBt->pTmpSpace = 0; + if( pBt->pTmpSpace ){ + pBt->pTmpSpace -= 4; + sqlite3PageFree(pBt->pTmpSpace); + pBt->pTmpSpace = 0; + } } /* @@ -56556,7 +63849,7 @@ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. */ - sqlite3BtreeRollback(p, SQLITE_OK); + sqlite3BtreeRollback(p, SQLITE_OK, 0); sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree @@ -56592,19 +63885,11 @@ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ } /* -** Change the limit on the number of pages allowed in the cache. -** -** The maximum number of cache pages is set to the absolute -** value of mxPage. If mxPage is negative, the pager will -** operate asynchronously - it will not stop to do fsync()s -** to insure data is written to the disk surface before -** continuing. Transactions still work if synchronous is off, -** and the database cannot be corrupted if this program -** crashes. But if the operating system crashes or there is -** an abrupt power failure when synchronous is off, the database -** could be left in an inconsistent and unrecoverable state. -** Synchronous is on by default so database corruption is not -** normally a worry. +** Change the "soft" limit on the number of pages in the cache. +** Unused and unmodified pages will be recycled when the number of +** pages in the cache exceeds this soft limit. But the size of the +** cache is allowed to grow larger than this limit if it contains +** dirty pages or pages still in active use. */ SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ BtShared *pBt = p->pBt; @@ -56615,6 +63900,26 @@ SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +/* +** Change the "spill" limit on the number of pages in the cache. +** If the number of pages exceeds this limit during a write transaction, +** the pager might attempt to "spill" pages to the journal early in +** order to free up memory. +** +** The value returned is the current spill size. If zero is passed +** as an argument, no changes are made to the spill size setting, so +** using mxPage of 0 is a way to query the current spill size. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree *p, int mxPage){ + BtShared *pBt = p->pBt; + int res; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + res = sqlite3PagerSetSpillsize(pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return res; +} + #if SQLITE_MAX_MMAP_SIZE>0 /* ** Change the limit on the amount of the database file that may be @@ -56652,21 +63957,6 @@ SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags( } #endif -/* -** Return TRUE if the given btree is set to safety level 1. In other -** words, return TRUE if no sync() occurs on the disk files. -*/ -SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree *p){ - BtShared *pBt = p->pBt; - int rc; - assert( sqlite3_mutex_held(p->db->mutex) ); - sqlite3BtreeEnter(p); - assert( pBt && pBt->pPager ); - rc = sqlite3PagerNosync(pBt->pPager); - sqlite3BtreeLeave(p); - return rc; -} - /* ** Change the default pages size and the number of reserved bytes per page. ** Or, if the page size has already been fixed, return SQLITE_READONLY @@ -56692,6 +63982,9 @@ SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, BtShared *pBt = p->pBt; assert( nReserve>=-1 && nReserve<=255 ); sqlite3BtreeEnter(p); +#if SQLITE_HAS_CODEC + if( nReserve>pBt->optimalReserve ) pBt->optimalReserve = (u8)nReserve; +#endif if( pBt->btsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); return SQLITE_READONLY; @@ -56703,7 +63996,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && ((pageSize-1)&pageSize)==0 ){ assert( (pageSize & 7)==0 ); - assert( !pBt->pPage1 && !pBt->pCursor ); + assert( !pBt->pCursor ); pBt->pageSize = (u32)pageSize; freeTempSpace(pBt); } @@ -56721,7 +64014,6 @@ SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree *p){ return p->pBt->pageSize; } -#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) /* ** This function is similar to sqlite3BtreeGetReserve(), except that it ** may only be called if it is guaranteed that the b-tree mutex is already @@ -56734,25 +64026,33 @@ SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree *p){ ** database handle that owns *p, causing undefined behavior. */ SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p){ + int n; assert( sqlite3_mutex_held(p->pBt->mutex) ); - return p->pBt->pageSize - p->pBt->usableSize; + n = p->pBt->pageSize - p->pBt->usableSize; + return n; } -#endif /* SQLITE_HAS_CODEC || SQLITE_DEBUG */ -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) /* ** Return the number of bytes of space at the end of every page that ** are intentually left unused. This is the "reserved" space that is ** sometimes used by extensions. +** +** If SQLITE_HAS_MUTEX is defined then the number returned is the +** greater of the current reserved space and the maximum requested +** reserve space. */ -SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree *p){ +SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree *p){ int n; sqlite3BtreeEnter(p); - n = p->pBt->pageSize - p->pBt->usableSize; + n = sqlite3BtreeGetReserveNoMutex(p); +#ifdef SQLITE_HAS_CODEC + if( npBt->optimalReserve ) n = p->pBt->optimalReserve; +#endif sqlite3BtreeLeave(p); return n; } + /* ** Set the maximum page count for a database if mxPage is positive. ** No changes are made if mxPage is 0 or negative. @@ -56783,7 +64083,6 @@ SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ sqlite3BtreeLeave(p); return b; } -#endif /* !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) */ /* ** Change the 'auto-vacuum' property of the database. If the 'autoVacuum' @@ -56868,6 +64167,9 @@ static int lockBtree(BtShared *pBt){ u32 usableSize; u8 *page1 = pPage1->aData; rc = SQLITE_NOTADB; + /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins + ** with the following 16 bytes (in hex): 53 51 4c 69 74 65 20 66 6f 72 6d + ** 61 74 20 33 00. */ if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; } @@ -56900,23 +64202,45 @@ static int lockBtree(BtShared *pBt){ rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); if( rc!=SQLITE_OK ){ goto page1_init_failed; - }else if( isOpen==0 ){ - releasePage(pPage1); - return SQLITE_OK; + }else{ +#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS + sqlite3 *db; + Db *pDb; + if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ + while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } + if( pDb->bSyncSet==0 + && pDb->safety_level==SQLITE_DEFAULT_SYNCHRONOUS+1 + ){ + pDb->safety_level = SQLITE_DEFAULT_WAL_SYNCHRONOUS+1; + sqlite3PagerSetFlags(pBt->pPager, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); + } + } +#endif + if( isOpen==0 ){ + releasePage(pPage1); + return SQLITE_OK; + } } rc = SQLITE_NOTADB; } #endif - /* The maximum embedded fraction must be exactly 25%. And the minimum - ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. + /* EVIDENCE-OF: R-15465-20813 The maximum and minimum embedded payload + ** fractions and the leaf payload fraction values must be 64, 32, and 32. + ** ** The original design allowed these amounts to vary, but as of ** version 3.6.0, we require them to be fixed. */ if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ goto page1_init_failed; } + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pageSize = (page1[16]<<8) | (page1[17]<<16); + /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two + ** between 512 and 65536 inclusive. */ if( ((pageSize-1)&pageSize)!=0 || pageSize>SQLITE_MAX_PAGE_SIZE || pageSize<=256 @@ -56924,6 +64248,13 @@ static int lockBtree(BtShared *pBt){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); + /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte + ** integer at offset 20 is the number of bytes of space at the end of + ** each page to reserve for extensions. + ** + ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ usableSize = pageSize - page1[20]; if( (u32)pageSize!=pBt->pageSize ){ /* After reading the first page of the database assuming a page size @@ -56944,6 +64275,9 @@ static int lockBtree(BtShared *pBt){ rc = SQLITE_CORRUPT_BKPT; goto page1_init_failed; } + /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to + ** be less than 480. In other words, if the page size is 512, then the + ** reserved space size cannot exceed 32. */ if( usableSize<480 ){ goto page1_init_failed; } @@ -56998,7 +64332,7 @@ page1_init_failed: ** false then all cursors are counted. ** ** For the purposes of this routine, a cursor is any cursor that -** is capable of reading or writing to the databse. Cursors that +** is capable of reading or writing to the database. Cursors that ** have been tripped into the CURSOR_FAULT state are not counted. */ static int countValidCursors(BtShared *pBt, int wrOnly){ @@ -57024,11 +64358,11 @@ static void unlockBtreeIfUnused(BtShared *pBt){ assert( sqlite3_mutex_held(pBt->mutex) ); assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ - assert( pBt->pPage1->aData ); + MemPage *pPage1 = pBt->pPage1; + assert( pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); - assert( pBt->pPage1->aData ); - releasePage(pBt->pPage1); pBt->pPage1 = 0; + releasePageNotNull(pPage1); } } @@ -57126,7 +64460,6 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** proceed. */ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ - sqlite3 *pBlock = 0; BtShared *pBt = p->pBt; int rc = SQLITE_OK; @@ -57149,27 +64482,30 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } #ifndef SQLITE_OMIT_SHARED_CACHE - /* If another database handle has already opened a write transaction - ** on this shared-btree structure and a second write transaction is - ** requested, return SQLITE_LOCKED. - */ - if( (wrflag && pBt->inTransaction==TRANS_WRITE) - || (pBt->btsFlags & BTS_PENDING)!=0 - ){ - pBlock = pBt->pWriter->db; - }else if( wrflag>1 ){ - BtLock *pIter; - for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ - if( pIter->pBtree!=p ){ - pBlock = pIter->pBtree->db; - break; + { + sqlite3 *pBlock = 0; + /* If another database handle has already opened a write transaction + ** on this shared-btree structure and a second write transaction is + ** requested, return SQLITE_LOCKED. + */ + if( (wrflag && pBt->inTransaction==TRANS_WRITE) + || (pBt->btsFlags & BTS_PENDING)!=0 + ){ + pBlock = pBt->pWriter->db; + }else if( wrflag>1 ){ + BtLock *pIter; + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->pBtree!=p ){ + pBlock = pIter->pBtree->db; + break; + } } } - } - if( pBlock ){ - sqlite3ConnectionBlocked(p->db, pBlock); - rc = SQLITE_LOCKED_SHAREDCACHE; - goto trans_begun; + if( pBlock ){ + sqlite3ConnectionBlocked(p->db, pBlock); + rc = SQLITE_LOCKED_SHAREDCACHE; + goto trans_begun; + } } #endif @@ -57333,20 +64669,22 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ u8 isInitOrig = pPage->isInit; int i; int nCell; + int rc; - btreeInitPage(pPage); + rc = btreeInitPage(pPage); + if( rc ) return rc; nCell = pPage->nCell; for(i=0; iaData+pPage->maskPage - && iFrom==get4byte(&pCell[info.iOverflow]) + pPage->xParseCell(pPage, pCell, &info); + if( info.nLocalaData+pPage->maskPage + && iFrom==get4byte(pCell+info.nSize-4) ){ - put4byte(&pCell[info.iOverflow], iTo); + put4byte(pCell+info.nSize-4, iTo); break; } }else{ @@ -57462,7 +64800,7 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); ** calling this function again), return SQLITE_DONE. Or, if an error ** occurs, return some other error code. ** -** More specificly, this function attempts to re-organize the database so +** More specifically, this function attempts to re-organize the database so ** that the last page of the file currently in use is no longer in use. ** ** Parameter nFin is the number of pages that this database would contain @@ -57470,7 +64808,7 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); ** ** If the bCommit parameter is non-zero, this function assumes that the ** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE -** or an error. bCommit is passed true for an auto-vacuum-on-commmit +** or an error. bCommit is passed true for an auto-vacuum-on-commit ** operation, or false for an incremental vacuum. */ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ @@ -57640,7 +64978,7 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){ static int autoVacuumCommit(BtShared *pBt){ int rc = SQLITE_OK; Pager *pPager = pBt->pPager; - VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) ); + VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); ) assert( sqlite3_mutex_held(pBt->mutex) ); invalidateAllOverflowCache(pBt); @@ -57824,6 +65162,7 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ sqlite3BtreeLeave(p); return rc; } + p->iDataVersion--; /* Compensate for pPager->iDataVersion++; */ pBt->inTransaction = TRANS_READ; btreeClearHasContent(pBt); } @@ -57849,60 +65188,91 @@ SQLITE_PRIVATE int sqlite3BtreeCommit(Btree *p){ /* ** This routine sets the state to CURSOR_FAULT and the error -** code to errCode for every cursor on BtShared that pBtree -** references. +** code to errCode for every cursor on any BtShared that pBtree +** references. Or if the writeOnly flag is set to 1, then only +** trip write cursors and leave read cursors unchanged. ** -** Every cursor is tripped, including cursors that belong -** to other database connections that happen to be sharing -** the cache with pBtree. +** Every cursor is a candidate to be tripped, including cursors +** that belong to other database connections that happen to be +** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. -** All cursors using the same cache must be tripped -** to prevent them from trying to use the btree after -** the rollback. The rollback may have deleted tables -** or moved root pages, so it is not sufficient to -** save the state of the cursor. The cursor must be -** invalidated. +** This routine gets called when a rollback occurs. If the writeOnly +** flag is true, then only write-cursors need be tripped - read-only +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are +** tripped. In general, writeOnly is false if the transaction being +** rolled back modified the database schema. In this case b-tree root +** pages may be moved or deleted from the database altogether, making +** it unsafe for read cursors to continue. +** +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, +** including all read-cursors are tripped. +** +** SQLITE_OK is returned if successful, or if an error occurs while +** saving a cursor position, an SQLite error code. */ -SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ +SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; - if( pBtree==0 ) return; - sqlite3BtreeEnter(pBtree); - for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - int i; - sqlite3BtreeClearCursor(p); - p->eState = CURSOR_FAULT; - p->skipNext = errCode; - for(i=0; i<=p->iPage; i++){ - releasePage(p->apPage[i]); - p->apPage[i] = 0; + int rc = SQLITE_OK; + + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); + if( pBtree ){ + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ + if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ + rc = saveCursorPosition(p); + if( rc!=SQLITE_OK ){ + (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); + break; + } + } + }else{ + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + } + for(i=0; i<=p->iPage; i++){ + releasePage(p->apPage[i]); + p->apPage[i] = 0; + } } + sqlite3BtreeLeave(pBtree); } - sqlite3BtreeLeave(pBtree); + return rc; } /* -** Rollback the transaction in progress. All cursors will be -** invalided by this operation. Any attempt to use a cursor -** that was open at the beginning of this operation will result -** in an error. +** Rollback the transaction in progress. +** +** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped). +** Only write cursors are tripped if writeOnly is true but all cursors are +** tripped if writeOnly is false. Any attempt to use +** a tripped cursor will result in an error. ** ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ -SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode){ +SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ int rc; BtShared *pBt = p->pBt; MemPage *pPage1; + assert( writeOnly==1 || writeOnly==0 ); + assert( tripCode==SQLITE_ABORT_ROLLBACK || tripCode==SQLITE_OK ); sqlite3BtreeEnter(p); if( tripCode==SQLITE_OK ){ rc = tripCode = saveAllCursors(pBt, 0, 0); + if( rc ) writeOnly = 0; }else{ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode); + int rc2 = sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + assert( rc==SQLITE_OK || (writeOnly==0 && rc2==SQLITE_OK) ); + if( rc2!=SQLITE_OK ) rc = rc2; } btreeIntegrity(p); @@ -57937,7 +65307,7 @@ SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode){ } /* -** Start a statement subtransaction. The subtransaction can can be rolled +** Start a statement subtransaction. The subtransaction can be rolled ** back independently of the main transaction. You must start a transaction ** before starting a subtransaction. The subtransaction is ended automatically ** if the main transaction commits or rolls back. @@ -58017,13 +65387,13 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ ** on the database already. If a write-cursor is requested, then ** the caller is assumed to have an open write transaction. ** -** If wrFlag==0, then the cursor can only be used for reading. -** If wrFlag==1, then the cursor can be used for reading or for -** writing if other conditions for writing are also met. These -** are the conditions that must be met in order for writing to -** be allowed: +** If the BTREE_WRCSR bit of wrFlag is clear, then the cursor can only +** be used for reading. If the BTREE_WRCSR bit is set, then the cursor +** can be used for reading or for writing if other conditions for writing +** are also met. These are the conditions that must be met in order +** for writing to be allowed: ** -** 1: The cursor must have been opened with wrFlag==1 +** 1: The cursor must have been opened with wrFlag containing BTREE_WRCSR ** ** 2: Other database connections that share the same pager cache ** but which are not in the READ_UNCOMMITTED state may not have @@ -58035,6 +65405,16 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ ** ** 4: There must be an active transaction. ** +** The BTREE_FORDELETE bit of wrFlag may optionally be set if BTREE_WRCSR +** is set. If FORDELETE is set, that is a hint to the implementation that +** this cursor will only be used to seek to and delete entries of an index +** as part of a larger DELETE statement. The FORDELETE hint is not used by +** this implementation. But in a hypothetical alternative storage engine +** in which index entries are automatically deleted when corresponding table +** rows are deleted, the FORDELETE flag is a hint that all SEEK and DELETE +** operations on this cursor can be no-ops and all READ operations can +** return a null row (2-bytes: 0x01 0x00). +** ** No checking is done to make sure that page iTable really is the ** root page of a b-tree. If it is not, then the cursor acquired ** will not work correctly. @@ -58050,24 +65430,30 @@ static int btreeCursor( BtCursor *pCur /* Space for new cursor */ ){ BtShared *pBt = p->pBt; /* Shared b-tree handle */ + BtCursor *pX; /* Looping over other all cursors */ assert( sqlite3BtreeHoldsMutex(p) ); - assert( wrFlag==0 || wrFlag==1 ); + assert( wrFlag==0 + || wrFlag==BTREE_WRCSR + || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) + ); /* The following assert statements verify that if this is a sharable ** b-tree database, the connection is holding the required table locks, ** and that no other connection has any open cursor that conflicts with ** this lock. */ - assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, wrFlag+1) ); + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) ); assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); /* Assert that the caller has opened the required transaction. */ assert( p->inTrans>TRANS_NONE ); assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); assert( pBt->pPage1 && pBt->pPage1->aData ); + assert( wrFlag==0 || (pBt->btsFlags & BTS_READ_ONLY)==0 ); - if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){ - return SQLITE_READONLY; + if( wrFlag ){ + allocateTempSpace(pBt); + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; } if( iTable==1 && btreePagecount(pBt)==0 ){ assert( wrFlag==0 ); @@ -58081,12 +65467,17 @@ static int btreeCursor( pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; pCur->pBt = pBt; - assert( wrFlag==0 || wrFlag==BTCF_WriteFlag ); - pCur->curFlags = wrFlag; - pCur->pNext = pBt->pCursor; - if( pCur->pNext ){ - pCur->pNext->pPrev = pCur; + pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; + pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; + /* If there are two or more cursors on the same btree, then all such + ** cursors *must* have the BTCF_Multiple flag set. */ + for(pX=pBt->pCursor; pX; pX=pX->pNext){ + if( pX->pgnoRoot==(Pgno)iTable ){ + pX->curFlags |= BTCF_Multiple; + pCur->curFlags |= BTCF_Multiple; + } } + pCur->pNext = pBt->pCursor; pBt->pCursor = pCur; pCur->eState = CURSOR_INVALID; return SQLITE_OK; @@ -58099,9 +65490,13 @@ SQLITE_PRIVATE int sqlite3BtreeCursor( BtCursor *pCur /* Write new cursor here */ ){ int rc; - sqlite3BtreeEnter(p); - rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); - sqlite3BtreeLeave(p); + if( iTable<1 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + } return rc; } @@ -58140,19 +65535,24 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ BtShared *pBt = pCur->pBt; sqlite3BtreeEnter(pBtree); sqlite3BtreeClearCursor(pCur); - if( pCur->pPrev ){ - pCur->pPrev->pNext = pCur->pNext; - }else{ + assert( pBt->pCursor!=0 ); + if( pBt->pCursor==pCur ){ pBt->pCursor = pCur->pNext; - } - if( pCur->pNext ){ - pCur->pNext->pPrev = pCur->pPrev; + }else{ + BtCursor *pPrev = pBt->pCursor; + do{ + if( pPrev->pNext==pCur ){ + pPrev->pNext = pCur->pNext; + break; + } + pPrev = pPrev->pNext; + }while( ALWAYS(pPrev) ); } for(i=0; i<=pCur->iPage; i++){ releasePage(pCur->apPage[i]); } unlockBtreeIfUnused(pBt); - sqlite3DbFree(pBtree->db, pCur->aOverflow); + sqlite3_free(pCur->aOverflow); /* sqlite3_free(pCur); */ sqlite3BtreeLeave(pBtree); } @@ -58166,13 +65566,6 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** ** BtCursor.info is a cache of the information in the current cell. ** Using this cache reduces the number of calls to btreeParseCell(). -** -** 2007-06-25: There is a bug in some versions of MSVC that cause the -** compiler to crash when getCellInfo() is implemented as a macro. -** But there is a measureable speed advantage to using the macro on gcc -** (when less compiler optimizations like -Os or -O0 are used and the -** compiler is not doing agressive inlining.) So we use a real function -** for MSVC and a macro for everything else. Ticket #2457. */ #ifndef NDEBUG static void assertCellInfo(BtCursor *pCur){ @@ -58185,28 +65578,15 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ #else #define assertCellInfo(x) #endif -#ifdef _MSC_VER - /* Use a real function in MSVC to work around bugs in that compiler. */ - static void getCellInfo(BtCursor *pCur){ - if( pCur->info.nSize==0 ){ - int iPage = pCur->iPage; - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); - pCur->curFlags |= BTCF_ValidNKey; - }else{ - assertCellInfo(pCur); - } +static SQLITE_NOINLINE void getCellInfo(BtCursor *pCur){ + if( pCur->info.nSize==0 ){ + int iPage = pCur->iPage; + pCur->curFlags |= BTCF_ValidNKey; + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); + }else{ + assertCellInfo(pCur); } -#else /* if not _MSC_VER */ - /* Use a macro in all other compilers so that the function is inlined */ -#define getCellInfo(pCur) \ - if( pCur->info.nSize==0 ){ \ - int iPage = pCur->iPage; \ - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ - pCur->curFlags |= BTCF_ValidNKey; \ - }else{ \ - assertCellInfo(pCur); \ - } -#endif /* _MSC_VER */ +} #ifndef NDEBUG /* The next routine used only within assert() statements */ /* @@ -58220,47 +65600,33 @@ SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor *pCur){ #endif /* NDEBUG */ /* -** Set *pSize to the size of the buffer needed to hold the value of -** the key for the current entry. If the cursor is not pointing -** to a valid entry, *pSize is set to 0. -** -** For a table with the INTKEY flag set, this routine returns the key -** itself, not the number of bytes in the key. -** -** The caller must position the cursor prior to invoking this routine. -** -** This routine cannot fail. It always returns SQLITE_OK. +** Return the value of the integer key or "rowid" for a table btree. +** This routine is only valid for a cursor that is pointing into a +** ordinary table btree. If the cursor points to an index btree or +** is invalid, the result of this routine is undefined. */ -SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ +SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID ); - if( pCur->eState!=CURSOR_VALID ){ - *pSize = 0; - }else{ - getCellInfo(pCur); - *pSize = pCur->info.nKey; - } - return SQLITE_OK; + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->curIntKey ); + getCellInfo(pCur); + return pCur->info.nKey; } /* -** Set *pSize to the number of bytes of data in the entry the -** cursor currently points to. +** Return the number of bytes of payload for the entry that pCur is +** currently pointing to. For table btrees, this will be the amount +** of data. For index btrees, this will be the size of the key. ** ** The caller must guarantee that the cursor is pointing to a non-NULL ** valid entry. In other words, the calling procedure must guarantee ** that the cursor has Cursor.eState==CURSOR_VALID. -** -** Failure is not possible. This function always returns SQLITE_OK. -** It might just as well be a procedure (returning void) but we continue -** to return an integer result code for historical reasons. */ -SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ +SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); getCellInfo(pCur); - *pSize = pCur->info.nData; - return SQLITE_OK; + return pCur->info.nPayload; } /* @@ -58388,7 +65754,7 @@ static int copyPayload( ** ** If the current cursor entry uses one or more overflow pages and the ** eOp argument is not 2, this function may allocate space for and lazily -** popluates the overflow page-list cache array (BtCursor.aOverflow). +** populates the overflow page-list cache array (BtCursor.aOverflow). ** Subsequent calls use this cache to make seeking to the supplied offset ** more efficient. ** @@ -58410,31 +65776,34 @@ static int accessPayload( ){ unsigned char *aPayload; int rc = SQLITE_OK; - u32 nKey; int iIdx = 0; MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - int bEnd; /* True if reading to end of data */ + unsigned char * const pBufStart = pBuf; + int bEnd; /* True if reading to end of data */ #endif assert( pPage ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); - assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ + assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); - aPayload = pCur->info.pCell + pCur->info.nHeader; - nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); + aPayload = pCur->info.pPayload; #ifdef SQLITE_DIRECT_OVERFLOW_READ - bEnd = (offset+amt==nKey+pCur->info.nData); + bEnd = offset+amt==pCur->info.nPayload; #endif + assert( offset+amt <= pCur->info.nPayload ); - if( NEVER(offset+amt > nKey+pCur->info.nData) - || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] - ){ - /* Trying to read or write past the end of the data is an error */ + assert( aPayload > pPage->aData ); + if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){ + /* Trying to read or write past the end of the data is an error. The + ** conditional above is really: + ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ** but is recast into its current form to avoid integer overflow problems + */ return SQLITE_CORRUPT_BKPT; } @@ -58452,6 +65821,7 @@ static int accessPayload( offset -= pCur->info.nLocal; } + if( rc==SQLITE_OK && amt>0 ){ const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ Pgno nextPage; @@ -58469,11 +65839,11 @@ static int accessPayload( if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; if( nOvfl>pCur->nOvflAlloc ){ - Pgno *aNew = (Pgno*)sqlite3DbRealloc( - pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno) + Pgno *aNew = (Pgno*)sqlite3Realloc( + pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; @@ -58489,7 +65859,9 @@ static int accessPayload( ** entry for the first required overflow page is valid, skip ** directly to it. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[offset/ovflSize] ){ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 + && pCur->aOverflow[offset/ovflSize] + ){ iIdx = (offset/ovflSize); nextPage = pCur->aOverflow[iIdx]; offset = (offset%ovflSize); @@ -58499,7 +65871,9 @@ static int accessPayload( /* If required, populate the overflow page-list cache. */ if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ - assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); pCur->aOverflow[iIdx] = nextPage; } @@ -58515,6 +65889,7 @@ static int accessPayload( */ assert( eOp!=2 ); assert( pCur->curFlags & BTCF_ValidOvfl ); + assert( pCur->pBtree->db==pBt->db ); if( pCur->aOverflow[iIdx+1] ){ nextPage = pCur->aOverflow[iIdx+1]; }else{ @@ -58542,6 +65917,7 @@ static int accessPayload( ** 4) there is no open write-transaction, and ** 5) the database is not a WAL database, ** 6) all data from the page is being read. + ** 7) at least 4 bytes have already been read into the output buffer ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds @@ -58552,10 +65928,12 @@ static int accessPayload( && (bEnd || a==ovflSize) /* (6) */ && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ - && pBt->pPage1->aData[19]==0x01 /* (5) */ + && 0==sqlite3PagerUseWal(pBt->pPager) /* (5) */ + && &pBuf[-4]>=pBufStart /* (7) */ ){ u8 aSave[4]; u8 *aWrite = &pBuf[-4]; + assert( aWrite>=pBufStart ); /* hence (7) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); nextPage = get4byte(aWrite); @@ -58565,7 +65943,7 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ @@ -58590,7 +65968,7 @@ static int accessPayload( /* ** Read part of the key associated with cursor pCur. Exactly -** "amt" bytes will be transfered into pBuf[]. The transfer +** "amt" bytes will be transferred into pBuf[]. The transfer ** begins at "offset". ** ** The caller must ensure that pCur is pointing to a valid row @@ -58626,7 +66004,7 @@ SQLITE_PRIVATE int sqlite3BtreeData(BtCursor *pCur, u32 offset, u32 amt, void *p } #endif - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); rc = restoreCursorPosition(pCur); if( rc==SQLITE_OK ){ assert( pCur->eState==CURSOR_VALID ); @@ -58660,14 +66038,19 @@ static const void *fetchPayload( BtCursor *pCur, /* Cursor pointing to entry to read from */ u32 *pAmt /* Write the number of available bytes here */ ){ + u32 amt; assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); assert( pCur->eState==CURSOR_VALID ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); assert( pCur->info.nSize>0 ); - *pAmt = pCur->info.nLocal; - return (void*)(pCur->info.pCell + pCur->info.nHeader); + assert( pCur->info.pPayload>pCur->apPage[pCur->iPage]->aData || CORRUPT_DB ); + assert( pCur->info.pPayloadapPage[pCur->iPage]->aDataEnd ||CORRUPT_DB); + amt = (int)(pCur->apPage[pCur->iPage]->aDataEnd - pCur->info.pPayload); + if( pCur->info.nLocalinfo.nLocal; + *pAmt = amt; + return (void*)pCur->info.pPayload; } @@ -58685,10 +66068,7 @@ static const void *fetchPayload( ** These routines is used to get quick access to key and data ** in the common case where no overflow pages are used. */ -SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor *pCur, u32 *pAmt){ - return fetchPayload(pCur, pAmt); -} -SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ +SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ return fetchPayload(pCur, pAmt); } @@ -58703,34 +66083,24 @@ SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ - int rc; - int i = pCur->iPage; - MemPage *pNewPage; BtShared *pBt = pCur->pBt; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageiPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage, - (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); - if( rc ) return rc; - pCur->apPage[i+1] = pNewPage; - pCur->aiIdx[i+1] = 0; - pCur->iPage++; - pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){ - return SQLITE_CORRUPT_BKPT; - } - return SQLITE_OK; + pCur->iPage++; + pCur->aiIdx[pCur->iPage] = 0; + return getAndInitPage(pBt, newPgno, &pCur->apPage[pCur->iPage], + pCur, pCur->curPagerFlags); } -#if 0 +#if SQLITE_DEBUG /* ** Page pParent is an internal (non-leaf) tree page. This function ** asserts that page number iChild is the left-child if the iIdx'th @@ -58739,6 +66109,8 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ ** the page. */ static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ + if( CORRUPT_DB ) return; /* The conditions tested below might not be true + ** in a corrupt database */ assert( iIdx<=pParent->nCell ); if( iIdx==pParent->nCell ){ assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild ); @@ -58759,29 +66131,19 @@ static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ ** the largest cell index. */ static void moveToParent(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage>0 ); assert( pCur->apPage[pCur->iPage] ); - - /* UPDATE: It is actually possible for the condition tested by the assert - ** below to be untrue if the database file is corrupt. This can occur if - ** one cursor has modified page pParent while a reference to it is held - ** by a second cursor. Which can only happen if a single page is linked - ** into more than one b-tree structure in a corrupt database. */ -#if 0 assertParentIndex( pCur->apPage[pCur->iPage-1], pCur->aiIdx[pCur->iPage-1], pCur->apPage[pCur->iPage]->pgno ); -#endif testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); - - releasePage(pCur->apPage[pCur->iPage]); - pCur->iPage--; pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + releasePageNotNull(pCur->apPage[pCur->iPage--]); } /* @@ -58809,7 +66171,7 @@ static int moveToRoot(BtCursor *pCur){ MemPage *pRoot; int rc = SQLITE_OK; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); @@ -58822,18 +66184,23 @@ static int moveToRoot(BtCursor *pCur){ } if( pCur->iPage>=0 ){ - while( pCur->iPage ) releasePage(pCur->apPage[pCur->iPage--]); + while( pCur->iPage ){ + assert( pCur->apPage[pCur->iPage]!=0 ); + releasePageNotNull(pCur->apPage[pCur->iPage--]); + } }else if( pCur->pgnoRoot==0 ){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ + assert( pCur->iPage==(-1) ); rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0], - (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); + 0, pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; } pCur->iPage = 0; + pCur->curIntKey = pCur->apPage[0]->intKey; } pRoot = pCur->apPage[0]; assert( pRoot->pgno==pCur->pgnoRoot ); @@ -58883,7 +66250,7 @@ static int moveToLeftmost(BtCursor *pCur){ int rc = SQLITE_OK; MemPage *pPage; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ assert( pCur->aiIdx[pCur->iPage]nCell ); @@ -58908,19 +66275,18 @@ static int moveToRightmost(BtCursor *pCur){ int rc = SQLITE_OK; MemPage *pPage = 0; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); - while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + while( !(pPage = pCur->apPage[pCur->iPage])->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); pCur->aiIdx[pCur->iPage] = pPage->nCell; rc = moveToChild(pCur, pgno); + if( rc ) return rc; } - if( rc==SQLITE_OK ){ - pCur->aiIdx[pCur->iPage] = pPage->nCell-1; - pCur->info.nSize = 0; - pCur->curFlags &= ~BTCF_ValidNKey; - } - return rc; + pCur->aiIdx[pCur->iPage] = pPage->nCell-1; + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & BTCF_ValidNKey)==0 ); + return SQLITE_OK; } /* Move the cursor to the first entry in the table. Return SQLITE_OK @@ -58930,7 +66296,7 @@ static int moveToRightmost(BtCursor *pCur){ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ int rc; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ @@ -58953,7 +66319,7 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ int rc; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); /* If the cursor already points to the last entry, this is a no-op. */ @@ -59018,6 +66384,8 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ ** *pRes>0 The cursor is left pointing at an entry that ** is larger than intKey/pIdxKey. ** +** For index tables, the pIdxKey->eqSeen field is set to 1 if there +** exists an entry in the table that exactly matches pIdxKey. */ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( BtCursor *pCur, /* The cursor to be moved */ @@ -59029,15 +66397,16 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( int rc; RecordCompare xRecordCompare; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( pRes ); assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); + assert( pCur->eState!=CURSOR_VALID || (pIdxKey==0)==(pCur->curIntKey!=0) ); /* If the cursor is already positioned at the point we are trying ** to move to, then just return without doing any work */ - if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 - && pCur->apPage[0]->intKey + if( pIdxKey==0 + && pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 ){ if( pCur->info.nKey==intKey ){ *pRes = 0; @@ -59051,7 +66420,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); - pIdxKey->isCorrupt = 0; + pIdxKey->errCode = 0; assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 @@ -59072,7 +66441,8 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); return SQLITE_OK; } - assert( pCur->apPage[0]->intKey || pIdxKey ); + assert( pCur->apPage[0]->intKey==pCur->curIntKey ); + assert( pCur->curIntKey || pIdxKey ); for(;;){ int lwr, upr, idx, c; Pgno chldPg; @@ -59095,8 +66465,8 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( if( xRecordCompare==0 ){ for(;;){ i64 nCellKey; - pCell = findCell(pPage, idx) + pPage->childPtrSize; - if( pPage->hasData ){ + pCell = findCellPastPtr(pPage, idx); + if( pPage->intKeyLeaf ){ while( 0x80 <= *(pCell++) ){ if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; } @@ -59127,8 +66497,8 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( } }else{ for(;;){ - int nCell; - pCell = findCell(pPage, idx) + pPage->childPtrSize; + int nCell; /* Size of the pCell cell in bytes */ + pCell = findCellPastPtr(pPage, idx); /* The maximum supported page-size is 65536 bytes. This means that ** the maximum number of record bytes stored on an index B-Tree @@ -59144,26 +66514,39 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ** single byte varint and the record fits entirely on the main ** b-tree page. */ testcase( pCell+nCell+1==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey, 0); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal ){ /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ testcase( pCell+nCell+2==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey, 0); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); }else{ /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated ** and accessPayload() used to retrieve the record into the - ** buffer before VdbeRecordCompare() can be called. */ + ** buffer before VdbeRecordCompare() can be called. + ** + ** If the record is corrupt, the xRecordCompare routine may read + ** up to two varints past the end of the buffer. An extra 18 + ** bytes of padding is allocated at the end of the buffer in + ** case this happens. */ void *pCellKey; u8 * const pCellBody = pCell - pPage->childPtrSize; - btreeParseCellPtr(pPage, pCellBody, &pCur->info); + pPage->xParseCell(pPage, pCellBody, &pCur->info); nCell = (int)pCur->info.nKey; - pCellKey = sqlite3Malloc( nCell ); + testcase( nCell<0 ); /* True if key size is 2^32 or more */ + testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ + testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ + testcase( nCell==2 ); /* Minimum legal index key size */ + if( nCell<2 ){ + rc = SQLITE_CORRUPT_BKPT; + goto moveto_finish; + } + pCellKey = sqlite3Malloc( nCell+18 ); if( pCellKey==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto moveto_finish; } pCur->aiIdx[pCur->iPage] = (u16)idx; @@ -59172,10 +66555,13 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( sqlite3_free(pCellKey); goto moveto_finish; } - c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); + c = xRecordCompare(nCell, pCellKey, pIdxKey); sqlite3_free(pCellKey); } - assert( pIdxKey->isCorrupt==0 || c==0 ); + assert( + (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) + && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed) + ); if( c<0 ){ lwr = idx+1; }else if( c>0 ){ @@ -59185,7 +66571,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( *pRes = 0; rc = SQLITE_OK; pCur->aiIdx[pCur->iPage] = (u16)idx; - if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT; + if( pIdxKey->errCode ) rc = SQLITE_CORRUPT; goto moveto_finish; } if( lwr>upr ) break; @@ -59240,6 +66626,12 @@ SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor *pCur){ ** was already pointing to the last entry in the database before ** this routine was called, then set *pRes=1. ** +** The main entry point is sqlite3BtreeNext(). That routine is optimized +** for the common case of merely incrementing the cell counter BtCursor.aiIdx +** to the next cell on the current page. The (slower) btreeNext() helper +** routine is called when it is necessary to move to a different page or +** to restore the cursor. +** ** The calling function will set *pRes to 0 or 1. The initial *pRes value ** will be 1 if the cursor being stepped corresponds to an SQL index and ** if this routine could have been skipped if that SQL index had been @@ -59249,20 +66641,18 @@ SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor *pCur){ ** SQLite btree implementation does not. (Note that the comdb2 btree ** implementation does use this hint, however.) */ -SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ +static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ int rc; int idx; MemPage *pPage; - assert( cursorHoldsMutex(pCur) ); - assert( pRes!=0 ); - assert( *pRes==0 || *pRes==1 ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + assert( *pRes==0 ); if( pCur->eState!=CURSOR_VALID ){ - invalidateOverflowCache(pCur); + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ - *pRes = 0; return rc; } if( CURSOR_INVALID==pCur->eState ){ @@ -59274,7 +66664,6 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ pCur->eState = CURSOR_VALID; if( pCur->skipNext>0 ){ pCur->skipNext = 0; - *pRes = 0; return SQLITE_OK; } pCur->skipNext = 0; @@ -59292,18 +66681,11 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** page into more than one b-tree structure. */ testcase( idx>pPage->nCell ); - pCur->info.nSize = 0; - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( idx>=pPage->nCell ){ if( !pPage->leaf ){ rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); - if( rc ){ - *pRes = 0; - return rc; - } - rc = moveToLeftmost(pCur); - *pRes = 0; - return rc; + if( rc ) return rc; + return moveToLeftmost(pCur); } do{ if( pCur->iPage==0 ){ @@ -59314,22 +66696,39 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ moveToParent(pCur); pPage = pCur->apPage[pCur->iPage]; }while( pCur->aiIdx[pCur->iPage]>=pPage->nCell ); - *pRes = 0; if( pPage->intKey ){ - rc = sqlite3BtreeNext(pCur, pRes); + return sqlite3BtreeNext(pCur, pRes); }else{ - rc = SQLITE_OK; + return SQLITE_OK; } - return rc; } - *pRes = 0; if( pPage->leaf ){ return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); + } +} +SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ + MemPage *pPage; + assert( cursorOwnsBtShared(pCur) ); + assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); + assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + *pRes = 0; + if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur, pRes); + pPage = pCur->apPage[pCur->iPage]; + if( (++pCur->aiIdx[pCur->iPage])>=pPage->nCell ){ + pCur->aiIdx[pCur->iPage]--; + return btreeNext(pCur, pRes); + } + if( pPage->leaf ){ + return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); } - rc = moveToLeftmost(pCur); - return rc; } - /* ** Step the cursor to the back to the previous entry in the database. If @@ -59337,6 +66736,12 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** was already pointing to the first entry in the database before ** this routine was called, then set *pRes=1. ** +** The main entry point is sqlite3BtreePrevious(). That routine is optimized +** for the common case of merely decrementing the cell counter BtCursor.aiIdx +** to the previous cell on the current page. The (slower) btreePrevious() +** helper routine is called when it is necessary to move to a different page +** or to restore the cursor. +** ** The calling function will set *pRes to 0 or 1. The initial *pRes value ** will be 1 if the cursor being stepped corresponds to an SQL index and ** if this routine could have been skipped if that SQL index had been @@ -59346,22 +66751,20 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** SQLite btree implementation does not. (Note that the comdb2 btree ** implementation does use this hint, however.) */ -SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ +static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ int rc; MemPage *pPage; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pRes!=0 ); - assert( *pRes==0 || *pRes==1 ); + assert( *pRes==0 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); - pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl); + assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 ); + assert( pCur->info.nSize==0 ); if( pCur->eState!=CURSOR_VALID ){ - if( ALWAYS(pCur->eState>=CURSOR_REQUIRESEEK) ){ - rc = btreeRestoreCursorPosition(pCur); - if( rc!=SQLITE_OK ){ - *pRes = 0; - return rc; - } + rc = restoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + return rc; } if( CURSOR_INVALID==pCur->eState ){ *pRes = 1; @@ -59372,7 +66775,6 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ pCur->eState = CURSOR_VALID; if( pCur->skipNext<0 ){ pCur->skipNext = 0; - *pRes = 0; return SQLITE_OK; } pCur->skipNext = 0; @@ -59384,10 +66786,7 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ if( !pPage->leaf ){ int idx = pCur->aiIdx[pCur->iPage]; rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); - if( rc ){ - *pRes = 0; - return rc; - } + if( rc ) return rc; rc = moveToRightmost(pCur); }else{ while( pCur->aiIdx[pCur->iPage]==0 ){ @@ -59398,8 +66797,8 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ } moveToParent(pCur); } - pCur->info.nSize = 0; - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & (BTCF_ValidNKey|BTCF_ValidOvfl))==0 ); pCur->aiIdx[pCur->iPage]--; pPage = pCur->apPage[pCur->iPage]; @@ -59409,9 +66808,25 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ rc = SQLITE_OK; } } - *pRes = 0; return rc; } +SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ + assert( cursorOwnsBtShared(pCur) ); + assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); + assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + *pRes = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey); + pCur->info.nSize = 0; + if( pCur->eState!=CURSOR_VALID + || pCur->aiIdx[pCur->iPage]==0 + || pCur->apPage[pCur->iPage]->leaf==0 + ){ + return btreePrevious(pCur, pRes); + } + pCur->aiIdx[pCur->iPage]--; + return SQLITE_OK; +} /* ** Allocate a new page from the database file. @@ -59422,8 +66837,7 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ ** sqlite3PagerUnref() on the new page when it is done. ** ** SQLITE_OK is returned on success. Any other return value indicates -** an error. *ppPage and *pPgno are undefined in the event of an error. -** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. +** an error. *ppPage is set to NULL in the event of an error. ** ** If the "nearby" parameter is not 0, then an effort is made to ** locate a page close to the page number "nearby". This can be used in an @@ -59455,6 +66869,8 @@ static int allocateBtreePage( assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); + /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 + ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ @@ -59464,6 +66880,7 @@ static int allocateBtreePage( /* There are pages on the freelist. Reuse one of those pages. */ Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ + u32 nSearch = 0; /* Count of the number of search attempts */ /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then @@ -59501,15 +66918,21 @@ static int allocateBtreePage( do { pPrevTrunk = pTrunk; if( pPrevTrunk ){ + /* EVIDENCE-OF: R-01506-11053 The first integer on a freelist trunk page + ** is the page number of the next freelist trunk page in the list or + ** zero if this is the last freelist trunk page. */ iTrunk = get4byte(&pPrevTrunk->aData[0]); }else{ + /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32 + ** stores the page number of the first page of the freelist, or zero if + ** the freelist is empty. */ iTrunk = get4byte(&pPage1->aData[32]); } testcase( iTrunk==mxPage ); - if( iTrunk>mxPage ){ + if( iTrunk>mxPage || nSearch++ > n ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0); } if( rc ){ pTrunk = 0; @@ -59517,8 +66940,9 @@ static int allocateBtreePage( } assert( pTrunk!=0 ); assert( pTrunk->aData!=0 ); - - k = get4byte(&pTrunk->aData[4]); /* # of leaves on this trunk page */ + /* EVIDENCE-OF: R-13523-04394 The second integer on a freelist trunk page + ** is the number of leaf page pointers to follow. */ + k = get4byte(&pTrunk->aData[4]); if( k==0 && !searchList ){ /* The trunk has no leaves and the list is not being searched. ** So extract the trunk page itself and use it as the newly @@ -59573,7 +66997,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetUnusedPage(pBt, iNewTrunk, &pNewTrunk, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -59652,12 +67076,13 @@ static int allocateBtreePage( memcpy(&aData[8+closest*4], &aData[4+k*4], 4); } put4byte(&aData[4], k-1); - noContent = !btreeGetHasContent(pBt, *pPgno) ? PAGER_GET_NOCONTENT : 0; - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + noContent = !btreeGetHasContent(pBt, *pPgno)? PAGER_GET_NOCONTENT : 0; + rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, noContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); + *ppPage = 0; } } searchList = 0; @@ -59685,7 +67110,7 @@ static int allocateBtreePage( ** here are confined to those pages that lie between the end of the ** database image and the end of the database file. */ - int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate)) ? PAGER_GET_NOCONTENT : 0; + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate))? PAGER_GET_NOCONTENT:0; rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc ) return rc; @@ -59701,7 +67126,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); + rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -59715,11 +67140,12 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); + rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, bNoContent); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); + *ppPage = 0; } TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); } @@ -59729,17 +67155,8 @@ static int allocateBtreePage( end_allocate_page: releasePage(pTrunk); releasePage(pPrevTrunk); - if( rc==SQLITE_OK ){ - if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ - releasePage(*ppPage); - *ppPage = 0; - return SQLITE_CORRUPT_BKPT; - } - (*ppPage)->isInit = 0; - }else{ - *ppPage = 0; - } - assert( rc!=SQLITE_OK || sqlite3PagerIswriteable((*ppPage)->pDbPage) ); + assert( rc!=SQLITE_OK || sqlite3PagerPageRefcount((*ppPage)->pDbPage)<=1 ); + assert( rc!=SQLITE_OK || (*ppPage)->isInit==0 ); return rc; } @@ -59764,9 +67181,10 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ int nFree; /* Initial number of pages on free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( iPage>1 ); + assert( CORRUPT_DB || iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); + if( iPage<2 ) return SQLITE_CORRUPT_BKPT; if( pMemPage ){ pPage = pMemPage; sqlite3PagerRef(pPage->pDbPage); @@ -59836,6 +67254,11 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** for now. At some point in the future (once everyone has upgraded ** to 3.6.0 or later) we should consider fixing the conditional above ** to read "usableSize/4-2" instead of "usableSize/4-8". + ** + ** EVIDENCE-OF: R-19920-11576 However, newer versions of SQLite still + ** avoid using the last six entries in the freelist trunk page array in + ** order that database files created by newer versions of SQLite can be + ** read by older versions of SQLite. */ rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc==SQLITE_OK ){ @@ -59884,9 +67307,15 @@ static void freePage(MemPage *pPage, int *pRC){ } /* -** Free any overflow pages associated with the given Cell. +** Free any overflow pages associated with the given Cell. Write the +** local Cell size (the number of bytes on the original page, omitting +** overflow) into *pnSize. */ -static int clearCell(MemPage *pPage, unsigned char *pCell){ +static int clearCell( + MemPage *pPage, /* The page that contains the Cell */ + unsigned char *pCell, /* First byte of the Cell */ + u16 *pnSize /* Write the size of the Cell here */ +){ BtShared *pBt = pPage->pBt; CellInfo info; Pgno ovflPgno; @@ -59895,18 +67324,21 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ u32 ovflPageSize; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - btreeParseCellPtr(pPage, pCell, &info); - if( info.iOverflow==0 ){ + pPage->xParseCell(pPage, pCell, &info); + *pnSize = info.nSize; + if( info.nLocal==info.nPayload ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } - if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){ + if( pCell+info.nSize-1 > pPage->aData+pPage->maskPage ){ return SQLITE_CORRUPT_BKPT; /* Cell extends past end of page */ } - ovflPgno = get4byte(&pCell[info.iOverflow]); + ovflPgno = get4byte(pCell + info.nSize - 4); assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; - assert( ovflPgno==0 || nOvfl>0 ); + assert( nOvfl>0 || + (CORRUPT_DB && (info.nPayload + ovflPageSize)pBt; Pgno pgnoOvfl = 0; int nHeader; - CellInfo info; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -59989,40 +67418,66 @@ static int fillInCell( || sqlite3PagerIswriteable(pPage->pDbPage) ); /* Fill in the header. */ - nHeader = 0; - if( !pPage->leaf ){ - nHeader += 4; - } - if( pPage->hasData ){ - nHeader += putVarint32(&pCell[nHeader], nData+nZero); + nHeader = pPage->childPtrSize; + if( pPage->intKey ){ + nPayload = pX->nData + pX->nZero; + pSrc = pX->pData; + nSrc = pX->nData; + assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */ + nHeader += putVarint32(&pCell[nHeader], nPayload); + nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey); }else{ - nData = nZero = 0; + assert( pX->nKey<=0x7fffffff && pX->pKey!=0 ); + nSrc = nPayload = (int)pX->nKey; + pSrc = pX->pKey; + nHeader += putVarint32(&pCell[nHeader], nPayload); } - nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); - btreeParseCellPtr(pPage, pCell, &info); - assert( info.nHeader==nHeader ); - assert( info.nKey==nKey ); - assert( info.nData==(u32)(nData+nZero) ); /* Fill in the payload */ - nPayload = nData + nZero; - if( pPage->intKey ){ - pSrc = pData; - nSrc = nData; - nData = 0; - }else{ - if( NEVER(nKey>0x7fffffff || pKey==0) ){ - return SQLITE_CORRUPT_BKPT; - } - nPayload += (int)nKey; - pSrc = pKey; - nSrc = (int)nKey; + if( nPayload<=pPage->maxLocal ){ + n = nHeader + nPayload; + testcase( n==3 ); + testcase( n==4 ); + if( n<4 ) n = 4; + *pnSize = n; + spaceLeft = nPayload; + pPrior = pCell; + }else{ + int mn = pPage->minLocal; + n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); + testcase( n==pPage->maxLocal ); + testcase( n==pPage->maxLocal+1 ); + if( n > pPage->maxLocal ) n = mn; + spaceLeft = n; + *pnSize = n + nHeader + 4; + pPrior = &pCell[nHeader+n]; } - *pnSize = info.nSize; - spaceLeft = info.nLocal; pPayload = &pCell[nHeader]; - pPrior = &pCell[info.iOverflow]; + /* At this point variables should be set as follows: + ** + ** nPayload Total payload size in bytes + ** pPayload Begin writing payload here + ** spaceLeft Space available at pPayload. If nPayload>spaceLeft, + ** that means content must spill into overflow pages. + ** *pnSize Size of the local cell (not counting overflow pages) + ** pPrior Where to write the pgno of the first overflow page + ** + ** Use a call to btreeParseCellPtr() to verify that the values above + ** were computed correctly. + */ +#if SQLITE_DEBUG + { + CellInfo info; + pPage->xParseCell(pPage, pCell, &info); + assert( nHeader==(int)(info.pPayload - pCell) ); + assert( info.nKey==pX->nKey ); + assert( *pnSize == info.nSize ); + assert( spaceLeft == info.nLocal ); + } +#endif + + /* Write the payload into the local Cell and any extra into overflow pages */ while( nPayload>0 ){ if( spaceLeft==0 ){ #ifndef SQLITE_OMIT_AUTOVACUUM @@ -60101,10 +67556,6 @@ static int fillInCell( pSrc += n; nSrc -= n; spaceLeft -= n; - if( nSrc==0 ){ - nSrc = nData; - pSrc = pData; - } } releasePage(pToRelease); return SQLITE_OK; @@ -60128,7 +67579,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ if( *pRC ) return; assert( idx>=0 && idxnCell ); - assert( sz==cellSize(pPage, idx) ); + assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); data = pPage->aData; @@ -60147,9 +67598,17 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ return; } pPage->nCell--; - memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); - put2byte(&data[hdr+3], pPage->nCell); - pPage->nFree += 2; + if( pPage->nCell==0 ){ + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pPage->pBt->usableSize); + pPage->nFree = pPage->pBt->usableSize - pPage->hdrOffset + - pPage->childPtrSize - 8; + }else{ + memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); + put2byte(&data[hdr+3], pPage->nCell); + pPage->nFree += 2; + } } /* @@ -60164,10 +67623,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. ** -** If nSkip is non-zero, then do not copy the first nSkip bytes of the -** cell. The caller will overwrite them after this function returns. If -** nSkip is non-zero, then pCell may not point to an invalid memory location -** (but pCell+nSkip is always valid). +** *pRC must be SQLITE_OK when this routine is called. */ static void insertCell( MemPage *pPage, /* Page into which we are copying */ @@ -60180,14 +67636,10 @@ static void insertCell( ){ int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ - int end; /* First byte past the last cell pointer in data[] */ - int ins; /* Index in data[] where new cell pointer is inserted */ - int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ - int nSkip = (iChild ? 4 : 0); - - if( *pRC ) return; + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ + assert( *pRC==SQLITE_OK ); assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); @@ -60199,10 +67651,10 @@ static void insertCell( ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size ** might be less than 8 (leaf-size + pointer) on the interior node. Hence ** the term after the || in the following assert(). */ - assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); + assert( sz==pPage->xCellSize(pPage, pCell) || (sz==8 && iChild>0) ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ - memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip); + memcpy(pTemp, pCell, sz); pCell = pTemp; } if( iChild ){ @@ -60212,6 +67664,14 @@ static void insertCell( assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) ); pPage->apOvfl[j] = pCell; pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ @@ -60220,24 +67680,26 @@ static void insertCell( } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; - cellOffset = pPage->cellOffset; - end = cellOffset + 2*pPage->nCell; - ins = cellOffset + 2*i; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); rc = allocateSpace(pPage, sz, &idx); if( rc ){ *pRC = rc; return; } - /* The allocateSpace() routine guarantees the following two properties - ** if it returns success */ - assert( idx >= end+2 ); + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); - pPage->nCell++; pPage->nFree -= (u16)(2 + sz); - memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip); + memcpy(&data[idx], pCell, sz); if( iChild ){ put4byte(&data[idx], iChild); } - memmove(&data[ins+2], &data[ins], end-ins); - put2byte(&data[ins], idx); - put2byte(&data[pPage->hdrOffset+3], pPage->nCell); + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ /* The cell may contain a pointer to an overflow page. If so, write @@ -60250,45 +67712,328 @@ static void insertCell( } /* -** Add a list of cells to a page. The page should be initially empty. -** The cells are guaranteed to fit on the page. +** A CellArray object contains a cache of pointers and sizes for a +** consecutive sequence of cells that might be held on multiple pages. */ -static void assemblePage( - MemPage *pPage, /* The page to be assemblied */ - int nCell, /* The number of cells to add to this page */ - u8 **apCell, /* Pointers to cell bodies */ - u16 *aSize /* Sizes of the cells */ -){ - int i; /* Loop counter */ - u8 *pCellptr; /* Address of next cell pointer */ - int cellbody; /* Address of next cell body */ - u8 * const data = pPage->aData; /* Pointer to data for pPage */ - const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ - const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ +typedef struct CellArray CellArray; +struct CellArray { + int nCell; /* Number of cells in apCell[] */ + MemPage *pRef; /* Reference page */ + u8 **apCell; /* All cells begin balanced */ + u16 *szCell; /* Local size of all cells in apCell[] */ +}; - assert( pPage->nOverflow==0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt) - && (int)MX_CELL(pPage->pBt)<=10921); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - - /* Check that the page has just been zeroed by zeroPage() */ - assert( pPage->nCell==0 ); - assert( get2byteNotZero(&data[hdr+5])==nUsable ); - - pCellptr = &pPage->aCellIdx[nCell*2]; - cellbody = nUsable; - for(i=nCell-1; i>=0; i--){ - u16 sz = aSize[i]; - pCellptr -= 2; - cellbody -= sz; - put2byte(pCellptr, cellbody); - memcpy(&data[cellbody], apCell[i], sz); +/* +** Make sure the cell sizes at idx, idx+1, ..., idx+N-1 have been +** computed. +*/ +static void populateCellCache(CellArray *p, int idx, int N){ + assert( idx>=0 && idx+N<=p->nCell ); + while( N>0 ){ + assert( p->apCell[idx]!=0 ); + if( p->szCell[idx]==0 ){ + p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + }else{ + assert( CORRUPT_DB || + p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + } + idx++; + N--; } - put2byte(&data[hdr+3], nCell); - put2byte(&data[hdr+5], cellbody); - pPage->nFree -= (nCell*2 + nUsable - cellbody); - pPage->nCell = (u16)nCell; +} + +/* +** Return the size of the Nth element of the cell array +*/ +static SQLITE_NOINLINE u16 computeCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + assert( p->szCell[N]==0 ); + p->szCell[N] = p->pRef->xCellSize(p->pRef, p->apCell[N]); + return p->szCell[N]; +} +static u16 cachedCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + if( p->szCell[N] ) return p->szCell[N]; + return computeCellSize(p, N); +} + +/* +** Array apCell[] contains pointers to nCell b-tree page cells. The +** szCell[] array contains the size in bytes of each cell. This function +** replaces the current contents of page pPg with the contents of the cell +** array. +** +** Some of the cells in apCell[] may currently be stored in pPg. This +** function works around problems caused by this by making a copy of any +** such cells before overwriting the page data. +** +** The MemPage.nFree field is invalidated by this function. It is the +** responsibility of the caller to set it correctly. +*/ +static int rebuildPage( + MemPage *pPg, /* Edit this page */ + int nCell, /* Final number of cells on page */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ + u8 * const aData = pPg->aData; /* Pointer to data for pPg */ + const int usableSize = pPg->pBt->usableSize; + u8 * const pEnd = &aData[usableSize]; + int i; + u8 *pCellptr = pPg->aCellIdx; + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + u8 *pData; + + i = get2byte(&aData[hdr+5]); + memcpy(&pTmp[i], &aData[i], usableSize - i); + + pData = pEnd; + for(i=0; ixCellSize(pPg, pCell) || CORRUPT_DB ); + testcase( szCell[i]!=pPg->xCellSize(pPg,pCell) ); + } + + /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ + pPg->nCell = nCell; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+1], 0); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + aData[hdr+7] = 0x00; + return SQLITE_OK; +} + +/* +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell +** contains the size in bytes of each such cell. This function attempts to +** add the cells stored in the array to page pPg. If it cannot (because +** the page needs to be defragmented before the cells will fit), non-zero +** is returned. Otherwise, if the cells are added successfully, zero is +** returned. +** +** Argument pCellptr points to the first entry in the cell-pointer array +** (part of page pPg) to populate. After cell apCell[0] is written to the +** page body, a 16-bit offset is written to pCellptr. And so on, for each +** cell in the array. It is the responsibility of the caller to ensure +** that it is safe to overwrite this part of the cell-pointer array. +** +** When this function is called, *ppData points to the start of the +** content area on page pPg. If the size of the content area is extended, +** *ppData is updated to point to the new start of the content area +** before returning. +** +** Finally, argument pBegin points to the byte immediately following the +** end of the space required by this page for the cell-pointer area (for +** all cells - not just those inserted by the current call). If the content +** area must be extended to before this point in order to accomodate all +** cells in apCell[], then the cells do not fit and non-zero is returned. +*/ +static int pageInsertArray( + MemPage *pPg, /* Page to add cells to */ + u8 *pBegin, /* End of cell-pointer array */ + u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 *pCellptr, /* Pointer to cell-pointer area */ + int iFirst, /* Index of first cell to add */ + int nCell, /* Number of cells to add to pPg */ + CellArray *pCArray /* Array of cells */ +){ + int i; + u8 *aData = pPg->aData; + u8 *pData = *ppData; + int iEnd = iFirst + nCell; + assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ + for(i=iFirst; iapCell[i] will never overlap on a well-formed + ** database. But they might for a corrupt database. Hence use memmove() + ** since memcpy() sends SIGABORT with overlapping buffers on OpenBSD */ + assert( (pSlot+sz)<=pCArray->apCell[i] + || pSlot>=(pCArray->apCell[i]+sz) + || CORRUPT_DB ); + memmove(pSlot, pCArray->apCell[i], sz); + put2byte(pCellptr, (pSlot - aData)); + pCellptr += 2; + } + *ppData = pData; + return 0; +} + +/* +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell +** contains the size in bytes of each such cell. This function adds the +** space associated with each cell in the array that is currently stored +** within the body of pPg to the pPg free-list. The cell-pointers and other +** fields of the page are not updated. +** +** This function returns the total number of cells added to the free-list. +*/ +static int pageFreeArray( + MemPage *pPg, /* Page to edit */ + int iFirst, /* First cell to delete */ + int nCell, /* Cells to delete */ + CellArray *pCArray /* Array of cells */ +){ + u8 * const aData = pPg->aData; + u8 * const pEnd = &aData[pPg->pBt->usableSize]; + u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; + int nRet = 0; + int i; + int iEnd = iFirst + nCell; + u8 *pFree = 0; + int szFree = 0; + + for(i=iFirst; iapCell[i]; + if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ + int sz; + /* No need to use cachedCellSize() here. The sizes of all cells that + ** are to be freed have already been computing while deciding which + ** cells need freeing */ + sz = pCArray->szCell[i]; assert( sz>0 ); + if( pFree!=(pCell + sz) ){ + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + pFree = pCell; + szFree = sz; + if( pFree+sz>pEnd ) return 0; + }else{ + pFree = pCell; + szFree += sz; + } + nRet++; + } + } + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + return nRet; +} + +/* +** apCell[] and szCell[] contains pointers to and sizes of all cells in the +** pages being balanced. The current page, pPg, has pPg->nCell cells starting +** with apCell[iOld]. After balancing, this page should hold nNew cells +** starting at apCell[iNew]. +** +** This routine makes the necessary adjustments to pPg so that it contains +** the correct cells after being balanced. +** +** The pPg->nFree field is invalid when this function returns. It is the +** responsibility of the caller to set it correctly. +*/ +static int editPage( + MemPage *pPg, /* Edit this page */ + int iOld, /* Index of first cell currently on page */ + int iNew, /* Index of new first cell on page */ + int nNew, /* Final number of cells on page */ + CellArray *pCArray /* Array of cells and sizes */ +){ + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + int iNewEnd = iNew + nNew; + +#ifdef SQLITE_DEBUG + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); +#endif + + /* Remove cells from the start and end of the page */ + if( iOldaCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } + if( iNewEnd < iOldEnd ){ + nCell -= pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + } + + pData = &aData[get2byteNotZero(&aData[hdr+5])]; + if( pDataaCellIdx; + memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iNew, nAdd, pCArray + ) ) goto editpage_fail; + nCell += nAdd; + } + + /* Add any overflow cells */ + for(i=0; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + nCell++; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iCell+iNew, 1, pCArray + ) ) goto editpage_fail; + } + } + + /* Append cells to the end of the page */ + pCellptr = &pPg->aCellIdx[nCell*2]; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iNew+nCell, nNew-nCell, pCArray + ) ) goto editpage_fail; + + pPg->nCell = nNew; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + +#ifdef SQLITE_DEBUG + for(i=0; iapCell[i+iNew]; + int iOff = get2byteAligned(&pPg->aCellIdx[i*2]); + if( SQLITE_WITHIN(pCell, aData, &aData[pPg->pBt->usableSize]) ){ + pCell = &pTmp[pCell - aData]; + } + assert( 0==memcmp(pCell, &aData[iOff], + pCArray->pRef->xCellSize(pCArray->pRef, pCArray->apCell[i+iNew])) ); + } +#endif + + return SQLITE_OK; + editpage_fail: + /* Unable to edit this page. Rebuild it from scratch instead. */ + populateCellCache(pCArray, iNew, nNew); + return rebuildPage(pPg, nNew, &pCArray->apCell[iNew], &pCArray->szCell[iNew]); } /* @@ -60342,7 +68087,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( pPage->nOverflow==1 ); /* This error condition is now caught prior to reaching this function */ - if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; + if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell @@ -60354,13 +68099,15 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ u8 *pOut = &pSpace[4]; u8 *pCell = pPage->apOvfl[0]; - u16 szCell = cellSizePtr(pPage, pCell); + u16 szCell = pPage->xCellSize(pPage, pCell); u8 *pStop; assert( sqlite3PagerIswriteable(pNew->pDbPage) ); assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - assemblePage(pNew, 1, &pCell, &szCell); + rc = rebuildPage(pNew, 1, &pCell, &szCell); + if( NEVER(rc) ) return rc; + pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map ** with entries for the new page, and any pointer from the @@ -60398,8 +68145,10 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ while( ((*(pOut++) = *(pCell++))&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); + if( rc==SQLITE_OK ){ + insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno, &rc); + } /* Set the right-child pointer of pParent to point to the new page. */ put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); @@ -60432,9 +68181,9 @@ static int ptrmapCheckPages(MemPage **apPage, int nPage){ u8 *z; z = findCell(pPage, j); - btreeParseCellPtr(pPage, z, &info); - if( info.iOverflow ){ - Pgno ovfl = get4byte(&z[info.iOverflow]); + pPage->xParseCell(pPage, z, &info); + if( info.nLocalpgno && e==PTRMAP_OVERFLOW1 ); } @@ -60552,9 +68301,6 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** If aOvflSpace is set to a null pointer, this function returns ** SQLITE_NOMEM. */ -#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) -#pragma optimize("", off) -#endif static int balance_nonroot( MemPage *pParent, /* Parent page of siblings being balanced */ int iParentIdx, /* Index of "the page" in pParent */ @@ -60563,7 +68309,6 @@ static int balance_nonroot( int bBulk /* True if this call is part of a bulk load */ ){ BtShared *pBt; /* The whole database */ - int nCell = 0; /* Number of cells in apCell[] */ int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ int nNew = 0; /* Number of pages in apNew[] */ int nOld; /* Number of pages in apOld[] */ @@ -60574,22 +68319,27 @@ static int balance_nonroot( int leafData; /* True if pPage is a leaf of a LEAFDATA tree */ int usableSpace; /* Bytes in pPage beyond the header */ int pageFlags; /* Value of pPage->aData[0] */ - int subtotal; /* Subtotal of bytes in cells on one page */ int iSpace1 = 0; /* First unused byte of aSpace1[] */ int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ int szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ - MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ - int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ - int szNew[NB+2]; /* Combined size of cells place on i-th page */ - u8 **apCell = 0; /* All cells begin balanced */ - u16 *szCell; /* Local size of all cells in apCell[] */ + int cntNew[NB+2]; /* Index in b.paCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in b.apCell[] */ + int szNew[NB+2]; /* Combined size of cells placed on i-th page */ u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ + u8 abDone[NB+2]; /* True after i'th new page is populated */ + Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ + u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ + CellArray b; /* Parsed information on cells being balanced */ + memset(abDone, 0, sizeof(abDone)); + b.nCell = 0; + b.apCell = 0; pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -60607,7 +68357,7 @@ static int balance_nonroot( assert( pParent->nOverflow==0 || pParent->aiOvfl[0]==iParentIdx ); if( !aOvflSpace ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* Find the sibling pages to balance. Also locate the cells in pParent @@ -60631,7 +68381,6 @@ static int balance_nonroot( }else if( iParentIdx==i ){ nxDiv = i-2+bBulk; }else{ - assert( bBulk==0 ); nxDiv = iParentIdx-1; } i = 2-bBulk; @@ -60644,7 +68393,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i], 0); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -60655,12 +68404,12 @@ static int balance_nonroot( if( i+nxDiv==pParent->aiOvfl[0] && pParent->nOverflow ){ apDiv[i] = pParent->apOvfl[0]; pgno = get4byte(apDiv[i]); - szNew[i] = cellSizePtr(pParent, apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); pParent->nOverflow = 0; }else{ apDiv[i] = findCell(pParent, i+nxDiv-pParent->nOverflow); pgno = get4byte(apDiv[i]); - szNew[i] = cellSizePtr(pParent, apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); /* Drop the cell from the parent page. apDiv[i] still points to ** the cell within the parent, even though it has been dropped. @@ -60698,138 +68447,208 @@ static int balance_nonroot( /* ** Allocate space for memory structures */ - k = pBt->pageSize + ROUND8(sizeof(MemPage)); szScratch = - nMaxCells*sizeof(u8*) /* apCell */ - + nMaxCells*sizeof(u16) /* szCell */ - + pBt->pageSize /* aSpace1 */ - + k*nOld; /* Page copies (apCopy) */ - apCell = sqlite3ScratchMalloc( szScratch ); - if( apCell==0 ){ - rc = SQLITE_NOMEM; + nMaxCells*sizeof(u8*) /* b.apCell */ + + nMaxCells*sizeof(u16) /* b.szCell */ + + pBt->pageSize; /* aSpace1 */ + + /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer + ** that is more than 6 times the database page size. */ + assert( szScratch<=6*(int)pBt->pageSize ); + b.apCell = sqlite3ScratchMalloc( szScratch ); + if( b.apCell==0 ){ + rc = SQLITE_NOMEM_BKPT; goto balance_cleanup; } - szCell = (u16*)&apCell[nMaxCells]; - aSpace1 = (u8*)&szCell[nMaxCells]; + b.szCell = (u16*)&b.apCell[nMaxCells]; + aSpace1 = (u8*)&b.szCell[nMaxCells]; assert( EIGHT_BYTE_ALIGNMENT(aSpace1) ); /* ** Load pointers to all cells on sibling pages and the divider cells - ** into the local apCell[] array. Make copies of the divider cells - ** into space obtained from aSpace1[] and remove the divider cells - ** from pParent. + ** into the local b.apCell[] array. Make copies of the divider cells + ** into space obtained from aSpace1[]. The divider cells have already + ** been removed from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied - ** into aSpace1[]. In this way, all cells in apCell[] are without + ** into aSpace1[]. In this way, all cells in b.apCell[] are without ** child pointers. If siblings are not leaves, then all cell in - ** apCell[] include child pointers. Either way, all cells in apCell[] + ** b.apCell[] include child pointers. Either way, all cells in b.apCell[] ** are alike. ** ** leafCorrection: 4 if pPage is a leaf. 0 if pPage is not a leaf. ** leafData: 1 if pPage holds key+data and pParent holds only keys. */ - leafCorrection = apOld[0]->leaf*4; - leafData = apOld[0]->hasData; + b.pRef = apOld[0]; + leafCorrection = b.pRef->leaf*4; + leafData = b.pRef->intKeyLeaf; for(i=0; ipageSize + k*i]; - memcpy(pOld, apOld[i], sizeof(MemPage)); - pOld->aData = (void*)&pOld[1]; - memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); + MemPage *pOld = apOld[i]; + int limit = pOld->nCell; + u8 *aData = pOld->aData; + u16 maskPage = pOld->maskPage; + u8 *piCell = aData + pOld->cellOffset; + u8 *piEnd; - limit = pOld->nCell+pOld->nOverflow; + /* Verify that all sibling pages are of the same "type" (table-leaf, + ** table-interior, index-leaf, or index-interior). + */ + if( pOld->aData[0]!=apOld[0]->aData[0] ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + + /* Load b.apCell[] with pointers to all cells in pOld. If pOld + ** constains overflow cells, include them in the b.apCell[] array + ** in the correct spot. + ** + ** Note that when there are multiple overflow cells, it is always the + ** case that they are sequential and adjacent. This invariant arises + ** because multiple overflows can only occurs when inserting divider + ** cells into a parent on a prior balance, and divider cells are always + ** adjacent and are inserted in order. There is an assert() tagged + ** with "NOTE 1" in the overflow cell insertion loop to prove this + ** invariant. + ** + ** This must be done in advance. Once the balance starts, the cell + ** offset section of the btree page will be overwritten and we will no + ** long be able to find the cells if a pointer to each cell is not saved + ** first. + */ + memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ + limit = pOld->aiOvfl[0]; for(j=0; jaData; - u16 maskPage = pOld->maskPage; - u16 cellOffset = pOld->cellOffset; - for(j=0; jnOverflow; k++){ + assert( k==0 || pOld->aiOvfl[k-1]+1==pOld->aiOvfl[k] );/* NOTE 1 */ + b.apCell[b.nCell] = pOld->apOvfl[k]; + b.nCell++; } - } + } + piEnd = aData + pOld->cellOffset + 2*pOld->nCell; + while( piCellmaxLocal+23 ); assert( iSpace1 <= (int)pBt->pageSize ); memcpy(pTemp, apDiv[i], sz); - apCell[nCell] = pTemp+leafCorrection; + b.apCell[b.nCell] = pTemp+leafCorrection; assert( leafCorrection==0 || leafCorrection==4 ); - szCell[nCell] = szCell[nCell] - leafCorrection; + b.szCell[b.nCell] = b.szCell[b.nCell] - leafCorrection; if( !pOld->leaf ){ assert( leafCorrection==0 ); assert( pOld->hdrOffset==0 ); /* The right pointer of the child page pOld becomes the left ** pointer of the divider cell */ - memcpy(apCell[nCell], &pOld->aData[8], 4); + memcpy(b.apCell[b.nCell], &pOld->aData[8], 4); }else{ assert( leafCorrection==4 ); - if( szCell[nCell]<4 ){ - /* Do not allow any cells smaller than 4 bytes. */ - szCell[nCell] = 4; + while( b.szCell[b.nCell]<4 ){ + /* Do not allow any cells smaller than 4 bytes. If a smaller cell + ** does exist, pad it with 0x00 bytes. */ + assert( b.szCell[b.nCell]==3 || CORRUPT_DB ); + assert( b.apCell[b.nCell]==&aSpace1[iSpace1-3] || CORRUPT_DB ); + aSpace1[iSpace1++] = 0x00; + b.szCell[b.nCell]++; } } - nCell++; + b.nCell++; } } /* - ** Figure out the number of pages needed to hold all nCell cells. + ** Figure out the number of pages needed to hold all b.nCell cells. ** Store this number in "k". Also compute szNew[] which is the total ** size of all cells on the i-th page and cntNew[] which is the index - ** in apCell[] of the cell that divides page i from page i+1. - ** cntNew[k] should equal nCell. + ** in b.apCell[] of the cell that divides page i from page i+1. + ** cntNew[k] should equal b.nCell. ** ** Values computed by this block: ** ** k: The total number of sibling pages ** szNew[i]: Spaced used on the i-th sibling page. - ** cntNew[i]: Index in apCell[] and szCell[] for the first cell to + ** cntNew[i]: Index in b.apCell[] and b.szCell[] for the first cell to ** the right of the i-th sibling page. ** usableSpace: Number of bytes of space available on each sibling. ** */ usableSpace = pBt->usableSize - 12 + leafCorrection; - for(subtotal=k=i=0; i usableSpace ){ - szNew[k] = subtotal - szCell[i]; - cntNew[k] = i; - if( leafData ){ i--; } - subtotal = 0; - k++; - if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + for(i=0; inFree; + if( szNew[i]<0 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + for(j=0; jnOverflow; j++){ + szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); + } + cntNew[i] = cntOld[i]; + } + k = nOld; + for(i=0; iusableSpace ){ + if( i+1>=k ){ + k = i+2; + if( k>NB+2 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + szNew[k-1] = 0; + cntNew[k-1] = b.nCell; + } + sz = 2 + cachedCellSize(&b, cntNew[i]-1); + szNew[i] -= sz; + if( !leafData ){ + if( cntNew[i]usableSpace ) break; + szNew[i] += sz; + cntNew[i]++; + if( !leafData ){ + if( cntNew[i]=b.nCell ){ + k = i+1; + }else if( cntNew[i] <= (i>0 ? cntNew[i-1] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; } } - szNew[k] = subtotal; - cntNew[k] = nCell; - k++; /* ** The packing computed by the previous block is biased toward the siblings - ** on the left side. The left siblings are always nearly full, while the - ** right-most sibling might be nearly empty. This block of code attempts - ** to adjust the packing of siblings to get a better balance. + ** on the left side (siblings with smaller keys). The left siblings are + ** always nearly full, while the right-most sibling might be nearly empty. + ** The next block of code attempts to adjust the packing of siblings to + ** get a better balance. ** ** This adjustment is more than an optimization. The packing above might ** be so out of balance as to be illegal. For example, the right-most @@ -60843,46 +68662,46 @@ static int balance_nonroot( r = cntNew[i-1] - 1; d = r + 1 - leafData; - assert( d szLeft-(b.szCell[r]+(i==k-1?0:2)))){ + break; + } + szRight += b.szCell[d] + 2; + szLeft -= b.szCell[r] + 2; + cntNew[i-1] = r; + r--; + d--; + }while( r>=0 ); szNew[i] = szRight; szNew[i-1] = szLeft; + if( cntNew[i-1] <= (i>1 ? cntNew[i-2] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } } - /* Either we found one or more cells (cntnew[0])>0) or pPage is - ** a virtual root page. A virtual root page is when the real root - ** page is page 1 and we are the only child of that page. - ** - ** UPDATE: The assert() below is not necessarily true if the database - ** file is corrupt. The corruption will be detected and reported later - ** in this procedure so there is no need to act upon it now. + /* Sanity check: For a non-corrupt database file one of the follwing + ** must be true: + ** (1) We found one or more cells (cntNew[0])>0), or + ** (2) pPage is a virtual root page. A virtual root page is when + ** the real root page is page 1 and we are the only child of + ** that page. */ -#if 0 - assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); -#endif - - TRACE(("BALANCE: old: %d %d %d ", - apOld[0]->pgno, - nOld>=2 ? apOld[1]->pgno : 0, - nOld>=3 ? apOld[2]->pgno : 0 + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); + TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + apOld[0]->pgno, apOld[0]->nCell, + nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, + nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 )); /* ** Allocate k new pages. Reuse old pages where possible. */ - if( apOld[0]->pgno<=1 ){ - rc = SQLITE_CORRUPT_BKPT; - goto balance_cleanup; - } pageFlags = apOld[0]->aData[0]; for(i=0; i0 ); rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); if( rc ) goto balance_cleanup; + zeroPage(pNew, pageFlags); apNew[i] = pNew; nNew++; + cntOld[i] = b.nCell; /* Set the pointer-map entry for the new sibling page. */ if( ISAUTOVACUUM ){ @@ -60909,135 +68730,249 @@ static int balance_nonroot( } } - /* Free any old pages that were not reused as new pages. - */ - while( ipgno; - int minI = i; - for(j=i+1; jpgno<(unsigned)minV ){ - minI = j; - minV = apNew[j]->pgno; + for(i=0; ipgno; + aPgFlags[i] = apNew[i]->pDbPage->flags; + for(j=0; ji ){ - MemPage *pT; - pT = apNew[i]; - apNew[i] = apNew[minI]; - apNew[minI] = pT; + } + for(i=0; ii ){ + sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); + } + sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); + apNew[i]->pgno = pgno; } } - TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", - apNew[0]->pgno, szNew[0], + + TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " + "%d(%d nc=%d) %d(%d nc=%d)\n", + apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0, nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, - nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); + nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0, + nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0 + )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - /* - ** Evenly distribute the data in apCell[] across the new pages. - ** Insert divider cells into pParent as necessary. + /* If the sibling pages are not leaves, ensure that the right-child pointer + ** of the right-most new sibling page is set to the value that was + ** originally in the same field of the right-most old sibling page. */ + if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ + MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; + memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); + } + + /* Make any required updates to pointer map entries associated with + ** cells stored on sibling pages following the balance operation. Pointer + ** map entries associated with divider cells are set by the insertCell() + ** routine. The associated pointer map entries are: + ** + ** a) if the cell contains a reference to an overflow chain, the + ** entry associated with the first page in the overflow chain, and + ** + ** b) if the sibling pages are not leaves, the child page associated + ** with the cell. + ** + ** If the sibling pages are not leaves, then the pointer map entry + ** associated with the right-child of each sibling may also need to be + ** updated. This happens below, after the sibling pages have been + ** populated, not here. */ - j = 0; - for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); - assert( pNew->nOverflow==0 ); + if( ISAUTOVACUUM ){ + MemPage *pNew = apNew[0]; + u8 *aOld = pNew->aData; + int cntOldNext = pNew->nCell + pNew->nOverflow; + int usableSize = pBt->usableSize; + int iNew = 0; + int iOld = 0; - j = cntNew[i]; - - /* If the sibling page assembled above was not the right-most sibling, - ** insert a divider cell into the parent page. - */ - assert( ileaf ){ - memcpy(&pNew->aData[8], pCell, 4); - }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of - ** the sibling-page assembled above only. - */ - CellInfo info; - j--; - btreeParseCellPtr(pNew, apCell[j], &info); - pCell = pTemp; - sz = 4 + putVarint(&pCell[4], info.nKey); - pTemp = 0; - }else{ - pCell -= 4; - /* Obscure case for non-leaf-data trees: If the cell at pCell was - ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this - ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to - ** insertCell(), so reparse the cell now. - ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. - */ - if( szCell[j]==4 ){ - assert(leafCorrection==4); - sz = cellSizePtr(pParent, pCell); - } + for(i=0; inCell + pOld->nOverflow + !leafData; + aOld = pOld->aData; + } + if( i==cntNew[iNew] ){ + pNew = apNew[++iNew]; + if( !leafData ) continue; } - iOvflSpace += sz; - assert( sz<=pBt->maxLocal+23 ); - assert( iOvflSpace <= (int)pBt->pageSize ); - insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); - if( rc!=SQLITE_OK ) goto balance_cleanup; - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - j++; - nxDiv++; + /* Cell pCell is destined for new sibling page pNew. Originally, it + ** was either part of sibling page iOld (possibly an overflow cell), + ** or else the divider cell to the left of sibling page iOld. So, + ** if sibling page iOld had the same page number as pNew, and if + ** pCell really was a part of sibling page iOld (not a divider or + ** overflow cell), we can skip updating the pointer map entries. */ + if( iOld>=nNew + || pNew->pgno!=aPgno[iOld] + || !SQLITE_WITHIN(pCell,aOld,&aOld[usableSize]) + ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( cachedCellSize(&b,i)>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pCell, &rc); + } + if( rc ) goto balance_cleanup; + } } } - assert( j==nCell ); + + /* Insert new divider cells into pParent. */ + for(i=0; ileaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in b.apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + pNew->xParseCell(pNew, b.apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** This can only happen for b-trees used to evaluate "IN (SELECT ...)" + ** and WITHOUT ROWID tables with exactly one column which is the + ** primary key. + */ + if( b.szCell[j]==4 ){ + assert(leafCorrection==4); + sz = pParent->xCellSize(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + } + + /* Now update the actual sibling pages. The order in which they are updated + ** is important, as this code needs to avoid disrupting any page from which + ** cells may still to be read. In practice, this means: + ** + ** (1) If cells are moving left (from apNew[iPg] to apNew[iPg-1]) + ** then it is not safe to update page apNew[iPg] until after + ** the left-hand sibling apNew[iPg-1] has been updated. + ** + ** (2) If cells are moving right (from apNew[iPg] to apNew[iPg+1]) + ** then it is not safe to update page apNew[iPg] until after + ** the right-hand sibling apNew[iPg+1] has been updated. + ** + ** If neither of the above apply, the page is safe to update. + ** + ** The iPg value in the following loop starts at nNew-1 goes down + ** to 0, then back up to nNew-1 again, thus making two passes over + ** the pages. On the initial downward pass, only condition (1) above + ** needs to be tested because (2) will always be true from the previous + ** step. On the upward pass, both conditions are always true, so the + ** upwards pass simply processes pages that were missed on the downward + ** pass. + */ + for(i=1-nNew; i=0 && iPg=0 /* On the upwards pass, or... */ + || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ + ){ + int iNew; + int iOld; + int nNewCell; + + /* Verify condition (1): If cells are moving left, update iPg + ** only after iPg-1 has already been updated. */ + assert( iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1] ); + + /* Verify condition (2): If cells are moving right, update iPg + ** only after iPg+1 has already been updated. */ + assert( cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1] ); + + if( iPg==0 ){ + iNew = iOld = 0; + nNewCell = cntNew[0]; + }else{ + iOld = iPgnFree = usableSpace-szNew[iPg]; + assert( apNew[iPg]->nOverflow==0 ); + assert( apNew[iPg]->nCell==nNewCell ); + } + } + + /* All pages have been processed exactly once */ + assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); + assert( nOld>0 ); assert( nNew>0 ); - if( (pageFlags & PTF_LEAF)==0 ){ - u8 *zChild = &apCopy[nOld-1]->aData[8]; - memcpy(&apNew[nNew-1]->aData[8], zChild, 4); - } if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ /* The root page of the b-tree now contains no cells. The only sibling @@ -61050,132 +68985,56 @@ static int balance_nonroot( ** sets all pointer-map entries corresponding to database image pages ** for which the pointer is stored within the content being copied. ** - ** The second assert below verifies that the child page is defragmented - ** (it must be, as it was just reconstructed using assemblePage()). This - ** is important if the parent page happens to be page 1 of the database - ** image. */ - assert( nNew==1 ); + ** It is critical that the child page be defragmented before being + ** copied into the parent, because if the parent is page 1 then it will + ** by smaller than the child due to the database header, and so all the + ** free space needs to be up front. + */ + assert( nNew==1 || CORRUPT_DB ); + rc = defragmentPage(apNew[0]); + testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + || rc!=SQLITE_OK ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM ){ - /* Fix the pointer-map entries for all the cells that were shifted around. - ** There are several different types of pointer-map entries that need to - ** be dealt with by this routine. Some of these have been set already, but - ** many have not. The following is a summary: - ** - ** 1) The entries associated with new sibling pages that were not - ** siblings when this function was called. These have already - ** been set. We don't need to worry about old siblings that were - ** moved to the free-list - the freePage() code has taken care - ** of those. - ** - ** 2) The pointer-map entries associated with the first overflow - ** page in any overflow chains used by new divider cells. These - ** have also already been taken care of by the insertCell() code. - ** - ** 3) If the sibling pages are not leaves, then the child pages of - ** cells stored on the sibling pages may need to be updated. - ** - ** 4) If the sibling pages are not internal intkey nodes, then any - ** overflow pages used by these cells may need to be updated - ** (internal intkey nodes never contain pointers to overflow pages). - ** - ** 5) If the sibling pages are not leaves, then the pointer-map - ** entries for the right-child pages of each sibling may need - ** to be updated. - ** - ** Cases 1 and 2 are dealt with above by other code. The next - ** block deals with cases 3 and 4 and the one after that, case 5. Since - ** setting a pointer map entry is a relatively expensive operation, this - ** code only sets pointer map entries for child or overflow pages that have - ** actually moved between pages. */ - MemPage *pNew = apNew[0]; - MemPage *pOld = apCopy[0]; - int nOverflow = pOld->nOverflow; - int iNextOld = pOld->nCell + nOverflow; - int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1); - j = 0; /* Current 'old' sibling page */ - k = 0; /* Current 'new' sibling page */ - for(i=0; inCell + pOld->nOverflow; - if( pOld->nOverflow ){ - nOverflow = pOld->nOverflow; - iOverflow = i + !leafData + pOld->aiOvfl[0]; - } - isDivider = !leafData; - } - - assert(nOverflow>0 || iOverflowaiOvfl[0]==pOld->aiOvfl[1]-1); - assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1); - if( i==iOverflow ){ - isDivider = 1; - if( (--nOverflow)>0 ){ - iOverflow++; - } - } - - if( i==cntNew[k] ){ - /* Cell i is the cell immediately following the last cell on new - ** sibling page k. If the siblings are not leaf pages of an - ** intkey b-tree, then cell i is a divider cell. */ - pNew = apNew[++k]; - if( !leafData ) continue; - } - assert( jpgno!=pNew->pgno ){ - if( !leafCorrection ){ - ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); - } - if( szCell[i]>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, apCell[i], &rc); - } - } + }else if( ISAUTOVACUUM && !leafCorrection ){ + /* Fix the pointer map entries associated with the right-child of each + ** sibling page. All other pointer map entries have already been taken + ** care of. */ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); } + } - if( !leafCorrection ){ - for(i=0; iaData[8]); - ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); - } - } + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, b.nCell)); + + /* Free any old pages that were not reused as new pages. + */ + for(i=nNew; iisInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); ptrmapCheckPages(&pParent, 1); -#endif } - - assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, nCell)); +#endif /* ** Cleanup before returning. */ balance_cleanup: - sqlite3ScratchFree(apCell); + sqlite3ScratchFree(b.apCell); for(i=0; i= 1700 && defined(_M_ARM) -#pragma optimize("", on) -#endif /* @@ -61272,8 +69128,8 @@ static int balance(BtCursor *pCur){ u8 aBalanceQuickSpace[13]; u8 *pFree = 0; - TESTONLY( int balance_quick_called = 0 ); - TESTONLY( int balance_deeper_called = 0 ); + VVA_ONLY( int balance_quick_called = 0 ); + VVA_ONLY( int balance_deeper_called = 0 ); do { int iPage = pCur->iPage; @@ -61286,7 +69142,8 @@ static int balance(BtCursor *pCur){ ** and copy the current contents of the root-page to it. The ** next iteration of the do-loop will balance the child page. */ - assert( (balance_deeper_called++)==0 ); + assert( balance_deeper_called==0 ); + VVA_ONLY( balance_deeper_called++ ); rc = balance_deeper(pPage, &pCur->apPage[1]); if( rc==SQLITE_OK ){ pCur->iPage = 1; @@ -61306,7 +69163,7 @@ static int balance(BtCursor *pCur){ rc = sqlite3PagerWrite(pParent->pDbPage); if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_QUICKBALANCE - if( pPage->hasData + if( pPage->intKeyLeaf && pPage->nOverflow==1 && pPage->aiOvfl[0]==pPage->nCell && pParent->pgno!=1 @@ -61315,7 +69172,7 @@ static int balance(BtCursor *pCur){ /* Call balance_quick() to create a new sibling of pPage on which ** to store the overflow cell. balance_quick() inserts a new cell ** into pParent, which may cause pParent overflow. If this - ** happens, the next interation of the do-loop will balance pParent + ** happens, the next iteration of the do-loop will balance pParent ** use either balance_nonroot() or balance_deeper(). Until this ** happens, the overflow cell is stored in the aBalanceQuickSpace[] ** buffer. @@ -61325,7 +69182,8 @@ static int balance(BtCursor *pCur){ ** function. If this were not verified, a subtle bug involving reuse ** of the aBalanceQuickSpace[] might sneak in. */ - assert( (balance_quick_called++)==0 ); + assert( balance_quick_called==0 ); + VVA_ONLY( balance_quick_called++ ); rc = balance_quick(pParent, pPage, aBalanceQuickSpace); }else #endif @@ -61348,7 +69206,8 @@ static int balance(BtCursor *pCur){ ** pSpace buffer passed to the latter call to balance_nonroot(). */ u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize); - rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, pCur->hints); + rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, + pCur->hints&BTREE_BULKLOAD); if( pFree ){ /* If pFree is not NULL, it points to the pSpace buffer used ** by a previous call to balance_nonroot(). Its contents are @@ -61369,6 +69228,7 @@ static int balance(BtCursor *pCur){ /* The next iteration of the do-loop balances the parent page. */ releasePage(pPage); pCur->iPage--; + assert( pCur->iPage>=0 ); } }while( rc==SQLITE_OK ); @@ -61380,19 +69240,25 @@ static int balance(BtCursor *pCur){ /* -** Insert a new record into the BTree. The key is given by (pKey,nKey) -** and the data is given by (pData,nData). The cursor is used only to -** define what table the record should be inserted into. The cursor -** is left pointing at a random location. +** Insert a new record into the BTree. The content of the new record +** is described by the pX object. The pCur cursor is used only to +** define what table the record should be inserted into, and is left +** pointing at a random location. ** -** For an INTKEY table, only the nKey value of the key is used. pKey is -** ignored. For a ZERODATA table, the pData and nData are both ignored. +** For a table btree (used for rowid tables), only the pX.nKey value of +** the key is used. The pX.pKey value must be NULL. The pX.nKey is the +** rowid or INTEGER PRIMARY KEY of the row. The pX.nData,pData,nZero fields +** hold the content of the row. +** +** For an index btree (used for indexes and WITHOUT ROWID tables), the +** key is an arbitrary byte sequence stored in pX.pKey,nKey. The +** pX.pData,nData,nZero fields must be zero. ** ** If the seekResult parameter is non-zero, then a successful call to ** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already ** been performed. seekResult is the search result returned (a negative ** number if pCur points at an entry that is smaller than (pKey, nKey), or -** a positive value if pCur points at an etry that is larger than +** a positive value if pCur points at an entry that is larger than ** (pKey, nKey)). ** ** If the seekResult parameter is non-zero, then the caller guarantees that @@ -61403,9 +69269,7 @@ static int balance(BtCursor *pCur){ */ SQLITE_PRIVATE int sqlite3BtreeInsert( BtCursor *pCur, /* Insert data into the table of this cursor */ - const void *pKey, i64 nKey, /* The key of the new record */ - const void *pData, int nData, /* The data of the new record */ - int nZero, /* Number of extra 0 bytes to append to data */ + const BtreePayload *pX, /* Content of the row to be inserted */ int appendBias, /* True if this is likely an append */ int seekResult /* Result of prior MovetoUnpacked() call */ ){ @@ -61424,8 +69288,9 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( return pCur->skipNext; } - assert( cursorHoldsMutex(pCur) ); - assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE + assert( cursorOwnsBtShared(pCur) ); + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 + && pBt->inTransaction==TRANS_WRITE && (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); @@ -61434,7 +69299,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** keys with no associated data. If the cursor was opened expecting an ** intkey table, the caller should be inserting integer keys with a ** blob of associated data. */ - assert( (pKey==0)==(pCur->pKeyInfo==0) ); + assert( (pX->pKey==0)==(pCur->pKeyInfo==0) ); /* Save the positions of any other cursors open on this table. ** @@ -61447,42 +69312,46 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** doing any work. To avoid thwarting these optimizations, it is important ** not to clear the cursor here. */ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; - - if( pCur->pKeyInfo==0 ){ - /* If this is an insert into a table b-tree, invalidate any incrblob - ** cursors open on the row being replaced */ - invalidateIncrblobCursors(p, nKey, 0); - - /* If the cursor is currently on the last row and we are appending a - ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() - ** call */ - if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){ - loc = -1; - } + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; } - if( !loc ){ - rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); + if( pCur->pKeyInfo==0 ){ + assert( pX->pKey==0 ); + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced */ + invalidateIncrblobCursors(p, pX->nKey, 0); + + /* If the cursor is currently on the last row and we are appending a + ** new row onto the end, set the "loc" to avoid an unnecessary + ** btreeMoveto() call */ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0 + && pCur->info.nKey==pX->nKey-1 ){ + loc = -1; + }else if( loc==0 ){ + rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, appendBias, &loc); + if( rc ) return rc; + } + }else if( loc==0 ){ + rc = btreeMoveto(pCur, pX->pKey, pX->nKey, appendBias, &loc); if( rc ) return rc; } assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); pPage = pCur->apPage[pCur->iPage]; - assert( pPage->intKey || nKey>=0 ); + assert( pPage->intKey || pX->nKey>=0 ); assert( pPage->leaf || !pPage->intKey ); TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", - pCur->pgnoRoot, nKey, nData, pPage->pgno, + pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit ); - allocateTempSpace(pBt); newCell = pBt->pTmpSpace; - if( newCell==0 ) return SQLITE_NOMEM; - rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); + assert( newCell!=0 ); + rc = fillInCell(pPage, newCell, pX, &szNew); if( rc ) goto end_insert; - assert( szNew==cellSizePtr(pPage, newCell) ); + assert( szNew==pPage->xCellSize(pPage, newCell) ); assert( szNew <= MX_CELL_SIZE(pBt) ); idx = pCur->aiIdx[pCur->iPage]; if( loc==0 ){ @@ -61496,8 +69365,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( !pPage->leaf ){ memcpy(newCell, oldCell, 4); } - szOld = cellSizePtr(pPage, oldCell); - rc = clearCell(pPage, oldCell); + rc = clearCell(pPage, oldCell, &szOld); dropCell(pPage, idx, szOld, &rc); if( rc ) goto end_insert; }else if( loc<0 && pPage->nCell>0 ){ @@ -61507,6 +69375,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( assert( pPage->leaf ); } insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); /* If no error has occurred and pPage has an overflow cell, call balance() @@ -61530,7 +69399,8 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** row without seeking the cursor. This can be a big performance boost. */ pCur->info.nSize = 0; - if( rc==SQLITE_OK && pPage->nOverflow ){ + if( pPage->nOverflow ){ + assert( rc==SQLITE_OK ); pCur->curFlags &= ~(BTCF_ValidNKey); rc = balance(pCur); @@ -61548,10 +69418,23 @@ end_insert: } /* -** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at a arbitrary location. +** Delete the entry that the cursor is pointing to. +** +** If the BTREE_SAVEPOSITION bit of the flags parameter is zero, then +** the cursor is left pointing at an arbitrary location after the delete. +** But if that bit is set, then the cursor is left in a state such that +** the next call to BtreeNext() or BtreePrev() moves it to the same row +** as it would have been on if the call to BtreeDelete() had been omitted. +** +** The BTREE_AUXDELETE bit of flags indicates that is one of several deletes +** associated with a single table entry and its indexes. Only one of those +** deletes is considered the "primary" delete. The primary delete occurs +** on a cursor that is not a BTREE_FORDELETE cursor. All but one delete +** operation on non-FORDELETE cursors is tagged with the AUXDELETE flag. +** The BTREE_AUXDELETE bit is a hint that is not used by this implementation, +** but which might be used by alternative storage engines. */ -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ Btree *p = pCur->pBtree; BtShared *pBt = p->pBt; int rc; /* Return code */ @@ -61559,25 +69442,47 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ unsigned char *pCell; /* Pointer to cell to delete */ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ + u16 szCell; /* Size of the cell being deleted */ + int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ + u8 bPreserve = flags & BTREE_SAVEPOSITION; /* Keep cursor valid */ - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); - - if( NEVER(pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell) - || NEVER(pCur->eState!=CURSOR_VALID) - ){ - return SQLITE_ERROR; /* Something has gone awry. */ - } + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + assert( pCur->eState==CURSOR_VALID ); + assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; pPage = pCur->apPage[iCellDepth]; pCell = findCell(pPage, iCellIdx); + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + /* If the page containing the entry to delete is not a leaf page, move ** the cursor to the largest entry in the tree that is smaller than ** the entry being deleted. This cell will replace the cell being deleted @@ -61592,12 +69497,11 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ } /* Save the positions of any other cursors open on this table before - ** making any modifications. Make the page containing the entry to be - ** deleted writable. Then free any overflow pages associated with the - ** entry and finally remove the cell itself from within the page. - */ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; + ** making any modifications. */ + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + } /* If this is a delete operation to remove a row from a table b-tree, ** invalidate any incrblob cursors open on the row being deleted. */ @@ -61605,10 +69509,13 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ invalidateIncrblobCursors(p, pCur->info.nKey, 0); } + /* Make the page containing the entry to be deleted writable. Then free any + ** overflow pages associated with the entry and finally remove the cell + ** itself from within the page. */ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; - rc = clearCell(pPage, pCell); - dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc); + rc = clearCell(pPage, pCell, &szCell); + dropCell(pPage, iCellIdx, szCell, &rc); if( rc ) return rc; /* If the cell deleted was not located on a leaf page, then the cursor @@ -61623,14 +69530,15 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ unsigned char *pTmp; pCell = findCell(pLeaf, pLeaf->nCell-1); - nCell = cellSizePtr(pLeaf, pCell); + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); - - allocateTempSpace(pBt); pTmp = pBt->pTmpSpace; - + assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + if( rc==SQLITE_OK ){ + insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + } dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; } @@ -61659,7 +69567,23 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ } if( rc==SQLITE_OK ){ - moveToRoot(pCur); + if( bSkipnext ){ + assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); + assert( pPage==pCur->apPage[pCur->iPage] || CORRUPT_DB ); + assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); + pCur->eState = CURSOR_SKIPNEXT; + if( iCellIdx>=pPage->nCell ){ + pCur->skipNext = -1; + pCur->aiIdx[iCellDepth] = pPage->nCell-1; + }else{ + pCur->skipNext = 1; + } + }else{ + rc = moveToRoot(pCur); + if( bPreserve ){ + pCur->eState = CURSOR_REQUIRESEEK; + } + } } return rc; } @@ -61717,7 +69641,8 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ pgnoRoot++; } - assert( pgnoRoot>=3 ); + assert( pgnoRoot>=3 || CORRUPT_DB ); + testcase( pgnoRoot<3 ); /* Allocate a page. The page that currently resides at pgnoRoot will ** be moved to the allocated page (unless the allocated page happens @@ -61840,14 +69765,19 @@ static int clearDatabasePage( unsigned char *pCell; int i; int hdr; + u16 szCell; assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ return SQLITE_CORRUPT_BKPT; } - - rc = getAndInitPage(pBt, pgno, &pPage, 0); + rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); if( rc ) return rc; + if( pPage->bBusy ){ + rc = SQLITE_CORRUPT_BKPT; + goto cleardatabasepage_out; + } + pPage->bBusy = 1; hdr = pPage->hdrOffset; for(i=0; inCell; i++){ pCell = findCell(pPage, i); @@ -61855,14 +69785,15 @@ static int clearDatabasePage( rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); if( rc ) goto cleardatabasepage_out; } - rc = clearCell(pPage, pCell); + rc = clearCell(pPage, pCell, &szCell); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); if( rc ) goto cleardatabasepage_out; }else if( pnChange ){ - assert( pPage->intKey ); + assert( pPage->intKey || CORRUPT_DB ); + testcase( !pPage->intKey ); *pnChange += pPage->nCell; } if( freePageFlag ){ @@ -61872,6 +69803,7 @@ static int clearDatabasePage( } cleardatabasepage_out: + pPage->bBusy = 0; releasePage(pPage); return rc; } @@ -61958,6 +69890,14 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } + /* + ** It is illegal to drop the sqlite_master table on page 1. But again, + ** this error is caught long before reaching this point. + */ + if( NEVER(iTable<2) ){ + return SQLITE_CORRUPT_BKPT; + } + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); @@ -61968,76 +69908,67 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ *piMoved = 0; - if( iTable>1 ){ #ifdef SQLITE_OMIT_AUTOVACUUM - freePage(pPage, &rc); - releasePage(pPage); + freePage(pPage, &rc); + releasePage(pPage); #else - if( pBt->autoVacuum ){ - Pgno maxRootPgno; - sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); + if( pBt->autoVacuum ){ + Pgno maxRootPgno; + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); - if( iTable==maxRootPgno ){ - /* If the table being dropped is the table with the largest root-page - ** number in the database, put the root page on the free list. - */ - freePage(pPage, &rc); - releasePage(pPage); - if( rc!=SQLITE_OK ){ - return rc; - } - }else{ - /* The table being dropped does not have the largest root-page - ** number in the database. So move the page that does into the - ** gap left by the deleted root-page. - */ - MemPage *pMove; - releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; - } - pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - freePage(pMove, &rc); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; - } - *piMoved = maxRootPgno; - } - - /* Set the new 'max-root-page' value in the database header. This - ** is the old value less one, less one more if that happens to - ** be a root-page number, less one again if that is the - ** PENDING_BYTE_PAGE. + if( iTable==maxRootPgno ){ + /* If the table being dropped is the table with the largest root-page + ** number in the database, put the root page on the free list. */ - maxRootPgno--; - while( maxRootPgno==PENDING_BYTE_PAGE(pBt) - || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ - maxRootPgno--; - } - assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); - - rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); - }else{ freePage(pPage, &rc); releasePage(pPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + /* The table being dropped does not have the largest root-page + ** number in the database. So move the page that does into the + ** gap left by the deleted root-page. + */ + MemPage *pMove; + releasePage(pPage); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + pMove = 0; + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + freePage(pMove, &rc); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + *piMoved = maxRootPgno; } -#endif - }else{ - /* If sqlite3BtreeDropTable was called on page 1. - ** This really never should happen except in a corrupt - ** database. + + /* Set the new 'max-root-page' value in the database header. This + ** is the old value less one, less one more if that happens to + ** be a root-page number, less one again if that is the + ** PENDING_BYTE_PAGE. */ - zeroPage(pPage, PTF_INTKEY|PTF_LEAF ); + maxRootPgno--; + while( maxRootPgno==PENDING_BYTE_PAGE(pBt) + || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ + maxRootPgno--; + } + assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); + + rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); + }else{ + freePage(pPage, &rc); releasePage(pPage); } +#endif return rc; } SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ @@ -62061,6 +69992,13 @@ SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ ** The schema layer numbers meta values differently. At the schema ** layer (and the SetCookie and ReadCookie opcodes) the number of ** free pages is not visible. So Cookie[0] is the same as Meta[1]. +** +** This routine treats Meta[BTREE_DATA_VERSION] as a special case. Instead +** of reading the value out of the header, it instead loads the "DataVersion" +** from the pager. The BTREE_DATA_VERSION value is not actually stored in the +** database file. It is a number computed by the pager. But its access +** pattern is the same as header meta values, and so it is convenient to +** read it from this routine. */ SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ BtShared *pBt = p->pBt; @@ -62071,7 +70009,11 @@ SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ assert( pBt->pPage1 ); assert( idx>=0 && idx<=15 ); - *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + if( idx==BTREE_DATA_VERSION ){ + *pMeta = sqlite3PagerDataVersion(pBt->pPager) + p->iDataVersion; + }else{ + *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + } /* If auto-vacuum is disabled in this build and this is an auto-vacuum ** database, mark the database as read-only. */ @@ -62162,7 +70104,7 @@ SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ if( pCur->iPage==0 ){ /* All pages of the b-tree have been visited. Return successfully. */ *pnEntry = nEntry; - return SQLITE_OK; + return moveToRoot(pCur); } moveToParent(pCur); }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell ); @@ -62201,7 +70143,6 @@ SQLITE_PRIVATE Pager *sqlite3BtreePager(Btree *p){ */ static void checkAppendMsg( IntegrityCk *pCheck, - char *zMsg1, const char *zFormat, ... ){ @@ -62213,10 +70154,10 @@ static void checkAppendMsg( if( pCheck->errMsg.nChar ){ sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); } - if( zMsg1 ){ - sqlite3StrAccumAppendAll(&pCheck->errMsg, zMsg1); + if( pCheck->zPfx ){ + sqlite3XPrintf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); } - sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); + sqlite3VXPrintf(&pCheck->errMsg, zFormat, ap); va_end(ap); if( pCheck->errMsg.accError==STRACCUM_NOMEM ){ pCheck->mallocFailed = 1; @@ -62247,19 +70188,19 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ /* ** Add 1 to the reference count for page iPage. If this is the second ** reference to the page, add an error message to pCheck->zErrMsg. -** Return 1 if there are 2 ore more references to the page and 0 if +** Return 1 if there are 2 or more references to the page and 0 if ** if this is the first reference to the page. ** ** Also check that the page number is in bounds. */ -static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ +static int checkRef(IntegrityCk *pCheck, Pgno iPage){ if( iPage==0 ) return 1; if( iPage>pCheck->nPage ){ - checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); + checkAppendMsg(pCheck, "invalid page number %d", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ - checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage); + checkAppendMsg(pCheck, "2nd reference to page %d", iPage); return 1; } setPageReferenced(pCheck, iPage); @@ -62276,8 +70217,7 @@ static void checkPtrmap( IntegrityCk *pCheck, /* Integrity check context */ Pgno iChild, /* Child page number */ u8 eType, /* Expected pointer map type */ - Pgno iParent, /* Expected pointer map parent page number */ - char *zContext /* Context description (used for error msg) */ + Pgno iParent /* Expected pointer map parent page number */ ){ int rc; u8 ePtrmapType; @@ -62286,12 +70226,12 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->mallocFailed = 1; - checkAppendMsg(pCheck, zContext, "Failed to read ptrmap key=%d", iChild); + checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } @@ -62306,8 +70246,7 @@ static void checkList( IntegrityCk *pCheck, /* Integrity checking context */ int isFreeList, /* True for a freelist. False for overflow page list */ int iPage, /* Page number for first page in the list */ - int N, /* Expected number of pages in the list */ - char *zContext /* Context for error messages */ + int N /* Expected number of pages in the list */ ){ int i; int expected = N; @@ -62316,14 +70255,14 @@ static void checkList( DbPage *pOvflPage; unsigned char *pOvflData; if( iPage<1 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "%d of %d pages missing from overflow list starting at %d", N+1, expected, iFirst); break; } - if( checkRef(pCheck, iPage, zContext) ) break; - if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){ - checkAppendMsg(pCheck, zContext, "failed to get page %d", iPage); + if( checkRef(pCheck, iPage) ) break; + if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ + checkAppendMsg(pCheck, "failed to get page %d", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); @@ -62331,11 +70270,11 @@ static void checkList( int n = get4byte(&pOvflData[4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); + checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0); } #endif if( n>(int)pCheck->pBt->usableSize/4-2 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "freelist leaf count too big on page %d", iPage); N--; }else{ @@ -62343,10 +70282,10 @@ static void checkList( Pgno iFreePage = get4byte(&pOvflData[8+i*4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext); + checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0); } #endif - checkRef(pCheck, iFreePage, zContext); + checkRef(pCheck, iFreePage); } N -= n; } @@ -62359,16 +70298,71 @@ static void checkList( */ if( pCheck->pBt->autoVacuum && N>0 ){ i = get4byte(pOvflData); - checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext); + checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage); } } #endif iPage = get4byte(pOvflData); sqlite3PagerUnref(pOvflPage); + + if( isFreeList && N<(iPage!=0) ){ + checkAppendMsg(pCheck, "free-page count in header is too small"); + } } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ +/* +** An implementation of a min-heap. +** +** aHeap[0] is the number of elements on the heap. aHeap[1] is the +** root element. The daughter nodes of aHeap[N] are aHeap[N*2] +** and aHeap[N*2+1]. +** +** The heap property is this: Every node is less than or equal to both +** of its daughter nodes. A consequence of the heap property is that the +** root node aHeap[1] is always the minimum value currently in the heap. +** +** The btreeHeapInsert() routine inserts an unsigned 32-bit number onto +** the heap, preserving the heap property. The btreeHeapPull() routine +** removes the root element from the heap (the minimum value in the heap) +** and then moves other nodes around as necessary to preserve the heap +** property. +** +** This heap is used for cell overlap and coverage testing. Each u32 +** entry represents the span of a cell or freeblock on a btree page. +** The upper 16 bits are the index of the first byte of a range and the +** lower 16 bits are the index of the last byte of that range. +*/ +static void btreeHeapInsert(u32 *aHeap, u32 x){ + u32 j, i = ++aHeap[0]; + aHeap[i] = x; + while( (j = i/2)>0 && aHeap[j]>aHeap[i] ){ + x = aHeap[j]; + aHeap[j] = aHeap[i]; + aHeap[i] = x; + i = j; + } +} +static int btreeHeapPull(u32 *aHeap, u32 *pOut){ + u32 j, i, x; + if( (x = aHeap[0])==0 ) return 0; + *pOut = aHeap[1]; + aHeap[1] = aHeap[x]; + aHeap[x] = 0xffffffff; + aHeap[0]--; + i = 1; + while( (j = i*2)<=aHeap[0] ){ + if( aHeap[j]>aHeap[j+1] ) j++; + if( aHeap[i]zPfx; + int saved_v1 = pCheck->v1; + int saved_v2 = pCheck->v2; + u8 savedIsInit = 0; /* Check that the page exists */ pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; - if( checkRef(pCheck, iPage, zParentContext) ) return 0; + if( checkRef(pCheck, iPage) ) return 0; + pCheck->zPfx = "Page %d: "; + pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); - return 0; + goto end_of_check; } /* Clear MemPage.isInit to make sure the corruption detection code in ** btreeInitPage() is executed. */ + savedIsInit = pPage->isInit; pPage->isInit = 0; if( (rc = btreeInitPage(pPage))!=0 ){ assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "btreeInitPage() returns error code %d", rc); - releasePage(pPage); - return 0; + goto end_of_check; + } + data = pPage->aData; + hdr = pPage->hdrOffset; + + /* Set up for cell analysis */ + pCheck->zPfx = "On tree page %d cell %d: "; + contentOffset = get2byteNotZero(&data[hdr+5]); + assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ + + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + nCell = get2byte(&data[hdr+3]); + assert( pPage->nCell==nCell ); + + /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page + ** immediately follows the b-tree page header. */ + cellStart = hdr + 12 - 4*pPage->leaf; + assert( pPage->aCellIdx==&data[cellStart] ); + pCellIdx = &data[cellStart + 2*(nCell-1)]; + + if( !pPage->leaf ){ + /* Analyze the right-child page of internal pages */ + pgno = get4byte(&data[hdr+8]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + pCheck->zPfx = "On page %d at right child: "; + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); + } +#endif + depth = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + }else{ + /* For leaf pages, the coverage check will occur in the same loop + ** as the other cell checks, so initialize the heap. */ + heap = pCheck->heap; + heap[0] = 0; } - /* Check out all the cells. - */ - depth = 0; - for(i=0; inCell && pCheck->mxErr; i++){ - u8 *pCell; - u32 sz; + /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte + ** integer offsets to the cell contents. */ + for(i=nCell-1; i>=0 && pCheck->mxErr; i--){ CellInfo info; - /* Check payload overflow pages - */ - sqlite3_snprintf(sizeof(zContext), zContext, - "On tree page %d cell %d: ", iPage, i); - pCell = findCell(pPage,i); - btreeParseCellPtr(pPage, pCell, &info); - sz = info.nData; - if( !pPage->intKey ) sz += (int)info.nKey; - /* For intKey pages, check that the keys are in order. - */ - else if( i==0 ) nMinKey = nMaxKey = info.nKey; - else{ - if( info.nKey <= nMaxKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); - } - nMaxKey = info.nKey; + /* Check cell size */ + pCheck->v2 = i; + assert( pCellIdx==&data[cellStart + i*2] ); + pc = get2byteAligned(pCellIdx); + pCellIdx -= 2; + if( pcusableSize-4 ){ + checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + pc, contentOffset, usableSize-4); + doCoverageCheck = 0; + continue; } - assert( sz==info.nPayload ); - if( (sz>info.nLocal) - && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) - ){ - int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4); - Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext); - } -#endif - checkList(pCheck, 0, pgnoOvfl, nPage, zContext); + pCell = &data[pc]; + pPage->xParseCell(pPage, pCell, &info); + if( pc+info.nSize>usableSize ){ + checkAppendMsg(pCheck, "Extends off end of page"); + doCoverageCheck = 0; + continue; + } + + /* Check for integer primary key out of range */ + if( pPage->intKey ){ + if( keyCanBeEqual ? (info.nKey > maxKey) : (info.nKey >= maxKey) ){ + checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey); + } + maxKey = info.nKey; + } + + /* Check the content overflow list */ + if( info.nPayload>info.nLocal ){ + int nPage; /* Number of pages on the overflow chain */ + Pgno pgnoOvfl; /* First page of the overflow chain */ + assert( pc + info.nSize - 4 <= usableSize ); + nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); + pgnoOvfl = get4byte(&pCell[info.nSize - 4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); + } +#endif + checkList(pCheck, 0, pgnoOvfl, nPage); } - /* Check sanity of left child page. - */ if( !pPage->leaf ){ + /* Check sanity of left child page for internal pages */ pgno = get4byte(pCell); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0 ? NULL : &nMaxKey); - if( i>0 && d2!=depth ){ - checkAppendMsg(pCheck, zContext, "Child page depth differs"); - } - depth = d2; - } - } - - if( !pPage->leaf ){ - pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - sqlite3_snprintf(sizeof(zContext), zContext, - "On page %d at right child: ", iPage); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); - } -#endif - checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell ? NULL : &nMaxKey); - } - - /* For intKey leaf pages, check that the min/max keys are in order - ** with any left/parent/right pages. - */ - if( pPage->leaf && pPage->intKey ){ - /* if we are a left child page */ - if( pnParentMinKey ){ - /* if we are the left most child page */ - if( !pnParentMaxKey ){ - if( nMaxKey > *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (max larger than parent min of %lld)", - nMaxKey, *pnParentMinKey); - } - }else{ - if( nMinKey <= *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (min less than parent min of %lld)", - nMinKey, *pnParentMinKey); - } - if( nMaxKey > *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (max larger than parent max of %lld)", - nMaxKey, *pnParentMaxKey); - } - *pnParentMinKey = nMaxKey; - } - /* else if we're a right child page */ - } else if( pnParentMaxKey ){ - if( nMinKey <= *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (min less than parent max of %lld)", - nMinKey, *pnParentMaxKey); + d2 = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + if( d2!=depth ){ + checkAppendMsg(pCheck, "Child page depth differs"); + depth = d2; } + }else{ + /* Populate the coverage-checking heap for leaf pages */ + btreeHeapInsert(heap, (pc<<16)|(pc+info.nSize-1)); } } + *piMinKey = maxKey; /* Check for complete coverage of the page */ - data = pPage->aData; - hdr = pPage->hdrOffset; - hit = sqlite3PageMalloc( pBt->pageSize ); - if( hit==0 ){ - pCheck->mallocFailed = 1; - }else{ - int contentOffset = get2byteNotZero(&data[hdr+5]); - assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ - memset(hit+contentOffset, 0, usableSize-contentOffset); - memset(hit, 1, contentOffset); - nCell = get2byte(&data[hdr+3]); - cellStart = hdr + 12 - 4*pPage->leaf; - for(i=0; i=usableSize ){ - checkAppendMsg(pCheck, 0, - "Corruption detected in cell %d on page %d",i,iPage); - }else{ - for(j=pc+size-1; j>=pc; j--) hit[j]++; + pCheck->zPfx = 0; + if( doCoverageCheck && pCheck->mxErr>0 ){ + /* For leaf pages, the min-heap has already been initialized and the + ** cells have already been inserted. But for internal pages, that has + ** not yet been done, so do it now */ + if( !pPage->leaf ){ + heap = pCheck->heap; + heap[0] = 0; + for(i=nCell-1; i>=0; i--){ + u32 size; + pc = get2byteAligned(&data[cellStart+i*2]); + size = pPage->xCellSize(pPage, &data[pc]); + btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } + /* Add the freeblocks to the min-heap + ** + ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header + ** is the offset of the first freeblock, or zero if there are no + ** freeblocks on the page. + */ i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; - assert( i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)i<=usableSize-4 ); /* Enforced by btreeInitPage() */ size = get2byte(&data[i+2]); - assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ - for(j=i+size-1; j>=i; j--) hit[j]++; + assert( (u32)(i+size)<=usableSize ); /* Enforced by btreeInitPage() */ + btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); + /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a + ** big-endian integer which is the offset in the b-tree page of the next + ** freeblock in the chain, or zero if the freeblock is the last on the + ** chain. */ j = get2byte(&data[i]); + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ - assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)j<=usableSize-4 ); /* Enforced by btreeInitPage() */ i = j; } - for(i=cnt=0; i1 ){ - checkAppendMsg(pCheck, 0, - "Multiple uses for byte %d of page %d", i, iPage); + /* Analyze the min-heap looking for overlap between cells and/or + ** freeblocks, and counting the number of untracked bytes in nFrag. + ** + ** Each min-heap entry is of the form: (start_address<<16)|end_address. + ** There is an implied first entry the covers the page header, the cell + ** pointer index, and the gap between the cell pointer index and the start + ** of cell content. + ** + ** The loop below pulls entries from the min-heap in order and compares + ** the start_address against the previous end_address. If there is an + ** overlap, that means bytes are used multiple times. If there is a gap, + ** that gap is added to the fragmentation count. + */ + nFrag = 0; + prev = contentOffset - 1; /* Implied first min-heap entry */ + while( btreeHeapPull(heap,&x) ){ + if( (prev&0xffff)>=(x>>16) ){ + checkAppendMsg(pCheck, + "Multiple uses for byte %u of page %d", x>>16, iPage); break; + }else{ + nFrag += (x>>16) - (prev&0xffff) - 1; + prev = x; } } - if( cnt!=data[hdr+7] ){ - checkAppendMsg(pCheck, 0, + nFrag += usableSize - (prev&0xffff) - 1; + /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments + ** is stored in the fifth field of the b-tree page header. + ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the + ** number of fragmented free bytes within the cell content area. + */ + if( heap[0]==0 && nFrag!=data[hdr+7] ){ + checkAppendMsg(pCheck, "Fragmentation of %d bytes reported as %d on page %d", - cnt, data[hdr+7], iPage); + nFrag, data[hdr+7], iPage); } } - sqlite3PageFree(hit); + +end_of_check: + if( !doCoverageCheck ) pPage->isInit = savedIsInit; releasePage(pPage); + pCheck->zPfx = saved_zPfx; + pCheck->v1 = saved_v1; + pCheck->v2 = saved_v2; return depth+1; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -62620,60 +70650,76 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( int *pnErr /* Write number of errors seen to this variable */ ){ Pgno i; - int nRef; IntegrityCk sCheck; BtShared *pBt = p->pBt; + int savedDbFlags = pBt->db->flags; char zErr[100]; + VVA_ONLY( int nRef ); sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); - nRef = sqlite3PagerRefcount(pBt->pPager); + VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); + assert( nRef>=0 ); sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); sCheck.mxErr = mxErr; sCheck.nErr = 0; sCheck.mallocFailed = 0; - *pnErr = 0; + sCheck.zPfx = 0; + sCheck.v1 = 0; + sCheck.v2 = 0; + sCheck.aPgRef = 0; + sCheck.heap = 0; + sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); + sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; if( sCheck.nPage==0 ){ - sqlite3BtreeLeave(p); - return 0; + goto integrity_ck_cleanup; } sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); if( !sCheck.aPgRef ){ - *pnErr = 1; - sqlite3BtreeLeave(p); - return 0; + sCheck.mallocFailed = 1; + goto integrity_ck_cleanup; } + sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); + if( sCheck.heap==0 ){ + sCheck.mallocFailed = 1; + goto integrity_ck_cleanup; + } + i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); - sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); - sCheck.errMsg.useMalloc = 2; /* Check the integrity of the freelist */ + sCheck.zPfx = "Main freelist: "; checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), - get4byte(&pBt->pPage1->aData[36]), "Main freelist: "); + get4byte(&pBt->pPage1->aData[36])); + sCheck.zPfx = 0; /* Check all the tables. */ + testcase( pBt->db->flags & SQLITE_CellSizeCk ); + pBt->db->flags &= ~SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); } #endif - checkTreePage(&sCheck, aRoot[i], "List of tree roots: ", NULL, NULL); + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); } + pBt->db->flags = savedDbFlags; /* Make sure every page in the file is referenced */ for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %d is never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain @@ -62681,37 +70727,29 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( */ if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %d is never used", i); } if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i); + checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); } #endif } - /* Make sure this analysis did not leave any unref() pages. - ** This is an internal consistency check; an integrity check - ** of the integrity check. - */ - if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ - checkAppendMsg(&sCheck, 0, - "Outstanding page count goes from %d to %d during this analysis", - nRef, sqlite3PagerRefcount(pBt->pPager) - ); - } - /* Clean up and report errors. */ - sqlite3BtreeLeave(p); +integrity_ck_cleanup: + sqlite3PageFree(sCheck.heap); sqlite3_free(sCheck.aPgRef); if( sCheck.mallocFailed ){ sqlite3StrAccumReset(&sCheck.errMsg); - *pnErr = sCheck.nErr+1; - return 0; + sCheck.nErr++; } *pnErr = sCheck.nErr; if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg); + /* Make sure this analysis did not leave any unref() pages. */ + assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); + sqlite3BtreeLeave(p); return sqlite3StrAccumFinish(&sCheck.errMsg); } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -62874,7 +70912,7 @@ SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ */ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ int rc; - assert( cursorHoldsMutex(pCsr) ); + assert( cursorOwnsBtShared(pCsr) ); assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); assert( pCsr->curFlags & BTCF_Incrblob ); @@ -62891,7 +70929,7 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void ** required in case any of them are holding references to an xFetch ** version of the b-tree page modified by the accessPayload call below. ** - ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** Note that pCsr must be open on a INTKEY table and saveCursorPosition() ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence ** saveAllCursors can only return SQLITE_OK. */ @@ -62922,6 +70960,7 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void */ SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ pCur->curFlags |= BTCF_Incrblob; + pCur->pBtree->hasIncrblobCur = 1; } #endif @@ -62962,12 +71001,11 @@ SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ } /* -** set the mask of hint flags for cursor pCsr. Currently the only valid -** values are 0 and BTREE_BULKLOAD. +** Return true if the cursor has a hint specified. This routine is +** only used from within assert() statements */ -SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ - assert( mask==BTREE_BULKLOAD || mask==0 ); - pCsr->hints = mask; +SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor *pCsr, unsigned int mask){ + return (pCsr->hints & mask)!=0; } /* @@ -62977,6 +71015,30 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ return (p->pBt->btsFlags & BTS_READ_ONLY)!=0; } +/* +** Return the size of the header added to each page by this module. +*/ +SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } + +#if !defined(SQLITE_OMIT_SHARED_CACHE) +/* +** Return true if the Btree passed as the only argument is sharable. +*/ +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ + return p->sharable; +} + +/* +** Return the number of connections to the BtShared object accessed by +** the Btree handle passed as the only argument. For private caches +** this is always 1. For shared caches it may be 1 or greater. +*/ +SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree *p){ + testcase( p->sharable ); + return p->pBt->nRef; +} +#endif + /************** End of btree.c ***********************************************/ /************** Begin file backup.c ******************************************/ /* @@ -62993,6 +71055,8 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ ** This file contains the implementation of the sqlite3_backup_XXX() ** API functions and the related features. */ +/* #include "sqliteInt.h" */ +/* #include "btreeInt.h" */ /* ** Structure allocated for each backup operation. @@ -63062,29 +71126,23 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ int i = sqlite3FindDbName(pDb, zDb); if( i==1 ){ - Parse *pParse; + Parse sParse; int rc = 0; - pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); - if( pParse==0 ){ - sqlite3Error(pErrorDb, SQLITE_NOMEM, "out of memory"); - rc = SQLITE_NOMEM; - }else{ - pParse->db = pDb; - if( sqlite3OpenTempDatabase(pParse) ){ - sqlite3Error(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); - rc = SQLITE_ERROR; - } - sqlite3DbFree(pErrorDb, pParse->zErrMsg); - sqlite3ParserReset(pParse); - sqlite3StackFree(pErrorDb, pParse); + memset(&sParse, 0, sizeof(sParse)); + sParse.db = pDb; + if( sqlite3OpenTempDatabase(&sParse) ){ + sqlite3ErrorWithMsg(pErrorDb, sParse.rc, "%s", sParse.zErrMsg); + rc = SQLITE_ERROR; } + sqlite3DbFree(pErrorDb, sParse.zErrMsg); + sqlite3ParserReset(&sParse); if( rc ){ return 0; } } if( i<0 ){ - sqlite3Error(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); + sqlite3ErrorWithMsg(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); return 0; } @@ -63101,6 +71159,20 @@ static int setDestPgsz(sqlite3_backup *p){ return rc; } +/* +** Check that there is no open read-transaction on the b-tree passed as the +** second argument. If there is not, return SQLITE_OK. Otherwise, if there +** is an open read-transaction, return SQLITE_ERROR and leave an error +** message in database handle db. +*/ +static int checkReadTransaction(sqlite3 *db, Btree *p){ + if( sqlite3BtreeIsInReadTrans(p) ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + /* ** Create an sqlite3_backup process to copy the contents of zSrcDb from ** connection handle pSrcDb to zDestDb in pDestDb. If successful, return @@ -63117,6 +71189,13 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( ){ sqlite3_backup *p; /* Value to return */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + /* Lock the source database handle. The destination database ** handle is not locked in this routine, but it is locked in ** sqlite3_backup_step(). The user is required to ensure that no @@ -63129,7 +71208,7 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( sqlite3_mutex_enter(pDestDb->mutex); if( pSrcDb==pDestDb ){ - sqlite3Error( + sqlite3ErrorWithMsg( pDestDb, SQLITE_ERROR, "source and destination must be distinct" ); p = 0; @@ -63140,7 +71219,7 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( ** sqlite3_backup_finish(). */ p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); if( !p ){ - sqlite3Error(pDestDb, SQLITE_NOMEM, 0); + sqlite3Error(pDestDb, SQLITE_NOMEM_BKPT); } } @@ -63153,12 +71232,14 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( p->iNext = 1; p->isAttached = 0; - if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){ + if( 0==p->pSrc || 0==p->pDest + || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK + ){ /* One (or both) of the named databases did not exist or an OOM - ** error was hit. The error has already been written into the - ** pDestDb handle. All that is left to do here is free the - ** sqlite3_backup structure. - */ + ** error was hit. Or there is a transaction open on the destination + ** database. The error has already been written into the pDestDb + ** handle. All that is left to do here is free the sqlite3_backup + ** structure. */ sqlite3_free(p); p = 0; } @@ -63202,7 +71283,7 @@ static int backupOnePage( ** guaranteed that the shared-mutex is held by this thread, handle ** p->pSrc may not actually be the owner. */ int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); - int nDestReserve = sqlite3BtreeGetReserve(p->pDest); + int nDestReserve = sqlite3BtreeGetOptimalReserve(p->pDest); #endif int rc = SQLITE_OK; i64 iOff; @@ -63248,7 +71329,7 @@ static int backupOnePage( DbPage *pDestPg = 0; Pgno iDest = (Pgno)(iOff/nDestPgsz)+1; if( iDest==PENDING_BYTE_PAGE(p->pDest->pBt) ) continue; - if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg)) + if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg, 0)) && SQLITE_OK==(rc = sqlite3PagerWrite(pDestPg)) ){ const u8 *zIn = &zSrcData[iOff%nSrcPgsz]; @@ -63313,6 +71394,9 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ int pgszSrc = 0; /* Source page size */ int pgszDest = 0; /* Destination page size */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(p->pSrcDb->mutex); sqlite3BtreeEnter(p->pSrc); if( p->pDestDb ){ @@ -63336,14 +71420,6 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = SQLITE_OK; } - /* Lock the destination database, if it is not locked already. */ - if( SQLITE_OK==rc && p->bDestLocked==0 - && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) - ){ - p->bDestLocked = 1; - sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); - } - /* If there is no open read-transaction on the source database, open ** one now. If a transaction is opened here, then it will be closed ** before this function exits. @@ -63353,6 +71429,24 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ bCloseTrans = 1; } + /* If the destination database has not yet been locked (i.e. if this + ** is the first call to backup_step() for the current backup operation), + ** try to set its page size to the same as the source database. This + ** is especially important on ZipVFS systems, as in that case it is + ** not possible to create a database file that uses one page size by + ** writing to it with another. */ + if( p->bDestLocked==0 && rc==SQLITE_OK && setDestPgsz(p)==SQLITE_NOMEM ){ + rc = SQLITE_NOMEM; + } + + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) + ){ + p->bDestLocked = 1; + sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); + } + /* Do not allow backup if the destination database is in WAL mode ** and the page sizes are different between source and destination */ pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); @@ -63371,8 +71465,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, - PAGER_GET_READONLY); + rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg,PAGER_GET_READONLY); if( rc==SQLITE_OK ){ rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); @@ -63472,7 +71565,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ DbPage *pPg; - rc = sqlite3PagerGet(pDestPager, iPg, &pPg); + rc = sqlite3PagerGet(pDestPager, iPg, &pPg, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg); sqlite3PagerUnref(pPg); @@ -63492,7 +71585,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ ){ PgHdr *pSrcPg = 0; const Pgno iSrcPg = (Pgno)((iOff/pgszSrc)+1); - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg, 0); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); @@ -63534,7 +71627,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ } if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } p->rc = rc; } @@ -63576,12 +71669,12 @@ SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ } /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest, SQLITE_OK); + sqlite3BtreeRollback(p->pDest, SQLITE_OK, 0); /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; if( p->pDestDb ){ - sqlite3Error(p->pDestDb, rc, 0); + sqlite3Error(p->pDestDb, rc); /* Exit the mutexes and free the backup context structure. */ sqlite3LeaveMutexAndCloseZombie(p->pDestDb); @@ -63602,6 +71695,12 @@ SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ ** call to sqlite3_backup_step(). */ SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nRemaining; } @@ -63610,6 +71709,12 @@ SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ ** recent call to sqlite3_backup_step(). */ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nPagecount; } @@ -63625,9 +71730,13 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ ** corresponding to the source database is held when this function is ** called. */ -SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ - sqlite3_backup *p; /* Iterator variable */ - for(p=pBackup; p; p=p->pNext){ +static SQLITE_NOINLINE void backupUpdate( + sqlite3_backup *p, + Pgno iPage, + const u8 *aData +){ + assert( p!=0 ); + do{ assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); if( !isFatalError(p->rc) && iPageiNext ){ /* The backup process p has already copied page iPage. But now it @@ -63644,7 +71753,10 @@ SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, con p->rc = rc; } } - } + }while( (p = p->pNext)!=0 ); +} +SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ + if( pBackup ) backupUpdate(pBackup, iPage, aData); } /* @@ -63702,15 +71814,19 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ b.pDest = pTo; b.iNext = 1; +#ifdef SQLITE_HAS_CODEC + sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); +#endif + /* 0x7FFFFFFF is the hard limit for the number of pages in a database ** file. By passing this as the number of pages to copy to ** sqlite3_backup_step(), we can guarantee that the copy finishes ** within a single call (unless an error occurs). The assert() statement ** checks this assumption - (p->rc) should be set to either SQLITE_DONE - ** or an error code. - */ + ** or an error code. */ sqlite3_backup_step(&b, 0x7FFFFFFF); assert( b.rc!=SQLITE_OK ); + rc = sqlite3_backup_finish(&b); if( rc==SQLITE_OK ){ pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; @@ -63745,6 +71861,8 @@ copy_finished: ** only within the VDBE. Interface routines refer to a Mem using the ** name sqlite_value */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifdef SQLITE_DEBUG /* @@ -63754,29 +71872,40 @@ copy_finished: ** this: assert( sqlite3VdbeCheckMemInvariants(pMem) ); */ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ - /* The MEM_Dyn bit is set if and only if Mem.xDel is a non-NULL destructor - ** function for Mem.z + /* If MEM_Dyn is set then Mem.xDel!=0. + ** Mem.xDel is might not be initialized if MEM_Dyn is clear. */ assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); - assert( (p->flags & MEM_Dyn)!=0 || p->xDel==0 ); + + /* MEM_Dyn may only be set if Mem.szMalloc==0. In this way we + ** ensure that if Mem.szMalloc>0 then it is safe to do + ** Mem.z = Mem.zMalloc without having to check Mem.flags&MEM_Dyn. + ** That saves a few cycles in inner loops. */ + assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); + + /* Cannot be both MEM_Int and MEM_Real at the same time */ + assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); + + /* The szMalloc field holds the correct memory allocation size */ + assert( p->szMalloc==0 + || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc) ); /* If p holds a string or blob, the Mem.z must point to exactly ** one of the following: ** ** (1) Memory in Mem.zMalloc and managed by the Mem object ** (2) Memory to be freed using Mem.xDel - ** (3) An ephermal string or blob + ** (3) An ephemeral string or blob ** (4) A static string or blob */ - if( (p->flags & (MEM_Str|MEM_Blob)) && p->z!=0 ){ + if( (p->flags & (MEM_Str|MEM_Blob)) && p->n>0 ){ assert( - ((p->z==p->zMalloc)? 1 : 0) + + ((p->szMalloc>0 && p->z==p->zMalloc)? 1 : 0) + ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + ((p->flags&MEM_Static)!=0 ? 1 : 0) == 1 ); } - return 1; } #endif @@ -63830,33 +71959,38 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ ** blob if bPreserve is true. If bPreserve is false, any prior content ** in pMem->z is discarded. */ -SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( sqlite3VdbeCheckMemInvariants(pMem) ); assert( (pMem->flags&MEM_RowSet)==0 ); + testcase( pMem->db==0 ); /* If the bPreserve flag is set to true, then the memory cell must already ** contain a valid string or blob value. */ assert( bPreserve==0 || pMem->flags&(MEM_Blob|MEM_Str) ); testcase( bPreserve && pMem->z==0 ); - if( pMem->zMalloc==0 || sqlite3DbMallocSize(pMem->db, pMem->zMalloc)szMalloc==0 + || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) ); + if( pMem->szMallocz==pMem->zMalloc ){ + if( bPreserve && pMem->szMalloc>0 && pMem->z==pMem->zMalloc ){ pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); bPreserve = 0; }else{ - sqlite3DbFree(pMem->db, pMem->zMalloc); + if( pMem->szMalloc>0 ) sqlite3DbFree(pMem->db, pMem->zMalloc); pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); } if( pMem->zMalloc==0 ){ - VdbeMemRelease(pMem); + sqlite3VdbeMemSetNull(pMem); pMem->z = 0; - pMem->flags = MEM_Null; - return SQLITE_NOMEM; + pMem->szMalloc = 0; + return SQLITE_NOMEM_BKPT; + }else{ + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); } } - if( pMem->z && bPreserve && pMem->z!=pMem->zMalloc ){ + if( bPreserve && pMem->z && pMem->z!=pMem->zMalloc ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( (pMem->flags&MEM_Dyn)!=0 ){ @@ -63866,35 +72000,58 @@ SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ pMem->z = pMem->zMalloc; pMem->flags &= ~(MEM_Dyn|MEM_Ephem|MEM_Static); - pMem->xDel = 0; return SQLITE_OK; } /* -** Make the given Mem object MEM_Dyn. In other words, make it so -** that any TEXT or BLOB content is stored in memory obtained from -** malloc(). In this way, we know that the memory is safe to be -** overwritten or altered. +** Change the pMem->zMalloc allocation to be at least szNew bytes. +** If pMem->zMalloc already meets or exceeds the requested size, this +** routine is a no-op. +** +** Any prior string or blob content in the pMem object may be discarded. +** The pMem->xDel destructor is called, if it exists. Though MEM_Str +** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, and MEM_Null +** values are preserved. +** +** Return SQLITE_OK on success or an error code (probably SQLITE_NOMEM) +** if unable to complete the resizing. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ + assert( szNew>0 ); + assert( (pMem->flags & MEM_Dyn)==0 || pMem->szMalloc==0 ); + if( pMem->szMallocflags & MEM_Dyn)==0 ); + pMem->z = pMem->zMalloc; + pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); + return SQLITE_OK; +} + +/* +** Change pMem so that its MEM_Str or MEM_Blob value is stored in +** MEM.zMalloc, where it can be safely written. ** ** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails. */ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ - int f; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( (pMem->flags&MEM_RowSet)==0 ); - ExpandBlob(pMem); - f = pMem->flags; - if( (f&(MEM_Str|MEM_Blob)) && pMem->z!=pMem->zMalloc ){ - if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ - return SQLITE_NOMEM; + if( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ){ + if( ExpandBlob(pMem) ) return SQLITE_NOMEM; + if( pMem->szMalloc==0 || pMem->z!=pMem->zMalloc ){ + if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ + return SQLITE_NOMEM_BKPT; + } + pMem->z[pMem->n] = 0; + pMem->z[pMem->n+1] = 0; + pMem->flags |= MEM_Term; } - pMem->z[pMem->n] = 0; - pMem->z[pMem->n+1] = 0; - pMem->flags |= MEM_Term; -#ifdef SQLITE_DEBUG - pMem->pScopyFrom = 0; -#endif } + pMem->flags &= ~MEM_Ephem; +#ifdef SQLITE_DEBUG + pMem->pScopyFrom = 0; +#endif return SQLITE_OK; } @@ -63905,40 +72062,35 @@ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ */ #ifndef SQLITE_OMIT_INCRBLOB SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ - if( pMem->flags & MEM_Zero ){ - int nByte; - assert( pMem->flags&MEM_Blob ); - assert( (pMem->flags&MEM_RowSet)==0 ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + int nByte; + assert( pMem->flags & MEM_Zero ); + assert( pMem->flags&MEM_Blob ); + assert( (pMem->flags&MEM_RowSet)==0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - /* Set nByte to the number of bytes required to store the expanded blob. */ - nByte = pMem->n + pMem->u.nZero; - if( nByte<=0 ){ - nByte = 1; - } - if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ - return SQLITE_NOMEM; - } - - memset(&pMem->z[pMem->n], 0, pMem->u.nZero); - pMem->n += pMem->u.nZero; - pMem->flags &= ~(MEM_Zero|MEM_Term); + /* Set nByte to the number of bytes required to store the expanded blob. */ + nByte = pMem->n + pMem->u.nZero; + if( nByte<=0 ){ + nByte = 1; } + if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ + return SQLITE_NOMEM_BKPT; + } + + memset(&pMem->z[pMem->n], 0, pMem->u.nZero); + pMem->n += pMem->u.nZero; + pMem->flags &= ~(MEM_Zero|MEM_Term); return SQLITE_OK; } #endif - /* -** Make sure the given Mem is \u0000 terminated. +** It is already known that pMem contains an unterminated string. +** Add the zero terminator. */ -SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - if( (pMem->flags & MEM_Term)!=0 || (pMem->flags & MEM_Str)==0 ){ - return SQLITE_OK; /* Nothing to do */ - } +static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; @@ -63946,21 +72098,35 @@ SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ return SQLITE_OK; } +/* +** Make sure the given Mem is \u0000 terminated. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + testcase( (pMem->flags & (MEM_Term|MEM_Str))==(MEM_Term|MEM_Str) ); + testcase( (pMem->flags & (MEM_Term|MEM_Str))==0 ); + if( (pMem->flags & (MEM_Term|MEM_Str))!=MEM_Str ){ + return SQLITE_OK; /* Nothing to do */ + }else{ + return vdbeMemAddTerminator(pMem); + } +} + /* ** Add MEM_Str to the set of representations for the given Mem. Numbers ** are converted using sqlite3_snprintf(). Converting a BLOB to a string ** is a no-op. ** -** Existing representations MEM_Int and MEM_Real are *not* invalidated. +** Existing representations MEM_Int and MEM_Real are invalidated if +** bForce is true but are retained if bForce is false. ** ** A MEM_Null value will never be passed to this function. This function is ** used for converting values to text for returning to the user (i.e. via ** sqlite3_value_text()), or for ensuring that values to be used as btree ** keys are strings. In the former case a NULL pointer is returned the -** user and the later is an internal programming error. +** user and the latter is an internal programming error. */ -SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, int enc){ - int rc = SQLITE_OK; +SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ int fg = pMem->flags; const int nByte = 32; @@ -63972,11 +72138,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, int enc){ assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - if( sqlite3VdbeMemGrow(pMem, nByte, 0) ){ - return SQLITE_NOMEM; + if( sqlite3VdbeMemClearAndResize(pMem, nByte) ){ + pMem->enc = 0; + return SQLITE_NOMEM_BKPT; } - /* For a Real or Integer, use sqlite3_mprintf() to produce the UTF-8 + /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 ** string representation of the value. Then, if the required encoding ** is UTF-16le or UTF-16be do a translation. ** @@ -63986,13 +72153,14 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, int enc){ sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); }else{ assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->r); + sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); } pMem->n = sqlite3Strlen30(pMem->z); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); sqlite3VdbeChangeEncoding(pMem, enc); - return rc; + return SQLITE_OK; } /* @@ -64007,59 +72175,90 @@ SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ int rc = SQLITE_OK; if( ALWAYS(pFunc && pFunc->xFinalize) ){ sqlite3_context ctx; + Mem t; assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); memset(&ctx, 0, sizeof(ctx)); - ctx.s.flags = MEM_Null; - ctx.s.db = pMem->db; + memset(&t, 0, sizeof(t)); + t.flags = MEM_Null; + t.db = pMem->db; + ctx.pOut = &t; ctx.pMem = pMem; ctx.pFunc = pFunc; pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ - assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); - sqlite3DbFree(pMem->db, pMem->zMalloc); - memcpy(pMem, &ctx.s, sizeof(ctx.s)); + assert( (pMem->flags & MEM_Dyn)==0 ); + if( pMem->szMalloc>0 ) sqlite3DbFree(pMem->db, pMem->zMalloc); + memcpy(pMem, &t, sizeof(t)); rc = ctx.isError; } return rc; } /* -** If the memory cell contains a string value that must be freed by -** invoking an external callback, free it now. Calling this function -** does not free any Mem.zMalloc buffer. +** If the memory cell contains a value that must be freed by +** invoking the external callback in Mem.xDel, then this routine +** will free that value. It also sets Mem.flags to MEM_Null. +** +** This is a helper routine for sqlite3VdbeMemSetNull() and +** for sqlite3VdbeMemRelease(). Use those other routines as the +** entry point for releasing Mem resources. */ -SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p){ +static SQLITE_NOINLINE void vdbeMemClearExternAndSetNull(Mem *p){ assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); + assert( VdbeMemDynamic(p) ); if( p->flags&MEM_Agg ){ sqlite3VdbeMemFinalize(p, p->u.pDef); assert( (p->flags & MEM_Agg)==0 ); - sqlite3VdbeMemRelease(p); - }else if( p->flags&MEM_Dyn ){ + testcase( p->flags & MEM_Dyn ); + } + if( p->flags&MEM_Dyn ){ assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); p->xDel((void *)p->z); - p->xDel = 0; }else if( p->flags&MEM_RowSet ){ sqlite3RowSetClear(p->u.pRowSet); }else if( p->flags&MEM_Frame ){ - sqlite3VdbeMemSetNull(p); + VdbeFrame *pFrame = p->u.pFrame; + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; } + p->flags = MEM_Null; } /* -** Release any memory held by the Mem. This may leave the Mem in an -** inconsistent state, for example with (Mem.z==0) and -** (Mem.flags==MEM_Str). +** Release memory held by the Mem p, both external memory cleared +** by p->xDel and memory in p->zMalloc. +** +** This is a helper routine invoked by sqlite3VdbeMemRelease() in +** the unusual case where there really is memory in p that needs +** to be freed. +*/ +static SQLITE_NOINLINE void vdbeMemClear(Mem *p){ + if( VdbeMemDynamic(p) ){ + vdbeMemClearExternAndSetNull(p); + } + if( p->szMalloc ){ + sqlite3DbFree(p->db, p->zMalloc); + p->szMalloc = 0; + } + p->z = 0; +} + +/* +** Release any memory resources held by the Mem. Both the memory that is +** free by Mem.xDel and the Mem.zMalloc allocation are freed. +** +** Use this routine prior to clean up prior to abandoning a Mem, or to +** reset a Mem back to its minimum memory utilization. +** +** Use sqlite3VdbeMemSetNull() to release just the Mem.xDel space +** prior to inserting new content into the Mem. */ SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p){ assert( sqlite3VdbeCheckMemInvariants(p) ); - VdbeMemRelease(p); - if( p->zMalloc ){ - sqlite3DbFree(p->db, p->zMalloc); - p->zMalloc = 0; + if( VdbeMemDynamic(p) || p->szMalloc ){ + vdbeMemClear(p); } - p->z = 0; - assert( p->xDel==0 ); /* Zeroed by VdbeMemRelease() above */ } /* @@ -64098,7 +72297,7 @@ static i64 doubleToInt64(double r){ ** If pMem is an integer, then the value is exact. If pMem is ** a floating-point then the value returned is the integer part. ** If pMem is a string or blob, then we make an attempt to convert -** it into a integer and return that. If pMem represents an +** it into an integer and return that. If pMem represents an ** an SQL-NULL value, return 0. ** ** If pMem represents a string value, its encoding might be changed. @@ -64111,11 +72310,10 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ if( flags & MEM_Int ){ return pMem->u.i; }else if( flags & MEM_Real ){ - return doubleToInt64(pMem->r); + return doubleToInt64(pMem->u.r); }else if( flags & (MEM_Str|MEM_Blob) ){ i64 value = 0; assert( pMem->z || pMem->n==0 ); - testcase( pMem->z==0 ); sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); return value; }else{ @@ -64133,7 +72331,7 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); if( pMem->flags & MEM_Real ){ - return pMem->r; + return pMem->u.r; }else if( pMem->flags & MEM_Int ){ return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ @@ -64152,12 +72350,13 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ ** MEM_Int if we can. */ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ + i64 ix; assert( pMem->flags & MEM_Real ); assert( (pMem->flags & MEM_RowSet)==0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - pMem->u.i = doubleToInt64(pMem->r); + ix = doubleToInt64(pMem->u.r); /* Only mark the value as an integer if ** @@ -64169,11 +72368,9 @@ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ ** the second condition under the assumption that addition overflow causes ** values to wrap around. */ - if( pMem->r==(double)pMem->u.i - && pMem->u.i>SMALLEST_INT64 - && pMem->u.iflags |= MEM_Int; + if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; + MemSetTypeFlag(pMem, MEM_Int); } } @@ -64198,7 +72395,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - pMem->r = sqlite3VdbeRealValue(pMem); + pMem->u.r = sqlite3VdbeRealValue(pMem); MemSetTypeFlag(pMem, MEM_Real); return SQLITE_OK; } @@ -64218,29 +72415,91 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ MemSetTypeFlag(pMem, MEM_Int); }else{ - pMem->r = sqlite3VdbeRealValue(pMem); + pMem->u.r = sqlite3VdbeRealValue(pMem); MemSetTypeFlag(pMem, MEM_Real); sqlite3VdbeIntegerAffinity(pMem); } } assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); - pMem->flags &= ~(MEM_Str|MEM_Blob); + pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero); return SQLITE_OK; } +/* +** Cast the datatype of the value in pMem according to the affinity +** "aff". Casting is different from applying affinity in that a cast +** is forced. In other words, the value is converted into the desired +** affinity even if that results in loss of data. This routine is +** used (for example) to implement the SQL "cast()" operator. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ + if( pMem->flags & MEM_Null ) return; + switch( aff ){ + case SQLITE_AFF_BLOB: { /* Really a cast to BLOB */ + if( (pMem->flags & MEM_Blob)==0 ){ + sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); + assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); + if( pMem->flags & MEM_Str ) MemSetTypeFlag(pMem, MEM_Blob); + }else{ + pMem->flags &= ~(MEM_TypeMask&~MEM_Blob); + } + break; + } + case SQLITE_AFF_NUMERIC: { + sqlite3VdbeMemNumerify(pMem); + break; + } + case SQLITE_AFF_INTEGER: { + sqlite3VdbeMemIntegerify(pMem); + break; + } + case SQLITE_AFF_REAL: { + sqlite3VdbeMemRealify(pMem); + break; + } + default: { + assert( aff==SQLITE_AFF_TEXT ); + assert( MEM_Str==(MEM_Blob>>3) ); + pMem->flags |= (pMem->flags&MEM_Blob)>>3; + sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); + assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + break; + } + } +} + +/* +** Initialize bulk memory to be a consistent Mem object. +** +** The minimum amount of initialization feasible is performed. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemInit(Mem *pMem, sqlite3 *db, u16 flags){ + assert( (flags & ~MEM_TypeMask)==0 ); + pMem->flags = flags; + pMem->db = db; + pMem->szMalloc = 0; +} + + /* ** Delete any previous value and set the value stored in *pMem to NULL. +** +** This routine calls the Mem.xDel destructor to dispose of values that +** require the destructor. But it preserves the Mem.zMalloc memory allocation. +** To free all resources, use sqlite3VdbeMemRelease(), which both calls this +** routine to invoke the destructor and deallocates Mem.zMalloc. +** +** Use this routine to reset the Mem prior to insert a new value. +** +** Use sqlite3VdbeMemRelease() to complete erase the Mem prior to abandoning it. */ SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){ - if( pMem->flags & MEM_Frame ){ - VdbeFrame *pFrame = pMem->u.pFrame; - pFrame->pParent = pFrame->v->pDelFrame; - pFrame->v->pDelFrame = pFrame; + if( VdbeMemDynamic(pMem) ){ + vdbeMemClearExternAndSetNull(pMem); + }else{ + pMem->flags = MEM_Null; } - if( pMem->flags & MEM_RowSet ){ - sqlite3RowSetClear(pMem->u.pRowSet); - } - MemSetTypeFlag(pMem, MEM_Null); } SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value *p){ sqlite3VdbeMemSetNull((Mem*)p); @@ -64257,14 +72516,18 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ if( n<0 ) n = 0; pMem->u.nZero = n; pMem->enc = SQLITE_UTF8; + pMem->z = 0; +} -#ifdef SQLITE_OMIT_INCRBLOB - sqlite3VdbeMemGrow(pMem, n, 0); - if( pMem->z ){ - pMem->n = n; - memset(pMem->z, 0, n); - } -#endif +/* +** The pMem is known to contain content that needs to be destroyed prior +** to a value change. So invoke the destructor, then set the value to +** a 64-bit integer. +*/ +static SQLITE_NOINLINE void vdbeReleaseAndSetInt64(Mem *pMem, i64 val){ + sqlite3VdbeMemSetNull(pMem); + pMem->u.i = val; + pMem->flags = MEM_Int; } /* @@ -64272,9 +72535,12 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ ** manifest type INTEGER. */ SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ - sqlite3VdbeMemRelease(pMem); - pMem->u.i = val; - pMem->flags = MEM_Int; + if( VdbeMemDynamic(pMem) ){ + vdbeReleaseAndSetInt64(pMem, val); + }else{ + pMem->u.i = val; + pMem->flags = MEM_Int; + } } #ifndef SQLITE_OMIT_FLOATING_POINT @@ -64283,11 +72549,9 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ ** manifest type REAL. */ SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ - if( sqlite3IsNaN(val) ){ - sqlite3VdbeMemSetNull(pMem); - }else{ - sqlite3VdbeMemRelease(pMem); - pMem->r = val; + sqlite3VdbeMemSetNull(pMem); + if( !sqlite3IsNaN(val) ){ + pMem->u.r = val; pMem->flags = MEM_Real; } } @@ -64302,13 +72566,14 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem *pMem){ assert( db!=0 ); assert( (pMem->flags & MEM_RowSet)==0 ); sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = sqlite3DbMallocRaw(db, 64); + pMem->zMalloc = sqlite3DbMallocRawNN(db, 64); if( db->mallocFailed ){ pMem->flags = MEM_Null; + pMem->szMalloc = 0; }else{ assert( pMem->zMalloc ); - pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, - sqlite3DbMallocSize(db, pMem->zMalloc)); + pMem->szMalloc = sqlite3DbMallocSize(db, pMem->zMalloc); + pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, pMem->szMalloc); assert( pMem->u.pRowSet!=0 ); pMem->flags = MEM_RowSet; } @@ -64332,7 +72597,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ #ifdef SQLITE_DEBUG /* -** This routine prepares a memory cell for modication by breaking +** This routine prepares a memory cell for modification by breaking ** its link to a shallow copy and by marking any current shallow ** copies of this cell as invalid. ** @@ -64342,7 +72607,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ int i; Mem *pX; - for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ + for(i=0, pX=pVdbe->aMem; inMem; i++, pX++){ if( pX->pScopyFrom==pMem ){ pX->flags |= MEM_Undefined; pX->pScopyFrom = 0; @@ -64352,10 +72617,6 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ } #endif /* SQLITE_DEBUG */ -/* -** Size of struct Mem not including the Mem.zMalloc member. -*/ -#define MEMCELLSIZE offsetof(Mem,zMalloc) /* ** Make an shallow copy of pFrom into pTo. Prior contents of @@ -64363,11 +72624,16 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ ** pFrom->z is used, then pTo->z points to the same thing as pFrom->z ** and flags gets srcType (either MEM_Ephem or MEM_Static). */ +static SQLITE_NOINLINE void vdbeClrCopy(Mem *pTo, const Mem *pFrom, int eType){ + vdbeMemClearExternAndSetNull(pTo); + assert( !VdbeMemDynamic(pTo) ); + sqlite3VdbeMemShallowCopy(pTo, pFrom, eType); +} SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); - VdbeMemRelease(pTo); + assert( pTo->db==pFrom->db ); + if( VdbeMemDynamic(pTo) ){ vdbeClrCopy(pTo,pFrom,srcType); return; } memcpy(pTo, pFrom, MEMCELLSIZE); - pTo->xDel = 0; if( (pFrom->flags&MEM_Static)==0 ){ pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); assert( srcType==MEM_Ephem || srcType==MEM_Static ); @@ -64383,11 +72649,9 @@ SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; assert( (pFrom->flags & MEM_RowSet)==0 ); - VdbeMemRelease(pTo); + if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; - pTo->xDel = 0; - if( pTo->flags&(MEM_Str|MEM_Blob) ){ if( 0==(pFrom->flags&MEM_Static) ){ pTo->flags |= MEM_Ephem; @@ -64412,8 +72676,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ sqlite3VdbeMemRelease(pTo); memcpy(pTo, pFrom, sizeof(Mem)); pFrom->flags = MEM_Null; - pFrom->xDel = 0; - pFrom->zMalloc = 0; + pFrom->szMalloc = 0; } /* @@ -64460,7 +72723,8 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( if( nByte<0 ){ assert( enc!=0 ); if( enc==SQLITE_UTF8 ){ - for(nByte=0; nByte<=iLimit && z[nByte]; nByte++){} + nByte = sqlite3Strlen30(z); + if( nByte>iLimit ) nByte = iLimit+1; }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } @@ -64479,14 +72743,17 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( if( nByte>iLimit ){ return SQLITE_TOOBIG; } - if( sqlite3VdbeMemGrow(pMem, nAlloc, 0) ){ - return SQLITE_NOMEM; + testcase( nAlloc==0 ); + testcase( nAlloc==31 ); + testcase( nAlloc==32 ); + if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){ + return SQLITE_NOMEM_BKPT; } memcpy(pMem->z, z, nAlloc); }else if( xDel==SQLITE_DYNAMIC ){ sqlite3VdbeMemRelease(pMem); pMem->zMalloc = pMem->z = (char *)z; - pMem->xDel = 0; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); }else{ sqlite3VdbeMemRelease(pMem); pMem->z = (char *)z; @@ -64500,7 +72767,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( #ifndef SQLITE_OMIT_UTF16 if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } #endif @@ -64518,41 +72785,25 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ** key is true to get the key or false to get data. The result is written ** into the pMem element. ** -** The pMem structure is assumed to be uninitialized. Any prior content -** is overwritten without being freed. +** The pMem object must have been initialized. This routine will use +** pMem->zMalloc to hold the content from the btree, if possible. New +** pMem->zMalloc space will be allocated if necessary. The calling routine +** is responsible for making sure that the pMem object is eventually +** destroyed. ** ** If this routine fails for any reason (malloc returns NULL or unable ** to read from the disk) then the pMem is left in an inconsistent state. */ -SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( +static SQLITE_NOINLINE int vdbeMemFromBtreeResize( BtCursor *pCur, /* Cursor pointing at record to retrieve. */ u32 offset, /* Offset from the start of data to return bytes from. */ u32 amt, /* Number of bytes to return. */ int key, /* If true, retrieve from the btree key, not data. */ Mem *pMem /* OUT: Return data in this Mem structure. */ ){ - char *zData; /* Data from the btree layer */ - u32 available = 0; /* Number of bytes available on the local btree page */ - int rc = SQLITE_OK; /* Return code */ - - assert( sqlite3BtreeCursorIsValid(pCur) ); - - /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() - ** that both the BtShared and database handle mutexes are held. */ - assert( (pMem->flags & MEM_RowSet)==0 ); - if( key ){ - zData = (char *)sqlite3BtreeKeyFetch(pCur, &available); - }else{ - zData = (char *)sqlite3BtreeDataFetch(pCur, &available); - } - assert( zData!=0 ); - - if( offset+amt<=available ){ - sqlite3VdbeMemRelease(pMem); - pMem->z = &zData[offset]; - pMem->flags = MEM_Blob|MEM_Ephem; - pMem->n = (int)amt; - }else if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ + int rc; + pMem->flags = MEM_Null; + if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ if( key ){ rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); }else{ @@ -64567,10 +72818,75 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( sqlite3VdbeMemRelease(pMem); } } + return rc; +} +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( + BtCursor *pCur, /* Cursor pointing at record to retrieve. */ + u32 offset, /* Offset from the start of data to return bytes from. */ + u32 amt, /* Number of bytes to return. */ + int key, /* If true, retrieve from the btree key, not data. */ + Mem *pMem /* OUT: Return data in this Mem structure. */ +){ + char *zData; /* Data from the btree layer */ + u32 available = 0; /* Number of bytes available on the local btree page */ + int rc = SQLITE_OK; /* Return code */ + + assert( sqlite3BtreeCursorIsValid(pCur) ); + assert( !VdbeMemDynamic(pMem) ); + + /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() + ** that both the BtShared and database handle mutexes are held. */ + assert( (pMem->flags & MEM_RowSet)==0 ); + zData = (char *)sqlite3BtreePayloadFetch(pCur, &available); + assert( zData!=0 ); + + if( offset+amt<=available ){ + pMem->z = &zData[offset]; + pMem->flags = MEM_Blob|MEM_Ephem; + pMem->n = (int)amt; + }else{ + rc = vdbeMemFromBtreeResize(pCur, offset, amt, key, pMem); + } return rc; } +/* +** The pVal argument is known to be a value other than NULL. +** Convert it into a string with encoding enc and return a pointer +** to a zero-terminated version of that string. +*/ +static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ + assert( pVal!=0 ); + assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); + assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); + assert( (pVal->flags & MEM_RowSet)==0 ); + assert( (pVal->flags & (MEM_Null))==0 ); + if( pVal->flags & (MEM_Blob|MEM_Str) ){ + pVal->flags |= MEM_Str; + if( pVal->enc != (enc & ~SQLITE_UTF16_ALIGNED) ){ + sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); + } + if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ + assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); + if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ + return 0; + } + } + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ + }else{ + sqlite3VdbeMemStringify(pVal, enc, 0); + assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); + } + assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 + || pVal->db->mallocFailed ); + if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ + return pVal->z; + }else{ + return 0; + } +} + /* This function is only available internally, it is not part of the ** external API. It works in a similar way to sqlite3_value_text(), ** except the data returned is in the encoding specified by the second @@ -64583,38 +72899,16 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( */ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ if( !pVal ) return 0; - assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); assert( (pVal->flags & MEM_RowSet)==0 ); - + if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ + return pVal->z; + } if( pVal->flags&MEM_Null ){ return 0; } - assert( (MEM_Blob>>3) == MEM_Str ); - pVal->flags |= (pVal->flags & MEM_Blob)>>3; - ExpandBlob(pVal); - if( pVal->flags&MEM_Str ){ - sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); - if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ - assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); - if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ - return 0; - } - } - sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ - }else{ - assert( (pVal->flags&MEM_Blob)==0 ); - sqlite3VdbeMemStringify(pVal, enc); - assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); - } - assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 - || pVal->db->mallocFailed ); - if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ - return pVal->z; - }else{ - return 0; - } + return valueToText(pVal, enc); } /* @@ -64648,7 +72942,7 @@ struct ValueNewStat4Ctx { ** Otherwise, if the second argument is non-zero, then this function is ** being called indirectly by sqlite3Stat4ProbeSetValue(). If it has not ** already been allocated, allocate the UnpackedRecord structure that -** that function will return to its caller here. Then return a pointer +** that function will return to its caller here. Then return a pointer to ** an sqlite3_value within the UnpackedRecord.a[] array. */ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ @@ -64692,6 +72986,111 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ return sqlite3ValueNew(db); } +/* +** The expression object indicated by the second argument is guaranteed +** to be a scalar SQL function. If +** +** * all function arguments are SQL literals, +** * one of the SQLITE_FUNC_CONSTANT or _SLOCHNG function flags is set, and +** * the SQLITE_FUNC_NEEDCOLL function flag is not set, +** +** then this routine attempts to invoke the SQL function. Assuming no +** error occurs, output parameter (*ppVal) is set to point to a value +** object containing the result before returning SQLITE_OK. +** +** Affinity aff is applied to the result of the function before returning. +** If the result is a text value, the sqlite3_value object uses encoding +** enc. +** +** If the conditions above are not met, this function returns SQLITE_OK +** and sets (*ppVal) to NULL. Or, if an error occurs, (*ppVal) is set to +** NULL and an SQLite error code returned. +*/ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +static int valueFromFunction( + sqlite3 *db, /* The database connection */ + Expr *p, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 aff, /* Affinity to use */ + sqlite3_value **ppVal, /* Write the new value here */ + struct ValueNewStat4Ctx *pCtx /* Second argument for valueNew() */ +){ + sqlite3_context ctx; /* Context object for function invocation */ + sqlite3_value **apVal = 0; /* Function arguments */ + int nVal = 0; /* Size of apVal[] array */ + FuncDef *pFunc = 0; /* Function definition */ + sqlite3_value *pVal = 0; /* New value */ + int rc = SQLITE_OK; /* Return code */ + ExprList *pList = 0; /* Function arguments */ + int i; /* Iterator variable */ + + assert( pCtx!=0 ); + assert( (p->flags & EP_TokenOnly)==0 ); + pList = p->x.pList; + if( pList ) nVal = pList->nExpr; + pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); + assert( pFunc ); + if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + ){ + return SQLITE_OK; + } + + if( pList ){ + apVal = (sqlite3_value**)sqlite3DbMallocZero(db, sizeof(apVal[0]) * nVal); + if( apVal==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto value_from_function_out; + } + for(i=0; ia[i].pExpr, enc, aff, &apVal[i]); + if( apVal[i]==0 || rc!=SQLITE_OK ) goto value_from_function_out; + } + } + + pVal = valueNew(db, pCtx); + if( pVal==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto value_from_function_out; + } + + assert( pCtx->pParse->rc==SQLITE_OK ); + memset(&ctx, 0, sizeof(ctx)); + ctx.pOut = pVal; + ctx.pFunc = pFunc; + pFunc->xSFunc(&ctx, nVal, apVal); + if( ctx.isError ){ + rc = ctx.isError; + sqlite3ErrorMsg(pCtx->pParse, "%s", sqlite3_value_text(pVal)); + }else{ + sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); + assert( rc==SQLITE_OK ); + rc = sqlite3VdbeChangeEncoding(pVal, enc); + if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){ + rc = SQLITE_TOOBIG; + pCtx->pParse->nErr++; + } + } + pCtx->pParse->rc = rc; + + value_from_function_out: + if( rc!=SQLITE_OK ){ + pVal = 0; + } + if( apVal ){ + for(i=0; iop; + assert( pExpr!=0 ); + while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; + /* Compressed expressions only appear when parsing the DEFAULT clause + ** on a table column definition, and hence only when pCtx==0. This + ** check ensures that an EP_TokenOnly expression is never passed down + ** into valueFromFunction(). */ + assert( (pExpr->flags & EP_TokenOnly)==0 || pCtx==0 ); + + if( op==TK_CAST ){ + u8 aff = sqlite3AffinityType(pExpr->u.zToken,0); + rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); + testcase( rc!=SQLITE_OK ); + if( *ppVal ){ + sqlite3VdbeMemCast(*ppVal, aff, SQLITE_UTF8); + sqlite3ValueApplyAffinity(*ppVal, affinity, SQLITE_UTF8); + } + return rc; + } + /* Handle negative integers in a single step. This is needed in the ** case when the value is -9223372036854775808. */ @@ -64745,7 +73158,7 @@ static int valueFromExpr( if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ + if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); @@ -64760,14 +73173,14 @@ static int valueFromExpr( && pVal!=0 ){ sqlite3VdbeMemNumerify(pVal); - if( pVal->u.i==SMALLEST_INT64 ){ - pVal->flags &= ~MEM_Int; - pVal->flags |= MEM_Real; - pVal->r = (double)SMALLEST_INT64; + if( pVal->flags & MEM_Real ){ + pVal->u.r = -pVal->u.r; + }else if( pVal->u.i==SMALLEST_INT64 ){ + pVal->u.r = -(double)SMALLEST_INT64; + MemSetTypeFlag(pVal, MEM_Real); }else{ pVal->u.i = -pVal->u.i; } - pVal->r = -pVal->r; sqlite3ValueApplyAffinity(pVal, affinity, enc); } }else if( op==TK_NULL ){ @@ -64789,11 +73202,17 @@ static int valueFromExpr( } #endif +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + else if( op==TK_FUNCTION && pCtx!=0 ){ + rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx); + } +#endif + *ppVal = pVal; return rc; no_mem: - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, zVal); assert( *ppVal==0 ); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -64801,7 +73220,7 @@ no_mem: #else assert( pCtx==0 ); sqlite3ValueFree(pVal); #endif - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* @@ -64821,7 +73240,7 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( u8 affinity, /* Affinity to use */ sqlite3_value **ppVal /* Write the new value here */ ){ - return valueFromExpr(db, pExpr, enc, affinity, ppVal, 0); + return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0; } #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -64839,26 +73258,25 @@ static void recordFunc( sqlite3_value **argv ){ const int file_format = 1; - int iSerial; /* Serial type */ + u32 iSerial; /* Serial type */ int nSerial; /* Bytes of space for iSerial as varint */ - int nVal; /* Bytes of space required for argv[0] */ + u32 nVal; /* Bytes of space required for argv[0] */ int nRet; sqlite3 *db; u8 *aRet; UNUSED_PARAMETER( argc ); - iSerial = sqlite3VdbeSerialType(argv[0], file_format); + iSerial = sqlite3VdbeSerialType(argv[0], file_format, &nVal); nSerial = sqlite3VarintLen(iSerial); - nVal = sqlite3VdbeSerialTypeLen(iSerial); db = sqlite3_context_db_handle(context); nRet = 1 + nSerial + nVal; - aRet = sqlite3DbMallocRaw(db, nRet); + aRet = sqlite3DbMallocRawNN(db, nRet); if( aRet==0 ){ sqlite3_result_error_nomem(context); }else{ aRet[0] = nSerial+1; - sqlite3PutVarint(&aRet[1], iSerial); + putVarint32(&aRet[1], iSerial); sqlite3VdbeSerialPut(&aRet[1+nSerial], argv[0], iSerial); sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); sqlite3DbFree(db, aRet); @@ -64869,15 +73287,10 @@ static void recordFunc( ** Register built-in functions used to help read ANALYZE data. */ SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void){ - static SQLITE_WSD FuncDef aAnalyzeTableFuncs[] = { + static FuncDef aAnalyzeTableFuncs[] = { FUNCTION(sqlite_record, 1, 0, 0, recordFunc), }; - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAnalyzeTableFuncs); - for(i=0; iop!=TK_SELECT ){ + int i; + struct ValueNewStat4Ctx alloc; - rc = stat4ValueFromExpr(pParse, pExpr, affinity, &alloc, &pVal); - assert( pVal==0 || pVal->db==pParse->db ); - *pbOk = (pVal!=0); + alloc.pParse = pParse; + alloc.pIdx = pIdx; + alloc.ppRec = ppRec; + + for(i=0; idb, pIdx, iVal+i); + alloc.iVal = iVal+i; + rc = stat4ValueFromExpr(pParse, pElem, aff, &alloc, &pVal); + if( !pVal ) break; + nExtract++; + } + } + + *pnExtract = nExtract; return rc; } @@ -65056,7 +73484,7 @@ SQLITE_PRIVATE int sqlite3Stat4Column( if( iField>nRec ) return SQLITE_CORRUPT_BKPT; if( pMem==0 ){ pMem = *ppVal = sqlite3ValueNew(db); - if( pMem==0 ) return SQLITE_NOMEM; + if( pMem==0 ) return SQLITE_NOMEM_BKPT; } sqlite3VdbeSerialGet(&a[iField-szField], t, pMem); pMem->enc = ENC(db); @@ -65075,7 +73503,7 @@ SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ Mem *aMem = pRec->aMem; sqlite3 *db = aMem[0].db; for(i=0; ipKeyInfo); sqlite3DbFree(db, pRec); @@ -65106,19 +73534,28 @@ SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value *v){ } /* -** Return the number of bytes in the sqlite3_value object assuming -** that it uses the encoding "enc" +** The sqlite3ValueBytes() routine returns the number of bytes in the +** sqlite3_value object assuming that it uses the encoding "enc". +** The valueBytes() routine is a helper function. */ +static SQLITE_NOINLINE int valueBytes(sqlite3_value *pVal, u8 enc){ + return valueToText(pVal, enc)!=0 ? pVal->n : 0; +} SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ Mem *p = (Mem*)pVal; - if( (p->flags & MEM_Blob)!=0 || sqlite3ValueText(pVal, enc) ){ + assert( (p->flags & MEM_Null)==0 || (p->flags & (MEM_Str|MEM_Blob))==0 ); + if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ + return p->n; + } + if( (p->flags & MEM_Blob)!=0 ){ if( p->flags & MEM_Zero ){ return p->n + p->u.nZero; }else{ return p->n; } } - return 0; + if( p->flags & MEM_Null ) return 0; + return valueBytes(pVal, enc); } /************** End of vdbemem.c *********************************************/ @@ -65135,10 +73572,10 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ ** ************************************************************************* ** This file contains code used for creating, destroying, and populating -** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior -** to version 2.8.7, all this code was combined into the vdbe.c source file. -** But that file was getting too big so this subroutines were split out. +** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ /* ** Create a new virtual database engine. @@ -65146,8 +73583,9 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ sqlite3 *db = pParse->db; Vdbe *p; - p = sqlite3DbMallocZero(db, sizeof(Vdbe) ); + p = sqlite3DbMallocRawNN(db, sizeof(Vdbe) ); if( p==0 ) return 0; + memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp)); p->db = db; if( db->pVdbe ){ db->pVdbe->pPrev = p; @@ -65160,9 +73598,21 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ assert( pParse->aLabel==0 ); assert( pParse->nLabel==0 ); assert( pParse->nOpAlloc==0 ); + assert( pParse->szOpAlloc==0 ); return p; } +/* +** Change the error string stored in Vdbe.zErrMsg +*/ +SQLITE_PRIVATE void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){ + va_list ap; + sqlite3DbFree(p->db, p->zErrMsg); + va_start(ap, zFormat); + p->zErrMsg = sqlite3VMPrintf(p->db, zFormat, ap); + va_end(ap); +} + /* ** Remember the SQL string for a prepared statement. */ @@ -65177,20 +73627,13 @@ SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepa p->isPrepareV2 = (u8)isPrepareV2; } -/* -** Return the SQL associated with a prepared statement -*/ -SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt){ - Vdbe *p = (Vdbe *)pStmt; - return (p && p->isPrepareV2) ? p->zSql : 0; -} - /* ** Swap all content between two VDBE structures. */ SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ Vdbe tmp, *pTmp; char *zTmp; + assert( pA->db==pB->db ); tmp = *pA; *pA = *pB; *pB = tmp; @@ -65238,10 +73681,11 @@ static int growOpArray(Vdbe *v, int nOp){ assert( nNew>=(p->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ - p->nOpAlloc = sqlite3DbMallocSize(p->db, pNew)/sizeof(Op); + p->szOpAlloc = sqlite3DbMallocSize(p->db, pNew); + p->nOpAlloc = p->szOpAlloc/sizeof(Op); v->aOp = pNew; } - return (pNew ? SQLITE_OK : SQLITE_NOMEM); + return (pNew ? SQLITE_OK : SQLITE_NOMEM_BKPT); } #ifdef SQLITE_DEBUG @@ -65271,17 +73715,21 @@ static void test_addop_breakpoint(void){ ** the sqlite3VdbeChangeP4() function to change the value of the P4 ** operand. */ +static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ + assert( p->pParse->nOpAlloc<=p->nOp ); + if( growOpArray(p, 1) ) return 1; + assert( p->pParse->nOpAlloc>p->nOp ); + return sqlite3VdbeAddOp3(p, op, p1, p2, p3); +} SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ int i; VdbeOp *pOp; i = p->nOp; assert( p->magic==VDBE_MAGIC_INIT ); - assert( op>0 && op<0xff ); + assert( op>=0 && op<0xff ); if( p->pParse->nOpAlloc<=i ){ - if( growOpArray(p, 1) ){ - return 1; - } + return growOp3(p, op, p1, p2, p3); } p->nOp++; pOp = &p->aOp[i]; @@ -65299,9 +73747,8 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ if( p->db->flags & SQLITE_VdbeAddopTrace ){ int jj, kk; Parse *pParse = p->pParse; - for(jj=kk=0; jjnColCache; jj++){ struct yColCache *x = pParse->aColCache + jj; - if( x->iLevel>pParse->iCacheLevel || x->iReg==0 ) continue; printf(" r[%d]={%d:%d}", x->iReg, x->iTable, x->iColumn); kk++; } @@ -65329,6 +73776,43 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ return sqlite3VdbeAddOp3(p, op, p1, p2, 0); } +/* Generate code for an unconditional jump to instruction iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe *p, int iDest){ + return sqlite3VdbeAddOp3(p, OP_Goto, 0, iDest, 0); +} + +/* Generate code to cause the string zStr to be loaded into +** register iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe *p, int iDest, const char *zStr){ + return sqlite3VdbeAddOp4(p, OP_String8, 0, iDest, 0, zStr, 0); +} + +/* +** Generate code that initializes multiple registers to string or integer +** constants. The registers begin with iDest and increase consecutively. +** One register is initialized for each characgter in zTypes[]. For each +** "s" character in zTypes[], the register is a string if the argument is +** not NULL, or OP_Null if the value is a null pointer. For each "i" character +** in zTypes[], the register is initialized to an integer. +*/ +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe *p, int iDest, const char *zTypes, ...){ + va_list ap; + int i; + char c; + va_start(ap, zTypes); + for(i=0; (c = zTypes[i])!=0; i++){ + if( c=='s' ){ + const char *z = va_arg(ap, const char*); + sqlite3VdbeAddOp4(p, z==0 ? OP_Null : OP_String8, 0, iDest++, 0, z, 0); + }else{ + assert( c=='i' ); + sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest++); + } + } + va_end(ap); +} /* ** Add an opcode that includes the p4 value as a pointer. @@ -65347,6 +73831,24 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4( return addr; } +/* +** Add an opcode that includes the p4 value with a P4_INT64 or +** P4_REAL type. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + const u8 *zP4, /* The P4 operand */ + int p4type /* P4 operand type */ +){ + char *p4copy = sqlite3DbMallocRawNN(sqlite3VdbeDb(p), 8); + if( p4copy ) memcpy(p4copy, zP4, 8); + return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type); +} + /* ** Add an OP_ParseSchema opcode. This routine is broken out from ** sqlite3VdbeAddOp4() since it needs to also needs to mark all btrees @@ -65357,8 +73859,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4( */ SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ int j; - int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); - sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); } @@ -65378,6 +73879,21 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( return addr; } +/* Insert the end of a co-routine +*/ +SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){ + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + + /* Clear the temporary register cache, thereby ensuring that each + ** co-routine has its own independent set of registers, because co-routines + ** might expect their registers to be preserved across an OP_Yield, and + ** that could cause problems if two or more co-routines are using the same + ** temporary register. + */ + v->pParse->nTempReg = 0; + v->pParse->nRangeReg = 0; +} + /* ** Create a new symbolic label for an instruction that has yet to be ** coded. The symbolic label is really just a negative number. The @@ -65403,7 +73919,7 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ if( p->aLabel ){ p->aLabel[i] = -1; } - return -1-i; + return ADDR(i); } /* @@ -65413,13 +73929,13 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ */ SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ Parse *p = v->pParse; - int j = -1-x; + int j = ADDR(x); assert( v->magic==VDBE_MAGIC_INIT ); assert( jnLabel ); - if( ALWAYS(j>=0) && p->aLabel ){ + assert( j>=0 ); + if( p->aLabel ){ p->aLabel[j] = v->nOp; } - p->iFixedOp = v->nOp - 1; } /* @@ -65429,6 +73945,13 @@ SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe *p){ p->runOnlyOnce = 1; } +/* +** Mark the VDBE as one that can only be run multiple times. +*/ +SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe *p){ + p->runOnlyOnce = 0; +} + #ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ /* @@ -65511,6 +74034,7 @@ static Op *opIterNext(VdbeOpIter *p){ ** * OP_VUpdate ** * OP_VRename ** * OP_FkCounter with P2==0 (immediate foreign key constraint) +** * OP_CreateTable and OP_InitCoroutine (for CREATE TABLE AS SELECT ...) ** ** Then check that the value of Parse.mayAbort is true if an ** ABORT may be thrown, or false otherwise. Return true if it does @@ -65521,6 +74045,9 @@ static Op *opIterNext(VdbeOpIter *p){ */ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasAbort = 0; + int hasFkCounter = 0; + int hasCreateTable = 0; + int hasInitCoroutine = 0; Op *pOp; VdbeOpIter sIter; memset(&sIter, 0, sizeof(sIter)); @@ -65529,15 +74056,19 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ while( (pOp = opIterNext(&sIter))!=0 ){ int opcode = pOp->opcode; if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename -#ifndef SQLITE_OMIT_FOREIGN_KEY - || (opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1) -#endif || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; } + if( opcode==OP_CreateTable ) hasCreateTable = 1; + if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ + hasFkCounter = 1; + } +#endif } sqlite3DbFree(v->db, sIter.apSub); @@ -65546,94 +74077,105 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** through all opcodes and hasAbort may be set incorrectly. Return ** true for this case to prevent the assert() in the callers frame ** from failing. */ - return ( v->db->mallocFailed || hasAbort==mayAbort ); + return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter + || (hasCreateTable && hasInitCoroutine) ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ /* -** Loop through the program looking for P2 values that are negative -** on jump instructions. Each such value is a label. Resolve the -** label by setting the P2 value to its correct non-zero value. +** This routine is called after all opcodes have been inserted. It loops +** through all the opcodes and fixes up some details. ** -** This routine is called once after all opcodes have been inserted. +** (1) For each jump instruction with a negative P2 value (a label) +** resolve the P2 value to an actual address. ** -** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument -** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by -** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. +** (2) Compute the maximum number of arguments used by any SQL function +** and store that value in *pMaxFuncArgs. ** -** The Op.opflags field is set on all opcodes. +** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately +** indicate what the prepared statement actually does. +** +** (4) Initialize the p4.xAdvance pointer on opcodes that use it. +** +** (5) Reclaim the memory allocated for storing labels. +** +** This routine will only function correctly if the mkopcodeh.tcl generator +** script numbers the opcodes correctly. Changes to this routine must be +** coordinated with changes to mkopcodeh.tcl. */ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ - int i; int nMaxArgs = *pMaxFuncArgs; Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; p->readOnly = 1; p->bIsReader = 0; - for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){ - u8 opcode = pOp->opcode; + pOp = &p->aOp[p->nOp-1]; + while(1){ - /* NOTE: Be sure to update mkopcodeh.awk when adding or removing - ** cases from this switch! */ - switch( opcode ){ - case OP_Function: - case OP_AggStep: { - if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5; - break; - } - case OP_Transaction: { - if( pOp->p2!=0 ) p->readOnly = 0; - /* fall thru */ - } - case OP_AutoCommit: - case OP_Savepoint: { - p->bIsReader = 1; - break; - } + /* Only JUMP opcodes and the short list of special opcodes in the switch + ** below need to be considered. The mkopcodeh.tcl generator script groups + ** all these opcodes together near the front of the opcode list. Skip + ** any opcode that does not need processing by virtual of the fact that + ** it is larger than SQLITE_MX_JUMP_OPCODE, as a performance optimization. + */ + if( pOp->opcode<=SQLITE_MX_JUMP_OPCODE ){ + /* NOTE: Be sure to update mkopcodeh.tcl when adding or removing + ** cases from this switch! */ + switch( pOp->opcode ){ + case OP_Transaction: { + if( pOp->p2!=0 ) p->readOnly = 0; + /* fall thru */ + } + case OP_AutoCommit: + case OP_Savepoint: { + p->bIsReader = 1; + break; + } #ifndef SQLITE_OMIT_WAL - case OP_Checkpoint: + case OP_Checkpoint: #endif - case OP_Vacuum: - case OP_JournalMode: { - p->readOnly = 0; - p->bIsReader = 1; - break; - } + case OP_Vacuum: + case OP_JournalMode: { + p->readOnly = 0; + p->bIsReader = 1; + break; + } #ifndef SQLITE_OMIT_VIRTUALTABLE - case OP_VUpdate: { - if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; - break; - } - case OP_VFilter: { - int n; - assert( p->nOp - i >= 3 ); - assert( pOp[-1].opcode==OP_Integer ); - n = pOp[-1].p1; - if( n>nMaxArgs ) nMaxArgs = n; - break; - } + case OP_VUpdate: { + if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; + break; + } + case OP_VFilter: { + int n; + assert( (pOp - p->aOp) >= 3 ); + assert( pOp[-1].opcode==OP_Integer ); + n = pOp[-1].p1; + if( n>nMaxArgs ) nMaxArgs = n; + break; + } #endif - case OP_Next: - case OP_NextIfOpen: - case OP_SorterNext: { - pOp->p4.xAdvance = sqlite3BtreeNext; - pOp->p4type = P4_ADVANCE; - break; + case OP_Next: + case OP_NextIfOpen: + case OP_SorterNext: { + pOp->p4.xAdvance = sqlite3BtreeNext; + pOp->p4type = P4_ADVANCE; + break; + } + case OP_Prev: + case OP_PrevIfOpen: { + pOp->p4.xAdvance = sqlite3BtreePrevious; + pOp->p4type = P4_ADVANCE; + break; + } } - case OP_Prev: - case OP_PrevIfOpen: { - pOp->p4.xAdvance = sqlite3BtreePrevious; - pOp->p4type = P4_ADVANCE; - break; + if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 && pOp->p2<0 ){ + assert( ADDR(pOp->p2)nLabel ); + pOp->p2 = aLabel[ADDR(pOp->p2)]; } } - - pOp->opflags = sqlite3OpcodeProperty[opcode]; - if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){ - assert( -1-pOp->p2nLabel ); - pOp->p2 = aLabel[-1-pOp->p2]; - } + if( pOp==p->aOp ) break; + pOp--; } sqlite3DbFree(p->db, pParse->aLabel); pParse->aLabel = 0; @@ -65650,6 +74192,20 @@ SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ return p->nOp; } +/* +** Verify that at least N opcode slots are available in p without +** having to malloc for more space (except when compiled using +** SQLITE_TEST_REALLOC_STRESS). This interface is used during testing +** to verify that certain calls to sqlite3VdbeAddOpList() can never +** fail due to a OOM fault and hence that the return value from +** sqlite3VdbeAddOpList() will always be non-NULL. +*/ +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){ + assert( p->nOp + N <= p->pParse->nOpAlloc ); +} +#endif + /* ** This function returns a pointer to the array of opcodes associated with ** the Vdbe passed as the first argument. It is the callers responsibility @@ -65675,97 +74231,103 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg) } /* -** Add a whole list of operations to the operation stack. Return the -** address of the first operation added. +** Add a whole list of operations to the operation stack. Return a +** pointer to the first operation inserted. +** +** Non-zero P2 arguments to jump instructions are automatically adjusted +** so that the jump target is relative to the first operation inserted. */ -SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ - int addr; +SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList( + Vdbe *p, /* Add opcodes to the prepared statement */ + int nOp, /* Number of opcodes to add */ + VdbeOpList const *aOp, /* The opcodes to be added */ + int iLineno /* Source-file line number of first opcode */ +){ + int i; + VdbeOp *pOut, *pFirst; + assert( nOp>0 ); assert( p->magic==VDBE_MAGIC_INIT ); if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ return 0; } - addr = p->nOp; - if( ALWAYS(nOp>0) ){ - int i; - VdbeOpList const *pIn = aOp; - for(i=0; ip2; - VdbeOp *pOut = &p->aOp[i+addr]; - pOut->opcode = pIn->opcode; - pOut->p1 = pIn->p1; - if( p2<0 ){ - assert( sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP ); - pOut->p2 = addr + ADDR(p2); - }else{ - pOut->p2 = p2; - } - pOut->p3 = pIn->p3; - pOut->p4type = P4_NOTUSED; - pOut->p4.p = 0; - pOut->p5 = 0; + pFirst = pOut = &p->aOp[p->nOp]; + for(i=0; iopcode = aOp->opcode; + pOut->p1 = aOp->p1; + pOut->p2 = aOp->p2; + assert( aOp->p2>=0 ); + if( (sqlite3OpcodeProperty[aOp->opcode] & OPFLG_JUMP)!=0 && aOp->p2>0 ){ + pOut->p2 += p->nOp; + } + pOut->p3 = aOp->p3; + pOut->p4type = P4_NOTUSED; + pOut->p4.p = 0; + pOut->p5 = 0; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - pOut->zComment = 0; + pOut->zComment = 0; #endif #ifdef SQLITE_VDBE_COVERAGE - pOut->iSrcLine = iLineno+i; + pOut->iSrcLine = iLineno+i; #else - (void)iLineno; + (void)iLineno; #endif #ifdef SQLITE_DEBUG - if( p->db->flags & SQLITE_VdbeAddopTrace ){ - sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); - } -#endif + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i+p->nOp, &p->aOp[i+p->nOp]); } - p->nOp += nOp; +#endif } - return addr; + p->nOp += nOp; + return pFirst; } +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) /* -** Change the value of the P1 operand for a specific instruction. -** This routine is useful when a large program is loaded from a -** static array using sqlite3VdbeAddOpList but we want to make a -** few minor changes to the program. +** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus(). */ +SQLITE_PRIVATE void sqlite3VdbeScanStatus( + Vdbe *p, /* VM to add scanstatus() to */ + int addrExplain, /* Address of OP_Explain (or 0) */ + int addrLoop, /* Address of loop counter */ + int addrVisit, /* Address of rows visited counter */ + LogEst nEst, /* Estimated number of output rows */ + const char *zName /* Name of table or index being scanned */ +){ + int nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } +} +#endif + + +/* +** Change the value of the opcode, or P1, P2, P3, or P5 operands +** for a specific instruction. +*/ +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, u32 addr, u8 iNewOpcode){ + sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; +} SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p1 = val; - } + sqlite3VdbeGetOp(p,addr)->p1 = val; } - -/* -** Change the value of the P2 operand for a specific instruction. -** This routine is useful for setting a jump destination. -*/ SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p2 = val; - } + sqlite3VdbeGetOp(p,addr)->p2 = val; } - -/* -** Change the value of the P3 operand for a specific instruction. -*/ SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p3 = val; - } + sqlite3VdbeGetOp(p,addr)->p3 = val; } - -/* -** Change the value of the P5 operand for the most recently -** added operation. -*/ -SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ - assert( p!=0 ); - if( p->aOp ){ - assert( p->nOp>0 ); - p->aOp[p->nOp-1].p5 = val; - } +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ + assert( p->nOp>0 || p->db->mallocFailed ); + if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5; } /* @@ -65774,7 +74336,6 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ */ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ sqlite3VdbeChangeP2(p, addr, p->nOp); - p->pParse->iFixedOp = p->nOp - 1; } @@ -65783,7 +74344,7 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ ** the FuncDef is not ephermal, then do nothing. */ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ - if( ALWAYS(pDef) && (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ + if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ sqlite3DbFree(db, pDef); } } @@ -65793,43 +74354,57 @@ static void vdbeFreeOpArray(sqlite3 *, Op *, int); /* ** Delete a P4 value if necessary. */ +static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){ + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); + sqlite3DbFree(db, p); +} +static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ + freeEphemeralFunction(db, p->pFunc); + sqlite3DbFree(db, p); +} static void freeP4(sqlite3 *db, int p4type, void *p4){ - if( p4 ){ - assert( db ); - switch( p4type ){ - case P4_REAL: - case P4_INT64: - case P4_DYNAMIC: - case P4_INTARRAY: { - sqlite3DbFree(db, p4); - break; - } - case P4_KEYINFO: { - if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4); - break; - } - case P4_MPRINTF: { - if( db->pnBytesFreed==0 ) sqlite3_free(p4); - break; - } - case P4_FUNCDEF: { - freeEphemeralFunction(db, (FuncDef*)p4); - break; - } - case P4_MEM: { - if( db->pnBytesFreed==0 ){ - sqlite3ValueFree((sqlite3_value*)p4); - }else{ - Mem *p = (Mem*)p4; - sqlite3DbFree(db, p->zMalloc); - sqlite3DbFree(db, p); - } - break; - } - case P4_VTAB : { - if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); - break; + assert( db ); + switch( p4type ){ + case P4_FUNCCTX: { + freeP4FuncCtx(db, (sqlite3_context*)p4); + break; + } + case P4_REAL: + case P4_INT64: + case P4_DYNAMIC: + case P4_INTARRAY: { + sqlite3DbFree(db, p4); + break; + } + case P4_KEYINFO: { + if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4); + break; + } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + case P4_EXPR: { + sqlite3ExprDelete(db, (Expr*)p4); + break; + } +#endif + case P4_MPRINTF: { + if( db->pnBytesFreed==0 ) sqlite3_free(p4); + break; + } + case P4_FUNCDEF: { + freeEphemeralFunction(db, (FuncDef*)p4); + break; + } + case P4_MEM: { + if( db->pnBytesFreed==0 ){ + sqlite3ValueFree((sqlite3_value*)p4); + }else{ + freeP4Mem(db, (Mem*)p4); } + break; + } + case P4_VTAB : { + if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); + break; } } } @@ -65843,7 +74418,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( aOp ){ Op *pOp; for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ - freeP4(db, pOp->p4type, pOp->p4.p); + if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS sqlite3DbFree(db, pOp->zComment); #endif @@ -65865,24 +74440,25 @@ SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ /* ** Change the opcode at addr into OP_Noop */ -SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ - if( addrnOp ){ - VdbeOp *pOp = &p->aOp[addr]; - sqlite3 *db = p->db; - freeP4(db, pOp->p4type, pOp->p4.p); - memset(pOp, 0, sizeof(pOp[0])); - pOp->opcode = OP_Noop; - if( addr==p->nOp-1 ) p->nOp--; - } +SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ + VdbeOp *pOp; + if( p->db->mallocFailed ) return 0; + assert( addr>=0 && addrnOp ); + pOp = &p->aOp[addr]; + freeP4(p->db, pOp->p4type, pOp->p4.p); + pOp->p4type = P4_NOTUSED; + pOp->p4.z = 0; + pOp->opcode = OP_Noop; + return 1; } /* -** Remove the last opcode inserted +** If the last opcode is "op" and it is not a jump destination, +** then remove it. Return true if and only if an opcode was removed. */ SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ - if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ - sqlite3VdbeChangeToNoop(p, p->nOp-1); - return 1; + if( p->nOp>0 && p->aOp[p->nOp-1].opcode==op ){ + return sqlite3VdbeChangeToNoop(p, p->nOp-1); }else{ return 0; } @@ -65905,16 +74481,34 @@ SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ ** ** If addr<0 then change P4 on the most recently inserted instruction. */ +static void SQLITE_NOINLINE vdbeChangeP4Full( + Vdbe *p, + Op *pOp, + const char *zP4, + int n +){ + if( pOp->p4type ){ + freeP4(p->db, pOp->p4type, pOp->p4.p); + pOp->p4type = 0; + pOp->p4.p = 0; + } + if( n<0 ){ + sqlite3VdbeChangeP4(p, (int)(pOp - p->aOp), zP4, n); + }else{ + if( n==0 ) n = sqlite3Strlen30(zP4); + pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); + pOp->p4type = P4_DYNAMIC; + } +} SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ Op *pOp; sqlite3 *db; assert( p!=0 ); db = p->db; assert( p->magic==VDBE_MAGIC_INIT ); - if( p->aOp==0 || db->mallocFailed ){ - if( n!=P4_VTAB ){ - freeP4(db, n, (void*)*(char**)&zP4); - } + assert( p->aOp!=0 || db->mallocFailed ); + if( db->mallocFailed ){ + if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); return; } assert( p->nOp>0 ); @@ -65923,34 +74517,20 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int addr = p->nOp - 1; } pOp = &p->aOp[addr]; - assert( pOp->p4type==P4_NOTUSED - || pOp->p4type==P4_INT32 - || pOp->p4type==P4_KEYINFO ); - freeP4(db, pOp->p4type, pOp->p4.p); - pOp->p4.p = 0; + if( n>=0 || pOp->p4type ){ + vdbeChangeP4Full(p, pOp, zP4, n); + return; + } if( n==P4_INT32 ){ /* Note: this cast is safe, because the origin data point was an int ** that was cast to a (const char *). */ pOp->p4.i = SQLITE_PTR_TO_INT(zP4); pOp->p4type = P4_INT32; - }else if( zP4==0 ){ - pOp->p4.p = 0; - pOp->p4type = P4_NOTUSED; - }else if( n==P4_KEYINFO ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_KEYINFO; - }else if( n==P4_VTAB ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_VTAB; - sqlite3VtabLock((VTable *)zP4); - assert( ((VTable *)zP4)->db==p->db ); - }else if( n<0 ){ + }else if( zP4!=0 ){ + assert( n<0 ); pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; - }else{ - if( n==0 ) n = sqlite3Strlen30(zP4); - pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); - pOp->p4type = P4_DYNAMIC; + if( n==P4_VTAB ) sqlite3VtabLock((VTable*)zP4); } } @@ -66018,7 +74598,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ ** routine, then a pointer to a dummy VdbeOp will be returned. That opcode ** is readable but not writable, though it is cast to a writable value. ** The return of a dummy opcode allows the call to continue functioning -** after a OOM fault without having to check to see if the return from +** after an OOM fault without having to check to see if the return from ** this routine is a valid pointer. But because the dummy.opcode is 0, ** dummy will never be written to. This is verified by code inspection and ** by running with Valgrind. @@ -66075,12 +74655,21 @@ static int displayComment( const char *zSynopsis; int nOpName; int ii, jj; + char zAlt[50]; zOpName = sqlite3OpcodeName(pOp->opcode); nOpName = sqlite3Strlen30(zOpName); if( zOpName[nOpName+1] ){ int seenCom = 0; char c; zSynopsis = zOpName += nOpName + 1; + if( strncmp(zSynopsis,"IF ",3)==0 ){ + if( pOp->p5 & SQLITE_STOREP2 ){ + sqlite3_snprintf(sizeof(zAlt), zAlt, "r[P2] = (%s)", zSynopsis+3); + }else{ + sqlite3_snprintf(sizeof(zAlt), zAlt, "if %s goto P2", zSynopsis+3); + } + zSynopsis = zAlt; + } for(ii=jj=0; jjop ){ + case TK_STRING: + sqlite3XPrintf(p, "%Q", pExpr->u.zToken); + break; + case TK_INTEGER: + sqlite3XPrintf(p, "%d", pExpr->u.iValue); + break; + case TK_NULL: + sqlite3XPrintf(p, "NULL"); + break; + case TK_REGISTER: { + sqlite3XPrintf(p, "r[%d]", pExpr->iTable); + break; + } + case TK_COLUMN: { + if( pExpr->iColumn<0 ){ + sqlite3XPrintf(p, "rowid"); + }else{ + sqlite3XPrintf(p, "c%d", (int)pExpr->iColumn); + } + break; + } + case TK_LT: zOp = "LT"; break; + case TK_LE: zOp = "LE"; break; + case TK_GT: zOp = "GT"; break; + case TK_GE: zOp = "GE"; break; + case TK_NE: zOp = "NE"; break; + case TK_EQ: zOp = "EQ"; break; + case TK_IS: zOp = "IS"; break; + case TK_ISNOT: zOp = "ISNOT"; break; + case TK_AND: zOp = "AND"; break; + case TK_OR: zOp = "OR"; break; + case TK_PLUS: zOp = "ADD"; break; + case TK_STAR: zOp = "MUL"; break; + case TK_MINUS: zOp = "SUB"; break; + case TK_REM: zOp = "REM"; break; + case TK_BITAND: zOp = "BITAND"; break; + case TK_BITOR: zOp = "BITOR"; break; + case TK_SLASH: zOp = "DIV"; break; + case TK_LSHIFT: zOp = "LSHIFT"; break; + case TK_RSHIFT: zOp = "RSHIFT"; break; + case TK_CONCAT: zOp = "CONCAT"; break; + case TK_UMINUS: zOp = "MINUS"; break; + case TK_UPLUS: zOp = "PLUS"; break; + case TK_BITNOT: zOp = "BITNOT"; break; + case TK_NOT: zOp = "NOT"; break; + case TK_ISNULL: zOp = "ISNULL"; break; + case TK_NOTNULL: zOp = "NOTNULL"; break; -#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ - || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) + default: + sqlite3XPrintf(p, "%s", "expr"); + break; + } + + if( zOp ){ + sqlite3XPrintf(p, "%s(", zOp); + displayP4Expr(p, pExpr->pLeft); + if( pExpr->pRight ){ + sqlite3StrAccumAppend(p, ",", 1); + displayP4Expr(p, pExpr->pRight); + } + sqlite3StrAccumAppend(p, ")", 1); + } +} +#endif /* VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) */ + + +#if VDBE_DISPLAY_P4 /* ** Compute a string that describes the P4 parameter for an opcode. ** Use zTemp for any required temporary buffer space. */ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ char *zP4 = zTemp; + StrAccum x; assert( nTemp>=20 ); + sqlite3StrAccumInit(&x, 0, zTemp, nTemp, 0); switch( pOp->p4type ){ case P4_KEYINFO: { - int i, j; + int j; KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; assert( pKeyInfo->aSortOrder!=0 ); - sqlite3_snprintf(nTemp, zTemp, "k(%d", pKeyInfo->nField); - i = sqlite3Strlen30(zTemp); + sqlite3XPrintf(&x, "k(%d", pKeyInfo->nField); for(j=0; jnField; j++){ CollSeq *pColl = pKeyInfo->aColl[j]; - const char *zColl = pColl ? pColl->zName : "nil"; - int n = sqlite3Strlen30(zColl); - if( n==6 && memcmp(zColl,"BINARY",6)==0 ){ - zColl = "B"; - n = 1; - } - if( i+n>nTemp-6 ){ - memcpy(&zTemp[i],",...",4); - break; - } - zTemp[i++] = ','; - if( pKeyInfo->aSortOrder[j] ){ - zTemp[i++] = '-'; - } - memcpy(&zTemp[i], zColl, n+1); - i += n; + const char *zColl = pColl ? pColl->zName : ""; + if( strcmp(zColl, "BINARY")==0 ) zColl = "B"; + sqlite3XPrintf(&x, ",%s%s", pKeyInfo->aSortOrder[j] ? "-" : "", zColl); } - zTemp[i++] = ')'; - zTemp[i] = 0; - assert( ip4.pExpr); + break; + } +#endif case P4_COLLSEQ: { CollSeq *pColl = pOp->p4.pColl; - sqlite3_snprintf(nTemp, zTemp, "(%.20s)", pColl->zName); + sqlite3XPrintf(&x, "(%.20s)", pColl->zName); break; } case P4_FUNCDEF: { FuncDef *pDef = pOp->p4.pFunc; - sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); + sqlite3XPrintf(&x, "%s(%d)", pDef->zName, pDef->nArg); break; } +#ifdef SQLITE_DEBUG + case P4_FUNCCTX: { + FuncDef *pDef = pOp->p4.pCtx->pFunc; + sqlite3XPrintf(&x, "%s(%d)", pDef->zName, pDef->nArg); + break; + } +#endif case P4_INT64: { - sqlite3_snprintf(nTemp, zTemp, "%lld", *pOp->p4.pI64); + sqlite3XPrintf(&x, "%lld", *pOp->p4.pI64); break; } case P4_INT32: { - sqlite3_snprintf(nTemp, zTemp, "%d", pOp->p4.i); + sqlite3XPrintf(&x, "%d", pOp->p4.i); break; } case P4_REAL: { - sqlite3_snprintf(nTemp, zTemp, "%.16g", *pOp->p4.pReal); + sqlite3XPrintf(&x, "%.16g", *pOp->p4.pReal); break; } case P4_MEM: { @@ -66197,11 +74857,11 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ if( pMem->flags & MEM_Str ){ zP4 = pMem->z; }else if( pMem->flags & MEM_Int ){ - sqlite3_snprintf(nTemp, zTemp, "%lld", pMem->u.i); + sqlite3XPrintf(&x, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ - sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->r); + sqlite3XPrintf(&x, "%.16g", pMem->u.r); }else if( pMem->flags & MEM_Null ){ - sqlite3_snprintf(nTemp, zTemp, "NULL"); + zP4 = "NULL"; }else{ assert( pMem->flags & MEM_Blob ); zP4 = "(blob)"; @@ -66211,22 +74871,34 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ #ifndef SQLITE_OMIT_VIRTUALTABLE case P4_VTAB: { sqlite3_vtab *pVtab = pOp->p4.pVtab->pVtab; - sqlite3_snprintf(nTemp, zTemp, "vtab:%p:%p", pVtab, pVtab->pModule); + sqlite3XPrintf(&x, "vtab:%p", pVtab); break; } #endif case P4_INTARRAY: { - sqlite3_snprintf(nTemp, zTemp, "intarray"); + int i; + int *ai = pOp->p4.ai; + int n = ai[0]; /* The first element of an INTARRAY is always the + ** count of the number of elements to follow */ + for(i=1; ip4.pTab->zName); + break; + } default: { zP4 = pOp->p4.z; if( zP4==0 ){ @@ -66235,10 +74907,11 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ } } } + sqlite3StrAccumFinish(&x); assert( zP4!=0 ); return zP4; } -#endif +#endif /* VDBE_DISPLAY_P4 */ /* ** Declare to the Vdbe that the BTree object at db->aDb[i] is used. @@ -66257,7 +74930,7 @@ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ } } -#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +#if !defined(SQLITE_OMIT_SHARED_CACHE) /* ** If SQLite is compiled to support shared-cache mode and to be threadsafe, ** this routine obtains the mutex associated with each BtShared structure @@ -66300,12 +74973,11 @@ SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe *p){ /* ** Unlock all of the btrees previously locked by a call to sqlite3VdbeEnter(). */ -SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ +static SQLITE_NOINLINE void vdbeLeave(Vdbe *p){ int i; sqlite3 *db; Db *aDb; int nDb; - if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; @@ -66315,6 +74987,10 @@ SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ } } } +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ + vdbeLeave(p); +} #endif #if defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) @@ -66344,21 +75020,35 @@ SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ } #endif +/* +** Initialize an array of N Mem element. +*/ +static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){ + while( (N--)>0 ){ + p->db = db; + p->flags = flags; + p->szMalloc = 0; +#ifdef SQLITE_DEBUG + p->pScopyFrom = 0; +#endif + p++; + } +} + /* ** Release an array of N Mem elements */ static void releaseMemArray(Mem *p, int N){ if( p && N ){ - Mem *pEnd; + Mem *pEnd = &p[N]; sqlite3 *db = p->db; - u8 malloc_failed = db->mallocFailed; if( db->pnBytesFreed ){ - for(pEnd=&p[N]; pzMalloc); - } + do{ + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); + }while( (++p)flags & MEM_RowSet ); if( p->flags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ sqlite3VdbeMemRelease(p); - }else if( p->zMalloc ){ + }else if( p->szMalloc ){ sqlite3DbFree(db, p->zMalloc); - p->zMalloc = 0; + p->szMalloc = 0; } p->flags = MEM_Undefined; - } - db->mallocFailed = malloc_failed; + }while( (++p)v, apCsr[i]); } releaseMemArray(aMem, p->nChildMem); + sqlite3VdbeDeleteAuxData(p->v->db, &p->pAuxData, -1, 0); sqlite3DbFree(p->v->db, p); } @@ -66445,10 +75135,10 @@ SQLITE_PRIVATE int sqlite3VdbeList( releaseMemArray(pMem, 8); p->pResultSet = 0; - if( p->rc==SQLITE_NOMEM ){ + if( p->rc==SQLITE_NOMEM_BKPT ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ - db->mallocFailed = 1; + sqlite3OomFault(db); return SQLITE_ERROR; } @@ -66487,7 +75177,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( }else if( db->u1.isInterrupted ){ p->rc = SQLITE_INTERRUPT; rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(p->rc)); + sqlite3VdbeError(p, sqlite3ErrStr(p->rc)); }else{ char *zP4; Op *pOp; @@ -66549,13 +75239,14 @@ SQLITE_PRIVATE int sqlite3VdbeList( pMem->u.i = pOp->p3; /* P3 */ pMem++; - if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ + if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */ assert( p->db->mallocFailed ); return SQLITE_ERROR; } pMem->flags = MEM_Str|MEM_Term; - zP4 = displayP4(pOp, pMem->z, 32); + zP4 = displayP4(pOp, pMem->z, pMem->szMalloc); if( zP4!=pMem->z ){ + pMem->n = 0; sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); }else{ assert( pMem->z!=0 ); @@ -66565,7 +75256,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( pMem++; if( p->explain==1 ){ - if( sqlite3VdbeMemGrow(pMem, 4, 0) ){ + if( sqlite3VdbeMemClearAndResize(pMem, 4) ){ assert( p->db->mallocFailed ); return SQLITE_ERROR; } @@ -66576,7 +75267,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( pMem++; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - if( sqlite3VdbeMemGrow(pMem, 500, 0) ){ + if( sqlite3VdbeMemClearAndResize(pMem, 500) ){ assert( p->db->mallocFailed ); return SQLITE_ERROR; } @@ -66646,43 +75337,46 @@ SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ } #endif /* !SQLITE_OMIT_TRACE && SQLITE_ENABLE_IOTRACE */ -/* -** Allocate space from a fixed size buffer and return a pointer to -** that space. If insufficient space is available, return NULL. +/* An instance of this object describes bulk memory available for use +** by subcomponents of a prepared statement. Space is allocated out +** of a ReusableSpace object by the allocSpace() routine below. +*/ +struct ReusableSpace { + u8 *pSpace; /* Available memory */ + int nFree; /* Bytes of available memory */ + int nNeeded; /* Total bytes that could not be allocated */ +}; + +/* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf +** from the ReusableSpace object. Return a pointer to the allocated +** memory on success. If insufficient memory is available in the +** ReusableSpace object, increase the ReusableSpace.nNeeded +** value by the amount needed and return NULL. ** -** The pBuf parameter is the initial value of a pointer which will -** receive the new memory. pBuf is normally NULL. If pBuf is not -** NULL, it means that memory space has already been allocated and that -** this routine should not allocate any new memory. When pBuf is not -** NULL simply return pBuf. Only allocate new memory space when pBuf -** is NULL. +** If pBuf is not initially NULL, that means that the memory has already +** been allocated by a prior call to this routine, so just return a copy +** of pBuf and leave ReusableSpace unchanged. ** -** nByte is the number of bytes of space needed. -** -** *ppFrom points to available space and pEnd points to the end of the -** available space. When space is allocated, *ppFrom is advanced past -** the end of the allocated space. -** -** *pnByte is a counter of the number of bytes of space that have failed -** to allocate. If there is insufficient space in *ppFrom to satisfy the -** request, then increment *pnByte by the amount of the request. +** This allocator is employed to repurpose unused slots at the end of the +** opcode array of prepared state for other memory needs of the prepared +** statement. */ static void *allocSpace( - void *pBuf, /* Where return pointer will be stored */ - int nByte, /* Number of bytes to allocate */ - u8 **ppFrom, /* IN/OUT: Allocate from *ppFrom */ - u8 *pEnd, /* Pointer to 1 byte past the end of *ppFrom buffer */ - int *pnByte /* If allocation cannot be made, increment *pnByte */ + struct ReusableSpace *p, /* Bulk memory available for allocation */ + void *pBuf, /* Pointer to a prior allocation */ + int nByte /* Bytes of memory needed */ ){ - assert( EIGHT_BYTE_ALIGNMENT(*ppFrom) ); - if( pBuf ) return pBuf; - nByte = ROUND8(nByte); - if( &(*ppFrom)[nByte] <= pEnd ){ - pBuf = (void*)*ppFrom; - *ppFrom += nByte; - }else{ - *pnByte += nByte; + assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); + if( pBuf==0 ){ + nByte = ROUND8(nByte); + if( nByte <= p->nFree ){ + p->nFree -= nByte; + pBuf = &p->pSpace[p->nFree]; + }else{ + p->nNeeded += nByte; + } } + assert( EIGHT_BYTE_ALIGNMENT(pBuf) ); return pBuf; } @@ -66695,7 +75389,7 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ int i; #endif assert( p!=0 ); - assert( p->magic==VDBE_MAGIC_INIT ); + assert( p->magic==VDBE_MAGIC_INIT || p->magic==VDBE_MAGIC_RESET ); /* There should be at least one opcode. */ @@ -66705,14 +75399,13 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ p->magic = VDBE_MAGIC_RUN; #ifdef SQLITE_DEBUG - for(i=1; inMem; i++){ + for(i=0; inMem; i++){ assert( p->aMem[i].db==p->db ); } #endif p->pc = -1; p->rc = SQLITE_OK; p->errorAction = OE_Abort; - p->magic = VDBE_MAGIC_RUN; p->nChange = 0; p->cacheCtr = 1; p->minWriteFileFormat = 255; @@ -66729,13 +75422,13 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ /* ** Prepare a virtual machine for execution for the first time after ** creating the virtual machine. This involves things such -** as allocating stack space and initializing the program counter. +** as allocating registers and initializing the program counter. ** After the VDBE has be prepped, it can be executed by one or more ** calls to sqlite3VdbeExec(). ** -** This function may be called exact once on a each virtual machine. +** This function may be called exactly once on each virtual machine. ** After this routine is called the VM has been "packaged" and is ready -** to run. After this routine is called, futher calls to +** to run. After this routine is called, further calls to ** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects ** the Vdbe from the Parse object that helped generate it so that the ** the Vdbe becomes an independent entity and the Parse object can be @@ -66753,11 +75446,8 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( int nMem; /* Number of VM memory registers */ int nCursor; /* Number of cursors required */ int nArg; /* Number of arguments in subprograms */ - int nOnce; /* Number of OP_Once instructions */ int n; /* Loop counter */ - u8 *zCsr; /* Memory available for allocation */ - u8 *zEnd; /* First byte past allocated memory */ - int nByte; /* How much extra memory is needed */ + struct ReusableSpace x; /* Reusable bulk memory */ assert( p!=0 ); assert( p->nOp>0 ); @@ -66770,85 +75460,77 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( nMem = pParse->nMem; nCursor = pParse->nTab; nArg = pParse->nMaxArg; - nOnce = pParse->nOnce; - if( nOnce==0 ) nOnce = 1; /* Ensure at least one byte in p->aOnceFlag[] */ - /* For each cursor required, also allocate a memory cell. Memory - ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by - ** the vdbe program. Instead they are used to allocate space for - ** VdbeCursor/BtCursor structures. The blob of memory associated with - ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1) - ** stores the blob of memory associated with cursor 1, etc. - ** + /* Each cursor uses a memory cell. The first cursor (cursor 0) can + ** use aMem[0] which is not otherwise used by the VDBE program. Allocate + ** space at the end of aMem[] for cursors 1 and greater. ** See also: allocateCursor(). */ nMem += nCursor; + if( nCursor==0 && nMem>0 ) nMem++; /* Space for aMem[0] even if not used */ - /* Allocate space for memory registers, SQL variables, VDBE cursors and - ** an array to marshal SQL function arguments in. + /* Figure out how much reusable memory is available at the end of the + ** opcode array. This extra memory will be reallocated for other elements + ** of the prepared statement. */ - zCsr = (u8*)&p->aOp[p->nOp]; /* Memory avaliable for allocation */ - zEnd = (u8*)&p->aOp[pParse->nOpAlloc]; /* First byte past end of zCsr[] */ + n = ROUND8(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ + x.pSpace = &((u8*)p->aOp)[n]; /* Unused opcode memory */ + assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) ); + x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n); /* Bytes of unused memory */ + assert( x.nFree>=0 ); + assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) ); resolveP2Values(p, &nArg); p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); if( pParse->explain && nMem<10 ){ nMem = 10; } - memset(zCsr, 0, zEnd-zCsr); - zCsr += (zCsr - (u8*)0)&7; - assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); p->expired = 0; - /* Memory for registers, parameters, cursor, etc, is allocated in two - ** passes. On the first pass, we try to reuse unused space at the + /* Memory for registers, parameters, cursor, etc, is allocated in one or two + ** passes. On the first pass, we try to reuse unused memory at the ** end of the opcode array. If we are unable to satisfy all memory ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the rest using a fresh allocation. + ** pass will fill in the remainder using a fresh memory allocation. ** ** This two-pass approach that reuses as much memory as possible from - ** the leftover space at the end of the opcode array can significantly + ** the leftover memory at the end of the opcode array. This can significantly ** reduce the amount of memory held by a prepared statement. */ do { - nByte = 0; - p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); - p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); - p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); - p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); - p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), - &zCsr, zEnd, &nByte); - p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); - if( nByte ){ - p->pFree = sqlite3DbMallocZero(db, nByte); - } - zCsr = p->pFree; - zEnd = &zCsr[nByte]; - }while( nByte && !db->mallocFailed ); + x.nNeeded = 0; + p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); +#endif + if( x.nNeeded==0 ) break; + x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); + x.nFree = x.nNeeded; + }while( !db->mallocFailed ); - p->nCursor = nCursor; - p->nOnceFlag = nOnce; - if( p->aVar ){ - p->nVar = (ynVar)nVar; - for(n=0; naVar[n].flags = MEM_Null; - p->aVar[n].db = db; - } - } - if( p->azVar ){ - p->nzVar = pParse->nzVar; - memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); - memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); - } - if( p->aMem ){ - p->aMem--; /* aMem[] goes from 1..nMem */ - p->nMem = nMem; /* not from 0..nMem-1 */ - for(n=1; n<=nMem; n++){ - p->aMem[n].flags = MEM_Undefined; - p->aMem[n].db = db; - } - } + p->nzVar = pParse->nzVar; + p->azVar = pParse->azVar; + pParse->nzVar = 0; + pParse->azVar = 0; p->explain = pParse->explain; + if( db->mallocFailed ){ + p->nVar = 0; + p->nCursor = 0; + p->nMem = 0; + }else{ + p->nCursor = nCursor; + p->nVar = (ynVar)nVar; + initMemArray(p->aVar, nVar, db, MEM_Null); + p->nMem = nMem; + initMemArray(p->aMem, nMem, db, MEM_Undefined); + memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + memset(p->anExec, 0, p->nOp*sizeof(i64)); +#endif + } sqlite3VdbeRewind(p); } @@ -66860,23 +75542,50 @@ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ if( pCx==0 ){ return; } - sqlite3VdbeSorterClose(p->db, pCx); - if( pCx->pBt ){ - sqlite3BtreeClose(pCx->pBt); - /* The pCx->pCursor will be close automatically, if it exists, by - ** the call above. */ - }else if( pCx->pCursor ){ - sqlite3BtreeCloseCursor(pCx->pCursor); - } + assert( pCx->pBt==0 || pCx->eCurType==CURTYPE_BTREE ); + switch( pCx->eCurType ){ + case CURTYPE_SORTER: { + sqlite3VdbeSorterClose(p->db, pCx); + break; + } + case CURTYPE_BTREE: { + if( pCx->pBt ){ + sqlite3BtreeClose(pCx->pBt); + /* The pCx->pCursor will be close automatically, if it exists, by + ** the call above. */ + }else{ + assert( pCx->uc.pCursor!=0 ); + sqlite3BtreeCloseCursor(pCx->uc.pCursor); + } + break; + } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pCx->pVtabCursor ){ - sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; - const sqlite3_module *pModule = pVtabCursor->pVtab->pModule; - p->inVtabMethod = 1; - pModule->xClose(pVtabCursor); - p->inVtabMethod = 0; - } + case CURTYPE_VTAB: { + sqlite3_vtab_cursor *pVCur = pCx->uc.pVCur; + const sqlite3_module *pModule = pVCur->pVtab->pModule; + assert( pVCur->pVtab->nRef>0 ); + pVCur->pVtab->nRef--; + pModule->xClose(pVCur); + break; + } #endif + } +} + +/* +** Close all cursors in the current frame. +*/ +static void closeCursorsInFrame(Vdbe *p){ + if( p->apCsr ){ + int i; + for(i=0; inCursor; i++){ + VdbeCursor *pC = p->apCsr[i]; + if( pC ){ + sqlite3VdbeFreeCursor(p, pC); + p->apCsr[i] = 0; + } + } + } } /* @@ -66886,8 +75595,10 @@ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ */ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; - v->aOnceFlag = pFrame->aOnceFlag; - v->nOnceFlag = pFrame->nOnceFlag; + closeCursorsInFrame(v); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + v->anExec = pFrame->anExec; +#endif v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; @@ -66896,6 +75607,10 @@ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ v->nCursor = pFrame->nCursor; v->db->lastRowid = pFrame->lastRowid; v->nChange = pFrame->nChange; + v->db->nChange = pFrame->nDbChange; + sqlite3VdbeDeleteAuxData(v->db, &v->pAuxData, -1, 0); + v->pAuxData = pFrame->pAuxData; + pFrame->pAuxData = 0; return pFrame->pc; } @@ -66912,22 +75627,13 @@ static void closeAllCursors(Vdbe *p){ VdbeFrame *pFrame; for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); sqlite3VdbeFrameRestore(pFrame); + p->pFrame = 0; + p->nFrame = 0; } - p->pFrame = 0; - p->nFrame = 0; - - if( p->apCsr ){ - int i; - for(i=0; inCursor; i++){ - VdbeCursor *pC = p->apCsr[i]; - if( pC ){ - sqlite3VdbeFreeCursor(p, pC); - p->apCsr[i] = 0; - } - } - } + assert( p->nFrame==0 ); + closeCursorsInFrame(p); if( p->aMem ){ - releaseMemArray(&p->aMem[1], p->nMem); + releaseMemArray(p->aMem, p->nMem); } while( p->pDelFrame ){ VdbeFrame *pDel = p->pDelFrame; @@ -66936,16 +75642,12 @@ static void closeAllCursors(Vdbe *p){ } /* Delete any auxdata allocations made by the VM */ - sqlite3VdbeDeleteAuxData(p, -1, 0); + if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p->db, &p->pAuxData, -1, 0); assert( p->pAuxData==0 ); } /* -** Clean up the VM after execution. -** -** This routine will automatically close any cursors, lists, and/or -** sorters that were left open. It also deletes the values of -** variables in the aVar[] array. +** Clean up the VM after a single run. */ static void Cleanup(Vdbe *p){ sqlite3 *db = p->db; @@ -66956,7 +75658,7 @@ static void Cleanup(Vdbe *p){ int i; if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 ); if( p->aMem ){ - for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); + for(i=0; inMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); } #endif @@ -66980,13 +75682,9 @@ SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ sqlite3DbFree(db, p->aColName); n = nResColumn*COLNAME_N; p->nResColumn = (u16)nResColumn; - p->aColName = pColName = (Mem*)sqlite3DbMallocZero(db, sizeof(Mem)*n ); + p->aColName = pColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n ); if( p->aColName==0 ) return; - while( n-- > 0 ){ - pColName->flags = MEM_Null; - pColName->db = p->db; - pColName++; - } + initMemArray(p->aColName, n, p->db, MEM_Null); } /* @@ -67012,7 +75710,7 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( assert( vardb->mallocFailed ){ assert( !zName || xDel!=SQLITE_DYNAMIC ); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResColumn]); @@ -67029,7 +75727,9 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( */ static int vdbeCommit(sqlite3 *db, Vdbe *p){ int i; - int nTrans = 0; /* Number of databases with an active write-transaction */ + int nTrans = 0; /* Number of databases with an active write-transaction + ** that are candidates for a two-phase commit using a + ** master-journal */ int rc = SQLITE_OK; int needXcommit = 0; @@ -67057,10 +75757,28 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ + /* Whether or not a database might need a master journal depends upon + ** its journal mode (among other things). This matrix determines which + ** journal modes use a master journal and which do not */ + static const u8 aMJNeeded[] = { + /* DELETE */ 1, + /* PERSIST */ 1, + /* OFF */ 0, + /* TRUNCATE */ 1, + /* MEMORY */ 0, + /* WAL */ 0 + }; + Pager *pPager; /* Pager associated with pBt */ needXcommit = 1; - if( i!=1 ) nTrans++; sqlite3BtreeEnter(pBt); - rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); + pPager = sqlite3BtreePager(pBt); + if( db->aDb[i].safety_level!=PAGER_SYNCHRONOUS_OFF + && aMJNeeded[sqlite3PagerGetJournalMode(pPager)] + ){ + assert( i!=1 ); + nTrans++; + } + rc = sqlite3PagerExclusiveLock(pPager); sqlite3BtreeLeave(pBt); } } @@ -67113,12 +75831,11 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* The complex case - There is a multi-file write-transaction active. ** This requires a master journal file to ensure the transaction is - ** committed atomicly. + ** committed atomically. */ #ifndef SQLITE_OMIT_DISKIO else{ sqlite3_vfs *pVfs = db->pVfs; - int needSync = 0; char *zMaster = 0; /* File-name for the master journal */ char const *zMainFile = sqlite3BtreeGetFilename(db->aDb[0].pBt); sqlite3_file *pMaster = 0; @@ -67130,7 +75847,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* Select a master journal file name */ nMainFile = sqlite3Strlen30(zMainFile); zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XXz", zMainFile); - if( zMaster==0 ) return SQLITE_NOMEM; + if( zMaster==0 ) return SQLITE_NOMEM_BKPT; do { u32 iRandom; if( retryCount ){ @@ -67178,9 +75895,6 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ continue; /* Ignore TEMP and :memory: databases */ } assert( zFile[0]!=0 ); - if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ - needSync = 1; - } rc = sqlite3OsWrite(pMaster, zFile, sqlite3Strlen30(zFile)+1, offset); offset += sqlite3Strlen30(zFile)+1; if( rc!=SQLITE_OK ){ @@ -67195,8 +75909,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* Sync the master journal file. If the IOCAP_SEQUENTIAL device ** flag is set this is not required. */ - if( needSync - && 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) + if( 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) && SQLITE_OK!=(rc = sqlite3OsSync(pMaster, SQLITE_SYNC_NORMAL)) ){ sqlite3OsCloseFree(pMaster); @@ -67379,7 +76092,7 @@ SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ ){ p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; p->errorAction = OE_Abort; - sqlite3SetString(&p->zErrMsg, db, "FOREIGN KEY constraint failed"); + sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); return SQLITE_ERROR; } return SQLITE_OK; @@ -67419,10 +76132,9 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ ** one, or the complete transaction if there is no statement transaction. */ - if( p->db->mallocFailed ){ - p->rc = SQLITE_NOMEM; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM_BKPT; } - if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag); closeAllCursors(p); if( p->magic!=VDBE_MAGIC_RUN ){ return SQLITE_OK; @@ -67466,6 +76178,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } } @@ -67506,6 +76219,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ }else if( rc!=SQLITE_OK ){ p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; @@ -67514,6 +76228,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } }else{ sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; } db->nStatement = 0; }else if( eStatementOp==0 ){ @@ -67525,6 +76240,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } @@ -67545,6 +76261,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } @@ -67575,8 +76292,8 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } p->magic = VDBE_MAGIC_HALT; checkActiveVdbeCnt(db); - if( p->db->mallocFailed ){ - p->rc = SQLITE_NOMEM; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM_BKPT; } /* If the auto-commit flag is set to true, then any locks that were held @@ -67612,15 +76329,15 @@ SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p){ sqlite3 *db = p->db; int rc = p->rc; if( p->zErrMsg ){ - u8 mallocFailed = db->mallocFailed; + db->bBenignMalloc++; sqlite3BeginBenignMalloc(); if( db->pErr==0 ) db->pErr = sqlite3ValueNew(db); sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); sqlite3EndBenignMalloc(); - db->mallocFailed = mallocFailed; + db->bBenignMalloc--; db->errCode = rc; }else{ - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } return rc; } @@ -67683,7 +76400,7 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ ** to sqlite3_step(). For consistency (since sqlite3_step() was ** called), set the database error in this case as well. */ - sqlite3Error(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); + sqlite3ErrorWithMsg(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } @@ -67729,7 +76446,7 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ } #endif p->iCurrentTime = 0; - p->magic = VDBE_MAGIC_INIT; + p->magic = VDBE_MAGIC_RESET; return p->rc & db->errMask; } @@ -67761,10 +76478,9 @@ SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ ** from left to right), or ** ** * the corresponding bit in argument mask is clear (where the first -** function parameter corrsponds to bit 0 etc.). +** function parameter corresponds to bit 0 etc.). */ -SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){ - AuxData **pp = &pVdbe->pAuxData; +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, int mask){ while( *pp ){ AuxData *pAux = *pp; if( (iOp<0) @@ -67775,7 +76491,7 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){ pAux->xDelete(pAux->pAux); } *pp = pAux->pNext; - sqlite3DbFree(pVdbe->db, pAux); + sqlite3DbFree(db, pAux); }else{ pp= &pAux->pNext; } @@ -67794,21 +76510,26 @@ SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; int i; assert( p->db==0 || p->db==db ); - releaseMemArray(p->aVar, p->nVar); releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); for(pSub=p->pProgram; pSub; pSub=pNext){ pNext = pSub->pNext; vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); sqlite3DbFree(db, pSub); } - for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); + if( p->magic!=VDBE_MAGIC_INIT ){ + releaseMemArray(p->aVar, p->nVar); + for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); + sqlite3DbFree(db, p->azVar); + sqlite3DbFree(db, p->pFree); + } vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); - sqlite3DbFree(db, p->pFree); -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - sqlite3DbFree(db, p->zExplain); - sqlite3DbFree(db, p->pExplain); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + for(i=0; inScan; i++){ + sqlite3DbFree(db, p->aScan[i].zName); + } + sqlite3DbFree(db, p->aScan); #endif } @@ -67836,6 +76557,60 @@ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ sqlite3DbFree(db, p); } +/* +** The cursor "p" has a pending seek operation that has not yet been +** carried out. Seek the cursor now. If an error occurs, return +** the appropriate error code. +*/ +static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ + int res, rc; +#ifdef SQLITE_TEST + extern int sqlite3_search_count; +#endif + assert( p->deferredMoveto ); + assert( p->isTable ); + assert( p->eCurType==CURTYPE_BTREE ); + rc = sqlite3BtreeMovetoUnpacked(p->uc.pCursor, 0, p->movetoTarget, 0, &res); + if( rc ) return rc; + if( res!=0 ) return SQLITE_CORRUPT_BKPT; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + p->deferredMoveto = 0; + p->cacheStatus = CACHE_STALE; + return SQLITE_OK; +} + +/* +** Something has moved cursor "p" out of place. Maybe the row it was +** pointed to was deleted out from under it. Or maybe the btree was +** rebalanced. Whatever the cause, try to restore "p" to the place it +** is supposed to be pointing. If the row was deleted out from under the +** cursor, set the cursor to point to a NULL row. +*/ +static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ + int isDifferentRow, rc; + assert( p->eCurType==CURTYPE_BTREE ); + assert( p->uc.pCursor!=0 ); + assert( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ); + rc = sqlite3BtreeCursorRestore(p->uc.pCursor, &isDifferentRow); + p->cacheStatus = CACHE_STALE; + if( isDifferentRow ) p->nullRow = 1; + return rc; +} + +/* +** Check to ensure that the cursor is valid. Restore the cursor +** if need be. Return any I/O error from the restore operation. +*/ +SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor *p){ + assert( p->eCurType==CURTYPE_BTREE ); + if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ + return handleMovedCursor(p); + } + return SQLITE_OK; +} + /* ** Make sure the cursor p is ready to read or write the row to which it ** was last positioned. Return an error code if an OOM fault or I/O error @@ -67849,30 +76624,20 @@ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ ** If the cursor is already pointing to the correct row and that row has ** not been deleted out from under the cursor, then this routine is a no-op. */ -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ - if( p->deferredMoveto ){ - int res, rc; -#ifdef SQLITE_TEST - extern int sqlite3_search_count; -#endif - assert( p->isTable ); - rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); - if( rc ) return rc; - p->lastRowid = p->movetoTarget; - if( res!=0 ) return SQLITE_CORRUPT_BKPT; - p->rowidIsValid = 1; -#ifdef SQLITE_TEST - sqlite3_search_count++; -#endif - p->deferredMoveto = 0; - p->cacheStatus = CACHE_STALE; - }else if( p->pCursor ){ - int hasMoved; - int rc = sqlite3BtreeCursorHasMoved(p->pCursor, &hasMoved); - if( rc ) return rc; - if( hasMoved ){ - p->cacheStatus = CACHE_STALE; - if( hasMoved==2 ) p->nullRow = 1; +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ + VdbeCursor *p = *pp; + if( p->eCurType==CURTYPE_BTREE ){ + if( p->deferredMoveto ){ + int iMap; + if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){ + *pp = p->pAltCursor; + *piCol = iMap - 1; + return SQLITE_OK; + } + return handleDeferredMoveto(p); + } + if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ + return handleMovedCursor(p); } } return SQLITE_OK; @@ -67923,11 +76688,13 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ /* ** Return the serial-type for the value stored in pMem. */ -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ int flags = pMem->flags; u32 n; + assert( pLen!=0 ); if( flags&MEM_Null ){ + *pLen = 0; return 0; } if( flags&MEM_Int ){ @@ -67936,22 +76703,28 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ i64 i = pMem->u.i; u64 u; if( i<0 ){ - if( i<(-MAX_6BYTE) ) return 6; - /* Previous test prevents: u = -(-9223372036854775808) */ - u = -i; + u = ~i; }else{ u = i; } if( u<=127 ){ - return ((i&1)==i && file_format>=4) ? 8+(u32)u : 1; + if( (i&1)==i && file_format>=4 ){ + *pLen = 0; + return 8+(u32)u; + }else{ + *pLen = 1; + return 1; + } } - if( u<=32767 ) return 2; - if( u<=8388607 ) return 3; - if( u<=2147483647 ) return 4; - if( u<=MAX_6BYTE ) return 5; + if( u<=32767 ){ *pLen = 2; return 2; } + if( u<=8388607 ){ *pLen = 3; return 3; } + if( u<=2147483647 ){ *pLen = 4; return 4; } + if( u<=MAX_6BYTE ){ *pLen = 6; return 5; } + *pLen = 8; return 6; } if( flags&MEM_Real ){ + *pLen = 8; return 7; } assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); @@ -67960,20 +76733,46 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ if( flags & MEM_Zero ){ n += pMem->u.nZero; } + *pLen = n; return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } +/* +** The sizes for serial types less than 128 +*/ +static const u8 sqlite3SmallTypeSizes[] = { + /* 0 1 2 3 4 5 6 7 8 9 */ +/* 0 */ 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, +/* 10 */ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, +/* 20 */ 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, +/* 30 */ 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, +/* 40 */ 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, +/* 50 */ 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, +/* 60 */ 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, +/* 70 */ 29, 29, 30, 30, 31, 31, 32, 32, 33, 33, +/* 80 */ 34, 34, 35, 35, 36, 36, 37, 37, 38, 38, +/* 90 */ 39, 39, 40, 40, 41, 41, 42, 42, 43, 43, +/* 100 */ 44, 44, 45, 45, 46, 46, 47, 47, 48, 48, +/* 110 */ 49, 49, 50, 50, 51, 51, 52, 52, 53, 53, +/* 120 */ 54, 54, 55, 55, 56, 56, 57, 57 +}; + /* ** Return the length of the data corresponding to the supplied serial-type. */ SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ - if( serial_type>=12 ){ + if( serial_type>=128 ){ return (serial_type-12)/2; }else{ - static const u8 aSize[] = { 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 }; - return aSize[serial_type]; + assert( serial_type<12 + || sqlite3SmallTypeSizes[serial_type]==(serial_type - 12)/2 ); + return sqlite3SmallTypeSizes[serial_type]; } } +SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){ + assert( serial_type<128 ); + return sqlite3SmallTypeSizes[serial_type]; +} /* ** If we are on an architecture with mixed-endian floating @@ -68049,17 +76848,18 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ u64 v; u32 i; if( serial_type==7 ){ - assert( sizeof(v)==sizeof(pMem->r) ); - memcpy(&v, &pMem->r, sizeof(v)); + assert( sizeof(v)==sizeof(pMem->u.r) ); + memcpy(&v, &pMem->u.r, sizeof(v)); swapMixedEndianFloat(v); }else{ v = pMem->u.i; } - len = i = sqlite3VdbeSerialTypeLen(serial_type); - while( i-- ){ - buf[i] = (u8)(v&0xFF); + len = i = sqlite3SmallTypeSizes[serial_type]; + assert( i>0 ); + do{ + buf[--i] = (u8)(v&0xFF); v >>= 8; - } + }while( i ); return len; } @@ -68068,7 +76868,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) == (int)sqlite3VdbeSerialTypeLen(serial_type) ); len = pMem->n; - memcpy(buf, pMem->z, len); + if( len>0 ) memcpy(buf, pMem->z, len); return len; } @@ -68083,51 +76883,105 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ #define TWO_BYTE_INT(x) (256*(i8)((x)[0])|(x)[1]) #define THREE_BYTE_INT(x) (65536*(i8)((x)[0])|((x)[1]<<8)|(x)[2]) #define FOUR_BYTE_UINT(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) +#define FOUR_BYTE_INT(x) (16777216*(i8)((x)[0])|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) /* ** Deserialize the data blob pointed to by buf as serial type serial_type ** and store the result in pMem. Return the number of bytes read. +** +** This function is implemented as two separate routines for performance. +** The few cases that require local variables are broken out into a separate +** routine so that in most cases the overhead of moving the stack pointer +** is avoided. */ +static u32 SQLITE_NOINLINE serialGet( + const unsigned char *buf, /* Buffer to deserialize from */ + u32 serial_type, /* Serial type to deserialize */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; + if( serial_type==6 ){ + /* EVIDENCE-OF: R-29851-52272 Value is a big-endian 64-bit + ** twos-complement integer. */ + pMem->u.i = *(i64*)&x; + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + }else{ + /* EVIDENCE-OF: R-57343-49114 Value is a big-endian IEEE 754-2008 64-bit + ** floating point number. */ +#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) + /* Verify that integers and floating point values use the same + ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is + ** defined that 64-bit floating point values really are mixed + ** endian. + */ + static const u64 t1 = ((u64)0x3ff00000)<<32; + static const double r1 = 1.0; + u64 t2 = t1; + swapMixedEndianFloat(t2); + assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); +#endif + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); + pMem->flags = sqlite3IsNaN(pMem->u.r) ? MEM_Null : MEM_Real; + } + return 8; +} SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ ){ - u64 x; - u32 y; switch( serial_type ){ case 10: /* Reserved for future use */ case 11: /* Reserved for future use */ - case 0: { /* NULL */ + case 0: { /* Null */ + /* EVIDENCE-OF: R-24078-09375 Value is a NULL. */ pMem->flags = MEM_Null; break; } - case 1: { /* 1-byte signed integer */ + case 1: { + /* EVIDENCE-OF: R-44885-25196 Value is an 8-bit twos-complement + ** integer. */ pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 1; } case 2: { /* 2-byte signed integer */ + /* EVIDENCE-OF: R-49794-35026 Value is a big-endian 16-bit + ** twos-complement integer. */ pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 2; } case 3: { /* 3-byte signed integer */ + /* EVIDENCE-OF: R-37839-54301 Value is a big-endian 24-bit + ** twos-complement integer. */ pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 3; } case 4: { /* 4-byte signed integer */ - y = FOUR_BYTE_UINT(buf); - pMem->u.i = (i64)*(int*)&y; + /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit + ** twos-complement integer. */ + pMem->u.i = FOUR_BYTE_INT(buf); +#ifdef __HP_cc + /* Work around a sign-extension bug in the HP compiler for HP/UX */ + if( buf[0]&0x80 ) pMem->u.i |= 0xffffffff80000000LL; +#endif pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 4; } case 5: { /* 6-byte signed integer */ + /* EVIDENCE-OF: R-50385-09674 Value is a big-endian 48-bit + ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); @@ -68135,52 +76989,32 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( } case 6: /* 8-byte signed integer */ case 7: { /* IEEE floating point */ -#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) - /* Verify that integers and floating point values use the same - ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is - ** defined that 64-bit floating point values really are mixed - ** endian. - */ - static const u64 t1 = ((u64)0x3ff00000)<<32; - static const double r1 = 1.0; - u64 t2 = t1; - swapMixedEndianFloat(t2); - assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); -#endif - x = FOUR_BYTE_UINT(buf); - y = FOUR_BYTE_UINT(buf+4); - x = (x<<32) | y; - if( serial_type==6 ){ - pMem->u.i = *(i64*)&x; - pMem->flags = MEM_Int; - testcase( pMem->u.i<0 ); - }else{ - assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); - swapMixedEndianFloat(x); - memcpy(&pMem->r, &x, sizeof(x)); - pMem->flags = sqlite3IsNaN(pMem->r) ? MEM_Null : MEM_Real; - } - return 8; + /* These use local variables, so do them in a separate routine + ** to avoid having to move the frame pointer in the common case */ + return serialGet(buf,serial_type,pMem); } case 8: /* Integer 0 */ case 9: { /* Integer 1 */ + /* EVIDENCE-OF: R-12976-22893 Value is the integer 0. */ + /* EVIDENCE-OF: R-18143-12121 Value is the integer 1. */ pMem->u.i = serial_type-8; pMem->flags = MEM_Int; return 0; } default: { + /* EVIDENCE-OF: R-14606-31564 Value is a BLOB that is (N-12)/2 bytes in + ** length. + ** EVIDENCE-OF: R-28401-00140 Value is a string in the text encoding and + ** (N-13)/2 bytes in length. */ static const u16 aFlag[] = { MEM_Blob|MEM_Ephem, MEM_Str|MEM_Ephem }; - u32 len = (serial_type-12)/2; pMem->z = (char *)buf; - pMem->n = len; - pMem->xDel = 0; + pMem->n = (serial_type-12)/2; pMem->flags = aFlag[serial_type&1]; - return len; + return pMem->n; } } return 0; } - /* ** This routine is used to allocate sufficient space for an UnpackedRecord ** structure large enough to be used with sqlite3VdbeRecordUnpack() if @@ -68250,17 +77084,18 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( idx = getVarint32(aKey, szHdr); d = szHdr; u = 0; - while( idxnField && d<=nKey ){ + while( idxenc = pKeyInfo->enc; pMem->db = pKeyInfo->db; /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ - pMem->zMalloc = 0; + pMem->szMalloc = 0; + pMem->z = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); pMem++; - u++; + if( (++u)>=p->nField ) break; } assert( u<=pKeyInfo->nField + 1 ); p->nField = u; @@ -68274,10 +77109,14 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( ** sqlite3VdbeSerialGet() and sqlite3MemCompare() functions. It is used ** in assert() statements to ensure that the optimized code in ** sqlite3VdbeRecordCompare() returns results with these two primitives. +** +** Return true if the result of comparison is equivalent to desiredResult. +** Return false if there is a disagreement. */ static int vdbeRecordCompareDebug( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2 /* Right key */ + const UnpackedRecord *pPKey2, /* Right key */ + int desiredResult /* Correct answer */ ){ u32 d1; /* Offset into aKey[] of next data element */ u32 idx1; /* Offset into aKey[] of next header element */ @@ -68289,10 +77128,11 @@ static int vdbeRecordCompareDebug( Mem mem1; pKeyInfo = pPKey2->pKeyInfo; + if( pKeyInfo->db==0 ) return 1; mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ - VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ /* Compilers may complain that mem1.u.i is potentially uninitialized. ** We could initialize it, as shown here, to silence those complaints. @@ -68304,6 +77144,7 @@ static int vdbeRecordCompareDebug( /* mem1.u.i = 0; // not needed, here to silence compiler warning */ idx1 = getVarint32(aKey1, szHdr1); + if( szHdr1>98307 ) return SQLITE_CORRUPT; d1 = szHdr1; assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); assert( pKeyInfo->aSortOrder!=0 ); @@ -68335,11 +77176,11 @@ static int vdbeRecordCompareDebug( */ rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], pKeyInfo->aColl[i]); if( rc!=0 ){ - assert( mem1.zMalloc==0 ); /* See comment below */ + assert( mem1.szMalloc==0 ); /* See comment below */ if( pKeyInfo->aSortOrder[i] ){ rc = -rc; /* Invert the result for DESC sort order. */ } - return rc; + goto debugCompareEnd; } i++; }while( idx1nField ); @@ -68348,15 +77189,59 @@ static int vdbeRecordCompareDebug( ** the following assert(). If the assert() fails, it indicates a ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ - assert( mem1.zMalloc==0 ); + assert( mem1.szMalloc==0 ); /* rc==0 here means that one of the keys ran out of fields and - ** all the fields up to that point were equal. Return the the default_rc + ** all the fields up to that point were equal. Return the default_rc ** value. */ - return pPKey2->default_rc; + rc = pPKey2->default_rc; + +debugCompareEnd: + if( desiredResult==0 && rc==0 ) return 1; + if( desiredResult<0 && rc<0 ) return 1; + if( desiredResult>0 && rc>0 ) return 1; + if( CORRUPT_DB ) return 1; + if( pKeyInfo->db->mallocFailed ) return 1; + return 0; } #endif +#if SQLITE_DEBUG +/* +** Count the number of fields (a.k.a. columns) in the record given by +** pKey,nKey. The verify that this count is less than or equal to the +** limit given by pKeyInfo->nField + pKeyInfo->nXField. +** +** If this constraint is not satisfied, it means that the high-speed +** vdbeRecordCompareInt() and vdbeRecordCompareString() routines will +** not work correctly. If this assert() ever fires, it probably means +** that the KeyInfo.nField or KeyInfo.nXField values were computed +** incorrectly. +*/ +static void vdbeAssertFieldCountWithinLimits( + int nKey, const void *pKey, /* The record to verify */ + const KeyInfo *pKeyInfo /* Compare size with this KeyInfo */ +){ + int nField = 0; + u32 szHdr; + u32 idx; + u32 notUsed; + const unsigned char *aKey = (const unsigned char*)pKey; + + if( CORRUPT_DB ) return; + idx = getVarint32(aKey, szHdr); + assert( nKey>=0 ); + assert( szHdr<=(u32)nKey ); + while( idxnField+pKeyInfo->nXField ); +} +#else +# define vdbeAssertFieldCountWithinLimits(A,B,C) +#endif + /* ** Both *pMem1 and *pMem2 contain string values. Compare the two values ** using the collation sequence pColl. As usual, return a negative , zero @@ -68366,7 +77251,8 @@ static int vdbeRecordCompareDebug( static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, - const CollSeq *pColl + const CollSeq *pColl, + u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ ){ if( pMem1->enc==pColl->enc ){ /* The strings are already in the correct encoding. Call the @@ -68378,8 +77264,8 @@ static int vdbeCompareMemString( int n1, n2; Mem c1; Mem c2; - memset(&c1, 0, sizeof(c1)); - memset(&c2, 0, sizeof(c2)); + sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); + sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); @@ -68387,12 +77273,87 @@ static int vdbeCompareMemString( v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); n2 = v2==0 ? 0 : c2.n; rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); + if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM_BKPT; sqlite3VdbeMemRelease(&c1); sqlite3VdbeMemRelease(&c2); return rc; } } +/* +** The input pBlob is guaranteed to be a Blob that is not marked +** with MEM_Zero. Return true if it could be a zero-blob. +*/ +static int isAllZero(const char *z, int n){ + int i; + for(i=0; in; + int n2 = pB2->n; + + /* It is possible to have a Blob value that has some non-zero content + ** followed by zero content. But that only comes up for Blobs formed + ** by the OP_MakeRecord opcode, and such Blobs never get passed into + ** sqlite3MemCompare(). */ + assert( (pB1->flags & MEM_Zero)==0 || n1==0 ); + assert( (pB2->flags & MEM_Zero)==0 || n2==0 ); + + if( (pB1->flags|pB2->flags) & MEM_Zero ){ + if( pB1->flags & pB2->flags & MEM_Zero ){ + return pB1->u.nZero - pB2->u.nZero; + }else if( pB1->flags & MEM_Zero ){ + if( !isAllZero(pB2->z, pB2->n) ) return -1; + return pB1->u.nZero - n2; + }else{ + if( !isAllZero(pB1->z, pB1->n) ) return +1; + return n1 - pB2->u.nZero; + } + } + c = memcmp(pB1->z, pB2->z, n1>n2 ? n2 : n1); + if( c ) return c; + return n1 - n2; +} + +/* +** Do a comparison between a 64-bit signed integer and a 64-bit floating-point +** number. Return negative, zero, or positive if the first (i64) is less than, +** equal to, or greater than the second (double). +*/ +static int sqlite3IntFloatCompare(i64 i, double r){ + if( sizeof(LONGDOUBLE_TYPE)>8 ){ + LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i; + if( xr ) return +1; + return 0; + }else{ + i64 y; + double s; + if( r<-9223372036854775808.0 ) return +1; + if( r>9223372036854775807.0 ) return -1; + y = (i64)r; + if( iy ){ + if( y==SMALLEST_INT64 && r>0.0 ) return -1; + return +1; + } + s = (double)i; + if( sr ) return +1; + return 0; + } +} + /* ** Compare the values contained by the two memory cells, returning ** negative, zero or positive if pMem1 is less than, equal to, or greater @@ -68403,7 +77364,6 @@ static int vdbeCompareMemString( ** Two NULL values are considered equal by this function. */ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ - int rc; int f1, f2; int combined_flags; @@ -68419,34 +77379,34 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C return (f2&MEM_Null) - (f1&MEM_Null); } - /* If one value is a number and the other is not, the number is less. - ** If both are numbers, compare as reals if one is a real, or as integers - ** if both values are integers. + /* At least one of the two values is a number */ if( combined_flags&(MEM_Int|MEM_Real) ){ - double r1, r2; if( (f1 & f2 & MEM_Int)!=0 ){ if( pMem1->u.i < pMem2->u.i ) return -1; - if( pMem1->u.i > pMem2->u.i ) return 1; + if( pMem1->u.i > pMem2->u.i ) return +1; return 0; } + if( (f1 & f2 & MEM_Real)!=0 ){ + if( pMem1->u.r < pMem2->u.r ) return -1; + if( pMem1->u.r > pMem2->u.r ) return +1; + return 0; + } + if( (f1&MEM_Int)!=0 ){ + if( (f2&MEM_Real)!=0 ){ + return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r); + }else{ + return -1; + } + } if( (f1&MEM_Real)!=0 ){ - r1 = pMem1->r; - }else if( (f1&MEM_Int)!=0 ){ - r1 = (double)pMem1->u.i; - }else{ - return 1; + if( (f2&MEM_Int)!=0 ){ + return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r); + }else{ + return -1; + } } - if( (f2&MEM_Real)!=0 ){ - r2 = pMem2->r; - }else if( (f2&MEM_Int)!=0 ){ - r2 = (double)pMem2->u.i; - }else{ - return -1; - } - if( r1r2 ) return 1; - return 0; + return +1; } /* If one value is a string and the other is a blob, the string is less. @@ -68460,7 +77420,7 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C return -1; } - assert( pMem1->enc==pMem2->enc ); + assert( pMem1->enc==pMem2->enc || pMem1->db->mallocFailed ); assert( pMem1->enc==SQLITE_UTF8 || pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); @@ -68471,18 +77431,14 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C assert( !pColl || pColl->xCmp ); if( pColl ){ - return vdbeCompareMemString(pMem1, pMem2, pColl); + return vdbeCompareMemString(pMem1, pMem2, pColl, 0); } /* If a NULL pointer was passed as the collate function, fall through ** to the blob case and use memcmp(). */ } /* Both values must be blobs. Compare using memcmp(). */ - rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); - if( rc==0 ){ - rc = pMem1->n - pMem2->n; - } - return rc; + return sqlite3BlobCompare(pMem1, pMem2); } @@ -68532,7 +77488,7 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero ** or positive integer if key1 is less than, equal to or ** greater than key2. The {nKey1, pKey1} key must be a blob -** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 +** created by the OP_MakeRecord opcode of the VDBE. The pPKey2 ** key must be a parsed key such as obtained from ** sqlite3VdbeParseRecord. ** @@ -68543,10 +77499,12 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. ** -** If database corruption is discovered, set pPKey2->isCorrupt to non-zero -** and return 0. +** If database corruption is discovered, set pPKey2->errCode to +** SQLITE_CORRUPT and return 0. If an OOM error is encountered, +** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the +** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). */ -SQLITE_PRIVATE int sqlite3VdbeRecordCompare( +SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ @@ -68575,13 +77533,13 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; if( d1>(unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } i = 0; } - VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); assert( pPKey2->pKeyInfo->aSortOrder!=0 ); @@ -68594,18 +77552,13 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( if( pRhs->flags & MEM_Int ){ serial_type = aKey1[idx1]; testcase( serial_type==12 ); - if( serial_type>=12 ){ + if( serial_type>=10 ){ rc = +1; }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ - double rhs = (double)pRhs->u.i; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); - if( mem1.rrhs ){ - rc = +1; - } + rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); }else{ i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); i64 rhs = pRhs->u.i; @@ -68620,23 +77573,24 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( /* RHS is real */ else if( pRhs->flags & MEM_Real ){ serial_type = aKey1[idx1]; - if( serial_type>=12 ){ + if( serial_type>=10 ){ + /* Serial types 12 or greater are strings and blobs (greater than + ** numbers). Types 10 and 11 are currently "reserved for future + ** use", so it doesn't really matter what the results of comparing + ** them to numberic values are. */ rc = +1; }else if( serial_type==0 ){ rc = -1; }else{ - double rhs = pRhs->r; - double lhs; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - lhs = mem1.r; + if( mem1.u.ru.r ){ + rc = -1; + }else if( mem1.u.r>pRhs->u.r ){ + rc = +1; + } }else{ - lhs = (double)mem1.u.i; - } - if( lhsrhs ){ - rc = +1; + rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); } } } @@ -68654,14 +77608,16 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; mem1.flags = MEM_Str; mem1.z = (char*)&aKey1[d1]; - rc = vdbeCompareMemString(&mem1, pRhs, pKeyInfo->aColl[i]); + rc = vdbeCompareMemString( + &mem1, pRhs, pKeyInfo->aColl[i], &pPKey2->errCode + ); }else{ int nCmp = MIN(mem1.n, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -68672,6 +77628,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( /* RHS is a blob */ else if( pRhs->flags & MEM_Blob ){ + assert( (pRhs->flags & MEM_Zero)==0 || pRhs->n==0 ); getVarint32(&aKey1[idx1], serial_type); testcase( serial_type==12 ); if( serial_type<12 || (serial_type & 0x01) ){ @@ -68681,8 +77638,14 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( testcase( (d1+nStr)==(unsigned)nKey1 ); testcase( (d1+nStr+1)==(unsigned)nKey1 ); if( (d1+nStr) > (unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ + }else if( pRhs->flags & MEM_Zero ){ + if( !isAllZero((const char*)&aKey1[d1],nStr) ){ + rc = 1; + }else{ + rc = nStr - pRhs->u.nZero; + } }else{ int nCmp = MIN(nStr, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -68701,12 +77664,8 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; } - assert( CORRUPT_DB - || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || pKeyInfo->db->mallocFailed - ); - assert( mem1.zMalloc==0 ); /* See comment below */ + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); + assert( mem1.szMalloc==0 ); /* See comment below */ return rc; } @@ -68719,17 +77678,25 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ - assert( mem1.zMalloc==0 ); + assert( mem1.szMalloc==0 ); /* rc==0 here means that one or both of the keys ran out of fields and - ** all the fields up to that point were equal. Return the the default_rc + ** all the fields up to that point were equal. Return the default_rc ** value. */ assert( CORRUPT_DB - || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) + || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) || pKeyInfo->db->mallocFailed ); + pPKey2->eqSeen = 1; return pPKey2->default_rc; } +SQLITE_PRIVATE int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 0); +} + /* ** This function is an optimized version of sqlite3VdbeRecordCompare() @@ -68742,19 +77709,17 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( */ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2, /* Right key */ - int bSkip /* Ignored */ + UnpackedRecord *pPKey2 /* Right key */ ){ const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; int serial_type = ((const u8*)pKey1)[1]; int res; u32 y; u64 x; - i64 v = pPKey2->aMem[0].u.i; + i64 v; i64 lhs; - UNUSED_PARAMETER(bSkip); - assert( bSkip==0 ); + vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); assert( (*(u8*)pKey1)<=0x3F || CORRUPT_DB ); switch( serial_type ){ case 1: { /* 1-byte signed integer */ @@ -68804,12 +77769,13 @@ static int vdbeRecordCompareInt( ** (as gcc is clever enough to combine the two like cases). Other ** compilers might be similar. */ case 0: case 7: - return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); default: - return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); } + v = pPKey2->aMem[0].u.i; if( v>lhs ){ res = pPKey2->r1; }else if( vnField>1 ){ /* The first fields of the two keys are equal. Compare the trailing ** fields. */ - res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ /* The first fields of the two keys are equal and there are no trailing ** fields. Return pPKey2->default_rc in this case. */ res = pPKey2->default_rc; + pPKey2->eqSeen = 1; } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); return res; } @@ -68840,17 +77803,15 @@ static int vdbeRecordCompareInt( */ static int vdbeRecordCompareString( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2, /* Right key */ - int bSkip + UnpackedRecord *pPKey2 /* Right key */ ){ const u8 *aKey1 = (const u8*)pKey1; int serial_type; int res; - UNUSED_PARAMETER(bSkip); - assert( bSkip==0 ); + assert( pPKey2->aMem[0].flags & MEM_Str ); + vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); getVarint32(&aKey1[1], serial_type); - if( serial_type<12 ){ res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ }else if( !(serial_type & 0x01) ){ @@ -68862,7 +77823,7 @@ static int vdbeRecordCompareString( nStr = (serial_type-12) / 2; if( (szHdr + nStr) > nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } nCmp = MIN( pPKey2->aMem[0].n, nStr ); @@ -68872,9 +77833,10 @@ static int vdbeRecordCompareString( res = nStr - pPKey2->aMem[0].n; if( res==0 ){ if( pPKey2->nField>1 ){ - res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ res = pPKey2->default_rc; + pPKey2->eqSeen = 1; } }else if( res>0 ){ res = pPKey2->r2; @@ -68888,9 +77850,7 @@ static int vdbeRecordCompareString( } } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) || CORRUPT_DB || pPKey2->pKeyInfo->db->mallocFailed ); @@ -68956,20 +77916,17 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ u32 lenRowid; /* Size of the rowid */ Mem m, v; - UNUSED_PARAMETER(db); - /* Get the size of the index entry. Only indices entries of less ** than 2GiB are support - anything large must be database corruption. ** Any corruption is detected in sqlite3BtreeParseCellPtr(), though, so ** this code can safely assume that nCellKey is 32-bits */ assert( sqlite3BtreeCursorIsValid(pCur) ); - VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); - assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ + nCellKey = sqlite3BtreePayloadSize(pCur); assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); /* Read in the complete content of the index entry */ - memset(&m, 0, sizeof(m)); + sqlite3VdbeMemInit(&m, db, 0); rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m); if( rc ){ return rc; @@ -68997,7 +77954,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ if( unlikely(typeRowid<1 || typeRowid>9 || typeRowid==7) ){ goto idx_rowid_corruption; } - lenRowid = sqlite3VdbeSerialTypeLen(typeRowid); + lenRowid = sqlite3SmallTypeSizes[typeRowid]; testcase( (u32)m.n==szHdr+lenRowid ); if( unlikely((u32)m.npCursor; + BtCursor *pCur; Mem m; + assert( pC->eCurType==CURTYPE_BTREE ); + pCur = pC->uc.pCursor; assert( sqlite3BtreeCursorIsValid(pCur) ); - VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); - assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ + nCellKey = sqlite3BtreePayloadSize(pCur); /* nCellKey will always be between 0 and 0xffffffff because of the way ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ if( nCellKey<=0 || nCellKey>0x7fffffff ){ *res = 0; return SQLITE_CORRUPT_BKPT; } - memset(&m, 0, sizeof(m)); - rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (u32)nCellKey, 1, &m); + sqlite3VdbeMemInit(&m, db, 0); + rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m); if( rc ){ return rc; } - *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked, 0); + *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); sqlite3VdbeMemRelease(&m); return SQLITE_OK; } @@ -69144,14 +78103,100 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ ** in memory obtained from sqlite3DbMalloc). */ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ - sqlite3 *db = p->db; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); - sqlite3_free(pVtab->zErrMsg); - pVtab->zErrMsg = 0; + if( pVtab->zErrMsg ){ + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; + } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + +/* +** If the second argument is not NULL, release any allocations associated +** with the memory cells in the p->aMem[] array. Also free the UnpackedRecord +** structure itself, using sqlite3DbFree(). +** +** This function is used to free UnpackedRecord structures allocated by +** the vdbeUnpackRecord() function found in vdbeapi.c. +*/ +static void vdbeFreeUnpacked(sqlite3 *db, UnpackedRecord *p){ + if( p ){ + int i; + for(i=0; inField; i++){ + Mem *pMem = &p->aMem[i]; + if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem); + } + sqlite3DbFree(db, p); + } +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Invoke the pre-update hook. If this is an UPDATE or DELETE pre-update call, +** then cursor passed as the second argument should point to the row about +** to be update or deleted. If the application calls sqlite3_preupdate_old(), +** the required value will be read from the row the cursor points to. +*/ +SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( + Vdbe *v, /* Vdbe pre-update hook is invoked by */ + VdbeCursor *pCsr, /* Cursor to grab old.* values from */ + int op, /* SQLITE_INSERT, UPDATE or DELETE */ + const char *zDb, /* Database name */ + Table *pTab, /* Modified table */ + i64 iKey1, /* Initial key value */ + int iReg /* Register for new.* record */ +){ + sqlite3 *db = v->db; + i64 iKey2; + PreUpdate preupdate; + const char *zTbl = pTab->zName; + static const u8 fakeSortOrder = 0; + + assert( db->pPreUpdate==0 ); + memset(&preupdate, 0, sizeof(PreUpdate)); + if( op==SQLITE_UPDATE ){ + iKey2 = v->aMem[iReg].u.i; + }else{ + iKey2 = iKey1; + } + + assert( pCsr->nField==pTab->nCol + || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1) + ); + + preupdate.v = v; + preupdate.pCsr = pCsr; + preupdate.op = op; + preupdate.iNewReg = iReg; + preupdate.keyinfo.db = db; + preupdate.keyinfo.enc = ENC(db); + preupdate.keyinfo.nField = pTab->nCol; + preupdate.keyinfo.aSortOrder = (u8*)&fakeSortOrder; + preupdate.iKey1 = iKey1; + preupdate.iKey2 = iKey2; + preupdate.pTab = pTab; + + db->pPreUpdate = &preupdate; + db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); + db->pPreUpdate = 0; + sqlite3DbFree(db, preupdate.aRecord); + vdbeFreeUnpacked(db, preupdate.pUnpacked); + vdbeFreeUnpacked(db, preupdate.pNewUnpacked); + if( preupdate.aNew ){ + int i; + for(i=0; inField; i++){ + sqlite3VdbeMemRelease(&preupdate.aNew[i]); + } + sqlite3DbFree(db, preupdate.aNew); + } +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + /************** End of vdbeaux.c *********************************************/ /************** Begin file vdbeapi.c *****************************************/ /* @@ -69169,6 +78214,8 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ ** This file contains code use to implement APIs that are part of the ** VDBE. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_DEPRECATED /* @@ -69207,6 +78254,38 @@ static int vdbeSafetyNotNull(Vdbe *p){ } } +#ifndef SQLITE_OMIT_TRACE +/* +** Invoke the profile callback. This routine is only called if we already +** know that the profile callback is defined and needs to be invoked. +*/ +static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ + sqlite3_int64 iNow; + sqlite3_int64 iElapse; + assert( p->startTime>0 ); + assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 ); + assert( db->init.busy==0 ); + assert( p->zSql!=0 ); + sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); + iElapse = (iNow - p->startTime)*1000000; + if( db->xProfile ){ + db->xProfile(db->pProfileArg, p->zSql, iElapse); + } + if( db->mTrace & SQLITE_TRACE_PROFILE ){ + db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse); + } + p->startTime = 0; +} +/* +** The checkProfileCallback(DB,P) macro checks to see if a profile callback +** is needed, and it invokes the callback if it is needed. +*/ +# define checkProfileCallback(DB,P) \ + if( ((P)->startTime)>0 ){ invokeProfileCallback(DB,P); } +#else +# define checkProfileCallback(DB,P) /*no-op*/ +#endif + /* ** The following routine destroys a virtual machine that is created by ** the sqlite3_compile() routine. The integer returned is an SQLITE_ @@ -69227,6 +78306,7 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ sqlite3 *db = v->db; if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); + checkProfileCallback(db, v); rc = sqlite3VdbeFinalize(v); rc = sqlite3ApiExit(db, rc); sqlite3LeaveMutexAndCloseZombie(db); @@ -69248,12 +78328,14 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ rc = SQLITE_OK; }else{ Vdbe *v = (Vdbe*)pStmt; - sqlite3_mutex_enter(v->db->mutex); + sqlite3 *db = v->db; + sqlite3_mutex_enter(db->mutex); + checkProfileCallback(db, v); rc = sqlite3VdbeReset(v); sqlite3VdbeRewind(v); - assert( (rc & (v->db->errMask))==rc ); - rc = sqlite3ApiExit(v->db, rc); - sqlite3_mutex_leave(v->db->mutex); + assert( (rc & (db->errMask))==rc ); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); } return rc; } @@ -69288,7 +78370,10 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ Mem *p = (Mem*)pVal; if( p->flags & (MEM_Blob|MEM_Str) ){ - sqlite3VdbeMemExpandBlob(p); + if( ExpandBlob(p)!=SQLITE_OK ){ + assert( p->flags==MEM_Null && p->z==0 ); + return 0; + } p->flags |= MEM_Blob; return p->n ? p->z : 0; }else{ @@ -69310,6 +78395,10 @@ SQLITE_API int sqlite3_value_int(sqlite3_value *pVal){ SQLITE_API sqlite_int64 sqlite3_value_int64(sqlite3_value *pVal){ return sqlite3VdbeIntValue((Mem*)pVal); } +SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value *pVal){ + Mem *pMem = (Mem*)pVal; + return ((pMem->flags & MEM_Subtype) ? pMem->eSubtype : 0); +} SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value *pVal){ return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); } @@ -69324,6 +78413,10 @@ SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16LE); } #endif /* SQLITE_OMIT_UTF16 */ +/* EVIDENCE-OF: R-12793-43283 Every value in SQLite has one of five +** fundamental datatypes: 64-bit signed integer 64-bit IEEE floating +** point number string BLOB NULL +*/ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ static const u8 aType[] = { SQLITE_BLOB, /* 0x00 */ @@ -69362,13 +78455,46 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ return aType[pVal->flags&MEM_AffMask]; } +/* Make a copy of an sqlite3_value object +*/ +SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){ + sqlite3_value *pNew; + if( pOrig==0 ) return 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return 0; + memset(pNew, 0, sizeof(*pNew)); + memcpy(pNew, pOrig, MEMCELLSIZE); + pNew->flags &= ~MEM_Dyn; + pNew->db = 0; + if( pNew->flags&(MEM_Str|MEM_Blob) ){ + pNew->flags &= ~(MEM_Static|MEM_Dyn); + pNew->flags |= MEM_Ephem; + if( sqlite3VdbeMemMakeWriteable(pNew)!=SQLITE_OK ){ + sqlite3ValueFree(pNew); + pNew = 0; + } + } + return pNew; +} + +/* Destroy an sqlite3_value object previously obtained from +** sqlite3_value_dup(). +*/ +SQLITE_API void sqlite3_value_free(sqlite3_value *pOld){ + sqlite3ValueFree(pOld); +} + + /**************************** sqlite3_result_ ******************************* ** The following routines are used by user-defined functions to specify ** the function result. ** -** The setStrOrError() funtion calls sqlite3VdbeMemSetStr() to store the +** The setStrOrError() function calls sqlite3VdbeMemSetStr() to store the ** result as a string or blob but if the string or blob is too large, it ** then sets the error code to SQLITE_TOOBIG +** +** The invokeValueDestructor(P,X) routine invokes destructor function X() +** on value P is not going to be used and need to be destroyed. */ static void setResultStrOrError( sqlite3_context *pCtx, /* Function context */ @@ -69377,10 +78503,26 @@ static void setResultStrOrError( u8 enc, /* Encoding of z. 0 for BLOBs */ void (*xDel)(void*) /* Destructor function */ ){ - if( sqlite3VdbeMemSetStr(&pCtx->s, z, n, enc, xDel)==SQLITE_TOOBIG ){ + if( sqlite3VdbeMemSetStr(pCtx->pOut, z, n, enc, xDel)==SQLITE_TOOBIG ){ sqlite3_result_error_toobig(pCtx); } } +static int invokeValueDestructor( + const void *p, /* Value to destroy */ + void (*xDel)(void*), /* The destructor */ + sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if no NULL */ +){ + assert( xDel!=SQLITE_DYNAMIC ); + if( xDel==0 ){ + /* noop */ + }else if( xDel==SQLITE_TRANSIENT ){ + /* noop */ + }else{ + xDel((void*)p); + } + if( pCtx ) sqlite3_result_error_toobig(pCtx); + return SQLITE_TOOBIG; +} SQLITE_API void sqlite3_result_blob( sqlite3_context *pCtx, const void *z, @@ -69388,38 +78530,58 @@ SQLITE_API void sqlite3_result_blob( void (*xDel)(void *) ){ assert( n>=0 ); - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, 0, xDel); } +SQLITE_API void sqlite3_result_blob64( + sqlite3_context *pCtx, + const void *z, + sqlite3_uint64 n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + assert( xDel!=SQLITE_DYNAMIC ); + if( n>0x7fffffff ){ + (void)invokeValueDestructor(z, xDel, pCtx); + }else{ + setResultStrOrError(pCtx, z, (int)n, 0, xDel); + } +} SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetDouble(&pCtx->s, rVal); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); } SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); + sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); + sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); } #endif SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetInt64(&pCtx->s, (i64)iVal); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal); } SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetInt64(&pCtx->s, iVal); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetInt64(pCtx->pOut, iVal); } SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetNull(&pCtx->s); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetNull(pCtx->pOut); +} +SQLITE_API void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ + Mem *pOut = pCtx->pOut; + assert( sqlite3_mutex_held(pOut->db->mutex) ); + pOut->eSubtype = eSubtype & 0xff; + pOut->flags |= MEM_Subtype; } SQLITE_API void sqlite3_result_text( sqlite3_context *pCtx, @@ -69427,9 +78589,25 @@ SQLITE_API void sqlite3_result_text( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } +SQLITE_API void sqlite3_result_text64( + sqlite3_context *pCtx, + const char *z, + sqlite3_uint64 n, + void (*xDel)(void *), + unsigned char enc +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + assert( xDel!=SQLITE_DYNAMIC ); + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( n>0x7fffffff ){ + (void)invokeValueDestructor(z, xDel, pCtx); + }else{ + setResultStrOrError(pCtx, z, (int)n, enc, xDel); + } +} #ifndef SQLITE_OMIT_UTF16 SQLITE_API void sqlite3_result_text16( sqlite3_context *pCtx, @@ -69437,7 +78615,7 @@ SQLITE_API void sqlite3_result_text16( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); } SQLITE_API void sqlite3_result_text16be( @@ -69446,7 +78624,7 @@ SQLITE_API void sqlite3_result_text16be( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); } SQLITE_API void sqlite3_result_text16le( @@ -69455,43 +78633,55 @@ SQLITE_API void sqlite3_result_text16le( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemCopy(&pCtx->s, pValue); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemCopy(pCtx->pOut, pValue); } SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetZeroBlob(&pCtx->s, n); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetZeroBlob(pCtx->pOut, n); +} +SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ + Mem *pOut = pCtx->pOut; + assert( sqlite3_mutex_held(pOut->db->mutex) ); + if( n>(u64)pOut->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + return SQLITE_TOOBIG; + } + sqlite3VdbeMemSetZeroBlob(pCtx->pOut, (int)n); + return SQLITE_OK; } SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ pCtx->isError = errCode; pCtx->fErrorOrAux = 1; - if( pCtx->s.flags & MEM_Null ){ - sqlite3VdbeMemSetStr(&pCtx->s, sqlite3ErrStr(errCode), -1, +#ifdef SQLITE_DEBUG + if( pCtx->pVdbe ) pCtx->pVdbe->rcApp = errCode; +#endif + if( pCtx->pOut->flags & MEM_Null ){ + sqlite3VdbeMemSetStr(pCtx->pOut, sqlite3ErrStr(errCode), -1, SQLITE_UTF8, SQLITE_STATIC); } } /* Force an SQLITE_TOOBIG error. */ SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_TOOBIG; pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(&pCtx->s, "string or blob too big", -1, + sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, SQLITE_UTF8, SQLITE_STATIC); } /* An SQLITE_NOMEM error. */ SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetNull(&pCtx->s); - pCtx->isError = SQLITE_NOMEM; + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetNull(pCtx->pOut); + pCtx->isError = SQLITE_NOMEM_BKPT; pCtx->fErrorOrAux = 1; - pCtx->s.db->mallocFailed = 1; + sqlite3OomFault(pCtx->pOut->db); } /* @@ -69505,9 +78695,12 @@ static int doWalCallbacks(sqlite3 *db){ for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ - int nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + int nEntry; + sqlite3BtreeEnter(pBt); + nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ - rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry); + rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zDbSName, nEntry); } } } @@ -69515,6 +78708,7 @@ static int doWalCallbacks(sqlite3 *db){ return rc; } + /* ** Execute the statement pStmt, either until a row of data is ready, the ** statement is completely executed or an error occurs. @@ -69547,7 +78741,7 @@ static int sqlite3Step(Vdbe *p){ ** or SQLITE_BUSY error. */ #ifdef SQLITE_OMIT_AUTORESET - if( p->rc==SQLITE_BUSY || p->rc==SQLITE_LOCKED ){ + if( (rc = p->rc&0xff)==SQLITE_BUSY || rc==SQLITE_LOCKED ){ sqlite3_reset((sqlite3_stmt*)p); }else{ return SQLITE_MISUSE_BKPT; @@ -69561,7 +78755,7 @@ static int sqlite3Step(Vdbe *p){ db = p->db; if( db->mallocFailed ){ p->rc = SQLITE_NOMEM; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( p->pc<=0 && p->expired ){ @@ -69583,8 +78777,11 @@ static int sqlite3Step(Vdbe *p){ ); #ifndef SQLITE_OMIT_TRACE - if( db->xProfile && !db->init.busy ){ + if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0) + && !db->init.busy && p->zSql ){ sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); + }else{ + assert( p->startTime==0 ); } #endif @@ -69593,6 +78790,9 @@ static int sqlite3Step(Vdbe *p){ if( p->bIsReader ) db->nVdbeRead++; p->pc = 0; } +#ifdef SQLITE_DEBUG + p->rcApp = SQLITE_OK; +#endif #ifndef SQLITE_OMIT_EXPLAIN if( p->explain ){ rc = sqlite3VdbeList(p); @@ -69605,13 +78805,8 @@ static int sqlite3Step(Vdbe *p){ } #ifndef SQLITE_OMIT_TRACE - /* Invoke the profile callback if there is one - */ - if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ - sqlite3_int64 iNow; - sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); - db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); - } + /* If the statement completed successfully, invoke the profile callback */ + if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); #endif if( rc==SQLITE_DONE ){ @@ -69624,7 +78819,7 @@ static int sqlite3Step(Vdbe *p){ db->errCode = rc; if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ - p->rc = SQLITE_NOMEM; + p->rc = SQLITE_NOMEM_BKPT; } end_of_step: /* At this point local variable rc holds the value that should be @@ -69635,9 +78830,9 @@ end_of_step: ** were called on statement p. */ assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR - || rc==SQLITE_BUSY || rc==SQLITE_MISUSE + || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); - assert( p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE ); + assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ /* If this statement was prepared using sqlite3_prepare_v2(), and an ** error has occurred, then return the error code in p->rc to the @@ -69667,10 +78862,12 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ sqlite3_mutex_enter(db->mutex); v->doingRerun = 0; while( (rc = sqlite3Step(v))==SQLITE_SCHEMA - && cnt++ < SQLITE_MAX_SCHEMA_RETRY - && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ + && cnt++ < SQLITE_MAX_SCHEMA_RETRY ){ + int savedPc = v->pc; + rc2 = rc = sqlite3Reprepare(v); + if( rc!=SQLITE_OK) break; sqlite3_reset(pStmt); - v->doingRerun = 1; + if( savedPc>=0 ) v->doingRerun = 1; assert( v->expired==0 ); } if( rc2!=SQLITE_OK ){ @@ -69683,14 +78880,13 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ ** sqlite3_errmsg() and sqlite3_errcode(). */ const char *zErr = (const char *)sqlite3_value_text(db->pErr); - assert( zErr!=0 || db->mallocFailed ); sqlite3DbFree(db, v->zErrMsg); if( !db->mallocFailed ){ v->zErrMsg = sqlite3DbStrDup(db, zErr); v->rc = rc2; } else { v->zErrMsg = 0; - v->rc = rc = SQLITE_NOMEM; + v->rc = rc = SQLITE_NOMEM_BKPT; } } rc = sqlite3ApiExit(db, rc); @@ -69719,21 +78915,31 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ ** application defined function. */ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ - assert( p && p->pFunc ); - return p->s.db; + assert( p && p->pOut ); + return p->pOut->db; } /* -** Return the current time for a statement +** Return the current time for a statement. If the current time +** is requested more than once within the same run of a single prepared +** statement, the exact same time is returned for each invocation regardless +** of the amount of time that elapses between invocations. In other words, +** the time returned is always the time of the first call. */ SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ - Vdbe *v = p->pVdbe; int rc; - if( v->iCurrentTime==0 ){ - rc = sqlite3OsCurrentTimeInt64(p->s.db->pVfs, &v->iCurrentTime); - if( rc ) v->iCurrentTime = 0; +#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3_int64 *piTime = &p->pVdbe->iCurrentTime; + assert( p->pVdbe!=0 ); +#else + sqlite3_int64 iTime = 0; + sqlite3_int64 *piTime = p->pVdbe!=0 ? &p->pVdbe->iCurrentTime : &iTime; +#endif + if( *piTime==0 ){ + rc = sqlite3OsCurrentTimeInt64(p->pOut->db->pVfs, piTime); + if( rc ) *piTime = 0; } - return v->iCurrentTime; + return *piTime; } /* @@ -69759,41 +78965,55 @@ SQLITE_PRIVATE void sqlite3InvalidFunction( } /* -** Allocate or return the aggregate context for a user function. A new -** context is allocated on the first call. Subsequent calls return the -** same context that was returned on prior calls. +** Create a new aggregate context for p and return a pointer to +** its pMem->z element. */ -SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ - Mem *pMem; - assert( p && p->pFunc && p->pFunc->xStep ); - assert( sqlite3_mutex_held(p->s.db->mutex) ); - pMem = p->pMem; - testcase( nByte<0 ); - if( (pMem->flags & MEM_Agg)==0 ){ - if( nByte<=0 ){ - sqlite3VdbeMemReleaseExternal(pMem); - pMem->flags = MEM_Null; - pMem->z = 0; - }else{ - sqlite3VdbeMemGrow(pMem, nByte, 0); - pMem->flags = MEM_Agg; - pMem->u.pDef = p->pFunc; - if( pMem->z ){ - memset(pMem->z, 0, nByte); - } +static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ + Mem *pMem = p->pMem; + assert( (pMem->flags & MEM_Agg)==0 ); + if( nByte<=0 ){ + sqlite3VdbeMemSetNull(pMem); + pMem->z = 0; + }else{ + sqlite3VdbeMemClearAndResize(pMem, nByte); + pMem->flags = MEM_Agg; + pMem->u.pDef = p->pFunc; + if( pMem->z ){ + memset(pMem->z, 0, nByte); } } return (void*)pMem->z; } /* -** Return the auxilary data pointer, if any, for the iArg'th argument to +** Allocate or return the aggregate context for a user function. A new +** context is allocated on the first call. Subsequent calls return the +** same context that was returned on prior calls. +*/ +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ + assert( p && p->pFunc && p->pFunc->xFinalize ); + assert( sqlite3_mutex_held(p->pOut->db->mutex) ); + testcase( nByte<0 ); + if( (p->pMem->flags & MEM_Agg)==0 ){ + return createAggContext(p, nByte); + }else{ + return (void*)p->pMem->z; + } +} + +/* +** Return the auxiliary data pointer, if any, for the iArg'th argument to ** the user-function defined by pCtx. */ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); +#if SQLITE_ENABLE_STAT3_OR_STAT4 + if( pCtx->pVdbe==0 ) return 0; +#else + assert( pCtx->pVdbe!=0 ); +#endif for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; } @@ -69802,7 +79022,7 @@ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ } /* -** Set the auxilary data pointer and delete function, for the iArg'th +** Set the auxiliary data pointer and delete function, for the iArg'th ** argument to the user-function defined by pCtx. Any previous value is ** deleted by calling the delete function specified when it was set. */ @@ -69815,8 +79035,13 @@ SQLITE_API void sqlite3_set_auxdata( AuxData *pAuxData; Vdbe *pVdbe = pCtx->pVdbe; - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); if( iArg<0 ) goto failed; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( pVdbe==0 ) goto failed; +#else + assert( pVdbe!=0 ); +#endif for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; @@ -69848,7 +79073,7 @@ failed: #ifndef SQLITE_OMIT_DEPRECATED /* -** Return the number of times the Step function of a aggregate has been +** Return the number of times the Step function of an aggregate has been ** called. ** ** This function is deprecated. Do not use it for new code. It is @@ -69857,7 +79082,7 @@ failed: ** context. */ SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ - assert( p && p->pMem && p->pFunc && p->pFunc->xStep ); + assert( p && p->pMem && p->pFunc && p->pFunc->xFinalize ); return p->pMem->n; } #endif @@ -69897,11 +79122,23 @@ static const Mem *columnNullValue(void){ #if defined(SQLITE_DEBUG) && defined(__GNUC__) __attribute__((aligned(8))) #endif - = {0, "", (double)0, {0}, 0, MEM_Null, 0, + = { + /* .u = */ {0}, + /* .flags = */ (u16)MEM_Null, + /* .enc = */ (u8)0, + /* .eSubtype = */ (u8)0, + /* .n = */ (int)0, + /* .z = */ (char*)0, + /* .zMalloc = */ (char*)0, + /* .szMalloc = */ (int)0, + /* .uTemp = */ (u32)0, + /* .db = */ (sqlite3*)0, + /* .xDel = */ (void(*)(void*))0, #ifdef SQLITE_DEBUG - 0, 0, /* pScopyFrom, pFiller */ + /* .pScopyFrom = */ (Mem*)0, + /* .pFiller = */ (void*)0, #endif - 0, 0 }; + }; return &nullMem; } @@ -69916,14 +79153,13 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ Mem *pOut; pVm = (Vdbe *)pStmt; - if( pVm && pVm->pResultSet!=0 && inResColumn && i>=0 ){ - sqlite3_mutex_enter(pVm->db->mutex); + if( pVm==0 ) return (Mem*)columnNullValue(); + assert( pVm->db ); + sqlite3_mutex_enter(pVm->db->mutex); + if( pVm->pResultSet!=0 && inResColumn && i>=0 ){ pOut = &pVm->pResultSet[i]; }else{ - if( pVm && ALWAYS(pVm->db) ){ - sqlite3_mutex_enter(pVm->db->mutex); - sqlite3Error(pVm->db, SQLITE_RANGE, 0); - } + sqlite3Error(pVm->db, SQLITE_RANGE); pOut = (Mem*)columnNullValue(); } return pOut; @@ -69956,6 +79192,8 @@ static void columnMallocFailure(sqlite3_stmt *pStmt) */ Vdbe *p = (Vdbe *)pStmt; if( p ){ + assert( p->db!=0 ); + assert( sqlite3_mutex_held(p->db->mutex) ); p->rc = sqlite3ApiExit(p->db, p->rc); sqlite3_mutex_leave(p->db->mutex); } @@ -70049,11 +79287,19 @@ static const void *columnName( const void *(*xFunc)(Mem*), int useType ){ - const void *ret = 0; - Vdbe *p = (Vdbe *)pStmt; + const void *ret; + Vdbe *p; int n; - sqlite3 *db = p->db; - + sqlite3 *db; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + ret = 0; + p = (Vdbe *)pStmt; + db = p->db; assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N=0 ){ @@ -70065,7 +79311,7 @@ static const void *columnName( ** is the case, clear the mallocFailed flag and return NULL. */ if( db->mallocFailed ){ - db->mallocFailed = 0; + sqlite3OomClear(db); ret = 0; } sqlite3_mutex_leave(db->mutex); @@ -70118,7 +79364,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ /* ** Return the name of the database from which a result column derives. ** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ return columnName( @@ -70134,7 +79380,7 @@ SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N /* ** Return the name of the table from which a result column derives. ** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ return columnName( @@ -70150,7 +79396,7 @@ SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ /* ** Return the name of the table column from which a result column derives. ** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ return columnName( @@ -70187,14 +79433,14 @@ static int vdbeUnbind(Vdbe *p, int i){ } sqlite3_mutex_enter(p->db->mutex); if( p->magic!=VDBE_MAGIC_RUN || p->pc>=0 ){ - sqlite3Error(p->db, SQLITE_MISUSE, 0); + sqlite3Error(p->db, SQLITE_MISUSE); sqlite3_mutex_leave(p->db->mutex); sqlite3_log(SQLITE_MISUSE, "bind on a busy prepared statement: [%s]", p->zSql); return SQLITE_MISUSE_BKPT; } if( i<1 || i>p->nVar ){ - sqlite3Error(p->db, SQLITE_RANGE, 0); + sqlite3Error(p->db, SQLITE_RANGE); sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; } @@ -70202,7 +79448,7 @@ static int vdbeUnbind(Vdbe *p, int i){ pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; - sqlite3Error(p->db, SQLITE_OK, 0); + sqlite3Error(p->db, SQLITE_OK); /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. @@ -70244,7 +79490,7 @@ static int bindText( if( rc==SQLITE_OK && encoding!=0 ){ rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); } - sqlite3Error(p->db, rc, 0); + sqlite3Error(p->db, rc); rc = sqlite3ApiExit(p->db, rc); } sqlite3_mutex_leave(p->db->mutex); @@ -70265,8 +79511,25 @@ SQLITE_API int sqlite3_bind_blob( int nData, void (*xDel)(void*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( nData<0 ) return SQLITE_MISUSE_BKPT; +#endif return bindText(pStmt, i, zData, nData, xDel, 0); } +SQLITE_API int sqlite3_bind_blob64( + sqlite3_stmt *pStmt, + int i, + const void *zData, + sqlite3_uint64 nData, + void (*xDel)(void*) +){ + assert( xDel!=SQLITE_DYNAMIC ); + if( nData>0x7fffffff ){ + return invokeValueDestructor(zData, xDel, 0); + }else{ + return bindText(pStmt, i, zData, (int)nData, xDel, 0); + } +} SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; @@ -70308,6 +79571,22 @@ SQLITE_API int sqlite3_bind_text( ){ return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); } +SQLITE_API int sqlite3_bind_text64( + sqlite3_stmt *pStmt, + int i, + const char *zData, + sqlite3_uint64 nData, + void (*xDel)(void*), + unsigned char enc +){ + assert( xDel!=SQLITE_DYNAMIC ); + if( nData>0x7fffffff ){ + return invokeValueDestructor(zData, xDel, 0); + }else{ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + return bindText(pStmt, i, zData, (int)nData, xDel, enc); + } +} #ifndef SQLITE_OMIT_UTF16 SQLITE_API int sqlite3_bind_text16( sqlite3_stmt *pStmt, @@ -70327,7 +79606,7 @@ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_valu break; } case SQLITE_FLOAT: { - rc = sqlite3_bind_double(pStmt, i, pValue->r); + rc = sqlite3_bind_double(pStmt, i, pValue->u.r); break; } case SQLITE_BLOB: { @@ -70360,6 +79639,20 @@ SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ } return rc; } +SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + sqlite3_mutex_enter(p->db->mutex); + if( n>(u64)p->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + rc = SQLITE_TOOBIG; + }else{ + assert( (n & 0x7FFFFFFF)==n ); + rc = sqlite3_bind_zeroblob(pStmt, i, n); + } + rc = sqlite3ApiExit(p->db, rc); + sqlite3_mutex_leave(p->db->mutex); + return rc; +} /* ** Return the number of wildcards that can be potentially bound to. @@ -70430,7 +79723,7 @@ SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt ** Deprecated external interface. Internal/core SQLite code ** should call sqlite3TransferBindings. ** -** Is is misuse to call this routine with statements from different +** It is misuse to call this routine with statements from different ** database connections. But as this is a deprecated interface, we ** will not bother to check for that condition. ** @@ -70477,7 +79770,7 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ */ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; - return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN; + return v!=0 && v->magic==VDBE_MAGIC_RUN && v->pc>=0; } /* @@ -70488,6 +79781,12 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ */ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ sqlite3_stmt *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(pDb->mutex); if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; @@ -70503,11 +79802,305 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ */ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ Vdbe *pVdbe = (Vdbe*)pStmt; - u32 v = pVdbe->aCounter[op]; + u32 v; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + v = pVdbe->aCounter[op]; if( resetFlag ) pVdbe->aCounter[op] = 0; return (int)v; } +/* +** Return the SQL associated with a prepared statement +*/ +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + return p ? p->zSql : 0; +} + +/* +** Return the SQL associated with a prepared statement with +** bound parameters expanded. Space to hold the returned string is +** obtained from sqlite3_malloc(). The caller is responsible for +** freeing the returned string by passing it to sqlite3_free(). +** +** The SQLITE_TRACE_SIZE_LIMIT puts an upper bound on the size of +** expanded bound parameters. +*/ +SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){ +#ifdef SQLITE_OMIT_TRACE + return 0; +#else + char *z = 0; + const char *zSql = sqlite3_sql(pStmt); + if( zSql ){ + Vdbe *p = (Vdbe *)pStmt; + sqlite3_mutex_enter(p->db->mutex); + z = sqlite3VdbeExpandSql(p, zSql); + sqlite3_mutex_leave(p->db->mutex); + } + return z; +#endif +} + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Allocate and populate an UnpackedRecord structure based on the serialized +** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure +** if successful, or a NULL pointer if an OOM error is encountered. +*/ +static UnpackedRecord *vdbeUnpackRecord( + KeyInfo *pKeyInfo, + int nKey, + const void *pKey +){ + char *dummy; /* Dummy argument for AllocUnpackedRecord() */ + UnpackedRecord *pRet; /* Return value */ + + pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo, 0, 0, &dummy); + if( pRet ){ + memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nField+1)); + sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet); + } + return pRet; +} + +/* +** This function is called from within a pre-update callback to retrieve +** a field of the row currently being updated or deleted. +*/ +SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ + PreUpdate *p = db->pPreUpdate; + int rc = SQLITE_OK; + + /* Test that this call is being made from within an SQLITE_DELETE or + ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */ + if( !p || p->op==SQLITE_INSERT ){ + rc = SQLITE_MISUSE_BKPT; + goto preupdate_old_out; + } + if( iIdx>=p->pCsr->nField || iIdx<0 ){ + rc = SQLITE_RANGE; + goto preupdate_old_out; + } + + /* If the old.* record has not yet been loaded into memory, do so now. */ + if( p->pUnpacked==0 ){ + u32 nRec; + u8 *aRec; + + nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor); + aRec = sqlite3DbMallocRaw(db, nRec); + if( !aRec ) goto preupdate_old_out; + rc = sqlite3BtreeData(p->pCsr->uc.pCursor, 0, nRec, aRec); + if( rc==SQLITE_OK ){ + p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec); + if( !p->pUnpacked ) rc = SQLITE_NOMEM; + } + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, aRec); + goto preupdate_old_out; + } + p->aRecord = aRec; + } + + if( iIdx>=p->pUnpacked->nField ){ + *ppValue = (sqlite3_value *)columnNullValue(); + }else{ + Mem *pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; + *ppValue = &p->pUnpacked->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey1); + }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ + if( pMem->flags & MEM_Int ){ + sqlite3VdbeMemRealify(pMem); + } + } + } + + preupdate_old_out: + sqlite3Error(db, rc); + return sqlite3ApiExit(db, rc); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** This function is called from within a pre-update callback to retrieve +** the number of columns in the row being updated, deleted or inserted. +*/ +SQLITE_API int sqlite3_preupdate_count(sqlite3 *db){ + PreUpdate *p = db->pPreUpdate; + return (p ? p->keyinfo.nField : 0); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** This function is designed to be called from within a pre-update callback +** only. It returns zero if the change that caused the callback was made +** immediately by a user SQL statement. Or, if the change was made by a +** trigger program, it returns the number of trigger programs currently +** on the stack (1 for a top-level trigger, 2 for a trigger fired by a +** top-level trigger etc.). +** +** For the purposes of the previous paragraph, a foreign key CASCADE, SET NULL +** or SET DEFAULT action is considered a trigger. +*/ +SQLITE_API int sqlite3_preupdate_depth(sqlite3 *db){ + PreUpdate *p = db->pPreUpdate; + return (p ? p->v->nFrame : 0); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** This function is called from within a pre-update callback to retrieve +** a field of the row currently being updated or inserted. +*/ +SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ + PreUpdate *p = db->pPreUpdate; + int rc = SQLITE_OK; + Mem *pMem; + + if( !p || p->op==SQLITE_DELETE ){ + rc = SQLITE_MISUSE_BKPT; + goto preupdate_new_out; + } + if( iIdx>=p->pCsr->nField || iIdx<0 ){ + rc = SQLITE_RANGE; + goto preupdate_new_out; + } + + if( p->op==SQLITE_INSERT ){ + /* For an INSERT, memory cell p->iNewReg contains the serialized record + ** that is being inserted. Deserialize it. */ + UnpackedRecord *pUnpack = p->pNewUnpacked; + if( !pUnpack ){ + Mem *pData = &p->v->aMem[p->iNewReg]; + rc = ExpandBlob(pData); + if( rc!=SQLITE_OK ) goto preupdate_new_out; + pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z); + if( !pUnpack ){ + rc = SQLITE_NOMEM; + goto preupdate_new_out; + } + p->pNewUnpacked = pUnpack; + } + if( iIdx>=pUnpack->nField ){ + pMem = (sqlite3_value *)columnNullValue(); + }else{ + pMem = &pUnpack->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey2); + } + } + }else{ + /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required + ** value. Make a copy of the cell contents and return a pointer to it. + ** It is not safe to return a pointer to the memory cell itself as the + ** caller may modify the value text encoding. + */ + assert( p->op==SQLITE_UPDATE ); + if( !p->aNew ){ + p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem) * p->pCsr->nField); + if( !p->aNew ){ + rc = SQLITE_NOMEM; + goto preupdate_new_out; + } + } + assert( iIdx>=0 && iIdxpCsr->nField ); + pMem = &p->aNew[iIdx]; + if( pMem->flags==0 ){ + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey2); + }else{ + rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iIdx]); + if( rc!=SQLITE_OK ) goto preupdate_new_out; + } + } + } + *ppValue = pMem; + + preupdate_new_out: + sqlite3Error(db, rc); + return sqlite3ApiExit(db, rc); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Return status data for a single loop within query pStmt. +*/ +SQLITE_API int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement being queried */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ +){ + Vdbe *p = (Vdbe*)pStmt; + ScanStatus *pScan; + if( idx<0 || idx>=p->nScan ) return 1; + pScan = &p->aScan[idx]; + switch( iScanStatusOp ){ + case SQLITE_SCANSTAT_NLOOP: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + break; + } + case SQLITE_SCANSTAT_NVISIT: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + break; + } + case SQLITE_SCANSTAT_EST: { + double r = 1.0; + LogEst x = pScan->nEst; + while( x<100 ){ + x += 10; + r *= 0.5; + } + *(double*)pOut = r*sqlite3LogEstToInt(x); + break; + } + case SQLITE_SCANSTAT_NAME: { + *(const char**)pOut = pScan->zName; + break; + } + case SQLITE_SCANSTAT_EXPLAIN: { + if( pScan->addrExplain ){ + *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + }else{ + *(const char**)pOut = 0; + } + break; + } + case SQLITE_SCANSTAT_SELECTID: { + if( pScan->addrExplain ){ + *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + }else{ + *(int*)pOut = -1; + } + break; + } + default: { + return 1; + } + } + return 0; +} + +/* +** Zero all counters associated with the sqlite3_stmt_scanstatus() data. +*/ +SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + memset(p->anExec, 0, p->nOp * sizeof(i64)); +} +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ + /************** End of vdbeapi.c *********************************************/ /************** Begin file vdbetrace.c ***************************************/ /* @@ -70527,6 +80120,8 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ ** ** The Vdbe parse-tree explainer is also found here. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_TRACE @@ -70574,7 +80169,7 @@ static int findNextHostParameter(const char *zSql, int *pnToken){ ** ALGORITHM: Scan the input string looking for host parameters in any of ** these forms: ?, ?N, $A, @A, :A. Take care to avoid text within ** string literals, quoted identifier names, and comments. For text forms, -** the host parameter index is found by scanning the perpared +** the host parameter index is found by scanning the prepared ** statement for the corresponding OP_Variable opcode. Once the host ** parameter index is known, locate the value in p->aVar[]. Then render ** the value as a literal in place of the host parameter name. @@ -70591,12 +80186,14 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( int i; /* Loop counter */ Mem *pVar; /* Value of a host parameter */ StrAccum out; /* Accumulate the output here */ +#ifndef SQLITE_OMIT_UTF16 + Mem utf8; /* Used to convert UTF16 parameters into UTF8 for display */ +#endif char zBase[100]; /* Initial working space */ db = p->db; - sqlite3StrAccumInit(&out, zBase, sizeof(zBase), + sqlite3StrAccumInit(&out, 0, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); - out.db = db; if( db->nVdbeExec>1 ){ while( *zRawSql ){ const char *zStart = zRawSql; @@ -70605,6 +80202,8 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( assert( (zRawSql - zStart) > 0 ); sqlite3StrAccumAppend(&out, zStart, (int)(zRawSql-zStart)); } + }else if( p->nVar==0 ){ + sqlite3StrAccumAppend(&out, zRawSql, sqlite3Strlen30(zRawSql)); }else{ while( zRawSql[0] ){ n = findNextHostParameter(zRawSql, &nToken); @@ -70621,10 +80220,12 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( idx = nextIndex; } }else{ - assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' ); + assert( zRawSql[0]==':' || zRawSql[0]=='$' || + zRawSql[0]=='@' || zRawSql[0]=='#' ); testcase( zRawSql[0]==':' ); testcase( zRawSql[0]=='$' ); testcase( zRawSql[0]=='@' ); + testcase( zRawSql[0]=='#' ); idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken); assert( idx>0 ); } @@ -70635,19 +80236,21 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( if( pVar->flags & MEM_Null ){ sqlite3StrAccumAppend(&out, "NULL", 4); }else if( pVar->flags & MEM_Int ){ - sqlite3XPrintf(&out, 0, "%lld", pVar->u.i); + sqlite3XPrintf(&out, "%lld", pVar->u.i); }else if( pVar->flags & MEM_Real ){ - sqlite3XPrintf(&out, 0, "%!.15g", pVar->r); + sqlite3XPrintf(&out, "%!.15g", pVar->u.r); }else if( pVar->flags & MEM_Str ){ int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); - Mem utf8; if( enc!=SQLITE_UTF8 ){ memset(&utf8, 0, sizeof(utf8)); utf8.db = db; sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); - sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); + if( SQLITE_NOMEM==sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8) ){ + out.accError = STRACCUM_NOMEM; + out.nAlloc = 0; + } pVar = &utf8; } #endif @@ -70658,17 +80261,17 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( while( nOutn && (pVar->z[nOut]&0xc0)==0x80 ){ nOut++; } } #endif - sqlite3XPrintf(&out, 0, "'%.*q'", nOut, pVar->z); + sqlite3XPrintf(&out, "'%.*q'", nOut, pVar->z); #ifdef SQLITE_TRACE_SIZE_LIMIT if( nOutn ){ - sqlite3XPrintf(&out, 0, "/*+%d bytes*/", pVar->n-nOut); + sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); } #endif #ifndef SQLITE_OMIT_UTF16 if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); #endif }else if( pVar->flags & MEM_Zero ){ - sqlite3XPrintf(&out, 0, "zeroblob(%d)", pVar->u.nZero); + sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ int nOut; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); @@ -70678,137 +80281,23 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; #endif for(i=0; iz[i]&0xff); + sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); #ifdef SQLITE_TRACE_SIZE_LIMIT if( nOutn ){ - sqlite3XPrintf(&out, 0, "/*+%d bytes*/", pVar->n-nOut); + sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); } #endif } } } + if( out.accError ) sqlite3StrAccumReset(&out); return sqlite3StrAccumFinish(&out); } #endif /* #ifndef SQLITE_OMIT_TRACE */ -/***************************************************************************** -** The following code implements the data-structure explaining logic -** for the Vdbe. -*/ - -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - -/* -** Allocate a new Explain object -*/ -SQLITE_PRIVATE void sqlite3ExplainBegin(Vdbe *pVdbe){ - if( pVdbe ){ - Explain *p; - sqlite3BeginBenignMalloc(); - p = (Explain *)sqlite3MallocZero( sizeof(Explain) ); - if( p ){ - p->pVdbe = pVdbe; - sqlite3_free(pVdbe->pExplain); - pVdbe->pExplain = p; - sqlite3StrAccumInit(&p->str, p->zBase, sizeof(p->zBase), - SQLITE_MAX_LENGTH); - p->str.useMalloc = 2; - }else{ - sqlite3EndBenignMalloc(); - } - } -} - -/* -** Return true if the Explain ends with a new-line. -*/ -static int endsWithNL(Explain *p){ - return p && p->str.zText && p->str.nChar - && p->str.zText[p->str.nChar-1]=='\n'; -} - -/* -** Append text to the indentation -*/ -SQLITE_PRIVATE void sqlite3ExplainPrintf(Vdbe *pVdbe, const char *zFormat, ...){ - Explain *p; - if( pVdbe && (p = pVdbe->pExplain)!=0 ){ - va_list ap; - if( p->nIndent && endsWithNL(p) ){ - int n = p->nIndent; - if( n>ArraySize(p->aIndent) ) n = ArraySize(p->aIndent); - sqlite3AppendSpace(&p->str, p->aIndent[n-1]); - } - va_start(ap, zFormat); - sqlite3VXPrintf(&p->str, SQLITE_PRINTF_INTERNAL, zFormat, ap); - va_end(ap); - } -} - -/* -** Append a '\n' if there is not already one. -*/ -SQLITE_PRIVATE void sqlite3ExplainNL(Vdbe *pVdbe){ - Explain *p; - if( pVdbe && (p = pVdbe->pExplain)!=0 && !endsWithNL(p) ){ - sqlite3StrAccumAppend(&p->str, "\n", 1); - } -} - -/* -** Push a new indentation level. Subsequent lines will be indented -** so that they begin at the current cursor position. -*/ -SQLITE_PRIVATE void sqlite3ExplainPush(Vdbe *pVdbe){ - Explain *p; - if( pVdbe && (p = pVdbe->pExplain)!=0 ){ - if( p->str.zText && p->nIndentaIndent) ){ - const char *z = p->str.zText; - int i = p->str.nChar-1; - int x; - while( i>=0 && z[i]!='\n' ){ i--; } - x = (p->str.nChar - 1) - i; - if( p->nIndent && xaIndent[p->nIndent-1] ){ - x = p->aIndent[p->nIndent-1]; - } - p->aIndent[p->nIndent] = x; - } - p->nIndent++; - } -} - -/* -** Pop the indentation stack by one level. -*/ -SQLITE_PRIVATE void sqlite3ExplainPop(Vdbe *p){ - if( p && p->pExplain ) p->pExplain->nIndent--; -} - -/* -** Free the indentation structure -*/ -SQLITE_PRIVATE void sqlite3ExplainFinish(Vdbe *pVdbe){ - if( pVdbe && pVdbe->pExplain ){ - sqlite3_free(pVdbe->zExplain); - sqlite3ExplainNL(pVdbe); - pVdbe->zExplain = sqlite3StrAccumFinish(&pVdbe->pExplain->str); - sqlite3_free(pVdbe->pExplain); - pVdbe->pExplain = 0; - sqlite3EndBenignMalloc(); - } -} - -/* -** Return the explanation of a virtual machine. -*/ -SQLITE_PRIVATE const char *sqlite3VdbeExplanation(Vdbe *pVdbe){ - return (pVdbe && pVdbe->zExplain) ? pVdbe->zExplain : 0; -} -#endif /* defined(SQLITE_DEBUG) */ - /************** End of vdbetrace.c *******************************************/ /************** Begin file vdbe.c ********************************************/ /* @@ -70831,6 +80320,8 @@ SQLITE_PRIVATE const char *sqlite3VdbeExplanation(Vdbe *pVdbe){ ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ /* ** Invoke this macro on memory cells just prior to changing the @@ -70897,6 +80388,16 @@ static void updateMaxBlobsize(Mem *p){ } #endif +/* +** This macro evaluates to true if either the update hook or the preupdate +** hook are enabled for database connect DB. +*/ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +# define HAS_UPDATE_HOOK(DB) ((DB)->xPreUpdateCallback||(DB)->xUpdateCallback) +#else +# define HAS_UPDATE_HOOK(DB) ((DB)->xUpdateCallback) +#endif + /* ** The next global variable is incremented each time the OP_Found opcode ** is executed. This is used to test whether or not the foreign key @@ -70957,7 +80458,7 @@ SQLITE_API int sqlite3_found_count = 0; ** already. Return non-zero if a malloc() fails. */ #define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc)) \ + if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ { goto no_mem; } /* @@ -70976,7 +80477,7 @@ SQLITE_API int sqlite3_found_count = 0; && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ -#define isSorter(x) ((x)->pSorter!=0) +#define isSorter(x) ((x)->eCurType==CURTYPE_SORTER) /* ** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL @@ -70987,7 +80488,7 @@ static VdbeCursor *allocateCursor( int iCur, /* Index of the new VdbeCursor */ int nField, /* Number of fields in the table or index */ int iDb, /* Database the cursor belongs to, or -1 */ - int isBtreeCursor /* True for B-Tree. False for pseudo-table or vtab */ + u8 eCurType /* Type of the new cursor */ ){ /* Find the memory cell that will be used to store the blob of memory ** required for this VdbeCursor structure. It is convenient to use a @@ -71003,32 +80504,34 @@ static VdbeCursor *allocateCursor( ** be freed lazily via the sqlite3_release_memory() API. This ** minimizes the number of malloc calls made by the system. ** - ** Memory cells for cursors are allocated at the top of the address - ** space. Memory cell (p->nMem) corresponds to cursor 0. Space for - ** cursor 1 is managed by memory cell (p->nMem-1), etc. + ** The memory cell for cursor 0 is aMem[0]. The rest are allocated from + ** the top of the register space. Cursor 1 is at Mem[p->nMem-1]. + ** Cursor 2 is at Mem[p->nMem-2]. And so forth. */ - Mem *pMem = &p->aMem[p->nMem-iCur]; + Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem; int nByte; VdbeCursor *pCx = 0; nByte = ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + - (isBtreeCursor?sqlite3BtreeCursorSize():0); + (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); - assert( iCurnCursor ); - if( p->apCsr[iCur] ){ + assert( iCur>=0 && iCurnCursor ); + if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } - if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + if( SQLITE_OK==sqlite3VdbeMemClearAndResize(pMem, nByte) ){ p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; memset(pCx, 0, sizeof(VdbeCursor)); + pCx->eCurType = eCurType; pCx->iDb = iDb; pCx->nField = nField; - if( isBtreeCursor ){ - pCx->pCursor = (BtCursor*) + pCx->aOffset = &pCx->aType[nField]; + if( eCurType==CURTYPE_BTREE ){ + pCx->uc.pCursor = (BtCursor*) &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; - sqlite3BtreeCursorZero(pCx->pCursor); + sqlite3BtreeCursorZero(pCx->uc.pCursor); } } return pCx; @@ -71039,23 +80542,31 @@ static VdbeCursor *allocateCursor( ** do so without loss of information. In other words, if the string ** looks like a number, convert it into a number. If it does not ** look like a number, leave it alone. +** +** If the bTryForInt flag is true, then extra effort is made to give +** an integer representation. Strings that look like floating point +** values but which have no fractional component (example: '48.00') +** will have a MEM_Int representation when bTryForInt is true. +** +** If bTryForInt is false, then if the input string contains a decimal +** point or exponential notation, the result is only MEM_Real, even +** if there is an exact integer representation of the quantity. */ -static void applyNumericAffinity(Mem *pRec){ +static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; i64 iValue; u8 enc = pRec->enc; - if( (pRec->flags&MEM_Str)==0 ) return; + assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real))==MEM_Str ); if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ pRec->u.i = iValue; pRec->flags |= MEM_Int; }else{ - pRec->r = rValue; + pRec->u.r = rValue; pRec->flags |= MEM_Real; + if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec); } } -#define ApplyNumericAffinity(X) \ - if(((X)->flags&(MEM_Real|MEM_Int))==0){applyNumericAffinity(X);} /* ** Processing is determine by the affinity parameter: @@ -71072,7 +80583,7 @@ static void applyNumericAffinity(Mem *pRec){ ** SQLITE_AFF_TEXT: ** Convert pRec to a text representation. ** -** SQLITE_AFF_NONE: +** SQLITE_AFF_BLOB: ** No-op. pRec is unchanged. */ static void applyAffinity( @@ -71080,22 +80591,28 @@ static void applyAffinity( char affinity, /* The affinity to be applied */ u8 enc /* Use this text encoding */ ){ - if( affinity==SQLITE_AFF_TEXT ){ - /* Only attempt the conversion to TEXT if there is an integer or real - ** representation (blob and NULL do not get converted) but no string - ** representation. - */ - if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ - sqlite3VdbeMemStringify(pRec, enc); - } - pRec->flags &= ~(MEM_Real|MEM_Int); - }else if( affinity!=SQLITE_AFF_NONE ){ + if( affinity>=SQLITE_AFF_NUMERIC ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL || affinity==SQLITE_AFF_NUMERIC ); - ApplyNumericAffinity(pRec); - if( pRec->flags & MEM_Real ){ - sqlite3VdbeIntegerAffinity(pRec); + if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (pRec->flags & MEM_Real)==0 ){ + if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); + }else{ + sqlite3VdbeIntegerAffinity(pRec); + } } + }else if( affinity==SQLITE_AFF_TEXT ){ + /* Only attempt the conversion to TEXT if there is an integer or real + ** representation (blob and NULL do not get converted) but no string + ** representation. It would be harmless to repeat the conversion if + ** there is already a string rep, but it is pointless to waste those + ** CPU cycles. */ + if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ + if( (pRec->flags&(MEM_Real|MEM_Int)) ){ + sqlite3VdbeMemStringify(pRec, enc, 1); + } + } + pRec->flags &= ~(MEM_Real|MEM_Int); } } @@ -71109,7 +80626,7 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; - applyNumericAffinity(pMem); + applyNumericAffinity(pMem, 0); eType = sqlite3_value_type(pVal); } return eType; @@ -71127,25 +80644,37 @@ SQLITE_PRIVATE void sqlite3ValueApplyAffinity( applyAffinity((Mem *)pVal, affinity, enc); } +/* +** pMem currently only holds a string type (or maybe a BLOB that we can +** interpret as a string if we want to). Compute its corresponding +** numeric type, if has one. Set the pMem->u.r and pMem->u.i fields +** accordingly. +*/ +static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ + assert( (pMem->flags & (MEM_Int|MEM_Real))==0 ); + assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); + if( sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc)==0 ){ + return 0; + } + if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ + return MEM_Int; + } + return MEM_Real; +} + /* ** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or ** none. ** ** Unlike applyNumericAffinity(), this routine does not modify pMem->flags. -** But it does set pMem->r and pMem->u.i appropriately. +** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ if( pMem->flags & (MEM_Int|MEM_Real) ){ return pMem->flags & (MEM_Int|MEM_Real); } if( pMem->flags & (MEM_Str|MEM_Blob) ){ - if( sqlite3AtoF(pMem->z, &pMem->r, pMem->n, pMem->enc)==0 ){ - return 0; - } - if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ - return MEM_Int; - } - return MEM_Real; + return computeNumericType(pMem); } return 0; } @@ -71248,7 +80777,7 @@ static void memTracePrint(Mem *p){ printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT }else if( p->flags & MEM_Real ){ - printf(" r:%g", p->r); + printf(" r:%g", p->u.r); #endif }else if( p->flags & MEM_RowSet ){ printf(" (rowset)"); @@ -71257,6 +80786,7 @@ static void memTracePrint(Mem *p){ sqlite3VdbeMemPrettyPrint(p, zBuf); printf(" %s", zBuf); } + if( p->flags & MEM_Subtype ) printf(" subtype=0x%02x", p->eSubtype); } static void registerTrace(int iReg, Mem *p){ printf("REG[%d] = ", iReg); @@ -71295,8 +80825,8 @@ static void registerTrace(int iReg, Mem *p){ ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H /* ** The following routine only works on pentium-class (or newer) processors. @@ -71364,7 +80894,7 @@ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } #endif -#endif /* !defined(_HWTIME_H_) */ +#endif /* !defined(SQLITE_HWTIME_H) */ /************** End of hwtime.h **********************************************/ /************** Continuing where we left off in vdbe.c ***********************/ @@ -71391,6 +80921,29 @@ static int checkSavepointCount(sqlite3 *db){ } #endif +/* +** Return the register of pOp->p2 after first preparing it to be +** overwritten with an integer value. +*/ +static SQLITE_NOINLINE Mem *out2PrereleaseWithClear(Mem *pOut){ + sqlite3VdbeMemSetNull(pOut); + pOut->flags = MEM_Int; + return pOut; +} +static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ + Mem *pOut; + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); + pOut = &p->aMem[pOp->p2]; + memAboutToChange(p, pOut); + if( VdbeMemDynamic(pOut) ){ /*OPTIMIZATION-IF-FALSE*/ + return out2PrereleaseWithClear(pOut); + }else{ + pOut->flags = MEM_Int; + return pOut; + } +} + /* ** Execute as much of a VDBE program as we can. @@ -71399,14 +80952,19 @@ static int checkSavepointCount(sqlite3 *db){ SQLITE_PRIVATE int sqlite3VdbeExec( Vdbe *p /* The VDBE */ ){ - int pc=0; /* The program counter */ Op *aOp = p->aOp; /* Copy of p->aOp */ - Op *pOp; /* Current operation */ + Op *pOp = aOp; /* Current operation */ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + Op *pOrigOp; /* Value of pOp at the top of the loop */ +#endif +#ifdef SQLITE_DEBUG + int nExtraDelete = 0; /* Verifies FORDELETE and AUXDELETE flags */ +#endif int rc = SQLITE_OK; /* Value to return */ sqlite3 *db = p->db; /* The database */ u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */ u8 encoding = ENC(db); /* The database encoding */ - int iCompare = 0; /* Result of last OP_Compare operation */ + int iCompare = 0; /* Result of last comparison */ unsigned nVmStep = 0; /* Number of virtual machine steps */ #ifndef SQLITE_OMIT_PROGRESS_CALLBACK unsigned nProgressLimit = 0;/* Invoke xProgress() when nVmStep reaches this */ @@ -71430,7 +80988,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** sqlite3_column_text16() failed. */ goto no_mem; } - assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY ); + assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); assert( p->bIsReader || p->readOnly!=0 ); p->rc = SQLITE_OK; p->iCurrentTime = 0; @@ -71441,13 +80999,9 @@ SQLITE_PRIVATE int sqlite3VdbeExec( sqlite3VdbeIOTraceSql(p); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( db->xProgress ){ + u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; assert( 0 < db->nProgressOps ); - nProgressLimit = (unsigned)p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; - if( nProgressLimit==0 ){ - nProgressLimit = db->nProgressOps; - }else{ - nProgressLimit %= (unsigned)db->nProgressOps; - } + nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); } #endif #ifdef SQLITE_DEBUG @@ -71477,20 +81031,25 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } sqlite3EndBenignMalloc(); #endif - for(pc=p->pc; rc==SQLITE_OK; pc++){ - assert( pc>=0 && pcnOp ); - if( db->mallocFailed ) goto no_mem; + for(pOp=&aOp[p->pc]; 1; pOp++){ + /* Errors are detected by individual opcodes, with an immediate + ** jumps to abort_due_to_error. */ + assert( rc==SQLITE_OK ); + + assert( pOp>=aOp && pOp<&aOp[p->nOp]); #ifdef VDBE_PROFILE start = sqlite3Hwtime(); #endif nVmStep++; - pOp = &aOp[pc]; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + if( p->anExec ) p->anExec[(int)(pOp-aOp)]++; +#endif /* Only allow tracing if SQLITE_DEBUG is defined. */ #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeTrace ){ - sqlite3VdbePrintOp(stdout, pc, pOp); + sqlite3VdbePrintOp(stdout, (int)(pOp - aOp), pOp); } #endif @@ -71507,55 +81066,46 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } #endif - /* On any opcode with the "out2-prerelease" tag, free any - ** external allocations out of mem[p2] and set mem[p2] to be - ** an undefined integer. Opcodes will either fill in the integer - ** value or convert mem[p2] to a different type. - */ - assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); - if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); - pOut = &aMem[pOp->p2]; - memAboutToChange(p, pOut); - VdbeMemRelease(pOut); - pOut->flags = MEM_Int; - } - /* Sanity checking on other operands */ #ifdef SQLITE_DEBUG - if( (pOp->opflags & OPFLG_IN1)!=0 ){ - assert( pOp->p1>0 ); - assert( pOp->p1<=(p->nMem-p->nCursor) ); - assert( memIsValid(&aMem[pOp->p1]) ); - assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); - REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); - } - if( (pOp->opflags & OPFLG_IN2)!=0 ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); - assert( memIsValid(&aMem[pOp->p2]) ); - assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); - REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); - } - if( (pOp->opflags & OPFLG_IN3)!=0 ){ - assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem-p->nCursor) ); - assert( memIsValid(&aMem[pOp->p3]) ); - assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); - REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); - } - if( (pOp->opflags & OPFLG_OUT2)!=0 ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); - memAboutToChange(p, &aMem[pOp->p2]); - } - if( (pOp->opflags & OPFLG_OUT3)!=0 ){ - assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem-p->nCursor) ); - memAboutToChange(p, &aMem[pOp->p3]); + { + u8 opProperty = sqlite3OpcodeProperty[pOp->opcode]; + if( (opProperty & OPFLG_IN1)!=0 ){ + assert( pOp->p1>0 ); + assert( pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( memIsValid(&aMem[pOp->p1]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); + REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); + } + if( (opProperty & OPFLG_IN2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); + assert( memIsValid(&aMem[pOp->p2]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); + REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); + } + if( (opProperty & OPFLG_IN3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( memIsValid(&aMem[pOp->p3]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); + REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); + } + if( (opProperty & OPFLG_OUT2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); + memAboutToChange(p, &aMem[pOp->p2]); + } + if( (opProperty & OPFLG_OUT3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + memAboutToChange(p, &aMem[pOp->p3]); + } } #endif +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + pOrigOp = pOp; +#endif switch( pOp->opcode ){ @@ -71579,7 +81129,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** ** Other keywords in the comment that follows each case are used to ** construct the OPFLG_INITIALIZER value that initializes opcodeProperty[]. -** Keywords include: in1, in2, in3, out2_prerelease, out2, out3. See +** Keywords include: in1, in2, in3, out2, out3. See ** the mkopcodeh.awk script for additional information. ** ** Documentation about VDBE opcodes is generated by scanning this file @@ -71607,7 +81157,8 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** to the current line should be indented for EXPLAIN output. */ case OP_Goto: { /* jump */ - pc = pOp->p2 - 1; +jump_to_p2_and_check_for_interrupt: + pOp = &aOp[pOp->p2 - 1]; /* Opcodes that are used as the bottom of a loop (OP_Next, OP_Prev, ** OP_VNext, OP_RowSetNext, or OP_SorterNext) all jump here upon @@ -71633,7 +81184,7 @@ check_for_interrupt: nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); if( db->xProgress(db->pProgressArg) ){ rc = SQLITE_INTERRUPT; - goto vdbe_error_halt; + goto abort_due_to_error; } } #endif @@ -71647,14 +81198,18 @@ check_for_interrupt: ** and then jump to address P2. */ case OP_Gosub: { /* jump */ - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); memAboutToChange(p, pIn1); pIn1->flags = MEM_Int; - pIn1->u.i = pc; + pIn1->u.i = (int)(pOp-aOp); REGISTER_TRACE(pOp->p1, pIn1); - pc = pOp->p2 - 1; + + /* Most jump operations do a goto to this spot in order to update + ** the pOp pointer. */ +jump_to_p2: + pOp = &aOp[pOp->p2 - 1]; break; } @@ -71666,7 +81221,7 @@ case OP_Gosub: { /* jump */ case OP_Return: { /* in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags==MEM_Int ); - pc = (int)pIn1->u.i; + pOp = &aOp[pIn1->u.i]; pIn1->flags = MEM_Undefined; break; } @@ -71683,14 +81238,14 @@ case OP_Return: { /* in1 */ ** See also: EndCoroutine */ case OP_InitCoroutine: { /* jump */ - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( pOp->p2>=0 && pOp->p2nOp ); assert( pOp->p3>=0 && pOp->p3nOp ); pOut = &aMem[pOp->p1]; assert( !VdbeMemDynamic(pOut) ); pOut->u.i = pOp->p3 - 1; pOut->flags = MEM_Int; - if( pOp->p2 ) pc = pOp->p2 - 1; + if( pOp->p2 ) goto jump_to_p2; break; } @@ -71710,7 +81265,7 @@ case OP_EndCoroutine: { /* in1 */ pCaller = &aOp[pIn1->u.i]; assert( pCaller->opcode==OP_Yield ); assert( pCaller->p2>=0 && pCaller->p2nOp ); - pc = pCaller->p2 - 1; + pOp = &aOp[pCaller->p2 - 1]; pIn1->flags = MEM_Undefined; break; } @@ -71734,14 +81289,14 @@ case OP_Yield: { /* in1, jump */ assert( VdbeMemDynamic(pIn1)==0 ); pIn1->flags = MEM_Int; pcDest = (int)pIn1->u.i; - pIn1->u.i = pc; + pIn1->u.i = (int)(pOp - aOp); REGISTER_TRACE(pOp->p1, pIn1); - pc = pcDest; + pOp = &aOp[pcDest]; break; } /* Opcode: HaltIfNull P1 P2 P3 P4 P5 -** Synopsis: if r[P3]=null halt +** Synopsis: if r[P3]=null halt ** ** Check the value in register P3. If it is NULL then Halt using ** parameter P1, P2, and P4 as if this were a Halt instruction. If the @@ -71785,61 +81340,56 @@ case OP_HaltIfNull: { /* in3 */ ** is the same as executing Halt. */ case OP_Halt: { - const char *zType; - const char *zLogFmt; + VdbeFrame *pFrame; + int pcx; + pcx = (int)(pOp - aOp); if( pOp->p1==SQLITE_OK && p->pFrame ){ /* Halt the sub-program. Return control to the parent frame. */ - VdbeFrame *pFrame = p->pFrame; + pFrame = p->pFrame; p->pFrame = pFrame->pParent; p->nFrame--; sqlite3VdbeSetChanges(db, p->nChange); - pc = sqlite3VdbeFrameRestore(pFrame); + pcx = sqlite3VdbeFrameRestore(pFrame); lastRowid = db->lastRowid; if( pOp->p2==OE_Ignore ){ - /* Instruction pc is the OP_Program that invoked the sub-program + /* Instruction pcx is the OP_Program that invoked the sub-program ** currently being halted. If the p2 instruction of this OP_Halt ** instruction is set to OE_Ignore, then the sub-program is throwing ** an IGNORE exception. In this case jump to the address specified ** as the p2 of the calling OP_Program. */ - pc = p->aOp[pc].p2-1; + pcx = p->aOp[pcx].p2-1; } aOp = p->aOp; aMem = p->aMem; + pOp = &aOp[pcx]; break; } p->rc = pOp->p1; p->errorAction = (u8)pOp->p2; - p->pc = pc; + p->pc = pcx; + assert( pOp->p5>=0 && pOp->p5<=4 ); if( p->rc ){ if( pOp->p5 ){ static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK", "FOREIGN KEY" }; - assert( pOp->p5>=1 && pOp->p5<=4 ); testcase( pOp->p5==1 ); testcase( pOp->p5==2 ); testcase( pOp->p5==3 ); testcase( pOp->p5==4 ); - zType = azType[pOp->p5-1]; + sqlite3VdbeError(p, "%s constraint failed", azType[pOp->p5-1]); + if( pOp->p4.z ){ + p->zErrMsg = sqlite3MPrintf(db, "%z: %s", p->zErrMsg, pOp->p4.z); + } }else{ - zType = 0; + sqlite3VdbeError(p, "%s", pOp->p4.z); } - assert( zType!=0 || pOp->p4.z!=0 ); - zLogFmt = "abort at %d in [%s]: %s"; - if( zType && pOp->p4.z ){ - sqlite3SetString(&p->zErrMsg, db, "%s constraint failed: %s", - zType, pOp->p4.z); - }else if( pOp->p4.z ){ - sqlite3SetString(&p->zErrMsg, db, "%s", pOp->p4.z); - }else{ - sqlite3SetString(&p->zErrMsg, db, "%s constraint failed", zType); - } - sqlite3_log(pOp->p1, zLogFmt, pc, p->zSql, p->zErrMsg); + sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg); } rc = sqlite3VdbeHalt(p); assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); if( rc==SQLITE_BUSY ){ - p->rc = rc = SQLITE_BUSY; + p->rc = SQLITE_BUSY; }else{ assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ); assert( rc==SQLITE_OK || db->nDeferredCons>0 || db->nDeferredImmCons>0 ); @@ -71853,7 +81403,8 @@ case OP_Halt: { ** ** The 32-bit integer value P1 is written into register P2. */ -case OP_Integer: { /* out2-prerelease */ +case OP_Integer: { /* out2 */ + pOut = out2Prerelease(p, pOp); pOut->u.i = pOp->p1; break; } @@ -71864,7 +81415,8 @@ case OP_Integer: { /* out2-prerelease */ ** P4 is a pointer to a 64-bit integer value. ** Write that value into register P2. */ -case OP_Int64: { /* out2-prerelease */ +case OP_Int64: { /* out2 */ + pOut = out2Prerelease(p, pOp); assert( pOp->p4.pI64!=0 ); pOut->u.i = *pOp->p4.pI64; break; @@ -71877,10 +81429,11 @@ case OP_Int64: { /* out2-prerelease */ ** P4 is a pointer to a 64-bit floating point value. ** Write that value into register P2. */ -case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ +case OP_Real: { /* same as TK_FLOAT, out2 */ + pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Real; assert( !sqlite3IsNaN(*pOp->p4.pReal) ); - pOut->r = *pOp->p4.pReal; + pOut->u.r = *pOp->p4.pReal; break; } #endif @@ -71889,23 +81442,24 @@ case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ ** Synopsis: r[P2]='P4' ** ** P4 points to a nul terminated UTF-8 string. This opcode is transformed -** into a String before it is executed for the first time. During +** into a String opcode before it is executed for the first time. During ** this transformation, the length of string P4 is computed and stored ** as the P1 parameter. */ -case OP_String8: { /* same as TK_STRING, out2-prerelease */ +case OP_String8: { /* same as TK_STRING, out2 */ assert( pOp->p4.z!=0 ); + pOut = out2Prerelease(p, pOp); pOp->opcode = OP_String; pOp->p1 = sqlite3Strlen30(pOp->p4.z); #ifndef SQLITE_OMIT_UTF16 if( encoding!=SQLITE_UTF8 ){ rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); - if( rc==SQLITE_TOOBIG ) goto too_big; + assert( rc==SQLITE_OK || rc==SQLITE_TOOBIG ); if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; - assert( pOut->zMalloc==pOut->z ); + assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); assert( VdbeMemDynamic(pOut)==0 ); - pOut->zMalloc = 0; + pOut->szMalloc = 0; pOut->flags |= MEM_Static; if( pOp->p4type==P4_DYNAMIC ){ sqlite3DbFree(db, pOp->p4.z); @@ -71914,30 +81468,48 @@ case OP_String8: { /* same as TK_STRING, out2-prerelease */ pOp->p4.z = pOut->z; pOp->p1 = pOut->n; } + testcase( rc==SQLITE_TOOBIG ); #endif if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } + assert( rc==SQLITE_OK ); /* Fall through to the next case, OP_String */ } -/* Opcode: String P1 P2 * P4 * +/* Opcode: String P1 P2 P3 P4 P5 ** Synopsis: r[P2]='P4' (len=P1) ** ** The string value P4 of length P1 (bytes) is stored in register P2. +** +** If P3 is not zero and the content of register P3 is equal to P5, then +** the datatype of the register P2 is converted to BLOB. The content is +** the same sequence of bytes, it is merely interpreted as a BLOB instead +** of a string, as if it had been CAST. In other words: +** +** if( P3!=0 and reg[P3]==P5 ) reg[P2] := CAST(reg[P2] as BLOB) */ -case OP_String: { /* out2-prerelease */ +case OP_String: { /* out2 */ assert( pOp->p4.z!=0 ); + pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Str|MEM_Static|MEM_Term; pOut->z = pOp->p4.z; pOut->n = pOp->p1; pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( pOp->p3>0 ){ + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + pIn3 = &aMem[pOp->p3]; + assert( pIn3->flags & MEM_Int ); + if( pIn3->u.i==pOp->p5 ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term; + } +#endif break; } /* Opcode: Null P1 P2 P3 * * -** Synopsis: r[P2..P3]=NULL +** Synopsis: r[P2..P3]=NULL ** ** Write a NULL into registers P2. If P3 greater than P2, then also write ** NULL into register P3 and every register in between P2 and P3. If P3 @@ -71948,24 +81520,27 @@ case OP_String: { /* out2-prerelease */ ** NULL values will not compare equal even if SQLITE_NULLEQ is set on ** OP_Ne or OP_Eq. */ -case OP_Null: { /* out2-prerelease */ +case OP_Null: { /* out2 */ int cnt; u16 nullFlag; + pOut = out2Prerelease(p, pOp); cnt = pOp->p3-pOp->p2; - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; + pOut->n = 0; while( cnt>0 ){ pOut++; memAboutToChange(p, pOut); - VdbeMemRelease(pOut); + sqlite3VdbeMemSetNull(pOut); pOut->flags = nullFlag; + pOut->n = 0; cnt--; } break; } /* Opcode: SoftNull P1 * * * * -** Synopsis: r[P1]=NULL +** Synopsis: r[P1]=NULL ** ** Set register P1 to have the value NULL as seen by the OP_MakeRecord ** instruction, but do not free any string or blob memory associated with @@ -71973,7 +81548,7 @@ case OP_Null: { /* out2-prerelease */ ** previously copied using OP_SCopy, the copies will continue to be valid. */ case OP_SoftNull: { - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pOut = &aMem[pOp->p1]; pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined; break; @@ -71985,8 +81560,9 @@ case OP_SoftNull: { ** P4 points to a blob of data P1 bytes long. Store this ** blob in register P2. */ -case OP_Blob: { /* out2-prerelease */ +case OP_Blob: { /* out2 */ assert( pOp->p1 <= SQLITE_MAX_LENGTH ); + pOut = out2Prerelease(p, pOp); sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); @@ -72001,7 +81577,7 @@ case OP_Blob: { /* out2-prerelease */ ** If the parameter is named, then its name appears in P4. ** The P4 value is used by sqlite3_bind_parameter_name(). */ -case OP_Variable: { /* out2-prerelease */ +case OP_Variable: { /* out2 */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); @@ -72010,13 +81586,14 @@ case OP_Variable: { /* out2-prerelease */ if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; } + pOut = out2Prerelease(p, pOp); sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); UPDATE_MAX_BLOBSIZE(pOut); break; } /* Opcode: Move P1 P2 P3 * * -** Synopsis: r[P2@P3]=r[P1@P3] +** Synopsis: r[P2@P3]=r[P1@P3] ** ** Move the P3 values in register P1..P1+P3-1 over into ** registers P2..P2+P3-1. Registers P1..P1+P3-1 are @@ -72025,7 +81602,6 @@ case OP_Variable: { /* out2-prerelease */ ** for P3 to be less than 1. */ case OP_Move: { - char *zMalloc; /* Holding variable for allocated memory */ int n; /* Number of registers left to copy */ int p1; /* Register to copy from */ int p2; /* Register to copy to */ @@ -72039,21 +81615,17 @@ case OP_Move: { pIn1 = &aMem[p1]; pOut = &aMem[p2]; do{ - assert( pOut<=&aMem[(p->nMem-p->nCursor)] ); - assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); + assert( pOut<=&aMem[(p->nMem+1 - p->nCursor)] ); + assert( pIn1<=&aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); - VdbeMemRelease(pOut); - zMalloc = pOut->zMalloc; - memcpy(pOut, pIn1, sizeof(Mem)); + sqlite3VdbeMemMove(pOut, pIn1); #ifdef SQLITE_DEBUG - if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrom<&aMem[p1+pOp->p3] ){ - pOut->pScopyFrom += p1 - pOp->p2; + if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrompScopyFrom += pOp->p2 - p1; } #endif - pIn1->flags = MEM_Undefined; - pIn1->xDel = 0; - pIn1->zMalloc = zMalloc; + Deephemeralize(pOut); REGISTER_TRACE(p2++, pOut); pIn1++; pOut++; @@ -72114,8 +81686,24 @@ case OP_SCopy: { /* out2 */ break; } +/* Opcode: IntCopy P1 P2 * * * +** Synopsis: r[P2]=r[P1] +** +** Transfer the integer value held in register P1 into register P2. +** +** This is an optimized version of SCopy that works only for integer +** values. +*/ +case OP_IntCopy: { /* out2 */ + pIn1 = &aMem[pOp->p1]; + assert( (pIn1->flags & MEM_Int)!=0 ); + pOut = &aMem[pOp->p2]; + sqlite3VdbeMemSetInt64(pOut, pIn1->u.i); + break; +} + /* Opcode: ResultRow P1 P2 * * * -** Synopsis: output=r[P1@P2] +** Synopsis: output=r[P1@P2] ** ** The registers P1 through P1+P2-1 contain a single row of ** results. This opcode causes the sqlite3_step() call to terminate @@ -72128,7 +81716,7 @@ case OP_ResultRow: { int i; assert( p->nResColumn==pOp->p2 ); assert( pOp->p1>0 ); - assert( pOp->p1+pOp->p2<=(p->nMem-p->nCursor)+1 ); + assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* Run the progress counter just before returning. @@ -72138,7 +81726,7 @@ case OP_ResultRow: { && db->xProgress(db->pProgressArg)!=0 ){ rc = SQLITE_INTERRUPT; - goto vdbe_error_halt; + goto abort_due_to_error; } #endif @@ -72148,7 +81736,7 @@ case OP_ResultRow: { if( SQLITE_OK!=(rc = sqlite3VdbeCheckFk(p, 0)) ){ assert( db->flags&SQLITE_CountRows ); assert( p->usesStmtJournal ); - break; + goto abort_due_to_error; } /* If the SQLITE_CountRows flag is set in sqlite3.flags mask, then @@ -72168,9 +81756,7 @@ case OP_ResultRow: { */ assert( p->iStatement==0 || db->flags&SQLITE_CountRows ); rc = sqlite3VdbeCloseStatement(p, SAVEPOINT_RELEASE); - if( NEVER(rc!=SQLITE_OK) ){ - break; - } + assert( rc==SQLITE_OK ); /* Invalidate all ephemeral cursor row caches */ p->cacheCtr = (p->cacheCtr + 2)|1; @@ -72190,9 +81776,13 @@ case OP_ResultRow: { } if( db->mallocFailed ) goto no_mem; + if( db->mTrace & SQLITE_TRACE_ROW ){ + db->xTrace(SQLITE_TRACE_ROW, db->pTraceArg, p, 0); + } + /* Return SQLITE_ROW */ - p->pc = pc + 1; + p->pc = (int)(pOp - aOp) + 1; rc = SQLITE_ROW; goto vdbe_return; } @@ -72246,14 +81836,14 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ } /* Opcode: Add P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]+r[P2] +** Synopsis: r[P3]=r[P1]+r[P2] ** ** Add the value in register P1 to the value in register P2 ** and store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: Multiply P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]*r[P2] +** Synopsis: r[P3]=r[P1]*r[P2] ** ** ** Multiply the value in register P1 by the value in register P2 @@ -72261,14 +81851,14 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ ** If either input is NULL, the result is NULL. */ /* Opcode: Subtract P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]-r[P1] +** Synopsis: r[P3]=r[P2]-r[P1] ** ** Subtract the value in register P1 from the value in register P2 ** and store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: Divide P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]/r[P1] +** Synopsis: r[P3]=r[P2]/r[P1] ** ** Divide the value in register P1 by the value in register P2 ** and store the result in register P3 (P3=P2/P1). If the value in @@ -72276,7 +81866,7 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ ** NULL, the result is NULL. */ /* Opcode: Remainder P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]%r[P1] +** Synopsis: r[P3]=r[P2]%r[P1] ** ** Compute the remainder after integer register P2 is divided by ** register P1 and store the result in register P3. @@ -72358,7 +81948,7 @@ fp_math: if( sqlite3IsNaN(rB) ){ goto arithmetic_result_is_null; } - pOut->r = rB; + pOut->u.r = rB; MemSetTypeFlag(pOut, MEM_Real); if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ sqlite3VdbeIntegerAffinity(pOut); @@ -72385,7 +81975,7 @@ arithmetic_result_is_null: ** ** The interface used by the implementation of the aforementioned functions ** to retrieve the collation sequence set by this opcode is not available -** publicly, only to user functions defined in func.c. +** publicly. Only built-in functions have access to this feature. */ case OP_CollSeq: { assert( pOp->p4type==P4_COLLSEQ ); @@ -72395,10 +81985,10 @@ case OP_CollSeq: { break; } -/* Opcode: Function P1 P2 P3 P4 P5 +/* Opcode: Function0 P1 P2 P3 P4 P5 ** Synopsis: r[P3]=func(r[P2@P5]) ** -** Invoke a user function (P4 is a pointer to a Function structure that +** Invoke a user function (P4 is a pointer to a FuncDef object that ** defines the function) with P5 arguments taken from register P2 and ** successors. The result of the function is stored in register P3. ** Register P3 must not be one of the function inputs. @@ -72410,114 +82000,120 @@ case OP_CollSeq: { ** sqlite3_set_auxdata() API may be safely retained until the next ** invocation of this opcode. ** -** See also: AggStep and AggFinal +** See also: Function, AggStep, AggFinal */ -case OP_Function: { - int i; - Mem *pArg; - sqlite3_context ctx; - sqlite3_value **apVal; +/* Opcode: Function P1 P2 P3 P4 P5 +** Synopsis: r[P3]=func(r[P2@P5]) +** +** Invoke a user function (P4 is a pointer to an sqlite3_context object that +** contains a pointer to the function to be run) with P5 arguments taken +** from register P2 and successors. The result of the function is stored +** in register P3. Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** SQL functions are initially coded as OP_Function0 with P4 pointing +** to a FuncDef object. But on first evaluation, the P4 operand is +** automatically converted into an sqlite3_context object and the operation +** changed to this OP_Function opcode. In this way, the initialization of +** the sqlite3_context object occurs only once, rather than once for each +** evaluation of the function. +** +** See also: Function0, AggStep, AggFinal +*/ +case OP_Function0: { int n; - - n = pOp->p5; - apVal = p->apArg; - assert( apVal || n==0 ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - pOut = &aMem[pOp->p3]; - memAboutToChange(p, pOut); - - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); - assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pArg = &aMem[pOp->p2]; - for(i=0; ip2+i, pArg); - } + sqlite3_context *pCtx; assert( pOp->p4type==P4_FUNCDEF ); - ctx.pFunc = pOp->p4.pFunc; - ctx.iOp = pc; - ctx.pVdbe = p; + n = pOp->p5; + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); + assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); + pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + if( pCtx==0 ) goto no_mem; + pCtx->pOut = 0; + pCtx->pFunc = pOp->p4.pFunc; + pCtx->iOp = (int)(pOp - aOp); + pCtx->pVdbe = p; + pCtx->argc = n; + pOp->p4type = P4_FUNCCTX; + pOp->p4.pCtx = pCtx; + pOp->opcode = OP_Function; + /* Fall through into OP_Function */ +} +case OP_Function: { + int i; + sqlite3_context *pCtx; - /* The output cell may already have a buffer allocated. Move - ** the pointer to ctx.s so in case the user-function can use - ** the already allocated buffer instead of allocating a new one. - */ - memcpy(&ctx.s, pOut, sizeof(Mem)); - pOut->flags = MEM_Null; - pOut->xDel = 0; - pOut->zMalloc = 0; - MemSetTypeFlag(&ctx.s, MEM_Null); + assert( pOp->p4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; - ctx.fErrorOrAux = 0; - if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - ctx.pColl = pOp[-1].p4.pColl; + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relavant parts of the sqlite3_context object */ + pOut = &aMem[pOp->p3]; + if( pCtx->pOut != pOut ){ + pCtx->pOut = pOut; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } + + memAboutToChange(p, pCtx->pOut); +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + MemSetTypeFlag(pCtx->pOut, MEM_Null); + pCtx->fErrorOrAux = 0; db->lastRowid = lastRowid; - (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; - - if( db->mallocFailed ){ - /* Even though a malloc() has failed, the implementation of the - ** user function may have called an sqlite3_result_XXX() function - ** to return a value. The following call releases any resources - ** associated with such a value. - */ - sqlite3VdbeMemRelease(&ctx.s); - goto no_mem; - } + (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ + lastRowid = db->lastRowid; /* Remember rowid changes made by xSFunc */ /* If the function returned an error, throw an exception */ - if( ctx.fErrorOrAux ){ - if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); - rc = ctx.isError; + if( pCtx->fErrorOrAux ){ + if( pCtx->isError ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); + rc = pCtx->isError; } - sqlite3VdbeDeleteAuxData(p, pc, pOp->p1); + sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); + if( rc ) goto abort_due_to_error; } /* Copy the result of the function into register P3 */ - sqlite3VdbeChangeEncoding(&ctx.s, encoding); - assert( pOut->flags==MEM_Null ); - memcpy(pOut, &ctx.s, sizeof(Mem)); - if( sqlite3VdbeMemTooBig(pOut) ){ - goto too_big; + if( pOut->flags & (MEM_Str|MEM_Blob) ){ + sqlite3VdbeChangeEncoding(pCtx->pOut, encoding); + if( sqlite3VdbeMemTooBig(pCtx->pOut) ) goto too_big; } -#if 0 - /* The app-defined function has done something that as caused this - ** statement to expire. (Perhaps the function called sqlite3_exec() - ** with a CREATE TABLE statement.) - */ - if( p->expired ) rc = SQLITE_ABORT; -#endif - - REGISTER_TRACE(pOp->p3, pOut); - UPDATE_MAX_BLOBSIZE(pOut); + REGISTER_TRACE(pOp->p3, pCtx->pOut); + UPDATE_MAX_BLOBSIZE(pCtx->pOut); break; } /* Opcode: BitAnd P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]&r[P2] +** Synopsis: r[P3]=r[P1]&r[P2] ** ** Take the bit-wise AND of the values in register P1 and P2 and ** store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: BitOr P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]|r[P2] +** Synopsis: r[P3]=r[P1]|r[P2] ** ** Take the bit-wise OR of the values in register P1 and P2 and ** store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: ShiftLeft P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]<>r[P1] +** Synopsis: r[P3]=r[P2]>>r[P1] ** ** Shift the integer value in register P2 to the right by the ** number of bits specified by the integer in register P1. @@ -72585,7 +82181,7 @@ case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ } /* Opcode: AddImm P1 P2 * * * -** Synopsis: r[P1]=r[P1]+P2 +** Synopsis: r[P1]=r[P1]+P2 ** ** Add the constant P2 to the value in register P1. ** The result is always an integer. @@ -72617,8 +82213,7 @@ case OP_MustBeInt: { /* jump, in1 */ rc = SQLITE_MISMATCH; goto abort_due_to_error; }else{ - pc = pOp->p2 - 1; - break; + goto jump_to_p2; } } } @@ -72646,114 +82241,93 @@ case OP_RealAffinity: { /* in1 */ #endif #ifndef SQLITE_OMIT_CAST -/* Opcode: ToText P1 * * * * +/* Opcode: Cast P1 P2 * * * +** Synopsis: affinity(r[P1]) ** -** Force the value in register P1 to be text. -** If the value is numeric, convert it to a string using the -** equivalent of sprintf(). Blob values are unchanged and -** are afterwards simply interpreted as text. +** Force the value in register P1 to be the type defined by P2. +** +**
      +**
    • TEXT +**
    • BLOB +**
    • NUMERIC +**
    • INTEGER +**
    • REAL +**
    ** ** A NULL value is not changed by this routine. It remains NULL. */ -case OP_ToText: { /* same as TK_TO_TEXT, in1 */ +case OP_Cast: { /* in1 */ + assert( pOp->p2>=SQLITE_AFF_BLOB && pOp->p2<=SQLITE_AFF_REAL ); + testcase( pOp->p2==SQLITE_AFF_TEXT ); + testcase( pOp->p2==SQLITE_AFF_BLOB ); + testcase( pOp->p2==SQLITE_AFF_NUMERIC ); + testcase( pOp->p2==SQLITE_AFF_INTEGER ); + testcase( pOp->p2==SQLITE_AFF_REAL ); pIn1 = &aMem[pOp->p1]; memAboutToChange(p, pIn1); - if( pIn1->flags & MEM_Null ) break; - assert( MEM_Str==(MEM_Blob>>3) ); - pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; - applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); rc = ExpandBlob(pIn1); - assert( pIn1->flags & MEM_Str || db->mallocFailed ); - pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); UPDATE_MAX_BLOBSIZE(pIn1); - break; -} - -/* Opcode: ToBlob P1 * * * * -** -** Force the value in register P1 to be a BLOB. -** If the value is numeric, convert it to a string first. -** Strings are simply reinterpreted as blobs with no change -** to the underlying data. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ) break; - if( (pIn1->flags & MEM_Blob)==0 ){ - applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); - assert( pIn1->flags & MEM_Str || db->mallocFailed ); - MemSetTypeFlag(pIn1, MEM_Blob); - }else{ - pIn1->flags &= ~(MEM_TypeMask&~MEM_Blob); - } - UPDATE_MAX_BLOBSIZE(pIn1); - break; -} - -/* Opcode: ToNumeric P1 * * * * -** -** Force the value in register P1 to be numeric (either an -** integer or a floating-point number.) -** If the value is text or blob, try to convert it to an using the -** equivalent of atoi() or atof() and store 0 if no such conversion -** is possible. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ - pIn1 = &aMem[pOp->p1]; - sqlite3VdbeMemNumerify(pIn1); + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_CAST */ -/* Opcode: ToInt P1 * * * * +/* Opcode: Eq P1 P2 P3 P4 P5 +** Synopsis: IF r[P3]==r[P1] ** -** Force the value in register P1 to be an integer. If -** The value is currently a real number, drop its fractional part. -** If the value is text or blob, try to convert it to an integer using the -** equivalent of atoi() and store 0 if no such conversion is possible. +** Compare the values in register P1 and P3. If reg(P3)==reg(P1) then +** jump to address P2. Or if the SQLITE_STOREP2 flag is set in P5, then +** store the result of comparison in register P2. ** -** A NULL value is not changed by this routine. It remains NULL. +** The SQLITE_AFF_MASK portion of P5 must be an affinity character - +** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made +** to coerce both inputs according to this affinity before the +** comparison is made. If the SQLITE_AFF_MASK is 0x00, then numeric +** affinity is used. Note that the affinity conversions are stored +** back into the input registers P1 and P3. So this opcode can cause +** persistent changes to registers P1 and P3. +** +** Once any conversions have taken place, and neither value is NULL, +** the values are compared. If both values are blobs then memcmp() is +** used to determine the results of the comparison. If both values +** are text, then the appropriate collating function specified in +** P4 is used to do the comparison. If P4 is not specified then +** memcmp() is used to compare text string. If both values are +** numeric, then a numeric comparison is used. If the two values +** are of different types, then numbers are considered less than +** strings and strings are considered less than blobs. +** +** If SQLITE_NULLEQ is set in P5 then the result of comparison is always either +** true or false and is never NULL. If both operands are NULL then the result +** of comparison is true. If either operand is NULL then the result is false. +** If neither operand is NULL the result is the same as it would be if +** the SQLITE_NULLEQ flag were omitted from P5. +** +** If both SQLITE_STOREP2 and SQLITE_KEEPNULL flags are set then the +** content of r[P2] is only changed if the new value is NULL or 0 (false). +** In other words, a prior r[P2] value will not be overwritten by 1 (true). */ -case OP_ToInt: { /* same as TK_TO_INT, in1 */ - pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_Null)==0 ){ - sqlite3VdbeMemIntegerify(pIn1); - } - break; -} - -#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) -/* Opcode: ToReal P1 * * * * +/* Opcode: Ne P1 P2 P3 P4 P5 +** Synopsis: IF r[P3]!=r[P1] ** -** Force the value in register P1 to be a floating point number. -** If The value is currently an integer, convert it. -** If the value is text or blob, try to convert it to an integer using the -** equivalent of atoi() and store 0.0 if no such conversion is possible. +** This works just like the Eq opcode except that the jump is taken if +** the operands in registers P1 and P3 are not equal. See the Eq opcode for +** additional information. ** -** A NULL value is not changed by this routine. It remains NULL. +** If both SQLITE_STOREP2 and SQLITE_KEEPNULL flags are set then the +** content of r[P2] is only changed if the new value is NULL or 1 (true). +** In other words, a prior r[P2] value will not be overwritten by 0 (false). */ -case OP_ToReal: { /* same as TK_TO_REAL, in1 */ - pIn1 = &aMem[pOp->p1]; - memAboutToChange(p, pIn1); - if( (pIn1->flags & MEM_Null)==0 ){ - sqlite3VdbeMemRealify(pIn1); - } - break; -} -#endif /* !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) */ - /* Opcode: Lt P1 P2 P3 P4 P5 -** Synopsis: if r[P1]r[P3] goto P2 +** Synopsis: IF r[P3]>r[P1] ** ** This works just like the Lt opcode except that the jump is taken if ** the content of register P3 is greater than the content of ** register P1. See the Lt opcode for additional information. */ /* Opcode: Ge P1 P2 P3 P4 P5 -** Synopsis: if r[P1]>=r[P3] goto P2 +** Synopsis: IF r[P3]>=r[P1] ** ** This works just like the Lt opcode except that the jump is taken if ** the content of register P3 is greater than or equal to the content of @@ -72834,7 +82375,7 @@ case OP_Lt: /* same as TK_LT, jump, in1, in3 */ case OP_Le: /* same as TK_LE, jump, in1, in3 */ case OP_Gt: /* same as TK_GT, jump, in1, in3 */ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ - int res; /* Result of the comparison of pIn1 against pIn3 */ + int res, res2; /* Result of the comparison of pIn1 against pIn3 */ char affinity; /* Affinity to use for comparison */ u16 flags1; /* Copy of initial value of pIn1->flags */ u16 flags3; /* Copy of initial value of pIn3->flags */ @@ -72857,9 +82398,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ && (flags3&MEM_Null)!=0 && (flags3&MEM_Cleared)==0 ){ - res = 0; /* Results are equal */ + res = 0; /* Operands are equal */ }else{ - res = 1; /* Results are not equal */ + res = 1; /* Operands are not equal */ } }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, @@ -72868,12 +82409,14 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ */ if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; + iCompare = 1; /* Operands are not equal */ + memAboutToChange(p, pOut); MemSetTypeFlag(pOut, MEM_Null); REGISTER_TRACE(pOp->p2, pOut); }else{ VdbeBranchTaken(2,3); if( pOp->p5 & SQLITE_JUMPIFNULL ){ - pc = pOp->p2-1; + goto jump_to_p2; } } break; @@ -72881,44 +82424,112 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ }else{ /* Neither operand is NULL. Do a comparison. */ affinity = pOp->p5 & SQLITE_AFF_MASK; - if( affinity ){ - applyAffinity(pIn1, affinity, encoding); - applyAffinity(pIn3, affinity, encoding); - if( db->mallocFailed ) goto no_mem; + if( affinity>=SQLITE_AFF_NUMERIC ){ + if( (flags1 | flags3)&MEM_Str ){ + if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn1,0); + testcase( flags3!=pIn3->flags ); /* Possible if pIn1==pIn3 */ + flags3 = pIn3->flags; + } + if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn3,0); + } + } + /* Handle the common case of integer comparison here, as an + ** optimization, to avoid a call to sqlite3MemCompare() */ + if( (pIn1->flags & pIn3->flags & MEM_Int)!=0 ){ + if( pIn3->u.i > pIn1->u.i ){ res = +1; goto compare_op; } + if( pIn3->u.i < pIn1->u.i ){ res = -1; goto compare_op; } + res = 0; + goto compare_op; + } + }else if( affinity==SQLITE_AFF_TEXT ){ + if( (flags1 & MEM_Str)==0 && (flags1 & (MEM_Int|MEM_Real))!=0 ){ + testcase( pIn1->flags & MEM_Int ); + testcase( pIn1->flags & MEM_Real ); + sqlite3VdbeMemStringify(pIn1, encoding, 1); + testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); + flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); + assert( pIn1!=pIn3 ); + } + if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){ + testcase( pIn3->flags & MEM_Int ); + testcase( pIn3->flags & MEM_Real ); + sqlite3VdbeMemStringify(pIn3, encoding, 1); + testcase( (flags3&MEM_Dyn) != (pIn3->flags&MEM_Dyn) ); + flags3 = (pIn3->flags & ~MEM_TypeMask) | (flags3 & MEM_TypeMask); + } } - assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); - ExpandBlob(pIn1); - ExpandBlob(pIn3); res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); } +compare_op: switch( pOp->opcode ){ - case OP_Eq: res = res==0; break; - case OP_Ne: res = res!=0; break; - case OP_Lt: res = res<0; break; - case OP_Le: res = res<=0; break; - case OP_Gt: res = res>0; break; - default: res = res>=0; break; + case OP_Eq: res2 = res==0; break; + case OP_Ne: res2 = res; break; + case OP_Lt: res2 = res<0; break; + case OP_Le: res2 = res<=0; break; + case OP_Gt: res2 = res>0; break; + default: res2 = res>=0; break; } + /* Undo any changes made by applyAffinity() to the input registers. */ + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; + assert( (pIn3->flags & MEM_Dyn) == (flags3 & MEM_Dyn) ); + pIn3->flags = flags3; + if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; + iCompare = res; + res2 = res2!=0; /* For this path res2 must be exactly 0 or 1 */ + if( (pOp->p5 & SQLITE_KEEPNULL)!=0 ){ + /* The KEEPNULL flag prevents OP_Eq from overwriting a NULL with 1 + ** and prevents OP_Ne from overwriting NULL with 0. This flag + ** is only used in contexts where either: + ** (1) op==OP_Eq && (r[P2]==NULL || r[P2]==0) + ** (2) op==OP_Ne && (r[P2]==NULL || r[P2]==1) + ** Therefore it is not necessary to check the content of r[P2] for + ** NULL. */ + assert( pOp->opcode==OP_Ne || pOp->opcode==OP_Eq ); + assert( res2==0 || res2==1 ); + testcase( res2==0 && pOp->opcode==OP_Eq ); + testcase( res2==1 && pOp->opcode==OP_Eq ); + testcase( res2==0 && pOp->opcode==OP_Ne ); + testcase( res2==1 && pOp->opcode==OP_Ne ); + if( (pOp->opcode==OP_Eq)==res2 ) break; + } memAboutToChange(p, pOut); MemSetTypeFlag(pOut, MEM_Int); - pOut->u.i = res; + pOut->u.i = res2; REGISTER_TRACE(pOp->p2, pOut); }else{ VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); - if( res ){ - pc = pOp->p2-1; + if( res2 ){ + goto jump_to_p2; } } - /* Undo any changes made by applyAffinity() to the input registers. */ - pIn1->flags = (pIn1->flags&~MEM_TypeMask) | (flags1&MEM_TypeMask); - pIn3->flags = (pIn3->flags&~MEM_TypeMask) | (flags3&MEM_TypeMask); break; } +/* Opcode: ElseNotEq * P2 * * * +** +** This opcode must immediately follow an OP_Lt or OP_Gt comparison operator. +** If result of an OP_Eq comparison on the same two operands +** would have be NULL or false (0), then then jump to P2. +** If the result of an OP_Eq comparison on the two previous operands +** would have been true (1), then fall through. +*/ +case OP_ElseNotEq: { /* same as TK_ESCAPE, jump */ + assert( pOp>aOp ); + assert( pOp[-1].opcode==OP_Lt || pOp[-1].opcode==OP_Gt ); + assert( pOp[-1].p5 & SQLITE_STOREP2 ); + VdbeBranchTaken(iCompare!=0, 2); + if( iCompare!=0 ) goto jump_to_p2; + break; +} + + /* Opcode: Permutation * * * P4 * ** ** Set the permutation used by the OP_Compare operator to be the array @@ -72927,11 +82538,14 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** The permutation is only valid until the next OP_Compare that has ** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should ** occur immediately prior to the OP_Compare. +** +** The first integer in the P4 integer array is the length of the array +** and does not become part of the permutation. */ case OP_Permutation: { assert( pOp->p4type==P4_INTARRAY ); assert( pOp->p4.ai ); - aPermute = pOp->p4.ai; + aPermute = pOp->p4.ai + 1; break; } @@ -72976,11 +82590,11 @@ case OP_Compare: { if( aPermute ){ int k, mx = 0; for(k=0; kmx ) mx = aPermute[k]; - assert( p1>0 && p1+mx<=(p->nMem-p->nCursor)+1 ); - assert( p2>0 && p2+mx<=(p->nMem-p->nCursor)+1 ); + assert( p1>0 && p1+mx<=(p->nMem+1 - p->nCursor)+1 ); + assert( p2>0 && p2+mx<=(p->nMem+1 - p->nCursor)+1 ); }else{ - assert( p1>0 && p1+n<=(p->nMem-p->nCursor)+1 ); - assert( p2>0 && p2+n<=(p->nMem-p->nCursor)+1 ); + assert( p1>0 && p1+n<=(p->nMem+1 - p->nCursor)+1 ); + assert( p2>0 && p2+n<=(p->nMem+1 - p->nCursor)+1 ); } #endif /* SQLITE_DEBUG */ for(i=0; ip1 - 1; VdbeBranchTaken(0,3); + VdbeBranchTaken(0,3); pOp = &aOp[pOp->p1 - 1]; }else if( iCompare==0 ){ - pc = pOp->p2 - 1; VdbeBranchTaken(1,3); + VdbeBranchTaken(1,3); pOp = &aOp[pOp->p2 - 1]; }else{ - pc = pOp->p3 - 1; VdbeBranchTaken(2,3); + VdbeBranchTaken(2,3); pOp = &aOp[pOp->p3 - 1]; } break; } @@ -73083,10 +82697,10 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ case OP_Not: { /* same as TK_NOT, in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; - if( pIn1->flags & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - }else{ - sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1)); + sqlite3VdbeMemSetNull(pOut); + if( (pIn1->flags & MEM_Null)==0 ){ + pOut->flags = MEM_Int; + pOut->u.i = !sqlite3VdbeIntValue(pIn1); } break; } @@ -73101,32 +82715,28 @@ case OP_Not: { /* same as TK_NOT, in1, out2 */ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; - if( pIn1->flags & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - }else{ - sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1)); + sqlite3VdbeMemSetNull(pOut); + if( (pIn1->flags & MEM_Null)==0 ){ + pOut->flags = MEM_Int; + pOut->u.i = ~sqlite3VdbeIntValue(pIn1); } break; } /* Opcode: Once P1 P2 * * * ** -** Check the "once" flag number P1. If it is set, jump to instruction P2. -** Otherwise, set the flag and fall through to the next instruction. -** In other words, this opcode causes all following opcodes up through P2 -** (but not including P2) to run just once and to be skipped on subsequent -** times through the loop. -** -** All "once" flags are initially cleared whenever a prepared statement -** first begins to run. +** If the P1 value is equal to the P1 value on the OP_Init opcode at +** instruction 0, then jump to P2. If the two P1 values differ, then +** set the P1 value on this opcode to equal the P1 value on the OP_Init +** and fall through. */ case OP_Once: { /* jump */ - assert( pOp->p1nOnceFlag ); - VdbeBranchTaken(p->aOnceFlag[pOp->p1]!=0, 2); - if( p->aOnceFlag[pOp->p1] ){ - pc = pOp->p2-1; + assert( p->aOp[0].opcode==OP_Init ); + VdbeBranchTaken(p->aOp[0].p1==pOp->p1, 2); + if( p->aOp[0].p1==pOp->p1 ){ + goto jump_to_p2; }else{ - p->aOnceFlag[pOp->p1] = 1; + pOp->p1 = p->aOp[0].p1; } break; } @@ -73159,13 +82769,13 @@ case OP_IfNot: { /* jump, in1 */ } VdbeBranchTaken(c!=0, 2); if( c ){ - pc = pOp->p2-1; + goto jump_to_p2; } break; } /* Opcode: IsNull P1 P2 * * * -** Synopsis: if r[P1]==NULL goto P2 +** Synopsis: if r[P1]==NULL goto P2 ** ** Jump to P2 if the value in register P1 is NULL. */ @@ -73173,7 +82783,7 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ pIn1 = &aMem[pOp->p1]; VdbeBranchTaken( (pIn1->flags & MEM_Null)!=0, 2); if( (pIn1->flags & MEM_Null)!=0 ){ - pc = pOp->p2 - 1; + goto jump_to_p2; } break; } @@ -73187,13 +82797,13 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ pIn1 = &aMem[pOp->p1]; VdbeBranchTaken( (pIn1->flags & MEM_Null)==0, 2); if( (pIn1->flags & MEM_Null)==0 ){ - pc = pOp->p2 - 1; + goto jump_to_p2; } break; } /* Opcode: Column P1 P2 P3 P4 P5 -** Synopsis: r[P3]=PX +** Synopsis: r[P3]=PX ** ** Interpret the data that cursor P1 points to as a structure built using ** the MakeRecord instruction. (See the MakeRecord opcode for additional @@ -73218,11 +82828,9 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** skipped for length() and all content loading can be skipped for typeof(). */ case OP_Column: { - i64 payloadSize64; /* Number of bytes in the record */ int p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ BtCursor *pCrsr; /* The BTree cursor */ - u32 *aType; /* aType[i] holds the numeric type of the i-th column */ u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ int len; /* The length of the serialized data for the column */ int i; /* Loop counter */ @@ -73232,102 +82840,98 @@ case OP_Column: { const u8 *zHdr; /* Next unparsed byte of the header */ const u8 *zEndHdr; /* Pointer to first byte after the header */ u32 offset; /* Offset into the data */ - u32 szField; /* Number of bytes in the content of a field */ + u64 offset64; /* 64-bit offset */ u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ Mem *pReg; /* PseudoTable input register */ + pC = p->apCsr[pOp->p1]; p2 = pOp->p2; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + + /* If the cursor cache is stale, bring it up-to-date */ + rc = sqlite3VdbeCursorMoveto(&pC, &p2); + if( rc ) goto abort_due_to_error; + + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); assert( pOp->p1>=0 && pOp->p1nCursor ); - pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( p2nField ); - aType = pC->aType; - aOffset = aType + pC->nField; -#ifndef SQLITE_OMIT_VIRTUALTABLE - assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */ -#endif - pCrsr = pC->pCursor; - assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* pCrsr NULL on PseudoTables */ - assert( pCrsr!=0 || pC->nullRow ); /* pC->nullRow on PseudoTables */ + aOffset = pC->aOffset; + assert( pC->eCurType!=CURTYPE_VTAB ); + assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); + assert( pC->eCurType!=CURTYPE_SORTER ); + pCrsr = pC->uc.pCursor; - /* If the cursor cache is stale, bring it up-to-date */ - rc = sqlite3VdbeCursorMoveto(pC); - if( rc ) goto abort_due_to_error; - if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){ + if( pC->cacheStatus!=p->cacheCtr ){ /*OPTIMIZATION-IF-FALSE*/ if( pC->nullRow ){ - if( pCrsr==0 ){ - assert( pC->pseudoTableReg>0 ); - pReg = &aMem[pC->pseudoTableReg]; + if( pC->eCurType==CURTYPE_PSEUDO ){ + assert( pC->uc.pseudoTableReg>0 ); + pReg = &aMem[pC->uc.pseudoTableReg]; assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); pC->payloadSize = pC->szRow = avail = pReg->n; pC->aRow = (u8*)pReg->z; }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); goto op_column_out; } }else{ + assert( pC->eCurType==CURTYPE_BTREE ); assert( pCrsr ); - if( pC->isTable==0 ){ - assert( sqlite3BtreeCursorIsValid(pCrsr) ); - VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64); - assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ - /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the - ** payload size, so it is impossible for payloadSize64 to be - ** larger than 32 bits. */ - assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 ); - pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail); - pC->payloadSize = (u32)payloadSize64; - }else{ - assert( sqlite3BtreeCursorIsValid(pCrsr) ); - VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize); - assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ - pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail); - } + assert( sqlite3BtreeCursorIsValid(pCrsr) ); + pC->payloadSize = sqlite3BtreePayloadSize(pCrsr); + pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &avail); assert( avail<=65536 ); /* Maximum page size is 64KiB */ if( pC->payloadSize <= (u32)avail ){ pC->szRow = pC->payloadSize; + }else if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; }else{ pC->szRow = avail; } - if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } } pC->cacheStatus = p->cacheCtr; pC->iHdrOffset = getVarint32(pC->aRow, offset); pC->nHdrParsed = 0; aOffset[0] = offset; - if( availaRow does not have to hold the entire row, but it does at least ** need to cover the header of the record. If pC->aRow does not contain ** the complete header, then set it to zero, forcing the header to be ** dynamically allocated. */ pC->aRow = 0; pC->szRow = 0; - } - /* Make sure a corrupt database has not given us an oversize header. - ** Do this now to avoid an oversize memory allocation. - ** - ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte - ** types use so much data space that there can only be 4096 and 32 of - ** them, respectively. So the maximum header length results from a - ** 3-byte type for each of the maximum of 32768 columns plus three - ** extra bytes for the header length itself. 32768*3 + 3 = 98307. - */ - if( offset > 98307 || offset > pC->payloadSize ){ - rc = SQLITE_CORRUPT_BKPT; - goto op_column_error; + /* Make sure a corrupt database has not given us an oversize header. + ** Do this now to avoid an oversize memory allocation. + ** + ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte + ** types use so much data space that there can only be 4096 and 32 of + ** them, respectively. So the maximum header length results from a + ** 3-byte type for each of the maximum of 32768 columns plus three + ** extra bytes for the header length itself. 32768*3 + 3 = 98307. + */ + if( offset > 98307 || offset > pC->payloadSize ){ + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } + }else if( offset>0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* The following goto is an optimization. It can be omitted and + ** everything will still work. But OP_Column is measurably faster + ** by skipping the subsequent conditional, which is always true. + */ + zData = pC->aRow; + assert( pC->nHdrParsed<=p2 ); /* Conditional skipped */ + goto op_column_read_header; } } /* Make sure at least the first p2+1 entries of the header have been - ** parsed and valid information is in aOffset[] and aType[]. + ** parsed and valid information is in aOffset[] and pC->aType[]. */ if( pC->nHdrParsed<=p2 ){ /* If there is more header available for parsing in the record, try @@ -73337,62 +82941,52 @@ case OP_Column: { /* Make sure zData points to enough of the record to cover the header. */ if( pC->aRow==0 ){ memset(&sMem, 0, sizeof(sMem)); - rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], - !pC->isTable, &sMem); - if( rc!=SQLITE_OK ){ - goto op_column_error; - } + rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], !pC->isTable, &sMem); + if( rc!=SQLITE_OK ) goto abort_due_to_error; zData = (u8*)sMem.z; }else{ zData = pC->aRow; } - /* Fill in aType[i] and aOffset[i] values through the p2-th field. */ + /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */ + op_column_read_header: i = pC->nHdrParsed; - offset = aOffset[i]; + offset64 = aOffset[i]; zHdr = zData + pC->iHdrOffset; zEndHdr = zData + aOffset[0]; - assert( i<=p2 && zHdraType[i++] = t; + aOffset[i] = (u32)(offset64 & 0xffffffff); }while( i<=p2 && zHdr=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize)) + || (offset64 > pC->payloadSize) + ){ + if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } + pC->nHdrParsed = i; pC->iHdrOffset = (u32)(zHdr - zData); - if( pC->aRow==0 ){ - sqlite3VdbeMemRelease(&sMem); - sMem.flags = MEM_Null; - } - - /* If we have read more header data than was contained in the header, - ** or if the end of the last field appears to be past the end of the - ** record, or if the end of the last field appears to be before the end - ** of the record (when all fields present), then we must be dealing - ** with a corrupt database. - */ - if( (zHdr > zEndHdr) - || (offset > pC->payloadSize) - || (zHdr==zEndHdr && offset!=pC->payloadSize) - ){ - rc = SQLITE_CORRUPT_BKPT; - goto op_column_error; - } + if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); + }else{ + t = 0; } - /* If after trying to extra new entries from the header, nHdrParsed is + /* If after trying to extract new entries from the header, nHdrParsed is ** still not up to p2, that means that the record has fewer than p2 ** columns. So the result will be either the default value or a NULL. */ @@ -73400,69 +82994,75 @@ case OP_Column: { if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); } goto op_column_out; } + }else{ + t = pC->aType[p2]; } /* Extract the content for the p2+1-th column. Control can only - ** reach this point if aOffset[p2], aOffset[p2+1], and aType[p2] are + ** reach this point if aOffset[p2], aOffset[p2+1], and pC->aType[p2] are ** all valid. */ assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); assert( sqlite3VdbeCheckMemInvariants(pDest) ); + if( VdbeMemDynamic(pDest) ){ + sqlite3VdbeMemSetNull(pDest); + } + assert( t==pC->aType[p2] ); if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ - VdbeMemRelease(pDest); - sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest); + zData = pC->aRow + aOffset[p2]; + if( t<12 ){ + sqlite3VdbeSerialGet(zData, t, pDest); + }else{ + /* If the column value is a string, we need a persistent value, not + ** a MEM_Ephem value. This branch is a fast short-cut that is equivalent + ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize(). + */ + static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term }; + pDest->n = len = (t-12)/2; + pDest->enc = encoding; + if( pDest->szMalloc < len+2 ){ + pDest->flags = MEM_Null; + if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; + }else{ + pDest->z = pDest->zMalloc; + } + memcpy(pDest->z, zData, len); + pDest->z[len] = 0; + pDest->z[len+1] = 0; + pDest->flags = aFlag[t&1]; + } }else{ + pDest->enc = encoding; /* This branch happens only when content is on overflow pages */ - t = aType[p2]; if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) || (len = sqlite3VdbeSerialTypeLen(t))==0 ){ - /* Content is irrelevant for the typeof() function and for - ** the length(X) function if X is a blob. So we might as well use - ** bogus content rather than reading content from disk. NULL works - ** for text and blob and whatever is in the payloadSize64 variable - ** will work for everything else. Content is also irrelevant if - ** the content length is 0. */ - zData = t<=13 ? (u8*)&payloadSize64 : 0; - sMem.zMalloc = 0; + /* Content is irrelevant for + ** 1. the typeof() function, + ** 2. the length(X) function if X is a blob, and + ** 3. if the content length is zero. + ** So we might as well use bogus content rather than reading + ** content from disk. */ + static u8 aZero[8]; /* This is the bogus content */ + sqlite3VdbeSerialGet(aZero, t, pDest); }else{ - memset(&sMem, 0, sizeof(sMem)); - sqlite3VdbeMemMove(&sMem, pDest); rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, - &sMem); - if( rc!=SQLITE_OK ){ - goto op_column_error; - } - zData = (u8*)sMem.z; - } - sqlite3VdbeSerialGet(zData, t, pDest); - /* If we dynamically allocated space to hold the data (in the - ** sqlite3VdbeMemFromBtree() call above) then transfer control of that - ** dynamically allocated space over to the pDest structure. - ** This prevents a memory copy. */ - if( sMem.zMalloc ){ - assert( sMem.z==sMem.zMalloc ); - assert( VdbeMemDynamic(pDest)==0 ); - assert( (pDest->flags & (MEM_Blob|MEM_Str))==0 || pDest->z==sMem.z ); - pDest->flags &= ~(MEM_Ephem|MEM_Static); - pDest->flags |= MEM_Term; - pDest->z = sMem.z; - pDest->zMalloc = sMem.zMalloc; + pDest); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + pDest->flags &= ~MEM_Ephem; } } - pDest->enc = encoding; op_column_out: - Deephemeralize(pDest); -op_column_error: UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); break; @@ -73486,7 +83086,7 @@ case OP_Affinity: { assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; while( (cAff = *(zAffinity++))!=0 ){ - assert( pIn1 <= &p->aMem[(p->nMem-p->nCursor)] ); + assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); applyAffinity(pIn1, cAff, encoding); pIn1++; @@ -73508,7 +83108,7 @@ case OP_Affinity: { ** The mapping from character to affinity is given by the SQLITE_AFF_ ** macros defined in sqliteInt.h. ** -** If P4 is NULL then all index fields have the affinity NONE. +** If P4 is NULL then all index fields have the affinity BLOB. */ case OP_MakeRecord: { u8 *zNewRecord; /* A buffer to hold the data for the new record */ @@ -73516,7 +83116,7 @@ case OP_MakeRecord: { u64 nData; /* Number of bytes of data space */ int nHdr; /* Number of bytes of header space */ i64 nByte; /* Data space required for this record */ - int nZero; /* Number of zero bytes at the end of the record */ + i64 nZero; /* Number of zero bytes at the end of the record */ int nVarint; /* Number of bytes in a varint */ u32 serial_type; /* Type field */ Mem *pData0; /* First field to be combined into the record */ @@ -73526,7 +83126,7 @@ case OP_MakeRecord: { int file_format; /* File format to use for encoding */ int i; /* Space used in zNewRecord[] header */ int j; /* Space used in zNewRecord[] content */ - int len; /* Length of a field */ + u32 len; /* Length of a field */ /* Assuming the record contains N fields, the record format looks ** like this: @@ -73536,7 +83136,7 @@ case OP_MakeRecord: { ** ------------------------------------------------------------------------ ** ** Data(0) is taken from register P1. Data(1) comes from register P1+1 - ** and so froth. + ** and so forth. ** ** Each type field is a varint representing the serial type of the ** corresponding data element (see sqlite3VdbeSerialType()). The @@ -73548,7 +83148,7 @@ case OP_MakeRecord: { nZero = 0; /* Number of zero bytes at the end of the record */ nField = pOp->p1; zAffinity = pOp->p4.z; - assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem-p->nCursor)+1 ); + assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem+1 - p->nCursor)+1 ); pData0 = &aMem[nField]; nField = pOp->p2; pLast = &pData0[nField-1]; @@ -73576,11 +83176,10 @@ case OP_MakeRecord: { pRec = pLast; do{ assert( memIsValid(pRec) ); - serial_type = sqlite3VdbeSerialType(pRec, file_format); - len = sqlite3VdbeSerialTypeLen(serial_type); + pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format, &len); if( pRec->flags & MEM_Zero ){ if( nData ){ - sqlite3VdbeMemExpandBlob(pRec); + if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; }else{ nZero += pRec->u.nZero; len -= pRec->u.nZero; @@ -73590,9 +83189,14 @@ case OP_MakeRecord: { testcase( serial_type==127 ); testcase( serial_type==128 ); nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); - }while( (--pRec)>=pData0 ); + if( pRec==pData0 ) break; + pRec--; + }while(1); - /* Add the initial header varint and total the size */ + /* EVIDENCE-OF: R-22564-11647 The header begins with a single varint + ** which determines the total number of bytes in the header. The varint + ** value is the size of the header in bytes including the size varint + ** itself. */ testcase( nHdr==126 ); testcase( nHdr==127 ); if( nHdr<=126 ){ @@ -73605,16 +83209,16 @@ case OP_MakeRecord: { if( nVarintdb->aLimit[SQLITE_LIMIT_LENGTH] ){ + if( nByte+nZero>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } /* Make sure the output register has a buffer large enough to store ** the new record. The output register (pOp->p3) is not allowed to ** be one of the input registers (because the following call to - ** sqlite3VdbeMemGrow() could clobber the value before it is used). + ** sqlite3VdbeMemClearAndResize() could clobber the value before it is used). */ - if( sqlite3VdbeMemGrow(pOut, (int)nByte, 0) ){ + if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ goto no_mem; } zNewRecord = (u8 *)pOut->z; @@ -73625,17 +83229,20 @@ case OP_MakeRecord: { assert( pData0<=pLast ); pRec = pData0; do{ - serial_type = sqlite3VdbeSerialType(pRec, file_format); + serial_type = pRec->uTemp; + /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more + ** additional varints, one per column. */ i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ + /* EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** immediately follow the header. */ j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); assert( i==nHdr ); assert( j==nByte ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob; - pOut->xDel = 0; if( nZero ){ pOut->u.nZero = nZero; pOut->flags |= MEM_Zero; @@ -73653,14 +83260,17 @@ case OP_MakeRecord: { ** opened by cursor P1 in register P2 */ #ifndef SQLITE_OMIT_BTREECOUNT -case OP_Count: { /* out2-prerelease */ +case OP_Count: { /* out2 */ i64 nEntry; BtCursor *pCrsr; - pCrsr = p->apCsr[pOp->p1]->pCursor; + assert( p->apCsr[pOp->p1]->eCurType==CURTYPE_BTREE ); + pCrsr = p->apCsr[pOp->p1]->uc.pCursor; assert( pCrsr ); nEntry = 0; /* Not needed. Only used to silence a warning. */ rc = sqlite3BtreeCount(pCrsr, &nEntry); + if( rc ) goto abort_due_to_error; + pOut = out2Prerelease(p, pOp); pOut->u.i = nEntry; break; } @@ -73699,8 +83309,7 @@ case OP_Savepoint: { /* A new savepoint cannot be created if there are active write ** statements (i.e. open read/write incremental blob handles). */ - sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - " - "SQL statements in progress"); + sqlite3VdbeError(p, "cannot open savepoint - SQL statements in progress"); rc = SQLITE_BUSY; }else{ nName = sqlite3Strlen30(zName); @@ -73717,7 +83326,7 @@ case OP_Savepoint: { #endif /* Create a new savepoint structure. */ - pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+nName+1); + pNew = sqlite3DbMallocRawNN(db, sizeof(Savepoint)+nName+1); if( pNew ){ pNew->zName = (char *)&pNew[1]; memcpy(pNew->zName, zName, nName+1); @@ -73730,7 +83339,7 @@ case OP_Savepoint: { }else{ db->nSavepoint++; } - + /* Link the new savepoint into the database handle's list. */ pNew->pNext = db->pSavepoint; db->pSavepoint = pNew; @@ -73751,15 +83360,14 @@ case OP_Savepoint: { iSavepoint++; } if( !pSavepoint ){ - sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName); + sqlite3VdbeError(p, "no such savepoint: %s", zName); rc = SQLITE_ERROR; }else if( db->nVdbeWrite>0 && p1==SAVEPOINT_RELEASE ){ /* It is not possible to release (commit) a savepoint if there are ** active write statements. */ - sqlite3SetString(&p->zErrMsg, db, - "cannot release savepoint - SQL statements in progress" - ); + sqlite3VdbeError(p, "cannot release savepoint - " + "SQL statements in progress"); rc = SQLITE_BUSY; }else{ @@ -73774,7 +83382,7 @@ case OP_Savepoint: { } db->autoCommit = 1; if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = pc; + p->pc = (int)(pOp - aOp); db->autoCommit = 0; p->rc = rc = SQLITE_BUSY; goto vdbe_return; @@ -73782,11 +83390,18 @@ case OP_Savepoint: { db->isTransactionSavepoint = 0; rc = p->rc; }else{ + int isSchemaChange; iSavepoint = db->nSavepoint - iSavepoint - 1; if( p1==SAVEPOINT_ROLLBACK ){ + isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, + SQLITE_ABORT_ROLLBACK, + isSchemaChange==0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; } + }else{ + isSchemaChange = 0; } for(ii=0; iinDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); @@ -73794,7 +83409,7 @@ case OP_Savepoint: { goto abort_due_to_error; } } - if( p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ + if( isSchemaChange ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); db->flags = (db->flags | SQLITE_InternChanges); @@ -73826,12 +83441,13 @@ case OP_Savepoint: { db->nDeferredImmCons = pSavepoint->nDeferredImmCons; } - if( !isTransaction ){ + if( !isTransaction || p1==SAVEPOINT_ROLLBACK ){ rc = sqlite3VtabSavepoint(db, p1, iSavepoint); if( rc!=SQLITE_OK ) goto abort_due_to_error; } } } + if( rc ) goto abort_due_to_error; break; } @@ -73848,49 +83464,37 @@ case OP_Savepoint: { case OP_AutoCommit: { int desiredAutoCommit; int iRollback; - int turnOnAC; desiredAutoCommit = pOp->p1; iRollback = pOp->p2; - turnOnAC = desiredAutoCommit && !db->autoCommit; assert( desiredAutoCommit==1 || desiredAutoCommit==0 ); assert( desiredAutoCommit==1 || iRollback==0 ); assert( db->nVdbeActive>0 ); /* At least this one VM is active */ assert( p->bIsReader ); -#if 0 - if( turnOnAC && iRollback && db->nVdbeActive>1 ){ - /* If this instruction implements a ROLLBACK and other VMs are - ** still running, and a transaction is active, return an error indicating - ** that the other VMs must complete first. - */ - sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " - "SQL statements in progress"); - rc = SQLITE_BUSY; - }else -#endif - if( turnOnAC && !iRollback && db->nVdbeWrite>0 ){ - /* If this instruction implements a COMMIT and other VMs are writing - ** return an error indicating that the other VMs must complete first. - */ - sqlite3SetString(&p->zErrMsg, db, "cannot commit transaction - " - "SQL statements in progress"); - rc = SQLITE_BUSY; - }else if( desiredAutoCommit!=db->autoCommit ){ + if( desiredAutoCommit!=db->autoCommit ){ if( iRollback ){ assert( desiredAutoCommit==1 ); sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); db->autoCommit = 1; + }else if( desiredAutoCommit && db->nVdbeWrite>0 ){ + /* If this instruction implements a COMMIT and other VMs are writing + ** return an error indicating that the other VMs must complete first. + */ + sqlite3VdbeError(p, "cannot commit transaction - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + goto abort_due_to_error; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = pc; - db->autoCommit = (u8)(1-desiredAutoCommit); - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; - } + } + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + db->autoCommit = (u8)(1-desiredAutoCommit); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; } assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); @@ -73901,12 +83505,13 @@ case OP_AutoCommit: { } goto vdbe_return; }else{ - sqlite3SetString(&p->zErrMsg, db, + sqlite3VdbeError(p, (!desiredAutoCommit)?"cannot start a transaction within a transaction":( (iRollback)?"cannot rollback - no transaction is active": "cannot commit - no transaction is active")); rc = SQLITE_ERROR; + goto abort_due_to_error; } break; } @@ -73962,12 +83567,14 @@ case OP_Transaction: { if( pBt ){ rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); - if( rc==SQLITE_BUSY ){ - p->pc = pc; - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; - } + testcase( rc==SQLITE_BUSY_SNAPSHOT ); + testcase( rc==SQLITE_BUSY_RECOVERY ); if( rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + p->rc = rc; + goto vdbe_return; + } goto abort_due_to_error; } @@ -73993,7 +83600,11 @@ case OP_Transaction: { p->nStmtDefImmCons = db->nDeferredImmCons; } - /* Gather the schema version number for checking */ + /* Gather the schema version number for checking: + ** IMPLEMENTATION-OF: R-03189-51135 As each SQL statement runs, the schema + ** version is checked to ensure that the schema has not changed since the + ** SQL statement was prepared. + */ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta); iGen = db->aDb[pOp->p1].pSchema->iGeneration; }else{ @@ -74022,6 +83633,7 @@ case OP_Transaction: { p->expired = 1; rc = SQLITE_SCHEMA; } + if( rc ) goto abort_due_to_error; break; } @@ -74037,7 +83649,7 @@ case OP_Transaction: { ** must be started or there must be an open cursor) before ** executing this instruction. */ -case OP_ReadCookie: { /* out2-prerelease */ +case OP_ReadCookie: { /* out2 */ int iMeta; int iDb; int iCookie; @@ -74051,21 +83663,22 @@ case OP_ReadCookie: { /* out2-prerelease */ assert( DbMaskTest(p->btreeMask, iDb) ); sqlite3BtreeGetMeta(db->aDb[iDb].pBt, iCookie, (u32 *)&iMeta); + pOut = out2Prerelease(p, pOp); pOut->u.i = iMeta; break; } /* Opcode: SetCookie P1 P2 P3 * * ** -** Write the content of register P3 (interpreted as an integer) -** into cookie number P2 of database P1. P2==1 is the schema version. -** P2==2 is the database format. P2==3 is the recommended pager cache +** Write the integer value P3 into cookie number P2 of database P1. +** P2==1 is the schema version. P2==2 is the database format. +** P2==3 is the recommended pager cache ** size, and so forth. P1==0 is the main database file and P1==1 is the ** database file used to store temporary tables. ** ** A transaction must be started before executing this opcode. */ -case OP_SetCookie: { /* in3 */ +case OP_SetCookie: { Db *pDb; assert( pOp->p2p1>=0 && pOp->p1nDb ); @@ -74074,17 +83687,15 @@ case OP_SetCookie: { /* in3 */ pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); - pIn3 = &aMem[pOp->p3]; - sqlite3VdbeMemIntegerify(pIn3); /* See note about index shifting on OP_ReadCookie */ - rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); + rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3); if( pOp->p2==BTREE_SCHEMA_VERSION ){ /* When the schema cookie changes, record the new cookie internally */ - pDb->pSchema->schema_cookie = (int)pIn3->u.i; + pDb->pSchema->schema_cookie = pOp->p3; db->flags |= SQLITE_InternChanges; }else if( pOp->p2==BTREE_FILE_FORMAT ){ /* Record changes in the file format */ - pDb->pSchema->file_format = (u8)pIn3->u.i; + pDb->pSchema->file_format = pOp->p3; } if( pOp->p1==1 ){ /* Invalidate all prepared statements whenever the TEMP database @@ -74092,6 +83703,7 @@ case OP_SetCookie: { /* in3 */ sqlite3ExpirePreparedStatements(db); p->expired = 0; } + if( rc ) goto abort_due_to_error; break; } @@ -74161,20 +83773,6 @@ case OP_SetCookie: { /* in3 */ ** See also OpenRead. */ case OP_ReopenIdx: { - VdbeCursor *pCur; - - assert( pOp->p5==0 ); - assert( pOp->p4type==P4_KEYINFO ); - pCur = p->apCsr[pOp->p1]; - if( pCur && pCur->pgnoRoot==(u32)pOp->p2 ){ - assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ - break; - } - /* If the cursor is not currently open or is open on a different - ** index, then fall through into OP_OpenRead to force a reopen */ -} -case OP_OpenRead: -case OP_OpenWrite: { int nField; KeyInfo *pKeyInfo; int p2; @@ -74184,15 +83782,26 @@ case OP_OpenWrite: { VdbeCursor *pCur; Db *pDb; - assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR))==pOp->p5 ); - assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 ); + assert( pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); + assert( pOp->p4type==P4_KEYINFO ); + pCur = p->apCsr[pOp->p1]; + if( pCur && pCur->pgnoRoot==(u32)pOp->p2 ){ + assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ + goto open_cursor_set_hints; + } + /* If the cursor is not currently open or is open on a different + ** index, then fall through into OP_OpenRead to force a reopen */ +case OP_OpenRead: +case OP_OpenWrite: + + assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); assert( p->bIsReader ); assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx || p->readOnly==0 ); if( p->expired ){ - rc = SQLITE_ABORT; - break; + rc = SQLITE_ABORT_ROLLBACK; + goto abort_due_to_error; } nField = 0; @@ -74205,7 +83814,8 @@ case OP_OpenWrite: { pX = pDb->pBt; assert( pX!=0 ); if( pOp->opcode==OP_OpenWrite ){ - wrFlag = 1; + assert( OPFLAG_FORDELETE==BTREE_FORDELETE ); + wrFlag = BTREE_WRCSR | (pOp->p5 & OPFLAG_FORDELETE); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->file_format < p->minWriteFileFormat ){ p->minWriteFileFormat = pDb->pSchema->file_format; @@ -74215,7 +83825,7 @@ case OP_OpenWrite: { } if( pOp->p5 & OPFLAG_P2ISREG ){ assert( p2>0 ); - assert( p2<=(p->nMem-p->nCursor) ); + assert( p2<=(p->nMem+1 - p->nCursor) ); pIn2 = &aMem[p2]; assert( memIsValid(pIn2) ); assert( (pIn2->flags & MEM_Int)!=0 ); @@ -74225,10 +83835,7 @@ case OP_OpenWrite: { ** that opcode will always set the p2 value to 2 or more or else fail. ** If there were a failure, the prepared statement would have halted ** before reaching this instruction. */ - if( NEVER(p2<2) ) { - rc = SQLITE_CORRUPT_BKPT; - goto abort_due_to_error; - } + assert( p2>=2 ); } if( pOp->p4type==P4_KEYINFO ){ pKeyInfo = pOp->p4.pKeyInfo; @@ -74241,25 +83848,32 @@ case OP_OpenWrite: { assert( pOp->p1>=0 ); assert( nField>=0 ); testcase( nField==0 ); /* Table with INTEGER PRIMARY KEY and nothing else */ - pCur = allocateCursor(p, pOp->p1, nField, iDb, 1); + pCur = allocateCursor(p, pOp->p1, nField, iDb, CURTYPE_BTREE); if( pCur==0 ) goto no_mem; pCur->nullRow = 1; pCur->isOrdered = 1; pCur->pgnoRoot = p2; - rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); +#ifdef SQLITE_DEBUG + pCur->wrFlag = wrFlag; +#endif + rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->uc.pCursor); pCur->pKeyInfo = pKeyInfo; - assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); - sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR)); - - /* Since it performs no memory allocation or IO, the only value that - ** sqlite3BtreeCursor() may return is SQLITE_OK. */ - assert( rc==SQLITE_OK ); - /* Set the VdbeCursor.isTable variable. Previous versions of ** SQLite used to check if the root-page flags were sane at this point ** and report database corruption if they were not, but this check has ** since moved into the btree layer. */ pCur->isTable = pOp->p4type!=P4_KEYINFO; + +open_cursor_set_hints: + assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); + assert( OPFLAG_SEEKEQ==BTREE_SEEK_EQ ); + testcase( pOp->p5 & OPFLAG_BULKCSR ); +#ifdef SQLITE_ENABLE_CURSOR_HINTS + testcase( pOp->p2 & OPFLAG_SEEKEQ ); +#endif + sqlite3BtreeCursorHintFlags(pCur->uc.pCursor, + (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); + if( rc ) goto abort_due_to_error; break; } @@ -74302,7 +83916,7 @@ case OP_OpenEphemeral: { SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; pCx->isEphemeral = 1; @@ -74326,35 +83940,61 @@ case OP_OpenEphemeral: { assert( pKeyInfo->db==db ); assert( pKeyInfo->enc==ENC(db) ); pCx->pKeyInfo = pKeyInfo; - rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, pKeyInfo, pCx->pCursor); + rc = sqlite3BtreeCursor(pCx->pBt, pgno, BTREE_WRCSR, + pKeyInfo, pCx->uc.pCursor); } pCx->isTable = 0; }else{ - rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, 1, 0, pCx->pCursor); + rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, BTREE_WRCSR, + 0, pCx->uc.pCursor); pCx->isTable = 1; } } + if( rc ) goto abort_due_to_error; pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); break; } -/* Opcode: SorterOpen P1 P2 * P4 * +/* Opcode: SorterOpen P1 P2 P3 P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. +** +** If argument P3 is non-zero, then it indicates that the sorter may +** assume that a stable sort considering the first P3 fields of each +** key is sufficient to produce the required results. */ case OP_SorterOpen: { VdbeCursor *pCx; assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_SORTER); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); assert( pCx->pKeyInfo->enc==ENC(db) ); - rc = sqlite3VdbeSorterInit(db, pCx); + rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx); + if( rc ) goto abort_due_to_error; + break; +} + +/* Opcode: SequenceTest P1 P2 * * * +** Synopsis: if( cursor[P1].ctr++ ) pc = P2 +** +** P1 is a sorter cursor. If the sequence counter is currently zero, jump +** to P2. Regardless of whether or not the jump is taken, increment the +** the sequence value. +*/ +case OP_SequenceTest: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + if( (pC->seqCount++)==0 ){ + goto jump_to_p2; + } break; } @@ -74379,10 +84019,10 @@ case OP_OpenPseudo: { assert( pOp->p1>=0 ); assert( pOp->p3>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, 0); + pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, CURTYPE_PSEUDO); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; - pCx->pseudoTableReg = pOp->p2; + pCx->uc.pseudoTableReg = pOp->p2; pCx->isTable = 1; assert( pOp->p5==0 ); break; @@ -74400,6 +84040,26 @@ case OP_Close: { break; } +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK +/* Opcode: ColumnsUsed P1 * * P4 * +** +** This opcode (which only exists if SQLite was compiled with +** SQLITE_ENABLE_COLUMN_USED_MASK) identifies which columns of the +** table or index for cursor P1 are used. P4 is a 64-bit integer +** (P4_INT64) in which the first 63 bits are one for each of the +** first 63 columns of the table or index that are actually used +** by the cursor. The high-order bit is set if any column after +** the 64th is used. +*/ +case OP_ColumnsUsed: { + VdbeCursor *pC; + pC = p->apCsr[pOp->p1]; + assert( pC->eCurType==CURTYPE_BTREE ); + pC->maskUsed = *(u64*)pOp->p4.pI64; + break; +} +#endif + /* Opcode: SeekGE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** @@ -74412,6 +84072,13 @@ case OP_Close: { ** is greater than or equal to the key value. If there are no records ** greater than or equal to the key and P2 is not zero, then jump to P2. ** +** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this +** opcode will always land on a record that equally equals the key, or +** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this +** opcode must be followed by an IdxLE opcode with the same arguments. +** The IdxLE opcode will be skipped if this opcode succeeds, but the +** IdxLE opcode will be used on subsequent loop iterations. +** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. @@ -74470,42 +84137,56 @@ case OP_Close: { ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. ** +** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this +** opcode will always land on a record that equally equals the key, or +** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this +** opcode must be followed by an IdxGE opcode with the same arguments. +** The IdxGE opcode will be skipped if this opcode succeeds, but the +** IdxGE opcode will be used on subsequent loop iterations. +** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ case OP_SeekLT: /* jump, in3 */ case OP_SeekLE: /* jump, in3 */ case OP_SeekGE: /* jump, in3 */ case OP_SeekGT: { /* jump, in3 */ - int res; - int oc; - VdbeCursor *pC; - UnpackedRecord r; - int nField; - i64 iKey; /* The rowid we are to seek to */ + int res; /* Comparison result */ + int oc; /* Opcode */ + VdbeCursor *pC; /* The cursor to seek */ + UnpackedRecord r; /* The key to seek for */ + int nField; /* Number of columns or fields in the key */ + i64 iKey; /* The rowid we are to seek to */ + int eqOnly; /* Only interested in == results */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p2!=0 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pseudoTableReg==0 ); + assert( pC->eCurType==CURTYPE_BTREE ); assert( OP_SeekLE == OP_SeekLT+1 ); assert( OP_SeekGE == OP_SeekLT+2 ); assert( OP_SeekGT == OP_SeekLT+3 ); assert( pC->isOrdered ); - assert( pC->pCursor!=0 ); + assert( pC->uc.pCursor!=0 ); oc = pOp->opcode; + eqOnly = 0; pC->nullRow = 0; #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif + if( pC->isTable ){ + /* The BTREE_SEEK_EQ flag is only set on index cursors */ + assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 ); + /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do - ** the seek, so covert it. */ + ** the seek, so convert it. */ pIn3 = &aMem[pOp->p3]; - ApplyNumericAffinity(pIn3); + if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn3, 0); + } iKey = sqlite3VdbeIntValue(pIn3); - pC->rowidIsValid = 0; /* If the P3 value could not be converted into an integer without ** loss of information, then special processing is required... */ @@ -74513,7 +84194,7 @@ case OP_SeekGT: { /* jump, in3 */ if( (pIn3->flags & MEM_Real)==0 ){ /* If the P3 value cannot be converted into any kind of a number, ** then the seek is not possible, so jump to P2 */ - pc = pOp->p2 - 1; VdbeBranchTaken(1,2); + VdbeBranchTaken(1,2); goto jump_to_p2; break; } @@ -74524,7 +84205,7 @@ case OP_SeekGT: { /* jump, in3 */ ** (x > 4.9) -> (x >= 5) ** (x <= 4.9) -> (x < 5) */ - if( pIn3->r<(double)iKey ){ + if( pIn3->u.r<(double)iKey ){ assert( OP_SeekGE==(OP_SeekGT-1) ); assert( OP_SeekLT==(OP_SeekLE-1) ); assert( (OP_SeekLE & 0x0001)==(OP_SeekGT & 0x0001) ); @@ -74533,22 +84214,33 @@ case OP_SeekGT: { /* jump, in3 */ /* If the approximation iKey is smaller than the actual real search ** term, substitute <= for < and > for >=. */ - else if( pIn3->r>(double)iKey ){ + else if( pIn3->u.r>(double)iKey ){ assert( OP_SeekLE==(OP_SeekLT+1) ); assert( OP_SeekGT==(OP_SeekGE+1) ); assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) ); if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++; } } - rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res); + rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)iKey, 0, &res); + pC->movetoTarget = iKey; /* Used by OP_Delete */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - if( res==0 ){ - pC->rowidIsValid = 1; - pC->lastRowid = iKey; - } }else{ + /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and + ** OP_SeekLE opcodes are allowed, and these must be immediately followed + ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key. + */ + if( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ) ){ + eqOnly = 1; + assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE ); + assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + assert( pOp[1].p1==pOp[0].p1 ); + assert( pOp[1].p2==pOp[0].p2 ); + assert( pOp[1].p3==pOp[0].p3 ); + assert( pOp[1].p4.i==pOp[0].p4.i ); + } + nField = pOp->p4.i; assert( pOp->p4type==P4_INT32 ); assert( nField>0 ); @@ -74572,12 +84264,15 @@ case OP_SeekGT: { /* jump, in3 */ #ifdef SQLITE_DEBUG { int i; for(i=0; ipCursor, &r, 0, 0, &res); + r.eqSeen = 0; + rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, &r, 0, 0, &res); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - pC->rowidIsValid = 0; + if( eqOnly && r.eqSeen==0 ){ + assert( res!=0 ); + goto seek_not_found; + } } pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -74587,9 +84282,8 @@ case OP_SeekGT: { /* jump, in3 */ if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); if( res<0 || (res==0 && oc==OP_SeekGT) ){ res = 0; - rc = sqlite3BtreeNext(pC->pCursor, &res); + rc = sqlite3BtreeNext(pC->uc.pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; - pC->rowidIsValid = 0; }else{ res = 0; } @@ -74597,51 +84291,27 @@ case OP_SeekGT: { /* jump, in3 */ assert( oc==OP_SeekLT || oc==OP_SeekLE ); if( res>0 || (res==0 && oc==OP_SeekLT) ){ res = 0; - rc = sqlite3BtreePrevious(pC->pCursor, &res); + rc = sqlite3BtreePrevious(pC->uc.pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; - pC->rowidIsValid = 0; }else{ /* res might be negative because the table is empty. Check to ** see if this is the case. */ - res = sqlite3BtreeEof(pC->pCursor); + res = sqlite3BtreeEof(pC->uc.pCursor); } } +seek_not_found: assert( pOp->p2>0 ); VdbeBranchTaken(res!=0,2); if( res ){ - pc = pOp->p2 - 1; + goto jump_to_p2; + }else if( eqOnly ){ + assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + pOp++; /* Skip the OP_IdxLt or OP_IdxGT that follows */ } break; } -/* Opcode: Seek P1 P2 * * * -** Synopsis: intkey=r[P2] -** -** P1 is an open table cursor and P2 is a rowid integer. Arrange -** for P1 to move so that it points to the rowid given by P2. -** -** This is actually a deferred seek. Nothing actually happens until -** the cursor is used to read a record. That way, if no reads -** occur, no unnecessary I/O happens. -*/ -case OP_Seek: { /* in2 */ - VdbeCursor *pC; - - assert( pOp->p1>=0 && pOp->p1nCursor ); - pC = p->apCsr[pOp->p1]; - assert( pC!=0 ); - assert( pC->pCursor!=0 ); - assert( pC->isTable ); - pC->nullRow = 0; - pIn2 = &aMem[pOp->p2]; - pC->movetoTarget = sqlite3VdbeIntValue(pIn2); - pC->rowidIsValid = 0; - pC->deferredMoveto = 1; - break; -} - - /* Opcode: Found P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** @@ -74705,6 +84375,7 @@ case OP_NoConflict: /* jump, in3 */ case OP_NotFound: /* jump, in3 */ case OP_Found: { /* jump, in3 */ int alreadyExists; + int takeJump; int ii; VdbeCursor *pC; int res; @@ -74725,48 +84396,48 @@ case OP_Found: { /* jump, in3 */ pC->seekOp = pOp->opcode; #endif pIn3 = &aMem[pOp->p3]; - assert( pC->pCursor!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); assert( pC->isTable==0 ); - pFree = 0; /* Not needed. Only used to suppress a compiler warning. */ + pFree = 0; if( pOp->p4.i>0 ){ r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p4.i; r.aMem = pIn3; +#ifdef SQLITE_DEBUG for(ii=0; iip3+ii, &r.aMem[ii]); -#endif } +#endif pIdxKey = &r; }else{ pIdxKey = sqlite3VdbeAllocUnpackedRecord( pC->pKeyInfo, aTempRec, sizeof(aTempRec), &pFree - ); + ); if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); - assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ + (void)ExpandBlob(pIn3); sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); } pIdxKey->default_rc = 0; + takeJump = 0; if( pOp->opcode==OP_NoConflict ){ /* For the OP_NoConflict opcode, take the jump if any of the ** input fields are NULL, since any key with a NULL will not ** conflict */ - for(ii=0; iip2 - 1; VdbeBranchTaken(1,2); + for(ii=0; iinField; ii++){ + if( pIdxKey->aMem[ii].flags & MEM_Null ){ + takeJump = 1; break; } } } - rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, pIdxKey, 0, 0, &res); - if( pOp->p4.i==0 ){ - sqlite3DbFree(db, pFree); - } + rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, pIdxKey, 0, 0, &res); + sqlite3DbFree(db, pFree); if( rc!=SQLITE_OK ){ - break; + goto abort_due_to_error; } pC->seekResult = res; alreadyExists = (res==0); @@ -74775,22 +84446,28 @@ case OP_Found: { /* jump, in3 */ pC->cacheStatus = CACHE_STALE; if( pOp->opcode==OP_Found ){ VdbeBranchTaken(alreadyExists!=0,2); - if( alreadyExists ) pc = pOp->p2 - 1; + if( alreadyExists ) goto jump_to_p2; }else{ - VdbeBranchTaken(alreadyExists==0,2); - if( !alreadyExists ) pc = pOp->p2 - 1; + VdbeBranchTaken(takeJump||alreadyExists==0,2); + if( takeJump || !alreadyExists ) goto jump_to_p2; } break; } -/* Opcode: NotExists P1 P2 P3 * * +/* Opcode: SeekRowid P1 P2 P3 * * ** Synopsis: intkey=r[P3] ** ** P1 is the index of a cursor open on an SQL table btree (with integer -** keys). P3 is an integer rowid. If P1 does not contain a record with -** rowid P3 then jump immediately to P2. If P1 does contain a record -** with rowid P3 then leave the cursor pointing at that record and fall -** through to the next instruction. +** keys). If register P3 does not contain an integer or if P1 does not +** contain a record with rowid P3 then jump immediately to P2. +** Or, if P2 is 0, raise an SQLITE_CORRUPT error. If P1 does contain +** a record with rowid P3 then +** leave the cursor pointing at that record and fall through to the next +** instruction. +** +** The OP_NotExists opcode performs the same operation, but with OP_NotExists +** the P3 register must be guaranteed to contain an integer value. With this +** opcode, register P3 might not contain an integer. ** ** The OP_NotFound opcode performs the same operation on index btrees ** (with arbitrary multi-value keys). @@ -74799,14 +84476,44 @@ case OP_Found: { /* jump, in3 */ ** in either direction. In other words, the Next and Prev opcodes will ** not work following this opcode. ** -** See also: Found, NotFound, NoConflict +** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_NotExists: { /* jump, in3 */ +/* Opcode: NotExists P1 P2 P3 * * +** Synopsis: intkey=r[P3] +** +** P1 is the index of a cursor open on an SQL table btree (with integer +** keys). P3 is an integer rowid. If P1 does not contain a record with +** rowid P3 then jump immediately to P2. Or, if P2 is 0, raise an +** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then +** leave the cursor pointing at that record and fall through to the next +** instruction. +** +** The OP_SeekRowid opcode performs the same operation but also allows the +** P3 register to contain a non-integer value, in which case the jump is +** always taken. This opcode requires that P3 always contain an integer. +** +** The OP_NotFound opcode performs the same operation on index btrees +** (with arbitrary multi-value keys). +** +** This opcode leaves the cursor in a state where it cannot be advanced +** in either direction. In other words, the Next and Prev opcodes will +** not work following this opcode. +** +** See also: Found, NotFound, NoConflict, SeekRowid +*/ +case OP_SeekRowid: { /* jump, in3 */ VdbeCursor *pC; BtCursor *pCrsr; int res; u64 iKey; + pIn3 = &aMem[pOp->p3]; + if( (pIn3->flags & MEM_Int)==0 ){ + applyAffinity(pIn3, SQLITE_AFF_NUMERIC, encoding); + if( (pIn3->flags & MEM_Int)==0 ) goto jump_to_p2; + } + /* Fall through into OP_NotExists */ +case OP_NotExists: /* jump, in3 */ pIn3 = &aMem[pOp->p3]; assert( pIn3->flags & MEM_Int ); assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -74816,23 +84523,28 @@ case OP_NotExists: { /* jump, in3 */ pC->seekOp = 0; #endif assert( pC->isTable ); - assert( pC->pseudoTableReg==0 ); - pCrsr = pC->pCursor; + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; assert( pCrsr!=0 ); res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); - pC->lastRowid = pIn3->u.i; - pC->rowidIsValid = res==0 ?1:0; + assert( rc==SQLITE_OK || res==0 ); + pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; pC->deferredMoveto = 0; VdbeBranchTaken(res!=0,2); - if( res!=0 ){ - pc = pOp->p2 - 1; - assert( pC->rowidIsValid==0 ); - } pC->seekResult = res; + if( res!=0 ){ + assert( rc==SQLITE_OK ); + if( pOp->p2==0 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + goto jump_to_p2; + } + } + if( rc ) goto abort_due_to_error; break; } @@ -74844,9 +84556,11 @@ case OP_NotExists: { /* jump, in3 */ ** The sequence number on the cursor is incremented after this ** instruction. */ -case OP_Sequence: { /* out2-prerelease */ +case OP_Sequence: { /* out2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( p->apCsr[pOp->p1]!=0 ); + assert( p->apCsr[pOp->p1]->eCurType!=CURTYPE_VTAB ); + pOut = out2Prerelease(p, pOp); pOut->u.i = p->apCsr[pOp->p1]->seqCount++; break; } @@ -74867,7 +84581,7 @@ case OP_Sequence: { /* out2-prerelease */ ** generated record number. This P3 mechanism is used to help implement the ** AUTOINCREMENT feature. */ -case OP_NewRowid: { /* out2-prerelease */ +case OP_NewRowid: { /* out2 */ i64 v; /* The new rowid */ VdbeCursor *pC; /* Cursor of table to get the new rowid */ int res; /* Result of an sqlite3BtreeLast() */ @@ -74877,12 +84591,13 @@ case OP_NewRowid: { /* out2-prerelease */ v = 0; res = 0; + pOut = out2Prerelease(p, pOp); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - if( NEVER(pC->pCursor==0) ){ - /* The zero initialization above is all that is needed */ - }else{ + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + { /* The next rowid or record number (different terms for the same ** thing) is obtained in a two-step algorithm. ** @@ -74909,16 +84624,15 @@ case OP_NewRowid: { /* out2-prerelease */ #endif if( !pC->useRandomRowid ){ - rc = sqlite3BtreeLast(pC->pCursor, &res); + rc = sqlite3BtreeLast(pC->uc.pCursor, &res); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } if( res ){ v = 1; /* IMP: R-61914-48074 */ }else{ - assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); - rc = sqlite3BtreeKeySize(pC->pCursor, &v); - assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ + assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) ); + v = sqlite3BtreeIntegerKey(pC->uc.pCursor); if( v>=MAX_ROWID ){ pC->useRandomRowid = 1; }else{ @@ -74938,7 +84652,7 @@ case OP_NewRowid: { /* out2-prerelease */ pMem = &pFrame->aMem[pOp->p3]; }else{ /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pMem = &aMem[pOp->p3]; memAboutToChange(p, pMem); } @@ -74964,32 +84678,21 @@ case OP_NewRowid: { /* out2-prerelease */ ** it finds one that is not previously used. */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ - /* on the first attempt, simply do one more than previous */ - v = lastRowid; - v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - v++; /* ensure non-zero */ cnt = 0; - while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, + do{ + sqlite3_randomness(sizeof(v), &v); + v &= (MAX_ROWID>>1); v++; /* Ensure that v is greater than zero */ + }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)v, 0, &res))==SQLITE_OK) && (res==0) - && (++cnt<100)){ - /* collision - try another random rowid */ - sqlite3_randomness(sizeof(v), &v); - if( cnt<5 ){ - /* try "small" random rowids for the initial attempts */ - v &= 0xffffff; - }else{ - v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - } - v++; /* ensure non-zero */ - } - if( rc==SQLITE_OK && res==0 ){ + && (++cnt<100)); + if( rc ) goto abort_due_to_error; + if( res==0 ){ rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; } assert( v>0 ); /* EV: R-40812-03570 */ } - pC->rowidIsValid = 0; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } @@ -75012,10 +84715,12 @@ case OP_NewRowid: { /* out2-prerelease */ ** sqlite3_last_insert_rowid() function (otherwise it is unmodified). ** ** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of -** the last seek operation (OP_NotExists) was a success, then this +** the last seek operation (OP_NotExists or OP_SeekRowid) was a success, +** then this ** operation will not attempt to find the appropriate row before doing ** the insert but will instead overwrite the row that the cursor is -** currently pointing to. Presumably, the prior OP_NotExists opcode +** currently pointing to. Presumably, the prior OP_NotExists or +** OP_SeekRowid opcode ** has already positioned the cursor correctly. This is an optimization ** that boosts performance by avoiding redundant seeks. ** @@ -75024,9 +84729,9 @@ case OP_NewRowid: { /* out2-prerelease */ ** is part of an INSERT operation. The difference is only important to ** the update hook. ** -** Parameter P4 may point to a string containing the table-name, or -** may be NULL. If it is not NULL, then the update-hook -** (sqlite3.xUpdateCallback) is invoked following a successful insert. +** Parameter P4 may point to a Table structure, or may be NULL. If it is +** not NULL, then the update-hook (sqlite3.xUpdateCallback) is invoked +** following a successful insert. ** ** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically ** allocated, then ownership of P2 is transferred to the pseudo-cursor @@ -75038,7 +84743,7 @@ case OP_NewRowid: { /* out2-prerelease */ ** for indices is OP_IdxInsert. */ /* Opcode: InsertInt P1 P2 P3 P4 P5 -** Synopsis: intkey=P3 data=r[P2] +** Synopsis: intkey=P3 data=r[P2] ** ** This works exactly like OP_Insert except that the key is the ** integer value P3, not the value of the integer stored in register P3. @@ -75047,22 +84752,23 @@ case OP_Insert: case OP_InsertInt: { Mem *pData; /* MEM cell holding data for the record to be inserted */ Mem *pKey; /* MEM cell holding key for the record */ - i64 iKey; /* The integer ROWID or key for the record to be inserted */ VdbeCursor *pC; /* Cursor to table into which insert is written */ - int nZero; /* Number of zero-bytes to append */ int seekResult; /* Result of prior seek or 0 if no USESEEKRESULT flag */ const char *zDb; /* database name - used by the update hook */ - const char *zTbl; /* Table name - used by the opdate hook */ + Table *pTab; /* Table structure - used by update and pre-update hooks */ int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ + BtreePayload x; /* Payload to be inserted */ + op = 0; pData = &aMem[pOp->p2]; assert( pOp->p1>=0 && pOp->p1nCursor ); assert( memIsValid(pData) ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pCursor!=0 ); - assert( pC->pseudoTableReg==0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); assert( pC->isTable ); + assert( pOp->p4type==P4_TABLE || pOp->p4type>=P4_STATIC ); REGISTER_TRACE(pOp->p2, pData); if( pOp->opcode==OP_Insert ){ @@ -75070,97 +84776,188 @@ case OP_InsertInt: { assert( pKey->flags & MEM_Int ); assert( memIsValid(pKey) ); REGISTER_TRACE(pOp->p3, pKey); - iKey = pKey->u.i; + x.nKey = pKey->u.i; }else{ assert( pOp->opcode==OP_InsertInt ); - iKey = pOp->p3; + x.nKey = pOp->p3; } + if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ + assert( pC->isTable ); + assert( pC->iDb>=0 ); + zDb = db->aDb[pC->iDb].zDbSName; + pTab = pOp->p4.pTab; + assert( HasRowid(pTab) ); + op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); + }else{ + pTab = 0; /* Not needed. Silence a comiler warning. */ + zDb = 0; /* Not needed. Silence a compiler warning. */ + } + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + /* Invoke the pre-update hook, if any */ + if( db->xPreUpdateCallback + && pOp->p4type==P4_TABLE + && !(pOp->p5 & OPFLAG_ISUPDATE) + ){ + sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey, pOp->p2); + } +#endif + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = x.nKey; if( pData->flags & MEM_Null ){ - pData->z = 0; - pData->n = 0; + x.pData = 0; + x.nData = 0; }else{ assert( pData->flags & (MEM_Blob|MEM_Str) ); + x.pData = pData->z; + x.nData = pData->n; } seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0); if( pData->flags & MEM_Zero ){ - nZero = pData->u.nZero; + x.nZero = pData->u.nZero; }else{ - nZero = 0; + x.nZero = 0; } - rc = sqlite3BtreeInsert(pC->pCursor, 0, iKey, - pData->z, pData->n, nZero, + x.pKey = 0; + rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, (pOp->p5 & OPFLAG_APPEND)!=0, seekResult ); - pC->rowidIsValid = 0; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; /* Invoke the update-hook if required. */ - if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){ - zDb = db->aDb[pC->iDb].zName; - zTbl = pOp->p4.z; - op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); - assert( pC->isTable ); - db->xUpdateCallback(db->pUpdateArg, op, zDb, zTbl, iKey); - assert( pC->iDb>=0 ); + if( rc ) goto abort_due_to_error; + if( db->xUpdateCallback && op ){ + db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, x.nKey); } break; } -/* Opcode: Delete P1 P2 * P4 * +/* Opcode: Delete P1 P2 P3 P4 P5 ** ** Delete the record at which the P1 cursor is currently pointing. ** -** The cursor will be left pointing at either the next or the previous +** If the OPFLAG_SAVEPOSITION bit of the P5 parameter is set, then +** the cursor will be left pointing at either the next or the previous ** record in the table. If it is left pointing at the next record, then -** the next Next instruction will be a no-op. Hence it is OK to delete -** a record from within a Next loop. +** the next Next instruction will be a no-op. As a result, in this case +** it is ok to delete a record from within a Next loop. If +** OPFLAG_SAVEPOSITION bit of P5 is clear, then the cursor will be +** left in an undefined state. ** -** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is -** incremented (otherwise not). +** If the OPFLAG_AUXDELETE bit is set on P5, that indicates that this +** delete one of several associated with deleting a table row and all its +** associated index entries. Exactly one of those deletes is the "primary" +** delete. The others are all on OPFLAG_FORDELETE cursors or else are +** marked with the AUXDELETE flag. +** +** If the OPFLAG_NCHANGE flag of P2 (NB: P2 not P5) is set, then the row +** change count is incremented (otherwise not). ** ** P1 must not be pseudo-table. It has to be a real table with ** multiple rows. ** -** If P4 is not NULL, then it is the name of the table that P1 is -** pointing to. The update hook will be invoked, if it exists. -** If P4 is not NULL then the P1 cursor must have been positioned -** using OP_NotFound prior to invoking this opcode. +** If P4 is not NULL then it points to a Table object. In this case either +** the update or pre-update hook, or both, may be invoked. The P1 cursor must +** have been positioned using OP_NotFound prior to invoking this opcode in +** this case. Specifically, if one is configured, the pre-update hook is +** invoked if P4 is not NULL. The update-hook is invoked if one is configured, +** P4 is not NULL, and the OPFLAG_NCHANGE flag is set in P2. +** +** If the OPFLAG_ISUPDATE flag is set in P2, then P3 contains the address +** of the memory cell that contains the value that the rowid of the row will +** be set to by the update. */ case OP_Delete: { - i64 iKey; VdbeCursor *pC; + const char *zDb; + Table *pTab; + int opflags; + opflags = pOp->p2; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ - iKey = pC->lastRowid; /* Only used for the update hook */ - - /* The OP_Delete opcode always follows an OP_NotExists or OP_Last or - ** OP_Column on the same table without any intervening operations that - ** might move or invalidate the cursor. Hence cursor pC is always pointing - ** to the row to be deleted and the sqlite3VdbeCursorMoveto() operation - ** below is always a no-op and cannot fail. We will run it anyhow, though, - ** to guard against future changes to the code generator. - **/ + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); assert( pC->deferredMoveto==0 ); - rc = sqlite3VdbeCursorMoveto(pC); - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; - rc = sqlite3BtreeDelete(pC->pCursor); +#ifdef SQLITE_DEBUG + if( pOp->p4type==P4_TABLE && HasRowid(pOp->p4.pTab) && pOp->p5==0 ){ + /* If p5 is zero, the seek operation that positioned the cursor prior to + ** OP_Delete will have also set the pC->movetoTarget field to the rowid of + ** the row that is being deleted */ + i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor); + assert( pC->movetoTarget==iKey ); + } +#endif + + /* If the update-hook or pre-update-hook will be invoked, set zDb to + ** the name of the db to pass as to it. Also set local pTab to a copy + ** of p4.pTab. Finally, if p5 is true, indicating that this cursor was + ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set + ** VdbeCursor.movetoTarget to the current rowid. */ + if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ + assert( pC->iDb>=0 ); + assert( pOp->p4.pTab!=0 ); + zDb = db->aDb[pC->iDb].zDbSName; + pTab = pOp->p4.pTab; + if( (pOp->p5 & OPFLAG_SAVEPOSITION)!=0 && pC->isTable ){ + pC->movetoTarget = sqlite3BtreeIntegerKey(pC->uc.pCursor); + } + }else{ + zDb = 0; /* Not needed. Silence a compiler warning. */ + pTab = 0; /* Not needed. Silence a compiler warning. */ + } + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + /* Invoke the pre-update-hook if required. */ + if( db->xPreUpdateCallback && pOp->p4.pTab && HasRowid(pTab) ){ + assert( !(opflags & OPFLAG_ISUPDATE) || (aMem[pOp->p3].flags & MEM_Int) ); + sqlite3VdbePreUpdateHook(p, pC, + (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, + zDb, pTab, pC->movetoTarget, + pOp->p3 + ); + } + if( opflags & OPFLAG_ISNOOP ) break; +#endif + + /* Only flags that can be set are SAVEPOISTION and AUXDELETE */ + assert( (pOp->p5 & ~(OPFLAG_SAVEPOSITION|OPFLAG_AUXDELETE))==0 ); + assert( OPFLAG_SAVEPOSITION==BTREE_SAVEPOSITION ); + assert( OPFLAG_AUXDELETE==BTREE_AUXDELETE ); + +#ifdef SQLITE_DEBUG + if( p->pFrame==0 ){ + if( pC->isEphemeral==0 + && (pOp->p5 & OPFLAG_AUXDELETE)==0 + && (pC->wrFlag & OPFLAG_FORDELETE)==0 + ){ + nExtraDelete++; + } + if( pOp->p2 & OPFLAG_NCHANGE ){ + nExtraDelete--; + } + } +#endif + + rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5); pC->cacheStatus = CACHE_STALE; + if( rc ) goto abort_due_to_error; /* Invoke the update-hook if required. */ - if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){ - db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, - db->aDb[pC->iDb].zName, pOp->p4.z, iKey); - assert( pC->iDb>=0 ); + if( opflags & OPFLAG_NCHANGE ){ + p->nChange++; + if( db->xUpdateCallback && HasRowid(pTab) ){ + db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, pTab->zName, + pC->movetoTarget); + assert( pC->iDb>=0 ); + } } - if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++; + break; } /* Opcode: ResetCount * * * * * @@ -75177,7 +84974,7 @@ case OP_ResetCount: { } /* Opcode: SorterCompare P1 P2 P3 P4 -** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 +** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 ** ** P1 is a sorter cursor. This instruction compares a prefix of the ** record blob in register P3 against a prefix of the entry that @@ -75201,18 +84998,25 @@ case OP_SorterCompare: { assert( pOp->p4type==P4_INT32 ); pIn3 = &aMem[pOp->p3]; nKeyCol = pOp->p4.i; + res = 0; rc = sqlite3VdbeSorterCompare(pC, pIn3, nKeyCol, &res); VdbeBranchTaken(res!=0,2); - if( res ){ - pc = pOp->p2-1; - } + if( rc ) goto abort_due_to_error; + if( res ) goto jump_to_p2; break; }; -/* Opcode: SorterData P1 P2 * * * +/* Opcode: SorterData P1 P2 P3 * * ** Synopsis: r[P2]=data ** ** Write into register P2 the current sorter data for sorter cursor P1. +** Then clear the column header cache on cursor P3. +** +** This opcode is normally use to move a record out of the sorter and into +** a register that is the source for a pseudo-table cursor created using +** OpenPseudo. That pseudo-table cursor is the one that is identified by +** parameter P3. Clearing the P3 column cache as part of this opcode saves +** us from having to issue a separate NullRow instruction to clear that cache. */ case OP_SorterData: { VdbeCursor *pC; @@ -75222,6 +85026,9 @@ case OP_SorterData: { assert( isSorter(pC) ); rc = sqlite3VdbeSorterRowkey(pC, pOut); assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + if( rc ) goto abort_due_to_error; + p->apCsr[pOp->p3]->cacheStatus = CACHE_STALE; break; } @@ -75252,7 +85059,6 @@ case OP_RowData: { VdbeCursor *pC; BtCursor *pCrsr; u32 n; - i64 n64; pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); @@ -75260,41 +85066,36 @@ case OP_RowData: { /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); assert( isSorter(pC)==0 ); assert( pC->isTable || pOp->opcode!=OP_RowData ); assert( pC->isTable==0 || pOp->opcode==OP_RowData ); - assert( pC!=0 ); assert( pC->nullRow==0 ); - assert( pC->pseudoTableReg==0 ); - assert( pC->pCursor!=0 ); - pCrsr = pC->pCursor; - assert( sqlite3BtreeCursorIsValid(pCrsr) ); + assert( pC->uc.pCursor!=0 ); + pCrsr = pC->uc.pCursor; /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or - ** OP_Rewind/Op_Next with no intervening instructions that might invalidate - ** the cursor. Hence the following sqlite3VdbeCursorMoveto() call is always - ** a no-op and can never fail. But we leave it in place as a safety. + ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions + ** that might invalidate the cursor. + ** If this where not the case, on of the following assert()s + ** would fail. Should this ever change (because of changes in the code + ** generator) then the fix would be to insert a call to + ** sqlite3VdbeCursorMoveto(). */ assert( pC->deferredMoveto==0 ); + assert( sqlite3BtreeCursorIsValid(pCrsr) ); +#if 0 /* Not required due to the previous to assert() statements */ rc = sqlite3VdbeCursorMoveto(pC); - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + if( rc!=SQLITE_OK ) goto abort_due_to_error; +#endif - if( pC->isTable==0 ){ - assert( !pC->isTable ); - VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64); - assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ - if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } - n = (u32)n64; - }else{ - VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &n); - assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ - if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } + n = sqlite3BtreePayloadSize(pCrsr); + if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; } - if( sqlite3VdbeMemGrow(pOut, n, 0) ){ + testcase( n==0 ); + if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){ goto no_mem; } pOut->n = n; @@ -75304,6 +85105,7 @@ case OP_RowData: { }else{ rc = sqlite3BtreeData(pCrsr, 0, n, pOut->z); } + if( rc ) goto abort_due_to_error; pOut->enc = SQLITE_UTF8; /* In case the blob is ever cast to text */ UPDATE_MAX_BLOBSIZE(pOut); REGISTER_TRACE(pOp->p2, pOut); @@ -75320,39 +85122,42 @@ case OP_RowData: { ** be a separate OP_VRowid opcode for use with virtual tables, but this ** one opcode now works for both table types. */ -case OP_Rowid: { /* out2-prerelease */ +case OP_Rowid: { /* out2 */ VdbeCursor *pC; i64 v; sqlite3_vtab *pVtab; const sqlite3_module *pModule; + pOut = out2Prerelease(p, pOp); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pseudoTableReg==0 || pC->nullRow ); + assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); if( pC->nullRow ){ pOut->flags = MEM_Null; break; }else if( pC->deferredMoveto ){ v = pC->movetoTarget; #ifndef SQLITE_OMIT_VIRTUALTABLE - }else if( pC->pVtabCursor ){ - pVtab = pC->pVtabCursor->pVtab; + }else if( pC->eCurType==CURTYPE_VTAB ){ + assert( pC->uc.pVCur!=0 ); + pVtab = pC->uc.pVCur->pVtab; pModule = pVtab->pModule; assert( pModule->xRowid ); - rc = pModule->xRowid(pC->pVtabCursor, &v); + rc = pModule->xRowid(pC->uc.pVCur, &v); sqlite3VtabImportErrmsg(p, pVtab); + if( rc ) goto abort_due_to_error; #endif /* SQLITE_OMIT_VIRTUALTABLE */ }else{ - assert( pC->pCursor!=0 ); - rc = sqlite3VdbeCursorMoveto(pC); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + rc = sqlite3VdbeCursorRestore(pC); if( rc ) goto abort_due_to_error; - if( pC->rowidIsValid ){ - v = pC->lastRowid; - }else{ - rc = sqlite3BtreeKeySize(pC->pCursor, &v); - assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */ + if( pC->nullRow ){ + pOut->flags = MEM_Null; + break; } + v = sqlite3BtreeIntegerKey(pC->uc.pCursor); } pOut->u.i = v; break; @@ -75371,15 +85176,15 @@ case OP_NullRow: { pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pC->nullRow = 1; - pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; - if( pC->pCursor ){ - sqlite3BtreeClearCursor(pC->pCursor); + if( pC->eCurType==CURTYPE_BTREE ){ + assert( pC->uc.pCursor!=0 ); + sqlite3BtreeClearCursor(pC->uc.pCursor); } break; } -/* Opcode: Last P1 P2 * * * +/* Opcode: Last P1 P2 P3 * * ** ** The next use of the Rowid or Column or Prev instruction for P1 ** will refer to the last entry in the database table or index. @@ -75399,20 +85204,22 @@ case OP_Last: { /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - pCrsr = pC->pCursor; + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; res = 0; assert( pCrsr!=0 ); rc = sqlite3BtreeLast(pCrsr, &res); pC->nullRow = (u8)res; pC->deferredMoveto = 0; - pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; + pC->seekResult = pOp->p3; #ifdef SQLITE_DEBUG pC->seekOp = OP_Last; #endif + if( rc ) goto abort_due_to_error; if( pOp->p2>0 ){ VdbeBranchTaken(res!=0,2); - if( res ) pc = pOp->p2 - 1; + if( res ) goto jump_to_p2; } break; } @@ -75443,9 +85250,9 @@ case OP_Sort: { /* jump */ ** ** The next use of the Rowid or Column or Next instruction for P1 ** will refer to the first entry in the database table or index. -** If the table or index is empty and P2>0, then jump immediately to P2. -** If P2 is 0 or if the table or index is not empty, fall through -** to the following instruction. +** If the table or index is empty, jump immediately to P2. +** If the table or index is not empty, fall through to the following +** instruction. ** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is @@ -75465,21 +85272,20 @@ case OP_Rewind: { /* jump */ pC->seekOp = OP_Rewind; #endif if( isSorter(pC) ){ - rc = sqlite3VdbeSorterRewind(db, pC, &res); + rc = sqlite3VdbeSorterRewind(pC, &res); }else{ - pCrsr = pC->pCursor; + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; - pC->rowidIsValid = 0; } + if( rc ) goto abort_due_to_error; pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); VdbeBranchTaken(res!=0,2); - if( res ){ - pc = pOp->p2 - 1; - } + if( res ) goto jump_to_p2; break; } @@ -75567,7 +85373,7 @@ case OP_Next: /* jump */ res = pOp->p3; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); - assert( pC->pCursor ); + assert( pC->eCurType==CURTYPE_BTREE ); assert( res==0 || (res==1 && pC->isTable==0) ); testcase( res==1 ); assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); @@ -75584,21 +85390,21 @@ case OP_Next: /* jump */ || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE || pC->seekOp==OP_Last ); - rc = pOp->p4.xAdvance(pC->pCursor, &res); + rc = pOp->p4.xAdvance(pC->uc.pCursor, &res); next_tail: pC->cacheStatus = CACHE_STALE; VdbeBranchTaken(res==0,2); + if( rc ) goto abort_due_to_error; if( res==0 ){ pC->nullRow = 0; - pc = pOp->p2 - 1; p->aCounter[pOp->p5]++; #ifdef SQLITE_TEST sqlite3_search_count++; #endif + goto jump_to_p2_and_check_for_interrupt; }else{ pC->nullRow = 1; } - pC->rowidIsValid = 0; goto check_for_interrupt; } @@ -75626,9 +85432,7 @@ next_tail: case OP_SorterInsert: /* in2 */ case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; - BtCursor *pCrsr; - int nKey; - const char *zKey; + BtreePayload x; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -75636,24 +85440,23 @@ case OP_IdxInsert: { /* in2 */ assert( isSorter(pC)==(pOp->opcode==OP_SorterInsert) ); pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); - pCrsr = pC->pCursor; if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - assert( pCrsr!=0 ); + assert( pC->eCurType==CURTYPE_BTREE || pOp->opcode==OP_SorterInsert ); assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); - if( rc==SQLITE_OK ){ - if( isSorter(pC) ){ - rc = sqlite3VdbeSorterWrite(db, pC, pIn2); - }else{ - nKey = pIn2->n; - zKey = pIn2->z; - rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, - ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) - ); - assert( pC->deferredMoveto==0 ); - pC->cacheStatus = CACHE_STALE; - } + if( rc ) goto abort_due_to_error; + if( pOp->opcode==OP_SorterInsert ){ + rc = sqlite3VdbeSorterWrite(pC, pIn2); + }else{ + x.nKey = pIn2->n; + x.pKey = pIn2->z; + rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, pOp->p3, + ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) + ); + assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; } + if( rc) goto abort_due_to_error; break; } @@ -75671,29 +85474,48 @@ case OP_IdxDelete: { UnpackedRecord r; assert( pOp->p3>0 ); - assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem-p->nCursor)+1 ); + assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem+1 - p->nCursor)+1 ); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - pCrsr = pC->pCursor; + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; assert( pCrsr!=0 ); assert( pOp->p5==0 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p3; r.default_rc = 0; r.aMem = &aMem[pOp->p2]; -#ifdef SQLITE_DEBUG - { int i; for(i=0; ideferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; break; } +/* Opcode: Seek P1 * P3 P4 * +** Synopsis: Move P3 to P1.rowid +** +** P1 is an open index cursor and P3 is a cursor on the corresponding +** table. This opcode does a deferred seek of the P3 table cursor +** to the row that corresponds to the current row of P1. +** +** This is a deferred seek. Nothing actually happens until +** the cursor is used to read a record. That way, if no reads +** occur, no unnecessary I/O happens. +** +** P4 may be an array of integers (type P4_INTARRAY) containing +** one entry for each column in the P3 table. If array entry a(i) +** is non-zero, then reading column a(i)-1 from cursor P3 is +** equivalent to performing the deferred seek and then reading column i +** from P1. This information is stored in P3 and used to redirect +** reads against P3 over to P1, thus possibly avoiding the need to +** seek and read cursor P3. +*/ /* Opcode: IdxRowid P1 P2 * * * ** Synopsis: r[P2]=rowid ** @@ -75703,29 +85525,57 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ -case OP_IdxRowid: { /* out2-prerelease */ - BtCursor *pCrsr; - VdbeCursor *pC; - i64 rowid; +case OP_Seek: +case OP_IdxRowid: { /* out2 */ + VdbeCursor *pC; /* The P1 index cursor */ + VdbeCursor *pTabCur; /* The P2 table cursor (OP_Seek only) */ + i64 rowid; /* Rowid that P1 current points to */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - pCrsr = pC->pCursor; - assert( pCrsr!=0 ); - pOut->flags = MEM_Null; - rc = sqlite3VdbeCursorMoveto(pC); - if( NEVER(rc) ) goto abort_due_to_error; - assert( pC->deferredMoveto==0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); assert( pC->isTable==0 ); + assert( pC->deferredMoveto==0 ); + assert( !pC->nullRow || pOp->opcode==OP_IdxRowid ); + + /* The IdxRowid and Seek opcodes are combined because of the commonality + ** of sqlite3VdbeCursorRestore() and sqlite3VdbeIdxRowid(). */ + rc = sqlite3VdbeCursorRestore(pC); + + /* sqlite3VbeCursorRestore() can only fail if the record has been deleted + ** out from under the cursor. That will never happens for an IdxRowid + ** or Seek opcode */ + if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ - rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid); + rc = sqlite3VdbeIdxRowid(db, pC->uc.pCursor, &rowid); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - pOut->u.i = rowid; - pOut->flags = MEM_Int; + if( pOp->opcode==OP_Seek ){ + assert( pOp->p3>=0 && pOp->p3nCursor ); + pTabCur = p->apCsr[pOp->p3]; + assert( pTabCur!=0 ); + assert( pTabCur->eCurType==CURTYPE_BTREE ); + assert( pTabCur->uc.pCursor!=0 ); + assert( pTabCur->isTable ); + pTabCur->nullRow = 0; + pTabCur->movetoTarget = rowid; + pTabCur->deferredMoveto = 1; + assert( pOp->p4type==P4_INTARRAY || pOp->p4.ai==0 ); + pTabCur->aAltMap = pOp->p4.ai; + pTabCur->pAltCursor = pC; + }else{ + pOut = out2Prerelease(p, pOp); + pOut->u.i = rowid; + pOut->flags = MEM_Int; + } + }else{ + assert( pOp->opcode==OP_IdxRowid ); + sqlite3VdbeMemSetNull(&aMem[pOp->p2]); } break; } @@ -75786,7 +85636,8 @@ case OP_IdxGE: { /* jump */ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->isOrdered ); - assert( pC->pCursor!=0); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0); assert( pC->deferredMoveto==0 ); assert( pOp->p5==0 || pOp->p5==1 ); assert( pOp->p4type==P4_INT32 ); @@ -75804,7 +85655,7 @@ case OP_IdxGE: { /* jump */ { int i; for(i=0; iopcode&1)==(OP_IdxLT&1) ){ assert( pOp->opcode==OP_IdxLE || pOp->opcode==OP_IdxLT ); @@ -75814,9 +85665,8 @@ case OP_IdxGE: { /* jump */ res++; } VdbeBranchTaken(res>0,2); - if( res>0 ){ - pc = pOp->p2 - 1 ; - } + if( rc ) goto abort_due_to_error; + if( res>0 ) goto jump_to_p2; break; } @@ -75840,39 +85690,28 @@ case OP_IdxGE: { /* jump */ ** ** See also: Clear */ -case OP_Destroy: { /* out2-prerelease */ +case OP_Destroy: { /* out2 */ int iMoved; - int iCnt; - Vdbe *pVdbe; int iDb; assert( p->readOnly==0 ); -#ifndef SQLITE_OMIT_VIRTUALTABLE - iCnt = 0; - for(pVdbe=db->pVdbe; pVdbe; pVdbe = pVdbe->pNext){ - if( pVdbe->magic==VDBE_MAGIC_RUN && pVdbe->bIsReader - && pVdbe->inVtabMethod<2 && pVdbe->pc>=0 - ){ - iCnt++; - } - } -#else - iCnt = db->nVdbeRead; -#endif + assert( pOp->p1>1 ); + pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Null; - if( iCnt>1 ){ + if( db->nVdbeRead > db->nVDestroy+1 ){ rc = SQLITE_LOCKED; p->errorAction = OE_Abort; + goto abort_due_to_error; }else{ iDb = pOp->p3; - assert( iCnt==1 ); assert( DbMaskTest(p->btreeMask, iDb) ); iMoved = 0; /* Not needed. Only to silence a warning. */ rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved); pOut->flags = MEM_Int; pOut->u.i = iMoved; + if( rc ) goto abort_due_to_error; #ifndef SQLITE_OMIT_AUTOVACUUM - if( rc==SQLITE_OK && iMoved!=0 ){ + if( iMoved!=0 ){ sqlite3RootPageMoved(db, iDb, iMoved, pOp->p1); /* All OP_Destroy operations occur on the same btree */ assert( resetSchemaOnFault==0 || resetSchemaOnFault==iDb+1 ); @@ -75918,6 +85757,7 @@ case OP_Clear: { aMem[pOp->p3].u.i += nChange; } } + if( rc ) goto abort_due_to_error; break; } @@ -75935,11 +85775,13 @@ case OP_ResetSorter: { assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - if( pC->pSorter ){ - sqlite3VdbeSorterReset(db, pC->pSorter); + if( isSorter(pC) ){ + sqlite3VdbeSorterReset(db, pC->uc.pSorter); }else{ + assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->isEphemeral ); - rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + rc = sqlite3BtreeClearTableOfCursor(pC->uc.pCursor); + if( rc ) goto abort_due_to_error; } break; } @@ -75968,12 +85810,13 @@ case OP_ResetSorter: { ** ** See documentation on OP_CreateTable for additional information. */ -case OP_CreateIndex: /* out2-prerelease */ -case OP_CreateTable: { /* out2-prerelease */ +case OP_CreateIndex: /* out2 */ +case OP_CreateTable: { /* out2 */ int pgno; int flags; Db *pDb; + pOut = out2Prerelease(p, pOp); pgno = 0; assert( pOp->p1>=0 && pOp->p1nDb ); assert( DbMaskTest(p->btreeMask, pOp->p1) ); @@ -75987,6 +85830,7 @@ case OP_CreateTable: { /* out2-prerelease */ flags = BTREE_BLOBKEY; } rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, flags); + if( rc ) goto abort_due_to_error; pOut->u.i = pgno; break; } @@ -76025,9 +85869,9 @@ case OP_ParseSchema: { initData.pzErrMsg = &p->zErrMsg; zSql = sqlite3MPrintf(db, "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", - db->aDb[iDb].zName, zMaster, pOp->p4.z); + db->aDb[iDb].zDbSName, zMaster, pOp->p4.z); if( zSql==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ assert( db->init.busy==0 ); db->init.busy = 1; @@ -76039,9 +85883,12 @@ case OP_ParseSchema: { db->init.busy = 0; } } - if( rc ) sqlite3ResetAllSchemasOfConnection(db); - if( rc==SQLITE_NOMEM ){ - goto no_mem; + if( rc ){ + sqlite3ResetAllSchemasOfConnection(db); + if( rc==SQLITE_NOMEM ){ + goto no_mem; + } + goto abort_due_to_error; } break; } @@ -76056,6 +85903,7 @@ case OP_ParseSchema: { case OP_LoadAnalysis: { assert( pOp->p1>=0 && pOp->p1nDb ); rc = sqlite3AnalysisLoad(db, pOp->p1); + if( rc ) goto abort_due_to_error; break; } #endif /* !defined(SQLITE_OMIT_ANALYZE) */ @@ -76101,7 +85949,7 @@ case OP_DropTrigger: { #ifndef SQLITE_OMIT_INTEGRITY_CHECK -/* Opcode: IntegrityCk P1 P2 P3 * P5 +/* Opcode: IntegrityCk P1 P2 P3 P4 P5 ** ** Do an analysis of the currently open database. Store in ** register P1 the text of an error message describing any problems. @@ -76112,9 +85960,8 @@ case OP_DropTrigger: { ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. ** -** The root page numbers of all tables in the database are integer -** stored in reg(P1), reg(P1+1), reg(P1+2), .... There are P2 tables -** total. +** The root page numbers of all tables in the database are integers +** stored in P4_INTARRAY argument. ** ** If P5 is not zero, the check is done on the auxiliary database ** file, not the main database file. @@ -76124,30 +85971,24 @@ case OP_DropTrigger: { case OP_IntegrityCk: { int nRoot; /* Number of tables to check. (Number of root pages.) */ int *aRoot; /* Array of rootpage numbers for tables to be checked */ - int j; /* Loop counter */ int nErr; /* Number of errors reported */ char *z; /* Text of the error report */ Mem *pnErr; /* Register keeping track of errors remaining */ assert( p->bIsReader ); nRoot = pOp->p2; + aRoot = pOp->p4.ai; assert( nRoot>0 ); - aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(nRoot+1) ); - if( aRoot==0 ) goto no_mem; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( aRoot[nRoot]==0 ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pnErr = &aMem[pOp->p3]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); pIn1 = &aMem[pOp->p1]; - for(j=0; jp5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, (int)pnErr->u.i, &nErr); - sqlite3DbFree(db, aRoot); pnErr->u.i -= nErr; sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ @@ -76164,7 +86005,7 @@ case OP_IntegrityCk: { #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ /* Opcode: RowSetAdd P1 P2 * * * -** Synopsis: rowset(P1)=r[P2] +** Synopsis: rowset(P1)=r[P2] ** ** Insert the integer value held by register P2 into a boolean index ** held in register P1. @@ -76184,7 +86025,7 @@ case OP_RowSetAdd: { /* in1, in2 */ } /* Opcode: RowSetRead P1 P2 P3 * * -** Synopsis: r[P3]=rowset(P1) +** Synopsis: r[P3]=rowset(P1) ** ** Extract the smallest value from boolean index P1 and put that value into ** register P3. Or, if boolean index P1 is initially empty, leave P3 @@ -76199,12 +86040,12 @@ case OP_RowSetRead: { /* jump, in1, out3 */ ){ /* The boolean index is empty */ sqlite3VdbeMemSetNull(pIn1); - pc = pOp->p2 - 1; VdbeBranchTaken(1,2); + goto jump_to_p2_and_check_for_interrupt; }else{ /* A value was pulled from the index */ - sqlite3VdbeMemSetInt64(&aMem[pOp->p3], val); VdbeBranchTaken(0,2); + sqlite3VdbeMemSetInt64(&aMem[pOp->p3], val); } goto check_for_interrupt; } @@ -76255,10 +86096,7 @@ case OP_RowSetTest: { /* jump, in1, in3 */ if( iSet ){ exists = sqlite3RowSetTest(pIn1->u.pRowSet, iSet, pIn3->u.i); VdbeBranchTaken(exists!=0,2); - if( exists ){ - pc = pOp->p2 - 1; - break; - } + if( exists ) goto jump_to_p2; } if( iSet>=0 ){ sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i); @@ -76317,8 +86155,8 @@ case OP_Program: { /* jump */ if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){ rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion"); - break; + sqlite3VdbeError(p, "too many levels of trigger recursion"); + goto abort_due_to_error; } /* Register pRt is used to store the memory required to save the state @@ -76332,10 +86170,11 @@ case OP_Program: { /* jump */ ** variable nMem (and later, VdbeFrame.nChildMem) to this value. */ nMem = pProgram->nMem + pProgram->nCsr; + assert( nMem>0 ); + if( pProgram->nCsr==0 ) nMem++; nByte = ROUND8(sizeof(VdbeFrame)) + nMem * sizeof(Mem) - + pProgram->nCsr * sizeof(VdbeCursor *) - + pProgram->nOnce * sizeof(u8); + + pProgram->nCsr * sizeof(VdbeCursor *); pFrame = sqlite3DbMallocZero(db, nByte); if( !pFrame ){ goto no_mem; @@ -76347,7 +86186,7 @@ case OP_Program: { /* jump */ pFrame->v = p; pFrame->nChildMem = nMem; pFrame->nChildCsr = pProgram->nCsr; - pFrame->pc = pc; + pFrame->pc = (int)(pOp - aOp); pFrame->aMem = p->aMem; pFrame->nMem = p->nMem; pFrame->apCsr = p->apCsr; @@ -76355,8 +86194,9 @@ case OP_Program: { /* jump */ pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; - pFrame->aOnceFlag = p->aOnceFlag; - pFrame->nOnceFlag = p->nOnceFlag; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pFrame->anExec = p->anExec; +#endif pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ @@ -76365,27 +86205,32 @@ case OP_Program: { /* jump */ } }else{ pFrame = pRt->u.pFrame; - assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem ); + assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem + || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) ); assert( pProgram->nCsr==pFrame->nChildCsr ); - assert( pc==pFrame->pc ); + assert( (int)(pOp - aOp)==pFrame->pc ); } p->nFrame++; pFrame->pParent = p->pFrame; pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; + pFrame->nDbChange = p->db->nChange; + assert( pFrame->pAuxData==0 ); + pFrame->pAuxData = p->pAuxData; + p->pAuxData = 0; p->nChange = 0; p->pFrame = pFrame; - p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; + p->aMem = aMem = VdbeFrameMem(pFrame); p->nMem = pFrame->nChildMem; p->nCursor = (u16)pFrame->nChildCsr; - p->apCsr = (VdbeCursor **)&aMem[p->nMem+1]; + p->apCsr = (VdbeCursor **)&aMem[p->nMem]; p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; - p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; - p->nOnceFlag = pProgram->nOnce; - pc = -1; - memset(p->aOnceFlag, 0, p->nOnceFlag); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = 0; +#endif + pOp = &aOp[-1]; break; } @@ -76402,9 +86247,10 @@ case OP_Program: { /* jump */ ** the value of the P1 argument to the value of the P1 argument to the ** calling OP_Program instruction. */ -case OP_Param: { /* out2-prerelease */ +case OP_Param: { /* out2 */ VdbeFrame *pFrame; Mem *pIn; + pOut = out2Prerelease(p, pOp); pFrame = p->pFrame; pIn = &pFrame->aMem[pOp->p1 + pFrame->aOp[pFrame->pc].p1]; sqlite3VdbeMemShallowCopy(pOut, pIn, MEM_Ephem); @@ -76448,10 +86294,10 @@ case OP_FkCounter: { case OP_FkIfZero: { /* jump */ if( pOp->p1 ){ VdbeBranchTaken(db->nDeferredCons==0 && db->nDeferredImmCons==0, 2); - if( db->nDeferredCons==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1; + if( db->nDeferredCons==0 && db->nDeferredImmCons==0 ) goto jump_to_p2; }else{ VdbeBranchTaken(p->nFkConstraint==0 && db->nDeferredImmCons==0, 2); - if( p->nFkConstraint==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1; + if( p->nFkConstraint==0 && db->nDeferredImmCons==0 ) goto jump_to_p2; } break; } @@ -76488,118 +86334,186 @@ case OP_MemMax: { /* in2 */ } #endif /* SQLITE_OMIT_AUTOINCREMENT */ -/* Opcode: IfPos P1 P2 * * * -** Synopsis: if r[P1]>0 goto P2 +/* Opcode: IfPos P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 ** -** If the value of register P1 is 1 or greater, jump to P2. +** Register P1 must contain an integer. +** If the value of register P1 is 1 or greater, subtract P3 from the +** value in P1 and jump to P2. ** -** It is illegal to use this instruction on a register that does -** not contain an integer. An assertion fault will result if you try. +** If the initial value of register P1 is less than 1, then the +** value is unchanged and control passes through to the next instruction. */ case OP_IfPos: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken( pIn1->u.i>0, 2); if( pIn1->u.i>0 ){ - pc = pOp->p2 - 1; + pIn1->u.i -= pOp->p3; + goto jump_to_p2; } break; } -/* Opcode: IfNeg P1 P2 P3 * * -** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 +/* Opcode: OffsetLimit P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) ** -** Register P1 must contain an integer. Add literal P3 to the value in -** register P1 then if the value of register P1 is less than zero, jump to P2. +** This opcode performs a commonly used computation associated with +** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] +** holds the offset counter. The opcode computes the combined value +** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] +** value computed is the total number of rows that will need to be +** visited in order to complete the query. +** +** If r[P3] is zero or negative, that means there is no OFFSET +** and r[P2] is set to be the value of the LIMIT, r[P1]. +** +** if r[P1] is zero or negative, that means there is no LIMIT +** and r[P2] is set to -1. +** +** Otherwise, r[P2] is set to the sum of r[P1] and r[P3]. */ -case OP_IfNeg: { /* jump, in1 */ +case OP_OffsetLimit: { /* in1, out2, in3 */ + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; + pOut = out2Prerelease(p, pOp); + assert( pIn1->flags & MEM_Int ); + assert( pIn3->flags & MEM_Int ); + pOut->u.i = pIn1->u.i<=0 ? -1 : pIn1->u.i+(pIn3->u.i>0?pIn3->u.i:0); + break; +} + +/* Opcode: IfNotZero P1 P2 P3 * * +** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 +** +** Register P1 must contain an integer. If the content of register P1 is +** initially nonzero, then subtract P3 from the value in register P1 and +** jump to P2. If register P1 is initially zero, leave it unchanged +** and fall through. +*/ +case OP_IfNotZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); - pIn1->u.i += pOp->p3; VdbeBranchTaken(pIn1->u.i<0, 2); - if( pIn1->u.i<0 ){ - pc = pOp->p2 - 1; + if( pIn1->u.i ){ + pIn1->u.i -= pOp->p3; + goto jump_to_p2; } break; } -/* Opcode: IfZero P1 P2 P3 * * -** Synopsis: r[P1]+=P3, if r[P1]==0 goto P2 +/* Opcode: DecrJumpZero P1 P2 * * * +** Synopsis: if (--r[P1])==0 goto P2 ** -** The register P1 must contain an integer. Add literal P3 to the -** value in register P1. If the result is exactly 0, jump to P2. +** Register P1 must hold an integer. Decrement the value in register P1 +** then jump to P2 if the new value is exactly zero. */ -case OP_IfZero: { /* jump, in1 */ +case OP_DecrJumpZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); - pIn1->u.i += pOp->p3; + pIn1->u.i--; VdbeBranchTaken(pIn1->u.i==0, 2); - if( pIn1->u.i==0 ){ - pc = pOp->p2 - 1; - } + if( pIn1->u.i==0 ) goto jump_to_p2; break; } -/* Opcode: AggStep * P2 P3 P4 P5 + +/* Opcode: AggStep0 * P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** ** Execute the step function for an aggregate. The ** function has P5 arguments. P4 is a pointer to the FuncDef -** structure that specifies the function. Use register -** P3 as the accumulator. +** structure that specifies the function. Register P3 is the +** accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. */ -case OP_AggStep: { +/* Opcode: AggStep * P2 P3 P4 P5 +** Synopsis: accum=r[P3] step(r[P2@P5]) +** +** Execute the step function for an aggregate. The +** function has P5 arguments. P4 is a pointer to an sqlite3_context +** object that is used to run the function. Register P3 is +** as the accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +** +** This opcode is initially coded as OP_AggStep0. On first evaluation, +** the FuncDef stored in P4 is converted into an sqlite3_context and +** the opcode is changed. In this way, the initialization of the +** sqlite3_context only happens once, instead of on each call to the +** step function. +*/ +case OP_AggStep0: { int n; - int i; - Mem *pMem; - Mem *pRec; - sqlite3_context ctx; - sqlite3_value **apVal; + sqlite3_context *pCtx; + assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; - assert( n>=0 ); - pRec = &aMem[pOp->p2]; - apVal = p->apArg; - assert( apVal || n==0 ); - for(i=0; ip3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); + assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); + pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + if( pCtx==0 ) goto no_mem; + pCtx->pMem = 0; + pCtx->pFunc = pOp->p4.pFunc; + pCtx->iOp = (int)(pOp - aOp); + pCtx->pVdbe = p; + pCtx->argc = n; + pOp->p4type = P4_FUNCCTX; + pOp->p4.pCtx = pCtx; + pOp->opcode = OP_AggStep; + /* Fall through into OP_AggStep */ +} +case OP_AggStep: { + int i; + sqlite3_context *pCtx; + Mem *pMem; + Mem t; + + assert( pOp->p4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; + pMem = &aMem[pOp->p3]; + + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relavant parts of the sqlite3_context object */ + if( pCtx->pMem != pMem ){ + pCtx->pMem = pMem; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } - ctx.pFunc = pOp->p4.pFunc; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - ctx.pMem = pMem = &aMem[pOp->p3]; + +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + pMem->n++; - ctx.s.flags = MEM_Null; - ctx.s.z = 0; - ctx.s.zMalloc = 0; - ctx.s.xDel = 0; - ctx.s.db = db; - ctx.isError = 0; - ctx.pColl = 0; - ctx.skipFlag = 0; - if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>p->aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - ctx.pColl = pOp[-1].p4.pColl; + sqlite3VdbeMemInit(&t, db, MEM_Null); + pCtx->pOut = &t; + pCtx->fErrorOrAux = 0; + pCtx->skipFlag = 0; + (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ + if( pCtx->fErrorOrAux ){ + if( pCtx->isError ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(&t)); + rc = pCtx->isError; + } + sqlite3VdbeMemRelease(&t); + if( rc ) goto abort_due_to_error; + }else{ + assert( t.flags==MEM_Null ); } - (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */ - if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); - rc = ctx.isError; - } - if( ctx.skipFlag ){ + if( pCtx->skipFlag ){ assert( pOp[-1].opcode==OP_CollSeq ); i = pOp[-1].p1; if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); } - - sqlite3VdbeMemRelease(&ctx.s); - break; } @@ -76618,12 +86532,13 @@ case OP_AggStep: { */ case OP_AggFinal: { Mem *pMem; - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pMem = &aMem[pOp->p1]; assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); if( rc ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(pMem)); + sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem)); + goto abort_due_to_error; } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); @@ -76637,8 +86552,8 @@ case OP_AggFinal: { /* Opcode: Checkpoint P1 P2 P3 * * ** ** Checkpoint database P1. This is a no-op if P1 is not currently in -** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL -** or RESTART. Write 1 or 0 into mem[P3] if the checkpoint returns +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL, +** RESTART, or TRUNCATE. Write 1 or 0 into mem[P3] if the checkpoint returns ** SQLITE_BUSY or not, respectively. Write the number of pages in the ** WAL after the checkpoint into mem[P3+1] and the number of pages ** in the WAL that have been checkpointed after the checkpoint @@ -76656,9 +86571,11 @@ case OP_Checkpoint: { assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART + || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); - if( rc==SQLITE_BUSY ){ + if( rc ){ + if( rc!=SQLITE_BUSY ) goto abort_due_to_error; rc = SQLITE_OK; aRes[0] = 1; } @@ -76681,7 +86598,7 @@ case OP_Checkpoint: { ** ** Write a string containing the final journal-mode to register P2. */ -case OP_JournalMode: { /* out2-prerelease */ +case OP_JournalMode: { /* out2 */ Btree *pBt; /* Btree to change journal mode of */ Pager *pPager; /* Pager associated with pBt */ int eNew; /* New journal mode */ @@ -76690,6 +86607,7 @@ case OP_JournalMode: { /* out2-prerelease */ const char *zFilename; /* Name of database file for pPager */ #endif + pOut = out2Prerelease(p, pOp); eNew = pOp->p3; assert( eNew==PAGER_JOURNALMODE_DELETE || eNew==PAGER_JOURNALMODE_TRUNCATE @@ -76726,11 +86644,11 @@ case OP_JournalMode: { /* out2-prerelease */ ){ if( !db->autoCommit || db->nVdbeRead>1 ){ rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, + sqlite3VdbeError(p, "cannot change %s wal mode from within a transaction", (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") ); - break; + goto abort_due_to_error; }else{ if( eOld==PAGER_JOURNALMODE_WAL ){ @@ -76760,31 +86678,29 @@ case OP_JournalMode: { /* out2-prerelease */ } #endif /* ifndef SQLITE_OMIT_WAL */ - if( rc ){ - eNew = eOld; - } + if( rc ) eNew = eOld; eNew = sqlite3PagerSetJournalMode(pPager, eNew); - pOut = &aMem[pOp->p2]; pOut->flags = MEM_Str|MEM_Static|MEM_Term; pOut->z = (char *)sqlite3JournalModename(eNew); pOut->n = sqlite3Strlen30(pOut->z); pOut->enc = SQLITE_UTF8; sqlite3VdbeChangeEncoding(pOut, encoding); + if( rc ) goto abort_due_to_error; break; }; #endif /* SQLITE_OMIT_PRAGMA */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* Opcode: Vacuum * * * * * +/* Opcode: Vacuum P1 * * * * ** -** Vacuum the entire database. This opcode will cause other virtual -** machines to be created and run. It may not be called from within -** a transaction. +** Vacuum the entire database P1. P1 is 0 for "main", and 2 or more +** for an attached database. The "temp" database may not be vacuumed. */ case OP_Vacuum: { assert( p->readOnly==0 ); - rc = sqlite3RunVacuum(&p->zErrMsg, db); + rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1); + if( rc ) goto abort_due_to_error; break; } #endif @@ -76805,9 +86721,10 @@ case OP_IncrVacuum: { /* jump */ pBt = db->aDb[pOp->p1].pBt; rc = sqlite3BtreeIncrVacuum(pBt); VdbeBranchTaken(rc==SQLITE_DONE,2); - if( rc==SQLITE_DONE ){ - pc = pOp->p2 - 1; + if( rc ){ + if( rc!=SQLITE_DONE ) goto abort_due_to_error; rc = SQLITE_OK; + goto jump_to_p2; } break; } @@ -76856,9 +86773,12 @@ case OP_TableLock: { assert( DbMaskTest(p->btreeMask, p1) ); assert( isWriteLock==0 || isWriteLock==1 ); rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); - if( (rc&0xFF)==SQLITE_LOCKED ){ - const char *z = pOp->p4.z; - sqlite3SetString(&p->zErrMsg, db, "database table is locked: %s", z); + if( rc ){ + if( (rc&0xFF)==SQLITE_LOCKED ){ + const char *z = pOp->p4.z; + sqlite3VdbeError(p, "database table is locked: %s", z); + } + goto abort_due_to_error; } } break; @@ -76880,18 +86800,36 @@ case OP_VBegin: { pVTab = pOp->p4.pVtab; rc = sqlite3VtabBegin(db, pVTab); if( pVTab ) sqlite3VtabImportErrmsg(p, pVTab->pVtab); + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ #ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VCreate P1 * * P4 * +/* Opcode: VCreate P1 P2 * * * ** -** P4 is the name of a virtual table in database P1. Call the xCreate method -** for that table. +** P2 is a register that holds the name of a virtual table in database +** P1. Call the xCreate method for that table. */ case OP_VCreate: { - rc = sqlite3VtabCallCreate(db, pOp->p1, pOp->p4.z, &p->zErrMsg); + Mem sMem; /* For storing the record being decoded */ + const char *zTab; /* Name of the virtual table */ + + memset(&sMem, 0, sizeof(sMem)); + sMem.db = db; + /* Because P2 is always a static string, it is impossible for the + ** sqlite3VdbeMemCopy() to fail */ + assert( (aMem[pOp->p2].flags & MEM_Str)!=0 ); + assert( (aMem[pOp->p2].flags & MEM_Static)!=0 ); + rc = sqlite3VdbeMemCopy(&sMem, &aMem[pOp->p2]); + assert( rc==SQLITE_OK ); + zTab = (const char*)sqlite3_value_text(&sMem); + assert( zTab || db->mallocFailed ); + if( zTab ){ + rc = sqlite3VtabCallCreate(db, pOp->p1, zTab, &p->zErrMsg); + } + sqlite3VdbeMemRelease(&sMem); + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -76903,9 +86841,10 @@ case OP_VCreate: { ** of that table. */ case OP_VDestroy: { - p->inVtabMethod = 2; + db->nVDestroy++; rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); - p->inVtabMethod = 0; + db->nVDestroy--; + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -76919,30 +86858,35 @@ case OP_VDestroy: { */ case OP_VOpen: { VdbeCursor *pCur; - sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; - sqlite3_module *pModule; + const sqlite3_module *pModule; assert( p->bIsReader ); pCur = 0; - pVtabCursor = 0; + pVCur = 0; pVtab = pOp->p4.pVtab->pVtab; - pModule = (sqlite3_module *)pVtab->pModule; - assert(pVtab && pModule); - rc = pModule->xOpen(pVtab, &pVtabCursor); + if( pVtab==0 || NEVER(pVtab->pModule==0) ){ + rc = SQLITE_LOCKED; + goto abort_due_to_error; + } + pModule = pVtab->pModule; + rc = pModule->xOpen(pVtab, &pVCur); sqlite3VtabImportErrmsg(p, pVtab); - if( SQLITE_OK==rc ){ - /* Initialize sqlite3_vtab_cursor base class */ - pVtabCursor->pVtab = pVtab; + if( rc ) goto abort_due_to_error; - /* Initialize vdbe cursor object */ - pCur = allocateCursor(p, pOp->p1, 0, -1, 0); - if( pCur ){ - pCur->pVtabCursor = pVtabCursor; - }else{ - db->mallocFailed = 1; - pModule->xClose(pVtabCursor); - } + /* Initialize sqlite3_vtab_cursor base class */ + pVCur->pVtab = pVtab; + + /* Initialize vdbe cursor object */ + pCur = allocateCursor(p, pOp->p1, 0, -1, CURTYPE_VTAB); + if( pCur ){ + pCur->uc.pVCur = pVCur; + pVtab->nRef++; + }else{ + assert( db->mallocFailed ); + pModule->xClose(pVCur); + goto no_mem; } break; } @@ -76974,7 +86918,7 @@ case OP_VFilter: { /* jump */ const sqlite3_module *pModule; Mem *pQuery; Mem *pArgc; - sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; VdbeCursor *pCur; int res; @@ -76986,9 +86930,9 @@ case OP_VFilter: { /* jump */ pCur = p->apCsr[pOp->p1]; assert( memIsValid(pQuery) ); REGISTER_TRACE(pOp->p3, pQuery); - assert( pCur->pVtabCursor ); - pVtabCursor = pCur->pVtabCursor; - pVtab = pVtabCursor->pVtab; + assert( pCur->eCurType==CURTYPE_VTAB ); + pVCur = pCur->uc.pVCur; + pVtab = pVCur->pVtab; pModule = pVtab->pModule; /* Grab the index number and argc parameters */ @@ -76997,27 +86941,18 @@ case OP_VFilter: { /* jump */ iQuery = (int)pQuery->u.i; /* Invoke the xFilter method */ - { - res = 0; - apArg = p->apArg; - for(i = 0; iinVtabMethod = 1; - rc = pModule->xFilter(pVtabCursor, iQuery, pOp->p4.z, nArg, apArg); - p->inVtabMethod = 0; - sqlite3VtabImportErrmsg(p, pVtab); - if( rc==SQLITE_OK ){ - res = pModule->xEof(pVtabCursor); - } - VdbeBranchTaken(res!=0,2); - if( res ){ - pc = pOp->p2 - 1; - } + res = 0; + apArg = p->apArg; + for(i = 0; ixFilter(pVCur, iQuery, pOp->p4.z, nArg, apArg); + sqlite3VtabImportErrmsg(p, pVtab); + if( rc ) goto abort_due_to_error; + res = pModule->xEof(pVCur); pCur->nullRow = 0; - + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -77037,45 +86972,33 @@ case OP_VColumn: { sqlite3_context sContext; VdbeCursor *pCur = p->apCsr[pOp->p1]; - assert( pCur->pVtabCursor ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( pCur->eCurType==CURTYPE_VTAB ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); if( pCur->nullRow ){ sqlite3VdbeMemSetNull(pDest); break; } - pVtab = pCur->pVtabCursor->pVtab; + pVtab = pCur->uc.pVCur->pVtab; pModule = pVtab->pModule; assert( pModule->xColumn ); memset(&sContext, 0, sizeof(sContext)); - - /* The output cell may already have a buffer allocated. Move - ** the current contents to sContext.s so in case the user-function - ** can use the already allocated buffer instead of allocating a - ** new one. - */ - sqlite3VdbeMemMove(&sContext.s, pDest); - MemSetTypeFlag(&sContext.s, MEM_Null); - - rc = pModule->xColumn(pCur->pVtabCursor, &sContext, pOp->p2); + sContext.pOut = pDest; + MemSetTypeFlag(pDest, MEM_Null); + rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2); sqlite3VtabImportErrmsg(p, pVtab); if( sContext.isError ){ rc = sContext.isError; } - - /* Copy the result of the function to the P3 register. We - ** do this regardless of whether or not an error occurred to ensure any - ** dynamic allocation in sContext.s (a Mem struct) is released. - */ - sqlite3VdbeChangeEncoding(&sContext.s, encoding); - sqlite3VdbeMemMove(pDest, &sContext.s); + sqlite3VdbeChangeEncoding(pDest, encoding); REGISTER_TRACE(pOp->p3, pDest); UPDATE_MAX_BLOBSIZE(pDest); if( sqlite3VdbeMemTooBig(pDest) ){ goto too_big; } + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -77095,11 +87018,11 @@ case OP_VNext: { /* jump */ res = 0; pCur = p->apCsr[pOp->p1]; - assert( pCur->pVtabCursor ); + assert( pCur->eCurType==CURTYPE_VTAB ); if( pCur->nullRow ){ break; } - pVtab = pCur->pVtabCursor->pVtab; + pVtab = pCur->uc.pVCur->pVtab; pModule = pVtab->pModule; assert( pModule->xNext ); @@ -77109,17 +87032,14 @@ case OP_VNext: { /* jump */ ** data is available) and the error code returned when xColumn or ** some other method is next invoked on the save virtual table cursor. */ - p->inVtabMethod = 1; - rc = pModule->xNext(pCur->pVtabCursor); - p->inVtabMethod = 0; + rc = pModule->xNext(pCur->uc.pVCur); sqlite3VtabImportErrmsg(p, pVtab); - if( rc==SQLITE_OK ){ - res = pModule->xEof(pCur->pVtabCursor); - } + if( rc ) goto abort_due_to_error; + res = pModule->xEof(pCur->uc.pVCur); VdbeBranchTaken(!res,2); if( !res ){ /* If there is data, jump to P2 */ - pc = pOp->p2 - 1; + goto jump_to_p2_and_check_for_interrupt; } goto check_for_interrupt; } @@ -77147,11 +87067,11 @@ case OP_VRename: { testcase( pName->enc==SQLITE_UTF16BE ); testcase( pName->enc==SQLITE_UTF16LE ); rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); - if( rc==SQLITE_OK ){ - rc = pVtab->pModule->xRename(pVtab, pName->z); - sqlite3VtabImportErrmsg(p, pVtab); - p->expired = 0; - } + if( rc ) goto abort_due_to_error; + rc = pVtab->pModule->xRename(pVtab, pName->z); + sqlite3VtabImportErrmsg(p, pVtab); + p->expired = 0; + if( rc ) goto abort_due_to_error; break; } #endif @@ -77186,7 +87106,7 @@ case OP_VRename: { */ case OP_VUpdate: { sqlite3_vtab *pVtab; - sqlite3_module *pModule; + const sqlite3_module *pModule; int nArg; int i; sqlite_int64 rowid; @@ -77198,7 +87118,11 @@ case OP_VUpdate: { ); assert( p->readOnly==0 ); pVtab = pOp->p4.pVtab->pVtab; - pModule = (sqlite3_module *)pVtab->pModule; + if( pVtab==0 || NEVER(pVtab->pModule==0) ){ + rc = SQLITE_LOCKED; + goto abort_due_to_error; + } + pModule = pVtab->pModule; nArg = pOp->p2; assert( pOp->p4type==P4_VTAB ); if( ALWAYS(pModule->xUpdate) ){ @@ -77228,6 +87152,7 @@ case OP_VUpdate: { }else{ p->nChange++; } + if( rc ) goto abort_due_to_error; } break; } @@ -77238,7 +87163,8 @@ case OP_VUpdate: { ** ** Write the current number of pages in database P1 to memory cell P2. */ -case OP_Pagecount: { /* out2-prerelease */ +case OP_Pagecount: { /* out2 */ + pOut = out2Prerelease(p, pOp); pOut->u.i = sqlite3BtreeLastPage(db->aDb[pOp->p1].pBt); break; } @@ -77254,10 +87180,11 @@ case OP_Pagecount: { /* out2-prerelease */ ** ** Store the maximum page count after the change in register P2. */ -case OP_MaxPgcnt: { /* out2-prerelease */ +case OP_MaxPgcnt: { /* out2 */ unsigned int newMax; Btree *pBt; + pOut = out2Prerelease(p, pOp); pBt = db->aDb[pOp->p1].pBt; newMax = 0; if( pOp->p3 ){ @@ -77270,8 +87197,8 @@ case OP_MaxPgcnt: { /* out2-prerelease */ #endif -/* Opcode: Init * P2 * P4 * -** Synopsis: Start at P2 +/* Opcode: Init P1 P2 * P4 * +** Synopsis: Start at P2 ** ** Programs contain a single instance of this opcode as the very first ** opcode. @@ -77281,30 +87208,50 @@ case OP_MaxPgcnt: { /* out2-prerelease */ ** Or if P4 is blank, use the string returned by sqlite3_sql(). ** ** If P2 is not zero, jump to instruction P2. +** +** Increment the value of P1 so that OP_Once opcodes will jump the +** first time they are evaluated for this run. */ case OP_Init: { /* jump */ char *zTrace; - char *z; + int i; + + /* If the P4 argument is not NULL, then it must be an SQL comment string. + ** The "--" string is broken up to prevent false-positives with srcck1.c. + ** + ** This assert() provides evidence for: + ** EVIDENCE-OF: R-50676-09860 The callback can compute the same text that + ** would have been returned by the legacy sqlite3_trace() interface by + ** using the X argument when X begins with "--" and invoking + ** sqlite3_expanded_sql(P) otherwise. + */ + assert( pOp->p4.z==0 || strncmp(pOp->p4.z, "-" "- ", 3)==0 ); + assert( pOp==p->aOp ); /* Always instruction 0 */ - if( pOp->p2 ){ - pc = pOp->p2 - 1; - } #ifndef SQLITE_OMIT_TRACE - if( db->xTrace + if( (db->mTrace & (SQLITE_TRACE_STMT|SQLITE_TRACE_LEGACY))!=0 && !p->doingRerun && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ - z = sqlite3VdbeExpandSql(p, zTrace); - db->xTrace(db->pTraceArg, z); - sqlite3DbFree(db, z); +#ifndef SQLITE_OMIT_DEPRECATED + if( db->mTrace & SQLITE_TRACE_LEGACY ){ + void (*x)(void*,const char*) = (void(*)(void*,const char*))db->xTrace; + char *z = sqlite3VdbeExpandSql(p, zTrace); + x(db->pTraceArg, z); + sqlite3_free(z); + }else +#endif + { + (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace); + } } #ifdef SQLITE_USE_FCNTL_TRACE zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); if( zTrace ){ - int i; - for(i=0; inDb; i++){ - if( DbMaskTest(p->btreeMask, i)==0 ) continue; - sqlite3_file_control(db, db->aDb[i].zName, SQLITE_FCNTL_TRACE, zTrace); + int j; + for(j=0; jnDb; j++){ + if( DbMaskTest(p->btreeMask, j)==0 ) continue; + sqlite3_file_control(db, db->aDb[j].zDbSName, SQLITE_FCNTL_TRACE, zTrace); } } #endif /* SQLITE_USE_FCNTL_TRACE */ @@ -77316,9 +87263,39 @@ case OP_Init: { /* jump */ } #endif /* SQLITE_DEBUG */ #endif /* SQLITE_OMIT_TRACE */ - break; + assert( pOp->p2>0 ); + if( pOp->p1>=sqlite3GlobalConfig.iOnceResetThreshold ){ + for(i=1; inOp; i++){ + if( p->aOp[i].opcode==OP_Once ) p->aOp[i].p1 = 0; + } + pOp->p1 = 0; + } + pOp->p1++; + goto jump_to_p2; } +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* Opcode: CursorHint P1 * * P4 * +** +** Provide a hint to cursor P1 that it only needs to return rows that +** satisfy the Expr in P4. TK_REGISTER terms in the P4 expression refer +** to values currently held in registers. TK_COLUMN terms in the P4 +** expression refer to columns in the b-tree to which cursor P1 is pointing. +*/ +case OP_CursorHint: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_EXPR ); + pC = p->apCsr[pOp->p1]; + if( pC ){ + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3BtreeCursorHint(pC->uc.pCursor, BTREE_HINT_RANGE, + pOp->p4.pExpr, aMem); + } + break; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ /* Opcode: Noop * * * * * ** @@ -77347,8 +87324,8 @@ default: { /* This is really OP_Noop and OP_Explain */ #ifdef VDBE_PROFILE { u64 endTime = sqlite3Hwtime(); - if( endTime>start ) pOp->cycles += endTime - start; - pOp->cnt++; + if( endTime>start ) pOrigOp->cycles += endTime - start; + pOrigOp->cnt++; } #endif @@ -77358,16 +87335,17 @@ default: { /* This is really OP_Noop and OP_Explain */ ** the evaluator loop. So we can leave it out when NDEBUG is defined. */ #ifndef NDEBUG - assert( pc>=-1 && pcnOp ); + assert( pOp>=&aOp[-1] && pOp<&aOp[p->nOp-1] ); #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeTrace ){ + u8 opProperty = sqlite3OpcodeProperty[pOrigOp->opcode]; if( rc!=0 ) printf("rc=%d\n",rc); - if( pOp->opflags & (OPFLG_OUT2_PRERELEASE|OPFLG_OUT2) ){ - registerTrace(pOp->p2, &aMem[pOp->p2]); + if( opProperty & (OPFLG_OUT2) ){ + registerTrace(pOrigOp->p2, &aMem[pOrigOp->p2]); } - if( pOp->opflags & OPFLG_OUT3 ){ - registerTrace(pOp->p3, &aMem[pOp->p3]); + if( opProperty & OPFLG_OUT3 ){ + registerTrace(pOrigOp->p3, &aMem[pOrigOp->p3]); } } #endif /* SQLITE_DEBUG */ @@ -77377,14 +87355,19 @@ default: { /* This is really OP_Noop and OP_Explain */ /* If we reach this point, it means that execution is finished with ** an error of some kind. */ -vdbe_error_halt: +abort_due_to_error: + if( db->mallocFailed ) rc = SQLITE_NOMEM_BKPT; assert( rc ); + if( p->zErrMsg==0 && rc!=SQLITE_IOERR_NOMEM ){ + sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); + } p->rc = rc; + sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(rc, "statement aborts at %d: [%s] %s", - pc, p->zSql, p->zErrMsg); + (int)(pOp - aOp), p->zSql, p->zErrMsg); sqlite3VdbeHalt(p); - if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); rc = SQLITE_ERROR; if( resetSchemaOnFault>0 ){ sqlite3ResetOneSchema(db, resetSchemaOnFault-1); @@ -77398,44 +87381,36 @@ vdbe_return: testcase( nVmStep>0 ); p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; sqlite3VdbeLeave(p); + assert( rc!=SQLITE_OK || nExtraDelete==0 + || sqlite3_strlike("DELETE%",p->zSql,0)!=0 + ); return rc; /* Jump to here if a string or blob larger than SQLITE_MAX_LENGTH ** is encountered. */ too_big: - sqlite3SetString(&p->zErrMsg, db, "string or blob too big"); + sqlite3VdbeError(p, "string or blob too big"); rc = SQLITE_TOOBIG; - goto vdbe_error_halt; + goto abort_due_to_error; /* Jump to here if a malloc() fails. */ no_mem: - db->mallocFailed = 1; - sqlite3SetString(&p->zErrMsg, db, "out of memory"); - rc = SQLITE_NOMEM; - goto vdbe_error_halt; - - /* Jump to here for any other kind of fatal error. The "rc" variable - ** should hold the error number. - */ -abort_due_to_error: - assert( p->zErrMsg==0 ); - if( db->mallocFailed ) rc = SQLITE_NOMEM; - if( rc!=SQLITE_IOERR_NOMEM ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); - } - goto vdbe_error_halt; + sqlite3OomFault(db); + sqlite3VdbeError(p, "out of memory"); + rc = SQLITE_NOMEM_BKPT; + goto abort_due_to_error; /* Jump to here if the sqlite3_interrupt() API sets the interrupt ** flag. */ abort_due_to_interrupt: assert( db->u1.isInterrupted ); - rc = SQLITE_INTERRUPT; + rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; p->rc = rc; - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); - goto vdbe_error_halt; + sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); + goto abort_due_to_error; } @@ -77456,6 +87431,8 @@ abort_due_to_interrupt: ** This file contains code used to implement incremental BLOB I/O. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_INCRBLOB @@ -77471,6 +87448,8 @@ struct Incrblob { BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ + char *zDb; /* Database name */ + Table *pTab; /* Table object */ }; @@ -77517,7 +87496,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ }else{ p->iOffset = pC->aType[p->iCol + pC->nField]; p->nByte = sqlite3VdbeSerialTypeLen(type); - p->pCsr = pC->pCursor; + p->pCsr = pC->uc.pCursor; sqlite3BtreeIncrblobCursor(p->pCsr); } } @@ -77556,46 +87535,24 @@ SQLITE_API int sqlite3_blob_open( ){ int nAttempt = 0; int iCol; /* Index of zColumn in row-record */ - - /* This VDBE program seeks a btree cursor to the identified - ** db/table/row entry. The reason for using a vdbe program instead - ** of writing code to use the b-tree layer directly is that the - ** vdbe program will take advantage of the various transaction, - ** locking and error handling infrastructure built into the vdbe. - ** - ** After seeking the cursor, the vdbe executes an OP_ResultRow. - ** Code external to the Vdbe then "borrows" the b-tree cursor and - ** uses it to implement the blob_read(), blob_write() and - ** blob_bytes() functions. - ** - ** The sqlite3_blob_close() function finalizes the vdbe program, - ** which closes the b-tree cursor and (possibly) commits the - ** transaction. - */ - static const int iLn = VDBE_OFFSET_LINENO(4); - static const VdbeOpList openBlob[] = { - /* {OP_Transaction, 0, 0, 0}, // 0: Inserted separately */ - {OP_TableLock, 0, 0, 0}, /* 1: Acquire a read or write lock */ - /* One of the following two instructions is replaced by an OP_Noop. */ - {OP_OpenRead, 0, 0, 0}, /* 2: Open cursor 0 for reading */ - {OP_OpenWrite, 0, 0, 0}, /* 3: Open cursor 0 for read/write */ - {OP_Variable, 1, 1, 1}, /* 4: Push the rowid to the stack */ - {OP_NotExists, 0, 10, 1}, /* 5: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 6 */ - {OP_ResultRow, 1, 0, 0}, /* 7 */ - {OP_Goto, 0, 4, 0}, /* 8 */ - {OP_Close, 0, 0, 0}, /* 9 */ - {OP_Halt, 0, 0, 0}, /* 10 */ - }; - int rc = SQLITE_OK; char *zErr = 0; Table *pTab; Parse *pParse = 0; Incrblob *pBlob = 0; - flags = !!flags; /* flags = (flags ? 1 : 0); */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppBlob==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif *ppBlob = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + flags = !!flags; /* flags = (flags ? 1 : 0); */ sqlite3_mutex_enter(db->mutex); @@ -77636,6 +87593,8 @@ SQLITE_API int sqlite3_blob_open( sqlite3BtreeLeaveAll(db); goto blob_open_out; } + pBlob->pTab = pTab; + pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; /* Now search pTab for the exact column. */ for(iCol=0; iColnCol; iCol++) { @@ -77678,7 +87637,8 @@ SQLITE_API int sqlite3_blob_open( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int j; for(j=0; jnKeyCol; j++){ - if( pIdx->aiColumn[j]==iCol ){ + /* FIXME: Be smarter about indexes that use expressions */ + if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==XN_EXPR ){ zFault = "indexed"; } } @@ -77695,45 +87655,78 @@ SQLITE_API int sqlite3_blob_open( pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(pParse); assert( pBlob->pStmt || db->mallocFailed ); if( pBlob->pStmt ){ + + /* This VDBE program seeks a btree cursor to the identified + ** db/table/row entry. The reason for using a vdbe program instead + ** of writing code to use the b-tree layer directly is that the + ** vdbe program will take advantage of the various transaction, + ** locking and error handling infrastructure built into the vdbe. + ** + ** After seeking the cursor, the vdbe executes an OP_ResultRow. + ** Code external to the Vdbe then "borrows" the b-tree cursor and + ** uses it to implement the blob_read(), blob_write() and + ** blob_bytes() functions. + ** + ** The sqlite3_blob_close() function finalizes the vdbe program, + ** which closes the b-tree cursor and (possibly) commits the + ** transaction. + */ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList openBlob[] = { + {OP_TableLock, 0, 0, 0}, /* 0: Acquire a read or write lock */ + {OP_OpenRead, 0, 0, 0}, /* 1: Open a cursor */ + {OP_Variable, 1, 1, 0}, /* 2: Move ?1 into reg[1] */ + {OP_NotExists, 0, 7, 1}, /* 3: Seek the cursor */ + {OP_Column, 0, 0, 1}, /* 4 */ + {OP_ResultRow, 1, 0, 0}, /* 5 */ + {OP_Goto, 0, 2, 0}, /* 6 */ + {OP_Close, 0, 0, 0}, /* 7 */ + {OP_Halt, 0, 0, 0}, /* 8 */ + }; Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - + VdbeOp *aOp; sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, pTab->pSchema->schema_cookie, pTab->pSchema->iGeneration); sqlite3VdbeChangeP5(v, 1); - sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); + aOp = sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); /* Make sure a mutex is held on the table to be accessed */ sqlite3VdbeUsesBtree(v, iDb); - /* Configure the OP_TableLock instruction */ + if( db->mallocFailed==0 ){ + assert( aOp!=0 ); + /* Configure the OP_TableLock instruction */ #ifdef SQLITE_OMIT_SHARED_CACHE - sqlite3VdbeChangeToNoop(v, 1); + aOp[0].opcode = OP_Noop; #else - sqlite3VdbeChangeP1(v, 1, iDb); - sqlite3VdbeChangeP2(v, 1, pTab->tnum); - sqlite3VdbeChangeP3(v, 1, flags); - sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); + aOp[0].p1 = iDb; + aOp[0].p2 = pTab->tnum; + aOp[0].p3 = flags; + sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); + } + if( db->mallocFailed==0 ){ #endif - /* Remove either the OP_OpenWrite or OpenRead. Set the P2 - ** parameter of the other to pTab->tnum. */ - sqlite3VdbeChangeToNoop(v, 3 - flags); - sqlite3VdbeChangeP2(v, 2 + flags, pTab->tnum); - sqlite3VdbeChangeP3(v, 2 + flags, iDb); + /* Remove either the OP_OpenWrite or OpenRead. Set the P2 + ** parameter of the other to pTab->tnum. */ + if( flags ) aOp[1].opcode = OP_OpenWrite; + aOp[1].p2 = pTab->tnum; + aOp[1].p3 = iDb; + + /* Configure the number of columns. Configure the cursor to + ** think that the table has one more column than it really + ** does. An OP_Column to retrieve this imaginary column will + ** always return an SQL NULL. This is useful because it means + ** we can invoke OP_Column to fill in the vdbe cursors type + ** and offset cache without causing any IO. + */ + aOp[1].p4type = P4_INT32; + aOp[1].p4.i = pTab->nCol+1; + aOp[4].p2 = pTab->nCol; - /* Configure the number of columns. Configure the cursor to - ** think that the table has one more column than it really - ** does. An OP_Column to retrieve this imaginary column will - ** always return an SQL NULL. This is useful because it means - ** we can invoke OP_Column to fill in the vdbe cursors type - ** and offset cache without causing any IO. - */ - sqlite3VdbeChangeP4(v, 2+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); - sqlite3VdbeChangeP2(v, 6, pTab->nCol); - if( !db->mallocFailed ){ pParse->nVar = 1; pParse->nMem = 1; pParse->nTab = 1; @@ -77759,7 +87752,7 @@ blob_open_out: if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); sqlite3DbFree(db, pBlob); } - sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); sqlite3ParserReset(pParse); sqlite3StackFree(db, pParse); @@ -77809,10 +87802,9 @@ static int blobReadWrite( sqlite3_mutex_enter(db->mutex); v = (Vdbe*)p->pStmt; - if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ + if( n<0 || iOffset<0 || ((sqlite3_int64)iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR, 0); }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. @@ -77824,16 +87816,40 @@ static int blobReadWrite( */ assert( db == v->db ); sqlite3BtreeEnterCursor(p->pCsr); + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( xCall==sqlite3BtreePutData && db->xPreUpdateCallback ){ + /* If a pre-update hook is registered and this is a write cursor, + ** invoke it here. + ** + ** TODO: The preupdate-hook is passed SQLITE_DELETE, even though this + ** operation should really be an SQLITE_UPDATE. This is probably + ** incorrect, but is convenient because at this point the new.* values + ** are not easily obtainable. And for the sessions module, an + ** SQLITE_UPDATE where the PK columns do not change is handled in the + ** same way as an SQLITE_DELETE (the SQLITE_DELETE code is actually + ** slightly more efficient). Since you cannot write to a PK column + ** using the incremental-blob API, this works. For the sessions module + ** anyhow. + */ + sqlite3_int64 iKey; + iKey = sqlite3BtreeIntegerKey(p->pCsr); + sqlite3VdbePreUpdateHook( + v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1 + ); + } +#endif + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); sqlite3BtreeLeaveCursor(p->pCsr); if( rc==SQLITE_ABORT ){ sqlite3VdbeFinalize(v); p->pStmt = 0; }else{ - db->errCode = rc; v->rc = rc; } } + sqlite3Error(db, rc); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -77892,7 +87908,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ char *zErr; rc = blobSeekToRow(p, iRow, &zErr); if( rc!=SQLITE_OK ){ - sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); } assert( rc!=SQLITE_SCHEMA ); @@ -77909,7 +87925,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ /************** End of vdbeblob.c ********************************************/ /************** Begin file vdbesort.c ****************************************/ /* -** 2011 July 9 +** 2011-07-09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -77920,42 +87936,205 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ** ************************************************************************* ** This file contains code for the VdbeSorter object, used in concert with -** a VdbeCursor to sort large numbers of keys (as may be required, for -** example, by CREATE INDEX statements on tables too large to fit in main -** memory). +** a VdbeCursor to sort large numbers of keys for CREATE INDEX statements +** or by SELECT statements with ORDER BY clauses that cannot be satisfied +** using indexes and without LIMIT clauses. +** +** The VdbeSorter object implements a multi-threaded external merge sort +** algorithm that is efficient even if the number of elements being sorted +** exceeds the available memory. +** +** Here is the (internal, non-API) interface between this module and the +** rest of the SQLite system: +** +** sqlite3VdbeSorterInit() Create a new VdbeSorter object. +** +** sqlite3VdbeSorterWrite() Add a single new row to the VdbeSorter +** object. The row is a binary blob in the +** OP_MakeRecord format that contains both +** the ORDER BY key columns and result columns +** in the case of a SELECT w/ ORDER BY, or +** the complete record for an index entry +** in the case of a CREATE INDEX. +** +** sqlite3VdbeSorterRewind() Sort all content previously added. +** Position the read cursor on the +** first sorted element. +** +** sqlite3VdbeSorterNext() Advance the read cursor to the next sorted +** element. +** +** sqlite3VdbeSorterRowkey() Return the complete binary blob for the +** row currently under the read cursor. +** +** sqlite3VdbeSorterCompare() Compare the binary blob for the row +** currently under the read cursor against +** another binary blob X and report if +** X is strictly less than the read cursor. +** Used to enforce uniqueness in a +** CREATE UNIQUE INDEX statement. +** +** sqlite3VdbeSorterClose() Close the VdbeSorter object and reclaim +** all resources. +** +** sqlite3VdbeSorterReset() Refurbish the VdbeSorter for reuse. This +** is like Close() followed by Init() only +** much faster. +** +** The interfaces above must be called in a particular order. Write() can +** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and +** Compare() can only occur in between Rewind() and Close()/Reset(). i.e. +** +** Init() +** for each record: Write() +** Rewind() +** Rowkey()/Compare() +** Next() +** Close() +** +** Algorithm: +** +** Records passed to the sorter via calls to Write() are initially held +** unsorted in main memory. Assuming the amount of memory used never exceeds +** a threshold, when Rewind() is called the set of records is sorted using +** an in-memory merge sort. In this case, no temporary files are required +** and subsequent calls to Rowkey(), Next() and Compare() read records +** directly from main memory. +** +** If the amount of space used to store records in main memory exceeds the +** threshold, then the set of records currently in memory are sorted and +** written to a temporary file in "Packed Memory Array" (PMA) format. +** A PMA created at this point is known as a "level-0 PMA". Higher levels +** of PMAs may be created by merging existing PMAs together - for example +** merging two or more level-0 PMAs together creates a level-1 PMA. +** +** The threshold for the amount of main memory to use before flushing +** records to a PMA is roughly the same as the limit configured for the +** page-cache of the main database. Specifically, the threshold is set to +** the value returned by "PRAGMA main.page_size" multipled by +** that returned by "PRAGMA main.cache_size", in bytes. +** +** If the sorter is running in single-threaded mode, then all PMAs generated +** are appended to a single temporary file. Or, if the sorter is running in +** multi-threaded mode then up to (N+1) temporary files may be opened, where +** N is the configured number of worker threads. In this case, instead of +** sorting the records and writing the PMA to a temporary file itself, the +** calling thread usually launches a worker thread to do so. Except, if +** there are already N worker threads running, the main thread does the work +** itself. +** +** The sorter is running in multi-threaded mode if (a) the library was built +** with pre-processor symbol SQLITE_MAX_WORKER_THREADS set to a value greater +** than zero, and (b) worker threads have been enabled at runtime by calling +** "PRAGMA threads=N" with some value of N greater than 0. +** +** When Rewind() is called, any data remaining in memory is flushed to a +** final PMA. So at this point the data is stored in some number of sorted +** PMAs within temporary files on disk. +** +** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the +** sorter is running in single-threaded mode, then these PMAs are merged +** incrementally as keys are retreived from the sorter by the VDBE. The +** MergeEngine object, described in further detail below, performs this +** merge. +** +** Or, if running in multi-threaded mode, then a background thread is +** launched to merge the existing PMAs. Once the background thread has +** merged T bytes of data into a single sorted PMA, the main thread +** begins reading keys from that PMA while the background thread proceeds +** with merging the next T bytes of data. And so on. +** +** Parameter T is set to half the value of the memory threshold used +** by Write() above to determine when to create a new PMA. +** +** If there are more than SORTER_MAX_MERGE_COUNT PMAs in total when +** Rewind() is called, then a hierarchy of incremental-merges is used. +** First, T bytes of data from the first SORTER_MAX_MERGE_COUNT PMAs on +** disk are merged together. Then T bytes of data from the second set, and +** so on, such that no operation ever merges more than SORTER_MAX_MERGE_COUNT +** PMAs at a time. This done is to improve locality. +** +** If running in multi-threaded mode and there are more than +** SORTER_MAX_MERGE_COUNT PMAs on disk when Rewind() is called, then more +** than one background thread may be created. Specifically, there may be +** one background thread for each temporary file on disk, and one background +** thread to merge the output of each of the others to a single PMA for +** the main thread to read from. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ - - -typedef struct VdbeSorterIter VdbeSorterIter; -typedef struct SorterRecord SorterRecord; -typedef struct FileWriter FileWriter; +/* +** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various +** messages to stderr that may be helpful in understanding the performance +** characteristics of the sorter in multi-threaded mode. +*/ +#if 0 +# define SQLITE_DEBUG_SORTER_THREADS 1 +#endif /* -** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: +** Hard-coded maximum amount of data to accumulate in memory before flushing +** to a level 0 PMA. The purpose of this limit is to prevent various integer +** overflows. 512MiB. +*/ +#define SQLITE_MAX_PMASZ (1<<29) + +/* +** Private objects used by the sorter +*/ +typedef struct MergeEngine MergeEngine; /* Merge PMAs together */ +typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ +typedef struct PmaWriter PmaWriter; /* Incrementally write one PMA */ +typedef struct SorterRecord SorterRecord; /* A record being sorted */ +typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ +typedef struct SorterFile SorterFile; /* Temporary file object wrapper */ +typedef struct SorterList SorterList; /* In-memory list of records */ +typedef struct IncrMerger IncrMerger; /* Read & merge multiple PMAs */ + +/* +** A container for a temp file handle and the current amount of data +** stored in the file. +*/ +struct SorterFile { + sqlite3_file *pFd; /* File handle */ + i64 iEof; /* Bytes of data stored in pFd */ +}; + +/* +** An in-memory list of objects to be sorted. ** -** As keys are added to the sorter, they are written to disk in a series -** of sorted packed-memory-arrays (PMAs). The size of each PMA is roughly -** the same as the cache-size allowed for temporary databases. In order -** to allow the caller to extract keys from the sorter in sorted order, -** all PMAs currently stored on disk must be merged together. This comment -** describes the data structure used to do so. The structure supports -** merging any number of arrays in a single pass with no redundant comparison -** operations. +** If aMemory==0 then each object is allocated separately and the objects +** are connected using SorterRecord.u.pNext. If aMemory!=0 then all objects +** are stored in the aMemory[] bulk memory, one right after the other, and +** are connected using SorterRecord.u.iNext. +*/ +struct SorterList { + SorterRecord *pList; /* Linked list of records */ + u8 *aMemory; /* If non-NULL, bulk memory to hold pList */ + int szPMA; /* Size of pList as PMA in bytes */ +}; + +/* +** The MergeEngine object is used to combine two or more smaller PMAs into +** one big PMA using a merge operation. Separate PMAs all need to be +** combined into one big PMA in order to be able to step through the sorted +** records in order. ** -** The aIter[] array contains an iterator for each of the PMAs being merged. -** An aIter[] iterator either points to a valid key or else is at EOF. For -** the purposes of the paragraphs below, we assume that the array is actually -** N elements in size, where N is the smallest power of 2 greater to or equal -** to the number of iterators being merged. The extra aIter[] elements are -** treated as if they are empty (always at EOF). +** The aReadr[] array contains a PmaReader object for each of the PMAs being +** merged. An aReadr[] object either points to a valid key or else is at EOF. +** ("EOF" means "End Of File". When aReadr[] is at EOF there is no more data.) +** For the purposes of the paragraphs below, we assume that the array is +** actually N elements in size, where N is the smallest power of 2 greater +** to or equal to the number of PMAs being merged. The extra aReadr[] elements +** are treated as if they are empty (always at EOF). ** ** The aTree[] array is also N elements in size. The value of N is stored in -** the VdbeSorter.nTree variable. +** the MergeEngine.nTree variable. ** ** The final (N/2) elements of aTree[] contain the results of comparing -** pairs of iterator keys together. Element i contains the result of -** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the +** pairs of PMA keys together. Element i contains the result of +** comparing aReadr[2*i-N] and aReadr[2*i-N+1]. Whichever key is smaller, the ** aTree element is set to the index of it. ** ** For the purposes of this comparison, EOF is considered greater than any @@ -77963,34 +88142,34 @@ typedef struct FileWriter FileWriter; ** values), it doesn't matter which index is stored. ** ** The (N/4) elements of aTree[] that precede the final (N/2) described -** above contains the index of the smallest of each block of 4 iterators. -** And so on. So that aTree[1] contains the index of the iterator that +** above contains the index of the smallest of each block of 4 PmaReaders +** And so on. So that aTree[1] contains the index of the PmaReader that ** currently points to the smallest key value. aTree[0] is unused. ** ** Example: ** -** aIter[0] -> Banana -** aIter[1] -> Feijoa -** aIter[2] -> Elderberry -** aIter[3] -> Currant -** aIter[4] -> Grapefruit -** aIter[5] -> Apple -** aIter[6] -> Durian -** aIter[7] -> EOF +** aReadr[0] -> Banana +** aReadr[1] -> Feijoa +** aReadr[2] -> Elderberry +** aReadr[3] -> Currant +** aReadr[4] -> Grapefruit +** aReadr[5] -> Apple +** aReadr[6] -> Durian +** aReadr[7] -> EOF ** ** aTree[] = { X, 5 0, 5 0, 3, 5, 6 } ** ** The current element is "Apple" (the value of the key indicated by -** iterator 5). When the Next() operation is invoked, iterator 5 will +** PmaReader 5). When the Next() operation is invoked, PmaReader 5 will ** be advanced to the next key in its segment. Say the next key is ** "Eggplant": ** -** aIter[5] -> Eggplant +** aReadr[5] -> Eggplant ** -** The contents of aTree[] are updated first by comparing the new iterator -** 5 key to the current key of iterator 4 (still "Grapefruit"). The iterator +** The contents of aTree[] are updated first by comparing the new PmaReader +** 5 key to the current key of PmaReader 4 (still "Grapefruit"). The PmaReader ** 5 value is still smaller, so aTree[6] is set to 5. And so on up the tree. -** The value of iterator 6 - "Durian" - is now smaller than that of iterator +** The value of PmaReader 6 - "Durian" - is now smaller than that of PmaReader ** 5, so aTree[3] is set to 6. Key 0 is smaller than key 6 (Bananafile2. And instead of using a +** background thread to prepare data for the PmaReader, with a single +** threaded IncrMerger the allocate part of pTask->file2 is "refilled" with +** keys from pMerger by the calling thread whenever the PmaReader runs out +** of data. */ -struct VdbeSorterIter { - i64 iReadOff; /* Current read offset */ - i64 iEof; /* 1 byte past EOF for this iterator */ - int nAlloc; /* Bytes of space at aAlloc */ - int nKey; /* Number of bytes in key */ - sqlite3_file *pFile; /* File iterator is reading from */ - u8 *aAlloc; /* Allocated space */ - u8 *aKey; /* Pointer to current key */ - u8 *aBuffer; /* Current read buffer */ - int nBuffer; /* Size of read buffer in bytes */ +struct IncrMerger { + SortSubtask *pTask; /* Task that owns this merger */ + MergeEngine *pMerger; /* Merge engine thread reads data from */ + i64 iStartOff; /* Offset to start writing file at */ + int mxSz; /* Maximum bytes of data to store */ + int bEof; /* Set to true when merge is finished */ + int bUseThread; /* True to use a bg thread for this object */ + SorterFile aFile[2]; /* aFile[0] for reading, [1] for writing */ }; /* -** An instance of this structure is used to organize the stream of records -** being written to files by the merge-sort code into aligned, page-sized -** blocks. Doing all I/O in aligned page-sized blocks helps I/O to go -** faster on many operating systems. +** An instance of this object is used for writing a PMA. +** +** The PMA is written one record at a time. Each record is of an arbitrary +** size. But I/O is more efficient if it occurs in page-sized blocks where +** each block is aligned on a page boundary. This object caches writes to +** the PMA so that aligned, page-size blocks are written. */ -struct FileWriter { +struct PmaWriter { int eFWErr; /* Non-zero if in an error state */ u8 *aBuffer; /* Pointer to write buffer */ int nBuffer; /* Size of write buffer in bytes */ int iBufStart; /* First byte of buffer to write */ int iBufEnd; /* Last byte of buffer to write */ i64 iWriteOff; /* Offset of start of buffer in file */ - sqlite3_file *pFile; /* File to write to */ + sqlite3_file *pFd; /* File handle to write to */ }; /* -** A structure to store a single record. All in-memory records are connected -** together into a linked list headed at VdbeSorter.pRecord using the -** SorterRecord.pNext pointer. +** This object is the header on a single record while that record is being +** held in memory and prior to being written out as part of a PMA. +** +** How the linked list is connected depends on how memory is being managed +** by this module. If using a separate allocation for each in-memory record +** (VdbeSorter.list.aMemory==0), then the list is always connected using the +** SorterRecord.u.pNext pointers. +** +** Or, if using the single large allocation method (VdbeSorter.list.aMemory!=0), +** then while records are being accumulated the list is linked using the +** SorterRecord.u.iNext offset. This is because the aMemory[] array may +** be sqlite3Realloc()ed while records are being accumulated. Once the VM +** has finished passing records to the sorter, or when the in-memory buffer +** is full, the list is sorted. As part of the sorting process, it is +** converted to use the SorterRecord.u.pNext pointers. See function +** vdbeSorterSort() for details. */ struct SorterRecord { - void *pVal; - int nVal; - SorterRecord *pNext; + int nVal; /* Size of the record in bytes */ + union { + SorterRecord *pNext; /* Pointer to next record in list */ + int iNext; /* Offset within aMemory of next record */ + } u; + /* The data for the record immediately follows this header */ }; -/* Minimum allowable value for the VdbeSorter.nWorking variable */ -#define SORTER_MIN_WORKING 10 +/* Return a pointer to the buffer containing the record data for SorterRecord +** object p. Should be used as if: +** +** void *SRVAL(SorterRecord *p) { return (void*)&p[1]; } +*/ +#define SRVAL(p) ((void*)((SorterRecord*)(p) + 1)) -/* Maximum number of segments to merge in a single pass. */ + +/* Maximum number of PMAs that a single MergeEngine can merge */ #define SORTER_MAX_MERGE_COUNT 16 +static int vdbeIncrSwap(IncrMerger*); +static void vdbeIncrFree(IncrMerger *); + /* -** Free all memory belonging to the VdbeSorterIter object passed as the second +** Free all memory belonging to the PmaReader object passed as the ** argument. All structure fields are set to zero before returning. */ -static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ - sqlite3DbFree(db, pIter->aAlloc); - sqlite3DbFree(db, pIter->aBuffer); - memset(pIter, 0, sizeof(VdbeSorterIter)); +static void vdbePmaReaderClear(PmaReader *pReadr){ + sqlite3_free(pReadr->aAlloc); + sqlite3_free(pReadr->aBuffer); + if( pReadr->aMap ) sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); + vdbeIncrFree(pReadr->pIncr); + memset(pReadr, 0, sizeof(PmaReader)); } /* -** Read nByte bytes of data from the stream of data iterated by object p. +** Read the next nByte bytes of data from the PMA p. ** If successful, set *ppOut to point to a buffer containing the data ** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite ** error code. ** -** The buffer indicated by *ppOut may only be considered valid until the +** The buffer returned in *ppOut is only valid until the ** next call to this function. */ -static int vdbeSorterIterRead( - sqlite3 *db, /* Database handle (for malloc) */ - VdbeSorterIter *p, /* Iterator */ +static int vdbePmaReadBlob( + PmaReader *p, /* PmaReader from which to take the blob */ int nByte, /* Bytes of data to read */ u8 **ppOut /* OUT: Pointer to buffer containing data */ ){ int iBuf; /* Offset within buffer to read from */ int nAvail; /* Bytes of data available in buffer */ + + if( p->aMap ){ + *ppOut = &p->aMap[p->iReadOff]; + p->iReadOff += nByte; + return SQLITE_OK; + } + assert( p->aBuffer ); /* If there is no more data to be read from the buffer, read the next @@ -78109,8 +88441,8 @@ static int vdbeSorterIterRead( } assert( nRead>0 ); - /* Read data from the file. Return early if an error occurs. */ - rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff); + /* Readr data from the file. Return early if an error occurs. */ + rc = sqlite3OsRead(p->pFd, p->aBuffer, nRead, p->iReadOff); assert( rc!=SQLITE_IOERR_SHORT_READ ); if( rc!=SQLITE_OK ) return rc; } @@ -78130,11 +88462,13 @@ static int vdbeSorterIterRead( /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2; + u8 *aNew; + int nNew = MAX(128, p->nAlloc*2); while( nByte>nNew ) nNew = nNew*2; - p->aAlloc = sqlite3DbReallocOrFree(db, p->aAlloc, nNew); - if( !p->aAlloc ) return SQLITE_NOMEM; + aNew = sqlite3Realloc(p->aAlloc, nNew); + if( !aNew ) return SQLITE_NOMEM_BKPT; p->nAlloc = nNew; + p->aAlloc = aNew; } /* Copy as much data as is available in the buffer into the start of @@ -78146,13 +88480,13 @@ static int vdbeSorterIterRead( /* The following loop copies up to p->nBuffer bytes per iteration into ** the p->aAlloc[] buffer. */ while( nRem>0 ){ - int rc; /* vdbeSorterIterRead() return code */ + int rc; /* vdbePmaReadBlob() return code */ int nCopy; /* Number of bytes to copy */ u8 *aNext; /* Pointer to buffer to copy data from */ nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; - rc = vdbeSorterIterRead(db, p, nCopy, &aNext); + rc = vdbePmaReadBlob(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); @@ -78169,234 +88503,453 @@ static int vdbeSorterIterRead( ** Read a varint from the stream of data accessed by p. Set *pnOut to ** the value read. */ -static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ +static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ int iBuf; - iBuf = p->iReadOff % p->nBuffer; - if( iBuf && (p->nBuffer-iBuf)>=9 ){ - p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + if( p->aMap ){ + p->iReadOff += sqlite3GetVarint(&p->aMap[p->iReadOff], pnOut); }else{ - u8 aVarint[16], *a; - int i = 0, rc; - do{ - rc = vdbeSorterIterRead(db, p, 1, &a); - if( rc ) return rc; - aVarint[(i++)&0xf] = a[0]; - }while( (a[0]&0x80)!=0 ); - sqlite3GetVarint(aVarint, pnOut); + iBuf = p->iReadOff % p->nBuffer; + if( iBuf && (p->nBuffer-iBuf)>=9 ){ + p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + }else{ + u8 aVarint[16], *a; + int i = 0, rc; + do{ + rc = vdbePmaReadBlob(p, 1, &a); + if( rc ) return rc; + aVarint[(i++)&0xf] = a[0]; + }while( (a[0]&0x80)!=0 ); + sqlite3GetVarint(aVarint, pnOut); + } } return SQLITE_OK; } - /* -** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if -** no error occurs, or an SQLite error code if one does. +** Attempt to memory map file pFile. If successful, set *pp to point to the +** new mapping and return SQLITE_OK. If the mapping is not attempted +** (because the file is too large or the VFS layer is configured not to use +** mmap), return SQLITE_OK and set *pp to NULL. +** +** Or, if an error occurs, return an SQLite error code. The final value of +** *pp is undefined in this case. */ -static int vdbeSorterIterNext( - sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */ - VdbeSorterIter *pIter /* Iterator to advance */ -){ - int rc; /* Return Code */ - u64 nRec = 0; /* Size of record in bytes */ - - if( pIter->iReadOff>=pIter->iEof ){ - /* This is an EOF condition */ - vdbeSorterIterZero(db, pIter); - return SQLITE_OK; +static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ + int rc = SQLITE_OK; + if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ + sqlite3_file *pFd = pFile->pFd; + if( pFd->pMethods->iVersion>=3 ){ + rc = sqlite3OsFetch(pFd, 0, (int)pFile->iEof, (void**)pp); + testcase( rc!=SQLITE_OK ); + } } - - rc = vdbeSorterIterVarint(db, pIter, &nRec); - if( rc==SQLITE_OK ){ - pIter->nKey = (int)nRec; - rc = vdbeSorterIterRead(db, pIter, (int)nRec, &pIter->aKey); - } - return rc; } /* -** Initialize iterator pIter to scan through the PMA stored in file pFile -** starting at offset iStart and ending at offset iEof-1. This function -** leaves the iterator pointing to the first key in the PMA (or EOF if the -** PMA is empty). +** Attach PmaReader pReadr to file pFile (if it is not already attached to +** that file) and seek it to offset iOff within the file. Return SQLITE_OK +** if successful, or an SQLite error code if an error occurs. */ -static int vdbeSorterIterInit( - sqlite3 *db, /* Database handle */ - const VdbeSorter *pSorter, /* Sorter object */ - i64 iStart, /* Start offset in pFile */ - VdbeSorterIter *pIter, /* Iterator to populate */ - i64 *pnByte /* IN/OUT: Increment this value by PMA size */ +static int vdbePmaReaderSeek( + SortSubtask *pTask, /* Task context */ + PmaReader *pReadr, /* Reader whose cursor is to be moved */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iOff /* Offset in pFile */ ){ int rc = SQLITE_OK; - int nBuf; - nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + assert( pReadr->pIncr==0 || pReadr->pIncr->bEof==0 ); - assert( pSorter->iWriteOff>iStart ); - assert( pIter->aAlloc==0 ); - assert( pIter->aBuffer==0 ); - pIter->pFile = pSorter->pTemp1; - pIter->iReadOff = iStart; - pIter->nAlloc = 128; - pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); - pIter->nBuffer = nBuf; - pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + if( sqlite3FaultSim(201) ) return SQLITE_IOERR_READ; + if( pReadr->aMap ){ + sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); + pReadr->aMap = 0; + } + pReadr->iReadOff = iOff; + pReadr->iEof = pFile->iEof; + pReadr->pFd = pFile->pFd; - if( !pIter->aBuffer ){ - rc = SQLITE_NOMEM; - }else{ - int iBuf; - - iBuf = iStart % nBuf; - if( iBuf ){ - int nRead = nBuf - iBuf; - if( (iStart + nRead) > pSorter->iWriteOff ){ - nRead = (int)(pSorter->iWriteOff - iStart); + rc = vdbeSorterMapFile(pTask, pFile, &pReadr->aMap); + if( rc==SQLITE_OK && pReadr->aMap==0 ){ + int pgsz = pTask->pSorter->pgsz; + int iBuf = pReadr->iReadOff % pgsz; + if( pReadr->aBuffer==0 ){ + pReadr->aBuffer = (u8*)sqlite3Malloc(pgsz); + if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM_BKPT; + pReadr->nBuffer = pgsz; + } + if( rc==SQLITE_OK && iBuf ){ + int nRead = pgsz - iBuf; + if( (pReadr->iReadOff + nRead) > pReadr->iEof ){ + nRead = (int)(pReadr->iEof - pReadr->iReadOff); } rc = sqlite3OsRead( - pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + pReadr->pFd, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff ); + testcase( rc!=SQLITE_OK ); + } + } + + return rc; +} + +/* +** Advance PmaReader pReadr to the next key in its PMA. Return SQLITE_OK if +** no error occurs, or an SQLite error code if one does. +*/ +static int vdbePmaReaderNext(PmaReader *pReadr){ + int rc = SQLITE_OK; /* Return Code */ + u64 nRec = 0; /* Size of record in bytes */ + + + if( pReadr->iReadOff>=pReadr->iEof ){ + IncrMerger *pIncr = pReadr->pIncr; + int bEof = 1; + if( pIncr ){ + rc = vdbeIncrSwap(pIncr); + if( rc==SQLITE_OK && pIncr->bEof==0 ){ + rc = vdbePmaReaderSeek( + pIncr->pTask, pReadr, &pIncr->aFile[0], pIncr->iStartOff + ); + bEof = 0; + } } - if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pSorter->iWriteOff; - rc = vdbeSorterIterVarint(db, pIter, &nByte); - pIter->iEof = pIter->iReadOff + nByte; - *pnByte += nByte; + if( bEof ){ + /* This is an EOF condition */ + vdbePmaReaderClear(pReadr); + testcase( rc!=SQLITE_OK ); + return rc; } } if( rc==SQLITE_OK ){ - rc = vdbeSorterIterNext(db, pIter); + rc = vdbePmaReadVarint(pReadr, &nRec); + } + if( rc==SQLITE_OK ){ + pReadr->nKey = (int)nRec; + rc = vdbePmaReadBlob(pReadr, (int)nRec, &pReadr->aKey); + testcase( rc!=SQLITE_OK ); + } + + return rc; +} + +/* +** Initialize PmaReader pReadr to scan through the PMA stored in file pFile +** starting at offset iStart and ending at offset iEof-1. This function +** leaves the PmaReader pointing to the first key in the PMA (or EOF if the +** PMA is empty). +** +** If the pnByte parameter is NULL, then it is assumed that the file +** contains a single PMA, and that that PMA omits the initial length varint. +*/ +static int vdbePmaReaderInit( + SortSubtask *pTask, /* Task context */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iStart, /* Start offset in pFile */ + PmaReader *pReadr, /* PmaReader to populate */ + i64 *pnByte /* IN/OUT: Increment this value by PMA size */ +){ + int rc; + + assert( pFile->iEof>iStart ); + assert( pReadr->aAlloc==0 && pReadr->nAlloc==0 ); + assert( pReadr->aBuffer==0 ); + assert( pReadr->aMap==0 ); + + rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); + if( rc==SQLITE_OK ){ + u64 nByte = 0; /* Size of PMA in bytes */ + rc = vdbePmaReadVarint(pReadr, &nByte); + pReadr->iEof = pReadr->iReadOff + nByte; + *pnByte += nByte; + } + + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderNext(pReadr); } return rc; } +/* +** A version of vdbeSorterCompare() that assumes that it has already been +** determined that the first field of key1 is equal to the first field of +** key2. +*/ +static int vdbeSorterCompareTail( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + UnpackedRecord *r2 = pTask->pUnpacked; + if( *pbKey2Cached==0 ){ + sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + *pbKey2Cached = 1; + } + return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, r2, 1); +} /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, -** size nKey2 bytes). Argument pKeyInfo supplies the collation functions -** used by the comparison. If an error occurs, return an SQLite error code. -** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive -** value, depending on whether key1 is smaller, equal to or larger than key2. +** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences +** used by the comparison. Return the result of the comparison. ** -** If the bOmitRowid argument is non-zero, assume both keys end in a rowid -** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid -** is true and key1 contains even a single NULL value, it is considered to -** be less than key2. Even if key2 also contains NULL values. +** If IN/OUT parameter *pbKey2Cached is true when this function is called, +** it is assumed that (pTask->pUnpacked) contains the unpacked version +** of key2. If it is false, (pTask->pUnpacked) is populated with the unpacked +** version of key2 and *pbKey2Cached set to true before returning. ** -** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace -** has been allocated and contains an unpacked record that is used as key2. +** If an OOM error is encountered, (pTask->pUnpacked->error_rc) is set +** to SQLITE_NOMEM. */ -static void vdbeSorterCompare( - const VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ - int nKeyCol, /* Num of columns. 0 means "all" */ +static int vdbeSorterCompare( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ const void *pKey1, int nKey1, /* Left side of comparison */ - const void *pKey2, int nKey2, /* Right side of comparison */ - int *pRes /* OUT: Result of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ ){ - KeyInfo *pKeyInfo = pCsr->pKeyInfo; - VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->pUnpacked; - int i; - - if( pKey2 ){ - sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); + UnpackedRecord *r2 = pTask->pUnpacked; + if( !*pbKey2Cached ){ + sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + *pbKey2Cached = 1; } - - if( nKeyCol ){ - r2->nField = nKeyCol; - for(i=0; iaMem[i].flags & MEM_Null ){ - *pRes = -1; - return; - } - } - assert( r2->default_rc==0 ); - } - - *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); } /* -** This function is called to compare two iterator keys when merging -** multiple b-tree segments. Parameter iOut is the index of the aTree[] -** value to recalculate. +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is a TEXT value and that the collation +** sequence to compare them with is BINARY. */ -static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ - VdbeSorter *pSorter = pCsr->pSorter; - int i1; - int i2; - int iRes; - VdbeSorterIter *p1; - VdbeSorterIter *p2; +static int vdbeSorterCompareText( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ - assert( iOutnTree && iOut>0 ); + int n1; + int n2; + int res; - if( iOut>=(pSorter->nTree/2) ){ - i1 = (iOut - pSorter->nTree/2) * 2; - i2 = i1 + 1; - }else{ - i1 = pSorter->aTree[iOut*2]; - i2 = pSorter->aTree[iOut*2+1]; + getVarint32(&p1[1], n1); n1 = (n1 - 13) / 2; + getVarint32(&p2[1], n2); n2 = (n2 - 13) / 2; + res = memcmp(v1, v2, MIN(n1, n2)); + if( res==0 ){ + res = n1 - n2; } - p1 = &pSorter->aIter[i1]; - p2 = &pSorter->aIter[i2]; - - if( p1->pFile==0 ){ - iRes = i2; - }else if( p2->pFile==0 ){ - iRes = i1; + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nField>1 ){ + res = vdbeSorterCompareTail( + pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 + ); + } }else{ - int res; - assert( pCsr->pSorter->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ - vdbeSorterCompare( - pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res - ); - if( res<=0 ){ - iRes = i1; - }else{ - iRes = i2; + if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + res = res * -1; } } - pSorter->aTree[iOut] = iRes; - return SQLITE_OK; + return res; +} + +/* +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is an INTEGER value. +*/ +static int vdbeSorterCompareInt( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const int s1 = p1[1]; /* Left hand serial type */ + const int s2 = p2[1]; /* Right hand serial type */ + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ + int res; /* Return value */ + + assert( (s1>0 && s1<7) || s1==8 || s1==9 ); + assert( (s2>0 && s2<7) || s2==8 || s2==9 ); + + if( s1>7 && s2>7 ){ + res = s1 - s2; + }else{ + if( s1==s2 ){ + if( (*v1 ^ *v2) & 0x80 ){ + /* The two values have different signs */ + res = (*v1 & 0x80) ? -1 : +1; + }else{ + /* The two values have the same sign. Compare using memcmp(). */ + static const u8 aLen[] = {0, 1, 2, 3, 4, 6, 8 }; + int i; + res = 0; + for(i=0; i7 ){ + res = +1; + }else if( s1>7 ){ + res = -1; + }else{ + res = s1 - s2; + } + assert( res!=0 ); + + if( res>0 ){ + if( *v1 & 0x80 ) res = -1; + }else{ + if( *v2 & 0x80 ) res = +1; + } + } + } + + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nField>1 ){ + res = vdbeSorterCompareTail( + pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 + ); + } + }else if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + res = res * -1; + } + + return res; } /* ** Initialize the temporary index cursor just opened as a sorter cursor. +** +** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField) +** to determine the number of fields that should be compared from the +** records being sorted. However, if the value passed as argument nField +** is non-zero and the sorter is able to guarantee a stable sort, nField +** is used instead. This is used when sorting records for a CREATE INDEX +** statement. In this case, keys are always delivered to the sorter in +** order of the primary key, which happens to be make up the final part +** of the records being sorted. So if the sort is stable, there is never +** any reason to compare PK fields and they can be ignored for a small +** performance boost. +** +** The sorter can guarantee a stable sort when running in single-threaded +** mode, but not in multi-threaded mode. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ +SQLITE_PRIVATE int sqlite3VdbeSorterInit( + sqlite3 *db, /* Database connection (for malloc()) */ + int nField, /* Number of key fields in each record */ + VdbeCursor *pCsr /* Cursor that holds the new sorter */ +){ int pgsz; /* Page size of main database */ - int mxCache; /* Cache size */ + int i; /* Used to iterate through aTask[] */ VdbeSorter *pSorter; /* The new sorter */ - char *d; /* Dummy */ + KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ + int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ + int sz; /* Size of pSorter in bytes */ + int rc = SQLITE_OK; +#if SQLITE_MAX_WORKER_THREADS==0 +# define nWorker 0 +#else + int nWorker; +#endif + + /* Initialize the upper limit on the number of worker threads */ +#if SQLITE_MAX_WORKER_THREADS>0 + if( sqlite3TempInMemory(db) || sqlite3GlobalConfig.bCoreMutex==0 ){ + nWorker = 0; + }else{ + nWorker = db->aLimit[SQLITE_LIMIT_WORKER_THREADS]; + } +#endif + + /* Do not allow the total number of threads (main thread + all workers) + ** to exceed the maximum merge count */ +#if SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT + if( nWorker>=SORTER_MAX_MERGE_COUNT ){ + nWorker = SORTER_MAX_MERGE_COUNT-1; + } +#endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); - pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); + assert( pCsr->eCurType==CURTYPE_SORTER ); + szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); + sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); + + pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); + pCsr->uc.pSorter = pSorter; if( pSorter==0 ){ - return SQLITE_NOMEM; - } - - pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); - if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; - assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + rc = SQLITE_NOMEM_BKPT; + }else{ + pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); + memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); + pKeyInfo->db = 0; + if( nField && nWorker==0 ){ + pKeyInfo->nXField += (pKeyInfo->nField - nField); + pKeyInfo->nField = nField; + } + pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + pSorter->nTask = nWorker + 1; + pSorter->iPrev = (u8)(nWorker - 1); + pSorter->bUseThreads = (pSorter->nTask>1); + pSorter->db = db; + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + pTask->pSorter = pSorter; + } - if( !sqlite3TempInMemory(db) ){ - pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; - mxCache = db->aDb[0].pSchema->cache_size; - if( mxCachemxPmaSize = mxCache * pgsz; + if( !sqlite3TempInMemory(db) ){ + i64 mxCache; /* Cache size in bytes*/ + u32 szPma = sqlite3GlobalConfig.szPma; + pSorter->mnPmaSize = szPma * pgsz; + + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCache<0 ){ + /* A negative cache-size value C indicates that the cache is abs(C) + ** KiB in size. */ + mxCache = mxCache * -1024; + }else{ + mxCache = mxCache * pgsz; + } + mxCache = MIN(mxCache, SQLITE_MAX_PMASZ); + pSorter->mxPmaSize = MAX(pSorter->mnPmaSize, (int)mxCache); + + /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of + ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary + ** large heap allocations. + */ + if( sqlite3GlobalConfig.pScratch==0 ){ + assert( pSorter->iMemory==0 ); + pSorter->nMemory = pgsz; + pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); + if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM_BKPT; + } + } + + if( (pKeyInfo->nField+pKeyInfo->nXField)<13 + && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl) + ){ + pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT; + } } - return SQLITE_OK; + return rc; } +#undef nWorker /* Defined at the top of this function */ /* ** Free the list of sorted records starting at pRecord. @@ -78405,121 +88958,406 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ SorterRecord *p; SorterRecord *pNext; for(p=pRecord; p; p=pNext){ - pNext = p->pNext; + pNext = p->u.pNext; sqlite3DbFree(db, p); } } +/* +** Free all resources owned by the object indicated by argument pTask. All +** fields of *pTask are zeroed before returning. +*/ +static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ + sqlite3DbFree(db, pTask->pUnpacked); +#if SQLITE_MAX_WORKER_THREADS>0 + /* pTask->list.aMemory can only be non-zero if it was handed memory + ** from the main thread. That only occurs SQLITE_MAX_WORKER_THREADS>0 */ + if( pTask->list.aMemory ){ + sqlite3_free(pTask->list.aMemory); + }else +#endif + { + assert( pTask->list.aMemory==0 ); + vdbeSorterRecordFree(0, pTask->list.pList); + } + if( pTask->file.pFd ){ + sqlite3OsCloseFree(pTask->file.pFd); + } + if( pTask->file2.pFd ){ + sqlite3OsCloseFree(pTask->file2.pFd); + } + memset(pTask, 0, sizeof(SortSubtask)); +} + +#ifdef SQLITE_DEBUG_SORTER_THREADS +static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); + fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterRewindDebug(const char *zEvent){ + i64 t; + sqlite3OsCurrentTimeInt64(sqlite3_vfs_find(0), &t); + fprintf(stderr, "%lld:X %s\n", t, zEvent); +} +static void vdbeSorterPopulateDebug( + SortSubtask *pTask, + const char *zEvent +){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); + fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterBlockDebug( + SortSubtask *pTask, + int bBlocked, + const char *zEvent +){ + if( bBlocked ){ + i64 t; + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); + fprintf(stderr, "%lld:main %s\n", t, zEvent); + } +} +#else +# define vdbeSorterWorkDebug(x,y) +# define vdbeSorterRewindDebug(y) +# define vdbeSorterPopulateDebug(x,y) +# define vdbeSorterBlockDebug(x,y,z) +#endif + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** Join thread pTask->thread. +*/ +static int vdbeSorterJoinThread(SortSubtask *pTask){ + int rc = SQLITE_OK; + if( pTask->pThread ){ +#ifdef SQLITE_DEBUG_SORTER_THREADS + int bDone = pTask->bDone; +#endif + void *pRet = SQLITE_INT_TO_PTR(SQLITE_ERROR); + vdbeSorterBlockDebug(pTask, !bDone, "enter"); + (void)sqlite3ThreadJoin(pTask->pThread, &pRet); + vdbeSorterBlockDebug(pTask, !bDone, "exit"); + rc = SQLITE_PTR_TO_INT(pRet); + assert( pTask->bDone==1 ); + pTask->bDone = 0; + pTask->pThread = 0; + } + return rc; +} + +/* +** Launch a background thread to run xTask(pIn). +*/ +static int vdbeSorterCreateThread( + SortSubtask *pTask, /* Thread will use this task object */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + assert( pTask->pThread==0 && pTask->bDone==0 ); + return sqlite3ThreadCreate(&pTask->pThread, xTask, pIn); +} + +/* +** Join all outstanding threads launched by SorterWrite() to create +** level-0 PMAs. +*/ +static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ + int rc = rcin; + int i; + + /* This function is always called by the main user thread. + ** + ** If this function is being called after SorterRewind() has been called, + ** it is possible that thread pSorter->aTask[pSorter->nTask-1].pThread + ** is currently attempt to join one of the other threads. To avoid a race + ** condition where this thread also attempts to join the same object, join + ** thread pSorter->aTask[pSorter->nTask-1].pThread first. */ + for(i=pSorter->nTask-1; i>=0; i--){ + SortSubtask *pTask = &pSorter->aTask[i]; + int rc2 = vdbeSorterJoinThread(pTask); + if( rc==SQLITE_OK ) rc = rc2; + } + return rc; +} +#else +# define vdbeSorterJoinAll(x,rcin) (rcin) +# define vdbeSorterJoinThread(pTask) SQLITE_OK +#endif + +/* +** Allocate a new MergeEngine object capable of handling up to +** nReader PmaReader inputs. +** +** nReader is automatically rounded up to the next power of two. +** nReader may not exceed SORTER_MAX_MERGE_COUNT even after rounding up. +*/ +static MergeEngine *vdbeMergeEngineNew(int nReader){ + int N = 2; /* Smallest power of two >= nReader */ + int nByte; /* Total bytes of space to allocate */ + MergeEngine *pNew; /* Pointer to allocated object to return */ + + assert( nReader<=SORTER_MAX_MERGE_COUNT ); + + while( NnTree = N; + pNew->pTask = 0; + pNew->aReadr = (PmaReader*)&pNew[1]; + pNew->aTree = (int*)&pNew->aReadr[N]; + } + return pNew; +} + +/* +** Free the MergeEngine object passed as the only argument. +*/ +static void vdbeMergeEngineFree(MergeEngine *pMerger){ + int i; + if( pMerger ){ + for(i=0; inTree; i++){ + vdbePmaReaderClear(&pMerger->aReadr[i]); + } + } + sqlite3_free(pMerger); +} + +/* +** Free all resources associated with the IncrMerger object indicated by +** the first argument. +*/ +static void vdbeIncrFree(IncrMerger *pIncr){ + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + vdbeSorterJoinThread(pIncr->pTask); + if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); + if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); + } +#endif + vdbeMergeEngineFree(pIncr->pMerger); + sqlite3_free(pIncr); + } +} + /* ** Reset a sorting cursor back to its original empty state. */ SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ - if( pSorter->aIter ){ - int i; - for(i=0; inTree; i++){ - vdbeSorterIterZero(db, &pSorter->aIter[i]); - } - sqlite3DbFree(db, pSorter->aIter); - pSorter->aIter = 0; + int i; + (void)vdbeSorterJoinAll(pSorter, SQLITE_OK); + assert( pSorter->bUseThreads || pSorter->pReader==0 ); +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->pReader ){ + vdbePmaReaderClear(pSorter->pReader); + sqlite3DbFree(db, pSorter->pReader); + pSorter->pReader = 0; } - if( pSorter->pTemp1 ){ - sqlite3OsCloseFree(pSorter->pTemp1); - pSorter->pTemp1 = 0; +#endif + vdbeMergeEngineFree(pSorter->pMerger); + pSorter->pMerger = 0; + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + vdbeSortSubtaskCleanup(db, pTask); + pTask->pSorter = pSorter; } - vdbeSorterRecordFree(db, pSorter->pRecord); - pSorter->pRecord = 0; - pSorter->iWriteOff = 0; - pSorter->iReadOff = 0; - pSorter->nInMemory = 0; - pSorter->nTree = 0; - pSorter->nPMA = 0; - pSorter->aTree = 0; + if( pSorter->list.aMemory==0 ){ + vdbeSorterRecordFree(0, pSorter->list.pList); + } + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; + pSorter->bUsePMA = 0; + pSorter->iMemory = 0; + pSorter->mxKeysize = 0; + sqlite3DbFree(db, pSorter->pUnpacked); + pSorter->pUnpacked = 0; } - /* ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); - sqlite3DbFree(db, pSorter->pUnpacked); + sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); - pCsr->pSorter = 0; + pCsr->uc.pSorter = 0; } } +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** The first argument is a file-handle open on a temporary file. The file +** is guaranteed to be nByte bytes or smaller in size. This function +** attempts to extend the file to nByte bytes in size and to ensure that +** the VFS has memory mapped it. +** +** Whether or not the file does end up memory mapped of course depends on +** the specific VFS implementation. +*/ +static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ + if( nByte<=(i64)(db->nMaxSorterMmap) && pFd->pMethods->iVersion>=3 ){ + void *p = 0; + int chunksize = 4*1024; + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_CHUNK_SIZE, &chunksize); + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_SIZE_HINT, &nByte); + sqlite3OsFetch(pFd, 0, (int)nByte, &p); + sqlite3OsUnfetch(pFd, 0, p); + } +} +#else +# define vdbeSorterExtendFile(x,y,z) +#endif + /* ** Allocate space for a file-handle and open a temporary file. If successful, -** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. -** Otherwise, set *ppFile to 0 and return an SQLite error code. +** set *ppFd to point to the malloc'd file-handle and return SQLITE_OK. +** Otherwise, set *ppFd to 0 and return an SQLite error code. */ -static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ - int dummy; - return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, +static int vdbeSorterOpenTempFile( + sqlite3 *db, /* Database handle doing sort */ + i64 nExtend, /* Attempt to extend file to this size */ + sqlite3_file **ppFd +){ + int rc; + if( sqlite3FaultSim(202) ) return SQLITE_IOERR_ACCESS; + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFd, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); + if( rc==SQLITE_OK ){ + i64 max = SQLITE_MAX_MMAP_SIZE; + sqlite3OsFileControlHint(*ppFd, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + if( nExtend>0 ){ + vdbeSorterExtendFile(db, *ppFd, nExtend); + } + } + return rc; } +/* +** If it has not already been allocated, allocate the UnpackedRecord +** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or +** if no allocation was required), or SQLITE_NOMEM otherwise. +*/ +static int vdbeSortAllocUnpacked(SortSubtask *pTask){ + if( pTask->pUnpacked==0 ){ + char *pFree; + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pTask->pSorter->pKeyInfo, 0, 0, &pFree + ); + assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); + if( pFree==0 ) return SQLITE_NOMEM_BKPT; + pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField; + pTask->pUnpacked->errCode = 0; + } + return SQLITE_OK; +} + + /* ** Merge the two sorted lists p1 and p2 into a single list. -** Set *ppOut to the head of the new list. */ -static void vdbeSorterMerge( - const VdbeCursor *pCsr, /* For pKeyInfo */ +static SorterRecord *vdbeSorterMerge( + SortSubtask *pTask, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ - SorterRecord *p2, /* Second list to merge */ - SorterRecord **ppOut /* OUT: Head of merged list */ + SorterRecord *p2 /* Second list to merge */ ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - void *pVal2 = p2 ? p2->pVal : 0; + int bCached = 0; - while( p1 && p2 ){ + assert( p1!=0 && p2!=0 ); + for(;;){ int res; - vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + res = pTask->xCompare( + pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal + ); + if( res<=0 ){ *pp = p1; - pp = &p1->pNext; - p1 = p1->pNext; - pVal2 = 0; + pp = &p1->u.pNext; + p1 = p1->u.pNext; + if( p1==0 ){ + *pp = p2; + break; + } }else{ *pp = p2; - pp = &p2->pNext; - p2 = p2->pNext; - if( p2==0 ) break; - pVal2 = p2->pVal; + pp = &p2->u.pNext; + p2 = p2->u.pNext; + bCached = 0; + if( p2==0 ){ + *pp = p1; + break; + } } } - *pp = p1 ? p1 : p2; - *ppOut = pFinal; + return pFinal; } /* -** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK -** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error -** occurs. +** Return the SorterCompare function to compare values collected by the +** sorter object passed as the only argument. */ -static int vdbeSorterSort(const VdbeCursor *pCsr){ +static SorterCompare vdbeSorterGetCompare(VdbeSorter *p){ + if( p->typeMask==SORTER_TYPE_INTEGER ){ + return vdbeSorterCompareInt; + }else if( p->typeMask==SORTER_TYPE_TEXT ){ + return vdbeSorterCompareText; + } + return vdbeSorterCompare; +} + +/* +** Sort the linked list of records headed at pTask->pList. Return +** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if +** an error occurs. +*/ +static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ int i; SorterRecord **aSlot; SorterRecord *p; - VdbeSorter *pSorter = pCsr->pSorter; + int rc; + + rc = vdbeSortAllocUnpacked(pTask); + if( rc!=SQLITE_OK ) return rc; + + p = pList->pList; + pTask->xCompare = vdbeSorterGetCompare(pTask->pSorter); aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } - p = pSorter->pRecord; while( p ){ - SorterRecord *pNext = p->pNext; - p->pNext = 0; + SorterRecord *pNext; + if( pList->aMemory ){ + if( (u8*)p==pList->aMemory ){ + pNext = 0; + }else{ + assert( p->u.iNextaMemory) ); + pNext = (SorterRecord*)&pList->aMemory[p->u.iNext]; + } + }else{ + pNext = p->u.pNext; + } + + p->u.pNext = 0; for(i=0; aSlot[i]; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); + p = vdbeSorterMerge(pTask, p, aSlot[i]); aSlot[i] = 0; } aSlot[i] = p; @@ -78528,42 +89366,44 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ p = 0; for(i=0; i<64; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); + if( aSlot[i]==0 ) continue; + p = p ? vdbeSorterMerge(pTask, p, aSlot[i]) : aSlot[i]; } - pSorter->pRecord = p; + pList->pList = p; sqlite3_free(aSlot); - return SQLITE_OK; + assert( pTask->pUnpacked->errCode==SQLITE_OK + || pTask->pUnpacked->errCode==SQLITE_NOMEM + ); + return pTask->pUnpacked->errCode; } /* -** Initialize a file-writer object. +** Initialize a PMA-writer object. */ -static void fileWriterInit( - sqlite3 *db, /* Database (for malloc) */ - sqlite3_file *pFile, /* File to write to */ - FileWriter *p, /* Object to populate */ - i64 iStart /* Offset of pFile to begin writing at */ +static void vdbePmaWriterInit( + sqlite3_file *pFd, /* File handle to write to */ + PmaWriter *p, /* Object to populate */ + int nBuf, /* Buffer size */ + i64 iStart /* Offset of pFd to begin writing at */ ){ - int nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - - memset(p, 0, sizeof(FileWriter)); - p->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + memset(p, 0, sizeof(PmaWriter)); + p->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !p->aBuffer ){ - p->eFWErr = SQLITE_NOMEM; + p->eFWErr = SQLITE_NOMEM_BKPT; }else{ p->iBufEnd = p->iBufStart = (iStart % nBuf); p->iWriteOff = iStart - p->iBufStart; p->nBuffer = nBuf; - p->pFile = pFile; + p->pFd = pFd; } } /* -** Write nData bytes of data to the file-write object. Return SQLITE_OK +** Write nData bytes of data to the PMA. Return SQLITE_OK ** if successful, or an SQLite error code if an error occurs. */ -static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ +static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ int nRem = nData; while( nRem>0 && p->eFWErr==0 ){ int nCopy = nRem; @@ -78574,7 +89414,7 @@ static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy); p->iBufEnd += nCopy; if( p->iBufEnd==p->nBuffer ){ - p->eFWErr = sqlite3OsWrite(p->pFile, + p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); @@ -78588,43 +89428,44 @@ static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ } /* -** Flush any buffered data to disk and clean up the file-writer object. -** The results of using the file-writer after this call are undefined. +** Flush any buffered data to disk and clean up the PMA-writer object. +** The results of using the PMA-writer after this call are undefined. ** Return SQLITE_OK if flushing the buffered data succeeds or is not ** required. Otherwise, return an SQLite error code. ** ** Before returning, set *piEof to the offset immediately following the ** last byte written to the file. */ -static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){ +static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ - p->eFWErr = sqlite3OsWrite(p->pFile, + p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); } *piEof = (p->iWriteOff + p->iBufEnd); - sqlite3DbFree(db, p->aBuffer); + sqlite3_free(p->aBuffer); rc = p->eFWErr; - memset(p, 0, sizeof(FileWriter)); + memset(p, 0, sizeof(PmaWriter)); return rc; } /* -** Write value iVal encoded as a varint to the file-write object. Return +** Write value iVal encoded as a varint to the PMA. Return ** SQLITE_OK if successful, or an SQLite error code if an error occurs. */ -static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ +static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ int nByte; u8 aByte[10]; nByte = sqlite3PutVarint(aByte, iVal); - fileWriterWrite(p, aByte, nByte); + vdbePmaWriteBlob(p, aByte, nByte); } /* -** Write the current contents of the in-memory linked-list to a PMA. Return -** SQLITE_OK if successful, or an SQLite error code otherwise. +** Write the current contents of in-memory linked-list pList to a level-0 +** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if +** successful, or an SQLite error code otherwise. ** ** The format of a PMA is: ** @@ -78635,76 +89476,256 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ +static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ + sqlite3 *db = pTask->pSorter->db; int rc = SQLITE_OK; /* Return code */ - VdbeSorter *pSorter = pCsr->pSorter; - FileWriter writer; + PmaWriter writer; /* Object used to write to the file */ - memset(&writer, 0, sizeof(FileWriter)); +#ifdef SQLITE_DEBUG + /* Set iSz to the expected size of file pTask->file after writing the PMA. + ** This is used by an assert() statement at the end of this function. */ + i64 iSz = pList->szPMA + sqlite3VarintLen(pList->szPMA) + pTask->file.iEof; +#endif - if( pSorter->nInMemory==0 ){ - assert( pSorter->pRecord==0 ); - return rc; - } - - rc = vdbeSorterSort(pCsr); + vdbeSorterWorkDebug(pTask, "enter"); + memset(&writer, 0, sizeof(PmaWriter)); + assert( pList->szPMA>0 ); /* If the first temporary PMA file has not been opened, open it now. */ - if( rc==SQLITE_OK && pSorter->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); - assert( rc!=SQLITE_OK || pSorter->pTemp1 ); - assert( pSorter->iWriteOff==0 ); - assert( pSorter->nPMA==0 ); + if( pTask->file.pFd==0 ){ + rc = vdbeSorterOpenTempFile(db, 0, &pTask->file.pFd); + assert( rc!=SQLITE_OK || pTask->file.pFd ); + assert( pTask->file.iEof==0 ); + assert( pTask->nPMA==0 ); + } + + /* Try to get the file to memory map */ + if( rc==SQLITE_OK ){ + vdbeSorterExtendFile(db, pTask->file.pFd, pTask->file.iEof+pList->szPMA+9); + } + + /* Sort the list */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterSort(pTask, pList); } if( rc==SQLITE_OK ){ SorterRecord *p; SorterRecord *pNext = 0; - fileWriterInit(db, pSorter->pTemp1, &writer, pSorter->iWriteOff); - pSorter->nPMA++; - fileWriterWriteVarint(&writer, pSorter->nInMemory); - for(p=pSorter->pRecord; p; p=pNext){ - pNext = p->pNext; - fileWriterWriteVarint(&writer, p->nVal); - fileWriterWrite(&writer, p->pVal, p->nVal); - sqlite3DbFree(db, p); + vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pSorter->pgsz, + pTask->file.iEof); + pTask->nPMA++; + vdbePmaWriteVarint(&writer, pList->szPMA); + for(p=pList->pList; p; p=pNext){ + pNext = p->u.pNext; + vdbePmaWriteVarint(&writer, p->nVal); + vdbePmaWriteBlob(&writer, SRVAL(p), p->nVal); + if( pList->aMemory==0 ) sqlite3_free(p); + } + pList->pList = p; + rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); + } + + vdbeSorterWorkDebug(pTask, "exit"); + assert( rc!=SQLITE_OK || pList->pList==0 ); + assert( rc!=SQLITE_OK || pTask->file.iEof==iSz ); + return rc; +} + +/* +** Advance the MergeEngine to its next entry. +** Set *pbEof to true there is no next entry because +** the MergeEngine has reached the end of all its inputs. +** +** Return SQLITE_OK if successful or an error code if an error occurs. +*/ +static int vdbeMergeEngineStep( + MergeEngine *pMerger, /* The merge engine to advance to the next row */ + int *pbEof /* Set TRUE at EOF. Set false for more content */ +){ + int rc; + int iPrev = pMerger->aTree[1];/* Index of PmaReader to advance */ + SortSubtask *pTask = pMerger->pTask; + + /* Advance the current PmaReader */ + rc = vdbePmaReaderNext(&pMerger->aReadr[iPrev]); + + /* Update contents of aTree[] */ + if( rc==SQLITE_OK ){ + int i; /* Index of aTree[] to recalculate */ + PmaReader *pReadr1; /* First PmaReader to compare */ + PmaReader *pReadr2; /* Second PmaReader to compare */ + int bCached = 0; + + /* Find the first two PmaReaders to compare. The one that was just + ** advanced (iPrev) and the one next to it in the array. */ + pReadr1 = &pMerger->aReadr[(iPrev & 0xFFFE)]; + pReadr2 = &pMerger->aReadr[(iPrev | 0x0001)]; + + for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ + /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ + int iRes; + if( pReadr1->pFd==0 ){ + iRes = +1; + }else if( pReadr2->pFd==0 ){ + iRes = -1; + }else{ + iRes = pTask->xCompare(pTask, &bCached, + pReadr1->aKey, pReadr1->nKey, pReadr2->aKey, pReadr2->nKey + ); + } + + /* If pReadr1 contained the smaller value, set aTree[i] to its index. + ** Then set pReadr2 to the next PmaReader to compare to pReadr1. In this + ** case there is no cache of pReadr2 in pTask->pUnpacked, so set + ** pKey2 to point to the record belonging to pReadr2. + ** + ** Alternatively, if pReadr2 contains the smaller of the two values, + ** set aTree[i] to its index and update pReadr1. If vdbeSorterCompare() + ** was actually called above, then pTask->pUnpacked now contains + ** a value equivalent to pReadr2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pReadr2 again. + ** + ** If the two values were equal, then the value from the oldest + ** PMA should be considered smaller. The VdbeSorter.aReadr[] array + ** is sorted from oldest to newest, so pReadr1 contains older values + ** than pReadr2 iff (pReadr1aTree[i] = (int)(pReadr1 - pMerger->aReadr); + pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; + bCached = 0; + }else{ + if( pReadr1->pFd ) bCached = 0; + pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); + pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; + } + } + *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFd==0); + } + + return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for background threads that write level-0 PMAs. +*/ +static void *vdbeSorterFlushThread(void *pCtx){ + SortSubtask *pTask = (SortSubtask*)pCtx; + int rc; /* Return code */ + assert( pTask->bDone==0 ); + rc = vdbeSorterListToPMA(pTask, &pTask->list); + pTask->bDone = 1; + return SQLITE_INT_TO_PTR(rc); +} +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ + +/* +** Flush the current contents of VdbeSorter.list to a new PMA, possibly +** using a background thread. +*/ +static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ +#if SQLITE_MAX_WORKER_THREADS==0 + pSorter->bUsePMA = 1; + return vdbeSorterListToPMA(&pSorter->aTask[0], &pSorter->list); +#else + int rc = SQLITE_OK; + int i; + SortSubtask *pTask = 0; /* Thread context used to create new PMA */ + int nWorker = (pSorter->nTask-1); + + /* Set the flag to indicate that at least one PMA has been written. + ** Or will be, anyhow. */ + pSorter->bUsePMA = 1; + + /* Select a sub-task to sort and flush the current list of in-memory + ** records to disk. If the sorter is running in multi-threaded mode, + ** round-robin between the first (pSorter->nTask-1) tasks. Except, if + ** the background thread from a sub-tasks previous turn is still running, + ** skip it. If the first (pSorter->nTask-1) sub-tasks are all still busy, + ** fall back to using the final sub-task. The first (pSorter->nTask-1) + ** sub-tasks are prefered as they use background threads - the final + ** sub-task uses the main thread. */ + for(i=0; iiPrev + i + 1) % nWorker; + pTask = &pSorter->aTask[iTest]; + if( pTask->bDone ){ + rc = vdbeSorterJoinThread(pTask); + } + if( rc!=SQLITE_OK || pTask->pThread==0 ) break; + } + + if( rc==SQLITE_OK ){ + if( i==nWorker ){ + /* Use the foreground thread for this operation */ + rc = vdbeSorterListToPMA(&pSorter->aTask[nWorker], &pSorter->list); + }else{ + /* Launch a background thread for this operation */ + u8 *aMem = pTask->list.aMemory; + void *pCtx = (void*)pTask; + + assert( pTask->pThread==0 && pTask->bDone==0 ); + assert( pTask->list.pList==0 ); + assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); + + pSorter->iPrev = (u8)(pTask - pSorter->aTask); + pTask->list = pSorter->list; + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; + if( aMem ){ + pSorter->list.aMemory = aMem; + pSorter->nMemory = sqlite3MallocSize(aMem); + }else if( pSorter->list.aMemory ){ + pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); + if( !pSorter->list.aMemory ) return SQLITE_NOMEM_BKPT; + } + + rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx); } - pSorter->pRecord = p; - rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } return rc; +#endif /* SQLITE_MAX_WORKER_THREADS!=0 */ } /* ** Add a record to the sorter. */ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( - sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ + int bFlush; /* True to flush contents of memory to PMA */ + int nReq; /* Bytes of memory required */ + int nPMA; /* Bytes of PMA space required */ + int t; /* serial type of first record field */ - assert( pSorter ); - pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); - if( pNew==0 ){ - rc = SQLITE_NOMEM; + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + getVarint32((const u8*)&pVal->z[1], t); + if( t>0 && t<10 && t!=7 ){ + pSorter->typeMask &= SORTER_TYPE_INTEGER; + }else if( t>10 && (t & 0x01) ){ + pSorter->typeMask &= SORTER_TYPE_TEXT; }else{ - pNew->pVal = (void *)&pNew[1]; - memcpy(pNew->pVal, pVal->z, pVal->n); - pNew->nVal = pVal->n; - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; + pSorter->typeMask = 0; } - /* See if the contents of the sorter should now be written out. They - ** are written out when either of the following are true: + assert( pSorter ); + + /* Figure out whether or not the current contents of memory should be + ** flushed to a PMA before continuing. If so, do so. + ** + ** If using the single large allocation mode (pSorter->aMemory!=0), then + ** flush the contents of memory to a new PMA if (a) at least one value is + ** already in memory and (b) the new value will not fit in memory. + ** + ** Or, if using separate allocations for each record, flush the contents + ** of memory to a PMA if either of the following are true: ** ** * The total memory allocated for the in-memory list is greater ** than (page-size * cache-size), or @@ -78712,161 +89733,812 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( - (pSorter->nInMemory>pSorter->mxPmaSize) - || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - )){ -#ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; + nReq = pVal->n + sizeof(SorterRecord); + nPMA = pVal->n + sqlite3VarintLen(pVal->n); + if( pSorter->mxPmaSize ){ + if( pSorter->list.aMemory ){ + bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; + }else{ + bFlush = ( + (pSorter->list.szPMA > pSorter->mxPmaSize) + || (pSorter->list.szPMA > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + ); + } + if( bFlush ){ + rc = vdbeSorterFlushPMA(pSorter); + pSorter->list.szPMA = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || pSorter->list.pList==0 ); + } + } + + pSorter->list.szPMA += nPMA; + if( nPMA>pSorter->mxKeysize ){ + pSorter->mxKeysize = nPMA; + } + + if( pSorter->list.aMemory ){ + int nMin = pSorter->iMemory + nReq; + + if( nMin>pSorter->nMemory ){ + u8 *aNew; + int iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; + int nNew = pSorter->nMemory * 2; + while( nNew < nMin ) nNew = nNew*2; + if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; + if( nNew < nMin ) nNew = nMin; + + aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); + if( !aNew ) return SQLITE_NOMEM_BKPT; + pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; + pSorter->list.aMemory = aNew; + pSorter->nMemory = nNew; + } + + pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; + pSorter->iMemory += ROUND8(nReq); + if( pSorter->list.pList ){ + pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); + } + }else{ + pNew = (SorterRecord *)sqlite3Malloc(nReq); + if( pNew==0 ){ + return SQLITE_NOMEM_BKPT; + } + pNew->u.pNext = pSorter->list.pList; + } + + memcpy(SRVAL(pNew), pVal->z, pVal->n); + pNew->nVal = pVal->n; + pSorter->list.pList = pNew; + + return rc; +} + +/* +** Read keys from pIncr->pMerger and populate pIncr->aFile[1]. The format +** of the data stored in aFile[1] is the same as that used by regular PMAs, +** except that the number-of-bytes varint is omitted from the start. +*/ +static int vdbeIncrPopulate(IncrMerger *pIncr){ + int rc = SQLITE_OK; + int rc2; + i64 iStart = pIncr->iStartOff; + SorterFile *pOut = &pIncr->aFile[1]; + SortSubtask *pTask = pIncr->pTask; + MergeEngine *pMerger = pIncr->pMerger; + PmaWriter writer; + assert( pIncr->bEof==0 ); + + vdbeSorterPopulateDebug(pTask, "enter"); + + vdbePmaWriterInit(pOut->pFd, &writer, pTask->pSorter->pgsz, iStart); + while( rc==SQLITE_OK ){ + int dummy; + PmaReader *pReader = &pMerger->aReadr[ pMerger->aTree[1] ]; + int nKey = pReader->nKey; + i64 iEof = writer.iWriteOff + writer.iBufEnd; + + /* Check if the output file is full or if the input has been exhausted. + ** In either case exit the loop. */ + if( pReader->pFd==0 ) break; + if( (iEof + nKey + sqlite3VarintLen(nKey))>(iStart + pIncr->mxSz) ) break; + + /* Write the next key to the output. */ + vdbePmaWriteVarint(&writer, nKey); + vdbePmaWriteBlob(&writer, pReader->aKey, nKey); + assert( pIncr->pMerger->pTask==pTask ); + rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy); + } + + rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); + if( rc==SQLITE_OK ) rc = rc2; + vdbeSorterPopulateDebug(pTask, "exit"); + return rc; +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for background threads that populate aFile[1] of +** multi-threaded IncrMerger objects. +*/ +static void *vdbeIncrPopulateThread(void *pCtx){ + IncrMerger *pIncr = (IncrMerger*)pCtx; + void *pRet = SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); + pIncr->pTask->bDone = 1; + return pRet; +} + +/* +** Launch a background thread to populate aFile[1] of pIncr. +*/ +static int vdbeIncrBgPopulate(IncrMerger *pIncr){ + void *p = (void*)pIncr; + assert( pIncr->bUseThread ); + return vdbeSorterCreateThread(pIncr->pTask, vdbeIncrPopulateThread, p); +} #endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + +/* +** This function is called when the PmaReader corresponding to pIncr has +** finished reading the contents of aFile[0]. Its purpose is to "refill" +** aFile[0] such that the PmaReader should start rereading it from the +** beginning. +** +** For single-threaded objects, this is accomplished by literally reading +** keys from pIncr->pMerger and repopulating aFile[0]. +** +** For multi-threaded objects, all that is required is to wait until the +** background thread is finished (if it is not already) and then swap +** aFile[0] and aFile[1] in place. If the contents of pMerger have not +** been exhausted, this function also launches a new background thread +** to populate the new aFile[1]. +** +** SQLITE_OK is returned on success, or an SQLite error code otherwise. +*/ +static int vdbeIncrSwap(IncrMerger *pIncr){ + int rc = SQLITE_OK; + +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + rc = vdbeSorterJoinThread(pIncr->pTask); + + if( rc==SQLITE_OK ){ + SorterFile f0 = pIncr->aFile[0]; + pIncr->aFile[0] = pIncr->aFile[1]; + pIncr->aFile[1] = f0; + } + + if( rc==SQLITE_OK ){ + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ + pIncr->bEof = 1; + }else{ + rc = vdbeIncrBgPopulate(pIncr); + } + } + }else +#endif + { + rc = vdbeIncrPopulate(pIncr); + pIncr->aFile[0] = pIncr->aFile[1]; + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ + pIncr->bEof = 1; + } } return rc; } /* -** Helper function for sqlite3VdbeSorterRewind(). +** Allocate and return a new IncrMerger object to read data from pMerger. +** +** If an OOM condition is encountered, return NULL. In this case free the +** pMerger argument before returning. */ -static int vdbeSorterInitMerge( - sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Cursor handle for this sorter */ - i64 *pnByte /* Sum of bytes in all opened PMAs */ +static int vdbeIncrMergerNew( + SortSubtask *pTask, /* The thread that will be using the new IncrMerger */ + MergeEngine *pMerger, /* The MergeEngine that the IncrMerger will control */ + IncrMerger **ppOut /* Write the new IncrMerger here */ +){ + int rc = SQLITE_OK; + IncrMerger *pIncr = *ppOut = (IncrMerger*) + (sqlite3FaultSim(100) ? 0 : sqlite3MallocZero(sizeof(*pIncr))); + if( pIncr ){ + pIncr->pMerger = pMerger; + pIncr->pTask = pTask; + pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); + pTask->file2.iEof += pIncr->mxSz; + }else{ + vdbeMergeEngineFree(pMerger); + rc = SQLITE_NOMEM_BKPT; + } + return rc; +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** Set the "use-threads" flag on object pIncr. +*/ +static void vdbeIncrMergerSetThreads(IncrMerger *pIncr){ + pIncr->bUseThread = 1; + pIncr->pTask->file2.iEof -= pIncr->mxSz; +} +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ + + + +/* +** Recompute pMerger->aTree[iOut] by comparing the next keys on the +** two PmaReaders that feed that entry. Neither of the PmaReaders +** are advanced. This routine merely does the comparison. +*/ +static void vdbeMergeEngineCompare( + MergeEngine *pMerger, /* Merge engine containing PmaReaders to compare */ + int iOut /* Store the result in pMerger->aTree[iOut] */ +){ + int i1; + int i2; + int iRes; + PmaReader *p1; + PmaReader *p2; + + assert( iOutnTree && iOut>0 ); + + if( iOut>=(pMerger->nTree/2) ){ + i1 = (iOut - pMerger->nTree/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pMerger->aTree[iOut*2]; + i2 = pMerger->aTree[iOut*2+1]; + } + + p1 = &pMerger->aReadr[i1]; + p2 = &pMerger->aReadr[i2]; + + if( p1->pFd==0 ){ + iRes = i2; + }else if( p2->pFd==0 ){ + iRes = i1; + }else{ + SortSubtask *pTask = pMerger->pTask; + int bCached = 0; + int res; + assert( pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ + res = pTask->xCompare( + pTask, &bCached, p1->aKey, p1->nKey, p2->aKey, p2->nKey + ); + if( res<=0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pMerger->aTree[iOut] = iRes; +} + +/* +** Allowed values for the eMode parameter to vdbeMergeEngineInit() +** and vdbePmaReaderIncrMergeInit(). +** +** Only INCRINIT_NORMAL is valid in single-threaded builds (when +** SQLITE_MAX_WORKER_THREADS==0). The other values are only used +** when there exists one or more separate worker threads. +*/ +#define INCRINIT_NORMAL 0 +#define INCRINIT_TASK 1 +#define INCRINIT_ROOT 2 + +/* +** Forward reference required as the vdbeIncrMergeInit() and +** vdbePmaReaderIncrInit() routines are called mutually recursively when +** building a merge tree. +*/ +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); + +/* +** Initialize the MergeEngine object passed as the second argument. Once this +** function returns, the first key of merged data may be read from the +** MergeEngine object in the usual fashion. +** +** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge +** objects attached to the PmaReader objects that the merger reads from have +** already been populated, but that they have not yet populated aFile[0] and +** set the PmaReader objects up to read from it. In this case all that is +** required is to call vdbePmaReaderNext() on each PmaReader to point it at +** its first key. +** +** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use +** vdbePmaReaderIncrMergeInit() to initialize each PmaReader that feeds data +** to pMerger. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int vdbeMergeEngineInit( + SortSubtask *pTask, /* Thread that will run pMerger */ + MergeEngine *pMerger, /* MergeEngine to initialize */ + int eMode /* One of the INCRINIT_XXX constants */ ){ - VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; /* Return code */ - int i; /* Used to iterator through aIter[] */ - i64 nByte = 0; /* Total bytes in all opened PMAs */ + int i; /* For looping over PmaReader objects */ + int nTree = pMerger->nTree; - /* Initialize the iterators. */ - for(i=0; iaIter[i]; - rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); - pSorter->iReadOff = pIter->iEof; - assert( rc!=SQLITE_OK || pSorter->iReadOff<=pSorter->iWriteOff ); - if( rc!=SQLITE_OK || pSorter->iReadOff>=pSorter->iWriteOff ) break; + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + + /* Verify that the MergeEngine is assigned to a single thread */ + assert( pMerger->pTask==0 ); + pMerger->pTask = pTask; + + for(i=0; i0 && eMode==INCRINIT_ROOT ){ + /* PmaReaders should be normally initialized in order, as if they are + ** reading from the same temp file this makes for more linear file IO. + ** However, in the INCRINIT_ROOT case, if PmaReader aReadr[nTask-1] is + ** in use it will block the vdbePmaReaderNext() call while it uses + ** the main thread to fill its buffer. So calling PmaReaderNext() + ** on this PmaReader before any of the multi-threaded PmaReaders takes + ** better advantage of multi-processor hardware. */ + rc = vdbePmaReaderNext(&pMerger->aReadr[nTree-i-1]); + }else{ + rc = vdbePmaReaderIncrInit(&pMerger->aReadr[i], INCRINIT_NORMAL); + } + if( rc!=SQLITE_OK ) return rc; } - /* Initialize the aTree[] array. */ - for(i=pSorter->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pCsr, i); + for(i=pMerger->nTree-1; i>0; i--){ + vdbeMergeEngineCompare(pMerger, i); + } + return pTask->pUnpacked->errCode; +} + +/* +** The PmaReader passed as the first argument is guaranteed to be an +** incremental-reader (pReadr->pIncr!=0). This function serves to open +** and/or initialize the temp file related fields of the IncrMerge +** object at (pReadr->pIncr). +** +** If argument eMode is set to INCRINIT_NORMAL, then all PmaReaders +** in the sub-tree headed by pReadr are also initialized. Data is then +** loaded into the buffers belonging to pReadr and it is set to point to +** the first key in its range. +** +** If argument eMode is set to INCRINIT_TASK, then pReadr is guaranteed +** to be a multi-threaded PmaReader and this function is being called in a +** background thread. In this case all PmaReaders in the sub-tree are +** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to +** pReadr is populated. However, pReadr itself is not set up to point +** to its first key. A call to vdbePmaReaderNext() is still required to do +** that. +** +** The reason this function does not call vdbePmaReaderNext() immediately +** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that it has +** to block on thread (pTask->thread) before accessing aFile[1]. But, since +** this entire function is being run by thread (pTask->thread), that will +** lead to the current background thread attempting to join itself. +** +** Finally, if argument eMode is set to INCRINIT_ROOT, it may be assumed +** that pReadr->pIncr is a multi-threaded IncrMerge objects, and that all +** child-trees have already been initialized using IncrInit(INCRINIT_TASK). +** In this case vdbePmaReaderNext() is called on all child PmaReaders and +** the current PmaReader set to point to the first key in its range. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ + int rc = SQLITE_OK; + IncrMerger *pIncr = pReadr->pIncr; + SortSubtask *pTask = pIncr->pTask; + sqlite3 *db = pTask->pSorter->db; + + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + + rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); + + /* Set up the required files for pIncr. A multi-theaded IncrMerge object + ** requires two temp files to itself, whereas a single-threaded object + ** only requires a region of pTask->file2. */ + if( rc==SQLITE_OK ){ + int mxSz = pIncr->mxSz; +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); + } + }else +#endif + /*if( !pIncr->bUseThread )*/{ + if( pTask->file2.pFd==0 ){ + assert( pTask->file2.iEof>0 ); + rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); + pTask->file2.iEof = 0; + } + if( rc==SQLITE_OK ){ + pIncr->aFile[1].pFd = pTask->file2.pFd; + pIncr->iStartOff = pTask->file2.iEof; + pTask->file2.iEof += mxSz; + } + } } - *pnByte = nByte; +#if SQLITE_MAX_WORKER_THREADS>0 + if( rc==SQLITE_OK && pIncr->bUseThread ){ + /* Use the current thread to populate aFile[1], even though this + ** PmaReader is multi-threaded. If this is an INCRINIT_TASK object, + ** then this function is already running in background thread + ** pIncr->pTask->thread. + ** + ** If this is the INCRINIT_ROOT object, then it is running in the + ** main VDBE thread. But that is Ok, as that thread cannot return + ** control to the VDBE or proceed with anything useful until the + ** first results are ready from this merger object anyway. + */ + assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); + rc = vdbeIncrPopulate(pIncr); + } +#endif + + if( rc==SQLITE_OK && (SQLITE_MAX_WORKER_THREADS==0 || eMode!=INCRINIT_TASK) ){ + rc = vdbePmaReaderNext(pReadr); + } + + return rc; +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for vdbePmaReaderIncrMergeInit() operations run in +** background threads. +*/ +static void *vdbePmaReaderBgIncrInit(void *pCtx){ + PmaReader *pReader = (PmaReader*)pCtx; + void *pRet = SQLITE_INT_TO_PTR( + vdbePmaReaderIncrMergeInit(pReader,INCRINIT_TASK) + ); + pReader->pIncr->pTask->bDone = 1; + return pRet; +} +#endif + +/* +** If the PmaReader passed as the first argument is not an incremental-reader +** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it invokes +** the vdbePmaReaderIncrMergeInit() function with the parameters passed to +** this routine to initialize the incremental merge. +** +** If the IncrMerger object is multi-threaded (IncrMerger.bUseThread==1), +** then a background thread is launched to call vdbePmaReaderIncrMergeInit(). +** Or, if the IncrMerger is single threaded, the same function is called +** using the current thread. +*/ +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ + IncrMerger *pIncr = pReadr->pIncr; /* Incremental merger */ + int rc = SQLITE_OK; /* Return code */ + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + assert( pIncr->bUseThread==0 || eMode==INCRINIT_TASK ); + if( pIncr->bUseThread ){ + void *pCtx = (void*)pReadr; + rc = vdbeSorterCreateThread(pIncr->pTask, vdbePmaReaderBgIncrInit, pCtx); + }else +#endif + { + rc = vdbePmaReaderIncrMergeInit(pReadr, eMode); + } + } return rc; } /* -** Once the sorter has been populated, this function is called to prepare -** for iterating through its contents in sorted order. +** Allocate a new MergeEngine object to merge the contents of nPMA level-0 +** PMAs from pTask->file. If no error occurs, set *ppOut to point to +** the new object and return SQLITE_OK. Or, if an error does occur, set *ppOut +** to NULL and return an SQLite error code. +** +** When this function is called, *piOffset is set to the offset of the +** first PMA to read from pTask->file. Assuming no error occurs, it is +** set to the offset immediately following the last byte of the last +** PMA before returning. If an error does occur, then the final value of +** *piOffset is undefined. */ -SQLITE_PRIVATE int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ - VdbeSorter *pSorter = pCsr->pSorter; - int rc; /* Return code */ - sqlite3_file *pTemp2 = 0; /* Second temp file to use */ - i64 iWrite2 = 0; /* Write offset for pTemp2 */ - int nIter; /* Number of iterators used */ - int nByte; /* Bytes of space required for aIter/aTree */ - int N = 2; /* Power of 2 >= nIter */ +static int vdbeMergeEngineLevel0( + SortSubtask *pTask, /* Sorter task to read from */ + int nPMA, /* Number of PMAs to read */ + i64 *piOffset, /* IN/OUT: Readr offset in pTask->file */ + MergeEngine **ppOut /* OUT: New merge-engine */ +){ + MergeEngine *pNew; /* Merge engine to return */ + i64 iOff = *piOffset; + int i; + int rc = SQLITE_OK; + *ppOut = pNew = vdbeMergeEngineNew(nPMA); + if( pNew==0 ) rc = SQLITE_NOMEM_BKPT; + + for(i=0; iaReadr[i]; + rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pReadr, &nDummy); + iOff = pReadr->iEof; + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pNew); + *ppOut = 0; + } + *piOffset = iOff; + return rc; +} + +/* +** Return the depth of a tree comprising nPMA PMAs, assuming a fanout of +** SORTER_MAX_MERGE_COUNT. The returned value does not include leaf nodes. +** +** i.e. +** +** nPMA<=16 -> TreeDepth() == 0 +** nPMA<=256 -> TreeDepth() == 1 +** nPMA<=65536 -> TreeDepth() == 2 +*/ +static int vdbeSorterTreeDepth(int nPMA){ + int nDepth = 0; + i64 nDiv = SORTER_MAX_MERGE_COUNT; + while( nDiv < (i64)nPMA ){ + nDiv = nDiv * SORTER_MAX_MERGE_COUNT; + nDepth++; + } + return nDepth; +} + +/* +** pRoot is the root of an incremental merge-tree with depth nDepth (according +** to vdbeSorterTreeDepth()). pLeaf is the iSeq'th leaf to be added to the +** tree, counting from zero. This function adds pLeaf to the tree. +** +** If successful, SQLITE_OK is returned. If an error occurs, an SQLite error +** code is returned and pLeaf is freed. +*/ +static int vdbeSorterAddToTree( + SortSubtask *pTask, /* Task context */ + int nDepth, /* Depth of tree according to TreeDepth() */ + int iSeq, /* Sequence number of leaf within tree */ + MergeEngine *pRoot, /* Root of tree */ + MergeEngine *pLeaf /* Leaf to add to tree */ +){ + int rc = SQLITE_OK; + int nDiv = 1; + int i; + MergeEngine *p = pRoot; + IncrMerger *pIncr; + + rc = vdbeIncrMergerNew(pTask, pLeaf, &pIncr); + + for(i=1; iaReadr[iIter]; + + if( pReadr->pIncr==0 ){ + MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pNew==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = vdbeIncrMergerNew(pTask, pNew, &pReadr->pIncr); + } + } + if( rc==SQLITE_OK ){ + p = pReadr->pIncr->pMerger; + nDiv = nDiv / SORTER_MAX_MERGE_COUNT; + } + } + + if( rc==SQLITE_OK ){ + p->aReadr[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; + }else{ + vdbeIncrFree(pIncr); + } + return rc; +} + +/* +** This function is called as part of a SorterRewind() operation on a sorter +** that has already written two or more level-0 PMAs to one or more temp +** files. It builds a tree of MergeEngine/IncrMerger/PmaReader objects that +** can be used to incrementally merge all PMAs on disk. +** +** If successful, SQLITE_OK is returned and *ppOut set to point to the +** MergeEngine object at the root of the tree before returning. Or, if an +** error occurs, an SQLite error code is returned and the final value +** of *ppOut is undefined. +*/ +static int vdbeSorterMergeTreeBuild( + VdbeSorter *pSorter, /* The VDBE cursor that implements the sort */ + MergeEngine **ppOut /* Write the MergeEngine here */ +){ + MergeEngine *pMain = 0; + int rc = SQLITE_OK; + int iTask; + +#if SQLITE_MAX_WORKER_THREADS>0 + /* If the sorter uses more than one task, then create the top-level + ** MergeEngine here. This MergeEngine will read data from exactly + ** one PmaReader per sub-task. */ + assert( pSorter->bUseThreads || pSorter->nTask==1 ); + if( pSorter->nTask>1 ){ + pMain = vdbeMergeEngineNew(pSorter->nTask); + if( pMain==0 ) rc = SQLITE_NOMEM_BKPT; + } +#endif + + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + SortSubtask *pTask = &pSorter->aTask[iTask]; + assert( pTask->nPMA>0 || SQLITE_MAX_WORKER_THREADS>0 ); + if( SQLITE_MAX_WORKER_THREADS==0 || pTask->nPMA ){ + MergeEngine *pRoot = 0; /* Root node of tree for this task */ + int nDepth = vdbeSorterTreeDepth(pTask->nPMA); + i64 iReadOff = 0; + + if( pTask->nPMA<=SORTER_MAX_MERGE_COUNT ){ + rc = vdbeMergeEngineLevel0(pTask, pTask->nPMA, &iReadOff, &pRoot); + }else{ + int i; + int iSeq = 0; + pRoot = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pRoot==0 ) rc = SQLITE_NOMEM_BKPT; + for(i=0; inPMA && rc==SQLITE_OK; i += SORTER_MAX_MERGE_COUNT){ + MergeEngine *pMerger = 0; /* New level-0 PMA merger */ + int nReader; /* Number of level-0 PMAs to merge */ + + nReader = MIN(pTask->nPMA - i, SORTER_MAX_MERGE_COUNT); + rc = vdbeMergeEngineLevel0(pTask, nReader, &iReadOff, &pMerger); + if( rc==SQLITE_OK ){ + rc = vdbeSorterAddToTree(pTask, nDepth, iSeq++, pRoot, pMerger); + } + } + } + + if( rc==SQLITE_OK ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pMain!=0 ){ + rc = vdbeIncrMergerNew(pTask, pRoot, &pMain->aReadr[iTask].pIncr); + }else +#endif + { + assert( pMain==0 ); + pMain = pRoot; + } + }else{ + vdbeMergeEngineFree(pRoot); + } + } + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMain); + pMain = 0; + } + *ppOut = pMain; + return rc; +} + +/* +** This function is called as part of an sqlite3VdbeSorterRewind() operation +** on a sorter that has written two or more PMAs to temporary files. It sets +** up either VdbeSorter.pMerger (for single threaded sorters) or pReader +** (for multi-threaded sorters) so that it can be used to iterate through +** all records stored in the sorter. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ + int rc; /* Return code */ + SortSubtask *pTask0 = &pSorter->aTask[0]; + MergeEngine *pMain = 0; +#if SQLITE_MAX_WORKER_THREADS + sqlite3 *db = pTask0->pSorter->db; + int i; + SorterCompare xCompare = vdbeSorterGetCompare(pSorter); + for(i=0; inTask; i++){ + pSorter->aTask[i].xCompare = xCompare; + } +#endif + + rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); + if( rc==SQLITE_OK ){ +#if SQLITE_MAX_WORKER_THREADS + assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); + if( pSorter->bUseThreads ){ + int iTask; + PmaReader *pReadr = 0; + SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; + rc = vdbeSortAllocUnpacked(pLast); + if( rc==SQLITE_OK ){ + pReadr = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); + pSorter->pReader = pReadr; + if( pReadr==0 ) rc = SQLITE_NOMEM_BKPT; + } + if( rc==SQLITE_OK ){ + rc = vdbeIncrMergerNew(pLast, pMain, &pReadr->pIncr); + if( rc==SQLITE_OK ){ + vdbeIncrMergerSetThreads(pReadr->pIncr); + for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ + IncrMerger *pIncr; + if( (pIncr = pMain->aReadr[iTask].pIncr) ){ + vdbeIncrMergerSetThreads(pIncr); + assert( pIncr->pTask!=pLast ); + } + } + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + /* Check that: + ** + ** a) The incremental merge object is configured to use the + ** right task, and + ** b) If it is using task (nTask-1), it is configured to run + ** in single-threaded mode. This is important, as the + ** root merge (INCRINIT_ROOT) will be using the same task + ** object. + */ + PmaReader *p = &pMain->aReadr[iTask]; + assert( p->pIncr==0 || ( + (p->pIncr->pTask==&pSorter->aTask[iTask]) /* a */ + && (iTask!=pSorter->nTask-1 || p->pIncr->bUseThread==0) /* b */ + )); + rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); + } + } + pMain = 0; + } + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderIncrMergeInit(pReadr, INCRINIT_ROOT); + } + }else +#endif + { + rc = vdbeMergeEngineInit(pTask0, pMain, INCRINIT_NORMAL); + pSorter->pMerger = pMain; + pMain = 0; + } + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMain); + } + return rc; +} + + +/* +** Once the sorter has been populated by calls to sqlite3VdbeSorterWrite, +** this function is called to prepare for iterating through the records +** in sorted order. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ + VdbeSorter *pSorter; + int rc = SQLITE_OK; /* Return code */ + + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; assert( pSorter ); /* If no data has been written to disk, then do not do so now. Instead, ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly ** from the in-memory list. */ - if( pSorter->nPMA==0 ){ - *pbEof = !pSorter->pRecord; - assert( pSorter->aTree==0 ); - return vdbeSorterSort(pCsr); - } - - /* Write the current in-memory list to a PMA. */ - rc = vdbeSorterListToPMA(db, pCsr); - if( rc!=SQLITE_OK ) return rc; - - /* Allocate space for aIter[] and aTree[]. */ - nIter = pSorter->nPMA; - if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT; - assert( nIter>0 ); - while( NaIter = (VdbeSorterIter *)sqlite3DbMallocZero(db, nByte); - if( !pSorter->aIter ) return SQLITE_NOMEM; - pSorter->aTree = (int *)&pSorter->aIter[N]; - pSorter->nTree = N; - - do { - int iNew; /* Index of new, merged, PMA */ - - for(iNew=0; - rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNTnPMA; - iNew++ - ){ - int rc2; /* Return code from fileWriterFinish() */ - FileWriter writer; /* Object used to write to disk */ - i64 nWrite; /* Number of bytes in new PMA */ - - memset(&writer, 0, sizeof(FileWriter)); - - /* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1, - ** initialize an iterator for each of them and break out of the loop. - ** These iterators will be incrementally merged as the VDBE layer calls - ** sqlite3VdbeSorterNext(). - ** - ** Otherwise, if pTemp1 contains more than SORTER_MAX_MERGE_COUNT PMAs, - ** initialize interators for SORTER_MAX_MERGE_COUNT of them. These PMAs - ** are merged into a single PMA that is written to file pTemp2. - */ - rc = vdbeSorterInitMerge(db, pCsr, &nWrite); - assert( rc!=SQLITE_OK || pSorter->aIter[ pSorter->aTree[1] ].pFile ); - if( rc!=SQLITE_OK || pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ - break; - } - - /* Open the second temp file, if it is not already open. */ - if( pTemp2==0 ){ - assert( iWrite2==0 ); - rc = vdbeSorterOpenTempFile(db, &pTemp2); - } - - if( rc==SQLITE_OK ){ - int bEof = 0; - fileWriterInit(db, pTemp2, &writer, iWrite2); - fileWriterWriteVarint(&writer, nWrite); - while( rc==SQLITE_OK && bEof==0 ){ - VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - assert( pIter->pFile ); - - fileWriterWriteVarint(&writer, pIter->nKey); - fileWriterWrite(&writer, pIter->aKey, pIter->nKey); - rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); - } - rc2 = fileWriterFinish(db, &writer, &iWrite2); - if( rc==SQLITE_OK ) rc = rc2; - } - } - - if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ - break; + if( pSorter->bUsePMA==0 ){ + if( pSorter->list.pList ){ + *pbEof = 0; + rc = vdbeSorterSort(&pSorter->aTask[0], &pSorter->list); }else{ - sqlite3_file *pTmp = pSorter->pTemp1; - pSorter->nPMA = iNew; - pSorter->pTemp1 = pTemp2; - pTemp2 = pTmp; - pSorter->iWriteOff = iWrite2; - pSorter->iReadOff = 0; - iWrite2 = 0; + *pbEof = 1; } - }while( rc==SQLITE_OK ); - - if( pTemp2 ){ - sqlite3OsCloseFree(pTemp2); + return rc; } - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + + /* Write the current in-memory list to a PMA. When the VdbeSorterWrite() + ** function flushes the contents of memory to disk, it immediately always + ** creates a new list consisting of a single key immediately afterwards. + ** So the list is never empty at this point. */ + assert( pSorter->list.pList ); + rc = vdbeSorterFlushPMA(pSorter); + + /* Join all threads */ + rc = vdbeSorterJoinAll(pSorter, rc); + + vdbeSorterRewindDebug("rewind"); + + /* Assuming no errors have occurred, set up a merger structure to + ** incrementally read and merge all remaining PMAs. */ + assert( pSorter->pReader==0 ); + if( rc==SQLITE_OK ){ + rc = vdbeSorterSetupMerge(pSorter); + *pbEof = 0; + } + + vdbeSorterRewindDebug("rewinddone"); return rc; } @@ -78874,66 +90546,33 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, ** Advance to the next element in the sorter. */ SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; int rc; /* Return code */ - if( pSorter->aTree ){ - int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - if( rc==SQLITE_OK ){ - int i; /* Index of aTree[] to recalculate */ - VdbeSorterIter *pIter1; /* First iterator to compare */ - VdbeSorterIter *pIter2; /* Second iterator to compare */ - u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ - - /* Find the first two iterators to compare. The one that was just - ** advanced (iPrev) and the one next to it in the array. */ - pIter1 = &pSorter->aIter[(iPrev & 0xFFFE)]; - pIter2 = &pSorter->aIter[(iPrev | 0x0001)]; - pKey2 = pIter2->aKey; - - for(i=(pSorter->nTree+iPrev)/2; i>0; i=i/2){ - /* Compare pIter1 and pIter2. Store the result in variable iRes. */ - int iRes; - if( pIter1->pFile==0 ){ - iRes = +1; - }else if( pIter2->pFile==0 ){ - iRes = -1; - }else{ - vdbeSorterCompare(pCsr, 0, - pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes - ); - } - - /* If pIter1 contained the smaller value, set aTree[i] to its index. - ** Then set pIter2 to the next iterator to compare to pIter1. In this - ** case there is no cache of pIter2 in pSorter->pUnpacked, so set - ** pKey2 to point to the record belonging to pIter2. - ** - ** Alternatively, if pIter2 contains the smaller of the two values, - ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() - ** was actually called above, then pSorter->pUnpacked now contains - ** a value equivalent to pIter2. So set pKey2 to NULL to prevent - ** vdbeSorterCompare() from decoding pIter2 again. */ - if( iRes<=0 ){ - pSorter->aTree[i] = (int)(pIter1 - pSorter->aIter); - pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; - pKey2 = pIter2->aKey; - }else{ - if( pIter1->pFile ) pKey2 = 0; - pSorter->aTree[i] = (int)(pIter2 - pSorter->aIter); - pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; - } - - } - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + assert( pSorter->bUsePMA || (pSorter->pReader==0 && pSorter->pMerger==0) ); + if( pSorter->bUsePMA ){ + assert( pSorter->pReader==0 || pSorter->pMerger==0 ); + assert( pSorter->bUseThreads==0 || pSorter->pReader ); + assert( pSorter->bUseThreads==1 || pSorter->pMerger ); +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->bUseThreads ){ + rc = vdbePmaReaderNext(pSorter->pReader); + *pbEof = (pSorter->pReader->pFd==0); + }else +#endif + /*if( !pSorter->bUseThreads )*/ { + assert( pSorter->pMerger!=0 ); + assert( pSorter->pMerger->pTask==(&pSorter->aTask[0]) ); + rc = vdbeMergeEngineStep(pSorter->pMerger, pbEof); } }else{ - SorterRecord *pFree = pSorter->pRecord; - pSorter->pRecord = pFree->pNext; - pFree->pNext = 0; - vdbeSorterRecordFree(db, pFree); - *pbEof = !pSorter->pRecord; + SorterRecord *pFree = pSorter->list.pList; + pSorter->list.pList = pFree->u.pNext; + pFree->u.pNext = 0; + if( pSorter->list.aMemory==0 ) vdbeSorterRecordFree(db, pFree); + *pbEof = !pSorter->list.pList; rc = SQLITE_OK; } return rc; @@ -78948,14 +90587,21 @@ static void *vdbeSorterRowkey( int *pnKey /* OUT: Size of current key in bytes */ ){ void *pKey; - if( pSorter->aTree ){ - VdbeSorterIter *pIter; - pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - *pnKey = pIter->nKey; - pKey = pIter->aKey; + if( pSorter->bUsePMA ){ + PmaReader *pReader; +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->bUseThreads ){ + pReader = pSorter->pReader; + }else +#endif + /*if( !pSorter->bUseThreads )*/{ + pReader = &pSorter->pMerger->aReadr[pSorter->pMerger->aTree[1]]; + } + *pnKey = pReader->nKey; + pKey = pReader->aKey; }else{ - *pnKey = pSorter->pRecord->nVal; - pKey = pSorter->pRecord->pVal; + *pnKey = pSorter->list.pList->nVal; + pKey = SRVAL(pSorter->list.pList); } return pKey; } @@ -78964,12 +90610,14 @@ static void *vdbeSorterRowkey( ** Copy the current sorter key into the memory cell pOut. */ SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; void *pKey; int nKey; /* Sorter key to copy into pOut */ + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; pKey = vdbeSorterRowkey(pSorter, &nKey); - if( sqlite3VdbeMemGrow(pOut, nKey, 0) ){ - return SQLITE_NOMEM; + if( sqlite3VdbeMemClearAndResize(pOut, nKey) ){ + return SQLITE_NOMEM_BKPT; } pOut->n = nKey; MemSetTypeFlag(pOut, MEM_Blob); @@ -78983,284 +90631,56 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ ** passed as the first argument currently points to. For the purposes of ** the comparison, ignore the rowid field at the end of each record. ** +** If the sorter cursor key contains any NULL values, consider it to be +** less than pVal. Even if pVal also contains NULL values. +** ** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). ** Otherwise, set *pRes to a negative, zero or positive value if the ** key in pVal is smaller than, equal to or larger than the current sorter ** key. +** +** This routine forms the core of the OP_SorterCompare opcode, which in +** turn is used to verify uniqueness when constructing a UNIQUE INDEX. */ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal, /* Value to compare to current sorter key */ - int nKeyCol, /* Only compare this many fields */ + int nKeyCol, /* Compare this many columns */ int *pRes /* OUT: Result of comparison */ ){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; + UnpackedRecord *r2; + KeyInfo *pKeyInfo; + int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + r2 = pSorter->pUnpacked; + pKeyInfo = pCsr->pKeyInfo; + if( r2==0 ){ + char *p; + r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p); + assert( pSorter->pUnpacked==(UnpackedRecord*)p ); + if( r2==0 ) return SQLITE_NOMEM_BKPT; + r2->nField = nKeyCol; + } + assert( r2->nField==nKeyCol ); + pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pCsr, nKeyCol, pVal->z, pVal->n, pKey, nKey, pRes); + sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + for(i=0; iaMem[i].flags & MEM_Null ){ + *pRes = -1; + return SQLITE_OK; + } + } + + *pRes = sqlite3VdbeRecordCompare(pVal->n, pVal->z, r2); return SQLITE_OK; } /************** End of vdbesort.c ********************************************/ -/************** Begin file journal.c *****************************************/ -/* -** 2007 August 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements a special kind of sqlite3_file object used -** by SQLite to create journal files if the atomic-write optimization -** is enabled. -** -** The distinctive characteristic of this sqlite3_file is that the -** actual on disk file is created lazily. When the file is created, -** the caller specifies a buffer size for an in-memory buffer to -** be used to service read() and write() requests. The actual file -** on disk is not created or populated until either: -** -** 1) The in-memory representation grows too large for the allocated -** buffer, or -** 2) The sqlite3JournalCreate() function is called. -*/ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - - -/* -** A JournalFile object is a subclass of sqlite3_file used by -** as an open file handle for journal files. -*/ -struct JournalFile { - sqlite3_io_methods *pMethod; /* I/O methods on journal files */ - int nBuf; /* Size of zBuf[] in bytes */ - char *zBuf; /* Space to buffer journal writes */ - int iSize; /* Amount of zBuf[] currently used */ - int flags; /* xOpen flags */ - sqlite3_vfs *pVfs; /* The "real" underlying VFS */ - sqlite3_file *pReal; /* The "real" underlying file descriptor */ - const char *zJournal; /* Name of the journal file */ -}; -typedef struct JournalFile JournalFile; - -/* -** If it does not already exists, create and populate the on-disk file -** for JournalFile p. -*/ -static int createFile(JournalFile *p){ - int rc = SQLITE_OK; - if( !p->pReal ){ - sqlite3_file *pReal = (sqlite3_file *)&p[1]; - rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); - if( rc==SQLITE_OK ){ - p->pReal = pReal; - if( p->iSize>0 ){ - assert(p->iSize<=p->nBuf); - rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); - } - if( rc!=SQLITE_OK ){ - /* If an error occurred while writing to the file, close it before - ** returning. This way, SQLite uses the in-memory journal data to - ** roll back changes made to the internal page-cache before this - ** function was called. */ - sqlite3OsClose(pReal); - p->pReal = 0; - } - } - } - return rc; -} - -/* -** Close the file. -*/ -static int jrnlClose(sqlite3_file *pJfd){ - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - sqlite3OsClose(p->pReal); - } - sqlite3_free(p->zBuf); - return SQLITE_OK; -} - -/* -** Read data from the file. -*/ -static int jrnlRead( - sqlite3_file *pJfd, /* The journal file from which to read */ - void *zBuf, /* Put the results here */ - int iAmt, /* Number of bytes to read */ - sqlite_int64 iOfst /* Begin reading at this offset */ -){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); - }else if( (iAmt+iOfst)>p->iSize ){ - rc = SQLITE_IOERR_SHORT_READ; - }else{ - memcpy(zBuf, &p->zBuf[iOfst], iAmt); - } - return rc; -} - -/* -** Write data to the file. -*/ -static int jrnlWrite( - sqlite3_file *pJfd, /* The journal file into which to write */ - const void *zBuf, /* Take data to be written from here */ - int iAmt, /* Number of bytes to write */ - sqlite_int64 iOfst /* Begin writing at this offset into the file */ -){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( !p->pReal && (iOfst+iAmt)>p->nBuf ){ - rc = createFile(p); - } - if( rc==SQLITE_OK ){ - if( p->pReal ){ - rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); - }else{ - memcpy(&p->zBuf[iOfst], zBuf, iAmt); - if( p->iSize<(iOfst+iAmt) ){ - p->iSize = (iOfst+iAmt); - } - } - } - return rc; -} - -/* -** Truncate the file. -*/ -static int jrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsTruncate(p->pReal, size); - }else if( sizeiSize ){ - p->iSize = size; - } - return rc; -} - -/* -** Sync the file. -*/ -static int jrnlSync(sqlite3_file *pJfd, int flags){ - int rc; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsSync(p->pReal, flags); - }else{ - rc = SQLITE_OK; - } - return rc; -} - -/* -** Query the size of the file in bytes. -*/ -static int jrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsFileSize(p->pReal, pSize); - }else{ - *pSize = (sqlite_int64) p->iSize; - } - return rc; -} - -/* -** Table of methods for JournalFile sqlite3_file object. -*/ -static struct sqlite3_io_methods JournalFileMethods = { - 1, /* iVersion */ - jrnlClose, /* xClose */ - jrnlRead, /* xRead */ - jrnlWrite, /* xWrite */ - jrnlTruncate, /* xTruncate */ - jrnlSync, /* xSync */ - jrnlFileSize, /* xFileSize */ - 0, /* xLock */ - 0, /* xUnlock */ - 0, /* xCheckReservedLock */ - 0, /* xFileControl */ - 0, /* xSectorSize */ - 0, /* xDeviceCharacteristics */ - 0, /* xShmMap */ - 0, /* xShmLock */ - 0, /* xShmBarrier */ - 0 /* xShmUnmap */ -}; - -/* -** Open a journal file. -*/ -SQLITE_PRIVATE int sqlite3JournalOpen( - sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ - const char *zName, /* Name of the journal file */ - sqlite3_file *pJfd, /* Preallocated, blank file handle */ - int flags, /* Opening flags */ - int nBuf /* Bytes buffered before opening the file */ -){ - JournalFile *p = (JournalFile *)pJfd; - memset(p, 0, sqlite3JournalSize(pVfs)); - if( nBuf>0 ){ - p->zBuf = sqlite3MallocZero(nBuf); - if( !p->zBuf ){ - return SQLITE_NOMEM; - } - }else{ - return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); - } - p->pMethod = &JournalFileMethods; - p->nBuf = nBuf; - p->flags = flags; - p->zJournal = zName; - p->pVfs = pVfs; - return SQLITE_OK; -} - -/* -** If the argument p points to a JournalFile structure, and the underlying -** file has not yet been created, create it now. -*/ -SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ - if( p->pMethods!=&JournalFileMethods ){ - return SQLITE_OK; - } - return createFile((JournalFile *)p); -} - -/* -** The file-handle passed as the only argument is guaranteed to be an open -** file. It may or may not be of class JournalFile. If the file is a -** JournalFile, and the underlying file on disk has not yet been opened, -** return 0. Otherwise, return 1. -*/ -SQLITE_PRIVATE int sqlite3JournalExists(sqlite3_file *p){ - return (p->pMethods!=&JournalFileMethods || ((JournalFile *)p)->pReal!=0); -} - -/* -** Return the number of bytes required to store a JournalFile that uses vfs -** pVfs to create the underlying on-disk files. -*/ -SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ - return (pVfs->szOsFile+sizeof(JournalFile)); -} -#endif - -/************** End of journal.c *********************************************/ /************** Begin file memjournal.c **************************************/ /* ** 2008 October 7 @@ -79277,31 +90697,45 @@ SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ ** This file contains code use to implement an in-memory rollback journal. ** The in-memory rollback journal is used to journal transactions for ** ":memory:" databases and when the journal_mode=MEMORY pragma is used. +** +** Update: The in-memory journal is also used to temporarily cache +** smaller journals that are not critical for power-loss recovery. +** For example, statement journals that are not too big will be held +** entirely in memory, thus reducing the number of file I/O calls, and +** more importantly, reducing temporary file creation events. If these +** journals become too large for memory, they are spilled to disk. But +** in the common case, they are usually small and no file I/O needs to +** occur. */ +/* #include "sqliteInt.h" */ /* Forward references to internal structures */ typedef struct MemJournal MemJournal; typedef struct FilePoint FilePoint; typedef struct FileChunk FileChunk; -/* Space to hold the rollback journal is allocated in increments of -** this many bytes. -** -** The size chosen is a little less than a power of two. That way, -** the FileChunk object will have a size that almost exactly fills -** a power-of-two allocation. This mimimizes wasted space in power-of-two -** memory allocators. -*/ -#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) - /* ** The rollback journal is composed of a linked list of these structures. +** +** The zChunk array is always at least 8 bytes in size - usually much more. +** Its actual size is stored in the MemJournal.nChunkSize variable. */ struct FileChunk { FileChunk *pNext; /* Next chunk in the journal */ - u8 zChunk[JOURNAL_CHUNKSIZE]; /* Content of this chunk */ + u8 zChunk[8]; /* Content of this chunk */ }; +/* +** By default, allocate this many bytes of memory for each FileChunk object. +*/ +#define MEMJOURNAL_DFLT_FILECHUNKSIZE 1024 + +/* +** For chunk size nChunkSize, return the number of bytes that should +** be allocated for each FileChunk structure. +*/ +#define fileChunkSize(nChunkSize) (sizeof(FileChunk) + ((nChunkSize)-8)) + /* ** An instance of this object serves as a cursor into the rollback journal. ** The cursor can be either for reading or writing. @@ -79312,14 +90746,22 @@ struct FilePoint { }; /* -** This subclass is a subclass of sqlite3_file. Each open memory-journal +** This structure is a subclass of sqlite3_file. Each open memory-journal ** is an instance of this class. */ struct MemJournal { - sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + const sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + int nChunkSize; /* In-memory chunk-size */ + + int nSpill; /* Bytes of data before flushing */ + int nSize; /* Bytes of data currently in memory */ FileChunk *pFirst; /* Head of in-memory chunk-list */ FilePoint endpoint; /* Pointer to the end of the file */ FilePoint readpoint; /* Pointer to the end of the last xRead() */ + + int flags; /* xOpen flags */ + sqlite3_vfs *pVfs; /* The "real" underlying VFS */ + const char *zJournal; /* Name of the journal file */ }; /* @@ -79338,36 +90780,94 @@ static int memjrnlRead( int iChunkOffset; FileChunk *pChunk; - /* SQLite never tries to read past the end of a rollback journal file */ - assert( iOfst+iAmt<=p->endpoint.iOffset ); +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + if( (iAmt+iOfst)>p->endpoint.iOffset ){ + return SQLITE_IOERR_SHORT_READ; + } +#endif + assert( (iAmt+iOfst)<=p->endpoint.iOffset ); + assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ sqlite3_int64 iOff = 0; for(pChunk=p->pFirst; - ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst; + ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst; pChunk=pChunk->pNext ){ - iOff += JOURNAL_CHUNKSIZE; + iOff += p->nChunkSize; } }else{ pChunk = p->readpoint.pChunk; + assert( pChunk!=0 ); } - iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE); + iChunkOffset = (int)(iOfst%p->nChunkSize); do { - int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset; - int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset)); - memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); + int iSpace = p->nChunkSize - iChunkOffset; + int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset)); + memcpy(zOut, (u8*)pChunk->zChunk + iChunkOffset, nCopy); zOut += nCopy; nRead -= iSpace; iChunkOffset = 0; } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); - p->readpoint.iOffset = iOfst+iAmt; + p->readpoint.iOffset = pChunk ? iOfst+iAmt : 0; p->readpoint.pChunk = pChunk; return SQLITE_OK; } +/* +** Free the list of FileChunk structures headed at MemJournal.pFirst. +*/ +static void memjrnlFreeChunks(MemJournal *p){ + FileChunk *pIter; + FileChunk *pNext; + for(pIter=p->pFirst; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } + p->pFirst = 0; +} + +/* +** Flush the contents of memory to a real file on disk. +*/ +static int memjrnlCreateFile(MemJournal *p){ + int rc; + sqlite3_file *pReal = (sqlite3_file*)p; + MemJournal copy = *p; + + memset(p, 0, sizeof(MemJournal)); + rc = sqlite3OsOpen(copy.pVfs, copy.zJournal, pReal, copy.flags, 0); + if( rc==SQLITE_OK ){ + int nChunk = copy.nChunkSize; + i64 iOff = 0; + FileChunk *pIter; + for(pIter=copy.pFirst; pIter; pIter=pIter->pNext){ + if( iOff + nChunk > copy.endpoint.iOffset ){ + nChunk = copy.endpoint.iOffset - iOff; + } + rc = sqlite3OsWrite(pReal, (u8*)pIter->zChunk, nChunk, iOff); + if( rc ) break; + iOff += nChunk; + } + if( rc==SQLITE_OK ){ + /* No error has occurred. Free the in-memory buffers. */ + memjrnlFreeChunks(©); + } + } + if( rc!=SQLITE_OK ){ + /* If an error occurred while creating or writing to the file, restore + ** the original before returning. This way, SQLite uses the in-memory + ** journal data to roll back changes made to the internal page-cache + ** before this function was called. */ + sqlite3OsClose(pReal); + *p = copy; + } + return rc; +} + + /* ** Write data to the file. */ @@ -79381,38 +90881,62 @@ static int memjrnlWrite( int nWrite = iAmt; u8 *zWrite = (u8 *)zBuf; - /* An in-memory journal file should only ever be appended to. Random - ** access writes are not required by sqlite. - */ - assert( iOfst==p->endpoint.iOffset ); - UNUSED_PARAMETER(iOfst); - - while( nWrite>0 ){ - FileChunk *pChunk = p->endpoint.pChunk; - int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE); - int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset); - - if( iChunkOffset==0 ){ - /* New chunk is required to extend the file. */ - FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk)); - if( !pNew ){ - return SQLITE_IOERR_NOMEM; - } - pNew->pNext = 0; - if( pChunk ){ - assert( p->pFirst ); - pChunk->pNext = pNew; - }else{ - assert( !p->pFirst ); - p->pFirst = pNew; - } - p->endpoint.pChunk = pNew; + /* If the file should be created now, create it and write the new data + ** into the file on disk. */ + if( p->nSpill>0 && (iAmt+iOfst)>p->nSpill ){ + int rc = memjrnlCreateFile(p); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pJfd, zBuf, iAmt, iOfst); } + return rc; + } - memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); - zWrite += iSpace; - nWrite -= iSpace; - p->endpoint.iOffset += iSpace; + /* If the contents of this write should be stored in memory */ + else{ + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required. The only exception to this is when + ** the in-memory journal is being used by a connection using the + ** atomic-write optimization. In this case the first 28 bytes of the + ** journal file may be written as part of committing the transaction. */ + assert( iOfst==p->endpoint.iOffset || iOfst==0 ); +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + if( iOfst==0 && p->pFirst ){ + assert( p->nChunkSize>iAmt ); + memcpy((u8*)p->pFirst->zChunk, zBuf, iAmt); + }else +#else + assert( iOfst>0 || p->pFirst==0 ); +#endif + { + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); + int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); + + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; + } + + memcpy((u8*)p->endpoint.pChunk->zChunk + iChunkOffset, zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; + } + p->nSize = iAmt + iOfst; + } } return SQLITE_OK; @@ -79420,19 +90944,21 @@ static int memjrnlWrite( /* ** Truncate the file. +** +** If the journal file is already on disk, truncate it there. Or, if it +** is still in main memory but is being truncated to zero bytes in size, +** ignore */ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ MemJournal *p = (MemJournal *)pJfd; - FileChunk *pChunk; - assert(size==0); - UNUSED_PARAMETER(size); - pChunk = p->pFirst; - while( pChunk ){ - FileChunk *pTmp = pChunk; - pChunk = pChunk->pNext; - sqlite3_free(pTmp); + if( ALWAYS(size==0) ){ + memjrnlFreeChunks(p); + p->nSize = 0; + p->endpoint.pChunk = 0; + p->endpoint.iOffset = 0; + p->readpoint.pChunk = 0; + p->readpoint.iOffset = 0; } - sqlite3MemJournalOpen(pJfd); return SQLITE_OK; } @@ -79440,21 +90966,19 @@ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ ** Close the file. */ static int memjrnlClose(sqlite3_file *pJfd){ - memjrnlTruncate(pJfd, 0); + MemJournal *p = (MemJournal *)pJfd; + memjrnlFreeChunks(p); return SQLITE_OK; } - /* ** Sync the file. ** -** Syncing an in-memory journal is a no-op. And, in fact, this routine -** is never called in a working implementation. This implementation -** exists purely as a contingency, in case some malfunction in some other -** part of SQLite causes Sync to be called by mistake. +** If the real file has been created, call its xSync method. Otherwise, +** syncing an in-memory journal is a no-op. */ -static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); +static int memjrnlSync(sqlite3_file *pJfd, int flags){ + UNUSED_PARAMETER2(pJfd, flags); return SQLITE_OK; } @@ -79493,28 +91017,88 @@ static const struct sqlite3_io_methods MemJournalMethods = { }; /* -** Open a journal file. +** Open a journal file. +** +** The behaviour of the journal file depends on the value of parameter +** nSpill. If nSpill is 0, then the journal file is always create and +** accessed using the underlying VFS. If nSpill is less than zero, then +** all content is always stored in main-memory. Finally, if nSpill is a +** positive value, then the journal file is initially created in-memory +** but may be flushed to disk later on. In this case the journal file is +** flushed to disk either when it grows larger than nSpill bytes in size, +** or when sqlite3JournalCreate() is called. */ -SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *pJfd){ - MemJournal *p = (MemJournal *)pJfd; - assert( EIGHT_BYTE_ALIGNMENT(p) ); - memset(p, 0, sqlite3MemJournalSize()); - p->pMethod = (sqlite3_io_methods*)&MemJournalMethods; +SQLITE_PRIVATE int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nSpill /* Bytes buffered before opening the file */ +){ + MemJournal *p = (MemJournal*)pJfd; + + /* Zero the file-handle object. If nSpill was passed zero, initialize + ** it using the sqlite3OsOpen() function of the underlying VFS. In this + ** case none of the code in this module is executed as a result of calls + ** made on the journal file-handle. */ + memset(p, 0, sizeof(MemJournal)); + if( nSpill==0 ){ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + } + + if( nSpill>0 ){ + p->nChunkSize = nSpill; + }else{ + p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk); + assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) ); + } + + p->pMethod = (const sqlite3_io_methods*)&MemJournalMethods; + p->nSpill = nSpill; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; } /* -** Return true if the file-handle passed as an argument is -** an in-memory journal +** Open an in-memory journal file. */ -SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *pJfd){ - return pJfd->pMethods==&MemJournalMethods; +SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *pJfd){ + sqlite3JournalOpen(0, 0, pJfd, 0, -1); +} + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE +/* +** If the argument p points to a MemJournal structure that is not an +** in-memory-only journal file (i.e. is one that was opened with a +ve +** nSpill parameter), and the underlying file has not yet been created, +** create it now. +*/ +SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ + int rc = SQLITE_OK; + if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nSpill>0 ){ + rc = memjrnlCreateFile((MemJournal*)p); + } + return rc; +} +#endif + +/* +** The file-handle passed as the only argument is open on a journal file. +** Return true if this "journal file" is currently stored in heap memory, +** or false otherwise. +*/ +SQLITE_PRIVATE int sqlite3JournalIsInMemory(sqlite3_file *p){ + return p->pMethods==&MemJournalMethods; } /* -** Return the number of bytes required to store a MemJournal file descriptor. +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. */ -SQLITE_PRIVATE int sqlite3MemJournalSize(void){ - return sizeof(MemJournal); +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return MAX(pVfs->szOsFile, (int)sizeof(MemJournal)); } /************** End of memjournal.c ******************************************/ @@ -79533,13 +91117,14 @@ SQLITE_PRIVATE int sqlite3MemJournalSize(void){ ** This file contains routines used for walking the parser tree for ** an SQL statement. */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ /* ** Walk an expression tree. Invoke the callback once for each node -** of the expression, while decending. (In other words, the callback +** of the expression, while descending. (In other words, the callback ** is invoked before visiting children.) ** ** The return value from the callback should be one of the WRC_* @@ -79556,23 +91141,25 @@ SQLITE_PRIVATE int sqlite3MemJournalSize(void){ ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ -SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ +static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ int rc; - if( pExpr==0 ) return WRC_Continue; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); rc = pWalker->xExprCallback(pWalker, pExpr); - if( rc==WRC_Continue - && !ExprHasProperty(pExpr,EP_TokenOnly) ){ - if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; - }else{ - if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; - } + if( rc || ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + return rc & WRC_Abort; } - return rc & WRC_Abort; + if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; + if( pExpr->pRight && walkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; + }else if( pExpr->x.pList ){ + if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + } + return WRC_Continue; +} +SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ + return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; } /* @@ -79625,6 +91212,11 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ return WRC_Abort; } + if( pItem->fg.isTabFunc + && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } } } return WRC_Continue; @@ -79691,6 +91283,7 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ ** resolve all identifiers by associating them with a particular ** table and column. */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -79704,7 +91297,7 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ ** is a helper function - a callback for the tree walker. */ static int incrAggDepth(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.i; + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; return WRC_Continue; } static void incrAggFunctionDepth(Expr *pExpr, int N){ @@ -79712,7 +91305,7 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ Walker w; memset(&w, 0, sizeof(w)); w.xExprCallback = incrAggDepth; - w.u.i = N; + w.u.n = N; sqlite3WalkExpr(&w, pExpr); } } @@ -79721,30 +91314,6 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** Turn the pExpr expression into an alias for the iCol-th column of the ** result set in pEList. ** -** If the result set column is a simple column reference, then this routine -** makes an exact copy. But for any other kind of expression, this -** routine make a copy of the result set column as the argument to the -** TK_AS operator. The TK_AS operator causes the expression to be -** evaluated just once and then reused for each alias. -** -** The reason for suppressing the TK_AS term when the expression is a simple -** column reference is so that the column reference will be recognized as -** usable by indices within the WHERE clause processing logic. -** -** The TK_AS operator is inhibited if zType[0]=='G'. This means -** that in a GROUP BY clause, the expression is evaluated twice. Hence: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x -** -** Is equivalent to: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 -** -** The result of random()%5 in the GROUP BY clause is probably different -** from the result in the result-set. On the other hand Standard SQL does -** not allow the GROUP BY clause to contain references to result-set columns. -** So this should never come up in well-formed queries. -** ** If the reference is followed by a COLLATE operator, then make sure ** the COLLATE operator is preserved. For example: ** @@ -79755,7 +91324,7 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase; ** ** The nSubquery parameter specifies how many levels of subquery the -** alias is removed from the original expression. The usually value is +** alias is removed from the original expression. The usual value is ** zero but it might be more if the alias is contained within a subquery ** of the original expression. The Expr.op2 field of TK_AGG_FUNCTION ** structures must be increased by the nSubquery amount. @@ -79775,23 +91344,14 @@ static void resolveAlias( assert( iCol>=0 && iColnExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); - assert( pOrig->flags & EP_Resolved ); db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( pDup==0 ) return; - if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ - incrAggFunctionDepth(pDup, nSubquery); - pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); - if( pDup==0 ) return; - ExprSetProperty(pDup, EP_Skip); - if( pEList->a[iCol].u.x.iAlias==0 ){ - pEList->a[iCol].u.x.iAlias = (u16)(++pParse->nAlias); - } - pDup->iTable = pEList->a[iCol].u.x.iAlias; - } + if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } + ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, @@ -79923,14 +91483,15 @@ static int lookupName( testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IsCheck ); if( (pNC->ncFlags & (NC_PartIdx|NC_IsCheck))!=0 ){ - /* Silently ignore database qualifiers inside CHECK constraints and partial - ** indices. Do not raise errors because that might break legacy and - ** because it does not hurt anything to just ignore the database name. */ + /* Silently ignore database qualifiers inside CHECK constraints and + ** partial indices. Do not raise errors because that might break + ** legacy and because it does not hurt anything to just ignore the + ** database name. */ zDb = 0; }else{ for(i=0; inDb; i++){ - assert( db->aDb[i].zName ); - if( sqlite3StrICmp(db->aDb[i].zName,zDb)==0 ){ + assert( db->aDb[i].zDbSName ); + if( sqlite3StrICmp(db->aDb[i].zDbSName,zDb)==0 ){ pSchema = db->aDb[i].pSchema; break; } @@ -79982,7 +91543,7 @@ static int lookupName( ** USING clause, then skip this match. */ if( cnt==1 ){ - if( pItem->jointype & JT_NATURAL ) continue; + if( pItem->fg.jointype & JT_NATURAL ) continue; if( nameInUsingClause(pItem->pUsing, zCol) ) continue; } cnt++; @@ -79996,6 +91557,11 @@ static int lookupName( if( pMatch ){ pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; + /* RIGHT JOIN not (yet) supported */ + assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); + if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ + ExprSetProperty(pExpr, EP_CanBeNull); + } pSchema = pExpr->pTab->pSchema; } } /* if( pSrcList ) */ @@ -80029,9 +91595,8 @@ static int lookupName( break; } } - if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && HasRowid(pTab) ){ + if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ /* IMP: R-51414-32910 */ - /* IMP: R-44911-55124 */ iCol = -1; } if( iColnCol ){ @@ -80058,10 +91623,15 @@ static int lookupName( /* ** Perhaps the name is a reference to the ROWID */ - if( cnt==0 && cntTab==1 && pMatch && sqlite3IsRowid(zCol) - && HasRowid(pMatch->pTab) ){ + if( cnt==0 + && cntTab==1 + && pMatch + && (pNC->ncFlags & NC_IdxExpr)==0 + && sqlite3IsRowid(zCol) + && VisibleRowid(pMatch->pTab) + ){ cnt = 1; - pExpr->iColumn = -1; /* IMP: R-44911-55124 */ + pExpr->iColumn = -1; pExpr->affinity = SQLITE_AFF_INTEGER; } @@ -80078,9 +91648,9 @@ static int lookupName( ** resolved by the time the WHERE clause is resolved. ** ** The ability to use an output result-set column in the WHERE, GROUP BY, - ** or HAVING clauses, or as part of a larger expression in the ORDRE BY + ** or HAVING clauses, or as part of a larger expression in the ORDER BY ** clause is not standard SQL. This is a (goofy) SQLite extension, that - ** is supported for backwards compatibility only. TO DO: Issue a warning + ** is supported for backwards compatibility only. Hence, we issue a warning ** on sqlite3_log() whenever the capability is used. */ if( (pEList = pNC->pEList)!=0 @@ -80099,6 +91669,10 @@ static int lookupName( sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); return WRC_Abort; } + if( sqlite3ExprVectorSize(pOrig)!=1 ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return WRC_Abort; + } resolveAlias(pParse, pEList, j, pExpr, "", nSubquery); cnt = 1; pMatch = 0; @@ -80177,7 +91751,7 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - if( pExpr->op!=TK_AS ){ + if( !ExprHasProperty(pExpr, EP_Alias) ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } /* Increment the nRef value on all name contexts from TopNC up to @@ -80218,36 +91792,25 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSr } /* -** Report an error that an expression is not valid for a partial index WHERE -** clause. +** Report an error that an expression is not valid for some set of +** pNC->ncFlags values determined by validMask. */ -static void notValidPartIdxWhere( +static void notValid( Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ + const char *zMsg, /* Type of error */ + int validMask /* Set of contexts for which prohibited */ ){ - if( (pNC->ncFlags & NC_PartIdx)!=0 ){ - sqlite3ErrorMsg(pParse, "%s prohibited in partial index WHERE clauses", - zMsg); - } -} - + assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); + if( (pNC->ncFlags & validMask)!=0 ){ + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK -/* -** Report an error that an expression is not valid for a CHECK constraint. -*/ -static void notValidCheckConstraint( - Parse *pParse, /* Leave error message here */ - NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ -){ - if( (pNC->ncFlags & NC_IsCheck)!=0 ){ - sqlite3ErrorMsg(pParse,"%s prohibited in CHECK constraints", zMsg); + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); } } -#else -# define notValidCheckConstraint(P,N,M) -#endif /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -80260,7 +91823,7 @@ static int exprProbability(Expr *p){ sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); assert( r>=0.0 ); if( r>1.0 ) return -1; - return (int)(r*1000.0); + return (int)(r*134217728.0); } /* @@ -80313,7 +91876,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->affinity = SQLITE_AFF_INTEGER; break; } -#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) + && !defined(SQLITE_OMIT_SUBQUERY) */ /* A lone identifier is the name of a column. */ @@ -80331,6 +91895,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Expr *pRight; /* if( pSrcList==0 ) break; */ + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; @@ -80353,69 +91918,84 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ int no_such_func = 0; /* True if no such function exists */ int wrong_num_args = 0; /* True if wrong number of arguments */ int is_agg = 0; /* True if is an aggregate function */ - int auth; /* Authorization to use the function */ int nId; /* Number of characters in function name */ const char *zId; /* The function name. */ FuncDef *pDef; /* Information about the function */ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValidPartIdxWhere(pParse, pNC, "functions"); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); - pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); if( pDef==0 ){ - pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, -2, enc, 0); if( pDef==0 ){ no_such_func = 1; }else{ wrong_num_args = 1; } }else{ - is_agg = pDef->xFunc==0; + is_agg = pDef->xFinalize!=0; if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ ExprSetProperty(pExpr, EP_Unlikely|EP_Skip); if( n==2 ){ pExpr->iTable = exprProbability(pList->a[1].pExpr); if( pExpr->iTable<0 ){ - sqlite3ErrorMsg(pParse, "second argument to likelihood() must be a " - "constant between 0.0 and 1.0"); + sqlite3ErrorMsg(pParse, + "second argument to likelihood() must be a " + "constant between 0.0 and 1.0"); pNC->nErr++; } }else{ - /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is equivalent to - ** likelihood(X, 0.0625). - ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is short-hand for - ** likelihood(X,0.0625). - ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand for - ** likelihood(X,0.9375). - ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to - ** likelihood(X,0.9375). */ + /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is + ** equivalent to likelihood(X, 0.0625). + ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is + ** short-hand for likelihood(X,0.0625). + ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand + ** for likelihood(X,0.9375). + ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent + ** to likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ - pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; + pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; } } - } #ifndef SQLITE_OMIT_AUTHORIZATION - if( pDef ){ - auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); - if( auth!=SQLITE_OK ){ - if( auth==SQLITE_DENY ){ - sqlite3ErrorMsg(pParse, "not authorized to use function: %s", - pDef->zName); - pNC->nErr++; + { + int auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0,pDef->zName,0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %s", + pDef->zName); + pNC->nErr++; + } + pExpr->op = TK_NULL; + return WRC_Prune; } - pExpr->op = TK_NULL; - return WRC_Prune; } - if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ) ExprSetProperty(pExpr,EP_Constant); - } #endif + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ + /* For the purposes of the EP_ConstFunc flag, date and time + ** functions and other functions that change slowly are considered + ** constant because they are constant for the duration of one query */ + ExprSetProperty(pExpr,EP_ConstFunc); + } + if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ + /* Date/time functions that use 'now', and other functions like + ** sqlite_version() that might change over time cannot be used + ** in an index. */ + notValid(pParse, pNC, "non-deterministic functions", + NC_IdxExpr|NC_PartIdx); + } + } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; is_agg = 0; - }else if( no_such_func && pParse->db->init.busy==0 ){ + }else if( no_such_func && pParse->db->init.busy==0 +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + && pParse->explain==0 +#endif + ){ sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); pNC->nErr++; }else if( wrong_num_args ){ @@ -80433,7 +92013,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->op2++; pNC2 = pNC2->pNext; } - if( pNC2 ) pNC2->ncFlags |= NC_HasAgg; + assert( pDef!=0 ); + if( pNC2 ){ + assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); + testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); + pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); + + } pNC->ncFlags |= NC_AllowAgg; } /* FIX ME: Compute pExpr->affinity based on the expected return @@ -80449,21 +92035,56 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValidCheckConstraint(pParse, pNC, "subqueries"); - notValidPartIdxWhere(pParse, pNC, "subqueries"); + notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); + pNC->ncFlags |= NC_VarSelect; } } break; } case TK_VARIABLE: { - notValidCheckConstraint(pParse, pNC, "parameters"); - notValidPartIdxWhere(pParse, pNC, "parameters"); + notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } + case TK_BETWEEN: + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_IS: + case TK_ISNOT: { + int nLeft, nRight; + if( pParse->db->mallocFailed ) break; + assert( pExpr->pLeft!=0 ); + nLeft = sqlite3ExprVectorSize(pExpr->pLeft); + if( pExpr->op==TK_BETWEEN ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[0].pExpr); + if( nRight==nLeft ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[1].pExpr); + } + }else{ + assert( pExpr->pRight!=0 ); + nRight = sqlite3ExprVectorSize(pExpr->pRight); + } + if( nLeft!=nRight ){ + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + testcase( pExpr->op==TK_BETWEEN ); + sqlite3ErrorMsg(pParse, "row value misused"); + } + break; + } } return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; } @@ -80655,9 +92276,11 @@ static int resolveCompoundOrderBy( if( pItem->pExpr==pE ){ pItem->pExpr = pNew; }else{ - assert( pItem->pExpr->op==TK_COLLATE ); - assert( pItem->pExpr->pLeft==pE ); - pItem->pExpr->pLeft = pNew; + Expr *pParent = pItem->pExpr; + assert( pParent->op==TK_COLLATE ); + while( pParent->pLeft->op==TK_COLLATE ) pParent = pParent->pLeft; + assert( pParent->pLeft==pE ); + pParent->pLeft = pNew; } sqlite3ExprDelete(db, pE); pItem->u.x.iOrderByCol = (u16)iCol; @@ -80714,7 +92337,8 @@ SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); return 1; } - resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, zType,0); + resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, + zType,0); } } return 0; @@ -80794,7 +92418,7 @@ static int resolveOrderGroupBy( } /* -** Resolve names in the SELECT statement p and all of its descendents. +** Resolve names in the SELECT statement p and all of its descendants. */ static int resolveSelectStep(Walker *pWalker, Select *p){ NameContext *pOuterNC; /* Context that contains this SELECT */ @@ -80802,7 +92426,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ int isCompound; /* True if p is a compound select */ int nCompound; /* Number of compound terms processed so far */ Parse *pParse; /* Parsing context */ - ExprList *pEList; /* Result set expression list */ int i; /* Loop counter */ ExprList *pGroupBy; /* The GROUP BY clause */ Select *pLeftmost; /* Left-most of SELECT of a compound */ @@ -80847,6 +92470,20 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sqlite3ResolveExprNames(&sNC, p->pOffset) ){ return WRC_Abort; } + + /* If the SF_Converted flags is set, then this Select object was + ** was created by the convertCompoundSelectToSubquery() function. + ** In this case the ORDER BY clause (p->pOrderBy) should be resolved + ** as if it were part of the sub-query, not the parent. This block + ** moves the pOrderBy down to the sub-query. It will be moved back + ** after the names have been resolved. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + assert( p->pSrc->nSrc==1 && p->pOrderBy ); + assert( pSub->pPrior && pSub->pOrderBy==0 ); + pSub->pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + } /* Recursively resolve names in all subqueries */ @@ -80861,7 +92498,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** parent contexts. After resolving references to expressions in ** pItem->pSelect, check if this value has changed. If so, then ** SELECT statement pItem->pSelect must be correlated. Set the - ** pItem->isCorrelated flag if this is the case. */ + ** pItem->fg.isCorrelated flag if this is the case. */ for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; if( pItem->zName ) pParse->zAuthContext = pItem->zName; @@ -80870,8 +92507,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pParse->nErr || db->mallocFailed ) return WRC_Abort; for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; - assert( pItem->isCorrelated==0 && nRef<=0 ); - pItem->isCorrelated = (nRef!=0); + assert( pItem->fg.isCorrelated==0 && nRef<=0 ); + pItem->fg.isCorrelated = (nRef!=0); } } @@ -80883,14 +92520,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sNC.pNext = pOuterNC; /* Resolve names in the result set. */ - pEList = p->pEList; - assert( pEList!=0 ); - for(i=0; inExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ResolveExprNames(&sNC, pX) ){ - return WRC_Abort; - } - } + if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. @@ -80898,7 +92528,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ assert( (p->selFlags & SF_Aggregate)==0 ); pGroupBy = p->pGroupBy; if( pGroupBy || (sNC.ncFlags & NC_HasAgg)!=0 ){ - p->selFlags |= SF_Aggregate; + assert( NC_MinMaxAgg==SF_MinMaxAgg ); + p->selFlags |= SF_Aggregate | (sNC.ncFlags&NC_MinMaxAgg); }else{ sNC.ncFlags &= ~NC_AllowAgg; } @@ -80922,18 +92553,46 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + /* Resolve names in table-valued-function arguments */ + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + if( pItem->fg.isTabFunc + && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } + } + /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries */ sNC.pNext = 0; sNC.ncFlags |= NC_AllowAgg; + /* If this is a converted compound query, move the ORDER BY clause from + ** the sub-query back to the parent query. At this point each term + ** within the ORDER BY clause has been transformed to an integer value. + ** These integers will be replaced by copies of the corresponding result + ** set expressions by the call to resolveOrderGroupBy() below. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + p->pOrderBy = pSub->pOrderBy; + pSub->pOrderBy = 0; + } + /* Process the ORDER BY clause for singleton SELECT statements. ** The ORDER BY clause for compounds SELECT statements is handled ** below, after all of the result-sets for all of the elements of ** the compound have been resolved. + ** + ** If there is an ORDER BY clause on a term of a compound-select other + ** than the right-most term, then that is a syntax error. But the error + ** is not detected until much later, and so we need to go ahead and + ** resolve those symbols on the incorrect ORDER BY for consistency. */ - if( !isCompound && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") ){ + if( isCompound<=nCompound /* Defer right-most ORDER BY of a compound */ + && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") + ){ return WRC_Abort; } if( db->mallocFailed ){ @@ -80958,6 +92617,13 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } + /* If this is part of a compound SELECT, check that it has the right + ** number of expressions in the select list. */ + if( p->pNext && p->pEList->nExpr!=p->pNext->pEList->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, p->pNext); + return WRC_Abort; + } + /* Advance to the next term of the compound */ p = p->pPrior; @@ -81026,7 +92692,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( NameContext *pNC, /* Namespace to resolve expressions in. */ Expr *pExpr /* The expression to be analyzed. */ ){ - u8 savedHasAgg; + u16 savedHasAgg; Walker w; if( pExpr==0 ) return 0; @@ -81039,12 +92705,14 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( pParse->nHeight += pExpr->nHeight; } #endif - savedHasAgg = pNC->ncFlags & NC_HasAgg; - pNC->ncFlags &= ~NC_HasAgg; - memset(&w, 0, sizeof(w)); + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg); + w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; - w.pParse = pNC->pParse; + w.xSelectCallback2 = 0; + w.walkerDepth = 0; + w.eCode = 0; w.u.pNC = pNC; sqlite3WalkExpr(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 @@ -81055,12 +92723,28 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( } if( pNC->ncFlags & NC_HasAgg ){ ExprSetProperty(pExpr, EP_Agg); - }else if( savedHasAgg ){ - pNC->ncFlags |= NC_HasAgg; } + pNC->ncFlags |= savedHasAgg; return ExprHasProperty(pExpr, EP_Error); } +/* +** Resolve all names for all expression in an expression list. This is +** just like sqlite3ResolveExprNames() except that it works for an expression +** list rather than a single expression. +*/ +SQLITE_PRIVATE int sqlite3ResolveExprListNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + ExprList *pList /* The expression list to be analyzed. */ +){ + int i; + if( pList ){ + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort; + } + } + return WRC_Continue; +} /* ** Resolve all names in all expressions of a SELECT and in all @@ -81104,15 +92788,14 @@ SQLITE_PRIVATE void sqlite3ResolveSelectNames( SQLITE_PRIVATE void sqlite3ResolveSelfReference( Parse *pParse, /* Parsing context */ Table *pTab, /* The table being referenced */ - int type, /* NC_IsCheck or NC_PartIdx */ + int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */ Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NUL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ - int i; /* Loop counter */ - assert( type==NC_IsCheck || type==NC_PartIdx ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); sSrc.nSrc = 1; @@ -81123,13 +92806,7 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( sNC.pSrcList = &sSrc; sNC.ncFlags = type; if( sqlite3ResolveExprNames(&sNC, pExpr) ) return; - if( pList ){ - for(i=0; inExpr; i++){ - if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ - return; - } - } - } + if( pList ) sqlite3ResolveExprListNames(&sNC, pList); } /************** End of resolve.c *********************************************/ @@ -81148,6 +92825,19 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( ** This file contains routines used for analyzing expressions and ** for generating VDBE code that evaluates expressions in SQLite. */ +/* #include "sqliteInt.h" */ + +/* Forward declarations */ +static void exprCodeBetween(Parse*,Expr*,int,void(*)(Parse*,Expr*,int,int),int); +static int exprCodeVector(Parse *pParse, Expr *p, int *piToFree); + +/* +** Return the affinity character for a single column of a table. +*/ +SQLITE_PRIVATE char sqlite3TableColumnAffinity(Table *pTab, int iCol){ + assert( iColnCol ); + return iCol>=0 ? pTab->aCol[iCol].affinity : SQLITE_AFF_INTEGER; +} /* ** Return the 'affinity' of the expression pExpr if any. @@ -81157,7 +92847,7 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( ** affinity of that column is returned. Otherwise, 0x00 is returned, ** indicating no affinity for the expression. ** -** i.e. the WHERE clause expresssions in the following statements all +** i.e. the WHERE clause expressions in the following statements all ** have an affinity: ** ** CREATE TABLE t1(a); @@ -81174,21 +92864,21 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ assert( pExpr->flags&EP_xIsSelect ); return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); } + if( op==TK_REGISTER ) op = pExpr->op2; #ifndef SQLITE_OMIT_CAST if( op==TK_CAST ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); return sqlite3AffinityType(pExpr->u.zToken, 0); } #endif - if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) - && pExpr->pTab!=0 - ){ - /* op==TK_REGISTER && pExpr->pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = pExpr->iColumn; - if( j<0 ) return SQLITE_AFF_INTEGER; - assert( pExpr->pTab && jpTab->nCol ); - return pExpr->pTab->aCol[j].affinity; + if( op==TK_AGG_COLUMN || op==TK_COLUMN ){ + return sqlite3TableColumnAffinity(pExpr->pTab, pExpr->iColumn); + } + if( op==TK_SELECT_COLUMN ){ + assert( pExpr->pLeft->flags&EP_xIsSelect ); + return sqlite3ExprAffinity( + pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr + ); } return pExpr->affinity; } @@ -81204,10 +92894,11 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( Parse *pParse, /* Parsing context */ Expr *pExpr, /* Add the "COLLATE" clause to this expression */ - const Token *pCollName /* Name of collating sequence */ + const Token *pCollName, /* Name of collating sequence */ + int dequote /* True to dequote pCollName */ ){ if( pCollName->n>0 ){ - Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, 1); + Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, dequote); if( pNew ){ pNew->pLeft = pExpr; pNew->flags |= EP_Collate|EP_Skip; @@ -81219,13 +92910,12 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ Token s; assert( zC!=0 ); - s.z = zC; - s.n = sqlite3Strlen30(s.z); - return sqlite3ExprAddCollateToken(pParse, pExpr, &s); + sqlite3TokenInit(&s, (char*)zC); + return sqlite3ExprAddCollateToken(pParse, pExpr, &s, 0); } /* -** Skip over any TK_COLLATE or TK_AS operators and any unlikely() +** Skip over any TK_COLLATE operators and any unlikely() ** or likelihood() function at the root of an expression. */ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ @@ -81236,7 +92926,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ); + assert( pExpr->op==TK_COLLATE ); pExpr = pExpr->pLeft; } } @@ -81267,9 +92957,9 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); break; } - if( p->pTab!=0 - && (op==TK_AGG_COLUMN || op==TK_COLUMN + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER || op==TK_TRIGGER) + && p->pTab!=0 ){ /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally ** a TK_COLUMN but was previously evaluated and cached in a register */ @@ -81281,10 +92971,25 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ break; } if( p->flags & EP_Collate ){ - if( ALWAYS(p->pLeft) && (p->pLeft->flags & EP_Collate)!=0 ){ + if( p->pLeft && (p->pLeft->flags & EP_Collate)!=0 ){ p = p->pLeft; }else{ - p = p->pRight; + Expr *pNext = p->pRight; + /* The Expr.x union is never used at the same time as Expr.pRight */ + assert( p->x.pList==0 || p->pRight==0 ); + /* p->flags holds EP_Collate and p->pLeft->flags does not. And + ** p->x.pSelect cannot. So if p->x.pLeft exists, it must hold at + ** least one EP_Collate. Thus the following two ALWAYS. */ + if( p->x.pList!=0 && ALWAYS(!ExprHasProperty(p, EP_xIsSelect)) ){ + int i; + for(i=0; ALWAYS(ix.pList->nExpr); i++){ + if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ + pNext = p->x.pList->a[i].pExpr; + break; + } + } + } + p = pNext; } }else{ break; @@ -81310,13 +93015,13 @@ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ if( sqlite3IsNumericAffinity(aff1) || sqlite3IsNumericAffinity(aff2) ){ return SQLITE_AFF_NUMERIC; }else{ - return SQLITE_AFF_NONE; + return SQLITE_AFF_BLOB; } }else if( !aff1 && !aff2 ){ /* Neither side of the comparison is a column. Compare the ** results directly. */ - return SQLITE_AFF_NONE; + return SQLITE_AFF_BLOB; }else{ /* One side is a column, the other is not. Use the columns affinity. */ assert( aff1==0 || aff2==0 ); @@ -81339,8 +93044,8 @@ static char comparisonAffinity(Expr *pExpr){ aff = sqlite3CompareAffinity(pExpr->pRight, aff); }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); - }else if( !aff ){ - aff = SQLITE_AFF_NONE; + }else if( NEVER(aff==0) ){ + aff = SQLITE_AFF_BLOB; } return aff; } @@ -81354,7 +93059,7 @@ static char comparisonAffinity(Expr *pExpr){ SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ char aff = comparisonAffinity(pExpr); switch( aff ){ - case SQLITE_AFF_NONE: + case SQLITE_AFF_BLOB: return 1; case SQLITE_AFF_TEXT: return idx_affinity==SQLITE_AFF_TEXT; @@ -81429,6 +93134,270 @@ static int codeCompare( return addr; } +/* +** Return true if expression pExpr is a vector, or false otherwise. +** +** A vector is defined as any expression that results in two or more +** columns of result. Every TK_VECTOR node is an vector because the +** parser will not generate a TK_VECTOR with fewer than two entries. +** But a TK_SELECT might be either a vector or a scalar. It is only +** considered a vector if it has two or more result columns. +*/ +SQLITE_PRIVATE int sqlite3ExprIsVector(Expr *pExpr){ + return sqlite3ExprVectorSize(pExpr)>1; +} + +/* +** If the expression passed as the only argument is of type TK_VECTOR +** return the number of expressions in the vector. Or, if the expression +** is a sub-select, return the number of columns in the sub-select. For +** any other type of expression, return 1. +*/ +SQLITE_PRIVATE int sqlite3ExprVectorSize(Expr *pExpr){ + u8 op = pExpr->op; + if( op==TK_REGISTER ) op = pExpr->op2; + if( op==TK_VECTOR ){ + return pExpr->x.pList->nExpr; + }else if( op==TK_SELECT ){ + return pExpr->x.pSelect->pEList->nExpr; + }else{ + return 1; + } +} + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Return a pointer to a subexpression of pVector that is the i-th +** column of the vector (numbered starting with 0). The caller must +** ensure that i is within range. +** +** If pVector is really a scalar (and "scalar" here includes subqueries +** that return a single column!) then return pVector unmodified. +** +** pVector retains ownership of the returned subexpression. +** +** If the vector is a (SELECT ...) then the expression returned is +** just the expression for the i-th term of the result set, and may +** not be ready for evaluation because the table cursor has not yet +** been positioned. +*/ +SQLITE_PRIVATE Expr *sqlite3VectorFieldSubexpr(Expr *pVector, int i){ + assert( iop2==0 || pVector->op==TK_REGISTER ); + if( pVector->op==TK_SELECT || pVector->op2==TK_SELECT ){ + return pVector->x.pSelect->pEList->a[i].pExpr; + }else{ + return pVector->x.pList->a[i].pExpr; + } + } + return pVector; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Compute and return a new Expr object which when passed to +** sqlite3ExprCode() will generate all necessary code to compute +** the iField-th column of the vector expression pVector. +** +** It is ok for pVector to be a scalar (as long as iField==0). +** In that case, this routine works like sqlite3ExprDup(). +** +** The caller owns the returned Expr object and is responsible for +** ensuring that the returned value eventually gets freed. +** +** The caller retains ownership of pVector. If pVector is a TK_SELECT, +** then the returned object will reference pVector and so pVector must remain +** valid for the life of the returned object. If pVector is a TK_VECTOR +** or a scalar expression, then it can be deleted as soon as this routine +** returns. +** +** A trick to cause a TK_SELECT pVector to be deleted together with +** the returned Expr object is to attach the pVector to the pRight field +** of the returned TK_SELECT_COLUMN Expr object. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprForVectorField( + Parse *pParse, /* Parsing context */ + Expr *pVector, /* The vector. List of expressions or a sub-SELECT */ + int iField /* Which column of the vector to return */ +){ + Expr *pRet; + if( pVector->op==TK_SELECT ){ + assert( pVector->flags & EP_xIsSelect ); + /* The TK_SELECT_COLUMN Expr node: + ** + ** pLeft: pVector containing TK_SELECT + ** pRight: not used. But recursively deleted. + ** iColumn: Index of a column in pVector + ** pLeft->iTable: First in an array of register holding result, or 0 + ** if the result is not yet computed. + ** + ** sqlite3ExprDelete() specifically skips the recursive delete of + ** pLeft on TK_SELECT_COLUMN nodes. But pRight is followed, so pVector + ** can be attached to pRight to cause this node to take ownership of + ** pVector. Typically there will be multiple TK_SELECT_COLUMN nodes + ** with the same pLeft pointer to the pVector, but only one of them + ** will own the pVector. + */ + pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0, 0); + if( pRet ){ + pRet->iColumn = iField; + pRet->pLeft = pVector; + } + assert( pRet==0 || pRet->iTable==0 ); + }else{ + if( pVector->op==TK_VECTOR ) pVector = pVector->x.pList->a[iField].pExpr; + pRet = sqlite3ExprDup(pParse->db, pVector, 0); + } + return pRet; +} +#endif /* !define(SQLITE_OMIT_SUBQUERY) */ + +/* +** If expression pExpr is of type TK_SELECT, generate code to evaluate +** it. Return the register in which the result is stored (or, if the +** sub-select returns more than one column, the first in an array +** of registers in which the result is stored). +** +** If pExpr is not a TK_SELECT expression, return 0. +*/ +static int exprCodeSubselect(Parse *pParse, Expr *pExpr){ + int reg = 0; +#ifndef SQLITE_OMIT_SUBQUERY + if( pExpr->op==TK_SELECT ){ + reg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); + } +#endif + return reg; +} + +/* +** Argument pVector points to a vector expression - either a TK_VECTOR +** or TK_SELECT that returns more than one column. This function returns +** the register number of a register that contains the value of +** element iField of the vector. +** +** If pVector is a TK_SELECT expression, then code for it must have +** already been generated using the exprCodeSubselect() routine. In this +** case parameter regSelect should be the first in an array of registers +** containing the results of the sub-select. +** +** If pVector is of type TK_VECTOR, then code for the requested field +** is generated. In this case (*pRegFree) may be set to the number of +** a temporary register to be freed by the caller before returning. +** +** Before returning, output parameter (*ppExpr) is set to point to the +** Expr object corresponding to element iElem of the vector. +*/ +static int exprVectorRegister( + Parse *pParse, /* Parse context */ + Expr *pVector, /* Vector to extract element from */ + int iField, /* Field to extract from pVector */ + int regSelect, /* First in array of registers */ + Expr **ppExpr, /* OUT: Expression element */ + int *pRegFree /* OUT: Temp register to free */ +){ + u8 op = pVector->op; + assert( op==TK_VECTOR || op==TK_REGISTER || op==TK_SELECT ); + if( op==TK_REGISTER ){ + *ppExpr = sqlite3VectorFieldSubexpr(pVector, iField); + return pVector->iTable+iField; + } + if( op==TK_SELECT ){ + *ppExpr = pVector->x.pSelect->pEList->a[iField].pExpr; + return regSelect+iField; + } + *ppExpr = pVector->x.pList->a[iField].pExpr; + return sqlite3ExprCodeTemp(pParse, *ppExpr, pRegFree); +} + +/* +** Expression pExpr is a comparison between two vector values. Compute +** the result of the comparison (1, 0, or NULL) and write that +** result into register dest. +** +** The caller must satisfy the following preconditions: +** +** if pExpr->op==TK_IS: op==TK_EQ and p5==SQLITE_NULLEQ +** if pExpr->op==TK_ISNOT: op==TK_NE and p5==SQLITE_NULLEQ +** otherwise: op==pExpr->op and p5==0 +*/ +static void codeVectorCompare( + Parse *pParse, /* Code generator context */ + Expr *pExpr, /* The comparison operation */ + int dest, /* Write results into this register */ + u8 op, /* Comparison operator */ + u8 p5 /* SQLITE_NULLEQ or zero */ +){ + Vdbe *v = pParse->pVdbe; + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + int nLeft = sqlite3ExprVectorSize(pLeft); + int i; + int regLeft = 0; + int regRight = 0; + u8 opx = op; + int addrDone = sqlite3VdbeMakeLabel(v); + + assert( nLeft==sqlite3ExprVectorSize(pRight) ); + assert( pExpr->op==TK_EQ || pExpr->op==TK_NE + || pExpr->op==TK_IS || pExpr->op==TK_ISNOT + || pExpr->op==TK_LT || pExpr->op==TK_GT + || pExpr->op==TK_LE || pExpr->op==TK_GE + ); + assert( pExpr->op==op || (pExpr->op==TK_IS && op==TK_EQ) + || (pExpr->op==TK_ISNOT && op==TK_NE) ); + assert( p5==0 || pExpr->op!=op ); + assert( p5==SQLITE_NULLEQ || pExpr->op==op ); + + p5 |= SQLITE_STOREP2; + if( opx==TK_LE ) opx = TK_LT; + if( opx==TK_GE ) opx = TK_GT; + + regLeft = exprCodeSubselect(pParse, pLeft); + regRight = exprCodeSubselect(pParse, pRight); + + for(i=0; 1 /*Loop exits by "break"*/; i++){ + int regFree1 = 0, regFree2 = 0; + Expr *pL, *pR; + int r1, r2; + assert( i>=0 && i0 ) sqlite3ExprCachePush(pParse); + r1 = exprVectorRegister(pParse, pLeft, i, regLeft, &pL, ®Free1); + r2 = exprVectorRegister(pParse, pRight, i, regRight, &pR, ®Free2); + codeCompare(pParse, pL, pR, opx, r1, r2, dest, p5); + testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); + if( i>0 ) sqlite3ExprCachePop(pParse); + if( i==nLeft-1 ){ + break; + } + if( opx==TK_EQ ){ + sqlite3VdbeAddOp2(v, OP_IfNot, dest, addrDone); VdbeCoverage(v); + p5 |= SQLITE_KEEPNULL; + }else if( opx==TK_NE ){ + sqlite3VdbeAddOp2(v, OP_If, dest, addrDone); VdbeCoverage(v); + p5 |= SQLITE_KEEPNULL; + }else{ + assert( op==TK_LT || op==TK_GT || op==TK_LE || op==TK_GE ); + sqlite3VdbeAddOp2(v, OP_ElseNotEq, 0, addrDone); + VdbeCoverageIf(v, op==TK_LT); + VdbeCoverageIf(v, op==TK_GT); + VdbeCoverageIf(v, op==TK_LE); + VdbeCoverageIf(v, op==TK_GE); + if( i==nLeft-2 ) opx = op; + } + } + sqlite3VdbeResolveLabel(v, addrDone); +} + #if SQLITE_MAX_EXPR_DEPTH>0 /* ** Check that argument nHeight is less than or equal to the maximum @@ -81490,6 +93459,9 @@ static void heightOfSelect(Select *p, int *pnHeight){ ** Expr.pSelect member has a height of 1. Any other expression ** has a height equal to the maximum height of any other ** referenced Expr plus one. +** +** Also propagate EP_Propagate flags up from Expr.x.pList to Expr.flags, +** if appropriate. */ static void exprSetHeight(Expr *p){ int nHeight = 0; @@ -81497,8 +93469,9 @@ static void exprSetHeight(Expr *p){ heightOfExpr(p->pRight, &nHeight); if( ExprHasProperty(p, EP_xIsSelect) ){ heightOfSelect(p->x.pSelect, &nHeight); - }else{ + }else if( p->x.pList ){ heightOfExprList(p->x.pList, &nHeight); + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); } p->nHeight = nHeight + 1; } @@ -81507,8 +93480,12 @@ static void exprSetHeight(Expr *p){ ** Set the Expr.nHeight variable using the exprSetHeight() function. If ** the height is greater than the maximum allowed expression depth, ** leave an error in pParse. +** +** Also propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. */ -SQLITE_PRIVATE void sqlite3ExprSetHeight(Parse *pParse, Expr *p){ +SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ + if( pParse->nErr ) return; exprSetHeight(p); sqlite3ExprCheckHeight(pParse, p->nHeight); } @@ -81522,8 +93499,17 @@ SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ heightOfSelect(p, &nHeight); return nHeight; } -#else - #define exprSetHeight(y) +#else /* ABOVE: Height enforcement enabled. BELOW: Height enforcement off */ +/* +** Propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. +*/ +SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ + if( p && p->x.pList && !ExprHasProperty(p, EP_xIsSelect) ){ + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); + } +} +#define exprSetHeight(y) #endif /* SQLITE_MAX_EXPR_DEPTH>0 */ /* @@ -81535,7 +93521,7 @@ SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ ** is responsible for making sure the node eventually gets freed. ** ** If dequote is true, then the token (if it exists) is dequoted. -** If dequote is false, no dequoting is performance. The deQuote +** If dequote is false, no dequoting is performed. The deQuote ** parameter is ignored if pToken is NULL or if the token does not ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. @@ -81547,7 +93533,7 @@ SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ ** is allocated to hold the integer text and the dequote flag is ignored. */ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( - sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ int op, /* Expression opcode */ const Token *pToken, /* Token argument. Might be NULL */ int dequote /* True to dequote */ @@ -81556,6 +93542,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( int nExtra = 0; int iValue = 0; + assert( db!=0 ); if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ @@ -81563,8 +93550,9 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( assert( iValue>=0 ); } } - pNew = sqlite3DbMallocZero(db, sizeof(Expr)+nExtra); + pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); if( pNew ){ + memset(pNew, 0, sizeof(Expr)); pNew->op = (u8)op; pNew->iAgg = -1; if( pToken ){ @@ -81572,15 +93560,13 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( pNew->flags |= EP_IntValue; pNew->u.iValue = iValue; }else{ - int c; pNew->u.zToken = (char*)&pNew[1]; assert( pToken->z!=0 || pToken->n==0 ); if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; - if( dequote && nExtra>=3 - && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ + if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ + if( pNew->u.zToken[0]=='"' ) pNew->flags |= EP_DblQuoted; sqlite3Dequote(pNew->u.zToken); - if( c=='"' ) pNew->flags |= EP_DblQuoted; } } } @@ -81625,18 +93611,18 @@ SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( }else{ if( pRight ){ pRoot->pRight = pRight; - pRoot->flags |= EP_Collate & pRight->flags; + pRoot->flags |= EP_Propagate & pRight->flags; } if( pLeft ){ pRoot->pLeft = pLeft; - pRoot->flags |= EP_Collate & pLeft->flags; + pRoot->flags |= EP_Propagate & pLeft->flags; } exprSetHeight(pRoot); } } /* -** Allocate a Expr node which joins as many as two subtrees. +** Allocate an Expr node which joins as many as two subtrees. ** ** One or both of the subtrees can be NULL. Return a pointer to the new ** Expr node. Or, if an OOM error occurs, set pParse->db->mallocFailed, @@ -81650,11 +93636,11 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( const Token *pToken /* Argument token */ ){ Expr *p; - if( op==TK_AND && pLeft && pRight ){ + if( op==TK_AND && pParse->nErr==0 ){ /* Take advantage of short-circuit false optimization for AND */ p = sqlite3ExprAnd(pParse->db, pLeft, pRight); }else{ - p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); + p = sqlite3ExprAlloc(pParse->db, op & TKFLG_MASK, pToken, 1); sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); } if( p ) { @@ -81663,6 +93649,22 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( return p; } +/* +** Add pSelect to the Expr.x.pSelect field. Or, if pExpr is NULL (due +** do a memory allocation failure) then delete the pSelect object. +*/ +SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pSelect){ + if( pExpr ){ + pExpr->x.pSelect = pSelect; + ExprSetProperty(pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, pExpr); + }else{ + assert( pParse->db->mallocFailed ); + sqlite3SelectDelete(pParse->db, pSelect); + } +} + + /* ** If the expression is always either TRUE or FALSE (respectively), ** then return 1. If one cannot determine the truth value of the @@ -81729,7 +93731,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token * } pNew->x.pList = pList; assert( !ExprHasProperty(pNew, EP_xIsSelect) ); - sqlite3ExprSetHeight(pParse, pNew); + sqlite3ExprSetHeightAndFlags(pParse, pNew); return pNew; } @@ -81746,10 +93748,10 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token * ** ** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number ** as the previous instance of the same wildcard. Or if this is the first -** instance of the wildcard, the next sequenial variable number is +** instance of the wildcard, the next sequential variable number is ** assigned. */ -SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ sqlite3 *db = pParse->db; const char *z; @@ -81758,19 +93760,19 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ z = pExpr->u.zToken; assert( z!=0 ); assert( z[0]!=0 ); + assert( n==sqlite3Strlen30(z) ); if( z[1]==0 ){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); pExpr->iColumn = (ynVar)(++pParse->nVar); }else{ - ynVar x = 0; - u32 n = sqlite3Strlen30(z); + ynVar x; if( z[0]=='?' ){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ i64 i; int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); - pExpr->iColumn = x = (ynVar)i; + x = (ynVar)i; testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); @@ -81778,7 +93780,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - x = 0; + return; } if( i>pParse->nVar ){ pParse->nVar = (int)i; @@ -81789,30 +93791,31 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ ** has never appeared before, reuse the same variable number */ ynVar i; - for(i=0; inzVar; i++){ + for(i=x=0; inzVar; i++){ if( pParse->azVar[i] && strcmp(pParse->azVar[i],z)==0 ){ - pExpr->iColumn = x = (ynVar)i+1; + x = (ynVar)i+1; break; } } - if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); + if( x==0 ) x = (ynVar)(++pParse->nVar); } - if( x>0 ){ - if( x>pParse->nzVar ){ - char **a; - a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); - if( a==0 ) return; /* Error reported through db->mallocFailed */ - pParse->azVar = a; - memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); - pParse->nzVar = x; - } - if( z[0]!='?' || pParse->azVar[x-1]==0 ){ - sqlite3DbFree(db, pParse->azVar[x-1]); - pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); + pExpr->iColumn = x; + if( x>pParse->nzVar ){ + char **a; + a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); + if( a==0 ){ + assert( db->mallocFailed ); /* Error reported through mallocFailed */ + return; } + pParse->azVar = a; + memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); + pParse->nzVar = x; + } + if( pParse->azVar[x-1]==0 ){ + pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); } } - if( !pParse->nErr && pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + if( pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "too many SQL variables"); } } @@ -81820,26 +93823,36 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ /* ** Recursively delete an expression tree. */ -SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ - if( p==0 ) return; +static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ + assert( p!=0 ); /* Sanity check: Assert that the IntValue is non-negative if it exists */ assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); - if( !ExprHasProperty(p, EP_TokenOnly) ){ +#ifdef SQLITE_DEBUG + if( ExprHasProperty(p, EP_Leaf) && !ExprHasProperty(p, EP_TokenOnly) ){ + assert( p->pLeft==0 ); + assert( p->pRight==0 ); + assert( p->x.pSelect==0 ); + } +#endif + if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( p->x.pList==0 || p->pRight==0 ); - sqlite3ExprDelete(db, p->pLeft); + if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); sqlite3ExprDelete(db, p->pRight); - if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); if( ExprHasProperty(p, EP_xIsSelect) ){ sqlite3SelectDelete(db, p->x.pSelect); }else{ sqlite3ExprListDelete(db, p->x.pList); } } + if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbFree(db, p); } } +SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p ) sqlite3ExprDeleteNN(db, p); +} /* ** Return the number of bytes allocated for the expression structure @@ -81881,7 +93894,7 @@ static int exprStructSize(Expr *p){ ** During expression analysis, extra information is computed and moved into ** later parts of teh Expr object and that extra information might get chopped ** off if the expression is reduced. Note also that it does not work to -** make a EXPRDUP_REDUCE copy of a reduced expression. It is only legal +** make an EXPRDUP_REDUCE copy of a reduced expression. It is only legal ** to reduce a pristine expression tree from the parser. The implementation ** of dupedExprStructSize() contain multiple assert() statements that attempt ** to enforce this constraint. @@ -81891,7 +93904,7 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==(flags&EXPRDUP_REDUCE) ){ + if( 0==flags ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); @@ -81950,87 +93963,95 @@ static int dupedExprSize(Expr *p, int flags){ ** is not NULL then *pzBuffer is assumed to point to a buffer large enough ** to store the copy of expression p, the copies of p->u.zToken ** (if applicable), and the copies of the p->pLeft and p->pRight expressions, -** if any. Before returning, *pzBuffer is set to the first byte passed the +** if any. Before returning, *pzBuffer is set to the first byte past the ** portion of the buffer copied into by this function. */ -static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ - Expr *pNew = 0; /* Value to return */ - if( p ){ - const int isReduced = (flags&EXPRDUP_REDUCE); - u8 *zAlloc; - u32 staticFlag = 0; +static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ + Expr *pNew; /* Value to return */ + u8 *zAlloc; /* Memory space from which to build Expr object */ + u32 staticFlag; /* EP_Static if space not obtained from malloc */ - assert( pzBuffer==0 || isReduced ); + assert( db!=0 ); + assert( p ); + assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); + assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); - /* Figure out where to write the new Expr structure. */ - if( pzBuffer ){ - zAlloc = *pzBuffer; - staticFlag = EP_Static; + /* Figure out where to write the new Expr structure. */ + if( pzBuffer ){ + zAlloc = *pzBuffer; + staticFlag = EP_Static; + }else{ + zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); + staticFlag = 0; + } + pNew = (Expr *)zAlloc; + + if( pNew ){ + /* Set nNewSize to the size allocated for the structure pointed to + ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or + ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed + ** by the copy of the p->u.zToken string (if any). + */ + const unsigned nStructSize = dupedExprStructSize(p, dupFlags); + const int nNewSize = nStructSize & 0xfff; + int nToken; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; }else{ - zAlloc = sqlite3DbMallocRaw(db, dupedExprSize(p, flags)); + nToken = 0; } - pNew = (Expr *)zAlloc; - - if( pNew ){ - /* Set nNewSize to the size allocated for the structure pointed to - ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or - ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed - ** by the copy of the p->u.zToken string (if any). - */ - const unsigned nStructSize = dupedExprStructSize(p, flags); - const int nNewSize = nStructSize & 0xfff; - int nToken; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nToken = sqlite3Strlen30(p->u.zToken) + 1; - }else{ - nToken = 0; - } - if( isReduced ){ - assert( ExprHasProperty(p, EP_Reduced)==0 ); - memcpy(zAlloc, p, nNewSize); - }else{ - int nSize = exprStructSize(p); - memcpy(zAlloc, p, nSize); + if( dupFlags ){ + assert( ExprHasProperty(p, EP_Reduced)==0 ); + memcpy(zAlloc, p, nNewSize); + }else{ + u32 nSize = (u32)exprStructSize(p); + memcpy(zAlloc, p, nSize); + if( nSizeflags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); - pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); - pNew->flags |= staticFlag; + /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); + pNew->flags |= staticFlag; - /* Copy the p->u.zToken string, if any. */ - if( nToken ){ - char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; - memcpy(zToken, p->u.zToken, nToken); - } + /* Copy the p->u.zToken string, if any. */ + if( nToken ){ + char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + memcpy(zToken, p->u.zToken, nToken); + } - if( 0==((p->flags|pNew->flags) & EP_TokenOnly) ){ - /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ - if( ExprHasProperty(p, EP_xIsSelect) ){ - pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, isReduced); - }else{ - pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, isReduced); - } - } - - /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ - zAlloc += dupedExprNodeSize(p, flags); - if( ExprHasProperty(pNew, EP_Reduced) ){ - pNew->pLeft = exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc); - pNew->pRight = exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc); - } - if( pzBuffer ){ - *pzBuffer = zAlloc; - } + if( 0==((p->flags|pNew->flags) & (EP_TokenOnly|EP_Leaf)) ){ + /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ + if( ExprHasProperty(p, EP_xIsSelect) ){ + pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); }else{ - if( !ExprHasProperty(p, EP_TokenOnly) ){ - pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); - pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); - } + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); } + } + /* Fill in pNew->pLeft and pNew->pRight. */ + if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ + zAlloc += dupedExprNodeSize(p, dupFlags); + if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ + pNew->pLeft = p->pLeft ? + exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; + pNew->pRight = p->pRight ? + exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; + } + if( pzBuffer ){ + *pzBuffer = zAlloc; + } + }else{ + if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + if( pNew->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; + }else{ + pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); + } + pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); + } } } return pNew; @@ -82081,18 +94102,20 @@ static With *withDup(sqlite3 *db, With *p){ ** part of the in-memory representation of the database schema. */ SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ - return exprDup(db, p, flags, 0); + assert( flags==0 || flags==EXPRDUP_REDUCE ); + return p ? exprDup(db, p, flags, 0) : 0; } SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ ExprList *pNew; struct ExprList_item *pItem, *pOldItem; int i; + assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); if( pNew==0 ) return 0; pNew->nExpr = i = p->nExpr; if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; inExpr; i+=i){} - pNew->a = pItem = sqlite3DbMallocRaw(db, i*sizeof(p->a[0]) ); + pNew->a = pItem = sqlite3DbMallocRawNN(db, i*sizeof(p->a[0]) ); if( pItem==0 ){ sqlite3DbFree(db, pNew); return 0; @@ -82123,9 +94146,10 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ SrcList *pNew; int i; int nByte; + assert( db!=0 ); if( p==0 ) return 0; nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqlite3DbMallocRaw(db, nByte ); + pNew = sqlite3DbMallocRawNN(db, nByte ); if( pNew==0 ) return 0; pNew->nSrc = pNew->nAlloc = p->nSrc; for(i=0; inSrc; i++){ @@ -82136,16 +94160,18 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->jointype = pOldItem->jointype; + pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; - pNewItem->isCorrelated = pOldItem->isCorrelated; - pNewItem->viaCoroutine = pOldItem->viaCoroutine; - pNewItem->isRecursive = pOldItem->isRecursive; - pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); - pNewItem->notIndexed = pOldItem->notIndexed; - pNewItem->pIndex = pOldItem->pIndex; + if( pNewItem->fg.isIndexedBy ){ + pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + } + pNewItem->pIBIndex = pOldItem->pIBIndex; + if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + } pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ pTab->nRef++; @@ -82160,11 +94186,12 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ IdList *pNew; int i; + assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); if( pNew==0 ) return 0; pNew->nId = p->nId; - pNew->a = sqlite3DbMallocRaw(db, p->nId*sizeof(p->a[0]) ); + pNew->a = sqlite3DbMallocRawNN(db, p->nId*sizeof(p->a[0]) ); if( pNew->a==0 ){ sqlite3DbFree(db, pNew); return 0; @@ -82182,8 +94209,9 @@ SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ } SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ Select *pNew, *pPrior; + assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); if( pNew==0 ) return 0; pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); @@ -82204,6 +94232,7 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); + sqlite3SelectSetName(pNew, p->zSelName); return pNew; } #else @@ -82228,12 +94257,14 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( Expr *pExpr /* Expression to be appended. Might be NULL */ ){ sqlite3 *db = pParse->db; + assert( db!=0 ); if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(ExprList) ); + pList = sqlite3DbMallocRawNN(db, sizeof(ExprList) ); if( pList==0 ){ goto no_mem; } - pList->a = sqlite3DbMallocRaw(db, sizeof(pList->a[0])); + pList->nExpr = 0; + pList->a = sqlite3DbMallocRawNN(db, sizeof(pList->a[0])); if( pList->a==0 ) goto no_mem; }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ struct ExprList_item *a; @@ -82259,6 +94290,74 @@ no_mem: return 0; } +/* +** pColumns and pExpr form a vector assignment which is part of the SET +** clause of an UPDATE statement. Like this: +** +** (a,b,c) = (expr1,expr2,expr3) +** Or: (a,b,c) = (SELECT x,y,z FROM ....) +** +** For each term of the vector assignment, append new entries to the +** expression list pList. In the case of a subquery on the LHS, append +** TK_SELECT_COLUMN expressions. +*/ +SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + IdList *pColumns, /* List of names of LHS of the assignment */ + Expr *pExpr /* Vector expression to be appended. Might be NULL */ +){ + sqlite3 *db = pParse->db; + int n; + int i; + int iFirst = pList ? pList->nExpr : 0; + /* pColumns can only be NULL due to an OOM but an OOM will cause an + ** exit prior to this routine being invoked */ + if( NEVER(pColumns==0) ) goto vector_append_error; + if( pExpr==0 ) goto vector_append_error; + n = sqlite3ExprVectorSize(pExpr); + if( pColumns->nId!=n ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pColumns->nId, n); + goto vector_append_error; + } + for(i=0; inExpr==iFirst+i+1 ); + pList->a[pList->nExpr-1].zName = pColumns->a[i].zName; + pColumns->a[i].zName = 0; + } + } + if( pExpr->op==TK_SELECT ){ + if( pList && pList->a[iFirst].pExpr ){ + assert( pList->a[iFirst].pExpr->op==TK_SELECT_COLUMN ); + pList->a[iFirst].pExpr->pRight = pExpr; + pExpr = 0; + } + } + +vector_append_error: + sqlite3ExprDelete(db, pExpr); + sqlite3IdListDelete(db, pColumns); + return pList; +} + +/* +** Set the sort order for the last element on the given ExprList. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ + if( p==0 ) return; + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC>=0 && SQLITE_SO_DESC>0 ); + assert( p->nExpr>0 ); + if( iSortOrder<0 ){ + assert( p->a[p->nExpr-1].sortOrder==SQLITE_SO_ASC ); + return; + } + p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; +} + /* ** Set the ExprList.a[].zName element of the most recently added item ** on the expression list. @@ -82280,7 +94379,7 @@ SQLITE_PRIVATE void sqlite3ExprListSetName( pItem = &pList->a[pList->nExpr-1]; assert( pItem->zName==0 ); pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); - if( dequote && pItem->zName ) sqlite3Dequote(pItem->zName); + if( dequote ) sqlite3Dequote(pItem->zName); } } @@ -82329,10 +94428,9 @@ SQLITE_PRIVATE void sqlite3ExprListCheckLength( /* ** Delete an entire expression list. */ -SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ +static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ int i; struct ExprList_item *pItem; - if( pList==0 ) return; assert( pList->a!=0 || pList->nExpr==0 ); for(pItem=pList->a, i=0; inExpr; i++, pItem++){ sqlite3ExprDelete(db, pItem->pExpr); @@ -82342,39 +94440,73 @@ SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ sqlite3DbFree(db, pList->a); sqlite3DbFree(db, pList); } +SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ + if( pList ) exprListDeleteNN(db, pList); +} /* -** These routines are Walker callbacks. Walker.u.pi is a pointer -** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.pi to 0 if the expression is -** not constant. +** Return the bitwise-OR of all Expr.flags fields in the given +** ExprList. +*/ +SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList *pList){ + int i; + u32 m = 0; + if( pList ){ + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + assert( pExpr!=0 ); + m |= pExpr->flags; + } + } + return m; +} + +/* +** These routines are Walker callbacks used to check expressions to +** see if they are "constant" for some definition of constant. The +** Walker.eCode value determines the type of "constant" we are looking +** for. ** ** These callback routines are used to implement the following: ** -** sqlite3ExprIsConstant() -** sqlite3ExprIsConstantNotJoin() -** sqlite3ExprIsConstantOrFunction() +** sqlite3ExprIsConstant() pWalker->eCode==1 +** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 +** sqlite3ExprIsTableConstant() pWalker->eCode==3 +** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 ** +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression +** is found to not be a constant. +** +** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions +** in a CREATE TABLE statement. The Walker.eCode value is 5 when parsing +** an existing schema and 4 when processing a new statement. A bound +** parameter raises an error for new statements, but is silently converted +** to NULL for existing schemas. This allows sqlite_master tables that +** contain a bound parameter because they were generated by older versions +** of SQLite to be parsed by newer versions of SQLite without raising a +** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ - /* If pWalker->u.i is 3 then any term of the expression that comes from - ** the ON or USING clauses of a join disqualifies the expression + /* If pWalker->eCode is 2 then any term of the expression that comes from + ** the ON or USING clauses of a left join disqualifies the expression ** from being considered constant. */ - if( pWalker->u.i==3 && ExprHasProperty(pExpr, EP_FromJoin) ){ - pWalker->u.i = 0; + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->eCode = 0; return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->u.i==2 or the function as the SQLITE_FUNC_CONST - ** flag. */ + ** and either pWalker->eCode==4 or 5 or the function has the + ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->u.i==2 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc) ){ return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; } - /* Fall through */ case TK_ID: case TK_COLUMN: case TK_AGG_FUNCTION: @@ -82383,8 +94515,25 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); - pWalker->u.i = 0; - return WRC_Abort; + if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ + return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; + } + case TK_VARIABLE: + if( pWalker->eCode==5 ){ + /* Silently convert bound parameters that appear inside of CREATE + ** statements into a NULL when parsing the CREATE statement text out + ** of the sqlite_master table */ + pExpr->op = TK_NULL; + }else if( pWalker->eCode==4 ){ + /* A bound parameter in a CREATE statement that originates from + ** sqlite3_prepare() causes an error */ + pWalker->eCode = 0; + return WRC_Abort; + } + /* Fall through */ default: testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */ testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */ @@ -82393,21 +94542,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ } static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ UNUSED_PARAMETER(NotUsed); - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } -static int exprIsConst(Expr *p, int initFlag){ +static int exprIsConst(Expr *p, int initFlag, int iCur){ Walker w; memset(&w, 0, sizeof(w)); - w.u.i = initFlag; + w.eCode = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; + w.u.iCur = iCur; sqlite3WalkExpr(&w, p); - return w.u.i; + return w.eCode; } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** and 0 if it involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") @@ -82415,21 +94565,31 @@ static int exprIsConst(Expr *p, int initFlag){ ** a constant. */ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1); + return exprIsConst(p, 1, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** that does no originate from the ON or USING clauses of a join. ** Return 0 if it involves variables or function calls or terms from ** an ON or USING clause. */ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 3); + return exprIsConst(p, 2, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant +** for any single row of the table with cursor iCur. In other words, the +** expression must not refer to any non-deterministic function nor any +** table other than iCur. +*/ +SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + return exprIsConst(p, 3, iCur); +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant ** or a function call with constant arguments. Return and 0 if there ** are any variables. ** @@ -82437,10 +94597,27 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. */ -SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p){ - return exprIsConst(p, 2); +SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ + assert( isInit==0 || isInit==1 ); + return exprIsConst(p, 4+isInit, 0); } +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Walk an expression tree. Return 1 if the expression contains a +** subquery of some kind. Return 0 if there are no subqueries. +*/ +SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 1; + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = selectNodeIsConstant; + sqlite3WalkExpr(&w, p); + return w.eCode==0; +} +#endif + /* ** If the expression p codes a constant integer that is small enough ** to fit in a 32-bit integer, return 1 and put the value of the integer @@ -82505,7 +94682,8 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( p->pTab!=0 ); - return p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0; + return ExprHasProperty(p, EP_CanBeNull) || + (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0); default: return 1; } @@ -82523,7 +94701,7 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ u8 op; - if( aff==SQLITE_AFF_NONE ) return 1; + if( aff==SQLITE_AFF_BLOB ) return 1; while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } op = p->op; if( op==TK_REGISTER ) op = p->op2; @@ -82562,23 +94740,22 @@ SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ } /* -** Return true if we are able to the IN operator optimization on a -** query of the form -** -** x IN (SELECT ...) -** -** Where the SELECT... clause is as specified by the parameter to this -** routine. -** -** The Select object passed in has already been preprocessed and no -** errors have been found. +** pX is the RHS of an IN operator. If pX is a SELECT statement +** that can be simplified to a direct table access, then return +** a pointer to the SELECT statement. If pX is not a SELECT statement, +** or if the SELECT statement needs to be manifested into a transient +** table, then return NULL. */ #ifndef SQLITE_OMIT_SUBQUERY -static int isCandidateForInOpt(Select *p){ +static Select *isCandidateForInOpt(Expr *pX){ + Select *p; SrcList *pSrc; ExprList *pEList; Table *pTab; - if( p==0 ) return 0; /* right-hand side of IN is SELECT */ + int i; + if( !ExprHasProperty(pX, EP_xIsSelect) ) return 0; /* Not a subquery */ + if( ExprHasProperty(pX, EP_VarSelect) ) return 0; /* Correlated subq */ + p = pX->x.pSelect; if( p->pPrior ) return 0; /* Not a compound SELECT */ if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); @@ -82594,25 +94771,22 @@ static int isCandidateForInOpt(Select *p){ if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ pTab = pSrc->a[0].pTab; - if( NEVER(pTab==0) ) return 0; + assert( pTab!=0 ); assert( pTab->pSelect==0 ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ pEList = p->pEList; - if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ - if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */ - return 1; + assert( pEList!=0 ); + /* All SELECT results must be columns. */ + for(i=0; inExpr; i++){ + Expr *pRes = pEList->a[i].pExpr; + if( pRes->op!=TK_COLUMN ) return 0; + assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ + } + return p; } #endif /* SQLITE_OMIT_SUBQUERY */ -/* -** Code an OP_Once instruction and allocate space for its flag. Return the -** address of the new instruction. -*/ -SQLITE_PRIVATE int sqlite3CodeOnce(Parse *pParse){ - Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ - return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++); -} - +#ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code that checks the left-most column of index table iCur to see if ** it contains any NULL entries. Cause the register at regHasNull to be set @@ -82620,14 +94794,15 @@ SQLITE_PRIVATE int sqlite3CodeOnce(Parse *pParse){ ** to be set to NULL if iCur contains one or more NULL values. */ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ - int j1; + int addr1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); - j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); VdbeComment((v, "first_entry_in(%d)", iCur)); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } +#endif #ifndef SQLITE_OMIT_SUBQUERY @@ -82672,11 +94847,11 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: ** -** SELECT FROM +** SELECT , ... FROM
    ** ** If the RHS of the IN operator is a list or a more complex subquery, then ** an ephemeral table might need to be generated from the RHS and then -** pX->iTable made to point to the ephermeral table instead of an +** pX->iTable made to point to the ephemeral table instead of an ** existing table. ** ** The inFlags parameter must contain exactly one of the bits @@ -82688,14 +94863,14 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate ** through the set members) then the b-tree must not contain duplicates. -** An epheremal table must be used unless the selected is guaranteed -** to be unique - either because it is an INTEGER PRIMARY KEY or it -** has a UNIQUE constraint or UNIQUE index. +** An epheremal table must be used unless the selected columns are guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or due to +** a UNIQUE constraint or index. ** ** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used ** for fast set membership tests) then an epheremal table must -** be used unless is an INTEGER PRIMARY KEY or an index can -** be found with as its left-most column. +** be used unless is a single INTEGER PRIMARY KEY column or an +** index can be found with the specified as its left-most. ** ** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and ** if the RHS of the IN operator is a list (not a subquery) then this @@ -82716,9 +94891,26 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** the value in that register will be NULL if the b-tree contains one or more ** NULL values, and it will be some non-NULL value if the b-tree contains no ** NULL values. +** +** If the aiMap parameter is not NULL, it must point to an array containing +** one element for each column returned by the SELECT statement on the RHS +** of the IN(...) operator. The i'th entry of the array is populated with the +** offset of the index column that matches the i'th column returned by the +** SELECT. For example, if the expression and selected index are: +** +** (?,?,?) IN (SELECT a, b, c FROM t1) +** CREATE INDEX i1 ON t1(b, c, a); +** +** then aiMap[] is populated with {2, 0, 1}. */ #ifndef SQLITE_OMIT_SUBQUERY -SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ +SQLITE_PRIVATE int sqlite3FindInIndex( + Parse *pParse, /* Parsing context */ + Expr *pX, /* The right-hand side (RHS) of the IN operator */ + u32 inFlags, /* IN_INDEX_LOOP, _MEMBERSHIP, and/or _NOOP_OK */ + int *prRhsHasNull, /* Register holding NULL status. See notes */ + int *aiMap /* Mapping from Index fields to RHS fields */ +){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ @@ -82728,38 +94920,46 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int assert( pX->op==TK_IN ); mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; + /* If the RHS of this IN(...) operator is a SELECT, and if it matters + ** whether or not the SELECT result contains NULL values, check whether + ** or not NULL is actually possible (it may not be, for example, due + ** to NOT NULL constraints in the schema). If no NULL values are possible, + ** set prRhsHasNull to 0 before continuing. */ + if( prRhsHasNull && (pX->flags & EP_xIsSelect) ){ + int i; + ExprList *pEList = pX->x.pSelect->pEList; + for(i=0; inExpr; i++){ + if( sqlite3ExprCanBeNull(pEList->a[i].pExpr) ) break; + } + if( i==pEList->nExpr ){ + prRhsHasNull = 0; + } + } + /* Check to see if an existing table or index can be used to ** satisfy the query. This is preferable to generating a new - ** ephemeral table. - */ - p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); - if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) ){ + ** ephemeral table. */ + if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){ sqlite3 *db = pParse->db; /* Database connection */ Table *pTab; /* Table
    . */ - Expr *pExpr; /* Expression */ - i16 iCol; /* Index of column */ i16 iDb; /* Database idx for pTab */ + ExprList *pEList = p->pEList; + int nExpr = pEList->nExpr; - assert( p ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ pTab = p->pSrc->a[0].pTab; - pExpr = p->pEList->a[0].pExpr; - iCol = (i16)pExpr->iColumn; - + /* Code an OP_Transaction and OP_TableLock for
    . */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - /* This function is only called from two places. In both cases the vdbe - ** has already been allocated. So assume sqlite3GetVdbe() is always - ** successful here. - */ - assert(v); - if( iCol<0 ){ - int iAddr = sqlite3CodeOnce(pParse); + assert(v); /* sqlite3GetVdbe() has always been previously called */ + if( nExpr==1 && pEList->a[0].pExpr->iColumn<0 ){ + /* The "x IN (SELECT rowid FROM table)" case */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); @@ -82768,45 +94968,115 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int sqlite3VdbeJumpHere(v, iAddr); }else{ Index *pIdx; /* Iterator variable */ + int affinity_ok = 1; + int i; - /* The collation sequence used by the comparison. If an index is to - ** be used in place of a temp-table, it must be ordered according - ** to this collation sequence. */ - CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); - - /* Check that the affinity that will be used to perform the - ** comparison is the same as the affinity of the column. If - ** it is not, it is not possible to use any index. - */ - int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); - - for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ - if( (pIdx->aiColumn[0]==iCol) - && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq - && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx))) - ){ - int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); - sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "%s", pIdx->zName)); - assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); - eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - - if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ - *prRhsHasNull = ++pParse->nMem; - sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); - } - sqlite3VdbeJumpHere(v, iAddr); + /* Check that the affinity that will be used to perform each + ** comparison is the same as the affinity of each column in table + ** on the RHS of the IN operator. If it not, it is not possible to + ** use any index of the RHS table. */ + for(i=0; ipLeft, i); + int iCol = pEList->a[i].pExpr->iColumn; + char idxaff = sqlite3TableColumnAffinity(pTab,iCol); /* RHS table */ + char cmpaff = sqlite3CompareAffinity(pLhs, idxaff); + testcase( cmpaff==SQLITE_AFF_BLOB ); + testcase( cmpaff==SQLITE_AFF_TEXT ); + switch( cmpaff ){ + case SQLITE_AFF_BLOB: + break; + case SQLITE_AFF_TEXT: + /* sqlite3CompareAffinity() only returns TEXT if one side or the + ** other has no affinity and the other side is TEXT. Hence, + ** the only way for cmpaff to be TEXT is for idxaff to be TEXT + ** and for the term on the LHS of the IN to have no affinity. */ + assert( idxaff==SQLITE_AFF_TEXT ); + break; + default: + affinity_ok = sqlite3IsNumericAffinity(idxaff); } } - } - } + + if( affinity_ok ){ + /* Search for an existing index that will work for this IN operator */ + for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){ + Bitmask colUsed; /* Columns of the index used */ + Bitmask mCol; /* Mask for the current column */ + if( pIdx->nColumnnColumn==BMS-2 ); + testcase( pIdx->nColumn==BMS-1 ); + if( pIdx->nColumn>=BMS-1 ) continue; + if( mustBeUnique ){ + if( pIdx->nKeyCol>nExpr + ||(pIdx->nColumn>nExpr && !IsUniqueIndex(pIdx)) + ){ + continue; /* This index is not unique over the IN RHS columns */ + } + } + + colUsed = 0; /* Columns of index used so far */ + for(i=0; ipLeft, i); + Expr *pRhs = pEList->a[i].pExpr; + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + int j; + + assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr ); + for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; + assert( pIdx->azColl[j] ); + if( pReq!=0 && sqlite3StrICmp(pReq->zName, pIdx->azColl[j])!=0 ){ + continue; + } + break; + } + if( j==nExpr ) break; + mCol = MASKBIT(j); + if( mCol & colUsed ) break; /* Each column used only once */ + colUsed |= mCol; + if( aiMap ) aiMap[i] = j; + } + + assert( i==nExpr || colUsed!=(MASKBIT(nExpr)-1) ); + if( colUsed==(MASKBIT(nExpr)-1) ){ + /* If we reach this point, that means the index pIdx is usable */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); +#ifndef SQLITE_OMIT_EXPLAIN + sqlite3VdbeAddOp4(v, OP_Explain, 0, 0, 0, + sqlite3MPrintf(db, "USING INDEX %s FOR IN-OPERATOR",pIdx->zName), + P4_DYNAMIC); +#endif + sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; + + if( prRhsHasNull ){ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + i64 mask = (1<nMem; + if( nExpr==1 ){ + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); + } + } + sqlite3VdbeJumpHere(v, iAddr); + } + } /* End loop over indexes */ + } /* End if( affinity_ok ) */ + } /* End if not an rowid index */ + } /* End attempt to optimize using an index */ /* If no preexisting index is available for the IN clause ** and IN_INDEX_NOOP is an allowed reply ** and the RHS of the IN operator is a list, not a subquery - ** and the RHS is not contant or has two or fewer terms, - ** then it is not worth creating an ephermeral table to evaluate + ** and the RHS is not constant or has two or fewer terms, + ** then it is not worth creating an ephemeral table to evaluate ** the IN operator so return IN_INDEX_NOOP. */ if( eType==0 @@ -82816,7 +95086,6 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int ){ eType = IN_INDEX_NOOP; } - if( eType==0 ){ /* Could not find an existing table or index to use as the RHS b-tree. @@ -82838,10 +95107,63 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int }else{ pX->iTable = iTab; } + + if( aiMap && eType!=IN_INDEX_INDEX_ASC && eType!=IN_INDEX_INDEX_DESC ){ + int i, n; + n = sqlite3ExprVectorSize(pX->pLeft); + for(i=0; ipLeft; + int nVal = sqlite3ExprVectorSize(pLeft); + Select *pSelect = (pExpr->flags & EP_xIsSelect) ? pExpr->x.pSelect : 0; + char *zRet; + + assert( pExpr->op==TK_IN ); + zRet = sqlite3DbMallocZero(pParse->db, nVal+1); + if( zRet ){ + int i; + for(i=0; ipEList->a[i].pExpr, a); + }else{ + zRet[i] = a; + } + } + zRet[nVal] = '\0'; + } + return zRet; +} +#endif + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Load the Parse object passed as the first argument with an error +** message of the form: +** +** "sub-select returns N columns - expected M" +*/ +SQLITE_PRIVATE void sqlite3SubselectError(Parse *pParse, int nActual, int nExpect){ + const char *zFmt = "sub-select returns %d columns - expected %d"; + sqlite3ErrorMsg(pParse, zFmt, nActual, nExpect); +} +#endif + /* ** Generate code for scalar subqueries used as a subquery expression, EXISTS, ** or IN operators. Examples: @@ -82867,7 +95189,9 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int ** value to non-NULL if the RHS is NULL-free. ** ** For a SELECT or EXISTS operator, return the register that holds the -** result. For IN operators or if an error occurs, the return value is 0. +** result. For a multi-column SELECT, the result is stored in a contiguous +** array of registers and the return value is the register of the left-most +** result column. Return 0 for IN operators or if an error occurs. */ #ifndef SQLITE_OMIT_SUBQUERY SQLITE_PRIVATE int sqlite3CodeSubselect( @@ -82882,8 +95206,8 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( if( NEVER(v==0) ) return 0; sqlite3ExprCachePush(pParse); - /* This code must be run in its entirety every time it is encountered - ** if any of the following is true: + /* The evaluation of the IN/EXISTS/SELECT must be repeated every time it + ** is encountered if any of the following is true: ** ** * The right-hand side is a correlated subquery ** * The right-hand side is an expression list containing variables @@ -82893,14 +95217,15 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** save the results, and reuse the same result on subsequent invocations. */ if( !ExprHasProperty(pExpr, EP_VarSelect) ){ - jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v); + jmpIfDynamic = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } #ifndef SQLITE_OMIT_EXPLAIN if( pParse->explain==2 ){ - char *zMsg = sqlite3MPrintf( - pParse->db, "EXECUTE %s%s SUBQUERY %d", jmpIfDynamic>=0?"":"CORRELATED ", - pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId + char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %s%s SUBQUERY %d", + jmpIfDynamic>=0?"":"CORRELATED ", + pExpr->op==TK_IN?"LIST":"SCALAR", + pParse->iNextSelectId ); sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); } @@ -82908,17 +95233,18 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( switch( pExpr->op ){ case TK_IN: { - char affinity; /* Affinity of the LHS of the IN */ int addr; /* Address of OP_OpenEphemeral instruction */ Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ KeyInfo *pKeyInfo = 0; /* Key information */ - - affinity = sqlite3ExprAffinity(pLeft); + int nVal; /* Size of vector pLeft */ + + nVal = sqlite3ExprVectorSize(pLeft); + assert( !isRowid || nVal==1 ); /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' ** expression it is handled the same way. An ephemeral table is - ** filled with single-field index keys representing the results - ** from the SELECT or the . + ** filled with index keys representing the results from the + ** SELECT or the . ** ** If the 'x' expression is a column value, or the SELECT... ** statement returns a column value, then the affinity of that @@ -82929,8 +95255,9 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** is used. */ pExpr->iTable = pParse->nTab++; - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); - pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, 1, 1); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, + pExpr->iTable, (isRowid?0:nVal)); + pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, nVal, 1); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -82939,28 +95266,37 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** table allocated and opened above. */ Select *pSelect = pExpr->x.pSelect; - SelectDest dest; - ExprList *pEList; + ExprList *pEList = pSelect->pEList; assert( !isRowid ); - sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); - dest.affSdst = (u8)affinity; - assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - pSelect->iLimit = 0; - testcase( pSelect->selFlags & SF_Distinct ); - pSelect->selFlags &= ~SF_Distinct; - testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ - if( sqlite3Select(pParse, pSelect, &dest) ){ - sqlite3KeyInfoUnref(pKeyInfo); - return 0; + /* If the LHS and RHS of the IN operator do not match, that + ** error will have been caught long before we reach this point. */ + if( ALWAYS(pEList->nExpr==nVal) ){ + SelectDest dest; + int i; + sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); + dest.zAffSdst = exprINAffinity(pParse, pExpr); + assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + if( sqlite3Select(pParse, pSelect, &dest) ){ + sqlite3DbFree(pParse->db, dest.zAffSdst); + sqlite3KeyInfoUnref(pKeyInfo); + return 0; + } + sqlite3DbFree(pParse->db, dest.zAffSdst); + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + for(i=0; iaColl[i] = sqlite3BinaryCompareCollSeq( + pParse, p, pEList->a[i].pExpr + ); + } } - pEList = pSelect->pEList; - assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ - assert( pEList!=0 ); - assert( pEList->nExpr>0 ); - assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); - pKeyInfo->aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, - pEList->a[0].pExpr); }else if( ALWAYS(pExpr->x.pList!=0) ){ /* Case 2: expr IN (exprlist) ** @@ -82969,13 +95305,15 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** that columns affinity when building index keys. If is not ** a column, use numeric affinity. */ + char affinity; /* Affinity of the LHS of the IN */ int i; ExprList *pList = pExpr->x.pList; struct ExprList_item *pItem; int r1, r2, r3; + affinity = sqlite3ExprAffinity(pLeft); if( !affinity ){ - affinity = SQLITE_AFF_NONE; + affinity = SQLITE_AFF_BLOB; } if( pKeyInfo ){ assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); @@ -83029,25 +95367,37 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( case TK_EXISTS: case TK_SELECT: default: { - /* If this has to be a scalar SELECT. Generate code to put the - ** value of this select in a memory cell and record the number - ** of the memory cell in iColumn. If this is an EXISTS, write - ** an integer 0 (not exists) or 1 (exists) into a memory cell - ** and record that memory cell in iColumn. + /* Case 3: (SELECT ... FROM ...) + ** or: EXISTS(SELECT ... FROM ...) + ** + ** For a SELECT, generate code to put the values for all columns of + ** the first row into an array of registers and return the index of + ** the first register. + ** + ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) + ** into a register and return that register number. + ** + ** In both cases, the query is augmented with "LIMIT 1". Any + ** preexisting limit is discarded in place of the new LIMIT 1. */ Select *pSel; /* SELECT statement to encode */ - SelectDest dest; /* How to deal with SELECt result */ + SelectDest dest; /* How to deal with SELECT result */ + int nReg; /* Registers to allocate */ testcase( pExpr->op==TK_EXISTS ); testcase( pExpr->op==TK_SELECT ); assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); - assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + pSel = pExpr->x.pSelect; - sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); + nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; + sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); + pParse->nMem += nReg; if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; - sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); + dest.iSdst = dest.iSDParm; + dest.nSdst = nReg; + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); VdbeComment((v, "Init subquery result")); }else{ dest.eDest = SRT_Exists; @@ -83055,9 +95405,10 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( VdbeComment((v, "Init EXISTS result")); } sqlite3ExprDelete(pParse->db, pSel->pLimit); - pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, - &sqlite3IntTokens[1]); + pSel->pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER, + &sqlite3IntTokens[1], 0); pSel->iLimit = 0; + pSel->selFlags &= ~SF_MultiValue; if( sqlite3Select(pParse, pSel, &dest) ){ return 0; } @@ -83080,6 +95431,32 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( } #endif /* SQLITE_OMIT_SUBQUERY */ +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Expr pIn is an IN(...) expression. This function checks that the +** sub-select on the RHS of the IN() operator has the same number of +** columns as the vector on the LHS. Or, if the RHS of the IN() is not +** a sub-query, that the LHS is a vector of size 1. +*/ +SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ + int nVector = sqlite3ExprVectorSize(pIn->pLeft); + if( (pIn->flags & EP_xIsSelect) ){ + if( nVector!=pIn->x.pSelect->pEList->nExpr ){ + sqlite3SubselectError(pParse, pIn->x.pSelect->pEList->nExpr, nVector); + return 1; + } + }else if( nVector!=1 ){ + if( (pIn->pLeft->flags & EP_xIsSelect) ){ + sqlite3SubselectError(pParse, nVector, 1); + }else{ + sqlite3ErrorMsg(pParse, "row value misused"); + } + return 1; + } + return 0; +} +#endif + #ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code for an IN expression. @@ -83087,16 +95464,24 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** x IN (SELECT ...) ** x IN (value, value, ...) ** -** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS) -** is an array of zero or more values. The expression is true if the LHS is -** contained within the RHS. The value of the expression is unknown (NULL) -** if the LHS is NULL or if the LHS is not contained within the RHS and the -** RHS contains one or more NULL values. +** The left-hand side (LHS) is a scalar or vector expression. The +** right-hand side (RHS) is an array of zero or more scalar values, or a +** subquery. If the RHS is a subquery, the number of result columns must +** match the number of columns in the vector on the LHS. If the RHS is +** a list of values, the LHS must be a scalar. +** +** The IN operator is true if the LHS value is contained within the RHS. +** The result is false if the LHS is definitely not in the RHS. The +** result is NULL if the presence of the LHS in the RHS cannot be +** determined due to NULLs. ** ** This routine generates code that jumps to destIfFalse if the LHS is not ** contained within the RHS. If due to NULLs we cannot determine if the LHS ** is contained in the RHS then jump to destIfNull. If the LHS is contained ** within the RHS then fall through. +** +** See the separate in-operator.md documentation file in the canonical +** SQLite source tree for additional information. */ static void sqlite3ExprCodeIN( Parse *pParse, /* Parsing and code generating context */ @@ -83105,36 +95490,83 @@ static void sqlite3ExprCodeIN( int destIfNull /* Jump here if the results are unknown due to NULLs */ ){ int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ - char affinity; /* Comparison affinity to use */ int eType; /* Type of the RHS */ - int r1; /* Temporary use register */ + int rLhs; /* Register(s) holding the LHS values */ + int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ Vdbe *v; /* Statement under construction */ + int *aiMap = 0; /* Map from vector field to index column */ + char *zAff = 0; /* Affinity string for comparisons */ + int nVector; /* Size of vectors for this IN operator */ + int iDummy; /* Dummy parameter to exprCodeVector() */ + Expr *pLeft; /* The LHS of the IN operator */ + int i; /* loop counter */ + int destStep2; /* Where to jump when NULLs seen in step 2 */ + int destStep6 = 0; /* Start of code for Step 6 */ + int addrTruthOp; /* Address of opcode that determines the IN is true */ + int destNotNull; /* Jump here if a comparison is not true in step 6 */ + int addrTop; /* Top of the step-6 loop */ - /* Compute the RHS. After this step, the table with cursor - ** pExpr->iTable will contains the values that make up the RHS. - */ + pLeft = pExpr->pLeft; + if( sqlite3ExprCheckIN(pParse, pExpr) ) return; + zAff = exprINAffinity(pParse, pExpr); + nVector = sqlite3ExprVectorSize(pExpr->pLeft); + aiMap = (int*)sqlite3DbMallocZero( + pParse->db, nVector*(sizeof(int) + sizeof(char)) + 1 + ); + if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; + + /* Attempt to compute the RHS. After this step, if anything other than + ** IN_INDEX_NOOP is returned, the table opened ith cursor pExpr->iTable + ** contains the values that make up the RHS. If IN_INDEX_NOOP is returned, + ** the RHS has not yet been coded. */ v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); eType = sqlite3FindInIndex(pParse, pExpr, IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, - destIfFalse==destIfNull ? 0 : &rRhsHasNull); + destIfFalse==destIfNull ? 0 : &rRhsHasNull, aiMap); - /* Figure out the affinity to use to create a key from the results - ** of the expression. affinityStr stores a static string suitable for - ** P4 of OP_MakeRecord. - */ - affinity = comparisonAffinity(pExpr); + assert( pParse->nErr || nVector==1 || eType==IN_INDEX_EPH + || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC + ); +#ifdef SQLITE_DEBUG + /* Confirm that aiMap[] contains nVector integer values between 0 and + ** nVector-1. */ + for(i=0; i from " IN (...)". + /* Code the LHS, the from " IN (...)". If the LHS is a + ** vector, then it is stored in an array of nVector registers starting + ** at r1. + ** + ** sqlite3FindInIndex() might have reordered the fields of the LHS vector + ** so that the fields are in the same order as an existing index. The + ** aiMap[] array contains a mapping from the original LHS field order to + ** the field order that matches the RHS index. */ sqlite3ExprCachePush(pParse); - r1 = sqlite3GetTempReg(pParse); - sqlite3ExprCode(pParse, pExpr->pLeft, r1); + rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + for(i=0; ix.pList; @@ -83146,7 +95578,7 @@ static void sqlite3ExprCodeIN( assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); if( destIfNull!=destIfFalse ){ regCkNull = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull); + sqlite3VdbeAddOp3(v, OP_BitAnd, rLhs, rLhs, regCkNull); } for(ii=0; iinExpr; ii++){ r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); @@ -83154,111 +95586,135 @@ static void sqlite3ExprCodeIN( sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); } if( iinExpr-1 || destIfNull!=destIfFalse ){ - sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, + sqlite3VdbeAddOp4(v, OP_Eq, rLhs, labelOk, r2, (void*)pColl, P4_COLLSEQ); VdbeCoverageIf(v, iinExpr-1); VdbeCoverageIf(v, ii==pList->nExpr-1); - sqlite3VdbeChangeP5(v, affinity); + sqlite3VdbeChangeP5(v, zAff[0]); }else{ assert( destIfNull==destIfFalse ); - sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2, + sqlite3VdbeAddOp4(v, OP_Ne, rLhs, destIfFalse, r2, (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL); + sqlite3VdbeChangeP5(v, zAff[0] | SQLITE_JUMPIFNULL); } sqlite3ReleaseTempReg(pParse, regToFree); } if( regCkNull ){ sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeGoto(v, destIfFalse); } sqlite3VdbeResolveLabel(v, labelOk); sqlite3ReleaseTempReg(pParse, regCkNull); + goto sqlite3ExprCodeIN_finished; + } + + /* Step 2: Check to see if the LHS contains any NULL columns. If the + ** LHS does contain NULLs then the result must be either FALSE or NULL. + ** We will then skip the binary search of the RHS. + */ + if( destIfNull==destIfFalse ){ + destStep2 = destIfFalse; }else{ - - /* If the LHS is NULL, then the result is either false or NULL depending - ** on whether the RHS is empty or not, respectively. - */ - if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ - if( destIfNull==destIfFalse ){ - /* Shortcut for the common case where the false and NULL outcomes are - ** the same. */ - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); - }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); - sqlite3VdbeJumpHere(v, addr1); - } - } - - if( eType==IN_INDEX_ROWID ){ - /* In this case, the RHS is the ROWID of table b-tree - */ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + destStep2 = destStep6 = sqlite3VdbeMakeLabel(v); + } + for(i=0; ipLeft, i); + if( sqlite3ExprCanBeNull(p) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, rLhs+i, destStep2); VdbeCoverage(v); - }else{ - /* In this case, the RHS is an index b-tree. - */ - sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); - - /* If the set membership test fails, then the result of the - ** "x IN (...)" expression must be either 0 or NULL. If the set - ** contains no NULL values, then the result is 0. If the set - ** contains one or more NULL values, then the result of the - ** expression is also NULL. - */ - assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); - if( rRhsHasNull==0 ){ - /* This branch runs if it is known at compile time that the RHS - ** cannot contain NULL values. This happens as the result - ** of a "NOT NULL" constraint in the database schema. - ** - ** Also run this branch if NULL is equivalent to FALSE - ** for this particular IN operator. - */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); - VdbeCoverage(v); - }else{ - /* In this branch, the RHS of the IN might contain a NULL and - ** the presence of a NULL on the RHS makes a difference in the - ** outcome. - */ - int j1; - - /* First check to see if the LHS is contained in the RHS. If so, - ** then the answer is TRUE the presence of NULLs in the RHS does - ** not matter. If the LHS is not contained in the RHS, then the - ** answer is NULL if the RHS contains NULLs and the answer is - ** FALSE if the RHS is NULL-free. - */ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); - sqlite3VdbeJumpHere(v, j1); - } } } - sqlite3ReleaseTempReg(pParse, r1); + + /* Step 3. The LHS is now known to be non-NULL. Do the binary search + ** of the RHS using the LHS as a probe. If found, the result is + ** true. + */ + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree and so we also + ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 + ** into a single opcode. */ + sqlite3VdbeAddOp3(v, OP_SeekRowid, pExpr->iTable, destIfFalse, rLhs); + VdbeCoverage(v); + addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ + }else{ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + if( destIfFalse==destIfNull ){ + /* Combine Step 3 and Step 5 into a single opcode */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + goto sqlite3ExprCodeIN_finished; + } + /* Ordinary Step 3, for the case where FALSE and NULL are distinct */ + addrTruthOp = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, + rLhs, nVector); VdbeCoverage(v); + } + + /* Step 4. If the RHS is known to be non-NULL and we did not find + ** an match on the search above, then the result must be FALSE. + */ + if( rRhsHasNull && nVector==1 ){ + sqlite3VdbeAddOp2(v, OP_NotNull, rRhsHasNull, destIfFalse); + VdbeCoverage(v); + } + + /* Step 5. If we do not care about the difference between NULL and + ** FALSE, then just return false. + */ + if( destIfFalse==destIfNull ) sqlite3VdbeGoto(v, destIfFalse); + + /* Step 6: Loop through rows of the RHS. Compare each row to the LHS. + ** If any comparison is NULL, then the result is NULL. If all + ** comparisons are FALSE then the final result is FALSE. + ** + ** For a scalar LHS, it is sufficient to check just the first row + ** of the RHS. + */ + if( destStep6 ) sqlite3VdbeResolveLabel(v, destStep6); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); + if( nVector>1 ){ + destNotNull = sqlite3VdbeMakeLabel(v); + }else{ + /* For nVector==1, combine steps 6 and 7 by immediately returning + ** FALSE if the first comparison is not NULL */ + destNotNull = destIfFalse; + } + for(i=0; iiTable, i, r3); + sqlite3VdbeAddOp4(v, OP_Ne, rLhs+i, destNotNull, r3, + (void*)pColl, P4_COLLSEQ); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, r3); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + if( nVector>1 ){ + sqlite3VdbeResolveLabel(v, destNotNull); + sqlite3VdbeAddOp2(v, OP_Next, pExpr->iTable, addrTop+1); + VdbeCoverage(v); + + /* Step 7: If we reach this point, we know that the result must + ** be false. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + } + + /* Jumps here in order to return true. */ + sqlite3VdbeJumpHere(v, addrTruthOp); + +sqlite3ExprCodeIN_finished: + if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); sqlite3ExprCachePop(pParse); VdbeComment((v, "end IN expr")); +sqlite3ExprCodeIN_oom_error: + sqlite3DbFree(pParse->db, aiMap); + sqlite3DbFree(pParse->db, zAff); } #endif /* SQLITE_OMIT_SUBQUERY */ -/* -** Duplicate an 8-byte value -*/ -static char *dup8bytes(Vdbe *v, const char *in){ - char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8); - if( out ){ - memcpy(out, in, 8); - } - return out; -} - #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Generate an instruction that will put the floating point @@ -83271,12 +95727,10 @@ static char *dup8bytes(Vdbe *v, const char *in){ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; - char *zV; sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; - zV = dup8bytes(v, (char*)&value); - sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); + sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); } } #endif @@ -83302,10 +95756,8 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ assert( z!=0 ); c = sqlite3DecOrHexToI64(z, &value); if( c==0 || (c==2 && negFlag) ){ - char *zV; if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } - zV = dup8bytes(v, (char*)&value); - sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); }else{ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); @@ -83324,14 +95776,17 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ } /* -** Clear a cache entry. +** Erase column-cache entry number i */ -static void cacheEntryClear(Parse *pParse, struct yColCache *p){ - if( p->tempReg ){ +static void cacheEntryClear(Parse *pParse, int i){ + if( pParse->aColCache[i].tempReg ){ if( pParse->nTempRegaTempReg) ){ - pParse->aTempReg[pParse->nTempReg++] = p->iReg; + pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; } - p->tempReg = 0; + } + pParse->nColCache--; + if( inColCache ){ + pParse->aColCache[i] = pParse->aColCache[pParse->nColCache]; } } @@ -83346,7 +95801,8 @@ SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int int idxLru; struct yColCache *p; - assert( iReg>0 ); /* Register numbers are always positive */ + /* Unless an error has occurred, register numbers are always positive. */ + assert( iReg>0 || pParse->nErr || pParse->db->mallocFailed ); assert( iCol>=-1 && iCol<32768 ); /* Finite column numbers */ /* The SQLITE_ColumnCache flag disables the column cache. This is used @@ -83361,43 +95817,33 @@ SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int ** that the object will never already be in cache. Verify this guarantee. */ #ifndef NDEBUG - for(i=0, p=pParse->aColCache; iiReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); + for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + assert( p->iTable!=iTab || p->iColumn!=iCol ); } #endif - /* Find an empty slot and replace it */ - for(i=0, p=pParse->aColCache; iiReg==0 ){ - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - return; + /* If the cache is already full, delete the least recently used entry */ + if( pParse->nColCache>=SQLITE_N_COLCACHE ){ + minLru = 0x7fffffff; + idxLru = -1; + for(i=0, p=pParse->aColCache; ilrulru; + } } + p = &pParse->aColCache[idxLru]; + }else{ + p = &pParse->aColCache[pParse->nColCache++]; } - /* Replace the last recently used */ - minLru = 0x7fffffff; - idxLru = -1; - for(i=0, p=pParse->aColCache; ilrulru; - } - } - if( ALWAYS(idxLru>=0) ){ - p = &pParse->aColCache[idxLru]; - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - return; - } + /* Add the new entry to the end of the cache */ + p->iLevel = pParse->iCacheLevel; + p->iTable = iTab; + p->iColumn = iCol; + p->iReg = iReg; + p->tempReg = 0; + p->lru = pParse->iCacheCnt++; } /* @@ -83405,14 +95851,13 @@ SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int ** Purge the range of registers from the column cache. */ SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ - int i; - int iLast = iReg + nReg - 1; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg; - if( r>=iReg && r<=iLast ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + int i = 0; + while( inColCache ){ + struct yColCache *p = &pParse->aColCache[i]; + if( p->iReg >= iReg && p->iReg < iReg+nReg ){ + cacheEntryClear(pParse, i); + }else{ + i++; } } } @@ -83437,8 +95882,7 @@ SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ ** the cache to the state it was in prior the most recent Push. */ SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ - int i; - struct yColCache *p; + int i = 0; assert( pParse->iCacheLevel>=1 ); pParse->iCacheLevel--; #ifdef SQLITE_DEBUG @@ -83446,10 +95890,11 @@ SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ printf("POP to %d\n", pParse->iCacheLevel); } #endif - for(i=0, p=pParse->aColCache; iiReg && p->iLevel>pParse->iCacheLevel ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + while( inColCache ){ + if( pParse->aColCache[i].iLevel>pParse->iCacheLevel ){ + cacheEntryClear(pParse, i); + }else{ + i++; } } } @@ -83463,13 +95908,35 @@ SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iaColCache; inColCache; i++, p++){ if( p->iReg==iReg ){ p->tempReg = 0; } } } +/* Generate code that will load into register regOut a value that is +** appropriate for the iIdxCol-th column of index pIdx. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn( + Parse *pParse, /* The parsing context */ + Index *pIdx, /* The index whose column is to be loaded */ + int iTabCur, /* Cursor pointing to a table row */ + int iIdxCol, /* The column of the index to be loaded */ + int regOut /* Store the index column value in this register */ +){ + i16 iTabCol = pIdx->aiColumn[iIdxCol]; + if( iTabCol==XN_EXPR ){ + assert( pIdx->aColExpr ); + assert( pIdx->aColExpr->nExpr>iIdxCol ); + pParse->iSelfTab = iTabCur; + sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); + }else{ + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, + iTabCol, regOut); + } +} + /* ** Generate code to extract the value of the iCol-th column of a table. */ @@ -83485,7 +95952,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( }else{ int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; int x = iCol; - if( !HasRowid(pTab) ){ + if( !HasRowid(pTab) && !IsVirtual(pTab) ){ x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol); } sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); @@ -83497,9 +95964,12 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( /* ** Generate code that will extract the iColumn-th column from -** table pTab and store the column value in a register. An effort -** is made to store the column value in register iReg, but this is -** not guaranteed. The location of the column value is returned. +** table pTab and store the column value in a register. +** +** An effort is made to store the column value in register iReg. This +** is not garanteeed for GetColumn() - the result can be stored in +** any register. But the result is guaranteed to land in register iReg +** for GetColumnToReg(). ** ** There must be an open cursor to pTab in iTable when this routine ** is called. If iColumn<0 then code is generated that extracts the rowid. @@ -83510,14 +95980,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ int iReg, /* Store results here */ - u8 p5 /* P5 value for OP_Column */ + u8 p5 /* P5 value for OP_Column + FLAGS */ ){ Vdbe *v = pParse->pVdbe; int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg>0 && p->iTable==iTable && p->iColumn==iColumn ){ + for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + if( p->iTable==iTable && p->iColumn==iColumn ){ p->lru = pParse->iCacheCnt++; sqlite3ExprCachePinRegister(pParse, p->iReg); return p->iReg; @@ -83532,25 +96002,37 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( } return iReg; } +SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Description of the table we are reading from */ + int iColumn, /* Index of the table column */ + int iTable, /* The cursor pointing to the table */ + int iReg /* Store results here */ +){ + int r1 = sqlite3ExprCodeGetColumn(pParse, pTab, iColumn, iTable, iReg, 0); + if( r1!=iReg ) sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, r1, iReg); +} + /* ** Clear all column cache entries. */ SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse *pParse){ int i; - struct yColCache *p; #if SQLITE_DEBUG if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ printf("CLEAR\n"); } #endif - for(i=0, p=pParse->aColCache; iiReg ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + for(i=0; inColCache; i++){ + if( pParse->aColCache[i].tempReg + && pParse->nTempRegaTempReg) + ){ + pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; } } + pParse->nColCache = 0; } /* @@ -83566,16 +96048,9 @@ SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, in ** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. */ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ - int i; - struct yColCache *p; assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); - for(i=0, p=pParse->aColCache; iiReg; - if( x>=iFrom && xiReg += iTo-iFrom; - } - } + sqlite3ExprCacheRemove(pParse, iFrom, nReg); } #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) @@ -83589,7 +96064,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int n static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iaColCache; inColCache; i++, p++){ int r = p->iReg; if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ } @@ -83597,8 +96072,11 @@ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ } #endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ + /* -** Convert an expression node to a TK_REGISTER +** Convert a scalar expression node to a TK_REGISTER referencing +** register iReg. The caller must ensure that iReg already contains +** the correct value for the expression. */ static void exprToRegister(Expr *p, int iReg){ p->op2 = p->op; @@ -83607,6 +96085,38 @@ static void exprToRegister(Expr *p, int iReg){ ExprClearProperty(p, EP_Skip); } +/* +** Evaluate an expression (either a vector or a scalar expression) and store +** the result in continguous temporary registers. Return the index of +** the first register used to store the result. +** +** If the returned result register is a temporary scalar, then also write +** that register number into *piFreeable. If the returned result register +** is not a temporary or if the expression is a vector set *piFreeable +** to 0. +*/ +static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ + int iResult; + int nResult = sqlite3ExprVectorSize(p); + if( nResult==1 ){ + iResult = sqlite3ExprCodeTemp(pParse, p, piFreeable); + }else{ + *piFreeable = 0; + if( p->op==TK_SELECT ){ + iResult = sqlite3CodeSubselect(pParse, p, 0, 0); + }else{ + int i; + iResult = pParse->nMem+1; + pParse->nMem += nResult; + for(i=0; ix.pList->a[i].pExpr, i+iResult); + } + } + } + return iResult; +} + + /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -83624,9 +96134,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) int inReg = target; /* Results stored in register inReg */ int regFree1 = 0; /* If non-zero free this temporary register */ int regFree2 = 0; /* If non-zero free this temporary register */ - int r1, r2, r3, r4; /* Various register numbers */ - sqlite3 *db = pParse->db; /* The database connection */ + int r1, r2; /* Various register numbers */ Expr tempX; /* Temporary expression node */ + int p5 = 0; assert( target>0 && target<=pParse->nMem ); if( v==0 ){ @@ -83645,12 +96155,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ assert( pCol->iMem>0 ); - inReg = pCol->iMem; - break; + return pCol->iMem; }else if( pAggInfo->useSortingIdx ){ sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - break; + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ } @@ -83659,37 +96168,36 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( iTab<0 ){ if( pParse->ckBase>0 ){ /* Generating CHECK constraints or inserting into partial index */ - inReg = pExpr->iColumn + pParse->ckBase; - break; + return pExpr->iColumn + pParse->ckBase; }else{ - /* Deleting from a partial index */ - iTab = pParse->iPartIdxTab; + /* Coding an expression that is part of an index where column names + ** in the index refer to the table to which the index belongs */ + iTab = pParse->iSelfTab; } } - inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + return sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, pExpr->iColumn, iTab, target, pExpr->op2); - break; } case TK_INTEGER: { codeInteger(pParse, pExpr, 0, target); - break; + return target; } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pExpr->u.zToken, 0, target); - break; + return target; } #endif case TK_STRING: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0, pExpr->u.zToken, 0); - break; + sqlite3VdbeLoadString(v, target, pExpr->u.zToken); + return target; } case TK_NULL: { sqlite3VdbeAddOp2(v, OP_Null, 0, target); - break; + return target; } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { @@ -83704,7 +96212,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( z[n]=='\'' ); zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); - break; + return target; } #endif case TK_VARIABLE: { @@ -83717,77 +96225,54 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); } - break; + return target; } case TK_REGISTER: { - inReg = pExpr->iTable; - break; - } - case TK_AS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; + return pExpr->iTable; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - int aff, to_op; inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - aff = sqlite3AffinityType(pExpr->u.zToken, 0); - to_op = aff - SQLITE_AFF_TEXT + OP_ToText; - assert( to_op==OP_ToText || aff!=SQLITE_AFF_TEXT ); - assert( to_op==OP_ToBlob || aff!=SQLITE_AFF_NONE ); - assert( to_op==OP_ToNumeric || aff!=SQLITE_AFF_NUMERIC ); - assert( to_op==OP_ToInt || aff!=SQLITE_AFF_INTEGER ); - assert( to_op==OP_ToReal || aff!=SQLITE_AFF_REAL ); - testcase( to_op==OP_ToText ); - testcase( to_op==OP_ToBlob ); - testcase( to_op==OP_ToNumeric ); - testcase( to_op==OP_ToInt ); - testcase( to_op==OP_ToReal ); if( inReg!=target ){ sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); inReg = target; } - sqlite3VdbeAddOp1(v, to_op, inReg); + sqlite3VdbeAddOp2(v, OP_Cast, target, + sqlite3AffinityType(pExpr->u.zToken, 0)); testcase( usedAsColumnCache(pParse, inReg, inReg) ); sqlite3ExprCacheAffinityChange(pParse, inReg, 1); - break; + return inReg; } #endif /* SQLITE_OMIT_CAST */ + case TK_IS: + case TK_ISNOT: + op = (op==TK_IS) ? TK_EQ : TK_NE; + p5 = SQLITE_NULLEQ; + /* fall-through */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2); - assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); - assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); - assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); - assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); - testcase( regFree1==0 ); - testcase( regFree2==0 ); + Expr *pLeft = pExpr->pLeft; + if( sqlite3ExprIsVector(pLeft) ){ + codeVectorCompare(pParse, pExpr, target, op, p5); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2 | p5); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + } break; } case TK_AND: @@ -83825,10 +96310,12 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( pLeft ); if( pLeft->op==TK_INTEGER ){ codeInteger(pParse, pLeft, 1, target); + return target; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( pLeft->op==TK_FLOAT ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pLeft->u.zToken, 1, target); + return target; #endif }else{ tempX.op = TK_INTEGER; @@ -83839,7 +96326,6 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); testcase( regFree2==0 ); } - inReg = target; break; } case TK_BITNOT: @@ -83848,7 +96334,6 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); - inReg = target; sqlite3VdbeAddOp2(v, op, r1, inReg); break; } @@ -83873,7 +96358,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); }else{ - inReg = pInfo->aFunc[pExpr->iAgg].iMem; + return pInfo->aFunc[pExpr->iAgg].iMem; } break; } @@ -83881,10 +96366,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ExprList *pFarg; /* List of function arguments */ int nFarg; /* Number of function arguments */ FuncDef *pDef; /* The function definition object */ - int nId; /* Length of the function name in bytes */ const char *zId; /* The function name */ u32 constMask = 0; /* Mask of function arguments that are constant */ int i; /* Loop counter */ + sqlite3 *db = pParse->db; /* The database connection */ u8 enc = ENC(db); /* The text encoding used by this database */ CollSeq *pColl = 0; /* A collating sequence */ @@ -83897,15 +96382,19 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) nFarg = pFarg ? pFarg->nExpr : 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); zId = pExpr->u.zToken; - nId = sqlite3Strlen30(zId); - pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); - if( pDef==0 || pDef->xFunc==0 ){ - sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); + pDef = sqlite3FindFunction(db, zId, nFarg, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 && pParse->explain ){ + pDef = sqlite3FindFunction(db, "unknown", nFarg, enc, 0); + } +#endif + if( pDef==0 || pDef->xFinalize!=0 ){ + sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); break; } /* Attempt a direct implementation of the built-in COALESCE() and - ** IFNULL() functions. This avoids unnecessary evalation of + ** IFNULL() functions. This avoids unnecessary evaluation of ** arguments past the first non-NULL argument. */ if( pDef->funcFlags & SQLITE_FUNC_COALESCE ){ @@ -83929,8 +96418,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) */ if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ assert( nFarg>=1 ); - sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); - break; + return sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); } for(i=0; ipDfltColl; sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_Function, constMask, r1, target, + sqlite3VdbeAddOp4(v, OP_Function0, constMask, r1, target, (char*)pDef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nFarg); if( nFarg && constMask==0 ){ sqlite3ReleaseTempRange(pParse, r1, nFarg); } - break; + return target; } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: case TK_SELECT: { + int nCol; testcase( op==TK_EXISTS ); testcase( op==TK_SELECT ); - inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); + if( op==TK_SELECT && (nCol = pExpr->x.pSelect->pEList->nExpr)!=1 ){ + sqlite3SubselectError(pParse, nCol, 1); + }else{ + return sqlite3CodeSubselect(pParse, pExpr, 0, 0); + } break; } + case TK_SELECT_COLUMN: { + if( pExpr->pLeft->iTable==0 ){ + pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft, 0, 0); + } + return pExpr->pLeft->iTable + pExpr->iColumn; + } case TK_IN: { int destIfFalse = sqlite3VdbeMakeLabel(v); int destIfNull = sqlite3VdbeMakeLabel(v); @@ -84024,7 +96523,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3VdbeResolveLabel(v, destIfFalse); sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); sqlite3VdbeResolveLabel(v, destIfNull); - break; + return target; } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -84041,34 +96540,13 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** Z is stored in pExpr->pList->a[1].pExpr. */ case TK_BETWEEN: { - Expr *pLeft = pExpr->pLeft; - struct ExprList_item *pLItem = pExpr->x.pList->a; - Expr *pRight = pLItem->pExpr; - - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - r3 = sqlite3GetTempReg(pParse); - r4 = sqlite3GetTempReg(pParse); - codeCompare(pParse, pLeft, pRight, OP_Ge, - r1, r2, r3, SQLITE_STOREP2); VdbeCoverage(v); - pLItem++; - pRight = pLItem->pExpr; - sqlite3ReleaseTempReg(pParse, regFree2); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree2==0 ); - codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); - sqlite3ReleaseTempReg(pParse, r3); - sqlite3ReleaseTempReg(pParse, r4); - break; + exprCodeBetween(pParse, pExpr, target, 0, 0); + return target; } + case TK_SPAN: case TK_COLLATE: case TK_UPLUS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; + return sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); } case TK_TRIGGER: { @@ -84114,7 +96592,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) #ifndef SQLITE_OMIT_FLOATING_POINT /* If the column has REAL affinity, it may currently be stored as an - ** integer. Use OP_RealAffinity to make sure it is really real. */ + ** integer. Use OP_RealAffinity to make sure it is really real. + ** + ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to + ** floating point when extracting it from the record. */ if( pExpr->iColumn>=0 && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL ){ @@ -84124,6 +96605,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) break; } + case TK_VECTOR: { + sqlite3ErrorMsg(pParse, "row value misused"); + break; + } /* ** Form A: @@ -84167,8 +96652,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( (pX = pExpr->pLeft)!=0 ){ tempX = *pX; testcase( pX->op==TK_COLUMN ); - exprToRegister(&tempX, sqlite3ExprCodeTemp(pParse, pX, ®Free1)); + exprToRegister(&tempX, exprCodeVector(pParse, &tempX, ®Free1)); testcase( regFree1==0 ); + memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; opCompare.pLeft = &tempX; pTest = &opCompare; @@ -84191,7 +96677,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); + sqlite3VdbeGoto(v, endLabel); sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } @@ -84202,7 +96688,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } - assert( db->mallocFailed || pParse->nErr>0 + assert( pParse->db->mallocFailed || pParse->nErr>0 || pParse->iCacheLevel==iCacheLevel ); sqlite3VdbeResolveLabel(v, endLabel); break; @@ -84322,13 +96808,25 @@ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); }else{ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe || pParse->db->mallocFailed ); + assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); if( inReg!=target && pParse->pVdbe ){ sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); } } } +/* +** Make a transient copy of expression pExpr and then code it using +** sqlite3ExprCode(). This routine works just like sqlite3ExprCode() +** except that the input expression is guaranteed to be unchanged. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ + sqlite3 *db = pParse->db; + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( !db->mallocFailed ) sqlite3ExprCode(pParse, pExpr, target); + sqlite3ExprDelete(db, pExpr); +} + /* ** Generate code that will evaluate expression pExpr and store the ** results in register target. The results are guaranteed to appear @@ -84344,7 +96842,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int ta } /* -** Generate code that evalutes the given expression and puts the result +** Generate code that evaluates the given expression and puts the result ** in register target. ** ** Also make a copy of the expression results into another "cache" register @@ -84367,278 +96865,6 @@ SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int targ exprToRegister(pExpr, iMem); } -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) -/* -** Generate a human-readable explanation of an expression tree. -*/ -SQLITE_PRIVATE void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ - int op; /* The opcode being coded */ - const char *zBinOp = 0; /* Binary operator */ - const char *zUniOp = 0; /* Unary operator */ - if( pExpr==0 ){ - op = TK_NULL; - }else{ - op = pExpr->op; - } - switch( op ){ - case TK_AGG_COLUMN: { - sqlite3ExplainPrintf(pOut, "AGG{%d:%d}", - pExpr->iTable, pExpr->iColumn); - break; - } - case TK_COLUMN: { - if( pExpr->iTable<0 ){ - /* This only happens when coding check constraints */ - sqlite3ExplainPrintf(pOut, "COLUMN(%d)", pExpr->iColumn); - }else{ - sqlite3ExplainPrintf(pOut, "{%d:%d}", - pExpr->iTable, pExpr->iColumn); - } - break; - } - case TK_INTEGER: { - if( pExpr->flags & EP_IntValue ){ - sqlite3ExplainPrintf(pOut, "%d", pExpr->u.iValue); - }else{ - sqlite3ExplainPrintf(pOut, "%s", pExpr->u.zToken); - } - break; - } -#ifndef SQLITE_OMIT_FLOATING_POINT - case TK_FLOAT: { - sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); - break; - } -#endif - case TK_STRING: { - sqlite3ExplainPrintf(pOut,"%Q", pExpr->u.zToken); - break; - } - case TK_NULL: { - sqlite3ExplainPrintf(pOut,"NULL"); - break; - } -#ifndef SQLITE_OMIT_BLOB_LITERAL - case TK_BLOB: { - sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); - break; - } -#endif - case TK_VARIABLE: { - sqlite3ExplainPrintf(pOut,"VARIABLE(%s,%d)", - pExpr->u.zToken, pExpr->iColumn); - break; - } - case TK_REGISTER: { - sqlite3ExplainPrintf(pOut,"REGISTER(%d)", pExpr->iTable); - break; - } - case TK_AS: { - sqlite3ExplainExpr(pOut, pExpr->pLeft); - break; - } -#ifndef SQLITE_OMIT_CAST - case TK_CAST: { - /* Expressions of the form: CAST(pLeft AS token) */ - const char *zAff = "unk"; - switch( sqlite3AffinityType(pExpr->u.zToken, 0) ){ - case SQLITE_AFF_TEXT: zAff = "TEXT"; break; - case SQLITE_AFF_NONE: zAff = "NONE"; break; - case SQLITE_AFF_NUMERIC: zAff = "NUMERIC"; break; - case SQLITE_AFF_INTEGER: zAff = "INTEGER"; break; - case SQLITE_AFF_REAL: zAff = "REAL"; break; - } - sqlite3ExplainPrintf(pOut, "CAST-%s(", zAff); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut, ")"); - break; - } -#endif /* SQLITE_OMIT_CAST */ - case TK_LT: zBinOp = "LT"; break; - case TK_LE: zBinOp = "LE"; break; - case TK_GT: zBinOp = "GT"; break; - case TK_GE: zBinOp = "GE"; break; - case TK_NE: zBinOp = "NE"; break; - case TK_EQ: zBinOp = "EQ"; break; - case TK_IS: zBinOp = "IS"; break; - case TK_ISNOT: zBinOp = "ISNOT"; break; - case TK_AND: zBinOp = "AND"; break; - case TK_OR: zBinOp = "OR"; break; - case TK_PLUS: zBinOp = "ADD"; break; - case TK_STAR: zBinOp = "MUL"; break; - case TK_MINUS: zBinOp = "SUB"; break; - case TK_REM: zBinOp = "REM"; break; - case TK_BITAND: zBinOp = "BITAND"; break; - case TK_BITOR: zBinOp = "BITOR"; break; - case TK_SLASH: zBinOp = "DIV"; break; - case TK_LSHIFT: zBinOp = "LSHIFT"; break; - case TK_RSHIFT: zBinOp = "RSHIFT"; break; - case TK_CONCAT: zBinOp = "CONCAT"; break; - - case TK_UMINUS: zUniOp = "UMINUS"; break; - case TK_UPLUS: zUniOp = "UPLUS"; break; - case TK_BITNOT: zUniOp = "BITNOT"; break; - case TK_NOT: zUniOp = "NOT"; break; - case TK_ISNULL: zUniOp = "ISNULL"; break; - case TK_NOTNULL: zUniOp = "NOTNULL"; break; - - case TK_COLLATE: { - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut,".COLLATE(%s)",pExpr->u.zToken); - break; - } - - case TK_AGG_FUNCTION: - case TK_FUNCTION: { - ExprList *pFarg; /* List of function arguments */ - if( ExprHasProperty(pExpr, EP_TokenOnly) ){ - pFarg = 0; - }else{ - pFarg = pExpr->x.pList; - } - if( op==TK_AGG_FUNCTION ){ - sqlite3ExplainPrintf(pOut, "AGG_FUNCTION%d:%s(", - pExpr->op2, pExpr->u.zToken); - }else{ - sqlite3ExplainPrintf(pOut, "FUNCTION:%s(", pExpr->u.zToken); - } - if( pFarg ){ - sqlite3ExplainExprList(pOut, pFarg); - } - sqlite3ExplainPrintf(pOut, ")"); - break; - } -#ifndef SQLITE_OMIT_SUBQUERY - case TK_EXISTS: { - sqlite3ExplainPrintf(pOut, "EXISTS("); - sqlite3ExplainSelect(pOut, pExpr->x.pSelect); - sqlite3ExplainPrintf(pOut,")"); - break; - } - case TK_SELECT: { - sqlite3ExplainPrintf(pOut, "("); - sqlite3ExplainSelect(pOut, pExpr->x.pSelect); - sqlite3ExplainPrintf(pOut, ")"); - break; - } - case TK_IN: { - sqlite3ExplainPrintf(pOut, "IN("); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut, ","); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - sqlite3ExplainSelect(pOut, pExpr->x.pSelect); - }else{ - sqlite3ExplainExprList(pOut, pExpr->x.pList); - } - sqlite3ExplainPrintf(pOut, ")"); - break; - } -#endif /* SQLITE_OMIT_SUBQUERY */ - - /* - ** x BETWEEN y AND z - ** - ** This is equivalent to - ** - ** x>=y AND x<=z - ** - ** X is stored in pExpr->pLeft. - ** Y is stored in pExpr->pList->a[0].pExpr. - ** Z is stored in pExpr->pList->a[1].pExpr. - */ - case TK_BETWEEN: { - Expr *pX = pExpr->pLeft; - Expr *pY = pExpr->x.pList->a[0].pExpr; - Expr *pZ = pExpr->x.pList->a[1].pExpr; - sqlite3ExplainPrintf(pOut, "BETWEEN("); - sqlite3ExplainExpr(pOut, pX); - sqlite3ExplainPrintf(pOut, ","); - sqlite3ExplainExpr(pOut, pY); - sqlite3ExplainPrintf(pOut, ","); - sqlite3ExplainExpr(pOut, pZ); - sqlite3ExplainPrintf(pOut, ")"); - break; - } - case TK_TRIGGER: { - /* If the opcode is TK_TRIGGER, then the expression is a reference - ** to a column in the new.* or old.* pseudo-tables available to - ** trigger programs. In this case Expr.iTable is set to 1 for the - ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn - ** is set to the column of the pseudo-table to read, or to -1 to - ** read the rowid field. - */ - sqlite3ExplainPrintf(pOut, "%s(%d)", - pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); - break; - } - case TK_CASE: { - sqlite3ExplainPrintf(pOut, "CASE("); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut, ","); - sqlite3ExplainExprList(pOut, pExpr->x.pList); - break; - } -#ifndef SQLITE_OMIT_TRIGGER - case TK_RAISE: { - const char *zType = "unk"; - switch( pExpr->affinity ){ - case OE_Rollback: zType = "rollback"; break; - case OE_Abort: zType = "abort"; break; - case OE_Fail: zType = "fail"; break; - case OE_Ignore: zType = "ignore"; break; - } - sqlite3ExplainPrintf(pOut, "RAISE-%s(%s)", zType, pExpr->u.zToken); - break; - } -#endif - } - if( zBinOp ){ - sqlite3ExplainPrintf(pOut,"%s(", zBinOp); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut,","); - sqlite3ExplainExpr(pOut, pExpr->pRight); - sqlite3ExplainPrintf(pOut,")"); - }else if( zUniOp ){ - sqlite3ExplainPrintf(pOut,"%s(", zUniOp); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut,")"); - } -} -#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ - -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) -/* -** Generate a human-readable explanation of an expression list. -*/ -SQLITE_PRIVATE void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ - int i; - if( pList==0 || pList->nExpr==0 ){ - sqlite3ExplainPrintf(pOut, "(empty-list)"); - return; - }else if( pList->nExpr==1 ){ - sqlite3ExplainExpr(pOut, pList->a[0].pExpr); - }else{ - sqlite3ExplainPush(pOut); - for(i=0; inExpr; i++){ - sqlite3ExplainPrintf(pOut, "item[%d] = ", i); - sqlite3ExplainPush(pOut); - sqlite3ExplainExpr(pOut, pList->a[i].pExpr); - sqlite3ExplainPop(pOut); - if( pList->a[i].zName ){ - sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); - } - if( pList->a[i].bSpanIsTab ){ - sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); - } - if( inExpr-1 ){ - sqlite3ExplainNL(pOut); - } - } - sqlite3ExplainPop(pOut); - } -} -#endif /* SQLITE_DEBUG */ - /* ** Generate code that pushes the value of every element of the given ** expression list into a sequence of registers beginning at target. @@ -84650,16 +96876,22 @@ SQLITE_PRIVATE void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ ** ** The SQLITE_ECEL_FACTOR argument allows constant arguments to be ** factored out into initialization code. +** +** The SQLITE_ECEL_REF flag means that expressions in the list with +** ExprList.a[].u.x.iOrderByCol>0 have already been evaluated and stored +** in registers at srcReg, and so the value can be copied from there. */ SQLITE_PRIVATE int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* The expression list to be coded */ int target, /* Where to write results */ + int srcReg, /* Source registers if SQLITE_ECEL_REF */ u8 flags /* SQLITE_ECEL_* flags */ ){ struct ExprList_item *pItem; - int i, n; + int i, j, n; u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; + Vdbe *v = pParse->pVdbe; assert( pList!=0 ); assert( target>0 ); assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ @@ -84667,13 +96899,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; - if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pList->a[i].u.x.iOrderByCol)>0 ){ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); if( inReg!=target+i ){ VdbeOp *pOp; - Vdbe *v = pParse->pVdbe; if( copyOp==OP_Copy && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg @@ -84699,21 +96932,34 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( ** x>=y AND x<=z ** ** Code it as such, taking care to do the common subexpression -** elementation of x. +** elimination of x. +** +** The xJumpIf parameter determines details: +** +** NULL: Store the boolean result in reg[dest] +** sqlite3ExprIfTrue: Jump to dest if true +** sqlite3ExprIfFalse: Jump to dest if false +** +** The jumpIfNull parameter is ignored if xJumpIf is NULL. */ static void exprCodeBetween( Parse *pParse, /* Parsing and code generating context */ Expr *pExpr, /* The BETWEEN expression */ - int dest, /* Jump here if the jump is taken */ - int jumpIfTrue, /* Take the jump if the BETWEEN is true */ + int dest, /* Jump destination or storage location */ + void (*xJump)(Parse*,Expr*,int,int), /* Action to take */ int jumpIfNull /* Take the jump if the BETWEEN is NULL */ ){ - Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ Expr compLeft; /* The x>=y term */ Expr compRight; /* The x<=z term */ Expr exprX; /* The x subexpression */ int regFree1 = 0; /* Temporary use register */ + + memset(&compLeft, 0, sizeof(Expr)); + memset(&compRight, 0, sizeof(Expr)); + memset(&exprAnd, 0, sizeof(Expr)); + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); exprX = *pExpr->pLeft; exprAnd.op = TK_AND; @@ -84725,23 +96971,25 @@ static void exprCodeBetween( compRight.op = TK_LE; compRight.pLeft = &exprX; compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(&exprX, sqlite3ExprCodeTemp(pParse, &exprX, ®Free1)); - if( jumpIfTrue ){ - sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); + exprToRegister(&exprX, exprCodeVector(pParse, &exprX, ®Free1)); + if( xJump ){ + xJump(pParse, &exprAnd, dest, jumpIfNull); }else{ - sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); + exprX.flags |= EP_FromJoin; + sqlite3ExprCodeTarget(pParse, &exprAnd, dest); } sqlite3ReleaseTempReg(pParse, regFree1); /* Ensure adequate test coverage */ - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==0 ); } /* @@ -84793,12 +97041,20 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + op = (op==TK_IS) ? TK_EQ : TK_NE; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); @@ -84808,23 +97064,12 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -84842,7 +97087,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfTrue, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -84850,14 +97095,15 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int int destIfFalse = sqlite3VdbeMakeLabel(v); int destIfNull = jumpIfNull ? dest : destIfFalse; sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); sqlite3VdbeResolveLabel(v, destIfFalse); break; } #endif default: { + default_expr: if( exprAlwaysTrue(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); }else if( exprAlwaysFalse(pExpr) ){ /* No-op */ }else{ @@ -84949,12 +97195,20 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); @@ -84964,23 +97218,12 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -84996,7 +97239,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfFalse, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -85012,8 +97255,9 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int } #endif default: { + default_expr: if( exprAlwaysFalse(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); }else if( exprAlwaysTrue(pExpr) ){ /* no-op */ }else{ @@ -85030,6 +97274,21 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ReleaseTempReg(pParse, regFree2); } +/* +** Like sqlite3ExprIfFalse() except that a copy is made of pExpr before +** code generation, and that copy is deleted after code generation. This +** ensures that the original pExpr is unchanged. +*/ +SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,int jumpIfNull){ + sqlite3 *db = pParse->db; + Expr *pCopy = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed==0 ){ + sqlite3ExprIfFalse(pParse, pCopy, dest, jumpIfNull); + } + sqlite3ExprDelete(db, pCopy); +} + + /* ** Do a deep comparison of two expression trees. Return 0 if the two ** expressions are completely identical. Return 1 if they differ only @@ -85073,8 +97332,10 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ } return 2; } - if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken ){ - if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ + if( pA->op==TK_FUNCTION ){ + if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; + }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return pA->op==TK_COLLATE ? 1 : 2; } } @@ -85084,7 +97345,7 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ if( sqlite3ExprCompare(pA->pLeft, pB->pLeft, iTab) ) return 2; if( sqlite3ExprCompare(pA->pRight, pB->pRight, iTab) ) return 2; if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2; - if( ALWAYS((combinedFlags & EP_Reduced)==0) ){ + if( ALWAYS((combinedFlags & EP_Reduced)==0) && pA->op!=TK_STRING ){ if( pA->iColumn!=pB->iColumn ) return 2; if( pA->iTable!=pB->iTable && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2; @@ -85161,6 +97422,61 @@ SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Expr *pE1, Expr *pE2, int iTab){ return 0; } +/* +** An instance of the following structure is used by the tree walker +** to determine if an expression can be evaluated by reference to the +** index only, without having to do a search for the corresponding +** table entry. The IdxCover.pIdx field is the index. IdxCover.iCur +** is the cursor for the table. +*/ +struct IdxCover { + Index *pIdx; /* The index to be tested for coverage */ + int iCur; /* Cursor number for the table corresponding to the index */ +}; + +/* +** Check to see if there are references to columns in table +** pWalker->u.pIdxCover->iCur can be satisfied using the index +** pWalker->u.pIdxCover->pIdx. +*/ +static int exprIdxCover(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pWalker->u.pIdxCover->iCur + && sqlite3ColumnOfIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Continue; +} + +/* +** Determine if an index pIdx on table with cursor iCur contains will +** the expression pExpr. Return true if the index does cover the +** expression and false if the pExpr expression references table columns +** that are not found in the index pIdx. +** +** An index covering an expression means that the expression can be +** evaluated using only the index and without having to lookup the +** corresponding table entry. +*/ +SQLITE_PRIVATE int sqlite3ExprCoveredByIndex( + Expr *pExpr, /* The index to be tested */ + int iCur, /* The cursor number for the corresponding table */ + Index *pIdx /* The index that might be used for coverage */ +){ + Walker w; + struct IdxCover xcov; + memset(&w, 0, sizeof(w)); + xcov.iCur = iCur; + xcov.pIdx = pIdx; + w.xExprCallback = exprIdxCover; + w.u.pIdxCover = &xcov; + sqlite3WalkExpr(&w, pExpr); + return !w.eCode; +} + + /* ** An instance of the following structure is used by the tree walker ** to count references to table columns in the arguments of an @@ -85186,10 +97502,11 @@ static int exprSrcCount(Walker *pWalker, Expr *pExpr){ int i; struct SrcCount *p = pWalker->u.pSrcCount; SrcList *pSrc = p->pSrc; - for(i=0; inSrc; i++){ + int nSrc = pSrc ? pSrc->nSrc : 0; + for(i=0; iiTable==pSrc->a[i].iCursor ) break; } - if( inSrc ){ + if( inThis++; }else{ p->nOther++; @@ -85357,7 +97674,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ pItem->iMem = ++pParse->nMem; assert( !ExprHasProperty(pExpr, EP_IntValue) ); pItem->pFunc = sqlite3FindFunction(pParse->db, - pExpr->u.zToken, sqlite3Strlen30(pExpr->u.zToken), + pExpr->u.zToken, pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); if( pExpr->flags & EP_Distinct ){ pItem->iDistinct = pParse->nTab++; @@ -85436,14 +97753,14 @@ SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ ** purpose. ** ** If a register is currently being used by the column cache, then -** the dallocation is deferred until the column cache line that uses +** the deallocation is deferred until the column cache line that uses ** the register becomes stale. */ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ if( iReg && pParse->nTempRegaTempReg) ){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iaColCache; inColCache; i++, p++){ if( p->iReg==iReg ){ p->tempReg = 1; return; @@ -85454,10 +97771,11 @@ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ } /* -** Allocate or deallocate a block of nReg consecutive registers +** Allocate or deallocate a block of nReg consecutive registers. */ SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ int i, n; + if( nReg==1 ) return sqlite3GetTempReg(pParse); i = pParse->iRangeReg; n = pParse->nRangeReg; if( nReg<=n ){ @@ -85471,6 +97789,10 @@ SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ return i; } SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, iReg); + return; + } sqlite3ExprCacheRemove(pParse, iReg, nReg); if( nReg>pParse->nRangeReg ){ pParse->nRangeReg = nReg; @@ -85486,6 +97808,29 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nRangeReg = 0; } +/* +** Validate that no temporary register falls within the range of +** iFirst..iLast, inclusive. This routine is only call from within assert() +** statements. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ + int i; + if( pParse->nRangeReg>0 + && pParse->iRangeReg+pParse->nRangeRegiRangeReg>=iFirst + ){ + return 0; + } + for(i=0; inTempReg; i++){ + if( pParse->aTempReg[i]>=iFirst && pParse->aTempReg[i]<=iLast ){ + return 0; + } + } + return 1; +} +#endif /* SQLITE_DEBUG */ + /************** End of expr.c ************************************************/ /************** Begin file alter.c *******************************************/ /* @@ -85502,6 +97847,7 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ ** This file contains C code routines that used to generate VDBE code ** that implements the ALTER TABLE command. */ +/* #include "sqliteInt.h" */ /* ** The code in this file only exists if we are not omitting the @@ -85615,6 +97961,7 @@ static void renameParentFunc( n = sqlite3GetToken(z, &token); }while( token==TK_SPACE ); + if( token==TK_ILLEGAL ) break; zParent = sqlite3DbStrNDup(db, (const char *)z, n); if( zParent==0 ) break; sqlite3Dequote(zParent); @@ -85663,8 +98010,8 @@ static void renameTriggerFunc( UNUSED_PARAMETER(NotUsed); /* The principle used to locate the table name in the CREATE TRIGGER - ** statement is that the table name is the first token that is immediatedly - ** preceded by either TK_ON or TK_DOT and immediatedly followed by one + ** statement is that the table name is the first token that is immediately + ** preceded by either TK_ON or TK_DOT and immediately followed by one ** of TK_WHEN, TK_BEGIN or TK_FOR. */ if( zSql ){ @@ -85717,7 +98064,7 @@ static void renameTriggerFunc( ** Register built-in functions used to help implement ALTER TABLE */ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ - static SQLITE_WSD FuncDef aAlterTableFuncs[] = { + static FuncDef aAlterTableFuncs[] = { FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), #ifndef SQLITE_OMIT_TRIGGER FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), @@ -85726,13 +98073,7 @@ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), #endif }; - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAlterTableFuncs); - - for(i=0; ia[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; db->flags |= SQLITE_PreferBuiltin; /* Get a NULL terminated version of the new table name. */ @@ -85979,7 +98320,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( #ifndef SQLITE_OMIT_VIRTUALTABLE if( pVTab ){ int i = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0); + sqlite3VdbeLoadString(v, i, zName); sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); sqlite3MayAbort(pParse); } @@ -86076,33 +98417,6 @@ exit_rename_table: db->flags = savedDbFlags; } - -/* -** Generate code to make sure the file format number is at least minFormat. -** The generated code will increase the file format number if necessary. -*/ -SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minFormat){ - Vdbe *v; - v = sqlite3GetVdbe(pParse); - /* The VDBE should have been allocated before this routine is called. - ** If that allocation failed, we would have quit before reaching this - ** point */ - if( ALWAYS(v) ){ - int r1 = sqlite3GetTempReg(pParse); - int r2 = sqlite3GetTempReg(pParse); - int j1; - sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); - sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); - j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); - sqlite3VdbeJumpHere(v, j1); - sqlite3ReleaseTempReg(pParse, r1); - sqlite3ReleaseTempReg(pParse, r2); - } -} - /* ** This function is called after an "ALTER TABLE ... ADD" statement ** has been parsed. Argument pColDef contains the text of the new @@ -86121,15 +98435,18 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ Column *pCol; /* The new column */ Expr *pDflt; /* Default value for the new column */ sqlite3 *db; /* The database connection; */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + int r1; /* Temporary registers */ db = pParse->db; if( pParse->nErr || db->mallocFailed ) return; + assert( v!=0 ); pNew = pParse->pNewTable; assert( pNew ); assert( sqlite3BtreeHoldsAllMutexes(db) ); iDb = sqlite3SchemaToIndex(db, pNew->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ pCol = &pNew->aCol[pNew->nCol-1]; pDflt = pCol->pDflt; @@ -86147,7 +98464,8 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** literal NULL, then set pDflt to 0. This simplifies checking ** for an SQL NULL default below. */ - if( pDflt && pDflt->op==TK_NULL ){ + assert( pDflt==0 || pDflt->op==TK_SPAN ); + if( pDflt && pDflt->pLeft->op==TK_NULL ){ pDflt = 0; } @@ -86179,8 +98497,11 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ */ if( pDflt ){ sqlite3_value *pVal = 0; - if( sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){ - db->mallocFailed = 1; + int rc; + rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + if( rc!=SQLITE_OK ){ + assert( db->mallocFailed == 1 ); return; } if( !pVal ){ @@ -86210,11 +98531,18 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ db->flags = savedDbFlags; } - /* If the default value of the new column is NULL, then set the file - ** format to 2. If the default value of the new column is not NULL, - ** the file format becomes 3. + /* Make sure the schema version is at least 3. But do not upgrade + ** from less than 3 to 4, as that will corrupt any preexisting DESC + ** index. */ - sqlite3MinimumFileFormat(pParse, iDb, pDflt ? 3 : 2); + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); + sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); + sqlite3ReleaseTempReg(pParse, r1); /* Reload the schema of the modified table. */ reloadTableSchema(pParse, pTab, pTab->zName); @@ -86288,7 +98616,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); if( !pNew->aCol || !pNew->zName ){ - db->mallocFailed = 1; + assert( db->mallocFailed ); goto exit_begin_add_column; } memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); @@ -86296,9 +98624,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ Column *pCol = &pNew->aCol[i]; pCol->zName = sqlite3DbStrDup(db, pCol->zName); pCol->zColl = 0; - pCol->zType = 0; pCol->pDflt = 0; - pCol->zDflt = 0; } pNew->pSchema = db->aDb[iDb].pSchema; pNew->addColOffset = pTab->addColOffset; @@ -86355,7 +98681,7 @@ exit_begin_add_column: ** not possible to enable both STAT3 and STAT4 at the same time. If they ** are both enabled, then STAT4 takes precedence. ** -** For most applications, sqlite_stat1 provides all the statisics required +** For most applications, sqlite_stat1 provides all the statistics required ** for the query planner to make good choices. ** ** Format of sqlite_stat1: @@ -86460,6 +98786,7 @@ exit_begin_add_column: ** integer in the equivalent columns in sqlite_stat4. */ #ifndef SQLITE_OMIT_ANALYZE +/* #include "sqliteInt.h" */ #if defined(SQLITE_ENABLE_STAT4) # define IsStat4 1 @@ -86529,14 +98856,14 @@ static void openStatTable( for(i=0; izName))==0 ){ + if( (pStat = sqlite3FindTable(db, zTab, pDb->zDbSName))==0 ){ if( aTable[i].zCols ){ /* The sqlite_statN table does not exist. Create it. Note that a ** side-effect of the CREATE TABLE statement is to leave the rootpage ** of the new table in register pParse->regRoot. This is important ** because the OpenWrite opcode below will be needing it. */ sqlite3NestedParse(pParse, - "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols + "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols ); aRoot[i] = pParse->regRoot; aCreateTbl[i] = OPFLAG_P2ISREG; @@ -86551,7 +98878,7 @@ static void openStatTable( if( zWhere ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE %s=%Q", - pDb->zName, zTab, zWhereType, zWhere + pDb->zDbSName, zTab, zWhereType, zWhere ); }else{ /* The sqlite_stat[134] table already exists. Delete all rows. */ @@ -86632,7 +98959,7 @@ static void sampleClear(sqlite3 *db, Stat4Sample *p){ static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); - p->u.aRowid = sqlite3DbMallocRaw(db, n); + p->u.aRowid = sqlite3DbMallocRawNN(db, n); if( p->u.aRowid ){ p->nRowid = n; memcpy(p->u.aRowid, pData, n); @@ -86706,8 +99033,9 @@ static void stat4Destructor(void *pOld){ ** original WITHOUT ROWID table as N==K as a special case. ** ** This routine allocates the Stat4Accum object in heap memory. The return -** value is a pointer to the the Stat4Accum object encoded as a blob (i.e. -** the size of the blob is sizeof(void*) bytes). +** value is a pointer to the Stat4Accum object. The datatype of the +** return value is BLOB, but it is really just a pointer to the Stat4Accum +** object. */ static void statInit( sqlite3_context *context, @@ -86766,7 +99094,7 @@ static void statInit( p->mxSample = mxSample; p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); p->current.anLt = &p->current.anEq[nColUp]; - p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[2])*0xd0944565; + p->iPrn = 0x689e962d*(u32)nCol ^ 0xd0944565*(u32)sqlite3_value_int(argv[2]); /* Set up the Stat4Accum.a[] and aBest[] arrays */ p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; @@ -86785,20 +99113,21 @@ static void statInit( } #endif - /* Return a pointer to the allocated object to the caller */ - sqlite3_result_blob(context, p, sizeof(p), stat4Destructor); + /* Return a pointer to the allocated object to the caller. Note that + ** only the pointer (the 2nd parameter) matters. The size of the object + ** (given by the 3rd parameter) is never used and can be any positive + ** value. */ + sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor); } static const FuncDef statInitFuncdef = { 2+IsStat34, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statInit, /* xFunc */ - 0, /* xStep */ + statInit, /* xSFunc */ 0, /* xFinalize */ "stat_init", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; #ifdef SQLITE_ENABLE_STAT4 @@ -87094,12 +99423,10 @@ static const FuncDef statPushFuncdef = { SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statPush, /* xFunc */ - 0, /* xStep */ + statPush, /* xSFunc */ 0, /* xFinalize */ "stat_push", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; #define STAT_GET_STAT1 0 /* "stat" column of stat1 table */ @@ -87112,7 +99439,7 @@ static const FuncDef statPushFuncdef = { ** Implementation of the stat_get(P,J) SQL function. This routine is ** used to query statistical information that has been gathered into ** the Stat4Accum object by prior calls to stat_push(). The P parameter -** is a BLOB which is decoded into a pointer to the Stat4Accum objects. +** has type BLOB but it is really just a pointer to the Stat4Accum object. ** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** @@ -87241,12 +99568,10 @@ static const FuncDef statGetFuncdef = { SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statGet, /* xFunc */ - 0, /* xStep */ + statGet, /* xSFunc */ 0, /* xFinalize */ "stat_get", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ @@ -87258,8 +99583,8 @@ static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ #else UNUSED_PARAMETER( iParam ); #endif - sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regOut); - sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4, regOut, + (char*)&statGetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 1 + IsStat34); } @@ -87305,7 +99630,7 @@ static void analyzeOneTable( /* Do not gather statistics on views or virtual tables */ return; } - if( sqlite3_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3_strlike("sqlite_%", pTab->zName, 0)==0 ){ /* Do not gather statistics on system tables */ return; } @@ -87315,7 +99640,7 @@ static void analyzeOneTable( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, - db->aDb[iDb].zName ) ){ + db->aDb[iDb].zDbSName ) ){ return; } #endif @@ -87329,7 +99654,7 @@ static void analyzeOneTable( iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); + sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ @@ -87351,7 +99676,7 @@ static void analyzeOneTable( } /* Populate the register containing the index name. */ - sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); + sqlite3VdbeLoadString(v, regIdxname, zIdxName); VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); /* @@ -87413,8 +99738,8 @@ static void analyzeOneTable( #endif sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); - sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); - sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4+1, regStat4, + (char*)&statInitFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); /* Implementation of the following: @@ -87433,7 +99758,7 @@ static void analyzeOneTable( if( nColTest>0 ){ int endDistinctTest = sqlite3VdbeMakeLabel(v); int *aGotoChng; /* Array of jump instruction addresses */ - aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); + aGotoChng = sqlite3DbMallocRawNN(db, sizeof(int)*nColTest); if( aGotoChng==0 ) continue; /* @@ -87465,7 +99790,7 @@ static void analyzeOneTable( VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); + sqlite3VdbeGoto(v, endDistinctTest); /* @@ -87501,6 +99826,7 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knCol ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } @@ -87509,14 +99835,15 @@ static void analyzeOneTable( } #endif assert( regChng==(regStat4+1) ); - sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); - sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_Function0, 1, regStat4, regTemp, + (char*)&statPushFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); /* Add the entry to the stat1 table. */ callStatGet(v, regStat4, STAT_GET_STAT1, regStat1); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -87549,12 +99876,10 @@ static void analyzeOneTable( ** be taken */ VdbeCoverageNeverTaken(v); #ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, - pIdx->aiColumn[0], regSample); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); #else for(i=0; iaiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, i, regCol+i); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif @@ -87579,7 +99904,8 @@ static void analyzeOneTable( sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -87704,7 +100030,7 @@ SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ /* Form 3: Analyze the fully qualified table name */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; z = sqlite3NameFromToken(db, pTableName); if( z ){ if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ @@ -87750,7 +100076,7 @@ static void decodeIntArray( #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( z==0 ) z = ""; #else - if( NEVER(z==0) ) z = ""; + assert( z!=0 ); #endif for(i=0; *z && ibUnordered = 1; - }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); - } + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; + } #ifdef SQLITE_ENABLE_COSTMULT - else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ - pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); - } + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } #endif - while( z[0]!=0 && z[0]!=' ' ) z++; - while( z[0]==' ' ) z++; + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; + } } } @@ -87829,8 +100158,20 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ + tRowcnt *aiRowEst = 0; + int nCol = pIndex->nKeyCol+1; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + /* Index.aiRowEst may already be set here if there are duplicate + ** sqlite_stat1 entries for this index. In that case just clobber + ** the old data with the new instead of allocating a new array. */ + if( pIndex->aiRowEst==0 ){ + pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero(sizeof(tRowcnt) * nCol); + if( pIndex->aiRowEst==0 ) sqlite3OomFault(pInfo->db); + } + aiRowEst = pIndex->aiRowEst; +#endif pIndex->bUnordered = 0; - decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ Index fakeIdx; @@ -87889,25 +100230,39 @@ static void initAvgEq(Index *pIdx){ pIdx->aAvgEq[nCol] = 1; } for(iCol=0; iColnSample; int i; /* Used to iterate through samples */ tRowcnt sumEq = 0; /* Sum of the nEq values */ - tRowcnt nSum = 0; /* Number of terms contributing to sumEq */ tRowcnt avgEq = 0; - tRowcnt nDLt = pFinal->anDLt[iCol]; + tRowcnt nRow; /* Number of rows in index */ + i64 nSum100 = 0; /* Number of terms contributing to sumEq */ + i64 nDist100; /* Number of distinct values in index */ + + if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){ + nRow = pFinal->anLt[iCol]; + nDist100 = (i64)100 * pFinal->anDLt[iCol]; + nSample--; + }else{ + nRow = pIdx->aiRowEst[0]; + nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; + } + pIdx->nRowEst0 = nRow; /* Set nSum to the number of distinct (iCol+1) field prefixes that - ** occur in the stat4 table for this index before pFinal. Set - ** sumEq to the sum of the nEq values for column iCol for the same - ** set (adding the value only once where there exist dupicate - ** prefixes). */ - for(i=0; i<(pIdx->nSample-1); i++){ - if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){ + ** occur in the stat4 table for this index. Set sumEq to the sum of + ** the nEq values for column iCol for the same set (adding the value + ** only once where there exist duplicate prefixes). */ + for(i=0; inSample-1) + || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] + ){ sumEq += aSample[i].anEq[iCol]; - nSum++; + nSum100 += 100; } } - if( nDLt>nSum ){ - avgEq = (pFinal->anLt[iCol] - sumEq)/(nDLt - nSum); + + if( nDist100>nSum100 ){ + avgEq = ((i64)100 * (nRow - sumEq))/(nDist100 - nSum100); } if( avgEq==0 ) avgEq = 1; pIdx->aAvgEq[iCol] = avgEq; @@ -87958,10 +100313,10 @@ static int loadStatTbl( Index *pPrevIdx = 0; /* Previous index in the loop */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ - assert( db->lookaside.bEnabled==0 ); + assert( db->lookaside.bDisable ); zSql = sqlite3MPrintf(db, zSql1, zDb); if( !zSql ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); sqlite3DbFree(db, zSql); @@ -88001,7 +100356,7 @@ static int loadStatTbl( pIdx->aSample = sqlite3DbMallocZero(db, nByte); if( pIdx->aSample==0 ){ sqlite3_finalize(pStmt); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; pIdx->aAvgEq = pSpace; pSpace += nIdxCol; @@ -88017,7 +100372,7 @@ static int loadStatTbl( zSql = sqlite3MPrintf(db, zSql2, zDb); if( !zSql ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); sqlite3DbFree(db, zSql); @@ -88055,7 +100410,7 @@ static int loadStatTbl( pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); if( pSample->p==0 ){ sqlite3_finalize(pStmt); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); pIdx->nSample++; @@ -88072,7 +100427,7 @@ static int loadStatTbl( static int loadStat4(sqlite3 *db, const char *zDb){ int rc = SQLITE_OK; /* Result codes from subroutines */ - assert( db->lookaside.bEnabled==0 ); + assert( db->lookaside.bDisable ); if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ rc = loadStatTbl(db, 0, "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", @@ -88117,7 +100472,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ analysisInfo sInfo; HashElem *i; char *zSql; - int rc; + int rc = SQLITE_OK; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); @@ -88126,43 +100481,50 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - sqlite3DefaultRowEst(pIdx); + pIdx->aiRowLogEst[0] = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif } - /* Check to make sure the sqlite_stat1 table exists */ - sInfo.db = db; - sInfo.zDatabase = db->aDb[iDb].zName; - if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)==0 ){ - return SQLITE_ERROR; - } - /* Load new statistics out of the sqlite_stat1 table */ - zSql = sqlite3MPrintf(db, - "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); - sqlite3DbFree(db, zSql); + sInfo.db = db; + sInfo.zDatabase = db->aDb[iDb].zDbSName; + if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)!=0 ){ + zSql = sqlite3MPrintf(db, + "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + if( zSql==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); + sqlite3DbFree(db, zSql); + } } + /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + if( pIdx->aiRowLogEst[0]==0 ) sqlite3DefaultRowEst(pIdx); + } /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK ){ - int lookasideEnabled = db->lookaside.bEnabled; - db->lookaside.bEnabled = 0; + if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ + db->lookaside.bDisable++; rc = loadStat4(db, sInfo.zDatabase); - db->lookaside.bEnabled = lookasideEnabled; + db->lookaside.bDisable--; + } + for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3_free(pIdx->aiRowEst); + pIdx->aiRowEst = 0; } #endif if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } return rc; } @@ -88185,6 +100547,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ ************************************************************************* ** This file contains code used to implement the ATTACH and DETACH commands. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_ATTACH /* @@ -88270,7 +100633,7 @@ static void attachFunc( goto attach_error; } for(i=0; inDb; i++){ - char *z = db->aDb[i].zName; + char *z = db->aDb[i].zDbSName; assert( z && zName ); if( sqlite3StrICmp(z, zName)==0 ){ zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); @@ -88282,7 +100645,7 @@ static void attachFunc( ** hash tables. */ if( db->aDb==db->aDbStatic ){ - aNew = sqlite3DbMallocRaw(db, sizeof(db->aDb[0])*3 ); + aNew = sqlite3DbMallocRawNN(db, sizeof(db->aDb[0])*3 ); if( aNew==0 ) return; memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); }else{ @@ -88300,7 +100663,7 @@ static void attachFunc( flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); return; @@ -88317,24 +100680,27 @@ static void attachFunc( Pager *pPager; aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt); if( !aNew->pSchema ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){ zErrDyn = sqlite3MPrintf(db, "attached databases must use the same text encoding as main database"); rc = SQLITE_ERROR; } + sqlite3BtreeEnter(aNew->pBt); pPager = sqlite3BtreePager(aNew->pBt); sqlite3PagerLockingMode(pPager, db->dfltLockMode); sqlite3BtreeSecureDelete(aNew->pBt, sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); #ifndef SQLITE_OMIT_PAGER_PRAGMAS - sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK)); + sqlite3BtreeSetPagerFlags(aNew->pBt, + PAGER_SYNCHRONOUS_FULL | (db->flags & PAGER_FLAGS_MASK)); #endif + sqlite3BtreeLeave(aNew->pBt); } - aNew->safety_level = 3; - aNew->zName = sqlite3DbStrDup(db, zName); - if( rc==SQLITE_OK && aNew->zName==0 ){ - rc = SQLITE_NOMEM; + aNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + aNew->zDbSName = sqlite3DbStrDup(db, zName); + if( rc==SQLITE_OK && aNew->zDbSName==0 ){ + rc = SQLITE_NOMEM_BKPT; } @@ -88362,7 +100728,7 @@ static void attachFunc( case SQLITE_NULL: /* No key specified. Use the key from the main database */ sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - if( nKey>0 || sqlite3BtreeGetReserve(db->aDb[0].pBt)>0 ){ + if( nKey || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); } break; @@ -88380,6 +100746,15 @@ static void attachFunc( rc = sqlite3Init(db, &zErrDyn); sqlite3BtreeLeaveAll(db); } +#ifdef SQLITE_USER_AUTHENTICATION + if( rc==SQLITE_OK ){ + u8 newAuth = 0; + rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); + if( newAuthauth.authLevel ){ + rc = SQLITE_AUTH_USER; + } + } +#endif if( rc ){ int iDb = db->nDb - 1; assert( iDb>=2 ); @@ -88391,7 +100766,7 @@ static void attachFunc( sqlite3ResetAllSchemasOfConnection(db); db->nDb = iDb; if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, zErrDyn); zErrDyn = sqlite3MPrintf(db, "out of memory"); }else if( zErrDyn==0 ){ @@ -88436,7 +100811,7 @@ static void detachFunc( for(i=0; inDb; i++){ pDb = &db->aDb[i]; if( pDb->pBt==0 ) continue; - if( sqlite3StrICmp(pDb->zName, zName)==0 ) break; + if( sqlite3StrICmp(pDb->zDbSName, zName)==0 ) break; } if( i>=db->nDb ){ @@ -88460,7 +100835,7 @@ static void detachFunc( sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; pDb->pSchema = 0; - sqlite3ResetAllSchemasOfConnection(db); + sqlite3CollapseDatabaseArray(db); return; detach_error: @@ -88486,6 +100861,7 @@ static void codeAttach( sqlite3* db = pParse->db; int regArgs; + if( pParse->nErr ) goto attach_end; memset(&sName, 0, sizeof(NameContext)); sName.pParse = pParse; @@ -88494,7 +100870,6 @@ static void codeAttach( SQLITE_OK!=(rc = resolveAttachExpr(&sName, pDbname)) || SQLITE_OK!=(rc = resolveAttachExpr(&sName, pKey)) ){ - pParse->nErr++; goto attach_end; } @@ -88522,11 +100897,11 @@ static void codeAttach( assert( v || db->mallocFailed ); if( v ){ - sqlite3VdbeAddOp3(v, OP_Function, 0, regArgs+3-pFunc->nArg, regArgs+3); + sqlite3VdbeAddOp4(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3, + (char *)pFunc, P4_FUNCDEF); assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); - sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); - + /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this ** statement only). For DETACH, set it to false (expire all existing ** statements). @@ -88551,12 +100926,10 @@ SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - detachFunc, /* xFunc */ - 0, /* xStep */ + detachFunc, /* xSFunc */ 0, /* xFinalize */ "sqlite_detach", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); } @@ -88572,12 +100945,10 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - attachFunc, /* xFunc */ - 0, /* xStep */ + attachFunc, /* xSFunc */ 0, /* xFinalize */ "sqlite_attach", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); } @@ -88599,7 +100970,7 @@ SQLITE_PRIVATE void sqlite3FixInit( db = pParse->db; assert( db->nDb>iDb ); pFix->pParse = pParse; - pFix->zDb = db->aDb[iDb].zName; + pFix->zDb = db->aDb[iDb].zDbSName; pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; @@ -88696,7 +101067,7 @@ SQLITE_PRIVATE int sqlite3FixExpr( return 1; } } - if( ExprHasProperty(pExpr, EP_TokenOnly) ) break; + if( ExprHasProperty(pExpr, EP_TokenOnly|EP_Leaf) ) break; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; }else{ @@ -88764,6 +101135,7 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep( ** systems that do not need this facility may omit it by recompiling ** the library with -DSQLITE_OMIT_AUTHORIZATION=1 */ +/* #include "sqliteInt.h" */ /* ** All of the code in this file may be omitted by defining a single @@ -88821,8 +101193,11 @@ SQLITE_API int sqlite3_set_authorizer( int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); - db->xAuth = xAuth; + db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; sqlite3ExpirePreparedStatements(db); sqlite3_mutex_leave(db->mutex); @@ -88853,11 +101228,16 @@ SQLITE_PRIVATE int sqlite3AuthReadCol( const char *zCol, /* Column name */ int iDb /* Index of containing database. */ ){ - sqlite3 *db = pParse->db; /* Database handle */ - char *zDb = db->aDb[iDb].zName; /* Name of attached database */ - int rc; /* Auth callback return code */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zDb = db->aDb[iDb].zDbSName; /* Schema name of attached database */ + int rc; /* Auth callback return code */ - rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext); + if( db->init.busy ) return SQLITE_OK; + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); if( rc==SQLITE_DENY ){ if( db->nDb>2 || iDb!=0 ){ sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol); @@ -88957,7 +101337,11 @@ SQLITE_PRIVATE int sqlite3AuthCheck( if( db->xAuth==0 ){ return SQLITE_OK; } - rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext); + rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); if( rc==SQLITE_DENY ){ sqlite3ErrorMsg(pParse, "not authorized"); pParse->rc = SQLITE_AUTH; @@ -89023,15 +101407,7 @@ SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ ** COMMIT ** ROLLBACK */ - -/* -** This routine is called when a new SQL statement is beginning to -** be parsed. Initialize the pParse structure as needed. -*/ -SQLITE_PRIVATE void sqlite3BeginParse(Parse *pParse, int explainFlag){ - pParse->explain = (u8)explainFlag; - pParse->nVar = 0; -} +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_SHARED_CACHE /* @@ -89087,7 +101463,7 @@ SQLITE_PRIVATE void sqlite3TableLock( p->zName = zName; }else{ pToplevel->nTableLock = 0; - pToplevel->db->mallocFailed = 1; + sqlite3OomFault(pToplevel->db); } } @@ -89142,9 +101518,11 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ assert( pParse->pToplevel==0 ); db = pParse->db; - if( db->mallocFailed ) return; if( pParse->nested ) return; - if( pParse->nErr ) return; + if( db->mallocFailed || pParse->nErr ){ + if( pParse->rc==SQLITE_OK ) pParse->rc = SQLITE_ERROR; + return; + } /* Begin by generating some termination code at the end of the ** vdbe program @@ -89153,9 +101531,19 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ - while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} sqlite3VdbeAddOp0(v, OP_Halt); +#if SQLITE_USER_AUTHENTICATION + if( pParse->nTableLock>0 && db->init.busy==0 ){ + sqlite3UserAuthInit(db); + if( db->auth.authLevelrc = SQLITE_AUTH_USER; + return; + } + } +#endif + /* The cookie mask contains one bit for each database file open. ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are ** set for each database that is used. Generate code to start a @@ -89169,16 +101557,20 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); sqlite3VdbeJumpHere(v, 0); for(iDb=0; iDbnDb; iDb++){ + Schema *pSchema; if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; sqlite3VdbeAddOp4Int(v, OP_Transaction, /* Opcode */ iDb, /* P1 */ DbMaskTest(pParse->writeMask,iDb), /* P2 */ - pParse->cookieValue[iDb], /* P3 */ - db->aDb[iDb].pSchema->iGeneration /* P4 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ ); if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); } #ifndef SQLITE_OMIT_VIRTUALTABLE for(i=0; inVtabLock; i++){ @@ -89208,29 +101600,23 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, 1); + sqlite3VdbeGoto(v, 1); } } /* Get the VDBE program ready for execution */ - if( v && ALWAYS(pParse->nErr==0) && !db->mallocFailed ){ + if( v && pParse->nErr==0 && !db->mallocFailed ){ assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; - pParse->colNamesSet = 0; }else{ pParse->rc = SQLITE_ERROR; } - pParse->nTab = 0; - pParse->nMem = 0; - pParse->nSet = 0; - pParse->nVar = 0; - DbMaskZero(pParse->cookieMask); } /* @@ -89250,8 +101636,7 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ char *zSql; char *zErrMsg = 0; sqlite3 *db = pParse->db; -# define SAVE_SZ (sizeof(Parse) - offsetof(Parse,nVar)) - char saveBuf[SAVE_SZ]; + char saveBuf[PARSE_TAIL_SZ]; if( pParse->nErr ) return; assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ @@ -89262,15 +101647,25 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ return; /* A malloc must have failed */ } pParse->nested++; - memcpy(saveBuf, &pParse->nVar, SAVE_SZ); - memset(&pParse->nVar, 0, SAVE_SZ); + memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); + memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); sqlite3RunParser(pParse, zSql, &zErrMsg); sqlite3DbFree(db, zErrMsg); sqlite3DbFree(db, zSql); - memcpy(&pParse->nVar, saveBuf, SAVE_SZ); + memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); pParse->nested--; } +#if SQLITE_USER_AUTHENTICATION +/* +** Return TRUE if zTable is the name of the system table that stores the +** list of users and their access credentials. +*/ +SQLITE_PRIVATE int sqlite3UserAuthTable(const char *zTable){ + return sqlite3_stricmp(zTable, "sqlite_user")==0; +} +#endif + /* ** Locate the in-memory structure that describes a particular database ** table given the name of that table and (optionally) the name of the @@ -89286,17 +101681,23 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ Table *p = 0; int i; - int nName; - assert( zName!=0 ); - nName = sqlite3Strlen30(zName); + /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); +#if SQLITE_USER_AUTHENTICATION + /* Only the admin user is allowed to know that the sqlite_user table + ** exists */ + if( db->auth.authLevelnDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; - assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); - if( p ) break; + if( zDatabase==0 || sqlite3StrICmp(zDatabase, db->aDb[j].zDbSName)==0 ){ + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); + if( p ) break; + } } return p; } @@ -89313,7 +101714,7 @@ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const cha */ SQLITE_PRIVATE Table *sqlite3LocateTable( Parse *pParse, /* context in which to report errors */ - int isView, /* True if looking for a VIEW rather than a TABLE */ + u32 flags, /* LOCATE_VIEW or LOCATE_NOERR */ const char *zName, /* Name of the table we are looking for */ const char *zDbase /* Name of the database. Might be NULL */ ){ @@ -89327,14 +101728,28 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( p = sqlite3FindTable(pParse->db, zName, zDbase); if( p==0 ){ - const char *zMsg = isView ? "no such view" : "no such table"; - if( zDbase ){ - sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); - }else{ - sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table"; +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); + if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + return pMod->pEpoTab; + } + } +#endif + if( (flags & LOCATE_NOERR)==0 ){ + if( zDbase ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + } + pParse->checkSchema = 1; } - pParse->checkSchema = 1; } + return p; } @@ -89349,18 +101764,18 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( */ SQLITE_PRIVATE Table *sqlite3LocateTableItem( Parse *pParse, - int isView, + u32 flags, struct SrcList_item *p ){ const char *zDb; assert( p->pSchema==0 || p->zDatabase==0 ); if( p->pSchema ){ int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); - zDb = pParse->db->aDb[iDb].zName; + zDb = pParse->db->aDb[iDb].zDbSName; }else{ zDb = p->zDatabase; } - return sqlite3LocateTable(pParse, isView, p->zName, zDb); + return sqlite3LocateTable(pParse, flags, p->zName, zDb); } /* @@ -89378,16 +101793,15 @@ SQLITE_PRIVATE Table *sqlite3LocateTableItem( SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ Index *p = 0; int i; - int nName = sqlite3Strlen30(zName); /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ Schema *pSchema = db->aDb[j].pSchema; assert( pSchema ); - if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; + if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zDbSName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&pSchema->idxHash, zName, nName); + p = sqlite3HashFind(&pSchema->idxHash, zName); if( p ) break; } return p; @@ -89400,10 +101814,13 @@ static void freeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE sqlite3DeleteIndexSamples(db, p); #endif - if( db==0 || db->pnBytesFreed==0 ) sqlite3KeyInfoUnref(p->pKeyInfo); sqlite3ExprDelete(db, p->pPartIdxWhere); + sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); - if( p->isResized ) sqlite3DbFree(db, p->azColl); + if( p->isResized ) sqlite3DbFree(db, (void *)p->azColl); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3_free(p->aiRowEst); +#endif sqlite3DbFree(db, p); } @@ -89415,13 +101832,11 @@ static void freeIndex(sqlite3 *db, Index *p){ */ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ Index *pIndex; - int len; Hash *pHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pHash = &db->aDb[iDb].pSchema->idxHash; - len = sqlite3Strlen30(zIdxName); - pIndex = sqlite3HashInsert(pHash, zIdxName, len, 0); + pIndex = sqlite3HashInsert(pHash, zIdxName, 0); if( ALWAYS(pIndex) ){ if( pIndex->pTable->pIndex==pIndex ){ pIndex->pTable->pIndex = pIndex->pNext; @@ -89453,8 +101868,8 @@ SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3 *db){ for(i=j=2; inDb; i++){ struct Db *pDb = &db->aDb[i]; if( pDb->pBt==0 ){ - sqlite3DbFree(db, pDb->zName); - pDb->zName = 0; + sqlite3DbFree(db, pDb->zDbSName); + pDb->zDbSName = 0; continue; } if( jaDb[j], 0, (db->nDb-j)*sizeof(db->aDb[j])); db->nDb = j; if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); @@ -89527,7 +101941,7 @@ SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ ** Delete memory allocated for the column names of a table or view (the ** Table.aCol[] array). */ -static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); @@ -89535,8 +101949,6 @@ static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ for(i=0; inCol; i++, pCol++){ sqlite3DbFree(db, pCol->zName); sqlite3ExprDelete(db, pCol->pDflt); - sqlite3DbFree(db, pCol->zDflt); - sqlite3DbFree(db, pCol->zType); sqlite3DbFree(db, pCol->zColl); } sqlite3DbFree(db, pTable->aCol); @@ -89558,16 +101970,10 @@ static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ ** db parameter can be used with db->pnBytesFreed to measure the memory ** used by the Table object. */ -SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ +static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */ - assert( !pTable || pTable->nRef>0 ); - - /* Do not delete the table until the reference count reaches zero. */ - if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; - /* Record the number of outstanding lookaside allocations in schema Tables ** prior to doing any free() operations. Since schema Tables do not use ** lookaside, this number should not change. */ @@ -89577,11 +101983,12 @@ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; - assert( pIndex->pSchema==pTable->pSchema ); - if( !db || db->pnBytesFreed==0 ){ + assert( pIndex->pSchema==pTable->pSchema + || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); + if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( - &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + &pIndex->pSchema->idxHash, zName, 0 ); assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); assert( pOld==pIndex || pOld==0 ); @@ -89594,13 +102001,11 @@ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Delete the Table structure itself. */ - sqliteDeleteColumnNames(db, pTable); + sqlite3DeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); -#ifndef SQLITE_OMIT_CHECK sqlite3ExprListDelete(db, pTable->pCheck); -#endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); #endif @@ -89609,6 +102014,13 @@ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Verify that no lookaside memory was used by schema tables */ assert( nLookaside==0 || nLookaside==db->lookaside.nOut ); } +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ + /* Do not delete the table until the reference count reaches zero. */ + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; + deleteTable(db, pTable); +} + /* ** Unlink the given table from the hash tables and the delete the @@ -89624,8 +102036,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ pDb = &db->aDb[iDb]; - p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, - sqlite3Strlen30(zTabName),0); + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); sqlite3DeleteTable(db, p); db->flags |= SQLITE_InternChanges; } @@ -89677,12 +102088,8 @@ SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *db, const char *zName){ int i = -1; /* Database number */ if( zName ){ Db *pDb; - int n = sqlite3Strlen30(zName); for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ - if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite3Strlen30(pDb->zName) && - 0==sqlite3StrICmp(pDb->zName, zName) ){ - break; - } + if( 0==sqlite3StrICmp(pDb->zDbSName, zName) ) break; } } return i; @@ -89728,21 +102135,20 @@ SQLITE_PRIVATE int sqlite3TwoPartName( int iDb; /* Database holding the object */ sqlite3 *db = pParse->db; - if( ALWAYS(pName2!=0) && pName2->n>0 ){ + assert( pName2!=0 ); + if( pName2->n>0 ){ if( db->init.busy ) { sqlite3ErrorMsg(pParse, "corrupt database"); - pParse->nErr++; return -1; } *pUnqual = pName2; iDb = sqlite3FindDb(db, pName1); if( iDb<0 ){ sqlite3ErrorMsg(pParse, "unknown database %T", pName1); - pParse->nErr++; return -1; } }else{ - assert( db->init.iDb==0 || db->init.busy ); + assert( db->init.iDb==0 || db->init.busy || (db->flags & SQLITE_Vacuum)!=0); iDb = db->init.iDb; *pUnqual = pName1; } @@ -89819,62 +102225,46 @@ SQLITE_PRIVATE void sqlite3StartTable( int iDb; /* Database number to create the table in */ Token *pName; /* Unqualified name of the table to create */ - /* The table or view name to create is passed to this routine via tokens - ** pName1 and pName2. If the table name was fully qualified, for example: - ** - ** CREATE TABLE xxx.yyy (...); - ** - ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if - ** the table name is not fully qualified, i.e.: - ** - ** CREATE TABLE yyy(...); - ** - ** Then pName1 is set to "yyy" and pName2 is "". - ** - ** The call below sets the pName pointer to point at the token (pName1 or - ** pName2) that stores the unqualified table name. The variable iDb is - ** set to the index of the database that the table or view is to be - ** created in. - */ - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); - if( iDb<0 ) return; - if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ - /* If creating a temp table, the name may not be qualified. Unless - ** the database name is "temp" anyway. */ - sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); - return; + if( db->init.busy && db->init.newTnum==1 ){ + /* Special case: Parsing the sqlite_master or sqlite_temp_master schema */ + iDb = db->init.iDb; + zName = sqlite3DbStrDup(db, SCHEMA_TABLE(iDb)); + pName = pName1; + }else{ + /* The common case */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) return; + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ + sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); + return; + } + if( !OMIT_TEMPDB && isTemp ) iDb = 1; + zName = sqlite3NameFromToken(db, pName); } - if( !OMIT_TEMPDB && isTemp ) iDb = 1; - pParse->sNameToken = *pName; - zName = sqlite3NameFromToken(db, pName); if( zName==0 ) return; if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto begin_table_error; } if( db->init.iDb==1 ) isTemp = 1; #ifndef SQLITE_OMIT_AUTHORIZATION - assert( (isTemp & 1)==isTemp ); + assert( isTemp==0 || isTemp==1 ); + assert( isView==0 || isView==1 ); { - int code; - char *zDb = db->aDb[iDb].zName; + static const u8 aCode[] = { + SQLITE_CREATE_TABLE, + SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_VIEW, + SQLITE_CREATE_TEMP_VIEW + }; + char *zDb = db->aDb[iDb].zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ goto begin_table_error; } - if( isView ){ - if( !OMIT_TEMPDB && isTemp ){ - code = SQLITE_CREATE_TEMP_VIEW; - }else{ - code = SQLITE_CREATE_VIEW; - } - }else{ - if( !OMIT_TEMPDB && isTemp ){ - code = SQLITE_CREATE_TEMP_TABLE; - }else{ - code = SQLITE_CREATE_TABLE; - } - } - if( !isVirtual && sqlite3AuthCheck(pParse, code, zName, 0, zDb) ){ + if( !isVirtual && sqlite3AuthCheck(pParse, (int)aCode[isTemp+2*isView], + zName, 0, zDb) ){ goto begin_table_error; } } @@ -89888,7 +102278,7 @@ SQLITE_PRIVATE void sqlite3StartTable( ** collisions. */ if( !IN_DECLARE_VTAB ){ - char *zDb = db->aDb[iDb].zName; + char *zDb = db->aDb[iDb].zDbSName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; } @@ -89897,7 +102287,7 @@ SQLITE_PRIVATE void sqlite3StartTable( if( !noErr ){ sqlite3ErrorMsg(pParse, "table %T already exists", pName); }else{ - assert( !db->init.busy ); + assert( !db->init.busy || CORRUPT_DB ); sqlite3CodeVerifySchema(pParse, iDb); } goto begin_table_error; @@ -89910,8 +102300,8 @@ SQLITE_PRIVATE void sqlite3StartTable( pTable = sqlite3DbMallocZero(db, sizeof(Table)); if( pTable==0 ){ - db->mallocFailed = 1; - pParse->rc = SQLITE_NOMEM; + assert( db->mallocFailed ); + pParse->rc = SQLITE_NOMEM_BKPT; pParse->nErr++; goto begin_table_error; } @@ -89943,10 +102333,12 @@ SQLITE_PRIVATE void sqlite3StartTable( ** now. */ if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ - int j1; + int addr1; int fileFormat; int reg1, reg2, reg3; - sqlite3BeginWriteOperation(pParse, 0, iDb); + /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ + static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; + sqlite3BeginWriteOperation(pParse, 1, iDb); #ifndef SQLITE_OMIT_VIRTUALTABLE if( isVirtual ){ @@ -89962,14 +102354,12 @@ SQLITE_PRIVATE void sqlite3StartTable( reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); - j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? 1 : SQLITE_MAX_FILE_FORMAT; - sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, reg3); - sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, reg3); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, fileFormat); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, ENC(db)); + sqlite3VdbeJumpHere(v, addr1); /* This just creates a place-holder record in the sqlite_master table. ** The record created does not contain anything yet. It will be replaced @@ -89990,7 +102380,7 @@ SQLITE_PRIVATE void sqlite3StartTable( } sqlite3OpenMasterTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); - sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); + sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); @@ -90005,18 +102395,19 @@ begin_table_error: return; } -/* -** This macro is used to compare two strings in a case-insensitive manner. -** It is slightly faster than calling sqlite3StrICmp() directly, but -** produces larger code. -** -** WARNING: This macro is not compatible with the strcmp() family. It -** returns true if the two strings are equal, otherwise false. +/* Set properties of a table column based on the (magical) +** name of the column. */ -#define STRICMP(x, y) (\ -sqlite3UpperToLower[*(unsigned char *)(x)]== \ -sqlite3UpperToLower[*(unsigned char *)(y)] \ -&& sqlite3StrICmp((x)+1,(y)+1)==0 ) +#if SQLITE_ENABLE_HIDDEN_COLUMNS +SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ + if( sqlite3_strnicmp(pCol->zName, "__hidden__", 10)==0 ){ + pCol->colFlags |= COLFLAG_HIDDEN; + }else if( pTab && pCol!=pTab->aCol && (pCol[-1].colFlags & COLFLAG_HIDDEN) ){ + pTab->tabFlags |= TF_OOOHidden; + } +} +#endif + /* ** Add a new column to the table currently being constructed. @@ -90026,10 +102417,11 @@ sqlite3UpperToLower[*(unsigned char *)(y)] \ ** first to get things going. Then this routine is called for each ** column. */ -SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ +SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ Table *p; int i; char *z; + char *zType; Column *pCol; sqlite3 *db = pParse->db; if( (p = pParse->pNewTable)==0 ) return; @@ -90039,10 +102431,13 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ return; } #endif - z = sqlite3NameFromToken(db, pName); + z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); if( z==0 ) return; + memcpy(z, pName->z, pName->n); + z[pName->n] = 0; + sqlite3Dequote(z); for(i=0; inCol; i++){ - if( STRICMP(z, p->aCol[i].zName) ){ + if( sqlite3_stricmp(z, p->aCol[i].zName)==0 ){ sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); sqlite3DbFree(db, z); return; @@ -90060,14 +102455,23 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ pCol = &p->aCol[p->nCol]; memset(pCol, 0, sizeof(p->aCol[0])); pCol->zName = z; + sqlite3ColumnPropertiesFromName(p, pCol); - /* If there is no type specified, columns have the default affinity - ** 'NONE'. If there is a type specified, then sqlite3AddColumnType() will - ** be called next to set pCol->affinity correctly. - */ - pCol->affinity = SQLITE_AFF_NONE; - pCol->szEst = 1; + if( pType->n==0 ){ + /* If there is no type specified, columns have the default affinity + ** 'BLOB'. */ + pCol->affinity = SQLITE_AFF_BLOB; + pCol->szEst = 1; + }else{ + zType = z + sqlite3Strlen30(z) + 1; + memcpy(zType, pType->z, pType->n); + zType[pType->n] = 0; + sqlite3Dequote(zType); + pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst); + pCol->colFlags |= COLFLAG_HASTYPE; + } p->nCol++; + pParse->constraintName.n = 0; } /* @@ -90100,7 +102504,7 @@ SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ ** 'CHAR' | SQLITE_AFF_TEXT ** 'CLOB' | SQLITE_AFF_TEXT ** 'TEXT' | SQLITE_AFF_TEXT -** 'BLOB' | SQLITE_AFF_NONE +** 'BLOB' | SQLITE_AFF_BLOB ** 'REAL' | SQLITE_AFF_REAL ** 'FLOA' | SQLITE_AFF_REAL ** 'DOUB' | SQLITE_AFF_REAL @@ -90113,7 +102517,7 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ char aff = SQLITE_AFF_NUMERIC; const char *zChar = 0; - if( zIn==0 ) return aff; + assert( zIn!=0 ); while( zIn[0] ){ h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; zIn++; @@ -90126,7 +102530,7 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ aff = SQLITE_AFF_TEXT; }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ - aff = SQLITE_AFF_NONE; + aff = SQLITE_AFF_BLOB; if( zIn[0]=='(' ) zChar = zIn; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ @@ -90149,7 +102553,7 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ ** estimate is scaled so that the size of an integer is 1. */ if( pszEst ){ *pszEst = 1; /* default size is approx 4 bytes */ - if( aff<=SQLITE_AFF_NONE ){ + if( affpNewTable; - if( p==0 || NEVER(p->nCol<1) ) return; - pCol = &p->aCol[p->nCol-1]; - assert( pCol->zType==0 ); - pCol->zType = sqlite3NameFromToken(pParse->db, pType); - pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst); -} - /* ** The expression is the default value for the most recently added column ** of the table currently under construction. @@ -90208,7 +102591,7 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ p = pParse->pNewTable; if( p!=0 ){ pCol = &(p->aCol[p->nCol-1]); - if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){ + if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr, db->init.busy) ){ sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", pCol->zName); }else{ @@ -90216,16 +102599,45 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ ** tokens that point to volatile memory. The 'span' of the expression ** is required by pragma table_info. */ + Expr x; sqlite3ExprDelete(db, pCol->pDflt); - pCol->pDflt = sqlite3ExprDup(db, pSpan->pExpr, EXPRDUP_REDUCE); - sqlite3DbFree(db, pCol->zDflt); - pCol->zDflt = sqlite3DbStrNDup(db, (char*)pSpan->zStart, - (int)(pSpan->zEnd - pSpan->zStart)); + memset(&x, 0, sizeof(x)); + x.op = TK_SPAN; + x.u.zToken = sqlite3DbStrNDup(db, (char*)pSpan->zStart, + (int)(pSpan->zEnd - pSpan->zStart)); + x.pLeft = pSpan->pExpr; + x.flags = EP_Skip; + pCol->pDflt = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE); + sqlite3DbFree(db, x.u.zToken); } } sqlite3ExprDelete(db, pSpan->pExpr); } +/* +** Backwards Compatibility Hack: +** +** Historical versions of SQLite accepted strings as column names in +** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: +** +** CREATE TABLE xyz(a,b,c,d,e,PRIMARY KEY('a'),UNIQUE('b','c' COLLATE trim) +** CREATE INDEX abc ON xyz('c','d' DESC,'e' COLLATE nocase DESC); +** +** This is goofy. But to preserve backwards compatibility we continue to +** accept it. This routine does the necessary conversion. It converts +** the expression given in its argument from a TK_STRING into a TK_ID +** if the expression is just a TK_STRING with an optional COLLATE clause. +** If the epxression is anything other than TK_STRING, the expression is +** unchanged. +*/ +static void sqlite3StringToId(Expr *p){ + if( p->op==TK_STRING ){ + p->op = TK_ID; + }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ + p->pLeft->op = TK_ID; + } +} + /* ** Designate the PRIMARY KEY for the table. pList is a list of names ** of columns that form the primary key. If pList is NULL, then the @@ -90252,10 +102664,10 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ ){ Table *pTab = pParse->pNewTable; - char *zType = 0; + Column *pCol = 0; int iCol = -1, i; int nTerm; - if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; + if( pTab==0 ) goto primary_key_exit; if( pTab->tabFlags & TF_HasPrimaryKey ){ sqlite3ErrorMsg(pParse, "table \"%s\" has more than one primary key", pTab->zName); @@ -90264,24 +102676,31 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zType = pTab->aCol[iCol].zType; + pCol = &pTab->aCol[iCol]; + pCol->colFlags |= COLFLAG_PRIMKEY; nTerm = 1; }else{ nTerm = pList->nExpr; for(i=0; inCol; iCol++){ - if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zType = pTab->aCol[iCol].zType; - break; + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[i].pExpr); + assert( pCExpr!=0 ); + sqlite3StringToId(pCExpr); + if( pCExpr->op==TK_ID ){ + const char *zCName = pCExpr->u.zToken; + for(iCol=0; iColnCol; iCol++){ + if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ + pCol = &pTab->aCol[iCol]; + pCol->colFlags |= COLFLAG_PRIMKEY; + break; + } } } } } if( nTerm==1 - && zType && sqlite3StrICmp(zType, "INTEGER")==0 - && sortOrder==SQLITE_SO_ASC + && pCol + && sqlite3StrICmp(sqlite3ColumnType(pCol,""), "INTEGER")==0 + && sortOrder!=SQLITE_SO_DESC ){ pTab->iPKey = iCol; pTab->keyConf = (u8)onError; @@ -90294,15 +102713,8 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( "INTEGER PRIMARY KEY"); #endif }else{ - Vdbe *v = pParse->pVdbe; - Index *p; - if( v ) pParse->addrSkipPK = sqlite3VdbeAddOp0(v, OP_Noop); - p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, - 0, sortOrder, 0); - if( p ){ - p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; - if( v ) sqlite3VdbeJumpHere(v, pParse->addrSkipPK); - } + sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, + 0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY); pList = 0; } @@ -90421,15 +102833,16 @@ SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ ** set back to prior value. But schema changes are infrequent ** and the probability of hitting the same cookie value is only ** 1 chance in 2^32. So we're safe enough. +** +** IMPLEMENTATION-OF: R-34230-56049 SQLite automatically increments +** the schema-version whenever the schema changes. */ SQLITE_PRIVATE void sqlite3ChangeCookie(Parse *pParse, int iDb){ - int r1 = sqlite3GetTempReg(pParse); sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - sqlite3VdbeAddOp2(v, OP_Integer, db->aDb[iDb].pSchema->schema_cookie+1, r1); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, r1); - sqlite3ReleaseTempReg(pParse, r1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, + db->aDb[iDb].pSchema->schema_cookie+1); } /* @@ -90511,7 +102924,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ n += 35 + 6*p->nCol; zStmt = sqlite3DbMallocRaw(0, n); if( zStmt==0 ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return 0; } sqlite3_snprintf(n, zStmt, "CREATE TABLE "); @@ -90520,8 +102933,8 @@ static char *createTableStmt(sqlite3 *db, Table *p){ zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ static const char * const azType[] = { + /* SQLITE_AFF_BLOB */ "", /* SQLITE_AFF_TEXT */ " TEXT", - /* SQLITE_AFF_NONE */ "", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", /* SQLITE_AFF_REAL */ " REAL" @@ -90533,17 +102946,17 @@ static char *createTableStmt(sqlite3 *db, Table *p){ k += sqlite3Strlen30(&zStmt[k]); zSep = zSep2; identPut(zStmt, &k, pCol->zName); - assert( pCol->affinity-SQLITE_AFF_TEXT >= 0 ); - assert( pCol->affinity-SQLITE_AFF_TEXT < ArraySize(azType) ); + assert( pCol->affinity-SQLITE_AFF_BLOB >= 0 ); + assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); + testcase( pCol->affinity==SQLITE_AFF_BLOB ); testcase( pCol->affinity==SQLITE_AFF_TEXT ); - testcase( pCol->affinity==SQLITE_AFF_NONE ); testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); - zType = azType[pCol->affinity - SQLITE_AFF_TEXT]; + zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; len = sqlite3Strlen30(zType); - assert( pCol->affinity==SQLITE_AFF_NONE + assert( pCol->affinity==SQLITE_AFF_BLOB || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; @@ -90564,9 +102977,9 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ assert( pIdx->isResized==0 ); nByte = (sizeof(char*) + sizeof(i16) + 1)*N; zExtra = sqlite3DbMallocZero(db, nByte); - if( zExtra==0 ) return SQLITE_NOMEM; + if( zExtra==0 ) return SQLITE_NOMEM_BKPT; memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); - pIdx->azColl = (char**)zExtra; + pIdx->azColl = (const char**)zExtra; zExtra += sizeof(char*)*N; memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn); pIdx->aiColumn = (i16*)zExtra; @@ -90621,21 +103034,23 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ ** are appropriate for a WITHOUT ROWID table instead of a rowid table. ** Changes include: ** -** (1) Convert the OP_CreateTable into an OP_CreateIndex. There is +** (1) Set all columns of the PRIMARY KEY schema object to be NOT NULL. +** (2) Convert the OP_CreateTable into an OP_CreateIndex. There is ** no rowid btree for a WITHOUT ROWID. Instead, the canonical ** data storage is a covering index btree. -** (2) Bypass the creation of the sqlite_master table entry -** for the PRIMARY KEY as the the primary key index is now +** (3) Bypass the creation of the sqlite_master table entry +** for the PRIMARY KEY as the primary key index is now ** identified by the sqlite_master table entry of the table itself. -** (3) Set the Index.tnum of the PRIMARY KEY Index object in the +** (4) Set the Index.tnum of the PRIMARY KEY Index object in the ** schema to the rootpage from the main table. -** (4) Set all columns of the PRIMARY KEY schema object to be NOT NULL. ** (5) Add all table columns to the PRIMARY KEY Index object ** so that the PRIMARY KEY is a covering index. The surplus ** columns are part of KeyInfo.nXField and are not used for ** sorting or lookup or uniqueness checks. ** (6) Replace the rowid tail on all automatically generated UNIQUE ** indices with the PRIMARY KEY columns. +** +** For virtual tables, only (1) is performed. */ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ Index *pIdx; @@ -90645,21 +103060,27 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; + /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables) + */ + if( !db->init.imposterTable ){ + for(i=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 ){ + pTab->aCol[i].notNull = OE_Abort; + } + } + } + + /* The remaining transformations only apply to b-tree tables, not to + ** virtual tables */ + if( IN_DECLARE_VTAB ) return; + /* Convert the OP_CreateTable opcode that would normally create the - ** root-page for the table into a OP_CreateIndex opcode. The index + ** root-page for the table into an OP_CreateIndex opcode. The index ** created will become the PRIMARY KEY index. */ if( pParse->addrCrTab ){ assert( v ); - sqlite3VdbeGetOp(v, pParse->addrCrTab)->opcode = OP_CreateIndex; - } - - /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master - ** table entry. - */ - if( pParse->addrSkipPK ){ - assert( v ); - sqlite3VdbeGetOp(v, pParse->addrSkipPK)->opcode = OP_Goto; + sqlite3VdbeChangeOpcode(v, pParse->addrCrTab, OP_CreateIndex); } /* Locate the PRIMARY KEY index. Or, if this table was originally @@ -90667,28 +103088,48 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ if( pTab->iPKey>=0 ){ ExprList *pList; - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token ipkToken; + sqlite3TokenInit(&ipkToken, pTab->aCol[pTab->iPKey].zName); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); if( pList==0 ) return; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->iPKey].zName); pList->a[0].sortOrder = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); - pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); - if( pPk==0 ) return; - pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY; + sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, + SQLITE_IDXTYPE_PRIMARYKEY); + if( db->mallocFailed ) return; + pPk = sqlite3PrimaryKeyIndex(pTab); pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); - } - pPk->isCovering = 1; - assert( pPk!=0 ); - nPk = pPk->nKeyCol; - /* Make sure every column of the PRIMARY KEY is NOT NULL */ - for(i=0; iaCol[pPk->aiColumn[i]].notNull = 1; + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master + ** table entry. This is only required if currently generating VDBE + ** code for a CREATE TABLE (not when parsing one as part of reading + ** a database schema). */ + if( v ){ + assert( db->init.busy==0 ); + sqlite3VdbeChangeOpcode(v, pPk->tnum, OP_Goto); + } + + /* + ** Remove all redundant columns from the PRIMARY KEY. For example, change + ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)". Later + ** code assumes the PRIMARY KEY contains no repeated columns. + */ + for(i=j=1; inKeyCol; i++){ + if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){ + pPk->nColumn--; + }else{ + pPk->aiColumn[j++] = pPk->aiColumn[i]; + } + } + pPk->nKeyCol = j; } - pPk->uniqNotNull = 1; + assert( pPk!=0 ); + pPk->isCovering = 1; + if( !db->init.imposterTable ) pPk->uniqNotNull = 1; + nPk = pPk->nKeyCol; /* The root page of the PRIMARY KEY is the table root page */ pPk->tnum = pTab->tnum; @@ -90727,7 +103168,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( !hasColumn(pPk->aiColumn, j, i) ){ assert( jnColumn ); pPk->aiColumn[j] = i; - pPk->azColl[j] = "BINARY"; + pPk->azColl[j] = sqlite3StrBINARY; j++; } } @@ -90770,9 +103211,10 @@ SQLITE_PRIVATE void sqlite3EndTable( int iDb; /* Database in which the table lives */ Index *pIdx; /* An implied index of the table */ - if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ + if( pEnd==0 && pSelect==0 ){ return; } + assert( !db->mallocFailed ); p = pParse->pNewTable; if( p==0 ) return; @@ -90783,9 +103225,13 @@ SQLITE_PRIVATE void sqlite3EndTable( ** So do not write to the disk again. Extract the root page number ** for the table from the db->init.newTnum field. (The page number ** should have been put there by the sqliteOpenCb routine.) + ** + ** If the root page number is 1, that means this is the sqlite_master + ** table itself. So mark it read-only. */ if( db->init.busy ){ p->tnum = db->init.newTnum; + if( p->tnum==1 ) p->tabFlags |= TF_Readonly; } /* Special processing for WITHOUT ROWID Tables */ @@ -90798,7 +103244,7 @@ SQLITE_PRIVATE void sqlite3EndTable( if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); }else{ - p->tabFlags |= TF_WithoutRowid; + p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; convertToWithoutRowidTable(pParse, p); } } @@ -90866,26 +103312,46 @@ SQLITE_PRIVATE void sqlite3EndTable( ** be redundant. */ if( pSelect ){ - SelectDest dest; - Table *pSelTab; + SelectDest dest; /* Where the SELECT should store results */ + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int regRec; /* A record to be insert into the new table */ + int regRowid; /* Rowid of the next row to insert */ + int addrInsLoop; /* Top of the loop for inserting rows */ + Table *pSelTab; /* A table that describes the SELECT results */ + regYield = ++pParse->nMem; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; assert(pParse->nTab==1); + sqlite3MayAbort(pParse); sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); pParse->nTab = 2; - sqlite3SelectDestInit(&dest, SRT_Table, 1); + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); sqlite3Select(pParse, pSelect, &dest); + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); + if( pParse->nErr ) return; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); + if( pSelTab==0 ) return; + assert( p->aCol==0 ); + p->nCol = pSelTab->nCol; + p->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); + sqlite3TableAffinity(v, p, 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeGoto(v, addrInsLoop); + sqlite3VdbeJumpHere(v, addrInsLoop); sqlite3VdbeAddOp1(v, OP_Close, 1); - if( pParse->nErr==0 ){ - pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); - if( pSelTab==0 ) return; - assert( p->aCol==0 ); - p->nCol = pSelTab->nCol; - p->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - } } /* Compute the complete text of the CREATE statement */ @@ -90908,7 +103374,7 @@ SQLITE_PRIVATE void sqlite3EndTable( "UPDATE %Q.%s " "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q " "WHERE rowid=#%d", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), zType, p->zName, p->zName, @@ -90923,13 +103389,13 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Check to see if we need to create an sqlite_sequence table for ** keeping track of autoincrement keys. */ - if( p->tabFlags & TF_Autoincrement ){ + if( (p->tabFlags & TF_Autoincrement)!=0 ){ Db *pDb = &db->aDb[iDb]; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->pSeqTab==0 ){ sqlite3NestedParse(pParse, "CREATE TABLE %Q.sqlite_sequence(name,seq)", - pDb->zName + pDb->zDbSName ); } } @@ -90947,11 +103413,10 @@ SQLITE_PRIVATE void sqlite3EndTable( Table *pOld; Schema *pSchema = p->pSchema; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, - sqlite3Strlen30(p->zName),p); + pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); if( pOld ){ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ - db->mallocFailed = 1; + sqlite3OomFault(db); return; } pParse->pNewTable = 0; @@ -90981,6 +103446,7 @@ SQLITE_PRIVATE void sqlite3CreateView( Token *pBegin, /* The CREATE token that begins the statement */ Token *pName1, /* The token that holds the name of the view */ Token *pName2, /* The token that holds the name of the view */ + ExprList *pCNames, /* Optional list of view column names */ Select *pSelect, /* A SELECT statement that will become the new view */ int isTemp, /* TRUE for a TEMPORARY view */ int noErr /* Suppress error messages if VIEW already exists */ @@ -90996,22 +103462,15 @@ SQLITE_PRIVATE void sqlite3CreateView( if( pParse->nVar>0 ){ sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); - sqlite3SelectDelete(db, pSelect); - return; + goto create_view_fail; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; - if( p==0 || pParse->nErr ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( p==0 || pParse->nErr ) goto create_view_fail; sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); - if( sqlite3FixSelect(&sFix, pSelect) ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; /* Make a copy of the entire SELECT statement that defines the view. ** This will force all the Expr.token.z values to be dynamically @@ -91019,30 +103478,31 @@ SQLITE_PRIVATE void sqlite3CreateView( ** they will persist after the current sqlite3_exec() call returns. */ p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); - sqlite3SelectDelete(db, pSelect); - if( db->mallocFailed ){ - return; - } - if( !db->init.busy ){ - sqlite3ViewGetColumnNames(pParse, p); - } + p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); + if( db->mallocFailed ) goto create_view_fail; /* Locate the end of the CREATE VIEW statement. Make sEnd point to ** the end. */ sEnd = pParse->sLastToken; - if( ALWAYS(sEnd.z[0]!=0) && sEnd.z[0]!=';' ){ + assert( sEnd.z[0]!=0 ); + if( sEnd.z[0]!=';' ){ sEnd.z += sEnd.n; } sEnd.n = 0; n = (int)(sEnd.z - pBegin->z); + assert( n>0 ); z = pBegin->z; - while( ALWAYS(n>0) && sqlite3Isspace(z[n-1]) ){ n--; } + while( sqlite3Isspace(z[n-1]) ){ n--; } sEnd.z = &z[n-1]; sEnd.n = 1; /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ sqlite3EndTable(pParse, 0, &sEnd, 0, 0); + +create_view_fail: + sqlite3SelectDelete(db, pSelect); + sqlite3ExprListDelete(db, pCNames); return; } #endif /* SQLITE_OMIT_VIEW */ @@ -91059,7 +103519,9 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int nErr = 0; /* Number of errors encountered */ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth; /* Saved xAuth pointer */ +#endif assert( pTable ); @@ -91107,11 +103569,10 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable->pSelect ); pSel = sqlite3SelectDup(db, pTable->pSelect, 0); if( pSel ){ - u8 enableLookaside = db->lookaside.bEnabled; n = pParse->nTab; sqlite3SrcListAssignCursors(pParse, pSel->pSrc); pTable->nCol = -1; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable++; #ifndef SQLITE_OMIT_AUTHORIZATION xAuth = db->xAuth; db->xAuth = 0; @@ -91120,25 +103581,43 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #else pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); #endif - db->lookaside.bEnabled = enableLookaside; pParse->nTab = n; - if( pSelTab ){ + if( pTable->pCheck ){ + /* CREATE VIEW name(arglist) AS ... + ** The names of the columns in the table are taken from + ** arglist which is stored in pTable->pCheck. The pCheck field + ** normally holds CHECK constraints on an ordinary table, but for + ** a VIEW it holds the list of column names. + */ + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + &pTable->nCol, &pTable->aCol); + if( db->mallocFailed==0 + && pParse->nErr==0 + && pTable->nCol==pSel->pEList->nExpr + ){ + sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel); + } + }else if( pSelTab ){ + /* CREATE VIEW name AS... without an argument list. Construct + ** the column names from the SELECT statement that defines the view. + */ assert( pTable->aCol==0 ); pTable->nCol = pSelTab->nCol; pTable->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - pTable->pSchema->schemaFlags |= DB_UnresetViews; }else{ pTable->nCol = 0; nErr++; } + sqlite3DeleteTable(db, pSelTab); sqlite3SelectDelete(db, pSel); + db->lookaside.bDisable--; } else { nErr++; } + pTable->pSchema->schemaFlags |= DB_UnresetViews; #endif /* SQLITE_OMIT_VIEW */ return nErr; } @@ -91155,7 +103634,7 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); if( pTab->pSelect ){ - sqliteDeleteColumnNames(db, pTab); + sqlite3DeleteColumnNames(db, pTab); pTab->aCol = 0; pTab->nCol = 0; } @@ -91217,6 +103696,7 @@ SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, int iFrom, int iT static void destroyRootPage(Parse *pParse, int iTable, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); int r1 = sqlite3GetTempReg(pParse); + assert( iTable>1 ); sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); sqlite3MayAbort(pParse); #ifndef SQLITE_OMIT_AUTOVACUUM @@ -91231,7 +103711,7 @@ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ */ sqlite3NestedParse(pParse, "UPDATE %Q.%s SET rootpage=%d WHERE #%d AND rootpage=#%d", - pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable, r1, r1); + pParse->db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), iTable, r1, r1); #endif sqlite3ReleaseTempReg(pParse, r1); } @@ -91307,7 +103787,7 @@ static void sqlite3ClearStatTables( const char *zName /* Name of index or table */ ){ int i; - const char *zDbName = pParse->db->aDb[iDb].zName; + const char *zDbName = pParse->db->aDb[iDb].zDbSName; for(i=1; i<=4; i++){ char zTab[24]; sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); @@ -91360,7 +103840,7 @@ SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, in if( pTab->tabFlags & TF_Autoincrement ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", - pDb->zName, pTab->zName + pDb->zDbSName, pTab->zName ); } #endif @@ -91374,7 +103854,7 @@ SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, in */ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", - pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + pDb->zDbSName, SCHEMA_TABLE(iDb), pTab->zName); if( !isView && !IsVirtual(pTab) ){ destroyTable(pParse, pTab); } @@ -91405,7 +103885,9 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, } assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); + if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; + assert( isView==0 || isView==LOCATE_VIEW ); pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; @@ -91426,7 +103908,7 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, { int code; const char *zTab = SCHEMA_TABLE(iDb); - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zArg2 = 0; if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ goto exit_drop_table; @@ -91598,10 +104080,10 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, - pFKey->zTo, sqlite3Strlen30(pFKey->zTo), (void *)pFKey + pFKey->zTo, (void *)pFKey ); if( pNextTo==pFKey ){ - db->mallocFailed = 1; + sqlite3OomFault(db); goto fk_end; } if( pNextTo ){ @@ -91661,13 +104143,13 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int iPartIdxLabel; /* Jump to this label to skip a row */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ - int regRecord; /* Register holding assemblied index record */ + int regRecord; /* Register holding assembled index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, - db->aDb[iDb].zName ) ){ + db->aDb[iDb].zDbSName ) ){ return; } #endif @@ -91683,10 +104165,11 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ tnum = pIndex->tnum; } pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); + assert( pKey!=0 || db->mallocFailed || pParse->nErr ); /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*) + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*) sqlite3KeyInfoRef(pKey), P4_KEYINFO); /* Open the table. Loop through all rows of the table, inserting index @@ -91706,10 +104189,9 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); - assert( pKey!=0 || db->mallocFailed || pParse->nErr ); - if( IsUniqueIndex(pIndex) && pKey!=0 ){ + if( IsUniqueIndex(pIndex) ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + sqlite3VdbeGoto(v, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, pIndex->nKeyCol); VdbeCoverage(v); @@ -91717,8 +104199,9 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ }else{ addr2 = sqlite3VdbeCurrentAddr(v); } - sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); + sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); + sqlite3VdbeAddOp3(v, OP_Last, iIdx, 0, -1); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); @@ -91753,7 +104236,7 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( p = sqlite3DbMallocZero(db, nByte + nExtra); if( p ){ char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); - p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->azColl = (const char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; @@ -91775,12 +104258,8 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( ** pList is a list of columns to be indexed. pList will be NULL if this ** is a primary key or unique-constraint on the most recent column added ** to the table currently under construction. -** -** If the index is created successfully, return a pointer to the new Index -** structure. This is used by sqlite3AddPrimaryKey() to mark the index -** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY) */ -SQLITE_PRIVATE Index *sqlite3CreateIndex( +SQLITE_PRIVATE void sqlite3CreateIndex( Parse *pParse, /* All information about this parse */ Token *pName1, /* First part of index name. May be NULL */ Token *pName2, /* Second part of index name. May be NULL */ @@ -91790,9 +104269,9 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( Token *pStart, /* The CREATE token that begins this statement */ Expr *pPIWhere, /* WHERE clause for partial indices */ int sortOrder, /* Sort order of primary key when pList==NULL */ - int ifNotExist /* Omit error if index already exists */ + int ifNotExist, /* Omit error if index already exists */ + u8 idxType /* The index type */ ){ - Index *pRet = 0; /* Pointer to return */ Table *pTab = 0; /* Table to be indexed */ Index *pIndex = 0; /* The index to be created */ char *zName = 0; /* Name of the index */ @@ -91805,14 +104284,15 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( int iDb; /* Index of the database that is being written */ Token *pName = 0; /* Unqualified name of the index to create */ struct ExprList_item *pListItem; /* For looping over pList */ - const Column *pTabCol; /* A column in the table */ int nExtra = 0; /* Space allocated for zExtra[] */ int nExtraCol; /* Number of extra columns needed */ char *zExtra = 0; /* Extra space after the Index object */ Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ - assert( pParse->nErr==0 ); /* Never called with prior errors */ - if( db->mallocFailed || IN_DECLARE_VTAB ){ + if( db->mallocFailed || pParse->nErr>0 ){ + goto exit_create_index; + } + if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){ goto exit_create_index; } if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ @@ -91874,6 +104354,10 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( assert( pTab!=0 ); assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && db->init.busy==0 +#if SQLITE_USER_AUTHENTICATION + && sqlite3UserAuthTable(pTab->zName)==0 +#endif && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; @@ -91917,7 +104401,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( goto exit_create_index; } } - if( sqlite3FindIndex(db, zName, pDb->zName)!=0 ){ + if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ if( !ifNotExist ){ sqlite3ErrorMsg(pParse, "index %s already exists", zName); }else{ @@ -91934,13 +104418,20 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( if( zName==0 ){ goto exit_create_index; } + + /* Automatic index names generated from within sqlite3_declare_vtab() + ** must have names that are distinct from normal automatic index names. + ** The following statement converts "sqlite3_autoindex..." into + ** "sqlite3_butoindex..." in order to make the names distinct. + ** The "vtab_err.test" test demonstrates the need of this statement. */ + if( IN_DECLARE_VTAB ) zName[7]++; } /* Check for authorization to create an index. */ #ifndef SQLITE_OMIT_AUTHORIZATION { - const char *zDb = pDb->zName; + const char *zDb = pDb->zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ goto exit_create_index; } @@ -91957,11 +104448,15 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** So create a fake list to simulate this. */ if( pList==0 ){ - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token prevCol; + sqlite3TokenInit(&prevCol, pTab->aCol[pTab->nCol-1].zName); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->nCol-1].zName); - pList->a[0].sortOrder = (u8)sortOrder; + assert( pList->nExpr==1 ); + sqlite3ExprListSetSortOrder(pList, sortOrder); + }else{ + sqlite3ExprListCheckLength(pParse, pList, "index"); } /* Figure out how many bytes of space are required to store explicitly @@ -91969,8 +104464,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( */ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( pExpr ){ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr!=0 ); + if( pExpr->op==TK_COLLATE ){ nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -91993,7 +104488,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pIndex->pTable = pTab; pIndex->onError = (u8)onError; pIndex->uniqNotNull = onError!=OE_None; - pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE; + pIndex->idxType = idxType; pIndex->pSchema = db->aDb[iDb].pSchema; pIndex->nKeyCol = pList->nExpr; if( pPIWhere ){ @@ -92011,35 +104506,54 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( sortOrderMask = 0; /* Ignore DESC */ } - /* Scan the names of the columns of the table to be indexed and - ** load the column indices into the Index structure. Report an error - ** if any column is not found. + /* Analyze the list of expressions that form the terms of the index and + ** report any errors. In the common case where the expression is exactly + ** a table column, store that column in aiColumn[]. For general expressions, + ** populate pIndex->aColExpr and store XN_EXPR (-2) in aiColumn[]. ** - ** TODO: Add a test to make sure that the same column is not named - ** more than once within the same index. Only the first instance of - ** the column will ever be used by the optimizer. Note that using the - ** same column more than once cannot be an error because that would - ** break backwards compatibility - it needs to be a warning. + ** TODO: Issue a warning if two or more columns of the index are identical. + ** TODO: Issue a warning if the table primary key is used as part of the + ** index key. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ - const char *zColName = pListItem->zName; - int requestedSortOrder; - char *zColl; /* Collation sequence name */ + Expr *pCExpr; /* The i-th index expression */ + int requestedSortOrder; /* ASC or DESC on the i-th expression */ + const char *zColl; /* Collation sequence name */ - for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ - if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; + sqlite3StringToId(pListItem->pExpr); + sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); + if( pParse->nErr ) goto exit_create_index; + pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); + if( pCExpr->op!=TK_COLUMN ){ + if( pTab==pParse->pNewTable ){ + sqlite3ErrorMsg(pParse, "expressions prohibited in PRIMARY KEY and " + "UNIQUE constraints"); + goto exit_create_index; + } + if( pIndex->aColExpr==0 ){ + ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); + pIndex->aColExpr = pCopy; + if( !db->mallocFailed ){ + assert( pCopy!=0 ); + pListItem = &pCopy->a[i]; + } + } + j = XN_EXPR; + pIndex->aiColumn[i] = XN_EXPR; + pIndex->uniqNotNull = 0; + }else{ + j = pCExpr->iColumn; + assert( j<=0x7fff ); + if( j<0 ){ + j = pTab->iPKey; + }else if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + pIndex->aiColumn[i] = (i16)j; } - if( j>=pTab->nCol ){ - sqlite3ErrorMsg(pParse, "table %s has no column named %s", - pTab->zName, zColName); - pParse->checkSchema = 1; - goto exit_create_index; - } - assert( pTab->nCol<=0x7fff && j<=0x7fff ); - pIndex->aiColumn[i] = (i16)j; - if( pListItem->pExpr ){ + zColl = 0; + if( pListItem->pExpr->op==TK_COLLATE ){ int nColl; - assert( pListItem->pExpr->op==TK_COLLATE ); zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); @@ -92047,21 +104561,26 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( zColl = zExtra; zExtra += nColl; nExtra -= nColl; - }else{ + }else if( j>=0 ){ zColl = pTab->aCol[j].zColl; - if( !zColl ) zColl = "BINARY"; } + if( !zColl ) zColl = sqlite3StrBINARY; if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; } pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; - if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; } + + /* Append the table key to the end of the index. For WITHOUT ROWID + ** tables (when pPk!=0) this will be the declared PRIMARY KEY. For + ** normal tables (when pPk==0) this will be the rowid. + */ if( pPk ){ for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; + assert( x>=0 ); if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ pIndex->nColumn--; }else{ @@ -92073,12 +104592,26 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( } assert( i==pIndex->nColumn ); }else{ - pIndex->aiColumn[i] = -1; - pIndex->azColl[i] = "BINARY"; + pIndex->aiColumn[i] = XN_ROWID; + pIndex->azColl[i] = sqlite3StrBINARY; } sqlite3DefaultRowEst(pIndex); if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); + /* If this index contains every column of its table, then mark + ** it as a covering index */ + assert( HasRowid(pTab) + || pTab->iPKey<0 || sqlite3ColumnOfIndex(pIndex, pTab->iPKey)>=0 ); + if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ + pIndex->isCovering = 1; + for(j=0; jnCol; j++){ + if( j==pTab->iPKey ) continue; + if( sqlite3ColumnOfIndex(pIndex,j)>=0 ) continue; + pIndex->isCovering = 0; + break; + } + } + if( pTab==pParse->pNewTable ){ /* This routine has been called to create an automatic index as a ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or @@ -92112,10 +104645,11 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( for(k=0; knKeyCol; k++){ const char *z1; const char *z2; + assert( pIdx->aiColumn[k]>=0 ); if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; - if( z1!=z2 && sqlite3StrICmp(z1, z2) ) break; + if( sqlite3StrICmp(z1, z2) ) break; } if( k==pIdx->nKeyCol ){ if( pIdx->onError!=pIndex->onError ){ @@ -92134,6 +104668,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pIdx->onError = pIndex->onError; } } + if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType; goto exit_create_index; } } @@ -92142,15 +104677,16 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( /* Link the new Index structure to its table and to the other ** in-memory database structures. */ + assert( pParse->nErr==0 ); if( db->init.busy ){ Index *p; + assert( !IN_DECLARE_VTAB ); assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); p = sqlite3HashInsert(&pIndex->pSchema->idxHash, - pIndex->zName, sqlite3Strlen30(pIndex->zName), - pIndex); + pIndex->zName, pIndex); if( p ){ assert( p==pIndex ); /* Malloc must have failed */ - db->mallocFailed = 1; + sqlite3OomFault(db); goto exit_create_index; } db->flags |= SQLITE_InternChanges; @@ -92172,7 +104708,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** has just been created, it contains no data and the index initialization ** step can be skipped. */ - else if( pParse->nErr==0 && (HasRowid(pTab) || pTblName!=0) ){ + else if( HasRowid(pTab) || pTblName!=0 ){ Vdbe *v; char *zStmt; int iMem = ++pParse->nMem; @@ -92180,10 +104716,15 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( v = sqlite3GetVdbe(pParse); if( v==0 ) goto exit_create_index; - - /* Create the rootpage for the index - */ sqlite3BeginWriteOperation(pParse, 1, iDb); + + /* Create the rootpage for the index using CreateIndex. But before + ** doing so, code a Noop instruction and store its address in + ** Index.tnum. This is required in case this index is actually a + ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In + ** that case the convertToWithoutRowidTable() routine will replace + ** the Noop with a Goto to jump over the VDBE code generated below. */ + pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); /* Gather the complete text of the CREATE INDEX statement into @@ -92205,7 +104746,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( */ sqlite3NestedParse(pParse, "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), pIndex->zName, pTab->zName, iMem, @@ -92221,8 +104762,10 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddParseSchemaOp(v, iDb, sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); - sqlite3VdbeAddOp1(v, OP_Expire, 0); + sqlite3VdbeAddOp0(v, OP_Expire); } + + sqlite3VdbeJumpHere(v, pIndex->tnum); } /* When adding an index to the list of indices for a table, make @@ -92244,7 +104787,6 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pIndex->pNext = pOther->pNext; pOther->pNext = pIndex; } - pRet = pIndex; pIndex = 0; } @@ -92255,14 +104797,13 @@ exit_create_index: sqlite3ExprListDelete(db, pList); sqlite3SrcListDelete(db, pTblName); sqlite3DbFree(db, zName); - return pRet; } /* ** Fill the Index.aiRowEst[] array with default information - information ** to be used when we have not run the ANALYZE command. ** -** aiRowEst[0] is suppose to contain the number of elements in the index. +** aiRowEst[0] is supposed to contain the number of elements in the index. ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number @@ -92284,10 +104825,11 @@ SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ int i; /* Set the first entry (number of rows in the index) to the estimated - ** number of rows in the table. Or 10, if the estimated number of rows - ** in the table is less than that. */ + ** number of rows in the table, or half the number of rows in the table + ** for a partial index. But do not let the estimate drop below 10. */ a[0] = pIdx->pTable->nRowLogEst; - if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + if( pIdx->pPartIdxWhere!=0 ) a[0] -= 10; assert( 10==sqlite3LogEst(2) ); + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is ** 6 and each subsequent value (if any) is 5. */ @@ -92338,7 +104880,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists { int code = SQLITE_DROP_INDEX; Table *pTab = pIndex->pTable; - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zTab = SCHEMA_TABLE(iDb); if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ goto exit_drop_index; @@ -92356,7 +104898,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName + db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), pIndex->zName ); sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); sqlite3ChangeCookie(pParse, iDb); @@ -92572,10 +105114,12 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( ){ struct SrcList_item *pItem; assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ + assert( db!=0 ); if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(SrcList) ); + pList = sqlite3DbMallocRawNN(db, sizeof(SrcList) ); if( pList==0 ) return 0; pList->nAlloc = 1; + pList->nSrc = 0; } pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); if( db->mallocFailed ){ @@ -92625,7 +105169,8 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbFree(db, pItem->zDatabase); sqlite3DbFree(db, pItem->zName); sqlite3DbFree(db, pItem->zAlias); - sqlite3DbFree(db, pItem->zIndex); + if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); + if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); sqlite3SelectDelete(db, pItem->pSelect); sqlite3ExprDelete(db, pItem->pOn); @@ -92641,7 +105186,7 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ ** if this is the first term of the FROM clause. pTable and pDatabase ** are the name of the table and database named in the FROM clause term. ** pDatabase is NULL if the database name qualifier is missing - the -** usual case. If the term has a alias, then pAlias points to the +** usual case. If the term has an alias, then pAlias points to the ** alias token. If the term is a subquery, then pSubquery is the ** SELECT statement that the subquery encodes. The pTable and ** pDatabase parameters are NULL for subqueries. The pOn and pUsing @@ -92698,17 +105243,37 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI assert( pIndexedBy!=0 ); if( p && ALWAYS(p->nSrc>0) ){ struct SrcList_item *pItem = &p->a[p->nSrc-1]; - assert( pItem->notIndexed==0 && pItem->zIndex==0 ); + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ - pItem->notIndexed = 1; + pItem->fg.notIndexed = 1; }else{ - pItem->zIndex = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0); } } } +/* +** Add the list of function arguments to the SrcList entry for a +** table-valued-function. +*/ +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){ + if( p ){ + struct SrcList_item *pItem = &p->a[p->nSrc-1]; + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); + pItem->u1.pFuncArg = pList; + pItem->fg.isTabFunc = 1; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + /* ** When building up a FROM clause in the parser, the join operator ** is initially attached to the left operand. But the code generator @@ -92727,16 +105292,15 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ if( p ){ int i; - assert( p->a || p->nSrc==0 ); for(i=p->nSrc-1; i>0; i--){ - p->a[i].jointype = p->a[i-1].jointype; + p->a[i].fg.jointype = p->a[i-1].fg.jointype; } - p->a[0].jointype = 0; + p->a[0].fg.jointype = 0; } } /* -** Begin a transaction +** Generate VDBE code for a BEGIN statement. */ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ sqlite3 *db; @@ -92746,7 +105310,6 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ assert( pParse!=0 ); db = pParse->db; assert( db!=0 ); -/* if( db->aDb[0].pBt==0 ) return; */ if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ){ return; } @@ -92758,11 +105321,11 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ sqlite3VdbeUsesBtree(v, i); } } - sqlite3VdbeAddOp2(v, OP_AutoCommit, 0, 0); + sqlite3VdbeAddOp0(v, OP_AutoCommit); } /* -** Commit a transaction +** Generate VDBE code for a COMMIT statement. */ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ Vdbe *v; @@ -92774,12 +105337,12 @@ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ } v = sqlite3GetVdbe(pParse); if( v ){ - sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 0); + sqlite3VdbeAddOp1(v, OP_AutoCommit, 1); } } /* -** Rollback a transaction +** Generate VDBE code for a ROLLBACK statement. */ SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse *pParse){ Vdbe *v; @@ -92841,7 +105404,7 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ db->aDb[1].pBt = pBt; assert( db->aDb[1].pSchema ); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return 1; } } @@ -92856,15 +105419,13 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ */ SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); - sqlite3 *db = pToplevel->db; - assert( iDb>=0 && iDbnDb ); - assert( db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDb>=0 && iDbdb->nDb ); + assert( pParse->db->aDb[iDb].pBt!=0 || iDb==1 ); assert( iDbdb, iDb, 0) ); if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){ DbMaskSet(pToplevel->cookieMask, iDb); - pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; if( !OMIT_TEMPDB && iDb==1 ){ sqlite3OpenTempDatabase(pToplevel); } @@ -92880,7 +105441,7 @@ SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb) int i; for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; - if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zName)) ){ + if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zDbSName)) ){ sqlite3CodeVerifySchema(pParse, i); } } @@ -92958,7 +105519,7 @@ SQLITE_PRIVATE void sqlite3HaltConstraint( sqlite3MayAbort(pParse); } sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); - if( p5Errmsg ) sqlite3VdbeChangeP5(v, p5Errmsg); + sqlite3VdbeChangeP5(v, p5Errmsg); } /* @@ -92974,14 +105535,17 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( StrAccum errMsg; Table *pTab = pIdx->pTable; - sqlite3StrAccumInit(&errMsg, 0, 0, 200); - errMsg.db = pParse->db; - for(j=0; jnKeyCol; j++){ - char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); - sqlite3StrAccumAppendAll(&errMsg, pTab->zName); - sqlite3StrAccumAppend(&errMsg, ".", 1); - sqlite3StrAccumAppendAll(&errMsg, zCol); + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); + if( pIdx->aColExpr ){ + sqlite3XPrintf(&errMsg, "index '%q'", pIdx->zName); + }else{ + for(j=0; jnKeyCol; j++){ + char *zCol; + assert( pIdx->aiColumn[j]>=0 ); + zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); + sqlite3XPrintf(&errMsg, "%s.%s", pTab->zName, zCol); + } } zErr = sqlite3StrAccumFinish(&errMsg); sqlite3HaltConstraint(pParse, @@ -93124,7 +105688,7 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ if( iDb<0 ) return; z = sqlite3NameFromToken(db, pObjName); if( z==0 ) return; - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; pTab = sqlite3FindTable(db, z, zDb); if( pTab ){ reindexTable(pParse, pTab, 0); @@ -93145,48 +105709,34 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ /* ** Return a KeyInfo structure that is appropriate for the given Index. ** -** The KeyInfo structure for an index is cached in the Index object. -** So there might be multiple references to the returned pointer. The -** caller should not try to modify the KeyInfo object. -** ** The caller should invoke sqlite3KeyInfoUnref() on the returned object ** when it has finished using it. */ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ + int i; + int nCol = pIdx->nColumn; + int nKey = pIdx->nKeyCol; + KeyInfo *pKey; if( pParse->nErr ) return 0; -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pIdx->pKeyInfo && pIdx->pKeyInfo->db!=pParse->db ){ - sqlite3KeyInfoUnref(pIdx->pKeyInfo); - pIdx->pKeyInfo = 0; + if( pIdx->uniqNotNull ){ + pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); + }else{ + pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); } -#endif - if( pIdx->pKeyInfo==0 ){ - int i; - int nCol = pIdx->nColumn; - int nKey = pIdx->nKeyCol; - KeyInfo *pKey; - if( pIdx->uniqNotNull ){ - pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); - }else{ - pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); + if( pKey ){ + assert( sqlite3KeyInfoIsWriteable(pKey) ); + for(i=0; iazColl[i]; + pKey->aColl[i] = zColl==sqlite3StrBINARY ? 0 : + sqlite3LocateCollSeq(pParse, zColl); + pKey->aSortOrder[i] = pIdx->aSortOrder[i]; } - if( pKey ){ - assert( sqlite3KeyInfoIsWriteable(pKey) ); - for(i=0; iazColl[i]; - assert( zColl!=0 ); - pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : - sqlite3LocateCollSeq(pParse, zColl); - pKey->aSortOrder[i] = pIdx->aSortOrder[i]; - } - if( pParse->nErr ){ - sqlite3KeyInfoUnref(pKey); - }else{ - pIdx->pKeyInfo = pKey; - } + if( pParse->nErr ){ + sqlite3KeyInfoUnref(pKey); + pKey = 0; } } - return sqlite3KeyInfoRef(pIdx->pKeyInfo); + return pKey; } #ifndef SQLITE_OMIT_CTE @@ -93223,10 +105773,9 @@ SQLITE_PRIVATE With *sqlite3WithAdd( }else{ pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); } - assert( zName!=0 || pNew==0 ); - assert( db->mallocFailed==0 || pNew==0 ); + assert( (pNew!=0 && zName!=0) || db->mallocFailed ); - if( pNew==0 ){ + if( db->mallocFailed ){ sqlite3ExprListDelete(db, pArglist); sqlite3SelectDelete(db, pQuery); sqlite3DbFree(db, zName); @@ -93235,7 +105784,7 @@ SQLITE_PRIVATE With *sqlite3WithAdd( pNew->a[pNew->nCte].pSelect = pQuery; pNew->a[pNew->nCte].pCols = pArglist; pNew->a[pNew->nCte].zName = zName; - pNew->a[pNew->nCte].zErr = 0; + pNew->a[pNew->nCte].zCteErr = 0; pNew->nCte++; } @@ -93277,6 +105826,7 @@ SQLITE_PRIVATE void sqlite3WithDelete(sqlite3 *db, With *pWith){ ** of user defined functions and collation sequences. */ +/* #include "sqliteInt.h" */ /* ** Invoke the 'collation needed' callback to request a collation sequence @@ -93404,7 +105954,7 @@ SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ ** ** Each pointer stored in the sqlite3.aCollSeq hash table contains an ** array of three CollSeq structures. The first is the collation sequence -** prefferred for UTF-8, the second UTF-16le, and the third UTF-16be. +** preferred for UTF-8, the second UTF-16le, and the third UTF-16be. ** ** Stored immediately after the three collation sequences is a copy of ** the collation sequence name. A pointer to this string is stored in @@ -93416,11 +105966,11 @@ static CollSeq *findCollSeqEntry( int create /* Create a new entry if true */ ){ CollSeq *pColl; - int nName = sqlite3Strlen30(zName); - pColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + pColl = sqlite3HashFind(&db->aCollSeq, zName); if( 0==pColl && create ){ - pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 ); + int nName = sqlite3Strlen30(zName); + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1); if( pColl ){ CollSeq *pDel = 0; pColl[0].zName = (char*)&pColl[3]; @@ -93431,7 +105981,7 @@ static CollSeq *findCollSeqEntry( pColl[2].enc = SQLITE_UTF16BE; memcpy(pColl[0].zName, zName, nName); pColl[0].zName[nName] = 0; - pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, nName, pColl); + pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl); /* If a malloc() failure occurred in sqlite3HashInsert(), it will ** return the pColl pointer to be deleted (because it wasn't added @@ -93439,7 +105989,7 @@ static CollSeq *findCollSeqEntry( */ assert( pDel==0 || pDel==pColl ); if( pDel!=0 ){ - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, pDel); pColl = 0; } @@ -93505,8 +106055,8 @@ SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( ** 5: UTF16 byte order conversion required - argument count matches exactly ** 6: Perfect match: encoding and argument count match exactly. ** -** If nArg==(-2) then any function with a non-null xStep or xFunc is -** a perfect match and any function with both xStep and xFunc NULL is +** If nArg==(-2) then any function with a non-null xSFunc is +** a perfect match and any function with xSFunc NULL is ** a non-match. */ #define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ @@ -93518,7 +106068,7 @@ static int matchQuality( int match; /* nArg of -2 is a special case */ - if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH; + if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; /* Wrong number of arguments means "no match" */ if( p->nArg!=nArg && p->nArg>=0 ) return 0; @@ -93546,14 +106096,12 @@ static int matchQuality( ** a pointer to the matching FuncDef if found, or 0 if there is no match. */ static FuncDef *functionSearch( - FuncDefHash *pHash, /* Hash table to search */ int h, /* Hash of the name */ - const char *zFunc, /* Name of function */ - int nFunc /* Number of bytes in zFunc */ + const char *zFunc /* Name of function */ ){ FuncDef *p; - for(p=pHash->a[h]; p; p=p->pHash){ - if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ + for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ + if( sqlite3StrICmp(p->zName, zFunc)==0 ){ return p; } } @@ -93563,23 +106111,26 @@ static FuncDef *functionSearch( /* ** Insert a new FuncDef into a FuncDefHash hash table. */ -SQLITE_PRIVATE void sqlite3FuncDefInsert( - FuncDefHash *pHash, /* The hash table into which to insert */ - FuncDef *pDef /* The function definition to insert */ +SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( + FuncDef *aDef, /* List of global functions to be inserted */ + int nDef /* Length of the apDef[] list */ ){ - FuncDef *pOther; - int nName = sqlite3Strlen30(pDef->zName); - u8 c1 = (u8)pDef->zName[0]; - int h = (sqlite3UpperToLower[c1] + nName) % ArraySize(pHash->a); - pOther = functionSearch(pHash, h, pDef->zName, nName); - if( pOther ){ - assert( pOther!=pDef && pOther->pNext!=pDef ); - pDef->pNext = pOther->pNext; - pOther->pNext = pDef; - }else{ - pDef->pNext = 0; - pDef->pHash = pHash->a[h]; - pHash->a[h] = pDef; + int i; + for(i=0; ipNext!=&aDef[i] ); + aDef[i].pNext = pOther->pNext; + pOther->pNext = &aDef[i]; + }else{ + aDef[i].pNext = 0; + aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h]; + sqlite3BuiltinFunctions.a[h] = &aDef[i]; + } } } @@ -93596,7 +106147,7 @@ SQLITE_PRIVATE void sqlite3FuncDefInsert( ** no matching function previously existed. ** ** If nArg is -2, then the first valid function found is returned. A -** function is valid if either xFunc or xStep is non-zero. The nArg==(-2) +** function is valid if xSFunc is non-zero. The nArg==(-2) ** case is used to see if zName is a valid function name for some number ** of arguments. If nArg is -2, then createFlag must be 0. ** @@ -93606,8 +106157,7 @@ SQLITE_PRIVATE void sqlite3FuncDefInsert( */ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( sqlite3 *db, /* An open database */ - const char *zName, /* Name of the function. Not null-terminated */ - int nName, /* Number of characters in the name */ + const char *zName, /* Name of the function. zero-terminated */ int nArg, /* Number of arguments. -1 means any number */ u8 enc, /* Preferred text encoding */ u8 createFlag /* Create new entry if true and does not otherwise exist */ @@ -93616,14 +106166,15 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( FuncDef *pBest = 0; /* Best match found so far */ int bestScore = 0; /* Score of best match */ int h; /* Hash value */ + int nName; /* Length of the name */ assert( nArg>=(-2) ); assert( nArg>=(-1) || createFlag==0 ); - h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); + nName = sqlite3Strlen30(zName); /* First search for a match amongst the application-defined functions. */ - p = functionSearch(&db->aFunc, h, zName, nName); + p = (FuncDef*)sqlite3HashFind(&db->aFunc, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -93646,9 +106197,9 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( ** So we must not search for built-ins when creating a new function. */ if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); bestScore = 0; - p = functionSearch(pHash, h, zName, nName); + h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ; + p = functionSearch(h, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -93665,15 +106216,22 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( */ if( createFlag && bestScorezName = (char *)&pBest[1]; + FuncDef *pOther; + pBest->zName = (const char*)&pBest[1]; pBest->nArg = (u16)nArg; pBest->funcFlags = enc; - memcpy(pBest->zName, zName, nName); - pBest->zName[nName] = 0; - sqlite3FuncDefInsert(&db->aFunc, pBest); + memcpy((char*)&pBest[1], zName, nName+1); + pOther = (FuncDef*)sqlite3HashInsert(&db->aFunc, pBest->zName, pBest); + if( pOther==pBest ){ + sqlite3DbFree(db, pBest); + sqlite3OomFault(db); + return 0; + }else{ + pBest->pNext = pOther; + } } - if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){ + if( pBest && (pBest->xSFunc || createFlag) ){ return pBest; } return 0; @@ -93727,7 +106285,7 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); } if( !p ){ - db->mallocFailed = 1; + sqlite3OomFault(db); }else if ( 0==p->file_format ){ sqlite3HashInit(&p->tblHash); sqlite3HashInit(&p->idxHash); @@ -93754,6 +106312,7 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ ** This file contains C code routines that are called by the parser ** in order to generate code for DELETE FROM statements. */ +/* #include "sqliteInt.h" */ /* ** While a SrcList can in general represent multiple tables and subqueries @@ -93831,7 +106390,7 @@ SQLITE_PRIVATE void sqlite3MaterializeView( Parse *pParse, /* Parsing context */ Table *pView, /* View definition */ Expr *pWhere, /* Optional WHERE clause to be added */ - int iCur /* Cursor number for ephemerial table */ + int iCur /* Cursor number for ephemeral table */ ){ SelectDest dest; Select *pSel; @@ -93843,11 +106402,12 @@ SQLITE_PRIVATE void sqlite3MaterializeView( if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); assert( pFrom->a[0].pOn==0 ); assert( pFrom->a[0].pUsing==0 ); } - pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, + SF_IncludeHidden, 0, 0); sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); sqlite3Select(pParse, pSel, &dest); sqlite3SelectDelete(db, pSel); @@ -93883,7 +106443,7 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( */ if( pOrderBy && (pLimit == 0) ) { sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); - goto limit_where_cleanup_2; + goto limit_where_cleanup; } /* We only need to generate a select expression if there @@ -93905,16 +106465,16 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( */ pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pSelectRowid == 0 ) goto limit_where_cleanup_2; + if( pSelectRowid == 0 ) goto limit_where_cleanup; pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid); - if( pEList == 0 ) goto limit_where_cleanup_2; + if( pEList == 0 ) goto limit_where_cleanup; /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); if( pSelectSrc == 0 ) { sqlite3ExprListDelete(pParse->db, pEList); - goto limit_where_cleanup_2; + goto limit_where_cleanup; } /* generate the SELECT expression tree. */ @@ -93924,21 +106484,11 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pWhereRowid == 0 ) goto limit_where_cleanup_1; - pInClause = sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0); - if( pInClause == 0 ) goto limit_where_cleanup_1; - - pInClause->x.pSelect = pSelect; - pInClause->flags |= EP_xIsSelect; - sqlite3ExprSetHeight(pParse, pInClause); + pInClause = pWhereRowid ? sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0) : 0; + sqlite3PExprAddSelect(pParse, pInClause, pSelect); return pInClause; - /* something went wrong. clean up anything allocated. */ -limit_where_cleanup_1: - sqlite3SelectDelete(pParse->db, pSelect); - return 0; - -limit_where_cleanup_2: +limit_where_cleanup: sqlite3ExprDelete(pParse->db, pWhere); sqlite3ExprListDelete(pParse->db, pOrderBy); sqlite3ExprDelete(pParse->db, pLimit); @@ -93962,13 +106512,12 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ){ Vdbe *v; /* The virtual database engine */ Table *pTab; /* The table from which records will be deleted */ - const char *zDb; /* Name of database holding pTab */ int i; /* Loop counter */ WhereInfo *pWInfo; /* Information about the WHERE clause */ Index *pIdx; /* For looping over indices of the table */ int iTabCur; /* Cursor number for the table */ - int iDataCur; /* VDBE cursor for the canonical data source */ - int iIdxCur; /* Cursor number of the first index */ + int iDataCur = 0; /* VDBE cursor for the canonical data source */ + int iIdxCur = 0; /* Cursor number of the first index */ int nIdx; /* Number of indices */ sqlite3 *db; /* Main database structure */ AuthContext sContext; /* Authorization context */ @@ -93976,7 +106525,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iDb; /* Database number */ int memCnt = -1; /* Memory cell used for change counting */ int rcauth; /* Value returned by authorization callback */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ + int eOnePass; /* ONEPASS_OFF or _SINGLE or _MULTI */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ Index *pPk; /* The PRIMARY KEY index on the table */ @@ -93988,8 +106537,9 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iRowSet = 0; /* Register for rowset of rows to delete */ int addrBypass = 0; /* Address of jump over the delete logic */ int addrLoop = 0; /* Top of the delete loop */ - int addrDelete = 0; /* Jump directly to the delete logic */ - int addrEphOpen = 0; /* Instruction to open the Ephermeral table */ + int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ + int bComplex; /* True if there are triggers or FKs or + ** subqueries in the WHERE clause */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ @@ -94017,6 +106567,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); isView = pTab->pSelect!=0; + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #else # define pTrigger 0 # define isView 0 @@ -94037,8 +106588,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - zDb = db->aDb[iDb].zName; - rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb); + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, + db->aDb[iDb].zDbSName); assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); if( rcauth==SQLITE_DENY ){ goto delete_from_cleanup; @@ -94069,7 +106620,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( sqlite3BeginWriteOperation(pParse, 1, iDb); /* If we are trying to delete from a view, realize that view into - ** a ephemeral table. + ** an ephemeral table. */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ @@ -94100,8 +106651,13 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** It is easier just to erase the whole table. Prior to version 3.6.5, ** this optimization caused the row change count (the value returned by ** API function sqlite3_count_changes) to be set incorrectly. */ - if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) - && 0==sqlite3FkRequired(pParse, pTab, 0, 0) + if( rcauth==SQLITE_OK + && pWhere==0 + && !bComplex + && !IsVirtual(pTab) +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + && db->xPreUpdateCallback==0 +#endif ){ assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); @@ -94116,6 +106672,9 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK|WHERE_SEEK_TABLE; + if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; + wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; @@ -94123,7 +106682,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( iRowSet = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); }else{ - /* For a WITHOUT ROWID table, create an ephermeral table used to + /* For a WITHOUT ROWID table, create an ephemeral table used to ** hold all primary keys for rows to be deleted. */ pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk!=0 ); @@ -94136,13 +106695,18 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } /* Construct a query to find the rowid or primary key for every row - ** to be deleted, based on the WHERE clause. + ** to be deleted, based on the WHERE clause. Set variable eOnePass + ** to indicate the strategy used to implement this delete: + ** + ** ONEPASS_OFF: Two-pass approach - use a FIFO for rowids/PK values. + ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. + ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, - iTabCur+1); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); /* Keep track of the number of rows to be deleted */ if( db->flags & SQLITE_CountRows ){ @@ -94152,6 +106716,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; iaiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pPk->aiColumn[i], iPk+i); } @@ -94162,13 +106727,12 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( iKey>pParse->nMem ) pParse->nMem = iKey; } - if( okOnePass ){ - /* For ONEPASS, no need to store the rowid/primary-key. There is only + if( eOnePass!=ONEPASS_OFF ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only ** one, so just keep it in its register(s) and fall through to the - ** delete code. - */ + ** delete code. */ nKey = nPk; /* OP_Found will use an unpacked key */ - aToOpen = sqlite3DbMallocRaw(db, nIdx+2); + aToOpen = sqlite3DbMallocRawNN(db, nIdx+2); if( aToOpen==0 ){ sqlite3WhereEnd(pWInfo); goto delete_from_cleanup; @@ -94178,27 +106742,27 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); - addrDelete = sqlite3VdbeAddOp0(v, OP_Goto); /* Jump to DELETE logic */ - }else if( pPk ){ - /* Construct a composite key for the row to be deleted and remember it */ - iKey = ++pParse->nMem; - nKey = 0; /* Zero tells OP_Found to use a composite key */ - sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, - sqlite3IndexAffinityStr(v, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); }else{ - /* Get the rowid of the row to be deleted and remember it in the RowSet */ - nKey = 1; /* OP_Seek always uses a single rowid */ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + if( pPk ){ + /* Add the PK key for this row to the temporary table */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(pParse->db, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + }else{ + /* Add the rowid of the row to be deleted to the RowSet */ + nKey = 1; /* OP_Seek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } } - /* End of the WHERE loop */ - sqlite3WhereEnd(pWInfo); - if( okOnePass ){ - /* Bypass the delete logic below if the WHERE loop found zero rows */ + /* If this DELETE cannot use the ONEPASS strategy, this is the + ** end of the WHERE loop */ + if( eOnePass!=ONEPASS_OFF ){ addrBypass = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBypass); - sqlite3VdbeJumpHere(v, addrDelete); + }else{ + sqlite3WhereEnd(pWInfo); } /* Unless this is a view, open cursors for the table we are @@ -94207,20 +106771,25 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** triggers. */ if( !isView ){ - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, - &iDataCur, &iIdxCur); - assert( pPk || iDataCur==iTabCur ); - assert( pPk || iIdxCur==iDataCur+1 ); + int iAddrOnce = 0; + if( eOnePass==ONEPASS_MULTI ){ + iAddrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + testcase( IsVirtual(pTab) ); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, OPFLAG_FORDELETE, + iTabCur, aToOpen, &iDataCur, &iIdxCur); + assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); + assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); + if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce); } /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ - if( okOnePass ){ - /* Just one row. Hence the top-of-loop is a no-op */ - assert( nKey==nPk ); /* OP_Found will use an unpacked key */ - if( aToOpen[iDataCur-iTabCur] ){ - assert( pPk!=0 ); + if( eOnePass!=ONEPASS_OFF ){ + assert( nKey==nPk ); /* OP_Found will use an unpacked key */ + if( !IsVirtual(pTab) && aToOpen[iDataCur-iTabCur] ){ + assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } @@ -94241,23 +106810,32 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, OE_Abort); + assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); sqlite3MayAbort(pParse); + if( eOnePass==ONEPASS_SINGLE && sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } }else #endif { int count = (pParse->nested==0); /* True to count changes */ + int iIdxNoSeek = -1; + if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ + iIdxNoSeek = aiCurOnePass[1]; + } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, okOnePass); + iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); } /* End of the loop over all rowids/primary-keys. */ - if( okOnePass ){ + if( eOnePass!=ONEPASS_OFF ){ sqlite3VdbeResolveLabel(v, addrBypass); + sqlite3WhereEnd(pWInfo); }else if( pPk ){ sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrLoop); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrLoop); + sqlite3VdbeGoto(v, addrLoop); sqlite3VdbeJumpHere(v, addrLoop); } @@ -94296,7 +106874,7 @@ delete_from_cleanup: return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView @@ -94324,6 +106902,25 @@ delete_from_cleanup: ** sequence of nPk memory cells starting at iPk. If nPk==0 that means ** that a search record formed from OP_MakeRecord is contained in the ** single memory location iPk. +** +** eMode: +** Parameter eMode may be passed either ONEPASS_OFF (0), ONEPASS_SINGLE, or +** ONEPASS_MULTI. If eMode is not ONEPASS_OFF, then the cursor +** iDataCur already points to the row to delete. If eMode is ONEPASS_OFF +** then this function must seek iDataCur to the entry identified by iPk +** and nPk before reading from it. +** +** If eMode is ONEPASS_MULTI, then this call is being made as part +** of a ONEPASS delete that affects multiple rows. In this case, if +** iIdxNoSeek is a valid cursor number (>=0), then its position should +** be preserved following the delete operation. Or, if iIdxNoSeek is not +** a valid cursor number, the position of iDataCur should be preserved +** instead. +** +** iIdxNoSeek: +** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an +** index cursor (from within array of cursors starting at iIdxCur) that +** already points to the index entry to be deleted. */ SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -94335,7 +106932,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( i16 nPk, /* Number of PRIMARY KEY memory cells */ u8 count, /* If non-zero, increment the row change counter */ u8 onconf, /* Default ON CONFLICT policy for triggers */ - u8 bNoSeek /* iDataCur is already pointing to the row to delete */ + u8 eMode, /* ONEPASS_OFF, _SINGLE, or _MULTI. See above */ + int iIdxNoSeek /* Cursor number of cursor that does not need seeking */ ){ Vdbe *v = pParse->pVdbe; /* Vdbe */ int iOld = 0; /* First register in OLD.* array */ @@ -94352,7 +106950,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(v); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - if( !bNoSeek ){ + if( eMode==ONEPASS_OFF ){ sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); VdbeCoverageIf(v, opSeek==OP_NotExists); VdbeCoverageIf(v, opSeek==OP_NotFound); @@ -94410,13 +107008,27 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( /* Delete the index and table entries. Skip this step if pTab is really ** a view (in which case the only effect of the DELETE statement is to - ** fire the INSTEAD OF triggers). */ + ** fire the INSTEAD OF triggers). + ** + ** If variable 'count' is non-zero, then this OP_Delete instruction should + ** invoke the update-hook. The pre-update-hook, on the other hand should + ** be invoked unless table pTab is a system table. The difference is that + ** the update-hook is not invoked for rows removed by REPLACE, but the + ** pre-update-hook is. + */ if( pTab->pSelect==0 ){ - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + u8 p5 = 0; + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); - if( count ){ - sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); + sqlite3VdbeChangeP4(v, -1, (char*)pTab, P4_TABLE); + if( eMode!=ONEPASS_OFF ){ + sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); } + if( iIdxNoSeek>=0 ){ + sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); + } + if( eMode==ONEPASS_MULTI ) p5 |= OPFLAG_SAVEPOSITION; + sqlite3VdbeChangeP5(v, p5); } /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to @@ -94459,7 +107071,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( Table *pTab, /* Table containing the row to be deleted */ int iDataCur, /* Cursor of table holding data. */ int iIdxCur, /* First index cursor */ - int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int *aRegIdx, /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int iIdxNoSeek /* Do not delete from this cursor */ ){ int i; /* Index loop counter */ int r1 = -1; /* Register holding an index key */ @@ -94475,11 +107088,12 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( assert( iIdxCur+i!=iDataCur || pPk==pIdx ); if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; if( pIdx==pPk ) continue; + if( iIdxCur+i==iIdxNoSeek ) continue; VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, - &iPartIdxLabel, pPrior, r1); + &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -94528,17 +107142,16 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( ){ Vdbe *v = pParse->pVdbe; int j; - Table *pTab = pIdx->pTable; int regBase; int nCol; if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); - pParse->iPartIdxTab = iDataCur; + pParse->iSelfTab = iDataCur; sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, - SQLITE_JUMPIFNULL); + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + SQLITE_JUMPIFNULL); }else{ *piPartIdxLabel = 0; } @@ -94547,9 +107160,14 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( regBase = sqlite3GetTempRange(pParse, nCol); if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] ) continue; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], - regBase+j); + if( pPrior + && pPrior->aiColumn[j]==pIdx->aiColumn[j] + && pPrior->aiColumn[j]!=XN_EXPR + ){ + /* This column was already computed by the previous index */ + continue; + } + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); /* If the column affinity is REAL but the number is an integer, then it ** might be stored in the table as an integer (using a compact ** representation) then converted to REAL by an OP_RealAffinity opcode. @@ -94590,18 +107208,25 @@ SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the C-language implementions for many of the SQL +** This file contains the C-language implementations for many of the SQL ** functions of SQLite. (Some function, and in particular the date and ** time functions, are implemented separately.) */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ +/* #include "vdbeInt.h" */ /* ** Return the collating function associated with a function. */ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ - return context->pColl; + VdbeOp *pOp; + assert( context->pVdbe!=0 ); + pOp = &context->pVdbe->aOp[context->iOp-1]; + assert( pOp->opcode==OP_CollSeq ); + assert( pOp->p4type==P4_COLLSEQ ); + return pOp->p4.pColl; } /* @@ -94733,8 +107358,8 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ default: { /* Because sqlite3_value_double() returns 0.0 if the argument is not ** something that can be converted into a number, we have: - ** IMP: R-57326-31541 Abs(X) return 0.0 if X is a string or blob that - ** cannot be converted to a numeric value. + ** IMP: R-01992-00519 Abs(X) returns 0.0 if X is a string or blob + ** that cannot be converted to a numeric value. */ double rVal = sqlite3_value_double(argv[0]); if( rVal<0 ) rVal = -rVal; @@ -94782,6 +107407,8 @@ static void instrFunc( zHaystack = sqlite3_value_text(argv[0]); zNeedle = sqlite3_value_text(argv[1]); isText = 1; + if( zNeedle==0 ) return; + assert( zHaystack ); } while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ N++; @@ -94806,14 +107433,15 @@ static void printfFunc( StrAccum str; const char *zFormat; int n; + sqlite3 *db = sqlite3_context_db_handle(context); if( argc>=1 && (zFormat = (const char*)sqlite3_value_text(argv[0]))!=0 ){ x.nArg = argc-1; x.nUsed = 0; x.apArg = argv+1; - sqlite3StrAccumInit(&str, 0, 0, SQLITE_MAX_LENGTH); - str.db = sqlite3_context_db_handle(context); - sqlite3XPrintf(&str, SQLITE_PRINTF_SQLFUNC, zFormat, &x); + sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); + str.printfFlags = SQLITE_PRINTF_SQLFUNC; + sqlite3XPrintf(&str, zFormat, &x); n = str.nChar; sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, SQLITE_DYNAMIC); @@ -94867,6 +107495,14 @@ static void substrFunc( } } } +#ifdef SQLITE_SUBSTR_COMPATIBILITY + /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as + ** as substr(X,1,N) - it returns the first N characters of X. This + ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] + ** from 2009-02-02 for compatibility of applications that exploited the + ** old buggy behavior. */ + if( p1==0 ) p1 = 1; /* */ +#endif if( argc==3 ){ p2 = sqlite3_value_int(argv[2]); if( p2<0 ){ @@ -94904,13 +107540,14 @@ static void substrFunc( for(z2=z; *z2 && p2; p2--){ SQLITE_SKIP_UTF8(z2); } - sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT); + sqlite3_result_text64(context, (char*)z, z2-z, SQLITE_TRANSIENT, + SQLITE_UTF8); }else{ if( p1+p2>len ){ p2 = len-p1; if( p2<0 ) p2 = 0; } - sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT); + sqlite3_result_blob64(context, (char*)&z[p1], (u64)p2, SQLITE_TRANSIENT); } } @@ -94953,7 +107590,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ #endif /* -** Allocate nByte bytes of space using sqlite3_malloc(). If the +** Allocate nByte bytes of space using sqlite3Malloc(). If the ** allocation fails, call sqlite3_result_error_nomem() to notify ** the database handle that malloc() has failed and return NULL. ** If nByte is larger than the maximum string or blob length, then @@ -94969,7 +107606,7 @@ static void *contextMalloc(sqlite3_context *context, i64 nByte){ sqlite3_result_error_toobig(context); z = 0; }else{ - z = sqlite3Malloc((int)nByte); + z = sqlite3Malloc(nByte); if( !z ){ sqlite3_result_error_nomem(context); } @@ -95132,23 +107769,23 @@ static void total_changes( ** A structure defining how to do GLOB-style comparisons. */ struct compareInfo { - u8 matchAll; - u8 matchOne; - u8 matchSet; - u8 noCase; + u8 matchAll; /* "*" or "%" */ + u8 matchOne; /* "?" or "_" */ + u8 matchSet; /* "[" or 0 */ + u8 noCase; /* true to ignore case differences */ }; /* ** For LIKE and GLOB matching on EBCDIC machines, assume that every -** character is exactly one byte in size. Also, all characters are -** able to participate in upper-case-to-lower-case mappings in EBCDIC -** whereas only characters less than 0x80 do in ASCII. +** character is exactly one byte in size. Also, provde the Utf8Read() +** macro for fast reading of the next character in the common case where +** the next character is ASCII. */ #if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A) (*((*A)++)) -# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] +# define sqlite3Utf8Read(A) (*((*A)++)) +# define Utf8Read(A) (*(A++)) #else -# define GlobUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } +# define Utf8Read(A) (A[0]<0x80?*(A++):sqlite3Utf8Read(&A)) #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -95161,7 +107798,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; /* ** Compare two UTF-8 strings for equality where the first string can -** potentially be a "glob" expression. Return true (1) if they +** potentially be a "glob" or "like" expression. Return true (1) if they ** are the same and false (0) if they are different. ** ** Globbing rules: @@ -95181,116 +107818,134 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; ** "[a-z]" matches any single lower-case letter. To match a '-', make ** it the last character in the list. ** +** Like matching rules: +** +** '%' Matches any sequence of zero or more characters +** +*** '_' Matches any one character +** +** Ec Where E is the "esc" character and c is any other +** character, including '%', '_', and esc, match exactly c. +** +** The comments within this routine usually assume glob matching. +** ** This routine is usually quick, but can be N**2 in the worst case. -** -** Hints: to match '*' or '?', put them in "[]". Like this: -** -** abc[*]xyz Matches "abc*xyz" only */ static int patternCompare( const u8 *zPattern, /* The glob pattern */ const u8 *zString, /* The string to compare against the glob */ const struct compareInfo *pInfo, /* Information about how to do the compare */ - u32 esc /* The escape character */ + u32 matchOther /* The escape char (LIKE) or '[' (GLOB) */ ){ - u32 c, c2; - int invert; - int seen; - u8 matchOne = pInfo->matchOne; - u8 matchAll = pInfo->matchAll; - u8 matchSet = pInfo->matchSet; - u8 noCase = pInfo->noCase; - int prevEscape = 0; /* True if the previous character was 'escape' */ - - while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ - if( c==matchAll && !prevEscape ){ - while( (c=sqlite3Utf8Read(&zPattern)) == matchAll - || c == matchOne ){ + u32 c, c2; /* Next pattern and input string chars */ + u32 matchOne = pInfo->matchOne; /* "?" or "_" */ + u32 matchAll = pInfo->matchAll; /* "*" or "%" */ + u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ + const u8 *zEscaped = 0; /* One past the last escaped input char */ + + while( (c = Utf8Read(zPattern))!=0 ){ + if( c==matchAll ){ /* Match "*" */ + /* Skip over multiple "*" characters in the pattern. If there + ** are also "?" characters, skip those as well, but consume a + ** single character of the input string for each "?" skipped */ + while( (c=Utf8Read(zPattern)) == matchAll || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ return 0; } } if( c==0 ){ - return 1; - }else if( c==esc ){ - c = sqlite3Utf8Read(&zPattern); - if( c==0 ){ - return 0; - } - }else if( c==matchSet ){ - assert( esc==0 ); /* This is GLOB, not LIKE */ - assert( matchSet<0x80 ); /* '[' is a single-byte character */ - while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ - SQLITE_SKIP_UTF8(zString); - } - return *zString!=0; - } - while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ - if( noCase ){ - GlobUpperToLower(c2); - GlobUpperToLower(c); - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(&zString); - GlobUpperToLower(c2); - } + return 1; /* "*" at the end of the pattern matches */ + }else if( c==matchOther ){ + if( pInfo->matchSet==0 ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; }else{ - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(&zString); + /* "[...]" immediately follows the "*". We have to do a slow + ** recursive search in this case, but it is an unusual case. */ + assert( matchOther<0x80 ); /* '[' is a single-byte character */ + while( *zString + && patternCompare(&zPattern[-1],zString,pInfo,matchOther)==0 ){ + SQLITE_SKIP_UTF8(zString); } + return *zString!=0; + } + } + + /* At this point variable c contains the first character of the + ** pattern string past the "*". Search in the input string for the + ** first matching character and recursively contine the match from + ** that point. + ** + ** For a case-insensitive search, set variable cx to be the same as + ** c but in the other case and search the input string for either + ** c or cx. + */ + if( c<=0x80 ){ + u32 cx; + if( noCase ){ + cx = sqlite3Toupper(c); + c = sqlite3Tolower(c); + }else{ + cx = c; + } + while( (c2 = *(zString++))!=0 ){ + if( c2!=c && c2!=cx ) continue; + if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; + } + }else{ + while( (c2 = Utf8Read(zString))!=0 ){ + if( c2!=c ) continue; + if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; } - if( c2==0 ) return 0; - if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( c==matchOne && !prevEscape ){ - if( sqlite3Utf8Read(&zString)==0 ){ - return 0; - } - }else if( c==matchSet ){ - u32 prior_c = 0; - assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ - seen = 0; - invert = 0; - c = sqlite3Utf8Read(&zString); - if( c==0 ) return 0; - c2 = sqlite3Utf8Read(&zPattern); - if( c2=='^' ){ - invert = 1; - c2 = sqlite3Utf8Read(&zPattern); - } - if( c2==']' ){ - if( c==']' ) seen = 1; - c2 = sqlite3Utf8Read(&zPattern); - } - while( c2 && c2!=']' ){ - if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ - c2 = sqlite3Utf8Read(&zPattern); - if( c>=prior_c && c<=c2 ) seen = 1; - prior_c = 0; - }else{ - if( c==c2 ){ - seen = 1; - } - prior_c = c2; - } - c2 = sqlite3Utf8Read(&zPattern); - } - if( c2==0 || (seen ^ invert)==0 ){ - return 0; - } - }else if( esc==c && !prevEscape ){ - prevEscape = 1; - }else{ - c2 = sqlite3Utf8Read(&zString); - if( noCase ){ - GlobUpperToLower(c); - GlobUpperToLower(c2); - } - if( c!=c2 ){ - return 0; - } - prevEscape = 0; } + if( c==matchOther ){ + if( pInfo->matchSet==0 ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; + zEscaped = zPattern; + }else{ + u32 prior_c = 0; + int seen = 0; + int invert = 0; + c = sqlite3Utf8Read(&zString); + if( c==0 ) return 0; + c2 = sqlite3Utf8Read(&zPattern); + if( c2=='^' ){ + invert = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ + c2 = sqlite3Utf8Read(&zPattern); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==0 || (seen ^ invert)==0 ){ + return 0; + } + continue; + } + } + c2 = Utf8Read(zString); + if( c==c2 ) continue; + if( noCase && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){ + continue; + } + if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; + return 0; } return *zString==0; } @@ -95299,7 +107954,14 @@ static int patternCompare( ** The sqlite3_strglob() interface. */ SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ - return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '[')==0; +} + +/* +** The sqlite3_strlike() interface. +*/ +SQLITE_API int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc)==0; } /* @@ -95330,10 +107992,22 @@ static void likeFunc( sqlite3_value **argv ){ const unsigned char *zA, *zB; - u32 escape = 0; + u32 escape; int nPat; sqlite3 *db = sqlite3_context_db_handle(context); + struct compareInfo *pInfo = sqlite3_user_data(context); +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + || sqlite3_value_type(argv[1])==SQLITE_BLOB + ){ +#ifdef SQLITE_TEST + sqlite3_like_count++; +#endif + sqlite3_result_int(context, 0); + return; + } +#endif zB = sqlite3_value_text(argv[0]); zA = sqlite3_value_text(argv[1]); @@ -95361,13 +108035,13 @@ static void likeFunc( return; } escape = sqlite3Utf8Read(&zEsc); + }else{ + escape = pInfo->matchSet; } if( zA && zB ){ - struct compareInfo *pInfo = sqlite3_user_data(context); #ifdef SQLITE_TEST sqlite3_like_count++; #endif - sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)); } } @@ -95593,7 +108267,7 @@ static void charFunc( ){ unsigned char *z, *zOut; int i; - zOut = z = sqlite3_malloc( argc*4+1 ); + zOut = z = sqlite3_malloc64( argc*4+1 ); if( z==0 ){ sqlite3_result_error_nomem(context); return; @@ -95620,7 +108294,7 @@ static void charFunc( *zOut++ = 0x80 + (u8)(c & 0x3F); } \ } - sqlite3_result_text(context, (char*)z, (int)(zOut-z), sqlite3_free); + sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); } /* @@ -95661,16 +108335,14 @@ static void zeroblobFunc( sqlite3_value **argv ){ i64 n; - sqlite3 *db = sqlite3_context_db_handle(context); + int rc; assert( argc==1 ); UNUSED_PARAMETER(argc); n = sqlite3_value_int64(argv[0]); - testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); - if( n>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - }else{ - sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */ + if( n<0 ) n = 0; + rc = sqlite3_result_zeroblob64(context, n); /* IMP: R-00293-64994 */ + if( rc ){ + sqlite3_result_error_code(context, rc); } } @@ -95741,7 +108413,7 @@ static void replaceFunc( return; } zOld = zOut; - zOut = sqlite3_realloc(zOut, (int)nOut); + zOut = sqlite3_realloc64(zOut, (int)nOut); if( zOut==0 ){ sqlite3_result_error_nomem(context); sqlite3_free(zOld); @@ -95845,6 +108517,26 @@ static void trimFunc( } +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +/* +** The "unknown" function is automatically substituted in place of +** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN +** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used. +** When the "sqlite3" command-line shell is built using this functionality, +** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries +** involving application-defined functions to be examined in a generic +** sqlite3 shell. +*/ +static void unknownFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + /* no-op */ +} +#endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ + + /* IMP: R-25361-16150 This function is omitted from SQLite by default. It ** is only available if the SQLITE_SOUNDEX compile-time option is used ** when SQLite is built. @@ -95915,6 +108607,14 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3 *db = sqlite3_context_db_handle(context); char *zErrMsg = 0; + /* Disallow the load_extension() SQL function unless the SQLITE_LoadExtFunc + ** flag is set. See the sqlite3_enable_load_extension() API. + */ + if( (db->flags & SQLITE_LoadExtFunc)==0 ){ + sqlite3_result_error(context, "not authorized", -1); + return; + } + if( argc==2 ){ zProc = (const char *)sqlite3_value_text(argv[1]); }else{ @@ -96070,6 +108770,7 @@ static void minmaxStep( sqlite3SkipAccumulatorLoad(context); } }else{ + pBest->db = sqlite3_context_db_handle(context); sqlite3VdbeMemCopy(pBest, pArg); } } @@ -96102,8 +108803,7 @@ static void groupConcatStep( if( pAccum ){ sqlite3 *db = sqlite3_context_db_handle(context); - int firstTerm = pAccum->useMalloc==0; - pAccum->useMalloc = 2; + int firstTerm = pAccum->mxAlloc==0; pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; if( !firstTerm ){ if( argc==2 ){ @@ -96140,7 +108840,7 @@ static void groupConcatFinalize(sqlite3_context *context){ ** of the built-in functions above are part of the global function set. ** This routine only deals with those that are not global. */ -SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ +SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ int rc = sqlite3_overload_function(db, "MATCH", 2); /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC @@ -96151,7 +108851,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ /* END SQLCIPHER */ assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC @@ -96167,8 +108867,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ */ static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ FuncDef *pDef; - pDef = sqlite3FindFunction(db, zName, sqlite3Strlen30(zName), - 2, SQLITE_UTF8, 0); + pDef = sqlite3FindFunction(db, zName, 2, SQLITE_UTF8, 0); if( ALWAYS(pDef) ){ pDef->funcFlags |= flagVal; } @@ -96201,6 +108900,11 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive) ** then set aWc[0] through aWc[2] to the wildcard characters and ** return TRUE. If the function is not a LIKE-style function then ** return FALSE. +** +** *pIsNocase is set to true if uppercase and lowercase are equivalent for +** the function (default for LIKE). If the function makes the distinction +** between uppercase and lowercase (as does GLOB) then *pIsNocase is set to +** false. */ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ FuncDef *pDef; @@ -96211,9 +108915,7 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas return 0; } assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - pDef = sqlite3FindFunction(db, pExpr->u.zToken, - sqlite3Strlen30(pExpr->u.zToken), - 2, SQLITE_UTF8, 0); + pDef = sqlite3FindFunction(db, pExpr->u.zToken, 2, SQLITE_UTF8, 0); if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ return 0; } @@ -96231,13 +108933,13 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas } /* -** All all of the FuncDef structures in the aBuiltinFunc[] array above +** All of the FuncDef structures in the aBuiltinFunc[] array above ** to the global function hash table. This occurs at start-time (as ** a consequence of calling sqlite3_initialize()). ** ** After this routine runs */ -SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ /* ** The following array holds FuncDef structures for all of the functions ** defined in this file. @@ -96245,8 +108947,27 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** The array cannot be constant since changes are made to the ** FuncDef.pHash elements at start-time. The elements of this array ** are read-only after initialization is complete. + ** + ** For peak efficiency, put the most frequently used function last. */ - static SQLITE_WSD FuncDef aBuiltinFunc[] = { + static FuncDef aBuiltinFunc[] = { +#ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION + VFUNCTION(load_extension, 1, 0, 0, loadExt ), + VFUNCTION(load_extension, 2, 0, 0, loadExt ), +#endif +#if SQLITE_USER_AUTHENTICATION + FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), +#endif +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), FUNCTION(ltrim, 1, 1, 0, trimFunc ), FUNCTION(ltrim, 2, 1, 0, trimFunc ), FUNCTION(rtrim, 1, 2, 0, trimFunc ), @@ -96255,15 +108976,15 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION(trim, 2, 3, 0, trimFunc ), FUNCTION(min, -1, 0, 1, minmaxFunc ), FUNCTION(min, 0, 0, 1, 0 ), - AGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize ), + AGGREGATE2(min, 1, 0, 1, minmaxStep, minMaxFinalize, + SQLITE_FUNC_MINMAX ), FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), - AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), + AGGREGATE2(max, 1, 1, 1, minmaxStep, minMaxFinalize, + SQLITE_FUNC_MINMAX ), FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(instr, 2, 0, 0, instrFunc ), - FUNCTION(substr, 2, 0, 0, substrFunc ), - FUNCTION(substr, 3, 0, 0, substrFunc ), FUNCTION(printf, -1, 0, 0, printfFunc ), FUNCTION(unicode, 1, 0, 0, unicodeFunc ), FUNCTION(char, -1, 0, 0, charFunc ), @@ -96274,69 +108995,69 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ #endif FUNCTION(upper, 1, 0, 0, upperFunc ), FUNCTION(lower, 1, 0, 0, lowerFunc ), - FUNCTION(coalesce, 1, 0, 0, 0 ), - FUNCTION(coalesce, 0, 0, 0, 0 ), - FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), - FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), - FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), - FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), - FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), VFUNCTION(changes, 0, 0, 0, changes ), VFUNCTION(total_changes, 0, 0, 0, total_changes ), FUNCTION(replace, 3, 0, 0, replaceFunc ), FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), - #ifdef SQLITE_SOUNDEX - FUNCTION(soundex, 1, 0, 0, soundexFunc ), - #endif - #ifndef SQLITE_OMIT_LOAD_EXTENSION - FUNCTION(load_extension, 1, 0, 0, loadExt ), - FUNCTION(load_extension, 2, 0, 0, loadExt ), - #endif + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), - /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ - {0,SQLITE_UTF8|SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, + AGGREGATE2(count, 0, 0, 0, countStep, countFinalize, + SQLITE_FUNC_COUNT ), AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), - #ifdef SQLITE_CASE_SENSITIVE_LIKE +#ifdef SQLITE_CASE_SENSITIVE_LIKE LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), - #else +#else LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), - #endif +#endif +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + FUNCTION(unknown, -1, 0, 0, unknownFunc ), +#endif + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, 0, 0, 0, 0 ), + FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), }; - - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aBuiltinFunc); - - for(i=0; iu.pHash){ + int n = sqlite3Strlen30(p->zName); + int h = p->zName[0] + n; + printf(" %s(%d)", p->zName, h); + } + printf("\n"); + } + } #endif } @@ -96355,6 +109076,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** This file contains code used by the compiler to add foreign key ** support to compiled SQL statements. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_FOREIGN_KEY #ifndef SQLITE_OMIT_TRIGGER @@ -96516,7 +109238,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** ** 4) No parent key columns were provided explicitly as part of the ** foreign key definition, and the PRIMARY KEY of the parent table -** consists of a a different number of columns to the child key in +** consists of a different number of columns to the child key in ** the child table. ** ** then non-zero is returned, and a "foreign key mismatch" error loaded @@ -96562,7 +109284,7 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( } }else if( paiCol ){ assert( nCol>1 ); - aiCol = (int *)sqlite3DbMallocRaw(pParse->db, nCol*sizeof(int)); + aiCol = (int *)sqlite3DbMallocRawNN(pParse->db, nCol*sizeof(int)); if( !aiCol ) return 1; *paiCol = aiCol; } @@ -96592,16 +109314,16 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( int i, j; for(i=0; iaiColumn[i]; /* Index of column in parent tbl */ - char *zDfltColl; /* Def. collation for column */ + const char *zDfltColl; /* Def. collation for column */ char *zIdxCol; /* Name of indexed column */ + if( iCol<0 ) break; /* No foreign keys against expression indexes */ + /* If the index uses a collation sequence that is different from ** the default collation sequence for the column, this index is ** unusable. Bail out early in this case. */ zDfltColl = pParent->aCol[iCol].zColl; - if( !zDfltColl ){ - zDfltColl = "BINARY"; - } + if( !zDfltColl ) zDfltColl = sqlite3StrBINARY; if( sqlite3StrICmp(pIdx->azColl[i], zDfltColl) ) break; zIdxCol = pParent->aCol[iCol].zName; @@ -96717,7 +109439,7 @@ static void fkLookupParent( sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeGoto(v, iOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); sqlite3VdbeJumpHere(v, iMustBeInt); sqlite3ReleaseTempReg(pParse, regTemp); @@ -96747,6 +109469,7 @@ static void fkLookupParent( for(i=0; iaiColumn[i]+1+regData; + assert( pIdx->aiColumn[i]>=0 ); assert( aiCol[i]!=pTab->iPKey ); if( pIdx->aiColumn[i]==pTab->iPKey ){ /* The parent key is a composite key that includes the IPK column */ @@ -96755,11 +109478,11 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeGoto(v, iOk); } sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, - sqlite3IndexAffinityStr(v,pIdx), nCol); + sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, regRec); @@ -96780,7 +109503,7 @@ static void fkLookupParent( OE_Abort, 0, P4_STATIC, P5_ConstraintFK); }else{ if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; + sqlite3MayAbort(pParse); } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); } @@ -96852,6 +109575,10 @@ static Expr *exprTableColumn( ** code for an SQL UPDATE operation, this function may be called twice - ** once to "delete" the old row and once to "insert" the new row. ** +** Parameter nIncr is passed -1 when inserting a row (as this may decrease +** the number of FK violations in the db) or +1 when deleting one (as this +** may increase the number of FK constraint problems). +** ** The code generated by this function scans through the rows in the child ** table that correspond to the parent table row being deleted or inserted. ** For each child row found, one of the following actions is taken: @@ -96951,6 +109678,7 @@ static void fkScanChildren( assert( pIdx!=0 ); for(i=0; inKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; + assert( iCol>=0 ); pLeft = exprTableRegister(pParse, pTab, regData, iCol); pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); @@ -96968,13 +109696,9 @@ static void fkScanChildren( sqlite3ResolveExprNames(&sNameContext, pWhere); /* Create VDBE to loop through the entries in pSrc that match the WHERE - ** clause. If the constraint is not deferred, throw an exception for - ** each row found. Otherwise, for deferred constraints, increment the - ** deferred constraint counter by nIncr for each row selected. */ + ** clause. For each row found, increment either the deferred or immediate + ** foreign key constraint counter. */ pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); - if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; - } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); if( pWInfo ){ sqlite3WhereEnd(pWInfo); @@ -97002,8 +109726,7 @@ static void fkScanChildren( ** table). */ SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *pTab){ - int nName = sqlite3Strlen30(pTab->zName); - return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName, nName); + return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName); } /* @@ -97154,6 +109877,24 @@ static int fkParentIsModified( return 0; } +/* +** Return true if the parser passed as the first argument is being +** used to code a trigger that is really a "SET NULL" action belonging +** to trigger pFKey. +*/ +static int isSetNullAction(Parse *pParse, FKey *pFKey){ + Parse *pTop = sqlite3ParseToplevel(pParse); + if( pTop->pTriggerPrg ){ + Trigger *p = pTop->pTriggerPrg->pTrigger; + if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) + || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) + ){ + return 1; + } + } + return 0; +} + /* ** This function is called when inserting, deleting or updating a row of ** table pTab to generate VDBE code to perform foreign key constraint @@ -97195,7 +109936,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( if( (db->flags&SQLITE_ForeignKeys)==0 ) return; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; /* Loop through all the foreign key constraints for which pTab is the ** child table (the table that the foreign key definition is part of). */ @@ -97206,7 +109947,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( int *aiCol; int iCol; int i; - int isIgnore = 0; + int bIgnore = 0; if( aChange && sqlite3_stricmp(pTab->zName, pFKey->zTo)!=0 @@ -97257,6 +109998,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( if( aiCol[i]==pTab->iPKey ){ aiCol[i] = -1; } + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Request permission to read the parent key columns. If the ** authorization callback returns SQLITE_IGNORE, behave as if any @@ -97265,7 +110007,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( int rcauth; char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zName; rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); - isIgnore = (rcauth==SQLITE_IGNORE); + bIgnore = (rcauth==SQLITE_IGNORE); } #endif } @@ -97280,12 +110022,18 @@ SQLITE_PRIVATE void sqlite3FkCheck( /* A row is being removed from the child table. Search for the parent. ** If the parent does not exist, removing the child row resolves an ** outstanding foreign key constraint violation. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1,isIgnore); + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1, bIgnore); } - if( regNew!=0 ){ + if( regNew!=0 && !isSetNullAction(pParse, pFKey) ){ /* A row is being added to the child table. If a parent row cannot - ** be found, adding the child row has violated the FK constraint. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1,isIgnore); + ** be found, adding the child row has violated the FK constraint. + ** + ** If this operation is being performed as part of a trigger program + ** that is actually a "SET NULL" action belonging to this very + ** foreign key, then omit this scan altogether. As all child key + ** values are guaranteed to be NULL, it is not possible for adding + ** this row to cause an FK violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1, bIgnore); } sqlite3DbFree(db, aiFree); @@ -97306,8 +110054,8 @@ SQLITE_PRIVATE void sqlite3FkCheck( && !pParse->pToplevel && !pParse->isMultiWrite ){ assert( regOld==0 && regNew!=0 ); - /* Inserting a single row into a parent table cannot cause an immediate - ** foreign key violation. So do nothing in this case. */ + /* Inserting a single row into a parent table cannot cause (or fix) + ** an immediate foreign key violation. So do nothing in this case. */ continue; } @@ -97331,13 +110079,28 @@ SQLITE_PRIVATE void sqlite3FkCheck( fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); } if( regOld!=0 ){ - /* If there is a RESTRICT action configured for the current operation - ** on the parent table of this FK, then throw an exception - ** immediately if the FK constraint is violated, even if this is a - ** deferred trigger. That's what RESTRICT means. To defer checking - ** the constraint, the FK should specify NO ACTION (represented - ** using OE_None). NO ACTION is the default. */ + int eAction = pFKey->aAction[aChange!=0]; fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); + /* If this is a deferred FK constraint, or a CASCADE or SET NULL + ** action applies, then any foreign key violations caused by + ** removing the parent key will be rectified by the action trigger. + ** So do not set the "may-abort" flag in this case. + ** + ** Note 1: If the FK is declared "ON UPDATE CASCADE", then the + ** may-abort flag will eventually be set on this statement anyway + ** (when this function is called as part of processing the UPDATE + ** within the action trigger). + ** + ** Note 2: At first glance it may seem like SQLite could simply omit + ** all OP_FkCounter related scans when either CASCADE or SET NULL + ** applies. The trouble starts if the CASCADE or SET NULL action + ** trigger causes other triggers or action rules attached to the + ** child table to fire. In these cases the fk constraint counters + ** might be set incorrectly if any OP_FkCounter related scans are + ** omitted. */ + if( !pFKey->isDeferred && eAction!=OE_Cascade && eAction!=OE_SetNull ){ + sqlite3MayAbort(pParse); + } } pItem->zName = 0; sqlite3SrcListDelete(db, pSrc); @@ -97367,7 +110130,10 @@ SQLITE_PRIVATE u32 sqlite3FkOldmask( Index *pIdx = 0; sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ - for(i=0; inKeyCol; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); + for(i=0; inKeyCol; i++){ + assert( pIdx->aiColumn[i]>=0 ); + mask |= COLUMN_MASK(pIdx->aiColumn[i]); + } } } } @@ -97461,10 +110227,12 @@ static Trigger *fkActionTrigger( int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ action = pFKey->aAction[iAction]; + if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ + return 0; + } pTrigger = pFKey->apTrigger[iAction]; if( action!=OE_None && !pTrigger ){ - u8 enableLookaside; /* Copy of db->lookaside.bEnabled */ char const *zFrom; /* Name of child table */ int nFrom; /* Length in bytes of zFrom */ Index *pIdx = 0; /* Parent key index for this FK */ @@ -97489,11 +110257,11 @@ static Trigger *fkActionTrigger( iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iFromCol>=0 ); - tToCol.z = pIdx ? pTab->aCol[pIdx->aiColumn[i]].zName : "oid"; - tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; - - tToCol.n = sqlite3Strlen30(tToCol.z); - tFromCol.n = sqlite3Strlen30(tFromCol.z); + assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); + sqlite3TokenInit(&tToCol, + pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName); + sqlite3TokenInit(&tFromCol, pFKey->pFrom->aCol[iFromCol].zName); /* Create the expression "OLD.zToCol = zFromCol". It is important ** that the "OLD.zToCol" term is on the LHS of the = operator, so @@ -97501,10 +110269,10 @@ static Trigger *fkActionTrigger( ** parent table are used for the comparison. */ pEq = sqlite3PExpr(pParse, TK_EQ, sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) + sqlite3ExprAlloc(db, TK_ID, &tOld, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0) , 0), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tFromCol) + sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0) , 0); pWhere = sqlite3ExprAnd(db, pWhere, pEq); @@ -97516,12 +110284,12 @@ static Trigger *fkActionTrigger( if( pChanges ){ pEq = sqlite3PExpr(pParse, TK_IS, sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), + sqlite3ExprAlloc(db, TK_ID, &tOld, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0), 0), sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), + sqlite3ExprAlloc(db, TK_ID, &tNew, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0), 0), 0); pWhen = sqlite3ExprAnd(db, pWhen, pEq); @@ -97531,18 +110299,18 @@ static Trigger *fkActionTrigger( Expr *pNew; if( action==OE_Cascade ){ pNew = sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) + sqlite3ExprAlloc(db, TK_ID, &tNew, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0) , 0); }else if( action==OE_SetDflt ){ Expr *pDflt = pFKey->pFrom->aCol[iFromCol].pDflt; if( pDflt ){ pNew = sqlite3ExprDup(db, pDflt, 0); }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); } }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); } pList = sqlite3ExprListAppend(pParse, pList, pNew); sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); @@ -97573,19 +110341,17 @@ static Trigger *fkActionTrigger( } /* Disable lookaside memory allocation */ - enableLookaside = db->lookaside.bEnabled; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable++; pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ sizeof(TriggerStep) + /* Single step in trigger program */ - nFrom + 1 /* Space for pStep->target.z */ + nFrom + 1 /* Space for pStep->zTarget */ ); if( pTrigger ){ pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; - pStep->target.z = (char *)&pStep[1]; - pStep->target.n = nFrom; - memcpy((char *)pStep->target.z, zFrom, nFrom); + pStep->zTarget = (char *)&pStep[1]; + memcpy((char *)pStep->zTarget, zFrom, nFrom); pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); @@ -97597,7 +110363,7 @@ static Trigger *fkActionTrigger( } /* Re-enable the lookaside buffer, if it was disabled earlier. */ - db->lookaside.bEnabled = enableLookaside; + db->lookaside.bDisable--; sqlite3ExprDelete(db, pWhere); sqlite3ExprDelete(db, pWhen); @@ -97671,7 +110437,8 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pFKey; /* Iterator variable */ FKey *pNext; /* Copy of pFKey->pNextFrom */ - assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); + assert( db==0 || IsVirtual(pTab) + || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ /* Remove the FK from the fkeyHash hash table. */ @@ -97681,7 +110448,7 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ }else{ void *p = (void *)pFKey->pNextTo; const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); } if( pFKey->pNextTo ){ pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; @@ -97721,6 +110488,7 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ ** This file contains C code routines that are called by the parser ** to handle INSERT statements in SQLite. */ +/* #include "sqliteInt.h" */ /* ** Generate code that will @@ -97750,7 +110518,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk!=0 ); - assert( pPk->tnum=pTab->tnum ); + assert( pPk->tnum==pTab->tnum ); sqlite3VdbeAddOp3(v, opcode, iCur, pPk->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pPk); VdbeComment((v, "%s", pTab->zName)); @@ -97764,20 +110532,20 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** ** Character Column affinity ** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL +** 'A' BLOB +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'F' REAL ** -** An extra 'd' is appended to the end of the string to cover the +** An extra 'D' is appended to the end of the string to cover the ** rowid that appears as the last column in every index. ** ** Memory for the buffer containing the column index affinity string ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ if( !pIdx->zColAff ){ /* The first time a column affinity string for a particular index is ** required, it is allocated and populated here. It is then stored as @@ -97789,15 +110557,25 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ */ int n; Table *pTab = pIdx->pTable; - sqlite3 *db = sqlite3VdbeDb(v); pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return 0; } for(n=0; nnColumn; n++){ i16 x = pIdx->aiColumn[n]; - pIdx->zColAff[n] = x<0 ? SQLITE_AFF_INTEGER : pTab->aCol[x].affinity; + if( x>=0 ){ + pIdx->zColAff[n] = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + pIdx->zColAff[n] = SQLITE_AFF_INTEGER; + }else{ + char aff; + assert( x==XN_EXPR ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + if( aff==0 ) aff = SQLITE_AFF_BLOB; + pIdx->zColAff[n] = aff; + } } pIdx->zColAff[n] = 0; } @@ -97807,9 +110585,9 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ /* ** Compute the affinity string for table pTab, if it has not already been -** computed. As an optimization, omit trailing SQLITE_AFF_NONE affinities. +** computed. As an optimization, omit trailing SQLITE_AFF_BLOB affinities. ** -** If the affinity exists (if it is no entirely SQLITE_AFF_NONE values) and +** If the affinity exists (if it is no entirely SQLITE_AFF_BLOB values) and ** if iReg>0 then code an OP_Affinity opcode that will set the affinities ** for register iReg and following. Or if affinities exists and iReg==0, ** then just set the P4 operand of the previous opcode (which should be @@ -97819,11 +110597,11 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ ** ** Character Column affinity ** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL +** 'A' BLOB +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'E' REAL */ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ int i; @@ -97832,7 +110610,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ sqlite3 *db = sqlite3VdbeDb(v); zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return; } @@ -97841,7 +110619,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } do{ zColAff[i--] = 0; - }while( i>=0 && zColAff[i]==SQLITE_AFF_NONE ); + }while( i>=0 && zColAff[i]==SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } i = sqlite3Strlen30(zColAff); @@ -97898,7 +110676,9 @@ static int readsTable(Parse *p, int iDb, Table *pTab){ /* ** Locate or create an AutoincInfo structure associated with table pTab ** which is in database iDb. Return the register number for the register -** that holds the maximum rowid. +** that holds the maximum rowid. Return zero if pTab is not an AUTOINCREMENT +** table. (Also return zero when doing a VACUUM since we do not want to +** update the AUTOINCREMENT counters during a VACUUM.) ** ** There is at most one AutoincInfo structure per table even if the ** same table is autoincremented multiple times due to inserts within @@ -97921,14 +110701,16 @@ static int autoIncBegin( Table *pTab /* The table we are writing to */ ){ int memId = 0; /* Register holding maximum rowid */ - if( pTab->tabFlags & TF_Autoincrement ){ + if( (pTab->tabFlags & TF_Autoincrement)!=0 + && (pParse->db->flags & SQLITE_Vacuum)==0 + ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); AutoincInfo *pInfo; pInfo = pToplevel->pAinc; while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } if( pInfo==0 ){ - pInfo = sqlite3DbMallocRaw(pParse->db, sizeof(*pInfo)); + pInfo = sqlite3DbMallocRawNN(pParse->db, sizeof(*pInfo)); if( pInfo==0 ) return 0; pInfo->pNext = pToplevel->pAinc; pToplevel->pAinc = pInfo; @@ -97952,43 +110734,55 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* Database only autoinc table */ int memId; /* Register holding max rowid */ - int addr; /* A VDBE address */ Vdbe *v = pParse->pVdbe; /* VDBE under construction */ /* This routine is never called during trigger-generation. It is ** only called from the top-level */ assert( pParse->pTriggerTab==0 ); - assert( pParse==sqlite3ParseToplevel(pParse) ); + assert( sqlite3IsToplevel(pParse) ); assert( v ); /* We failed long ago if this is not so */ for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoInc[] = { + /* 0 */ {OP_Null, 0, 0, 0}, + /* 1 */ {OP_Rewind, 0, 9, 0}, + /* 2 */ {OP_Column, 0, 0, 0}, + /* 3 */ {OP_Ne, 0, 7, 0}, + /* 4 */ {OP_Rowid, 0, 0, 0}, + /* 5 */ {OP_Column, 0, 1, 0}, + /* 6 */ {OP_Goto, 0, 9, 0}, + /* 7 */ {OP_Next, 0, 2, 0}, + /* 8 */ {OP_Integer, 0, 0, 0}, + /* 9 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; pDb = &db->aDb[p->iDb]; memId = p->regCtr; assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); - addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); - sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); - sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); - sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); - sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); - sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); - sqlite3VdbeAddOp0(v, OP_Close); + sqlite3VdbeLoadString(v, memId-1, p->pTab->zName); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoInc), autoInc, iLn); + if( aOp==0 ) break; + aOp[0].p2 = memId; + aOp[0].p3 = memId+1; + aOp[2].p3 = memId; + aOp[3].p1 = memId-1; + aOp[3].p3 = memId; + aOp[3].p5 = SQLITE_JUMPIFNULL; + aOp[4].p2 = memId+1; + aOp[5].p3 = memId; + aOp[8].p2 = memId; } } /* ** Update the maximum rowid for an autoincrement calculation. ** -** This routine should be called when the top of the stack holds a +** This routine should be called when the regRowid register holds a ** new rowid that is about to be inserted. If that new rowid is ** larger than the maximum rowid in the memId memory cell, then the -** memory cell is updated. The stack is unchanged. +** memory cell is updated. */ static void autoIncStep(Parse *pParse, int memId, int regRowid){ if( memId>0 ){ @@ -98003,31 +110797,44 @@ static void autoIncStep(Parse *pParse, int memId, int regRowid){ ** table (either directly or through triggers) needs to call this ** routine just before the "exit" code. */ -SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ +static SQLITE_NOINLINE void autoIncrementEnd(Parse *pParse){ AutoincInfo *p; Vdbe *v = pParse->pVdbe; sqlite3 *db = pParse->db; assert( v ); for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoIncEnd[] = { + /* 0 */ {OP_NotNull, 0, 2, 0}, + /* 1 */ {OP_NewRowid, 0, 0, 0}, + /* 2 */ {OP_MakeRecord, 0, 2, 0}, + /* 3 */ {OP_Insert, 0, 0, 0}, + /* 4 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; Db *pDb = &db->aDb[p->iDb]; - int j1; int iRec; int memId = p->regCtr; iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); - sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); - sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - sqlite3VdbeAddOp0(v, OP_Close); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoIncEnd), autoIncEnd, iLn); + if( aOp==0 ) break; + aOp[0].p1 = memId+1; + aOp[1].p2 = memId+1; + aOp[2].p1 = memId-1; + aOp[2].p3 = iRec; + aOp[3].p2 = iRec; + aOp[3].p3 = memId+1; + aOp[3].p5 = OPFLAG_APPEND; sqlite3ReleaseTempReg(pParse, iRec); } } +SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ + if( pParse->pAinc ) autoIncrementEnd(pParse); +} #else /* ** If SQLITE_OMIT_AUTOINCREMENT is defined, then the three routines @@ -98050,20 +110857,23 @@ static int xferOptimization( /* ** This routine is called to handle SQL of the following forms: ** -** insert into TABLE (IDLIST) values(EXPRLIST) +** insert into TABLE (IDLIST) values(EXPRLIST),(EXPRLIST),... ** insert into TABLE (IDLIST) select +** insert into TABLE (IDLIST) default values ** ** The IDLIST following the table name is always optional. If omitted, -** then a list of all columns for the table is substituted. The IDLIST -** appears in the pColumn parameter. pColumn is NULL if IDLIST is omitted. +** then a list of all (non-hidden) columns for the table is substituted. +** The IDLIST appears in the pColumn parameter. pColumn is NULL if IDLIST +** is omitted. ** -** The pList parameter holds EXPRLIST in the first form of the INSERT -** statement above, and pSelect is NULL. For the second form, pList is -** NULL and pSelect is a pointer to the select statement used to generate -** data for the insert. +** For the pSelect parameter holds the values to be inserted for the +** first two forms shown above. A VALUES clause is really just short-hand +** for a SELECT statement that omits the FROM clause and everything else +** that follows. If the pSelect parameter is NULL, that means that the +** DEFAULT VALUES form of the INSERT statement is intended. ** ** The code generated follows one of four templates. For a simple -** insert with data coming from a VALUES clause, the code executes +** insert with data coming from a single-row VALUES clause, the code executes ** once straight down through. Pseudo-code follows (we call this ** the "1st template"): ** @@ -98118,7 +110928,7 @@ static int xferOptimization( ** The 4th template is used if the insert statement takes its ** values from a SELECT but the data is being inserted into a table ** that is also read as part of the SELECT. In the third form, -** we have to use a intermediate table to store the results of +** we have to use an intermediate table to store the results of ** the select. The template is like this: ** ** X <- A @@ -98151,7 +110961,6 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3 *db; /* The main database structure */ Table *pTab; /* The table to insert into. aka TABLE */ char *zTab; /* Name of the table into which we are inserting */ - const char *zDb; /* Name of the database holding this table */ int i, j, idx; /* Loop counters */ Vdbe *v; /* Generate code into this virtual machine */ Index *pIdx; /* For looping over indices of the table */ @@ -98166,11 +110975,10 @@ SQLITE_PRIVATE void sqlite3Insert( int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ - Db *pDb; /* The database containing table being inserted into */ u8 useTempTable = 0; /* Store SELECT results in intermediate table */ u8 appendFlag = 0; /* True if the insert is likely to be an append */ u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ - u8 bIdListInOrder = 1; /* True if IDLIST is in table order */ + u8 bIdListInOrder; /* True if IDLIST is in table order */ ExprList *pList = 0; /* List of VALUES() to be inserted */ /* Register allocations */ @@ -98195,8 +111003,8 @@ SQLITE_PRIVATE void sqlite3Insert( } /* If the Select object is really just a simple VALUES() list with a - ** single row values (the common case) then keep that one row of values - ** and go ahead and discard the Select object + ** single row (the common case) then keep that one row of values + ** and discard the other (unused) parts of the pSelect object */ if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){ pList = pSelect->pEList; @@ -98216,9 +111024,8 @@ SQLITE_PRIVATE void sqlite3Insert( } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - pDb = &db->aDb[iDb]; - zDb = pDb->zName; - if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, zDb) ){ + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, + db->aDb[iDb].zDbSName) ){ goto insert_cleanup; } withoutRowid = !HasRowid(pTab); @@ -98283,7 +111090,7 @@ SQLITE_PRIVATE void sqlite3Insert( regAutoinc = autoIncBegin(pParse, iDb, pTab); /* Allocate registers for holding the rowid of the new row, - ** the content of the new row, and the assemblied row record. + ** the content of the new row, and the assembled row record. */ regRowid = regIns = pParse->nMem+1; pParse->nMem += pTab->nCol + 1; @@ -98304,6 +111111,7 @@ SQLITE_PRIVATE void sqlite3Insert( ** is appears in the original table. (The index of the INTEGER ** PRIMARY KEY in the original table is pTab->iPKey.) */ + bIdListInOrder = (pTab->tabFlags & TF_OOOHidden)==0; if( pColumn ){ for(i=0; inId; i++){ pColumn->a[i].idx = -1; @@ -98339,7 +111147,8 @@ SQLITE_PRIVATE void sqlite3Insert( ** co-routine is the common header to the 3rd and 4th templates. */ if( pSelect ){ - /* Data is coming from a SELECT. Generate a co-routine to run the SELECT */ + /* Data is coming from a SELECT or from a multi-row VALUES clause. + ** Generate a co-routine to run the SELECT. */ int regYield; /* Register holding co-routine entry-point */ int addrTop; /* Top of the co-routine */ int rc; /* Result code */ @@ -98352,9 +111161,8 @@ SQLITE_PRIVATE void sqlite3Insert( dest.nSdst = pTab->nCol; rc = sqlite3Select(pParse, pSelect, &dest); regFromSelect = dest.iSdst; - assert( pParse->nErr==0 || rc ); - if( rc || db->mallocFailed ) goto insert_cleanup; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup; + sqlite3VdbeEndCoroutine(v, regYield); sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; @@ -98395,25 +111203,27 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrL); + sqlite3VdbeGoto(v, addrL); sqlite3VdbeJumpHere(v, addrL); sqlite3ReleaseTempReg(pParse, regRec); sqlite3ReleaseTempReg(pParse, regTempRowid); } }else{ - /* This is the case if the data for the INSERT is coming from a VALUES - ** clause + /* This is the case if the data for the INSERT is coming from a + ** single-row VALUES clause */ NameContext sNC; memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; srcTab = -1; assert( useTempTable==0 ); - nColumn = pList ? pList->nExpr : 0; - for(i=0; ia[i].pExpr) ){ + if( pList ){ + nColumn = pList->nExpr; + if( sqlite3ResolveExprListNames(&sNC, pList) ){ goto insert_cleanup; } + }else{ + nColumn = 0; } } @@ -98428,10 +111238,8 @@ SQLITE_PRIVATE void sqlite3Insert( /* Make sure the number of columns in the source data matches the number ** of columns to be inserted into the table. */ - if( IsVirtual(pTab) ){ - for(i=0; inCol; i++){ - nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); - } + for(i=0; inCol; i++){ + nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); } if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ sqlite3ErrorMsg(pParse, @@ -98454,9 +111262,9 @@ SQLITE_PRIVATE void sqlite3Insert( /* If this is not a view, open the table and and all indices */ if( !isView ){ int nIdx; - nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, -1, 0, + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, &iDataCur, &iIdxCur); - aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); + aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+1)); if( aRegIdx==0 ){ goto insert_cleanup; } @@ -98506,7 +111314,7 @@ SQLITE_PRIVATE void sqlite3Insert( if( ipkColumn<0 ){ sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); }else{ - int j1; + int addr1; assert( !withoutRowid ); if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); @@ -98514,9 +111322,9 @@ SQLITE_PRIVATE void sqlite3Insert( assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); } - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); } @@ -98527,15 +111335,14 @@ SQLITE_PRIVATE void sqlite3Insert( /* Create the new column data */ - for(i=0; inCol; i++){ - if( pColumn==0 ){ - j = i; - }else{ + for(i=j=0; inCol; i++){ + if( pColumn ){ for(j=0; jnId; j++){ if( pColumn->a[j].idx==i ) break; } } - if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) ){ + if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) + || (pColumn==0 && IsOrdinaryHiddenColumn(&pTab->aCol[i])) ){ sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1); }else if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); @@ -98543,6 +111350,7 @@ SQLITE_PRIVATE void sqlite3Insert( assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1); } + if( pColumn==0 && !IsOrdinaryHiddenColumn(&pTab->aCol[i]) ) j++; } /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, @@ -98590,14 +111398,14 @@ SQLITE_PRIVATE void sqlite3Insert( ** to generate a unique primary key value. */ if( !appendFlag ){ - int j1; + int addr1; if( !IsVirtual(pTab) ){ - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); }else{ - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, addr1+2); VdbeCoverage(v); } sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); } @@ -98626,7 +111434,6 @@ SQLITE_PRIVATE void sqlite3Insert( } if( pColumn==0 ){ if( IsHiddenColumn(&pTab->aCol[i]) ){ - assert( IsVirtual(pTab) ); j = -1; nHidden++; }else{ @@ -98665,7 +111472,7 @@ SQLITE_PRIVATE void sqlite3Insert( { int isReplace; /* Set to true if constraints may cause a replace */ sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, - regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace + regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0 ); sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, @@ -98694,7 +111501,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeJumpHere(v, addrInsTop); sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrCont); + sqlite3VdbeGoto(v, addrCont); sqlite3VdbeJumpHere(v, addrInsTop); } @@ -98735,7 +111542,7 @@ insert_cleanup: } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView @@ -98747,6 +111554,59 @@ insert_cleanup: #undef tmask #endif +/* +** Meanings of bits in of pWalker->eCode for checkConstraintUnchanged() +*/ +#define CKCNSTRNT_COLUMN 0x01 /* CHECK constraint uses a changing column */ +#define CKCNSTRNT_ROWID 0x02 /* CHECK constraint references the ROWID */ + +/* This is the Walker callback from checkConstraintUnchanged(). Set +** bit 0x01 of pWalker->eCode if +** pWalker->eCode to 0 if this expression node references any of the +** columns that are being modifed by an UPDATE statement. +*/ +static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN ){ + assert( pExpr->iColumn>=0 || pExpr->iColumn==-1 ); + if( pExpr->iColumn>=0 ){ + if( pWalker->u.aiCol[pExpr->iColumn]>=0 ){ + pWalker->eCode |= CKCNSTRNT_COLUMN; + } + }else{ + pWalker->eCode |= CKCNSTRNT_ROWID; + } + } + return WRC_Continue; +} + +/* +** pExpr is a CHECK constraint on a row that is being UPDATE-ed. The +** only columns that are modified by the UPDATE are those for which +** aiChng[i]>=0, and also the ROWID is modified if chngRowid is true. +** +** Return true if CHECK constraint pExpr does not use any of the +** changing columns (or the rowid if it is changing). In other words, +** return true if this CHECK constraint can be skipped when validating +** the new row in the UPDATE statement. +*/ +static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 0; + w.xExprCallback = checkConstraintExprNode; + w.u.aiCol = aiChng; + sqlite3WalkExpr(&w, pExpr); + if( !chngRowid ){ + testcase( (w.eCode & CKCNSTRNT_ROWID)!=0 ); + w.eCode &= ~CKCNSTRNT_ROWID; + } + testcase( w.eCode==0 ); + testcase( w.eCode==CKCNSTRNT_COLUMN ); + testcase( w.eCode==CKCNSTRNT_ROWID ); + testcase( w.eCode==(CKCNSTRNT_ROWID|CKCNSTRNT_COLUMN) ); + return !w.eCode; +} + /* ** Generate code to do constraint checks prior to an INSERT or an UPDATE ** on table pTab. @@ -98841,7 +111701,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( u8 pkChng, /* Non-zero if the rowid or PRIMARY KEY changed */ u8 overrideError, /* Override onError to this if not OE_Default */ int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ - int *pbMayReplace /* OUT: Set to true if constraint may cause a replace */ + int *pbMayReplace, /* OUT: Set to true if constraint may cause a replace */ + int *aiChng /* column i is unchanged if aiChng[i]<0 */ ){ Vdbe *v; /* VDBE under constrution */ Index *pIdx; /* Pointer to one of the indices */ @@ -98851,7 +111712,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int j1; /* Addresss of jump instruction */ + int addr1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ int ipkTop = 0; /* Top of the rowid change constraint check */ @@ -98887,10 +111748,14 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( */ for(i=0; iiPKey ){ + continue; /* ROWID is never NULL */ + } + if( aiChng && aiChng[i]<0 ){ + /* Don't bother checking for NOT NULL on columns that do not change */ continue; } onError = pTab->aCol[i].notNull; - if( onError==OE_None ) continue; + if( onError==OE_None ) continue; /* This column is allowed to be NULL */ if( overrideError!=OE_Default ){ onError = overrideError; }else if( onError==OE_Default ){ @@ -98922,9 +111787,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } default: { assert( onError==OE_Replace ); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); + VdbeCoverage(v); sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); break; } } @@ -98938,10 +111804,13 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pParse->ckBase = regNewData+1; onError = overrideError!=OE_Default ? overrideError : OE_Abort; for(i=0; inExpr; i++){ - int allOk = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); + int allOk; + Expr *pExpr = pCheck->a[i].pExpr; + if( aiChng && checkConstraintUnchanged(pExpr, aiChng, pkChng) ) continue; + allOk = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfTrue(pParse, pExpr, allOk, SQLITE_JUMPIFNULL); if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); }else{ char *zName = pCheck->a[i].zName; if( zName==0 ) zName = pTab->zName; @@ -99039,17 +111908,29 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regNewData, 1, 0, OE_Replace, 1); - }else if( pTab->pIndex ){ - sqlite3MultiWrite(pParse); - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + regNewData, 1, 0, OE_Replace, 1, -1); + }else{ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( HasRowid(pTab) ){ + /* This OP_Delete opcode fires the pre-update-hook only. It does + ** not modify the b-tree. It is more efficient to let the coming + ** OP_Insert replace the existing entry than it is to delete the + ** existing entry and then insert a new one. */ + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP); + sqlite3VdbeChangeP4(v, -1, (char *)pTab, P4_TABLE); + } +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); + } } seenReplace = 1; break; } case OE_Ignore: { /*assert( seenReplace==0 );*/ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); break; } } @@ -99085,8 +111966,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); pParse->ckBase = regNewData+1; - sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, addrUniqueOk, - SQLITE_JUMPIFNULL); + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk, + SQLITE_JUMPIFNULL); pParse->ckBase = 0; } @@ -99097,15 +111978,22 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; - if( iField<0 || iField==pTab->iPKey ){ - if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ - x = regNewData; - regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + if( iField==XN_EXPR ){ + pParse->ckBase = regNewData+1; + sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); + pParse->ckBase = 0; + VdbeComment((v, "%s column %d", pIdx->zName, i)); }else{ - x = iField + regNewData + 1; + if( iField==XN_ROWID || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ + x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + }else{ + x = iField + regNewData + 1; + } + sqlite3VdbeAddOp2(v, iField<0 ? OP_IntCopy : OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } - sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); @@ -99155,6 +112043,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** store it in registers regR..regR+nPk-1 */ if( pIdx!=pPk ){ for(i=0; inKeyCol; i++){ + assert( pPk->aiColumn[i]>=0 ); x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); VdbeComment((v, "%s.%s", pTab->zName, @@ -99176,6 +112065,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; inKeyCol; i++){ char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); x = pPk->aiColumn[i]; + assert( x>=0 ); if( i==(pPk->nKeyCol-1) ){ addrJump = addrUniqueOk; op = OP_Eq; @@ -99202,7 +112092,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( break; } case OE_Ignore: { - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); break; } default: { @@ -99213,7 +112103,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regR, nPkField, 0, OE_Replace, pIdx==pPk); + regR, nPkField, 0, OE_Replace, + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); seenReplace = 1; break; } @@ -99223,7 +112114,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); } if( ipkTop ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ipkTop+1); + sqlite3VdbeGoto(v, ipkTop+1); sqlite3VdbeJumpHere(v, ipkBottom); } @@ -99255,7 +112146,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( Index *pIdx; /* An index being inserted or updated */ u8 pik_flags; /* flag values passed to the btree insert */ int regData; /* Content registers (after the rowid) */ - int regRec; /* Register holding assemblied record for the table */ + int regRec; /* Register holding assembled record for the table */ int i; /* Loop counter */ u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ @@ -99276,7 +112167,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; } - if( pik_flags ) sqlite3VdbeChangeP5(v, pik_flags); + sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; regData = regNewData + 1; @@ -99298,7 +112189,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( } sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, regRec, regNewData); if( !pParse->nested ){ - sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); + sqlite3VdbeChangeP4(v, -1, (char *)pTab, P4_TABLE); } sqlite3VdbeChangeP5(v, pik_flags); } @@ -99320,11 +112211,15 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( ** For a WITHOUT ROWID table, *piDataCur will be somewhere in the range ** of *piIdxCurs, depending on where the PRIMARY KEY index appears on the ** pTab->pIndex list. +** +** If pTab is a virtual table, then this routine is a no-op and the +** *piDataCur and *piIdxCur values are left uninitialized. */ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( Parse *pParse, /* Parsing context */ Table *pTab, /* Table to be opened */ int op, /* OP_OpenRead or OP_OpenWrite */ + u8 p5, /* P5 value for OP_Open* opcodes (except on WITHOUT ROWID) */ int iBase, /* Use this for the table cursor, if there is one */ u8 *aToOpen, /* If not NULL: boolean for each table and index */ int *piDataCur, /* Write the database source cursor number here */ @@ -99337,10 +112232,11 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( Vdbe *v; assert( op==OP_OpenRead || op==OP_OpenWrite ); + assert( op==OP_OpenWrite || p5==0 ); if( IsVirtual(pTab) ){ - assert( aToOpen==0 ); - *piDataCur = 0; - *piIdxCur = 1; + /* This routine is a no-op for virtual tables. Leave the output + ** variables *piDataCur and *piIdxCur uninitialized so that valgrind + ** can detect if they are used by mistake in the caller. */ return 0; } iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); @@ -99358,12 +112254,14 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); - if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) && piDataCur ){ - *piDataCur = iIdxCur; + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + if( piDataCur ) *piDataCur = iIdxCur; + p5 = 0; } if( aToOpen==0 || aToOpen[i+1] ){ sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + sqlite3VdbeChangeP5(v, p5); VdbeComment((v, "%s", pIdx->zName)); } } @@ -99377,27 +112275,13 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( ** The following global variable is incremented whenever the ** transfer optimization is used. This is used for testing ** purposes only - to make sure the transfer optimization really -** is happening when it is suppose to. +** is happening when it is supposed to. */ SQLITE_API int sqlite3_xferopt_count; #endif /* SQLITE_TEST */ #ifndef SQLITE_OMIT_XFER_OPT -/* -** Check to collation names to see if they are compatible. -*/ -static int xferCompatibleCollation(const char *z1, const char *z2){ - if( z1==0 ){ - return z2==0; - } - if( z2==0 ){ - return 0; - } - return sqlite3StrICmp(z1, z2)==0; -} - - /* ** Check to see if index pSrc is compatible as a source of data ** for index pDest in an insert transfer optimization. The rules @@ -99423,10 +112307,17 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ return 0; /* Different columns indexed */ } + if( pSrc->aiColumn[i]==XN_EXPR ){ + assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); + if( sqlite3ExprCompare(pSrc->aColExpr->a[i].pExpr, + pDest->aColExpr->a[i].pExpr, -1)!=0 ){ + return 0; /* Different expressions in the index */ + } + } if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ return 0; /* Different sort orders */ } - if( !xferCompatibleCollation(pSrc->azColl[i],pDest->azColl[i]) ){ + if( sqlite3_stricmp(pSrc->azColl[i],pDest->azColl[i])!=0 ){ return 0; /* Different collating sequences */ } } @@ -99444,7 +112335,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ ** INSERT INTO tab1 SELECT * FROM tab2; ** ** The xfer optimization transfers raw records from tab2 over to tab1. -** Columns are not decoded and reassemblied, which greatly improves +** Columns are not decoded and reassembled, which greatly improves ** performance. Raw index records are transferred in the same way. ** ** The xfer optimization is only attempted if tab1 and tab2 are compatible. @@ -99470,6 +112361,7 @@ static int xferOptimization( int onError, /* How to handle constraint errors */ int iDbDest /* The database of pDest */ ){ + sqlite3 *db = pParse->db; ExprList *pEList; /* The result set of the SELECT */ Table *pSrc; /* The table in the FROM clause of SELECT */ Index *pSrcIdx, *pDestIdx; /* Source and destination indices */ @@ -99540,7 +112432,7 @@ static int xferOptimization( return 0; /* The result set must have exactly one column */ } assert( pEList->a[0].pExpr ); - if( pEList->a[0].pExpr->op!=TK_ALL ){ + if( pEList->a[0].pExpr->op!=TK_ASTERISK ){ return 0; /* The result set must be the special operator "*" */ } @@ -99576,21 +112468,32 @@ static int xferOptimization( for(i=0; inCol; i++){ Column *pDestCol = &pDest->aCol[i]; Column *pSrcCol = &pSrc->aCol[i]; +#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS + if( (db->flags & SQLITE_Vacuum)==0 + && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN + ){ + return 0; /* Neither table may have __hidden__ columns */ + } +#endif if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } - if( !xferCompatibleCollation(pDestCol->zColl, pSrcCol->zColl) ){ + if( sqlite3_stricmp(pDestCol->zColl, pSrcCol->zColl)!=0 ){ return 0; /* Collating sequence must be the same on all columns */ } if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } /* Default values for second and subsequent columns need to match. */ - if( i>0 - && ((pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) - || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)!=0)) - ){ - return 0; /* Default values must be the same for all columns */ + if( i>0 ){ + assert( pDestCol->pDflt==0 || pDestCol->pDflt->op==TK_SPAN ); + assert( pSrcCol->pDflt==0 || pSrcCol->pDflt->op==TK_SPAN ); + if( (pDestCol->pDflt==0)!=(pSrcCol->pDflt==0) + || (pDestCol->pDflt && strcmp(pDestCol->pDflt->u.zToken, + pSrcCol->pDflt->u.zToken)!=0) + ){ + return 0; /* Default values must be the same for all columns */ + } } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ @@ -99617,11 +112520,11 @@ static int xferOptimization( ** the extra complication to make this rule less restrictive is probably ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] */ - if( (pParse->db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ + if( (db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ return 0; } #endif - if( (pParse->db->flags & SQLITE_CountRows)!=0 ){ + if( (db->flags & SQLITE_CountRows)!=0 ){ return 0; /* xfer opt does not play well with PRAGMA count_changes */ } @@ -99632,7 +112535,7 @@ static int xferOptimization( #ifdef SQLITE_TEST sqlite3_xferopt_count++; #endif - iDbSrc = sqlite3SchemaToIndex(pParse->db, pSrc->pSchema); + iDbSrc = sqlite3SchemaToIndex(db, pSrc->pSchema); v = sqlite3GetVdbe(pParse); sqlite3CodeVerifySchema(pParse, iDbSrc); iSrc = pParse->nTab++; @@ -99642,14 +112545,18 @@ static int xferOptimization( regRowid = sqlite3GetTempReg(pParse); sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); assert( HasRowid(pDest) || destHasUniqueIdx ); - if( (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ + if( (db->flags & SQLITE_Vacuum)==0 && ( + (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ || destHasUniqueIdx /* (2) */ || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ - ){ + )){ /* In some circumstances, we are able to run the xfer optimization - ** only if the destination table is initially empty. This code makes - ** that determination. Conditions under which the destination must - ** be empty: + ** only if the destination table is initially empty. Unless the + ** SQLITE_Vacuum flag is set, this block generates code to make + ** that determination. If SQLITE_Vacuum is set, then the destination + ** table is always empty. + ** + ** Conditions under which the destination must be empty: ** ** (1) There is no INTEGER PRIMARY KEY but there are indices. ** (If the destination is not initially empty, the rowid fields @@ -99661,7 +112568,7 @@ static int xferOptimization( ** (3) onError is something other than OE_Abort and OE_Rollback. */ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); - emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + emptyDestTest = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); } if( HasRowid(pSrc) ){ @@ -99681,9 +112588,9 @@ static int xferOptimization( assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); - sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid, + (char*)pDest, P4_TABLE); sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); - sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); @@ -99692,6 +112599,7 @@ static int xferOptimization( sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + u8 idxInsFlags = 0; for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; } @@ -99705,7 +112613,37 @@ static int xferOptimization( VdbeComment((v, "%s", pDestIdx->zName)); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); + if( db->flags & SQLITE_Vacuum ){ + /* This INSERT command is part of a VACUUM operation, which guarantees + ** that the destination table is empty. If all indexed columns use + ** collation sequence BINARY, then it can also be assumed that the + ** index will be populated by inserting keys in strictly sorted + ** order. In this case, instead of seeking within the b-tree as part + ** of every OP_IdxInsert opcode, an OP_Last is added before the + ** OP_IdxInsert to seek to the point within the b-tree where each key + ** should be inserted. This is faster. + ** + ** If any of the indexed columns use a collation sequence other than + ** BINARY, this optimization is disabled. This is because the user + ** might change the definition of a collation sequence and then run + ** a VACUUM command. In that case keys may not be written in strictly + ** sorted order. */ + for(i=0; inColumn; i++){ + const char *zColl = pSrcIdx->azColl[i]; + assert( sqlite3_stricmp(sqlite3StrBINARY, zColl)!=0 + || sqlite3StrBINARY==zColl ); + if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; + } + if( i==pSrcIdx->nColumn ){ + idxInsFlags = OPFLAG_USESEEKRESULT; + sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); + } + } + if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ + idxInsFlags |= OPFLAG_NCHANGE; + } sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); + sqlite3VdbeChangeP5(v, idxInsFlags); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); @@ -99715,6 +112653,7 @@ static int xferOptimization( sqlite3ReleaseTempReg(pParse, regRowid); sqlite3ReleaseTempReg(pParse, regData); if( emptyDestTest ){ + sqlite3AutoincrementEnd(pParse); sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); sqlite3VdbeJumpHere(v, emptyDestTest); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); @@ -99744,6 +112683,7 @@ static int xferOptimization( ** accessed by users of the library. */ +/* #include "sqliteInt.h" */ /* ** Execute SQL code. Return one of the SQLITE_ success/failure @@ -99772,7 +112712,7 @@ SQLITE_API int sqlite3_exec( if( zSql==0 ) zSql = ""; sqlite3_mutex_enter(db->mutex); - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; @@ -99818,7 +112758,7 @@ SQLITE_API int sqlite3_exec( for(i=0; imallocFailed = 1; + sqlite3OomFault(db); goto exec_out; } } @@ -99830,7 +112770,7 @@ SQLITE_API int sqlite3_exec( rc = SQLITE_ABORT; sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - sqlite3Error(db, SQLITE_ABORT, 0); + sqlite3Error(db, SQLITE_ABORT); goto exec_out; } } @@ -99853,14 +112793,14 @@ exec_out: sqlite3DbFree(db, azCols); rc = sqlite3ApiExit(db, rc); - if( rc!=SQLITE_OK && ALWAYS(rc==sqlite3_errcode(db)) && pzErrMsg ){ + if( rc!=SQLITE_OK && pzErrMsg ){ int nErrMsg = 1 + sqlite3Strlen30(sqlite3_errmsg(db)); *pzErrMsg = sqlite3Malloc(nErrMsg); if( *pzErrMsg ){ memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); }else{ - rc = SQLITE_NOMEM; - sqlite3Error(db, SQLITE_NOMEM, 0); + rc = SQLITE_NOMEM_BKPT; + sqlite3Error(db, SQLITE_NOMEM); } }else if( pzErrMsg ){ *pzErrMsg = 0; @@ -99910,10 +112850,9 @@ exec_out: ** as extensions by SQLite should #include this file instead of ** sqlite3.h. */ -#ifndef _SQLITE3EXT_H_ -#define _SQLITE3EXT_H_ - -typedef struct sqlite3_api_routines sqlite3_api_routines; +#ifndef SQLITE3EXT_H +#define SQLITE3EXT_H +/* #include "sqlite3.h" */ /* ** The following structure holds pointers to all of the SQLite API @@ -99922,7 +112861,7 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** WARNING: In order to maintain backwards compatibility, add new ** interfaces to the end of this structure only. If you insert new ** interfaces in the middle of this structure, then older different -** versions of SQLite will not be able to load each others' shared +** versions of SQLite will not be able to load each other's shared ** libraries! */ struct sqlite3_api_routines { @@ -100144,11 +113083,55 @@ struct sqlite3_api_routines { const char *(*uri_parameter)(const char*,const char*); char *(*vsnprintf)(int,char*,const char*,va_list); int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); + /* Version 3.8.7 and later */ + int (*auto_extension)(void(*)(void)); + int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, + void(*)(void*)); + int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64, + void(*)(void*),unsigned char); + int (*cancel_auto_extension)(void(*)(void)); + int (*load_extension)(sqlite3*,const char*,const char*,char**); + void *(*malloc64)(sqlite3_uint64); + sqlite3_uint64 (*msize)(void*); + void *(*realloc64)(void*,sqlite3_uint64); + void (*reset_auto_extension)(void); + void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64, + void(*)(void*)); + void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, + void(*)(void*), unsigned char); + int (*strglob)(const char*,const char*); + /* Version 3.8.11 and later */ + sqlite3_value *(*value_dup)(const sqlite3_value*); + void (*value_free)(sqlite3_value*); + int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); + int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); + /* Version 3.9.0 and later */ + unsigned int (*value_subtype)(sqlite3_value*); + void (*result_subtype)(sqlite3_context*,unsigned int); + /* Version 3.10.0 and later */ + int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int); + int (*strlike)(const char*,const char*,unsigned int); + int (*db_cacheflush)(sqlite3*); + /* Version 3.12.0 and later */ + int (*system_errno)(sqlite3*); + /* Version 3.14.0 and later */ + int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*); + char *(*expanded_sql)(sqlite3_stmt*); }; +/* +** This is the function signature used for all extension entry points. It +** is also defined in the file "loadext.c". +*/ +typedef int (*sqlite3_loadext_entry)( + sqlite3 *db, /* Handle to the database. */ + char **pzErrMsg, /* Used to set error string on failure. */ + const sqlite3_api_routines *pThunk /* Extension API function pointers. */ +); + /* ** The following macros redefine the API routines so that they are -** redirected throught the global sqlite3_api structure. +** redirected through the global sqlite3_api structure. ** ** This header file is also used by the loadext.c source file ** (part of the main SQLite library - not an extension) so that @@ -100157,7 +113140,7 @@ struct sqlite3_api_routines { ** the API. So the redefinition macros are only valid if the ** SQLITE_CORE macros is undefined. */ -#ifndef SQLITE_CORE +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) #define sqlite3_aggregate_context sqlite3_api->aggregate_context #ifndef SQLITE_OMIT_DEPRECATED #define sqlite3_aggregate_count sqlite3_api->aggregate_count @@ -100284,6 +113267,7 @@ struct sqlite3_api_routines { #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_vsnprintf sqlite3_api->vsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 @@ -100361,9 +113345,39 @@ struct sqlite3_api_routines { #define sqlite3_uri_parameter sqlite3_api->uri_parameter #define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 -#endif /* SQLITE_CORE */ +/* Version 3.8.7 and later */ +#define sqlite3_auto_extension sqlite3_api->auto_extension +#define sqlite3_bind_blob64 sqlite3_api->bind_blob64 +#define sqlite3_bind_text64 sqlite3_api->bind_text64 +#define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension +#define sqlite3_load_extension sqlite3_api->load_extension +#define sqlite3_malloc64 sqlite3_api->malloc64 +#define sqlite3_msize sqlite3_api->msize +#define sqlite3_realloc64 sqlite3_api->realloc64 +#define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension +#define sqlite3_result_blob64 sqlite3_api->result_blob64 +#define sqlite3_result_text64 sqlite3_api->result_text64 +#define sqlite3_strglob sqlite3_api->strglob +/* Version 3.8.11 and later */ +#define sqlite3_value_dup sqlite3_api->value_dup +#define sqlite3_value_free sqlite3_api->value_free +#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 +#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 +/* Version 3.9.0 and later */ +#define sqlite3_value_subtype sqlite3_api->value_subtype +#define sqlite3_result_subtype sqlite3_api->result_subtype +/* Version 3.10.0 and later */ +#define sqlite3_status64 sqlite3_api->status64 +#define sqlite3_strlike sqlite3_api->strlike +#define sqlite3_db_cacheflush sqlite3_api->db_cacheflush +/* Version 3.12.0 and later */ +#define sqlite3_system_errno sqlite3_api->system_errno +/* Version 3.14.0 and later */ +#define sqlite3_trace_v2 sqlite3_api->trace_v2 +#define sqlite3_expanded_sql sqlite3_api->expanded_sql +#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ -#ifndef SQLITE_CORE +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) /* This case when the file really is being compiled as a loadable ** extension */ # define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; @@ -100378,14 +113392,14 @@ struct sqlite3_api_routines { # define SQLITE_EXTENSION_INIT3 /*no-op*/ #endif -#endif /* _SQLITE3EXT_H_ */ +#endif /* SQLITE3EXT_H */ /************** End of sqlite3ext.h ******************************************/ /************** Continuing where we left off in loadext.c ********************/ +/* #include "sqliteInt.h" */ /* #include */ #ifndef SQLITE_OMIT_LOAD_EXTENSION - /* ** Some API routines are omitted when various features are ** excluded from a build of SQLite. Substitute a NULL pointer @@ -100398,7 +113412,6 @@ struct sqlite3_api_routines { # define sqlite3_column_table_name16 0 # define sqlite3_column_origin_name 0 # define sqlite3_column_origin_name16 0 -# define sqlite3_table_column_metadata 0 #endif #ifdef SQLITE_OMIT_AUTHORIZATION @@ -100456,7 +113469,7 @@ struct sqlite3_api_routines { # define sqlite3_enable_shared_cache 0 #endif -#ifdef SQLITE_OMIT_TRACE +#if defined(SQLITE_OMIT_TRACE) || defined(SQLITE_OMIT_DEPRECATED) # define sqlite3_profile 0 # define sqlite3_trace 0 #endif @@ -100476,6 +113489,10 @@ struct sqlite3_api_routines { #define sqlite3_blob_reopen 0 #endif +#if defined(SQLITE_OMIT_TRACE) +# define sqlite3_trace_v2 0 +#endif + /* ** The following structure contains pointers to all SQLite API routines. ** A pointer to this structure is passed into extensions when they are @@ -100754,7 +113771,37 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_uri_int64, sqlite3_uri_parameter, sqlite3_vsnprintf, - sqlite3_wal_checkpoint_v2 + sqlite3_wal_checkpoint_v2, + /* Version 3.8.7 and later */ + sqlite3_auto_extension, + sqlite3_bind_blob64, + sqlite3_bind_text64, + sqlite3_cancel_auto_extension, + sqlite3_load_extension, + sqlite3_malloc64, + sqlite3_msize, + sqlite3_realloc64, + sqlite3_reset_auto_extension, + sqlite3_result_blob64, + sqlite3_result_text64, + sqlite3_strglob, + /* Version 3.8.11 and later */ + (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, + sqlite3_value_free, + sqlite3_result_zeroblob64, + sqlite3_bind_zeroblob64, + /* Version 3.9.0 and later */ + sqlite3_value_subtype, + sqlite3_result_subtype, + /* Version 3.10.0 and later */ + sqlite3_status64, + sqlite3_strlike, + sqlite3_db_cacheflush, + /* Version 3.12.0 and later */ + sqlite3_system_errno, + /* Version 3.14.0 and later */ + sqlite3_trace_v2, + sqlite3_expanded_sql }; /* @@ -100777,13 +113824,14 @@ static int sqlite3LoadExtension( ){ sqlite3_vfs *pVfs = db->pVfs; void *handle; - int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + sqlite3_loadext_entry xInit; char *zErrmsg = 0; const char *zEntry; char *zAltEntry = 0; void **aHandle; - int nMsg = 300 + sqlite3Strlen30(zFile); + u64 nMsg = 300 + sqlite3Strlen30(zFile); int ii; + int rc; /* Shared library endings to try if zFile cannot be loaded as written */ static const char *azEndings[] = { @@ -100802,8 +113850,9 @@ static int sqlite3LoadExtension( /* Ticket #1863. To avoid a creating security problems for older ** applications that relink against newer versions of SQLite, the ** ability to run load_extension is turned off by default. One - ** must call sqlite3_enable_load_extension() to turn on extension - ** loading. Otherwise you get the following error. + ** must call either sqlite3_enable_load_extension(db) or + ** sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, 0) + ** to turn on extension loading. */ if( (db->flags & SQLITE_LoadExtension)==0 ){ if( pzErrMsg ){ @@ -100818,14 +113867,14 @@ static int sqlite3LoadExtension( #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; ii=0 && zFile[iFile]!='/'; iFile--){} @@ -100867,13 +113915,12 @@ static int sqlite3LoadExtension( } memcpy(zAltEntry+iEntry, "_init", 6); zEntry = zAltEntry; - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - sqlite3OsDlSym(pVfs, handle, zEntry); + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); } if( xInit==0 ){ if( pzErrMsg ){ nMsg += sqlite3Strlen30(zEntry); - *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "no entry point [%s] in shared library [%s]", zEntry, zFile); @@ -100885,7 +113932,9 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } sqlite3_free(zAltEntry); - if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + rc = xInit(db, &zErrmsg, &sqlite3Apis); + if( rc ){ + if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK; if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } @@ -100897,7 +113946,7 @@ static int sqlite3LoadExtension( /* Append the new shared library handle to the db->aExtension array. */ aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); if( aHandle==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( db->nExtension>0 ){ memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); @@ -100942,26 +113991,15 @@ SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3 *db){ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ sqlite3_mutex_enter(db->mutex); if( onoff ){ - db->flags |= SQLITE_LoadExtension; + db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; }else{ - db->flags &= ~SQLITE_LoadExtension; + db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ - -/* -** The auto-extension code added regardless of whether or not extension -** loading is supported. We need a dummy sqlite3Apis pointer for that -** code if regular extension loading is not available. This is that -** dummy pointer. -*/ -#ifdef SQLITE_OMIT_LOAD_EXTENSION -static const sqlite3_api_routines sqlite3Apis = { 0 }; -#endif - +#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */ /* ** The following object holds the list of automatically loaded @@ -100972,7 +114010,7 @@ static const sqlite3_api_routines sqlite3Apis = { 0 }; */ typedef struct sqlite3AutoExtList sqlite3AutoExtList; static SQLITE_WSD struct sqlite3AutoExtList { - int nExt; /* Number of entries in aExt[] */ + u32 nExt; /* Number of entries in aExt[] */ void (**aExt)(void); /* Pointers to the extension init functions */ } sqlite3Autoext = { 0, 0 }; @@ -100996,7 +114034,9 @@ static SQLITE_WSD struct sqlite3AutoExtList { ** Register a statically linked extension that is automatically ** loaded by every new database connection. */ -SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ +SQLITE_API int sqlite3_auto_extension( + void (*xInit)(void) +){ int rc = SQLITE_OK; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -101005,7 +114045,7 @@ SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ }else #endif { - int i; + u32 i; #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif @@ -101015,11 +114055,11 @@ SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ if( wsdAutoext.aExt[i]==xInit ) break; } if( i==wsdAutoext.nExt ){ - int nByte = (wsdAutoext.nExt+1)*sizeof(wsdAutoext.aExt[0]); + u64 nByte = (wsdAutoext.nExt+1)*sizeof(wsdAutoext.aExt[0]); void (**aNew)(void); - aNew = sqlite3_realloc(wsdAutoext.aExt, nByte); + aNew = sqlite3_realloc64(wsdAutoext.aExt, nByte); if( aNew==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ wsdAutoext.aExt = aNew; wsdAutoext.aExt[wsdAutoext.nExt] = xInit; @@ -101041,7 +114081,9 @@ SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ ** Return 1 if xInit was found on the list and removed. Return 0 if xInit ** was not on the list. */ -SQLITE_API int sqlite3_cancel_auto_extension(void (*xInit)(void)){ +SQLITE_API int sqlite3_cancel_auto_extension( + void (*xInit)(void) +){ #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif @@ -101049,7 +114091,7 @@ SQLITE_API int sqlite3_cancel_auto_extension(void (*xInit)(void)){ int n = 0; wsdAutoextInit; sqlite3_mutex_enter(mutex); - for(i=wsdAutoext.nExt-1; i>=0; i--){ + for(i=(int)wsdAutoext.nExt-1; i>=0; i--){ if( wsdAutoext.aExt[i]==xInit ){ wsdAutoext.nExt--; wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt]; @@ -101087,10 +114129,10 @@ SQLITE_API void sqlite3_reset_auto_extension(void){ ** If anything goes wrong, set an error in the database connection. */ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ - int i; + u32 i; int go = 1; int rc; - int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + sqlite3_loadext_entry xInit; wsdAutoextInit; if( wsdAutoext.nExt==0 ){ @@ -101101,19 +114143,23 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ char *zErrmsg; #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + const sqlite3_api_routines *pThunk = 0; +#else + const sqlite3_api_routines *pThunk = &sqlite3Apis; #endif sqlite3_mutex_enter(mutex); if( i>=wsdAutoext.nExt ){ xInit = 0; go = 0; }else{ - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - wsdAutoext.aExt[i]; + xInit = (sqlite3_loadext_entry)wsdAutoext.aExt[i]; } sqlite3_mutex_leave(mutex); zErrmsg = 0; - if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ - sqlite3Error(db, rc, + if( xInit && (rc = xInit(db, &zErrmsg, pThunk))!=0 ){ + sqlite3ErrorWithMsg(db, rc, "automatic extension loading failed: %s", zErrmsg); go = 0; } @@ -101136,6 +114182,7 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ************************************************************************* ** This file contains code used to implement the PRAGMA command. */ +/* #include "sqliteInt.h" */ #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) @@ -101146,54 +114193,64 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ #endif /*************************************************************************** -** The next block of code, including the PragTyp_XXXX macro definitions and -** the aPragmaName[] object is composed of generated code. DO NOT EDIT. -** -** To add new pragmas, edit the code in ../tool/mkpragmatab.tcl and rerun -** that script. Then copy/paste the output in place of the following: +** The "pragma.h" include file is an automatically generated file that +** that includes the PragType_XXXX macro definitions and the aPragmaName[] +** object. This ensures that the aPragmaName[] table is arranged in +** lexicographical order to facility a binary search of the pragma name. +** Do not edit pragma.h directly. Edit and rerun the script in at +** ../tool/mkpragmatab.tcl. */ +/************** Include pragma.h in the middle of pragma.c *******************/ +/************** Begin file pragma.h ******************************************/ +/* DO NOT EDIT! +** This file is automatically generated by the script at +** ../tool/mkpragmatab.tcl. To update the set of pragmas, edit +** that script and rerun it. */ #define PragTyp_HEADER_VALUE 0 #define PragTyp_AUTO_VACUUM 1 #define PragTyp_FLAG 2 #define PragTyp_BUSY_TIMEOUT 3 #define PragTyp_CACHE_SIZE 4 -#define PragTyp_CASE_SENSITIVE_LIKE 5 -#define PragTyp_COLLATION_LIST 6 -#define PragTyp_COMPILE_OPTIONS 7 -#define PragTyp_DATA_STORE_DIRECTORY 8 -#define PragTyp_DATABASE_LIST 9 -#define PragTyp_DEFAULT_CACHE_SIZE 10 -#define PragTyp_ENCODING 11 -#define PragTyp_FOREIGN_KEY_CHECK 12 -#define PragTyp_FOREIGN_KEY_LIST 13 -#define PragTyp_INCREMENTAL_VACUUM 14 -#define PragTyp_INDEX_INFO 15 -#define PragTyp_INDEX_LIST 16 -#define PragTyp_INTEGRITY_CHECK 17 -#define PragTyp_JOURNAL_MODE 18 -#define PragTyp_JOURNAL_SIZE_LIMIT 19 -#define PragTyp_LOCK_PROXY_FILE 20 -#define PragTyp_LOCKING_MODE 21 -#define PragTyp_PAGE_COUNT 22 -#define PragTyp_MMAP_SIZE 23 -#define PragTyp_PAGE_SIZE 24 -#define PragTyp_SECURE_DELETE 25 -#define PragTyp_SHRINK_MEMORY 26 -#define PragTyp_SOFT_HEAP_LIMIT 27 -#define PragTyp_STATS 28 -#define PragTyp_SYNCHRONOUS 29 -#define PragTyp_TABLE_INFO 30 -#define PragTyp_TEMP_STORE 31 -#define PragTyp_TEMP_STORE_DIRECTORY 32 -#define PragTyp_WAL_AUTOCHECKPOINT 33 -#define PragTyp_WAL_CHECKPOINT 34 -#define PragTyp_ACTIVATE_EXTENSIONS 35 -#define PragTyp_HEXKEY 36 -#define PragTyp_KEY 37 -#define PragTyp_REKEY 38 -#define PragTyp_LOCK_STATUS 39 -#define PragTyp_PARSER_TRACE 40 +#define PragTyp_CACHE_SPILL 5 +#define PragTyp_CASE_SENSITIVE_LIKE 6 +#define PragTyp_COLLATION_LIST 7 +#define PragTyp_COMPILE_OPTIONS 8 +#define PragTyp_DATA_STORE_DIRECTORY 9 +#define PragTyp_DATABASE_LIST 10 +#define PragTyp_DEFAULT_CACHE_SIZE 11 +#define PragTyp_ENCODING 12 +#define PragTyp_FOREIGN_KEY_CHECK 13 +#define PragTyp_FOREIGN_KEY_LIST 14 +#define PragTyp_INCREMENTAL_VACUUM 15 +#define PragTyp_INDEX_INFO 16 +#define PragTyp_INDEX_LIST 17 +#define PragTyp_INTEGRITY_CHECK 18 +#define PragTyp_JOURNAL_MODE 19 +#define PragTyp_JOURNAL_SIZE_LIMIT 20 +#define PragTyp_LOCK_PROXY_FILE 21 +#define PragTyp_LOCKING_MODE 22 +#define PragTyp_PAGE_COUNT 23 +#define PragTyp_MMAP_SIZE 24 +#define PragTyp_PAGE_SIZE 25 +#define PragTyp_SECURE_DELETE 26 +#define PragTyp_SHRINK_MEMORY 27 +#define PragTyp_SOFT_HEAP_LIMIT 28 +#define PragTyp_STATS 29 +#define PragTyp_SYNCHRONOUS 30 +#define PragTyp_TABLE_INFO 31 +#define PragTyp_TEMP_STORE 32 +#define PragTyp_TEMP_STORE_DIRECTORY 33 +#define PragTyp_THREADS 34 +#define PragTyp_WAL_AUTOCHECKPOINT 35 +#define PragTyp_WAL_CHECKPOINT 36 +#define PragTyp_ACTIVATE_EXTENSIONS 37 +#define PragTyp_HEXKEY 38 +#define PragTyp_KEY 39 +#define PragTyp_REKEY 40 +#define PragTyp_LOCK_STATUS 41 +#define PragTyp_PARSER_TRACE 42 #define PragFlag_NeedSchema 0x01 +#define PragFlag_ReadOnly 0x02 static const struct sPragmaNames { const char *const zName; /* Name of pragma */ u8 ePragTyp; /* PragTyp_XXX value */ @@ -101210,7 +114267,7 @@ static const struct sPragmaNames { { /* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_APPLICATION_ID }, #endif #if !defined(SQLITE_OMIT_AUTOVACUUM) { /* zName: */ "auto_vacuum", @@ -101238,14 +114295,18 @@ static const struct sPragmaNames { #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "cache_spill", - /* ePragTyp: */ PragTyp_FLAG, + /* ePragTyp: */ PragTyp_CACHE_SPILL, /* ePragFlag: */ 0, - /* iArg: */ SQLITE_CacheSpill }, + /* iArg: */ 0 }, #endif { /* zName: */ "case_sensitive_like", /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, /* ePragFlag: */ 0, /* iArg: */ 0 }, + { /* zName: */ "cell_size_check", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_CellSizeCk }, #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "checkpoint_fullfsync", /* ePragTyp: */ PragTyp_FLAG, @@ -101276,6 +114337,12 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "data_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ PragFlag_ReadOnly, + /* iArg: */ BTREE_DATA_VERSION }, +#endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) { /* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, @@ -101331,8 +114398,8 @@ static const struct sPragmaNames { #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) { /* zName: */ "freelist_count", /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* ePragFlag: */ PragFlag_ReadOnly, + /* iArg: */ BTREE_FREE_PAGE_COUNT }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "full_column_names", @@ -101377,6 +114444,10 @@ static const struct sPragmaNames { /* ePragTyp: */ PragTyp_INDEX_LIST, /* ePragFlag: */ PragFlag_NeedSchema, /* iArg: */ 0 }, + { /* zName: */ "index_xinfo", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) { /* zName: */ "integrity_check", @@ -101440,7 +114511,7 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_PARSER_TRACE) { /* zName: */ "parser_trace", /* ePragTyp: */ PragTyp_PARSER_TRACE, /* ePragFlag: */ 0, @@ -101484,7 +114555,7 @@ static const struct sPragmaNames { { /* zName: */ "schema_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_SCHEMA_VERSION }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "secure_delete", @@ -101542,11 +114613,15 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif + { /* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) { /* zName: */ "user_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_USER_VERSION }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if defined(SQLITE_DEBUG) @@ -101589,14 +114664,15 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 56 on by default, 69 total. */ -/* End of the automatically generated pragma table. -***************************************************************************/ +/* Number of pragmas: 60 on by default, 73 total. */ + +/************** End of pragma.h **********************************************/ +/************** Continuing where we left off in pragma.c *********************/ /* ** Interpret the given string as a safety level. Return 0 for OFF, -** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or -** unrecognized string argument. The FULL option is disallowed +** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or +** unrecognized string argument. The FULL and EXTRA option is disallowed ** if the omitFull parameter it 1. ** ** Note that the values returned are one less that the values that @@ -101605,18 +114681,21 @@ static const struct sPragmaNames { ** and older scripts may have used numbers 0 for OFF and 1 for ON. */ static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ - /* 123456789 123456789 */ - static const char zText[] = "onoffalseyestruefull"; - static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; - static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 4}; - static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; + /* 123456789 123456789 123 */ + static const char zText[] = "onoffalseyestruextrafull"; + static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 15, 20}; + static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 5, 4}; + static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 3, 2}; + /* on no off false yes true extra full */ int i, n; if( sqlite3Isdigit(*z) ){ return (u8)sqlite3Atoi(z); } n = sqlite3Strlen30(z); - for(i=0; inMem; - i64 *pI64 = sqlite3DbMallocRaw(pParse->db, sizeof(value)); - if( pI64 ){ - memcpy(pI64, &value, sizeof(value)); +static void returnSingleInt(Vdbe *v, const char *zLabel, i64 value){ + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, 1, 0, (const u8*)&value, P4_INT64); + setOneColumnName(v, zLabel); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); +} + +/* +** Generate code to return a single text value. +*/ +static void returnSingleText( + Vdbe *v, /* Prepared statement under construction */ + const char *zLabel, /* Name of the result column */ + const char *zValue /* Value to be returned */ +){ + if( zValue ){ + sqlite3VdbeLoadString(v, 1, (const char*)zValue); + setOneColumnName(v, zLabel); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } - sqlite3VdbeAddOp4(v, OP_Int64, 0, mem, 0, (char*)pI64, P4_INT64); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); } @@ -101816,7 +114921,7 @@ SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ ** ** Pragmas are of this form: ** -** PRAGMA [database.]id [= value] +** PRAGMA [schema.]id [= value] ** ** The identifier might also be a string. The value is a string, and ** identifier, or a number. If minusFlag is true, then the value is @@ -101828,8 +114933,8 @@ SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ */ SQLITE_PRIVATE void sqlite3Pragma( Parse *pParse, - Token *pId1, /* First part of [database.]id field */ - Token *pId2, /* Second part of [database.]id field, or NULL */ + Token *pId1, /* First part of [schema.]id field */ + Token *pId2, /* Second part of [schema.]id field, or NULL */ Token *pValue, /* Token for , or NULL */ int minusFlag /* True if a '-' sign preceded */ ){ @@ -101839,23 +114944,23 @@ SQLITE_PRIVATE void sqlite3Pragma( Token *pId; /* Pointer to token */ char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ int iDb; /* Database index for */ - int lwr, upr, mid; /* Binary search bounds */ + int lwr, upr, mid = 0; /* Binary search bounds */ int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ + const struct sPragmaNames *pPragma; /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC extern int sqlcipher_codec_pragma(sqlite3*, int, Parse *, const char *, const char *); #endif /* END SQLCIPHER */ - if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); pParse->nMem = 2; - /* Interpret the [database.] part of the pragma statement. iDb is the + /* Interpret the [schema.] part of the pragma statement. iDb is the ** index of the database this pragma is being applied to in db.aDb[]. */ iDb = sqlite3TwoPartName(pParse, pId1, pId2, &pId); if( iDb<0 ) return; @@ -101877,7 +114982,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } assert( pId2 ); - zDb = pId2->n>0 ? pDb->zName : 0; + zDb = pId2->n>0 ? pDb->zDbSName : 0; if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } @@ -101885,6 +114990,17 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Send an SQLITE_FCNTL_PRAGMA file-control to the underlying VFS ** connection. If it returns SQLITE_OK, then assume that the VFS ** handled the pragma and generate a no-op prepared statement. + ** + ** IMPLEMENTATION-OF: R-12238-55120 Whenever a PRAGMA statement is parsed, + ** an SQLITE_FCNTL_PRAGMA file control is sent to the open sqlite3_file + ** object corresponding to the database file to which the pragma + ** statement refers. + ** + ** IMPLEMENTATION-OF: R-29875-31678 The argument to the SQLITE_FCNTL_PRAGMA + ** file control is an array of pointers to strings (char**) in which the + ** second element of the array is the name of the pragma and the third + ** element is the argument to the pragma or NULL if the pragma has no + ** argument. */ aFcntl[0] = 0; aFcntl[1] = zLeft; @@ -101893,14 +115009,8 @@ SQLITE_PRIVATE void sqlite3Pragma( db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ - if( aFcntl[0] ){ - int mem = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, mem, 0, aFcntl[0], 0); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); - sqlite3_free(aFcntl[0]); - } + returnSingleText(v, "result", aFcntl[0]); + sqlite3_free(aFcntl[0]); goto pragma_out; } if( rc!=SQLITE_NOTFOUND ){ @@ -101937,19 +115047,20 @@ SQLITE_PRIVATE void sqlite3Pragma( } } if( lwr>upr ) goto pragma_out; + pPragma = &aPragmaNames[mid]; /* Make sure the database schema is loaded if the pragma requires that */ - if( (aPragmaNames[mid].mPragFlag & PragFlag_NeedSchema)!=0 ){ + if( (pPragma->mPragFlag & PragFlag_NeedSchema)!=0 ){ if( sqlite3ReadSchema(pParse) ) goto pragma_out; } /* Jump to the appropriate pragma handler */ - switch( aPragmaNames[mid].ePragTyp ){ + switch( pPragma->ePragTyp ){ #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* - ** PRAGMA [database.]default_cache_size - ** PRAGMA [database.]default_cache_size=N + ** PRAGMA [schema.]default_cache_size + ** PRAGMA [schema.]default_cache_size=N ** ** The first form reports the current persistent setting for the ** page cache size. The value returned is the maximum number of @@ -101976,21 +115087,21 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; - int addr; + VdbeOp *aOp; sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); + setOneColumnName(v, "cache_size"); pParse->nMem += 2; - addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, iDb); - sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[6].p1 = SQLITE_DEFAULT_CACHE_SIZE; }else{ int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3VdbeAddOp2(v, OP_Integer, size, 1); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, size); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); @@ -102001,8 +115112,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) /* - ** PRAGMA [database.]page_size - ** PRAGMA [database.]page_size=N + ** PRAGMA [schema.]page_size + ** PRAGMA [schema.]page_size=N ** ** The first form reports the current setting for the ** database page size in bytes. The second form sets the @@ -102014,22 +115125,22 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( pBt!=0 ); if( !zRight ){ int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; - returnSingleInt(pParse, "page_size", size); + returnSingleInt(v, "page_size", size); }else{ /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). */ db->nextPagesize = sqlite3Atoi(zRight); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,-1,0) ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } } break; } /* - ** PRAGMA [database.]secure_delete - ** PRAGMA [database.]secure_delete=ON/OFF + ** PRAGMA [schema.]secure_delete + ** PRAGMA [schema.]secure_delete=ON/OFF ** ** The first form reports the current setting for the ** secure_delete flag. The second form changes the secure_delete @@ -102049,13 +115160,13 @@ SQLITE_PRIVATE void sqlite3Pragma( } } b = sqlite3BtreeSecureDelete(pBt, b); - returnSingleInt(pParse, "secure_delete", b); + returnSingleInt(v, "secure_delete", b); break; } /* - ** PRAGMA [database.]max_page_count - ** PRAGMA [database.]max_page_count=N + ** PRAGMA [schema.]max_page_count + ** PRAGMA [schema.]max_page_count=N ** ** The first form reports the current setting for the ** maximum number of pages in the database file. The @@ -102066,7 +115177,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** change. The only purpose is to provide an easy way to test ** the sqlite3AbsInt32() function. ** - ** PRAGMA [database.]page_count + ** PRAGMA [schema.]page_count ** ** Return the number of pages in the specified database. */ @@ -102087,8 +115198,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } /* - ** PRAGMA [database.]locking_mode - ** PRAGMA [database.]locking_mode = (normal|exclusive) + ** PRAGMA [schema.]locking_mode + ** PRAGMA [schema.]locking_mode = (normal|exclusive) */ case PragTyp_LOCKING_MODE: { const char *zRet = "normal"; @@ -102128,25 +115239,20 @@ SQLITE_PRIVATE void sqlite3Pragma( if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ zRet = "exclusive"; } - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + returnSingleText(v, "locking_mode", zRet); break; } /* - ** PRAGMA [database.]journal_mode - ** PRAGMA [database.]journal_mode = + ** PRAGMA [schema.]journal_mode + ** PRAGMA [schema.]journal_mode = ** (delete|persist|off|truncate|memory|wal|off) */ case PragTyp_JOURNAL_MODE: { int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ int ii; /* Loop counter */ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); - + setOneColumnName(v, "journal_mode"); if( zRight==0 ){ /* If there is no "=MODE" part of the pragma, do a query for the ** current mode */ @@ -102179,8 +115285,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } /* - ** PRAGMA [database.]journal_size_limit - ** PRAGMA [database.]journal_size_limit=N + ** PRAGMA [schema.]journal_size_limit + ** PRAGMA [schema.]journal_size_limit=N ** ** Get or set the size limit on rollback journal files. */ @@ -102192,15 +115298,15 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); - returnSingleInt(pParse, "journal_size_limit", iLimit); + returnSingleInt(v, "journal_size_limit", iLimit); break; } #endif /* SQLITE_OMIT_PAGER_PRAGMAS */ /* - ** PRAGMA [database.]auto_vacuum - ** PRAGMA [database.]auto_vacuum=N + ** PRAGMA [schema.]auto_vacuum + ** PRAGMA [schema.]auto_vacuum=N ** ** Get or set the value of the database 'auto-vacuum' parameter. ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL @@ -102210,7 +115316,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( !zRight ){ - returnSingleInt(pParse, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); + returnSingleInt(v, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); }else{ int eAuto = getAutoVacuum(zRight); assert( eAuto>=0 && eAuto<=2 ); @@ -102233,16 +115339,18 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, { OP_If, 1, 0, 0}, /* 2 */ { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ - { OP_Integer, 0, 1, 0}, /* 4 */ - { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ + { OP_SetCookie, 0, BTREE_INCR_VACUUM, 0}, /* 4 */ }; - int iAddr; - iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); - sqlite3VdbeChangeP1(v, iAddr, iDb); - sqlite3VdbeChangeP1(v, iAddr+1, iDb); - sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); - sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); - sqlite3VdbeChangeP1(v, iAddr+5, iDb); + VdbeOp *aOp; + int iAddr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setMeta6)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[2].p2 = iAddr+4; + aOp[4].p1 = iDb; + aOp[4].p3 = eAuto - 1; sqlite3VdbeUsesBtree(v, iDb); } } @@ -102251,7 +115359,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* - ** PRAGMA [database.]incremental_vacuum(N) + ** PRAGMA [schema.]incremental_vacuum(N) ** ** Do N steps of incremental vacuuming on a database. */ @@ -102274,8 +115382,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_PAGER_PRAGMAS /* - ** PRAGMA [database.]cache_size - ** PRAGMA [database.]cache_size=N + ** PRAGMA [schema.]cache_size + ** PRAGMA [schema.]cache_size=N ** ** The first form reports the current local setting for the ** page cache size. The second form sets the local @@ -102287,7 +115395,7 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ - returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); + returnSingleInt(v, "cache_size", pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; @@ -102297,7 +115405,50 @@ SQLITE_PRIVATE void sqlite3Pragma( } /* - ** PRAGMA [database.]mmap_size(N) + ** PRAGMA [schema.]cache_spill + ** PRAGMA cache_spill=BOOLEAN + ** PRAGMA [schema.]cache_spill=N + ** + ** The first form reports the current local setting for the + ** page cache spill size. The second form turns cache spill on + ** or off. When turnning cache spill on, the size is set to the + ** current cache_size. The third form sets a spill size that + ** may be different form the cache size. + ** If N is positive then that is the + ** number of pages in the cache. If N is negative, then the + ** number of pages is adjusted so that the cache uses -N kibibytes + ** of memory. + ** + ** If the number of cache_spill pages is less then the number of + ** cache_size pages, no spilling occurs until the page count exceeds + ** the number of cache_size pages. + ** + ** The cache_spill=BOOLEAN setting applies to all attached schemas, + ** not just the schema specified. + */ + case PragTyp_CACHE_SPILL: { + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(v, "cache_spill", + (db->flags & SQLITE_CacheSpill)==0 ? 0 : + sqlite3BtreeSetSpillSize(pDb->pBt,0)); + }else{ + int size = 1; + if( sqlite3GetInt32(zRight, &size) ){ + sqlite3BtreeSetSpillSize(pDb->pBt, size); + } + if( sqlite3GetBoolean(zRight, size!=0) ){ + db->flags |= SQLITE_CacheSpill; + }else{ + db->flags &= ~SQLITE_CacheSpill; + } + setAllPagerFlags(db); + } + break; + } + + /* + ** PRAGMA [schema.]mmap_size(N) ** ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the @@ -102332,7 +115483,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = SQLITE_OK; #endif if( rc==SQLITE_OK ){ - returnSingleInt(pParse, "mmap_size", sz); + returnSingleInt(v, "mmap_size", sz); }else if( rc!=SQLITE_NOTFOUND ){ pParse->nErr++; pParse->rc = rc; @@ -102353,7 +115504,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE: { if( !zRight ){ - returnSingleInt(pParse, "temp_store", db->temp_store); + returnSingleInt(v, "temp_store", db->temp_store); }else{ changeTempStorage(pParse, zRight); } @@ -102372,13 +115523,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE_DIRECTORY: { if( !zRight ){ - if( sqlite3_temp_directory ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "temp_store_directory", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_temp_directory, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "temp_store_directory", sqlite3_temp_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -102422,13 +115567,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_DATA_STORE_DIRECTORY: { if( !zRight ){ - if( sqlite3_data_directory ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "data_store_directory", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_data_directory, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "data_store_directory", sqlite3_data_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -102453,8 +115592,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #if SQLITE_ENABLE_LOCKING_STYLE /* - ** PRAGMA [database.]lock_proxy_file - ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" + ** PRAGMA [schema.]lock_proxy_file + ** PRAGMA [schema.]lock_proxy_file = ":auto:"|"lock_file_path" ** ** Return or set the value of the lock_proxy_file flag. Changing ** the value sets a specific file to be used for database access locks. @@ -102467,14 +115606,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3_file *pFile = sqlite3PagerFile(pPager); sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); - - if( proxy_file_path ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "lock_proxy_file", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, proxy_file_path, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "lock_proxy_file", proxy_file_path); }else{ Pager *pPager = sqlite3BtreePager(pDb->pBt); sqlite3_file *pFile = sqlite3PagerFile(pPager); @@ -102496,8 +115628,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* SQLITE_ENABLE_LOCKING_STYLE */ /* - ** PRAGMA [database.]synchronous - ** PRAGMA [database.]synchronous=OFF|ON|NORMAL|FULL + ** PRAGMA [schema.]synchronous + ** PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL|EXTRA ** ** Return or set the local value of the synchronous flag. Changing ** the local value does not make changes to the disk file and the @@ -102506,13 +115638,16 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_SYNCHRONOUS: { if( !zRight ){ - returnSingleInt(pParse, "synchronous", pDb->safety_level-1); + returnSingleInt(v, "synchronous", pDb->safety_level-1); }else{ if( !db->autoCommit ){ sqlite3ErrorMsg(pParse, "Safety level may not be changed inside a transaction"); }else{ - pDb->safety_level = getSafetyLevel(zRight,0,1)+1; + int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; + if( iLevel==0 ) iLevel = 1; + pDb->safety_level = iLevel; + pDb->bSyncSet = 1; setAllPagerFlags(db); } } @@ -102523,15 +115658,20 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_FLAG_PRAGMAS case PragTyp_FLAG: { if( zRight==0 ){ - returnSingleInt(pParse, aPragmaNames[mid].zName, - (db->flags & aPragmaNames[mid].iArg)!=0 ); + returnSingleInt(v, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); }else{ - int mask = aPragmaNames[mid].iArg; /* Mask of bits to set or clear. */ + int mask = pPragma->iArg; /* Mask of bits to set or clear. */ if( db->autoCommit==0 ){ /* Foreign key support may not be enabled or disabled while not ** in auto-commit mode. */ mask &= ~(SQLITE_ForeignKeys); } +#if SQLITE_USER_AUTHENTICATION + if( db->auth.authLevel==UAUTH_User ){ + /* Do not allow non-admin users to modify the schema arbitrarily */ + mask &= ~(SQLITE_WriteSchema); + } +#endif if( sqlite3GetBoolean(zRight, 0) ){ db->flags |= mask; @@ -102544,7 +115684,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** compiler (eg. count_changes). So add an opcode to expire all ** compiled SQL statements after modifying a pragma value. */ - sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + sqlite3VdbeAddOp0(v, OP_Expire); setAllPagerFlags(db); } break; @@ -102566,45 +115706,39 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TABLE_INFO: if( zRight ){ Table *pTab; - pTab = sqlite3FindTable(db, zRight, zDb); + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); if( pTab ){ + static const char *azCol[] = { + "cid", "name", "type", "notnull", "dflt_value", "pk" + }; int i, k; int nHidden = 0; Column *pCol; Index *pPk = sqlite3PrimaryKeyIndex(pTab); - sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", SQLITE_STATIC); + setAllColumnNames(v, 6, azCol); assert( 6==ArraySize(azCol) ); sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ if( IsHiddenColumn(pCol) ){ nHidden++; continue; } - sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - pCol->zType ? pCol->zType : "", 0); - sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4); - if( pCol->zDflt ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pCol->zDflt, 0); - }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, 5); - } if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ k = 0; }else if( pPk==0 ){ k = 1; }else{ - for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} } - sqlite3VdbeAddOp2(v, OP_Integer, k, 6); + assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN ); + sqlite3VdbeMultiLoad(v, 1, "issisi", + i-nHidden, + pCol->zName, + sqlite3ColumnType(pCol,""), + pCol->notNull ? 1 : 0, + pCol->pDflt ? pCol->pDflt->u.zToken : 0, + k); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } @@ -102612,31 +115746,26 @@ SQLITE_PRIVATE void sqlite3Pragma( break; case PragTyp_STATS: { + static const char *azCol[] = { "table", "index", "width", "height" }; Index *pIdx; HashElem *i; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 4); pParse->nMem = 4; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "index", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "width", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "height", SQLITE_STATIC); + setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, pTab->zName, 0); - sqlite3VdbeAddOp2(v, OP_Null, 0, 2); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); + sqlite3VdbeMultiLoad(v, 1, "ssii", + pTab->zName, + 0, + pTab->szTabRow, + pTab->nRowLogEst); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); + sqlite3VdbeMultiLoad(v, 2, "sii", + pIdx->zName, + pIdx->szIdxRow, + pIdx->aiRowLogEst[0]); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } @@ -102648,21 +115777,35 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); if( pIdx ){ + static const char *azCol[] = { + "seqno", "cid", "name", "desc", "coll", "key" + }; int i; + int mx; + if( pPragma->iArg ){ + /* PRAGMA index_xinfo (newer version with more rows and columns) */ + mx = pIdx->nColumn; + pParse->nMem = 6; + }else{ + /* PRAGMA index_info (legacy version) */ + mx = pIdx->nKeyCol; + pParse->nMem = 3; + } pTab = pIdx->pTable; - sqlite3VdbeSetNumCols(v, 3); - pParse->nMem = 3; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); - for(i=0; inKeyCol; i++){ + assert( pParse->nMem<=ArraySize(azCol) ); + setAllColumnNames(v, pParse->nMem, azCol); + for(i=0; iaiColumn[i]; - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); - assert( pTab->nCol>cnum ); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pTab->aCol[cnum].zName, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + sqlite3VdbeMultiLoad(v, 1, "iis", i, cnum, + cnum<0 ? 0 : pTab->aCol[cnum].zName); + if( pPragma->iArg ){ + sqlite3VdbeMultiLoad(v, 4, "isi", + pIdx->aSortOrder[i], + pIdx->azColl[i], + inKeyCol); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, pParse->nMem); } } } @@ -102674,53 +115817,53 @@ SQLITE_PRIVATE void sqlite3Pragma( int i; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ + static const char *azCol[] = { + "seq", "name", "unique", "origin", "partial" + }; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 3); - pParse->nMem = 3; + pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); + setAllColumnNames(v, 5, azCol); assert( 5==ArraySize(azCol) ); for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, IsUniqueIndex(pIdx), 3); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + const char *azOrigin[] = { "c", "u", "pk" }; + sqlite3VdbeMultiLoad(v, 1, "isisi", + i, + pIdx->zName, + IsUniqueIndex(pIdx), + azOrigin[pIdx->idxType], + pIdx->pPartIdxWhere!=0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } } break; case PragTyp_DATABASE_LIST: { + static const char *azCol[] = { "seq", "name", "file" }; int i; - sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", SQLITE_STATIC); + setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); for(i=0; inDb; i++){ if( db->aDb[i].pBt==0 ) continue; - assert( db->aDb[i].zName!=0 ); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, db->aDb[i].zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); + assert( db->aDb[i].zDbSName!=0 ); + sqlite3VdbeMultiLoad(v, 1, "iss", + i, + db->aDb[i].zDbSName, + sqlite3BtreeGetFilename(db->aDb[i].pBt)); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } break; case PragTyp_COLLATION_LIST: { + static const char *azCol[] = { "seq", "name" }; int i = 0; HashElem *p; - sqlite3VdbeSetNumCols(v, 2); pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ CollSeq *pColl = (CollSeq *)sqliteHashData(p); - sqlite3VdbeAddOp2(v, OP_Integer, i++, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); + sqlite3VdbeMultiLoad(v, 1, "is", i++, pColl->zName); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } } @@ -102736,33 +115879,26 @@ SQLITE_PRIVATE void sqlite3Pragma( v = sqlite3GetVdbe(pParse); pFK = pTab->pFKey; if( pFK ){ + static const char *azCol[] = { + "id", "seq", "table", "from", "to", "on_update", "on_delete", + "match" + }; int i = 0; - sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "on_update", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 6, COLNAME_NAME, "on_delete", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 7, COLNAME_NAME, "match", SQLITE_STATIC); + setAllColumnNames(v, 8, azCol); assert( 8==ArraySize(azCol) ); while(pFK){ int j; for(j=0; jnCol; j++){ - char *zCol = pFK->aCol[j].zCol; - char *zOnDelete = (char *)actionName(pFK->aAction[0]); - char *zOnUpdate = (char *)actionName(pFK->aAction[1]); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, j, 2); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pFK->zTo, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - pTab->aCol[pFK->aCol[j].iFrom].zName, 0); - sqlite3VdbeAddOp4(v, zCol ? OP_String8 : OP_Null, 0, 5, 0, zCol, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 6, 0, zOnUpdate, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 7, 0, zOnDelete, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 8, 0, "NONE", 0); + sqlite3VdbeMultiLoad(v, 1, "iissssss", + i, + j, + pFK->zTo, + pTab->aCol[pFK->aCol[j].iFrom].zName, + pFK->aCol[j].zCol, + actionName(pFK->aAction[1]), /* ON UPDATE */ + actionName(pFK->aAction[0]), /* ON DELETE */ + "NONE"); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); } ++i; @@ -102791,17 +115927,14 @@ SQLITE_PRIVATE void sqlite3Pragma( int addrTop; /* Top of a loop checking foreign keys */ int addrOk; /* Jump here if the key is OK */ int *aiCols; /* child to parent column mapping */ + static const char *azCol[] = { "table", "rowid", "parent", "fkid" }; regResult = pParse->nMem+1; pParse->nMem += 4; regKey = ++pParse->nMem; regRow = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 4); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); sqlite3CodeVerifySchema(pParse, iDb); k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); while( k ){ @@ -102816,8 +115949,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, - P4_TRANSIENT); + sqlite3VdbeLoadString(v, regResult, pTab->zName); for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ pParent = sqlite3FindTable(db, pFK->zTo, zDb); if( pParent==0 ) continue; @@ -102856,13 +115988,11 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); sqlite3ColumnDefault(v, pTab, iKey, regRow); sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, - sqlite3VdbeCurrentAddr(v)+3); VdbeCoverage(v); }else{ sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); } - sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeAddOp3(v, OP_SeekRowid, i, 0, regRow); VdbeCoverage(v); + sqlite3VdbeGoto(v, addrOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); }else{ for(j=0; jnCol; j++){ @@ -102872,15 +116002,13 @@ SQLITE_PRIVATE void sqlite3Pragma( } if( pParent ){ sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, - sqlite3IndexAffinityStr(v,pIdx), pFK->nCol); + sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); VdbeCoverage(v); } } sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); - sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, - pFK->zTo, P4_TRANSIENT); - sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeMultiLoad(v, regResult+2, "si", pFK->zTo, i-1); sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); sqlite3VdbeResolveLabel(v, addrOk); sqlite3DbFree(db, aiCols); @@ -102897,7 +116025,7 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_PARSER_TRACE: { if( zRight ){ if( sqlite3GetBoolean(zRight, 0) ){ - sqlite3ParserTrace(stderr, "parser: "); + sqlite3ParserTrace(stdout, "parser: "); }else{ sqlite3ParserTrace(0, 0); } @@ -102928,17 +116056,6 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_INTEGRITY_CHECK: { int i, j, addr, mxErr; - /* Code that appears at the end of the integrity check. If no error - ** messages have been generated, output OK. Otherwise output the - ** error message - */ - static const int iLn = VDBE_OFFSET_LINENO(2); - static const VdbeOpList endCode[] = { - { OP_IfNeg, 1, 0, 0}, /* 0 */ - { OP_String8, 0, 3, 0}, /* 1 */ - { OP_ResultRow, 3, 1, 0}, - }; - int isQuick = (sqlite3Tolower(zLeft[0])=='q'); /* If the PRAGMA command was of the form "PRAGMA .integrity_check", @@ -102956,8 +116073,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Initialize the VDBE program */ pParse->nMem = 6; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); + setOneColumnName(v, "integrity_check"); /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; @@ -102973,7 +116089,10 @@ SQLITE_PRIVATE void sqlite3Pragma( for(i=0; inDb; i++){ HashElem *x; Hash *pTbls; + int *aRoot; int cnt = 0; + int mxIdx = 0; + int nIdx; if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -102986,35 +116105,39 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Do an integrity check of the B-Tree ** - ** Begin by filling registers 2, 3, ... with the root pages numbers + ** Begin by finding the root pages numbers ** for all tables and indices in the database. */ assert( sqlite3SchemaMutexHeld(db, i, 0) ); pTbls = &db->aDb[i].pSchema->tblHash; - for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - if( HasRowid(pTab) ){ - sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); - VdbeComment((v, "%s", pTab->zName)); - cnt++; - } + if( HasRowid(pTab) ) cnt++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } + if( nIdx>mxIdx ) mxIdx = nIdx; + } + aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(cnt+1)); + if( aRoot==0 ) break; + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( HasRowid(pTab) ) aRoot[cnt++] = pTab->tnum; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); - VdbeComment((v, "%s", pIdx->zName)); - cnt++; + aRoot[cnt++] = pIdx->tnum; } } + aRoot[cnt] = 0; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, cnt+8 ); + pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), P4_DYNAMIC); sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); @@ -103038,13 +116161,14 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); sqlite3ExprCacheClear(pParse); - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ } - pParse->nMem = MAX(pParse->nMem, 8+j); + assert( pParse->nMem>=8+j ); + assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); /* Verify that all NOT NULL columns really are NOT NULL */ @@ -103079,13 +116203,11 @@ SQLITE_PRIVATE void sqlite3Pragma( jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); + sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - " missing from index ", P4_STATIC); + sqlite3VdbeLoadString(v, 4, " missing from index "); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - jmp5 = sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - pIdx->zName, P4_TRANSIENT); + jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); @@ -103100,20 +116222,19 @@ SQLITE_PRIVATE void sqlite3Pragma( int kk; for(kk=0; kknKeyCol; kk++){ int iCol = pIdx->aiColumn[kk]; - assert( iCol>=0 && iColnCol ); - if( pTab->aCol[iCol].notNull ) continue; + assert( iCol!=XN_ROWID && iColnCol ); + if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); VdbeCoverage(v); } jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, uniqOk); + sqlite3VdbeGoto(v, uniqOk); sqlite3VdbeJumpHere(v, jmp6); sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, pIdx->nKeyCol); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - "non-unique entry in index ", P4_STATIC); - sqlite3VdbeAddOp2(v, OP_Goto, 0, jmp5); + sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); + sqlite3VdbeGoto(v, jmp5); sqlite3VdbeResolveLabel(v, uniqOk); } sqlite3VdbeJumpHere(v, jmp4); @@ -103122,8 +116243,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); #ifndef SQLITE_OMIT_BTREECOUNT - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, - "wrong # of entries in index ", P4_STATIC); + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ if( pPk==pIdx ) continue; addr = sqlite3VdbeCurrentAddr(v); @@ -103133,17 +116253,30 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pIdx->zName, P4_TRANSIENT); + sqlite3VdbeLoadString(v, 3, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); } #endif /* SQLITE_OMIT_BTREECOUNT */ } } - addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); - sqlite3VdbeChangeP3(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); + { + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList endCode[] = { + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_If, 1, 4, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ + { OP_ResultRow, 3, 1, 0}, /* 3 */ + }; + VdbeOp *aOp; + + aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); + if( aOp ){ + aOp[0].p2 = -mxErr; + aOp[2].p4type = P4_STATIC; + aOp[2].p4.z = "ok"; + } + } } break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -103189,14 +116322,10 @@ SQLITE_PRIVATE void sqlite3Pragma( const struct EncName *pEnc; if( !zRight ){ /* "PRAGMA encoding" */ if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_String8, 0, 1); assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); - sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + returnSingleText(v, "encoding", encnames[ENC(pParse->db)].zName); }else{ /* "PRAGMA encoding = XXX" */ /* Only change the value of sqlite.enc if the database handle is not ** initialized. If the main database exists, the new sqlite.enc value @@ -103209,7 +116338,8 @@ SQLITE_PRIVATE void sqlite3Pragma( ){ for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ - ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + SCHEMA_ENC(db) = ENC(db) = + pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; break; } } @@ -103224,16 +116354,18 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS /* - ** PRAGMA [database.]schema_version - ** PRAGMA [database.]schema_version = + ** PRAGMA [schema.]schema_version + ** PRAGMA [schema.]schema_version = ** - ** PRAGMA [database.]user_version - ** PRAGMA [database.]user_version = + ** PRAGMA [schema.]user_version + ** PRAGMA [schema.]user_version = ** - ** PRAGMA [database.]freelist_count = + ** PRAGMA [schema.]freelist_count ** - ** PRAGMA [database.]application_id - ** PRAGMA [database.]application_id = + ** PRAGMA [schema.]data_version + ** + ** PRAGMA [schema.]application_id + ** PRAGMA [schema.]application_id = ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both @@ -103254,35 +116386,22 @@ SQLITE_PRIVATE void sqlite3Pragma( ** applications for any purpose. */ case PragTyp_HEADER_VALUE: { - int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ + int iCookie = pPragma->iArg; /* Which cookie to read or write */ sqlite3VdbeUsesBtree(v, iDb); - switch( zLeft[0] ){ - case 'a': case 'A': - iCookie = BTREE_APPLICATION_ID; - break; - case 'f': case 'F': - iCookie = BTREE_FREE_PAGE_COUNT; - break; - case 's': case 'S': - iCookie = BTREE_SCHEMA_VERSION; - break; - default: - iCookie = BTREE_USER_VERSION; - break; - } - - if( zRight && iCookie!=BTREE_FREE_PAGE_COUNT ){ + if( zRight && (pPragma->mPragFlag & PragFlag_ReadOnly)==0 ){ /* Write the specified cookie value */ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ - { OP_Integer, 0, 1, 0}, /* 1 */ - { OP_SetCookie, 0, 0, 1}, /* 2 */ + { OP_SetCookie, 0, 0, 0}, /* 1 */ }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); - sqlite3VdbeChangeP1(v, addr+2, iDb); - sqlite3VdbeChangeP2(v, addr+2, iCookie); + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[1].p2 = iCookie; + aOp[1].p3 = sqlite3Atoi(zRight); }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { @@ -103290,12 +116409,16 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_ReadCookie, 0, 1, 0}, /* 1 */ { OP_ResultRow, 1, 1, 0} }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie, 0); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, iDb); - sqlite3VdbeChangeP3(v, addr+1, iCookie); + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(readCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(readCookie),readCookie,0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[1].p3 = iCookie; sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); + sqlite3VdbeReusable(v); } } break; @@ -103311,24 +116434,25 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_COMPILE_OPTIONS: { int i = 0; const char *zOpt; - sqlite3VdbeSetNumCols(v, 1); pParse->nMem = 1; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE_STATIC); + setOneColumnName(v, "compile_option"); while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); + sqlite3VdbeLoadString(v, 1, zOpt); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } + sqlite3VdbeReusable(v); } break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]wal_checkpoint = passive|full|restart + ** PRAGMA [schema.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ case PragTyp_WAL_CHECKPOINT: { + static const char *azCol[] = { "busy", "log", "checkpointed" }; int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); int eMode = SQLITE_CHECKPOINT_PASSIVE; if( zRight ){ @@ -103336,14 +116460,12 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_FULL; }else if( sqlite3StrICmp(zRight, "restart")==0 ){ eMode = SQLITE_CHECKPOINT_RESTART; + }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ + eMode = SQLITE_CHECKPOINT_TRUNCATE; } } - sqlite3VdbeSetNumCols(v, 3); + setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC); - sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } @@ -103361,7 +116483,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight ){ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } - returnSingleInt(pParse, "wal_autocheckpoint", + returnSingleInt(v, "wal_autocheckpoint", db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); } @@ -103371,8 +116493,9 @@ SQLITE_PRIVATE void sqlite3Pragma( /* ** PRAGMA shrink_memory ** - ** This pragma attempts to free as much memory as possible from the - ** current database connection. + ** IMPLEMENTATION-OF: R-23445-46109 This pragma causes the database + ** connection on which it is invoked to free up as much memory as it + ** can, by calling sqlite3_db_release_memory(). */ case PragTyp_SHRINK_MEMORY: { sqlite3_db_release_memory(db); @@ -103389,11 +116512,11 @@ SQLITE_PRIVATE void sqlite3Pragma( ** disables the timeout. */ /*case PragTyp_BUSY_TIMEOUT*/ default: { - assert( aPragmaNames[mid].ePragTyp==PragTyp_BUSY_TIMEOUT ); + assert( pPragma->ePragTyp==PragTyp_BUSY_TIMEOUT ); if( zRight ){ sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); } - returnSingleInt(pParse, "timeout", db->busyTimeout); + returnSingleInt(v, "timeout", db->busyTimeout); break; } @@ -103401,15 +116524,39 @@ SQLITE_PRIVATE void sqlite3Pragma( ** PRAGMA soft_heap_limit ** PRAGMA soft_heap_limit = N ** - ** Call sqlite3_soft_heap_limit64(N). Return the result. If N is omitted, - ** use -1. + ** IMPLEMENTATION-OF: R-26343-45930 This pragma invokes the + ** sqlite3_soft_heap_limit64() interface with the argument N, if N is + ** specified and is a non-negative integer. + ** IMPLEMENTATION-OF: R-64451-07163 The soft_heap_limit pragma always + ** returns the same integer that would be returned by the + ** sqlite3_soft_heap_limit64(-1) C-language function. */ case PragTyp_SOFT_HEAP_LIMIT: { sqlite3_int64 N; if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } - returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + returnSingleInt(v, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + break; + } + + /* + ** PRAGMA threads + ** PRAGMA threads = N + ** + ** Configure the maximum number of worker threads. Return the new + ** maximum, which might be less than requested. + */ + case PragTyp_THREADS: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); + } + returnSingleInt(v, "threads", + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); break; } @@ -103421,25 +116568,23 @@ SQLITE_PRIVATE void sqlite3Pragma( static const char *const azLockName[] = { "unlocked", "shared", "reserved", "pending", "exclusive" }; + static const char *azCol[] = { "database", "status" }; int i; - sqlite3VdbeSetNumCols(v, 2); + setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; inDb; i++){ Btree *pBt; const char *zState = "unknown"; int j; - if( db->aDb[i].zName==0 ) continue; - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); + if( db->aDb[i].zDbSName==0 ) continue; pBt = db->aDb[i].pBt; if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; - }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, + }else if( sqlite3_file_control(db, i ? db->aDb[i].zDbSName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ zState = azLockName[j]; } - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); + sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zDbSName, zState); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } break; @@ -103515,6 +116660,7 @@ pragma_out: ** interface, and routines that contribute to loading the database schema ** from disk. */ +/* #include "sqliteInt.h" */ /* ** Fill the InitData structure with an error message that indicates @@ -103527,15 +116673,14 @@ static void corruptSchema( ){ sqlite3 *db = pData->db; if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ + char *z; if( zObj==0 ) zObj = "?"; - sqlite3SetString(pData->pzErrMsg, db, - "malformed database schema (%s)", zObj); - if( zExtra ){ - *pData->pzErrMsg = sqlite3MAppendf(db, *pData->pzErrMsg, - "%s - %s", *pData->pzErrMsg, zExtra); - } + z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); + if( zExtra ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); + sqlite3DbFree(db, *pData->pzErrMsg); + *pData->pzErrMsg = z; } - pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT_BKPT; + pData->rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_CORRUPT_BKPT; } /* @@ -103568,13 +116713,14 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ if( argv[1]==0 ){ corruptSchema(pData, argv[0], 0); - }else if( argv[2] && argv[2][0] ){ + }else if( sqlite3_strnicmp(argv[2],"create ",7)==0 ){ /* Call the parser to process a CREATE TABLE, INDEX or VIEW. ** But because db->init.busy is set to 1, no VDBE code is generated ** or executed. All the parser does is build the internal data ** structures that describe the table, index, or view. */ int rc; + u8 saved_iDb = db->init.iDb; sqlite3_stmt *pStmt; TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ @@ -103585,22 +116731,23 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); - db->init.iDb = 0; + db->init.iDb = saved_iDb; + assert( saved_iDb==0 || (db->flags & SQLITE_Vacuum)!=0 ); if( SQLITE_OK!=rc ){ if( db->init.orphanTrigger ){ assert( iDb==1 ); }else{ pData->rc = rc; if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ corruptSchema(pData, argv[0], sqlite3_errmsg(db)); } } } sqlite3_finalize(pStmt); - }else if( argv[0]==0 ){ - corruptSchema(pData, 0, 0); + }else if( argv[0]==0 || (argv[2]!=0 && argv[2][0]!=0) ){ + corruptSchema(pData, argv[0], 0); }else{ /* If the SQL column is blank it means this is an index that ** was created to be the PRIMARY KEY or to fulfill a UNIQUE @@ -103609,7 +116756,7 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char ** to do here is record the root page number for that index. */ Index *pIndex; - pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zName); + pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zDbSName); if( pIndex==0 ){ /* This can occur if there exists an index on a TEMP table which ** has the same name as another index on a permanent index. Since @@ -103638,61 +116785,27 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ #ifndef SQLITE_OMIT_DEPRECATED int size; #endif - Table *pTab; Db *pDb; char const *azArg[4]; int meta[5]; InitData initData; - char const *zMasterSchema; - char const *zMasterName; + const char *zMasterName; int openedTransaction = 0; - /* - ** The master database table has a structure like this - */ - static const char master_schema[] = - "CREATE TABLE sqlite_master(\n" - " type text,\n" - " name text,\n" - " tbl_name text,\n" - " rootpage integer,\n" - " sql text\n" - ")" - ; -#ifndef SQLITE_OMIT_TEMPDB - static const char temp_master_schema[] = - "CREATE TEMP TABLE sqlite_temp_master(\n" - " type text,\n" - " name text,\n" - " tbl_name text,\n" - " rootpage integer,\n" - " sql text\n" - ")" - ; -#else - #define temp_master_schema 0 -#endif - assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pSchema ); assert( sqlite3_mutex_held(db->mutex) ); assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); - /* zMasterSchema and zInitScript are set to point at the master schema - ** and initialisation script appropriate for the database being - ** initialized. zMasterName is the name of the master table. - */ - if( !OMIT_TEMPDB && iDb==1 ){ - zMasterSchema = temp_master_schema; - }else{ - zMasterSchema = master_schema; - } - zMasterName = SCHEMA_TABLE(iDb); - - /* Construct the schema tables. */ - azArg[0] = zMasterName; + /* Construct the in-memory representation schema tables (sqlite_master or + ** sqlite_temp_master) by invoking the parser directly. The appropriate + ** table name will be inserted automatically by the parser so we can just + ** use the abbreviation "x" here. The parser will also automatically tag + ** the schema table as read-only. */ + azArg[0] = zMasterName = SCHEMA_TABLE(iDb); azArg[1] = "1"; - azArg[2] = zMasterSchema; + azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text," + "rootpage integer,sql text)"; azArg[3] = 0; initData.db = db; initData.iDb = iDb; @@ -103703,10 +116816,6 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ rc = initData.rc; goto error_out; } - pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName); - if( ALWAYS(pTab) ){ - pTab->tabFlags |= TF_Readonly; - } /* Create a cursor to hold the database open */ @@ -103725,7 +116834,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); if( rc!=SQLITE_OK ){ - sqlite3SetString(pzErrMsg, db, "%s", sqlite3ErrStr(rc)); + sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc)); goto initone_error_out; } openedTransaction = 1; @@ -103825,11 +116934,11 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ { char *zSql; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid", - db->aDb[iDb].zName, zMasterName); + "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid", + db->aDb[iDb].zDbSName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + sqlite3_xauth xAuth; xAuth = db->xAuth; db->xAuth = 0; #endif @@ -103847,7 +116956,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ #endif } if( db->mallocFailed ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; sqlite3ResetAllSchemasOfConnection(db); } if( rc==SQLITE_OK || (db->flags&SQLITE_RecoveryMode)){ @@ -103875,7 +116984,7 @@ initone_error_out: error_out: if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } return rc; } @@ -103895,8 +117004,11 @@ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int commit_internal = !(db->flags&SQLITE_InternChanges); assert( sqlite3_mutex_held(db->mutex) ); + assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) ); + assert( db->init.busy==0 ); rc = SQLITE_OK; db->init.busy = 1; + ENC(db) = SCHEMA_ENC(db); for(i=0; rc==SQLITE_OK && inDb; i++){ if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; rc = sqlite3InitOne(db, i, pzErrMsg); @@ -103910,8 +117022,8 @@ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ ** schema may contain references to objects in other databases. */ #ifndef SQLITE_OMIT_TEMPDB - if( rc==SQLITE_OK && ALWAYS(db->nDb>1) - && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ + assert( db->nDb>1 ); + if( rc==SQLITE_OK && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ rc = sqlite3InitOne(db, 1, pzErrMsg); if( rc ){ sqlite3ResetOneSchema(db, 1); @@ -103970,7 +117082,7 @@ static void schemaIsValid(Parse *pParse){ if( !sqlite3BtreeIsInReadTrans(pBt) ){ rc = sqlite3BtreeBeginTrans(pBt, 0); if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } if( rc!=SQLITE_OK ) return; openedTransaction = 1; @@ -104033,6 +117145,11 @@ SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ sqlite3 *db = pParse->db; sqlite3DbFree(db, pParse->aLabel); sqlite3ExprListDelete(db, pParse->pConstExpr); + if( db ){ + assert( db->lookaside.bDisable >= pParse->disableLookaside ); + db->lookaside.bDisable -= pParse->disableLookaside; + } + pParse->disableLookaside = 0; } } @@ -104048,20 +117165,16 @@ static int sqlite3Prepare( sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ - Parse *pParse; /* Parsing context */ char *zErrMsg = 0; /* Error message */ int rc = SQLITE_OK; /* Result code */ int i; /* Loop counter */ + Parse sParse; /* Parsing context */ - /* Allocate the parsing context */ - pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM; - goto end_prepare; - } - pParse->pReprepare = pReprepare; + memset(&sParse, 0, PARSE_HDR_SZ); + memset(PARSE_TAIL(&sParse), 0, PARSE_TAIL_SZ); + sParse.pReprepare = pReprepare; assert( ppStmt && *ppStmt==0 ); - assert( !db->mallocFailed ); + /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ assert( sqlite3_mutex_held(db->mutex) ); /* Check to verify that it is possible to get a read lock on all @@ -104093,8 +117206,8 @@ static int sqlite3Prepare( assert( sqlite3BtreeHoldsMutex(pBt) ); rc = sqlite3BtreeSchemaLocked(pBt); if( rc ){ - const char *zDb = db->aDb[i].zName; - sqlite3Error(db, rc, "database schema is locked: %s", zDb); + const char *zDb = db->aDb[i].zDbSName; + sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); testcase( db->flags & SQLITE_ReadUncommitted ); goto end_prepare; } @@ -104103,98 +117216,93 @@ static int sqlite3Prepare( sqlite3VtabUnlockList(db); - pParse->db = db; - pParse->nQueryLoop = 0; /* Logarithmic, so 0 really means 1 */ + sParse.db = db; if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; testcase( nBytes==mxLen ); testcase( nBytes==mxLen+1 ); if( nBytes>mxLen ){ - sqlite3Error(db, SQLITE_TOOBIG, "statement too long"); + sqlite3ErrorWithMsg(db, SQLITE_TOOBIG, "statement too long"); rc = sqlite3ApiExit(db, SQLITE_TOOBIG); goto end_prepare; } zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); if( zSqlCopy ){ - sqlite3RunParser(pParse, zSqlCopy, &zErrMsg); + sqlite3RunParser(&sParse, zSqlCopy, &zErrMsg); + sParse.zTail = &zSql[sParse.zTail-zSqlCopy]; sqlite3DbFree(db, zSqlCopy); - pParse->zTail = &zSql[pParse->zTail-zSqlCopy]; }else{ - pParse->zTail = &zSql[nBytes]; + sParse.zTail = &zSql[nBytes]; } }else{ - sqlite3RunParser(pParse, zSql, &zErrMsg); + sqlite3RunParser(&sParse, zSql, &zErrMsg); } - assert( 0==pParse->nQueryLoop ); + assert( 0==sParse.nQueryLoop ); - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; - } - if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; - if( pParse->checkSchema ){ - schemaIsValid(pParse); + if( sParse.rc==SQLITE_DONE ) sParse.rc = SQLITE_OK; + if( sParse.checkSchema ){ + schemaIsValid(&sParse); } if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; + sParse.rc = SQLITE_NOMEM_BKPT; } if( pzTail ){ - *pzTail = pParse->zTail; + *pzTail = sParse.zTail; } - rc = pParse->rc; + rc = sParse.rc; #ifndef SQLITE_OMIT_EXPLAIN - if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ + if( rc==SQLITE_OK && sParse.pVdbe && sParse.explain ){ static const char * const azColName[] = { "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", "selectid", "order", "from", "detail" }; int iFirst, mx; - if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(pParse->pVdbe, 4); + if( sParse.explain==2 ){ + sqlite3VdbeSetNumCols(sParse.pVdbe, 4); iFirst = 8; mx = 12; }else{ - sqlite3VdbeSetNumCols(pParse->pVdbe, 8); + sqlite3VdbeSetNumCols(sParse.pVdbe, 8); iFirst = 0; mx = 8; } for(i=iFirst; ipVdbe, i-iFirst, COLNAME_NAME, + sqlite3VdbeSetColName(sParse.pVdbe, i-iFirst, COLNAME_NAME, azColName[i], SQLITE_STATIC); } } #endif if( db->init.busy==0 ){ - Vdbe *pVdbe = pParse->pVdbe; - sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); + Vdbe *pVdbe = sParse.pVdbe; + sqlite3VdbeSetSql(pVdbe, zSql, (int)(sParse.zTail-zSql), saveSqlFlag); } - if( pParse->pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(pParse->pVdbe); + if( sParse.pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ + sqlite3VdbeFinalize(sParse.pVdbe); assert(!(*ppStmt)); }else{ - *ppStmt = (sqlite3_stmt*)pParse->pVdbe; + *ppStmt = (sqlite3_stmt*)sParse.pVdbe; } if( zErrMsg ){ - sqlite3Error(db, rc, "%s", zErrMsg); + sqlite3ErrorWithMsg(db, rc, "%s", zErrMsg); sqlite3DbFree(db, zErrMsg); }else{ - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } /* Delete any TriggerPrg structures allocated while parsing this statement. */ - while( pParse->pTriggerPrg ){ - TriggerPrg *pT = pParse->pTriggerPrg; - pParse->pTriggerPrg = pT->pNext; + while( sParse.pTriggerPrg ){ + TriggerPrg *pT = sParse.pTriggerPrg; + sParse.pTriggerPrg = pT->pNext; sqlite3DbFree(db, pT); } end_prepare: - sqlite3ParserReset(pParse); - sqlite3StackFree(db, pParse); + sqlite3ParserReset(&sParse); rc = sqlite3ApiExit(db, rc); assert( (rc&db->errMask)==rc ); return rc; @@ -104209,9 +117317,12 @@ static int sqlite3LockAndPrepare( const char **pzTail /* OUT: End of parsed string */ ){ int rc; - assert( ppStmt!=0 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -104249,7 +117360,7 @@ SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ rc = sqlite3LockAndPrepare(db, zSql, -1, 0, p, &pNew, 0); if( rc ){ if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } assert( pNew==0 ); return rc; @@ -104318,9 +117429,11 @@ static int sqlite3Prepare16( const char *zTail8 = 0; int rc = SQLITE_OK; - assert( ppStmt ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } if( nBytes>=0 ){ @@ -104401,6 +117514,22 @@ SQLITE_API int sqlite3_prepare16_v2( ** This file contains C code routines that are called by the parser ** to handle SELECT statements in SQLite. */ +/* #include "sqliteInt.h" */ + +/* +** Trace output macros +*/ +#if SELECTTRACE_ENABLED +/***/ int sqlite3SelectTrace = 0; +# define SELECTTRACE(K,P,S,X) \ + if(sqlite3SelectTrace&(K)) \ + sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",\ + (S)->zSelName,(S)),\ + sqlite3DebugPrintf X +#else +# define SELECTTRACE(K,P,S,X) +#endif + /* ** An instance of the following object is used to record information about @@ -104427,25 +117556,32 @@ struct SortCtx { int regReturn; /* Register holding block-output return address */ int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ + u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* -** Delete all the content of a Select structure but do not deallocate -** the select structure itself. +** Delete all the content of a Select structure. Deallocate the structure +** itself only if bFree is true. */ -static void clearSelect(sqlite3 *db, Select *p){ - sqlite3ExprListDelete(db, p->pEList); - sqlite3SrcListDelete(db, p->pSrc); - sqlite3ExprDelete(db, p->pWhere); - sqlite3ExprListDelete(db, p->pGroupBy); - sqlite3ExprDelete(db, p->pHaving); - sqlite3ExprListDelete(db, p->pOrderBy); - sqlite3SelectDelete(db, p->pPrior); - sqlite3ExprDelete(db, p->pLimit); - sqlite3ExprDelete(db, p->pOffset); - sqlite3WithDelete(db, p->pWith); +static void clearSelect(sqlite3 *db, Select *p, int bFree){ + while( p ){ + Select *pPrior = p->pPrior; + sqlite3ExprListDelete(db, p->pEList); + sqlite3SrcListDelete(db, p->pSrc); + sqlite3ExprDelete(db, p->pWhere); + sqlite3ExprListDelete(db, p->pGroupBy); + sqlite3ExprDelete(db, p->pHaving); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3ExprDelete(db, p->pLimit); + sqlite3ExprDelete(db, p->pOffset); + if( p->pWith ) sqlite3WithDelete(db, p->pWith); + if( bFree ) sqlite3DbFree(db, p); + p = pPrior; + bFree = 1; + } } /* @@ -104454,7 +117590,7 @@ static void clearSelect(sqlite3 *db, Select *p){ SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ pDest->eDest = (u8)eDest; pDest->iSDParm = iParm; - pDest->affSdst = 0; + pDest->zAffSdst = 0; pDest->iSdst = 0; pDest->nSdst = 0; } @@ -104472,40 +117608,46 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - u16 selFlags, /* Flag parameters, such as SF_Distinct */ + u32 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ Select *pNew; Select standin; sqlite3 *db = pParse->db; - pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); - assert( db->mallocFailed || !pOffset || pLimit ); /* OFFSET implies LIMIT */ + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); if( pNew==0 ){ assert( db->mallocFailed ); pNew = &standin; - memset(pNew, 0, sizeof(*pNew)); } if( pEList==0 ){ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0)); + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0)); } pNew->pEList = pEList; + pNew->op = TK_SELECT; + pNew->selFlags = selFlags; + pNew->iLimit = 0; + pNew->iOffset = 0; +#if SELECTTRACE_ENABLED + pNew->zSelName[0] = 0; +#endif + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = 0; if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; - pNew->selFlags = selFlags; - pNew->op = TK_SELECT; + pNew->pPrior = 0; + pNew->pNext = 0; pNew->pLimit = pLimit; pNew->pOffset = pOffset; - assert( pOffset==0 || pLimit!=0 ); - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; + pNew->pWith = 0; + assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || db->mallocFailed!=0 ); if( db->mallocFailed ) { - clearSelect(db, pNew); - if( pNew!=&standin ) sqlite3DbFree(db, pNew); + clearSelect(db, pNew, pNew!=&standin); pNew = 0; }else{ assert( pNew->pSrc!=0 || pParse->nErr>0 ); @@ -104514,14 +117656,23 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( return pNew; } +#if SELECTTRACE_ENABLED +/* +** Set the name of a Select object +*/ +SQLITE_PRIVATE void sqlite3SelectSetName(Select *p, const char *zName){ + if( p && zName ){ + sqlite3_snprintf(sizeof(p->zSelName), p->zSelName, "%s", zName); + } +} +#endif + + /* ** Delete the given Select structure and all of its substructures. */ SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ - if( p ){ - clearSelect(db, p); - sqlite3DbFree(db, p); - } + if( p ) clearSelect(db, p, 1); } /* @@ -104727,6 +117878,12 @@ static void setJoinExpr(Expr *p, int iTable){ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(p, EP_NoReduce); p->iRightJoinTable = (i16)iTable; + if( p->op==TK_FUNCTION && p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + setJoinExpr(p->x.pList->a[i].pExpr, iTable); + } + } setJoinExpr(p->pLeft, iTable); p = p->pRight; } @@ -104761,12 +117918,12 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ int isOuter; if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; - isOuter = (pRight->jointype & JT_OUTER)!=0; + isOuter = (pRight->fg.jointype & JT_OUTER)!=0; /* When the NATURAL keyword is present, add WHERE clause terms for ** every column that the two tables have in common. */ - if( pRight->jointype & JT_NATURAL ){ + if( pRight->fg.jointype & JT_NATURAL ){ if( pRight->pOn || pRight->pUsing ){ sqlite3ErrorMsg(pParse, "a NATURAL join may not have " "an ON or USING clause", 0); @@ -104843,28 +118000,49 @@ static KeyInfo *keyInfoFromExprList( ); /* -** Insert code into "v" that will push the record in register regData -** into the sorter. +** Generate code that will push the record in registers regData +** through regData+nData-1 onto the sorter. */ static void pushOntoSorter( Parse *pParse, /* Parser context */ SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ - int regData /* Register holding data to be sorted */ + int regData, /* First register holding data to be sorted */ + int regOrigData, /* First register holding data before packing */ + int nData, /* Number of elements in the data array */ + int nPrefixReg /* No. of reg prior to regData available for use */ ){ - Vdbe *v = pParse->pVdbe; - int nExpr = pSort->pOrderBy->nExpr; - int regRecord = ++pParse->nMem; - int regBase = pParse->nMem+1; - int nOBSat = pSort->nOBSat; - int op; + Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); + int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ + int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ + int regBase; /* Regs for sorter record */ + int regRecord = ++pParse->nMem; /* Assembled sorter record */ + int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + int iLimit; /* LIMIT counter */ - pParse->nMem += nExpr+2; /* nExpr+2 registers allocated at regBase */ - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat,regRecord); + assert( bSeq==0 || bSeq==1 ); + assert( nData==1 || regData==regOrigData ); + if( nPrefixReg ){ + assert( nPrefixReg==nExpr+bSeq ); + regBase = regData - nExpr - bSeq; + }else{ + regBase = pParse->nMem + 1; + pParse->nMem += nBase; + } + assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); + iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; + pSort->labelDone = sqlite3VdbeMakeLabel(v); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, + SQLITE_ECEL_DUP|SQLITE_ECEL_REF); + if( bSeq ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); + } + if( nPrefixReg==0 ){ + sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -104875,24 +118053,35 @@ static void pushOntoSorter( regPrevKey = pParse->nMem+1; pParse->nMem += pSort->nOBSat; - nKey = nExpr - pSort->nOBSat + 1; - addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + nKey = nExpr - pSort->nOBSat + bSeq; + if( bSeq ){ + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + }else{ + addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); + } + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); if( pParse->db->mallocFailed ) return; - pOp->p2 = nKey + 1; + pOp->p2 = nKey + nData; pKI = pOp->p4.pKeyInfo; memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); - pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); + testcase( pKI->nXField>2 ); + pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, + pKI->nXField-1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); pSort->labelBkOut = sqlite3VdbeMakeLabel(v); pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); + if( iLimit ){ + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone); + VdbeCoverage(v); + } sqlite3VdbeJumpHere(v, addrFirst); - sqlite3VdbeAddOp3(v, OP_Move, regBase, regPrevKey, pSort->nOBSat); + sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); } if( pSort->sortFlags & SORTFLAG_UseSorter ){ @@ -104901,21 +118090,33 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( pSelect->iLimit ){ - int addr1, addr2; - int iLimit; - if( pSelect->iOffset ){ - iLimit = pSelect->iOffset+1; - }else{ - iLimit = pSelect->iLimit; - } - addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); - addr2 = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, addr1); + if( iLimit ){ + int addr; + int r1 = 0; + /* Fill the sorter until it contains LIMIT+OFFSET entries. (The iLimit + ** register is initialized with value of LIMIT+OFFSET.) After the sorter + ** fills up, delete the least entry in the sorter after each insert. + ** Thus we never hold more than the LIMIT+OFFSET rows in memory at once */ + addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); + if( pSort->bOrderedInnerLoop ){ + r1 = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_Column, pSort->iECursor, nExpr, r1); + VdbeComment((v, "seq")); + } sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); - sqlite3VdbeJumpHere(v, addr2); + if( pSort->bOrderedInnerLoop ){ + /* If the inner loop is driven by an index such that values from + ** the same iteration of the inner loop are in sorted order, then + ** immediately jump to the next iteration of an inner loop if the + ** entry from the current iteration does not fit into the top + ** LIMIT+OFFSET entries of the sorter. */ + int iBrk = sqlite3VdbeCurrentAddr(v) + 2; + sqlite3VdbeAddOp3(v, OP_Eq, regBase+nExpr, iBrk, r1); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } + sqlite3VdbeJumpHere(v, addr); } } @@ -104928,11 +118129,8 @@ static void codeOffset( int iContinue /* Jump here to skip the current record */ ){ if( iOffset>0 ){ - int addr; - addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); - VdbeComment((v, "skip OFFSET records")); - sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v); + VdbeComment((v, "OFFSET")); } } @@ -104963,30 +118161,6 @@ static void codeDistinct( sqlite3ReleaseTempReg(pParse, r1); } -#ifndef SQLITE_OMIT_SUBQUERY -/* -** Generate an error message when a SELECT is used within a subexpression -** (example: "a IN (SELECT * FROM table)") but it has more than 1 result -** column. We do this in a subroutine because the error used to occur -** in multiple places. (The error only occurs in one place now, but we -** retain the subroutine to minimize code disruption.) -*/ -static int checkForMultiColumnSelectError( - Parse *pParse, /* Parse context. */ - SelectDest *pDest, /* Destination of SELECT results */ - int nExpr /* Number of result columns returned by SELECT */ -){ - int eDest = pDest->eDest; - if( nExpr>1 && (eDest==SRT_Mem || eDest==SRT_Set) ){ - sqlite3ErrorMsg(pParse, "only a single result allowed for " - "a SELECT that is part of an expression"); - return 1; - }else{ - return 0; - } -} -#endif - /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -105014,6 +118188,7 @@ static void selectInnerLoop( int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ + int nPrefixReg = 0; /* Number of extra registers before regResult */ assert( v ); assert( pEList!=0 ); @@ -105029,6 +118204,11 @@ static void selectInnerLoop( nResultCol = pEList->nExpr; if( pDest->iSdst==0 ){ + if( pSort ){ + nPrefixReg = pSort->pOrderBy->nExpr; + if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++; + pParse->nMem += nPrefixReg; + } pDest->iSdst = pParse->nMem+1; pParse->nMem += nResultCol; }else if( pDest->iSdst+nResultCol > pParse->nMem ){ @@ -105050,8 +118230,13 @@ static void selectInnerLoop( /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ - sqlite3ExprCodeExprList(pParse, pEList, regResult, - (eDest==SRT_Output||eDest==SRT_Coroutine)?SQLITE_ECEL_DUP:0); + u8 ecelFlags; + if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){ + ecelFlags = SQLITE_ECEL_DUP; + }else{ + ecelFlags = 0; + } + sqlite3ExprCodeExprList(pParse, pEList, regResult, 0, ecelFlags); } /* If the DISTINCT keyword was present on the SELECT statement @@ -105106,7 +118291,8 @@ static void selectInnerLoop( default: { assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, + regResult); break; } } @@ -105145,10 +118331,12 @@ static void selectInnerLoop( case SRT_DistFifo: case SRT_Table: case SRT_EphemTab: { - int r1 = sqlite3GetTempReg(pParse); + int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + testcase( eDest==SRT_Fifo ); + testcase( eDest==SRT_DistFifo ); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -105157,13 +118345,14 @@ static void selectInnerLoop( ** current row to the index and proceed with writing it to the ** output table as well. */ int addr = sqlite3VdbeCurrentAddr(v) + 4; - sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); assert( pSort==0 ); } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -105171,7 +118360,7 @@ static void selectInnerLoop( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); } - sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1); break; } @@ -105181,19 +118370,19 @@ static void selectInnerLoop( ** item into the set table with bogus data. */ case SRT_Set: { - assert( nResultCol==1 ); - pDest->affSdst = - sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pSort ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter( + pParse, pSort, p, regResult, regResult, nResultCol, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); - sqlite3ExprCacheAffinityChange(pParse, regResult, 1); + assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, + r1, pDest->zAffSdst, nResultCol); + sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); } @@ -105209,15 +118398,16 @@ static void selectInnerLoop( } /* If this is a scalar select that is part of an expression, then - ** store the results in the appropriate memory cell and break out - ** of the scan loop. + ** store the results in the appropriate memory cell or array of + ** memory cells and break out of the scan loop. */ case SRT_Mem: { - assert( nResultCol==1 ); + assert( nResultCol==pDest->nSdst ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter( + pParse, pSort, p, regResult, regResult, nResultCol, nPrefixReg); }else{ - sqlite3ExprCodeMove(pParse, regResult, iParm, 1); + assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ } break; @@ -105229,10 +118419,8 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - pushOntoSorter(pParse, pSort, p, r1); - sqlite3ReleaseTempReg(pParse, r1); + pushOntoSorter(pParse, pSort, p, regResult, regResult, nResultCol, + nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -105309,7 +118497,7 @@ static void selectInnerLoop( ** the output for us. */ if( pSort==0 && p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v); } } @@ -105318,8 +118506,8 @@ static void selectInnerLoop( ** X extra columns. */ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ - KeyInfo *p = sqlite3DbMallocZero(0, - sizeof(KeyInfo) + (N+X)*(sizeof(CollSeq*)+1)); + int nExtra = (N+X)*(sizeof(CollSeq*)+1); + KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); if( p ){ p->aSortOrder = (u8*)&p->aColl[N+X]; p->nField = (u16)N; @@ -105327,8 +118515,9 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ p->enc = ENC(db); p->db = db; p->nRef = 1; + memset(&p[1], 0, nExtra); }else{ - db->mallocFailed = 1; + sqlite3OomFault(db); } return p; } @@ -105340,7 +118529,7 @@ SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){ if( p ){ assert( p->nRef>0 ); p->nRef--; - if( p->nRef==0 ) sqlite3DbFree(0, p); + if( p->nRef==0 ) sqlite3DbFree(p->db, p); } } @@ -105375,7 +118564,7 @@ SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** then the KeyInfo structure is appropriate for initializing a virtual ** index to implement a DISTINCT test. ** -** Space to hold the KeyInfo structure is obtain from malloc. The calling +** Space to hold the KeyInfo structure is obtained from malloc. The calling ** function is responsible for seeing that this structure is eventually ** freed. */ @@ -105392,7 +118581,7 @@ static KeyInfo *keyInfoFromExprList( int i; nExpr = pList->nExpr; - pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra-iStart, 1); + pInfo = sqlite3KeyInfoAlloc(db, nExpr-iStart, nExtra+1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=iStart, pItem=pList->a+iStart; ipVdbe; /* The prepared statement */ - int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ + int addrBreak = pSort->labelDone; /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; int addrOnce = 0; int iTab; - int pseudoTab = 0; ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; int iParm = pDest->iSDParm; int regRow; int regRowid; int nKey; + int iSortTab; /* Sorter cursor to read from */ + int nSortData; /* Trailing values to read from sorter */ + int i; + int bSeq; /* True if sorter record includes seq. no. */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + struct ExprList_item *aOutEx = p->pEList->a; +#endif + assert( addrBreak<0 ); if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); + sqlite3VdbeGoto(v, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); - addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); } iTab = pSort->iECursor; - regRow = sqlite3GetTempReg(pParse); - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - pseudoTab = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); + if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ regRowid = 0; + regRow = pDest->iSdst; + nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); + regRow = sqlite3GetTempRange(pParse, nColumn); + nSortData = nColumn; } nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; - int ptab2 = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2); + iSortTab = pParse->nTab++; + if( pSort->labelBkOut ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow); - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); + bSeq = 0; }else{ - if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); + iSortTab = iTab; + bSeq = 1; + } + for(i=0; iaffSdst, 1); - sqlite3ExprCacheAffinityChange(pParse, regRow, 1); + assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid, + pDest->zAffSdst, nColumn); + sqlite3ExprCacheAffinityChange(pParse, regRow, nColumn); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); break; } case SRT_Mem: { - assert( nColumn==1 ); - sqlite3ExprCodeMove(pParse, regRow, iParm, 1); /* The LIMIT clause will terminate the loop for us */ break; } #endif default: { - int i; assert( eDest==SRT_Output || eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); testcase( eDest==SRT_Coroutine ); - for(i=0; iiSdst+i ); - sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iSdst+i); - if( i==0 ){ - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); - } - } if( eDest==SRT_Output ){ sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); @@ -105600,9 +118787,14 @@ static void generateSortTail( break; } } - sqlite3ReleaseTempReg(pParse, regRow); - sqlite3ReleaseTempReg(pParse, regRowid); - + if( regRowid ){ + if( eDest==SRT_Set ){ + sqlite3ReleaseTempRange(pParse, regRow, nColumn); + }else{ + sqlite3ReleaseTempReg(pParse, regRow); + } + sqlite3ReleaseTempReg(pParse, regRowid); + } /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); @@ -105641,30 +118833,30 @@ static void generateSortTail( */ #ifdef SQLITE_ENABLE_COLUMN_METADATA # define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,C,D,E,F) +#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ +# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) +#endif static const char *columnTypeImpl( NameContext *pNC, Expr *pExpr, +#ifdef SQLITE_ENABLE_COLUMN_METADATA const char **pzOrigDb, const char **pzOrigTab, const char **pzOrigCol, +#endif u8 *pEstWidth ){ - char const *zOrigDb = 0; - char const *zOrigTab = 0; - char const *zOrigCol = 0; -#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ -# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) -static const char *columnTypeImpl( - NameContext *pNC, - Expr *pExpr, - u8 *pEstWidth -){ -#endif /* !defined(SQLITE_ENABLE_COLUMN_METADATA) */ char const *zType = 0; int j; u8 estWidth = 1; +#ifdef SQLITE_ENABLE_COLUMN_METADATA + char const *zOrigDb = 0; + char const *zOrigTab = 0; + char const *zOrigCol = 0; +#endif - if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; + assert( pExpr!=0 ); + assert( pNC->pSrcList!=0 ); switch( pExpr->op ){ case TK_AGG_COLUMN: case TK_COLUMN: { @@ -105719,6 +118911,9 @@ static const char *columnTypeImpl( /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. + ** + ** The ALWAYS() is because iCol>=pS->pEList->nExpr will have been + ** caught already by name resolution. */ NameContext sNC; Expr *p = pS->pEList->a[iCol].pExpr; @@ -105737,20 +118932,20 @@ static const char *columnTypeImpl( zType = "INTEGER"; zOrigCol = "rowid"; }else{ - zType = pTab->aCol[iCol].zType; zOrigCol = pTab->aCol[iCol].zName; + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); estWidth = pTab->aCol[iCol].szEst; } zOrigTab = pTab->zName; if( pNC->pParse ){ int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); - zOrigDb = pNC->pParse->db->aDb[iDb].zName; + zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName; } #else if( iCol<0 ){ zType = "INTEGER"; }else{ - zType = pTab->aCol[iCol].zType; + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); estWidth = pTab->aCol[iCol].szEst; } #endif @@ -105849,7 +119044,9 @@ static void generateColumnNames( } #endif - if( pParse->colNamesSet || NEVER(v==0) || db->mallocFailed ) return; + if( pParse->colNamesSet || db->mallocFailed ) return; + assert( v!=0 ); + assert( pTabList!=0 ); pParse->colNamesSet = 1; fullNames = (db->flags & SQLITE_FullColNames)!=0; shortNames = (db->flags & SQLITE_ShortColNames)!=0; @@ -105861,7 +119058,7 @@ static void generateColumnNames( if( pEList->a[i].zName ){ char *zName = pEList->a[i].zName; sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); - }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && pTabList ){ + }else if( p->op==TK_COLUMN || p->op==TK_AGG_COLUMN ){ Table *pTab; char *zCol; int iCol = p->iColumn; @@ -105897,7 +119094,7 @@ static void generateColumnNames( } /* -** Given a an expression list (which is really the list of expressions +** Given an expression list (which is really the list of expressions ** that form the result set of a SELECT statement) compute appropriate ** column names for a table that would hold the expression list. ** @@ -105909,7 +119106,7 @@ static void generateColumnNames( ** Return SQLITE_OK on success. If a memory allocation error occurs, ** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. */ -static int selectColumnsFromExprList( +SQLITE_PRIVATE int sqlite3ColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ i16 *pnCol, /* Write the number of columns here */ @@ -105917,13 +119114,15 @@ static int selectColumnsFromExprList( ){ sqlite3 *db = pParse->db; /* Database connection */ int i, j; /* Loop counters */ - int cnt; /* Index added to make the name unique */ + u32 cnt; /* Index added to make the name unique */ Column *aCol, *pCol; /* For looping over result columns */ int nCol; /* Number of columns in the result set */ Expr *p; /* Expression for a single result column */ char *zName; /* Column name */ int nName; /* Size of name in zName[] */ + Hash ht; /* Hash table of column names */ + sqlite3HashInit(&ht); if( pEList ){ nCol = pEList->nExpr; aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); @@ -105932,16 +119131,16 @@ static int selectColumnsFromExprList( nCol = 0; aCol = 0; } + assert( nCol==(i16)nCol ); *pnCol = nCol; *paCol = aCol; - for(i=0, pCol=aCol; imallocFailed; i++, pCol++){ /* Get an appropriate name for the column */ p = sqlite3ExprSkipCollate(pEList->a[i].pExpr); if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS " phrase, use as the name */ - zName = sqlite3DbStrDup(db, zName); }else{ Expr *pColExpr = p; /* The expression that is the result column name */ Table *pTab; /* Table associated with this expression */ @@ -105954,41 +119153,37 @@ static int selectColumnsFromExprList( int iCol = pColExpr->iColumn; pTab = pColExpr->pTab; if( iCol<0 ) iCol = pTab->iPKey; - zName = sqlite3MPrintf(db, "%s", - iCol>=0 ? pTab->aCol[iCol].zName : "rowid"); + zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; }else if( pColExpr->op==TK_ID ){ assert( !ExprHasProperty(pColExpr, EP_IntValue) ); - zName = sqlite3MPrintf(db, "%s", pColExpr->u.zToken); + zName = pColExpr->u.zToken; }else{ /* Use the original text of the column expression as its name */ - zName = sqlite3MPrintf(db, "%s", pEList->a[i].zSpan); + zName = pEList->a[i].zSpan; } } - if( db->mallocFailed ){ - sqlite3DbFree(db, zName); - break; - } + zName = sqlite3MPrintf(db, "%s", zName); /* Make sure the column name is unique. If the name is not unique, - ** append a integer to the name so that it becomes unique. + ** append an integer to the name so that it becomes unique. */ - nName = sqlite3Strlen30(zName); - for(j=cnt=0; j1 && sqlite3Isdigit(zName[k]); k--){} - if( k>=0 && zName[k]==':' ) nName = k; - zName[nName] = 0; - zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); - sqlite3DbFree(db, zName); - zName = zNewName; - j = -1; - if( zName==0 ) break; + cnt = 0; + while( zName && sqlite3HashFind(&ht, zName)!=0 ){ + nName = sqlite3Strlen30(zName); + if( nName>0 ){ + for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} + if( zName[j]==':' ) nName = j; } + zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); + if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); } pCol->zName = zName; + sqlite3ColumnPropertiesFromName(0, pCol); + if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ + sqlite3OomFault(db); + } } + sqlite3HashClear(&ht); if( db->mallocFailed ){ for(j=0; jpSrc; a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + const char *zType; + int n, m; p = a[i].pExpr; - pCol->zType = sqlite3DbStrDup(db, columnType(&sNC, p,0,0,0, &pCol->szEst)); + zType = columnType(&sNC, p, 0, 0, 0, &pCol->szEst); szAll += pCol->szEst; pCol->affinity = sqlite3ExprAffinity(p); - if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_NONE; + if( zType && (m = sqlite3Strlen30(zType))>0 ){ + n = sqlite3Strlen30(pCol->zName); + pCol->zName = sqlite3DbReallocOrFree(db, pCol->zName, n+m+2); + if( pCol->zName ){ + memcpy(&pCol->zName[n+1], zType, m+1); + pCol->colFlags |= COLFLAG_HASTYPE; + } + } + if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; pColl = sqlite3ExprCollSeq(pParse, p); - if( pColl ){ + if( pColl && pCol->zColl==0 ){ pCol->zColl = sqlite3DbStrDup(db, pColl->zName); } } @@ -106069,12 +119274,12 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ } /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside ** is disabled */ - assert( db->lookaside.bEnabled==0 ); + assert( db->lookaside.bDisable ); pTab->nRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - selectAddColumnTypeAndCollation(pParse, pTab, pSelect); + sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -106087,20 +119292,20 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ ** Get a VDBE for the given parser context. Create a new one if necessary. ** If an error occurs, return NULL and leave a message in pParse. */ -SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ - Vdbe *v = pParse->pVdbe; - if( v==0 ){ - v = pParse->pVdbe = sqlite3VdbeCreate(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Init); - if( pParse->pToplevel==0 - && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) - ){ - pParse->okConstFactor = 1; - } - +static SQLITE_NOINLINE Vdbe *allocVdbe(Parse *pParse){ + Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(pParse); + if( v ) sqlite3VdbeAddOp2(v, OP_Init, 0, 1); + if( pParse->pToplevel==0 + && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) + ){ + pParse->okConstFactor = 1; } return v; } +SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ + Vdbe *v = pParse->pVdbe; + return v ? v : allocVdbe(pParse); +} /* @@ -106130,7 +119335,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ Vdbe *v = 0; int iLimit = 0; int iOffset; - int addr1, n; + int n; if( p->iLimit ) return; /* @@ -106149,15 +119354,16 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); - }else if( n>=0 && p->nSelectRow>(u64)n ){ - p->nSelectRow = n; + sqlite3VdbeGoto(v, iBreak); + }else if( n>=0 && p->nSelectRow>sqlite3LogEst((u64)n) ){ + p->nSelectRow = sqlite3LogEst((u64)n); + p->selFlags |= SF_FixedLimit; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); VdbeComment((v, "LIMIT counter")); - sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v); } if( p->pOffset ){ p->iOffset = iOffset = ++pParse->nMem; @@ -106165,14 +119371,8 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3ExprCode(pParse, p->pOffset, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); - sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); + sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset); VdbeComment((v, "LIMIT+OFFSET")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); - sqlite3VdbeJumpHere(v, addr1); } } } @@ -106194,7 +119394,10 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ pRet = 0; } assert( iCol>=0 ); - if( pRet==0 && iColpEList->nExpr ){ + /* iCol must be less than p->pEList->nExpr. Otherwise an error would + ** have been thrown during name resolution and we would not have gotten + ** this far */ + if( pRet==0 && ALWAYS(iColpEList->nExpr) ){ pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr); } return pRet; @@ -106249,7 +119452,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ** ** ** There is exactly one reference to the recursive-table in the FROM clause -** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag. ** ** The setup-query runs once to generate an initial set of rows that go ** into a Queue table. Rows are extracted from the Queue table one by @@ -106314,7 +119517,7 @@ static void generateWithRecursiveQuery( /* Locate the cursor number of the Current table */ for(i=0; ALWAYS(inSrc); i++){ - if( pSrc->a[i].isRecursive ){ + if( pSrc->a[i].fg.isRecursive ){ iCurrent = pSrc->a[i].iCursor; break; } @@ -106376,7 +119579,7 @@ static void generateWithRecursiveQuery( selectInnerLoop(pParse, p, p->pEList, iCurrent, 0, 0, pDest, addrCont, addrBreak); if( regLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, regLimit, addrBreak); VdbeCoverage(v); } sqlite3VdbeResolveLabel(v, addrCont); @@ -106384,13 +119587,17 @@ static void generateWithRecursiveQuery( /* Execute the recursive SELECT taking the single row in Current as ** the value for the recursive-table. Store the results in the Queue. */ - p->pPrior = 0; - sqlite3Select(pParse, p, &destQueue); - assert( p->pPrior==0 ); - p->pPrior = pSetup; + if( p->selFlags & SF_Aggregate ){ + sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); + }else{ + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; + } /* Keep running the loop until the Queue is empty */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeGoto(v, addrTop); sqlite3VdbeResolveLabel(v, addrBreak); end_of_recursive_query: @@ -106409,6 +119616,48 @@ static int multiSelectOrderBy( SelectDest *pDest /* What to do with query results */ ); +/* +** Handle the special case of a compound-select that originates from a +** VALUES clause. By handling this as a special case, we avoid deep +** recursion, and thus do not need to enforce the SQLITE_LIMIT_COMPOUND_SELECT +** on a VALUES clause. +** +** Because the Select object originates from a VALUES clause: +** (1) It has no LIMIT or OFFSET +** (2) All terms are UNION ALL +** (3) There is no ORDER BY clause +*/ +static int multiSelectValues( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + Select *pPrior; + int nRow = 1; + int rc = 0; + assert( p->selFlags & SF_MultiValue ); + do{ + assert( p->selFlags & SF_Values ); + assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) ); + assert( p->pLimit==0 ); + assert( p->pOffset==0 ); + assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr ); + if( p->pPrior==0 ) break; + assert( p->pPrior->pNext==p ); + p = p->pPrior; + nRow++; + }while(1); + while( p ){ + pPrior = p->pPrior; + p->pPrior = 0; + rc = sqlite3Select(pParse, p, pDest); + p->pPrior = pPrior; + if( rc ) break; + p->nSelectRow = nRow; + p = p->pNext; + } + return rc; +} /* ** This routine is called to process a compound query form from @@ -106486,24 +119735,21 @@ static int multiSelect( if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr); - sqlite3VdbeChangeP5(v, BTREE_UNORDERED); dest.eDest = SRT_Table; } + /* Special handling for a compound-select that originates as a VALUES clause. + */ + if( p->selFlags & SF_MultiValue ){ + rc = multiSelectValues(pParse, p, &dest); + goto multi_select_end; + } + /* Make sure all SELECTs in the statement have the same number of elements ** in their result sets. */ assert( p->pEList && pPrior->pEList ); - if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ - if( p->selFlags & SF_Values ){ - sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); - }else{ - sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); - } - rc = 1; - goto multi_select_end; - } + assert( p->pEList->nExpr==pPrior->pEList->nExpr ); #ifndef SQLITE_OMIT_CTE if( p->selFlags & SF_Recursive ){ @@ -106539,20 +119785,24 @@ static int multiSelect( p->iLimit = pPrior->iLimit; p->iOffset = pPrior->iOffset; if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); VdbeCoverage(v); + addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); + } } explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; - p->nSelectRow += pPrior->nSelectRow; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( pPrior->pLimit && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && nLimit>0 && p->nSelectRow > (u64)nLimit + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ - p->nSelectRow = nLimit; + p->nSelectRow = sqlite3LogEst((u64)nLimit); } if( addr ){ sqlite3VdbeJumpHere(v, addr); @@ -106624,7 +119874,9 @@ static int multiSelect( pDelete = p->pPrior; p->pPrior = pPrior; p->pOrderBy = 0; - if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; + if( p->op==TK_UNION ){ + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->pOffset = pOffset; @@ -106641,7 +119893,7 @@ static int multiSelect( if( dest.eDest==SRT_Output ){ Select *pFirst = p; while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); + generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); } iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); @@ -106716,7 +119968,7 @@ static int multiSelect( if( dest.eDest==SRT_Output ){ Select *pFirst = p; while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); + generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); } iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); @@ -106759,7 +120011,7 @@ static int multiSelect( nCol = p->pEList->nExpr; pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); if( !pKeyInfo ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto multi_select_end; } for(i=0, apColl=pKeyInfo->aColl; iselFlags & SF_Values ){ + sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); + } +} + /* ** Code an output subroutine for a coroutine implementation of a ** SELECT statment. @@ -106835,12 +120100,12 @@ static int generateOutputSubroutine( /* Suppress duplicates for UNION, EXCEPT, and INTERSECT */ if( regPrev ){ - int j1, j2; - j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); - j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + int addr1, addr2; + addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } @@ -106850,15 +120115,14 @@ static int generateOutputSubroutine( */ codeOffset(v, p->iOffset, iContinue); + assert( pDest->eDest!=SRT_Exists ); + assert( pDest->eDest!=SRT_Table ); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ - case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); - testcase( pDest->eDest==SRT_Table ); - testcase( pDest->eDest==SRT_EphemTab ); sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); @@ -106869,39 +120133,26 @@ static int generateOutputSubroutine( } #ifndef SQLITE_OMIT_SUBQUERY - /* If we are creating a set for an "expr IN (SELECT ...)" construct, - ** then there should be a single item on the stack. Write this - ** item into the set table with bogus data. + /* If we are creating a set for an "expr IN (SELECT ...)". */ case SRT_Set: { int r1; - assert( pIn->nSdst==1 ); - pDest->affSdst = - sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); + testcase( pIn->nSdst>1 ); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, + r1, pDest->zAffSdst, pIn->nSdst); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; } -#if 0 /* Never occurs on an ORDER BY query */ - /* If any row exist in the result set, record that fact and abort. - */ - case SRT_Exists: { - sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); - /* The LIMIT clause will terminate the loop for us */ - break; - } -#endif - /* If this is a scalar select that is part of an expression, then ** store the results in the appropriate memory cell and break out ** of the scan loop. */ case SRT_Mem: { - assert( pIn->nSdst==1 ); + assert( pIn->nSdst==1 || pParse->nErr>0 ); testcase( pIn->nSdst!=1 ); sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, 1); /* The LIMIT clause will jump out of the loop for us */ break; @@ -106916,7 +120167,7 @@ static int generateOutputSubroutine( pDest->iSdst = sqlite3GetTempRange(pParse, pIn->nSdst); pDest->nSdst = pIn->nSdst; } - sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pDest->nSdst); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pIn->nSdst); sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); break; } @@ -106940,7 +120191,7 @@ static int generateOutputSubroutine( /* Jump to the end of the loop if the LIMIT is reached. */ if( p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v); } /* Generate the subroutine return @@ -107068,7 +120319,7 @@ static int multiSelectOrderBy( int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int j1; /* Jump instructions that get retargetted */ + int addr1; /* Jump instructions that get retargetted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ @@ -107112,7 +120363,7 @@ static int multiSelectOrderBy( } if( j==nOrderBy ){ Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); - if( pNew==0 ) return SQLITE_NOMEM; + if( pNew==0 ) return SQLITE_NOMEM_BKPT; pNew->flags |= EP_IntValue; pNew->u.iValue = i; pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); @@ -107128,12 +120379,13 @@ static int multiSelectOrderBy( ** to the right and the left are evaluated, they use the correct ** collation. */ - aPermute = sqlite3DbMallocRaw(db, sizeof(int)*nOrderBy); + aPermute = sqlite3DbMallocRawNN(db, sizeof(int)*(nOrderBy + 1)); if( aPermute ){ struct ExprList_item *pItem; - for(i=0, pItem=pOrderBy->a; iu.x.iOrderByCol>0 - && pItem->u.x.iOrderByCol<=p->pEList->nExpr ); + aPermute[0] = nOrderBy; + for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ + assert( pItem->u.x.iOrderByCol>0 ); + assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; } pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); @@ -107204,19 +120456,19 @@ static int multiSelectOrderBy( ** left of the compound operator - the "A" select. */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeEndCoroutine(v, regAddrA); + sqlite3VdbeJumpHere(v, addr1); /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); VdbeComment((v, "right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; @@ -107226,7 +120478,7 @@ static int multiSelectOrderBy( sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; p->iOffset = savedOffset; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrB); + sqlite3VdbeEndCoroutine(v, regAddrB); /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. @@ -107257,8 +120509,8 @@ static int multiSelectOrderBy( addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); - p->nSelectRow += pPrior->nSelectRow; + sqlite3VdbeGoto(v, addrEofA); + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } /* Generate a subroutine to run when the results from select B @@ -107271,7 +120523,7 @@ static int multiSelectOrderBy( VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); + sqlite3VdbeGoto(v, addrEofB); } /* Generate code to handle the case of AB @@ -107303,11 +120555,11 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); } sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); + sqlite3VdbeGoto(v, labelCmpr); /* This code runs once to initialize everything. */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); @@ -107329,7 +120581,7 @@ static int multiSelectOrderBy( if( pDest->eDest==SRT_Output ){ Select *pFirst = pPrior; while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); + generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); } /* Reassembly the compound query so that it will be freed correctly @@ -107343,14 +120595,14 @@ static int multiSelectOrderBy( /*** TBD: Insert subroutine calls to close cursors on incomplete **** subqueries ****/ explainComposite(pParse, p->op, iSub1, iSub2, 0); - return SQLITE_OK; + return pParse->nErr!=0; } #endif #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Forward Declarations */ static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList *); +static void substSelect(sqlite3*, Select *, int, ExprList*, int); /* ** Scan through the expression pExpr. Replace every reference to @@ -107387,7 +120639,7 @@ static Expr *substExpr( pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList); + substSelect(db, pExpr->x.pSelect, iTable, pEList, 1); }else{ substExprList(db, pExpr->x.pList, iTable, pEList); } @@ -107410,25 +120662,28 @@ static void substSelect( sqlite3 *db, /* Report malloc errors here */ Select *p, /* SELECT statement in which to make substitutions */ int iTable, /* Table to be replaced */ - ExprList *pEList /* Substitute values */ + ExprList *pEList, /* Substitute values */ + int doPrior /* Do substitutes on p->pPrior too */ ){ SrcList *pSrc; struct SrcList_item *pItem; int i; if( !p ) return; - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); - substSelect(db, p->pPrior, iTable, pEList); - pSrc = p->pSrc; - assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ - if( ALWAYS(pSrc) ){ + do{ + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(db, p->pHaving, iTable, pEList); + p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + pSrc = p->pSrc; + assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList); + substSelect(db, pItem->pSelect, iTable, pEList, 1); + if( pItem->fg.isTabFunc ){ + substExprList(db, pItem->u1.pFuncArg, iTable, pEList); + } } - } + }while( doPrior && (p = p->pPrior)!=0 ); } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -107454,7 +120709,7 @@ static void substSelect( ** ** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5 ** -** The code generated for this simpification gives the same result +** The code generated for this simplification gives the same result ** but only has to scan the data once. And because indices might ** exist on the table t1, a complete scan of the data might be ** avoided. @@ -107463,7 +120718,10 @@ static void substSelect( ** ** (1) The subquery and the outer query do not both use aggregates. ** -** (2) The subquery is not an aggregate or the outer query is not a join. +** (2) The subquery is not an aggregate or (2a) the outer query is not a join +** and (2b) the outer query does not use subqueries other than the one +** FROM-clause subquery that is a candidate for flattening. (2b is +** due to ticket [2f7170d73bf9abf80] from 2015-02-09.) ** ** (3) The subquery is not the right operand of a left outer join ** (Originally ticket #306. Strengthened by ticket #3300) @@ -107487,8 +120745,10 @@ static void substSelect( ** (9) The subquery does not use LIMIT or the outer query does not use ** aggregates. ** -** (10) The subquery does not use aggregates or the outer query does not -** use LIMIT. +** (**) Restriction (10) was removed from the code on 2005-02-05 but we +** accidently carried the comment forward until 2014-09-15. Original +** text: "The subquery does not use aggregates or the outer query +** does not use LIMIT." ** ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** @@ -107551,6 +120811,11 @@ static void substSelect( ** parent to a compound query confuses the code that handles ** recursive queries in multiSelect(). ** +** (24) The subquery is not an aggregate that uses the built-in min() or +** or max() functions. (Without this restriction, a query like: +** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily +** return the value X for which Y was maximal.) +** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query @@ -107570,7 +120835,7 @@ static int flattenSubquery( int subqueryIsAgg /* True if the subquery uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; - Select *pParent; + Select *pParent; /* Current UNION ALL term of the other query */ Select *pSub; /* The inner query or "subquery" */ Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ @@ -107593,12 +120858,21 @@ static int flattenSubquery( iParent = pSubitem->iCursor; pSub = pSubitem->pSelect; assert( pSub!=0 ); - if( isAgg && subqueryIsAgg ) return 0; /* Restriction (1) */ - if( subqueryIsAgg && pSrc->nSrc>1 ) return 0; /* Restriction (2) */ + if( subqueryIsAgg ){ + if( isAgg ) return 0; /* Restriction (1) */ + if( pSrc->nSrc>1 ) return 0; /* Restriction (2a) */ + if( (p->pWhere && ExprHasProperty(p->pWhere,EP_Subquery)) + || (sqlite3ExprListFlags(p->pEList) & EP_Subquery)!=0 + || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0 + ){ + return 0; /* Restriction (2b) */ + } + } + pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, - ** not arbitrary expresssions, we allowed some combining of LIMIT and OFFSET + ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET ** because they could be computed at compile-time. But when LIMIT and OFFSET ** became arbitrary expressions, we were forced to add restrictions (13) ** and (14). */ @@ -107623,8 +120897,14 @@ static int flattenSubquery( if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } - if( pSub->selFlags & SF_Recursive ) return 0; /* Restriction (22) */ - if( (p->selFlags & SF_Recursive) && pSub->pPrior ) return 0; /* (23) */ + testcase( pSub->selFlags & SF_Recursive ); + testcase( pSub->selFlags & SF_MinMaxAgg ); + if( pSub->selFlags & (SF_Recursive|SF_MinMaxAgg) ){ + return 0; /* Restrictions (22) and (24) */ + } + if( (p->selFlags & SF_Recursive) && pSub->pPrior ){ + return 0; /* Restriction (23) */ + } /* OBSOLETE COMMENT 1: ** Restriction 3: If the subquery is a join, make sure the subquery is @@ -107658,7 +120938,7 @@ static int flattenSubquery( ** is fraught with danger. Best to avoid the whole thing. If the ** subquery is the right term of a LEFT JOIN, then do not flatten. */ - if( (pSubitem->jointype & JT_OUTER)!=0 ){ + if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ return 0; } @@ -107678,10 +120958,10 @@ static int flattenSubquery( testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); assert( pSub->pSrc!=0 ); + assert( pSub->pEList->nExpr==pSub1->pEList->nExpr ); if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 || (pSub1->pPrior && pSub1->op!=TK_ALL) || pSub1->pSrc->nSrc<1 - || pSub->pEList->nExpr!=pSub1->pEList->nExpr ){ return 0; } @@ -107698,6 +120978,8 @@ static int flattenSubquery( } /***** If we reach this point, flattening is permitted. *****/ + SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n", + pSub->zSelName, pSub, iFrom)); /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; @@ -107750,6 +121032,7 @@ static int flattenSubquery( p->pLimit = 0; p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); + sqlite3SelectSetName(pNew, pSub->zSelName); p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; @@ -107762,6 +121045,9 @@ static int flattenSubquery( if( pPrior ) pPrior->pNext = pNew; pNew->pNext = p; p->pPrior = pNew; + SELECTTRACE(2,pParse,p, + ("compound-subquery flattener creates %s.%p as peer\n", + pNew->zSelName, pNew)); } if( db->mallocFailed ) return 1; } @@ -107823,7 +121109,7 @@ static int flattenSubquery( if( pSrc ){ assert( pParent==p ); /* First time through the loop */ - jointype = pSubitem->jointype; + jointype = pSubitem->fg.jointype; }else{ assert( pParent!=p ); /* 2nd and subsequent times through the loop */ pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); @@ -107844,9 +121130,9 @@ static int flattenSubquery( ** ** The outer query has 3 slots in its FROM clause. One slot of the ** outer query (the middle slot) is used by the subquery. The next - ** block of code will expand the out query to 4 slots. The middle - ** slot is expanded to two slots in order to make space for the - ** two elements in the FROM clause of the subquery. + ** block of code will expand the outer query FROM clause to 4 slots. + ** The middle slot is expanded to two slots in order to make space + ** for the two elements in the FROM clause of the subquery. */ if( nSubSrc>1 ){ pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); @@ -107860,10 +121146,11 @@ static int flattenSubquery( */ for(i=0; ia[i+iFrom].pUsing); + assert( pSrc->a[i+iFrom].fg.isTabFunc==0 ); pSrc->a[i+iFrom] = pSubSrc->a[i]; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].jointype = jointype; + pSrc->a[iFrom].fg.jointype = jointype; /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. @@ -107885,36 +121172,40 @@ static int flattenSubquery( pList->a[i].zName = zName; } } - substExprList(db, pParent->pEList, iParent, pSub->pEList); - if( isAgg ){ - substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); - } if( pSub->pOrderBy ){ + /* At this point, any non-zero iOrderByCol values indicate that the + ** ORDER BY column expression is identical to the iOrderByCol'th + ** expression returned by SELECT statement pSub. Since these values + ** do not necessarily correspond to columns in SELECT statement pParent, + ** zero them before transfering the ORDER BY clause. + ** + ** Not doing this may cause an error if a subsequent call to this + ** function attempts to flatten a compound sub-query into pParent + ** (the only way this can happen is if the compound sub-query is + ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ + ExprList *pOrderBy = pSub->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } assert( pParent->pOrderBy==0 ); - pParent->pOrderBy = pSub->pOrderBy; + assert( pSub->pPrior==0 ); + pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; - }else if( pParent->pOrderBy ){ - substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); - } - if( pSub->pWhere ){ - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); - }else{ - pWhere = 0; } + pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); if( subqueryIsAgg ){ assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); - pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, - sqlite3ExprDup(db, pSub->pHaving, 0)); + pParent->pHaving = sqlite3ExprAnd(db, + sqlite3ExprDup(db, pSub->pHaving, 0), pParent->pHaving + ); assert( pParent->pGroupBy==0 ); pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ - pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); - pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); + pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); } + substSelect(db, pParent, iParent, pSub->pEList, 0); /* The flattened query is distinct if either the inner or the ** outer query is distinct. @@ -107938,10 +121229,94 @@ static int flattenSubquery( */ sqlite3SelectDelete(db, pSub1); +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After flattening:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + return 1; } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** Make copies of relevant WHERE clause terms of the outer query into +** the WHERE clause of subquery. Example: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1) WHERE x=5 AND y=10; +** +** Transformed into: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1 WHERE a=5 AND c-d=10) +** WHERE x=5 AND y=10; +** +** The hope is that the terms added to the inner query will make it more +** efficient. +** +** Do not attempt this optimization if: +** +** (1) The inner query is an aggregate. (In that case, we'd really want +** to copy the outer WHERE-clause terms onto the HAVING clause of the +** inner query. But they probably won't help there so do not bother.) +** +** (2) The inner query is the recursive part of a common table expression. +** +** (3) The inner query has a LIMIT clause (since the changes to the WHERE +** close would change the meaning of the LIMIT). +** +** (4) The inner query is the right operand of a LEFT JOIN. (The caller +** enforces this restriction since this routine does not have enough +** information to know.) +** +** (5) The WHERE clause expression originates in the ON or USING clause +** of a LEFT JOIN. +** +** Return 0 if no changes are made and non-zero if one or more WHERE clause +** terms are duplicated into the subquery. +*/ +static int pushDownWhereTerms( + sqlite3 *db, /* The database connection (for malloc()) */ + Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ + Expr *pWhere, /* The WHERE clause of the outer query */ + int iCursor /* Cursor number of the subquery */ +){ + Expr *pNew; + int nChng = 0; + Select *pX; /* For looping over compound SELECTs in pSubq */ + if( pWhere==0 ) return 0; + for(pX=pSubq; pX; pX=pX->pPrior){ + if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ + testcase( pX->selFlags & SF_Aggregate ); + testcase( pX->selFlags & SF_Recursive ); + testcase( pX!=pSubq ); + return 0; /* restrictions (1) and (2) */ + } + } + if( pSubq->pLimit!=0 ){ + return 0; /* restriction (3) */ + } + while( pWhere->op==TK_AND ){ + nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); + pWhere = pWhere->pLeft; + } + if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ + if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ + nChng++; + while( pSubq ){ + pNew = sqlite3ExprDup(db, pWhere, 0); + pNew = substExpr(db, pNew, iCursor, pSubq->pEList); + pSubq->pWhere = sqlite3ExprAnd(db, pSubq->pWhere, pNew); + pSubq = pSubq->pPrior; + } + } + return nChng; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + /* ** Based on the contents of the AggInfo structure indicated by the first ** argument, this function checks if the following are true: @@ -107984,7 +121359,7 @@ static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ /* ** The select statement passed as the first argument is an aggregate query. -** The second argment is the associated aggregate-info object. This +** The second argument is the associated aggregate-info object. This ** function tests if the SELECT is of the form: ** ** SELECT count(*) FROM @@ -108025,20 +121400,20 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ ** pFrom->pIndex and return SQLITE_OK. */ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ - if( pFrom->pTab && pFrom->zIndex ){ + if( pFrom->pTab && pFrom->fg.isIndexedBy ){ Table *pTab = pFrom->pTab; - char *zIndex = pFrom->zIndex; + char *zIndexedBy = pFrom->u1.zIndexedBy; Index *pIdx; for(pIdx=pTab->pIndex; - pIdx && sqlite3StrICmp(pIdx->zName, zIndex); + pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); pIdx=pIdx->pNext ); if( !pIdx ){ - sqlite3ErrorMsg(pParse, "no such index: %s", zIndex, 0); + sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0); pParse->checkSchema = 1; return SQLITE_ERROR; } - pFrom->pIndex = pIdx; + pFrom->pIBIndex = pIdx; } return SQLITE_OK; } @@ -108094,7 +121469,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ if( pNewSrc==0 ) return WRC_Abort; *pNew = *p; p->pSrc = pNewSrc; - p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); p->op = TK_SELECT; p->pWhere = 0; pNew->pGroupBy = 0; @@ -108102,7 +121477,10 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ pNew->pOrderBy = 0; p->pPrior = 0; p->pNext = 0; + p->pWith = 0; p->selFlags &= ~SF_Compound; + assert( (p->selFlags & SF_Converted)==0 ); + p->selFlags |= SF_Converted; assert( pNew->pPrior!=0 ); pNew->pPrior->pNext = pNew; pNew->pLimit = 0; @@ -108110,6 +121488,19 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ return WRC_Continue; } +/* +** Check to see if the FROM clause term pFrom has table-valued function +** arguments. If it does, leave an error message in pParse and return +** non-zero, since pFrom is not allowed to be a table-valued function. +*/ +static int cannotBeFunction(Parse *pParse, struct SrcList_item *pFrom){ + if( pFrom->fg.isTabFunc ){ + sqlite3ErrorMsg(pParse, "'%s' is not a function", pFrom->zName); + return 1; + } + return 0; +} + #ifndef SQLITE_OMIT_CTE /* ** Argument pWith (which may be NULL) points to a linked list of nested @@ -108122,7 +121513,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ ** object that the returned CTE belongs to. */ static struct Cte *searchWith( - With *pWith, /* Current outermost WITH clause */ + With *pWith, /* Current innermost WITH clause */ struct SrcList_item *pItem, /* FROM clause element to resolve */ With **ppContext /* OUT: WITH clause return value belongs to */ ){ @@ -108153,11 +121544,12 @@ static struct Cte *searchWith( ** statement with which it is associated. */ SQLITE_PRIVATE void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ - assert( bFree==0 || pParse->pWith==0 ); + assert( bFree==0 || (pParse->pWith==0 && pParse->pWithToFree==0) ); if( pWith ){ + assert( pParse->pWith!=pWith ); pWith->pOuter = pParse->pWith; pParse->pWith = pWith; - pParse->bFreeWith = bFree; + if( bFree ) pParse->pWithToFree = pWith; } } @@ -108196,14 +121588,15 @@ static int withExpand( int bMayRecursive; /* True if compound joined by UNION [ALL] */ With *pSavedWith; /* Initial value of pParse->pWith */ - /* If pCte->zErr is non-NULL at this point, then this is an illegal + /* If pCte->zCteErr is non-NULL at this point, then this is an illegal ** recursive reference to CTE pCte. Leave an error in pParse and return - ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** early. If pCte->zCteErr is NULL, then this is not a recursive reference. ** In this case, proceed. */ - if( pCte->zErr ){ - sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + if( pCte->zCteErr ){ + sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName); return SQLITE_ERROR; } + if( cannotBeFunction(pParse, pFrom) ) return SQLITE_ERROR; assert( pFrom->pTab==0 ); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); @@ -108212,9 +121605,9 @@ static int withExpand( pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - pTab->tabFlags |= TF_Ephemeral; + pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); - if( db->mallocFailed ) return SQLITE_NOMEM; + if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; assert( pFrom->pSelect ); /* Check if this is a recursive CTE. */ @@ -108230,7 +121623,7 @@ static int withExpand( && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ pItem->pTab = pTab; - pItem->isRecursive = 1; + pItem->fg.isRecursive = 1; pTab->nRef++; pSel->selFlags |= SF_Recursive; } @@ -108246,15 +121639,16 @@ static int withExpand( } assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); - pCte->zErr = "circular reference: %s"; + pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + pParse->pWith = pWith; for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); pEList = pLeft->pEList; if( pCte->pCols ){ - if( pEList->nExpr!=pCte->pCols->nExpr ){ + if( pEList && pEList->nExpr!=pCte->pCols->nExpr ){ sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", pCte->zName, pEList->nExpr, pCte->pCols->nExpr ); @@ -108264,16 +121658,16 @@ static int withExpand( pEList = pCte->pCols; } - selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); if( bMayRecursive ){ if( pSel->selFlags & SF_Recursive ){ - pCte->zErr = "multiple recursive references: %s"; + pCte->zCteErr = "multiple recursive references: %s"; }else{ - pCte->zErr = "recursive reference in a subquery: %s"; + pCte->zCteErr = "recursive reference in a subquery: %s"; } sqlite3WalkSelect(pWalker, pSel); } - pCte->zErr = 0; + pCte->zCteErr = 0; pParse->pWith = pSavedWith; } @@ -108314,10 +121708,10 @@ static void selectPopWith(Walker *pWalker, Select *p){ ** fill pTabList->a[].pSelect with a copy of the SELECT statement ** that implements the view. A copy is made of the view's SELECT ** statement so that we can freely modify or delete that statement -** without worrying about messing up the presistent representation +** without worrying about messing up the persistent representation ** of the view. ** -** (3) Add terms to the WHERE clause to accomodate the NATURAL keyword +** (3) Add terms to the WHERE clause to accommodate the NATURAL keyword ** on joins and the ON and USING clause of joins. ** ** (4) Scan the list of columns in the result set (pEList) looking @@ -108345,7 +121739,9 @@ static int selectExpander(Walker *pWalker, Select *p){ } pTabList = p->pSrc; pEList = p->pEList; - sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); + if( pWalker->xSelectCallback2==selectPopWith ){ + sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); + } /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. @@ -108358,17 +121754,9 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->isRecursive==0 || pFrom->pTab ); - if( pFrom->isRecursive ) continue; - if( pFrom->pTab!=0 ){ - /* This statement has already been prepared. There is no need - ** to go further. */ - assert( i==0 ); -#ifndef SQLITE_OMIT_CTE - selectPopWith(pWalker, p); -#endif - return WRC_Prune; - } + assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); + if( pFrom->fg.isRecursive ) continue; + assert( pFrom->pTab==0 ); #ifndef SQLITE_OMIT_CTE if( withExpand(pWalker, pFrom) ) return WRC_Abort; if( pFrom->pTab ) {} else @@ -108379,13 +121767,13 @@ static int selectExpander(Walker *pWalker, Select *p){ /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); assert( pFrom->pTab==0 ); - sqlite3WalkSelect(pWalker, pSel); + if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } - selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; @@ -108402,13 +121790,20 @@ static int selectExpander(Walker *pWalker, Select *p){ return WRC_Abort; } pTab->nRef++; + if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){ + return WRC_Abort; + } #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) - if( pTab->pSelect || IsVirtual(pTab) ){ - /* We reach here if the named table is a really a view */ + if( IsVirtual(pTab) || pTab->pSelect ){ + i16 nCol; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); + sqlite3SelectSetName(pFrom->pSelect, pTab->zName); + nCol = pTab->nCol; + pTab->nCol = -1; sqlite3WalkSelect(pWalker, pFrom->pSelect); + pTab->nCol = nCol; } #endif } @@ -108428,19 +121823,20 @@ static int selectExpander(Walker *pWalker, Select *p){ /* For every "*" that occurs in the column list, insert the names of ** all columns in all tables. And for every TABLE.* insert the names ** of all columns in TABLE. The parser inserted a special expression - ** with the TK_ALL operator for each "*" that it found in the column list. - ** The following code just has to locate the TK_ALL expressions and expand - ** each one to the list of all columns in all tables. + ** with the TK_ASTERISK operator for each "*" that it found in the column + ** list. The following code just has to locate the TK_ASTERISK + ** expressions and expand each one to the list of all columns in + ** all tables. ** ** The first loop just checks to see if there are any "*" operators ** that need expanding. */ for(k=0; knExpr; k++){ pE = pEList->a[k].pExpr; - if( pE->op==TK_ALL ) break; + if( pE->op==TK_ASTERISK ) break; assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); - if( pE->op==TK_DOT && pE->pRight->op==TK_ALL ) break; + if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break; } if( knExpr ){ /* @@ -108454,18 +121850,13 @@ static int selectExpander(Walker *pWalker, Select *p){ int longNames = (flags & SQLITE_FullColNames)!=0 && (flags & SQLITE_ShortColNames)==0; - /* When processing FROM-clause subqueries, it is always the case - ** that full_column_names=OFF and short_column_names=ON. The - ** sqlite3ResultSetOfSelect() routine makes it so. */ - assert( (p->selFlags & SF_NestedFrom)==0 - || ((flags & SQLITE_FullColNames)==0 && - (flags & SQLITE_ShortColNames)!=0) ); - for(k=0; knExpr; k++){ pE = a[k].pExpr; pRight = pE->pRight; assert( pE->op!=TK_DOT || pRight!=0 ); - if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){ + if( pE->op!=TK_ASTERISK + && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK) + ){ /* This particular expression does not need to be expanded. */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); @@ -108502,7 +121893,7 @@ static int selectExpander(Walker *pWalker, Select *p){ continue; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; + zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*"; } for(j=0; jnCol; j++){ char *zName = pTab->aCol[j].zName; @@ -108517,18 +121908,19 @@ static int selectExpander(Walker *pWalker, Select *p){ continue; } - /* If a column is marked as 'hidden' (currently only possible - ** for virtual tables), do not include it in the expanded - ** result-set list. + /* If a column is marked as 'hidden', omit it from the expanded + ** result-set list unless the SELECT has the SF_IncludeHidden + ** bit set. */ - if( IsHiddenColumn(&pTab->aCol[j]) ){ - assert(IsVirtual(pTab)); + if( (p->selFlags & SF_IncludeHidden)==0 + && IsHiddenColumn(&pTab->aCol[j]) + ){ continue; } tableSeen = 1; if( i>0 && zTName==0 ){ - if( (pFrom->jointype & JT_NATURAL)!=0 + if( (pFrom->fg.jointype & JT_NATURAL)!=0 && tableAndColumnIndex(pTabList, i, zName, 0, 0) ){ /* In a NATURAL join, omit the join columns from the @@ -108560,8 +121952,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pExpr = pRight; } pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); - sColname.z = zColname; - sColname.n = sqlite3Strlen30(zColname); + sqlite3TokenInit(&sColname, zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; @@ -108593,6 +121984,7 @@ static int selectExpander(Walker *pWalker, Select *p){ #if SQLITE_MAX_COLUMN if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many columns in result set"); + return WRC_Abort; } #endif return WRC_Continue; @@ -108607,7 +121999,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** Walker.xSelectCallback is set to do something useful for every ** subquery in the parser tree. */ -static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ +SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ UNUSED_PARAMETER2(NotUsed, NotUsed2); return WRC_Continue; } @@ -108628,14 +122020,16 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; memset(&w, 0, sizeof(w)); - w.xExprCallback = exprWalkNoop; + w.xExprCallback = sqlite3ExprWalkNoop; w.pParse = pParse; if( pParse->hasCompound ){ w.xSelectCallback = convertCompoundSelectToSubquery; sqlite3WalkSelect(&w, pSelect); } w.xSelectCallback = selectExpander; - w.xSelectCallback2 = selectPopWith; + if( (pSelect->selFlags & SF_MultiValue)==0 ){ + w.xSelectCallback2 = selectPopWith; + } sqlite3WalkSelect(&w, pSelect); } @@ -108661,19 +122055,19 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ struct SrcList_item *pFrom; assert( p->selFlags & SF_Resolved ); - if( (p->selFlags & SF_HasTypeInfo)==0 ){ - p->selFlags |= SF_HasTypeInfo; - pParse = pWalker->pParse; - pTabList = p->pSrc; - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ - /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); - } + assert( (p->selFlags & SF_HasTypeInfo)==0 ); + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + assert( pTab!=0 ); + if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel); } } } @@ -108693,7 +122087,7 @@ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ Walker w; memset(&w, 0, sizeof(w)); w.xSelectCallback2 = selectAddSubqueryTypeInfo; - w.xExprCallback = exprWalkNoop; + w.xExprCallback = sqlite3ExprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); #endif @@ -108812,14 +122206,15 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, SQLITE_ECEL_DUP); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ nArg = 0; regAgg = 0; } if( pF->iDistinct>=0 ){ addrNext = sqlite3VdbeMakeLabel(v); - assert( nArg==1 ); + testcase( nArg==0 ); /* Error condition */ + testcase( nArg>1 ); /* Also an error */ codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); } if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ @@ -108836,7 +122231,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, + sqlite3VdbeAddOp4(v, OP_AggStep0, 0, regAgg, pF->iMem, (void*)pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); @@ -108919,7 +122314,7 @@ SQLITE_PRIVATE int sqlite3Select( WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ Vdbe *v; /* The virtual machine under construction */ int isAgg; /* True for select lists like "count(*)" */ - ExprList *pEList; /* List of columns to extract. */ + ExprList *pEList = 0; /* List of columns to extract. */ SrcList *pTabList; /* List of tables to select from */ Expr *pWhere; /* The WHERE clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ @@ -108942,6 +122337,13 @@ SQLITE_PRIVATE int sqlite3Select( } if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); +#if SELECTTRACE_ENABLED + pParse->nSelectIndent++; + SELECTTRACE(1,pParse,p, ("begin processing:\n")); + if( sqlite3SelectTrace & 0x100 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); @@ -108962,36 +122364,80 @@ SQLITE_PRIVATE int sqlite3Select( memset(&sSort, 0, sizeof(sSort)); sSort.pOrderBy = p->pOrderBy; pTabList = p->pSrc; - pEList = p->pEList; if( pParse->nErr || db->mallocFailed ){ goto select_end; } + assert( p->pEList!=0 ); isAgg = (p->selFlags & SF_Aggregate)!=0; - assert( pEList!=0 ); +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p, ("after name resolution:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif - /* Begin generating code. + /* Try to flatten subqueries in the FROM clause up into the main query */ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + for(i=0; !p->pPrior && inSrc; i++){ + struct SrcList_item *pItem = &pTabList->a[i]; + Select *pSub = pItem->pSelect; + int isAggSub; + Table *pTab = pItem->pTab; + if( pSub==0 ) continue; + + /* Catch mismatch in the declared columns of a view and the number of + ** columns in the SELECT on the RHS */ + if( pTab->nCol!=pSub->pEList->nExpr ){ + sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", + pTab->nCol, pTab->zName, pSub->pEList->nExpr); + goto select_end; + } + + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; + if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + /* This subquery can be absorbed into its parent. */ + if( isAggSub ){ + isAgg = 1; + p->selFlags |= SF_Aggregate; + } + i = -1; + } + pTabList = p->pSrc; + if( db->mallocFailed ) goto select_end; + if( !IgnorableOrderby(pDest) ){ + sSort.pOrderBy = p->pOrderBy; + } + } +#endif + + /* Get a pointer the VDBE under construction, allocating a new VDBE if one + ** does not already exist */ v = sqlite3GetVdbe(pParse); if( v==0 ) goto select_end; - /* If writing to memory or generating a set - ** only a single column may be output. +#ifndef SQLITE_OMIT_COMPOUND_SELECT + /* Handle compound SELECT statements using the separate multiSelect() + ** procedure. */ -#ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ - goto select_end; + if( p->pPrior ){ + rc = multiSelect(pParse, p, pDest); + explainSetInteger(pParse->iSelectId, iRestoreSelectId); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end compound-select processing\n")); + pParse->nSelectIndent--; +#endif + return rc; } #endif /* Generate code for all sub-queries in the FROM clause */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) - for(i=0; !p->pPrior && inSrc; i++){ + for(i=0; inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; SelectDest dest; Select *pSub = pItem->pSelect; - int isAggSub; - if( pSub==0 ) continue; /* Sometimes the code for a subquery will be generated more than @@ -109001,7 +122447,7 @@ SQLITE_PRIVATE int sqlite3Select( ** is sufficient, though the subroutine to manifest the view does need ** to be invoked again. */ if( pItem->addrFillSub ){ - if( pItem->viaCoroutine==0 ){ + if( pItem->fg.viaCoroutine==0 ){ sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); } continue; @@ -109016,16 +122462,39 @@ SQLITE_PRIVATE int sqlite3Select( */ pParse->nHeight += sqlite3SelectExprHeight(p); - isAggSub = (pSub->selFlags & SF_Aggregate)!=0; - if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ - /* This subquery can be absorbed into its parent. */ - if( isAggSub ){ - isAgg = 1; - p->selFlags |= SF_Aggregate; + /* Make copies of constant WHERE-clause terms in the outer query down + ** inside the subquery. This can help the subquery to run more efficiently. + */ + if( (pItem->fg.jointype & JT_OUTER)==0 + && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n")); + sqlite3TreeViewSelect(0, p, 0); } - i = -1; - }else if( pTabList->nSrc==1 - && OptimizationEnabled(db, SQLITE_SubqCoroutine) +#endif + } + + /* Generate code to implement the subquery + ** + ** The subquery is implemented as a co-routine if all of these are true: + ** (1) The subquery is guaranteed to be the outer loop (so that it + ** does not need to be computed more than once) + ** (2) The ALL keyword after SELECT is omitted. (Applications are + ** allowed to say "SELECT ALL" instead of just "SELECT" to disable + ** the use of co-routines.) + ** (3) Co-routines are not disabled using sqlite3_test_control() + ** with SQLITE_TESTCTRL_OPTIMIZATIONS. + ** + ** TODO: Are there other reasons beside (1) to use a co-routine + ** implementation? + */ + if( i==0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ + && (p->selFlags & SF_All)==0 /* (2) */ + && OptimizationEnabled(db, SQLITE_SubqCoroutine) /* (3) */ ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. @@ -109038,10 +122507,10 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); - pItem->viaCoroutine = 1; + pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->fg.viaCoroutine = 1; pItem->regResult = dest.iSdst; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); + sqlite3VdbeEndCoroutine(v, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else{ @@ -109057,11 +122526,11 @@ SQLITE_PRIVATE int sqlite3Select( pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; - if( pItem->isCorrelated==0 ){ + if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ - onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); }else{ VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); @@ -109069,36 +122538,30 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); sqlite3VdbeChangeP1(v, topAddr, retAddr); sqlite3ClearTempRegCache(pParse); } - if( /*pParse->nErr ||*/ db->mallocFailed ){ - goto select_end; - } + if( db->mallocFailed ) goto select_end; pParse->nHeight -= sqlite3SelectExprHeight(p); - pTabList = p->pSrc; - if( !IgnorableOrderby(pDest) ){ - sSort.pOrderBy = p->pOrderBy; - } } - pEList = p->pEList; #endif + + /* Various elements of the SELECT copied into local variables for + ** convenience */ + pEList = p->pEList; pWhere = p->pWhere; pGroupBy = p->pGroupBy; pHaving = p->pHaving; sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; -#ifndef SQLITE_OMIT_COMPOUND_SELECT - /* If there is are a sequence of queries, do the earlier ones first. - */ - if( p->pPrior ){ - rc = multiSelect(pParse, p, pDest); - explainSetInteger(pParse->iSelectId, iRestoreSelectId); - return rc; +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); + sqlite3TreeViewSelect(0, p, 0); } #endif @@ -109110,7 +122573,7 @@ SQLITE_PRIVATE int sqlite3Select( ** ** is transformed to: ** - ** SELECT xyz FROM ... GROUP BY xyz + ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz ** ** The second form is preferred as a single index (or temp-table) may be ** used for both the ORDER BY and DISTINCT processing. As originally @@ -109118,33 +122581,40 @@ SQLITE_PRIVATE int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 + && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 ){ p->selFlags &= ~SF_Distinct; - p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); - pGroupBy = p->pGroupBy; - sSort.pOrderBy = 0; + pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif } - /* If there is an ORDER BY clause, then this sorting - ** index might end up being unused if the data can be - ** extracted in pre-sorted order. If that is the case, then the - ** OP_OpenEphemeral instruction will be changed to an OP_Noop once - ** we figure out that the sorting index is not needed. The addrSortIndex - ** variable is used to facilitate that change. + /* If there is an ORDER BY clause, then create an ephemeral index to + ** do the sorting. But this sorting ephemeral index might end up + ** being unused if the data can be extracted in pre-sorted order. + ** If that is the case, then the OP_OpenEphemeral instruction will be + ** changed to an OP_Noop once we figure out that the sorting index is + ** not needed. The sSort.addrSortIndex variable is used to facilitate + ** that change. */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, 0); + pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, pEList->nExpr); sSort.iECursor = pParse->nTab++; sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sSort.iECursor, sSort.pOrderBy->nExpr+2, 0, - (char*)pKeyInfo, P4_KEYINFO); + sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0, + (char*)pKeyInfo, P4_KEYINFO + ); }else{ sSort.addrSortIndex = -1; } @@ -109158,21 +122628,21 @@ SQLITE_PRIVATE int sqlite3Select( /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = LARGEST_INT64; + p->nSelectRow = 320; /* 4 billion rows */ computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ - sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; + sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); sSort.sortFlags |= SORTFLAG_UseSorter; } - /* Open a virtual index to use for the distinct set. + /* Open an ephemeral index to use for the distinct set. */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList,0,0), - P4_KEYINFO); + sDistinct.tabTnct, 0, 0, + (char*)keyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ @@ -109182,10 +122652,12 @@ SQLITE_PRIVATE int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + assert( WHERE_USE_LIMIT==SF_FixedLimit ); + wctrlFlags |= p->selFlags & SF_FixedLimit; /* Begin the database scan. */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, - p->pEList, wctrlFlags, 0); + p->pEList, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -109195,6 +122667,7 @@ SQLITE_PRIVATE int sqlite3Select( } if( sSort.pOrderBy ){ sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } @@ -109245,16 +122718,17 @@ SQLITE_PRIVATE int sqlite3Select( for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ pItem->u.x.iAlias = 0; } - if( p->nSelectRow>100 ) p->nSelectRow = 100; + assert( 66==sqlite3LogEst(100) ); + if( p->nSelectRow>66 ) p->nSelectRow = 66; }else{ - p->nSelectRow = 1; + assert( 0==sqlite3LogEst(1) ); + p->nSelectRow = 0; } - /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then it may be possible to disable the ORDER BY clause ** on the grounds that the GROUP BY will cause elements to come out - ** in the correct order. It also may not - the GROUP BY may use a + ** in the correct order. It also may not - the GROUP BY might use a ** database index that causes rows to be grouped together as required ** but not actually sorted. Either way, record the fact that the ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp @@ -109275,7 +122749,7 @@ SQLITE_PRIVATE int sqlite3Select( sNC.pSrcList = pTabList; sNC.pAggInfo = &sAggInfo; sAggInfo.mnReg = pParse->nMem+1; - sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; + sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; sAggInfo.pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); @@ -109297,7 +122771,7 @@ SQLITE_PRIVATE int sqlite3Select( */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int j1; /* A-vs-B comparision jump */ + int addr1; /* A-vs-B comparision jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ @@ -109312,7 +122786,7 @@ SQLITE_PRIVATE int sqlite3Select( ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, 0); + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, sAggInfo.nColumn); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -109368,8 +122842,8 @@ SQLITE_PRIVATE int sqlite3Select( groupBySort = 1; nGroupBy = pGroupBy->nExpr; - nCol = nGroupBy + 1; - j = nGroupBy+1; + nCol = nGroupBy; + j = nGroupBy; for(i=0; i=j ){ nCol++; @@ -109378,20 +122852,14 @@ SQLITE_PRIVATE int sqlite3Select( } regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); - j = nGroupBy+1; + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); + j = nGroupBy; for(i=0; iiSorterColumn>=j ){ int r1 = j + regBase; - int r2; - - r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); - if( r1!=r2 ){ - sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); - } + sqlite3ExprCodeGetColumnToReg(pParse, + pCol->pTab, pCol->iColumn, pCol->iTable, r1); j++; } } @@ -109433,12 +122901,12 @@ SQLITE_PRIVATE int sqlite3Select( addrTopOfLoop = sqlite3VdbeCurrentAddr(v); sqlite3ExprCacheClear(pParse); if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut); + sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, + sortOut, sortPTab); } for(j=0; jnExpr; j++){ if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); - if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ sAggInfo.directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); @@ -109446,8 +122914,8 @@ SQLITE_PRIVATE int sqlite3Select( } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v); /* Generate code that runs whenever the GROUP BY changes. ** Changes in the GROUP BY are detected by the previous code @@ -109469,7 +122937,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Update the aggregate accumulators based on the content of ** the current row */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); updateAccumulator(pParse, &sAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -109491,7 +122959,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Jump over the subroutines */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEnd); + sqlite3VdbeGoto(v, addrEnd); /* Generate a subroutine that outputs a single row of the result ** set. This subroutine first looks at the iUseFlag. If iUseFlag @@ -109506,7 +122974,8 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); sqlite3VdbeResolveLabel(v, addrOutputRow); addrOutputRow = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); + VdbeCoverage(v); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); @@ -109625,7 +123094,8 @@ SQLITE_PRIVATE int sqlite3Select( if( flag ){ pMinMax = sqlite3ExprListDup(db, pMinMax, 0); pDel = pMinMax; - if( pMinMax && !db->mallocFailed ){ + assert( db->mallocFailed || pMinMax!=0 ); + if( !db->mallocFailed ){ pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; pMinMax->a[0].pExpr->op = TK_COLUMN; } @@ -109644,7 +123114,7 @@ SQLITE_PRIVATE int sqlite3Select( updateAccumulator(pParse, &sAggInfo); assert( pMinMax==0 || pMinMax->nExpr==1 ); if( sqlite3WhereIsOrdered(pWInfo)>0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); + sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); } @@ -109670,7 +123140,8 @@ SQLITE_PRIVATE int sqlite3Select( ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + explainTempTable(pParse, + sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } @@ -109678,10 +123149,9 @@ SQLITE_PRIVATE int sqlite3Select( */ sqlite3VdbeResolveLabel(v, iEnd); - /* The SELECT was successfully coded. Set the return code to 0 - ** to indicate no errors. - */ - rc = 0; + /* The SELECT has been coded. If there is an error in the Parse structure, + ** set the return code to 1. Otherwise 0. */ + rc = (pParse->nErr>0); /* Control jumps to here if an error is encountered above, or upon ** successful coding of the SELECT. @@ -109697,104 +123167,13 @@ select_end: sqlite3DbFree(db, sAggInfo.aCol); sqlite3DbFree(db, sAggInfo.aFunc); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end processing\n")); + pParse->nSelectIndent--; +#endif return rc; } -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) -/* -** Generate a human-readable description of a the Select object. -*/ -static void explainOneSelect(Vdbe *pVdbe, Select *p){ - sqlite3ExplainPrintf(pVdbe, "SELECT "); - if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ - if( p->selFlags & SF_Distinct ){ - sqlite3ExplainPrintf(pVdbe, "DISTINCT "); - } - if( p->selFlags & SF_Aggregate ){ - sqlite3ExplainPrintf(pVdbe, "agg_flag "); - } - sqlite3ExplainNL(pVdbe); - sqlite3ExplainPrintf(pVdbe, " "); - } - sqlite3ExplainExprList(pVdbe, p->pEList); - sqlite3ExplainNL(pVdbe); - if( p->pSrc && p->pSrc->nSrc ){ - int i; - sqlite3ExplainPrintf(pVdbe, "FROM "); - sqlite3ExplainPush(pVdbe); - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - sqlite3ExplainPrintf(pVdbe, "{%d,*} = ", pItem->iCursor); - if( pItem->pSelect ){ - sqlite3ExplainSelect(pVdbe, pItem->pSelect); - if( pItem->pTab ){ - sqlite3ExplainPrintf(pVdbe, " (tabname=%s)", pItem->pTab->zName); - } - }else if( pItem->zName ){ - sqlite3ExplainPrintf(pVdbe, "%s", pItem->zName); - } - if( pItem->zAlias ){ - sqlite3ExplainPrintf(pVdbe, " (AS %s)", pItem->zAlias); - } - if( pItem->jointype & JT_LEFT ){ - sqlite3ExplainPrintf(pVdbe, " LEFT-JOIN"); - } - sqlite3ExplainNL(pVdbe); - } - sqlite3ExplainPop(pVdbe); - } - if( p->pWhere ){ - sqlite3ExplainPrintf(pVdbe, "WHERE "); - sqlite3ExplainExpr(pVdbe, p->pWhere); - sqlite3ExplainNL(pVdbe); - } - if( p->pGroupBy ){ - sqlite3ExplainPrintf(pVdbe, "GROUPBY "); - sqlite3ExplainExprList(pVdbe, p->pGroupBy); - sqlite3ExplainNL(pVdbe); - } - if( p->pHaving ){ - sqlite3ExplainPrintf(pVdbe, "HAVING "); - sqlite3ExplainExpr(pVdbe, p->pHaving); - sqlite3ExplainNL(pVdbe); - } - if( p->pOrderBy ){ - sqlite3ExplainPrintf(pVdbe, "ORDERBY "); - sqlite3ExplainExprList(pVdbe, p->pOrderBy); - sqlite3ExplainNL(pVdbe); - } - if( p->pLimit ){ - sqlite3ExplainPrintf(pVdbe, "LIMIT "); - sqlite3ExplainExpr(pVdbe, p->pLimit); - sqlite3ExplainNL(pVdbe); - } - if( p->pOffset ){ - sqlite3ExplainPrintf(pVdbe, "OFFSET "); - sqlite3ExplainExpr(pVdbe, p->pOffset); - sqlite3ExplainNL(pVdbe); - } -} -SQLITE_PRIVATE void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ - if( p==0 ){ - sqlite3ExplainPrintf(pVdbe, "(null-select)"); - return; - } - sqlite3ExplainPush(pVdbe); - while( p ){ - explainOneSelect(pVdbe, p); - p = p->pNext; - if( p==0 ) break; - sqlite3ExplainNL(pVdbe); - sqlite3ExplainPrintf(pVdbe, "%s\n", selectOpName(p->op)); - } - sqlite3ExplainPrintf(pVdbe, "END"); - sqlite3ExplainPop(pVdbe); -} - -/* End of the structure debug printing code -*****************************************************************************/ -#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ - /************** End of select.c **********************************************/ /************** Begin file table.c *******************************************/ /* @@ -109815,6 +123194,7 @@ SQLITE_PRIVATE void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ ** These routines are in a separate files so that they will not be linked ** if they are not used. */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -109827,10 +123207,10 @@ SQLITE_PRIVATE void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ typedef struct TabResult { char **azResult; /* Accumulated output */ char *zErrMsg; /* Error message text, if an error occurs */ - int nAlloc; /* Slots allocated for azResult[] */ - int nRow; /* Number of rows in the result */ - int nColumn; /* Number of columns in the result */ - int nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ + u32 nAlloc; /* Slots allocated for azResult[] */ + u32 nRow; /* Number of rows in the result */ + u32 nColumn; /* Number of columns in the result */ + u32 nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ int rc; /* Return code from sqlite3_exec() */ } TabResult; @@ -109856,7 +123236,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( p->nData + need > p->nAlloc ){ char **azNew; p->nAlloc = p->nAlloc*2 + need; - azNew = sqlite3_realloc( p->azResult, sizeof(char*)*p->nAlloc ); + azNew = sqlite3_realloc64( p->azResult, sizeof(char*)*p->nAlloc ); if( azNew==0 ) goto malloc_failed; p->azResult = azNew; } @@ -109871,7 +123251,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( z==0 ) goto malloc_failed; p->azResult[p->nData++] = z; } - }else if( p->nColumn!=nCol ){ + }else if( (int)p->nColumn!=nCol ){ sqlite3_free(p->zErrMsg); p->zErrMsg = sqlite3_mprintf( "sqlite3_get_table() called with two or more incompatible queries" @@ -109888,7 +123268,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ z = 0; }else{ int n = sqlite3Strlen30(argv[i])+1; - z = sqlite3_malloc( n ); + z = sqlite3_malloc64( n ); if( z==0 ) goto malloc_failed; memcpy(z, argv[i], n); } @@ -109899,7 +123279,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ return 0; malloc_failed: - p->rc = SQLITE_NOMEM; + p->rc = SQLITE_NOMEM_BKPT; return 1; } @@ -109924,6 +123304,9 @@ SQLITE_API int sqlite3_get_table( int rc; TabResult res; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pazResult==0 ) return SQLITE_MISUSE_BKPT; +#endif *pazResult = 0; if( pnColumn ) *pnColumn = 0; if( pnRow ) *pnRow = 0; @@ -109934,10 +123317,10 @@ SQLITE_API int sqlite3_get_table( res.nData = 1; res.nAlloc = 20; res.rc = SQLITE_OK; - res.azResult = sqlite3_malloc(sizeof(char*)*res.nAlloc ); + res.azResult = sqlite3_malloc64(sizeof(char*)*res.nAlloc ); if( res.azResult==0 ){ db->errCode = SQLITE_NOMEM; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } res.azResult[0] = 0; rc = sqlite3_exec(db, zSql, sqlite3_get_table_cb, &res, pzErrMsg); @@ -109962,11 +123345,11 @@ SQLITE_API int sqlite3_get_table( } if( res.nAlloc>res.nData ){ char **azNew; - azNew = sqlite3_realloc( res.azResult, sizeof(char*)*res.nData ); + azNew = sqlite3_realloc64( res.azResult, sizeof(char*)*res.nData ); if( azNew==0 ){ sqlite3_free_table(&res.azResult[1]); db->errCode = SQLITE_NOMEM; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } res.azResult = azNew; } @@ -109980,7 +123363,7 @@ SQLITE_API int sqlite3_get_table( ** This routine frees the space the sqlite3_get_table() malloced. */ SQLITE_API void sqlite3_free_table( - char **azResult /* Result returned from from sqlite3_get_table() */ + char **azResult /* Result returned from sqlite3_get_table() */ ){ if( azResult ){ int i, n; @@ -110008,6 +123391,7 @@ SQLITE_API void sqlite3_free_table( ************************************************************************* ** This file contains the implementation for TRIGGERs */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_TRIGGER /* @@ -110093,7 +123477,6 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( int iDb; /* The database to store the trigger in */ Token *pName; /* The unqualified db name */ DbFixer sFix; /* State vector for the DB fixer */ - int iTabDb; /* Index of the database holding pTab */ assert( pName1!=0 ); /* pName1->z might be NULL, but not pName1 itself */ assert( pName2!=0 ); @@ -110124,7 +123507,7 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( ** ^^^^^^^^ ** ** To maintain backwards compatibility, ignore the database - ** name on pTableName if we are reparsing our of SQLITE_MASTER. + ** name on pTableName if we are reparsing out of SQLITE_MASTER. */ if( db->init.busy && iDb!=1 ){ sqlite3DbFree(db, pTableName->a[0].zDatabase); @@ -110177,8 +123560,7 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( goto trigger_cleanup; } assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash), - zName, sqlite3Strlen30(zName)) ){ + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ if( !noErr ){ sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); }else{ @@ -110191,7 +123573,6 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( /* Do not create a trigger on a system table */ if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); - pParse->nErr++; goto trigger_cleanup; } @@ -110208,13 +123589,13 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( " trigger on table: %S", pTableName, 0); goto trigger_cleanup; } - iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); #ifndef SQLITE_OMIT_AUTHORIZATION { + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); int code = SQLITE_CREATE_TRIGGER; - const char *zDb = db->aDb[iTabDb].zName; - const char *zDbTrig = isTemp ? db->aDb[1].zName : zDb; + const char *zDb = db->aDb[iTabDb].zDbSName; + const char *zDbTrig = isTemp ? db->aDb[1].zDbSName : zDb; if( iTabDb==1 || isTemp ) code = SQLITE_CREATE_TEMP_TRIGGER; if( sqlite3AuthCheck(pParse, code, zName, pTab->zName, zDbTrig) ){ goto trigger_cleanup; @@ -110286,8 +123667,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( pStepList->pTrig = pTrig; pStepList = pStepList->pNext; } - nameToken.z = pTrig->zName; - nameToken.n = sqlite3Strlen30(nameToken.z); + sqlite3TokenInit(&nameToken, pTrig->zName); sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken); if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) || sqlite3FixExpr(&sFix, pTrig->pWhen) @@ -110309,7 +123689,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n); sqlite3NestedParse(pParse, "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName, + db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), zName, pTrig->table, z); sqlite3DbFree(db, z); sqlite3ChangeCookie(pParse, iDb); @@ -110321,13 +123701,12 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( Trigger *pLink = pTrig; Hash *pHash = &db->aDb[iDb].pSchema->trigHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - pTrig = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), pTrig); + pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ - db->mallocFailed = 1; + sqlite3OomFault(db); }else if( pLink->pSchema==pLink->pTabSchema ){ Table *pTab; - int n = sqlite3Strlen30(pLink->table); - pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table, n); + pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table); assert( pTab!=0 ); pLink->pNext = pTab->pTrigger; pTab->pTrigger = pLink; @@ -110372,12 +123751,12 @@ static TriggerStep *triggerStepAllocate( ){ TriggerStep *pTriggerStep; - pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n); + pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); if( pTriggerStep ){ char *z = (char*)&pTriggerStep[1]; memcpy(z, pName->z, pName->n); - pTriggerStep->target.z = z; - pTriggerStep->target.n = pName->n; + sqlite3Dequote(z); + pTriggerStep->zTarget = z; pTriggerStep->op = op; } return pTriggerStep; @@ -110486,7 +123865,6 @@ SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr) int i; const char *zDb; const char *zName; - int nName; sqlite3 *db = pParse->db; if( db->mallocFailed ) goto drop_trigger_cleanup; @@ -110497,13 +123875,12 @@ SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr) assert( pName->nSrc==1 ); zDb = pName->a[0].zDatabase; zName = pName->a[0].zName; - nName = sqlite3Strlen30(zName); assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue; + if( zDb && sqlite3StrICmp(db->aDb[j].zDbSName, zDb) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName, nName); + pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName); if( pTrigger ) break; } if( !pTrigger ){ @@ -110526,8 +123903,7 @@ drop_trigger_cleanup: ** is set on. */ static Table *tableOfTrigger(Trigger *pTrigger){ - int n = sqlite3Strlen30(pTrigger->table); - return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table, n); + return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table); } @@ -110548,7 +123924,7 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ #ifndef SQLITE_OMIT_AUTHORIZATION { int code = SQLITE_DROP_TRIGGER; - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zTab = SCHEMA_TABLE(iDb); if( iDb==1 ) code = SQLITE_DROP_TEMP_TRIGGER; if( sqlite3AuthCheck(pParse, code, pTrigger->zName, pTable->zName, zDb) || @@ -110562,31 +123938,12 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ */ assert( pTable!=0 ); if( (v = sqlite3GetVdbe(pParse))!=0 ){ - int base; - static const int iLn = VDBE_OFFSET_LINENO(2); - static const VdbeOpList dropTrigger[] = { - { OP_Rewind, 0, ADDR(9), 0}, - { OP_String8, 0, 1, 0}, /* 1 */ - { OP_Column, 0, 1, 2}, - { OP_Ne, 2, ADDR(8), 1}, - { OP_String8, 0, 1, 0}, /* 4: "trigger" */ - { OP_Column, 0, 0, 2}, - { OP_Ne, 2, ADDR(8), 1}, - { OP_Delete, 0, 0, 0}, - { OP_Next, 0, ADDR(1), 0}, /* 8 */ - }; - - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3OpenMasterTable(pParse, iDb); - base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger, iLn); - sqlite3VdbeChangeP4(v, base+1, pTrigger->zName, P4_TRANSIENT); - sqlite3VdbeChangeP4(v, base+4, "trigger", P4_STATIC); + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE name=%Q AND type='trigger'", + db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), pTrigger->zName + ); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp2(v, OP_Close, 0, 0); sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0); - if( pParse->nMem<3 ){ - pParse->nMem = 3; - } } } @@ -110599,7 +123956,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const ch assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pHash = &(db->aDb[iDb].pSchema->trigHash); - pTrigger = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), 0); + pTrigger = sqlite3HashInsert(pHash, zName, 0); if( ALWAYS(pTrigger) ){ if( pTrigger->pSchema==pTrigger->pTabSchema ){ Table *pTab = tableOfTrigger(pTrigger); @@ -110663,7 +124020,7 @@ SQLITE_PRIVATE Trigger *sqlite3TriggersExist( } /* -** Convert the pStep->target token into a SrcList and return a pointer +** Convert the pStep->zTarget string into a SrcList and return a pointer ** to that SrcList. ** ** This routine adds a specific database name, if needed, to the target when @@ -110676,18 +124033,20 @@ static SrcList *targetSrcList( Parse *pParse, /* The parsing context */ TriggerStep *pStep /* The trigger containing the target token */ ){ + sqlite3 *db = pParse->db; int iDb; /* Index of the database to use */ SrcList *pSrc; /* SrcList to be returned */ - pSrc = sqlite3SrcListAppend(pParse->db, 0, &pStep->target, 0); + pSrc = sqlite3SrcListAppend(db, 0, 0, 0); if( pSrc ){ assert( pSrc->nSrc>0 ); - assert( pSrc->a!=0 ); - iDb = sqlite3SchemaToIndex(pParse->db, pStep->pTrig->pSchema); + pSrc->a[pSrc->nSrc-1].zName = sqlite3DbStrDup(db, pStep->zTarget); + iDb = sqlite3SchemaToIndex(db, pStep->pTrig->pSchema); if( iDb==0 || iDb>=2 ){ - sqlite3 *db = pParse->db; - assert( iDbdb->nDb ); - pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + const char *zDb; + assert( iDbnDb ); + zDb = db->aDb[iDb].zDbSName; + pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, zDb); } } return pSrc; @@ -110798,6 +124157,7 @@ static void transferParseError(Parse *pTo, Parse *pFrom){ if( pTo->nErr==0 ){ pTo->zErrMsg = pFrom->zErrMsg; pTo->nErr = pFrom->nErr; + pTo->rc = pFrom->rc; }else{ sqlite3DbFree(pFrom->db, pFrom->zErrMsg); } @@ -110900,7 +124260,6 @@ static TriggerPrg *codeRowTrigger( } pProgram->nMem = pSubParse->nMem; pProgram->nCsr = pSubParse->nTab; - pProgram->nOnce = pSubParse->nOnce; pProgram->token = (void *)pTrigger; pPrg->aColmask[0] = pSubParse->oldmask; pPrg->aColmask[1] = pSubParse->newmask; @@ -110973,8 +124332,8 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( if( pPrg ){ int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); - sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); - sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); + sqlite3VdbeAddOp4(v, OP_Program, reg, ignoreJump, ++pParse->nMem, + (const char *)pPrg->pProgram, P4_SUBPROGRAM); VdbeComment( (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); @@ -111136,6 +124495,7 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask( ** This file contains C code routines that are called by the parser ** to handle UPDATE statements. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_VIRTUALTABLE /* Forward declaration */ @@ -111257,9 +124617,9 @@ SQLITE_PRIVATE void sqlite3Update( /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ - int regOldRowid; /* The old rowid */ - int regNewRowid; /* The new rowid */ - int regNew; /* Content of the NEW.* table in triggers */ + int regOldRowid = 0; /* The old rowid */ + int regNewRowid = 0; /* The new rowid */ + int regNew = 0; /* Content of the NEW.* table in triggers */ int regOld = 0; /* Content of OLD.* table in triggers */ int regRowSet = 0; /* Rowset of rows to be updated */ int regKey = 0; /* composite PRIMARY KEY value */ @@ -111320,7 +124680,7 @@ SQLITE_PRIVATE void sqlite3Update( /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. ** Initialize aXRef[] and aToOpen[] to their default values. */ - aXRef = sqlite3DbMallocRaw(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); + aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); if( aXRef==0 ) goto update_cleanup; aRegIdx = aXRef+pTab->nCol; aToOpen = (u8*)(aRegIdx+nIdx); @@ -111372,7 +124732,7 @@ SQLITE_PRIVATE void sqlite3Update( int rc; rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, j<0 ? "ROWID" : pTab->aCol[j].zName, - db->aDb[iDb].zName); + db->aDb[iDb].zDbSName); if( rc==SQLITE_DENY ){ goto update_cleanup; }else if( rc==SQLITE_IGNORE ){ @@ -111386,16 +124746,20 @@ SQLITE_PRIVATE void sqlite3Update( assert( chngPk==0 || chngPk==1 ); chngKey = chngRowid + chngPk; - /* The SET expressions are not actually used inside the WHERE loop. - ** So reset the colUsed mask + /* The SET expressions are not actually used inside the WHERE loop. + ** So reset the colUsed mask. Unless this is a virtual table. In that + ** case, set all bits of the colUsed mask (to ensure that the virtual + ** table implementation makes all columns available). */ - pTabList->a[0].colUsed = 0; + pTabList->a[0].colUsed = IsVirtual(pTab) ? ALLBITS : 0; hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngKey); /* There is one entry in the aRegIdx[] array for each index on the table ** being updated. Fill in aRegIdx[] with a register number that will hold - ** the key for accessing each index. + ** the key for accessing each index. + ** + ** FIXME: Be smarter about omitting indexes that use expressions. */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; @@ -111404,7 +124768,8 @@ SQLITE_PRIVATE void sqlite3Update( }else{ reg = 0; for(i=0; inKeyCol; i++){ - if( aXRef[pIdx->aiColumn[i]]>=0 ){ + i16 iIdxCol = pIdx->aiColumn[i]; + if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ reg = ++pParse->nMem; break; } @@ -111420,29 +124785,20 @@ SQLITE_PRIVATE void sqlite3Update( if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); sqlite3BeginWriteOperation(pParse, 1, iDb); -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Virtual tables must be handled separately */ - if( IsVirtual(pTab) ){ - updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere, onError); - pWhere = 0; - pTabList = 0; - goto update_cleanup; - } -#endif - /* Allocate required registers. */ - regRowSet = ++pParse->nMem; - regOldRowid = regNewRowid = ++pParse->nMem; - if( chngPk || pTrigger || hasFK ){ - regOld = pParse->nMem + 1; + if( !IsVirtual(pTab) ){ + regRowSet = ++pParse->nMem; + regOldRowid = regNewRowid = ++pParse->nMem; + if( chngPk || pTrigger || hasFK ){ + regOld = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } + if( chngKey || pTrigger || hasFK ){ + regNewRowid = ++pParse->nMem; + } + regNew = pParse->nMem + 1; pParse->nMem += pTab->nCol; } - if( chngKey || pTrigger || hasFK ){ - regNewRowid = ++pParse->nMem; - } - regNew = pParse->nMem + 1; - pParse->nMem += pTab->nCol; /* Start the view context. */ if( isView ){ @@ -111450,7 +124806,7 @@ SQLITE_PRIVATE void sqlite3Update( } /* If we are trying to update a view, realize that view into - ** a ephemeral table. + ** an ephemeral table. */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ @@ -111465,12 +124821,22 @@ SQLITE_PRIVATE void sqlite3Update( goto update_cleanup; } +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Virtual tables must be handled separately */ + if( IsVirtual(pTab) ){ + updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, + pWhere, onError); + goto update_cleanup; + } +#endif + /* Begin the database scan */ if( HasRowid(pTab) ){ sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); pWInfo = sqlite3WhereBegin( - pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur + pParse, pTabList, pWhere, 0, 0, + WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE, iIdxCur ); if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); @@ -111504,6 +124870,7 @@ SQLITE_PRIVATE void sqlite3Update( if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); for(i=0; iaiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], iPk+i); } @@ -111513,7 +124880,7 @@ SQLITE_PRIVATE void sqlite3Update( regKey = iPk; }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, - sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3IndexAffinityStr(db, pPk), nPk); sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); } sqlite3WhereEnd(pWInfo); @@ -111548,14 +124915,14 @@ SQLITE_PRIVATE void sqlite3Update( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; } - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iBaseCur, aToOpen, + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen, 0, 0); } /* Top of the update loop */ if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] ){ - assert( pPk!=0 ); + if( aToOpen[iDataCur-iBaseCur] && !isView ){ + assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); VdbeCoverageNeverTaken(v); } @@ -111611,7 +124978,7 @@ SQLITE_PRIVATE void sqlite3Update( } /* Populate the array of registers beginning at regNew with the new - ** row data. This array is used to check constaints, create the new + ** row data. This array is used to check constants, create the new ** table and index records, and as the values for any new.* references ** made by triggers. ** @@ -111626,7 +124993,6 @@ SQLITE_PRIVATE void sqlite3Update( newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); - /*sqlite3VdbeAddOp3(v, OP_Null, 0, regNew, regNew+pTab->nCol-1);*/ for(i=0; inCol; i++){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); @@ -111642,7 +125008,7 @@ SQLITE_PRIVATE void sqlite3Update( */ testcase( i==31 ); testcase( i==32 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + sqlite3ExprCodeGetColumnToReg(pParse, pTab, i, iDataCur, regNew+i); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); } @@ -111684,13 +125050,14 @@ SQLITE_PRIVATE void sqlite3Update( } if( !isView ){ - int j1 = 0; /* Address of jump instruction */ + int addr1 = 0; /* Address of jump instruction */ int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Do constraint checks. */ assert( regOldRowid>0 ); sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, - regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace); + regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace, + aXRef); /* Do FK constraint checks. */ if( hasFK ){ @@ -111700,20 +125067,39 @@ SQLITE_PRIVATE void sqlite3Update( /* Delete the index entries associated with the current record. */ if( bReplace || chngKey ){ if( pPk ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + addr1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); }else{ - j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + addr1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); } VdbeCoverageNeverTaken(v); } - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx); - - /* If changing the record number, delete the old record. */ + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); + + /* If changing the rowid value, or if there are foreign key constraints + ** to process, delete the old record. Otherwise, add a noop OP_Delete + ** to invoke the pre-update hook. + ** + ** That (regNew==regnewRowid+1) is true is also important for the + ** pre-update hook. If the caller invokes preupdate_new(), the returned + ** value is copied from memory cell (regNewRowid+1+iCol), where iCol + ** is the column index supplied by the user. + */ + assert( regNew==regNewRowid+1 ); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + sqlite3VdbeAddOp3(v, OP_Delete, iDataCur, + OPFLAG_ISUPDATE | ((hasFK || chngKey || pPk!=0) ? 0 : OPFLAG_ISNOOP), + regNewRowid + ); + if( !pParse->nested ){ + sqlite3VdbeChangeP4(v, -1, (char*)pTab, P4_TABLE); + } +#else if( hasFK || chngKey || pPk!=0 ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } +#endif if( bReplace || chngKey ){ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } if( hasFK ){ @@ -111750,7 +125136,7 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeResolveLabel(v, labelContinue); sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelContinue); + sqlite3VdbeGoto(v, labelContinue); } sqlite3VdbeResolveLabel(v, labelBreak); @@ -111791,7 +125177,7 @@ update_cleanup: return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView @@ -111804,21 +125190,23 @@ update_cleanup: /* ** Generate code for an UPDATE of a virtual table. ** -** The strategy is that we create an ephemerial table that contains +** There are two possible strategies - the default and the special +** "onepass" strategy. Onepass is only used if the virtual table +** implementation indicates that pWhere may match at most one row. +** +** The default strategy is to create an ephemeral table that contains ** for each row to be changed: ** ** (A) The original rowid of that row. -** (B) The revised rowid for the row. (note1) +** (B) The revised rowid for the row. ** (C) The content of every column in the row. ** -** Then we loop over this ephemeral table and for each row in -** the ephermeral table call VUpdate. +** Then loop through the contents of this ephemeral table executing a +** VUpdate for each row. When finished, drop the ephemeral table. ** -** When finished, drop the ephemeral table. -** -** (note1) Actually, if we know in advance that (A) is always the same -** as (B) we only store (A), then duplicate (A) when pulling -** it out of the ephemeral table before calling VUpdate. +** The "onepass" strategy does not use an ephemeral table. Instead, it +** stores the same values (A, B and C above) in a register array and +** makes a single invocation of VUpdate. */ static void updateVirtualTable( Parse *pParse, /* The parsing context */ @@ -111831,68 +125219,96 @@ static void updateVirtualTable( int onError /* ON CONFLICT strategy */ ){ Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ - ExprList *pEList = 0; /* The result set of the SELECT statement */ - Select *pSelect = 0; /* The SELECT statement */ - Expr *pExpr; /* Temporary expression */ int ephemTab; /* Table holding the result of the SELECT */ int i; /* Loop counter */ - int addr; /* Address of top of loop */ - int iReg; /* First register in set passed to OP_VUpdate */ sqlite3 *db = pParse->db; /* Database connection */ const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); - SelectDest dest; + WhereInfo *pWInfo; + int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ + int regArg; /* First register in VUpdate arg array */ + int regRec; /* Register in which to assemble record */ + int regRowid; /* Register for ephem table rowid */ + int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ + int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ + int bOnePass; /* True to use onepass strategy */ + int addr; /* Address of OP_OpenEphemeral */ - /* Construct the SELECT statement that will find the new values for - ** all updated rows. - */ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, "_rowid_")); - if( pRowid ){ - pEList = sqlite3ExprListAppend(pParse, pEList, - sqlite3ExprDup(db, pRowid, 0)); - } - assert( pTab->iPKey<0 ); - for(i=0; inCol; i++){ - if( aXRef[i]>=0 ){ - pExpr = sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0); - }else{ - pExpr = sqlite3Expr(db, TK_ID, pTab->aCol[i].zName); - } - pEList = sqlite3ExprListAppend(pParse, pEList, pExpr); - } - pSelect = sqlite3SelectNew(pParse, pEList, pSrc, pWhere, 0, 0, 0, 0, 0, 0); - - /* Create the ephemeral table into which the update results will - ** be stored. - */ + /* Allocate nArg registers to martial the arguments to VUpdate. Then + ** create and open the ephemeral table in which the records created from + ** these arguments will be temporarily stored. */ assert( v ); ephemTab = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); - sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + addr= sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, nArg); + regArg = pParse->nMem + 1; + pParse->nMem += nArg; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; - /* fill the ephemeral table - */ - sqlite3SelectDestInit(&dest, SRT_Table, ephemTab); - sqlite3Select(pParse, pSelect, &dest); + /* Start scanning the virtual table */ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); + if( pWInfo==0 ) return; - /* Generate code to scan the ephemeral table and call VUpdate. */ - iReg = ++pParse->nMem; - pParse->nMem += pTab->nCol+1; - addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); + /* Populate the argument registers. */ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); + if( pRowid ){ + sqlite3ExprCode(pParse, pRowid, regArg+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); + } for(i=0; inCol; i++){ - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); + if( aXRef[i]>=0 ){ + sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); + }else{ + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); + } + } + + bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); + + if( bOnePass ){ + /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded + ** above. Also, if this is a top-level parse (not a trigger), clear the + ** multi-write flag so that the VM does not open a statement journal */ + sqlite3VdbeChangeToNoop(v, addr); + if( sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } + }else{ + /* Create a record from the argument register contents and insert it into + ** the ephemeral table. */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid); + } + + + if( bOnePass==0 ){ + /* End the virtual table scan */ + sqlite3WhereEnd(pWInfo); + + /* Begin scannning through the ephemeral table. */ + addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); + + /* Extract arguments from the current row of the ephemeral table and + ** invoke the VUpdate method. */ + for(i=0; inCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, nArg, regArg, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); - /* Cleanup */ - sqlite3SelectDelete(db, pSelect); + /* End of the ephemeral table scan. Or, if using the onepass strategy, + ** jump to here if the scan visited zero rows. */ + if( bOnePass==0 ){ + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); + }else{ + sqlite3WhereEnd(pWInfo); + } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -111914,59 +125330,56 @@ static void updateVirtualTable( ** Most of the code in this file may be omitted by defining the ** SQLITE_OMIT_VACUUM macro. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* -** Finalize a prepared statement. If there was an error, store the -** text of the error message in *pzErrMsg. Return the result code. -*/ -static int vacuumFinalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ - int rc; - rc = sqlite3VdbeFinalize((Vdbe*)pStmt); - if( rc ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - } - return rc; -} /* -** Execute zSql on database db. Return an error code. +** Execute zSql on database db. +** +** If zSql returns rows, then each row will have exactly one +** column. (This will only happen if zSql begins with "SELECT".) +** Take each row of result and call execSql() again recursively. +** +** The execSqlF() routine does the same thing, except it accepts +** a format string as its third argument */ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - VVA_ONLY( int rc; ) - if( !zSql ){ - return SQLITE_NOMEM; - } - if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - return sqlite3_errcode(db); - } - VVA_ONLY( rc = ) sqlite3_step(pStmt); - assert( rc!=SQLITE_ROW || (db->flags&SQLITE_CountRows) ); - return vacuumFinalize(db, pStmt, pzErrMsg); -} - -/* -** Execute zSql on database db. The statement returns exactly -** one column. Execute this as SQL on the same database. -*/ -static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ sqlite3_stmt *pStmt; int rc; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + /* printf("SQL: [%s]\n", zSql); fflush(stdout); */ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - rc = execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); - if( rc!=SQLITE_OK ){ - vacuumFinalize(db, pStmt, pzErrMsg); - return rc; + while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0); + assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 ); + if( zSubSql ){ + assert( zSubSql[0]!='S' ); + rc = execSql(db, pzErrMsg, zSubSql); + if( rc!=SQLITE_OK ) break; } } - - return vacuumFinalize(db, pStmt, pzErrMsg); + assert( rc!=SQLITE_ROW ); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + (void)sqlite3_finalize(pStmt); + return rc; +} +static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ + char *z; + va_list ap; + int rc; + va_start(ap, zSql); + z = sqlite3VMPrintf(db, zSql, ap); + va_end(ap); + if( z==0 ) return SQLITE_NOMEM; + rc = execSql(db, pzErrMsg, z); + sqlite3DbFree(db, z); + return rc; } /* @@ -111985,25 +125398,26 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ ** step (3) requires additional temporary disk space approximately equal ** to the size of the original database for the rollback journal. ** Hence, temporary disk space that is approximately 2x the size of the -** orginal database is required. Every page of the database is written +** original database is required. Every page of the database is written ** approximately 3 times: Once for step (2) and twice for step (3). ** Two writes per page are required in step (3) because the original ** database content must be written into the rollback journal prior to ** overwriting the database with the vacuumed content. ** ** Only 1x temporary space and only 1x writes would be required if -** the copy of step (3) were replace by deleting the original database +** the copy of step (3) were replaced by deleting the original database ** and renaming the transient database as the original. But that will ** not work if other processes are attached to the original database. ** And a power loss in between deleting the original and renaming the ** transient would cause the database file to appear to be deleted ** following reboot. */ -SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse){ +SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm){ Vdbe *v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0); - sqlite3VdbeUsesBtree(v, 0); + int iDb = pNm ? sqlite3TwoPartName(pParse, pNm, pNm, &pNm) : 0; + if( v && (iDb>=2 || iDb==0) ){ + sqlite3VdbeAddOp1(v, OP_Vacuum, iDb); + sqlite3VdbeUsesBtree(v, iDb); } return; } @@ -112011,19 +125425,19 @@ SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse){ /* ** This routine implements the OP_Vacuum opcode of the VDBE. */ -SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ +SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ Btree *pTemp; /* The temporary database we vacuum into */ - char *zSql = 0; /* SQL statements */ int saved_flags; /* Saved value of the db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ - void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ + u8 saved_mTrace; /* Saved trace settings */ Db *pDb = 0; /* Database to detach at end of vacuum */ int isMemDb; /* True if vacuuming a :memory: database */ int nRes; /* Bytes of reserved space at the end of each page */ int nDb; /* Number of attached databases */ + const char *zDbMain; /* Schema name of database to vacuum */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -112040,12 +125454,14 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ saved_flags = db->flags; saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; - saved_xTrace = db->xTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; - db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); - db->xTrace = 0; + saved_mTrace = db->mTrace; + db->flags |= (SQLITE_WriteSchema | SQLITE_IgnoreChecks + | SQLITE_PreferBuiltin | SQLITE_Vacuum); + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_CountRows); + db->mTrace = 0; - pMain = db->aDb[0].pBt; + zDbMain = db->aDb[iDb].zDbSName; + pMain = db->aDb[iDb].pBt; isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); /* Attach the temporary database as 'vacuum_db'. The synchronous pragma @@ -112063,18 +125479,12 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** to write the journal header file. */ nDb = db->nDb; - if( sqlite3TempInMemory(db) ){ - zSql = "ATTACH ':memory:' AS vacuum_db;"; - }else{ - zSql = "ATTACH '' AS vacuum_db;"; - } - rc = execSql(db, pzErrMsg, zSql); - if( db->nDb>nDb ){ - pDb = &db->aDb[db->nDb-1]; - assert( strcmp(pDb->zName,"vacuum_db")==0 ); - } + rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db"); if( rc!=SQLITE_OK ) goto end_of_vacuum; - pTemp = db->aDb[db->nDb-1].pBt; + assert( (db->nDb-1)==nDb ); + pDb = &db->aDb[nDb]; + assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); + pTemp = pDb->pBt; /* The call to execSql() to attach the temp database has left the file ** locked (as there was more than one active statement when the transaction @@ -112082,7 +125492,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** cause problems for the call to BtreeSetPageSize() below. */ sqlite3BtreeCommit(pTemp); - nRes = sqlite3BtreeGetReserve(pMain); + nRes = sqlite3BtreeGetOptimalReserve(pMain); /* A VACUUM cannot change the pagesize of an encrypted database. */ #ifdef SQLITE_HAS_CODEC @@ -112095,14 +125505,15 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ } #endif - rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); - if( rc!=SQLITE_OK ) goto end_of_vacuum; + sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); + sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); + sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL); /* Begin a transaction and take an exclusive lock on the main database ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, ** to ensure that we do not try to change the page-size on a WAL database. */ - rc = execSql(db, pzErrMsg, "BEGIN;"); + rc = execSql(db, pzErrMsg, "BEGIN"); if( rc!=SQLITE_OK ) goto end_of_vacuum; rc = sqlite3BtreeBeginTrans(pMain, 2); if( rc!=SQLITE_OK ) goto end_of_vacuum; @@ -112117,7 +125528,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ || (!isMemDb && sqlite3BtreeSetPageSize(pTemp, db->nextPagesize, nRes, 0)) || NEVER(db->mallocFailed) ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto end_of_vacuum; } @@ -112129,60 +125540,48 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ /* Query the schema of the main database. Create a mirror schema ** in the temporary database. */ - rc = execExecSql(db, pzErrMsg, - "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) " - " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" - " AND coalesce(rootpage,1)>0" + db->init.iDb = nDb; /* force new CREATE statements into vacuum_db */ + rc = execSqlF(db, pzErrMsg, + "SELECT sql FROM \"%w\".sqlite_master" + " WHERE type='table'AND name<>'sqlite_sequence'" + " AND coalesce(rootpage,1)>0", + zDbMain ); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)" - " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %' "); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) " - " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %'"); + rc = execSqlF(db, pzErrMsg, + "SELECT sql FROM \"%w\".sqlite_master" + " WHERE type='index' AND length(sql)>10", + zDbMain + ); if( rc!=SQLITE_OK ) goto end_of_vacuum; + db->init.iDb = 0; /* Loop through the tables in the main database. For each, do ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy ** the contents to the temporary database. */ - rc = execExecSql(db, pzErrMsg, - "SELECT 'INSERT INTO vacuum_db.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';'" - "FROM main.sqlite_master " - "WHERE type = 'table' AND name!='sqlite_sequence' " - " AND coalesce(rootpage,1)>0" + rc = execSqlF(db, pzErrMsg, + "SELECT'INSERT INTO vacuum_db.'||quote(name)" + "||' SELECT*FROM\"%w\".'||quote(name)" + "FROM vacuum_db.sqlite_master " + "WHERE type='table'AND coalesce(rootpage,1)>0", + zDbMain ); + assert( (db->flags & SQLITE_Vacuum)!=0 ); + db->flags &= ~SQLITE_Vacuum; if( rc!=SQLITE_OK ) goto end_of_vacuum; - /* Copy over the sequence table - */ - rc = execExecSql(db, pzErrMsg, - "SELECT 'DELETE FROM vacuum_db.' || quote(name) || ';' " - "FROM vacuum_db.sqlite_master WHERE name='sqlite_sequence' " - ); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'INSERT INTO vacuum_db.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';' " - "FROM vacuum_db.sqlite_master WHERE name=='sqlite_sequence';" - ); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - - /* Copy the triggers, views, and virtual tables from the main database ** over to the temporary database. None of these objects has any ** associated storage, so all we have to do is copy their entries ** from the SQLITE_MASTER table. */ - rc = execSql(db, pzErrMsg, - "INSERT INTO vacuum_db.sqlite_master " - " SELECT type, name, tbl_name, rootpage, sql" - " FROM main.sqlite_master" - " WHERE type='view' OR type='trigger'" - " OR (type='table' AND rootpage=0)" + rc = execSqlF(db, pzErrMsg, + "INSERT INTO vacuum_db.sqlite_master" + " SELECT*FROM \"%w\".sqlite_master" + " WHERE type IN('view','trigger')" + " OR(type='table'AND rootpage=0)", + zDbMain ); if( rc ) goto end_of_vacuum; @@ -112236,10 +125635,11 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ end_of_vacuum: /* Restore the original value of db->flags */ + db->init.iDb = 0; db->flags = saved_flags; db->nChange = saved_nChange; db->nTotalChange = saved_nTotalChange; - db->xTrace = saved_xTrace; + db->mTrace = saved_mTrace; sqlite3BtreeSetPageSize(pMain, -1, -1, 1); /* Currently there is an SQL level transaction open on the vacuum @@ -112282,6 +125682,7 @@ end_of_vacuum: ** This file contains code used to help implement virtual tables. */ #ifndef SQLITE_OMIT_VIRTUALTABLE +/* #include "sqliteInt.h" */ /* ** Before a virtual table xCreate() or xConnect() method is invoked, the @@ -112293,6 +125694,8 @@ end_of_vacuum: struct VtabCtx { VTable *pVTable; /* The virtual table being constructed */ Table *pTab; /* The Table object to which the virtual table belongs */ + VtabCtx *pPrior; /* Parent context (if any) */ + int bDeclared; /* True after sqlite3_declare_vtab() is called */ }; /* @@ -112312,11 +125715,11 @@ static int createModule( sqlite3_mutex_enter(db->mutex); nName = sqlite3Strlen30(zName); - if( sqlite3HashFind(&db->aModule, zName, nName) ){ + if( sqlite3HashFind(&db->aModule, zName) ){ rc = SQLITE_MISUSE_BKPT; }else{ Module *pMod; - pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); + pMod = (Module *)sqlite3DbMallocRawNN(db, sizeof(Module) + nName + 1); if( pMod ){ Module *pDel; char *zCopy = (char *)(&pMod[1]); @@ -112325,10 +125728,11 @@ static int createModule( pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; - pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,nName,(void*)pMod); + pMod->pEpoTab = 0; + pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); assert( pDel==0 || pDel==pMod ); if( pDel ){ - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, pDel); } } @@ -112350,6 +125754,9 @@ SQLITE_API int sqlite3_create_module( const sqlite3_module *pModule, /* The definition of the module */ void *pAux /* Context pointer for xCreate/xConnect */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, 0); } @@ -112363,6 +125770,9 @@ SQLITE_API int sqlite3_create_module_v2( void *pAux, /* Context pointer for xCreate/xConnect */ void (*xDestroy)(void *) /* Module destructor function */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, xDestroy); } @@ -112546,23 +125956,17 @@ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ ** deleted. */ static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int i = pTable->nModuleArg++; - int nBytes = sizeof(char *)*(1+pTable->nModuleArg); + int nBytes = sizeof(char *)*(2+pTable->nModuleArg); char **azModuleArg; azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); if( azModuleArg==0 ){ - int j; - for(j=0; jazModuleArg[j]); - } sqlite3DbFree(db, zArg); - sqlite3DbFree(db, pTable->azModuleArg); - pTable->nModuleArg = 0; }else{ + int i = pTable->nModuleArg++; azModuleArg[i] = zArg; azModuleArg[i+1] = 0; + pTable->azModuleArg = azModuleArg; } - pTable->azModuleArg = azModuleArg; } /* @@ -112595,7 +125999,12 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); - pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) + || (pParse->sNameToken.z==pName1->z && pName2->z==0) + ); + pParse->sNameToken.n = (int)( + &pModuleName->z[pModuleName->n] - pParse->sNameToken.z + ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Creating a virtual table invokes the authorization callback twice. @@ -112605,7 +126014,7 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( */ if( pTable->azModuleArg ){ sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, - pTable->azModuleArg[0], pParse->db->aDb[iDb].zName); + pTable->azModuleArg[0], pParse->db->aDb[iDb].zDbSName); } #endif } @@ -112647,6 +126056,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ char *zStmt; char *zWhere; int iDb; + int iReg; Vdbe *v; /* Compute the complete text of the CREATE VIRTUAL TABLE statement */ @@ -112668,7 +126078,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ "UPDATE %Q.%s " "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q " "WHERE rowid=#%d", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), pTab->zName, pTab->zName, zStmt, @@ -112678,11 +126088,13 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ v = sqlite3GetVdbe(pParse); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + sqlite3VdbeAddOp0(v, OP_Expire); zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); - sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, - pTab->zName, sqlite3Strlen30(pTab->zName) + 1); + + iReg = ++pParse->nMem; + sqlite3VdbeLoadString(v, iReg, pTab->zName); + sqlite3VdbeAddOp2(v, OP_VCreate, iDb, iReg); } /* If we are rereading the sqlite_master table create the in-memory @@ -112694,11 +126106,10 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ Table *pOld; Schema *pSchema = pTab->pSchema; const char *zName = pTab->zName; - int nName = sqlite3Strlen30(zName); assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); - pOld = sqlite3HashInsert(&pSchema->tblHash, zName, nName, pTab); + pOld = sqlite3HashInsert(&pSchema->tblHash, zName, pTab); if( pOld ){ - db->mallocFailed = 1; + sqlite3OomFault(db); assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ return; } @@ -112726,7 +126137,7 @@ SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){ pArg->z = p->z; pArg->n = p->n; }else{ - assert(pArg->z < p->z); + assert(pArg->z <= p->z); pArg->n = (int)(&p->z[p->n] - pArg->z); } } @@ -112743,40 +126154,54 @@ static int vtabCallConstructor( int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), char **pzErr ){ - VtabCtx sCtx, *pPriorCtx; + VtabCtx sCtx; VTable *pVTable; int rc; const char *const*azArg = (const char *const*)pTab->azModuleArg; int nArg = pTab->nModuleArg; char *zErr = 0; - char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + char *zModuleName; int iDb; + VtabCtx *pCtx; + /* Check that the virtual-table is not already being initialized */ + for(pCtx=db->pVtabCtx; pCtx; pCtx=pCtx->pPrior){ + if( pCtx->pTab==pTab ){ + *pzErr = sqlite3MPrintf(db, + "vtable constructor called recursively: %s", pTab->zName + ); + return SQLITE_LOCKED; + } + } + + zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); if( !zModuleName ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pVTable = sqlite3DbMallocZero(db, sizeof(VTable)); if( !pVTable ){ sqlite3DbFree(db, zModuleName); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pVTable->db = db; pVTable->pMod = pMod; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - pTab->azModuleArg[1] = db->aDb[iDb].zName; + pTab->azModuleArg[1] = db->aDb[iDb].zDbSName; /* Invoke the virtual table constructor */ assert( &db->pVtabCtx ); assert( xConstruct ); sCtx.pTab = pTab; sCtx.pVTable = pVTable; - pPriorCtx = db->pVtabCtx; + sCtx.pPrior = db->pVtabCtx; + sCtx.bDeclared = 0; db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); - db->pVtabCtx = pPriorCtx; - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + db->pVtabCtx = sCtx.pPrior; + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + assert( sCtx.pTab==pTab ); if( SQLITE_OK!=rc ){ if( zErr==0 ){ @@ -112789,15 +126214,17 @@ static int vtabCallConstructor( }else if( ALWAYS(pVTable->pVtab) ){ /* Justification of ALWAYS(): A correct vtab constructor must allocate ** the sqlite3_vtab object if successful. */ + memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; - if( sCtx.pTab ){ + if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ int iCol; + u8 oooHidden = 0; /* If everything went according to plan, link the new VTable structure ** into the linked list headed by pTab->pVTable. Then loop through the ** columns of the table to see if any of them contain the token "hidden". @@ -112807,19 +126234,16 @@ static int vtabCallConstructor( pTab->pVTable = pVTable; for(iCol=0; iColnCol; iCol++){ - char *zType = pTab->aCol[iCol].zType; + char *zType = sqlite3ColumnType(&pTab->aCol[iCol], ""); int nType; int i = 0; - if( !zType ) continue; nType = sqlite3Strlen30(zType); - if( sqlite3StrNICmp("hidden", zType, 6)||(zType[6] && zType[6]!=' ') ){ - for(i=0; iaCol[iCol].colFlags |= COLFLAG_HIDDEN; + oooHidden = TF_OOOHidden; + }else{ + pTab->tabFlags |= oooHidden; } } } @@ -112862,7 +126289,7 @@ SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); if( !pMod ){ const char *zModule = pTab->azModuleArg[0]; @@ -112892,7 +126319,7 @@ static int growVTrans(sqlite3 *db){ int nBytes = sizeof(sqlite3_vtab *) * (db->nVTrans + ARRAY_INCR); aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); if( !aVTrans ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(&aVTrans[db->nVTrans], 0, sizeof(sqlite3_vtab *)*ARRAY_INCR); db->aVTrans = aVTrans; @@ -112915,7 +126342,7 @@ static void addToVTrans(sqlite3 *db, VTable *pVTab){ ** This function is invoked by the vdbe to call the xCreate method ** of the virtual table named zTab in database iDb. ** -** If an error occurs, *pzErr is set to point an an English language +** If an error occurs, *pzErr is set to point to an English language ** description of the error and an SQLITE_XXX error code is returned. ** In this case the caller must call sqlite3DbFree(db, ) on *pzErr. */ @@ -112925,18 +126352,18 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, Module *pMod; const char *zMod; - pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable ); /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); /* If the module has been registered and includes a Create method, ** invoke it now. If the module has not been registered, return an ** error. Otherwise, do nothing. */ - if( !pMod ){ + if( pMod==0 || pMod->pModule->xCreate==0 || pMod->pModule->xDestroy==0 ){ *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); rc = SQLITE_ERROR; }else{ @@ -112961,23 +126388,30 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, ** virtual table module. */ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ + VtabCtx *pCtx; Parse *pParse; - int rc = SQLITE_OK; Table *pTab; char *zErr = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); - if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ - sqlite3Error(db, SQLITE_MISUSE, 0); + pCtx = db->pVtabCtx; + if( !pCtx || pCtx->bDeclared ){ + sqlite3Error(db, SQLITE_MISUSE); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } + pTab = pCtx->pTab; assert( (pTab->tabFlags & TF_Virtual)!=0 ); pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); if( pParse==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ pParse->declareVtab = 1; pParse->db = db; @@ -112990,14 +126424,28 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ && (pParse->pNewTable->tabFlags & TF_Virtual)==0 ){ if( !pTab->aCol ){ - pTab->aCol = pParse->pNewTable->aCol; - pTab->nCol = pParse->pNewTable->nCol; - pParse->pNewTable->nCol = 0; - pParse->pNewTable->aCol = 0; + Table *pNew = pParse->pNewTable; + Index *pIdx; + pTab->aCol = pNew->aCol; + pTab->nCol = pNew->nCol; + pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid); + pNew->nCol = 0; + pNew->aCol = 0; + assert( pTab->pIndex==0 ); + if( !HasRowid(pNew) && pCtx->pVTable->pMod->pModule->xUpdate!=0 ){ + rc = SQLITE_ERROR; + } + pIdx = pNew->pIndex; + if( pIdx ){ + assert( pIdx->pNext==0 ); + pTab->pIndex = pIdx; + pNew->pIndex = 0; + pIdx->pTable = pTab; + } } - db->pVtabCtx->pTab = 0; + pCtx->bDeclared = 1; }else{ - sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } @@ -113028,13 +126476,20 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab int rc = SQLITE_OK; Table *pTab; - pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); - if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ - VTable *p = vtabDisconnectAll(db, pTab); - - assert( rc==SQLITE_OK ); - rc = p->pMod->pModule->xDestroy(p->pVtab); - + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); + if( pTab!=0 && ALWAYS(pTab->pVTable!=0) ){ + VTable *p; + int (*xDestroy)(sqlite3_vtab *); + for(p=pTab->pVTable; p; p=p->pNext){ + assert( p->pVtab ); + if( p->pVtab->nRef>0 ){ + return SQLITE_LOCKED; + } + } + p = vtabDisconnectAll(db, pTab); + xDestroy = p->pMod->pModule->xDestroy; + assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ assert( pTab->pVTable==p && p->pNext==0 ); @@ -113058,8 +126513,10 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab static void callFinaliser(sqlite3 *db, int offset){ int i; if( db->aVTrans ){ + VTable **aVTrans = db->aVTrans; + db->aVTrans = 0; for(i=0; inVTrans; i++){ - VTable *pVTab = db->aVTrans[i]; + VTable *pVTab = aVTrans[i]; sqlite3_vtab *p = pVTab->pVtab; if( p ){ int (*x)(sqlite3_vtab *); @@ -113069,9 +126526,8 @@ static void callFinaliser(sqlite3 *db, int offset){ pVTab->iSavepoint = 0; sqlite3VtabUnlock(pVTab); } - sqlite3DbFree(db, db->aVTrans); + sqlite3DbFree(db, aVTrans); db->nVTrans = 0; - db->aVTrans = 0; } } @@ -113159,7 +126615,12 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ if( rc==SQLITE_OK ){ rc = pModule->xBegin(pVTab->pVtab); if( rc==SQLITE_OK ){ + int iSvpt = db->nStatement + db->nSavepoint; addToVTrans(db, pVTab); + if( iSvpt && pModule->xSavepoint ){ + pVTab->iSavepoint = iSvpt; + rc = pModule->xSavepoint(pVTab->pVtab, iSvpt-1); + } } } } @@ -113185,7 +126646,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ int rc = SQLITE_OK; assert( op==SAVEPOINT_RELEASE||op==SAVEPOINT_ROLLBACK||op==SAVEPOINT_BEGIN ); - assert( iSavepoint>=0 ); + assert( iSavepoint>=-1 ); if( db->aVTrans ){ int i; for(i=0; rc==SQLITE_OK && inVTrans; i++){ @@ -113236,7 +126697,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( Table *pTab; sqlite3_vtab *pVtab; sqlite3_module *pMod; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**) = 0; + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**) = 0; void *pArg = 0; FuncDef *pNew; int rc = 0; @@ -113264,7 +126725,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( for(z=(unsigned char*)zLowerName; *z; z++){ *z = sqlite3UpperToLower[*z]; } - rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xFunc, &pArg); + rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xSFunc, &pArg); sqlite3DbFree(db, zLowerName); } if( rc==0 ){ @@ -113279,9 +126740,9 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( return pDef; } *pNew = *pDef; - pNew->zName = (char *)&pNew[1]; - memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1); - pNew->xFunc = xFunc; + pNew->zName = (const char*)&pNew[1]; + memcpy((char*)&pNew[1], pDef->zName, sqlite3Strlen30(pDef->zName)+1); + pNew->xSFunc = xSFunc; pNew->pUserData = pArg; pNew->funcFlags |= SQLITE_FUNC_EPHEM; return pNew; @@ -113303,12 +126764,76 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ if( pTab==pToplevel->apVtabLock[i] ) return; } n = (pToplevel->nVtabLock+1)*sizeof(pToplevel->apVtabLock[0]); - apVtabLock = sqlite3_realloc(pToplevel->apVtabLock, n); + apVtabLock = sqlite3_realloc64(pToplevel->apVtabLock, n); if( apVtabLock ){ pToplevel->apVtabLock = apVtabLock; pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; }else{ - pToplevel->db->mallocFailed = 1; + sqlite3OomFault(pToplevel->db); + } +} + +/* +** Check to see if virtual table module pMod can be have an eponymous +** virtual table instance. If it can, create one if one does not already +** exist. Return non-zero if the eponymous virtual table instance exists +** when this routine returns, and return zero if it does not exist. +** +** An eponymous virtual table instance is one that is named after its +** module, and more importantly, does not require a CREATE VIRTUAL TABLE +** statement in order to come into existance. Eponymous virtual table +** instances always exist. They cannot be DROP-ed. +** +** Any virtual table module for which xConnect and xCreate are the same +** method can have an eponymous virtual table instance. +*/ +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ + const sqlite3_module *pModule = pMod->pModule; + Table *pTab; + char *zErr = 0; + int rc; + sqlite3 *db = pParse->db; + if( pMod->pEpoTab ) return 1; + if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0; + pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return 0; + pTab->zName = sqlite3DbStrDup(db, pMod->zName); + if( pTab->zName==0 ){ + sqlite3DbFree(db, pTab); + return 0; + } + pMod->pEpoTab = pTab; + pTab->nRef = 1; + pTab->pSchema = db->aDb[0].pSchema; + pTab->tabFlags |= TF_Virtual; + pTab->nModuleArg = 0; + pTab->iPKey = -1; + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(db, pTab, 0); + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); + if( rc ){ + sqlite3ErrorMsg(pParse, "%s", zErr); + sqlite3DbFree(db, zErr); + sqlite3VtabEponymousTableClear(db, pMod); + return 0; + } + return 1; +} + +/* +** Erase the eponymous virtual table instance associated with +** virtual table module pMod, if it exists. +*/ +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ + Table *pTab = pMod->pEpoTab; + if( pTab!=0 ){ + /* Mark the table as Ephemeral prior to deleting it, so that the + ** sqlite3DeleteTable() routine will know that it is not stored in + ** the schema. */ + pTab->tabFlags |= TF_Ephemeral; + sqlite3DeleteTable(db, pTab); + pMod->pEpoTab = 0; } } @@ -113323,6 +126848,9 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ static const unsigned char aMap[] = { SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE }; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); assert( OE_Ignore==4 && OE_Replace==5 ); assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); @@ -113338,8 +126866,10 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ va_list ap; int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); - va_start(ap, op); switch( op ){ case SQLITE_VTAB_CONSTRAINT_SUPPORT: { @@ -113358,7 +126888,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ } va_end(ap); - if( rc!=SQLITE_OK ) sqlite3Error(db, rc, 0); + if( rc!=SQLITE_OK ) sqlite3Error(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } @@ -113366,9 +126896,9 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ #endif /* SQLITE_OMIT_VIRTUALTABLE */ /************** End of vtab.c ************************************************/ -/************** Begin file where.c *******************************************/ +/************** Begin file wherecode.c ***************************************/ /* -** 2001 September 15 +** 2015-06-06 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -113379,13 +126909,15 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** ************************************************************************* ** This module contains C code that generates VDBE code used to process -** the WHERE clause of SQL statements. This module is responsible for -** generating the code that loops through a table looking for applicable -** rows. Indices are selected and used to speed the search when doing -** so is applicable. Because this module is responsible for selecting -** indices, you might also think of this module as the "query optimizer". +** the WHERE clause of SQL statements. +** +** This file was split off from where.c on 2015-06-06 in order to reduce the +** size of where.c and make it easier to edit. This file contains the routines +** that actually generate the bulk of the WHERE loop code. The original where.c +** file retains the code that does query planning and analysis. */ -/************** Include whereInt.h in the middle of where.c ******************/ +/* #include "sqliteInt.h" */ +/************** Include whereInt.h in the middle of wherecode.c **************/ /************** Begin file whereInt.h ****************************************/ /* ** 2013-11-12 @@ -113408,7 +126940,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** Trace output macros */ #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -/***/ int sqlite3WhereTrace = 0; +/***/ int sqlite3WhereTrace; #endif #if defined(SQLITE_DEBUG) \ && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) @@ -113458,6 +126990,10 @@ struct WhereLevel { int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */ + int addrLikeRep; /* LIKE range processing address */ +#endif u8 iFrom; /* Which entry in the FROM clause */ u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ int p1, p2; /* Operands of the opcode used to ends the loop */ @@ -113474,6 +127010,9 @@ struct WhereLevel { } u; struct WhereLoop *pWLoop; /* The selected WhereLoop object */ Bitmask notReady; /* FROM entries not usable at this level */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrVisit; /* Address at which row is visited */ +#endif }; /* @@ -113504,7 +127043,8 @@ struct WhereLoop { union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ - u16 nSkip; /* Number of initial index columns to skip */ + u16 nBtm; /* Size of BTM vector */ + u16 nTop; /* Size of TOP vector */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -113517,12 +127057,13 @@ struct WhereLoop { } u; u32 wsFlags; /* WHERE_* flags describing the plan */ u16 nLTerm; /* Number of entries in aLTerm[] */ + u16 nSkip; /* Number of NULL aLTerm[] entries */ /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ - WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ + WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ }; /* This object holds the prerequisites and the cost of running a @@ -113545,10 +127086,6 @@ struct WhereOrSet { WhereOrCost a[N_OR_COST]; /* Set of best costs */ }; - -/* Forward declaration of methods */ -static int whereLoopResize(sqlite3*, WhereLoop*, int); - /* ** Each instance of this object holds a sequence of WhereLoop objects ** that implement some or all of a query plan. @@ -113565,7 +127102,7 @@ static int whereLoopResize(sqlite3*, WhereLoop*, int); ** 1. Then using those as a basis to compute the N best WherePath objects ** of length 2. And so forth until the length of WherePaths equals the ** number of nodes in the FROM clause. The best (lowest cost) WherePath -** at the end is the choosen query plan. +** at the end is the chosen query plan. */ struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ @@ -113630,18 +127167,20 @@ struct WherePath { */ struct WhereTerm { Expr *pExpr; /* Pointer to the subexpression that is this term */ + WhereClause *pWC; /* The clause this term is part of */ + LogEst truthProb; /* Probability of truth for this expression */ + u16 wtFlags; /* TERM_xxx bit flags. See below */ + u16 eOperator; /* A WO_xx value describing */ + u8 nChild; /* Number of children that must disable us */ + u8 eMatchOp; /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */ int iParent; /* Disable pWC->a[iParent] when this term disabled */ int leftCursor; /* Cursor number of X in "X " */ + int iField; /* Field in (?,?,?) IN (SELECT...) vector */ union { int leftColumn; /* Column number of X in "X " */ WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; - LogEst truthProb; /* Probability of truth for this expression */ - u16 eOperator; /* A WO_xx value describing */ - u8 wtFlags; /* TERM_xxx bit flags. See below */ - u8 nChild; /* Number of children that must disable us */ - WhereClause *pWC; /* The clause this term is part of */ Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ }; @@ -113661,6 +127200,10 @@ struct WhereTerm { #else # define TERM_VNULL 0x00 /* Disabled if not using stat3 */ #endif +#define TERM_LIKEOPT 0x100 /* Virtual terms from the LIKE optimization */ +#define TERM_LIKECOND 0x200 /* Conditionally this LIKE operator term */ +#define TERM_LIKE 0x400 /* The original LIKE operator */ +#define TERM_IS 0x800 /* Term.pExpr is an IS operator */ /* ** An instance of the WhereScan object is used as an iterator for locating @@ -113669,13 +127212,15 @@ struct WhereTerm { struct WhereScan { WhereClause *pOrigWC; /* Original, innermost WhereClause */ WhereClause *pWC; /* WhereClause currently being scanned */ - char *zCollName; /* Required collating sequence, if not NULL */ + const char *zCollName; /* Required collating sequence, if not NULL */ + Expr *pIdxExpr; /* Search for this index expression */ char idxaff; /* Must match this affinity, if zCollName!=NULL */ unsigned char nEquiv; /* Number of entries in aEquiv[] */ unsigned char iEquiv; /* Next unused slot in aEquiv[] */ u32 opMask; /* Acceptable operators */ int k; /* Resume scanning at this->pWC->a[this->k] */ - int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ + int aiCur[11]; /* Cursors in the equivalence class */ + i16 aiColumn[11]; /* Corresponding column number in the eq-class */ }; /* @@ -113752,6 +127297,11 @@ struct WhereMaskSet { int ix[BMS]; /* Cursor assigned to each bit */ }; +/* +** Initialize a WhereMaskSet object +*/ +#define initMaskSet(P) (P)->n=0 + /* ** This object is a convenience wrapper holding all information needed ** to construct WhereLoop objects for a particular query. @@ -113782,48 +127332,119 @@ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ - ExprList *pResultSet; /* Result set. DISTINCT operates on these */ - WhereLoop *pLoops; /* List of all WhereLoop objects */ - Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ - LogEst nRowOut; /* Estimated number of output rows */ - u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ - u8 sorted; /* True if really sorted (not just grouped) */ - u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ - u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ - u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ - u8 nLevel; /* Number of nested loop */ - int iTop; /* The very beginning of the WHERE loop */ + ExprList *pDistinctSet; /* DISTINCT over all these values */ + LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ + int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ - int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ - WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ + u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + u8 nLevel; /* Number of nested loop */ + i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u8 sorted; /* True if really sorted (not just grouped) */ + u8 eOnePass; /* ONEPASS_OFF, or _SINGLE, or _MULTI */ + u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values */ + u8 bOrderedInnerLoop; /* True if only the inner-most loop is ordered */ + int iTop; /* The very beginning of the WHERE loop */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ + LogEst nRowOut; /* Estimated number of output rows */ WhereClause sWC; /* Decomposition of the WHERE clause */ + WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; +/* +** Private interfaces - callable only by other where.c routines. +** +** where.c: +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); +#ifdef WHERETRACE_ENABLED +SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC); +#endif +SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ +); + +/* wherecode.c: */ +#ifndef SQLITE_OMIT_EXPLAIN +SQLITE_PRIVATE int sqlite3WhereExplainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +); +#else +# define sqlite3WhereExplainOneScan(u,v,w,x,y,z) 0 +#endif /* SQLITE_OMIT_EXPLAIN */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +SQLITE_PRIVATE void sqlite3WhereAddScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ +); +#else +# define sqlite3WhereAddScanStatus(a, b, c, d) ((void)d) +#endif +SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + Bitmask notReady /* Which tables are currently available */ +); + +/* whereexpr.c: */ +SQLITE_PRIVATE void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); +SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause*); +SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); +SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); +SQLITE_PRIVATE void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); + + + + + /* ** Bitmasks for the operators on WhereTerm objects. These are all ** operators that are of interest to the query planner. An ** OR-ed combination of these values can be used when searching for ** particular WhereTerms within a WhereClause. +** +** Value constraints: +** WO_EQ == SQLITE_INDEX_CONSTRAINT_EQ +** WO_LT == SQLITE_INDEX_CONSTRAINT_LT +** WO_LE == SQLITE_INDEX_CONSTRAINT_LE +** WO_GT == SQLITE_INDEX_CONSTRAINT_GT +** WO_GE == SQLITE_INDEX_CONSTRAINT_GE +** WO_MATCH == SQLITE_INDEX_CONSTRAINT_MATCH */ -#define WO_IN 0x001 -#define WO_EQ 0x002 +#define WO_IN 0x0001 +#define WO_EQ 0x0002 #define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) #define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) #define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) #define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) -#define WO_MATCH 0x040 -#define WO_ISNULL 0x080 -#define WO_OR 0x100 /* Two or more OR-connected terms */ -#define WO_AND 0x200 /* Two or more AND-connected terms */ -#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ -#define WO_NOOP 0x800 /* This term does not restrict search space */ +#define WO_MATCH 0x0040 +#define WO_IS 0x0080 +#define WO_ISNULL 0x0100 +#define WO_OR 0x0200 /* Two or more OR-connected terms */ +#define WO_AND 0x0400 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x0800 /* Of the form A==B, both columns */ +#define WO_NOOP 0x1000 /* This term does not restrict search space */ -#define WO_ALL 0xfff /* Mask of all possible WO_* values */ -#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ +#define WO_ALL 0x1fff /* Mask of all possible WO_* values */ +#define WO_SINGLE 0x01ff /* Mask of all non-compound WO_* values */ /* ** These are definitions of bits in the WhereLoop.wsFlags field. @@ -113848,140 +127469,2059 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ /************** End of whereInt.h ********************************************/ -/************** Continuing where we left off in where.c **********************/ +/************** Continuing where we left off in wherecode.c ******************/ + +#ifndef SQLITE_OMIT_EXPLAIN /* -** Return the estimated number of output rows from a WHERE clause +** Return the name of the i-th column of the pIdx index. */ -SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ - return sqlite3LogEstToInt(pWInfo->nRowOut); +static const char *explainIndexColumnName(Index *pIdx, int i){ + i = pIdx->aiColumn[i]; + if( i==XN_EXPR ) return ""; + if( i==XN_ROWID ) return "rowid"; + return pIdx->pTable->aCol[i].zName; } /* -** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this -** WHERE clause returns outputs for DISTINCT processing. -*/ -SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ - return pWInfo->eDistinct; -} - -/* -** Return TRUE if the WHERE clause returns rows in ORDER BY order. -** Return FALSE if the output needs to be sorted. -*/ -SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat; -} - -/* -** Return the VDBE address or label to jump to in order to continue -** immediately with the next row of a WHERE clause. -*/ -SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ - assert( pWInfo->iContinue!=0 ); - return pWInfo->iContinue; -} - -/* -** Return the VDBE address or label to jump to in order to break -** out of a WHERE loop. -*/ -SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ - return pWInfo->iBreak; -} - -/* -** Return TRUE if an UPDATE or DELETE statement can operate directly on -** the rowids returned by a WHERE clause. Return FALSE if doing an -** UPDATE or DELETE might change subsequent WHERE clause results. +** This routine is a helper for explainIndexRange() below ** -** If the ONEPASS optimization is used (if this routine returns true) -** then also write the indices of open cursors used by ONEPASS -** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data -** table and iaCur[1] gets the cursor used by an auxiliary index. -** Either value may be -1, indicating that cursor is not used. -** Any cursors returned will have been opened for writing. -** -** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is -** unable to use the ONEPASS optimization. +** pStr holds the text of an expression that we are building up one term +** at a time. This routine adds a new term to the end of the expression. +** Terms are separated by AND so add the "AND" text for second and subsequent +** terms only. */ -SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ - memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); - return pWInfo->okOnePass; -} - -/* -** Move the content of pSrc into pDest -*/ -static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ - pDest->n = pSrc->n; - memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); -} - -/* -** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. -** -** The new entry might overwrite an existing entry, or it might be -** appended, or it might be discarded. Do whatever is the right thing -** so that pSet keeps the N_OR_COST best entries seen so far. -*/ -static int whereOrInsert( - WhereOrSet *pSet, /* The WhereOrSet to be updated */ - Bitmask prereq, /* Prerequisites of the new entry */ - LogEst rRun, /* Run-cost of the new entry */ - LogEst nOut /* Number of outputs for the new entry */ +static void explainAppendTerm( + StrAccum *pStr, /* The text expression being built */ + Index *pIdx, /* Index to read column names from */ + int nTerm, /* Number of terms */ + int iTerm, /* Zero-based index of first term. */ + int bAnd, /* Non-zero to append " AND " */ + const char *zOp /* Name of the operator */ ){ - u16 i; - WhereOrCost *p; - for(i=pSet->n, p=pSet->a; i>0; i--, p++){ - if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ - goto whereOrInsert_done; - } - if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ - return 0; - } + int i; + + assert( nTerm>=1 ); + if( bAnd ) sqlite3StrAccumAppend(pStr, " AND ", 5); + + if( nTerm>1 ) sqlite3StrAccumAppend(pStr, "(", 1); + for(i=0; ina[pSet->n++]; - p->nOut = nOut; + if( nTerm>1 ) sqlite3StrAccumAppend(pStr, ")", 1); + + sqlite3StrAccumAppend(pStr, zOp, 1); + + if( nTerm>1 ) sqlite3StrAccumAppend(pStr, "(", 1); + for(i=0; i1 ) sqlite3StrAccumAppend(pStr, ")", 1); +} + +/* +** Argument pLevel describes a strategy for scanning table pTab. This +** function appends text to pStr that describes the subset of table +** rows scanned by the strategy in the form of an SQL expression. +** +** For example, if the query: +** +** SELECT * FROM t1 WHERE a=1 AND b>2; +** +** is run and there is an index on (a, b), then this function returns a +** string similar to: +** +** "a=? AND b>?" +*/ +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ + Index *pIndex = pLoop->u.btree.pIndex; + u16 nEq = pLoop->u.btree.nEq; + u16 nSkip = pLoop->nSkip; + int i, j; + + if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; + sqlite3StrAccumAppend(pStr, " (", 2); + for(i=0; i=nSkip ? "%s=?" : "ANY(%s)", z); + } + + j = i; + if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ + explainAppendTerm(pStr, pIndex, pLoop->u.btree.nBtm, j, i, ">"); + i = 1; + } + if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ + explainAppendTerm(pStr, pIndex, pLoop->u.btree.nTop, j, i, "<"); + } + sqlite3StrAccumAppend(pStr, ")", 1); +} + +/* +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was +** defined at compile-time. If it is not a no-op, a single OP_Explain opcode +** is added to the output to describe the table scan strategy in pLevel. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. +*/ +SQLITE_PRIVATE int sqlite3WhereExplainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +){ + int ret = 0; +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pParse->explain==2 ) +#endif + { + struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + int iId = pParse->iSelectId; /* Select id (left-most output column) */ + int isSearch; /* True for a SEARCH. False for SCAN. */ + WhereLoop *pLoop; /* The controlling WhereLoop object */ + u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ + + pLoop = pLevel->pWLoop; + flags = pLoop->wsFlags; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0; + + isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + + sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); + if( pItem->pSelect ){ + sqlite3XPrintf(&str, " SUBQUERY %d", pItem->iSelectId); + }else{ + sqlite3XPrintf(&str, " TABLE %s", pItem->zName); + } + + if( pItem->zAlias ){ + sqlite3XPrintf(&str, " AS %s", pItem->zAlias); + } + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + const char *zFmt = 0; + Index *pIdx; + + assert( pLoop->u.btree.pIndex!=0 ); + pIdx = pLoop->u.btree.pIndex; + assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); + if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( isSearch ){ + zFmt = "PRIMARY KEY"; + } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; + }else if( flags & WHERE_AUTO_INDEX ){ + zFmt = "AUTOMATIC COVERING INDEX"; + }else if( flags & WHERE_IDX_ONLY ){ + zFmt = "COVERING INDEX %s"; + }else{ + zFmt = "INDEX %s"; + } + if( zFmt ){ + sqlite3StrAccumAppend(&str, " USING ", 7); + sqlite3XPrintf(&str, zFmt, pIdx->zName); + explainIndexRange(&str, pLoop); + } + }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ + const char *zRangeOp; + if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ + zRangeOp = "="; + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zRangeOp = ">? AND rowid<"; + }else if( flags&WHERE_BTM_LIMIT ){ + zRangeOp = ">"; + }else{ + assert( flags&WHERE_TOP_LIMIT); + zRangeOp = "<"; + } + sqlite3XPrintf(&str, " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ + sqlite3XPrintf(&str, " VIRTUAL TABLE INDEX %d:%s", + pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); + } +#endif +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + if( pLoop->nOut>=10 ){ + sqlite3XPrintf(&str, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + }else{ + sqlite3StrAccumAppend(&str, " (~1 row)", 9); + } +#endif + zMsg = sqlite3StrAccumFinish(&str); + ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); + } + return ret; +} +#endif /* SQLITE_OMIT_EXPLAIN */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Configure the VM passed as the first argument with an +** sqlite3_stmt_scanstatus() entry corresponding to the scan used to +** implement level pLvl. Argument pSrclist is a pointer to the FROM +** clause that the scan reads data from. +** +** If argument addrExplain is not 0, it must be the address of an +** OP_Explain instruction that describes the same loop. +*/ +SQLITE_PRIVATE void sqlite3WhereAddScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ +){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; }else{ - p = pSet->a; - for(i=1; in; i++){ - if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; - } - if( p->rRun<=rRun ) return 0; + zObj = pSrclist->a[pLvl->iFrom].zName; + } + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); +} +#endif + + +/* +** Disable a term in the WHERE clause. Except, do not disable the term +** if it controls a LEFT OUTER JOIN and it did not originate in the ON +** or USING clause of that join. +** +** Consider the term t2.z='ok' in the following queries: +** +** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' +** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' +** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' +** +** The t2.z='ok' is disabled in the in (2) because it originates +** in the ON clause. The term is disabled in (3) because it is not part +** of a LEFT OUTER JOIN. In (1), the term is not disabled. +** +** Disabling a term causes that term to not be tested in the inner loop +** of the join. Disabling is an optimization. When terms are satisfied +** by indices, we disable them to prevent redundant tests in the inner +** loop. We would get the correct results if nothing were ever disabled, +** but joins might run a little slower. The trick is to disable as much +** as we can without disabling too much. If we disabled in (1), we'd get +** the wrong answer. See ticket #813. +** +** If all the children of a term are disabled, then that term is also +** automatically disabled. In this way, terms get disabled if derived +** virtual terms are tested first. For example: +** +** x GLOB 'abc*' AND x>='abc' AND x<'acd' +** \___________/ \______/ \_____/ +** parent child1 child2 +** +** Only the parent term was in the original WHERE clause. The child1 +** and child2 terms were added by the LIKE optimization. If both of +** the virtual child terms are valid, then testing of the parent can be +** skipped. +** +** Usually the parent term is marked as TERM_CODED. But if the parent +** term was originally TERM_LIKE, then the parent gets TERM_LIKECOND instead. +** The TERM_LIKECOND marking indicates that the term should be coded inside +** a conditional such that is only evaluated on the second pass of a +** LIKE-optimization loop, when scanning BLOBs instead of strings. +*/ +static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ + int nLoop = 0; + while( ALWAYS(pTerm!=0) + && (pTerm->wtFlags & TERM_CODED)==0 + && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pLevel->notReady & pTerm->prereqAll)==0 + ){ + if( nLoop && (pTerm->wtFlags & TERM_LIKE)!=0 ){ + pTerm->wtFlags |= TERM_LIKECOND; + }else{ + pTerm->wtFlags |= TERM_CODED; + } + if( pTerm->iParent<0 ) break; + pTerm = &pTerm->pWC->a[pTerm->iParent]; + pTerm->nChild--; + if( pTerm->nChild!=0 ) break; + nLoop++; } -whereOrInsert_done: - p->prereq = prereq; - p->rRun = rRun; - if( p->nOut>nOut ) p->nOut = nOut; - return 1; } /* -** Initialize a preallocated WhereClause structure. +** Code an OP_Affinity opcode to apply the column affinity string zAff +** to the n registers starting at base. +** +** As an optimization, SQLITE_AFF_BLOB entries (which are no-ops) at the +** beginning and end of zAff are ignored. If all entries in zAff are +** SQLITE_AFF_BLOB, then no code gets generated. +** +** This routine makes its own copy of zAff so that the caller is free +** to modify zAff after this routine returns. */ -static void whereClauseInit( - WhereClause *pWC, /* The WhereClause to be initialized */ - WhereInfo *pWInfo /* The WHERE processing context */ -){ - pWC->pWInfo = pWInfo; - pWC->pOuter = 0; - pWC->nTerm = 0; - pWC->nSlot = ArraySize(pWC->aStatic); - pWC->a = pWC->aStatic; +static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ + Vdbe *v = pParse->pVdbe; + if( zAff==0 ){ + assert( pParse->db->mallocFailed ); + return; + } + assert( v!=0 ); + + /* Adjust base and n to skip over SQLITE_AFF_BLOB entries at the beginning + ** and end of the affinity string. + */ + while( n>0 && zAff[0]==SQLITE_AFF_BLOB ){ + n--; + base++; + zAff++; + } + while( n>1 && zAff[n-1]==SQLITE_AFF_BLOB ){ + n--; + } + + /* Code the OP_Affinity opcode if there is anything left to do. */ + if( n>0 ){ + sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n); + sqlite3ExprCacheAffinityChange(pParse, base, n); + } } -/* Forward reference */ -static void whereClauseClear(WhereClause*); +/* +** Expression pRight, which is the RHS of a comparison operation, is +** either a vector of n elements or, if n==1, a scalar expression. +** Before the comparison operation, affinity zAff is to be applied +** to the pRight values. This function modifies characters within the +** affinity string to SQLITE_AFF_BLOB if either: +** +** * the comparison will be performed with no affinity, or +** * the affinity change in zAff is guaranteed not to change the value. +*/ +static void updateRangeAffinityStr( + Expr *pRight, /* RHS of comparison */ + int n, /* Number of vector elements in comparison */ + char *zAff /* Affinity string to modify */ +){ + int i; + for(i=0; ipExpr; + Vdbe *v = pParse->pVdbe; + int iReg; /* Register holding results */ + + assert( pLevel->pWLoop->aLTerm[iEq]==pTerm ); + assert( iTarget>0 ); + if( pX->op==TK_EQ || pX->op==TK_IS ){ + iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); + }else if( pX->op==TK_ISNULL ){ + iReg = iTarget; + sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); +#ifndef SQLITE_OMIT_SUBQUERY + }else{ + int eType = IN_INDEX_NOOP; + int iTab; + struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + int i; + int nEq = 0; + int *aiMap = 0; + + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } + assert( pX->op==TK_IN ); + iReg = iTarget; + + for(i=0; iaLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){ + disableTerm(pLevel, pTerm); + return iTarget; + } + } + for(i=iEq;inLTerm; i++){ + if( ALWAYS(pLoop->aLTerm[i]) && pLoop->aLTerm[i]->pExpr==pX ) nEq++; + } + + if( (pX->flags & EP_xIsSelect)==0 || pX->x.pSelect->pEList->nExpr==1 ){ + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0); + }else{ + Select *pSelect = pX->x.pSelect; + sqlite3 *db = pParse->db; + u16 savedDbOptFlags = db->dbOptFlags; + ExprList *pOrigRhs = pSelect->pEList; + ExprList *pOrigLhs = pX->pLeft->x.pList; + ExprList *pRhs = 0; /* New Select.pEList for RHS */ + ExprList *pLhs = 0; /* New pX->pLeft vector */ + + for(i=iEq;inLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iField = pLoop->aLTerm[i]->iField - 1; + Expr *pNewRhs = sqlite3ExprDup(db, pOrigRhs->a[iField].pExpr, 0); + Expr *pNewLhs = sqlite3ExprDup(db, pOrigLhs->a[iField].pExpr, 0); + + pRhs = sqlite3ExprListAppend(pParse, pRhs, pNewRhs); + pLhs = sqlite3ExprListAppend(pParse, pLhs, pNewLhs); + } + } + if( !db->mallocFailed ){ + Expr *pLeft = pX->pLeft; + + if( pSelect->pOrderBy ){ + /* If the SELECT statement has an ORDER BY clause, zero the + ** iOrderByCol variables. These are set to non-zero when an + ** ORDER BY term exactly matches one of the terms of the + ** result-set. Since the result-set of the SELECT statement may + ** have been modified or reordered, these variables are no longer + ** set correctly. Since setting them is just an optimization, + ** it's easiest just to zero them here. */ + ExprList *pOrderBy = pSelect->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } + } + + /* Take care here not to generate a TK_VECTOR containing only a + ** single value. Since the parser never creates such a vector, some + ** of the subroutines do not handle this case. */ + if( pLhs->nExpr==1 ){ + pX->pLeft = pLhs->a[0].pExpr; + }else{ + pLeft->x.pList = pLhs; + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int) * nEq); + testcase( aiMap==0 ); + } + pSelect->pEList = pRhs; + db->dbOptFlags |= SQLITE_QueryFlattener; + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap); + db->dbOptFlags = savedDbOptFlags; + testcase( aiMap!=0 && aiMap[0]!=0 ); + pSelect->pEList = pOrigRhs; + pLeft->x.pList = pOrigLhs; + pX->pLeft = pLeft; + } + sqlite3ExprListDelete(pParse->db, pLhs); + sqlite3ExprListDelete(pParse->db, pRhs); + } + + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + iTab = pX->iTable; + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + } + + i = pLevel->u.in.nIn; + pLevel->u.in.nIn += nEq; + pLevel->u.in.aInLoop = + sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + int iMap = 0; /* Index in aiMap[] */ + pIn += i; + for(i=iEq;inLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iOut = iReg + i - iEq; + if( eType==IN_INDEX_ROWID ){ + testcase( nEq>1 ); /* Happens with a UNIQUE index on ROWID */ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); + }else{ + int iCol = aiMap ? aiMap[iMap++] : 0; + pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut); + } + sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); + if( i==iEq ){ + pIn->iCur = iTab; + pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; + }else{ + pIn->eEndLoopOp = OP_Noop; + } + pIn++; + } + } + }else{ + pLevel->u.in.nIn = 0; + } + sqlite3DbFree(pParse->db, aiMap); +#endif + } + disableTerm(pLevel, pTerm); + return iReg; +} + +/* +** Generate code that will evaluate all == and IN constraints for an +** index scan. +** +** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). +** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 +** The index has as many as three equality constraints, but in this +** example, the third "c" value is an inequality. So only two +** constraints are coded. This routine will generate code to evaluate +** a==5 and b IN (1,2,3). The current values for a and b will be stored +** in consecutive registers and the index of the first register is returned. +** +** In the example above nEq==2. But this subroutine works for any value +** of nEq including 0. If nEq==0, this routine is nearly a no-op. +** The only thing it does is allocate the pLevel->iMem memory cell and +** compute the affinity string. +** +** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints +** are == or IN and are covered by the nEq. nExtraReg is 1 if there is +** an inequality constraint (such as the "c>=5 AND c<10" in the example) that +** occurs after the nEq quality constraints. +** +** This routine allocates a range of nEq+nExtraReg memory cells and returns +** the index of the first memory cell in that range. The code that +** calls this routine will use that memory range to store keys for +** start and termination conditions of the loop. +** key value of the loop. If one or more IN operators appear, then +** this routine allocates an additional nEq memory cells for internal +** use. +** +** Before returning, *pzAff is set to point to a buffer containing a +** copy of the column affinity string of the index allocated using +** sqlite3DbMalloc(). Except, entries in the copy of the string associated +** with equality constraints that use BLOB or NONE affinity are set to +** SQLITE_AFF_BLOB. This is to deal with SQL such as the following: +** +** CREATE TABLE t1(a TEXT PRIMARY KEY, b); +** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; +** +** In the example above, the index on t1(a) has TEXT affinity. But since +** the right hand side of the equality constraint (t2.b) has BLOB/NONE affinity, +** no conversion should be attempted before using a t2.b value as part of +** a key to search the index. Hence the first byte in the returned affinity +** string in this example would be set to SQLITE_AFF_BLOB. +*/ +static int codeAllEqualityTerms( + Parse *pParse, /* Parsing context */ + WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ + int bRev, /* Reverse the order of IN operators */ + int nExtraReg, /* Number of extra registers to allocate */ + char **pzAff /* OUT: Set to point to affinity string */ +){ + u16 nEq; /* The number of == or IN constraints to code */ + u16 nSkip; /* Number of left-most columns to skip */ + Vdbe *v = pParse->pVdbe; /* The vm under construction */ + Index *pIdx; /* The index being used for this loop */ + WhereTerm *pTerm; /* A single constraint term */ + WhereLoop *pLoop; /* The WhereLoop object */ + int j; /* Loop counter */ + int regBase; /* Base register */ + int nReg; /* Number of registers to allocate */ + char *zAff; /* Affinity string to return */ + + /* This module is only called on query plans that use an index. */ + pLoop = pLevel->pWLoop; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + nEq = pLoop->u.btree.nEq; + nSkip = pLoop->nSkip; + pIdx = pLoop->u.btree.pIndex; + assert( pIdx!=0 ); + + /* Figure out how many memory cells we will need then allocate them. + */ + regBase = pParse->nMem + 1; + nReg = pLoop->u.btree.nEq + nExtraReg; + pParse->nMem += nReg; + + zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); + assert( zAff!=0 || pParse->db->mallocFailed ); + + if( nSkip ){ + int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); + j = sqlite3VdbeAddOp0(v, OP_Goto); + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), + iIdxCur, 0, regBase, nSkip); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + sqlite3VdbeJumpHere(v, j); + for(j=0; jaiColumn[j]==XN_EXPR ); + VdbeComment((v, "%s", explainIndexColumnName(pIdx, j))); + } + } + + /* Evaluate the equality constraints + */ + assert( zAff==0 || (int)strlen(zAff)>=nEq ); + for(j=nSkip; jaLTerm[j]; + assert( pTerm!=0 ); + /* The following testcase is true for indices with redundant columns. + ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ + testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); + if( r1!=regBase+j ){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, regBase); + regBase = r1; + }else{ + sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); + } + } + if( pTerm->eOperator & WO_IN ){ + if( pTerm->pExpr->flags & EP_xIsSelect ){ + /* No affinity ever needs to be (or should be) applied to a value + ** from the RHS of an "? IN (SELECT ...)" expression. The + ** sqlite3FindInIndex() routine has already ensured that the + ** affinity of the comparison has been applied to the value. */ + if( zAff ) zAff[j] = SQLITE_AFF_BLOB; + } + }else if( (pTerm->eOperator & WO_ISNULL)==0 ){ + Expr *pRight = pTerm->pExpr->pRight; + if( (pTerm->wtFlags & TERM_IS)==0 && sqlite3ExprCanBeNull(pRight) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); + VdbeCoverage(v); + } + if( zAff ){ + if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_BLOB ){ + zAff[j] = SQLITE_AFF_BLOB; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ + zAff[j] = SQLITE_AFF_BLOB; + } + } + } + } + *pzAff = zAff; + return regBase; +} + +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS +/* +** If the most recently coded instruction is a constant range constraint +** (a string literal) that originated from the LIKE optimization, then +** set P3 and P5 on the OP_String opcode so that the string will be cast +** to a BLOB at appropriate times. +** +** The LIKE optimization trys to evaluate "x LIKE 'abc%'" as a range +** expression: "x>='ABC' AND x<'abd'". But this requires that the range +** scan loop run twice, once for strings and a second time for BLOBs. +** The OP_String opcodes on the second pass convert the upper and lower +** bound string constants to blobs. This routine makes the necessary changes +** to the OP_String opcodes for that to happen. +** +** Except, of course, if SQLITE_LIKE_DOESNT_MATCH_BLOBS is defined, then +** only the one pass through the string space is required, so this routine +** becomes a no-op. +*/ +static void whereLikeOptimizationStringFixup( + Vdbe *v, /* prepared statement under construction */ + WhereLevel *pLevel, /* The loop that contains the LIKE operator */ + WhereTerm *pTerm /* The upper or lower bound just coded */ +){ + if( pTerm->wtFlags & TERM_LIKEOPT ){ + VdbeOp *pOp; + assert( pLevel->iLikeRepCntr>0 ); + pOp = sqlite3VdbeGetOp(v, -1); + assert( pOp!=0 ); + assert( pOp->opcode==OP_String8 + || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); + pOp->p3 = (int)(pLevel->iLikeRepCntr>>1); /* Register holding counter */ + pOp->p5 = (u8)(pLevel->iLikeRepCntr&1); /* ASC or DESC */ + } +} +#else +# define whereLikeOptimizationStringFixup(A,B,C) +#endif + +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Information is passed from codeCursorHint() down to individual nodes of +** the expression tree (by sqlite3WalkExpr()) using an instance of this +** structure. +*/ +struct CCurHint { + int iTabCur; /* Cursor for the main table */ + int iIdxCur; /* Cursor for the index, if pIdx!=0. Unused otherwise */ + Index *pIdx; /* The index used to access the table */ +}; + +/* +** This function is called for every node of an expression that is a candidate +** for a cursor hint on an index cursor. For TK_COLUMN nodes that reference +** the table CCurHint.iTabCur, verify that the same column can be +** accessed through the index. If it cannot, then set pWalker->eCode to 1. +*/ +static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){ + struct CCurHint *pHint = pWalker->u.pCCurHint; + assert( pHint->pIdx!=0 ); + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pHint->iTabCur + && sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + } + return WRC_Continue; +} + +/* +** Test whether or not expression pExpr, which was part of a WHERE clause, +** should be included in the cursor-hint for a table that is on the rhs +** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the +** expression is not suitable. +** +** An expression is unsuitable if it might evaluate to non NULL even if +** a TK_COLUMN node that does affect the value of the expression is set +** to NULL. For example: +** +** col IS NULL +** col IS NOT NULL +** coalesce(col, 1) +** CASE WHEN col THEN 0 ELSE 1 END +*/ +static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_IS + || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT + || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE + ){ + pWalker->eCode = 1; + }else if( pExpr->op==TK_FUNCTION ){ + int d1; + char d2[3]; + if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){ + pWalker->eCode = 1; + } + } + + return WRC_Continue; +} + + +/* +** This function is called on every node of an expression tree used as an +** argument to the OP_CursorHint instruction. If the node is a TK_COLUMN +** that accesses any table other than the one identified by +** CCurHint.iTabCur, then do the following: +** +** 1) allocate a register and code an OP_Column instruction to read +** the specified column into the new register, and +** +** 2) transform the expression node to a TK_REGISTER node that reads +** from the newly populated register. +** +** Also, if the node is a TK_COLUMN that does access the table idenified +** by pCCurHint.iTabCur, and an index is being used (which we will +** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into +** an access of the index rather than the original table. +*/ +static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ + int rc = WRC_Continue; + struct CCurHint *pHint = pWalker->u.pCCurHint; + if( pExpr->op==TK_COLUMN ){ + if( pExpr->iTable!=pHint->iTabCur ){ + Vdbe *v = pWalker->pParse->pVdbe; + int reg = ++pWalker->pParse->nMem; /* Register for column value */ + sqlite3ExprCodeGetColumnOfTable( + v, pExpr->pTab, pExpr->iTable, pExpr->iColumn, reg + ); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; + }else if( pHint->pIdx!=0 ){ + pExpr->iTable = pHint->iIdxCur; + pExpr->iColumn = sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn); + assert( pExpr->iColumn>=0 ); + } + }else if( pExpr->op==TK_AGG_FUNCTION ){ + /* An aggregate function in the WHERE clause of a query means this must + ** be a correlated sub-query, and expression pExpr is an aggregate from + ** the parent context. Do not walk the function arguments in this case. + ** + ** todo: It should be possible to replace this node with a TK_REGISTER + ** expression, as the result of the expression must be stored in a + ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + rc = WRC_Prune; + } + return rc; +} + +/* +** Insert an OP_CursorHint instruction if it is appropriate to do so. +*/ +static void codeCursorHint( + struct SrcList_item *pTabItem, /* FROM clause item */ + WhereInfo *pWInfo, /* The where clause */ + WhereLevel *pLevel, /* Which loop to provide hints for */ + WhereTerm *pEndRange /* Hint this end-of-scan boundary term if not NULL */ +){ + Parse *pParse = pWInfo->pParse; + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + Expr *pExpr = 0; + WhereLoop *pLoop = pLevel->pWLoop; + int iCur; + WhereClause *pWC; + WhereTerm *pTerm; + int i, j; + struct CCurHint sHint; + Walker sWalker; + + if( OptimizationDisabled(db, SQLITE_CursorHints) ) return; + iCur = pLevel->iTabCur; + assert( iCur==pWInfo->pTabList->a[pLevel->iFrom].iCursor ); + sHint.iTabCur = iCur; + sHint.iIdxCur = pLevel->iIdxCur; + sHint.pIdx = pLoop->u.btree.pIndex; + memset(&sWalker, 0, sizeof(sWalker)); + sWalker.pParse = pParse; + sWalker.u.pCCurHint = &sHint; + pWC = &pWInfo->sWC; + for(i=0; inTerm; i++){ + pTerm = &pWC->a[i]; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->prereqAll & pLevel->notReady ) continue; + + /* Any terms specified as part of the ON(...) clause for any LEFT + ** JOIN for which the current table is not the rhs are omitted + ** from the cursor-hint. + ** + ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms + ** that were specified as part of the WHERE clause must be excluded. + ** This is to address the following: + ** + ** SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL; + ** + ** Say there is a single row in t2 that matches (t1.a=t2.b), but its + ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is + ** pushed down to the cursor, this row is filtered out, causing + ** SQLite to synthesize a row of NULL values. Which does match the + ** WHERE clause, and so the query returns a row. Which is incorrect. + ** + ** For the same reason, WHERE terms such as: + ** + ** WHERE 1 = (t2.c IS NULL) + ** + ** are also excluded. See codeCursorHintIsOrFunction() for details. + */ + if( pTabItem->fg.jointype & JT_LEFT ){ + Expr *pExpr = pTerm->pExpr; + if( !ExprHasProperty(pExpr, EP_FromJoin) + || pExpr->iRightJoinTable!=pTabItem->iCursor + ){ + sWalker.eCode = 0; + sWalker.xExprCallback = codeCursorHintIsOrFunction; + sqlite3WalkExpr(&sWalker, pTerm->pExpr); + if( sWalker.eCode ) continue; + } + }else{ + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue; + } + + /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize + ** the cursor. These terms are not needed as hints for a pure range + ** scan (that has no == terms) so omit them. */ + if( pLoop->u.btree.nEq==0 && pTerm!=pEndRange ){ + for(j=0; jnLTerm && pLoop->aLTerm[j]!=pTerm; j++){} + if( jnLTerm ) continue; + } + + /* No subqueries or non-deterministic functions allowed */ + if( sqlite3ExprContainsSubquery(pTerm->pExpr) ) continue; + + /* For an index scan, make sure referenced columns are actually in + ** the index. */ + if( sHint.pIdx!=0 ){ + sWalker.eCode = 0; + sWalker.xExprCallback = codeCursorHintCheckExpr; + sqlite3WalkExpr(&sWalker, pTerm->pExpr); + if( sWalker.eCode ) continue; + } + + /* If we survive all prior tests, that means this term is worth hinting */ + pExpr = sqlite3ExprAnd(db, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); + } + if( pExpr!=0 ){ + sWalker.xExprCallback = codeCursorHintFixExpr; + sqlite3WalkExpr(&sWalker, pExpr); + sqlite3VdbeAddOp4(v, OP_CursorHint, + (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, + (const char*)pExpr, P4_EXPR); + } +} +#else +# define codeCursorHint(A,B,C,D) /* No-op */ +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + +/* +** Cursor iCur is open on an intkey b-tree (a table). Register iRowid contains +** a rowid value just read from cursor iIdxCur, open on index pIdx. This +** function generates code to do a deferred seek of cursor iCur to the +** rowid stored in register iRowid. +** +** Normally, this is just: +** +** OP_Seek $iCur $iRowid +** +** However, if the scan currently being coded is a branch of an OR-loop and +** the statement currently being coded is a SELECT, then P3 of the OP_Seek +** is set to iIdxCur and P4 is set to point to an array of integers +** containing one entry for each column of the table cursor iCur is open +** on. For each table column, if the column is the i'th column of the +** index, then the corresponding array entry is set to (i+1). If the column +** does not appear in the index at all, the array entry is set to 0. +*/ +static void codeDeferredSeek( + WhereInfo *pWInfo, /* Where clause context */ + Index *pIdx, /* Index scan is using */ + int iCur, /* Cursor for IPK b-tree */ + int iIdxCur /* Index cursor */ +){ + Parse *pParse = pWInfo->pParse; /* Parse context */ + Vdbe *v = pParse->pVdbe; /* Vdbe to generate code within */ + + assert( iIdxCur>0 ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 ); + + sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur); + if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE) + && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask) + ){ + int i; + Table *pTab = pIdx->pTable; + int *ai = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*(pTab->nCol+1)); + if( ai ){ + ai[0] = pTab->nCol; + for(i=0; inColumn-1; i++){ + assert( pIdx->aiColumn[i]nCol ); + if( pIdx->aiColumn[i]>=0 ) ai[pIdx->aiColumn[i]+1] = i+1; + } + sqlite3VdbeChangeP4(v, -1, (char*)ai, P4_INTARRAY); + } + } +} + +/* +** If the expression passed as the second argument is a vector, generate +** code to write the first nReg elements of the vector into an array +** of registers starting with iReg. +** +** If the expression is not a vector, then nReg must be passed 1. In +** this case, generate code to evaluate the expression and leave the +** result in register iReg. +*/ +static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){ + assert( nReg>0 ); + if( sqlite3ExprIsVector(p) ){ +#ifndef SQLITE_OMIT_SUBQUERY + if( (p->flags & EP_xIsSelect) ){ + Vdbe *v = pParse->pVdbe; + int iSelect = sqlite3CodeSubselect(pParse, p, 0, 0); + sqlite3VdbeAddOp3(v, OP_Copy, iSelect, iReg, nReg-1); + }else +#endif + { + int i; + ExprList *pList = p->x.pList; + assert( nReg<=pList->nExpr ); + for(i=0; ia[i].pExpr, iReg+i); + } + } + }else{ + assert( nReg==1 ); + sqlite3ExprCode(pParse, p, iReg); + } +} + +/* +** Generate code for the start of the iLevel-th loop in the WHERE clause +** implementation described by pWInfo. +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + Bitmask notReady /* Which tables are currently available */ +){ + int j, k; /* Loop counters */ + int iCur; /* The VDBE cursor for the table */ + int addrNxt; /* Where to jump to continue with the next IN case */ + int omitTable; /* True if we use the index only */ + int bRev; /* True if we need to scan in reverse order */ + WhereLevel *pLevel; /* The where level to be coded */ + WhereLoop *pLoop; /* The WhereLoop object being coded */ + WhereClause *pWC; /* Decomposition of the entire WHERE clause */ + WhereTerm *pTerm; /* A WHERE clause term */ + Parse *pParse; /* Parsing context */ + sqlite3 *db; /* Database connection */ + Vdbe *v; /* The prepared stmt under constructions */ + struct SrcList_item *pTabItem; /* FROM clause term being coded */ + int addrBrk; /* Jump here to break out of the loop */ + int addrCont; /* Jump here to continue with next cycle */ + int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ + int iReleaseReg = 0; /* Temp register to free before returning */ + + pParse = pWInfo->pParse; + v = pParse->pVdbe; + pWC = &pWInfo->sWC; + db = pParse->db; + pLevel = &pWInfo->a[iLevel]; + pLoop = pLevel->pWLoop; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + iCur = pTabItem->iCursor; + pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); + bRev = (pWInfo->revMask>>iLevel)&1; + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0; + VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); + + /* Create labels for the "break" and "continue" instructions + ** for the current loop. Jump to addrBrk to break out of a loop. + ** Jump to cont to go immediately to the next iteration of the + ** loop. + ** + ** When there is an IN operator, we also have a "addrNxt" label that + ** means to continue with the next IN value combination. When + ** there are no IN operators in the constraints, the "addrNxt" label + ** is the same as "addrBrk". + */ + addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + + /* If this is the right table of a LEFT OUTER JOIN, allocate and + ** initialize a memory cell that records if this table matches any + ** row of the left table of the join. + */ + if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ + pLevel->iLeftJoin = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); + VdbeComment((v, "init LEFT JOIN no-match flag")); + } + + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->fg.viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + pLevel->op = OP_Goto; + }else + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 1: The table is a virtual-table. Use the VFilter and VNext + ** to access the data. + */ + int iReg; /* P3 Value for OP_VFilter */ + int addrNotFound; + int nConstraint = pLoop->nLTerm; + int iIn; /* Counter for IN constraints */ + + sqlite3ExprCachePush(pParse); + iReg = sqlite3GetTempRange(pParse, nConstraint+2); + addrNotFound = pLevel->addrBrk; + for(j=0; jaLTerm[j]; + if( NEVER(pTerm==0) ) continue; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + Expr *pRight = pTerm->pExpr->pRight; + codeExprOrVector(pParse, pRight, iTarget, 1); + } + } + sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); + sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, + pLoop->u.vtab.idxStr, + pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + VdbeCoverage(v); + pLoop->u.vtab.needFree = 0; + pLevel->p1 = iCur; + pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + iIn = pLevel->u.in.nIn; + for(j=nConstraint-1; j>=0; j--){ + pTerm = pLoop->aLTerm[j]; + if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pTerm); + }else if( (pTerm->eOperator & WO_IN)!=0 ){ + Expr *pCompare; /* The comparison operator */ + Expr *pRight; /* RHS of the comparison */ + VdbeOp *pOp; /* Opcode to access the value of the IN constraint */ + + /* Reload the constraint value into reg[iReg+j+2]. The same value + ** was loaded into the same register prior to the OP_VFilter, but + ** the xFilter implementation might have changed the datatype or + ** encoding of the value in the register, so it *must* be reloaded. */ + assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed ); + if( !db->mallocFailed ){ + assert( iIn>0 ); + pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[--iIn].addrInTop); + assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid ); + assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 ); + assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 ); + testcase( pOp->opcode==OP_Rowid ); + sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); + } + + /* Generate code that will continue to the next row if + ** the IN constraint is not satisfied */ + pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0, 0); + assert( pCompare!=0 || db->mallocFailed ); + if( pCompare ){ + pCompare->pLeft = pTerm->pExpr->pLeft; + pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0); + if( pRight ){ + pRight->iTable = iReg+j+2; + sqlite3ExprIfFalse(pParse, pCompare, pLevel->addrCont, 0); + } + pCompare->pLeft = 0; + sqlite3ExprDelete(db, pCompare); + } + } + } + /* These registers need to be preserved in case there is an IN operator + ** loop. So we could deallocate the registers here (and potentially + ** reuse them later) if (pLoop->wsFlags & WHERE_IN_ABLE)==0. But it seems + ** simpler and safer to simply not reuse the registers. + ** + ** sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); + */ + sqlite3ExprCachePop(pParse); + }else +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 + ){ + /* Case 2: We can directly reference a single row using an + ** equality comparison against the ROWID field. Or + ** we reference multiple rows using a "rowid IN (...)" + ** construct. + */ + assert( pLoop->u.btree.nEq==1 ); + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + iReleaseReg = ++pParse->nMem; + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); + addrNxt = pLevel->addrNxt; + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg); + VdbeCoverage(v); + sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + VdbeComment((v, "pk")); + pLevel->op = OP_Noop; + }else if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 + ){ + /* Case 3: We have an inequality comparison against the ROWID field. + */ + int testOp = OP_Noop; + int start; + int memEndValue = 0; + WhereTerm *pStart, *pEnd; + + assert( omitTable==0 ); + j = 0; + pStart = pEnd = 0; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; + assert( pStart!=0 || pEnd!=0 ); + if( bRev ){ + pTerm = pStart; + pStart = pEnd; + pEnd = pTerm; + } + codeCursorHint(pTabItem, pWInfo, pLevel, pEnd); + if( pStart ){ + Expr *pX; /* The expression that defines the start bound */ + int r1, rTemp; /* Registers for holding the start boundary */ + int op; /* Cursor seek operation */ + + /* The following constant maps TK_xx codes into corresponding + ** seek opcodes. It depends on a particular ordering of TK_xx + */ + const u8 aMoveOp[] = { + /* TK_GT */ OP_SeekGT, + /* TK_LE */ OP_SeekLE, + /* TK_LT */ OP_SeekLT, + /* TK_GE */ OP_SeekGE + }; + assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ + assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ + assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + + assert( (pStart->wtFlags & TERM_VNULL)==0 ); + testcase( pStart->wtFlags & TERM_VIRTUAL ); + pX = pStart->pExpr; + assert( pX!=0 ); + testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ + if( sqlite3ExprIsVector(pX->pRight) ){ + r1 = rTemp = sqlite3GetTempReg(pParse); + codeExprOrVector(pParse, pX->pRight, r1, 1); + op = aMoveOp[(pX->op - TK_GT) | 0x0001]; + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); + disableTerm(pLevel, pStart); + op = aMoveOp[(pX->op - TK_GT)]; + } + sqlite3VdbeAddOp3(v, op, iCur, addrBrk, r1); + VdbeComment((v, "pk")); + VdbeCoverageIf(v, pX->op==TK_GT); + VdbeCoverageIf(v, pX->op==TK_LE); + VdbeCoverageIf(v, pX->op==TK_LT); + VdbeCoverageIf(v, pX->op==TK_GE); + sqlite3ExprCacheAffinityChange(pParse, r1, 1); + sqlite3ReleaseTempReg(pParse, rTemp); + }else{ + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + } + if( pEnd ){ + Expr *pX; + pX = pEnd->pExpr; + assert( pX!=0 ); + assert( (pEnd->wtFlags & TERM_VNULL)==0 ); + testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ + testcase( pEnd->wtFlags & TERM_VIRTUAL ); + memEndValue = ++pParse->nMem; + codeExprOrVector(pParse, pX->pRight, memEndValue, 1); + if( 0==sqlite3ExprIsVector(pX->pRight) + && (pX->op==TK_LT || pX->op==TK_GT) + ){ + testOp = bRev ? OP_Le : OP_Ge; + }else{ + testOp = bRev ? OP_Lt : OP_Gt; + } + if( 0==sqlite3ExprIsVector(pX->pRight) ){ + disableTerm(pLevel, pEnd); + } + } + start = sqlite3VdbeCurrentAddr(v); + pLevel->op = bRev ? OP_Prev : OP_Next; + pLevel->p1 = iCur; + pLevel->p2 = start; + assert( pLevel->p5==0 ); + if( testOp!=OP_Noop ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + VdbeCoverageIf(v, testOp==OP_Le); + VdbeCoverageIf(v, testOp==OP_Lt); + VdbeCoverageIf(v, testOp==OP_Ge); + VdbeCoverageIf(v, testOp==OP_Gt); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); + } + }else if( pLoop->wsFlags & WHERE_INDEXED ){ + /* Case 4: A scan using an index. + ** + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==" and "IN" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: + ** + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 + ** + ** The z<10 term of the following cannot be used, only + ** the x=5 term: + ** + ** x=5 AND z<10 + ** + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. + ** + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. + */ + static const u8 aStartOp[] = { + 0, + 0, + OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ + OP_Last, /* 3: (!start_constraints && startEq && bRev) */ + OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ + OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ + OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ + OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ + }; + static const u8 aEndOp[] = { + OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ + OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ + OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ + OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ + }; + u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + u16 nBtm = pLoop->u.btree.nBtm; /* Length of BTM vector */ + u16 nTop = pLoop->u.btree.nTop; /* Length of TOP vector */ + int regBase; /* Base register holding constraint values */ + WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ + WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ + int startEq; /* True if range start uses ==, >= or <= */ + int endEq; /* True if range end uses ==, >= or <= */ + int start_constraints; /* Start of range is constrained */ + int nConstraint; /* Number of constraint terms */ + Index *pIdx; /* The index we will be using */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ + char *zStartAff; /* Affinity for start of range constraint */ + char *zEndAff = 0; /* Affinity for end of range constraint */ + u8 bSeekPastNull = 0; /* True to seek past initial nulls */ + u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ + + pIdx = pLoop->u.btree.pIndex; + iIdxCur = pLevel->iIdxCur; + assert( nEq>=pLoop->nSkip ); + + /* If this loop satisfies a sort order (pOrderBy) request that + ** was passed to this function to implement a "SELECT min(x) ..." + ** query, then the caller will only allow the loop to run for + ** a single iteration. This means that the first row returned + ** should not have a NULL value stored in 'x'. If column 'x' is + ** the first one after the nEq equality constraints in the index, + ** this requires some special handling. + */ + assert( pWInfo->pOrderBy==0 + || pWInfo->pOrderBy->nExpr==1 + || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); + if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 + && pWInfo->nOBSat>0 + && (pIdx->nKeyCol>nEq) + ){ + assert( pLoop->nSkip==0 ); + bSeekPastNull = 1; + nExtraReg = 1; + } + + /* Find any inequality constraint terms for the start and end + ** of the range. + */ + j = nEq; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = pLoop->aLTerm[j++]; + nExtraReg = MAX(nExtraReg, pLoop->u.btree.nBtm); + /* Like optimization range constraints always occur in pairs */ + assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || + (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 ); + } + if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = pLoop->aLTerm[j++]; + nExtraReg = MAX(nExtraReg, pLoop->u.btree.nTop); +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){ + assert( pRangeStart!=0 ); /* LIKE opt constraints */ + assert( pRangeStart->wtFlags & TERM_LIKEOPT ); /* occur in pairs */ + pLevel->iLikeRepCntr = (u32)++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 1, (int)pLevel->iLikeRepCntr); + VdbeComment((v, "LIKE loop counter")); + pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); + /* iLikeRepCntr actually stores 2x the counter register number. The + ** bottom bit indicates whether the search order is ASC or DESC. */ + testcase( bRev ); + testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC ); + assert( (bRev & ~1)==0 ); + pLevel->iLikeRepCntr <<=1; + pLevel->iLikeRepCntr |= bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC); + } +#endif + if( pRangeStart==0 ){ + j = pIdx->aiColumn[nEq]; + if( (j>=0 && pIdx->pTable->aCol[j].notNull==0) || j==XN_EXPR ){ + bSeekPastNull = 1; + } + } + } + assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); + + /* If we are doing a reverse order scan on an ascending index, or + ** a forward order scan on a descending index, interchange the + ** start and end terms (pRangeStart and pRangeEnd). + */ + if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) + || (bRev && pIdx->nKeyCol==nEq) + ){ + SWAP(WhereTerm *, pRangeEnd, pRangeStart); + SWAP(u8, bSeekPastNull, bStopAtNull); + SWAP(u8, nBtm, nTop); + } + + /* Generate code to evaluate all constraint terms using == or IN + ** and store the values of those terms in an array of registers + ** starting at regBase. + */ + codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd); + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); + assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); + if( zStartAff && nTop ){ + zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]); + } + addrNxt = pLevel->addrNxt; + + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); + startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); + endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); + start_constraints = pRangeStart || nEq>0; + + /* Seek the index cursor to the start of the range. */ + nConstraint = nEq; + if( pRangeStart ){ + Expr *pRight = pRangeStart->pExpr->pRight; + codeExprOrVector(pParse, pRight, regBase+nEq, nBtm); + whereLikeOptimizationStringFixup(v, pLevel, pRangeStart); + if( (pRangeStart->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); + } + if( zStartAff ){ + updateRangeAffinityStr(pRight, nBtm, &zStartAff[nEq]); + } + nConstraint += nBtm; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); + if( sqlite3ExprIsVector(pRight)==0 ){ + disableTerm(pLevel, pRangeStart); + }else{ + startEq = 1; + } + bSeekPastNull = 0; + }else if( bSeekPastNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + nConstraint++; + startEq = 0; + start_constraints = 1; + } + codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); + if( pLoop->nSkip>0 && nConstraint==pLoop->nSkip ){ + /* The skip-scan logic inside the call to codeAllEqualityConstraints() + ** above has already left the cursor sitting on the correct row, + ** so no further seeking is needed */ + }else{ + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; + assert( op!=0 ); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + } + + /* Load the value for the inequality constraint at the end of the + ** range (if any). + */ + nConstraint = nEq; + if( pRangeEnd ){ + Expr *pRight = pRangeEnd->pExpr->pRight; + sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); + codeExprOrVector(pParse, pRight, regBase+nEq, nTop); + whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); + if( (pRangeEnd->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); + } + if( zEndAff ){ + updateRangeAffinityStr(pRight, nTop, zEndAff); + codeApplyAffinity(pParse, regBase+nEq, nTop, zEndAff); + }else{ + assert( pParse->db->mallocFailed ); + } + nConstraint += nTop; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + + if( sqlite3ExprIsVector(pRight)==0 ){ + disableTerm(pLevel, pRangeEnd); + }else{ + endEq = 1; + } + }else if( bStopAtNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + nConstraint++; + } + sqlite3DbFree(db, zStartAff); + sqlite3DbFree(db, zEndAff); + + /* Top of the loop body */ + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + + /* Check if the index cursor is past the end of the range. */ + if( nConstraint ){ + op = aEndOp[bRev*2 + endEq]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); + } + + /* Seek the table cursor, if required */ + if( omitTable ){ + /* pIdx is a covering index. No need to access the main table. */ + }else if( HasRowid(pIdx->pTable) ){ + if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE)!=0 ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg); + VdbeCoverage(v); + }else{ + codeDeferredSeek(pWInfo, pIdx, iCur, iIdxCur); + } + }else if( iCur!=iIdxCur ){ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); + } + sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, + iRowidReg, pPk->nKeyCol); VdbeCoverage(v); + } + + /* Record the instruction used to terminate the loop. */ + if( pLoop->wsFlags & WHERE_ONEROW ){ + pLevel->op = OP_Noop; + }else if( bRev ){ + pLevel->op = OP_Prev; + }else{ + pLevel->op = OP_Next; + } + pLevel->p1 = iIdxCur; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; + if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } + }else + +#ifndef SQLITE_OMIT_OR_OPTIMIZATION + if( pLoop->wsFlags & WHERE_MULTI_OR ){ + /* Case 5: Two or more separately indexed terms connected by OR + ** + ** Example: + ** + ** CREATE TABLE t1(a,b,c,d); + ** CREATE INDEX i1 ON t1(a); + ** CREATE INDEX i2 ON t1(b); + ** CREATE INDEX i3 ON t1(c); + ** + ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) + ** + ** In the example, there are three indexed terms connected by OR. + ** The top of the loop looks like this: + ** + ** Null 1 # Zero the rowset in reg 1 + ** + ** Then, for each indexed term, the following. The arguments to + ** RowSetTest are such that the rowid of the current row is inserted + ** into the RowSet. If it is already present, control skips the + ** Gosub opcode and jumps straight to the code generated by WhereEnd(). + ** + ** sqlite3WhereBegin() + ** RowSetTest # Insert rowid into rowset + ** Gosub 2 A + ** sqlite3WhereEnd() + ** + ** Following the above, code to terminate the loop. Label A, the target + ** of the Gosub above, jumps to the instruction right after the Goto. + ** + ** Null 1 # Zero the rowset in reg 1 + ** Goto B # The loop is finished. + ** + ** A: # Return data, whatever. + ** + ** Return 2 # Jump back to the Gosub + ** + ** B: + ** + ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then + ** use an ephemeral index instead of a RowSet to record the primary + ** keys of the rows we have already seen. + ** + */ + WhereClause *pOrWc; /* The OR-clause broken out into subterms */ + SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + Index *pCov = 0; /* Potential covering index (or NULL) */ + int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ + + int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ + int regRowset = 0; /* Register for RowSet object */ + int regRowid = 0; /* Register holding rowid */ + int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ + int iRetInit; /* Address of regReturn init */ + int untestedTerms = 0; /* Some terms not completely tested */ + int ii; /* Loop counter */ + u16 wctrlFlags; /* Flags for sub-WHERE clause */ + Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ + Table *pTab = pTabItem->pTab; + + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->eOperator & WO_OR ); + assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); + pOrWc = &pTerm->u.pOrInfo->wc; + pLevel->op = OP_Return; + pLevel->p1 = regReturn; + + /* Set up a new SrcList in pOrTab containing the table being scanned + ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. + ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). + */ + if( pWInfo->nLevel>1 ){ + int nNotReady; /* The number of notReady tables */ + struct SrcList_item *origSrc; /* Original list of tables */ + nNotReady = pWInfo->nLevel - iLevel - 1; + pOrTab = sqlite3StackAllocRaw(db, + sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + if( pOrTab==0 ) return notReady; + pOrTab->nAlloc = (u8)(nNotReady + 1); + pOrTab->nSrc = pOrTab->nAlloc; + memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); + origSrc = pWInfo->pTabList->a; + for(k=1; k<=nNotReady; k++){ + memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); + } + }else{ + pOrTab = pWInfo->pTabList; + } + + /* Initialize the rowset register to contain NULL. An SQL NULL is + ** equivalent to an empty rowset. Or, create an ephemeral index + ** capable of holding primary keys in the case of a WITHOUT ROWID. + ** + ** Also initialize regReturn to contain the address of the instruction + ** immediately following the OP_Return at the bottom of the loop. This + ** is required in a few obscure LEFT JOIN cases where control jumps + ** over the top of the loop into the body of it. In this case the + ** correct response for the end-of-loop code (the OP_Return) is to + ** fall through to the next instruction, just as an OP_Next does if + ** called on an uninitialized cursor. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( HasRowid(pTab) ){ + regRowset = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + regRowset = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + regRowid = ++pParse->nMem; + } + iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); + + /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y + ** Then for every term xN, evaluate as the subexpression: xN AND z + ** That way, terms in y that are factored into the disjunction will + ** be picked up by the recursive calls to sqlite3WhereBegin() below. + ** + ** Actually, each subexpression is converted to "xN AND w" where w is + ** the "interesting" terms of z - terms that did not originate in the + ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 + */ + if( pWC->nTerm>1 ){ + int iTerm; + for(iTerm=0; iTermnTerm; iTerm++){ + Expr *pExpr = pWC->a[iTerm].pExpr; + if( &pWC->a[iTerm] == pTerm ) continue; + if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); + testcase( pWC->a[iTerm].wtFlags & TERM_CODED ); + if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED))!=0 ) continue; + if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + pExpr = sqlite3ExprDup(db, pExpr, 0); + pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); + } + if( pAndExpr ){ + pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr, 0); + } + } + + /* Run a separate WHERE clause for each term of the OR clause. After + ** eliminating duplicates from other WHERE clauses, the action for each + ** sub-WHERE clause is to to invoke the main loop body as a subroutine. + */ + wctrlFlags = WHERE_OR_SUBCLAUSE | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE); + for(ii=0; iinTerm; ii++){ + WhereTerm *pOrTerm = &pOrWc->a[ii]; + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ + WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ + int jmp1 = 0; /* Address of jump operation */ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ + pAndExpr->pLeft = pOrExpr; + pOrExpr = pAndExpr; + } + /* Loop through table entries that match term pOrTerm. */ + WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + wctrlFlags, iCovCur); + assert( pSubWInfo || pParse->nErr || db->mallocFailed ); + if( pSubWInfo ){ + WhereLoop *pSubLoop; + int addrExplain = sqlite3WhereExplainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); + sqlite3WhereAddScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); + + /* This is the sub-WHERE clause body. First skip over + ** duplicate rows from prior sub-WHERE clauses, and record the + ** rowid (or PRIMARY KEY) for the current row so that the same + ** row will be skipped in subsequent sub-WHERE clauses. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + int r; + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + if( HasRowid(pTab) ){ + r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, + r,iSet); + VdbeCoverage(v); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk; + + /* Read the PK into an array of temp registers. */ + r = sqlite3GetTempRange(pParse, nPk); + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumnToReg(pParse, pTab, iCol, iCur, r+iPk); + } + + /* Check if the temp table already contains this key. If so, + ** the row has already been included in the result set and + ** can be ignored (by jumping past the Gosub below). Otherwise, + ** insert the key into the temp table and proceed with processing + ** the row. + ** + ** Use some of the same optimizations as OP_RowSetTest: If iSet + ** is zero, assume that the key cannot already be present in + ** the temp table. And if iSet is -1, assume that there is no + ** need to insert the key into the temp table, as it will never + ** be tested for. */ + if( iSet ){ + jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + VdbeCoverage(v); + } + if( iSet>=0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); + sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); + if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + + /* Release the array of temp registers */ + sqlite3ReleaseTempRange(pParse, r, nPk); + } + } + + /* Invoke the main loop body as a subroutine */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + + /* Jump here (skipping the main loop body subroutine) if the + ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ + if( jmp1 ) sqlite3VdbeJumpHere(v, jmp1); + + /* The pSubWInfo->untestedTerms flag means that this OR term + ** contained one or more AND term from a notReady table. The + ** terms from the notReady table could not be tested and will + ** need to be tested later. + */ + if( pSubWInfo->untestedTerms ) untestedTerms = 1; + + /* If all of the OR-connected terms are optimized using the same + ** index, and the index is opened using the same cursor number + ** by each call to sqlite3WhereBegin() made by this loop, it may + ** be possible to use that index as a covering index. + ** + ** If the call to sqlite3WhereBegin() above resulted in a scan that + ** uses an index, and this is either the first OR-connected term + ** processed or the index is the same as that used by all previous + ** terms, set pCov to the candidate covering index. Otherwise, set + ** pCov to NULL to indicate that no candidate covering index will + ** be available. + */ + pSubLoop = pSubWInfo->a[0].pWLoop; + assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 + && (ii==0 || pSubLoop->u.btree.pIndex==pCov) + && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) + ){ + assert( pSubWInfo->a[0].iIdxCur==iCovCur ); + pCov = pSubLoop->u.btree.pIndex; + }else{ + pCov = 0; + } + + /* Finish the loop through table entries that match term pOrTerm. */ + sqlite3WhereEnd(pSubWInfo); + } + } + } + pLevel->u.pCovidx = pCov; + if( pCov ) pLevel->iIdxCur = iCovCur; + if( pAndExpr ){ + pAndExpr->pLeft = 0; + sqlite3ExprDelete(db, pAndExpr); + } + sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeGoto(v, pLevel->addrBrk); + sqlite3VdbeResolveLabel(v, iLoopBody); + + if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); + if( !untestedTerms ) disableTerm(pLevel, pTerm); + }else +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + + { + /* Case 6: There is no usable index. We must do a complete + ** scan of the entire table. + */ + static const u8 aStep[] = { OP_Next, OP_Prev }; + static const u8 aStart[] = { OP_Rewind, OP_Last }; + assert( bRev==0 || bRev==1 ); + if( pTabItem->fg.isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + codeCursorHint(pTabItem, pWInfo, pLevel, 0); + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } + } + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); +#endif + + /* Insert code to test every subexpression that can be completely + ** computed using the current set of tables. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + int skipLikeAddr = 0; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + continue; + } + if( pTerm->wtFlags & TERM_LIKECOND ){ + /* If the TERM_LIKECOND flag is set, that means that the range search + ** is sufficient to guarantee that the LIKE operator is true, so we + ** can skip the call to the like(A,B) function. But this only works + ** for strings. So do not skip the call to the function on the pass + ** that compares BLOBs. */ +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + continue; +#else + u32 x = pLevel->iLikeRepCntr; + assert( x>0 ); + skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)? OP_IfNot : OP_If, (int)(x>>1)); + VdbeCoverage(v); +#endif + } + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); + pTerm->wtFlags |= TERM_CODED; + } + + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE, sEAlt; + WhereTerm *pAlt; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue; + if( (pTerm->eOperator & WO_EQUIV)==0 ) continue; + if( pTerm->leftCursor!=iCur ) continue; + if( pLevel->iLeftJoin ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); + pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.leftColumn, notReady, + WO_EQ|WO_IN|WO_IS, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + testcase( pAlt->eOperator & WO_EQ ); + testcase( pAlt->eOperator & WO_IS ); + testcase( pAlt->eOperator & WO_IN ); + VdbeModuleComment((v, "begin transitive constraint")); + sEAlt = *pAlt->pExpr; + sEAlt.pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, &sEAlt, addrCont, SQLITE_JUMPIFNULL); + } + + /* For a LEFT OUTER JOIN, generate code that will record the fact that + ** at least one row of the right table has matched the left table. + */ + if( pLevel->iLeftJoin ){ + pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); + VdbeComment((v, "record LEFT JOIN hit")); + sqlite3ExprCacheClear(pParse); + for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + assert( pWInfo->untestedTerms ); + continue; + } + assert( pTerm->pExpr ); + sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); + pTerm->wtFlags |= TERM_CODED; + } + } + + return pLevel->notReady; +} + +/************** End of wherecode.c *******************************************/ +/************** Begin file whereexpr.c ***************************************/ +/* +** 2015-06-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. +** +** This file was originally part of where.c but was split out to improve +** readability and editabiliity. This file contains utility routines for +** analyzing Expr objects in the WHERE clause. +*/ +/* #include "sqliteInt.h" */ +/* #include "whereInt.h" */ + +/* Forward declarations */ +static void exprAnalyze(SrcList*, WhereClause*, int); /* ** Deallocate all memory associated with a WhereOrInfo object. */ static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ - whereClauseClear(&p->wc); + sqlite3WhereClauseClear(&p->wc); sqlite3DbFree(db, p); } @@ -113989,33 +129529,10 @@ static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ ** Deallocate all memory associated with a WhereAndInfo object. */ static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ - whereClauseClear(&p->wc); + sqlite3WhereClauseClear(&p->wc); sqlite3DbFree(db, p); } -/* -** Deallocate a WhereClause structure. The WhereClause structure -** itself is not freed. This routine is the inverse of whereClauseInit(). -*/ -static void whereClauseClear(WhereClause *pWC){ - int i; - WhereTerm *a; - sqlite3 *db = pWC->pWInfo->pParse->db; - for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ - if( a->wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, a->pExpr); - } - if( a->wtFlags & TERM_ORINFO ){ - whereOrInfoDelete(db, a->u.pOrInfo); - }else if( a->wtFlags & TERM_ANDINFO ){ - whereAndInfoDelete(db, a->u.pAndInfo); - } - } - if( pWC->a!=pWC->aStatic ){ - sqlite3DbFree(db, pWC->a); - } -} - /* ** Add a single new WhereTerm entry to the WhereClause object pWC. ** The new WhereTerm object is constructed from Expr p and with wtFlags. @@ -114035,14 +129552,14 @@ static void whereClauseClear(WhereClause *pWC){ ** calling this routine. Such pointers may be reinitialized by referencing ** the pWC->a[] array. */ -static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ +static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ WhereTerm *pTerm; int idx; testcase( wtFlags & TERM_VIRTUAL ); if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; sqlite3 *db = pWC->pWInfo->pParse->db; - pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + pWC->a = sqlite3DbMallocRawNN(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); if( pWC->a==0 ){ if( wtFlags & TERM_DYNAMIC ){ sqlite3ExprDelete(db, p); @@ -114058,7 +129575,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ pTerm->truthProb = 1; } @@ -114066,142 +129583,24 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ pTerm->wtFlags = wtFlags; pTerm->pWC = pWC; pTerm->iParent = -1; + memset(&pTerm->eOperator, 0, + sizeof(WhereTerm) - offsetof(WhereTerm,eOperator)); return idx; } -/* -** This routine identifies subexpressions in the WHERE clause where -** each subexpression is separated by the AND operator or some other -** operator specified in the op parameter. The WhereClause structure -** is filled with pointers to subexpressions. For example: -** -** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) -** \________/ \_______________/ \________________/ -** slot[0] slot[1] slot[2] -** -** The original WHERE clause in pExpr is unaltered. All this routine -** does is make slot[] entries point to substructure within pExpr. -** -** In the previous sentence and in the diagram, "slot[]" refers to -** the WhereClause.a[] array. The slot[] array grows as needed to contain -** all terms of the WHERE clause. -*/ -static void whereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ - pWC->op = op; - if( pExpr==0 ) return; - if( pExpr->op!=op ){ - whereClauseInsert(pWC, pExpr, 0); - }else{ - whereSplit(pWC, pExpr->pLeft, op); - whereSplit(pWC, pExpr->pRight, op); - } -} - -/* -** Initialize a WhereMaskSet object -*/ -#define initMaskSet(P) (P)->n=0 - -/* -** Return the bitmask for the given cursor number. Return 0 if -** iCursor is not in the set. -*/ -static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ - int i; - assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); - for(i=0; in; i++){ - if( pMaskSet->ix[i]==iCursor ){ - return MASKBIT(i); - } - } - return 0; -} - -/* -** Create a new mask for cursor iCursor. -** -** There is one cursor per table in the FROM clause. The number of -** tables in the FROM clause is limited by a test early in the -** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] -** array will never overflow. -*/ -static void createMask(WhereMaskSet *pMaskSet, int iCursor){ - assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); - pMaskSet->ix[pMaskSet->n++] = iCursor; -} - -/* -** These routines walk (recursively) an expression tree and generate -** a bitmask indicating which tables are used in that expression -** tree. -*/ -static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*); -static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*); -static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){ - Bitmask mask = 0; - if( p==0 ) return 0; - if( p->op==TK_COLUMN ){ - mask = getMask(pMaskSet, p->iTable); - return mask; - } - mask = exprTableUsage(pMaskSet, p->pRight); - mask |= exprTableUsage(pMaskSet, p->pLeft); - if( ExprHasProperty(p, EP_xIsSelect) ){ - mask |= exprSelectTableUsage(pMaskSet, p->x.pSelect); - }else{ - mask |= exprListTableUsage(pMaskSet, p->x.pList); - } - return mask; -} -static Bitmask exprListTableUsage(WhereMaskSet *pMaskSet, ExprList *pList){ - int i; - Bitmask mask = 0; - if( pList ){ - for(i=0; inExpr; i++){ - mask |= exprTableUsage(pMaskSet, pList->a[i].pExpr); - } - } - return mask; -} -static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ - Bitmask mask = 0; - while( pS ){ - SrcList *pSrc = pS->pSrc; - mask |= exprListTableUsage(pMaskSet, pS->pEList); - mask |= exprListTableUsage(pMaskSet, pS->pGroupBy); - mask |= exprListTableUsage(pMaskSet, pS->pOrderBy); - mask |= exprTableUsage(pMaskSet, pS->pWhere); - mask |= exprTableUsage(pMaskSet, pS->pHaving); - if( ALWAYS(pSrc!=0) ){ - int i; - for(i=0; inSrc; i++){ - mask |= exprSelectTableUsage(pMaskSet, pSrc->a[i].pSelect); - mask |= exprTableUsage(pMaskSet, pSrc->a[i].pOn); - } - } - pS = pS->pPrior; - } - return mask; -} - /* ** Return TRUE if the given operator is one of the operators that is ** allowed for an indexable WHERE clause term. The allowed operators are -** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" +** "=", "<", ">", "<=", ">=", "IN", "IS", and "IS NULL" */ static int allowedOp(int op){ assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL; + return op==TK_IN || (op>=TK_EQ && op<=TK_GE) || op==TK_ISNULL || op==TK_IS; } -/* -** Swap two objects of type TYPE. -*/ -#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} - /* ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". @@ -114252,6 +129651,8 @@ static u16 operatorMask(int op){ c = WO_IN; }else if( op==TK_ISNULL ){ c = WO_ISNULL; + }else if( op==TK_IS ){ + c = WO_IS; }else{ assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); c = (u16)(WO_EQ<<(op-TK_EQ)); @@ -114263,199 +129664,10 @@ static u16 operatorMask(int op){ assert( op!=TK_LE || c==WO_LE ); assert( op!=TK_GT || c==WO_GT ); assert( op!=TK_GE || c==WO_GE ); + assert( op!=TK_IS || c==WO_IS ); return c; } -/* -** Advance to the next WhereTerm that matches according to the criteria -** established when the pScan object was initialized by whereScanInit(). -** Return NULL if there are no more matching WhereTerms. -*/ -static WhereTerm *whereScanNext(WhereScan *pScan){ - int iCur; /* The cursor on the LHS of the term */ - int iColumn; /* The column on the LHS of the term. -1 for IPK */ - Expr *pX; /* An expression being tested */ - WhereClause *pWC; /* Shorthand for pScan->pWC */ - WhereTerm *pTerm; /* The term being tested */ - int k = pScan->k; /* Where to start scanning */ - - while( pScan->iEquiv<=pScan->nEquiv ){ - iCur = pScan->aEquiv[pScan->iEquiv-2]; - iColumn = pScan->aEquiv[pScan->iEquiv-1]; - while( (pWC = pScan->pWC)!=0 ){ - for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ - if( pTerm->leftCursor==iCur - && pTerm->u.leftColumn==iColumn - && (pScan->iEquiv<=2 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) - ){ - if( (pTerm->eOperator & WO_EQUIV)!=0 - && pScan->nEquivaEquiv) - ){ - int j; - pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); - assert( pX->op==TK_COLUMN ); - for(j=0; jnEquiv; j+=2){ - if( pScan->aEquiv[j]==pX->iTable - && pScan->aEquiv[j+1]==pX->iColumn ){ - break; - } - } - if( j==pScan->nEquiv ){ - pScan->aEquiv[j] = pX->iTable; - pScan->aEquiv[j+1] = pX->iColumn; - pScan->nEquiv += 2; - } - } - if( (pTerm->eOperator & pScan->opMask)!=0 ){ - /* Verify the affinity and collating sequence match */ - if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; - Parse *pParse = pWC->pWInfo->pParse; - pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; - } - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, - pX->pLeft, pX->pRight); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ - continue; - } - } - if( (pTerm->eOperator & WO_EQ)!=0 - && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN - && pX->iTable==pScan->aEquiv[0] - && pX->iColumn==pScan->aEquiv[1] - ){ - continue; - } - pScan->k = k+1; - return pTerm; - } - } - } - pScan->pWC = pScan->pWC->pOuter; - k = 0; - } - pScan->pWC = pScan->pOrigWC; - k = 0; - pScan->iEquiv += 2; - } - return 0; -} - -/* -** Initialize a WHERE clause scanner object. Return a pointer to the -** first match. Return NULL if there are no matches. -** -** The scanner will be searching the WHERE clause pWC. It will look -** for terms of the form "X " where X is column iColumn of table -** iCur. The must be one of the operators described by opMask. -** -** If the search is for X and the WHERE clause contains terms of the -** form X=Y then this routine might also return terms of the form -** "Y ". The number of levels of transitivity is limited, -** but is enough to handle most commonly occurring SQL statements. -** -** If X is not the INTEGER PRIMARY KEY then X must be compatible with -** index pIdx. -*/ -static WhereTerm *whereScanInit( - WhereScan *pScan, /* The WhereScan object being initialized */ - WhereClause *pWC, /* The WHERE clause to be scanned */ - int iCur, /* Cursor to scan for */ - int iColumn, /* Column to scan for */ - u32 opMask, /* Operator(s) to scan for */ - Index *pIdx /* Must be compatible with this index */ -){ - int j; - - /* memset(pScan, 0, sizeof(*pScan)); */ - pScan->pOrigWC = pWC; - pScan->pWC = pWC; - if( pIdx && iColumn>=0 ){ - pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>pIdx->nColumn) ) return 0; - } - pScan->zCollName = pIdx->azColl[j]; - }else{ - pScan->idxaff = 0; - pScan->zCollName = 0; - } - pScan->opMask = opMask; - pScan->k = 0; - pScan->aEquiv[0] = iCur; - pScan->aEquiv[1] = iColumn; - pScan->nEquiv = 2; - pScan->iEquiv = 2; - return whereScanNext(pScan); -} - -/* -** Search for a term in the WHERE clause that is of the form "X " -** where X is a reference to the iColumn of table iCur and is one of -** the WO_xx operator codes specified by the op parameter. -** Return a pointer to the term. Return 0 if not found. -** -** The term returned might by Y= if there is another constraint in -** the WHERE clause that specifies that X=Y. Any such constraints will be -** identified by the WO_EQUIV bit in the pTerm->eOperator field. The -** aEquiv[] array holds X and all its equivalents, with each SQL variable -** taking up two slots in aEquiv[]. The first slot is for the cursor number -** and the second is for the column number. There are 22 slots in aEquiv[] -** so that means we can look for X plus up to 10 other equivalent values. -** Hence a search for X will return if X=A1 and A1=A2 and A2=A3 -** and ... and A9=A10 and A10=. -** -** If there are multiple terms in the WHERE clause of the form "X " -** then try for the one with no dependencies on - in other words where -** is a constant expression of some kind. Only return entries of -** the form "X Y" where Y is a column in another table if no terms of -** the form "X " exist. If no terms with a constant RHS -** exist, try to return a term that does not use WO_EQUIV. -*/ -static WhereTerm *findTerm( - WhereClause *pWC, /* The WHERE clause to be searched */ - int iCur, /* Cursor number of LHS */ - int iColumn, /* Column number of LHS */ - Bitmask notReady, /* RHS must not overlap with this mask */ - u32 op, /* Mask of WO_xx values describing operator */ - Index *pIdx /* Must be compatible with this index, if not NULL */ -){ - WhereTerm *pResult = 0; - WhereTerm *p; - WhereScan scan; - - p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); - while( p ){ - if( (p->prereqRight & notReady)==0 ){ - if( p->prereqRight==0 && (p->eOperator&WO_EQ)!=0 ){ - return p; - } - if( pResult==0 ) pResult = p; - } - p = whereScanNext(&scan); - } - return pResult; -} - -/* Forward reference */ -static void exprAnalyze(SrcList*, WhereClause*, int); - -/* -** Call exprAnalyze on all terms in a WHERE clause. -*/ -static void exprAnalyzeAll( - SrcList *pTabList, /* the FROM clause */ - WhereClause *pWC /* the WHERE clause to be analyzed */ -){ - int i; - for(i=pWC->nTerm-1; i>=0; i--){ - exprAnalyze(pTabList, pWC, i); - } -} #ifndef SQLITE_OMIT_LIKE_OPTIMIZATION /* @@ -114464,7 +129676,11 @@ static void exprAnalyzeAll( ** so and false if not. ** ** In order for the operator to be optimizible, the RHS must be a string -** literal that does not begin with a wildcard. +** literal that does not begin with a wildcard. The LHS must be a column +** that may only be NULL, a string, or a BLOB, never a number. (This means +** that virtual tables cannot participate in the LIKE optimization.) The +** collating sequence for the column on the LHS must be appropriate for +** the operator. */ static int isLikeOrGlob( Parse *pParse, /* Parsing and code generating context */ @@ -114482,6 +129698,7 @@ static int isLikeOrGlob( sqlite3 *db = pParse->db; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ + int rc; /* Result code to return */ if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ return 0; @@ -114493,7 +129710,7 @@ static int isLikeOrGlob( pLeft = pList->a[1].pExpr; if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->pTab) + || IsVirtual(pLeft->pTab) /* Value might be numeric */ ){ /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must ** be the name of an indexed column with TEXT affinity. */ @@ -114506,7 +129723,7 @@ static int isLikeOrGlob( if( op==TK_VARIABLE ){ Vdbe *pReprepare = pParse->pReprepare; int iCol = pRight->iColumn; - pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_NONE); + pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_BLOB); if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ z = (char *)sqlite3_value_text(pVal); } @@ -114534,7 +129751,7 @@ static int isLikeOrGlob( ** value of the variable means there is no need to invoke the LIKE ** function, then no OP_Variable will be added to the program. ** This causes problems for the sqlite3_bind_parameter_name() - ** API. To workaround them, add a dummy OP_Variable here. + ** API. To work around them, add a dummy OP_Variable here. */ int r1 = sqlite3GetTempReg(pParse); sqlite3ExprCodeTarget(pParse, pRight, r1); @@ -114547,8 +129764,9 @@ static int isLikeOrGlob( } } + rc = (z!=0); sqlite3ValueFree(pVal); - return (z!=0); + return rc; } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ @@ -114557,29 +129775,48 @@ static int isLikeOrGlob( /* ** Check to see if the given expression is of the form ** -** column MATCH expr +** column OP expr +** +** where OP is one of MATCH, GLOB, LIKE or REGEXP and "column" is a +** column of a virtual table. ** ** If it is then return TRUE. If not, return FALSE. */ static int isMatchOfColumn( - Expr *pExpr /* Test this expression */ + Expr *pExpr, /* Test this expression */ + unsigned char *peOp2 /* OUT: 0 for MATCH, or else an op2 value */ ){ + static const struct Op2 { + const char *zOp; + unsigned char eOp2; + } aOp[] = { + { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, + { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, + { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, + { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } + }; ExprList *pList; + Expr *pCol; /* Column reference */ + int i; if( pExpr->op!=TK_FUNCTION ){ return 0; } - if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ - return 0; - } pList = pExpr->x.pList; - if( pList->nExpr!=2 ){ + if( pList==0 || pList->nExpr!=2 ){ return 0; } - if( pList->a[1].pExpr->op != TK_COLUMN ){ + pCol = pList->a[1].pExpr; + if( pCol->op!=TK_COLUMN || !IsVirtual(pCol->pTab) ){ return 0; } - return 1; + for(i=0; iu.zToken, aOp[i].zOp)==0 ){ + *peOp2 = aOp[i].eOp2; + return 1; + } + } + return 0; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -114594,6 +129831,88 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ } } +/* +** Mark term iChild as being a child of term iParent +*/ +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} + +/* +** Return the N-th AND-connected subterm of pTerm. Or if pTerm is not +** a conjunction, then return just pTerm when N==0. If N is exceeds +** the number of available subterms, return NULL. +*/ +static WhereTerm *whereNthSubterm(WhereTerm *pTerm, int N){ + if( pTerm->eOperator!=WO_AND ){ + return N==0 ? pTerm : 0; + } + if( Nu.pAndInfo->wc.nTerm ){ + return &pTerm->u.pAndInfo->wc.a[N]; + } + return 0; +} + +/* +** Subterms pOne and pTwo are contained within WHERE clause pWC. The +** two subterms are in disjunction - they are OR-ed together. +** +** If these two terms are both of the form: "A op B" with the same +** A and B values but different operators and if the operators are +** compatible (if one is = and the other is <, for example) then +** add a new virtual AND term to pWC that is the combination of the +** two. +** +** Some examples: +** +** x x<=y +** x=y OR x=y --> x=y +** x<=y OR x x<=y +** +** The following is NOT generated: +** +** xy --> x!=y +*/ +static void whereCombineDisjuncts( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* The complete WHERE clause */ + WhereTerm *pOne, /* First disjunct */ + WhereTerm *pTwo /* Second disjunct */ +){ + u16 eOp = pOne->eOperator | pTwo->eOperator; + sqlite3 *db; /* Database connection (for malloc) */ + Expr *pNew; /* New virtual expression */ + int op; /* Operator for the combined expression */ + int idxNew; /* Index in pWC of the next virtual term */ + + if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp + && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; + assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); + assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); + if( sqlite3ExprCompare(pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; + if( sqlite3ExprCompare(pOne->pExpr->pRight, pTwo->pExpr->pRight, -1) )return; + /* If we reach this point, it means the two subterms can be combined */ + if( (eOp & (eOp-1))!=0 ){ + if( eOp & (WO_LT|WO_LE) ){ + eOp = WO_LE; + }else{ + assert( eOp & (WO_GT|WO_GE) ); + eOp = WO_GE; + } + } + db = pWC->pWInfo->pParse->db; + pNew = sqlite3ExprDup(db, pOne->pExpr, 0); + if( pNew==0 ) return; + for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( opop = op; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + exprAnalyze(pSrc, pWC, idxNew); +} + #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* ** Analyze a term that consists of two or more OR-connected @@ -114618,6 +129937,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) ** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') ** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) +** (F) x>A OR (x=A AND y>=B) ** ** CASE 1: ** @@ -114634,6 +129954,16 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** ** CASE 2: ** +** If there are exactly two disjuncts and one side has x>A and the other side +** has x=A (for the same x and A) then add a new virtual conjunct term to the +** WHERE clause of the form "x>=A". Example: +** +** x>A OR (x=A AND y>B) adds: x>=A +** +** The added conjunct can sometimes be helpful in query planning. +** +** CASE 3: +** ** If all subterms are indexable by a single table T, then set ** ** WhereTerm.eOperator = WO_OR @@ -114653,22 +129983,22 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** is decided elsewhere. This analysis only looks at whether subterms ** appropriate for indexing exist. ** -** All examples A through E above satisfy case 2. But if a term -** also statisfies case 1 (such as B) we know that the optimizer will -** always prefer case 1, so in that case we pretend that case 2 is not +** All examples A through E above satisfy case 3. But if a term +** also satisfies case 1 (such as B) we know that the optimizer will +** always prefer case 1, so in that case we pretend that case 3 is not ** satisfied. ** ** It might be the case that multiple tables are indexable. For example, ** (E) above is indexable on tables P, Q, and R. ** -** Terms that satisfy case 2 are candidates for lookup by using +** Terms that satisfy case 3 are candidates for lookup by using ** separate indices to find rowids for each subterm and composing ** the union of all rowids using a RowSet object. This is similar ** to "bitmap indices" in other database engines. ** ** OTHERWISE: ** -** If neither case 1 nor case 2 apply, then leave the eOperator set to +** If none of cases 1, 2, or 3 apply, then leave the eOperator set to ** zero. This term is not useful for search. */ static void exprAnalyzeOrTerm( @@ -114699,14 +130029,15 @@ static void exprAnalyzeOrTerm( if( pOrInfo==0 ) return; pTerm->wtFlags |= TERM_ORINFO; pOrWc = &pOrInfo->wc; - whereClauseInit(pOrWc, pWInfo); - whereSplit(pOrWc, pExpr, TK_OR); - exprAnalyzeAll(pSrc, pOrWc); + memset(pOrWc->aStatic, 0, sizeof(pOrWc->aStatic)); + sqlite3WhereClauseInit(pOrWc, pWInfo); + sqlite3WhereSplit(pOrWc, pExpr, TK_OR); + sqlite3WhereExprAnalyze(pSrc, pOrWc); if( db->mallocFailed ) return; assert( pOrWc->nTerm>=2 ); /* - ** Compute the set of tables that might satisfy cases 1 or 2. + ** Compute the set of tables that might satisfy cases 1 or 3. */ indexable = ~(Bitmask)0; chngToIN = ~(Bitmask)0; @@ -114715,7 +130046,7 @@ static void exprAnalyzeOrTerm( WhereAndInfo *pAndInfo; assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); chngToIN = 0; - pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); + pAndInfo = sqlite3DbMallocRawNN(db, sizeof(*pAndInfo)); if( pAndInfo ){ WhereClause *pAndWC; WhereTerm *pAndTerm; @@ -114725,16 +130056,18 @@ static void exprAnalyzeOrTerm( pOrTerm->wtFlags |= TERM_ANDINFO; pOrTerm->eOperator = WO_AND; pAndWC = &pAndInfo->wc; - whereClauseInit(pAndWC, pWC->pWInfo); - whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); - exprAnalyzeAll(pSrc, pAndWC); + memset(pAndWC->aStatic, 0, sizeof(pAndWC->aStatic)); + sqlite3WhereClauseInit(pAndWC, pWC->pWInfo); + sqlite3WhereSplit(pAndWC, pOrTerm->pExpr, TK_AND); + sqlite3WhereExprAnalyze(pSrc, pAndWC); pAndWC->pOuter = pWC; - testcase( db->mallocFailed ); if( !db->mallocFailed ){ for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ assert( pAndTerm->pExpr ); - if( allowedOp(pAndTerm->pExpr->op) ){ - b |= getMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); + if( allowedOp(pAndTerm->pExpr->op) + || pAndTerm->eOperator==WO_MATCH + ){ + b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); } } } @@ -114745,10 +130078,10 @@ static void exprAnalyzeOrTerm( ** corresponding TERM_VIRTUAL term */ }else{ Bitmask b; - b = getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); + b = sqlite3WhereGetMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); if( pOrTerm->wtFlags & TERM_VIRTUAL ){ WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; - b |= getMask(&pWInfo->sMaskSet, pOther->leftCursor); + b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pOther->leftCursor); } indexable &= b; if( (pOrTerm->eOperator & WO_EQ)==0 ){ @@ -114760,12 +130093,26 @@ static void exprAnalyzeOrTerm( } /* - ** Record the set of tables that satisfy case 2. The set might be + ** Record the set of tables that satisfy case 3. The set might be ** empty. */ pOrInfo->indexable = indexable; pTerm->eOperator = indexable==0 ? 0 : WO_OR; + /* For a two-way OR, attempt to implementation case 2. + */ + if( indexable && pOrWc->nTerm==2 ){ + int iOne = 0; + WhereTerm *pOne; + while( (pOne = whereNthSubterm(&pOrWc->a[0],iOne++))!=0 ){ + int iTwo = 0; + WhereTerm *pTwo; + while( (pTwo = whereNthSubterm(&pOrWc->a[1],iTwo++))!=0 ){ + whereCombineDisjuncts(pSrc, pWC, pOne, pTwo); + } + } + } + /* ** chngToIN holds a set of tables that *might* satisfy case 1. But ** we have to do some additional checking to see if case 1 really @@ -114810,9 +130157,10 @@ static void exprAnalyzeOrTerm( assert( j==1 ); continue; } - if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ + if( (chngToIN & sqlite3WhereGetMask(&pWInfo->sMaskSet, + pOrTerm->leftCursor))==0 ){ /* This term must be of the form t1.a==t2.b where t2 is in the - ** chngToIN set but t1 is not. This term will be either preceeded + ** chngToIN set but t1 is not. This term will be either preceded ** or follwed by an inverted copy (t2.b==t1.a). Skip this term ** and use its inversion. */ testcase( pOrTerm->wtFlags & TERM_COPIED ); @@ -114829,7 +130177,7 @@ static void exprAnalyzeOrTerm( ** on the second iteration */ assert( j==1 ); assert( IsPowerOfTwo(chngToIN) ); - assert( chngToIN==getMask(&pWInfo->sMaskSet, iCursor) ); + assert( chngToIN==sqlite3WhereGetMask(&pWInfo->sMaskSet, iCursor) ); break; } testcase( j==1 ); @@ -114891,17 +130239,138 @@ static void exprAnalyzeOrTerm( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; - pTerm->nChild = 1; + markTermAsChild(pWC, idxNew, idxTerm); }else{ sqlite3ExprListDelete(db, pList); } - pTerm->eOperator = WO_NOOP; /* case 1 trumps case 2 */ + pTerm->eOperator = WO_NOOP; /* case 1 trumps case 3 */ } } } #endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ +/* +** We already know that pExpr is a binary operator where both operands are +** column references. This routine checks to see if pExpr is an equivalence +** relation: +** 1. The SQLITE_Transitive optimization must be enabled +** 2. Must be either an == or an IS operator +** 3. Not originating in the ON clause of an OUTER JOIN +** 4. The affinities of A and B must be compatible +** 5a. Both operands use the same collating sequence OR +** 5b. The overall collating sequence is BINARY +** If this routine returns TRUE, that means that the RHS can be substituted +** for the LHS anyplace else in the WHERE clause where the LHS column occurs. +** This is an optimization. No harm comes from returning 0. But if 1 is +** returned when it should not be, then incorrect answers might result. +*/ +static int termIsEquivalence(Parse *pParse, Expr *pExpr){ + char aff1, aff2; + CollSeq *pColl; + const char *zColl1, *zColl2; + if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; + if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; + if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0; + aff1 = sqlite3ExprAffinity(pExpr->pLeft); + aff2 = sqlite3ExprAffinity(pExpr->pRight); + if( aff1!=aff2 + && (!sqlite3IsNumericAffinity(aff1) || !sqlite3IsNumericAffinity(aff2)) + ){ + return 0; + } + pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight); + if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1; + pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + zColl1 = pColl ? pColl->zName : 0; + pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); + zColl2 = pColl ? pColl->zName : 0; + return sqlite3_stricmp(zColl1, zColl2)==0; +} + +/* +** Recursively walk the expressions of a SELECT statement and generate +** a bitmask indicating which tables are used in that expression +** tree. +*/ +static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ + Bitmask mask = 0; + while( pS ){ + SrcList *pSrc = pS->pSrc; + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pEList); + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pGroupBy); + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pOrderBy); + mask |= sqlite3WhereExprUsage(pMaskSet, pS->pWhere); + mask |= sqlite3WhereExprUsage(pMaskSet, pS->pHaving); + if( ALWAYS(pSrc!=0) ){ + int i; + for(i=0; inSrc; i++){ + mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); + mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn); + } + } + pS = pS->pPrior; + } + return mask; +} + +/* +** Expression pExpr is one operand of a comparison operator that might +** be useful for indexing. This routine checks to see if pExpr appears +** in any index. Return TRUE (1) if pExpr is an indexed term and return +** FALSE (0) if not. If TRUE is returned, also set *piCur to the cursor +** number of the table that is indexed and *piColumn to the column number +** of the column that is indexed, or XN_EXPR (-2) if an expression is being +** indexed. +** +** If pExpr is a TK_COLUMN column reference, then this routine always returns +** true even if that particular column is not indexed, because the column +** might be added to an automatic index later. +*/ +static int exprMightBeIndexed( + SrcList *pFrom, /* The FROM clause */ + int op, /* The specific comparison operator */ + Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ + Expr *pExpr, /* An operand of a comparison operator */ + int *piCur, /* Write the referenced table cursor number here */ + int *piColumn /* Write the referenced table column number here */ +){ + Index *pIdx; + int i; + int iCur; + + /* If this expression is a vector to the left or right of a + ** inequality constraint (>, <, >= or <=), perform the processing + ** on the first element of the vector. */ + assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE ); + assert( TK_ISop==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){ + pExpr = pExpr->x.pList->a[0].pExpr; + } + + if( pExpr->op==TK_COLUMN ){ + *piCur = pExpr->iTable; + *piColumn = pExpr->iColumn; + return 1; + } + if( mPrereq==0 ) return 0; /* No table references */ + if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ + for(i=0; mPrereq>1; i++, mPrereq>>=1){} + iCur = pFrom->a[i].iCursor; + for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; inKeyCol; i++){ + if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ + *piCur = iCur; + *piColumn = XN_EXPR; + return 1; + } + } + } + return 0; +} + /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -114934,10 +130403,11 @@ static void exprAnalyze( Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ - int noCase = 0; /* LIKE/GLOB distinguishes case */ + int noCase = 0; /* uppercase equivalent to lowercase */ int op; /* Top-level operator. pExpr->op */ Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection */ + unsigned char eOp2; /* op2 value for LIKE/REGEXP/GLOB */ if( db->mallocFailed ){ return; @@ -114946,23 +130416,24 @@ static void exprAnalyze( pMaskSet = &pWInfo->sMaskSet; pExpr = pTerm->pExpr; assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); - prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); + prereqLeft = sqlite3WhereExprUsage(pMaskSet, pExpr->pLeft); op = pExpr->op; if( op==TK_IN ){ assert( pExpr->pRight==0 ); + if( sqlite3ExprCheckIN(pParse, pExpr) ) return; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - pTerm->prereqRight = exprSelectTableUsage(pMaskSet, pExpr->x.pSelect); + pTerm->prereqRight = exprSelectUsage(pMaskSet, pExpr->x.pSelect); }else{ - pTerm->prereqRight = exprListTableUsage(pMaskSet, pExpr->x.pList); + pTerm->prereqRight = sqlite3WhereExprListUsage(pMaskSet, pExpr->x.pList); } }else if( op==TK_ISNULL ){ pTerm->prereqRight = 0; }else{ - pTerm->prereqRight = exprTableUsage(pMaskSet, pExpr->pRight); + pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); } - prereqAll = exprTableUsage(pMaskSet, pExpr); + prereqAll = sqlite3WhereExprUsage(pMaskSet, pExpr); if( ExprHasProperty(pExpr, EP_FromJoin) ){ - Bitmask x = getMask(pMaskSet, pExpr->iRightJoinTable); + Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); prereqAll |= x; extraRight = x-1; /* ON clause terms may not be used with an index ** on left table of a LEFT JOIN. Ticket #3015 */ @@ -114972,18 +130443,30 @@ static void exprAnalyze( pTerm->iParent = -1; pTerm->eOperator = 0; if( allowedOp(op) ){ + int iCur, iColumn; Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; - if( pLeft->op==TK_COLUMN ){ - pTerm->leftCursor = pLeft->iTable; - pTerm->u.leftColumn = pLeft->iColumn; + + if( pTerm->iField>0 ){ + assert( op==TK_IN ); + assert( pLeft->op==TK_VECTOR ); + pLeft = pLeft->x.pList->a[pTerm->iField-1].pExpr; + } + + if( exprMightBeIndexed(pSrc, op, prereqLeft, pLeft, &iCur, &iColumn) ){ + pTerm->leftCursor = iCur; + pTerm->u.leftColumn = iColumn; pTerm->eOperator = operatorMask(op) & opMask; } - if( pRight && pRight->op==TK_COLUMN ){ + if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; + if( pRight + && exprMightBeIndexed(pSrc, op, pTerm->prereqRight, pRight, &iCur,&iColumn) + ){ WhereTerm *pNew; Expr *pDup; u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ + assert( pTerm->iField==0 ); if( pTerm->leftCursor>=0 ){ int idxNew; pDup = sqlite3ExprDup(db, pExpr, 0); @@ -114994,14 +130477,12 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); if( idxNew==0 ) return; pNew = &pWC->a[idxNew]; - pNew->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); + if( op==TK_IS ) pNew->wtFlags |= TERM_IS; pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; - if( pExpr->op==TK_EQ - && !ExprHasProperty(pExpr, EP_FromJoin) - && OptimizationEnabled(db, SQLITE_Transitive) - ){ + + if( termIsEquivalence(pParse, pDup) ){ pTerm->eOperator |= WO_EQUIV; eExtraOp = WO_EQUIV; } @@ -115010,9 +130491,8 @@ static void exprAnalyze( pNew = pTerm; } exprCommute(pParse, pDup); - pLeft = sqlite3ExprSkipCollate(pDup->pLeft); - pNew->leftCursor = pLeft->iTable; - pNew->u.leftColumn = pLeft->iColumn; + pNew->leftCursor = iCur; + pNew->u.leftColumn = iColumn; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; @@ -115053,9 +130533,8 @@ static void exprAnalyze( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); } - pTerm->nChild = 2; } #endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ @@ -115074,12 +130553,15 @@ static void exprAnalyze( /* Add constraints to reduce the search space on a LIKE or GLOB ** operator. ** - ** A like pattern of the form "x LIKE 'abc%'" is changed into constraints + ** A like pattern of the form "x LIKE 'aBc%'" is changed into constraints ** - ** x>='abc' AND x<'abd' AND x LIKE 'abc%' + ** x>='ABC' AND x<'abd' AND x LIKE 'aBc%' ** ** The last character of the prefix "abc" is incremented to form the - ** termination condition "abd". + ** termination condition "abd". If case is not significant (the default + ** for LIKE) then the lower-bound is made all uppercase and the upper- + ** bound is made all lowercase so that the bounds also work when comparing + ** BLOBs. */ if( pWC->op==TK_AND && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) @@ -115090,10 +130572,26 @@ static void exprAnalyze( Expr *pNewExpr2; int idxNew1; int idxNew2; - Token sCollSeqName; /* Name of collating sequence */ + const char *zCollSeqName; /* Name of collating sequence */ + const u16 wtFlags = TERM_LIKEOPT | TERM_VIRTUAL | TERM_DYNAMIC; pLeft = pExpr->x.pList->a[1].pExpr; pStr2 = sqlite3ExprDup(db, pStr1, 0); + + /* Convert the lower bound to upper-case and the upper bound to + ** lower-case (upper-case is less than lower-case in ASCII) so that + ** the range constraints also work for BLOBs + */ + if( noCase && !pParse->db->mallocFailed ){ + int i; + char c; + pTerm->wtFlags |= TERM_LIKE; + for(i=0; (c = pStr1->u.zToken[i])!=0; i++){ + pStr1->u.zToken[i] = sqlite3Toupper(c); + pStr2->u.zToken[i] = sqlite3Tolower(c); + } + } + if( !db->mallocFailed ){ u8 c, *pC; /* Last character before the first wildcard */ pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; @@ -115110,29 +130608,27 @@ static void exprAnalyze( } *pC = c + 1; } - sCollSeqName.z = noCase ? "NOCASE" : "BINARY"; - sCollSeqName.n = 6; + zCollSeqName = noCase ? "NOCASE" : "BINARY"; pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); - pNewExpr1 = sqlite3PExpr(pParse, TK_GE, - sqlite3ExprAddCollateToken(pParse,pNewExpr1,&sCollSeqName), + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), pStr1, 0); transferJoinMarkings(pNewExpr1, pExpr); - idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); + idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); testcase( idxNew1==0 ); exprAnalyze(pSrc, pWC, idxNew1); pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); pNewExpr2 = sqlite3PExpr(pParse, TK_LT, - sqlite3ExprAddCollateToken(pParse,pNewExpr2,&sCollSeqName), + sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), pStr2, 0); transferJoinMarkings(pNewExpr2, pExpr); - idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); + idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); testcase( idxNew2==0 ); exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ - pWC->a[idxNew1].iParent = idxTerm; - pWC->a[idxNew2].iParent = idxTerm; - pTerm->nChild = 2; + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); } } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ @@ -115144,7 +130640,7 @@ static void exprAnalyze( ** virtual tables. The native query optimizer does not attempt ** to do anything with MATCH functions. */ - if( isMatchOfColumn(pExpr) ){ + if( pWC->op==TK_AND && isMatchOfColumn(pExpr, &eOp2) ){ int idxNew; Expr *pRight, *pLeft; WhereTerm *pNewTerm; @@ -115152,8 +130648,8 @@ static void exprAnalyze( pRight = pExpr->x.pList->a[0].pExpr; pLeft = pExpr->x.pList->a[1].pExpr; - prereqExpr = exprTableUsage(pMaskSet, pRight); - prereqColumn = exprTableUsage(pMaskSet, pLeft); + prereqExpr = sqlite3WhereExprUsage(pMaskSet, pRight); + prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft); if( (prereqExpr & prereqColumn)==0 ){ Expr *pNewExpr; pNewExpr = sqlite3PExpr(pParse, TK_MATCH, @@ -115165,30 +130661,81 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_MATCH; - pNewTerm->iParent = idxTerm; + pNewTerm->eMatchOp = eOp2; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ + /* If there is a vector == or IS term - e.g. "(a, b) == (?, ?)" - create + ** new terms for each component comparison - "a = ?" and "b = ?". The + ** new terms completely replace the original vector comparison, which is + ** no longer used. + ** + ** This is only required if at least one side of the comparison operation + ** is not a sub-select. */ + if( pWC->op==TK_AND + && (pExpr->op==TK_EQ || pExpr->op==TK_IS) + && sqlite3ExprIsVector(pExpr->pLeft) + && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 + || (pExpr->pRight->flags & EP_xIsSelect)==0 + )){ + int nLeft = sqlite3ExprVectorSize(pExpr->pLeft); + int i; + assert( nLeft==sqlite3ExprVectorSize(pExpr->pRight) ); + for(i=0; ipLeft, i); + Expr *pRight = sqlite3ExprForVectorField(pParse, pExpr->pRight, i); + + pNew = sqlite3PExpr(pParse, pExpr->op, pLeft, pRight, 0); + transferJoinMarkings(pNew, pExpr); + idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC); + exprAnalyze(pSrc, pWC, idxNew); + } + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags = TERM_CODED|TERM_VIRTUAL; /* Disable the original */ + pTerm->eOperator = 0; + } + + /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create + ** a virtual term for each vector component. The expression object + ** used by each such virtual term is pExpr (the full vector IN(...) + ** expression). The WhereTerm.iField variable identifies the index within + ** the vector on the LHS that the virtual term represents. + ** + ** This only works if the RHS is a simple SELECT, not a compound + */ + if( pWC->op==TK_AND && pExpr->op==TK_IN && pTerm->iField==0 + && pExpr->pLeft->op==TK_VECTOR + && pExpr->x.pSelect->pPrior==0 + ){ + int i; + for(i=0; ipLeft); i++){ + int idxNew; + idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL); + pWC->a[idxNew].iField = i+1; + exprAnalyze(pSrc, pWC, idxNew); + markTermAsChild(pWC, idxNew, idxTerm); + } + } + #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* When sqlite_stat3 histogram data is available an operator of the ** form "x IS NOT NULL" can sometimes be evaluated more efficiently ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a ** virtual term of that form. ** - ** Note that the virtual term must be tagged with TERM_VNULL. This - ** TERM_VNULL tag will suppress the not-null check at the beginning - ** of the loop. Without the TERM_VNULL flag, the not-null check at - ** the start of the loop will prevent any results from being returned. + ** Note that the virtual term must be tagged with TERM_VNULL. */ if( pExpr->op==TK_NOTNULL && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat3) + && OptimizationEnabled(db, SQLITE_Stat34) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -115197,7 +130744,7 @@ static void exprAnalyze( pNewExpr = sqlite3PExpr(pParse, TK_GT, sqlite3ExprDup(db, pLeft, 0), - sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0); + sqlite3ExprAlloc(db, TK_NULL, 0, 0), 0); idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); @@ -115207,9 +130754,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_GT; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } @@ -115219,11 +130765,556 @@ static void exprAnalyze( /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. */ + testcase( pTerm!=&pWC->a[idxTerm] ); + pTerm = &pWC->a[idxTerm]; pTerm->prereqRight |= extraRight; } +/*************************************************************************** +** Routines with file scope above. Interface to the rest of the where.c +** subsystem follows. +***************************************************************************/ + /* -** This function searches pList for a entry that matches the iCol-th column +** This routine identifies subexpressions in the WHERE clause where +** each subexpression is separated by the AND operator or some other +** operator specified in the op parameter. The WhereClause structure +** is filled with pointers to subexpressions. For example: +** +** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) +** \________/ \_______________/ \________________/ +** slot[0] slot[1] slot[2] +** +** The original WHERE clause in pExpr is unaltered. All this routine +** does is make slot[] entries point to substructure within pExpr. +** +** In the previous sentence and in the diagram, "slot[]" refers to +** the WhereClause.a[] array. The slot[] array grows as needed to contain +** all terms of the WHERE clause. +*/ +SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ + Expr *pE2 = sqlite3ExprSkipCollate(pExpr); + pWC->op = op; + if( pE2==0 ) return; + if( pE2->op!=op ){ + whereClauseInsert(pWC, pExpr, 0); + }else{ + sqlite3WhereSplit(pWC, pE2->pLeft, op); + sqlite3WhereSplit(pWC, pE2->pRight, op); + } +} + +/* +** Initialize a preallocated WhereClause structure. +*/ +SQLITE_PRIVATE void sqlite3WhereClauseInit( + WhereClause *pWC, /* The WhereClause to be initialized */ + WhereInfo *pWInfo /* The WHERE processing context */ +){ + pWC->pWInfo = pWInfo; + pWC->pOuter = 0; + pWC->nTerm = 0; + pWC->nSlot = ArraySize(pWC->aStatic); + pWC->a = pWC->aStatic; +} + +/* +** Deallocate a WhereClause structure. The WhereClause structure +** itself is not freed. This routine is the inverse of +** sqlite3WhereClauseInit(). +*/ +SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ + int i; + WhereTerm *a; + sqlite3 *db = pWC->pWInfo->pParse->db; + for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ + if( a->wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, a->pExpr); + } + if( a->wtFlags & TERM_ORINFO ){ + whereOrInfoDelete(db, a->u.pOrInfo); + }else if( a->wtFlags & TERM_ANDINFO ){ + whereAndInfoDelete(db, a->u.pAndInfo); + } + } + if( pWC->a!=pWC->aStatic ){ + sqlite3DbFree(db, pWC->a); + } +} + + +/* +** These routines walk (recursively) an expression tree and generate +** a bitmask indicating which tables are used in that expression +** tree. +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ + Bitmask mask; + if( p==0 ) return 0; + if( p->op==TK_COLUMN ){ + mask = sqlite3WhereGetMask(pMaskSet, p->iTable); + return mask; + } + assert( !ExprHasProperty(p, EP_TokenOnly) ); + mask = p->pRight ? sqlite3WhereExprUsage(pMaskSet, p->pRight) : 0; + if( p->pLeft ) mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft); + if( ExprHasProperty(p, EP_xIsSelect) ){ + mask |= exprSelectUsage(pMaskSet, p->x.pSelect); + }else if( p->x.pList ){ + mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList); + } + return mask; +} +SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){ + int i; + Bitmask mask = 0; + if( pList ){ + for(i=0; inExpr; i++){ + mask |= sqlite3WhereExprUsage(pMaskSet, pList->a[i].pExpr); + } + } + return mask; +} + + +/* +** Call exprAnalyze on all terms in a WHERE clause. +** +** Note that exprAnalyze() might add new virtual terms onto the +** end of the WHERE clause. We do not want to analyze these new +** virtual terms, so start analyzing at the end and work forward +** so that the added virtual terms are never processed. +*/ +SQLITE_PRIVATE void sqlite3WhereExprAnalyze( + SrcList *pTabList, /* the FROM clause */ + WhereClause *pWC /* the WHERE clause to be analyzed */ +){ + int i; + for(i=pWC->nTerm-1; i>=0; i--){ + exprAnalyze(pTabList, pWC, i); + } +} + +/* +** For table-valued-functions, transform the function arguments into +** new WHERE clause terms. +** +** Each function argument translates into an equality constraint against +** a HIDDEN column in the table. +*/ +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( + Parse *pParse, /* Parsing context */ + struct SrcList_item *pItem, /* The FROM clause term to process */ + WhereClause *pWC /* Xfer function arguments to here */ +){ + Table *pTab; + int j, k; + ExprList *pArgs; + Expr *pColRef; + Expr *pTerm; + if( pItem->fg.isTabFunc==0 ) return; + pTab = pItem->pTab; + assert( pTab!=0 ); + pArgs = pItem->u1.pFuncArg; + if( pArgs==0 ) return; + for(j=k=0; jnExpr; j++){ + while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;} + if( k>=pTab->nCol ){ + sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", + pTab->zName, j); + return; + } + pColRef = sqlite3ExprAlloc(pParse->db, TK_COLUMN, 0, 0); + if( pColRef==0 ) return; + pColRef->iTable = pItem->iCursor; + pColRef->iColumn = k++; + pColRef->pTab = pTab; + pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, + sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); + } +} + +/************** End of whereexpr.c *******************************************/ +/************** Begin file where.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. This module is responsible for +** generating the code that loops through a table looking for applicable +** rows. Indices are selected and used to speed the search when doing +** so is applicable. Because this module is responsible for selecting +** indices, you might also think of this module as the "query optimizer". +*/ +/* #include "sqliteInt.h" */ +/* #include "whereInt.h" */ + +/* Forward declaration of methods */ +static int whereLoopResize(sqlite3*, WhereLoop*, int); + +/* Test variable that can be set to enable WHERE tracing */ +#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) +/***/ int sqlite3WhereTrace = 0; +#endif + + +/* +** Return the estimated number of output rows from a WHERE clause +*/ +SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ + return pWInfo->nRowOut; +} + +/* +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this +** WHERE clause returns outputs for DISTINCT processing. +*/ +SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ + return pWInfo->eDistinct; +} + +/* +** Return TRUE if the WHERE clause returns rows in ORDER BY order. +** Return FALSE if the output needs to be sorted. +*/ +SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ + return pWInfo->nOBSat; +} + +/* +** Return TRUE if the innermost loop of the WHERE clause implementation +** returns rows in ORDER BY order for complete run of the inner loop. +** +** Across multiple iterations of outer loops, the output rows need not be +** sorted. As long as rows are sorted for just the innermost loop, this +** routine can return TRUE. +*/ +SQLITE_PRIVATE int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){ + return pWInfo->bOrderedInnerLoop; +} + +/* +** Return the VDBE address or label to jump to in order to continue +** immediately with the next row of a WHERE clause. +*/ +SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); + return pWInfo->iContinue; +} + +/* +** Return the VDBE address or label to jump to in order to break +** out of a WHERE loop. +*/ +SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ + return pWInfo->iBreak; +} + +/* +** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to +** operate directly on the rowis returned by a WHERE clause. Return +** ONEPASS_SINGLE (1) if the statement can operation directly because only +** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass +** optimization can be used on multiple +** +** If the ONEPASS optimization is used (if this routine returns true) +** then also write the indices of open cursors used by ONEPASS +** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data +** table and iaCur[1] gets the cursor used by an auxiliary index. +** Either value may be -1, indicating that cursor is not used. +** Any cursors returned will have been opened for writing. +** +** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is +** unable to use the ONEPASS optimization. +*/ +SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ + memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){ + sqlite3DebugPrintf("%s cursors: %d %d\n", + pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI", + aiCur[0], aiCur[1]); + } +#endif + return pWInfo->eOnePass; +} + +/* +** Move the content of pSrc into pDest +*/ +static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ + pDest->n = pSrc->n; + memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); +} + +/* +** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. +** +** The new entry might overwrite an existing entry, or it might be +** appended, or it might be discarded. Do whatever is the right thing +** so that pSet keeps the N_OR_COST best entries seen so far. +*/ +static int whereOrInsert( + WhereOrSet *pSet, /* The WhereOrSet to be updated */ + Bitmask prereq, /* Prerequisites of the new entry */ + LogEst rRun, /* Run-cost of the new entry */ + LogEst nOut /* Number of outputs for the new entry */ +){ + u16 i; + WhereOrCost *p; + for(i=pSet->n, p=pSet->a; i>0; i--, p++){ + if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ + goto whereOrInsert_done; + } + if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ + return 0; + } + } + if( pSet->na[pSet->n++]; + p->nOut = nOut; + }else{ + p = pSet->a; + for(i=1; in; i++){ + if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; + } + if( p->rRun<=rRun ) return 0; + } +whereOrInsert_done: + p->prereq = prereq; + p->rRun = rRun; + if( p->nOut>nOut ) p->nOut = nOut; + return 1; +} + +/* +** Return the bitmask for the given cursor number. Return 0 if +** iCursor is not in the set. +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ + int i; + assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); + for(i=0; in; i++){ + if( pMaskSet->ix[i]==iCursor ){ + return MASKBIT(i); + } + } + return 0; +} + +/* +** Create a new mask for cursor iCursor. +** +** There is one cursor per table in the FROM clause. The number of +** tables in the FROM clause is limited by a test early in the +** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] +** array will never overflow. +*/ +static void createMask(WhereMaskSet *pMaskSet, int iCursor){ + assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); + pMaskSet->ix[pMaskSet->n++] = iCursor; +} + +/* +** Advance to the next WhereTerm that matches according to the criteria +** established when the pScan object was initialized by whereScanInit(). +** Return NULL if there are no more matching WhereTerms. +*/ +static WhereTerm *whereScanNext(WhereScan *pScan){ + int iCur; /* The cursor on the LHS of the term */ + i16 iColumn; /* The column on the LHS of the term. -1 for IPK */ + Expr *pX; /* An expression being tested */ + WhereClause *pWC; /* Shorthand for pScan->pWC */ + WhereTerm *pTerm; /* The term being tested */ + int k = pScan->k; /* Where to start scanning */ + + while( pScan->iEquiv<=pScan->nEquiv ){ + iCur = pScan->aiCur[pScan->iEquiv-1]; + iColumn = pScan->aiColumn[pScan->iEquiv-1]; + if( iColumn==XN_EXPR && pScan->pIdxExpr==0 ) return 0; + while( (pWC = pScan->pWC)!=0 ){ + for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + && (iColumn!=XN_EXPR + || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) + && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + ){ + if( (pTerm->eOperator & WO_EQUIV)!=0 + && pScan->nEquivaiCur) + && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN + ){ + int j; + for(j=0; jnEquiv; j++){ + if( pScan->aiCur[j]==pX->iTable + && pScan->aiColumn[j]==pX->iColumn ){ + break; + } + } + if( j==pScan->nEquiv ){ + pScan->aiCur[j] = pX->iTable; + pScan->aiColumn[j] = pX->iColumn; + pScan->nEquiv++; + } + } + if( (pTerm->eOperator & pScan->opMask)!=0 ){ + /* Verify the affinity and collating sequence match */ + if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + Parse *pParse = pWC->pWInfo->pParse; + pX = pTerm->pExpr; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse, + pX->pLeft, pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + continue; + } + } + if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 + && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && pX->iTable==pScan->aiCur[0] + && pX->iColumn==pScan->aiColumn[0] + ){ + testcase( pTerm->eOperator & WO_IS ); + continue; + } + pScan->k = k+1; + return pTerm; + } + } + } + pScan->pWC = pScan->pWC->pOuter; + k = 0; + } + pScan->pWC = pScan->pOrigWC; + k = 0; + pScan->iEquiv++; + } + return 0; +} + +/* +** Initialize a WHERE clause scanner object. Return a pointer to the +** first match. Return NULL if there are no matches. +** +** The scanner will be searching the WHERE clause pWC. It will look +** for terms of the form "X " where X is column iColumn of table +** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx +** must be one of the indexes of table iCur. +** +** The must be one of the operators described by opMask. +** +** If the search is for X and the WHERE clause contains terms of the +** form X=Y then this routine might also return terms of the form +** "Y ". The number of levels of transitivity is limited, +** but is enough to handle most commonly occurring SQL statements. +** +** If X is not the INTEGER PRIMARY KEY then X must be compatible with +** index pIdx. +*/ +static WhereTerm *whereScanInit( + WhereScan *pScan, /* The WhereScan object being initialized */ + WhereClause *pWC, /* The WHERE clause to be scanned */ + int iCur, /* Cursor to scan for */ + int iColumn, /* Column to scan for */ + u32 opMask, /* Operator(s) to scan for */ + Index *pIdx /* Must be compatible with this index */ +){ + int j = 0; + + /* memset(pScan, 0, sizeof(*pScan)); */ + pScan->pOrigWC = pWC; + pScan->pWC = pWC; + pScan->pIdxExpr = 0; + if( pIdx ){ + j = iColumn; + iColumn = pIdx->aiColumn[j]; + if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + if( iColumn==pIdx->pTable->iPKey ) iColumn = XN_ROWID; + } + if( pIdx && iColumn>=0 ){ + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + pScan->zCollName = pIdx->azColl[j]; + }else{ + pScan->idxaff = 0; + pScan->zCollName = 0; + } + pScan->opMask = opMask; + pScan->k = 0; + pScan->aiCur[0] = iCur; + pScan->aiColumn[0] = iColumn; + pScan->nEquiv = 1; + pScan->iEquiv = 1; + return whereScanNext(pScan); +} + +/* +** Search for a term in the WHERE clause that is of the form "X " +** where X is a reference to the iColumn of table iCur or of index pIdx +** if pIdx!=0 and is one of the WO_xx operator codes specified by +** the op parameter. Return a pointer to the term. Return 0 if not found. +** +** If pIdx!=0 then it must be one of the indexes of table iCur. +** Search for terms matching the iColumn-th column of pIdx +** rather than the iColumn-th column of table iCur. +** +** The term returned might by Y= if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11 +** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10 +** other equivalent values. Hence a search for X will return if X=A1 +** and A1=A2 and A2=A3 and ... and A9=A10 and A10=. +** +** If there are multiple terms in the WHERE clause of the form "X " +** then try for the one with no dependencies on - in other words where +** is a constant expression of some kind. Only return entries of +** the form "X Y" where Y is a column in another table if no terms of +** the form "X " exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. +*/ +SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ +){ + WhereTerm *pResult = 0; + WhereTerm *p; + WhereScan scan; + + p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); + op &= WO_EQ|WO_IS; + while( p ){ + if( (p->prereqRight & notReady)==0 ){ + if( p->prereqRight==0 && (p->eOperator&op)!=0 ){ + testcase( p->eOperator & WO_IS ); + return p; + } + if( pResult==0 ) pResult = p; + } + p = whereScanNext(&scan); + } + return pResult; +} + +/* +** This function searches pList for an entry that matches the iCol-th column ** of index pIdx. ** ** If such an expression is found, its index in pList->a[] is returned. If @@ -115246,7 +131337,7 @@ static int findIndexCol( && p->iTable==iBase ){ CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); - if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){ return i; } } @@ -115255,12 +131346,31 @@ static int findIndexCol( return -1; } +/* +** Return TRUE if the iCol-th column of index pIdx is NOT NULL +*/ +static int indexColumnNotNull(Index *pIdx, int iCol){ + int j; + assert( pIdx!=0 ); + assert( iCol>=0 && iColnColumn ); + j = pIdx->aiColumn[iCol]; + if( j>=0 ){ + return pIdx->pTable->aCol[j].notNull; + }else if( j==(-1) ){ + return 1; + }else{ + assert( j==(-2) ); + return 0; /* Assume an indexed expression can always yield a NULL */ + + } +} + /* ** Return true if the DISTINCT expression-list passed as the third argument ** is redundant. ** -** A DISTINCT list is redundant if the database contains some subset of -** columns that are unique and non-null. +** A DISTINCT list is redundant if any subset of the columns in the +** DISTINCT list are collectively unique and individually non-null. */ static int isDistinctRedundant( Parse *pParse, /* Parsing context */ @@ -115305,12 +131415,9 @@ static int isDistinctRedundant( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( !IsUniqueIndex(pIdx) ) continue; for(i=0; inKeyCol; i++){ - i16 iCol = pIdx->aiColumn[i]; - if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ - int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); - if( iIdxCol<0 || pTab->aCol[iCol].notNull==0 ){ - break; - } + if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ + if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; + if( indexColumnNotNull(pIdx, i)==0 ) break; } } if( i==pIdx->nKeyCol ){ @@ -115330,6 +131437,49 @@ static LogEst estLog(LogEst N){ return N<=10 ? 0 : sqlite3LogEst(N) - 33; } +/* +** Convert OP_Column opcodes to OP_Copy in previously generated code. +** +** This routine runs over generated VDBE code and translates OP_Column +** opcodes into OP_Copy when the table is being accessed via co-routine +** instead of via table lookup. +** +** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on +** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero, +** then each OP_Rowid is transformed into an instruction to increment the +** value stored in its output register. +*/ +static void translateColumnToCopy( + Vdbe *v, /* The VDBE containing code to translate */ + int iStart, /* Translate from this opcode to the end */ + int iTabCur, /* OP_Column/OP_Rowid references to this table */ + int iRegister, /* The first column is in this register */ + int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ +){ + VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); + int iEnd = sqlite3VdbeCurrentAddr(v); + for(; iStartp1!=iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + pOp->opcode = OP_Copy; + pOp->p1 = pOp->p2 + iRegister; + pOp->p2 = pOp->p3; + pOp->p3 = 0; + }else if( pOp->opcode==OP_Rowid ){ + if( bIncrRowid ){ + /* Increment the value stored in the P2 operand of the OP_Rowid. */ + pOp->opcode = OP_AddImm; + pOp->p1 = pOp->p2; + pOp->p2 = 1; + }else{ + pOp->opcode = OP_Null; + pOp->p1 = 0; + pOp->p3 = 0; + } + } + } +} + /* ** Two routines for printing the content of an sqlite3_index_info ** structure. Used for testing and debugging only. If neither @@ -115388,11 +131538,12 @@ static int termCanDriveIndex( ){ char aff; if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( (pTerm->eOperator & WO_EQ)==0 ) return 0; + if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; if( (pTerm->prereqRight & notReady)!=0 ) return 0; if( pTerm->u.leftColumn<0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; + testcase( pTerm->pExpr->op==TK_IS ); return 1; } #endif @@ -115429,12 +131580,17 @@ static void constructAutomaticIndex( Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ + struct SrcList_item *pTabItem; /* FROM clause term being indexed */ + int addrCounter = 0; /* Address where integer counter is initialized */ + int regBase; /* Array of registers where record is assembled */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ v = pParse->pVdbe; assert( v!=0 ); - addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); + addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); /* Count the number of columns that will be added to the index ** and used to match WHERE clause constraints */ @@ -115444,6 +131600,17 @@ static void constructAutomaticIndex( pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermpExpr; + assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */ + || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */ + || pLoop->prereq!=0 ); /* table of a LEFT JOIN */ + if( pLoop->prereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 + && !ExprHasProperty(pExpr, EP_FromJoin) + && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse->db, pPartial, + sqlite3ExprDup(pParse->db, pExpr, 0)); + } if( termCanDriveIndex(pTerm, pSrc, notReady) ){ int iCol = pTerm->u.leftColumn; Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); @@ -115456,7 +131623,9 @@ static void constructAutomaticIndex( sentWarning = 1; } if( (idxCols & cMask)==0 ){ - if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return; + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; + } pLoop->aLTerm[nKeyCol++] = pTerm; idxCols |= cMask; } @@ -115476,7 +131645,7 @@ static void constructAutomaticIndex( ** if they go out of sync. */ extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); - mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; + mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); for(i=0; icolUsed & MASKBIT(BMS-1) ){ nKeyCol += pTable->nCol - BMS + 1; } - pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); - if( pIdx==0 ) return; + if( pIdx==0 ) goto end_auto_index_create; pLoop->u.btree.pIndex = pIdx; pIdx->zName = "auto-index"; pIdx->pTable = pTable; @@ -115506,7 +131674,7 @@ static void constructAutomaticIndex( idxCols |= cMask; pIdx->aiColumn[n] = pTerm->u.leftColumn; pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - pIdx->azColl[n] = ALWAYS(pColl) ? pColl->zName : "BINARY"; + pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; n++; } } @@ -115518,20 +131686,20 @@ static void constructAutomaticIndex( for(i=0; iaiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; + pIdx->azColl[n] = sqlite3StrBINARY; n++; } } if( pSrc->colUsed & MASKBIT(BMS-1) ){ for(i=BMS-1; inCol; i++){ pIdx->aiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; + pIdx->azColl[n] = sqlite3StrBINARY; n++; } } assert( n==nKeyCol ); - pIdx->aiColumn[n] = -1; - pIdx->azColl[n] = "BINARY"; + pIdx->aiColumn[n] = XN_ROWID; + pIdx->azColl[n] = sqlite3StrBINARY; /* Create the automatic index */ assert( pLevel->iIdxCur>=0 ); @@ -115541,18 +131709,48 @@ static void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + sqlite3ExprCachePush(pParse); + pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; + if( pTabItem->fg.viaCoroutine ){ + int regYield = pTabItem->regReturn; + addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + }else{ + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + } + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; + } regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); + regBase = sqlite3GenerateIndexKey( + pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 + ); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); + if( pTabItem->fg.viaCoroutine ){ + sqlite3VdbeChangeP2(v, addrCounter, regBase+n); + translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1); + sqlite3VdbeGoto(v, addrTop); + pTabItem->fg.viaCoroutine = 0; + }else{ + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); + } sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ExprCachePop(pParse); /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ @@ -115565,8 +131763,10 @@ static void constructAutomaticIndex( static sqlite3_index_info *allocateIndexInfo( Parse *pParse, WhereClause *pWC, + Bitmask mUnusable, /* Ignore terms with these prereqs */ struct SrcList_item *pSrc, - ExprList *pOrderBy + ExprList *pOrderBy, + u16 *pmNoOmit /* Mask of terms not to omit */ ){ int i, j; int nTerm; @@ -115576,17 +131776,21 @@ static sqlite3_index_info *allocateIndexInfo( WhereTerm *pTerm; int nOrderBy; sqlite3_index_info *pIdxInfo; + u16 mNoOmit = 0; /* Count the number of possible WHERE clause constraints referring ** to this virtual table */ for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); testcase( pTerm->eOperator & WO_IN ); testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IS ); testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( pTerm->u.leftColumn>=(-1) ); nTerm++; } @@ -115634,16 +131838,22 @@ static sqlite3_index_info *allocateIndexInfo( for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ u8 op; if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_IS ); testcase( pTerm->eOperator & WO_ISNULL ); testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( pTerm->u.leftColumn>=(-1) ); pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; op = (u8)pTerm->eOperator & WO_ALL; if( op==WO_IN ) op = WO_EQ; + if( op==WO_MATCH ){ + op = pTerm->eMatchOp; + } pIdxCons[j].op = op; /* The direct assignment in the previous line is possible only because ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The @@ -115655,6 +131865,15 @@ static sqlite3_index_info *allocateIndexInfo( assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + + if( op & (WO_LT|WO_LE|WO_GT|WO_GE) + && sqlite3ExprIsVector(pTerm->pExpr->pRight) + ){ + if( i<16 ) mNoOmit |= (1 << i); + if( op==WO_LT ) pIdxCons[j].op = WO_LE; + if( op==WO_GT ) pIdxCons[j].op = WO_GE; + } + j++; } for(i=0; ia[i].sortOrder; } + *pmNoOmit = mNoOmit; return pIdxInfo; } @@ -115682,7 +131902,6 @@ static sqlite3_index_info *allocateIndexInfo( */ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; - int i; int rc; TRACE_IDX_INPUTS(p); @@ -115691,7 +131910,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ){ - pParse->db->mallocFailed = 1; + sqlite3OomFault(pParse->db); }else if( !pVtab->zErrMsg ){ sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); }else{ @@ -115701,29 +131920,36 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; +#if 0 + /* This error is now caught by the caller. + ** Search for "xBestIndex malfunction" below */ for(i=0; inConstraint; i++){ if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ sqlite3ErrorMsg(pParse, "table %s: xBestIndex returned an invalid plan", pTab->zName); } } +#endif return pParse->nErr; } #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ - #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Estimate the location of a particular key among all keys in an ** index. Store the results in aStat as follows: ** -** aStat[0] Est. number of rows less than pVal -** aStat[1] Est. number of rows equal to pVal +** aStat[0] Est. number of rows less than pRec +** aStat[1] Est. number of rows equal to pRec ** -** Return SQLITE_OK on success. +** Return the index of the sample that is the smallest sample that +** is greater than or equal to pRec. Note that this index is not an index +** into the aSample[] array - it is an index into a virtual set of samples +** based on the contents of aSample[] and the number of fields in record +** pRec. */ -static void whereKeyStats( +static int whereKeyStats( Parse *pParse, /* Database connection */ Index *pIdx, /* Index to consider domain of */ UnpackedRecord *pRec, /* Vector of values to consider */ @@ -115732,67 +131958,158 @@ static void whereKeyStats( ){ IndexSample *aSample = pIdx->aSample; int iCol; /* Index of required stats in anEq[] etc. */ + int i; /* Index of first sample >= pRec */ + int iSample; /* Smallest sample larger than or equal to pRec */ int iMin = 0; /* Smallest sample not yet tested */ - int i = pIdx->nSample; /* Smallest sample larger than or equal to pRec */ int iTest; /* Next sample to test */ int res; /* Result of comparison operation */ + int nField; /* Number of fields in pRec */ + tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */ #ifndef SQLITE_DEBUG UNUSED_PARAMETER( pParse ); #endif assert( pRec!=0 ); - iCol = pRec->nField - 1; assert( pIdx->nSample>0 ); - assert( pRec->nField>0 && iColnSampleCol ); + assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); + + /* Do a binary search to find the first sample greater than or equal + ** to pRec. If pRec contains a single field, the set of samples to search + ** is simply the aSample[] array. If the samples in aSample[] contain more + ** than one fields, all fields following the first are ignored. + ** + ** If pRec contains N fields, where N is more than one, then as well as the + ** samples in aSample[] (truncated to N fields), the search also has to + ** consider prefixes of those samples. For example, if the set of samples + ** in aSample is: + ** + ** aSample[0] = (a, 5) + ** aSample[1] = (a, 10) + ** aSample[2] = (b, 5) + ** aSample[3] = (c, 100) + ** aSample[4] = (c, 105) + ** + ** Then the search space should ideally be the samples above and the + ** unique prefixes [a], [b] and [c]. But since that is hard to organize, + ** the code actually searches this set: + ** + ** 0: (a) + ** 1: (a, 5) + ** 2: (a, 10) + ** 3: (a, 10) + ** 4: (b) + ** 5: (b, 5) + ** 6: (c) + ** 7: (c, 100) + ** 8: (c, 105) + ** 9: (c, 105) + ** + ** For each sample in the aSample[] array, N samples are present in the + ** effective sample array. In the above, samples 0 and 1 are based on + ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. + ** + ** Often, sample i of each block of N effective samples has (i+1) fields. + ** Except, each sample may be extended to ensure that it is greater than or + ** equal to the previous sample in the array. For example, in the above, + ** sample 2 is the first sample of a block of N samples, so at first it + ** appears that it should be 1 field in size. However, that would make it + ** smaller than sample 1, so the binary search would not work. As a result, + ** it is extended to two fields. The duplicates that this creates do not + ** cause any problems. + */ + nField = pRec->nField; + iCol = 0; + iSample = pIdx->nSample * nField; do{ - iTest = (iMin+i)/2; - res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec, 0); - if( res<0 ){ - iMin = iTest+1; + int iSamp; /* Index in aSample[] of test sample */ + int n; /* Number of fields in test sample */ + + iTest = (iMin+iSample)/2; + iSamp = iTest / nField; + if( iSamp>0 ){ + /* The proposed effective sample is a prefix of sample aSample[iSamp]. + ** Specifically, the shortest prefix of at least (1 + iTest%nField) + ** fields that is greater than the previous effective sample. */ + for(n=(iTest % nField) + 1; nnField = n; + res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec); + if( res<0 ){ + iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1]; + iMin = iTest+1; + }else if( res==0 && nnSample ); - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0) - || pParse->db->mallocFailed ); - }else{ - /* Otherwise, pRec must be smaller than sample $i and larger than - ** sample ($i-1). */ - assert( i==pIdx->nSample - || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0)>0 - || pParse->db->mallocFailed ); - assert( i==0 - || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec, 0)<0 - || pParse->db->mallocFailed ); + if( pParse->db->mallocFailed==0 ){ + if( res==0 ){ + /* If (res==0) is true, then pRec must be equal to sample i. */ + assert( inSample ); + assert( iCol==nField-1 ); + pRec->nField = nField; + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + || pParse->db->mallocFailed + ); + }else{ + /* Unless i==pIdx->nSample, indicating that pRec is larger than + ** all samples in the aSample[] array, pRec must be smaller than the + ** (iCol+1) field prefix of sample i. */ + assert( i<=pIdx->nSample && i>=0 ); + pRec->nField = iCol+1; + assert( i==pIdx->nSample + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 + || pParse->db->mallocFailed ); + + /* if i==0 and iCol==0, then record pRec is smaller than all samples + ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must + ** be greater than or equal to the (iCol) field prefix of sample i. + ** If (i>0), then pRec must also be greater than sample (i-1). */ + if( iCol>0 ){ + pRec->nField = iCol; + assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 + || pParse->db->mallocFailed ); + } + if( i>0 ){ + pRec->nField = nField; + assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 + || pParse->db->mallocFailed ); + } + } } #endif /* ifdef SQLITE_DEBUG */ - /* At this point, aSample[i] is the first sample that is greater than - ** or equal to pVal. Or if i==pIdx->nSample, then all samples are less - ** than pVal. If aSample[i]==pVal, then res==0. - */ if( res==0 ){ + /* Record pRec is equal to sample i */ + assert( iCol==nField-1 ); aStat[0] = aSample[i].anLt[iCol]; aStat[1] = aSample[i].anEq[iCol]; }else{ - tRowcnt iLower, iUpper, iGap; - if( i==0 ){ - iLower = 0; - iUpper = aSample[0].anLt[iCol]; + /* At this point, the (iCol+1) field prefix of aSample[i] is the first + ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec + ** is larger than all samples in the array. */ + tRowcnt iUpper, iGap; + if( i>=pIdx->nSample ){ + iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); }else{ - i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); - iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; - iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; + iUpper = aSample[i].anLt[iCol]; } - aStat[1] = pIdx->aAvgEq[iCol]; + if( iLower>=iUpper ){ iGap = 0; }else{ @@ -115804,7 +132121,12 @@ static void whereKeyStats( iGap = iGap/3; } aStat[0] = iLower + iGap; + aStat[1] = pIdx->aAvgEq[iCol]; } + + /* Restore the pRec->nField value before returning. */ + pRec->nField = nField; + return i; } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ @@ -115831,6 +132153,21 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ return nRet; } + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Return the affinity for a single column of an index. +*/ +SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ + assert( iCol>=0 && iColnColumn ); + if( !pIdx->zColAff ){ + if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; + } + return pIdx->zColAff[iCol]; +} +#endif + + #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** This function is called to estimate the number of rows visited by a @@ -115880,8 +132217,7 @@ static int whereRangeSkipScanEst( int nLower = -1; int nUpper = p->nSample+1; int rc = SQLITE_OK; - int iCol = p->aiColumn[nEq]; - u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; + u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); CollSeq *pColl; sqlite3_value *p1 = 0; /* Value extracted from pLower */ @@ -115955,7 +132291,7 @@ static int whereRangeSkipScanEst( ** If either of the upper or lower bound is not present, then NULL is passed in ** place of the corresponding WhereTerm. ** -** The value in (pBuilder->pNew->u.btree.nEq) is the index of the index +** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index ** column subject to the range constraint. Or, equivalently, the number of ** equality constraints optimized by the proposed index scan. For example, ** assuming index p is on t1(a, b), and the SQL query is: @@ -115971,9 +132307,9 @@ static int whereRangeSkipScanEst( ** ** When this function is called, *pnOut is set to the sqlite3LogEst() of the ** number of rows that the index scan is expected to visit without -** considering the range constraints. If nEq is 0, this is the number of +** considering the range constraints. If nEq is 0, then *pnOut is the number of ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) -** to account for the range contraints pLower and pUpper. +** to account for the range constraints pLower and pUpper. ** ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be ** used, a single range inequality reduces the search space by a factor of 4. @@ -115995,14 +132331,12 @@ static int whereRangeScanEst( Index *p = pLoop->u.btree.pIndex; int nEq = pLoop->u.btree.nEq; - if( p->nSample>0 - && nEqnSampleCol - && OptimizationEnabled(pParse->db, SQLITE_Stat3) - ){ + if( p->nSample>0 && nEqnSampleCol ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; tRowcnt a[2]; - u8 aff; + int nBtm = pLoop->u.btree.nBtm; + int nTop = pLoop->u.btree.nTop; /* Variable iLower will be set to the estimate of the number of rows in ** the index that are less than the lower bound of the range query. The @@ -116014,25 +132348,28 @@ static int whereRangeScanEst( ** is not a simple variable or literal value), the lower bound of the ** range is $P. Due to a quirk in the way whereKeyStats() works, even ** if $L is available, whereKeyStats() is called for both ($P) and - ** ($P:$L) and the larger of the two returned values used. + ** ($P:$L) and the larger of the two returned values is used. ** ** Similarly, iUpper is to be set to the estimate of the number of rows ** less than the upper bound of the range query. Where the upper bound ** is either ($P) or ($P:$U). Again, even if $U is available, both values ** of iUpper are requested of whereKeyStats() and the smaller used. + ** + ** The number of rows between the two bounds is then just iUpper-iLower. */ - tRowcnt iLower; - tRowcnt iUpper; + tRowcnt iLower; /* Rows less than the lower bound */ + tRowcnt iUpper; /* Rows less than the upper bound */ + int iLwrIdx = -2; /* aSample[] for the lower bound */ + int iUprIdx = -1; /* aSample[] for the upper bound */ - if( nEq==p->nKeyCol ){ - aff = SQLITE_AFF_INTEGER; - }else{ - aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; + if( pRec ){ + testcase( pRec->nField!=pBuilder->nRecValid ); + pRec->nField = pBuilder->nRecValid; } /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); + iUpper = p->nRowEst0; }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ @@ -116041,33 +132378,46 @@ static int whereRangeScanEst( iUpper = a[0] + a[1]; } + assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + assert( p->aSortOrder!=0 ); + if( p->aSortOrder[nEq] ){ + /* The roles of pLower and pUpper are swapped for a DESC index */ + SWAP(WhereTerm*, pLower, pUpper); + SWAP(int, nBtm, nTop); + } + /* If possible, improve on the iLower estimate using ($P:$L). */ if( pLower ){ - int bOk; /* True if value is extracted from pExpr */ + int n; /* Values extracted from pExpr */ Expr *pExpr = pLower->pExpr->pRight; - assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n); + if( rc==SQLITE_OK && n ){ tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 0, a); - iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); + u16 mask = WO_GT|WO_LE; + if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); + iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0); if( iNew>iLower ) iLower = iNew; nOut--; + pLower = 0; } } /* If possible, improve on the iUpper estimate using ($P:$U). */ if( pUpper ){ - int bOk; /* True if value is extracted from pExpr */ + int n; /* Values extracted from pExpr */ Expr *pExpr = pUpper->pExpr->pRight; - assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n); + if( rc==SQLITE_OK && n ){ tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 1, a); - iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); + u16 mask = WO_GT|WO_LE; + if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); + iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0); if( iNewiLower ){ nNew = sqlite3LogEst(iUpper - iLower); + /* TUNING: If both iUpper and iLower are derived from the same + ** sample, then assume they are 4x more selective. This brings + ** the estimated selectivity more in line with what it would be + ** if estimated without the use of STAT3/4 tables. */ + if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); }else{ nNew = 10; assert( 10==sqlite3LogEst(2) ); } if( nNewnOut = (LogEst)nOut; - WHERETRACE(0x10, ("range scan regions: %u..%u est=%d\n", + WHERETRACE(0x10, ("STAT4 range scan: %u..%u est=%d\n", (u32)iLower, (u32)iUpper, nOut)); - return SQLITE_OK; } }else{ int bDone = 0; @@ -116095,22 +132448,31 @@ static int whereRangeScanEst( #else UNUSED_PARAMETER(pParse); UNUSED_PARAMETER(pBuilder); -#endif assert( pLower || pUpper ); +#endif assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); - /* TUNING: If there is both an upper and lower limit, assume the range is + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ - if( pLower && pUpper ) nNew -= 20; + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; + } nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewnOut>nOut ){ + WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + pLoop->nOut, nOut)); + } +#endif pLoop->nOut = (LogEst)nOut; return rc; } @@ -116142,7 +132504,6 @@ static int whereEqualScanEst( Index *p = pBuilder->pNew->u.btree.pIndex; int nEq = pBuilder->pNew->u.btree.nEq; UnpackedRecord *pRec = pBuilder->pRec; - u8 aff; /* Column affinity */ int rc; /* Subfunction return code */ tRowcnt a[2]; /* Statistics */ int bOk; @@ -116166,15 +132527,15 @@ static int whereEqualScanEst( return SQLITE_OK; } - aff = p->pTable->aCol[p->aiColumn[nEq-1]].affinity; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk); pBuilder->pRec = pRec; if( rc!=SQLITE_OK ) return rc; if( bOk==0 ) return SQLITE_NOTFOUND; pBuilder->nRecValid = nEq; whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1])); + WHERETRACE(0x10,("equality scan regions %s(%d): %d\n", + p->zName, nEq-1, (int)a[1])); *pnRow = a[1]; return rc; @@ -116223,1382 +132584,62 @@ static int whereInScanEst( if( rc==SQLITE_OK ){ if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); + WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); } assert( pBuilder->nRecValid==nRecValid ); return rc; } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +#ifdef WHERETRACE_ENABLED /* -** Disable a term in the WHERE clause. Except, do not disable the term -** if it controls a LEFT OUTER JOIN and it did not originate in the ON -** or USING clause of that join. -** -** Consider the term t2.z='ok' in the following queries: -** -** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' -** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' -** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' -** -** The t2.z='ok' is disabled in the in (2) because it originates -** in the ON clause. The term is disabled in (3) because it is not part -** of a LEFT OUTER JOIN. In (1), the term is not disabled. -** -** Disabling a term causes that term to not be tested in the inner loop -** of the join. Disabling is an optimization. When terms are satisfied -** by indices, we disable them to prevent redundant tests in the inner -** loop. We would get the correct results if nothing were ever disabled, -** but joins might run a little slower. The trick is to disable as much -** as we can without disabling too much. If we disabled in (1), we'd get -** the wrong answer. See ticket #813. +** Print the content of a WhereTerm object */ -static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ - if( pTerm - && (pTerm->wtFlags & TERM_CODED)==0 - && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) - && (pLevel->notReady & pTerm->prereqAll)==0 - ){ - pTerm->wtFlags |= TERM_CODED; - if( pTerm->iParent>=0 ){ - WhereTerm *pOther = &pTerm->pWC->a[pTerm->iParent]; - if( (--pOther->nChild)==0 ){ - disableTerm(pLevel, pOther); - } - } - } -} - -/* -** Code an OP_Affinity opcode to apply the column affinity string zAff -** to the n registers starting at base. -** -** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the -** beginning and end of zAff are ignored. If all entries in zAff are -** SQLITE_AFF_NONE, then no code gets generated. -** -** This routine makes its own copy of zAff so that the caller is free -** to modify zAff after this routine returns. -*/ -static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ - Vdbe *v = pParse->pVdbe; - if( zAff==0 ){ - assert( pParse->db->mallocFailed ); - return; - } - assert( v!=0 ); - - /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning - ** and end of the affinity string. - */ - while( n>0 && zAff[0]==SQLITE_AFF_NONE ){ - n--; - base++; - zAff++; - } - while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){ - n--; - } - - /* Code the OP_Affinity opcode if there is anything left to do. */ - if( n>0 ){ - sqlite3VdbeAddOp2(v, OP_Affinity, base, n); - sqlite3VdbeChangeP4(v, -1, zAff, n); - sqlite3ExprCacheAffinityChange(pParse, base, n); - } -} - - -/* -** Generate code for a single equality term of the WHERE clause. An equality -** term can be either X=expr or X IN (...). pTerm is the term to be -** coded. -** -** The current value for the constraint is left in register iReg. -** -** For a constraint of the form X=expr, the expression is evaluated and its -** result is left on the stack. For constraints of the form X IN (...) -** this routine sets up a loop that will iterate over all values of X. -*/ -static int codeEqualityTerm( - Parse *pParse, /* The parsing context */ - WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* The level of the FROM clause we are working on */ - int iEq, /* Index of the equality term within this level */ - int bRev, /* True for reverse-order IN operations */ - int iTarget /* Attempt to leave results in this register */ -){ - Expr *pX = pTerm->pExpr; - Vdbe *v = pParse->pVdbe; - int iReg; /* Register holding results */ - - assert( iTarget>0 ); - if( pX->op==TK_EQ ){ - iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); - }else if( pX->op==TK_ISNULL ){ - iReg = iTarget; - sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); -#ifndef SQLITE_OMIT_SUBQUERY +static void whereTermPrint(WhereTerm *pTerm, int iTerm){ + if( pTerm==0 ){ + sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); }else{ - int eType; - int iTab; - struct InLoop *pIn; - WhereLoop *pLoop = pLevel->pWLoop; - - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 - && pLoop->u.btree.pIndex!=0 - && pLoop->u.btree.pIndex->aSortOrder[iEq] - ){ - testcase( iEq==0 ); - testcase( bRev ); - bRev = !bRev; - } - assert( pX->op==TK_IN ); - iReg = iTarget; - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); - if( eType==IN_INDEX_INDEX_DESC ){ - testcase( bRev ); - bRev = !bRev; - } - iTab = pX->iTable; - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); - VdbeCoverageIf(v, bRev); - VdbeCoverageIf(v, !bRev); - assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); - pLoop->wsFlags |= WHERE_IN_ABLE; - if( pLevel->u.in.nIn==0 ){ - pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - } - pLevel->u.in.nIn++; - pLevel->u.in.aInLoop = - sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, - sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); - pIn = pLevel->u.in.aInLoop; - if( pIn ){ - pIn += pLevel->u.in.nIn - 1; - pIn->iCur = iTab; - if( eType==IN_INDEX_ROWID ){ - pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); - }else{ - pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); - } - pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; - sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); + char zType[4]; + char zLeft[50]; + memcpy(zType, "...", 4); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + if( pTerm->eOperator & WO_SINGLE ){ + sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}", + pTerm->leftCursor, pTerm->u.leftColumn); + }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){ + sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%lld", + pTerm->u.pOrInfo->indexable); }else{ - pLevel->u.in.nIn = 0; + sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor); } + sqlite3DebugPrintf( + "TERM-%-3d %p %s %-12s prob=%-3d op=0x%03x wtFlags=0x%04x", + iTerm, pTerm, zType, zLeft, pTerm->truthProb, + pTerm->eOperator, pTerm->wtFlags); + if( pTerm->iField ){ + sqlite3DebugPrintf(" iField=%d\n", pTerm->iField); + }else{ + sqlite3DebugPrintf("\n"); + } + sqlite3TreeViewExpr(0, pTerm->pExpr, 0); + } +} #endif - } - disableTerm(pLevel, pTerm); - return iReg; -} +#ifdef WHERETRACE_ENABLED /* -** Generate code that will evaluate all == and IN constraints for an -** index scan. -** -** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). -** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 -** The index has as many as three equality constraints, but in this -** example, the third "c" value is an inequality. So only two -** constraints are coded. This routine will generate code to evaluate -** a==5 and b IN (1,2,3). The current values for a and b will be stored -** in consecutive registers and the index of the first register is returned. -** -** In the example above nEq==2. But this subroutine works for any value -** of nEq including 0. If nEq==0, this routine is nearly a no-op. -** The only thing it does is allocate the pLevel->iMem memory cell and -** compute the affinity string. -** -** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints -** are == or IN and are covered by the nEq. nExtraReg is 1 if there is -** an inequality constraint (such as the "c>=5 AND c<10" in the example) that -** occurs after the nEq quality constraints. -** -** This routine allocates a range of nEq+nExtraReg memory cells and returns -** the index of the first memory cell in that range. The code that -** calls this routine will use that memory range to store keys for -** start and termination conditions of the loop. -** key value of the loop. If one or more IN operators appear, then -** this routine allocates an additional nEq memory cells for internal -** use. -** -** Before returning, *pzAff is set to point to a buffer containing a -** copy of the column affinity string of the index allocated using -** sqlite3DbMalloc(). Except, entries in the copy of the string associated -** with equality constraints that use NONE affinity are set to -** SQLITE_AFF_NONE. This is to deal with SQL such as the following: -** -** CREATE TABLE t1(a TEXT PRIMARY KEY, b); -** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; -** -** In the example above, the index on t1(a) has TEXT affinity. But since -** the right hand side of the equality constraint (t2.b) has NONE affinity, -** no conversion should be attempted before using a t2.b value as part of -** a key to search the index. Hence the first byte in the returned affinity -** string in this example would be set to SQLITE_AFF_NONE. +** Show the complete content of a WhereClause */ -static int codeAllEqualityTerms( - Parse *pParse, /* Parsing context */ - WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ - int bRev, /* Reverse the order of IN operators */ - int nExtraReg, /* Number of extra registers to allocate */ - char **pzAff /* OUT: Set to point to affinity string */ -){ - u16 nEq; /* The number of == or IN constraints to code */ - u16 nSkip; /* Number of left-most columns to skip */ - Vdbe *v = pParse->pVdbe; /* The vm under construction */ - Index *pIdx; /* The index being used for this loop */ - WhereTerm *pTerm; /* A single constraint term */ - WhereLoop *pLoop; /* The WhereLoop object */ - int j; /* Loop counter */ - int regBase; /* Base register */ - int nReg; /* Number of registers to allocate */ - char *zAff; /* Affinity string to return */ - - /* This module is only called on query plans that use an index. */ - pLoop = pLevel->pWLoop; - assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); - nEq = pLoop->u.btree.nEq; - nSkip = pLoop->u.btree.nSkip; - pIdx = pLoop->u.btree.pIndex; - assert( pIdx!=0 ); - - /* Figure out how many memory cells we will need then allocate them. - */ - regBase = pParse->nMem + 1; - nReg = pLoop->u.btree.nEq + nExtraReg; - pParse->nMem += nReg; - - zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); - if( !zAff ){ - pParse->db->mallocFailed = 1; +SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ + int i; + for(i=0; inTerm; i++){ + whereTermPrint(&pWC->a[i], i); } - - if( nSkip ){ - int iIdxCur = pLevel->iIdxCur; - sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); - j = sqlite3VdbeAddOp0(v, OP_Goto); - pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), - iIdxCur, 0, regBase, nSkip); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - sqlite3VdbeJumpHere(v, j); - for(j=0; jaiColumn[j]>=0 ); - VdbeComment((v, "%s", pIdx->pTable->aCol[pIdx->aiColumn[j]].zName)); - } - } - - /* Evaluate the equality constraints - */ - assert( zAff==0 || (int)strlen(zAff)>=nEq ); - for(j=nSkip; jaLTerm[j]; - assert( pTerm!=0 ); - /* The following testcase is true for indices with redundant columns. - ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ - testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); - if( r1!=regBase+j ){ - if( nReg==1 ){ - sqlite3ReleaseTempReg(pParse, regBase); - regBase = r1; - }else{ - sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); - } - } - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_IN ); - if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ - Expr *pRight = pTerm->pExpr->pRight; - if( sqlite3ExprCanBeNull(pRight) ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); - VdbeCoverage(v); - } - if( zAff ){ - if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ - zAff[j] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ - zAff[j] = SQLITE_AFF_NONE; - } - } - } - } - *pzAff = zAff; - return regBase; } - -#ifndef SQLITE_OMIT_EXPLAIN -/* -** This routine is a helper for explainIndexRange() below -** -** pStr holds the text of an expression that we are building up one term -** at a time. This routine adds a new term to the end of the expression. -** Terms are separated by AND so add the "AND" text for second and subsequent -** terms only. -*/ -static void explainAppendTerm( - StrAccum *pStr, /* The text expression being built */ - int iTerm, /* Index of this term. First is zero */ - const char *zColumn, /* Name of the column */ - const char *zOp /* Name of the operator */ -){ - if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); - sqlite3StrAccumAppendAll(pStr, zColumn); - sqlite3StrAccumAppend(pStr, zOp, 1); - sqlite3StrAccumAppend(pStr, "?", 1); -} - -/* -** Argument pLevel describes a strategy for scanning table pTab. This -** function returns a pointer to a string buffer containing a description -** of the subset of table rows scanned by the strategy in the form of an -** SQL expression. Or, if all rows are scanned, NULL is returned. -** -** For example, if the query: -** -** SELECT * FROM t1 WHERE a=1 AND b>2; -** -** is run and there is an index on (a, b), then this function returns a -** string similar to: -** -** "a=? AND b>?" -** -** The returned pointer points to memory obtained from sqlite3DbMalloc(). -** It is the responsibility of the caller to free the buffer when it is -** no longer required. -*/ -static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ - Index *pIndex = pLoop->u.btree.pIndex; - u16 nEq = pLoop->u.btree.nEq; - u16 nSkip = pLoop->u.btree.nSkip; - int i, j; - Column *aCol = pTab->aCol; - i16 *aiColumn = pIndex->aiColumn; - StrAccum txt; - - if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ - return 0; - } - sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); - txt.db = db; - sqlite3StrAccumAppend(&txt, " (", 2); - for(i=0; i=nSkip ){ - explainAppendTerm(&txt, i, z, "="); - }else{ - if( i ) sqlite3StrAccumAppend(&txt, " AND ", 5); - sqlite3StrAccumAppend(&txt, "ANY(", 4); - sqlite3StrAccumAppendAll(&txt, z); - sqlite3StrAccumAppend(&txt, ")", 1); - } - } - - j = i; - if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(&txt, i++, z, ">"); - } - if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(&txt, i, z, "<"); - } - sqlite3StrAccumAppend(&txt, ")", 1); - return sqlite3StrAccumFinish(&txt); -} - -/* -** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single -** record is added to the output to describe the table scan strategy in -** pLevel. -*/ -static void explainOneScan( - Parse *pParse, /* Parse context */ - SrcList *pTabList, /* Table list this loop refers to */ - WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ - int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ - u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ -){ -#ifndef SQLITE_DEBUG - if( pParse->explain==2 ) #endif - { - struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; - Vdbe *v = pParse->pVdbe; /* VM being constructed */ - sqlite3 *db = pParse->db; /* Database handle */ - char *zMsg; /* Text to add to EQP output */ - int iId = pParse->iSelectId; /* Select id (left-most output column) */ - int isSearch; /* True for a SEARCH. False for SCAN. */ - WhereLoop *pLoop; /* The controlling WhereLoop object */ - u32 flags; /* Flags that describe this loop */ - - pLoop = pLevel->pWLoop; - flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; - - isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) - || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); - - zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); - if( pItem->pSelect ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); - }else{ - zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName); - } - - if( pItem->zAlias ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); - } - if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 - && ALWAYS(pLoop->u.btree.pIndex!=0) - ){ - const char *zFmt; - Index *pIdx = pLoop->u.btree.pIndex; - char *zWhere = explainIndexRange(db, pLoop, pItem->pTab); - assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); - if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ - zFmt = zWhere ? "%s USING PRIMARY KEY%.0s%s" : "%s%.0s%s"; - }else if( flags & WHERE_AUTO_INDEX ){ - zFmt = "%s USING AUTOMATIC COVERING INDEX%.0s%s"; - }else if( flags & WHERE_IDX_ONLY ){ - zFmt = "%s USING COVERING INDEX %s%s"; - }else{ - zFmt = "%s USING INDEX %s%s"; - } - zMsg = sqlite3MAppendf(db, zMsg, zFmt, zMsg, pIdx->zName, zWhere); - sqlite3DbFree(db, zWhere); - }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); - - if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); - }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid?)", zMsg); - }else if( ALWAYS(flags&WHERE_TOP_LIMIT) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowidu.vtab.idxNum, pLoop->u.vtab.idxStr); - } -#endif - zMsg = sqlite3MAppendf(db, zMsg, "%s", zMsg); - sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); - } -} -#else -# define explainOneScan(u,v,w,x,y,z) -#endif /* SQLITE_OMIT_EXPLAIN */ - - -/* -** Generate code for the start of the iLevel-th loop in the WHERE clause -** implementation described by pWInfo. -*/ -static Bitmask codeOneLoopStart( - WhereInfo *pWInfo, /* Complete information about the WHERE clause */ - int iLevel, /* Which level of pWInfo->a[] should be coded */ - Bitmask notReady /* Which tables are currently available */ -){ - int j, k; /* Loop counters */ - int iCur; /* The VDBE cursor for the table */ - int addrNxt; /* Where to jump to continue with the next IN case */ - int omitTable; /* True if we use the index only */ - int bRev; /* True if we need to scan in reverse order */ - WhereLevel *pLevel; /* The where level to be coded */ - WhereLoop *pLoop; /* The WhereLoop object being coded */ - WhereClause *pWC; /* Decomposition of the entire WHERE clause */ - WhereTerm *pTerm; /* A WHERE clause term */ - Parse *pParse; /* Parsing context */ - sqlite3 *db; /* Database connection */ - Vdbe *v; /* The prepared stmt under constructions */ - struct SrcList_item *pTabItem; /* FROM clause term being coded */ - int addrBrk; /* Jump here to break out of the loop */ - int addrCont; /* Jump here to continue with next cycle */ - int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ - int iReleaseReg = 0; /* Temp register to free before returning */ - - pParse = pWInfo->pParse; - v = pParse->pVdbe; - pWC = &pWInfo->sWC; - db = pParse->db; - pLevel = &pWInfo->a[iLevel]; - pLoop = pLevel->pWLoop; - pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; - iCur = pTabItem->iCursor; - pLevel->notReady = notReady & ~getMask(&pWInfo->sMaskSet, iCur); - bRev = (pWInfo->revMask>>iLevel)&1; - omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 - && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; - VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); - - /* Create labels for the "break" and "continue" instructions - ** for the current loop. Jump to addrBrk to break out of a loop. - ** Jump to cont to go immediately to the next iteration of the - ** loop. - ** - ** When there is an IN operator, we also have a "addrNxt" label that - ** means to continue with the next IN value combination. When - ** there are no IN operators in the constraints, the "addrNxt" label - ** is the same as "addrBrk". - */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); - - /* If this is the right table of a LEFT OUTER JOIN, allocate and - ** initialize a memory cell that records if this table matches any - ** row of the left table of the join. - */ - if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ - pLevel->iLeftJoin = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); - } - - /* Special case of a FROM clause subquery implemented as a co-routine */ - if( pTabItem->viaCoroutine ){ - int regYield = pTabItem->regReturn; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); - pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); - VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); - pLevel->op = OP_Goto; - }else - -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - /* Case 1: The table is a virtual-table. Use the VFilter and VNext - ** to access the data. - */ - int iReg; /* P3 Value for OP_VFilter */ - int addrNotFound; - int nConstraint = pLoop->nLTerm; - - sqlite3ExprCachePush(pParse); - iReg = sqlite3GetTempRange(pParse, nConstraint+2); - addrNotFound = pLevel->addrBrk; - for(j=0; jaLTerm[j]; - if( pTerm==0 ) continue; - if( pTerm->eOperator & WO_IN ){ - codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); - addrNotFound = pLevel->addrNxt; - }else{ - sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); - } - } - sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); - sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, - pLoop->u.vtab.idxStr, - pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); - VdbeCoverage(v); - pLoop->u.vtab.needFree = 0; - for(j=0; ju.vtab.omitMask>>j)&1 ){ - disableTerm(pLevel, pLoop->aLTerm[j]); - } - } - pLevel->op = OP_VNext; - pLevel->p1 = iCur; - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); - sqlite3ExprCachePop(pParse); - }else -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - - if( (pLoop->wsFlags & WHERE_IPK)!=0 - && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 - ){ - /* Case 2: We can directly reference a single row using an - ** equality comparison against the ROWID field. Or - ** we reference multiple rows using a "rowid IN (...)" - ** construct. - */ - assert( pLoop->u.btree.nEq==1 ); - pTerm = pLoop->aLTerm[0]; - assert( pTerm!=0 ); - assert( pTerm->pExpr!=0 ); - assert( omitTable==0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - iReleaseReg = ++pParse->nMem; - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); - if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); - addrNxt = pLevel->addrNxt; - sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); - VdbeCoverage(v); - sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - VdbeComment((v, "pk")); - pLevel->op = OP_Noop; - }else if( (pLoop->wsFlags & WHERE_IPK)!=0 - && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 - ){ - /* Case 3: We have an inequality comparison against the ROWID field. - */ - int testOp = OP_Noop; - int start; - int memEndValue = 0; - WhereTerm *pStart, *pEnd; - - assert( omitTable==0 ); - j = 0; - pStart = pEnd = 0; - if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; - if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; - assert( pStart!=0 || pEnd!=0 ); - if( bRev ){ - pTerm = pStart; - pStart = pEnd; - pEnd = pTerm; - } - if( pStart ){ - Expr *pX; /* The expression that defines the start bound */ - int r1, rTemp; /* Registers for holding the start boundary */ - - /* The following constant maps TK_xx codes into corresponding - ** seek opcodes. It depends on a particular ordering of TK_xx - */ - const u8 aMoveOp[] = { - /* TK_GT */ OP_SeekGT, - /* TK_LE */ OP_SeekLE, - /* TK_LT */ OP_SeekLT, - /* TK_GE */ OP_SeekGE - }; - assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ - assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ - assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ - - assert( (pStart->wtFlags & TERM_VNULL)==0 ); - testcase( pStart->wtFlags & TERM_VIRTUAL ); - pX = pStart->pExpr; - assert( pX!=0 ); - testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ - r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); - sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); - VdbeComment((v, "pk")); - VdbeCoverageIf(v, pX->op==TK_GT); - VdbeCoverageIf(v, pX->op==TK_LE); - VdbeCoverageIf(v, pX->op==TK_LT); - VdbeCoverageIf(v, pX->op==TK_GE); - sqlite3ExprCacheAffinityChange(pParse, r1, 1); - sqlite3ReleaseTempReg(pParse, rTemp); - disableTerm(pLevel, pStart); - }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - } - if( pEnd ){ - Expr *pX; - pX = pEnd->pExpr; - assert( pX!=0 ); - assert( (pEnd->wtFlags & TERM_VNULL)==0 ); - testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ - testcase( pEnd->wtFlags & TERM_VIRTUAL ); - memEndValue = ++pParse->nMem; - sqlite3ExprCode(pParse, pX->pRight, memEndValue); - if( pX->op==TK_LT || pX->op==TK_GT ){ - testOp = bRev ? OP_Le : OP_Ge; - }else{ - testOp = bRev ? OP_Lt : OP_Gt; - } - disableTerm(pLevel, pEnd); - } - start = sqlite3VdbeCurrentAddr(v); - pLevel->op = bRev ? OP_Prev : OP_Next; - pLevel->p1 = iCur; - pLevel->p2 = start; - assert( pLevel->p5==0 ); - if( testOp!=OP_Noop ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); - VdbeCoverageIf(v, testOp==OP_Le); - VdbeCoverageIf(v, testOp==OP_Lt); - VdbeCoverageIf(v, testOp==OP_Ge); - VdbeCoverageIf(v, testOp==OP_Gt); - sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); - } - }else if( pLoop->wsFlags & WHERE_INDEXED ){ - /* Case 4: A scan using an index. - ** - ** The WHERE clause may contain zero or more equality - ** terms ("==" or "IN" operators) that refer to the N - ** left-most columns of the index. It may also contain - ** inequality constraints (>, <, >= or <=) on the indexed - ** column that immediately follows the N equalities. Only - ** the right-most column can be an inequality - the rest must - ** use the "==" and "IN" operators. For example, if the - ** index is on (x,y,z), then the following clauses are all - ** optimized: - ** - ** x=5 - ** x=5 AND y=10 - ** x=5 AND y<10 - ** x=5 AND y>5 AND y<10 - ** x=5 AND y=5 AND z<=10 - ** - ** The z<10 term of the following cannot be used, only - ** the x=5 term: - ** - ** x=5 AND z<10 - ** - ** N may be zero if there are inequality constraints. - ** If there are no inequality constraints, then N is at - ** least one. - ** - ** This case is also used when there are no WHERE clause - ** constraints but an index is selected anyway, in order - ** to force the output order to conform to an ORDER BY. - */ - static const u8 aStartOp[] = { - 0, - 0, - OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ - OP_Last, /* 3: (!start_constraints && startEq && bRev) */ - OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ - OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ - OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ - OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ - }; - static const u8 aEndOp[] = { - OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ - OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ - OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ - OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ - }; - u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ - int regBase; /* Base register holding constraint values */ - WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ - WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ - int startEq; /* True if range start uses ==, >= or <= */ - int endEq; /* True if range end uses ==, >= or <= */ - int start_constraints; /* Start of range is constrained */ - int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ - int iIdxCur; /* The VDBE cursor for the index */ - int nExtraReg = 0; /* Number of extra registers needed */ - int op; /* Instruction opcode */ - char *zStartAff; /* Affinity for start of range constraint */ - char cEndAff = 0; /* Affinity for end of range constraint */ - u8 bSeekPastNull = 0; /* True to seek past initial nulls */ - u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ - - pIdx = pLoop->u.btree.pIndex; - iIdxCur = pLevel->iIdxCur; - assert( nEq>=pLoop->u.btree.nSkip ); - - /* If this loop satisfies a sort order (pOrderBy) request that - ** was passed to this function to implement a "SELECT min(x) ..." - ** query, then the caller will only allow the loop to run for - ** a single iteration. This means that the first row returned - ** should not have a NULL value stored in 'x'. If column 'x' is - ** the first one after the nEq equality constraints in the index, - ** this requires some special handling. - */ - assert( pWInfo->pOrderBy==0 - || pWInfo->pOrderBy->nExpr==1 - || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); - if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && pWInfo->nOBSat>0 - && (pIdx->nKeyCol>nEq) - ){ - assert( pLoop->u.btree.nSkip==0 ); - bSeekPastNull = 1; - nExtraReg = 1; - } - - /* Find any inequality constraint terms for the start and end - ** of the range. - */ - j = nEq; - if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ - pRangeStart = pLoop->aLTerm[j++]; - nExtraReg = 1; - } - if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ - pRangeEnd = pLoop->aLTerm[j++]; - nExtraReg = 1; - if( pRangeStart==0 - && (j = pIdx->aiColumn[nEq])>=0 - && pIdx->pTable->aCol[j].notNull==0 - ){ - bSeekPastNull = 1; - } - } - assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); - - /* Generate code to evaluate all constraint terms using == or IN - ** and store the values of those terms in an array of registers - ** starting at regBase. - */ - regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); - assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); - if( zStartAff ) cEndAff = zStartAff[nEq]; - addrNxt = pLevel->addrNxt; - - /* If we are doing a reverse order scan on an ascending index, or - ** a forward order scan on a descending index, interchange the - ** start and end terms (pRangeStart and pRangeEnd). - */ - if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) - || (bRev && pIdx->nKeyCol==nEq) - ){ - SWAP(WhereTerm *, pRangeEnd, pRangeStart); - SWAP(u8, bSeekPastNull, bStopAtNull); - } - - testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); - testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); - testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); - testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); - startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); - endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); - start_constraints = pRangeStart || nEq>0; - - /* Seek the index cursor to the start of the range. */ - nConstraint = nEq; - if( pRangeStart ){ - Expr *pRight = pRangeStart->pExpr->pRight; - sqlite3ExprCode(pParse, pRight, regBase+nEq); - if( (pRangeStart->wtFlags & TERM_VNULL)==0 - && sqlite3ExprCanBeNull(pRight) - ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - VdbeCoverage(v); - } - if( zStartAff ){ - if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){ - /* Since the comparison is to be performed with no conversions - ** applied to the operands, set the affinity to apply to pRight to - ** SQLITE_AFF_NONE. */ - zStartAff[nEq] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ - zStartAff[nEq] = SQLITE_AFF_NONE; - } - } - nConstraint++; - testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); - }else if( bSeekPastNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - nConstraint++; - startEq = 0; - start_constraints = 1; - } - codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); - op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; - assert( op!=0 ); - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - VdbeCoverage(v); - VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); - VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); - VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); - VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); - VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); - VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); - - /* Load the value for the inequality constraint at the end of the - ** range (if any). - */ - nConstraint = nEq; - if( pRangeEnd ){ - Expr *pRight = pRangeEnd->pExpr->pRight; - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); - sqlite3ExprCode(pParse, pRight, regBase+nEq); - if( (pRangeEnd->wtFlags & TERM_VNULL)==0 - && sqlite3ExprCanBeNull(pRight) - ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - VdbeCoverage(v); - } - if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_NONE - && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) - ){ - codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff); - } - nConstraint++; - testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); - }else if( bStopAtNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - endEq = 0; - nConstraint++; - } - sqlite3DbFree(db, zStartAff); - - /* Top of the loop body */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - - /* Check if the index cursor is past the end of the range. */ - if( nConstraint ){ - op = aEndOp[bRev*2 + endEq]; - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); - testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); - testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); - testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); - } - - /* Seek the table cursor, if required */ - disableTerm(pLevel, pRangeStart); - disableTerm(pLevel, pRangeEnd); - if( omitTable ){ - /* pIdx is a covering index. No need to access the main table. */ - }else if( HasRowid(pIdx->pTable) ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ - }else if( iCur!=iIdxCur ){ - Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); - iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); - for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); - } - sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, - iRowidReg, pPk->nKeyCol); VdbeCoverage(v); - } - - /* Record the instruction used to terminate the loop. Disable - ** WHERE clause terms made redundant by the index range scan. - */ - if( pLoop->wsFlags & WHERE_ONEROW ){ - pLevel->op = OP_Noop; - }else if( bRev ){ - pLevel->op = OP_Prev; - }else{ - pLevel->op = OP_Next; - } - pLevel->p1 = iIdxCur; - pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; - if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - }else{ - assert( pLevel->p5==0 ); - } - }else - -#ifndef SQLITE_OMIT_OR_OPTIMIZATION - if( pLoop->wsFlags & WHERE_MULTI_OR ){ - /* Case 5: Two or more separately indexed terms connected by OR - ** - ** Example: - ** - ** CREATE TABLE t1(a,b,c,d); - ** CREATE INDEX i1 ON t1(a); - ** CREATE INDEX i2 ON t1(b); - ** CREATE INDEX i3 ON t1(c); - ** - ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) - ** - ** In the example, there are three indexed terms connected by OR. - ** The top of the loop looks like this: - ** - ** Null 1 # Zero the rowset in reg 1 - ** - ** Then, for each indexed term, the following. The arguments to - ** RowSetTest are such that the rowid of the current row is inserted - ** into the RowSet. If it is already present, control skips the - ** Gosub opcode and jumps straight to the code generated by WhereEnd(). - ** - ** sqlite3WhereBegin() - ** RowSetTest # Insert rowid into rowset - ** Gosub 2 A - ** sqlite3WhereEnd() - ** - ** Following the above, code to terminate the loop. Label A, the target - ** of the Gosub above, jumps to the instruction right after the Goto. - ** - ** Null 1 # Zero the rowset in reg 1 - ** Goto B # The loop is finished. - ** - ** A: # Return data, whatever. - ** - ** Return 2 # Jump back to the Gosub - ** - ** B: - ** - ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then - ** use an ephermeral index instead of a RowSet to record the primary - ** keys of the rows we have already seen. - ** - */ - WhereClause *pOrWc; /* The OR-clause broken out into subterms */ - SrcList *pOrTab; /* Shortened table list or OR-clause generation */ - Index *pCov = 0; /* Potential covering index (or NULL) */ - int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ - - int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ - int regRowset = 0; /* Register for RowSet object */ - int regRowid = 0; /* Register holding rowid */ - int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ - int iRetInit; /* Address of regReturn init */ - int untestedTerms = 0; /* Some terms not completely tested */ - int ii; /* Loop counter */ - u16 wctrlFlags; /* Flags for sub-WHERE clause */ - Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - Table *pTab = pTabItem->pTab; - - pTerm = pLoop->aLTerm[0]; - assert( pTerm!=0 ); - assert( pTerm->eOperator & WO_OR ); - assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); - pOrWc = &pTerm->u.pOrInfo->wc; - pLevel->op = OP_Return; - pLevel->p1 = regReturn; - - /* Set up a new SrcList in pOrTab containing the table being scanned - ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. - ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). - */ - if( pWInfo->nLevel>1 ){ - int nNotReady; /* The number of notReady tables */ - struct SrcList_item *origSrc; /* Original list of tables */ - nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(db, - sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); - if( pOrTab==0 ) return notReady; - pOrTab->nAlloc = (u8)(nNotReady + 1); - pOrTab->nSrc = pOrTab->nAlloc; - memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); - origSrc = pWInfo->pTabList->a; - for(k=1; k<=nNotReady; k++){ - memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); - } - }else{ - pOrTab = pWInfo->pTabList; - } - - /* Initialize the rowset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. Or, create an ephermeral index - ** capable of holding primary keys in the case of a WITHOUT ROWID. - ** - ** Also initialize regReturn to contain the address of the instruction - ** immediately following the OP_Return at the bottom of the loop. This - ** is required in a few obscure LEFT JOIN cases where control jumps - ** over the top of the loop into the body of it. In this case the - ** correct response for the end-of-loop code (the OP_Return) is to - ** fall through to the next instruction, just as an OP_Next does if - ** called on an uninitialized cursor. - */ - if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - if( HasRowid(pTab) ){ - regRowset = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - regRowset = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); - sqlite3VdbeSetP4KeyInfo(pParse, pPk); - } - regRowid = ++pParse->nMem; - } - iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); - - /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y - ** Then for every term xN, evaluate as the subexpression: xN AND z - ** That way, terms in y that are factored into the disjunction will - ** be picked up by the recursive calls to sqlite3WhereBegin() below. - ** - ** Actually, each subexpression is converted to "xN AND w" where w is - ** the "interesting" terms of z - terms that did not originate in the - ** ON or USING clause of a LEFT JOIN, and terms that are usable as - ** indices. - ** - ** This optimization also only applies if the (x1 OR x2 OR ...) term - ** is not contained in the ON clause of a LEFT JOIN. - ** See ticket http://www.sqlite.org/src/info/f2369304e4 - */ - if( pWC->nTerm>1 ){ - int iTerm; - for(iTerm=0; iTermnTerm; iTerm++){ - Expr *pExpr = pWC->a[iTerm].pExpr; - if( &pWC->a[iTerm] == pTerm ) continue; - if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); - testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); - if( pWC->a[iTerm].wtFlags & (TERM_ORINFO|TERM_VIRTUAL) ) continue; - if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; - pExpr = sqlite3ExprDup(db, pExpr, 0); - pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); - } - if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); - } - } - - /* Run a separate WHERE clause for each term of the OR clause. After - ** eliminating duplicates from other WHERE clauses, the action for each - ** sub-WHERE clause is to to invoke the main loop body as a subroutine. - */ - wctrlFlags = WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | - WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY; - for(ii=0; iinTerm; ii++){ - WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ - WhereInfo *pSubWInfo; /* Info for single OR-term scan */ - Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ - int j1 = 0; /* Address of jump operation */ - if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ - pAndExpr->pLeft = pOrExpr; - pOrExpr = pAndExpr; - } - /* Loop through table entries that match term pOrTerm. */ - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - wctrlFlags, iCovCur); - assert( pSubWInfo || pParse->nErr || db->mallocFailed ); - if( pSubWInfo ){ - WhereLoop *pSubLoop; - explainOneScan( - pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 - ); - /* This is the sub-WHERE clause body. First skip over - ** duplicate rows from prior sub-WHERE clauses, and record the - ** rowid (or PRIMARY KEY) for the current row so that the same - ** row will be skipped in subsequent sub-WHERE clauses. - */ - if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int r; - int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); - if( HasRowid(pTab) ){ - r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); - j1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, r,iSet); - VdbeCoverage(v); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - int nPk = pPk->nKeyCol; - int iPk; - - /* Read the PK into an array of temp registers. */ - r = sqlite3GetTempRange(pParse, nPk); - for(iPk=0; iPkaiColumn[iPk]; - sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur, r+iPk, 0); - } - - /* Check if the temp table already contains this key. If so, - ** the row has already been included in the result set and - ** can be ignored (by jumping past the Gosub below). Otherwise, - ** insert the key into the temp table and proceed with processing - ** the row. - ** - ** Use some of the same optimizations as OP_RowSetTest: If iSet - ** is zero, assume that the key cannot already be present in - ** the temp table. And if iSet is -1, assume that there is no - ** need to insert the key into the temp table, as it will never - ** be tested for. */ - if( iSet ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); - VdbeCoverage(v); - } - if( iSet>=0 ){ - sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); - sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); - if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - } - - /* Release the array of temp registers */ - sqlite3ReleaseTempRange(pParse, r, nPk); - } - } - - /* Invoke the main loop body as a subroutine */ - sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); - - /* Jump here (skipping the main loop body subroutine) if the - ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ - if( j1 ) sqlite3VdbeJumpHere(v, j1); - - /* The pSubWInfo->untestedTerms flag means that this OR term - ** contained one or more AND term from a notReady table. The - ** terms from the notReady table could not be tested and will - ** need to be tested later. - */ - if( pSubWInfo->untestedTerms ) untestedTerms = 1; - - /* If all of the OR-connected terms are optimized using the same - ** index, and the index is opened using the same cursor number - ** by each call to sqlite3WhereBegin() made by this loop, it may - ** be possible to use that index as a covering index. - ** - ** If the call to sqlite3WhereBegin() above resulted in a scan that - ** uses an index, and this is either the first OR-connected term - ** processed or the index is the same as that used by all previous - ** terms, set pCov to the candidate covering index. Otherwise, set - ** pCov to NULL to indicate that no candidate covering index will - ** be available. - */ - pSubLoop = pSubWInfo->a[0].pWLoop; - assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); - if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 - && (ii==0 || pSubLoop->u.btree.pIndex==pCov) - && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) - ){ - assert( pSubWInfo->a[0].iIdxCur==iCovCur ); - pCov = pSubLoop->u.btree.pIndex; - wctrlFlags |= WHERE_REOPEN_IDX; - }else{ - pCov = 0; - } - - /* Finish the loop through table entries that match term pOrTerm. */ - sqlite3WhereEnd(pSubWInfo); - } - } - } - pLevel->u.pCovidx = pCov; - if( pCov ) pLevel->iIdxCur = iCovCur; - if( pAndExpr ){ - pAndExpr->pLeft = 0; - sqlite3ExprDelete(db, pAndExpr); - } - sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); - sqlite3VdbeResolveLabel(v, iLoopBody); - - if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); - if( !untestedTerms ) disableTerm(pLevel, pTerm); - }else -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ - - { - /* Case 6: There is no usable index. We must do a complete - ** scan of the entire table. - */ - static const u8 aStep[] = { OP_Next, OP_Prev }; - static const u8 aStart[] = { OP_Rewind, OP_Last }; - assert( bRev==0 || bRev==1 ); - if( pTabItem->isRecursive ){ - /* Tables marked isRecursive have only a single row that is stored in - ** a pseudo-cursor. No need to Rewind or Next such cursors. */ - pLevel->op = OP_Noop; - }else{ - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - } - } - - /* Insert code to test every subexpression that can be completely - ** computed using the current set of tables. - */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - testcase( pWInfo->untestedTerms==0 - && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); - pWInfo->untestedTerms = 1; - continue; - } - pE = pTerm->pExpr; - assert( pE!=0 ); - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ - continue; - } - sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); - pTerm->wtFlags |= TERM_CODED; - } - - /* Insert code to test for implied constraints based on transitivity - ** of the "==" operator. - ** - ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" - ** and we are coding the t1 loop and the t2 loop has not yet coded, - ** then we cannot use the "t1.a=t2.b" constraint, but we can code - ** the implied "t1.a=123" constraint. - */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE, *pEAlt; - WhereTerm *pAlt; - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; - if( pTerm->leftCursor!=iCur ) continue; - if( pLevel->iLeftJoin ) continue; - pE = pTerm->pExpr; - assert( !ExprHasProperty(pE, EP_FromJoin) ); - assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); - pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); - if( pAlt==0 ) continue; - if( pAlt->wtFlags & (TERM_CODED) ) continue; - testcase( pAlt->eOperator & WO_EQ ); - testcase( pAlt->eOperator & WO_IN ); - VdbeModuleComment((v, "begin transitive constraint")); - pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt)); - if( pEAlt ){ - *pEAlt = *pAlt->pExpr; - pEAlt->pLeft = pE->pLeft; - sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL); - sqlite3StackFree(db, pEAlt); - } - } - - /* For a LEFT OUTER JOIN, generate code that will record the fact that - ** at least one row of the right table has matched the left table. - */ - if( pLevel->iLeftJoin ){ - pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); - VdbeComment((v, "record LEFT JOIN hit")); - sqlite3ExprCacheClear(pParse); - for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - assert( pWInfo->untestedTerms ); - continue; - } - assert( pTerm->pExpr ); - sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); - pTerm->wtFlags |= TERM_CODED; - } - } - - return pLevel->notReady; -} - -#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN) -/* -** Generate "Explanation" text for a WhereTerm. -*/ -static void whereExplainTerm(Vdbe *v, WhereTerm *pTerm){ - char zType[4]; - memcpy(zType, "...", 4); - if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; - if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; - sqlite3ExplainPrintf(v, "%s ", zType); - sqlite3ExplainExpr(v, pTerm->pExpr); -} -#endif /* WHERETRACE_ENABLED && SQLITE_ENABLE_TREE_EXPLAIN */ - #ifdef WHERETRACE_ENABLED /* @@ -117606,16 +132647,17 @@ static void whereExplainTerm(Vdbe *v, WhereTerm *pTerm){ */ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ WhereInfo *pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+7)/8; + int nb = 1+(pWInfo->pTabList->nSrc+3)/4; struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; Table *pTab = pItem->pTab; + Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, - p->iTab, nb, p->maskSelf, nb, p->prereq); + p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); sqlite3DebugPrintf(" %12s", pItem->zAlias ? pItem->zAlias : pTab->zName); if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - const char *zName; - if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ + const char *zName; + if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ int i = sqlite3Strlen30(zName) - 1; while( zName[i]!='_' ) i--; @@ -117636,29 +132678,18 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" %-19s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); - sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); -#ifdef SQLITE_ENABLE_TREE_EXPLAIN - /* If the 0x100 bit of wheretracing is set, then show all of the constraint - ** expressions in the WhereLoop.aLTerm[] array. - */ - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ /* WHERETRACE 0x100 */ - int i; - Vdbe *v = pWInfo->pParse->pVdbe; - sqlite3ExplainBegin(v); - for(i=0; inLTerm; i++){ - WhereTerm *pTerm = p->aLTerm[i]; - if( pTerm==0 ) continue; - sqlite3ExplainPrintf(v, " (%d) #%-2d ", i+1, (int)(pTerm-pWC->a)); - sqlite3ExplainPush(v); - whereExplainTerm(v, pTerm); - sqlite3ExplainPop(v); - sqlite3ExplainNL(v); - } - sqlite3ExplainFinish(v); - sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v)); + if( p->wsFlags & WHERE_SKIPSCAN ){ + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); + }else{ + sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + } + sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); + if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + int i; + for(i=0; inLTerm; i++){ + whereTermPrint(p->aLTerm[i], i); + } } -#endif } #endif @@ -117684,7 +132715,6 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ p->u.vtab.idxStr = 0; }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ sqlite3DbFree(db, p->u.btree.pIndex->zColAff); - sqlite3KeyInfoUnref(p->u.btree.pIndex->pKeyInfo); sqlite3DbFree(db, p->u.btree.pIndex); p->u.btree.pIndex = 0; } @@ -117707,8 +132737,8 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ WhereTerm **paNew; if( p->nLSlot>=n ) return SQLITE_OK; n = (n+7)&~7; - paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n); - if( paNew==0 ) return SQLITE_NOMEM; + paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n); + if( paNew==0 ) return SQLITE_NOMEM_BKPT; memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); p->aLTerm = paNew; @@ -117723,7 +132753,7 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ whereLoopClearUnion(db, pTo); if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ memset(&pTo->u, 0, sizeof(pTo->u)); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); @@ -117748,7 +132778,14 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ if( ALWAYS(pWInfo) ){ - whereClauseClear(&pWInfo->sWC); + int i; + for(i=0; inLevel; i++){ + WhereLevel *pLevel = &pWInfo->a[i]; + if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ + sqlite3DbFree(db, pLevel->u.in.aInLoop); + } + } + sqlite3WhereClauseClear(&pWInfo->sWC); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; pWInfo->pLoops = p->pNextLoop; @@ -117759,10 +132796,11 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Return TRUE if both of the following are true: +** Return TRUE if all of the following are true: ** ** (1) X has the same or lower cost that Y ** (2) X is a proper subset of Y +** (3) X skips at least as many columns as Y ** ** By "proper subset" we mean that X uses fewer WHERE clause terms ** than Y and that every WHERE clause term used by X is also used @@ -117770,19 +132808,25 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** ** If X is a proper subset of Y then Y is a better choice and ought ** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. +** relationship is inverted and needs to be adjusted. The third rule +** was added because if X uses skip-scan less than Y it still might +** deserve a lower cost even if it is a proper subset of Y. */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; - if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ + } + if( pY->nSkip > pX->nSkip ) return 0; if( pX->rRun >= pY->rRun ){ if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } @@ -117804,33 +132848,24 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. -** -** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the -** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE -** clause terms covered, since some of the first nLTerm entries in aLTerm[] -** will be NULL (because they are skipped). That makes it more difficult -** to compare the loops. We could add extra code to do the comparison, and -** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this -** adjustment is sufficient minor, that it is very difficult to construct -** a test case where the extra code would improve the query plan. Better -** to avoid the added complexity and just omit cost adjustments to SKIPSCAN -** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p */ + ** subset p. */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut - 1; }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ /* Adjust pTemplate cost upward so that it is costlier than p since ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut + 1; } @@ -117875,8 +132910,9 @@ static WhereLoop **whereLoopFindLesser( /* Any loop using an appliation-defined index (or PRIMARY KEY or ** UNIQUE constraint) with one or more == constraints is better - ** than an automatic index. */ + ** than an automatic index. Unless it is a skip-scan. */ if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->nSkip)==0 && (pTemplate->wsFlags & WHERE_INDEXED)!=0 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 && (p->prereq & pTemplate->prereq)==pTemplate->prereq @@ -117941,23 +132977,26 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p; WhereInfo *pWInfo = pBuilder->pWInfo; sqlite3 *db = pWInfo->pParse->db; + int rc; /* If pBuilder->pOrSet is defined, then only keep track of the costs ** and prereqs. */ if( pBuilder->pOrSet!=0 ){ + if( pTemplate->nLTerm ){ #if WHERETRACE_ENABLED - u16 n = pBuilder->pOrSet->n; - int x = + u16 n = pBuilder->pOrSet->n; + int x = #endif - whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, pTemplate->nOut); #if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); - } + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } #endif + } return SQLITE_OK; } @@ -117971,7 +133010,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** than pTemplate, so just ignore pTemplate */ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-noop: "); + sqlite3DebugPrintf(" skip: "); whereLoopPrint(pTemplate, pBuilder->pWC); } #endif @@ -117987,17 +133026,17 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ if( p!=0 ){ - sqlite3DebugPrintf("ins-del: "); + sqlite3DebugPrintf("replace: "); whereLoopPrint(p, pBuilder->pWC); } - sqlite3DebugPrintf("ins-new: "); + sqlite3DebugPrintf(" add: "); whereLoopPrint(pTemplate, pBuilder->pWC); } #endif if( p==0 ){ /* Allocate a new WhereLoop to add to the end of the list */ - *ppPrev = p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); - if( p==0 ) return SQLITE_NOMEM; + *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop)); + if( p==0 ) return SQLITE_NOMEM_BKPT; whereLoopInit(p); p->pNextLoop = 0; }else{ @@ -118014,40 +133053,63 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ *ppTail = pToDel->pNextLoop; #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-del: "); + sqlite3DebugPrintf(" delete: "); whereLoopPrint(pToDel, pBuilder->pWC); } #endif whereLoopDelete(db, pToDel); } } - whereLoopXfer(db, p, pTemplate); + rc = whereLoopXfer(db, p, pTemplate); if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ Index *pIndex = p->u.btree.pIndex; if( pIndex && pIndex->tnum==0 ){ p->u.btree.pIndex = 0; } } - return SQLITE_OK; + return rc; } /* ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an ** index. +* +** For every WHERE clause term that is not used by the index +** and which has a truth probability assigned by one of the likelihood(), +** likely(), or unlikely() SQL functions, reduce the estimated number +** of output rows by the probability specified. ** -** In the current implementation, the first extra WHERE clause term reduces -** the number of output rows by a factor of 10 and each additional term -** reduces the number of output rows by sqrt(2). +** TUNING: For every WHERE clause term that is not used by the index +** and which does not have an assigned truth probability, heuristics +** described below are used to try to estimate the truth probability. +** TODO --> Perhaps this is something that could be improved by better +** table statistics. +** +** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% +** value corresponds to -1 in LogEst notation, so this means decrement +** the WhereLoop.nOut field for every such WHERE clause term. +** +** Heuristic 2: If there exists one or more WHERE clause terms of the +** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the +** final output row estimate is no greater than 1/4 of the total number +** of rows in the table. In other words, assume that x==EXPR will filter +** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the +** "x" column is boolean or else -1 or 0 or 1 is a common default value +** on the "x" column and so in that case only cap the output row estimate +** at 1/2 instead of 1/4. */ -static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ +static void whereLoopOutputAdjust( + WhereClause *pWC, /* The WHERE clause */ + WhereLoop *pLoop, /* The loop to adjust downward */ + LogEst nRow /* Number of rows in the entire table */ +){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j; + int i, j, k; + LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ - if( !OptimizationEnabled(pWC->pWInfo->pParse->db, SQLITE_AdjustOutEst) ){ - return; - } + assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; @@ -118059,9 +133121,94 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } if( j<0 ){ - pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + if( pTerm->truthProb<=0 ){ + /* If a truth probability is specified using the likelihood() hints, + ** then use the probability provided by the application. */ + pLoop->nOut += pTerm->truthProb; + }else{ + /* In the absence of explicit truth probabilities, use heuristics to + ** guess a reasonable truth probability. */ + pLoop->nOut--; + if( pTerm->eOperator&(WO_EQ|WO_IS) ){ + Expr *pRight = pTerm->pExpr->pRight; + testcase( pTerm->pExpr->op==TK_IS ); + if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + k = 10; + }else{ + k = 20; + } + if( iReducenOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; +} + +/* +** Term pTerm is a vector range comparison operation. The first comparison +** in the vector can be optimized using column nEq of the index. This +** function returns the total number of vector elements that can be used +** as part of the range comparison. +** +** For example, if the query is: +** +** WHERE a = ? AND (b, c, d) > (?, ?, ?) +** +** and the index: +** +** CREATE INDEX ... ON (a, b, c, d, e) +** +** then this function would be invoked with nEq=1. The value returned in +** this case is 3. +*/ +static int whereRangeVectorLen( + Parse *pParse, /* Parsing context */ + int iCur, /* Cursor open on pIdx */ + Index *pIdx, /* The index to be used for a inequality constraint */ + int nEq, /* Number of prior equality constraints on same index */ + WhereTerm *pTerm /* The vector inequality constraint */ +){ + int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft); + int i; + + nCmp = MIN(nCmp, (pIdx->nColumn - nEq)); + for(i=1; ipExpr->pLeft->x.pList->a[i].pExpr; + Expr *pRhs = pTerm->pExpr->pRight; + if( pRhs->flags & EP_xIsSelect ){ + pRhs = pRhs->x.pSelect->pEList->a[i].pExpr; + }else{ + pRhs = pRhs->x.pList->a[i].pExpr; + } + + /* Check that the LHS of the comparison is a column reference to + ** the right column of the right source table. And that the sort + ** order of the index column is the same as the sort order of the + ** leftmost index column. */ + if( pLhs->op!=TK_COLUMN + || pLhs->iTable!=iCur + || pLhs->iColumn!=pIdx->aiColumn[i+nEq] + || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq] + ){ + break; + } + + testcase( pLhs->iColumn==XN_ROWID ); + aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs)); + idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); + if( aff!=idxaff ) break; + + pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + if( pColl==0 ) break; + if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; + } + return i; } /* @@ -118102,69 +133249,46 @@ static int whereLoopAddBtreeIndex( Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ - u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ + u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */ + u16 saved_nTop; /* Original value of pNew->u.btree.nTop */ + u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ - int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ + LogEst rSize; /* Number of rows in the table */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ pNew = pBuilder->pNew; - if( db->mallocFailed ) return SQLITE_NOMEM; + if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; + WHERETRACE(0x800, ("BEGIN addBtreeIdx(%s), nEq=%d\n", + pProbe->zName, pNew->u.btree.nEq)); assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); if( pNew->wsFlags & WHERE_BTM_LIMIT ){ opMask = WO_LT|WO_LE; - }else if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ - opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; }else{ - opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; + assert( pNew->u.btree.nBtm==0 ); + opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; } if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); assert( pNew->u.btree.nEqnColumn ); - iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, - opMask, pProbe); saved_nEq = pNew->u.btree.nEq; - saved_nSkip = pNew->u.btree.nSkip; + saved_nBtm = pNew->u.btree.nBtm; + saved_nTop = pNew->u.btree.nTop; + saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq, + opMask, pProbe); pNew->rSetup = 0; - rLogSize = estLog(pProbe->aiRowLogEst[0]); - - /* Consider using a skip-scan if there are no WHERE clause constraints - ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. - ** - ** The magic number 18 is selected on the basis that scanning 17 rows - ** is almost always quicker than an index seek (even though if the index - ** contains fewer than 2^17 rows we assume otherwise in other parts of - ** the code). And, even if it is not, it should not be too much slower. - ** On the other hand, the extra seeks could end up being significantly - ** more expensive. */ - assert( 42==sqlite3LogEst(18) ); - if( pTerm==0 - && saved_nEq==saved_nSkip - && saved_nEq+1nKeyCol - && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ - && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK - ){ - LogEst nIter; - pNew->u.btree.nEq++; - pNew->u.btree.nSkip++; - pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - pNew->nOut -= nIter; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); - pNew->nOut = saved_nOut; - } + rSize = pProbe->aiRowLogEst[0]; + rLogSize = estLog(rSize); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; @@ -118174,14 +133298,32 @@ static int whereLoopAddBtreeIndex( int nRecValid = pBuilder->nRecValid; #endif if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) - && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) + && indexColumnNotNull(pProbe, saved_nEq) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } if( pTerm->prereqRight & pNew->maskSelf ) continue; + /* Do not allow the upper bound of a LIKE optimization range constraint + ** to mix with a lower range bound from some other source */ + if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; + + /* Do not allow IS constraints from the WHERE clause to be used by the + ** right table of a LEFT JOIN. Only constraints in the ON clause are + ** allowed */ + if( (pSrc->fg.jointype & JT_LEFT)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + && (eOp & (WO_IS|WO_ISNULL))!=0 + ){ + testcase( eOp & WO_IS ); + testcase( eOp & WO_ISNULL ); + continue; + } + pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nBtm = saved_nBtm; + pNew->u.btree.nTop = saved_nTop; pNew->nLTerm = saved_nLTerm; if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ pNew->aLTerm[pNew->nLTerm++] = pTerm; @@ -118198,18 +133340,31 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ + int i; nIn = 46; assert( 46==sqlite3LogEst(25) ); + + /* The expression may actually be of the form (x, y) IN (SELECT...). + ** In this case there is a separate term for each of (x) and (y). + ** However, the nIn multiplier should only be applied once, not once + ** for each such term. The following loop checks that pTerm is the + ** first such term in use, and sets nIn back to 0 if it is not. */ + for(i=0; inLTerm-1; i++){ + if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0; + } }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ nIn = sqlite3LogEst(pExpr->x.pList->nExpr); + assert( nIn>0 ); /* RHS always has 2 or more terms... The parser + ** changes "x IN (?)" into "x=?". */ } - assert( nIn>0 ); /* RHS always has 2 or more terms... The parser - ** changes "x IN (?)" into "x=?". */ - - }else if( eOp & (WO_EQ) ){ + }else if( eOp & (WO_EQ|WO_IS) ){ + int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ - if( iCol>=0 && !IsUniqueIndex(pProbe) ){ + assert( saved_nEq==pNew->u.btree.nEq ); + if( iCol==XN_ROWID + || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) + ){ + if( iCol>=0 && pProbe->uniqNotNull==0 ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; @@ -118221,13 +133376,31 @@ static int whereLoopAddBtreeIndex( testcase( eOp & WO_GT ); testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->u.btree.nBtm = whereRangeVectorLen( + pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm + ); pBtm = pTerm; pTop = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + /* Range contraints that come from the LIKE optimization are + ** always used in pairs. */ + pTop = &pTerm[1]; + assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); + assert( pTop->wtFlags & TERM_LIKEOPT ); + assert( pTop->eOperator==WO_LT ); + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTop; + pNew->wsFlags |= WHERE_TOP_LIMIT; + pNew->u.btree.nTop = 1; + } }else{ assert( eOp & (WO_LT|WO_LE) ); testcase( eOp & WO_LT ); testcase( eOp & WO_LE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->u.btree.nTop = whereRangeVectorLen( + pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm + ); pTop = pTerm; pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? pNew->aLTerm[pNew->nLTerm-2] : 0; @@ -118245,10 +133418,10 @@ static int whereLoopAddBtreeIndex( whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); }else{ int nEq = ++pNew->u.btree.nEq; - assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) ); assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 && iCol>=0 ){ + if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); testcase( eOp & WO_IN ); pNew->nOut += pTerm->truthProb; @@ -118259,12 +133432,12 @@ static int whereLoopAddBtreeIndex( if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) ){ Expr *pExpr = pTerm->pExpr; - if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ testcase( eOp & WO_EQ ); + testcase( eOp & WO_IS ); testcase( eOp & WO_ISNULL ); rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); }else{ @@ -118306,7 +133479,7 @@ static int whereLoopAddBtreeIndex( nOutUnadjusted = pNew->nOut; pNew->rRun += nInMul + nIn; pNew->nOut += nInMul + nIn; - whereLoopOutputAdjust(pBuilder->pWC, pNew); + whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ @@ -118327,10 +133500,49 @@ static int whereLoopAddBtreeIndex( } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; + pNew->u.btree.nBtm = saved_nBtm; + pNew->u.btree.nTop = saved_nTop; + pNew->nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; pNew->nLTerm = saved_nLTerm; + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->noSkipScan==0 + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + } + + WHERETRACE(0x800, ("END addBtreeIdx(%s), nEq=%d, rc=%d\n", + pProbe->zName, saved_nEq, rc)); return rc; } @@ -118348,17 +133560,25 @@ static int indexMightHelpWithOrderBy( int iCursor ){ ExprList *pOB; + ExprList *aColExpr; int ii, jj; if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); - if( pExpr->op!=TK_COLUMN ) return 0; - if( pExpr->iTable==iCursor ){ + if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ + if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; } + }else if( (aColExpr = pIndex->aColExpr)!=0 ){ + for(jj=0; jjnKeyCol; jj++){ + if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; + if( sqlite3ExprCompare(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ + return 1; + } + } } } return 0; @@ -118388,15 +133608,24 @@ static Bitmask columnsInIndex(Index *pIdx){ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ int i; WhereTerm *pTerm; + while( pWhere->op==TK_AND ){ + if( !whereUsablePartialIndex(iTab,pWC,pWhere->pLeft) ) return 0; + pWhere = pWhere->pRight; + } for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - if( sqlite3ExprImpliesExpr(pTerm->pExpr, pWhere, iTab) ) return 1; + Expr *pExpr = pTerm->pExpr; + if( sqlite3ExprImpliesExpr(pExpr, pWhere, iTab) + && (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab) + ){ + return 1; + } } return 0; } /* ** Add all WhereLoop objects for a single table of the join where the table -** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be +** is identified by pBuilder->pNew->iTab. That table is guaranteed to be ** a b-tree table, not a virtual table. ** ** The costs (WhereLoop.rRun) of the b-tree loops added by this function @@ -118432,7 +133661,7 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mExtra /* Extra prerequesites for using this table */ + Bitmask mPrereq /* Extra prerequesites for using this table */ ){ WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ @@ -118458,9 +133687,9 @@ static int whereLoopAddBtree( pWC = pBuilder->pWC; assert( !IsVirtual(pSrc->pTab) ); - if( pSrc->pIndex ){ + if( pSrc->pIBIndex ){ /* An INDEXED BY clause specifies a particular index to use */ - pProbe = pSrc->pIndex; + pProbe = pSrc->pIBIndex; }else if( !HasRowid(pTab) ){ pProbe = pTab->pIndex; }else{ @@ -118480,7 +133709,7 @@ static int whereLoopAddBtree( aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; - if( pSrc->notIndexed==0 ){ + if( pSrc->fg.notIndexed==0 ){ /* The real indices of the table are only considered if the ** NOT INDEXED qualifier is omitted from the FROM clause */ sPk.pNext = pFirst; @@ -118492,14 +133721,14 @@ static int whereLoopAddBtree( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* Automatic indexes */ - if( !pBuilder->pOrSet + if( !pBuilder->pOrSet /* Not part of an OR optimization */ + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 - && pSrc->pIndex==0 - && !pSrc->viaCoroutine - && !pSrc->notIndexed - && HasRowid(pTab) - && !pSrc->isCorrelated - && !pSrc->isRecursive + && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ + && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ + && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ + && !pSrc->fg.isCorrelated /* Not a correlated subquery */ + && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ ){ /* Generate auto-index WhereLoops */ WhereTerm *pTerm; @@ -118508,23 +133737,32 @@ static int whereLoopAddBtree( if( pTerm->prereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; /* TUNING: One-time cost for computing the automatic index is - ** approximately 7*N*log2(N) where N is the number of rows in - ** the table being indexed. */ - pNew->rSetup = rLogSize + rSize + 28; assert( 28==sqlite3LogEst(7) ); + ** estimated to be X*N*log2(N) where N is the number of rows in + ** the table being indexed and where X is 7 (LogEst=28) for normal + ** tables or 1.375 (LogEst=4) for views and subqueries. The value + ** of X is smaller for views and subqueries so that the query planner + ** will be more aggressive about generating automatic indexes for + ** those objects, since there is no opportunity to add schema + ** indexes on subqueries and views. */ + pNew->rSetup = rLogSize + rSize + 4; + if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){ + pNew->rSetup += 24; + } ApplyCostMultiplier(pNew->rSetup, pTab->costMult); + if( pNew->rSetup<0 ) pNew->rSetup = 0; /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way - ** of knowning how selective the index will ultimately be. It would + ** of knowing how selective the index will ultimately be. It would ** not be unreasonable to make this value much larger. */ pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); pNew->wsFlags = WHERE_AUTO_INDEX; - pNew->prereq = mExtra | pTerm->prereqRight; + pNew->prereq = mPrereq | pTerm->prereqRight; rc = whereLoopInsert(pBuilder, pNew); } } @@ -118535,16 +133773,19 @@ static int whereLoopAddBtree( */ for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ if( pProbe->pPartIdxWhere!=0 - && !whereUsablePartialIndex(pNew->iTab, pWC, pProbe->pPartIdxWhere) ){ + && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ + testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ continue; /* Partial index inappropriate for this query */ } rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; - pNew->u.btree.nSkip = 0; + pNew->u.btree.nBtm = 0; + pNew->u.btree.nTop = 0; + pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; - pNew->prereq = mExtra; + pNew->prereq = mPrereq; pNew->nOut = rSize; pNew->u.btree.pIndex = pProbe; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); @@ -118559,7 +133800,7 @@ static int whereLoopAddBtree( /* TUNING: Cost of full table scan is (N*3.0). */ pNew->rRun = rSize + 16; ApplyCostMultiplier(pNew->rRun, pTab->costMult); - whereLoopOutputAdjust(pWC, pNew); + whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; @@ -118576,6 +133817,7 @@ static int whereLoopAddBtree( /* Full scan via index */ if( b || !HasRowid(pTab) + || pProbe->pPartIdxWhere!=0 || ( m==0 && pProbe->bUnordered==0 && (pProbe->szIdxRowszTabRow) @@ -118588,14 +133830,37 @@ static int whereLoopAddBtree( /* The cost of visiting the index rows is N*K, where K is ** between 1.1 and 3.0, depending on the relative sizes of the - ** index and table rows. If this is a non-covering index scan, - ** also add the cost of visiting table rows (N*3.0). */ + ** index and table rows. */ pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; if( m!=0 ){ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); + /* If this is a non-covering index scan, add in the cost of + ** doing table lookups. The cost will be 3x the number of + ** lookups. Take into account WHERE clause terms that can be + ** satisfied using just the index, and that do not require a + ** table lookup. */ + LogEst nLookup = rSize + 16; /* Base cost: N*3 */ + int ii; + int iCur = pSrc->iCursor; + WhereClause *pWC2 = &pWInfo->sWC; + for(ii=0; iinTerm; ii++){ + WhereTerm *pTerm = &pWC2->a[ii]; + if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){ + break; + } + /* pTerm can be evaluated using just the index. So reduce + ** the expected number of table lookups accordingly */ + if( pTerm->truthProb<=0 ){ + nLookup += pTerm->truthProb; + }else{ + nLookup--; + if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19; + } + } + + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup); } ApplyCostMultiplier(pNew->rRun, pTab->costMult); - whereLoopOutputAdjust(pWC, pNew); + whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; @@ -118611,176 +133876,300 @@ static int whereLoopAddBtree( /* If there was an INDEXED BY clause, then only that one index is ** considered. */ - if( pSrc->pIndex ) break; + if( pSrc->pIBIndex ) break; } return rc; } #ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Argument pIdxInfo is already populated with all constraints that may +** be used by the virtual table identified by pBuilder->pNew->iTab. This +** function marks a subset of those constraints usable, invokes the +** xBestIndex method and adds the returned plan to pBuilder. +** +** A constraint is marked usable if: +** +** * Argument mUsable indicates that its prerequisites are available, and +** +** * It is not one of the operators specified in the mExclude mask passed +** as the fourth argument (which in practice is either WO_IN or 0). +** +** Argument mPrereq is a mask of tables that must be scanned before the +** virtual table in question. These are added to the plans prerequisites +** before it is added to pBuilder. +** +** Output parameter *pbIn is set to true if the plan added to pBuilder +** uses one or more WO_IN terms, or false otherwise. +*/ +static int whereLoopAddVirtualOne( + WhereLoopBuilder *pBuilder, + Bitmask mPrereq, /* Mask of tables that must be used. */ + Bitmask mUsable, /* Mask of usable tables */ + u16 mExclude, /* Exclude terms using these operators */ + sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */ + u16 mNoOmit, /* Do not omit these constraints */ + int *pbIn /* OUT: True if plan uses an IN(...) op */ +){ + WhereClause *pWC = pBuilder->pWC; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage; + int i; + int mxTerm; + int rc = SQLITE_OK; + WhereLoop *pNew = pBuilder->pNew; + Parse *pParse = pBuilder->pWInfo->pParse; + struct SrcList_item *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab]; + int nConstraint = pIdxInfo->nConstraint; + + assert( (mUsable & mPrereq)==mPrereq ); + *pbIn = 0; + pNew->prereq = mPrereq; + + /* Set the usable flag on the subset of constraints identified by + ** arguments mUsable and mExclude. */ + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; ia[pIdxCons->iTermOffset]; + pIdxCons->usable = 0; + if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight + && (pTerm->eOperator & mExclude)==0 + ){ + pIdxCons->usable = 1; + } + } + + /* Initialize the output fields of the sqlite3_index_info structure */ + memset(pUsage, 0, sizeof(pUsage[0])*nConstraint); + assert( pIdxInfo->needToFreeIdxStr==0 ); + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->orderByConsumed = 0; + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; + pIdxInfo->estimatedRows = 25; + pIdxInfo->idxFlags = 0; + pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; + + /* Invoke the virtual table xBestIndex() method */ + rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); + if( rc ) return rc; + + mxTerm = -1; + assert( pNew->nLSlot>=nConstraint ); + for(i=0; iaLTerm[i] = 0; + pNew->u.vtab.omitMask = 0; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; i=0 ){ + WhereTerm *pTerm; + int j = pIdxCons->iTermOffset; + if( iTerm>=nConstraint + || j<0 + || j>=pWC->nTerm + || pNew->aLTerm[iTerm]!=0 + || pIdxCons->usable==0 + ){ + rc = SQLITE_ERROR; + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); + return rc; + } + testcase( iTerm==nConstraint-1 ); + testcase( j==0 ); + testcase( j==pWC->nTerm-1 ); + pTerm = &pWC->a[j]; + pNew->prereq |= pTerm->prereqRight; + assert( iTermnLSlot ); + pNew->aLTerm[iTerm] = pTerm; + if( iTerm>mxTerm ) mxTerm = iTerm; + testcase( iTerm==15 ); + testcase( iTerm==16 ); + if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; + *pbIn = 1; assert( (mExclude & WO_IN)==0 ); + } + } + } + pNew->u.vtab.omitMask &= ~mNoOmit; + + pNew->nLTerm = mxTerm+1; + assert( pNew->nLTerm<=pNew->nLSlot ); + pNew->u.vtab.idxNum = pIdxInfo->idxNum; + pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; + pIdxInfo->needToFreeIdxStr = 0; + pNew->u.vtab.idxStr = pIdxInfo->idxStr; + pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? + pIdxInfo->nOrderBy : 0); + pNew->rSetup = 0; + pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); + pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); + + /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated + ** that the scan will visit at most one row. Clear it otherwise. */ + if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){ + pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags &= ~WHERE_ONEROW; + } + rc = whereLoopInsert(pBuilder, pNew); + if( pNew->u.vtab.needFree ){ + sqlite3_free(pNew->u.vtab.idxStr); + pNew->u.vtab.needFree = 0; + } + WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", + *pbIn, (sqlite3_uint64)mPrereq, + (sqlite3_uint64)(pNew->prereq & ~mPrereq))); + + return rc; +} + + /* ** Add all WhereLoop objects for a table of the join identified by ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. +** +** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and +** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause +** entries that occur before the virtual table in the FROM clause and are +** separated from it by at least one LEFT or CROSS JOIN. Similarly, the +** mUnusable mask contains all FROM clause entries that occur after the +** virtual table and are separated from it by at least one LEFT or +** CROSS JOIN. +** +** For example, if the query were: +** +** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6; +** +** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6). +** +** All the tables in mPrereq must be scanned before the current virtual +** table. So any terms for which all prerequisites are satisfied by +** mPrereq may be specified as "usable" in all calls to xBestIndex. +** Conversely, all tables in mUnusable must be scanned after the current +** virtual table, so any terms for which the prerequisites overlap with +** mUnusable should always be configured as "not-usable" for xBestIndex. */ static int whereLoopAddVirtual( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mExtra + Bitmask mPrereq, /* Tables that must be scanned before this one */ + Bitmask mUnusable /* Tables that must be scanned after this one */ ){ + int rc = SQLITE_OK; /* Return code */ WhereInfo *pWInfo; /* WHERE analysis context */ Parse *pParse; /* The parsing context */ WhereClause *pWC; /* The WHERE clause */ struct SrcList_item *pSrc; /* The FROM clause term to search */ - Table *pTab; - sqlite3 *db; - sqlite3_index_info *pIdxInfo; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int i, j; - int iTerm, mxTerm; - int nConstraint; - int seenIn = 0; /* True if an IN operator is seen */ - int seenVar = 0; /* True if a non-constant constraint is seen */ - int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ + sqlite3_index_info *p; /* Object to pass to xBestIndex() */ + int nConstraint; /* Number of constraints in p */ + int bIn; /* True if plan uses IN(...) operator */ WhereLoop *pNew; - int rc = SQLITE_OK; + Bitmask mBest; /* Tables used by best possible plan */ + u16 mNoOmit; + assert( (mPrereq & mUnusable)==0 ); pWInfo = pBuilder->pWInfo; pParse = pWInfo->pParse; - db = pParse->db; pWC = pBuilder->pWC; pNew = pBuilder->pNew; pSrc = &pWInfo->pTabList->a[pNew->iTab]; - pTab = pSrc->pTab; - assert( IsVirtual(pTab) ); - pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); - if( pIdxInfo==0 ) return SQLITE_NOMEM; - pNew->prereq = 0; + assert( IsVirtual(pSrc->pTab) ); + p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy, + &mNoOmit); + if( p==0 ) return SQLITE_NOMEM_BKPT; pNew->rSetup = 0; pNew->wsFlags = WHERE_VIRTUALTABLE; pNew->nLTerm = 0; pNew->u.vtab.needFree = 0; - pUsage = pIdxInfo->aConstraintUsage; - nConstraint = pIdxInfo->nConstraint; - if( whereLoopResize(db, pNew, nConstraint) ){ - sqlite3DbFree(db, pIdxInfo); - return SQLITE_NOMEM; + nConstraint = p->nConstraint; + if( whereLoopResize(pParse->db, pNew, nConstraint) ){ + sqlite3DbFree(pParse->db, p); + return SQLITE_NOMEM_BKPT; } - for(iPhase=0; iPhase<=3; iPhase++){ - if( !seenIn && (iPhase&1)!=0 ){ - iPhase++; - if( iPhase>3 ) break; - } - if( !seenVar && iPhase>1 ) break; - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; inConstraint; i++, pIdxCons++){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - switch( iPhase ){ - case 0: /* Constants without IN operator */ - pIdxCons->usable = 0; - if( (pTerm->eOperator & WO_IN)!=0 ){ - seenIn = 1; - } - if( pTerm->prereqRight!=0 ){ - seenVar = 1; - }else if( (pTerm->eOperator & WO_IN)==0 ){ - pIdxCons->usable = 1; - } - break; - case 1: /* Constants with IN operators */ - assert( seenIn ); - pIdxCons->usable = (pTerm->prereqRight==0); - break; - case 2: /* Variables without IN */ - assert( seenVar ); - pIdxCons->usable = (pTerm->eOperator & WO_IN)==0; - break; - default: /* Variables with IN */ - assert( seenVar && seenIn ); - pIdxCons->usable = 1; - break; - } - } - memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); - if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->needToFreeIdxStr = 0; - pIdxInfo->orderByConsumed = 0; - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; - pIdxInfo->estimatedRows = 25; - rc = vtabBestIndex(pParse, pTab, pIdxInfo); - if( rc ) goto whereLoopAddVtab_exit; - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - pNew->prereq = mExtra; - mxTerm = -1; - assert( pNew->nLSlot>=nConstraint ); - for(i=0; iaLTerm[i] = 0; - pNew->u.vtab.omitMask = 0; - for(i=0; i=0 ){ - j = pIdxCons->iTermOffset; - if( iTerm>=nConstraint - || j<0 - || j>=pWC->nTerm - || pNew->aLTerm[iTerm]!=0 - ){ - rc = SQLITE_ERROR; - sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName); - goto whereLoopAddVtab_exit; - } - testcase( iTerm==nConstraint-1 ); - testcase( j==0 ); - testcase( j==pWC->nTerm-1 ); - pTerm = &pWC->a[j]; - pNew->prereq |= pTerm->prereqRight; - assert( iTermnLSlot ); - pNew->aLTerm[iTerm] = pTerm; - if( iTerm>mxTerm ) mxTerm = iTerm; - testcase( iTerm==15 ); - testcase( iTerm==16 ); - if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ - if( pUsage[i].omit==0 ){ - /* Do not attempt to use an IN constraint if the virtual table - ** says that the equivalent EQ constraint cannot be safely omitted. - ** If we do attempt to use such a constraint, some rows might be - ** repeated in the output. */ - break; - } - /* A virtual table that is constrained by an IN clause may not - ** consume the ORDER BY clause because (1) the order of IN terms - ** is not necessarily related to the order of output terms and - ** (2) Multiple outputs from a single IN value will not merge - ** together. */ - pIdxInfo->orderByConsumed = 0; - } - } - } - if( i>=nConstraint ){ - pNew->nLTerm = mxTerm+1; - assert( pNew->nLTerm<=pNew->nLSlot ); - pNew->u.vtab.idxNum = pIdxInfo->idxNum; - pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; - pIdxInfo->needToFreeIdxStr = 0; - pNew->u.vtab.idxStr = pIdxInfo->idxStr; - pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? - pIdxInfo->nOrderBy : 0); - pNew->rSetup = 0; - pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); - pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); - whereLoopInsert(pBuilder, pNew); - if( pNew->u.vtab.needFree ){ - sqlite3_free(pNew->u.vtab.idxStr); - pNew->u.vtab.needFree = 0; - } - } - } + /* First call xBestIndex() with all constraints usable. */ + WHERETRACE(0x40, (" VirtualOne: all usable\n")); + rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn); -whereLoopAddVtab_exit: - if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); - sqlite3DbFree(db, pIdxInfo); + /* If the call to xBestIndex() with all terms enabled produced a plan + ** that does not require any source tables (IOW: a plan with mBest==0), + ** then there is no point in making any further calls to xBestIndex() + ** since they will all return the same result (if the xBestIndex() + ** implementation is sane). */ + if( rc==SQLITE_OK && (mBest = (pNew->prereq & ~mPrereq))!=0 ){ + int seenZero = 0; /* True if a plan with no prereqs seen */ + int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */ + Bitmask mPrev = 0; + Bitmask mBestNoIn = 0; + + /* If the plan produced by the earlier call uses an IN(...) term, call + ** xBestIndex again, this time with IN(...) terms disabled. */ + if( bIn ){ + WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn); + assert( bIn==0 ); + mBestNoIn = pNew->prereq & ~mPrereq; + if( mBestNoIn==0 ){ + seenZero = 1; + seenZeroNoIN = 1; + } + } + + /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) + ** in the set of terms that apply to the current virtual table. */ + while( rc==SQLITE_OK ){ + int i; + Bitmask mNext = ALLBITS; + assert( mNext>0 ); + for(i=0; ia[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq + ); + if( mThis>mPrev && mThisprereq==mPrereq ){ + seenZero = 1; + if( bIn==0 ) seenZeroNoIN = 1; + } + } + + /* If the calls to xBestIndex() in the above loop did not find a plan + ** that requires no source tables at all (i.e. one guaranteed to be + ** usable), make a call here with all source tables disabled */ + if( rc==SQLITE_OK && seenZero==0 ){ + WHERETRACE(0x40, (" VirtualOne: all disabled\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn); + if( bIn==0 ) seenZeroNoIN = 1; + } + + /* If the calls to xBestIndex() have so far failed to find a plan + ** that requires no source tables at all and does not use an IN(...) + ** operator, make a final call to obtain one here. */ + if( rc==SQLITE_OK && seenZeroNoIN==0 ){ + WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn); + } + } + + if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr); + sqlite3DbFree(pParse->db, p); return rc; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -118789,7 +134178,11 @@ whereLoopAddVtab_exit: ** Add WhereLoop entries to handle OR terms. This works for either ** btrees or virtual tables. */ -static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ +static int whereLoopAddOr( + WhereLoopBuilder *pBuilder, + Bitmask mPrereq, + Bitmask mUnusable +){ WhereInfo *pWInfo = pBuilder->pWInfo; WhereClause *pWC; WhereLoop *pNew; @@ -118802,7 +134195,6 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ struct SrcList_item *pItem; pWC = pBuilder->pWC; - if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; pWCEnd = pWC->a + pWC->nTerm; pNew = pBuilder->pNew; memset(&sSum, 0, sizeof(sSum)); @@ -118823,6 +134215,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ sSubBuild.pOrderBy = 0; sSubBuild.pOrSet = &sCur; + WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; @@ -118837,13 +134230,23 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ continue; } sCur.n = 0; +#ifdef WHERETRACE_ENABLED + WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); + if( sqlite3WhereTrace & 0x400 ){ + sqlite3WhereClausePrint(sSubBuild.pWC); + } +#endif #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(&sSubBuild, mExtra); + rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); }else #endif { - rc = whereLoopAddBtree(&sSubBuild, mExtra); + rc = whereLoopAddBtree(&sSubBuild, mPrereq); + } + if( rc==SQLITE_OK ){ + rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); } assert( rc==SQLITE_OK || sCur.n==0 ); if( sCur.n==0 ){ @@ -118889,6 +134292,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); } + WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); } } return rc; @@ -118899,38 +134303,51 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ */ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ WhereInfo *pWInfo = pBuilder->pWInfo; - Bitmask mExtra = 0; + Bitmask mPrereq = 0; Bitmask mPrior = 0; int iTab; SrcList *pTabList = pWInfo->pTabList; struct SrcList_item *pItem; + struct SrcList_item *pEnd = &pTabList->a[pWInfo->nLevel]; sqlite3 *db = pWInfo->pParse->db; - int nTabList = pWInfo->nLevel; int rc = SQLITE_OK; - u8 priorJoinType = 0; WhereLoop *pNew; + u8 priorJointype = 0; /* Loop over the tables in the join, from left to right */ pNew = pBuilder->pNew; whereLoopInit(pNew); - for(iTab=0, pItem=pTabList->a; iTaba; pItemiTab = iTab; - pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); - if( ((pItem->jointype|priorJoinType) & (JT_LEFT|JT_CROSS))!=0 ){ - mExtra = mPrior; + pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); + if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ + /* This condition is true when pItem is the FROM clause term on the + ** right-hand-side of a LEFT or CROSS JOIN. */ + mPrereq = mPrior; } - priorJoinType = pItem->jointype; + priorJointype = pItem->fg.jointype; +#ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(pBuilder, mExtra); - }else{ - rc = whereLoopAddBtree(pBuilder, mExtra); + struct SrcList_item *p; + for(p=&pItem[1]; pfg.jointype & (JT_LEFT|JT_CROSS)) ){ + mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); + } + } + rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable); + }else +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + { + rc = whereLoopAddBtree(pBuilder, mPrereq); } if( rc==SQLITE_OK ){ - rc = whereLoopAddOr(pBuilder, mExtra); + rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); } mPrior |= pNew->maskSelf; if( rc || db->mallocFailed ) break; } + whereLoopClear(db, pNew); return rc; } @@ -118948,7 +134365,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** strict. With GROUP BY and DISTINCT the only requirement is that ** equivalent rows appear immediately adjacent to one another. GROUP BY ** and DISTINCT do not require rows to appear in any particular order as long -** as equivelent rows are grouped together. Thus for GROUP BY and DISTINCT +** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT ** the pOrderBy terms can be matched in any order. With ORDER BY, the ** pOrderBy terms must be matched in strict left-to-right order. */ @@ -118956,7 +134373,7 @@ static i8 wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ WherePath *pPath, /* The WherePath to check */ - u16 wctrlFlags, /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */ + u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */ u16 nLoop, /* Number of entries in pPath->aLoop[] */ WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ @@ -118967,6 +134384,7 @@ static i8 wherePathSatisfiesOrderBy( u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ + u16 eqOpMask; /* Allowed equality operators */ u16 nKeyCol; /* Number of key columns in pIndex */ u16 nColumn; /* Total number of ordered columns in the index */ u16 nOrderBy; /* Number terms in the ORDER BY clause */ @@ -119017,9 +134435,16 @@ static i8 wherePathSatisfiesOrderBy( obDone = MASKBIT(nOrderBy)-1; orderDistinctMask = 0; ready = 0; + eqOpMask = WO_EQ | WO_IS | WO_ISNULL; + if( wctrlFlags & WHERE_ORDERBY_LIMIT ) eqOpMask |= WO_IN; for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; - pLoop = iLoopaLoop[iLoop] : pLast; + if( iLoopaLoop[iLoop]; + if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue; + }else{ + pLoop = pLast; + } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ if( pLoop->u.vtab.isOrdered ) obSat = obDone; break; @@ -119036,10 +134461,18 @@ static i8 wherePathSatisfiesOrderBy( pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr); if( pOBExpr->op!=TK_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; - pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, - ~ready, WO_EQ|WO_ISNULL, 0); + pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, + ~ready, eqOpMask, 0); if( pTerm==0 ) continue; - if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){ + if( pTerm->eOperator==WO_IN ){ + /* IN terms are only valid for sorting in the ORDER BY LIMIT + ** optimization, and then only if they are actually used + ** by the query plan */ + assert( wctrlFlags & WHERE_ORDERBY_LIMIT ); + for(j=0; jnLTerm && pTerm!=pLoop->aLTerm[j]; j++){} + if( j>=pLoop->nLTerm ) continue; + } + if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ const char *z1, *z2; pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; @@ -119048,6 +134481,7 @@ static i8 wherePathSatisfiesOrderBy( if( !pColl ) pColl = db->pDfltColl; z2 = pColl->zName; if( sqlite3StrICmp(z1, z2)!=0 ) continue; + testcase( pTerm->pExpr->op==TK_IS ); } obSat |= MASKBIT(i); } @@ -119063,7 +134497,8 @@ static i8 wherePathSatisfiesOrderBy( nKeyCol = pIndex->nKeyCol; nColumn = pIndex->nColumn; assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); - assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); + assert( pIndex->aiColumn[nColumn-1]==XN_ROWID + || !HasRowid(pIndex->pTable)); isOrderDistinct = IsUniqueIndex(pIndex); } @@ -119073,18 +134508,42 @@ static i8 wherePathSatisfiesOrderBy( rev = revSet = 0; distinctColumns = 0; for(j=0; ju.btree.nEq - && pLoop->u.btree.nSkip==0 - && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 - ){ - if( i & WO_ISNULL ){ - testcase( isOrderDistinct ); - isOrderDistinct = 0; + assert( j>=pLoop->u.btree.nEq + || (pLoop->aLTerm[j]==0)==(jnSkip) + ); + if( ju.btree.nEq && j>=pLoop->nSkip ){ + u16 eOp = pLoop->aLTerm[j]->eOperator; + + /* Skip over == and IS and ISNULL terms. (Also skip IN terms when + ** doing WHERE_ORDERBY_LIMIT processing). + ** + ** If the current term is a column of an ((?,?) IN (SELECT...)) + ** expression for which the SELECT returns more than one column, + ** check that it is the only column used by this loop. Otherwise, + ** if it is one of two or more, none of the columns can be + ** considered to match an ORDER BY term. */ + if( (eOp & eqOpMask)!=0 ){ + if( eOp & WO_ISNULL ){ + testcase( isOrderDistinct ); + isOrderDistinct = 0; + } + continue; + }else if( ALWAYS(eOp & WO_IN) ){ + /* ALWAYS() justification: eOp is an equality operator due to the + ** ju.btree.nEq constraint above. Any equality other + ** than WO_IN is captured by the previous "if". So this one + ** always has to be WO_IN. */ + Expr *pX = pLoop->aLTerm[j]->pExpr; + for(i=j+1; iu.btree.nEq; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + assert( (pLoop->aLTerm[i]->eOperator & WO_IN) ); + bOnce = 0; + break; + } + } } - continue; } /* Get the column number in the table (iColumn) and sort order @@ -119095,7 +134554,7 @@ static i8 wherePathSatisfiesOrderBy( revIdx = pIndex->aSortOrder[j]; if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; }else{ - iColumn = -1; + iColumn = XN_ROWID; revIdx = 0; } @@ -119113,7 +134572,6 @@ static i8 wherePathSatisfiesOrderBy( /* Find the ORDER BY term that corresponds to the j-th column ** of the index and mark that ORDER BY term off */ - bOnce = 1; isMatch = 0; for(i=0; bOnce && iop!=TK_COLUMN ) continue; - if( pOBExpr->iTable!=iCur ) continue; - if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=(-1) ){ + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + }else{ + if( sqlite3ExprCompare(pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){ + continue; + } + } if( iColumn>=0 ){ pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; @@ -119132,7 +134596,7 @@ static i8 wherePathSatisfiesOrderBy( isMatch = 1; break; } - if( isMatch && (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ /* Make sure the sort order is compatible in an ORDER BY clause. ** Sort order is irrelevant for a GROUP BY clause. */ if( revSet ){ @@ -119144,7 +134608,7 @@ static i8 wherePathSatisfiesOrderBy( } } if( isMatch ){ - if( iColumn<0 ){ + if( iColumn==XN_ROWID ){ testcase( distinctColumns==0 ); distinctColumns = 1; } @@ -119172,7 +134636,7 @@ static i8 wherePathSatisfiesOrderBy( Bitmask mTerm; if( MASKBIT(i) & obSat ) continue; p = pOrderBy->a[i].pExpr; - mTerm = exprTableUsage(&pWInfo->sMaskSet,p); + mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); @@ -119260,15 +134724,14 @@ static LogEst whereSortingCost( LogEst rScale, rSortCost; assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + estLog(nRow) + rScale + 16; + rSortCost = nRow + rScale + 16; - /* TUNING: The cost of implementing DISTINCT using a B-TREE is - ** similar but with a larger constant of proportionality. - ** Multiply by an additional factor of 3.0. */ - if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ - rSortCost += 16; + /* Multiple by log(M) where M is the number of output rows. + ** Use the LIMIT for M if it is smaller */ + if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimitiLimit; } - + rSortCost += estLog(nRow); return rSortCost; } @@ -119330,8 +134793,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; nSpace += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRaw(db, nSpace); - if( pSpace==0 ) return SQLITE_NOMEM; + pSpace = sqlite3DbMallocRawNN(db, nSpace); + if( pSpace==0 ) return SQLITE_NOMEM_BKPT; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; memset(aFrom, 0, sizeof(aFrom[0])); @@ -119354,10 +134817,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Seed the search with a single WherePath containing zero WhereLoops. ** - ** TUNING: Do not let the number of iterations go above 25. If the cost - ** of computing an automatic index is not paid back within the first 25 + ** TUNING: Do not let the number of iterations go above 28. If the cost + ** of computing an automatic index is not paid back within the first 28 ** rows, then do not use the automatic index. */ - aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==sqlite3LogEst(25) ); + aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) ); nFrom = 1; assert( aFrom[0].isOrdered==0 ); if( nOrderBy ){ @@ -119386,6 +134849,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<10 ){ + /* Do not use an automatic index if the this loop is expected + ** to run less than 2 times. */ + assert( 10==sqlite3LogEst(2) ); + continue; + } /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); @@ -119531,7 +135000,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } #ifdef WHERETRACE_ENABLED /* >=2 */ - if( sqlite3WhereTrace>=2 ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; iipResultSet, pFrom, + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pDistinctSet, pFrom, WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); - if( rc==pWInfo->pResultSet->nExpr ){ + if( rc==pWInfo->pDistinctSet->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } } @@ -119591,18 +135060,39 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } }else{ pWInfo->nOBSat = pFrom->isOrdered; - if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; pWInfo->revMask = pFrom->revLoop; + if( pWInfo->nOBSat<=0 ){ + pWInfo->nOBSat = 0; + if( nLoop>0 ){ + u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags; + if( (wsFlags & WHERE_ONEROW)==0 + && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN) + ){ + Bitmask m = 0; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, + WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m); + testcase( wsFlags & WHERE_IPK ); + testcase( wsFlags & WHERE_COLUMN_IN ); + if( rc==pWInfo->pOrderBy->nExpr ){ + pWInfo->bOrderedInnerLoop = 1; + pWInfo->revMask = m; + } + } + } + } } if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) - && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr + && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0 ){ - Bitmask notUsed = 0; + Bitmask revMask = 0; int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, - pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used + pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask ); assert( pWInfo->sorted==0 ); - pWInfo->sorted = (nOrder==pWInfo->pOrderBy->nExpr); + if( nOrder==pWInfo->pOrderBy->nExpr ){ + pWInfo->sorted = 1; + pWInfo->revMask = revMask; + } } } @@ -119635,21 +135125,22 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ int j; Table *pTab; Index *pIdx; - + pWInfo = pBuilder->pWInfo; - if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0; + if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0; assert( pWInfo->pTabList->nSrc>=1 ); pItem = pWInfo->pTabList->a; pTab = pItem->pTab; if( IsVirtual(pTab) ) return 0; - if( pItem->zIndex ) return 0; + if( pItem->fg.isIndexedBy ) return 0; iCur = pItem->iCursor; pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; - pLoop->u.btree.nSkip = 0; - pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); + pLoop->nSkip = 0; + pTerm = sqlite3WhereFindTerm(pWC, iCur, -1, 0, WO_EQ|WO_IS, 0); if( pTerm ){ + testcase( pTerm->eOperator & WO_IS ); pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; pLoop->aLTerm[0] = pTerm; pLoop->nLTerm = 1; @@ -119658,15 +135149,17 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int opMask; assert( pLoop->aLTermSpace==pLoop->aLTerm ); - assert( ArraySize(pLoop->aLTermSpace)==4 ); if( !IsUniqueIndex(pIdx) || pIdx->pPartIdxWhere!=0 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) ) continue; + opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; for(j=0; jnKeyCol; j++){ - pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx); + pTerm = sqlite3WhereFindTerm(pWC, iCur, j, 0, opMask, pIdx); if( pTerm==0 ) break; + testcase( pTerm->eOperator & WO_IS ); pLoop->aLTerm[j] = pTerm; } if( j!=pIdx->nKeyCol ) continue; @@ -119685,7 +135178,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ if( pLoop->wsFlags ){ pLoop->nOut = (LogEst)1; pWInfo->a[0].pWLoop = pLoop; - pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); + pLoop->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); pWInfo->a[0].iTabCur = iCur; pWInfo->nRowOut = 1; if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; @@ -119781,7 +135274,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. ** ** The iIdxCur parameter is the cursor number of an index. If -** WHERE_ONETABLE_ONLY is set, iIdxCur is the cursor number of an index +** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index ** to use for OR clause processing. The WHERE clause should use this ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is ** the first cursor in an array of cursors for all indices. iIdxCur should @@ -119789,13 +135282,14 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ ** used. */ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( - Parse *pParse, /* The parser context */ - SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ - Expr *pWhere, /* The WHERE clause */ - ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ - ExprList *pResultSet, /* Result set of the query */ - u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ - int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ + Expr *pWhere, /* The WHERE clause */ + ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ + ExprList *pDistinctSet, /* Try not to output two rows that duplicate these */ + u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ + int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number + ** If WHERE_USE_LIMIT, then the limit amount */ ){ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ int nTabList; /* Number of elements in pTabList */ @@ -119809,7 +135303,16 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ int rc; /* Return code */ + u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */ + assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( + (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + )); + + /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */ + assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + || (wctrlFlags & WHERE_USE_LIMIT)==0 ); /* Variable initialization */ db = pParse->db; @@ -119836,11 +135339,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } /* This function normally generates a nested loop for all tables in - ** pTabList. But if the WHERE_ONETABLE_ONLY flag is set, then we should + ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should ** only generate code for the first table in pTabList and assume that ** any cursors associated with subsequent tables are uninitialized. */ - nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc; + nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc; /* Allocate and initialize the WhereInfo structure that will become the ** return value. A single allocation is used to store the WhereInfo @@ -119850,21 +135353,26 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** some architectures. Hence the ROUND8() below. */ nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); - pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereLoop)); + pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); pWInfo = 0; goto whereBeginError; } - pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; - pWInfo->nLevel = nTabList; pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; - pWInfo->pResultSet = pResultSet; + pWInfo->pDistinctSet = pDistinctSet; + pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; + pWInfo->nLevel = nTabList; pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; + pWInfo->iLimit = iAuxArg; pWInfo->savedNQueryLoop = pParse->nQueryLoop; + memset(&pWInfo->nOBSat, 0, + offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); + memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); + assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ pMaskSet = &pWInfo->sMaskSet; sWLB.pWInfo = pWInfo; sWLB.pWC = &pWInfo->sWC; @@ -119879,8 +135387,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(&pWInfo->sWC, pWInfo); - whereSplit(&pWInfo->sWC, pWhere, TK_AND); + sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo); + sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND); /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -119904,85 +135412,68 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Assign a bit from the bitmask to every term in the FROM clause. ** - ** When assigning bitmask values to FROM clause cursors, it must be - ** the case that if X is the bitmask for the N-th FROM clause term then - ** the bitmask for all FROM clause terms to the left of the N-th term - ** is (X-1). An expression from the ON clause of a LEFT JOIN can use - ** its Expr.iRightJoinTable value to find the bitmask of the right table - ** of the join. Subtracting one from the right table bitmask gives a - ** bitmask for all tables to the left of the join. Knowing the bitmask - ** for all tables to the left of a left join is important. Ticket #3015. + ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally ** equal to pTabList->nSrc but might be shortened to 1 if the - ** WHERE_ONETABLE_ONLY flag is set. + ** WHERE_OR_SUBCLAUSE flag is set. */ for(ii=0; iinSrc; ii++){ createMask(pMaskSet, pTabList->a[ii].iCursor); + sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); } -#ifndef NDEBUG - { - Bitmask toTheLeft = 0; - for(ii=0; iinSrc; ii++){ - Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); - assert( (m-1)==toTheLeft ); - toTheLeft |= m; - } +#ifdef SQLITE_DEBUG + for(ii=0; iinSrc; ii++){ + Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); + assert( m==MASKBIT(ii) ); } #endif - /* Analyze all of the subexpressions. Note that exprAnalyze() might - ** add new virtual terms onto the end of the WHERE clause. We do not - ** want to analyze these virtual terms, so start analyzing at the end - ** and work forward so that the added virtual terms are never processed. - */ - exprAnalyzeAll(pTabList, &pWInfo->sWC); - if( db->mallocFailed ){ - goto whereBeginError; - } + /* Analyze all of the subexpressions. */ + sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); + if( db->mallocFailed ) goto whereBeginError; if( wctrlFlags & WHERE_WANT_DISTINCT ){ - if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ + if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pDistinctSet) ){ /* The DISTINCT marking is pointless. Ignore it. */ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; }else if( pOrderBy==0 ){ /* Try to ORDER BY the result set to make distinct processing easier */ pWInfo->wctrlFlags |= WHERE_DISTINCTBY; - pWInfo->pOrderBy = pResultSet; + pWInfo->pOrderBy = pDistinctSet; } } /* Construct the WhereLoop objects */ - WHERETRACE(0xffff,("*** Optimizer Start ***\n")); - /* Display all terms of the WHERE clause */ -#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN) - if( sqlite3WhereTrace & 0x100 ){ - int i; - Vdbe *v = pParse->pVdbe; - sqlite3ExplainBegin(v); - for(i=0; inTerm; i++){ - sqlite3ExplainPrintf(v, "#%-2d ", i); - sqlite3ExplainPush(v); - whereExplainTerm(v, &sWLB.pWC->a[i]); - sqlite3ExplainPop(v); - sqlite3ExplainNL(v); +#if defined(WHERETRACE_ENABLED) + if( sqlite3WhereTrace & 0xffff ){ + sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); + if( wctrlFlags & WHERE_USE_LIMIT ){ + sqlite3DebugPrintf(", limit: %d", iAuxArg); } - sqlite3ExplainFinish(v); - sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v)); + sqlite3DebugPrintf(")\n"); + } + if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ + sqlite3WhereClausePrint(sWLB.pWC); } #endif + if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; - /* Display all of the WhereLoop objects if wheretrace is enabled */ -#ifdef WHERETRACE_ENABLED /* !=0 */ - if( sqlite3WhereTrace ){ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ WhereLoop *p; int i; - static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" - "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ p->cId = zLabel[i%sizeof(zLabel)]; whereLoopPrint(p, sWLB.pWC); @@ -119998,14 +135489,13 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } } if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ - pWInfo->revMask = (Bitmask)(-1); + pWInfo->revMask = ALLBITS; } if( pParse->nErr || NEVER(db->mallocFailed) ){ goto whereBeginError; } -#ifdef WHERETRACE_ENABLED /* !=0 */ +#ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ - int ii; sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); if( pWInfo->nOBSat>0 ){ sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); @@ -120032,15 +135522,17 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( #endif /* Attempt to omit tables from the join that do not effect the result */ if( pWInfo->nLevel>=2 - && pResultSet!=0 + && pDistinctSet!=0 && OptimizationEnabled(db, SQLITE_OmitNoopJoin) ){ - Bitmask tabUsed = exprListTableUsage(pMaskSet, pResultSet); - if( sWLB.pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, sWLB.pOrderBy); + Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pDistinctSet); + if( sWLB.pOrderBy ){ + tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); + } while( pWInfo->nLevel>=2 ){ WhereTerm *pTerm, *pEnd; pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; - if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break; if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 && (pLoop->wsFlags & WHERE_ONEROW)==0 ){ @@ -120066,22 +135558,28 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. - ** The one-pass algorithm only works if the WHERE clause constrains - ** the statement to update a single row. */ assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 - && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ - pWInfo->okOnePass = 1; - if( HasRowid(pTabList->a[0].pTab) ){ - pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ + int wsFlags = pWInfo->a[0].pWLoop->wsFlags; + int bOnerow = (wsFlags & WHERE_ONEROW)!=0; + if( bOnerow + || ((wctrlFlags & WHERE_ONEPASS_MULTIROW)!=0 + && 0==(wsFlags & WHERE_VIRTUALTABLE)) + ){ + pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; + if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ + if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ + bFordelete = OPFLAG_FORDELETE; + } + pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY); + } } } /* Open all tables in the pTabList and any indices selected for ** searching those tables. */ - notReady = ~(Bitmask)0; for(ii=0, pLevel=pWInfo->a; iiwsFlags & WHERE_IDX_ONLY)==0 - && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ + && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){ int op = OP_OpenRead; - if( pWInfo->okOnePass ){ + if( pWInfo->eOnePass!=ONEPASS_OFF ){ op = OP_OpenWrite; pWInfo->aiCurOnePass[0] = pTabItem->iCursor; }; sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); assert( pTabItem->iCursor==pLevel->iTabCur ); - testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 ); - testcase( !pWInfo->okOnePass && pTab->nCol==BMS ); - if( !pWInfo->okOnePass && pTab->nColeOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); + testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); + if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nColcolUsed; int n = 0; for(; b; b=b>>1, n++){} - sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, - SQLITE_INT_TO_PTR(n), P4_INT32); + sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); assert( n<=pTab->nCol ); } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + if( pLoop->u.btree.pIndex!=0 ){ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); + }else +#endif + { + sqlite3VdbeChangeP5(v, bFordelete); + } +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, + (const u8*)&pTabItem->colUsed, P4_INT64); +#endif }else{ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } @@ -120129,18 +135638,18 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( Index *pIx = pLoop->u.btree.pIndex; int iIndexCur; int op = OP_OpenRead; - /* iIdxCur is always set if to a positive value if ONEPASS is possible */ - assert( iIdxCur!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); + /* iAuxArg is always set if to a positive value if ONEPASS is possible */ + assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) - && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 + && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){ /* This is one term of an OR-optimization using the PRIMARY KEY of a ** WITHOUT ROWID table. No need for a separate index */ iIndexCur = pLevel->iTabCur; op = 0; - }else if( pWInfo->okOnePass ){ + }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ Index *pJ = pTabItem->pTab->pIndex; - iIndexCur = iIdxCur; + iIndexCur = iAuxArg; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); while( ALWAYS(pJ) && pJ!=pIx ){ iIndexCur++; @@ -120148,9 +135657,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } op = OP_OpenWrite; pWInfo->aiCurOnePass[1] = iIndexCur; - }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ - iIndexCur = iIdxCur; - if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; + }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){ + iIndexCur = iAuxArg; + op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; } @@ -120160,11 +135669,31 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( op ){ sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIx); + if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 + && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 + ){ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ + } VdbeComment((v, "%s", pIx->zName)); +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + { + u64 colUsed = 0; + int ii, jj; + for(ii=0; iinColumn; ii++){ + jj = pIx->aiColumn[ii]; + if( jj<0 ) continue; + if( jj>63 ) jj = 63; + if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue; + colUsed |= ((u64)1)<<(ii<63 ? ii : 63); + } + sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0, + (u8*)&colUsed, P4_INT64); + } +#endif /* SQLITE_ENABLE_COLUMN_USED_MASK */ } } if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(&pWInfo->sMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -120175,7 +135704,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( */ notReady = ~(Bitmask)0; for(ii=0; iia[ii]; + wsFlags = pLevel->pWLoop->wsFlags; #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ constructAutomaticIndex(pParse, &pWInfo->sWC, @@ -120183,10 +135715,15 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; } #endif - explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + addrExplain = sqlite3WhereExplainOneScan( + pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags + ); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); - notReady = codeOneLoopStart(pWInfo, ii, notReady); + notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; + if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){ + sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain); + } } /* Done. */ @@ -120238,35 +135775,45 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); - sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); - VdbeCoverage(v); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); + if( pIn->eEndLoopOp!=OP_Noop ){ + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); + VdbeCoverage(v); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); + } sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } - sqlite3DbFree(db, pLevel->u.in.aInLoop); } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->addrSkip ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + sqlite3VdbeGoto(v, pLevel->addrSkip); VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); sqlite3VdbeJumpHere(v, pLevel->addrSkip); sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); } +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( pLevel->addrLikeRep ){ + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1), + pLevel->addrLikeRep); + VdbeCoverage(v); + } +#endif if( pLevel->iLeftJoin ){ + int ws = pLoop->wsFlags; addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); - if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){ + assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); + if( (ws & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); } - if( pLoop->wsFlags & WHERE_INDEXED ){ + if( (ws & WHERE_INDEXED) + || ((ws & WHERE_MULTI_OR) && pLevel->u.pCovidx) + ){ sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); } if( pLevel->op==OP_Return ){ sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrFirst); + sqlite3VdbeGoto(v, pLevel->addrFirst); } sqlite3VdbeJumpHere(v, addr); } @@ -120290,40 +135837,26 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pLoop = pLevel->pWLoop; /* For a co-routine, change all OP_Column references to the table of - ** the co-routine into OP_SCopy of result contained in a register. + ** the co-routine into OP_Copy of result contained in a register. ** OP_Rowid becomes OP_Null. */ - if( pTabItem->viaCoroutine && !db->mallocFailed ){ - last = sqlite3VdbeCurrentAddr(v); - k = pLevel->addrBody; - pOp = sqlite3VdbeGetOp(v, k); - for(; kp1!=pLevel->iTabCur ) continue; - if( pOp->opcode==OP_Column ){ - pOp->opcode = OP_Copy; - pOp->p1 = pOp->p2 + pTabItem->regResult; - pOp->p2 = pOp->p3; - pOp->p3 = 0; - }else if( pOp->opcode==OP_Rowid ){ - pOp->opcode = OP_Null; - pOp->p1 = 0; - pOp->p3 = 0; - } - } + if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ + translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, + pTabItem->regResult, 0); continue; } /* Close all of the cursors that were opened by sqlite3WhereBegin. ** Except, do not close cursors that will be reused by the OR optimization - ** (WHERE_OMIT_OPEN_CLOSE). And do not close the OP_OpenWrite cursors + ** (WHERE_OR_SUBCLAUSE). And do not close the OP_OpenWrite cursors ** created for the ONEPASS optimization. */ if( (pTab->tabFlags & TF_Ephemeral)==0 && pTab->pSelect==0 - && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){ int ws = pLoop->wsFlags; - if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ + if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); } if( (ws & WHERE_INDEXED)!=0 @@ -120350,7 +135883,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ pIdx = pLevel->u.pCovidx; } - if( pIdx && !db->mallocFailed ){ + if( pIdx + && (pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable)) + && !db->mallocFailed + ){ last = sqlite3VdbeCurrentAddr(v); k = pLevel->addrBody; pOp = sqlite3VdbeGetOp(v, k); @@ -120362,6 +135898,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ if( !HasRowid(pTab) ){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; + assert( x>=0 ); } x = sqlite3ColumnOfIndex(pIdx, x); if( x>=0 ){ @@ -120386,19 +135923,34 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ /************** End of where.c ***********************************************/ /************** Begin file parse.c *******************************************/ -/* Driver template for the LEMON parser generator. -** The author disclaims copyright to this source code. +/* +** 2000-05-29 ** -** This version of "lempar.c" is modified, slightly, for use by SQLite. -** The only modifications are the addition of a couple of NEVER() -** macros to disable tests that are needed in the case of a general -** LALR(1) grammar but which are always false in the -** specific grammar used by SQLite. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: */ -/* First off, code is included that follows the "include" declaration -** in the input grammar file. */ /* #include */ +/************ Begin %include sections from the grammar ************************/ +/* #include "sqliteInt.h" */ /* ** Disable all error recovery processing in the parser push-down @@ -120411,6 +135963,18 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ */ #define yytestcase(X) testcase(X) +/* +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. +*/ +#define YYPARSEFREENEVERNULL 1 + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE u64 + /* ** An instance of this structure holds information about the ** LIMIT clause of a SELECT statement. @@ -120420,15 +135984,6 @@ struct LimitVal { Expr *pOffset; /* The OFFSET expression. NULL if there is none */ }; -/* -** An instance of this structure is used to store the LIKE, -** GLOB, NOT LIKE, and NOT GLOB operators. -*/ -struct LikeOp { - Token eOperator; /* "like" or "glob" or "regexp" */ - int bNot; /* True if the NOT keyword is present */ -}; - /* ** An instance of the following structure describes the event of a ** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, @@ -120441,11 +135996,37 @@ struct LikeOp { struct TrigEvent { int a; IdList * b; }; /* -** An instance of this structure holds the ATTACH key and the key type. +** Disable lookaside memory allocation for objects that might be +** shared across database connections. */ -struct AttachKey { int type; Token key; }; +static void disableLookaside(Parse *pParse){ + pParse->disableLookaside++; + pParse->db->lookaside.bDisable++; +} + /* + ** For a compound SELECT statement, make sure p->pPrior->pNext==p for + ** all elements in the list. And make sure list length does not exceed + ** SQLITE_LIMIT_COMPOUND_SELECT. + */ + static void parserDoubleLinkSelect(Parse *pParse, Select *p){ + if( p->pPrior ){ + Select *pNext = 0, *pLoop; + int mxSelect, cnt = 0; + for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ + pLoop->pNext = pNext; + pLoop->selFlags |= SF_Compound; + } + if( (p->selFlags & SF_MultiValue)==0 && + (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && + cnt>mxSelect + ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); + } + } + } + /* This is a utility routine used to set the ExprSpan.zStart and ** ExprSpan.zEnd values of pOut so that the span covers the complete ** range of text beginning with pStart and going to the end of pEnd. @@ -120459,46 +136040,68 @@ struct AttachKey { int type; Token key; }; ** new Expr to populate pOut. Set the span of pOut to be the identifier ** that created the expression. */ - static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){ - pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, pValue); - pOut->zStart = pValue->z; - pOut->zEnd = &pValue->z[pValue->n]; + static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token t){ + Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); + if( p ){ + memset(p, 0, sizeof(Expr)); + p->op = (u8)op; + p->flags = EP_Leaf; + p->iAgg = -1; + p->u.zToken = (char*)&p[1]; + memcpy(p->u.zToken, t.z, t.n); + p->u.zToken[t.n] = 0; + if( sqlite3Isquote(p->u.zToken[0]) ){ + if( p->u.zToken[0]=='"' ) p->flags |= EP_DblQuoted; + sqlite3Dequote(p->u.zToken); + } +#if SQLITE_MAX_EXPR_DEPTH>0 + p->nHeight = 1; +#endif + } + pOut->pExpr = p; + pOut->zStart = t.z; + pOut->zEnd = &t.z[t.n]; } /* This routine constructs a binary expression node out of two ExprSpan ** objects and uses the result to populate a new ExprSpan object. */ static void spanBinaryExpr( - ExprSpan *pOut, /* Write the result here */ Parse *pParse, /* The parsing context. Errors accumulate here */ int op, /* The binary operation */ - ExprSpan *pLeft, /* The left operand */ + ExprSpan *pLeft, /* The left operand, and output */ ExprSpan *pRight /* The right operand */ ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); - pOut->zStart = pLeft->zStart; - pOut->zEnd = pRight->zEnd; + pLeft->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); + pLeft->zEnd = pRight->zEnd; + } + + /* If doNot is true, then add a TK_NOT Expr-node wrapper around the + ** outside of *ppExpr. + */ + static void exprNot(Parse *pParse, int doNot, ExprSpan *pSpan){ + if( doNot ){ + pSpan->pExpr = sqlite3PExpr(pParse, TK_NOT, pSpan->pExpr, 0, 0); + } } /* Construct an expression node for a unary postfix operator */ static void spanUnaryPostfix( - ExprSpan *pOut, /* Write the new expression node here */ Parse *pParse, /* Parsing context to record errors */ int op, /* The operator */ - ExprSpan *pOperand, /* The operand */ + ExprSpan *pOperand, /* The operand, and output */ Token *pPostOp /* The operand token for setting the span */ ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOut->zStart = pOperand->zStart; - pOut->zEnd = &pPostOp->z[pPostOp->n]; + pOperand->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOperand->zEnd = &pPostOp->z[pPostOp->n]; } /* A routine to convert a binary TK_IS or TK_ISNOT expression into a ** unary TK_ISNULL or TK_NOTNULL expression. */ static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ sqlite3 *db = pParse->db; - if( db->mallocFailed==0 && pY->op==TK_NULL ){ + if( pA && pY && pY->op==TK_NULL ){ pA->op = (u8)op; sqlite3ExprDelete(db, pA->pRight); pA->pRight = 0; @@ -120514,82 +136117,111 @@ struct AttachKey { int type; Token key; }; ExprSpan *pOperand, /* The operand */ Token *pPreOp /* The operand token for setting the span */ ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); pOut->zStart = pPreOp->z; + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); pOut->zEnd = pOperand->zEnd; } -/* Next is all token values, in a form suitable for use by makeheaders. -** This section will be null unless lemon is run with the -m switch. -*/ -/* -** These constants (all generated automatically by the parser generator) -** specify the various kinds of tokens (terminals) that the parser -** understands. -** -** Each symbol here is a terminal symbol in the grammar. -*/ -/* Make sure the INTERFACE macro is defined. -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/* The next thing included is series of defines which control + + /* Add a single new term to an ExprList that is used to store a + ** list of identifiers. Report an error if the ID list contains + ** a COLLATE clause or an ASC or DESC keyword, except ignore the + ** error while parsing a legacy schema. + */ + static ExprList *parserAddExprIdListTerm( + Parse *pParse, + ExprList *pPrior, + Token *pIdToken, + int hasCollate, + int sortOrder + ){ + ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); + if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) + && pParse->db->init.busy==0 + ){ + sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", + pIdToken->n, pIdToken->z); + } + sqlite3ExprListSetName(pParse, p, pIdToken, 1); + return p; + } +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols +** in a format understandable to "makeheaders". This section is blank unless +** "lemon" is run with the "-m" command-line option. +***************** Begin makeheaders token definitions *************************/ +/**************** End makeheaders token definitions ***************************/ + +/* The next sections is a series of control #defines. ** various aspects of the generated parser. -** YYCODETYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 terminals -** and nonterminals. "int" is used otherwise. -** YYNOCODE is a number of type YYCODETYPE which corresponds -** to no legal terminal or nonterminal number. This -** number is used to fill in empty slots of the hash -** table. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. ** YYFALLBACK If defined, this indicates that one or more tokens -** have fall-back values which should be used if the -** original value of the token will not parse. -** YYACTIONTYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 rules and -** states combined. "int" is used otherwise. -** sqlite3ParserTOKENTYPE is the data type used for minor tokens given -** directly to the parser from the tokenizer. -** YYMINORTYPE is the data type used for all minor tokens. +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** sqlite3ParserTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. ** This is typically a union of many types, one of ** which is sqlite3ParserTOKENTYPE. The entry in the union -** for base tokens is called "yy0". +** for terminal symbols is called "yy0". ** YYSTACKDEPTH is the maximum depth of the parser's stack. If ** zero the stack is dynamically sized using realloc() ** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument ** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument ** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser ** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_MIN_REDUCE Maximum value for reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op */ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 254 +#define YYNOCODE 252 #define YYACTIONTYPE unsigned short int -#define YYWILDCARD 70 +#define YYWILDCARD 96 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - Select* yy3; - ExprList* yy14; - With* yy59; - SrcList* yy65; - struct LikeOp yy96; - Expr* yy132; - u8 yy186; - int yy328; - ExprSpan yy346; - struct TrigEvent yy378; - u16 yy381; - IdList* yy408; - struct {int value; int mask;} yy429; - TriggerStep* yy473; - struct LimitVal yy476; + Expr* yy72; + TriggerStep* yy145; + ExprList* yy148; + SrcList* yy185; + ExprSpan yy190; + int yy194; + Select* yy243; + IdList* yy254; + With* yy285; + struct TrigEvent yy332; + struct LimitVal yy354; + struct {int value; int mask;} yy497; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -120598,16 +136230,18 @@ typedef union { #define sqlite3ParserARG_PDECL ,Parse *pParse #define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse #define sqlite3ParserARG_STORE yypParser->pParse = pParse -#define YYNSTATE 642 -#define YYNRULE 327 #define YYFALLBACK 1 -#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) -#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) -#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) - -/* The yyzerominor constant is used to initialize instances of -** YYMINORTYPE objects to zero. */ -static const YYMINORTYPE yyzerominor = { 0 }; +#define YYNSTATE 456 +#define YYNRULE 332 +#define YY_MAX_SHIFT 455 +#define YY_MIN_SHIFTREDUCE 668 +#define YY_MAX_SHIFTREDUCE 999 +#define YY_MIN_REDUCE 1000 +#define YY_MAX_REDUCE 1331 +#define YY_ERROR_ACTION 1332 +#define YY_ACCEPT_ACTION 1333 +#define YY_NO_ACTION 1334 +/************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -120630,29 +136264,37 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N == YYNSTATE+YYNRULE A syntax error has occurred. +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE ** -** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** N == YY_ERROR_ACTION A syntax error has occurred. ** -** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** N == YY_ACCEPT_ACTION The parser accepts its input. +** +** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** ** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as +** Given state S and lookahead X, the action is computed as either: ** -** yy_action[ yy_shift_ofst[S] + X ] +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] ** -** If the index value yy_shift_ofst[S]+X is out of range or if the value -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] -** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table -** and that yy_default[S] should be used instead. +** The (A) formula is preferred. The B formula is used instead if: +** (1) The yy_shift_ofst[S]+X value is out of range, or +** (2) yy_lookahead[yy_shift_ofst[S]+X] is not equal to X, or +** (3) yy_shift_ofst[S] equal YY_SHIFT_USE_DFLT. +** (Implementation note: YY_SHIFT_USE_DFLT is chosen so that +** YY_SHIFT_USE_DFLT+X will be out of range for all possible lookaheads X. +** Hence only tests (1) and (2) need to be evaluated.) ** -** The formula above is for computing the action when the lookahead is +** The formulas above are for computing the action when the lookahead is ** a terminal symbol. If the lookahead is a non-terminal (as occurs after ** a reduce action) then the yy_reduce_ofst[] array is used in place of ** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of @@ -120668,468 +136310,470 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** yy_reduce_ofst[] For each state, the offset into yy_action for ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. -*/ -#define YY_ACTTAB_COUNT (1497) +** +*********** Begin parsing tables **********************************************/ +#define YY_ACTTAB_COUNT (1567) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 306, 212, 432, 955, 639, 191, 955, 295, 559, 88, - /* 10 */ 88, 88, 88, 81, 86, 86, 86, 86, 85, 85, - /* 20 */ 84, 84, 84, 83, 330, 185, 184, 183, 635, 635, - /* 30 */ 292, 606, 606, 88, 88, 88, 88, 683, 86, 86, - /* 40 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 16, - /* 50 */ 436, 597, 89, 90, 80, 600, 599, 601, 601, 87, - /* 60 */ 87, 88, 88, 88, 88, 684, 86, 86, 86, 86, - /* 70 */ 85, 85, 84, 84, 84, 83, 330, 306, 559, 84, - /* 80 */ 84, 84, 83, 330, 65, 86, 86, 86, 86, 85, - /* 90 */ 85, 84, 84, 84, 83, 330, 635, 635, 634, 633, - /* 100 */ 182, 682, 550, 379, 376, 375, 17, 322, 606, 606, - /* 110 */ 371, 198, 479, 91, 374, 82, 79, 165, 85, 85, - /* 120 */ 84, 84, 84, 83, 330, 598, 635, 635, 107, 89, - /* 130 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 140 */ 88, 88, 186, 86, 86, 86, 86, 85, 85, 84, - /* 150 */ 84, 84, 83, 330, 306, 594, 594, 142, 328, 327, - /* 160 */ 484, 249, 344, 238, 635, 635, 634, 633, 585, 448, - /* 170 */ 526, 525, 229, 388, 1, 394, 450, 584, 449, 635, - /* 180 */ 635, 635, 635, 319, 395, 606, 606, 199, 157, 273, - /* 190 */ 382, 268, 381, 187, 635, 635, 634, 633, 311, 555, - /* 200 */ 266, 593, 593, 266, 347, 588, 89, 90, 80, 600, - /* 210 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 478, - /* 220 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, - /* 230 */ 330, 306, 272, 536, 634, 633, 146, 610, 197, 310, - /* 240 */ 575, 182, 482, 271, 379, 376, 375, 506, 21, 634, - /* 250 */ 633, 634, 633, 635, 635, 374, 611, 574, 548, 440, - /* 260 */ 111, 563, 606, 606, 634, 633, 324, 479, 608, 608, - /* 270 */ 608, 300, 435, 573, 119, 407, 210, 162, 562, 883, - /* 280 */ 592, 592, 306, 89, 90, 80, 600, 599, 601, 601, - /* 290 */ 87, 87, 88, 88, 88, 88, 506, 86, 86, 86, - /* 300 */ 86, 85, 85, 84, 84, 84, 83, 330, 620, 111, - /* 310 */ 635, 635, 361, 606, 606, 358, 249, 349, 248, 433, - /* 320 */ 243, 479, 586, 634, 633, 195, 611, 93, 119, 221, - /* 330 */ 575, 497, 534, 534, 89, 90, 80, 600, 599, 601, - /* 340 */ 601, 87, 87, 88, 88, 88, 88, 574, 86, 86, - /* 350 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 306, - /* 360 */ 77, 429, 638, 573, 589, 530, 240, 230, 242, 105, - /* 370 */ 249, 349, 248, 515, 588, 208, 460, 529, 564, 173, - /* 380 */ 634, 633, 970, 144, 430, 2, 424, 228, 380, 557, - /* 390 */ 606, 606, 190, 153, 159, 158, 514, 51, 632, 631, - /* 400 */ 630, 71, 536, 432, 954, 196, 610, 954, 614, 45, - /* 410 */ 18, 89, 90, 80, 600, 599, 601, 601, 87, 87, - /* 420 */ 88, 88, 88, 88, 261, 86, 86, 86, 86, 85, - /* 430 */ 85, 84, 84, 84, 83, 330, 306, 608, 608, 608, - /* 440 */ 542, 424, 402, 385, 241, 506, 451, 320, 211, 543, - /* 450 */ 164, 436, 386, 293, 451, 587, 108, 496, 111, 334, - /* 460 */ 391, 591, 424, 614, 27, 452, 453, 606, 606, 72, - /* 470 */ 257, 70, 259, 452, 339, 342, 564, 582, 68, 415, - /* 480 */ 469, 328, 327, 62, 614, 45, 110, 393, 89, 90, - /* 490 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, - /* 500 */ 88, 152, 86, 86, 86, 86, 85, 85, 84, 84, - /* 510 */ 84, 83, 330, 306, 110, 499, 520, 538, 402, 389, - /* 520 */ 424, 110, 566, 500, 593, 593, 454, 82, 79, 165, - /* 530 */ 424, 591, 384, 564, 340, 615, 188, 162, 424, 350, - /* 540 */ 616, 424, 614, 44, 606, 606, 445, 582, 300, 434, - /* 550 */ 151, 19, 614, 9, 568, 580, 348, 615, 469, 567, - /* 560 */ 614, 26, 616, 614, 45, 89, 90, 80, 600, 599, - /* 570 */ 601, 601, 87, 87, 88, 88, 88, 88, 411, 86, - /* 580 */ 86, 86, 86, 85, 85, 84, 84, 84, 83, 330, - /* 590 */ 306, 579, 110, 578, 521, 282, 433, 398, 400, 255, - /* 600 */ 486, 82, 79, 165, 487, 164, 82, 79, 165, 488, - /* 610 */ 488, 364, 387, 424, 544, 544, 509, 350, 362, 155, - /* 620 */ 191, 606, 606, 559, 642, 640, 333, 82, 79, 165, - /* 630 */ 305, 564, 507, 312, 357, 614, 45, 329, 596, 595, - /* 640 */ 194, 337, 89, 90, 80, 600, 599, 601, 601, 87, - /* 650 */ 87, 88, 88, 88, 88, 424, 86, 86, 86, 86, - /* 660 */ 85, 85, 84, 84, 84, 83, 330, 306, 20, 323, - /* 670 */ 150, 263, 211, 543, 421, 596, 595, 614, 22, 424, - /* 680 */ 193, 424, 284, 424, 391, 424, 509, 424, 577, 424, - /* 690 */ 186, 335, 424, 559, 424, 313, 120, 546, 606, 606, - /* 700 */ 67, 614, 47, 614, 50, 614, 48, 614, 100, 614, - /* 710 */ 99, 614, 101, 576, 614, 102, 614, 109, 326, 89, - /* 720 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 730 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, - /* 740 */ 84, 84, 83, 330, 306, 424, 311, 424, 585, 54, - /* 750 */ 424, 516, 517, 590, 614, 112, 424, 584, 424, 572, - /* 760 */ 424, 195, 424, 571, 424, 67, 424, 614, 94, 614, - /* 770 */ 98, 424, 614, 97, 264, 606, 606, 195, 614, 46, - /* 780 */ 614, 96, 614, 30, 614, 49, 614, 115, 614, 114, - /* 790 */ 418, 229, 388, 614, 113, 306, 89, 90, 80, 600, - /* 800 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 424, - /* 810 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, - /* 820 */ 330, 119, 424, 590, 110, 372, 606, 606, 195, 53, - /* 830 */ 250, 614, 29, 195, 472, 438, 729, 190, 302, 498, - /* 840 */ 14, 523, 641, 2, 614, 43, 306, 89, 90, 80, - /* 850 */ 600, 599, 601, 601, 87, 87, 88, 88, 88, 88, - /* 860 */ 424, 86, 86, 86, 86, 85, 85, 84, 84, 84, - /* 870 */ 83, 330, 424, 613, 964, 964, 354, 606, 606, 420, - /* 880 */ 312, 64, 614, 42, 391, 355, 283, 437, 301, 255, - /* 890 */ 414, 410, 495, 492, 614, 28, 471, 306, 89, 90, - /* 900 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, - /* 910 */ 88, 424, 86, 86, 86, 86, 85, 85, 84, 84, - /* 920 */ 84, 83, 330, 424, 110, 110, 110, 110, 606, 606, - /* 930 */ 110, 254, 13, 614, 41, 532, 531, 283, 481, 531, - /* 940 */ 457, 284, 119, 561, 356, 614, 40, 284, 306, 89, - /* 950 */ 78, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 960 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, - /* 970 */ 84, 84, 83, 330, 110, 424, 341, 220, 555, 606, - /* 980 */ 606, 351, 555, 318, 614, 95, 413, 255, 83, 330, - /* 990 */ 284, 284, 255, 640, 333, 356, 255, 614, 39, 306, - /* 1000 */ 356, 90, 80, 600, 599, 601, 601, 87, 87, 88, - /* 1010 */ 88, 88, 88, 424, 86, 86, 86, 86, 85, 85, - /* 1020 */ 84, 84, 84, 83, 330, 424, 317, 316, 141, 465, - /* 1030 */ 606, 606, 219, 619, 463, 614, 10, 417, 462, 255, - /* 1040 */ 189, 510, 553, 351, 207, 363, 161, 614, 38, 315, - /* 1050 */ 218, 255, 255, 80, 600, 599, 601, 601, 87, 87, - /* 1060 */ 88, 88, 88, 88, 424, 86, 86, 86, 86, 85, - /* 1070 */ 85, 84, 84, 84, 83, 330, 76, 419, 255, 3, - /* 1080 */ 878, 461, 424, 247, 331, 331, 614, 37, 217, 76, - /* 1090 */ 419, 390, 3, 216, 215, 422, 4, 331, 331, 424, - /* 1100 */ 547, 12, 424, 545, 614, 36, 424, 541, 422, 424, - /* 1110 */ 540, 424, 214, 424, 408, 424, 539, 403, 605, 605, - /* 1120 */ 237, 614, 25, 119, 614, 24, 588, 408, 614, 45, - /* 1130 */ 118, 614, 35, 614, 34, 614, 33, 614, 23, 588, - /* 1140 */ 60, 223, 603, 602, 513, 378, 73, 74, 140, 139, - /* 1150 */ 424, 110, 265, 75, 426, 425, 59, 424, 610, 73, - /* 1160 */ 74, 549, 402, 404, 424, 373, 75, 426, 425, 604, - /* 1170 */ 138, 610, 614, 11, 392, 76, 419, 181, 3, 614, - /* 1180 */ 32, 271, 369, 331, 331, 493, 614, 31, 149, 608, - /* 1190 */ 608, 608, 607, 15, 422, 365, 614, 8, 137, 489, - /* 1200 */ 136, 190, 608, 608, 608, 607, 15, 485, 176, 135, - /* 1210 */ 7, 252, 477, 408, 174, 133, 175, 474, 57, 56, - /* 1220 */ 132, 130, 119, 76, 419, 588, 3, 468, 245, 464, - /* 1230 */ 171, 331, 331, 125, 123, 456, 447, 122, 446, 104, - /* 1240 */ 336, 231, 422, 166, 154, 73, 74, 332, 116, 431, - /* 1250 */ 121, 309, 75, 426, 425, 222, 106, 610, 308, 637, - /* 1260 */ 204, 408, 629, 627, 628, 6, 200, 428, 427, 290, - /* 1270 */ 203, 622, 201, 588, 62, 63, 289, 66, 419, 399, - /* 1280 */ 3, 401, 288, 92, 143, 331, 331, 287, 608, 608, - /* 1290 */ 608, 607, 15, 73, 74, 227, 422, 325, 69, 416, - /* 1300 */ 75, 426, 425, 612, 412, 610, 192, 61, 569, 209, - /* 1310 */ 396, 226, 278, 225, 383, 408, 527, 558, 276, 533, - /* 1320 */ 552, 528, 321, 523, 370, 508, 180, 588, 494, 179, - /* 1330 */ 366, 117, 253, 269, 522, 503, 608, 608, 608, 607, - /* 1340 */ 15, 551, 502, 58, 274, 524, 178, 73, 74, 304, - /* 1350 */ 501, 368, 303, 206, 75, 426, 425, 491, 360, 610, - /* 1360 */ 213, 177, 483, 131, 345, 298, 297, 296, 202, 294, - /* 1370 */ 480, 490, 466, 134, 172, 129, 444, 346, 470, 128, - /* 1380 */ 314, 459, 103, 127, 126, 148, 124, 167, 443, 235, - /* 1390 */ 608, 608, 608, 607, 15, 442, 439, 623, 234, 299, - /* 1400 */ 145, 583, 291, 377, 581, 160, 119, 156, 270, 636, - /* 1410 */ 971, 169, 279, 626, 520, 625, 473, 624, 170, 621, - /* 1420 */ 618, 119, 168, 55, 409, 423, 537, 609, 286, 285, - /* 1430 */ 405, 570, 560, 556, 5, 52, 458, 554, 147, 267, - /* 1440 */ 519, 504, 518, 406, 262, 239, 260, 512, 343, 511, - /* 1450 */ 258, 353, 565, 256, 224, 251, 359, 277, 275, 476, - /* 1460 */ 475, 246, 352, 244, 467, 455, 236, 233, 232, 307, - /* 1470 */ 441, 281, 205, 163, 397, 280, 535, 505, 330, 617, - /* 1480 */ 971, 971, 971, 971, 367, 971, 971, 971, 971, 971, - /* 1490 */ 971, 971, 971, 971, 971, 971, 338, + /* 0 */ 325, 832, 351, 825, 5, 203, 203, 819, 99, 100, + /* 10 */ 90, 842, 842, 854, 857, 846, 846, 97, 97, 98, + /* 20 */ 98, 98, 98, 301, 96, 96, 96, 96, 95, 95, + /* 30 */ 94, 94, 94, 93, 351, 325, 977, 977, 824, 824, + /* 40 */ 826, 947, 354, 99, 100, 90, 842, 842, 854, 857, + /* 50 */ 846, 846, 97, 97, 98, 98, 98, 98, 338, 96, + /* 60 */ 96, 96, 96, 95, 95, 94, 94, 94, 93, 351, + /* 70 */ 95, 95, 94, 94, 94, 93, 351, 791, 977, 977, + /* 80 */ 325, 94, 94, 94, 93, 351, 792, 75, 99, 100, + /* 90 */ 90, 842, 842, 854, 857, 846, 846, 97, 97, 98, + /* 100 */ 98, 98, 98, 450, 96, 96, 96, 96, 95, 95, + /* 110 */ 94, 94, 94, 93, 351, 1333, 155, 155, 2, 325, + /* 120 */ 275, 146, 132, 52, 52, 93, 351, 99, 100, 90, + /* 130 */ 842, 842, 854, 857, 846, 846, 97, 97, 98, 98, + /* 140 */ 98, 98, 101, 96, 96, 96, 96, 95, 95, 94, + /* 150 */ 94, 94, 93, 351, 958, 958, 325, 268, 428, 413, + /* 160 */ 411, 61, 752, 752, 99, 100, 90, 842, 842, 854, + /* 170 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 60, + /* 180 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, + /* 190 */ 351, 325, 270, 329, 273, 277, 959, 960, 250, 99, + /* 200 */ 100, 90, 842, 842, 854, 857, 846, 846, 97, 97, + /* 210 */ 98, 98, 98, 98, 301, 96, 96, 96, 96, 95, + /* 220 */ 95, 94, 94, 94, 93, 351, 325, 938, 1326, 698, + /* 230 */ 706, 1326, 242, 412, 99, 100, 90, 842, 842, 854, + /* 240 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 347, + /* 250 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, + /* 260 */ 351, 325, 938, 1327, 384, 699, 1327, 381, 379, 99, + /* 270 */ 100, 90, 842, 842, 854, 857, 846, 846, 97, 97, + /* 280 */ 98, 98, 98, 98, 701, 96, 96, 96, 96, 95, + /* 290 */ 95, 94, 94, 94, 93, 351, 325, 92, 89, 178, + /* 300 */ 833, 936, 373, 700, 99, 100, 90, 842, 842, 854, + /* 310 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 375, + /* 320 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, + /* 330 */ 351, 325, 1276, 947, 354, 818, 936, 739, 739, 99, + /* 340 */ 100, 90, 842, 842, 854, 857, 846, 846, 97, 97, + /* 350 */ 98, 98, 98, 98, 230, 96, 96, 96, 96, 95, + /* 360 */ 95, 94, 94, 94, 93, 351, 325, 969, 227, 92, + /* 370 */ 89, 178, 373, 300, 99, 100, 90, 842, 842, 854, + /* 380 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 921, + /* 390 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, + /* 400 */ 351, 325, 449, 447, 447, 447, 147, 737, 737, 99, + /* 410 */ 100, 90, 842, 842, 854, 857, 846, 846, 97, 97, + /* 420 */ 98, 98, 98, 98, 296, 96, 96, 96, 96, 95, + /* 430 */ 95, 94, 94, 94, 93, 351, 325, 419, 231, 958, + /* 440 */ 958, 158, 25, 422, 99, 100, 90, 842, 842, 854, + /* 450 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 450, + /* 460 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, + /* 470 */ 351, 443, 224, 224, 420, 958, 958, 962, 325, 52, + /* 480 */ 52, 959, 960, 176, 415, 78, 99, 100, 90, 842, + /* 490 */ 842, 854, 857, 846, 846, 97, 97, 98, 98, 98, + /* 500 */ 98, 379, 96, 96, 96, 96, 95, 95, 94, 94, + /* 510 */ 94, 93, 351, 325, 428, 418, 298, 959, 960, 962, + /* 520 */ 81, 99, 88, 90, 842, 842, 854, 857, 846, 846, + /* 530 */ 97, 97, 98, 98, 98, 98, 717, 96, 96, 96, + /* 540 */ 96, 95, 95, 94, 94, 94, 93, 351, 325, 843, + /* 550 */ 843, 855, 858, 996, 318, 343, 379, 100, 90, 842, + /* 560 */ 842, 854, 857, 846, 846, 97, 97, 98, 98, 98, + /* 570 */ 98, 450, 96, 96, 96, 96, 95, 95, 94, 94, + /* 580 */ 94, 93, 351, 325, 350, 350, 350, 260, 377, 340, + /* 590 */ 929, 52, 52, 90, 842, 842, 854, 857, 846, 846, + /* 600 */ 97, 97, 98, 98, 98, 98, 361, 96, 96, 96, + /* 610 */ 96, 95, 95, 94, 94, 94, 93, 351, 86, 445, + /* 620 */ 847, 3, 1203, 361, 360, 378, 344, 813, 958, 958, + /* 630 */ 1300, 86, 445, 729, 3, 212, 169, 287, 405, 282, + /* 640 */ 404, 199, 232, 450, 300, 760, 83, 84, 280, 245, + /* 650 */ 262, 365, 251, 85, 352, 352, 92, 89, 178, 83, + /* 660 */ 84, 242, 412, 52, 52, 448, 85, 352, 352, 246, + /* 670 */ 959, 960, 194, 455, 670, 402, 399, 398, 448, 243, + /* 680 */ 221, 114, 434, 776, 361, 450, 397, 268, 747, 224, + /* 690 */ 224, 132, 132, 198, 832, 434, 452, 451, 428, 427, + /* 700 */ 819, 415, 734, 713, 132, 52, 52, 832, 268, 452, + /* 710 */ 451, 734, 194, 819, 363, 402, 399, 398, 450, 1271, + /* 720 */ 1271, 23, 958, 958, 86, 445, 397, 3, 228, 429, + /* 730 */ 895, 824, 824, 826, 827, 19, 203, 720, 52, 52, + /* 740 */ 428, 408, 439, 249, 824, 824, 826, 827, 19, 229, + /* 750 */ 403, 153, 83, 84, 761, 177, 241, 450, 721, 85, + /* 760 */ 352, 352, 120, 157, 959, 960, 58, 977, 409, 355, + /* 770 */ 330, 448, 268, 428, 430, 320, 790, 32, 32, 86, + /* 780 */ 445, 776, 3, 341, 98, 98, 98, 98, 434, 96, + /* 790 */ 96, 96, 96, 95, 95, 94, 94, 94, 93, 351, + /* 800 */ 832, 120, 452, 451, 813, 887, 819, 83, 84, 977, + /* 810 */ 813, 132, 410, 920, 85, 352, 352, 132, 407, 789, + /* 820 */ 958, 958, 92, 89, 178, 917, 448, 262, 370, 261, + /* 830 */ 82, 914, 80, 262, 370, 261, 776, 824, 824, 826, + /* 840 */ 827, 19, 934, 434, 96, 96, 96, 96, 95, 95, + /* 850 */ 94, 94, 94, 93, 351, 832, 74, 452, 451, 958, + /* 860 */ 958, 819, 959, 960, 120, 92, 89, 178, 945, 2, + /* 870 */ 918, 965, 268, 1, 976, 76, 445, 762, 3, 708, + /* 880 */ 901, 901, 387, 958, 958, 757, 919, 371, 740, 778, + /* 890 */ 756, 257, 824, 824, 826, 827, 19, 417, 741, 450, + /* 900 */ 24, 959, 960, 83, 84, 369, 958, 958, 177, 226, + /* 910 */ 85, 352, 352, 885, 315, 314, 313, 215, 311, 10, + /* 920 */ 10, 683, 448, 349, 348, 959, 960, 909, 777, 157, + /* 930 */ 120, 958, 958, 337, 776, 416, 711, 310, 450, 434, + /* 940 */ 450, 321, 450, 791, 103, 200, 175, 450, 959, 960, + /* 950 */ 908, 832, 792, 452, 451, 9, 9, 819, 10, 10, + /* 960 */ 52, 52, 51, 51, 180, 716, 248, 10, 10, 171, + /* 970 */ 170, 167, 339, 959, 960, 247, 984, 702, 702, 450, + /* 980 */ 715, 233, 686, 982, 889, 983, 182, 914, 824, 824, + /* 990 */ 826, 827, 19, 183, 256, 423, 132, 181, 394, 10, + /* 1000 */ 10, 889, 891, 749, 958, 958, 917, 268, 985, 198, + /* 1010 */ 985, 349, 348, 425, 415, 299, 817, 832, 326, 825, + /* 1020 */ 120, 332, 133, 819, 268, 98, 98, 98, 98, 91, + /* 1030 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, + /* 1040 */ 351, 157, 810, 371, 382, 359, 959, 960, 358, 268, + /* 1050 */ 450, 918, 368, 324, 824, 824, 826, 450, 709, 450, + /* 1060 */ 264, 380, 889, 450, 877, 746, 253, 919, 255, 433, + /* 1070 */ 36, 36, 234, 450, 234, 120, 269, 37, 37, 12, + /* 1080 */ 12, 334, 272, 27, 27, 450, 330, 118, 450, 162, + /* 1090 */ 742, 280, 450, 38, 38, 450, 985, 356, 985, 450, + /* 1100 */ 709, 1210, 450, 132, 450, 39, 39, 450, 40, 40, + /* 1110 */ 450, 362, 41, 41, 450, 42, 42, 450, 254, 28, + /* 1120 */ 28, 450, 29, 29, 31, 31, 450, 43, 43, 450, + /* 1130 */ 44, 44, 450, 714, 45, 45, 450, 11, 11, 767, + /* 1140 */ 450, 46, 46, 450, 268, 450, 105, 105, 450, 47, + /* 1150 */ 47, 450, 48, 48, 450, 237, 33, 33, 450, 172, + /* 1160 */ 49, 49, 450, 50, 50, 34, 34, 274, 122, 122, + /* 1170 */ 450, 123, 123, 450, 124, 124, 450, 898, 56, 56, + /* 1180 */ 450, 897, 35, 35, 450, 267, 450, 817, 450, 817, + /* 1190 */ 106, 106, 450, 53, 53, 385, 107, 107, 450, 817, + /* 1200 */ 108, 108, 817, 450, 104, 104, 121, 121, 119, 119, + /* 1210 */ 450, 117, 112, 112, 450, 276, 450, 225, 111, 111, + /* 1220 */ 450, 730, 450, 109, 109, 450, 673, 674, 675, 912, + /* 1230 */ 110, 110, 317, 998, 55, 55, 57, 57, 692, 331, + /* 1240 */ 54, 54, 26, 26, 696, 30, 30, 317, 937, 197, + /* 1250 */ 196, 195, 335, 281, 336, 446, 331, 745, 689, 436, + /* 1260 */ 440, 444, 120, 72, 386, 223, 175, 345, 757, 933, + /* 1270 */ 20, 286, 319, 756, 815, 372, 374, 202, 202, 202, + /* 1280 */ 263, 395, 285, 74, 208, 21, 696, 719, 718, 884, + /* 1290 */ 120, 120, 120, 120, 120, 754, 278, 828, 77, 74, + /* 1300 */ 726, 727, 785, 783, 880, 202, 999, 208, 894, 893, + /* 1310 */ 894, 893, 694, 816, 763, 116, 774, 1290, 431, 432, + /* 1320 */ 302, 999, 390, 303, 823, 697, 691, 680, 159, 289, + /* 1330 */ 679, 884, 681, 952, 291, 218, 293, 7, 316, 828, + /* 1340 */ 173, 805, 259, 364, 252, 911, 376, 713, 295, 435, + /* 1350 */ 308, 168, 955, 993, 135, 400, 990, 284, 882, 881, + /* 1360 */ 205, 928, 926, 59, 333, 62, 144, 156, 130, 72, + /* 1370 */ 802, 366, 367, 393, 137, 185, 189, 160, 139, 383, + /* 1380 */ 67, 896, 140, 141, 142, 148, 389, 812, 775, 266, + /* 1390 */ 219, 190, 154, 391, 913, 876, 271, 406, 191, 322, + /* 1400 */ 682, 733, 192, 342, 732, 724, 731, 711, 723, 421, + /* 1410 */ 705, 71, 323, 6, 204, 771, 288, 79, 297, 346, + /* 1420 */ 772, 704, 290, 283, 703, 770, 292, 294, 967, 239, + /* 1430 */ 769, 102, 862, 438, 426, 240, 424, 442, 73, 213, + /* 1440 */ 688, 238, 22, 453, 953, 214, 217, 216, 454, 677, + /* 1450 */ 676, 671, 753, 125, 115, 235, 126, 669, 353, 166, + /* 1460 */ 127, 244, 179, 357, 306, 304, 305, 307, 113, 892, + /* 1470 */ 327, 890, 811, 328, 134, 128, 136, 138, 743, 258, + /* 1480 */ 907, 184, 143, 129, 910, 186, 63, 64, 145, 187, + /* 1490 */ 906, 65, 8, 66, 13, 188, 202, 899, 265, 149, + /* 1500 */ 987, 388, 150, 685, 161, 392, 285, 193, 279, 396, + /* 1510 */ 151, 401, 68, 14, 15, 722, 69, 236, 831, 131, + /* 1520 */ 830, 860, 70, 751, 16, 414, 755, 4, 174, 220, + /* 1530 */ 222, 784, 201, 152, 779, 77, 74, 17, 18, 875, + /* 1540 */ 861, 859, 916, 864, 915, 207, 206, 942, 163, 437, + /* 1550 */ 948, 943, 164, 209, 1002, 441, 863, 165, 210, 829, + /* 1560 */ 695, 87, 312, 211, 1292, 1291, 309, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 19, 22, 22, 23, 1, 24, 26, 15, 27, 80, - /* 10 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 20 */ 91, 92, 93, 94, 95, 108, 109, 110, 27, 28, - /* 30 */ 23, 50, 51, 80, 81, 82, 83, 122, 85, 86, - /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 22, - /* 50 */ 70, 23, 71, 72, 73, 74, 75, 76, 77, 78, - /* 60 */ 79, 80, 81, 82, 83, 122, 85, 86, 87, 88, - /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 91, - /* 80 */ 92, 93, 94, 95, 26, 85, 86, 87, 88, 89, - /* 90 */ 90, 91, 92, 93, 94, 95, 27, 28, 97, 98, - /* 100 */ 99, 122, 211, 102, 103, 104, 79, 19, 50, 51, - /* 110 */ 19, 122, 59, 55, 113, 224, 225, 226, 89, 90, - /* 120 */ 91, 92, 93, 94, 95, 23, 27, 28, 26, 71, - /* 130 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 140 */ 82, 83, 51, 85, 86, 87, 88, 89, 90, 91, - /* 150 */ 92, 93, 94, 95, 19, 132, 133, 58, 89, 90, - /* 160 */ 21, 108, 109, 110, 27, 28, 97, 98, 33, 100, - /* 170 */ 7, 8, 119, 120, 22, 19, 107, 42, 109, 27, - /* 180 */ 28, 27, 28, 95, 28, 50, 51, 99, 100, 101, - /* 190 */ 102, 103, 104, 105, 27, 28, 97, 98, 107, 152, - /* 200 */ 112, 132, 133, 112, 65, 69, 71, 72, 73, 74, - /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 11, - /* 220 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 230 */ 95, 19, 101, 97, 97, 98, 24, 101, 122, 157, - /* 240 */ 12, 99, 103, 112, 102, 103, 104, 152, 22, 97, - /* 250 */ 98, 97, 98, 27, 28, 113, 27, 29, 91, 164, - /* 260 */ 165, 124, 50, 51, 97, 98, 219, 59, 132, 133, - /* 270 */ 134, 22, 23, 45, 66, 47, 212, 213, 124, 140, - /* 280 */ 132, 133, 19, 71, 72, 73, 74, 75, 76, 77, - /* 290 */ 78, 79, 80, 81, 82, 83, 152, 85, 86, 87, - /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 164, 165, - /* 310 */ 27, 28, 230, 50, 51, 233, 108, 109, 110, 70, - /* 320 */ 16, 59, 23, 97, 98, 26, 97, 22, 66, 185, - /* 330 */ 12, 187, 27, 28, 71, 72, 73, 74, 75, 76, - /* 340 */ 77, 78, 79, 80, 81, 82, 83, 29, 85, 86, - /* 350 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 19, - /* 360 */ 22, 148, 149, 45, 23, 47, 62, 154, 64, 156, - /* 370 */ 108, 109, 110, 37, 69, 23, 163, 59, 26, 26, - /* 380 */ 97, 98, 144, 145, 146, 147, 152, 200, 52, 23, - /* 390 */ 50, 51, 26, 22, 89, 90, 60, 210, 7, 8, - /* 400 */ 9, 138, 97, 22, 23, 26, 101, 26, 174, 175, - /* 410 */ 197, 71, 72, 73, 74, 75, 76, 77, 78, 79, - /* 420 */ 80, 81, 82, 83, 16, 85, 86, 87, 88, 89, - /* 430 */ 90, 91, 92, 93, 94, 95, 19, 132, 133, 134, - /* 440 */ 23, 152, 208, 209, 140, 152, 152, 111, 195, 196, - /* 450 */ 98, 70, 163, 160, 152, 23, 22, 164, 165, 246, - /* 460 */ 207, 27, 152, 174, 175, 171, 172, 50, 51, 137, - /* 470 */ 62, 139, 64, 171, 172, 222, 124, 27, 138, 24, - /* 480 */ 163, 89, 90, 130, 174, 175, 197, 163, 71, 72, - /* 490 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - /* 500 */ 83, 22, 85, 86, 87, 88, 89, 90, 91, 92, - /* 510 */ 93, 94, 95, 19, 197, 181, 182, 23, 208, 209, - /* 520 */ 152, 197, 26, 189, 132, 133, 232, 224, 225, 226, - /* 530 */ 152, 97, 91, 26, 232, 116, 212, 213, 152, 222, - /* 540 */ 121, 152, 174, 175, 50, 51, 243, 97, 22, 23, - /* 550 */ 22, 234, 174, 175, 177, 23, 239, 116, 163, 177, - /* 560 */ 174, 175, 121, 174, 175, 71, 72, 73, 74, 75, - /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 24, 85, - /* 580 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 590 */ 19, 23, 197, 11, 23, 227, 70, 208, 220, 152, - /* 600 */ 31, 224, 225, 226, 35, 98, 224, 225, 226, 108, - /* 610 */ 109, 110, 115, 152, 117, 118, 27, 222, 49, 123, - /* 620 */ 24, 50, 51, 27, 0, 1, 2, 224, 225, 226, - /* 630 */ 166, 124, 168, 169, 239, 174, 175, 170, 171, 172, - /* 640 */ 22, 194, 71, 72, 73, 74, 75, 76, 77, 78, - /* 650 */ 79, 80, 81, 82, 83, 152, 85, 86, 87, 88, - /* 660 */ 89, 90, 91, 92, 93, 94, 95, 19, 22, 208, - /* 670 */ 24, 23, 195, 196, 170, 171, 172, 174, 175, 152, - /* 680 */ 26, 152, 152, 152, 207, 152, 97, 152, 23, 152, - /* 690 */ 51, 244, 152, 97, 152, 247, 248, 23, 50, 51, - /* 700 */ 26, 174, 175, 174, 175, 174, 175, 174, 175, 174, - /* 710 */ 175, 174, 175, 23, 174, 175, 174, 175, 188, 71, - /* 720 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 730 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, - /* 740 */ 92, 93, 94, 95, 19, 152, 107, 152, 33, 24, - /* 750 */ 152, 100, 101, 27, 174, 175, 152, 42, 152, 23, - /* 760 */ 152, 26, 152, 23, 152, 26, 152, 174, 175, 174, - /* 770 */ 175, 152, 174, 175, 23, 50, 51, 26, 174, 175, - /* 780 */ 174, 175, 174, 175, 174, 175, 174, 175, 174, 175, - /* 790 */ 163, 119, 120, 174, 175, 19, 71, 72, 73, 74, - /* 800 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 152, - /* 810 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 820 */ 95, 66, 152, 97, 197, 23, 50, 51, 26, 53, - /* 830 */ 23, 174, 175, 26, 23, 23, 23, 26, 26, 26, - /* 840 */ 36, 106, 146, 147, 174, 175, 19, 71, 72, 73, - /* 850 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, - /* 860 */ 152, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 870 */ 94, 95, 152, 196, 119, 120, 19, 50, 51, 168, - /* 880 */ 169, 26, 174, 175, 207, 28, 152, 249, 250, 152, - /* 890 */ 163, 163, 163, 163, 174, 175, 163, 19, 71, 72, - /* 900 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - /* 910 */ 83, 152, 85, 86, 87, 88, 89, 90, 91, 92, - /* 920 */ 93, 94, 95, 152, 197, 197, 197, 197, 50, 51, - /* 930 */ 197, 194, 36, 174, 175, 191, 192, 152, 191, 192, - /* 940 */ 163, 152, 66, 124, 152, 174, 175, 152, 19, 71, - /* 950 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 960 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, - /* 970 */ 92, 93, 94, 95, 197, 152, 100, 188, 152, 50, - /* 980 */ 51, 152, 152, 188, 174, 175, 252, 152, 94, 95, - /* 990 */ 152, 152, 152, 1, 2, 152, 152, 174, 175, 19, - /* 1000 */ 152, 72, 73, 74, 75, 76, 77, 78, 79, 80, - /* 1010 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, - /* 1020 */ 91, 92, 93, 94, 95, 152, 188, 188, 22, 194, - /* 1030 */ 50, 51, 240, 173, 194, 174, 175, 252, 194, 152, - /* 1040 */ 36, 181, 28, 152, 23, 219, 122, 174, 175, 219, - /* 1050 */ 221, 152, 152, 73, 74, 75, 76, 77, 78, 79, - /* 1060 */ 80, 81, 82, 83, 152, 85, 86, 87, 88, 89, - /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 152, 22, - /* 1080 */ 23, 194, 152, 240, 27, 28, 174, 175, 240, 19, - /* 1090 */ 20, 26, 22, 194, 194, 38, 22, 27, 28, 152, - /* 1100 */ 23, 22, 152, 116, 174, 175, 152, 23, 38, 152, - /* 1110 */ 23, 152, 221, 152, 57, 152, 23, 163, 50, 51, - /* 1120 */ 194, 174, 175, 66, 174, 175, 69, 57, 174, 175, - /* 1130 */ 40, 174, 175, 174, 175, 174, 175, 174, 175, 69, - /* 1140 */ 22, 53, 74, 75, 30, 53, 89, 90, 22, 22, - /* 1150 */ 152, 197, 23, 96, 97, 98, 22, 152, 101, 89, - /* 1160 */ 90, 91, 208, 209, 152, 53, 96, 97, 98, 101, - /* 1170 */ 22, 101, 174, 175, 152, 19, 20, 105, 22, 174, - /* 1180 */ 175, 112, 19, 27, 28, 20, 174, 175, 24, 132, - /* 1190 */ 133, 134, 135, 136, 38, 44, 174, 175, 107, 61, - /* 1200 */ 54, 26, 132, 133, 134, 135, 136, 54, 107, 22, - /* 1210 */ 5, 140, 1, 57, 36, 111, 122, 28, 79, 79, - /* 1220 */ 131, 123, 66, 19, 20, 69, 22, 1, 16, 20, - /* 1230 */ 125, 27, 28, 123, 111, 120, 23, 131, 23, 16, - /* 1240 */ 68, 142, 38, 15, 22, 89, 90, 3, 167, 4, - /* 1250 */ 248, 251, 96, 97, 98, 180, 180, 101, 251, 151, - /* 1260 */ 6, 57, 151, 13, 151, 26, 25, 151, 161, 202, - /* 1270 */ 153, 162, 153, 69, 130, 128, 203, 19, 20, 127, - /* 1280 */ 22, 126, 204, 129, 22, 27, 28, 205, 132, 133, - /* 1290 */ 134, 135, 136, 89, 90, 231, 38, 95, 137, 179, - /* 1300 */ 96, 97, 98, 206, 179, 101, 122, 107, 159, 159, - /* 1310 */ 125, 231, 216, 228, 107, 57, 184, 217, 216, 176, - /* 1320 */ 217, 176, 48, 106, 18, 184, 158, 69, 159, 158, - /* 1330 */ 46, 71, 237, 176, 176, 176, 132, 133, 134, 135, - /* 1340 */ 136, 217, 176, 137, 216, 178, 158, 89, 90, 179, - /* 1350 */ 176, 159, 179, 159, 96, 97, 98, 159, 159, 101, - /* 1360 */ 5, 158, 202, 22, 18, 10, 11, 12, 13, 14, - /* 1370 */ 190, 238, 17, 190, 158, 193, 41, 159, 202, 193, - /* 1380 */ 159, 202, 245, 193, 193, 223, 190, 32, 159, 34, - /* 1390 */ 132, 133, 134, 135, 136, 159, 39, 155, 43, 150, - /* 1400 */ 223, 177, 201, 178, 177, 186, 66, 199, 177, 152, - /* 1410 */ 253, 56, 215, 152, 182, 152, 202, 152, 63, 152, - /* 1420 */ 152, 66, 67, 242, 229, 152, 174, 152, 152, 152, - /* 1430 */ 152, 152, 152, 152, 199, 242, 202, 152, 198, 152, - /* 1440 */ 152, 152, 183, 192, 152, 215, 152, 183, 215, 183, - /* 1450 */ 152, 241, 214, 152, 211, 152, 152, 211, 211, 152, - /* 1460 */ 152, 241, 152, 152, 152, 152, 152, 152, 152, 114, - /* 1470 */ 152, 152, 235, 152, 152, 152, 174, 187, 95, 174, - /* 1480 */ 253, 253, 253, 253, 236, 253, 253, 253, 253, 253, - /* 1490 */ 253, 253, 253, 253, 253, 253, 141, + /* 0 */ 19, 95, 53, 97, 22, 24, 24, 101, 27, 28, + /* 10 */ 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + /* 20 */ 39, 40, 41, 152, 43, 44, 45, 46, 47, 48, + /* 30 */ 49, 50, 51, 52, 53, 19, 55, 55, 132, 133, + /* 40 */ 134, 1, 2, 27, 28, 29, 30, 31, 32, 33, + /* 50 */ 34, 35, 36, 37, 38, 39, 40, 41, 187, 43, + /* 60 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 70 */ 47, 48, 49, 50, 51, 52, 53, 61, 97, 97, + /* 80 */ 19, 49, 50, 51, 52, 53, 70, 26, 27, 28, + /* 90 */ 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + /* 100 */ 39, 40, 41, 152, 43, 44, 45, 46, 47, 48, + /* 110 */ 49, 50, 51, 52, 53, 144, 145, 146, 147, 19, + /* 120 */ 16, 22, 92, 172, 173, 52, 53, 27, 28, 29, + /* 130 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + /* 140 */ 40, 41, 81, 43, 44, 45, 46, 47, 48, 49, + /* 150 */ 50, 51, 52, 53, 55, 56, 19, 152, 207, 208, + /* 160 */ 115, 24, 117, 118, 27, 28, 29, 30, 31, 32, + /* 170 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 79, + /* 180 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 190 */ 53, 19, 88, 157, 90, 23, 97, 98, 193, 27, + /* 200 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, + /* 210 */ 38, 39, 40, 41, 152, 43, 44, 45, 46, 47, + /* 220 */ 48, 49, 50, 51, 52, 53, 19, 22, 23, 172, + /* 230 */ 23, 26, 119, 120, 27, 28, 29, 30, 31, 32, + /* 240 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 187, + /* 250 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 260 */ 53, 19, 22, 23, 228, 23, 26, 231, 152, 27, + /* 270 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, + /* 280 */ 38, 39, 40, 41, 172, 43, 44, 45, 46, 47, + /* 290 */ 48, 49, 50, 51, 52, 53, 19, 221, 222, 223, + /* 300 */ 23, 96, 152, 172, 27, 28, 29, 30, 31, 32, + /* 310 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 152, + /* 320 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 330 */ 53, 19, 0, 1, 2, 23, 96, 190, 191, 27, + /* 340 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, + /* 350 */ 38, 39, 40, 41, 238, 43, 44, 45, 46, 47, + /* 360 */ 48, 49, 50, 51, 52, 53, 19, 185, 218, 221, + /* 370 */ 222, 223, 152, 152, 27, 28, 29, 30, 31, 32, + /* 380 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 241, + /* 390 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 400 */ 53, 19, 152, 168, 169, 170, 22, 190, 191, 27, + /* 410 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, + /* 420 */ 38, 39, 40, 41, 152, 43, 44, 45, 46, 47, + /* 430 */ 48, 49, 50, 51, 52, 53, 19, 19, 218, 55, + /* 440 */ 56, 24, 22, 152, 27, 28, 29, 30, 31, 32, + /* 450 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 152, + /* 460 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 470 */ 53, 250, 194, 195, 56, 55, 56, 55, 19, 172, + /* 480 */ 173, 97, 98, 152, 206, 138, 27, 28, 29, 30, + /* 490 */ 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + /* 500 */ 41, 152, 43, 44, 45, 46, 47, 48, 49, 50, + /* 510 */ 51, 52, 53, 19, 207, 208, 152, 97, 98, 97, + /* 520 */ 138, 27, 28, 29, 30, 31, 32, 33, 34, 35, + /* 530 */ 36, 37, 38, 39, 40, 41, 181, 43, 44, 45, + /* 540 */ 46, 47, 48, 49, 50, 51, 52, 53, 19, 30, + /* 550 */ 31, 32, 33, 247, 248, 19, 152, 28, 29, 30, + /* 560 */ 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + /* 570 */ 41, 152, 43, 44, 45, 46, 47, 48, 49, 50, + /* 580 */ 51, 52, 53, 19, 168, 169, 170, 238, 19, 53, + /* 590 */ 152, 172, 173, 29, 30, 31, 32, 33, 34, 35, + /* 600 */ 36, 37, 38, 39, 40, 41, 152, 43, 44, 45, + /* 610 */ 46, 47, 48, 49, 50, 51, 52, 53, 19, 20, + /* 620 */ 101, 22, 23, 169, 170, 56, 207, 85, 55, 56, + /* 630 */ 23, 19, 20, 26, 22, 99, 100, 101, 102, 103, + /* 640 */ 104, 105, 238, 152, 152, 210, 47, 48, 112, 152, + /* 650 */ 108, 109, 110, 54, 55, 56, 221, 222, 223, 47, + /* 660 */ 48, 119, 120, 172, 173, 66, 54, 55, 56, 152, + /* 670 */ 97, 98, 99, 148, 149, 102, 103, 104, 66, 154, + /* 680 */ 23, 156, 83, 26, 230, 152, 113, 152, 163, 194, + /* 690 */ 195, 92, 92, 30, 95, 83, 97, 98, 207, 208, + /* 700 */ 101, 206, 179, 180, 92, 172, 173, 95, 152, 97, + /* 710 */ 98, 188, 99, 101, 219, 102, 103, 104, 152, 119, + /* 720 */ 120, 196, 55, 56, 19, 20, 113, 22, 193, 163, + /* 730 */ 11, 132, 133, 134, 135, 136, 24, 65, 172, 173, + /* 740 */ 207, 208, 250, 152, 132, 133, 134, 135, 136, 193, + /* 750 */ 78, 84, 47, 48, 49, 98, 199, 152, 86, 54, + /* 760 */ 55, 56, 196, 152, 97, 98, 209, 55, 163, 244, + /* 770 */ 107, 66, 152, 207, 208, 164, 175, 172, 173, 19, + /* 780 */ 20, 124, 22, 111, 38, 39, 40, 41, 83, 43, + /* 790 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 800 */ 95, 196, 97, 98, 85, 152, 101, 47, 48, 97, + /* 810 */ 85, 92, 207, 193, 54, 55, 56, 92, 49, 175, + /* 820 */ 55, 56, 221, 222, 223, 12, 66, 108, 109, 110, + /* 830 */ 137, 163, 139, 108, 109, 110, 26, 132, 133, 134, + /* 840 */ 135, 136, 152, 83, 43, 44, 45, 46, 47, 48, + /* 850 */ 49, 50, 51, 52, 53, 95, 26, 97, 98, 55, + /* 860 */ 56, 101, 97, 98, 196, 221, 222, 223, 146, 147, + /* 870 */ 57, 171, 152, 22, 26, 19, 20, 49, 22, 179, + /* 880 */ 108, 109, 110, 55, 56, 116, 73, 219, 75, 124, + /* 890 */ 121, 152, 132, 133, 134, 135, 136, 163, 85, 152, + /* 900 */ 232, 97, 98, 47, 48, 237, 55, 56, 98, 5, + /* 910 */ 54, 55, 56, 193, 10, 11, 12, 13, 14, 172, + /* 920 */ 173, 17, 66, 47, 48, 97, 98, 152, 124, 152, + /* 930 */ 196, 55, 56, 186, 124, 152, 106, 160, 152, 83, + /* 940 */ 152, 164, 152, 61, 22, 211, 212, 152, 97, 98, + /* 950 */ 152, 95, 70, 97, 98, 172, 173, 101, 172, 173, + /* 960 */ 172, 173, 172, 173, 60, 181, 62, 172, 173, 47, + /* 970 */ 48, 123, 186, 97, 98, 71, 100, 55, 56, 152, + /* 980 */ 181, 186, 21, 107, 152, 109, 82, 163, 132, 133, + /* 990 */ 134, 135, 136, 89, 16, 207, 92, 93, 19, 172, + /* 1000 */ 173, 169, 170, 195, 55, 56, 12, 152, 132, 30, + /* 1010 */ 134, 47, 48, 186, 206, 225, 152, 95, 114, 97, + /* 1020 */ 196, 245, 246, 101, 152, 38, 39, 40, 41, 42, + /* 1030 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 1040 */ 53, 152, 163, 219, 152, 141, 97, 98, 193, 152, + /* 1050 */ 152, 57, 91, 164, 132, 133, 134, 152, 55, 152, + /* 1060 */ 152, 237, 230, 152, 103, 193, 88, 73, 90, 75, + /* 1070 */ 172, 173, 183, 152, 185, 196, 152, 172, 173, 172, + /* 1080 */ 173, 217, 152, 172, 173, 152, 107, 22, 152, 24, + /* 1090 */ 193, 112, 152, 172, 173, 152, 132, 242, 134, 152, + /* 1100 */ 97, 140, 152, 92, 152, 172, 173, 152, 172, 173, + /* 1110 */ 152, 100, 172, 173, 152, 172, 173, 152, 140, 172, + /* 1120 */ 173, 152, 172, 173, 172, 173, 152, 172, 173, 152, + /* 1130 */ 172, 173, 152, 152, 172, 173, 152, 172, 173, 213, + /* 1140 */ 152, 172, 173, 152, 152, 152, 172, 173, 152, 172, + /* 1150 */ 173, 152, 172, 173, 152, 210, 172, 173, 152, 26, + /* 1160 */ 172, 173, 152, 172, 173, 172, 173, 152, 172, 173, + /* 1170 */ 152, 172, 173, 152, 172, 173, 152, 59, 172, 173, + /* 1180 */ 152, 63, 172, 173, 152, 193, 152, 152, 152, 152, + /* 1190 */ 172, 173, 152, 172, 173, 77, 172, 173, 152, 152, + /* 1200 */ 172, 173, 152, 152, 172, 173, 172, 173, 172, 173, + /* 1210 */ 152, 22, 172, 173, 152, 152, 152, 22, 172, 173, + /* 1220 */ 152, 152, 152, 172, 173, 152, 7, 8, 9, 163, + /* 1230 */ 172, 173, 22, 23, 172, 173, 172, 173, 166, 167, + /* 1240 */ 172, 173, 172, 173, 55, 172, 173, 22, 23, 108, + /* 1250 */ 109, 110, 217, 152, 217, 166, 167, 163, 163, 163, + /* 1260 */ 163, 163, 196, 130, 217, 211, 212, 217, 116, 23, + /* 1270 */ 22, 101, 26, 121, 23, 23, 23, 26, 26, 26, + /* 1280 */ 23, 23, 112, 26, 26, 37, 97, 100, 101, 55, + /* 1290 */ 196, 196, 196, 196, 196, 23, 23, 55, 26, 26, + /* 1300 */ 7, 8, 23, 152, 23, 26, 96, 26, 132, 132, + /* 1310 */ 134, 134, 23, 152, 152, 26, 152, 122, 152, 191, + /* 1320 */ 152, 96, 234, 152, 152, 152, 152, 152, 197, 210, + /* 1330 */ 152, 97, 152, 152, 210, 233, 210, 198, 150, 97, + /* 1340 */ 184, 201, 239, 214, 214, 201, 239, 180, 214, 227, + /* 1350 */ 200, 198, 155, 67, 243, 176, 69, 175, 175, 175, + /* 1360 */ 122, 159, 159, 240, 159, 240, 22, 220, 27, 130, + /* 1370 */ 201, 18, 159, 18, 189, 158, 158, 220, 192, 159, + /* 1380 */ 137, 236, 192, 192, 192, 189, 74, 189, 159, 235, + /* 1390 */ 159, 158, 22, 177, 201, 201, 159, 107, 158, 177, + /* 1400 */ 159, 174, 158, 76, 174, 182, 174, 106, 182, 125, + /* 1410 */ 174, 107, 177, 22, 159, 216, 215, 137, 159, 53, + /* 1420 */ 216, 176, 215, 174, 174, 216, 215, 215, 174, 229, + /* 1430 */ 216, 129, 224, 177, 126, 229, 127, 177, 128, 25, + /* 1440 */ 162, 226, 26, 161, 13, 153, 6, 153, 151, 151, + /* 1450 */ 151, 151, 205, 165, 178, 178, 165, 4, 3, 22, + /* 1460 */ 165, 142, 15, 94, 202, 204, 203, 201, 16, 23, + /* 1470 */ 249, 23, 120, 249, 246, 111, 131, 123, 20, 16, + /* 1480 */ 1, 125, 123, 111, 56, 64, 37, 37, 131, 122, + /* 1490 */ 1, 37, 5, 37, 22, 107, 26, 80, 140, 80, + /* 1500 */ 87, 72, 107, 20, 24, 19, 112, 105, 23, 79, + /* 1510 */ 22, 79, 22, 22, 22, 58, 22, 79, 23, 68, + /* 1520 */ 23, 23, 26, 116, 22, 26, 23, 22, 122, 23, + /* 1530 */ 23, 56, 64, 22, 124, 26, 26, 64, 64, 23, + /* 1540 */ 23, 23, 23, 11, 23, 22, 26, 23, 22, 24, + /* 1550 */ 1, 23, 22, 26, 251, 24, 23, 22, 122, 23, + /* 1560 */ 23, 22, 15, 122, 122, 122, 23, }; -#define YY_SHIFT_USE_DFLT (-86) -#define YY_SHIFT_COUNT (429) -#define YY_SHIFT_MIN (-85) -#define YY_SHIFT_MAX (1383) +#define YY_SHIFT_USE_DFLT (1567) +#define YY_SHIFT_COUNT (455) +#define YY_SHIFT_MIN (-94) +#define YY_SHIFT_MAX (1549) static const short yy_shift_ofst[] = { - /* 0 */ 992, 1057, 1355, 1156, 1204, 1204, 1, 262, -19, 135, - /* 10 */ 135, 776, 1204, 1204, 1204, 1204, 69, 69, 53, 208, - /* 20 */ 283, 755, 58, 725, 648, 571, 494, 417, 340, 263, - /* 30 */ 212, 827, 827, 827, 827, 827, 827, 827, 827, 827, - /* 40 */ 827, 827, 827, 827, 827, 827, 878, 827, 929, 980, - /* 50 */ 980, 1070, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 60 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 70 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 80 */ 1258, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 90 */ 1204, 1204, 1204, 1204, -71, -47, -47, -47, -47, -47, - /* 100 */ 0, 29, -12, 283, 283, 139, 91, 392, 392, 894, - /* 110 */ 672, 726, 1383, -86, -86, -86, 88, 318, 318, 99, - /* 120 */ 381, -20, 283, 283, 283, 283, 283, 283, 283, 283, - /* 130 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 140 */ 283, 283, 283, 283, 624, 876, 726, 672, 1340, 1340, - /* 150 */ 1340, 1340, 1340, 1340, -86, -86, -86, 305, 136, 136, - /* 160 */ 142, 167, 226, 154, 137, 152, 283, 283, 283, 283, - /* 170 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 180 */ 283, 283, 283, 336, 336, 336, 283, 283, 352, 283, - /* 190 */ 283, 283, 283, 283, 228, 283, 283, 283, 283, 283, - /* 200 */ 283, 283, 283, 283, 283, 501, 569, 596, 596, 596, - /* 210 */ 507, 497, 441, 391, 353, 156, 156, 857, 353, 857, - /* 220 */ 735, 813, 639, 715, 156, 332, 715, 715, 496, 419, - /* 230 */ 646, 1357, 1184, 1184, 1335, 1335, 1184, 1341, 1260, 1144, - /* 240 */ 1346, 1346, 1346, 1346, 1184, 1306, 1144, 1341, 1260, 1260, - /* 250 */ 1144, 1184, 1306, 1206, 1284, 1184, 1184, 1306, 1184, 1306, - /* 260 */ 1184, 1306, 1262, 1207, 1207, 1207, 1274, 1262, 1207, 1217, - /* 270 */ 1207, 1274, 1207, 1207, 1185, 1200, 1185, 1200, 1185, 1200, - /* 280 */ 1184, 1184, 1161, 1262, 1202, 1202, 1262, 1154, 1155, 1147, - /* 290 */ 1152, 1144, 1241, 1239, 1250, 1250, 1254, 1254, 1254, 1254, - /* 300 */ -86, -86, -86, -86, -86, -86, 1068, 304, 526, 249, - /* 310 */ 408, -83, 434, 812, 27, 811, 807, 802, 751, 589, - /* 320 */ 651, 163, 131, 674, 366, 450, 299, 148, 23, 102, - /* 330 */ 229, -21, 1245, 1244, 1222, 1099, 1228, 1172, 1223, 1215, - /* 340 */ 1213, 1115, 1106, 1123, 1110, 1209, 1105, 1212, 1226, 1098, - /* 350 */ 1089, 1140, 1139, 1104, 1189, 1178, 1094, 1211, 1205, 1187, - /* 360 */ 1101, 1071, 1153, 1175, 1146, 1138, 1151, 1091, 1164, 1165, - /* 370 */ 1163, 1069, 1072, 1148, 1112, 1134, 1127, 1129, 1126, 1092, - /* 380 */ 1114, 1118, 1088, 1090, 1093, 1087, 1084, 987, 1079, 1077, - /* 390 */ 1074, 1065, 924, 1021, 1014, 1004, 1006, 819, 739, 896, - /* 400 */ 855, 804, 739, 740, 736, 690, 654, 665, 618, 582, - /* 410 */ 568, 528, 554, 379, 532, 479, 455, 379, 432, 371, - /* 420 */ 341, 28, 338, 116, -11, -57, -85, 7, -8, 3, + /* 0 */ 40, 599, 904, 612, 760, 760, 760, 760, 725, -19, + /* 10 */ 16, 16, 100, 760, 760, 760, 760, 760, 760, 760, + /* 20 */ 876, 876, 573, 542, 719, 600, 61, 137, 172, 207, + /* 30 */ 242, 277, 312, 347, 382, 417, 459, 459, 459, 459, + /* 40 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 50 */ 459, 459, 459, 494, 459, 529, 564, 564, 705, 760, + /* 60 */ 760, 760, 760, 760, 760, 760, 760, 760, 760, 760, + /* 70 */ 760, 760, 760, 760, 760, 760, 760, 760, 760, 760, + /* 80 */ 760, 760, 760, 760, 760, 760, 760, 760, 760, 760, + /* 90 */ 856, 760, 760, 760, 760, 760, 760, 760, 760, 760, + /* 100 */ 760, 760, 760, 760, 987, 746, 746, 746, 746, 746, + /* 110 */ 801, 23, 32, 949, 961, 979, 964, 964, 949, 73, + /* 120 */ 113, -51, 1567, 1567, 1567, 536, 536, 536, 99, 99, + /* 130 */ 813, 813, 667, 205, 240, 949, 949, 949, 949, 949, + /* 140 */ 949, 949, 949, 949, 949, 949, 949, 949, 949, 949, + /* 150 */ 949, 949, 949, 949, 949, 332, 1011, 422, 422, 113, + /* 160 */ 30, 30, 30, 30, 30, 30, 1567, 1567, 1567, 922, + /* 170 */ -94, -94, 384, 613, 828, 420, 765, 804, 851, 949, + /* 180 */ 949, 949, 949, 949, 949, 949, 949, 949, 949, 949, + /* 190 */ 949, 949, 949, 949, 949, 672, 672, 672, 949, 949, + /* 200 */ 657, 949, 949, 949, -18, 949, 949, 994, 949, 949, + /* 210 */ 949, 949, 949, 949, 949, 949, 949, 949, 772, 1118, + /* 220 */ 712, 712, 712, 810, 45, 769, 1219, 1133, 418, 418, + /* 230 */ 569, 1133, 569, 830, 607, 663, 882, 418, 693, 882, + /* 240 */ 882, 848, 1152, 1065, 1286, 1238, 1238, 1287, 1287, 1238, + /* 250 */ 1344, 1341, 1239, 1353, 1353, 1353, 1353, 1238, 1355, 1239, + /* 260 */ 1344, 1341, 1341, 1239, 1238, 1355, 1243, 1312, 1238, 1238, + /* 270 */ 1355, 1370, 1238, 1355, 1238, 1355, 1370, 1290, 1290, 1290, + /* 280 */ 1327, 1370, 1290, 1301, 1290, 1327, 1290, 1290, 1284, 1304, + /* 290 */ 1284, 1304, 1284, 1304, 1284, 1304, 1238, 1391, 1238, 1280, + /* 300 */ 1370, 1366, 1366, 1370, 1302, 1308, 1310, 1309, 1239, 1414, + /* 310 */ 1416, 1431, 1431, 1440, 1440, 1440, 1440, 1567, 1567, 1567, + /* 320 */ 1567, 1567, 1567, 1567, 1567, 519, 978, 1210, 1225, 104, + /* 330 */ 1141, 1189, 1246, 1248, 1251, 1252, 1253, 1257, 1258, 1273, + /* 340 */ 1003, 1187, 1293, 1170, 1272, 1279, 1234, 1281, 1176, 1177, + /* 350 */ 1289, 1242, 1195, 1453, 1455, 1437, 1319, 1447, 1369, 1452, + /* 360 */ 1446, 1448, 1352, 1345, 1364, 1354, 1458, 1356, 1463, 1479, + /* 370 */ 1359, 1357, 1449, 1450, 1454, 1456, 1372, 1428, 1421, 1367, + /* 380 */ 1489, 1487, 1472, 1388, 1358, 1417, 1470, 1419, 1413, 1429, + /* 390 */ 1395, 1480, 1483, 1486, 1394, 1402, 1488, 1430, 1490, 1491, + /* 400 */ 1485, 1492, 1432, 1457, 1494, 1438, 1451, 1495, 1497, 1498, + /* 410 */ 1496, 1407, 1502, 1503, 1505, 1499, 1406, 1506, 1507, 1475, + /* 420 */ 1468, 1511, 1410, 1509, 1473, 1510, 1474, 1516, 1509, 1517, + /* 430 */ 1518, 1519, 1520, 1521, 1523, 1532, 1524, 1526, 1525, 1527, + /* 440 */ 1528, 1530, 1531, 1527, 1533, 1535, 1536, 1537, 1539, 1436, + /* 450 */ 1441, 1442, 1443, 1543, 1547, 1549, }; -#define YY_REDUCE_USE_DFLT (-110) -#define YY_REDUCE_COUNT (305) -#define YY_REDUCE_MIN (-109) -#define YY_REDUCE_MAX (1323) +#define YY_REDUCE_USE_DFLT (-130) +#define YY_REDUCE_COUNT (324) +#define YY_REDUCE_MIN (-129) +#define YY_REDUCE_MAX (1300) static const short yy_reduce_ofst[] = { - /* 0 */ 238, 954, 213, 289, 310, 234, 144, 317, -109, 382, - /* 10 */ 377, 303, 461, 389, 378, 368, 302, 294, 253, 395, - /* 20 */ 293, 324, 403, 403, 403, 403, 403, 403, 403, 403, - /* 30 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, - /* 40 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, - /* 50 */ 403, 1022, 1012, 1005, 998, 963, 961, 959, 957, 950, - /* 60 */ 947, 930, 912, 873, 861, 823, 810, 771, 759, 720, - /* 70 */ 708, 670, 657, 619, 614, 612, 610, 608, 606, 604, - /* 80 */ 598, 595, 593, 580, 542, 540, 537, 535, 533, 531, - /* 90 */ 529, 527, 503, 386, 403, 403, 403, 403, 403, 403, - /* 100 */ 403, 403, 403, 95, 447, 82, 334, 504, 467, 403, - /* 110 */ 477, 464, 403, 403, 403, 403, 860, 747, 744, 785, - /* 120 */ 638, 638, 926, 891, 900, 899, 887, 844, 840, 835, - /* 130 */ 848, 830, 843, 829, 792, 839, 826, 737, 838, 795, - /* 140 */ 789, 47, 734, 530, 696, 777, 711, 677, 733, 730, - /* 150 */ 729, 728, 727, 627, 448, 64, 187, 1305, 1302, 1252, - /* 160 */ 1290, 1273, 1323, 1322, 1321, 1319, 1318, 1316, 1315, 1314, - /* 170 */ 1313, 1312, 1311, 1310, 1308, 1307, 1304, 1303, 1301, 1298, - /* 180 */ 1294, 1292, 1289, 1266, 1264, 1259, 1288, 1287, 1238, 1285, - /* 190 */ 1281, 1280, 1279, 1278, 1251, 1277, 1276, 1275, 1273, 1268, - /* 200 */ 1267, 1265, 1263, 1261, 1257, 1248, 1237, 1247, 1246, 1243, - /* 210 */ 1238, 1240, 1235, 1249, 1234, 1233, 1230, 1220, 1214, 1210, - /* 220 */ 1225, 1219, 1232, 1231, 1197, 1195, 1227, 1224, 1201, 1208, - /* 230 */ 1242, 1137, 1236, 1229, 1193, 1181, 1221, 1177, 1196, 1179, - /* 240 */ 1191, 1190, 1186, 1182, 1218, 1216, 1176, 1162, 1183, 1180, - /* 250 */ 1160, 1199, 1203, 1133, 1095, 1198, 1194, 1188, 1192, 1171, - /* 260 */ 1169, 1168, 1173, 1174, 1166, 1159, 1141, 1170, 1158, 1167, - /* 270 */ 1157, 1132, 1145, 1143, 1124, 1128, 1103, 1102, 1100, 1096, - /* 280 */ 1150, 1149, 1085, 1125, 1080, 1064, 1120, 1097, 1082, 1078, - /* 290 */ 1073, 1067, 1109, 1107, 1119, 1117, 1116, 1113, 1111, 1108, - /* 300 */ 1007, 1000, 1002, 1076, 1075, 1081, + /* 0 */ -29, 566, 525, 605, -49, 307, 491, 533, 668, 435, + /* 10 */ 601, 644, 148, 747, 786, 795, 419, 788, 827, 790, + /* 20 */ 454, 832, 889, 495, 824, 734, 76, 76, 76, 76, + /* 30 */ 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, + /* 40 */ 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, + /* 50 */ 76, 76, 76, 76, 76, 76, 76, 76, 783, 898, + /* 60 */ 905, 907, 911, 921, 933, 936, 940, 943, 947, 950, + /* 70 */ 952, 955, 958, 962, 965, 969, 974, 977, 980, 984, + /* 80 */ 988, 991, 993, 996, 999, 1002, 1006, 1010, 1018, 1021, + /* 90 */ 1024, 1028, 1032, 1034, 1036, 1040, 1046, 1051, 1058, 1062, + /* 100 */ 1064, 1068, 1070, 1073, 76, 76, 76, 76, 76, 76, + /* 110 */ 76, 76, 76, 855, 36, 523, 235, 416, 777, 76, + /* 120 */ 278, 76, 76, 76, 76, 700, 700, 700, 150, 220, + /* 130 */ 147, 217, 221, 306, 306, 611, 5, 535, 556, 620, + /* 140 */ 720, 872, 897, 116, 864, 349, 1035, 1037, 404, 1047, + /* 150 */ 992, -129, 1050, 492, 62, 722, 879, 1072, 1089, 808, + /* 160 */ 1066, 1094, 1095, 1096, 1097, 1098, 776, 1054, 557, 57, + /* 170 */ 112, 131, 167, 182, 250, 272, 291, 331, 364, 438, + /* 180 */ 497, 517, 591, 653, 690, 739, 775, 798, 892, 908, + /* 190 */ 924, 930, 1015, 1063, 1069, 355, 784, 799, 981, 1101, + /* 200 */ 926, 1151, 1161, 1162, 945, 1164, 1166, 1128, 1168, 1171, + /* 210 */ 1172, 250, 1173, 1174, 1175, 1178, 1180, 1181, 1088, 1102, + /* 220 */ 1119, 1124, 1126, 926, 1131, 1139, 1188, 1140, 1129, 1130, + /* 230 */ 1103, 1144, 1107, 1179, 1156, 1167, 1182, 1134, 1122, 1183, + /* 240 */ 1184, 1150, 1153, 1197, 1111, 1202, 1203, 1123, 1125, 1205, + /* 250 */ 1147, 1185, 1169, 1186, 1190, 1191, 1192, 1213, 1217, 1193, + /* 260 */ 1157, 1196, 1198, 1194, 1220, 1218, 1145, 1154, 1229, 1231, + /* 270 */ 1233, 1216, 1237, 1240, 1241, 1244, 1222, 1227, 1230, 1232, + /* 280 */ 1223, 1235, 1236, 1245, 1249, 1226, 1250, 1254, 1199, 1201, + /* 290 */ 1204, 1207, 1209, 1211, 1214, 1212, 1255, 1208, 1259, 1215, + /* 300 */ 1256, 1200, 1206, 1260, 1247, 1261, 1263, 1262, 1266, 1278, + /* 310 */ 1282, 1292, 1294, 1297, 1298, 1299, 1300, 1221, 1224, 1228, + /* 320 */ 1288, 1291, 1276, 1277, 1295, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 647, 964, 964, 964, 878, 878, 969, 964, 774, 802, - /* 10 */ 802, 938, 969, 969, 969, 876, 969, 969, 969, 964, - /* 20 */ 969, 778, 808, 969, 969, 969, 969, 969, 969, 969, - /* 30 */ 969, 937, 939, 816, 815, 918, 789, 813, 806, 810, - /* 40 */ 879, 872, 873, 871, 875, 880, 969, 809, 841, 856, - /* 50 */ 840, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 60 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 70 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 80 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 90 */ 969, 969, 969, 969, 850, 855, 862, 854, 851, 843, - /* 100 */ 842, 844, 845, 969, 969, 673, 739, 969, 969, 846, - /* 110 */ 969, 685, 847, 859, 858, 857, 680, 969, 969, 969, - /* 120 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 130 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 140 */ 969, 969, 969, 969, 647, 964, 969, 969, 964, 964, - /* 150 */ 964, 964, 964, 964, 956, 778, 768, 969, 969, 969, - /* 160 */ 969, 969, 969, 969, 969, 969, 969, 944, 942, 969, - /* 170 */ 891, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 180 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 190 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 200 */ 969, 969, 969, 969, 653, 969, 911, 774, 774, 774, - /* 210 */ 776, 754, 766, 655, 812, 791, 791, 923, 812, 923, - /* 220 */ 710, 733, 707, 802, 791, 874, 802, 802, 775, 766, - /* 230 */ 969, 949, 782, 782, 941, 941, 782, 821, 743, 812, - /* 240 */ 750, 750, 750, 750, 782, 670, 812, 821, 743, 743, - /* 250 */ 812, 782, 670, 917, 915, 782, 782, 670, 782, 670, - /* 260 */ 782, 670, 884, 741, 741, 741, 725, 884, 741, 710, - /* 270 */ 741, 725, 741, 741, 795, 790, 795, 790, 795, 790, - /* 280 */ 782, 782, 969, 884, 888, 888, 884, 807, 796, 805, - /* 290 */ 803, 812, 676, 728, 663, 663, 652, 652, 652, 652, - /* 300 */ 961, 961, 956, 712, 712, 695, 969, 969, 969, 969, - /* 310 */ 969, 969, 687, 969, 893, 969, 969, 969, 969, 969, - /* 320 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 330 */ 969, 828, 969, 648, 951, 969, 969, 948, 969, 969, - /* 340 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 350 */ 969, 969, 969, 969, 969, 969, 921, 969, 969, 969, - /* 360 */ 969, 969, 969, 914, 913, 969, 969, 969, 969, 969, - /* 370 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 380 */ 969, 969, 969, 969, 969, 969, 969, 757, 969, 969, - /* 390 */ 969, 761, 969, 969, 969, 969, 969, 969, 804, 969, - /* 400 */ 797, 969, 877, 969, 969, 969, 969, 969, 969, 969, - /* 410 */ 969, 969, 969, 966, 969, 969, 969, 965, 969, 969, - /* 420 */ 969, 969, 969, 830, 969, 829, 833, 969, 661, 969, - /* 430 */ 644, 649, 960, 963, 962, 959, 958, 957, 952, 950, - /* 440 */ 947, 946, 945, 943, 940, 936, 897, 895, 902, 901, - /* 450 */ 900, 899, 898, 896, 894, 892, 818, 817, 814, 811, - /* 460 */ 753, 935, 890, 752, 749, 748, 669, 953, 920, 929, - /* 470 */ 928, 927, 822, 926, 925, 924, 922, 919, 906, 820, - /* 480 */ 819, 744, 882, 881, 672, 910, 909, 908, 912, 916, - /* 490 */ 907, 784, 751, 671, 668, 675, 679, 731, 732, 740, - /* 500 */ 738, 737, 736, 735, 734, 730, 681, 686, 724, 709, - /* 510 */ 708, 717, 716, 722, 721, 720, 719, 718, 715, 714, - /* 520 */ 713, 706, 705, 711, 704, 727, 726, 723, 703, 747, - /* 530 */ 746, 745, 742, 702, 701, 700, 833, 699, 698, 838, - /* 540 */ 837, 866, 826, 755, 759, 758, 762, 763, 771, 770, - /* 550 */ 769, 780, 781, 793, 792, 824, 823, 794, 779, 773, - /* 560 */ 772, 788, 787, 786, 785, 777, 767, 799, 798, 868, - /* 570 */ 783, 867, 865, 934, 933, 932, 931, 930, 870, 967, - /* 580 */ 968, 887, 889, 886, 801, 800, 885, 869, 839, 836, - /* 590 */ 690, 691, 905, 904, 903, 693, 692, 689, 688, 863, - /* 600 */ 860, 852, 864, 861, 853, 849, 848, 834, 832, 831, - /* 610 */ 827, 835, 760, 756, 825, 765, 764, 697, 696, 694, - /* 620 */ 678, 677, 674, 667, 665, 664, 666, 662, 660, 659, - /* 630 */ 658, 657, 656, 684, 683, 682, 654, 651, 650, 646, - /* 640 */ 645, 643, + /* 0 */ 1281, 1271, 1271, 1271, 1203, 1203, 1203, 1203, 1271, 1096, + /* 10 */ 1125, 1125, 1255, 1332, 1332, 1332, 1332, 1332, 1332, 1202, + /* 20 */ 1332, 1332, 1332, 1332, 1271, 1100, 1131, 1332, 1332, 1332, + /* 30 */ 1332, 1204, 1205, 1332, 1332, 1332, 1254, 1256, 1141, 1140, + /* 40 */ 1139, 1138, 1237, 1112, 1136, 1129, 1133, 1204, 1198, 1199, + /* 50 */ 1197, 1201, 1205, 1332, 1132, 1167, 1182, 1166, 1332, 1332, + /* 60 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 70 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 80 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 90 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 100 */ 1332, 1332, 1332, 1332, 1176, 1181, 1188, 1180, 1177, 1169, + /* 110 */ 1168, 1170, 1171, 1332, 1019, 1067, 1332, 1332, 1332, 1172, + /* 120 */ 1332, 1173, 1185, 1184, 1183, 1262, 1289, 1288, 1332, 1332, + /* 130 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 140 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 150 */ 1332, 1332, 1332, 1332, 1332, 1281, 1271, 1025, 1025, 1332, + /* 160 */ 1271, 1271, 1271, 1271, 1271, 1271, 1267, 1100, 1091, 1332, + /* 170 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 180 */ 1259, 1257, 1332, 1218, 1332, 1332, 1332, 1332, 1332, 1332, + /* 190 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 200 */ 1332, 1332, 1332, 1332, 1096, 1332, 1332, 1332, 1332, 1332, + /* 210 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1283, 1332, 1232, + /* 220 */ 1096, 1096, 1096, 1098, 1080, 1090, 1004, 1135, 1114, 1114, + /* 230 */ 1321, 1135, 1321, 1042, 1303, 1039, 1125, 1114, 1200, 1125, + /* 240 */ 1125, 1097, 1090, 1332, 1324, 1105, 1105, 1323, 1323, 1105, + /* 250 */ 1146, 1070, 1135, 1076, 1076, 1076, 1076, 1105, 1016, 1135, + /* 260 */ 1146, 1070, 1070, 1135, 1105, 1016, 1236, 1318, 1105, 1105, + /* 270 */ 1016, 1211, 1105, 1016, 1105, 1016, 1211, 1068, 1068, 1068, + /* 280 */ 1057, 1211, 1068, 1042, 1068, 1057, 1068, 1068, 1118, 1113, + /* 290 */ 1118, 1113, 1118, 1113, 1118, 1113, 1105, 1206, 1105, 1332, + /* 300 */ 1211, 1215, 1215, 1211, 1130, 1119, 1128, 1126, 1135, 1022, + /* 310 */ 1060, 1286, 1286, 1282, 1282, 1282, 1282, 1329, 1329, 1267, + /* 320 */ 1298, 1298, 1044, 1044, 1298, 1332, 1332, 1332, 1332, 1332, + /* 330 */ 1332, 1293, 1332, 1220, 1332, 1332, 1332, 1332, 1332, 1332, + /* 340 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 350 */ 1332, 1332, 1152, 1332, 1000, 1264, 1332, 1332, 1263, 1332, + /* 360 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 370 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1320, + /* 380 */ 1332, 1332, 1332, 1332, 1332, 1332, 1235, 1234, 1332, 1332, + /* 390 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 400 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, + /* 410 */ 1332, 1082, 1332, 1332, 1332, 1307, 1332, 1332, 1332, 1332, + /* 420 */ 1332, 1332, 1332, 1127, 1332, 1120, 1332, 1332, 1311, 1332, + /* 430 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1273, + /* 440 */ 1332, 1332, 1332, 1272, 1332, 1332, 1332, 1332, 1332, 1154, + /* 450 */ 1332, 1153, 1157, 1332, 1010, 1332, }; +/********** End of lemon-generated parsing tables *****************************/ -/* The next table maps tokens into fallback tokens. If a construct -** like the following: +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: ** ** %fallback ID X Y Z. ** @@ -121137,79 +136781,109 @@ static const YYACTIONTYPE yy_default[] = { ** and Z. Whenever one of the tokens X, Y, or Z is input to the parser ** but it does not parse, the type of the token is changed to ID and ** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. */ #ifdef YYFALLBACK static const YYCODETYPE yyFallback[] = { 0, /* $ => nothing */ 0, /* SEMI => nothing */ - 27, /* EXPLAIN => ID */ - 27, /* QUERY => ID */ - 27, /* PLAN => ID */ - 27, /* BEGIN => ID */ + 55, /* EXPLAIN => ID */ + 55, /* QUERY => ID */ + 55, /* PLAN => ID */ + 55, /* BEGIN => ID */ 0, /* TRANSACTION => nothing */ - 27, /* DEFERRED => ID */ - 27, /* IMMEDIATE => ID */ - 27, /* EXCLUSIVE => ID */ + 55, /* DEFERRED => ID */ + 55, /* IMMEDIATE => ID */ + 55, /* EXCLUSIVE => ID */ 0, /* COMMIT => nothing */ - 27, /* END => ID */ - 27, /* ROLLBACK => ID */ - 27, /* SAVEPOINT => ID */ - 27, /* RELEASE => ID */ + 55, /* END => ID */ + 55, /* ROLLBACK => ID */ + 55, /* SAVEPOINT => ID */ + 55, /* RELEASE => ID */ 0, /* TO => nothing */ 0, /* TABLE => nothing */ 0, /* CREATE => nothing */ - 27, /* IF => ID */ + 55, /* IF => ID */ 0, /* NOT => nothing */ 0, /* EXISTS => nothing */ - 27, /* TEMP => ID */ + 55, /* TEMP => ID */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* AS => nothing */ - 27, /* WITHOUT => ID */ + 55, /* WITHOUT => ID */ 0, /* COMMA => nothing */ + 0, /* OR => nothing */ + 0, /* AND => nothing */ + 0, /* IS => nothing */ + 55, /* MATCH => ID */ + 55, /* LIKE_KW => ID */ + 0, /* BETWEEN => nothing */ + 0, /* IN => nothing */ + 0, /* ISNULL => nothing */ + 0, /* NOTNULL => nothing */ + 0, /* NE => nothing */ + 0, /* EQ => nothing */ + 0, /* GT => nothing */ + 0, /* LE => nothing */ + 0, /* LT => nothing */ + 0, /* GE => nothing */ + 0, /* ESCAPE => nothing */ + 0, /* BITAND => nothing */ + 0, /* BITOR => nothing */ + 0, /* LSHIFT => nothing */ + 0, /* RSHIFT => nothing */ + 0, /* PLUS => nothing */ + 0, /* MINUS => nothing */ + 0, /* STAR => nothing */ + 0, /* SLASH => nothing */ + 0, /* REM => nothing */ + 0, /* CONCAT => nothing */ + 0, /* COLLATE => nothing */ + 0, /* BITNOT => nothing */ 0, /* ID => nothing */ 0, /* INDEXED => nothing */ - 27, /* ABORT => ID */ - 27, /* ACTION => ID */ - 27, /* AFTER => ID */ - 27, /* ANALYZE => ID */ - 27, /* ASC => ID */ - 27, /* ATTACH => ID */ - 27, /* BEFORE => ID */ - 27, /* BY => ID */ - 27, /* CASCADE => ID */ - 27, /* CAST => ID */ - 27, /* COLUMNKW => ID */ - 27, /* CONFLICT => ID */ - 27, /* DATABASE => ID */ - 27, /* DESC => ID */ - 27, /* DETACH => ID */ - 27, /* EACH => ID */ - 27, /* FAIL => ID */ - 27, /* FOR => ID */ - 27, /* IGNORE => ID */ - 27, /* INITIALLY => ID */ - 27, /* INSTEAD => ID */ - 27, /* LIKE_KW => ID */ - 27, /* MATCH => ID */ - 27, /* NO => ID */ - 27, /* KEY => ID */ - 27, /* OF => ID */ - 27, /* OFFSET => ID */ - 27, /* PRAGMA => ID */ - 27, /* RAISE => ID */ - 27, /* RECURSIVE => ID */ - 27, /* REPLACE => ID */ - 27, /* RESTRICT => ID */ - 27, /* ROW => ID */ - 27, /* TRIGGER => ID */ - 27, /* VACUUM => ID */ - 27, /* VIEW => ID */ - 27, /* VIRTUAL => ID */ - 27, /* WITH => ID */ - 27, /* REINDEX => ID */ - 27, /* RENAME => ID */ - 27, /* CTIME_KW => ID */ + 55, /* ABORT => ID */ + 55, /* ACTION => ID */ + 55, /* AFTER => ID */ + 55, /* ANALYZE => ID */ + 55, /* ASC => ID */ + 55, /* ATTACH => ID */ + 55, /* BEFORE => ID */ + 55, /* BY => ID */ + 55, /* CASCADE => ID */ + 55, /* CAST => ID */ + 55, /* COLUMNKW => ID */ + 55, /* CONFLICT => ID */ + 55, /* DATABASE => ID */ + 55, /* DESC => ID */ + 55, /* DETACH => ID */ + 55, /* EACH => ID */ + 55, /* FAIL => ID */ + 55, /* FOR => ID */ + 55, /* IGNORE => ID */ + 55, /* INITIALLY => ID */ + 55, /* INSTEAD => ID */ + 55, /* NO => ID */ + 55, /* KEY => ID */ + 55, /* OF => ID */ + 55, /* OFFSET => ID */ + 55, /* PRAGMA => ID */ + 55, /* RAISE => ID */ + 55, /* RECURSIVE => ID */ + 55, /* REPLACE => ID */ + 55, /* RESTRICT => ID */ + 55, /* ROW => ID */ + 55, /* TRIGGER => ID */ + 55, /* VACUUM => ID */ + 55, /* VIEW => ID */ + 55, /* VIRTUAL => ID */ + 55, /* WITH => ID */ + 55, /* REINDEX => ID */ + 55, /* RENAME => ID */ + 55, /* CTIME_KW => ID */ }; #endif /* YYFALLBACK */ @@ -121224,9 +136898,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -121237,15 +136915,18 @@ typedef struct yyStackEntry yyStackEntry; /* The state of the parser is completely contained in an instance of ** the following structure */ struct yyParser { - int yyidx; /* Index of top element in stack */ + yyStackEntry *yytos; /* Pointer to top element of the stack */ #ifdef YYTRACKMAXSTACKDEPTH - int yyidxMax; /* Maximum value of yyidx */ + int yyhwm; /* High-water mark of the stack */ #endif +#ifndef YYNOERRORRECOVERY int yyerrcnt; /* Shifts left before out of the error */ +#endif sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ + yyStackEntry yystk0; /* First stack entry */ #else yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ #endif @@ -121294,25 +136975,25 @@ static const char *const yyTokenName[] = { "ROLLBACK", "SAVEPOINT", "RELEASE", "TO", "TABLE", "CREATE", "IF", "NOT", "EXISTS", "TEMP", "LP", "RP", - "AS", "WITHOUT", "COMMA", "ID", + "AS", "WITHOUT", "COMMA", "OR", + "AND", "IS", "MATCH", "LIKE_KW", + "BETWEEN", "IN", "ISNULL", "NOTNULL", + "NE", "EQ", "GT", "LE", + "LT", "GE", "ESCAPE", "BITAND", + "BITOR", "LSHIFT", "RSHIFT", "PLUS", + "MINUS", "STAR", "SLASH", "REM", + "CONCAT", "COLLATE", "BITNOT", "ID", "INDEXED", "ABORT", "ACTION", "AFTER", "ANALYZE", "ASC", "ATTACH", "BEFORE", "BY", "CASCADE", "CAST", "COLUMNKW", "CONFLICT", "DATABASE", "DESC", "DETACH", "EACH", "FAIL", "FOR", "IGNORE", - "INITIALLY", "INSTEAD", "LIKE_KW", "MATCH", - "NO", "KEY", "OF", "OFFSET", - "PRAGMA", "RAISE", "RECURSIVE", "REPLACE", - "RESTRICT", "ROW", "TRIGGER", "VACUUM", - "VIEW", "VIRTUAL", "WITH", "REINDEX", - "RENAME", "CTIME_KW", "ANY", "OR", - "AND", "IS", "BETWEEN", "IN", - "ISNULL", "NOTNULL", "NE", "EQ", - "GT", "LE", "LT", "GE", - "ESCAPE", "BITAND", "BITOR", "LSHIFT", - "RSHIFT", "PLUS", "MINUS", "STAR", - "SLASH", "REM", "CONCAT", "COLLATE", - "BITNOT", "STRING", "JOIN_KW", "CONSTRAINT", + "INITIALLY", "INSTEAD", "NO", "KEY", + "OF", "OFFSET", "PRAGMA", "RAISE", + "RECURSIVE", "REPLACE", "RESTRICT", "ROW", + "TRIGGER", "VACUUM", "VIEW", "VIRTUAL", + "WITH", "REINDEX", "RENAME", "CTIME_KW", + "ANY", "STRING", "JOIN_KW", "CONSTRAINT", "DEFAULT", "NULL", "PRIMARY", "UNIQUE", "CHECK", "REFERENCES", "AUTOINCR", "ON", "INSERT", "DELETE", "UPDATE", "SET", @@ -121321,7 +137002,7 @@ static const char *const yyTokenName[] = { "VALUES", "DISTINCT", "DOT", "FROM", "JOIN", "USING", "ORDER", "GROUP", "HAVING", "LIMIT", "WHERE", "INTO", - "INTEGER", "FLOAT", "BLOB", "VARIABLE", + "FLOAT", "BLOB", "INTEGER", "VARIABLE", "CASE", "WHEN", "THEN", "ELSE", "INDEX", "ALTER", "ADD", "error", "input", "cmdlist", "ecmd", "explain", @@ -121329,29 +137010,28 @@ static const char *const yyTokenName[] = { "nm", "savepoint_opt", "create_table", "create_table_args", "createkw", "temp", "ifnotexists", "dbnm", "columnlist", "conslist_opt", "table_options", "select", - "column", "columnid", "type", "carglist", - "typetoken", "typename", "signed", "plus_num", - "minus_num", "ccons", "term", "expr", - "onconf", "sortorder", "autoinc", "idxlist_opt", - "refargs", "defer_subclause", "refarg", "refact", - "init_deferred_pred_opt", "conslist", "tconscomma", "tcons", - "idxlist", "defer_subclause_opt", "orconf", "resolvetype", - "raisetype", "ifexists", "fullname", "selectnowith", - "oneselect", "with", "multiselect_op", "distinct", - "selcollist", "from", "where_opt", "groupby_opt", - "having_opt", "orderby_opt", "limit_opt", "values", - "nexprlist", "exprlist", "sclp", "as", - "seltablist", "stl_prefix", "joinop", "indexed_opt", - "on_opt", "using_opt", "joinop2", "idlist", - "sortlist", "setlist", "insert_cmd", "inscollist_opt", - "likeop", "between_op", "in_op", "case_operand", - "case_exprlist", "case_else", "uniqueflag", "collate", - "nmnum", "trigger_decl", "trigger_cmd_list", "trigger_time", - "trigger_event", "foreach_clause", "when_clause", "trigger_cmd", - "trnm", "tridxby", "database_kw_opt", "key_opt", - "add_column_fullname", "kwcolumn_opt", "create_vtab", "vtabarglist", - "vtabarg", "vtabargtoken", "lp", "anylist", - "wqlist", + "columnname", "carglist", "typetoken", "typename", + "signed", "plus_num", "minus_num", "ccons", + "term", "expr", "onconf", "sortorder", + "autoinc", "eidlist_opt", "refargs", "defer_subclause", + "refarg", "refact", "init_deferred_pred_opt", "conslist", + "tconscomma", "tcons", "sortlist", "eidlist", + "defer_subclause_opt", "orconf", "resolvetype", "raisetype", + "ifexists", "fullname", "selectnowith", "oneselect", + "with", "multiselect_op", "distinct", "selcollist", + "from", "where_opt", "groupby_opt", "having_opt", + "orderby_opt", "limit_opt", "values", "nexprlist", + "exprlist", "sclp", "as", "seltablist", + "stl_prefix", "joinop", "indexed_opt", "on_opt", + "using_opt", "idlist", "setlist", "insert_cmd", + "idlist_opt", "likeop", "between_op", "in_op", + "paren_exprlist", "case_operand", "case_exprlist", "case_else", + "uniqueflag", "collate", "nmnum", "trigger_decl", + "trigger_cmd_list", "trigger_time", "trigger_event", "foreach_clause", + "when_clause", "trigger_cmd", "trnm", "tridxby", + "database_kw_opt", "key_opt", "add_column_fullname", "kwcolumn_opt", + "create_vtab", "vtabarglist", "vtabarg", "vtabargtoken", + "lp", "anylist", "wqlist", }; #endif /* NDEBUG */ @@ -121359,360 +137039,384 @@ static const char *const yyTokenName[] = { /* For tracing reduce actions, the names of all rules are required. */ static const char *const yyRuleName[] = { - /* 0 */ "input ::= cmdlist", - /* 1 */ "cmdlist ::= cmdlist ecmd", - /* 2 */ "cmdlist ::= ecmd", - /* 3 */ "ecmd ::= SEMI", - /* 4 */ "ecmd ::= explain cmdx SEMI", - /* 5 */ "explain ::=", - /* 6 */ "explain ::= EXPLAIN", - /* 7 */ "explain ::= EXPLAIN QUERY PLAN", - /* 8 */ "cmdx ::= cmd", - /* 9 */ "cmd ::= BEGIN transtype trans_opt", - /* 10 */ "trans_opt ::=", - /* 11 */ "trans_opt ::= TRANSACTION", - /* 12 */ "trans_opt ::= TRANSACTION nm", - /* 13 */ "transtype ::=", - /* 14 */ "transtype ::= DEFERRED", - /* 15 */ "transtype ::= IMMEDIATE", - /* 16 */ "transtype ::= EXCLUSIVE", - /* 17 */ "cmd ::= COMMIT trans_opt", - /* 18 */ "cmd ::= END trans_opt", - /* 19 */ "cmd ::= ROLLBACK trans_opt", - /* 20 */ "savepoint_opt ::= SAVEPOINT", - /* 21 */ "savepoint_opt ::=", - /* 22 */ "cmd ::= SAVEPOINT nm", - /* 23 */ "cmd ::= RELEASE savepoint_opt nm", - /* 24 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", - /* 25 */ "cmd ::= create_table create_table_args", - /* 26 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", - /* 27 */ "createkw ::= CREATE", - /* 28 */ "ifnotexists ::=", - /* 29 */ "ifnotexists ::= IF NOT EXISTS", - /* 30 */ "temp ::= TEMP", - /* 31 */ "temp ::=", - /* 32 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", - /* 33 */ "create_table_args ::= AS select", - /* 34 */ "table_options ::=", - /* 35 */ "table_options ::= WITHOUT nm", - /* 36 */ "columnlist ::= columnlist COMMA column", - /* 37 */ "columnlist ::= column", - /* 38 */ "column ::= columnid type carglist", - /* 39 */ "columnid ::= nm", - /* 40 */ "nm ::= ID|INDEXED", - /* 41 */ "nm ::= STRING", - /* 42 */ "nm ::= JOIN_KW", - /* 43 */ "type ::=", - /* 44 */ "type ::= typetoken", - /* 45 */ "typetoken ::= typename", - /* 46 */ "typetoken ::= typename LP signed RP", - /* 47 */ "typetoken ::= typename LP signed COMMA signed RP", - /* 48 */ "typename ::= ID|STRING", - /* 49 */ "typename ::= typename ID|STRING", - /* 50 */ "signed ::= plus_num", - /* 51 */ "signed ::= minus_num", - /* 52 */ "carglist ::= carglist ccons", - /* 53 */ "carglist ::=", - /* 54 */ "ccons ::= CONSTRAINT nm", - /* 55 */ "ccons ::= DEFAULT term", - /* 56 */ "ccons ::= DEFAULT LP expr RP", - /* 57 */ "ccons ::= DEFAULT PLUS term", - /* 58 */ "ccons ::= DEFAULT MINUS term", - /* 59 */ "ccons ::= DEFAULT ID|INDEXED", - /* 60 */ "ccons ::= NULL onconf", - /* 61 */ "ccons ::= NOT NULL onconf", - /* 62 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", - /* 63 */ "ccons ::= UNIQUE onconf", - /* 64 */ "ccons ::= CHECK LP expr RP", - /* 65 */ "ccons ::= REFERENCES nm idxlist_opt refargs", - /* 66 */ "ccons ::= defer_subclause", - /* 67 */ "ccons ::= COLLATE ID|STRING", - /* 68 */ "autoinc ::=", - /* 69 */ "autoinc ::= AUTOINCR", - /* 70 */ "refargs ::=", - /* 71 */ "refargs ::= refargs refarg", - /* 72 */ "refarg ::= MATCH nm", - /* 73 */ "refarg ::= ON INSERT refact", - /* 74 */ "refarg ::= ON DELETE refact", - /* 75 */ "refarg ::= ON UPDATE refact", - /* 76 */ "refact ::= SET NULL", - /* 77 */ "refact ::= SET DEFAULT", - /* 78 */ "refact ::= CASCADE", - /* 79 */ "refact ::= RESTRICT", - /* 80 */ "refact ::= NO ACTION", - /* 81 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", - /* 82 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", - /* 83 */ "init_deferred_pred_opt ::=", - /* 84 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", - /* 85 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", - /* 86 */ "conslist_opt ::=", - /* 87 */ "conslist_opt ::= COMMA conslist", - /* 88 */ "conslist ::= conslist tconscomma tcons", - /* 89 */ "conslist ::= tcons", - /* 90 */ "tconscomma ::= COMMA", - /* 91 */ "tconscomma ::=", - /* 92 */ "tcons ::= CONSTRAINT nm", - /* 93 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", - /* 94 */ "tcons ::= UNIQUE LP idxlist RP onconf", - /* 95 */ "tcons ::= CHECK LP expr RP onconf", - /* 96 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", - /* 97 */ "defer_subclause_opt ::=", - /* 98 */ "defer_subclause_opt ::= defer_subclause", - /* 99 */ "onconf ::=", - /* 100 */ "onconf ::= ON CONFLICT resolvetype", - /* 101 */ "orconf ::=", - /* 102 */ "orconf ::= OR resolvetype", - /* 103 */ "resolvetype ::= raisetype", - /* 104 */ "resolvetype ::= IGNORE", - /* 105 */ "resolvetype ::= REPLACE", - /* 106 */ "cmd ::= DROP TABLE ifexists fullname", - /* 107 */ "ifexists ::= IF EXISTS", - /* 108 */ "ifexists ::=", - /* 109 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select", - /* 110 */ "cmd ::= DROP VIEW ifexists fullname", - /* 111 */ "cmd ::= select", - /* 112 */ "select ::= with selectnowith", - /* 113 */ "selectnowith ::= oneselect", - /* 114 */ "selectnowith ::= selectnowith multiselect_op oneselect", - /* 115 */ "multiselect_op ::= UNION", - /* 116 */ "multiselect_op ::= UNION ALL", - /* 117 */ "multiselect_op ::= EXCEPT|INTERSECT", - /* 118 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", - /* 119 */ "oneselect ::= values", - /* 120 */ "values ::= VALUES LP nexprlist RP", - /* 121 */ "values ::= values COMMA LP exprlist RP", - /* 122 */ "distinct ::= DISTINCT", - /* 123 */ "distinct ::= ALL", - /* 124 */ "distinct ::=", - /* 125 */ "sclp ::= selcollist COMMA", - /* 126 */ "sclp ::=", - /* 127 */ "selcollist ::= sclp expr as", - /* 128 */ "selcollist ::= sclp STAR", - /* 129 */ "selcollist ::= sclp nm DOT STAR", - /* 130 */ "as ::= AS nm", - /* 131 */ "as ::= ID|STRING", - /* 132 */ "as ::=", - /* 133 */ "from ::=", - /* 134 */ "from ::= FROM seltablist", - /* 135 */ "stl_prefix ::= seltablist joinop", - /* 136 */ "stl_prefix ::=", - /* 137 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 138 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 139 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 140 */ "dbnm ::=", - /* 141 */ "dbnm ::= DOT nm", - /* 142 */ "fullname ::= nm dbnm", - /* 143 */ "joinop ::= COMMA|JOIN", - /* 144 */ "joinop ::= JOIN_KW JOIN", - /* 145 */ "joinop ::= JOIN_KW nm JOIN", - /* 146 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 147 */ "on_opt ::= ON expr", - /* 148 */ "on_opt ::=", - /* 149 */ "indexed_opt ::=", - /* 150 */ "indexed_opt ::= INDEXED BY nm", - /* 151 */ "indexed_opt ::= NOT INDEXED", - /* 152 */ "using_opt ::= USING LP idlist RP", - /* 153 */ "using_opt ::=", - /* 154 */ "orderby_opt ::=", - /* 155 */ "orderby_opt ::= ORDER BY sortlist", - /* 156 */ "sortlist ::= sortlist COMMA expr sortorder", - /* 157 */ "sortlist ::= expr sortorder", - /* 158 */ "sortorder ::= ASC", - /* 159 */ "sortorder ::= DESC", - /* 160 */ "sortorder ::=", - /* 161 */ "groupby_opt ::=", - /* 162 */ "groupby_opt ::= GROUP BY nexprlist", - /* 163 */ "having_opt ::=", - /* 164 */ "having_opt ::= HAVING expr", - /* 165 */ "limit_opt ::=", - /* 166 */ "limit_opt ::= LIMIT expr", - /* 167 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 168 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 169 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", - /* 170 */ "where_opt ::=", - /* 171 */ "where_opt ::= WHERE expr", - /* 172 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", - /* 173 */ "setlist ::= setlist COMMA nm EQ expr", - /* 174 */ "setlist ::= nm EQ expr", - /* 175 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt select", - /* 176 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", - /* 177 */ "insert_cmd ::= INSERT orconf", - /* 178 */ "insert_cmd ::= REPLACE", - /* 179 */ "inscollist_opt ::=", - /* 180 */ "inscollist_opt ::= LP idlist RP", - /* 181 */ "idlist ::= idlist COMMA nm", - /* 182 */ "idlist ::= nm", - /* 183 */ "expr ::= term", - /* 184 */ "expr ::= LP expr RP", - /* 185 */ "term ::= NULL", - /* 186 */ "expr ::= ID|INDEXED", - /* 187 */ "expr ::= JOIN_KW", - /* 188 */ "expr ::= nm DOT nm", - /* 189 */ "expr ::= nm DOT nm DOT nm", - /* 190 */ "term ::= INTEGER|FLOAT|BLOB", - /* 191 */ "term ::= STRING", - /* 192 */ "expr ::= VARIABLE", - /* 193 */ "expr ::= expr COLLATE ID|STRING", - /* 194 */ "expr ::= CAST LP expr AS typetoken RP", - /* 195 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 196 */ "expr ::= ID|INDEXED LP STAR RP", - /* 197 */ "term ::= CTIME_KW", - /* 198 */ "expr ::= expr AND expr", - /* 199 */ "expr ::= expr OR expr", - /* 200 */ "expr ::= expr LT|GT|GE|LE expr", - /* 201 */ "expr ::= expr EQ|NE expr", - /* 202 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 203 */ "expr ::= expr PLUS|MINUS expr", - /* 204 */ "expr ::= expr STAR|SLASH|REM expr", - /* 205 */ "expr ::= expr CONCAT expr", - /* 206 */ "likeop ::= LIKE_KW|MATCH", - /* 207 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 208 */ "expr ::= expr likeop expr", - /* 209 */ "expr ::= expr likeop expr ESCAPE expr", - /* 210 */ "expr ::= expr ISNULL|NOTNULL", - /* 211 */ "expr ::= expr NOT NULL", - /* 212 */ "expr ::= expr IS expr", - /* 213 */ "expr ::= expr IS NOT expr", - /* 214 */ "expr ::= NOT expr", - /* 215 */ "expr ::= BITNOT expr", - /* 216 */ "expr ::= MINUS expr", - /* 217 */ "expr ::= PLUS expr", - /* 218 */ "between_op ::= BETWEEN", - /* 219 */ "between_op ::= NOT BETWEEN", - /* 220 */ "expr ::= expr between_op expr AND expr", - /* 221 */ "in_op ::= IN", - /* 222 */ "in_op ::= NOT IN", - /* 223 */ "expr ::= expr in_op LP exprlist RP", - /* 224 */ "expr ::= LP select RP", - /* 225 */ "expr ::= expr in_op LP select RP", - /* 226 */ "expr ::= expr in_op nm dbnm", - /* 227 */ "expr ::= EXISTS LP select RP", - /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 230 */ "case_exprlist ::= WHEN expr THEN expr", - /* 231 */ "case_else ::= ELSE expr", - /* 232 */ "case_else ::=", - /* 233 */ "case_operand ::= expr", - /* 234 */ "case_operand ::=", - /* 235 */ "exprlist ::= nexprlist", - /* 236 */ "exprlist ::=", - /* 237 */ "nexprlist ::= nexprlist COMMA expr", - /* 238 */ "nexprlist ::= expr", - /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt", - /* 240 */ "uniqueflag ::= UNIQUE", - /* 241 */ "uniqueflag ::=", - /* 242 */ "idxlist_opt ::=", - /* 243 */ "idxlist_opt ::= LP idxlist RP", - /* 244 */ "idxlist ::= idxlist COMMA nm collate sortorder", - /* 245 */ "idxlist ::= nm collate sortorder", - /* 246 */ "collate ::=", - /* 247 */ "collate ::= COLLATE ID|STRING", - /* 248 */ "cmd ::= DROP INDEX ifexists fullname", - /* 249 */ "cmd ::= VACUUM", - /* 250 */ "cmd ::= VACUUM nm", - /* 251 */ "cmd ::= PRAGMA nm dbnm", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 256 */ "nmnum ::= plus_num", - /* 257 */ "nmnum ::= nm", - /* 258 */ "nmnum ::= ON", - /* 259 */ "nmnum ::= DELETE", - /* 260 */ "nmnum ::= DEFAULT", - /* 261 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 262 */ "plus_num ::= INTEGER|FLOAT", - /* 263 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 264 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 265 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 266 */ "trigger_time ::= BEFORE", - /* 267 */ "trigger_time ::= AFTER", - /* 268 */ "trigger_time ::= INSTEAD OF", - /* 269 */ "trigger_time ::=", - /* 270 */ "trigger_event ::= DELETE|INSERT", - /* 271 */ "trigger_event ::= UPDATE", - /* 272 */ "trigger_event ::= UPDATE OF idlist", - /* 273 */ "foreach_clause ::=", - /* 274 */ "foreach_clause ::= FOR EACH ROW", - /* 275 */ "when_clause ::=", - /* 276 */ "when_clause ::= WHEN expr", - /* 277 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 278 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 279 */ "trnm ::= nm", - /* 280 */ "trnm ::= nm DOT nm", - /* 281 */ "tridxby ::=", - /* 282 */ "tridxby ::= INDEXED BY nm", - /* 283 */ "tridxby ::= NOT INDEXED", - /* 284 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", - /* 285 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", - /* 286 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", - /* 287 */ "trigger_cmd ::= select", - /* 288 */ "expr ::= RAISE LP IGNORE RP", - /* 289 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 290 */ "raisetype ::= ROLLBACK", - /* 291 */ "raisetype ::= ABORT", - /* 292 */ "raisetype ::= FAIL", - /* 293 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 294 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 295 */ "cmd ::= DETACH database_kw_opt expr", - /* 296 */ "key_opt ::=", - /* 297 */ "key_opt ::= KEY expr", - /* 298 */ "database_kw_opt ::= DATABASE", - /* 299 */ "database_kw_opt ::=", - /* 300 */ "cmd ::= REINDEX", - /* 301 */ "cmd ::= REINDEX nm dbnm", - /* 302 */ "cmd ::= ANALYZE", - /* 303 */ "cmd ::= ANALYZE nm dbnm", - /* 304 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 305 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", - /* 306 */ "add_column_fullname ::= fullname", - /* 307 */ "kwcolumn_opt ::=", - /* 308 */ "kwcolumn_opt ::= COLUMNKW", - /* 309 */ "cmd ::= create_vtab", - /* 310 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 311 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 312 */ "vtabarglist ::= vtabarg", - /* 313 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 314 */ "vtabarg ::=", - /* 315 */ "vtabarg ::= vtabarg vtabargtoken", - /* 316 */ "vtabargtoken ::= ANY", - /* 317 */ "vtabargtoken ::= lp anylist RP", - /* 318 */ "lp ::= LP", - /* 319 */ "anylist ::=", - /* 320 */ "anylist ::= anylist LP anylist RP", - /* 321 */ "anylist ::= anylist ANY", - /* 322 */ "with ::=", - /* 323 */ "with ::= WITH wqlist", - /* 324 */ "with ::= WITH RECURSIVE wqlist", - /* 325 */ "wqlist ::= nm idxlist_opt AS LP select RP", - /* 326 */ "wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP", + /* 0 */ "explain ::= EXPLAIN", + /* 1 */ "explain ::= EXPLAIN QUERY PLAN", + /* 2 */ "cmdx ::= cmd", + /* 3 */ "cmd ::= BEGIN transtype trans_opt", + /* 4 */ "transtype ::=", + /* 5 */ "transtype ::= DEFERRED", + /* 6 */ "transtype ::= IMMEDIATE", + /* 7 */ "transtype ::= EXCLUSIVE", + /* 8 */ "cmd ::= COMMIT trans_opt", + /* 9 */ "cmd ::= END trans_opt", + /* 10 */ "cmd ::= ROLLBACK trans_opt", + /* 11 */ "cmd ::= SAVEPOINT nm", + /* 12 */ "cmd ::= RELEASE savepoint_opt nm", + /* 13 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", + /* 14 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", + /* 15 */ "createkw ::= CREATE", + /* 16 */ "ifnotexists ::=", + /* 17 */ "ifnotexists ::= IF NOT EXISTS", + /* 18 */ "temp ::= TEMP", + /* 19 */ "temp ::=", + /* 20 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", + /* 21 */ "create_table_args ::= AS select", + /* 22 */ "table_options ::=", + /* 23 */ "table_options ::= WITHOUT nm", + /* 24 */ "columnname ::= nm typetoken", + /* 25 */ "typetoken ::=", + /* 26 */ "typetoken ::= typename LP signed RP", + /* 27 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 28 */ "typename ::= typename ID|STRING", + /* 29 */ "ccons ::= CONSTRAINT nm", + /* 30 */ "ccons ::= DEFAULT term", + /* 31 */ "ccons ::= DEFAULT LP expr RP", + /* 32 */ "ccons ::= DEFAULT PLUS term", + /* 33 */ "ccons ::= DEFAULT MINUS term", + /* 34 */ "ccons ::= DEFAULT ID|INDEXED", + /* 35 */ "ccons ::= NOT NULL onconf", + /* 36 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 37 */ "ccons ::= UNIQUE onconf", + /* 38 */ "ccons ::= CHECK LP expr RP", + /* 39 */ "ccons ::= REFERENCES nm eidlist_opt refargs", + /* 40 */ "ccons ::= defer_subclause", + /* 41 */ "ccons ::= COLLATE ID|STRING", + /* 42 */ "autoinc ::=", + /* 43 */ "autoinc ::= AUTOINCR", + /* 44 */ "refargs ::=", + /* 45 */ "refargs ::= refargs refarg", + /* 46 */ "refarg ::= MATCH nm", + /* 47 */ "refarg ::= ON INSERT refact", + /* 48 */ "refarg ::= ON DELETE refact", + /* 49 */ "refarg ::= ON UPDATE refact", + /* 50 */ "refact ::= SET NULL", + /* 51 */ "refact ::= SET DEFAULT", + /* 52 */ "refact ::= CASCADE", + /* 53 */ "refact ::= RESTRICT", + /* 54 */ "refact ::= NO ACTION", + /* 55 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 56 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 57 */ "init_deferred_pred_opt ::=", + /* 58 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 59 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 60 */ "conslist_opt ::=", + /* 61 */ "tconscomma ::= COMMA", + /* 62 */ "tcons ::= CONSTRAINT nm", + /* 63 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", + /* 64 */ "tcons ::= UNIQUE LP sortlist RP onconf", + /* 65 */ "tcons ::= CHECK LP expr RP onconf", + /* 66 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", + /* 67 */ "defer_subclause_opt ::=", + /* 68 */ "onconf ::=", + /* 69 */ "onconf ::= ON CONFLICT resolvetype", + /* 70 */ "orconf ::=", + /* 71 */ "orconf ::= OR resolvetype", + /* 72 */ "resolvetype ::= IGNORE", + /* 73 */ "resolvetype ::= REPLACE", + /* 74 */ "cmd ::= DROP TABLE ifexists fullname", + /* 75 */ "ifexists ::= IF EXISTS", + /* 76 */ "ifexists ::=", + /* 77 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", + /* 78 */ "cmd ::= DROP VIEW ifexists fullname", + /* 79 */ "cmd ::= select", + /* 80 */ "select ::= with selectnowith", + /* 81 */ "selectnowith ::= selectnowith multiselect_op oneselect", + /* 82 */ "multiselect_op ::= UNION", + /* 83 */ "multiselect_op ::= UNION ALL", + /* 84 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 85 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 86 */ "values ::= VALUES LP nexprlist RP", + /* 87 */ "values ::= values COMMA LP exprlist RP", + /* 88 */ "distinct ::= DISTINCT", + /* 89 */ "distinct ::= ALL", + /* 90 */ "distinct ::=", + /* 91 */ "sclp ::=", + /* 92 */ "selcollist ::= sclp expr as", + /* 93 */ "selcollist ::= sclp STAR", + /* 94 */ "selcollist ::= sclp nm DOT STAR", + /* 95 */ "as ::= AS nm", + /* 96 */ "as ::=", + /* 97 */ "from ::=", + /* 98 */ "from ::= FROM seltablist", + /* 99 */ "stl_prefix ::= seltablist joinop", + /* 100 */ "stl_prefix ::=", + /* 101 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 102 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", + /* 103 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 104 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 105 */ "dbnm ::=", + /* 106 */ "dbnm ::= DOT nm", + /* 107 */ "fullname ::= nm dbnm", + /* 108 */ "joinop ::= COMMA|JOIN", + /* 109 */ "joinop ::= JOIN_KW JOIN", + /* 110 */ "joinop ::= JOIN_KW nm JOIN", + /* 111 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 112 */ "on_opt ::= ON expr", + /* 113 */ "on_opt ::=", + /* 114 */ "indexed_opt ::=", + /* 115 */ "indexed_opt ::= INDEXED BY nm", + /* 116 */ "indexed_opt ::= NOT INDEXED", + /* 117 */ "using_opt ::= USING LP idlist RP", + /* 118 */ "using_opt ::=", + /* 119 */ "orderby_opt ::=", + /* 120 */ "orderby_opt ::= ORDER BY sortlist", + /* 121 */ "sortlist ::= sortlist COMMA expr sortorder", + /* 122 */ "sortlist ::= expr sortorder", + /* 123 */ "sortorder ::= ASC", + /* 124 */ "sortorder ::= DESC", + /* 125 */ "sortorder ::=", + /* 126 */ "groupby_opt ::=", + /* 127 */ "groupby_opt ::= GROUP BY nexprlist", + /* 128 */ "having_opt ::=", + /* 129 */ "having_opt ::= HAVING expr", + /* 130 */ "limit_opt ::=", + /* 131 */ "limit_opt ::= LIMIT expr", + /* 132 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 133 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 134 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", + /* 135 */ "where_opt ::=", + /* 136 */ "where_opt ::= WHERE expr", + /* 137 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", + /* 138 */ "setlist ::= setlist COMMA nm EQ expr", + /* 139 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 140 */ "setlist ::= nm EQ expr", + /* 141 */ "setlist ::= LP idlist RP EQ expr", + /* 142 */ "cmd ::= with insert_cmd INTO fullname idlist_opt select", + /* 143 */ "cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES", + /* 144 */ "insert_cmd ::= INSERT orconf", + /* 145 */ "insert_cmd ::= REPLACE", + /* 146 */ "idlist_opt ::=", + /* 147 */ "idlist_opt ::= LP idlist RP", + /* 148 */ "idlist ::= idlist COMMA nm", + /* 149 */ "idlist ::= nm", + /* 150 */ "expr ::= LP expr RP", + /* 151 */ "term ::= NULL", + /* 152 */ "expr ::= ID|INDEXED", + /* 153 */ "expr ::= JOIN_KW", + /* 154 */ "expr ::= nm DOT nm", + /* 155 */ "expr ::= nm DOT nm DOT nm", + /* 156 */ "term ::= FLOAT|BLOB", + /* 157 */ "term ::= STRING", + /* 158 */ "term ::= INTEGER", + /* 159 */ "expr ::= VARIABLE", + /* 160 */ "expr ::= expr COLLATE ID|STRING", + /* 161 */ "expr ::= CAST LP expr AS typetoken RP", + /* 162 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 163 */ "expr ::= ID|INDEXED LP STAR RP", + /* 164 */ "term ::= CTIME_KW", + /* 165 */ "expr ::= LP nexprlist COMMA expr RP", + /* 166 */ "expr ::= expr AND expr", + /* 167 */ "expr ::= expr OR expr", + /* 168 */ "expr ::= expr LT|GT|GE|LE expr", + /* 169 */ "expr ::= expr EQ|NE expr", + /* 170 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 171 */ "expr ::= expr PLUS|MINUS expr", + /* 172 */ "expr ::= expr STAR|SLASH|REM expr", + /* 173 */ "expr ::= expr CONCAT expr", + /* 174 */ "likeop ::= LIKE_KW|MATCH", + /* 175 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 176 */ "expr ::= expr likeop expr", + /* 177 */ "expr ::= expr likeop expr ESCAPE expr", + /* 178 */ "expr ::= expr ISNULL|NOTNULL", + /* 179 */ "expr ::= expr NOT NULL", + /* 180 */ "expr ::= expr IS expr", + /* 181 */ "expr ::= expr IS NOT expr", + /* 182 */ "expr ::= NOT expr", + /* 183 */ "expr ::= BITNOT expr", + /* 184 */ "expr ::= MINUS expr", + /* 185 */ "expr ::= PLUS expr", + /* 186 */ "between_op ::= BETWEEN", + /* 187 */ "between_op ::= NOT BETWEEN", + /* 188 */ "expr ::= expr between_op expr AND expr", + /* 189 */ "in_op ::= IN", + /* 190 */ "in_op ::= NOT IN", + /* 191 */ "expr ::= expr in_op LP exprlist RP", + /* 192 */ "expr ::= LP select RP", + /* 193 */ "expr ::= expr in_op LP select RP", + /* 194 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 195 */ "expr ::= EXISTS LP select RP", + /* 196 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 197 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 198 */ "case_exprlist ::= WHEN expr THEN expr", + /* 199 */ "case_else ::= ELSE expr", + /* 200 */ "case_else ::=", + /* 201 */ "case_operand ::= expr", + /* 202 */ "case_operand ::=", + /* 203 */ "exprlist ::=", + /* 204 */ "nexprlist ::= nexprlist COMMA expr", + /* 205 */ "nexprlist ::= expr", + /* 206 */ "paren_exprlist ::=", + /* 207 */ "paren_exprlist ::= LP exprlist RP", + /* 208 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 209 */ "uniqueflag ::= UNIQUE", + /* 210 */ "uniqueflag ::=", + /* 211 */ "eidlist_opt ::=", + /* 212 */ "eidlist_opt ::= LP eidlist RP", + /* 213 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 214 */ "eidlist ::= nm collate sortorder", + /* 215 */ "collate ::=", + /* 216 */ "collate ::= COLLATE ID|STRING", + /* 217 */ "cmd ::= DROP INDEX ifexists fullname", + /* 218 */ "cmd ::= VACUUM", + /* 219 */ "cmd ::= VACUUM nm", + /* 220 */ "cmd ::= PRAGMA nm dbnm", + /* 221 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 222 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 223 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 224 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 225 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 226 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 227 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 228 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 229 */ "trigger_time ::= BEFORE", + /* 230 */ "trigger_time ::= AFTER", + /* 231 */ "trigger_time ::= INSTEAD OF", + /* 232 */ "trigger_time ::=", + /* 233 */ "trigger_event ::= DELETE|INSERT", + /* 234 */ "trigger_event ::= UPDATE", + /* 235 */ "trigger_event ::= UPDATE OF idlist", + /* 236 */ "when_clause ::=", + /* 237 */ "when_clause ::= WHEN expr", + /* 238 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 239 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 240 */ "trnm ::= nm DOT nm", + /* 241 */ "tridxby ::= INDEXED BY nm", + /* 242 */ "tridxby ::= NOT INDEXED", + /* 243 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", + /* 244 */ "trigger_cmd ::= insert_cmd INTO trnm idlist_opt select", + /* 245 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", + /* 246 */ "trigger_cmd ::= select", + /* 247 */ "expr ::= RAISE LP IGNORE RP", + /* 248 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 249 */ "raisetype ::= ROLLBACK", + /* 250 */ "raisetype ::= ABORT", + /* 251 */ "raisetype ::= FAIL", + /* 252 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 253 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 254 */ "cmd ::= DETACH database_kw_opt expr", + /* 255 */ "key_opt ::=", + /* 256 */ "key_opt ::= KEY expr", + /* 257 */ "cmd ::= REINDEX", + /* 258 */ "cmd ::= REINDEX nm dbnm", + /* 259 */ "cmd ::= ANALYZE", + /* 260 */ "cmd ::= ANALYZE nm dbnm", + /* 261 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 262 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 263 */ "add_column_fullname ::= fullname", + /* 264 */ "cmd ::= create_vtab", + /* 265 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 266 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 267 */ "vtabarg ::=", + /* 268 */ "vtabargtoken ::= ANY", + /* 269 */ "vtabargtoken ::= lp anylist RP", + /* 270 */ "lp ::= LP", + /* 271 */ "with ::=", + /* 272 */ "with ::= WITH wqlist", + /* 273 */ "with ::= WITH RECURSIVE wqlist", + /* 274 */ "wqlist ::= nm eidlist_opt AS LP select RP", + /* 275 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", + /* 276 */ "input ::= cmdlist", + /* 277 */ "cmdlist ::= cmdlist ecmd", + /* 278 */ "cmdlist ::= ecmd", + /* 279 */ "ecmd ::= SEMI", + /* 280 */ "ecmd ::= explain cmdx SEMI", + /* 281 */ "explain ::=", + /* 282 */ "trans_opt ::=", + /* 283 */ "trans_opt ::= TRANSACTION", + /* 284 */ "trans_opt ::= TRANSACTION nm", + /* 285 */ "savepoint_opt ::= SAVEPOINT", + /* 286 */ "savepoint_opt ::=", + /* 287 */ "cmd ::= create_table create_table_args", + /* 288 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 289 */ "columnlist ::= columnname carglist", + /* 290 */ "nm ::= ID|INDEXED", + /* 291 */ "nm ::= STRING", + /* 292 */ "nm ::= JOIN_KW", + /* 293 */ "typetoken ::= typename", + /* 294 */ "typename ::= ID|STRING", + /* 295 */ "signed ::= plus_num", + /* 296 */ "signed ::= minus_num", + /* 297 */ "carglist ::= carglist ccons", + /* 298 */ "carglist ::=", + /* 299 */ "ccons ::= NULL onconf", + /* 300 */ "conslist_opt ::= COMMA conslist", + /* 301 */ "conslist ::= conslist tconscomma tcons", + /* 302 */ "conslist ::= tcons", + /* 303 */ "tconscomma ::=", + /* 304 */ "defer_subclause_opt ::= defer_subclause", + /* 305 */ "resolvetype ::= raisetype", + /* 306 */ "selectnowith ::= oneselect", + /* 307 */ "oneselect ::= values", + /* 308 */ "sclp ::= selcollist COMMA", + /* 309 */ "as ::= ID|STRING", + /* 310 */ "expr ::= term", + /* 311 */ "exprlist ::= nexprlist", + /* 312 */ "nmnum ::= plus_num", + /* 313 */ "nmnum ::= nm", + /* 314 */ "nmnum ::= ON", + /* 315 */ "nmnum ::= DELETE", + /* 316 */ "nmnum ::= DEFAULT", + /* 317 */ "plus_num ::= INTEGER|FLOAT", + /* 318 */ "foreach_clause ::=", + /* 319 */ "foreach_clause ::= FOR EACH ROW", + /* 320 */ "trnm ::= nm", + /* 321 */ "tridxby ::=", + /* 322 */ "database_kw_opt ::= DATABASE", + /* 323 */ "database_kw_opt ::=", + /* 324 */ "kwcolumn_opt ::=", + /* 325 */ "kwcolumn_opt ::= COLUMNKW", + /* 326 */ "vtabarglist ::= vtabarg", + /* 327 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 328 */ "vtabarg ::= vtabarg vtabargtoken", + /* 329 */ "anylist ::=", + /* 330 */ "anylist ::= anylist LP anylist RP", + /* 331 */ "anylist ::= anylist ANY", }; #endif /* NDEBUG */ #if YYSTACKDEPTH<=0 /* -** Try to increase the size of the parser stack. +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. */ -static void yyGrowStack(yyParser *p){ +static int yyGrowStack(yyParser *p){ int newSize; + int idx; yyStackEntry *pNew; newSize = p->yystksz*2 + 100; - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; + if( p->yystack==&p->yystk0 ){ + pNew = malloc(newSize*sizeof(pNew[0])); + if( pNew ) pNew[0] = p->yystk0; + }else{ + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + } if( pNew ){ p->yystack = pNew; - p->yystksz = newSize; + p->yytos = &p->yystack[idx]; #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows to %d entries!\n", - yyTracePrompt, p->yystksz); + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, p->yystksz, newSize); } #endif + p->yystksz = newSize; } + return pNew==0; } #endif +/* Datatype of the argument to the memory allocated passed as the +** second argument to sqlite3ParserAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like @@ -121725,27 +137429,38 @@ static void yyGrowStack(yyParser *p){ ** A pointer to a parser. This pointer is used in subsequent calls ** to sqlite3Parser and sqlite3ParserFree. */ -SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(size_t)){ +SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); + pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); if( pParser ){ - pParser->yyidx = -1; #ifdef YYTRACKMAXSTACKDEPTH - pParser->yyidxMax = 0; + pParser->yyhwm = 0; #endif #if YYSTACKDEPTH<=0 + pParser->yytos = NULL; pParser->yystack = NULL; pParser->yystksz = 0; - yyGrowStack(pParser); + if( yyGrowStack(pParser) ){ + pParser->yystack = &pParser->yystk0; + pParser->yystksz = 1; + } #endif +#ifndef YYNOERRORRECOVERY + pParser->yyerrcnt = -1; +#endif + pParser->yytos = pParser->yystack; + pParser->yystack[0].stateno = 0; + pParser->yystack[0].major = 0; } return pParser; } -/* The following function deletes the value associated with a -** symbol. The symbol can be either a terminal or nonterminal. -** "yymajor" is the symbol code, and "yypminor" is a pointer to -** the value. +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. */ static void yy_destructor( yyParser *yypParser, /* The parser */ @@ -121761,81 +137476,84 @@ static void yy_destructor( ** being destroyed before it is finished parsing. ** ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are not used + ** which appear on the RHS of the rule, but which are *not* used ** inside the C code. */ +/********* Begin destructor definitions ***************************************/ case 163: /* select */ - case 195: /* selectnowith */ - case 196: /* oneselect */ - case 207: /* values */ + case 194: /* selectnowith */ + case 195: /* oneselect */ + case 206: /* values */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy3)); +sqlite3SelectDelete(pParse->db, (yypminor->yy243)); } break; - case 174: /* term */ - case 175: /* expr */ + case 172: /* term */ + case 173: /* expr */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy346).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy190).pExpr); } break; - case 179: /* idxlist_opt */ - case 188: /* idxlist */ - case 200: /* selcollist */ - case 203: /* groupby_opt */ - case 205: /* orderby_opt */ - case 208: /* nexprlist */ - case 209: /* exprlist */ - case 210: /* sclp */ - case 220: /* sortlist */ - case 221: /* setlist */ - case 228: /* case_exprlist */ + case 177: /* eidlist_opt */ + case 186: /* sortlist */ + case 187: /* eidlist */ + case 199: /* selcollist */ + case 202: /* groupby_opt */ + case 204: /* orderby_opt */ + case 207: /* nexprlist */ + case 208: /* exprlist */ + case 209: /* sclp */ + case 218: /* setlist */ + case 224: /* paren_exprlist */ + case 226: /* case_exprlist */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy148)); } break; - case 194: /* fullname */ - case 201: /* from */ - case 212: /* seltablist */ - case 213: /* stl_prefix */ + case 193: /* fullname */ + case 200: /* from */ + case 211: /* seltablist */ + case 212: /* stl_prefix */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy65)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy185)); } break; - case 197: /* with */ - case 252: /* wqlist */ + case 196: /* with */ + case 250: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy59)); +sqlite3WithDelete(pParse->db, (yypminor->yy285)); } break; - case 202: /* where_opt */ - case 204: /* having_opt */ - case 216: /* on_opt */ - case 227: /* case_operand */ - case 229: /* case_else */ - case 238: /* when_clause */ - case 243: /* key_opt */ + case 201: /* where_opt */ + case 203: /* having_opt */ + case 215: /* on_opt */ + case 225: /* case_operand */ + case 227: /* case_else */ + case 236: /* when_clause */ + case 241: /* key_opt */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy132)); +sqlite3ExprDelete(pParse->db, (yypminor->yy72)); } break; - case 217: /* using_opt */ - case 219: /* idlist */ - case 223: /* inscollist_opt */ + case 216: /* using_opt */ + case 217: /* idlist */ + case 220: /* idlist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy408)); +sqlite3IdListDelete(pParse->db, (yypminor->yy254)); } break; - case 234: /* trigger_cmd_list */ - case 239: /* trigger_cmd */ + case 232: /* trigger_cmd_list */ + case 237: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy473)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy145)); } break; - case 236: /* trigger_event */ + case 234: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy378).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy332).b); } break; +/********* End destructor definitions *****************************************/ default: break; /* If no destructor action specified: do nothing */ } } @@ -121845,52 +137563,41 @@ sqlite3IdListDelete(pParse->db, (yypminor->yy378).b); ** ** If there is a destructor routine associated with the token which ** is popped from the stack, then call it. -** -** Return the major token number for the symbol popped. */ -static int yy_pop_parser_stack(yyParser *pParser){ - YYCODETYPE yymajor; - yyStackEntry *yytos = &pParser->yystack[pParser->yyidx]; - - /* There is no mechanism by which the parser stack can be popped below - ** empty in SQLite. */ - if( NEVER(pParser->yyidx<0) ) return 0; +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; #ifndef NDEBUG - if( yyTraceFILE && pParser->yyidx>=0 ){ + if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sPopping %s\n", yyTracePrompt, yyTokenName[yytos->major]); } #endif - yymajor = yytos->major; - yy_destructor(pParser, yymajor, &yytos->minor); - pParser->yyidx--; - return yymajor; + yy_destructor(pParser, yytos->major, &yytos->minor); } /* -** Deallocate and destroy a parser. Destructors are all called for +** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. ** -** Inputs: -**
      -**
    • A pointer to the parser. This should be a pointer -** obtained from sqlite3ParserAlloc. -**
    • A pointer to a function used to reclaim memory obtained -** from malloc. -**
    +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. */ SQLITE_PRIVATE void sqlite3ParserFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ yyParser *pParser = (yyParser*)p; - /* In SQLite, we never try to destroy a parser that was not successfully - ** created in the first place. */ - if( NEVER(pParser==0) ) return; - while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); +#ifndef YYPARSEFREENEVERNULL + if( pParser==0 ) return; +#endif + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 - free(pParser->yystack); + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); #endif (*freeProc)((void*)pParser); } @@ -121901,33 +137608,28 @@ SQLITE_PRIVATE void sqlite3ParserFree( #ifdef YYTRACKMAXSTACKDEPTH SQLITE_PRIVATE int sqlite3ParserStackPeak(void *p){ yyParser *pParser = (yyParser*)p; - return pParser->yyidxMax; + return pParser->yyhwm; } #endif /* ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. */ -static int yy_find_shift_action( +static unsigned int yy_find_shift_action( yyParser *pParser, /* The parser */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; - int stateno = pParser->yystack[pParser->yyidx].stateno; + int stateno = pParser->yytos->stateno; - if( stateno>YY_SHIFT_COUNT - || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ - return yy_default[stateno]; - } - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - if( iLookAhead>0 ){ + if( stateno>=YY_MIN_REDUCE ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); + do{ + i = yy_shift_ofst[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ if( iLookAhead=YY_ACTTAB_COUNT j0 ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[YYWILDCARD]); + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); } #endif /* NDEBUG */ return yy_action[j]; } } #endif /* YYWILDCARD */ + return yy_default[stateno]; + }else{ + return yy_action[i]; } - return yy_default[stateno]; - }else{ - return yy_action[i]; - } + }while(1); } /* ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. */ static int yy_find_reduce_action( int stateno, /* Current state number */ @@ -122008,23 +137709,44 @@ static int yy_find_reduce_action( /* ** The following routine is called if the stack overflows. */ -static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ +static void yyStackOverflow(yyParser *yypParser){ sqlite3ParserARG_FETCH; - yypParser->yyidx--; + yypParser->yytos--; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ - UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ sqlite3ErrorMsg(pParse, "parser stack overflow"); +/******** End %stack_overflow code ********************************************/ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ } +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + if( yyNewStateyytos->major], + yyNewState); + }else{ + fprintf(yyTraceFILE,"%sShift '%s'\n", + yyTracePrompt,yyTokenName[yypParser->yytos->major]); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + /* ** Perform a shift action. */ @@ -122032,43 +137754,37 @@ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ - YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ + sqlite3ParserTOKENTYPE yyMinor /* The minor token to shift in */ ){ yyStackEntry *yytos; - yypParser->yyidx++; + yypParser->yytos++; #ifdef YYTRACKMAXSTACKDEPTH - if( yypParser->yyidx>yypParser->yyidxMax ){ - yypParser->yyidxMax = yypParser->yyidx; + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); } #endif #if YYSTACKDEPTH>0 - if( yypParser->yyidx>=YYSTACKDEPTH ){ - yyStackOverflow(yypParser, yypMinor); + if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH] ){ + yyStackOverflow(yypParser); return; } #else - if( yypParser->yyidx>=yypParser->yystksz ){ - yyGrowStack(yypParser); - if( yypParser->yyidx>=yypParser->yystksz ){ - yyStackOverflow(yypParser, yypMinor); + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); return; } } #endif - yytos = &yypParser->yystack[yypParser->yyidx]; + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos = yypParser->yytos; yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; - yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); - fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - } -#endif + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -122078,19 +137794,10 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ unsigned char nrhs; /* Number of right-hand side symbols in the rule */ } yyRuleInfo[] = { - { 144, 1 }, - { 145, 2 }, - { 145, 1 }, - { 146, 1 }, - { 146, 3 }, - { 147, 0 }, { 147, 1 }, { 147, 3 }, { 148, 1 }, { 149, 3 }, - { 151, 0 }, - { 151, 1 }, - { 151, 2 }, { 150, 0 }, { 150, 1 }, { 150, 1 }, @@ -122098,12 +137805,9 @@ static const struct { { 149, 2 }, { 149, 2 }, { 149, 2 }, - { 153, 1 }, - { 153, 0 }, { 149, 2 }, { 149, 3 }, { 149, 5 }, - { 149, 2 }, { 154, 6 }, { 156, 1 }, { 158, 0 }, @@ -122114,218 +137818,199 @@ static const struct { { 155, 2 }, { 162, 0 }, { 162, 2 }, - { 160, 3 }, - { 160, 1 }, - { 164, 3 }, - { 165, 1 }, - { 152, 1 }, - { 152, 1 }, - { 152, 1 }, + { 164, 2 }, { 166, 0 }, - { 166, 1 }, - { 168, 1 }, - { 168, 4 }, - { 168, 6 }, - { 169, 1 }, - { 169, 2 }, - { 170, 1 }, - { 170, 1 }, + { 166, 4 }, + { 166, 6 }, { 167, 2 }, - { 167, 0 }, - { 173, 2 }, - { 173, 2 }, - { 173, 4 }, - { 173, 3 }, - { 173, 3 }, - { 173, 2 }, - { 173, 2 }, - { 173, 3 }, - { 173, 5 }, - { 173, 2 }, - { 173, 4 }, - { 173, 4 }, - { 173, 1 }, - { 173, 2 }, - { 178, 0 }, - { 178, 1 }, - { 180, 0 }, - { 180, 2 }, - { 182, 2 }, - { 182, 3 }, - { 182, 3 }, - { 182, 3 }, - { 183, 2 }, - { 183, 2 }, - { 183, 1 }, - { 183, 1 }, - { 183, 2 }, - { 181, 3 }, - { 181, 2 }, - { 184, 0 }, - { 184, 2 }, - { 184, 2 }, - { 161, 0 }, - { 161, 2 }, - { 185, 3 }, - { 185, 1 }, - { 186, 1 }, - { 186, 0 }, - { 187, 2 }, - { 187, 7 }, - { 187, 5 }, - { 187, 5 }, - { 187, 10 }, - { 189, 0 }, - { 189, 1 }, + { 171, 2 }, + { 171, 2 }, + { 171, 4 }, + { 171, 3 }, + { 171, 3 }, + { 171, 2 }, + { 171, 3 }, + { 171, 5 }, + { 171, 2 }, + { 171, 4 }, + { 171, 4 }, + { 171, 1 }, + { 171, 2 }, { 176, 0 }, - { 176, 3 }, - { 190, 0 }, - { 190, 2 }, - { 191, 1 }, - { 191, 1 }, - { 191, 1 }, + { 176, 1 }, + { 178, 0 }, + { 178, 2 }, + { 180, 2 }, + { 180, 3 }, + { 180, 3 }, + { 180, 3 }, + { 181, 2 }, + { 181, 2 }, + { 181, 1 }, + { 181, 1 }, + { 181, 2 }, + { 179, 3 }, + { 179, 2 }, + { 182, 0 }, + { 182, 2 }, + { 182, 2 }, + { 161, 0 }, + { 184, 1 }, + { 185, 2 }, + { 185, 7 }, + { 185, 5 }, + { 185, 5 }, + { 185, 10 }, + { 188, 0 }, + { 174, 0 }, + { 174, 3 }, + { 189, 0 }, + { 189, 2 }, + { 190, 1 }, + { 190, 1 }, { 149, 4 }, - { 193, 2 }, - { 193, 0 }, - { 149, 8 }, + { 192, 2 }, + { 192, 0 }, + { 149, 9 }, { 149, 4 }, { 149, 1 }, { 163, 2 }, - { 195, 1 }, - { 195, 3 }, + { 194, 3 }, + { 197, 1 }, + { 197, 2 }, + { 197, 1 }, + { 195, 9 }, + { 206, 4 }, + { 206, 5 }, { 198, 1 }, - { 198, 2 }, { 198, 1 }, - { 196, 9 }, - { 196, 1 }, - { 207, 4 }, - { 207, 5 }, - { 199, 1 }, - { 199, 1 }, - { 199, 0 }, + { 198, 0 }, + { 209, 0 }, + { 199, 3 }, + { 199, 2 }, + { 199, 4 }, { 210, 2 }, { 210, 0 }, - { 200, 3 }, + { 200, 0 }, { 200, 2 }, - { 200, 4 }, - { 211, 2 }, - { 211, 1 }, - { 211, 0 }, - { 201, 0 }, - { 201, 2 }, - { 213, 2 }, - { 213, 0 }, - { 212, 7 }, - { 212, 7 }, - { 212, 7 }, + { 212, 2 }, + { 212, 0 }, + { 211, 7 }, + { 211, 9 }, + { 211, 7 }, + { 211, 7 }, { 159, 0 }, { 159, 2 }, - { 194, 2 }, - { 214, 1 }, - { 214, 2 }, - { 214, 3 }, - { 214, 4 }, - { 216, 2 }, - { 216, 0 }, - { 215, 0 }, - { 215, 3 }, + { 193, 2 }, + { 213, 1 }, + { 213, 2 }, + { 213, 3 }, + { 213, 4 }, { 215, 2 }, - { 217, 4 }, - { 217, 0 }, - { 205, 0 }, - { 205, 3 }, - { 220, 4 }, - { 220, 2 }, - { 177, 1 }, - { 177, 1 }, - { 177, 0 }, - { 203, 0 }, - { 203, 3 }, + { 215, 0 }, + { 214, 0 }, + { 214, 3 }, + { 214, 2 }, + { 216, 4 }, + { 216, 0 }, { 204, 0 }, - { 204, 2 }, - { 206, 0 }, - { 206, 2 }, - { 206, 4 }, - { 206, 4 }, - { 149, 6 }, + { 204, 3 }, + { 186, 4 }, + { 186, 2 }, + { 175, 1 }, + { 175, 1 }, + { 175, 0 }, { 202, 0 }, - { 202, 2 }, + { 202, 3 }, + { 203, 0 }, + { 203, 2 }, + { 205, 0 }, + { 205, 2 }, + { 205, 4 }, + { 205, 4 }, + { 149, 6 }, + { 201, 0 }, + { 201, 2 }, { 149, 8 }, - { 221, 5 }, - { 221, 3 }, + { 218, 5 }, + { 218, 7 }, + { 218, 3 }, + { 218, 5 }, { 149, 6 }, { 149, 7 }, - { 222, 2 }, - { 222, 1 }, - { 223, 0 }, - { 223, 3 }, - { 219, 3 }, + { 219, 2 }, { 219, 1 }, - { 175, 1 }, - { 175, 3 }, - { 174, 1 }, - { 175, 1 }, - { 175, 1 }, - { 175, 3 }, - { 175, 5 }, - { 174, 1 }, - { 174, 1 }, - { 175, 1 }, - { 175, 3 }, - { 175, 6 }, - { 175, 5 }, - { 175, 4 }, - { 174, 1 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 224, 1 }, - { 224, 2 }, - { 175, 3 }, - { 175, 5 }, - { 175, 2 }, - { 175, 3 }, - { 175, 3 }, - { 175, 4 }, - { 175, 2 }, - { 175, 2 }, - { 175, 2 }, - { 175, 2 }, - { 225, 1 }, - { 225, 2 }, - { 175, 5 }, - { 226, 1 }, - { 226, 2 }, - { 175, 5 }, - { 175, 3 }, - { 175, 5 }, - { 175, 4 }, - { 175, 4 }, - { 175, 5 }, - { 228, 5 }, - { 228, 4 }, - { 229, 2 }, - { 229, 0 }, - { 227, 1 }, + { 220, 0 }, + { 220, 3 }, + { 217, 3 }, + { 217, 1 }, + { 173, 3 }, + { 172, 1 }, + { 173, 1 }, + { 173, 1 }, + { 173, 3 }, + { 173, 5 }, + { 172, 1 }, + { 172, 1 }, + { 172, 1 }, + { 173, 1 }, + { 173, 3 }, + { 173, 6 }, + { 173, 5 }, + { 173, 4 }, + { 172, 1 }, + { 173, 5 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 221, 1 }, + { 221, 2 }, + { 173, 3 }, + { 173, 5 }, + { 173, 2 }, + { 173, 3 }, + { 173, 3 }, + { 173, 4 }, + { 173, 2 }, + { 173, 2 }, + { 173, 2 }, + { 173, 2 }, + { 222, 1 }, + { 222, 2 }, + { 173, 5 }, + { 223, 1 }, + { 223, 2 }, + { 173, 5 }, + { 173, 3 }, + { 173, 5 }, + { 173, 5 }, + { 173, 4 }, + { 173, 5 }, + { 226, 5 }, + { 226, 4 }, + { 227, 2 }, { 227, 0 }, - { 209, 1 }, - { 209, 0 }, - { 208, 3 }, - { 208, 1 }, + { 225, 1 }, + { 225, 0 }, + { 208, 0 }, + { 207, 3 }, + { 207, 1 }, + { 224, 0 }, + { 224, 3 }, { 149, 12 }, - { 230, 1 }, - { 230, 0 }, - { 179, 0 }, - { 179, 3 }, - { 188, 5 }, - { 188, 3 }, - { 231, 0 }, - { 231, 2 }, + { 228, 1 }, + { 228, 0 }, + { 177, 0 }, + { 177, 3 }, + { 187, 5 }, + { 187, 3 }, + { 229, 0 }, + { 229, 2 }, { 149, 4 }, { 149, 1 }, { 149, 2 }, @@ -122334,77 +138019,113 @@ static const struct { { 149, 6 }, { 149, 5 }, { 149, 6 }, - { 232, 1 }, - { 232, 1 }, - { 232, 1 }, - { 232, 1 }, - { 232, 1 }, - { 171, 2 }, - { 171, 1 }, - { 172, 2 }, + { 169, 2 }, + { 170, 2 }, { 149, 5 }, - { 233, 11 }, - { 235, 1 }, - { 235, 1 }, - { 235, 2 }, - { 235, 0 }, - { 236, 1 }, - { 236, 1 }, - { 236, 3 }, - { 237, 0 }, - { 237, 3 }, - { 238, 0 }, - { 238, 2 }, + { 231, 11 }, + { 233, 1 }, + { 233, 1 }, + { 233, 2 }, + { 233, 0 }, + { 234, 1 }, + { 234, 1 }, { 234, 3 }, - { 234, 2 }, - { 240, 1 }, - { 240, 3 }, + { 236, 0 }, + { 236, 2 }, + { 232, 3 }, + { 232, 2 }, + { 238, 3 }, + { 239, 3 }, + { 239, 2 }, + { 237, 7 }, + { 237, 5 }, + { 237, 5 }, + { 237, 1 }, + { 173, 4 }, + { 173, 6 }, + { 191, 1 }, + { 191, 1 }, + { 191, 1 }, + { 149, 4 }, + { 149, 6 }, + { 149, 3 }, { 241, 0 }, - { 241, 3 }, { 241, 2 }, - { 239, 7 }, - { 239, 5 }, - { 239, 5 }, - { 239, 1 }, - { 175, 4 }, - { 175, 6 }, - { 192, 1 }, - { 192, 1 }, - { 192, 1 }, - { 149, 4 }, - { 149, 6 }, + { 149, 1 }, { 149, 3 }, - { 243, 0 }, - { 243, 2 }, + { 149, 1 }, + { 149, 3 }, + { 149, 6 }, + { 149, 7 }, { 242, 1 }, - { 242, 0 }, - { 149, 1 }, - { 149, 3 }, - { 149, 1 }, - { 149, 3 }, - { 149, 6 }, - { 149, 6 }, - { 244, 1 }, - { 245, 0 }, - { 245, 1 }, { 149, 1 }, { 149, 4 }, - { 246, 8 }, + { 244, 8 }, + { 246, 0 }, { 247, 1 }, { 247, 3 }, - { 248, 0 }, - { 248, 2 }, - { 249, 1 }, - { 249, 3 }, - { 250, 1 }, - { 251, 0 }, - { 251, 4 }, - { 251, 2 }, - { 197, 0 }, - { 197, 2 }, - { 197, 3 }, - { 252, 6 }, - { 252, 8 }, + { 248, 1 }, + { 196, 0 }, + { 196, 2 }, + { 196, 3 }, + { 250, 6 }, + { 250, 8 }, + { 144, 1 }, + { 145, 2 }, + { 145, 1 }, + { 146, 1 }, + { 146, 3 }, + { 147, 0 }, + { 151, 0 }, + { 151, 1 }, + { 151, 2 }, + { 153, 1 }, + { 153, 0 }, + { 149, 2 }, + { 160, 4 }, + { 160, 2 }, + { 152, 1 }, + { 152, 1 }, + { 152, 1 }, + { 166, 1 }, + { 167, 1 }, + { 168, 1 }, + { 168, 1 }, + { 165, 2 }, + { 165, 0 }, + { 171, 2 }, + { 161, 2 }, + { 183, 3 }, + { 183, 1 }, + { 184, 0 }, + { 188, 1 }, + { 190, 1 }, + { 194, 1 }, + { 195, 1 }, + { 209, 2 }, + { 210, 1 }, + { 173, 1 }, + { 208, 1 }, + { 230, 1 }, + { 230, 1 }, + { 230, 1 }, + { 230, 1 }, + { 230, 1 }, + { 169, 1 }, + { 235, 0 }, + { 235, 3 }, + { 238, 1 }, + { 239, 0 }, + { 240, 1 }, + { 240, 0 }, + { 243, 0 }, + { 243, 1 }, + { 245, 1 }, + { 245, 3 }, + { 246, 2 }, + { 249, 0 }, + { 249, 4 }, + { 249, 2 }, }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -122415,40 +138136,47 @@ static void yy_accept(yyParser*); /* Forward Declaration */ */ static void yy_reduce( yyParser *yypParser, /* The parser */ - int yyruleno /* Number of the rule by which to reduce */ + unsigned int yyruleno /* Number of the rule by which to reduce */ ){ int yygoto; /* The next state */ int yyact; /* The next action */ - YYMINORTYPE yygotominor; /* The LHS of the rule reduced */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ sqlite3ParserARG_FETCH; - yymsp = &yypParser->yystack[yypParser->yyidx]; + yymsp = yypParser->yytos; #ifndef NDEBUG - if( yyTraceFILE && yyruleno>=0 - && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, - yyRuleName[yyruleno]); + if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + yysize = yyRuleInfo[yyruleno].nrhs; + fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt, + yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ - /* Silence complaints from purify about yygotominor being uninitialized - ** in some cases when it is copied into the stack after the following - ** switch. yygotominor is uninitialized when a rule reduces that does - ** not set the value of its left-hand side nonterminal. Leaving the - ** value of the nonterminal uninitialized is utterly harmless as long - ** as the value is never used. So really the only thing this code - ** accomplishes is to quieten purify. - ** - ** 2007-01-16: The wireshark project (www.wireshark.org) reports that - ** without this code, their parser segfaults. I'm not sure what there - ** parser is doing to make this happen. This is the second bug report - ** from wireshark this week. Clearly they are stressing Lemon in ways - ** that it has not been previously stressed... (SQLite ticket #2172) - */ - /*memset(&yygotominor, 0, sizeof(yygotominor));*/ - yygotominor = yyzerominor; - + /* Check that the stack is large enough to grow by a single entry + ** if the RHS of the rule is empty. This ensures that there is room + ** enough on the stack to push the LHS value */ + if( yyRuleInfo[yyruleno].nrhs==0 ){ +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH-1] ){ + yyStackOverflow(yypParser); + return; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + return; + } + yymsp = yypParser->yytos; + } +#endif + } switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -122459,785 +138187,813 @@ static void yy_reduce( ** #line ** break; */ - case 5: /* explain ::= */ -{ sqlite3BeginParse(pParse, 0); } +/********** Begin reduce actions **********************************************/ + YYMINORTYPE yylhsminor; + case 0: /* explain ::= EXPLAIN */ +{ pParse->explain = 1; } break; - case 6: /* explain ::= EXPLAIN */ -{ sqlite3BeginParse(pParse, 1); } + case 1: /* explain ::= EXPLAIN QUERY PLAN */ +{ pParse->explain = 2; } break; - case 7: /* explain ::= EXPLAIN QUERY PLAN */ -{ sqlite3BeginParse(pParse, 2); } - break; - case 8: /* cmdx ::= cmd */ + case 2: /* cmdx ::= cmd */ { sqlite3FinishCoding(pParse); } break; - case 9: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy328);} + case 3: /* cmd ::= BEGIN transtype trans_opt */ +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy194);} break; - case 13: /* transtype ::= */ -{yygotominor.yy328 = TK_DEFERRED;} + case 4: /* transtype ::= */ +{yymsp[1].minor.yy194 = TK_DEFERRED;} break; - case 14: /* transtype ::= DEFERRED */ - case 15: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==15); - case 16: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==16); - case 115: /* multiselect_op ::= UNION */ yytestcase(yyruleno==115); - case 117: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==117); -{yygotominor.yy328 = yymsp[0].major;} + case 5: /* transtype ::= DEFERRED */ + case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); + case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); +{yymsp[0].minor.yy194 = yymsp[0].major; /*A-overwrites-X*/} break; - case 17: /* cmd ::= COMMIT trans_opt */ - case 18: /* cmd ::= END trans_opt */ yytestcase(yyruleno==18); + case 8: /* cmd ::= COMMIT trans_opt */ + case 9: /* cmd ::= END trans_opt */ yytestcase(yyruleno==9); {sqlite3CommitTransaction(pParse);} break; - case 19: /* cmd ::= ROLLBACK trans_opt */ + case 10: /* cmd ::= ROLLBACK trans_opt */ {sqlite3RollbackTransaction(pParse);} break; - case 22: /* cmd ::= SAVEPOINT nm */ + case 11: /* cmd ::= SAVEPOINT nm */ { sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); } break; - case 23: /* cmd ::= RELEASE savepoint_opt nm */ + case 12: /* cmd ::= RELEASE savepoint_opt nm */ { sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); } break; - case 24: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + case 13: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ { sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); } break; - case 26: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + case 14: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy328,0,0,yymsp[-2].minor.yy328); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy194,0,0,yymsp[-2].minor.yy194); } break; - case 27: /* createkw ::= CREATE */ + case 15: /* createkw ::= CREATE */ +{disableLookaside(pParse);} + break; + case 16: /* ifnotexists ::= */ + case 19: /* temp ::= */ yytestcase(yyruleno==19); + case 22: /* table_options ::= */ yytestcase(yyruleno==22); + case 42: /* autoinc ::= */ yytestcase(yyruleno==42); + case 57: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==57); + case 67: /* defer_subclause_opt ::= */ yytestcase(yyruleno==67); + case 76: /* ifexists ::= */ yytestcase(yyruleno==76); + case 90: /* distinct ::= */ yytestcase(yyruleno==90); + case 215: /* collate ::= */ yytestcase(yyruleno==215); +{yymsp[1].minor.yy194 = 0;} + break; + case 17: /* ifnotexists ::= IF NOT EXISTS */ +{yymsp[-2].minor.yy194 = 1;} + break; + case 18: /* temp ::= TEMP */ + case 43: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==43); +{yymsp[0].minor.yy194 = 1;} + break; + case 20: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ { - pParse->db->lookaside.bEnabled = 0; - yygotominor.yy0 = yymsp[0].minor.yy0; + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy194,0); } break; - case 28: /* ifnotexists ::= */ - case 31: /* temp ::= */ yytestcase(yyruleno==31); - case 68: /* autoinc ::= */ yytestcase(yyruleno==68); - case 81: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ yytestcase(yyruleno==81); - case 83: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==83); - case 85: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==85); - case 97: /* defer_subclause_opt ::= */ yytestcase(yyruleno==97); - case 108: /* ifexists ::= */ yytestcase(yyruleno==108); - case 218: /* between_op ::= BETWEEN */ yytestcase(yyruleno==218); - case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); -{yygotominor.yy328 = 0;} - break; - case 29: /* ifnotexists ::= IF NOT EXISTS */ - case 30: /* temp ::= TEMP */ yytestcase(yyruleno==30); - case 69: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==69); - case 84: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==84); - case 107: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==107); - case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); - case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); -{yygotominor.yy328 = 1;} - break; - case 32: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ + case 21: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy186,0); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy243); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy243); } break; - case 33: /* create_table_args ::= AS select */ -{ - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy3); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); -} - break; - case 34: /* table_options ::= */ -{yygotominor.yy186 = 0;} - break; - case 35: /* table_options ::= WITHOUT nm */ + case 23: /* table_options ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yygotominor.yy186 = TF_WithoutRowid; + yymsp[-1].minor.yy194 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yygotominor.yy186 = 0; + yymsp[-1].minor.yy194 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } break; - case 38: /* column ::= columnid type carglist */ + case 24: /* columnname ::= nm typetoken */ +{sqlite3AddColumn(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} + break; + case 25: /* typetoken ::= */ + case 60: /* conslist_opt ::= */ yytestcase(yyruleno==60); + case 96: /* as ::= */ yytestcase(yyruleno==96); +{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} + break; + case 26: /* typetoken ::= typename LP signed RP */ { - yygotominor.yy0.z = yymsp[-2].minor.yy0.z; - yygotominor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-2].minor.yy0.z) + pParse->sLastToken.n; + yymsp[-3].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); } break; - case 39: /* columnid ::= nm */ + case 27: /* typetoken ::= typename LP signed COMMA signed RP */ { - sqlite3AddColumn(pParse,&yymsp[0].minor.yy0); - yygotominor.yy0 = yymsp[0].minor.yy0; - pParse->constraintName.n = 0; + yymsp[-5].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); } break; - case 40: /* nm ::= ID|INDEXED */ - case 41: /* nm ::= STRING */ yytestcase(yyruleno==41); - case 42: /* nm ::= JOIN_KW */ yytestcase(yyruleno==42); - case 45: /* typetoken ::= typename */ yytestcase(yyruleno==45); - case 48: /* typename ::= ID|STRING */ yytestcase(yyruleno==48); - case 130: /* as ::= AS nm */ yytestcase(yyruleno==130); - case 131: /* as ::= ID|STRING */ yytestcase(yyruleno==131); - case 141: /* dbnm ::= DOT nm */ yytestcase(yyruleno==141); - case 150: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==150); - case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); - case 256: /* nmnum ::= plus_num */ yytestcase(yyruleno==256); - case 257: /* nmnum ::= nm */ yytestcase(yyruleno==257); - case 258: /* nmnum ::= ON */ yytestcase(yyruleno==258); - case 259: /* nmnum ::= DELETE */ yytestcase(yyruleno==259); - case 260: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==260); - case 261: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==261); - case 262: /* plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==262); - case 263: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==263); - case 279: /* trnm ::= nm */ yytestcase(yyruleno==279); -{yygotominor.yy0 = yymsp[0].minor.yy0;} + case 28: /* typename ::= typename ID|STRING */ +{yymsp[-1].minor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} break; - case 44: /* type ::= typetoken */ -{sqlite3AddColumnType(pParse,&yymsp[0].minor.yy0);} - break; - case 46: /* typetoken ::= typename LP signed RP */ -{ - yygotominor.yy0.z = yymsp[-3].minor.yy0.z; - yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); -} - break; - case 47: /* typetoken ::= typename LP signed COMMA signed RP */ -{ - yygotominor.yy0.z = yymsp[-5].minor.yy0.z; - yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); -} - break; - case 49: /* typename ::= typename ID|STRING */ -{yygotominor.yy0.z=yymsp[-1].minor.yy0.z; yygotominor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} - break; - case 54: /* ccons ::= CONSTRAINT nm */ - case 92: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==92); + case 29: /* ccons ::= CONSTRAINT nm */ + case 62: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==62); {pParse->constraintName = yymsp[0].minor.yy0;} break; - case 55: /* ccons ::= DEFAULT term */ - case 57: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==57); -{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy346);} + case 30: /* ccons ::= DEFAULT term */ + case 32: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==32); +{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy190);} break; - case 56: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy346);} + case 31: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy190);} break; - case 58: /* ccons ::= DEFAULT MINUS term */ + case 33: /* ccons ::= DEFAULT MINUS term */ { ExprSpan v; - v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy346.pExpr, 0, 0); + v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy190.pExpr, 0, 0); v.zStart = yymsp[-1].minor.yy0.z; - v.zEnd = yymsp[0].minor.yy346.zEnd; + v.zEnd = yymsp[0].minor.yy190.zEnd; sqlite3AddDefaultValue(pParse,&v); } break; - case 59: /* ccons ::= DEFAULT ID|INDEXED */ + case 34: /* ccons ::= DEFAULT ID|INDEXED */ { ExprSpan v; - spanExpr(&v, pParse, TK_STRING, &yymsp[0].minor.yy0); + spanExpr(&v, pParse, TK_STRING, yymsp[0].minor.yy0); sqlite3AddDefaultValue(pParse,&v); } break; - case 61: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy328);} + case 35: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy194);} break; - case 62: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy328,yymsp[0].minor.yy328,yymsp[-2].minor.yy328);} + case 36: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy194,yymsp[0].minor.yy194,yymsp[-2].minor.yy194);} break; - case 63: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy328,0,0,0,0);} + case 37: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy194,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} break; - case 64: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy346.pExpr);} + case 38: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy190.pExpr);} break; - case 65: /* ccons ::= REFERENCES nm idxlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy328);} + case 39: /* ccons ::= REFERENCES nm eidlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy148,yymsp[0].minor.yy194);} break; - case 66: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy328);} + case 40: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy194);} break; - case 67: /* ccons ::= COLLATE ID|STRING */ + case 41: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; - case 70: /* refargs ::= */ -{ yygotominor.yy328 = OE_None*0x0101; /* EV: R-19803-45884 */} + case 44: /* refargs ::= */ +{ yymsp[1].minor.yy194 = OE_None*0x0101; /* EV: R-19803-45884 */} break; - case 71: /* refargs ::= refargs refarg */ -{ yygotominor.yy328 = (yymsp[-1].minor.yy328 & ~yymsp[0].minor.yy429.mask) | yymsp[0].minor.yy429.value; } + case 45: /* refargs ::= refargs refarg */ +{ yymsp[-1].minor.yy194 = (yymsp[-1].minor.yy194 & ~yymsp[0].minor.yy497.mask) | yymsp[0].minor.yy497.value; } break; - case 72: /* refarg ::= MATCH nm */ - case 73: /* refarg ::= ON INSERT refact */ yytestcase(yyruleno==73); -{ yygotominor.yy429.value = 0; yygotominor.yy429.mask = 0x000000; } + case 46: /* refarg ::= MATCH nm */ +{ yymsp[-1].minor.yy497.value = 0; yymsp[-1].minor.yy497.mask = 0x000000; } break; - case 74: /* refarg ::= ON DELETE refact */ -{ yygotominor.yy429.value = yymsp[0].minor.yy328; yygotominor.yy429.mask = 0x0000ff; } + case 47: /* refarg ::= ON INSERT refact */ +{ yymsp[-2].minor.yy497.value = 0; yymsp[-2].minor.yy497.mask = 0x000000; } break; - case 75: /* refarg ::= ON UPDATE refact */ -{ yygotominor.yy429.value = yymsp[0].minor.yy328<<8; yygotominor.yy429.mask = 0x00ff00; } + case 48: /* refarg ::= ON DELETE refact */ +{ yymsp[-2].minor.yy497.value = yymsp[0].minor.yy194; yymsp[-2].minor.yy497.mask = 0x0000ff; } break; - case 76: /* refact ::= SET NULL */ -{ yygotominor.yy328 = OE_SetNull; /* EV: R-33326-45252 */} + case 49: /* refarg ::= ON UPDATE refact */ +{ yymsp[-2].minor.yy497.value = yymsp[0].minor.yy194<<8; yymsp[-2].minor.yy497.mask = 0x00ff00; } break; - case 77: /* refact ::= SET DEFAULT */ -{ yygotominor.yy328 = OE_SetDflt; /* EV: R-33326-45252 */} + case 50: /* refact ::= SET NULL */ +{ yymsp[-1].minor.yy194 = OE_SetNull; /* EV: R-33326-45252 */} break; - case 78: /* refact ::= CASCADE */ -{ yygotominor.yy328 = OE_Cascade; /* EV: R-33326-45252 */} + case 51: /* refact ::= SET DEFAULT */ +{ yymsp[-1].minor.yy194 = OE_SetDflt; /* EV: R-33326-45252 */} break; - case 79: /* refact ::= RESTRICT */ -{ yygotominor.yy328 = OE_Restrict; /* EV: R-33326-45252 */} + case 52: /* refact ::= CASCADE */ +{ yymsp[0].minor.yy194 = OE_Cascade; /* EV: R-33326-45252 */} break; - case 80: /* refact ::= NO ACTION */ -{ yygotominor.yy328 = OE_None; /* EV: R-33326-45252 */} + case 53: /* refact ::= RESTRICT */ +{ yymsp[0].minor.yy194 = OE_Restrict; /* EV: R-33326-45252 */} break; - case 82: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - case 98: /* defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==98); - case 100: /* onconf ::= ON CONFLICT resolvetype */ yytestcase(yyruleno==100); - case 103: /* resolvetype ::= raisetype */ yytestcase(yyruleno==103); -{yygotominor.yy328 = yymsp[0].minor.yy328;} + case 54: /* refact ::= NO ACTION */ +{ yymsp[-1].minor.yy194 = OE_None; /* EV: R-33326-45252 */} break; - case 86: /* conslist_opt ::= */ -{yygotominor.yy0.n = 0; yygotominor.yy0.z = 0;} + case 55: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ +{yymsp[-2].minor.yy194 = 0;} break; - case 87: /* conslist_opt ::= COMMA conslist */ -{yygotominor.yy0 = yymsp[-1].minor.yy0;} + case 56: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 71: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==71); + case 144: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==144); +{yymsp[-1].minor.yy194 = yymsp[0].minor.yy194;} break; - case 90: /* tconscomma ::= COMMA */ + case 58: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ + case 75: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==75); + case 187: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==187); + case 190: /* in_op ::= NOT IN */ yytestcase(yyruleno==190); + case 216: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==216); +{yymsp[-1].minor.yy194 = 1;} + break; + case 59: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ +{yymsp[-1].minor.yy194 = 0;} + break; + case 61: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; - case 93: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy328,yymsp[-2].minor.yy328,0);} + case 63: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy148,yymsp[0].minor.yy194,yymsp[-2].minor.yy194,0);} break; - case 94: /* tcons ::= UNIQUE LP idxlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy328,0,0,0,0);} + case 64: /* tcons ::= UNIQUE LP sortlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy148,yymsp[0].minor.yy194,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} break; - case 95: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy346.pExpr);} + case 65: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy190.pExpr);} break; - case 96: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ + case 66: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy328); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy328); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy148, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy148, yymsp[-1].minor.yy194); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy194); } break; - case 99: /* onconf ::= */ -{yygotominor.yy328 = OE_Default;} + case 68: /* onconf ::= */ + case 70: /* orconf ::= */ yytestcase(yyruleno==70); +{yymsp[1].minor.yy194 = OE_Default;} break; - case 101: /* orconf ::= */ -{yygotominor.yy186 = OE_Default;} + case 69: /* onconf ::= ON CONFLICT resolvetype */ +{yymsp[-2].minor.yy194 = yymsp[0].minor.yy194;} break; - case 102: /* orconf ::= OR resolvetype */ -{yygotominor.yy186 = (u8)yymsp[0].minor.yy328;} + case 72: /* resolvetype ::= IGNORE */ +{yymsp[0].minor.yy194 = OE_Ignore;} break; - case 104: /* resolvetype ::= IGNORE */ -{yygotominor.yy328 = OE_Ignore;} + case 73: /* resolvetype ::= REPLACE */ + case 145: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==145); +{yymsp[0].minor.yy194 = OE_Replace;} break; - case 105: /* resolvetype ::= REPLACE */ -{yygotominor.yy328 = OE_Replace;} - break; - case 106: /* cmd ::= DROP TABLE ifexists fullname */ + case 74: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy65, 0, yymsp[-1].minor.yy328); + sqlite3DropTable(pParse, yymsp[0].minor.yy185, 0, yymsp[-1].minor.yy194); } break; - case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ + case 77: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy3, yymsp[-6].minor.yy328, yymsp[-4].minor.yy328); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy148, yymsp[0].minor.yy243, yymsp[-7].minor.yy194, yymsp[-5].minor.yy194); } break; - case 110: /* cmd ::= DROP VIEW ifexists fullname */ + case 78: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy65, 1, yymsp[-1].minor.yy328); + sqlite3DropTable(pParse, yymsp[0].minor.yy185, 1, yymsp[-1].minor.yy194); } break; - case 111: /* cmd ::= select */ + case 79: /* cmd ::= select */ { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy3, &dest); - sqlite3ExplainBegin(pParse->pVdbe); - sqlite3ExplainSelect(pParse->pVdbe, yymsp[0].minor.yy3); - sqlite3ExplainFinish(pParse->pVdbe); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); + sqlite3Select(pParse, yymsp[0].minor.yy243, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy243); } break; - case 112: /* select ::= with selectnowith */ + case 80: /* select ::= with selectnowith */ { - Select *p = yymsp[0].minor.yy3, *pNext, *pLoop; + Select *p = yymsp[0].minor.yy243; if( p ){ - int cnt = 0, mxSelect; - p->pWith = yymsp[-1].minor.yy59; - if( p->pPrior ){ - pNext = 0; - for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ - pLoop->pNext = pNext; - pLoop->selFlags |= SF_Compound; - } - mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; - if( mxSelect && cnt>mxSelect ){ - sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - } - } + p->pWith = yymsp[-1].minor.yy285; + parserDoubleLinkSelect(pParse, p); }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy59); + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy285); } - yygotominor.yy3 = p; + yymsp[-1].minor.yy243 = p; /*A-overwrites-W*/ } break; - case 113: /* selectnowith ::= oneselect */ - case 119: /* oneselect ::= values */ yytestcase(yyruleno==119); -{yygotominor.yy3 = yymsp[0].minor.yy3;} - break; - case 114: /* selectnowith ::= selectnowith multiselect_op oneselect */ + case 81: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy3; + Select *pRhs = yymsp[0].minor.yy243; + Select *pLhs = yymsp[-2].minor.yy243; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; x.n = 0; + parserDoubleLinkSelect(pParse, pRhs); pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy328; - pRhs->pPrior = yymsp[-2].minor.yy3; - if( yymsp[-1].minor.yy328!=TK_ALL ) pParse->hasCompound = 1; + pRhs->op = (u8)yymsp[-1].minor.yy194; + pRhs->pPrior = pLhs; + if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; + pRhs->selFlags &= ~SF_MultiValue; + if( yymsp[-1].minor.yy194!=TK_ALL ) pParse->hasCompound = 1; }else{ - sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy3); + sqlite3SelectDelete(pParse->db, pLhs); } - yygotominor.yy3 = pRhs; + yymsp[-2].minor.yy243 = pRhs; } break; - case 116: /* multiselect_op ::= UNION ALL */ -{yygotominor.yy328 = TK_ALL;} + case 82: /* multiselect_op ::= UNION */ + case 84: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==84); +{yymsp[0].minor.yy194 = yymsp[0].major; /*A-overwrites-OP*/} break; - case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + case 83: /* multiselect_op ::= UNION ALL */ +{yymsp[-1].minor.yy194 = TK_ALL;} + break; + case 85: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy65,yymsp[-4].minor.yy132,yymsp[-3].minor.yy14,yymsp[-2].minor.yy132,yymsp[-1].minor.yy14,yymsp[-7].minor.yy381,yymsp[0].minor.yy476.pLimit,yymsp[0].minor.yy476.pOffset); +#if SELECTTRACE_ENABLED + Token s = yymsp[-8].minor.yy0; /*A-overwrites-S*/ +#endif + yymsp[-8].minor.yy243 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy148,yymsp[-5].minor.yy185,yymsp[-4].minor.yy72,yymsp[-3].minor.yy148,yymsp[-2].minor.yy72,yymsp[-1].minor.yy148,yymsp[-7].minor.yy194,yymsp[0].minor.yy354.pLimit,yymsp[0].minor.yy354.pOffset); +#if SELECTTRACE_ENABLED + /* Populate the Select.zSelName[] string that is used to help with + ** query planner debugging, to differentiate between multiple Select + ** objects in a complex query. + ** + ** If the SELECT keyword is immediately followed by a C-style comment + ** then extract the first few alphanumeric characters from within that + ** comment to be the zSelName value. Otherwise, the label is #N where + ** is an integer that is incremented with each SELECT statement seen. + */ + if( yymsp[-8].minor.yy243!=0 ){ + const char *z = s.z+6; + int i; + sqlite3_snprintf(sizeof(yymsp[-8].minor.yy243->zSelName), yymsp[-8].minor.yy243->zSelName, "#%d", + ++pParse->nSelect); + while( z[0]==' ' ) z++; + if( z[0]=='/' && z[1]=='*' ){ + z += 2; + while( z[0]==' ' ) z++; + for(i=0; sqlite3Isalnum(z[i]); i++){} + sqlite3_snprintf(sizeof(yymsp[-8].minor.yy243->zSelName), yymsp[-8].minor.yy243->zSelName, "%.*s", i, z); + } + } +#endif /* SELECTRACE_ENABLED */ } break; - case 120: /* values ::= VALUES LP nexprlist RP */ + case 86: /* values ::= VALUES LP nexprlist RP */ { - yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0); + yymsp[-3].minor.yy243 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy148,0,0,0,0,0,SF_Values,0,0); } break; - case 121: /* values ::= values COMMA LP exprlist RP */ + case 87: /* values ::= values COMMA LP exprlist RP */ { - Select *pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0); + Select *pRight, *pLeft = yymsp[-4].minor.yy243; + pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy148,0,0,0,0,0,SF_Values|SF_MultiValue,0,0); + if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; if( pRight ){ pRight->op = TK_ALL; - pRight->pPrior = yymsp[-4].minor.yy3; - yygotominor.yy3 = pRight; + pRight->pPrior = pLeft; + yymsp[-4].minor.yy243 = pRight; }else{ - yygotominor.yy3 = yymsp[-4].minor.yy3; + yymsp[-4].minor.yy243 = pLeft; } } break; - case 122: /* distinct ::= DISTINCT */ -{yygotominor.yy381 = SF_Distinct;} + case 88: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy194 = SF_Distinct;} break; - case 123: /* distinct ::= ALL */ - case 124: /* distinct ::= */ yytestcase(yyruleno==124); -{yygotominor.yy381 = 0;} + case 89: /* distinct ::= ALL */ +{yymsp[0].minor.yy194 = SF_All;} break; - case 125: /* sclp ::= selcollist COMMA */ - case 243: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==243); -{yygotominor.yy14 = yymsp[-1].minor.yy14;} + case 91: /* sclp ::= */ + case 119: /* orderby_opt ::= */ yytestcase(yyruleno==119); + case 126: /* groupby_opt ::= */ yytestcase(yyruleno==126); + case 203: /* exprlist ::= */ yytestcase(yyruleno==203); + case 206: /* paren_exprlist ::= */ yytestcase(yyruleno==206); + case 211: /* eidlist_opt ::= */ yytestcase(yyruleno==211); +{yymsp[1].minor.yy148 = 0;} break; - case 126: /* sclp ::= */ - case 154: /* orderby_opt ::= */ yytestcase(yyruleno==154); - case 161: /* groupby_opt ::= */ yytestcase(yyruleno==161); - case 236: /* exprlist ::= */ yytestcase(yyruleno==236); - case 242: /* idxlist_opt ::= */ yytestcase(yyruleno==242); -{yygotominor.yy14 = 0;} - break; - case 127: /* selcollist ::= sclp expr as */ + case 92: /* selcollist ::= sclp expr as */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, yymsp[-1].minor.yy346.pExpr); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yygotominor.yy14,&yymsp[-1].minor.yy346); + yymsp[-2].minor.yy148 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy148, yymsp[-1].minor.yy190.pExpr); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-2].minor.yy148, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-2].minor.yy148,&yymsp[-1].minor.yy190); } break; - case 128: /* selcollist ::= sclp STAR */ + case 93: /* selcollist ::= sclp STAR */ { - Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0); - yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy14, p); + Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); + yymsp[-1].minor.yy148 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy148, p); } break; - case 129: /* selcollist ::= sclp nm DOT STAR */ + case 94: /* selcollist ::= sclp nm DOT STAR */ { - Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0); + Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0, 0); Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, pDot); + yymsp[-3].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy148, pDot); } break; - case 132: /* as ::= */ -{yygotominor.yy0.n = 0;} + case 95: /* as ::= AS nm */ + case 106: /* dbnm ::= DOT nm */ yytestcase(yyruleno==106); + case 225: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==225); + case 226: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==226); +{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; - case 133: /* from ::= */ -{yygotominor.yy65 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy65));} + case 97: /* from ::= */ +{yymsp[1].minor.yy185 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy185));} break; - case 134: /* from ::= FROM seltablist */ + case 98: /* from ::= FROM seltablist */ { - yygotominor.yy65 = yymsp[0].minor.yy65; - sqlite3SrcListShiftJoinType(yygotominor.yy65); + yymsp[-1].minor.yy185 = yymsp[0].minor.yy185; + sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy185); } break; - case 135: /* stl_prefix ::= seltablist joinop */ + case 99: /* stl_prefix ::= seltablist joinop */ { - yygotominor.yy65 = yymsp[-1].minor.yy65; - if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].jointype = (u8)yymsp[0].minor.yy328; + if( ALWAYS(yymsp[-1].minor.yy185 && yymsp[-1].minor.yy185->nSrc>0) ) yymsp[-1].minor.yy185->a[yymsp[-1].minor.yy185->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy194; } break; - case 136: /* stl_prefix ::= */ -{yygotominor.yy65 = 0;} + case 100: /* stl_prefix ::= */ +{yymsp[1].minor.yy185 = 0;} break; - case 137: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + case 101: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ { - yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); - sqlite3SrcListIndexedBy(pParse, yygotominor.yy65, &yymsp[-2].minor.yy0); + yymsp[-6].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy185,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); + sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy185, &yymsp[-2].minor.yy0); } break; - case 138: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + case 102: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ { - yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy3,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + yymsp[-8].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy185,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); + sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy185, yymsp[-4].minor.yy148); +} + break; + case 103: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ +{ + yymsp[-6].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy185,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy243,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); } break; - case 139: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + case 104: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ { - if( yymsp[-6].minor.yy65==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy132==0 && yymsp[0].minor.yy408==0 ){ - yygotominor.yy65 = yymsp[-4].minor.yy65; - }else if( yymsp[-4].minor.yy65->nSrc==1 ){ - yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); - if( yygotominor.yy65 ){ - struct SrcList_item *pNew = &yygotominor.yy65->a[yygotominor.yy65->nSrc-1]; - struct SrcList_item *pOld = yymsp[-4].minor.yy65->a; + if( yymsp[-6].minor.yy185==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy72==0 && yymsp[0].minor.yy254==0 ){ + yymsp[-6].minor.yy185 = yymsp[-4].minor.yy185; + }else if( yymsp[-4].minor.yy185->nSrc==1 ){ + yymsp[-6].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy185,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); + if( yymsp[-6].minor.yy185 ){ + struct SrcList_item *pNew = &yymsp[-6].minor.yy185->a[yymsp[-6].minor.yy185->nSrc-1]; + struct SrcList_item *pOld = yymsp[-4].minor.yy185->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy65); + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy185); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy65); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy65,0,0,0,0,SF_NestedFrom,0,0); - yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy185); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy185,0,0,0,0,SF_NestedFrom,0,0); + yymsp[-6].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy185,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); } } break; - case 140: /* dbnm ::= */ - case 149: /* indexed_opt ::= */ yytestcase(yyruleno==149); -{yygotominor.yy0.z=0; yygotominor.yy0.n=0;} + case 105: /* dbnm ::= */ + case 114: /* indexed_opt ::= */ yytestcase(yyruleno==114); +{yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; - case 142: /* fullname ::= nm dbnm */ -{yygotominor.yy65 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} + case 107: /* fullname ::= nm dbnm */ +{yymsp[-1].minor.yy185 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 143: /* joinop ::= COMMA|JOIN */ -{ yygotominor.yy328 = JT_INNER; } + case 108: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy194 = JT_INNER; } break; - case 144: /* joinop ::= JOIN_KW JOIN */ -{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } + case 109: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy194 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} break; - case 145: /* joinop ::= JOIN_KW nm JOIN */ -{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } + case 110: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy194 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} break; - case 146: /* joinop ::= JOIN_KW nm nm JOIN */ -{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } + case 111: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy194 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} break; - case 147: /* on_opt ::= ON expr */ - case 164: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==164); - case 171: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==171); - case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); - case 233: /* case_operand ::= expr */ yytestcase(yyruleno==233); -{yygotominor.yy132 = yymsp[0].minor.yy346.pExpr;} + case 112: /* on_opt ::= ON expr */ + case 129: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==129); + case 136: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==136); + case 199: /* case_else ::= ELSE expr */ yytestcase(yyruleno==199); +{yymsp[-1].minor.yy72 = yymsp[0].minor.yy190.pExpr;} break; - case 148: /* on_opt ::= */ - case 163: /* having_opt ::= */ yytestcase(yyruleno==163); - case 170: /* where_opt ::= */ yytestcase(yyruleno==170); - case 232: /* case_else ::= */ yytestcase(yyruleno==232); - case 234: /* case_operand ::= */ yytestcase(yyruleno==234); -{yygotominor.yy132 = 0;} + case 113: /* on_opt ::= */ + case 128: /* having_opt ::= */ yytestcase(yyruleno==128); + case 135: /* where_opt ::= */ yytestcase(yyruleno==135); + case 200: /* case_else ::= */ yytestcase(yyruleno==200); + case 202: /* case_operand ::= */ yytestcase(yyruleno==202); +{yymsp[1].minor.yy72 = 0;} break; - case 151: /* indexed_opt ::= NOT INDEXED */ -{yygotominor.yy0.z=0; yygotominor.yy0.n=1;} + case 115: /* indexed_opt ::= INDEXED BY nm */ +{yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} break; - case 152: /* using_opt ::= USING LP idlist RP */ - case 180: /* inscollist_opt ::= LP idlist RP */ yytestcase(yyruleno==180); -{yygotominor.yy408 = yymsp[-1].minor.yy408;} + case 116: /* indexed_opt ::= NOT INDEXED */ +{yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} break; - case 153: /* using_opt ::= */ - case 179: /* inscollist_opt ::= */ yytestcase(yyruleno==179); -{yygotominor.yy408 = 0;} + case 117: /* using_opt ::= USING LP idlist RP */ +{yymsp[-3].minor.yy254 = yymsp[-1].minor.yy254;} break; - case 155: /* orderby_opt ::= ORDER BY sortlist */ - case 162: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==162); - case 235: /* exprlist ::= nexprlist */ yytestcase(yyruleno==235); -{yygotominor.yy14 = yymsp[0].minor.yy14;} + case 118: /* using_opt ::= */ + case 146: /* idlist_opt ::= */ yytestcase(yyruleno==146); +{yymsp[1].minor.yy254 = 0;} break; - case 156: /* sortlist ::= sortlist COMMA expr sortorder */ + case 120: /* orderby_opt ::= ORDER BY sortlist */ + case 127: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==127); +{yymsp[-2].minor.yy148 = yymsp[0].minor.yy148;} + break; + case 121: /* sortlist ::= sortlist COMMA expr sortorder */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14,yymsp[-1].minor.yy346.pExpr); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yymsp[-3].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy148,yymsp[-1].minor.yy190.pExpr); + sqlite3ExprListSetSortOrder(yymsp[-3].minor.yy148,yymsp[0].minor.yy194); } break; - case 157: /* sortlist ::= expr sortorder */ + case 122: /* sortlist ::= expr sortorder */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy346.pExpr); - if( yygotominor.yy14 && ALWAYS(yygotominor.yy14->a) ) yygotominor.yy14->a[0].sortOrder = (u8)yymsp[0].minor.yy328; + yymsp[-1].minor.yy148 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy190.pExpr); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-1].minor.yy148,yymsp[0].minor.yy194); } break; - case 158: /* sortorder ::= ASC */ - case 160: /* sortorder ::= */ yytestcase(yyruleno==160); -{yygotominor.yy328 = SQLITE_SO_ASC;} + case 123: /* sortorder ::= ASC */ +{yymsp[0].minor.yy194 = SQLITE_SO_ASC;} break; - case 159: /* sortorder ::= DESC */ -{yygotominor.yy328 = SQLITE_SO_DESC;} + case 124: /* sortorder ::= DESC */ +{yymsp[0].minor.yy194 = SQLITE_SO_DESC;} break; - case 165: /* limit_opt ::= */ -{yygotominor.yy476.pLimit = 0; yygotominor.yy476.pOffset = 0;} + case 125: /* sortorder ::= */ +{yymsp[1].minor.yy194 = SQLITE_SO_UNDEFINED;} break; - case 166: /* limit_opt ::= LIMIT expr */ -{yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr; yygotominor.yy476.pOffset = 0;} + case 130: /* limit_opt ::= */ +{yymsp[1].minor.yy354.pLimit = 0; yymsp[1].minor.yy354.pOffset = 0;} break; - case 167: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yygotominor.yy476.pLimit = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pOffset = yymsp[0].minor.yy346.pExpr;} + case 131: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy354.pLimit = yymsp[0].minor.yy190.pExpr; yymsp[-1].minor.yy354.pOffset = 0;} break; - case 168: /* limit_opt ::= LIMIT expr COMMA expr */ -{yygotominor.yy476.pOffset = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr;} + case 132: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy354.pLimit = yymsp[-2].minor.yy190.pExpr; yymsp[-3].minor.yy354.pOffset = yymsp[0].minor.yy190.pExpr;} break; - case 169: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ + case 133: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy354.pOffset = yymsp[-2].minor.yy190.pExpr; yymsp[-3].minor.yy354.pLimit = yymsp[0].minor.yy190.pExpr;} + break; + case 134: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ { - sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy65, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy65,yymsp[0].minor.yy132); + sqlite3WithPush(pParse, yymsp[-5].minor.yy285, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy185, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy185,yymsp[0].minor.yy72); } break; - case 172: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ + case 137: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ { - sqlite3WithPush(pParse, yymsp[-7].minor.yy59, 1); - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy65, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy14,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy65,yymsp[-1].minor.yy14,yymsp[0].minor.yy132,yymsp[-5].minor.yy186); + sqlite3WithPush(pParse, yymsp[-7].minor.yy285, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy185, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy148,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy185,yymsp[-1].minor.yy148,yymsp[0].minor.yy72,yymsp[-5].minor.yy194); } break; - case 173: /* setlist ::= setlist COMMA nm EQ expr */ + case 138: /* setlist ::= setlist COMMA nm EQ expr */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy346.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy148 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy148, yymsp[0].minor.yy190.pExpr); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy148, &yymsp[-2].minor.yy0, 1); } break; - case 174: /* setlist ::= nm EQ expr */ + case 139: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy346.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); + yymsp[-6].minor.yy148 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy148, yymsp[-3].minor.yy254, yymsp[0].minor.yy190.pExpr); } break; - case 175: /* cmd ::= with insert_cmd INTO fullname inscollist_opt select */ + case 140: /* setlist ::= nm EQ expr */ { - sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); - sqlite3Insert(pParse, yymsp[-2].minor.yy65, yymsp[0].minor.yy3, yymsp[-1].minor.yy408, yymsp[-4].minor.yy186); + yylhsminor.yy148 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy190.pExpr); + sqlite3ExprListSetName(pParse, yylhsminor.yy148, &yymsp[-2].minor.yy0, 1); +} + yymsp[-2].minor.yy148 = yylhsminor.yy148; + break; + case 141: /* setlist ::= LP idlist RP EQ expr */ +{ + yymsp[-4].minor.yy148 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy254, yymsp[0].minor.yy190.pExpr); } break; - case 176: /* cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ + case 142: /* cmd ::= with insert_cmd INTO fullname idlist_opt select */ { - sqlite3WithPush(pParse, yymsp[-6].minor.yy59, 1); - sqlite3Insert(pParse, yymsp[-3].minor.yy65, 0, yymsp[-2].minor.yy408, yymsp[-5].minor.yy186); + sqlite3WithPush(pParse, yymsp[-5].minor.yy285, 1); + sqlite3Insert(pParse, yymsp[-2].minor.yy185, yymsp[0].minor.yy243, yymsp[-1].minor.yy254, yymsp[-4].minor.yy194); } break; - case 177: /* insert_cmd ::= INSERT orconf */ -{yygotominor.yy186 = yymsp[0].minor.yy186;} - break; - case 178: /* insert_cmd ::= REPLACE */ -{yygotominor.yy186 = OE_Replace;} - break; - case 181: /* idlist ::= idlist COMMA nm */ -{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy408,&yymsp[0].minor.yy0);} - break; - case 182: /* idlist ::= nm */ -{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} - break; - case 183: /* expr ::= term */ -{yygotominor.yy346 = yymsp[0].minor.yy346;} - break; - case 184: /* expr ::= LP expr RP */ -{yygotominor.yy346.pExpr = yymsp[-1].minor.yy346.pExpr; spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} - break; - case 185: /* term ::= NULL */ - case 190: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==190); - case 191: /* term ::= STRING */ yytestcase(yyruleno==191); -{spanExpr(&yygotominor.yy346, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} - break; - case 186: /* expr ::= ID|INDEXED */ - case 187: /* expr ::= JOIN_KW */ yytestcase(yyruleno==187); -{spanExpr(&yygotominor.yy346, pParse, TK_ID, &yymsp[0].minor.yy0);} - break; - case 188: /* expr ::= nm DOT nm */ + case 143: /* cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES */ { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); - spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + sqlite3WithPush(pParse, yymsp[-6].minor.yy285, 1); + sqlite3Insert(pParse, yymsp[-3].minor.yy185, 0, yymsp[-2].minor.yy254, yymsp[-5].minor.yy194); } break; - case 189: /* expr ::= nm DOT nm DOT nm */ + case 147: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy254 = yymsp[-1].minor.yy254;} + break; + case 148: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy254 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy254,&yymsp[0].minor.yy0);} + break; + case 149: /* idlist ::= nm */ +{yymsp[0].minor.yy254 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + break; + case 150: /* expr ::= LP expr RP */ +{spanSet(&yymsp[-2].minor.yy190,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ yymsp[-2].minor.yy190.pExpr = yymsp[-1].minor.yy190.pExpr;} + break; + case 151: /* term ::= NULL */ + case 156: /* term ::= FLOAT|BLOB */ yytestcase(yyruleno==156); + case 157: /* term ::= STRING */ yytestcase(yyruleno==157); +{spanExpr(&yymsp[0].minor.yy190,pParse,yymsp[0].major,yymsp[0].minor.yy0);/*A-overwrites-X*/} + break; + case 152: /* expr ::= ID|INDEXED */ + case 153: /* expr ::= JOIN_KW */ yytestcase(yyruleno==153); +{spanExpr(&yymsp[0].minor.yy190,pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 154: /* expr ::= nm DOT nm */ { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); - Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); + spanSet(&yymsp[-2].minor.yy190,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-2].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); +} + break; + case 155: /* expr ::= nm DOT nm DOT nm */ +{ + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-4].minor.yy0, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); - spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + spanSet(&yymsp[-4].minor.yy190,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); } break; - case 192: /* expr ::= VARIABLE */ + case 158: /* term ::= INTEGER */ { - if( yymsp[0].minor.yy0.n>=2 && yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1]) ){ + yylhsminor.yy190.pExpr = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + yylhsminor.yy190.zStart = yymsp[0].minor.yy0.z; + yylhsminor.yy190.zEnd = yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n; + if( yylhsminor.yy190.pExpr ) yylhsminor.yy190.pExpr->flags |= EP_Leaf; +} + yymsp[0].minor.yy190 = yylhsminor.yy190; + break; + case 159: /* expr ::= VARIABLE */ +{ + if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ + u32 n = yymsp[0].minor.yy0.n; + spanExpr(&yymsp[0].minor.yy190, pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy190.pExpr, n); + }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers ** in the virtual machine. #N is the N-th register. */ + Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/ + assert( t.n>=2 ); + spanSet(&yymsp[0].minor.yy190, &t, &t); if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0); - yygotominor.yy346.pExpr = 0; + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); + yymsp[0].minor.yy190.pExpr = 0; }else{ - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); - if( yygotominor.yy346.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy346.pExpr->iTable); + yymsp[0].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, 0); + if( yymsp[0].minor.yy190.pExpr ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy190.pExpr->iTable); } - }else{ - spanExpr(&yygotominor.yy346, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yygotominor.yy346.pExpr); } - spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; - case 193: /* expr ::= expr COLLATE ID|STRING */ + case 160: /* expr ::= expr COLLATE ID|STRING */ { - yygotominor.yy346.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy346.pExpr, &yymsp[0].minor.yy0); - yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yymsp[-2].minor.yy190.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy190.pExpr, &yymsp[0].minor.yy0, 1); + yymsp[-2].minor.yy190.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 194: /* expr ::= CAST LP expr AS typetoken RP */ + case 161: /* expr ::= CAST LP expr AS typetoken RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy346.pExpr, 0, &yymsp[-1].minor.yy0); - spanSet(&yygotominor.yy346,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); + spanSet(&yymsp[-5].minor.yy190,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-5].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy190.pExpr, 0, &yymsp[-1].minor.yy0); } break; - case 195: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 162: /* expr ::= ID|INDEXED LP distinct exprlist RP */ { - if( yymsp[-1].minor.yy14 && yymsp[-1].minor.yy14->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + if( yymsp[-1].minor.yy148 && yymsp[-1].minor.yy148->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); } - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); - spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy381 && yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->flags |= EP_Distinct; + yylhsminor.yy190.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy148, &yymsp[-4].minor.yy0); + spanSet(&yylhsminor.yy190,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + if( yymsp[-2].minor.yy194==SF_Distinct && yylhsminor.yy190.pExpr ){ + yylhsminor.yy190.pExpr->flags |= EP_Distinct; } } + yymsp[-4].minor.yy190 = yylhsminor.yy190; break; - case 196: /* expr ::= ID|INDEXED LP STAR RP */ + case 163: /* expr ::= ID|INDEXED LP STAR RP */ { - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); - spanSet(&yygotominor.yy346,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); + yylhsminor.yy190.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); + spanSet(&yylhsminor.yy190,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); } + yymsp[-3].minor.yy190 = yylhsminor.yy190; break; - case 197: /* term ::= CTIME_KW */ + case 164: /* term ::= CTIME_KW */ { - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); - spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); + yylhsminor.yy190.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); + spanSet(&yylhsminor.yy190, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy190 = yylhsminor.yy190; break; - case 198: /* expr ::= expr AND expr */ - case 199: /* expr ::= expr OR expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==201); - case 202: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==202); - case 203: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==203); - case 204: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==204); - case 205: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==205); -{spanBinaryExpr(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);} + case 165: /* expr ::= LP nexprlist COMMA expr RP */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy148, yymsp[-1].minor.yy190.pExpr); + yylhsminor.yy190.pExpr = sqlite3PExpr(pParse, TK_VECTOR, 0, 0, 0); + if( yylhsminor.yy190.pExpr ){ + yylhsminor.yy190.pExpr->x.pList = pList; + spanSet(&yylhsminor.yy190, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0); + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + yymsp[-4].minor.yy190 = yylhsminor.yy190; break; - case 206: /* likeop ::= LIKE_KW|MATCH */ -{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 0;} + case 166: /* expr ::= expr AND expr */ + case 167: /* expr ::= expr OR expr */ yytestcase(yyruleno==167); + case 168: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==168); + case 169: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==169); + case 170: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==170); + case 171: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==171); + case 172: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==172); + case 173: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==173); +{spanBinaryExpr(pParse,yymsp[-1].major,&yymsp[-2].minor.yy190,&yymsp[0].minor.yy190);} break; - case 207: /* likeop ::= NOT LIKE_KW|MATCH */ -{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 1;} + case 174: /* likeop ::= LIKE_KW|MATCH */ +{yymsp[0].minor.yy0=yymsp[0].minor.yy0;/*A-overwrites-X*/} break; - case 208: /* expr ::= expr likeop expr */ + case 175: /* likeop ::= NOT LIKE_KW|MATCH */ +{yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} + break; + case 176: /* expr ::= expr likeop expr */ { ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy346.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy346.pExpr); - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy96.eOperator); - if( yymsp[-1].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; - yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; - if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; + int bNot = yymsp[-1].minor.yy0.n & 0x80000000; + yymsp[-1].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy190.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy190.pExpr); + yymsp[-2].minor.yy190.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0); + exprNot(pParse, bNot, &yymsp[-2].minor.yy190); + yymsp[-2].minor.yy190.zEnd = yymsp[0].minor.yy190.zEnd; + if( yymsp[-2].minor.yy190.pExpr ) yymsp[-2].minor.yy190.pExpr->flags |= EP_InfixFunc; } break; - case 209: /* expr ::= expr likeop expr ESCAPE expr */ + case 177: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy346.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy96.eOperator); - if( yymsp[-3].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; - yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; - if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; + int bNot = yymsp[-3].minor.yy0.n & 0x80000000; + yymsp[-3].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy190.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy190.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy190.pExpr); + yymsp[-4].minor.yy190.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0); + exprNot(pParse, bNot, &yymsp[-4].minor.yy190); + yymsp[-4].minor.yy190.zEnd = yymsp[0].minor.yy190.zEnd; + if( yymsp[-4].minor.yy190.pExpr ) yymsp[-4].minor.yy190.pExpr->flags |= EP_InfixFunc; } break; - case 210: /* expr ::= expr ISNULL|NOTNULL */ -{spanUnaryPostfix(&yygotominor.yy346,pParse,yymsp[0].major,&yymsp[-1].minor.yy346,&yymsp[0].minor.yy0);} + case 178: /* expr ::= expr ISNULL|NOTNULL */ +{spanUnaryPostfix(pParse,yymsp[0].major,&yymsp[-1].minor.yy190,&yymsp[0].minor.yy0);} break; - case 211: /* expr ::= expr NOT NULL */ -{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);} + case 179: /* expr ::= expr NOT NULL */ +{spanUnaryPostfix(pParse,TK_NOTNULL,&yymsp[-2].minor.yy190,&yymsp[0].minor.yy0);} break; - case 212: /* expr ::= expr IS expr */ + case 180: /* expr ::= expr IS expr */ { - spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL); + spanBinaryExpr(pParse,TK_IS,&yymsp[-2].minor.yy190,&yymsp[0].minor.yy190); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy190.pExpr, yymsp[-2].minor.yy190.pExpr, TK_ISNULL); } break; - case 213: /* expr ::= expr IS NOT expr */ + case 181: /* expr ::= expr IS NOT expr */ { - spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL); + spanBinaryExpr(pParse,TK_ISNOT,&yymsp[-3].minor.yy190,&yymsp[0].minor.yy190); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy190.pExpr, yymsp[-3].minor.yy190.pExpr, TK_NOTNULL); } break; - case 214: /* expr ::= NOT expr */ - case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); -{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + case 182: /* expr ::= NOT expr */ + case 183: /* expr ::= BITNOT expr */ yytestcase(yyruleno==183); +{spanUnaryPrefix(&yymsp[-1].minor.yy190,pParse,yymsp[-1].major,&yymsp[0].minor.yy190,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} break; - case 216: /* expr ::= MINUS expr */ -{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UMINUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + case 184: /* expr ::= MINUS expr */ +{spanUnaryPrefix(&yymsp[-1].minor.yy190,pParse,TK_UMINUS,&yymsp[0].minor.yy190,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} break; - case 217: /* expr ::= PLUS expr */ -{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UPLUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + case 185: /* expr ::= PLUS expr */ +{spanUnaryPrefix(&yymsp[-1].minor.yy190,pParse,TK_UPLUS,&yymsp[0].minor.yy190,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} break; - case 220: /* expr ::= expr between_op expr AND expr */ + case 186: /* between_op ::= BETWEEN */ + case 189: /* in_op ::= IN */ yytestcase(yyruleno==189); +{yymsp[0].minor.yy194 = 0;} + break; + case 188: /* expr ::= expr between_op expr AND expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy346.pExpr, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy190.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy190.pExpr); + yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy190.pExpr, 0, 0); + if( yymsp[-4].minor.yy190.pExpr ){ + yymsp[-4].minor.yy190.pExpr->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; - yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; + exprNot(pParse, yymsp[-3].minor.yy194, &yymsp[-4].minor.yy190); + yymsp[-4].minor.yy190.zEnd = yymsp[0].minor.yy190.zEnd; } break; - case 223: /* expr ::= expr in_op LP exprlist RP */ + case 191: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy14==0 ){ + if( yymsp[-1].minor.yy148==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -123246,9 +139002,9 @@ static void yy_reduce( ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy328]); - sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy346.pExpr); - }else if( yymsp[-1].minor.yy14->nExpr==1 ){ + sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy190.pExpr); + yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy194]); + }else if( yymsp[-1].minor.yy148->nExpr==1 ){ /* Expressions of the form: ** ** expr1 IN (?1) @@ -123265,433 +139021,419 @@ static void yy_reduce( ** affinity or the collating sequence to use for comparison. Otherwise, ** the semantics would be subtly different from IN or NOT IN. */ - Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; - yymsp[-1].minor.yy14->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + Expr *pRHS = yymsp[-1].minor.yy148->a[0].pExpr; + yymsp[-1].minor.yy148->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy148); /* pRHS cannot be NULL because a malloc error would have been detected ** before now and control would have never reached this point */ if( ALWAYS(pRHS) ){ pRHS->flags &= ~EP_Collate; pRHS->flags |= EP_Generic; } - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy328 ? TK_NE : TK_EQ, yymsp[-4].minor.yy346.pExpr, pRHS, 0); + yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy194 ? TK_NE : TK_EQ, yymsp[-4].minor.yy190.pExpr, pRHS, 0); }else{ - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy14; - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy190.pExpr, 0, 0); + if( yymsp[-4].minor.yy190.pExpr ){ + yymsp[-4].minor.yy190.pExpr->x.pList = yymsp[-1].minor.yy148; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy190.pExpr); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy148); } - if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + exprNot(pParse, yymsp[-3].minor.yy194, &yymsp[-4].minor.yy190); } - yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yymsp[-4].minor.yy190.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 224: /* expr ::= LP select RP */ + case 192: /* expr ::= LP select RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; - ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); - } - yygotominor.yy346.zStart = yymsp[-2].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + spanSet(&yymsp[-2].minor.yy190,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ + yymsp[-2].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy190.pExpr, yymsp[-1].minor.yy243); } break; - case 225: /* expr ::= expr in_op LP select RP */ + case 193: /* expr ::= expr in_op LP select RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; - ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); - } - if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy190.pExpr, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy190.pExpr, yymsp[-1].minor.yy243); + exprNot(pParse, yymsp[-3].minor.yy194, &yymsp[-4].minor.yy190); + yymsp[-4].minor.yy190.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 226: /* expr ::= expr in_op nm dbnm */ + case 194: /* expr ::= expr in_op nm dbnm paren_exprlist */ { - SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy346.pExpr, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); - }else{ - sqlite3SrcListDelete(pParse->db, pSrc); - } - if( yymsp[-2].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-3].minor.yy346.zStart; - yygotominor.yy346.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; + SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); + Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + if( yymsp[0].minor.yy148 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy148); + yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy190.pExpr, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy190.pExpr, pSelect); + exprNot(pParse, yymsp[-3].minor.yy194, &yymsp[-4].minor.yy190); + yymsp[-4].minor.yy190.zEnd = yymsp[-1].minor.yy0.z ? &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n] : &yymsp[-2].minor.yy0.z[yymsp[-2].minor.yy0.n]; } break; - case 227: /* expr ::= EXISTS LP select RP */ + case 195: /* expr ::= EXISTS LP select RP */ { - Expr *p = yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); - if( p ){ - p->x.pSelect = yymsp[-1].minor.yy3; - ExprSetProperty(p, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, p); - }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); - } - yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + Expr *p; + spanSet(&yymsp[-3].minor.yy190,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ + p = yymsp[-3].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy243); } break; - case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 196: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy132, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy132 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy132) : yymsp[-2].minor.yy14; - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + spanSet(&yymsp[-4].minor.yy190,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-C*/ + yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy72, 0, 0); + if( yymsp[-4].minor.yy190.pExpr ){ + yymsp[-4].minor.yy190.pExpr->x.pList = yymsp[-1].minor.yy72 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy148,yymsp[-1].minor.yy72) : yymsp[-2].minor.yy148; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy190.pExpr); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy132); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy148); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy72); } - yygotominor.yy346.zStart = yymsp[-4].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 197: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy346.pExpr); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); + yymsp[-4].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy148, yymsp[-2].minor.yy190.pExpr); + yymsp[-4].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy148, yymsp[0].minor.yy190.pExpr); } break; - case 230: /* case_exprlist ::= WHEN expr THEN expr */ + case 198: /* case_exprlist ::= WHEN expr THEN expr */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); + yymsp[-3].minor.yy148 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy190.pExpr); + yymsp[-3].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy148, yymsp[0].minor.yy190.pExpr); } break; - case 237: /* nexprlist ::= nexprlist COMMA expr */ -{yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy346.pExpr);} + case 201: /* case_operand ::= expr */ +{yymsp[0].minor.yy72 = yymsp[0].minor.yy190.pExpr; /*A-overwrites-X*/} break; - case 238: /* nexprlist ::= expr */ -{yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy346.pExpr);} + case 204: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy148,yymsp[0].minor.yy190.pExpr);} break; - case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt */ + case 205: /* nexprlist ::= expr */ +{yymsp[0].minor.yy148 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy190.pExpr); /*A-overwrites-Y*/} + break; + case 207: /* paren_exprlist ::= LP exprlist RP */ + case 212: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==212); +{yymsp[-2].minor.yy148 = yymsp[-1].minor.yy148;} + break; + case 208: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy328, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy132, SQLITE_SO_ASC, yymsp[-8].minor.yy328); + sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy148, yymsp[-10].minor.yy194, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy72, SQLITE_SO_ASC, yymsp[-8].minor.yy194, SQLITE_IDXTYPE_APPDEF); } break; - case 240: /* uniqueflag ::= UNIQUE */ - case 291: /* raisetype ::= ABORT */ yytestcase(yyruleno==291); -{yygotominor.yy328 = OE_Abort;} + case 209: /* uniqueflag ::= UNIQUE */ + case 250: /* raisetype ::= ABORT */ yytestcase(yyruleno==250); +{yymsp[0].minor.yy194 = OE_Abort;} break; - case 241: /* uniqueflag ::= */ -{yygotominor.yy328 = OE_None;} + case 210: /* uniqueflag ::= */ +{yymsp[1].minor.yy194 = OE_None;} break; - case 244: /* idxlist ::= idxlist COMMA nm collate sortorder */ + case 213: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, p); - sqlite3ExprListSetName(pParse,yygotominor.yy14,&yymsp[-2].minor.yy0,1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yymsp[-4].minor.yy148 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy148, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy194, yymsp[0].minor.yy194); } break; - case 245: /* idxlist ::= nm collate sortorder */ + case 214: /* eidlist ::= nm collate sortorder */ { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, p); - sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yymsp[-2].minor.yy148 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy194, yymsp[0].minor.yy194); /*A-overwrites-Y*/ } break; - case 246: /* collate ::= */ -{yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} + case 217: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy185, yymsp[-1].minor.yy194);} break; - case 248: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy65, yymsp[-1].minor.yy328);} + case 218: /* cmd ::= VACUUM */ +{sqlite3Vacuum(pParse,0);} break; - case 249: /* cmd ::= VACUUM */ - case 250: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==250); -{sqlite3Vacuum(pParse);} + case 219: /* cmd ::= VACUUM nm */ +{sqlite3Vacuum(pParse,&yymsp[0].minor.yy0);} break; - case 251: /* cmd ::= PRAGMA nm dbnm */ + case 220: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 221: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 222: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 223: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 224: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 264: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 227: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy473, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy145, &all); } break; - case 265: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 228: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy328, yymsp[-4].minor.yy378.a, yymsp[-4].minor.yy378.b, yymsp[-2].minor.yy65, yymsp[0].minor.yy132, yymsp[-10].minor.yy328, yymsp[-8].minor.yy328); - yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy194, yymsp[-4].minor.yy332.a, yymsp[-4].minor.yy332.b, yymsp[-2].minor.yy185, yymsp[0].minor.yy72, yymsp[-10].minor.yy194, yymsp[-8].minor.yy194); + yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 266: /* trigger_time ::= BEFORE */ - case 269: /* trigger_time ::= */ yytestcase(yyruleno==269); -{ yygotominor.yy328 = TK_BEFORE; } + case 229: /* trigger_time ::= BEFORE */ +{ yymsp[0].minor.yy194 = TK_BEFORE; } break; - case 267: /* trigger_time ::= AFTER */ -{ yygotominor.yy328 = TK_AFTER; } + case 230: /* trigger_time ::= AFTER */ +{ yymsp[0].minor.yy194 = TK_AFTER; } break; - case 268: /* trigger_time ::= INSTEAD OF */ -{ yygotominor.yy328 = TK_INSTEAD;} + case 231: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy194 = TK_INSTEAD;} break; - case 270: /* trigger_event ::= DELETE|INSERT */ - case 271: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==271); -{yygotominor.yy378.a = yymsp[0].major; yygotominor.yy378.b = 0;} + case 232: /* trigger_time ::= */ +{ yymsp[1].minor.yy194 = TK_BEFORE; } break; - case 272: /* trigger_event ::= UPDATE OF idlist */ -{yygotominor.yy378.a = TK_UPDATE; yygotominor.yy378.b = yymsp[0].minor.yy408;} + case 233: /* trigger_event ::= DELETE|INSERT */ + case 234: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==234); +{yymsp[0].minor.yy332.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy332.b = 0;} break; - case 275: /* when_clause ::= */ - case 296: /* key_opt ::= */ yytestcase(yyruleno==296); -{ yygotominor.yy132 = 0; } + case 235: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy332.a = TK_UPDATE; yymsp[-2].minor.yy332.b = yymsp[0].minor.yy254;} break; - case 276: /* when_clause ::= WHEN expr */ - case 297: /* key_opt ::= KEY expr */ yytestcase(yyruleno==297); -{ yygotominor.yy132 = yymsp[0].minor.yy346.pExpr; } + case 236: /* when_clause ::= */ + case 255: /* key_opt ::= */ yytestcase(yyruleno==255); +{ yymsp[1].minor.yy72 = 0; } break; - case 277: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 237: /* when_clause ::= WHEN expr */ + case 256: /* key_opt ::= KEY expr */ yytestcase(yyruleno==256); +{ yymsp[-1].minor.yy72 = yymsp[0].minor.yy190.pExpr; } + break; + case 238: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy473!=0 ); - yymsp[-2].minor.yy473->pLast->pNext = yymsp[-1].minor.yy473; - yymsp[-2].minor.yy473->pLast = yymsp[-1].minor.yy473; - yygotominor.yy473 = yymsp[-2].minor.yy473; + assert( yymsp[-2].minor.yy145!=0 ); + yymsp[-2].minor.yy145->pLast->pNext = yymsp[-1].minor.yy145; + yymsp[-2].minor.yy145->pLast = yymsp[-1].minor.yy145; } break; - case 278: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 239: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy473!=0 ); - yymsp[-1].minor.yy473->pLast = yymsp[-1].minor.yy473; - yygotominor.yy473 = yymsp[-1].minor.yy473; + assert( yymsp[-1].minor.yy145!=0 ); + yymsp[-1].minor.yy145->pLast = yymsp[-1].minor.yy145; } break; - case 280: /* trnm ::= nm DOT nm */ + case 240: /* trnm ::= nm DOT nm */ { - yygotominor.yy0 = yymsp[0].minor.yy0; + yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, "qualified table names are not allowed on INSERT, UPDATE, and DELETE " "statements within triggers"); } break; - case 282: /* tridxby ::= INDEXED BY nm */ + case 241: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 283: /* tridxby ::= NOT INDEXED */ + case 242: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 284: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ -{ yygotominor.yy473 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy14, yymsp[0].minor.yy132, yymsp[-5].minor.yy186); } + case 243: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ +{yymsp[-6].minor.yy145 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy148, yymsp[0].minor.yy72, yymsp[-5].minor.yy194);} break; - case 285: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ -{yygotominor.yy473 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy408, yymsp[0].minor.yy3, yymsp[-4].minor.yy186);} + case 244: /* trigger_cmd ::= insert_cmd INTO trnm idlist_opt select */ +{yymsp[-4].minor.yy145 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy254, yymsp[0].minor.yy243, yymsp[-4].minor.yy194);/*A-overwrites-R*/} break; - case 286: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ -{yygotominor.yy473 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy132);} + case 245: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ +{yymsp[-4].minor.yy145 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy72);} break; - case 287: /* trigger_cmd ::= select */ -{yygotominor.yy473 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy3); } + case 246: /* trigger_cmd ::= select */ +{yymsp[0].minor.yy145 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy243); /*A-overwrites-X*/} break; - case 288: /* expr ::= RAISE LP IGNORE RP */ + case 247: /* expr ::= RAISE LP IGNORE RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->affinity = OE_Ignore; + spanSet(&yymsp[-3].minor.yy190,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-3].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); + if( yymsp[-3].minor.yy190.pExpr ){ + yymsp[-3].minor.yy190.pExpr->affinity = OE_Ignore; } - yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 289: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 248: /* expr ::= RAISE LP raisetype COMMA nm RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); - if( yygotominor.yy346.pExpr ) { - yygotominor.yy346.pExpr->affinity = (char)yymsp[-3].minor.yy328; + spanSet(&yymsp[-5].minor.yy190,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-5].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); + if( yymsp[-5].minor.yy190.pExpr ) { + yymsp[-5].minor.yy190.pExpr->affinity = (char)yymsp[-3].minor.yy194; } - yygotominor.yy346.zStart = yymsp[-5].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 290: /* raisetype ::= ROLLBACK */ -{yygotominor.yy328 = OE_Rollback;} + case 249: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy194 = OE_Rollback;} break; - case 292: /* raisetype ::= FAIL */ -{yygotominor.yy328 = OE_Fail;} + case 251: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy194 = OE_Fail;} break; - case 293: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 252: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy65,yymsp[-1].minor.yy328); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy185,yymsp[-1].minor.yy194); } break; - case 294: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 253: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy346.pExpr, yymsp[-1].minor.yy346.pExpr, yymsp[0].minor.yy132); + sqlite3Attach(pParse, yymsp[-3].minor.yy190.pExpr, yymsp[-1].minor.yy190.pExpr, yymsp[0].minor.yy72); } break; - case 295: /* cmd ::= DETACH database_kw_opt expr */ + case 254: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy346.pExpr); + sqlite3Detach(pParse, yymsp[0].minor.yy190.pExpr); } break; - case 300: /* cmd ::= REINDEX */ + case 257: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 301: /* cmd ::= REINDEX nm dbnm */ + case 258: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 302: /* cmd ::= ANALYZE */ + case 259: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 303: /* cmd ::= ANALYZE nm dbnm */ + case 260: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 304: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 261: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy65,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy185,&yymsp[0].minor.yy0); } break; - case 305: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ + case 262: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { - sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); + yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; + sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 306: /* add_column_fullname ::= fullname */ + case 263: /* add_column_fullname ::= fullname */ { - pParse->db->lookaside.bEnabled = 0; - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy65); + disableLookaside(pParse); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy185); } break; - case 309: /* cmd ::= create_vtab */ + case 264: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 310: /* cmd ::= create_vtab LP vtabarglist RP */ + case 265: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 311: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 266: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy328); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy194); } break; - case 314: /* vtabarg ::= */ + case 267: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 316: /* vtabargtoken ::= ANY */ - case 317: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==317); - case 318: /* lp ::= LP */ yytestcase(yyruleno==318); + case 268: /* vtabargtoken ::= ANY */ + case 269: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==269); + case 270: /* lp ::= LP */ yytestcase(yyruleno==270); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 322: /* with ::= */ -{yygotominor.yy59 = 0;} + case 271: /* with ::= */ +{yymsp[1].minor.yy285 = 0;} break; - case 323: /* with ::= WITH wqlist */ - case 324: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==324); -{ yygotominor.yy59 = yymsp[0].minor.yy59; } + case 272: /* with ::= WITH wqlist */ +{ yymsp[-1].minor.yy285 = yymsp[0].minor.yy285; } break; - case 325: /* wqlist ::= nm idxlist_opt AS LP select RP */ + case 273: /* with ::= WITH RECURSIVE wqlist */ +{ yymsp[-2].minor.yy285 = yymsp[0].minor.yy285; } + break; + case 274: /* wqlist ::= nm eidlist_opt AS LP select RP */ { - yygotominor.yy59 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); + yymsp[-5].minor.yy285 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy148, yymsp[-1].minor.yy243); /*A-overwrites-X*/ } break; - case 326: /* wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP */ + case 275: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ { - yygotominor.yy59 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy59, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); + yymsp[-7].minor.yy285 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy285, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy148, yymsp[-1].minor.yy243); } break; default: - /* (0) input ::= cmdlist */ yytestcase(yyruleno==0); - /* (1) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==1); - /* (2) cmdlist ::= ecmd */ yytestcase(yyruleno==2); - /* (3) ecmd ::= SEMI */ yytestcase(yyruleno==3); - /* (4) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==4); - /* (10) trans_opt ::= */ yytestcase(yyruleno==10); - /* (11) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==11); - /* (12) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==12); - /* (20) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==20); - /* (21) savepoint_opt ::= */ yytestcase(yyruleno==21); - /* (25) cmd ::= create_table create_table_args */ yytestcase(yyruleno==25); - /* (36) columnlist ::= columnlist COMMA column */ yytestcase(yyruleno==36); - /* (37) columnlist ::= column */ yytestcase(yyruleno==37); - /* (43) type ::= */ yytestcase(yyruleno==43); - /* (50) signed ::= plus_num */ yytestcase(yyruleno==50); - /* (51) signed ::= minus_num */ yytestcase(yyruleno==51); - /* (52) carglist ::= carglist ccons */ yytestcase(yyruleno==52); - /* (53) carglist ::= */ yytestcase(yyruleno==53); - /* (60) ccons ::= NULL onconf */ yytestcase(yyruleno==60); - /* (88) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==88); - /* (89) conslist ::= tcons */ yytestcase(yyruleno==89); - /* (91) tconscomma ::= */ yytestcase(yyruleno==91); - /* (273) foreach_clause ::= */ yytestcase(yyruleno==273); - /* (274) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==274); - /* (281) tridxby ::= */ yytestcase(yyruleno==281); - /* (298) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==298); - /* (299) database_kw_opt ::= */ yytestcase(yyruleno==299); - /* (307) kwcolumn_opt ::= */ yytestcase(yyruleno==307); - /* (308) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==308); - /* (312) vtabarglist ::= vtabarg */ yytestcase(yyruleno==312); - /* (313) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==313); - /* (315) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==315); - /* (319) anylist ::= */ yytestcase(yyruleno==319); - /* (320) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==320); - /* (321) anylist ::= anylist ANY */ yytestcase(yyruleno==321); + /* (276) input ::= cmdlist */ yytestcase(yyruleno==276); + /* (277) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==277); + /* (278) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=278); + /* (279) ecmd ::= SEMI */ yytestcase(yyruleno==279); + /* (280) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==280); + /* (281) explain ::= */ yytestcase(yyruleno==281); + /* (282) trans_opt ::= */ yytestcase(yyruleno==282); + /* (283) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==283); + /* (284) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==284); + /* (285) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==285); + /* (286) savepoint_opt ::= */ yytestcase(yyruleno==286); + /* (287) cmd ::= create_table create_table_args */ yytestcase(yyruleno==287); + /* (288) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==288); + /* (289) columnlist ::= columnname carglist */ yytestcase(yyruleno==289); + /* (290) nm ::= ID|INDEXED */ yytestcase(yyruleno==290); + /* (291) nm ::= STRING */ yytestcase(yyruleno==291); + /* (292) nm ::= JOIN_KW */ yytestcase(yyruleno==292); + /* (293) typetoken ::= typename */ yytestcase(yyruleno==293); + /* (294) typename ::= ID|STRING */ yytestcase(yyruleno==294); + /* (295) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=295); + /* (296) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=296); + /* (297) carglist ::= carglist ccons */ yytestcase(yyruleno==297); + /* (298) carglist ::= */ yytestcase(yyruleno==298); + /* (299) ccons ::= NULL onconf */ yytestcase(yyruleno==299); + /* (300) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==300); + /* (301) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==301); + /* (302) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=302); + /* (303) tconscomma ::= */ yytestcase(yyruleno==303); + /* (304) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=304); + /* (305) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=305); + /* (306) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=306); + /* (307) oneselect ::= values */ yytestcase(yyruleno==307); + /* (308) sclp ::= selcollist COMMA */ yytestcase(yyruleno==308); + /* (309) as ::= ID|STRING */ yytestcase(yyruleno==309); + /* (310) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=310); + /* (311) exprlist ::= nexprlist */ yytestcase(yyruleno==311); + /* (312) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=312); + /* (313) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=313); + /* (314) nmnum ::= ON */ yytestcase(yyruleno==314); + /* (315) nmnum ::= DELETE */ yytestcase(yyruleno==315); + /* (316) nmnum ::= DEFAULT */ yytestcase(yyruleno==316); + /* (317) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==317); + /* (318) foreach_clause ::= */ yytestcase(yyruleno==318); + /* (319) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==319); + /* (320) trnm ::= nm */ yytestcase(yyruleno==320); + /* (321) tridxby ::= */ yytestcase(yyruleno==321); + /* (322) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==322); + /* (323) database_kw_opt ::= */ yytestcase(yyruleno==323); + /* (324) kwcolumn_opt ::= */ yytestcase(yyruleno==324); + /* (325) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==325); + /* (326) vtabarglist ::= vtabarg */ yytestcase(yyruleno==326); + /* (327) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==327); + /* (328) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==328); + /* (329) anylist ::= */ yytestcase(yyruleno==329); + /* (330) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==330); + /* (331) anylist ::= anylist ANY */ yytestcase(yyruleno==331); break; +/********** End reduce actions ************************************************/ }; - assert( yyruleno>=0 && yyrulenoyyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YYNSTATE ){ -#ifdef NDEBUG - /* If we are not debugging and the reduce action popped at least - ** one element off the stack, then we can push the new element back - ** onto the stack here, and skip the stack overflow test in yy_shift(). - ** That gives a significant speed improvement. */ - if( yysize ){ - yypParser->yyidx++; - yymsp -= yysize-1; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yymsp->minor = yygotominor; - }else -#endif - { - yy_shift(yypParser,yyact,yygoto,&yygotominor); + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ){ + yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; } + yymsp -= yysize-1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact); }else{ - assert( yyact == YYNSTATE + YYNRULE + 1 ); + assert( yyact == YY_ACCEPT_ACTION ); + yypParser->yytos -= yysize; yy_accept(yypParser); } } @@ -123709,9 +139451,11 @@ static void yy_parse_failed( fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +/************ End %parse_failure code *****************************************/ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } #endif /* YYNOERRORRECOVERY */ @@ -123722,14 +139466,16 @@ static void yy_parse_failed( static void yy_syntax_error( yyParser *yypParser, /* The parser */ int yymajor, /* The major type of the error token */ - YYMINORTYPE yyminor /* The minor type of the error token */ + sqlite3ParserTOKENTYPE yyminor /* The minor type of the error token */ ){ sqlite3ParserARG_FETCH; -#define TOKEN (yyminor.yy0) +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); +/************ End %syntax_error code ******************************************/ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } @@ -123745,9 +139491,14 @@ static void yy_accept( fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); /* Here code is inserted which will be executed whenever the ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +/*********** End %parse_accept code *******************************************/ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } @@ -123777,7 +139528,7 @@ SQLITE_PRIVATE void sqlite3Parser( sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; - int yyact; /* The parser action. */ + unsigned int yyact; /* The parser action. */ #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ #endif @@ -123786,23 +139537,8 @@ SQLITE_PRIVATE void sqlite3Parser( #endif yyParser *yypParser; /* The parser */ - /* (re)initialize the parser, if necessary */ yypParser = (yyParser*)yyp; - if( yypParser->yyidx<0 ){ -#if YYSTACKDEPTH<=0 - if( yypParser->yystksz <=0 ){ - /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ - yyminorunion = yyzerominor; - yyStackOverflow(yypParser, &yyminorunion); - return; - } -#endif - yypParser->yyidx = 0; - yypParser->yyerrcnt = -1; - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; - } - yyminorunion.yy0 = yyminor; + assert( yypParser->yytos!=0 ); #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); #endif @@ -123810,20 +139546,23 @@ SQLITE_PRIVATE void sqlite3Parser( #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); + fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]); } #endif do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyactyyerrcnt--; +#endif yymajor = YYNOCODE; - }else if( yyact < YYNSTATE + YYNRULE ){ - yy_reduce(yypParser,yyact-YYNSTATE); + }else if( yyact <= YY_MAX_REDUCE ){ + yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; #ifdef YYERRORSYMBOL int yymx; #endif @@ -123853,9 +139592,9 @@ SQLITE_PRIVATE void sqlite3Parser( ** */ if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_syntax_error(yypParser,yymajor,yyminor); } - yymx = yypParser->yystack[yypParser->yyidx].major; + yymx = yypParser->yytos->major; if( yymx==YYERRORSYMBOL || yyerrorhit ){ #ifndef NDEBUG if( yyTraceFILE ){ @@ -123863,26 +139602,26 @@ SQLITE_PRIVATE void sqlite3Parser( yyTracePrompt,yyTokenName[yymajor]); } #endif - yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); yymajor = YYNOCODE; }else{ - while( - yypParser->yyidx >= 0 && - yymx != YYERRORSYMBOL && - (yyact = yy_find_reduce_action( - yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YYNSTATE + while( yypParser->yytos >= yypParser->yystack + && yymx != YYERRORSYMBOL + && (yyact = yy_find_reduce_action( + yypParser->yytos->stateno, + YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } - if( yypParser->yyidx < 0 || yymajor==0 ){ + if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif yymajor = YYNOCODE; }else if( yymx!=YYERRORSYMBOL ){ - YYMINORTYPE u2; - u2.YYERRSYMDT = 0; - yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); } } yypParser->yyerrcnt = 3; @@ -123895,7 +139634,7 @@ SQLITE_PRIVATE void sqlite3Parser( ** Applications can set this macro (for example inside %include) if ** they intend to abandon the parse upon the first syntax error seen. */ - yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_syntax_error(yypParser,yymajor, yyminor); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yymajor = YYNOCODE; @@ -123910,17 +139649,32 @@ SQLITE_PRIVATE void sqlite3Parser( ** three input tokens have been successfully shifted. */ if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_syntax_error(yypParser,yymajor, yyminor); } yypParser->yyerrcnt = 3; yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); if( yyendofinput ){ yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif } yymajor = YYNOCODE; #endif } - }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); + }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack ); +#ifndef NDEBUG + if( yyTraceFILE ){ + yyStackEntry *i; + char cDiv = '['; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } + fprintf(yyTraceFILE,"]\n"); + } +#endif return; } @@ -123943,14 +139697,95 @@ SQLITE_PRIVATE void sqlite3Parser( ** individual tokens and sends those tokens one-by-one over to the ** parser for analysis. */ +/* #include "sqliteInt.h" */ /* #include */ +/* Character classes for tokenizing +** +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented +** using a lookup table, whereas a switch() directly on c uses a binary search. +** The lookup table is much faster. To maximize speed, and to ensure that +** a lookup table is used, all of the classes need to be small integers and +** all of them need to be used within the switch. +*/ +#define CC_X 0 /* The letter 'x', or start of BLOB literal */ +#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ +#define CC_ID 2 /* unicode characters usable in IDs */ +#define CC_DIGIT 3 /* Digits */ +#define CC_DOLLAR 4 /* '$' */ +#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ +#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ +#define CC_SPACE 7 /* Space characters */ +#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ +#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ +#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ +#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ +#define CC_LT 12 /* '<'. Part of < or <= or <> */ +#define CC_GT 13 /* '>'. Part of > or >= */ +#define CC_EQ 14 /* '='. Part of = or == */ +#define CC_BANG 15 /* '!'. Part of != */ +#define CC_SLASH 16 /* '/'. / or c-style comment */ +#define CC_LP 17 /* '(' */ +#define CC_RP 18 /* ')' */ +#define CC_SEMI 19 /* ';' */ +#define CC_PLUS 20 /* '+' */ +#define CC_STAR 21 /* '*' */ +#define CC_PERCENT 22 /* '%' */ +#define CC_COMMA 23 /* ',' */ +#define CC_AND 24 /* '&' */ +#define CC_TILDA 25 /* '~' */ +#define CC_DOT 26 /* '.' */ +#define CC_ILLEGAL 27 /* Illegal character */ + +static const unsigned char aiClass[] = { +#ifdef SQLITE_ASCII +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, +/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, +/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +#endif +#ifdef SQLITE_EBCDIC +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 12, 17, 20, 10, +/* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27, +/* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 7, +/* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8, +/* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* 9x */ 25, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, +/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27, +/* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, +/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27, +#endif +}; + /* -** The charMap() macro maps alphabetic characters into their +** The charMap() macro maps alphabetic characters (only) into their ** lower-case ASCII equivalent. On ASCII machines, this is just ** an upper-to-lower case map. On EBCDIC machines we also need -** to adjust the encoding. Only alphabetic characters and underscores -** need to be translated. +** to adjust the encoding. The mapping is only valid for alphabetics +** which are the only characters for which this feature is used. +** +** Used by keywordhash.h */ #ifdef SQLITE_ASCII # define charMap(X) sqlite3UpperToLower[(unsigned char)X] @@ -123984,7 +139819,7 @@ const unsigned char ebcdicToAscii[] = { ** returned. If the input is not a keyword, TK_ID is returned. ** ** The implementation of this routine was generated by a program, -** mkkeywordhash.h, located in the tool subdirectory of the distribution. +** mkkeywordhash.c, located in the tool subdirectory of the distribution. ** The output of the mkkeywordhash.c program is written into a file ** named keywordhash.h and then included into this source file by ** the #include below. @@ -124005,7 +139840,7 @@ const unsigned char ebcdicToAscii[] = { ** on platforms with limited memory. */ /* Hash score: 182 */ -static int keywordCode(const char *z, int n){ +static int keywordCode(const char *z, int n, int *pType){ /* zText[] encodes 834 bytes of keywords in 554 bytes */ /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ @@ -124125,13 +139960,21 @@ static int keywordCode(const char *z, int n){ TK_JOIN_KW, TK_ROLLBACK, TK_ROW, TK_UNION, TK_USING, TK_VACUUM, TK_VIEW, TK_INITIALLY, TK_ALL, }; - int h, i; - if( n<2 ) return TK_ID; - h = ((charMap(z[0])*4) ^ - (charMap(z[n-1])*3) ^ - n) % 127; - for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){ - if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){ + int i, j; + const char *zKW; + if( n>=2 ){ + i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) % 127; + for(i=((int)aHash[i])-1; i>=0; i=((int)aNext[i])-1){ + if( aLen[i]!=n ) continue; + j = 0; + zKW = &zText[aOffset[i]]; +#ifdef SQLITE_ASCII + while( j=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) #endif +/* Make the IdChar function accessible from ctime.c */ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } +#endif + /* -** Return the length of the token that begins at z[0]. +** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ int i, c; - switch( *z ){ - case ' ': case '\t': case '\n': case '\f': case '\r': { + switch( aiClass[*z] ){ /* Switch on the character-class of the first byte + ** of the token. See the comment on the CC_ defines + ** above. */ + case CC_SPACE: { testcase( z[0]==' ' ); testcase( z[0]=='\t' ); testcase( z[0]=='\n' ); @@ -124326,7 +140179,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_SPACE; return i; } - case '-': { + case CC_MINUS: { if( z[1]=='-' ){ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ @@ -124335,27 +140188,27 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_MINUS; return 1; } - case '(': { + case CC_LP: { *tokenType = TK_LP; return 1; } - case ')': { + case CC_RP: { *tokenType = TK_RP; return 1; } - case ';': { + case CC_SEMI: { *tokenType = TK_SEMI; return 1; } - case '+': { + case CC_PLUS: { *tokenType = TK_PLUS; return 1; } - case '*': { + case CC_STAR: { *tokenType = TK_STAR; return 1; } - case '/': { + case CC_SLASH: { if( z[1]!='*' || z[2]==0 ){ *tokenType = TK_SLASH; return 1; @@ -124365,15 +140218,15 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ return i; } - case '%': { + case CC_PERCENT: { *tokenType = TK_REM; return 1; } - case '=': { + case CC_EQ: { *tokenType = TK_EQ; return 1 + (z[1]=='='); } - case '<': { + case CC_LT: { if( (c=z[1])=='=' ){ *tokenType = TK_LE; return 2; @@ -124388,7 +140241,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return 1; } } - case '>': { + case CC_GT: { if( (c=z[1])=='=' ){ *tokenType = TK_GE; return 2; @@ -124400,16 +140253,16 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return 1; } } - case '!': { + case CC_BANG: { if( z[1]!='=' ){ *tokenType = TK_ILLEGAL; - return 2; + return 1; }else{ *tokenType = TK_NE; return 2; } } - case '|': { + case CC_PIPE: { if( z[1]!='|' ){ *tokenType = TK_BITOR; return 1; @@ -124418,21 +140271,19 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return 2; } } - case ',': { + case CC_COMMA: { *tokenType = TK_COMMA; return 1; } - case '&': { + case CC_AND: { *tokenType = TK_BITAND; return 1; } - case '~': { + case CC_TILDA: { *tokenType = TK_BITNOT; return 1; } - case '`': - case '\'': - case '"': { + case CC_QUOTE: { int delim = z[0]; testcase( delim=='`' ); testcase( delim=='\'' ); @@ -124457,7 +140308,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return i; } } - case '.': { + case CC_DOT: { #ifndef SQLITE_OMIT_FLOATING_POINT if( !sqlite3Isdigit(z[1]) ) #endif @@ -124468,8 +140319,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ /* If the next character is a digit, this is a floating point ** number that begins with ".". Fall thru into the next case */ } - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': { + case CC_DIGIT: { testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); @@ -124504,22 +140354,18 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ } return i; } - case '[': { + case CC_QUOTE2: { for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} *tokenType = c==']' ? TK_ID : TK_ILLEGAL; return i; } - case '?': { + case CC_VARNUM: { *tokenType = TK_VARIABLE; for(i=1; sqlite3Isdigit(z[i]); i++){} return i; } -#ifndef SQLITE_OMIT_TCL_VARIABLE - case '$': -#endif - case '@': /* For compatibility with MS SQL Server */ - case '#': - case ':': { + case CC_DOLLAR: + case CC_VARALPHA: { int n = 0; testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); testcase( z[0]=='#' ); @@ -124548,8 +140394,20 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( n==0 ) *tokenType = TK_ILLEGAL; return i; } + case CC_KYWD: { + for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( IdChar(z[i]) ){ + /* This token started out using characters that can appear in keywords, + ** but z[i] is a character not allowed within keywords, so this must + ** be an identifier instead */ + i++; + break; + } + *tokenType = TK_ID; + return keywordCode((char*)z, i, tokenType); + } + case CC_X: { #ifndef SQLITE_OMIT_BLOB_LITERAL - case 'x': case 'X': { testcase( z[0]=='x' ); testcase( z[0]=='X' ); if( z[1]=='\'' ){ *tokenType = TK_BLOB; @@ -124561,20 +140419,22 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( z[i] ) i++; return i; } - /* Otherwise fall through to the next case */ - } #endif + /* If it is not a BLOB literal, then it must be an ID, since no + ** SQL keywords start with the letter 'x'. Fall through */ + } + case CC_ID: { + i = 1; + break; + } default: { - if( !IdChar(*z) ){ - break; - } - for(i=1; IdChar(z[i]); i++){} - *tokenType = keywordCode((char*)z, i); - return i; + *tokenType = TK_ILLEGAL; + return 1; } } - *tokenType = TK_ILLEGAL; - return 1; + while( IdChar(z[i]) ){ i++; } + *tokenType = TK_ID; + return i; } /* @@ -124590,11 +140450,10 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr void *pEngine; /* The LEMON-generated LALR(1) parser */ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ - u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ sqlite3 *db = pParse->db; /* The database connection */ int mxSqlLen; /* Max length of an SQL string */ - + assert( zSql!=0 ); mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; if( db->nVdbeActive==0 ){ db->u1.isInterrupted = 0; @@ -124603,77 +140462,70 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr pParse->zTail = zSql; i = 0; assert( pzErrMsg!=0 ); - pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc); + /* sqlite3ParserTrace(stdout, "parser: "); */ + pEngine = sqlite3ParserAlloc(sqlite3Malloc); if( pEngine==0 ){ - db->mallocFailed = 1; - return SQLITE_NOMEM; + sqlite3OomFault(db); + return SQLITE_NOMEM_BKPT; } assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); assert( pParse->nzVar==0 ); assert( pParse->azVar==0 ); - enableLookaside = db->lookaside.bEnabled; - if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; - while( !db->mallocFailed && zSql[i]!=0 ){ + while( 1 ){ assert( i>=0 ); - pParse->sLastToken.z = &zSql[i]; - pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); - i += pParse->sLastToken.n; - if( i>mxSqlLen ){ - pParse->rc = SQLITE_TOOBIG; - break; - } - switch( tokenType ){ - case TK_SPACE: { - if( db->u1.isInterrupted ){ - sqlite3ErrorMsg(pParse, "interrupt"); - pParse->rc = SQLITE_INTERRUPT; - goto abort_parse; - } + if( zSql[i]!=0 ){ + pParse->sLastToken.z = &zSql[i]; + pParse->sLastToken.n = sqlite3GetToken((u8*)&zSql[i],&tokenType); + i += pParse->sLastToken.n; + if( i>mxSqlLen ){ + pParse->rc = SQLITE_TOOBIG; break; } - case TK_ILLEGAL: { - sqlite3DbFree(db, *pzErrMsg); - *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"", + }else{ + /* Upon reaching the end of input, call the parser two more times + ** with tokens TK_SEMI and 0, in that order. */ + if( lastTokenParsed==TK_SEMI ){ + tokenType = 0; + }else if( lastTokenParsed==0 ){ + break; + }else{ + tokenType = TK_SEMI; + } + } + if( tokenType>=TK_SPACE ){ + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); + if( db->u1.isInterrupted ){ + pParse->rc = SQLITE_INTERRUPT; + break; + } + if( tokenType==TK_ILLEGAL ){ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &pParse->sLastToken); - nErr++; - goto abort_parse; - } - case TK_SEMI: { - pParse->zTail = &zSql[i]; - /* Fall thru into the default case */ - } - default: { - sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); - lastTokenParsed = tokenType; - if( pParse->rc!=SQLITE_OK ){ - goto abort_parse; - } break; } + }else{ + sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); + lastTokenParsed = tokenType; + if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; } } -abort_parse: - if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ - if( lastTokenParsed!=TK_SEMI ){ - sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); - pParse->zTail = &zSql[i]; - } - sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); - } + assert( nErr==0 ); + pParse->zTail = &zSql[i]; #ifdef YYTRACKMAXSTACKDEPTH - sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, + sqlite3_mutex_enter(sqlite3MallocMutex()); + sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, sqlite3ParserStackPeak(pEngine) ); + sqlite3_mutex_leave(sqlite3MallocMutex()); #endif /* YYDEBUG */ sqlite3ParserFree(pEngine, sqlite3_free); - db->lookaside.bEnabled = enableLookaside; if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; + pParse->rc = SQLITE_NOMEM_BKPT; } if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ - sqlite3SetString(&pParse->zErrMsg, db, "%s", sqlite3ErrStr(pParse->rc)); + pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); } assert( pzErrMsg!=0 ); if( pParse->zErrMsg ){ @@ -124705,7 +140557,7 @@ abort_parse: sqlite3DeleteTable(db, pParse->pNewTable); } - if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); + if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree); sqlite3DeleteTrigger(db, pParse->pNewTrigger); for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); sqlite3DbFree(db, pParse->azVar); @@ -124719,9 +140571,7 @@ abort_parse: pParse->pZombieTab = p->pNextZombie; sqlite3DeleteTable(db, p); } - if( nErr>0 && pParse->rc==SQLITE_OK ){ - pParse->rc = SQLITE_ERROR; - } + assert( nErr==0 || pParse->rc!=SQLITE_OK ); return nErr; } @@ -124745,6 +140595,7 @@ abort_parse: ** separating it out, the code will be automatically omitted from ** static links that do not use it. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_COMPLETE /* @@ -124798,7 +140649,7 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; ** a statement. ** ** (4) CREATE The keyword CREATE has been seen at the beginning of a -** statement, possibly preceeded by EXPLAIN and/or followed by +** statement, possibly preceded by EXPLAIN and/or followed by ** TEMP or TEMPORARY ** ** (5) TRIGGER We are in the middle of a trigger definition that must be @@ -124808,7 +140659,7 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; ** the end of a trigger definition. ** ** (7) END We've seen the ";END" of the ";END;" that occurs at the end -** of a trigger difinition. +** of a trigger definition. ** ** Transitions between states above are determined by tokens extracted ** from the input. The following tokens are significant: @@ -124851,7 +140702,7 @@ SQLITE_API int sqlite3_complete(const char *zSql){ }; #else /* If triggers are not supported by this compile then the statement machine - ** used to detect the end of a statement is much simplier + ** used to detect the end of a statement is much simpler */ static const u8 trans[3][3] = { /* Token: */ @@ -124862,6 +140713,13 @@ SQLITE_API int sqlite3_complete(const char *zSql){ }; #endif /* SQLITE_OMIT_TRIGGER */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + while( *zSql ){ switch( *zSql ){ case ';': { /* A semicolon */ @@ -124990,7 +140848,7 @@ SQLITE_API int sqlite3_complete(const char *zSql){ SQLITE_API int sqlite3_complete16(const void *zSql){ sqlite3_value *pVal; char const *zSql8; - int rc = SQLITE_NOMEM; + int rc; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -125002,10 +140860,10 @@ SQLITE_API int sqlite3_complete16(const void *zSql){ if( zSql8 ){ rc = sqlite3_complete(zSql8); }else{ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } sqlite3ValueFree(pVal); - return sqlite3ApiExit(0, rc); + return rc & 0xff; } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_OMIT_COMPLETE */ @@ -125028,6 +140886,7 @@ SQLITE_API int sqlite3_complete16(const void *zSql){ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. */ +/* #include "sqliteInt.h" */ #ifdef SQLITE_ENABLE_FTS3 /************** Include fts3.h in the middle of main.c ***********************/ @@ -125047,6 +140906,7 @@ SQLITE_API int sqlite3_complete16(const void *zSql){ ** This header file is used by programs that want to link against the ** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. */ +/* #include "sqlite3.h" */ #if 0 extern "C" { @@ -125079,6 +140939,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); ** This header file is used by programs that want to link against the ** RTREE library. All it does is declare the sqlite3RtreeInit() interface. */ +/* #include "sqlite3.h" */ #if 0 extern "C" { @@ -125111,6 +140972,7 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); ** This header file is used by programs that want to link against the ** ICU extension. All it does is declare the sqlite3IcuInit() interface. */ +/* #include "sqlite3.h" */ #if 0 extern "C" { @@ -125126,6 +140988,12 @@ SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); /************** End of sqliteicu.h *******************************************/ /************** Continuing where we left off in main.c ***********************/ #endif +#ifdef SQLITE_ENABLE_JSON1 +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS5 +SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); +#endif #ifndef SQLITE_AMALGAMATION /* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant @@ -125156,6 +141024,18 @@ SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } */ SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } +/* +** When compiling the test fixture or with debugging enabled (on Win32), +** this variable being set to non-zero will cause OSTRACE macros to emit +** extra diagnostic information. +*/ +#ifdef SQLITE_HAVE_OS_TRACE +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +#endif + #if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) /* ** If the following function pointer is not NULL and if @@ -125163,7 +141043,7 @@ SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } ** I/O active are written using this function. These messages ** are intended for debugging activity only. */ -SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*, ...) = 0; +SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; #endif /* @@ -125229,6 +141109,11 @@ SQLITE_API int sqlite3_initialize(void){ } #endif + /* If the following assert() fails on some obscure processor/compiler + ** combination, the work-around is to set the correct pointer + ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ + assert( SQLITE_PTRSIZE==sizeof(char*) ); + /* If SQLite is already completely initialized, then this call ** to sqlite3_initialize() should be a no-op. But the initialization ** must be complete. So isInit must not be set until the very end @@ -125265,7 +141150,7 @@ SQLITE_API int sqlite3_initialize(void){ sqlite3GlobalConfig.pInitMutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } } @@ -125296,10 +141181,15 @@ SQLITE_API int sqlite3_initialize(void){ */ sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); sqlite3GlobalConfig.inProgress = 1; - memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); - sqlite3RegisterGlobalFunctions(); +#ifdef SQLITE_ENABLE_SQLLOG + { + extern void sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); + } +#endif + memset(&sqlite3BuiltinFunctions, 0, sizeof(sqlite3BuiltinFunctions)); + sqlite3RegisterBuiltinFunctions(); if( sqlite3GlobalConfig.isPCacheInit==0 ){ rc = sqlite3PcacheInitialize(); } @@ -125372,6 +141262,13 @@ SQLITE_API int sqlite3_initialize(void){ ** when this routine is invoked, then this routine is a harmless no-op. */ SQLITE_API int sqlite3_shutdown(void){ +#ifdef SQLITE_OMIT_WSD + int rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + if( sqlite3GlobalConfig.isInit ){ #ifdef SQLITE_EXTRA_SHUTDOWN void SQLITE_EXTRA_SHUTDOWN(void); @@ -125430,33 +141327,43 @@ SQLITE_API int sqlite3_config(int op, ...){ switch( op ){ /* Mutex configuration options are only available in a threadsafe - ** compile. + ** compile. */ -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ case SQLITE_CONFIG_SINGLETHREAD: { - /* Disable all mutexing */ - sqlite3GlobalConfig.bCoreMutex = 0; - sqlite3GlobalConfig.bFullMutex = 0; + /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to + ** Single-thread. */ + sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ case SQLITE_CONFIG_MULTITHREAD: { - /* Disable mutexing of database connections */ - /* Enable mutexing of core data structures */ - sqlite3GlobalConfig.bCoreMutex = 1; - sqlite3GlobalConfig.bFullMutex = 0; + /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to + ** Multi-thread. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ case SQLITE_CONFIG_SERIALIZED: { - /* Enable all mutexing */ - sqlite3GlobalConfig.bCoreMutex = 1; - sqlite3GlobalConfig.bFullMutex = 1; + /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to + ** Serialized. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ case SQLITE_CONFIG_MUTEX: { /* Specify an alternative mutex implementation */ sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ case SQLITE_CONFIG_GETMUTEX: { /* Retrieve the current mutex implementation */ *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; @@ -125464,37 +141371,62 @@ SQLITE_API int sqlite3_config(int op, ...){ } #endif - case SQLITE_CONFIG_MALLOC: { - /* Specify an alternative malloc implementation */ + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); break; } case SQLITE_CONFIG_GETMALLOC: { - /* Retrieve the current malloc() implementation */ + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; break; } case SQLITE_CONFIG_MEMSTATUS: { - /* Enable or disable the malloc status collection */ + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ sqlite3GlobalConfig.bMemstat = va_arg(ap, int); break; } case SQLITE_CONFIG_SCRATCH: { - /* Designate a buffer for scratch memory space */ + /* EVIDENCE-OF: R-08404-60887 There are three arguments to + ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from + ** which the scratch allocations will be drawn, the size of each scratch + ** allocation (sz), and the maximum number of scratch allocations (N). */ sqlite3GlobalConfig.pScratch = va_arg(ap, void*); sqlite3GlobalConfig.szScratch = va_arg(ap, int); sqlite3GlobalConfig.nScratch = va_arg(ap, int); break; } case SQLITE_CONFIG_PAGECACHE: { - /* Designate a buffer for page cache memory space */ + /* EVIDENCE-OF: R-18761-36601 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory (pMem), + ** the size of each page cache line (sz), and the number of cache lines + ** (N). */ sqlite3GlobalConfig.pPage = va_arg(ap, void*); sqlite3GlobalConfig.szPage = va_arg(ap, int); sqlite3GlobalConfig.nPage = va_arg(ap, int); break; } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } case SQLITE_CONFIG_PCACHE: { /* no-op */ @@ -125507,11 +141439,18 @@ SQLITE_API int sqlite3_config(int op, ...){ } case SQLITE_CONFIG_PCACHE2: { - /* Specify an alternative page cache implementation */ + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); break; } case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ if( sqlite3GlobalConfig.pcache2.xInit==0 ){ sqlite3PCacheSetDefault(); } @@ -125519,9 +141458,15 @@ SQLITE_API int sqlite3_config(int op, ...){ break; } +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) case SQLITE_CONFIG_HEAP: { - /* Designate a buffer for heap memory space */ + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. + */ sqlite3GlobalConfig.pHeap = va_arg(ap, void*); sqlite3GlobalConfig.nHeap = va_arg(ap, int); sqlite3GlobalConfig.mnReq = va_arg(ap, int); @@ -125534,17 +141479,19 @@ SQLITE_API int sqlite3_config(int op, ...){ } if( sqlite3GlobalConfig.pHeap==0 ){ - /* If the heap pointer is NULL, then restore the malloc implementation - ** back to NULL pointers too. This will cause the malloc to go - ** back to its default implementation when sqlite3_initialize() is - ** run. + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run */ memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); }else{ - /* The heap pointer is not NULL, then install one of the - ** mem5.c/mem3.c methods. The enclosing #if guarantees at - ** least one of these methods is currently enabled. - */ + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ #ifdef SQLITE_ENABLE_MEMSYS3 sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); #endif @@ -125577,12 +141524,25 @@ SQLITE_API int sqlite3_config(int op, ...){ break; } + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); break; } case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ sqlite3GlobalConfig.bUseCis = va_arg(ap, int); break; } @@ -125597,25 +141557,50 @@ SQLITE_API int sqlite3_config(int op, ...){ #endif case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ mxMmap = SQLITE_MAX_MMAP_SIZE; } - sqlite3GlobalConfig.mxMmap = mxMmap; if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; sqlite3GlobalConfig.szMmap = szMmap; break; } -#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ sqlite3GlobalConfig.nHeap = va_arg(ap, int); break; } #endif + case SQLITE_CONFIG_PMASZ: { + sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int); + break; + } + + case SQLITE_CONFIG_STMTJRNL_SPILL: { + sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int); + break; + } + default: { rc = SQLITE_ERROR; break; @@ -125637,6 +141622,7 @@ SQLITE_API int sqlite3_config(int op, ...){ ** the lookaside memory. */ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ +#ifndef SQLITE_OMIT_LOOKASIDE void *pStart; if( db->lookaside.nOut ){ return SQLITE_BUSY; @@ -125679,14 +141665,15 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ p = (LookasideSlot*)&((u8*)p)[sz]; } db->lookaside.pEnd = p; - db->lookaside.bEnabled = 1; + db->lookaside.bDisable = 0; db->lookaside.bMalloced = pBuf==0 ?1:0; }else{ db->lookaside.pStart = db; db->lookaside.pEnd = db; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable = 1; db->lookaside.bMalloced = 0; } +#endif /* SQLITE_OMIT_LOOKASIDE */ return SQLITE_OK; } @@ -125694,6 +141681,12 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ ** Return the mutex associated with a database connection. */ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->mutex; } @@ -125703,6 +141696,10 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ */ SQLITE_API int sqlite3_db_release_memory(sqlite3 *db){ int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ @@ -125717,6 +141714,36 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3 *db){ return SQLITE_OK; } +/* +** Flush any dirty pages in the pager-cache for any attached database +** to disk. +*/ +SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ + int i; + int rc = SQLITE_OK; + int bSeenBusy = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeIsInTrans(pBt) ){ + Pager *pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerFlush(pPager); + if( rc==SQLITE_BUSY ){ + bSeenBusy = 1; + rc = SQLITE_OK; + } + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return ((rc==SQLITE_OK && bSeenBusy) ? SQLITE_BUSY : rc); +} + /* ** Configuration settings for an individual database connection */ @@ -125725,6 +141752,11 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ int rc; va_start(ap, op); switch( op ){ + case SQLITE_DBCONFIG_MAINDBNAME: { + db->aDb[0].zDbSName = va_arg(ap,char*); + rc = SQLITE_OK; + break; + } case SQLITE_DBCONFIG_LOOKASIDE: { void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ @@ -125737,8 +141769,10 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ int op; /* The opcode */ u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ } aFlagOp[] = { - { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, - { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, + { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -125792,13 +141826,20 @@ static int binCollFunc( ){ int rc, n; n = nKey1lastRowid; } @@ -125840,6 +141887,12 @@ SQLITE_API sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ ** Return the number of changes in the most recent call to sqlite3_exec(). */ SQLITE_API int sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nChange; } @@ -125847,6 +141900,12 @@ SQLITE_API int sqlite3_changes(sqlite3 *db){ ** Return the number of changes since the database handle was opened. */ SQLITE_API int sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nTotalChange; } @@ -125873,7 +141932,7 @@ SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ ** with SQLITE_ANY as the encoding. */ static void functionDestroy(sqlite3 *db, FuncDef *p){ - FuncDestructor *pDestructor = p->pDestructor; + FuncDestructor *pDestructor = p->u.pDestructor; if( pDestructor ){ pDestructor->nRef--; if( pDestructor->nRef==0 ){ @@ -125890,17 +141949,23 @@ static void functionDestroy(sqlite3 *db, FuncDef *p){ static void disconnectAllVtab(sqlite3 *db){ #ifndef SQLITE_OMIT_VIRTUALTABLE int i; + HashElem *p; sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ Schema *pSchema = db->aDb[i].pSchema; if( db->aDb[i].pSchema ){ - HashElem *p; for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ Table *pTab = (Table *)sqliteHashData(p); if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); } } } + for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ + Module *pMod = (Module *)sqliteHashData(p); + if( pMod->pEpoTab ){ + sqlite3VtabDisconnect(db, pMod->pEpoTab); + } + } sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); #else @@ -125936,6 +142001,9 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); + if( db->mTrace & SQLITE_TRACE_CLOSE ){ + db->xTrace(SQLITE_TRACE_CLOSE, db->pTraceArg, db, 0); + } /* Force xDisconnect calls on all virtual tables */ disconnectAllVtab(db); @@ -125953,7 +142021,7 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ ** SQLITE_BUSY if the connection can not be closed immediately. */ if( !forceZombie && connectionIsBusy(db) ){ - sqlite3Error(db, SQLITE_BUSY, "unable to close due to unfinalized " + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " "statements or unfinished backups"); sqlite3_mutex_leave(db->mutex); return SQLITE_BUSY; @@ -126049,18 +142117,17 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ */ sqlite3ConnectionClosed(db); - for(j=0; jaFunc.a); j++){ - FuncDef *pNext, *pHash, *p; - for(p=db->aFunc.a[j]; p; p=pHash){ - pHash = p->pHash; - while( p ){ - functionDestroy(db, p); - pNext = p->pNext; - sqlite3DbFree(db, p); - p = pNext; - } - } + for(i=sqliteHashFirst(&db->aFunc); i; i=sqliteHashNext(i)){ + FuncDef *pNext, *p; + p = sqliteHashData(i); + do{ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + }while( p ); } + sqlite3HashClear(&db->aFunc); for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ CollSeq *pColl = (CollSeq *)sqliteHashData(i); /* Invoke any destructors registered for collation sequence user data. */ @@ -126078,14 +142145,19 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ if( pMod->xDestroy ){ pMod->xDestroy(pMod->pAux); } + sqlite3VtabEponymousTableClear(db, pMod); sqlite3DbFree(db, pMod); } sqlite3HashClear(&db->aModule); #endif - sqlite3Error(db, SQLITE_OK, 0); /* Deallocates any cached error strings. */ + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ sqlite3ValueFree(db->pErr); sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif db->magic = SQLITE_MAGIC_ERROR; @@ -126108,13 +142180,15 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* ** Rollback all database files. If tripCode is not SQLITE_OK, then -** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** any write cursors are invalidated ("tripped" - as in "tripping a circuit ** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. */ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; + int schemaChange; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); @@ -126125,6 +142199,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** the database rollback and schema reset, which can cause false ** corruption reports in some cases. */ sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; @@ -126132,7 +142207,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(p, tripCode); + sqlite3BtreeRollback(p, tripCode, !schemaChange); } } sqlite3VtabRollback(db); @@ -126159,7 +142234,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** Return a static string containing the name corresponding to the error code ** specified in the argument. */ -#if (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) || defined(SQLITE_TEST) +#if defined(SQLITE_NEED_ERR_NAME) SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ const char *zName = 0; int i, origRc = rc; @@ -126325,7 +142400,7 @@ static int sqliteDefaultBusyCallback( void *ptr, /* Database connection */ int count /* Number of times table has been busy */ ){ -#if SQLITE_OS_WIN || (defined(HAVE_USLEEP) && HAVE_USLEEP) +#if SQLITE_OS_WIN || HAVE_USLEEP static const u8 delays[] = { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; static const u8 totals[] = @@ -126388,6 +142463,9 @@ SQLITE_API int sqlite3_busy_handler( int (*xBusy)(void*,int), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->busyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; @@ -126409,6 +142487,12 @@ SQLITE_API void sqlite3_progress_handler( int (*xProgress)(void*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif sqlite3_mutex_enter(db->mutex); if( nOps>0 ){ db->xProgress = xProgress; @@ -126429,6 +142513,9 @@ SQLITE_API void sqlite3_progress_handler( ** specified number of milliseconds before returning 0. */ SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( ms>0 ){ sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); db->busyTimeout = ms; @@ -126442,6 +142529,12 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ ** Cause any pending operation to stop at its earliest opportunity. */ SQLITE_API void sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif db->u1.isInterrupted = 1; } @@ -126458,7 +142551,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( int nArg, int enc, void *pUserData, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*), FuncDestructor *pDestructor @@ -126469,9 +142562,9 @@ SQLITE_PRIVATE int sqlite3CreateFunc( assert( sqlite3_mutex_held(db->mutex) ); if( zFunctionName==0 || - (xFunc && (xFinal || xStep)) || - (!xFunc && (xFinal && !xStep)) || - (!xFunc && (!xFinal && xStep)) || + (xSFunc && (xFinal || xStep)) || + (!xSFunc && (xFinal && !xStep)) || + (!xSFunc && (!xFinal && xStep)) || (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || (255<(nName = sqlite3Strlen30( zFunctionName))) ){ return SQLITE_MISUSE_BKPT; @@ -126494,10 +142587,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( }else if( enc==SQLITE_ANY ){ int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); + pUserData, xSFunc, xStep, xFinal, pDestructor); if( rc==SQLITE_OK ){ rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); + pUserData, xSFunc, xStep, xFinal, pDestructor); } if( rc!=SQLITE_OK ){ return rc; @@ -126513,10 +142606,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( ** is being overridden/deleted but there are no active VMs, allow the ** operation to continue but invalidate all precompiled statements. */ - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ if( db->nVdbeActive ){ - sqlite3Error(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify user-function due to active statements"); assert( !db->mallocFailed ); return SQLITE_BUSY; @@ -126525,10 +142618,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( } } - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 1); assert(p || db->mallocFailed); if( !p ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* If an older version of the function with a configured destructor is @@ -126538,11 +142631,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( if( pDestructor ){ pDestructor->nRef++; } - p->pDestructor = pDestructor; + p->u.pDestructor = pDestructor; p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; testcase( p->funcFlags & SQLITE_DETERMINISTIC ); - p->xFunc = xFunc; - p->xStep = xStep; + p->xSFunc = xSFunc ? xSFunc : xStep; p->xFinalize = xFinal; p->pUserData = pUserData; p->nArg = (u16)nArg; @@ -126558,11 +142650,11 @@ SQLITE_API int sqlite3_create_function( int nArg, int enc, void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*) ){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, 0); } @@ -126572,13 +142664,19 @@ SQLITE_API int sqlite3_create_function_v2( int nArg, int enc, void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*), void (*xDestroy)(void *) ){ int rc = SQLITE_ERROR; FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); if( xDestroy ){ pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); @@ -126589,7 +142687,7 @@ SQLITE_API int sqlite3_create_function_v2( pArg->xDestroy = xDestroy; pArg->pUserData = p; } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, pArg); if( pArg && pArg->nRef==0 ){ assert( rc!=SQLITE_OK ); xDestroy(p); @@ -126609,16 +142707,20 @@ SQLITE_API int sqlite3_create_function16( int nArg, int eTextRep, void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ){ int rc; char *zFunc8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0); sqlite3DbFree(db, zFunc8); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -126644,10 +142746,15 @@ SQLITE_API int sqlite3_overload_function( const char *zName, int nArg ){ - int nName = sqlite3Strlen30(zName); int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); - if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ + if( sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)==0 ){ rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, 0, sqlite3InvalidFunction, 0, 0, 0); } @@ -126665,15 +142772,50 @@ SQLITE_API int sqlite3_overload_function( ** trace is a pointer to a function that is invoked at the start of each ** SQL statement. */ -SQLITE_API void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_API void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; - db->xTrace = xTrace; + db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; + db->xTrace = (int(*)(u32,void*,void*,void*))xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); return pOld; } +#endif /* SQLITE_OMIT_DEPRECATED */ + +/* Register a trace callback using the version-2 interface. +*/ +SQLITE_API int sqlite3_trace_v2( + sqlite3 *db, /* Trace this connection */ + unsigned mTrace, /* Mask of events to be traced */ + int(*xTrace)(unsigned,void*,void*,void*), /* Callback to invoke */ + void *pArg /* Context */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( mTrace==0 ) xTrace = 0; + if( xTrace==0 ) mTrace = 0; + db->mTrace = mTrace; + db->xTrace = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED /* ** Register a profile function. The pArg from the previously registered ** profile function is returned. @@ -126688,6 +142830,13 @@ SQLITE_API void *sqlite3_profile( void *pArg ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pProfileArg; db->xProfile = xProfile; @@ -126695,6 +142844,7 @@ SQLITE_API void *sqlite3_profile( sqlite3_mutex_leave(db->mutex); return pOld; } +#endif /* SQLITE_OMIT_DEPRECATED */ #endif /* SQLITE_OMIT_TRACE */ /* @@ -126708,6 +142858,13 @@ SQLITE_API void *sqlite3_commit_hook( void *pArg /* Argument to the function */ ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pCommitArg; db->xCommitCallback = xCallback; @@ -126726,6 +142883,13 @@ SQLITE_API void *sqlite3_update_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pUpdateArg; db->xUpdateCallback = xCallback; @@ -126744,6 +142908,13 @@ SQLITE_API void *sqlite3_rollback_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pRollbackArg; db->xRollbackCallback = xCallback; @@ -126752,6 +142923,27 @@ SQLITE_API void *sqlite3_rollback_hook( return pRet; } +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +SQLITE_API void *sqlite3_preupdate_hook( + sqlite3 *db, /* Attach the hook to this database */ + void(*xCallback)( /* Callback function */ + void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64), + void *pArg /* First callback argument */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pPreUpdateArg; + db->xPreUpdateCallback = xCallback; + db->pPreUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + #ifndef SQLITE_OMIT_WAL /* ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). @@ -126790,6 +142982,9 @@ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ UNUSED_PARAMETER(db); UNUSED_PARAMETER(nFrame); #else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( nFrame>0 ){ sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); }else{ @@ -126810,6 +143005,12 @@ SQLITE_API void *sqlite3_wal_hook( ){ #ifndef SQLITE_OMIT_WAL void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pWalArg; db->xWalCallback = xCallback; @@ -126837,14 +143038,21 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( int rc; /* Return code */ int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + /* Initialize the output variables to -1 in case an error occurs. */ if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; - assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE ); - assert( SQLITE_CHECKPOINT_FULLSQLITE_CHECKPOINT_RESTART ){ + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ return SQLITE_MISUSE; } @@ -126854,10 +143062,11 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( } if( iDb<0 ){ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); }else{ + db->busyHandler.nBusy = 0; rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -126872,7 +143081,9 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ** checkpointed. */ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ - return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0); + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); } #ifndef SQLITE_OMIT_WAL @@ -126947,9 +143158,11 @@ SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ return ( db->temp_store!=1 ); #endif #if SQLITE_TEMP_STORE==3 + UNUSED_PARAMETER(db); return 1; #endif #if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 + UNUSED_PARAMETER(db); return 0; #endif } @@ -126961,14 +143174,14 @@ SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ const char *z; if( !db ){ - return sqlite3ErrStr(SQLITE_NOMEM); + return sqlite3ErrStr(SQLITE_NOMEM_BKPT); } if( !sqlite3SafetyCheckSickOrOk(db) ){ return sqlite3ErrStr(SQLITE_MISUSE_BKPT); } sqlite3_mutex_enter(db->mutex); if( db->mallocFailed ){ - z = sqlite3ErrStr(SQLITE_NOMEM); + z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); }else{ testcase( db->pErr==0 ); z = (char*)sqlite3_value_text(db->pErr); @@ -127012,7 +143225,7 @@ SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ }else{ z = sqlite3_value_text16(db->pErr); if( z==0 ){ - sqlite3Error(db, db->errCode, sqlite3ErrStr(db->errCode)); + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); z = sqlite3_value_text16(db->pErr); } /* A malloc() may have failed within the call to sqlite3_value_text16() @@ -127020,7 +143233,7 @@ SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ ** be cleared before returning. Do this directly, instead of via ** sqlite3ApiExit(), to avoid setting the database handle error message. */ - db->mallocFailed = 0; + sqlite3OomClear(db); } sqlite3_mutex_leave(db->mutex); return z; @@ -127036,7 +143249,7 @@ SQLITE_API int sqlite3_errcode(sqlite3 *db){ return SQLITE_MISUSE_BKPT; } if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } return db->errCode & db->errMask; } @@ -127045,10 +143258,13 @@ SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ return SQLITE_MISUSE_BKPT; } if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } return db->errCode; } +SQLITE_API int sqlite3_system_errno(sqlite3 *db){ + return db ? db->iSysErrno : 0; +} /* ** Return a string that describes the kind of error specified in the @@ -127059,32 +143275,6 @@ SQLITE_API const char *sqlite3_errstr(int rc){ return sqlite3ErrStr(rc); } -/* -** Invalidate all cached KeyInfo objects for database connection "db" -*/ -static void invalidateCachedKeyInfo(sqlite3 *db){ - Db *pDb; /* A single database */ - int iDb; /* The database index number */ - HashElem *k; /* For looping over tables in pDb */ - Table *pTab; /* A table in the database */ - Index *pIdx; /* Each index */ - - for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ - if( pDb->pBt==0 ) continue; - sqlite3BtreeEnter(pDb->pBt); - for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ - pTab = (Table*)sqliteHashData(k); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->pKeyInfo && pIdx->pKeyInfo->db==db ){ - sqlite3KeyInfoUnref(pIdx->pKeyInfo); - pIdx->pKeyInfo = 0; - } - } - } - sqlite3BtreeLeave(pDb->pBt); - } -} - /* ** Create a new collating function for database "db". The name is zName ** and the encoding is enc. @@ -127099,7 +143289,6 @@ static int createCollation( ){ CollSeq *pColl; int enc2; - int nName = sqlite3Strlen30(zName); assert( sqlite3_mutex_held(db->mutex) ); @@ -127124,12 +143313,11 @@ static int createCollation( pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); if( pColl && pColl->xCmp ){ if( db->nVdbeActive ){ - sqlite3Error(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify collation sequence due to active statements"); return SQLITE_BUSY; } sqlite3ExpirePreparedStatements(db); - invalidateCachedKeyInfo(db); /* If collation sequence pColl was created directly by a call to ** sqlite3_create_collation, and not generated by synthCollSeq(), @@ -127138,7 +143326,7 @@ static int createCollation( ** to be called. */ if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); int j; for(j=0; j<3; j++){ CollSeq *p = &aColl[j]; @@ -127153,12 +143341,12 @@ static int createCollation( } pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl==0 ) return SQLITE_NOMEM; + if( pColl==0 ) return SQLITE_NOMEM_BKPT; pColl->xCmp = xCompare; pColl->pUser = pCtx; pColl->xDel = xDel; pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); return SQLITE_OK; } @@ -127180,6 +143368,7 @@ static const int aHardLimit[] = { SQLITE_MAX_LIKE_PATTERN_LENGTH, SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, }; /* @@ -127200,8 +143389,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_VDBE_OP<40 # error SQLITE_MAX_VDBE_OP must be at least 40 #endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 #endif #if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 # error SQLITE_MAX_ATTACHED must be between 0 and 125 @@ -127215,6 +143404,9 @@ static const int aHardLimit[] = { #if SQLITE_MAX_TRIGGER_DEPTH<1 # error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 #endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif /* @@ -127230,6 +143422,12 @@ static const int aHardLimit[] = { SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ int oldLimit; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME ** there is a hard upper bound set at compile-time by a C preprocessor @@ -127248,7 +143446,8 @@ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ SQLITE_MAX_LIKE_PATTERN_LENGTH ); assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -127305,25 +143504,38 @@ SQLITE_PRIVATE int sqlite3ParseUri( assert( *pzErrMsg==0 ); - if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) - && nUri>=5 && memcmp(zUri, "file:", 5)==0 + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; int eState; /* Parser state when parsing URI */ int iIn; /* Input character index */ int iOut = 0; /* Output character index */ - int nByte = nUri+2; /* Bytes of space to allocate */ + u64 nByte = nUri+2; /* Bytes of space to allocate */ /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen ** method that there may be extra parameters following the file-name. */ flags |= SQLITE_OPEN_URI; for(iIn=0; iInaLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; + db->nMaxSorterMmap = 0x7FFFFFFF; db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill #if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX | SQLITE_AutoIndex #endif +#if SQLITE_DEFAULT_CKPTFULLFSYNC + | SQLITE_CkptFullFSync +#endif #if SQLITE_DEFAULT_FILE_FORMAT<4 | SQLITE_LegacyFileFmt #endif @@ -127614,6 +143836,15 @@ static int openDatabase( #endif #if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS | SQLITE_ForeignKeys +#endif +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) + | SQLITE_ReverseOrder +#endif +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + | SQLITE_CellSizeCk +#endif +#if defined(SQLITE_ENABLE_FTS3_TOKENIZER) + | SQLITE_Fts3Tokenizer #endif ; sqlite3HashInit(&db->aCollSeq); @@ -127624,26 +143855,30 @@ static int openDatabase( /* Add the default collation sequence BINARY. BINARY works for both UTF-8 ** and UTF-16, so add a version for each to avoid any unnecessary ** conversions. The only error that can occur here is a malloc() failure. + ** + ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating + ** functions: */ - createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); if( db->mallocFailed ){ goto opendb_out; } - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ + db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, sqlite3StrBINARY, 0); assert( db->pDfltColl!=0 ); - /* Also add a UTF-8 case-insensitive collation sequence. */ - createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); - /* Parse the filename/URI argument. */ db->openFlags = flags; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; - sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); sqlite3_free(zErrMsg); goto opendb_out; } @@ -127653,22 +143888,24 @@ static int openDatabase( flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); goto opendb_out; } + sqlite3BtreeEnter(db->aDb[0].pBt); db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); + sqlite3BtreeLeave(db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - - /* The default safety_level for the main database is 'full'; for the temp - ** database it is 'NONE'. This matches the pager layer defaults. + /* The default safety_level for the main database is FULL; for the temp + ** database it is OFF. This matches the pager layer defaults. */ - db->aDb[0].zName = "main"; - db->aDb[0].safety_level = 3; - db->aDb[1].zName = "temp"; - db->aDb[1].safety_level = 1; + db->aDb[0].zDbSName = "main"; + db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + db->aDb[1].zDbSName = "temp"; + db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; db->magic = SQLITE_MAGIC_OPEN; if( db->mallocFailed ){ @@ -127679,13 +143916,22 @@ static int openDatabase( ** database schema yet. This is delayed until the first time the database ** is accessed. */ - sqlite3Error(db, SQLITE_OK, 0); - sqlite3RegisterBuiltinFunctions(db); + sqlite3Error(db, SQLITE_OK); + sqlite3RegisterPerConnectionBuiltinFunctions(db); + rc = sqlite3_errcode(db); + +#ifdef SQLITE_ENABLE_FTS5 + /* Register any built-in FTS5 module before loading the automatic + ** extensions. This allows automatic extensions to register FTS5 + ** tokenizers and auxiliary functions. */ + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts5Init(db); + } +#endif /* Load automatic extensions - extensions that have been registered ** using the sqlite3_automatic_extension() API. */ - rc = sqlite3_errcode(db); if( rc==SQLITE_OK ){ sqlite3AutoLoadExtensions(db); rc = sqlite3_errcode(db); @@ -127708,7 +143954,7 @@ static int openDatabase( } #endif -#ifdef SQLITE_ENABLE_FTS3 +#ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3Fts3Init(db); } @@ -127726,6 +143972,18 @@ static int openDatabase( } #endif +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3DbstatRegister(db); + } +#endif + +#ifdef SQLITE_ENABLE_JSON1 + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3Json1Init(db); + } +#endif + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking ** mode. Doing nothing at all also makes NORMAL the default. @@ -127736,7 +143994,7 @@ static int openDatabase( SQLITE_DEFAULT_LOCKING_MODE); #endif - if( rc ) sqlite3Error(db, rc, 0); + if( rc ) sqlite3Error(db, rc); /* Enable the lookaside-malloc subsystem */ setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, @@ -127745,9 +144003,9 @@ static int openDatabase( sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); opendb_out: - sqlite3_free(zOpen); if( db ){ - assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); + assert( db->mutex!=0 || isThreadsafe==0 + || sqlite3GlobalConfig.bFullMutex==0 ); sqlite3_mutex_leave(db->mutex); } rc = sqlite3_errcode(db); @@ -127766,7 +144024,23 @@ opendb_out: sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); } #endif - return sqlite3ApiExit(0, rc); +#if defined(SQLITE_HAS_CODEC) + if( rc==SQLITE_OK ){ + const char *zHexKey = sqlite3_uri_parameter(zOpen, "hexkey"); + if( zHexKey && zHexKey[0] ){ + u8 iByte; + int i; + char zKey[40]; + for(i=0, iByte=0; imutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); } /* @@ -127857,6 +144127,10 @@ SQLITE_API int sqlite3_create_collation_v2( void(*xDel)(void*) ){ int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); @@ -127878,6 +144152,10 @@ SQLITE_API int sqlite3_create_collation16( ){ int rc = SQLITE_OK; char *zName8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); @@ -127900,6 +144178,9 @@ SQLITE_API int sqlite3_collation_needed( void *pCollNeededArg, void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = xCollNeeded; db->xCollNeeded16 = 0; @@ -127918,6 +144199,9 @@ SQLITE_API int sqlite3_collation_needed16( void *pCollNeededArg, void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = 0; db->xCollNeeded16 = xCollNeeded16; @@ -127944,13 +144228,19 @@ SQLITE_API int sqlite3_global_recover(void){ ** by the next COMMIT or ROLLBACK. */ SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->autoCommit; } /* -** The following routines are subtitutes for constants SQLITE_CORRUPT, -** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error -** constants. They server two purposes: +** The following routines are substitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_NOMEM and possibly other error +** constants. They serve two purposes: ** ** 1. Serve as a convenient place to set a breakpoint in a debugger ** to detect when version error conditions occurs. @@ -127958,28 +144248,33 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ ** 2. Invoke sqlite3_log() to provide the source code location where ** a low-level error is first detected. */ +static int reportError(int iErr, int lineno, const char *zType){ + sqlite3_log(iErr, "%s at line %d of [%.10s]", + zType, lineno, 20+sqlite3_sourceid()); + return iErr; +} SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CORRUPT, - "database corruption at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CORRUPT; + return reportError(SQLITE_CORRUPT, lineno, "database corruption"); } SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_MISUSE, - "misuse at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_MISUSE; + return reportError(SQLITE_MISUSE, lineno, "misuse"); } SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CANTOPEN, - "cannot open file at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CANTOPEN; + return reportError(SQLITE_CANTOPEN, lineno, "cannot open file"); } - +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return reportError(SQLITE_NOMEM, lineno, "OOM"); +} +SQLITE_PRIVATE int sqlite3IoerrnomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return reportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); +} +#endif #ifndef SQLITE_OMIT_DEPRECATED /* @@ -127997,7 +144292,6 @@ SQLITE_API void sqlite3_thread_cleanup(void){ ** Return meta information about a specific column of a database table. ** See comment in sqlite3.h (sqlite.h.in) for details. */ -#ifdef SQLITE_ENABLE_COLUMN_METADATA SQLITE_API int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ const char *zDbName, /* Database name or NULL */ @@ -128013,14 +144307,20 @@ SQLITE_API int sqlite3_table_column_metadata( char *zErrMsg = 0; Table *pTab = 0; Column *pCol = 0; - int iCol; - + int iCol = 0; char const *zDataType = 0; char const *zCollSeq = 0; int notnull = 0; int primarykey = 0; int autoinc = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + /* Ensure the database schema has been loaded */ sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); @@ -128037,11 +144337,8 @@ SQLITE_API int sqlite3_table_column_metadata( } /* Find the column for which info is requested */ - if( sqlite3IsRowid(zColumnName) ){ - iCol = pTab->iPKey; - if( iCol>=0 ){ - pCol = &pTab->aCol[iCol]; - } + if( zColumnName==0 ){ + /* Query for existance of table only */ }else{ for(iCol=0; iColnCol; iCol++){ pCol = &pTab->aCol[iCol]; @@ -128050,8 +144347,13 @@ SQLITE_API int sqlite3_table_column_metadata( } } if( iCol==pTab->nCol ){ - pTab = 0; - goto error_out; + if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; + }else{ + pTab = 0; + goto error_out; + } } } @@ -128066,7 +144368,7 @@ SQLITE_API int sqlite3_table_column_metadata( ** explicitly declared column. Copy meta information from *pCol. */ if( pCol ){ - zDataType = pCol->zType; + zDataType = sqlite3ColumnType(pCol,0); zCollSeq = pCol->zColl; notnull = pCol->notNull!=0; primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; @@ -128076,7 +144378,7 @@ SQLITE_API int sqlite3_table_column_metadata( primarykey = 1; } if( !zCollSeq ){ - zCollSeq = "BINARY"; + zCollSeq = sqlite3StrBINARY; } error_out: @@ -128098,13 +144400,12 @@ error_out: zColumnName); rc = SQLITE_ERROR; } - sqlite3Error(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); sqlite3DbFree(db, zErrMsg); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } -#endif /* ** Sleep for a little while. Return the amount of time slept. @@ -128126,6 +144427,9 @@ SQLITE_API int sqlite3_sleep(int ms){ ** Enable or disable the extended result codes. */ SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->errMask = onoff ? 0xffffffff : 0xff; sqlite3_mutex_leave(db->mutex); @@ -128139,6 +144443,9 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo int rc = SQLITE_ERROR; Btree *pBtree; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); pBtree = sqlite3DbNameToBtree(db, zDbName); if( pBtree ){ @@ -128152,6 +144459,12 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo if( op==SQLITE_FCNTL_FILE_POINTER ){ *(sqlite3_file**)pArg = fd; rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_VFS_POINTER ){ + *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ + *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); + rc = SQLITE_OK; }else if( fd->pMethods ){ rc = sqlite3OsFileControl(fd, op, pArg); }else{ @@ -128160,7 +144473,7 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo sqlite3BtreeLeave(pBtree); } sqlite3_mutex_leave(db->mutex); - return rc; + return rc; } /* @@ -128168,7 +144481,9 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo */ SQLITE_API int sqlite3_test_control(int op, ...){ int rc = 0; -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifdef SQLITE_OMIT_BUILTIN_TEST + UNUSED_PARAMETER(op); +#else va_list ap; va_start(ap, op); switch( op ){ @@ -128264,7 +144579,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in ** an incompatible database file format. Changing the PENDING byte ** while any database connection is open results in undefined and - ** dileterious behavior. + ** deleterious behavior. */ case SQLITE_TESTCTRL_PENDING_BYTE: { rc = PENDING_BYTE; @@ -128290,7 +144605,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_ASSERT: { volatile int x = 0; - assert( (x = va_arg(ap,int))!=0 ); + assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); rc = x; break; } @@ -128419,22 +144734,6 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - /* sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT, - ** sqlite3_stmt*,const char**); - ** - ** If compiled with SQLITE_ENABLE_TREE_EXPLAIN, each sqlite3_stmt holds - ** a string that describes the optimized parse tree. This test-control - ** returns a pointer to that string. - */ - case SQLITE_TESTCTRL_EXPLAIN_STMT: { - sqlite3_stmt *pStmt = va_arg(ap, sqlite3_stmt*); - const char **pzRet = va_arg(ap, const char**); - *pzRet = sqlite3VdbeExplanation((Vdbe*)pStmt); - break; - } -#endif - /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); ** ** Set or clear a flag that indicates that the database file is always well- @@ -128448,6 +144747,15 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* Set the threshold at which OP_Once counters reset back to zero. + ** By default this is 0x7ffffffe (over 2 billion), but that value is + ** too big to test in a reasonable amount of time, so this control is + ** provided to set a small and easily reachable reset value. + */ + case SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD: { + sqlite3GlobalConfig.iOnceResetThreshold = va_arg(ap, int); + break; + } /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); ** @@ -128463,6 +144771,13 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; + } + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); ** ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if @@ -128473,6 +144788,34 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); + ** + ** This test control is used to create imposter tables. "db" is a pointer + ** to the database connection. dbName is the database name (ex: "main" or + ** "temp") which will receive the imposter. "onOff" turns imposter mode on + ** or off. "tnum" is the root page of the b-tree to which the imposter + ** table should connect. + ** + ** Enable imposter mode only when the schema has already been parsed. Then + ** run a single CREATE TABLE statement to construct the imposter table in + ** the parsed schema. Then turn imposter mode back off again. + ** + ** If onOff==0 and tnum>0 then reset the schema for all databases, causing + ** the schema to be reparsed the next time it is needed. This has the + ** effect of erasing all imposter tables. + */ + case SQLITE_TESTCTRL_IMPOSTER: { + sqlite3 *db = va_arg(ap, sqlite3*); + sqlite3_mutex_enter(db->mutex); + db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); + db->init.busy = db->init.imposterTable = va_arg(ap,int); + db->init.newTnum = va_arg(ap,int); + if( db->init.busy==0 && db->init.newTnum>0 ){ + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3_mutex_leave(db->mutex); + break; + } } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ @@ -128491,7 +144834,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** returns a NULL pointer. */ SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ - if( zFilename==0 ) return 0; + if( zFilename==0 || zParam==0 ) return 0; zFilename += sqlite3Strlen30(zFilename) + 1; while( zFilename[0] ){ int x = strcmp(zFilename, zParam); @@ -128534,7 +144877,7 @@ SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ int i; for(i=0; inDb; i++){ if( db->aDb[i].pBt - && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) + && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zDbSName)==0) ){ return db->aDb[i].pBt; } @@ -128547,7 +144890,14 @@ SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ ** connection. */ SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ - Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeGetFilename(pBt) : 0; } @@ -128556,10 +144906,99 @@ SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ ** no such database exists. */ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ - Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** Obtain a snapshot handle for the snapshot of database zDb currently +** being read by handle db. +*/ +SQLITE_API int sqlite3_snapshot_get( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot **ppSnapshot +){ + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL + int iDb; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInTrans(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + } + } + } + + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Open a read-transaction on the snapshot idendified by pSnapshot. +*/ +SQLITE_API int sqlite3_snapshot_open( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot *pSnapshot +){ + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( db->autoCommit==0 ){ + int iDb; + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ + rc = sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), pSnapshot); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pBt, 0); + sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), 0); + } + } + } + } + + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Free a snapshot handle obtained from sqlite3_snapshot_get(). +*/ +SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ + sqlite3_free(pSnapshot); +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + /************** End of main.c ************************************************/ /************** Begin file notify.c ******************************************/ /* @@ -128577,6 +145016,8 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ ** This file contains the implementation of the sqlite3_unlock_notify() ** API method and its associated functionality. */ +/* #include "sqliteInt.h" */ +/* #include "btreeInt.h" */ /* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY @@ -128746,7 +145187,7 @@ SQLITE_API int sqlite3_unlock_notify( leaveMutex(); assert( !db->mallocFailed ); - sqlite3Error(db, rc, (rc?"database is deadlocked":0)); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); sqlite3_mutex_leave(db->mutex); return rc; } @@ -129207,6 +145648,12 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ # define NDEBUG 1 #endif +/* FTS3/FTS4 require virtual tables */ +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_FTS3 +# undef SQLITE_ENABLE_FTS4 +#endif + /* ** FTS4 is really an extension for FTS3. It is enabled using the ** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all @@ -129220,9 +145667,11 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ /* If not building as part of the core, include sqlite3ext.h. */ #ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT3 #endif +/* #include "sqlite3.h" */ /************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ /************** Begin file fts3_tokenizer.h **********************************/ /* @@ -129251,6 +145700,7 @@ SQLITE_EXTENSION_INIT3 ** If tokenizers are to be allowed to call sqlite3_*() functions, then ** we will need a way to register the API consistently. */ +/* #include "sqlite3.h" */ /* ** Structures used by the tokenizer interface. When a new tokenizer @@ -129601,6 +146051,11 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi #ifdef SQLITE_COVERAGE_TEST # define ALWAYS(x) (1) # define NEVER(X) (0) +#elif defined(SQLITE_DEBUG) +# define ALWAYS(x) sqlite3Fts3Always((x)!=0) +# define NEVER(x) sqlite3Fts3Never((x)!=0) +SQLITE_PRIVATE int sqlite3Fts3Always(int b); +SQLITE_PRIVATE int sqlite3Fts3Never(int b); #else # define ALWAYS(x) (x) # define NEVER(x) (x) @@ -129659,6 +146114,8 @@ typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; typedef struct Fts3MultiSegReader Fts3MultiSegReader; +typedef struct MatchinfoBuffer MatchinfoBuffer; + /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance @@ -129724,6 +146181,7 @@ struct Fts3Table { int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ int iPrevLangid; /* Langid of recently inserted document */ + int bPrevDelete; /* True if last operation was a delete */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* State variables used for validating that the transaction control @@ -129768,9 +146226,7 @@ struct Fts3Cursor { i64 iMinDocid; /* Minimum docid to return */ i64 iMaxDocid; /* Maximum docid to return */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ - u32 *aMatchinfo; /* Information about most recent match */ - int nMatchinfo; /* Number of elements in aMatchinfo[] */ - char *zMatchinfo; /* Matchinfo specification */ + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ }; #define FTS3_EVAL_FILTER 0 @@ -129842,6 +146298,11 @@ struct Fts3Phrase { int bIncr; /* True if doclist is loaded incrementally */ int iDoclistToken; + /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an + ** OR condition. */ + char *pOrPoslist; + i64 iOrDocid; + /* Variables below this point are populated by fts3_expr.c when parsing ** a MATCH expression. Everything above is part of the evaluation phase. */ @@ -129885,7 +146346,9 @@ struct Fts3Expr { u8 bStart; /* True if iDocid is valid */ u8 bDeferred; /* True if this expression is entirely deferred */ - u32 *aMI; + /* The following are used by the fts3_snippet.c module. */ + int iPhrase; /* Index of this phrase in matchinfo() results */ + u32 *aMI; /* See above */ }; /* @@ -129996,6 +146459,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); ) /* fts3.c */ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); @@ -130005,6 +146469,7 @@ SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,i SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); /* fts3_tokenizer.c */ SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); @@ -130020,6 +146485,7 @@ SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const ch const char *, const char *, int, int ); SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); /* fts3_expr.c */ SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, @@ -130076,7 +146542,9 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); /* #include */ /* #include */ +/* #include "fts3.h" */ #ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #endif @@ -130085,6 +146553,13 @@ static int fts3EvalStart(Fts3Cursor *pCsr); static int fts3TermSegReaderCursor( Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); +#ifndef SQLITE_AMALGAMATION +# if defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } +SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } +# endif +#endif + /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. @@ -130194,7 +146669,7 @@ SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ /* If the first byte was a '[', then the close-quote character is a ']' */ if( quote=='[' ) quote = ']'; - while( ALWAYS(z[iIn]) ){ + while( z[iIn] ){ if( z[iIn]==quote ){ if( z[iIn+1]!=quote ) break; z[iOut++] = quote; @@ -130273,6 +146748,17 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ return SQLITE_OK; } +/* +** Write an error message into *pzErr +*/ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ + va_list ap; + sqlite3_free(*pzErr); + va_start(ap, zFormat); + *pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + /* ** Construct one or more SQL statements from the format string given ** and then evaluate those statements. The success code is written @@ -130682,11 +147168,16 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ ** This function is used when parsing the "prefix=" FTS4 parameter. */ static int fts3GobbleInt(const char **pp, int *pnOut){ + const int MAX_NPREFIX = 10000000; const char *p; /* Iterator pointer */ int nInt = 0; /* Output value */ for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ nInt = nInt * 10 + (p[0] - '0'); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + break; + } } if( p==*pp ) return SQLITE_ERROR; *pnOut = nInt; @@ -130729,7 +147220,6 @@ static int fts3PrefixParameter( aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); *apIndex = aIndex; - *pnIndex = nIndex; if( !aIndex ){ return SQLITE_NOMEM; } @@ -130739,13 +147229,20 @@ static int fts3PrefixParameter( const char *p = zParam; int i; for(i=1; i=0 ); + if( nPrefix==0 ){ + nIndex--; + i--; + }else{ + aIndex[i].nPrefix = nPrefix; + } p++; } } + *pnIndex = nIndex; return SQLITE_OK; } @@ -130780,7 +147277,8 @@ static int fts3ContentColumns( const char *zTbl, /* Name of content table */ const char ***pazCol, /* OUT: Malloc'd array of column names */ int *pnCol, /* OUT: Size of array *pazCol */ - int *pnStr /* OUT: Bytes of string content */ + int *pnStr, /* OUT: Bytes of string content */ + char **pzErr /* OUT: error message */ ){ int rc = SQLITE_OK; /* Return code */ char *zSql; /* "SELECT *" statement on zTbl */ @@ -130791,6 +147289,9 @@ static int fts3ContentColumns( rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); + } } sqlite3_free(zSql); @@ -130869,7 +147370,7 @@ static int fts3InitVtab( const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ - int nIndex; /* Size of aIndex[] array */ + int nIndex = 0; /* Size of aIndex[] array */ struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ /* The results of parsing supported FTS4 key=value options: */ @@ -130957,13 +147458,13 @@ static int fts3InitVtab( } } if( iOpt==SizeofArray(aFts4Opt) ){ - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); rc = SQLITE_ERROR; }else{ switch( iOpt ){ case 0: /* MATCHINFO */ if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); rc = SQLITE_ERROR; } bNoDocsize = 1; @@ -130991,7 +147492,7 @@ static int fts3InitVtab( if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) ){ - *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); rc = SQLITE_ERROR; } bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); @@ -131042,7 +147543,7 @@ static int fts3InitVtab( if( nCol==0 ){ sqlite3_free((void*)aCol); aCol = 0; - rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); /* If a languageid= option was specified, remove the language id ** column from the aCol[] array. */ @@ -131077,7 +147578,7 @@ static int fts3InitVtab( rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex); if( rc==SQLITE_ERROR ){ assert( zPrefix ); - *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); + sqlite3Fts3ErrMsg(pzErr, "error parsing prefix parameter: %s", zPrefix); } if( rc!=SQLITE_OK ) goto fts3_init_out; @@ -131159,7 +147660,7 @@ static int fts3InitVtab( } for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); @@ -131256,6 +147757,19 @@ static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ #endif } +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 + if( sqlite3_libversion_number()>=3008012 ){ + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + /* ** Implementation of the xBestIndex method for FTS3 tables. There ** are three possible strategies, in order of preference: @@ -131346,6 +147860,9 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } } + /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ + if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); + iIdx = 1; if( iCons>=0 ){ pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; @@ -131414,7 +147931,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; @@ -131625,7 +148142,7 @@ static int fts3SelectLeaf( sqlite3_int64 *piLeaf, /* Selected leaf node */ sqlite3_int64 *piLeaf2 /* Selected leaf node */ ){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int iHeight; /* Height of this node in tree */ assert( piLeaf || piLeaf2 ); @@ -131636,7 +148153,7 @@ static int fts3SelectLeaf( if( rc==SQLITE_OK && iHeight>1 ){ char *zBlob = 0; /* Blob read from %_segments table */ - int nBlob; /* Size of zBlob in bytes */ + int nBlob = 0; /* Size of zBlob in bytes */ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); @@ -132263,26 +148780,33 @@ static int fts3DoclistOrMerge( ** ** The right-hand input doclist is overwritten by this function. */ -static void fts3DoclistPhraseMerge( +static int fts3DoclistPhraseMerge( int bDescDoclist, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ char *aLeft, int nLeft, /* Left doclist */ - char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ + char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; + char *aRight = *paRight; char *pEnd1 = &aLeft[nLeft]; char *pEnd2 = &aRight[*pnRight]; char *p1 = aLeft; char *p2 = aRight; char *p; int bFirstOut = 0; - char *aOut = aRight; + char *aOut; assert( nDist>0 ); - + if( bDescDoclist ){ + aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); + if( aOut==0 ) return SQLITE_NOMEM; + }else{ + aOut = aRight; + } p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); @@ -132311,6 +148835,12 @@ static void fts3DoclistPhraseMerge( } *pnRight = (int)(p - aOut); + if( bDescDoclist ){ + sqlite3_free(aRight); + *paRight = aOut; + } + + return SQLITE_OK; } /* @@ -132435,8 +148965,22 @@ static int fts3TermSelectMerge( ){ if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist); + ** buffer using memcpy(). + ** + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** allocation. This is so as to ensure that the buffer is big enough + ** to hold the current doclist AND'd with any other doclist. If the + ** doclists are stored in order=ASC order, this padding would not be + ** required (since the size of [doclistA AND doclistB] is always less + ** than or equal to the size of [doclistA] in that case). But this is + ** not true for order=DESC. For example, a doclist containing (1, -1) + ** may be smaller than (-1), as in the first example the -1 may be stored + ** as a single-byte delta, whereas in the second it must be stored as a + ** FTS3_VARINT_MAX byte varint. + ** + ** Similar padding is added in the fts3DoclistOrMerge() function. + */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); @@ -132533,7 +149077,7 @@ static int fts3SegReaderCursor( ** calls out here. */ if( iLevel<0 && p->aIndex ){ Fts3SegReader *pSeg = 0; - rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); if( rc==SQLITE_OK && pSeg ){ rc = fts3SegReaderCursorAppend(pCsr, pSeg); } @@ -132858,7 +149402,7 @@ static int fts3FilterMethod( int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - int rc; + int rc = SQLITE_OK; char *zSql; /* SQL statement used to access %_content */ int eSearch; Fts3Table *p = (Fts3Table *)pCursor->pVtab; @@ -132888,6 +149432,7 @@ static int fts3FilterMethod( /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); @@ -132935,10 +149480,17 @@ static int fts3FilterMethod( ** row by docid. */ if( eSearch==FTS3_FULLSCAN_SEARCH ){ - zSql = sqlite3_mprintf( - "SELECT %s ORDER BY rowid %s", - p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") - ); + if( pDocidGe || pDocidLe ){ + zSql = sqlite3_mprintf( + "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", + p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, + (pCsr->bDesc ? "DESC" : "ASC") + ); + }else{ + zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + } if( zSql ){ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); sqlite3_free(zSql); @@ -133174,11 +149726,31 @@ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ char *p = &(*ppPoslist)[-2]; char c = 0; + /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ while( p>pStart && (c=*p--)==0 ); + + /* Search backwards for a varint with value zero (the end of the previous + ** poslist). This is an 0x00 byte preceded by some byte that does not + ** have the 0x80 bit set. */ while( p>pStart && (*p & 0x80) | c ){ c = *p--; } - if( p>pStart ){ p = &p[2]; } + assert( p==pStart || c==0 ); + + /* At this point p points to that preceding byte without the 0x80 bit + ** set. So to find the start of the poslist, skip forward 2 bytes then + ** over a varint. + ** + ** Normally. The other case is that p==pStart and the poslist to return + ** is the first in the doclist. In this case do not skip forward 2 bytes. + ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) + ** is required for cases where the first byte of a doclist and the + ** doclist is empty. For example, if the first docid is 10, a doclist + ** that begins with: + ** + ** 0x0A 0x00 + */ + if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } while( *p++&0x80 ); *ppPoslist = p; } @@ -133249,6 +149821,8 @@ static void fts3SnippetFunc( } if( !zEllipsis || !zEnd || !zStart ){ sqlite3_result_error_nomem(pContext); + }else if( nToken==0 ){ + sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); } @@ -133684,14 +150258,17 @@ static void fts3EvalAllocateReaders( ** This function assumes that pList points to a buffer allocated using ** sqlite3_malloc(). This function takes responsibility for eventually ** freeing the buffer. +** +** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. */ -static void fts3EvalPhraseMergeToken( +static int fts3EvalPhraseMergeToken( Fts3Table *pTab, /* FTS Table pointer */ Fts3Phrase *p, /* Phrase to merge pList/nList into */ int iToken, /* Token pList/nList corresponds to */ char *pList, /* Pointer to doclist */ int nList /* Number of bytes in pList */ ){ + int rc = SQLITE_OK; assert( iToken!=p->iDoclistToken ); if( pList==0 ){ @@ -133730,13 +150307,16 @@ static void fts3EvalPhraseMergeToken( nDiff = p->iDoclistToken - iToken; } - fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); + rc = fts3DoclistPhraseMerge( + pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight + ); sqlite3_free(pLeft); p->doclist.aAll = pRight; p->doclist.nAll = nRight; } if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + return rc; } /* @@ -133762,7 +150342,7 @@ static int fts3EvalPhraseLoad( char *pThis = 0; rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); if( rc==SQLITE_OK ){ - fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } } assert( pToken->pSegcsr==0 ); @@ -133907,7 +150487,6 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int bIncrOk = (bOptOk && pCsr->bDesc==pTab->bDescIdx && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 - && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 #ifdef SQLITE_TEST && pTab->bNoIncrDoclist==0 #endif @@ -134027,6 +150606,7 @@ SQLITE_PRIVATE void sqlite3Fts3DoclistNext( p += sqlite3Fts3GetVarint(p, piDocid); }else{ fts3PoslistCopy(0, &p); + while( p<&aDoclist[nDoclist] && *p==0 ) p++; if( p>=&aDoclist[nDoclist] ){ *pbEof = 1; }else{ @@ -134198,7 +150778,7 @@ static int fts3EvalIncrPhraseNext( bMaxSet = 1; } } - assert( rc!=SQLITE_OK || a[p->nToken-1].bIgnore==0 ); + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); assert( rc!=SQLITE_OK || bMaxSet ); /* Keep advancing iterators until they all point to the same document */ @@ -134304,12 +150884,14 @@ static void fts3EvalStartReaders( ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ - int i; int nToken = pExpr->pPhrase->nToken; - for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + if( nToken ){ + int i; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); } - pExpr->bDeferred = (i==nToken); *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); }else{ fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); @@ -134564,9 +151146,13 @@ static int fts3EvalSelectDeferred( char *pList = 0; rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken( + pTab, pTC->pPhrase, pTC->iToken,pList,nList + ); + } if( rc==SQLITE_OK ){ int nCount; - fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); nCount = fts3DoclistCountDocids( pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll ); @@ -134744,7 +151330,7 @@ static int fts3EvalNearTrim( ** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is ** advanced to point to the next row that matches "x AND y". ** -** See fts3EvalTestDeferredAndNear() for details on testing if a row is +** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is ** really a match, taking into account deferred tokens and NEAR operators. */ static void fts3EvalNextRow( @@ -134791,6 +151377,22 @@ static void fts3EvalNextRow( } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); + if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ + if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pRight->pPhrase->doclist; + while( *pRc==SQLITE_OK && pRight->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pLeft->pPhrase->doclist; + while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pLeft, pRc); + } + } + } } break; } @@ -134948,7 +151550,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ } /* -** This function is a helper function for fts3EvalTestDeferredAndNear(). +** This function is a helper function for sqlite3Fts3EvalTestDeferred(). ** Assuming no error occurs or has occurred, It returns non-zero if the ** expression passed as the second argument matches the row that pCsr ** currently points to, or zero if it does not. @@ -135069,7 +151671,7 @@ static int fts3EvalTestExpr( ** Or, if no error occurs and it seems the current row does match the FTS ** query, return 0. */ -static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){ +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; int bMiss = 0; if( rc==SQLITE_OK ){ @@ -135116,7 +151718,7 @@ static int fts3EvalNext(Fts3Cursor *pCsr){ pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pExpr->iDocid; - }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); + }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); } /* Check if the cursor is past the end of the docid range specified @@ -135163,6 +151765,7 @@ static void fts3EvalRestart( } pPhrase->doclist.pNextDocid = 0; pPhrase->doclist.iDocid = 0; + pPhrase->pOrPoslist = 0; } pExpr->iDocid = 0; @@ -135276,7 +151879,7 @@ static int fts3EvalGatherStats( pCsr->iPrevId = pRoot->iDocid; }while( pCsr->isEof==0 && pRoot->eType==FTSQUERY_NEAR - && fts3EvalTestDeferredAndNear(pCsr, &rc) + && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); if( rc==SQLITE_OK && pCsr->isEof==0 ){ @@ -135301,7 +151904,6 @@ static int fts3EvalGatherStats( fts3EvalNextRow(pCsr, pRoot, &rc); assert( pRoot->bEof==0 ); }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); - fts3EvalTestDeferredAndNear(pCsr, &rc); } } return rc; @@ -135408,13 +152010,13 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( iDocid = pExpr->iDocid; pIter = pPhrase->doclist.pList; if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int rc = SQLITE_OK; int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ - int iMul; /* +1 if csr dir matches index dir, else -1 */ int bOr = 0; - u8 bEof = 0; u8 bTreeEof = 0; Fts3Expr *p; /* Used to iterate from pExpr to root */ Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + int bMatch; /* Check if this phrase descends from an OR expression node. If not, ** return NULL. Otherwise, the entry that corresponds to docid @@ -135433,74 +152035,62 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( ** an incremental phrase. Load the entire doclist for the phrase ** into memory in this case. */ if( pPhrase->bIncr ){ - int rc = SQLITE_OK; - int bEofSave = pExpr->bEof; - fts3EvalRestart(pCsr, pExpr, &rc); - while( rc==SQLITE_OK && !pExpr->bEof ){ - fts3EvalNextRow(pCsr, pExpr, &rc); - if( bEofSave==0 && pExpr->iDocid==iDocid ) break; + int bEofSave = pNear->bEof; + fts3EvalRestart(pCsr, pNear, &rc); + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + if( bEofSave==0 && pNear->iDocid==iDocid ) break; } - pIter = pPhrase->doclist.pList; assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); - if( rc!=SQLITE_OK ) return rc; } - - iMul = ((pCsr->bDesc==bDescDoclist) ? 1 : -1); - while( bTreeEof==1 - && pNear->bEof==0 - && (DOCID_CMP(pNear->iDocid, pCsr->iPrevId) * iMul)<0 - ){ - int rc = SQLITE_OK; - fts3EvalNextRow(pCsr, pExpr, &rc); - if( rc!=SQLITE_OK ) return rc; - iDocid = pExpr->iDocid; - pIter = pPhrase->doclist.pList; + if( bTreeEof ){ + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + } } + if( rc!=SQLITE_OK ) return rc; - bEof = (pPhrase->doclist.nAll==0); - assert( bDescDoclist==0 || bDescDoclist==1 ); - assert( pCsr->bDesc==0 || pCsr->bDesc==1 ); + bMatch = 1; + for(p=pNear; p; p=p->pLeft){ + u8 bEof = 0; + Fts3Expr *pTest = p; + Fts3Phrase *pPh; + assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); + if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; + assert( pTest->eType==FTSQUERY_PHRASE ); + pPh = pTest->pPhrase; - if( bEof==0 ){ + pIter = pPh->pOrPoslist; + iDocid = pPh->iOrDocid; if( pCsr->bDesc==bDescDoclist ){ - int dummy; - if( pNear->bEof ){ - /* This expression is already at EOF. So position it to point to the - ** last entry in the doclist at pPhrase->doclist.aAll[]. Variable - ** iDocid is already set for this entry, so all that is required is - ** to set pIter to point to the first byte of the last position-list - ** in the doclist. - ** - ** It would also be correct to set pIter and iDocid to zero. In - ** this case, the first call to sqltie3Fts4DoclistPrev() below - ** would also move the iterator to point to the last entry in the - ** doclist. However, this is expensive, as to do so it has to - ** iterate through the entire doclist from start to finish (since - ** it does not know the docid for the last entry). */ - pIter = &pPhrase->doclist.aAll[pPhrase->doclist.nAll-1]; - fts3ReversePoslist(pPhrase->doclist.aAll, &pIter); - } - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ - sqlite3Fts3DoclistPrev( - bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, - &pIter, &iDocid, &dummy, &bEof - ); - } - }else{ - if( pNear->bEof ){ - pIter = 0; - iDocid = 0; - } + bEof = !pPh->doclist.nAll || + (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ sqlite3Fts3DoclistNext( - bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, &pIter, &iDocid, &bEof ); } + }else{ + bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + int dummy; + sqlite3Fts3DoclistPrev( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } } + pPh->pOrPoslist = pIter; + pPh->iOrDocid = iDocid; + if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; } - if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0; + if( bMatch ){ + pIter = pPhrase->pOrPoslist; + }else{ + pIter = 0; + } } if( pIter==0 ) return SQLITE_OK; @@ -135512,10 +152102,13 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( } while( iThis */ @@ -135689,7 +152283,7 @@ static int fts3auxConnectMethod( return SQLITE_OK; bad_args: - *pzErr = sqlite3_mprintf("invalid arguments to fts4aux constructor"); + sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); return SQLITE_ERROR; } @@ -136141,6 +152735,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ ** syntax is relatively simple, the whole tokenizer/parser system is ** hand-coded. */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* @@ -136315,7 +152910,7 @@ static int getNextToken( /* Set variable i to the maximum number of bytes of input to tokenize. */ for(i=0; ieType==eType; p=p->pLeft){ - assert( p->pParent==0 || p->pParent->pLeft==p ); - assert( p->pLeft && p->pRight ); - } - - /* This loop runs once for each leaf in the tree of eType nodes. */ - while( 1 ){ - int iLvl; - Fts3Expr *pParent = p->pParent; /* Current parent of p */ - - assert( pParent==0 || pParent->pLeft==p ); - p->pParent = 0; - if( pParent ){ - pParent->pLeft = 0; - }else{ - pRoot = 0; - } - rc = fts3ExprBalance(&p, nMaxDepth-1); - if( rc!=SQLITE_OK ) break; - - for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; - pFree->pRight = p; - pFree->pLeft->pParent = pFree; - pFree->pRight->pParent = pFree; - - p = pFree; - pFree = pFree->pParent; - p->pParent = 0; - apLeaf[iLvl] = 0; - } - } - if( p ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_TOOBIG; - break; - } - - /* If that was the last leaf node, break out of the loop */ - if( pParent==0 ) break; - - /* Set $p to point to the next leaf in the tree of eType nodes */ - for(p=pParent->pRight; p->eType==eType; p=p->pLeft); - - /* Remove pParent from the original tree. */ - assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); - pParent->pRight->pParent = pParent->pParent; - if( pParent->pParent ){ - pParent->pParent->pLeft = pParent->pRight; - }else{ - assert( pParent==pRoot ); - pRoot = pParent->pRight; - } - - /* Link pParent into the free node list. It will be used as an - ** internal node of the new tree. */ - pParent->pParent = pFree; - pFree = pParent; + if( rc==SQLITE_OK ){ + if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); } if( rc==SQLITE_OK ){ - p = 0; - for(i=0; ipParent = 0; + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; pFree->pRight = p; - pFree->pLeft = apLeaf[i]; pFree->pLeft->pParent = pFree; pFree->pRight->pParent = pFree; p = pFree; pFree = pFree->pParent; p->pParent = 0; + apLeaf[iLvl] = 0; } } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; } - pRoot = p; - }else{ - /* An error occurred. Delete the contents of the apLeaf[] array - ** and pFree list. Everything else is cleaned up by the call to - ** sqlite3Fts3ExprFree(pRoot) below. */ - Fts3Expr *pDel; - for(i=0; ipParent; - sqlite3_free(pDel); + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + }else if( eType==FTSQUERY_NOT ){ + Fts3Expr *pLeft = pRoot->pLeft; + Fts3Expr *pRight = pRoot->pRight; + + pRoot->pLeft = 0; + pRoot->pRight = 0; + pLeft->pParent = 0; + pRight->pParent = 0; + + rc = fts3ExprBalance(&pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprBalance(&pRight, nMaxDepth-1); } - assert( pFree==0 ); - sqlite3_free( apLeaf ); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRight); + sqlite3Fts3ExprFree(pLeft); + }else{ + assert( pLeft && pRight ); + pRoot->pLeft = pLeft; + pLeft->pParent = pRoot; + pRoot->pRight = pRight; + pRight->pParent = pRoot; + } } } - + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(pRoot); pRoot = 0; @@ -137147,13 +153768,13 @@ SQLITE_PRIVATE int sqlite3Fts3ExprParse( sqlite3Fts3ExprFree(*ppExpr); *ppExpr = 0; if( rc==SQLITE_TOOBIG ){ - *pzErr = sqlite3_mprintf( + sqlite3Fts3ErrMsg(pzErr, "FTS expression tree is too large (maximum depth %d)", SQLITE_FTS3_MAX_EXPR_DEPTH ); rc = SQLITE_ERROR; }else if( rc==SQLITE_ERROR ){ - *pzErr = sqlite3_mprintf("malformed MATCH expression: [%s]", z); + sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); } } @@ -137434,12 +154055,14 @@ SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* #include */ /* #include */ /* #include */ +/* #include "fts3_hash.h" */ /* ** Malloc and Free functions @@ -137817,6 +154440,7 @@ SQLITE_PRIVATE void *sqlite3Fts3HashInsert( ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* #include */ @@ -137824,6 +154448,7 @@ SQLITE_PRIVATE void *sqlite3Fts3HashInsert( /* #include */ /* #include */ +/* #include "fts3_tokenizer.h" */ /* ** Class derived from sqlite3_tokenizer @@ -137976,7 +154601,7 @@ static int isVowel(const char *z){ ** by a consonant. ** ** In this routine z[] is in reverse order. So we are really looking -** for an instance of of a consonant followed by a vowel. +** for an instance of a consonant followed by a vowel. */ static int m_gt_0(const char *z){ while( isVowel(z) ){ z++; } @@ -138481,11 +155106,24 @@ SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* #include */ /* #include */ +/* +** Return true if the two-argument version of fts3_tokenizer() +** has been activated via a prior call to sqlite3_db_config(db, +** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); +*/ +static int fts3TokenizerEnabled(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + int isEnabled = 0; + sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); + return isEnabled; +} + /* ** Implementation of the SQL scalar function for accessing the underlying ** hash table. This function may be called as follows: @@ -138506,7 +155144,7 @@ SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( ** is a blob containing the pointer stored as the hash data corresponding ** to string (after the hash-table is updated, if applicable). */ -static void scalarFunc( +static void fts3TokenizerFunc( sqlite3_context *context, int argc, sqlite3_value **argv @@ -138524,20 +155162,26 @@ static void scalarFunc( nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ - void *pOld; - int n = sqlite3_value_bytes(argv[1]); - if( n!=sizeof(pPtr) ){ - sqlite3_result_error(context, "argument type mismatch", -1); - return; - } - pPtr = *(void **)sqlite3_value_blob(argv[1]); - pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); - if( pOld==pPtr ){ - sqlite3_result_error(context, "out of memory", -1); + if( fts3TokenizerEnabled(context) ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( zName==0 || n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + } + }else{ + sqlite3_result_error(context, "fts3tokenize disabled", -1); return; } }else{ - pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + if( zName ){ + pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + } if( !pPtr ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); sqlite3_result_error(context, zErr, -1); @@ -138545,7 +155189,6 @@ static void scalarFunc( return; } } - sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } @@ -138618,12 +155261,16 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( zEnd = &zCopy[strlen(zCopy)]; z = (char *)sqlite3Fts3NextToken(zCopy, &n); + if( z==0 ){ + assert( n==0 ); + z = zCopy; + } z[n] = '\0'; sqlite3Fts3Dequote(z); m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); if( !m ){ - *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); rc = SQLITE_ERROR; }else{ char const **aArg = 0; @@ -138646,7 +155293,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( rc = m->xCreate(iArg, aArg, ppTok); assert( rc!=SQLITE_OK || *ppTok ); if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("unknown tokenizer"); + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); }else{ (*ppTok)->pModule = m; } @@ -138660,7 +155307,11 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( #ifdef SQLITE_TEST -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif /* #include */ /* @@ -138730,9 +155381,9 @@ static void testFunc( p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); if( !p ){ - char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); + char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr2, -1); + sqlite3_free(zErr2); return; } @@ -138801,6 +155452,7 @@ int registerTokenizer( return sqlite3_finalize(pStmt); } + static int queryTokenizer( sqlite3 *db, @@ -138871,11 +155523,13 @@ static void intTestFunc( assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); /* Test the storage function */ - rc = registerTokenizer(db, "nosuchtokenizer", p1); - assert( rc==SQLITE_OK ); - rc = queryTokenizer(db, "nosuchtokenizer", &p2); - assert( rc==SQLITE_OK ); - assert( p2==p1 ); + if( fts3TokenizerEnabled(context) ){ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); + } sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); } @@ -138891,7 +155545,7 @@ static void intTestFunc( ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); ** ** This function adds a scalar function (see header comment above -** scalarFunc() in this file for details) and, if ENABLE_TABLE is +** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is ** defined at compilation time, a temporary virtual table (see header ** comment above struct HashTableVtab) to the database schema. Both ** provide read/write access to the contents of *pHash. @@ -138920,10 +155574,10 @@ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( #endif if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); + rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); } if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); + rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); } #ifdef SQLITE_TEST if( SQLITE_OK==rc ){ @@ -138970,6 +155624,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* #include */ @@ -138977,6 +155632,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( /* #include */ /* #include */ +/* #include "fts3_tokenizer.h" */ typedef struct simple_tokenizer { sqlite3_tokenizer base; @@ -139221,6 +155877,7 @@ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( ** pos: Token offset of token within input. ** */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* #include */ @@ -139267,7 +155924,7 @@ static int fts3tokQueryTokenizer( p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); if( !p ){ - *pzErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); return SQLITE_ERROR; } @@ -139345,7 +156002,7 @@ static int fts3tokConnectMethod( sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ char **pzErr /* OUT: sqlite3_malloc'd error message */ ){ - Fts3tokTable *pTab; + Fts3tokTable *pTab = 0; const sqlite3_tokenizer_module *pMod = 0; sqlite3_tokenizer *pTok = 0; int rc; @@ -139656,6 +156313,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ ** code in fts3.c. */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* #include */ @@ -139964,14 +156622,15 @@ static int fts3SqlStmt( /* 25 */ "", /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", -/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'", +/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", /* This statement is used to determine which level to read the input from ** when performing an incremental merge. It returns the absolute level number ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ -/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " + " GROUP BY level HAVING cnt>=?" " ORDER BY (level %% 1024) ASC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment @@ -140498,10 +157157,12 @@ static int fts3PendingTermsAdd( */ static int fts3PendingTermsDocid( Fts3Table *p, /* Full-text table handle */ + int bDelete, /* True if this op is a delete */ int iLangid, /* Language id of row being written */ sqlite_int64 iDocid /* Docid of row being written */ ){ assert( iLangid>=0 ); + assert( bDelete==1 || bDelete==0 ); /* TODO(shess) Explore whether partially flushing the buffer on ** forced-flush would provide better performance. I suspect that if @@ -140509,7 +157170,8 @@ static int fts3PendingTermsDocid( ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocid<=p->iPrevDocid + if( iDocidiPrevDocid + || (iDocid==p->iPrevDocid && p->bPrevDelete==0) || p->iPrevLangid!=iLangid || p->nPendingData>p->nMaxPendingData ){ @@ -140518,6 +157180,7 @@ static int fts3PendingTermsDocid( } p->iPrevDocid = iDocid; p->iPrevLangid = iLangid; + p->bPrevDelete = bDelete; return SQLITE_OK; } @@ -140707,7 +157370,8 @@ static void fts3DeleteTerms( if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; int iLangid = langidFromSelect(p, pSelect); - rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); + i64 iDocid = sqlite3_column_int64(pSelect, 0); + rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ int iCol = i-1; if( p->abNotindexed[iCol]==0 ){ @@ -140955,14 +157619,19 @@ static int fts3SegReaderNext( if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); - if( pElem==0 ){ - pReader->aNode = 0; - }else{ + sqlite3_free(pReader->aNode); + pReader->aNode = 0; + if( pElem ){ + char *aCopy; PendingList *pList = (PendingList *)fts3HashData(pElem); + int nCopy = pList->nData+1; pReader->zTerm = (char *)fts3HashKey(pElem); pReader->nTerm = fts3HashKeysize(pElem); - pReader->nNode = pReader->nDoclist = pList->nData + 1; - pReader->aNode = pReader->aDoclist = pList->aData; + aCopy = (char*)sqlite3_malloc(nCopy); + if( !aCopy ) return SQLITE_NOMEM; + memcpy(aCopy, pList->aData, nCopy); + pReader->nNode = pReader->nDoclist = nCopy; + pReader->aNode = pReader->aDoclist = aCopy; pReader->ppNextElem++; assert( pReader->aNode ); } @@ -141202,12 +157871,14 @@ SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( ** second argument. */ SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ - if( pReader && !fts3SegReaderIsPending(pReader) ){ - sqlite3_free(pReader->zTerm); + if( pReader ){ + if( !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + } if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); - sqlite3_blob_close(pReader->pBlob); } + sqlite3_blob_close(pReader->pBlob); } sqlite3_free(pReader); } @@ -141263,7 +157934,10 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( ** an array of pending terms by term. This occurs as part of flushing ** the contents of the pending-terms hash table to the database. */ -static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ +static int SQLITE_CDECL fts3CompareElemByTerm( + const void *lhs, + const void *rhs +){ char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); @@ -142720,8 +159394,8 @@ static int fts3PromoteSegments( if( bOk ){ int iIdx = 0; - sqlite3_stmt *pUpdate1; - sqlite3_stmt *pUpdate2; + sqlite3_stmt *pUpdate1 = 0; + sqlite3_stmt *pUpdate2 = 0; if( rc==SQLITE_OK ){ rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); @@ -142817,7 +159491,7 @@ static int fts3SegmentMerge( ** segment. The level of the new segment is equal to the numerically ** greatest segment level currently present in the database for this ** index. The idx of the new segment is always 0. */ - if( csr.nSegment==1 ){ + if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ rc = SQLITE_DONE; goto finished; } @@ -143079,7 +159753,8 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); if( rc==SQLITE_OK ){ int rc2; - sqlite3_bind_int(pAllLangid, 1, p->nIndex); + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ int i; int iLangid = sqlite3_column_int(pAllLangid, 0); @@ -143146,7 +159821,7 @@ static int fts3DoRebuild(Fts3Table *p){ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int iCol; int iLangid = langidFromSelect(p, pStmt); - rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); + rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ if( p->abNotindexed[iCol]==0 ){ @@ -144411,7 +161086,7 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ pHint->n = i; i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); i += fts3GetVarint32(&pHint->a[i], pnInput); - if( i!=nHint ) return SQLITE_CORRUPT_VTAB; + if( i!=nHint ) return FTS_CORRUPT_VTAB; return SQLITE_OK; } @@ -144458,10 +161133,11 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** set nSeg to -1. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - sqlite3_bind_int(pFindLevel, 1, nMin); + sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); - nSeg = nMin; + nSeg = sqlite3_column_int(pFindLevel, 1); + assert( nSeg>=2 ); }else{ nSeg = -1; } @@ -144779,7 +161455,8 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); if( rc==SQLITE_OK ){ int rc2; - sqlite3_bind_int(pAllLangid, 1, p->nIndex); + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ int iLangid = sqlite3_column_int(pAllLangid, 0); int i; @@ -144792,7 +161469,6 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ } /* This block calculates the checksum according to the %_content table */ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); if( rc==SQLITE_OK ){ sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; sqlite3_stmt *pStmt = 0; @@ -144889,7 +161565,7 @@ static int fts3DoIntegrityCheck( int rc; int bOk = 0; rc = fts3IntegrityCheck(p, &bOk); - if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB; + if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; return rc; } @@ -145251,7 +161927,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( } } if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ - rc = fts3PendingTermsDocid(p, iLangid, *pRowid); + rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); } if( rc==SQLITE_OK ){ assert( p->iPrevDocid==*pRowid ); @@ -145312,6 +161988,7 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ ****************************************************************************** */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* #include */ @@ -145327,6 +162004,8 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ #define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ #define FTS3_MATCHINFO_LCS 's' /* nCol values */ #define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ /* ** The default value for the second argument to matchinfo(). @@ -145388,9 +162067,22 @@ struct MatchInfo { int nCol; /* Number of columns in table */ int nPhrase; /* Number of matchable phrases in query */ sqlite3_int64 nDoc; /* Number of docs in database */ + char flag; u32 *aMatchinfo; /* Pre-allocated buffer */ }; +/* +** An instance of this structure is used to manage a pair of buffers, each +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below +** for details. +*/ +struct MatchinfoBuffer { + u8 aRef[3]; + int nElem; + int bGlobal; /* Set if global data is loaded */ + char *zMatchinfo; + u32 aMatchinfo[1]; +}; /* @@ -145406,6 +162098,97 @@ struct StrBuffer { }; +/************************************************************************* +** Start of MatchinfoBuffer code. +*/ + +/* +** Allocate a two-slot MatchinfoBuffer object. +*/ +static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ + MatchinfoBuffer *pRet; + int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer); + int nStr = (int)strlen(zMatchinfo); + + pRet = sqlite3_malloc(nByte + nStr+1); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); + pRet->nElem = nElem; + pRet->zMatchinfo = ((char*)pRet) + nByte; + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); + pRet->aRef[0] = 1; + } + + return pRet; +} + +static void fts3MIBufferFree(void *p){ + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); + + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + ); + if( (u32*)p==&pBuf->aMatchinfo[1] ){ + pBuf->aRef[1] = 0; + }else{ + pBuf->aRef[2] = 0; + } + + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ + sqlite3_free(pBuf); + } +} + +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ + void (*xRet)(void*) = 0; + u32 *aOut = 0; + + if( p->aRef[1]==0 ){ + p->aRef[1] = 1; + aOut = &p->aMatchinfo[1]; + xRet = fts3MIBufferFree; + } + else if( p->aRef[2]==0 ){ + p->aRef[2] = 1; + aOut = &p->aMatchinfo[p->nElem+2]; + xRet = fts3MIBufferFree; + }else{ + aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); + if( aOut ){ + xRet = sqlite3_free; + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + } + } + + *paOut = aOut; + return xRet; +} + +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ + p->bGlobal = 1; + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); +} + +/* +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() +*/ +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ + if( p ){ + assert( p->aRef[0]==1 ); + p->aRef[0] = 0; + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ + sqlite3_free(p); + } + } +} + +/* +** End of MatchinfoBuffer code. +*************************************************************************/ + + /* ** This function is used to help iterate through a position-list. A position ** list is a list of unique integers, sorted from smallest to largest. Each @@ -145442,7 +162225,7 @@ static int fts3ExprIterate2( void *pCtx /* Second argument to pass to callback */ ){ int rc; /* Return code */ - int eType = pExpr->eType; /* Type of expression node pExpr */ + int eType = pExpr->eType; /* Type of expression node pExpr */ if( eType!=FTSQUERY_PHRASE ){ assert( pExpr->pLeft && pExpr->pRight ); @@ -145476,6 +162259,7 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } + /* ** This is an fts3ExprIterate() callback used while loading the doclists ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also @@ -145520,8 +162304,7 @@ static int fts3ExprLoadDoclists( static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ (*(int *)ctx)++; - UNUSED_PARAMETER(pExpr); - UNUSED_PARAMETER(iPhrase); + pExpr->iPhrase = iPhrase; return SQLITE_OK; } static int fts3ExprPhraseCount(Fts3Expr *pExpr){ @@ -145742,37 +162525,39 @@ static int fts3BestSnippet( sIter.nSnippet = nSnippet; sIter.nPhrase = nList; sIter.iCurrent = -1; - (void)fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter); + rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); + if( rc==SQLITE_OK ){ - /* Set the *pmSeen output variable. */ - for(i=0; iiCol = iCol; - while( !fts3SnippetNextCandidate(&sIter) ){ - int iPos; - int iScore; - u64 mCover; - u64 mHighlight; - fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover, &mHighlight); - assert( iScore>=0 ); - if( iScore>iBestScore ){ - pFragment->iPos = iPos; - pFragment->hlmask = mHighlight; - pFragment->covered = mCover; - iBestScore = iScore; + /* Loop through all candidate snippets. Store the best snippet in + ** *pFragment. Store its associated 'score' in iBestScore. + */ + pFragment->iCol = iCol; + while( !fts3SnippetNextCandidate(&sIter) ){ + int iPos; + int iScore; + u64 mCover; + u64 mHighlite; + fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); + assert( iScore>=0 ); + if( iScore>iBestScore ){ + pFragment->iPos = iPos; + pFragment->hlmask = mHighlite; + pFragment->covered = mCover; + iBestScore = iScore; + } } - } + *piScore = iBestScore; + } sqlite3_free(sIter.aPhrase); - *piScore = iBestScore; - return SQLITE_OK; + return rc; } @@ -145980,8 +162765,12 @@ static int fts3SnippetText( ** required. They are required if (a) this is not the first fragment, ** or (b) this fragment does not begin at position 0 of its column. */ - if( rc==SQLITE_OK && (iPos>0 || iFragment>0) ){ - rc = fts3StringAppend(pOut, zEllipsis, -1); + if( rc==SQLITE_OK ){ + if( iPos>0 || iFragment>0 ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + }else if( iBegin ){ + rc = fts3StringAppend(pOut, zDoc, iBegin); + } } if( rc!=SQLITE_OK || iCurrentpCursor->base.pVtab; + int iStart; + Fts3Phrase *pPhrase = pExpr->pPhrase; + char *pIter = pPhrase->doclist.pList; + int iCol = 0; + + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); + if( p->flag==FTS3_MATCHINFO_LHITS ){ + iStart = pExpr->iPhrase * p->nCol; + }else{ + iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); + } + + while( 1 ){ + int nHit = fts3ColumnlistCount(&pIter); + if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ + if( p->flag==FTS3_MATCHINFO_LHITS ){ + p->aMatchinfo[iStart + iCol] = (u32)nHit; + }else if( nHit ){ + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); + } + } + assert( *pIter==0x00 || *pIter==0x01 ); + if( *pIter!=0x01 ) break; + pIter++; + pIter += fts3GetVarint32(pIter, &iCol); + } +} + +/* +** Gather the results for matchinfo directives 'y' and 'b'. +*/ +static void fts3ExprLHitGather( + Fts3Expr *pExpr, + MatchInfo *p +){ + assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); + if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ + if( pExpr->pLeft ){ + fts3ExprLHitGather(pExpr->pLeft, p); + fts3ExprLHitGather(pExpr->pRight, p); + }else{ + fts3ExprLHits(pExpr, p); + } + } +} + /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats ** for a single query. @@ -146115,10 +162958,12 @@ static int fts3MatchinfoCheck( || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) || (cArg==FTS3_MATCHINFO_LCS) || (cArg==FTS3_MATCHINFO_HITS) + || (cArg==FTS3_MATCHINFO_LHITS) + || (cArg==FTS3_MATCHINFO_LHITS_BM) ){ return SQLITE_OK; } - *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg); + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); return SQLITE_ERROR; } @@ -146138,6 +162983,14 @@ static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ nVal = pInfo->nCol; break; + case FTS3_MATCHINFO_LHITS: + nVal = pInfo->nCol * pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_LHITS_BM: + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + break; + default: assert( cArg==FTS3_MATCHINFO_HITS ); nVal = pInfo->nCol * pInfo->nPhrase * 3; @@ -146332,7 +163185,7 @@ static int fts3MatchinfoValues( sqlite3_stmt *pSelect = 0; for(i=0; rc==SQLITE_OK && zArg[i]; i++){ - + pInfo->flag = zArg[i]; switch( zArg[i] ){ case FTS3_MATCHINFO_NPHRASE: if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; @@ -146392,6 +163245,14 @@ static int fts3MatchinfoValues( } break; + case FTS3_MATCHINFO_LHITS_BM: + case FTS3_MATCHINFO_LHITS: { + int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + memset(pInfo->aMatchinfo, 0, nZero); + fts3ExprLHitGather(pCsr->pExpr, pInfo); + break; + } + default: { Fts3Expr *pExpr; assert( zArg[i]==FTS3_MATCHINFO_HITS ); @@ -146404,6 +163265,7 @@ static int fts3MatchinfoValues( if( rc!=SQLITE_OK ) break; } rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + sqlite3Fts3EvalTestDeferred(pCsr, &rc); if( rc!=SQLITE_OK ) break; } (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); @@ -146423,7 +163285,8 @@ static int fts3MatchinfoValues( ** Populate pCsr->aMatchinfo[] with data for the current row. The ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). */ -static int fts3GetMatchinfo( +static void fts3GetMatchinfo( + sqlite3_context *pCtx, /* Return results here */ Fts3Cursor *pCsr, /* FTS3 Cursor object */ const char *zArg /* Second argument to matchinfo() function */ ){ @@ -146432,6 +163295,9 @@ static int fts3GetMatchinfo( int rc = SQLITE_OK; int bGlobal = 0; /* Collect 'global' stats as well as local */ + u32 *aOut = 0; + void (*xDestroyOut)(void*) = 0; + memset(&sInfo, 0, sizeof(MatchInfo)); sInfo.pCursor = pCsr; sInfo.nCol = pTab->nColumn; @@ -146439,21 +163305,18 @@ static int fts3GetMatchinfo( /* If there is cached matchinfo() data, but the format string for the ** cache does not match the format string for this request, discard ** the cached data. */ - if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){ - assert( pCsr->aMatchinfo ); - sqlite3_free(pCsr->aMatchinfo); - pCsr->zMatchinfo = 0; - pCsr->aMatchinfo = 0; + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + pCsr->pMIBuffer = 0; } - /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the ** matchinfo function has been called for this query. In this case ** allocate the array used to accumulate the matchinfo data and ** initialize those elements that are constant for every row. */ - if( pCsr->aMatchinfo==0 ){ + if( pCsr->pMIBuffer==0 ){ int nMatchinfo = 0; /* Number of u32 elements in match-info */ - int nArg; /* Bytes in zArg */ int i; /* Used to iterate through zArg */ /* Determine the number of phrases in the query */ @@ -146462,30 +163325,46 @@ static int fts3GetMatchinfo( /* Determine the number of integers in the buffer returned by this call. */ for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); } /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ - nArg = (int)strlen(zArg); - pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1); - if( !pCsr->aMatchinfo ) return SQLITE_NOMEM; + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; - pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo]; - pCsr->nMatchinfo = nMatchinfo; - memcpy(pCsr->zMatchinfo, zArg, nArg+1); - memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo); pCsr->isMatchinfoNeeded = 1; bGlobal = 1; } - sInfo.aMatchinfo = pCsr->aMatchinfo; - sInfo.nPhrase = pCsr->nPhrase; - if( pCsr->isMatchinfoNeeded ){ - rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); - pCsr->isMatchinfoNeeded = 0; + if( rc==SQLITE_OK ){ + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); + if( xDestroyOut==0 ){ + rc = SQLITE_NOMEM; + } } - return rc; + if( rc==SQLITE_OK ){ + sInfo.aMatchinfo = aOut; + sInfo.nPhrase = pCsr->nPhrase; + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); + if( bGlobal ){ + fts3MIBufferSetGlobal(pCsr->pMIBuffer); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + if( xDestroyOut ) xDestroyOut(aOut); + }else{ + int n = pCsr->pMIBuffer->nElem * sizeof(u32); + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); + } } /* @@ -146547,7 +163426,7 @@ SQLITE_PRIVATE void sqlite3Fts3Snippet( */ for(iRead=0; iReadnColumn; iRead++){ SnippetFragment sF = {0, 0, 0, 0}; - int iS; + int iS = 0; if( iCol>=0 && iRead!=iCol ) continue; /* Find the best snippet of nFToken tokens in column iRead. */ @@ -146691,7 +163570,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( */ sCtx.iCol = iCol; sCtx.iTerm = 0; - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx); + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); /* Retreive the text stored in column iCol. If an SQL NULL is stored ** in column iCol, jump immediately to the next iteration of the loop. @@ -146783,19 +163662,9 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo( const char *zArg /* Second arg to matchinfo() function */ ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc; - int i; const char *zFormat; if( zArg ){ - for(i=0; zArg[i]; i++){ - char *zErr = 0; - if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ - sqlite3_result_error(pContext, zErr, -1); - sqlite3_free(zErr); - return; - } - } zFormat = zArg; }else{ zFormat = FTS3_MATCHINFO_DEFAULT; @@ -146804,17 +163673,10 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo( if( !pCsr->pExpr ){ sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); return; - } - - /* Retrieve matchinfo() data. */ - rc = fts3GetMatchinfo(pCsr, zFormat); - sqlite3Fts3SegmentsClose(pTab); - - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pContext, rc); }else{ - int n = pCsr->nMatchinfo * sizeof(u32); - sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); + /* Retrieve matchinfo() data. */ + fts3GetMatchinfo(pContext, pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); } } @@ -146839,6 +163701,7 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo( #ifndef SQLITE_DISABLE_FTS3_UNICODE +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* #include */ @@ -146846,6 +163709,7 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo( /* #include */ /* #include */ +/* #include "fts3_tokenizer.h" */ /* ** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied @@ -147641,8 +164505,10 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) #ifndef SQLITE_CORE +/* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #else +/* #include "sqlite3.h" */ #endif /* #include */ @@ -147935,6 +164801,7 @@ struct RtreeMatchArg { u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ RtreeGeomCallback cb; /* Info about the callback functions */ int nParam; /* Number of parameters to the SQL function */ + sqlite3_value **apSqlParam; /* Original SQL parameter values */ RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ }; @@ -147953,13 +164820,12 @@ static int readInt16(u8 *p){ return (p[0]<<8) + p[1]; } static void readCoord(u8 *p, RtreeCoord *pCoord){ - u32 i = ( + pCoord->u = ( (((u32)p[0]) << 24) + (((u32)p[1]) << 16) + (((u32)p[2]) << 8) + (((u32)p[3]) << 0) ); - *(u32 *)pCoord = i; } static i64 readInt64(u8 *p){ return ( @@ -147988,7 +164854,7 @@ static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); - i = *(u32 *)pCoord; + i = pCoord->u; p[0] = (i>>24)&0xFF; p[1] = (i>>16)&0xFF; p[2] = (i>> 8)&0xFF; @@ -148319,14 +165185,13 @@ static void nodeGetCell( RtreeCell *pCell /* OUT: Write the cell contents here */ ){ u8 *pData; - u8 *pEnd; RtreeCoord *pCoord; + int ii; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); - pEnd = pData + pRtree->nDim*8; pCoord = pCell->aCoord; - for(; pDatanDim*2; ii++){ + readCoord(&pData[ii*4], &pCoord[ii]); } } @@ -148766,7 +165631,7 @@ static RtreeSearchPoint *rtreeEnqueue( pNew = pCur->aPoint + i; pNew->rScore = rScore; pNew->iLevel = iLevel; - assert( iLevel>=0 && iLevel<=RTREE_MAX_DEPTH ); + assert( iLevel<=RTREE_MAX_DEPTH ); while( i>0 ){ RtreeSearchPoint *pParent; j = (i-1)/2; @@ -149068,9 +165933,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ /* Check that the blob is roughly the right size. */ nBlob = sqlite3_value_bytes(pValue); - if( nBlob<(int)sizeof(RtreeMatchArg) - || ((nBlob-sizeof(RtreeMatchArg))%sizeof(RtreeDValue))!=0 - ){ + if( nBlob<(int)sizeof(RtreeMatchArg) ){ return SQLITE_ERROR; } @@ -149081,6 +165944,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); nExpected = (int)(sizeof(RtreeMatchArg) + + pBlob->nParam*sizeof(sqlite3_value*) + (pBlob->nParam-1)*sizeof(RtreeDValue)); if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ sqlite3_free(pInfo); @@ -149089,6 +165953,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ pInfo->pContext = pBlob->cb.pContext; pInfo->nParam = pBlob->nParam; pInfo->aParam = pBlob->aParam; + pInfo->apSqlParam = pBlob->apSqlParam; if( pBlob->cb.xGeom ){ pCons->u.xGeom = pBlob->cb.xGeom; @@ -149127,7 +165992,7 @@ static int rtreeFilter( if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ - RtreeSearchPoint *p; /* Search point for the the leaf */ + RtreeSearchPoint *p; /* Search point for the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); i64 iNode = 0; rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); @@ -149255,17 +166120,30 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ Rtree *pRtree = (Rtree*)tab; int rc = SQLITE_OK; int ii; + int bMatch = 0; /* True if there exists a MATCH constraint */ i64 nRow; /* Estimated rows returned by this scan */ int iIdx = 0; char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; memset(zIdxStr, 0, sizeof(zIdxStr)); + /* Check if there exists a MATCH constraint - even an unusable one. If there + ** is, do not consider the lookup-by-rowid plan as using such a plan would + ** require the VDBE to evaluate the MATCH constraint, which is not currently + ** possible. */ + for(ii=0; iinConstraint; ii++){ + if( pIdxInfo->aConstraint[ii].op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + bMatch = 1; + } + } + assert( pIdxInfo->idxStr==0 ); for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; - if( p->usable && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + if( bMatch==0 && p->usable + && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ /* We have an equality constraint on the rowid. Use strategy 1. */ int jj; for(jj=0; jjnRowEst / (iIdx + 1); + nRow = pRtree->nRowEst >> (iIdx/2); pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; setEstimatedRows(pIdxInfo, nRow); @@ -150372,6 +167250,53 @@ static RtreeValue rtreeValueUp(sqlite3_value *v){ } #endif /* !defined(SQLITE_RTREE_INT_ONLY) */ +/* +** A constraint has failed while inserting a row into an rtree table. +** Assuming no OOM error occurs, this function sets the error message +** (at pRtree->base.zErrMsg) to an appropriate value and returns +** SQLITE_CONSTRAINT. +** +** Parameter iCol is the index of the leftmost column involved in the +** constraint failure. If it is 0, then the constraint that failed is +** the unique constraint on the id column. Otherwise, it is the rtree +** (c1<=c2) constraint on columns iCol and iCol+1 that has failed. +** +** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT. +*/ +static int rtreeConstraintError(Rtree *pRtree, int iCol){ + sqlite3_stmt *pStmt = 0; + char *zSql; + int rc; + + assert( iCol==0 || iCol%2 ); + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName); + if( zSql ){ + rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + if( iCol==0 ){ + const char *zCol = sqlite3_column_name(pStmt, 0); + pRtree->base.zErrMsg = sqlite3_mprintf( + "UNIQUE constraint failed: %s.%s", pRtree->zName, zCol + ); + }else{ + const char *zCol1 = sqlite3_column_name(pStmt, iCol); + const char *zCol2 = sqlite3_column_name(pStmt, iCol+1); + pRtree->base.zErrMsg = sqlite3_mprintf( + "rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2 + ); + } + } + + sqlite3_finalize(pStmt); + return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc); +} + + /* ** The xUpdate method for rtree module virtual tables. @@ -150390,6 +167315,8 @@ static int rtreeUpdate( rtreeReference(pRtree); assert(nData>=1); + cell.iRowid = 0; /* Used only to suppress a compiler warning */ + /* Constraint handling. A write operation on an r-tree table may return ** SQLITE_CONSTRAINT for two reasons: ** @@ -150404,26 +167331,34 @@ static int rtreeUpdate( if( nData>1 ){ int ii; - /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */ - assert( nData==(pRtree->nDim*2 + 3) ); + /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. + ** + ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared + ** with "column" that are interpreted as table constraints. + ** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5)); + ** This problem was discovered after years of use, so we silently ignore + ** these kinds of misdeclared tables to avoid breaking any legacy. + */ + assert( nData<=(pRtree->nDim*2 + 3) ); + #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + for(ii=0; iicell.aCoord[ii+1].f ){ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, ii+1); goto constraint; } } }else #endif { - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + for(ii=0; iicell.aCoord[ii+1].i ){ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, ii+1); goto constraint; } } @@ -150444,7 +167379,7 @@ static int rtreeUpdate( if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ rc = rtreeDeleteRowid(pRtree, cell.iRowid); }else{ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, 0); goto constraint; } } @@ -150527,6 +167462,13 @@ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ int rc; i64 nRow = 0; + rc = sqlite3_table_column_metadata( + db, pRtree->zDb, "sqlite_stat1",0,0,0,0,0,0 + ); + if( rc!=SQLITE_OK ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + return rc==SQLITE_ERROR ? SQLITE_OK : rc; + } zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); if( zSql==0 ){ rc = SQLITE_NOMEM; @@ -150948,6 +167890,18 @@ static void rtreeFreeCallback(void *p){ sqlite3_free(p); } +/* +** This routine frees the BLOB that is returned by geomCallback(). +*/ +static void rtreeMatchArgFree(void *pArg){ + int i; + RtreeMatchArg *p = (RtreeMatchArg*)pArg; + for(i=0; inParam; i++){ + sqlite3_value_free(p->apSqlParam[i]); + } + sqlite3_free(p); +} + /* ** Each call to sqlite3_rtree_geometry_callback() or ** sqlite3_rtree_query_callback() creates an ordinary SQLite @@ -150966,8 +167920,10 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); RtreeMatchArg *pBlob; int nBlob; + int memErr = 0; - nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue); + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue) + + nArg*sizeof(sqlite3_value*); pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); if( !pBlob ){ sqlite3_result_error_nomem(ctx); @@ -150975,15 +167931,23 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ int i; pBlob->magic = RTREE_GEOMETRY_MAGIC; pBlob->cb = pGeomCtx[0]; + pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg]; pBlob->nParam = nArg; for(i=0; iapSqlParam[i] = sqlite3_value_dup(aArg[i]); + if( pBlob->apSqlParam[i]==0 ) memErr = 1; #ifdef SQLITE_RTREE_INT_ONLY pBlob->aParam[i] = sqlite3_value_int64(aArg[i]); #else pBlob->aParam[i] = sqlite3_value_double(aArg[i]); #endif } - sqlite3_result_blob(ctx, pBlob, nBlob, sqlite3_free); + if( memErr ){ + sqlite3_result_error_nomem(ctx); + rtreeMatchArgFree(pBlob); + }else{ + sqlite3_result_blob(ctx, pBlob, nBlob, rtreeMatchArgFree); + } } } @@ -151094,8 +168058,10 @@ SQLITE_API int sqlite3_rtree_init( /* #include */ #ifndef SQLITE_CORE +/* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #else +/* #include "sqlite3.h" */ #endif /* @@ -151113,6 +168079,38 @@ static void xFree(void *p){ sqlite3_free(p); } +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. It is copied here from SQLite source +** code file utf8.c. +*/ +static const unsigned char icuUtf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define SQLITE_ICU_READ_UTF8(zIn, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = icuUtf8Trans1[c-0xc0]; \ + while( (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + } + +#define SQLITE_ICU_SKIP_UTF8(zIn) \ + assert( *zIn ); \ + if( *(zIn++)>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){zIn++;} \ + } + + /* ** Compare two UTF-8 strings for equality where the first string is ** a "LIKE" expression. Return true (1) if they are the same and @@ -151126,17 +168124,14 @@ static int icuLikeCompare( static const int MATCH_ONE = (UChar32)'_'; static const int MATCH_ALL = (UChar32)'%'; - int iPattern = 0; /* Current byte index in zPattern */ - int iString = 0; /* Current byte index in zString */ - int prevEscape = 0; /* True if the previous character was uEsc */ - while( zPattern[iPattern]!=0 ){ + while( 1 ){ /* Read (and consume) the next character from the input pattern. */ UChar32 uPattern; - U8_NEXT_UNSAFE(zPattern, iPattern, uPattern); - assert(uPattern!=0); + SQLITE_ICU_READ_UTF8(zPattern, uPattern); + if( uPattern==0 ) break; /* There are now 4 possibilities: ** @@ -151153,28 +168148,28 @@ static int icuLikeCompare( ** MATCH_ALL. For each MATCH_ONE, skip one character in the ** test string. */ - while( (c=zPattern[iPattern]) == MATCH_ALL || c == MATCH_ONE ){ + while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ if( c==MATCH_ONE ){ - if( zString[iString]==0 ) return 0; - U8_FWD_1_UNSAFE(zString, iString); + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); } - iPattern++; + zPattern++; } - if( zPattern[iPattern]==0 ) return 1; + if( *zPattern==0 ) return 1; - while( zString[iString] ){ - if( icuLikeCompare(&zPattern[iPattern], &zString[iString], uEsc) ){ + while( *zString ){ + if( icuLikeCompare(zPattern, zString, uEsc) ){ return 1; } - U8_FWD_1_UNSAFE(zString, iString); + SQLITE_ICU_SKIP_UTF8(zString); } return 0; }else if( !prevEscape && uPattern==MATCH_ONE ){ /* Case 2. */ - if( zString[iString]==0 ) return 0; - U8_FWD_1_UNSAFE(zString, iString); + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); }else if( !prevEscape && uPattern==uEsc){ /* Case 3. */ @@ -151183,7 +168178,7 @@ static int icuLikeCompare( }else{ /* Case 4. */ UChar32 uString; - U8_NEXT_UNSAFE(zString, iString, uString); + SQLITE_ICU_READ_UTF8(zString, uString); uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT); uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT); if( uString!=uPattern ){ @@ -151193,7 +168188,7 @@ static int icuLikeCompare( } } - return zString[iString]==0; + return *zString==0; } /* @@ -151373,20 +168368,22 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ ** of upper() or lower(). ** ** lower('I', 'en_us') -> 'i' -** lower('I', 'tr_tr') -> 'ı' (small dotless i) +** lower('I', 'tr_tr') -> '\u131' (small dotless i) ** ** http://www.icu-project.org/userguide/posix.html#case_mappings */ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ - const UChar *zInput; - UChar *zOutput; - int nInput; - int nOutput; - - UErrorCode status = U_ZERO_ERROR; + const UChar *zInput; /* Pointer to input string */ + UChar *zOutput = 0; /* Pointer to output buffer */ + int nInput; /* Size of utf-16 input string in bytes */ + int nOut; /* Size of output buffer in bytes */ + int cnt; + int bToUpper; /* True for toupper(), false for tolower() */ + UErrorCode status; const char *zLocale = 0; assert(nArg==1 || nArg==2); + bToUpper = (sqlite3_user_data(p)!=0); if( nArg==2 ){ zLocale = (const char *)sqlite3_value_text(apArg[1]); } @@ -151395,26 +168392,38 @@ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ if( !zInput ){ return; } - nInput = sqlite3_value_bytes16(apArg[0]); - - nOutput = nInput * 2 + 2; - zOutput = sqlite3_malloc(nOutput); - if( !zOutput ){ + nOut = nInput = sqlite3_value_bytes16(apArg[0]); + if( nOut==0 ){ + sqlite3_result_text16(p, "", 0, SQLITE_STATIC); return; } - if( sqlite3_user_data(p) ){ - u_strToUpper(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); - }else{ - u_strToLower(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); - } + for(cnt=0; cnt<2; cnt++){ + UChar *zNew = sqlite3_realloc(zOutput, nOut); + if( zNew==0 ){ + sqlite3_free(zOutput); + sqlite3_result_error_nomem(p); + return; + } + zOutput = zNew; + status = U_ZERO_ERROR; + if( bToUpper ){ + nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + }else{ + nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + } - if( !U_SUCCESS(status) ){ - icuFunctionError(p, "u_strToLower()/u_strToUpper", status); + if( U_SUCCESS(status) ){ + sqlite3_result_text16(p, zOutput, nOut, xFree); + }else if( status==U_BUFFER_OVERFLOW_ERROR ){ + assert( cnt==0 ); + continue; + }else{ + icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); + } return; } - - sqlite3_result_text16(p, zOutput, -1, xFree); + assert( 0 ); /* Unreachable */ } /* @@ -151475,6 +168484,7 @@ static void icuLoadCollation( int rc; /* Return code from sqlite3_create_collation_x() */ assert(nArg==2); + (void)nArg; /* Unused parameter */ zLocale = (const char *)sqlite3_value_text(apArg[0]); zName = (const char *)sqlite3_value_text(apArg[1]); @@ -151509,20 +168519,20 @@ SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ void *pContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { - {"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc}, + {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, - {"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF16, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF16, (void*)1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"lower", 1, SQLITE_UTF8, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF8, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF8, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF8, (void*)1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"like", 2, SQLITE_UTF8, 0, icuLikeFunc}, - {"like", 3, SQLITE_UTF8, 0, icuLikeFunc}, + {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, }; @@ -151571,11 +168581,13 @@ SQLITE_API int sqlite3_icu_init( ************************************************************************* ** This file implements a tokenizer for fts3 based on the ICU library. */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_ENABLE_ICU /* #include */ /* #include */ +/* #include "fts3_tokenizer.h" */ #include /* #include */ @@ -151798,12 +168810,13 @@ static int icuNext( ** The set of routines that implement the simple tokenizer */ static const sqlite3_tokenizer_module icuTokenizerModule = { - 0, /* iVersion */ - icuCreate, /* xCreate */ - icuDestroy, /* xCreate */ - icuOpen, /* xOpen */ - icuClose, /* xClose */ - icuNext, /* xNext */ + 0, /* iVersion */ + icuCreate, /* xCreate */ + icuDestroy, /* xCreate */ + icuOpen, /* xOpen */ + icuClose, /* xClose */ + icuNext, /* xNext */ + 0, /* xLanguageid */ }; /* @@ -151819,3 +168832,33804 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ /************** End of fts3_icu.c ********************************************/ +/************** Begin file sqlite3rbu.c **************************************/ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** +** OVERVIEW +** +** The RBU extension requires that the RBU update be packaged as an +** SQLite database. The tables it expects to find are described in +** sqlite3rbu.h. Essentially, for each table xyz in the target database +** that the user wishes to write to, a corresponding data_xyz table is +** created in the RBU database and populated with one row for each row to +** update, insert or delete from the target table. +** +** The update proceeds in three stages: +** +** 1) The database is updated. The modified database pages are written +** to a *-oal file. A *-oal file is just like a *-wal file, except +** that it is named "-oal" instead of "-wal". +** Because regular SQLite clients do not look for file named +** "-oal", they go on using the original database in +** rollback mode while the *-oal file is being generated. +** +** During this stage RBU does not update the database by writing +** directly to the target tables. Instead it creates "imposter" +** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses +** to update each b-tree individually. All updates required by each +** b-tree are completed before moving on to the next, and all +** updates are done in sorted key order. +** +** 2) The "-oal" file is moved to the equivalent "-wal" +** location using a call to rename(2). Before doing this the RBU +** module takes an EXCLUSIVE lock on the database file, ensuring +** that there are no other active readers. +** +** Once the EXCLUSIVE lock is released, any other database readers +** detect the new *-wal file and read the database in wal mode. At +** this point they see the new version of the database - including +** the updates made as part of the RBU update. +** +** 3) The new *-wal file is checkpointed. This proceeds in the same way +** as a regular database checkpoint, except that a single frame is +** checkpointed each time sqlite3rbu_step() is called. If the RBU +** handle is closed before the entire *-wal file is checkpointed, +** the checkpoint progress is saved in the RBU database and the +** checkpoint can be resumed by another RBU client at some point in +** the future. +** +** POTENTIAL PROBLEMS +** +** The rename() call might not be portable. And RBU is not currently +** syncing the directory after renaming the file. +** +** When state is saved, any commit to the *-oal file and the commit to +** the RBU update database are not atomic. So if the power fails at the +** wrong moment they might get out of sync. As the main database will be +** committed before the RBU update database this will likely either just +** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE +** constraint violations). +** +** If some client does modify the target database mid RBU update, or some +** other error occurs, the RBU extension will keep throwing errors. It's +** not really clear how to get out of this state. The system could just +** by delete the RBU update database and *-oal file and have the device +** download the update again and start over. +** +** At present, for an UPDATE, both the new.* and old.* records are +** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all +** fields are collected. This means we're probably writing a lot more +** data to disk when saving the state of an ongoing update to the RBU +** update database than is strictly necessary. +** +*/ + +/* #include */ +/* #include */ +/* #include */ + +/* #include "sqlite3.h" */ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) +/************** Include sqlite3rbu.h in the middle of sqlite3rbu.c ***********/ +/************** Begin file sqlite3rbu.h **************************************/ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the public interface for the RBU extension. +*/ + +/* +** SUMMARY +** +** Writing a transaction containing a large number of operations on +** b-tree indexes that are collectively larger than the available cache +** memory can be very inefficient. +** +** The problem is that in order to update a b-tree, the leaf page (at least) +** containing the entry being inserted or deleted must be modified. If the +** working set of leaves is larger than the available cache memory, then a +** single leaf that is modified more than once as part of the transaction +** may be loaded from or written to the persistent media multiple times. +** Additionally, because the index updates are likely to be applied in +** random order, access to pages within the database is also likely to be in +** random order, which is itself quite inefficient. +** +** One way to improve the situation is to sort the operations on each index +** by index key before applying them to the b-tree. This leads to an IO +** pattern that resembles a single linear scan through the index b-tree, +** and all but guarantees each modified leaf page is loaded and stored +** exactly once. SQLite uses this trick to improve the performance of +** CREATE INDEX commands. This extension allows it to be used to improve +** the performance of large transactions on existing databases. +** +** Additionally, this extension allows the work involved in writing the +** large transaction to be broken down into sub-transactions performed +** sequentially by separate processes. This is useful if the system cannot +** guarantee that a single update process will run for long enough to apply +** the entire update, for example because the update is being applied on a +** mobile device that is frequently rebooted. Even after the writer process +** has committed one or more sub-transactions, other database clients continue +** to read from the original database snapshot. In other words, partially +** applied transactions are not visible to other clients. +** +** "RBU" stands for "Resumable Bulk Update". As in a large database update +** transmitted via a wireless network to a mobile device. A transaction +** applied using this extension is hence refered to as an "RBU update". +** +** +** LIMITATIONS +** +** An "RBU update" transaction is subject to the following limitations: +** +** * The transaction must consist of INSERT, UPDATE and DELETE operations +** only. +** +** * INSERT statements may not use any default values. +** +** * UPDATE and DELETE statements must identify their target rows by +** non-NULL PRIMARY KEY values. Rows with NULL values stored in PRIMARY +** KEY fields may not be updated or deleted. If the table being written +** has no PRIMARY KEY, affected rows must be identified by rowid. +** +** * UPDATE statements may not modify PRIMARY KEY columns. +** +** * No triggers will be fired. +** +** * No foreign key violations are detected or reported. +** +** * CHECK constraints are not enforced. +** +** * No constraint handling mode except for "OR ROLLBACK" is supported. +** +** +** PREPARATION +** +** An "RBU update" is stored as a separate SQLite database. A database +** containing an RBU update is an "RBU database". For each table in the +** target database to be updated, the RBU database should contain a table +** named "data_" containing the same set of columns as the +** target table, and one more - "rbu_control". The data_% table should +** have no PRIMARY KEY or UNIQUE constraints, but each column should have +** the same type as the corresponding column in the target database. +** The "rbu_control" column should have no type at all. For example, if +** the target database contains: +** +** CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c UNIQUE); +** +** Then the RBU database should contain: +** +** CREATE TABLE data_t1(a INTEGER, b TEXT, c, rbu_control); +** +** The order of the columns in the data_% table does not matter. +** +** Instead of a regular table, the RBU database may also contain virtual +** tables or view named using the data_ naming scheme. +** +** Instead of the plain data_ naming scheme, RBU database tables +** may also be named data_, where is any sequence +** of zero or more numeric characters (0-9). This can be significant because +** tables within the RBU database are always processed in order sorted by +** name. By judicious selection of the portion of the names +** of the RBU tables the user can therefore control the order in which they +** are processed. This can be useful, for example, to ensure that "external +** content" FTS4 tables are updated before their underlying content tables. +** +** If the target database table is a virtual table or a table that has no +** PRIMARY KEY declaration, the data_% table must also contain a column +** named "rbu_rowid". This column is mapped to the tables implicit primary +** key column - "rowid". Virtual tables for which the "rowid" column does +** not function like a primary key value cannot be updated using RBU. For +** example, if the target db contains either of the following: +** +** CREATE VIRTUAL TABLE x1 USING fts3(a, b); +** CREATE TABLE x1(a, b) +** +** then the RBU database should contain: +** +** CREATE TABLE data_x1(a, b, rbu_rowid, rbu_control); +** +** All non-hidden columns (i.e. all columns matched by "SELECT *") of the +** target table must be present in the input table. For virtual tables, +** hidden columns are optional - they are updated by RBU if present in +** the input table, or not otherwise. For example, to write to an fts4 +** table with a hidden languageid column such as: +** +** CREATE VIRTUAL TABLE ft1 USING fts4(a, b, languageid='langid'); +** +** Either of the following input table schemas may be used: +** +** CREATE TABLE data_ft1(a, b, langid, rbu_rowid, rbu_control); +** CREATE TABLE data_ft1(a, b, rbu_rowid, rbu_control); +** +** For each row to INSERT into the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain integer value 0. The +** other columns should be set to the values that make up the new record +** to insert. +** +** If the target database table has an INTEGER PRIMARY KEY, it is not +** possible to insert a NULL value into the IPK column. Attempting to +** do so results in an SQLITE_MISMATCH error. +** +** For each row to DELETE from the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain integer value 1. The +** real primary key values of the row to delete should be stored in the +** corresponding columns of the data_% table. The values stored in the +** other columns are not used. +** +** For each row to UPDATE from the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain a value of type text. +** The real primary key values identifying the row to update should be +** stored in the corresponding columns of the data_% table row, as should +** the new values of all columns being update. The text value in the +** "rbu_control" column must contain the same number of characters as +** there are columns in the target database table, and must consist entirely +** of 'x' and '.' characters (or in some special cases 'd' - see below). For +** each column that is being updated, the corresponding character is set to +** 'x'. For those that remain as they are, the corresponding character of the +** rbu_control value should be set to '.'. For example, given the tables +** above, the update statement: +** +** UPDATE t1 SET c = 'usa' WHERE a = 4; +** +** is represented by the data_t1 row created by: +** +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..x'); +** +** Instead of an 'x' character, characters of the rbu_control value specified +** for UPDATEs may also be set to 'd'. In this case, instead of updating the +** target table with the value stored in the corresponding data_% column, the +** user-defined SQL function "rbu_delta()" is invoked and the result stored in +** the target table column. rbu_delta() is invoked with two arguments - the +** original value currently stored in the target table column and the +** value specified in the data_xxx table. +** +** For example, this row: +** +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..d'); +** +** is similar to an UPDATE statement such as: +** +** UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4; +** +** Finally, if an 'f' character appears in place of a 'd' or 's' in an +** ota_control string, the contents of the data_xxx table column is assumed +** to be a "fossil delta" - a patch to be applied to a blob value in the +** format used by the fossil source-code management system. In this case +** the existing value within the target database table must be of type BLOB. +** It is replaced by the result of applying the specified fossil delta to +** itself. +** +** If the target database table is a virtual table or a table with no PRIMARY +** KEY, the rbu_control value should not include a character corresponding +** to the rbu_rowid value. For example, this: +** +** INSERT INTO data_ft1(a, b, rbu_rowid, rbu_control) +** VALUES(NULL, 'usa', 12, '.x'); +** +** causes a result similar to: +** +** UPDATE ft1 SET b = 'usa' WHERE rowid = 12; +** +** The data_xxx tables themselves should have no PRIMARY KEY declarations. +** However, RBU is more efficient if reading the rows in from each data_xxx +** table in "rowid" order is roughly the same as reading them sorted by +** the PRIMARY KEY of the corresponding target database table. In other +** words, rows should be sorted using the destination table PRIMARY KEY +** fields before they are inserted into the data_xxx tables. +** +** USAGE +** +** The API declared below allows an application to apply an RBU update +** stored on disk to an existing target database. Essentially, the +** application: +** +** 1) Opens an RBU handle using the sqlite3rbu_open() function. +** +** 2) Registers any required virtual table modules with the database +** handle returned by sqlite3rbu_db(). Also, if required, register +** the rbu_delta() implementation. +** +** 3) Calls the sqlite3rbu_step() function one or more times on +** the new handle. Each call to sqlite3rbu_step() performs a single +** b-tree operation, so thousands of calls may be required to apply +** a complete update. +** +** 4) Calls sqlite3rbu_close() to close the RBU update handle. If +** sqlite3rbu_step() has been called enough times to completely +** apply the update to the target database, then the RBU database +** is marked as fully applied. Otherwise, the state of the RBU +** update application is saved in the RBU database for later +** resumption. +** +** See comments below for more detail on APIs. +** +** If an update is only partially applied to the target database by the +** time sqlite3rbu_close() is called, various state information is saved +** within the RBU database. This allows subsequent processes to automatically +** resume the RBU update from where it left off. +** +** To remove all RBU extension state information, returning an RBU database +** to its original contents, it is sufficient to drop all tables that begin +** with the prefix "rbu_" +** +** DATABASE LOCKING +** +** An RBU update may not be applied to a database in WAL mode. Attempting +** to do so is an error (SQLITE_ERROR). +** +** While an RBU handle is open, a SHARED lock may be held on the target +** database file. This means it is possible for other clients to read the +** database, but not to write it. +** +** If an RBU update is started and then suspended before it is completed, +** then an external client writes to the database, then attempting to resume +** the suspended RBU update is also an error (SQLITE_BUSY). +*/ + +#ifndef _SQLITE3RBU_H +#define _SQLITE3RBU_H + +/* #include "sqlite3.h" ** Required for error code definitions ** */ + +#if 0 +extern "C" { +#endif + +typedef struct sqlite3rbu sqlite3rbu; + +/* +** Open an RBU handle. +** +** Argument zTarget is the path to the target database. Argument zRbu is +** the path to the RBU database. Each call to this function must be matched +** by a call to sqlite3rbu_close(). When opening the databases, RBU passes +** the SQLITE_CONFIG_URI flag to sqlite3_open_v2(). So if either zTarget +** or zRbu begin with "file:", it will be interpreted as an SQLite +** database URI, not a regular file name. +** +** If the zState argument is passed a NULL value, the RBU extension stores +** the current state of the update (how many rows have been updated, which +** indexes are yet to be updated etc.) within the RBU database itself. This +** can be convenient, as it means that the RBU application does not need to +** organize removing a separate state file after the update is concluded. +** Or, if zState is non-NULL, it must be a path to a database file in which +** the RBU extension can store the state of the update. +** +** When resuming an RBU update, the zState argument must be passed the same +** value as when the RBU update was started. +** +** Once the RBU update is finished, the RBU extension does not +** automatically remove any zState database file, even if it created it. +** +** By default, RBU uses the default VFS to access the files on disk. To +** use a VFS other than the default, an SQLite "file:" URI containing a +** "vfs=..." option may be passed as the zTarget option. +** +** IMPORTANT NOTE FOR ZIPVFS USERS: The RBU extension works with all of +** SQLite's built-in VFSs, including the multiplexor VFS. However it does +** not work out of the box with zipvfs. Refer to the comment describing +** the zipvfs_create_vfs() API below for details on using RBU with zipvfs. +*/ +SQLITE_API sqlite3rbu *sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +); + +/* +** Open an RBU handle to perform an RBU vacuum on database file zTarget. +** An RBU vacuum is similar to SQLite's built-in VACUUM command, except +** that it can be suspended and resumed like an RBU update. +** +** The second argument to this function identifies a database in which +** to store the state of the RBU vacuum operation if it is suspended. The +** first time sqlite3rbu_vacuum() is called, to start an RBU vacuum +** operation, the state database should either not exist or be empty +** (contain no tables). If an RBU vacuum is suspended by calling +** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has +** returned SQLITE_DONE, the vacuum state is stored in the state database. +** The vacuum can be resumed by calling this function to open a new RBU +** handle specifying the same target and state databases. +** +** If the second argument passed to this function is NULL, then the +** name of the state database is "-vacuum", where +** is the name of the target database file. In this case, on UNIX, if the +** state database is not already present in the file-system, it is created +** with the same permissions as the target db is made. +** +** This function does not delete the state database after an RBU vacuum +** is completed, even if it created it. However, if the call to +** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents +** of the state tables within the state database are zeroed. This way, +** the next call to sqlite3rbu_vacuum() opens a handle that starts a +** new RBU vacuum operation. +** +** As with sqlite3rbu_open(), Zipvfs users should rever to the comment +** describing the sqlite3rbu_create_vfs() API function below for +** a description of the complications associated with using RBU with +** zipvfs databases. +*/ +SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +); + +/* +** Internally, each RBU connection uses a separate SQLite database +** connection to access the target and rbu update databases. This +** API allows the application direct access to these database handles. +** +** The first argument passed to this function must be a valid, open, RBU +** handle. The second argument should be passed zero to access the target +** database handle, or non-zero to access the rbu update database handle. +** Accessing the underlying database handles may be useful in the +** following scenarios: +** +** * If any target tables are virtual tables, it may be necessary to +** call sqlite3_create_module() on the target database handle to +** register the required virtual table implementations. +** +** * If the data_xxx tables in the RBU source database are virtual +** tables, the application may need to call sqlite3_create_module() on +** the rbu update db handle to any required virtual table +** implementations. +** +** * If the application uses the "rbu_delta()" feature described above, +** it must use sqlite3_create_function() or similar to register the +** rbu_delta() implementation with the target database handle. +** +** If an error has occurred, either while opening or stepping the RBU object, +** this function may return NULL. The error code and message may be collected +** when sqlite3rbu_close() is called. +** +** Database handles returned by this function remain valid until the next +** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db(). +*/ +SQLITE_API sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu); + +/* +** Do some work towards applying the RBU update to the target db. +** +** Return SQLITE_DONE if the update has been completely applied, or +** SQLITE_OK if no error occurs but there remains work to do to apply +** the RBU update. If an error does occur, some other error code is +** returned. +** +** Once a call to sqlite3rbu_step() has returned a value other than +** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops +** that immediately return the same value. +*/ +SQLITE_API int sqlite3rbu_step(sqlite3rbu *pRbu); + +/* +** Force RBU to save its state to disk. +** +** If a power failure or application crash occurs during an update, following +** system recovery RBU may resume the update from the point at which the state +** was last saved. In other words, from the most recent successful call to +** sqlite3rbu_close() or this function. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *pRbu); + +/* +** Close an RBU handle. +** +** If the RBU update has been completely applied, mark the RBU database +** as fully applied. Otherwise, assuming no error has occurred, save the +** current state of the RBU update appliation to the RBU database. +** +** If an error has already occurred as part of an sqlite3rbu_step() +** or sqlite3rbu_open() call, or if one occurs within this function, an +** SQLite error code is returned. Additionally, *pzErrmsg may be set to +** point to a buffer containing a utf-8 formatted English language error +** message. It is the responsibility of the caller to eventually free any +** such buffer using sqlite3_free(). +** +** Otherwise, if no error occurs, this function returns SQLITE_OK if the +** update has been partially applied, or SQLITE_DONE if it has been +** completely applied. +*/ +SQLITE_API int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); + +/* +** Return the total number of key-value operations (inserts, deletes or +** updates) that have been performed on the target database since the +** current RBU update was started. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); + +/* +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** progress indications for the two stages of an RBU update. This API may +** be useful for driving GUI progress indicators and similar. +** +** An RBU update is divided into two stages: +** +** * Stage 1, in which changes are accumulated in an oal/wal file, and +** * Stage 2, in which the contents of the wal file are copied into the +** main database. +** +** The update is visible to non-RBU clients during stage 2. During stage 1 +** non-RBU reader clients may see the original database. +** +** If this API is called during stage 2 of the update, output variable +** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) +** to a value between 0 and 10000 to indicate the permyriadage progress of +** stage 2. A value of 5000 indicates that stage 2 is half finished, +** 9000 indicates that it is 90% finished, and so on. +** +** If this API is called during stage 1 of the update, output variable +** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it +** exists, must contain the same columns as the following: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There must be one row in the table for each source (data_xxx) table within +** the RBU database. The 'tbl' column should contain the name of the source +** table. The 'cnt' column should contain the number of rows within the +** source table. +** +** If the rbu_count table is present and populated correctly and this +** API is called during stage 1, the *pnOne output variable is set to the +** permyriadage progress of the same stage. If the rbu_count table does +** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count +** table exists but is not correctly populated, the value of the *pnOne +** output variable during stage 1 is undefined. +*/ +SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); + +/* +** Obtain an indication as to the current stage of an RBU update or vacuum. +** This function always returns one of the SQLITE_RBU_STATE_XXX constants +** defined in this file. Return values should be interpreted as follows: +** +** SQLITE_RBU_STATE_OAL: +** RBU is currently building a *-oal file. The next call to sqlite3rbu_step() +** may either add further data to the *-oal file, or compute data that will +** be added by a subsequent call. +** +** SQLITE_RBU_STATE_MOVE: +** RBU has finished building the *-oal file. The next call to sqlite3rbu_step() +** will move the *-oal file to the equivalent *-wal path. If the current +** operation is an RBU update, then the updated version of the database +** file will become visible to ordinary SQLite clients following the next +** call to sqlite3rbu_step(). +** +** SQLITE_RBU_STATE_CHECKPOINT: +** RBU is currently performing an incremental checkpoint. The next call to +** sqlite3rbu_step() will copy a page of data from the *-wal file into +** the target database file. +** +** SQLITE_RBU_STATE_DONE: +** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step() +** will immediately return SQLITE_DONE. +** +** SQLITE_RBU_STATE_ERROR: +** An error has occurred. Any subsequent calls to sqlite3rbu_step() will +** immediately return the SQLite error code associated with the error. +*/ +#define SQLITE_RBU_STATE_OAL 1 +#define SQLITE_RBU_STATE_MOVE 2 +#define SQLITE_RBU_STATE_CHECKPOINT 3 +#define SQLITE_RBU_STATE_DONE 4 +#define SQLITE_RBU_STATE_ERROR 5 + +SQLITE_API int sqlite3rbu_state(sqlite3rbu *pRbu); + +/* +** Create an RBU VFS named zName that accesses the underlying file-system +** via existing VFS zParent. Or, if the zParent parameter is passed NULL, +** then the new RBU VFS uses the default system VFS to access the file-system. +** The new object is registered as a non-default VFS with SQLite before +** returning. +** +** Part of the RBU implementation uses a custom VFS object. Usually, this +** object is created and deleted automatically by RBU. +** +** The exception is for applications that also use zipvfs. In this case, +** the custom VFS must be explicitly created by the user before the RBU +** handle is opened. The RBU VFS should be installed so that the zipvfs +** VFS uses the RBU VFS, which in turn uses any other VFS layers in use +** (for example multiplexor) to access the file-system. For example, +** to assemble an RBU enabled VFS stack that uses both zipvfs and +** multiplexor (error checking omitted): +** +** // Create a VFS named "multiplex" (not the default). +** sqlite3_multiplex_initialize(0, 0); +** +** // Create an rbu VFS named "rbu" that uses multiplexor. If the +** // second argument were replaced with NULL, the "rbu" VFS would +** // access the file-system via the system default VFS, bypassing the +** // multiplexor. +** sqlite3rbu_create_vfs("rbu", "multiplex"); +** +** // Create a zipvfs VFS named "zipvfs" that uses rbu. +** zipvfs_create_vfs_v3("zipvfs", "rbu", 0, xCompressorAlgorithmDetector); +** +** // Make zipvfs the default VFS. +** sqlite3_vfs_register(sqlite3_vfs_find("zipvfs"), 1); +** +** Because the default VFS created above includes a RBU functionality, it +** may be used by RBU clients. Attempting to use RBU with a zipvfs VFS stack +** that does not include the RBU layer results in an error. +** +** The overhead of adding the "rbu" VFS to the system is negligible for +** non-RBU users. There is no harm in an application accessing the +** file-system via "rbu" all the time, even if it only uses RBU functionality +** occasionally. +*/ +SQLITE_API int sqlite3rbu_create_vfs(const char *zName, const char *zParent); + +/* +** Deregister and destroy an RBU vfs created by an earlier call to +** sqlite3rbu_create_vfs(). +** +** VFS objects are not reference counted. If a VFS object is destroyed +** before all database handles that use it have been closed, the results +** are undefined. +*/ +SQLITE_API void sqlite3rbu_destroy_vfs(const char *zName); + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _SQLITE3RBU_H */ + +/************** End of sqlite3rbu.h ******************************************/ +/************** Continuing where we left off in sqlite3rbu.c *****************/ + +#if defined(_WIN32_WCE) +/* #include "windows.h" */ +#endif + +/* Maximum number of prepared UPDATE statements held by this module */ +#define SQLITE_RBU_UPDATE_CACHESIZE 16 + +/* +** Swap two objects of type TYPE. +*/ +#if !defined(SQLITE_AMALGAMATION) +# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} +#endif + +/* +** The rbu_state table is used to save the state of a partially applied +** update so that it can be resumed later. The table consists of integer +** keys mapped to values as follows: +** +** RBU_STATE_STAGE: +** May be set to integer values 1, 2, 4 or 5. As follows: +** 1: the *-rbu file is currently under construction. +** 2: the *-rbu file has been constructed, but not yet moved +** to the *-wal path. +** 4: the checkpoint is underway. +** 5: the rbu update has been checkpointed. +** +** RBU_STATE_TBL: +** Only valid if STAGE==1. The target database name of the table +** currently being written. +** +** RBU_STATE_IDX: +** Only valid if STAGE==1. The target database name of the index +** currently being written, or NULL if the main table is currently being +** updated. +** +** RBU_STATE_ROW: +** Only valid if STAGE==1. Number of rows already processed for the current +** table/index. +** +** RBU_STATE_PROGRESS: +** Trbul number of sqlite3rbu_step() calls made so far as part of this +** rbu update. +** +** RBU_STATE_CKPT: +** Valid if STAGE==4. The 64-bit checksum associated with the wal-index +** header created by recovering the *-wal file. This is used to detect +** cases when another client appends frames to the *-wal file in the +** middle of an incremental checkpoint (an incremental checkpoint cannot +** be continued if this happens). +** +** RBU_STATE_COOKIE: +** Valid if STAGE==1. The current change-counter cookie value in the +** target db file. +** +** RBU_STATE_OALSZ: +** Valid if STAGE==1. The size in bytes of the *-oal file. +*/ +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 + +#define RBU_STAGE_OAL 1 +#define RBU_STAGE_MOVE 2 +#define RBU_STAGE_CAPTURE 3 +#define RBU_STAGE_CKPT 4 +#define RBU_STAGE_DONE 5 + + +#define RBU_CREATE_STATE \ + "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)" + +typedef struct RbuFrame RbuFrame; +typedef struct RbuObjIter RbuObjIter; +typedef struct RbuState RbuState; +typedef struct rbu_vfs rbu_vfs; +typedef struct rbu_file rbu_file; +typedef struct RbuUpdateStmt RbuUpdateStmt; + +#if !defined(SQLITE_AMALGAMATION) +typedef unsigned int u32; +typedef unsigned short u16; +typedef unsigned char u8; +typedef sqlite3_int64 i64; +#endif + +/* +** These values must match the values defined in wal.c for the equivalent +** locks. These are not magic numbers as they are part of the SQLite file +** format. +*/ +#define WAL_LOCK_WRITE 0 +#define WAL_LOCK_CKPT 1 +#define WAL_LOCK_READ0 3 + +#define SQLITE_FCNTL_RBUCNT 5149216 + +/* +** A structure to store values read from the rbu_state table in memory. +*/ +struct RbuState { + int eStage; + char *zTbl; + char *zIdx; + i64 iWalCksum; + int nRow; + i64 nProgress; + u32 iCookie; + i64 iOalSz; + i64 nPhaseOneStep; +}; + +struct RbuUpdateStmt { + char *zMask; /* Copy of update mask used with pUpdate */ + sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ + RbuUpdateStmt *pNext; +}; + +/* +** An iterator of this type is used to iterate through all objects in +** the target database that require updating. For each such table, the +** iterator visits, in order: +** +** * the table itself, +** * each index of the table (zero or more points to visit), and +** * a special "cleanup table" state. +** +** abIndexed: +** If the table has no indexes on it, abIndexed is set to NULL. Otherwise, +** it points to an array of flags nTblCol elements in size. The flag is +** set for each column that is either a part of the PK or a part of an +** index. Or clear otherwise. +** +*/ +struct RbuObjIter { + sqlite3_stmt *pTblIter; /* Iterate through tables */ + sqlite3_stmt *pIdxIter; /* Index iterator */ + int nTblCol; /* Size of azTblCol[] array */ + char **azTblCol; /* Array of unquoted target column names */ + char **azTblType; /* Array of target column types */ + int *aiSrcOrder; /* src table col -> target table col */ + u8 *abTblPk; /* Array of flags, set on target PK columns */ + u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ + u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ + int eType; /* Table type - an RBU_PK_XXX value */ + + /* Output variables. zTbl==0 implies EOF. */ + int bCleanup; /* True in "cleanup" state */ + const char *zTbl; /* Name of target db table */ + const char *zDataTbl; /* Name of rbu db table (or null) */ + const char *zIdx; /* Name of target db index (or null) */ + int iTnum; /* Root page of current object */ + int iPkTnum; /* If eType==EXTERNAL, root of PK index */ + int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ + + /* Statements created by rbuObjIterPrepareAll() */ + int nCol; /* Number of columns in current object */ + sqlite3_stmt *pSelect; /* Source data */ + sqlite3_stmt *pInsert; /* Statement for INSERT operations */ + sqlite3_stmt *pDelete; /* Statement for DELETE ops */ + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ + + /* Last UPDATE used (for PK b-tree updates only), or NULL. */ + RbuUpdateStmt *pRbuUpdate; +}; + +/* +** Values for RbuObjIter.eType +** +** 0: Table does not exist (error) +** 1: Table has an implicit rowid. +** 2: Table has an explicit IPK column. +** 3: Table has an external PK index. +** 4: Table is WITHOUT ROWID. +** 5: Table is a virtual table. +*/ +#define RBU_PK_NOTABLE 0 +#define RBU_PK_NONE 1 +#define RBU_PK_IPK 2 +#define RBU_PK_EXTERNAL 3 +#define RBU_PK_WITHOUT_ROWID 4 +#define RBU_PK_VTAB 5 + + +/* +** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs +** one of the following operations. +*/ +#define RBU_INSERT 1 /* Insert on a main table b-tree */ +#define RBU_DELETE 2 /* Delete a row from a main table b-tree */ +#define RBU_REPLACE 3 /* Delete and then insert a row */ +#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */ +#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */ + +#define RBU_UPDATE 6 /* Update a row in a main table b-tree */ + +/* +** A single step of an incremental checkpoint - frame iWalFrame of the wal +** file should be copied to page iDbPage of the database file. +*/ +struct RbuFrame { + u32 iDbPage; + u32 iWalFrame; +}; + +/* +** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. +*/ +struct sqlite3rbu { + int eStage; /* Value of RBU_STATE_STAGE field */ + sqlite3 *dbMain; /* target database handle */ + sqlite3 *dbRbu; /* rbu database handle */ + char *zTarget; /* Path to target db */ + char *zRbu; /* Path to rbu db */ + char *zState; /* Path to state db (or NULL if zRbu) */ + char zStateDb[5]; /* Db name for state ("stat" or "main") */ + int rc; /* Value returned by last rbu_step() call */ + char *zErrmsg; /* Error message if rc!=SQLITE_OK */ + int nStep; /* Rows processed for current object */ + int nProgress; /* Rows processed for all objects */ + RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ + const char *zVfsName; /* Name of automatically created rbu vfs */ + rbu_file *pTargetFd; /* File handle open on target db */ + i64 iOalSz; + i64 nPhaseOneStep; + + /* The following state variables are used as part of the incremental + ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding + ** function rbuSetupCheckpoint() for details. */ + u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ + u32 mLock; + int nFrame; /* Entries in aFrame[] array */ + int nFrameAlloc; /* Allocated size of aFrame[] array */ + RbuFrame *aFrame; + int pgsz; + u8 *aBuf; + i64 iWalCksum; + + /* Used in RBU vacuum mode only */ + int nRbu; /* Number of RBU VFS in the stack */ + rbu_file *pRbuFd; /* Fd for main db of dbRbu */ +}; + +/* +** An rbu VFS is implemented using an instance of this structure. +*/ +struct rbu_vfs { + sqlite3_vfs base; /* rbu VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; /* Mutex to protect pMain */ + rbu_file *pMain; /* Linked list of main db files */ +}; + +/* +** Each file opened by an rbu VFS is represented by an instance of +** the following structure. +*/ +struct rbu_file { + sqlite3_file base; /* sqlite3_file methods */ + sqlite3_file *pReal; /* Underlying file handle */ + rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ + sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ + + int openFlags; /* Flags this file was opened with */ + u32 iCookie; /* Cookie value for main db files */ + u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; /* True to fail EXCLUSIVE locks */ + + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + char *zDel; /* Delete this when closing file */ + + const char *zWal; /* Wal filename for this main db file */ + rbu_file *pWalFd; /* Wal file descriptor for this main db */ + rbu_file *pMainNext; /* Next MAIN_DB file */ +}; + +/* +** True for an RBU vacuum handle, or false otherwise. +*/ +#define rbuIsVacuum(p) ((p)->zTarget==0) + + +/************************************************************************* +** The following three functions, found below: +** +** rbuDeltaGetInt() +** rbuDeltaChecksum() +** rbuDeltaApply() +** +** are lifted from the fossil source code (http://fossil-scm.org). They +** are used to implement the scalar SQL function rbu_fossil_delta(). +*/ + +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} + +/* +** Apply a delta. +** +** The output buffer should be big enough to hold the whole output +** file and a NUL terminator at the end. The delta_output_size() +** routine will determine this size for you. +** +** The delta string should be null-terminated. But the delta string +** may contain embedded NUL characters (if the input and output are +** binary files) so we also have to pass in the length of the delta in +** the lenDelta parameter. +** +** This function returns the size of the output file in bytes (excluding +** the final NUL terminator character). Except, if the delta string is +** malformed or intended for use with a source file other than zSrc, +** then this routine returns -1. +** +** Refer to the delta_create() documentation above for a description +** of the delta file format. +*/ +static int rbuDeltaApply( + const char *zSrc, /* The source or pattern file */ + int lenSrc, /* Length of the source file */ + const char *zDelta, /* Delta to apply to the pattern */ + int lenDelta, /* Length of the delta */ + char *zOut /* Write the output into this preallocated buffer */ +){ + unsigned int limit; + unsigned int total = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + char *zOrigOut = zOut; +#endif + + limit = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + unsigned int cnt, ofst; + cnt = rbuDeltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = rbuDeltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( (int)(ofst+cnt) > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( (int)cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; + } + } + } + /* ERROR: unterminated delta */ + return -1; +} + +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ + int size; + size = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + return size; +} + +/* +** End of code taken from fossil. +*************************************************************************/ + +/* +** Implementation of SQL scalar function rbu_fossil_delta(). +** +** This function applies a fossil delta patch to a blob. Exactly two +** arguments must be passed to this function. The first is the blob to +** patch and the second the patch to apply. If no error occurs, this +** function returns the patched blob. +*/ +static void rbuFossilDeltaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aDelta; + int nDelta; + const char *aOrig; + int nOrig; + + int nOut; + int nOut2; + char *aOut; + + assert( argc==2 ); + + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); + + /* Figure out the size of the output */ + nOut = rbuDeltaOutputSize(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + } + + aOut = sqlite3_malloc(nOut+1); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); + if( nOut2!=nOut ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + + +/* +** Prepare the SQL statement in buffer zSql against database handle db. +** If successful, set *ppStmt to point to the new statement and return +** SQLITE_OK. +** +** Otherwise, if an error does occur, set *ppStmt to NULL and return +** an SQLite error code. Additionally, set output variable *pzErrmsg to +** point to a buffer containing an error message. It is the responsibility +** of the caller to (eventually) free this buffer using sqlite3_free(). +*/ +static int prepareAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + const char *zSql +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + *ppStmt = 0; + } + return rc; +} + +/* +** Reset the SQL statement passed as the first argument. Return a copy +** of the value returned by sqlite3_reset(). +** +** If an error has occurred, then set *pzErrmsg to point to a buffer +** containing an error message. It is the responsibility of the caller +** to eventually free this buffer using sqlite3_free(). +*/ +static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); + } + return rc; +} + +/* +** Unless it is NULL, argument zSql points to a buffer allocated using +** sqlite3_malloc containing an SQL statement. This function prepares the SQL +** statement against database db and frees the buffer. If statement +** compilation is successful, *ppStmt is set to point to the new statement +** handle and SQLITE_OK is returned. +** +** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code +** returned. In this case, *pzErrmsg may also be set to point to an error +** message. It is the responsibility of the caller to free this error message +** buffer using sqlite3_free(). +** +** If argument zSql is NULL, this function assumes that an OOM has occurred. +** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. +*/ +static int prepareFreeAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + char *zSql +){ + int rc; + assert( *pzErrmsg==0 ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + *ppStmt = 0; + }else{ + rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); + } + return rc; +} + +/* +** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated +** by an earlier call to rbuObjIterCacheTableInfo(). +*/ +static void rbuObjIterFreeCols(RbuObjIter *pIter){ + int i; + for(i=0; inTblCol; i++){ + sqlite3_free(pIter->azTblCol[i]); + sqlite3_free(pIter->azTblType[i]); + } + sqlite3_free(pIter->azTblCol); + pIter->azTblCol = 0; + pIter->azTblType = 0; + pIter->aiSrcOrder = 0; + pIter->abTblPk = 0; + pIter->abNotNull = 0; + pIter->nTblCol = 0; + pIter->eType = 0; /* Invalid value */ +} + +/* +** Finalize all statements and free all allocations that are specific to +** the current object (table/index pair). +*/ +static void rbuObjIterClearStatements(RbuObjIter *pIter){ + RbuUpdateStmt *pUp; + + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); + sqlite3_finalize(pIter->pTmpInsert); + pUp = pIter->pRbuUpdate; + while( pUp ){ + RbuUpdateStmt *pTmp = pUp->pNext; + sqlite3_finalize(pUp->pUpdate); + sqlite3_free(pUp); + pUp = pTmp; + } + + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->pDelete = 0; + pIter->pRbuUpdate = 0; + pIter->pTmpInsert = 0; + pIter->nCol = 0; +} + +/* +** Clean up any resources allocated as part of the iterator object passed +** as the only argument. +*/ +static void rbuObjIterFinalize(RbuObjIter *pIter){ + rbuObjIterClearStatements(pIter); + sqlite3_finalize(pIter->pTblIter); + sqlite3_finalize(pIter->pIdxIter); + rbuObjIterFreeCols(pIter); + memset(pIter, 0, sizeof(RbuObjIter)); +} + +/* +** Advance the iterator to the next position. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the next entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ + int rc = p->rc; + if( rc==SQLITE_OK ){ + + /* Free any SQLite statements used while processing the previous object */ + rbuObjIterClearStatements(pIter); + if( pIter->zIdx==0 ){ + rc = sqlite3_exec(p->dbMain, + "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;" + , 0, 0, &p->zErrmsg + ); + } + + if( rc==SQLITE_OK ){ + if( pIter->bCleanup ){ + rbuObjIterFreeCols(pIter); + pIter->bCleanup = 0; + rc = sqlite3_step(pIter->pTblIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); + pIter->zTbl = 0; + }else{ + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; + } + }else{ + if( pIter->zIdx==0 ){ + sqlite3_stmt *pIdx = pIter->pIdxIter; + rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pIter->pIdxIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg); + pIter->bCleanup = 1; + pIter->zIdx = 0; + }else{ + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); + rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM; + } + } + } + } + } + + if( rc!=SQLITE_OK ){ + rbuObjIterFinalize(pIter); + p->rc = rc; + } + return rc; +} + + +/* +** The implementation of the rbu_target_name() SQL function. This function +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: +** +** data[0-9]_ +** +** where is any sequence of 1 or more characters, is returned. +** Otherwise, if the only argument does not match the above pattern, an SQL +** NULL is returned. +** +** "data_t1" -> "t1" +** "data0123_t2" -> "t2" +** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). +*/ +static void rbuTargetNameFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + const char *zIn; + assert( argc==1 || argc==2 ); + + zIn = (const char*)sqlite3_value_text(argv[0]); + if( zIn ){ + if( rbuIsVacuum(p) ){ + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); + } + }else{ + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); + } + } + } + } +} + +/* +** Initialize the iterator structure passed as the second argument. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the first entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ + int rc; + memset(pIter, 0, sizeof(RbuObjIter)); + + rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + sqlite3_mprintf( + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_master " + "WHERE type IN ('table', 'view') AND target IS NOT NULL " + " %s " + "ORDER BY name" + , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, + "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " + " FROM main.sqlite_master " + " WHERE type='index' AND tbl_name = ?" + ); + } + + pIter->bCleanup = 1; + p->rc = rc; + return rbuObjIterNext(p, pIter); +} + +/* +** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, +** an error code is stored in the RBU handle passed as the first argument. +** +** If an error has already occurred (p->rc is already set to something other +** than SQLITE_OK), then this function returns NULL without modifying the +** stored error code. In this case it still calls sqlite3_free() on any +** printf() parameters associated with %z conversions. +*/ +static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ + char *zSql = 0; + va_list ap; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ) p->rc = SQLITE_NOMEM; + }else{ + sqlite3_free(zSql); + zSql = 0; + } + va_end(ap); + return zSql; +} + +/* +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing +** arguments are the usual subsitution values. This function performs +** the printf() style substitutions and executes the result as an SQL +** statement on the RBU handles database. +** +** If an error occurs, an error code and error message is stored in the +** RBU handle. If an error has already occurred when this function is +** called, it is a no-op. +*/ +static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ + va_list ap; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg); + } + } + sqlite3_free(zSql); + va_end(ap); + return p->rc; +} + +/* +** Attempt to allocate and return a pointer to a zeroed block of nByte +** bytes. +** +** If an error (i.e. an OOM condition) occurs, return NULL and leave an +** error code in the rbu handle passed as the first argument. Or, if an +** error has already occurred when this function is called, return NULL +** immediately without attempting the allocation or modifying the stored +** error code. +*/ +static void *rbuMalloc(sqlite3rbu *p, int nByte){ + void *pRet = 0; + if( p->rc==SQLITE_OK ){ + assert( nByte>0 ); + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + + +/* +** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that +** there is room for at least nCol elements. If an OOM occurs, store an +** error code in the RBU handle passed as the first argument. +*/ +static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ + int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + char **azNew; + + azNew = (char**)rbuMalloc(p, nByte); + if( azNew ){ + pIter->azTblCol = azNew; + pIter->azTblType = &azNew[nCol]; + pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; + pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; + pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; + pIter->abIndexed = (u8*)&pIter->abNotNull[nCol]; + } +} + +/* +** The first argument must be a nul-terminated string. This function +** returns a copy of the string in memory obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free this memory +** using sqlite3_free(). +** +** If an OOM condition is encountered when attempting to allocate memory, +** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise, +** if the allocation succeeds, (*pRc) is left unchanged. +*/ +static char *rbuStrndup(const char *zStr, int *pRc){ + char *zRet = 0; + + assert( *pRc==SQLITE_OK ); + if( zStr ){ + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } + } + + return zRet; +} + +/* +** Finalize the statement passed as the second argument. +** +** If the sqlite3_finalize() call indicates that an error occurs, and the +** rbu handle error code is not already set, set the error code and error +** message accordingly. +*/ +static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + p->rc = rc; + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } +} + +/* Determine the type of a table. +** +** peType is of type (int*), a pointer to an output parameter of type +** (int). This call sets the output parameter as follows, depending +** on the type of the table specified by parameters dbName and zTbl. +** +** RBU_PK_NOTABLE: No such table. +** RBU_PK_NONE: Table has an implicit rowid. +** RBU_PK_IPK: Table has an explicit IPK column. +** RBU_PK_EXTERNAL: Table has an external PK index. +** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. +** RBU_PK_VTAB: Table is a virtual table. +** +** Argument *piPk is also of type (int*), and also points to an output +** parameter. Unless the table has an external primary key index +** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, +** if the table does have an external primary key index, then *piPk +** is set to the root page number of the primary key index before +** returning. +** +** ALGORITHM: +** +** if( no entry exists in sqlite_master ){ +** return RBU_PK_NOTABLE +** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ +** return RBU_PK_VTAB +** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ +** if( the index that is the pk exists in sqlite_master ){ +** *piPK = rootpage of that index. +** return RBU_PK_EXTERNAL +** }else{ +** return RBU_PK_WITHOUT_ROWID +** } +** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ +** return RBU_PK_IPK +** }else{ +** return RBU_PK_NONE +** } +*/ +static void rbuTableType( + sqlite3rbu *p, + const char *zTab, + int *peType, + int *piTnum, + int *piPk +){ + /* + ** 0) SELECT count(*) FROM sqlite_master where name=%Q AND IsVirtual(%Q) + ** 1) PRAGMA index_list = ? + ** 2) SELECT count(*) FROM sqlite_master where name=%Q + ** 3) PRAGMA table_info = ? + */ + sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; + + *peType = RBU_PK_NOTABLE; + *piPk = 0; + + assert( p->rc==SQLITE_OK ); + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, + sqlite3_mprintf( + "SELECT (sql LIKE 'create virtual%%'), rootpage" + " FROM sqlite_master" + " WHERE name=%Q", zTab + )); + if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ + /* Either an error, or no such table. */ + goto rbuTableType_end; + } + if( sqlite3_column_int(aStmt[0], 0) ){ + *peType = RBU_PK_VTAB; /* virtual table */ + goto rbuTableType_end; + } + *piTnum = sqlite3_column_int(aStmt[0], 1); + + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, + sqlite3_mprintf("PRAGMA index_list=%Q",zTab) + ); + if( p->rc ) goto rbuTableType_end; + while( sqlite3_step(aStmt[1])==SQLITE_ROW ){ + const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); + const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); + if( zOrig && zIdx && zOrig[0]=='p' ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, + sqlite3_mprintf( + "SELECT rootpage FROM sqlite_master WHERE name = %Q", zIdx + )); + if( p->rc==SQLITE_OK ){ + if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ + *piPk = sqlite3_column_int(aStmt[2], 0); + *peType = RBU_PK_EXTERNAL; + }else{ + *peType = RBU_PK_WITHOUT_ROWID; + } + } + goto rbuTableType_end; + } + } + + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info=%Q",zTab) + ); + if( p->rc==SQLITE_OK ){ + while( sqlite3_step(aStmt[3])==SQLITE_ROW ){ + if( sqlite3_column_int(aStmt[3],5)>0 ){ + *peType = RBU_PK_IPK; /* explicit IPK column */ + goto rbuTableType_end; + } + } + *peType = RBU_PK_NONE; + } + +rbuTableType_end: { + unsigned int i; + for(i=0; iabIndexed[] array. +*/ +static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pList = 0; + int bIndex = 0; + + if( p->rc==SQLITE_OK ){ + memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol); + p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + } + + pIter->nIndex = 0; + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ + const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + sqlite3_stmt *pXInfo = 0; + if( zIdx==0 ) break; + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + if( iCid>=0 ) pIter->abIndexed[iCid] = 1; + } + rbuFinalize(p, pXInfo); + bIndex = 1; + pIter->nIndex++; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + /* "PRAGMA index_list" includes the main PK b-tree */ + pIter->nIndex--; + } + + rbuFinalize(p, pList); + if( bIndex==0 ) pIter->abIndexed = 0; +} + + +/* +** If they are not already populated, populate the pIter->azTblCol[], +** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to +** the table (not index) that the iterator currently points to. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. If +** an error does occur, an error code and error message are also left in +** the RBU handle. +*/ +static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ + if( pIter->azTblCol==0 ){ + sqlite3_stmt *pStmt = 0; + int nCol = 0; + int i; /* for() loop iterator variable */ + int bRbuRowid = 0; /* If input table has column "rbu_rowid" */ + int iOrder = 0; + int iTnum = 0; + + /* Figure out the type of table this step will deal with. */ + assert( pIter->eType==0 ); + rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum); + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl); + } + if( p->rc ) return p->rc; + if( pIter->zIdx==0 ) pIter->iTnum = iTnum; + + assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK + || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID + || pIter->eType==RBU_PK_VTAB + ); + + /* Populate the azTblCol[] and nTblCol variables based on the columns + ** of the input table. Ignore any input table columns that begin with + ** "rbu_". */ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) + ); + if( p->rc==SQLITE_OK ){ + nCol = sqlite3_column_count(pStmt); + rbuAllocateIterArrays(p, pIter, nCol); + } + for(i=0; p->rc==SQLITE_OK && irc); + pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol; + pIter->azTblCol[pIter->nTblCol++] = zCopy; + } + else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){ + bRbuRowid = 1; + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + + if( p->rc==SQLITE_OK + && rbuIsVacuum(p)==0 + && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) + ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "table %q %s rbu_rowid column", pIter->zDataTbl, + (bRbuRowid ? "may not have" : "requires") + ); + } + + /* Check that all non-HIDDEN columns in the destination table are also + ** present in the input table. Populate the abTblPk[], azTblType[] and + ** aiTblOrder[] arrays at the same time. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) + ); + } + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ + for(i=iOrder; inTblCol; i++){ + if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; + } + if( i==pIter->nTblCol ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", + pIter->zDataTbl, zName + ); + }else{ + int iPk = sqlite3_column_int(pStmt, 5); + int bNotNull = sqlite3_column_int(pStmt, 3); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + + if( i!=iOrder ){ + SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]); + SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]); + } + + pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); + pIter->abTblPk[iOrder] = (iPk!=0); + pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); + iOrder++; + } + } + + rbuFinalize(p, pStmt); + rbuObjIterCacheIndexedCols(p, pIter); + assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); + assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); + } + + return p->rc; +} + +/* +** This function constructs and returns a pointer to a nul-terminated +** string containing some SQL clause or list based on one or more of the +** column names currently stored in the pIter->azTblCol[] array. +*/ +static char *rbuObjIterGetCollist( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter /* Object iterator for column names */ +){ + char *zList = 0; + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + const char *z = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z); + zSep = ", "; + } + return zList; +} + +/* +** This function is used to create a SELECT list (the list of SQL +** expressions that follows a SELECT keyword) for a SELECT statement +** used to read from an data_xxx or rbu_tmp_xxx table while updating the +** index object currently indicated by the iterator object passed as the +** second argument. A "PRAGMA index_xinfo = " statement is used +** to obtain the required information. +** +** If the index is of the following form: +** +** CREATE INDEX i1 ON t1(c, b COLLATE nocase); +** +** and "t1" is a table with an explicit INTEGER PRIMARY KEY column +** "ipk", the returned string is: +** +** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" +** +** As well as the returned string, three other malloc'd strings are +** returned via output parameters. As follows: +** +** pzImposterCols: ... +** pzImposterPk: ... +** pzWhere: ... +*/ +static char *rbuObjIterGetIndexCols( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + char **pzImposterCols, /* OUT: Columns for imposter table */ + char **pzImposterPk, /* OUT: Imposter PK clause */ + char **pzWhere, /* OUT: WHERE clause */ + int *pnBind /* OUT: Trbul number of columns */ +){ + int rc = p->rc; /* Error code */ + int rc2; /* sqlite3_finalize() return code */ + char *zRet = 0; /* String to return */ + char *zImpCols = 0; /* String to return via *pzImposterCols */ + char *zImpPK = 0; /* String to return via *pzImposterPK */ + char *zWhere = 0; /* String to return via *pzWhere */ + int nBind = 0; /* Value to return via *pnBind */ + const char *zCom = ""; /* Set to ", " later on */ + const char *zAnd = ""; /* Set to " AND " later on */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ + + if( rc==SQLITE_OK ){ + assert( p->zErrmsg==0 ); + rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol; + const char *zType; + + if( iCid<0 ){ + /* An integer primary key. If the table has an explicit IPK, use + ** its name. Otherwise, use "rbu_rowid". */ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; + }else{ + zCol = "rbu_rowid"; + } + zType = "INTEGER"; + }else{ + zCol = pIter->azTblCol[iCid]; + zType = pIter->azTblType[iCid]; + } + + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); + if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ + const char *zOrder = (bDesc ? " DESC" : ""); + zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", + zImpPK, zCom, nBind, zCol, zOrder + ); + } + zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", + zImpCols, zCom, nBind, zCol, zType, zCollate + ); + zWhere = sqlite3_mprintf( + "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol + ); + if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; + zCom = ", "; + zAnd = " AND "; + nBind++; + } + + rc2 = sqlite3_finalize(pXInfo); + if( rc==SQLITE_OK ) rc = rc2; + + if( rc!=SQLITE_OK ){ + sqlite3_free(zRet); + sqlite3_free(zImpCols); + sqlite3_free(zImpPK); + sqlite3_free(zWhere); + zRet = 0; + zImpCols = 0; + zImpPK = 0; + zWhere = 0; + p->rc = rc; + } + + *pzImposterCols = zImpCols; + *pzImposterPk = zImpPK; + *pzWhere = zWhere; + *pnBind = nBind; + return zRet; +} + +/* +** Assuming the current table columns are "a", "b" and "c", and the zObj +** paramter is passed "old", return a string of the form: +** +** "old.a, old.b, old.b" +** +** With the column names escaped. +** +** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append +** the text ", old._rowid_" to the returned value. +*/ +static char *rbuObjIterGetOldlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zObj +){ + char *zList = 0; + if( p->rc==SQLITE_OK && pIter->abIndexed ){ + const char *zS = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abIndexed[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol); + }else{ + zList = sqlite3_mprintf("%z%sNULL", zList, zS); + } + zS = ", "; + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + } + + /* For a table with implicit rowids, append "old._rowid_" to the list. */ + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj); + } + } + return zList; +} + +/* +** Return an expression that can be used in a WHERE clause to match the +** primary key of the current table. For example, if the table is: +** +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)); +** +** Return the string: +** +** "b = ?1 AND c = ?2" +*/ +static char *rbuObjIterGetWhere( + sqlite3rbu *p, + RbuObjIter *pIter +){ + char *zList = 0; + if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); + }else if( pIter->eType==RBU_PK_EXTERNAL ){ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1); + zSep = " AND "; + } + } + zList = rbuMPrintf(p, + "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList + ); + + }else{ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1); + zSep = " AND "; + } + } + } + return zList; +} + +/* +** The SELECT statement iterating through the keys for the current object +** (p->objiter.pSelect) currently points to a valid row. However, there +** is something wrong with the rbu_control value in the rbu_control value +** stored in the (p->nCol+1)'th column. Set the error code and error message +** of the RBU handle to something reflecting this. +*/ +static void rbuBadControlError(sqlite3rbu *p){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid rbu_control value"); +} + + +/* +** Return a nul-terminated string containing the comma separated list of +** assignments that should be included following the "SET" keyword of +** an UPDATE statement used to update the table object that the iterator +** passed as the second argument currently points to if the rbu_control +** column of the data_xxx table entry is set to zMask. +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. +*/ +static char *rbuObjIterGetSetlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zMask +){ + char *zList = 0; + if( p->rc==SQLITE_OK ){ + int i; + + if( (int)strlen(zMask)!=pIter->nTblCol ){ + rbuBadControlError(p); + }else{ + const char *zSep = ""; + for(i=0; inTblCol; i++){ + char c = zMask[pIter->aiSrcOrder[i]]; + if( c=='x' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", + zList, zSep, pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='d' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='f' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + } + } + } + return zList; +} + +/* +** Return a nul-terminated string consisting of nByte comma separated +** "?" expressions. For example, if nByte is 3, return a pointer to +** a buffer containing the string "?,?,?". +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. +*/ +static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ + char *zRet = 0; + int nByte = nBind*2 + 1; + + zRet = (char*)rbuMalloc(p, nByte); + if( zRet ){ + int i; + for(i=0; izIdx==0 ); + if( p->rc==SQLITE_OK ){ + const char *zSep = "PRIMARY KEY("; + sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){ + const char *zOrig = (const char*)sqlite3_column_text(pXList,3); + if( zOrig && strcmp(zOrig, "pk")==0 ){ + const char *zIdx = (const char*)sqlite3_column_text(pXList,1); + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + break; + } + } + rbuFinalize(p, pXList); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + if( sqlite3_column_int(pXInfo, 5) ){ + /* int iCid = sqlite3_column_int(pXInfo, 0); */ + const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); + const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; + z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); + zSep = ", "; + } + } + z = rbuMPrintf(p, "%z)", z); + rbuFinalize(p, pXInfo); + } + return z; +} + +/* +** This function creates the second imposter table used when writing to +** a table b-tree where the table has an external primary key. If the +** iterator passed as the second argument does not currently point to +** a table (not index) with an external primary key, this function is a +** no-op. +** +** Assuming the iterator does point to a table with an external PK, this +** function creates a WITHOUT ROWID imposter table named "rbu_imposter2" +** used to access that PK index. For example, if the target table is +** declared as follows: +** +** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c)); +** +** then the imposter table schema is: +** +** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID; +** +*/ +static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){ + int tnum = pIter->iPkTnum; /* Root page of PK index */ + sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */ + const char *zIdx = 0; /* Name of PK index */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */ + const char *zComma = ""; + char *zCols = 0; /* Used to build up list of table cols */ + char *zPk = 0; /* Used to build up table PK declaration */ + + /* Figure out the name of the primary key index for the current table. + ** This is needed for the argument to "PRAGMA index_xinfo". Set + ** zIdx to point to a nul-terminated string containing this name. */ + p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, + "SELECT name FROM sqlite_master WHERE rootpage = ?" + ); + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(pQuery, 1, tnum); + if( SQLITE_ROW==sqlite3_step(pQuery) ){ + zIdx = (const char*)sqlite3_column_text(pQuery, 0); + } + } + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + rbuFinalize(p, pQuery); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int bKey = sqlite3_column_int(pXInfo, 5); + if( bKey ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma, + iCid, pIter->azTblType[iCid], zCollate + ); + zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); + zComma = ", "; + } + } + zCols = rbuMPrintf(p, "%z, id INTEGER", zCols); + rbuFinalize(p, pXInfo); + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", + zCols, zPk + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + } +} + +/* +** If an error has already occurred when this function is called, it +** immediately returns zero (without doing any work). Or, if an error +** occurs during the execution of this function, it sets the error code +** in the sqlite3rbu object indicated by the first argument and returns +** zero. +** +** The iterator passed as the second argument is guaranteed to point to +** a table (not an index) when this function is called. This function +** attempts to create any imposter table required to write to the main +** table b-tree of the table before returning. Non-zero is returned if +** an imposter table are created, or zero otherwise. +** +** An imposter table is required in all cases except RBU_PK_VTAB. Only +** virtual tables are written to directly. The imposter table has the +** same schema as the actual target table (less any UNIQUE constraints). +** More precisely, the "same schema" means the same columns, types, +** collation sequences. For tables that do not have an external PRIMARY +** KEY, it also means the same PRIMARY KEY declaration. +*/ +static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){ + int tnum = pIter->iTnum; + const char *zComma = ""; + char *zSql = 0; + int iCol; + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + + for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ + const char *zPk = ""; + const char *zCol = pIter->azTblCol[iCol]; + const char *zColl = 0; + + p->rc = sqlite3_table_column_metadata( + p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 + ); + + if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){ + /* If the target table column is an "INTEGER PRIMARY KEY", add + ** "PRIMARY KEY" to the imposter table column declaration. */ + zPk = "PRIMARY KEY "; + } + zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s", + zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, + (pIter->abNotNull[iCol] ? " NOT NULL" : "") + ); + zComma = ", "; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + char *zPk = rbuWithoutRowidPK(p, pIter); + if( zPk ){ + zSql = rbuMPrintf(p, "%z, %z", zSql, zPk); + } + } + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", + pIter->zTbl, zSql, + (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + } +} + +/* +** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table. +** Specifically a statement of the form: +** +** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...); +** +** The number of bound variables is equal to the number of columns in +** the target table, plus one (for the rbu_control column), plus one more +** (for the rbu_rowid column) if the target table is an implicit IPK or +** virtual table. +*/ +static void rbuObjIterPrepareTmpInsert( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zCollist, + const char *zRbuRowid +){ + int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); + char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); + if( zBind ){ + assert( pIter->pTmpInsert==0 ); + p->rc = prepareFreeAndCollectError( + p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( + "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", + p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind + )); + } +} + +static void rbuTmpInsertFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + int rc = SQLITE_OK; + int i; + + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } + + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(p->objiter.pTmpInsert); + rc = sqlite3_reset(p->objiter.pTmpInsert); + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } +} + +/* +** Ensure that the SQLite statement handles required to update the +** target database object currently indicated by the iterator passed +** as the second argument are available. +*/ +static int rbuObjIterPrepareAll( + sqlite3rbu *p, + RbuObjIter *pIter, + int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ +){ + assert( pIter->bCleanup==0 ); + if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ + const int tnum = pIter->iTnum; + char *zCollist = 0; /* List of indexed columns */ + char **pz = &p->zErrmsg; + const char *zIdx = pIter->zIdx; + char *zLimit = 0; + + if( nOffset ){ + zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset); + if( !zLimit ) p->rc = SQLITE_NOMEM; + } + + if( zIdx ){ + const char *zTbl = pIter->zTbl; + char *zImposterCols = 0; /* Columns for imposter table */ + char *zImposterPK = 0; /* Primary key declaration for imposter */ + char *zWhere = 0; /* WHERE clause on PK columns */ + char *zBind = 0; + int nBind = 0; + + assert( pIter->eType!=RBU_PK_VTAB ); + zCollist = rbuObjIterGetIndexCols( + p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind + ); + zBind = rbuObjIterGetBindlist(p, nBind); + + /* Create the imposter table used to write to this index. */ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", + zTbl, zImposterCols, zImposterPK + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + + /* Create the statement to insert index entries */ + pIter->nCol = nBind; + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pInsert, &p->zErrmsg, + sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind) + ); + } + + /* And to delete index entries */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pDelete, &p->zErrmsg, + sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) + ); + } + + /* Create the SELECT statement to read keys in sorted order */ + if( p->rc==SQLITE_OK ){ + char *zSql; + if( rbuIsVacuum(p) ){ + zSql = sqlite3_mprintf( + "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", + zCollist, + pIter->zDataTbl, + zCollist, zLimit + ); + }else + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, zLimit + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " + "UNION ALL " + "SELECT %s, rbu_control FROM '%q' " + "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " + "ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, pIter->zDataTbl, + zCollist, zLimit + ); + } + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql); + } + + sqlite3_free(zImposterCols); + sqlite3_free(zImposterPK); + sqlite3_free(zWhere); + sqlite3_free(zBind); + }else{ + int bRbuRowid = (pIter->eType==RBU_PK_VTAB) + ||(pIter->eType==RBU_PK_NONE) + ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); + const char *zTbl = pIter->zTbl; /* Table this step applies to */ + const char *zWrite; /* Imposter table name */ + + char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid); + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old"); + char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new"); + + zCollist = rbuObjIterGetCollist(p, pIter); + pIter->nCol = pIter->nTblCol; + + /* Create the imposter table or tables (if required). */ + rbuCreateImposterTable(p, pIter); + rbuCreateImposterTable2(p, pIter); + zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_"); + + /* Create the INSERT statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz, + sqlite3_mprintf( + "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", + zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings + ) + ); + } + + /* Create the DELETE statement to write to the target PK b-tree. + ** Because it only performs INSERT operations, this is not required for + ** an rbu vacuum handle. */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, + sqlite3_mprintf( + "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere + ) + ); + } + + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ + const char *zRbuRowid = ""; + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zRbuRowid = ", rbu_rowid"; + } + + /* Create the rbu_tmp_xxx table and the triggers to populate it. */ + rbuMPrintfExec(p, p->dbRbu, + "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " + "SELECT *%s FROM '%q' WHERE 0;" + , p->zStateDb, pIter->zDataTbl + , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") + , pIter->zDataTbl + ); + + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(4, %s);" + "END;", + zWrite, zTbl, zOldlist, + zWrite, zTbl, zOldlist, + zWrite, zTbl, zNewlist + ); + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(0, %s);" + "END;", + zWrite, zTbl, zNewlist + ); + } + + rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); + } + + /* Create the SELECT statement to read keys from data_xxx */ + if( p->rc==SQLITE_OK ){ + const char *zRbuRowid = ""; + if( bRbuRowid ){ + zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + } + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s,%s rbu_control%s FROM '%q'%s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, + pIter->zDataTbl, zLimit + ) + ); + } + + sqlite3_free(zWhere); + sqlite3_free(zOldlist); + sqlite3_free(zNewlist); + sqlite3_free(zBindings); + } + sqlite3_free(zCollist); + sqlite3_free(zLimit); + } + + return p->rc; +} + +/* +** Set output variable *ppStmt to point to an UPDATE statement that may +** be used to update the imposter table for the main table b-tree of the +** table object that pIter currently points to, assuming that the +** rbu_control column of the data_xyz table contains zMask. +** +** If the zMask string does not specify any columns to update, then this +** is not an error. Output variable *ppStmt is set to NULL in this case. +*/ +static int rbuGetUpdateStmt( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* Object iterator */ + const char *zMask, /* rbu_control value ('x.x.') */ + sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */ +){ + RbuUpdateStmt **pp; + RbuUpdateStmt *pUp = 0; + int nUp = 0; + + /* In case an error occurs */ + *ppStmt = 0; + + /* Search for an existing statement. If one is found, shift it to the front + ** of the LRU queue and return immediately. Otherwise, leave nUp pointing + ** to the number of statements currently in the cache and pUp to the + ** last object in the list. */ + for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){ + pUp = *pp; + if( strcmp(pUp->zMask, zMask)==0 ){ + *pp = pUp->pNext; + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + *ppStmt = pUp->pUpdate; + return SQLITE_OK; + } + nUp++; + } + assert( pUp==0 || pUp->pNext==0 ); + + if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){ + for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext)); + *pp = 0; + sqlite3_finalize(pUp->pUpdate); + pUp->pUpdate = 0; + }else{ + pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1); + } + + if( pUp ){ + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zSet = rbuObjIterGetSetlist(p, pIter, zMask); + char *zUpdate = 0; + + pUp->zMask = (char*)&pUp[1]; + memcpy(pUp->zMask, zMask, pIter->nTblCol); + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + + if( zSet ){ + const char *zPrefix = ""; + + if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_"; + zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", + zPrefix, pIter->zTbl, zSet, zWhere + ); + p->rc = prepareFreeAndCollectError( + p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate + ); + *ppStmt = pUp->pUpdate; + } + sqlite3_free(zWhere); + sqlite3_free(zSet); + } + + return p->rc; +} + +static sqlite3 *rbuOpenDbhandle( + sqlite3rbu *p, + const char *zName, + int bUseVfs +){ + sqlite3 *db = 0; + if( p->rc==SQLITE_OK ){ + const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; + p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); + if( p->rc ){ + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + sqlite3_close(db); + db = 0; + } + } + return db; +} + +/* +** Free an RbuState object allocated by rbuLoadState(). +*/ +static void rbuFreeState(RbuState *p){ + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the rbu handle +** and return NULL. +*/ +static RbuState *rbuLoadState(sqlite3rbu *p){ + RbuState *pRet = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int rc2; + + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); + if( pRet==0 ) return 0; + + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case RBU_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=RBU_STAGE_OAL + && pRet->eStage!=RBU_STAGE_MOVE + && pRet->eStage!=RBU_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; + } + break; + + case RBU_STATE_TBL: + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_IDX: + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case RBU_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_CKPT: + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_COOKIE: + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_OALSZ: + pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + + default: + rc = SQLITE_CORRUPT; + break; + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + p->rc = rc; + return pRet; +} + + +/* +** Open the database handle and attach the RBU database as "rbu". If an +** error occurs, leave an error code and message in the RBU handle. +*/ +static void rbuOpenDatabase(sqlite3rbu *p){ + assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); + assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); + + /* Open the RBU database */ + p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( p->zState==0 ){ + const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); + p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); + } + } + + /* If using separate RBU and state databases, attach the state database to + ** the RBU db handle now. */ + if( p->zState ){ + rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState); + memcpy(p->zStateDb, "stat", 4); + }else{ + memcpy(p->zStateDb, "main", 4); + } + +#if 0 + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + } +#endif + + /* If it has not already been created, create the rbu_state table */ + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + +#if 0 + if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK ){ + int rc2; + int bOk = 0; + sqlite3_stmt *pCnt = 0; + p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, + "SELECT count(*) FROM stat.sqlite_master" + ); + if( p->rc==SQLITE_OK + && sqlite3_step(pCnt)==SQLITE_ROW + && 1==sqlite3_column_int(pCnt, 0) + ){ + bOk = 1; + } + rc2 = sqlite3_finalize(pCnt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bOk==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid state database"); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + } + } + } +#endif + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + int bOpen = 0; + int rc; + p->nRbu = 0; + p->pRbuFd = 0; + rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( rc!=SQLITE_NOTFOUND ) p->rc = rc; + if( p->eStage>=RBU_STAGE_MOVE ){ + bOpen = 1; + }else{ + RbuState *pState = rbuLoadState(p); + if( pState ){ + bOpen = (pState->eStage>RBU_STAGE_MOVE); + rbuFreeState(pState); + } + } + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); + } + + p->eStage = 0; + if( p->rc==SQLITE_OK && p->dbMain==0 ){ + if( !rbuIsVacuum(p) ){ + p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + }else if( p->pRbuFd->pWalFd ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); + }else{ + char *zTarget; + char *zExtra = 0; + if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ + zExtra = &p->zRbu[5]; + while( *zExtra ){ + if( *zExtra++=='?' ) break; + } + if( *zExtra=='\0' ) zExtra = 0; + } + + zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", + sqlite3_db_filename(p->dbRbu, "main"), + (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) + ); + + if( zTarget==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); + sqlite3_free(zTarget); + } + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master"); + + /* Mark the database file just opened as an RBU target database. If + ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use. + ** This is an error. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("rbu vfs not found"); + } +} + +/* +** This routine is a copy of the sqlite3FileSuffix3() routine from the core. +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. +** +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa +*/ +static void rbuFileSuffix3(const char *zBase, char *z){ +#ifdef SQLITE_ENABLE_8_3_NAMES +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = (int)strlen(z)&0xffffff; + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); + } +#endif +} + +/* +** Return the current wal-index header checksum for the target database +** as a 64-bit integer. +** +** The checksum is store in the first page of xShmMap memory as an 8-byte +** blob starting at byte offset 40. +*/ +static i64 rbuShmChecksum(sqlite3rbu *p){ + i64 iRet = 0; + if( p->rc==SQLITE_OK ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + u32 volatile *ptr; + p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); + if( p->rc==SQLITE_OK ){ + iRet = ((i64)ptr[10] << 32) + ptr[11]; + } + } + return iRet; +} + +/* +** This function is called as part of initializing or reinitializing an +** incremental checkpoint. +** +** It populates the sqlite3rbu.aFrame[] array with the set of +** (wal frame -> db page) copy operations required to checkpoint the +** current wal file, and obtains the set of shm locks required to safely +** perform the copy operations directly on the file-system. +** +** If argument pState is not NULL, then the incremental checkpoint is +** being resumed. In this case, if the checksum of the wal-index-header +** following recovery is not the same as the checksum saved in the RbuState +** object, then the rbu handle is set to DONE state. This occurs if some +** other client appends a transaction to the wal file in the middle of +** an incremental checkpoint. +*/ +static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ + + /* If pState is NULL, then the wal file may not have been opened and + ** recovered. Running a read-statement here to ensure that doing so + ** does not interfere with the "capture" process below. */ + if( pState==0 ){ + p->eStage = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_master", 0, 0, 0); + } + } + + /* Assuming no error has occurred, run a "restart" checkpoint with the + ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following + ** special behaviour in the rbu VFS: + ** + ** * If the exclusive shm WRITER or READ0 lock cannot be obtained, + ** the checkpoint fails with SQLITE_BUSY (normally SQLite would + ** proceed with running a passive checkpoint instead of failing). + ** + ** * Attempts to read from the *-wal file or write to the database file + ** do not perform any IO. Instead, the frame/page combinations that + ** would be read/written are recorded in the sqlite3rbu.aFrame[] + ** array. + ** + ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, + ** READ0 and CHECKPOINT locks taken as part of the checkpoint are + ** no-ops. These locks will not be released until the connection + ** is closed. + ** + ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** error. + ** + ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the + ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] + ** array populated with a set of (frame -> page) mappings. Because the + ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy + ** data from the wal file into the database file according to the + ** contents of aFrame[]. + */ + if( p->rc==SQLITE_OK ){ + int rc2; + p->eStage = RBU_STAGE_CAPTURE; + rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); + if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + } + + if( p->rc==SQLITE_OK ){ + p->eStage = RBU_STAGE_CKPT; + p->nStep = (pState ? pState->nRow : 0); + p->aBuf = rbuMalloc(p, p->pgsz); + p->iWalCksum = rbuShmChecksum(p); + } + + if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + } +} + +/* +** Called when iAmt bytes are read from offset iOff of the wal file while +** the rbu object is in capture mode. Record the frame number of the frame +** being read in the aFrame[] array. +*/ +static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ + const u32 mReq = (1<mLock!=mReq ){ + pRbu->rc = SQLITE_BUSY; + return SQLITE_INTERNAL; + } + + pRbu->pgsz = iAmt; + if( pRbu->nFrame==pRbu->nFrameAlloc ){ + int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; + RbuFrame *aNew; + aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); + if( aNew==0 ) return SQLITE_NOMEM; + pRbu->aFrame = aNew; + pRbu->nFrameAlloc = nNew; + } + + iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; + if( pRbu->iMaxFrame